diff --git "a/sft/665K_Memory/Full_smoe/checkpoint-8316/trainer_state.json" "b/sft/665K_Memory/Full_smoe/checkpoint-8316/trainer_state.json" new file mode 100644--- /dev/null +++ "b/sft/665K_Memory/Full_smoe/checkpoint-8316/trainer_state.json" @@ -0,0 +1,141405 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9999398785546805, + "eval_steps": 500, + "global_step": 8316, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "auxiliary_loss_clip": 0.05084274, + "auxiliary_loss_mlp": 0.02254236, + "balance_loss_clip": 2.43166804, + "balance_loss_mlp": 1.82355666, + "epoch": 0.00012024289063909097, + "flos": 24932483919360.0, + "grad_norm": 41.044738699628475, + "language_loss": 2.57854795, + "learning_rate": 0.0, + "loss": 1.90186608, + "num_input_tokens_seen": 20375, + "router_z_loss_clip": 26.515625, + "router_z_loss_mlp": 4.30859375, + "step": 1, + "time_per_iteration": 18.68628764152527 + }, + { + "auxiliary_loss_clip": 0.03334969, + "auxiliary_loss_mlp": 0.01466922, + "balance_loss_clip": 1.62000775, + "balance_loss_mlp": 1.18406177, + "epoch": 0.00024048578127818193, + "flos": 30664624377600.0, + "grad_norm": 55.44981146749522, + "language_loss": 1.88882363, + "learning_rate": 5.021476677069823e-07, + "loss": 1.93684268, + "num_input_tokens_seen": 39035, + "router_z_loss_clip": 17.15625, + "router_z_loss_mlp": 2.828125, + "step": 2, + "time_per_iteration": 2.83567476272583 + }, + { + "auxiliary_loss_clip": 0.03359117, + "auxiliary_loss_mlp": 0.01433325, + "balance_loss_clip": 1.62643743, + "balance_loss_mlp": 1.15828574, + "epoch": 0.0003607286719172729, + "flos": 19026227969280.0, + "grad_norm": 40.46974269385572, + "language_loss": 1.6152178, + "learning_rate": 7.958852231401551e-07, + "loss": 1.6631422, + "num_input_tokens_seen": 57600, + "router_z_loss_clip": 17.3125, + "router_z_loss_mlp": 2.75195312, + "step": 3, + "time_per_iteration": 2.6939709186553955 + }, + { + "auxiliary_loss_clip": 0.0330364, + "auxiliary_loss_mlp": 0.01430006, + "balance_loss_clip": 1.61410177, + "balance_loss_mlp": 1.17403984, + "epoch": 0.00048097156255636386, + "flos": 19316314206720.0, + "grad_norm": 38.528943753089614, + "language_loss": 1.64560199, + "learning_rate": 1.0042953354139647e-06, + "loss": 1.69293833, + "num_input_tokens_seen": 76465, + "router_z_loss_clip": 16.875, + "router_z_loss_mlp": 2.56054688, + "step": 4, + "time_per_iteration": 2.746034860610962 + }, + { + "auxiliary_loss_clip": 0.03368603, + "auxiliary_loss_mlp": 0.0146408, + "balance_loss_clip": 1.61420786, + "balance_loss_mlp": 1.1989584, + "epoch": 0.0006012144531954548, + "flos": 13991264893440.0, + "grad_norm": 55.07474226703095, + "language_loss": 1.93626249, + "learning_rate": 1.1659507774310057e-06, + "loss": 1.98458934, + "num_input_tokens_seen": 94350, + "router_z_loss_clip": 17.546875, + "router_z_loss_mlp": 2.65234375, + "step": 5, + "time_per_iteration": 3.195141553878784 + }, + { + "auxiliary_loss_clip": 0.03384568, + "auxiliary_loss_mlp": 0.01441662, + "balance_loss_clip": 1.61453962, + "balance_loss_mlp": 1.16471481, + "epoch": 0.0007214573438345458, + "flos": 23148988225920.0, + "grad_norm": 45.82930637340719, + "language_loss": 1.61077058, + "learning_rate": 1.2980328908471373e-06, + "loss": 1.65903294, + "num_input_tokens_seen": 114595, + "router_z_loss_clip": 17.6875, + "router_z_loss_mlp": 2.77148438, + "step": 6, + "time_per_iteration": 3.0958359241485596 + }, + { + "auxiliary_loss_clip": 0.03977933, + "auxiliary_loss_mlp": 0.02072711, + "balance_loss_clip": 1.62635601, + "balance_loss_mlp": 1.83619964, + "epoch": 0.0008417002344736367, + "flos": 67663246170240.0, + "grad_norm": 4.689406335973038, + "language_loss": 0.81523812, + "learning_rate": 1.4097067265369432e-06, + "loss": 0.87574446, + "num_input_tokens_seen": 179590, + "router_z_loss_clip": 23.5, + "router_z_loss_mlp": 2.359375, + "step": 7, + "time_per_iteration": 3.4423916339874268 + }, + { + "auxiliary_loss_clip": 0.03323958, + "auxiliary_loss_mlp": 0.01489969, + "balance_loss_clip": 1.60676885, + "balance_loss_mlp": 1.20577455, + "epoch": 0.0009619431251127277, + "flos": 21281381504640.0, + "grad_norm": 40.8552640965295, + "language_loss": 1.5828104, + "learning_rate": 1.506443003120947e-06, + "loss": 1.63094974, + "num_input_tokens_seen": 195090, + "router_z_loss_clip": 17.15625, + "router_z_loss_mlp": 2.83984375, + "step": 8, + "time_per_iteration": 3.0266616344451904 + }, + { + "auxiliary_loss_clip": 0.03323212, + "auxiliary_loss_mlp": 0.01470381, + "balance_loss_clip": 1.61805439, + "balance_loss_mlp": 1.2050693, + "epoch": 0.0010821860157518186, + "flos": 23331342597120.0, + "grad_norm": 17.323955838605087, + "language_loss": 1.4795593, + "learning_rate": 1.5917704462803102e-06, + "loss": 1.52749527, + "num_input_tokens_seen": 211635, + "router_z_loss_clip": 17.046875, + "router_z_loss_mlp": 2.65429688, + "step": 9, + "time_per_iteration": 2.9909744262695312 + }, + { + "auxiliary_loss_clip": 0.03279818, + "auxiliary_loss_mlp": 0.01447096, + "balance_loss_clip": 1.62625647, + "balance_loss_mlp": 1.183882, + "epoch": 0.0012024289063909096, + "flos": 17010166337280.0, + "grad_norm": 13.198644690813767, + "language_loss": 1.5319171, + "learning_rate": 1.6680984451379884e-06, + "loss": 1.5791862, + "num_input_tokens_seen": 224705, + "router_z_loss_clip": 16.53125, + "router_z_loss_mlp": 2.6328125, + "step": 10, + "time_per_iteration": 3.1162338256835938 + }, + { + "auxiliary_loss_clip": 0.03267288, + "auxiliary_loss_mlp": 0.01435606, + "balance_loss_clip": 1.61303782, + "balance_loss_mlp": 1.17353678, + "epoch": 0.0013226717970300007, + "flos": 21288133261440.0, + "grad_norm": 13.630961789732135, + "language_loss": 1.32703912, + "learning_rate": 1.7371455188905097e-06, + "loss": 1.37406802, + "num_input_tokens_seen": 244635, + "router_z_loss_clip": 16.5078125, + "router_z_loss_mlp": 2.62109375, + "step": 11, + "time_per_iteration": 3.0227913856506348 + }, + { + "auxiliary_loss_clip": 0.03286451, + "auxiliary_loss_mlp": 0.01432224, + "balance_loss_clip": 1.60440612, + "balance_loss_mlp": 1.16271567, + "epoch": 0.0014429146876690916, + "flos": 27237884935680.0, + "grad_norm": 12.945355994849772, + "language_loss": 1.25471604, + "learning_rate": 1.8001805585541196e-06, + "loss": 1.30190277, + "num_input_tokens_seen": 265765, + "router_z_loss_clip": 16.796875, + "router_z_loss_mlp": 2.69335938, + "step": 12, + "time_per_iteration": 2.9958655834198 + }, + { + "auxiliary_loss_clip": 0.03250948, + "auxiliary_loss_mlp": 0.01406064, + "balance_loss_clip": 1.60671043, + "balance_loss_mlp": 1.1523869, + "epoch": 0.0015631575783081825, + "flos": 19062174504960.0, + "grad_norm": 6.942390953473345, + "language_loss": 1.29154921, + "learning_rate": 1.8581671739548328e-06, + "loss": 1.33811927, + "num_input_tokens_seen": 283500, + "router_z_loss_clip": 16.40625, + "router_z_loss_mlp": 2.53515625, + "step": 13, + "time_per_iteration": 3.018801689147949 + }, + { + "auxiliary_loss_clip": 0.03251088, + "auxiliary_loss_mlp": 0.0142243, + "balance_loss_clip": 1.60724497, + "balance_loss_mlp": 1.15196836, + "epoch": 0.0016834004689472734, + "flos": 48139473985920.0, + "grad_norm": 6.180867116127311, + "language_loss": 1.13513136, + "learning_rate": 1.9118543942439254e-06, + "loss": 1.18186665, + "num_input_tokens_seen": 305685, + "router_z_loss_clip": 16.4140625, + "router_z_loss_mlp": 2.703125, + "step": 14, + "time_per_iteration": 3.1455399990081787 + }, + { + "auxiliary_loss_clip": 0.03229054, + "auxiliary_loss_mlp": 0.01418256, + "balance_loss_clip": 1.61617851, + "balance_loss_mlp": 1.15446973, + "epoch": 0.0018036433595863645, + "flos": 34970026314240.0, + "grad_norm": 5.637589854047485, + "language_loss": 1.12667012, + "learning_rate": 1.961836000571161e-06, + "loss": 1.17314315, + "num_input_tokens_seen": 327340, + "router_z_loss_clip": 16.1328125, + "router_z_loss_mlp": 2.63671875, + "step": 15, + "time_per_iteration": 3.0643255710601807 + }, + { + "auxiliary_loss_clip": 0.03604912, + "auxiliary_loss_mlp": 0.01692374, + "balance_loss_clip": 1.62985969, + "balance_loss_mlp": 1.50469112, + "epoch": 0.0019238862502254555, + "flos": 59768284440960.0, + "grad_norm": 3.8453246723150643, + "language_loss": 0.64657366, + "learning_rate": 2.0085906708279293e-06, + "loss": 0.69954658, + "num_input_tokens_seen": 382710, + "router_z_loss_clip": 19.75, + "router_z_loss_mlp": 1.875, + "step": 16, + "time_per_iteration": 3.375112771987915 + }, + { + "auxiliary_loss_clip": 0.03118591, + "auxiliary_loss_mlp": 0.01407766, + "balance_loss_clip": 1.60131264, + "balance_loss_mlp": 1.16438782, + "epoch": 0.0020441291408645466, + "flos": 20814543417600.0, + "grad_norm": 4.382552884506777, + "language_loss": 1.16036248, + "learning_rate": 2.0525099325728135e-06, + "loss": 1.20562613, + "num_input_tokens_seen": 400890, + "router_z_loss_clip": 15.1796875, + "router_z_loss_mlp": 2.43359375, + "step": 17, + "time_per_iteration": 3.007169723510742 + }, + { + "auxiliary_loss_clip": 0.03450867, + "auxiliary_loss_mlp": 0.01585548, + "balance_loss_clip": 1.63156712, + "balance_loss_mlp": 1.41312361, + "epoch": 0.0021643720315036373, + "flos": 63857001582720.0, + "grad_norm": 3.57700145055464, + "language_loss": 0.72177064, + "learning_rate": 2.0939181139872922e-06, + "loss": 0.77213478, + "num_input_tokens_seen": 462605, + "router_z_loss_clip": 18.25, + "router_z_loss_mlp": 1.7265625, + "step": 18, + "time_per_iteration": 5.094023942947388 + }, + { + "auxiliary_loss_clip": 0.03125408, + "auxiliary_loss_mlp": 0.01381735, + "balance_loss_clip": 1.59844708, + "balance_loss_mlp": 1.13625956, + "epoch": 0.0022846149221427284, + "flos": 31284981192960.0, + "grad_norm": 4.899593681362477, + "language_loss": 1.01559424, + "learning_rate": 2.1330868934640175e-06, + "loss": 1.06066561, + "num_input_tokens_seen": 483280, + "router_z_loss_clip": 15.2734375, + "router_z_loss_mlp": 2.45507812, + "step": 19, + "time_per_iteration": 3.276538372039795 + }, + { + "auxiliary_loss_clip": 0.0326723, + "auxiliary_loss_mlp": 0.01446403, + "balance_loss_clip": 1.6353873, + "balance_loss_mlp": 1.29305255, + "epoch": 0.002404857812781819, + "flos": 51083648161920.0, + "grad_norm": 3.56950967762513, + "language_loss": 0.76371729, + "learning_rate": 2.170246112844971e-06, + "loss": 0.8108536, + "num_input_tokens_seen": 537620, + "router_z_loss_clip": 16.25, + "router_z_loss_mlp": 1.53125, + "step": 20, + "time_per_iteration": 3.1794235706329346 + }, + { + "auxiliary_loss_clip": 0.03028423, + "auxiliary_loss_mlp": 0.01344386, + "balance_loss_clip": 1.59932256, + "balance_loss_mlp": 1.1193186, + "epoch": 0.0025251007034209102, + "flos": 15815347309440.0, + "grad_norm": 4.220661581621062, + "language_loss": 1.0154624, + "learning_rate": 2.2055919496770983e-06, + "loss": 1.05919051, + "num_input_tokens_seen": 555760, + "router_z_loss_clip": 14.28125, + "router_z_loss_mlp": 2.25195312, + "step": 21, + "time_per_iteration": 3.023585081100464 + }, + { + "auxiliary_loss_clip": 0.02990415, + "auxiliary_loss_mlp": 0.01368281, + "balance_loss_clip": 1.58223557, + "balance_loss_mlp": 1.13663316, + "epoch": 0.0026453435940600014, + "flos": 37851857458560.0, + "grad_norm": 7.306539480442498, + "language_loss": 0.89430702, + "learning_rate": 2.2392931865974923e-06, + "loss": 0.93789405, + "num_input_tokens_seen": 578450, + "router_z_loss_clip": 14.078125, + "router_z_loss_mlp": 2.31347656, + "step": 22, + "time_per_iteration": 3.0990617275238037 + }, + { + "auxiliary_loss_clip": 0.0288333, + "auxiliary_loss_mlp": 0.0134525, + "balance_loss_clip": 1.56907177, + "balance_loss_mlp": 1.12247205, + "epoch": 0.002765586484699092, + "flos": 21141976821120.0, + "grad_norm": 4.571134363820792, + "language_loss": 1.02142406, + "learning_rate": 2.271496085962064e-06, + "loss": 1.06370986, + "num_input_tokens_seen": 596145, + "router_z_loss_clip": 13.1484375, + "router_z_loss_mlp": 2.22949219, + "step": 23, + "time_per_iteration": 3.1621828079223633 + }, + { + "auxiliary_loss_clip": 0.02849723, + "auxiliary_loss_mlp": 0.01317375, + "balance_loss_clip": 1.56254506, + "balance_loss_mlp": 1.11443329, + "epoch": 0.002885829375338183, + "flos": 20667381396480.0, + "grad_norm": 4.648227328156693, + "language_loss": 1.02786911, + "learning_rate": 2.3023282262611022e-06, + "loss": 1.06954002, + "num_input_tokens_seen": 614920, + "router_z_loss_clip": 12.8671875, + "router_z_loss_mlp": 2.03125, + "step": 24, + "time_per_iteration": 3.0294697284698486 + }, + { + "auxiliary_loss_clip": 0.02819309, + "auxiliary_loss_mlp": 0.01337584, + "balance_loss_clip": 1.56551504, + "balance_loss_mlp": 1.13483334, + "epoch": 0.003006072265977274, + "flos": 34823869873920.0, + "grad_norm": 3.095629635897947, + "language_loss": 0.92717338, + "learning_rate": 2.3319015548620114e-06, + "loss": 0.96874225, + "num_input_tokens_seen": 636060, + "router_z_loss_clip": 12.5390625, + "router_z_loss_mlp": 2.02539062, + "step": 25, + "time_per_iteration": 3.13999605178833 + }, + { + "auxiliary_loss_clip": 0.02776519, + "auxiliary_loss_mlp": 0.01317341, + "balance_loss_clip": 1.56399632, + "balance_loss_mlp": 1.12689209, + "epoch": 0.003126315156616365, + "flos": 24422021118720.0, + "grad_norm": 2.6048280651621734, + "language_loss": 0.93057168, + "learning_rate": 2.3603148416618152e-06, + "loss": 0.97151029, + "num_input_tokens_seen": 655575, + "router_z_loss_clip": 12.140625, + "router_z_loss_mlp": 1.90429688, + "step": 26, + "time_per_iteration": 3.0115253925323486 + }, + { + "auxiliary_loss_clip": 0.02695193, + "auxiliary_loss_mlp": 0.01309715, + "balance_loss_clip": 1.55204654, + "balance_loss_mlp": 1.12556088, + "epoch": 0.003246558047255456, + "flos": 23622326674560.0, + "grad_norm": 2.5450516405245165, + "language_loss": 1.00968039, + "learning_rate": 2.3876556694204647e-06, + "loss": 1.04972947, + "num_input_tokens_seen": 675730, + "router_z_loss_clip": 11.453125, + "router_z_loss_mlp": 1.84277344, + "step": 27, + "time_per_iteration": 3.0386805534362793 + }, + { + "auxiliary_loss_clip": 0.02655187, + "auxiliary_loss_mlp": 0.01322416, + "balance_loss_clip": 1.5357604, + "balance_loss_mlp": 1.13177657, + "epoch": 0.003366800937894547, + "flos": 17820275725440.0, + "grad_norm": 2.8421390060741167, + "language_loss": 0.90679151, + "learning_rate": 2.414002061950908e-06, + "loss": 0.94656754, + "num_input_tokens_seen": 694605, + "router_z_loss_clip": 11.1875, + "router_z_loss_mlp": 1.90820312, + "step": 28, + "time_per_iteration": 3.0462887287139893 + }, + { + "auxiliary_loss_clip": 0.02650066, + "auxiliary_loss_mlp": 0.01313125, + "balance_loss_clip": 1.53687334, + "balance_loss_mlp": 1.13183117, + "epoch": 0.003487043828533638, + "flos": 24426115269120.0, + "grad_norm": 2.4372971737909914, + "language_loss": 1.00069487, + "learning_rate": 2.4394238264681557e-06, + "loss": 1.04032671, + "num_input_tokens_seen": 714340, + "router_z_loss_clip": 11.1328125, + "router_z_loss_mlp": 1.81152344, + "step": 29, + "time_per_iteration": 3.1337966918945312 + }, + { + "auxiliary_loss_clip": 0.02584271, + "auxiliary_loss_mlp": 0.01287557, + "balance_loss_clip": 1.52507627, + "balance_loss_mlp": 1.11923313, + "epoch": 0.003607286719172729, + "flos": 26140311002880.0, + "grad_norm": 2.0285235511734494, + "language_loss": 0.99673903, + "learning_rate": 2.4639836682781433e-06, + "loss": 1.03545725, + "num_input_tokens_seen": 734470, + "router_z_loss_clip": 10.6015625, + "router_z_loss_mlp": 1.68359375, + "step": 30, + "time_per_iteration": 3.018602132797241 + }, + { + "auxiliary_loss_clip": 0.02525776, + "auxiliary_loss_mlp": 0.0133199, + "balance_loss_clip": 1.51534343, + "balance_loss_mlp": 1.16624165, + "epoch": 0.00372752960981182, + "flos": 20593082113920.0, + "grad_norm": 2.620776065856624, + "language_loss": 1.00396776, + "learning_rate": 2.487738122623307e-06, + "loss": 1.04254544, + "num_input_tokens_seen": 753380, + "router_z_loss_clip": 10.0859375, + "router_z_loss_mlp": 1.65820312, + "step": 31, + "time_per_iteration": 2.961134433746338 + }, + { + "auxiliary_loss_clip": 0.02490627, + "auxiliary_loss_mlp": 0.01281683, + "balance_loss_clip": 1.49572897, + "balance_loss_mlp": 1.11870003, + "epoch": 0.003847772500450911, + "flos": 22674608282880.0, + "grad_norm": 2.7789077109578924, + "language_loss": 0.99132276, + "learning_rate": 2.510738338534912e-06, + "loss": 1.02904594, + "num_input_tokens_seen": 772105, + "router_z_loss_clip": 9.9609375, + "router_z_loss_mlp": 1.62890625, + "step": 32, + "time_per_iteration": 3.019542932510376 + }, + { + "auxiliary_loss_clip": 0.02400187, + "auxiliary_loss_mlp": 0.01320344, + "balance_loss_clip": 1.47976053, + "balance_loss_mlp": 1.1545949, + "epoch": 0.003968015391090002, + "flos": 17967796882560.0, + "grad_norm": 3.2556998467615723, + "language_loss": 1.02549839, + "learning_rate": 2.5330307420306648e-06, + "loss": 1.06270361, + "num_input_tokens_seen": 788955, + "router_z_loss_clip": 9.203125, + "router_z_loss_mlp": 1.65722656, + "step": 33, + "time_per_iteration": 2.963702917098999 + }, + { + "auxiliary_loss_clip": 0.0235084, + "auxiliary_loss_mlp": 0.01271843, + "balance_loss_clip": 1.47628021, + "balance_loss_mlp": 1.11849213, + "epoch": 0.004088258281729093, + "flos": 27304103658240.0, + "grad_norm": 2.6552973792466474, + "language_loss": 0.8807956, + "learning_rate": 2.554657600279796e-06, + "loss": 0.91702247, + "num_input_tokens_seen": 810230, + "router_z_loss_clip": 8.75, + "router_z_loss_mlp": 1.53417969, + "step": 34, + "time_per_iteration": 3.0041751861572266 + }, + { + "auxiliary_loss_clip": 0.02314898, + "auxiliary_loss_mlp": 0.01323971, + "balance_loss_clip": 1.46887398, + "balance_loss_mlp": 1.17653298, + "epoch": 0.004208501172368184, + "flos": 23258587599360.0, + "grad_norm": 2.2495040729803435, + "language_loss": 1.03354609, + "learning_rate": 2.5756575039679493e-06, + "loss": 1.06993484, + "num_input_tokens_seen": 829780, + "router_z_loss_clip": 8.46875, + "router_z_loss_mlp": 1.47363281, + "step": 35, + "time_per_iteration": 3.0076639652252197 + }, + { + "auxiliary_loss_clip": 0.02246696, + "auxiliary_loss_mlp": 0.01281247, + "balance_loss_clip": 1.44652772, + "balance_loss_mlp": 1.14448977, + "epoch": 0.0043287440630072746, + "flos": 17312104062720.0, + "grad_norm": 1.8393629027674236, + "language_loss": 0.95138299, + "learning_rate": 2.5960657816942747e-06, + "loss": 0.98666239, + "num_input_tokens_seen": 848695, + "router_z_loss_clip": 8.00390625, + "router_z_loss_mlp": 1.3671875, + "step": 36, + "time_per_iteration": 2.9506449699401855 + }, + { + "auxiliary_loss_clip": 0.02052101, + "auxiliary_loss_mlp": 0.01706101, + "balance_loss_clip": 1.58429229, + "balance_loss_mlp": 1.65116966, + "epoch": 0.004448986953646365, + "flos": 53092491160320.0, + "grad_norm": 1.4112955045194189, + "language_loss": 0.60926062, + "learning_rate": 2.6159148575788668e-06, + "loss": 0.6468426, + "num_input_tokens_seen": 906730, + "router_z_loss_clip": 4.6875, + "router_z_loss_mlp": 0.55078125, + "step": 37, + "time_per_iteration": 3.367563009262085 + }, + { + "auxiliary_loss_clip": 0.02193487, + "auxiliary_loss_mlp": 0.01311646, + "balance_loss_clip": 1.430902, + "balance_loss_mlp": 1.17503178, + "epoch": 0.004569229844285457, + "flos": 13444165866240.0, + "grad_norm": 2.327447079959112, + "language_loss": 0.98927188, + "learning_rate": 2.635234561171e-06, + "loss": 1.02432323, + "num_input_tokens_seen": 925125, + "router_z_loss_clip": 7.62890625, + "router_z_loss_mlp": 1.36572266, + "step": 38, + "time_per_iteration": 3.005049467086792 + }, + { + "auxiliary_loss_clip": 0.02133155, + "auxiliary_loss_mlp": 0.01239222, + "balance_loss_clip": 1.42124927, + "balance_loss_mlp": 1.12678373, + "epoch": 0.0046894727349245475, + "flos": 16209609966720.0, + "grad_norm": 2.3180938186492357, + "language_loss": 0.94272143, + "learning_rate": 2.6540523970949877e-06, + "loss": 0.9764452, + "num_input_tokens_seen": 939970, + "router_z_loss_clip": 7.1171875, + "router_z_loss_mlp": 1.12402344, + "step": 39, + "time_per_iteration": 3.005795478820801 + }, + { + "auxiliary_loss_clip": 0.02121965, + "auxiliary_loss_mlp": 0.01270031, + "balance_loss_clip": 1.42247331, + "balance_loss_mlp": 1.15806961, + "epoch": 0.004809715625563638, + "flos": 23914244505600.0, + "grad_norm": 2.633350280439338, + "language_loss": 0.92482781, + "learning_rate": 2.6723937805519533e-06, + "loss": 0.95874774, + "num_input_tokens_seen": 957470, + "router_z_loss_clip": 6.98828125, + "router_z_loss_mlp": 1.11865234, + "step": 40, + "time_per_iteration": 3.0632331371307373 + }, + { + "auxiliary_loss_clip": 0.0210941, + "auxiliary_loss_mlp": 0.01254074, + "balance_loss_clip": 1.41106093, + "balance_loss_mlp": 1.1398716, + "epoch": 0.00492995851620273, + "flos": 20773030273920.0, + "grad_norm": 2.0469113582990612, + "language_loss": 0.92972255, + "learning_rate": 2.690282243737839e-06, + "loss": 0.96335739, + "num_input_tokens_seen": 976405, + "router_z_loss_clip": 6.984375, + "router_z_loss_mlp": 1.14208984, + "step": 41, + "time_per_iteration": 2.9929702281951904 + }, + { + "auxiliary_loss_clip": 0.020741, + "auxiliary_loss_mlp": 0.0123894, + "balance_loss_clip": 1.39628446, + "balance_loss_mlp": 1.12626338, + "epoch": 0.0050502014068418205, + "flos": 20338655103360.0, + "grad_norm": 8.274198622841128, + "language_loss": 0.9931891, + "learning_rate": 2.7077396173840807e-06, + "loss": 1.0263195, + "num_input_tokens_seen": 994690, + "router_z_loss_clip": 6.7734375, + "router_z_loss_mlp": 1.12744141, + "step": 42, + "time_per_iteration": 2.9886550903320312 + }, + { + "auxiliary_loss_clip": 0.02023051, + "auxiliary_loss_mlp": 0.0123411, + "balance_loss_clip": 1.38858426, + "balance_loss_mlp": 1.13235307, + "epoch": 0.005170444297480911, + "flos": 25994872834560.0, + "grad_norm": 2.0940761628682965, + "language_loss": 0.92787063, + "learning_rate": 2.7247861909342594e-06, + "loss": 0.96044225, + "num_input_tokens_seen": 1015615, + "router_z_loss_clip": 6.34765625, + "router_z_loss_mlp": 1.01708984, + "step": 43, + "time_per_iteration": 3.019174575805664 + }, + { + "auxiliary_loss_clip": 0.02004936, + "auxiliary_loss_mlp": 0.01230631, + "balance_loss_clip": 1.38602233, + "balance_loss_mlp": 1.13473868, + "epoch": 0.005290687188120003, + "flos": 20954055841920.0, + "grad_norm": 2.2025539307047017, + "language_loss": 0.82815325, + "learning_rate": 2.7414408543044743e-06, + "loss": 0.86050886, + "num_input_tokens_seen": 1031255, + "router_z_loss_clip": 6.1875, + "router_z_loss_mlp": 0.95996094, + "step": 44, + "time_per_iteration": 3.020174026489258 + }, + { + "auxiliary_loss_clip": 0.02007926, + "auxiliary_loss_mlp": 0.01249538, + "balance_loss_clip": 1.38400531, + "balance_loss_mlp": 1.15560126, + "epoch": 0.005410930078759093, + "flos": 15851401585920.0, + "grad_norm": 5.135366334579933, + "language_loss": 0.79367465, + "learning_rate": 2.7577212237113157e-06, + "loss": 0.8262493, + "num_input_tokens_seen": 1048295, + "router_z_loss_clip": 6.23828125, + "router_z_loss_mlp": 0.93945312, + "step": 45, + "time_per_iteration": 3.0140926837921143 + }, + { + "auxiliary_loss_clip": 0.0196765, + "auxiliary_loss_mlp": 0.01226745, + "balance_loss_clip": 1.380669, + "balance_loss_mlp": 1.13662243, + "epoch": 0.005531172969398184, + "flos": 21104988791040.0, + "grad_norm": 2.268982232243637, + "language_loss": 1.04250884, + "learning_rate": 2.7736437536690466e-06, + "loss": 1.07445264, + "num_input_tokens_seen": 1067925, + "router_z_loss_clip": 5.8671875, + "router_z_loss_mlp": 0.90087891, + "step": 46, + "time_per_iteration": 3.9459030628204346 + }, + { + "auxiliary_loss_clip": 0.01968763, + "auxiliary_loss_mlp": 0.01246127, + "balance_loss_clip": 1.37142229, + "balance_loss_mlp": 1.1559577, + "epoch": 0.005651415860037276, + "flos": 20844887431680.0, + "grad_norm": 2.6214303737148774, + "language_loss": 1.07962418, + "learning_rate": 2.789223836941131e-06, + "loss": 1.11177301, + "num_input_tokens_seen": 1088060, + "router_z_loss_clip": 5.9765625, + "router_z_loss_mlp": 0.90185547, + "step": 47, + "time_per_iteration": 3.866034984588623 + }, + { + "auxiliary_loss_clip": 0.01931572, + "auxiliary_loss_mlp": 0.01243499, + "balance_loss_clip": 1.36423516, + "balance_loss_mlp": 1.15433049, + "epoch": 0.005771658750676366, + "flos": 13260195383040.0, + "grad_norm": 2.3614567655768717, + "language_loss": 1.08812654, + "learning_rate": 2.8044758939680847e-06, + "loss": 1.11987722, + "num_input_tokens_seen": 1104130, + "router_z_loss_clip": 5.66796875, + "router_z_loss_mlp": 0.89208984, + "step": 48, + "time_per_iteration": 3.0406956672668457 + }, + { + "auxiliary_loss_clip": 0.01921496, + "auxiliary_loss_mlp": 0.01219467, + "balance_loss_clip": 1.36729121, + "balance_loss_mlp": 1.13692594, + "epoch": 0.005891901641315457, + "flos": 24425396997120.0, + "grad_norm": 3.011693056893026, + "language_loss": 1.02023935, + "learning_rate": 2.8194134530738863e-06, + "loss": 1.05164909, + "num_input_tokens_seen": 1122900, + "router_z_loss_clip": 5.5390625, + "router_z_loss_mlp": 0.82519531, + "step": 49, + "time_per_iteration": 3.0730714797973633 + }, + { + "auxiliary_loss_clip": 0.01899192, + "auxiliary_loss_mlp": 0.01240706, + "balance_loss_clip": 1.3565675, + "balance_loss_mlp": 1.16369724, + "epoch": 0.006012144531954548, + "flos": 23076197314560.0, + "grad_norm": 3.053733393205025, + "language_loss": 0.90169656, + "learning_rate": 2.834049222568994e-06, + "loss": 0.93309557, + "num_input_tokens_seen": 1140250, + "router_z_loss_clip": 5.42578125, + "router_z_loss_mlp": 0.77001953, + "step": 50, + "time_per_iteration": 2.958300828933716 + }, + { + "auxiliary_loss_clip": 0.01895383, + "auxiliary_loss_mlp": 0.01200393, + "balance_loss_clip": 1.35181499, + "balance_loss_mlp": 1.1253866, + "epoch": 0.006132387422593639, + "flos": 22528775064960.0, + "grad_norm": 6.035623198571278, + "language_loss": 0.92545462, + "learning_rate": 2.848395155712969e-06, + "loss": 0.95641243, + "num_input_tokens_seen": 1160470, + "router_z_loss_clip": 5.43359375, + "router_z_loss_mlp": 0.75, + "step": 51, + "time_per_iteration": 3.04852557182312 + }, + { + "auxiliary_loss_clip": 0.01891859, + "auxiliary_loss_mlp": 0.01193904, + "balance_loss_clip": 1.35870075, + "balance_loss_mlp": 1.11904073, + "epoch": 0.00625263031323273, + "flos": 27628340751360.0, + "grad_norm": 2.2109567577438027, + "language_loss": 0.974051, + "learning_rate": 2.8624625093687977e-06, + "loss": 1.00490868, + "num_input_tokens_seen": 1177605, + "router_z_loss_clip": 5.33203125, + "router_z_loss_mlp": 0.74853516, + "step": 52, + "time_per_iteration": 3.0058486461639404 + }, + { + "auxiliary_loss_clip": 0.0186512, + "auxiliary_loss_mlp": 0.01191258, + "balance_loss_clip": 1.33977652, + "balance_loss_mlp": 1.1153934, + "epoch": 0.006372873203871821, + "flos": 23110671392640.0, + "grad_norm": 2.086332119123683, + "language_loss": 0.89035356, + "learning_rate": 2.876261897070029e-06, + "loss": 0.92091733, + "num_input_tokens_seen": 1197735, + "router_z_loss_clip": 5.25, + "router_z_loss_mlp": 0.7578125, + "step": 53, + "time_per_iteration": 2.976712942123413 + }, + { + "auxiliary_loss_clip": 0.01871673, + "auxiliary_loss_mlp": 0.01180626, + "balance_loss_clip": 1.34286976, + "balance_loss_mlp": 1.10809922, + "epoch": 0.006493116094510912, + "flos": 22856028900480.0, + "grad_norm": 5.1653702467508955, + "language_loss": 0.92357409, + "learning_rate": 2.889803337127447e-06, + "loss": 0.95409703, + "num_input_tokens_seen": 1216335, + "router_z_loss_clip": 5.29296875, + "router_z_loss_mlp": 0.72509766, + "step": 54, + "time_per_iteration": 3.0579512119293213 + }, + { + "auxiliary_loss_clip": 0.01869527, + "auxiliary_loss_mlp": 0.01180139, + "balance_loss_clip": 1.34385455, + "balance_loss_mlp": 1.10250986, + "epoch": 0.006613358985150003, + "flos": 23071708114560.0, + "grad_norm": 2.5160256141975506, + "language_loss": 0.84810036, + "learning_rate": 2.903096296321516e-06, + "loss": 0.87859702, + "num_input_tokens_seen": 1234480, + "router_z_loss_clip": 5.2578125, + "router_z_loss_mlp": 0.77636719, + "step": 55, + "time_per_iteration": 2.9621596336364746 + }, + { + "auxiliary_loss_clip": 0.01837799, + "auxiliary_loss_mlp": 0.01201751, + "balance_loss_clip": 1.3328073, + "balance_loss_mlp": 1.12550426, + "epoch": 0.006733601875789094, + "flos": 26537662229760.0, + "grad_norm": 2.371070742313244, + "language_loss": 0.91750091, + "learning_rate": 2.9161497296578907e-06, + "loss": 0.94789642, + "num_input_tokens_seen": 1253870, + "router_z_loss_clip": 5.05078125, + "router_z_loss_mlp": 0.76220703, + "step": 56, + "time_per_iteration": 3.0195860862731934 + }, + { + "auxiliary_loss_clip": 0.01832863, + "auxiliary_loss_mlp": 0.01176928, + "balance_loss_clip": 1.32630491, + "balance_loss_mlp": 1.10130191, + "epoch": 0.006853844766428185, + "flos": 15523178083200.0, + "grad_norm": 2.356388891149499, + "language_loss": 0.85852635, + "learning_rate": 2.928972116604173e-06, + "loss": 0.88862419, + "num_input_tokens_seen": 1270145, + "router_z_loss_clip": 5.0625, + "router_z_loss_mlp": 0.75634766, + "step": 57, + "time_per_iteration": 2.9248955249786377 + }, + { + "auxiliary_loss_clip": 0.01817558, + "auxiliary_loss_mlp": 0.01146529, + "balance_loss_clip": 1.32132387, + "balance_loss_mlp": 1.07414532, + "epoch": 0.006974087657067276, + "flos": 24243760897920.0, + "grad_norm": 1.9547451524743407, + "language_loss": 1.0207684, + "learning_rate": 2.9415714941751377e-06, + "loss": 1.05040932, + "num_input_tokens_seen": 1291365, + "router_z_loss_clip": 4.96484375, + "router_z_loss_mlp": 0.72412109, + "step": 58, + "time_per_iteration": 3.007565498352051 + }, + { + "auxiliary_loss_clip": 0.01809867, + "auxiliary_loss_mlp": 0.01156543, + "balance_loss_clip": 1.31724977, + "balance_loss_mlp": 1.0823946, + "epoch": 0.007094330547706367, + "flos": 25772513690880.0, + "grad_norm": 2.008015329789242, + "language_loss": 0.93546939, + "learning_rate": 2.9539554871897396e-06, + "loss": 0.96513355, + "num_input_tokens_seen": 1311535, + "router_z_loss_clip": 4.9296875, + "router_z_loss_mlp": 0.7421875, + "step": 59, + "time_per_iteration": 3.116995334625244 + }, + { + "auxiliary_loss_clip": 0.01804785, + "auxiliary_loss_mlp": 0.01170965, + "balance_loss_clip": 1.31995177, + "balance_loss_mlp": 1.10191941, + "epoch": 0.007214573438345458, + "flos": 21319015979520.0, + "grad_norm": 1.9906534147795103, + "language_loss": 0.97464496, + "learning_rate": 2.9661313359851253e-06, + "loss": 1.0044024, + "num_input_tokens_seen": 1329420, + "router_z_loss_clip": 4.8515625, + "router_z_loss_mlp": 0.69140625, + "step": 60, + "time_per_iteration": 3.0450363159179688 + }, + { + "auxiliary_loss_clip": 0.01808237, + "auxiliary_loss_mlp": 0.01157417, + "balance_loss_clip": 1.31868613, + "balance_loss_mlp": 1.08703542, + "epoch": 0.007334816328984549, + "flos": 24937088192640.0, + "grad_norm": 2.715441381426959, + "language_loss": 0.93982166, + "learning_rate": 2.978105921839922e-06, + "loss": 0.96947819, + "num_input_tokens_seen": 1349965, + "router_z_loss_clip": 4.89453125, + "router_z_loss_mlp": 0.70361328, + "step": 61, + "time_per_iteration": 3.1619839668273926 + }, + { + "auxiliary_loss_clip": 0.01794847, + "auxiliary_loss_mlp": 0.01180884, + "balance_loss_clip": 1.31310546, + "balance_loss_mlp": 1.10854769, + "epoch": 0.00745505921962364, + "flos": 18510586277760.0, + "grad_norm": 2.1075497852818623, + "language_loss": 0.72196615, + "learning_rate": 2.9898857903302893e-06, + "loss": 0.75172341, + "num_input_tokens_seen": 1368915, + "router_z_loss_clip": 4.8046875, + "router_z_loss_mlp": 0.72314453, + "step": 62, + "time_per_iteration": 3.102274179458618 + }, + { + "auxiliary_loss_clip": 0.01804258, + "auxiliary_loss_mlp": 0.01180182, + "balance_loss_clip": 1.31780934, + "balance_loss_mlp": 1.11237609, + "epoch": 0.007575302110262731, + "flos": 18477656484480.0, + "grad_norm": 2.84017546461817, + "language_loss": 0.87945938, + "learning_rate": 3.001477172817253e-06, + "loss": 0.90930378, + "num_input_tokens_seen": 1386805, + "router_z_loss_clip": 4.87109375, + "router_z_loss_mlp": 0.67822266, + "step": 63, + "time_per_iteration": 3.025564193725586 + }, + { + "auxiliary_loss_clip": 0.01780732, + "auxiliary_loss_mlp": 0.01172485, + "balance_loss_clip": 1.30913508, + "balance_loss_mlp": 1.10296214, + "epoch": 0.007695545000901822, + "flos": 24973178382720.0, + "grad_norm": 2.720091457478954, + "language_loss": 0.96441114, + "learning_rate": 3.012886006241894e-06, + "loss": 0.99394321, + "num_input_tokens_seen": 1406190, + "router_z_loss_clip": 4.71875, + "router_z_loss_mlp": 0.6953125, + "step": 64, + "time_per_iteration": 3.036529541015625 + }, + { + "auxiliary_loss_clip": 0.01771262, + "auxiliary_loss_mlp": 0.01162741, + "balance_loss_clip": 1.30227637, + "balance_loss_mlp": 1.09202611, + "epoch": 0.007815787891540913, + "flos": 21324223451520.0, + "grad_norm": 2.060223495016894, + "language_loss": 0.88033372, + "learning_rate": 3.0241179513858383e-06, + "loss": 0.90967375, + "num_input_tokens_seen": 1425500, + "router_z_loss_clip": 4.69140625, + "router_z_loss_mlp": 0.70800781, + "step": 65, + "time_per_iteration": 3.0000221729278564 + }, + { + "auxiliary_loss_clip": 0.01766821, + "auxiliary_loss_mlp": 0.01156866, + "balance_loss_clip": 1.29879475, + "balance_loss_mlp": 1.08424377, + "epoch": 0.007936030782180003, + "flos": 21575777374080.0, + "grad_norm": 2.0045426543901557, + "language_loss": 0.87786323, + "learning_rate": 3.035178409737647e-06, + "loss": 0.90710008, + "num_input_tokens_seen": 1442950, + "router_z_loss_clip": 4.68359375, + "router_z_loss_mlp": 0.7265625, + "step": 66, + "time_per_iteration": 3.1387715339660645 + }, + { + "auxiliary_loss_clip": 0.01745586, + "auxiliary_loss_mlp": 0.0113517, + "balance_loss_clip": 1.28609908, + "balance_loss_mlp": 1.0689851, + "epoch": 0.008056273672819095, + "flos": 20120785159680.0, + "grad_norm": 2.4972893466397137, + "language_loss": 0.88551086, + "learning_rate": 3.046072539090907e-06, + "loss": 0.91431838, + "num_input_tokens_seen": 1460915, + "router_z_loss_clip": 4.59375, + "router_z_loss_mlp": 0.66210938, + "step": 67, + "time_per_iteration": 3.0142083168029785 + }, + { + "auxiliary_loss_clip": 0.01750108, + "auxiliary_loss_mlp": 0.01147547, + "balance_loss_clip": 1.28532231, + "balance_loss_mlp": 1.07454312, + "epoch": 0.008176516563458186, + "flos": 18333116156160.0, + "grad_norm": 2.273881367186026, + "language_loss": 1.04626846, + "learning_rate": 3.056805267986779e-06, + "loss": 1.07524502, + "num_input_tokens_seen": 1478385, + "router_z_loss_clip": 4.64453125, + "router_z_loss_mlp": 0.72998047, + "step": 68, + "time_per_iteration": 3.0432093143463135 + }, + { + "auxiliary_loss_clip": 0.01745615, + "auxiliary_loss_mlp": 0.01151854, + "balance_loss_clip": 1.28906322, + "balance_loss_mlp": 1.08032799, + "epoch": 0.008296759454097276, + "flos": 21872076664320.0, + "grad_norm": 2.5553609252344085, + "language_loss": 0.95452088, + "learning_rate": 3.0673813091022194e-06, + "loss": 0.98349559, + "num_input_tokens_seen": 1497605, + "router_z_loss_clip": 4.5703125, + "router_z_loss_mlp": 0.71533203, + "step": 69, + "time_per_iteration": 2.9853739738464355 + }, + { + "auxiliary_loss_clip": 0.01874969, + "auxiliary_loss_mlp": 0.01524489, + "balance_loss_clip": 1.4666338, + "balance_loss_mlp": 1.46421659, + "epoch": 0.008417002344736368, + "flos": 63408228036480.0, + "grad_norm": 1.3737515863831862, + "language_loss": 0.62123716, + "learning_rate": 3.0778051716749317e-06, + "loss": 0.65523177, + "num_input_tokens_seen": 1561150, + "router_z_loss_clip": 4.09375, + "router_z_loss_mlp": 0.6015625, + "step": 70, + "time_per_iteration": 3.4771549701690674 + }, + { + "auxiliary_loss_clip": 0.01729505, + "auxiliary_loss_mlp": 0.01165542, + "balance_loss_clip": 1.2786535, + "balance_loss_mlp": 1.09616208, + "epoch": 0.008537245235375458, + "flos": 22966454286720.0, + "grad_norm": 2.0109457723298925, + "language_loss": 0.9052301, + "learning_rate": 3.0880811730470094e-06, + "loss": 0.93418056, + "num_input_tokens_seen": 1580605, + "router_z_loss_clip": 4.50390625, + "router_z_loss_mlp": 0.69384766, + "step": 71, + "time_per_iteration": 2.9994542598724365 + }, + { + "auxiliary_loss_clip": 0.01822197, + "auxiliary_loss_mlp": 0.01371454, + "balance_loss_clip": 1.45077991, + "balance_loss_mlp": 1.30927491, + "epoch": 0.008657488126014549, + "flos": 61984046712960.0, + "grad_norm": 1.2152243225705044, + "language_loss": 0.58660442, + "learning_rate": 3.098213449401257e-06, + "loss": 0.61854094, + "num_input_tokens_seen": 1647535, + "router_z_loss_clip": 3.71875, + "router_z_loss_mlp": 0.62109375, + "step": 72, + "time_per_iteration": 3.355011463165283 + }, + { + "auxiliary_loss_clip": 0.01718398, + "auxiliary_loss_mlp": 0.01149924, + "balance_loss_clip": 1.2663672, + "balance_loss_mlp": 1.07916129, + "epoch": 0.00877773101665364, + "flos": 30296791152000.0, + "grad_norm": 2.6523125787684263, + "language_loss": 0.9897418, + "learning_rate": 3.1082059657570015e-06, + "loss": 1.01842499, + "num_input_tokens_seen": 1666770, + "router_z_loss_clip": 4.51953125, + "router_z_loss_mlp": 0.70703125, + "step": 73, + "time_per_iteration": 3.0120937824249268 + }, + { + "auxiliary_loss_clip": 0.01708528, + "auxiliary_loss_mlp": 0.01145451, + "balance_loss_clip": 1.26497638, + "balance_loss_mlp": 1.07111239, + "epoch": 0.00889797390729273, + "flos": 23514056104320.0, + "grad_norm": 2.155041504581898, + "language_loss": 0.96699798, + "learning_rate": 3.1180625252858496e-06, + "loss": 0.99553782, + "num_input_tokens_seen": 1685200, + "router_z_loss_clip": 4.4296875, + "router_z_loss_mlp": 0.74316406, + "step": 74, + "time_per_iteration": 3.0244619846343994 + }, + { + "auxiliary_loss_clip": 0.01704382, + "auxiliary_loss_mlp": 0.01167826, + "balance_loss_clip": 1.263309, + "balance_loss_mlp": 1.0975883, + "epoch": 0.009018216797931822, + "flos": 23075838178560.0, + "grad_norm": 3.0968932310894988, + "language_loss": 0.80140704, + "learning_rate": 3.1277867780021663e-06, + "loss": 0.83012915, + "num_input_tokens_seen": 1701835, + "router_z_loss_clip": 4.4140625, + "router_z_loss_mlp": 0.70263672, + "step": 75, + "time_per_iteration": 3.9621777534484863 + }, + { + "auxiliary_loss_clip": 0.01693653, + "auxiliary_loss_mlp": 0.01178303, + "balance_loss_clip": 1.25667906, + "balance_loss_mlp": 1.10754073, + "epoch": 0.009138459688570914, + "flos": 15918877284480.0, + "grad_norm": 1.9363899886658749, + "language_loss": 0.95738608, + "learning_rate": 3.1373822288779824e-06, + "loss": 0.98610562, + "num_input_tokens_seen": 1718415, + "router_z_loss_clip": 4.3671875, + "router_z_loss_mlp": 0.70751953, + "step": 76, + "time_per_iteration": 2.941425323486328 + }, + { + "auxiliary_loss_clip": 0.0170907, + "auxiliary_loss_mlp": 0.01206211, + "balance_loss_clip": 1.25932252, + "balance_loss_mlp": 1.13292158, + "epoch": 0.009258702579210003, + "flos": 27016531372800.0, + "grad_norm": 2.068392713912233, + "language_loss": 0.79520172, + "learning_rate": 3.1468522454274533e-06, + "loss": 0.82435453, + "num_input_tokens_seen": 1738770, + "router_z_loss_clip": 4.5, + "router_z_loss_mlp": 0.73242188, + "step": 77, + "time_per_iteration": 2.9839978218078613 + }, + { + "auxiliary_loss_clip": 0.01692183, + "auxiliary_loss_mlp": 0.01186009, + "balance_loss_clip": 1.25900936, + "balance_loss_mlp": 1.11333883, + "epoch": 0.009378945469849095, + "flos": 26903196984960.0, + "grad_norm": 2.741966565968236, + "language_loss": 0.9168523, + "learning_rate": 3.15620006480197e-06, + "loss": 0.94563425, + "num_input_tokens_seen": 1758040, + "router_z_loss_clip": 4.33203125, + "router_z_loss_mlp": 0.72753906, + "step": 78, + "time_per_iteration": 3.053584575653076 + }, + { + "auxiliary_loss_clip": 0.01706032, + "auxiliary_loss_mlp": 0.01205728, + "balance_loss_clip": 1.25660372, + "balance_loss_mlp": 1.13181865, + "epoch": 0.009499188360488187, + "flos": 35694236327040.0, + "grad_norm": 2.950021080536837, + "language_loss": 0.74696088, + "learning_rate": 3.1654288004333087e-06, + "loss": 0.77607846, + "num_input_tokens_seen": 1776705, + "router_z_loss_clip": 4.4921875, + "router_z_loss_mlp": 0.73974609, + "step": 79, + "time_per_iteration": 3.0750582218170166 + }, + { + "auxiliary_loss_clip": 0.0167208, + "auxiliary_loss_mlp": 0.01163047, + "balance_loss_clip": 1.24988091, + "balance_loss_mlp": 1.08751631, + "epoch": 0.009619431251127276, + "flos": 21503201944320.0, + "grad_norm": 2.5956058508964053, + "language_loss": 0.76016223, + "learning_rate": 3.1745414482589353e-06, + "loss": 0.78851348, + "num_input_tokens_seen": 1795915, + "router_z_loss_clip": 4.21875, + "router_z_loss_mlp": 0.75537109, + "step": 80, + "time_per_iteration": 2.9796981811523438 + }, + { + "auxiliary_loss_clip": 0.01658589, + "auxiliary_loss_mlp": 0.01194569, + "balance_loss_clip": 1.24251294, + "balance_loss_mlp": 1.12557125, + "epoch": 0.009739674141766368, + "flos": 17421056991360.0, + "grad_norm": 3.0278087491307604, + "language_loss": 0.87142509, + "learning_rate": 3.1835408925606204e-06, + "loss": 0.8999567, + "num_input_tokens_seen": 1814055, + "router_z_loss_clip": 4.15820312, + "router_z_loss_mlp": 0.69042969, + "step": 81, + "time_per_iteration": 2.9523160457611084 + }, + { + "auxiliary_loss_clip": 0.01653487, + "auxiliary_loss_mlp": 0.01170681, + "balance_loss_clip": 1.24138486, + "balance_loss_mlp": 1.09925044, + "epoch": 0.00985991703240546, + "flos": 27527109246720.0, + "grad_norm": 2.1536533304519097, + "language_loss": 0.89142883, + "learning_rate": 3.1924299114448214e-06, + "loss": 0.91967046, + "num_input_tokens_seen": 1834535, + "router_z_loss_clip": 4.12109375, + "router_z_loss_mlp": 0.71484375, + "step": 82, + "time_per_iteration": 2.9844141006469727 + }, + { + "auxiliary_loss_clip": 0.0165765, + "auxiliary_loss_mlp": 0.01175004, + "balance_loss_clip": 1.2365768, + "balance_loss_mlp": 1.10018849, + "epoch": 0.00998015992304455, + "flos": 13808084509440.0, + "grad_norm": 4.740408806056361, + "language_loss": 0.83589143, + "learning_rate": 3.2012111819909055e-06, + "loss": 0.86421794, + "num_input_tokens_seen": 1851865, + "router_z_loss_clip": 4.2109375, + "router_z_loss_mlp": 0.74755859, + "step": 83, + "time_per_iteration": 2.9658730030059814 + }, + { + "auxiliary_loss_clip": 0.01639251, + "auxiliary_loss_mlp": 0.01184798, + "balance_loss_clip": 1.22846675, + "balance_loss_mlp": 1.1137017, + "epoch": 0.010100402813683641, + "flos": 20191385341440.0, + "grad_norm": 2.360184443003567, + "language_loss": 0.94949436, + "learning_rate": 3.2098872850910627e-06, + "loss": 0.97773486, + "num_input_tokens_seen": 1868540, + "router_z_loss_clip": 4.109375, + "router_z_loss_mlp": 0.71142578, + "step": 84, + "time_per_iteration": 2.965941905975342 + }, + { + "auxiliary_loss_clip": 0.01643634, + "auxiliary_loss_mlp": 0.01203649, + "balance_loss_clip": 1.22987497, + "balance_loss_mlp": 1.13360167, + "epoch": 0.010220645704322733, + "flos": 17201642762880.0, + "grad_norm": 1.9265768482517969, + "language_loss": 0.89249468, + "learning_rate": 3.2184607100038194e-06, + "loss": 0.92096746, + "num_input_tokens_seen": 1887180, + "router_z_loss_clip": 4.140625, + "router_z_loss_mlp": 0.70117188, + "step": 85, + "time_per_iteration": 3.021751880645752 + }, + { + "auxiliary_loss_clip": 0.0163443, + "auxiliary_loss_mlp": 0.01151316, + "balance_loss_clip": 1.22730303, + "balance_loss_mlp": 1.0783596, + "epoch": 0.010340888594961822, + "flos": 21470415805440.0, + "grad_norm": 2.5636078721584146, + "language_loss": 0.93093193, + "learning_rate": 3.2269338586412414e-06, + "loss": 0.95878947, + "num_input_tokens_seen": 1904765, + "router_z_loss_clip": 4.06835938, + "router_z_loss_mlp": 0.72949219, + "step": 86, + "time_per_iteration": 3.068331003189087 + }, + { + "auxiliary_loss_clip": 0.01627215, + "auxiliary_loss_mlp": 0.01169599, + "balance_loss_clip": 1.22380936, + "balance_loss_mlp": 1.09964728, + "epoch": 0.010461131485600914, + "flos": 23002831785600.0, + "grad_norm": 2.341876734099171, + "language_loss": 0.96356261, + "learning_rate": 3.2353090496083106e-06, + "loss": 0.99153078, + "num_input_tokens_seen": 1922600, + "router_z_loss_clip": 4.02929688, + "router_z_loss_mlp": 0.69970703, + "step": 87, + "time_per_iteration": 3.067192316055298 + }, + { + "auxiliary_loss_clip": 0.01613229, + "auxiliary_loss_mlp": 0.01171776, + "balance_loss_clip": 1.22365808, + "balance_loss_mlp": 1.1028254, + "epoch": 0.010581374376240005, + "flos": 33546850571520.0, + "grad_norm": 1.851280785048922, + "language_loss": 0.81272745, + "learning_rate": 3.2435885220114572e-06, + "loss": 0.84057748, + "num_input_tokens_seen": 1943950, + "router_z_loss_clip": 3.89453125, + "router_z_loss_mlp": 0.68945312, + "step": 88, + "time_per_iteration": 3.0062694549560547 + }, + { + "auxiliary_loss_clip": 0.01615731, + "auxiliary_loss_mlp": 0.01180581, + "balance_loss_clip": 1.22373366, + "balance_loss_mlp": 1.10843599, + "epoch": 0.010701617266879095, + "flos": 21763087822080.0, + "grad_norm": 2.0987824160852866, + "language_loss": 0.93971241, + "learning_rate": 3.2517744390519113e-06, + "loss": 0.96767545, + "num_input_tokens_seen": 1962815, + "router_z_loss_clip": 3.91796875, + "router_z_loss_mlp": 0.72167969, + "step": 89, + "time_per_iteration": 2.972834348678589 + }, + { + "auxiliary_loss_clip": 0.01609912, + "auxiliary_loss_mlp": 0.01141755, + "balance_loss_clip": 1.21663618, + "balance_loss_mlp": 1.07447338, + "epoch": 0.010821860157518187, + "flos": 19060199256960.0, + "grad_norm": 3.177029303206305, + "language_loss": 0.75048518, + "learning_rate": 3.259868891418298e-06, + "loss": 0.77800179, + "num_input_tokens_seen": 1980580, + "router_z_loss_clip": 3.93554688, + "router_z_loss_mlp": 0.67285156, + "step": 90, + "time_per_iteration": 2.9419209957122803 + }, + { + "auxiliary_loss_clip": 0.01612173, + "auxiliary_loss_mlp": 0.01161818, + "balance_loss_clip": 1.21955204, + "balance_loss_mlp": 1.09272444, + "epoch": 0.010942103048157278, + "flos": 25447378757760.0, + "grad_norm": 2.544773798593532, + "language_loss": 0.85045075, + "learning_rate": 3.2678739004917757e-06, + "loss": 0.87819064, + "num_input_tokens_seen": 2000315, + "router_z_loss_clip": 3.9296875, + "router_z_loss_mlp": 0.69042969, + "step": 91, + "time_per_iteration": 2.9226434230804443 + }, + { + "auxiliary_loss_clip": 0.01603624, + "auxiliary_loss_mlp": 0.01159296, + "balance_loss_clip": 1.21857059, + "balance_loss_mlp": 1.08939123, + "epoch": 0.011062345938796368, + "flos": 27493928058240.0, + "grad_norm": 1.6337866454912782, + "language_loss": 0.92093962, + "learning_rate": 3.275791421376029e-06, + "loss": 0.94856882, + "num_input_tokens_seen": 2023760, + "router_z_loss_clip": 3.85546875, + "router_z_loss_mlp": 0.69921875, + "step": 92, + "time_per_iteration": 2.9918739795684814 + }, + { + "auxiliary_loss_clip": 0.01587786, + "auxiliary_loss_mlp": 0.01154098, + "balance_loss_clip": 1.2069701, + "balance_loss_mlp": 1.08552885, + "epoch": 0.01118258882943546, + "flos": 16071210864000.0, + "grad_norm": 2.095418117954405, + "language_loss": 0.96024704, + "learning_rate": 3.2836233457634622e-06, + "loss": 0.98766577, + "num_input_tokens_seen": 2041895, + "router_z_loss_clip": 3.80664062, + "router_z_loss_mlp": 0.68554688, + "step": 93, + "time_per_iteration": 2.872424602508545 + }, + { + "auxiliary_loss_clip": 0.01593058, + "auxiliary_loss_mlp": 0.01165818, + "balance_loss_clip": 1.20999515, + "balance_loss_mlp": 1.10006213, + "epoch": 0.011302831720074551, + "flos": 20668602458880.0, + "grad_norm": 2.2773538525968537, + "language_loss": 0.85282648, + "learning_rate": 3.2913715046481135e-06, + "loss": 0.8804152, + "num_input_tokens_seen": 2061640, + "router_z_loss_clip": 3.828125, + "router_z_loss_mlp": 0.65722656, + "step": 94, + "time_per_iteration": 2.917630910873413 + }, + { + "auxiliary_loss_clip": 0.01590247, + "auxiliary_loss_mlp": 0.01152393, + "balance_loss_clip": 1.20505595, + "balance_loss_mlp": 1.08620811, + "epoch": 0.011423074610713641, + "flos": 13072238490240.0, + "grad_norm": 2.065283084557949, + "language_loss": 0.88831753, + "learning_rate": 3.299037670895023e-06, + "loss": 0.91574395, + "num_input_tokens_seen": 2078255, + "router_z_loss_clip": 3.8515625, + "router_z_loss_mlp": 0.66162109, + "step": 95, + "time_per_iteration": 2.9178388118743896 + }, + { + "auxiliary_loss_clip": 0.01580885, + "auxiliary_loss_mlp": 0.01154544, + "balance_loss_clip": 1.20098317, + "balance_loss_mlp": 1.09083903, + "epoch": 0.011543317501352733, + "flos": 30335646689280.0, + "grad_norm": 2.1373942574375677, + "language_loss": 0.80251801, + "learning_rate": 3.3066235616750667e-06, + "loss": 0.82987225, + "num_input_tokens_seen": 2099490, + "router_z_loss_clip": 3.79492188, + "router_z_loss_mlp": 0.63720703, + "step": 96, + "time_per_iteration": 2.973956346511841 + }, + { + "auxiliary_loss_clip": 0.0157316, + "auxiliary_loss_mlp": 0.0115798, + "balance_loss_clip": 1.2017194, + "balance_loss_mlp": 1.08750391, + "epoch": 0.011663560391991824, + "flos": 15522962601600.0, + "grad_norm": 2.9981828547442793, + "language_loss": 0.9218033, + "learning_rate": 3.3141308407736276e-06, + "loss": 0.94911468, + "num_input_tokens_seen": 2116125, + "router_z_loss_clip": 3.71484375, + "router_z_loss_mlp": 0.70458984, + "step": 97, + "time_per_iteration": 2.9525508880615234 + }, + { + "auxiliary_loss_clip": 0.01581998, + "auxiliary_loss_mlp": 0.01154525, + "balance_loss_clip": 1.20096791, + "balance_loss_mlp": 1.08776784, + "epoch": 0.011783803282630914, + "flos": 19902125116800.0, + "grad_norm": 2.0591853006193355, + "language_loss": 0.86520737, + "learning_rate": 3.321561120780869e-06, + "loss": 0.89257264, + "num_input_tokens_seen": 2134835, + "router_z_loss_clip": 3.81054688, + "router_z_loss_mlp": 0.66796875, + "step": 98, + "time_per_iteration": 3.0181884765625 + }, + { + "auxiliary_loss_clip": 0.01573669, + "auxiliary_loss_mlp": 0.01142245, + "balance_loss_clip": 1.20295954, + "balance_loss_mlp": 1.07520127, + "epoch": 0.011904046173270006, + "flos": 22340674517760.0, + "grad_norm": 2.540706170255913, + "language_loss": 1.01509833, + "learning_rate": 3.3289159651708192e-06, + "loss": 1.04225755, + "num_input_tokens_seen": 2152410, + "router_z_loss_clip": 3.70507812, + "router_z_loss_mlp": 0.67041016, + "step": 99, + "time_per_iteration": 3.0317883491516113 + }, + { + "auxiliary_loss_clip": 0.01558804, + "auxiliary_loss_mlp": 0.01133188, + "balance_loss_clip": 1.19142354, + "balance_loss_mlp": 1.07043588, + "epoch": 0.012024289063909096, + "flos": 19100060375040.0, + "grad_norm": 1.9286959336897072, + "language_loss": 0.97327709, + "learning_rate": 3.3361968902759768e-06, + "loss": 1.00019693, + "num_input_tokens_seen": 2172090, + "router_z_loss_clip": 3.67773438, + "router_z_loss_mlp": 0.62646484, + "step": 100, + "time_per_iteration": 3.003039598464966 + }, + { + "auxiliary_loss_clip": 0.01554624, + "auxiliary_loss_mlp": 0.0114126, + "balance_loss_clip": 1.19224715, + "balance_loss_mlp": 1.07846081, + "epoch": 0.012144531954548187, + "flos": 15012205159680.0, + "grad_norm": 2.2809775647723853, + "language_loss": 0.93885159, + "learning_rate": 3.343405367163663e-06, + "loss": 0.96581042, + "num_input_tokens_seen": 2189020, + "router_z_loss_clip": 3.62304688, + "router_z_loss_mlp": 0.62792969, + "step": 101, + "time_per_iteration": 2.918335199356079 + }, + { + "auxiliary_loss_clip": 0.0156017, + "auxiliary_loss_mlp": 0.01145818, + "balance_loss_clip": 1.19278085, + "balance_loss_mlp": 1.0763433, + "epoch": 0.012264774845187279, + "flos": 15122020014720.0, + "grad_norm": 2.798655642929105, + "language_loss": 0.81142342, + "learning_rate": 3.350542823419951e-06, + "loss": 0.83848333, + "num_input_tokens_seen": 2205620, + "router_z_loss_clip": 3.67773438, + "router_z_loss_mlp": 0.69433594, + "step": 102, + "time_per_iteration": 2.9368460178375244 + }, + { + "auxiliary_loss_clip": 0.01568629, + "auxiliary_loss_mlp": 0.01155416, + "balance_loss_clip": 1.19910991, + "balance_loss_mlp": 1.09428596, + "epoch": 0.012385017735826368, + "flos": 13949248959360.0, + "grad_norm": 3.5819559524380225, + "language_loss": 0.87476891, + "learning_rate": 3.3576106448465615e-06, + "loss": 0.90200937, + "num_input_tokens_seen": 2219000, + "router_z_loss_clip": 3.69726562, + "router_z_loss_mlp": 0.61132812, + "step": 103, + "time_per_iteration": 2.9018137454986572 + }, + { + "auxiliary_loss_clip": 0.015573, + "auxiliary_loss_mlp": 0.01143632, + "balance_loss_clip": 1.18825948, + "balance_loss_mlp": 1.07692301, + "epoch": 0.01250526062646546, + "flos": 23623260428160.0, + "grad_norm": 1.9481145059124876, + "language_loss": 0.88242471, + "learning_rate": 3.3646101770757797e-06, + "loss": 0.90943402, + "num_input_tokens_seen": 2237790, + "router_z_loss_clip": 3.68945312, + "router_z_loss_mlp": 0.66748047, + "step": 104, + "time_per_iteration": 4.707855463027954 + }, + { + "auxiliary_loss_clip": 0.01543407, + "auxiliary_loss_mlp": 0.01156674, + "balance_loss_clip": 1.18489587, + "balance_loss_mlp": 1.08705533, + "epoch": 0.012625503517104552, + "flos": 34640078958720.0, + "grad_norm": 1.7234756550988553, + "language_loss": 0.85515547, + "learning_rate": 3.371542727108104e-06, + "loss": 0.88215631, + "num_input_tokens_seen": 2259965, + "router_z_loss_clip": 3.59179688, + "router_z_loss_mlp": 0.69580078, + "step": 105, + "time_per_iteration": 3.044633388519287 + }, + { + "auxiliary_loss_clip": 0.01559234, + "auxiliary_loss_mlp": 0.01164098, + "balance_loss_clip": 1.18938994, + "balance_loss_mlp": 1.09824646, + "epoch": 0.012745746407743641, + "flos": 17821891837440.0, + "grad_norm": 4.311698984485291, + "language_loss": 0.90019572, + "learning_rate": 3.3784095647770114e-06, + "loss": 0.92742908, + "num_input_tokens_seen": 2278610, + "router_z_loss_clip": 3.69921875, + "router_z_loss_mlp": 0.65869141, + "step": 106, + "time_per_iteration": 3.012056589126587 + }, + { + "auxiliary_loss_clip": 0.01546633, + "auxiliary_loss_mlp": 0.0115716, + "balance_loss_clip": 1.18528056, + "balance_loss_mlp": 1.09068942, + "epoch": 0.012865989298382733, + "flos": 20595057361920.0, + "grad_norm": 2.038092356019365, + "language_loss": 0.88718891, + "learning_rate": 3.3852119241449547e-06, + "loss": 0.91422683, + "num_input_tokens_seen": 2297730, + "router_z_loss_clip": 3.61523438, + "router_z_loss_mlp": 0.66455078, + "step": 107, + "time_per_iteration": 2.936929225921631 + }, + { + "auxiliary_loss_clip": 0.01552529, + "auxiliary_loss_mlp": 0.0116734, + "balance_loss_clip": 1.18580604, + "balance_loss_mlp": 1.10334897, + "epoch": 0.012986232189021825, + "flos": 23948969978880.0, + "grad_norm": 2.325271620559335, + "language_loss": 0.96145326, + "learning_rate": 3.3919510048344295e-06, + "loss": 0.98865199, + "num_input_tokens_seen": 2315740, + "router_z_loss_clip": 3.66601562, + "router_z_loss_mlp": 0.64013672, + "step": 108, + "time_per_iteration": 2.9943044185638428 + }, + { + "auxiliary_loss_clip": 0.01538209, + "auxiliary_loss_mlp": 0.01162622, + "balance_loss_clip": 1.18146634, + "balance_loss_mlp": 1.09791529, + "epoch": 0.013106475079660914, + "flos": 23725425686400.0, + "grad_norm": 2.3679984279166524, + "language_loss": 0.86779338, + "learning_rate": 3.3986279732976907e-06, + "loss": 0.89480168, + "num_input_tokens_seen": 2334215, + "router_z_loss_clip": 3.56640625, + "router_z_loss_mlp": 0.64697266, + "step": 109, + "time_per_iteration": 3.030904531478882 + }, + { + "auxiliary_loss_clip": 0.01542634, + "auxiliary_loss_mlp": 0.01140884, + "balance_loss_clip": 1.18138468, + "balance_loss_mlp": 1.07484245, + "epoch": 0.013226717970300006, + "flos": 21102438925440.0, + "grad_norm": 1.9680180713955717, + "language_loss": 0.95311749, + "learning_rate": 3.4052439640284983e-06, + "loss": 0.97995257, + "num_input_tokens_seen": 2353130, + "router_z_loss_clip": 3.61132812, + "router_z_loss_mlp": 0.66064453, + "step": 110, + "time_per_iteration": 2.9525468349456787 + }, + { + "auxiliary_loss_clip": 0.01533454, + "auxiliary_loss_mlp": 0.011451, + "balance_loss_clip": 1.18495631, + "balance_loss_mlp": 1.08191895, + "epoch": 0.013346960860939098, + "flos": 24863902231680.0, + "grad_norm": 1.7596872449397392, + "language_loss": 0.81292117, + "learning_rate": 3.4118000807190217e-06, + "loss": 0.83970672, + "num_input_tokens_seen": 2374010, + "router_z_loss_clip": 3.48828125, + "router_z_loss_mlp": 0.63134766, + "step": 111, + "time_per_iteration": 2.9435031414031982 + }, + { + "auxiliary_loss_clip": 0.01536724, + "auxiliary_loss_mlp": 0.01139059, + "balance_loss_clip": 1.17855215, + "balance_loss_mlp": 1.07478094, + "epoch": 0.013467203751578187, + "flos": 28181940140160.0, + "grad_norm": 2.7717688147409296, + "language_loss": 0.75834918, + "learning_rate": 3.4182973973648723e-06, + "loss": 0.78510702, + "num_input_tokens_seen": 2395220, + "router_z_loss_clip": 3.578125, + "router_z_loss_mlp": 0.64257812, + "step": 112, + "time_per_iteration": 2.9137139320373535 + }, + { + "auxiliary_loss_clip": 0.01538517, + "auxiliary_loss_mlp": 0.0116724, + "balance_loss_clip": 1.1813587, + "balance_loss_mlp": 1.10458326, + "epoch": 0.013587446642217279, + "flos": 18916233546240.0, + "grad_norm": 3.258007201885123, + "language_loss": 0.94930017, + "learning_rate": 3.424736959321014e-06, + "loss": 0.9763577, + "num_input_tokens_seen": 2413025, + "router_z_loss_clip": 3.57421875, + "router_z_loss_mlp": 0.62646484, + "step": 113, + "time_per_iteration": 2.9096920490264893 + }, + { + "auxiliary_loss_clip": 0.01534053, + "auxiliary_loss_mlp": 0.01170291, + "balance_loss_clip": 1.17904234, + "balance_loss_mlp": 1.10157883, + "epoch": 0.01370768953285637, + "flos": 23988615615360.0, + "grad_norm": 3.80416764307014, + "language_loss": 0.88556546, + "learning_rate": 3.431119784311155e-06, + "loss": 0.91260892, + "num_input_tokens_seen": 2432700, + "router_z_loss_clip": 3.55273438, + "router_z_loss_mlp": 0.68652344, + "step": 114, + "time_per_iteration": 2.9052140712738037 + }, + { + "auxiliary_loss_clip": 0.01521521, + "auxiliary_loss_mlp": 0.01131761, + "balance_loss_clip": 1.17990291, + "balance_loss_mlp": 1.06929493, + "epoch": 0.01382793242349546, + "flos": 39202565512320.0, + "grad_norm": 1.6848975223833487, + "language_loss": 0.77566415, + "learning_rate": 3.43744686339307e-06, + "loss": 0.80219698, + "num_input_tokens_seen": 2455020, + "router_z_loss_clip": 3.41601562, + "router_z_loss_mlp": 0.62451172, + "step": 115, + "time_per_iteration": 3.064577579498291 + }, + { + "auxiliary_loss_clip": 0.01516845, + "auxiliary_loss_mlp": 0.01115951, + "balance_loss_clip": 1.16534352, + "balance_loss_mlp": 1.05377126, + "epoch": 0.013948175314134552, + "flos": 41353506714240.0, + "grad_norm": 2.01593206502531, + "language_loss": 0.9098112, + "learning_rate": 3.44371916188212e-06, + "loss": 0.93613911, + "num_input_tokens_seen": 2475775, + "router_z_loss_clip": 3.51757812, + "router_z_loss_mlp": 0.62109375, + "step": 116, + "time_per_iteration": 3.1251604557037354 + }, + { + "auxiliary_loss_clip": 0.01510156, + "auxiliary_loss_mlp": 0.01123596, + "balance_loss_clip": 1.1660223, + "balance_loss_mlp": 1.06060588, + "epoch": 0.014068418204773643, + "flos": 22453542028800.0, + "grad_norm": 5.645158850903745, + "language_loss": 0.86084473, + "learning_rate": 3.449937620235143e-06, + "loss": 0.88718224, + "num_input_tokens_seen": 2496370, + "router_z_loss_clip": 3.44140625, + "router_z_loss_mlp": 0.62988281, + "step": 117, + "time_per_iteration": 2.9611551761627197 + }, + { + "auxiliary_loss_clip": 0.01512205, + "auxiliary_loss_mlp": 0.01143349, + "balance_loss_clip": 1.16711187, + "balance_loss_mlp": 1.07740283, + "epoch": 0.014188661095412733, + "flos": 23805147922560.0, + "grad_norm": 1.8883144708790771, + "language_loss": 0.89281958, + "learning_rate": 3.456103154896722e-06, + "loss": 0.91937512, + "num_input_tokens_seen": 2517645, + "router_z_loss_clip": 3.44921875, + "router_z_loss_mlp": 0.66015625, + "step": 118, + "time_per_iteration": 2.9543378353118896 + }, + { + "auxiliary_loss_clip": 0.01510878, + "auxiliary_loss_mlp": 0.01135425, + "balance_loss_clip": 1.16649365, + "balance_loss_mlp": 1.07224441, + "epoch": 0.014308903986051825, + "flos": 23660248458240.0, + "grad_norm": 1.7963279742245584, + "language_loss": 0.92198026, + "learning_rate": 3.462216659109757e-06, + "loss": 0.94844329, + "num_input_tokens_seen": 2537825, + "router_z_loss_clip": 3.4453125, + "router_z_loss_mlp": 0.63134766, + "step": 119, + "time_per_iteration": 2.99044132232666 + }, + { + "auxiliary_loss_clip": 0.01523545, + "auxiliary_loss_mlp": 0.01138691, + "balance_loss_clip": 1.17024446, + "balance_loss_mlp": 1.07202888, + "epoch": 0.014429146876690916, + "flos": 20667991927680.0, + "grad_norm": 2.9952279963276816, + "language_loss": 0.85227144, + "learning_rate": 3.4682790036921077e-06, + "loss": 0.87889373, + "num_input_tokens_seen": 2556485, + "router_z_loss_clip": 3.53125, + "router_z_loss_mlp": 0.66650391, + "step": 120, + "time_per_iteration": 2.953275203704834 + }, + { + "auxiliary_loss_clip": 0.01494903, + "auxiliary_loss_mlp": 0.01129083, + "balance_loss_clip": 1.16258776, + "balance_loss_mlp": 1.06528234, + "epoch": 0.014549389767330006, + "flos": 20229199384320.0, + "grad_norm": 1.8883134796062466, + "language_loss": 0.83077943, + "learning_rate": 3.4742910377810193e-06, + "loss": 0.85701931, + "num_input_tokens_seen": 2573945, + "router_z_loss_clip": 3.3203125, + "router_z_loss_mlp": 0.63720703, + "step": 121, + "time_per_iteration": 2.964303970336914 + }, + { + "auxiliary_loss_clip": 0.01502055, + "auxiliary_loss_mlp": 0.01138428, + "balance_loss_clip": 1.16268432, + "balance_loss_mlp": 1.07853758, + "epoch": 0.014669632657969098, + "flos": 18004174381440.0, + "grad_norm": 2.924451879636504, + "language_loss": 0.88650948, + "learning_rate": 3.4802535895469042e-06, + "loss": 0.91291434, + "num_input_tokens_seen": 2592695, + "router_z_loss_clip": 3.39257812, + "router_z_loss_mlp": 0.59790039, + "step": 122, + "time_per_iteration": 2.912899971008301 + }, + { + "auxiliary_loss_clip": 0.01505411, + "auxiliary_loss_mlp": 0.01131883, + "balance_loss_clip": 1.1638577, + "balance_loss_mlp": 1.0702275, + "epoch": 0.01478987554860819, + "flos": 22741796672640.0, + "grad_norm": 2.1967445301089557, + "language_loss": 0.8954187, + "learning_rate": 3.4861674668779934e-06, + "loss": 0.92179161, + "num_input_tokens_seen": 2610925, + "router_z_loss_clip": 3.4140625, + "router_z_loss_mlp": 0.6171875, + "step": 123, + "time_per_iteration": 2.9446659088134766 + }, + { + "auxiliary_loss_clip": 0.01494506, + "auxiliary_loss_mlp": 0.01136053, + "balance_loss_clip": 1.15819788, + "balance_loss_mlp": 1.07554197, + "epoch": 0.01491011843924728, + "flos": 17198590106880.0, + "grad_norm": 2.036640454574605, + "language_loss": 0.83901995, + "learning_rate": 3.492033458037272e-06, + "loss": 0.86532557, + "num_input_tokens_seen": 2629495, + "router_z_loss_clip": 3.36132812, + "router_z_loss_mlp": 0.60498047, + "step": 124, + "time_per_iteration": 2.8890652656555176 + }, + { + "auxiliary_loss_clip": 0.01502032, + "auxiliary_loss_mlp": 0.01146757, + "balance_loss_clip": 1.16119695, + "balance_loss_mlp": 1.08052421, + "epoch": 0.01503036132988637, + "flos": 17673867889920.0, + "grad_norm": 2.2912866300288566, + "language_loss": 0.86857152, + "learning_rate": 3.497852332293018e-06, + "loss": 0.89505947, + "num_input_tokens_seen": 2645070, + "router_z_loss_clip": 3.40625, + "router_z_loss_mlp": 0.66210938, + "step": 125, + "time_per_iteration": 2.949651002883911 + }, + { + "auxiliary_loss_clip": 0.01503147, + "auxiliary_loss_mlp": 0.01135508, + "balance_loss_clip": 1.16028893, + "balance_loss_mlp": 1.07056248, + "epoch": 0.015150604220525462, + "flos": 18878239935360.0, + "grad_norm": 2.0078979294866195, + "language_loss": 0.96447134, + "learning_rate": 3.5036248405242356e-06, + "loss": 0.99085784, + "num_input_tokens_seen": 2663825, + "router_z_loss_clip": 3.42773438, + "router_z_loss_mlp": 0.64941406, + "step": 126, + "time_per_iteration": 2.844184637069702 + }, + { + "auxiliary_loss_clip": 0.01493106, + "auxiliary_loss_mlp": 0.01151038, + "balance_loss_clip": 1.15630627, + "balance_loss_mlp": 1.08599699, + "epoch": 0.015270847111164552, + "flos": 39420184060800.0, + "grad_norm": 2.089971102255192, + "language_loss": 0.82708871, + "learning_rate": 3.509351715802146e-06, + "loss": 0.85353017, + "num_input_tokens_seen": 2684710, + "router_z_loss_clip": 3.36914062, + "router_z_loss_mlp": 0.65039062, + "step": 127, + "time_per_iteration": 3.0489492416381836 + }, + { + "auxiliary_loss_clip": 0.01489692, + "auxiliary_loss_mlp": 0.01125042, + "balance_loss_clip": 1.15435696, + "balance_loss_mlp": 1.06071663, + "epoch": 0.015391090001803644, + "flos": 43762466286720.0, + "grad_norm": 2.07455325367136, + "language_loss": 0.78574616, + "learning_rate": 3.5150336739488763e-06, + "loss": 0.81189346, + "num_input_tokens_seen": 2706995, + "router_z_loss_clip": 3.35546875, + "router_z_loss_mlp": 0.64306641, + "step": 128, + "time_per_iteration": 3.0933775901794434 + }, + { + "auxiliary_loss_clip": 0.01471653, + "auxiliary_loss_mlp": 0.01119857, + "balance_loss_clip": 1.15157044, + "balance_loss_mlp": 1.05753481, + "epoch": 0.015511332892442733, + "flos": 18916341287040.0, + "grad_norm": 1.8857143133016254, + "language_loss": 0.83949822, + "learning_rate": 3.5206714140744143e-06, + "loss": 0.86541331, + "num_input_tokens_seen": 2727050, + "router_z_loss_clip": 3.203125, + "router_z_loss_mlp": 0.62353516, + "step": 129, + "time_per_iteration": 2.8794360160827637 + }, + { + "auxiliary_loss_clip": 0.01503537, + "auxiliary_loss_mlp": 0.0114311, + "balance_loss_clip": 1.16599512, + "balance_loss_mlp": 1.08064389, + "epoch": 0.015631575783081827, + "flos": 24535283679360.0, + "grad_norm": 2.680599572864758, + "language_loss": 0.87263668, + "learning_rate": 3.5262656190928208e-06, + "loss": 0.89910316, + "num_input_tokens_seen": 2745350, + "router_z_loss_clip": 3.37695312, + "router_z_loss_mlp": 0.62402344, + "step": 130, + "time_per_iteration": 2.9429612159729004 + }, + { + "auxiliary_loss_clip": 0.01482706, + "auxiliary_loss_mlp": 0.01270466, + "balance_loss_clip": 1.2795912, + "balance_loss_mlp": 1.23098373, + "epoch": 0.015751818673720917, + "flos": 62328536098560.0, + "grad_norm": 1.099388760559183, + "language_loss": 0.71524894, + "learning_rate": 3.5318169562186737e-06, + "loss": 0.74278069, + "num_input_tokens_seen": 2814195, + "router_z_loss_clip": 2.03125, + "router_z_loss_mlp": 0.39453125, + "step": 131, + "time_per_iteration": 3.4439022541046143 + }, + { + "auxiliary_loss_clip": 0.01487027, + "auxiliary_loss_mlp": 0.01132936, + "balance_loss_clip": 1.15873826, + "balance_loss_mlp": 1.07018387, + "epoch": 0.015872061564360006, + "flos": 23878549365120.0, + "grad_norm": 1.9548884588682844, + "language_loss": 0.81950045, + "learning_rate": 3.5373260774446292e-06, + "loss": 0.84570014, + "num_input_tokens_seen": 2834645, + "router_z_loss_clip": 3.27929688, + "router_z_loss_mlp": 0.62695312, + "step": 132, + "time_per_iteration": 2.9629015922546387 + }, + { + "auxiliary_loss_clip": 0.01472063, + "auxiliary_loss_mlp": 0.01133803, + "balance_loss_clip": 1.14592135, + "balance_loss_mlp": 1.07019305, + "epoch": 0.0159923044549991, + "flos": 23367899664000.0, + "grad_norm": 5.08235821139688, + "language_loss": 0.9020136, + "learning_rate": 3.542793620000961e-06, + "loss": 0.92807227, + "num_input_tokens_seen": 2854120, + "router_z_loss_clip": 3.25976562, + "router_z_loss_mlp": 0.63623047, + "step": 133, + "time_per_iteration": 4.6977691650390625 + }, + { + "auxiliary_loss_clip": 0.01471003, + "auxiliary_loss_mlp": 0.01134295, + "balance_loss_clip": 1.15027094, + "balance_loss_mlp": 1.07092381, + "epoch": 0.01611254734563819, + "flos": 17858305249920.0, + "grad_norm": 2.762914462814705, + "language_loss": 0.87072647, + "learning_rate": 3.5482202067978894e-06, + "loss": 0.89677954, + "num_input_tokens_seen": 2871330, + "router_z_loss_clip": 3.20898438, + "router_z_loss_mlp": 0.63330078, + "step": 134, + "time_per_iteration": 2.909289598464966 + }, + { + "auxiliary_loss_clip": 0.01477685, + "auxiliary_loss_mlp": 0.01143054, + "balance_loss_clip": 1.14771438, + "balance_loss_mlp": 1.07849002, + "epoch": 0.01623279023627728, + "flos": 20954774113920.0, + "grad_norm": 2.737169473438629, + "language_loss": 0.75995219, + "learning_rate": 3.553606446851471e-06, + "loss": 0.78615963, + "num_input_tokens_seen": 2888070, + "router_z_loss_clip": 3.29882812, + "router_z_loss_mlp": 0.64550781, + "step": 135, + "time_per_iteration": 2.8910279273986816 + }, + { + "auxiliary_loss_clip": 0.01462767, + "auxiliary_loss_mlp": 0.0115345, + "balance_loss_clip": 1.14291942, + "balance_loss_mlp": 1.09146166, + "epoch": 0.016353033126916373, + "flos": 15742412743680.0, + "grad_norm": 1.8548238671419919, + "language_loss": 0.8331933, + "learning_rate": 3.5589529356937613e-06, + "loss": 0.85935545, + "num_input_tokens_seen": 2906465, + "router_z_loss_clip": 3.203125, + "router_z_loss_mlp": 0.61962891, + "step": 136, + "time_per_iteration": 2.896120548248291 + }, + { + "auxiliary_loss_clip": 0.01472426, + "auxiliary_loss_mlp": 0.01145219, + "balance_loss_clip": 1.14622736, + "balance_loss_mlp": 1.08260989, + "epoch": 0.016473276017555463, + "flos": 18807280617600.0, + "grad_norm": 2.261916056546978, + "language_loss": 0.76959991, + "learning_rate": 3.5642602557679627e-06, + "loss": 0.79577637, + "num_input_tokens_seen": 2924915, + "router_z_loss_clip": 3.26171875, + "router_z_loss_mlp": 0.62548828, + "step": 137, + "time_per_iteration": 2.9285857677459717 + }, + { + "auxiliary_loss_clip": 0.01464632, + "auxiliary_loss_mlp": 0.01154361, + "balance_loss_clip": 1.15037978, + "balance_loss_mlp": 1.09280157, + "epoch": 0.016593518908194552, + "flos": 24352641999360.0, + "grad_norm": 2.447467669689162, + "language_loss": 0.84139758, + "learning_rate": 3.569528976809202e-06, + "loss": 0.86758757, + "num_input_tokens_seen": 2942130, + "router_z_loss_clip": 3.140625, + "router_z_loss_mlp": 0.61572266, + "step": 138, + "time_per_iteration": 2.891411066055298 + }, + { + "auxiliary_loss_clip": 0.01470048, + "auxiliary_loss_mlp": 0.01150097, + "balance_loss_clip": 1.14149535, + "balance_loss_mlp": 1.08920467, + "epoch": 0.016713761798833646, + "flos": 22346133384960.0, + "grad_norm": 1.92681766683308, + "language_loss": 0.8970468, + "learning_rate": 3.5747596562115522e-06, + "loss": 0.92324829, + "num_input_tokens_seen": 2962745, + "router_z_loss_clip": 3.2890625, + "router_z_loss_mlp": 0.60888672, + "step": 139, + "time_per_iteration": 2.962127923965454 + }, + { + "auxiliary_loss_clip": 0.01476241, + "auxiliary_loss_mlp": 0.01167517, + "balance_loss_clip": 1.14317346, + "balance_loss_mlp": 1.10218978, + "epoch": 0.016834004689472735, + "flos": 17821820010240.0, + "grad_norm": 2.1775151638621386, + "language_loss": 0.9096669, + "learning_rate": 3.5799528393819138e-06, + "loss": 0.93610448, + "num_input_tokens_seen": 2981825, + "router_z_loss_clip": 3.33203125, + "router_z_loss_mlp": 0.65332031, + "step": 140, + "time_per_iteration": 2.925212860107422 + }, + { + "auxiliary_loss_clip": 0.01458343, + "auxiliary_loss_mlp": 0.01137156, + "balance_loss_clip": 1.14324307, + "balance_loss_mlp": 1.07793307, + "epoch": 0.016954247580111825, + "flos": 20519501103360.0, + "grad_norm": 2.3465420170999343, + "language_loss": 0.87819672, + "learning_rate": 3.585109060081286e-06, + "loss": 0.90415174, + "num_input_tokens_seen": 3001625, + "router_z_loss_clip": 3.15039062, + "router_z_loss_mlp": 0.59228516, + "step": 141, + "time_per_iteration": 3.0069375038146973 + }, + { + "auxiliary_loss_clip": 0.01469021, + "auxiliary_loss_mlp": 0.01144822, + "balance_loss_clip": 1.14151239, + "balance_loss_mlp": 1.08187985, + "epoch": 0.017074490470750915, + "flos": 22088869200000.0, + "grad_norm": 2.379570779471642, + "language_loss": 0.78548217, + "learning_rate": 3.590228840753992e-06, + "loss": 0.81162059, + "num_input_tokens_seen": 3022055, + "router_z_loss_clip": 3.27734375, + "router_z_loss_mlp": 0.62939453, + "step": 142, + "time_per_iteration": 2.9405415058135986 + }, + { + "auxiliary_loss_clip": 0.01459357, + "auxiliary_loss_mlp": 0.01160598, + "balance_loss_clip": 1.14194024, + "balance_loss_mlp": 1.09589159, + "epoch": 0.01719473336139001, + "flos": 15997270717440.0, + "grad_norm": 2.264136948861057, + "language_loss": 0.8722105, + "learning_rate": 3.5953126928453423e-06, + "loss": 0.89841008, + "num_input_tokens_seen": 3039605, + "router_z_loss_clip": 3.171875, + "router_z_loss_mlp": 0.64648438, + "step": 143, + "time_per_iteration": 2.9257638454437256 + }, + { + "auxiliary_loss_clip": 0.01461956, + "auxiliary_loss_mlp": 0.0115157, + "balance_loss_clip": 1.14059949, + "balance_loss_mlp": 1.08776951, + "epoch": 0.017314976252029098, + "flos": 22492038430080.0, + "grad_norm": 4.656832575022329, + "language_loss": 0.80524969, + "learning_rate": 3.600361117108239e-06, + "loss": 0.8313849, + "num_input_tokens_seen": 3059405, + "router_z_loss_clip": 3.21484375, + "router_z_loss_mlp": 0.63769531, + "step": 144, + "time_per_iteration": 3.0421385765075684 + }, + { + "auxiliary_loss_clip": 0.01460738, + "auxiliary_loss_mlp": 0.01149376, + "balance_loss_clip": 1.14126885, + "balance_loss_mlp": 1.08958089, + "epoch": 0.017435219142668188, + "flos": 22018053536640.0, + "grad_norm": 2.1199316398925703, + "language_loss": 0.97132552, + "learning_rate": 3.6053746038991616e-06, + "loss": 0.99742663, + "num_input_tokens_seen": 3078490, + "router_z_loss_clip": 3.1953125, + "router_z_loss_mlp": 0.59765625, + "step": 145, + "time_per_iteration": 2.909471273422241 + }, + { + "auxiliary_loss_clip": 0.01420854, + "auxiliary_loss_mlp": 0.0126775, + "balance_loss_clip": 1.20374298, + "balance_loss_mlp": 1.23322713, + "epoch": 0.01755546203330728, + "flos": 72240526149120.0, + "grad_norm": 1.0875051378192087, + "language_loss": 0.5842613, + "learning_rate": 3.6103536334639843e-06, + "loss": 0.6111474, + "num_input_tokens_seen": 3131755, + "router_z_loss_clip": 2.171875, + "router_z_loss_mlp": 0.34570312, + "step": 146, + "time_per_iteration": 3.386589288711548 + }, + { + "auxiliary_loss_clip": 0.01449338, + "auxiliary_loss_mlp": 0.01145647, + "balance_loss_clip": 1.13997889, + "balance_loss_mlp": 1.08499348, + "epoch": 0.01767570492394637, + "flos": 25337061112320.0, + "grad_norm": 2.1914660352616018, + "language_loss": 0.85414845, + "learning_rate": 3.615298676214041e-06, + "loss": 0.88009828, + "num_input_tokens_seen": 3152035, + "router_z_loss_clip": 3.09570312, + "router_z_loss_mlp": 0.60595703, + "step": 147, + "time_per_iteration": 3.096822738647461 + }, + { + "auxiliary_loss_clip": 0.0145037, + "auxiliary_loss_mlp": 0.01132248, + "balance_loss_clip": 1.13666975, + "balance_loss_mlp": 1.07078409, + "epoch": 0.01779594781458546, + "flos": 20449188230400.0, + "grad_norm": 1.9288312812463717, + "language_loss": 0.88862538, + "learning_rate": 3.6202101929928317e-06, + "loss": 0.9144516, + "num_input_tokens_seen": 3170625, + "router_z_loss_clip": 3.140625, + "router_z_loss_mlp": 0.61474609, + "step": 148, + "time_per_iteration": 2.9374279975891113 + }, + { + "auxiliary_loss_clip": 0.01439383, + "auxiliary_loss_mlp": 0.01121699, + "balance_loss_clip": 1.13636827, + "balance_loss_mlp": 1.06466937, + "epoch": 0.017916190705224554, + "flos": 16253601148800.0, + "grad_norm": 2.0565779393672257, + "language_loss": 0.88408893, + "learning_rate": 3.6250886353337413e-06, + "loss": 0.90969974, + "num_input_tokens_seen": 3188155, + "router_z_loss_clip": 3.03125, + "router_z_loss_mlp": 0.57080078, + "step": 149, + "time_per_iteration": 2.936248302459717 + }, + { + "auxiliary_loss_clip": 0.01446617, + "auxiliary_loss_mlp": 0.01124469, + "balance_loss_clip": 1.13891447, + "balance_loss_mlp": 1.06352901, + "epoch": 0.018036433595863644, + "flos": 23330588411520.0, + "grad_norm": 1.8831534639342202, + "language_loss": 0.86444163, + "learning_rate": 3.6299344457091488e-06, + "loss": 0.89015257, + "num_input_tokens_seen": 3209015, + "router_z_loss_clip": 3.07421875, + "router_z_loss_mlp": 0.60986328, + "step": 150, + "time_per_iteration": 2.9168620109558105 + }, + { + "auxiliary_loss_clip": 0.01442604, + "auxiliary_loss_mlp": 0.01127176, + "balance_loss_clip": 1.13253903, + "balance_loss_mlp": 1.06866837, + "epoch": 0.018156676486502734, + "flos": 18588010043520.0, + "grad_norm": 2.23811209584669, + "language_loss": 0.93759078, + "learning_rate": 3.634748057771256e-06, + "loss": 0.96328866, + "num_input_tokens_seen": 3224955, + "router_z_loss_clip": 3.09960938, + "router_z_loss_mlp": 0.58496094, + "step": 151, + "time_per_iteration": 2.8655946254730225 + }, + { + "auxiliary_loss_clip": 0.01442634, + "auxiliary_loss_mlp": 0.01122427, + "balance_loss_clip": 1.13450158, + "balance_loss_mlp": 1.06291771, + "epoch": 0.018276919377141827, + "flos": 25448707560960.0, + "grad_norm": 1.8255962257596756, + "language_loss": 0.85637271, + "learning_rate": 3.639529896584965e-06, + "loss": 0.88202327, + "num_input_tokens_seen": 3246330, + "router_z_loss_clip": 3.08203125, + "router_z_loss_mlp": 0.59521484, + "step": 152, + "time_per_iteration": 3.053980588912964 + }, + { + "auxiliary_loss_clip": 0.01440701, + "auxiliary_loss_mlp": 0.01127901, + "balance_loss_clip": 1.13174868, + "balance_loss_mlp": 1.0693934, + "epoch": 0.018397162267780917, + "flos": 20047311889920.0, + "grad_norm": 2.660790806175467, + "language_loss": 0.88885307, + "learning_rate": 3.6442803788531233e-06, + "loss": 0.9145391, + "num_input_tokens_seen": 3264290, + "router_z_loss_clip": 3.08984375, + "router_z_loss_mlp": 0.58496094, + "step": 153, + "time_per_iteration": 2.902595281600952 + }, + { + "auxiliary_loss_clip": 0.01444889, + "auxiliary_loss_mlp": 0.01128796, + "balance_loss_clip": 1.13344121, + "balance_loss_mlp": 1.07028794, + "epoch": 0.018517405158420007, + "flos": 27565282425600.0, + "grad_norm": 2.157089883890426, + "language_loss": 0.95897108, + "learning_rate": 3.6489999131344357e-06, + "loss": 0.98470795, + "num_input_tokens_seen": 3287065, + "router_z_loss_clip": 3.109375, + "router_z_loss_mlp": 0.58447266, + "step": 154, + "time_per_iteration": 2.948387861251831 + }, + { + "auxiliary_loss_clip": 0.01442348, + "auxiliary_loss_mlp": 0.01121397, + "balance_loss_clip": 1.13939404, + "balance_loss_mlp": 1.05525923, + "epoch": 0.0186376480490591, + "flos": 19354056422400.0, + "grad_norm": 1.8546010455255704, + "language_loss": 0.90638649, + "learning_rate": 3.653688900054313e-06, + "loss": 0.93202388, + "num_input_tokens_seen": 3305595, + "router_z_loss_clip": 3.03125, + "router_z_loss_mlp": 0.66113281, + "step": 155, + "time_per_iteration": 2.957883834838867 + }, + { + "auxiliary_loss_clip": 0.01436771, + "auxiliary_loss_mlp": 0.01130008, + "balance_loss_clip": 1.12854123, + "balance_loss_mlp": 1.07207179, + "epoch": 0.01875789093969819, + "flos": 26687840993280.0, + "grad_norm": 2.1092004010038683, + "language_loss": 0.76040554, + "learning_rate": 3.6583477325089526e-06, + "loss": 0.78607339, + "num_input_tokens_seen": 3326135, + "router_z_loss_clip": 3.08789062, + "router_z_loss_mlp": 0.57861328, + "step": 156, + "time_per_iteration": 3.0574800968170166 + }, + { + "auxiliary_loss_clip": 0.01429094, + "auxiliary_loss_mlp": 0.01144772, + "balance_loss_clip": 1.1277945, + "balance_loss_mlp": 1.08645535, + "epoch": 0.01887813383033728, + "flos": 24353001135360.0, + "grad_norm": 2.5915377439193605, + "language_loss": 1.04324424, + "learning_rate": 3.6629767958628916e-06, + "loss": 1.06898284, + "num_input_tokens_seen": 3343510, + "router_z_loss_clip": 3.015625, + "router_z_loss_mlp": 0.58251953, + "step": 157, + "time_per_iteration": 2.945383310317993 + }, + { + "auxiliary_loss_clip": 0.01453487, + "auxiliary_loss_mlp": 0.01152509, + "balance_loss_clip": 1.13784361, + "balance_loss_mlp": 1.0950501, + "epoch": 0.018998376720976373, + "flos": 14647532330880.0, + "grad_norm": 2.53168289162869, + "language_loss": 0.85661495, + "learning_rate": 3.667576468140291e-06, + "loss": 0.88267493, + "num_input_tokens_seen": 3361325, + "router_z_loss_clip": 3.15625, + "router_z_loss_mlp": 0.57470703, + "step": 158, + "time_per_iteration": 2.9435536861419678 + }, + { + "auxiliary_loss_clip": 0.01427815, + "auxiliary_loss_mlp": 0.01120077, + "balance_loss_clip": 1.12468934, + "balance_loss_mlp": 1.06032979, + "epoch": 0.019118619611615463, + "flos": 29305261146240.0, + "grad_norm": 2.8786399268982947, + "language_loss": 0.88887173, + "learning_rate": 3.672147120210184e-06, + "loss": 0.91435069, + "num_input_tokens_seen": 3377925, + "router_z_loss_clip": 3.03125, + "router_z_loss_mlp": 0.59765625, + "step": 159, + "time_per_iteration": 2.943326950073242 + }, + { + "auxiliary_loss_clip": 0.01429136, + "auxiliary_loss_mlp": 0.01133608, + "balance_loss_clip": 1.13004255, + "balance_loss_mlp": 1.07591045, + "epoch": 0.019238862502254553, + "flos": 20886723797760.0, + "grad_norm": 2.233573455047195, + "language_loss": 0.86303782, + "learning_rate": 3.6766891159659177e-06, + "loss": 0.8886652, + "num_input_tokens_seen": 3396335, + "router_z_loss_clip": 2.99609375, + "router_z_loss_mlp": 0.57666016, + "step": 160, + "time_per_iteration": 2.9435837268829346 + }, + { + "auxiliary_loss_clip": 0.01424523, + "auxiliary_loss_mlp": 0.01122531, + "balance_loss_clip": 1.12526321, + "balance_loss_mlp": 1.06387997, + "epoch": 0.019359105392893646, + "flos": 21360672777600.0, + "grad_norm": 3.926086330613333, + "language_loss": 0.87646759, + "learning_rate": 3.6812028124990075e-06, + "loss": 0.90193808, + "num_input_tokens_seen": 3413605, + "router_z_loss_clip": 2.99414062, + "router_z_loss_mlp": 0.5859375, + "step": 161, + "time_per_iteration": 2.8979170322418213 + }, + { + "auxiliary_loss_clip": 0.01430306, + "auxiliary_loss_mlp": 0.01129185, + "balance_loss_clip": 1.12942898, + "balance_loss_mlp": 1.07058215, + "epoch": 0.019479348283532736, + "flos": 16283729681280.0, + "grad_norm": 5.41651513109003, + "language_loss": 0.81642193, + "learning_rate": 3.6856885602676016e-06, + "loss": 0.84201688, + "num_input_tokens_seen": 3429640, + "router_z_loss_clip": 3.00976562, + "router_z_loss_mlp": 0.58691406, + "step": 162, + "time_per_iteration": 4.6559765338897705 + }, + { + "auxiliary_loss_clip": 0.01430695, + "auxiliary_loss_mlp": 0.0111948, + "balance_loss_clip": 1.12529826, + "balance_loss_mlp": 1.06097162, + "epoch": 0.019599591174171826, + "flos": 22091239497600.0, + "grad_norm": 2.5375968047921362, + "language_loss": 0.93969083, + "learning_rate": 3.6901467032597733e-06, + "loss": 0.96519256, + "num_input_tokens_seen": 3448125, + "router_z_loss_clip": 3.0546875, + "router_z_loss_mlp": 0.58447266, + "step": 163, + "time_per_iteration": 2.9305577278137207 + }, + { + "auxiliary_loss_clip": 0.01429915, + "auxiliary_loss_mlp": 0.01132826, + "balance_loss_clip": 1.12637615, + "balance_loss_mlp": 1.07479453, + "epoch": 0.01971983406481092, + "flos": 19609668581760.0, + "grad_norm": 2.849290732742838, + "language_loss": 0.87405109, + "learning_rate": 3.694577579151804e-06, + "loss": 0.89967847, + "num_input_tokens_seen": 3466535, + "router_z_loss_clip": 3.03320312, + "router_z_loss_mlp": 0.58056641, + "step": 164, + "time_per_iteration": 2.900273084640503 + }, + { + "auxiliary_loss_clip": 0.01425817, + "auxiliary_loss_mlp": 0.01126931, + "balance_loss_clip": 1.12771344, + "balance_loss_mlp": 1.06503725, + "epoch": 0.01984007695545001, + "flos": 19099342103040.0, + "grad_norm": 2.533456500430269, + "language_loss": 0.73533332, + "learning_rate": 3.6989815194616703e-06, + "loss": 0.7608608, + "num_input_tokens_seen": 3483730, + "router_z_loss_clip": 2.984375, + "router_z_loss_mlp": 0.61816406, + "step": 165, + "time_per_iteration": 2.8389322757720947 + }, + { + "auxiliary_loss_clip": 0.01430945, + "auxiliary_loss_mlp": 0.01135915, + "balance_loss_clip": 1.11979735, + "balance_loss_mlp": 1.07378328, + "epoch": 0.0199603198460891, + "flos": 20848406964480.0, + "grad_norm": 2.3607271192602517, + "language_loss": 0.79960406, + "learning_rate": 3.703358849697888e-06, + "loss": 0.82527268, + "num_input_tokens_seen": 3503640, + "router_z_loss_clip": 3.11328125, + "router_z_loss_mlp": 0.62084961, + "step": 166, + "time_per_iteration": 2.8887999057769775 + }, + { + "auxiliary_loss_clip": 0.01423553, + "auxiliary_loss_mlp": 0.01130291, + "balance_loss_clip": 1.12409854, + "balance_loss_mlp": 1.06629992, + "epoch": 0.020080562736728192, + "flos": 21870747861120.0, + "grad_norm": 1.936141896169769, + "language_loss": 0.82798398, + "learning_rate": 3.7077098895038803e-06, + "loss": 0.85352248, + "num_input_tokens_seen": 3523010, + "router_z_loss_clip": 2.99414062, + "router_z_loss_mlp": 0.63964844, + "step": 167, + "time_per_iteration": 2.894052028656006 + }, + { + "auxiliary_loss_clip": 0.01418286, + "auxiliary_loss_mlp": 0.0112215, + "balance_loss_clip": 1.12263656, + "balance_loss_mlp": 1.06614602, + "epoch": 0.020200805627367282, + "flos": 21688788539520.0, + "grad_norm": 2.229082677432085, + "language_loss": 0.97026205, + "learning_rate": 3.712034952798045e-06, + "loss": 0.99566638, + "num_input_tokens_seen": 3541125, + "router_z_loss_clip": 2.95703125, + "router_z_loss_mlp": 0.55981445, + "step": 168, + "time_per_iteration": 2.9857068061828613 + }, + { + "auxiliary_loss_clip": 0.01422036, + "auxiliary_loss_mlp": 0.01109429, + "balance_loss_clip": 1.11983538, + "balance_loss_mlp": 1.05378246, + "epoch": 0.02032104851800637, + "flos": 33543043729920.0, + "grad_norm": 2.8508644903442106, + "language_loss": 0.84745342, + "learning_rate": 3.7163343479096656e-06, + "loss": 0.87276804, + "num_input_tokens_seen": 3562700, + "router_z_loss_clip": 3.01953125, + "router_z_loss_mlp": 0.55712891, + "step": 169, + "time_per_iteration": 3.0172085762023926 + }, + { + "auxiliary_loss_clip": 0.0141041, + "auxiliary_loss_mlp": 0.01123235, + "balance_loss_clip": 1.12180829, + "balance_loss_mlp": 1.06863737, + "epoch": 0.020441291408645465, + "flos": 31686965274240.0, + "grad_norm": 2.1795941396815177, + "language_loss": 0.82911229, + "learning_rate": 3.720608377710802e-06, + "loss": 0.8544488, + "num_input_tokens_seen": 3582790, + "router_z_loss_clip": 2.88476562, + "router_z_loss_mlp": 0.54614258, + "step": 170, + "time_per_iteration": 2.9845495223999023 + }, + { + "auxiliary_loss_clip": 0.01421081, + "auxiliary_loss_mlp": 0.01105679, + "balance_loss_clip": 1.11977053, + "balance_loss_mlp": 1.04860175, + "epoch": 0.020561534299284555, + "flos": 20886687884160.0, + "grad_norm": 3.024340411882313, + "language_loss": 0.86597127, + "learning_rate": 3.7248573397443277e-06, + "loss": 0.89123893, + "num_input_tokens_seen": 3601715, + "router_z_loss_clip": 3.01171875, + "router_z_loss_mlp": 0.57128906, + "step": 171, + "time_per_iteration": 2.9248995780944824 + }, + { + "auxiliary_loss_clip": 0.01405423, + "auxiliary_loss_mlp": 0.01115792, + "balance_loss_clip": 1.12079167, + "balance_loss_mlp": 1.0596211, + "epoch": 0.020681777189923645, + "flos": 20996610480000.0, + "grad_norm": 1.9363333295740324, + "language_loss": 0.97195029, + "learning_rate": 3.729081526348224e-06, + "loss": 0.99716246, + "num_input_tokens_seen": 3620245, + "router_z_loss_clip": 2.84765625, + "router_z_loss_mlp": 0.56201172, + "step": 172, + "time_per_iteration": 2.8436806201934814 + }, + { + "auxiliary_loss_clip": 0.01415837, + "auxiliary_loss_mlp": 0.01116563, + "balance_loss_clip": 1.12359381, + "balance_loss_mlp": 1.06156027, + "epoch": 0.020802020080562738, + "flos": 28257532312320.0, + "grad_norm": 3.0418605138861854, + "language_loss": 0.84732026, + "learning_rate": 3.7332812247762777e-06, + "loss": 0.87264425, + "num_input_tokens_seen": 3641545, + "router_z_loss_clip": 2.92382812, + "router_z_loss_mlp": 0.55053711, + "step": 173, + "time_per_iteration": 2.905719518661499 + }, + { + "auxiliary_loss_clip": 0.01422528, + "auxiliary_loss_mlp": 0.01132137, + "balance_loss_clip": 1.12497771, + "balance_loss_mlp": 1.07262778, + "epoch": 0.020922262971201828, + "flos": 19681274344320.0, + "grad_norm": 2.740331726607427, + "language_loss": 0.95450538, + "learning_rate": 3.737456717315293e-06, + "loss": 0.98005199, + "num_input_tokens_seen": 3660510, + "router_z_loss_clip": 2.9765625, + "router_z_loss_mlp": 0.59423828, + "step": 174, + "time_per_iteration": 2.8965060710906982 + }, + { + "auxiliary_loss_clip": 0.01412549, + "auxiliary_loss_mlp": 0.01131065, + "balance_loss_clip": 1.12453592, + "balance_loss_mlp": 1.07570434, + "epoch": 0.021042505861840918, + "flos": 15666353694720.0, + "grad_norm": 2.6153846744518234, + "language_loss": 0.9052583, + "learning_rate": 3.7416082813989552e-06, + "loss": 0.93069446, + "num_input_tokens_seen": 3677505, + "router_z_loss_clip": 2.88085938, + "router_z_loss_mlp": 0.55419922, + "step": 175, + "time_per_iteration": 2.8755741119384766 + }, + { + "auxiliary_loss_clip": 0.01415496, + "auxiliary_loss_mlp": 0.01119765, + "balance_loss_clip": 1.11853981, + "balance_loss_mlp": 1.06521451, + "epoch": 0.02116274875248001, + "flos": 21142012734720.0, + "grad_norm": 2.425563412453339, + "language_loss": 0.89585161, + "learning_rate": 3.745736189718439e-06, + "loss": 0.92120427, + "num_input_tokens_seen": 3696760, + "router_z_loss_clip": 2.96875, + "router_z_loss_mlp": 0.5456543, + "step": 176, + "time_per_iteration": 2.9009530544281006 + }, + { + "auxiliary_loss_clip": 0.01408696, + "auxiliary_loss_mlp": 0.01116854, + "balance_loss_clip": 1.12082469, + "balance_loss_mlp": 1.0625664, + "epoch": 0.0212829916431191, + "flos": 24715770543360.0, + "grad_norm": 3.4483919003042285, + "language_loss": 0.72634935, + "learning_rate": 3.749840710329894e-06, + "loss": 0.75160491, + "num_input_tokens_seen": 3717465, + "router_z_loss_clip": 2.87695312, + "router_z_loss_mlp": 0.54248047, + "step": 177, + "time_per_iteration": 2.9000957012176514 + }, + { + "auxiliary_loss_clip": 0.01416207, + "auxiliary_loss_mlp": 0.01125114, + "balance_loss_clip": 1.12147057, + "balance_loss_mlp": 1.07156503, + "epoch": 0.02140323453375819, + "flos": 16645493508480.0, + "grad_norm": 2.7644274601957, + "language_loss": 0.98210347, + "learning_rate": 3.7539221067588938e-06, + "loss": 1.00751662, + "num_input_tokens_seen": 3731440, + "router_z_loss_clip": 2.9453125, + "router_z_loss_mlp": 0.53515625, + "step": 178, + "time_per_iteration": 2.9815216064453125 + }, + { + "auxiliary_loss_clip": 0.01410403, + "auxiliary_loss_mlp": 0.01107038, + "balance_loss_clip": 1.11641395, + "balance_loss_mlp": 1.04762399, + "epoch": 0.021523477424397284, + "flos": 20299332689280.0, + "grad_norm": 3.489560202396641, + "language_loss": 0.93685454, + "learning_rate": 3.757980638101964e-06, + "loss": 0.96202892, + "num_input_tokens_seen": 3744935, + "router_z_loss_clip": 2.9375, + "router_z_loss_mlp": 0.59423828, + "step": 179, + "time_per_iteration": 2.9206037521362305 + }, + { + "auxiliary_loss_clip": 0.01416303, + "auxiliary_loss_mlp": 0.011167, + "balance_loss_clip": 1.12237096, + "balance_loss_mlp": 1.06200635, + "epoch": 0.021643720315036374, + "flos": 26104005331200.0, + "grad_norm": 2.8036929695262045, + "language_loss": 0.8957262, + "learning_rate": 3.7620165591252806e-06, + "loss": 0.92105627, + "num_input_tokens_seen": 3763035, + "router_z_loss_clip": 2.94140625, + "router_z_loss_mlp": 0.546875, + "step": 180, + "time_per_iteration": 2.991920232772827 + }, + { + "auxiliary_loss_clip": 0.01399173, + "auxiliary_loss_mlp": 0.01109748, + "balance_loss_clip": 1.11790061, + "balance_loss_mlp": 1.05670023, + "epoch": 0.021763963205675464, + "flos": 24787663614720.0, + "grad_norm": 1.8322125850394697, + "language_loss": 0.94391561, + "learning_rate": 3.766030120360636e-06, + "loss": 0.96900481, + "num_input_tokens_seen": 3782665, + "router_z_loss_clip": 2.81054688, + "router_z_loss_mlp": 0.53076172, + "step": 181, + "time_per_iteration": 2.9808084964752197 + }, + { + "auxiliary_loss_clip": 0.01396988, + "auxiliary_loss_mlp": 0.01110928, + "balance_loss_clip": 1.11437559, + "balance_loss_mlp": 1.05528069, + "epoch": 0.021884206096314557, + "flos": 25813559957760.0, + "grad_norm": 3.0723271859504986, + "language_loss": 0.90163004, + "learning_rate": 3.7700215681987578e-06, + "loss": 0.92670918, + "num_input_tokens_seen": 3802435, + "router_z_loss_clip": 2.828125, + "router_z_loss_mlp": 0.55664062, + "step": 182, + "time_per_iteration": 2.8875668048858643 + }, + { + "auxiliary_loss_clip": 0.01398606, + "auxiliary_loss_mlp": 0.01122968, + "balance_loss_clip": 1.11392772, + "balance_loss_mlp": 1.06693935, + "epoch": 0.022004448986953647, + "flos": 20082719721600.0, + "grad_norm": 2.4733810990707585, + "language_loss": 0.82465529, + "learning_rate": 3.7739911449800767e-06, + "loss": 0.84987104, + "num_input_tokens_seen": 3822490, + "router_z_loss_clip": 2.8515625, + "router_z_loss_mlp": 0.56054688, + "step": 183, + "time_per_iteration": 2.9232099056243896 + }, + { + "auxiliary_loss_clip": 0.01402371, + "auxiliary_loss_mlp": 0.01121021, + "balance_loss_clip": 1.11481571, + "balance_loss_mlp": 1.06537366, + "epoch": 0.022124691877592736, + "flos": 20480609652480.0, + "grad_norm": 2.557444732313563, + "language_loss": 0.8074398, + "learning_rate": 3.7779390890830114e-06, + "loss": 0.83267367, + "num_input_tokens_seen": 3841140, + "router_z_loss_clip": 2.87304688, + "router_z_loss_mlp": 0.55664062, + "step": 184, + "time_per_iteration": 2.858337640762329 + }, + { + "auxiliary_loss_clip": 0.01395192, + "auxiliary_loss_mlp": 0.01122896, + "balance_loss_clip": 1.11132765, + "balance_loss_mlp": 1.06734431, + "epoch": 0.02224493476823183, + "flos": 23586847015680.0, + "grad_norm": 2.1439968655446378, + "language_loss": 0.85954428, + "learning_rate": 3.7818656350098723e-06, + "loss": 0.88472509, + "num_input_tokens_seen": 3862090, + "router_z_loss_clip": 2.83789062, + "router_z_loss_mlp": 0.55615234, + "step": 185, + "time_per_iteration": 2.911672592163086 + }, + { + "auxiliary_loss_clip": 0.01404778, + "auxiliary_loss_mlp": 0.0112067, + "balance_loss_clip": 1.11439466, + "balance_loss_mlp": 1.06588173, + "epoch": 0.02236517765887092, + "flos": 16909940413440.0, + "grad_norm": 3.51817585158861, + "language_loss": 0.77469873, + "learning_rate": 3.7857710134704447e-06, + "loss": 0.79995322, + "num_input_tokens_seen": 3881025, + "router_z_loss_clip": 2.90234375, + "router_z_loss_mlp": 0.54785156, + "step": 186, + "time_per_iteration": 2.883776903152466 + }, + { + "auxiliary_loss_clip": 0.01383276, + "auxiliary_loss_mlp": 0.01122356, + "balance_loss_clip": 1.11160636, + "balance_loss_mlp": 1.06802058, + "epoch": 0.02248542054951001, + "flos": 43508182930560.0, + "grad_norm": 1.9867871442555676, + "language_loss": 0.78980255, + "learning_rate": 3.7896554514633234e-06, + "loss": 0.81485879, + "num_input_tokens_seen": 3905310, + "router_z_loss_clip": 2.71875, + "router_z_loss_mlp": 0.54321289, + "step": 187, + "time_per_iteration": 2.998124837875366 + }, + { + "auxiliary_loss_clip": 0.01388498, + "auxiliary_loss_mlp": 0.01112108, + "balance_loss_clip": 1.11077619, + "balance_loss_mlp": 1.06060934, + "epoch": 0.022605663440149103, + "flos": 23367648268800.0, + "grad_norm": 2.0879716734673828, + "language_loss": 0.83987778, + "learning_rate": 3.7935191723550955e-06, + "loss": 0.8648839, + "num_input_tokens_seen": 3924265, + "router_z_loss_clip": 2.7734375, + "router_z_loss_mlp": 0.51489258, + "step": 188, + "time_per_iteration": 2.9051599502563477 + }, + { + "auxiliary_loss_clip": 0.01395381, + "auxiliary_loss_mlp": 0.0111287, + "balance_loss_clip": 1.11728311, + "balance_loss_mlp": 1.05805767, + "epoch": 0.022725906330788193, + "flos": 29019915504000.0, + "grad_norm": 2.015857265065446, + "language_loss": 0.88551426, + "learning_rate": 3.797362395957408e-06, + "loss": 0.91059673, + "num_input_tokens_seen": 3944830, + "router_z_loss_clip": 2.78515625, + "router_z_loss_mlp": 0.54760742, + "step": 189, + "time_per_iteration": 2.878861904144287 + }, + { + "auxiliary_loss_clip": 0.01396412, + "auxiliary_loss_mlp": 0.01108764, + "balance_loss_clip": 1.11353278, + "balance_loss_mlp": 1.05638385, + "epoch": 0.022846149221427282, + "flos": 24496176746880.0, + "grad_norm": 2.1794831107158577, + "language_loss": 0.78366238, + "learning_rate": 3.8011853386020055e-06, + "loss": 0.80871415, + "num_input_tokens_seen": 3965735, + "router_z_loss_clip": 2.83007812, + "router_z_loss_mlp": 0.52416992, + "step": 190, + "time_per_iteration": 2.9309449195861816 + }, + { + "auxiliary_loss_clip": 0.01392278, + "auxiliary_loss_mlp": 0.01107672, + "balance_loss_clip": 1.10953426, + "balance_loss_mlp": 1.05860519, + "epoch": 0.022966392112066376, + "flos": 15523537219200.0, + "grad_norm": 3.2261099892274694, + "language_loss": 0.89820963, + "learning_rate": 3.804988213213804e-06, + "loss": 0.92320913, + "num_input_tokens_seen": 3983975, + "router_z_loss_clip": 2.828125, + "router_z_loss_mlp": 0.49047852, + "step": 191, + "time_per_iteration": 4.802253246307373 + }, + { + "auxiliary_loss_clip": 0.01328325, + "auxiliary_loss_mlp": 0.01227055, + "balance_loss_clip": 1.16128397, + "balance_loss_mlp": 1.19920778, + "epoch": 0.023086635002705466, + "flos": 55650408433920.0, + "grad_norm": 1.0496261564244653, + "language_loss": 0.63227791, + "learning_rate": 3.808771229382049e-06, + "loss": 0.65783173, + "num_input_tokens_seen": 4043440, + "router_z_loss_clip": 1.671875, + "router_z_loss_mlp": 0.27929688, + "step": 192, + "time_per_iteration": 3.3340694904327393 + }, + { + "auxiliary_loss_clip": 0.01387686, + "auxiliary_loss_mlp": 0.01106144, + "balance_loss_clip": 1.11135554, + "balance_loss_mlp": 1.05188012, + "epoch": 0.023206877893344555, + "flos": 19313441118720.0, + "grad_norm": 2.165163635510412, + "language_loss": 0.84384882, + "learning_rate": 3.8125345934296324e-06, + "loss": 0.86878711, + "num_input_tokens_seen": 4061750, + "router_z_loss_clip": 2.76171875, + "router_z_loss_mlp": 0.54296875, + "step": 193, + "time_per_iteration": 2.8750720024108887 + }, + { + "auxiliary_loss_clip": 0.01392247, + "auxiliary_loss_mlp": 0.01117033, + "balance_loss_clip": 1.10807037, + "balance_loss_mlp": 1.06253016, + "epoch": 0.02332712078398365, + "flos": 23072965090560.0, + "grad_norm": 2.1503725633887414, + "language_loss": 0.87829077, + "learning_rate": 3.81627850848061e-06, + "loss": 0.90338361, + "num_input_tokens_seen": 4082345, + "router_z_loss_clip": 2.84179688, + "router_z_loss_mlp": 0.54492188, + "step": 194, + "time_per_iteration": 2.9170925617218018 + }, + { + "auxiliary_loss_clip": 0.01387076, + "auxiliary_loss_mlp": 0.01105371, + "balance_loss_clip": 1.11001468, + "balance_loss_mlp": 1.05318093, + "epoch": 0.02344736367462274, + "flos": 24425971614720.0, + "grad_norm": 2.239240828481543, + "language_loss": 0.86087298, + "learning_rate": 3.820003174525994e-06, + "loss": 0.8857975, + "num_input_tokens_seen": 4101770, + "router_z_loss_clip": 2.76953125, + "router_z_loss_mlp": 0.52172852, + "step": 195, + "time_per_iteration": 3.054203987121582 + }, + { + "auxiliary_loss_clip": 0.01392199, + "auxiliary_loss_mlp": 0.01108058, + "balance_loss_clip": 1.11494267, + "balance_loss_mlp": 1.05360341, + "epoch": 0.02356760656526183, + "flos": 21579799697280.0, + "grad_norm": 3.144653794691587, + "language_loss": 0.82852817, + "learning_rate": 3.823708788487851e-06, + "loss": 0.85353076, + "num_input_tokens_seen": 4118770, + "router_z_loss_clip": 2.7734375, + "router_z_loss_mlp": 0.54492188, + "step": 196, + "time_per_iteration": 3.0319948196411133 + }, + { + "auxiliary_loss_clip": 0.01387218, + "auxiliary_loss_mlp": 0.01109468, + "balance_loss_clip": 1.11268973, + "balance_loss_mlp": 1.05854166, + "epoch": 0.02368784945590092, + "flos": 25193598192000.0, + "grad_norm": 2.025565045980547, + "language_loss": 0.84701538, + "learning_rate": 3.827395544281781e-06, + "loss": 0.87198222, + "num_input_tokens_seen": 4141110, + "router_z_loss_clip": 2.74804688, + "router_z_loss_mlp": 0.50927734, + "step": 197, + "time_per_iteration": 2.9733994007110596 + }, + { + "auxiliary_loss_clip": 0.01395369, + "auxiliary_loss_mlp": 0.01115912, + "balance_loss_clip": 1.11396503, + "balance_loss_mlp": 1.06500995, + "epoch": 0.02380809234654001, + "flos": 27562481164800.0, + "grad_norm": 2.0054478955310224, + "language_loss": 0.79112291, + "learning_rate": 3.831063632877802e-06, + "loss": 0.81623566, + "num_input_tokens_seen": 4161430, + "router_z_loss_clip": 2.8125, + "router_z_loss_mlp": 0.50952148, + "step": 198, + "time_per_iteration": 2.9878087043762207 + }, + { + "auxiliary_loss_clip": 0.01380029, + "auxiliary_loss_mlp": 0.01114495, + "balance_loss_clip": 1.11389077, + "balance_loss_mlp": 1.0614233, + "epoch": 0.0239283352371791, + "flos": 18259786540800.0, + "grad_norm": 2.7956642587704215, + "language_loss": 0.7586453, + "learning_rate": 3.834713242359712e-06, + "loss": 0.78359056, + "num_input_tokens_seen": 4179260, + "router_z_loss_clip": 2.66015625, + "router_z_loss_mlp": 0.53076172, + "step": 199, + "time_per_iteration": 2.939791202545166 + }, + { + "auxiliary_loss_clip": 0.01385413, + "auxiliary_loss_mlp": 0.01114518, + "balance_loss_clip": 1.11231852, + "balance_loss_mlp": 1.06132662, + "epoch": 0.02404857812781819, + "flos": 21395110942080.0, + "grad_norm": 1.886883121560149, + "language_loss": 0.87281096, + "learning_rate": 3.838344557982959e-06, + "loss": 0.89781028, + "num_input_tokens_seen": 4200640, + "router_z_loss_clip": 2.72851562, + "router_z_loss_mlp": 0.53222656, + "step": 200, + "time_per_iteration": 2.9297869205474854 + }, + { + "auxiliary_loss_clip": 0.01385035, + "auxiliary_loss_mlp": 0.01093118, + "balance_loss_clip": 1.10693979, + "balance_loss_mlp": 1.04376554, + "epoch": 0.024168821018457284, + "flos": 16654256426880.0, + "grad_norm": 2.591175284269284, + "language_loss": 0.84686053, + "learning_rate": 3.841957762231063e-06, + "loss": 0.87164211, + "num_input_tokens_seen": 4218170, + "router_z_loss_clip": 2.77929688, + "router_z_loss_mlp": 0.49389648, + "step": 201, + "time_per_iteration": 2.9427168369293213 + }, + { + "auxiliary_loss_clip": 0.0137573, + "auxiliary_loss_mlp": 0.01098227, + "balance_loss_clip": 1.10515106, + "balance_loss_mlp": 1.047611, + "epoch": 0.024289063909096374, + "flos": 22820872464000.0, + "grad_norm": 1.9650299091500794, + "language_loss": 0.8763839, + "learning_rate": 3.8455530348706454e-06, + "loss": 0.90112352, + "num_input_tokens_seen": 4237770, + "router_z_loss_clip": 2.70507812, + "router_z_loss_mlp": 0.50585938, + "step": 202, + "time_per_iteration": 2.9115045070648193 + }, + { + "auxiliary_loss_clip": 0.01377122, + "auxiliary_loss_mlp": 0.01098539, + "balance_loss_clip": 1.11006904, + "balance_loss_mlp": 1.04847074, + "epoch": 0.024409306799735464, + "flos": 17748598135680.0, + "grad_norm": 1.9233601596843368, + "language_loss": 0.77438408, + "learning_rate": 3.849130553005099e-06, + "loss": 0.79914063, + "num_input_tokens_seen": 4255985, + "router_z_loss_clip": 2.671875, + "router_z_loss_mlp": 0.50097656, + "step": 203, + "time_per_iteration": 2.878140926361084 + }, + { + "auxiliary_loss_clip": 0.01384038, + "auxiliary_loss_mlp": 0.01101919, + "balance_loss_clip": 1.10772514, + "balance_loss_mlp": 1.04999185, + "epoch": 0.024529549690374557, + "flos": 21616213109760.0, + "grad_norm": 1.7541502380109302, + "language_loss": 0.83628982, + "learning_rate": 3.852690491126933e-06, + "loss": 0.86114943, + "num_input_tokens_seen": 4276035, + "router_z_loss_clip": 2.7578125, + "router_z_loss_mlp": 0.51977539, + "step": 204, + "time_per_iteration": 2.916992664337158 + }, + { + "auxiliary_loss_clip": 0.01379666, + "auxiliary_loss_mlp": 0.01102382, + "balance_loss_clip": 1.10571599, + "balance_loss_mlp": 1.05267167, + "epoch": 0.024649792581013647, + "flos": 25551662918400.0, + "grad_norm": 3.52177878625982, + "language_loss": 0.91323984, + "learning_rate": 3.856233021168845e-06, + "loss": 0.93806034, + "num_input_tokens_seen": 4295730, + "router_z_loss_clip": 2.7421875, + "router_z_loss_mlp": 0.49731445, + "step": 205, + "time_per_iteration": 2.877096652984619 + }, + { + "auxiliary_loss_clip": 0.01362002, + "auxiliary_loss_mlp": 0.01113015, + "balance_loss_clip": 1.10050392, + "balance_loss_mlp": 1.0653553, + "epoch": 0.024770035471652737, + "flos": 34495574544000.0, + "grad_norm": 8.498511694504229, + "language_loss": 0.91093373, + "learning_rate": 3.859758312553544e-06, + "loss": 0.93568391, + "num_input_tokens_seen": 4317950, + "router_z_loss_clip": 2.61328125, + "router_z_loss_mlp": 0.4765625, + "step": 206, + "time_per_iteration": 2.9783034324645996 + }, + { + "auxiliary_loss_clip": 0.01381438, + "auxiliary_loss_mlp": 0.01106763, + "balance_loss_clip": 1.11047411, + "balance_loss_mlp": 1.05750537, + "epoch": 0.02489027836229183, + "flos": 21505428587520.0, + "grad_norm": 2.2750771566407946, + "language_loss": 0.91903669, + "learning_rate": 3.8632665322423735e-06, + "loss": 0.9439187, + "num_input_tokens_seen": 4337605, + "router_z_loss_clip": 2.70703125, + "router_z_loss_mlp": 0.49291992, + "step": 207, + "time_per_iteration": 2.818882465362549 + }, + { + "auxiliary_loss_clip": 0.01371102, + "auxiliary_loss_mlp": 0.01112693, + "balance_loss_clip": 1.10852528, + "balance_loss_mlp": 1.06133747, + "epoch": 0.02501052125293092, + "flos": 23219013790080.0, + "grad_norm": 2.7796303793762904, + "language_loss": 0.85795856, + "learning_rate": 3.866757844782762e-06, + "loss": 0.88279653, + "num_input_tokens_seen": 4358110, + "router_z_loss_clip": 2.62890625, + "router_z_loss_mlp": 0.51342773, + "step": 208, + "time_per_iteration": 2.886105537414551 + }, + { + "auxiliary_loss_clip": 0.01369791, + "auxiliary_loss_mlp": 0.01105452, + "balance_loss_clip": 1.10603058, + "balance_loss_mlp": 1.05552721, + "epoch": 0.02513076414357001, + "flos": 26388920010240.0, + "grad_norm": 2.9309677374562715, + "language_loss": 0.91438651, + "learning_rate": 3.870232412354527e-06, + "loss": 0.93913901, + "num_input_tokens_seen": 4374955, + "router_z_loss_clip": 2.63671875, + "router_z_loss_mlp": 0.49975586, + "step": 209, + "time_per_iteration": 2.908290147781372 + }, + { + "auxiliary_loss_clip": 0.01374703, + "auxiliary_loss_mlp": 0.01100578, + "balance_loss_clip": 1.10544443, + "balance_loss_mlp": 1.04958045, + "epoch": 0.025251007034209103, + "flos": 13590430047360.0, + "grad_norm": 2.149452040484826, + "language_loss": 0.92511368, + "learning_rate": 3.873690394815086e-06, + "loss": 0.94986653, + "num_input_tokens_seen": 4391535, + "router_z_loss_clip": 2.69335938, + "router_z_loss_mlp": 0.51049805, + "step": 210, + "time_per_iteration": 2.843076229095459 + }, + { + "auxiliary_loss_clip": 0.01371141, + "auxiliary_loss_mlp": 0.01113751, + "balance_loss_clip": 1.101264, + "balance_loss_mlp": 1.06294394, + "epoch": 0.025371249924848193, + "flos": 15049229103360.0, + "grad_norm": 3.286549776299636, + "language_loss": 0.91079307, + "learning_rate": 3.877131949743587e-06, + "loss": 0.935642, + "num_input_tokens_seen": 4408400, + "router_z_loss_clip": 2.6953125, + "router_z_loss_mlp": 0.5078125, + "step": 211, + "time_per_iteration": 2.9232518672943115 + }, + { + "auxiliary_loss_clip": 0.0136888, + "auxiliary_loss_mlp": 0.01107065, + "balance_loss_clip": 1.10169792, + "balance_loss_mlp": 1.06000102, + "epoch": 0.025491492815487283, + "flos": 25553853648000.0, + "grad_norm": 2.340432352860666, + "language_loss": 0.78187156, + "learning_rate": 3.880557232483993e-06, + "loss": 0.80663109, + "num_input_tokens_seen": 4427840, + "router_z_loss_clip": 2.671875, + "router_z_loss_mlp": 0.47094727, + "step": 212, + "time_per_iteration": 2.9409701824188232 + }, + { + "auxiliary_loss_clip": 0.01366222, + "auxiliary_loss_mlp": 0.01102121, + "balance_loss_clip": 1.10177207, + "balance_loss_mlp": 1.05064631, + "epoch": 0.025611735706126376, + "flos": 20630752502400.0, + "grad_norm": 2.004066008972625, + "language_loss": 0.86991996, + "learning_rate": 3.883966396187164e-06, + "loss": 0.89460337, + "num_input_tokens_seen": 4447110, + "router_z_loss_clip": 2.6484375, + "router_z_loss_mlp": 0.51416016, + "step": 213, + "time_per_iteration": 2.8822405338287354 + }, + { + "auxiliary_loss_clip": 0.0136862, + "auxiliary_loss_mlp": 0.01099565, + "balance_loss_clip": 1.10327947, + "balance_loss_mlp": 1.05088043, + "epoch": 0.025731978596765466, + "flos": 19062282245760.0, + "grad_norm": 2.11706624049736, + "language_loss": 0.90246642, + "learning_rate": 3.887359591851937e-06, + "loss": 0.92714834, + "num_input_tokens_seen": 4464715, + "router_z_loss_clip": 2.65039062, + "router_z_loss_mlp": 0.48706055, + "step": 214, + "time_per_iteration": 2.881404161453247 + }, + { + "auxiliary_loss_clip": 0.01365551, + "auxiliary_loss_mlp": 0.01095058, + "balance_loss_clip": 1.10662341, + "balance_loss_mlp": 1.04525256, + "epoch": 0.025852221487404556, + "flos": 22163814927360.0, + "grad_norm": 1.7434620119409412, + "language_loss": 0.92242861, + "learning_rate": 3.890736968365265e-06, + "loss": 0.94703472, + "num_input_tokens_seen": 4485030, + "router_z_loss_clip": 2.58984375, + "router_z_loss_mlp": 0.49829102, + "step": 215, + "time_per_iteration": 2.9210195541381836 + }, + { + "auxiliary_loss_clip": 0.01364482, + "auxiliary_loss_mlp": 0.01082366, + "balance_loss_clip": 1.10147738, + "balance_loss_mlp": 1.03747177, + "epoch": 0.02597246437804365, + "flos": 26541971861760.0, + "grad_norm": 1.8918313663167419, + "language_loss": 0.84826571, + "learning_rate": 3.894098672541412e-06, + "loss": 0.87273425, + "num_input_tokens_seen": 4505935, + "router_z_loss_clip": 2.62890625, + "router_z_loss_mlp": 0.44921875, + "step": 216, + "time_per_iteration": 2.9940028190612793 + }, + { + "auxiliary_loss_clip": 0.01366637, + "auxiliary_loss_mlp": 0.01091973, + "balance_loss_clip": 1.10385275, + "balance_loss_mlp": 1.04509985, + "epoch": 0.02609270726868274, + "flos": 32671671696000.0, + "grad_norm": 1.8362889159846931, + "language_loss": 0.7527684, + "learning_rate": 3.89744484916025e-06, + "loss": 0.77735448, + "num_input_tokens_seen": 4527045, + "router_z_loss_clip": 2.63085938, + "router_z_loss_mlp": 0.46850586, + "step": 217, + "time_per_iteration": 2.931548833847046 + }, + { + "auxiliary_loss_clip": 0.01365316, + "auxiliary_loss_mlp": 0.01096482, + "balance_loss_clip": 1.10087943, + "balance_loss_mlp": 1.04488802, + "epoch": 0.02621295015932183, + "flos": 26243553669120.0, + "grad_norm": 1.9018787748091757, + "language_loss": 0.87374616, + "learning_rate": 3.900775641004673e-06, + "loss": 0.89836413, + "num_input_tokens_seen": 4546360, + "router_z_loss_clip": 2.64453125, + "router_z_loss_mlp": 0.51635742, + "step": 218, + "time_per_iteration": 2.892521619796753 + }, + { + "auxiliary_loss_clip": 0.01381293, + "auxiliary_loss_mlp": 0.01100765, + "balance_loss_clip": 1.10716033, + "balance_loss_mlp": 1.05396366, + "epoch": 0.026333193049960922, + "flos": 42921402353280.0, + "grad_norm": 4.007286400098493, + "language_loss": 0.73748839, + "learning_rate": 3.904091188897156e-06, + "loss": 0.76230896, + "num_input_tokens_seen": 4565495, + "router_z_loss_clip": 2.7421875, + "router_z_loss_mlp": 0.46777344, + "step": 219, + "time_per_iteration": 3.019831657409668 + }, + { + "auxiliary_loss_clip": 0.01367569, + "auxiliary_loss_mlp": 0.01105214, + "balance_loss_clip": 1.10670841, + "balance_loss_mlp": 1.05564642, + "epoch": 0.026453435940600012, + "flos": 17963846386560.0, + "grad_norm": 2.152835875189963, + "language_loss": 0.81691134, + "learning_rate": 3.90739163173548e-06, + "loss": 0.84163916, + "num_input_tokens_seen": 4583330, + "router_z_loss_clip": 2.609375, + "router_z_loss_mlp": 0.49609375, + "step": 220, + "time_per_iteration": 3.7967052459716797 + }, + { + "auxiliary_loss_clip": 0.01366215, + "auxiliary_loss_mlp": 0.01104031, + "balance_loss_clip": 1.10335779, + "balance_loss_mlp": 1.05422521, + "epoch": 0.026573678831239102, + "flos": 18984319776000.0, + "grad_norm": 2.6523969380718473, + "language_loss": 0.88308275, + "learning_rate": 3.910677106527646e-06, + "loss": 0.90778518, + "num_input_tokens_seen": 4600520, + "router_z_loss_clip": 2.63085938, + "router_z_loss_mlp": 0.49804688, + "step": 221, + "time_per_iteration": 2.8970894813537598 + }, + { + "auxiliary_loss_clip": 0.01359609, + "auxiliary_loss_mlp": 0.01107648, + "balance_loss_clip": 1.09895241, + "balance_loss_mlp": 1.05858183, + "epoch": 0.026693921721878195, + "flos": 29241448634880.0, + "grad_norm": 2.671015177172478, + "language_loss": 0.84176195, + "learning_rate": 3.913947748426004e-06, + "loss": 0.86643451, + "num_input_tokens_seen": 4617340, + "router_z_loss_clip": 2.60742188, + "router_z_loss_mlp": 0.49072266, + "step": 222, + "time_per_iteration": 2.9431238174438477 + }, + { + "auxiliary_loss_clip": 0.01362867, + "auxiliary_loss_mlp": 0.01106073, + "balance_loss_clip": 1.10226774, + "balance_loss_mlp": 1.0547893, + "epoch": 0.026814164612517285, + "flos": 14128083797760.0, + "grad_norm": 2.985248989959295, + "language_loss": 0.76513314, + "learning_rate": 3.9172036907606136e-06, + "loss": 0.78982258, + "num_input_tokens_seen": 4630820, + "router_z_loss_clip": 2.60546875, + "router_z_loss_mlp": 0.51245117, + "step": 223, + "time_per_iteration": 2.8840904235839844 + }, + { + "auxiliary_loss_clip": 0.01366539, + "auxiliary_loss_mlp": 0.01111122, + "balance_loss_clip": 1.10428715, + "balance_loss_mlp": 1.05914629, + "epoch": 0.026934407503156375, + "flos": 23511973115520.0, + "grad_norm": 2.805036489672205, + "language_loss": 0.94933349, + "learning_rate": 3.920445065071855e-06, + "loss": 0.97411013, + "num_input_tokens_seen": 4651985, + "router_z_loss_clip": 2.62109375, + "router_z_loss_mlp": 0.51953125, + "step": 224, + "time_per_iteration": 2.862391948699951 + }, + { + "auxiliary_loss_clip": 0.01357061, + "auxiliary_loss_mlp": 0.0109414, + "balance_loss_clip": 1.10024154, + "balance_loss_mlp": 1.04359531, + "epoch": 0.027054650393795468, + "flos": 28950356816640.0, + "grad_norm": 2.6571048171365725, + "language_loss": 0.79912651, + "learning_rate": 3.923672001142322e-06, + "loss": 0.82363856, + "num_input_tokens_seen": 4672295, + "router_z_loss_clip": 2.56835938, + "router_z_loss_mlp": 0.50585938, + "step": 225, + "time_per_iteration": 2.891935348510742 + }, + { + "auxiliary_loss_clip": 0.01357432, + "auxiliary_loss_mlp": 0.01107804, + "balance_loss_clip": 1.09776878, + "balance_loss_mlp": 1.05919063, + "epoch": 0.027174893284434558, + "flos": 31431568596480.0, + "grad_norm": 2.0458329441507783, + "language_loss": 0.84330189, + "learning_rate": 3.926884627027996e-06, + "loss": 0.86795425, + "num_input_tokens_seen": 4696065, + "router_z_loss_clip": 2.59765625, + "router_z_loss_mlp": 0.48583984, + "step": 226, + "time_per_iteration": 2.9243528842926025 + }, + { + "auxiliary_loss_clip": 0.01359916, + "auxiliary_loss_mlp": 0.01107112, + "balance_loss_clip": 1.10047102, + "balance_loss_mlp": 1.05966723, + "epoch": 0.027295136175073648, + "flos": 22054466949120.0, + "grad_norm": 2.084954347062194, + "language_loss": 0.77379161, + "learning_rate": 3.930083069088744e-06, + "loss": 0.79846191, + "num_input_tokens_seen": 4716065, + "router_z_loss_clip": 2.59179688, + "router_z_loss_mlp": 0.47436523, + "step": 227, + "time_per_iteration": 2.9734184741973877 + }, + { + "auxiliary_loss_clip": 0.01322164, + "auxiliary_loss_mlp": 0.01025873, + "balance_loss_clip": 1.16176081, + "balance_loss_mlp": 1.00107706, + "epoch": 0.02741537906571274, + "flos": 60800752972800.0, + "grad_norm": 0.9879355857137758, + "language_loss": 0.59272993, + "learning_rate": 3.933267452018137e-06, + "loss": 0.61621022, + "num_input_tokens_seen": 4775860, + "router_z_loss_clip": 1.609375, + "router_z_loss_mlp": 0.24707031, + "step": 228, + "time_per_iteration": 3.4130313396453857 + }, + { + "auxiliary_loss_clip": 0.01354039, + "auxiliary_loss_mlp": 0.01095518, + "balance_loss_clip": 1.1008296, + "balance_loss_mlp": 1.04912138, + "epoch": 0.02753562195635183, + "flos": 24606278910720.0, + "grad_norm": 2.0162526654349, + "language_loss": 0.84305894, + "learning_rate": 3.936437898872622e-06, + "loss": 0.86755455, + "num_input_tokens_seen": 4795835, + "router_z_loss_clip": 2.53515625, + "router_z_loss_mlp": 0.46411133, + "step": 229, + "time_per_iteration": 2.8974955081939697 + }, + { + "auxiliary_loss_clip": 0.01363173, + "auxiliary_loss_mlp": 0.01097896, + "balance_loss_clip": 1.10272455, + "balance_loss_mlp": 1.05076075, + "epoch": 0.02765586484699092, + "flos": 34094236907520.0, + "grad_norm": 2.61064649200512, + "language_loss": 0.79580694, + "learning_rate": 3.9395945311000525e-06, + "loss": 0.82041764, + "num_input_tokens_seen": 4817460, + "router_z_loss_clip": 2.59960938, + "router_z_loss_mlp": 0.47143555, + "step": 230, + "time_per_iteration": 2.9391427040100098 + }, + { + "auxiliary_loss_clip": 0.01361203, + "auxiliary_loss_mlp": 0.01105326, + "balance_loss_clip": 1.1022898, + "balance_loss_mlp": 1.05707002, + "epoch": 0.027776107737630014, + "flos": 14829922615680.0, + "grad_norm": 2.2547041114100614, + "language_loss": 0.90836895, + "learning_rate": 3.942737468567608e-06, + "loss": 0.93303418, + "num_input_tokens_seen": 4835475, + "router_z_loss_clip": 2.58789062, + "router_z_loss_mlp": 0.4831543, + "step": 231, + "time_per_iteration": 2.8288025856018066 + }, + { + "auxiliary_loss_clip": 0.01360536, + "auxiliary_loss_mlp": 0.01094452, + "balance_loss_clip": 1.09967756, + "balance_loss_mlp": 1.04776967, + "epoch": 0.027896350628269104, + "flos": 47920347066240.0, + "grad_norm": 2.2493263662478054, + "language_loss": 0.8611595, + "learning_rate": 3.9458668295891026e-06, + "loss": 0.8857094, + "num_input_tokens_seen": 4857760, + "router_z_loss_clip": 2.61328125, + "router_z_loss_mlp": 0.46679688, + "step": 232, + "time_per_iteration": 3.1508216857910156 + }, + { + "auxiliary_loss_clip": 0.01354691, + "auxiliary_loss_mlp": 0.01107412, + "balance_loss_clip": 1.09801316, + "balance_loss_mlp": 1.0579164, + "epoch": 0.028016593518908194, + "flos": 21684550734720.0, + "grad_norm": 2.37831150476217, + "language_loss": 0.86762422, + "learning_rate": 3.948982730951712e-06, + "loss": 0.89224529, + "num_input_tokens_seen": 4875855, + "router_z_loss_clip": 2.56835938, + "router_z_loss_mlp": 0.49487305, + "step": 233, + "time_per_iteration": 2.8510706424713135 + }, + { + "auxiliary_loss_clip": 0.01359753, + "auxiliary_loss_mlp": 0.01100127, + "balance_loss_clip": 1.09810901, + "balance_loss_mlp": 1.05353999, + "epoch": 0.028136836409547287, + "flos": 18439483305600.0, + "grad_norm": 2.211183237050757, + "language_loss": 0.82141781, + "learning_rate": 3.9520852879421254e-06, + "loss": 0.84601665, + "num_input_tokens_seen": 4893200, + "router_z_loss_clip": 2.6171875, + "router_z_loss_mlp": 0.46606445, + "step": 234, + "time_per_iteration": 2.8086907863616943 + }, + { + "auxiliary_loss_clip": 0.01348044, + "auxiliary_loss_mlp": 0.01098253, + "balance_loss_clip": 1.09717381, + "balance_loss_mlp": 1.0506649, + "epoch": 0.028257079300186377, + "flos": 31576934937600.0, + "grad_norm": 2.504406447511031, + "language_loss": 0.81658483, + "learning_rate": 3.955174614372137e-06, + "loss": 0.84104776, + "num_input_tokens_seen": 4912965, + "router_z_loss_clip": 2.50976562, + "router_z_loss_mlp": 0.47583008, + "step": 235, + "time_per_iteration": 3.020660638809204 + }, + { + "auxiliary_loss_clip": 0.01353673, + "auxiliary_loss_mlp": 0.01093192, + "balance_loss_clip": 1.09495938, + "balance_loss_mlp": 1.04558003, + "epoch": 0.028377322190825467, + "flos": 23513337832320.0, + "grad_norm": 2.7615921901701834, + "language_loss": 0.84324712, + "learning_rate": 3.9582508226037045e-06, + "loss": 0.86771572, + "num_input_tokens_seen": 4933105, + "router_z_loss_clip": 2.58984375, + "router_z_loss_mlp": 0.47607422, + "step": 236, + "time_per_iteration": 2.9464917182922363 + }, + { + "auxiliary_loss_clip": 0.01359688, + "auxiliary_loss_mlp": 0.01103516, + "balance_loss_clip": 1.09780872, + "balance_loss_mlp": 1.05201793, + "epoch": 0.02849756508146456, + "flos": 20479604071680.0, + "grad_norm": 2.47656780788673, + "language_loss": 0.94076985, + "learning_rate": 3.9613140235734636e-06, + "loss": 0.96540195, + "num_input_tokens_seen": 4950085, + "router_z_loss_clip": 2.6171875, + "router_z_loss_mlp": 0.51489258, + "step": 237, + "time_per_iteration": 2.8452396392822266 + }, + { + "auxiliary_loss_clip": 0.01356116, + "auxiliary_loss_mlp": 0.01098217, + "balance_loss_clip": 1.09938419, + "balance_loss_mlp": 1.04948378, + "epoch": 0.02861780797210365, + "flos": 14283362292480.0, + "grad_norm": 1.9292703587075313, + "language_loss": 0.81045419, + "learning_rate": 3.96436432681674e-06, + "loss": 0.83499753, + "num_input_tokens_seen": 4968075, + "router_z_loss_clip": 2.56640625, + "router_z_loss_mlp": 0.48754883, + "step": 238, + "time_per_iteration": 2.867485523223877 + }, + { + "auxiliary_loss_clip": 0.01355976, + "auxiliary_loss_mlp": 0.0110955, + "balance_loss_clip": 1.09683919, + "balance_loss_mlp": 1.06124628, + "epoch": 0.02873805086274274, + "flos": 25808532053760.0, + "grad_norm": 2.2155860516574446, + "language_loss": 0.89301252, + "learning_rate": 3.967401840491044e-06, + "loss": 0.91766781, + "num_input_tokens_seen": 4987355, + "router_z_loss_clip": 2.59179688, + "router_z_loss_mlp": 0.4831543, + "step": 239, + "time_per_iteration": 2.97391676902771 + }, + { + "auxiliary_loss_clip": 0.01345611, + "auxiliary_loss_mlp": 0.01099582, + "balance_loss_clip": 1.09876466, + "balance_loss_mlp": 1.05640483, + "epoch": 0.028858293753381833, + "flos": 17304238984320.0, + "grad_norm": 2.1541041311498357, + "language_loss": 0.88105834, + "learning_rate": 3.97042667139909e-06, + "loss": 0.90551019, + "num_input_tokens_seen": 5004680, + "router_z_loss_clip": 2.46679688, + "router_z_loss_mlp": 0.43164062, + "step": 240, + "time_per_iteration": 2.999467611312866 + }, + { + "auxiliary_loss_clip": 0.0135141, + "auxiliary_loss_mlp": 0.01093469, + "balance_loss_clip": 1.09755111, + "balance_loss_mlp": 1.04564178, + "epoch": 0.028978536644020923, + "flos": 23038347358080.0, + "grad_norm": 2.104675267468764, + "language_loss": 0.87273383, + "learning_rate": 3.973438925011327e-06, + "loss": 0.89718258, + "num_input_tokens_seen": 5022965, + "router_z_loss_clip": 2.54101562, + "router_z_loss_mlp": 0.47827148, + "step": 241, + "time_per_iteration": 2.944638729095459 + }, + { + "auxiliary_loss_clip": 0.01345687, + "auxiliary_loss_mlp": 0.01101599, + "balance_loss_clip": 1.09212542, + "balance_loss_mlp": 1.05372405, + "epoch": 0.029098779534660012, + "flos": 28329712692480.0, + "grad_norm": 2.333093350265628, + "language_loss": 0.91618013, + "learning_rate": 3.976438705488002e-06, + "loss": 0.94065297, + "num_input_tokens_seen": 5042625, + "router_z_loss_clip": 2.53515625, + "router_z_loss_mlp": 0.47851562, + "step": 242, + "time_per_iteration": 3.0530903339385986 + }, + { + "auxiliary_loss_clip": 0.01349209, + "auxiliary_loss_mlp": 0.01100344, + "balance_loss_clip": 1.09831941, + "balance_loss_mlp": 1.05347157, + "epoch": 0.029219022425299106, + "flos": 13881665520000.0, + "grad_norm": 2.2992510399215416, + "language_loss": 0.93198508, + "learning_rate": 3.9794261157007744e-06, + "loss": 0.95648062, + "num_input_tokens_seen": 5060380, + "router_z_loss_clip": 2.50585938, + "router_z_loss_mlp": 0.46875, + "step": 243, + "time_per_iteration": 3.0478124618530273 + }, + { + "auxiliary_loss_clip": 0.01353224, + "auxiliary_loss_mlp": 0.0110674, + "balance_loss_clip": 1.09500957, + "balance_loss_mlp": 1.05855584, + "epoch": 0.029339265315938196, + "flos": 19422501788160.0, + "grad_norm": 2.1976219551452454, + "language_loss": 0.84959412, + "learning_rate": 3.982401257253887e-06, + "loss": 0.87419373, + "num_input_tokens_seen": 5078720, + "router_z_loss_clip": 2.58007812, + "router_z_loss_mlp": 0.48242188, + "step": 244, + "time_per_iteration": 2.8775794506073 + }, + { + "auxiliary_loss_clip": 0.01346376, + "auxiliary_loss_mlp": 0.01096428, + "balance_loss_clip": 1.09589136, + "balance_loss_mlp": 1.04752827, + "epoch": 0.029459508206577285, + "flos": 15669550005120.0, + "grad_norm": 2.661654034282831, + "language_loss": 0.89452219, + "learning_rate": 3.985364230504893e-06, + "loss": 0.9189502, + "num_input_tokens_seen": 5096605, + "router_z_loss_clip": 2.50585938, + "router_z_loss_mlp": 0.48876953, + "step": 245, + "time_per_iteration": 2.845710039138794 + }, + { + "auxiliary_loss_clip": 0.01347089, + "auxiliary_loss_mlp": 0.01113669, + "balance_loss_clip": 1.09431374, + "balance_loss_mlp": 1.06405413, + "epoch": 0.02957975109721638, + "flos": 28220975245440.0, + "grad_norm": 1.961629174583271, + "language_loss": 0.84560525, + "learning_rate": 3.988315134584976e-06, + "loss": 0.87021279, + "num_input_tokens_seen": 5116285, + "router_z_loss_clip": 2.52539062, + "router_z_loss_mlp": 0.49633789, + "step": 246, + "time_per_iteration": 2.8610281944274902 + }, + { + "auxiliary_loss_clip": 0.01352253, + "auxiliary_loss_mlp": 0.01095911, + "balance_loss_clip": 1.09710407, + "balance_loss_mlp": 1.04915679, + "epoch": 0.02969999398785547, + "flos": 24315869450880.0, + "grad_norm": 1.7037703406764981, + "language_loss": 0.80404282, + "learning_rate": 3.991254067418851e-06, + "loss": 0.82852447, + "num_input_tokens_seen": 5136825, + "router_z_loss_clip": 2.546875, + "router_z_loss_mlp": 0.46728516, + "step": 247, + "time_per_iteration": 2.851442575454712 + }, + { + "auxiliary_loss_clip": 0.01343304, + "auxiliary_loss_mlp": 0.01097565, + "balance_loss_clip": 1.09583402, + "balance_loss_mlp": 1.05188417, + "epoch": 0.02982023687849456, + "flos": 35078584193280.0, + "grad_norm": 2.7537249416822807, + "language_loss": 0.82724404, + "learning_rate": 3.994181125744254e-06, + "loss": 0.85165262, + "num_input_tokens_seen": 5158630, + "router_z_loss_clip": 2.48046875, + "router_z_loss_mlp": 0.45654297, + "step": 248, + "time_per_iteration": 2.9314932823181152 + }, + { + "auxiliary_loss_clip": 0.01338235, + "auxiliary_loss_mlp": 0.01098665, + "balance_loss_clip": 1.09448743, + "balance_loss_mlp": 1.05086231, + "epoch": 0.02994047976913365, + "flos": 26177155378560.0, + "grad_norm": 1.7662213806733948, + "language_loss": 0.73860705, + "learning_rate": 3.99709640513106e-06, + "loss": 0.76297605, + "num_input_tokens_seen": 5179510, + "router_z_loss_clip": 2.4375, + "router_z_loss_mlp": 0.47827148, + "step": 249, + "time_per_iteration": 5.351762533187866 + }, + { + "auxiliary_loss_clip": 0.01347167, + "auxiliary_loss_mlp": 0.01105745, + "balance_loss_clip": 1.09317327, + "balance_loss_mlp": 1.05939674, + "epoch": 0.03006072265977274, + "flos": 25625028447360.0, + "grad_norm": 2.11206820043969, + "language_loss": 0.85669994, + "learning_rate": 4e-06, + "loss": 0.88122904, + "num_input_tokens_seen": 5199345, + "router_z_loss_clip": 2.54296875, + "router_z_loss_mlp": 0.46386719, + "step": 250, + "time_per_iteration": 2.8502559661865234 + }, + { + "auxiliary_loss_clip": 0.01342189, + "auxiliary_loss_mlp": 0.01104689, + "balance_loss_clip": 1.09443378, + "balance_loss_mlp": 1.05829251, + "epoch": 0.03018096555041183, + "flos": 22127078292480.0, + "grad_norm": 2.9356565591280734, + "language_loss": 0.88543081, + "learning_rate": 3.999999848300794e-06, + "loss": 0.90989953, + "num_input_tokens_seen": 5218330, + "router_z_loss_clip": 2.47851562, + "router_z_loss_mlp": 0.46362305, + "step": 251, + "time_per_iteration": 2.885157346725464 + }, + { + "auxiliary_loss_clip": 0.01335971, + "auxiliary_loss_mlp": 0.01085167, + "balance_loss_clip": 1.09002757, + "balance_loss_mlp": 1.03955746, + "epoch": 0.030301208441050925, + "flos": 30188197359360.0, + "grad_norm": 1.6277377439640277, + "language_loss": 0.89051169, + "learning_rate": 3.999999393203203e-06, + "loss": 0.9147231, + "num_input_tokens_seen": 5240740, + "router_z_loss_clip": 2.45898438, + "router_z_loss_mlp": 0.45629883, + "step": 252, + "time_per_iteration": 2.9858968257904053 + }, + { + "auxiliary_loss_clip": 0.01342143, + "auxiliary_loss_mlp": 0.01104152, + "balance_loss_clip": 1.0920043, + "balance_loss_mlp": 1.05792284, + "epoch": 0.030421451331690014, + "flos": 23621392920960.0, + "grad_norm": 1.9616590660993007, + "language_loss": 0.84747398, + "learning_rate": 3.999998634707293e-06, + "loss": 0.87193698, + "num_input_tokens_seen": 5260290, + "router_z_loss_clip": 2.5, + "router_z_loss_mlp": 0.46240234, + "step": 253, + "time_per_iteration": 2.8723764419555664 + }, + { + "auxiliary_loss_clip": 0.01348603, + "auxiliary_loss_mlp": 0.01116346, + "balance_loss_clip": 1.09625936, + "balance_loss_mlp": 1.06739831, + "epoch": 0.030541694222329104, + "flos": 27928446883200.0, + "grad_norm": 2.205867077133445, + "language_loss": 0.9658339, + "learning_rate": 3.999997572813182e-06, + "loss": 0.99048328, + "num_input_tokens_seen": 5278100, + "router_z_loss_clip": 2.52148438, + "router_z_loss_mlp": 0.48950195, + "step": 254, + "time_per_iteration": 2.907452344894409 + }, + { + "auxiliary_loss_clip": 0.01337594, + "auxiliary_loss_mlp": 0.0109183, + "balance_loss_clip": 1.09179592, + "balance_loss_mlp": 1.04500496, + "epoch": 0.030661937112968194, + "flos": 18588441006720.0, + "grad_norm": 2.1481727086545135, + "language_loss": 0.87606114, + "learning_rate": 3.999996207521028e-06, + "loss": 0.9003554, + "num_input_tokens_seen": 5296810, + "router_z_loss_clip": 2.45703125, + "router_z_loss_mlp": 0.46850586, + "step": 255, + "time_per_iteration": 2.963449239730835 + }, + { + "auxiliary_loss_clip": 0.01347456, + "auxiliary_loss_mlp": 0.01099714, + "balance_loss_clip": 1.09280479, + "balance_loss_mlp": 1.05579758, + "epoch": 0.030782180003607287, + "flos": 12969139478400.0, + "grad_norm": 2.491910980438938, + "language_loss": 0.82150054, + "learning_rate": 3.999994538831039e-06, + "loss": 0.84597218, + "num_input_tokens_seen": 5313395, + "router_z_loss_clip": 2.54492188, + "router_z_loss_mlp": 0.43920898, + "step": 256, + "time_per_iteration": 2.833160400390625 + }, + { + "auxiliary_loss_clip": 0.01341663, + "auxiliary_loss_mlp": 0.01103966, + "balance_loss_clip": 1.0902307, + "balance_loss_mlp": 1.05563831, + "epoch": 0.030902422894246377, + "flos": 23335364920320.0, + "grad_norm": 2.997453649231333, + "language_loss": 0.85643893, + "learning_rate": 3.99999256674347e-06, + "loss": 0.8808952, + "num_input_tokens_seen": 5333545, + "router_z_loss_clip": 2.515625, + "router_z_loss_mlp": 0.4831543, + "step": 257, + "time_per_iteration": 2.8840653896331787 + }, + { + "auxiliary_loss_clip": 0.01286323, + "auxiliary_loss_mlp": 0.01045557, + "balance_loss_clip": 1.12358046, + "balance_loss_mlp": 1.02333665, + "epoch": 0.031022665784885467, + "flos": 55094151438720.0, + "grad_norm": 1.0167057831591217, + "language_loss": 0.53543228, + "learning_rate": 3.999990291258618e-06, + "loss": 0.55875105, + "num_input_tokens_seen": 5392235, + "router_z_loss_clip": 1.625, + "router_z_loss_mlp": 0.22265625, + "step": 258, + "time_per_iteration": 3.359858989715576 + }, + { + "auxiliary_loss_clip": 0.0133613, + "auxiliary_loss_mlp": 0.01100229, + "balance_loss_clip": 1.09110594, + "balance_loss_mlp": 1.05521584, + "epoch": 0.03114290867552456, + "flos": 19317786664320.0, + "grad_norm": 2.355098461276531, + "language_loss": 0.86844009, + "learning_rate": 3.999987712376829e-06, + "loss": 0.89280373, + "num_input_tokens_seen": 5410555, + "router_z_loss_clip": 2.44921875, + "router_z_loss_mlp": 0.45019531, + "step": 259, + "time_per_iteration": 2.9019975662231445 + }, + { + "auxiliary_loss_clip": 0.01338519, + "auxiliary_loss_mlp": 0.01093985, + "balance_loss_clip": 1.09373808, + "balance_loss_mlp": 1.04904342, + "epoch": 0.031263151566163654, + "flos": 20959442881920.0, + "grad_norm": 1.9075497169964803, + "language_loss": 0.82117188, + "learning_rate": 3.999984830098494e-06, + "loss": 0.84549689, + "num_input_tokens_seen": 5430135, + "router_z_loss_clip": 2.44921875, + "router_z_loss_mlp": 0.44897461, + "step": 260, + "time_per_iteration": 2.8474433422088623 + }, + { + "auxiliary_loss_clip": 0.01340336, + "auxiliary_loss_mlp": 0.01097833, + "balance_loss_clip": 1.0923512, + "balance_loss_mlp": 1.04936206, + "epoch": 0.03138339445680274, + "flos": 14793006412800.0, + "grad_norm": 2.691301100675591, + "language_loss": 0.97780192, + "learning_rate": 3.999981644424051e-06, + "loss": 1.00218368, + "num_input_tokens_seen": 5444935, + "router_z_loss_clip": 2.48242188, + "router_z_loss_mlp": 0.484375, + "step": 261, + "time_per_iteration": 2.8440663814544678 + }, + { + "auxiliary_loss_clip": 0.01337156, + "auxiliary_loss_mlp": 0.01109728, + "balance_loss_clip": 1.08978033, + "balance_loss_mlp": 1.06249702, + "epoch": 0.03150363734744183, + "flos": 11655599022720.0, + "grad_norm": 2.4391268166895674, + "language_loss": 0.85842824, + "learning_rate": 3.999978155353982e-06, + "loss": 0.88289714, + "num_input_tokens_seen": 5462080, + "router_z_loss_clip": 2.47460938, + "router_z_loss_mlp": 0.47167969, + "step": 262, + "time_per_iteration": 2.9015965461730957 + }, + { + "auxiliary_loss_clip": 0.01337991, + "auxiliary_loss_mlp": 0.0108892, + "balance_loss_clip": 1.0876708, + "balance_loss_mlp": 1.04302406, + "epoch": 0.03162388023808092, + "flos": 33727732485120.0, + "grad_norm": 2.2279897635666215, + "language_loss": 0.80197126, + "learning_rate": 3.9999743628888186e-06, + "loss": 0.82624036, + "num_input_tokens_seen": 5483870, + "router_z_loss_clip": 2.50585938, + "router_z_loss_mlp": 0.45922852, + "step": 263, + "time_per_iteration": 3.024226427078247 + }, + { + "auxiliary_loss_clip": 0.01338832, + "auxiliary_loss_mlp": 0.01098453, + "balance_loss_clip": 1.0907104, + "balance_loss_mlp": 1.05055451, + "epoch": 0.03174412312872001, + "flos": 20810952057600.0, + "grad_norm": 2.3728456263306876, + "language_loss": 0.89596498, + "learning_rate": 3.999970267029133e-06, + "loss": 0.92033786, + "num_input_tokens_seen": 5502830, + "router_z_loss_clip": 2.48242188, + "router_z_loss_mlp": 0.47875977, + "step": 264, + "time_per_iteration": 3.0038228034973145 + }, + { + "auxiliary_loss_clip": 0.01332578, + "auxiliary_loss_mlp": 0.01105205, + "balance_loss_clip": 1.09086883, + "balance_loss_mlp": 1.05861819, + "epoch": 0.0318643660193591, + "flos": 23727939638400.0, + "grad_norm": 2.8548272531687964, + "language_loss": 0.80044955, + "learning_rate": 3.999965867775548e-06, + "loss": 0.82482737, + "num_input_tokens_seen": 5523225, + "router_z_loss_clip": 2.41796875, + "router_z_loss_mlp": 0.46557617, + "step": 265, + "time_per_iteration": 2.833587646484375 + }, + { + "auxiliary_loss_clip": 0.01336421, + "auxiliary_loss_mlp": 0.01099873, + "balance_loss_clip": 1.09356308, + "balance_loss_mlp": 1.0548358, + "epoch": 0.0319846089099982, + "flos": 13917863450880.0, + "grad_norm": 115.99151803999999, + "language_loss": 0.86653101, + "learning_rate": 3.9999611651287315e-06, + "loss": 0.89089394, + "num_input_tokens_seen": 5541380, + "router_z_loss_clip": 2.42773438, + "router_z_loss_mlp": 0.45019531, + "step": 266, + "time_per_iteration": 2.860980987548828 + }, + { + "auxiliary_loss_clip": 0.0133129, + "auxiliary_loss_mlp": 0.01090193, + "balance_loss_clip": 1.08924031, + "balance_loss_mlp": 1.04494071, + "epoch": 0.03210485180063729, + "flos": 14753253035520.0, + "grad_norm": 2.3883393059956575, + "language_loss": 0.78580809, + "learning_rate": 3.999956159089396e-06, + "loss": 0.81002295, + "num_input_tokens_seen": 5558830, + "router_z_loss_clip": 2.421875, + "router_z_loss_mlp": 0.45263672, + "step": 267, + "time_per_iteration": 2.8110334873199463 + }, + { + "auxiliary_loss_clip": 0.01332718, + "auxiliary_loss_mlp": 0.01098355, + "balance_loss_clip": 1.08984208, + "balance_loss_mlp": 1.05481935, + "epoch": 0.03222509469127638, + "flos": 28913153304960.0, + "grad_norm": 2.500390205953751, + "language_loss": 0.79818535, + "learning_rate": 3.999950849658302e-06, + "loss": 0.82249606, + "num_input_tokens_seen": 5577750, + "router_z_loss_clip": 2.43164062, + "router_z_loss_mlp": 0.43579102, + "step": 268, + "time_per_iteration": 2.928755760192871 + }, + { + "auxiliary_loss_clip": 0.01337363, + "auxiliary_loss_mlp": 0.01089819, + "balance_loss_clip": 1.09109473, + "balance_loss_mlp": 1.04494882, + "epoch": 0.03234533758191547, + "flos": 16946389739520.0, + "grad_norm": 2.083848430451074, + "language_loss": 0.84539902, + "learning_rate": 3.999945236836254e-06, + "loss": 0.86967075, + "num_input_tokens_seen": 5596715, + "router_z_loss_clip": 2.46484375, + "router_z_loss_mlp": 0.44873047, + "step": 269, + "time_per_iteration": 2.853557825088501 + }, + { + "auxiliary_loss_clip": 0.01339545, + "auxiliary_loss_mlp": 0.01100244, + "balance_loss_clip": 1.093086, + "balance_loss_mlp": 1.05358589, + "epoch": 0.03246558047255456, + "flos": 18989096284800.0, + "grad_norm": 2.700350822032456, + "language_loss": 0.94743377, + "learning_rate": 3.999939320624103e-06, + "loss": 0.97183162, + "num_input_tokens_seen": 5611865, + "router_z_loss_clip": 2.47070312, + "router_z_loss_mlp": 0.46655273, + "step": 270, + "time_per_iteration": 2.8581244945526123 + }, + { + "auxiliary_loss_clip": 0.01328112, + "auxiliary_loss_mlp": 0.01091091, + "balance_loss_clip": 1.08996797, + "balance_loss_mlp": 1.04607737, + "epoch": 0.03258582336319365, + "flos": 23728334688000.0, + "grad_norm": 2.407151221294766, + "language_loss": 0.89955539, + "learning_rate": 3.999933101022749e-06, + "loss": 0.92374742, + "num_input_tokens_seen": 5632270, + "router_z_loss_clip": 2.3828125, + "router_z_loss_mlp": 0.45043945, + "step": 271, + "time_per_iteration": 2.977768659591675 + }, + { + "auxiliary_loss_clip": 0.01328677, + "auxiliary_loss_mlp": 0.01097392, + "balance_loss_clip": 1.08961582, + "balance_loss_mlp": 1.05321312, + "epoch": 0.032706066253832745, + "flos": 27670823562240.0, + "grad_norm": 1.8665294111224398, + "language_loss": 0.87177074, + "learning_rate": 3.999926578033132e-06, + "loss": 0.8960315, + "num_input_tokens_seen": 5652085, + "router_z_loss_clip": 2.39257812, + "router_z_loss_mlp": 0.44165039, + "step": 272, + "time_per_iteration": 2.86548113822937 + }, + { + "auxiliary_loss_clip": 0.01337887, + "auxiliary_loss_mlp": 0.01101802, + "balance_loss_clip": 1.08716357, + "balance_loss_mlp": 1.05604935, + "epoch": 0.032826309144471835, + "flos": 45624685968000.0, + "grad_norm": 2.4170783312556274, + "language_loss": 0.63101351, + "learning_rate": 3.999919751656244e-06, + "loss": 0.65541041, + "num_input_tokens_seen": 5678985, + "router_z_loss_clip": 2.5078125, + "router_z_loss_mlp": 0.45751953, + "step": 273, + "time_per_iteration": 3.068521022796631 + }, + { + "auxiliary_loss_clip": 0.01330241, + "auxiliary_loss_mlp": 0.01102639, + "balance_loss_clip": 1.08814669, + "balance_loss_mlp": 1.05760217, + "epoch": 0.032946552035110925, + "flos": 25812374808960.0, + "grad_norm": 2.2238078532029046, + "language_loss": 0.75419807, + "learning_rate": 3.9999126218931195e-06, + "loss": 0.7785269, + "num_input_tokens_seen": 5697020, + "router_z_loss_clip": 2.421875, + "router_z_loss_mlp": 0.45043945, + "step": 274, + "time_per_iteration": 2.8856101036071777 + }, + { + "auxiliary_loss_clip": 0.01340141, + "auxiliary_loss_mlp": 0.0109777, + "balance_loss_clip": 1.0950098, + "balance_loss_mlp": 1.05018139, + "epoch": 0.033066794925750015, + "flos": 15121984101120.0, + "grad_norm": 2.3539968888628886, + "language_loss": 0.89642155, + "learning_rate": 3.99990518874484e-06, + "loss": 0.92080069, + "num_input_tokens_seen": 5713460, + "router_z_loss_clip": 2.453125, + "router_z_loss_mlp": 0.47583008, + "step": 275, + "time_per_iteration": 2.9140937328338623 + }, + { + "auxiliary_loss_clip": 0.01328095, + "auxiliary_loss_mlp": 0.0110501, + "balance_loss_clip": 1.09065127, + "balance_loss_mlp": 1.05925798, + "epoch": 0.033187037816389105, + "flos": 22776593973120.0, + "grad_norm": 4.142648831992689, + "language_loss": 0.92460299, + "learning_rate": 3.999897452212534e-06, + "loss": 0.94893402, + "num_input_tokens_seen": 5730790, + "router_z_loss_clip": 2.37695312, + "router_z_loss_mlp": 0.45727539, + "step": 276, + "time_per_iteration": 3.06204891204834 + }, + { + "auxiliary_loss_clip": 0.01329422, + "auxiliary_loss_mlp": 0.01110511, + "balance_loss_clip": 1.0870167, + "balance_loss_mlp": 1.06366205, + "epoch": 0.033307280707028195, + "flos": 23331414424320.0, + "grad_norm": 3.6091751800493777, + "language_loss": 1.00166357, + "learning_rate": 3.999889412297374e-06, + "loss": 1.02606285, + "num_input_tokens_seen": 5750215, + "router_z_loss_clip": 2.42578125, + "router_z_loss_mlp": 0.46875, + "step": 277, + "time_per_iteration": 4.556630611419678 + }, + { + "auxiliary_loss_clip": 0.01329478, + "auxiliary_loss_mlp": 0.0108647, + "balance_loss_clip": 1.08893728, + "balance_loss_mlp": 1.0411948, + "epoch": 0.03342752359766729, + "flos": 28840290566400.0, + "grad_norm": 2.4106605246849524, + "language_loss": 0.79293042, + "learning_rate": 3.999881069000581e-06, + "loss": 0.81708986, + "num_input_tokens_seen": 5769945, + "router_z_loss_clip": 2.40429688, + "router_z_loss_mlp": 0.45263672, + "step": 278, + "time_per_iteration": 4.724735260009766 + }, + { + "auxiliary_loss_clip": 0.01330191, + "auxiliary_loss_mlp": 0.01097599, + "balance_loss_clip": 1.08679318, + "balance_loss_mlp": 1.0512985, + "epoch": 0.03354776648830638, + "flos": 19384544090880.0, + "grad_norm": 2.6760283773415776, + "language_loss": 0.86728776, + "learning_rate": 3.99987242232342e-06, + "loss": 0.89156568, + "num_input_tokens_seen": 5784950, + "router_z_loss_clip": 2.43164062, + "router_z_loss_mlp": 0.46289062, + "step": 279, + "time_per_iteration": 2.8629140853881836 + }, + { + "auxiliary_loss_clip": 0.01328648, + "auxiliary_loss_mlp": 0.01102733, + "balance_loss_clip": 1.09097946, + "balance_loss_mlp": 1.056265, + "epoch": 0.03366800937894547, + "flos": 17858628472320.0, + "grad_norm": 3.3348205574972254, + "language_loss": 0.7982707, + "learning_rate": 3.9998634722672026e-06, + "loss": 0.82258451, + "num_input_tokens_seen": 5805005, + "router_z_loss_clip": 2.375, + "router_z_loss_mlp": 0.46459961, + "step": 280, + "time_per_iteration": 2.9843103885650635 + }, + { + "auxiliary_loss_clip": 0.01332682, + "auxiliary_loss_mlp": 0.01112566, + "balance_loss_clip": 1.09170532, + "balance_loss_mlp": 1.06783879, + "epoch": 0.03378825226958456, + "flos": 35951033635200.0, + "grad_norm": 2.1575440277457494, + "language_loss": 0.7850672, + "learning_rate": 3.999854218833286e-06, + "loss": 0.80951965, + "num_input_tokens_seen": 5825825, + "router_z_loss_clip": 2.4140625, + "router_z_loss_mlp": 0.44726562, + "step": 281, + "time_per_iteration": 2.9940943717956543 + }, + { + "auxiliary_loss_clip": 0.01326391, + "auxiliary_loss_mlp": 0.01091153, + "balance_loss_clip": 1.08893466, + "balance_loss_mlp": 1.04616344, + "epoch": 0.03390849516022365, + "flos": 25702488126720.0, + "grad_norm": 1.969074911722007, + "language_loss": 0.81809294, + "learning_rate": 3.999844662023075e-06, + "loss": 0.84226841, + "num_input_tokens_seen": 5845700, + "router_z_loss_clip": 2.37304688, + "router_z_loss_mlp": 0.45019531, + "step": 282, + "time_per_iteration": 3.0389773845672607 + }, + { + "auxiliary_loss_clip": 0.0131181, + "auxiliary_loss_mlp": 0.01100502, + "balance_loss_clip": 1.08301926, + "balance_loss_mlp": 1.05398607, + "epoch": 0.03402873805086274, + "flos": 21284505987840.0, + "grad_norm": 1.8723251303991113, + "language_loss": 0.92080319, + "learning_rate": 3.999834801838018e-06, + "loss": 0.94492632, + "num_input_tokens_seen": 5864680, + "router_z_loss_clip": 2.2890625, + "router_z_loss_mlp": 0.46459961, + "step": 283, + "time_per_iteration": 2.860685110092163 + }, + { + "auxiliary_loss_clip": 0.01319626, + "auxiliary_loss_mlp": 0.01091927, + "balance_loss_clip": 1.08800578, + "balance_loss_mlp": 1.0489403, + "epoch": 0.03414898094150183, + "flos": 22710913954560.0, + "grad_norm": 1.9490751284125245, + "language_loss": 0.73859084, + "learning_rate": 3.9998246382796115e-06, + "loss": 0.7627064, + "num_input_tokens_seen": 5884260, + "router_z_loss_clip": 2.31640625, + "router_z_loss_mlp": 0.4296875, + "step": 284, + "time_per_iteration": 2.8803513050079346 + }, + { + "auxiliary_loss_clip": 0.01329377, + "auxiliary_loss_mlp": 0.01094243, + "balance_loss_clip": 1.08750176, + "balance_loss_mlp": 1.04949188, + "epoch": 0.03426922383214093, + "flos": 18879927874560.0, + "grad_norm": 2.440847152386148, + "language_loss": 0.90915191, + "learning_rate": 3.999814171349399e-06, + "loss": 0.93338811, + "num_input_tokens_seen": 5902120, + "router_z_loss_clip": 2.41992188, + "router_z_loss_mlp": 0.44750977, + "step": 285, + "time_per_iteration": 2.8699703216552734 + }, + { + "auxiliary_loss_clip": 0.01323465, + "auxiliary_loss_mlp": 0.01112205, + "balance_loss_clip": 1.09039152, + "balance_loss_mlp": 1.06459284, + "epoch": 0.03438946672278002, + "flos": 34752012716160.0, + "grad_norm": 3.015570313321507, + "language_loss": 0.73601294, + "learning_rate": 3.9998034010489655e-06, + "loss": 0.76036966, + "num_input_tokens_seen": 5925810, + "router_z_loss_clip": 2.328125, + "router_z_loss_mlp": 0.47631836, + "step": 286, + "time_per_iteration": 3.0027830600738525 + }, + { + "auxiliary_loss_clip": 0.01325934, + "auxiliary_loss_mlp": 0.01105657, + "balance_loss_clip": 1.09117889, + "balance_loss_mlp": 1.05988038, + "epoch": 0.03450970961341911, + "flos": 22164102236160.0, + "grad_norm": 2.1306413380246925, + "language_loss": 0.76111925, + "learning_rate": 3.999792327379946e-06, + "loss": 0.78543514, + "num_input_tokens_seen": 5945185, + "router_z_loss_clip": 2.34960938, + "router_z_loss_mlp": 0.45776367, + "step": 287, + "time_per_iteration": 2.8870036602020264 + }, + { + "auxiliary_loss_clip": 0.01320519, + "auxiliary_loss_mlp": 0.01098281, + "balance_loss_clip": 1.08852828, + "balance_loss_mlp": 1.05372036, + "epoch": 0.034629952504058197, + "flos": 21725740656000.0, + "grad_norm": 2.3678388738834966, + "language_loss": 0.96358645, + "learning_rate": 3.999780950344021e-06, + "loss": 0.98777437, + "num_input_tokens_seen": 5963375, + "router_z_loss_clip": 2.3203125, + "router_z_loss_mlp": 0.44580078, + "step": 288, + "time_per_iteration": 2.8304975032806396 + }, + { + "auxiliary_loss_clip": 0.01330345, + "auxiliary_loss_mlp": 0.01099622, + "balance_loss_clip": 1.09105849, + "balance_loss_mlp": 1.0566113, + "epoch": 0.034750195394697286, + "flos": 20047994248320.0, + "grad_norm": 2.762235964168876, + "language_loss": 0.82612079, + "learning_rate": 3.999769269942916e-06, + "loss": 0.85042048, + "num_input_tokens_seen": 5983415, + "router_z_loss_clip": 2.390625, + "router_z_loss_mlp": 0.4296875, + "step": 289, + "time_per_iteration": 2.8709053993225098 + }, + { + "auxiliary_loss_clip": 0.01317199, + "auxiliary_loss_mlp": 0.01094714, + "balance_loss_clip": 1.08680034, + "balance_loss_mlp": 1.05303824, + "epoch": 0.034870438285336376, + "flos": 27965865876480.0, + "grad_norm": 2.497686467625699, + "language_loss": 0.80837363, + "learning_rate": 3.999757286178402e-06, + "loss": 0.83249277, + "num_input_tokens_seen": 6005850, + "router_z_loss_clip": 2.30273438, + "router_z_loss_mlp": 0.41674805, + "step": 290, + "time_per_iteration": 2.8392765522003174 + }, + { + "auxiliary_loss_clip": 0.01323904, + "auxiliary_loss_mlp": 0.01094397, + "balance_loss_clip": 1.0888567, + "balance_loss_mlp": 1.05048013, + "epoch": 0.03499068117597547, + "flos": 22017514832640.0, + "grad_norm": 4.213928686016807, + "language_loss": 0.90744591, + "learning_rate": 3.999744999052299e-06, + "loss": 0.93162894, + "num_input_tokens_seen": 6027240, + "router_z_loss_clip": 2.34960938, + "router_z_loss_mlp": 0.43920898, + "step": 291, + "time_per_iteration": 2.8728408813476562 + }, + { + "auxiliary_loss_clip": 0.0126258, + "auxiliary_loss_mlp": 0.01046431, + "balance_loss_clip": 1.12354636, + "balance_loss_mlp": 1.02592707, + "epoch": 0.03511092406661456, + "flos": 57242147725440.0, + "grad_norm": 0.9581905648500227, + "language_loss": 0.61167818, + "learning_rate": 3.9997324085664675e-06, + "loss": 0.63476831, + "num_input_tokens_seen": 6087470, + "router_z_loss_clip": 1.390625, + "router_z_loss_mlp": 0.20507812, + "step": 292, + "time_per_iteration": 3.321810722351074 + }, + { + "auxiliary_loss_clip": 0.01324079, + "auxiliary_loss_mlp": 0.01102159, + "balance_loss_clip": 1.0893743, + "balance_loss_mlp": 1.05786109, + "epoch": 0.03523116695725365, + "flos": 22928065626240.0, + "grad_norm": 2.5165771898269873, + "language_loss": 0.92261207, + "learning_rate": 3.999719514722821e-06, + "loss": 0.9468745, + "num_input_tokens_seen": 6107600, + "router_z_loss_clip": 2.34570312, + "router_z_loss_mlp": 0.44287109, + "step": 293, + "time_per_iteration": 2.963682174682617 + }, + { + "auxiliary_loss_clip": 0.01314415, + "auxiliary_loss_mlp": 0.01105805, + "balance_loss_clip": 1.08270168, + "balance_loss_mlp": 1.06367624, + "epoch": 0.03535140984789274, + "flos": 36903241226880.0, + "grad_norm": 2.3632438157255127, + "language_loss": 0.74769753, + "learning_rate": 3.999706317523314e-06, + "loss": 0.7718997, + "num_input_tokens_seen": 6126160, + "router_z_loss_clip": 2.31640625, + "router_z_loss_mlp": 0.42163086, + "step": 294, + "time_per_iteration": 2.969752311706543 + }, + { + "auxiliary_loss_clip": 0.01315426, + "auxiliary_loss_mlp": 0.01103779, + "balance_loss_clip": 1.08747137, + "balance_loss_mlp": 1.06253266, + "epoch": 0.03547165273853183, + "flos": 20449152316800.0, + "grad_norm": 2.1696241791498596, + "language_loss": 0.86367595, + "learning_rate": 3.999692816969948e-06, + "loss": 0.88786793, + "num_input_tokens_seen": 6145695, + "router_z_loss_clip": 2.28125, + "router_z_loss_mlp": 0.41235352, + "step": 295, + "time_per_iteration": 2.896135091781616 + }, + { + "auxiliary_loss_clip": 0.01249642, + "auxiliary_loss_mlp": 0.01022284, + "balance_loss_clip": 1.11361289, + "balance_loss_mlp": 1.00235271, + "epoch": 0.03559189562917092, + "flos": 69850564871040.0, + "grad_norm": 1.002675732800829, + "language_loss": 0.69421911, + "learning_rate": 3.999679013064772e-06, + "loss": 0.71693838, + "num_input_tokens_seen": 6212440, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.19921875, + "step": 296, + "time_per_iteration": 3.559966802597046 + }, + { + "auxiliary_loss_clip": 0.01322031, + "auxiliary_loss_mlp": 0.01093106, + "balance_loss_clip": 1.08476937, + "balance_loss_mlp": 1.04690075, + "epoch": 0.03571213851981002, + "flos": 21651944163840.0, + "grad_norm": 8.044463981340874, + "language_loss": 0.85469496, + "learning_rate": 3.99966490580988e-06, + "loss": 0.87884635, + "num_input_tokens_seen": 6229800, + "router_z_loss_clip": 2.37109375, + "router_z_loss_mlp": 0.4621582, + "step": 297, + "time_per_iteration": 3.156543254852295 + }, + { + "auxiliary_loss_clip": 0.0132347, + "auxiliary_loss_mlp": 0.01094865, + "balance_loss_clip": 1.09079528, + "balance_loss_mlp": 1.05190206, + "epoch": 0.03583238141044911, + "flos": 43945610757120.0, + "grad_norm": 2.5209684045257488, + "language_loss": 0.66068971, + "learning_rate": 3.999650495207411e-06, + "loss": 0.6848731, + "num_input_tokens_seen": 6255825, + "router_z_loss_clip": 2.32617188, + "router_z_loss_mlp": 0.4296875, + "step": 298, + "time_per_iteration": 3.169337272644043 + }, + { + "auxiliary_loss_clip": 0.01317654, + "auxiliary_loss_mlp": 0.01108579, + "balance_loss_clip": 1.09012175, + "balance_loss_mlp": 1.06683183, + "epoch": 0.0359526243010882, + "flos": 18910810592640.0, + "grad_norm": 2.555444181655102, + "language_loss": 0.90084028, + "learning_rate": 3.999635781259553e-06, + "loss": 0.92510265, + "num_input_tokens_seen": 6271090, + "router_z_loss_clip": 2.27539062, + "router_z_loss_mlp": 0.41748047, + "step": 299, + "time_per_iteration": 2.8822126388549805 + }, + { + "auxiliary_loss_clip": 0.01234471, + "auxiliary_loss_mlp": 0.01025509, + "balance_loss_clip": 1.10525835, + "balance_loss_mlp": 1.00633991, + "epoch": 0.03607286719172729, + "flos": 61668892782720.0, + "grad_norm": 0.9218215362662133, + "language_loss": 0.52329814, + "learning_rate": 3.999620763968535e-06, + "loss": 0.54589796, + "num_input_tokens_seen": 6329965, + "router_z_loss_clip": 1.296875, + "router_z_loss_mlp": 0.19140625, + "step": 300, + "time_per_iteration": 3.227180004119873 + }, + { + "auxiliary_loss_clip": 0.01311127, + "auxiliary_loss_mlp": 0.01103375, + "balance_loss_clip": 1.0868715, + "balance_loss_mlp": 1.0608654, + "epoch": 0.03619311008236638, + "flos": 27819062991360.0, + "grad_norm": 1.6849746130115335, + "language_loss": 0.86188185, + "learning_rate": 3.999605443336638e-06, + "loss": 0.88602686, + "num_input_tokens_seen": 6352095, + "router_z_loss_clip": 2.24023438, + "router_z_loss_mlp": 0.42504883, + "step": 301, + "time_per_iteration": 2.8993687629699707 + }, + { + "auxiliary_loss_clip": 0.01324346, + "auxiliary_loss_mlp": 0.01103356, + "balance_loss_clip": 1.08545899, + "balance_loss_mlp": 1.06046402, + "epoch": 0.03631335297300547, + "flos": 13621133197440.0, + "grad_norm": 2.3859953803660314, + "language_loss": 0.89508128, + "learning_rate": 3.999589819366185e-06, + "loss": 0.91935831, + "num_input_tokens_seen": 6365885, + "router_z_loss_clip": 2.38867188, + "router_z_loss_mlp": 0.42871094, + "step": 302, + "time_per_iteration": 2.817627191543579 + }, + { + "auxiliary_loss_clip": 0.01321831, + "auxiliary_loss_mlp": 0.0109322, + "balance_loss_clip": 1.08563972, + "balance_loss_mlp": 1.04837382, + "epoch": 0.036433595863644565, + "flos": 27631788456960.0, + "grad_norm": 1.9476312254483532, + "language_loss": 0.84774959, + "learning_rate": 3.999573892059547e-06, + "loss": 0.87190008, + "num_input_tokens_seen": 6385015, + "router_z_loss_clip": 2.3671875, + "router_z_loss_mlp": 0.44873047, + "step": 303, + "time_per_iteration": 2.9734606742858887 + }, + { + "auxiliary_loss_clip": 0.0133177, + "auxiliary_loss_mlp": 0.0110328, + "balance_loss_clip": 1.08767498, + "balance_loss_mlp": 1.05876744, + "epoch": 0.036553838754283655, + "flos": 24572020314240.0, + "grad_norm": 1.9563045799747303, + "language_loss": 0.8090663, + "learning_rate": 3.999557661419138e-06, + "loss": 0.83341682, + "num_input_tokens_seen": 6405165, + "router_z_loss_clip": 2.44140625, + "router_z_loss_mlp": 0.44506836, + "step": 304, + "time_per_iteration": 2.8774499893188477 + }, + { + "auxiliary_loss_clip": 0.01323575, + "auxiliary_loss_mlp": 0.01105117, + "balance_loss_clip": 1.08980381, + "balance_loss_mlp": 1.06298864, + "epoch": 0.036674081644922744, + "flos": 23404313076480.0, + "grad_norm": 2.246585061933438, + "language_loss": 0.81358004, + "learning_rate": 3.9995411274474225e-06, + "loss": 0.83786702, + "num_input_tokens_seen": 6424445, + "router_z_loss_clip": 2.3359375, + "router_z_loss_mlp": 0.42138672, + "step": 305, + "time_per_iteration": 2.8564672470092773 + }, + { + "auxiliary_loss_clip": 0.01314769, + "auxiliary_loss_mlp": 0.01109845, + "balance_loss_clip": 1.08336425, + "balance_loss_mlp": 1.0661428, + "epoch": 0.036794324535561834, + "flos": 27489690253440.0, + "grad_norm": 1.8399404981321463, + "language_loss": 0.8136189, + "learning_rate": 3.999524290146908e-06, + "loss": 0.83786505, + "num_input_tokens_seen": 6444650, + "router_z_loss_clip": 2.31640625, + "router_z_loss_mlp": 0.43701172, + "step": 306, + "time_per_iteration": 3.8126323223114014 + }, + { + "auxiliary_loss_clip": 0.0132645, + "auxiliary_loss_mlp": 0.01104037, + "balance_loss_clip": 1.08897638, + "balance_loss_mlp": 1.06052589, + "epoch": 0.036914567426200924, + "flos": 19463476227840.0, + "grad_norm": 3.046442452820517, + "language_loss": 0.92408001, + "learning_rate": 3.9995071495201485e-06, + "loss": 0.94838494, + "num_input_tokens_seen": 6461755, + "router_z_loss_clip": 2.37695312, + "router_z_loss_mlp": 0.43530273, + "step": 307, + "time_per_iteration": 3.622136354446411 + }, + { + "auxiliary_loss_clip": 0.0131768, + "auxiliary_loss_mlp": 0.01105485, + "balance_loss_clip": 1.08456254, + "balance_loss_mlp": 1.06192625, + "epoch": 0.037034810316840014, + "flos": 22309324922880.0, + "grad_norm": 2.383425912496452, + "language_loss": 0.97434807, + "learning_rate": 3.999489705569744e-06, + "loss": 0.99857974, + "num_input_tokens_seen": 6479455, + "router_z_loss_clip": 2.33203125, + "router_z_loss_mlp": 0.43579102, + "step": 308, + "time_per_iteration": 2.831721067428589 + }, + { + "auxiliary_loss_clip": 0.01315367, + "auxiliary_loss_mlp": 0.01095252, + "balance_loss_clip": 1.08381915, + "balance_loss_mlp": 1.05205011, + "epoch": 0.03715505320747911, + "flos": 18588333265920.0, + "grad_norm": 2.132981150872624, + "language_loss": 0.86471748, + "learning_rate": 3.999471958298341e-06, + "loss": 0.88882363, + "num_input_tokens_seen": 6498365, + "router_z_loss_clip": 2.3125, + "router_z_loss_mlp": 0.43212891, + "step": 309, + "time_per_iteration": 2.87056827545166 + }, + { + "auxiliary_loss_clip": 0.01328003, + "auxiliary_loss_mlp": 0.01099028, + "balance_loss_clip": 1.09034789, + "balance_loss_mlp": 1.05372882, + "epoch": 0.0372752960981182, + "flos": 35955343267200.0, + "grad_norm": 1.9352264958457837, + "language_loss": 0.75673449, + "learning_rate": 3.999453907708631e-06, + "loss": 0.78100485, + "num_input_tokens_seen": 6520770, + "router_z_loss_clip": 2.37695312, + "router_z_loss_mlp": 0.45288086, + "step": 310, + "time_per_iteration": 3.013923168182373 + }, + { + "auxiliary_loss_clip": 0.01313879, + "auxiliary_loss_mlp": 0.01087139, + "balance_loss_clip": 1.08682954, + "balance_loss_mlp": 1.04408097, + "epoch": 0.03739553898875729, + "flos": 20814040627200.0, + "grad_norm": 1.8662047996653832, + "language_loss": 0.81543374, + "learning_rate": 3.999435553803353e-06, + "loss": 0.83944386, + "num_input_tokens_seen": 6540170, + "router_z_loss_clip": 2.27148438, + "router_z_loss_mlp": 0.43066406, + "step": 311, + "time_per_iteration": 2.9051876068115234 + }, + { + "auxiliary_loss_clip": 0.01314686, + "auxiliary_loss_mlp": 0.01098207, + "balance_loss_clip": 1.08267832, + "balance_loss_mlp": 1.05707967, + "epoch": 0.03751578187939638, + "flos": 20264140339200.0, + "grad_norm": 2.503343719567116, + "language_loss": 0.83331019, + "learning_rate": 3.999416896585292e-06, + "loss": 0.85743916, + "num_input_tokens_seen": 6557200, + "router_z_loss_clip": 2.31640625, + "router_z_loss_mlp": 0.41088867, + "step": 312, + "time_per_iteration": 2.8866825103759766 + }, + { + "auxiliary_loss_clip": 0.01319223, + "auxiliary_loss_mlp": 0.01092566, + "balance_loss_clip": 1.08557773, + "balance_loss_mlp": 1.04884005, + "epoch": 0.03763602477003547, + "flos": 20668063754880.0, + "grad_norm": 3.0294582923768245, + "language_loss": 0.86266303, + "learning_rate": 3.9993979360572775e-06, + "loss": 0.88678092, + "num_input_tokens_seen": 6577340, + "router_z_loss_clip": 2.33984375, + "router_z_loss_mlp": 0.4375, + "step": 313, + "time_per_iteration": 2.9167325496673584 + }, + { + "auxiliary_loss_clip": 0.01320715, + "auxiliary_loss_mlp": 0.01102178, + "balance_loss_clip": 1.08723366, + "balance_loss_mlp": 1.06047809, + "epoch": 0.03775626766067456, + "flos": 16691352197760.0, + "grad_norm": 3.3603948555127308, + "language_loss": 0.8269037, + "learning_rate": 3.999378672222185e-06, + "loss": 0.85113263, + "num_input_tokens_seen": 6595125, + "router_z_loss_clip": 2.33789062, + "router_z_loss_mlp": 0.41699219, + "step": 314, + "time_per_iteration": 2.842595100402832 + }, + { + "auxiliary_loss_clip": 0.01316056, + "auxiliary_loss_mlp": 0.01095959, + "balance_loss_clip": 1.08383918, + "balance_loss_mlp": 1.0537827, + "epoch": 0.03787651055131366, + "flos": 21141797253120.0, + "grad_norm": 1.8962507153427404, + "language_loss": 0.82988697, + "learning_rate": 3.9993591050829385e-06, + "loss": 0.85400707, + "num_input_tokens_seen": 6612990, + "router_z_loss_clip": 2.3203125, + "router_z_loss_mlp": 0.42211914, + "step": 315, + "time_per_iteration": 2.88918137550354 + }, + { + "auxiliary_loss_clip": 0.0132376, + "auxiliary_loss_mlp": 0.01085586, + "balance_loss_clip": 1.0908289, + "balance_loss_mlp": 1.04040599, + "epoch": 0.037996753441952746, + "flos": 22018089450240.0, + "grad_norm": 1.7791957930497724, + "language_loss": 0.79239082, + "learning_rate": 3.999339234642506e-06, + "loss": 0.81648433, + "num_input_tokens_seen": 6632740, + "router_z_loss_clip": 2.33203125, + "router_z_loss_mlp": 0.45141602, + "step": 316, + "time_per_iteration": 2.933166027069092 + }, + { + "auxiliary_loss_clip": 0.01312882, + "auxiliary_loss_mlp": 0.01087772, + "balance_loss_clip": 1.08358908, + "balance_loss_mlp": 1.04669237, + "epoch": 0.038116996332591836, + "flos": 27709391790720.0, + "grad_norm": 2.2505483506780797, + "language_loss": 0.83427799, + "learning_rate": 3.9993190609038994e-06, + "loss": 0.85828459, + "num_input_tokens_seen": 6651505, + "router_z_loss_clip": 2.29492188, + "router_z_loss_mlp": 0.41088867, + "step": 317, + "time_per_iteration": 2.871861696243286 + }, + { + "auxiliary_loss_clip": 0.0131556, + "auxiliary_loss_mlp": 0.01113115, + "balance_loss_clip": 1.08281362, + "balance_loss_mlp": 1.06986582, + "epoch": 0.038237239223230926, + "flos": 21178067011200.0, + "grad_norm": 1.8999192754329606, + "language_loss": 0.8331123, + "learning_rate": 3.999298583870182e-06, + "loss": 0.85739911, + "num_input_tokens_seen": 6671090, + "router_z_loss_clip": 2.32421875, + "router_z_loss_mlp": 0.43261719, + "step": 318, + "time_per_iteration": 2.9012467861175537 + }, + { + "auxiliary_loss_clip": 0.0131243, + "auxiliary_loss_mlp": 0.01096828, + "balance_loss_clip": 1.08316636, + "balance_loss_mlp": 1.05336428, + "epoch": 0.038357482113870016, + "flos": 25556618995200.0, + "grad_norm": 1.8198035132464452, + "language_loss": 0.77472192, + "learning_rate": 3.999277803544458e-06, + "loss": 0.79881454, + "num_input_tokens_seen": 6691245, + "router_z_loss_clip": 2.29296875, + "router_z_loss_mlp": 0.43457031, + "step": 319, + "time_per_iteration": 2.8703575134277344 + }, + { + "auxiliary_loss_clip": 0.01220705, + "auxiliary_loss_mlp": 0.01017822, + "balance_loss_clip": 1.09608412, + "balance_loss_mlp": 1.00199115, + "epoch": 0.038477725004509106, + "flos": 59227578034560.0, + "grad_norm": 0.9586537682082787, + "language_loss": 0.62373722, + "learning_rate": 3.999256719929882e-06, + "loss": 0.64612246, + "num_input_tokens_seen": 6752520, + "router_z_loss_clip": 1.25, + "router_z_loss_mlp": 0.15820312, + "step": 320, + "time_per_iteration": 3.352203369140625 + }, + { + "auxiliary_loss_clip": 0.01216385, + "auxiliary_loss_mlp": 0.01016454, + "balance_loss_clip": 1.09279037, + "balance_loss_mlp": 1.00109982, + "epoch": 0.0385979678951482, + "flos": 67317676398720.0, + "grad_norm": 1.2219387716244619, + "language_loss": 0.67144036, + "learning_rate": 3.999235333029651e-06, + "loss": 0.69376874, + "num_input_tokens_seen": 6806460, + "router_z_loss_clip": 1.234375, + "router_z_loss_mlp": 0.15332031, + "step": 321, + "time_per_iteration": 3.308277130126953 + }, + { + "auxiliary_loss_clip": 0.01313576, + "auxiliary_loss_mlp": 0.01091262, + "balance_loss_clip": 1.08576846, + "balance_loss_mlp": 1.04858541, + "epoch": 0.03871821078578729, + "flos": 22746752749440.0, + "grad_norm": 1.9665813231198905, + "language_loss": 0.82030737, + "learning_rate": 3.999213642847009e-06, + "loss": 0.84435576, + "num_input_tokens_seen": 6827045, + "router_z_loss_clip": 2.27734375, + "router_z_loss_mlp": 0.42700195, + "step": 322, + "time_per_iteration": 3.0097124576568604 + }, + { + "auxiliary_loss_clip": 0.01311066, + "auxiliary_loss_mlp": 0.01100333, + "balance_loss_clip": 1.08322215, + "balance_loss_mlp": 1.05832326, + "epoch": 0.03883845367642638, + "flos": 26280613526400.0, + "grad_norm": 1.7398890356851866, + "language_loss": 0.9101547, + "learning_rate": 3.999191649385247e-06, + "loss": 0.93426871, + "num_input_tokens_seen": 6848220, + "router_z_loss_clip": 2.27929688, + "router_z_loss_mlp": 0.42016602, + "step": 323, + "time_per_iteration": 2.907682418823242 + }, + { + "auxiliary_loss_clip": 0.01201719, + "auxiliary_loss_mlp": 0.01020049, + "balance_loss_clip": 1.07962763, + "balance_loss_mlp": 1.00536287, + "epoch": 0.03895869656706547, + "flos": 56962835568000.0, + "grad_norm": 0.9388555233751852, + "language_loss": 0.59794438, + "learning_rate": 3.999169352647702e-06, + "loss": 0.62016201, + "num_input_tokens_seen": 6909400, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.14648438, + "step": 324, + "time_per_iteration": 3.2652814388275146 + }, + { + "auxiliary_loss_clip": 0.01321804, + "auxiliary_loss_mlp": 0.01103057, + "balance_loss_clip": 1.0886749, + "balance_loss_mlp": 1.05809093, + "epoch": 0.03907893945770456, + "flos": 24863363527680.0, + "grad_norm": 1.9615025944147213, + "language_loss": 0.83097583, + "learning_rate": 3.999146752637755e-06, + "loss": 0.85522443, + "num_input_tokens_seen": 6930445, + "router_z_loss_clip": 2.33007812, + "router_z_loss_mlp": 0.44970703, + "step": 325, + "time_per_iteration": 2.875084161758423 + }, + { + "auxiliary_loss_clip": 0.0130948, + "auxiliary_loss_mlp": 0.01092859, + "balance_loss_clip": 1.08353639, + "balance_loss_mlp": 1.05027699, + "epoch": 0.03919918234834365, + "flos": 18368595815040.0, + "grad_norm": 2.221482633976766, + "language_loss": 0.89611459, + "learning_rate": 3.999123849358836e-06, + "loss": 0.920138, + "num_input_tokens_seen": 6948110, + "router_z_loss_clip": 2.26171875, + "router_z_loss_mlp": 0.42602539, + "step": 326, + "time_per_iteration": 2.980517625808716 + }, + { + "auxiliary_loss_clip": 0.01313057, + "auxiliary_loss_mlp": 0.01104641, + "balance_loss_clip": 1.08415067, + "balance_loss_mlp": 1.06251287, + "epoch": 0.03931942523898275, + "flos": 25225414663680.0, + "grad_norm": 2.94317361355456, + "language_loss": 0.74700546, + "learning_rate": 3.999100642814418e-06, + "loss": 0.77118242, + "num_input_tokens_seen": 6968550, + "router_z_loss_clip": 2.29101562, + "router_z_loss_mlp": 0.42138672, + "step": 327, + "time_per_iteration": 2.8611202239990234 + }, + { + "auxiliary_loss_clip": 0.01314991, + "auxiliary_loss_mlp": 0.01092396, + "balance_loss_clip": 1.08601284, + "balance_loss_mlp": 1.05134034, + "epoch": 0.03943966812962184, + "flos": 23257905240960.0, + "grad_norm": 2.74424933540695, + "language_loss": 0.88567346, + "learning_rate": 3.999077133008022e-06, + "loss": 0.90974736, + "num_input_tokens_seen": 6987135, + "router_z_loss_clip": 2.28515625, + "router_z_loss_mlp": 0.41064453, + "step": 328, + "time_per_iteration": 2.8264997005462646 + }, + { + "auxiliary_loss_clip": 0.01316518, + "auxiliary_loss_mlp": 0.01088075, + "balance_loss_clip": 1.08803451, + "balance_loss_mlp": 1.04701936, + "epoch": 0.03955991102026093, + "flos": 29168837291520.0, + "grad_norm": 2.6215837364598116, + "language_loss": 0.90468132, + "learning_rate": 3.9990533199432145e-06, + "loss": 0.92872727, + "num_input_tokens_seen": 7008630, + "router_z_loss_clip": 2.28710938, + "router_z_loss_mlp": 0.41015625, + "step": 329, + "time_per_iteration": 2.8674237728118896 + }, + { + "auxiliary_loss_clip": 0.01313138, + "auxiliary_loss_mlp": 0.01103344, + "balance_loss_clip": 1.082304, + "balance_loss_mlp": 1.06259787, + "epoch": 0.03968015391090002, + "flos": 17602441695360.0, + "grad_norm": 2.3794682662446793, + "language_loss": 0.75779647, + "learning_rate": 3.999029203623608e-06, + "loss": 0.78196126, + "num_input_tokens_seen": 7026350, + "router_z_loss_clip": 2.30859375, + "router_z_loss_mlp": 0.4074707, + "step": 330, + "time_per_iteration": 2.8144466876983643 + }, + { + "auxiliary_loss_clip": 0.01312929, + "auxiliary_loss_mlp": 0.01098287, + "balance_loss_clip": 1.08751035, + "balance_loss_mlp": 1.05663478, + "epoch": 0.03980039680153911, + "flos": 21799285752960.0, + "grad_norm": 1.9838702498998508, + "language_loss": 0.86976939, + "learning_rate": 3.99900478405286e-06, + "loss": 0.89388156, + "num_input_tokens_seen": 7045660, + "router_z_loss_clip": 2.25390625, + "router_z_loss_mlp": 0.41650391, + "step": 331, + "time_per_iteration": 2.8172152042388916 + }, + { + "auxiliary_loss_clip": 0.01313637, + "auxiliary_loss_mlp": 0.01113392, + "balance_loss_clip": 1.08924937, + "balance_loss_mlp": 1.07121575, + "epoch": 0.0399206396921782, + "flos": 15195134148480.0, + "grad_norm": 2.651930655102787, + "language_loss": 0.8262893, + "learning_rate": 3.998980061234676e-06, + "loss": 0.85055959, + "num_input_tokens_seen": 7063575, + "router_z_loss_clip": 2.24414062, + "router_z_loss_mlp": 0.42211914, + "step": 332, + "time_per_iteration": 2.875100612640381 + }, + { + "auxiliary_loss_clip": 0.01314599, + "auxiliary_loss_mlp": 0.01084367, + "balance_loss_clip": 1.08592939, + "balance_loss_mlp": 1.04445565, + "epoch": 0.040040882582817294, + "flos": 14422910630400.0, + "grad_norm": 2.2482130796721274, + "language_loss": 0.75791013, + "learning_rate": 3.9989550351728055e-06, + "loss": 0.78189969, + "num_input_tokens_seen": 7080505, + "router_z_loss_clip": 2.28710938, + "router_z_loss_mlp": 0.39916992, + "step": 333, + "time_per_iteration": 2.8926684856414795 + }, + { + "auxiliary_loss_clip": 0.0130693, + "auxiliary_loss_mlp": 0.01086109, + "balance_loss_clip": 1.08394694, + "balance_loss_mlp": 1.04688907, + "epoch": 0.040161125473456384, + "flos": 19280906375040.0, + "grad_norm": 3.057588502451204, + "language_loss": 0.84552199, + "learning_rate": 3.998929705871046e-06, + "loss": 0.86945242, + "num_input_tokens_seen": 7097860, + "router_z_loss_clip": 2.23046875, + "router_z_loss_mlp": 0.39233398, + "step": 334, + "time_per_iteration": 2.965385913848877 + }, + { + "auxiliary_loss_clip": 0.01309895, + "auxiliary_loss_mlp": 0.01085257, + "balance_loss_clip": 1.08634305, + "balance_loss_mlp": 1.04217458, + "epoch": 0.040281368364095474, + "flos": 17821101738240.0, + "grad_norm": 2.585887434574877, + "language_loss": 0.89274913, + "learning_rate": 3.99890407333324e-06, + "loss": 0.9167006, + "num_input_tokens_seen": 7116390, + "router_z_loss_clip": 2.23242188, + "router_z_loss_mlp": 0.43066406, + "step": 335, + "time_per_iteration": 4.538498878479004 + }, + { + "auxiliary_loss_clip": 0.013057, + "auxiliary_loss_mlp": 0.01093383, + "balance_loss_clip": 1.08170152, + "balance_loss_mlp": 1.05280411, + "epoch": 0.040401611254734564, + "flos": 19573757959680.0, + "grad_norm": 1.7228288346503635, + "language_loss": 0.87171131, + "learning_rate": 3.998878137563275e-06, + "loss": 0.89570218, + "num_input_tokens_seen": 7135940, + "router_z_loss_clip": 2.23828125, + "router_z_loss_mlp": 0.40576172, + "step": 336, + "time_per_iteration": 2.8407392501831055 + }, + { + "auxiliary_loss_clip": 0.01311204, + "auxiliary_loss_mlp": 0.010836, + "balance_loss_clip": 1.08236575, + "balance_loss_mlp": 1.04328346, + "epoch": 0.040521854145373654, + "flos": 22054466949120.0, + "grad_norm": 2.1371979807352144, + "language_loss": 0.8495748, + "learning_rate": 3.998851898565085e-06, + "loss": 0.87352282, + "num_input_tokens_seen": 7155745, + "router_z_loss_clip": 2.28515625, + "router_z_loss_mlp": 0.40307617, + "step": 337, + "time_per_iteration": 2.888571262359619 + }, + { + "auxiliary_loss_clip": 0.01312087, + "auxiliary_loss_mlp": 0.01090707, + "balance_loss_clip": 1.08781886, + "balance_loss_mlp": 1.05041456, + "epoch": 0.04064209703601274, + "flos": 22674644196480.0, + "grad_norm": 1.978838812622485, + "language_loss": 0.82912797, + "learning_rate": 3.998825356342653e-06, + "loss": 0.85315597, + "num_input_tokens_seen": 7175920, + "router_z_loss_clip": 2.2421875, + "router_z_loss_mlp": 0.40283203, + "step": 338, + "time_per_iteration": 2.8379054069519043 + }, + { + "auxiliary_loss_clip": 0.01314565, + "auxiliary_loss_mlp": 0.01093786, + "balance_loss_clip": 1.08380818, + "balance_loss_mlp": 1.05211043, + "epoch": 0.04076233992665183, + "flos": 38582172783360.0, + "grad_norm": 2.271642583811842, + "language_loss": 0.73428804, + "learning_rate": 3.998798510900003e-06, + "loss": 0.75837153, + "num_input_tokens_seen": 7198720, + "router_z_loss_clip": 2.3125, + "router_z_loss_mlp": 0.41674805, + "step": 339, + "time_per_iteration": 3.016974449157715 + }, + { + "auxiliary_loss_clip": 0.01307328, + "auxiliary_loss_mlp": 0.01103454, + "balance_loss_clip": 1.08377957, + "balance_loss_mlp": 1.06339979, + "epoch": 0.04088258281729093, + "flos": 25885309374720.0, + "grad_norm": 2.727215518522093, + "language_loss": 0.83820599, + "learning_rate": 3.998771362241207e-06, + "loss": 0.86231387, + "num_input_tokens_seen": 7219125, + "router_z_loss_clip": 2.23632812, + "router_z_loss_mlp": 0.40039062, + "step": 340, + "time_per_iteration": 2.9057486057281494 + }, + { + "auxiliary_loss_clip": 0.0130185, + "auxiliary_loss_mlp": 0.01091547, + "balance_loss_clip": 1.08315325, + "balance_loss_mlp": 1.05149221, + "epoch": 0.04100282570793002, + "flos": 19789832223360.0, + "grad_norm": 1.745298253398918, + "language_loss": 0.87752068, + "learning_rate": 3.998743910370385e-06, + "loss": 0.90145469, + "num_input_tokens_seen": 7237985, + "router_z_loss_clip": 2.18945312, + "router_z_loss_mlp": 0.40039062, + "step": 341, + "time_per_iteration": 2.806321144104004 + }, + { + "auxiliary_loss_clip": 0.01310438, + "auxiliary_loss_mlp": 0.01101702, + "balance_loss_clip": 1.08851588, + "balance_loss_mlp": 1.06198108, + "epoch": 0.04112306859856911, + "flos": 22565152563840.0, + "grad_norm": 2.097520627307105, + "language_loss": 0.73260701, + "learning_rate": 3.998716155291702e-06, + "loss": 0.75672847, + "num_input_tokens_seen": 7255825, + "router_z_loss_clip": 2.21875, + "router_z_loss_mlp": 0.39697266, + "step": 342, + "time_per_iteration": 2.8437798023223877 + }, + { + "auxiliary_loss_clip": 0.01309837, + "auxiliary_loss_mlp": 0.01086182, + "balance_loss_clip": 1.08900726, + "balance_loss_mlp": 1.04426801, + "epoch": 0.0412433114892082, + "flos": 25040654081280.0, + "grad_norm": 1.677993342444368, + "language_loss": 0.90476716, + "learning_rate": 3.998688097009366e-06, + "loss": 0.92872733, + "num_input_tokens_seen": 7276590, + "router_z_loss_clip": 2.20996094, + "router_z_loss_mlp": 0.41918945, + "step": 343, + "time_per_iteration": 2.826319456100464 + }, + { + "auxiliary_loss_clip": 0.01305695, + "auxiliary_loss_mlp": 0.01088814, + "balance_loss_clip": 1.08602524, + "balance_loss_mlp": 1.04918861, + "epoch": 0.04136355437984729, + "flos": 25191371548800.0, + "grad_norm": 2.2181283978419493, + "language_loss": 0.80164087, + "learning_rate": 3.998659735527636e-06, + "loss": 0.82558596, + "num_input_tokens_seen": 7295680, + "router_z_loss_clip": 2.1953125, + "router_z_loss_mlp": 0.39599609, + "step": 344, + "time_per_iteration": 2.8685333728790283 + }, + { + "auxiliary_loss_clip": 0.01299465, + "auxiliary_loss_mlp": 0.01086842, + "balance_loss_clip": 1.08169341, + "balance_loss_mlp": 1.04466605, + "epoch": 0.04148379727048638, + "flos": 22966777509120.0, + "grad_norm": 1.7997381520173013, + "language_loss": 0.77858615, + "learning_rate": 3.998631070850813e-06, + "loss": 0.80244923, + "num_input_tokens_seen": 7316300, + "router_z_loss_clip": 2.1796875, + "router_z_loss_mlp": 0.421875, + "step": 345, + "time_per_iteration": 2.9941601753234863 + }, + { + "auxiliary_loss_clip": 0.01307119, + "auxiliary_loss_mlp": 0.01095277, + "balance_loss_clip": 1.08732295, + "balance_loss_mlp": 1.0540787, + "epoch": 0.041604040161125476, + "flos": 14063481187200.0, + "grad_norm": 2.892636541367324, + "language_loss": 0.83506626, + "learning_rate": 3.9986021029832455e-06, + "loss": 0.85909021, + "num_input_tokens_seen": 7333615, + "router_z_loss_clip": 2.19726562, + "router_z_loss_mlp": 0.41162109, + "step": 346, + "time_per_iteration": 2.850679636001587 + }, + { + "auxiliary_loss_clip": 0.01305719, + "auxiliary_loss_mlp": 0.01087566, + "balance_loss_clip": 1.0817349, + "balance_loss_mlp": 1.0466063, + "epoch": 0.041724283051764566, + "flos": 12091877614080.0, + "grad_norm": 2.5684521830821754, + "language_loss": 0.91350257, + "learning_rate": 3.9985728319293285e-06, + "loss": 0.93743545, + "num_input_tokens_seen": 7347590, + "router_z_loss_clip": 2.2421875, + "router_z_loss_mlp": 0.40966797, + "step": 347, + "time_per_iteration": 2.7874958515167236 + }, + { + "auxiliary_loss_clip": 0.01312723, + "auxiliary_loss_mlp": 0.01096747, + "balance_loss_clip": 1.0845288, + "balance_loss_mlp": 1.05504799, + "epoch": 0.041844525942403656, + "flos": 12385303816320.0, + "grad_norm": 2.6621134639108175, + "language_loss": 0.85265386, + "learning_rate": 3.998543257693501e-06, + "loss": 0.8767485, + "num_input_tokens_seen": 7364345, + "router_z_loss_clip": 2.28125, + "router_z_loss_mlp": 0.41723633, + "step": 348, + "time_per_iteration": 2.8784139156341553 + }, + { + "auxiliary_loss_clip": 0.01305716, + "auxiliary_loss_mlp": 0.01092874, + "balance_loss_clip": 1.08752131, + "balance_loss_mlp": 1.05515575, + "epoch": 0.041964768833042745, + "flos": 23769345041280.0, + "grad_norm": 2.172157550796351, + "language_loss": 0.87872231, + "learning_rate": 3.998513380280251e-06, + "loss": 0.90270817, + "num_input_tokens_seen": 7384625, + "router_z_loss_clip": 2.18554688, + "router_z_loss_mlp": 0.37695312, + "step": 349, + "time_per_iteration": 2.9222412109375 + }, + { + "auxiliary_loss_clip": 0.01308354, + "auxiliary_loss_mlp": 0.01102273, + "balance_loss_clip": 1.08274722, + "balance_loss_mlp": 1.0610745, + "epoch": 0.042085011723681835, + "flos": 11875336473600.0, + "grad_norm": 2.808301070004309, + "language_loss": 0.95122558, + "learning_rate": 3.99848319969411e-06, + "loss": 0.9753319, + "num_input_tokens_seen": 7402225, + "router_z_loss_clip": 2.25585938, + "router_z_loss_mlp": 0.41186523, + "step": 350, + "time_per_iteration": 2.8237998485565186 + }, + { + "auxiliary_loss_clip": 0.01314671, + "auxiliary_loss_mlp": 0.01106295, + "balance_loss_clip": 1.08911574, + "balance_loss_mlp": 1.06712317, + "epoch": 0.042205254614320925, + "flos": 16873957964160.0, + "grad_norm": 2.0606685206781132, + "language_loss": 0.79636657, + "learning_rate": 3.9984527159396564e-06, + "loss": 0.82057625, + "num_input_tokens_seen": 7420865, + "router_z_loss_clip": 2.25585938, + "router_z_loss_mlp": 0.39160156, + "step": 351, + "time_per_iteration": 2.8394813537597656 + }, + { + "auxiliary_loss_clip": 0.01298547, + "auxiliary_loss_mlp": 0.0107922, + "balance_loss_clip": 1.07967448, + "balance_loss_mlp": 1.04035759, + "epoch": 0.04232549750496002, + "flos": 25118508810240.0, + "grad_norm": 2.3795458254495614, + "language_loss": 0.84546268, + "learning_rate": 3.9984219290215154e-06, + "loss": 0.86924028, + "num_input_tokens_seen": 7441040, + "router_z_loss_clip": 2.1875, + "router_z_loss_mlp": 0.38891602, + "step": 352, + "time_per_iteration": 3.005821704864502 + }, + { + "auxiliary_loss_clip": 0.01303397, + "auxiliary_loss_mlp": 0.01101096, + "balance_loss_clip": 1.08820796, + "balance_loss_mlp": 1.06206727, + "epoch": 0.04244574039559911, + "flos": 26724541714560.0, + "grad_norm": 1.964129973626022, + "language_loss": 0.89065516, + "learning_rate": 3.998390838944356e-06, + "loss": 0.91470015, + "num_input_tokens_seen": 7462545, + "router_z_loss_clip": 2.15039062, + "router_z_loss_mlp": 0.39038086, + "step": 353, + "time_per_iteration": 2.9011995792388916 + }, + { + "auxiliary_loss_clip": 0.0130589, + "auxiliary_loss_mlp": 0.01108932, + "balance_loss_clip": 1.08589816, + "balance_loss_mlp": 1.06882954, + "epoch": 0.0425659832862382, + "flos": 20923244951040.0, + "grad_norm": 2.6010461950043395, + "language_loss": 0.9013862, + "learning_rate": 3.998359445712895e-06, + "loss": 0.92553443, + "num_input_tokens_seen": 7481650, + "router_z_loss_clip": 2.203125, + "router_z_loss_mlp": 0.40087891, + "step": 354, + "time_per_iteration": 2.8913321495056152 + }, + { + "auxiliary_loss_clip": 0.01309685, + "auxiliary_loss_mlp": 0.01079986, + "balance_loss_clip": 1.08317971, + "balance_loss_mlp": 1.04186261, + "epoch": 0.04268622617687729, + "flos": 23331127115520.0, + "grad_norm": 2.3957078411325714, + "language_loss": 0.81095773, + "learning_rate": 3.9983277493318955e-06, + "loss": 0.83485448, + "num_input_tokens_seen": 7500945, + "router_z_loss_clip": 2.26367188, + "router_z_loss_mlp": 0.38134766, + "step": 355, + "time_per_iteration": 2.942420721054077 + }, + { + "auxiliary_loss_clip": 0.01305859, + "auxiliary_loss_mlp": 0.01096107, + "balance_loss_clip": 1.08555889, + "balance_loss_mlp": 1.05617166, + "epoch": 0.04280646906751638, + "flos": 25994010908160.0, + "grad_norm": 1.6453292289717945, + "language_loss": 0.81136239, + "learning_rate": 3.998295749806165e-06, + "loss": 0.83538204, + "num_input_tokens_seen": 7522170, + "router_z_loss_clip": 2.20703125, + "router_z_loss_mlp": 0.39941406, + "step": 356, + "time_per_iteration": 2.9436182975769043 + }, + { + "auxiliary_loss_clip": 0.01304384, + "auxiliary_loss_mlp": 0.01097271, + "balance_loss_clip": 1.08300304, + "balance_loss_mlp": 1.0574075, + "epoch": 0.04292671195815547, + "flos": 26906824258560.0, + "grad_norm": 1.8644331671266194, + "language_loss": 0.83596796, + "learning_rate": 3.998263447140558e-06, + "loss": 0.85998452, + "num_input_tokens_seen": 7542370, + "router_z_loss_clip": 2.21289062, + "router_z_loss_mlp": 0.3984375, + "step": 357, + "time_per_iteration": 2.9441115856170654 + }, + { + "auxiliary_loss_clip": 0.01304297, + "auxiliary_loss_mlp": 0.01075739, + "balance_loss_clip": 1.07990491, + "balance_loss_mlp": 1.03759193, + "epoch": 0.04304695484879457, + "flos": 39457315745280.0, + "grad_norm": 2.3343896269628015, + "language_loss": 0.81982839, + "learning_rate": 3.998230841339976e-06, + "loss": 0.8436287, + "num_input_tokens_seen": 7564380, + "router_z_loss_clip": 2.2421875, + "router_z_loss_mlp": 0.3815918, + "step": 358, + "time_per_iteration": 3.022880792617798 + }, + { + "auxiliary_loss_clip": 0.01303441, + "auxiliary_loss_mlp": 0.01096868, + "balance_loss_clip": 1.08381689, + "balance_loss_mlp": 1.05757666, + "epoch": 0.04316719773943366, + "flos": 19646297475840.0, + "grad_norm": 2.3563517789921313, + "language_loss": 0.84831893, + "learning_rate": 3.998197932409363e-06, + "loss": 0.87232202, + "num_input_tokens_seen": 7582390, + "router_z_loss_clip": 2.19335938, + "router_z_loss_mlp": 0.39282227, + "step": 359, + "time_per_iteration": 2.929996967315674 + }, + { + "auxiliary_loss_clip": 0.01296331, + "auxiliary_loss_mlp": 0.01084973, + "balance_loss_clip": 1.0833993, + "balance_loss_mlp": 1.0462544, + "epoch": 0.04328744063007275, + "flos": 22452320966400.0, + "grad_norm": 2.259128270612277, + "language_loss": 0.86205316, + "learning_rate": 3.9981647203537125e-06, + "loss": 0.88586617, + "num_input_tokens_seen": 7599890, + "router_z_loss_clip": 2.12890625, + "router_z_loss_mlp": 0.38696289, + "step": 360, + "time_per_iteration": 2.7970376014709473 + }, + { + "auxiliary_loss_clip": 0.01304619, + "auxiliary_loss_mlp": 0.0110107, + "balance_loss_clip": 1.0839839, + "balance_loss_mlp": 1.06218433, + "epoch": 0.04340768352071184, + "flos": 21283033530240.0, + "grad_norm": 1.990565797891639, + "language_loss": 0.95854628, + "learning_rate": 3.998131205178063e-06, + "loss": 0.98260313, + "num_input_tokens_seen": 7618360, + "router_z_loss_clip": 2.20703125, + "router_z_loss_mlp": 0.38891602, + "step": 361, + "time_per_iteration": 2.808269500732422 + }, + { + "auxiliary_loss_clip": 0.01313305, + "auxiliary_loss_mlp": 0.0110508, + "balance_loss_clip": 1.08716106, + "balance_loss_mlp": 1.06354785, + "epoch": 0.04352792641135093, + "flos": 11583705951360.0, + "grad_norm": 3.1039465097671983, + "language_loss": 0.76591206, + "learning_rate": 3.998097386887498e-06, + "loss": 0.79009593, + "num_input_tokens_seen": 7635435, + "router_z_loss_clip": 2.25976562, + "router_z_loss_mlp": 0.4152832, + "step": 362, + "time_per_iteration": 2.8076870441436768 + }, + { + "auxiliary_loss_clip": 0.01296853, + "auxiliary_loss_mlp": 0.01094833, + "balance_loss_clip": 1.08540392, + "balance_loss_mlp": 1.05327702, + "epoch": 0.04364816930199002, + "flos": 23623547736960.0, + "grad_norm": 1.6676648949391424, + "language_loss": 0.85121268, + "learning_rate": 3.998063265487148e-06, + "loss": 0.87512958, + "num_input_tokens_seen": 7656485, + "router_z_loss_clip": 2.11328125, + "router_z_loss_mlp": 0.41577148, + "step": 363, + "time_per_iteration": 2.8422088623046875 + }, + { + "auxiliary_loss_clip": 0.01301147, + "auxiliary_loss_mlp": 0.01091275, + "balance_loss_clip": 1.08351374, + "balance_loss_mlp": 1.05155444, + "epoch": 0.043768412192629114, + "flos": 14429734214400.0, + "grad_norm": 2.8987001529556253, + "language_loss": 0.80942631, + "learning_rate": 3.99802884098219e-06, + "loss": 0.83335054, + "num_input_tokens_seen": 7674595, + "router_z_loss_clip": 2.17578125, + "router_z_loss_mlp": 0.39746094, + "step": 364, + "time_per_iteration": 3.731372594833374 + }, + { + "auxiliary_loss_clip": 0.01301218, + "auxiliary_loss_mlp": 0.01092297, + "balance_loss_clip": 1.08239436, + "balance_loss_mlp": 1.05250478, + "epoch": 0.043888655083268203, + "flos": 26468893641600.0, + "grad_norm": 2.7272793791047136, + "language_loss": 0.82376182, + "learning_rate": 3.997994113377845e-06, + "loss": 0.8476969, + "num_input_tokens_seen": 7693495, + "router_z_loss_clip": 2.1875, + "router_z_loss_mlp": 0.39819336, + "step": 365, + "time_per_iteration": 2.9016103744506836 + }, + { + "auxiliary_loss_clip": 0.01300947, + "auxiliary_loss_mlp": 0.01089929, + "balance_loss_clip": 1.08055997, + "balance_loss_mlp": 1.04777718, + "epoch": 0.04400889797390729, + "flos": 27235263242880.0, + "grad_norm": 4.597190102397278, + "language_loss": 0.83403665, + "learning_rate": 3.9979590826793815e-06, + "loss": 0.85794544, + "num_input_tokens_seen": 7714685, + "router_z_loss_clip": 2.20117188, + "router_z_loss_mlp": 0.42163086, + "step": 366, + "time_per_iteration": 3.010021448135376 + }, + { + "auxiliary_loss_clip": 0.01311795, + "auxiliary_loss_mlp": 0.01090996, + "balance_loss_clip": 1.08519363, + "balance_loss_mlp": 1.04803324, + "epoch": 0.04412914086454638, + "flos": 20119528183680.0, + "grad_norm": 2.5814968755736034, + "language_loss": 0.81133628, + "learning_rate": 3.997923748892113e-06, + "loss": 0.83536416, + "num_input_tokens_seen": 7734005, + "router_z_loss_clip": 2.26757812, + "router_z_loss_mlp": 0.42944336, + "step": 367, + "time_per_iteration": 2.828580617904663 + }, + { + "auxiliary_loss_clip": 0.01300793, + "auxiliary_loss_mlp": 0.01107918, + "balance_loss_clip": 1.08516765, + "balance_loss_mlp": 1.06872237, + "epoch": 0.04424938375518547, + "flos": 22604618632320.0, + "grad_norm": 1.8834749210057378, + "language_loss": 0.88738942, + "learning_rate": 3.9978881120214015e-06, + "loss": 0.91147661, + "num_input_tokens_seen": 7755525, + "router_z_loss_clip": 2.15429688, + "router_z_loss_mlp": 0.3918457, + "step": 368, + "time_per_iteration": 2.860997438430786 + }, + { + "auxiliary_loss_clip": 0.0129642, + "auxiliary_loss_mlp": 0.01090846, + "balance_loss_clip": 1.08082831, + "balance_loss_mlp": 1.05348551, + "epoch": 0.04436962664582456, + "flos": 24132365844480.0, + "grad_norm": 1.8393971245750462, + "language_loss": 0.7928248, + "learning_rate": 3.997852172072652e-06, + "loss": 0.81669748, + "num_input_tokens_seen": 7776740, + "router_z_loss_clip": 2.15234375, + "router_z_loss_mlp": 0.37353516, + "step": 369, + "time_per_iteration": 2.945208787918091 + }, + { + "auxiliary_loss_clip": 0.01301343, + "auxiliary_loss_mlp": 0.01093537, + "balance_loss_clip": 1.08367193, + "balance_loss_mlp": 1.05260074, + "epoch": 0.04448986953646366, + "flos": 18222906251520.0, + "grad_norm": 2.3241323291676745, + "language_loss": 0.89258081, + "learning_rate": 3.9978159290513155e-06, + "loss": 0.9165296, + "num_input_tokens_seen": 7794820, + "router_z_loss_clip": 2.17578125, + "router_z_loss_mlp": 0.40942383, + "step": 370, + "time_per_iteration": 3.0000433921813965 + }, + { + "auxiliary_loss_clip": 0.01307402, + "auxiliary_loss_mlp": 0.01092197, + "balance_loss_clip": 1.08394384, + "balance_loss_mlp": 1.05307221, + "epoch": 0.04461011242710275, + "flos": 30117920400000.0, + "grad_norm": 1.739930672233972, + "language_loss": 0.80273873, + "learning_rate": 3.997779382962892e-06, + "loss": 0.82673466, + "num_input_tokens_seen": 7817705, + "router_z_loss_clip": 2.23242188, + "router_z_loss_mlp": 0.39111328, + "step": 371, + "time_per_iteration": 2.908229112625122 + }, + { + "auxiliary_loss_clip": 0.01296163, + "auxiliary_loss_mlp": 0.01100224, + "balance_loss_clip": 1.08304095, + "balance_loss_mlp": 1.05931163, + "epoch": 0.04473035531774184, + "flos": 29752529299200.0, + "grad_norm": 2.060632569383266, + "language_loss": 0.73490655, + "learning_rate": 3.997742533812924e-06, + "loss": 0.75887048, + "num_input_tokens_seen": 7840970, + "router_z_loss_clip": 2.13085938, + "router_z_loss_mlp": 0.40917969, + "step": 372, + "time_per_iteration": 2.9047181606292725 + }, + { + "auxiliary_loss_clip": 0.01298267, + "auxiliary_loss_mlp": 0.01097047, + "balance_loss_clip": 1.0828166, + "balance_loss_mlp": 1.05532432, + "epoch": 0.04485059820838093, + "flos": 13151565676800.0, + "grad_norm": 2.5580122227523576, + "language_loss": 0.92267752, + "learning_rate": 3.997705381607001e-06, + "loss": 0.94663072, + "num_input_tokens_seen": 7857785, + "router_z_loss_clip": 2.15429688, + "router_z_loss_mlp": 0.41699219, + "step": 373, + "time_per_iteration": 2.9285027980804443 + }, + { + "auxiliary_loss_clip": 0.01179246, + "auxiliary_loss_mlp": 0.01016609, + "balance_loss_clip": 1.06734443, + "balance_loss_mlp": 1.00511682, + "epoch": 0.04497084109902002, + "flos": 68094209548800.0, + "grad_norm": 1.007865382882687, + "language_loss": 0.60320348, + "learning_rate": 3.997667926350761e-06, + "loss": 0.62516207, + "num_input_tokens_seen": 7916115, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.11474609, + "step": 374, + "time_per_iteration": 3.2754979133605957 + }, + { + "auxiliary_loss_clip": 0.01178973, + "auxiliary_loss_mlp": 0.01015929, + "balance_loss_clip": 1.06661749, + "balance_loss_mlp": 1.00477123, + "epoch": 0.04509108398965911, + "flos": 64342263346560.0, + "grad_norm": 0.9093432679263538, + "language_loss": 0.57734871, + "learning_rate": 3.997630168049886e-06, + "loss": 0.59929776, + "num_input_tokens_seen": 7974480, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.11181641, + "step": 375, + "time_per_iteration": 3.3481485843658447 + }, + { + "auxiliary_loss_clip": 0.01308776, + "auxiliary_loss_mlp": 0.01094368, + "balance_loss_clip": 1.08956587, + "balance_loss_mlp": 1.05393243, + "epoch": 0.045211326880298205, + "flos": 22271115830400.0, + "grad_norm": 1.9873624332653075, + "language_loss": 0.77266347, + "learning_rate": 3.997592106710101e-06, + "loss": 0.79669487, + "num_input_tokens_seen": 7993940, + "router_z_loss_clip": 2.19335938, + "router_z_loss_mlp": 0.40429688, + "step": 376, + "time_per_iteration": 2.88820219039917 + }, + { + "auxiliary_loss_clip": 0.01296822, + "auxiliary_loss_mlp": 0.01074908, + "balance_loss_clip": 1.08187032, + "balance_loss_mlp": 1.03866816, + "epoch": 0.045331569770937295, + "flos": 32159441796480.0, + "grad_norm": 2.7488334128269534, + "language_loss": 0.65849936, + "learning_rate": 3.997553742337182e-06, + "loss": 0.68221664, + "num_input_tokens_seen": 8013365, + "router_z_loss_clip": 2.1484375, + "router_z_loss_mlp": 0.36254883, + "step": 377, + "time_per_iteration": 2.8897111415863037 + }, + { + "auxiliary_loss_clip": 0.0129523, + "auxiliary_loss_mlp": 0.01085098, + "balance_loss_clip": 1.07980418, + "balance_loss_mlp": 1.04726148, + "epoch": 0.045451812661576385, + "flos": 22163455791360.0, + "grad_norm": 7.314905505616255, + "language_loss": 0.91375178, + "learning_rate": 3.997515074936949e-06, + "loss": 0.93755507, + "num_input_tokens_seen": 8034240, + "router_z_loss_clip": 2.15625, + "router_z_loss_mlp": 0.37841797, + "step": 378, + "time_per_iteration": 2.850905418395996 + }, + { + "auxiliary_loss_clip": 0.01295385, + "auxiliary_loss_mlp": 0.0109078, + "balance_loss_clip": 1.08036613, + "balance_loss_mlp": 1.05146444, + "epoch": 0.045572055552215475, + "flos": 16581968305920.0, + "grad_norm": 3.044231510935958, + "language_loss": 0.86996889, + "learning_rate": 3.997476104515268e-06, + "loss": 0.89383054, + "num_input_tokens_seen": 8052430, + "router_z_loss_clip": 2.15136719, + "router_z_loss_mlp": 0.39282227, + "step": 379, + "time_per_iteration": 2.9071741104125977 + }, + { + "auxiliary_loss_clip": 0.01292758, + "auxiliary_loss_mlp": 0.01096276, + "balance_loss_clip": 1.0817275, + "balance_loss_mlp": 1.05483866, + "epoch": 0.045692298442854565, + "flos": 17603375448960.0, + "grad_norm": 2.1482151002577314, + "language_loss": 0.77438998, + "learning_rate": 3.9974368310780485e-06, + "loss": 0.79828036, + "num_input_tokens_seen": 8069605, + "router_z_loss_clip": 2.11230469, + "router_z_loss_mlp": 0.41455078, + "step": 380, + "time_per_iteration": 2.848284959793091 + }, + { + "auxiliary_loss_clip": 0.01311454, + "auxiliary_loss_mlp": 0.0109643, + "balance_loss_clip": 1.08598626, + "balance_loss_mlp": 1.05339527, + "epoch": 0.045812541333493655, + "flos": 26761098781440.0, + "grad_norm": 2.400028288594748, + "language_loss": 0.74731088, + "learning_rate": 3.997397254631251e-06, + "loss": 0.77138966, + "num_input_tokens_seen": 8090225, + "router_z_loss_clip": 2.25390625, + "router_z_loss_mlp": 0.43017578, + "step": 381, + "time_per_iteration": 2.90189266204834 + }, + { + "auxiliary_loss_clip": 0.0117664, + "auxiliary_loss_mlp": 0.01015747, + "balance_loss_clip": 1.06731343, + "balance_loss_mlp": 1.00497043, + "epoch": 0.04593278422413275, + "flos": 60250349894400.0, + "grad_norm": 0.8330838197983456, + "language_loss": 0.60061538, + "learning_rate": 3.997357375180878e-06, + "loss": 0.62253928, + "num_input_tokens_seen": 8154505, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.10791016, + "step": 382, + "time_per_iteration": 3.4608676433563232 + }, + { + "auxiliary_loss_clip": 0.01291905, + "auxiliary_loss_mlp": 0.0108863, + "balance_loss_clip": 1.07811236, + "balance_loss_mlp": 1.05034041, + "epoch": 0.04605302711477184, + "flos": 21799249839360.0, + "grad_norm": 1.7898459209388313, + "language_loss": 0.74937272, + "learning_rate": 3.997317192732979e-06, + "loss": 0.7731781, + "num_input_tokens_seen": 8173285, + "router_z_loss_clip": 2.13671875, + "router_z_loss_mlp": 0.38305664, + "step": 383, + "time_per_iteration": 2.8513259887695312 + }, + { + "auxiliary_loss_clip": 0.01295304, + "auxiliary_loss_mlp": 0.01090201, + "balance_loss_clip": 1.08047771, + "balance_loss_mlp": 1.05062377, + "epoch": 0.04617327000541093, + "flos": 19459705299840.0, + "grad_norm": 2.007463973643379, + "language_loss": 0.8234368, + "learning_rate": 3.99727670729365e-06, + "loss": 0.84729183, + "num_input_tokens_seen": 8191845, + "router_z_loss_clip": 2.15039062, + "router_z_loss_mlp": 0.39550781, + "step": 384, + "time_per_iteration": 2.829347848892212 + }, + { + "auxiliary_loss_clip": 0.0129762, + "auxiliary_loss_mlp": 0.010927, + "balance_loss_clip": 1.08503127, + "balance_loss_mlp": 1.05314612, + "epoch": 0.04629351289605002, + "flos": 25411468135680.0, + "grad_norm": 2.0857381204510976, + "language_loss": 0.77930427, + "learning_rate": 3.997235918869033e-06, + "loss": 0.80320746, + "num_input_tokens_seen": 8212880, + "router_z_loss_clip": 2.12304688, + "router_z_loss_mlp": 0.39550781, + "step": 385, + "time_per_iteration": 2.879145383834839 + }, + { + "auxiliary_loss_clip": 0.01294817, + "auxiliary_loss_mlp": 0.01078787, + "balance_loss_clip": 1.08324862, + "balance_loss_mlp": 1.04257059, + "epoch": 0.04641375578668911, + "flos": 20558284813440.0, + "grad_norm": 1.9306795196460482, + "language_loss": 0.82632536, + "learning_rate": 3.997194827465315e-06, + "loss": 0.85006142, + "num_input_tokens_seen": 8231475, + "router_z_loss_clip": 2.11523438, + "router_z_loss_mlp": 0.36181641, + "step": 386, + "time_per_iteration": 2.8672115802764893 + }, + { + "auxiliary_loss_clip": 0.01295342, + "auxiliary_loss_mlp": 0.01080977, + "balance_loss_clip": 1.08244777, + "balance_loss_mlp": 1.04004097, + "epoch": 0.0465339986773282, + "flos": 13188661447680.0, + "grad_norm": 2.7107138294856186, + "language_loss": 0.90646279, + "learning_rate": 3.997153433088728e-06, + "loss": 0.93022597, + "num_input_tokens_seen": 8248600, + "router_z_loss_clip": 2.13085938, + "router_z_loss_mlp": 0.40942383, + "step": 387, + "time_per_iteration": 2.808152675628662 + }, + { + "auxiliary_loss_clip": 0.01297814, + "auxiliary_loss_mlp": 0.01095357, + "balance_loss_clip": 1.08253026, + "balance_loss_mlp": 1.05413425, + "epoch": 0.0466542415679673, + "flos": 25556547168000.0, + "grad_norm": 2.113814478883475, + "language_loss": 0.81219029, + "learning_rate": 3.997111735745554e-06, + "loss": 0.83612192, + "num_input_tokens_seen": 8271570, + "router_z_loss_clip": 2.15429688, + "router_z_loss_mlp": 0.41210938, + "step": 388, + "time_per_iteration": 2.860483169555664 + }, + { + "auxiliary_loss_clip": 0.01290917, + "auxiliary_loss_mlp": 0.0108257, + "balance_loss_clip": 1.07963634, + "balance_loss_mlp": 1.04318345, + "epoch": 0.04677448445860639, + "flos": 22236749493120.0, + "grad_norm": 1.963383348594844, + "language_loss": 0.82653642, + "learning_rate": 3.997069735442118e-06, + "loss": 0.85027128, + "num_input_tokens_seen": 8291265, + "router_z_loss_clip": 2.11523438, + "router_z_loss_mlp": 0.39379883, + "step": 389, + "time_per_iteration": 2.7814762592315674 + }, + { + "auxiliary_loss_clip": 0.01294427, + "auxiliary_loss_mlp": 0.01090928, + "balance_loss_clip": 1.08284807, + "balance_loss_mlp": 1.05249453, + "epoch": 0.04689472734924548, + "flos": 28147825198080.0, + "grad_norm": 1.8687424900672451, + "language_loss": 0.8011021, + "learning_rate": 3.997027432184792e-06, + "loss": 0.8249557, + "num_input_tokens_seen": 8315925, + "router_z_loss_clip": 2.11523438, + "router_z_loss_mlp": 0.38427734, + "step": 390, + "time_per_iteration": 2.9355721473693848 + }, + { + "auxiliary_loss_clip": 0.01298015, + "auxiliary_loss_mlp": 0.01086799, + "balance_loss_clip": 1.08320141, + "balance_loss_mlp": 1.05067885, + "epoch": 0.04701497023988457, + "flos": 23148952312320.0, + "grad_norm": 1.8086530110823085, + "language_loss": 0.89299226, + "learning_rate": 3.99698482597999e-06, + "loss": 0.91684043, + "num_input_tokens_seen": 8333605, + "router_z_loss_clip": 2.15039062, + "router_z_loss_mlp": 0.36132812, + "step": 391, + "time_per_iteration": 2.899014711380005 + }, + { + "auxiliary_loss_clip": 0.01160823, + "auxiliary_loss_mlp": 0.0101301, + "balance_loss_clip": 1.05423522, + "balance_loss_mlp": 1.00333047, + "epoch": 0.04713521313052366, + "flos": 64827668764800.0, + "grad_norm": 0.906352507132043, + "language_loss": 0.63835561, + "learning_rate": 3.99694191683418e-06, + "loss": 0.6600939, + "num_input_tokens_seen": 8394405, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.09667969, + "step": 392, + "time_per_iteration": 3.5216777324676514 + }, + { + "auxiliary_loss_clip": 0.01303551, + "auxiliary_loss_mlp": 0.01107142, + "balance_loss_clip": 1.08363676, + "balance_loss_mlp": 1.0651567, + "epoch": 0.047255456021162746, + "flos": 18771585477120.0, + "grad_norm": 1.9111467386938952, + "language_loss": 0.8196134, + "learning_rate": 3.996898704753867e-06, + "loss": 0.84372032, + "num_input_tokens_seen": 8412355, + "router_z_loss_clip": 2.19921875, + "router_z_loss_mlp": 0.41967773, + "step": 393, + "time_per_iteration": 5.570013999938965 + }, + { + "auxiliary_loss_clip": 0.01293753, + "auxiliary_loss_mlp": 0.01080345, + "balance_loss_clip": 1.08136201, + "balance_loss_mlp": 1.04560709, + "epoch": 0.04737569891180184, + "flos": 22053820504320.0, + "grad_norm": 2.21300220566762, + "language_loss": 0.87502575, + "learning_rate": 3.996855189745609e-06, + "loss": 0.89876676, + "num_input_tokens_seen": 8431620, + "router_z_loss_clip": 2.12402344, + "router_z_loss_mlp": 0.34765625, + "step": 394, + "time_per_iteration": 2.8907604217529297 + }, + { + "auxiliary_loss_clip": 0.01291299, + "auxiliary_loss_mlp": 0.0108756, + "balance_loss_clip": 1.07906532, + "balance_loss_mlp": 1.04771996, + "epoch": 0.04749594180244093, + "flos": 29057370410880.0, + "grad_norm": 4.196159301069701, + "language_loss": 0.92494786, + "learning_rate": 3.996811371816007e-06, + "loss": 0.94873643, + "num_input_tokens_seen": 8454045, + "router_z_loss_clip": 2.12109375, + "router_z_loss_mlp": 0.39892578, + "step": 395, + "time_per_iteration": 3.0485849380493164 + }, + { + "auxiliary_loss_clip": 0.01292607, + "auxiliary_loss_mlp": 0.0108673, + "balance_loss_clip": 1.0838176, + "balance_loss_mlp": 1.04951251, + "epoch": 0.04761618469308002, + "flos": 35112268172160.0, + "grad_norm": 2.282282763837405, + "language_loss": 0.77881575, + "learning_rate": 3.996767250971707e-06, + "loss": 0.80260915, + "num_input_tokens_seen": 8476785, + "router_z_loss_clip": 2.0859375, + "router_z_loss_mlp": 0.37231445, + "step": 396, + "time_per_iteration": 2.99396014213562 + }, + { + "auxiliary_loss_clip": 0.0130071, + "auxiliary_loss_mlp": 0.01089323, + "balance_loss_clip": 1.0862639, + "balance_loss_mlp": 1.04817247, + "epoch": 0.04773642758371911, + "flos": 25630702796160.0, + "grad_norm": 2.537052433479449, + "language_loss": 0.867962, + "learning_rate": 3.996722827219403e-06, + "loss": 0.89186227, + "num_input_tokens_seen": 8498400, + "router_z_loss_clip": 2.14257812, + "router_z_loss_mlp": 0.41137695, + "step": 397, + "time_per_iteration": 2.9002466201782227 + }, + { + "auxiliary_loss_clip": 0.01300119, + "auxiliary_loss_mlp": 0.01095158, + "balance_loss_clip": 1.08270979, + "balance_loss_mlp": 1.05701137, + "epoch": 0.0478566704743582, + "flos": 20631506688000.0, + "grad_norm": 3.090134722399495, + "language_loss": 0.82568109, + "learning_rate": 3.996678100565833e-06, + "loss": 0.84963381, + "num_input_tokens_seen": 8517455, + "router_z_loss_clip": 2.171875, + "router_z_loss_mlp": 0.3815918, + "step": 398, + "time_per_iteration": 2.8752713203430176 + }, + { + "auxiliary_loss_clip": 0.01289102, + "auxiliary_loss_mlp": 0.01090132, + "balance_loss_clip": 1.08124852, + "balance_loss_mlp": 1.04943371, + "epoch": 0.04797691336499729, + "flos": 18835721210880.0, + "grad_norm": 2.5225911778327976, + "language_loss": 0.88529539, + "learning_rate": 3.996633071017783e-06, + "loss": 0.90908772, + "num_input_tokens_seen": 8534085, + "router_z_loss_clip": 2.08203125, + "router_z_loss_mlp": 0.40722656, + "step": 399, + "time_per_iteration": 2.845191240310669 + }, + { + "auxiliary_loss_clip": 0.01287354, + "auxiliary_loss_mlp": 0.01087848, + "balance_loss_clip": 1.08098865, + "balance_loss_mlp": 1.04951072, + "epoch": 0.04809715625563638, + "flos": 21099673578240.0, + "grad_norm": 2.23907255206895, + "language_loss": 0.81713217, + "learning_rate": 3.996587738582084e-06, + "loss": 0.84088421, + "num_input_tokens_seen": 8550885, + "router_z_loss_clip": 2.06152344, + "router_z_loss_mlp": 0.38354492, + "step": 400, + "time_per_iteration": 2.8361282348632812 + }, + { + "auxiliary_loss_clip": 0.01294146, + "auxiliary_loss_mlp": 0.01079543, + "balance_loss_clip": 1.08110952, + "balance_loss_mlp": 1.04242134, + "epoch": 0.04821739914627548, + "flos": 23805650712960.0, + "grad_norm": 2.423438117189525, + "language_loss": 0.86291748, + "learning_rate": 3.9965421032656115e-06, + "loss": 0.88665438, + "num_input_tokens_seen": 8570815, + "router_z_loss_clip": 2.12695312, + "router_z_loss_mlp": 0.37133789, + "step": 401, + "time_per_iteration": 2.8322949409484863 + }, + { + "auxiliary_loss_clip": 0.01291244, + "auxiliary_loss_mlp": 0.01084795, + "balance_loss_clip": 1.07599688, + "balance_loss_mlp": 1.04803109, + "epoch": 0.04833764203691457, + "flos": 22200587475840.0, + "grad_norm": 2.9412135282775886, + "language_loss": 0.94423634, + "learning_rate": 3.99649616507529e-06, + "loss": 0.96799672, + "num_input_tokens_seen": 8589910, + "router_z_loss_clip": 2.15625, + "router_z_loss_mlp": 0.36791992, + "step": 402, + "time_per_iteration": 2.872709035873413 + }, + { + "auxiliary_loss_clip": 0.01159707, + "auxiliary_loss_mlp": 0.01012405, + "balance_loss_clip": 1.05423641, + "balance_loss_mlp": 1.00310659, + "epoch": 0.04845788492755366, + "flos": 65904376896000.0, + "grad_norm": 0.885651618742271, + "language_loss": 0.63143224, + "learning_rate": 3.996449924018088e-06, + "loss": 0.65315336, + "num_input_tokens_seen": 8650370, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.09277344, + "step": 403, + "time_per_iteration": 3.2719905376434326 + }, + { + "auxiliary_loss_clip": 0.0128808, + "auxiliary_loss_mlp": 0.01085398, + "balance_loss_clip": 1.08016539, + "balance_loss_mlp": 1.04994535, + "epoch": 0.04857812781819275, + "flos": 19281301424640.0, + "grad_norm": 1.9832518154127112, + "language_loss": 0.79261583, + "learning_rate": 3.99640338010102e-06, + "loss": 0.81635058, + "num_input_tokens_seen": 8669475, + "router_z_loss_clip": 2.078125, + "router_z_loss_mlp": 0.35449219, + "step": 404, + "time_per_iteration": 2.868795156478882 + }, + { + "auxiliary_loss_clip": 0.0128812, + "auxiliary_loss_mlp": 0.01071215, + "balance_loss_clip": 1.07977998, + "balance_loss_mlp": 1.03602421, + "epoch": 0.04869837070883184, + "flos": 24062376193920.0, + "grad_norm": 1.7368925728253053, + "language_loss": 0.78245461, + "learning_rate": 3.996356533331146e-06, + "loss": 0.80604792, + "num_input_tokens_seen": 8691345, + "router_z_loss_clip": 2.08789062, + "router_z_loss_mlp": 0.35180664, + "step": 405, + "time_per_iteration": 2.869645595550537 + }, + { + "auxiliary_loss_clip": 0.01298464, + "auxiliary_loss_mlp": 0.0108069, + "balance_loss_clip": 1.08117557, + "balance_loss_mlp": 1.04390192, + "epoch": 0.04881861359947093, + "flos": 25187169657600.0, + "grad_norm": 2.2179863918891587, + "language_loss": 0.61575365, + "learning_rate": 3.996309383715573e-06, + "loss": 0.63954514, + "num_input_tokens_seen": 8710125, + "router_z_loss_clip": 2.171875, + "router_z_loss_mlp": 0.36767578, + "step": 406, + "time_per_iteration": 2.9185712337493896 + }, + { + "auxiliary_loss_clip": 0.01286957, + "auxiliary_loss_mlp": 0.01086298, + "balance_loss_clip": 1.07665515, + "balance_loss_mlp": 1.04850912, + "epoch": 0.048938856490110025, + "flos": 16362913213440.0, + "grad_norm": 2.0148531865095887, + "language_loss": 0.73940051, + "learning_rate": 3.996261931261454e-06, + "loss": 0.76313305, + "num_input_tokens_seen": 8728705, + "router_z_loss_clip": 2.10351562, + "router_z_loss_mlp": 0.37792969, + "step": 407, + "time_per_iteration": 2.807619333267212 + }, + { + "auxiliary_loss_clip": 0.01288562, + "auxiliary_loss_mlp": 0.01070141, + "balance_loss_clip": 1.08244252, + "balance_loss_mlp": 1.03309095, + "epoch": 0.049059099380749115, + "flos": 29895094379520.0, + "grad_norm": 1.5864915241503623, + "language_loss": 0.86602563, + "learning_rate": 3.996214175975987e-06, + "loss": 0.88961267, + "num_input_tokens_seen": 8749225, + "router_z_loss_clip": 2.06054688, + "router_z_loss_mlp": 0.37011719, + "step": 408, + "time_per_iteration": 2.869940757751465 + }, + { + "auxiliary_loss_clip": 0.01290861, + "auxiliary_loss_mlp": 0.0108828, + "balance_loss_clip": 1.0804708, + "balance_loss_mlp": 1.04822612, + "epoch": 0.049179342271388204, + "flos": 35918858027520.0, + "grad_norm": 2.607503766611645, + "language_loss": 0.79102951, + "learning_rate": 3.996166117866417e-06, + "loss": 0.81482089, + "num_input_tokens_seen": 8771160, + "router_z_loss_clip": 2.10546875, + "router_z_loss_mlp": 0.40039062, + "step": 409, + "time_per_iteration": 3.051870822906494 + }, + { + "auxiliary_loss_clip": 0.01286589, + "auxiliary_loss_mlp": 0.01082006, + "balance_loss_clip": 1.07804656, + "balance_loss_mlp": 1.04414535, + "epoch": 0.049299585162027294, + "flos": 14611226659200.0, + "grad_norm": 1.922356496292733, + "language_loss": 0.86570084, + "learning_rate": 3.996117756940035e-06, + "loss": 0.88938689, + "num_input_tokens_seen": 8787845, + "router_z_loss_clip": 2.0859375, + "router_z_loss_mlp": 0.37841797, + "step": 410, + "time_per_iteration": 2.9872984886169434 + }, + { + "auxiliary_loss_clip": 0.01286409, + "auxiliary_loss_mlp": 0.01080139, + "balance_loss_clip": 1.07739329, + "balance_loss_mlp": 1.04296923, + "epoch": 0.049419828052666384, + "flos": 19567939956480.0, + "grad_norm": 1.950405827730567, + "language_loss": 0.97518921, + "learning_rate": 3.996069093204175e-06, + "loss": 0.99885464, + "num_input_tokens_seen": 8803805, + "router_z_loss_clip": 2.08984375, + "router_z_loss_mlp": 0.37182617, + "step": 411, + "time_per_iteration": 2.85245943069458 + }, + { + "auxiliary_loss_clip": 0.0129992, + "auxiliary_loss_mlp": 0.01092704, + "balance_loss_clip": 1.08696413, + "balance_loss_mlp": 1.05486715, + "epoch": 0.049540070943305474, + "flos": 13659916907520.0, + "grad_norm": 2.4551663404299364, + "language_loss": 0.8841328, + "learning_rate": 3.996020126666221e-06, + "loss": 0.908059, + "num_input_tokens_seen": 8820785, + "router_z_loss_clip": 2.12597656, + "router_z_loss_mlp": 0.37866211, + "step": 412, + "time_per_iteration": 2.8661296367645264 + }, + { + "auxiliary_loss_clip": 0.01292375, + "auxiliary_loss_mlp": 0.0107522, + "balance_loss_clip": 1.08157051, + "balance_loss_mlp": 1.03724003, + "epoch": 0.04966031383394457, + "flos": 21832035978240.0, + "grad_norm": 2.1248964513709843, + "language_loss": 0.82011819, + "learning_rate": 3.995970857333601e-06, + "loss": 0.84379411, + "num_input_tokens_seen": 8841195, + "router_z_loss_clip": 2.109375, + "router_z_loss_mlp": 0.37939453, + "step": 413, + "time_per_iteration": 2.917175769805908 + }, + { + "auxiliary_loss_clip": 0.0129095, + "auxiliary_loss_mlp": 0.01075441, + "balance_loss_clip": 1.07977521, + "balance_loss_mlp": 1.03848612, + "epoch": 0.04978055672458366, + "flos": 28618793349120.0, + "grad_norm": 1.8054204944115162, + "language_loss": 0.79824281, + "learning_rate": 3.995921285213789e-06, + "loss": 0.82190669, + "num_input_tokens_seen": 8861455, + "router_z_loss_clip": 2.11328125, + "router_z_loss_mlp": 0.36914062, + "step": 414, + "time_per_iteration": 3.0222575664520264 + }, + { + "auxiliary_loss_clip": 0.01285535, + "auxiliary_loss_mlp": 0.01073909, + "balance_loss_clip": 1.07998943, + "balance_loss_mlp": 1.03945732, + "epoch": 0.04990079961522275, + "flos": 19828220883840.0, + "grad_norm": 2.1579862512481305, + "language_loss": 0.80574131, + "learning_rate": 3.995871410314305e-06, + "loss": 0.82933569, + "num_input_tokens_seen": 8880015, + "router_z_loss_clip": 2.05371094, + "router_z_loss_mlp": 0.34399414, + "step": 415, + "time_per_iteration": 2.8232576847076416 + }, + { + "auxiliary_loss_clip": 0.01160084, + "auxiliary_loss_mlp": 0.01010785, + "balance_loss_clip": 1.05515552, + "balance_loss_mlp": 1.00181997, + "epoch": 0.05002104250586184, + "flos": 62735045293440.0, + "grad_norm": 0.9087367395506412, + "language_loss": 0.59664208, + "learning_rate": 3.995821232642714e-06, + "loss": 0.6183508, + "num_input_tokens_seen": 8938420, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.08984375, + "step": 416, + "time_per_iteration": 3.448927879333496 + }, + { + "auxiliary_loss_clip": 0.01286884, + "auxiliary_loss_mlp": 0.01085145, + "balance_loss_clip": 1.08077002, + "balance_loss_mlp": 1.04830921, + "epoch": 0.05014128539650093, + "flos": 27928518710400.0, + "grad_norm": 2.2958392902463833, + "language_loss": 0.82528991, + "learning_rate": 3.995770752206629e-06, + "loss": 0.84901023, + "num_input_tokens_seen": 8959495, + "router_z_loss_clip": 2.05859375, + "router_z_loss_mlp": 0.36865234, + "step": 417, + "time_per_iteration": 2.9588968753814697 + }, + { + "auxiliary_loss_clip": 0.01287357, + "auxiliary_loss_mlp": 0.01088562, + "balance_loss_clip": 1.07994425, + "balance_loss_mlp": 1.0513922, + "epoch": 0.05026152828714002, + "flos": 17705576620800.0, + "grad_norm": 1.8310534029528363, + "language_loss": 0.97196448, + "learning_rate": 3.995719969013709e-06, + "loss": 0.99572361, + "num_input_tokens_seen": 8976675, + "router_z_loss_clip": 2.07617188, + "router_z_loss_mlp": 0.37133789, + "step": 418, + "time_per_iteration": 2.872931957244873 + }, + { + "auxiliary_loss_clip": 0.01287529, + "auxiliary_loss_mlp": 0.01080812, + "balance_loss_clip": 1.07681644, + "balance_loss_mlp": 1.04333282, + "epoch": 0.05038177117777912, + "flos": 19133277477120.0, + "grad_norm": 3.193133515516051, + "language_loss": 0.85480618, + "learning_rate": 3.995668883071655e-06, + "loss": 0.87848961, + "num_input_tokens_seen": 8992900, + "router_z_loss_clip": 2.109375, + "router_z_loss_mlp": 0.37426758, + "step": 419, + "time_per_iteration": 2.811211109161377 + }, + { + "auxiliary_loss_clip": 0.01282244, + "auxiliary_loss_mlp": 0.01089099, + "balance_loss_clip": 1.07740879, + "balance_loss_mlp": 1.05054653, + "epoch": 0.050502014068418206, + "flos": 20667704618880.0, + "grad_norm": 2.758618071778993, + "language_loss": 0.90647054, + "learning_rate": 3.995617494388219e-06, + "loss": 0.93018395, + "num_input_tokens_seen": 9011020, + "router_z_loss_clip": 2.04980469, + "router_z_loss_mlp": 0.38574219, + "step": 420, + "time_per_iteration": 2.7869491577148438 + }, + { + "auxiliary_loss_clip": 0.01287944, + "auxiliary_loss_mlp": 0.01078809, + "balance_loss_clip": 1.07819688, + "balance_loss_mlp": 1.04342806, + "epoch": 0.050622256959057296, + "flos": 21361103740800.0, + "grad_norm": 2.0369259943964617, + "language_loss": 0.80751729, + "learning_rate": 3.995565802971196e-06, + "loss": 0.83118486, + "num_input_tokens_seen": 9030995, + "router_z_loss_clip": 2.09765625, + "router_z_loss_mlp": 0.35400391, + "step": 421, + "time_per_iteration": 2.845320701599121 + }, + { + "auxiliary_loss_clip": 0.01282771, + "auxiliary_loss_mlp": 0.01078401, + "balance_loss_clip": 1.07967782, + "balance_loss_mlp": 1.04278159, + "epoch": 0.050742499849696386, + "flos": 27673588909440.0, + "grad_norm": 1.928540989755047, + "language_loss": 0.6731112, + "learning_rate": 3.995513808828427e-06, + "loss": 0.69672287, + "num_input_tokens_seen": 9053790, + "router_z_loss_clip": 2.03222656, + "router_z_loss_mlp": 0.35644531, + "step": 422, + "time_per_iteration": 4.594061374664307 + }, + { + "auxiliary_loss_clip": 0.01285041, + "auxiliary_loss_mlp": 0.01075031, + "balance_loss_clip": 1.07988429, + "balance_loss_mlp": 1.03860104, + "epoch": 0.050862742740335476, + "flos": 19865999013120.0, + "grad_norm": 1.9817889962112987, + "language_loss": 0.76562572, + "learning_rate": 3.9954615119678e-06, + "loss": 0.78922641, + "num_input_tokens_seen": 9072345, + "router_z_loss_clip": 2.05078125, + "router_z_loss_mlp": 0.36425781, + "step": 423, + "time_per_iteration": 2.827711343765259 + }, + { + "auxiliary_loss_clip": 0.01279123, + "auxiliary_loss_mlp": 0.01071805, + "balance_loss_clip": 1.07728028, + "balance_loss_mlp": 1.03606629, + "epoch": 0.050982985630974566, + "flos": 22085098272000.0, + "grad_norm": 2.0795157079049487, + "language_loss": 0.81006092, + "learning_rate": 3.995408912397248e-06, + "loss": 0.83357012, + "num_input_tokens_seen": 9090240, + "router_z_loss_clip": 2.01953125, + "router_z_loss_mlp": 0.35742188, + "step": 424, + "time_per_iteration": 2.82315993309021 + }, + { + "auxiliary_loss_clip": 0.01290209, + "auxiliary_loss_mlp": 0.01093593, + "balance_loss_clip": 1.08134103, + "balance_loss_mlp": 1.05477858, + "epoch": 0.05110322852161366, + "flos": 20740962407040.0, + "grad_norm": 2.37744690242463, + "language_loss": 0.93210852, + "learning_rate": 3.99535601012475e-06, + "loss": 0.95594656, + "num_input_tokens_seen": 9105570, + "router_z_loss_clip": 2.08886719, + "router_z_loss_mlp": 0.38842773, + "step": 425, + "time_per_iteration": 2.7972288131713867 + }, + { + "auxiliary_loss_clip": 0.01273698, + "auxiliary_loss_mlp": 0.0108577, + "balance_loss_clip": 1.0772779, + "balance_loss_mlp": 1.05098498, + "epoch": 0.05122347141225275, + "flos": 28547295327360.0, + "grad_norm": 1.5259652363472729, + "language_loss": 0.75493491, + "learning_rate": 3.995302805158333e-06, + "loss": 0.77852958, + "num_input_tokens_seen": 9128225, + "router_z_loss_clip": 1.96777344, + "router_z_loss_mlp": 0.34814453, + "step": 426, + "time_per_iteration": 2.8902618885040283 + }, + { + "auxiliary_loss_clip": 0.01287853, + "auxiliary_loss_mlp": 0.01085081, + "balance_loss_clip": 1.0776968, + "balance_loss_mlp": 1.05010533, + "epoch": 0.05134371430289184, + "flos": 19722679747200.0, + "grad_norm": 2.542212459091435, + "language_loss": 0.83196032, + "learning_rate": 3.9952492975060665e-06, + "loss": 0.85568964, + "num_input_tokens_seen": 9148295, + "router_z_loss_clip": 2.10351562, + "router_z_loss_mlp": 0.3503418, + "step": 427, + "time_per_iteration": 2.8435096740722656 + }, + { + "auxiliary_loss_clip": 0.01279355, + "auxiliary_loss_mlp": 0.010733, + "balance_loss_clip": 1.07631326, + "balance_loss_mlp": 1.03863406, + "epoch": 0.05146395719353093, + "flos": 34458945649920.0, + "grad_norm": 2.675058870219991, + "language_loss": 0.8510977, + "learning_rate": 3.995195487176067e-06, + "loss": 0.87462419, + "num_input_tokens_seen": 9168525, + "router_z_loss_clip": 2.03125, + "router_z_loss_mlp": 0.34692383, + "step": 428, + "time_per_iteration": 2.9119937419891357 + }, + { + "auxiliary_loss_clip": 0.01285926, + "auxiliary_loss_mlp": 0.01091446, + "balance_loss_clip": 1.07771003, + "balance_loss_mlp": 1.05189216, + "epoch": 0.05158420008417002, + "flos": 21760286561280.0, + "grad_norm": 1.7633593060740322, + "language_loss": 0.85634726, + "learning_rate": 3.995141374176499e-06, + "loss": 0.88012093, + "num_input_tokens_seen": 9186920, + "router_z_loss_clip": 2.07910156, + "router_z_loss_mlp": 0.39550781, + "step": 429, + "time_per_iteration": 2.8081789016723633 + }, + { + "auxiliary_loss_clip": 0.01154618, + "auxiliary_loss_mlp": 0.01012976, + "balance_loss_clip": 1.05289996, + "balance_loss_mlp": 1.00372553, + "epoch": 0.05170444297480911, + "flos": 72553956226560.0, + "grad_norm": 0.8834234329659374, + "language_loss": 0.63147497, + "learning_rate": 3.995086958515572e-06, + "loss": 0.65315092, + "num_input_tokens_seen": 9244940, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.09228516, + "step": 430, + "time_per_iteration": 3.3790416717529297 + }, + { + "auxiliary_loss_clip": 0.01152588, + "auxiliary_loss_mlp": 0.01013728, + "balance_loss_clip": 1.05142248, + "balance_loss_mlp": 1.00447702, + "epoch": 0.05182468586544821, + "flos": 62416159326720.0, + "grad_norm": 0.8891144176397627, + "language_loss": 0.59893262, + "learning_rate": 3.995032240201538e-06, + "loss": 0.62059569, + "num_input_tokens_seen": 9307335, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.09228516, + "step": 431, + "time_per_iteration": 3.3025660514831543 + }, + { + "auxiliary_loss_clip": 0.01151097, + "auxiliary_loss_mlp": 0.01009014, + "balance_loss_clip": 1.05034852, + "balance_loss_mlp": 1.0004307, + "epoch": 0.0519449287560873, + "flos": 41225989432320.0, + "grad_norm": 0.9338607786207265, + "language_loss": 0.63148248, + "learning_rate": 3.9949772192427e-06, + "loss": 0.65308356, + "num_input_tokens_seen": 9353960, + "router_z_loss_clip": 1.0078125, + "router_z_loss_mlp": 0.0859375, + "step": 432, + "time_per_iteration": 3.0403058528900146 + }, + { + "auxiliary_loss_clip": 0.01281051, + "auxiliary_loss_mlp": 0.01084168, + "balance_loss_clip": 1.07692254, + "balance_loss_mlp": 1.04804778, + "epoch": 0.05206517164672639, + "flos": 17494530261120.0, + "grad_norm": 1.980868365294641, + "language_loss": 0.79605341, + "learning_rate": 3.994921895647405e-06, + "loss": 0.81970561, + "num_input_tokens_seen": 9372130, + "router_z_loss_clip": 2.04101562, + "router_z_loss_mlp": 0.36108398, + "step": 433, + "time_per_iteration": 2.875852346420288 + }, + { + "auxiliary_loss_clip": 0.01145551, + "auxiliary_loss_mlp": 0.0100777, + "balance_loss_clip": 1.04688811, + "balance_loss_mlp": 0.99932975, + "epoch": 0.05218541453736548, + "flos": 64002762973440.0, + "grad_norm": 0.8439373752505914, + "language_loss": 0.55382842, + "learning_rate": 3.994866269424043e-06, + "loss": 0.57536161, + "num_input_tokens_seen": 9428500, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.08447266, + "step": 434, + "time_per_iteration": 3.2602474689483643 + }, + { + "auxiliary_loss_clip": 0.01283401, + "auxiliary_loss_mlp": 0.01074961, + "balance_loss_clip": 1.07747364, + "balance_loss_mlp": 1.03802979, + "epoch": 0.05230565742800457, + "flos": 19317319787520.0, + "grad_norm": 2.287408795940489, + "language_loss": 0.7809031, + "learning_rate": 3.9948103405810545e-06, + "loss": 0.80448675, + "num_input_tokens_seen": 9447450, + "router_z_loss_clip": 2.06054688, + "router_z_loss_mlp": 0.36938477, + "step": 435, + "time_per_iteration": 2.8400659561157227 + }, + { + "auxiliary_loss_clip": 0.01277709, + "auxiliary_loss_mlp": 0.01086609, + "balance_loss_clip": 1.07905841, + "balance_loss_mlp": 1.05029798, + "epoch": 0.05242590031864366, + "flos": 25298636538240.0, + "grad_norm": 2.11709082061653, + "language_loss": 0.85945547, + "learning_rate": 3.994754109126923e-06, + "loss": 0.88309872, + "num_input_tokens_seen": 9468945, + "router_z_loss_clip": 1.984375, + "router_z_loss_mlp": 0.36328125, + "step": 436, + "time_per_iteration": 2.9707093238830566 + }, + { + "auxiliary_loss_clip": 0.01281449, + "auxiliary_loss_mlp": 0.01081768, + "balance_loss_clip": 1.07970953, + "balance_loss_mlp": 1.04672062, + "epoch": 0.052546143209282754, + "flos": 26211629456640.0, + "grad_norm": 1.954343541858056, + "language_loss": 0.93402487, + "learning_rate": 3.994697575070181e-06, + "loss": 0.95765704, + "num_input_tokens_seen": 9488405, + "router_z_loss_clip": 2.01660156, + "router_z_loss_mlp": 0.3503418, + "step": 437, + "time_per_iteration": 2.9050071239471436 + }, + { + "auxiliary_loss_clip": 0.01285952, + "auxiliary_loss_mlp": 0.01081505, + "balance_loss_clip": 1.08108783, + "balance_loss_mlp": 1.0455749, + "epoch": 0.052666386099921844, + "flos": 22158140578560.0, + "grad_norm": 1.654232985564461, + "language_loss": 0.91277152, + "learning_rate": 3.994640738419402e-06, + "loss": 0.93644613, + "num_input_tokens_seen": 9507780, + "router_z_loss_clip": 2.04492188, + "router_z_loss_mlp": 0.359375, + "step": 438, + "time_per_iteration": 2.879221200942993 + }, + { + "auxiliary_loss_clip": 0.01275588, + "auxiliary_loss_mlp": 0.01073491, + "balance_loss_clip": 1.07510591, + "balance_loss_mlp": 1.04047012, + "epoch": 0.052786628990560934, + "flos": 23881817502720.0, + "grad_norm": 2.307064276187363, + "language_loss": 0.80876327, + "learning_rate": 3.9945835991832075e-06, + "loss": 0.83225405, + "num_input_tokens_seen": 9529665, + "router_z_loss_clip": 2.00488281, + "router_z_loss_mlp": 0.33032227, + "step": 439, + "time_per_iteration": 2.909921407699585 + }, + { + "auxiliary_loss_clip": 0.01281388, + "auxiliary_loss_mlp": 0.01089789, + "balance_loss_clip": 1.07842481, + "balance_loss_mlp": 1.05409777, + "epoch": 0.052906871881200024, + "flos": 24605021934720.0, + "grad_norm": 2.1115005573088976, + "language_loss": 0.92911178, + "learning_rate": 3.994526157370268e-06, + "loss": 0.95282358, + "num_input_tokens_seen": 9548280, + "router_z_loss_clip": 2.03222656, + "router_z_loss_mlp": 0.35693359, + "step": 440, + "time_per_iteration": 2.898608922958374 + }, + { + "auxiliary_loss_clip": 0.01139475, + "auxiliary_loss_mlp": 0.01009915, + "balance_loss_clip": 1.0434258, + "balance_loss_mlp": 1.00204754, + "epoch": 0.053027114771839114, + "flos": 56461631143680.0, + "grad_norm": 0.9076411855656592, + "language_loss": 0.59283698, + "learning_rate": 3.994468412989296e-06, + "loss": 0.61433089, + "num_input_tokens_seen": 9609690, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.07861328, + "step": 441, + "time_per_iteration": 3.410731792449951 + }, + { + "auxiliary_loss_clip": 0.01278659, + "auxiliary_loss_mlp": 0.01086773, + "balance_loss_clip": 1.07920337, + "balance_loss_mlp": 1.05103397, + "epoch": 0.053147357662478203, + "flos": 17311098481920.0, + "grad_norm": 2.804206157460881, + "language_loss": 0.92870426, + "learning_rate": 3.994410366049052e-06, + "loss": 0.95235854, + "num_input_tokens_seen": 9627550, + "router_z_loss_clip": 1.9921875, + "router_z_loss_mlp": 0.35766602, + "step": 442, + "time_per_iteration": 2.862488031387329 + }, + { + "auxiliary_loss_clip": 0.01275727, + "auxiliary_loss_mlp": 0.01091455, + "balance_loss_clip": 1.07576168, + "balance_loss_mlp": 1.05464315, + "epoch": 0.0532676005531173, + "flos": 17164977955200.0, + "grad_norm": 2.0835097378293943, + "language_loss": 0.83129531, + "learning_rate": 3.994352016558341e-06, + "loss": 0.85496712, + "num_input_tokens_seen": 9644855, + "router_z_loss_clip": 2.0, + "router_z_loss_mlp": 0.36791992, + "step": 443, + "time_per_iteration": 2.8549134731292725 + }, + { + "auxiliary_loss_clip": 0.01274426, + "auxiliary_loss_mlp": 0.01067622, + "balance_loss_clip": 1.07756388, + "balance_loss_mlp": 1.03250337, + "epoch": 0.05338784344375639, + "flos": 27819960831360.0, + "grad_norm": 2.0247773077091265, + "language_loss": 0.74091697, + "learning_rate": 3.994293364526014e-06, + "loss": 0.76433742, + "num_input_tokens_seen": 9665740, + "router_z_loss_clip": 1.96972656, + "router_z_loss_mlp": 0.35107422, + "step": 444, + "time_per_iteration": 2.86647629737854 + }, + { + "auxiliary_loss_clip": 0.01272158, + "auxiliary_loss_mlp": 0.01081484, + "balance_loss_clip": 1.07822192, + "balance_loss_mlp": 1.04817712, + "epoch": 0.05350808633439548, + "flos": 21507691144320.0, + "grad_norm": 1.985591749894498, + "language_loss": 0.84860533, + "learning_rate": 3.99423440996097e-06, + "loss": 0.87214172, + "num_input_tokens_seen": 9685280, + "router_z_loss_clip": 1.93945312, + "router_z_loss_mlp": 0.33276367, + "step": 445, + "time_per_iteration": 2.813153028488159 + }, + { + "auxiliary_loss_clip": 0.01277824, + "auxiliary_loss_mlp": 0.0108314, + "balance_loss_clip": 1.0778718, + "balance_loss_mlp": 1.04778242, + "epoch": 0.05362832922503457, + "flos": 20084299920000.0, + "grad_norm": 2.9954653537072025, + "language_loss": 0.81730807, + "learning_rate": 3.994175152872152e-06, + "loss": 0.84091771, + "num_input_tokens_seen": 9704365, + "router_z_loss_clip": 2.00195312, + "router_z_loss_mlp": 0.35351562, + "step": 446, + "time_per_iteration": 2.8272480964660645 + }, + { + "auxiliary_loss_clip": 0.01273374, + "auxiliary_loss_mlp": 0.01090231, + "balance_loss_clip": 1.07852173, + "balance_loss_mlp": 1.05678105, + "epoch": 0.05374857211567366, + "flos": 26137222433280.0, + "grad_norm": 4.628491079135401, + "language_loss": 0.7862283, + "learning_rate": 3.994115593268548e-06, + "loss": 0.8098644, + "num_input_tokens_seen": 9724145, + "router_z_loss_clip": 1.94824219, + "router_z_loss_mlp": 0.33422852, + "step": 447, + "time_per_iteration": 2.8554012775421143 + }, + { + "auxiliary_loss_clip": 0.01278917, + "auxiliary_loss_mlp": 0.01085223, + "balance_loss_clip": 1.07790613, + "balance_loss_mlp": 1.04955518, + "epoch": 0.05386881500631275, + "flos": 27486817165440.0, + "grad_norm": 5.38325724268846, + "language_loss": 0.82310456, + "learning_rate": 3.994055731159195e-06, + "loss": 0.84674597, + "num_input_tokens_seen": 9741615, + "router_z_loss_clip": 2.00976562, + "router_z_loss_mlp": 0.35668945, + "step": 448, + "time_per_iteration": 2.8493521213531494 + }, + { + "auxiliary_loss_clip": 0.01283625, + "auxiliary_loss_mlp": 0.0109828, + "balance_loss_clip": 1.08442044, + "balance_loss_mlp": 1.06370962, + "epoch": 0.053989057896951846, + "flos": 23585087249280.0, + "grad_norm": 1.867219147084601, + "language_loss": 0.86795193, + "learning_rate": 3.993995566553172e-06, + "loss": 0.89177096, + "num_input_tokens_seen": 9760580, + "router_z_loss_clip": 1.99414062, + "router_z_loss_mlp": 0.34570312, + "step": 449, + "time_per_iteration": 2.8969004154205322 + }, + { + "auxiliary_loss_clip": 0.01269742, + "auxiliary_loss_mlp": 0.01069217, + "balance_loss_clip": 1.07264733, + "balance_loss_mlp": 1.03569508, + "epoch": 0.054109300787590936, + "flos": 25228862369280.0, + "grad_norm": 1.660864176689129, + "language_loss": 0.76951772, + "learning_rate": 3.993935099459607e-06, + "loss": 0.79290724, + "num_input_tokens_seen": 9782195, + "router_z_loss_clip": 1.97070312, + "router_z_loss_mlp": 0.33520508, + "step": 450, + "time_per_iteration": 2.8888537883758545 + }, + { + "auxiliary_loss_clip": 0.01267642, + "auxiliary_loss_mlp": 0.01068802, + "balance_loss_clip": 1.07343721, + "balance_loss_mlp": 1.03556681, + "epoch": 0.054229543678230026, + "flos": 23841525421440.0, + "grad_norm": 1.9570959125982814, + "language_loss": 0.73874152, + "learning_rate": 3.993874329887673e-06, + "loss": 0.762106, + "num_input_tokens_seen": 9800850, + "router_z_loss_clip": 1.93945312, + "router_z_loss_mlp": 0.33227539, + "step": 451, + "time_per_iteration": 4.685229301452637 + }, + { + "auxiliary_loss_clip": 0.012786, + "auxiliary_loss_mlp": 0.01075727, + "balance_loss_clip": 1.07833803, + "balance_loss_mlp": 1.03984451, + "epoch": 0.054349786568869116, + "flos": 16320933192960.0, + "grad_norm": 2.1407395466024344, + "language_loss": 0.86420083, + "learning_rate": 3.993813257846589e-06, + "loss": 0.88774407, + "num_input_tokens_seen": 9817605, + "router_z_loss_clip": 2.00488281, + "router_z_loss_mlp": 0.35913086, + "step": 452, + "time_per_iteration": 2.8893158435821533 + }, + { + "auxiliary_loss_clip": 0.01278992, + "auxiliary_loss_mlp": 0.01077927, + "balance_loss_clip": 1.07912624, + "balance_loss_mlp": 1.04199731, + "epoch": 0.054470029459508205, + "flos": 18660729127680.0, + "grad_norm": 2.7547546186362757, + "language_loss": 0.9284178, + "learning_rate": 3.993751883345619e-06, + "loss": 0.95198697, + "num_input_tokens_seen": 9835965, + "router_z_loss_clip": 2.0, + "router_z_loss_mlp": 0.35913086, + "step": 453, + "time_per_iteration": 3.0393941402435303 + }, + { + "auxiliary_loss_clip": 0.01279614, + "auxiliary_loss_mlp": 0.01089336, + "balance_loss_clip": 1.0796206, + "balance_loss_mlp": 1.05447924, + "epoch": 0.054590272350147295, + "flos": 17785298856960.0, + "grad_norm": 2.571747910199935, + "language_loss": 0.87789178, + "learning_rate": 3.993690206394073e-06, + "loss": 0.90158129, + "num_input_tokens_seen": 9852265, + "router_z_loss_clip": 2.0, + "router_z_loss_mlp": 0.34863281, + "step": 454, + "time_per_iteration": 2.873096466064453 + }, + { + "auxiliary_loss_clip": 0.01282292, + "auxiliary_loss_mlp": 0.01082551, + "balance_loss_clip": 1.0818547, + "balance_loss_mlp": 1.04907715, + "epoch": 0.054710515240786385, + "flos": 17785945301760.0, + "grad_norm": 2.0850114042789567, + "language_loss": 0.87448704, + "learning_rate": 3.993628227001307e-06, + "loss": 0.89813548, + "num_input_tokens_seen": 9870465, + "router_z_loss_clip": 2.00292969, + "router_z_loss_mlp": 0.33496094, + "step": 455, + "time_per_iteration": 2.792245626449585 + }, + { + "auxiliary_loss_clip": 0.01272312, + "auxiliary_loss_mlp": 0.01078231, + "balance_loss_clip": 1.07933426, + "balance_loss_mlp": 1.0459969, + "epoch": 0.05483075813142548, + "flos": 48210900180480.0, + "grad_norm": 2.0271698234904805, + "language_loss": 0.71381843, + "learning_rate": 3.993565945176726e-06, + "loss": 0.73732388, + "num_input_tokens_seen": 9891490, + "router_z_loss_clip": 1.93066406, + "router_z_loss_mlp": 0.32226562, + "step": 456, + "time_per_iteration": 3.0659313201904297 + }, + { + "auxiliary_loss_clip": 0.01275447, + "auxiliary_loss_mlp": 0.01092117, + "balance_loss_clip": 1.08099508, + "balance_loss_mlp": 1.05785644, + "epoch": 0.05495100102206457, + "flos": 19682244011520.0, + "grad_norm": 2.0191414370075718, + "language_loss": 0.84279454, + "learning_rate": 3.993503360929776e-06, + "loss": 0.86647016, + "num_input_tokens_seen": 9910375, + "router_z_loss_clip": 1.94726562, + "router_z_loss_mlp": 0.34277344, + "step": 457, + "time_per_iteration": 3.038893938064575 + }, + { + "auxiliary_loss_clip": 0.01281629, + "auxiliary_loss_mlp": 0.01076434, + "balance_loss_clip": 1.07983208, + "balance_loss_mlp": 1.04164886, + "epoch": 0.05507124391270366, + "flos": 26360048453760.0, + "grad_norm": 1.7076438808394532, + "language_loss": 0.81226051, + "learning_rate": 3.99344047426995e-06, + "loss": 0.83584112, + "num_input_tokens_seen": 9931635, + "router_z_loss_clip": 2.01757812, + "router_z_loss_mlp": 0.34790039, + "step": 458, + "time_per_iteration": 2.869516611099243 + }, + { + "auxiliary_loss_clip": 0.01282509, + "auxiliary_loss_mlp": 0.01069591, + "balance_loss_clip": 1.07973838, + "balance_loss_mlp": 1.03561687, + "epoch": 0.05519148680334275, + "flos": 22601314581120.0, + "grad_norm": 2.142937872111043, + "language_loss": 0.93645328, + "learning_rate": 3.993377285206789e-06, + "loss": 0.95997429, + "num_input_tokens_seen": 9951420, + "router_z_loss_clip": 2.02929688, + "router_z_loss_mlp": 0.33984375, + "step": 459, + "time_per_iteration": 2.81506609916687 + }, + { + "auxiliary_loss_clip": 0.01285743, + "auxiliary_loss_mlp": 0.01084918, + "balance_loss_clip": 1.08387506, + "balance_loss_mlp": 1.05015683, + "epoch": 0.05531172969398184, + "flos": 40552519380480.0, + "grad_norm": 1.878254338216777, + "language_loss": 0.86530244, + "learning_rate": 3.99331379374988e-06, + "loss": 0.88900912, + "num_input_tokens_seen": 9975025, + "router_z_loss_clip": 2.01953125, + "router_z_loss_mlp": 0.34765625, + "step": 460, + "time_per_iteration": 2.963397979736328 + }, + { + "auxiliary_loss_clip": 0.01279486, + "auxiliary_loss_mlp": 0.01075264, + "balance_loss_clip": 1.08084178, + "balance_loss_mlp": 1.04236221, + "epoch": 0.05543197258462093, + "flos": 23477894087040.0, + "grad_norm": 2.1142249009985363, + "language_loss": 0.79747033, + "learning_rate": 3.993249999908852e-06, + "loss": 0.82101786, + "num_input_tokens_seen": 9995175, + "router_z_loss_clip": 1.98632812, + "router_z_loss_mlp": 0.32910156, + "step": 461, + "time_per_iteration": 2.826519012451172 + }, + { + "auxiliary_loss_clip": 0.01274181, + "auxiliary_loss_mlp": 0.01076085, + "balance_loss_clip": 1.07511997, + "balance_loss_mlp": 1.04168153, + "epoch": 0.05555221547526003, + "flos": 18624603024000.0, + "grad_norm": 2.0461480297233745, + "language_loss": 0.87297976, + "learning_rate": 3.993185903693384e-06, + "loss": 0.89648235, + "num_input_tokens_seen": 10011975, + "router_z_loss_clip": 1.98925781, + "router_z_loss_mlp": 0.34399414, + "step": 462, + "time_per_iteration": 2.771583080291748 + }, + { + "auxiliary_loss_clip": 0.01276938, + "auxiliary_loss_mlp": 0.01076198, + "balance_loss_clip": 1.07775092, + "balance_loss_mlp": 1.04210448, + "epoch": 0.05567245836589912, + "flos": 23587098410880.0, + "grad_norm": 2.1517386431657974, + "language_loss": 0.82174194, + "learning_rate": 3.9931215051131995e-06, + "loss": 0.84527332, + "num_input_tokens_seen": 10032620, + "router_z_loss_clip": 1.9921875, + "router_z_loss_mlp": 0.34082031, + "step": 463, + "time_per_iteration": 2.912605047225952 + }, + { + "auxiliary_loss_clip": 0.01274218, + "auxiliary_loss_mlp": 0.01076674, + "balance_loss_clip": 1.07726455, + "balance_loss_mlp": 1.04262757, + "epoch": 0.05579270125653821, + "flos": 27746667129600.0, + "grad_norm": 1.6700184727266096, + "language_loss": 0.79877979, + "learning_rate": 3.993056804178068e-06, + "loss": 0.82228869, + "num_input_tokens_seen": 10054165, + "router_z_loss_clip": 1.96972656, + "router_z_loss_mlp": 0.34057617, + "step": 464, + "time_per_iteration": 2.878239870071411 + }, + { + "auxiliary_loss_clip": 0.01277929, + "auxiliary_loss_mlp": 0.01082227, + "balance_loss_clip": 1.0765698, + "balance_loss_mlp": 1.04908657, + "epoch": 0.0559129441471773, + "flos": 27014161075200.0, + "grad_norm": 2.3026788257274706, + "language_loss": 0.84376413, + "learning_rate": 3.992991800897803e-06, + "loss": 0.86736566, + "num_input_tokens_seen": 10073970, + "router_z_loss_clip": 2.00976562, + "router_z_loss_mlp": 0.33105469, + "step": 465, + "time_per_iteration": 2.875335216522217 + }, + { + "auxiliary_loss_clip": 0.01270116, + "auxiliary_loss_mlp": 0.01077412, + "balance_loss_clip": 1.07521856, + "balance_loss_mlp": 1.04260254, + "epoch": 0.05603318703781639, + "flos": 15229787794560.0, + "grad_norm": 2.4563493631802857, + "language_loss": 0.89427829, + "learning_rate": 3.9929264952822665e-06, + "loss": 0.91775358, + "num_input_tokens_seen": 10091505, + "router_z_loss_clip": 1.94921875, + "router_z_loss_mlp": 0.34814453, + "step": 466, + "time_per_iteration": 2.797480583190918 + }, + { + "auxiliary_loss_clip": 0.01276615, + "auxiliary_loss_mlp": 0.0107829, + "balance_loss_clip": 1.07969904, + "balance_loss_mlp": 1.04493546, + "epoch": 0.05615342992845548, + "flos": 22266482976000.0, + "grad_norm": 2.533353751745479, + "language_loss": 0.8810786, + "learning_rate": 3.992860887341366e-06, + "loss": 0.90462756, + "num_input_tokens_seen": 10109675, + "router_z_loss_clip": 1.96777344, + "router_z_loss_mlp": 0.33349609, + "step": 467, + "time_per_iteration": 2.836127519607544 + }, + { + "auxiliary_loss_clip": 0.01275866, + "auxiliary_loss_mlp": 0.01082807, + "balance_loss_clip": 1.07755482, + "balance_loss_mlp": 1.04814076, + "epoch": 0.056273672819094574, + "flos": 23584979508480.0, + "grad_norm": 1.9331864916839643, + "language_loss": 0.80930388, + "learning_rate": 3.992794977085052e-06, + "loss": 0.83289063, + "num_input_tokens_seen": 10127675, + "router_z_loss_clip": 1.98339844, + "router_z_loss_mlp": 0.34643555, + "step": 468, + "time_per_iteration": 2.8113293647766113 + }, + { + "auxiliary_loss_clip": 0.01279344, + "auxiliary_loss_mlp": 0.01078833, + "balance_loss_clip": 1.07941341, + "balance_loss_mlp": 1.04597914, + "epoch": 0.056393915709733664, + "flos": 19858708552320.0, + "grad_norm": 2.222769516888797, + "language_loss": 0.8470782, + "learning_rate": 3.992728764523326e-06, + "loss": 0.87065995, + "num_input_tokens_seen": 10146620, + "router_z_loss_clip": 1.99804688, + "router_z_loss_mlp": 0.32836914, + "step": 469, + "time_per_iteration": 2.838845729827881 + }, + { + "auxiliary_loss_clip": 0.01277113, + "auxiliary_loss_mlp": 0.0107414, + "balance_loss_clip": 1.08112419, + "balance_loss_mlp": 1.0393784, + "epoch": 0.05651415860037275, + "flos": 22163779013760.0, + "grad_norm": 1.6844342884455035, + "language_loss": 0.8068555, + "learning_rate": 3.99266224966623e-06, + "loss": 0.83036804, + "num_input_tokens_seen": 10167535, + "router_z_loss_clip": 1.9609375, + "router_z_loss_mlp": 0.34790039, + "step": 470, + "time_per_iteration": 2.861034870147705 + }, + { + "auxiliary_loss_clip": 0.01271703, + "auxiliary_loss_mlp": 0.01085255, + "balance_loss_clip": 1.0790329, + "balance_loss_mlp": 1.05011189, + "epoch": 0.05663440149101184, + "flos": 19463548055040.0, + "grad_norm": 1.8740267909004134, + "language_loss": 0.87645262, + "learning_rate": 3.992595432523855e-06, + "loss": 0.90002227, + "num_input_tokens_seen": 10184825, + "router_z_loss_clip": 1.92382812, + "router_z_loss_mlp": 0.35131836, + "step": 471, + "time_per_iteration": 2.838747024536133 + }, + { + "auxiliary_loss_clip": 0.01273501, + "auxiliary_loss_mlp": 0.01076561, + "balance_loss_clip": 1.08051848, + "balance_loss_mlp": 1.04373121, + "epoch": 0.05675464438165093, + "flos": 22670226823680.0, + "grad_norm": 1.8942577305055623, + "language_loss": 0.8587327, + "learning_rate": 3.992528313106338e-06, + "loss": 0.88223332, + "num_input_tokens_seen": 10203025, + "router_z_loss_clip": 1.93164062, + "router_z_loss_mlp": 0.328125, + "step": 472, + "time_per_iteration": 2.841459274291992 + }, + { + "auxiliary_loss_clip": 0.01268739, + "auxiliary_loss_mlp": 0.01070473, + "balance_loss_clip": 1.07743311, + "balance_loss_mlp": 1.03831053, + "epoch": 0.05687488727229002, + "flos": 16901177495040.0, + "grad_norm": 3.6727812297424314, + "language_loss": 0.82301772, + "learning_rate": 3.9924608914238595e-06, + "loss": 0.84640992, + "num_input_tokens_seen": 10218020, + "router_z_loss_clip": 1.91503906, + "router_z_loss_mlp": 0.32177734, + "step": 473, + "time_per_iteration": 2.8094961643218994 + }, + { + "auxiliary_loss_clip": 0.01278172, + "auxiliary_loss_mlp": 0.01072053, + "balance_loss_clip": 1.0794189, + "balance_loss_mlp": 1.03888941, + "epoch": 0.05699513016292912, + "flos": 29168980945920.0, + "grad_norm": 2.4964364602714566, + "language_loss": 0.83754766, + "learning_rate": 3.992393167486648e-06, + "loss": 0.86104989, + "num_input_tokens_seen": 10237170, + "router_z_loss_clip": 1.98925781, + "router_z_loss_mlp": 0.33154297, + "step": 474, + "time_per_iteration": 2.889543056488037 + }, + { + "auxiliary_loss_clip": 0.01280223, + "auxiliary_loss_mlp": 0.01079002, + "balance_loss_clip": 1.08036661, + "balance_loss_mlp": 1.04624319, + "epoch": 0.05711537305356821, + "flos": 18916197632640.0, + "grad_norm": 2.2629103586205135, + "language_loss": 0.80713379, + "learning_rate": 3.992325141304977e-06, + "loss": 0.83072603, + "num_input_tokens_seen": 10255125, + "router_z_loss_clip": 1.99707031, + "router_z_loss_mlp": 0.32739258, + "step": 475, + "time_per_iteration": 2.8372297286987305 + }, + { + "auxiliary_loss_clip": 0.01272103, + "auxiliary_loss_mlp": 0.0106942, + "balance_loss_clip": 1.07859015, + "balance_loss_mlp": 1.03668475, + "epoch": 0.0572356159442073, + "flos": 26758979879040.0, + "grad_norm": 2.348453660531453, + "language_loss": 0.8687458, + "learning_rate": 3.992256812889166e-06, + "loss": 0.89216101, + "num_input_tokens_seen": 10271230, + "router_z_loss_clip": 1.9375, + "router_z_loss_mlp": 0.32739258, + "step": 476, + "time_per_iteration": 2.949171304702759 + }, + { + "auxiliary_loss_clip": 0.0127323, + "auxiliary_loss_mlp": 0.01065347, + "balance_loss_clip": 1.07968569, + "balance_loss_mlp": 1.03606927, + "epoch": 0.05735585883484639, + "flos": 35116146840960.0, + "grad_norm": 2.400126613332942, + "language_loss": 0.76791561, + "learning_rate": 3.992188182249582e-06, + "loss": 0.79130137, + "num_input_tokens_seen": 10293125, + "router_z_loss_clip": 1.93652344, + "router_z_loss_mlp": 0.29260254, + "step": 477, + "time_per_iteration": 3.0331497192382812 + }, + { + "auxiliary_loss_clip": 0.012738, + "auxiliary_loss_mlp": 0.01071663, + "balance_loss_clip": 1.07743382, + "balance_loss_mlp": 1.03864205, + "epoch": 0.05747610172548548, + "flos": 18734381965440.0, + "grad_norm": 2.1959996096980734, + "language_loss": 0.90598488, + "learning_rate": 3.992119249396633e-06, + "loss": 0.92943943, + "num_input_tokens_seen": 10311810, + "router_z_loss_clip": 1.96289062, + "router_z_loss_mlp": 0.33032227, + "step": 478, + "time_per_iteration": 2.8833162784576416 + }, + { + "auxiliary_loss_clip": 0.01271185, + "auxiliary_loss_mlp": 0.01080349, + "balance_loss_clip": 1.07756901, + "balance_loss_mlp": 1.04704177, + "epoch": 0.05759634461612457, + "flos": 27964752554880.0, + "grad_norm": 1.8162181810749514, + "language_loss": 0.82229173, + "learning_rate": 3.992050014340778e-06, + "loss": 0.84580708, + "num_input_tokens_seen": 10332165, + "router_z_loss_clip": 1.93457031, + "router_z_loss_mlp": 0.33325195, + "step": 479, + "time_per_iteration": 2.8458988666534424 + }, + { + "auxiliary_loss_clip": 0.0114683, + "auxiliary_loss_mlp": 0.01026976, + "balance_loss_clip": 1.05537581, + "balance_loss_mlp": 1.01801193, + "epoch": 0.057716587506763666, + "flos": 69292009405440.0, + "grad_norm": 0.8363649296244348, + "language_loss": 0.55018139, + "learning_rate": 3.99198047709252e-06, + "loss": 0.57191944, + "num_input_tokens_seen": 10393685, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.08984375, + "step": 480, + "time_per_iteration": 5.77389931678772 + }, + { + "auxiliary_loss_clip": 0.01268866, + "auxiliary_loss_mlp": 0.01067859, + "balance_loss_clip": 1.07404447, + "balance_loss_mlp": 1.03450441, + "epoch": 0.057836830397402755, + "flos": 25009196745600.0, + "grad_norm": 1.973263700152789, + "language_loss": 0.78669798, + "learning_rate": 3.991910637662408e-06, + "loss": 0.81006521, + "num_input_tokens_seen": 10413975, + "router_z_loss_clip": 1.94824219, + "router_z_loss_mlp": 0.33349609, + "step": 481, + "time_per_iteration": 2.8270814418792725 + }, + { + "auxiliary_loss_clip": 0.01266398, + "auxiliary_loss_mlp": 0.01087183, + "balance_loss_clip": 1.0769676, + "balance_loss_mlp": 1.05542564, + "epoch": 0.057957073288041845, + "flos": 25593894334080.0, + "grad_norm": 1.8149066683306188, + "language_loss": 0.8079356, + "learning_rate": 3.9918404960610355e-06, + "loss": 0.83147138, + "num_input_tokens_seen": 10433005, + "router_z_loss_clip": 1.89453125, + "router_z_loss_mlp": 0.31762695, + "step": 482, + "time_per_iteration": 2.840852737426758 + }, + { + "auxiliary_loss_clip": 0.01270672, + "auxiliary_loss_mlp": 0.01071358, + "balance_loss_clip": 1.07794678, + "balance_loss_mlp": 1.03709745, + "epoch": 0.058077316178680935, + "flos": 20777411733120.0, + "grad_norm": 2.2524842904574425, + "language_loss": 0.77611172, + "learning_rate": 3.991770052299043e-06, + "loss": 0.79953206, + "num_input_tokens_seen": 10451235, + "router_z_loss_clip": 1.92773438, + "router_z_loss_mlp": 0.34277344, + "step": 483, + "time_per_iteration": 2.7782089710235596 + }, + { + "auxiliary_loss_clip": 0.01270136, + "auxiliary_loss_mlp": 0.01072571, + "balance_loss_clip": 1.07806146, + "balance_loss_mlp": 1.03981185, + "epoch": 0.058197559069320025, + "flos": 18916484941440.0, + "grad_norm": 2.1700110472392056, + "language_loss": 0.87353313, + "learning_rate": 3.991699306387118e-06, + "loss": 0.8969602, + "num_input_tokens_seen": 10469705, + "router_z_loss_clip": 1.91992188, + "router_z_loss_mlp": 0.32763672, + "step": 484, + "time_per_iteration": 2.7750608921051025 + }, + { + "auxiliary_loss_clip": 0.01270944, + "auxiliary_loss_mlp": 0.01074746, + "balance_loss_clip": 1.07762289, + "balance_loss_mlp": 1.04432368, + "epoch": 0.058317801959959115, + "flos": 24863327614080.0, + "grad_norm": 1.7783472616558582, + "language_loss": 0.78147393, + "learning_rate": 3.991628258335991e-06, + "loss": 0.80493081, + "num_input_tokens_seen": 10491910, + "router_z_loss_clip": 1.93457031, + "router_z_loss_mlp": 0.30407715, + "step": 485, + "time_per_iteration": 2.808518886566162 + }, + { + "auxiliary_loss_clip": 0.01273768, + "auxiliary_loss_mlp": 0.01080858, + "balance_loss_clip": 1.07969213, + "balance_loss_mlp": 1.04688358, + "epoch": 0.05843804485059821, + "flos": 23257977068160.0, + "grad_norm": 2.694445647765252, + "language_loss": 0.87732041, + "learning_rate": 3.991556908156442e-06, + "loss": 0.90086669, + "num_input_tokens_seen": 10508435, + "router_z_loss_clip": 1.9375, + "router_z_loss_mlp": 0.33984375, + "step": 486, + "time_per_iteration": 2.8263001441955566 + }, + { + "auxiliary_loss_clip": 0.01270678, + "auxiliary_loss_mlp": 0.01073928, + "balance_loss_clip": 1.07503867, + "balance_loss_mlp": 1.04181349, + "epoch": 0.0585582877412373, + "flos": 23150532510720.0, + "grad_norm": 2.062064248156741, + "language_loss": 0.8770678, + "learning_rate": 3.9914852558592914e-06, + "loss": 0.90051389, + "num_input_tokens_seen": 10529485, + "router_z_loss_clip": 1.95507812, + "router_z_loss_mlp": 0.32128906, + "step": 487, + "time_per_iteration": 2.840501070022583 + }, + { + "auxiliary_loss_clip": 0.01273603, + "auxiliary_loss_mlp": 0.01073008, + "balance_loss_clip": 1.07694566, + "balance_loss_mlp": 1.0383662, + "epoch": 0.05867853063187639, + "flos": 23506406507520.0, + "grad_norm": 3.817268321103502, + "language_loss": 0.81233984, + "learning_rate": 3.991413301455413e-06, + "loss": 0.83580595, + "num_input_tokens_seen": 10545935, + "router_z_loss_clip": 1.96582031, + "router_z_loss_mlp": 0.34667969, + "step": 488, + "time_per_iteration": 2.9035849571228027 + }, + { + "auxiliary_loss_clip": 0.0126488, + "auxiliary_loss_mlp": 0.01077607, + "balance_loss_clip": 1.07261717, + "balance_loss_mlp": 1.04580164, + "epoch": 0.05879877352251548, + "flos": 29495803818240.0, + "grad_norm": 2.7391151770751794, + "language_loss": 0.77767062, + "learning_rate": 3.991341044955719e-06, + "loss": 0.80109543, + "num_input_tokens_seen": 10565690, + "router_z_loss_clip": 1.92480469, + "router_z_loss_mlp": 0.31787109, + "step": 489, + "time_per_iteration": 2.8730199337005615 + }, + { + "auxiliary_loss_clip": 0.01270851, + "auxiliary_loss_mlp": 0.01076618, + "balance_loss_clip": 1.07652962, + "balance_loss_mlp": 1.0430491, + "epoch": 0.05891901641315457, + "flos": 20157485880960.0, + "grad_norm": 2.0052496951255234, + "language_loss": 0.81662655, + "learning_rate": 3.991268486371172e-06, + "loss": 0.8401013, + "num_input_tokens_seen": 10584245, + "router_z_loss_clip": 1.94335938, + "router_z_loss_mlp": 0.33569336, + "step": 490, + "time_per_iteration": 2.826496124267578 + }, + { + "auxiliary_loss_clip": 0.01272741, + "auxiliary_loss_mlp": 0.01067076, + "balance_loss_clip": 1.07512105, + "balance_loss_mlp": 1.03412652, + "epoch": 0.05903925930379366, + "flos": 24644200694400.0, + "grad_norm": 2.6565598177474725, + "language_loss": 0.88229179, + "learning_rate": 3.991195625712779e-06, + "loss": 0.90568995, + "num_input_tokens_seen": 10601210, + "router_z_loss_clip": 1.97558594, + "router_z_loss_mlp": 0.3293457, + "step": 491, + "time_per_iteration": 2.891686201095581 + }, + { + "auxiliary_loss_clip": 0.01265669, + "auxiliary_loss_mlp": 0.01060284, + "balance_loss_clip": 1.07568407, + "balance_loss_mlp": 1.02983761, + "epoch": 0.05915950219443276, + "flos": 21250391045760.0, + "grad_norm": 2.2552318154064577, + "language_loss": 0.81541574, + "learning_rate": 3.991122462991592e-06, + "loss": 0.83867532, + "num_input_tokens_seen": 10620730, + "router_z_loss_clip": 1.90234375, + "router_z_loss_mlp": 0.3046875, + "step": 492, + "time_per_iteration": 2.8551905155181885 + }, + { + "auxiliary_loss_clip": 0.0127468, + "auxiliary_loss_mlp": 0.01076047, + "balance_loss_clip": 1.07597613, + "balance_loss_mlp": 1.04331255, + "epoch": 0.05927974508507185, + "flos": 9902727319680.0, + "grad_norm": 3.617697477910007, + "language_loss": 0.81136853, + "learning_rate": 3.991048998218712e-06, + "loss": 0.83487582, + "num_input_tokens_seen": 10634035, + "router_z_loss_clip": 1.98828125, + "router_z_loss_mlp": 0.32739258, + "step": 493, + "time_per_iteration": 2.8374388217926025 + }, + { + "auxiliary_loss_clip": 0.01266349, + "auxiliary_loss_mlp": 0.01084958, + "balance_loss_clip": 1.07593155, + "balance_loss_mlp": 1.052724, + "epoch": 0.05939998797571094, + "flos": 18259499232000.0, + "grad_norm": 2.1547399052227605, + "language_loss": 0.76772612, + "learning_rate": 3.990975231405281e-06, + "loss": 0.79123914, + "num_input_tokens_seen": 10652485, + "router_z_loss_clip": 1.90527344, + "router_z_loss_mlp": 0.32226562, + "step": 494, + "time_per_iteration": 2.8534224033355713 + }, + { + "auxiliary_loss_clip": 0.01269107, + "auxiliary_loss_mlp": 0.01079503, + "balance_loss_clip": 1.07903624, + "balance_loss_mlp": 1.04619575, + "epoch": 0.05952023086635003, + "flos": 28256598558720.0, + "grad_norm": 1.8479562666967333, + "language_loss": 0.78803694, + "learning_rate": 3.990901162562491e-06, + "loss": 0.81152308, + "num_input_tokens_seen": 10673175, + "router_z_loss_clip": 1.90039062, + "router_z_loss_mlp": 0.33325195, + "step": 495, + "time_per_iteration": 2.8958795070648193 + }, + { + "auxiliary_loss_clip": 0.01260001, + "auxiliary_loss_mlp": 0.01078588, + "balance_loss_clip": 1.07373893, + "balance_loss_mlp": 1.04513824, + "epoch": 0.05964047375698912, + "flos": 14902498045440.0, + "grad_norm": 1.944714994178927, + "language_loss": 0.90840805, + "learning_rate": 3.9908267917015765e-06, + "loss": 0.93179405, + "num_input_tokens_seen": 10691235, + "router_z_loss_clip": 1.86230469, + "router_z_loss_mlp": 0.33447266, + "step": 496, + "time_per_iteration": 2.9456915855407715 + }, + { + "auxiliary_loss_clip": 0.01269861, + "auxiliary_loss_mlp": 0.01091272, + "balance_loss_clip": 1.07639027, + "balance_loss_mlp": 1.05636752, + "epoch": 0.059760716647628206, + "flos": 23185581206400.0, + "grad_norm": 2.6378108830927696, + "language_loss": 0.92953008, + "learning_rate": 3.990752118833821e-06, + "loss": 0.95314145, + "num_input_tokens_seen": 10708675, + "router_z_loss_clip": 1.93554688, + "router_z_loss_mlp": 0.34887695, + "step": 497, + "time_per_iteration": 3.0237793922424316 + }, + { + "auxiliary_loss_clip": 0.01277973, + "auxiliary_loss_mlp": 0.01071356, + "balance_loss_clip": 1.08119524, + "balance_loss_mlp": 1.03773868, + "epoch": 0.0598809595382673, + "flos": 22746968231040.0, + "grad_norm": 2.8647040491769635, + "language_loss": 0.77859354, + "learning_rate": 3.990677143970553e-06, + "loss": 0.80208683, + "num_input_tokens_seen": 10729485, + "router_z_loss_clip": 1.97167969, + "router_z_loss_mlp": 0.33642578, + "step": 498, + "time_per_iteration": 3.0364158153533936 + }, + { + "auxiliary_loss_clip": 0.01270124, + "auxiliary_loss_mlp": 0.01081837, + "balance_loss_clip": 1.07691741, + "balance_loss_mlp": 1.04819655, + "epoch": 0.06000120242890639, + "flos": 22127221946880.0, + "grad_norm": 2.2364662546776444, + "language_loss": 0.81191868, + "learning_rate": 3.990601867123144e-06, + "loss": 0.83543831, + "num_input_tokens_seen": 10749210, + "router_z_loss_clip": 1.93359375, + "router_z_loss_mlp": 0.33618164, + "step": 499, + "time_per_iteration": 2.8679990768432617 + }, + { + "auxiliary_loss_clip": 0.0126647, + "auxiliary_loss_mlp": 0.01077735, + "balance_loss_clip": 1.07670927, + "balance_loss_mlp": 1.04359341, + "epoch": 0.06012144531954548, + "flos": 19171773878400.0, + "grad_norm": 2.4617239175794405, + "language_loss": 0.85091126, + "learning_rate": 3.990526288303014e-06, + "loss": 0.87435329, + "num_input_tokens_seen": 10768000, + "router_z_loss_clip": 1.89648438, + "router_z_loss_mlp": 0.34130859, + "step": 500, + "time_per_iteration": 2.8320066928863525 + }, + { + "auxiliary_loss_clip": 0.01258592, + "auxiliary_loss_mlp": 0.01072236, + "balance_loss_clip": 1.07585645, + "balance_loss_mlp": 1.0378083, + "epoch": 0.06024168821018457, + "flos": 22783345729920.0, + "grad_norm": 1.7735605767621174, + "language_loss": 0.90736729, + "learning_rate": 3.9904504075216295e-06, + "loss": 0.93067551, + "num_input_tokens_seen": 10788760, + "router_z_loss_clip": 1.828125, + "router_z_loss_mlp": 0.34448242, + "step": 501, + "time_per_iteration": 2.877471923828125 + }, + { + "auxiliary_loss_clip": 0.012747, + "auxiliary_loss_mlp": 0.01079613, + "balance_loss_clip": 1.07604289, + "balance_loss_mlp": 1.04647231, + "epoch": 0.06036193110082366, + "flos": 18770687637120.0, + "grad_norm": 2.348353991557166, + "language_loss": 0.9389019, + "learning_rate": 3.990374224790501e-06, + "loss": 0.96244502, + "num_input_tokens_seen": 10806965, + "router_z_loss_clip": 1.98632812, + "router_z_loss_mlp": 0.33129883, + "step": 502, + "time_per_iteration": 2.880167007446289 + }, + { + "auxiliary_loss_clip": 0.01271268, + "auxiliary_loss_mlp": 0.01084612, + "balance_loss_clip": 1.07860231, + "balance_loss_mlp": 1.05171001, + "epoch": 0.06048217399146275, + "flos": 17201570935680.0, + "grad_norm": 2.1186735051982444, + "language_loss": 0.70977837, + "learning_rate": 3.990297740121185e-06, + "loss": 0.73333716, + "num_input_tokens_seen": 10824900, + "router_z_loss_clip": 1.92675781, + "router_z_loss_mlp": 0.32861328, + "step": 503, + "time_per_iteration": 2.7999322414398193 + }, + { + "auxiliary_loss_clip": 0.01270479, + "auxiliary_loss_mlp": 0.01079871, + "balance_loss_clip": 1.07689309, + "balance_loss_mlp": 1.04829276, + "epoch": 0.06060241688210185, + "flos": 24024131187840.0, + "grad_norm": 1.7331602674849498, + "language_loss": 0.78221971, + "learning_rate": 3.990220953525284e-06, + "loss": 0.80572325, + "num_input_tokens_seen": 10842010, + "router_z_loss_clip": 1.93457031, + "router_z_loss_mlp": 0.31604004, + "step": 504, + "time_per_iteration": 2.8248400688171387 + }, + { + "auxiliary_loss_clip": 0.01268584, + "auxiliary_loss_mlp": 0.01073069, + "balance_loss_clip": 1.07395685, + "balance_loss_mlp": 1.03909492, + "epoch": 0.06072265977274094, + "flos": 14611190745600.0, + "grad_norm": 2.526980551684787, + "language_loss": 0.74510139, + "learning_rate": 3.9901438650144465e-06, + "loss": 0.76851797, + "num_input_tokens_seen": 10858260, + "router_z_loss_clip": 1.94628906, + "router_z_loss_mlp": 0.33935547, + "step": 505, + "time_per_iteration": 2.8474555015563965 + }, + { + "auxiliary_loss_clip": 0.01266568, + "auxiliary_loss_mlp": 0.01077147, + "balance_loss_clip": 1.07531333, + "balance_loss_mlp": 1.0460577, + "epoch": 0.06084290266338003, + "flos": 20558284813440.0, + "grad_norm": 2.186464706231832, + "language_loss": 0.91692263, + "learning_rate": 3.990066474600367e-06, + "loss": 0.94035977, + "num_input_tokens_seen": 10876230, + "router_z_loss_clip": 1.91113281, + "router_z_loss_mlp": 0.31054688, + "step": 506, + "time_per_iteration": 2.797410249710083 + }, + { + "auxiliary_loss_clip": 0.01266504, + "auxiliary_loss_mlp": 0.01075637, + "balance_loss_clip": 1.07523835, + "balance_loss_mlp": 1.04269934, + "epoch": 0.06096314555401912, + "flos": 22309217182080.0, + "grad_norm": 1.8734477469774513, + "language_loss": 0.67752957, + "learning_rate": 3.989988782294786e-06, + "loss": 0.70095098, + "num_input_tokens_seen": 10896320, + "router_z_loss_clip": 1.91015625, + "router_z_loss_mlp": 0.32971191, + "step": 507, + "time_per_iteration": 2.8435580730438232 + }, + { + "auxiliary_loss_clip": 0.01259036, + "auxiliary_loss_mlp": 0.01070648, + "balance_loss_clip": 1.07190537, + "balance_loss_mlp": 1.0378654, + "epoch": 0.06108338844465821, + "flos": 19131374056320.0, + "grad_norm": 2.1218548699706465, + "language_loss": 0.94884551, + "learning_rate": 3.989910788109489e-06, + "loss": 0.97214234, + "num_input_tokens_seen": 10912970, + "router_z_loss_clip": 1.87011719, + "router_z_loss_mlp": 0.32763672, + "step": 508, + "time_per_iteration": 3.682764768600464 + }, + { + "auxiliary_loss_clip": 0.01266152, + "auxiliary_loss_mlp": 0.01076204, + "balance_loss_clip": 1.07557034, + "balance_loss_mlp": 1.04339719, + "epoch": 0.0612036313352973, + "flos": 33584018169600.0, + "grad_norm": 2.465526647527464, + "language_loss": 0.74597842, + "learning_rate": 3.989832492056307e-06, + "loss": 0.76940197, + "num_input_tokens_seen": 10933995, + "router_z_loss_clip": 1.90429688, + "router_z_loss_mlp": 0.32788086, + "step": 509, + "time_per_iteration": 5.507188320159912 + }, + { + "auxiliary_loss_clip": 0.01266086, + "auxiliary_loss_mlp": 0.01076631, + "balance_loss_clip": 1.07395244, + "balance_loss_mlp": 1.04258513, + "epoch": 0.06132387422593639, + "flos": 27490552179840.0, + "grad_norm": 2.807959808373916, + "language_loss": 0.80561042, + "learning_rate": 3.989753894147119e-06, + "loss": 0.82903755, + "num_input_tokens_seen": 10954120, + "router_z_loss_clip": 1.92285156, + "router_z_loss_mlp": 0.34057617, + "step": 510, + "time_per_iteration": 2.8900253772735596 + }, + { + "auxiliary_loss_clip": 0.01263259, + "auxiliary_loss_mlp": 0.01084092, + "balance_loss_clip": 1.08027697, + "balance_loss_mlp": 1.05192935, + "epoch": 0.061444117116575485, + "flos": 25885057979520.0, + "grad_norm": 1.725479457467174, + "language_loss": 0.79929566, + "learning_rate": 3.989674994393846e-06, + "loss": 0.82276917, + "num_input_tokens_seen": 10973595, + "router_z_loss_clip": 1.828125, + "router_z_loss_mlp": 0.3215332, + "step": 511, + "time_per_iteration": 2.914921760559082 + }, + { + "auxiliary_loss_clip": 0.01259899, + "auxiliary_loss_mlp": 0.01063206, + "balance_loss_clip": 1.07514524, + "balance_loss_mlp": 1.03030443, + "epoch": 0.061564360007214575, + "flos": 28512031150080.0, + "grad_norm": 2.4716296689780894, + "language_loss": 0.93863976, + "learning_rate": 3.98959579280846e-06, + "loss": 0.96187079, + "num_input_tokens_seen": 10991995, + "router_z_loss_clip": 1.84863281, + "router_z_loss_mlp": 0.32910156, + "step": 512, + "time_per_iteration": 2.920933723449707 + }, + { + "auxiliary_loss_clip": 0.01267406, + "auxiliary_loss_mlp": 0.01071222, + "balance_loss_clip": 1.07952261, + "balance_loss_mlp": 1.04165816, + "epoch": 0.061684602897853665, + "flos": 12094355652480.0, + "grad_norm": 2.2766629039932034, + "language_loss": 0.82847047, + "learning_rate": 3.989516289402973e-06, + "loss": 0.85185677, + "num_input_tokens_seen": 11007625, + "router_z_loss_clip": 1.87890625, + "router_z_loss_mlp": 0.2956543, + "step": 513, + "time_per_iteration": 2.8173434734344482 + }, + { + "auxiliary_loss_clip": 0.01261315, + "auxiliary_loss_mlp": 0.01067833, + "balance_loss_clip": 1.07599807, + "balance_loss_mlp": 1.03493154, + "epoch": 0.061804845788492754, + "flos": 19532639865600.0, + "grad_norm": 2.2404858634721836, + "language_loss": 0.80234087, + "learning_rate": 3.989436484189447e-06, + "loss": 0.82563233, + "num_input_tokens_seen": 11025570, + "router_z_loss_clip": 1.8515625, + "router_z_loss_mlp": 0.32910156, + "step": 514, + "time_per_iteration": 2.8711466789245605 + }, + { + "auxiliary_loss_clip": 0.01263666, + "auxiliary_loss_mlp": 0.01075063, + "balance_loss_clip": 1.07783103, + "balance_loss_mlp": 1.04278088, + "epoch": 0.061925088679131844, + "flos": 15341111020800.0, + "grad_norm": 2.59696026957107, + "language_loss": 0.806651, + "learning_rate": 3.9893563771799885e-06, + "loss": 0.83003831, + "num_input_tokens_seen": 11042045, + "router_z_loss_clip": 1.86035156, + "router_z_loss_mlp": 0.32275391, + "step": 515, + "time_per_iteration": 2.834165573120117 + }, + { + "auxiliary_loss_clip": 0.01268567, + "auxiliary_loss_mlp": 0.01069422, + "balance_loss_clip": 1.07781291, + "balance_loss_mlp": 1.03682971, + "epoch": 0.062045331569770934, + "flos": 25919927107200.0, + "grad_norm": 2.2651200119789077, + "language_loss": 0.86371875, + "learning_rate": 3.989275968386749e-06, + "loss": 0.88709867, + "num_input_tokens_seen": 11059955, + "router_z_loss_clip": 1.90917969, + "router_z_loss_mlp": 0.32543945, + "step": 516, + "time_per_iteration": 2.8714890480041504 + }, + { + "auxiliary_loss_clip": 0.01260287, + "auxiliary_loss_mlp": 0.01066298, + "balance_loss_clip": 1.07555056, + "balance_loss_mlp": 1.03434992, + "epoch": 0.06216557446041003, + "flos": 28110621686400.0, + "grad_norm": 2.4260078944471064, + "language_loss": 0.76738417, + "learning_rate": 3.989195257821926e-06, + "loss": 0.79065001, + "num_input_tokens_seen": 11078440, + "router_z_loss_clip": 1.84960938, + "router_z_loss_mlp": 0.31933594, + "step": 517, + "time_per_iteration": 2.8795382976531982 + }, + { + "auxiliary_loss_clip": 0.01262798, + "auxiliary_loss_mlp": 0.01071253, + "balance_loss_clip": 1.07439578, + "balance_loss_mlp": 1.03787422, + "epoch": 0.06228581735104912, + "flos": 23478181395840.0, + "grad_norm": 2.041417394838123, + "language_loss": 0.84276837, + "learning_rate": 3.989114245497765e-06, + "loss": 0.86610889, + "num_input_tokens_seen": 11098240, + "router_z_loss_clip": 1.88574219, + "router_z_loss_mlp": 0.33349609, + "step": 518, + "time_per_iteration": 2.889742136001587 + }, + { + "auxiliary_loss_clip": 0.01261659, + "auxiliary_loss_mlp": 0.01065551, + "balance_loss_clip": 1.07391858, + "balance_loss_mlp": 1.03343582, + "epoch": 0.06240606024168821, + "flos": 15195205975680.0, + "grad_norm": 2.304851179395053, + "language_loss": 0.94988161, + "learning_rate": 3.989032931426554e-06, + "loss": 0.97315371, + "num_input_tokens_seen": 11115395, + "router_z_loss_clip": 1.87695312, + "router_z_loss_mlp": 0.32104492, + "step": 519, + "time_per_iteration": 2.8961257934570312 + }, + { + "auxiliary_loss_clip": 0.01258703, + "auxiliary_loss_mlp": 0.01069038, + "balance_loss_clip": 1.07264042, + "balance_loss_mlp": 1.03816259, + "epoch": 0.06252630313232731, + "flos": 20631829910400.0, + "grad_norm": 2.4427265486743273, + "language_loss": 0.86923563, + "learning_rate": 3.9889513156206295e-06, + "loss": 0.89251304, + "num_input_tokens_seen": 11134835, + "router_z_loss_clip": 1.86132812, + "router_z_loss_mlp": 0.30883789, + "step": 520, + "time_per_iteration": 2.888510227203369 + }, + { + "auxiliary_loss_clip": 0.01266839, + "auxiliary_loss_mlp": 0.01078354, + "balance_loss_clip": 1.07699168, + "balance_loss_mlp": 1.0452143, + "epoch": 0.06264654602296639, + "flos": 20778058177920.0, + "grad_norm": 2.8471907274732, + "language_loss": 0.73907983, + "learning_rate": 3.988869398092371e-06, + "loss": 0.76253176, + "num_input_tokens_seen": 11154745, + "router_z_loss_clip": 1.89648438, + "router_z_loss_mlp": 0.33178711, + "step": 521, + "time_per_iteration": 2.852036714553833 + }, + { + "auxiliary_loss_clip": 0.01267309, + "auxiliary_loss_mlp": 0.01074363, + "balance_loss_clip": 1.07508135, + "balance_loss_mlp": 1.04010201, + "epoch": 0.06276678891360549, + "flos": 29605798241280.0, + "grad_norm": 2.1935080924284853, + "language_loss": 0.78819335, + "learning_rate": 3.988787178854206e-06, + "loss": 0.81161004, + "num_input_tokens_seen": 11174280, + "router_z_loss_clip": 1.92285156, + "router_z_loss_mlp": 0.34228516, + "step": 522, + "time_per_iteration": 3.0284512042999268 + }, + { + "auxiliary_loss_clip": 0.01263237, + "auxiliary_loss_mlp": 0.01064838, + "balance_loss_clip": 1.07477641, + "balance_loss_mlp": 1.03091156, + "epoch": 0.06288703180424457, + "flos": 22126288193280.0, + "grad_norm": 5.006342079953184, + "language_loss": 0.87562406, + "learning_rate": 3.988704657918608e-06, + "loss": 0.89890486, + "num_input_tokens_seen": 11193340, + "router_z_loss_clip": 1.88476562, + "router_z_loss_mlp": 0.33935547, + "step": 523, + "time_per_iteration": 2.8953938484191895 + }, + { + "auxiliary_loss_clip": 0.01259095, + "auxiliary_loss_mlp": 0.01070076, + "balance_loss_clip": 1.0760417, + "balance_loss_mlp": 1.04041719, + "epoch": 0.06300727469488367, + "flos": 14976689587200.0, + "grad_norm": 3.0352308825750764, + "language_loss": 0.79438186, + "learning_rate": 3.988621835298094e-06, + "loss": 0.81767356, + "num_input_tokens_seen": 11210555, + "router_z_loss_clip": 1.83105469, + "router_z_loss_mlp": 0.29663086, + "step": 524, + "time_per_iteration": 2.8022327423095703 + }, + { + "auxiliary_loss_clip": 0.01259066, + "auxiliary_loss_mlp": 0.01070458, + "balance_loss_clip": 1.07468212, + "balance_loss_mlp": 1.03929615, + "epoch": 0.06312751758552275, + "flos": 24535391420160.0, + "grad_norm": 2.05574556380029, + "language_loss": 0.91290814, + "learning_rate": 3.988538711005229e-06, + "loss": 0.93620336, + "num_input_tokens_seen": 11230010, + "router_z_loss_clip": 1.84179688, + "router_z_loss_mlp": 0.31152344, + "step": 525, + "time_per_iteration": 2.82475209236145 + }, + { + "auxiliary_loss_clip": 0.01258393, + "auxiliary_loss_mlp": 0.01073552, + "balance_loss_clip": 1.07431817, + "balance_loss_mlp": 1.04229486, + "epoch": 0.06324776047616185, + "flos": 21507008785920.0, + "grad_norm": 2.2169185439993546, + "language_loss": 0.8783195, + "learning_rate": 3.988455285052622e-06, + "loss": 0.90163898, + "num_input_tokens_seen": 11246190, + "router_z_loss_clip": 1.83984375, + "router_z_loss_mlp": 0.31225586, + "step": 526, + "time_per_iteration": 2.8322200775146484 + }, + { + "auxiliary_loss_clip": 0.01259086, + "auxiliary_loss_mlp": 0.01064213, + "balance_loss_clip": 1.07638669, + "balance_loss_mlp": 1.03383839, + "epoch": 0.06336800336680094, + "flos": 21688034353920.0, + "grad_norm": 3.042497174425248, + "language_loss": 0.84186792, + "learning_rate": 3.98837155745293e-06, + "loss": 0.86510092, + "num_input_tokens_seen": 11264230, + "router_z_loss_clip": 1.82617188, + "router_z_loss_mlp": 0.30358887, + "step": 527, + "time_per_iteration": 2.796778440475464 + }, + { + "auxiliary_loss_clip": 0.01262472, + "auxiliary_loss_mlp": 0.01066038, + "balance_loss_clip": 1.07463145, + "balance_loss_mlp": 1.03466249, + "epoch": 0.06348824625744003, + "flos": 19500895221120.0, + "grad_norm": 2.5596253552097084, + "language_loss": 0.76315755, + "learning_rate": 3.988287528218854e-06, + "loss": 0.78644264, + "num_input_tokens_seen": 11283015, + "router_z_loss_clip": 1.87988281, + "router_z_loss_mlp": 0.31347656, + "step": 528, + "time_per_iteration": 2.818898916244507 + }, + { + "auxiliary_loss_clip": 0.01259272, + "auxiliary_loss_mlp": 0.01074187, + "balance_loss_clip": 1.07771599, + "balance_loss_mlp": 1.04402697, + "epoch": 0.06360848914807912, + "flos": 15481233976320.0, + "grad_norm": 2.0337799461432677, + "language_loss": 0.90586245, + "learning_rate": 3.98820319736314e-06, + "loss": 0.92919707, + "num_input_tokens_seen": 11299630, + "router_z_loss_clip": 1.81835938, + "router_z_loss_mlp": 0.30175781, + "step": 529, + "time_per_iteration": 2.8528764247894287 + }, + { + "auxiliary_loss_clip": 0.01256879, + "auxiliary_loss_mlp": 0.01060441, + "balance_loss_clip": 1.07193518, + "balance_loss_mlp": 1.03061438, + "epoch": 0.0637287320387182, + "flos": 20593369422720.0, + "grad_norm": 1.6811222588758306, + "language_loss": 0.85470182, + "learning_rate": 3.988118564898582e-06, + "loss": 0.87787497, + "num_input_tokens_seen": 11319170, + "router_z_loss_clip": 1.84960938, + "router_z_loss_mlp": 0.2980957, + "step": 530, + "time_per_iteration": 2.936223030090332 + }, + { + "auxiliary_loss_clip": 0.01275521, + "auxiliary_loss_mlp": 0.01073545, + "balance_loss_clip": 1.08200383, + "balance_loss_mlp": 1.03966618, + "epoch": 0.0638489749293573, + "flos": 17412222245760.0, + "grad_norm": 3.194479424976505, + "language_loss": 0.8951956, + "learning_rate": 3.988033630838019e-06, + "loss": 0.91868627, + "num_input_tokens_seen": 11333210, + "router_z_loss_clip": 1.93554688, + "router_z_loss_mlp": 0.33886719, + "step": 531, + "time_per_iteration": 2.8134727478027344 + }, + { + "auxiliary_loss_clip": 0.01263215, + "auxiliary_loss_mlp": 0.01065451, + "balance_loss_clip": 1.08042431, + "balance_loss_mlp": 1.03464723, + "epoch": 0.0639692178199964, + "flos": 23807661874560.0, + "grad_norm": 1.7661662259800532, + "language_loss": 0.87877381, + "learning_rate": 3.987948395194334e-06, + "loss": 0.90206051, + "num_input_tokens_seen": 11355590, + "router_z_loss_clip": 1.828125, + "router_z_loss_mlp": 0.30810547, + "step": 532, + "time_per_iteration": 2.8909807205200195 + }, + { + "auxiliary_loss_clip": 0.01252165, + "auxiliary_loss_mlp": 0.01076311, + "balance_loss_clip": 1.07215214, + "balance_loss_mlp": 1.04576921, + "epoch": 0.06408946071063548, + "flos": 18477225521280.0, + "grad_norm": 3.068892725138999, + "language_loss": 0.76620829, + "learning_rate": 3.987862857980458e-06, + "loss": 0.78949302, + "num_input_tokens_seen": 11371535, + "router_z_loss_clip": 1.80175781, + "router_z_loss_mlp": 0.30541992, + "step": 533, + "time_per_iteration": 2.897017002105713 + }, + { + "auxiliary_loss_clip": 0.01263167, + "auxiliary_loss_mlp": 0.01069263, + "balance_loss_clip": 1.07698536, + "balance_loss_mlp": 1.039294, + "epoch": 0.06420970360127458, + "flos": 27162220936320.0, + "grad_norm": 2.122866564860215, + "language_loss": 0.76901054, + "learning_rate": 3.987777019209368e-06, + "loss": 0.79233491, + "num_input_tokens_seen": 11392050, + "router_z_loss_clip": 1.86035156, + "router_z_loss_mlp": 0.29943848, + "step": 534, + "time_per_iteration": 2.873312473297119 + }, + { + "auxiliary_loss_clip": 0.01261141, + "auxiliary_loss_mlp": 0.01081025, + "balance_loss_clip": 1.07776427, + "balance_loss_mlp": 1.04814744, + "epoch": 0.06432994649191366, + "flos": 23659673840640.0, + "grad_norm": 2.025295500530719, + "language_loss": 0.81263256, + "learning_rate": 3.987690878894084e-06, + "loss": 0.83605415, + "num_input_tokens_seen": 11411765, + "router_z_loss_clip": 1.83203125, + "router_z_loss_mlp": 0.32836914, + "step": 535, + "time_per_iteration": 2.867352247238159 + }, + { + "auxiliary_loss_clip": 0.01264371, + "auxiliary_loss_mlp": 0.01071234, + "balance_loss_clip": 1.07667804, + "balance_loss_mlp": 1.03716445, + "epoch": 0.06445018938255276, + "flos": 23403953940480.0, + "grad_norm": 2.5230780601745586, + "language_loss": 0.84881234, + "learning_rate": 3.987604437047673e-06, + "loss": 0.87216836, + "num_input_tokens_seen": 11431565, + "router_z_loss_clip": 1.87597656, + "router_z_loss_mlp": 0.34057617, + "step": 536, + "time_per_iteration": 2.922762155532837 + }, + { + "auxiliary_loss_clip": 0.01257188, + "auxiliary_loss_mlp": 0.01076642, + "balance_loss_clip": 1.07517993, + "balance_loss_mlp": 1.04528952, + "epoch": 0.06457043227319184, + "flos": 19646692525440.0, + "grad_norm": 2.1408141868328285, + "language_loss": 0.77914691, + "learning_rate": 3.987517693683251e-06, + "loss": 0.80248523, + "num_input_tokens_seen": 11450140, + "router_z_loss_clip": 1.82128906, + "router_z_loss_mlp": 0.31323242, + "step": 537, + "time_per_iteration": 3.847666025161743 + }, + { + "auxiliary_loss_clip": 0.01255491, + "auxiliary_loss_mlp": 0.01083321, + "balance_loss_clip": 1.07364893, + "balance_loss_mlp": 1.05339956, + "epoch": 0.06469067516383094, + "flos": 16978744915200.0, + "grad_norm": 2.643373645594482, + "language_loss": 0.96084094, + "learning_rate": 3.9874306488139745e-06, + "loss": 0.98422909, + "num_input_tokens_seen": 11465400, + "router_z_loss_clip": 1.81933594, + "router_z_loss_mlp": 0.29907227, + "step": 538, + "time_per_iteration": 3.657226800918579 + }, + { + "auxiliary_loss_clip": 0.01260983, + "auxiliary_loss_mlp": 0.01076052, + "balance_loss_clip": 1.07638478, + "balance_loss_mlp": 1.04329371, + "epoch": 0.06481091805447003, + "flos": 23296401642240.0, + "grad_norm": 2.011147402016149, + "language_loss": 0.87947738, + "learning_rate": 3.987343302453049e-06, + "loss": 0.90284771, + "num_input_tokens_seen": 11486675, + "router_z_loss_clip": 1.84570312, + "router_z_loss_mlp": 0.32788086, + "step": 539, + "time_per_iteration": 2.834256649017334 + }, + { + "auxiliary_loss_clip": 0.01260077, + "auxiliary_loss_mlp": 0.01074987, + "balance_loss_clip": 1.07599461, + "balance_loss_mlp": 1.04619813, + "epoch": 0.06493116094510912, + "flos": 29172356824320.0, + "grad_norm": 1.6284611326107978, + "language_loss": 0.82507706, + "learning_rate": 3.987255654613724e-06, + "loss": 0.84842771, + "num_input_tokens_seen": 11510440, + "router_z_loss_clip": 1.84277344, + "router_z_loss_mlp": 0.28771973, + "step": 540, + "time_per_iteration": 2.874563455581665 + }, + { + "auxiliary_loss_clip": 0.01258964, + "auxiliary_loss_mlp": 0.01085377, + "balance_loss_clip": 1.07440615, + "balance_loss_mlp": 1.05512154, + "epoch": 0.06505140383574821, + "flos": 19865065259520.0, + "grad_norm": 3.7348034446759373, + "language_loss": 0.70561159, + "learning_rate": 3.987167705309296e-06, + "loss": 0.72905493, + "num_input_tokens_seen": 11529715, + "router_z_loss_clip": 1.84472656, + "router_z_loss_mlp": 0.30273438, + "step": 541, + "time_per_iteration": 2.907188892364502 + }, + { + "auxiliary_loss_clip": 0.0125975, + "auxiliary_loss_mlp": 0.01073177, + "balance_loss_clip": 1.07894111, + "balance_loss_mlp": 1.04289734, + "epoch": 0.0651716467263873, + "flos": 17924703540480.0, + "grad_norm": 2.1263595549891203, + "language_loss": 0.95369256, + "learning_rate": 3.987079454553108e-06, + "loss": 0.97702187, + "num_input_tokens_seen": 11547665, + "router_z_loss_clip": 1.80761719, + "router_z_loss_mlp": 0.30297852, + "step": 542, + "time_per_iteration": 2.818065881729126 + }, + { + "auxiliary_loss_clip": 0.01260972, + "auxiliary_loss_mlp": 0.01066272, + "balance_loss_clip": 1.07686174, + "balance_loss_mlp": 1.0354445, + "epoch": 0.0652918896170264, + "flos": 20842840356480.0, + "grad_norm": 2.132324580793799, + "language_loss": 0.91157013, + "learning_rate": 3.986990902358546e-06, + "loss": 0.93484259, + "num_input_tokens_seen": 11564605, + "router_z_loss_clip": 1.84082031, + "router_z_loss_mlp": 0.30810547, + "step": 543, + "time_per_iteration": 2.84346866607666 + }, + { + "auxiliary_loss_clip": 0.01255916, + "auxiliary_loss_mlp": 0.01061801, + "balance_loss_clip": 1.07204998, + "balance_loss_mlp": 1.03037763, + "epoch": 0.06541213250766549, + "flos": 21872507627520.0, + "grad_norm": 1.9580833007665324, + "language_loss": 0.93313527, + "learning_rate": 3.986902048739045e-06, + "loss": 0.95631242, + "num_input_tokens_seen": 11584550, + "router_z_loss_clip": 1.83886719, + "router_z_loss_mlp": 0.31445312, + "step": 544, + "time_per_iteration": 2.80627179145813 + }, + { + "auxiliary_loss_clip": 0.01265055, + "auxiliary_loss_mlp": 0.01078593, + "balance_loss_clip": 1.07543421, + "balance_loss_mlp": 1.04714537, + "epoch": 0.06553237539830457, + "flos": 23110743219840.0, + "grad_norm": 2.5641316613204377, + "language_loss": 0.80258167, + "learning_rate": 3.986812893708082e-06, + "loss": 0.8260181, + "num_input_tokens_seen": 11600740, + "router_z_loss_clip": 1.89160156, + "router_z_loss_mlp": 0.31445312, + "step": 545, + "time_per_iteration": 2.890017032623291 + }, + { + "auxiliary_loss_clip": 0.01259349, + "auxiliary_loss_mlp": 0.01064671, + "balance_loss_clip": 1.07449043, + "balance_loss_mlp": 1.03362882, + "epoch": 0.06565261828894367, + "flos": 17923769786880.0, + "grad_norm": 3.34738956686934, + "language_loss": 0.8107245, + "learning_rate": 3.9867234372791826e-06, + "loss": 0.83396471, + "num_input_tokens_seen": 11618695, + "router_z_loss_clip": 1.84960938, + "router_z_loss_mlp": 0.31030273, + "step": 546, + "time_per_iteration": 2.9202349185943604 + }, + { + "auxiliary_loss_clip": 0.01251544, + "auxiliary_loss_mlp": 0.01065792, + "balance_loss_clip": 1.0720818, + "balance_loss_mlp": 1.03508401, + "epoch": 0.06577286117958275, + "flos": 22783058421120.0, + "grad_norm": 2.335645080356585, + "language_loss": 0.87277317, + "learning_rate": 3.986633679465918e-06, + "loss": 0.8959465, + "num_input_tokens_seen": 11638850, + "router_z_loss_clip": 1.79589844, + "router_z_loss_mlp": 0.30688477, + "step": 547, + "time_per_iteration": 2.8470194339752197 + }, + { + "auxiliary_loss_clip": 0.0125785, + "auxiliary_loss_mlp": 0.01060768, + "balance_loss_clip": 1.07456779, + "balance_loss_mlp": 1.03094149, + "epoch": 0.06589310407022185, + "flos": 23696194993920.0, + "grad_norm": 2.4780545208604083, + "language_loss": 0.80300236, + "learning_rate": 3.986543620281904e-06, + "loss": 0.82618856, + "num_input_tokens_seen": 11658500, + "router_z_loss_clip": 1.83203125, + "router_z_loss_mlp": 0.2980957, + "step": 548, + "time_per_iteration": 2.8700504302978516 + }, + { + "auxiliary_loss_clip": 0.01253545, + "auxiliary_loss_mlp": 0.01071323, + "balance_loss_clip": 1.07373941, + "balance_loss_mlp": 1.04206872, + "epoch": 0.06601334696086093, + "flos": 26864772410880.0, + "grad_norm": 1.695841808990939, + "language_loss": 0.90802115, + "learning_rate": 3.986453259740802e-06, + "loss": 0.93126982, + "num_input_tokens_seen": 11676670, + "router_z_loss_clip": 1.79785156, + "router_z_loss_mlp": 0.29248047, + "step": 549, + "time_per_iteration": 2.8714723587036133 + }, + { + "auxiliary_loss_clip": 0.01260437, + "auxiliary_loss_mlp": 0.01071906, + "balance_loss_clip": 1.07704985, + "balance_loss_mlp": 1.04346311, + "epoch": 0.06613358985150003, + "flos": 12567694101120.0, + "grad_norm": 2.3532160560953197, + "language_loss": 0.79436636, + "learning_rate": 3.986362597856319e-06, + "loss": 0.81768978, + "num_input_tokens_seen": 11693170, + "router_z_loss_clip": 1.83398438, + "router_z_loss_mlp": 0.28430176, + "step": 550, + "time_per_iteration": 2.9325571060180664 + }, + { + "auxiliary_loss_clip": 0.01262549, + "auxiliary_loss_mlp": 0.01064341, + "balance_loss_clip": 1.07709241, + "balance_loss_mlp": 1.03406167, + "epoch": 0.06625383274213913, + "flos": 18332505624960.0, + "grad_norm": 3.2111558201880213, + "language_loss": 0.81616336, + "learning_rate": 3.986271634642211e-06, + "loss": 0.8394323, + "num_input_tokens_seen": 11710150, + "router_z_loss_clip": 1.85449219, + "router_z_loss_mlp": 0.30249023, + "step": 551, + "time_per_iteration": 2.8611629009246826 + }, + { + "auxiliary_loss_clip": 0.01259026, + "auxiliary_loss_mlp": 0.01067549, + "balance_loss_clip": 1.07732332, + "balance_loss_mlp": 1.03862846, + "epoch": 0.06637407563277821, + "flos": 15375585098880.0, + "grad_norm": 2.092979151023399, + "language_loss": 0.81450081, + "learning_rate": 3.986180370112274e-06, + "loss": 0.83776659, + "num_input_tokens_seen": 11726670, + "router_z_loss_clip": 1.81835938, + "router_z_loss_mlp": 0.28930664, + "step": 552, + "time_per_iteration": 2.8534021377563477 + }, + { + "auxiliary_loss_clip": 0.01257707, + "auxiliary_loss_mlp": 0.01072142, + "balance_loss_clip": 1.07273316, + "balance_loss_mlp": 1.0404799, + "epoch": 0.0664943185234173, + "flos": 24025244509440.0, + "grad_norm": 2.0169741962195755, + "language_loss": 0.74544728, + "learning_rate": 3.986088804280354e-06, + "loss": 0.76874578, + "num_input_tokens_seen": 11746400, + "router_z_loss_clip": 1.84960938, + "router_z_loss_mlp": 0.31665039, + "step": 553, + "time_per_iteration": 2.868607521057129 + }, + { + "auxiliary_loss_clip": 0.01258947, + "auxiliary_loss_mlp": 0.01071803, + "balance_loss_clip": 1.07591033, + "balance_loss_mlp": 1.04223907, + "epoch": 0.06661456141405639, + "flos": 20957503547520.0, + "grad_norm": 2.20681030085663, + "language_loss": 0.93833458, + "learning_rate": 3.985996937160342e-06, + "loss": 0.96164215, + "num_input_tokens_seen": 11765590, + "router_z_loss_clip": 1.83007812, + "router_z_loss_mlp": 0.2956543, + "step": 554, + "time_per_iteration": 2.820960521697998 + }, + { + "auxiliary_loss_clip": 0.0125848, + "auxiliary_loss_mlp": 0.01078251, + "balance_loss_clip": 1.07898283, + "balance_loss_mlp": 1.04868698, + "epoch": 0.06673480430469549, + "flos": 52223953322880.0, + "grad_norm": 4.218759438146005, + "language_loss": 0.68697697, + "learning_rate": 3.985904768766173e-06, + "loss": 0.71034425, + "num_input_tokens_seen": 11788365, + "router_z_loss_clip": 1.796875, + "router_z_loss_mlp": 0.2956543, + "step": 555, + "time_per_iteration": 3.0704429149627686 + }, + { + "auxiliary_loss_clip": 0.01260367, + "auxiliary_loss_mlp": 0.01067463, + "balance_loss_clip": 1.07724059, + "balance_loss_mlp": 1.0348711, + "epoch": 0.06685504719533458, + "flos": 16217079995520.0, + "grad_norm": 2.9330853435179227, + "language_loss": 0.7585274, + "learning_rate": 3.98581229911183e-06, + "loss": 0.78180575, + "num_input_tokens_seen": 11807285, + "router_z_loss_clip": 1.82910156, + "router_z_loss_mlp": 0.32617188, + "step": 556, + "time_per_iteration": 2.823739767074585 + }, + { + "auxiliary_loss_clip": 0.01257224, + "auxiliary_loss_mlp": 0.0106944, + "balance_loss_clip": 1.07301998, + "balance_loss_mlp": 1.03930402, + "epoch": 0.06697529008597367, + "flos": 22491535639680.0, + "grad_norm": 1.8125901733030003, + "language_loss": 0.91944247, + "learning_rate": 3.985719528211341e-06, + "loss": 0.94270909, + "num_input_tokens_seen": 11826655, + "router_z_loss_clip": 1.83984375, + "router_z_loss_mlp": 0.30126953, + "step": 557, + "time_per_iteration": 2.90340256690979 + }, + { + "auxiliary_loss_clip": 0.01144887, + "auxiliary_loss_mlp": 0.01036414, + "balance_loss_clip": 1.05644441, + "balance_loss_mlp": 1.02945256, + "epoch": 0.06709553297661276, + "flos": 62688216936960.0, + "grad_norm": 0.8469398083543384, + "language_loss": 0.6306442, + "learning_rate": 3.985626456078777e-06, + "loss": 0.65245718, + "num_input_tokens_seen": 11891310, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.06982422, + "step": 558, + "time_per_iteration": 3.4448912143707275 + }, + { + "auxiliary_loss_clip": 0.01260074, + "auxiliary_loss_mlp": 0.01068448, + "balance_loss_clip": 1.07749701, + "balance_loss_mlp": 1.03762019, + "epoch": 0.06721577586725185, + "flos": 11216590997760.0, + "grad_norm": 2.2936931265117915, + "language_loss": 0.86068392, + "learning_rate": 3.985533082728259e-06, + "loss": 0.88396919, + "num_input_tokens_seen": 11906965, + "router_z_loss_clip": 1.82714844, + "router_z_loss_mlp": 0.30810547, + "step": 559, + "time_per_iteration": 2.8898088932037354 + }, + { + "auxiliary_loss_clip": 0.01258288, + "auxiliary_loss_mlp": 0.01069079, + "balance_loss_clip": 1.07206666, + "balance_loss_mlp": 1.03796542, + "epoch": 0.06733601875789094, + "flos": 25922189664000.0, + "grad_norm": 1.8472910304778805, + "language_loss": 0.74502468, + "learning_rate": 3.985439408173951e-06, + "loss": 0.76829839, + "num_input_tokens_seen": 11927190, + "router_z_loss_clip": 1.86230469, + "router_z_loss_mlp": 0.31103516, + "step": 560, + "time_per_iteration": 2.8588991165161133 + }, + { + "auxiliary_loss_clip": 0.01263584, + "auxiliary_loss_mlp": 0.01075151, + "balance_loss_clip": 1.07587707, + "balance_loss_mlp": 1.04487216, + "epoch": 0.06745626164853002, + "flos": 20813645577600.0, + "grad_norm": 1.9630470363853294, + "language_loss": 0.70842379, + "learning_rate": 3.9853454324300634e-06, + "loss": 0.73181105, + "num_input_tokens_seen": 11946400, + "router_z_loss_clip": 1.87402344, + "router_z_loss_mlp": 0.30273438, + "step": 561, + "time_per_iteration": 2.879528760910034 + }, + { + "auxiliary_loss_clip": 0.01257342, + "auxiliary_loss_mlp": 0.01065881, + "balance_loss_clip": 1.07684422, + "balance_loss_mlp": 1.03680599, + "epoch": 0.06757650453916912, + "flos": 19829262378240.0, + "grad_norm": 2.106171790196712, + "language_loss": 0.77786952, + "learning_rate": 3.985251155510852e-06, + "loss": 0.8011018, + "num_input_tokens_seen": 11965430, + "router_z_loss_clip": 1.8046875, + "router_z_loss_mlp": 0.29040527, + "step": 562, + "time_per_iteration": 2.7990012168884277 + }, + { + "auxiliary_loss_clip": 0.01258465, + "auxiliary_loss_mlp": 0.01070404, + "balance_loss_clip": 1.07554317, + "balance_loss_mlp": 1.03952861, + "epoch": 0.06769674742980822, + "flos": 25739224761600.0, + "grad_norm": 1.8893049263441295, + "language_loss": 0.80450308, + "learning_rate": 3.98515657743062e-06, + "loss": 0.82779181, + "num_input_tokens_seen": 11984895, + "router_z_loss_clip": 1.82910156, + "router_z_loss_mlp": 0.30883789, + "step": 563, + "time_per_iteration": 2.8245060443878174 + }, + { + "auxiliary_loss_clip": 0.01256751, + "auxiliary_loss_mlp": 0.01069024, + "balance_loss_clip": 1.07563436, + "balance_loss_mlp": 1.03762388, + "epoch": 0.0678169903204473, + "flos": 13074788355840.0, + "grad_norm": 1.9255458208464649, + "language_loss": 0.77950168, + "learning_rate": 3.985061698203711e-06, + "loss": 0.80275941, + "num_input_tokens_seen": 12002010, + "router_z_loss_clip": 1.8125, + "router_z_loss_mlp": 0.31396484, + "step": 564, + "time_per_iteration": 2.7933542728424072 + }, + { + "auxiliary_loss_clip": 0.01139474, + "auxiliary_loss_mlp": 0.01008851, + "balance_loss_clip": 1.05324745, + "balance_loss_mlp": 1.00227094, + "epoch": 0.0679372332110864, + "flos": 70865830788480.0, + "grad_norm": 0.8856386963242474, + "language_loss": 0.63803822, + "learning_rate": 3.984966517844523e-06, + "loss": 0.65952146, + "num_input_tokens_seen": 12057255, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.06591797, + "step": 565, + "time_per_iteration": 3.3007266521453857 + }, + { + "auxiliary_loss_clip": 0.01256808, + "auxiliary_loss_mlp": 0.0106225, + "balance_loss_clip": 1.07224703, + "balance_loss_mlp": 1.03061211, + "epoch": 0.06805747610172548, + "flos": 28256418990720.0, + "grad_norm": 2.396845559770285, + "language_loss": 0.80318308, + "learning_rate": 3.984871036367492e-06, + "loss": 0.82637364, + "num_input_tokens_seen": 12077280, + "router_z_loss_clip": 1.84570312, + "router_z_loss_mlp": 0.31640625, + "step": 566, + "time_per_iteration": 3.8903560638427734 + }, + { + "auxiliary_loss_clip": 0.01254791, + "auxiliary_loss_mlp": 0.01065965, + "balance_loss_clip": 1.07685244, + "balance_loss_mlp": 1.03716362, + "epoch": 0.06817771899236458, + "flos": 20120533764480.0, + "grad_norm": 1.8689890567323315, + "language_loss": 0.8310653, + "learning_rate": 3.984775253787102e-06, + "loss": 0.85427284, + "num_input_tokens_seen": 12095570, + "router_z_loss_clip": 1.78027344, + "router_z_loss_mlp": 0.28771973, + "step": 567, + "time_per_iteration": 4.455020189285278 + }, + { + "auxiliary_loss_clip": 0.01257731, + "auxiliary_loss_mlp": 0.0106684, + "balance_loss_clip": 1.07530844, + "balance_loss_mlp": 1.03646493, + "epoch": 0.06829796188300366, + "flos": 17930629284480.0, + "grad_norm": 2.965870132155644, + "language_loss": 0.88222837, + "learning_rate": 3.984679170117885e-06, + "loss": 0.90547413, + "num_input_tokens_seen": 12111775, + "router_z_loss_clip": 1.82519531, + "router_z_loss_mlp": 0.30322266, + "step": 568, + "time_per_iteration": 2.8116061687469482 + }, + { + "auxiliary_loss_clip": 0.01255595, + "auxiliary_loss_mlp": 0.01062883, + "balance_loss_clip": 1.07153225, + "balance_loss_mlp": 1.03243697, + "epoch": 0.06841820477364276, + "flos": 14501627285760.0, + "grad_norm": 3.177753657130695, + "language_loss": 0.78699583, + "learning_rate": 3.984582785374415e-06, + "loss": 0.81018054, + "num_input_tokens_seen": 12129215, + "router_z_loss_clip": 1.84082031, + "router_z_loss_mlp": 0.30444336, + "step": 569, + "time_per_iteration": 2.823869228363037 + }, + { + "auxiliary_loss_clip": 0.01256013, + "auxiliary_loss_mlp": 0.01070923, + "balance_loss_clip": 1.07572889, + "balance_loss_mlp": 1.04222894, + "epoch": 0.06853844766428185, + "flos": 21938474954880.0, + "grad_norm": 1.8104057140383036, + "language_loss": 0.80627143, + "learning_rate": 3.9844860995713155e-06, + "loss": 0.82954073, + "num_input_tokens_seen": 12148755, + "router_z_loss_clip": 1.80273438, + "router_z_loss_mlp": 0.28723145, + "step": 570, + "time_per_iteration": 2.9419970512390137 + }, + { + "auxiliary_loss_clip": 0.01251567, + "auxiliary_loss_mlp": 0.01086226, + "balance_loss_clip": 1.07753766, + "balance_loss_mlp": 1.05872464, + "epoch": 0.06865869055492094, + "flos": 16800628348800.0, + "grad_norm": 3.941802896235585, + "language_loss": 0.82961541, + "learning_rate": 3.9843891127232524e-06, + "loss": 0.85299337, + "num_input_tokens_seen": 12166290, + "router_z_loss_clip": 1.7421875, + "router_z_loss_mlp": 0.27490234, + "step": 571, + "time_per_iteration": 2.9613850116729736 + }, + { + "auxiliary_loss_clip": 0.0125106, + "auxiliary_loss_mlp": 0.01072431, + "balance_loss_clip": 1.07028449, + "balance_loss_mlp": 1.04258108, + "epoch": 0.06877893344556003, + "flos": 19937281553280.0, + "grad_norm": 3.602993891656203, + "language_loss": 0.66984946, + "learning_rate": 3.984291824844938e-06, + "loss": 0.69308436, + "num_input_tokens_seen": 12181385, + "router_z_loss_clip": 1.80859375, + "router_z_loss_mlp": 0.29833984, + "step": 572, + "time_per_iteration": 2.811089515686035 + }, + { + "auxiliary_loss_clip": 0.01257672, + "auxiliary_loss_mlp": 0.01066503, + "balance_loss_clip": 1.07285416, + "balance_loss_mlp": 1.03791642, + "epoch": 0.06889917633619912, + "flos": 23039388852480.0, + "grad_norm": 2.5223392582748985, + "language_loss": 0.84967071, + "learning_rate": 3.984194235951132e-06, + "loss": 0.87291241, + "num_input_tokens_seen": 12197530, + "router_z_loss_clip": 1.84667969, + "router_z_loss_mlp": 0.28613281, + "step": 573, + "time_per_iteration": 2.87249755859375 + }, + { + "auxiliary_loss_clip": 0.01258893, + "auxiliary_loss_mlp": 0.01074255, + "balance_loss_clip": 1.07700264, + "balance_loss_mlp": 1.0433799, + "epoch": 0.06901941922683821, + "flos": 20960556203520.0, + "grad_norm": 3.6109994988494982, + "language_loss": 0.84499741, + "learning_rate": 3.9840963460566375e-06, + "loss": 0.86832893, + "num_input_tokens_seen": 12216310, + "router_z_loss_clip": 1.81933594, + "router_z_loss_mlp": 0.30859375, + "step": 574, + "time_per_iteration": 2.8125622272491455 + }, + { + "auxiliary_loss_clip": 0.01248718, + "auxiliary_loss_mlp": 0.01064942, + "balance_loss_clip": 1.07002962, + "balance_loss_mlp": 1.03742826, + "epoch": 0.06913966211747731, + "flos": 24821850384000.0, + "grad_norm": 1.6583955699785626, + "language_loss": 0.89546049, + "learning_rate": 3.983998155176305e-06, + "loss": 0.9185971, + "num_input_tokens_seen": 12236670, + "router_z_loss_clip": 1.78417969, + "router_z_loss_mlp": 0.27502441, + "step": 575, + "time_per_iteration": 2.823007822036743 + }, + { + "auxiliary_loss_clip": 0.01130433, + "auxiliary_loss_mlp": 0.01005661, + "balance_loss_clip": 1.04517961, + "balance_loss_mlp": 0.99936658, + "epoch": 0.06925990500811639, + "flos": 58367446957440.0, + "grad_norm": 0.8181397702663685, + "language_loss": 0.57031071, + "learning_rate": 3.9838996633250305e-06, + "loss": 0.59167159, + "num_input_tokens_seen": 12297185, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.06298828, + "step": 576, + "time_per_iteration": 3.369112730026245 + }, + { + "auxiliary_loss_clip": 0.01247106, + "auxiliary_loss_mlp": 0.01071911, + "balance_loss_clip": 1.07207394, + "balance_loss_mlp": 1.0427407, + "epoch": 0.06938014789875549, + "flos": 12749940731520.0, + "grad_norm": 2.233258447445147, + "language_loss": 0.88226664, + "learning_rate": 3.983800870517753e-06, + "loss": 0.90545678, + "num_input_tokens_seen": 12313975, + "router_z_loss_clip": 1.75, + "router_z_loss_mlp": 0.29138184, + "step": 577, + "time_per_iteration": 2.7746496200561523 + }, + { + "auxiliary_loss_clip": 0.01251418, + "auxiliary_loss_mlp": 0.01071171, + "balance_loss_clip": 1.07482076, + "balance_loss_mlp": 1.04134536, + "epoch": 0.06950039078939457, + "flos": 22820226019200.0, + "grad_norm": 5.889893460468581, + "language_loss": 0.7843411, + "learning_rate": 3.983701776769463e-06, + "loss": 0.807567, + "num_input_tokens_seen": 12331385, + "router_z_loss_clip": 1.76464844, + "router_z_loss_mlp": 0.2980957, + "step": 578, + "time_per_iteration": 2.850876569747925 + }, + { + "auxiliary_loss_clip": 0.01247988, + "auxiliary_loss_mlp": 0.01066491, + "balance_loss_clip": 1.07476032, + "balance_loss_mlp": 1.03592563, + "epoch": 0.06962063368003367, + "flos": 21941348042880.0, + "grad_norm": 2.029227803341837, + "language_loss": 0.85463804, + "learning_rate": 3.9836023820951885e-06, + "loss": 0.87778282, + "num_input_tokens_seen": 12350600, + "router_z_loss_clip": 1.73242188, + "router_z_loss_mlp": 0.30566406, + "step": 579, + "time_per_iteration": 2.836656332015991 + }, + { + "auxiliary_loss_clip": 0.01250727, + "auxiliary_loss_mlp": 0.01068405, + "balance_loss_clip": 1.07178688, + "balance_loss_mlp": 1.03767323, + "epoch": 0.06974087657067275, + "flos": 20706021452160.0, + "grad_norm": 2.0778372679587434, + "language_loss": 0.68375671, + "learning_rate": 3.983502686510011e-06, + "loss": 0.70694804, + "num_input_tokens_seen": 12371430, + "router_z_loss_clip": 1.79003906, + "router_z_loss_mlp": 0.30712891, + "step": 580, + "time_per_iteration": 2.8339052200317383 + }, + { + "auxiliary_loss_clip": 0.01245555, + "auxiliary_loss_mlp": 0.0106052, + "balance_loss_clip": 1.06896043, + "balance_loss_mlp": 1.03322148, + "epoch": 0.06986111946131185, + "flos": 22638230784000.0, + "grad_norm": 1.9456669965971578, + "language_loss": 0.73562604, + "learning_rate": 3.9834026900290525e-06, + "loss": 0.75868684, + "num_input_tokens_seen": 12390825, + "router_z_loss_clip": 1.765625, + "router_z_loss_mlp": 0.27319336, + "step": 581, + "time_per_iteration": 2.9487743377685547 + }, + { + "auxiliary_loss_clip": 0.01253986, + "auxiliary_loss_mlp": 0.01074621, + "balance_loss_clip": 1.07558525, + "balance_loss_mlp": 1.04598725, + "epoch": 0.06998136235195095, + "flos": 26943453152640.0, + "grad_norm": 2.064935443344456, + "language_loss": 1.00015664, + "learning_rate": 3.983302392667482e-06, + "loss": 1.02344275, + "num_input_tokens_seen": 12411670, + "router_z_loss_clip": 1.78320312, + "router_z_loss_mlp": 0.28613281, + "step": 582, + "time_per_iteration": 2.8878445625305176 + }, + { + "auxiliary_loss_clip": 0.0125668, + "auxiliary_loss_mlp": 0.01061215, + "balance_loss_clip": 1.0774045, + "balance_loss_mlp": 1.0334388, + "epoch": 0.07010160524259003, + "flos": 22492505306880.0, + "grad_norm": 1.785328533375704, + "language_loss": 0.93900245, + "learning_rate": 3.983201794440517e-06, + "loss": 0.96218139, + "num_input_tokens_seen": 12431245, + "router_z_loss_clip": 1.79296875, + "router_z_loss_mlp": 0.27783203, + "step": 583, + "time_per_iteration": 2.9126434326171875 + }, + { + "auxiliary_loss_clip": 0.01252384, + "auxiliary_loss_mlp": 0.01062479, + "balance_loss_clip": 1.07308555, + "balance_loss_mlp": 1.03298712, + "epoch": 0.07022184813322913, + "flos": 18332541538560.0, + "grad_norm": 1.7897793454772455, + "language_loss": 0.67832208, + "learning_rate": 3.9831008953634165e-06, + "loss": 0.70147073, + "num_input_tokens_seen": 12450535, + "router_z_loss_clip": 1.79101562, + "router_z_loss_mlp": 0.29492188, + "step": 584, + "time_per_iteration": 2.754196882247925 + }, + { + "auxiliary_loss_clip": 0.01255079, + "auxiliary_loss_mlp": 0.01063577, + "balance_loss_clip": 1.07623696, + "balance_loss_mlp": 1.0345372, + "epoch": 0.07034209102386821, + "flos": 24675550289280.0, + "grad_norm": 3.0457863328087913, + "language_loss": 0.81248289, + "learning_rate": 3.9829996954514864e-06, + "loss": 0.8356694, + "num_input_tokens_seen": 12469675, + "router_z_loss_clip": 1.7890625, + "router_z_loss_mlp": 0.29016113, + "step": 585, + "time_per_iteration": 2.839601755142212 + }, + { + "auxiliary_loss_clip": 0.01248766, + "auxiliary_loss_mlp": 0.01071136, + "balance_loss_clip": 1.0726645, + "balance_loss_mlp": 1.04206085, + "epoch": 0.0704623339145073, + "flos": 25995878415360.0, + "grad_norm": 2.1925026491909305, + "language_loss": 0.84199023, + "learning_rate": 3.982898194720079e-06, + "loss": 0.86518925, + "num_input_tokens_seen": 12490405, + "router_z_loss_clip": 1.75878906, + "router_z_loss_mlp": 0.29064941, + "step": 586, + "time_per_iteration": 2.991182565689087 + }, + { + "auxiliary_loss_clip": 0.0126109, + "auxiliary_loss_mlp": 0.0106943, + "balance_loss_clip": 1.07821381, + "balance_loss_mlp": 1.03891218, + "epoch": 0.0705825768051464, + "flos": 25338318088320.0, + "grad_norm": 2.08722164646074, + "language_loss": 0.82299215, + "learning_rate": 3.982796393184592e-06, + "loss": 0.84629738, + "num_input_tokens_seen": 12509485, + "router_z_loss_clip": 1.828125, + "router_z_loss_mlp": 0.30517578, + "step": 587, + "time_per_iteration": 2.8425583839416504 + }, + { + "auxiliary_loss_clip": 0.0112505, + "auxiliary_loss_mlp": 0.01006289, + "balance_loss_clip": 1.04277194, + "balance_loss_mlp": 1.00047147, + "epoch": 0.07070281969578548, + "flos": 66047552507520.0, + "grad_norm": 0.7915813984021753, + "language_loss": 0.62663162, + "learning_rate": 3.98269429086047e-06, + "loss": 0.64794499, + "num_input_tokens_seen": 12567325, + "router_z_loss_clip": 0.82421875, + "router_z_loss_mlp": 0.05810547, + "step": 588, + "time_per_iteration": 3.251762866973877 + }, + { + "auxiliary_loss_clip": 0.01260041, + "auxiliary_loss_mlp": 0.01077668, + "balance_loss_clip": 1.07594705, + "balance_loss_mlp": 1.04388392, + "epoch": 0.07082306258642458, + "flos": 23653568528640.0, + "grad_norm": 2.555717298723728, + "language_loss": 0.86263847, + "learning_rate": 3.982591887763199e-06, + "loss": 0.88601553, + "num_input_tokens_seen": 12584785, + "router_z_loss_clip": 1.84179688, + "router_z_loss_mlp": 0.33764648, + "step": 589, + "time_per_iteration": 2.851876974105835 + }, + { + "auxiliary_loss_clip": 0.01251358, + "auxiliary_loss_mlp": 0.01067091, + "balance_loss_clip": 1.07100391, + "balance_loss_mlp": 1.03795671, + "epoch": 0.07094330547706366, + "flos": 13880049408000.0, + "grad_norm": 2.1014072427772796, + "language_loss": 0.82001066, + "learning_rate": 3.982489183908316e-06, + "loss": 0.84319514, + "num_input_tokens_seen": 12601205, + "router_z_loss_clip": 1.80175781, + "router_z_loss_mlp": 0.29150391, + "step": 590, + "time_per_iteration": 2.9811177253723145 + }, + { + "auxiliary_loss_clip": 0.01248057, + "auxiliary_loss_mlp": 0.01081425, + "balance_loss_clip": 1.07346082, + "balance_loss_mlp": 1.05326724, + "epoch": 0.07106354836770276, + "flos": 24645098534400.0, + "grad_norm": 2.023269210526972, + "language_loss": 0.84726703, + "learning_rate": 3.982386179311399e-06, + "loss": 0.87056184, + "num_input_tokens_seen": 12621725, + "router_z_loss_clip": 1.74609375, + "router_z_loss_mlp": 0.28125, + "step": 591, + "time_per_iteration": 2.888762950897217 + }, + { + "auxiliary_loss_clip": 0.01262761, + "auxiliary_loss_mlp": 0.01073812, + "balance_loss_clip": 1.07541442, + "balance_loss_mlp": 1.04403377, + "epoch": 0.07118379125834184, + "flos": 16217223649920.0, + "grad_norm": 2.436152710239187, + "language_loss": 0.87691081, + "learning_rate": 3.982282873988075e-06, + "loss": 0.9002766, + "num_input_tokens_seen": 12639600, + "router_z_loss_clip": 1.875, + "router_z_loss_mlp": 0.29760742, + "step": 592, + "time_per_iteration": 2.9066805839538574 + }, + { + "auxiliary_loss_clip": 0.01248406, + "auxiliary_loss_mlp": 0.010711, + "balance_loss_clip": 1.07278883, + "balance_loss_mlp": 1.04175067, + "epoch": 0.07130403414898094, + "flos": 19719986227200.0, + "grad_norm": 1.7950935325190154, + "language_loss": 0.86781204, + "learning_rate": 3.982179267954016e-06, + "loss": 0.89100707, + "num_input_tokens_seen": 12660030, + "router_z_loss_clip": 1.75390625, + "router_z_loss_mlp": 0.29370117, + "step": 593, + "time_per_iteration": 2.926499605178833 + }, + { + "auxiliary_loss_clip": 0.01253851, + "auxiliary_loss_mlp": 0.01071054, + "balance_loss_clip": 1.07438397, + "balance_loss_mlp": 1.04137051, + "epoch": 0.07142427703962004, + "flos": 21871933009920.0, + "grad_norm": 2.276917763834906, + "language_loss": 0.95897317, + "learning_rate": 3.982075361224937e-06, + "loss": 0.9822222, + "num_input_tokens_seen": 12678395, + "router_z_loss_clip": 1.79492188, + "router_z_loss_mlp": 0.29663086, + "step": 594, + "time_per_iteration": 2.9342496395111084 + }, + { + "auxiliary_loss_clip": 0.01245761, + "auxiliary_loss_mlp": 0.01072205, + "balance_loss_clip": 1.07164311, + "balance_loss_mlp": 1.04407191, + "epoch": 0.07154451993025912, + "flos": 18296595002880.0, + "grad_norm": 1.9263059482967642, + "language_loss": 0.87852788, + "learning_rate": 3.981971153816602e-06, + "loss": 0.90170753, + "num_input_tokens_seen": 12696000, + "router_z_loss_clip": 1.74121094, + "router_z_loss_mlp": 0.28112793, + "step": 595, + "time_per_iteration": 3.7406005859375 + }, + { + "auxiliary_loss_clip": 0.01253957, + "auxiliary_loss_mlp": 0.01075068, + "balance_loss_clip": 1.07800472, + "balance_loss_mlp": 1.04531348, + "epoch": 0.07166476282089822, + "flos": 22160690444160.0, + "grad_norm": 1.7846148451656927, + "language_loss": 0.96047127, + "learning_rate": 3.981866645744819e-06, + "loss": 0.98376155, + "num_input_tokens_seen": 12716715, + "router_z_loss_clip": 1.76074219, + "router_z_loss_mlp": 0.29736328, + "step": 596, + "time_per_iteration": 4.552423715591431 + }, + { + "auxiliary_loss_clip": 0.01259376, + "auxiliary_loss_mlp": 0.01081472, + "balance_loss_clip": 1.07515001, + "balance_loss_mlp": 1.05019128, + "epoch": 0.0717850057115373, + "flos": 14136343925760.0, + "grad_norm": 2.524580341484676, + "language_loss": 0.81454444, + "learning_rate": 3.9817618370254416e-06, + "loss": 0.83795285, + "num_input_tokens_seen": 12733370, + "router_z_loss_clip": 1.84179688, + "router_z_loss_mlp": 0.31274414, + "step": 597, + "time_per_iteration": 2.7602641582489014 + }, + { + "auxiliary_loss_clip": 0.01255745, + "auxiliary_loss_mlp": 0.01072952, + "balance_loss_clip": 1.07368541, + "balance_loss_mlp": 1.0440793, + "epoch": 0.0719052486021764, + "flos": 30917794412160.0, + "grad_norm": 2.7571386794757142, + "language_loss": 0.86937463, + "learning_rate": 3.9816567276743684e-06, + "loss": 0.89266163, + "num_input_tokens_seen": 12753235, + "router_z_loss_clip": 1.81933594, + "router_z_loss_mlp": 0.2890625, + "step": 598, + "time_per_iteration": 2.8836662769317627 + }, + { + "auxiliary_loss_clip": 0.01251414, + "auxiliary_loss_mlp": 0.0108148, + "balance_loss_clip": 1.07438803, + "balance_loss_mlp": 1.05255926, + "epoch": 0.0720254914928155, + "flos": 21287019939840.0, + "grad_norm": 2.6411744898782707, + "language_loss": 0.77706587, + "learning_rate": 3.9815513177075466e-06, + "loss": 0.80039489, + "num_input_tokens_seen": 12772020, + "router_z_loss_clip": 1.77148438, + "router_z_loss_mlp": 0.2890625, + "step": 599, + "time_per_iteration": 2.8085098266601562 + }, + { + "auxiliary_loss_clip": 0.01244071, + "auxiliary_loss_mlp": 0.01067906, + "balance_loss_clip": 1.07380462, + "balance_loss_mlp": 1.04157269, + "epoch": 0.07214573438345458, + "flos": 27819170732160.0, + "grad_norm": 1.5221610100063854, + "language_loss": 0.70092714, + "learning_rate": 3.9814456071409646e-06, + "loss": 0.72404695, + "num_input_tokens_seen": 12792555, + "router_z_loss_clip": 1.70214844, + "router_z_loss_mlp": 0.2635498, + "step": 600, + "time_per_iteration": 2.837631940841675 + }, + { + "auxiliary_loss_clip": 0.01263996, + "auxiliary_loss_mlp": 0.01081546, + "balance_loss_clip": 1.07617259, + "balance_loss_mlp": 1.05059934, + "epoch": 0.07226597727409367, + "flos": 25483576688640.0, + "grad_norm": 2.5051962598184554, + "language_loss": 0.85247421, + "learning_rate": 3.981339595990659e-06, + "loss": 0.87592959, + "num_input_tokens_seen": 12811085, + "router_z_loss_clip": 1.87988281, + "router_z_loss_mlp": 0.30957031, + "step": 601, + "time_per_iteration": 2.8547658920288086 + }, + { + "auxiliary_loss_clip": 0.01254118, + "auxiliary_loss_mlp": 0.01063725, + "balance_loss_clip": 1.07255673, + "balance_loss_mlp": 1.03529358, + "epoch": 0.07238622016473276, + "flos": 23513840622720.0, + "grad_norm": 4.084766130146227, + "language_loss": 0.80894464, + "learning_rate": 3.981233284272713e-06, + "loss": 0.83212304, + "num_input_tokens_seen": 12830830, + "router_z_loss_clip": 1.81640625, + "router_z_loss_mlp": 0.28442383, + "step": 602, + "time_per_iteration": 2.858340263366699 + }, + { + "auxiliary_loss_clip": 0.01249218, + "auxiliary_loss_mlp": 0.01079765, + "balance_loss_clip": 1.07704043, + "balance_loss_mlp": 1.05074978, + "epoch": 0.07250646305537185, + "flos": 25453519983360.0, + "grad_norm": 2.0345518126874405, + "language_loss": 0.90172851, + "learning_rate": 3.981126672003253e-06, + "loss": 0.92501831, + "num_input_tokens_seen": 12853505, + "router_z_loss_clip": 1.72265625, + "router_z_loss_mlp": 0.2902832, + "step": 603, + "time_per_iteration": 2.8835253715515137 + }, + { + "auxiliary_loss_clip": 0.0125526, + "auxiliary_loss_mlp": 0.01063515, + "balance_loss_clip": 1.07457972, + "balance_loss_mlp": 1.03423703, + "epoch": 0.07262670594601094, + "flos": 27155038216320.0, + "grad_norm": 2.38072911987299, + "language_loss": 0.78375846, + "learning_rate": 3.981019759198451e-06, + "loss": 0.80694622, + "num_input_tokens_seen": 12872455, + "router_z_loss_clip": 1.80371094, + "router_z_loss_mlp": 0.29272461, + "step": 604, + "time_per_iteration": 2.8334295749664307 + }, + { + "auxiliary_loss_clip": 0.01257305, + "auxiliary_loss_mlp": 0.01072336, + "balance_loss_clip": 1.07631803, + "balance_loss_mlp": 1.04318976, + "epoch": 0.07274694883665003, + "flos": 26651607148800.0, + "grad_norm": 2.4443800848580652, + "language_loss": 0.84319502, + "learning_rate": 3.980912545874528e-06, + "loss": 0.86649144, + "num_input_tokens_seen": 12892620, + "router_z_loss_clip": 1.80859375, + "router_z_loss_mlp": 0.29089355, + "step": 605, + "time_per_iteration": 2.8764333724975586 + }, + { + "auxiliary_loss_clip": 0.01258762, + "auxiliary_loss_mlp": 0.01068794, + "balance_loss_clip": 1.07598031, + "balance_loss_mlp": 1.03790724, + "epoch": 0.07286719172728913, + "flos": 29862344154240.0, + "grad_norm": 2.407642481881653, + "language_loss": 0.85756433, + "learning_rate": 3.980805032047746e-06, + "loss": 0.88083982, + "num_input_tokens_seen": 12914090, + "router_z_loss_clip": 1.82714844, + "router_z_loss_mlp": 0.30895996, + "step": 606, + "time_per_iteration": 2.954806089401245 + }, + { + "auxiliary_loss_clip": 0.01256743, + "auxiliary_loss_mlp": 0.0108614, + "balance_loss_clip": 1.07704425, + "balance_loss_mlp": 1.05667162, + "epoch": 0.07298743461792821, + "flos": 17382057799680.0, + "grad_norm": 1.9831272179178154, + "language_loss": 0.81114578, + "learning_rate": 3.980697217734415e-06, + "loss": 0.83457458, + "num_input_tokens_seen": 12931830, + "router_z_loss_clip": 1.79589844, + "router_z_loss_mlp": 0.29492188, + "step": 607, + "time_per_iteration": 2.913006544113159 + }, + { + "auxiliary_loss_clip": 0.01254642, + "auxiliary_loss_mlp": 0.01064771, + "balance_loss_clip": 1.07421529, + "balance_loss_mlp": 1.03575504, + "epoch": 0.07310767750856731, + "flos": 19498201701120.0, + "grad_norm": 2.4350683610910515, + "language_loss": 0.91795164, + "learning_rate": 3.980589102950891e-06, + "loss": 0.94114578, + "num_input_tokens_seen": 12949995, + "router_z_loss_clip": 1.8046875, + "router_z_loss_mlp": 0.29003906, + "step": 608, + "time_per_iteration": 2.8779456615448 + }, + { + "auxiliary_loss_clip": 0.01254074, + "auxiliary_loss_mlp": 0.01066783, + "balance_loss_clip": 1.07570553, + "balance_loss_mlp": 1.03861439, + "epoch": 0.07322792039920639, + "flos": 29168693637120.0, + "grad_norm": 64.41960008015032, + "language_loss": 0.75701463, + "learning_rate": 3.9804806877135755e-06, + "loss": 0.78022325, + "num_input_tokens_seen": 12968040, + "router_z_loss_clip": 1.78417969, + "router_z_loss_mlp": 0.28161621, + "step": 609, + "time_per_iteration": 2.8901281356811523 + }, + { + "auxiliary_loss_clip": 0.01253666, + "auxiliary_loss_mlp": 0.01070491, + "balance_loss_clip": 1.0720067, + "balance_loss_mlp": 1.03844762, + "epoch": 0.07334816328984549, + "flos": 23477822259840.0, + "grad_norm": 2.7039829458564473, + "language_loss": 0.86006558, + "learning_rate": 3.980371972038915e-06, + "loss": 0.88330716, + "num_input_tokens_seen": 12988530, + "router_z_loss_clip": 1.81640625, + "router_z_loss_mlp": 0.3203125, + "step": 610, + "time_per_iteration": 2.8851659297943115 + }, + { + "auxiliary_loss_clip": 0.0125754, + "auxiliary_loss_mlp": 0.0108712, + "balance_loss_clip": 1.07727385, + "balance_loss_mlp": 1.05316877, + "epoch": 0.07346840618048459, + "flos": 22962467877120.0, + "grad_norm": 1.6980561901278084, + "language_loss": 0.84210289, + "learning_rate": 3.980262955943399e-06, + "loss": 0.8655495, + "num_input_tokens_seen": 13008195, + "router_z_loss_clip": 1.80273438, + "router_z_loss_mlp": 0.33935547, + "step": 611, + "time_per_iteration": 2.8721442222595215 + }, + { + "auxiliary_loss_clip": 0.01247102, + "auxiliary_loss_mlp": 0.01075848, + "balance_loss_clip": 1.07338953, + "balance_loss_mlp": 1.04502034, + "epoch": 0.07358864907112367, + "flos": 17673903803520.0, + "grad_norm": 3.556406927119434, + "language_loss": 0.86876297, + "learning_rate": 3.980153639443569e-06, + "loss": 0.89199257, + "num_input_tokens_seen": 13024180, + "router_z_loss_clip": 1.73632812, + "router_z_loss_mlp": 0.30810547, + "step": 612, + "time_per_iteration": 2.8819663524627686 + }, + { + "auxiliary_loss_clip": 0.01255946, + "auxiliary_loss_mlp": 0.01073545, + "balance_loss_clip": 1.07694685, + "balance_loss_mlp": 1.04321837, + "epoch": 0.07370889196176277, + "flos": 24097029840000.0, + "grad_norm": 2.035351506572023, + "language_loss": 0.80141342, + "learning_rate": 3.980044022556005e-06, + "loss": 0.82470828, + "num_input_tokens_seen": 13043865, + "router_z_loss_clip": 1.78808594, + "router_z_loss_mlp": 0.3034668, + "step": 613, + "time_per_iteration": 2.9010426998138428 + }, + { + "auxiliary_loss_clip": 0.0125236, + "auxiliary_loss_mlp": 0.01063782, + "balance_loss_clip": 1.07408619, + "balance_loss_mlp": 1.03169096, + "epoch": 0.07382913485240185, + "flos": 25885919905920.0, + "grad_norm": 2.140809444428899, + "language_loss": 0.72787565, + "learning_rate": 3.9799341052973375e-06, + "loss": 0.75103706, + "num_input_tokens_seen": 13063700, + "router_z_loss_clip": 1.78320312, + "router_z_loss_mlp": 0.32080078, + "step": 614, + "time_per_iteration": 2.869396209716797 + }, + { + "auxiliary_loss_clip": 0.0125679, + "auxiliary_loss_mlp": 0.01076886, + "balance_loss_clip": 1.07538378, + "balance_loss_mlp": 1.04450881, + "epoch": 0.07394937774304094, + "flos": 16873850223360.0, + "grad_norm": 2.378819930584001, + "language_loss": 0.75279129, + "learning_rate": 3.979823887684241e-06, + "loss": 0.77612805, + "num_input_tokens_seen": 13082640, + "router_z_loss_clip": 1.81445312, + "router_z_loss_mlp": 0.32348633, + "step": 615, + "time_per_iteration": 2.9008982181549072 + }, + { + "auxiliary_loss_clip": 0.01254697, + "auxiliary_loss_mlp": 0.0107158, + "balance_loss_clip": 1.07707286, + "balance_loss_mlp": 1.04354191, + "epoch": 0.07406962063368003, + "flos": 20703471586560.0, + "grad_norm": 2.476629614884971, + "language_loss": 0.84673852, + "learning_rate": 3.979713369733434e-06, + "loss": 0.87000132, + "num_input_tokens_seen": 13100505, + "router_z_loss_clip": 1.77734375, + "router_z_loss_mlp": 0.28051758, + "step": 616, + "time_per_iteration": 2.853400230407715 + }, + { + "auxiliary_loss_clip": 0.0125929, + "auxiliary_loss_mlp": 0.01079613, + "balance_loss_clip": 1.07580113, + "balance_loss_mlp": 1.04864275, + "epoch": 0.07418986352431912, + "flos": 21430985650560.0, + "grad_norm": 2.1969480915313846, + "language_loss": 0.85095489, + "learning_rate": 3.979602551461683e-06, + "loss": 0.87434387, + "num_input_tokens_seen": 13121285, + "router_z_loss_clip": 1.83300781, + "router_z_loss_mlp": 0.30957031, + "step": 617, + "time_per_iteration": 2.8052186965942383 + }, + { + "auxiliary_loss_clip": 0.01253483, + "auxiliary_loss_mlp": 0.01074715, + "balance_loss_clip": 1.07598054, + "balance_loss_mlp": 1.04536533, + "epoch": 0.07431010641495822, + "flos": 12021133777920.0, + "grad_norm": 2.2607643407485196, + "language_loss": 0.91635239, + "learning_rate": 3.979491432885799e-06, + "loss": 0.93963432, + "num_input_tokens_seen": 13137550, + "router_z_loss_clip": 1.77636719, + "router_z_loss_mlp": 0.29370117, + "step": 618, + "time_per_iteration": 2.800868511199951 + }, + { + "auxiliary_loss_clip": 0.01248587, + "auxiliary_loss_mlp": 0.01076408, + "balance_loss_clip": 1.07483733, + "balance_loss_mlp": 1.04926443, + "epoch": 0.0744303493055973, + "flos": 20957575374720.0, + "grad_norm": 2.28721172129101, + "language_loss": 0.82668513, + "learning_rate": 3.97938001402264e-06, + "loss": 0.84993505, + "num_input_tokens_seen": 13156675, + "router_z_loss_clip": 1.73925781, + "router_z_loss_mlp": 0.27160645, + "step": 619, + "time_per_iteration": 2.8429667949676514 + }, + { + "auxiliary_loss_clip": 0.01252802, + "auxiliary_loss_mlp": 0.01060718, + "balance_loss_clip": 1.07542515, + "balance_loss_mlp": 1.03098702, + "epoch": 0.0745505921962364, + "flos": 16253134272000.0, + "grad_norm": 2.7005462731100254, + "language_loss": 0.79505551, + "learning_rate": 3.979268294889105e-06, + "loss": 0.81819063, + "num_input_tokens_seen": 13172225, + "router_z_loss_clip": 1.77148438, + "router_z_loss_mlp": 0.296875, + "step": 620, + "time_per_iteration": 2.8120269775390625 + }, + { + "auxiliary_loss_clip": 0.01252026, + "auxiliary_loss_mlp": 0.01070883, + "balance_loss_clip": 1.07523084, + "balance_loss_mlp": 1.04127181, + "epoch": 0.07467083508687548, + "flos": 50944635550080.0, + "grad_norm": 1.9088982252978675, + "language_loss": 0.74008095, + "learning_rate": 3.979156275502143e-06, + "loss": 0.76331007, + "num_input_tokens_seen": 13195885, + "router_z_loss_clip": 1.76855469, + "router_z_loss_mlp": 0.29589844, + "step": 621, + "time_per_iteration": 3.0470058917999268 + }, + { + "auxiliary_loss_clip": 0.01252699, + "auxiliary_loss_mlp": 0.0106386, + "balance_loss_clip": 1.07664633, + "balance_loss_mlp": 1.03532171, + "epoch": 0.07479107797751458, + "flos": 17529686697600.0, + "grad_norm": 2.5254017987191943, + "language_loss": 0.91199869, + "learning_rate": 3.979043955878749e-06, + "loss": 0.93516433, + "num_input_tokens_seen": 13213730, + "router_z_loss_clip": 1.76171875, + "router_z_loss_mlp": 0.28503418, + "step": 622, + "time_per_iteration": 2.814612627029419 + }, + { + "auxiliary_loss_clip": 0.01254038, + "auxiliary_loss_mlp": 0.01067892, + "balance_loss_clip": 1.07441664, + "balance_loss_mlp": 1.03842354, + "epoch": 0.07491132086815366, + "flos": 23473943591040.0, + "grad_norm": 2.111610018044393, + "language_loss": 0.82821691, + "learning_rate": 3.978931336035959e-06, + "loss": 0.8514362, + "num_input_tokens_seen": 13232540, + "router_z_loss_clip": 1.79589844, + "router_z_loss_mlp": 0.29467773, + "step": 623, + "time_per_iteration": 2.8827505111694336 + }, + { + "auxiliary_loss_clip": 0.01255905, + "auxiliary_loss_mlp": 0.01076038, + "balance_loss_clip": 1.07812655, + "balance_loss_mlp": 1.04656959, + "epoch": 0.07503156375879276, + "flos": 20157557708160.0, + "grad_norm": 2.3480173535087148, + "language_loss": 0.82135844, + "learning_rate": 3.9788184159908595e-06, + "loss": 0.84467787, + "num_input_tokens_seen": 13249670, + "router_z_loss_clip": 1.77832031, + "router_z_loss_mlp": 0.29492188, + "step": 624, + "time_per_iteration": 3.662674903869629 + }, + { + "auxiliary_loss_clip": 0.01246626, + "auxiliary_loss_mlp": 0.01081172, + "balance_loss_clip": 1.07655096, + "balance_loss_mlp": 1.05157256, + "epoch": 0.07515180664943186, + "flos": 15115519653120.0, + "grad_norm": 4.34996557321472, + "language_loss": 0.82778668, + "learning_rate": 3.97870519576058e-06, + "loss": 0.85106462, + "num_input_tokens_seen": 13266095, + "router_z_loss_clip": 1.70019531, + "router_z_loss_mlp": 0.29577637, + "step": 625, + "time_per_iteration": 4.530543088912964 + }, + { + "auxiliary_loss_clip": 0.01253478, + "auxiliary_loss_mlp": 0.01081102, + "balance_loss_clip": 1.07709289, + "balance_loss_mlp": 1.05110884, + "epoch": 0.07527204954007094, + "flos": 21287702298240.0, + "grad_norm": 2.335263698705836, + "language_loss": 0.80965662, + "learning_rate": 3.978591675362295e-06, + "loss": 0.83300245, + "num_input_tokens_seen": 13284810, + "router_z_loss_clip": 1.76269531, + "router_z_loss_mlp": 0.29943848, + "step": 626, + "time_per_iteration": 2.8834991455078125 + }, + { + "auxiliary_loss_clip": 0.01247586, + "auxiliary_loss_mlp": 0.01074296, + "balance_loss_clip": 1.07321548, + "balance_loss_mlp": 1.04498279, + "epoch": 0.07539229243071004, + "flos": 21324187537920.0, + "grad_norm": 1.671325836871207, + "language_loss": 0.87605381, + "learning_rate": 3.978477854813226e-06, + "loss": 0.89927262, + "num_input_tokens_seen": 13304150, + "router_z_loss_clip": 1.74121094, + "router_z_loss_mlp": 0.29333496, + "step": 627, + "time_per_iteration": 2.7917442321777344 + }, + { + "auxiliary_loss_clip": 0.01254231, + "auxiliary_loss_mlp": 0.01065146, + "balance_loss_clip": 1.07815206, + "balance_loss_mlp": 1.0365833, + "epoch": 0.07551253532134912, + "flos": 13042540920960.0, + "grad_norm": 2.337471430999979, + "language_loss": 0.82198632, + "learning_rate": 3.97836373413064e-06, + "loss": 0.84518015, + "num_input_tokens_seen": 13322205, + "router_z_loss_clip": 1.75976562, + "router_z_loss_mlp": 0.28540039, + "step": 628, + "time_per_iteration": 2.8511176109313965 + }, + { + "auxiliary_loss_clip": 0.01253517, + "auxiliary_loss_mlp": 0.01069905, + "balance_loss_clip": 1.07481825, + "balance_loss_mlp": 1.04046071, + "epoch": 0.07563277821198822, + "flos": 19208761908480.0, + "grad_norm": 2.1027783620231424, + "language_loss": 0.74529934, + "learning_rate": 3.978249313331848e-06, + "loss": 0.76853359, + "num_input_tokens_seen": 13340435, + "router_z_loss_clip": 1.78710938, + "router_z_loss_mlp": 0.29443359, + "step": 629, + "time_per_iteration": 2.987269878387451 + }, + { + "auxiliary_loss_clip": 0.01250855, + "auxiliary_loss_mlp": 0.0106034, + "balance_loss_clip": 1.07238293, + "balance_loss_mlp": 1.03230214, + "epoch": 0.07575302110262731, + "flos": 19537200892800.0, + "grad_norm": 8.027348562529339, + "language_loss": 0.62481666, + "learning_rate": 3.978134592434208e-06, + "loss": 0.6479286, + "num_input_tokens_seen": 13358185, + "router_z_loss_clip": 1.78515625, + "router_z_loss_mlp": 0.28027344, + "step": 630, + "time_per_iteration": 2.993469476699829 + }, + { + "auxiliary_loss_clip": 0.01133725, + "auxiliary_loss_mlp": 0.010053, + "balance_loss_clip": 1.05203223, + "balance_loss_mlp": 0.99960214, + "epoch": 0.0758732639932664, + "flos": 67961808017280.0, + "grad_norm": 1.0248031832891606, + "language_loss": 0.59350741, + "learning_rate": 3.978019571455123e-06, + "loss": 0.61489761, + "num_input_tokens_seen": 13410130, + "router_z_loss_clip": 0.81640625, + "router_z_loss_mlp": 0.05688477, + "step": 631, + "time_per_iteration": 3.389319896697998 + }, + { + "auxiliary_loss_clip": 0.01244412, + "auxiliary_loss_mlp": 0.01062478, + "balance_loss_clip": 1.07331014, + "balance_loss_mlp": 1.03582251, + "epoch": 0.07599350688390549, + "flos": 18989204025600.0, + "grad_norm": 7.072675888851559, + "language_loss": 0.84121454, + "learning_rate": 3.977904250412042e-06, + "loss": 0.86428338, + "num_input_tokens_seen": 13429085, + "router_z_loss_clip": 1.7109375, + "router_z_loss_mlp": 0.2668457, + "step": 632, + "time_per_iteration": 2.8448476791381836 + }, + { + "auxiliary_loss_clip": 0.01250513, + "auxiliary_loss_mlp": 0.01071347, + "balance_loss_clip": 1.07966614, + "balance_loss_mlp": 1.04342878, + "epoch": 0.07611374977454458, + "flos": 21069006341760.0, + "grad_norm": 2.1438789142651813, + "language_loss": 0.85692883, + "learning_rate": 3.97778862932246e-06, + "loss": 0.8801474, + "num_input_tokens_seen": 13446250, + "router_z_loss_clip": 1.70898438, + "router_z_loss_mlp": 0.27954102, + "step": 633, + "time_per_iteration": 2.97288179397583 + }, + { + "auxiliary_loss_clip": 0.01247238, + "auxiliary_loss_mlp": 0.01064764, + "balance_loss_clip": 1.07269311, + "balance_loss_mlp": 1.03557014, + "epoch": 0.07623399266518367, + "flos": 18514536773760.0, + "grad_norm": 2.3463481649298665, + "language_loss": 0.94115317, + "learning_rate": 3.9776727082039144e-06, + "loss": 0.96427327, + "num_input_tokens_seen": 13463220, + "router_z_loss_clip": 1.74609375, + "router_z_loss_mlp": 0.29187012, + "step": 634, + "time_per_iteration": 2.895465850830078 + }, + { + "auxiliary_loss_clip": 0.01127521, + "auxiliary_loss_mlp": 0.01005684, + "balance_loss_clip": 1.04665327, + "balance_loss_mlp": 0.99984258, + "epoch": 0.07635423555582276, + "flos": 44663036077440.0, + "grad_norm": 0.801658126118076, + "language_loss": 0.55505955, + "learning_rate": 3.977556487073991e-06, + "loss": 0.57639164, + "num_input_tokens_seen": 13517775, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.05834961, + "step": 635, + "time_per_iteration": 3.251782178878784 + }, + { + "auxiliary_loss_clip": 0.01250783, + "auxiliary_loss_mlp": 0.01072839, + "balance_loss_clip": 1.07558632, + "balance_loss_mlp": 1.04400194, + "epoch": 0.07647447844646185, + "flos": 21761148487680.0, + "grad_norm": 1.7792398340906097, + "language_loss": 0.81497741, + "learning_rate": 3.97743996595032e-06, + "loss": 0.83821356, + "num_input_tokens_seen": 13537815, + "router_z_loss_clip": 1.75292969, + "router_z_loss_mlp": 0.28808594, + "step": 636, + "time_per_iteration": 2.842989444732666 + }, + { + "auxiliary_loss_clip": 0.01248568, + "auxiliary_loss_mlp": 0.01063025, + "balance_loss_clip": 1.07453847, + "balance_loss_mlp": 1.03455746, + "epoch": 0.07659472133710095, + "flos": 23806799948160.0, + "grad_norm": 2.105540895187914, + "language_loss": 0.816818, + "learning_rate": 3.9773231448505804e-06, + "loss": 0.83993399, + "num_input_tokens_seen": 13559605, + "router_z_loss_clip": 1.74023438, + "router_z_loss_mlp": 0.2845459, + "step": 637, + "time_per_iteration": 2.810682773590088 + }, + { + "auxiliary_loss_clip": 0.01249745, + "auxiliary_loss_mlp": 0.01075897, + "balance_loss_clip": 1.07411647, + "balance_loss_mlp": 1.04568934, + "epoch": 0.07671496422774003, + "flos": 21469984842240.0, + "grad_norm": 17.92631014068769, + "language_loss": 0.78142667, + "learning_rate": 3.977206023792491e-06, + "loss": 0.80468309, + "num_input_tokens_seen": 13579495, + "router_z_loss_clip": 1.75585938, + "router_z_loss_mlp": 0.30249023, + "step": 638, + "time_per_iteration": 2.7800960540771484 + }, + { + "auxiliary_loss_clip": 0.01251632, + "auxiliary_loss_mlp": 0.01068396, + "balance_loss_clip": 1.07813501, + "balance_loss_mlp": 1.03942847, + "epoch": 0.07683520711837913, + "flos": 16980971558400.0, + "grad_norm": 2.329870421321427, + "language_loss": 0.81049752, + "learning_rate": 3.97708860279382e-06, + "loss": 0.8336978, + "num_input_tokens_seen": 13597605, + "router_z_loss_clip": 1.73535156, + "router_z_loss_mlp": 0.28955078, + "step": 639, + "time_per_iteration": 2.7809367179870605 + }, + { + "auxiliary_loss_clip": 0.01249535, + "auxiliary_loss_mlp": 0.01073041, + "balance_loss_clip": 1.07493138, + "balance_loss_mlp": 1.04464531, + "epoch": 0.07695545000901821, + "flos": 23476744851840.0, + "grad_norm": 1.7639347494413045, + "language_loss": 0.78148341, + "learning_rate": 3.97697088187238e-06, + "loss": 0.8047092, + "num_input_tokens_seen": 13618120, + "router_z_loss_clip": 1.74707031, + "router_z_loss_mlp": 0.28405762, + "step": 640, + "time_per_iteration": 2.8028905391693115 + }, + { + "auxiliary_loss_clip": 0.01240646, + "auxiliary_loss_mlp": 0.01062539, + "balance_loss_clip": 1.07122195, + "balance_loss_mlp": 1.03504908, + "epoch": 0.07707569289965731, + "flos": 17634258167040.0, + "grad_norm": 3.18159640486435, + "language_loss": 0.92161679, + "learning_rate": 3.976852861046029e-06, + "loss": 0.94464862, + "num_input_tokens_seen": 13634735, + "router_z_loss_clip": 1.69238281, + "router_z_loss_mlp": 0.27490234, + "step": 641, + "time_per_iteration": 2.7694497108459473 + }, + { + "auxiliary_loss_clip": 0.01246721, + "auxiliary_loss_mlp": 0.01069019, + "balance_loss_clip": 1.07404387, + "balance_loss_mlp": 1.04067063, + "epoch": 0.0771959357902964, + "flos": 25775674087680.0, + "grad_norm": 1.8231957448499216, + "language_loss": 0.80071521, + "learning_rate": 3.97673454033267e-06, + "loss": 0.82387257, + "num_input_tokens_seen": 13656835, + "router_z_loss_clip": 1.72753906, + "router_z_loss_mlp": 0.28369141, + "step": 642, + "time_per_iteration": 2.856812000274658 + }, + { + "auxiliary_loss_clip": 0.01245836, + "auxiliary_loss_mlp": 0.01073025, + "balance_loss_clip": 1.072276, + "balance_loss_mlp": 1.04310298, + "epoch": 0.07731617868093549, + "flos": 19828651847040.0, + "grad_norm": 1.9543661259799376, + "language_loss": 0.82533479, + "learning_rate": 3.976615919750254e-06, + "loss": 0.84852338, + "num_input_tokens_seen": 13674535, + "router_z_loss_clip": 1.73730469, + "router_z_loss_mlp": 0.29907227, + "step": 643, + "time_per_iteration": 2.8931479454040527 + }, + { + "auxiliary_loss_clip": 0.01249577, + "auxiliary_loss_mlp": 0.01081261, + "balance_loss_clip": 1.07440042, + "balance_loss_mlp": 1.05198359, + "epoch": 0.07743642157157458, + "flos": 21324654414720.0, + "grad_norm": 1.9043605338350404, + "language_loss": 0.86626536, + "learning_rate": 3.976496999316775e-06, + "loss": 0.88957369, + "num_input_tokens_seen": 13693290, + "router_z_loss_clip": 1.75195312, + "router_z_loss_mlp": 0.29248047, + "step": 644, + "time_per_iteration": 2.824524402618408 + }, + { + "auxiliary_loss_clip": 0.01252018, + "auxiliary_loss_mlp": 0.01068591, + "balance_loss_clip": 1.0774585, + "balance_loss_mlp": 1.04051757, + "epoch": 0.07755666446221367, + "flos": 19969133938560.0, + "grad_norm": 2.199361855232336, + "language_loss": 0.84340417, + "learning_rate": 3.976377779050271e-06, + "loss": 0.86661029, + "num_input_tokens_seen": 13711420, + "router_z_loss_clip": 1.74511719, + "router_z_loss_mlp": 0.28088379, + "step": 645, + "time_per_iteration": 2.7540571689605713 + }, + { + "auxiliary_loss_clip": 0.0125419, + "auxiliary_loss_mlp": 0.0106869, + "balance_loss_clip": 1.07474971, + "balance_loss_mlp": 1.04075968, + "epoch": 0.07767690735285276, + "flos": 23623224514560.0, + "grad_norm": 2.1370850069512226, + "language_loss": 0.84489566, + "learning_rate": 3.976258258968831e-06, + "loss": 0.86812449, + "num_input_tokens_seen": 13729965, + "router_z_loss_clip": 1.79589844, + "router_z_loss_mlp": 0.27941895, + "step": 646, + "time_per_iteration": 2.8671956062316895 + }, + { + "auxiliary_loss_clip": 0.0124792, + "auxiliary_loss_mlp": 0.01075535, + "balance_loss_clip": 1.07649064, + "balance_loss_mlp": 1.04794979, + "epoch": 0.07779715024349185, + "flos": 22236246702720.0, + "grad_norm": 2.7011234227421723, + "language_loss": 0.74632299, + "learning_rate": 3.976138439090583e-06, + "loss": 0.76955748, + "num_input_tokens_seen": 13748045, + "router_z_loss_clip": 1.71386719, + "router_z_loss_mlp": 0.27600098, + "step": 647, + "time_per_iteration": 2.827615737915039 + }, + { + "auxiliary_loss_clip": 0.01248719, + "auxiliary_loss_mlp": 0.01071666, + "balance_loss_clip": 1.07452273, + "balance_loss_mlp": 1.04098117, + "epoch": 0.07791739313413094, + "flos": 20955097336320.0, + "grad_norm": 2.368180169217483, + "language_loss": 0.85128516, + "learning_rate": 3.976018319433706e-06, + "loss": 0.87448901, + "num_input_tokens_seen": 13765590, + "router_z_loss_clip": 1.74121094, + "router_z_loss_mlp": 0.30664062, + "step": 648, + "time_per_iteration": 2.8235583305358887 + }, + { + "auxiliary_loss_clip": 0.01253789, + "auxiliary_loss_mlp": 0.01058137, + "balance_loss_clip": 1.07743585, + "balance_loss_mlp": 1.02845418, + "epoch": 0.07803763602477004, + "flos": 19312327797120.0, + "grad_norm": 2.683364355543874, + "language_loss": 0.9148525, + "learning_rate": 3.9758979000164205e-06, + "loss": 0.93797171, + "num_input_tokens_seen": 13782410, + "router_z_loss_clip": 1.76464844, + "router_z_loss_mlp": 0.29663086, + "step": 649, + "time_per_iteration": 2.9286956787109375 + }, + { + "auxiliary_loss_clip": 0.01251977, + "auxiliary_loss_mlp": 0.01069073, + "balance_loss_clip": 1.07527494, + "balance_loss_mlp": 1.03848398, + "epoch": 0.07815787891540912, + "flos": 22710806213760.0, + "grad_norm": 1.9294266745274056, + "language_loss": 0.72156072, + "learning_rate": 3.975777180856995e-06, + "loss": 0.74477124, + "num_input_tokens_seen": 13801530, + "router_z_loss_clip": 1.77050781, + "router_z_loss_mlp": 0.3059082, + "step": 650, + "time_per_iteration": 2.835505962371826 + }, + { + "auxiliary_loss_clip": 0.01254372, + "auxiliary_loss_mlp": 0.01068029, + "balance_loss_clip": 1.07405353, + "balance_loss_mlp": 1.03963375, + "epoch": 0.07827812180604822, + "flos": 22711129436160.0, + "grad_norm": 2.3613039428607494, + "language_loss": 0.86191928, + "learning_rate": 3.975656161973742e-06, + "loss": 0.88514334, + "num_input_tokens_seen": 13820615, + "router_z_loss_clip": 1.80175781, + "router_z_loss_mlp": 0.28417969, + "step": 651, + "time_per_iteration": 2.8826892375946045 + }, + { + "auxiliary_loss_clip": 0.012556, + "auxiliary_loss_mlp": 0.0107057, + "balance_loss_clip": 1.07423294, + "balance_loss_mlp": 1.04103017, + "epoch": 0.0783983646966873, + "flos": 21725597001600.0, + "grad_norm": 2.5017896620161935, + "language_loss": 0.88383925, + "learning_rate": 3.9755348433850194e-06, + "loss": 0.90710092, + "num_input_tokens_seen": 13835955, + "router_z_loss_clip": 1.81152344, + "router_z_loss_mlp": 0.2956543, + "step": 652, + "time_per_iteration": 2.9247210025787354 + }, + { + "auxiliary_loss_clip": 0.01129955, + "auxiliary_loss_mlp": 0.01012999, + "balance_loss_clip": 1.05081654, + "balance_loss_mlp": 1.00715816, + "epoch": 0.0785186075873264, + "flos": 60640877537280.0, + "grad_norm": 0.9842448621361841, + "language_loss": 0.63613796, + "learning_rate": 3.975413225109232e-06, + "loss": 0.6575675, + "num_input_tokens_seen": 13896505, + "router_z_loss_clip": 0.7890625, + "router_z_loss_mlp": 0.05834961, + "step": 653, + "time_per_iteration": 5.074479579925537 + }, + { + "auxiliary_loss_clip": 0.01250129, + "auxiliary_loss_mlp": 0.01067825, + "balance_loss_clip": 1.07332826, + "balance_loss_mlp": 1.03661561, + "epoch": 0.0786388504779655, + "flos": 23877902920320.0, + "grad_norm": 3.0971945627165227, + "language_loss": 0.93573046, + "learning_rate": 3.975291307164829e-06, + "loss": 0.95890999, + "num_input_tokens_seen": 13915150, + "router_z_loss_clip": 1.765625, + "router_z_loss_mlp": 0.31225586, + "step": 654, + "time_per_iteration": 4.622396469116211 + }, + { + "auxiliary_loss_clip": 0.01244019, + "auxiliary_loss_mlp": 0.01056376, + "balance_loss_clip": 1.07471609, + "balance_loss_mlp": 1.02745605, + "epoch": 0.07875909336860458, + "flos": 15158684822400.0, + "grad_norm": 2.26204125888344, + "language_loss": 0.85395354, + "learning_rate": 3.975169089570306e-06, + "loss": 0.87695748, + "num_input_tokens_seen": 13933525, + "router_z_loss_clip": 1.69335938, + "router_z_loss_mlp": 0.2890625, + "step": 655, + "time_per_iteration": 2.7798140048980713 + }, + { + "auxiliary_loss_clip": 0.01248256, + "auxiliary_loss_mlp": 0.01070078, + "balance_loss_clip": 1.07566166, + "balance_loss_mlp": 1.04089499, + "epoch": 0.07887933625924368, + "flos": 22236857233920.0, + "grad_norm": 2.2623265701668345, + "language_loss": 0.91866112, + "learning_rate": 3.975046572344202e-06, + "loss": 0.94184446, + "num_input_tokens_seen": 13949985, + "router_z_loss_clip": 1.72558594, + "router_z_loss_mlp": 0.29174805, + "step": 656, + "time_per_iteration": 2.8214473724365234 + }, + { + "auxiliary_loss_clip": 0.01247706, + "auxiliary_loss_mlp": 0.0106348, + "balance_loss_clip": 1.07424486, + "balance_loss_mlp": 1.03435731, + "epoch": 0.07899957914988276, + "flos": 20777734955520.0, + "grad_norm": 1.7005393824199746, + "language_loss": 0.70992351, + "learning_rate": 3.974923755505103e-06, + "loss": 0.73303533, + "num_input_tokens_seen": 13969215, + "router_z_loss_clip": 1.73535156, + "router_z_loss_mlp": 0.2911377, + "step": 657, + "time_per_iteration": 2.8047244548797607 + }, + { + "auxiliary_loss_clip": 0.01244663, + "auxiliary_loss_mlp": 0.01068384, + "balance_loss_clip": 1.07428753, + "balance_loss_mlp": 1.04224098, + "epoch": 0.07911982204052186, + "flos": 23003047267200.0, + "grad_norm": 1.6718765709389565, + "language_loss": 0.91174108, + "learning_rate": 3.974800639071641e-06, + "loss": 0.93487155, + "num_input_tokens_seen": 13989935, + "router_z_loss_clip": 1.703125, + "router_z_loss_mlp": 0.26159668, + "step": 658, + "time_per_iteration": 2.781578540802002 + }, + { + "auxiliary_loss_clip": 0.01245796, + "auxiliary_loss_mlp": 0.01071758, + "balance_loss_clip": 1.07132387, + "balance_loss_mlp": 1.0415262, + "epoch": 0.07924006493116094, + "flos": 23111389664640.0, + "grad_norm": 2.129047222639126, + "language_loss": 1.00290537, + "learning_rate": 3.974677223062492e-06, + "loss": 1.02608085, + "num_input_tokens_seen": 14007150, + "router_z_loss_clip": 1.74511719, + "router_z_loss_mlp": 0.30224609, + "step": 659, + "time_per_iteration": 2.7979300022125244 + }, + { + "auxiliary_loss_clip": 0.0125345, + "auxiliary_loss_mlp": 0.0107453, + "balance_loss_clip": 1.07753706, + "balance_loss_mlp": 1.04799414, + "epoch": 0.07936030782180004, + "flos": 16472153450880.0, + "grad_norm": 2.6091534406831696, + "language_loss": 0.74369347, + "learning_rate": 3.974553507496378e-06, + "loss": 0.76697326, + "num_input_tokens_seen": 14025725, + "router_z_loss_clip": 1.75976562, + "router_z_loss_mlp": 0.265625, + "step": 660, + "time_per_iteration": 2.82218074798584 + }, + { + "auxiliary_loss_clip": 0.01256938, + "auxiliary_loss_mlp": 0.01070085, + "balance_loss_clip": 1.07576895, + "balance_loss_mlp": 1.03897095, + "epoch": 0.07948055071243913, + "flos": 23733290764800.0, + "grad_norm": 5.335286350124414, + "language_loss": 0.89387631, + "learning_rate": 3.974429492392068e-06, + "loss": 0.91714656, + "num_input_tokens_seen": 14045750, + "router_z_loss_clip": 1.81054688, + "router_z_loss_mlp": 0.3112793, + "step": 661, + "time_per_iteration": 2.775465726852417 + }, + { + "auxiliary_loss_clip": 0.0124742, + "auxiliary_loss_mlp": 0.010663, + "balance_loss_clip": 1.07607853, + "balance_loss_mlp": 1.03635478, + "epoch": 0.07960079360307822, + "flos": 19573326996480.0, + "grad_norm": 3.1950561018948815, + "language_loss": 0.9111743, + "learning_rate": 3.974305177768373e-06, + "loss": 0.93431151, + "num_input_tokens_seen": 14063960, + "router_z_loss_clip": 1.71484375, + "router_z_loss_mlp": 0.29956055, + "step": 662, + "time_per_iteration": 2.799206495285034 + }, + { + "auxiliary_loss_clip": 0.01251042, + "auxiliary_loss_mlp": 0.01068055, + "balance_loss_clip": 1.07831919, + "balance_loss_mlp": 1.03844333, + "epoch": 0.07972103649371731, + "flos": 23513409659520.0, + "grad_norm": 2.6661966966050716, + "language_loss": 0.86512423, + "learning_rate": 3.974180563644152e-06, + "loss": 0.8883152, + "num_input_tokens_seen": 14082525, + "router_z_loss_clip": 1.7265625, + "router_z_loss_mlp": 0.29589844, + "step": 663, + "time_per_iteration": 2.8901824951171875 + }, + { + "auxiliary_loss_clip": 0.01247726, + "auxiliary_loss_mlp": 0.01076164, + "balance_loss_clip": 1.07552338, + "balance_loss_mlp": 1.04423964, + "epoch": 0.0798412793843564, + "flos": 16726867770240.0, + "grad_norm": 2.2914177137547735, + "language_loss": 0.88782001, + "learning_rate": 3.97405565003831e-06, + "loss": 0.9110589, + "num_input_tokens_seen": 14098610, + "router_z_loss_clip": 1.72167969, + "router_z_loss_mlp": 0.3190918, + "step": 664, + "time_per_iteration": 2.844447612762451 + }, + { + "auxiliary_loss_clip": 0.01245925, + "auxiliary_loss_mlp": 0.0106754, + "balance_loss_clip": 1.07320821, + "balance_loss_mlp": 1.03811884, + "epoch": 0.07996152227499549, + "flos": 18223337214720.0, + "grad_norm": 2.0519817458161445, + "language_loss": 0.78256804, + "learning_rate": 3.973930436969794e-06, + "loss": 0.80570269, + "num_input_tokens_seen": 14117065, + "router_z_loss_clip": 1.72558594, + "router_z_loss_mlp": 0.29431152, + "step": 665, + "time_per_iteration": 2.916292905807495 + }, + { + "auxiliary_loss_clip": 0.01253029, + "auxiliary_loss_mlp": 0.01086964, + "balance_loss_clip": 1.07590961, + "balance_loss_mlp": 1.05575478, + "epoch": 0.08008176516563459, + "flos": 20594877793920.0, + "grad_norm": 2.085791204846022, + "language_loss": 0.85989451, + "learning_rate": 3.973804924457602e-06, + "loss": 0.88329434, + "num_input_tokens_seen": 14135145, + "router_z_loss_clip": 1.77246094, + "router_z_loss_mlp": 0.31176758, + "step": 666, + "time_per_iteration": 2.829164505004883 + }, + { + "auxiliary_loss_clip": 0.01254328, + "auxiliary_loss_mlp": 0.01079286, + "balance_loss_clip": 1.08023548, + "balance_loss_mlp": 1.04806554, + "epoch": 0.08020200805627367, + "flos": 31834306863360.0, + "grad_norm": 1.8153682371263664, + "language_loss": 0.85836554, + "learning_rate": 3.973679112520771e-06, + "loss": 0.88170165, + "num_input_tokens_seen": 14156860, + "router_z_loss_clip": 1.73828125, + "router_z_loss_mlp": 0.31213379, + "step": 667, + "time_per_iteration": 3.0255069732666016 + }, + { + "auxiliary_loss_clip": 0.01245068, + "auxiliary_loss_mlp": 0.010782, + "balance_loss_clip": 1.07430971, + "balance_loss_mlp": 1.05111551, + "epoch": 0.08032225094691277, + "flos": 17783503176960.0, + "grad_norm": 2.0560065372228125, + "language_loss": 0.98897707, + "learning_rate": 3.973553001178389e-06, + "loss": 1.01220965, + "num_input_tokens_seen": 14174365, + "router_z_loss_clip": 1.70703125, + "router_z_loss_mlp": 0.27075195, + "step": 668, + "time_per_iteration": 2.874849319458008 + }, + { + "auxiliary_loss_clip": 0.01245607, + "auxiliary_loss_mlp": 0.01069421, + "balance_loss_clip": 1.07460535, + "balance_loss_mlp": 1.03825927, + "epoch": 0.08044249383755185, + "flos": 24061693835520.0, + "grad_norm": 2.0351058264229644, + "language_loss": 0.75848478, + "learning_rate": 3.973426590449585e-06, + "loss": 0.78163505, + "num_input_tokens_seen": 14192320, + "router_z_loss_clip": 1.71191406, + "router_z_loss_mlp": 0.31152344, + "step": 669, + "time_per_iteration": 2.826569080352783 + }, + { + "auxiliary_loss_clip": 0.01249475, + "auxiliary_loss_mlp": 0.01072951, + "balance_loss_clip": 1.0744921, + "balance_loss_mlp": 1.04455566, + "epoch": 0.08056273672819095, + "flos": 18223624523520.0, + "grad_norm": 1.967534540741977, + "language_loss": 0.75152045, + "learning_rate": 3.9732998803535364e-06, + "loss": 0.77474475, + "num_input_tokens_seen": 14210380, + "router_z_loss_clip": 1.74902344, + "router_z_loss_mlp": 0.28393555, + "step": 670, + "time_per_iteration": 2.822230577468872 + }, + { + "auxiliary_loss_clip": 0.01243153, + "auxiliary_loss_mlp": 0.01071487, + "balance_loss_clip": 1.07255244, + "balance_loss_mlp": 1.04423547, + "epoch": 0.08068297961883003, + "flos": 19676856971520.0, + "grad_norm": 2.575184140006771, + "language_loss": 0.85440761, + "learning_rate": 3.973172870909465e-06, + "loss": 0.877554, + "num_input_tokens_seen": 14225145, + "router_z_loss_clip": 1.70703125, + "router_z_loss_mlp": 0.2722168, + "step": 671, + "time_per_iteration": 2.8308207988739014 + }, + { + "auxiliary_loss_clip": 0.01248853, + "auxiliary_loss_mlp": 0.01062952, + "balance_loss_clip": 1.07468438, + "balance_loss_mlp": 1.03336382, + "epoch": 0.08080322250946913, + "flos": 23148736830720.0, + "grad_norm": 2.5761881761733583, + "language_loss": 0.81319082, + "learning_rate": 3.973045562136638e-06, + "loss": 0.83630896, + "num_input_tokens_seen": 14241960, + "router_z_loss_clip": 1.74316406, + "router_z_loss_mlp": 0.29614258, + "step": 672, + "time_per_iteration": 2.868079423904419 + }, + { + "auxiliary_loss_clip": 0.01247544, + "auxiliary_loss_mlp": 0.01064933, + "balance_loss_clip": 1.07528436, + "balance_loss_mlp": 1.03798008, + "epoch": 0.08092346540010822, + "flos": 21763626526080.0, + "grad_norm": 2.053737416907425, + "language_loss": 0.91178942, + "learning_rate": 3.972917954054368e-06, + "loss": 0.93491423, + "num_input_tokens_seen": 14260515, + "router_z_loss_clip": 1.72265625, + "router_z_loss_mlp": 0.26928711, + "step": 673, + "time_per_iteration": 2.957719326019287 + }, + { + "auxiliary_loss_clip": 0.01258154, + "auxiliary_loss_mlp": 0.01073852, + "balance_loss_clip": 1.07826328, + "balance_loss_mlp": 1.04481292, + "epoch": 0.08104370829074731, + "flos": 21032485188480.0, + "grad_norm": 2.3165818448096642, + "language_loss": 0.82054305, + "learning_rate": 3.972790046682013e-06, + "loss": 0.84386319, + "num_input_tokens_seen": 14279190, + "router_z_loss_clip": 1.79882812, + "router_z_loss_mlp": 0.29052734, + "step": 674, + "time_per_iteration": 2.9950692653656006 + }, + { + "auxiliary_loss_clip": 0.01246875, + "auxiliary_loss_mlp": 0.01063031, + "balance_loss_clip": 1.07282472, + "balance_loss_mlp": 1.03446841, + "epoch": 0.0811639511813864, + "flos": 20083186598400.0, + "grad_norm": 1.8301874463027, + "language_loss": 0.7891016, + "learning_rate": 3.972661840038977e-06, + "loss": 0.81220067, + "num_input_tokens_seen": 14299480, + "router_z_loss_clip": 1.73925781, + "router_z_loss_mlp": 0.28564453, + "step": 675, + "time_per_iteration": 2.7818706035614014 + }, + { + "auxiliary_loss_clip": 0.01254459, + "auxiliary_loss_mlp": 0.01068697, + "balance_loss_clip": 1.07808304, + "balance_loss_mlp": 1.04155278, + "epoch": 0.08128419407202549, + "flos": 16836718538880.0, + "grad_norm": 2.162292823818419, + "language_loss": 0.83402807, + "learning_rate": 3.972533334144707e-06, + "loss": 0.85725957, + "num_input_tokens_seen": 14316405, + "router_z_loss_clip": 1.76269531, + "router_z_loss_mlp": 0.27160645, + "step": 676, + "time_per_iteration": 2.775923252105713 + }, + { + "auxiliary_loss_clip": 0.01246473, + "auxiliary_loss_mlp": 0.01060133, + "balance_loss_clip": 1.07338047, + "balance_loss_mlp": 1.03351378, + "epoch": 0.08140443696266458, + "flos": 23769273214080.0, + "grad_norm": 2.1359521186320487, + "language_loss": 0.78186333, + "learning_rate": 3.972404529018699e-06, + "loss": 0.80492938, + "num_input_tokens_seen": 14336265, + "router_z_loss_clip": 1.72949219, + "router_z_loss_mlp": 0.26623535, + "step": 677, + "time_per_iteration": 2.841609001159668 + }, + { + "auxiliary_loss_clip": 0.01246063, + "auxiliary_loss_mlp": 0.01074259, + "balance_loss_clip": 1.07176852, + "balance_loss_mlp": 1.04696035, + "epoch": 0.08152467985330367, + "flos": 24390132819840.0, + "grad_norm": 2.0891975072262903, + "language_loss": 0.85221094, + "learning_rate": 3.972275424680493e-06, + "loss": 0.87541413, + "num_input_tokens_seen": 14356375, + "router_z_loss_clip": 1.74316406, + "router_z_loss_mlp": 0.27294922, + "step": 678, + "time_per_iteration": 2.9037699699401855 + }, + { + "auxiliary_loss_clip": 0.01244352, + "auxiliary_loss_mlp": 0.01063197, + "balance_loss_clip": 1.07422912, + "balance_loss_mlp": 1.0360415, + "epoch": 0.08164492274394276, + "flos": 19317750750720.0, + "grad_norm": 2.0313741207967744, + "language_loss": 0.9187085, + "learning_rate": 3.972146021149673e-06, + "loss": 0.94178402, + "num_input_tokens_seen": 14374650, + "router_z_loss_clip": 1.70117188, + "router_z_loss_mlp": 0.27185059, + "step": 679, + "time_per_iteration": 2.844205856323242 + }, + { + "auxiliary_loss_clip": 0.01243199, + "auxiliary_loss_mlp": 0.01057435, + "balance_loss_clip": 1.07363033, + "balance_loss_mlp": 1.03122091, + "epoch": 0.08176516563458186, + "flos": 14830461319680.0, + "grad_norm": 2.2507681169419715, + "language_loss": 0.78592277, + "learning_rate": 3.972016318445868e-06, + "loss": 0.80892909, + "num_input_tokens_seen": 14392650, + "router_z_loss_clip": 1.69433594, + "router_z_loss_mlp": 0.2623291, + "step": 680, + "time_per_iteration": 2.8534834384918213 + }, + { + "auxiliary_loss_clip": 0.01242741, + "auxiliary_loss_mlp": 0.01065668, + "balance_loss_clip": 1.07093513, + "balance_loss_mlp": 1.03829765, + "epoch": 0.08188540852522094, + "flos": 22602320161920.0, + "grad_norm": 2.1023931547896297, + "language_loss": 0.9207536, + "learning_rate": 3.971886316588757e-06, + "loss": 0.9438377, + "num_input_tokens_seen": 14413155, + "router_z_loss_clip": 1.71875, + "router_z_loss_mlp": 0.27368164, + "step": 681, + "time_per_iteration": 2.8256523609161377 + }, + { + "auxiliary_loss_clip": 0.01251113, + "auxiliary_loss_mlp": 0.01073982, + "balance_loss_clip": 1.07557273, + "balance_loss_mlp": 1.04625416, + "epoch": 0.08200565141586004, + "flos": 19463727623040.0, + "grad_norm": 2.40878901437028, + "language_loss": 0.73343736, + "learning_rate": 3.9717560155980595e-06, + "loss": 0.7566883, + "num_input_tokens_seen": 14428805, + "router_z_loss_clip": 1.75488281, + "router_z_loss_mlp": 0.27722168, + "step": 682, + "time_per_iteration": 4.476180553436279 + }, + { + "auxiliary_loss_clip": 0.01249293, + "auxiliary_loss_mlp": 0.01069522, + "balance_loss_clip": 1.07444561, + "balance_loss_mlp": 1.04205668, + "epoch": 0.08212589430649912, + "flos": 20594662312320.0, + "grad_norm": 2.2477996640830735, + "language_loss": 0.91778553, + "learning_rate": 3.971625415493542e-06, + "loss": 0.94097376, + "num_input_tokens_seen": 14447125, + "router_z_loss_clip": 1.75, + "router_z_loss_mlp": 0.27453613, + "step": 683, + "time_per_iteration": 3.6286752223968506 + }, + { + "auxiliary_loss_clip": 0.01242913, + "auxiliary_loss_mlp": 0.01061548, + "balance_loss_clip": 1.07328355, + "balance_loss_mlp": 1.03554797, + "epoch": 0.08224613719713822, + "flos": 25953611086080.0, + "grad_norm": 1.8474185300703487, + "language_loss": 0.87783271, + "learning_rate": 3.971494516295017e-06, + "loss": 0.9008773, + "num_input_tokens_seen": 14466575, + "router_z_loss_clip": 1.69726562, + "router_z_loss_mlp": 0.26013184, + "step": 684, + "time_per_iteration": 2.8878185749053955 + }, + { + "auxiliary_loss_clip": 0.0124431, + "auxiliary_loss_mlp": 0.01069764, + "balance_loss_clip": 1.07411647, + "balance_loss_mlp": 1.04074836, + "epoch": 0.08236638008777732, + "flos": 23768734510080.0, + "grad_norm": 2.3654791666370616, + "language_loss": 0.85276634, + "learning_rate": 3.971363318022341e-06, + "loss": 0.87590706, + "num_input_tokens_seen": 14487915, + "router_z_loss_clip": 1.70117188, + "router_z_loss_mlp": 0.2902832, + "step": 685, + "time_per_iteration": 2.936729669570923 + }, + { + "auxiliary_loss_clip": 0.01247223, + "auxiliary_loss_mlp": 0.01068108, + "balance_loss_clip": 1.07383585, + "balance_loss_mlp": 1.04030883, + "epoch": 0.0824866229784164, + "flos": 38799144887040.0, + "grad_norm": 2.2711793899875428, + "language_loss": 0.68395138, + "learning_rate": 3.971231820695417e-06, + "loss": 0.70710468, + "num_input_tokens_seen": 14511530, + "router_z_loss_clip": 1.73339844, + "router_z_loss_mlp": 0.27758789, + "step": 686, + "time_per_iteration": 2.9349706172943115 + }, + { + "auxiliary_loss_clip": 0.01242679, + "auxiliary_loss_mlp": 0.01056856, + "balance_loss_clip": 1.07240677, + "balance_loss_mlp": 1.03068972, + "epoch": 0.0826068658690555, + "flos": 23107762391040.0, + "grad_norm": 2.199907464006476, + "language_loss": 0.81591135, + "learning_rate": 3.971100024334193e-06, + "loss": 0.83890665, + "num_input_tokens_seen": 14529050, + "router_z_loss_clip": 1.70117188, + "router_z_loss_mlp": 0.26159668, + "step": 687, + "time_per_iteration": 2.86897611618042 + }, + { + "auxiliary_loss_clip": 0.01237839, + "auxiliary_loss_mlp": 0.01061811, + "balance_loss_clip": 1.07087922, + "balance_loss_mlp": 1.03451228, + "epoch": 0.08272710875969458, + "flos": 21136374299520.0, + "grad_norm": 2.0445100845528654, + "language_loss": 0.86361992, + "learning_rate": 3.970967928958663e-06, + "loss": 0.88661647, + "num_input_tokens_seen": 14546165, + "router_z_loss_clip": 1.66992188, + "router_z_loss_mlp": 0.27294922, + "step": 688, + "time_per_iteration": 2.853106737136841 + }, + { + "auxiliary_loss_clip": 0.01244037, + "auxiliary_loss_mlp": 0.010733, + "balance_loss_clip": 1.0765661, + "balance_loss_mlp": 1.04522634, + "epoch": 0.08284735165033368, + "flos": 19063000517760.0, + "grad_norm": 1.6302356217757972, + "language_loss": 0.83458197, + "learning_rate": 3.970835534588865e-06, + "loss": 0.8577553, + "num_input_tokens_seen": 14563660, + "router_z_loss_clip": 1.67578125, + "router_z_loss_mlp": 0.28051758, + "step": 689, + "time_per_iteration": 2.821593999862671 + }, + { + "auxiliary_loss_clip": 0.01248948, + "auxiliary_loss_mlp": 0.01082431, + "balance_loss_clip": 1.07776237, + "balance_loss_mlp": 1.05389237, + "epoch": 0.08296759454097276, + "flos": 16727442387840.0, + "grad_norm": 1.797494933021298, + "language_loss": 0.85803097, + "learning_rate": 3.970702841244883e-06, + "loss": 0.8813448, + "num_input_tokens_seen": 14581980, + "router_z_loss_clip": 1.71191406, + "router_z_loss_mlp": 0.28564453, + "step": 690, + "time_per_iteration": 2.921100616455078 + }, + { + "auxiliary_loss_clip": 0.01248377, + "auxiliary_loss_mlp": 0.01070661, + "balance_loss_clip": 1.07534766, + "balance_loss_mlp": 1.04238439, + "epoch": 0.08308783743161186, + "flos": 18004928567040.0, + "grad_norm": 1.7472376821364075, + "language_loss": 0.82682979, + "learning_rate": 3.970569848946847e-06, + "loss": 0.85002017, + "num_input_tokens_seen": 14601795, + "router_z_loss_clip": 1.72949219, + "router_z_loss_mlp": 0.28295898, + "step": 691, + "time_per_iteration": 2.916417121887207 + }, + { + "auxiliary_loss_clip": 0.01245156, + "auxiliary_loss_mlp": 0.01073164, + "balance_loss_clip": 1.07590461, + "balance_loss_mlp": 1.04337394, + "epoch": 0.08320808032225095, + "flos": 15079788599040.0, + "grad_norm": 2.2903409896909395, + "language_loss": 0.83458209, + "learning_rate": 3.970436557714932e-06, + "loss": 0.85776532, + "num_input_tokens_seen": 14618315, + "router_z_loss_clip": 1.69433594, + "router_z_loss_mlp": 0.29772949, + "step": 692, + "time_per_iteration": 2.8977210521698 + }, + { + "auxiliary_loss_clip": 0.0124719, + "auxiliary_loss_mlp": 0.01063276, + "balance_loss_clip": 1.07288408, + "balance_loss_mlp": 1.0351187, + "epoch": 0.08332832321289003, + "flos": 22383085501440.0, + "grad_norm": 2.137640172167897, + "language_loss": 0.86813188, + "learning_rate": 3.970302967569358e-06, + "loss": 0.89123654, + "num_input_tokens_seen": 14636905, + "router_z_loss_clip": 1.74511719, + "router_z_loss_mlp": 0.28149414, + "step": 693, + "time_per_iteration": 2.991401195526123 + }, + { + "auxiliary_loss_clip": 0.01248176, + "auxiliary_loss_mlp": 0.01080158, + "balance_loss_clip": 1.07606816, + "balance_loss_mlp": 1.05171502, + "epoch": 0.08344856610352913, + "flos": 24717386655360.0, + "grad_norm": 2.6745886040432056, + "language_loss": 0.6843456, + "learning_rate": 3.9701690785303896e-06, + "loss": 0.70762897, + "num_input_tokens_seen": 14656100, + "router_z_loss_clip": 1.72265625, + "router_z_loss_mlp": 0.28442383, + "step": 694, + "time_per_iteration": 2.860116481781006 + }, + { + "auxiliary_loss_clip": 0.01250878, + "auxiliary_loss_mlp": 0.01074408, + "balance_loss_clip": 1.07638001, + "balance_loss_mlp": 1.04439092, + "epoch": 0.08356880899416821, + "flos": 25370206387200.0, + "grad_norm": 2.236356777024055, + "language_loss": 0.88369352, + "learning_rate": 3.970034890618339e-06, + "loss": 0.90694636, + "num_input_tokens_seen": 14675790, + "router_z_loss_clip": 1.74511719, + "router_z_loss_mlp": 0.30004883, + "step": 695, + "time_per_iteration": 2.894423007965088 + }, + { + "auxiliary_loss_clip": 0.01246552, + "auxiliary_loss_mlp": 0.0106992, + "balance_loss_clip": 1.07365572, + "balance_loss_mlp": 1.04294264, + "epoch": 0.08368905188480731, + "flos": 24353072962560.0, + "grad_norm": 3.6308145518396455, + "language_loss": 0.8825351, + "learning_rate": 3.969900403853562e-06, + "loss": 0.90569985, + "num_input_tokens_seen": 14694830, + "router_z_loss_clip": 1.72753906, + "router_z_loss_mlp": 0.26940918, + "step": 696, + "time_per_iteration": 2.814730167388916 + }, + { + "auxiliary_loss_clip": 0.01248344, + "auxiliary_loss_mlp": 0.01071557, + "balance_loss_clip": 1.07679999, + "balance_loss_mlp": 1.04316127, + "epoch": 0.08380929477544641, + "flos": 18037319656320.0, + "grad_norm": 1.700549168474178, + "language_loss": 0.78011739, + "learning_rate": 3.96976561825646e-06, + "loss": 0.80331635, + "num_input_tokens_seen": 14711920, + "router_z_loss_clip": 1.71484375, + "router_z_loss_mlp": 0.28442383, + "step": 697, + "time_per_iteration": 2.7716758251190186 + }, + { + "auxiliary_loss_clip": 0.01242412, + "auxiliary_loss_mlp": 0.01066425, + "balance_loss_clip": 1.07387686, + "balance_loss_mlp": 1.03827918, + "epoch": 0.08392953766608549, + "flos": 26286287875200.0, + "grad_norm": 1.9254779062913092, + "language_loss": 0.87354499, + "learning_rate": 3.969630533847479e-06, + "loss": 0.89663339, + "num_input_tokens_seen": 14730880, + "router_z_loss_clip": 1.68457031, + "router_z_loss_mlp": 0.28137207, + "step": 698, + "time_per_iteration": 2.8368546962738037 + }, + { + "auxiliary_loss_clip": 0.01245979, + "auxiliary_loss_mlp": 0.0106094, + "balance_loss_clip": 1.07625175, + "balance_loss_mlp": 1.03269947, + "epoch": 0.08404978055672459, + "flos": 22492146170880.0, + "grad_norm": 4.4787667007578635, + "language_loss": 0.84047914, + "learning_rate": 3.969495150647113e-06, + "loss": 0.8635484, + "num_input_tokens_seen": 14749050, + "router_z_loss_clip": 1.6953125, + "router_z_loss_mlp": 0.28222656, + "step": 699, + "time_per_iteration": 2.8168325424194336 + }, + { + "auxiliary_loss_clip": 0.0125066, + "auxiliary_loss_mlp": 0.0106897, + "balance_loss_clip": 1.07869554, + "balance_loss_mlp": 1.04113412, + "epoch": 0.08417002344736367, + "flos": 24826878288000.0, + "grad_norm": 1.7002680948841349, + "language_loss": 0.76522219, + "learning_rate": 3.969359468675899e-06, + "loss": 0.78841853, + "num_input_tokens_seen": 14769180, + "router_z_loss_clip": 1.71972656, + "router_z_loss_mlp": 0.27819824, + "step": 700, + "time_per_iteration": 2.83823823928833 + }, + { + "auxiliary_loss_clip": 0.01243561, + "auxiliary_loss_mlp": 0.01051007, + "balance_loss_clip": 1.07402468, + "balance_loss_mlp": 1.02431607, + "epoch": 0.08429026633800277, + "flos": 16945922862720.0, + "grad_norm": 1.9087855166501926, + "language_loss": 0.89559454, + "learning_rate": 3.969223487954418e-06, + "loss": 0.91854024, + "num_input_tokens_seen": 14786640, + "router_z_loss_clip": 1.6953125, + "router_z_loss_mlp": 0.26696777, + "step": 701, + "time_per_iteration": 2.8094818592071533 + }, + { + "auxiliary_loss_clip": 0.01240729, + "auxiliary_loss_mlp": 0.01075697, + "balance_loss_clip": 1.07336211, + "balance_loss_mlp": 1.04891062, + "epoch": 0.08441050922864185, + "flos": 23841920471040.0, + "grad_norm": 3.19277976173539, + "language_loss": 0.82670557, + "learning_rate": 3.969087208503301e-06, + "loss": 0.84986985, + "num_input_tokens_seen": 14806720, + "router_z_loss_clip": 1.67480469, + "router_z_loss_mlp": 0.26757812, + "step": 702, + "time_per_iteration": 2.792001724243164 + }, + { + "auxiliary_loss_clip": 0.01247075, + "auxiliary_loss_mlp": 0.01059581, + "balance_loss_clip": 1.07457554, + "balance_loss_mlp": 1.0304935, + "epoch": 0.08453075211928095, + "flos": 25520205582720.0, + "grad_norm": 2.3313679128065985, + "language_loss": 0.84696198, + "learning_rate": 3.968950630343219e-06, + "loss": 0.8700285, + "num_input_tokens_seen": 14823705, + "router_z_loss_clip": 1.72558594, + "router_z_loss_mlp": 0.29101562, + "step": 703, + "time_per_iteration": 2.8245058059692383 + }, + { + "auxiliary_loss_clip": 0.0124284, + "auxiliary_loss_mlp": 0.01067882, + "balance_loss_clip": 1.06953776, + "balance_loss_mlp": 1.04072642, + "epoch": 0.08465099500992004, + "flos": 19532496211200.0, + "grad_norm": 1.9662017254892992, + "language_loss": 0.93852299, + "learning_rate": 3.968813753494892e-06, + "loss": 0.96163017, + "num_input_tokens_seen": 14841865, + "router_z_loss_clip": 1.73242188, + "router_z_loss_mlp": 0.27172852, + "step": 704, + "time_per_iteration": 2.8416736125946045 + }, + { + "auxiliary_loss_clip": 0.01241916, + "auxiliary_loss_mlp": 0.01063445, + "balance_loss_clip": 1.07398319, + "balance_loss_mlp": 1.03689671, + "epoch": 0.08477123790055913, + "flos": 29351299403520.0, + "grad_norm": 3.2152471030557326, + "language_loss": 0.75640583, + "learning_rate": 3.968676577979084e-06, + "loss": 0.77945942, + "num_input_tokens_seen": 14861415, + "router_z_loss_clip": 1.67773438, + "router_z_loss_mlp": 0.265625, + "step": 705, + "time_per_iteration": 2.946885585784912 + }, + { + "auxiliary_loss_clip": 0.01244398, + "auxiliary_loss_mlp": 0.01057702, + "balance_loss_clip": 1.07569885, + "balance_loss_mlp": 1.0304625, + "epoch": 0.08489148079119822, + "flos": 18624495283200.0, + "grad_norm": 2.095086496881759, + "language_loss": 0.78163403, + "learning_rate": 3.968539103816605e-06, + "loss": 0.80465508, + "num_input_tokens_seen": 14879215, + "router_z_loss_clip": 1.68554688, + "router_z_loss_mlp": 0.2722168, + "step": 706, + "time_per_iteration": 2.8976144790649414 + }, + { + "auxiliary_loss_clip": 0.01245144, + "auxiliary_loss_mlp": 0.0106937, + "balance_loss_clip": 1.07702184, + "balance_loss_mlp": 1.04191577, + "epoch": 0.0850117236818373, + "flos": 23471393725440.0, + "grad_norm": 2.3592323084355127, + "language_loss": 0.89033031, + "learning_rate": 3.9684013310283085e-06, + "loss": 0.91347551, + "num_input_tokens_seen": 14897900, + "router_z_loss_clip": 1.68261719, + "router_z_loss_mlp": 0.27429199, + "step": 707, + "time_per_iteration": 2.903339147567749 + }, + { + "auxiliary_loss_clip": 0.01243669, + "auxiliary_loss_mlp": 0.01060489, + "balance_loss_clip": 1.07588005, + "balance_loss_mlp": 1.03376186, + "epoch": 0.0851319665724764, + "flos": 40625058896640.0, + "grad_norm": 1.7849963409179872, + "language_loss": 0.63868237, + "learning_rate": 3.9682632596350956e-06, + "loss": 0.66172397, + "num_input_tokens_seen": 14919065, + "router_z_loss_clip": 1.67675781, + "router_z_loss_mlp": 0.26708984, + "step": 708, + "time_per_iteration": 3.0063064098358154 + }, + { + "auxiliary_loss_clip": 0.01243193, + "auxiliary_loss_mlp": 0.01065691, + "balance_loss_clip": 1.07580614, + "balance_loss_mlp": 1.03662777, + "epoch": 0.0852522094631155, + "flos": 15879554870400.0, + "grad_norm": 1.933426088065015, + "language_loss": 0.78370321, + "learning_rate": 3.968124889657911e-06, + "loss": 0.80679208, + "num_input_tokens_seen": 14934165, + "router_z_loss_clip": 1.67578125, + "router_z_loss_mlp": 0.29064941, + "step": 709, + "time_per_iteration": 2.849815845489502 + }, + { + "auxiliary_loss_clip": 0.01241035, + "auxiliary_loss_mlp": 0.01066624, + "balance_loss_clip": 1.07267237, + "balance_loss_mlp": 1.0398016, + "epoch": 0.08537245235375458, + "flos": 14567091822720.0, + "grad_norm": 2.3683235406440795, + "language_loss": 0.90469313, + "learning_rate": 3.967986221117746e-06, + "loss": 0.92776978, + "num_input_tokens_seen": 14950105, + "router_z_loss_clip": 1.68164062, + "router_z_loss_mlp": 0.26794434, + "step": 710, + "time_per_iteration": 2.8335037231445312 + }, + { + "auxiliary_loss_clip": 0.01242455, + "auxiliary_loss_mlp": 0.01073037, + "balance_loss_clip": 1.07455587, + "balance_loss_mlp": 1.0449754, + "epoch": 0.08549269524439368, + "flos": 26468929555200.0, + "grad_norm": 17.459458633446108, + "language_loss": 0.86671495, + "learning_rate": 3.967847254035635e-06, + "loss": 0.88986987, + "num_input_tokens_seen": 14969490, + "router_z_loss_clip": 1.68066406, + "router_z_loss_mlp": 0.28076172, + "step": 711, + "time_per_iteration": 4.530505418777466 + }, + { + "auxiliary_loss_clip": 0.01248609, + "auxiliary_loss_mlp": 0.01060282, + "balance_loss_clip": 1.07547736, + "balance_loss_mlp": 1.03195822, + "epoch": 0.08561293813503276, + "flos": 13590214565760.0, + "grad_norm": 4.05945088197534, + "language_loss": 0.86658889, + "learning_rate": 3.967707988432661e-06, + "loss": 0.88967776, + "num_input_tokens_seen": 14987195, + "router_z_loss_clip": 1.72949219, + "router_z_loss_mlp": 0.2833252, + "step": 712, + "time_per_iteration": 3.6208865642547607 + }, + { + "auxiliary_loss_clip": 0.01244272, + "auxiliary_loss_mlp": 0.01074726, + "balance_loss_clip": 1.07326126, + "balance_loss_mlp": 1.0467118, + "epoch": 0.08573318102567186, + "flos": 26943524979840.0, + "grad_norm": 2.490171394003899, + "language_loss": 0.8799178, + "learning_rate": 3.967568424329949e-06, + "loss": 0.90310776, + "num_input_tokens_seen": 15007620, + "router_z_loss_clip": 1.7109375, + "router_z_loss_mlp": 0.28027344, + "step": 713, + "time_per_iteration": 2.869950294494629 + }, + { + "auxiliary_loss_clip": 0.01153484, + "auxiliary_loss_mlp": 0.01029007, + "balance_loss_clip": 1.07486975, + "balance_loss_mlp": 1.02247453, + "epoch": 0.08585342391631094, + "flos": 67302739319040.0, + "grad_norm": 0.8346704877813445, + "language_loss": 0.55506879, + "learning_rate": 3.967428561748671e-06, + "loss": 0.57689375, + "num_input_tokens_seen": 15075590, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.06542969, + "step": 714, + "time_per_iteration": 3.4733831882476807 + }, + { + "auxiliary_loss_clip": 0.01246871, + "auxiliary_loss_mlp": 0.01062864, + "balance_loss_clip": 1.07271671, + "balance_loss_mlp": 1.03439689, + "epoch": 0.08597366680695004, + "flos": 22456594684800.0, + "grad_norm": 2.0380683462608657, + "language_loss": 0.87726974, + "learning_rate": 3.967288400710045e-06, + "loss": 0.90036708, + "num_input_tokens_seen": 15095055, + "router_z_loss_clip": 1.74121094, + "router_z_loss_mlp": 0.28466797, + "step": 715, + "time_per_iteration": 2.8984766006469727 + }, + { + "auxiliary_loss_clip": 0.01245289, + "auxiliary_loss_mlp": 0.01074385, + "balance_loss_clip": 1.07727575, + "balance_loss_mlp": 1.04646635, + "epoch": 0.08609390969758914, + "flos": 23550505430400.0, + "grad_norm": 1.9786583673433862, + "language_loss": 0.88431263, + "learning_rate": 3.9671479412353335e-06, + "loss": 0.90750933, + "num_input_tokens_seen": 15113520, + "router_z_loss_clip": 1.6796875, + "router_z_loss_mlp": 0.27893066, + "step": 716, + "time_per_iteration": 2.836470127105713 + }, + { + "auxiliary_loss_clip": 0.01243297, + "auxiliary_loss_mlp": 0.01060037, + "balance_loss_clip": 1.07185364, + "balance_loss_mlp": 1.03303635, + "epoch": 0.08621415258822822, + "flos": 25885848078720.0, + "grad_norm": 2.085334051529574, + "language_loss": 0.74287874, + "learning_rate": 3.967007183345843e-06, + "loss": 0.76591206, + "num_input_tokens_seen": 15133375, + "router_z_loss_clip": 1.71484375, + "router_z_loss_mlp": 0.27026367, + "step": 717, + "time_per_iteration": 2.947204113006592 + }, + { + "auxiliary_loss_clip": 0.01244831, + "auxiliary_loss_mlp": 0.01062631, + "balance_loss_clip": 1.0774498, + "balance_loss_mlp": 1.03409195, + "epoch": 0.08633439547886732, + "flos": 13589568120960.0, + "grad_norm": 2.0876918059047447, + "language_loss": 0.89276004, + "learning_rate": 3.966866127062927e-06, + "loss": 0.91583461, + "num_input_tokens_seen": 15150500, + "router_z_loss_clip": 1.67285156, + "router_z_loss_mlp": 0.28540039, + "step": 718, + "time_per_iteration": 2.897085189819336 + }, + { + "auxiliary_loss_clip": 0.01141335, + "auxiliary_loss_mlp": 0.01008571, + "balance_loss_clip": 1.06286645, + "balance_loss_mlp": 1.00241983, + "epoch": 0.0864546383695064, + "flos": 57767342434560.0, + "grad_norm": 0.9239419435913888, + "language_loss": 0.62690252, + "learning_rate": 3.966724772407982e-06, + "loss": 0.64840156, + "num_input_tokens_seen": 15208015, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.06152344, + "step": 719, + "time_per_iteration": 3.178297519683838 + }, + { + "auxiliary_loss_clip": 0.01241831, + "auxiliary_loss_mlp": 0.01070699, + "balance_loss_clip": 1.07428813, + "balance_loss_mlp": 1.04320979, + "epoch": 0.0865748812601455, + "flos": 20046952753920.0, + "grad_norm": 2.1928945517832688, + "language_loss": 0.88900405, + "learning_rate": 3.966583119402454e-06, + "loss": 0.91212934, + "num_input_tokens_seen": 15224780, + "router_z_loss_clip": 1.67578125, + "router_z_loss_mlp": 0.27490234, + "step": 720, + "time_per_iteration": 2.8183321952819824 + }, + { + "auxiliary_loss_clip": 0.01234488, + "auxiliary_loss_mlp": 0.01061849, + "balance_loss_clip": 1.0701561, + "balance_loss_mlp": 1.03574193, + "epoch": 0.08669512415078459, + "flos": 35262446935680.0, + "grad_norm": 1.6407166830366178, + "language_loss": 0.81970465, + "learning_rate": 3.9664411680678305e-06, + "loss": 0.842668, + "num_input_tokens_seen": 15246535, + "router_z_loss_clip": 1.64160156, + "router_z_loss_mlp": 0.26123047, + "step": 721, + "time_per_iteration": 3.020448923110962 + }, + { + "auxiliary_loss_clip": 0.01133834, + "auxiliary_loss_mlp": 0.01009327, + "balance_loss_clip": 1.05596817, + "balance_loss_mlp": 1.00334251, + "epoch": 0.08681536704142367, + "flos": 65654870048640.0, + "grad_norm": 0.8425465596672486, + "language_loss": 0.61402011, + "learning_rate": 3.966298918425644e-06, + "loss": 0.63545173, + "num_input_tokens_seen": 15304025, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.05981445, + "step": 722, + "time_per_iteration": 3.2227067947387695 + }, + { + "auxiliary_loss_clip": 0.01242981, + "auxiliary_loss_mlp": 0.01072375, + "balance_loss_clip": 1.07449174, + "balance_loss_mlp": 1.04381239, + "epoch": 0.08693560993206277, + "flos": 34529940881280.0, + "grad_norm": 1.8372425247867818, + "language_loss": 0.82727462, + "learning_rate": 3.966156370497476e-06, + "loss": 0.85042816, + "num_input_tokens_seen": 15327635, + "router_z_loss_clip": 1.68457031, + "router_z_loss_mlp": 0.28552246, + "step": 723, + "time_per_iteration": 2.8812777996063232 + }, + { + "auxiliary_loss_clip": 0.01240205, + "auxiliary_loss_mlp": 0.01060829, + "balance_loss_clip": 1.07319617, + "balance_loss_mlp": 1.03269553, + "epoch": 0.08705585282270185, + "flos": 23149419189120.0, + "grad_norm": 1.7622398364945797, + "language_loss": 0.88359714, + "learning_rate": 3.96601352430495e-06, + "loss": 0.90660745, + "num_input_tokens_seen": 15347405, + "router_z_loss_clip": 1.66992188, + "router_z_loss_mlp": 0.28161621, + "step": 724, + "time_per_iteration": 2.868250846862793 + }, + { + "auxiliary_loss_clip": 0.01238415, + "auxiliary_loss_mlp": 0.01058907, + "balance_loss_clip": 1.07263219, + "balance_loss_mlp": 1.03188229, + "epoch": 0.08717609571334095, + "flos": 29497599498240.0, + "grad_norm": 1.4797737131176096, + "language_loss": 0.83048069, + "learning_rate": 3.965870379869735e-06, + "loss": 0.85345387, + "num_input_tokens_seen": 15369450, + "router_z_loss_clip": 1.65722656, + "router_z_loss_mlp": 0.26989746, + "step": 725, + "time_per_iteration": 2.9146220684051514 + }, + { + "auxiliary_loss_clip": 0.01236825, + "auxiliary_loss_mlp": 0.01059053, + "balance_loss_clip": 1.07144046, + "balance_loss_mlp": 1.03282738, + "epoch": 0.08729633860398003, + "flos": 20667489137280.0, + "grad_norm": 2.5507433185687662, + "language_loss": 0.87130642, + "learning_rate": 3.965726937213547e-06, + "loss": 0.89426517, + "num_input_tokens_seen": 15388085, + "router_z_loss_clip": 1.65332031, + "router_z_loss_mlp": 0.26220703, + "step": 726, + "time_per_iteration": 2.8720223903656006 + }, + { + "auxiliary_loss_clip": 0.01244204, + "auxiliary_loss_mlp": 0.01064418, + "balance_loss_clip": 1.07029712, + "balance_loss_mlp": 1.03356671, + "epoch": 0.08741658149461913, + "flos": 18369493655040.0, + "grad_norm": 2.0972198046232773, + "language_loss": 0.8111397, + "learning_rate": 3.965583196358144e-06, + "loss": 0.83422589, + "num_input_tokens_seen": 15407120, + "router_z_loss_clip": 1.73925781, + "router_z_loss_mlp": 0.30859375, + "step": 727, + "time_per_iteration": 2.8660178184509277 + }, + { + "auxiliary_loss_clip": 0.01237946, + "auxiliary_loss_mlp": 0.01065553, + "balance_loss_clip": 1.06799865, + "balance_loss_mlp": 1.03755057, + "epoch": 0.08753682438525823, + "flos": 18729677283840.0, + "grad_norm": 2.8004953008710363, + "language_loss": 0.74185324, + "learning_rate": 3.965439157325335e-06, + "loss": 0.76488823, + "num_input_tokens_seen": 15424485, + "router_z_loss_clip": 1.69726562, + "router_z_loss_mlp": 0.28015137, + "step": 728, + "time_per_iteration": 2.8016769886016846 + }, + { + "auxiliary_loss_clip": 0.01240407, + "auxiliary_loss_mlp": 0.01075289, + "balance_loss_clip": 1.07054353, + "balance_loss_mlp": 1.04684615, + "epoch": 0.08765706727589731, + "flos": 27776113303680.0, + "grad_norm": 5.8664426132342875, + "language_loss": 0.76037109, + "learning_rate": 3.965294820136968e-06, + "loss": 0.78352803, + "num_input_tokens_seen": 15446285, + "router_z_loss_clip": 1.69921875, + "router_z_loss_mlp": 0.28466797, + "step": 729, + "time_per_iteration": 2.8554463386535645 + }, + { + "auxiliary_loss_clip": 0.01241659, + "auxiliary_loss_mlp": 0.01074872, + "balance_loss_clip": 1.07078004, + "balance_loss_mlp": 1.04720354, + "epoch": 0.08777731016653641, + "flos": 24389127239040.0, + "grad_norm": 2.0771254602588503, + "language_loss": 0.86917782, + "learning_rate": 3.965150184814938e-06, + "loss": 0.89234304, + "num_input_tokens_seen": 15465770, + "router_z_loss_clip": 1.70800781, + "router_z_loss_mlp": 0.2767334, + "step": 730, + "time_per_iteration": 2.819544553756714 + }, + { + "auxiliary_loss_clip": 0.01240929, + "auxiliary_loss_mlp": 0.01077976, + "balance_loss_clip": 1.07450175, + "balance_loss_mlp": 1.05085635, + "epoch": 0.08789755305717549, + "flos": 21981855605760.0, + "grad_norm": 2.281094040438165, + "language_loss": 0.76074457, + "learning_rate": 3.965005251381189e-06, + "loss": 0.78393364, + "num_input_tokens_seen": 15483705, + "router_z_loss_clip": 1.66503906, + "router_z_loss_mlp": 0.27148438, + "step": 731, + "time_per_iteration": 2.9437918663024902 + }, + { + "auxiliary_loss_clip": 0.0112879, + "auxiliary_loss_mlp": 0.0102359, + "balance_loss_clip": 1.05225611, + "balance_loss_mlp": 1.01824939, + "epoch": 0.08801779594781459, + "flos": 58360120583040.0, + "grad_norm": 0.9021521704514546, + "language_loss": 0.64569962, + "learning_rate": 3.964860019857705e-06, + "loss": 0.66722345, + "num_input_tokens_seen": 15548620, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.0534668, + "step": 732, + "time_per_iteration": 3.344991683959961 + }, + { + "auxiliary_loss_clip": 0.01243247, + "auxiliary_loss_mlp": 0.01067223, + "balance_loss_clip": 1.07574987, + "balance_loss_mlp": 1.04091382, + "epoch": 0.08813803883845367, + "flos": 23294785530240.0, + "grad_norm": 1.8729311057864038, + "language_loss": 0.84001738, + "learning_rate": 3.964714490266518e-06, + "loss": 0.86312211, + "num_input_tokens_seen": 15569265, + "router_z_loss_clip": 1.67675781, + "router_z_loss_mlp": 0.26318359, + "step": 733, + "time_per_iteration": 2.8600881099700928 + }, + { + "auxiliary_loss_clip": 0.01124694, + "auxiliary_loss_mlp": 0.01012581, + "balance_loss_clip": 1.04836893, + "balance_loss_mlp": 1.00716901, + "epoch": 0.08825828172909277, + "flos": 63424924882560.0, + "grad_norm": 0.9203430423148096, + "language_loss": 0.64616942, + "learning_rate": 3.964568662629706e-06, + "loss": 0.66754222, + "num_input_tokens_seen": 15630570, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.05419922, + "step": 734, + "time_per_iteration": 3.2401766777038574 + }, + { + "auxiliary_loss_clip": 0.01240998, + "auxiliary_loss_mlp": 0.01070231, + "balance_loss_clip": 1.0702827, + "balance_loss_mlp": 1.04119122, + "epoch": 0.08837852461973186, + "flos": 26720986268160.0, + "grad_norm": 2.5005275883679086, + "language_loss": 0.84247792, + "learning_rate": 3.9644225369693895e-06, + "loss": 0.86559021, + "num_input_tokens_seen": 15650870, + "router_z_loss_clip": 1.70703125, + "router_z_loss_mlp": 0.2902832, + "step": 735, + "time_per_iteration": 2.8638808727264404 + }, + { + "auxiliary_loss_clip": 0.01240465, + "auxiliary_loss_mlp": 0.01076879, + "balance_loss_clip": 1.07387829, + "balance_loss_mlp": 1.04874539, + "epoch": 0.08849876751037095, + "flos": 27265427688960.0, + "grad_norm": 2.567623791951311, + "language_loss": 0.86748481, + "learning_rate": 3.964276113307735e-06, + "loss": 0.89065826, + "num_input_tokens_seen": 15670835, + "router_z_loss_clip": 1.66503906, + "router_z_loss_mlp": 0.28137207, + "step": 736, + "time_per_iteration": 2.9111576080322266 + }, + { + "auxiliary_loss_clip": 0.01246884, + "auxiliary_loss_mlp": 0.01071053, + "balance_loss_clip": 1.07506478, + "balance_loss_mlp": 1.04127502, + "epoch": 0.08861901040101004, + "flos": 19828759587840.0, + "grad_norm": 1.841045913908637, + "language_loss": 0.80860901, + "learning_rate": 3.9641293916669574e-06, + "loss": 0.83178836, + "num_input_tokens_seen": 15689795, + "router_z_loss_clip": 1.71972656, + "router_z_loss_mlp": 0.29785156, + "step": 737, + "time_per_iteration": 2.825613260269165 + }, + { + "auxiliary_loss_clip": 0.01243376, + "auxiliary_loss_mlp": 0.01072813, + "balance_loss_clip": 1.07262707, + "balance_loss_mlp": 1.04441702, + "epoch": 0.08873925329164913, + "flos": 23658704173440.0, + "grad_norm": 1.7102656743442488, + "language_loss": 0.8263433, + "learning_rate": 3.9639823720693115e-06, + "loss": 0.84950519, + "num_input_tokens_seen": 15711650, + "router_z_loss_clip": 1.70703125, + "router_z_loss_mlp": 0.28417969, + "step": 738, + "time_per_iteration": 2.8378448486328125 + }, + { + "auxiliary_loss_clip": 0.01122173, + "auxiliary_loss_mlp": 0.01017967, + "balance_loss_clip": 1.04585481, + "balance_loss_mlp": 1.01253128, + "epoch": 0.08885949618228822, + "flos": 71831541893760.0, + "grad_norm": 0.8695904285159075, + "language_loss": 0.60012949, + "learning_rate": 3.963835054537102e-06, + "loss": 0.62153089, + "num_input_tokens_seen": 15780615, + "router_z_loss_clip": 0.76171875, + "router_z_loss_mlp": 0.05444336, + "step": 739, + "time_per_iteration": 3.4096925258636475 + }, + { + "auxiliary_loss_clip": 0.01242736, + "auxiliary_loss_mlp": 0.01063064, + "balance_loss_clip": 1.07618022, + "balance_loss_mlp": 1.03547955, + "epoch": 0.08897973907292732, + "flos": 22346169298560.0, + "grad_norm": 2.033884508221597, + "language_loss": 0.61180651, + "learning_rate": 3.963687439092676e-06, + "loss": 0.63486445, + "num_input_tokens_seen": 15801300, + "router_z_loss_clip": 1.66308594, + "router_z_loss_mlp": 0.27587891, + "step": 740, + "time_per_iteration": 4.544021844863892 + }, + { + "auxiliary_loss_clip": 0.0124059, + "auxiliary_loss_mlp": 0.01066159, + "balance_loss_clip": 1.07227385, + "balance_loss_mlp": 1.03782296, + "epoch": 0.0890999819635664, + "flos": 21251827589760.0, + "grad_norm": 2.2631534250234866, + "language_loss": 0.81011039, + "learning_rate": 3.963539525758427e-06, + "loss": 0.83317792, + "num_input_tokens_seen": 15820860, + "router_z_loss_clip": 1.68261719, + "router_z_loss_mlp": 0.28356934, + "step": 741, + "time_per_iteration": 3.638625383377075 + }, + { + "auxiliary_loss_clip": 0.01243769, + "auxiliary_loss_mlp": 0.01062432, + "balance_loss_clip": 1.07512283, + "balance_loss_mlp": 1.03584814, + "epoch": 0.0892202248542055, + "flos": 25370888745600.0, + "grad_norm": 1.8620125862339392, + "language_loss": 0.67269111, + "learning_rate": 3.9633913145567925e-06, + "loss": 0.6957531, + "num_input_tokens_seen": 15841350, + "router_z_loss_clip": 1.6875, + "router_z_loss_mlp": 0.26611328, + "step": 742, + "time_per_iteration": 2.8239145278930664 + }, + { + "auxiliary_loss_clip": 0.01240553, + "auxiliary_loss_mlp": 0.0106284, + "balance_loss_clip": 1.07511353, + "balance_loss_mlp": 1.03589869, + "epoch": 0.08934046774484458, + "flos": 24457895827200.0, + "grad_norm": 1.8235085281243986, + "language_loss": 0.8153429, + "learning_rate": 3.9632428055102575e-06, + "loss": 0.83837682, + "num_input_tokens_seen": 15861360, + "router_z_loss_clip": 1.65332031, + "router_z_loss_mlp": 0.26940918, + "step": 743, + "time_per_iteration": 2.8967437744140625 + }, + { + "auxiliary_loss_clip": 0.01237952, + "auxiliary_loss_mlp": 0.01066511, + "balance_loss_clip": 1.0708704, + "balance_loss_mlp": 1.03788853, + "epoch": 0.08946071063548368, + "flos": 35772773414400.0, + "grad_norm": 2.2839343507772147, + "language_loss": 0.66353887, + "learning_rate": 3.9630939986413495e-06, + "loss": 0.68658352, + "num_input_tokens_seen": 15883160, + "router_z_loss_clip": 1.66992188, + "router_z_loss_mlp": 0.28625488, + "step": 744, + "time_per_iteration": 2.9551029205322266 + }, + { + "auxiliary_loss_clip": 0.01239238, + "auxiliary_loss_mlp": 0.01072951, + "balance_loss_clip": 1.07188439, + "balance_loss_mlp": 1.04755926, + "epoch": 0.08958095352612276, + "flos": 14356584167040.0, + "grad_norm": 1.7311288797402027, + "language_loss": 0.78185254, + "learning_rate": 3.962944893972643e-06, + "loss": 0.80497444, + "num_input_tokens_seen": 15901610, + "router_z_loss_clip": 1.67285156, + "router_z_loss_mlp": 0.25415039, + "step": 745, + "time_per_iteration": 2.834460496902466 + }, + { + "auxiliary_loss_clip": 0.01236588, + "auxiliary_loss_mlp": 0.01059083, + "balance_loss_clip": 1.06958032, + "balance_loss_mlp": 1.03149772, + "epoch": 0.08970119641676186, + "flos": 17853277345920.0, + "grad_norm": 2.487338062744293, + "language_loss": 0.90691447, + "learning_rate": 3.962795491526756e-06, + "loss": 0.9298712, + "num_input_tokens_seen": 15918770, + "router_z_loss_clip": 1.67089844, + "router_z_loss_mlp": 0.27575684, + "step": 746, + "time_per_iteration": 2.8548073768615723 + }, + { + "auxiliary_loss_clip": 0.01249731, + "auxiliary_loss_mlp": 0.01069813, + "balance_loss_clip": 1.07496786, + "balance_loss_mlp": 1.04208493, + "epoch": 0.08982143930740095, + "flos": 20811670329600.0, + "grad_norm": 2.5169334007827713, + "language_loss": 0.89473093, + "learning_rate": 3.962645791326354e-06, + "loss": 0.91792643, + "num_input_tokens_seen": 15938025, + "router_z_loss_clip": 1.74902344, + "router_z_loss_mlp": 0.27746582, + "step": 747, + "time_per_iteration": 2.8316690921783447 + }, + { + "auxiliary_loss_clip": 0.01234558, + "auxiliary_loss_mlp": 0.01059566, + "balance_loss_clip": 1.07172227, + "balance_loss_mlp": 1.03369725, + "epoch": 0.08994168219804004, + "flos": 24097712198400.0, + "grad_norm": 1.8237973568814458, + "language_loss": 0.83280104, + "learning_rate": 3.962495793394146e-06, + "loss": 0.85574228, + "num_input_tokens_seen": 15957215, + "router_z_loss_clip": 1.62792969, + "router_z_loss_mlp": 0.25854492, + "step": 748, + "time_per_iteration": 2.904447078704834 + }, + { + "auxiliary_loss_clip": 0.01115133, + "auxiliary_loss_mlp": 0.0101715, + "balance_loss_clip": 1.04102921, + "balance_loss_mlp": 1.01207161, + "epoch": 0.09006192508867913, + "flos": 57188893812480.0, + "grad_norm": 0.744133967297468, + "language_loss": 0.61225605, + "learning_rate": 3.9623454977528864e-06, + "loss": 0.6335789, + "num_input_tokens_seen": 16015870, + "router_z_loss_clip": 0.7421875, + "router_z_loss_mlp": 0.05078125, + "step": 749, + "time_per_iteration": 3.206287145614624 + }, + { + "auxiliary_loss_clip": 0.01239581, + "auxiliary_loss_mlp": 0.01067031, + "balance_loss_clip": 1.07214117, + "balance_loss_mlp": 1.03932726, + "epoch": 0.09018216797931822, + "flos": 20487505063680.0, + "grad_norm": 2.6252082173146287, + "language_loss": 0.84834361, + "learning_rate": 3.962194904425375e-06, + "loss": 0.87140977, + "num_input_tokens_seen": 16036500, + "router_z_loss_clip": 1.67285156, + "router_z_loss_mlp": 0.27722168, + "step": 750, + "time_per_iteration": 2.921603202819824 + }, + { + "auxiliary_loss_clip": 0.0124059, + "auxiliary_loss_mlp": 0.0106674, + "balance_loss_clip": 1.07340884, + "balance_loss_mlp": 1.03954864, + "epoch": 0.09030241086995731, + "flos": 22638123043200.0, + "grad_norm": 2.5435562138713874, + "language_loss": 0.67695898, + "learning_rate": 3.9620440134344566e-06, + "loss": 0.70003229, + "num_input_tokens_seen": 16054655, + "router_z_loss_clip": 1.671875, + "router_z_loss_mlp": 0.27209473, + "step": 751, + "time_per_iteration": 2.9057228565216064 + }, + { + "auxiliary_loss_clip": 0.01242744, + "auxiliary_loss_mlp": 0.01063343, + "balance_loss_clip": 1.07205439, + "balance_loss_mlp": 1.0355078, + "epoch": 0.09042265376059641, + "flos": 21871502046720.0, + "grad_norm": 2.5900117311178676, + "language_loss": 0.82115161, + "learning_rate": 3.9618928248030215e-06, + "loss": 0.84421253, + "num_input_tokens_seen": 16074165, + "router_z_loss_clip": 1.70898438, + "router_z_loss_mlp": 0.27856445, + "step": 752, + "time_per_iteration": 2.8620784282684326 + }, + { + "auxiliary_loss_clip": 0.012395, + "auxiliary_loss_mlp": 0.01070011, + "balance_loss_clip": 1.07277679, + "balance_loss_mlp": 1.04094779, + "epoch": 0.0905428966512355, + "flos": 24316192673280.0, + "grad_norm": 2.0245554373588375, + "language_loss": 0.825104, + "learning_rate": 3.961741338554005e-06, + "loss": 0.84819913, + "num_input_tokens_seen": 16092505, + "router_z_loss_clip": 1.66699219, + "router_z_loss_mlp": 0.29040527, + "step": 753, + "time_per_iteration": 2.782762050628662 + }, + { + "auxiliary_loss_clip": 0.01243769, + "auxiliary_loss_mlp": 0.01066117, + "balance_loss_clip": 1.07213652, + "balance_loss_mlp": 1.03849578, + "epoch": 0.09066313954187459, + "flos": 35845061535360.0, + "grad_norm": 2.0537405687047756, + "language_loss": 0.75976741, + "learning_rate": 3.9615895547103865e-06, + "loss": 0.7828663, + "num_input_tokens_seen": 16116150, + "router_z_loss_clip": 1.71386719, + "router_z_loss_mlp": 0.27624512, + "step": 754, + "time_per_iteration": 2.9138336181640625 + }, + { + "auxiliary_loss_clip": 0.01234536, + "auxiliary_loss_mlp": 0.01063965, + "balance_loss_clip": 1.07067323, + "balance_loss_mlp": 1.03721476, + "epoch": 0.09078338243251367, + "flos": 29168729550720.0, + "grad_norm": 2.1533780029843843, + "language_loss": 0.77561635, + "learning_rate": 3.961437473295193e-06, + "loss": 0.79860139, + "num_input_tokens_seen": 16136295, + "router_z_loss_clip": 1.63867188, + "router_z_loss_mlp": 0.26757812, + "step": 755, + "time_per_iteration": 2.845536708831787 + }, + { + "auxiliary_loss_clip": 0.01230457, + "auxiliary_loss_mlp": 0.0106122, + "balance_loss_clip": 1.06894565, + "balance_loss_mlp": 1.03513718, + "epoch": 0.09090362532315277, + "flos": 21907699977600.0, + "grad_norm": 2.5134081876291963, + "language_loss": 0.72006232, + "learning_rate": 3.961285094331495e-06, + "loss": 0.74297905, + "num_input_tokens_seen": 16154210, + "router_z_loss_clip": 1.61523438, + "router_z_loss_mlp": 0.26098633, + "step": 756, + "time_per_iteration": 2.851923704147339 + }, + { + "auxiliary_loss_clip": 0.01236798, + "auxiliary_loss_mlp": 0.01059208, + "balance_loss_clip": 1.070912, + "balance_loss_mlp": 1.03342342, + "epoch": 0.09102386821379185, + "flos": 27344503480320.0, + "grad_norm": 1.8828971485284862, + "language_loss": 0.85892189, + "learning_rate": 3.961132417842406e-06, + "loss": 0.88188195, + "num_input_tokens_seen": 16173995, + "router_z_loss_clip": 1.65917969, + "router_z_loss_mlp": 0.2578125, + "step": 757, + "time_per_iteration": 2.825104236602783 + }, + { + "auxiliary_loss_clip": 0.01238302, + "auxiliary_loss_mlp": 0.01066525, + "balance_loss_clip": 1.07300019, + "balance_loss_mlp": 1.04050219, + "epoch": 0.09114411110443095, + "flos": 20813501923200.0, + "grad_norm": 3.1795911406121804, + "language_loss": 0.74836797, + "learning_rate": 3.960979443851089e-06, + "loss": 0.77141625, + "num_input_tokens_seen": 16191020, + "router_z_loss_clip": 1.65429688, + "router_z_loss_mlp": 0.26049805, + "step": 758, + "time_per_iteration": 2.8537819385528564 + }, + { + "auxiliary_loss_clip": 0.01239542, + "auxiliary_loss_mlp": 0.01073532, + "balance_loss_clip": 1.07243693, + "balance_loss_mlp": 1.04563713, + "epoch": 0.09126435399507005, + "flos": 26145949438080.0, + "grad_norm": 1.655677124180333, + "language_loss": 0.78680581, + "learning_rate": 3.96082617238075e-06, + "loss": 0.80993652, + "num_input_tokens_seen": 16213645, + "router_z_loss_clip": 1.66992188, + "router_z_loss_mlp": 0.27905273, + "step": 759, + "time_per_iteration": 2.8188679218292236 + }, + { + "auxiliary_loss_clip": 0.01231419, + "auxiliary_loss_mlp": 0.01065939, + "balance_loss_clip": 1.07103169, + "balance_loss_mlp": 1.03956985, + "epoch": 0.09138459688570913, + "flos": 24388911757440.0, + "grad_norm": 2.194701392259253, + "language_loss": 0.79896069, + "learning_rate": 3.960672603454639e-06, + "loss": 0.82193422, + "num_input_tokens_seen": 16233625, + "router_z_loss_clip": 1.60351562, + "router_z_loss_mlp": 0.26416016, + "step": 760, + "time_per_iteration": 2.816995143890381 + }, + { + "auxiliary_loss_clip": 0.0123975, + "auxiliary_loss_mlp": 0.01061437, + "balance_loss_clip": 1.06951427, + "balance_loss_mlp": 1.03348207, + "epoch": 0.09150483977634823, + "flos": 21032664756480.0, + "grad_norm": 3.3537633860658, + "language_loss": 0.77410328, + "learning_rate": 3.960518737096054e-06, + "loss": 0.79711515, + "num_input_tokens_seen": 16253255, + "router_z_loss_clip": 1.703125, + "router_z_loss_mlp": 0.27954102, + "step": 761, + "time_per_iteration": 2.928011655807495 + }, + { + "auxiliary_loss_clip": 0.01236005, + "auxiliary_loss_mlp": 0.01068474, + "balance_loss_clip": 1.06977868, + "balance_loss_mlp": 1.04295111, + "epoch": 0.09162508266698731, + "flos": 22856998567680.0, + "grad_norm": 2.745009442559782, + "language_loss": 0.73135614, + "learning_rate": 3.960364573328334e-06, + "loss": 0.75440097, + "num_input_tokens_seen": 16272580, + "router_z_loss_clip": 1.66308594, + "router_z_loss_mlp": 0.25549316, + "step": 762, + "time_per_iteration": 2.9507341384887695 + }, + { + "auxiliary_loss_clip": 0.01242088, + "auxiliary_loss_mlp": 0.01054735, + "balance_loss_clip": 1.07214475, + "balance_loss_mlp": 1.0274483, + "epoch": 0.0917453255576264, + "flos": 21724411852800.0, + "grad_norm": 6.10295976439343, + "language_loss": 0.88812125, + "learning_rate": 3.9602101121748675e-06, + "loss": 0.91108942, + "num_input_tokens_seen": 16293075, + "router_z_loss_clip": 1.69921875, + "router_z_loss_mlp": 0.27307129, + "step": 763, + "time_per_iteration": 2.934645652770996 + }, + { + "auxiliary_loss_clip": 0.01237025, + "auxiliary_loss_mlp": 0.01060657, + "balance_loss_clip": 1.07166255, + "balance_loss_mlp": 1.03290486, + "epoch": 0.0918655684482655, + "flos": 14609215497600.0, + "grad_norm": 2.069874924687905, + "language_loss": 0.7238313, + "learning_rate": 3.960055353659085e-06, + "loss": 0.74680817, + "num_input_tokens_seen": 16310185, + "router_z_loss_clip": 1.65527344, + "router_z_loss_mlp": 0.27734375, + "step": 764, + "time_per_iteration": 2.8732283115386963 + }, + { + "auxiliary_loss_clip": 0.01233977, + "auxiliary_loss_mlp": 0.01077064, + "balance_loss_clip": 1.07087779, + "balance_loss_mlp": 1.04935956, + "epoch": 0.09198581133890459, + "flos": 23435016226560.0, + "grad_norm": 2.1469705896260862, + "language_loss": 0.83863044, + "learning_rate": 3.959900297804465e-06, + "loss": 0.86174089, + "num_input_tokens_seen": 16330355, + "router_z_loss_clip": 1.63085938, + "router_z_loss_mlp": 0.27697754, + "step": 765, + "time_per_iteration": 2.942153215408325 + }, + { + "auxiliary_loss_clip": 0.01231724, + "auxiliary_loss_mlp": 0.01054379, + "balance_loss_clip": 1.07168078, + "balance_loss_mlp": 1.02948761, + "epoch": 0.09210605422954368, + "flos": 16795887753600.0, + "grad_norm": 9.989483239290022, + "language_loss": 0.77407885, + "learning_rate": 3.9597449446345276e-06, + "loss": 0.79693985, + "num_input_tokens_seen": 16347600, + "router_z_loss_clip": 1.59960938, + "router_z_loss_mlp": 0.24890137, + "step": 766, + "time_per_iteration": 2.8764939308166504 + }, + { + "auxiliary_loss_clip": 0.01236697, + "auxiliary_loss_mlp": 0.01066588, + "balance_loss_clip": 1.07310402, + "balance_loss_mlp": 1.04114926, + "epoch": 0.09222629712018277, + "flos": 22674249146880.0, + "grad_norm": 3.628955152465072, + "language_loss": 0.83374506, + "learning_rate": 3.95958929417284e-06, + "loss": 0.85677791, + "num_input_tokens_seen": 16365755, + "router_z_loss_clip": 1.63769531, + "router_z_loss_mlp": 0.25439453, + "step": 767, + "time_per_iteration": 2.9468891620635986 + }, + { + "auxiliary_loss_clip": 0.01119277, + "auxiliary_loss_mlp": 0.01010405, + "balance_loss_clip": 1.04646528, + "balance_loss_mlp": 1.00506485, + "epoch": 0.09234654001082186, + "flos": 69976756327680.0, + "grad_norm": 0.7415470043678712, + "language_loss": 0.58754385, + "learning_rate": 3.9594333464430145e-06, + "loss": 0.6088407, + "num_input_tokens_seen": 16435245, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.0534668, + "step": 768, + "time_per_iteration": 4.326439619064331 + }, + { + "auxiliary_loss_clip": 0.01237053, + "auxiliary_loss_mlp": 0.01062978, + "balance_loss_clip": 1.07349586, + "balance_loss_mlp": 1.03797984, + "epoch": 0.09246678290146094, + "flos": 20011437181440.0, + "grad_norm": 1.9797475768811412, + "language_loss": 0.8799566, + "learning_rate": 3.959277101468709e-06, + "loss": 0.90295696, + "num_input_tokens_seen": 16454795, + "router_z_loss_clip": 1.63671875, + "router_z_loss_mlp": 0.25024414, + "step": 769, + "time_per_iteration": 4.505715847015381 + }, + { + "auxiliary_loss_clip": 0.01232986, + "auxiliary_loss_mlp": 0.01064662, + "balance_loss_clip": 1.07174945, + "balance_loss_mlp": 1.03842437, + "epoch": 0.09258702579210004, + "flos": 17747448900480.0, + "grad_norm": 2.648959661479479, + "language_loss": 0.7883901, + "learning_rate": 3.959120559273624e-06, + "loss": 0.81136662, + "num_input_tokens_seen": 16472580, + "router_z_loss_clip": 1.61035156, + "router_z_loss_mlp": 0.2623291, + "step": 770, + "time_per_iteration": 3.657043218612671 + }, + { + "auxiliary_loss_clip": 0.01239057, + "auxiliary_loss_mlp": 0.010671, + "balance_loss_clip": 1.07374752, + "balance_loss_mlp": 1.04095733, + "epoch": 0.09270726868273914, + "flos": 20886544229760.0, + "grad_norm": 1.981308138761431, + "language_loss": 0.83134794, + "learning_rate": 3.958963719881509e-06, + "loss": 0.85440952, + "num_input_tokens_seen": 16490670, + "router_z_loss_clip": 1.65332031, + "router_z_loss_mlp": 0.26159668, + "step": 771, + "time_per_iteration": 2.8545494079589844 + }, + { + "auxiliary_loss_clip": 0.01241793, + "auxiliary_loss_mlp": 0.01068224, + "balance_loss_clip": 1.07802081, + "balance_loss_mlp": 1.04332149, + "epoch": 0.09282751157337822, + "flos": 17015697031680.0, + "grad_norm": 2.803644450786704, + "language_loss": 0.93862128, + "learning_rate": 3.958806583316154e-06, + "loss": 0.96172142, + "num_input_tokens_seen": 16508640, + "router_z_loss_clip": 1.63769531, + "router_z_loss_mlp": 0.24926758, + "step": 772, + "time_per_iteration": 2.8721611499786377 + }, + { + "auxiliary_loss_clip": 0.01243279, + "auxiliary_loss_mlp": 0.01065075, + "balance_loss_clip": 1.07670307, + "balance_loss_mlp": 1.03752613, + "epoch": 0.09294775446401732, + "flos": 32523647748480.0, + "grad_norm": 2.250459832297949, + "language_loss": 0.78721964, + "learning_rate": 3.9586491496013985e-06, + "loss": 0.81030321, + "num_input_tokens_seen": 16531035, + "router_z_loss_clip": 1.66503906, + "router_z_loss_mlp": 0.27575684, + "step": 773, + "time_per_iteration": 2.9472811222076416 + }, + { + "auxiliary_loss_clip": 0.01244253, + "auxiliary_loss_mlp": 0.01072015, + "balance_loss_clip": 1.07567668, + "balance_loss_mlp": 1.04600346, + "epoch": 0.0930679973546564, + "flos": 18259750627200.0, + "grad_norm": 2.4159742622536795, + "language_loss": 0.82900369, + "learning_rate": 3.958491418761124e-06, + "loss": 0.85216641, + "num_input_tokens_seen": 16548605, + "router_z_loss_clip": 1.68457031, + "router_z_loss_mlp": 0.26049805, + "step": 774, + "time_per_iteration": 2.847702980041504 + }, + { + "auxiliary_loss_clip": 0.01239743, + "auxiliary_loss_mlp": 0.01067509, + "balance_loss_clip": 1.07278073, + "balance_loss_mlp": 1.04103231, + "epoch": 0.0931882402452955, + "flos": 21099745405440.0, + "grad_norm": 2.122015959664602, + "language_loss": 0.7261343, + "learning_rate": 3.958333390819258e-06, + "loss": 0.74920678, + "num_input_tokens_seen": 16565535, + "router_z_loss_clip": 1.66699219, + "router_z_loss_mlp": 0.26452637, + "step": 775, + "time_per_iteration": 2.8609561920166016 + }, + { + "auxiliary_loss_clip": 0.01237427, + "auxiliary_loss_mlp": 0.01082129, + "balance_loss_clip": 1.07298398, + "balance_loss_mlp": 1.05473506, + "epoch": 0.0933084831359346, + "flos": 24207275658240.0, + "grad_norm": 2.1288927419391452, + "language_loss": 0.80135393, + "learning_rate": 3.9581750657997754e-06, + "loss": 0.8245495, + "num_input_tokens_seen": 16584900, + "router_z_loss_clip": 1.64550781, + "router_z_loss_mlp": 0.27368164, + "step": 776, + "time_per_iteration": 2.859048843383789 + }, + { + "auxiliary_loss_clip": 0.01234096, + "auxiliary_loss_mlp": 0.01057875, + "balance_loss_clip": 1.07165706, + "balance_loss_mlp": 1.03229308, + "epoch": 0.09342872602657368, + "flos": 25480272637440.0, + "grad_norm": 1.7494085868157114, + "language_loss": 0.89436519, + "learning_rate": 3.95801644372669e-06, + "loss": 0.91728491, + "num_input_tokens_seen": 16604805, + "router_z_loss_clip": 1.62402344, + "router_z_loss_mlp": 0.25598145, + "step": 777, + "time_per_iteration": 2.8309519290924072 + }, + { + "auxiliary_loss_clip": 0.01239066, + "auxiliary_loss_mlp": 0.01059767, + "balance_loss_clip": 1.07354152, + "balance_loss_mlp": 1.03506672, + "epoch": 0.09354896891721277, + "flos": 23149060053120.0, + "grad_norm": 2.0232753942053665, + "language_loss": 0.84374309, + "learning_rate": 3.957857524624068e-06, + "loss": 0.86673141, + "num_input_tokens_seen": 16623685, + "router_z_loss_clip": 1.65722656, + "router_z_loss_mlp": 0.24731445, + "step": 778, + "time_per_iteration": 2.811640739440918 + }, + { + "auxiliary_loss_clip": 0.01232478, + "auxiliary_loss_mlp": 0.01072875, + "balance_loss_clip": 1.06973672, + "balance_loss_mlp": 1.04624391, + "epoch": 0.09366921180785186, + "flos": 24279563779200.0, + "grad_norm": 1.6409544099667461, + "language_loss": 0.89333916, + "learning_rate": 3.957698308516016e-06, + "loss": 0.91639268, + "num_input_tokens_seen": 16644985, + "router_z_loss_clip": 1.62695312, + "router_z_loss_mlp": 0.26623535, + "step": 779, + "time_per_iteration": 2.7991700172424316 + }, + { + "auxiliary_loss_clip": 0.01237575, + "auxiliary_loss_mlp": 0.01085626, + "balance_loss_clip": 1.07463944, + "balance_loss_mlp": 1.06186759, + "epoch": 0.09378945469849095, + "flos": 18730036419840.0, + "grad_norm": 1.962713895377959, + "language_loss": 0.82467538, + "learning_rate": 3.957538795426688e-06, + "loss": 0.84790736, + "num_input_tokens_seen": 16662410, + "router_z_loss_clip": 1.63085938, + "router_z_loss_mlp": 0.23754883, + "step": 780, + "time_per_iteration": 2.7937488555908203 + }, + { + "auxiliary_loss_clip": 0.01237832, + "auxiliary_loss_mlp": 0.01072314, + "balance_loss_clip": 1.07132697, + "balance_loss_mlp": 1.04620719, + "epoch": 0.09390969758913004, + "flos": 23218834222080.0, + "grad_norm": 2.5336305659214995, + "language_loss": 0.77140468, + "learning_rate": 3.9573789853802804e-06, + "loss": 0.79450619, + "num_input_tokens_seen": 16680885, + "router_z_loss_clip": 1.66503906, + "router_z_loss_mlp": 0.26159668, + "step": 781, + "time_per_iteration": 2.7886128425598145 + }, + { + "auxiliary_loss_clip": 0.01234043, + "auxiliary_loss_mlp": 0.01074632, + "balance_loss_clip": 1.06879961, + "balance_loss_mlp": 1.04769111, + "epoch": 0.09402994047976913, + "flos": 19646728439040.0, + "grad_norm": 1.9804834095057267, + "language_loss": 0.74615216, + "learning_rate": 3.957218878401037e-06, + "loss": 0.76923895, + "num_input_tokens_seen": 16699375, + "router_z_loss_clip": 1.65234375, + "router_z_loss_mlp": 0.26928711, + "step": 782, + "time_per_iteration": 2.770129680633545 + }, + { + "auxiliary_loss_clip": 0.01235607, + "auxiliary_loss_mlp": 0.01074478, + "balance_loss_clip": 1.0725503, + "balance_loss_mlp": 1.04748917, + "epoch": 0.09415018337040823, + "flos": 29420463041280.0, + "grad_norm": 1.9369751138282838, + "language_loss": 0.89401227, + "learning_rate": 3.957058474513246e-06, + "loss": 0.91711318, + "num_input_tokens_seen": 16719230, + "router_z_loss_clip": 1.63085938, + "router_z_loss_mlp": 0.26940918, + "step": 783, + "time_per_iteration": 2.8714733123779297 + }, + { + "auxiliary_loss_clip": 0.01234998, + "auxiliary_loss_mlp": 0.01067445, + "balance_loss_clip": 1.07181573, + "balance_loss_mlp": 1.04214847, + "epoch": 0.09427042626104731, + "flos": 24572092141440.0, + "grad_norm": 1.9017903543932502, + "language_loss": 0.78666139, + "learning_rate": 3.956897773741241e-06, + "loss": 0.80968583, + "num_input_tokens_seen": 16738220, + "router_z_loss_clip": 1.63085938, + "router_z_loss_mlp": 0.25317383, + "step": 784, + "time_per_iteration": 2.8721494674682617 + }, + { + "auxiliary_loss_clip": 0.01234075, + "auxiliary_loss_mlp": 0.01053851, + "balance_loss_clip": 1.07005727, + "balance_loss_mlp": 1.02750611, + "epoch": 0.09439066915168641, + "flos": 26359581576960.0, + "grad_norm": 1.9099742943118012, + "language_loss": 0.71768916, + "learning_rate": 3.956736776109398e-06, + "loss": 0.74056846, + "num_input_tokens_seen": 16759395, + "router_z_loss_clip": 1.640625, + "router_z_loss_mlp": 0.2635498, + "step": 785, + "time_per_iteration": 2.8608791828155518 + }, + { + "auxiliary_loss_clip": 0.01230413, + "auxiliary_loss_mlp": 0.01055237, + "balance_loss_clip": 1.07013822, + "balance_loss_mlp": 1.03087008, + "epoch": 0.09451091204232549, + "flos": 19427278296960.0, + "grad_norm": 1.8969388433501955, + "language_loss": 0.83788115, + "learning_rate": 3.956575481642143e-06, + "loss": 0.86073768, + "num_input_tokens_seen": 16778285, + "router_z_loss_clip": 1.60449219, + "router_z_loss_mlp": 0.24353027, + "step": 786, + "time_per_iteration": 2.867993116378784 + }, + { + "auxiliary_loss_clip": 0.01237151, + "auxiliary_loss_mlp": 0.01066418, + "balance_loss_clip": 1.07410038, + "balance_loss_mlp": 1.03996575, + "epoch": 0.09463115493296459, + "flos": 25368051571200.0, + "grad_norm": 2.5125902122441204, + "language_loss": 0.74706113, + "learning_rate": 3.956413890363943e-06, + "loss": 0.7700969, + "num_input_tokens_seen": 16795265, + "router_z_loss_clip": 1.62890625, + "router_z_loss_mlp": 0.26452637, + "step": 787, + "time_per_iteration": 2.977175235748291 + }, + { + "auxiliary_loss_clip": 0.01237932, + "auxiliary_loss_mlp": 0.01065307, + "balance_loss_clip": 1.07106996, + "balance_loss_mlp": 1.03968906, + "epoch": 0.09475139782360369, + "flos": 10123254869760.0, + "grad_norm": 2.184408880404461, + "language_loss": 0.81346107, + "learning_rate": 3.956252002299312e-06, + "loss": 0.83649349, + "num_input_tokens_seen": 16811165, + "router_z_loss_clip": 1.66992188, + "router_z_loss_mlp": 0.25610352, + "step": 788, + "time_per_iteration": 2.807070732116699 + }, + { + "auxiliary_loss_clip": 0.01234842, + "auxiliary_loss_mlp": 0.01060988, + "balance_loss_clip": 1.06940317, + "balance_loss_mlp": 1.03283095, + "epoch": 0.09487164071424277, + "flos": 17231088936960.0, + "grad_norm": 2.3117210771827503, + "language_loss": 0.90779078, + "learning_rate": 3.956089817472807e-06, + "loss": 0.93074912, + "num_input_tokens_seen": 16828470, + "router_z_loss_clip": 1.65625, + "router_z_loss_mlp": 0.28137207, + "step": 789, + "time_per_iteration": 2.831697940826416 + }, + { + "auxiliary_loss_clip": 0.01237279, + "auxiliary_loss_mlp": 0.01058057, + "balance_loss_clip": 1.07291937, + "balance_loss_mlp": 1.03297544, + "epoch": 0.09499188360488187, + "flos": 30849564528000.0, + "grad_norm": 2.7336913074949027, + "language_loss": 0.85186183, + "learning_rate": 3.955927335909032e-06, + "loss": 0.87481523, + "num_input_tokens_seen": 16851680, + "router_z_loss_clip": 1.64355469, + "router_z_loss_mlp": 0.25048828, + "step": 790, + "time_per_iteration": 2.8742854595184326 + }, + { + "auxiliary_loss_clip": 0.01232593, + "auxiliary_loss_mlp": 0.01064075, + "balance_loss_clip": 1.07272279, + "balance_loss_mlp": 1.03819418, + "epoch": 0.09511212649552095, + "flos": 29351694453120.0, + "grad_norm": 3.0713660537528296, + "language_loss": 0.7596283, + "learning_rate": 3.955764557632634e-06, + "loss": 0.78259498, + "num_input_tokens_seen": 16871490, + "router_z_loss_clip": 1.59765625, + "router_z_loss_mlp": 0.25915527, + "step": 791, + "time_per_iteration": 2.9184131622314453 + }, + { + "auxiliary_loss_clip": 0.01228523, + "auxiliary_loss_mlp": 0.01070648, + "balance_loss_clip": 1.07059288, + "balance_loss_mlp": 1.04412436, + "epoch": 0.09523236938616005, + "flos": 10378687461120.0, + "grad_norm": 2.701539713052265, + "language_loss": 0.94876027, + "learning_rate": 3.955601482668309e-06, + "loss": 0.97175199, + "num_input_tokens_seen": 16889350, + "router_z_loss_clip": 1.57714844, + "router_z_loss_mlp": 0.26501465, + "step": 792, + "time_per_iteration": 2.857133150100708 + }, + { + "auxiliary_loss_clip": 0.01238207, + "auxiliary_loss_mlp": 0.01071383, + "balance_loss_clip": 1.07289791, + "balance_loss_mlp": 1.04264188, + "epoch": 0.09535261227679913, + "flos": 19061815368960.0, + "grad_norm": 3.2663644212227343, + "language_loss": 0.88626802, + "learning_rate": 3.955438111040794e-06, + "loss": 0.90936393, + "num_input_tokens_seen": 16907625, + "router_z_loss_clip": 1.65332031, + "router_z_loss_mlp": 0.28747559, + "step": 793, + "time_per_iteration": 2.8303849697113037 + }, + { + "auxiliary_loss_clip": 0.01234052, + "auxiliary_loss_mlp": 0.01064481, + "balance_loss_clip": 1.07066607, + "balance_loss_mlp": 1.03773093, + "epoch": 0.09547285516743823, + "flos": 20922993555840.0, + "grad_norm": 1.9284238511698748, + "language_loss": 0.80081713, + "learning_rate": 3.955274442774873e-06, + "loss": 0.82380247, + "num_input_tokens_seen": 16926205, + "router_z_loss_clip": 1.63476562, + "router_z_loss_mlp": 0.26721191, + "step": 794, + "time_per_iteration": 2.8070385456085205 + }, + { + "auxiliary_loss_clip": 0.01237226, + "auxiliary_loss_mlp": 0.01057603, + "balance_loss_clip": 1.07387948, + "balance_loss_mlp": 1.03266406, + "epoch": 0.09559309805807732, + "flos": 30154405639680.0, + "grad_norm": 2.3486084146215536, + "language_loss": 0.70516485, + "learning_rate": 3.9551104778953725e-06, + "loss": 0.72811311, + "num_input_tokens_seen": 16946500, + "router_z_loss_clip": 1.63378906, + "router_z_loss_mlp": 0.24926758, + "step": 795, + "time_per_iteration": 2.816829204559326 + }, + { + "auxiliary_loss_clip": 0.01240187, + "auxiliary_loss_mlp": 0.01052828, + "balance_loss_clip": 1.0752157, + "balance_loss_mlp": 1.02790141, + "epoch": 0.0957133409487164, + "flos": 21066743784960.0, + "grad_norm": 1.9133216865980676, + "language_loss": 0.85503525, + "learning_rate": 3.954946216427167e-06, + "loss": 0.87796533, + "num_input_tokens_seen": 16966960, + "router_z_loss_clip": 1.64941406, + "router_z_loss_mlp": 0.24938965, + "step": 796, + "time_per_iteration": 2.794435739517212 + }, + { + "auxiliary_loss_clip": 0.0110971, + "auxiliary_loss_mlp": 0.01018869, + "balance_loss_clip": 1.03626871, + "balance_loss_mlp": 1.0142436, + "epoch": 0.0958335838393555, + "flos": 71297979315840.0, + "grad_norm": 0.8779821053058887, + "language_loss": 0.61561096, + "learning_rate": 3.954781658395176e-06, + "loss": 0.63689673, + "num_input_tokens_seen": 17023215, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.04614258, + "step": 797, + "time_per_iteration": 4.134121417999268 + }, + { + "auxiliary_loss_clip": 0.01237161, + "auxiliary_loss_mlp": 0.0107875, + "balance_loss_clip": 1.07314944, + "balance_loss_mlp": 1.05090249, + "epoch": 0.09595382672999458, + "flos": 21872974504320.0, + "grad_norm": 2.1723026336242213, + "language_loss": 0.92305875, + "learning_rate": 3.95461680382436e-06, + "loss": 0.94621795, + "num_input_tokens_seen": 17042140, + "router_z_loss_clip": 1.63867188, + "router_z_loss_mlp": 0.27832031, + "step": 798, + "time_per_iteration": 3.8332955837249756 + }, + { + "auxiliary_loss_clip": 0.01233325, + "auxiliary_loss_mlp": 0.01065369, + "balance_loss_clip": 1.07072568, + "balance_loss_mlp": 1.03929842, + "epoch": 0.09607406962063368, + "flos": 18695562341760.0, + "grad_norm": 2.9685375537223306, + "language_loss": 0.86219037, + "learning_rate": 3.9544516527397295e-06, + "loss": 0.88517725, + "num_input_tokens_seen": 17058490, + "router_z_loss_clip": 1.62792969, + "router_z_loss_mlp": 0.2611084, + "step": 799, + "time_per_iteration": 3.6448254585266113 + }, + { + "auxiliary_loss_clip": 0.01235654, + "auxiliary_loss_mlp": 0.01066327, + "balance_loss_clip": 1.07236528, + "balance_loss_mlp": 1.03940916, + "epoch": 0.09619431251127276, + "flos": 22568456615040.0, + "grad_norm": 1.9265544697146815, + "language_loss": 0.80872309, + "learning_rate": 3.954286205166338e-06, + "loss": 0.83174288, + "num_input_tokens_seen": 17079655, + "router_z_loss_clip": 1.63378906, + "router_z_loss_mlp": 0.2689209, + "step": 800, + "time_per_iteration": 2.830772638320923 + }, + { + "auxiliary_loss_clip": 0.01243635, + "auxiliary_loss_mlp": 0.01072801, + "balance_loss_clip": 1.07522106, + "balance_loss_mlp": 1.04444122, + "epoch": 0.09631455540191186, + "flos": 14246230608000.0, + "grad_norm": 2.780638674051375, + "language_loss": 0.83880407, + "learning_rate": 3.954120461129282e-06, + "loss": 0.86196846, + "num_input_tokens_seen": 17097065, + "router_z_loss_clip": 1.68261719, + "router_z_loss_mlp": 0.28393555, + "step": 801, + "time_per_iteration": 2.7645580768585205 + }, + { + "auxiliary_loss_clip": 0.01237586, + "auxiliary_loss_mlp": 0.01067297, + "balance_loss_clip": 1.07531703, + "balance_loss_mlp": 1.04062963, + "epoch": 0.09643479829255096, + "flos": 20740387789440.0, + "grad_norm": 1.9701917081865863, + "language_loss": 0.83571494, + "learning_rate": 3.953954420653706e-06, + "loss": 0.85876375, + "num_input_tokens_seen": 17114090, + "router_z_loss_clip": 1.62304688, + "router_z_loss_mlp": 0.2668457, + "step": 802, + "time_per_iteration": 2.777182102203369 + }, + { + "auxiliary_loss_clip": 0.01229149, + "auxiliary_loss_mlp": 0.01058174, + "balance_loss_clip": 1.07152331, + "balance_loss_mlp": 1.0311017, + "epoch": 0.09655504118319004, + "flos": 24420476833920.0, + "grad_norm": 2.149189714551957, + "language_loss": 0.88291472, + "learning_rate": 3.953788083764798e-06, + "loss": 0.90578806, + "num_input_tokens_seen": 17133325, + "router_z_loss_clip": 1.57617188, + "router_z_loss_mlp": 0.27099609, + "step": 803, + "time_per_iteration": 2.8743865489959717 + }, + { + "auxiliary_loss_clip": 0.01233285, + "auxiliary_loss_mlp": 0.01068285, + "balance_loss_clip": 1.07325745, + "balance_loss_mlp": 1.04067659, + "epoch": 0.09667528407382914, + "flos": 18441961344000.0, + "grad_norm": 1.8840663914543905, + "language_loss": 0.92185652, + "learning_rate": 3.953621450487792e-06, + "loss": 0.94487214, + "num_input_tokens_seen": 17151945, + "router_z_loss_clip": 1.59960938, + "router_z_loss_mlp": 0.27600098, + "step": 804, + "time_per_iteration": 2.9641880989074707 + }, + { + "auxiliary_loss_clip": 0.01107629, + "auxiliary_loss_mlp": 0.01006768, + "balance_loss_clip": 1.03441238, + "balance_loss_mlp": 1.00238061, + "epoch": 0.09679552696446822, + "flos": 70816455544320.0, + "grad_norm": 0.8419995601499987, + "language_loss": 0.61226922, + "learning_rate": 3.953454520847964e-06, + "loss": 0.6334132, + "num_input_tokens_seen": 17216790, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.04394531, + "step": 805, + "time_per_iteration": 3.5454800128936768 + }, + { + "auxiliary_loss_clip": 0.01240717, + "auxiliary_loss_mlp": 0.01070834, + "balance_loss_clip": 1.07189536, + "balance_loss_mlp": 1.04408383, + "epoch": 0.09691576985510732, + "flos": 21945514020480.0, + "grad_norm": 2.576696156523815, + "language_loss": 0.73532152, + "learning_rate": 3.9532872948706395e-06, + "loss": 0.75843704, + "num_input_tokens_seen": 17236285, + "router_z_loss_clip": 1.68945312, + "router_z_loss_mlp": 0.26757812, + "step": 806, + "time_per_iteration": 2.8663432598114014 + }, + { + "auxiliary_loss_clip": 0.01233113, + "auxiliary_loss_mlp": 0.01065473, + "balance_loss_clip": 1.07196891, + "balance_loss_mlp": 1.03973556, + "epoch": 0.09703601274574641, + "flos": 17965211103360.0, + "grad_norm": 2.6362340253516736, + "language_loss": 0.82676661, + "learning_rate": 3.9531197725811845e-06, + "loss": 0.84975243, + "num_input_tokens_seen": 17251670, + "router_z_loss_clip": 1.61132812, + "router_z_loss_mlp": 0.25756836, + "step": 807, + "time_per_iteration": 2.8849940299987793 + }, + { + "auxiliary_loss_clip": 0.01230664, + "auxiliary_loss_mlp": 0.0105455, + "balance_loss_clip": 1.07329643, + "balance_loss_mlp": 1.02859783, + "epoch": 0.0971562556363855, + "flos": 22162162901760.0, + "grad_norm": 1.7718189761896488, + "language_loss": 0.88203573, + "learning_rate": 3.952951954005013e-06, + "loss": 0.90488791, + "num_input_tokens_seen": 17271355, + "router_z_loss_clip": 1.57324219, + "router_z_loss_mlp": 0.25964355, + "step": 808, + "time_per_iteration": 2.87640118598938 + }, + { + "auxiliary_loss_clip": 0.01231435, + "auxiliary_loss_mlp": 0.01054637, + "balance_loss_clip": 1.07243454, + "balance_loss_mlp": 1.02971029, + "epoch": 0.0972764985270246, + "flos": 25848716394240.0, + "grad_norm": 1.6776307717233683, + "language_loss": 0.84557438, + "learning_rate": 3.952783839167584e-06, + "loss": 0.86843503, + "num_input_tokens_seen": 17291400, + "router_z_loss_clip": 1.59082031, + "router_z_loss_mlp": 0.24938965, + "step": 809, + "time_per_iteration": 2.9709417819976807 + }, + { + "auxiliary_loss_clip": 0.0123358, + "auxiliary_loss_mlp": 0.01055366, + "balance_loss_clip": 1.06830084, + "balance_loss_mlp": 1.02835345, + "epoch": 0.09739674141766368, + "flos": 20339373375360.0, + "grad_norm": 3.2481247478887036, + "language_loss": 0.74284768, + "learning_rate": 3.952615428094398e-06, + "loss": 0.76573718, + "num_input_tokens_seen": 17310920, + "router_z_loss_clip": 1.65332031, + "router_z_loss_mlp": 0.27026367, + "step": 810, + "time_per_iteration": 2.857330799102783 + }, + { + "auxiliary_loss_clip": 0.01237139, + "auxiliary_loss_mlp": 0.01072534, + "balance_loss_clip": 1.07227647, + "balance_loss_mlp": 1.04381633, + "epoch": 0.09751698430830277, + "flos": 15743059188480.0, + "grad_norm": 1.795026259208035, + "language_loss": 0.73598415, + "learning_rate": 3.952446720811004e-06, + "loss": 0.75908083, + "num_input_tokens_seen": 17329245, + "router_z_loss_clip": 1.6484375, + "router_z_loss_mlp": 0.28710938, + "step": 811, + "time_per_iteration": 2.839057445526123 + }, + { + "auxiliary_loss_clip": 0.01104589, + "auxiliary_loss_mlp": 0.01019183, + "balance_loss_clip": 1.03287792, + "balance_loss_mlp": 1.01467705, + "epoch": 0.09763722719894186, + "flos": 63716806800000.0, + "grad_norm": 0.843197109468753, + "language_loss": 0.63592958, + "learning_rate": 3.952277717342995e-06, + "loss": 0.65716732, + "num_input_tokens_seen": 17395680, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.04516602, + "step": 812, + "time_per_iteration": 3.459240436553955 + }, + { + "auxiliary_loss_clip": 0.01241104, + "auxiliary_loss_mlp": 0.01068715, + "balance_loss_clip": 1.07506466, + "balance_loss_mlp": 1.0400331, + "epoch": 0.09775747008958095, + "flos": 22090916275200.0, + "grad_norm": 2.1989286869790754, + "language_loss": 0.85724127, + "learning_rate": 3.952108417716009e-06, + "loss": 0.8803395, + "num_input_tokens_seen": 17415135, + "router_z_loss_clip": 1.66015625, + "router_z_loss_mlp": 0.28686523, + "step": 813, + "time_per_iteration": 2.7858612537384033 + }, + { + "auxiliary_loss_clip": 0.01239841, + "auxiliary_loss_mlp": 0.01072964, + "balance_loss_clip": 1.075719, + "balance_loss_mlp": 1.04418635, + "epoch": 0.09787771298022005, + "flos": 21286050272640.0, + "grad_norm": 1.9470620638333025, + "language_loss": 0.84611452, + "learning_rate": 3.951938821955727e-06, + "loss": 0.86924255, + "num_input_tokens_seen": 17434535, + "router_z_loss_clip": 1.64160156, + "router_z_loss_mlp": 0.28747559, + "step": 814, + "time_per_iteration": 2.830993890762329 + }, + { + "auxiliary_loss_clip": 0.01233351, + "auxiliary_loss_mlp": 0.01065084, + "balance_loss_clip": 1.07222462, + "balance_loss_mlp": 1.03706956, + "epoch": 0.09799795587085913, + "flos": 22054574689920.0, + "grad_norm": 1.733301175248864, + "language_loss": 0.76448441, + "learning_rate": 3.9517689300878786e-06, + "loss": 0.78746873, + "num_input_tokens_seen": 17454270, + "router_z_loss_clip": 1.61425781, + "router_z_loss_mlp": 0.27978516, + "step": 815, + "time_per_iteration": 2.8048973083496094 + }, + { + "auxiliary_loss_clip": 0.01234766, + "auxiliary_loss_mlp": 0.01069928, + "balance_loss_clip": 1.07182205, + "balance_loss_mlp": 1.04258108, + "epoch": 0.09811819876149823, + "flos": 22163743100160.0, + "grad_norm": 1.8115480072824348, + "language_loss": 0.78613734, + "learning_rate": 3.951598742138236e-06, + "loss": 0.80918431, + "num_input_tokens_seen": 17472995, + "router_z_loss_clip": 1.62890625, + "router_z_loss_mlp": 0.2734375, + "step": 816, + "time_per_iteration": 2.7860970497131348 + }, + { + "auxiliary_loss_clip": 0.01233632, + "auxiliary_loss_mlp": 0.01059724, + "balance_loss_clip": 1.07098913, + "balance_loss_mlp": 1.03180575, + "epoch": 0.09823844165213731, + "flos": 22231111057920.0, + "grad_norm": 2.1407893755465377, + "language_loss": 0.7997297, + "learning_rate": 3.951428258132615e-06, + "loss": 0.82266325, + "num_input_tokens_seen": 17491115, + "router_z_loss_clip": 1.62695312, + "router_z_loss_mlp": 0.27978516, + "step": 817, + "time_per_iteration": 2.7821648120880127 + }, + { + "auxiliary_loss_clip": 0.01236481, + "auxiliary_loss_mlp": 0.01061438, + "balance_loss_clip": 1.07027102, + "balance_loss_mlp": 1.03498578, + "epoch": 0.09835868454277641, + "flos": 22487728798080.0, + "grad_norm": 1.9057432972049015, + "language_loss": 0.84674168, + "learning_rate": 3.951257478096879e-06, + "loss": 0.86972094, + "num_input_tokens_seen": 17509480, + "router_z_loss_clip": 1.66113281, + "router_z_loss_mlp": 0.26464844, + "step": 818, + "time_per_iteration": 2.828969955444336 + }, + { + "auxiliary_loss_clip": 0.01234841, + "auxiliary_loss_mlp": 0.01062077, + "balance_loss_clip": 1.07205582, + "balance_loss_mlp": 1.03514755, + "epoch": 0.0984789274334155, + "flos": 16362554077440.0, + "grad_norm": 3.560916794828956, + "language_loss": 0.68176329, + "learning_rate": 3.951086402056936e-06, + "loss": 0.70473254, + "num_input_tokens_seen": 17524080, + "router_z_loss_clip": 1.62792969, + "router_z_loss_mlp": 0.26928711, + "step": 819, + "time_per_iteration": 2.755676031112671 + }, + { + "auxiliary_loss_clip": 0.01236113, + "auxiliary_loss_mlp": 0.01076726, + "balance_loss_clip": 1.07506871, + "balance_loss_mlp": 1.04890275, + "epoch": 0.09859917032405459, + "flos": 24243545416320.0, + "grad_norm": 1.6403527222567635, + "language_loss": 0.83413923, + "learning_rate": 3.950915030038735e-06, + "loss": 0.85726762, + "num_input_tokens_seen": 17543875, + "router_z_loss_clip": 1.61035156, + "router_z_loss_mlp": 0.27832031, + "step": 820, + "time_per_iteration": 2.8649139404296875 + }, + { + "auxiliary_loss_clip": 0.01231101, + "auxiliary_loss_mlp": 0.01053581, + "balance_loss_clip": 1.07247877, + "balance_loss_mlp": 1.02816594, + "epoch": 0.09871941321469369, + "flos": 17420195064960.0, + "grad_norm": 2.272004737408264, + "language_loss": 0.83675969, + "learning_rate": 3.9507433620682765e-06, + "loss": 0.8596065, + "num_input_tokens_seen": 17560810, + "router_z_loss_clip": 1.58691406, + "router_z_loss_mlp": 0.25427246, + "step": 821, + "time_per_iteration": 2.8482003211975098 + }, + { + "auxiliary_loss_clip": 0.01232296, + "auxiliary_loss_mlp": 0.01076498, + "balance_loss_clip": 1.06842017, + "balance_loss_mlp": 1.04824543, + "epoch": 0.09883965610533277, + "flos": 28477341590400.0, + "grad_norm": 1.6131263779090064, + "language_loss": 0.88284445, + "learning_rate": 3.9505713981716e-06, + "loss": 0.90593231, + "num_input_tokens_seen": 17583640, + "router_z_loss_clip": 1.63671875, + "router_z_loss_mlp": 0.28234863, + "step": 822, + "time_per_iteration": 2.8813207149505615 + }, + { + "auxiliary_loss_clip": 0.01232121, + "auxiliary_loss_mlp": 0.0106378, + "balance_loss_clip": 1.07495582, + "balance_loss_mlp": 1.03824568, + "epoch": 0.09895989899597187, + "flos": 23693932437120.0, + "grad_norm": 1.943422659414967, + "language_loss": 0.81322277, + "learning_rate": 3.950399138374795e-06, + "loss": 0.83618176, + "num_input_tokens_seen": 17602720, + "router_z_loss_clip": 1.56933594, + "router_z_loss_mlp": 0.25524902, + "step": 823, + "time_per_iteration": 2.848886251449585 + }, + { + "auxiliary_loss_clip": 0.01232365, + "auxiliary_loss_mlp": 0.01064779, + "balance_loss_clip": 1.07105243, + "balance_loss_mlp": 1.03767085, + "epoch": 0.09908014188661095, + "flos": 24679608526080.0, + "grad_norm": 1.7516949202412806, + "language_loss": 0.74542987, + "learning_rate": 3.95022658270399e-06, + "loss": 0.76840127, + "num_input_tokens_seen": 17623085, + "router_z_loss_clip": 1.61230469, + "router_z_loss_mlp": 0.27124023, + "step": 824, + "time_per_iteration": 2.8759074211120605 + }, + { + "auxiliary_loss_clip": 0.01232748, + "auxiliary_loss_mlp": 0.01064217, + "balance_loss_clip": 1.07185936, + "balance_loss_mlp": 1.03729975, + "epoch": 0.09920038477725004, + "flos": 14064307200000.0, + "grad_norm": 2.1999675157975664, + "language_loss": 0.7830584, + "learning_rate": 3.9500537311853635e-06, + "loss": 0.80602801, + "num_input_tokens_seen": 17641040, + "router_z_loss_clip": 1.60742188, + "router_z_loss_mlp": 0.26916504, + "step": 825, + "time_per_iteration": 2.8034818172454834 + }, + { + "auxiliary_loss_clip": 0.01235638, + "auxiliary_loss_mlp": 0.01069193, + "balance_loss_clip": 1.07062268, + "balance_loss_mlp": 1.04089284, + "epoch": 0.09932062766788914, + "flos": 13407070095360.0, + "grad_norm": 3.0125906192113767, + "language_loss": 0.83736289, + "learning_rate": 3.949880583845136e-06, + "loss": 0.86041123, + "num_input_tokens_seen": 17659115, + "router_z_loss_clip": 1.6484375, + "router_z_loss_mlp": 0.28320312, + "step": 826, + "time_per_iteration": 3.6897802352905273 + }, + { + "auxiliary_loss_clip": 0.012337, + "auxiliary_loss_mlp": 0.01059878, + "balance_loss_clip": 1.07246244, + "balance_loss_mlp": 1.03288925, + "epoch": 0.09944087055852822, + "flos": 19500751566720.0, + "grad_norm": 1.7701711056802796, + "language_loss": 0.81344044, + "learning_rate": 3.949707140709575e-06, + "loss": 0.83637625, + "num_input_tokens_seen": 17678845, + "router_z_loss_clip": 1.61132812, + "router_z_loss_mlp": 0.26989746, + "step": 827, + "time_per_iteration": 4.542106628417969 + }, + { + "auxiliary_loss_clip": 0.01234566, + "auxiliary_loss_mlp": 0.01068086, + "balance_loss_clip": 1.07009315, + "balance_loss_mlp": 1.0415976, + "epoch": 0.09956111344916732, + "flos": 17749100926080.0, + "grad_norm": 2.4957064299947556, + "language_loss": 0.83755875, + "learning_rate": 3.949533401804991e-06, + "loss": 0.86058527, + "num_input_tokens_seen": 17695750, + "router_z_loss_clip": 1.64453125, + "router_z_loss_mlp": 0.26501465, + "step": 828, + "time_per_iteration": 3.7639081478118896 + }, + { + "auxiliary_loss_clip": 0.01230388, + "auxiliary_loss_mlp": 0.01063935, + "balance_loss_clip": 1.07126272, + "balance_loss_mlp": 1.03688669, + "epoch": 0.0996813563398064, + "flos": 17967581400960.0, + "grad_norm": 2.1485473169225084, + "language_loss": 0.9078126, + "learning_rate": 3.949359367157739e-06, + "loss": 0.93075579, + "num_input_tokens_seen": 17714445, + "router_z_loss_clip": 1.59277344, + "router_z_loss_mlp": 0.27050781, + "step": 829, + "time_per_iteration": 2.814370632171631 + }, + { + "auxiliary_loss_clip": 0.01235044, + "auxiliary_loss_mlp": 0.0107538, + "balance_loss_clip": 1.07169819, + "balance_loss_mlp": 1.04667401, + "epoch": 0.0998015992304455, + "flos": 17457039440640.0, + "grad_norm": 1.95494897318834, + "language_loss": 0.75232875, + "learning_rate": 3.949185036794222e-06, + "loss": 0.775433, + "num_input_tokens_seen": 17732455, + "router_z_loss_clip": 1.63378906, + "router_z_loss_mlp": 0.28686523, + "step": 830, + "time_per_iteration": 2.7943732738494873 + }, + { + "auxiliary_loss_clip": 0.01228828, + "auxiliary_loss_mlp": 0.01059366, + "balance_loss_clip": 1.07131267, + "balance_loss_mlp": 1.0346179, + "epoch": 0.0999218421210846, + "flos": 25888757080320.0, + "grad_norm": 1.9578117410221065, + "language_loss": 0.78922033, + "learning_rate": 3.949010410740884e-06, + "loss": 0.8121022, + "num_input_tokens_seen": 17755280, + "router_z_loss_clip": 1.57324219, + "router_z_loss_mlp": 0.24768066, + "step": 831, + "time_per_iteration": 2.879431962966919 + }, + { + "auxiliary_loss_clip": 0.01234976, + "auxiliary_loss_mlp": 0.01074783, + "balance_loss_clip": 1.07030189, + "balance_loss_mlp": 1.04687572, + "epoch": 0.10004208501172368, + "flos": 21215916967680.0, + "grad_norm": 1.723104798495314, + "language_loss": 0.86411822, + "learning_rate": 3.948835489024216e-06, + "loss": 0.88721579, + "num_input_tokens_seen": 17775015, + "router_z_loss_clip": 1.64746094, + "router_z_loss_mlp": 0.27893066, + "step": 832, + "time_per_iteration": 2.842841386795044 + }, + { + "auxiliary_loss_clip": 0.01224896, + "auxiliary_loss_mlp": 0.01068392, + "balance_loss_clip": 1.06684208, + "balance_loss_mlp": 1.04251218, + "epoch": 0.10016232790236278, + "flos": 17348409734400.0, + "grad_norm": 1.9890275947985212, + "language_loss": 0.90506327, + "learning_rate": 3.948660271670755e-06, + "loss": 0.92799616, + "num_input_tokens_seen": 17792165, + "router_z_loss_clip": 1.58203125, + "router_z_loss_mlp": 0.25866699, + "step": 833, + "time_per_iteration": 2.8445327281951904 + }, + { + "auxiliary_loss_clip": 0.01226394, + "auxiliary_loss_mlp": 0.01063399, + "balance_loss_clip": 1.06794751, + "balance_loss_mlp": 1.03593349, + "epoch": 0.10028257079300186, + "flos": 25666541591040.0, + "grad_norm": 2.0680356278302483, + "language_loss": 0.84018826, + "learning_rate": 3.948484758707079e-06, + "loss": 0.8630861, + "num_input_tokens_seen": 17811765, + "router_z_loss_clip": 1.58496094, + "router_z_loss_mlp": 0.2746582, + "step": 834, + "time_per_iteration": 2.9180500507354736 + }, + { + "auxiliary_loss_clip": 0.01235996, + "auxiliary_loss_mlp": 0.01069473, + "balance_loss_clip": 1.07224298, + "balance_loss_mlp": 1.04133916, + "epoch": 0.10040281368364096, + "flos": 25156035544320.0, + "grad_norm": 2.209314584410341, + "language_loss": 0.83684492, + "learning_rate": 3.948308950159815e-06, + "loss": 0.85989964, + "num_input_tokens_seen": 17830445, + "router_z_loss_clip": 1.63671875, + "router_z_loss_mlp": 0.28088379, + "step": 835, + "time_per_iteration": 2.8636698722839355 + }, + { + "auxiliary_loss_clip": 0.01232069, + "auxiliary_loss_mlp": 0.01076551, + "balance_loss_clip": 1.06638169, + "balance_loss_mlp": 1.04825115, + "epoch": 0.10052305657428004, + "flos": 17603303621760.0, + "grad_norm": 2.660177990279231, + "language_loss": 0.75873262, + "learning_rate": 3.9481328460556326e-06, + "loss": 0.78181875, + "num_input_tokens_seen": 17847665, + "router_z_loss_clip": 1.65722656, + "router_z_loss_mlp": 0.28320312, + "step": 836, + "time_per_iteration": 2.7865185737609863 + }, + { + "auxiliary_loss_clip": 0.01229214, + "auxiliary_loss_mlp": 0.01065498, + "balance_loss_clip": 1.06815398, + "balance_loss_mlp": 1.0401895, + "epoch": 0.10064329946491914, + "flos": 18660154510080.0, + "grad_norm": 2.2191087041285056, + "language_loss": 0.89646184, + "learning_rate": 3.9479564464212455e-06, + "loss": 0.91940892, + "num_input_tokens_seen": 17866825, + "router_z_loss_clip": 1.61132812, + "router_z_loss_mlp": 0.2532959, + "step": 837, + "time_per_iteration": 2.852883815765381 + }, + { + "auxiliary_loss_clip": 0.01238444, + "auxiliary_loss_mlp": 0.01070279, + "balance_loss_clip": 1.07173252, + "balance_loss_mlp": 1.04131079, + "epoch": 0.10076354235555823, + "flos": 17199056983680.0, + "grad_norm": 2.370125846003246, + "language_loss": 0.76092494, + "learning_rate": 3.947779751283414e-06, + "loss": 0.7840122, + "num_input_tokens_seen": 17883995, + "router_z_loss_clip": 1.66699219, + "router_z_loss_mlp": 0.28955078, + "step": 838, + "time_per_iteration": 2.8521673679351807 + }, + { + "auxiliary_loss_clip": 0.01234115, + "auxiliary_loss_mlp": 0.01070735, + "balance_loss_clip": 1.0767951, + "balance_loss_mlp": 1.04388881, + "epoch": 0.10088378524619732, + "flos": 22962252395520.0, + "grad_norm": 1.9856031299110863, + "language_loss": 0.76337206, + "learning_rate": 3.947602760668944e-06, + "loss": 0.78642058, + "num_input_tokens_seen": 17903785, + "router_z_loss_clip": 1.57226562, + "router_z_loss_mlp": 0.2689209, + "step": 839, + "time_per_iteration": 2.8144779205322266 + }, + { + "auxiliary_loss_clip": 0.01231686, + "auxiliary_loss_mlp": 0.01060453, + "balance_loss_clip": 1.07050705, + "balance_loss_mlp": 1.0327723, + "epoch": 0.10100402813683641, + "flos": 37885828746240.0, + "grad_norm": 2.0798162974696646, + "language_loss": 0.71168661, + "learning_rate": 3.947425474604684e-06, + "loss": 0.73460799, + "num_input_tokens_seen": 17927720, + "router_z_loss_clip": 1.61132812, + "router_z_loss_mlp": 0.27685547, + "step": 840, + "time_per_iteration": 2.888725519180298 + }, + { + "auxiliary_loss_clip": 0.01229215, + "auxiliary_loss_mlp": 0.01058967, + "balance_loss_clip": 1.07175779, + "balance_loss_mlp": 1.0318234, + "epoch": 0.1011242710274755, + "flos": 21543458112000.0, + "grad_norm": 2.193087509320428, + "language_loss": 0.92213601, + "learning_rate": 3.947247893117528e-06, + "loss": 0.94501781, + "num_input_tokens_seen": 17946225, + "router_z_loss_clip": 1.57519531, + "router_z_loss_mlp": 0.27148438, + "step": 841, + "time_per_iteration": 2.8349318504333496 + }, + { + "auxiliary_loss_clip": 0.01239567, + "auxiliary_loss_mlp": 0.01064612, + "balance_loss_clip": 1.07246566, + "balance_loss_mlp": 1.03819537, + "epoch": 0.10124451391811459, + "flos": 13621456419840.0, + "grad_norm": 2.9736120827672226, + "language_loss": 0.69640338, + "learning_rate": 3.947070016234413e-06, + "loss": 0.71944517, + "num_input_tokens_seen": 17962015, + "router_z_loss_clip": 1.66894531, + "router_z_loss_mlp": 0.26416016, + "step": 842, + "time_per_iteration": 2.8293886184692383 + }, + { + "auxiliary_loss_clip": 0.01232184, + "auxiliary_loss_mlp": 0.01076911, + "balance_loss_clip": 1.0711832, + "balance_loss_mlp": 1.04857457, + "epoch": 0.10136475680875369, + "flos": 16649228522880.0, + "grad_norm": 2.2244468973777485, + "language_loss": 0.7502147, + "learning_rate": 3.946891843982326e-06, + "loss": 0.77330565, + "num_input_tokens_seen": 17979680, + "router_z_loss_clip": 1.61035156, + "router_z_loss_mlp": 0.28320312, + "step": 843, + "time_per_iteration": 2.8821346759796143 + }, + { + "auxiliary_loss_clip": 0.01235515, + "auxiliary_loss_mlp": 0.01067, + "balance_loss_clip": 1.07308626, + "balance_loss_mlp": 1.03865242, + "epoch": 0.10148499969939277, + "flos": 19461034103040.0, + "grad_norm": 1.9580445669495334, + "language_loss": 0.74616987, + "learning_rate": 3.9467133763882935e-06, + "loss": 0.76919502, + "num_input_tokens_seen": 17998145, + "router_z_loss_clip": 1.62402344, + "router_z_loss_mlp": 0.2833252, + "step": 844, + "time_per_iteration": 2.8069093227386475 + }, + { + "auxiliary_loss_clip": 0.01232474, + "auxiliary_loss_mlp": 0.01068893, + "balance_loss_clip": 1.07248914, + "balance_loss_mlp": 1.04247665, + "epoch": 0.10160524259003187, + "flos": 21104988791040.0, + "grad_norm": 2.3971482970591187, + "language_loss": 0.86369365, + "learning_rate": 3.9465346134793905e-06, + "loss": 0.88670737, + "num_input_tokens_seen": 18017955, + "router_z_loss_clip": 1.59960938, + "router_z_loss_mlp": 0.2644043, + "step": 845, + "time_per_iteration": 2.871105432510376 + }, + { + "auxiliary_loss_clip": 0.01231018, + "auxiliary_loss_mlp": 0.01061291, + "balance_loss_clip": 1.07093847, + "balance_loss_mlp": 1.03510094, + "epoch": 0.10172548548067095, + "flos": 17712687513600.0, + "grad_norm": 2.222489616385698, + "language_loss": 0.79495615, + "learning_rate": 3.9463555552827335e-06, + "loss": 0.8178792, + "num_input_tokens_seen": 18035125, + "router_z_loss_clip": 1.59863281, + "router_z_loss_mlp": 0.26196289, + "step": 846, + "time_per_iteration": 2.9193618297576904 + }, + { + "auxiliary_loss_clip": 0.01229503, + "auxiliary_loss_mlp": 0.0106692, + "balance_loss_clip": 1.06979823, + "balance_loss_mlp": 1.04003835, + "epoch": 0.10184572837131005, + "flos": 21104845136640.0, + "grad_norm": 3.707473584944694, + "language_loss": 0.86213267, + "learning_rate": 3.946176201825487e-06, + "loss": 0.88509691, + "num_input_tokens_seen": 18053160, + "router_z_loss_clip": 1.59667969, + "router_z_loss_mlp": 0.26855469, + "step": 847, + "time_per_iteration": 2.8850719928741455 + }, + { + "auxiliary_loss_clip": 0.01229307, + "auxiliary_loss_mlp": 0.0106586, + "balance_loss_clip": 1.07047951, + "balance_loss_mlp": 1.03760743, + "epoch": 0.10196597126194913, + "flos": 26067591918720.0, + "grad_norm": 2.6190184392269575, + "language_loss": 0.8352949, + "learning_rate": 3.9459965531348575e-06, + "loss": 0.85824656, + "num_input_tokens_seen": 18072815, + "router_z_loss_clip": 1.58886719, + "router_z_loss_mlp": 0.28259277, + "step": 848, + "time_per_iteration": 2.924980401992798 + }, + { + "auxiliary_loss_clip": 0.01229369, + "auxiliary_loss_mlp": 0.01067373, + "balance_loss_clip": 1.06906056, + "balance_loss_mlp": 1.04064596, + "epoch": 0.10208621415258823, + "flos": 29314634595840.0, + "grad_norm": 2.488601558604994, + "language_loss": 0.85132825, + "learning_rate": 3.945816609238098e-06, + "loss": 0.87429559, + "num_input_tokens_seen": 18092225, + "router_z_loss_clip": 1.6015625, + "router_z_loss_mlp": 0.26708984, + "step": 849, + "time_per_iteration": 3.0816049575805664 + }, + { + "auxiliary_loss_clip": 0.01233561, + "auxiliary_loss_mlp": 0.01072216, + "balance_loss_clip": 1.07079768, + "balance_loss_mlp": 1.04340315, + "epoch": 0.10220645704322733, + "flos": 23805794367360.0, + "grad_norm": 2.028566631074141, + "language_loss": 0.85523194, + "learning_rate": 3.945636370162507e-06, + "loss": 0.8782897, + "num_input_tokens_seen": 18112335, + "router_z_loss_clip": 1.62695312, + "router_z_loss_mlp": 0.28796387, + "step": 850, + "time_per_iteration": 2.863640785217285 + }, + { + "auxiliary_loss_clip": 0.01229719, + "auxiliary_loss_mlp": 0.01081258, + "balance_loss_clip": 1.07253194, + "balance_loss_mlp": 1.05578327, + "epoch": 0.10232669993386641, + "flos": 23218546913280.0, + "grad_norm": 1.9578194614463784, + "language_loss": 0.79123211, + "learning_rate": 3.945455835935425e-06, + "loss": 0.81434184, + "num_input_tokens_seen": 18131520, + "router_z_loss_clip": 1.5703125, + "router_z_loss_mlp": 0.25488281, + "step": 851, + "time_per_iteration": 2.8370699882507324 + }, + { + "auxiliary_loss_clip": 0.01237595, + "auxiliary_loss_mlp": 0.01071138, + "balance_loss_clip": 1.07485795, + "balance_loss_mlp": 1.0412879, + "epoch": 0.1024469428245055, + "flos": 22922929981440.0, + "grad_norm": 2.641243525680755, + "language_loss": 0.75235808, + "learning_rate": 3.94527500658424e-06, + "loss": 0.77544546, + "num_input_tokens_seen": 18149185, + "router_z_loss_clip": 1.62695312, + "router_z_loss_mlp": 0.29833984, + "step": 852, + "time_per_iteration": 2.7988364696502686 + }, + { + "auxiliary_loss_clip": 0.01235709, + "auxiliary_loss_mlp": 0.01069125, + "balance_loss_clip": 1.0751431, + "balance_loss_mlp": 1.04236221, + "epoch": 0.10256718571514459, + "flos": 31359495957120.0, + "grad_norm": 1.7324258633947596, + "language_loss": 0.80967641, + "learning_rate": 3.945093882136382e-06, + "loss": 0.83272481, + "num_input_tokens_seen": 18172960, + "router_z_loss_clip": 1.60644531, + "router_z_loss_mlp": 0.26782227, + "step": 853, + "time_per_iteration": 2.9446589946746826 + }, + { + "auxiliary_loss_clip": 0.01226147, + "auxiliary_loss_mlp": 0.01069724, + "balance_loss_clip": 1.07123971, + "balance_loss_mlp": 1.04446363, + "epoch": 0.10268742860578368, + "flos": 23474877344640.0, + "grad_norm": 1.8712959402887228, + "language_loss": 0.84950531, + "learning_rate": 3.944912462619329e-06, + "loss": 0.87246394, + "num_input_tokens_seen": 18191925, + "router_z_loss_clip": 1.54882812, + "router_z_loss_mlp": 0.25256348, + "step": 854, + "time_per_iteration": 2.78914213180542 + }, + { + "auxiliary_loss_clip": 0.01239373, + "auxiliary_loss_mlp": 0.01067973, + "balance_loss_clip": 1.07183516, + "balance_loss_mlp": 1.04044795, + "epoch": 0.10280767149642277, + "flos": 25520313323520.0, + "grad_norm": 2.5895688480337897, + "language_loss": 0.80328774, + "learning_rate": 3.9447307480606025e-06, + "loss": 0.82636118, + "num_input_tokens_seen": 18212010, + "router_z_loss_clip": 1.67480469, + "router_z_loss_mlp": 0.27502441, + "step": 855, + "time_per_iteration": 3.639925956726074 + }, + { + "auxiliary_loss_clip": 0.01230029, + "auxiliary_loss_mlp": 0.01069507, + "balance_loss_clip": 1.06951153, + "balance_loss_mlp": 1.04088438, + "epoch": 0.10292791438706186, + "flos": 17347691462400.0, + "grad_norm": 3.230028811123399, + "language_loss": 0.89861727, + "learning_rate": 3.944548738487767e-06, + "loss": 0.92161274, + "num_input_tokens_seen": 18229525, + "router_z_loss_clip": 1.60546875, + "router_z_loss_mlp": 0.28625488, + "step": 856, + "time_per_iteration": 4.5189924240112305 + }, + { + "auxiliary_loss_clip": 0.01238821, + "auxiliary_loss_mlp": 0.01071108, + "balance_loss_clip": 1.07431316, + "balance_loss_mlp": 1.04447651, + "epoch": 0.10304815727770096, + "flos": 27052693390080.0, + "grad_norm": 1.876722197805783, + "language_loss": 0.90763998, + "learning_rate": 3.944366433928434e-06, + "loss": 0.93073928, + "num_input_tokens_seen": 18249505, + "router_z_loss_clip": 1.64355469, + "router_z_loss_mlp": 0.26647949, + "step": 857, + "time_per_iteration": 2.970466136932373 + }, + { + "auxiliary_loss_clip": 0.01224046, + "auxiliary_loss_mlp": 0.01069506, + "balance_loss_clip": 1.07063794, + "balance_loss_mlp": 1.0446384, + "epoch": 0.10316840016834004, + "flos": 22782591544320.0, + "grad_norm": 1.5582987304134137, + "language_loss": 0.83663619, + "learning_rate": 3.9441838344102594e-06, + "loss": 0.8595717, + "num_input_tokens_seen": 18269230, + "router_z_loss_clip": 1.53417969, + "router_z_loss_mlp": 0.24865723, + "step": 858, + "time_per_iteration": 2.846785068511963 + }, + { + "auxiliary_loss_clip": 0.01231899, + "auxiliary_loss_mlp": 0.01076595, + "balance_loss_clip": 1.07105803, + "balance_loss_mlp": 1.04923654, + "epoch": 0.10328864305897914, + "flos": 20704584908160.0, + "grad_norm": 2.884673309898748, + "language_loss": 0.67215669, + "learning_rate": 3.944000939960943e-06, + "loss": 0.69524157, + "num_input_tokens_seen": 18287955, + "router_z_loss_clip": 1.61035156, + "router_z_loss_mlp": 0.27355957, + "step": 859, + "time_per_iteration": 2.9128994941711426 + }, + { + "auxiliary_loss_clip": 0.01230836, + "auxiliary_loss_mlp": 0.01065467, + "balance_loss_clip": 1.07200336, + "balance_loss_mlp": 1.04075551, + "epoch": 0.10340888594961822, + "flos": 28478814048000.0, + "grad_norm": 1.5413862736396837, + "language_loss": 0.79855222, + "learning_rate": 3.943817750608229e-06, + "loss": 0.8215152, + "num_input_tokens_seen": 18310505, + "router_z_loss_clip": 1.58886719, + "router_z_loss_mlp": 0.24731445, + "step": 860, + "time_per_iteration": 2.8573265075683594 + }, + { + "auxiliary_loss_clip": 0.01231104, + "auxiliary_loss_mlp": 0.01072992, + "balance_loss_clip": 1.07357621, + "balance_loss_mlp": 1.04676616, + "epoch": 0.10352912884025732, + "flos": 13370333460480.0, + "grad_norm": 3.002903054494438, + "language_loss": 0.82060134, + "learning_rate": 3.943634266379908e-06, + "loss": 0.84364229, + "num_input_tokens_seen": 18327400, + "router_z_loss_clip": 1.57617188, + "router_z_loss_mlp": 0.26245117, + "step": 861, + "time_per_iteration": 2.837770938873291 + }, + { + "auxiliary_loss_clip": 0.01225189, + "auxiliary_loss_mlp": 0.01059054, + "balance_loss_clip": 1.06818223, + "balance_loss_mlp": 1.0331974, + "epoch": 0.10364937173089642, + "flos": 25558558329600.0, + "grad_norm": 1.990309325795108, + "language_loss": 0.85242814, + "learning_rate": 3.943450487303815e-06, + "loss": 0.87527061, + "num_input_tokens_seen": 18347895, + "router_z_loss_clip": 1.56933594, + "router_z_loss_mlp": 0.25878906, + "step": 862, + "time_per_iteration": 2.9377288818359375 + }, + { + "auxiliary_loss_clip": 0.01226216, + "auxiliary_loss_mlp": 0.01058902, + "balance_loss_clip": 1.07000804, + "balance_loss_mlp": 1.03360629, + "epoch": 0.1037696146215355, + "flos": 21215486004480.0, + "grad_norm": 2.0410395883371097, + "language_loss": 0.85291433, + "learning_rate": 3.943266413407827e-06, + "loss": 0.87576556, + "num_input_tokens_seen": 18367170, + "router_z_loss_clip": 1.56152344, + "router_z_loss_mlp": 0.25292969, + "step": 863, + "time_per_iteration": 2.8335089683532715 + }, + { + "auxiliary_loss_clip": 0.01224825, + "auxiliary_loss_mlp": 0.01062572, + "balance_loss_clip": 1.07106614, + "balance_loss_mlp": 1.03721595, + "epoch": 0.1038898575121746, + "flos": 25807382818560.0, + "grad_norm": 2.722458644034902, + "language_loss": 0.84882605, + "learning_rate": 3.94308204471987e-06, + "loss": 0.87170005, + "num_input_tokens_seen": 18386185, + "router_z_loss_clip": 1.5390625, + "router_z_loss_mlp": 0.25366211, + "step": 864, + "time_per_iteration": 2.8240890502929688 + }, + { + "auxiliary_loss_clip": 0.01225813, + "auxiliary_loss_mlp": 0.01061137, + "balance_loss_clip": 1.06949222, + "balance_loss_mlp": 1.03524506, + "epoch": 0.10401010040281368, + "flos": 19062425900160.0, + "grad_norm": 2.5616656517716327, + "language_loss": 0.74715233, + "learning_rate": 3.942897381267912e-06, + "loss": 0.7700218, + "num_input_tokens_seen": 18402550, + "router_z_loss_clip": 1.56542969, + "router_z_loss_mlp": 0.25939941, + "step": 865, + "time_per_iteration": 2.778686761856079 + }, + { + "auxiliary_loss_clip": 0.012372, + "auxiliary_loss_mlp": 0.01062549, + "balance_loss_clip": 1.07251549, + "balance_loss_mlp": 1.0359056, + "epoch": 0.10413034329345278, + "flos": 16355119962240.0, + "grad_norm": 2.3540726568756773, + "language_loss": 0.66153383, + "learning_rate": 3.942712423079965e-06, + "loss": 0.68453133, + "num_input_tokens_seen": 18418940, + "router_z_loss_clip": 1.6484375, + "router_z_loss_mlp": 0.26672363, + "step": 866, + "time_per_iteration": 2.8045451641082764 + }, + { + "auxiliary_loss_clip": 0.01221249, + "auxiliary_loss_mlp": 0.01058966, + "balance_loss_clip": 1.06957304, + "balance_loss_mlp": 1.03380084, + "epoch": 0.10425058618409186, + "flos": 17236511890560.0, + "grad_norm": 2.9172670288863136, + "language_loss": 0.90497983, + "learning_rate": 3.942527170184088e-06, + "loss": 0.92778194, + "num_input_tokens_seen": 18435560, + "router_z_loss_clip": 1.51855469, + "router_z_loss_mlp": 0.25170898, + "step": 867, + "time_per_iteration": 2.860116481781006 + }, + { + "auxiliary_loss_clip": 0.01233944, + "auxiliary_loss_mlp": 0.01063448, + "balance_loss_clip": 1.07300663, + "balance_loss_mlp": 1.03666162, + "epoch": 0.10437082907473096, + "flos": 17967365919360.0, + "grad_norm": 2.313192065233821, + "language_loss": 0.77318347, + "learning_rate": 3.942341622608385e-06, + "loss": 0.79615736, + "num_input_tokens_seen": 18452590, + "router_z_loss_clip": 1.61035156, + "router_z_loss_mlp": 0.26843262, + "step": 868, + "time_per_iteration": 2.8330698013305664 + }, + { + "auxiliary_loss_clip": 0.01227025, + "auxiliary_loss_mlp": 0.01058695, + "balance_loss_clip": 1.07353878, + "balance_loss_mlp": 1.03239703, + "epoch": 0.10449107196537005, + "flos": 36283315374720.0, + "grad_norm": 1.5776427093097778, + "language_loss": 0.77962399, + "learning_rate": 3.942155780381001e-06, + "loss": 0.80248117, + "num_input_tokens_seen": 18476325, + "router_z_loss_clip": 1.53613281, + "router_z_loss_mlp": 0.26281738, + "step": 869, + "time_per_iteration": 2.918342351913452 + }, + { + "auxiliary_loss_clip": 0.01231669, + "auxiliary_loss_mlp": 0.01061329, + "balance_loss_clip": 1.07064557, + "balance_loss_mlp": 1.03473377, + "epoch": 0.10461131485600914, + "flos": 23802095266560.0, + "grad_norm": 1.8927877865405172, + "language_loss": 0.75787425, + "learning_rate": 3.94196964353013e-06, + "loss": 0.78080428, + "num_input_tokens_seen": 18495775, + "router_z_loss_clip": 1.609375, + "router_z_loss_mlp": 0.26574707, + "step": 870, + "time_per_iteration": 2.9246408939361572 + }, + { + "auxiliary_loss_clip": 0.01228181, + "auxiliary_loss_mlp": 0.01059343, + "balance_loss_clip": 1.07370281, + "balance_loss_mlp": 1.03520358, + "epoch": 0.10473155774664823, + "flos": 18405476104320.0, + "grad_norm": 2.0213915579840434, + "language_loss": 0.80621088, + "learning_rate": 3.941783212084008e-06, + "loss": 0.82908612, + "num_input_tokens_seen": 18513530, + "router_z_loss_clip": 1.54394531, + "router_z_loss_mlp": 0.24145508, + "step": 871, + "time_per_iteration": 2.8268158435821533 + }, + { + "auxiliary_loss_clip": 0.01228303, + "auxiliary_loss_mlp": 0.01063793, + "balance_loss_clip": 1.07408524, + "balance_loss_mlp": 1.0385921, + "epoch": 0.10485180063728732, + "flos": 25592637358080.0, + "grad_norm": 2.640943235577881, + "language_loss": 0.78718334, + "learning_rate": 3.941596486070916e-06, + "loss": 0.81010431, + "num_input_tokens_seen": 18531575, + "router_z_loss_clip": 1.54296875, + "router_z_loss_mlp": 0.2520752, + "step": 872, + "time_per_iteration": 2.8015425205230713 + }, + { + "auxiliary_loss_clip": 0.01226, + "auxiliary_loss_mlp": 0.01063936, + "balance_loss_clip": 1.06856823, + "balance_loss_mlp": 1.0355525, + "epoch": 0.10497204352792641, + "flos": 27088747666560.0, + "grad_norm": 2.3347789610271086, + "language_loss": 0.58561718, + "learning_rate": 3.941409465519182e-06, + "loss": 0.60851657, + "num_input_tokens_seen": 18552100, + "router_z_loss_clip": 1.57421875, + "router_z_loss_mlp": 0.28405762, + "step": 873, + "time_per_iteration": 2.8265037536621094 + }, + { + "auxiliary_loss_clip": 0.01224969, + "auxiliary_loss_mlp": 0.0105735, + "balance_loss_clip": 1.06911314, + "balance_loss_mlp": 1.03232813, + "epoch": 0.10509228641856551, + "flos": 32858479353600.0, + "grad_norm": 1.7039654517617322, + "language_loss": 0.85320783, + "learning_rate": 3.941222150457176e-06, + "loss": 0.87603104, + "num_input_tokens_seen": 18575355, + "router_z_loss_clip": 1.55761719, + "router_z_loss_mlp": 0.25012207, + "step": 874, + "time_per_iteration": 2.8860390186309814 + }, + { + "auxiliary_loss_clip": 0.01228085, + "auxiliary_loss_mlp": 0.01055826, + "balance_loss_clip": 1.07158303, + "balance_loss_mlp": 1.03209162, + "epoch": 0.10521252930920459, + "flos": 14319165173760.0, + "grad_norm": 4.19915504925697, + "language_loss": 0.71586442, + "learning_rate": 3.941034540913311e-06, + "loss": 0.73870349, + "num_input_tokens_seen": 18592885, + "router_z_loss_clip": 1.56445312, + "router_z_loss_mlp": 0.23730469, + "step": 875, + "time_per_iteration": 2.9138505458831787 + }, + { + "auxiliary_loss_clip": 0.01224386, + "auxiliary_loss_mlp": 0.01052174, + "balance_loss_clip": 1.07047081, + "balance_loss_mlp": 1.02684164, + "epoch": 0.10533277219984369, + "flos": 21687028773120.0, + "grad_norm": 1.6899038097106995, + "language_loss": 0.82303369, + "learning_rate": 3.940846636916051e-06, + "loss": 0.84579933, + "num_input_tokens_seen": 18612920, + "router_z_loss_clip": 1.54003906, + "router_z_loss_mlp": 0.25354004, + "step": 876, + "time_per_iteration": 2.82624888420105 + }, + { + "auxiliary_loss_clip": 0.0122172, + "auxiliary_loss_mlp": 0.01061813, + "balance_loss_clip": 1.069929, + "balance_loss_mlp": 1.03689873, + "epoch": 0.10545301509048277, + "flos": 22269787027200.0, + "grad_norm": 2.031771206797871, + "language_loss": 0.86467868, + "learning_rate": 3.940658438493899e-06, + "loss": 0.88751405, + "num_input_tokens_seen": 18630765, + "router_z_loss_clip": 1.51855469, + "router_z_loss_mlp": 0.24926758, + "step": 877, + "time_per_iteration": 2.81404972076416 + }, + { + "auxiliary_loss_clip": 0.01233744, + "auxiliary_loss_mlp": 0.01060203, + "balance_loss_clip": 1.06984186, + "balance_loss_mlp": 1.03278446, + "epoch": 0.10557325798112187, + "flos": 22199725549440.0, + "grad_norm": 5.332658419879542, + "language_loss": 0.76190311, + "learning_rate": 3.940469945675405e-06, + "loss": 0.78484261, + "num_input_tokens_seen": 18649150, + "router_z_loss_clip": 1.63964844, + "router_z_loss_mlp": 0.27404785, + "step": 878, + "time_per_iteration": 2.858741044998169 + }, + { + "auxiliary_loss_clip": 0.0122482, + "auxiliary_loss_mlp": 0.01058191, + "balance_loss_clip": 1.07069707, + "balance_loss_mlp": 1.03228652, + "epoch": 0.10569350087176095, + "flos": 25775889569280.0, + "grad_norm": 2.02736173528745, + "language_loss": 0.9140898, + "learning_rate": 3.940281158489163e-06, + "loss": 0.93691993, + "num_input_tokens_seen": 18668380, + "router_z_loss_clip": 1.54394531, + "router_z_loss_mlp": 0.25878906, + "step": 879, + "time_per_iteration": 2.820115089416504 + }, + { + "auxiliary_loss_clip": 0.01222209, + "auxiliary_loss_mlp": 0.01059891, + "balance_loss_clip": 1.0685997, + "balance_loss_mlp": 1.03288972, + "epoch": 0.10581374376240005, + "flos": 17311385790720.0, + "grad_norm": 1.7389458529147295, + "language_loss": 0.82781994, + "learning_rate": 3.940092076963812e-06, + "loss": 0.85064095, + "num_input_tokens_seen": 18685875, + "router_z_loss_clip": 1.53710938, + "router_z_loss_mlp": 0.27001953, + "step": 880, + "time_per_iteration": 2.8146581649780273 + }, + { + "auxiliary_loss_clip": 0.01230398, + "auxiliary_loss_mlp": 0.01060817, + "balance_loss_clip": 1.07297421, + "balance_loss_mlp": 1.03472221, + "epoch": 0.10593398665303914, + "flos": 34349454017280.0, + "grad_norm": 2.0607963309828987, + "language_loss": 0.78784126, + "learning_rate": 3.9399027011280355e-06, + "loss": 0.81075346, + "num_input_tokens_seen": 18707970, + "router_z_loss_clip": 1.57421875, + "router_z_loss_mlp": 0.26135254, + "step": 881, + "time_per_iteration": 2.8633315563201904 + }, + { + "auxiliary_loss_clip": 0.0123049, + "auxiliary_loss_mlp": 0.01065216, + "balance_loss_clip": 1.07094717, + "balance_loss_mlp": 1.03827477, + "epoch": 0.10605422954367823, + "flos": 23257977068160.0, + "grad_norm": 1.9840484975192807, + "language_loss": 0.77165663, + "learning_rate": 3.939713031010561e-06, + "loss": 0.79461372, + "num_input_tokens_seen": 18726335, + "router_z_loss_clip": 1.59472656, + "router_z_loss_mlp": 0.26940918, + "step": 882, + "time_per_iteration": 2.835906744003296 + }, + { + "auxiliary_loss_clip": 0.01229655, + "auxiliary_loss_mlp": 0.01063627, + "balance_loss_clip": 1.07133341, + "balance_loss_mlp": 1.03722262, + "epoch": 0.10617447243431732, + "flos": 22820118278400.0, + "grad_norm": 3.0949036631169533, + "language_loss": 0.77888685, + "learning_rate": 3.939523066640163e-06, + "loss": 0.80181974, + "num_input_tokens_seen": 18745230, + "router_z_loss_clip": 1.58300781, + "router_z_loss_mlp": 0.26403809, + "step": 883, + "time_per_iteration": 2.887148857116699 + }, + { + "auxiliary_loss_clip": 0.01225602, + "auxiliary_loss_mlp": 0.01055668, + "balance_loss_clip": 1.06960177, + "balance_loss_mlp": 1.0280472, + "epoch": 0.10629471532495641, + "flos": 24386577373440.0, + "grad_norm": 1.7681517477015496, + "language_loss": 0.80628622, + "learning_rate": 3.939332808045657e-06, + "loss": 0.82909888, + "num_input_tokens_seen": 18764880, + "router_z_loss_clip": 1.5625, + "router_z_loss_mlp": 0.27612305, + "step": 884, + "time_per_iteration": 3.7129416465759277 + }, + { + "auxiliary_loss_clip": 0.0122602, + "auxiliary_loss_mlp": 0.01058531, + "balance_loss_clip": 1.06905973, + "balance_loss_mlp": 1.03290129, + "epoch": 0.1064149582155955, + "flos": 21105491581440.0, + "grad_norm": 2.49478126071319, + "language_loss": 0.84780192, + "learning_rate": 3.939142255255906e-06, + "loss": 0.87064749, + "num_input_tokens_seen": 18785765, + "router_z_loss_clip": 1.56933594, + "router_z_loss_mlp": 0.2565918, + "step": 885, + "time_per_iteration": 5.309668302536011 + }, + { + "auxiliary_loss_clip": 0.01224881, + "auxiliary_loss_mlp": 0.010695, + "balance_loss_clip": 1.07156277, + "balance_loss_mlp": 1.04383445, + "epoch": 0.1065352011062346, + "flos": 20702035042560.0, + "grad_norm": 3.054689167981753, + "language_loss": 0.86986172, + "learning_rate": 3.938951408299817e-06, + "loss": 0.89280546, + "num_input_tokens_seen": 18804605, + "router_z_loss_clip": 1.53417969, + "router_z_loss_mlp": 0.2565918, + "step": 886, + "time_per_iteration": 2.9606010913848877 + }, + { + "auxiliary_loss_clip": 0.01102527, + "auxiliary_loss_mlp": 0.01023631, + "balance_loss_clip": 1.03469002, + "balance_loss_mlp": 1.01931536, + "epoch": 0.10665544399687368, + "flos": 62659632689280.0, + "grad_norm": 0.8018658671833503, + "language_loss": 0.54427481, + "learning_rate": 3.938760267206342e-06, + "loss": 0.56553638, + "num_input_tokens_seen": 18866425, + "router_z_loss_clip": 0.6796875, + "router_z_loss_mlp": 0.04321289, + "step": 887, + "time_per_iteration": 3.2304275035858154 + }, + { + "auxiliary_loss_clip": 0.01228404, + "auxiliary_loss_mlp": 0.01052869, + "balance_loss_clip": 1.07192159, + "balance_loss_mlp": 1.02716744, + "epoch": 0.10677568688751278, + "flos": 26140382830080.0, + "grad_norm": 2.0771197695483052, + "language_loss": 0.78465766, + "learning_rate": 3.938568832004475e-06, + "loss": 0.80747032, + "num_input_tokens_seen": 18885130, + "router_z_loss_clip": 1.5625, + "router_z_loss_mlp": 0.25671387, + "step": 888, + "time_per_iteration": 2.9539499282836914 + }, + { + "auxiliary_loss_clip": 0.01224914, + "auxiliary_loss_mlp": 0.01064999, + "balance_loss_clip": 1.06817937, + "balance_loss_mlp": 1.03768778, + "epoch": 0.10689592977815186, + "flos": 12786533712000.0, + "grad_norm": 2.0881694713796595, + "language_loss": 0.75269759, + "learning_rate": 3.938377102723257e-06, + "loss": 0.77559674, + "num_input_tokens_seen": 18902265, + "router_z_loss_clip": 1.56835938, + "router_z_loss_mlp": 0.27331543, + "step": 889, + "time_per_iteration": 2.8947067260742188 + }, + { + "auxiliary_loss_clip": 0.0123172, + "auxiliary_loss_mlp": 0.01072301, + "balance_loss_clip": 1.06894577, + "balance_loss_mlp": 1.0432018, + "epoch": 0.10701617266879096, + "flos": 22126683242880.0, + "grad_norm": 2.3673765540360376, + "language_loss": 0.83301032, + "learning_rate": 3.938185079391774e-06, + "loss": 0.85605055, + "num_input_tokens_seen": 18919310, + "router_z_loss_clip": 1.62792969, + "router_z_loss_mlp": 0.29064941, + "step": 890, + "time_per_iteration": 2.820202112197876 + }, + { + "auxiliary_loss_clip": 0.01224908, + "auxiliary_loss_mlp": 0.0106407, + "balance_loss_clip": 1.07120156, + "balance_loss_mlp": 1.03789127, + "epoch": 0.10713641555943004, + "flos": 19745625559680.0, + "grad_norm": 5.807750597614754, + "language_loss": 1.05871034, + "learning_rate": 3.937992762039157e-06, + "loss": 1.08160007, + "num_input_tokens_seen": 18932635, + "router_z_loss_clip": 1.53808594, + "router_z_loss_mlp": 0.26184082, + "step": 891, + "time_per_iteration": 2.775355100631714 + }, + { + "auxiliary_loss_clip": 0.01224156, + "auxiliary_loss_mlp": 0.01062653, + "balance_loss_clip": 1.07028079, + "balance_loss_mlp": 1.03771484, + "epoch": 0.10725665845006914, + "flos": 23952992302080.0, + "grad_norm": 1.9822592524276916, + "language_loss": 0.803738, + "learning_rate": 3.937800150694577e-06, + "loss": 0.82660615, + "num_input_tokens_seen": 18953810, + "router_z_loss_clip": 1.5390625, + "router_z_loss_mlp": 0.24951172, + "step": 892, + "time_per_iteration": 2.9213039875030518 + }, + { + "auxiliary_loss_clip": 0.0122352, + "auxiliary_loss_mlp": 0.01068818, + "balance_loss_clip": 1.06838751, + "balance_loss_mlp": 1.04076791, + "epoch": 0.10737690134070824, + "flos": 18551704371840.0, + "grad_norm": 2.1997831861123416, + "language_loss": 0.75894564, + "learning_rate": 3.937607245387255e-06, + "loss": 0.78186899, + "num_input_tokens_seen": 18973175, + "router_z_loss_clip": 1.55273438, + "router_z_loss_mlp": 0.28039551, + "step": 893, + "time_per_iteration": 2.918353319168091 + }, + { + "auxiliary_loss_clip": 0.01221696, + "auxiliary_loss_mlp": 0.01060779, + "balance_loss_clip": 1.06899524, + "balance_loss_mlp": 1.03601933, + "epoch": 0.10749714423134732, + "flos": 22707609903360.0, + "grad_norm": 1.8005715442563748, + "language_loss": 0.72611165, + "learning_rate": 3.937414046146455e-06, + "loss": 0.74893636, + "num_input_tokens_seen": 18991130, + "router_z_loss_clip": 1.52929688, + "router_z_loss_mlp": 0.24780273, + "step": 894, + "time_per_iteration": 2.9335286617279053 + }, + { + "auxiliary_loss_clip": 0.01226443, + "auxiliary_loss_mlp": 0.01064148, + "balance_loss_clip": 1.07093537, + "balance_loss_mlp": 1.03780293, + "epoch": 0.10761738712198642, + "flos": 21106066199040.0, + "grad_norm": 2.064199826512719, + "language_loss": 0.75313157, + "learning_rate": 3.9372205530014845e-06, + "loss": 0.77603745, + "num_input_tokens_seen": 19009610, + "router_z_loss_clip": 1.55566406, + "router_z_loss_mlp": 0.26342773, + "step": 895, + "time_per_iteration": 2.7931456565856934 + }, + { + "auxiliary_loss_clip": 0.01230311, + "auxiliary_loss_mlp": 0.01068412, + "balance_loss_clip": 1.06849754, + "balance_loss_mlp": 1.04054058, + "epoch": 0.1077376300126255, + "flos": 23766723348480.0, + "grad_norm": 2.0602883529856784, + "language_loss": 0.71692324, + "learning_rate": 3.937026765981696e-06, + "loss": 0.73991042, + "num_input_tokens_seen": 19029680, + "router_z_loss_clip": 1.61816406, + "router_z_loss_mlp": 0.27880859, + "step": 896, + "time_per_iteration": 2.7951762676239014 + }, + { + "auxiliary_loss_clip": 0.0123242, + "auxiliary_loss_mlp": 0.01068971, + "balance_loss_clip": 1.07552266, + "balance_loss_mlp": 1.04241109, + "epoch": 0.1078578729032646, + "flos": 20919581763840.0, + "grad_norm": 1.87307030354297, + "language_loss": 0.79321492, + "learning_rate": 3.936832685116488e-06, + "loss": 0.81622881, + "num_input_tokens_seen": 19047775, + "router_z_loss_clip": 1.5703125, + "router_z_loss_mlp": 0.265625, + "step": 897, + "time_per_iteration": 2.9315366744995117 + }, + { + "auxiliary_loss_clip": 0.01227741, + "auxiliary_loss_mlp": 0.01059042, + "balance_loss_clip": 1.06951571, + "balance_loss_mlp": 1.03256536, + "epoch": 0.10797811579390369, + "flos": 14829886702080.0, + "grad_norm": 2.5264942378470385, + "language_loss": 0.89900684, + "learning_rate": 3.936638310435301e-06, + "loss": 0.92187464, + "num_input_tokens_seen": 19065640, + "router_z_loss_clip": 1.58203125, + "router_z_loss_mlp": 0.26477051, + "step": 898, + "time_per_iteration": 2.789151191711426 + }, + { + "auxiliary_loss_clip": 0.01224093, + "auxiliary_loss_mlp": 0.01061254, + "balance_loss_clip": 1.07106781, + "balance_loss_mlp": 1.03643465, + "epoch": 0.10809835868454278, + "flos": 19536985411200.0, + "grad_norm": 1.834009639350958, + "language_loss": 0.81284529, + "learning_rate": 3.936443641967623e-06, + "loss": 0.83569872, + "num_input_tokens_seen": 19084470, + "router_z_loss_clip": 1.52929688, + "router_z_loss_mlp": 0.24829102, + "step": 899, + "time_per_iteration": 2.766681671142578 + }, + { + "auxiliary_loss_clip": 0.01223079, + "auxiliary_loss_mlp": 0.01062062, + "balance_loss_clip": 1.06844282, + "balance_loss_mlp": 1.03563344, + "epoch": 0.10821860157518187, + "flos": 18442320480000.0, + "grad_norm": 2.1649219900206678, + "language_loss": 0.83092499, + "learning_rate": 3.936248679742983e-06, + "loss": 0.8537764, + "num_input_tokens_seen": 19102965, + "router_z_loss_clip": 1.546875, + "router_z_loss_mlp": 0.26452637, + "step": 900, + "time_per_iteration": 2.8375961780548096 + }, + { + "auxiliary_loss_clip": 0.01102387, + "auxiliary_loss_mlp": 0.01013721, + "balance_loss_clip": 1.03443384, + "balance_loss_mlp": 1.00954878, + "epoch": 0.10833884446582095, + "flos": 49359468447360.0, + "grad_norm": 1.0904210893781283, + "language_loss": 0.70147139, + "learning_rate": 3.936053423790959e-06, + "loss": 0.72263247, + "num_input_tokens_seen": 19151285, + "router_z_loss_clip": 0.6796875, + "router_z_loss_mlp": 0.04174805, + "step": 901, + "time_per_iteration": 3.1143593788146973 + }, + { + "auxiliary_loss_clip": 0.01222966, + "auxiliary_loss_mlp": 0.01063401, + "balance_loss_clip": 1.06872416, + "balance_loss_mlp": 1.0351963, + "epoch": 0.10845908735646005, + "flos": 20411912891520.0, + "grad_norm": 1.702247482467135, + "language_loss": 0.77223539, + "learning_rate": 3.935857874141168e-06, + "loss": 0.79509908, + "num_input_tokens_seen": 19170120, + "router_z_loss_clip": 1.54199219, + "router_z_loss_mlp": 0.28210449, + "step": 902, + "time_per_iteration": 2.8192124366760254 + }, + { + "auxiliary_loss_clip": 0.01226758, + "auxiliary_loss_mlp": 0.01063385, + "balance_loss_clip": 1.07113934, + "balance_loss_mlp": 1.03489399, + "epoch": 0.10857933024709913, + "flos": 14027750133120.0, + "grad_norm": 2.127461905324595, + "language_loss": 0.83645272, + "learning_rate": 3.935662030823279e-06, + "loss": 0.85935414, + "num_input_tokens_seen": 19186305, + "router_z_loss_clip": 1.55664062, + "router_z_loss_mlp": 0.28466797, + "step": 903, + "time_per_iteration": 2.814724922180176 + }, + { + "auxiliary_loss_clip": 0.01219574, + "auxiliary_loss_mlp": 0.01061332, + "balance_loss_clip": 1.06537127, + "balance_loss_mlp": 1.03493905, + "epoch": 0.10869957313773823, + "flos": 13369004657280.0, + "grad_norm": 2.945598154919342, + "language_loss": 0.72898602, + "learning_rate": 3.935465893866998e-06, + "loss": 0.75179511, + "num_input_tokens_seen": 19204530, + "router_z_loss_clip": 1.54003906, + "router_z_loss_mlp": 0.26391602, + "step": 904, + "time_per_iteration": 2.796267509460449 + }, + { + "auxiliary_loss_clip": 0.01225677, + "auxiliary_loss_mlp": 0.01059425, + "balance_loss_clip": 1.06855965, + "balance_loss_mlp": 1.03242469, + "epoch": 0.10881981602837733, + "flos": 25807095509760.0, + "grad_norm": 2.0896453922436047, + "language_loss": 0.80405891, + "learning_rate": 3.935269463302079e-06, + "loss": 0.8269099, + "num_input_tokens_seen": 19222735, + "router_z_loss_clip": 1.57128906, + "router_z_loss_mlp": 0.26989746, + "step": 905, + "time_per_iteration": 2.8705942630767822 + }, + { + "auxiliary_loss_clip": 0.01231283, + "auxiliary_loss_mlp": 0.01062719, + "balance_loss_clip": 1.07172751, + "balance_loss_mlp": 1.03706551, + "epoch": 0.10894005891901641, + "flos": 20777555387520.0, + "grad_norm": 2.2449998474501864, + "language_loss": 0.76421648, + "learning_rate": 3.935072739158322e-06, + "loss": 0.78715652, + "num_input_tokens_seen": 19242445, + "router_z_loss_clip": 1.59570312, + "router_z_loss_mlp": 0.25646973, + "step": 906, + "time_per_iteration": 2.866529941558838 + }, + { + "auxiliary_loss_clip": 0.0122684, + "auxiliary_loss_mlp": 0.01070738, + "balance_loss_clip": 1.06878901, + "balance_loss_mlp": 1.04223561, + "epoch": 0.10906030180965551, + "flos": 26649883296000.0, + "grad_norm": 1.6099111292289545, + "language_loss": 0.79982281, + "learning_rate": 3.934875721465569e-06, + "loss": 0.82279861, + "num_input_tokens_seen": 19262865, + "router_z_loss_clip": 1.57910156, + "router_z_loss_mlp": 0.28503418, + "step": 907, + "time_per_iteration": 2.8565728664398193 + }, + { + "auxiliary_loss_clip": 0.01228134, + "auxiliary_loss_mlp": 0.01060326, + "balance_loss_clip": 1.06885695, + "balance_loss_mlp": 1.03303921, + "epoch": 0.10918054470029459, + "flos": 36534402420480.0, + "grad_norm": 2.7347224066030003, + "language_loss": 0.7199533, + "learning_rate": 3.9346784102537076e-06, + "loss": 0.74283791, + "num_input_tokens_seen": 19285000, + "router_z_loss_clip": 1.59082031, + "router_z_loss_mlp": 0.27307129, + "step": 908, + "time_per_iteration": 2.933709144592285 + }, + { + "auxiliary_loss_clip": 0.01223263, + "auxiliary_loss_mlp": 0.01063687, + "balance_loss_clip": 1.0667901, + "balance_loss_mlp": 1.03625679, + "epoch": 0.10930078759093369, + "flos": 21762549118080.0, + "grad_norm": 1.8024246595413957, + "language_loss": 0.78228927, + "learning_rate": 3.934480805552669e-06, + "loss": 0.80515873, + "num_input_tokens_seen": 19306010, + "router_z_loss_clip": 1.56542969, + "router_z_loss_mlp": 0.27429199, + "step": 909, + "time_per_iteration": 2.817837715148926 + }, + { + "auxiliary_loss_clip": 0.01218189, + "auxiliary_loss_mlp": 0.01071212, + "balance_loss_clip": 1.06585646, + "balance_loss_mlp": 1.04294789, + "epoch": 0.10942103048157277, + "flos": 22601781457920.0, + "grad_norm": 2.0006864975502894, + "language_loss": 0.88232917, + "learning_rate": 3.93428290739243e-06, + "loss": 0.90522325, + "num_input_tokens_seen": 19325380, + "router_z_loss_clip": 1.52441406, + "router_z_loss_mlp": 0.28259277, + "step": 910, + "time_per_iteration": 2.909151792526245 + }, + { + "auxiliary_loss_clip": 0.01219477, + "auxiliary_loss_mlp": 0.0106679, + "balance_loss_clip": 1.06650615, + "balance_loss_mlp": 1.04069519, + "epoch": 0.10954127337221187, + "flos": 15045781397760.0, + "grad_norm": 2.267817526574841, + "language_loss": 0.79882407, + "learning_rate": 3.9340847158030125e-06, + "loss": 0.82168674, + "num_input_tokens_seen": 19338960, + "router_z_loss_clip": 1.53027344, + "router_z_loss_mlp": 0.26098633, + "step": 911, + "time_per_iteration": 2.8587656021118164 + }, + { + "auxiliary_loss_clip": 0.0122132, + "auxiliary_loss_mlp": 0.01062448, + "balance_loss_clip": 1.06781495, + "balance_loss_mlp": 1.03785539, + "epoch": 0.10966151626285096, + "flos": 21650974496640.0, + "grad_norm": 1.9283201190277126, + "language_loss": 0.75529402, + "learning_rate": 3.9338862308144814e-06, + "loss": 0.77813172, + "num_input_tokens_seen": 19357780, + "router_z_loss_clip": 1.53417969, + "router_z_loss_mlp": 0.24584961, + "step": 912, + "time_per_iteration": 2.8251771926879883 + }, + { + "auxiliary_loss_clip": 0.01224305, + "auxiliary_loss_mlp": 0.01069841, + "balance_loss_clip": 1.06895351, + "balance_loss_mlp": 1.04250658, + "epoch": 0.10978175915349005, + "flos": 20121359777280.0, + "grad_norm": 1.7473582641332672, + "language_loss": 0.84654522, + "learning_rate": 3.933687452456946e-06, + "loss": 0.86948669, + "num_input_tokens_seen": 19377680, + "router_z_loss_clip": 1.5546875, + "router_z_loss_mlp": 0.27319336, + "step": 913, + "time_per_iteration": 4.479077339172363 + }, + { + "auxiliary_loss_clip": 0.01226091, + "auxiliary_loss_mlp": 0.01063637, + "balance_loss_clip": 1.06642842, + "balance_loss_mlp": 1.03676772, + "epoch": 0.10990200204412914, + "flos": 20412667077120.0, + "grad_norm": 2.163925324126049, + "language_loss": 0.86563838, + "learning_rate": 3.933488380760562e-06, + "loss": 0.88853562, + "num_input_tokens_seen": 19397040, + "router_z_loss_clip": 1.59667969, + "router_z_loss_mlp": 0.26867676, + "step": 914, + "time_per_iteration": 3.6210720539093018 + }, + { + "auxiliary_loss_clip": 0.01223649, + "auxiliary_loss_mlp": 0.010644, + "balance_loss_clip": 1.06882441, + "balance_loss_mlp": 1.03757811, + "epoch": 0.11002224493476823, + "flos": 17530117660800.0, + "grad_norm": 2.014378612137697, + "language_loss": 0.86909473, + "learning_rate": 3.9332890157555286e-06, + "loss": 0.89197522, + "num_input_tokens_seen": 19413975, + "router_z_loss_clip": 1.54882812, + "router_z_loss_mlp": 0.26831055, + "step": 915, + "time_per_iteration": 2.7814323902130127 + }, + { + "auxiliary_loss_clip": 0.01228014, + "auxiliary_loss_mlp": 0.01061453, + "balance_loss_clip": 1.07166314, + "balance_loss_mlp": 1.0362165, + "epoch": 0.11014248782540732, + "flos": 12203093099520.0, + "grad_norm": 1.9855423220715454, + "language_loss": 0.76641417, + "learning_rate": 3.933089357472088e-06, + "loss": 0.78930885, + "num_input_tokens_seen": 19432005, + "router_z_loss_clip": 1.56542969, + "router_z_loss_mlp": 0.25244141, + "step": 916, + "time_per_iteration": 2.799757242202759 + }, + { + "auxiliary_loss_clip": 0.01221876, + "auxiliary_loss_mlp": 0.01055801, + "balance_loss_clip": 1.07130706, + "balance_loss_mlp": 1.0301708, + "epoch": 0.11026273071604642, + "flos": 22382977760640.0, + "grad_norm": 1.8634157313421489, + "language_loss": 0.85975254, + "learning_rate": 3.932889405940529e-06, + "loss": 0.88252938, + "num_input_tokens_seen": 19450100, + "router_z_loss_clip": 1.50585938, + "router_z_loss_mlp": 0.25622559, + "step": 917, + "time_per_iteration": 2.8378183841705322 + }, + { + "auxiliary_loss_clip": 0.0123298, + "auxiliary_loss_mlp": 0.0106276, + "balance_loss_clip": 1.07569957, + "balance_loss_mlp": 1.03560448, + "epoch": 0.1103829736066855, + "flos": 19829046896640.0, + "grad_norm": 2.718724479801503, + "language_loss": 0.80100739, + "learning_rate": 3.932689161191184e-06, + "loss": 0.82396483, + "num_input_tokens_seen": 19467805, + "router_z_loss_clip": 1.57324219, + "router_z_loss_mlp": 0.27148438, + "step": 918, + "time_per_iteration": 2.8636109828948975 + }, + { + "auxiliary_loss_clip": 0.01227196, + "auxiliary_loss_mlp": 0.01055266, + "balance_loss_clip": 1.06946945, + "balance_loss_mlp": 1.02853966, + "epoch": 0.1105032164973246, + "flos": 22669616292480.0, + "grad_norm": 2.1119611323689793, + "language_loss": 0.88342559, + "learning_rate": 3.93248862325443e-06, + "loss": 0.90625012, + "num_input_tokens_seen": 19486710, + "router_z_loss_clip": 1.57714844, + "router_z_loss_mlp": 0.26721191, + "step": 919, + "time_per_iteration": 2.81836199760437 + }, + { + "auxiliary_loss_clip": 0.01100942, + "auxiliary_loss_mlp": 0.01010564, + "balance_loss_clip": 1.03403747, + "balance_loss_mlp": 1.00651097, + "epoch": 0.11062345938796368, + "flos": 66483507876480.0, + "grad_norm": 0.9380330923547545, + "language_loss": 0.64483774, + "learning_rate": 3.932287792160688e-06, + "loss": 0.6659528, + "num_input_tokens_seen": 19545170, + "router_z_loss_clip": 0.66796875, + "router_z_loss_mlp": 0.04052734, + "step": 920, + "time_per_iteration": 3.252040386199951 + }, + { + "auxiliary_loss_clip": 0.01229337, + "auxiliary_loss_mlp": 0.01065808, + "balance_loss_clip": 1.06984973, + "balance_loss_mlp": 1.03909302, + "epoch": 0.11074370227860278, + "flos": 21907771804800.0, + "grad_norm": 2.4418061408523126, + "language_loss": 0.8059749, + "learning_rate": 3.932086667940424e-06, + "loss": 0.82892632, + "num_input_tokens_seen": 19561875, + "router_z_loss_clip": 1.59472656, + "router_z_loss_mlp": 0.26696777, + "step": 921, + "time_per_iteration": 2.9398319721221924 + }, + { + "auxiliary_loss_clip": 0.01222443, + "auxiliary_loss_mlp": 0.01059323, + "balance_loss_clip": 1.07022381, + "balance_loss_mlp": 1.0344317, + "epoch": 0.11086394516924186, + "flos": 28658115763200.0, + "grad_norm": 2.0651565055548486, + "language_loss": 0.81764412, + "learning_rate": 3.93188525062415e-06, + "loss": 0.84046179, + "num_input_tokens_seen": 19582340, + "router_z_loss_clip": 1.5234375, + "router_z_loss_mlp": 0.24914551, + "step": 922, + "time_per_iteration": 2.8376145362854004 + }, + { + "auxiliary_loss_clip": 0.01229585, + "auxiliary_loss_mlp": 0.01071372, + "balance_loss_clip": 1.06831646, + "balance_loss_mlp": 1.0444901, + "epoch": 0.11098418805988096, + "flos": 24535247765760.0, + "grad_norm": 1.92820082505824, + "language_loss": 0.86084569, + "learning_rate": 3.931683540242418e-06, + "loss": 0.88385522, + "num_input_tokens_seen": 19603405, + "router_z_loss_clip": 1.61230469, + "router_z_loss_mlp": 0.2689209, + "step": 923, + "time_per_iteration": 2.842477321624756 + }, + { + "auxiliary_loss_clip": 0.01220097, + "auxiliary_loss_mlp": 0.01062172, + "balance_loss_clip": 1.06760538, + "balance_loss_mlp": 1.03489733, + "epoch": 0.11110443095052006, + "flos": 22960384888320.0, + "grad_norm": 2.8075519989655793, + "language_loss": 0.90792304, + "learning_rate": 3.9314815368258295e-06, + "loss": 0.93074578, + "num_input_tokens_seen": 19619885, + "router_z_loss_clip": 1.5234375, + "router_z_loss_mlp": 0.27294922, + "step": 924, + "time_per_iteration": 2.8189685344696045 + }, + { + "auxiliary_loss_clip": 0.01224369, + "auxiliary_loss_mlp": 0.01076066, + "balance_loss_clip": 1.07068312, + "balance_loss_mlp": 1.04968512, + "epoch": 0.11122467384115914, + "flos": 18950025265920.0, + "grad_norm": 1.6743530725888864, + "language_loss": 0.78706485, + "learning_rate": 3.9312792404050275e-06, + "loss": 0.8100692, + "num_input_tokens_seen": 19637940, + "router_z_loss_clip": 1.53808594, + "router_z_loss_mlp": 0.26379395, + "step": 925, + "time_per_iteration": 2.8125979900360107 + }, + { + "auxiliary_loss_clip": 0.01219027, + "auxiliary_loss_mlp": 0.01068578, + "balance_loss_clip": 1.06964815, + "balance_loss_mlp": 1.04387748, + "epoch": 0.11134491673179824, + "flos": 25082957324160.0, + "grad_norm": 1.905239726622071, + "language_loss": 0.77231908, + "learning_rate": 3.9310766510107e-06, + "loss": 0.7951951, + "num_input_tokens_seen": 19657115, + "router_z_loss_clip": 1.49316406, + "router_z_loss_mlp": 0.24694824, + "step": 926, + "time_per_iteration": 2.8554844856262207 + }, + { + "auxiliary_loss_clip": 0.01225156, + "auxiliary_loss_mlp": 0.01062446, + "balance_loss_clip": 1.07093418, + "balance_loss_mlp": 1.03695893, + "epoch": 0.11146515962243732, + "flos": 24499121662080.0, + "grad_norm": 1.8927503919101896, + "language_loss": 0.92270225, + "learning_rate": 3.9308737686735806e-06, + "loss": 0.94557834, + "num_input_tokens_seen": 19677075, + "router_z_loss_clip": 1.54101562, + "router_z_loss_mlp": 0.25488281, + "step": 927, + "time_per_iteration": 2.857689619064331 + }, + { + "auxiliary_loss_clip": 0.01219795, + "auxiliary_loss_mlp": 0.0104959, + "balance_loss_clip": 1.06688118, + "balance_loss_mlp": 1.02338827, + "epoch": 0.11158540251307641, + "flos": 22343763087360.0, + "grad_norm": 2.503479137042067, + "language_loss": 0.8276726, + "learning_rate": 3.9306705934244455e-06, + "loss": 0.85036647, + "num_input_tokens_seen": 19697155, + "router_z_loss_clip": 1.53027344, + "router_z_loss_mlp": 0.26208496, + "step": 928, + "time_per_iteration": 2.800297737121582 + }, + { + "auxiliary_loss_clip": 0.01221924, + "auxiliary_loss_mlp": 0.01071911, + "balance_loss_clip": 1.06935358, + "balance_loss_mlp": 1.04595888, + "epoch": 0.11170564540371551, + "flos": 19902304684800.0, + "grad_norm": 2.3948745338785797, + "language_loss": 0.88180041, + "learning_rate": 3.930467125294116e-06, + "loss": 0.90473878, + "num_input_tokens_seen": 19716705, + "router_z_loss_clip": 1.52734375, + "router_z_loss_mlp": 0.25952148, + "step": 929, + "time_per_iteration": 2.8371167182922363 + }, + { + "auxiliary_loss_clip": 0.01095555, + "auxiliary_loss_mlp": 0.01003097, + "balance_loss_clip": 1.02854395, + "balance_loss_mlp": 0.99956876, + "epoch": 0.1118258882943546, + "flos": 64586239499520.0, + "grad_norm": 0.9293157609345939, + "language_loss": 0.60400546, + "learning_rate": 3.930263364313458e-06, + "loss": 0.62499201, + "num_input_tokens_seen": 19767275, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.03540039, + "step": 930, + "time_per_iteration": 3.1484789848327637 + }, + { + "auxiliary_loss_clip": 0.01224557, + "auxiliary_loss_mlp": 0.01065233, + "balance_loss_clip": 1.06849778, + "balance_loss_mlp": 1.03859019, + "epoch": 0.11194613118499369, + "flos": 17201965985280.0, + "grad_norm": 2.0065447537958674, + "language_loss": 0.82910752, + "learning_rate": 3.930059310513384e-06, + "loss": 0.85200542, + "num_input_tokens_seen": 19786315, + "router_z_loss_clip": 1.56054688, + "router_z_loss_mlp": 0.26660156, + "step": 931, + "time_per_iteration": 2.8204455375671387 + }, + { + "auxiliary_loss_clip": 0.01225511, + "auxiliary_loss_mlp": 0.01066691, + "balance_loss_clip": 1.07137251, + "balance_loss_mlp": 1.04230058, + "epoch": 0.11206637407563277, + "flos": 31863465728640.0, + "grad_norm": 2.599857183236204, + "language_loss": 0.8399325, + "learning_rate": 3.929854963924846e-06, + "loss": 0.86285448, + "num_input_tokens_seen": 19806580, + "router_z_loss_clip": 1.54199219, + "router_z_loss_mlp": 0.24414062, + "step": 932, + "time_per_iteration": 2.9498207569122314 + }, + { + "auxiliary_loss_clip": 0.01217251, + "auxiliary_loss_mlp": 0.01051863, + "balance_loss_clip": 1.06621647, + "balance_loss_mlp": 1.02632856, + "epoch": 0.11218661696627187, + "flos": 21945621761280.0, + "grad_norm": 1.8598253094616857, + "language_loss": 0.77630305, + "learning_rate": 3.929650324578845e-06, + "loss": 0.79899418, + "num_input_tokens_seen": 19826045, + "router_z_loss_clip": 1.51171875, + "router_z_loss_mlp": 0.25537109, + "step": 933, + "time_per_iteration": 2.792572259902954 + }, + { + "auxiliary_loss_clip": 0.01236834, + "auxiliary_loss_mlp": 0.01062806, + "balance_loss_clip": 1.07294297, + "balance_loss_mlp": 1.03635347, + "epoch": 0.11230685985691095, + "flos": 25878198481920.0, + "grad_norm": 2.6321662330812514, + "language_loss": 0.82481259, + "learning_rate": 3.929445392506423e-06, + "loss": 0.84780908, + "num_input_tokens_seen": 19843985, + "router_z_loss_clip": 1.63867188, + "router_z_loss_mlp": 0.2644043, + "step": 934, + "time_per_iteration": 2.825624704360962 + }, + { + "auxiliary_loss_clip": 0.01223533, + "auxiliary_loss_mlp": 0.01061143, + "balance_loss_clip": 1.07320333, + "balance_loss_mlp": 1.03544188, + "epoch": 0.11242710274755005, + "flos": 22231506107520.0, + "grad_norm": 2.0202695357222433, + "language_loss": 0.75223464, + "learning_rate": 3.92924016773867e-06, + "loss": 0.7750814, + "num_input_tokens_seen": 19860480, + "router_z_loss_clip": 1.50195312, + "router_z_loss_mlp": 0.25720215, + "step": 935, + "time_per_iteration": 2.766517400741577 + }, + { + "auxiliary_loss_clip": 0.01220994, + "auxiliary_loss_mlp": 0.01056573, + "balance_loss_clip": 1.07010627, + "balance_loss_mlp": 1.03126526, + "epoch": 0.11254734563818915, + "flos": 17712184723200.0, + "grad_norm": 2.29921835763692, + "language_loss": 0.72985792, + "learning_rate": 3.9290346503067175e-06, + "loss": 0.75263357, + "num_input_tokens_seen": 19877145, + "router_z_loss_clip": 1.5078125, + "router_z_loss_mlp": 0.25317383, + "step": 936, + "time_per_iteration": 2.758547067642212 + }, + { + "auxiliary_loss_clip": 0.01219618, + "auxiliary_loss_mlp": 0.01065037, + "balance_loss_clip": 1.06656981, + "balance_loss_mlp": 1.03875113, + "epoch": 0.11266758852882823, + "flos": 54930397334400.0, + "grad_norm": 1.8284478671894613, + "language_loss": 0.78839433, + "learning_rate": 3.9288288402417415e-06, + "loss": 0.81124085, + "num_input_tokens_seen": 19903405, + "router_z_loss_clip": 1.52832031, + "router_z_loss_mlp": 0.26269531, + "step": 937, + "time_per_iteration": 3.184561252593994 + }, + { + "auxiliary_loss_clip": 0.01222072, + "auxiliary_loss_mlp": 0.01062746, + "balance_loss_clip": 1.06696951, + "balance_loss_mlp": 1.03603125, + "epoch": 0.11278783141946733, + "flos": 18878132194560.0, + "grad_norm": 3.4329295792615917, + "language_loss": 0.70426607, + "learning_rate": 3.928622737574964e-06, + "loss": 0.72711432, + "num_input_tokens_seen": 19918740, + "router_z_loss_clip": 1.55371094, + "router_z_loss_mlp": 0.26696777, + "step": 938, + "time_per_iteration": 2.8109304904937744 + }, + { + "auxiliary_loss_clip": 0.012203, + "auxiliary_loss_mlp": 0.01064544, + "balance_loss_clip": 1.06707132, + "balance_loss_mlp": 1.03745937, + "epoch": 0.11290807431010641, + "flos": 26469252777600.0, + "grad_norm": 1.8520056410313153, + "language_loss": 0.91003722, + "learning_rate": 3.928416342337652e-06, + "loss": 0.93288571, + "num_input_tokens_seen": 19938475, + "router_z_loss_clip": 1.53222656, + "router_z_loss_mlp": 0.27124023, + "step": 939, + "time_per_iteration": 2.81203293800354 + }, + { + "auxiliary_loss_clip": 0.01216495, + "auxiliary_loss_mlp": 0.01055414, + "balance_loss_clip": 1.06525135, + "balance_loss_mlp": 1.03130996, + "epoch": 0.1130283172007455, + "flos": 22710590732160.0, + "grad_norm": 1.8705345666888995, + "language_loss": 0.82749975, + "learning_rate": 3.928209654561113e-06, + "loss": 0.85021877, + "num_input_tokens_seen": 19959310, + "router_z_loss_clip": 1.51074219, + "router_z_loss_mlp": 0.24108887, + "step": 940, + "time_per_iteration": 2.811180830001831 + }, + { + "auxiliary_loss_clip": 0.01219129, + "auxiliary_loss_mlp": 0.01058506, + "balance_loss_clip": 1.06843042, + "balance_loss_mlp": 1.03354347, + "epoch": 0.1131485600913846, + "flos": 23219911630080.0, + "grad_norm": 2.269675905957549, + "language_loss": 0.81471145, + "learning_rate": 3.928002674276703e-06, + "loss": 0.83748782, + "num_input_tokens_seen": 19978700, + "router_z_loss_clip": 1.50585938, + "router_z_loss_mlp": 0.24963379, + "step": 941, + "time_per_iteration": 2.8867125511169434 + }, + { + "auxiliary_loss_clip": 0.01221347, + "auxiliary_loss_mlp": 0.01059596, + "balance_loss_clip": 1.06705403, + "balance_loss_mlp": 1.03219032, + "epoch": 0.11326880298202369, + "flos": 14064271286400.0, + "grad_norm": 2.2584152500970385, + "language_loss": 0.75367975, + "learning_rate": 3.92779540151582e-06, + "loss": 0.7764892, + "num_input_tokens_seen": 19995785, + "router_z_loss_clip": 1.54492188, + "router_z_loss_mlp": 0.27404785, + "step": 942, + "time_per_iteration": 4.663546800613403 + }, + { + "auxiliary_loss_clip": 0.0122057, + "auxiliary_loss_mlp": 0.01055976, + "balance_loss_clip": 1.06945765, + "balance_loss_mlp": 1.02957129, + "epoch": 0.11338904587266278, + "flos": 16325386479360.0, + "grad_norm": 1.7774894409343769, + "language_loss": 0.85777676, + "learning_rate": 3.927587836309907e-06, + "loss": 0.88054222, + "num_input_tokens_seen": 20013615, + "router_z_loss_clip": 1.51074219, + "router_z_loss_mlp": 0.26403809, + "step": 943, + "time_per_iteration": 3.736900568008423 + }, + { + "auxiliary_loss_clip": 0.01224772, + "auxiliary_loss_mlp": 0.01063203, + "balance_loss_clip": 1.06781566, + "balance_loss_mlp": 1.0352838, + "epoch": 0.11350928876330187, + "flos": 24426258923520.0, + "grad_norm": 2.165804242953634, + "language_loss": 0.78265822, + "learning_rate": 3.927379978690452e-06, + "loss": 0.80553794, + "num_input_tokens_seen": 20032880, + "router_z_loss_clip": 1.56835938, + "router_z_loss_mlp": 0.27966309, + "step": 944, + "time_per_iteration": 2.8539934158325195 + }, + { + "auxiliary_loss_clip": 0.01216994, + "auxiliary_loss_mlp": 0.01070798, + "balance_loss_clip": 1.06517744, + "balance_loss_mlp": 1.04361796, + "epoch": 0.11362953165394096, + "flos": 24497074586880.0, + "grad_norm": 4.360008633413507, + "language_loss": 0.87523425, + "learning_rate": 3.927171828688987e-06, + "loss": 0.89811218, + "num_input_tokens_seen": 20052405, + "router_z_loss_clip": 1.51660156, + "router_z_loss_mlp": 0.27172852, + "step": 945, + "time_per_iteration": 2.8767755031585693 + }, + { + "auxiliary_loss_clip": 0.01216581, + "auxiliary_loss_mlp": 0.01055045, + "balance_loss_clip": 1.06468606, + "balance_loss_mlp": 1.02940321, + "epoch": 0.11374977454458005, + "flos": 24060831909120.0, + "grad_norm": 2.2590411165131012, + "language_loss": 0.81899995, + "learning_rate": 3.926963386337088e-06, + "loss": 0.84171623, + "num_input_tokens_seen": 20070635, + "router_z_loss_clip": 1.515625, + "router_z_loss_mlp": 0.25646973, + "step": 946, + "time_per_iteration": 2.859081983566284 + }, + { + "auxiliary_loss_clip": 0.01224735, + "auxiliary_loss_mlp": 0.01063067, + "balance_loss_clip": 1.06839931, + "balance_loss_mlp": 1.03657854, + "epoch": 0.11387001743521914, + "flos": 39457638967680.0, + "grad_norm": 2.2773475352893473, + "language_loss": 0.70636237, + "learning_rate": 3.926754651666375e-06, + "loss": 0.72924036, + "num_input_tokens_seen": 20091195, + "router_z_loss_clip": 1.56347656, + "router_z_loss_mlp": 0.26513672, + "step": 947, + "time_per_iteration": 2.9745283126831055 + }, + { + "auxiliary_loss_clip": 0.01218553, + "auxiliary_loss_mlp": 0.0105803, + "balance_loss_clip": 1.06714976, + "balance_loss_mlp": 1.03170836, + "epoch": 0.11399026032585824, + "flos": 25082454533760.0, + "grad_norm": 2.734738256582372, + "language_loss": 0.78557992, + "learning_rate": 3.926545624708513e-06, + "loss": 0.80834574, + "num_input_tokens_seen": 20110435, + "router_z_loss_clip": 1.515625, + "router_z_loss_mlp": 0.2635498, + "step": 948, + "time_per_iteration": 2.8502659797668457 + }, + { + "auxiliary_loss_clip": 0.01219399, + "auxiliary_loss_mlp": 0.01066661, + "balance_loss_clip": 1.06786239, + "balance_loss_mlp": 1.0396961, + "epoch": 0.11411050321649732, + "flos": 17961835224960.0, + "grad_norm": 1.8930544870706736, + "language_loss": 0.85426128, + "learning_rate": 3.926336305495213e-06, + "loss": 0.87712193, + "num_input_tokens_seen": 20128995, + "router_z_loss_clip": 1.51367188, + "router_z_loss_mlp": 0.26989746, + "step": 949, + "time_per_iteration": 2.825044870376587 + }, + { + "auxiliary_loss_clip": 0.0122124, + "auxiliary_loss_mlp": 0.01065687, + "balance_loss_clip": 1.06865239, + "balance_loss_mlp": 1.04012799, + "epoch": 0.11423074610713642, + "flos": 22455409536000.0, + "grad_norm": 2.2987665291775534, + "language_loss": 0.88668537, + "learning_rate": 3.926126694058226e-06, + "loss": 0.90955466, + "num_input_tokens_seen": 20148145, + "router_z_loss_clip": 1.52636719, + "router_z_loss_mlp": 0.25549316, + "step": 950, + "time_per_iteration": 2.7658050060272217 + }, + { + "auxiliary_loss_clip": 0.01217649, + "auxiliary_loss_mlp": 0.01061011, + "balance_loss_clip": 1.06947899, + "balance_loss_mlp": 1.03553605, + "epoch": 0.1143509889977755, + "flos": 19717687756800.0, + "grad_norm": 1.7209551926150732, + "language_loss": 0.82138449, + "learning_rate": 3.92591679042935e-06, + "loss": 0.84417105, + "num_input_tokens_seen": 20168035, + "router_z_loss_clip": 1.47949219, + "router_z_loss_mlp": 0.25512695, + "step": 951, + "time_per_iteration": 2.813953161239624 + }, + { + "auxiliary_loss_clip": 0.01226497, + "auxiliary_loss_mlp": 0.01063424, + "balance_loss_clip": 1.07313585, + "balance_loss_mlp": 1.03726912, + "epoch": 0.1144712318884146, + "flos": 19822869757440.0, + "grad_norm": 1.6020778769398587, + "language_loss": 0.82289392, + "learning_rate": 3.92570659464043e-06, + "loss": 0.84579307, + "num_input_tokens_seen": 20186095, + "router_z_loss_clip": 1.53417969, + "router_z_loss_mlp": 0.26171875, + "step": 952, + "time_per_iteration": 2.8070225715637207 + }, + { + "auxiliary_loss_clip": 0.01218766, + "auxiliary_loss_mlp": 0.01059643, + "balance_loss_clip": 1.07011676, + "balance_loss_mlp": 1.03509784, + "epoch": 0.1145914747790537, + "flos": 14939198766720.0, + "grad_norm": 2.2442350214365616, + "language_loss": 0.79930949, + "learning_rate": 3.925496106723349e-06, + "loss": 0.82209349, + "num_input_tokens_seen": 20203535, + "router_z_loss_clip": 1.48632812, + "router_z_loss_mlp": 0.24572754, + "step": 953, + "time_per_iteration": 2.819443941116333 + }, + { + "auxiliary_loss_clip": 0.01218604, + "auxiliary_loss_mlp": 0.01065035, + "balance_loss_clip": 1.06951058, + "balance_loss_mlp": 1.0390588, + "epoch": 0.11471171766969278, + "flos": 19865029345920.0, + "grad_norm": 3.0156533175134657, + "language_loss": 0.8352505, + "learning_rate": 3.9252853267100405e-06, + "loss": 0.85808688, + "num_input_tokens_seen": 20222780, + "router_z_loss_clip": 1.49121094, + "router_z_loss_mlp": 0.25976562, + "step": 954, + "time_per_iteration": 2.8366737365722656 + }, + { + "auxiliary_loss_clip": 0.01222851, + "auxiliary_loss_mlp": 0.01061739, + "balance_loss_clip": 1.07071459, + "balance_loss_mlp": 1.03762269, + "epoch": 0.11483196056033187, + "flos": 22526476594560.0, + "grad_norm": 1.9364916497190168, + "language_loss": 0.83427954, + "learning_rate": 3.9250742546324786e-06, + "loss": 0.8571254, + "num_input_tokens_seen": 20243015, + "router_z_loss_clip": 1.52050781, + "router_z_loss_mlp": 0.24133301, + "step": 955, + "time_per_iteration": 2.8477373123168945 + }, + { + "auxiliary_loss_clip": 0.01224392, + "auxiliary_loss_mlp": 0.01064404, + "balance_loss_clip": 1.07020521, + "balance_loss_mlp": 1.03709304, + "epoch": 0.11495220345097096, + "flos": 28220292887040.0, + "grad_norm": 1.89948612321197, + "language_loss": 0.87116683, + "learning_rate": 3.924862890522683e-06, + "loss": 0.89405477, + "num_input_tokens_seen": 20263025, + "router_z_loss_clip": 1.54296875, + "router_z_loss_mlp": 0.27282715, + "step": 956, + "time_per_iteration": 2.809715747833252 + }, + { + "auxiliary_loss_clip": 0.0122638, + "auxiliary_loss_mlp": 0.01068887, + "balance_loss_clip": 1.07023776, + "balance_loss_mlp": 1.04056334, + "epoch": 0.11507244634161005, + "flos": 17492267704320.0, + "grad_norm": 2.27354585708013, + "language_loss": 0.86566198, + "learning_rate": 3.9246512344127174e-06, + "loss": 0.88861465, + "num_input_tokens_seen": 20280685, + "router_z_loss_clip": 1.56347656, + "router_z_loss_mlp": 0.28320312, + "step": 957, + "time_per_iteration": 2.8217620849609375 + }, + { + "auxiliary_loss_clip": 0.01216407, + "auxiliary_loss_mlp": 0.01064217, + "balance_loss_clip": 1.06419945, + "balance_loss_mlp": 1.03770447, + "epoch": 0.11519268923224914, + "flos": 22564937082240.0, + "grad_norm": 1.915286797912454, + "language_loss": 0.81928384, + "learning_rate": 3.9244392863346895e-06, + "loss": 0.84209007, + "num_input_tokens_seen": 20300090, + "router_z_loss_clip": 1.5234375, + "router_z_loss_mlp": 0.26550293, + "step": 958, + "time_per_iteration": 2.803431510925293 + }, + { + "auxiliary_loss_clip": 0.01223619, + "auxiliary_loss_mlp": 0.01063324, + "balance_loss_clip": 1.0695399, + "balance_loss_mlp": 1.03638268, + "epoch": 0.11531293212288823, + "flos": 16982839065600.0, + "grad_norm": 2.5387721340164737, + "language_loss": 0.92142487, + "learning_rate": 3.9242270463207524e-06, + "loss": 0.94429433, + "num_input_tokens_seen": 20318480, + "router_z_loss_clip": 1.54101562, + "router_z_loss_mlp": 0.26928711, + "step": 959, + "time_per_iteration": 2.7860798835754395 + }, + { + "auxiliary_loss_clip": 0.012196, + "auxiliary_loss_mlp": 0.01060762, + "balance_loss_clip": 1.06652308, + "balance_loss_mlp": 1.03434539, + "epoch": 0.11543317501352733, + "flos": 12422004537600.0, + "grad_norm": 2.62498820917161, + "language_loss": 0.85402763, + "learning_rate": 3.924014514403102e-06, + "loss": 0.87683123, + "num_input_tokens_seen": 20334635, + "router_z_loss_clip": 1.53320312, + "router_z_loss_mlp": 0.26428223, + "step": 960, + "time_per_iteration": 2.8475029468536377 + }, + { + "auxiliary_loss_clip": 0.0122086, + "auxiliary_loss_mlp": 0.01065292, + "balance_loss_clip": 1.06497073, + "balance_loss_mlp": 1.0384227, + "epoch": 0.11555341790416641, + "flos": 19821648695040.0, + "grad_norm": 2.111772186923692, + "language_loss": 0.91170037, + "learning_rate": 3.92380169061398e-06, + "loss": 0.93456191, + "num_input_tokens_seen": 20352415, + "router_z_loss_clip": 1.55859375, + "router_z_loss_mlp": 0.26867676, + "step": 961, + "time_per_iteration": 2.8558976650238037 + }, + { + "auxiliary_loss_clip": 0.01217822, + "auxiliary_loss_mlp": 0.01067247, + "balance_loss_clip": 1.06645131, + "balance_loss_mlp": 1.04038918, + "epoch": 0.11567366079480551, + "flos": 25738865625600.0, + "grad_norm": 2.2226841400798545, + "language_loss": 0.84255779, + "learning_rate": 3.9235885749856705e-06, + "loss": 0.86540854, + "num_input_tokens_seen": 20371095, + "router_z_loss_clip": 1.51367188, + "router_z_loss_mlp": 0.26867676, + "step": 962, + "time_per_iteration": 2.9512650966644287 + }, + { + "auxiliary_loss_clip": 0.01221941, + "auxiliary_loss_mlp": 0.01057016, + "balance_loss_clip": 1.06816638, + "balance_loss_mlp": 1.03193486, + "epoch": 0.1157939036854446, + "flos": 18223301301120.0, + "grad_norm": 9.593314064632775, + "language_loss": 0.8256712, + "learning_rate": 3.9233751675505035e-06, + "loss": 0.84846073, + "num_input_tokens_seen": 20389805, + "router_z_loss_clip": 1.5390625, + "router_z_loss_mlp": 0.25073242, + "step": 963, + "time_per_iteration": 2.844355344772339 + }, + { + "auxiliary_loss_clip": 0.0122595, + "auxiliary_loss_mlp": 0.01067028, + "balance_loss_clip": 1.07329249, + "balance_loss_mlp": 1.04199445, + "epoch": 0.11591414657608369, + "flos": 23073755189760.0, + "grad_norm": 1.9823226379173953, + "language_loss": 0.84879708, + "learning_rate": 3.923161468340853e-06, + "loss": 0.87172687, + "num_input_tokens_seen": 20409640, + "router_z_loss_clip": 1.52441406, + "router_z_loss_mlp": 0.25036621, + "step": 964, + "time_per_iteration": 2.857827663421631 + }, + { + "auxiliary_loss_clip": 0.01223099, + "auxiliary_loss_mlp": 0.01056397, + "balance_loss_clip": 1.06778777, + "balance_loss_mlp": 1.02823973, + "epoch": 0.11603438946672277, + "flos": 19461716461440.0, + "grad_norm": 1.8050827102853613, + "language_loss": 0.81454182, + "learning_rate": 3.9229474773891374e-06, + "loss": 0.83733678, + "num_input_tokens_seen": 20428180, + "router_z_loss_clip": 1.55175781, + "router_z_loss_mlp": 0.28137207, + "step": 965, + "time_per_iteration": 2.798386573791504 + }, + { + "auxiliary_loss_clip": 0.01219892, + "auxiliary_loss_mlp": 0.0106249, + "balance_loss_clip": 1.06564975, + "balance_loss_mlp": 1.03333175, + "epoch": 0.11615463235736187, + "flos": 26831986272000.0, + "grad_norm": 1.8756204048043825, + "language_loss": 0.83625317, + "learning_rate": 3.922733194727818e-06, + "loss": 0.85907698, + "num_input_tokens_seen": 20447975, + "router_z_loss_clip": 1.54492188, + "router_z_loss_mlp": 0.29174805, + "step": 966, + "time_per_iteration": 2.9126203060150146 + }, + { + "auxiliary_loss_clip": 0.01225043, + "auxiliary_loss_mlp": 0.01061335, + "balance_loss_clip": 1.07091749, + "balance_loss_mlp": 1.03373814, + "epoch": 0.11627487524800097, + "flos": 18580324533120.0, + "grad_norm": 2.1489893997693934, + "language_loss": 0.8770014, + "learning_rate": 3.922518620389402e-06, + "loss": 0.89986521, + "num_input_tokens_seen": 20464840, + "router_z_loss_clip": 1.54199219, + "router_z_loss_mlp": 0.27600098, + "step": 967, + "time_per_iteration": 2.8426826000213623 + }, + { + "auxiliary_loss_clip": 0.0121163, + "auxiliary_loss_mlp": 0.01060233, + "balance_loss_clip": 1.06781244, + "balance_loss_mlp": 1.0358901, + "epoch": 0.11639511813864005, + "flos": 18150474476160.0, + "grad_norm": 1.862732806998636, + "language_loss": 0.89779341, + "learning_rate": 3.922303754406439e-06, + "loss": 0.92051202, + "num_input_tokens_seen": 20482680, + "router_z_loss_clip": 1.43652344, + "router_z_loss_mlp": 0.2434082, + "step": 968, + "time_per_iteration": 2.8173508644104004 + }, + { + "auxiliary_loss_clip": 0.01217849, + "auxiliary_loss_mlp": 0.01062838, + "balance_loss_clip": 1.06883192, + "balance_loss_mlp": 1.03600419, + "epoch": 0.11651536102927915, + "flos": 20922023888640.0, + "grad_norm": 1.9084117350933278, + "language_loss": 0.79239082, + "learning_rate": 3.922088596811526e-06, + "loss": 0.81519771, + "num_input_tokens_seen": 20501810, + "router_z_loss_clip": 1.49023438, + "router_z_loss_mlp": 0.26843262, + "step": 969, + "time_per_iteration": 2.9288699626922607 + }, + { + "auxiliary_loss_clip": 0.01224418, + "auxiliary_loss_mlp": 0.01062625, + "balance_loss_clip": 1.07077146, + "balance_loss_mlp": 1.03678036, + "epoch": 0.11663560391991823, + "flos": 16508602776960.0, + "grad_norm": 2.22299456925315, + "language_loss": 0.87090713, + "learning_rate": 3.9218731476373e-06, + "loss": 0.89377749, + "num_input_tokens_seen": 20517995, + "router_z_loss_clip": 1.5390625, + "router_z_loss_mlp": 0.25866699, + "step": 970, + "time_per_iteration": 2.794959783554077 + }, + { + "auxiliary_loss_clip": 0.0122668, + "auxiliary_loss_mlp": 0.01065753, + "balance_loss_clip": 1.06925225, + "balance_loss_mlp": 1.03788221, + "epoch": 0.11675584681055733, + "flos": 19865029345920.0, + "grad_norm": 2.157315383755145, + "language_loss": 0.85068309, + "learning_rate": 3.9216574069164455e-06, + "loss": 0.87360752, + "num_input_tokens_seen": 20536970, + "router_z_loss_clip": 1.57421875, + "router_z_loss_mlp": 0.27868652, + "step": 971, + "time_per_iteration": 4.398606538772583 + }, + { + "auxiliary_loss_clip": 0.01216436, + "auxiliary_loss_mlp": 0.01060884, + "balance_loss_clip": 1.06716406, + "balance_loss_mlp": 1.03464651, + "epoch": 0.11687608970119642, + "flos": 21944364785280.0, + "grad_norm": 2.0543155922837766, + "language_loss": 0.8014375, + "learning_rate": 3.921441374681691e-06, + "loss": 0.8242107, + "num_input_tokens_seen": 20557030, + "router_z_loss_clip": 1.4921875, + "router_z_loss_mlp": 0.26245117, + "step": 972, + "time_per_iteration": 4.459091663360596 + }, + { + "auxiliary_loss_clip": 0.01212968, + "auxiliary_loss_mlp": 0.01069224, + "balance_loss_clip": 1.06450391, + "balance_loss_mlp": 1.0423305, + "epoch": 0.1169963325918355, + "flos": 24061155131520.0, + "grad_norm": 2.466991019995465, + "language_loss": 0.65037256, + "learning_rate": 3.921225050965808e-06, + "loss": 0.67319447, + "num_input_tokens_seen": 20576915, + "router_z_loss_clip": 1.484375, + "router_z_loss_mlp": 0.26879883, + "step": 973, + "time_per_iteration": 2.8390631675720215 + }, + { + "auxiliary_loss_clip": 0.01214506, + "auxiliary_loss_mlp": 0.01060238, + "balance_loss_clip": 1.06741917, + "balance_loss_mlp": 1.03426266, + "epoch": 0.1171165754824746, + "flos": 23368151059200.0, + "grad_norm": 2.018744000741985, + "language_loss": 0.75380528, + "learning_rate": 3.921008435801612e-06, + "loss": 0.77655268, + "num_input_tokens_seen": 20596000, + "router_z_loss_clip": 1.47167969, + "router_z_loss_mlp": 0.25976562, + "step": 974, + "time_per_iteration": 2.765216827392578 + }, + { + "auxiliary_loss_clip": 0.01220329, + "auxiliary_loss_mlp": 0.01061706, + "balance_loss_clip": 1.06704068, + "balance_loss_mlp": 1.0357188, + "epoch": 0.11723681837311369, + "flos": 18552243075840.0, + "grad_norm": 2.1617534261588265, + "language_loss": 0.76279545, + "learning_rate": 3.920791529221963e-06, + "loss": 0.78561586, + "num_input_tokens_seen": 20614675, + "router_z_loss_clip": 1.53222656, + "router_z_loss_mlp": 0.25964355, + "step": 975, + "time_per_iteration": 2.8023159503936768 + }, + { + "auxiliary_loss_clip": 0.01213095, + "auxiliary_loss_mlp": 0.01055812, + "balance_loss_clip": 1.06579256, + "balance_loss_mlp": 1.03044403, + "epoch": 0.11735706126375278, + "flos": 23550541344000.0, + "grad_norm": 1.8974087397332244, + "language_loss": 0.76202428, + "learning_rate": 3.920574331259768e-06, + "loss": 0.78471333, + "num_input_tokens_seen": 20635875, + "router_z_loss_clip": 1.47167969, + "router_z_loss_mlp": 0.25402832, + "step": 976, + "time_per_iteration": 2.826894521713257 + }, + { + "auxiliary_loss_clip": 0.01221577, + "auxiliary_loss_mlp": 0.01061189, + "balance_loss_clip": 1.07021534, + "balance_loss_mlp": 1.03535628, + "epoch": 0.11747730415439187, + "flos": 22381541216640.0, + "grad_norm": 2.2999596381102574, + "language_loss": 0.79735553, + "learning_rate": 3.9203568419479716e-06, + "loss": 0.82018322, + "num_input_tokens_seen": 20656430, + "router_z_loss_clip": 1.51171875, + "router_z_loss_mlp": 0.25830078, + "step": 977, + "time_per_iteration": 2.8572418689727783 + }, + { + "auxiliary_loss_clip": 0.0121963, + "auxiliary_loss_mlp": 0.01061713, + "balance_loss_clip": 1.06761956, + "balance_loss_mlp": 1.03670311, + "epoch": 0.11759754704503096, + "flos": 22200731130240.0, + "grad_norm": 2.1659536469657317, + "language_loss": 0.75156134, + "learning_rate": 3.92013906131957e-06, + "loss": 0.77437472, + "num_input_tokens_seen": 20675360, + "router_z_loss_clip": 1.52148438, + "router_z_loss_mlp": 0.25012207, + "step": 978, + "time_per_iteration": 2.815807819366455 + }, + { + "auxiliary_loss_clip": 0.0121659, + "auxiliary_loss_mlp": 0.01059698, + "balance_loss_clip": 1.06730223, + "balance_loss_mlp": 1.0337944, + "epoch": 0.11771778993567006, + "flos": 22309755886080.0, + "grad_norm": 1.7137262428919238, + "language_loss": 0.82781744, + "learning_rate": 3.9199209894076e-06, + "loss": 0.85058033, + "num_input_tokens_seen": 20695675, + "router_z_loss_clip": 1.49414062, + "router_z_loss_mlp": 0.2590332, + "step": 979, + "time_per_iteration": 2.8349950313568115 + }, + { + "auxiliary_loss_clip": 0.01225172, + "auxiliary_loss_mlp": 0.0106809, + "balance_loss_clip": 1.06805944, + "balance_loss_mlp": 1.04128003, + "epoch": 0.11783803282630914, + "flos": 21288169175040.0, + "grad_norm": 1.7879606898999554, + "language_loss": 0.89985347, + "learning_rate": 3.919702626245142e-06, + "loss": 0.92278606, + "num_input_tokens_seen": 20715330, + "router_z_loss_clip": 1.57324219, + "router_z_loss_mlp": 0.26831055, + "step": 980, + "time_per_iteration": 2.8405699729919434 + }, + { + "auxiliary_loss_clip": 0.01218826, + "auxiliary_loss_mlp": 0.01064754, + "balance_loss_clip": 1.06618202, + "balance_loss_mlp": 1.0387311, + "epoch": 0.11795827571694824, + "flos": 25371535190400.0, + "grad_norm": 1.9709712207943644, + "language_loss": 0.66305304, + "learning_rate": 3.919483971865322e-06, + "loss": 0.68588883, + "num_input_tokens_seen": 20735325, + "router_z_loss_clip": 1.52734375, + "router_z_loss_mlp": 0.26013184, + "step": 981, + "time_per_iteration": 2.8389649391174316 + }, + { + "auxiliary_loss_clip": 0.0121945, + "auxiliary_loss_mlp": 0.01060207, + "balance_loss_clip": 1.06736743, + "balance_loss_mlp": 1.03514874, + "epoch": 0.11807851860758732, + "flos": 23622218933760.0, + "grad_norm": 2.6014579636868804, + "language_loss": 0.87849563, + "learning_rate": 3.91926502630131e-06, + "loss": 0.9012922, + "num_input_tokens_seen": 20755940, + "router_z_loss_clip": 1.52148438, + "router_z_loss_mlp": 0.25036621, + "step": 982, + "time_per_iteration": 2.8639638423919678 + }, + { + "auxiliary_loss_clip": 0.0122314, + "auxiliary_loss_mlp": 0.01066499, + "balance_loss_clip": 1.07324612, + "balance_loss_mlp": 1.0414772, + "epoch": 0.11819876149822642, + "flos": 24972496024320.0, + "grad_norm": 1.7299155327954738, + "language_loss": 0.72015828, + "learning_rate": 3.91904578958632e-06, + "loss": 0.74305463, + "num_input_tokens_seen": 20775355, + "router_z_loss_clip": 1.49902344, + "router_z_loss_mlp": 0.25024414, + "step": 983, + "time_per_iteration": 2.842636823654175 + }, + { + "auxiliary_loss_clip": 0.01222313, + "auxiliary_loss_mlp": 0.0107369, + "balance_loss_clip": 1.06893623, + "balance_loss_mlp": 1.04472172, + "epoch": 0.11831900438886551, + "flos": 23003226835200.0, + "grad_norm": 2.227896905986338, + "language_loss": 0.84304249, + "learning_rate": 3.918826261753608e-06, + "loss": 0.8660025, + "num_input_tokens_seen": 20794935, + "router_z_loss_clip": 1.53320312, + "router_z_loss_mlp": 0.28930664, + "step": 984, + "time_per_iteration": 2.83119797706604 + }, + { + "auxiliary_loss_clip": 0.01216844, + "auxiliary_loss_mlp": 0.01063228, + "balance_loss_clip": 1.0656352, + "balance_loss_mlp": 1.03681159, + "epoch": 0.1184392472795046, + "flos": 27965147604480.0, + "grad_norm": 2.616446155148328, + "language_loss": 0.71312755, + "learning_rate": 3.918606442836478e-06, + "loss": 0.7359283, + "num_input_tokens_seen": 20817155, + "router_z_loss_clip": 1.51074219, + "router_z_loss_mlp": 0.2644043, + "step": 985, + "time_per_iteration": 2.8785030841827393 + }, + { + "auxiliary_loss_clip": 0.01212568, + "auxiliary_loss_mlp": 0.01060353, + "balance_loss_clip": 1.06929541, + "balance_loss_mlp": 1.0339241, + "epoch": 0.1185594901701437, + "flos": 19898497843200.0, + "grad_norm": 2.2426572530217905, + "language_loss": 0.77591693, + "learning_rate": 3.918386332868277e-06, + "loss": 0.79864609, + "num_input_tokens_seen": 20835125, + "router_z_loss_clip": 1.43066406, + "router_z_loss_mlp": 0.26428223, + "step": 986, + "time_per_iteration": 2.978776454925537 + }, + { + "auxiliary_loss_clip": 0.012219, + "auxiliary_loss_mlp": 0.01065788, + "balance_loss_clip": 1.06804943, + "balance_loss_mlp": 1.04124308, + "epoch": 0.11867973306078278, + "flos": 18912354877440.0, + "grad_norm": 1.8235881032621226, + "language_loss": 0.94651729, + "learning_rate": 3.918165931882394e-06, + "loss": 0.96939421, + "num_input_tokens_seen": 20853525, + "router_z_loss_clip": 1.5390625, + "router_z_loss_mlp": 0.24536133, + "step": 987, + "time_per_iteration": 2.769228935241699 + }, + { + "auxiliary_loss_clip": 0.0121614, + "auxiliary_loss_mlp": 0.01062533, + "balance_loss_clip": 1.06739211, + "balance_loss_mlp": 1.03699827, + "epoch": 0.11879997595142187, + "flos": 16982803152000.0, + "grad_norm": 2.319068192019609, + "language_loss": 0.75691777, + "learning_rate": 3.917945239912264e-06, + "loss": 0.77970445, + "num_input_tokens_seen": 20871000, + "router_z_loss_clip": 1.48730469, + "router_z_loss_mlp": 0.25537109, + "step": 988, + "time_per_iteration": 2.8169424533843994 + }, + { + "auxiliary_loss_clip": 0.01215346, + "auxiliary_loss_mlp": 0.01063184, + "balance_loss_clip": 1.06950474, + "balance_loss_mlp": 1.0379355, + "epoch": 0.11892021884206096, + "flos": 17530369056000.0, + "grad_norm": 3.0601948253625113, + "language_loss": 0.75456846, + "learning_rate": 3.917724256991367e-06, + "loss": 0.77735376, + "num_input_tokens_seen": 20889745, + "router_z_loss_clip": 1.45996094, + "router_z_loss_mlp": 0.25231934, + "step": 989, + "time_per_iteration": 2.8107588291168213 + }, + { + "auxiliary_loss_clip": 0.01219155, + "auxiliary_loss_mlp": 0.01057374, + "balance_loss_clip": 1.06933928, + "balance_loss_mlp": 1.03133881, + "epoch": 0.11904046173270005, + "flos": 30955895763840.0, + "grad_norm": 2.526732493567739, + "language_loss": 0.81540382, + "learning_rate": 3.9175029831532245e-06, + "loss": 0.8381691, + "num_input_tokens_seen": 20909260, + "router_z_loss_clip": 1.49707031, + "router_z_loss_mlp": 0.26062012, + "step": 990, + "time_per_iteration": 2.902053117752075 + }, + { + "auxiliary_loss_clip": 0.01215941, + "auxiliary_loss_mlp": 0.01059702, + "balance_loss_clip": 1.0668385, + "balance_loss_mlp": 1.03514504, + "epoch": 0.11916070462333915, + "flos": 20157234485760.0, + "grad_norm": 2.101353218451433, + "language_loss": 0.88848817, + "learning_rate": 3.917281418431404e-06, + "loss": 0.91124463, + "num_input_tokens_seen": 20928305, + "router_z_loss_clip": 1.49023438, + "router_z_loss_mlp": 0.2454834, + "step": 991, + "time_per_iteration": 2.886892318725586 + }, + { + "auxiliary_loss_clip": 0.01220827, + "auxiliary_loss_mlp": 0.01074499, + "balance_loss_clip": 1.07028735, + "balance_loss_mlp": 1.04675949, + "epoch": 0.11928094751397823, + "flos": 23551115961600.0, + "grad_norm": 2.234963735396069, + "language_loss": 0.77015626, + "learning_rate": 3.917059562859516e-06, + "loss": 0.79310948, + "num_input_tokens_seen": 20947630, + "router_z_loss_clip": 1.50585938, + "router_z_loss_mlp": 0.27758789, + "step": 992, + "time_per_iteration": 2.7847518920898438 + }, + { + "auxiliary_loss_clip": 0.01219876, + "auxiliary_loss_mlp": 0.01053555, + "balance_loss_clip": 1.07372308, + "balance_loss_mlp": 1.0295465, + "epoch": 0.11940119040461733, + "flos": 23908426502400.0, + "grad_norm": 2.0191427324303732, + "language_loss": 0.88739169, + "learning_rate": 3.916837416471218e-06, + "loss": 0.91012609, + "num_input_tokens_seen": 20964250, + "router_z_loss_clip": 1.46289062, + "router_z_loss_mlp": 0.24023438, + "step": 993, + "time_per_iteration": 2.799677610397339 + }, + { + "auxiliary_loss_clip": 0.0121893, + "auxiliary_loss_mlp": 0.01055951, + "balance_loss_clip": 1.06919682, + "balance_loss_mlp": 1.03059494, + "epoch": 0.11952143329525641, + "flos": 13844533835520.0, + "grad_norm": 2.4400357347292942, + "language_loss": 0.72339225, + "learning_rate": 3.916614979300207e-06, + "loss": 0.74614108, + "num_input_tokens_seen": 20979095, + "router_z_loss_clip": 1.49609375, + "router_z_loss_mlp": 0.25378418, + "step": 994, + "time_per_iteration": 2.8416907787323 + }, + { + "auxiliary_loss_clip": 0.01218534, + "auxiliary_loss_mlp": 0.01061096, + "balance_loss_clip": 1.06963968, + "balance_loss_mlp": 1.03582335, + "epoch": 0.11964167618589551, + "flos": 27015525792000.0, + "grad_norm": 1.5990627462306137, + "language_loss": 0.78410983, + "learning_rate": 3.9163922513802274e-06, + "loss": 0.80690616, + "num_input_tokens_seen": 21001430, + "router_z_loss_clip": 1.48828125, + "router_z_loss_mlp": 0.25280762, + "step": 995, + "time_per_iteration": 2.8569931983947754 + }, + { + "auxiliary_loss_clip": 0.01222245, + "auxiliary_loss_mlp": 0.01064562, + "balance_loss_clip": 1.06744277, + "balance_loss_mlp": 1.0378474, + "epoch": 0.1197619190765346, + "flos": 12567622273920.0, + "grad_norm": 3.0759758606045486, + "language_loss": 0.82422829, + "learning_rate": 3.916169232745067e-06, + "loss": 0.84709632, + "num_input_tokens_seen": 21019105, + "router_z_loss_clip": 1.546875, + "router_z_loss_mlp": 0.26672363, + "step": 996, + "time_per_iteration": 2.7686948776245117 + }, + { + "auxiliary_loss_clip": 0.0122815, + "auxiliary_loss_mlp": 0.01061578, + "balance_loss_clip": 1.06794763, + "balance_loss_mlp": 1.0344336, + "epoch": 0.11988216196717369, + "flos": 16909437623040.0, + "grad_norm": 2.6936225621719356, + "language_loss": 0.91790694, + "learning_rate": 3.915945923428559e-06, + "loss": 0.94080418, + "num_input_tokens_seen": 21035630, + "router_z_loss_clip": 1.6015625, + "router_z_loss_mlp": 0.27185059, + "step": 997, + "time_per_iteration": 2.8069472312927246 + }, + { + "auxiliary_loss_clip": 0.01220551, + "auxiliary_loss_mlp": 0.01054634, + "balance_loss_clip": 1.07025313, + "balance_loss_mlp": 1.02985048, + "epoch": 0.12000240485781279, + "flos": 16216577205120.0, + "grad_norm": 2.289581956332829, + "language_loss": 0.83290237, + "learning_rate": 3.915722323464577e-06, + "loss": 0.85565424, + "num_input_tokens_seen": 21054235, + "router_z_loss_clip": 1.50097656, + "router_z_loss_mlp": 0.24768066, + "step": 998, + "time_per_iteration": 2.7396044731140137 + }, + { + "auxiliary_loss_clip": 0.01218527, + "auxiliary_loss_mlp": 0.01066204, + "balance_loss_clip": 1.06734324, + "balance_loss_mlp": 1.04127741, + "epoch": 0.12012264774845187, + "flos": 49344887525760.0, + "grad_norm": 2.1682967024463276, + "language_loss": 0.70335716, + "learning_rate": 3.91549843288704e-06, + "loss": 0.72620451, + "num_input_tokens_seen": 21077915, + "router_z_loss_clip": 1.51171875, + "router_z_loss_mlp": 0.24926758, + "step": 999, + "time_per_iteration": 3.005383253097534 + }, + { + "auxiliary_loss_clip": 0.01223483, + "auxiliary_loss_mlp": 0.01055841, + "balance_loss_clip": 1.07405663, + "balance_loss_mlp": 1.03133154, + "epoch": 0.12024289063909097, + "flos": 26979435601920.0, + "grad_norm": 2.159255332199903, + "language_loss": 0.79453856, + "learning_rate": 3.915274251729916e-06, + "loss": 0.81733179, + "num_input_tokens_seen": 21099205, + "router_z_loss_clip": 1.49316406, + "router_z_loss_mlp": 0.24523926, + "step": 1000, + "time_per_iteration": 4.449343204498291 + }, + { + "auxiliary_loss_clip": 0.01222705, + "auxiliary_loss_mlp": 0.010601, + "balance_loss_clip": 1.07073247, + "balance_loss_mlp": 1.03413653, + "epoch": 0.12036313352973005, + "flos": 19537308633600.0, + "grad_norm": 2.358597159148701, + "language_loss": 0.90194023, + "learning_rate": 3.91504978002721e-06, + "loss": 0.92476833, + "num_input_tokens_seen": 21118260, + "router_z_loss_clip": 1.51953125, + "router_z_loss_mlp": 0.2598877, + "step": 1001, + "time_per_iteration": 3.6909329891204834 + }, + { + "auxiliary_loss_clip": 0.0121399, + "auxiliary_loss_mlp": 0.01065105, + "balance_loss_clip": 1.06885457, + "balance_loss_mlp": 1.0408814, + "epoch": 0.12048337642036915, + "flos": 17268256535040.0, + "grad_norm": 3.3144919894498868, + "language_loss": 0.76358294, + "learning_rate": 3.914825017812974e-06, + "loss": 0.78637391, + "num_input_tokens_seen": 21134910, + "router_z_loss_clip": 1.45117188, + "router_z_loss_mlp": 0.24230957, + "step": 1002, + "time_per_iteration": 3.2383904457092285 + }, + { + "auxiliary_loss_clip": 0.01213893, + "auxiliary_loss_mlp": 0.01064528, + "balance_loss_clip": 1.06690669, + "balance_loss_mlp": 1.03837347, + "epoch": 0.12060361931100824, + "flos": 22856962654080.0, + "grad_norm": 2.363161274949043, + "language_loss": 0.728019, + "learning_rate": 3.9145999651213065e-06, + "loss": 0.75080317, + "num_input_tokens_seen": 21154150, + "router_z_loss_clip": 1.46875, + "router_z_loss_mlp": 0.26147461, + "step": 1003, + "time_per_iteration": 2.920713186264038 + }, + { + "auxiliary_loss_clip": 0.01219979, + "auxiliary_loss_mlp": 0.01068157, + "balance_loss_clip": 1.06844151, + "balance_loss_mlp": 1.04153705, + "epoch": 0.12072386220164733, + "flos": 16726795943040.0, + "grad_norm": 2.497235214114884, + "language_loss": 0.88822001, + "learning_rate": 3.9143746219863465e-06, + "loss": 0.91110134, + "num_input_tokens_seen": 21171255, + "router_z_loss_clip": 1.51464844, + "router_z_loss_mlp": 0.26635742, + "step": 1004, + "time_per_iteration": 2.8654189109802246 + }, + { + "auxiliary_loss_clip": 0.01105659, + "auxiliary_loss_mlp": 0.01015796, + "balance_loss_clip": 1.0430057, + "balance_loss_mlp": 1.01186228, + "epoch": 0.12084410509228642, + "flos": 55144176105600.0, + "grad_norm": 0.9573341125040137, + "language_loss": 0.64727223, + "learning_rate": 3.914148988442278e-06, + "loss": 0.66848671, + "num_input_tokens_seen": 21227045, + "router_z_loss_clip": 0.625, + "router_z_loss_mlp": 0.03930664, + "step": 1005, + "time_per_iteration": 3.245840549468994 + }, + { + "auxiliary_loss_clip": 0.01221417, + "auxiliary_loss_mlp": 0.01062709, + "balance_loss_clip": 1.0686202, + "balance_loss_mlp": 1.03628016, + "epoch": 0.1209643479829255, + "flos": 26760236855040.0, + "grad_norm": 4.863288198246571, + "language_loss": 0.95788664, + "learning_rate": 3.91392306452333e-06, + "loss": 0.98072791, + "num_input_tokens_seen": 21244120, + "router_z_loss_clip": 1.52734375, + "router_z_loss_mlp": 0.26452637, + "step": 1006, + "time_per_iteration": 2.855140447616577 + }, + { + "auxiliary_loss_clip": 0.01222738, + "auxiliary_loss_mlp": 0.01056971, + "balance_loss_clip": 1.06968713, + "balance_loss_mlp": 1.02967179, + "epoch": 0.1210845908735646, + "flos": 11035026725760.0, + "grad_norm": 3.364955187225143, + "language_loss": 0.65910959, + "learning_rate": 3.913696850263774e-06, + "loss": 0.6819067, + "num_input_tokens_seen": 21258485, + "router_z_loss_clip": 1.53125, + "router_z_loss_mlp": 0.27307129, + "step": 1007, + "time_per_iteration": 2.8280210494995117 + }, + { + "auxiliary_loss_clip": 0.01221112, + "auxiliary_loss_mlp": 0.01067746, + "balance_loss_clip": 1.06836176, + "balance_loss_mlp": 1.04264045, + "epoch": 0.1212048337642037, + "flos": 20484631975680.0, + "grad_norm": 1.9294922790551272, + "language_loss": 0.79423171, + "learning_rate": 3.913470345697929e-06, + "loss": 0.81712031, + "num_input_tokens_seen": 21277115, + "router_z_loss_clip": 1.52734375, + "router_z_loss_mlp": 0.25109863, + "step": 1008, + "time_per_iteration": 2.8002562522888184 + }, + { + "auxiliary_loss_clip": 0.01219076, + "auxiliary_loss_mlp": 0.0106807, + "balance_loss_clip": 1.06892884, + "balance_loss_mlp": 1.0400914, + "epoch": 0.12132507665484278, + "flos": 22346061557760.0, + "grad_norm": 2.060877978927279, + "language_loss": 0.85611689, + "learning_rate": 3.913243550860153e-06, + "loss": 0.87898839, + "num_input_tokens_seen": 21294880, + "router_z_loss_clip": 1.50195312, + "router_z_loss_mlp": 0.27978516, + "step": 1009, + "time_per_iteration": 2.8407344818115234 + }, + { + "auxiliary_loss_clip": 0.01228102, + "auxiliary_loss_mlp": 0.01069601, + "balance_loss_clip": 1.0740844, + "balance_loss_mlp": 1.0429337, + "epoch": 0.12144531954548188, + "flos": 29314957818240.0, + "grad_norm": 1.9213458071918064, + "language_loss": 0.76069438, + "learning_rate": 3.913016465784852e-06, + "loss": 0.78367138, + "num_input_tokens_seen": 21315555, + "router_z_loss_clip": 1.5390625, + "router_z_loss_mlp": 0.26660156, + "step": 1010, + "time_per_iteration": 2.8354413509368896 + }, + { + "auxiliary_loss_clip": 0.0121781, + "auxiliary_loss_mlp": 0.01065694, + "balance_loss_clip": 1.06799042, + "balance_loss_mlp": 1.04161406, + "epoch": 0.12156556243612096, + "flos": 20485242506880.0, + "grad_norm": 4.497105788977735, + "language_loss": 0.72263288, + "learning_rate": 3.912789090506474e-06, + "loss": 0.74546796, + "num_input_tokens_seen": 21334815, + "router_z_loss_clip": 1.49707031, + "router_z_loss_mlp": 0.24108887, + "step": 1011, + "time_per_iteration": 2.906179428100586 + }, + { + "auxiliary_loss_clip": 0.01218988, + "auxiliary_loss_mlp": 0.01061167, + "balance_loss_clip": 1.06622505, + "balance_loss_mlp": 1.03608525, + "epoch": 0.12168580532676006, + "flos": 16472009796480.0, + "grad_norm": 2.371462287080131, + "language_loss": 0.72003448, + "learning_rate": 3.9125614250595114e-06, + "loss": 0.742836, + "num_input_tokens_seen": 21351025, + "router_z_loss_clip": 1.52539062, + "router_z_loss_mlp": 0.25097656, + "step": 1012, + "time_per_iteration": 2.775176763534546 + }, + { + "auxiliary_loss_clip": 0.01219588, + "auxiliary_loss_mlp": 0.01062168, + "balance_loss_clip": 1.06748724, + "balance_loss_mlp": 1.03604913, + "epoch": 0.12180604821739914, + "flos": 15341290588800.0, + "grad_norm": 2.7039434287879116, + "language_loss": 0.8893134, + "learning_rate": 3.912333469478502e-06, + "loss": 0.91213095, + "num_input_tokens_seen": 21368990, + "router_z_loss_clip": 1.52148438, + "router_z_loss_mlp": 0.26123047, + "step": 1013, + "time_per_iteration": 2.7954471111297607 + }, + { + "auxiliary_loss_clip": 0.01218482, + "auxiliary_loss_mlp": 0.01059204, + "balance_loss_clip": 1.06912386, + "balance_loss_mlp": 1.03476644, + "epoch": 0.12192629110803824, + "flos": 19318038059520.0, + "grad_norm": 2.0022685605867103, + "language_loss": 0.78339106, + "learning_rate": 3.912105223798025e-06, + "loss": 0.8061679, + "num_input_tokens_seen": 21388410, + "router_z_loss_clip": 1.49316406, + "router_z_loss_mlp": 0.2442627, + "step": 1014, + "time_per_iteration": 2.8436803817749023 + }, + { + "auxiliary_loss_clip": 0.01101917, + "auxiliary_loss_mlp": 0.0100439, + "balance_loss_clip": 1.03877497, + "balance_loss_mlp": 1.00057495, + "epoch": 0.12204653399867733, + "flos": 47725354085760.0, + "grad_norm": 1.0049909074068037, + "language_loss": 0.67694712, + "learning_rate": 3.9118766880527065e-06, + "loss": 0.69801021, + "num_input_tokens_seen": 21442845, + "router_z_loss_clip": 0.6328125, + "router_z_loss_mlp": 0.03808594, + "step": 1015, + "time_per_iteration": 3.272000551223755 + }, + { + "auxiliary_loss_clip": 0.01214371, + "auxiliary_loss_mlp": 0.01051966, + "balance_loss_clip": 1.06513572, + "balance_loss_mlp": 1.02680063, + "epoch": 0.12216677688931642, + "flos": 18221936584320.0, + "grad_norm": 1.6558756071370926, + "language_loss": 0.7357955, + "learning_rate": 3.9116478622772145e-06, + "loss": 0.75845885, + "num_input_tokens_seen": 21461420, + "router_z_loss_clip": 1.4921875, + "router_z_loss_mlp": 0.25195312, + "step": 1016, + "time_per_iteration": 2.7877743244171143 + }, + { + "auxiliary_loss_clip": 0.01215121, + "auxiliary_loss_mlp": 0.01053164, + "balance_loss_clip": 1.06675208, + "balance_loss_mlp": 1.02836847, + "epoch": 0.12228701977995551, + "flos": 27525636789120.0, + "grad_norm": 1.8061952986327046, + "language_loss": 0.88326108, + "learning_rate": 3.911418746506261e-06, + "loss": 0.90594393, + "num_input_tokens_seen": 21481550, + "router_z_loss_clip": 1.48339844, + "router_z_loss_mlp": 0.24780273, + "step": 1017, + "time_per_iteration": 2.8406572341918945 + }, + { + "auxiliary_loss_clip": 0.01227338, + "auxiliary_loss_mlp": 0.0105412, + "balance_loss_clip": 1.07436311, + "balance_loss_mlp": 1.0304811, + "epoch": 0.1224072626705946, + "flos": 21798136517760.0, + "grad_norm": 3.2205631154958465, + "language_loss": 0.7856853, + "learning_rate": 3.911189340774604e-06, + "loss": 0.80849987, + "num_input_tokens_seen": 21501680, + "router_z_loss_clip": 1.53027344, + "router_z_loss_mlp": 0.2364502, + "step": 1018, + "time_per_iteration": 2.8886513710021973 + }, + { + "auxiliary_loss_clip": 0.01217021, + "auxiliary_loss_mlp": 0.01061092, + "balance_loss_clip": 1.06922388, + "balance_loss_mlp": 1.03674936, + "epoch": 0.1225275055612337, + "flos": 20703758895360.0, + "grad_norm": 1.8448151170838532, + "language_loss": 0.79259419, + "learning_rate": 3.910959645117043e-06, + "loss": 0.81537533, + "num_input_tokens_seen": 21521015, + "router_z_loss_clip": 1.47753906, + "router_z_loss_mlp": 0.2434082, + "step": 1019, + "time_per_iteration": 2.8558530807495117 + }, + { + "auxiliary_loss_clip": 0.01100604, + "auxiliary_loss_mlp": 0.01002371, + "balance_loss_clip": 1.03761721, + "balance_loss_mlp": 0.99872285, + "epoch": 0.12264774845187278, + "flos": 57745294462080.0, + "grad_norm": 0.8161100147189675, + "language_loss": 0.56697524, + "learning_rate": 3.910729659568423e-06, + "loss": 0.58800501, + "num_input_tokens_seen": 21578200, + "router_z_loss_clip": 0.62890625, + "router_z_loss_mlp": 0.03637695, + "step": 1020, + "time_per_iteration": 3.3517277240753174 + }, + { + "auxiliary_loss_clip": 0.01216193, + "auxiliary_loss_mlp": 0.0106305, + "balance_loss_clip": 1.07098651, + "balance_loss_mlp": 1.03749204, + "epoch": 0.12276799134251187, + "flos": 26396282298240.0, + "grad_norm": 1.960857749713611, + "language_loss": 0.82342595, + "learning_rate": 3.9104993841636344e-06, + "loss": 0.84621835, + "num_input_tokens_seen": 21598770, + "router_z_loss_clip": 1.45214844, + "router_z_loss_mlp": 0.25598145, + "step": 1021, + "time_per_iteration": 2.883808135986328 + }, + { + "auxiliary_loss_clip": 0.01216988, + "auxiliary_loss_mlp": 0.01064622, + "balance_loss_clip": 1.07334089, + "balance_loss_mlp": 1.04037476, + "epoch": 0.12288823423315097, + "flos": 21064193919360.0, + "grad_norm": 1.8276773477821855, + "language_loss": 0.80751336, + "learning_rate": 3.910268818937608e-06, + "loss": 0.83032948, + "num_input_tokens_seen": 21616925, + "router_z_loss_clip": 1.43457031, + "router_z_loss_mlp": 0.24243164, + "step": 1022, + "time_per_iteration": 2.8570172786712646 + }, + { + "auxiliary_loss_clip": 0.01217111, + "auxiliary_loss_mlp": 0.01062691, + "balance_loss_clip": 1.06928062, + "balance_loss_mlp": 1.0382179, + "epoch": 0.12300847712379005, + "flos": 12312441077760.0, + "grad_norm": 2.4953420331003735, + "language_loss": 0.87490499, + "learning_rate": 3.9100379639253196e-06, + "loss": 0.89770299, + "num_input_tokens_seen": 21633645, + "router_z_loss_clip": 1.47753906, + "router_z_loss_mlp": 0.24499512, + "step": 1023, + "time_per_iteration": 2.8072402477264404 + }, + { + "auxiliary_loss_clip": 0.01218286, + "auxiliary_loss_mlp": 0.01057825, + "balance_loss_clip": 1.06921971, + "balance_loss_mlp": 1.03238535, + "epoch": 0.12312872001442915, + "flos": 16762239688320.0, + "grad_norm": 2.653076171829596, + "language_loss": 0.86555505, + "learning_rate": 3.909806819161791e-06, + "loss": 0.88831615, + "num_input_tokens_seen": 21649120, + "router_z_loss_clip": 1.49121094, + "router_z_loss_mlp": 0.2545166, + "step": 1024, + "time_per_iteration": 2.796818971633911 + }, + { + "auxiliary_loss_clip": 0.01221078, + "auxiliary_loss_mlp": 0.01061927, + "balance_loss_clip": 1.06973445, + "balance_loss_mlp": 1.03561735, + "epoch": 0.12324896290506823, + "flos": 18404937400320.0, + "grad_norm": 2.214109686516345, + "language_loss": 0.86317515, + "learning_rate": 3.909575384682086e-06, + "loss": 0.88600528, + "num_input_tokens_seen": 21668000, + "router_z_loss_clip": 1.51464844, + "router_z_loss_mlp": 0.26306152, + "step": 1025, + "time_per_iteration": 2.865687847137451 + }, + { + "auxiliary_loss_clip": 0.01218932, + "auxiliary_loss_mlp": 0.01062103, + "balance_loss_clip": 1.07138348, + "balance_loss_mlp": 1.04020405, + "epoch": 0.12336920579570733, + "flos": 18915407533440.0, + "grad_norm": 1.9072003726874878, + "language_loss": 0.69339895, + "learning_rate": 3.9093436605213144e-06, + "loss": 0.71620929, + "num_input_tokens_seen": 21688500, + "router_z_loss_clip": 1.4765625, + "router_z_loss_mlp": 0.21911621, + "step": 1026, + "time_per_iteration": 2.8962831497192383 + }, + { + "auxiliary_loss_clip": 0.0122033, + "auxiliary_loss_mlp": 0.01061494, + "balance_loss_clip": 1.06811607, + "balance_loss_mlp": 1.0353992, + "epoch": 0.12348944868634643, + "flos": 23878369797120.0, + "grad_norm": 2.1358387660656506, + "language_loss": 0.79292321, + "learning_rate": 3.909111646714627e-06, + "loss": 0.81574142, + "num_input_tokens_seen": 21709345, + "router_z_loss_clip": 1.52246094, + "router_z_loss_mlp": 0.26062012, + "step": 1027, + "time_per_iteration": 2.9315414428710938 + }, + { + "auxiliary_loss_clip": 0.01216591, + "auxiliary_loss_mlp": 0.0106658, + "balance_loss_clip": 1.06948102, + "balance_loss_mlp": 1.0421896, + "epoch": 0.12360969157698551, + "flos": 19026084314880.0, + "grad_norm": 2.2338241159203815, + "language_loss": 0.72524834, + "learning_rate": 3.9088793432972206e-06, + "loss": 0.74808002, + "num_input_tokens_seen": 21728165, + "router_z_loss_clip": 1.47363281, + "router_z_loss_mlp": 0.24389648, + "step": 1028, + "time_per_iteration": 3.6371145248413086 + }, + { + "auxiliary_loss_clip": 0.01220306, + "auxiliary_loss_mlp": 0.01070277, + "balance_loss_clip": 1.07137716, + "balance_loss_mlp": 1.04384804, + "epoch": 0.1237299344676246, + "flos": 13224607983360.0, + "grad_norm": 2.157900267318682, + "language_loss": 0.81928635, + "learning_rate": 3.908646750304336e-06, + "loss": 0.84219217, + "num_input_tokens_seen": 21745850, + "router_z_loss_clip": 1.48828125, + "router_z_loss_mlp": 0.2644043, + "step": 1029, + "time_per_iteration": 3.678088903427124 + }, + { + "auxiliary_loss_clip": 0.01223018, + "auxiliary_loss_mlp": 0.01065164, + "balance_loss_clip": 1.07335234, + "balance_loss_mlp": 1.03906941, + "epoch": 0.12385017735826369, + "flos": 20485673470080.0, + "grad_norm": 2.3731328024127705, + "language_loss": 0.8702811, + "learning_rate": 3.908413867771257e-06, + "loss": 0.89316291, + "num_input_tokens_seen": 21764760, + "router_z_loss_clip": 1.49804688, + "router_z_loss_mlp": 0.26123047, + "step": 1030, + "time_per_iteration": 3.669736862182617 + }, + { + "auxiliary_loss_clip": 0.01225721, + "auxiliary_loss_mlp": 0.0105964, + "balance_loss_clip": 1.07477081, + "balance_loss_mlp": 1.03617978, + "epoch": 0.12397042024890279, + "flos": 17347835116800.0, + "grad_norm": 1.8627089529617669, + "language_loss": 0.80475855, + "learning_rate": 3.908180695733311e-06, + "loss": 0.82761216, + "num_input_tokens_seen": 21784250, + "router_z_loss_clip": 1.50878906, + "router_z_loss_mlp": 0.23474121, + "step": 1031, + "time_per_iteration": 2.8037755489349365 + }, + { + "auxiliary_loss_clip": 0.01219731, + "auxiliary_loss_mlp": 0.01066841, + "balance_loss_clip": 1.06954312, + "balance_loss_mlp": 1.03839803, + "epoch": 0.12409066313954187, + "flos": 20412343854720.0, + "grad_norm": 1.8851530992818861, + "language_loss": 0.82794225, + "learning_rate": 3.907947234225871e-06, + "loss": 0.85080791, + "num_input_tokens_seen": 21803260, + "router_z_loss_clip": 1.50292969, + "router_z_loss_mlp": 0.2845459, + "step": 1032, + "time_per_iteration": 2.813004732131958 + }, + { + "auxiliary_loss_clip": 0.01220319, + "auxiliary_loss_mlp": 0.01063914, + "balance_loss_clip": 1.07163095, + "balance_loss_mlp": 1.03971434, + "epoch": 0.12421090603018096, + "flos": 20736688688640.0, + "grad_norm": 2.100772244819203, + "language_loss": 0.8710947, + "learning_rate": 3.907713483284352e-06, + "loss": 0.89393699, + "num_input_tokens_seen": 21822735, + "router_z_loss_clip": 1.48632812, + "router_z_loss_mlp": 0.2421875, + "step": 1033, + "time_per_iteration": 2.8347198963165283 + }, + { + "auxiliary_loss_clip": 0.01219985, + "auxiliary_loss_mlp": 0.01074515, + "balance_loss_clip": 1.0708766, + "balance_loss_mlp": 1.04818177, + "epoch": 0.12433114892082006, + "flos": 24498834353280.0, + "grad_norm": 2.2099615461719835, + "language_loss": 0.97297198, + "learning_rate": 3.907479442944216e-06, + "loss": 0.99591696, + "num_input_tokens_seen": 21841140, + "router_z_loss_clip": 1.49023438, + "router_z_loss_mlp": 0.26342773, + "step": 1034, + "time_per_iteration": 2.823256254196167 + }, + { + "auxiliary_loss_clip": 0.01220154, + "auxiliary_loss_mlp": 0.01051115, + "balance_loss_clip": 1.07414258, + "balance_loss_mlp": 1.02665305, + "epoch": 0.12445139181145914, + "flos": 19682315838720.0, + "grad_norm": 2.1988819295777753, + "language_loss": 0.92752939, + "learning_rate": 3.907245113240963e-06, + "loss": 0.95024204, + "num_input_tokens_seen": 21859260, + "router_z_loss_clip": 1.45898438, + "router_z_loss_mlp": 0.24462891, + "step": 1035, + "time_per_iteration": 2.8500864505767822 + }, + { + "auxiliary_loss_clip": 0.0122354, + "auxiliary_loss_mlp": 0.01067912, + "balance_loss_clip": 1.07087111, + "balance_loss_mlp": 1.04213929, + "epoch": 0.12457163470209824, + "flos": 46423087522560.0, + "grad_norm": 1.8613381130663016, + "language_loss": 0.73636699, + "learning_rate": 3.907010494210144e-06, + "loss": 0.75928152, + "num_input_tokens_seen": 21881920, + "router_z_loss_clip": 1.52636719, + "router_z_loss_mlp": 0.2578125, + "step": 1036, + "time_per_iteration": 2.98892879486084 + }, + { + "auxiliary_loss_clip": 0.01220889, + "auxiliary_loss_mlp": 0.01061333, + "balance_loss_clip": 1.07214868, + "balance_loss_mlp": 1.03557205, + "epoch": 0.12469187759273732, + "flos": 20376289578240.0, + "grad_norm": 2.0044813719038705, + "language_loss": 0.92218989, + "learning_rate": 3.9067755858873495e-06, + "loss": 0.94501209, + "num_input_tokens_seen": 21898720, + "router_z_loss_clip": 1.48632812, + "router_z_loss_mlp": 0.25756836, + "step": 1037, + "time_per_iteration": 2.8198442459106445 + }, + { + "auxiliary_loss_clip": 0.01102196, + "auxiliary_loss_mlp": 0.01009444, + "balance_loss_clip": 1.0388577, + "balance_loss_mlp": 1.0056771, + "epoch": 0.12481212048337642, + "flos": 69224641447680.0, + "grad_norm": 0.8632933171251542, + "language_loss": 0.62839639, + "learning_rate": 3.906540388308214e-06, + "loss": 0.64951277, + "num_input_tokens_seen": 21958305, + "router_z_loss_clip": 0.6328125, + "router_z_loss_mlp": 0.03759766, + "step": 1038, + "time_per_iteration": 3.381661891937256 + }, + { + "auxiliary_loss_clip": 0.01220796, + "auxiliary_loss_mlp": 0.01059194, + "balance_loss_clip": 1.07118678, + "balance_loss_mlp": 1.03396964, + "epoch": 0.12493236337401552, + "flos": 18223696350720.0, + "grad_norm": 1.6778094599814581, + "language_loss": 0.81473035, + "learning_rate": 3.906304901508417e-06, + "loss": 0.83753026, + "num_input_tokens_seen": 21977205, + "router_z_loss_clip": 1.49609375, + "router_z_loss_mlp": 0.25244141, + "step": 1039, + "time_per_iteration": 2.9607813358306885 + }, + { + "auxiliary_loss_clip": 0.01221718, + "auxiliary_loss_mlp": 0.01054338, + "balance_loss_clip": 1.07232499, + "balance_loss_mlp": 1.02923262, + "epoch": 0.12505260626465461, + "flos": 30044375303040.0, + "grad_norm": 2.3662742497285394, + "language_loss": 0.75891101, + "learning_rate": 3.9060691255236835e-06, + "loss": 0.78167152, + "num_input_tokens_seen": 21997770, + "router_z_loss_clip": 1.49511719, + "router_z_loss_mlp": 0.2512207, + "step": 1040, + "time_per_iteration": 2.9255752563476562 + }, + { + "auxiliary_loss_clip": 0.01219193, + "auxiliary_loss_mlp": 0.01064436, + "balance_loss_clip": 1.07080698, + "balance_loss_mlp": 1.0380429, + "epoch": 0.1251728491552937, + "flos": 24433980347520.0, + "grad_norm": 2.2624958634751327, + "language_loss": 0.80574834, + "learning_rate": 3.905833060389778e-06, + "loss": 0.82858467, + "num_input_tokens_seen": 22021890, + "router_z_loss_clip": 1.484375, + "router_z_loss_mlp": 0.26367188, + "step": 1041, + "time_per_iteration": 2.9350719451904297 + }, + { + "auxiliary_loss_clip": 0.01222198, + "auxiliary_loss_mlp": 0.0106314, + "balance_loss_clip": 1.07201207, + "balance_loss_mlp": 1.03696156, + "epoch": 0.12529309204593278, + "flos": 27119809952640.0, + "grad_norm": 2.2796027433174024, + "language_loss": 0.78532469, + "learning_rate": 3.905596706142513e-06, + "loss": 0.80817807, + "num_input_tokens_seen": 22043300, + "router_z_loss_clip": 1.50292969, + "router_z_loss_mlp": 0.26208496, + "step": 1042, + "time_per_iteration": 2.9037299156188965 + }, + { + "auxiliary_loss_clip": 0.01223728, + "auxiliary_loss_mlp": 0.0105848, + "balance_loss_clip": 1.07269549, + "balance_loss_mlp": 1.03299308, + "epoch": 0.12541333493657186, + "flos": 30774151923840.0, + "grad_norm": 5.344051701961909, + "language_loss": 0.85698014, + "learning_rate": 3.9053600628177435e-06, + "loss": 0.87980223, + "num_input_tokens_seen": 22062910, + "router_z_loss_clip": 1.51171875, + "router_z_loss_mlp": 0.25488281, + "step": 1043, + "time_per_iteration": 2.905543327331543 + }, + { + "auxiliary_loss_clip": 0.01214533, + "auxiliary_loss_mlp": 0.01055906, + "balance_loss_clip": 1.06811929, + "balance_loss_mlp": 1.0300976, + "epoch": 0.12553357782721097, + "flos": 23659566099840.0, + "grad_norm": 2.018246993033352, + "language_loss": 0.8524437, + "learning_rate": 3.905123130451367e-06, + "loss": 0.87514806, + "num_input_tokens_seen": 22084010, + "router_z_loss_clip": 1.46289062, + "router_z_loss_mlp": 0.25805664, + "step": 1044, + "time_per_iteration": 2.946794033050537 + }, + { + "auxiliary_loss_clip": 0.01221032, + "auxiliary_loss_mlp": 0.01066206, + "balance_loss_clip": 1.07067621, + "balance_loss_mlp": 1.0397892, + "epoch": 0.12565382071785006, + "flos": 24863758577280.0, + "grad_norm": 1.728808020448559, + "language_loss": 0.79412401, + "learning_rate": 3.904885909079326e-06, + "loss": 0.8169964, + "num_input_tokens_seen": 22102795, + "router_z_loss_clip": 1.50195312, + "router_z_loss_mlp": 0.26428223, + "step": 1045, + "time_per_iteration": 2.9512341022491455 + }, + { + "auxiliary_loss_clip": 0.01223907, + "auxiliary_loss_mlp": 0.01051994, + "balance_loss_clip": 1.07029319, + "balance_loss_mlp": 1.0258038, + "epoch": 0.12577406360848914, + "flos": 21360780518400.0, + "grad_norm": 2.315374913664564, + "language_loss": 0.77948844, + "learning_rate": 3.904648398737607e-06, + "loss": 0.80224746, + "num_input_tokens_seen": 22121360, + "router_z_loss_clip": 1.53613281, + "router_z_loss_mlp": 0.26208496, + "step": 1046, + "time_per_iteration": 3.1897246837615967 + }, + { + "auxiliary_loss_clip": 0.01222838, + "auxiliary_loss_mlp": 0.01052633, + "balance_loss_clip": 1.07283354, + "balance_loss_mlp": 1.02696729, + "epoch": 0.12589430649912825, + "flos": 36138056774400.0, + "grad_norm": 1.9049861414714468, + "language_loss": 0.78356504, + "learning_rate": 3.9044105994622406e-06, + "loss": 0.80631971, + "num_input_tokens_seen": 22142505, + "router_z_loss_clip": 1.5, + "router_z_loss_mlp": 0.25646973, + "step": 1047, + "time_per_iteration": 2.9216604232788086 + }, + { + "auxiliary_loss_clip": 0.01220498, + "auxiliary_loss_mlp": 0.01058622, + "balance_loss_clip": 1.06920981, + "balance_loss_mlp": 1.03343308, + "epoch": 0.12601454938976733, + "flos": 25337671643520.0, + "grad_norm": 2.12080931607647, + "language_loss": 0.81799293, + "learning_rate": 3.9041725112893005e-06, + "loss": 0.84078407, + "num_input_tokens_seen": 22163730, + "router_z_loss_clip": 1.51074219, + "router_z_loss_mlp": 0.25183105, + "step": 1048, + "time_per_iteration": 2.836897373199463 + }, + { + "auxiliary_loss_clip": 0.01223474, + "auxiliary_loss_mlp": 0.01055291, + "balance_loss_clip": 1.07235909, + "balance_loss_mlp": 1.0278132, + "epoch": 0.12613479228040642, + "flos": 15560094286080.0, + "grad_norm": 1.7862215371139805, + "language_loss": 0.75161612, + "learning_rate": 3.903934134254904e-06, + "loss": 0.77440369, + "num_input_tokens_seen": 22181520, + "router_z_loss_clip": 1.50976562, + "router_z_loss_mlp": 0.2746582, + "step": 1049, + "time_per_iteration": 2.8417978286743164 + }, + { + "auxiliary_loss_clip": 0.01222842, + "auxiliary_loss_mlp": 0.01062084, + "balance_loss_clip": 1.07171619, + "balance_loss_mlp": 1.0354166, + "epoch": 0.1262550351710455, + "flos": 21470595373440.0, + "grad_norm": 2.2375555109052114, + "language_loss": 0.85292649, + "learning_rate": 3.903695468395213e-06, + "loss": 0.87577569, + "num_input_tokens_seen": 22199390, + "router_z_loss_clip": 1.51074219, + "router_z_loss_mlp": 0.26660156, + "step": 1050, + "time_per_iteration": 2.950352668762207 + }, + { + "auxiliary_loss_clip": 0.01219, + "auxiliary_loss_mlp": 0.01072979, + "balance_loss_clip": 1.07041299, + "balance_loss_mlp": 1.04651427, + "epoch": 0.1263752780616846, + "flos": 31576719456000.0, + "grad_norm": 3.3846265881315265, + "language_loss": 0.55970889, + "learning_rate": 3.903456513746434e-06, + "loss": 0.58262867, + "num_input_tokens_seen": 22220365, + "router_z_loss_clip": 1.48535156, + "router_z_loss_mlp": 0.26489258, + "step": 1051, + "time_per_iteration": 2.9023423194885254 + }, + { + "auxiliary_loss_clip": 0.01219167, + "auxiliary_loss_mlp": 0.01059077, + "balance_loss_clip": 1.07128763, + "balance_loss_mlp": 1.03554535, + "epoch": 0.1264955209523237, + "flos": 28768217927040.0, + "grad_norm": 1.7951918553339112, + "language_loss": 0.87349766, + "learning_rate": 3.903217270344815e-06, + "loss": 0.89628005, + "num_input_tokens_seen": 22240615, + "router_z_loss_clip": 1.48046875, + "router_z_loss_mlp": 0.23535156, + "step": 1052, + "time_per_iteration": 2.850444793701172 + }, + { + "auxiliary_loss_clip": 0.01221001, + "auxiliary_loss_mlp": 0.01064337, + "balance_loss_clip": 1.07268834, + "balance_loss_mlp": 1.03690696, + "epoch": 0.12661576384296278, + "flos": 29241125412480.0, + "grad_norm": 1.7720447328620947, + "language_loss": 0.82160735, + "learning_rate": 3.902977738226648e-06, + "loss": 0.84446073, + "num_input_tokens_seen": 22261350, + "router_z_loss_clip": 1.48339844, + "router_z_loss_mlp": 0.27429199, + "step": 1053, + "time_per_iteration": 2.9066736698150635 + }, + { + "auxiliary_loss_clip": 0.01224335, + "auxiliary_loss_mlp": 0.01063832, + "balance_loss_clip": 1.06983376, + "balance_loss_mlp": 1.03798723, + "epoch": 0.12673600673360189, + "flos": 20850346298880.0, + "grad_norm": 1.8774942946070152, + "language_loss": 0.91378188, + "learning_rate": 3.902737917428273e-06, + "loss": 0.93666351, + "num_input_tokens_seen": 22279515, + "router_z_loss_clip": 1.54589844, + "router_z_loss_mlp": 0.25817871, + "step": 1054, + "time_per_iteration": 2.84057879447937 + }, + { + "auxiliary_loss_clip": 0.01223362, + "auxiliary_loss_mlp": 0.01069074, + "balance_loss_clip": 1.07168722, + "balance_loss_mlp": 1.04289615, + "epoch": 0.12685624962424097, + "flos": 25263695583360.0, + "grad_norm": 1.6088471939132385, + "language_loss": 0.840321, + "learning_rate": 3.902497807986068e-06, + "loss": 0.86324537, + "num_input_tokens_seen": 22299535, + "router_z_loss_clip": 1.51855469, + "router_z_loss_mlp": 0.26208496, + "step": 1055, + "time_per_iteration": 2.846930742263794 + }, + { + "auxiliary_loss_clip": 0.01222802, + "auxiliary_loss_mlp": 0.01057085, + "balance_loss_clip": 1.0714848, + "balance_loss_mlp": 1.03071606, + "epoch": 0.12697649251488005, + "flos": 27527109246720.0, + "grad_norm": 1.6527820658552264, + "language_loss": 0.8381952, + "learning_rate": 3.902257409936458e-06, + "loss": 0.8609941, + "num_input_tokens_seen": 22320300, + "router_z_loss_clip": 1.51464844, + "router_z_loss_mlp": 0.26367188, + "step": 1056, + "time_per_iteration": 2.919553279876709 + }, + { + "auxiliary_loss_clip": 0.01218452, + "auxiliary_loss_mlp": 0.0105058, + "balance_loss_clip": 1.07186866, + "balance_loss_mlp": 1.02604723, + "epoch": 0.12709673540551916, + "flos": 21251863503360.0, + "grad_norm": 1.8280086672701585, + "language_loss": 0.83949184, + "learning_rate": 3.902016723315912e-06, + "loss": 0.86218214, + "num_input_tokens_seen": 22338240, + "router_z_loss_clip": 1.46386719, + "router_z_loss_mlp": 0.24560547, + "step": 1057, + "time_per_iteration": 3.714864730834961 + }, + { + "auxiliary_loss_clip": 0.0121937, + "auxiliary_loss_mlp": 0.0106009, + "balance_loss_clip": 1.07107675, + "balance_loss_mlp": 1.0354135, + "epoch": 0.12721697829615825, + "flos": 25337707557120.0, + "grad_norm": 2.3640052746499634, + "language_loss": 0.69665247, + "learning_rate": 3.901775748160941e-06, + "loss": 0.71944708, + "num_input_tokens_seen": 22357420, + "router_z_loss_clip": 1.48242188, + "router_z_loss_mlp": 0.24694824, + "step": 1058, + "time_per_iteration": 4.544795513153076 + }, + { + "auxiliary_loss_clip": 0.01108293, + "auxiliary_loss_mlp": 0.01004984, + "balance_loss_clip": 1.04504609, + "balance_loss_mlp": 1.00126481, + "epoch": 0.12733722118679733, + "flos": 61943287754880.0, + "grad_norm": 0.8007281388667271, + "language_loss": 0.60878092, + "learning_rate": 3.901534484508101e-06, + "loss": 0.62991369, + "num_input_tokens_seen": 22420095, + "router_z_loss_clip": 0.6328125, + "router_z_loss_mlp": 0.03710938, + "step": 1059, + "time_per_iteration": 4.143107652664185 + }, + { + "auxiliary_loss_clip": 0.01218474, + "auxiliary_loss_mlp": 0.01052182, + "balance_loss_clip": 1.06902957, + "balance_loss_mlp": 1.02785146, + "epoch": 0.1274574640774364, + "flos": 26976742081920.0, + "grad_norm": 1.8775796615624625, + "language_loss": 0.74440575, + "learning_rate": 3.901292932393991e-06, + "loss": 0.76711237, + "num_input_tokens_seen": 22438975, + "router_z_loss_clip": 1.49414062, + "router_z_loss_mlp": 0.2434082, + "step": 1060, + "time_per_iteration": 2.9150540828704834 + }, + { + "auxiliary_loss_clip": 0.01223248, + "auxiliary_loss_mlp": 0.01061764, + "balance_loss_clip": 1.0737071, + "balance_loss_mlp": 1.03642058, + "epoch": 0.12757770696807552, + "flos": 22236318529920.0, + "grad_norm": 3.4131144942480804, + "language_loss": 0.84730136, + "learning_rate": 3.9010510918552555e-06, + "loss": 0.87015146, + "num_input_tokens_seen": 22458050, + "router_z_loss_clip": 1.49609375, + "router_z_loss_mlp": 0.2532959, + "step": 1061, + "time_per_iteration": 2.954798698425293 + }, + { + "auxiliary_loss_clip": 0.01225034, + "auxiliary_loss_mlp": 0.01060918, + "balance_loss_clip": 1.07309735, + "balance_loss_mlp": 1.03603888, + "epoch": 0.1276979498587146, + "flos": 28547905858560.0, + "grad_norm": 2.554500507885075, + "language_loss": 0.74385899, + "learning_rate": 3.900808962928581e-06, + "loss": 0.76671851, + "num_input_tokens_seen": 22475665, + "router_z_loss_clip": 1.51953125, + "router_z_loss_mlp": 0.24902344, + "step": 1062, + "time_per_iteration": 2.94381046295166 + }, + { + "auxiliary_loss_clip": 0.01221515, + "auxiliary_loss_mlp": 0.01077585, + "balance_loss_clip": 1.0751617, + "balance_loss_mlp": 1.05277801, + "epoch": 0.1278181927493537, + "flos": 17420338719360.0, + "grad_norm": 2.30847945429912, + "language_loss": 0.89456069, + "learning_rate": 3.900566545650698e-06, + "loss": 0.91755164, + "num_input_tokens_seen": 22493335, + "router_z_loss_clip": 1.46484375, + "router_z_loss_mlp": 0.24780273, + "step": 1063, + "time_per_iteration": 2.8805782794952393 + }, + { + "auxiliary_loss_clip": 0.01222126, + "auxiliary_loss_mlp": 0.0106424, + "balance_loss_clip": 1.07352328, + "balance_loss_mlp": 1.03701234, + "epoch": 0.1279384356399928, + "flos": 21138636856320.0, + "grad_norm": 2.1986827560013724, + "language_loss": 0.81600225, + "learning_rate": 3.900323840058381e-06, + "loss": 0.83886588, + "num_input_tokens_seen": 22511045, + "router_z_loss_clip": 1.48535156, + "router_z_loss_mlp": 0.27233887, + "step": 1064, + "time_per_iteration": 2.873199939727783 + }, + { + "auxiliary_loss_clip": 0.01220533, + "auxiliary_loss_mlp": 0.01071317, + "balance_loss_clip": 1.07250166, + "balance_loss_mlp": 1.0449723, + "epoch": 0.12805867853063188, + "flos": 26576733248640.0, + "grad_norm": 1.9240415200091066, + "language_loss": 0.81913924, + "learning_rate": 3.900080846188449e-06, + "loss": 0.8420577, + "num_input_tokens_seen": 22529635, + "router_z_loss_clip": 1.47949219, + "router_z_loss_mlp": 0.26367188, + "step": 1065, + "time_per_iteration": 2.916172981262207 + }, + { + "auxiliary_loss_clip": 0.01224059, + "auxiliary_loss_mlp": 0.01059101, + "balance_loss_clip": 1.07412052, + "balance_loss_mlp": 1.03350711, + "epoch": 0.12817892142127096, + "flos": 16436206915200.0, + "grad_norm": 1.831660315217107, + "language_loss": 0.81286663, + "learning_rate": 3.8998375640777625e-06, + "loss": 0.83569825, + "num_input_tokens_seen": 22547505, + "router_z_loss_clip": 1.50097656, + "router_z_loss_mlp": 0.25561523, + "step": 1066, + "time_per_iteration": 2.909848928451538 + }, + { + "auxiliary_loss_clip": 0.01101581, + "auxiliary_loss_mlp": 0.01016395, + "balance_loss_clip": 1.03983688, + "balance_loss_mlp": 1.01286685, + "epoch": 0.12829916431191005, + "flos": 60757049099520.0, + "grad_norm": 0.7025787707106084, + "language_loss": 0.52605927, + "learning_rate": 3.899593993763229e-06, + "loss": 0.54723907, + "num_input_tokens_seen": 22608465, + "router_z_loss_clip": 0.6171875, + "router_z_loss_mlp": 0.03540039, + "step": 1067, + "time_per_iteration": 3.242218017578125 + }, + { + "auxiliary_loss_clip": 0.01220253, + "auxiliary_loss_mlp": 0.01061772, + "balance_loss_clip": 1.06978667, + "balance_loss_mlp": 1.03593898, + "epoch": 0.12841940720254916, + "flos": 29786895636480.0, + "grad_norm": 2.0598910776575146, + "language_loss": 0.81518227, + "learning_rate": 3.899350135281796e-06, + "loss": 0.8380025, + "num_input_tokens_seen": 22629465, + "router_z_loss_clip": 1.50488281, + "router_z_loss_mlp": 0.25842285, + "step": 1068, + "time_per_iteration": 2.847844362258911 + }, + { + "auxiliary_loss_clip": 0.01220312, + "auxiliary_loss_mlp": 0.01068318, + "balance_loss_clip": 1.07423067, + "balance_loss_mlp": 1.04445231, + "epoch": 0.12853965009318824, + "flos": 25951851319680.0, + "grad_norm": 1.9410878457784726, + "language_loss": 0.79881454, + "learning_rate": 3.8991059886704585e-06, + "loss": 0.82170081, + "num_input_tokens_seen": 22648970, + "router_z_loss_clip": 1.46191406, + "router_z_loss_mlp": 0.23864746, + "step": 1069, + "time_per_iteration": 2.8462870121002197 + }, + { + "auxiliary_loss_clip": 0.01226395, + "auxiliary_loss_mlp": 0.01083568, + "balance_loss_clip": 1.07461739, + "balance_loss_mlp": 1.05736542, + "epoch": 0.12865989298382732, + "flos": 30846871008000.0, + "grad_norm": 2.971477550607749, + "language_loss": 0.830154, + "learning_rate": 3.898861553966252e-06, + "loss": 0.8532536, + "num_input_tokens_seen": 22668620, + "router_z_loss_clip": 1.51660156, + "router_z_loss_mlp": 0.26220703, + "step": 1070, + "time_per_iteration": 2.8739752769470215 + }, + { + "auxiliary_loss_clip": 0.01217517, + "auxiliary_loss_mlp": 0.01063336, + "balance_loss_clip": 1.06964958, + "balance_loss_mlp": 1.03911281, + "epoch": 0.12878013587446643, + "flos": 25885776251520.0, + "grad_norm": 1.7638129626830137, + "language_loss": 0.88213402, + "learning_rate": 3.898616831206257e-06, + "loss": 0.90494251, + "num_input_tokens_seen": 22689045, + "router_z_loss_clip": 1.4765625, + "router_z_loss_mlp": 0.2421875, + "step": 1071, + "time_per_iteration": 2.867886781692505 + }, + { + "auxiliary_loss_clip": 0.01221637, + "auxiliary_loss_mlp": 0.01053722, + "balance_loss_clip": 1.06907892, + "balance_loss_mlp": 1.02723384, + "epoch": 0.12890037876510552, + "flos": 23333138277120.0, + "grad_norm": 1.955552743741727, + "language_loss": 0.76817685, + "learning_rate": 3.8983718204276e-06, + "loss": 0.79093045, + "num_input_tokens_seen": 22711265, + "router_z_loss_clip": 1.52734375, + "router_z_loss_mlp": 0.26525879, + "step": 1072, + "time_per_iteration": 2.852308511734009 + }, + { + "auxiliary_loss_clip": 0.01215346, + "auxiliary_loss_mlp": 0.01053877, + "balance_loss_clip": 1.06948876, + "balance_loss_mlp": 1.02802002, + "epoch": 0.1290206216557446, + "flos": 23587242065280.0, + "grad_norm": 1.7347957130024234, + "language_loss": 0.82743692, + "learning_rate": 3.898126521667446e-06, + "loss": 0.85012919, + "num_input_tokens_seen": 22731420, + "router_z_loss_clip": 1.45898438, + "router_z_loss_mlp": 0.25878906, + "step": 1073, + "time_per_iteration": 3.0531370639801025 + }, + { + "auxiliary_loss_clip": 0.01222989, + "auxiliary_loss_mlp": 0.01057016, + "balance_loss_clip": 1.07240105, + "balance_loss_mlp": 1.03169572, + "epoch": 0.12914086454638368, + "flos": 24170610850560.0, + "grad_norm": 1.8396084228948226, + "language_loss": 0.83552539, + "learning_rate": 3.897880934963007e-06, + "loss": 0.85832542, + "num_input_tokens_seen": 22750970, + "router_z_loss_clip": 1.50683594, + "router_z_loss_mlp": 0.2532959, + "step": 1074, + "time_per_iteration": 2.9542765617370605 + }, + { + "auxiliary_loss_clip": 0.01214411, + "auxiliary_loss_mlp": 0.01063315, + "balance_loss_clip": 1.0684247, + "balance_loss_mlp": 1.03762519, + "epoch": 0.1292611074370228, + "flos": 20267157081600.0, + "grad_norm": 2.320986928391959, + "language_loss": 0.78567868, + "learning_rate": 3.89763506035154e-06, + "loss": 0.80845594, + "num_input_tokens_seen": 22768820, + "router_z_loss_clip": 1.45898438, + "router_z_loss_mlp": 0.25708008, + "step": 1075, + "time_per_iteration": 2.8423991203308105 + }, + { + "auxiliary_loss_clip": 0.01217383, + "auxiliary_loss_mlp": 0.01066315, + "balance_loss_clip": 1.07130313, + "balance_loss_mlp": 1.042068, + "epoch": 0.12938135032766188, + "flos": 27377684668800.0, + "grad_norm": 1.8381553929651326, + "language_loss": 0.81005859, + "learning_rate": 3.897388897870343e-06, + "loss": 0.83289564, + "num_input_tokens_seen": 22789460, + "router_z_loss_clip": 1.4609375, + "router_z_loss_mlp": 0.24243164, + "step": 1076, + "time_per_iteration": 2.9659945964813232 + }, + { + "auxiliary_loss_clip": 0.01220251, + "auxiliary_loss_mlp": 0.01060401, + "balance_loss_clip": 1.06865323, + "balance_loss_mlp": 1.03457999, + "epoch": 0.12950159321830096, + "flos": 29277107861760.0, + "grad_norm": 2.049718616061615, + "language_loss": 0.74590778, + "learning_rate": 3.89714244755676e-06, + "loss": 0.76871431, + "num_input_tokens_seen": 22810820, + "router_z_loss_clip": 1.515625, + "router_z_loss_mlp": 0.25830078, + "step": 1077, + "time_per_iteration": 2.8883237838745117 + }, + { + "auxiliary_loss_clip": 0.01220572, + "auxiliary_loss_mlp": 0.01063688, + "balance_loss_clip": 1.06978595, + "balance_loss_mlp": 1.03761744, + "epoch": 0.12962183610894007, + "flos": 24534888629760.0, + "grad_norm": 2.4279881122733493, + "language_loss": 0.86195517, + "learning_rate": 3.896895709448175e-06, + "loss": 0.88479769, + "num_input_tokens_seen": 22830570, + "router_z_loss_clip": 1.50878906, + "router_z_loss_mlp": 0.2611084, + "step": 1078, + "time_per_iteration": 2.8483057022094727 + }, + { + "auxiliary_loss_clip": 0.01216552, + "auxiliary_loss_mlp": 0.01060772, + "balance_loss_clip": 1.06524014, + "balance_loss_mlp": 1.03392577, + "epoch": 0.12974207899957915, + "flos": 11215944552960.0, + "grad_norm": 3.4127524751544276, + "language_loss": 0.77115756, + "learning_rate": 3.896648683582019e-06, + "loss": 0.79393077, + "num_input_tokens_seen": 22845905, + "router_z_loss_clip": 1.51269531, + "router_z_loss_mlp": 0.26879883, + "step": 1079, + "time_per_iteration": 2.853316307067871 + }, + { + "auxiliary_loss_clip": 0.01216813, + "auxiliary_loss_mlp": 0.01062883, + "balance_loss_clip": 1.06891859, + "balance_loss_mlp": 1.03833795, + "epoch": 0.12986232189021824, + "flos": 24717889445760.0, + "grad_norm": 2.2730991335811748, + "language_loss": 0.80582005, + "learning_rate": 3.896401369995766e-06, + "loss": 0.82861698, + "num_input_tokens_seen": 22865710, + "router_z_loss_clip": 1.47851562, + "router_z_loss_mlp": 0.24536133, + "step": 1080, + "time_per_iteration": 2.9168875217437744 + }, + { + "auxiliary_loss_clip": 0.01216649, + "auxiliary_loss_mlp": 0.01057209, + "balance_loss_clip": 1.07066083, + "balance_loss_mlp": 1.03240216, + "epoch": 0.12998256478085732, + "flos": 23915357827200.0, + "grad_norm": 2.449302873006513, + "language_loss": 0.79493964, + "learning_rate": 3.896153768726932e-06, + "loss": 0.81767827, + "num_input_tokens_seen": 22886020, + "router_z_loss_clip": 1.45898438, + "router_z_loss_mlp": 0.24816895, + "step": 1081, + "time_per_iteration": 2.8909008502960205 + }, + { + "auxiliary_loss_clip": 0.01218016, + "auxiliary_loss_mlp": 0.01060984, + "balance_loss_clip": 1.07084846, + "balance_loss_mlp": 1.03569973, + "epoch": 0.13010280767149643, + "flos": 18624207974400.0, + "grad_norm": 2.0966207350871273, + "language_loss": 0.8762008, + "learning_rate": 3.8959058798130806e-06, + "loss": 0.89899081, + "num_input_tokens_seen": 22903995, + "router_z_loss_clip": 1.47167969, + "router_z_loss_mlp": 0.25317383, + "step": 1082, + "time_per_iteration": 2.83665132522583 + }, + { + "auxiliary_loss_clip": 0.01222528, + "auxiliary_loss_mlp": 0.01052191, + "balance_loss_clip": 1.07243514, + "balance_loss_mlp": 1.02713299, + "epoch": 0.1302230505621355, + "flos": 22783992174720.0, + "grad_norm": 1.7665212861291393, + "language_loss": 0.7514255, + "learning_rate": 3.895657703291814e-06, + "loss": 0.77417266, + "num_input_tokens_seen": 22924100, + "router_z_loss_clip": 1.50292969, + "router_z_loss_mlp": 0.25073242, + "step": 1083, + "time_per_iteration": 2.869072675704956 + }, + { + "auxiliary_loss_clip": 0.01219209, + "auxiliary_loss_mlp": 0.0105756, + "balance_loss_clip": 1.06814551, + "balance_loss_mlp": 1.02990341, + "epoch": 0.1303432934527746, + "flos": 21323612920320.0, + "grad_norm": 3.026183690265573, + "language_loss": 0.79896533, + "learning_rate": 3.895409239200781e-06, + "loss": 0.821733, + "num_input_tokens_seen": 22939985, + "router_z_loss_clip": 1.50976562, + "router_z_loss_mlp": 0.27661133, + "step": 1084, + "time_per_iteration": 2.900327682495117 + }, + { + "auxiliary_loss_clip": 0.012173, + "auxiliary_loss_mlp": 0.01060769, + "balance_loss_clip": 1.06881428, + "balance_loss_mlp": 1.03457904, + "epoch": 0.1304635363434137, + "flos": 20922490765440.0, + "grad_norm": 2.4304370292658843, + "language_loss": 0.91379869, + "learning_rate": 3.895160487577673e-06, + "loss": 0.93657935, + "num_input_tokens_seen": 22957555, + "router_z_loss_clip": 1.484375, + "router_z_loss_mlp": 0.26196289, + "step": 1085, + "time_per_iteration": 2.918989419937134 + }, + { + "auxiliary_loss_clip": 0.01096428, + "auxiliary_loss_mlp": 0.01007861, + "balance_loss_clip": 1.03595829, + "balance_loss_mlp": 1.00409365, + "epoch": 0.1305837792340528, + "flos": 63245659080960.0, + "grad_norm": 0.7928396027688756, + "language_loss": 0.60926896, + "learning_rate": 3.894911448460226e-06, + "loss": 0.63031185, + "num_input_tokens_seen": 23016870, + "router_z_loss_clip": 0.6015625, + "router_z_loss_mlp": 0.03759766, + "step": 1086, + "time_per_iteration": 4.121317386627197 + }, + { + "auxiliary_loss_clip": 0.01211879, + "auxiliary_loss_mlp": 0.01058995, + "balance_loss_clip": 1.06633306, + "balance_loss_mlp": 1.03317392, + "epoch": 0.13070402212469187, + "flos": 26428852955520.0, + "grad_norm": 1.9798294320873284, + "language_loss": 0.726246, + "learning_rate": 3.8946621218862195e-06, + "loss": 0.74895471, + "num_input_tokens_seen": 23037870, + "router_z_loss_clip": 1.45703125, + "router_z_loss_mlp": 0.25805664, + "step": 1087, + "time_per_iteration": 4.540360450744629 + }, + { + "auxiliary_loss_clip": 0.01212617, + "auxiliary_loss_mlp": 0.01053333, + "balance_loss_clip": 1.06989884, + "balance_loss_mlp": 1.0302422, + "epoch": 0.13082426501533098, + "flos": 27673409341440.0, + "grad_norm": 2.2185348542695915, + "language_loss": 0.88786346, + "learning_rate": 3.894412507893475e-06, + "loss": 0.91052294, + "num_input_tokens_seen": 23058150, + "router_z_loss_clip": 1.42773438, + "router_z_loss_mlp": 0.23083496, + "step": 1088, + "time_per_iteration": 3.7273099422454834 + }, + { + "auxiliary_loss_clip": 0.01221974, + "auxiliary_loss_mlp": 0.01057039, + "balance_loss_clip": 1.07217169, + "balance_loss_mlp": 1.03062177, + "epoch": 0.13094450790597006, + "flos": 24826770547200.0, + "grad_norm": 3.413512365888209, + "language_loss": 0.72020292, + "learning_rate": 3.894162606519859e-06, + "loss": 0.74299306, + "num_input_tokens_seen": 23077100, + "router_z_loss_clip": 1.49707031, + "router_z_loss_mlp": 0.2644043, + "step": 1089, + "time_per_iteration": 2.9481422901153564 + }, + { + "auxiliary_loss_clip": 0.01216943, + "auxiliary_loss_mlp": 0.01065495, + "balance_loss_clip": 1.07027841, + "balance_loss_mlp": 1.04005539, + "epoch": 0.13106475079660915, + "flos": 19062605468160.0, + "grad_norm": 2.00064451679371, + "language_loss": 0.77459741, + "learning_rate": 3.893912417803282e-06, + "loss": 0.79742175, + "num_input_tokens_seen": 23096815, + "router_z_loss_clip": 1.46777344, + "router_z_loss_mlp": 0.2545166, + "step": 1090, + "time_per_iteration": 2.812952995300293 + }, + { + "auxiliary_loss_clip": 0.01219182, + "auxiliary_loss_mlp": 0.01052833, + "balance_loss_clip": 1.06870985, + "balance_loss_mlp": 1.02815652, + "epoch": 0.13118499368724823, + "flos": 28913189218560.0, + "grad_norm": 2.082383870988466, + "language_loss": 0.77046835, + "learning_rate": 3.8936619417816975e-06, + "loss": 0.79318851, + "num_input_tokens_seen": 23117145, + "router_z_loss_clip": 1.50585938, + "router_z_loss_mlp": 0.24658203, + "step": 1091, + "time_per_iteration": 2.826673984527588 + }, + { + "auxiliary_loss_clip": 0.01219072, + "auxiliary_loss_mlp": 0.01061245, + "balance_loss_clip": 1.07296371, + "balance_loss_mlp": 1.03710532, + "epoch": 0.13130523657788734, + "flos": 14283398206080.0, + "grad_norm": 1.9669568597753346, + "language_loss": 0.71297765, + "learning_rate": 3.8934111784931015e-06, + "loss": 0.73578084, + "num_input_tokens_seen": 23134595, + "router_z_loss_clip": 1.4609375, + "router_z_loss_mlp": 0.24133301, + "step": 1092, + "time_per_iteration": 2.7993969917297363 + }, + { + "auxiliary_loss_clip": 0.01092667, + "auxiliary_loss_mlp": 0.0100882, + "balance_loss_clip": 1.03204465, + "balance_loss_mlp": 1.00512445, + "epoch": 0.13142547946852642, + "flos": 70174155519360.0, + "grad_norm": 0.9182888373574901, + "language_loss": 0.59069246, + "learning_rate": 3.893160127975535e-06, + "loss": 0.61170727, + "num_input_tokens_seen": 23195285, + "router_z_loss_clip": 0.60546875, + "router_z_loss_mlp": 0.03686523, + "step": 1093, + "time_per_iteration": 3.4225547313690186 + }, + { + "auxiliary_loss_clip": 0.01211621, + "auxiliary_loss_mlp": 0.01058692, + "balance_loss_clip": 1.06788254, + "balance_loss_mlp": 1.03629303, + "epoch": 0.1315457223591655, + "flos": 45805998844800.0, + "grad_norm": 2.292404991597347, + "language_loss": 0.81168389, + "learning_rate": 3.8929087902670826e-06, + "loss": 0.83438706, + "num_input_tokens_seen": 23216915, + "router_z_loss_clip": 1.4375, + "router_z_loss_mlp": 0.22412109, + "step": 1094, + "time_per_iteration": 3.01657772064209 + }, + { + "auxiliary_loss_clip": 0.01090588, + "auxiliary_loss_mlp": 0.01006766, + "balance_loss_clip": 1.03058589, + "balance_loss_mlp": 1.00323761, + "epoch": 0.13166596524980462, + "flos": 62881165820160.0, + "grad_norm": 0.9365724438443207, + "language_loss": 0.60768449, + "learning_rate": 3.8926571654058715e-06, + "loss": 0.628658, + "num_input_tokens_seen": 23273560, + "router_z_loss_clip": 0.6015625, + "router_z_loss_mlp": 0.03540039, + "step": 1095, + "time_per_iteration": 3.2579538822174072 + }, + { + "auxiliary_loss_clip": 0.01214594, + "auxiliary_loss_mlp": 0.01061091, + "balance_loss_clip": 1.06762993, + "balance_loss_mlp": 1.03450727, + "epoch": 0.1317862081404437, + "flos": 23586523793280.0, + "grad_norm": 2.5245263114652787, + "language_loss": 0.77212131, + "learning_rate": 3.892405253430074e-06, + "loss": 0.79487818, + "num_input_tokens_seen": 23291080, + "router_z_loss_clip": 1.46875, + "router_z_loss_mlp": 0.26586914, + "step": 1096, + "time_per_iteration": 2.96484637260437 + }, + { + "auxiliary_loss_clip": 0.01215884, + "auxiliary_loss_mlp": 0.01065871, + "balance_loss_clip": 1.06962705, + "balance_loss_mlp": 1.04213667, + "epoch": 0.13190645103108278, + "flos": 20260764460800.0, + "grad_norm": 3.848595063756782, + "language_loss": 0.82396543, + "learning_rate": 3.892153054377904e-06, + "loss": 0.84678298, + "num_input_tokens_seen": 23308485, + "router_z_loss_clip": 1.46289062, + "router_z_loss_mlp": 0.23742676, + "step": 1097, + "time_per_iteration": 2.9088385105133057 + }, + { + "auxiliary_loss_clip": 0.01089996, + "auxiliary_loss_mlp": 0.01004268, + "balance_loss_clip": 1.03087068, + "balance_loss_mlp": 1.00093031, + "epoch": 0.13202669392172187, + "flos": 53455440136320.0, + "grad_norm": 0.9337835419365436, + "language_loss": 0.59379578, + "learning_rate": 3.891900568287619e-06, + "loss": 0.61473846, + "num_input_tokens_seen": 23360870, + "router_z_loss_clip": 0.58984375, + "router_z_loss_mlp": 0.03344727, + "step": 1098, + "time_per_iteration": 3.1879820823669434 + }, + { + "auxiliary_loss_clip": 0.01218967, + "auxiliary_loss_mlp": 0.01059655, + "balance_loss_clip": 1.06884348, + "balance_loss_mlp": 1.03296399, + "epoch": 0.13214693681236098, + "flos": 15851293845120.0, + "grad_norm": 2.6042645354761667, + "language_loss": 0.71935314, + "learning_rate": 3.891647795197523e-06, + "loss": 0.74213934, + "num_input_tokens_seen": 23376910, + "router_z_loss_clip": 1.50097656, + "router_z_loss_mlp": 0.2668457, + "step": 1099, + "time_per_iteration": 2.9357686042785645 + }, + { + "auxiliary_loss_clip": 0.01219889, + "auxiliary_loss_mlp": 0.01055512, + "balance_loss_clip": 1.06872129, + "balance_loss_mlp": 1.02897561, + "epoch": 0.13226717970300006, + "flos": 19353840940800.0, + "grad_norm": 2.27102782911683, + "language_loss": 0.68537349, + "learning_rate": 3.8913947351459605e-06, + "loss": 0.7081275, + "num_input_tokens_seen": 23394450, + "router_z_loss_clip": 1.51171875, + "router_z_loss_mlp": 0.26513672, + "step": 1100, + "time_per_iteration": 2.905196189880371 + }, + { + "auxiliary_loss_clip": 0.01212127, + "auxiliary_loss_mlp": 0.01067553, + "balance_loss_clip": 1.0691973, + "balance_loss_mlp": 1.04456973, + "epoch": 0.13238742259363914, + "flos": 20698084546560.0, + "grad_norm": 1.9014709635327371, + "language_loss": 0.67703509, + "learning_rate": 3.89114138817132e-06, + "loss": 0.69983196, + "num_input_tokens_seen": 23411115, + "router_z_loss_clip": 1.4296875, + "router_z_loss_mlp": 0.22973633, + "step": 1101, + "time_per_iteration": 2.8902106285095215 + }, + { + "auxiliary_loss_clip": 0.0120846, + "auxiliary_loss_mlp": 0.01052622, + "balance_loss_clip": 1.06683111, + "balance_loss_mlp": 1.03042579, + "epoch": 0.13250766548427825, + "flos": 21032449274880.0, + "grad_norm": 2.0057239084239638, + "language_loss": 0.84279138, + "learning_rate": 3.890887754312035e-06, + "loss": 0.86540222, + "num_input_tokens_seen": 23429360, + "router_z_loss_clip": 1.41992188, + "router_z_loss_mlp": 0.22216797, + "step": 1102, + "time_per_iteration": 2.9299962520599365 + }, + { + "auxiliary_loss_clip": 0.01213238, + "auxiliary_loss_mlp": 0.01061384, + "balance_loss_clip": 1.06815171, + "balance_loss_mlp": 1.0364697, + "epoch": 0.13262790837491734, + "flos": 22637871648000.0, + "grad_norm": 2.112063792098101, + "language_loss": 0.87422699, + "learning_rate": 3.890633833606581e-06, + "loss": 0.89697319, + "num_input_tokens_seen": 23449050, + "router_z_loss_clip": 1.45214844, + "router_z_loss_mlp": 0.24926758, + "step": 1103, + "time_per_iteration": 2.8499879837036133 + }, + { + "auxiliary_loss_clip": 0.01208277, + "auxiliary_loss_mlp": 0.01064025, + "balance_loss_clip": 1.06701148, + "balance_loss_mlp": 1.03949201, + "epoch": 0.13274815126555642, + "flos": 19683141851520.0, + "grad_norm": 1.7722820253437095, + "language_loss": 0.69237006, + "learning_rate": 3.890379626093477e-06, + "loss": 0.71509308, + "num_input_tokens_seen": 23468800, + "router_z_loss_clip": 1.4140625, + "router_z_loss_mlp": 0.2454834, + "step": 1104, + "time_per_iteration": 2.888648748397827 + }, + { + "auxiliary_loss_clip": 0.01210468, + "auxiliary_loss_mlp": 0.01052745, + "balance_loss_clip": 1.06373906, + "balance_loss_mlp": 1.02736497, + "epoch": 0.1328683941561955, + "flos": 21317687176320.0, + "grad_norm": 2.5775526393956576, + "language_loss": 0.92150044, + "learning_rate": 3.890125131811287e-06, + "loss": 0.94413263, + "num_input_tokens_seen": 23486850, + "router_z_loss_clip": 1.46875, + "router_z_loss_mlp": 0.25390625, + "step": 1105, + "time_per_iteration": 2.8201541900634766 + }, + { + "auxiliary_loss_clip": 0.01210505, + "auxiliary_loss_mlp": 0.01049154, + "balance_loss_clip": 1.0664978, + "balance_loss_mlp": 1.02533615, + "epoch": 0.1329886370468346, + "flos": 13699131580800.0, + "grad_norm": 2.076986434963479, + "language_loss": 0.75676835, + "learning_rate": 3.889870350798618e-06, + "loss": 0.77936494, + "num_input_tokens_seen": 23504195, + "router_z_loss_clip": 1.44042969, + "router_z_loss_mlp": 0.23815918, + "step": 1106, + "time_per_iteration": 2.834486246109009 + }, + { + "auxiliary_loss_clip": 0.0121371, + "auxiliary_loss_mlp": 0.01057313, + "balance_loss_clip": 1.0694716, + "balance_loss_mlp": 1.03194571, + "epoch": 0.1331088799374737, + "flos": 21032413361280.0, + "grad_norm": 1.997919277902125, + "language_loss": 0.7868886, + "learning_rate": 3.889615283094119e-06, + "loss": 0.8095988, + "num_input_tokens_seen": 23523385, + "router_z_loss_clip": 1.44335938, + "router_z_loss_mlp": 0.25378418, + "step": 1107, + "time_per_iteration": 2.792783498764038 + }, + { + "auxiliary_loss_clip": 0.01216334, + "auxiliary_loss_mlp": 0.01047009, + "balance_loss_clip": 1.06651855, + "balance_loss_mlp": 1.02314353, + "epoch": 0.13322912282811278, + "flos": 18260432985600.0, + "grad_norm": 2.3628781193951407, + "language_loss": 0.8451556, + "learning_rate": 3.889359928736485e-06, + "loss": 0.86778903, + "num_input_tokens_seen": 23541330, + "router_z_loss_clip": 1.49707031, + "router_z_loss_mlp": 0.23852539, + "step": 1108, + "time_per_iteration": 2.8282582759857178 + }, + { + "auxiliary_loss_clip": 0.01207932, + "auxiliary_loss_mlp": 0.01048957, + "balance_loss_clip": 1.06551456, + "balance_loss_mlp": 1.02474606, + "epoch": 0.1333493657187519, + "flos": 24460876656000.0, + "grad_norm": 2.257837236664186, + "language_loss": 0.9122923, + "learning_rate": 3.889104287764451e-06, + "loss": 0.93486118, + "num_input_tokens_seen": 23561705, + "router_z_loss_clip": 1.42382812, + "router_z_loss_mlp": 0.24230957, + "step": 1109, + "time_per_iteration": 2.934260368347168 + }, + { + "auxiliary_loss_clip": 0.01213942, + "auxiliary_loss_mlp": 0.01054402, + "balance_loss_clip": 1.06816173, + "balance_loss_mlp": 1.03022623, + "epoch": 0.13346960860939097, + "flos": 22158930677760.0, + "grad_norm": 2.1434455462745374, + "language_loss": 0.90392661, + "learning_rate": 3.888848360216798e-06, + "loss": 0.92660999, + "num_input_tokens_seen": 23579350, + "router_z_loss_clip": 1.45800781, + "router_z_loss_mlp": 0.24169922, + "step": 1110, + "time_per_iteration": 2.8208718299865723 + }, + { + "auxiliary_loss_clip": 0.01086672, + "auxiliary_loss_mlp": 0.01010393, + "balance_loss_clip": 1.0285244, + "balance_loss_mlp": 1.00712669, + "epoch": 0.13358985150003005, + "flos": 67931212608000.0, + "grad_norm": 0.8038720630616496, + "language_loss": 0.56539202, + "learning_rate": 3.888592146132351e-06, + "loss": 0.58636266, + "num_input_tokens_seen": 23640620, + "router_z_loss_clip": 0.578125, + "router_z_loss_mlp": 0.03271484, + "step": 1111, + "time_per_iteration": 3.4200518131256104 + }, + { + "auxiliary_loss_clip": 0.0121024, + "auxiliary_loss_mlp": 0.01062754, + "balance_loss_clip": 1.06813169, + "balance_loss_mlp": 1.03907943, + "epoch": 0.13371009439066917, + "flos": 26834284742400.0, + "grad_norm": 1.89279505462212, + "language_loss": 0.78385472, + "learning_rate": 3.888335645549978e-06, + "loss": 0.8065846, + "num_input_tokens_seen": 23661040, + "router_z_loss_clip": 1.42089844, + "router_z_loss_mlp": 0.23693848, + "step": 1112, + "time_per_iteration": 2.859663963317871 + }, + { + "auxiliary_loss_clip": 0.01209405, + "auxiliary_loss_mlp": 0.01056074, + "balance_loss_clip": 1.06620765, + "balance_loss_mlp": 1.0322324, + "epoch": 0.13383033728130825, + "flos": 26322844942080.0, + "grad_norm": 2.425159640623007, + "language_loss": 0.81312937, + "learning_rate": 3.888078858508588e-06, + "loss": 0.8357842, + "num_input_tokens_seen": 23680900, + "router_z_loss_clip": 1.43359375, + "router_z_loss_mlp": 0.23852539, + "step": 1113, + "time_per_iteration": 2.800175905227661 + }, + { + "auxiliary_loss_clip": 0.0120783, + "auxiliary_loss_mlp": 0.01061668, + "balance_loss_clip": 1.06490123, + "balance_loss_mlp": 1.03696764, + "epoch": 0.13395058017194733, + "flos": 22563931501440.0, + "grad_norm": 2.072064488345686, + "language_loss": 0.84292436, + "learning_rate": 3.8878217850471365e-06, + "loss": 0.8656193, + "num_input_tokens_seen": 23700815, + "router_z_loss_clip": 1.42871094, + "router_z_loss_mlp": 0.24694824, + "step": 1114, + "time_per_iteration": 2.8477776050567627 + }, + { + "auxiliary_loss_clip": 0.01218227, + "auxiliary_loss_mlp": 0.01057291, + "balance_loss_clip": 1.06979358, + "balance_loss_mlp": 1.03265083, + "epoch": 0.13407082306258641, + "flos": 25810938264960.0, + "grad_norm": 1.7575766113606432, + "language_loss": 0.73824322, + "learning_rate": 3.887564425204621e-06, + "loss": 0.76099837, + "num_input_tokens_seen": 23722500, + "router_z_loss_clip": 1.48632812, + "router_z_loss_mlp": 0.24658203, + "step": 1115, + "time_per_iteration": 3.7433764934539795 + }, + { + "auxiliary_loss_clip": 0.01085011, + "auxiliary_loss_mlp": 0.01002436, + "balance_loss_clip": 1.02733004, + "balance_loss_mlp": 0.99928904, + "epoch": 0.13419106595322552, + "flos": 68338365269760.0, + "grad_norm": 0.8363063420380341, + "language_loss": 0.54660225, + "learning_rate": 3.887306779020083e-06, + "loss": 0.56747675, + "num_input_tokens_seen": 23777155, + "router_z_loss_clip": 0.578125, + "router_z_loss_mlp": 0.03149414, + "step": 1116, + "time_per_iteration": 5.079431772232056 + }, + { + "auxiliary_loss_clip": 0.01213172, + "auxiliary_loss_mlp": 0.01051647, + "balance_loss_clip": 1.06851292, + "balance_loss_mlp": 1.02600527, + "epoch": 0.1343113088438646, + "flos": 20449080489600.0, + "grad_norm": 2.7232366934146506, + "language_loss": 0.70938587, + "learning_rate": 3.887048846532608e-06, + "loss": 0.73203409, + "num_input_tokens_seen": 23794130, + "router_z_loss_clip": 1.44726562, + "router_z_loss_mlp": 0.25610352, + "step": 1117, + "time_per_iteration": 3.649000644683838 + }, + { + "auxiliary_loss_clip": 0.01084908, + "auxiliary_loss_mlp": 0.01001129, + "balance_loss_clip": 1.02709568, + "balance_loss_mlp": 0.99795836, + "epoch": 0.1344315517345037, + "flos": 67389784951680.0, + "grad_norm": 0.7597342782459636, + "language_loss": 0.58118176, + "learning_rate": 3.8867906277813224e-06, + "loss": 0.6020422, + "num_input_tokens_seen": 23852285, + "router_z_loss_clip": 0.578125, + "router_z_loss_mlp": 0.03173828, + "step": 1118, + "time_per_iteration": 3.2539544105529785 + }, + { + "auxiliary_loss_clip": 0.01209598, + "auxiliary_loss_mlp": 0.01062995, + "balance_loss_clip": 1.06604552, + "balance_loss_mlp": 1.03814006, + "epoch": 0.1345517946251428, + "flos": 40734442788480.0, + "grad_norm": 2.1645491202257126, + "language_loss": 0.73543817, + "learning_rate": 3.886532122805399e-06, + "loss": 0.75816405, + "num_input_tokens_seen": 23874765, + "router_z_loss_clip": 1.43554688, + "router_z_loss_mlp": 0.24853516, + "step": 1119, + "time_per_iteration": 3.0317418575286865 + }, + { + "auxiliary_loss_clip": 0.01214203, + "auxiliary_loss_mlp": 0.01056631, + "balance_loss_clip": 1.06698394, + "balance_loss_mlp": 1.03178751, + "epoch": 0.13467203751578188, + "flos": 22816850140800.0, + "grad_norm": 2.2358841611050804, + "language_loss": 0.89767504, + "learning_rate": 3.886273331644053e-06, + "loss": 0.92038339, + "num_input_tokens_seen": 23893635, + "router_z_loss_clip": 1.47265625, + "router_z_loss_mlp": 0.24841309, + "step": 1120, + "time_per_iteration": 2.9182658195495605 + }, + { + "auxiliary_loss_clip": 0.01202954, + "auxiliary_loss_mlp": 0.01060842, + "balance_loss_clip": 1.06279922, + "balance_loss_mlp": 1.03673804, + "epoch": 0.13479228040642097, + "flos": 17091576512640.0, + "grad_norm": 2.202489881525613, + "language_loss": 0.82458109, + "learning_rate": 3.886014254336542e-06, + "loss": 0.84721899, + "num_input_tokens_seen": 23910110, + "router_z_loss_clip": 1.40136719, + "router_z_loss_mlp": 0.2409668, + "step": 1121, + "time_per_iteration": 2.7942161560058594 + }, + { + "auxiliary_loss_clip": 0.01209229, + "auxiliary_loss_mlp": 0.01057027, + "balance_loss_clip": 1.06774032, + "balance_loss_mlp": 1.03444862, + "epoch": 0.13491252329706005, + "flos": 23730525417600.0, + "grad_norm": 1.7140779202153014, + "language_loss": 0.92338723, + "learning_rate": 3.885754890922168e-06, + "loss": 0.94604981, + "num_input_tokens_seen": 23930440, + "router_z_loss_clip": 1.41503906, + "router_z_loss_mlp": 0.22583008, + "step": 1122, + "time_per_iteration": 2.876535177230835 + }, + { + "auxiliary_loss_clip": 0.01214846, + "auxiliary_loss_mlp": 0.01057041, + "balance_loss_clip": 1.06842041, + "balance_loss_mlp": 1.0328896, + "epoch": 0.13503276618769916, + "flos": 34127058960000.0, + "grad_norm": 1.7582335820530457, + "language_loss": 0.7829535, + "learning_rate": 3.885495241440277e-06, + "loss": 0.80567241, + "num_input_tokens_seen": 23954535, + "router_z_loss_clip": 1.46484375, + "router_z_loss_mlp": 0.24157715, + "step": 1123, + "time_per_iteration": 2.953747034072876 + }, + { + "auxiliary_loss_clip": 0.0120795, + "auxiliary_loss_mlp": 0.01052152, + "balance_loss_clip": 1.06417859, + "balance_loss_mlp": 1.02978885, + "epoch": 0.13515300907833824, + "flos": 17712328377600.0, + "grad_norm": 1.91652127406346, + "language_loss": 0.74237263, + "learning_rate": 3.885235305930257e-06, + "loss": 0.76497364, + "num_input_tokens_seen": 23972735, + "router_z_loss_clip": 1.43847656, + "router_z_loss_mlp": 0.22363281, + "step": 1124, + "time_per_iteration": 2.851022958755493 + }, + { + "auxiliary_loss_clip": 0.01207384, + "auxiliary_loss_mlp": 0.01063444, + "balance_loss_clip": 1.06904507, + "balance_loss_mlp": 1.04006732, + "epoch": 0.13527325196897733, + "flos": 20260872201600.0, + "grad_norm": 2.5498756035454426, + "language_loss": 0.85127676, + "learning_rate": 3.884975084431539e-06, + "loss": 0.87398511, + "num_input_tokens_seen": 23987685, + "router_z_loss_clip": 1.38378906, + "router_z_loss_mlp": 0.23364258, + "step": 1125, + "time_per_iteration": 2.7899410724639893 + }, + { + "auxiliary_loss_clip": 0.012102, + "auxiliary_loss_mlp": 0.01072925, + "balance_loss_clip": 1.06938243, + "balance_loss_mlp": 1.04787922, + "epoch": 0.13539349485961644, + "flos": 18186492839040.0, + "grad_norm": 2.372958341374972, + "language_loss": 0.91575903, + "learning_rate": 3.8847145769836e-06, + "loss": 0.93859029, + "num_input_tokens_seen": 24004105, + "router_z_loss_clip": 1.40820312, + "router_z_loss_mlp": 0.25061035, + "step": 1126, + "time_per_iteration": 2.8505859375 + }, + { + "auxiliary_loss_clip": 0.01212973, + "auxiliary_loss_mlp": 0.01063847, + "balance_loss_clip": 1.06838536, + "balance_loss_mlp": 1.03928971, + "epoch": 0.13551373775025552, + "flos": 19317463441920.0, + "grad_norm": 2.8929184953734723, + "language_loss": 0.66758835, + "learning_rate": 3.884453783625959e-06, + "loss": 0.69035661, + "num_input_tokens_seen": 24021715, + "router_z_loss_clip": 1.44628906, + "router_z_loss_mlp": 0.2454834, + "step": 1127, + "time_per_iteration": 2.76908802986145 + }, + { + "auxiliary_loss_clip": 0.01211752, + "auxiliary_loss_mlp": 0.01072749, + "balance_loss_clip": 1.06515551, + "balance_loss_mlp": 1.04747629, + "epoch": 0.1356339806408946, + "flos": 20850813175680.0, + "grad_norm": 2.178615787365668, + "language_loss": 0.84786689, + "learning_rate": 3.884192704398176e-06, + "loss": 0.87071186, + "num_input_tokens_seen": 24038915, + "router_z_loss_clip": 1.46386719, + "router_z_loss_mlp": 0.25268555, + "step": 1128, + "time_per_iteration": 2.844529867172241 + }, + { + "auxiliary_loss_clip": 0.01210054, + "auxiliary_loss_mlp": 0.0105499, + "balance_loss_clip": 1.06727052, + "balance_loss_mlp": 1.03081417, + "epoch": 0.13575422353153369, + "flos": 50476037696640.0, + "grad_norm": 1.7812571095620753, + "language_loss": 0.74730593, + "learning_rate": 3.883931339339858e-06, + "loss": 0.76995635, + "num_input_tokens_seen": 24063300, + "router_z_loss_clip": 1.4296875, + "router_z_loss_mlp": 0.24182129, + "step": 1129, + "time_per_iteration": 3.0093960762023926 + }, + { + "auxiliary_loss_clip": 0.0121749, + "auxiliary_loss_mlp": 0.01063315, + "balance_loss_clip": 1.07009768, + "balance_loss_mlp": 1.03561068, + "epoch": 0.1358744664221728, + "flos": 18150797698560.0, + "grad_norm": 2.3878414628736997, + "language_loss": 0.78883499, + "learning_rate": 3.883669688490654e-06, + "loss": 0.811643, + "num_input_tokens_seen": 24081070, + "router_z_loss_clip": 1.47558594, + "router_z_loss_mlp": 0.27709961, + "step": 1130, + "time_per_iteration": 2.8357832431793213 + }, + { + "auxiliary_loss_clip": 0.0120757, + "auxiliary_loss_mlp": 0.01060242, + "balance_loss_clip": 1.06781662, + "balance_loss_mlp": 1.03705585, + "epoch": 0.13599470931281188, + "flos": 18442966924800.0, + "grad_norm": 2.2082062588177145, + "language_loss": 0.85428131, + "learning_rate": 3.883407751890256e-06, + "loss": 0.87695932, + "num_input_tokens_seen": 24099675, + "router_z_loss_clip": 1.39648438, + "router_z_loss_mlp": 0.23193359, + "step": 1131, + "time_per_iteration": 2.840061902999878 + }, + { + "auxiliary_loss_clip": 0.01212505, + "auxiliary_loss_mlp": 0.01058579, + "balance_loss_clip": 1.06499934, + "balance_loss_mlp": 1.03294945, + "epoch": 0.13611495220345096, + "flos": 26680766014080.0, + "grad_norm": 1.8924680484232368, + "language_loss": 0.8575865, + "learning_rate": 3.8831455295783994e-06, + "loss": 0.88029742, + "num_input_tokens_seen": 24118925, + "router_z_loss_clip": 1.47558594, + "router_z_loss_mlp": 0.2565918, + "step": 1132, + "time_per_iteration": 2.8883957862854004 + }, + { + "auxiliary_loss_clip": 0.01214331, + "auxiliary_loss_mlp": 0.01065075, + "balance_loss_clip": 1.0683012, + "balance_loss_mlp": 1.0398742, + "epoch": 0.13623519509409007, + "flos": 21686238673920.0, + "grad_norm": 1.6869326968940734, + "language_loss": 0.74146825, + "learning_rate": 3.882883021594864e-06, + "loss": 0.76426232, + "num_input_tokens_seen": 24137065, + "router_z_loss_clip": 1.45898438, + "router_z_loss_mlp": 0.25231934, + "step": 1133, + "time_per_iteration": 3.0001745223999023 + }, + { + "auxiliary_loss_clip": 0.0120958, + "auxiliary_loss_mlp": 0.0106629, + "balance_loss_clip": 1.06686282, + "balance_loss_mlp": 1.04186428, + "epoch": 0.13635543798472916, + "flos": 14830389492480.0, + "grad_norm": 2.2249496028990765, + "language_loss": 0.86825436, + "learning_rate": 3.8826202279794705e-06, + "loss": 0.89101303, + "num_input_tokens_seen": 24154125, + "router_z_loss_clip": 1.42675781, + "router_z_loss_mlp": 0.24438477, + "step": 1134, + "time_per_iteration": 2.8278207778930664 + }, + { + "auxiliary_loss_clip": 0.01212254, + "auxiliary_loss_mlp": 0.01054585, + "balance_loss_clip": 1.06889927, + "balance_loss_mlp": 1.03111279, + "epoch": 0.13647568087536824, + "flos": 22890323410560.0, + "grad_norm": 1.9367559531373426, + "language_loss": 0.70545536, + "learning_rate": 3.882357148772085e-06, + "loss": 0.72812378, + "num_input_tokens_seen": 24171550, + "router_z_loss_clip": 1.43261719, + "router_z_loss_mlp": 0.23486328, + "step": 1135, + "time_per_iteration": 2.885462522506714 + }, + { + "auxiliary_loss_clip": 0.0121197, + "auxiliary_loss_mlp": 0.01062101, + "balance_loss_clip": 1.06815553, + "balance_loss_mlp": 1.03742433, + "epoch": 0.13659592376600732, + "flos": 19937927998080.0, + "grad_norm": 3.333879027153783, + "language_loss": 0.84323108, + "learning_rate": 3.882093784012617e-06, + "loss": 0.86597174, + "num_input_tokens_seen": 24190190, + "router_z_loss_clip": 1.43945312, + "router_z_loss_mlp": 0.24707031, + "step": 1136, + "time_per_iteration": 2.8596110343933105 + }, + { + "auxiliary_loss_clip": 0.01209139, + "auxiliary_loss_mlp": 0.01069595, + "balance_loss_clip": 1.06728649, + "balance_loss_mlp": 1.04520512, + "epoch": 0.13671616665664643, + "flos": 21428579439360.0, + "grad_norm": 2.3072302203566313, + "language_loss": 0.83912522, + "learning_rate": 3.881830133741019e-06, + "loss": 0.86191249, + "num_input_tokens_seen": 24209055, + "router_z_loss_clip": 1.41894531, + "router_z_loss_mlp": 0.24401855, + "step": 1137, + "time_per_iteration": 2.9201819896698 + }, + { + "auxiliary_loss_clip": 0.01208979, + "auxiliary_loss_mlp": 0.01059944, + "balance_loss_clip": 1.0682987, + "balance_loss_mlp": 1.03649509, + "epoch": 0.13683640954728551, + "flos": 22778138257920.0, + "grad_norm": 2.198937538816138, + "language_loss": 0.76250875, + "learning_rate": 3.881566197997285e-06, + "loss": 0.78519797, + "num_input_tokens_seen": 24225490, + "router_z_loss_clip": 1.40722656, + "router_z_loss_mlp": 0.23449707, + "step": 1138, + "time_per_iteration": 2.8606269359588623 + }, + { + "auxiliary_loss_clip": 0.0120362, + "auxiliary_loss_mlp": 0.01053578, + "balance_loss_clip": 1.06682599, + "balance_loss_mlp": 1.03085637, + "epoch": 0.1369566524379246, + "flos": 21725884310400.0, + "grad_norm": 1.6006640407241985, + "language_loss": 0.74663556, + "learning_rate": 3.881301976821456e-06, + "loss": 0.76920748, + "num_input_tokens_seen": 24245520, + "router_z_loss_clip": 1.36816406, + "router_z_loss_mlp": 0.22729492, + "step": 1139, + "time_per_iteration": 2.8556582927703857 + }, + { + "auxiliary_loss_clip": 0.01205563, + "auxiliary_loss_mlp": 0.01058579, + "balance_loss_clip": 1.06650054, + "balance_loss_mlp": 1.03473735, + "epoch": 0.1370768953285637, + "flos": 18624459369600.0, + "grad_norm": 2.1521552374578525, + "language_loss": 0.90583163, + "learning_rate": 3.881037470253612e-06, + "loss": 0.92847306, + "num_input_tokens_seen": 24265035, + "router_z_loss_clip": 1.38769531, + "router_z_loss_mlp": 0.23864746, + "step": 1140, + "time_per_iteration": 2.836233377456665 + }, + { + "auxiliary_loss_clip": 0.01204204, + "auxiliary_loss_mlp": 0.01059456, + "balance_loss_clip": 1.06461406, + "balance_loss_mlp": 1.03606701, + "epoch": 0.1371971382192028, + "flos": 14939521989120.0, + "grad_norm": 3.1710019898867956, + "language_loss": 0.80180711, + "learning_rate": 3.88077267833388e-06, + "loss": 0.8244437, + "num_input_tokens_seen": 24281550, + "router_z_loss_clip": 1.39550781, + "router_z_loss_mlp": 0.23400879, + "step": 1141, + "time_per_iteration": 2.8418986797332764 + }, + { + "auxiliary_loss_clip": 0.01209245, + "auxiliary_loss_mlp": 0.01066954, + "balance_loss_clip": 1.06572735, + "balance_loss_mlp": 1.04274297, + "epoch": 0.13731738110984187, + "flos": 19023785844480.0, + "grad_norm": 1.9624535090671476, + "language_loss": 0.8390919, + "learning_rate": 3.880507601102427e-06, + "loss": 0.86185396, + "num_input_tokens_seen": 24299485, + "router_z_loss_clip": 1.43652344, + "router_z_loss_mlp": 0.2421875, + "step": 1142, + "time_per_iteration": 2.853771448135376 + }, + { + "auxiliary_loss_clip": 0.01207451, + "auxiliary_loss_mlp": 0.010621, + "balance_loss_clip": 1.06891775, + "balance_loss_mlp": 1.03707755, + "epoch": 0.13743762400048098, + "flos": 18187462506240.0, + "grad_norm": 2.354310267494864, + "language_loss": 0.82538521, + "learning_rate": 3.880242238599467e-06, + "loss": 0.84808075, + "num_input_tokens_seen": 24316010, + "router_z_loss_clip": 1.38476562, + "router_z_loss_mlp": 0.25024414, + "step": 1143, + "time_per_iteration": 2.8726301193237305 + }, + { + "auxiliary_loss_clip": 0.01205715, + "auxiliary_loss_mlp": 0.01055612, + "balance_loss_clip": 1.06715417, + "balance_loss_mlp": 1.03256893, + "epoch": 0.13755786689112007, + "flos": 21031982398080.0, + "grad_norm": 1.6963824179849891, + "language_loss": 0.8343333, + "learning_rate": 3.879976590865254e-06, + "loss": 0.85694653, + "num_input_tokens_seen": 24335465, + "router_z_loss_clip": 1.38574219, + "router_z_loss_mlp": 0.23010254, + "step": 1144, + "time_per_iteration": 3.7768728733062744 + }, + { + "auxiliary_loss_clip": 0.0120553, + "auxiliary_loss_mlp": 0.01057196, + "balance_loss_clip": 1.06749785, + "balance_loss_mlp": 1.03402162, + "epoch": 0.13767810978175915, + "flos": 21360636864000.0, + "grad_norm": 2.830284366662402, + "language_loss": 0.87402201, + "learning_rate": 3.879710657940087e-06, + "loss": 0.8966493, + "num_input_tokens_seen": 24354415, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.23168945, + "step": 1145, + "time_per_iteration": 4.575195550918579 + }, + { + "auxiliary_loss_clip": 0.01213549, + "auxiliary_loss_mlp": 0.01061872, + "balance_loss_clip": 1.06965101, + "balance_loss_mlp": 1.0367192, + "epoch": 0.13779835267239823, + "flos": 30592084861440.0, + "grad_norm": 2.2152190661224553, + "language_loss": 0.70165521, + "learning_rate": 3.879444439864308e-06, + "loss": 0.72440946, + "num_input_tokens_seen": 24373990, + "router_z_loss_clip": 1.43945312, + "router_z_loss_mlp": 0.25183105, + "step": 1146, + "time_per_iteration": 3.7551770210266113 + }, + { + "auxiliary_loss_clip": 0.0120832, + "auxiliary_loss_mlp": 0.01060289, + "balance_loss_clip": 1.06557298, + "balance_loss_mlp": 1.03566074, + "epoch": 0.13791859556303734, + "flos": 22669867687680.0, + "grad_norm": 1.9492332878856165, + "language_loss": 0.85960245, + "learning_rate": 3.879177936678301e-06, + "loss": 0.88228852, + "num_input_tokens_seen": 24392995, + "router_z_loss_clip": 1.42773438, + "router_z_loss_mlp": 0.24609375, + "step": 1147, + "time_per_iteration": 2.780318021774292 + }, + { + "auxiliary_loss_clip": 0.01212727, + "auxiliary_loss_mlp": 0.01060268, + "balance_loss_clip": 1.06862056, + "balance_loss_mlp": 1.03653383, + "epoch": 0.13803883845367643, + "flos": 35224166016000.0, + "grad_norm": 1.7406589313009369, + "language_loss": 0.77002037, + "learning_rate": 3.878911148422496e-06, + "loss": 0.79275036, + "num_input_tokens_seen": 24414470, + "router_z_loss_clip": 1.43945312, + "router_z_loss_mlp": 0.23754883, + "step": 1148, + "time_per_iteration": 2.9360029697418213 + }, + { + "auxiliary_loss_clip": 0.01208572, + "auxiliary_loss_mlp": 0.01063782, + "balance_loss_clip": 1.06589723, + "balance_loss_mlp": 1.03879619, + "epoch": 0.1381590813443155, + "flos": 32014542332160.0, + "grad_norm": 2.0495387326320476, + "language_loss": 0.7044124, + "learning_rate": 3.878644075137364e-06, + "loss": 0.7271359, + "num_input_tokens_seen": 24435120, + "router_z_loss_clip": 1.42480469, + "router_z_loss_mlp": 0.24987793, + "step": 1149, + "time_per_iteration": 2.8608720302581787 + }, + { + "auxiliary_loss_clip": 0.01207446, + "auxiliary_loss_mlp": 0.01066844, + "balance_loss_clip": 1.06599307, + "balance_loss_mlp": 1.04164338, + "epoch": 0.13827932423495462, + "flos": 17821855923840.0, + "grad_norm": 2.0437040649181863, + "language_loss": 0.79179037, + "learning_rate": 3.878376716863418e-06, + "loss": 0.81453323, + "num_input_tokens_seen": 24451420, + "router_z_loss_clip": 1.41503906, + "router_z_loss_mlp": 0.25231934, + "step": 1150, + "time_per_iteration": 2.7943766117095947 + }, + { + "auxiliary_loss_clip": 0.01212141, + "auxiliary_loss_mlp": 0.01057168, + "balance_loss_clip": 1.06759501, + "balance_loss_mlp": 1.0339222, + "epoch": 0.1383995671255937, + "flos": 19427098728960.0, + "grad_norm": 2.2171114905720213, + "language_loss": 0.71446371, + "learning_rate": 3.878109073641219e-06, + "loss": 0.73715675, + "num_input_tokens_seen": 24470450, + "router_z_loss_clip": 1.4453125, + "router_z_loss_mlp": 0.23254395, + "step": 1151, + "time_per_iteration": 2.7878286838531494 + }, + { + "auxiliary_loss_clip": 0.01206302, + "auxiliary_loss_mlp": 0.01067944, + "balance_loss_clip": 1.06479943, + "balance_loss_mlp": 1.04400682, + "epoch": 0.13851981001623279, + "flos": 28296603331200.0, + "grad_norm": 1.56052821275613, + "language_loss": 0.81185269, + "learning_rate": 3.877841145511366e-06, + "loss": 0.83459508, + "num_input_tokens_seen": 24493190, + "router_z_loss_clip": 1.41601562, + "router_z_loss_mlp": 0.23937988, + "step": 1152, + "time_per_iteration": 2.8759617805480957 + }, + { + "auxiliary_loss_clip": 0.01207988, + "auxiliary_loss_mlp": 0.01058729, + "balance_loss_clip": 1.06609464, + "balance_loss_mlp": 1.03469658, + "epoch": 0.13864005290687187, + "flos": 21213079793280.0, + "grad_norm": 1.7846015862803886, + "language_loss": 0.82611299, + "learning_rate": 3.8775729325145035e-06, + "loss": 0.84878016, + "num_input_tokens_seen": 24512425, + "router_z_loss_clip": 1.41992188, + "router_z_loss_mlp": 0.24060059, + "step": 1153, + "time_per_iteration": 2.869659662246704 + }, + { + "auxiliary_loss_clip": 0.01092124, + "auxiliary_loss_mlp": 0.01012793, + "balance_loss_clip": 1.03370321, + "balance_loss_mlp": 1.0093596, + "epoch": 0.13876029579751098, + "flos": 71653389413760.0, + "grad_norm": 0.7907393028623309, + "language_loss": 0.64783645, + "learning_rate": 3.877304434691321e-06, + "loss": 0.66888559, + "num_input_tokens_seen": 24579275, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.03442383, + "step": 1154, + "time_per_iteration": 3.5231451988220215 + }, + { + "auxiliary_loss_clip": 0.01203495, + "auxiliary_loss_mlp": 0.01047599, + "balance_loss_clip": 1.06657505, + "balance_loss_mlp": 1.02496099, + "epoch": 0.13888053868815006, + "flos": 21941348042880.0, + "grad_norm": 1.8295859935685526, + "language_loss": 0.79724306, + "learning_rate": 3.877035652082548e-06, + "loss": 0.819754, + "num_input_tokens_seen": 24598720, + "router_z_loss_clip": 1.36914062, + "router_z_loss_mlp": 0.22644043, + "step": 1155, + "time_per_iteration": 2.9046249389648438 + }, + { + "auxiliary_loss_clip": 0.0121227, + "auxiliary_loss_mlp": 0.01056961, + "balance_loss_clip": 1.0683794, + "balance_loss_mlp": 1.03175998, + "epoch": 0.13900078157878915, + "flos": 19608627087360.0, + "grad_norm": 2.0081863861391978, + "language_loss": 0.85234988, + "learning_rate": 3.87676658472896e-06, + "loss": 0.8750422, + "num_input_tokens_seen": 24617530, + "router_z_loss_clip": 1.43945312, + "router_z_loss_mlp": 0.2520752, + "step": 1156, + "time_per_iteration": 2.9624459743499756 + }, + { + "auxiliary_loss_clip": 0.0121402, + "auxiliary_loss_mlp": 0.01058309, + "balance_loss_clip": 1.06819916, + "balance_loss_mlp": 1.03179669, + "epoch": 0.13912102446942826, + "flos": 22638051216000.0, + "grad_norm": 2.389941219022454, + "language_loss": 0.85279524, + "learning_rate": 3.876497232671372e-06, + "loss": 0.87551856, + "num_input_tokens_seen": 24637485, + "router_z_loss_clip": 1.45898438, + "router_z_loss_mlp": 0.26525879, + "step": 1157, + "time_per_iteration": 2.9306535720825195 + }, + { + "auxiliary_loss_clip": 0.01204463, + "auxiliary_loss_mlp": 0.01044479, + "balance_loss_clip": 1.06421626, + "balance_loss_mlp": 1.02201986, + "epoch": 0.13924126736006734, + "flos": 29643324975360.0, + "grad_norm": 2.265447618423487, + "language_loss": 0.8368578, + "learning_rate": 3.876227595950647e-06, + "loss": 0.85934716, + "num_input_tokens_seen": 24656915, + "router_z_loss_clip": 1.40234375, + "router_z_loss_mlp": 0.22460938, + "step": 1158, + "time_per_iteration": 2.947389841079712 + }, + { + "auxiliary_loss_clip": 0.01209631, + "auxiliary_loss_mlp": 0.01068429, + "balance_loss_clip": 1.06596565, + "balance_loss_mlp": 1.04232204, + "epoch": 0.13936151025070642, + "flos": 27417653527680.0, + "grad_norm": 1.8833685440542445, + "language_loss": 0.79119229, + "learning_rate": 3.875957674607686e-06, + "loss": 0.81397289, + "num_input_tokens_seen": 24679190, + "router_z_loss_clip": 1.43652344, + "router_z_loss_mlp": 0.26135254, + "step": 1159, + "time_per_iteration": 3.091599941253662 + }, + { + "auxiliary_loss_clip": 0.01206643, + "auxiliary_loss_mlp": 0.01057104, + "balance_loss_clip": 1.0624603, + "balance_loss_mlp": 1.03283262, + "epoch": 0.1394817531413455, + "flos": 16399326625920.0, + "grad_norm": 2.047806415972891, + "language_loss": 0.87570131, + "learning_rate": 3.8756874686834386e-06, + "loss": 0.89833879, + "num_input_tokens_seen": 24697405, + "router_z_loss_clip": 1.44140625, + "router_z_loss_mlp": 0.24291992, + "step": 1160, + "time_per_iteration": 2.856950521469116 + }, + { + "auxiliary_loss_clip": 0.01209781, + "auxiliary_loss_mlp": 0.0105801, + "balance_loss_clip": 1.06753707, + "balance_loss_mlp": 1.03310764, + "epoch": 0.13960199603198462, + "flos": 30922319525760.0, + "grad_norm": 1.7433877398277173, + "language_loss": 0.80282772, + "learning_rate": 3.875416978218893e-06, + "loss": 0.82550567, + "num_input_tokens_seen": 24720600, + "router_z_loss_clip": 1.421875, + "router_z_loss_mlp": 0.24902344, + "step": 1161, + "time_per_iteration": 2.925034761428833 + }, + { + "auxiliary_loss_clip": 0.01214745, + "auxiliary_loss_mlp": 0.01060304, + "balance_loss_clip": 1.07065082, + "balance_loss_mlp": 1.03418517, + "epoch": 0.1397222389226237, + "flos": 18113773754880.0, + "grad_norm": 2.8331313043125506, + "language_loss": 0.82525831, + "learning_rate": 3.8751462032550835e-06, + "loss": 0.84800881, + "num_input_tokens_seen": 24737605, + "router_z_loss_clip": 1.44140625, + "router_z_loss_mlp": 0.26123047, + "step": 1162, + "time_per_iteration": 2.8143246173858643 + }, + { + "auxiliary_loss_clip": 0.0120867, + "auxiliary_loss_mlp": 0.0105546, + "balance_loss_clip": 1.07010663, + "balance_loss_mlp": 1.03179693, + "epoch": 0.13984248181326278, + "flos": 16872772815360.0, + "grad_norm": 2.8696322126265468, + "language_loss": 0.82824326, + "learning_rate": 3.874875143833085e-06, + "loss": 0.85088462, + "num_input_tokens_seen": 24755845, + "router_z_loss_clip": 1.38378906, + "router_z_loss_mlp": 0.23669434, + "step": 1163, + "time_per_iteration": 2.8785977363586426 + }, + { + "auxiliary_loss_clip": 0.01208405, + "auxiliary_loss_mlp": 0.01061051, + "balance_loss_clip": 1.06381238, + "balance_loss_mlp": 1.03544474, + "epoch": 0.1399627247039019, + "flos": 54121401267840.0, + "grad_norm": 1.9019109637190121, + "language_loss": 0.68794501, + "learning_rate": 3.874603799994019e-06, + "loss": 0.71063954, + "num_input_tokens_seen": 24779380, + "router_z_loss_clip": 1.4453125, + "router_z_loss_mlp": 0.25622559, + "step": 1164, + "time_per_iteration": 3.084001064300537 + }, + { + "auxiliary_loss_clip": 0.01206534, + "auxiliary_loss_mlp": 0.01055817, + "balance_loss_clip": 1.06822526, + "balance_loss_mlp": 1.03353715, + "epoch": 0.14008296759454097, + "flos": 11765521618560.0, + "grad_norm": 2.1261916180421823, + "language_loss": 0.87124634, + "learning_rate": 3.874332171779046e-06, + "loss": 0.89386988, + "num_input_tokens_seen": 24794260, + "router_z_loss_clip": 1.38183594, + "router_z_loss_mlp": 0.22277832, + "step": 1165, + "time_per_iteration": 2.814661979675293 + }, + { + "auxiliary_loss_clip": 0.01203788, + "auxiliary_loss_mlp": 0.01055346, + "balance_loss_clip": 1.06456518, + "balance_loss_mlp": 1.02869081, + "epoch": 0.14020321048518006, + "flos": 22017514832640.0, + "grad_norm": 1.829808199052901, + "language_loss": 0.75459653, + "learning_rate": 3.874060259229373e-06, + "loss": 0.77718788, + "num_input_tokens_seen": 24815835, + "router_z_loss_clip": 1.39257812, + "router_z_loss_mlp": 0.26696777, + "step": 1166, + "time_per_iteration": 2.85758900642395 + }, + { + "auxiliary_loss_clip": 0.01215073, + "auxiliary_loss_mlp": 0.01062945, + "balance_loss_clip": 1.06993186, + "balance_loss_mlp": 1.03800666, + "epoch": 0.14032345337581917, + "flos": 23404313076480.0, + "grad_norm": 2.2837962434105035, + "language_loss": 0.93734503, + "learning_rate": 3.873788062386249e-06, + "loss": 0.96012521, + "num_input_tokens_seen": 24834095, + "router_z_loss_clip": 1.45019531, + "router_z_loss_mlp": 0.24951172, + "step": 1167, + "time_per_iteration": 2.891826868057251 + }, + { + "auxiliary_loss_clip": 0.01202558, + "auxiliary_loss_mlp": 0.01056923, + "balance_loss_clip": 1.06456041, + "balance_loss_mlp": 1.03230619, + "epoch": 0.14044369626645825, + "flos": 29645767100160.0, + "grad_norm": 4.887489213592365, + "language_loss": 0.82063353, + "learning_rate": 3.873515581290965e-06, + "loss": 0.84322834, + "num_input_tokens_seen": 24858900, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.24597168, + "step": 1168, + "time_per_iteration": 2.9216644763946533 + }, + { + "auxiliary_loss_clip": 0.01209253, + "auxiliary_loss_mlp": 0.01059995, + "balance_loss_clip": 1.06751883, + "balance_loss_mlp": 1.03590274, + "epoch": 0.14056393915709733, + "flos": 18332972501760.0, + "grad_norm": 2.6794001045452815, + "language_loss": 0.75867355, + "learning_rate": 3.8732428159848575e-06, + "loss": 0.78136605, + "num_input_tokens_seen": 24877875, + "router_z_loss_clip": 1.41796875, + "router_z_loss_mlp": 0.2409668, + "step": 1169, + "time_per_iteration": 2.8014206886291504 + }, + { + "auxiliary_loss_clip": 0.01213269, + "auxiliary_loss_mlp": 0.01059147, + "balance_loss_clip": 1.07111025, + "balance_loss_mlp": 1.03600848, + "epoch": 0.14068418204773642, + "flos": 26687517770880.0, + "grad_norm": 2.172710640316676, + "language_loss": 0.77955055, + "learning_rate": 3.872969766509304e-06, + "loss": 0.8022747, + "num_input_tokens_seen": 24898430, + "router_z_loss_clip": 1.42089844, + "router_z_loss_mlp": 0.23132324, + "step": 1170, + "time_per_iteration": 2.873619794845581 + }, + { + "auxiliary_loss_clip": 0.01085702, + "auxiliary_loss_mlp": 0.01007679, + "balance_loss_clip": 1.02905679, + "balance_loss_mlp": 1.00417447, + "epoch": 0.14080442493837553, + "flos": 65259314501760.0, + "grad_norm": 0.7705164840338276, + "language_loss": 0.55602026, + "learning_rate": 3.872696432905726e-06, + "loss": 0.57695413, + "num_input_tokens_seen": 24959250, + "router_z_loss_clip": 0.56640625, + "router_z_loss_mlp": 0.03515625, + "step": 1171, + "time_per_iteration": 3.3265304565429688 + }, + { + "auxiliary_loss_clip": 0.01210055, + "auxiliary_loss_mlp": 0.0105909, + "balance_loss_clip": 1.06534708, + "balance_loss_mlp": 1.03363836, + "epoch": 0.1409246678290146, + "flos": 25776715582080.0, + "grad_norm": 2.5468409960366754, + "language_loss": 0.71799982, + "learning_rate": 3.872422815215589e-06, + "loss": 0.74069118, + "num_input_tokens_seen": 24978330, + "router_z_loss_clip": 1.4453125, + "router_z_loss_mlp": 0.2545166, + "step": 1172, + "time_per_iteration": 2.8510689735412598 + }, + { + "auxiliary_loss_clip": 0.01212656, + "auxiliary_loss_mlp": 0.01071113, + "balance_loss_clip": 1.06500983, + "balance_loss_mlp": 1.04615068, + "epoch": 0.1410449107196537, + "flos": 21868521217920.0, + "grad_norm": 1.840241445014997, + "language_loss": 0.74525833, + "learning_rate": 3.8721489134803994e-06, + "loss": 0.76809597, + "num_input_tokens_seen": 24997120, + "router_z_loss_clip": 1.47558594, + "router_z_loss_mlp": 0.24963379, + "step": 1173, + "time_per_iteration": 3.6997201442718506 + }, + { + "auxiliary_loss_clip": 0.01205875, + "auxiliary_loss_mlp": 0.01049769, + "balance_loss_clip": 1.06270659, + "balance_loss_mlp": 1.02555704, + "epoch": 0.1411651536102928, + "flos": 16684133564160.0, + "grad_norm": 3.266024658691896, + "language_loss": 0.72775573, + "learning_rate": 3.871874727741707e-06, + "loss": 0.75031221, + "num_input_tokens_seen": 25014350, + "router_z_loss_clip": 1.43164062, + "router_z_loss_mlp": 0.24206543, + "step": 1174, + "time_per_iteration": 3.8471944332122803 + }, + { + "auxiliary_loss_clip": 0.01207245, + "auxiliary_loss_mlp": 0.0105761, + "balance_loss_clip": 1.07062817, + "balance_loss_mlp": 1.03463817, + "epoch": 0.1412853965009319, + "flos": 20992264934400.0, + "grad_norm": 1.8552107688229966, + "language_loss": 0.96808171, + "learning_rate": 3.871600258041108e-06, + "loss": 0.99073029, + "num_input_tokens_seen": 25033875, + "router_z_loss_clip": 1.36523438, + "router_z_loss_mlp": 0.2298584, + "step": 1175, + "time_per_iteration": 3.759263753890991 + }, + { + "auxiliary_loss_clip": 0.01213421, + "auxiliary_loss_mlp": 0.0105421, + "balance_loss_clip": 1.06964231, + "balance_loss_mlp": 1.03047526, + "epoch": 0.14140563939157097, + "flos": 20335279224960.0, + "grad_norm": 2.871255546078813, + "language_loss": 0.85504365, + "learning_rate": 3.871325504420238e-06, + "loss": 0.87772, + "num_input_tokens_seen": 25052865, + "router_z_loss_clip": 1.43945312, + "router_z_loss_mlp": 0.23742676, + "step": 1176, + "time_per_iteration": 3.0849268436431885 + }, + { + "auxiliary_loss_clip": 0.01204279, + "auxiliary_loss_mlp": 0.01054652, + "balance_loss_clip": 1.06412148, + "balance_loss_mlp": 1.03171611, + "epoch": 0.14152588228221005, + "flos": 21068826773760.0, + "grad_norm": 1.8502241104088286, + "language_loss": 0.8127144, + "learning_rate": 3.871050466920776e-06, + "loss": 0.83530372, + "num_input_tokens_seen": 25072770, + "router_z_loss_clip": 1.40136719, + "router_z_loss_mlp": 0.22924805, + "step": 1177, + "time_per_iteration": 2.907531976699829 + }, + { + "auxiliary_loss_clip": 0.01202405, + "auxiliary_loss_mlp": 0.01061701, + "balance_loss_clip": 1.06466532, + "balance_loss_mlp": 1.03701293, + "epoch": 0.14164612517284916, + "flos": 18223157646720.0, + "grad_norm": 1.9210209106191733, + "language_loss": 0.79933858, + "learning_rate": 3.870775145584447e-06, + "loss": 0.82197964, + "num_input_tokens_seen": 25090550, + "router_z_loss_clip": 1.37695312, + "router_z_loss_mlp": 0.24707031, + "step": 1178, + "time_per_iteration": 2.835472345352173 + }, + { + "auxiliary_loss_clip": 0.01216546, + "auxiliary_loss_mlp": 0.01064992, + "balance_loss_clip": 1.06662631, + "balance_loss_mlp": 1.03880167, + "epoch": 0.14176636806348825, + "flos": 22744454279040.0, + "grad_norm": 3.557948779619692, + "language_loss": 0.64861059, + "learning_rate": 3.8704995404530145e-06, + "loss": 0.671426, + "num_input_tokens_seen": 25106175, + "router_z_loss_clip": 1.49804688, + "router_z_loss_mlp": 0.26196289, + "step": 1179, + "time_per_iteration": 2.921135663986206 + }, + { + "auxiliary_loss_clip": 0.01202815, + "auxiliary_loss_mlp": 0.01057286, + "balance_loss_clip": 1.06512499, + "balance_loss_mlp": 1.03468382, + "epoch": 0.14188661095412733, + "flos": 22091095843200.0, + "grad_norm": 1.9044920526599471, + "language_loss": 0.8496263, + "learning_rate": 3.87022365156829e-06, + "loss": 0.87222725, + "num_input_tokens_seen": 25126890, + "router_z_loss_clip": 1.375, + "router_z_loss_mlp": 0.22607422, + "step": 1180, + "time_per_iteration": 2.8670814037323 + }, + { + "auxiliary_loss_clip": 0.01203788, + "auxiliary_loss_mlp": 0.01057851, + "balance_loss_clip": 1.06487918, + "balance_loss_mlp": 1.03261471, + "epoch": 0.14200685384476644, + "flos": 24352390604160.0, + "grad_norm": 2.0934149990238584, + "language_loss": 0.80929542, + "learning_rate": 3.869947478972123e-06, + "loss": 0.8319118, + "num_input_tokens_seen": 25147915, + "router_z_loss_clip": 1.38964844, + "router_z_loss_mlp": 0.25231934, + "step": 1181, + "time_per_iteration": 2.8630683422088623 + }, + { + "auxiliary_loss_clip": 0.01207648, + "auxiliary_loss_mlp": 0.01060587, + "balance_loss_clip": 1.06620896, + "balance_loss_mlp": 1.03468239, + "epoch": 0.14212709673540552, + "flos": 24022048199040.0, + "grad_norm": 2.4451299170614242, + "language_loss": 0.82510537, + "learning_rate": 3.869671022706412e-06, + "loss": 0.84778774, + "num_input_tokens_seen": 25166645, + "router_z_loss_clip": 1.41601562, + "router_z_loss_mlp": 0.2590332, + "step": 1182, + "time_per_iteration": 2.8971989154815674 + }, + { + "auxiliary_loss_clip": 0.01207025, + "auxiliary_loss_mlp": 0.01062226, + "balance_loss_clip": 1.06764507, + "balance_loss_mlp": 1.03789568, + "epoch": 0.1422473396260446, + "flos": 26431797870720.0, + "grad_norm": 2.0692419047091173, + "language_loss": 0.64910865, + "learning_rate": 3.869394282813092e-06, + "loss": 0.67180115, + "num_input_tokens_seen": 25185845, + "router_z_loss_clip": 1.39453125, + "router_z_loss_mlp": 0.24328613, + "step": 1183, + "time_per_iteration": 2.8834598064422607 + }, + { + "auxiliary_loss_clip": 0.01210169, + "auxiliary_loss_mlp": 0.0106377, + "balance_loss_clip": 1.06668365, + "balance_loss_mlp": 1.03895044, + "epoch": 0.1423675825166837, + "flos": 17055306754560.0, + "grad_norm": 2.414604389630668, + "language_loss": 0.89196134, + "learning_rate": 3.869117259334147e-06, + "loss": 0.91470069, + "num_input_tokens_seen": 25203770, + "router_z_loss_clip": 1.43652344, + "router_z_loss_mlp": 0.24816895, + "step": 1184, + "time_per_iteration": 2.884676218032837 + }, + { + "auxiliary_loss_clip": 0.01203238, + "auxiliary_loss_mlp": 0.01066838, + "balance_loss_clip": 1.06480944, + "balance_loss_mlp": 1.04172063, + "epoch": 0.1424878254073228, + "flos": 17929480049280.0, + "grad_norm": 1.7474811084523891, + "language_loss": 0.81818652, + "learning_rate": 3.868839952311599e-06, + "loss": 0.84088725, + "num_input_tokens_seen": 25221725, + "router_z_loss_clip": 1.38378906, + "router_z_loss_mlp": 0.2512207, + "step": 1185, + "time_per_iteration": 2.853506565093994 + }, + { + "auxiliary_loss_clip": 0.0120475, + "auxiliary_loss_mlp": 0.01056386, + "balance_loss_clip": 1.06437278, + "balance_loss_mlp": 1.03345013, + "epoch": 0.14260806829796188, + "flos": 20303606407680.0, + "grad_norm": 2.732250633091025, + "language_loss": 0.8024472, + "learning_rate": 3.868562361787516e-06, + "loss": 0.82505858, + "num_input_tokens_seen": 25240855, + "router_z_loss_clip": 1.40527344, + "router_z_loss_mlp": 0.22949219, + "step": 1186, + "time_per_iteration": 2.8000893592834473 + }, + { + "auxiliary_loss_clip": 0.01203645, + "auxiliary_loss_mlp": 0.01057036, + "balance_loss_clip": 1.06508327, + "balance_loss_mlp": 1.03317022, + "epoch": 0.14272831118860096, + "flos": 23185724860800.0, + "grad_norm": 2.1118386300073815, + "language_loss": 0.68982059, + "learning_rate": 3.868284487804009e-06, + "loss": 0.71242744, + "num_input_tokens_seen": 25260085, + "router_z_loss_clip": 1.38476562, + "router_z_loss_mlp": 0.23876953, + "step": 1187, + "time_per_iteration": 2.8877205848693848 + }, + { + "auxiliary_loss_clip": 0.01201198, + "auxiliary_loss_mlp": 0.01056847, + "balance_loss_clip": 1.06516957, + "balance_loss_mlp": 1.03187251, + "epoch": 0.14284855407924008, + "flos": 27232210586880.0, + "grad_norm": 1.6841087156638508, + "language_loss": 0.78110546, + "learning_rate": 3.86800633040323e-06, + "loss": 0.8036859, + "num_input_tokens_seen": 25280675, + "router_z_loss_clip": 1.36035156, + "router_z_loss_mlp": 0.24987793, + "step": 1188, + "time_per_iteration": 2.84309983253479 + }, + { + "auxiliary_loss_clip": 0.01209318, + "auxiliary_loss_mlp": 0.01074506, + "balance_loss_clip": 1.06864166, + "balance_loss_mlp": 1.04950774, + "epoch": 0.14296879696987916, + "flos": 28184202696960.0, + "grad_norm": 2.9343117705325676, + "language_loss": 0.7781508, + "learning_rate": 3.867727889627376e-06, + "loss": 0.80098903, + "num_input_tokens_seen": 25300290, + "router_z_loss_clip": 1.40917969, + "router_z_loss_mlp": 0.25, + "step": 1189, + "time_per_iteration": 2.8287501335144043 + }, + { + "auxiliary_loss_clip": 0.01208302, + "auxiliary_loss_mlp": 0.01064861, + "balance_loss_clip": 1.06568277, + "balance_loss_mlp": 1.04136443, + "epoch": 0.14308903986051824, + "flos": 19390290266880.0, + "grad_norm": 3.2920827796732306, + "language_loss": 0.78354782, + "learning_rate": 3.867449165518687e-06, + "loss": 0.80627942, + "num_input_tokens_seen": 25316760, + "router_z_loss_clip": 1.42871094, + "router_z_loss_mlp": 0.23522949, + "step": 1190, + "time_per_iteration": 2.8062264919281006 + }, + { + "auxiliary_loss_clip": 0.01209031, + "auxiliary_loss_mlp": 0.01058273, + "balance_loss_clip": 1.06484056, + "balance_loss_mlp": 1.03319097, + "epoch": 0.14320928275115732, + "flos": 17457506317440.0, + "grad_norm": 1.8390101149407387, + "language_loss": 0.71310985, + "learning_rate": 3.867170158119444e-06, + "loss": 0.73578286, + "num_input_tokens_seen": 25335760, + "router_z_loss_clip": 1.44335938, + "router_z_loss_mlp": 0.25073242, + "step": 1191, + "time_per_iteration": 2.862248182296753 + }, + { + "auxiliary_loss_clip": 0.01211498, + "auxiliary_loss_mlp": 0.01062796, + "balance_loss_clip": 1.06509805, + "balance_loss_mlp": 1.03776157, + "epoch": 0.14332952564179643, + "flos": 21466070259840.0, + "grad_norm": 2.002590113401848, + "language_loss": 0.75603008, + "learning_rate": 3.866890867471972e-06, + "loss": 0.77877301, + "num_input_tokens_seen": 25354230, + "router_z_loss_clip": 1.46484375, + "router_z_loss_mlp": 0.25036621, + "step": 1192, + "time_per_iteration": 2.8435757160186768 + }, + { + "auxiliary_loss_clip": 0.01215736, + "auxiliary_loss_mlp": 0.01066056, + "balance_loss_clip": 1.06877851, + "balance_loss_mlp": 1.04061627, + "epoch": 0.14344976853243552, + "flos": 16396992241920.0, + "grad_norm": 2.5432777359704772, + "language_loss": 0.89523256, + "learning_rate": 3.86661129361864e-06, + "loss": 0.91805053, + "num_input_tokens_seen": 25368720, + "router_z_loss_clip": 1.46972656, + "router_z_loss_mlp": 0.25427246, + "step": 1193, + "time_per_iteration": 2.767582416534424 + }, + { + "auxiliary_loss_clip": 0.01211373, + "auxiliary_loss_mlp": 0.0106442, + "balance_loss_clip": 1.06690121, + "balance_loss_mlp": 1.04013729, + "epoch": 0.1435700114230746, + "flos": 18916736336640.0, + "grad_norm": 1.8764240978676647, + "language_loss": 0.85799682, + "learning_rate": 3.866331436601859e-06, + "loss": 0.88075477, + "num_input_tokens_seen": 25386715, + "router_z_loss_clip": 1.4453125, + "router_z_loss_mlp": 0.24279785, + "step": 1194, + "time_per_iteration": 2.847517967224121 + }, + { + "auxiliary_loss_clip": 0.0121085, + "auxiliary_loss_mlp": 0.0106002, + "balance_loss_clip": 1.0699327, + "balance_loss_mlp": 1.03440166, + "epoch": 0.1436902543137137, + "flos": 19755394058880.0, + "grad_norm": 2.1318456665148653, + "language_loss": 0.73461837, + "learning_rate": 3.866051296464083e-06, + "loss": 0.75732708, + "num_input_tokens_seen": 25405550, + "router_z_loss_clip": 1.40722656, + "router_z_loss_mlp": 0.25598145, + "step": 1195, + "time_per_iteration": 2.8551948070526123 + }, + { + "auxiliary_loss_clip": 0.01210758, + "auxiliary_loss_mlp": 0.01057748, + "balance_loss_clip": 1.0664531, + "balance_loss_mlp": 1.03339362, + "epoch": 0.1438104972043528, + "flos": 14684807669760.0, + "grad_norm": 2.2337975074728162, + "language_loss": 0.85437018, + "learning_rate": 3.86577087324781e-06, + "loss": 0.87705517, + "num_input_tokens_seen": 25422040, + "router_z_loss_clip": 1.44140625, + "router_z_loss_mlp": 0.24377441, + "step": 1196, + "time_per_iteration": 2.8463590145111084 + }, + { + "auxiliary_loss_clip": 0.01206309, + "auxiliary_loss_mlp": 0.01062628, + "balance_loss_clip": 1.06941295, + "balance_loss_mlp": 1.03958511, + "epoch": 0.14393074009499188, + "flos": 17092330698240.0, + "grad_norm": 9.661847890435425, + "language_loss": 0.77886903, + "learning_rate": 3.865490166995578e-06, + "loss": 0.80155843, + "num_input_tokens_seen": 25440270, + "router_z_loss_clip": 1.37011719, + "router_z_loss_mlp": 0.23022461, + "step": 1197, + "time_per_iteration": 2.8399112224578857 + }, + { + "auxiliary_loss_clip": 0.01206719, + "auxiliary_loss_mlp": 0.01054157, + "balance_loss_clip": 1.0655911, + "balance_loss_mlp": 1.0295639, + "epoch": 0.144050982985631, + "flos": 30476200608000.0, + "grad_norm": 2.6814726644758253, + "language_loss": 0.84418619, + "learning_rate": 3.86520917774997e-06, + "loss": 0.86679494, + "num_input_tokens_seen": 25459705, + "router_z_loss_clip": 1.41308594, + "router_z_loss_mlp": 0.24584961, + "step": 1198, + "time_per_iteration": 2.96569561958313 + }, + { + "auxiliary_loss_clip": 0.01205963, + "auxiliary_loss_mlp": 0.01063803, + "balance_loss_clip": 1.06617808, + "balance_loss_mlp": 1.03882849, + "epoch": 0.14417122587627007, + "flos": 17858484817920.0, + "grad_norm": 2.9284776157257983, + "language_loss": 0.75558317, + "learning_rate": 3.864927905553614e-06, + "loss": 0.77828085, + "num_input_tokens_seen": 25477615, + "router_z_loss_clip": 1.39648438, + "router_z_loss_mlp": 0.24963379, + "step": 1199, + "time_per_iteration": 2.850595712661743 + }, + { + "auxiliary_loss_clip": 0.01203686, + "auxiliary_loss_mlp": 0.01061891, + "balance_loss_clip": 1.06497395, + "balance_loss_mlp": 1.03520012, + "epoch": 0.14429146876690915, + "flos": 21613914639360.0, + "grad_norm": 1.6282023237136931, + "language_loss": 0.88976175, + "learning_rate": 3.8646463504491765e-06, + "loss": 0.91241753, + "num_input_tokens_seen": 25497750, + "router_z_loss_clip": 1.38671875, + "router_z_loss_mlp": 0.26696777, + "step": 1200, + "time_per_iteration": 2.8369486331939697 + }, + { + "auxiliary_loss_clip": 0.01205272, + "auxiliary_loss_mlp": 0.01060697, + "balance_loss_clip": 1.06529045, + "balance_loss_mlp": 1.03553224, + "epoch": 0.14441171165754824, + "flos": 23258120722560.0, + "grad_norm": 1.7441725034189999, + "language_loss": 0.83423269, + "learning_rate": 3.8643645124793705e-06, + "loss": 0.85689235, + "num_input_tokens_seen": 25516650, + "router_z_loss_clip": 1.39941406, + "router_z_loss_mlp": 0.25195312, + "step": 1201, + "time_per_iteration": 2.8349874019622803 + }, + { + "auxiliary_loss_clip": 0.01206165, + "auxiliary_loss_mlp": 0.01053709, + "balance_loss_clip": 1.06727898, + "balance_loss_mlp": 1.03011751, + "epoch": 0.14453195454818735, + "flos": 42854213963520.0, + "grad_norm": 1.7184908930334668, + "language_loss": 0.74780941, + "learning_rate": 3.8640823916869515e-06, + "loss": 0.77040815, + "num_input_tokens_seen": 25540960, + "router_z_loss_clip": 1.38964844, + "router_z_loss_mlp": 0.23596191, + "step": 1202, + "time_per_iteration": 4.604949474334717 + }, + { + "auxiliary_loss_clip": 0.01204581, + "auxiliary_loss_mlp": 0.01057234, + "balance_loss_clip": 1.06582475, + "balance_loss_mlp": 1.03209233, + "epoch": 0.14465219743882643, + "flos": 27235873774080.0, + "grad_norm": 1.530505749464229, + "language_loss": 0.78269696, + "learning_rate": 3.863799988114714e-06, + "loss": 0.80531514, + "num_input_tokens_seen": 25562990, + "router_z_loss_clip": 1.38867188, + "router_z_loss_mlp": 0.25146484, + "step": 1203, + "time_per_iteration": 3.881486654281616 + }, + { + "auxiliary_loss_clip": 0.01210097, + "auxiliary_loss_mlp": 0.01068413, + "balance_loss_clip": 1.06580544, + "balance_loss_mlp": 1.04308116, + "epoch": 0.1447724403294655, + "flos": 16690705752960.0, + "grad_norm": 2.6476761555006245, + "language_loss": 0.70565987, + "learning_rate": 3.863517301805502e-06, + "loss": 0.72844493, + "num_input_tokens_seen": 25581380, + "router_z_loss_clip": 1.44335938, + "router_z_loss_mlp": 0.25354004, + "step": 1204, + "time_per_iteration": 2.781473159790039 + }, + { + "auxiliary_loss_clip": 0.0120866, + "auxiliary_loss_mlp": 0.01065759, + "balance_loss_clip": 1.06814599, + "balance_loss_mlp": 1.04144061, + "epoch": 0.14489268322010462, + "flos": 20073741321600.0, + "grad_norm": 2.7639762380150485, + "language_loss": 0.97002161, + "learning_rate": 3.863234332802196e-06, + "loss": 0.99276578, + "num_input_tokens_seen": 25593585, + "router_z_loss_clip": 1.40527344, + "router_z_loss_mlp": 0.24316406, + "step": 1205, + "time_per_iteration": 2.7641730308532715 + }, + { + "auxiliary_loss_clip": 0.01208385, + "auxiliary_loss_mlp": 0.01061761, + "balance_loss_clip": 1.06784606, + "balance_loss_mlp": 1.03683496, + "epoch": 0.1450129261107437, + "flos": 27125627955840.0, + "grad_norm": 2.0864073565155516, + "language_loss": 0.73588181, + "learning_rate": 3.862951081147723e-06, + "loss": 0.75858331, + "num_input_tokens_seen": 25613750, + "router_z_loss_clip": 1.40527344, + "router_z_loss_mlp": 0.24914551, + "step": 1206, + "time_per_iteration": 2.8126518726348877 + }, + { + "auxiliary_loss_clip": 0.01200199, + "auxiliary_loss_mlp": 0.01056815, + "balance_loss_clip": 1.06570923, + "balance_loss_mlp": 1.03467834, + "epoch": 0.1451331690013828, + "flos": 25702344472320.0, + "grad_norm": 2.5900995781778224, + "language_loss": 0.78457868, + "learning_rate": 3.862667546885053e-06, + "loss": 0.80714881, + "num_input_tokens_seen": 25632300, + "router_z_loss_clip": 1.34472656, + "router_z_loss_mlp": 0.22131348, + "step": 1207, + "time_per_iteration": 2.86525297164917 + }, + { + "auxiliary_loss_clip": 0.01207358, + "auxiliary_loss_mlp": 0.01060363, + "balance_loss_clip": 1.06510055, + "balance_loss_mlp": 1.03579354, + "epoch": 0.14525341189202187, + "flos": 25737393168000.0, + "grad_norm": 2.774253143494321, + "language_loss": 0.73297918, + "learning_rate": 3.8623837300571965e-06, + "loss": 0.75565636, + "num_input_tokens_seen": 25651285, + "router_z_loss_clip": 1.42285156, + "router_z_loss_mlp": 0.24572754, + "step": 1208, + "time_per_iteration": 2.8900673389434814 + }, + { + "auxiliary_loss_clip": 0.01204456, + "auxiliary_loss_mlp": 0.01063983, + "balance_loss_clip": 1.06425691, + "balance_loss_mlp": 1.03959274, + "epoch": 0.14537365478266098, + "flos": 23073898844160.0, + "grad_norm": 1.8666418534014506, + "language_loss": 0.84235835, + "learning_rate": 3.8620996307072085e-06, + "loss": 0.86504275, + "num_input_tokens_seen": 25671990, + "router_z_loss_clip": 1.40234375, + "router_z_loss_mlp": 0.24401855, + "step": 1209, + "time_per_iteration": 2.8420019149780273 + }, + { + "auxiliary_loss_clip": 0.01205776, + "auxiliary_loss_mlp": 0.01063814, + "balance_loss_clip": 1.0670507, + "balance_loss_mlp": 1.03999543, + "epoch": 0.14549389767330007, + "flos": 20595021448320.0, + "grad_norm": 1.9163588922114754, + "language_loss": 0.64592075, + "learning_rate": 3.861815248878188e-06, + "loss": 0.66861665, + "num_input_tokens_seen": 25689475, + "router_z_loss_clip": 1.38671875, + "router_z_loss_mlp": 0.23828125, + "step": 1210, + "time_per_iteration": 2.8131613731384277 + }, + { + "auxiliary_loss_clip": 0.0120532, + "auxiliary_loss_mlp": 0.01060884, + "balance_loss_clip": 1.06583107, + "balance_loss_mlp": 1.03650606, + "epoch": 0.14561414056393915, + "flos": 15121804533120.0, + "grad_norm": 2.3194531725796703, + "language_loss": 0.79817432, + "learning_rate": 3.861530584613274e-06, + "loss": 0.82083642, + "num_input_tokens_seen": 25707475, + "router_z_loss_clip": 1.39453125, + "router_z_loss_mlp": 0.24389648, + "step": 1211, + "time_per_iteration": 2.747100830078125 + }, + { + "auxiliary_loss_clip": 0.01214886, + "auxiliary_loss_mlp": 0.01057189, + "balance_loss_clip": 1.06795287, + "balance_loss_mlp": 1.03051019, + "epoch": 0.14573438345457826, + "flos": 19427493778560.0, + "grad_norm": 2.5352528629025235, + "language_loss": 0.82550895, + "learning_rate": 3.86124563795565e-06, + "loss": 0.84822977, + "num_input_tokens_seen": 25726290, + "router_z_loss_clip": 1.47070312, + "router_z_loss_mlp": 0.26708984, + "step": 1212, + "time_per_iteration": 2.8200695514678955 + }, + { + "auxiliary_loss_clip": 0.01206333, + "auxiliary_loss_mlp": 0.01060157, + "balance_loss_clip": 1.06742513, + "balance_loss_mlp": 1.03466964, + "epoch": 0.14585462634521734, + "flos": 24828422572800.0, + "grad_norm": 1.9081965287191482, + "language_loss": 0.69824874, + "learning_rate": 3.860960408948543e-06, + "loss": 0.72091371, + "num_input_tokens_seen": 25748040, + "router_z_loss_clip": 1.38769531, + "router_z_loss_mlp": 0.25512695, + "step": 1213, + "time_per_iteration": 2.8474042415618896 + }, + { + "auxiliary_loss_clip": 0.01200873, + "auxiliary_loss_mlp": 0.0106028, + "balance_loss_clip": 1.06711805, + "balance_loss_mlp": 1.03772593, + "epoch": 0.14597486923585642, + "flos": 15448627405440.0, + "grad_norm": 2.359101324695109, + "language_loss": 0.89682865, + "learning_rate": 3.860674897635222e-06, + "loss": 0.91944027, + "num_input_tokens_seen": 25764525, + "router_z_loss_clip": 1.33789062, + "router_z_loss_mlp": 0.22558594, + "step": 1214, + "time_per_iteration": 2.8958122730255127 + }, + { + "auxiliary_loss_clip": 0.01212599, + "auxiliary_loss_mlp": 0.01066255, + "balance_loss_clip": 1.07043505, + "balance_loss_mlp": 1.04154277, + "epoch": 0.1460951121264955, + "flos": 16655154266880.0, + "grad_norm": 3.969560702123191, + "language_loss": 0.8375231, + "learning_rate": 3.860389104058998e-06, + "loss": 0.86031163, + "num_input_tokens_seen": 25782755, + "router_z_loss_clip": 1.421875, + "router_z_loss_mlp": 0.24731445, + "step": 1215, + "time_per_iteration": 2.916810989379883 + }, + { + "auxiliary_loss_clip": 0.01200472, + "auxiliary_loss_mlp": 0.01057011, + "balance_loss_clip": 1.06218374, + "balance_loss_mlp": 1.03339577, + "epoch": 0.14621535501713462, + "flos": 24863291700480.0, + "grad_norm": 2.913812370950784, + "language_loss": 0.72599924, + "learning_rate": 3.860103028263227e-06, + "loss": 0.74857414, + "num_input_tokens_seen": 25805860, + "router_z_loss_clip": 1.38574219, + "router_z_loss_mlp": 0.23608398, + "step": 1216, + "time_per_iteration": 2.9216835498809814 + }, + { + "auxiliary_loss_clip": 0.01203153, + "auxiliary_loss_mlp": 0.0105514, + "balance_loss_clip": 1.06479859, + "balance_loss_mlp": 1.03060699, + "epoch": 0.1463355979077737, + "flos": 25228000442880.0, + "grad_norm": 2.2940344311119722, + "language_loss": 0.6966657, + "learning_rate": 3.859816670291304e-06, + "loss": 0.71924865, + "num_input_tokens_seen": 25824955, + "router_z_loss_clip": 1.38183594, + "router_z_loss_mlp": 0.24523926, + "step": 1217, + "time_per_iteration": 2.9214212894439697 + }, + { + "auxiliary_loss_clip": 0.01208676, + "auxiliary_loss_mlp": 0.01057773, + "balance_loss_clip": 1.06558037, + "balance_loss_mlp": 1.03424144, + "epoch": 0.14645584079841278, + "flos": 22054143726720.0, + "grad_norm": 2.513615583718106, + "language_loss": 0.89577937, + "learning_rate": 3.859530030186672e-06, + "loss": 0.91844392, + "num_input_tokens_seen": 25841965, + "router_z_loss_clip": 1.43066406, + "router_z_loss_mlp": 0.23535156, + "step": 1218, + "time_per_iteration": 2.879112720489502 + }, + { + "auxiliary_loss_clip": 0.01207844, + "auxiliary_loss_mlp": 0.01060769, + "balance_loss_clip": 1.06857002, + "balance_loss_mlp": 1.03583074, + "epoch": 0.1465760836890519, + "flos": 23623870959360.0, + "grad_norm": 2.3572804162156835, + "language_loss": 0.82943571, + "learning_rate": 3.859243107992813e-06, + "loss": 0.85212189, + "num_input_tokens_seen": 25860770, + "router_z_loss_clip": 1.39355469, + "router_z_loss_mlp": 0.24926758, + "step": 1219, + "time_per_iteration": 2.942093849182129 + }, + { + "auxiliary_loss_clip": 0.01210551, + "auxiliary_loss_mlp": 0.01062432, + "balance_loss_clip": 1.06538105, + "balance_loss_mlp": 1.03636146, + "epoch": 0.14669632657969098, + "flos": 37407893356800.0, + "grad_norm": 2.753319432140205, + "language_loss": 0.78123069, + "learning_rate": 3.858955903753252e-06, + "loss": 0.80396056, + "num_input_tokens_seen": 25879410, + "router_z_loss_clip": 1.453125, + "router_z_loss_mlp": 0.26086426, + "step": 1220, + "time_per_iteration": 3.0154988765716553 + }, + { + "auxiliary_loss_clip": 0.01203491, + "auxiliary_loss_mlp": 0.01051815, + "balance_loss_clip": 1.06692004, + "balance_loss_mlp": 1.02756763, + "epoch": 0.14681656947033006, + "flos": 28365910623360.0, + "grad_norm": 1.6240534276224605, + "language_loss": 0.83529377, + "learning_rate": 3.858668417511559e-06, + "loss": 0.85784686, + "num_input_tokens_seen": 25902160, + "router_z_loss_clip": 1.36523438, + "router_z_loss_mlp": 0.24230957, + "step": 1221, + "time_per_iteration": 2.952516555786133 + }, + { + "auxiliary_loss_clip": 0.01212287, + "auxiliary_loss_mlp": 0.01054143, + "balance_loss_clip": 1.07047975, + "balance_loss_mlp": 1.03003883, + "epoch": 0.14693681236096917, + "flos": 18479488078080.0, + "grad_norm": 2.1730089758732776, + "language_loss": 0.76328433, + "learning_rate": 3.8583806493113445e-06, + "loss": 0.78594863, + "num_input_tokens_seen": 25920505, + "router_z_loss_clip": 1.41796875, + "router_z_loss_mlp": 0.2409668, + "step": 1222, + "time_per_iteration": 2.8886029720306396 + }, + { + "auxiliary_loss_clip": 0.01206073, + "auxiliary_loss_mlp": 0.01057934, + "balance_loss_clip": 1.0652616, + "balance_loss_mlp": 1.03353214, + "epoch": 0.14705705525160825, + "flos": 20777806782720.0, + "grad_norm": 2.286215286115702, + "language_loss": 0.8233704, + "learning_rate": 3.858092599196263e-06, + "loss": 0.84601045, + "num_input_tokens_seen": 25938460, + "router_z_loss_clip": 1.40820312, + "router_z_loss_mlp": 0.24389648, + "step": 1223, + "time_per_iteration": 2.7576630115509033 + }, + { + "auxiliary_loss_clip": 0.01207693, + "auxiliary_loss_mlp": 0.01056199, + "balance_loss_clip": 1.06548977, + "balance_loss_mlp": 1.03252435, + "epoch": 0.14717729814224734, + "flos": 29932944336000.0, + "grad_norm": 2.6159080945090056, + "language_loss": 0.82413864, + "learning_rate": 3.857804267210012e-06, + "loss": 0.8467775, + "num_input_tokens_seen": 25957760, + "router_z_loss_clip": 1.421875, + "router_z_loss_mlp": 0.23669434, + "step": 1224, + "time_per_iteration": 2.8350541591644287 + }, + { + "auxiliary_loss_clip": 0.01203531, + "auxiliary_loss_mlp": 0.01057623, + "balance_loss_clip": 1.0658896, + "balance_loss_mlp": 1.03285122, + "epoch": 0.14729754103288642, + "flos": 20047491457920.0, + "grad_norm": 2.3677369960766623, + "language_loss": 0.8819766, + "learning_rate": 3.857515653396331e-06, + "loss": 0.90458822, + "num_input_tokens_seen": 25974970, + "router_z_loss_clip": 1.37597656, + "router_z_loss_mlp": 0.24743652, + "step": 1225, + "time_per_iteration": 2.7837963104248047 + }, + { + "auxiliary_loss_clip": 0.01205364, + "auxiliary_loss_mlp": 0.01052121, + "balance_loss_clip": 1.06693089, + "balance_loss_mlp": 1.02944708, + "epoch": 0.14741778392352553, + "flos": 19281516906240.0, + "grad_norm": 2.366879054706266, + "language_loss": 0.86926961, + "learning_rate": 3.857226757799002e-06, + "loss": 0.89184451, + "num_input_tokens_seen": 25992525, + "router_z_loss_clip": 1.38476562, + "router_z_loss_mlp": 0.22668457, + "step": 1226, + "time_per_iteration": 2.7766904830932617 + }, + { + "auxiliary_loss_clip": 0.01208891, + "auxiliary_loss_mlp": 0.01057244, + "balance_loss_clip": 1.06627178, + "balance_loss_mlp": 1.03210235, + "epoch": 0.1475380268141646, + "flos": 25411108999680.0, + "grad_norm": 2.4361287260085573, + "language_loss": 0.74719858, + "learning_rate": 3.85693758046185e-06, + "loss": 0.76985991, + "num_input_tokens_seen": 26010815, + "router_z_loss_clip": 1.42578125, + "router_z_loss_mlp": 0.25158691, + "step": 1227, + "time_per_iteration": 2.846977710723877 + }, + { + "auxiliary_loss_clip": 0.01209438, + "auxiliary_loss_mlp": 0.01067348, + "balance_loss_clip": 1.07064664, + "balance_loss_mlp": 1.04430509, + "epoch": 0.1476582697048037, + "flos": 20847652778880.0, + "grad_norm": 3.71369567267383, + "language_loss": 0.8260814, + "learning_rate": 3.8566481214287435e-06, + "loss": 0.8488493, + "num_input_tokens_seen": 26028935, + "router_z_loss_clip": 1.38867188, + "router_z_loss_mlp": 0.23034668, + "step": 1228, + "time_per_iteration": 2.9423062801361084 + }, + { + "auxiliary_loss_clip": 0.01201732, + "auxiliary_loss_mlp": 0.01065165, + "balance_loss_clip": 1.06671786, + "balance_loss_mlp": 1.0409894, + "epoch": 0.1477785125954428, + "flos": 14028109269120.0, + "grad_norm": 2.8433063850368026, + "language_loss": 0.90843028, + "learning_rate": 3.8563583807435935e-06, + "loss": 0.93109924, + "num_input_tokens_seen": 26045080, + "router_z_loss_clip": 1.34960938, + "router_z_loss_mlp": 0.24194336, + "step": 1229, + "time_per_iteration": 2.879518747329712 + }, + { + "auxiliary_loss_clip": 0.01210096, + "auxiliary_loss_mlp": 0.01059582, + "balance_loss_clip": 1.06516457, + "balance_loss_mlp": 1.03439283, + "epoch": 0.1478987554860819, + "flos": 20516699842560.0, + "grad_norm": 2.075865883188652, + "language_loss": 0.77587569, + "learning_rate": 3.856068358450353e-06, + "loss": 0.79857254, + "num_input_tokens_seen": 26065030, + "router_z_loss_clip": 1.44921875, + "router_z_loss_mlp": 0.25195312, + "step": 1230, + "time_per_iteration": 2.870284080505371 + }, + { + "auxiliary_loss_clip": 0.01203092, + "auxiliary_loss_mlp": 0.01060604, + "balance_loss_clip": 1.06618071, + "balance_loss_mlp": 1.03670251, + "epoch": 0.14801899837672097, + "flos": 17857012360320.0, + "grad_norm": 1.6816228667372657, + "language_loss": 0.85898507, + "learning_rate": 3.8557780545930186e-06, + "loss": 0.88162208, + "num_input_tokens_seen": 26083445, + "router_z_loss_clip": 1.36914062, + "router_z_loss_mlp": 0.23913574, + "step": 1231, + "time_per_iteration": 4.463048696517944 + }, + { + "auxiliary_loss_clip": 0.01208029, + "auxiliary_loss_mlp": 0.01053581, + "balance_loss_clip": 1.06712615, + "balance_loss_mlp": 1.0292027, + "epoch": 0.14813924126736006, + "flos": 20881408584960.0, + "grad_norm": 1.815388195247496, + "language_loss": 0.79310614, + "learning_rate": 3.855487469215628e-06, + "loss": 0.81572223, + "num_input_tokens_seen": 26102375, + "router_z_loss_clip": 1.40917969, + "router_z_loss_mlp": 0.24389648, + "step": 1232, + "time_per_iteration": 3.7211380004882812 + }, + { + "auxiliary_loss_clip": 0.01202319, + "auxiliary_loss_mlp": 0.01053005, + "balance_loss_clip": 1.0654248, + "balance_loss_mlp": 1.02972329, + "epoch": 0.14825948415799917, + "flos": 37414070496000.0, + "grad_norm": 2.381833631109626, + "language_loss": 0.72451293, + "learning_rate": 3.855196602362264e-06, + "loss": 0.74706614, + "num_input_tokens_seen": 26125295, + "router_z_loss_clip": 1.37011719, + "router_z_loss_mlp": 0.23291016, + "step": 1233, + "time_per_iteration": 2.9562888145446777 + }, + { + "auxiliary_loss_clip": 0.01204813, + "auxiliary_loss_mlp": 0.01064362, + "balance_loss_clip": 1.06528926, + "balance_loss_mlp": 1.04032969, + "epoch": 0.14837972704863825, + "flos": 22014641744640.0, + "grad_norm": 2.1026996418633135, + "language_loss": 0.94100595, + "learning_rate": 3.854905454077051e-06, + "loss": 0.96369767, + "num_input_tokens_seen": 26142905, + "router_z_loss_clip": 1.39453125, + "router_z_loss_mlp": 0.2401123, + "step": 1234, + "time_per_iteration": 2.8613474369049072 + }, + { + "auxiliary_loss_clip": 0.012076, + "auxiliary_loss_mlp": 0.01062287, + "balance_loss_clip": 1.0658803, + "balance_loss_mlp": 1.03715742, + "epoch": 0.14849996993927733, + "flos": 20996323171200.0, + "grad_norm": 2.031773959955537, + "language_loss": 0.87980139, + "learning_rate": 3.854614024404155e-06, + "loss": 0.90250027, + "num_input_tokens_seen": 26161215, + "router_z_loss_clip": 1.41601562, + "router_z_loss_mlp": 0.25146484, + "step": 1235, + "time_per_iteration": 2.8245863914489746 + }, + { + "auxiliary_loss_clip": 0.01205989, + "auxiliary_loss_mlp": 0.01057948, + "balance_loss_clip": 1.0673151, + "balance_loss_mlp": 1.03463125, + "epoch": 0.14862021282991644, + "flos": 20047994248320.0, + "grad_norm": 1.98064223156191, + "language_loss": 0.89225388, + "learning_rate": 3.8543223133877865e-06, + "loss": 0.91489327, + "num_input_tokens_seen": 26179810, + "router_z_loss_clip": 1.38476562, + "router_z_loss_mlp": 0.23327637, + "step": 1236, + "time_per_iteration": 2.8047890663146973 + }, + { + "auxiliary_loss_clip": 0.01209403, + "auxiliary_loss_mlp": 0.01063299, + "balance_loss_clip": 1.06724215, + "balance_loss_mlp": 1.03529644, + "epoch": 0.14874045572055553, + "flos": 22712027276160.0, + "grad_norm": 2.1591220875919195, + "language_loss": 0.88197863, + "learning_rate": 3.854030321072198e-06, + "loss": 0.90470564, + "num_input_tokens_seen": 26199715, + "router_z_loss_clip": 1.421875, + "router_z_loss_mlp": 0.27966309, + "step": 1237, + "time_per_iteration": 2.8736495971679688 + }, + { + "auxiliary_loss_clip": 0.0120574, + "auxiliary_loss_mlp": 0.01056992, + "balance_loss_clip": 1.06707788, + "balance_loss_mlp": 1.03252995, + "epoch": 0.1488606986111946, + "flos": 25411288567680.0, + "grad_norm": 2.1262376974471846, + "language_loss": 0.73616379, + "learning_rate": 3.853738047501682e-06, + "loss": 0.75879109, + "num_input_tokens_seen": 26220275, + "router_z_loss_clip": 1.38574219, + "router_z_loss_mlp": 0.24462891, + "step": 1238, + "time_per_iteration": 2.868652820587158 + }, + { + "auxiliary_loss_clip": 0.01210961, + "auxiliary_loss_mlp": 0.0105957, + "balance_loss_clip": 1.0694654, + "balance_loss_mlp": 1.0357995, + "epoch": 0.1489809415018337, + "flos": 17018749687680.0, + "grad_norm": 1.8716629047348594, + "language_loss": 0.7759856, + "learning_rate": 3.85344549272058e-06, + "loss": 0.79869092, + "num_input_tokens_seen": 26238255, + "router_z_loss_clip": 1.41503906, + "router_z_loss_mlp": 0.23791504, + "step": 1239, + "time_per_iteration": 2.991184949874878 + }, + { + "auxiliary_loss_clip": 0.01204991, + "auxiliary_loss_mlp": 0.01059224, + "balance_loss_clip": 1.06596971, + "balance_loss_mlp": 1.03506088, + "epoch": 0.1491011843924728, + "flos": 33659394860160.0, + "grad_norm": 1.7770135524130566, + "language_loss": 0.82067943, + "learning_rate": 3.853152656773269e-06, + "loss": 0.84332156, + "num_input_tokens_seen": 26259690, + "router_z_loss_clip": 1.390625, + "router_z_loss_mlp": 0.24182129, + "step": 1240, + "time_per_iteration": 2.9703471660614014 + }, + { + "auxiliary_loss_clip": 0.01207414, + "auxiliary_loss_mlp": 0.01062049, + "balance_loss_clip": 1.06618714, + "balance_loss_mlp": 1.03733706, + "epoch": 0.14922142728311188, + "flos": 21179000764800.0, + "grad_norm": 1.7772746788505316, + "language_loss": 0.84563535, + "learning_rate": 3.852859539704174e-06, + "loss": 0.86833, + "num_input_tokens_seen": 26278990, + "router_z_loss_clip": 1.41308594, + "router_z_loss_mlp": 0.24707031, + "step": 1241, + "time_per_iteration": 2.8716964721679688 + }, + { + "auxiliary_loss_clip": 0.01202261, + "auxiliary_loss_mlp": 0.01053477, + "balance_loss_clip": 1.06366968, + "balance_loss_mlp": 1.03012371, + "epoch": 0.14934167017375097, + "flos": 29860548474240.0, + "grad_norm": 1.850685972236678, + "language_loss": 0.76442885, + "learning_rate": 3.85256614155776e-06, + "loss": 0.78698623, + "num_input_tokens_seen": 26299120, + "router_z_loss_clip": 1.38476562, + "router_z_loss_mlp": 0.23339844, + "step": 1242, + "time_per_iteration": 2.964948892593384 + }, + { + "auxiliary_loss_clip": 0.01209177, + "auxiliary_loss_mlp": 0.01057002, + "balance_loss_clip": 1.06686139, + "balance_loss_mlp": 1.03164589, + "epoch": 0.14946191306439008, + "flos": 17019216564480.0, + "grad_norm": 2.080135217013069, + "language_loss": 0.74330032, + "learning_rate": 3.852272462378535e-06, + "loss": 0.76596212, + "num_input_tokens_seen": 26316995, + "router_z_loss_clip": 1.421875, + "router_z_loss_mlp": 0.25354004, + "step": 1243, + "time_per_iteration": 2.7652554512023926 + }, + { + "auxiliary_loss_clip": 0.01205973, + "auxiliary_loss_mlp": 0.0105037, + "balance_loss_clip": 1.06677127, + "balance_loss_mlp": 1.02592039, + "epoch": 0.14958215595502916, + "flos": 15669047214720.0, + "grad_norm": 2.2177302934240766, + "language_loss": 0.77796423, + "learning_rate": 3.85197850221105e-06, + "loss": 0.80052757, + "num_input_tokens_seen": 26333295, + "router_z_loss_clip": 1.390625, + "router_z_loss_mlp": 0.24462891, + "step": 1244, + "time_per_iteration": 2.821046829223633 + }, + { + "auxiliary_loss_clip": 0.01201079, + "auxiliary_loss_mlp": 0.01059469, + "balance_loss_clip": 1.06615961, + "balance_loss_mlp": 1.03607988, + "epoch": 0.14970239884566824, + "flos": 33108560818560.0, + "grad_norm": 2.0886193836866527, + "language_loss": 0.75458312, + "learning_rate": 3.851684261099899e-06, + "loss": 0.7771886, + "num_input_tokens_seen": 26355035, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.23388672, + "step": 1245, + "time_per_iteration": 2.863612413406372 + }, + { + "auxiliary_loss_clip": 0.01208593, + "auxiliary_loss_mlp": 0.01052555, + "balance_loss_clip": 1.06755519, + "balance_loss_mlp": 1.02735364, + "epoch": 0.14982264173630733, + "flos": 17821245392640.0, + "grad_norm": 2.0686047918173127, + "language_loss": 0.86418015, + "learning_rate": 3.851389739089718e-06, + "loss": 0.88679159, + "num_input_tokens_seen": 26371655, + "router_z_loss_clip": 1.41113281, + "router_z_loss_mlp": 0.25219727, + "step": 1246, + "time_per_iteration": 2.7789688110351562 + }, + { + "auxiliary_loss_clip": 0.01208317, + "auxiliary_loss_mlp": 0.01058097, + "balance_loss_clip": 1.06643963, + "balance_loss_mlp": 1.03302717, + "epoch": 0.14994288462694644, + "flos": 32409559175040.0, + "grad_norm": 2.253211157524904, + "language_loss": 0.80330777, + "learning_rate": 3.851094936225186e-06, + "loss": 0.8259719, + "num_input_tokens_seen": 26392540, + "router_z_loss_clip": 1.41894531, + "router_z_loss_mlp": 0.25073242, + "step": 1247, + "time_per_iteration": 2.9198479652404785 + }, + { + "auxiliary_loss_clip": 0.01204527, + "auxiliary_loss_mlp": 0.01051694, + "balance_loss_clip": 1.06675279, + "balance_loss_mlp": 1.02782845, + "epoch": 0.15006312751758552, + "flos": 31794661226880.0, + "grad_norm": 1.4692602372813426, + "language_loss": 0.76534963, + "learning_rate": 3.850799852551024e-06, + "loss": 0.78791189, + "num_input_tokens_seen": 26414960, + "router_z_loss_clip": 1.37792969, + "router_z_loss_mlp": 0.23864746, + "step": 1248, + "time_per_iteration": 2.8636975288391113 + }, + { + "auxiliary_loss_clip": 0.01203933, + "auxiliary_loss_mlp": 0.01054428, + "balance_loss_clip": 1.06486225, + "balance_loss_mlp": 1.03002548, + "epoch": 0.1501833704082246, + "flos": 16618022582400.0, + "grad_norm": 2.5620367940465556, + "language_loss": 0.8638202, + "learning_rate": 3.850504488111995e-06, + "loss": 0.8864038, + "num_input_tokens_seen": 26431635, + "router_z_loss_clip": 1.38964844, + "router_z_loss_mlp": 0.24401855, + "step": 1249, + "time_per_iteration": 2.7673580646514893 + }, + { + "auxiliary_loss_clip": 0.01202763, + "auxiliary_loss_mlp": 0.01050095, + "balance_loss_clip": 1.06649446, + "balance_loss_mlp": 1.02791035, + "epoch": 0.15030361329886371, + "flos": 23471178243840.0, + "grad_norm": 2.001789937928704, + "language_loss": 0.82657284, + "learning_rate": 3.850208842952907e-06, + "loss": 0.84910142, + "num_input_tokens_seen": 26450440, + "router_z_loss_clip": 1.36230469, + "router_z_loss_mlp": 0.22192383, + "step": 1250, + "time_per_iteration": 2.891822576522827 + }, + { + "auxiliary_loss_clip": 0.01207902, + "auxiliary_loss_mlp": 0.01062937, + "balance_loss_clip": 1.06620276, + "balance_loss_mlp": 1.03764057, + "epoch": 0.1504238561895028, + "flos": 25629409906560.0, + "grad_norm": 1.7725475682352354, + "language_loss": 0.79319745, + "learning_rate": 3.849912917118608e-06, + "loss": 0.81590581, + "num_input_tokens_seen": 26471480, + "router_z_loss_clip": 1.41699219, + "router_z_loss_mlp": 0.25317383, + "step": 1251, + "time_per_iteration": 2.8720128536224365 + }, + { + "auxiliary_loss_clip": 0.01096523, + "auxiliary_loss_mlp": 0.01012793, + "balance_loss_clip": 1.03908205, + "balance_loss_mlp": 1.0089066, + "epoch": 0.15054409908014188, + "flos": 52095146129280.0, + "grad_norm": 0.8859148314778601, + "language_loss": 0.59307379, + "learning_rate": 3.849616710653992e-06, + "loss": 0.61416692, + "num_input_tokens_seen": 26532950, + "router_z_loss_clip": 0.57421875, + "router_z_loss_mlp": 0.03881836, + "step": 1252, + "time_per_iteration": 3.330090045928955 + }, + { + "auxiliary_loss_clip": 0.01207182, + "auxiliary_loss_mlp": 0.01051474, + "balance_loss_clip": 1.0660764, + "balance_loss_mlp": 1.02733445, + "epoch": 0.150664341970781, + "flos": 18880251096960.0, + "grad_norm": 1.8052978168050022, + "language_loss": 0.74990809, + "learning_rate": 3.84932022360399e-06, + "loss": 0.77249467, + "num_input_tokens_seen": 26551615, + "router_z_loss_clip": 1.41210938, + "router_z_loss_mlp": 0.24157715, + "step": 1253, + "time_per_iteration": 2.840454339981079 + }, + { + "auxiliary_loss_clip": 0.01204886, + "auxiliary_loss_mlp": 0.01057741, + "balance_loss_clip": 1.06897664, + "balance_loss_mlp": 1.03366113, + "epoch": 0.15078458486142007, + "flos": 22163240309760.0, + "grad_norm": 2.629101858100689, + "language_loss": 0.84264874, + "learning_rate": 3.849023456013581e-06, + "loss": 0.86527503, + "num_input_tokens_seen": 26569175, + "router_z_loss_clip": 1.35839844, + "router_z_loss_mlp": 0.24108887, + "step": 1254, + "time_per_iteration": 2.8619463443756104 + }, + { + "auxiliary_loss_clip": 0.01209681, + "auxiliary_loss_mlp": 0.01053416, + "balance_loss_clip": 1.06804132, + "balance_loss_mlp": 1.02983606, + "epoch": 0.15090482775205916, + "flos": 26651894457600.0, + "grad_norm": 3.476931197563514, + "language_loss": 0.61967731, + "learning_rate": 3.848726407927784e-06, + "loss": 0.64230824, + "num_input_tokens_seen": 26589560, + "router_z_loss_clip": 1.41601562, + "router_z_loss_mlp": 0.23571777, + "step": 1255, + "time_per_iteration": 2.8787591457366943 + }, + { + "auxiliary_loss_clip": 0.01204798, + "auxiliary_loss_mlp": 0.01052705, + "balance_loss_clip": 1.06459928, + "balance_loss_mlp": 1.02925634, + "epoch": 0.15102507064269824, + "flos": 21798998444160.0, + "grad_norm": 2.2963436867479605, + "language_loss": 0.86349201, + "learning_rate": 3.84842907939166e-06, + "loss": 0.88606697, + "num_input_tokens_seen": 26608785, + "router_z_loss_clip": 1.40332031, + "router_z_loss_mlp": 0.23461914, + "step": 1256, + "time_per_iteration": 2.9074532985687256 + }, + { + "auxiliary_loss_clip": 0.0120482, + "auxiliary_loss_mlp": 0.01055897, + "balance_loss_clip": 1.06878173, + "balance_loss_mlp": 1.03262711, + "epoch": 0.15114531353333735, + "flos": 22820908377600.0, + "grad_norm": 2.611496934228274, + "language_loss": 0.71210015, + "learning_rate": 3.8481314704503146e-06, + "loss": 0.73470736, + "num_input_tokens_seen": 26628615, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.23266602, + "step": 1257, + "time_per_iteration": 2.8598952293395996 + }, + { + "auxiliary_loss_clip": 0.01204238, + "auxiliary_loss_mlp": 0.01057348, + "balance_loss_clip": 1.06839812, + "balance_loss_mlp": 1.03369713, + "epoch": 0.15126555642397643, + "flos": 19682674974720.0, + "grad_norm": 2.213880416575481, + "language_loss": 0.8796016, + "learning_rate": 3.847833581148895e-06, + "loss": 0.90221745, + "num_input_tokens_seen": 26647525, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.23657227, + "step": 1258, + "time_per_iteration": 2.8643434047698975 + }, + { + "auxiliary_loss_clip": 0.01207167, + "auxiliary_loss_mlp": 0.01062705, + "balance_loss_clip": 1.06449461, + "balance_loss_mlp": 1.03720593, + "epoch": 0.15138579931461552, + "flos": 28726022424960.0, + "grad_norm": 2.507724830537425, + "language_loss": 0.80516565, + "learning_rate": 3.84753541153259e-06, + "loss": 0.82786441, + "num_input_tokens_seen": 26667095, + "router_z_loss_clip": 1.42773438, + "router_z_loss_mlp": 0.25476074, + "step": 1259, + "time_per_iteration": 3.7204551696777344 + }, + { + "auxiliary_loss_clip": 0.01208201, + "auxiliary_loss_mlp": 0.01068335, + "balance_loss_clip": 1.06781852, + "balance_loss_mlp": 1.04357505, + "epoch": 0.15150604220525463, + "flos": 22127006465280.0, + "grad_norm": 1.8244372600155152, + "language_loss": 0.83233106, + "learning_rate": 3.847236961646633e-06, + "loss": 0.85509646, + "num_input_tokens_seen": 26686075, + "router_z_loss_clip": 1.40234375, + "router_z_loss_mlp": 0.24768066, + "step": 1260, + "time_per_iteration": 3.8226277828216553 + }, + { + "auxiliary_loss_clip": 0.01211245, + "auxiliary_loss_mlp": 0.01064153, + "balance_loss_clip": 1.07045054, + "balance_loss_mlp": 1.0400846, + "epoch": 0.1516262850958937, + "flos": 12968708515200.0, + "grad_norm": 2.4256617259691042, + "language_loss": 0.77760214, + "learning_rate": 3.846938231536296e-06, + "loss": 0.80035615, + "num_input_tokens_seen": 26701695, + "router_z_loss_clip": 1.40820312, + "router_z_loss_mlp": 0.24060059, + "step": 1261, + "time_per_iteration": 4.555985689163208 + }, + { + "auxiliary_loss_clip": 0.01208783, + "auxiliary_loss_mlp": 0.01058013, + "balance_loss_clip": 1.07086968, + "balance_loss_mlp": 1.03363442, + "epoch": 0.1517465279865328, + "flos": 21797130936960.0, + "grad_norm": 2.207403049037888, + "language_loss": 0.80869234, + "learning_rate": 3.8466392212468995e-06, + "loss": 0.83136034, + "num_input_tokens_seen": 26721885, + "router_z_loss_clip": 1.37890625, + "router_z_loss_mlp": 0.24414062, + "step": 1262, + "time_per_iteration": 2.9326703548431396 + }, + { + "auxiliary_loss_clip": 0.01093067, + "auxiliary_loss_mlp": 0.01037001, + "balance_loss_clip": 1.03568482, + "balance_loss_mlp": 1.03325772, + "epoch": 0.15186677087717187, + "flos": 58174569901440.0, + "grad_norm": 0.8178118777098191, + "language_loss": 0.61954355, + "learning_rate": 3.8463399308238e-06, + "loss": 0.64084423, + "num_input_tokens_seen": 26780990, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.03735352, + "step": 1263, + "time_per_iteration": 3.3311843872070312 + }, + { + "auxiliary_loss_clip": 0.01206968, + "auxiliary_loss_mlp": 0.01058918, + "balance_loss_clip": 1.06907916, + "balance_loss_mlp": 1.03599453, + "epoch": 0.15198701376781099, + "flos": 32669696448000.0, + "grad_norm": 1.985334289044448, + "language_loss": 0.64131504, + "learning_rate": 3.846040360312402e-06, + "loss": 0.66397393, + "num_input_tokens_seen": 26804250, + "router_z_loss_clip": 1.37695312, + "router_z_loss_mlp": 0.22949219, + "step": 1264, + "time_per_iteration": 3.0021817684173584 + }, + { + "auxiliary_loss_clip": 0.01211034, + "auxiliary_loss_mlp": 0.01059576, + "balance_loss_clip": 1.06866407, + "balance_loss_mlp": 1.03658092, + "epoch": 0.15210725665845007, + "flos": 28402575431040.0, + "grad_norm": 3.017725183216552, + "language_loss": 0.81228298, + "learning_rate": 3.8457405097581485e-06, + "loss": 0.83498907, + "num_input_tokens_seen": 26823240, + "router_z_loss_clip": 1.42382812, + "router_z_loss_mlp": 0.22998047, + "step": 1265, + "time_per_iteration": 2.8596692085266113 + }, + { + "auxiliary_loss_clip": 0.01210818, + "auxiliary_loss_mlp": 0.01055478, + "balance_loss_clip": 1.06924629, + "balance_loss_mlp": 1.03028893, + "epoch": 0.15222749954908915, + "flos": 19938179393280.0, + "grad_norm": 2.499646417451739, + "language_loss": 0.77787304, + "learning_rate": 3.8454403792065275e-06, + "loss": 0.80053604, + "num_input_tokens_seen": 26842060, + "router_z_loss_clip": 1.41699219, + "router_z_loss_mlp": 0.25183105, + "step": 1266, + "time_per_iteration": 2.772456407546997 + }, + { + "auxiliary_loss_clip": 0.01204086, + "auxiliary_loss_mlp": 0.01064395, + "balance_loss_clip": 1.06927085, + "balance_loss_mlp": 1.04060078, + "epoch": 0.15234774243972826, + "flos": 21324223451520.0, + "grad_norm": 2.2117942825290164, + "language_loss": 0.85461199, + "learning_rate": 3.845139968703068e-06, + "loss": 0.87729681, + "num_input_tokens_seen": 26859580, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.23791504, + "step": 1267, + "time_per_iteration": 2.80903697013855 + }, + { + "auxiliary_loss_clip": 0.01208531, + "auxiliary_loss_mlp": 0.01064202, + "balance_loss_clip": 1.06799269, + "balance_loss_mlp": 1.03832197, + "epoch": 0.15246798533036734, + "flos": 25957812977280.0, + "grad_norm": 2.0303884906033196, + "language_loss": 0.82841051, + "learning_rate": 3.844839278293342e-06, + "loss": 0.85113788, + "num_input_tokens_seen": 26880430, + "router_z_loss_clip": 1.40527344, + "router_z_loss_mlp": 0.25878906, + "step": 1268, + "time_per_iteration": 2.8146090507507324 + }, + { + "auxiliary_loss_clip": 0.01213492, + "auxiliary_loss_mlp": 0.01062685, + "balance_loss_clip": 1.07202673, + "balance_loss_mlp": 1.03895044, + "epoch": 0.15258822822100643, + "flos": 25811907932160.0, + "grad_norm": 2.3655012249705063, + "language_loss": 0.76790744, + "learning_rate": 3.8445383080229654e-06, + "loss": 0.7906692, + "num_input_tokens_seen": 26896445, + "router_z_loss_clip": 1.41601562, + "router_z_loss_mlp": 0.23742676, + "step": 1269, + "time_per_iteration": 2.8168745040893555 + }, + { + "auxiliary_loss_clip": 0.01209897, + "auxiliary_loss_mlp": 0.0106473, + "balance_loss_clip": 1.06713927, + "balance_loss_mlp": 1.04091215, + "epoch": 0.1527084711116455, + "flos": 25265455349760.0, + "grad_norm": 2.440979235946412, + "language_loss": 0.73339689, + "learning_rate": 3.844237057937593e-06, + "loss": 0.75614309, + "num_input_tokens_seen": 26915450, + "router_z_loss_clip": 1.42871094, + "router_z_loss_mlp": 0.23803711, + "step": 1270, + "time_per_iteration": 2.9479362964630127 + }, + { + "auxiliary_loss_clip": 0.01206979, + "auxiliary_loss_mlp": 0.01061587, + "balance_loss_clip": 1.06805444, + "balance_loss_mlp": 1.03917599, + "epoch": 0.15282871400228462, + "flos": 29240227572480.0, + "grad_norm": 8.606482212824462, + "language_loss": 0.77996868, + "learning_rate": 3.843935528082926e-06, + "loss": 0.80265439, + "num_input_tokens_seen": 26936475, + "router_z_loss_clip": 1.38867188, + "router_z_loss_mlp": 0.22412109, + "step": 1271, + "time_per_iteration": 2.8467955589294434 + }, + { + "auxiliary_loss_clip": 0.01206548, + "auxiliary_loss_mlp": 0.01063886, + "balance_loss_clip": 1.06836891, + "balance_loss_mlp": 1.03998482, + "epoch": 0.1529489568929237, + "flos": 20882952869760.0, + "grad_norm": 2.126505481453586, + "language_loss": 0.85033208, + "learning_rate": 3.843633718504704e-06, + "loss": 0.8730365, + "num_input_tokens_seen": 26954920, + "router_z_loss_clip": 1.38378906, + "router_z_loss_mlp": 0.23901367, + "step": 1272, + "time_per_iteration": 2.8527939319610596 + }, + { + "auxiliary_loss_clip": 0.01203569, + "auxiliary_loss_mlp": 0.01053063, + "balance_loss_clip": 1.06635165, + "balance_loss_mlp": 1.02968621, + "epoch": 0.1530691997835628, + "flos": 20083833043200.0, + "grad_norm": 5.0678336940323865, + "language_loss": 0.90067029, + "learning_rate": 3.843331629248715e-06, + "loss": 0.92323661, + "num_input_tokens_seen": 26972520, + "router_z_loss_clip": 1.37304688, + "router_z_loss_mlp": 0.23376465, + "step": 1273, + "time_per_iteration": 2.830941915512085 + }, + { + "auxiliary_loss_clip": 0.0120626, + "auxiliary_loss_mlp": 0.0105173, + "balance_loss_clip": 1.06862307, + "balance_loss_mlp": 1.02829361, + "epoch": 0.1531894426742019, + "flos": 28759814144640.0, + "grad_norm": 2.292496623137792, + "language_loss": 0.77008659, + "learning_rate": 3.843029260360782e-06, + "loss": 0.79266649, + "num_input_tokens_seen": 26990890, + "router_z_loss_clip": 1.37597656, + "router_z_loss_mlp": 0.23461914, + "step": 1274, + "time_per_iteration": 2.9540774822235107 + }, + { + "auxiliary_loss_clip": 0.01204631, + "auxiliary_loss_mlp": 0.01057195, + "balance_loss_clip": 1.06818986, + "balance_loss_mlp": 1.03403318, + "epoch": 0.15330968556484098, + "flos": 22236282616320.0, + "grad_norm": 1.9639863574920773, + "language_loss": 0.79149628, + "learning_rate": 3.8427266118867755e-06, + "loss": 0.81411451, + "num_input_tokens_seen": 27010640, + "router_z_loss_clip": 1.36523438, + "router_z_loss_mlp": 0.23132324, + "step": 1275, + "time_per_iteration": 2.894322395324707 + }, + { + "auxiliary_loss_clip": 0.01206176, + "auxiliary_loss_mlp": 0.0105092, + "balance_loss_clip": 1.06644249, + "balance_loss_mlp": 1.02654207, + "epoch": 0.15342992845548006, + "flos": 27527504296320.0, + "grad_norm": 2.0537823771337624, + "language_loss": 0.83101225, + "learning_rate": 3.842423683872608e-06, + "loss": 0.85358322, + "num_input_tokens_seen": 27031215, + "router_z_loss_clip": 1.39746094, + "router_z_loss_mlp": 0.24389648, + "step": 1276, + "time_per_iteration": 2.896949529647827 + }, + { + "auxiliary_loss_clip": 0.0120714, + "auxiliary_loss_mlp": 0.01059953, + "balance_loss_clip": 1.06865978, + "balance_loss_mlp": 1.03569436, + "epoch": 0.15355017134611917, + "flos": 19609596754560.0, + "grad_norm": 2.6790378286582572, + "language_loss": 0.77952498, + "learning_rate": 3.842120476364232e-06, + "loss": 0.80219591, + "num_input_tokens_seen": 27049665, + "router_z_loss_clip": 1.38378906, + "router_z_loss_mlp": 0.24267578, + "step": 1277, + "time_per_iteration": 2.8799245357513428 + }, + { + "auxiliary_loss_clip": 0.01205492, + "auxiliary_loss_mlp": 0.01053667, + "balance_loss_clip": 1.06574559, + "balance_loss_mlp": 1.02932477, + "epoch": 0.15367041423675826, + "flos": 18478590238080.0, + "grad_norm": 2.1294384442442187, + "language_loss": 0.83810711, + "learning_rate": 3.841816989407644e-06, + "loss": 0.8606987, + "num_input_tokens_seen": 27065155, + "router_z_loss_clip": 1.39648438, + "router_z_loss_mlp": 0.2434082, + "step": 1278, + "time_per_iteration": 2.8198487758636475 + }, + { + "auxiliary_loss_clip": 0.0120044, + "auxiliary_loss_mlp": 0.01051736, + "balance_loss_clip": 1.06637645, + "balance_loss_mlp": 1.02969432, + "epoch": 0.15379065712739734, + "flos": 41427662342400.0, + "grad_norm": 2.5378480530534575, + "language_loss": 0.76975191, + "learning_rate": 3.841513223048884e-06, + "loss": 0.79227364, + "num_input_tokens_seen": 27085840, + "router_z_loss_clip": 1.34082031, + "router_z_loss_mlp": 0.22033691, + "step": 1279, + "time_per_iteration": 3.036271810531616 + }, + { + "auxiliary_loss_clip": 0.01205354, + "auxiliary_loss_mlp": 0.01060773, + "balance_loss_clip": 1.06822109, + "balance_loss_mlp": 1.03535724, + "epoch": 0.15391090001803642, + "flos": 22054215553920.0, + "grad_norm": 2.383641297509887, + "language_loss": 0.78010762, + "learning_rate": 3.841209177334031e-06, + "loss": 0.80276889, + "num_input_tokens_seen": 27104200, + "router_z_loss_clip": 1.37109375, + "router_z_loss_mlp": 0.25427246, + "step": 1280, + "time_per_iteration": 2.8920464515686035 + }, + { + "auxiliary_loss_clip": 0.01200483, + "auxiliary_loss_mlp": 0.01055656, + "balance_loss_clip": 1.06385756, + "balance_loss_mlp": 1.03266108, + "epoch": 0.15403114290867553, + "flos": 15450351258240.0, + "grad_norm": 1.9694801444371983, + "language_loss": 0.74562883, + "learning_rate": 3.84090485230921e-06, + "loss": 0.76819021, + "num_input_tokens_seen": 27122440, + "router_z_loss_clip": 1.3671875, + "router_z_loss_mlp": 0.22998047, + "step": 1281, + "time_per_iteration": 2.928537130355835 + }, + { + "auxiliary_loss_clip": 0.01203728, + "auxiliary_loss_mlp": 0.01057622, + "balance_loss_clip": 1.06709766, + "balance_loss_mlp": 1.03373194, + "epoch": 0.15415138579931462, + "flos": 17929156826880.0, + "grad_norm": 2.966573676896128, + "language_loss": 0.76483935, + "learning_rate": 3.840600248020588e-06, + "loss": 0.78745282, + "num_input_tokens_seen": 27139380, + "router_z_loss_clip": 1.3671875, + "router_z_loss_mlp": 0.23876953, + "step": 1282, + "time_per_iteration": 2.9113824367523193 + }, + { + "auxiliary_loss_clip": 0.01209329, + "auxiliary_loss_mlp": 0.01055056, + "balance_loss_clip": 1.0682168, + "balance_loss_mlp": 1.02998626, + "epoch": 0.1542716286899537, + "flos": 11429325296640.0, + "grad_norm": 2.5152496463606937, + "language_loss": 0.79928464, + "learning_rate": 3.840295364514371e-06, + "loss": 0.8219285, + "num_input_tokens_seen": 27156760, + "router_z_loss_clip": 1.41113281, + "router_z_loss_mlp": 0.25085449, + "step": 1283, + "time_per_iteration": 2.8151731491088867 + }, + { + "auxiliary_loss_clip": 0.01202489, + "auxiliary_loss_mlp": 0.0104982, + "balance_loss_clip": 1.06587315, + "balance_loss_mlp": 1.02686, + "epoch": 0.1543918715805928, + "flos": 17420338719360.0, + "grad_norm": 3.9236946612679233, + "language_loss": 0.78527302, + "learning_rate": 3.83999020183681e-06, + "loss": 0.80779612, + "num_input_tokens_seen": 27175455, + "router_z_loss_clip": 1.36425781, + "router_z_loss_mlp": 0.22924805, + "step": 1284, + "time_per_iteration": 2.8275978565216064 + }, + { + "auxiliary_loss_clip": 0.01207746, + "auxiliary_loss_mlp": 0.01064965, + "balance_loss_clip": 1.06902051, + "balance_loss_mlp": 1.04059863, + "epoch": 0.1545121144712319, + "flos": 17786376264960.0, + "grad_norm": 2.3435583113127816, + "language_loss": 0.78359449, + "learning_rate": 3.839684760034199e-06, + "loss": 0.80632162, + "num_input_tokens_seen": 27193660, + "router_z_loss_clip": 1.38769531, + "router_z_loss_mlp": 0.24365234, + "step": 1285, + "time_per_iteration": 2.794523000717163 + }, + { + "auxiliary_loss_clip": 0.0120384, + "auxiliary_loss_mlp": 0.01054235, + "balance_loss_clip": 1.06680858, + "balance_loss_mlp": 1.03048861, + "epoch": 0.15463235736187098, + "flos": 28220185146240.0, + "grad_norm": 2.399764417042009, + "language_loss": 0.65379483, + "learning_rate": 3.8393790391528716e-06, + "loss": 0.67637563, + "num_input_tokens_seen": 27214355, + "router_z_loss_clip": 1.37011719, + "router_z_loss_mlp": 0.23754883, + "step": 1286, + "time_per_iteration": 2.888139247894287 + }, + { + "auxiliary_loss_clip": 0.01201406, + "auxiliary_loss_mlp": 0.01054167, + "balance_loss_clip": 1.06495094, + "balance_loss_mlp": 1.03026581, + "epoch": 0.15475260025251006, + "flos": 22856890826880.0, + "grad_norm": 2.3450398641685224, + "language_loss": 0.88868415, + "learning_rate": 3.8390730392392075e-06, + "loss": 0.91123986, + "num_input_tokens_seen": 27234335, + "router_z_loss_clip": 1.36425781, + "router_z_loss_mlp": 0.23901367, + "step": 1287, + "time_per_iteration": 2.895155191421509 + }, + { + "auxiliary_loss_clip": 0.01203068, + "auxiliary_loss_mlp": 0.01053356, + "balance_loss_clip": 1.06749785, + "balance_loss_mlp": 1.03100371, + "epoch": 0.15487284314314917, + "flos": 17602872658560.0, + "grad_norm": 2.3406458627942515, + "language_loss": 0.79257774, + "learning_rate": 3.838766760339626e-06, + "loss": 0.81514204, + "num_input_tokens_seen": 27252860, + "router_z_loss_clip": 1.35351562, + "router_z_loss_mlp": 0.22351074, + "step": 1288, + "time_per_iteration": 3.710615396499634 + }, + { + "auxiliary_loss_clip": 0.01197318, + "auxiliary_loss_mlp": 0.01045542, + "balance_loss_clip": 1.06277728, + "balance_loss_mlp": 1.02348828, + "epoch": 0.15499308603378825, + "flos": 20082037363200.0, + "grad_norm": 2.6179381022733064, + "language_loss": 0.79545265, + "learning_rate": 3.838460202500587e-06, + "loss": 0.81788123, + "num_input_tokens_seen": 27268650, + "router_z_loss_clip": 1.34472656, + "router_z_loss_mlp": 0.22045898, + "step": 1289, + "time_per_iteration": 3.642017364501953 + }, + { + "auxiliary_loss_clip": 0.01204485, + "auxiliary_loss_mlp": 0.01064023, + "balance_loss_clip": 1.06609416, + "balance_loss_mlp": 1.04031277, + "epoch": 0.15511332892442733, + "flos": 15918051271680.0, + "grad_norm": 2.230369916768756, + "language_loss": 0.74287367, + "learning_rate": 3.838153365768599e-06, + "loss": 0.76555878, + "num_input_tokens_seen": 27285160, + "router_z_loss_clip": 1.38476562, + "router_z_loss_mlp": 0.23742676, + "step": 1290, + "time_per_iteration": 3.6747262477874756 + }, + { + "auxiliary_loss_clip": 0.01207138, + "auxiliary_loss_mlp": 0.01059688, + "balance_loss_clip": 1.06739926, + "balance_loss_mlp": 1.03604841, + "epoch": 0.15523357181506645, + "flos": 41282475569280.0, + "grad_norm": 2.5599134082008774, + "language_loss": 0.74943841, + "learning_rate": 3.837846250190206e-06, + "loss": 0.77210671, + "num_input_tokens_seen": 27308025, + "router_z_loss_clip": 1.39550781, + "router_z_loss_mlp": 0.23657227, + "step": 1291, + "time_per_iteration": 2.978299856185913 + }, + { + "auxiliary_loss_clip": 0.01199178, + "auxiliary_loss_mlp": 0.01056818, + "balance_loss_clip": 1.06627274, + "balance_loss_mlp": 1.03395367, + "epoch": 0.15535381470570553, + "flos": 18478769806080.0, + "grad_norm": 2.171621756065342, + "language_loss": 0.76628983, + "learning_rate": 3.837538855811998e-06, + "loss": 0.78884983, + "num_input_tokens_seen": 27326200, + "router_z_loss_clip": 1.32910156, + "router_z_loss_mlp": 0.2286377, + "step": 1292, + "time_per_iteration": 2.8836610317230225 + }, + { + "auxiliary_loss_clip": 0.01203697, + "auxiliary_loss_mlp": 0.01058487, + "balance_loss_clip": 1.06637168, + "balance_loss_mlp": 1.03478825, + "epoch": 0.1554740575963446, + "flos": 13918150759680.0, + "grad_norm": 2.613297501291972, + "language_loss": 0.70694184, + "learning_rate": 3.837231182680606e-06, + "loss": 0.72956371, + "num_input_tokens_seen": 27344165, + "router_z_loss_clip": 1.37304688, + "router_z_loss_mlp": 0.23706055, + "step": 1293, + "time_per_iteration": 2.8039631843566895 + }, + { + "auxiliary_loss_clip": 0.01202708, + "auxiliary_loss_mlp": 0.01057725, + "balance_loss_clip": 1.06852865, + "balance_loss_mlp": 1.03587365, + "epoch": 0.1555943004869837, + "flos": 20847078161280.0, + "grad_norm": 1.7246014959100497, + "language_loss": 0.75821882, + "learning_rate": 3.836923230842706e-06, + "loss": 0.78082311, + "num_input_tokens_seen": 27363280, + "router_z_loss_clip": 1.34277344, + "router_z_loss_mlp": 0.21862793, + "step": 1294, + "time_per_iteration": 2.8145086765289307 + }, + { + "auxiliary_loss_clip": 0.01206337, + "auxiliary_loss_mlp": 0.01052883, + "balance_loss_clip": 1.06661344, + "balance_loss_mlp": 1.02997065, + "epoch": 0.1557145433776228, + "flos": 22085888371200.0, + "grad_norm": 2.2955113739232273, + "language_loss": 0.81175745, + "learning_rate": 3.836615000345011e-06, + "loss": 0.83434963, + "num_input_tokens_seen": 27381460, + "router_z_loss_clip": 1.39453125, + "router_z_loss_mlp": 0.22924805, + "step": 1295, + "time_per_iteration": 2.833531141281128 + }, + { + "auxiliary_loss_clip": 0.0120003, + "auxiliary_loss_mlp": 0.01053994, + "balance_loss_clip": 1.06622636, + "balance_loss_mlp": 1.03281069, + "epoch": 0.1558347862682619, + "flos": 19791987039360.0, + "grad_norm": 2.3365379186158513, + "language_loss": 0.7790671, + "learning_rate": 3.836306491234282e-06, + "loss": 0.80160737, + "num_input_tokens_seen": 27399310, + "router_z_loss_clip": 1.33886719, + "router_z_loss_mlp": 0.21179199, + "step": 1296, + "time_per_iteration": 2.8153653144836426 + }, + { + "auxiliary_loss_clip": 0.01200083, + "auxiliary_loss_mlp": 0.01052327, + "balance_loss_clip": 1.06659067, + "balance_loss_mlp": 1.03009415, + "epoch": 0.15595502915890097, + "flos": 17237086508160.0, + "grad_norm": 2.42980335473816, + "language_loss": 0.75661308, + "learning_rate": 3.835997703557317e-06, + "loss": 0.77913719, + "num_input_tokens_seen": 27416050, + "router_z_loss_clip": 1.33496094, + "router_z_loss_mlp": 0.22253418, + "step": 1297, + "time_per_iteration": 2.839857339859009 + }, + { + "auxiliary_loss_clip": 0.01199231, + "auxiliary_loss_mlp": 0.0104795, + "balance_loss_clip": 1.06713843, + "balance_loss_mlp": 1.02597952, + "epoch": 0.15607527204954008, + "flos": 19719519350400.0, + "grad_norm": 2.576002078176061, + "language_loss": 0.80053461, + "learning_rate": 3.83568863736096e-06, + "loss": 0.82300639, + "num_input_tokens_seen": 27434920, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.21960449, + "step": 1298, + "time_per_iteration": 2.8850200176239014 + }, + { + "auxiliary_loss_clip": 0.01206431, + "auxiliary_loss_mlp": 0.01058641, + "balance_loss_clip": 1.06872821, + "balance_loss_mlp": 1.03639674, + "epoch": 0.15619551494017916, + "flos": 18515650095360.0, + "grad_norm": 2.755692048128063, + "language_loss": 0.89175212, + "learning_rate": 3.8353792926920975e-06, + "loss": 0.91440284, + "num_input_tokens_seen": 27453570, + "router_z_loss_clip": 1.37597656, + "router_z_loss_mlp": 0.22241211, + "step": 1299, + "time_per_iteration": 2.9393742084503174 + }, + { + "auxiliary_loss_clip": 0.01202885, + "auxiliary_loss_mlp": 0.0106822, + "balance_loss_clip": 1.0676173, + "balance_loss_mlp": 1.04453301, + "epoch": 0.15631575783081825, + "flos": 19902125116800.0, + "grad_norm": 3.047366312309334, + "language_loss": 0.81069767, + "learning_rate": 3.835069669597655e-06, + "loss": 0.83340877, + "num_input_tokens_seen": 27471960, + "router_z_loss_clip": 1.35351562, + "router_z_loss_mlp": 0.23706055, + "step": 1300, + "time_per_iteration": 2.8707964420318604 + }, + { + "auxiliary_loss_clip": 0.01208468, + "auxiliary_loss_mlp": 0.01058491, + "balance_loss_clip": 1.06881881, + "balance_loss_mlp": 1.03524566, + "epoch": 0.15643600072145733, + "flos": 20777663128320.0, + "grad_norm": 2.3020121411802528, + "language_loss": 0.80128372, + "learning_rate": 3.834759768124603e-06, + "loss": 0.82395327, + "num_input_tokens_seen": 27490835, + "router_z_loss_clip": 1.39648438, + "router_z_loss_mlp": 0.23242188, + "step": 1301, + "time_per_iteration": 2.9487905502319336 + }, + { + "auxiliary_loss_clip": 0.01207519, + "auxiliary_loss_mlp": 0.01068458, + "balance_loss_clip": 1.07077575, + "balance_loss_mlp": 1.04207695, + "epoch": 0.15655624361209644, + "flos": 18546389159040.0, + "grad_norm": 3.3441471854005926, + "language_loss": 0.76416874, + "learning_rate": 3.834449588319953e-06, + "loss": 0.78692847, + "num_input_tokens_seen": 27508870, + "router_z_loss_clip": 1.36621094, + "router_z_loss_mlp": 0.26379395, + "step": 1302, + "time_per_iteration": 2.8598504066467285 + }, + { + "auxiliary_loss_clip": 0.01201006, + "auxiliary_loss_mlp": 0.01067474, + "balance_loss_clip": 1.0672648, + "balance_loss_mlp": 1.04494309, + "epoch": 0.15667648650273552, + "flos": 25229544727680.0, + "grad_norm": 3.8507120035224234, + "language_loss": 0.85140783, + "learning_rate": 3.834139130230758e-06, + "loss": 0.87409258, + "num_input_tokens_seen": 27528175, + "router_z_loss_clip": 1.33789062, + "router_z_loss_mlp": 0.2253418, + "step": 1303, + "time_per_iteration": 2.873013496398926 + }, + { + "auxiliary_loss_clip": 0.01202088, + "auxiliary_loss_mlp": 0.01053913, + "balance_loss_clip": 1.06383324, + "balance_loss_mlp": 1.03117967, + "epoch": 0.1567967293933746, + "flos": 24827093769600.0, + "grad_norm": 2.3353659093446586, + "language_loss": 0.81291109, + "learning_rate": 3.833828393904117e-06, + "loss": 0.83547109, + "num_input_tokens_seen": 27548455, + "router_z_loss_clip": 1.38183594, + "router_z_loss_mlp": 0.22741699, + "step": 1304, + "time_per_iteration": 2.8420283794403076 + }, + { + "auxiliary_loss_clip": 0.0120404, + "auxiliary_loss_mlp": 0.01051893, + "balance_loss_clip": 1.06846499, + "balance_loss_mlp": 1.02924371, + "epoch": 0.15691697228401372, + "flos": 19164555244800.0, + "grad_norm": 2.645201719000728, + "language_loss": 0.77395201, + "learning_rate": 3.833517379387165e-06, + "loss": 0.79651129, + "num_input_tokens_seen": 27564910, + "router_z_loss_clip": 1.35546875, + "router_z_loss_mlp": 0.22644043, + "step": 1305, + "time_per_iteration": 2.790764331817627 + }, + { + "auxiliary_loss_clip": 0.01200341, + "auxiliary_loss_mlp": 0.0105714, + "balance_loss_clip": 1.06657147, + "balance_loss_mlp": 1.03571808, + "epoch": 0.1570372151746528, + "flos": 24790931752320.0, + "grad_norm": 2.2508928448296968, + "language_loss": 0.88830721, + "learning_rate": 3.833206086727085e-06, + "loss": 0.910882, + "num_input_tokens_seen": 27584260, + "router_z_loss_clip": 1.33789062, + "router_z_loss_mlp": 0.2142334, + "step": 1306, + "time_per_iteration": 2.8509514331817627 + }, + { + "auxiliary_loss_clip": 0.01203269, + "auxiliary_loss_mlp": 0.01062388, + "balance_loss_clip": 1.06638169, + "balance_loss_mlp": 1.04014385, + "epoch": 0.15715745806529188, + "flos": 24863650836480.0, + "grad_norm": 2.236788543939057, + "language_loss": 0.70282221, + "learning_rate": 3.8328945159710994e-06, + "loss": 0.72547883, + "num_input_tokens_seen": 27604440, + "router_z_loss_clip": 1.3671875, + "router_z_loss_mlp": 0.22253418, + "step": 1307, + "time_per_iteration": 2.813605546951294 + }, + { + "auxiliary_loss_clip": 0.01206686, + "auxiliary_loss_mlp": 0.01054795, + "balance_loss_clip": 1.06875038, + "balance_loss_mlp": 1.03306293, + "epoch": 0.157277700955931, + "flos": 21872148491520.0, + "grad_norm": 2.322636429266961, + "language_loss": 0.88963449, + "learning_rate": 3.832582667166473e-06, + "loss": 0.91224933, + "num_input_tokens_seen": 27624250, + "router_z_loss_clip": 1.38183594, + "router_z_loss_mlp": 0.21740723, + "step": 1308, + "time_per_iteration": 2.8591055870056152 + }, + { + "auxiliary_loss_clip": 0.01205692, + "auxiliary_loss_mlp": 0.01051698, + "balance_loss_clip": 1.06799865, + "balance_loss_mlp": 1.02810669, + "epoch": 0.15739794384657008, + "flos": 24533344344960.0, + "grad_norm": 1.9257128291927372, + "language_loss": 0.81187099, + "learning_rate": 3.8322705403605125e-06, + "loss": 0.83444494, + "num_input_tokens_seen": 27644595, + "router_z_loss_clip": 1.37695312, + "router_z_loss_mlp": 0.2355957, + "step": 1309, + "time_per_iteration": 2.9093518257141113 + }, + { + "auxiliary_loss_clip": 0.01197148, + "auxiliary_loss_mlp": 0.0104719, + "balance_loss_clip": 1.06574297, + "balance_loss_mlp": 1.02569699, + "epoch": 0.15751818673720916, + "flos": 17745329998080.0, + "grad_norm": 2.5424113941500717, + "language_loss": 0.81184191, + "learning_rate": 3.831958135600568e-06, + "loss": 0.83428526, + "num_input_tokens_seen": 27662145, + "router_z_loss_clip": 1.31347656, + "router_z_loss_mlp": 0.21496582, + "step": 1310, + "time_per_iteration": 2.837747812271118 + }, + { + "auxiliary_loss_clip": 0.01203715, + "auxiliary_loss_mlp": 0.01057134, + "balance_loss_clip": 1.07112432, + "balance_loss_mlp": 1.03372157, + "epoch": 0.15763842962784824, + "flos": 17858520731520.0, + "grad_norm": 2.8142676332534062, + "language_loss": 0.79305017, + "learning_rate": 3.831645452934032e-06, + "loss": 0.81565869, + "num_input_tokens_seen": 27680575, + "router_z_loss_clip": 1.32617188, + "router_z_loss_mlp": 0.23425293, + "step": 1311, + "time_per_iteration": 2.9240591526031494 + }, + { + "auxiliary_loss_clip": 0.01204498, + "auxiliary_loss_mlp": 0.0105786, + "balance_loss_clip": 1.07010221, + "balance_loss_mlp": 1.03453112, + "epoch": 0.15775867251848735, + "flos": 26980908059520.0, + "grad_norm": 2.777244218231937, + "language_loss": 0.79891258, + "learning_rate": 3.831332492408336e-06, + "loss": 0.82153618, + "num_input_tokens_seen": 27701985, + "router_z_loss_clip": 1.34375, + "router_z_loss_mlp": 0.2331543, + "step": 1312, + "time_per_iteration": 2.947871685028076 + }, + { + "auxiliary_loss_clip": 0.01201226, + "auxiliary_loss_mlp": 0.01056934, + "balance_loss_clip": 1.06691015, + "balance_loss_mlp": 1.03341389, + "epoch": 0.15787891540912644, + "flos": 19240398812160.0, + "grad_norm": 1.8715361168996245, + "language_loss": 0.69111741, + "learning_rate": 3.831019254070957e-06, + "loss": 0.71369904, + "num_input_tokens_seen": 27719770, + "router_z_loss_clip": 1.34179688, + "router_z_loss_mlp": 0.23522949, + "step": 1313, + "time_per_iteration": 2.8477306365966797 + }, + { + "auxiliary_loss_clip": 0.01200495, + "auxiliary_loss_mlp": 0.01060412, + "balance_loss_clip": 1.06608033, + "balance_loss_mlp": 1.0380249, + "epoch": 0.15799915829976552, + "flos": 27271102037760.0, + "grad_norm": 2.3887301151602442, + "language_loss": 0.95089978, + "learning_rate": 3.8307057379694135e-06, + "loss": 0.97350889, + "num_input_tokens_seen": 27739105, + "router_z_loss_clip": 1.34472656, + "router_z_loss_mlp": 0.22387695, + "step": 1314, + "time_per_iteration": 2.912825345993042 + }, + { + "auxiliary_loss_clip": 0.0120561, + "auxiliary_loss_mlp": 0.01060555, + "balance_loss_clip": 1.06641841, + "balance_loss_mlp": 1.03580701, + "epoch": 0.15811940119040463, + "flos": 20405520270720.0, + "grad_norm": 2.249704957512844, + "language_loss": 0.82258284, + "learning_rate": 3.830391944151264e-06, + "loss": 0.84524453, + "num_input_tokens_seen": 27754985, + "router_z_loss_clip": 1.39160156, + "router_z_loss_mlp": 0.24743652, + "step": 1315, + "time_per_iteration": 2.8625001907348633 + }, + { + "auxiliary_loss_clip": 0.01205738, + "auxiliary_loss_mlp": 0.01059895, + "balance_loss_clip": 1.06728613, + "balance_loss_mlp": 1.03619623, + "epoch": 0.1582396440810437, + "flos": 32599347661440.0, + "grad_norm": 1.9866382021861326, + "language_loss": 0.67360592, + "learning_rate": 3.830077872664114e-06, + "loss": 0.69626224, + "num_input_tokens_seen": 27776110, + "router_z_loss_clip": 1.3828125, + "router_z_loss_mlp": 0.23718262, + "step": 1316, + "time_per_iteration": 2.8930158615112305 + }, + { + "auxiliary_loss_clip": 0.0120266, + "auxiliary_loss_mlp": 0.01052511, + "balance_loss_clip": 1.06887054, + "balance_loss_mlp": 1.0308032, + "epoch": 0.1583598869716828, + "flos": 33800559310080.0, + "grad_norm": 1.945366795176826, + "language_loss": 0.73261631, + "learning_rate": 3.829763523555604e-06, + "loss": 0.75516802, + "num_input_tokens_seen": 27796510, + "router_z_loss_clip": 1.33984375, + "router_z_loss_mlp": 0.21716309, + "step": 1317, + "time_per_iteration": 3.7654170989990234 + }, + { + "auxiliary_loss_clip": 0.01196276, + "auxiliary_loss_mlp": 0.0105271, + "balance_loss_clip": 1.06783283, + "balance_loss_mlp": 1.03014326, + "epoch": 0.15848012986232188, + "flos": 24681332378880.0, + "grad_norm": 3.0311789907778097, + "language_loss": 0.77929902, + "learning_rate": 3.829448896873423e-06, + "loss": 0.80178893, + "num_input_tokens_seen": 27815610, + "router_z_loss_clip": 1.28613281, + "router_z_loss_mlp": 0.22583008, + "step": 1318, + "time_per_iteration": 3.699639081954956 + }, + { + "auxiliary_loss_clip": 0.0120854, + "auxiliary_loss_mlp": 0.01055152, + "balance_loss_clip": 1.07150602, + "balance_loss_mlp": 1.03202569, + "epoch": 0.158600372752961, + "flos": 22602068766720.0, + "grad_norm": 2.2279047323673677, + "language_loss": 0.78899741, + "learning_rate": 3.829133992665299e-06, + "loss": 0.8116343, + "num_input_tokens_seen": 27834735, + "router_z_loss_clip": 1.37011719, + "router_z_loss_mlp": 0.23132324, + "step": 1319, + "time_per_iteration": 4.488276720046997 + }, + { + "auxiliary_loss_clip": 0.01207598, + "auxiliary_loss_mlp": 0.01054562, + "balance_loss_clip": 1.07160628, + "balance_loss_mlp": 1.03120863, + "epoch": 0.15872061564360007, + "flos": 27927944092800.0, + "grad_norm": 2.6038845911538626, + "language_loss": 0.89203608, + "learning_rate": 3.828818810979002e-06, + "loss": 0.91465771, + "num_input_tokens_seen": 27853065, + "router_z_loss_clip": 1.35742188, + "router_z_loss_mlp": 0.23352051, + "step": 1320, + "time_per_iteration": 2.838707685470581 + }, + { + "auxiliary_loss_clip": 0.01204095, + "auxiliary_loss_mlp": 0.01054012, + "balance_loss_clip": 1.07047725, + "balance_loss_mlp": 1.03121936, + "epoch": 0.15884085853423915, + "flos": 23696805525120.0, + "grad_norm": 2.025589341837576, + "language_loss": 0.80230844, + "learning_rate": 3.8285033518623454e-06, + "loss": 0.82488954, + "num_input_tokens_seen": 27873315, + "router_z_loss_clip": 1.33398438, + "router_z_loss_mlp": 0.22790527, + "step": 1321, + "time_per_iteration": 2.9738152027130127 + }, + { + "auxiliary_loss_clip": 0.01205921, + "auxiliary_loss_mlp": 0.01062752, + "balance_loss_clip": 1.06972551, + "balance_loss_mlp": 1.03898168, + "epoch": 0.15896110142487826, + "flos": 23112359331840.0, + "grad_norm": 4.110146173682072, + "language_loss": 0.80955225, + "learning_rate": 3.8281876153631845e-06, + "loss": 0.83223903, + "num_input_tokens_seen": 27890070, + "router_z_loss_clip": 1.36425781, + "router_z_loss_mlp": 0.23754883, + "step": 1322, + "time_per_iteration": 2.984339952468872 + }, + { + "auxiliary_loss_clip": 0.01201176, + "auxiliary_loss_mlp": 0.01056274, + "balance_loss_clip": 1.06808758, + "balance_loss_mlp": 1.03379083, + "epoch": 0.15908134431551735, + "flos": 14685238632960.0, + "grad_norm": 2.0453630017158235, + "language_loss": 0.64415181, + "learning_rate": 3.827871601529416e-06, + "loss": 0.66672623, + "num_input_tokens_seen": 27908590, + "router_z_loss_clip": 1.33105469, + "router_z_loss_mlp": 0.22485352, + "step": 1323, + "time_per_iteration": 2.8143069744110107 + }, + { + "auxiliary_loss_clip": 0.01201629, + "auxiliary_loss_mlp": 0.01060499, + "balance_loss_clip": 1.06964874, + "balance_loss_mlp": 1.03842163, + "epoch": 0.15920158720615643, + "flos": 20193611984640.0, + "grad_norm": 2.170746983064609, + "language_loss": 0.8049565, + "learning_rate": 3.827555310408979e-06, + "loss": 0.82757783, + "num_input_tokens_seen": 27927985, + "router_z_loss_clip": 1.3203125, + "router_z_loss_mlp": 0.22094727, + "step": 1324, + "time_per_iteration": 2.8831329345703125 + }, + { + "auxiliary_loss_clip": 0.01205917, + "auxiliary_loss_mlp": 0.01052059, + "balance_loss_clip": 1.06846738, + "balance_loss_mlp": 1.02888465, + "epoch": 0.1593218300967955, + "flos": 24826626892800.0, + "grad_norm": 1.8298282535267236, + "language_loss": 0.8288241, + "learning_rate": 3.827238742049854e-06, + "loss": 0.85140383, + "num_input_tokens_seen": 27948280, + "router_z_loss_clip": 1.375, + "router_z_loss_mlp": 0.23181152, + "step": 1325, + "time_per_iteration": 2.8659656047821045 + }, + { + "auxiliary_loss_clip": 0.01202967, + "auxiliary_loss_mlp": 0.01053792, + "balance_loss_clip": 1.0672121, + "balance_loss_mlp": 1.03107119, + "epoch": 0.15944207298743462, + "flos": 28328707111680.0, + "grad_norm": 2.7201940883976805, + "language_loss": 0.51720935, + "learning_rate": 3.826921896500066e-06, + "loss": 0.53977692, + "num_input_tokens_seen": 27969565, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.22729492, + "step": 1326, + "time_per_iteration": 2.8646931648254395 + }, + { + "auxiliary_loss_clip": 0.01206203, + "auxiliary_loss_mlp": 0.01058275, + "balance_loss_clip": 1.07066953, + "balance_loss_mlp": 1.03523219, + "epoch": 0.1595623158780737, + "flos": 22964838174720.0, + "grad_norm": 2.4033421297221453, + "language_loss": 0.7772516, + "learning_rate": 3.826604773807678e-06, + "loss": 0.79989642, + "num_input_tokens_seen": 27987540, + "router_z_loss_clip": 1.35449219, + "router_z_loss_mlp": 0.23046875, + "step": 1327, + "time_per_iteration": 2.7963197231292725 + }, + { + "auxiliary_loss_clip": 0.0120449, + "auxiliary_loss_mlp": 0.01053566, + "balance_loss_clip": 1.06692028, + "balance_loss_mlp": 1.03002274, + "epoch": 0.1596825587687128, + "flos": 19710540950400.0, + "grad_norm": 2.913086654031764, + "language_loss": 0.73987401, + "learning_rate": 3.826287374020798e-06, + "loss": 0.76245463, + "num_input_tokens_seen": 28002345, + "router_z_loss_clip": 1.37695312, + "router_z_loss_mlp": 0.23547363, + "step": 1328, + "time_per_iteration": 2.8011295795440674 + }, + { + "auxiliary_loss_clip": 0.01203115, + "auxiliary_loss_mlp": 0.01060792, + "balance_loss_clip": 1.07012999, + "balance_loss_mlp": 1.03771281, + "epoch": 0.1598028016593519, + "flos": 22637727993600.0, + "grad_norm": 2.4092023135433074, + "language_loss": 0.81792188, + "learning_rate": 3.825969697187575e-06, + "loss": 0.84056091, + "num_input_tokens_seen": 28021675, + "router_z_loss_clip": 1.33105469, + "router_z_loss_mlp": 0.23071289, + "step": 1329, + "time_per_iteration": 2.841264247894287 + }, + { + "auxiliary_loss_clip": 0.01207884, + "auxiliary_loss_mlp": 0.01055147, + "balance_loss_clip": 1.07001281, + "balance_loss_mlp": 1.03155589, + "epoch": 0.15992304454999098, + "flos": 20482908122880.0, + "grad_norm": 2.626646025252235, + "language_loss": 0.69676292, + "learning_rate": 3.8256517433562015e-06, + "loss": 0.71939325, + "num_input_tokens_seen": 28039615, + "router_z_loss_clip": 1.37695312, + "router_z_loss_mlp": 0.23596191, + "step": 1330, + "time_per_iteration": 2.773773193359375 + }, + { + "auxiliary_loss_clip": 0.01204466, + "auxiliary_loss_mlp": 0.010543, + "balance_loss_clip": 1.06960762, + "balance_loss_mlp": 1.03187668, + "epoch": 0.16004328744063007, + "flos": 17676094533120.0, + "grad_norm": 7.251606738783255, + "language_loss": 0.91898191, + "learning_rate": 3.82533351257491e-06, + "loss": 0.94156957, + "num_input_tokens_seen": 28057565, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.22436523, + "step": 1331, + "time_per_iteration": 2.817878246307373 + }, + { + "auxiliary_loss_clip": 0.012038, + "auxiliary_loss_mlp": 0.01054954, + "balance_loss_clip": 1.06941986, + "balance_loss_mlp": 1.03150582, + "epoch": 0.16016353033126918, + "flos": 24098717779200.0, + "grad_norm": 2.848468618828573, + "language_loss": 0.88800669, + "learning_rate": 3.825015004891975e-06, + "loss": 0.91059411, + "num_input_tokens_seen": 28076305, + "router_z_loss_clip": 1.34179688, + "router_z_loss_mlp": 0.23461914, + "step": 1332, + "time_per_iteration": 2.919675588607788 + }, + { + "auxiliary_loss_clip": 0.01202337, + "auxiliary_loss_mlp": 0.01051432, + "balance_loss_clip": 1.06960022, + "balance_loss_mlp": 1.03132141, + "epoch": 0.16028377322190826, + "flos": 27634841112960.0, + "grad_norm": 2.0309109399546443, + "language_loss": 0.75828719, + "learning_rate": 3.824696220355716e-06, + "loss": 0.7808249, + "num_input_tokens_seen": 28097895, + "router_z_loss_clip": 1.328125, + "router_z_loss_mlp": 0.20092773, + "step": 1333, + "time_per_iteration": 2.911494016647339 + }, + { + "auxiliary_loss_clip": 0.0120002, + "auxiliary_loss_mlp": 0.01060147, + "balance_loss_clip": 1.06416488, + "balance_loss_mlp": 1.03736639, + "epoch": 0.16040401611254734, + "flos": 20961202648320.0, + "grad_norm": 2.148749904785113, + "language_loss": 0.78889954, + "learning_rate": 3.824377159014491e-06, + "loss": 0.81150126, + "num_input_tokens_seen": 28118790, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.2277832, + "step": 1334, + "time_per_iteration": 2.916368246078491 + }, + { + "auxiliary_loss_clip": 0.0120115, + "auxiliary_loss_mlp": 0.01054568, + "balance_loss_clip": 1.06646907, + "balance_loss_mlp": 1.03113198, + "epoch": 0.16052425900318643, + "flos": 21247051080960.0, + "grad_norm": 1.802095212222278, + "language_loss": 0.85024565, + "learning_rate": 3.824057820916702e-06, + "loss": 0.87280285, + "num_input_tokens_seen": 28135995, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.23449707, + "step": 1335, + "time_per_iteration": 3.026961326599121 + }, + { + "auxiliary_loss_clip": 0.0120745, + "auxiliary_loss_mlp": 0.01050312, + "balance_loss_clip": 1.06609964, + "balance_loss_mlp": 1.02582622, + "epoch": 0.16064450189382554, + "flos": 15524004096000.0, + "grad_norm": 3.084186554925239, + "language_loss": 0.72007567, + "learning_rate": 3.8237382061107904e-06, + "loss": 0.74265325, + "num_input_tokens_seen": 28152715, + "router_z_loss_clip": 1.4140625, + "router_z_loss_mlp": 0.24487305, + "step": 1336, + "time_per_iteration": 2.9162378311157227 + }, + { + "auxiliary_loss_clip": 0.01200515, + "auxiliary_loss_mlp": 0.01056294, + "balance_loss_clip": 1.06577027, + "balance_loss_mlp": 1.03314352, + "epoch": 0.16076474478446462, + "flos": 21178497974400.0, + "grad_norm": 2.046514459940328, + "language_loss": 0.78566682, + "learning_rate": 3.823418314645243e-06, + "loss": 0.80823493, + "num_input_tokens_seen": 28171590, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.23132324, + "step": 1337, + "time_per_iteration": 2.8231263160705566 + }, + { + "auxiliary_loss_clip": 0.0119858, + "auxiliary_loss_mlp": 0.01051928, + "balance_loss_clip": 1.06593454, + "balance_loss_mlp": 1.03004098, + "epoch": 0.1608849876751037, + "flos": 18366476912640.0, + "grad_norm": 6.1768021056164315, + "language_loss": 0.75070494, + "learning_rate": 3.823098146568588e-06, + "loss": 0.77321005, + "num_input_tokens_seen": 28191295, + "router_z_loss_clip": 1.32617188, + "router_z_loss_mlp": 0.21887207, + "step": 1338, + "time_per_iteration": 2.839024305343628 + }, + { + "auxiliary_loss_clip": 0.01198318, + "auxiliary_loss_mlp": 0.01055403, + "balance_loss_clip": 1.06702316, + "balance_loss_mlp": 1.03507805, + "epoch": 0.1610052305657428, + "flos": 29497024880640.0, + "grad_norm": 2.323309880030204, + "language_loss": 0.71349877, + "learning_rate": 3.822777701929394e-06, + "loss": 0.73603594, + "num_input_tokens_seen": 28213120, + "router_z_loss_clip": 1.31152344, + "router_z_loss_mlp": 0.20336914, + "step": 1339, + "time_per_iteration": 2.89347243309021 + }, + { + "auxiliary_loss_clip": 0.01202341, + "auxiliary_loss_mlp": 0.01059776, + "balance_loss_clip": 1.06640148, + "balance_loss_mlp": 1.03612494, + "epoch": 0.1611254734563819, + "flos": 26797871329920.0, + "grad_norm": 1.9914112749921364, + "language_loss": 0.73289466, + "learning_rate": 3.8224569807762714e-06, + "loss": 0.75551581, + "num_input_tokens_seen": 28232440, + "router_z_loss_clip": 1.35839844, + "router_z_loss_mlp": 0.23657227, + "step": 1340, + "time_per_iteration": 2.8409481048583984 + }, + { + "auxiliary_loss_clip": 0.01199361, + "auxiliary_loss_mlp": 0.01056195, + "balance_loss_clip": 1.06663859, + "balance_loss_mlp": 1.03526163, + "epoch": 0.16124571634702098, + "flos": 22419570741120.0, + "grad_norm": 4.09880138923911, + "language_loss": 0.76402956, + "learning_rate": 3.822135983157873e-06, + "loss": 0.78658515, + "num_input_tokens_seen": 28251715, + "router_z_loss_clip": 1.32714844, + "router_z_loss_mlp": 0.20935059, + "step": 1341, + "time_per_iteration": 2.8440651893615723 + }, + { + "auxiliary_loss_clip": 0.01198007, + "auxiliary_loss_mlp": 0.01051136, + "balance_loss_clip": 1.06703031, + "balance_loss_mlp": 1.02953482, + "epoch": 0.16136595923766006, + "flos": 10999116103680.0, + "grad_norm": 3.833366509319771, + "language_loss": 0.84265512, + "learning_rate": 3.821814709122896e-06, + "loss": 0.86514652, + "num_input_tokens_seen": 28269765, + "router_z_loss_clip": 1.30957031, + "router_z_loss_mlp": 0.21606445, + "step": 1342, + "time_per_iteration": 2.897799253463745 + }, + { + "auxiliary_loss_clip": 0.01200541, + "auxiliary_loss_mlp": 0.01055497, + "balance_loss_clip": 1.07000566, + "balance_loss_mlp": 1.03363407, + "epoch": 0.16148620212829917, + "flos": 21214983214080.0, + "grad_norm": 4.678466636128661, + "language_loss": 0.84556955, + "learning_rate": 3.821493158720076e-06, + "loss": 0.86812997, + "num_input_tokens_seen": 28288870, + "router_z_loss_clip": 1.30664062, + "router_z_loss_mlp": 0.21862793, + "step": 1343, + "time_per_iteration": 2.898670196533203 + }, + { + "auxiliary_loss_clip": 0.01205954, + "auxiliary_loss_mlp": 0.01059138, + "balance_loss_clip": 1.0665493, + "balance_loss_mlp": 1.03515363, + "epoch": 0.16160644501893826, + "flos": 16758468760320.0, + "grad_norm": 3.6054236865823173, + "language_loss": 0.73392427, + "learning_rate": 3.821171331998191e-06, + "loss": 0.75657523, + "num_input_tokens_seen": 28305400, + "router_z_loss_clip": 1.39550781, + "router_z_loss_mlp": 0.2401123, + "step": 1344, + "time_per_iteration": 2.7742257118225098 + }, + { + "auxiliary_loss_clip": 0.0112801, + "auxiliary_loss_mlp": 0.01005605, + "balance_loss_clip": 1.06083369, + "balance_loss_mlp": 1.00043154, + "epoch": 0.16172668790957734, + "flos": 64444967308800.0, + "grad_norm": 0.724935444408229, + "language_loss": 0.54466015, + "learning_rate": 3.820849229006064e-06, + "loss": 0.56599635, + "num_input_tokens_seen": 28373150, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.05175781, + "step": 1345, + "time_per_iteration": 3.478536367416382 + }, + { + "auxiliary_loss_clip": 0.01204515, + "auxiliary_loss_mlp": 0.01054243, + "balance_loss_clip": 1.06909537, + "balance_loss_mlp": 1.0328567, + "epoch": 0.16184693080021645, + "flos": 23257689759360.0, + "grad_norm": 3.102070318115379, + "language_loss": 0.70560849, + "learning_rate": 3.8205268497925564e-06, + "loss": 0.72819614, + "num_input_tokens_seen": 28393620, + "router_z_loss_clip": 1.35449219, + "router_z_loss_mlp": 0.21374512, + "step": 1346, + "time_per_iteration": 3.6967833042144775 + }, + { + "auxiliary_loss_clip": 0.01200369, + "auxiliary_loss_mlp": 0.01056554, + "balance_loss_clip": 1.06552744, + "balance_loss_mlp": 1.0342257, + "epoch": 0.16196717369085553, + "flos": 17451113696640.0, + "grad_norm": 2.812030091998173, + "language_loss": 0.78866208, + "learning_rate": 3.8202041944065725e-06, + "loss": 0.81123132, + "num_input_tokens_seen": 28409440, + "router_z_loss_clip": 1.34960938, + "router_z_loss_mlp": 0.2232666, + "step": 1347, + "time_per_iteration": 4.525706768035889 + }, + { + "auxiliary_loss_clip": 0.01201591, + "auxiliary_loss_mlp": 0.01049764, + "balance_loss_clip": 1.06751418, + "balance_loss_mlp": 1.02817512, + "epoch": 0.16208741658149461, + "flos": 23873377806720.0, + "grad_norm": 2.7145761750760027, + "language_loss": 0.73639441, + "learning_rate": 3.819881262897061e-06, + "loss": 0.75890803, + "num_input_tokens_seen": 28427575, + "router_z_loss_clip": 1.33886719, + "router_z_loss_mlp": 0.21606445, + "step": 1348, + "time_per_iteration": 3.643770694732666 + }, + { + "auxiliary_loss_clip": 0.01209729, + "auxiliary_loss_mlp": 0.01052647, + "balance_loss_clip": 1.0716567, + "balance_loss_mlp": 1.02837563, + "epoch": 0.1622076594721337, + "flos": 25884806584320.0, + "grad_norm": 2.271521701027083, + "language_loss": 0.73218977, + "learning_rate": 3.819558055313008e-06, + "loss": 0.75481355, + "num_input_tokens_seen": 28448260, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.24291992, + "step": 1349, + "time_per_iteration": 2.8322343826293945 + }, + { + "auxiliary_loss_clip": 0.01201298, + "auxiliary_loss_mlp": 0.01050186, + "balance_loss_clip": 1.0665822, + "balance_loss_mlp": 1.02891922, + "epoch": 0.1623279023627728, + "flos": 21539759011200.0, + "grad_norm": 2.066712417397929, + "language_loss": 0.77494782, + "learning_rate": 3.819234571703444e-06, + "loss": 0.7974627, + "num_input_tokens_seen": 28467085, + "router_z_loss_clip": 1.34570312, + "router_z_loss_mlp": 0.21289062, + "step": 1350, + "time_per_iteration": 2.805093765258789 + }, + { + "auxiliary_loss_clip": 0.01205433, + "auxiliary_loss_mlp": 0.0104975, + "balance_loss_clip": 1.06690598, + "balance_loss_mlp": 1.02581263, + "epoch": 0.1624481452534119, + "flos": 22085421494400.0, + "grad_norm": 2.226244647712148, + "language_loss": 0.85460556, + "learning_rate": 3.8189108121174435e-06, + "loss": 0.87715733, + "num_input_tokens_seen": 28486850, + "router_z_loss_clip": 1.38378906, + "router_z_loss_mlp": 0.23950195, + "step": 1351, + "time_per_iteration": 2.887803077697754 + }, + { + "auxiliary_loss_clip": 0.0120597, + "auxiliary_loss_mlp": 0.01056118, + "balance_loss_clip": 1.06842709, + "balance_loss_mlp": 1.03399289, + "epoch": 0.16256838814405097, + "flos": 27087490690560.0, + "grad_norm": 2.223627183243943, + "language_loss": 0.83442748, + "learning_rate": 3.818586776604118e-06, + "loss": 0.85704833, + "num_input_tokens_seen": 28507490, + "router_z_loss_clip": 1.375, + "router_z_loss_mlp": 0.22131348, + "step": 1352, + "time_per_iteration": 3.111494541168213 + }, + { + "auxiliary_loss_clip": 0.01202568, + "auxiliary_loss_mlp": 0.0105463, + "balance_loss_clip": 1.06665695, + "balance_loss_mlp": 1.03273165, + "epoch": 0.16268863103469008, + "flos": 20120354196480.0, + "grad_norm": 2.118483272390919, + "language_loss": 0.61560339, + "learning_rate": 3.818262465212625e-06, + "loss": 0.63817537, + "num_input_tokens_seen": 28527615, + "router_z_loss_clip": 1.35839844, + "router_z_loss_mlp": 0.21911621, + "step": 1353, + "time_per_iteration": 2.8403799533843994 + }, + { + "auxiliary_loss_clip": 0.0120479, + "auxiliary_loss_mlp": 0.01053596, + "balance_loss_clip": 1.06906581, + "balance_loss_mlp": 1.03107691, + "epoch": 0.16280887392532917, + "flos": 18332792933760.0, + "grad_norm": 2.910649093766731, + "language_loss": 0.76960123, + "learning_rate": 3.817937877992161e-06, + "loss": 0.79218507, + "num_input_tokens_seen": 28544910, + "router_z_loss_clip": 1.35449219, + "router_z_loss_mlp": 0.22509766, + "step": 1354, + "time_per_iteration": 2.8571012020111084 + }, + { + "auxiliary_loss_clip": 0.01205233, + "auxiliary_loss_mlp": 0.0105242, + "balance_loss_clip": 1.06708455, + "balance_loss_mlp": 1.02906752, + "epoch": 0.16292911681596825, + "flos": 11874330892800.0, + "grad_norm": 3.218488725289176, + "language_loss": 0.85659575, + "learning_rate": 3.817613014991967e-06, + "loss": 0.87917233, + "num_input_tokens_seen": 28561050, + "router_z_loss_clip": 1.38085938, + "router_z_loss_mlp": 0.23339844, + "step": 1355, + "time_per_iteration": 2.871354341506958 + }, + { + "auxiliary_loss_clip": 0.01205046, + "auxiliary_loss_mlp": 0.01049031, + "balance_loss_clip": 1.06584024, + "balance_loss_mlp": 1.02501035, + "epoch": 0.16304935970660733, + "flos": 26103466627200.0, + "grad_norm": 2.6441791857685346, + "language_loss": 0.76494765, + "learning_rate": 3.817287876261323e-06, + "loss": 0.7874884, + "num_input_tokens_seen": 28581385, + "router_z_loss_clip": 1.39160156, + "router_z_loss_mlp": 0.2401123, + "step": 1356, + "time_per_iteration": 2.908704996109009 + }, + { + "auxiliary_loss_clip": 0.01206586, + "auxiliary_loss_mlp": 0.01048719, + "balance_loss_clip": 1.06725395, + "balance_loss_mlp": 1.02564001, + "epoch": 0.16316960259724644, + "flos": 29351945848320.0, + "grad_norm": 4.2596501690730175, + "language_loss": 0.79756129, + "learning_rate": 3.816962461849553e-06, + "loss": 0.82011431, + "num_input_tokens_seen": 28603255, + "router_z_loss_clip": 1.39257812, + "router_z_loss_mlp": 0.23083496, + "step": 1357, + "time_per_iteration": 2.877495765686035 + }, + { + "auxiliary_loss_clip": 0.01204314, + "auxiliary_loss_mlp": 0.01052655, + "balance_loss_clip": 1.06798053, + "balance_loss_mlp": 1.02964723, + "epoch": 0.16328984548788553, + "flos": 20886759711360.0, + "grad_norm": 1.9923969178539644, + "language_loss": 0.84428287, + "learning_rate": 3.8166367718060235e-06, + "loss": 0.86685258, + "num_input_tokens_seen": 28623145, + "router_z_loss_clip": 1.36523438, + "router_z_loss_mlp": 0.2298584, + "step": 1358, + "time_per_iteration": 2.9698290824890137 + }, + { + "auxiliary_loss_clip": 0.01202746, + "auxiliary_loss_mlp": 0.01044325, + "balance_loss_clip": 1.06626308, + "balance_loss_mlp": 1.02153254, + "epoch": 0.1634100883785246, + "flos": 18041090584320.0, + "grad_norm": 2.5776950763847206, + "language_loss": 0.7624265, + "learning_rate": 3.816310806180139e-06, + "loss": 0.78489721, + "num_input_tokens_seen": 28641555, + "router_z_loss_clip": 1.36328125, + "router_z_loss_mlp": 0.22790527, + "step": 1359, + "time_per_iteration": 2.9347341060638428 + }, + { + "auxiliary_loss_clip": 0.01209303, + "auxiliary_loss_mlp": 0.01059202, + "balance_loss_clip": 1.06745505, + "balance_loss_mlp": 1.03488278, + "epoch": 0.16353033126916372, + "flos": 24572128055040.0, + "grad_norm": 2.257534224529236, + "language_loss": 0.80661267, + "learning_rate": 3.81598456502135e-06, + "loss": 0.82929778, + "num_input_tokens_seen": 28661575, + "router_z_loss_clip": 1.41894531, + "router_z_loss_mlp": 0.24328613, + "step": 1360, + "time_per_iteration": 2.8234217166900635 + }, + { + "auxiliary_loss_clip": 0.01203897, + "auxiliary_loss_mlp": 0.01053511, + "balance_loss_clip": 1.06600058, + "balance_loss_mlp": 1.03000307, + "epoch": 0.1636505741598028, + "flos": 19892895321600.0, + "grad_norm": 2.31683366491185, + "language_loss": 0.86850297, + "learning_rate": 3.8156580483791455e-06, + "loss": 0.89107704, + "num_input_tokens_seen": 28676765, + "router_z_loss_clip": 1.37792969, + "router_z_loss_mlp": 0.23510742, + "step": 1361, + "time_per_iteration": 2.837268114089966 + }, + { + "auxiliary_loss_clip": 0.01201478, + "auxiliary_loss_mlp": 0.01048111, + "balance_loss_clip": 1.0658474, + "balance_loss_mlp": 1.02709484, + "epoch": 0.16377081705044189, + "flos": 28402611344640.0, + "grad_norm": 3.4261463445418947, + "language_loss": 0.76766169, + "learning_rate": 3.815331256303059e-06, + "loss": 0.7901575, + "num_input_tokens_seen": 28696795, + "router_z_loss_clip": 1.35644531, + "router_z_loss_mlp": 0.21008301, + "step": 1362, + "time_per_iteration": 2.8183343410491943 + }, + { + "auxiliary_loss_clip": 0.0120887, + "auxiliary_loss_mlp": 0.01058939, + "balance_loss_clip": 1.06982028, + "balance_loss_mlp": 1.03779113, + "epoch": 0.163891059941081, + "flos": 21908059113600.0, + "grad_norm": 2.821022282383648, + "language_loss": 0.76930463, + "learning_rate": 3.815004188842665e-06, + "loss": 0.79198277, + "num_input_tokens_seen": 28714835, + "router_z_loss_clip": 1.38964844, + "router_z_loss_mlp": 0.21130371, + "step": 1363, + "time_per_iteration": 2.8395724296569824 + }, + { + "auxiliary_loss_clip": 0.01198531, + "auxiliary_loss_mlp": 0.0105473, + "balance_loss_clip": 1.06253076, + "balance_loss_mlp": 1.03039896, + "epoch": 0.16401130283172008, + "flos": 26797619934720.0, + "grad_norm": 1.8806709775031618, + "language_loss": 0.7972182, + "learning_rate": 3.814676846047578e-06, + "loss": 0.81975085, + "num_input_tokens_seen": 28735710, + "router_z_loss_clip": 1.35839844, + "router_z_loss_mlp": 0.24316406, + "step": 1364, + "time_per_iteration": 2.807992458343506 + }, + { + "auxiliary_loss_clip": 0.01203652, + "auxiliary_loss_mlp": 0.01059061, + "balance_loss_clip": 1.06566668, + "balance_loss_mlp": 1.03692436, + "epoch": 0.16413154572235916, + "flos": 32997417160320.0, + "grad_norm": 2.363593502718129, + "language_loss": 0.69488168, + "learning_rate": 3.8143492279674565e-06, + "loss": 0.71750879, + "num_input_tokens_seen": 28758405, + "router_z_loss_clip": 1.38085938, + "router_z_loss_mlp": 0.22131348, + "step": 1365, + "time_per_iteration": 2.883754253387451 + }, + { + "auxiliary_loss_clip": 0.01143546, + "auxiliary_loss_mlp": 0.01005836, + "balance_loss_clip": 1.06820738, + "balance_loss_mlp": 1.00004232, + "epoch": 0.16425178861299825, + "flos": 40113622074240.0, + "grad_norm": 0.8949827364925056, + "language_loss": 0.58420342, + "learning_rate": 3.8140213346519997e-06, + "loss": 0.60569721, + "num_input_tokens_seen": 28809000, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.05786133, + "step": 1366, + "time_per_iteration": 3.0735766887664795 + }, + { + "auxiliary_loss_clip": 0.01202841, + "auxiliary_loss_mlp": 0.0105583, + "balance_loss_clip": 1.06667876, + "balance_loss_mlp": 1.03227413, + "epoch": 0.16437203150363736, + "flos": 25447486498560.0, + "grad_norm": 2.8452605942775446, + "language_loss": 0.76705348, + "learning_rate": 3.813693166150948e-06, + "loss": 0.78964019, + "num_input_tokens_seen": 28829210, + "router_z_loss_clip": 1.35839844, + "router_z_loss_mlp": 0.2355957, + "step": 1367, + "time_per_iteration": 2.8278489112854004 + }, + { + "auxiliary_loss_clip": 0.01209444, + "auxiliary_loss_mlp": 0.01059056, + "balance_loss_clip": 1.06970453, + "balance_loss_mlp": 1.0351783, + "epoch": 0.16449227439427644, + "flos": 23476888506240.0, + "grad_norm": 2.7039649661707585, + "language_loss": 0.85139704, + "learning_rate": 3.813364722514086e-06, + "loss": 0.87408209, + "num_input_tokens_seen": 28847545, + "router_z_loss_clip": 1.39746094, + "router_z_loss_mlp": 0.2388916, + "step": 1368, + "time_per_iteration": 2.806873083114624 + }, + { + "auxiliary_loss_clip": 0.01203753, + "auxiliary_loss_mlp": 0.01065095, + "balance_loss_clip": 1.06649745, + "balance_loss_mlp": 1.04209924, + "epoch": 0.16461251728491552, + "flos": 13545217802880.0, + "grad_norm": 2.1557112936795724, + "language_loss": 0.80104256, + "learning_rate": 3.8130360037912368e-06, + "loss": 0.82373106, + "num_input_tokens_seen": 28863990, + "router_z_loss_clip": 1.37402344, + "router_z_loss_mlp": 0.22973633, + "step": 1369, + "time_per_iteration": 2.7844741344451904 + }, + { + "auxiliary_loss_clip": 0.01199704, + "auxiliary_loss_mlp": 0.0106241, + "balance_loss_clip": 1.06260276, + "balance_loss_mlp": 1.03950977, + "epoch": 0.16473276017555463, + "flos": 23003298662400.0, + "grad_norm": 4.581931855190112, + "language_loss": 0.81947267, + "learning_rate": 3.812707010032268e-06, + "loss": 0.84209383, + "num_input_tokens_seen": 28883045, + "router_z_loss_clip": 1.37109375, + "router_z_loss_mlp": 0.22912598, + "step": 1370, + "time_per_iteration": 2.838718891143799 + }, + { + "auxiliary_loss_clip": 0.01203605, + "auxiliary_loss_mlp": 0.01061716, + "balance_loss_clip": 1.06761456, + "balance_loss_mlp": 1.03764725, + "epoch": 0.16485300306619372, + "flos": 24790680357120.0, + "grad_norm": 2.020918774185248, + "language_loss": 0.79304636, + "learning_rate": 3.8123777412870863e-06, + "loss": 0.81569958, + "num_input_tokens_seen": 28902545, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.24072266, + "step": 1371, + "time_per_iteration": 2.8796303272247314 + }, + { + "auxiliary_loss_clip": 0.01202738, + "auxiliary_loss_mlp": 0.01066109, + "balance_loss_clip": 1.06474137, + "balance_loss_mlp": 1.04192162, + "epoch": 0.1649732459568328, + "flos": 21106497162240.0, + "grad_norm": 2.024826911783341, + "language_loss": 0.78473192, + "learning_rate": 3.812048197605643e-06, + "loss": 0.80742043, + "num_input_tokens_seen": 28921440, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.24194336, + "step": 1372, + "time_per_iteration": 2.852020025253296 + }, + { + "auxiliary_loss_clip": 0.01201279, + "auxiliary_loss_mlp": 0.01064795, + "balance_loss_clip": 1.06357419, + "balance_loss_mlp": 1.04184723, + "epoch": 0.16509348884747188, + "flos": 20266726118400.0, + "grad_norm": 2.024642449059875, + "language_loss": 0.81596535, + "learning_rate": 3.8117183790379277e-06, + "loss": 0.83862615, + "num_input_tokens_seen": 28939890, + "router_z_loss_clip": 1.37890625, + "router_z_loss_mlp": 0.22961426, + "step": 1373, + "time_per_iteration": 2.912858724594116 + }, + { + "auxiliary_loss_clip": 0.01199382, + "auxiliary_loss_mlp": 0.01060085, + "balance_loss_clip": 1.06289339, + "balance_loss_mlp": 1.03849673, + "epoch": 0.165213731738111, + "flos": 11035493602560.0, + "grad_norm": 4.322072345010329, + "language_loss": 0.93950474, + "learning_rate": 3.811388285633976e-06, + "loss": 0.96209937, + "num_input_tokens_seen": 28955875, + "router_z_loss_clip": 1.36328125, + "router_z_loss_mlp": 0.21569824, + "step": 1374, + "time_per_iteration": 2.878694534301758 + }, + { + "auxiliary_loss_clip": 0.01199408, + "auxiliary_loss_mlp": 0.01052113, + "balance_loss_clip": 1.06495893, + "balance_loss_mlp": 1.02968955, + "epoch": 0.16533397462875007, + "flos": 29972051268480.0, + "grad_norm": 2.629895796712098, + "language_loss": 0.62087852, + "learning_rate": 3.811057917443861e-06, + "loss": 0.64339375, + "num_input_tokens_seen": 28975140, + "router_z_loss_clip": 1.34375, + "router_z_loss_mlp": 0.22412109, + "step": 1375, + "time_per_iteration": 4.580889940261841 + }, + { + "auxiliary_loss_clip": 0.011021, + "auxiliary_loss_mlp": 0.01015393, + "balance_loss_clip": 1.04296374, + "balance_loss_mlp": 1.01143539, + "epoch": 0.16545421751938916, + "flos": 65556763027200.0, + "grad_norm": 2.3764542667719897, + "language_loss": 0.68332553, + "learning_rate": 3.8107272745177e-06, + "loss": 0.70450044, + "num_input_tokens_seen": 29047470, + "router_z_loss_clip": 0.58984375, + "router_z_loss_mlp": 0.03955078, + "step": 1376, + "time_per_iteration": 4.3733811378479 + }, + { + "auxiliary_loss_clip": 0.0120186, + "auxiliary_loss_mlp": 0.01053181, + "balance_loss_clip": 1.06565082, + "balance_loss_mlp": 1.03057837, + "epoch": 0.16557446041002827, + "flos": 22492361652480.0, + "grad_norm": 2.1013892763830784, + "language_loss": 0.784374, + "learning_rate": 3.8103963569056513e-06, + "loss": 0.8069244, + "num_input_tokens_seen": 29066605, + "router_z_loss_clip": 1.36035156, + "router_z_loss_mlp": 0.22607422, + "step": 1377, + "time_per_iteration": 3.7191543579101562 + }, + { + "auxiliary_loss_clip": 0.01197335, + "auxiliary_loss_mlp": 0.01053195, + "balance_loss_clip": 1.06437588, + "balance_loss_mlp": 1.03164184, + "epoch": 0.16569470330066735, + "flos": 24602723464320.0, + "grad_norm": 1.9522013249021075, + "language_loss": 0.88110566, + "learning_rate": 3.8100651646579146e-06, + "loss": 0.90361094, + "num_input_tokens_seen": 29085815, + "router_z_loss_clip": 1.33007812, + "router_z_loss_mlp": 0.21557617, + "step": 1378, + "time_per_iteration": 2.97412109375 + }, + { + "auxiliary_loss_clip": 0.01199497, + "auxiliary_loss_mlp": 0.01052719, + "balance_loss_clip": 1.06612277, + "balance_loss_mlp": 1.03073645, + "epoch": 0.16581494619130643, + "flos": 15006207588480.0, + "grad_norm": 2.6483827432629945, + "language_loss": 0.92308861, + "learning_rate": 3.8097336978247317e-06, + "loss": 0.94561076, + "num_input_tokens_seen": 29102520, + "router_z_loss_clip": 1.33105469, + "router_z_loss_mlp": 0.21984863, + "step": 1379, + "time_per_iteration": 2.8385872840881348 + }, + { + "auxiliary_loss_clip": 0.01202135, + "auxiliary_loss_mlp": 0.01057805, + "balance_loss_clip": 1.06434989, + "balance_loss_mlp": 1.03379643, + "epoch": 0.16593518908194552, + "flos": 17420338719360.0, + "grad_norm": 2.2984989574074004, + "language_loss": 0.8852877, + "learning_rate": 3.8094019564563854e-06, + "loss": 0.9078871, + "num_input_tokens_seen": 29119450, + "router_z_loss_clip": 1.37695312, + "router_z_loss_mlp": 0.24023438, + "step": 1380, + "time_per_iteration": 2.848933458328247 + }, + { + "auxiliary_loss_clip": 0.01203954, + "auxiliary_loss_mlp": 0.01048265, + "balance_loss_clip": 1.0669744, + "balance_loss_mlp": 1.02642608, + "epoch": 0.16605543197258463, + "flos": 20412631163520.0, + "grad_norm": 3.000741640289808, + "language_loss": 0.75403786, + "learning_rate": 3.809069940603201e-06, + "loss": 0.77656001, + "num_input_tokens_seen": 29137405, + "router_z_loss_clip": 1.3671875, + "router_z_loss_mlp": 0.21838379, + "step": 1381, + "time_per_iteration": 2.8610963821411133 + }, + { + "auxiliary_loss_clip": 0.01199506, + "auxiliary_loss_mlp": 0.01056993, + "balance_loss_clip": 1.06563973, + "balance_loss_mlp": 1.03455782, + "epoch": 0.1661756748632237, + "flos": 14209745368320.0, + "grad_norm": 2.1058858622905947, + "language_loss": 0.7768364, + "learning_rate": 3.8087376503155452e-06, + "loss": 0.79940134, + "num_input_tokens_seen": 29154890, + "router_z_loss_clip": 1.33886719, + "router_z_loss_mlp": 0.22436523, + "step": 1382, + "time_per_iteration": 2.875270366668701 + }, + { + "auxiliary_loss_clip": 0.01103165, + "auxiliary_loss_mlp": 0.01005487, + "balance_loss_clip": 1.04418123, + "balance_loss_mlp": 1.00160122, + "epoch": 0.1662959177538628, + "flos": 66080877350400.0, + "grad_norm": 0.8983371605887198, + "language_loss": 0.56229961, + "learning_rate": 3.808405085643826e-06, + "loss": 0.58338606, + "num_input_tokens_seen": 29219770, + "router_z_loss_clip": 0.58984375, + "router_z_loss_mlp": 0.03881836, + "step": 1383, + "time_per_iteration": 3.3905138969421387 + }, + { + "auxiliary_loss_clip": 0.01198312, + "auxiliary_loss_mlp": 0.01052816, + "balance_loss_clip": 1.06603575, + "balance_loss_mlp": 1.03069127, + "epoch": 0.1664161606445019, + "flos": 20740567357440.0, + "grad_norm": 2.482996765034791, + "language_loss": 0.89007896, + "learning_rate": 3.8080722466384925e-06, + "loss": 0.91259027, + "num_input_tokens_seen": 29237620, + "router_z_loss_clip": 1.32226562, + "router_z_loss_mlp": 0.22119141, + "step": 1384, + "time_per_iteration": 2.851264715194702 + }, + { + "auxiliary_loss_clip": 0.01204398, + "auxiliary_loss_mlp": 0.01059667, + "balance_loss_clip": 1.06301785, + "balance_loss_mlp": 1.03552723, + "epoch": 0.166536403535141, + "flos": 25260930236160.0, + "grad_norm": 2.1677094465812377, + "language_loss": 0.70626694, + "learning_rate": 3.8077391333500376e-06, + "loss": 0.72890759, + "num_input_tokens_seen": 29256760, + "router_z_loss_clip": 1.41308594, + "router_z_loss_mlp": 0.24145508, + "step": 1385, + "time_per_iteration": 2.8569085597991943 + }, + { + "auxiliary_loss_clip": 0.01197659, + "auxiliary_loss_mlp": 0.01064307, + "balance_loss_clip": 1.06611919, + "balance_loss_mlp": 1.0418241, + "epoch": 0.16665664642578007, + "flos": 25447450584960.0, + "grad_norm": 1.8159650063114061, + "language_loss": 0.76138234, + "learning_rate": 3.8074057458289934e-06, + "loss": 0.78400201, + "num_input_tokens_seen": 29277450, + "router_z_loss_clip": 1.31542969, + "router_z_loss_mlp": 0.22497559, + "step": 1386, + "time_per_iteration": 2.867875099182129 + }, + { + "auxiliary_loss_clip": 0.0120509, + "auxiliary_loss_mlp": 0.01055682, + "balance_loss_clip": 1.06567502, + "balance_loss_mlp": 1.03098226, + "epoch": 0.16677688931641918, + "flos": 22200767043840.0, + "grad_norm": 3.840721474414177, + "language_loss": 0.82265157, + "learning_rate": 3.807072084125934e-06, + "loss": 0.84525931, + "num_input_tokens_seen": 29299300, + "router_z_loss_clip": 1.39648438, + "router_z_loss_mlp": 0.24719238, + "step": 1387, + "time_per_iteration": 2.832498073577881 + }, + { + "auxiliary_loss_clip": 0.01201938, + "auxiliary_loss_mlp": 0.01056239, + "balance_loss_clip": 1.06485939, + "balance_loss_mlp": 1.03256392, + "epoch": 0.16689713220705826, + "flos": 16945958776320.0, + "grad_norm": 2.339335302990875, + "language_loss": 0.8043263, + "learning_rate": 3.806738148291477e-06, + "loss": 0.82690811, + "num_input_tokens_seen": 29316125, + "router_z_loss_clip": 1.36914062, + "router_z_loss_mlp": 0.23669434, + "step": 1388, + "time_per_iteration": 2.781724691390991 + }, + { + "auxiliary_loss_clip": 0.01208256, + "auxiliary_loss_mlp": 0.01065662, + "balance_loss_clip": 1.06663895, + "balance_loss_mlp": 1.04201078, + "epoch": 0.16701737509769735, + "flos": 36244423923840.0, + "grad_norm": 2.3140330524133934, + "language_loss": 0.71126366, + "learning_rate": 3.8064039383762793e-06, + "loss": 0.73400283, + "num_input_tokens_seen": 29338490, + "router_z_loss_clip": 1.41503906, + "router_z_loss_mlp": 0.23669434, + "step": 1389, + "time_per_iteration": 2.92907977104187 + }, + { + "auxiliary_loss_clip": 0.01203112, + "auxiliary_loss_mlp": 0.01068341, + "balance_loss_clip": 1.06680036, + "balance_loss_mlp": 1.04341388, + "epoch": 0.16713761798833643, + "flos": 23258659426560.0, + "grad_norm": 2.0943118362395547, + "language_loss": 0.765306, + "learning_rate": 3.8060694544310396e-06, + "loss": 0.78802055, + "num_input_tokens_seen": 29357000, + "router_z_loss_clip": 1.36425781, + "router_z_loss_mlp": 0.24938965, + "step": 1390, + "time_per_iteration": 2.813217878341675 + }, + { + "auxiliary_loss_clip": 0.0120438, + "auxiliary_loss_mlp": 0.0105152, + "balance_loss_clip": 1.0671947, + "balance_loss_mlp": 1.02715397, + "epoch": 0.16725786087897554, + "flos": 25302515207040.0, + "grad_norm": 1.9176267909893476, + "language_loss": 0.78607994, + "learning_rate": 3.8057346965065006e-06, + "loss": 0.80863893, + "num_input_tokens_seen": 29378230, + "router_z_loss_clip": 1.37207031, + "router_z_loss_mlp": 0.24353027, + "step": 1391, + "time_per_iteration": 2.8775863647460938 + }, + { + "auxiliary_loss_clip": 0.01200436, + "auxiliary_loss_mlp": 0.01054579, + "balance_loss_clip": 1.0663482, + "balance_loss_mlp": 1.03216755, + "epoch": 0.16737810376961462, + "flos": 31831541516160.0, + "grad_norm": 1.9989744211393374, + "language_loss": 0.84277433, + "learning_rate": 3.805399664653443e-06, + "loss": 0.8653245, + "num_input_tokens_seen": 29400370, + "router_z_loss_clip": 1.33789062, + "router_z_loss_mlp": 0.22424316, + "step": 1392, + "time_per_iteration": 2.893843650817871 + }, + { + "auxiliary_loss_clip": 0.01205412, + "auxiliary_loss_mlp": 0.01063806, + "balance_loss_clip": 1.06843686, + "balance_loss_mlp": 1.03926039, + "epoch": 0.1674983466602537, + "flos": 27961843553280.0, + "grad_norm": 2.3487034033344356, + "language_loss": 0.74062073, + "learning_rate": 3.805064358922692e-06, + "loss": 0.76331294, + "num_input_tokens_seen": 29418660, + "router_z_loss_clip": 1.36914062, + "router_z_loss_mlp": 0.24572754, + "step": 1393, + "time_per_iteration": 2.96309232711792 + }, + { + "auxiliary_loss_clip": 0.01207332, + "auxiliary_loss_mlp": 0.01071395, + "balance_loss_clip": 1.06681705, + "balance_loss_mlp": 1.04618216, + "epoch": 0.16761858955089282, + "flos": 21762656858880.0, + "grad_norm": 2.609453436127768, + "language_loss": 0.81109893, + "learning_rate": 3.8047287793651136e-06, + "loss": 0.83388615, + "num_input_tokens_seen": 29440105, + "router_z_loss_clip": 1.40625, + "router_z_loss_mlp": 0.25231934, + "step": 1394, + "time_per_iteration": 2.972360372543335 + }, + { + "auxiliary_loss_clip": 0.01206073, + "auxiliary_loss_mlp": 0.01052671, + "balance_loss_clip": 1.06781542, + "balance_loss_mlp": 1.0305934, + "epoch": 0.1677388324415319, + "flos": 23805507058560.0, + "grad_norm": 2.105826781628503, + "language_loss": 0.88420022, + "learning_rate": 3.8043929260316137e-06, + "loss": 0.90678769, + "num_input_tokens_seen": 29458260, + "router_z_loss_clip": 1.3828125, + "router_z_loss_mlp": 0.22070312, + "step": 1395, + "time_per_iteration": 2.933654546737671 + }, + { + "auxiliary_loss_clip": 0.01207304, + "auxiliary_loss_mlp": 0.01067637, + "balance_loss_clip": 1.07208192, + "balance_loss_mlp": 1.04458213, + "epoch": 0.16785907533217098, + "flos": 20558859431040.0, + "grad_norm": 2.301169461983554, + "language_loss": 0.8296594, + "learning_rate": 3.8040567989731417e-06, + "loss": 0.85240883, + "num_input_tokens_seen": 29476205, + "router_z_loss_clip": 1.35253906, + "router_z_loss_mlp": 0.23071289, + "step": 1396, + "time_per_iteration": 2.9358444213867188 + }, + { + "auxiliary_loss_clip": 0.01195839, + "auxiliary_loss_mlp": 0.01055291, + "balance_loss_clip": 1.06626701, + "balance_loss_mlp": 1.03215289, + "epoch": 0.16797931822281006, + "flos": 15669657745920.0, + "grad_norm": 2.1892108828189305, + "language_loss": 0.79754853, + "learning_rate": 3.8037203982406876e-06, + "loss": 0.82005984, + "num_input_tokens_seen": 29494370, + "router_z_loss_clip": 1.29492188, + "router_z_loss_mlp": 0.23144531, + "step": 1397, + "time_per_iteration": 2.828748941421509 + }, + { + "auxiliary_loss_clip": 0.01206518, + "auxiliary_loss_mlp": 0.01053459, + "balance_loss_clip": 1.07064724, + "balance_loss_mlp": 1.02915227, + "epoch": 0.16809956111344918, + "flos": 16541101607040.0, + "grad_norm": 2.000754382730878, + "language_loss": 0.72945261, + "learning_rate": 3.8033837238852835e-06, + "loss": 0.75205237, + "num_input_tokens_seen": 29511070, + "router_z_loss_clip": 1.35644531, + "router_z_loss_mlp": 0.2434082, + "step": 1398, + "time_per_iteration": 2.8481764793395996 + }, + { + "auxiliary_loss_clip": 0.01199258, + "auxiliary_loss_mlp": 0.01061304, + "balance_loss_clip": 1.06695819, + "balance_loss_mlp": 1.03929806, + "epoch": 0.16821980400408826, + "flos": 23258084808960.0, + "grad_norm": 2.0326643561244673, + "language_loss": 0.694736, + "learning_rate": 3.8030467759580017e-06, + "loss": 0.7173416, + "num_input_tokens_seen": 29531990, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.22009277, + "step": 1399, + "time_per_iteration": 2.9073073863983154 + }, + { + "auxiliary_loss_clip": 0.01202252, + "auxiliary_loss_mlp": 0.01050104, + "balance_loss_clip": 1.06605577, + "balance_loss_mlp": 1.02714443, + "epoch": 0.16834004689472734, + "flos": 20774754126720.0, + "grad_norm": 2.6315365401172492, + "language_loss": 0.87045777, + "learning_rate": 3.802709554509958e-06, + "loss": 0.89298135, + "num_input_tokens_seen": 29549790, + "router_z_loss_clip": 1.36230469, + "router_z_loss_mlp": 0.22961426, + "step": 1400, + "time_per_iteration": 2.9445443153381348 + }, + { + "auxiliary_loss_clip": 0.01198804, + "auxiliary_loss_mlp": 0.01047293, + "balance_loss_clip": 1.0640167, + "balance_loss_mlp": 1.0246309, + "epoch": 0.16846028978536645, + "flos": 26687302289280.0, + "grad_norm": 2.175619435217408, + "language_loss": 0.79072517, + "learning_rate": 3.8023720595923083e-06, + "loss": 0.81318611, + "num_input_tokens_seen": 29569045, + "router_z_loss_clip": 1.34765625, + "router_z_loss_mlp": 0.2265625, + "step": 1401, + "time_per_iteration": 2.893744945526123 + }, + { + "auxiliary_loss_clip": 0.01198523, + "auxiliary_loss_mlp": 0.01055458, + "balance_loss_clip": 1.06399226, + "balance_loss_mlp": 1.03308201, + "epoch": 0.16858053267600553, + "flos": 18843298980480.0, + "grad_norm": 4.434666268557063, + "language_loss": 0.87556994, + "learning_rate": 3.80203429125625e-06, + "loss": 0.89810967, + "num_input_tokens_seen": 29587220, + "router_z_loss_clip": 1.34375, + "router_z_loss_mlp": 0.22375488, + "step": 1402, + "time_per_iteration": 2.7791359424591064 + }, + { + "auxiliary_loss_clip": 0.01199854, + "auxiliary_loss_mlp": 0.01053479, + "balance_loss_clip": 1.06679177, + "balance_loss_mlp": 1.03074551, + "epoch": 0.16870077556664462, + "flos": 27744548227200.0, + "grad_norm": 2.2248688970524992, + "language_loss": 0.70107806, + "learning_rate": 3.8016962495530225e-06, + "loss": 0.72361135, + "num_input_tokens_seen": 29606410, + "router_z_loss_clip": 1.32910156, + "router_z_loss_mlp": 0.22705078, + "step": 1403, + "time_per_iteration": 2.817056655883789 + }, + { + "auxiliary_loss_clip": 0.01205288, + "auxiliary_loss_mlp": 0.01055381, + "balance_loss_clip": 1.068488, + "balance_loss_mlp": 1.03218281, + "epoch": 0.1688210184572837, + "flos": 13730768484480.0, + "grad_norm": 2.522193701728338, + "language_loss": 0.7677781, + "learning_rate": 3.8013579345339063e-06, + "loss": 0.79038477, + "num_input_tokens_seen": 29621275, + "router_z_loss_clip": 1.36816406, + "router_z_loss_mlp": 0.23193359, + "step": 1404, + "time_per_iteration": 3.680701494216919 + }, + { + "auxiliary_loss_clip": 0.01200513, + "auxiliary_loss_mlp": 0.01054807, + "balance_loss_clip": 1.06447244, + "balance_loss_mlp": 1.03140652, + "epoch": 0.1689412613479228, + "flos": 26468785900800.0, + "grad_norm": 2.182426127572126, + "language_loss": 0.69182462, + "learning_rate": 3.801019346250224e-06, + "loss": 0.71437788, + "num_input_tokens_seen": 29641420, + "router_z_loss_clip": 1.36035156, + "router_z_loss_mlp": 0.23388672, + "step": 1405, + "time_per_iteration": 3.650635242462158 + }, + { + "auxiliary_loss_clip": 0.01201809, + "auxiliary_loss_mlp": 0.01061081, + "balance_loss_clip": 1.06440651, + "balance_loss_mlp": 1.03788304, + "epoch": 0.1690615042385619, + "flos": 21138852337920.0, + "grad_norm": 3.407813119637628, + "language_loss": 0.83553326, + "learning_rate": 3.8006804847533395e-06, + "loss": 0.85816216, + "num_input_tokens_seen": 29660935, + "router_z_loss_clip": 1.37207031, + "router_z_loss_mlp": 0.23193359, + "step": 1406, + "time_per_iteration": 3.6380410194396973 + }, + { + "auxiliary_loss_clip": 0.0120079, + "auxiliary_loss_mlp": 0.01062083, + "balance_loss_clip": 1.06365144, + "balance_loss_mlp": 1.0381459, + "epoch": 0.16918174712920098, + "flos": 20849340718080.0, + "grad_norm": 2.064865452154769, + "language_loss": 0.85697103, + "learning_rate": 3.8003413500946556e-06, + "loss": 0.87959975, + "num_input_tokens_seen": 29681045, + "router_z_loss_clip": 1.37011719, + "router_z_loss_mlp": 0.23925781, + "step": 1407, + "time_per_iteration": 2.8175597190856934 + }, + { + "auxiliary_loss_clip": 0.01204406, + "auxiliary_loss_mlp": 0.01069003, + "balance_loss_clip": 1.06908035, + "balance_loss_mlp": 1.04452944, + "epoch": 0.1693019900198401, + "flos": 16983270028800.0, + "grad_norm": 3.3630494710519807, + "language_loss": 0.83644748, + "learning_rate": 3.8000019423256216e-06, + "loss": 0.85918158, + "num_input_tokens_seen": 29698810, + "router_z_loss_clip": 1.35351562, + "router_z_loss_mlp": 0.24475098, + "step": 1408, + "time_per_iteration": 2.749532461166382 + }, + { + "auxiliary_loss_clip": 0.0119702, + "auxiliary_loss_mlp": 0.01056182, + "balance_loss_clip": 1.06628489, + "balance_loss_mlp": 1.03273332, + "epoch": 0.16942223291047917, + "flos": 26796901662720.0, + "grad_norm": 1.93641519280601, + "language_loss": 0.8780126, + "learning_rate": 3.7996622614977234e-06, + "loss": 0.90054458, + "num_input_tokens_seen": 29720000, + "router_z_loss_clip": 1.30664062, + "router_z_loss_mlp": 0.23461914, + "step": 1409, + "time_per_iteration": 2.8520939350128174 + }, + { + "auxiliary_loss_clip": 0.01196063, + "auxiliary_loss_mlp": 0.01056487, + "balance_loss_clip": 1.06708884, + "balance_loss_mlp": 1.03443384, + "epoch": 0.16954247580111825, + "flos": 18583700411520.0, + "grad_norm": 2.3691798682817056, + "language_loss": 0.78784364, + "learning_rate": 3.799322307662492e-06, + "loss": 0.81036913, + "num_input_tokens_seen": 29737820, + "router_z_loss_clip": 1.28710938, + "router_z_loss_mlp": 0.22045898, + "step": 1410, + "time_per_iteration": 2.770415782928467 + }, + { + "auxiliary_loss_clip": 0.01198641, + "auxiliary_loss_mlp": 0.01059484, + "balance_loss_clip": 1.06308055, + "balance_loss_mlp": 1.03505862, + "epoch": 0.16966271869175734, + "flos": 13983651210240.0, + "grad_norm": 2.514101964175186, + "language_loss": 0.83683836, + "learning_rate": 3.798982080871496e-06, + "loss": 0.85941958, + "num_input_tokens_seen": 29752960, + "router_z_loss_clip": 1.35742188, + "router_z_loss_mlp": 0.2442627, + "step": 1411, + "time_per_iteration": 2.8331069946289062 + }, + { + "auxiliary_loss_clip": 0.01202848, + "auxiliary_loss_mlp": 0.01055717, + "balance_loss_clip": 1.06717908, + "balance_loss_mlp": 1.03114796, + "epoch": 0.16978296158239645, + "flos": 37487328284160.0, + "grad_norm": 2.2960043586024037, + "language_loss": 0.67485321, + "learning_rate": 3.798641581176349e-06, + "loss": 0.69743884, + "num_input_tokens_seen": 29775240, + "router_z_loss_clip": 1.35644531, + "router_z_loss_mlp": 0.24560547, + "step": 1412, + "time_per_iteration": 2.957547903060913 + }, + { + "auxiliary_loss_clip": 0.01197091, + "auxiliary_loss_mlp": 0.01064306, + "balance_loss_clip": 1.06363499, + "balance_loss_mlp": 1.0393908, + "epoch": 0.16990320447303553, + "flos": 28328958506880.0, + "grad_norm": 2.3864521773503533, + "language_loss": 0.74635541, + "learning_rate": 3.7983008086287044e-06, + "loss": 0.76896936, + "num_input_tokens_seen": 29796560, + "router_z_loss_clip": 1.33496094, + "router_z_loss_mlp": 0.24914551, + "step": 1413, + "time_per_iteration": 2.833977699279785 + }, + { + "auxiliary_loss_clip": 0.0120476, + "auxiliary_loss_mlp": 0.01064345, + "balance_loss_clip": 1.06524563, + "balance_loss_mlp": 1.03938234, + "epoch": 0.1700234473636746, + "flos": 20188189031040.0, + "grad_norm": 2.5205071963976713, + "language_loss": 0.79633605, + "learning_rate": 3.797959763280257e-06, + "loss": 0.81902707, + "num_input_tokens_seen": 29815245, + "router_z_loss_clip": 1.39355469, + "router_z_loss_mlp": 0.24951172, + "step": 1414, + "time_per_iteration": 2.8760251998901367 + }, + { + "auxiliary_loss_clip": 0.01203202, + "auxiliary_loss_mlp": 0.0105558, + "balance_loss_clip": 1.06628406, + "balance_loss_mlp": 1.03186953, + "epoch": 0.17014369025431372, + "flos": 24858658846080.0, + "grad_norm": 2.1261163046799503, + "language_loss": 0.78758323, + "learning_rate": 3.797618445182743e-06, + "loss": 0.81017101, + "num_input_tokens_seen": 29836640, + "router_z_loss_clip": 1.36914062, + "router_z_loss_mlp": 0.23730469, + "step": 1415, + "time_per_iteration": 2.7949728965759277 + }, + { + "auxiliary_loss_clip": 0.01203186, + "auxiliary_loss_mlp": 0.01062262, + "balance_loss_clip": 1.06836832, + "balance_loss_mlp": 1.03778839, + "epoch": 0.1702639331449528, + "flos": 16467233287680.0, + "grad_norm": 2.5334560438885134, + "language_loss": 0.84903359, + "learning_rate": 3.79727685438794e-06, + "loss": 0.87168813, + "num_input_tokens_seen": 29850830, + "router_z_loss_clip": 1.34960938, + "router_z_loss_mlp": 0.24462891, + "step": 1416, + "time_per_iteration": 2.8169727325439453 + }, + { + "auxiliary_loss_clip": 0.01096663, + "auxiliary_loss_mlp": 0.01004733, + "balance_loss_clip": 1.03738642, + "balance_loss_mlp": 1.00098956, + "epoch": 0.1703841760355919, + "flos": 52508870979840.0, + "grad_norm": 1.3526475796810653, + "language_loss": 0.61595881, + "learning_rate": 3.796934990947667e-06, + "loss": 0.63697278, + "num_input_tokens_seen": 29912515, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.03735352, + "step": 1417, + "time_per_iteration": 3.3498196601867676 + }, + { + "auxiliary_loss_clip": 0.01096811, + "auxiliary_loss_mlp": 0.00999507, + "balance_loss_clip": 1.03723288, + "balance_loss_mlp": 0.99557316, + "epoch": 0.170504418926231, + "flos": 49370637576960.0, + "grad_norm": 0.8724572867381437, + "language_loss": 0.62421417, + "learning_rate": 3.7965928549137854e-06, + "loss": 0.64517736, + "num_input_tokens_seen": 29969330, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.03930664, + "step": 1418, + "time_per_iteration": 3.230722665786743 + }, + { + "auxiliary_loss_clip": 0.01207194, + "auxiliary_loss_mlp": 0.01058021, + "balance_loss_clip": 1.0652504, + "balance_loss_mlp": 1.03471601, + "epoch": 0.17062466181687008, + "flos": 25849219184640.0, + "grad_norm": 2.7373505920760137, + "language_loss": 0.77625656, + "learning_rate": 3.7962504463381953e-06, + "loss": 0.79890871, + "num_input_tokens_seen": 29990820, + "router_z_loss_clip": 1.41992188, + "router_z_loss_mlp": 0.23291016, + "step": 1419, + "time_per_iteration": 2.8671586513519287 + }, + { + "auxiliary_loss_clip": 0.01206814, + "auxiliary_loss_mlp": 0.01063977, + "balance_loss_clip": 1.06936288, + "balance_loss_mlp": 1.03980184, + "epoch": 0.17074490470750917, + "flos": 20960412549120.0, + "grad_norm": 2.014265090204997, + "language_loss": 0.78883696, + "learning_rate": 3.7959077652728412e-06, + "loss": 0.81154484, + "num_input_tokens_seen": 30009275, + "router_z_loss_clip": 1.37402344, + "router_z_loss_mlp": 0.24169922, + "step": 1420, + "time_per_iteration": 2.825772285461426 + }, + { + "auxiliary_loss_clip": 0.01201952, + "auxiliary_loss_mlp": 0.01055711, + "balance_loss_clip": 1.06722856, + "balance_loss_mlp": 1.03414631, + "epoch": 0.17086514759814825, + "flos": 20959766104320.0, + "grad_norm": 2.2751833707984734, + "language_loss": 0.76918912, + "learning_rate": 3.795564811769707e-06, + "loss": 0.79176575, + "num_input_tokens_seen": 30027630, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.21557617, + "step": 1421, + "time_per_iteration": 2.7936198711395264 + }, + { + "auxiliary_loss_clip": 0.0120229, + "auxiliary_loss_mlp": 0.01057841, + "balance_loss_clip": 1.06862831, + "balance_loss_mlp": 1.03528643, + "epoch": 0.17098539048878736, + "flos": 28474073452800.0, + "grad_norm": 2.735748941599682, + "language_loss": 0.78248358, + "learning_rate": 3.795221585880818e-06, + "loss": 0.80508494, + "num_input_tokens_seen": 30048310, + "router_z_loss_clip": 1.33496094, + "router_z_loss_mlp": 0.22570801, + "step": 1422, + "time_per_iteration": 2.917802333831787 + }, + { + "auxiliary_loss_clip": 0.01204709, + "auxiliary_loss_mlp": 0.01056668, + "balance_loss_clip": 1.0710268, + "balance_loss_mlp": 1.03467417, + "epoch": 0.17110563337942644, + "flos": 16290014561280.0, + "grad_norm": 2.428000340265605, + "language_loss": 0.91392219, + "learning_rate": 3.794878087658242e-06, + "loss": 0.93653601, + "num_input_tokens_seen": 30066080, + "router_z_loss_clip": 1.33691406, + "router_z_loss_mlp": 0.2199707, + "step": 1423, + "time_per_iteration": 2.7759644985198975 + }, + { + "auxiliary_loss_clip": 0.01202873, + "auxiliary_loss_mlp": 0.01053633, + "balance_loss_clip": 1.06747317, + "balance_loss_mlp": 1.029791, + "epoch": 0.17122587627006552, + "flos": 29674207693440.0, + "grad_norm": 1.9836543326828677, + "language_loss": 0.78751242, + "learning_rate": 3.7945343171540873e-06, + "loss": 0.81007749, + "num_input_tokens_seen": 30086955, + "router_z_loss_clip": 1.35546875, + "router_z_loss_mlp": 0.23840332, + "step": 1424, + "time_per_iteration": 2.850329637527466 + }, + { + "auxiliary_loss_clip": 0.01204932, + "auxiliary_loss_mlp": 0.01055365, + "balance_loss_clip": 1.06767452, + "balance_loss_mlp": 1.03328741, + "epoch": 0.17134611916070464, + "flos": 25338389915520.0, + "grad_norm": 2.1267905249342323, + "language_loss": 0.79066277, + "learning_rate": 3.7941902744205033e-06, + "loss": 0.81326574, + "num_input_tokens_seen": 30107990, + "router_z_loss_clip": 1.375, + "router_z_loss_mlp": 0.22070312, + "step": 1425, + "time_per_iteration": 2.854215621948242 + }, + { + "auxiliary_loss_clip": 0.01205772, + "auxiliary_loss_mlp": 0.01058268, + "balance_loss_clip": 1.06896973, + "balance_loss_mlp": 1.03512907, + "epoch": 0.17146636205134372, + "flos": 13953845900160.0, + "grad_norm": 2.5491348700173284, + "language_loss": 0.83743894, + "learning_rate": 3.7938459595096817e-06, + "loss": 0.86007929, + "num_input_tokens_seen": 30126535, + "router_z_loss_clip": 1.36816406, + "router_z_loss_mlp": 0.23156738, + "step": 1426, + "time_per_iteration": 2.8643651008605957 + }, + { + "auxiliary_loss_clip": 0.01209473, + "auxiliary_loss_mlp": 0.01062694, + "balance_loss_clip": 1.0688169, + "balance_loss_mlp": 1.03765988, + "epoch": 0.1715866049419828, + "flos": 23915214172800.0, + "grad_norm": 2.019534917092668, + "language_loss": 0.86356413, + "learning_rate": 3.7935013724738545e-06, + "loss": 0.88628584, + "num_input_tokens_seen": 30147035, + "router_z_loss_clip": 1.40917969, + "router_z_loss_mlp": 0.25036621, + "step": 1427, + "time_per_iteration": 2.7814433574676514 + }, + { + "auxiliary_loss_clip": 0.01199525, + "auxiliary_loss_mlp": 0.01053846, + "balance_loss_clip": 1.06495655, + "balance_loss_mlp": 1.03251898, + "epoch": 0.17170684783262188, + "flos": 22709369669760.0, + "grad_norm": 2.93195753183784, + "language_loss": 0.78093076, + "learning_rate": 3.7931565133652945e-06, + "loss": 0.80346447, + "num_input_tokens_seen": 30167110, + "router_z_loss_clip": 1.34570312, + "router_z_loss_mlp": 0.21313477, + "step": 1428, + "time_per_iteration": 2.8498001098632812 + }, + { + "auxiliary_loss_clip": 0.01196652, + "auxiliary_loss_mlp": 0.01057683, + "balance_loss_clip": 1.06080699, + "balance_loss_mlp": 1.03449726, + "epoch": 0.171827090723261, + "flos": 26613290315520.0, + "grad_norm": 2.631662127029358, + "language_loss": 0.68409145, + "learning_rate": 3.792811382236317e-06, + "loss": 0.70663482, + "num_input_tokens_seen": 30185620, + "router_z_loss_clip": 1.359375, + "router_z_loss_mlp": 0.23181152, + "step": 1429, + "time_per_iteration": 2.878784656524658 + }, + { + "auxiliary_loss_clip": 0.0120584, + "auxiliary_loss_mlp": 0.0105861, + "balance_loss_clip": 1.06774819, + "balance_loss_mlp": 1.03505456, + "epoch": 0.17194733361390008, + "flos": 28148507556480.0, + "grad_norm": 3.9650301069697633, + "language_loss": 0.78267062, + "learning_rate": 3.792465979139279e-06, + "loss": 0.80531514, + "num_input_tokens_seen": 30208225, + "router_z_loss_clip": 1.3828125, + "router_z_loss_mlp": 0.23547363, + "step": 1430, + "time_per_iteration": 2.958195209503174 + }, + { + "auxiliary_loss_clip": 0.01107995, + "auxiliary_loss_mlp": 0.01032307, + "balance_loss_clip": 1.04383612, + "balance_loss_mlp": 1.028373, + "epoch": 0.17206757650453916, + "flos": 65530689753600.0, + "grad_norm": 0.9286409258351193, + "language_loss": 0.65646464, + "learning_rate": 3.792120304126576e-06, + "loss": 0.67786765, + "num_input_tokens_seen": 30271600, + "router_z_loss_clip": 0.640625, + "router_z_loss_mlp": 0.03930664, + "step": 1431, + "time_per_iteration": 3.4127535820007324 + }, + { + "auxiliary_loss_clip": 0.0119939, + "auxiliary_loss_mlp": 0.01052918, + "balance_loss_clip": 1.06488037, + "balance_loss_mlp": 1.03054202, + "epoch": 0.17218781939517827, + "flos": 22273486128000.0, + "grad_norm": 2.597326047502173, + "language_loss": 0.83729792, + "learning_rate": 3.791774357250649e-06, + "loss": 0.85982096, + "num_input_tokens_seen": 30290430, + "router_z_loss_clip": 1.34375, + "router_z_loss_mlp": 0.22387695, + "step": 1432, + "time_per_iteration": 2.8445699214935303 + }, + { + "auxiliary_loss_clip": 0.01200974, + "auxiliary_loss_mlp": 0.01062042, + "balance_loss_clip": 1.06346941, + "balance_loss_mlp": 1.03734159, + "epoch": 0.17230806228581735, + "flos": 14137313592960.0, + "grad_norm": 4.242583482875892, + "language_loss": 0.78833318, + "learning_rate": 3.7914281385639757e-06, + "loss": 0.81096339, + "num_input_tokens_seen": 30308305, + "router_z_loss_clip": 1.37304688, + "router_z_loss_mlp": 0.24682617, + "step": 1433, + "time_per_iteration": 4.542450428009033 + }, + { + "auxiliary_loss_clip": 0.01194907, + "auxiliary_loss_mlp": 0.01056017, + "balance_loss_clip": 1.06146359, + "balance_loss_mlp": 1.03303337, + "epoch": 0.17242830517645644, + "flos": 20704836303360.0, + "grad_norm": 2.083855594016595, + "language_loss": 0.79786062, + "learning_rate": 3.7910816481190784e-06, + "loss": 0.82036984, + "num_input_tokens_seen": 30328120, + "router_z_loss_clip": 1.33203125, + "router_z_loss_mlp": 0.2298584, + "step": 1434, + "time_per_iteration": 3.7012383937835693 + }, + { + "auxiliary_loss_clip": 0.01198915, + "auxiliary_loss_mlp": 0.0104983, + "balance_loss_clip": 1.06315207, + "balance_loss_mlp": 1.02650094, + "epoch": 0.17254854806709552, + "flos": 30774582887040.0, + "grad_norm": 2.8232991891781176, + "language_loss": 0.74642849, + "learning_rate": 3.7907348859685193e-06, + "loss": 0.76891595, + "num_input_tokens_seen": 30349825, + "router_z_loss_clip": 1.35742188, + "router_z_loss_mlp": 0.2331543, + "step": 1435, + "time_per_iteration": 3.703270196914673 + }, + { + "auxiliary_loss_clip": 0.01195756, + "auxiliary_loss_mlp": 0.010558, + "balance_loss_clip": 1.06369352, + "balance_loss_mlp": 1.0334723, + "epoch": 0.17266879095773463, + "flos": 26614726859520.0, + "grad_norm": 2.5045854420857503, + "language_loss": 0.81018245, + "learning_rate": 3.790387852164902e-06, + "loss": 0.83269799, + "num_input_tokens_seen": 30370555, + "router_z_loss_clip": 1.3203125, + "router_z_loss_mlp": 0.2232666, + "step": 1436, + "time_per_iteration": 2.860485076904297 + }, + { + "auxiliary_loss_clip": 0.01198681, + "auxiliary_loss_mlp": 0.01046012, + "balance_loss_clip": 1.06581259, + "balance_loss_mlp": 1.02459002, + "epoch": 0.1727890338483737, + "flos": 20266295155200.0, + "grad_norm": 2.3261916436170043, + "language_loss": 0.76589298, + "learning_rate": 3.7900405467608707e-06, + "loss": 0.78833985, + "num_input_tokens_seen": 30390100, + "router_z_loss_clip": 1.33105469, + "router_z_loss_mlp": 0.21411133, + "step": 1437, + "time_per_iteration": 2.8136188983917236 + }, + { + "auxiliary_loss_clip": 0.01202604, + "auxiliary_loss_mlp": 0.01053001, + "balance_loss_clip": 1.06570482, + "balance_loss_mlp": 1.02974296, + "epoch": 0.1729092767390128, + "flos": 18179812909440.0, + "grad_norm": 3.204912192322694, + "language_loss": 0.79322433, + "learning_rate": 3.7896929698091114e-06, + "loss": 0.81578034, + "num_input_tokens_seen": 30402915, + "router_z_loss_clip": 1.36914062, + "router_z_loss_mlp": 0.23266602, + "step": 1438, + "time_per_iteration": 2.848728895187378 + }, + { + "auxiliary_loss_clip": 0.01201732, + "auxiliary_loss_mlp": 0.01049095, + "balance_loss_clip": 1.06742215, + "balance_loss_mlp": 1.02710068, + "epoch": 0.1730295196296519, + "flos": 26759518583040.0, + "grad_norm": 3.0713321414028143, + "language_loss": 0.6838026, + "learning_rate": 3.7893451213623518e-06, + "loss": 0.70631087, + "num_input_tokens_seen": 30420145, + "router_z_loss_clip": 1.34277344, + "router_z_loss_mlp": 0.2199707, + "step": 1439, + "time_per_iteration": 2.9851012229919434 + }, + { + "auxiliary_loss_clip": 0.01202748, + "auxiliary_loss_mlp": 0.01061638, + "balance_loss_clip": 1.0648973, + "balance_loss_mlp": 1.03941727, + "epoch": 0.173149762520291, + "flos": 23842531002240.0, + "grad_norm": 2.7545445488552733, + "language_loss": 0.82763171, + "learning_rate": 3.7889970014733606e-06, + "loss": 0.85027564, + "num_input_tokens_seen": 30439250, + "router_z_loss_clip": 1.37890625, + "router_z_loss_mlp": 0.22216797, + "step": 1440, + "time_per_iteration": 2.8194546699523926 + }, + { + "auxiliary_loss_clip": 0.01195496, + "auxiliary_loss_mlp": 0.01048928, + "balance_loss_clip": 1.06247783, + "balance_loss_mlp": 1.02566993, + "epoch": 0.17327000541093007, + "flos": 23368186972800.0, + "grad_norm": 1.9131187870582629, + "language_loss": 0.78267264, + "learning_rate": 3.7886486101949463e-06, + "loss": 0.80511689, + "num_input_tokens_seen": 30460430, + "router_z_loss_clip": 1.33203125, + "router_z_loss_mlp": 0.23266602, + "step": 1441, + "time_per_iteration": 2.8782882690429688 + }, + { + "auxiliary_loss_clip": 0.01198828, + "auxiliary_loss_mlp": 0.01055237, + "balance_loss_clip": 1.06667566, + "balance_loss_mlp": 1.03259933, + "epoch": 0.17339024830156918, + "flos": 18221290139520.0, + "grad_norm": 2.524115414093824, + "language_loss": 0.87798429, + "learning_rate": 3.7882999475799594e-06, + "loss": 0.90052497, + "num_input_tokens_seen": 30478465, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.22619629, + "step": 1442, + "time_per_iteration": 2.829793930053711 + }, + { + "auxiliary_loss_clip": 0.01200233, + "auxiliary_loss_mlp": 0.01056383, + "balance_loss_clip": 1.06561291, + "balance_loss_mlp": 1.03351855, + "epoch": 0.17351049119220827, + "flos": 23332024955520.0, + "grad_norm": 2.13187273731603, + "language_loss": 0.81391072, + "learning_rate": 3.787951013681293e-06, + "loss": 0.83647692, + "num_input_tokens_seen": 30496510, + "router_z_loss_clip": 1.34472656, + "router_z_loss_mlp": 0.2286377, + "step": 1443, + "time_per_iteration": 2.78722882270813 + }, + { + "auxiliary_loss_clip": 0.01202468, + "auxiliary_loss_mlp": 0.01051994, + "balance_loss_clip": 1.06650221, + "balance_loss_mlp": 1.0290345, + "epoch": 0.17363073408284735, + "flos": 23803495896960.0, + "grad_norm": 2.7615127513573903, + "language_loss": 0.77373618, + "learning_rate": 3.787601808551879e-06, + "loss": 0.7962808, + "num_input_tokens_seen": 30516325, + "router_z_loss_clip": 1.36035156, + "router_z_loss_mlp": 0.22961426, + "step": 1444, + "time_per_iteration": 2.816199779510498 + }, + { + "auxiliary_loss_clip": 0.01202177, + "auxiliary_loss_mlp": 0.01048417, + "balance_loss_clip": 1.06584239, + "balance_loss_mlp": 1.02548075, + "epoch": 0.17375097697348643, + "flos": 18515290959360.0, + "grad_norm": 3.252641527699548, + "language_loss": 0.84576178, + "learning_rate": 3.7872523322446926e-06, + "loss": 0.86826766, + "num_input_tokens_seen": 30535210, + "router_z_loss_clip": 1.36230469, + "router_z_loss_mlp": 0.22949219, + "step": 1445, + "time_per_iteration": 2.9175117015838623 + }, + { + "auxiliary_loss_clip": 0.01197983, + "auxiliary_loss_mlp": 0.010503, + "balance_loss_clip": 1.06491446, + "balance_loss_mlp": 1.02792454, + "epoch": 0.17387121986412554, + "flos": 38877897456000.0, + "grad_norm": 1.819935384208245, + "language_loss": 0.60408616, + "learning_rate": 3.7869025848127478e-06, + "loss": 0.62656903, + "num_input_tokens_seen": 30559405, + "router_z_loss_clip": 1.32910156, + "router_z_loss_mlp": 0.22375488, + "step": 1446, + "time_per_iteration": 2.945838451385498 + }, + { + "auxiliary_loss_clip": 0.01197596, + "auxiliary_loss_mlp": 0.01055857, + "balance_loss_clip": 1.06400418, + "balance_loss_mlp": 1.03343391, + "epoch": 0.17399146275476463, + "flos": 20375714960640.0, + "grad_norm": 4.470527819878048, + "language_loss": 0.80986375, + "learning_rate": 3.786552566309102e-06, + "loss": 0.8323983, + "num_input_tokens_seen": 30577615, + "router_z_loss_clip": 1.3359375, + "router_z_loss_mlp": 0.22424316, + "step": 1447, + "time_per_iteration": 2.8435592651367188 + }, + { + "auxiliary_loss_clip": 0.01200252, + "auxiliary_loss_mlp": 0.01058988, + "balance_loss_clip": 1.06510639, + "balance_loss_mlp": 1.03665972, + "epoch": 0.1741117056454037, + "flos": 19164339763200.0, + "grad_norm": 2.310121032260093, + "language_loss": 0.85986352, + "learning_rate": 3.7862022767868517e-06, + "loss": 0.88245589, + "num_input_tokens_seen": 30595205, + "router_z_loss_clip": 1.34960938, + "router_z_loss_mlp": 0.22314453, + "step": 1448, + "time_per_iteration": 2.802739381790161 + }, + { + "auxiliary_loss_clip": 0.01195426, + "auxiliary_loss_mlp": 0.0105409, + "balance_loss_clip": 1.06498384, + "balance_loss_mlp": 1.03183365, + "epoch": 0.17423194853604282, + "flos": 25374300537600.0, + "grad_norm": 5.046903935475531, + "language_loss": 0.84349632, + "learning_rate": 3.7858517162991367e-06, + "loss": 0.86599147, + "num_input_tokens_seen": 30615280, + "router_z_loss_clip": 1.30664062, + "router_z_loss_mlp": 0.22265625, + "step": 1449, + "time_per_iteration": 2.810229539871216 + }, + { + "auxiliary_loss_clip": 0.01202104, + "auxiliary_loss_mlp": 0.01055328, + "balance_loss_clip": 1.06673622, + "balance_loss_mlp": 1.03354824, + "epoch": 0.1743521914266819, + "flos": 25191874339200.0, + "grad_norm": 3.133449767725836, + "language_loss": 0.60481876, + "learning_rate": 3.7855008848991363e-06, + "loss": 0.62739313, + "num_input_tokens_seen": 30633485, + "router_z_loss_clip": 1.35253906, + "router_z_loss_mlp": 0.21777344, + "step": 1450, + "time_per_iteration": 2.8235480785369873 + }, + { + "auxiliary_loss_clip": 0.01191509, + "auxiliary_loss_mlp": 0.01050236, + "balance_loss_clip": 1.06404245, + "balance_loss_mlp": 1.02822948, + "epoch": 0.17447243431732098, + "flos": 25666577504640.0, + "grad_norm": 1.8667934258157615, + "language_loss": 0.77601296, + "learning_rate": 3.7851497826400714e-06, + "loss": 0.79843044, + "num_input_tokens_seen": 30653625, + "router_z_loss_clip": 1.2734375, + "router_z_loss_mlp": 0.22009277, + "step": 1451, + "time_per_iteration": 2.907879590988159 + }, + { + "auxiliary_loss_clip": 0.01199199, + "auxiliary_loss_mlp": 0.01058344, + "balance_loss_clip": 1.0649904, + "balance_loss_mlp": 1.03475237, + "epoch": 0.17459267720796007, + "flos": 36281950657920.0, + "grad_norm": 1.8990774456593558, + "language_loss": 0.76079077, + "learning_rate": 3.7847984095752034e-06, + "loss": 0.7833662, + "num_input_tokens_seen": 30677080, + "router_z_loss_clip": 1.34375, + "router_z_loss_mlp": 0.23608398, + "step": 1452, + "time_per_iteration": 2.981259346008301 + }, + { + "auxiliary_loss_clip": 0.01195324, + "auxiliary_loss_mlp": 0.01053195, + "balance_loss_clip": 1.06273353, + "balance_loss_mlp": 1.02993751, + "epoch": 0.17471292009859918, + "flos": 20011113959040.0, + "grad_norm": 2.451488259410039, + "language_loss": 0.802082, + "learning_rate": 3.784446765757836e-06, + "loss": 0.8245672, + "num_input_tokens_seen": 30695725, + "router_z_loss_clip": 1.32421875, + "router_z_loss_mlp": 0.23254395, + "step": 1453, + "time_per_iteration": 2.9032816886901855 + }, + { + "auxiliary_loss_clip": 0.01195849, + "auxiliary_loss_mlp": 0.01060363, + "balance_loss_clip": 1.06439936, + "balance_loss_mlp": 1.03642607, + "epoch": 0.17483316298923826, + "flos": 27819242559360.0, + "grad_norm": 2.532574494140639, + "language_loss": 0.77437824, + "learning_rate": 3.7840948512413133e-06, + "loss": 0.79694033, + "num_input_tokens_seen": 30713310, + "router_z_loss_clip": 1.31542969, + "router_z_loss_mlp": 0.23925781, + "step": 1454, + "time_per_iteration": 2.9011383056640625 + }, + { + "auxiliary_loss_clip": 0.01202628, + "auxiliary_loss_mlp": 0.01049651, + "balance_loss_clip": 1.064291, + "balance_loss_mlp": 1.02670312, + "epoch": 0.17495340587987734, + "flos": 44017934791680.0, + "grad_norm": 2.0950784004808223, + "language_loss": 0.78716528, + "learning_rate": 3.7837426660790196e-06, + "loss": 0.80968809, + "num_input_tokens_seen": 30734725, + "router_z_loss_clip": 1.38476562, + "router_z_loss_mlp": 0.22937012, + "step": 1455, + "time_per_iteration": 3.079136610031128 + }, + { + "auxiliary_loss_clip": 0.01200238, + "auxiliary_loss_mlp": 0.01048131, + "balance_loss_clip": 1.06565726, + "balance_loss_mlp": 1.02566016, + "epoch": 0.17507364877051645, + "flos": 20885825957760.0, + "grad_norm": 2.27499931710443, + "language_loss": 0.81656146, + "learning_rate": 3.783390210324382e-06, + "loss": 0.83904517, + "num_input_tokens_seen": 30754450, + "router_z_loss_clip": 1.34667969, + "router_z_loss_mlp": 0.22473145, + "step": 1456, + "time_per_iteration": 2.8697588443756104 + }, + { + "auxiliary_loss_clip": 0.01197636, + "auxiliary_loss_mlp": 0.01052073, + "balance_loss_clip": 1.06615746, + "balance_loss_mlp": 1.02875543, + "epoch": 0.17519389166115554, + "flos": 24717602136960.0, + "grad_norm": 2.230233702687628, + "language_loss": 0.72703516, + "learning_rate": 3.7830374840308676e-06, + "loss": 0.74953222, + "num_input_tokens_seen": 30774605, + "router_z_loss_clip": 1.31347656, + "router_z_loss_mlp": 0.23327637, + "step": 1457, + "time_per_iteration": 2.8564341068267822 + }, + { + "auxiliary_loss_clip": 0.01197359, + "auxiliary_loss_mlp": 0.01053742, + "balance_loss_clip": 1.06349397, + "balance_loss_mlp": 1.03085375, + "epoch": 0.17531413455179462, + "flos": 23798144770560.0, + "grad_norm": 3.28429965046973, + "language_loss": 0.8276099, + "learning_rate": 3.7826844872519842e-06, + "loss": 0.85012096, + "num_input_tokens_seen": 30792460, + "router_z_loss_clip": 1.33886719, + "router_z_loss_mlp": 0.22875977, + "step": 1458, + "time_per_iteration": 2.8471715450286865 + }, + { + "auxiliary_loss_clip": 0.01196705, + "auxiliary_loss_mlp": 0.01054082, + "balance_loss_clip": 1.06769657, + "balance_loss_mlp": 1.03391123, + "epoch": 0.1754343774424337, + "flos": 24572379450240.0, + "grad_norm": 3.0339979959493886, + "language_loss": 0.7271052, + "learning_rate": 3.782331220041282e-06, + "loss": 0.74961305, + "num_input_tokens_seen": 30812525, + "router_z_loss_clip": 1.2890625, + "router_z_loss_mlp": 0.20178223, + "step": 1459, + "time_per_iteration": 2.8660643100738525 + }, + { + "auxiliary_loss_clip": 0.0119975, + "auxiliary_loss_mlp": 0.01048467, + "balance_loss_clip": 1.06382644, + "balance_loss_mlp": 1.02662802, + "epoch": 0.17555462033307281, + "flos": 18114599767680.0, + "grad_norm": 2.4407829108515644, + "language_loss": 0.8311007, + "learning_rate": 3.7819776824523504e-06, + "loss": 0.85358286, + "num_input_tokens_seen": 30830390, + "router_z_loss_clip": 1.35839844, + "router_z_loss_mlp": 0.21850586, + "step": 1460, + "time_per_iteration": 2.8220510482788086 + }, + { + "auxiliary_loss_clip": 0.01196085, + "auxiliary_loss_mlp": 0.01059271, + "balance_loss_clip": 1.06441629, + "balance_loss_mlp": 1.03591835, + "epoch": 0.1756748632237119, + "flos": 28366018364160.0, + "grad_norm": 1.9398036981404834, + "language_loss": 0.83904809, + "learning_rate": 3.7816238745388213e-06, + "loss": 0.86160171, + "num_input_tokens_seen": 30849935, + "router_z_loss_clip": 1.31640625, + "router_z_loss_mlp": 0.23339844, + "step": 1461, + "time_per_iteration": 2.817469596862793 + }, + { + "auxiliary_loss_clip": 0.01194252, + "auxiliary_loss_mlp": 0.01049863, + "balance_loss_clip": 1.06203294, + "balance_loss_mlp": 1.0283699, + "epoch": 0.17579510611435098, + "flos": 25732939881600.0, + "grad_norm": 1.9490106661702196, + "language_loss": 0.87037849, + "learning_rate": 3.781269796354367e-06, + "loss": 0.89281964, + "num_input_tokens_seen": 30869555, + "router_z_loss_clip": 1.3203125, + "router_z_loss_mlp": 0.21472168, + "step": 1462, + "time_per_iteration": 3.695384979248047 + }, + { + "auxiliary_loss_clip": 0.01197855, + "auxiliary_loss_mlp": 0.01060541, + "balance_loss_clip": 1.06744206, + "balance_loss_mlp": 1.03714037, + "epoch": 0.1759153490049901, + "flos": 18588081870720.0, + "grad_norm": 2.259543245844553, + "language_loss": 0.85592902, + "learning_rate": 3.7809154479527006e-06, + "loss": 0.87851304, + "num_input_tokens_seen": 30888760, + "router_z_loss_clip": 1.30371094, + "router_z_loss_mlp": 0.23400879, + "step": 1463, + "time_per_iteration": 3.631796360015869 + }, + { + "auxiliary_loss_clip": 0.01195263, + "auxiliary_loss_mlp": 0.01049462, + "balance_loss_clip": 1.06641173, + "balance_loss_mlp": 1.02841008, + "epoch": 0.17603559189562917, + "flos": 18619323724800.0, + "grad_norm": 4.5446727980684996, + "language_loss": 0.84660023, + "learning_rate": 3.780560829387577e-06, + "loss": 0.86904746, + "num_input_tokens_seen": 30907260, + "router_z_loss_clip": 1.28808594, + "router_z_loss_mlp": 0.21044922, + "step": 1464, + "time_per_iteration": 3.6536455154418945 + }, + { + "auxiliary_loss_clip": 0.01108549, + "auxiliary_loss_mlp": 0.01023322, + "balance_loss_clip": 1.0499506, + "balance_loss_mlp": 1.01922154, + "epoch": 0.17615583478626826, + "flos": 60530775373440.0, + "grad_norm": 0.8564436447077362, + "language_loss": 0.57931936, + "learning_rate": 3.7802059407127915e-06, + "loss": 0.60063815, + "num_input_tokens_seen": 30965810, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.04101562, + "step": 1465, + "time_per_iteration": 3.416027545928955 + }, + { + "auxiliary_loss_clip": 0.01195811, + "auxiliary_loss_mlp": 0.0105648, + "balance_loss_clip": 1.06463146, + "balance_loss_mlp": 1.03518915, + "epoch": 0.17627607767690734, + "flos": 23616221362560.0, + "grad_norm": 2.7320801475394254, + "language_loss": 0.86091352, + "learning_rate": 3.7798507819821797e-06, + "loss": 0.88343644, + "num_input_tokens_seen": 30982935, + "router_z_loss_clip": 1.31152344, + "router_z_loss_mlp": 0.2130127, + "step": 1466, + "time_per_iteration": 2.88222599029541 + }, + { + "auxiliary_loss_clip": 0.01199274, + "auxiliary_loss_mlp": 0.01060267, + "balance_loss_clip": 1.06231213, + "balance_loss_mlp": 1.0375222, + "epoch": 0.17639632056754645, + "flos": 17639070589440.0, + "grad_norm": 2.408678917571998, + "language_loss": 0.79025775, + "learning_rate": 3.7794953532496197e-06, + "loss": 0.81285316, + "num_input_tokens_seen": 30998840, + "router_z_loss_clip": 1.37109375, + "router_z_loss_mlp": 0.22753906, + "step": 1467, + "time_per_iteration": 2.82926869392395 + }, + { + "auxiliary_loss_clip": 0.01104538, + "auxiliary_loss_mlp": 0.01004117, + "balance_loss_clip": 1.04595089, + "balance_loss_mlp": 1.00034952, + "epoch": 0.17651656345818553, + "flos": 57932604910080.0, + "grad_norm": 0.8685246986426892, + "language_loss": 0.57940531, + "learning_rate": 3.7791396545690295e-06, + "loss": 0.60049188, + "num_input_tokens_seen": 31060075, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.03759766, + "step": 1468, + "time_per_iteration": 3.3161067962646484 + }, + { + "auxiliary_loss_clip": 0.01194433, + "auxiliary_loss_mlp": 0.01055598, + "balance_loss_clip": 1.06345737, + "balance_loss_mlp": 1.03321052, + "epoch": 0.17663680634882462, + "flos": 22929502170240.0, + "grad_norm": 2.587891176412108, + "language_loss": 0.81037837, + "learning_rate": 3.7787836859943685e-06, + "loss": 0.83287865, + "num_input_tokens_seen": 31078800, + "router_z_loss_clip": 1.30957031, + "router_z_loss_mlp": 0.22387695, + "step": 1469, + "time_per_iteration": 2.8775274753570557 + }, + { + "auxiliary_loss_clip": 0.01195385, + "auxiliary_loss_mlp": 0.0105326, + "balance_loss_clip": 1.0634923, + "balance_loss_mlp": 1.03087199, + "epoch": 0.17675704923946373, + "flos": 22637979388800.0, + "grad_norm": 2.314125421075128, + "language_loss": 0.78094012, + "learning_rate": 3.7784274475796363e-06, + "loss": 0.80342656, + "num_input_tokens_seen": 31097430, + "router_z_loss_clip": 1.31738281, + "router_z_loss_mlp": 0.22387695, + "step": 1470, + "time_per_iteration": 2.8585150241851807 + }, + { + "auxiliary_loss_clip": 0.01197766, + "auxiliary_loss_mlp": 0.01055391, + "balance_loss_clip": 1.06131196, + "balance_loss_mlp": 1.0326097, + "epoch": 0.1768772921301028, + "flos": 27126525795840.0, + "grad_norm": 2.5624234399762695, + "language_loss": 0.76329082, + "learning_rate": 3.7780709393788745e-06, + "loss": 0.78582239, + "num_input_tokens_seen": 31117905, + "router_z_loss_clip": 1.36523438, + "router_z_loss_mlp": 0.22790527, + "step": 1471, + "time_per_iteration": 2.913438558578491 + }, + { + "auxiliary_loss_clip": 0.01195567, + "auxiliary_loss_mlp": 0.01057563, + "balance_loss_clip": 1.06414616, + "balance_loss_mlp": 1.03522325, + "epoch": 0.1769975350207419, + "flos": 19172133014400.0, + "grad_norm": 2.331755564565886, + "language_loss": 0.75367045, + "learning_rate": 3.777714161446165e-06, + "loss": 0.77620173, + "num_input_tokens_seen": 31137610, + "router_z_loss_clip": 1.31445312, + "router_z_loss_mlp": 0.22338867, + "step": 1472, + "time_per_iteration": 2.8783960342407227 + }, + { + "auxiliary_loss_clip": 0.01194148, + "auxiliary_loss_mlp": 0.01048017, + "balance_loss_clip": 1.06448162, + "balance_loss_mlp": 1.02737045, + "epoch": 0.177117777911381, + "flos": 36134932291200.0, + "grad_norm": 2.5549948635936865, + "language_loss": 0.69584423, + "learning_rate": 3.7773571138356304e-06, + "loss": 0.71826589, + "num_input_tokens_seen": 31157780, + "router_z_loss_clip": 1.29785156, + "router_z_loss_mlp": 0.20666504, + "step": 1473, + "time_per_iteration": 2.9264447689056396 + }, + { + "auxiliary_loss_clip": 0.0118966, + "auxiliary_loss_mlp": 0.01053598, + "balance_loss_clip": 1.06045914, + "balance_loss_mlp": 1.03204501, + "epoch": 0.17723802080202009, + "flos": 22090593052800.0, + "grad_norm": 2.3220824569959593, + "language_loss": 0.89201075, + "learning_rate": 3.776999796601435e-06, + "loss": 0.91444331, + "num_input_tokens_seen": 31176540, + "router_z_loss_clip": 1.29199219, + "router_z_loss_mlp": 0.2154541, + "step": 1474, + "time_per_iteration": 2.8479883670806885 + }, + { + "auxiliary_loss_clip": 0.01197914, + "auxiliary_loss_mlp": 0.01045852, + "balance_loss_clip": 1.06454253, + "balance_loss_mlp": 1.02303505, + "epoch": 0.17735826369265917, + "flos": 30222671437440.0, + "grad_norm": 2.7052429016751396, + "language_loss": 0.72857285, + "learning_rate": 3.776642209797783e-06, + "loss": 0.75101054, + "num_input_tokens_seen": 31198370, + "router_z_loss_clip": 1.33203125, + "router_z_loss_mlp": 0.22827148, + "step": 1475, + "time_per_iteration": 2.871823787689209 + }, + { + "auxiliary_loss_clip": 0.01198302, + "auxiliary_loss_mlp": 0.01065064, + "balance_loss_clip": 1.06209135, + "balance_loss_mlp": 1.04059041, + "epoch": 0.17747850658329825, + "flos": 21397588980480.0, + "grad_norm": 2.4037401013124033, + "language_loss": 0.78176403, + "learning_rate": 3.7762843534789205e-06, + "loss": 0.8043977, + "num_input_tokens_seen": 31217120, + "router_z_loss_clip": 1.36132812, + "router_z_loss_mlp": 0.24487305, + "step": 1476, + "time_per_iteration": 2.86333966255188 + }, + { + "auxiliary_loss_clip": 0.01196622, + "auxiliary_loss_mlp": 0.01051257, + "balance_loss_clip": 1.06246102, + "balance_loss_mlp": 1.02727175, + "epoch": 0.17759874947393736, + "flos": 16983341856000.0, + "grad_norm": 2.422317014140575, + "language_loss": 0.88636696, + "learning_rate": 3.7759262276991343e-06, + "loss": 0.90884572, + "num_input_tokens_seen": 31234730, + "router_z_loss_clip": 1.34277344, + "router_z_loss_mlp": 0.23999023, + "step": 1477, + "time_per_iteration": 2.831625461578369 + }, + { + "auxiliary_loss_clip": 0.01198513, + "auxiliary_loss_mlp": 0.01060613, + "balance_loss_clip": 1.06277716, + "balance_loss_mlp": 1.0386188, + "epoch": 0.17771899236457644, + "flos": 11546107390080.0, + "grad_norm": 3.394055517197106, + "language_loss": 0.8073523, + "learning_rate": 3.7755678325127506e-06, + "loss": 0.8299436, + "num_input_tokens_seen": 31252410, + "router_z_loss_clip": 1.35742188, + "router_z_loss_mlp": 0.21984863, + "step": 1478, + "time_per_iteration": 2.893341302871704 + }, + { + "auxiliary_loss_clip": 0.01194317, + "auxiliary_loss_mlp": 0.01049502, + "balance_loss_clip": 1.06115484, + "balance_loss_mlp": 1.02635145, + "epoch": 0.17783923525521553, + "flos": 18807747494400.0, + "grad_norm": 2.7040503712165234, + "language_loss": 0.75646627, + "learning_rate": 3.7752091679741393e-06, + "loss": 0.77890444, + "num_input_tokens_seen": 31270200, + "router_z_loss_clip": 1.33203125, + "router_z_loss_mlp": 0.23168945, + "step": 1479, + "time_per_iteration": 2.785860776901245 + }, + { + "auxiliary_loss_clip": 0.0119503, + "auxiliary_loss_mlp": 0.01049351, + "balance_loss_clip": 1.06017721, + "balance_loss_mlp": 1.02578306, + "epoch": 0.17795947814585464, + "flos": 30408365773440.0, + "grad_norm": 3.164680591633105, + "language_loss": 0.77591538, + "learning_rate": 3.774850234137708e-06, + "loss": 0.79835916, + "num_input_tokens_seen": 31287495, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.23596191, + "step": 1480, + "time_per_iteration": 2.8370907306671143 + }, + { + "auxiliary_loss_clip": 0.01194591, + "auxiliary_loss_mlp": 0.01062131, + "balance_loss_clip": 1.06184816, + "balance_loss_mlp": 1.03750205, + "epoch": 0.17807972103649372, + "flos": 24389055411840.0, + "grad_norm": 2.8869847363610885, + "language_loss": 0.82726043, + "learning_rate": 3.7744910310579076e-06, + "loss": 0.84982765, + "num_input_tokens_seen": 31306420, + "router_z_loss_clip": 1.328125, + "router_z_loss_mlp": 0.24633789, + "step": 1481, + "time_per_iteration": 2.8325324058532715 + }, + { + "auxiliary_loss_clip": 0.01192046, + "auxiliary_loss_mlp": 0.01054096, + "balance_loss_clip": 1.06175256, + "balance_loss_mlp": 1.03178036, + "epoch": 0.1781999639271328, + "flos": 20301559332480.0, + "grad_norm": 4.688704323607149, + "language_loss": 0.85506469, + "learning_rate": 3.774131558789229e-06, + "loss": 0.87752616, + "num_input_tokens_seen": 31325750, + "router_z_loss_clip": 1.30273438, + "router_z_loss_mlp": 0.22314453, + "step": 1482, + "time_per_iteration": 2.773388385772705 + }, + { + "auxiliary_loss_clip": 0.01194012, + "auxiliary_loss_mlp": 0.01051138, + "balance_loss_clip": 1.06166816, + "balance_loss_mlp": 1.027022, + "epoch": 0.1783202068177719, + "flos": 15924479806080.0, + "grad_norm": 8.15292243315849, + "language_loss": 0.70467567, + "learning_rate": 3.773771817386203e-06, + "loss": 0.72712719, + "num_input_tokens_seen": 31343080, + "router_z_loss_clip": 1.32226562, + "router_z_loss_mlp": 0.24121094, + "step": 1483, + "time_per_iteration": 2.9058353900909424 + }, + { + "auxiliary_loss_clip": 0.01190604, + "auxiliary_loss_mlp": 0.0105295, + "balance_loss_clip": 1.06043434, + "balance_loss_mlp": 1.032529, + "epoch": 0.178440449708411, + "flos": 20631758083200.0, + "grad_norm": 2.442304794097857, + "language_loss": 0.79351467, + "learning_rate": 3.773411806903403e-06, + "loss": 0.81595021, + "num_input_tokens_seen": 31362160, + "router_z_loss_clip": 1.30175781, + "router_z_loss_mlp": 0.20422363, + "step": 1484, + "time_per_iteration": 2.8205912113189697 + }, + { + "auxiliary_loss_clip": 0.01198888, + "auxiliary_loss_mlp": 0.01061578, + "balance_loss_clip": 1.06427646, + "balance_loss_mlp": 1.0375216, + "epoch": 0.17856069259905008, + "flos": 21686059105920.0, + "grad_norm": 2.1067595581681586, + "language_loss": 0.94871527, + "learning_rate": 3.7730515273954415e-06, + "loss": 0.97131985, + "num_input_tokens_seen": 31380770, + "router_z_loss_clip": 1.34570312, + "router_z_loss_mlp": 0.24047852, + "step": 1485, + "time_per_iteration": 2.803978443145752 + }, + { + "auxiliary_loss_clip": 0.01191024, + "auxiliary_loss_mlp": 0.01055255, + "balance_loss_clip": 1.06205332, + "balance_loss_mlp": 1.03335595, + "epoch": 0.17868093548968916, + "flos": 26572962320640.0, + "grad_norm": 1.9351228163153218, + "language_loss": 0.84901029, + "learning_rate": 3.772690978916973e-06, + "loss": 0.87147307, + "num_input_tokens_seen": 31400525, + "router_z_loss_clip": 1.29003906, + "router_z_loss_mlp": 0.21899414, + "step": 1486, + "time_per_iteration": 2.8869872093200684 + }, + { + "auxiliary_loss_clip": 0.01191846, + "auxiliary_loss_mlp": 0.01052429, + "balance_loss_clip": 1.05965149, + "balance_loss_mlp": 1.0298748, + "epoch": 0.17880117838032827, + "flos": 18581006891520.0, + "grad_norm": 2.738014214161832, + "language_loss": 0.8669675, + "learning_rate": 3.772330161522693e-06, + "loss": 0.88941026, + "num_input_tokens_seen": 31418435, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.22558594, + "step": 1487, + "time_per_iteration": 2.7760274410247803 + }, + { + "auxiliary_loss_clip": 0.01194406, + "auxiliary_loss_mlp": 0.01054228, + "balance_loss_clip": 1.06421781, + "balance_loss_mlp": 1.03187656, + "epoch": 0.17892142127096736, + "flos": 26541217676160.0, + "grad_norm": 2.8615450755984737, + "language_loss": 0.79653597, + "learning_rate": 3.7719690752673365e-06, + "loss": 0.8190223, + "num_input_tokens_seen": 31439230, + "router_z_loss_clip": 1.30273438, + "router_z_loss_mlp": 0.22351074, + "step": 1488, + "time_per_iteration": 2.836101531982422 + }, + { + "auxiliary_loss_clip": 0.01196042, + "auxiliary_loss_mlp": 0.0106146, + "balance_loss_clip": 1.06513548, + "balance_loss_mlp": 1.0378685, + "epoch": 0.17904166416160644, + "flos": 23872623621120.0, + "grad_norm": 2.1322941750395197, + "language_loss": 0.7862221, + "learning_rate": 3.7716077202056796e-06, + "loss": 0.80879712, + "num_input_tokens_seen": 31457705, + "router_z_loss_clip": 1.30859375, + "router_z_loss_mlp": 0.23596191, + "step": 1489, + "time_per_iteration": 2.848078966140747 + }, + { + "auxiliary_loss_clip": 0.01189757, + "auxiliary_loss_mlp": 0.01051437, + "balance_loss_clip": 1.0595212, + "balance_loss_mlp": 1.02977633, + "epoch": 0.17916190705224552, + "flos": 19134426712320.0, + "grad_norm": 2.39282397698144, + "language_loss": 0.93580079, + "learning_rate": 3.7712460963925404e-06, + "loss": 0.95821273, + "num_input_tokens_seen": 31473645, + "router_z_loss_clip": 1.30078125, + "router_z_loss_mlp": 0.2166748, + "step": 1490, + "time_per_iteration": 3.708394765853882 + }, + { + "auxiliary_loss_clip": 0.01193543, + "auxiliary_loss_mlp": 0.01056056, + "balance_loss_clip": 1.06395721, + "balance_loss_mlp": 1.03401375, + "epoch": 0.17928214994288463, + "flos": 25152120961920.0, + "grad_norm": 2.0427887176232677, + "language_loss": 0.75355864, + "learning_rate": 3.7708842038827775e-06, + "loss": 0.77605462, + "num_input_tokens_seen": 31492605, + "router_z_loss_clip": 1.296875, + "router_z_loss_mlp": 0.22033691, + "step": 1491, + "time_per_iteration": 3.6607954502105713 + }, + { + "auxiliary_loss_clip": 0.01193662, + "auxiliary_loss_mlp": 0.01056891, + "balance_loss_clip": 1.06012797, + "balance_loss_mlp": 1.03372848, + "epoch": 0.17940239283352372, + "flos": 22384629786240.0, + "grad_norm": 1.8627417065521552, + "language_loss": 0.85564399, + "learning_rate": 3.770522042731288e-06, + "loss": 0.87814951, + "num_input_tokens_seen": 31514500, + "router_z_loss_clip": 1.3359375, + "router_z_loss_mlp": 0.23181152, + "step": 1492, + "time_per_iteration": 3.656571388244629 + }, + { + "auxiliary_loss_clip": 0.01192536, + "auxiliary_loss_mlp": 0.01054595, + "balance_loss_clip": 1.06153131, + "balance_loss_mlp": 1.03099132, + "epoch": 0.1795226357241628, + "flos": 23178685795200.0, + "grad_norm": 2.6011233801689015, + "language_loss": 0.87929738, + "learning_rate": 3.7701596129930122e-06, + "loss": 0.90176868, + "num_input_tokens_seen": 31533225, + "router_z_loss_clip": 1.31054688, + "router_z_loss_mlp": 0.23620605, + "step": 1493, + "time_per_iteration": 3.665360450744629 + }, + { + "auxiliary_loss_clip": 0.01195193, + "auxiliary_loss_mlp": 0.01047613, + "balance_loss_clip": 1.06394553, + "balance_loss_mlp": 1.02344966, + "epoch": 0.1796428786148019, + "flos": 22090413484800.0, + "grad_norm": 2.635403838373051, + "language_loss": 0.7361964, + "learning_rate": 3.7697969147229315e-06, + "loss": 0.75862449, + "num_input_tokens_seen": 31551385, + "router_z_loss_clip": 1.31054688, + "router_z_loss_mlp": 0.24157715, + "step": 1494, + "time_per_iteration": 3.005850076675415 + }, + { + "auxiliary_loss_clip": 0.01193217, + "auxiliary_loss_mlp": 0.0105026, + "balance_loss_clip": 1.06308627, + "balance_loss_mlp": 1.02812314, + "epoch": 0.179763121505441, + "flos": 21324618501120.0, + "grad_norm": 2.359170150409962, + "language_loss": 0.84902489, + "learning_rate": 3.7694339479760647e-06, + "loss": 0.87145966, + "num_input_tokens_seen": 31570415, + "router_z_loss_clip": 1.30078125, + "router_z_loss_mlp": 0.22143555, + "step": 1495, + "time_per_iteration": 2.8264968395233154 + }, + { + "auxiliary_loss_clip": 0.01096189, + "auxiliary_loss_mlp": 0.01005554, + "balance_loss_clip": 1.0394454, + "balance_loss_mlp": 1.00219274, + "epoch": 0.17988336439608008, + "flos": 68161864815360.0, + "grad_norm": 0.794575302159059, + "language_loss": 0.57266605, + "learning_rate": 3.769070712807476e-06, + "loss": 0.59368348, + "num_input_tokens_seen": 31632445, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.03369141, + "step": 1496, + "time_per_iteration": 3.454038619995117 + }, + { + "auxiliary_loss_clip": 0.01191285, + "auxiliary_loss_mlp": 0.01058582, + "balance_loss_clip": 1.06031156, + "balance_loss_mlp": 1.03348839, + "epoch": 0.18000360728671919, + "flos": 21945047143680.0, + "grad_norm": 2.2676293137212777, + "language_loss": 0.78912926, + "learning_rate": 3.768707209272266e-06, + "loss": 0.81162792, + "num_input_tokens_seen": 31652575, + "router_z_loss_clip": 1.30859375, + "router_z_loss_mlp": 0.25109863, + "step": 1497, + "time_per_iteration": 2.821469783782959 + }, + { + "auxiliary_loss_clip": 0.01189131, + "auxiliary_loss_mlp": 0.01059539, + "balance_loss_clip": 1.05809784, + "balance_loss_mlp": 1.03692508, + "epoch": 0.18012385017735827, + "flos": 18986330937600.0, + "grad_norm": 2.8895766777656586, + "language_loss": 0.76629317, + "learning_rate": 3.768343437425579e-06, + "loss": 0.78877985, + "num_input_tokens_seen": 31671145, + "router_z_loss_clip": 1.31152344, + "router_z_loss_mlp": 0.22631836, + "step": 1498, + "time_per_iteration": 2.852229356765747 + }, + { + "auxiliary_loss_clip": 0.01191651, + "auxiliary_loss_mlp": 0.01061892, + "balance_loss_clip": 1.05738962, + "balance_loss_mlp": 1.03914642, + "epoch": 0.18024409306799735, + "flos": 19748103598080.0, + "grad_norm": 3.1705889175407544, + "language_loss": 0.86561531, + "learning_rate": 3.7679793973225987e-06, + "loss": 0.88815075, + "num_input_tokens_seen": 31686955, + "router_z_loss_clip": 1.34179688, + "router_z_loss_mlp": 0.22741699, + "step": 1499, + "time_per_iteration": 2.8029251098632812 + }, + { + "auxiliary_loss_clip": 0.01085579, + "auxiliary_loss_mlp": 0.01003816, + "balance_loss_clip": 1.02933156, + "balance_loss_mlp": 1.00062084, + "epoch": 0.18036433595863643, + "flos": 67227183060480.0, + "grad_norm": 0.845698964328265, + "language_loss": 0.6159091, + "learning_rate": 3.767615089018549e-06, + "loss": 0.63680303, + "num_input_tokens_seen": 31749300, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.03198242, + "step": 1500, + "time_per_iteration": 3.271089553833008 + }, + { + "auxiliary_loss_clip": 0.01193245, + "auxiliary_loss_mlp": 0.01059606, + "balance_loss_clip": 1.05977511, + "balance_loss_mlp": 1.03497732, + "epoch": 0.18048457884927555, + "flos": 18181464935040.0, + "grad_norm": 2.036220769452896, + "language_loss": 0.85978401, + "learning_rate": 3.7672505125686966e-06, + "loss": 0.88231254, + "num_input_tokens_seen": 31765665, + "router_z_loss_clip": 1.33496094, + "router_z_loss_mlp": 0.24621582, + "step": 1501, + "time_per_iteration": 2.841947078704834 + }, + { + "auxiliary_loss_clip": 0.01195645, + "auxiliary_loss_mlp": 0.01064049, + "balance_loss_clip": 1.06128442, + "balance_loss_mlp": 1.03965831, + "epoch": 0.18060482173991463, + "flos": 15813767111040.0, + "grad_norm": 3.6909983048929735, + "language_loss": 0.84442317, + "learning_rate": 3.7668856680283455e-06, + "loss": 0.86702013, + "num_input_tokens_seen": 31782690, + "router_z_loss_clip": 1.34375, + "router_z_loss_mlp": 0.24377441, + "step": 1502, + "time_per_iteration": 2.7818808555603027 + }, + { + "auxiliary_loss_clip": 0.01192205, + "auxiliary_loss_mlp": 0.01063289, + "balance_loss_clip": 1.06035936, + "balance_loss_mlp": 1.04125905, + "epoch": 0.1807250646305537, + "flos": 18587399512320.0, + "grad_norm": 2.32120205038914, + "language_loss": 0.82115138, + "learning_rate": 3.7665205554528437e-06, + "loss": 0.84370637, + "num_input_tokens_seen": 31802045, + "router_z_loss_clip": 1.31738281, + "router_z_loss_mlp": 0.22009277, + "step": 1503, + "time_per_iteration": 2.7750613689422607 + }, + { + "auxiliary_loss_clip": 0.01187467, + "auxiliary_loss_mlp": 0.01057928, + "balance_loss_clip": 1.05716002, + "balance_loss_mlp": 1.03711402, + "epoch": 0.18084530752119282, + "flos": 23149131880320.0, + "grad_norm": 1.9512449843816293, + "language_loss": 0.7394374, + "learning_rate": 3.7661551748975782e-06, + "loss": 0.76189137, + "num_input_tokens_seen": 31820220, + "router_z_loss_clip": 1.30273438, + "router_z_loss_mlp": 0.20800781, + "step": 1504, + "time_per_iteration": 2.7810306549072266 + }, + { + "auxiliary_loss_clip": 0.01080712, + "auxiliary_loss_mlp": 0.01027783, + "balance_loss_clip": 1.02564192, + "balance_loss_mlp": 1.02456462, + "epoch": 0.1809655504118319, + "flos": 59803153568640.0, + "grad_norm": 0.8169066278048652, + "language_loss": 0.60419619, + "learning_rate": 3.7657895264179772e-06, + "loss": 0.62528116, + "num_input_tokens_seen": 31876195, + "router_z_loss_clip": 0.55078125, + "router_z_loss_mlp": 0.03222656, + "step": 1505, + "time_per_iteration": 3.3108301162719727 + }, + { + "auxiliary_loss_clip": 0.01189178, + "auxiliary_loss_mlp": 0.01058195, + "balance_loss_clip": 1.05886829, + "balance_loss_mlp": 1.03539038, + "epoch": 0.181085793302471, + "flos": 44201941188480.0, + "grad_norm": 1.968424041460084, + "language_loss": 0.7435118, + "learning_rate": 3.765423610069509e-06, + "loss": 0.76598543, + "num_input_tokens_seen": 31901585, + "router_z_loss_clip": 1.30273438, + "router_z_loss_mlp": 0.22814941, + "step": 1506, + "time_per_iteration": 3.05012845993042 + }, + { + "auxiliary_loss_clip": 0.01187474, + "auxiliary_loss_mlp": 0.01061528, + "balance_loss_clip": 1.05911767, + "balance_loss_mlp": 1.03979611, + "epoch": 0.18120603619311007, + "flos": 34898384638080.0, + "grad_norm": 1.8952210187395753, + "language_loss": 0.72312093, + "learning_rate": 3.765057425907683e-06, + "loss": 0.74561095, + "num_input_tokens_seen": 31923045, + "router_z_loss_clip": 1.28320312, + "router_z_loss_mlp": 0.21728516, + "step": 1507, + "time_per_iteration": 2.868345022201538 + }, + { + "auxiliary_loss_clip": 0.01184044, + "auxiliary_loss_mlp": 0.01052987, + "balance_loss_clip": 1.05697417, + "balance_loss_mlp": 1.03032565, + "epoch": 0.18132627908374918, + "flos": 21506757390720.0, + "grad_norm": 1.9309902902728415, + "language_loss": 0.78397667, + "learning_rate": 3.764690973988048e-06, + "loss": 0.80634695, + "num_input_tokens_seen": 31943385, + "router_z_loss_clip": 1.26953125, + "router_z_loss_mlp": 0.22668457, + "step": 1508, + "time_per_iteration": 2.8185906410217285 + }, + { + "auxiliary_loss_clip": 0.01185014, + "auxiliary_loss_mlp": 0.01056764, + "balance_loss_clip": 1.05482602, + "balance_loss_mlp": 1.03438807, + "epoch": 0.18144652197438826, + "flos": 29057693633280.0, + "grad_norm": 1.8365367878577354, + "language_loss": 0.74203849, + "learning_rate": 3.7643242543661967e-06, + "loss": 0.76445627, + "num_input_tokens_seen": 31966045, + "router_z_loss_clip": 1.30078125, + "router_z_loss_mlp": 0.22375488, + "step": 1509, + "time_per_iteration": 2.8763973712921143 + }, + { + "auxiliary_loss_clip": 0.01078798, + "auxiliary_loss_mlp": 0.01010745, + "balance_loss_clip": 1.02446175, + "balance_loss_mlp": 1.00743055, + "epoch": 0.18156676486502735, + "flos": 68675064382080.0, + "grad_norm": 0.8201523418659395, + "language_loss": 0.6046555, + "learning_rate": 3.7639572670977573e-06, + "loss": 0.62555087, + "num_input_tokens_seen": 32021540, + "router_z_loss_clip": 0.54296875, + "router_z_loss_mlp": 0.03320312, + "step": 1510, + "time_per_iteration": 3.3523106575012207 + }, + { + "auxiliary_loss_clip": 0.01187362, + "auxiliary_loss_mlp": 0.01056824, + "balance_loss_clip": 1.05852652, + "balance_loss_mlp": 1.03404284, + "epoch": 0.18168700775566646, + "flos": 26471515334400.0, + "grad_norm": 1.839057297471527, + "language_loss": 0.76706004, + "learning_rate": 3.7635900122384042e-06, + "loss": 0.78950191, + "num_input_tokens_seen": 32044535, + "router_z_loss_clip": 1.2890625, + "router_z_loss_mlp": 0.2277832, + "step": 1511, + "time_per_iteration": 2.894991159439087 + }, + { + "auxiliary_loss_clip": 0.01189544, + "auxiliary_loss_mlp": 0.01051055, + "balance_loss_clip": 1.05541921, + "balance_loss_mlp": 1.02776146, + "epoch": 0.18180725064630554, + "flos": 15005668884480.0, + "grad_norm": 2.1191574260021526, + "language_loss": 0.86971223, + "learning_rate": 3.7632224898438477e-06, + "loss": 0.89211822, + "num_input_tokens_seen": 32061010, + "router_z_loss_clip": 1.34179688, + "router_z_loss_mlp": 0.23303223, + "step": 1512, + "time_per_iteration": 2.8773279190063477 + }, + { + "auxiliary_loss_clip": 0.01185179, + "auxiliary_loss_mlp": 0.01054569, + "balance_loss_clip": 1.05812156, + "balance_loss_mlp": 1.03282547, + "epoch": 0.18192749353694462, + "flos": 19682387665920.0, + "grad_norm": 1.6677137603957646, + "language_loss": 0.7876519, + "learning_rate": 3.762854699969842e-06, + "loss": 0.8100493, + "num_input_tokens_seen": 32081520, + "router_z_loss_clip": 1.27050781, + "router_z_loss_mlp": 0.21740723, + "step": 1513, + "time_per_iteration": 3.00171160697937 + }, + { + "auxiliary_loss_clip": 0.01184255, + "auxiliary_loss_mlp": 0.01048584, + "balance_loss_clip": 1.05762482, + "balance_loss_mlp": 1.02600598, + "epoch": 0.1820477364275837, + "flos": 20702717400960.0, + "grad_norm": 6.964100577824994, + "language_loss": 0.73440921, + "learning_rate": 3.762486642672179e-06, + "loss": 0.75673759, + "num_input_tokens_seen": 32098460, + "router_z_loss_clip": 1.26660156, + "router_z_loss_mlp": 0.22583008, + "step": 1514, + "time_per_iteration": 3.003552198410034 + }, + { + "auxiliary_loss_clip": 0.01190704, + "auxiliary_loss_mlp": 0.01048617, + "balance_loss_clip": 1.05988669, + "balance_loss_mlp": 1.02463198, + "epoch": 0.18216797931822282, + "flos": 17128708197120.0, + "grad_norm": 207.61274621206337, + "language_loss": 0.86832511, + "learning_rate": 3.7621183180066946e-06, + "loss": 0.89071834, + "num_input_tokens_seen": 32116420, + "router_z_loss_clip": 1.30664062, + "router_z_loss_mlp": 0.23999023, + "step": 1515, + "time_per_iteration": 2.8394298553466797 + }, + { + "auxiliary_loss_clip": 0.01183834, + "auxiliary_loss_mlp": 0.01054174, + "balance_loss_clip": 1.0551182, + "balance_loss_mlp": 1.03141689, + "epoch": 0.1822882222088619, + "flos": 29242561956480.0, + "grad_norm": 1.6344999619833438, + "language_loss": 0.73679578, + "learning_rate": 3.7617497260292625e-06, + "loss": 0.7591759, + "num_input_tokens_seen": 32138475, + "router_z_loss_clip": 1.28710938, + "router_z_loss_mlp": 0.22753906, + "step": 1516, + "time_per_iteration": 2.89084792137146 + }, + { + "auxiliary_loss_clip": 0.01186056, + "auxiliary_loss_mlp": 0.01062169, + "balance_loss_clip": 1.05561483, + "balance_loss_mlp": 1.0380528, + "epoch": 0.18240846509950098, + "flos": 17702739446400.0, + "grad_norm": 2.9255048352034154, + "language_loss": 0.78686118, + "learning_rate": 3.7613808667957967e-06, + "loss": 0.80934346, + "num_input_tokens_seen": 32151165, + "router_z_loss_clip": 1.30566406, + "router_z_loss_mlp": 0.24108887, + "step": 1517, + "time_per_iteration": 2.8954732418060303 + }, + { + "auxiliary_loss_clip": 0.01184664, + "auxiliary_loss_mlp": 0.01054874, + "balance_loss_clip": 1.05665159, + "balance_loss_mlp": 1.03178334, + "epoch": 0.1825287079901401, + "flos": 14790025584000.0, + "grad_norm": 1.9612131456204254, + "language_loss": 0.91516495, + "learning_rate": 3.7610117403622547e-06, + "loss": 0.93756032, + "num_input_tokens_seen": 32167725, + "router_z_loss_clip": 1.28125, + "router_z_loss_mlp": 0.23095703, + "step": 1518, + "time_per_iteration": 2.911646604537964 + }, + { + "auxiliary_loss_clip": 0.01186666, + "auxiliary_loss_mlp": 0.01051868, + "balance_loss_clip": 1.05770266, + "balance_loss_mlp": 1.02954042, + "epoch": 0.18264895088077918, + "flos": 21946232292480.0, + "grad_norm": 2.3186094213102475, + "language_loss": 0.90448952, + "learning_rate": 3.7606423467846313e-06, + "loss": 0.92687488, + "num_input_tokens_seen": 32187330, + "router_z_loss_clip": 1.2890625, + "router_z_loss_mlp": 0.22314453, + "step": 1519, + "time_per_iteration": 3.630556344985962 + }, + { + "auxiliary_loss_clip": 0.0119457, + "auxiliary_loss_mlp": 0.0105799, + "balance_loss_clip": 1.06150973, + "balance_loss_mlp": 1.03610301, + "epoch": 0.18276919377141826, + "flos": 20886759711360.0, + "grad_norm": 1.6231472850717514, + "language_loss": 0.79757905, + "learning_rate": 3.760272686118964e-06, + "loss": 0.82010466, + "num_input_tokens_seen": 32205550, + "router_z_loss_clip": 1.33105469, + "router_z_loss_mlp": 0.21875, + "step": 1520, + "time_per_iteration": 4.405120134353638 + }, + { + "auxiliary_loss_clip": 0.01184237, + "auxiliary_loss_mlp": 0.01065434, + "balance_loss_clip": 1.05507541, + "balance_loss_mlp": 1.04105544, + "epoch": 0.18288943666205737, + "flos": 21469877101440.0, + "grad_norm": 3.50069542903841, + "language_loss": 0.9267236, + "learning_rate": 3.7599027584213297e-06, + "loss": 0.9492203, + "num_input_tokens_seen": 32224430, + "router_z_loss_clip": 1.29199219, + "router_z_loss_mlp": 0.24389648, + "step": 1521, + "time_per_iteration": 2.755361795425415 + }, + { + "auxiliary_loss_clip": 0.01189633, + "auxiliary_loss_mlp": 0.0106563, + "balance_loss_clip": 1.05865049, + "balance_loss_mlp": 1.04233682, + "epoch": 0.18300967955269645, + "flos": 21539363961600.0, + "grad_norm": 2.0434109107787375, + "language_loss": 0.78067744, + "learning_rate": 3.7595325637478465e-06, + "loss": 0.80323005, + "num_input_tokens_seen": 32242455, + "router_z_loss_clip": 1.31054688, + "router_z_loss_mlp": 0.23278809, + "step": 1522, + "time_per_iteration": 3.649711847305298 + }, + { + "auxiliary_loss_clip": 0.01187617, + "auxiliary_loss_mlp": 0.01053225, + "balance_loss_clip": 1.06076646, + "balance_loss_mlp": 1.03149271, + "epoch": 0.18312992244333554, + "flos": 28876237102080.0, + "grad_norm": 2.024104400640799, + "language_loss": 0.81970304, + "learning_rate": 3.7591621021546723e-06, + "loss": 0.84211147, + "num_input_tokens_seen": 32264450, + "router_z_loss_clip": 1.26757812, + "router_z_loss_mlp": 0.21728516, + "step": 1523, + "time_per_iteration": 2.838627576828003 + }, + { + "auxiliary_loss_clip": 0.01189487, + "auxiliary_loss_mlp": 0.01073529, + "balance_loss_clip": 1.05748105, + "balance_loss_mlp": 1.04880548, + "epoch": 0.18325016533397462, + "flos": 20120102801280.0, + "grad_norm": 1.7138019641974984, + "language_loss": 0.812958, + "learning_rate": 3.7587913736980062e-06, + "loss": 0.83558816, + "num_input_tokens_seen": 32284090, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.24755859, + "step": 1524, + "time_per_iteration": 2.846134901046753 + }, + { + "auxiliary_loss_clip": 0.01185738, + "auxiliary_loss_mlp": 0.01056649, + "balance_loss_clip": 1.0570482, + "balance_loss_mlp": 1.0352633, + "epoch": 0.18337040822461373, + "flos": 23329187781120.0, + "grad_norm": 1.7434500873794918, + "language_loss": 0.84549153, + "learning_rate": 3.7584203784340865e-06, + "loss": 0.86791539, + "num_input_tokens_seen": 32303260, + "router_z_loss_clip": 1.28515625, + "router_z_loss_mlp": 0.21386719, + "step": 1525, + "time_per_iteration": 2.931500196456909 + }, + { + "auxiliary_loss_clip": 0.01188278, + "auxiliary_loss_mlp": 0.0105943, + "balance_loss_clip": 1.05771255, + "balance_loss_mlp": 1.03757906, + "epoch": 0.1834906511152528, + "flos": 25009555881600.0, + "grad_norm": 2.01939156029259, + "language_loss": 0.85902989, + "learning_rate": 3.7580491164191938e-06, + "loss": 0.88150704, + "num_input_tokens_seen": 32321570, + "router_z_loss_clip": 1.30566406, + "router_z_loss_mlp": 0.21838379, + "step": 1526, + "time_per_iteration": 3.115541934967041 + }, + { + "auxiliary_loss_clip": 0.01090277, + "auxiliary_loss_mlp": 0.01004578, + "balance_loss_clip": 1.03386223, + "balance_loss_mlp": 1.00145423, + "epoch": 0.1836108940058919, + "flos": 67251493589760.0, + "grad_norm": 0.755913865123029, + "language_loss": 0.61253953, + "learning_rate": 3.757677587709648e-06, + "loss": 0.63348806, + "num_input_tokens_seen": 32384835, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.03125, + "step": 1527, + "time_per_iteration": 3.483109951019287 + }, + { + "auxiliary_loss_clip": 0.01187617, + "auxiliary_loss_mlp": 0.0105569, + "balance_loss_clip": 1.05751479, + "balance_loss_mlp": 1.03240895, + "epoch": 0.183731136896531, + "flos": 25738721971200.0, + "grad_norm": 2.1921862635371667, + "language_loss": 0.75718898, + "learning_rate": 3.7573057923618095e-06, + "loss": 0.77962208, + "num_input_tokens_seen": 32404930, + "router_z_loss_clip": 1.30175781, + "router_z_loss_mlp": 0.23303223, + "step": 1528, + "time_per_iteration": 2.9139537811279297 + }, + { + "auxiliary_loss_clip": 0.01187279, + "auxiliary_loss_mlp": 0.01052041, + "balance_loss_clip": 1.05461812, + "balance_loss_mlp": 1.02912879, + "epoch": 0.1838513797871701, + "flos": 20449403712000.0, + "grad_norm": 2.166123290613184, + "language_loss": 0.74479473, + "learning_rate": 3.7569337304320793e-06, + "loss": 0.76718789, + "num_input_tokens_seen": 32424515, + "router_z_loss_clip": 1.32714844, + "router_z_loss_mlp": 0.22912598, + "step": 1529, + "time_per_iteration": 2.827972412109375 + }, + { + "auxiliary_loss_clip": 0.0109298, + "auxiliary_loss_mlp": 0.01001451, + "balance_loss_clip": 1.03610277, + "balance_loss_mlp": 0.99801791, + "epoch": 0.18397162267780917, + "flos": 68565141786240.0, + "grad_norm": 0.8405116888086148, + "language_loss": 0.64459932, + "learning_rate": 3.756561401976899e-06, + "loss": 0.66554368, + "num_input_tokens_seen": 32484220, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.03442383, + "step": 1530, + "time_per_iteration": 3.2178170680999756 + }, + { + "auxiliary_loss_clip": 0.01188096, + "auxiliary_loss_mlp": 0.01052121, + "balance_loss_clip": 1.05911219, + "balance_loss_mlp": 1.02954292, + "epoch": 0.18409186556844825, + "flos": 31941104976000.0, + "grad_norm": 2.427964779208639, + "language_loss": 0.8226006, + "learning_rate": 3.7561888070527514e-06, + "loss": 0.84500277, + "num_input_tokens_seen": 32506260, + "router_z_loss_clip": 1.29101562, + "router_z_loss_mlp": 0.22570801, + "step": 1531, + "time_per_iteration": 2.884925127029419 + }, + { + "auxiliary_loss_clip": 0.01182172, + "auxiliary_loss_mlp": 0.0104651, + "balance_loss_clip": 1.0546416, + "balance_loss_mlp": 1.02636325, + "epoch": 0.18421210845908736, + "flos": 20120533764480.0, + "grad_norm": 2.4371088453314305, + "language_loss": 0.80404401, + "learning_rate": 3.7558159457161577e-06, + "loss": 0.82633084, + "num_input_tokens_seen": 32524225, + "router_z_loss_clip": 1.2734375, + "router_z_loss_mlp": 0.20153809, + "step": 1532, + "time_per_iteration": 2.824777603149414 + }, + { + "auxiliary_loss_clip": 0.01186543, + "auxiliary_loss_mlp": 0.01053748, + "balance_loss_clip": 1.05903852, + "balance_loss_mlp": 1.03159845, + "epoch": 0.18433235134972645, + "flos": 23110491824640.0, + "grad_norm": 2.0823446075323346, + "language_loss": 0.77751267, + "learning_rate": 3.755442818023681e-06, + "loss": 0.79991555, + "num_input_tokens_seen": 32543850, + "router_z_loss_clip": 1.27539062, + "router_z_loss_mlp": 0.22131348, + "step": 1533, + "time_per_iteration": 2.87713360786438 + }, + { + "auxiliary_loss_clip": 0.01184181, + "auxiliary_loss_mlp": 0.0105728, + "balance_loss_clip": 1.05592823, + "balance_loss_mlp": 1.03603649, + "epoch": 0.18445259424036553, + "flos": 18291351617280.0, + "grad_norm": 2.3806175949476187, + "language_loss": 0.76296622, + "learning_rate": 3.7550694240319246e-06, + "loss": 0.78538084, + "num_input_tokens_seen": 32561725, + "router_z_loss_clip": 1.28222656, + "router_z_loss_mlp": 0.21264648, + "step": 1534, + "time_per_iteration": 2.856563091278076 + }, + { + "auxiliary_loss_clip": 0.01183346, + "auxiliary_loss_mlp": 0.01062462, + "balance_loss_clip": 1.05587196, + "balance_loss_mlp": 1.03939486, + "epoch": 0.18457283713100464, + "flos": 21324079797120.0, + "grad_norm": 2.912488885923634, + "language_loss": 0.76417637, + "learning_rate": 3.7546957637975326e-06, + "loss": 0.78663445, + "num_input_tokens_seen": 32579135, + "router_z_loss_clip": 1.27441406, + "router_z_loss_mlp": 0.23095703, + "step": 1535, + "time_per_iteration": 2.8858213424682617 + }, + { + "auxiliary_loss_clip": 0.01179819, + "auxiliary_loss_mlp": 0.01053573, + "balance_loss_clip": 1.05507326, + "balance_loss_mlp": 1.03168583, + "epoch": 0.18469308002164372, + "flos": 20375679047040.0, + "grad_norm": 1.5177842058606996, + "language_loss": 0.73920476, + "learning_rate": 3.7543218373771873e-06, + "loss": 0.76153868, + "num_input_tokens_seen": 32598460, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.21862793, + "step": 1536, + "time_per_iteration": 2.9734973907470703 + }, + { + "auxiliary_loss_clip": 0.01184127, + "auxiliary_loss_mlp": 0.01062397, + "balance_loss_clip": 1.05768621, + "balance_loss_mlp": 1.04188108, + "epoch": 0.1848133229122828, + "flos": 26435892021120.0, + "grad_norm": 1.4129102986504183, + "language_loss": 0.7783916, + "learning_rate": 3.753947644827615e-06, + "loss": 0.80085689, + "num_input_tokens_seen": 32621920, + "router_z_loss_clip": 1.26171875, + "router_z_loss_mlp": 0.2052002, + "step": 1537, + "time_per_iteration": 2.984855890274048 + }, + { + "auxiliary_loss_clip": 0.01102074, + "auxiliary_loss_mlp": 0.01075722, + "balance_loss_clip": 1.04112852, + "balance_loss_mlp": 1.07171655, + "epoch": 0.1849335658029219, + "flos": 70547447612160.0, + "grad_norm": 0.9598889609251111, + "language_loss": 0.57125592, + "learning_rate": 3.753573186205579e-06, + "loss": 0.59303391, + "num_input_tokens_seen": 32690040, + "router_z_loss_clip": 0.609375, + "router_z_loss_mlp": 0.04003906, + "step": 1538, + "time_per_iteration": 3.448486566543579 + }, + { + "auxiliary_loss_clip": 0.01179961, + "auxiliary_loss_mlp": 0.0106118, + "balance_loss_clip": 1.05483317, + "balance_loss_mlp": 1.03961504, + "epoch": 0.185053808693561, + "flos": 17384140788480.0, + "grad_norm": 2.0487591228786175, + "language_loss": 0.77736294, + "learning_rate": 3.753198461567885e-06, + "loss": 0.79977441, + "num_input_tokens_seen": 32707285, + "router_z_loss_clip": 1.25097656, + "router_z_loss_mlp": 0.21569824, + "step": 1539, + "time_per_iteration": 2.7849767208099365 + }, + { + "auxiliary_loss_clip": 0.01180016, + "auxiliary_loss_mlp": 0.01065089, + "balance_loss_clip": 1.05519772, + "balance_loss_mlp": 1.04404891, + "epoch": 0.18517405158420008, + "flos": 28986159697920.0, + "grad_norm": 1.8603217501286626, + "language_loss": 0.92208838, + "learning_rate": 3.7528234709713783e-06, + "loss": 0.94453943, + "num_input_tokens_seen": 32730030, + "router_z_loss_clip": 1.24804688, + "router_z_loss_mlp": 0.21032715, + "step": 1540, + "time_per_iteration": 2.8246548175811768 + }, + { + "auxiliary_loss_clip": 0.01180468, + "auxiliary_loss_mlp": 0.01054758, + "balance_loss_clip": 1.05513978, + "balance_loss_mlp": 1.0345397, + "epoch": 0.18529429447483917, + "flos": 26794962328320.0, + "grad_norm": 2.0922437399703315, + "language_loss": 0.84306729, + "learning_rate": 3.7524482144729447e-06, + "loss": 0.86541951, + "num_input_tokens_seen": 32749485, + "router_z_loss_clip": 1.25390625, + "router_z_loss_mlp": 0.20214844, + "step": 1541, + "time_per_iteration": 2.8131349086761475 + }, + { + "auxiliary_loss_clip": 0.01181343, + "auxiliary_loss_mlp": 0.01054771, + "balance_loss_clip": 1.05691457, + "balance_loss_mlp": 1.03299165, + "epoch": 0.18541453736547828, + "flos": 13581595301760.0, + "grad_norm": 2.7747446518363015, + "language_loss": 0.83259678, + "learning_rate": 3.7520726921295106e-06, + "loss": 0.85495794, + "num_input_tokens_seen": 32766205, + "router_z_loss_clip": 1.24511719, + "router_z_loss_mlp": 0.21801758, + "step": 1542, + "time_per_iteration": 2.8175439834594727 + }, + { + "auxiliary_loss_clip": 0.01183007, + "auxiliary_loss_mlp": 0.01057341, + "balance_loss_clip": 1.0549171, + "balance_loss_mlp": 1.03637183, + "epoch": 0.18553478025611736, + "flos": 24025424077440.0, + "grad_norm": 2.0572710033257056, + "language_loss": 0.72250974, + "learning_rate": 3.751696903998042e-06, + "loss": 0.74491316, + "num_input_tokens_seen": 32784840, + "router_z_loss_clip": 1.28125, + "router_z_loss_mlp": 0.20983887, + "step": 1543, + "time_per_iteration": 2.7873306274414062 + }, + { + "auxiliary_loss_clip": 0.01179646, + "auxiliary_loss_mlp": 0.01049747, + "balance_loss_clip": 1.05359292, + "balance_loss_mlp": 1.02876592, + "epoch": 0.18565502314675644, + "flos": 25885165720320.0, + "grad_norm": 1.8021022797665396, + "language_loss": 0.69902194, + "learning_rate": 3.7513208501355456e-06, + "loss": 0.72131586, + "num_input_tokens_seen": 32805945, + "router_z_loss_clip": 1.26074219, + "router_z_loss_mlp": 0.20983887, + "step": 1544, + "time_per_iteration": 2.8031063079833984 + }, + { + "auxiliary_loss_clip": 0.01182477, + "auxiliary_loss_mlp": 0.01050875, + "balance_loss_clip": 1.05744123, + "balance_loss_mlp": 1.03038323, + "epoch": 0.18577526603739553, + "flos": 19610063631360.0, + "grad_norm": 2.3667548238547993, + "language_loss": 0.83430207, + "learning_rate": 3.750944530599069e-06, + "loss": 0.85663557, + "num_input_tokens_seen": 32825515, + "router_z_loss_clip": 1.25195312, + "router_z_loss_mlp": 0.20471191, + "step": 1545, + "time_per_iteration": 2.7672955989837646 + }, + { + "auxiliary_loss_clip": 0.01184251, + "auxiliary_loss_mlp": 0.01046098, + "balance_loss_clip": 1.05569041, + "balance_loss_mlp": 1.02452171, + "epoch": 0.18589550892803464, + "flos": 18474891137280.0, + "grad_norm": 2.7584782898520914, + "language_loss": 0.81042153, + "learning_rate": 3.7505679454456992e-06, + "loss": 0.83272505, + "num_input_tokens_seen": 32842125, + "router_z_loss_clip": 1.28613281, + "router_z_loss_mlp": 0.21582031, + "step": 1546, + "time_per_iteration": 2.7853987216949463 + }, + { + "auxiliary_loss_clip": 0.01183979, + "auxiliary_loss_mlp": 0.01059463, + "balance_loss_clip": 1.05627358, + "balance_loss_mlp": 1.03782618, + "epoch": 0.18601575181867372, + "flos": 23549966726400.0, + "grad_norm": 2.7301784909417695, + "language_loss": 0.70263189, + "learning_rate": 3.750191094732564e-06, + "loss": 0.7250663, + "num_input_tokens_seen": 32862990, + "router_z_loss_clip": 1.27832031, + "router_z_loss_mlp": 0.21643066, + "step": 1547, + "time_per_iteration": 2.913642168045044 + }, + { + "auxiliary_loss_clip": 0.01184069, + "auxiliary_loss_mlp": 0.01049858, + "balance_loss_clip": 1.05594873, + "balance_loss_mlp": 1.02851963, + "epoch": 0.1861359947093128, + "flos": 26360192108160.0, + "grad_norm": 2.4593113866373173, + "language_loss": 0.74929368, + "learning_rate": 3.7498139785168313e-06, + "loss": 0.77163291, + "num_input_tokens_seen": 32883595, + "router_z_loss_clip": 1.28125, + "router_z_loss_mlp": 0.21337891, + "step": 1548, + "time_per_iteration": 3.7842094898223877 + }, + { + "auxiliary_loss_clip": 0.01182846, + "auxiliary_loss_mlp": 0.0104513, + "balance_loss_clip": 1.05670297, + "balance_loss_mlp": 1.02388668, + "epoch": 0.1862562375999519, + "flos": 23331198942720.0, + "grad_norm": 2.2829552587354858, + "language_loss": 0.77119148, + "learning_rate": 3.749436596855709e-06, + "loss": 0.79347128, + "num_input_tokens_seen": 32902895, + "router_z_loss_clip": 1.26171875, + "router_z_loss_mlp": 0.21264648, + "step": 1549, + "time_per_iteration": 3.713062047958374 + }, + { + "auxiliary_loss_clip": 0.01188214, + "auxiliary_loss_mlp": 0.01055838, + "balance_loss_clip": 1.0573287, + "balance_loss_mlp": 1.03441656, + "epoch": 0.186376480490591, + "flos": 16648222942080.0, + "grad_norm": 1.9678527320072998, + "language_loss": 0.90490818, + "learning_rate": 3.749058949806446e-06, + "loss": 0.92734873, + "num_input_tokens_seen": 32919620, + "router_z_loss_clip": 1.30957031, + "router_z_loss_mlp": 0.21435547, + "step": 1550, + "time_per_iteration": 2.8045666217803955 + }, + { + "auxiliary_loss_clip": 0.01181347, + "auxiliary_loss_mlp": 0.01045556, + "balance_loss_clip": 1.05474234, + "balance_loss_mlp": 1.02382374, + "epoch": 0.18649672338123008, + "flos": 21468656039040.0, + "grad_norm": 2.175758543354845, + "language_loss": 0.84894115, + "learning_rate": 3.748681037426331e-06, + "loss": 0.87121016, + "num_input_tokens_seen": 32938830, + "router_z_loss_clip": 1.26464844, + "router_z_loss_mlp": 0.21740723, + "step": 1551, + "time_per_iteration": 3.6662869453430176 + }, + { + "auxiliary_loss_clip": 0.01181599, + "auxiliary_loss_mlp": 0.01056384, + "balance_loss_clip": 1.05514348, + "balance_loss_mlp": 1.03337693, + "epoch": 0.1866169662718692, + "flos": 12312728386560.0, + "grad_norm": 3.243583740762649, + "language_loss": 0.91784179, + "learning_rate": 3.7483028597726936e-06, + "loss": 0.94022167, + "num_input_tokens_seen": 32955600, + "router_z_loss_clip": 1.26367188, + "router_z_loss_mlp": 0.2298584, + "step": 1552, + "time_per_iteration": 2.885305643081665 + }, + { + "auxiliary_loss_clip": 0.01187904, + "auxiliary_loss_mlp": 0.01054507, + "balance_loss_clip": 1.05647314, + "balance_loss_mlp": 1.03058147, + "epoch": 0.18673720916250827, + "flos": 23581280407680.0, + "grad_norm": 2.528164573003419, + "language_loss": 0.62585104, + "learning_rate": 3.7479244169029017e-06, + "loss": 0.64827514, + "num_input_tokens_seen": 32975390, + "router_z_loss_clip": 1.31542969, + "router_z_loss_mlp": 0.23925781, + "step": 1553, + "time_per_iteration": 2.9895451068878174 + }, + { + "auxiliary_loss_clip": 0.01183064, + "auxiliary_loss_mlp": 0.01047282, + "balance_loss_clip": 1.05421233, + "balance_loss_mlp": 1.02510905, + "epoch": 0.18685745205314735, + "flos": 19718370115200.0, + "grad_norm": 2.4719976402277752, + "language_loss": 0.73313355, + "learning_rate": 3.7475457088743658e-06, + "loss": 0.75543696, + "num_input_tokens_seen": 32992640, + "router_z_loss_clip": 1.2890625, + "router_z_loss_mlp": 0.22143555, + "step": 1554, + "time_per_iteration": 2.8549246788024902 + }, + { + "auxiliary_loss_clip": 0.01184018, + "auxiliary_loss_mlp": 0.01059249, + "balance_loss_clip": 1.05715501, + "balance_loss_mlp": 1.03545451, + "epoch": 0.18697769494378644, + "flos": 34204123589760.0, + "grad_norm": 2.0916558573366117, + "language_loss": 0.75085139, + "learning_rate": 3.7471667357445348e-06, + "loss": 0.77328402, + "num_input_tokens_seen": 33012470, + "router_z_loss_clip": 1.26757812, + "router_z_loss_mlp": 0.23791504, + "step": 1555, + "time_per_iteration": 2.920215129852295 + }, + { + "auxiliary_loss_clip": 0.01183352, + "auxiliary_loss_mlp": 0.01046263, + "balance_loss_clip": 1.05307341, + "balance_loss_mlp": 1.02425742, + "epoch": 0.18709793783442555, + "flos": 34241327101440.0, + "grad_norm": 2.42584095820183, + "language_loss": 0.72028279, + "learning_rate": 3.7467874975709e-06, + "loss": 0.74257898, + "num_input_tokens_seen": 33033275, + "router_z_loss_clip": 1.30371094, + "router_z_loss_mlp": 0.21972656, + "step": 1556, + "time_per_iteration": 2.9550352096557617 + }, + { + "auxiliary_loss_clip": 0.0118474, + "auxiliary_loss_mlp": 0.01049285, + "balance_loss_clip": 1.05857253, + "balance_loss_mlp": 1.02696919, + "epoch": 0.18721818072506463, + "flos": 40734550529280.0, + "grad_norm": 2.3228984805872224, + "language_loss": 0.7812016, + "learning_rate": 3.7464079944109904e-06, + "loss": 0.8035419, + "num_input_tokens_seen": 33055135, + "router_z_loss_clip": 1.26269531, + "router_z_loss_mlp": 0.22302246, + "step": 1557, + "time_per_iteration": 3.004136800765991 + }, + { + "auxiliary_loss_clip": 0.01179822, + "auxiliary_loss_mlp": 0.01044173, + "balance_loss_clip": 1.05275166, + "balance_loss_mlp": 1.02374065, + "epoch": 0.18733842361570371, + "flos": 22157386392960.0, + "grad_norm": 2.2030941244613116, + "language_loss": 0.77572656, + "learning_rate": 3.746028226322376e-06, + "loss": 0.79796648, + "num_input_tokens_seen": 33071015, + "router_z_loss_clip": 1.27050781, + "router_z_loss_mlp": 0.2043457, + "step": 1558, + "time_per_iteration": 2.8031532764434814 + }, + { + "auxiliary_loss_clip": 0.0117662, + "auxiliary_loss_mlp": 0.01053693, + "balance_loss_clip": 1.05421817, + "balance_loss_mlp": 1.03346348, + "epoch": 0.18745866650634282, + "flos": 18914940656640.0, + "grad_norm": 1.9513277199032788, + "language_loss": 0.75413895, + "learning_rate": 3.745648193362669e-06, + "loss": 0.77644205, + "num_input_tokens_seen": 33090370, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.20239258, + "step": 1559, + "time_per_iteration": 2.801870822906494 + }, + { + "auxiliary_loss_clip": 0.01181049, + "auxiliary_loss_mlp": 0.01045896, + "balance_loss_clip": 1.05504692, + "balance_loss_mlp": 1.02498651, + "epoch": 0.1875789093969819, + "flos": 19314626267520.0, + "grad_norm": 2.0599283475126025, + "language_loss": 0.72853506, + "learning_rate": 3.745267895589518e-06, + "loss": 0.75080454, + "num_input_tokens_seen": 33108910, + "router_z_loss_clip": 1.25976562, + "router_z_loss_mlp": 0.20910645, + "step": 1560, + "time_per_iteration": 2.7792956829071045 + }, + { + "auxiliary_loss_clip": 0.01184698, + "auxiliary_loss_mlp": 0.01050138, + "balance_loss_clip": 1.05362332, + "balance_loss_mlp": 1.02789378, + "epoch": 0.187699152287621, + "flos": 17018965169280.0, + "grad_norm": 2.318830989119944, + "language_loss": 0.82422334, + "learning_rate": 3.7448873330606154e-06, + "loss": 0.84657168, + "num_input_tokens_seen": 33126680, + "router_z_loss_clip": 1.31152344, + "router_z_loss_mlp": 0.22253418, + "step": 1561, + "time_per_iteration": 2.8309898376464844 + }, + { + "auxiliary_loss_clip": 0.01182684, + "auxiliary_loss_mlp": 0.01046746, + "balance_loss_clip": 1.05485344, + "balance_loss_mlp": 1.02541995, + "epoch": 0.18781939517826007, + "flos": 22346384780160.0, + "grad_norm": 2.2100469686458037, + "language_loss": 0.87276566, + "learning_rate": 3.7445065058336914e-06, + "loss": 0.89506, + "num_input_tokens_seen": 33145550, + "router_z_loss_clip": 1.27929688, + "router_z_loss_mlp": 0.21313477, + "step": 1562, + "time_per_iteration": 2.788634777069092 + }, + { + "auxiliary_loss_clip": 0.01180082, + "auxiliary_loss_mlp": 0.01055308, + "balance_loss_clip": 1.05573463, + "balance_loss_mlp": 1.0329082, + "epoch": 0.18793963806889918, + "flos": 14611478054400.0, + "grad_norm": 2.2970064773831673, + "language_loss": 0.86398745, + "learning_rate": 3.7441254139665176e-06, + "loss": 0.88634133, + "num_input_tokens_seen": 33161735, + "router_z_loss_clip": 1.2421875, + "router_z_loss_mlp": 0.22424316, + "step": 1563, + "time_per_iteration": 2.7377147674560547 + }, + { + "auxiliary_loss_clip": 0.01180089, + "auxiliary_loss_mlp": 0.01047281, + "balance_loss_clip": 1.05591059, + "balance_loss_mlp": 1.02597809, + "epoch": 0.18805988095953827, + "flos": 17457075354240.0, + "grad_norm": 1.80489153644141, + "language_loss": 0.82679611, + "learning_rate": 3.743744057516905e-06, + "loss": 0.84906983, + "num_input_tokens_seen": 33179795, + "router_z_loss_clip": 1.24121094, + "router_z_loss_mlp": 0.2130127, + "step": 1564, + "time_per_iteration": 2.8076250553131104 + }, + { + "auxiliary_loss_clip": 0.01183161, + "auxiliary_loss_mlp": 0.01044657, + "balance_loss_clip": 1.05435872, + "balance_loss_mlp": 1.02342629, + "epoch": 0.18818012385017735, + "flos": 15043877976960.0, + "grad_norm": 2.9574790281274224, + "language_loss": 0.87719715, + "learning_rate": 3.743362436542706e-06, + "loss": 0.89947534, + "num_input_tokens_seen": 33194485, + "router_z_loss_clip": 1.28808594, + "router_z_loss_mlp": 0.21228027, + "step": 1565, + "time_per_iteration": 2.730985164642334 + }, + { + "auxiliary_loss_clip": 0.01179729, + "auxiliary_loss_mlp": 0.01051742, + "balance_loss_clip": 1.05312788, + "balance_loss_mlp": 1.02878249, + "epoch": 0.18830036674081646, + "flos": 47551975136640.0, + "grad_norm": 1.9710549548324847, + "language_loss": 0.76647997, + "learning_rate": 3.7429805511018115e-06, + "loss": 0.78879464, + "num_input_tokens_seen": 33216145, + "router_z_loss_clip": 1.26464844, + "router_z_loss_mlp": 0.22961426, + "step": 1566, + "time_per_iteration": 3.0042576789855957 + }, + { + "auxiliary_loss_clip": 0.01188761, + "auxiliary_loss_mlp": 0.01055368, + "balance_loss_clip": 1.05782783, + "balance_loss_mlp": 1.03163362, + "epoch": 0.18842060963145554, + "flos": 30044626698240.0, + "grad_norm": 2.216000655649863, + "language_loss": 0.7819618, + "learning_rate": 3.7425984012521524e-06, + "loss": 0.80440307, + "num_input_tokens_seen": 33236345, + "router_z_loss_clip": 1.31054688, + "router_z_loss_mlp": 0.23718262, + "step": 1567, + "time_per_iteration": 2.830549716949463 + }, + { + "auxiliary_loss_clip": 0.01109902, + "auxiliary_loss_mlp": 0.01007823, + "balance_loss_clip": 1.04490399, + "balance_loss_mlp": 1.00346017, + "epoch": 0.18854085252209463, + "flos": 70318372625280.0, + "grad_norm": 0.7497237597452503, + "language_loss": 0.60448945, + "learning_rate": 3.7422159870517025e-06, + "loss": 0.62566674, + "num_input_tokens_seen": 33301600, + "router_z_loss_clip": 0.6484375, + "router_z_loss_mlp": 0.04370117, + "step": 1568, + "time_per_iteration": 3.3967952728271484 + }, + { + "auxiliary_loss_clip": 0.01178914, + "auxiliary_loss_mlp": 0.01047209, + "balance_loss_clip": 1.05390191, + "balance_loss_mlp": 1.02651477, + "epoch": 0.1886610954127337, + "flos": 21289318410240.0, + "grad_norm": 1.7066406300681798, + "language_loss": 0.78910351, + "learning_rate": 3.7418333085584717e-06, + "loss": 0.81136477, + "num_input_tokens_seen": 33322785, + "router_z_loss_clip": 1.25, + "router_z_loss_mlp": 0.20715332, + "step": 1569, + "time_per_iteration": 2.86647629737854 + }, + { + "auxiliary_loss_clip": 0.01180824, + "auxiliary_loss_mlp": 0.01046631, + "balance_loss_clip": 1.05557466, + "balance_loss_mlp": 1.02576935, + "epoch": 0.18878133830337282, + "flos": 17266819991040.0, + "grad_norm": 2.3427109362939005, + "language_loss": 0.90830886, + "learning_rate": 3.7414503658305128e-06, + "loss": 0.93058348, + "num_input_tokens_seen": 33340020, + "router_z_loss_clip": 1.25292969, + "router_z_loss_mlp": 0.20874023, + "step": 1570, + "time_per_iteration": 2.9673852920532227 + }, + { + "auxiliary_loss_clip": 0.01180522, + "auxiliary_loss_mlp": 0.01051949, + "balance_loss_clip": 1.05210721, + "balance_loss_mlp": 1.02989542, + "epoch": 0.1889015811940119, + "flos": 25775207210880.0, + "grad_norm": 2.581471822437154, + "language_loss": 0.77588332, + "learning_rate": 3.7410671589259185e-06, + "loss": 0.798208, + "num_input_tokens_seen": 33358620, + "router_z_loss_clip": 1.28320312, + "router_z_loss_mlp": 0.22070312, + "step": 1571, + "time_per_iteration": 2.8733882904052734 + }, + { + "auxiliary_loss_clip": 0.011808, + "auxiliary_loss_mlp": 0.01050928, + "balance_loss_clip": 1.0523963, + "balance_loss_mlp": 1.02908862, + "epoch": 0.18902182408465099, + "flos": 21032197879680.0, + "grad_norm": 1.9873099411940984, + "language_loss": 0.79605317, + "learning_rate": 3.7406836879028205e-06, + "loss": 0.81837046, + "num_input_tokens_seen": 33378845, + "router_z_loss_clip": 1.28515625, + "router_z_loss_mlp": 0.21838379, + "step": 1572, + "time_per_iteration": 2.839460611343384 + }, + { + "auxiliary_loss_clip": 0.01177196, + "auxiliary_loss_mlp": 0.01045474, + "balance_loss_clip": 1.0522815, + "balance_loss_mlp": 1.02542305, + "epoch": 0.1891420669752901, + "flos": 22272121411200.0, + "grad_norm": 2.2506711483472417, + "language_loss": 0.76507956, + "learning_rate": 3.7402999528193907e-06, + "loss": 0.78730631, + "num_input_tokens_seen": 33398345, + "router_z_loss_clip": 1.25097656, + "router_z_loss_mlp": 0.20056152, + "step": 1573, + "time_per_iteration": 2.9051146507263184 + }, + { + "auxiliary_loss_clip": 0.01180331, + "auxiliary_loss_mlp": 0.01061839, + "balance_loss_clip": 1.05295277, + "balance_loss_mlp": 1.03902209, + "epoch": 0.18926230986592918, + "flos": 22017802141440.0, + "grad_norm": 2.6847738267593093, + "language_loss": 0.8571915, + "learning_rate": 3.739915953733842e-06, + "loss": 0.87961322, + "num_input_tokens_seen": 33416390, + "router_z_loss_clip": 1.2734375, + "router_z_loss_mlp": 0.22827148, + "step": 1574, + "time_per_iteration": 2.8456761837005615 + }, + { + "auxiliary_loss_clip": 0.01177976, + "auxiliary_loss_mlp": 0.01061626, + "balance_loss_clip": 1.05172479, + "balance_loss_mlp": 1.03947663, + "epoch": 0.18938255275656826, + "flos": 24462672336000.0, + "grad_norm": 2.7284987570240746, + "language_loss": 0.81498361, + "learning_rate": 3.7395316907044264e-06, + "loss": 0.83737969, + "num_input_tokens_seen": 33437175, + "router_z_loss_clip": 1.26269531, + "router_z_loss_mlp": 0.22155762, + "step": 1575, + "time_per_iteration": 2.8589799404144287 + }, + { + "auxiliary_loss_clip": 0.01179661, + "auxiliary_loss_mlp": 0.01052604, + "balance_loss_clip": 1.04948425, + "balance_loss_mlp": 1.02997792, + "epoch": 0.18950279564720737, + "flos": 24427049022720.0, + "grad_norm": 2.0752329173299073, + "language_loss": 0.79437739, + "learning_rate": 3.7391471637894364e-06, + "loss": 0.81669998, + "num_input_tokens_seen": 33459440, + "router_z_loss_clip": 1.30078125, + "router_z_loss_mlp": 0.22619629, + "step": 1576, + "time_per_iteration": 2.868938446044922 + }, + { + "auxiliary_loss_clip": 0.01177575, + "auxiliary_loss_mlp": 0.0104731, + "balance_loss_clip": 1.05302525, + "balance_loss_mlp": 1.02495825, + "epoch": 0.18962303853784646, + "flos": 19756291898880.0, + "grad_norm": 1.8017638223679182, + "language_loss": 0.84692788, + "learning_rate": 3.738762373047205e-06, + "loss": 0.86917675, + "num_input_tokens_seen": 33479360, + "router_z_loss_clip": 1.24511719, + "router_z_loss_mlp": 0.22351074, + "step": 1577, + "time_per_iteration": 4.5850749015808105 + }, + { + "auxiliary_loss_clip": 0.01178225, + "auxiliary_loss_mlp": 0.01050057, + "balance_loss_clip": 1.05451286, + "balance_loss_mlp": 1.02925491, + "epoch": 0.18974328142848554, + "flos": 21032054225280.0, + "grad_norm": 2.7370654603827003, + "language_loss": 0.83260131, + "learning_rate": 3.738377318536103e-06, + "loss": 0.85488415, + "num_input_tokens_seen": 33499245, + "router_z_loss_clip": 1.23828125, + "router_z_loss_mlp": 0.20788574, + "step": 1578, + "time_per_iteration": 3.697662830352783 + }, + { + "auxiliary_loss_clip": 0.01173435, + "auxiliary_loss_mlp": 0.01054593, + "balance_loss_clip": 1.05246699, + "balance_loss_mlp": 1.03367138, + "epoch": 0.18986352431912462, + "flos": 12966122736000.0, + "grad_norm": 2.7475995466580936, + "language_loss": 0.71589249, + "learning_rate": 3.7379920003145447e-06, + "loss": 0.73817283, + "num_input_tokens_seen": 33513520, + "router_z_loss_clip": 1.20800781, + "router_z_loss_mlp": 0.20910645, + "step": 1579, + "time_per_iteration": 2.7718849182128906 + }, + { + "auxiliary_loss_clip": 0.01180278, + "auxiliary_loss_mlp": 0.01058616, + "balance_loss_clip": 1.05195403, + "balance_loss_mlp": 1.03602624, + "epoch": 0.18998376720976373, + "flos": 23767908497280.0, + "grad_norm": 1.9926413191904284, + "language_loss": 0.83886921, + "learning_rate": 3.7376064184409817e-06, + "loss": 0.86125815, + "num_input_tokens_seen": 33533100, + "router_z_loss_clip": 1.28417969, + "router_z_loss_mlp": 0.22607422, + "step": 1580, + "time_per_iteration": 3.638878345489502 + }, + { + "auxiliary_loss_clip": 0.01183574, + "auxiliary_loss_mlp": 0.01056028, + "balance_loss_clip": 1.05588293, + "balance_loss_mlp": 1.03441513, + "epoch": 0.19010401010040281, + "flos": 22966023323520.0, + "grad_norm": 1.7044072190194277, + "language_loss": 0.8701461, + "learning_rate": 3.7372205729739063e-06, + "loss": 0.89254212, + "num_input_tokens_seen": 33554915, + "router_z_loss_clip": 1.27832031, + "router_z_loss_mlp": 0.21606445, + "step": 1581, + "time_per_iteration": 2.7970340251922607 + }, + { + "auxiliary_loss_clip": 0.0118498, + "auxiliary_loss_mlp": 0.01050806, + "balance_loss_clip": 1.05808198, + "balance_loss_mlp": 1.02857351, + "epoch": 0.1902242529910419, + "flos": 19135647774720.0, + "grad_norm": 2.279568916741227, + "language_loss": 0.71667308, + "learning_rate": 3.7368344639718514e-06, + "loss": 0.73903096, + "num_input_tokens_seen": 33572850, + "router_z_loss_clip": 1.26757812, + "router_z_loss_mlp": 0.22241211, + "step": 1582, + "time_per_iteration": 2.802520990371704 + }, + { + "auxiliary_loss_clip": 0.01180683, + "auxiliary_loss_mlp": 0.01051647, + "balance_loss_clip": 1.05567002, + "balance_loss_mlp": 1.0294621, + "epoch": 0.190344495881681, + "flos": 25483935824640.0, + "grad_norm": 1.796551357002157, + "language_loss": 0.80720156, + "learning_rate": 3.7364480914933895e-06, + "loss": 0.82952487, + "num_input_tokens_seen": 33593090, + "router_z_loss_clip": 1.24707031, + "router_z_loss_mlp": 0.22167969, + "step": 1583, + "time_per_iteration": 2.8294358253479004 + }, + { + "auxiliary_loss_clip": 0.0118008, + "auxiliary_loss_mlp": 0.01050805, + "balance_loss_clip": 1.05394197, + "balance_loss_mlp": 1.02950287, + "epoch": 0.1904647387723201, + "flos": 26792843425920.0, + "grad_norm": 2.4911818283834193, + "language_loss": 0.80880487, + "learning_rate": 3.7360614555971325e-06, + "loss": 0.83111376, + "num_input_tokens_seen": 33612745, + "router_z_loss_clip": 1.26074219, + "router_z_loss_mlp": 0.2130127, + "step": 1584, + "time_per_iteration": 2.9041659832000732 + }, + { + "auxiliary_loss_clip": 0.01181568, + "auxiliary_loss_mlp": 0.01053537, + "balance_loss_clip": 1.05540419, + "balance_loss_mlp": 1.03124464, + "epoch": 0.19058498166295917, + "flos": 23987753688960.0, + "grad_norm": 2.1256488536606386, + "language_loss": 0.85119998, + "learning_rate": 3.735674556341733e-06, + "loss": 0.87355101, + "num_input_tokens_seen": 33632360, + "router_z_loss_clip": 1.25976562, + "router_z_loss_mlp": 0.22290039, + "step": 1585, + "time_per_iteration": 2.7851312160491943 + }, + { + "auxiliary_loss_clip": 0.01182733, + "auxiliary_loss_mlp": 0.01054256, + "balance_loss_clip": 1.05611336, + "balance_loss_mlp": 1.03105831, + "epoch": 0.19070522455359826, + "flos": 28293299280000.0, + "grad_norm": 2.929408147197599, + "language_loss": 0.82699108, + "learning_rate": 3.7352873937858835e-06, + "loss": 0.84936094, + "num_input_tokens_seen": 33653895, + "router_z_loss_clip": 1.265625, + "router_z_loss_mlp": 0.23181152, + "step": 1586, + "time_per_iteration": 2.8130950927734375 + }, + { + "auxiliary_loss_clip": 0.01182085, + "auxiliary_loss_mlp": 0.01055402, + "balance_loss_clip": 1.05396831, + "balance_loss_mlp": 1.03257298, + "epoch": 0.19082546744423737, + "flos": 25660220797440.0, + "grad_norm": 2.142209346505814, + "language_loss": 0.72077149, + "learning_rate": 3.734899967988316e-06, + "loss": 0.74314642, + "num_input_tokens_seen": 33672075, + "router_z_loss_clip": 1.28125, + "router_z_loss_mlp": 0.22814941, + "step": 1587, + "time_per_iteration": 2.876495838165283 + }, + { + "auxiliary_loss_clip": 0.0117847, + "auxiliary_loss_mlp": 0.01044743, + "balance_loss_clip": 1.05531406, + "balance_loss_mlp": 1.02310658, + "epoch": 0.19094571033487645, + "flos": 19719483436800.0, + "grad_norm": 2.212342239742156, + "language_loss": 0.83660924, + "learning_rate": 3.7345122790078026e-06, + "loss": 0.85884136, + "num_input_tokens_seen": 33689640, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.21643066, + "step": 1588, + "time_per_iteration": 2.913839340209961 + }, + { + "auxiliary_loss_clip": 0.01182176, + "auxiliary_loss_mlp": 0.01050397, + "balance_loss_clip": 1.05460274, + "balance_loss_mlp": 1.0272584, + "epoch": 0.19106595322551553, + "flos": 21616320850560.0, + "grad_norm": 2.747929170504145, + "language_loss": 0.93066597, + "learning_rate": 3.7341243269031556e-06, + "loss": 0.9529916, + "num_input_tokens_seen": 33708630, + "router_z_loss_clip": 1.27539062, + "router_z_loss_mlp": 0.23132324, + "step": 1589, + "time_per_iteration": 2.835106611251831 + }, + { + "auxiliary_loss_clip": 0.01176591, + "auxiliary_loss_mlp": 0.01053579, + "balance_loss_clip": 1.05508113, + "balance_loss_mlp": 1.03243136, + "epoch": 0.19118619611615464, + "flos": 29896890059520.0, + "grad_norm": 1.7461627748760162, + "language_loss": 0.77497995, + "learning_rate": 3.7337361117332275e-06, + "loss": 0.79728168, + "num_input_tokens_seen": 33730370, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.21142578, + "step": 1590, + "time_per_iteration": 2.928208827972412 + }, + { + "auxiliary_loss_clip": 0.01178155, + "auxiliary_loss_mlp": 0.01053037, + "balance_loss_clip": 1.05352855, + "balance_loss_mlp": 1.03244913, + "epoch": 0.19130643900679373, + "flos": 17273428093440.0, + "grad_norm": 3.933482988358306, + "language_loss": 0.76748687, + "learning_rate": 3.7333476335569087e-06, + "loss": 0.7897988, + "num_input_tokens_seen": 33748370, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.20581055, + "step": 1591, + "time_per_iteration": 2.815990686416626 + }, + { + "auxiliary_loss_clip": 0.0118329, + "auxiliary_loss_mlp": 0.01051077, + "balance_loss_clip": 1.05389571, + "balance_loss_mlp": 1.0287962, + "epoch": 0.1914266818974328, + "flos": 24826339584000.0, + "grad_norm": 2.539743416654349, + "language_loss": 0.67088592, + "learning_rate": 3.7329588924331325e-06, + "loss": 0.69322956, + "num_input_tokens_seen": 33769575, + "router_z_loss_clip": 1.29394531, + "router_z_loss_mlp": 0.22265625, + "step": 1592, + "time_per_iteration": 2.800143241882324 + }, + { + "auxiliary_loss_clip": 0.0118155, + "auxiliary_loss_mlp": 0.01047694, + "balance_loss_clip": 1.05579448, + "balance_loss_mlp": 1.0268091, + "epoch": 0.1915469247880719, + "flos": 18952467390720.0, + "grad_norm": 2.1186486422803874, + "language_loss": 0.82061321, + "learning_rate": 3.732569888420871e-06, + "loss": 0.84290564, + "num_input_tokens_seen": 33789110, + "router_z_loss_clip": 1.25585938, + "router_z_loss_mlp": 0.2088623, + "step": 1593, + "time_per_iteration": 2.8301637172698975 + }, + { + "auxiliary_loss_clip": 0.01181242, + "auxiliary_loss_mlp": 0.01059082, + "balance_loss_clip": 1.05217195, + "balance_loss_mlp": 1.0374099, + "epoch": 0.191667167678711, + "flos": 21032952065280.0, + "grad_norm": 2.505441232144283, + "language_loss": 0.83041555, + "learning_rate": 3.732180621579134e-06, + "loss": 0.85281879, + "num_input_tokens_seen": 33808325, + "router_z_loss_clip": 1.28808594, + "router_z_loss_mlp": 0.21691895, + "step": 1594, + "time_per_iteration": 2.9283671379089355 + }, + { + "auxiliary_loss_clip": 0.01183416, + "auxiliary_loss_mlp": 0.0106955, + "balance_loss_clip": 1.05670333, + "balance_loss_mlp": 1.04786587, + "epoch": 0.1917874105693501, + "flos": 34237663914240.0, + "grad_norm": 2.135592538420256, + "language_loss": 0.80902803, + "learning_rate": 3.7317910919669745e-06, + "loss": 0.83155775, + "num_input_tokens_seen": 33829520, + "router_z_loss_clip": 1.26757812, + "router_z_loss_mlp": 0.21655273, + "step": 1595, + "time_per_iteration": 2.9149105548858643 + }, + { + "auxiliary_loss_clip": 0.01180241, + "auxiliary_loss_mlp": 0.01057513, + "balance_loss_clip": 1.05356801, + "balance_loss_mlp": 1.03529215, + "epoch": 0.19190765345998917, + "flos": 23550613171200.0, + "grad_norm": 3.3270185859743586, + "language_loss": 0.76135921, + "learning_rate": 3.7314012996434826e-06, + "loss": 0.78373682, + "num_input_tokens_seen": 33848250, + "router_z_loss_clip": 1.26367188, + "router_z_loss_mlp": 0.22229004, + "step": 1596, + "time_per_iteration": 2.893444538116455 + }, + { + "auxiliary_loss_clip": 0.01178395, + "auxiliary_loss_mlp": 0.01048063, + "balance_loss_clip": 1.05353403, + "balance_loss_mlp": 1.02563977, + "epoch": 0.19202789635062828, + "flos": 19861330245120.0, + "grad_norm": 2.1417750757668794, + "language_loss": 0.80891097, + "learning_rate": 3.7310112446677907e-06, + "loss": 0.83117557, + "num_input_tokens_seen": 33866160, + "router_z_loss_clip": 1.24902344, + "router_z_loss_mlp": 0.22424316, + "step": 1597, + "time_per_iteration": 3.015789747238159 + }, + { + "auxiliary_loss_clip": 0.01180889, + "auxiliary_loss_mlp": 0.01048438, + "balance_loss_clip": 1.05327463, + "balance_loss_mlp": 1.0262531, + "epoch": 0.19214813924126736, + "flos": 20922957642240.0, + "grad_norm": 2.8090880277380887, + "language_loss": 0.69310862, + "learning_rate": 3.7306209270990695e-06, + "loss": 0.71540189, + "num_input_tokens_seen": 33884165, + "router_z_loss_clip": 1.27636719, + "router_z_loss_mlp": 0.22180176, + "step": 1598, + "time_per_iteration": 2.8254871368408203 + }, + { + "auxiliary_loss_clip": 0.01182731, + "auxiliary_loss_mlp": 0.01047841, + "balance_loss_clip": 1.05742681, + "balance_loss_mlp": 1.02742004, + "epoch": 0.19226838213190645, + "flos": 26359725231360.0, + "grad_norm": 1.882588232475419, + "language_loss": 0.86657465, + "learning_rate": 3.7302303469965292e-06, + "loss": 0.88888037, + "num_input_tokens_seen": 33903705, + "router_z_loss_clip": 1.25195312, + "router_z_loss_mlp": 0.20422363, + "step": 1599, + "time_per_iteration": 2.8711788654327393 + }, + { + "auxiliary_loss_clip": 0.01178207, + "auxiliary_loss_mlp": 0.01048154, + "balance_loss_clip": 1.05364096, + "balance_loss_mlp": 1.02772129, + "epoch": 0.19238862502254553, + "flos": 20850525866880.0, + "grad_norm": 1.8217418179001752, + "language_loss": 0.70354104, + "learning_rate": 3.7298395044194206e-06, + "loss": 0.72580469, + "num_input_tokens_seen": 33922515, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.2043457, + "step": 1600, + "time_per_iteration": 2.8017640113830566 + }, + { + "auxiliary_loss_clip": 0.01178031, + "auxiliary_loss_mlp": 0.01055786, + "balance_loss_clip": 1.05492258, + "balance_loss_mlp": 1.03408968, + "epoch": 0.19250886791318464, + "flos": 21726063878400.0, + "grad_norm": 2.2371745974858044, + "language_loss": 0.94340092, + "learning_rate": 3.7294483994270356e-06, + "loss": 0.96573907, + "num_input_tokens_seen": 33940840, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.21704102, + "step": 1601, + "time_per_iteration": 2.783379316329956 + }, + { + "auxiliary_loss_clip": 0.0117828, + "auxiliary_loss_mlp": 0.01049816, + "balance_loss_clip": 1.05417418, + "balance_loss_mlp": 1.02845335, + "epoch": 0.19262911080382372, + "flos": 23367827836800.0, + "grad_norm": 2.4940482316499892, + "language_loss": 0.78104651, + "learning_rate": 3.7290570320787033e-06, + "loss": 0.80332744, + "num_input_tokens_seen": 33960420, + "router_z_loss_clip": 1.24121094, + "router_z_loss_mlp": 0.21374512, + "step": 1602, + "time_per_iteration": 2.7827460765838623 + }, + { + "auxiliary_loss_clip": 0.01177101, + "auxiliary_loss_mlp": 0.01049429, + "balance_loss_clip": 1.05184722, + "balance_loss_mlp": 1.02789998, + "epoch": 0.1927493536944628, + "flos": 21943502858880.0, + "grad_norm": 9.791429837720289, + "language_loss": 0.70861113, + "learning_rate": 3.728665402433793e-06, + "loss": 0.73087645, + "num_input_tokens_seen": 33978990, + "router_z_loss_clip": 1.25195312, + "router_z_loss_mlp": 0.21533203, + "step": 1603, + "time_per_iteration": 2.754251480102539 + }, + { + "auxiliary_loss_clip": 0.01176308, + "auxiliary_loss_mlp": 0.01057711, + "balance_loss_clip": 1.05358815, + "balance_loss_mlp": 1.03751719, + "epoch": 0.19286959658510192, + "flos": 16545590807040.0, + "grad_norm": 3.0227660763833297, + "language_loss": 0.86135101, + "learning_rate": 3.7282735105517164e-06, + "loss": 0.88369113, + "num_input_tokens_seen": 33997115, + "router_z_loss_clip": 1.2265625, + "router_z_loss_mlp": 0.20202637, + "step": 1604, + "time_per_iteration": 2.8109419345855713 + }, + { + "auxiliary_loss_clip": 0.01180188, + "auxiliary_loss_mlp": 0.01048725, + "balance_loss_clip": 1.05411613, + "balance_loss_mlp": 1.02748156, + "epoch": 0.192989839475741, + "flos": 21616967295360.0, + "grad_norm": 2.1343675122489945, + "language_loss": 0.67438924, + "learning_rate": 3.727881356491922e-06, + "loss": 0.69667828, + "num_input_tokens_seen": 34015525, + "router_z_loss_clip": 1.26171875, + "router_z_loss_mlp": 0.21264648, + "step": 1605, + "time_per_iteration": 2.7730700969696045 + }, + { + "auxiliary_loss_clip": 0.01175132, + "auxiliary_loss_mlp": 0.01054699, + "balance_loss_clip": 1.05527711, + "balance_loss_mlp": 1.03444552, + "epoch": 0.19311008236638008, + "flos": 19281516906240.0, + "grad_norm": 2.1306073879952088, + "language_loss": 0.75690067, + "learning_rate": 3.7274889403139002e-06, + "loss": 0.77919906, + "num_input_tokens_seen": 34033150, + "router_z_loss_clip": 1.19824219, + "router_z_loss_mlp": 0.20251465, + "step": 1606, + "time_per_iteration": 4.492567777633667 + }, + { + "auxiliary_loss_clip": 0.01175079, + "auxiliary_loss_mlp": 0.01040352, + "balance_loss_clip": 1.05288184, + "balance_loss_mlp": 1.01982427, + "epoch": 0.1932303252570192, + "flos": 28652369587200.0, + "grad_norm": 2.3177441073817726, + "language_loss": 0.78909039, + "learning_rate": 3.727096262077179e-06, + "loss": 0.81124473, + "num_input_tokens_seen": 34052145, + "router_z_loss_clip": 1.22363281, + "router_z_loss_mlp": 0.20507812, + "step": 1607, + "time_per_iteration": 3.737048864364624 + }, + { + "auxiliary_loss_clip": 0.01178177, + "auxiliary_loss_mlp": 0.01062064, + "balance_loss_clip": 1.0540421, + "balance_loss_mlp": 1.0406183, + "epoch": 0.19335056814765827, + "flos": 18368990864640.0, + "grad_norm": 3.4447526612329074, + "language_loss": 0.85432076, + "learning_rate": 3.7267033218413285e-06, + "loss": 0.87672317, + "num_input_tokens_seen": 34069940, + "router_z_loss_clip": 1.24316406, + "router_z_loss_mlp": 0.21459961, + "step": 1608, + "time_per_iteration": 3.799928665161133 + }, + { + "auxiliary_loss_clip": 0.01176574, + "auxiliary_loss_mlp": 0.0105116, + "balance_loss_clip": 1.05193257, + "balance_loss_mlp": 1.03067994, + "epoch": 0.19347081103829736, + "flos": 13260877741440.0, + "grad_norm": 3.6865233269702604, + "language_loss": 0.8134104, + "learning_rate": 3.726310119665957e-06, + "loss": 0.83568776, + "num_input_tokens_seen": 34086275, + "router_z_loss_clip": 1.24511719, + "router_z_loss_mlp": 0.20471191, + "step": 1609, + "time_per_iteration": 2.9212498664855957 + }, + { + "auxiliary_loss_clip": 0.01177117, + "auxiliary_loss_mlp": 0.01046477, + "balance_loss_clip": 1.0542686, + "balance_loss_mlp": 1.02587795, + "epoch": 0.19359105392893644, + "flos": 20300122788480.0, + "grad_norm": 1.9534140696852325, + "language_loss": 0.85391325, + "learning_rate": 3.725916655610713e-06, + "loss": 0.87614918, + "num_input_tokens_seen": 34105605, + "router_z_loss_clip": 1.23046875, + "router_z_loss_mlp": 0.20593262, + "step": 1610, + "time_per_iteration": 2.8740077018737793 + }, + { + "auxiliary_loss_clip": 0.01180934, + "auxiliary_loss_mlp": 0.01056883, + "balance_loss_clip": 1.05369449, + "balance_loss_mlp": 1.03320825, + "epoch": 0.19371129681957555, + "flos": 20484596062080.0, + "grad_norm": 2.903219039643113, + "language_loss": 0.756172, + "learning_rate": 3.725522929735284e-06, + "loss": 0.77855015, + "num_input_tokens_seen": 34122540, + "router_z_loss_clip": 1.27148438, + "router_z_loss_mlp": 0.23681641, + "step": 1611, + "time_per_iteration": 2.8342626094818115 + }, + { + "auxiliary_loss_clip": 0.01178437, + "auxiliary_loss_mlp": 0.01058002, + "balance_loss_clip": 1.05395317, + "balance_loss_mlp": 1.03628182, + "epoch": 0.19383153971021463, + "flos": 30445497457920.0, + "grad_norm": 4.8559112828081465, + "language_loss": 0.73914737, + "learning_rate": 3.725128942099399e-06, + "loss": 0.7615118, + "num_input_tokens_seen": 34142940, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.21716309, + "step": 1612, + "time_per_iteration": 2.849015951156616 + }, + { + "auxiliary_loss_clip": 0.01177022, + "auxiliary_loss_mlp": 0.01047936, + "balance_loss_clip": 1.05012715, + "balance_loss_mlp": 1.02479732, + "epoch": 0.19395178260085372, + "flos": 24569937325440.0, + "grad_norm": 3.180304952479758, + "language_loss": 0.80161935, + "learning_rate": 3.7247346927628245e-06, + "loss": 0.82386893, + "num_input_tokens_seen": 34162875, + "router_z_loss_clip": 1.26757812, + "router_z_loss_mlp": 0.23144531, + "step": 1613, + "time_per_iteration": 2.829235792160034 + }, + { + "auxiliary_loss_clip": 0.011827, + "auxiliary_loss_mlp": 0.01048971, + "balance_loss_clip": 1.05354464, + "balance_loss_mlp": 1.0261544, + "epoch": 0.19407202549149283, + "flos": 28950608211840.0, + "grad_norm": 2.2994822784500344, + "language_loss": 0.79029602, + "learning_rate": 3.7243401817853694e-06, + "loss": 0.81261271, + "num_input_tokens_seen": 34183565, + "router_z_loss_clip": 1.2890625, + "router_z_loss_mlp": 0.22839355, + "step": 1614, + "time_per_iteration": 3.017498731613159 + }, + { + "auxiliary_loss_clip": 0.01176825, + "auxiliary_loss_mlp": 0.0104397, + "balance_loss_clip": 1.05391741, + "balance_loss_mlp": 1.02346599, + "epoch": 0.1941922683821319, + "flos": 18004497603840.0, + "grad_norm": 2.2020053186565907, + "language_loss": 0.72240412, + "learning_rate": 3.723945409226879e-06, + "loss": 0.7446121, + "num_input_tokens_seen": 34202055, + "router_z_loss_clip": 1.22949219, + "router_z_loss_mlp": 0.20507812, + "step": 1615, + "time_per_iteration": 2.8270697593688965 + }, + { + "auxiliary_loss_clip": 0.01175866, + "auxiliary_loss_mlp": 0.01049831, + "balance_loss_clip": 1.05168962, + "balance_loss_mlp": 1.02790833, + "epoch": 0.194312511272771, + "flos": 9720337034880.0, + "grad_norm": 2.5353256347052837, + "language_loss": 0.80173194, + "learning_rate": 3.723550375147241e-06, + "loss": 0.82398891, + "num_input_tokens_seen": 34216830, + "router_z_loss_clip": 1.24121094, + "router_z_loss_mlp": 0.21936035, + "step": 1616, + "time_per_iteration": 2.7620015144348145 + }, + { + "auxiliary_loss_clip": 0.0118002, + "auxiliary_loss_mlp": 0.01050091, + "balance_loss_clip": 1.05348945, + "balance_loss_mlp": 1.02777553, + "epoch": 0.19443275416341008, + "flos": 27016208150400.0, + "grad_norm": 2.2040174601417353, + "language_loss": 0.80348909, + "learning_rate": 3.7231550796063816e-06, + "loss": 0.82579017, + "num_input_tokens_seen": 34236840, + "router_z_loss_clip": 1.26660156, + "router_z_loss_mlp": 0.22338867, + "step": 1617, + "time_per_iteration": 2.7832705974578857 + }, + { + "auxiliary_loss_clip": 0.01181563, + "auxiliary_loss_mlp": 0.01053327, + "balance_loss_clip": 1.0550952, + "balance_loss_mlp": 1.03010476, + "epoch": 0.1945529970540492, + "flos": 15846625077120.0, + "grad_norm": 2.3275242132409586, + "language_loss": 0.64897132, + "learning_rate": 3.722759522664266e-06, + "loss": 0.6713202, + "num_input_tokens_seen": 34254140, + "router_z_loss_clip": 1.26269531, + "router_z_loss_mlp": 0.2322998, + "step": 1618, + "time_per_iteration": 2.7741169929504395 + }, + { + "auxiliary_loss_clip": 0.01178438, + "auxiliary_loss_mlp": 0.0104843, + "balance_loss_clip": 1.0547806, + "balance_loss_mlp": 1.02640045, + "epoch": 0.19467323994468827, + "flos": 19314985403520.0, + "grad_norm": 2.194414536794658, + "language_loss": 0.81942838, + "learning_rate": 3.7223637043809016e-06, + "loss": 0.84169704, + "num_input_tokens_seen": 34273120, + "router_z_loss_clip": 1.23632812, + "router_z_loss_mlp": 0.22021484, + "step": 1619, + "time_per_iteration": 2.807926893234253 + }, + { + "auxiliary_loss_clip": 0.01177815, + "auxiliary_loss_mlp": 0.0104511, + "balance_loss_clip": 1.05400145, + "balance_loss_mlp": 1.02446294, + "epoch": 0.19479348283532735, + "flos": 24133227770880.0, + "grad_norm": 1.8176669818904936, + "language_loss": 0.86300582, + "learning_rate": 3.7219676248163322e-06, + "loss": 0.88523507, + "num_input_tokens_seen": 34290285, + "router_z_loss_clip": 1.23828125, + "router_z_loss_mlp": 0.20654297, + "step": 1620, + "time_per_iteration": 2.8690435886383057 + }, + { + "auxiliary_loss_clip": 0.01178266, + "auxiliary_loss_mlp": 0.01056525, + "balance_loss_clip": 1.05473208, + "balance_loss_mlp": 1.03445995, + "epoch": 0.19491372572596646, + "flos": 25775638174080.0, + "grad_norm": 2.362105047275456, + "language_loss": 0.93677205, + "learning_rate": 3.721571284030643e-06, + "loss": 0.95911998, + "num_input_tokens_seen": 34310095, + "router_z_loss_clip": 1.23535156, + "router_z_loss_mlp": 0.22045898, + "step": 1621, + "time_per_iteration": 2.8659491539001465 + }, + { + "auxiliary_loss_clip": 0.01174864, + "auxiliary_loss_mlp": 0.01056078, + "balance_loss_clip": 1.05326033, + "balance_loss_mlp": 1.03296351, + "epoch": 0.19503396861660555, + "flos": 19645220067840.0, + "grad_norm": 2.1789844775844123, + "language_loss": 0.7879957, + "learning_rate": 3.7211746820839587e-06, + "loss": 0.81030512, + "num_input_tokens_seen": 34327190, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.2310791, + "step": 1622, + "time_per_iteration": 2.7477221488952637 + }, + { + "auxiliary_loss_clip": 0.01175615, + "auxiliary_loss_mlp": 0.01047367, + "balance_loss_clip": 1.05274582, + "balance_loss_mlp": 1.02540839, + "epoch": 0.19515421150724463, + "flos": 21033023892480.0, + "grad_norm": 1.811528281083671, + "language_loss": 0.80666101, + "learning_rate": 3.7207778190364437e-06, + "loss": 0.82889086, + "num_input_tokens_seen": 34345615, + "router_z_loss_clip": 1.22753906, + "router_z_loss_mlp": 0.21936035, + "step": 1623, + "time_per_iteration": 2.781398296356201 + }, + { + "auxiliary_loss_clip": 0.01175971, + "auxiliary_loss_mlp": 0.01047006, + "balance_loss_clip": 1.05685854, + "balance_loss_mlp": 1.02750325, + "epoch": 0.1952744543978837, + "flos": 32961255143040.0, + "grad_norm": 2.4447422531936516, + "language_loss": 0.73923099, + "learning_rate": 3.720380694948302e-06, + "loss": 0.76146078, + "num_input_tokens_seen": 34368500, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.19506836, + "step": 1624, + "time_per_iteration": 2.8639473915100098 + }, + { + "auxiliary_loss_clip": 0.01100866, + "auxiliary_loss_mlp": 0.01014486, + "balance_loss_clip": 1.04167295, + "balance_loss_mlp": 1.01045704, + "epoch": 0.19539469728852282, + "flos": 64044312030720.0, + "grad_norm": 1.0428255787193244, + "language_loss": 0.71256709, + "learning_rate": 3.719983309879777e-06, + "loss": 0.7337206, + "num_input_tokens_seen": 34428280, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.0402832, + "step": 1625, + "time_per_iteration": 3.2913057804107666 + }, + { + "auxiliary_loss_clip": 0.01176344, + "auxiliary_loss_mlp": 0.01051932, + "balance_loss_clip": 1.05484486, + "balance_loss_mlp": 1.03157115, + "epoch": 0.1955149401791619, + "flos": 13370908078080.0, + "grad_norm": 1.8297947893112918, + "language_loss": 0.77175081, + "learning_rate": 3.719585663891151e-06, + "loss": 0.79403353, + "num_input_tokens_seen": 34445815, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.20361328, + "step": 1626, + "time_per_iteration": 2.786674976348877 + }, + { + "auxiliary_loss_clip": 0.01179884, + "auxiliary_loss_mlp": 0.01054476, + "balance_loss_clip": 1.05482888, + "balance_loss_mlp": 1.0318979, + "epoch": 0.195635183069801, + "flos": 18728887184640.0, + "grad_norm": 3.325277682400377, + "language_loss": 0.7940104, + "learning_rate": 3.719187757042747e-06, + "loss": 0.81635404, + "num_input_tokens_seen": 34463635, + "router_z_loss_clip": 1.25195312, + "router_z_loss_mlp": 0.22558594, + "step": 1627, + "time_per_iteration": 2.7736847400665283 + }, + { + "auxiliary_loss_clip": 0.01095223, + "auxiliary_loss_mlp": 0.01009836, + "balance_loss_clip": 1.03684151, + "balance_loss_mlp": 1.00618827, + "epoch": 0.1957554259604401, + "flos": 69313952615040.0, + "grad_norm": 0.72759619764037, + "language_loss": 0.54914945, + "learning_rate": 3.7187895893949275e-06, + "loss": 0.57020003, + "num_input_tokens_seen": 34530105, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.03637695, + "step": 1628, + "time_per_iteration": 3.401878833770752 + }, + { + "auxiliary_loss_clip": 0.01177131, + "auxiliary_loss_mlp": 0.01052934, + "balance_loss_clip": 1.05185771, + "balance_loss_mlp": 1.02965295, + "epoch": 0.19587566885107918, + "flos": 21069257736960.0, + "grad_norm": 2.7142243265503714, + "language_loss": 0.76174235, + "learning_rate": 3.7183911610080937e-06, + "loss": 0.78404307, + "num_input_tokens_seen": 34546970, + "router_z_loss_clip": 1.25195312, + "router_z_loss_mlp": 0.23266602, + "step": 1629, + "time_per_iteration": 2.784621000289917 + }, + { + "auxiliary_loss_clip": 0.01179508, + "auxiliary_loss_mlp": 0.01050417, + "balance_loss_clip": 1.05391681, + "balance_loss_mlp": 1.02737379, + "epoch": 0.19599591174171827, + "flos": 22194661731840.0, + "grad_norm": 5.749443589461662, + "language_loss": 0.7469815, + "learning_rate": 3.7179924719426872e-06, + "loss": 0.76928067, + "num_input_tokens_seen": 34564865, + "router_z_loss_clip": 1.2578125, + "router_z_loss_mlp": 0.23046875, + "step": 1630, + "time_per_iteration": 2.889883518218994 + }, + { + "auxiliary_loss_clip": 0.01181336, + "auxiliary_loss_mlp": 0.01053632, + "balance_loss_clip": 1.05417764, + "balance_loss_mlp": 1.03122091, + "epoch": 0.19611615463235738, + "flos": 23768375374080.0, + "grad_norm": 7.6395094979805505, + "language_loss": 0.76047015, + "learning_rate": 3.7175935222591885e-06, + "loss": 0.78281987, + "num_input_tokens_seen": 34584165, + "router_z_loss_clip": 1.27441406, + "router_z_loss_mlp": 0.22436523, + "step": 1631, + "time_per_iteration": 2.8456530570983887 + }, + { + "auxiliary_loss_clip": 0.01182108, + "auxiliary_loss_mlp": 0.01058029, + "balance_loss_clip": 1.05600226, + "balance_loss_mlp": 1.035236, + "epoch": 0.19623639752299646, + "flos": 28618218731520.0, + "grad_norm": 1.9985745952569165, + "language_loss": 0.74440491, + "learning_rate": 3.717194312018118e-06, + "loss": 0.76680624, + "num_input_tokens_seen": 34603150, + "router_z_loss_clip": 1.25976562, + "router_z_loss_mlp": 0.22790527, + "step": 1632, + "time_per_iteration": 2.9632601737976074 + }, + { + "auxiliary_loss_clip": 0.01182781, + "auxiliary_loss_mlp": 0.01051715, + "balance_loss_clip": 1.05610204, + "balance_loss_mlp": 1.02865994, + "epoch": 0.19635664041363554, + "flos": 21032700670080.0, + "grad_norm": 4.839239961538564, + "language_loss": 0.76156723, + "learning_rate": 3.716794841280036e-06, + "loss": 0.78391218, + "num_input_tokens_seen": 34621855, + "router_z_loss_clip": 1.26367188, + "router_z_loss_mlp": 0.23034668, + "step": 1633, + "time_per_iteration": 2.8474435806274414 + }, + { + "auxiliary_loss_clip": 0.01177988, + "auxiliary_loss_mlp": 0.01049768, + "balance_loss_clip": 1.05229771, + "balance_loss_mlp": 1.02766633, + "epoch": 0.19647688330427462, + "flos": 18879748306560.0, + "grad_norm": 2.795505328712666, + "language_loss": 0.77704668, + "learning_rate": 3.7163951101055407e-06, + "loss": 0.79932421, + "num_input_tokens_seen": 34639915, + "router_z_loss_clip": 1.25488281, + "router_z_loss_mlp": 0.22119141, + "step": 1634, + "time_per_iteration": 2.8063085079193115 + }, + { + "auxiliary_loss_clip": 0.01184158, + "auxiliary_loss_mlp": 0.01065399, + "balance_loss_clip": 1.05609524, + "balance_loss_mlp": 1.04180765, + "epoch": 0.19659712619491373, + "flos": 24242503921920.0, + "grad_norm": 1.9156635462309346, + "language_loss": 0.79361284, + "learning_rate": 3.715995118555273e-06, + "loss": 0.81610835, + "num_input_tokens_seen": 34659890, + "router_z_loss_clip": 1.28125, + "router_z_loss_mlp": 0.23583984, + "step": 1635, + "time_per_iteration": 3.619079113006592 + }, + { + "auxiliary_loss_clip": 0.0118175, + "auxiliary_loss_mlp": 0.0105292, + "balance_loss_clip": 1.05539107, + "balance_loss_mlp": 1.02658629, + "epoch": 0.19671736908555282, + "flos": 24717422568960.0, + "grad_norm": 3.229519573260421, + "language_loss": 0.85935593, + "learning_rate": 3.71559486668991e-06, + "loss": 0.88170266, + "num_input_tokens_seen": 34678750, + "router_z_loss_clip": 1.26367188, + "router_z_loss_mlp": 0.26342773, + "step": 1636, + "time_per_iteration": 3.6464264392852783 + }, + { + "auxiliary_loss_clip": 0.01178401, + "auxiliary_loss_mlp": 0.01053208, + "balance_loss_clip": 1.05493379, + "balance_loss_mlp": 1.03148794, + "epoch": 0.1968376119761919, + "flos": 23842279607040.0, + "grad_norm": 3.100713960716103, + "language_loss": 0.77226394, + "learning_rate": 3.715194354570169e-06, + "loss": 0.79457998, + "num_input_tokens_seen": 34698755, + "router_z_loss_clip": 1.23535156, + "router_z_loss_mlp": 0.21728516, + "step": 1637, + "time_per_iteration": 3.6260974407196045 + }, + { + "auxiliary_loss_clip": 0.0117285, + "auxiliary_loss_mlp": 0.0104858, + "balance_loss_clip": 1.05330682, + "balance_loss_mlp": 1.02706301, + "epoch": 0.196957854866831, + "flos": 18113917409280.0, + "grad_norm": 2.2760549021774454, + "language_loss": 0.83164847, + "learning_rate": 3.714793582256809e-06, + "loss": 0.85386282, + "num_input_tokens_seen": 34715820, + "router_z_loss_clip": 1.19433594, + "router_z_loss_mlp": 0.21520996, + "step": 1638, + "time_per_iteration": 2.771577835083008 + }, + { + "auxiliary_loss_clip": 0.01175082, + "auxiliary_loss_mlp": 0.01052683, + "balance_loss_clip": 1.05384958, + "balance_loss_mlp": 1.03166628, + "epoch": 0.1970780977574701, + "flos": 21653129312640.0, + "grad_norm": 3.046224030335483, + "language_loss": 0.85782361, + "learning_rate": 3.7143925498106253e-06, + "loss": 0.88010132, + "num_input_tokens_seen": 34734360, + "router_z_loss_clip": 1.21191406, + "router_z_loss_mlp": 0.21020508, + "step": 1639, + "time_per_iteration": 2.882814645767212 + }, + { + "auxiliary_loss_clip": 0.0117965, + "auxiliary_loss_mlp": 0.01054038, + "balance_loss_clip": 1.05266047, + "balance_loss_mlp": 1.03137612, + "epoch": 0.19719834064810918, + "flos": 20811813984000.0, + "grad_norm": 2.4447599425618622, + "language_loss": 0.78867674, + "learning_rate": 3.7139912572924558e-06, + "loss": 0.81101364, + "num_input_tokens_seen": 34753390, + "router_z_loss_clip": 1.26855469, + "router_z_loss_mlp": 0.22692871, + "step": 1640, + "time_per_iteration": 2.85827898979187 + }, + { + "auxiliary_loss_clip": 0.01176278, + "auxiliary_loss_mlp": 0.01052202, + "balance_loss_clip": 1.05090618, + "balance_loss_mlp": 1.03014874, + "epoch": 0.19731858353874826, + "flos": 23434800744960.0, + "grad_norm": 3.5475546931665276, + "language_loss": 0.8037141, + "learning_rate": 3.7135897047631744e-06, + "loss": 0.8259989, + "num_input_tokens_seen": 34771275, + "router_z_loss_clip": 1.25488281, + "router_z_loss_mlp": 0.22033691, + "step": 1641, + "time_per_iteration": 2.8957834243774414 + }, + { + "auxiliary_loss_clip": 0.01177265, + "auxiliary_loss_mlp": 0.01048714, + "balance_loss_clip": 1.05350804, + "balance_loss_mlp": 1.0271852, + "epoch": 0.19743882642938737, + "flos": 23988184652160.0, + "grad_norm": 2.6702975992100066, + "language_loss": 0.75475585, + "learning_rate": 3.713187892283698e-06, + "loss": 0.77701569, + "num_input_tokens_seen": 34790885, + "router_z_loss_clip": 1.23828125, + "router_z_loss_mlp": 0.21520996, + "step": 1642, + "time_per_iteration": 2.8723347187042236 + }, + { + "auxiliary_loss_clip": 0.01173829, + "auxiliary_loss_mlp": 0.01046211, + "balance_loss_clip": 1.05022883, + "balance_loss_mlp": 1.02495563, + "epoch": 0.19755906932002645, + "flos": 15004340081280.0, + "grad_norm": 2.521728341665001, + "language_loss": 0.87167466, + "learning_rate": 3.71278581991498e-06, + "loss": 0.893875, + "num_input_tokens_seen": 34806745, + "router_z_loss_clip": 1.23730469, + "router_z_loss_mlp": 0.21240234, + "step": 1643, + "time_per_iteration": 2.8390069007873535 + }, + { + "auxiliary_loss_clip": 0.01185649, + "auxiliary_loss_mlp": 0.01053636, + "balance_loss_clip": 1.05747032, + "balance_loss_mlp": 1.03153431, + "epoch": 0.19767931221066554, + "flos": 19494466686720.0, + "grad_norm": 2.2207937794732477, + "language_loss": 0.79179555, + "learning_rate": 3.712383487718015e-06, + "loss": 0.81418836, + "num_input_tokens_seen": 34824985, + "router_z_loss_clip": 1.28125, + "router_z_loss_mlp": 0.22106934, + "step": 1644, + "time_per_iteration": 2.823596715927124 + }, + { + "auxiliary_loss_clip": 0.01176742, + "auxiliary_loss_mlp": 0.01053655, + "balance_loss_clip": 1.0569979, + "balance_loss_mlp": 1.03273416, + "epoch": 0.19779955510130465, + "flos": 25737895958400.0, + "grad_norm": 2.7989035787274563, + "language_loss": 0.86365277, + "learning_rate": 3.7119808957538365e-06, + "loss": 0.88595676, + "num_input_tokens_seen": 34843980, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.20922852, + "step": 1645, + "time_per_iteration": 2.845745801925659 + }, + { + "auxiliary_loss_clip": 0.01179866, + "auxiliary_loss_mlp": 0.01051359, + "balance_loss_clip": 1.05257487, + "balance_loss_mlp": 1.02997291, + "epoch": 0.19791979799194373, + "flos": 20777699041920.0, + "grad_norm": 2.1644549353733264, + "language_loss": 0.80237794, + "learning_rate": 3.711578044083517e-06, + "loss": 0.8246901, + "num_input_tokens_seen": 34860780, + "router_z_loss_clip": 1.27441406, + "router_z_loss_mlp": 0.21374512, + "step": 1646, + "time_per_iteration": 2.7853777408599854 + }, + { + "auxiliary_loss_clip": 0.01173279, + "auxiliary_loss_mlp": 0.01054742, + "balance_loss_clip": 1.0519805, + "balance_loss_mlp": 1.03271222, + "epoch": 0.1980400408825828, + "flos": 25589010084480.0, + "grad_norm": 2.155967744419093, + "language_loss": 0.74482155, + "learning_rate": 3.7111749327681698e-06, + "loss": 0.76710176, + "num_input_tokens_seen": 34880815, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.22033691, + "step": 1647, + "time_per_iteration": 2.880821466445923 + }, + { + "auxiliary_loss_clip": 0.01178612, + "auxiliary_loss_mlp": 0.01054893, + "balance_loss_clip": 1.05418551, + "balance_loss_mlp": 1.0333159, + "epoch": 0.1981602837732219, + "flos": 23513840622720.0, + "grad_norm": 2.099064491146368, + "language_loss": 0.8619892, + "learning_rate": 3.7107715618689455e-06, + "loss": 0.88432425, + "num_input_tokens_seen": 34899790, + "router_z_loss_clip": 1.24414062, + "router_z_loss_mlp": 0.21557617, + "step": 1648, + "time_per_iteration": 2.920295476913452 + }, + { + "auxiliary_loss_clip": 0.01176568, + "auxiliary_loss_mlp": 0.01051994, + "balance_loss_clip": 1.05260396, + "balance_loss_mlp": 1.03114414, + "epoch": 0.198280526663861, + "flos": 23185365724800.0, + "grad_norm": 3.0763215135609236, + "language_loss": 0.83470148, + "learning_rate": 3.710367931447035e-06, + "loss": 0.85698706, + "num_input_tokens_seen": 34921570, + "router_z_loss_clip": 1.23925781, + "router_z_loss_mlp": 0.20837402, + "step": 1649, + "time_per_iteration": 2.831386089324951 + }, + { + "auxiliary_loss_clip": 0.01182538, + "auxiliary_loss_mlp": 0.01058118, + "balance_loss_clip": 1.05593085, + "balance_loss_mlp": 1.03656554, + "epoch": 0.1984007695545001, + "flos": 21689470897920.0, + "grad_norm": 2.5981318767067973, + "language_loss": 0.86367327, + "learning_rate": 3.70996404156367e-06, + "loss": 0.88607985, + "num_input_tokens_seen": 34941205, + "router_z_loss_clip": 1.26757812, + "router_z_loss_mlp": 0.2154541, + "step": 1650, + "time_per_iteration": 2.930924415588379 + }, + { + "auxiliary_loss_clip": 0.01176393, + "auxiliary_loss_mlp": 0.0105062, + "balance_loss_clip": 1.05405831, + "balance_loss_mlp": 1.02833962, + "epoch": 0.19852101244513917, + "flos": 36064008887040.0, + "grad_norm": 1.935429043099152, + "language_loss": 0.7271049, + "learning_rate": 3.7095598922801187e-06, + "loss": 0.74937505, + "num_input_tokens_seen": 34963280, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.22277832, + "step": 1651, + "time_per_iteration": 2.9389400482177734 + }, + { + "auxiliary_loss_clip": 0.01175446, + "auxiliary_loss_mlp": 0.01051816, + "balance_loss_clip": 1.05107796, + "balance_loss_mlp": 1.03041792, + "epoch": 0.19864125533577828, + "flos": 23105894883840.0, + "grad_norm": 3.00192955196961, + "language_loss": 0.7618044, + "learning_rate": 3.7091554836576914e-06, + "loss": 0.78407699, + "num_input_tokens_seen": 34979955, + "router_z_loss_clip": 1.24511719, + "router_z_loss_mlp": 0.21411133, + "step": 1652, + "time_per_iteration": 2.8241748809814453 + }, + { + "auxiliary_loss_clip": 0.01172863, + "auxiliary_loss_mlp": 0.0104841, + "balance_loss_clip": 1.05087531, + "balance_loss_mlp": 1.02639234, + "epoch": 0.19876149822641737, + "flos": 24608505553920.0, + "grad_norm": 1.8671445610322037, + "language_loss": 0.82608372, + "learning_rate": 3.708750815757736e-06, + "loss": 0.84829652, + "num_input_tokens_seen": 35000725, + "router_z_loss_clip": 1.21777344, + "router_z_loss_mlp": 0.22021484, + "step": 1653, + "time_per_iteration": 2.9389560222625732 + }, + { + "auxiliary_loss_clip": 0.01184348, + "auxiliary_loss_mlp": 0.01064903, + "balance_loss_clip": 1.05536819, + "balance_loss_mlp": 1.04096603, + "epoch": 0.19888174111705645, + "flos": 32196645308160.0, + "grad_norm": 2.295356972833561, + "language_loss": 0.73158222, + "learning_rate": 3.7083458886416407e-06, + "loss": 0.75407475, + "num_input_tokens_seen": 35019920, + "router_z_loss_clip": 1.28808594, + "router_z_loss_mlp": 0.23950195, + "step": 1654, + "time_per_iteration": 2.8696820735931396 + }, + { + "auxiliary_loss_clip": 0.01176184, + "auxiliary_loss_mlp": 0.01051856, + "balance_loss_clip": 1.05109072, + "balance_loss_mlp": 1.03054142, + "epoch": 0.19900198400769553, + "flos": 24608469640320.0, + "grad_norm": 2.3409157702949965, + "language_loss": 0.87946665, + "learning_rate": 3.707940702370832e-06, + "loss": 0.90174705, + "num_input_tokens_seen": 35040765, + "router_z_loss_clip": 1.24804688, + "router_z_loss_mlp": 0.21325684, + "step": 1655, + "time_per_iteration": 2.7955358028411865 + }, + { + "auxiliary_loss_clip": 0.01090748, + "auxiliary_loss_mlp": 0.01023926, + "balance_loss_clip": 1.03232813, + "balance_loss_mlp": 1.01989675, + "epoch": 0.19912222689833464, + "flos": 67915805673600.0, + "grad_norm": 0.7661967917860376, + "language_loss": 0.58252931, + "learning_rate": 3.707535257006777e-06, + "loss": 0.60367596, + "num_input_tokens_seen": 35106390, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.0402832, + "step": 1656, + "time_per_iteration": 3.411257266998291 + }, + { + "auxiliary_loss_clip": 0.01176969, + "auxiliary_loss_mlp": 0.01045519, + "balance_loss_clip": 1.05242598, + "balance_loss_mlp": 1.02274954, + "epoch": 0.19924246978897373, + "flos": 15742340916480.0, + "grad_norm": 2.817474083452534, + "language_loss": 0.88952732, + "learning_rate": 3.707129552610981e-06, + "loss": 0.9117521, + "num_input_tokens_seen": 35125040, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.2277832, + "step": 1657, + "time_per_iteration": 2.822389841079712 + }, + { + "auxiliary_loss_clip": 0.01177787, + "auxiliary_loss_mlp": 0.01050073, + "balance_loss_clip": 1.05331695, + "balance_loss_mlp": 1.02786446, + "epoch": 0.1993627126796128, + "flos": 17566566986880.0, + "grad_norm": 2.0113042822480645, + "language_loss": 0.73983371, + "learning_rate": 3.70672358924499e-06, + "loss": 0.76211232, + "num_input_tokens_seen": 35144280, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.2220459, + "step": 1658, + "time_per_iteration": 2.8549468517303467 + }, + { + "auxiliary_loss_clip": 0.01176536, + "auxiliary_loss_mlp": 0.01056924, + "balance_loss_clip": 1.05326223, + "balance_loss_mlp": 1.03513265, + "epoch": 0.19948295557025192, + "flos": 40843826680320.0, + "grad_norm": 2.9947060446475726, + "language_loss": 0.78397, + "learning_rate": 3.706317366970386e-06, + "loss": 0.80630457, + "num_input_tokens_seen": 35165280, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.21813965, + "step": 1659, + "time_per_iteration": 2.973583459854126 + }, + { + "auxiliary_loss_clip": 0.01178522, + "auxiliary_loss_mlp": 0.01054519, + "balance_loss_clip": 1.05078661, + "balance_loss_mlp": 1.03189337, + "epoch": 0.199603198460891, + "flos": 25082418620160.0, + "grad_norm": 2.097288085439466, + "language_loss": 0.83933187, + "learning_rate": 3.705910885848795e-06, + "loss": 0.86166233, + "num_input_tokens_seen": 35183655, + "router_z_loss_clip": 1.27929688, + "router_z_loss_mlp": 0.22619629, + "step": 1660, + "time_per_iteration": 2.831862449645996 + }, + { + "auxiliary_loss_clip": 0.0117599, + "auxiliary_loss_mlp": 0.01049478, + "balance_loss_clip": 1.05151725, + "balance_loss_mlp": 1.02780604, + "epoch": 0.19972344135153008, + "flos": 20084120352000.0, + "grad_norm": 2.294397661234237, + "language_loss": 0.84916776, + "learning_rate": 3.705504145941879e-06, + "loss": 0.87142241, + "num_input_tokens_seen": 35201825, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.21679688, + "step": 1661, + "time_per_iteration": 2.8690402507781982 + }, + { + "auxiliary_loss_clip": 0.01179144, + "auxiliary_loss_mlp": 0.01055225, + "balance_loss_clip": 1.05311394, + "balance_loss_mlp": 1.03282511, + "epoch": 0.1998436842421692, + "flos": 23727472761600.0, + "grad_norm": 2.4154315668252657, + "language_loss": 0.78487325, + "learning_rate": 3.7050971473113403e-06, + "loss": 0.80721688, + "num_input_tokens_seen": 35221600, + "router_z_loss_clip": 1.26171875, + "router_z_loss_mlp": 0.22399902, + "step": 1662, + "time_per_iteration": 2.7922146320343018 + }, + { + "auxiliary_loss_clip": 0.01178446, + "auxiliary_loss_mlp": 0.01044569, + "balance_loss_clip": 1.05341673, + "balance_loss_mlp": 1.02342141, + "epoch": 0.19996392713280828, + "flos": 36102361633920.0, + "grad_norm": 1.9101294169104612, + "language_loss": 0.80098808, + "learning_rate": 3.7046898900189196e-06, + "loss": 0.82321823, + "num_input_tokens_seen": 35245935, + "router_z_loss_clip": 1.25, + "router_z_loss_mlp": 0.21130371, + "step": 1663, + "time_per_iteration": 3.721452474594116 + }, + { + "auxiliary_loss_clip": 0.01183014, + "auxiliary_loss_mlp": 0.0105206, + "balance_loss_clip": 1.05668974, + "balance_loss_mlp": 1.028754, + "epoch": 0.20008417002344736, + "flos": 23657662679040.0, + "grad_norm": 2.8644891563681387, + "language_loss": 0.8292141, + "learning_rate": 3.704282374126398e-06, + "loss": 0.85156482, + "num_input_tokens_seen": 35265615, + "router_z_loss_clip": 1.26367188, + "router_z_loss_mlp": 0.23327637, + "step": 1664, + "time_per_iteration": 3.637136697769165 + }, + { + "auxiliary_loss_clip": 0.01184419, + "auxiliary_loss_mlp": 0.01058158, + "balance_loss_clip": 1.05544424, + "balance_loss_mlp": 1.03516209, + "epoch": 0.20020441291408644, + "flos": 21872076664320.0, + "grad_norm": 4.423584459974016, + "language_loss": 0.87655318, + "learning_rate": 3.7038745996955954e-06, + "loss": 0.89897895, + "num_input_tokens_seen": 35284960, + "router_z_loss_clip": 1.29101562, + "router_z_loss_mlp": 0.23022461, + "step": 1665, + "time_per_iteration": 3.696326971054077 + }, + { + "auxiliary_loss_clip": 0.01186226, + "auxiliary_loss_mlp": 0.01053635, + "balance_loss_clip": 1.05528164, + "balance_loss_mlp": 1.03170037, + "epoch": 0.20032465580472555, + "flos": 23179691376000.0, + "grad_norm": 2.8306287794553016, + "language_loss": 0.72420704, + "learning_rate": 3.703466566788371e-06, + "loss": 0.74660563, + "num_input_tokens_seen": 35304090, + "router_z_loss_clip": 1.30957031, + "router_z_loss_mlp": 0.21936035, + "step": 1666, + "time_per_iteration": 3.6655995845794678 + }, + { + "auxiliary_loss_clip": 0.01178045, + "auxiliary_loss_mlp": 0.01052574, + "balance_loss_clip": 1.05300045, + "balance_loss_mlp": 1.02818346, + "epoch": 0.20044489869536464, + "flos": 23873521461120.0, + "grad_norm": 3.2788475677881186, + "language_loss": 0.74471742, + "learning_rate": 3.703058275466622e-06, + "loss": 0.76702362, + "num_input_tokens_seen": 35323325, + "router_z_loss_clip": 1.25097656, + "router_z_loss_mlp": 0.24377441, + "step": 1667, + "time_per_iteration": 2.9068260192871094 + }, + { + "auxiliary_loss_clip": 0.01176751, + "auxiliary_loss_mlp": 0.01042605, + "balance_loss_clip": 1.05370128, + "balance_loss_mlp": 1.02152872, + "epoch": 0.20056514158600372, + "flos": 21945226711680.0, + "grad_norm": 2.1610163380226184, + "language_loss": 0.77510297, + "learning_rate": 3.7026497257922877e-06, + "loss": 0.79729652, + "num_input_tokens_seen": 35343635, + "router_z_loss_clip": 1.22949219, + "router_z_loss_mlp": 0.21069336, + "step": 1668, + "time_per_iteration": 2.85569167137146 + }, + { + "auxiliary_loss_clip": 0.01178855, + "auxiliary_loss_mlp": 0.01053939, + "balance_loss_clip": 1.05390704, + "balance_loss_mlp": 1.03264821, + "epoch": 0.20068538447664283, + "flos": 23879159896320.0, + "grad_norm": 1.730754506899425, + "language_loss": 0.85226625, + "learning_rate": 3.7022409178273436e-06, + "loss": 0.87459421, + "num_input_tokens_seen": 35364615, + "router_z_loss_clip": 1.24707031, + "router_z_loss_mlp": 0.2130127, + "step": 1669, + "time_per_iteration": 2.835139036178589 + }, + { + "auxiliary_loss_clip": 0.01178761, + "auxiliary_loss_mlp": 0.01045098, + "balance_loss_clip": 1.05272722, + "balance_loss_mlp": 1.02386665, + "epoch": 0.2008056273672819, + "flos": 18442823270400.0, + "grad_norm": 2.7759131929716974, + "language_loss": 0.78601134, + "learning_rate": 3.7018318516338054e-06, + "loss": 0.80824995, + "num_input_tokens_seen": 35383775, + "router_z_loss_clip": 1.25976562, + "router_z_loss_mlp": 0.21240234, + "step": 1670, + "time_per_iteration": 2.8319108486175537 + }, + { + "auxiliary_loss_clip": 0.01178526, + "auxiliary_loss_mlp": 0.01048914, + "balance_loss_clip": 1.05444551, + "balance_loss_mlp": 1.02810025, + "epoch": 0.200925870257921, + "flos": 23659530186240.0, + "grad_norm": 3.2990403844375824, + "language_loss": 0.81678522, + "learning_rate": 3.7014225272737284e-06, + "loss": 0.83905959, + "num_input_tokens_seen": 35403000, + "router_z_loss_clip": 1.2421875, + "router_z_loss_mlp": 0.20825195, + "step": 1671, + "time_per_iteration": 2.8369526863098145 + }, + { + "auxiliary_loss_clip": 0.01181696, + "auxiliary_loss_mlp": 0.01047185, + "balance_loss_clip": 1.0563513, + "balance_loss_mlp": 1.02509534, + "epoch": 0.20104611314856008, + "flos": 16217115909120.0, + "grad_norm": 2.9916888113120628, + "language_loss": 0.74043906, + "learning_rate": 3.701012944809207e-06, + "loss": 0.76272786, + "num_input_tokens_seen": 35420115, + "router_z_loss_clip": 1.25292969, + "router_z_loss_mlp": 0.22094727, + "step": 1672, + "time_per_iteration": 2.8625214099884033 + }, + { + "auxiliary_loss_clip": 0.01177832, + "auxiliary_loss_mlp": 0.0106075, + "balance_loss_clip": 1.05272889, + "balance_loss_mlp": 1.03923297, + "epoch": 0.2011663560391992, + "flos": 21397373498880.0, + "grad_norm": 2.1186225698914907, + "language_loss": 0.79428685, + "learning_rate": 3.700603104302374e-06, + "loss": 0.81667268, + "num_input_tokens_seen": 35439925, + "router_z_loss_clip": 1.25097656, + "router_z_loss_mlp": 0.21520996, + "step": 1673, + "time_per_iteration": 2.8850889205932617 + }, + { + "auxiliary_loss_clip": 0.0110117, + "auxiliary_loss_mlp": 0.01019721, + "balance_loss_clip": 1.04090691, + "balance_loss_mlp": 1.0160737, + "epoch": 0.20128659892983827, + "flos": 62229459409920.0, + "grad_norm": 0.9093626064791194, + "language_loss": 0.55958825, + "learning_rate": 3.7001930058154027e-06, + "loss": 0.5807972, + "num_input_tokens_seen": 35504885, + "router_z_loss_clip": 0.6015625, + "router_z_loss_mlp": 0.03637695, + "step": 1674, + "time_per_iteration": 3.3705222606658936 + }, + { + "auxiliary_loss_clip": 0.01182538, + "auxiliary_loss_mlp": 0.0105293, + "balance_loss_clip": 1.05463767, + "balance_loss_mlp": 1.02973199, + "epoch": 0.20140684182047736, + "flos": 28438737448320.0, + "grad_norm": 2.967701264593375, + "language_loss": 0.79494405, + "learning_rate": 3.6997826494105037e-06, + "loss": 0.81729871, + "num_input_tokens_seen": 35525330, + "router_z_loss_clip": 1.27832031, + "router_z_loss_mlp": 0.23205566, + "step": 1675, + "time_per_iteration": 2.809213876724243 + }, + { + "auxiliary_loss_clip": 0.01176055, + "auxiliary_loss_mlp": 0.01042945, + "balance_loss_clip": 1.05162644, + "balance_loss_mlp": 1.02209496, + "epoch": 0.20152708471111647, + "flos": 28074064619520.0, + "grad_norm": 2.892498958981897, + "language_loss": 0.69745803, + "learning_rate": 3.6993720351499286e-06, + "loss": 0.719648, + "num_input_tokens_seen": 35546455, + "router_z_loss_clip": 1.24316406, + "router_z_loss_mlp": 0.20837402, + "step": 1676, + "time_per_iteration": 2.8546884059906006 + }, + { + "auxiliary_loss_clip": 0.01177918, + "auxiliary_loss_mlp": 0.01055837, + "balance_loss_clip": 1.05586982, + "balance_loss_mlp": 1.03291321, + "epoch": 0.20164732760175555, + "flos": 23549751244800.0, + "grad_norm": 1.947702645107485, + "language_loss": 0.76923418, + "learning_rate": 3.6989611630959666e-06, + "loss": 0.79157174, + "num_input_tokens_seen": 35565010, + "router_z_loss_clip": 1.22070312, + "router_z_loss_mlp": 0.22937012, + "step": 1677, + "time_per_iteration": 2.7624917030334473 + }, + { + "auxiliary_loss_clip": 0.01097994, + "auxiliary_loss_mlp": 0.01003189, + "balance_loss_clip": 1.03864956, + "balance_loss_mlp": 0.99942189, + "epoch": 0.20176757049239463, + "flos": 71100616037760.0, + "grad_norm": 0.6818444987404266, + "language_loss": 0.58268732, + "learning_rate": 3.6985500333109474e-06, + "loss": 0.60369915, + "num_input_tokens_seen": 35633340, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.03759766, + "step": 1678, + "time_per_iteration": 3.4034409523010254 + }, + { + "auxiliary_loss_clip": 0.0117846, + "auxiliary_loss_mlp": 0.01041126, + "balance_loss_clip": 1.05565929, + "balance_loss_mlp": 1.01822591, + "epoch": 0.20188781338303372, + "flos": 21430159637760.0, + "grad_norm": 2.598786957047838, + "language_loss": 0.76825964, + "learning_rate": 3.6981386458572385e-06, + "loss": 0.79045546, + "num_input_tokens_seen": 35651315, + "router_z_loss_clip": 1.22753906, + "router_z_loss_mlp": 0.22888184, + "step": 1679, + "time_per_iteration": 2.869328498840332 + }, + { + "auxiliary_loss_clip": 0.01179022, + "auxiliary_loss_mlp": 0.01053321, + "balance_loss_clip": 1.05213726, + "balance_loss_mlp": 1.02976561, + "epoch": 0.20200805627367283, + "flos": 11546215130880.0, + "grad_norm": 2.9968948106150477, + "language_loss": 0.76115716, + "learning_rate": 3.6977270007972468e-06, + "loss": 0.78348058, + "num_input_tokens_seen": 35668850, + "router_z_loss_clip": 1.26855469, + "router_z_loss_mlp": 0.2355957, + "step": 1680, + "time_per_iteration": 2.846592903137207 + }, + { + "auxiliary_loss_clip": 0.01174101, + "auxiliary_loss_mlp": 0.01052815, + "balance_loss_clip": 1.05174708, + "balance_loss_mlp": 1.03144133, + "epoch": 0.2021282991643119, + "flos": 28545391906560.0, + "grad_norm": 2.6321974787776483, + "language_loss": 0.72664976, + "learning_rate": 3.6973150981934196e-06, + "loss": 0.74891889, + "num_input_tokens_seen": 35690080, + "router_z_loss_clip": 1.22363281, + "router_z_loss_mlp": 0.21386719, + "step": 1681, + "time_per_iteration": 2.877919912338257 + }, + { + "auxiliary_loss_clip": 0.0118217, + "auxiliary_loss_mlp": 0.01044274, + "balance_loss_clip": 1.05435634, + "balance_loss_mlp": 1.02156496, + "epoch": 0.202248542054951, + "flos": 17923446564480.0, + "grad_norm": 2.971837323934504, + "language_loss": 0.83171409, + "learning_rate": 3.6969029381082415e-06, + "loss": 0.85397851, + "num_input_tokens_seen": 35706075, + "router_z_loss_clip": 1.27929688, + "router_z_loss_mlp": 0.22717285, + "step": 1682, + "time_per_iteration": 2.749774217605591 + }, + { + "auxiliary_loss_clip": 0.01169519, + "auxiliary_loss_mlp": 0.01054138, + "balance_loss_clip": 1.05160046, + "balance_loss_mlp": 1.03334761, + "epoch": 0.2023687849455901, + "flos": 19864634296320.0, + "grad_norm": 3.031328025192821, + "language_loss": 0.79533279, + "learning_rate": 3.696490520604237e-06, + "loss": 0.81756938, + "num_input_tokens_seen": 35724765, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.20788574, + "step": 1683, + "time_per_iteration": 2.7939939498901367 + }, + { + "auxiliary_loss_clip": 0.01170756, + "auxiliary_loss_mlp": 0.01047111, + "balance_loss_clip": 1.05061376, + "balance_loss_mlp": 1.02673805, + "epoch": 0.20248902783622919, + "flos": 22564721600640.0, + "grad_norm": 2.0798945192435747, + "language_loss": 0.80520719, + "learning_rate": 3.696077845743968e-06, + "loss": 0.8273859, + "num_input_tokens_seen": 35744355, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.20385742, + "step": 1684, + "time_per_iteration": 2.7973058223724365 + }, + { + "auxiliary_loss_clip": 0.01178309, + "auxiliary_loss_mlp": 0.01051279, + "balance_loss_clip": 1.05069566, + "balance_loss_mlp": 1.02729428, + "epoch": 0.20260927072686827, + "flos": 22709728805760.0, + "grad_norm": 2.695010498448091, + "language_loss": 0.73062855, + "learning_rate": 3.69566491359004e-06, + "loss": 0.75292438, + "num_input_tokens_seen": 35761000, + "router_z_loss_clip": 1.27734375, + "router_z_loss_mlp": 0.2401123, + "step": 1685, + "time_per_iteration": 2.9406824111938477 + }, + { + "auxiliary_loss_clip": 0.01176721, + "auxiliary_loss_mlp": 0.010582, + "balance_loss_clip": 1.05265903, + "balance_loss_mlp": 1.03571749, + "epoch": 0.20272951361750738, + "flos": 51023998650240.0, + "grad_norm": 12.136307179060593, + "language_loss": 0.6904521, + "learning_rate": 3.695251724205092e-06, + "loss": 0.71280128, + "num_input_tokens_seen": 35785360, + "router_z_loss_clip": 1.23828125, + "router_z_loss_mlp": 0.22473145, + "step": 1686, + "time_per_iteration": 3.0212209224700928 + }, + { + "auxiliary_loss_clip": 0.01171843, + "auxiliary_loss_mlp": 0.01053406, + "balance_loss_clip": 1.04910398, + "balance_loss_mlp": 1.03209138, + "epoch": 0.20284975650814646, + "flos": 26578133879040.0, + "grad_norm": 1.6765073233553642, + "language_loss": 0.86328316, + "learning_rate": 3.6948382776518054e-06, + "loss": 0.88553572, + "num_input_tokens_seen": 35806065, + "router_z_loss_clip": 1.22753906, + "router_z_loss_mlp": 0.2130127, + "step": 1687, + "time_per_iteration": 2.8298490047454834 + }, + { + "auxiliary_loss_clip": 0.01176023, + "auxiliary_loss_mlp": 0.01050205, + "balance_loss_clip": 1.05145955, + "balance_loss_mlp": 1.02825809, + "epoch": 0.20296999939878554, + "flos": 16034222833920.0, + "grad_norm": 3.072994777499447, + "language_loss": 0.7971549, + "learning_rate": 3.6944245739929e-06, + "loss": 0.81941718, + "num_input_tokens_seen": 35822225, + "router_z_loss_clip": 1.24511719, + "router_z_loss_mlp": 0.21936035, + "step": 1688, + "time_per_iteration": 2.81939435005188 + }, + { + "auxiliary_loss_clip": 0.01178399, + "auxiliary_loss_mlp": 0.01051241, + "balance_loss_clip": 1.05414438, + "balance_loss_mlp": 1.0276612, + "epoch": 0.20309024228942463, + "flos": 19203374868480.0, + "grad_norm": 3.039216612910752, + "language_loss": 0.71946681, + "learning_rate": 3.6940106132911332e-06, + "loss": 0.74176317, + "num_input_tokens_seen": 35839410, + "router_z_loss_clip": 1.24316406, + "router_z_loss_mlp": 0.23596191, + "step": 1689, + "time_per_iteration": 2.7803258895874023 + }, + { + "auxiliary_loss_clip": 0.01177816, + "auxiliary_loss_mlp": 0.01051225, + "balance_loss_clip": 1.05498958, + "balance_loss_mlp": 1.03017235, + "epoch": 0.20321048518006374, + "flos": 22821087945600.0, + "grad_norm": 6.008782458058675, + "language_loss": 0.88968283, + "learning_rate": 3.6935963956093037e-06, + "loss": 0.91197324, + "num_input_tokens_seen": 35859495, + "router_z_loss_clip": 1.22851562, + "router_z_loss_mlp": 0.21044922, + "step": 1690, + "time_per_iteration": 2.845179796218872 + }, + { + "auxiliary_loss_clip": 0.01174219, + "auxiliary_loss_mlp": 0.01045985, + "balance_loss_clip": 1.05476451, + "balance_loss_mlp": 1.02506375, + "epoch": 0.20333072807070282, + "flos": 19096397187840.0, + "grad_norm": 2.1520820663317086, + "language_loss": 0.68453968, + "learning_rate": 3.6931819210102474e-06, + "loss": 0.70674175, + "num_input_tokens_seen": 35878890, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.20910645, + "step": 1691, + "time_per_iteration": 2.860400438308716 + }, + { + "auxiliary_loss_clip": 0.01177251, + "auxiliary_loss_mlp": 0.01052696, + "balance_loss_clip": 1.05303288, + "balance_loss_mlp": 1.03027272, + "epoch": 0.2034509709613419, + "flos": 18180962144640.0, + "grad_norm": 1.9612722718850866, + "language_loss": 0.84228307, + "learning_rate": 3.6927671895568402e-06, + "loss": 0.86458254, + "num_input_tokens_seen": 35897950, + "router_z_loss_clip": 1.2421875, + "router_z_loss_mlp": 0.22436523, + "step": 1692, + "time_per_iteration": 3.861725091934204 + }, + { + "auxiliary_loss_clip": 0.01174234, + "auxiliary_loss_mlp": 0.01049349, + "balance_loss_clip": 1.05388236, + "balance_loss_mlp": 1.02625835, + "epoch": 0.20357121385198101, + "flos": 22923899648640.0, + "grad_norm": 2.2636550892834055, + "language_loss": 0.86912477, + "learning_rate": 3.692352201311996e-06, + "loss": 0.89136058, + "num_input_tokens_seen": 35916800, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.23071289, + "step": 1693, + "time_per_iteration": 3.6239373683929443 + }, + { + "auxiliary_loss_clip": 0.0117178, + "auxiliary_loss_mlp": 0.01047045, + "balance_loss_clip": 1.05121493, + "balance_loss_mlp": 1.02500343, + "epoch": 0.2036914567426201, + "flos": 20922131629440.0, + "grad_norm": 2.592743906298334, + "language_loss": 0.76704693, + "learning_rate": 3.6919369563386687e-06, + "loss": 0.78923523, + "num_input_tokens_seen": 35936600, + "router_z_loss_clip": 1.20703125, + "router_z_loss_mlp": 0.22045898, + "step": 1694, + "time_per_iteration": 3.657984972000122 + }, + { + "auxiliary_loss_clip": 0.01173703, + "auxiliary_loss_mlp": 0.01049399, + "balance_loss_clip": 1.05423355, + "balance_loss_mlp": 1.02662992, + "epoch": 0.20381169963325918, + "flos": 15519155760000.0, + "grad_norm": 2.176672312290422, + "language_loss": 0.79192674, + "learning_rate": 3.69152145469985e-06, + "loss": 0.81415778, + "num_input_tokens_seen": 35953645, + "router_z_loss_clip": 1.19335938, + "router_z_loss_mlp": 0.22766113, + "step": 1695, + "time_per_iteration": 3.5961549282073975 + }, + { + "auxiliary_loss_clip": 0.01181043, + "auxiliary_loss_mlp": 0.01065159, + "balance_loss_clip": 1.05582929, + "balance_loss_mlp": 1.04138851, + "epoch": 0.20393194252389826, + "flos": 28833143760000.0, + "grad_norm": 2.7096235709322234, + "language_loss": 0.817581, + "learning_rate": 3.691105696458572e-06, + "loss": 0.84004301, + "num_input_tokens_seen": 35970940, + "router_z_loss_clip": 1.25292969, + "router_z_loss_mlp": 0.23791504, + "step": 1696, + "time_per_iteration": 2.811199188232422 + }, + { + "auxiliary_loss_clip": 0.01175882, + "auxiliary_loss_mlp": 0.01050175, + "balance_loss_clip": 1.05566061, + "balance_loss_mlp": 1.02942097, + "epoch": 0.20405218541453737, + "flos": 22488554810880.0, + "grad_norm": 3.539007786147172, + "language_loss": 0.67817092, + "learning_rate": 3.690689681677904e-06, + "loss": 0.70043147, + "num_input_tokens_seen": 35989410, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.2076416, + "step": 1697, + "time_per_iteration": 2.7700769901275635 + }, + { + "auxiliary_loss_clip": 0.01173566, + "auxiliary_loss_mlp": 0.01054316, + "balance_loss_clip": 1.05310369, + "balance_loss_mlp": 1.03188086, + "epoch": 0.20417242830517646, + "flos": 25374408278400.0, + "grad_norm": 2.966941938216418, + "language_loss": 0.88769865, + "learning_rate": 3.690273410420956e-06, + "loss": 0.90997744, + "num_input_tokens_seen": 36009175, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.22436523, + "step": 1698, + "time_per_iteration": 2.798524856567383 + }, + { + "auxiliary_loss_clip": 0.01175934, + "auxiliary_loss_mlp": 0.01047637, + "balance_loss_clip": 1.05253398, + "balance_loss_mlp": 1.02601194, + "epoch": 0.20429267119581554, + "flos": 14793078240000.0, + "grad_norm": 3.576451985134146, + "language_loss": 0.7629953, + "learning_rate": 3.689856882750875e-06, + "loss": 0.78523099, + "num_input_tokens_seen": 36024375, + "router_z_loss_clip": 1.234375, + "router_z_loss_mlp": 0.21618652, + "step": 1699, + "time_per_iteration": 2.723285675048828 + }, + { + "auxiliary_loss_clip": 0.01171591, + "auxiliary_loss_mlp": 0.01047645, + "balance_loss_clip": 1.05444169, + "balance_loss_mlp": 1.02723658, + "epoch": 0.20441291408645465, + "flos": 17781851151360.0, + "grad_norm": 2.048217636377074, + "language_loss": 0.78441441, + "learning_rate": 3.6894400987308486e-06, + "loss": 0.80660677, + "num_input_tokens_seen": 36041895, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.20410156, + "step": 1700, + "time_per_iteration": 2.8723716735839844 + }, + { + "auxiliary_loss_clip": 0.01180749, + "auxiliary_loss_mlp": 0.01053146, + "balance_loss_clip": 1.05620813, + "balance_loss_mlp": 1.03115165, + "epoch": 0.20453315697709373, + "flos": 16435668211200.0, + "grad_norm": 2.2331799300995288, + "language_loss": 0.84738421, + "learning_rate": 3.6890230584241024e-06, + "loss": 0.86972314, + "num_input_tokens_seen": 36058825, + "router_z_loss_clip": 1.24511719, + "router_z_loss_mlp": 0.21984863, + "step": 1701, + "time_per_iteration": 2.7696099281311035 + }, + { + "auxiliary_loss_clip": 0.01089574, + "auxiliary_loss_mlp": 0.0100407, + "balance_loss_clip": 1.03136086, + "balance_loss_mlp": 0.99977893, + "epoch": 0.20465339986773282, + "flos": 66713085653760.0, + "grad_norm": 1.0797303287337938, + "language_loss": 0.66430455, + "learning_rate": 3.6886057618939016e-06, + "loss": 0.68524098, + "num_input_tokens_seen": 36121645, + "router_z_loss_clip": 0.58203125, + "router_z_loss_mlp": 0.04296875, + "step": 1702, + "time_per_iteration": 3.5179624557495117 + }, + { + "auxiliary_loss_clip": 0.01175445, + "auxiliary_loss_mlp": 0.01055509, + "balance_loss_clip": 1.05420089, + "balance_loss_mlp": 1.03414714, + "epoch": 0.2047736427583719, + "flos": 41974114924800.0, + "grad_norm": 2.2857404319455363, + "language_loss": 0.69429481, + "learning_rate": 3.6881882092035492e-06, + "loss": 0.71660435, + "num_input_tokens_seen": 36143030, + "router_z_loss_clip": 1.21191406, + "router_z_loss_mlp": 0.21374512, + "step": 1703, + "time_per_iteration": 3.062574625015259 + }, + { + "auxiliary_loss_clip": 0.01087536, + "auxiliary_loss_mlp": 0.01012528, + "balance_loss_clip": 1.02941716, + "balance_loss_mlp": 1.00830793, + "epoch": 0.204893885649011, + "flos": 69940878641280.0, + "grad_norm": 0.937606178387653, + "language_loss": 0.61253703, + "learning_rate": 3.6877704004163873e-06, + "loss": 0.63353765, + "num_input_tokens_seen": 36203435, + "router_z_loss_clip": 0.58203125, + "router_z_loss_mlp": 0.04223633, + "step": 1704, + "time_per_iteration": 3.495588779449463 + }, + { + "auxiliary_loss_clip": 0.0117949, + "auxiliary_loss_mlp": 0.01052692, + "balance_loss_clip": 1.05606151, + "balance_loss_mlp": 1.03299844, + "epoch": 0.2050141285396501, + "flos": 22200012858240.0, + "grad_norm": 2.140414114863707, + "language_loss": 0.77710915, + "learning_rate": 3.6873523355957984e-06, + "loss": 0.79943097, + "num_input_tokens_seen": 36222435, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.19714355, + "step": 1705, + "time_per_iteration": 2.813588857650757 + }, + { + "auxiliary_loss_clip": 0.01085709, + "auxiliary_loss_mlp": 0.01010097, + "balance_loss_clip": 1.02821255, + "balance_loss_mlp": 1.00594842, + "epoch": 0.20513437143028918, + "flos": 46283721730560.0, + "grad_norm": 0.9956191311371014, + "language_loss": 0.64061552, + "learning_rate": 3.686934014805201e-06, + "loss": 0.66157359, + "num_input_tokens_seen": 36273065, + "router_z_loss_clip": 0.57421875, + "router_z_loss_mlp": 0.04150391, + "step": 1706, + "time_per_iteration": 3.1648600101470947 + }, + { + "auxiliary_loss_clip": 0.01178274, + "auxiliary_loss_mlp": 0.01062282, + "balance_loss_clip": 1.05339861, + "balance_loss_mlp": 1.03993058, + "epoch": 0.20525461432092829, + "flos": 21904324099200.0, + "grad_norm": 2.380445713696863, + "language_loss": 0.8078469, + "learning_rate": 3.6865154381080552e-06, + "loss": 0.83025253, + "num_input_tokens_seen": 36293750, + "router_z_loss_clip": 1.24804688, + "router_z_loss_mlp": 0.22375488, + "step": 1707, + "time_per_iteration": 2.86378812789917 + }, + { + "auxiliary_loss_clip": 0.01182363, + "auxiliary_loss_mlp": 0.01056727, + "balance_loss_clip": 1.05752587, + "balance_loss_mlp": 1.03462553, + "epoch": 0.20537485721156737, + "flos": 21214264942080.0, + "grad_norm": 2.6328247612185423, + "language_loss": 0.82788891, + "learning_rate": 3.6860966055678585e-06, + "loss": 0.85027981, + "num_input_tokens_seen": 36310105, + "router_z_loss_clip": 1.24804688, + "router_z_loss_mlp": 0.22106934, + "step": 1708, + "time_per_iteration": 2.8303325176239014 + }, + { + "auxiliary_loss_clip": 0.01178469, + "auxiliary_loss_mlp": 0.01061239, + "balance_loss_clip": 1.05358863, + "balance_loss_mlp": 1.03966212, + "epoch": 0.20549510010220645, + "flos": 20191205773440.0, + "grad_norm": 2.016496835229542, + "language_loss": 0.86625612, + "learning_rate": 3.685677517248147e-06, + "loss": 0.88865316, + "num_input_tokens_seen": 36328995, + "router_z_loss_clip": 1.24707031, + "router_z_loss_mlp": 0.21594238, + "step": 1709, + "time_per_iteration": 2.8116397857666016 + }, + { + "auxiliary_loss_clip": 0.01178625, + "auxiliary_loss_mlp": 0.01053048, + "balance_loss_clip": 1.05684626, + "balance_loss_mlp": 1.03284228, + "epoch": 0.20561534299284553, + "flos": 17016702612480.0, + "grad_norm": 2.482193726579428, + "language_loss": 0.80443913, + "learning_rate": 3.6852581732124967e-06, + "loss": 0.82675582, + "num_input_tokens_seen": 36346340, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.20202637, + "step": 1710, + "time_per_iteration": 2.8216402530670166 + }, + { + "auxiliary_loss_clip": 0.01180247, + "auxiliary_loss_mlp": 0.0105378, + "balance_loss_clip": 1.05530953, + "balance_loss_mlp": 1.02977073, + "epoch": 0.20573558588348465, + "flos": 22890467064960.0, + "grad_norm": 2.767813917280227, + "language_loss": 0.76195633, + "learning_rate": 3.6848385735245213e-06, + "loss": 0.78429663, + "num_input_tokens_seen": 36365430, + "router_z_loss_clip": 1.25, + "router_z_loss_mlp": 0.2401123, + "step": 1711, + "time_per_iteration": 2.864773988723755 + }, + { + "auxiliary_loss_clip": 0.0117359, + "auxiliary_loss_mlp": 0.01051835, + "balance_loss_clip": 1.0537219, + "balance_loss_mlp": 1.0305922, + "epoch": 0.20585582877412373, + "flos": 24643123286400.0, + "grad_norm": 2.5551172600447187, + "language_loss": 0.86263406, + "learning_rate": 3.6844187182478734e-06, + "loss": 0.88488829, + "num_input_tokens_seen": 36386285, + "router_z_loss_clip": 1.20019531, + "router_z_loss_mlp": 0.21252441, + "step": 1712, + "time_per_iteration": 2.8864729404449463 + }, + { + "auxiliary_loss_clip": 0.01176315, + "auxiliary_loss_mlp": 0.01047116, + "balance_loss_clip": 1.05345941, + "balance_loss_mlp": 1.02472878, + "epoch": 0.2059760716647628, + "flos": 24206952435840.0, + "grad_norm": 4.111524325801285, + "language_loss": 0.75197303, + "learning_rate": 3.683998607446246e-06, + "loss": 0.77420729, + "num_input_tokens_seen": 36404935, + "router_z_loss_clip": 1.22851562, + "router_z_loss_mlp": 0.22387695, + "step": 1713, + "time_per_iteration": 2.7827916145324707 + }, + { + "auxiliary_loss_clip": 0.01174771, + "auxiliary_loss_mlp": 0.01049342, + "balance_loss_clip": 1.05492485, + "balance_loss_mlp": 1.02864742, + "epoch": 0.20609631455540192, + "flos": 20229522606720.0, + "grad_norm": 2.622238205473294, + "language_loss": 0.75170827, + "learning_rate": 3.6835782411833686e-06, + "loss": 0.77394938, + "num_input_tokens_seen": 36424455, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.20690918, + "step": 1714, + "time_per_iteration": 2.80721116065979 + }, + { + "auxiliary_loss_clip": 0.01174866, + "auxiliary_loss_mlp": 0.01051652, + "balance_loss_clip": 1.05623579, + "balance_loss_mlp": 1.03012276, + "epoch": 0.206216557446041, + "flos": 19864957518720.0, + "grad_norm": 3.244793558890574, + "language_loss": 0.74303824, + "learning_rate": 3.68315761952301e-06, + "loss": 0.76530337, + "num_input_tokens_seen": 36441685, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.21533203, + "step": 1715, + "time_per_iteration": 2.8091213703155518 + }, + { + "auxiliary_loss_clip": 0.01175177, + "auxiliary_loss_mlp": 0.01053446, + "balance_loss_clip": 1.05246043, + "balance_loss_mlp": 1.03220248, + "epoch": 0.2063368003366801, + "flos": 24096311568000.0, + "grad_norm": 2.510946090534771, + "language_loss": 0.83201468, + "learning_rate": 3.6827367425289797e-06, + "loss": 0.85430092, + "num_input_tokens_seen": 36461460, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.21240234, + "step": 1716, + "time_per_iteration": 2.8068413734436035 + }, + { + "auxiliary_loss_clip": 0.01179267, + "auxiliary_loss_mlp": 0.01053954, + "balance_loss_clip": 1.0540998, + "balance_loss_mlp": 1.03143549, + "epoch": 0.2064570432273192, + "flos": 20340163474560.0, + "grad_norm": 2.891891338048733, + "language_loss": 0.72912192, + "learning_rate": 3.6823156102651225e-06, + "loss": 0.75145411, + "num_input_tokens_seen": 36479615, + "router_z_loss_clip": 1.25097656, + "router_z_loss_mlp": 0.22497559, + "step": 1717, + "time_per_iteration": 2.7887117862701416 + }, + { + "auxiliary_loss_clip": 0.0117752, + "auxiliary_loss_mlp": 0.01045465, + "balance_loss_clip": 1.05309272, + "balance_loss_mlp": 1.02319646, + "epoch": 0.20657728611795828, + "flos": 20520363029760.0, + "grad_norm": 2.328973062389528, + "language_loss": 0.71270049, + "learning_rate": 3.6818942227953257e-06, + "loss": 0.7349304, + "num_input_tokens_seen": 36500160, + "router_z_loss_clip": 1.24511719, + "router_z_loss_mlp": 0.22290039, + "step": 1718, + "time_per_iteration": 2.7736711502075195 + }, + { + "auxiliary_loss_clip": 0.0117504, + "auxiliary_loss_mlp": 0.01049832, + "balance_loss_clip": 1.05381155, + "balance_loss_mlp": 1.02854156, + "epoch": 0.20669752900859736, + "flos": 21799285752960.0, + "grad_norm": 2.21891255307603, + "language_loss": 0.69131279, + "learning_rate": 3.681472580183512e-06, + "loss": 0.71356153, + "num_input_tokens_seen": 36518810, + "router_z_loss_clip": 1.2109375, + "router_z_loss_mlp": 0.2130127, + "step": 1719, + "time_per_iteration": 2.778719902038574 + }, + { + "auxiliary_loss_clip": 0.01173203, + "auxiliary_loss_mlp": 0.01048519, + "balance_loss_clip": 1.05177307, + "balance_loss_mlp": 1.02647769, + "epoch": 0.20681777189923645, + "flos": 15122020014720.0, + "grad_norm": 1.9418690173565214, + "language_loss": 0.86134183, + "learning_rate": 3.6810506824936455e-06, + "loss": 0.88355899, + "num_input_tokens_seen": 36536890, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.22021484, + "step": 1720, + "time_per_iteration": 2.770219326019287 + }, + { + "auxiliary_loss_clip": 0.01106517, + "auxiliary_loss_mlp": 0.01012653, + "balance_loss_clip": 1.04723775, + "balance_loss_mlp": 1.00876701, + "epoch": 0.20693801478987556, + "flos": 56481021509760.0, + "grad_norm": 1.0514770087903598, + "language_loss": 0.62498379, + "learning_rate": 3.680628529789726e-06, + "loss": 0.6461755, + "num_input_tokens_seen": 36589300, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.03881836, + "step": 1721, + "time_per_iteration": 4.831583738327026 + }, + { + "auxiliary_loss_clip": 0.01183482, + "auxiliary_loss_mlp": 0.01048688, + "balance_loss_clip": 1.05454016, + "balance_loss_mlp": 1.02526367, + "epoch": 0.20705825768051464, + "flos": 21614201948160.0, + "grad_norm": 3.0748314062241087, + "language_loss": 0.86627769, + "learning_rate": 3.680206122135796e-06, + "loss": 0.8885994, + "num_input_tokens_seen": 36609905, + "router_z_loss_clip": 1.28710938, + "router_z_loss_mlp": 0.23449707, + "step": 1722, + "time_per_iteration": 3.820984125137329 + }, + { + "auxiliary_loss_clip": 0.01173547, + "auxiliary_loss_mlp": 0.01047159, + "balance_loss_clip": 1.0521543, + "balance_loss_mlp": 1.02531993, + "epoch": 0.20717850057115372, + "flos": 25848895962240.0, + "grad_norm": 2.033236767755381, + "language_loss": 0.78639996, + "learning_rate": 3.6797834595959323e-06, + "loss": 0.80860698, + "num_input_tokens_seen": 36629805, + "router_z_loss_clip": 1.21484375, + "router_z_loss_mlp": 0.21838379, + "step": 1723, + "time_per_iteration": 2.8432631492614746 + }, + { + "auxiliary_loss_clip": 0.01176735, + "auxiliary_loss_mlp": 0.0104799, + "balance_loss_clip": 1.05199075, + "balance_loss_mlp": 1.02498293, + "epoch": 0.20729874346179283, + "flos": 29130807767040.0, + "grad_norm": 3.0686530544974087, + "language_loss": 0.77752054, + "learning_rate": 3.679360542234254e-06, + "loss": 0.79976773, + "num_input_tokens_seen": 36649150, + "router_z_loss_clip": 1.24707031, + "router_z_loss_mlp": 0.23010254, + "step": 1724, + "time_per_iteration": 3.6881258487701416 + }, + { + "auxiliary_loss_clip": 0.01171931, + "auxiliary_loss_mlp": 0.01045963, + "balance_loss_clip": 1.05148172, + "balance_loss_mlp": 1.02562571, + "epoch": 0.20741898635243192, + "flos": 29023363209600.0, + "grad_norm": 2.247923853150271, + "language_loss": 0.72304595, + "learning_rate": 3.678937370114916e-06, + "loss": 0.74522495, + "num_input_tokens_seen": 36668955, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.20349121, + "step": 1725, + "time_per_iteration": 2.867053508758545 + }, + { + "auxiliary_loss_clip": 0.01170258, + "auxiliary_loss_mlp": 0.01044602, + "balance_loss_clip": 1.05131173, + "balance_loss_mlp": 1.02351403, + "epoch": 0.207539229243071, + "flos": 15559447841280.0, + "grad_norm": 2.2808041466560747, + "language_loss": 0.79327738, + "learning_rate": 3.678513943302114e-06, + "loss": 0.81542599, + "num_input_tokens_seen": 36685730, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.2109375, + "step": 1726, + "time_per_iteration": 2.8768155574798584 + }, + { + "auxiliary_loss_clip": 0.01175896, + "auxiliary_loss_mlp": 0.01043656, + "balance_loss_clip": 1.05329227, + "balance_loss_mlp": 1.02342618, + "epoch": 0.20765947213371008, + "flos": 20521081301760.0, + "grad_norm": 3.470340670556002, + "language_loss": 0.85006875, + "learning_rate": 3.678090261860082e-06, + "loss": 0.87226427, + "num_input_tokens_seen": 36705460, + "router_z_loss_clip": 1.22558594, + "router_z_loss_mlp": 0.20227051, + "step": 1727, + "time_per_iteration": 2.85905122756958 + }, + { + "auxiliary_loss_clip": 0.01175048, + "auxiliary_loss_mlp": 0.0104802, + "balance_loss_clip": 1.05151308, + "balance_loss_mlp": 1.02634799, + "epoch": 0.2077797150243492, + "flos": 19354415558400.0, + "grad_norm": 2.3562710106904317, + "language_loss": 0.78055036, + "learning_rate": 3.6776663258530906e-06, + "loss": 0.80278111, + "num_input_tokens_seen": 36724110, + "router_z_loss_clip": 1.23632812, + "router_z_loss_mlp": 0.21655273, + "step": 1728, + "time_per_iteration": 2.9276514053344727 + }, + { + "auxiliary_loss_clip": 0.0117093, + "auxiliary_loss_mlp": 0.01041694, + "balance_loss_clip": 1.05184126, + "balance_loss_mlp": 1.0211308, + "epoch": 0.20789995791498828, + "flos": 21829952989440.0, + "grad_norm": 2.1585020787508737, + "language_loss": 0.71189177, + "learning_rate": 3.6772421353454516e-06, + "loss": 0.73401803, + "num_input_tokens_seen": 36742705, + "router_z_loss_clip": 1.19042969, + "router_z_loss_mlp": 0.20556641, + "step": 1729, + "time_per_iteration": 2.8297243118286133 + }, + { + "auxiliary_loss_clip": 0.0117611, + "auxiliary_loss_mlp": 0.01047138, + "balance_loss_clip": 1.05202675, + "balance_loss_mlp": 1.02428532, + "epoch": 0.20802020080562736, + "flos": 23148844571520.0, + "grad_norm": 3.576536754775992, + "language_loss": 0.88778245, + "learning_rate": 3.6768176904015153e-06, + "loss": 0.91001493, + "num_input_tokens_seen": 36762510, + "router_z_loss_clip": 1.23730469, + "router_z_loss_mlp": 0.22851562, + "step": 1730, + "time_per_iteration": 2.837725877761841 + }, + { + "auxiliary_loss_clip": 0.01173694, + "auxiliary_loss_mlp": 0.01047822, + "balance_loss_clip": 1.04940069, + "balance_loss_mlp": 1.02617323, + "epoch": 0.20814044369626647, + "flos": 23072677781760.0, + "grad_norm": 4.233215074402881, + "language_loss": 0.6007669, + "learning_rate": 3.6763929910856674e-06, + "loss": 0.62298214, + "num_input_tokens_seen": 36780960, + "router_z_loss_clip": 1.2421875, + "router_z_loss_mlp": 0.21630859, + "step": 1731, + "time_per_iteration": 2.7974302768707275 + }, + { + "auxiliary_loss_clip": 0.01174998, + "auxiliary_loss_mlp": 0.01050176, + "balance_loss_clip": 1.05313373, + "balance_loss_mlp": 1.02921915, + "epoch": 0.20826068658690555, + "flos": 19608016556160.0, + "grad_norm": 2.8693394838815705, + "language_loss": 0.77843702, + "learning_rate": 3.6759680374623365e-06, + "loss": 0.80068874, + "num_input_tokens_seen": 36798875, + "router_z_loss_clip": 1.22070312, + "router_z_loss_mlp": 0.20947266, + "step": 1732, + "time_per_iteration": 2.850964069366455 + }, + { + "auxiliary_loss_clip": 0.01175876, + "auxiliary_loss_mlp": 0.01054622, + "balance_loss_clip": 1.05376601, + "balance_loss_mlp": 1.03275943, + "epoch": 0.20838092947754464, + "flos": 25374049142400.0, + "grad_norm": 2.3968518761341078, + "language_loss": 0.75287831, + "learning_rate": 3.675542829595986e-06, + "loss": 0.77518332, + "num_input_tokens_seen": 36818540, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.21862793, + "step": 1733, + "time_per_iteration": 2.826902151107788 + }, + { + "auxiliary_loss_clip": 0.01171817, + "auxiliary_loss_mlp": 0.01058044, + "balance_loss_clip": 1.05129218, + "balance_loss_mlp": 1.03694379, + "epoch": 0.20850117236818372, + "flos": 24061729749120.0, + "grad_norm": 1.6384401395092707, + "language_loss": 0.7927587, + "learning_rate": 3.6751173675511213e-06, + "loss": 0.81505734, + "num_input_tokens_seen": 36840585, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.2109375, + "step": 1734, + "time_per_iteration": 2.8042895793914795 + }, + { + "auxiliary_loss_clip": 0.01169665, + "auxiliary_loss_mlp": 0.01050793, + "balance_loss_clip": 1.04964161, + "balance_loss_mlp": 1.02977657, + "epoch": 0.20862141525882283, + "flos": 20077799558400.0, + "grad_norm": 2.6813337112381785, + "language_loss": 0.8763293, + "learning_rate": 3.674691651392283e-06, + "loss": 0.89853382, + "num_input_tokens_seen": 36858255, + "router_z_loss_clip": 1.20117188, + "router_z_loss_mlp": 0.21020508, + "step": 1735, + "time_per_iteration": 2.824601650238037 + }, + { + "auxiliary_loss_clip": 0.01177305, + "auxiliary_loss_mlp": 0.01050619, + "balance_loss_clip": 1.05182528, + "balance_loss_mlp": 1.0297333, + "epoch": 0.2087416581494619, + "flos": 39015183237120.0, + "grad_norm": 2.0769322210117194, + "language_loss": 0.75896728, + "learning_rate": 3.674265681184053e-06, + "loss": 0.78124654, + "num_input_tokens_seen": 36881515, + "router_z_loss_clip": 1.25585938, + "router_z_loss_mlp": 0.20874023, + "step": 1736, + "time_per_iteration": 2.8983616828918457 + }, + { + "auxiliary_loss_clip": 0.01172897, + "auxiliary_loss_mlp": 0.0104546, + "balance_loss_clip": 1.050354, + "balance_loss_mlp": 1.0243721, + "epoch": 0.208861901040101, + "flos": 26101994169600.0, + "grad_norm": 2.08289357568028, + "language_loss": 0.86591947, + "learning_rate": 3.6738394569910504e-06, + "loss": 0.88810313, + "num_input_tokens_seen": 36902055, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.2109375, + "step": 1737, + "time_per_iteration": 2.813570976257324 + }, + { + "auxiliary_loss_clip": 0.01177959, + "auxiliary_loss_mlp": 0.01048828, + "balance_loss_clip": 1.05477118, + "balance_loss_mlp": 1.02638113, + "epoch": 0.2089821439307401, + "flos": 28398732675840.0, + "grad_norm": 2.8164729234338783, + "language_loss": 0.82566345, + "learning_rate": 3.6734129788779333e-06, + "loss": 0.84793127, + "num_input_tokens_seen": 36921230, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.2244873, + "step": 1738, + "time_per_iteration": 2.8865535259246826 + }, + { + "auxiliary_loss_clip": 0.01169635, + "auxiliary_loss_mlp": 0.01050633, + "balance_loss_clip": 1.05172455, + "balance_loss_mlp": 1.03030753, + "epoch": 0.2091023868213792, + "flos": 21069616872960.0, + "grad_norm": 1.8559381295893405, + "language_loss": 0.9027487, + "learning_rate": 3.6729862469093976e-06, + "loss": 0.92495143, + "num_input_tokens_seen": 36940325, + "router_z_loss_clip": 1.17773438, + "router_z_loss_mlp": 0.20336914, + "step": 1739, + "time_per_iteration": 2.80937123298645 + }, + { + "auxiliary_loss_clip": 0.01171178, + "auxiliary_loss_mlp": 0.01048052, + "balance_loss_clip": 1.05044389, + "balance_loss_mlp": 1.02782238, + "epoch": 0.20922262971201827, + "flos": 22455481363200.0, + "grad_norm": 5.925798996970402, + "language_loss": 0.82978213, + "learning_rate": 3.6725592611501782e-06, + "loss": 0.85197449, + "num_input_tokens_seen": 36959000, + "router_z_loss_clip": 1.20800781, + "router_z_loss_mlp": 0.20239258, + "step": 1740, + "time_per_iteration": 2.7852628231048584 + }, + { + "auxiliary_loss_clip": 0.01169653, + "auxiliary_loss_mlp": 0.01048313, + "balance_loss_clip": 1.04693794, + "balance_loss_mlp": 1.02760684, + "epoch": 0.20934287260265738, + "flos": 27852244179840.0, + "grad_norm": 3.9631001462370907, + "language_loss": 0.76138073, + "learning_rate": 3.6721320216650496e-06, + "loss": 0.7835604, + "num_input_tokens_seen": 36979615, + "router_z_loss_clip": 1.22753906, + "router_z_loss_mlp": 0.20703125, + "step": 1741, + "time_per_iteration": 2.8233368396759033 + }, + { + "auxiliary_loss_clip": 0.0116989, + "auxiliary_loss_mlp": 0.01046563, + "balance_loss_clip": 1.04765165, + "balance_loss_mlp": 1.02565324, + "epoch": 0.20946311549329646, + "flos": 16435309075200.0, + "grad_norm": 1.9785013182215294, + "language_loss": 0.83387303, + "learning_rate": 3.6717045285188215e-06, + "loss": 0.85603762, + "num_input_tokens_seen": 36997310, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.20910645, + "step": 1742, + "time_per_iteration": 2.846289873123169 + }, + { + "auxiliary_loss_clip": 0.01175714, + "auxiliary_loss_mlp": 0.01050361, + "balance_loss_clip": 1.04988503, + "balance_loss_mlp": 1.02799702, + "epoch": 0.20958335838393555, + "flos": 22492720788480.0, + "grad_norm": 6.6883424691514595, + "language_loss": 0.87095535, + "learning_rate": 3.671276781776346e-06, + "loss": 0.89321607, + "num_input_tokens_seen": 37015965, + "router_z_loss_clip": 1.25878906, + "router_z_loss_mlp": 0.22363281, + "step": 1743, + "time_per_iteration": 2.9409964084625244 + }, + { + "auxiliary_loss_clip": 0.01170482, + "auxiliary_loss_mlp": 0.01046699, + "balance_loss_clip": 1.04981828, + "balance_loss_mlp": 1.02584934, + "epoch": 0.20970360127457463, + "flos": 25224768218880.0, + "grad_norm": 3.5049129317133896, + "language_loss": 0.6751951, + "learning_rate": 3.6708487815025128e-06, + "loss": 0.69736695, + "num_input_tokens_seen": 37036545, + "router_z_loss_clip": 1.20703125, + "router_z_loss_mlp": 0.20837402, + "step": 1744, + "time_per_iteration": 2.851640462875366 + }, + { + "auxiliary_loss_clip": 0.01171903, + "auxiliary_loss_mlp": 0.01047923, + "balance_loss_clip": 1.0500958, + "balance_loss_mlp": 1.02709711, + "epoch": 0.20982384416521374, + "flos": 18479164855680.0, + "grad_norm": 7.485007347461538, + "language_loss": 0.74314296, + "learning_rate": 3.6704205277622463e-06, + "loss": 0.76534128, + "num_input_tokens_seen": 37054985, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.20825195, + "step": 1745, + "time_per_iteration": 2.7997381687164307 + }, + { + "auxiliary_loss_clip": 0.01167487, + "auxiliary_loss_mlp": 0.01041988, + "balance_loss_clip": 1.04761243, + "balance_loss_mlp": 1.02259254, + "epoch": 0.20994408705585282, + "flos": 25373546352000.0, + "grad_norm": 2.9801684420145538, + "language_loss": 0.80169445, + "learning_rate": 3.6699920206205146e-06, + "loss": 0.82378918, + "num_input_tokens_seen": 37075725, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.19396973, + "step": 1746, + "time_per_iteration": 3.017080307006836 + }, + { + "auxiliary_loss_clip": 0.01170189, + "auxiliary_loss_mlp": 0.01050067, + "balance_loss_clip": 1.04907084, + "balance_loss_mlp": 1.02934813, + "epoch": 0.2100643299464919, + "flos": 21320955313920.0, + "grad_norm": 2.2303878423837107, + "language_loss": 0.82309127, + "learning_rate": 3.669563260142321e-06, + "loss": 0.84529388, + "num_input_tokens_seen": 37094615, + "router_z_loss_clip": 1.21484375, + "router_z_loss_mlp": 0.20703125, + "step": 1747, + "time_per_iteration": 2.8312578201293945 + }, + { + "auxiliary_loss_clip": 0.01172149, + "auxiliary_loss_mlp": 0.01051963, + "balance_loss_clip": 1.04888821, + "balance_loss_mlp": 1.03020704, + "epoch": 0.21018457283713102, + "flos": 19354379644800.0, + "grad_norm": 2.573900103929288, + "language_loss": 0.84020001, + "learning_rate": 3.6691342463927083e-06, + "loss": 0.86244106, + "num_input_tokens_seen": 37113610, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.2175293, + "step": 1748, + "time_per_iteration": 2.807340621948242 + }, + { + "auxiliary_loss_clip": 0.01169763, + "auxiliary_loss_mlp": 0.01048354, + "balance_loss_clip": 1.04850483, + "balance_loss_mlp": 1.02801692, + "epoch": 0.2103048157277701, + "flos": 28330035914880.0, + "grad_norm": 1.8786472900321967, + "language_loss": 0.82006562, + "learning_rate": 3.668704979436758e-06, + "loss": 0.84224677, + "num_input_tokens_seen": 37133705, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.20336914, + "step": 1749, + "time_per_iteration": 2.86490797996521 + }, + { + "auxiliary_loss_clip": 0.01172153, + "auxiliary_loss_mlp": 0.01055416, + "balance_loss_clip": 1.04819381, + "balance_loss_mlp": 1.03323078, + "epoch": 0.21042505861840918, + "flos": 17457290835840.0, + "grad_norm": 2.2902186201838424, + "language_loss": 0.78753656, + "learning_rate": 3.668275459339588e-06, + "loss": 0.80981225, + "num_input_tokens_seen": 37152185, + "router_z_loss_clip": 1.23925781, + "router_z_loss_mlp": 0.22180176, + "step": 1750, + "time_per_iteration": 4.594646215438843 + }, + { + "auxiliary_loss_clip": 0.01173086, + "auxiliary_loss_mlp": 0.01047291, + "balance_loss_clip": 1.05162489, + "balance_loss_mlp": 1.02514148, + "epoch": 0.21054530150904827, + "flos": 14209817195520.0, + "grad_norm": 2.2681357718972017, + "language_loss": 0.79806662, + "learning_rate": 3.667845686166358e-06, + "loss": 0.82027042, + "num_input_tokens_seen": 37169110, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.22155762, + "step": 1751, + "time_per_iteration": 3.670454263687134 + }, + { + "auxiliary_loss_clip": 0.01171812, + "auxiliary_loss_mlp": 0.01050125, + "balance_loss_clip": 1.04913616, + "balance_loss_mlp": 1.02878618, + "epoch": 0.21066554439968738, + "flos": 18618210403200.0, + "grad_norm": 2.314287030362562, + "language_loss": 0.8572253, + "learning_rate": 3.6674156599822634e-06, + "loss": 0.8794446, + "num_input_tokens_seen": 37184905, + "router_z_loss_clip": 1.22558594, + "router_z_loss_mlp": 0.21350098, + "step": 1752, + "time_per_iteration": 2.740208625793457 + }, + { + "auxiliary_loss_clip": 0.01171615, + "auxiliary_loss_mlp": 0.01051703, + "balance_loss_clip": 1.05065036, + "balance_loss_mlp": 1.03036427, + "epoch": 0.21078578729032646, + "flos": 23658883741440.0, + "grad_norm": 2.4331692387661734, + "language_loss": 0.81841636, + "learning_rate": 3.666985380852539e-06, + "loss": 0.8406496, + "num_input_tokens_seen": 37203910, + "router_z_loss_clip": 1.2109375, + "router_z_loss_mlp": 0.21325684, + "step": 1753, + "time_per_iteration": 3.720263719558716 + }, + { + "auxiliary_loss_clip": 0.01176072, + "auxiliary_loss_mlp": 0.01048743, + "balance_loss_clip": 1.04958785, + "balance_loss_mlp": 1.02611685, + "epoch": 0.21090603018096554, + "flos": 29346379240320.0, + "grad_norm": 3.5763526906205167, + "language_loss": 0.74413085, + "learning_rate": 3.6665548488424576e-06, + "loss": 0.766379, + "num_input_tokens_seen": 37222670, + "router_z_loss_clip": 1.265625, + "router_z_loss_mlp": 0.22631836, + "step": 1754, + "time_per_iteration": 2.825761556625366 + }, + { + "auxiliary_loss_clip": 0.01174384, + "auxiliary_loss_mlp": 0.01052054, + "balance_loss_clip": 1.05187988, + "balance_loss_mlp": 1.0291065, + "epoch": 0.21102627307160465, + "flos": 23261245205760.0, + "grad_norm": 2.072253121208094, + "language_loss": 0.88015759, + "learning_rate": 3.6661240640173307e-06, + "loss": 0.90242195, + "num_input_tokens_seen": 37244140, + "router_z_loss_clip": 1.22558594, + "router_z_loss_mlp": 0.22949219, + "step": 1755, + "time_per_iteration": 2.8460323810577393 + }, + { + "auxiliary_loss_clip": 0.01117928, + "auxiliary_loss_mlp": 0.01031909, + "balance_loss_clip": 1.05264139, + "balance_loss_mlp": 1.02764165, + "epoch": 0.21114651596224374, + "flos": 54633454577280.0, + "grad_norm": 3.60481868461706, + "language_loss": 0.57920069, + "learning_rate": 3.6656930264425085e-06, + "loss": 0.60069907, + "num_input_tokens_seen": 37308185, + "router_z_loss_clip": 0.65625, + "router_z_loss_mlp": 0.04272461, + "step": 1756, + "time_per_iteration": 3.3578383922576904 + }, + { + "auxiliary_loss_clip": 0.01171721, + "auxiliary_loss_mlp": 0.01049349, + "balance_loss_clip": 1.05038023, + "balance_loss_mlp": 1.02910757, + "epoch": 0.21126675885288282, + "flos": 21543314457600.0, + "grad_norm": 2.5240236172122423, + "language_loss": 0.75556922, + "learning_rate": 3.665261736183378e-06, + "loss": 0.77778, + "num_input_tokens_seen": 37328220, + "router_z_loss_clip": 1.21484375, + "router_z_loss_mlp": 0.20239258, + "step": 1757, + "time_per_iteration": 2.7794342041015625 + }, + { + "auxiliary_loss_clip": 0.01174354, + "auxiliary_loss_mlp": 0.01046884, + "balance_loss_clip": 1.05190611, + "balance_loss_mlp": 1.02548623, + "epoch": 0.2113870017435219, + "flos": 10961876678400.0, + "grad_norm": 4.0245175328735225, + "language_loss": 0.8857528, + "learning_rate": 3.664830193305366e-06, + "loss": 0.90796518, + "num_input_tokens_seen": 37345995, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.21398926, + "step": 1758, + "time_per_iteration": 2.7897067070007324 + }, + { + "auxiliary_loss_clip": 0.01170931, + "auxiliary_loss_mlp": 0.01046096, + "balance_loss_clip": 1.04863131, + "balance_loss_mlp": 1.02534175, + "epoch": 0.211507244634161, + "flos": 16653825463680.0, + "grad_norm": 5.480448764586069, + "language_loss": 0.77422076, + "learning_rate": 3.6643983978739373e-06, + "loss": 0.79639101, + "num_input_tokens_seen": 37362610, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.20751953, + "step": 1759, + "time_per_iteration": 2.765592575073242 + }, + { + "auxiliary_loss_clip": 0.01174083, + "auxiliary_loss_mlp": 0.01052229, + "balance_loss_clip": 1.05286348, + "balance_loss_mlp": 1.03124857, + "epoch": 0.2116274875248001, + "flos": 20954091755520.0, + "grad_norm": 2.1379319789903732, + "language_loss": 0.82258499, + "learning_rate": 3.663966349954596e-06, + "loss": 0.84484804, + "num_input_tokens_seen": 37382790, + "router_z_loss_clip": 1.21191406, + "router_z_loss_mlp": 0.20983887, + "step": 1760, + "time_per_iteration": 2.825700283050537 + }, + { + "auxiliary_loss_clip": 0.01127497, + "auxiliary_loss_mlp": 0.01027405, + "balance_loss_clip": 1.05806375, + "balance_loss_mlp": 1.02244568, + "epoch": 0.21174773041543918, + "flos": 68196949424640.0, + "grad_norm": 0.8029398749175803, + "language_loss": 0.59658861, + "learning_rate": 3.6635340496128816e-06, + "loss": 0.61813766, + "num_input_tokens_seen": 37439720, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.04956055, + "step": 1761, + "time_per_iteration": 3.2705068588256836 + }, + { + "auxiliary_loss_clip": 0.01169378, + "auxiliary_loss_mlp": 0.01048282, + "balance_loss_clip": 1.05094504, + "balance_loss_mlp": 1.02902937, + "epoch": 0.2118679733060783, + "flos": 20668315150080.0, + "grad_norm": 2.660769490726527, + "language_loss": 0.92902815, + "learning_rate": 3.6631014969143747e-06, + "loss": 0.95120484, + "num_input_tokens_seen": 37459410, + "router_z_loss_clip": 1.18457031, + "router_z_loss_mlp": 0.19238281, + "step": 1762, + "time_per_iteration": 2.8907980918884277 + }, + { + "auxiliary_loss_clip": 0.01174984, + "auxiliary_loss_mlp": 0.01053572, + "balance_loss_clip": 1.05281687, + "balance_loss_mlp": 1.03308046, + "epoch": 0.21198821619671737, + "flos": 23223431162880.0, + "grad_norm": 2.336243270834891, + "language_loss": 0.88660967, + "learning_rate": 3.662668691924693e-06, + "loss": 0.90889525, + "num_input_tokens_seen": 37480460, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.20495605, + "step": 1763, + "time_per_iteration": 2.947497844696045 + }, + { + "auxiliary_loss_clip": 0.01174583, + "auxiliary_loss_mlp": 0.01056172, + "balance_loss_clip": 1.05151403, + "balance_loss_mlp": 1.03469062, + "epoch": 0.21210845908735645, + "flos": 24498547044480.0, + "grad_norm": 13.293476231741433, + "language_loss": 0.71161175, + "learning_rate": 3.6622356347094927e-06, + "loss": 0.73391938, + "num_input_tokens_seen": 37502025, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.21496582, + "step": 1764, + "time_per_iteration": 2.951974630355835 + }, + { + "auxiliary_loss_clip": 0.01175068, + "auxiliary_loss_mlp": 0.01056615, + "balance_loss_clip": 1.05337095, + "balance_loss_mlp": 1.03518152, + "epoch": 0.21222870197799554, + "flos": 27089789160960.0, + "grad_norm": 3.3890851804308526, + "language_loss": 0.7859484, + "learning_rate": 3.6618023253344684e-06, + "loss": 0.80826515, + "num_input_tokens_seen": 37520885, + "router_z_loss_clip": 1.21484375, + "router_z_loss_mlp": 0.21447754, + "step": 1765, + "time_per_iteration": 2.8894989490509033 + }, + { + "auxiliary_loss_clip": 0.01178282, + "auxiliary_loss_mlp": 0.01058502, + "balance_loss_clip": 1.0524013, + "balance_loss_mlp": 1.03740191, + "epoch": 0.21234894486863465, + "flos": 16873850223360.0, + "grad_norm": 2.010161874545481, + "language_loss": 0.83602297, + "learning_rate": 3.6613687638653527e-06, + "loss": 0.85839081, + "num_input_tokens_seen": 37539055, + "router_z_loss_clip": 1.2578125, + "router_z_loss_mlp": 0.21130371, + "step": 1766, + "time_per_iteration": 2.8490188121795654 + }, + { + "auxiliary_loss_clip": 0.01175781, + "auxiliary_loss_mlp": 0.01057428, + "balance_loss_clip": 1.05270624, + "balance_loss_mlp": 1.03682911, + "epoch": 0.21246918775927373, + "flos": 23474949171840.0, + "grad_norm": 2.6429894966040663, + "language_loss": 0.78404152, + "learning_rate": 3.660934950367916e-06, + "loss": 0.80637366, + "num_input_tokens_seen": 37558300, + "router_z_loss_clip": 1.22949219, + "router_z_loss_mlp": 0.20605469, + "step": 1767, + "time_per_iteration": 2.83758282661438 + }, + { + "auxiliary_loss_clip": 0.0117371, + "auxiliary_loss_mlp": 0.01054613, + "balance_loss_clip": 1.05110824, + "balance_loss_mlp": 1.03220177, + "epoch": 0.21258943064991281, + "flos": 22382295402240.0, + "grad_norm": 1.8926983649982452, + "language_loss": 0.83316714, + "learning_rate": 3.660500884907968e-06, + "loss": 0.85545039, + "num_input_tokens_seen": 37579040, + "router_z_loss_clip": 1.22753906, + "router_z_loss_mlp": 0.22399902, + "step": 1768, + "time_per_iteration": 2.871898889541626 + }, + { + "auxiliary_loss_clip": 0.0112641, + "auxiliary_loss_mlp": 0.01020292, + "balance_loss_clip": 1.06111968, + "balance_loss_mlp": 1.01592863, + "epoch": 0.21270967354055192, + "flos": 59440168679040.0, + "grad_norm": 0.9225661763560449, + "language_loss": 0.59959108, + "learning_rate": 3.660066567551356e-06, + "loss": 0.62105805, + "num_input_tokens_seen": 37639185, + "router_z_loss_clip": 0.65234375, + "router_z_loss_mlp": 0.04370117, + "step": 1769, + "time_per_iteration": 3.3208470344543457 + }, + { + "auxiliary_loss_clip": 0.01170291, + "auxiliary_loss_mlp": 0.01048998, + "balance_loss_clip": 1.05098581, + "balance_loss_mlp": 1.02862501, + "epoch": 0.212829916431191, + "flos": 21544032729600.0, + "grad_norm": 4.3224106962551385, + "language_loss": 0.84179997, + "learning_rate": 3.6596319983639657e-06, + "loss": 0.86399287, + "num_input_tokens_seen": 37657765, + "router_z_loss_clip": 1.19335938, + "router_z_loss_mlp": 0.20373535, + "step": 1770, + "time_per_iteration": 2.823721408843994 + }, + { + "auxiliary_loss_clip": 0.01174638, + "auxiliary_loss_mlp": 0.0104484, + "balance_loss_clip": 1.05242729, + "balance_loss_mlp": 1.02373958, + "epoch": 0.2129501593218301, + "flos": 28987739896320.0, + "grad_norm": 7.921253072641097, + "language_loss": 0.86046398, + "learning_rate": 3.6591971774117214e-06, + "loss": 0.88265872, + "num_input_tokens_seen": 37680740, + "router_z_loss_clip": 1.22167969, + "router_z_loss_mlp": 0.21105957, + "step": 1771, + "time_per_iteration": 2.8166892528533936 + }, + { + "auxiliary_loss_clip": 0.01173919, + "auxiliary_loss_mlp": 0.01051884, + "balance_loss_clip": 1.0531795, + "balance_loss_mlp": 1.03155875, + "epoch": 0.2130704022124692, + "flos": 18806993308800.0, + "grad_norm": 10.377608998009745, + "language_loss": 0.80405009, + "learning_rate": 3.6587621047605833e-06, + "loss": 0.82630813, + "num_input_tokens_seen": 37697910, + "router_z_loss_clip": 1.20800781, + "router_z_loss_mlp": 0.20336914, + "step": 1772, + "time_per_iteration": 2.8803422451019287 + }, + { + "auxiliary_loss_clip": 0.01165504, + "auxiliary_loss_mlp": 0.01049532, + "balance_loss_clip": 1.04905045, + "balance_loss_mlp": 1.02812147, + "epoch": 0.21319064510310828, + "flos": 13918150759680.0, + "grad_norm": 3.2000243689478536, + "language_loss": 0.86450422, + "learning_rate": 3.6583267804765542e-06, + "loss": 0.88665456, + "num_input_tokens_seen": 37712245, + "router_z_loss_clip": 1.1640625, + "router_z_loss_mlp": 0.21411133, + "step": 1773, + "time_per_iteration": 2.852029323577881 + }, + { + "auxiliary_loss_clip": 0.01176024, + "auxiliary_loss_mlp": 0.01054143, + "balance_loss_clip": 1.05271721, + "balance_loss_mlp": 1.03255427, + "epoch": 0.21331088799374737, + "flos": 20959694277120.0, + "grad_norm": 2.519548918868391, + "language_loss": 0.85734963, + "learning_rate": 3.6578912046256702e-06, + "loss": 0.87965137, + "num_input_tokens_seen": 37730765, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.21582031, + "step": 1774, + "time_per_iteration": 2.9445390701293945 + }, + { + "auxiliary_loss_clip": 0.01171538, + "auxiliary_loss_mlp": 0.01040092, + "balance_loss_clip": 1.05145645, + "balance_loss_mlp": 1.01923013, + "epoch": 0.21343113088438645, + "flos": 18624638937600.0, + "grad_norm": 2.3430975332408357, + "language_loss": 0.75829554, + "learning_rate": 3.6574553772740083e-06, + "loss": 0.78041184, + "num_input_tokens_seen": 37748695, + "router_z_loss_clip": 1.20117188, + "router_z_loss_mlp": 0.20861816, + "step": 1775, + "time_per_iteration": 2.824397563934326 + }, + { + "auxiliary_loss_clip": 0.01110969, + "auxiliary_loss_mlp": 0.01012759, + "balance_loss_clip": 1.04919434, + "balance_loss_mlp": 1.00880098, + "epoch": 0.21355137377502556, + "flos": 67413128791680.0, + "grad_norm": 0.9676530937211784, + "language_loss": 0.61881799, + "learning_rate": 3.657019298487684e-06, + "loss": 0.6400553, + "num_input_tokens_seen": 37813705, + "router_z_loss_clip": 0.6171875, + "router_z_loss_mlp": 0.03955078, + "step": 1776, + "time_per_iteration": 3.369626998901367 + }, + { + "auxiliary_loss_clip": 0.01172246, + "auxiliary_loss_mlp": 0.01041423, + "balance_loss_clip": 1.04972827, + "balance_loss_mlp": 1.02025175, + "epoch": 0.21367161666566464, + "flos": 34532095697280.0, + "grad_norm": 2.7021453203712813, + "language_loss": 0.83712673, + "learning_rate": 3.6565829683328495e-06, + "loss": 0.85926342, + "num_input_tokens_seen": 37836330, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.21166992, + "step": 1777, + "time_per_iteration": 2.955167055130005 + }, + { + "auxiliary_loss_clip": 0.01172059, + "auxiliary_loss_mlp": 0.01047765, + "balance_loss_clip": 1.05117798, + "balance_loss_mlp": 1.02672458, + "epoch": 0.21379185955630373, + "flos": 18989347680000.0, + "grad_norm": 3.5370015246887694, + "language_loss": 0.86219925, + "learning_rate": 3.6561463868756965e-06, + "loss": 0.88439751, + "num_input_tokens_seen": 37855030, + "router_z_loss_clip": 1.20996094, + "router_z_loss_mlp": 0.21032715, + "step": 1778, + "time_per_iteration": 2.8095431327819824 + }, + { + "auxiliary_loss_clip": 0.01171029, + "auxiliary_loss_mlp": 0.01043593, + "balance_loss_clip": 1.05183935, + "balance_loss_mlp": 1.02167034, + "epoch": 0.21391210244694284, + "flos": 28218497207040.0, + "grad_norm": 2.581300715271322, + "language_loss": 0.78124994, + "learning_rate": 3.655709554182452e-06, + "loss": 0.80339611, + "num_input_tokens_seen": 37875370, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.21911621, + "step": 1779, + "time_per_iteration": 4.560044288635254 + }, + { + "auxiliary_loss_clip": 0.01173282, + "auxiliary_loss_mlp": 0.01049821, + "balance_loss_clip": 1.05110991, + "balance_loss_mlp": 1.02860212, + "epoch": 0.21403234533758192, + "flos": 17455064192640.0, + "grad_norm": 2.0193421832786322, + "language_loss": 0.84759998, + "learning_rate": 3.6552724703193855e-06, + "loss": 0.86983097, + "num_input_tokens_seen": 37892560, + "router_z_loss_clip": 1.22167969, + "router_z_loss_mlp": 0.21228027, + "step": 1780, + "time_per_iteration": 3.6259098052978516 + }, + { + "auxiliary_loss_clip": 0.01102087, + "auxiliary_loss_mlp": 0.01007152, + "balance_loss_clip": 1.04280829, + "balance_loss_mlp": 1.00328946, + "epoch": 0.214152588228221, + "flos": 51637606686720.0, + "grad_norm": 0.7886354586443253, + "language_loss": 0.55897403, + "learning_rate": 3.654835135352801e-06, + "loss": 0.58006632, + "num_input_tokens_seen": 37947370, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.03857422, + "step": 1781, + "time_per_iteration": 3.2067697048187256 + }, + { + "auxiliary_loss_clip": 0.01166771, + "auxiliary_loss_mlp": 0.01047304, + "balance_loss_clip": 1.04791152, + "balance_loss_mlp": 1.02652633, + "epoch": 0.21427283111886009, + "flos": 19496154625920.0, + "grad_norm": 4.136371418145807, + "language_loss": 0.87272829, + "learning_rate": 3.654397549349043e-06, + "loss": 0.89486897, + "num_input_tokens_seen": 37964745, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.20751953, + "step": 1782, + "time_per_iteration": 3.6423895359039307 + }, + { + "auxiliary_loss_clip": 0.01169883, + "auxiliary_loss_mlp": 0.01051966, + "balance_loss_clip": 1.05045915, + "balance_loss_mlp": 1.0305922, + "epoch": 0.2143930740094992, + "flos": 20084802710400.0, + "grad_norm": 2.5828695338055, + "language_loss": 0.75213951, + "learning_rate": 3.653959712374491e-06, + "loss": 0.77435803, + "num_input_tokens_seen": 37982850, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.21362305, + "step": 1783, + "time_per_iteration": 2.884364604949951 + }, + { + "auxiliary_loss_clip": 0.01170447, + "auxiliary_loss_mlp": 0.01048916, + "balance_loss_clip": 1.04906678, + "balance_loss_mlp": 1.0293057, + "epoch": 0.21451331690013828, + "flos": 21798603394560.0, + "grad_norm": 2.07557706044272, + "language_loss": 0.8266905, + "learning_rate": 3.6535216244955663e-06, + "loss": 0.84888411, + "num_input_tokens_seen": 38002745, + "router_z_loss_clip": 1.21386719, + "router_z_loss_mlp": 0.19604492, + "step": 1784, + "time_per_iteration": 2.8602850437164307 + }, + { + "auxiliary_loss_clip": 0.01171578, + "auxiliary_loss_mlp": 0.01052658, + "balance_loss_clip": 1.05037093, + "balance_loss_mlp": 1.03080678, + "epoch": 0.21463355979077736, + "flos": 32853882412800.0, + "grad_norm": 2.376065276250656, + "language_loss": 0.7122159, + "learning_rate": 3.653083285778726e-06, + "loss": 0.73445827, + "num_input_tokens_seen": 38024115, + "router_z_loss_clip": 1.20996094, + "router_z_loss_mlp": 0.21887207, + "step": 1785, + "time_per_iteration": 2.984572172164917 + }, + { + "auxiliary_loss_clip": 0.01171762, + "auxiliary_loss_mlp": 0.01051341, + "balance_loss_clip": 1.04992092, + "balance_loss_mlp": 1.02984762, + "epoch": 0.21475380268141647, + "flos": 21543817248000.0, + "grad_norm": 3.086948432433903, + "language_loss": 0.81018227, + "learning_rate": 3.6526446962904653e-06, + "loss": 0.83241332, + "num_input_tokens_seen": 38042830, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.21484375, + "step": 1786, + "time_per_iteration": 2.8353428840637207 + }, + { + "auxiliary_loss_clip": 0.0116676, + "auxiliary_loss_mlp": 0.01044429, + "balance_loss_clip": 1.04948425, + "balance_loss_mlp": 1.0249145, + "epoch": 0.21487404557205556, + "flos": 32159082660480.0, + "grad_norm": 2.6062792205979193, + "language_loss": 0.73994875, + "learning_rate": 3.652205856097318e-06, + "loss": 0.76206064, + "num_input_tokens_seen": 38066015, + "router_z_loss_clip": 1.17285156, + "router_z_loss_mlp": 0.19519043, + "step": 1787, + "time_per_iteration": 2.846379280090332 + }, + { + "auxiliary_loss_clip": 0.01173228, + "auxiliary_loss_mlp": 0.0104703, + "balance_loss_clip": 1.04943275, + "balance_loss_mlp": 1.02408242, + "epoch": 0.21499428846269464, + "flos": 12673091583360.0, + "grad_norm": 2.533074315189678, + "language_loss": 0.79366827, + "learning_rate": 3.651766765265856e-06, + "loss": 0.81587082, + "num_input_tokens_seen": 38083025, + "router_z_loss_clip": 1.23828125, + "router_z_loss_mlp": 0.22937012, + "step": 1788, + "time_per_iteration": 2.817178726196289 + }, + { + "auxiliary_loss_clip": 0.01171793, + "auxiliary_loss_mlp": 0.01051883, + "balance_loss_clip": 1.05009294, + "balance_loss_mlp": 1.03191578, + "epoch": 0.21511453135333372, + "flos": 23471573293440.0, + "grad_norm": 4.522694489033548, + "language_loss": 0.80564874, + "learning_rate": 3.65132742386269e-06, + "loss": 0.82788551, + "num_input_tokens_seen": 38098245, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.19970703, + "step": 1789, + "time_per_iteration": 2.9176316261291504 + }, + { + "auxiliary_loss_clip": 0.01171695, + "auxiliary_loss_mlp": 0.01054357, + "balance_loss_clip": 1.05069661, + "balance_loss_mlp": 1.0333879, + "epoch": 0.21523477424397283, + "flos": 26943560893440.0, + "grad_norm": 2.2484889889064466, + "language_loss": 0.8453989, + "learning_rate": 3.6508878319544656e-06, + "loss": 0.86765945, + "num_input_tokens_seen": 38118460, + "router_z_loss_clip": 1.20898438, + "router_z_loss_mlp": 0.20959473, + "step": 1790, + "time_per_iteration": 2.8149194717407227 + }, + { + "auxiliary_loss_clip": 0.01167194, + "auxiliary_loss_mlp": 0.0104601, + "balance_loss_clip": 1.04895115, + "balance_loss_mlp": 1.02589989, + "epoch": 0.21535501713461191, + "flos": 18916161719040.0, + "grad_norm": 3.6061574552571756, + "language_loss": 0.81517851, + "learning_rate": 3.65044798960787e-06, + "loss": 0.83731061, + "num_input_tokens_seen": 38136800, + "router_z_loss_clip": 1.18164062, + "router_z_loss_mlp": 0.20129395, + "step": 1791, + "time_per_iteration": 2.7812235355377197 + }, + { + "auxiliary_loss_clip": 0.01165714, + "auxiliary_loss_mlp": 0.01051693, + "balance_loss_clip": 1.04872847, + "balance_loss_mlp": 1.03161812, + "epoch": 0.215475260025251, + "flos": 17895113712000.0, + "grad_norm": 2.0214453450649286, + "language_loss": 0.78157777, + "learning_rate": 3.650007896889627e-06, + "loss": 0.80375183, + "num_input_tokens_seen": 38155380, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.20080566, + "step": 1792, + "time_per_iteration": 2.7721102237701416 + }, + { + "auxiliary_loss_clip": 0.01167189, + "auxiliary_loss_mlp": 0.01054943, + "balance_loss_clip": 1.05093741, + "balance_loss_mlp": 1.0333066, + "epoch": 0.2155955029158901, + "flos": 16654292340480.0, + "grad_norm": 2.972037250006033, + "language_loss": 0.80446553, + "learning_rate": 3.6495675538664974e-06, + "loss": 0.82668686, + "num_input_tokens_seen": 38174395, + "router_z_loss_clip": 1.16113281, + "router_z_loss_mlp": 0.21643066, + "step": 1793, + "time_per_iteration": 2.8503777980804443 + }, + { + "auxiliary_loss_clip": 0.01169677, + "auxiliary_loss_mlp": 0.01054524, + "balance_loss_clip": 1.05009484, + "balance_loss_mlp": 1.03258955, + "epoch": 0.2157157458065292, + "flos": 23621213352960.0, + "grad_norm": 4.025673889488132, + "language_loss": 0.82560307, + "learning_rate": 3.649126960605282e-06, + "loss": 0.84784508, + "num_input_tokens_seen": 38195380, + "router_z_loss_clip": 1.19628906, + "router_z_loss_mlp": 0.21936035, + "step": 1794, + "time_per_iteration": 2.8055901527404785 + }, + { + "auxiliary_loss_clip": 0.01170715, + "auxiliary_loss_mlp": 0.01051301, + "balance_loss_clip": 1.04740763, + "balance_loss_mlp": 1.03072572, + "epoch": 0.21583598869716827, + "flos": 22127078292480.0, + "grad_norm": 2.7085810405989545, + "language_loss": 0.83523667, + "learning_rate": 3.6486861171728174e-06, + "loss": 0.85745686, + "num_input_tokens_seen": 38213775, + "router_z_loss_clip": 1.23339844, + "router_z_loss_mlp": 0.20568848, + "step": 1795, + "time_per_iteration": 2.8102879524230957 + }, + { + "auxiliary_loss_clip": 0.01170469, + "auxiliary_loss_mlp": 0.01048072, + "balance_loss_clip": 1.04898405, + "balance_loss_mlp": 1.02703166, + "epoch": 0.21595623158780738, + "flos": 23441229279360.0, + "grad_norm": 2.7297287427728256, + "language_loss": 0.78376985, + "learning_rate": 3.6482450236359803e-06, + "loss": 0.80595529, + "num_input_tokens_seen": 38235630, + "router_z_loss_clip": 1.21386719, + "router_z_loss_mlp": 0.21020508, + "step": 1796, + "time_per_iteration": 2.79868483543396 + }, + { + "auxiliary_loss_clip": 0.0116459, + "auxiliary_loss_mlp": 0.01041591, + "balance_loss_clip": 1.04788423, + "balance_loss_mlp": 1.02212429, + "epoch": 0.21607647447844647, + "flos": 26906501036160.0, + "grad_norm": 6.310579965932127, + "language_loss": 0.78039443, + "learning_rate": 3.647803680061683e-06, + "loss": 0.80245626, + "num_input_tokens_seen": 38256045, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.19470215, + "step": 1797, + "time_per_iteration": 2.8640615940093994 + }, + { + "auxiliary_loss_clip": 0.01167062, + "auxiliary_loss_mlp": 0.01051105, + "balance_loss_clip": 1.04732919, + "balance_loss_mlp": 1.03069639, + "epoch": 0.21619671736908555, + "flos": 14495378319360.0, + "grad_norm": 7.350289161303192, + "language_loss": 0.75173461, + "learning_rate": 3.6473620865168776e-06, + "loss": 0.77391624, + "num_input_tokens_seen": 38272915, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.20397949, + "step": 1798, + "time_per_iteration": 2.8042757511138916 + }, + { + "auxiliary_loss_clip": 0.01165918, + "auxiliary_loss_mlp": 0.01051234, + "balance_loss_clip": 1.04989731, + "balance_loss_mlp": 1.03158808, + "epoch": 0.21631696025972463, + "flos": 17931096161280.0, + "grad_norm": 2.211546945822639, + "language_loss": 0.81536567, + "learning_rate": 3.646920243068554e-06, + "loss": 0.83753717, + "num_input_tokens_seen": 38290810, + "router_z_loss_clip": 1.16015625, + "router_z_loss_mlp": 0.19628906, + "step": 1799, + "time_per_iteration": 2.809709072113037 + }, + { + "auxiliary_loss_clip": 0.01171074, + "auxiliary_loss_mlp": 0.01042785, + "balance_loss_clip": 1.05158651, + "balance_loss_mlp": 1.02040958, + "epoch": 0.21643720315036374, + "flos": 24462385027200.0, + "grad_norm": 2.02926452829271, + "language_loss": 0.74127799, + "learning_rate": 3.6464781497837384e-06, + "loss": 0.76341665, + "num_input_tokens_seen": 38312785, + "router_z_loss_clip": 1.19628906, + "router_z_loss_mlp": 0.22387695, + "step": 1800, + "time_per_iteration": 2.8461244106292725 + }, + { + "auxiliary_loss_clip": 0.01172177, + "auxiliary_loss_mlp": 0.01047659, + "balance_loss_clip": 1.04976034, + "balance_loss_mlp": 1.02690494, + "epoch": 0.21655744604100283, + "flos": 28474432588800.0, + "grad_norm": 2.545448332245645, + "language_loss": 0.72660172, + "learning_rate": 3.6460358067294965e-06, + "loss": 0.74880004, + "num_input_tokens_seen": 38334015, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.20739746, + "step": 1801, + "time_per_iteration": 2.8440442085266113 + }, + { + "auxiliary_loss_clip": 0.01169486, + "auxiliary_loss_mlp": 0.01053604, + "balance_loss_clip": 1.04607666, + "balance_loss_mlp": 1.03003693, + "epoch": 0.2166776889316419, + "flos": 20152960767360.0, + "grad_norm": 2.664600432241048, + "language_loss": 0.77752423, + "learning_rate": 3.645593213972932e-06, + "loss": 0.79975522, + "num_input_tokens_seen": 38352920, + "router_z_loss_clip": 1.23339844, + "router_z_loss_mlp": 0.23583984, + "step": 1802, + "time_per_iteration": 2.833416700363159 + }, + { + "auxiliary_loss_clip": 0.01173275, + "auxiliary_loss_mlp": 0.01049346, + "balance_loss_clip": 1.05112851, + "balance_loss_mlp": 1.02835345, + "epoch": 0.21679793182228102, + "flos": 15193482122880.0, + "grad_norm": 2.6852626495308103, + "language_loss": 0.7969355, + "learning_rate": 3.6451503715811852e-06, + "loss": 0.81916177, + "num_input_tokens_seen": 38371230, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.20996094, + "step": 1803, + "time_per_iteration": 2.8081440925598145 + }, + { + "auxiliary_loss_clip": 0.01165649, + "auxiliary_loss_mlp": 0.01050304, + "balance_loss_clip": 1.05135536, + "balance_loss_mlp": 1.03026474, + "epoch": 0.2169181747129201, + "flos": 17384464010880.0, + "grad_norm": 2.5625682608279234, + "language_loss": 0.80382848, + "learning_rate": 3.6447072796214345e-06, + "loss": 0.82598805, + "num_input_tokens_seen": 38389795, + "router_z_loss_clip": 1.14453125, + "router_z_loss_mlp": 0.20043945, + "step": 1804, + "time_per_iteration": 2.855802059173584 + }, + { + "auxiliary_loss_clip": 0.01097081, + "auxiliary_loss_mlp": 0.01030007, + "balance_loss_clip": 1.04216754, + "balance_loss_mlp": 1.02657366, + "epoch": 0.21703841760355919, + "flos": 58760955429120.0, + "grad_norm": 0.9452097521518298, + "language_loss": 0.63111031, + "learning_rate": 3.644263938160898e-06, + "loss": 0.65238118, + "num_input_tokens_seen": 38445760, + "router_z_loss_clip": 0.55078125, + "router_z_loss_mlp": 0.03442383, + "step": 1805, + "time_per_iteration": 3.280999183654785 + }, + { + "auxiliary_loss_clip": 0.01170979, + "auxiliary_loss_mlp": 0.01045727, + "balance_loss_clip": 1.05263996, + "balance_loss_mlp": 1.02485335, + "epoch": 0.21715866049419827, + "flos": 22418457419520.0, + "grad_norm": 2.1834276531012136, + "language_loss": 0.72173202, + "learning_rate": 3.6438203472668293e-06, + "loss": 0.74389911, + "num_input_tokens_seen": 38465405, + "router_z_loss_clip": 1.18457031, + "router_z_loss_mlp": 0.20849609, + "step": 1806, + "time_per_iteration": 2.8645689487457275 + }, + { + "auxiliary_loss_clip": 0.0116874, + "auxiliary_loss_mlp": 0.01044409, + "balance_loss_clip": 1.05072784, + "balance_loss_mlp": 1.02435756, + "epoch": 0.21727890338483738, + "flos": 17237732952960.0, + "grad_norm": 4.04320886566886, + "language_loss": 0.81858218, + "learning_rate": 3.6433765070065206e-06, + "loss": 0.84071374, + "num_input_tokens_seen": 38483195, + "router_z_loss_clip": 1.1796875, + "router_z_loss_mlp": 0.20056152, + "step": 1807, + "time_per_iteration": 2.795340061187744 + }, + { + "auxiliary_loss_clip": 0.01168837, + "auxiliary_loss_mlp": 0.01044095, + "balance_loss_clip": 1.04812264, + "balance_loss_mlp": 1.02239931, + "epoch": 0.21739914627547646, + "flos": 13434792416640.0, + "grad_norm": 2.3696857316829503, + "language_loss": 0.87483412, + "learning_rate": 3.6429324174473025e-06, + "loss": 0.89696342, + "num_input_tokens_seen": 38496735, + "router_z_loss_clip": 1.20703125, + "router_z_loss_mlp": 0.21716309, + "step": 1808, + "time_per_iteration": 4.446065187454224 + }, + { + "auxiliary_loss_clip": 0.01168106, + "auxiliary_loss_mlp": 0.01043134, + "balance_loss_clip": 1.05159187, + "balance_loss_mlp": 1.02320266, + "epoch": 0.21751938916611555, + "flos": 20959514709120.0, + "grad_norm": 2.797502081717461, + "language_loss": 0.84814465, + "learning_rate": 3.6424880786565425e-06, + "loss": 0.87025702, + "num_input_tokens_seen": 38512880, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.19921875, + "step": 1809, + "time_per_iteration": 3.66904616355896 + }, + { + "auxiliary_loss_clip": 0.01173364, + "auxiliary_loss_mlp": 0.01052508, + "balance_loss_clip": 1.05214524, + "balance_loss_mlp": 1.02916634, + "epoch": 0.21763963205675466, + "flos": 27599936071680.0, + "grad_norm": 2.447651564494252, + "language_loss": 0.80046183, + "learning_rate": 3.6420434907016482e-06, + "loss": 0.82272053, + "num_input_tokens_seen": 38532570, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.23327637, + "step": 1810, + "time_per_iteration": 2.8213980197906494 + }, + { + "auxiliary_loss_clip": 0.01166755, + "auxiliary_loss_mlp": 0.0104604, + "balance_loss_clip": 1.0512706, + "balance_loss_mlp": 1.02449942, + "epoch": 0.21775987494739374, + "flos": 21430411032960.0, + "grad_norm": 2.570387965760973, + "language_loss": 0.81132782, + "learning_rate": 3.6415986536500606e-06, + "loss": 0.8334558, + "num_input_tokens_seen": 38550900, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.21557617, + "step": 1811, + "time_per_iteration": 3.5996694564819336 + }, + { + "auxiliary_loss_clip": 0.01170088, + "auxiliary_loss_mlp": 0.01053229, + "balance_loss_clip": 1.05438077, + "balance_loss_mlp": 1.03198564, + "epoch": 0.21788011783803282, + "flos": 18332972501760.0, + "grad_norm": 2.2400443377057786, + "language_loss": 0.80941176, + "learning_rate": 3.641153567569263e-06, + "loss": 0.83164489, + "num_input_tokens_seen": 38569215, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.2121582, + "step": 1812, + "time_per_iteration": 2.7705087661743164 + }, + { + "auxiliary_loss_clip": 0.01162054, + "auxiliary_loss_mlp": 0.01047323, + "balance_loss_clip": 1.04752791, + "balance_loss_mlp": 1.02683091, + "epoch": 0.2180003607286719, + "flos": 30262748037120.0, + "grad_norm": 3.322601881634483, + "language_loss": 0.95416272, + "learning_rate": 3.640708232526774e-06, + "loss": 0.97625649, + "num_input_tokens_seen": 38587870, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.20507812, + "step": 1813, + "time_per_iteration": 2.826273202896118 + }, + { + "auxiliary_loss_clip": 0.01169851, + "auxiliary_loss_mlp": 0.01052199, + "balance_loss_clip": 1.05074406, + "balance_loss_mlp": 1.03109932, + "epoch": 0.21812060361931102, + "flos": 25480272637440.0, + "grad_norm": 2.7209355289680457, + "language_loss": 0.77662635, + "learning_rate": 3.6402626485901504e-06, + "loss": 0.79884684, + "num_input_tokens_seen": 38606965, + "router_z_loss_clip": 1.19042969, + "router_z_loss_mlp": 0.2109375, + "step": 1814, + "time_per_iteration": 2.828118324279785 + }, + { + "auxiliary_loss_clip": 0.01171285, + "auxiliary_loss_mlp": 0.01056165, + "balance_loss_clip": 1.05470824, + "balance_loss_mlp": 1.03634012, + "epoch": 0.2182408465099501, + "flos": 21908166854400.0, + "grad_norm": 2.4912529750971926, + "language_loss": 0.77958065, + "learning_rate": 3.639816815826988e-06, + "loss": 0.80185515, + "num_input_tokens_seen": 38626290, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.19824219, + "step": 1815, + "time_per_iteration": 2.9176290035247803 + }, + { + "auxiliary_loss_clip": 0.01167008, + "auxiliary_loss_mlp": 0.01055666, + "balance_loss_clip": 1.05111337, + "balance_loss_mlp": 1.03479278, + "epoch": 0.21836108940058918, + "flos": 23657339456640.0, + "grad_norm": 3.040335711771826, + "language_loss": 0.78167248, + "learning_rate": 3.6393707343049176e-06, + "loss": 0.80389917, + "num_input_tokens_seen": 38646620, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.2088623, + "step": 1816, + "time_per_iteration": 2.962430953979492 + }, + { + "auxiliary_loss_clip": 0.01168017, + "auxiliary_loss_mlp": 0.01048795, + "balance_loss_clip": 1.05044913, + "balance_loss_mlp": 1.02837443, + "epoch": 0.2184813322912283, + "flos": 24681009156480.0, + "grad_norm": 2.644374912168785, + "language_loss": 0.73629838, + "learning_rate": 3.6389244040916104e-06, + "loss": 0.75846648, + "num_input_tokens_seen": 38665695, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.20422363, + "step": 1817, + "time_per_iteration": 2.780703067779541 + }, + { + "auxiliary_loss_clip": 0.011727, + "auxiliary_loss_mlp": 0.01051073, + "balance_loss_clip": 1.05238342, + "balance_loss_mlp": 1.02805352, + "epoch": 0.21860157518186737, + "flos": 26574650259840.0, + "grad_norm": 2.2627426875445567, + "language_loss": 0.79346168, + "learning_rate": 3.6384778252547747e-06, + "loss": 0.81569934, + "num_input_tokens_seen": 38681575, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.22998047, + "step": 1818, + "time_per_iteration": 2.7910072803497314 + }, + { + "auxiliary_loss_clip": 0.01172225, + "auxiliary_loss_mlp": 0.01046892, + "balance_loss_clip": 1.05491054, + "balance_loss_mlp": 1.02653146, + "epoch": 0.21872181807250646, + "flos": 20886292834560.0, + "grad_norm": 2.829944140913883, + "language_loss": 0.77593541, + "learning_rate": 3.638030997862155e-06, + "loss": 0.79812658, + "num_input_tokens_seen": 38700510, + "router_z_loss_clip": 1.17285156, + "router_z_loss_mlp": 0.20336914, + "step": 1819, + "time_per_iteration": 2.8147974014282227 + }, + { + "auxiliary_loss_clip": 0.01100841, + "auxiliary_loss_mlp": 0.01036071, + "balance_loss_clip": 1.0445075, + "balance_loss_mlp": 1.03280437, + "epoch": 0.21884206096314554, + "flos": 61209452897280.0, + "grad_norm": 0.7777342970212214, + "language_loss": 0.59466314, + "learning_rate": 3.6375839219815356e-06, + "loss": 0.61603236, + "num_input_tokens_seen": 38758310, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.03271484, + "step": 1820, + "time_per_iteration": 3.3098323345184326 + }, + { + "auxiliary_loss_clip": 0.01173001, + "auxiliary_loss_mlp": 0.0105354, + "balance_loss_clip": 1.05370736, + "balance_loss_mlp": 1.03195083, + "epoch": 0.21896230385378465, + "flos": 23473835850240.0, + "grad_norm": 2.186178998482852, + "language_loss": 0.83013666, + "learning_rate": 3.6371365976807375e-06, + "loss": 0.85240203, + "num_input_tokens_seen": 38778705, + "router_z_loss_clip": 1.19335938, + "router_z_loss_mlp": 0.21582031, + "step": 1821, + "time_per_iteration": 3.0198497772216797 + }, + { + "auxiliary_loss_clip": 0.01170298, + "auxiliary_loss_mlp": 0.01055291, + "balance_loss_clip": 1.05213773, + "balance_loss_mlp": 1.03599155, + "epoch": 0.21908254674442373, + "flos": 25081915829760.0, + "grad_norm": 2.0895691740177305, + "language_loss": 0.83394659, + "learning_rate": 3.6366890250276185e-06, + "loss": 0.85620248, + "num_input_tokens_seen": 38799660, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.19311523, + "step": 1822, + "time_per_iteration": 2.857060194015503 + }, + { + "auxiliary_loss_clip": 0.01169959, + "auxiliary_loss_mlp": 0.01056546, + "balance_loss_clip": 1.05048895, + "balance_loss_mlp": 1.03456342, + "epoch": 0.21920278963506282, + "flos": 23513768795520.0, + "grad_norm": 2.589478599808277, + "language_loss": 0.90306211, + "learning_rate": 3.6362412040900764e-06, + "loss": 0.92532718, + "num_input_tokens_seen": 38819450, + "router_z_loss_clip": 1.19433594, + "router_z_loss_mlp": 0.2199707, + "step": 1823, + "time_per_iteration": 2.8379592895507812 + }, + { + "auxiliary_loss_clip": 0.01170052, + "auxiliary_loss_mlp": 0.01061225, + "balance_loss_clip": 1.05282974, + "balance_loss_mlp": 1.03964829, + "epoch": 0.21932303252570193, + "flos": 29242238734080.0, + "grad_norm": 1.9948803578453043, + "language_loss": 0.80447245, + "learning_rate": 3.635793134936044e-06, + "loss": 0.82678527, + "num_input_tokens_seen": 38840460, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.21569824, + "step": 1824, + "time_per_iteration": 2.8991646766662598 + }, + { + "auxiliary_loss_clip": 0.01172434, + "auxiliary_loss_mlp": 0.0106024, + "balance_loss_clip": 1.05536926, + "balance_loss_mlp": 1.03981912, + "epoch": 0.219443275416341, + "flos": 20806857907200.0, + "grad_norm": 9.129951176586154, + "language_loss": 0.73539245, + "learning_rate": 3.635344817633494e-06, + "loss": 0.75771922, + "num_input_tokens_seen": 38859775, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.20410156, + "step": 1825, + "time_per_iteration": 2.9295637607574463 + }, + { + "auxiliary_loss_clip": 0.01169168, + "auxiliary_loss_mlp": 0.01053148, + "balance_loss_clip": 1.0524404, + "balance_loss_mlp": 1.03230989, + "epoch": 0.2195635183069801, + "flos": 14501555458560.0, + "grad_norm": 3.8088445127189328, + "language_loss": 0.75663781, + "learning_rate": 3.634896252250436e-06, + "loss": 0.77886099, + "num_input_tokens_seen": 38876540, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.20837402, + "step": 1826, + "time_per_iteration": 2.8167967796325684 + }, + { + "auxiliary_loss_clip": 0.01177706, + "auxiliary_loss_mlp": 0.01062302, + "balance_loss_clip": 1.05500948, + "balance_loss_mlp": 1.04194093, + "epoch": 0.2196837611976192, + "flos": 24243473589120.0, + "grad_norm": 1.865397650595095, + "language_loss": 0.82051885, + "learning_rate": 3.6344474388549157e-06, + "loss": 0.84291899, + "num_input_tokens_seen": 38896195, + "router_z_loss_clip": 1.2265625, + "router_z_loss_mlp": 0.20349121, + "step": 1827, + "time_per_iteration": 2.834606409072876 + }, + { + "auxiliary_loss_clip": 0.01177054, + "auxiliary_loss_mlp": 0.010559, + "balance_loss_clip": 1.05437064, + "balance_loss_mlp": 1.03454912, + "epoch": 0.2198040040882583, + "flos": 18074523168000.0, + "grad_norm": 2.8121725044794013, + "language_loss": 0.80332124, + "learning_rate": 3.6339983775150183e-06, + "loss": 0.82565081, + "num_input_tokens_seen": 38912755, + "router_z_loss_clip": 1.22558594, + "router_z_loss_mlp": 0.21350098, + "step": 1828, + "time_per_iteration": 2.785050630569458 + }, + { + "auxiliary_loss_clip": 0.01171806, + "auxiliary_loss_mlp": 0.01053498, + "balance_loss_clip": 1.054057, + "balance_loss_mlp": 1.03223062, + "epoch": 0.21992424697889737, + "flos": 17784185535360.0, + "grad_norm": 3.3095138289828356, + "language_loss": 0.84506738, + "learning_rate": 3.6335490682988664e-06, + "loss": 0.86732042, + "num_input_tokens_seen": 38928365, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.21264648, + "step": 1829, + "time_per_iteration": 2.7477142810821533 + }, + { + "auxiliary_loss_clip": 0.01180101, + "auxiliary_loss_mlp": 0.01063947, + "balance_loss_clip": 1.05809045, + "balance_loss_mlp": 1.0423702, + "epoch": 0.22004448986953645, + "flos": 17638495971840.0, + "grad_norm": 2.1799826783793534, + "language_loss": 0.827052, + "learning_rate": 3.63309951127462e-06, + "loss": 0.84949249, + "num_input_tokens_seen": 38945275, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.21557617, + "step": 1830, + "time_per_iteration": 2.7933688163757324 + }, + { + "auxiliary_loss_clip": 0.0117839, + "auxiliary_loss_mlp": 0.01061076, + "balance_loss_clip": 1.05428767, + "balance_loss_mlp": 1.03744912, + "epoch": 0.22016473276017556, + "flos": 22275533203200.0, + "grad_norm": 2.5703354670814282, + "language_loss": 0.75299942, + "learning_rate": 3.6326497065104757e-06, + "loss": 0.77539408, + "num_input_tokens_seen": 38965740, + "router_z_loss_clip": 1.24121094, + "router_z_loss_mlp": 0.23632812, + "step": 1831, + "time_per_iteration": 2.769307851791382 + }, + { + "auxiliary_loss_clip": 0.01175932, + "auxiliary_loss_mlp": 0.01058779, + "balance_loss_clip": 1.05571818, + "balance_loss_mlp": 1.03661776, + "epoch": 0.22028497565081465, + "flos": 25556259859200.0, + "grad_norm": 2.252131174393196, + "language_loss": 0.77956951, + "learning_rate": 3.6321996540746697e-06, + "loss": 0.8019166, + "num_input_tokens_seen": 38984815, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.22155762, + "step": 1832, + "time_per_iteration": 2.8442201614379883 + }, + { + "auxiliary_loss_clip": 0.01176386, + "auxiliary_loss_mlp": 0.0106218, + "balance_loss_clip": 1.05746818, + "balance_loss_mlp": 1.03973293, + "epoch": 0.22040521854145373, + "flos": 36247332925440.0, + "grad_norm": 2.0529927236963643, + "language_loss": 0.80021548, + "learning_rate": 3.6317493540354733e-06, + "loss": 0.82260114, + "num_input_tokens_seen": 39008230, + "router_z_loss_clip": 1.18945312, + "router_z_loss_mlp": 0.22460938, + "step": 1833, + "time_per_iteration": 2.9949538707733154 + }, + { + "auxiliary_loss_clip": 0.01177209, + "auxiliary_loss_mlp": 0.01060534, + "balance_loss_clip": 1.05309319, + "balance_loss_mlp": 1.03715706, + "epoch": 0.22052546143209284, + "flos": 11838420270720.0, + "grad_norm": 5.979220061942037, + "language_loss": 0.76994157, + "learning_rate": 3.6312988064611976e-06, + "loss": 0.792319, + "num_input_tokens_seen": 39026540, + "router_z_loss_clip": 1.23925781, + "router_z_loss_mlp": 0.23400879, + "step": 1834, + "time_per_iteration": 2.8759102821350098 + }, + { + "auxiliary_loss_clip": 0.01172013, + "auxiliary_loss_mlp": 0.01060831, + "balance_loss_clip": 1.05206203, + "balance_loss_mlp": 1.04013586, + "epoch": 0.22064570432273192, + "flos": 24209250906240.0, + "grad_norm": 1.828875151335685, + "language_loss": 0.81733513, + "learning_rate": 3.6308480114201896e-06, + "loss": 0.83966362, + "num_input_tokens_seen": 39048460, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.20703125, + "step": 1835, + "time_per_iteration": 2.867523670196533 + }, + { + "auxiliary_loss_clip": 0.01173523, + "auxiliary_loss_mlp": 0.01064278, + "balance_loss_clip": 1.05463588, + "balance_loss_mlp": 1.04270101, + "epoch": 0.220765947213371, + "flos": 17931347556480.0, + "grad_norm": 2.1572023383768277, + "language_loss": 0.7612108, + "learning_rate": 3.630396968980835e-06, + "loss": 0.78358883, + "num_input_tokens_seen": 39066335, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.21569824, + "step": 1836, + "time_per_iteration": 3.6402273178100586 + }, + { + "auxiliary_loss_clip": 0.0117805, + "auxiliary_loss_mlp": 0.01068936, + "balance_loss_clip": 1.05636668, + "balance_loss_mlp": 1.04514194, + "epoch": 0.2208861901040101, + "flos": 26757040544640.0, + "grad_norm": 2.6366066114063935, + "language_loss": 0.83346057, + "learning_rate": 3.6299456792115575e-06, + "loss": 0.85593045, + "num_input_tokens_seen": 39087590, + "router_z_loss_clip": 1.21777344, + "router_z_loss_mlp": 0.23803711, + "step": 1837, + "time_per_iteration": 3.7040863037109375 + }, + { + "auxiliary_loss_clip": 0.01175018, + "auxiliary_loss_mlp": 0.01054402, + "balance_loss_clip": 1.05461013, + "balance_loss_mlp": 1.03181207, + "epoch": 0.2210064329946492, + "flos": 17817977255040.0, + "grad_norm": 3.618074069145801, + "language_loss": 0.81237322, + "learning_rate": 3.629494142180815e-06, + "loss": 0.83466738, + "num_input_tokens_seen": 39106335, + "router_z_loss_clip": 1.20507812, + "router_z_loss_mlp": 0.22583008, + "step": 1838, + "time_per_iteration": 3.630965232849121 + }, + { + "auxiliary_loss_clip": 0.01172514, + "auxiliary_loss_mlp": 0.0104901, + "balance_loss_clip": 1.05339026, + "balance_loss_mlp": 1.02878034, + "epoch": 0.22112667588528828, + "flos": 17967401832960.0, + "grad_norm": 2.7135416713908875, + "language_loss": 0.84648263, + "learning_rate": 3.6290423579571075e-06, + "loss": 0.86869788, + "num_input_tokens_seen": 39122875, + "router_z_loss_clip": 1.18945312, + "router_z_loss_mlp": 0.20239258, + "step": 1839, + "time_per_iteration": 3.6622140407562256 + }, + { + "auxiliary_loss_clip": 0.01176309, + "auxiliary_loss_mlp": 0.01065186, + "balance_loss_clip": 1.05518746, + "balance_loss_mlp": 1.04112983, + "epoch": 0.22124691877592736, + "flos": 18369206346240.0, + "grad_norm": 2.134389365113573, + "language_loss": 0.80094814, + "learning_rate": 3.6285903266089694e-06, + "loss": 0.82336307, + "num_input_tokens_seen": 39142150, + "router_z_loss_clip": 1.21191406, + "router_z_loss_mlp": 0.24035645, + "step": 1840, + "time_per_iteration": 2.8450870513916016 + }, + { + "auxiliary_loss_clip": 0.01173453, + "auxiliary_loss_mlp": 0.01054373, + "balance_loss_clip": 1.05096853, + "balance_loss_mlp": 1.03209329, + "epoch": 0.22136716166656648, + "flos": 20813286441600.0, + "grad_norm": 2.376134952510518, + "language_loss": 0.77288675, + "learning_rate": 3.628138048204974e-06, + "loss": 0.79516494, + "num_input_tokens_seen": 39162835, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.22290039, + "step": 1841, + "time_per_iteration": 2.8071329593658447 + }, + { + "auxiliary_loss_clip": 0.01179315, + "auxiliary_loss_mlp": 0.01051283, + "balance_loss_clip": 1.05533361, + "balance_loss_mlp": 1.027668, + "epoch": 0.22148740455720556, + "flos": 17675699483520.0, + "grad_norm": 2.05095492187532, + "language_loss": 0.76082295, + "learning_rate": 3.6276855228137304e-06, + "loss": 0.78312892, + "num_input_tokens_seen": 39181040, + "router_z_loss_clip": 1.23925781, + "router_z_loss_mlp": 0.2364502, + "step": 1842, + "time_per_iteration": 2.8161518573760986 + }, + { + "auxiliary_loss_clip": 0.01174007, + "auxiliary_loss_mlp": 0.01050947, + "balance_loss_clip": 1.05425406, + "balance_loss_mlp": 1.02742672, + "epoch": 0.22160764744784464, + "flos": 21726710323200.0, + "grad_norm": 2.6252638943383073, + "language_loss": 0.81892788, + "learning_rate": 3.6272327505038874e-06, + "loss": 0.84117746, + "num_input_tokens_seen": 39197505, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.23535156, + "step": 1843, + "time_per_iteration": 2.810844659805298 + }, + { + "auxiliary_loss_clip": 0.0117172, + "auxiliary_loss_mlp": 0.01049257, + "balance_loss_clip": 1.05269444, + "balance_loss_mlp": 1.02756071, + "epoch": 0.22172789033848372, + "flos": 23764712186880.0, + "grad_norm": 3.623176543501318, + "language_loss": 0.78431094, + "learning_rate": 3.626779731344131e-06, + "loss": 0.8065207, + "num_input_tokens_seen": 39217295, + "router_z_loss_clip": 1.19140625, + "router_z_loss_mlp": 0.21704102, + "step": 1844, + "time_per_iteration": 2.859506130218506 + }, + { + "auxiliary_loss_clip": 0.01167447, + "auxiliary_loss_mlp": 0.01045389, + "balance_loss_clip": 1.05097151, + "balance_loss_mlp": 1.02447939, + "epoch": 0.22184813322912283, + "flos": 16982300361600.0, + "grad_norm": 4.724772549898272, + "language_loss": 0.85148537, + "learning_rate": 3.6263264654031814e-06, + "loss": 0.87361377, + "num_input_tokens_seen": 39234195, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.20922852, + "step": 1845, + "time_per_iteration": 2.927419662475586 + }, + { + "auxiliary_loss_clip": 0.01099397, + "auxiliary_loss_mlp": 0.01020717, + "balance_loss_clip": 1.04926562, + "balance_loss_mlp": 1.01728368, + "epoch": 0.22196837611976192, + "flos": 61823740314240.0, + "grad_norm": 0.7236323649349802, + "language_loss": 0.59098029, + "learning_rate": 3.6258729527498008e-06, + "loss": 0.61218143, + "num_input_tokens_seen": 39295040, + "router_z_loss_clip": 0.5, + "router_z_loss_mlp": 0.03442383, + "step": 1846, + "time_per_iteration": 3.3489556312561035 + }, + { + "auxiliary_loss_clip": 0.01170381, + "auxiliary_loss_mlp": 0.01050259, + "balance_loss_clip": 1.05115879, + "balance_loss_mlp": 1.0285275, + "epoch": 0.222088619010401, + "flos": 25558019625600.0, + "grad_norm": 4.313169322984642, + "language_loss": 0.64658672, + "learning_rate": 3.6254191934527854e-06, + "loss": 0.66879308, + "num_input_tokens_seen": 39314395, + "router_z_loss_clip": 1.19335938, + "router_z_loss_mlp": 0.21728516, + "step": 1847, + "time_per_iteration": 2.8100101947784424 + }, + { + "auxiliary_loss_clip": 0.01173303, + "auxiliary_loss_mlp": 0.01056104, + "balance_loss_clip": 1.05205548, + "balance_loss_mlp": 1.03211904, + "epoch": 0.2222088619010401, + "flos": 19318612677120.0, + "grad_norm": 2.822093438053974, + "language_loss": 0.64484626, + "learning_rate": 3.6249651875809715e-06, + "loss": 0.66714031, + "num_input_tokens_seen": 39334275, + "router_z_loss_clip": 1.2109375, + "router_z_loss_mlp": 0.23962402, + "step": 1848, + "time_per_iteration": 2.7772793769836426 + }, + { + "auxiliary_loss_clip": 0.01172568, + "auxiliary_loss_mlp": 0.01056209, + "balance_loss_clip": 1.05487204, + "balance_loss_mlp": 1.03421545, + "epoch": 0.2223291047916792, + "flos": 19099342103040.0, + "grad_norm": 2.494330117713578, + "language_loss": 0.89323235, + "learning_rate": 3.62451093520323e-06, + "loss": 0.91552013, + "num_input_tokens_seen": 39352180, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.21984863, + "step": 1849, + "time_per_iteration": 2.7938666343688965 + }, + { + "auxiliary_loss_clip": 0.0116822, + "auxiliary_loss_mlp": 0.01041892, + "balance_loss_clip": 1.04943442, + "balance_loss_mlp": 1.02042186, + "epoch": 0.22244934768231828, + "flos": 20850418126080.0, + "grad_norm": 2.2853337231414663, + "language_loss": 0.90398055, + "learning_rate": 3.6240564363884714e-06, + "loss": 0.92608166, + "num_input_tokens_seen": 39372125, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.21472168, + "step": 1850, + "time_per_iteration": 2.7673346996307373 + }, + { + "auxiliary_loss_clip": 0.01169453, + "auxiliary_loss_mlp": 0.01055291, + "balance_loss_clip": 1.04936111, + "balance_loss_mlp": 1.0316515, + "epoch": 0.2225695905729574, + "flos": 15632921111040.0, + "grad_norm": 2.236093163408111, + "language_loss": 0.69984883, + "learning_rate": 3.623601691205643e-06, + "loss": 0.72209626, + "num_input_tokens_seen": 39391200, + "router_z_loss_clip": 1.20019531, + "router_z_loss_mlp": 0.23608398, + "step": 1851, + "time_per_iteration": 2.7569077014923096 + }, + { + "auxiliary_loss_clip": 0.01169783, + "auxiliary_loss_mlp": 0.01062917, + "balance_loss_clip": 1.0519979, + "balance_loss_mlp": 1.04139996, + "epoch": 0.22268983346359647, + "flos": 25373582265600.0, + "grad_norm": 2.369922881855373, + "language_loss": 0.81383705, + "learning_rate": 3.623146699723729e-06, + "loss": 0.83616406, + "num_input_tokens_seen": 39410660, + "router_z_loss_clip": 1.17773438, + "router_z_loss_mlp": 0.21508789, + "step": 1852, + "time_per_iteration": 2.826479911804199 + }, + { + "auxiliary_loss_clip": 0.01169986, + "auxiliary_loss_mlp": 0.01056372, + "balance_loss_clip": 1.05332112, + "balance_loss_mlp": 1.03453314, + "epoch": 0.22281007635423555, + "flos": 13261452359040.0, + "grad_norm": 2.0438678706863542, + "language_loss": 0.77493542, + "learning_rate": 3.6226914620117507e-06, + "loss": 0.79719901, + "num_input_tokens_seen": 39429280, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.21850586, + "step": 1853, + "time_per_iteration": 2.7714197635650635 + }, + { + "auxiliary_loss_clip": 0.01167837, + "auxiliary_loss_mlp": 0.0105251, + "balance_loss_clip": 1.05028713, + "balance_loss_mlp": 1.03186321, + "epoch": 0.22293031924487464, + "flos": 15340536403200.0, + "grad_norm": 2.4096472409711653, + "language_loss": 0.8041926, + "learning_rate": 3.622235978138768e-06, + "loss": 0.82639611, + "num_input_tokens_seen": 39446905, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.20654297, + "step": 1854, + "time_per_iteration": 2.786745071411133 + }, + { + "auxiliary_loss_clip": 0.01175536, + "auxiliary_loss_mlp": 0.01056638, + "balance_loss_clip": 1.0542177, + "balance_loss_mlp": 1.03513288, + "epoch": 0.22305056213551375, + "flos": 22564649773440.0, + "grad_norm": 2.0685547878980226, + "language_loss": 0.81546462, + "learning_rate": 3.621780248173877e-06, + "loss": 0.83778638, + "num_input_tokens_seen": 39465105, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.21508789, + "step": 1855, + "time_per_iteration": 2.781240224838257 + }, + { + "auxiliary_loss_clip": 0.01084183, + "auxiliary_loss_mlp": 0.01021165, + "balance_loss_clip": 1.03504348, + "balance_loss_mlp": 1.01792288, + "epoch": 0.22317080502615283, + "flos": 64880419887360.0, + "grad_norm": 0.8446998830338661, + "language_loss": 0.61100215, + "learning_rate": 3.6213242721862125e-06, + "loss": 0.63205564, + "num_input_tokens_seen": 39523560, + "router_z_loss_clip": 0.4921875, + "router_z_loss_mlp": 0.0324707, + "step": 1856, + "time_per_iteration": 3.400954008102417 + }, + { + "auxiliary_loss_clip": 0.01168758, + "auxiliary_loss_mlp": 0.01053043, + "balance_loss_clip": 1.05439246, + "balance_loss_mlp": 1.03137124, + "epoch": 0.2232910479167919, + "flos": 25775997310080.0, + "grad_norm": 3.009312865356955, + "language_loss": 0.75527602, + "learning_rate": 3.620868050244945e-06, + "loss": 0.77749407, + "num_input_tokens_seen": 39544040, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.2166748, + "step": 1857, + "time_per_iteration": 2.8875796794891357 + }, + { + "auxiliary_loss_clip": 0.01178858, + "auxiliary_loss_mlp": 0.01055776, + "balance_loss_clip": 1.05425012, + "balance_loss_mlp": 1.03299522, + "epoch": 0.22341129080743102, + "flos": 23251799928960.0, + "grad_norm": 2.489047740294057, + "language_loss": 0.77271813, + "learning_rate": 3.6204115824192817e-06, + "loss": 0.79506445, + "num_input_tokens_seen": 39561515, + "router_z_loss_clip": 1.24511719, + "router_z_loss_mlp": 0.22802734, + "step": 1858, + "time_per_iteration": 2.8328607082366943 + }, + { + "auxiliary_loss_clip": 0.01173583, + "auxiliary_loss_mlp": 0.01056262, + "balance_loss_clip": 1.05231655, + "balance_loss_mlp": 1.03187156, + "epoch": 0.2235315336980701, + "flos": 21214552250880.0, + "grad_norm": 3.1394265136734916, + "language_loss": 0.77077591, + "learning_rate": 3.619954868778471e-06, + "loss": 0.79307437, + "num_input_tokens_seen": 39578210, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.24401855, + "step": 1859, + "time_per_iteration": 2.7925214767456055 + }, + { + "auxiliary_loss_clip": 0.01173484, + "auxiliary_loss_mlp": 0.01061865, + "balance_loss_clip": 1.05451667, + "balance_loss_mlp": 1.04099131, + "epoch": 0.2236517765887092, + "flos": 19901945548800.0, + "grad_norm": 2.2647107154471127, + "language_loss": 0.82801658, + "learning_rate": 3.6194979093917944e-06, + "loss": 0.85037005, + "num_input_tokens_seen": 39597625, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.2088623, + "step": 1860, + "time_per_iteration": 2.957092046737671 + }, + { + "auxiliary_loss_clip": 0.01170791, + "auxiliary_loss_mlp": 0.01054632, + "balance_loss_clip": 1.0536952, + "balance_loss_mlp": 1.03288841, + "epoch": 0.22377201947934827, + "flos": 23214847812480.0, + "grad_norm": 4.189314141150163, + "language_loss": 0.87346721, + "learning_rate": 3.6190407043285724e-06, + "loss": 0.89572138, + "num_input_tokens_seen": 39615360, + "router_z_loss_clip": 1.17285156, + "router_z_loss_mlp": 0.21728516, + "step": 1861, + "time_per_iteration": 2.8264365196228027 + }, + { + "auxiliary_loss_clip": 0.0117927, + "auxiliary_loss_mlp": 0.01050999, + "balance_loss_clip": 1.05617905, + "balance_loss_mlp": 1.02802753, + "epoch": 0.22389226236998738, + "flos": 26794244056320.0, + "grad_norm": 1.8952124762828217, + "language_loss": 0.75679994, + "learning_rate": 3.618583253658163e-06, + "loss": 0.77910256, + "num_input_tokens_seen": 39635460, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.22949219, + "step": 1862, + "time_per_iteration": 2.8625645637512207 + }, + { + "auxiliary_loss_clip": 0.01171095, + "auxiliary_loss_mlp": 0.01051065, + "balance_loss_clip": 1.05340111, + "balance_loss_mlp": 1.02888036, + "epoch": 0.22401250526062647, + "flos": 24170359455360.0, + "grad_norm": 4.374689451413402, + "language_loss": 0.86516976, + "learning_rate": 3.618125557449961e-06, + "loss": 0.88739133, + "num_input_tokens_seen": 39653515, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.22192383, + "step": 1863, + "time_per_iteration": 2.862004280090332 + }, + { + "auxiliary_loss_clip": 0.01169414, + "auxiliary_loss_mlp": 0.01050049, + "balance_loss_clip": 1.05156851, + "balance_loss_mlp": 1.02712512, + "epoch": 0.22413274815126555, + "flos": 16759761649920.0, + "grad_norm": 2.366210998313254, + "language_loss": 0.83360887, + "learning_rate": 3.6176676157733983e-06, + "loss": 0.85580349, + "num_input_tokens_seen": 39668525, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.22937012, + "step": 1864, + "time_per_iteration": 2.8331382274627686 + }, + { + "auxiliary_loss_clip": 0.0117323, + "auxiliary_loss_mlp": 0.01055078, + "balance_loss_clip": 1.0527333, + "balance_loss_mlp": 1.03316724, + "epoch": 0.22425299104190466, + "flos": 21360205900800.0, + "grad_norm": 3.0553240021712025, + "language_loss": 0.75851691, + "learning_rate": 3.6172094286979443e-06, + "loss": 0.78079993, + "num_input_tokens_seen": 39685895, + "router_z_loss_clip": 1.20605469, + "router_z_loss_mlp": 0.21899414, + "step": 1865, + "time_per_iteration": 3.728037118911743 + }, + { + "auxiliary_loss_clip": 0.01168521, + "auxiliary_loss_mlp": 0.01040355, + "balance_loss_clip": 1.05076098, + "balance_loss_mlp": 1.01905203, + "epoch": 0.22437323393254374, + "flos": 32165547108480.0, + "grad_norm": 1.9379043414242247, + "language_loss": 0.81501508, + "learning_rate": 3.6167509962931064e-06, + "loss": 0.83710384, + "num_input_tokens_seen": 39711595, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.2130127, + "step": 1866, + "time_per_iteration": 3.7162930965423584 + }, + { + "auxiliary_loss_clip": 0.01175415, + "auxiliary_loss_mlp": 0.01045736, + "balance_loss_clip": 1.05522025, + "balance_loss_mlp": 1.02299058, + "epoch": 0.22449347682318282, + "flos": 18002809664640.0, + "grad_norm": 7.556252733815296, + "language_loss": 0.76861143, + "learning_rate": 3.6162923186284276e-06, + "loss": 0.79082292, + "num_input_tokens_seen": 39727555, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.22741699, + "step": 1867, + "time_per_iteration": 3.687222719192505 + }, + { + "auxiliary_loss_clip": 0.01169885, + "auxiliary_loss_mlp": 0.01051564, + "balance_loss_clip": 1.0512507, + "balance_loss_mlp": 1.02840137, + "epoch": 0.2246137197138219, + "flos": 18697286194560.0, + "grad_norm": 2.1937642823079537, + "language_loss": 0.86013377, + "learning_rate": 3.6158333957734888e-06, + "loss": 0.8823483, + "num_input_tokens_seen": 39746145, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.23181152, + "step": 1868, + "time_per_iteration": 3.6661548614501953 + }, + { + "auxiliary_loss_clip": 0.01168008, + "auxiliary_loss_mlp": 0.01058684, + "balance_loss_clip": 1.05292511, + "balance_loss_mlp": 1.03852606, + "epoch": 0.22473396260446102, + "flos": 15590653781760.0, + "grad_norm": 2.38990399077141, + "language_loss": 0.83000886, + "learning_rate": 3.6153742277979088e-06, + "loss": 0.85227573, + "num_input_tokens_seen": 39763575, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.20166016, + "step": 1869, + "time_per_iteration": 2.796008348464966 + }, + { + "auxiliary_loss_clip": 0.01171176, + "auxiliary_loss_mlp": 0.01048971, + "balance_loss_clip": 1.05294502, + "balance_loss_mlp": 1.02943265, + "epoch": 0.2248542054951001, + "flos": 14465501182080.0, + "grad_norm": 2.12438965092086, + "language_loss": 0.77990496, + "learning_rate": 3.6149148147713434e-06, + "loss": 0.80210644, + "num_input_tokens_seen": 39781810, + "router_z_loss_clip": 1.18359375, + "router_z_loss_mlp": 0.1953125, + "step": 1870, + "time_per_iteration": 2.7897675037384033 + }, + { + "auxiliary_loss_clip": 0.01173522, + "auxiliary_loss_mlp": 0.01049709, + "balance_loss_clip": 1.05570877, + "balance_loss_mlp": 1.02776289, + "epoch": 0.22497444838573918, + "flos": 19243882431360.0, + "grad_norm": 2.0314121350095324, + "language_loss": 0.86424041, + "learning_rate": 3.614455156763484e-06, + "loss": 0.8864727, + "num_input_tokens_seen": 39800115, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.21948242, + "step": 1871, + "time_per_iteration": 2.755356550216675 + }, + { + "auxiliary_loss_clip": 0.01170583, + "auxiliary_loss_mlp": 0.01052951, + "balance_loss_clip": 1.05181181, + "balance_loss_mlp": 1.02978837, + "epoch": 0.2250946912763783, + "flos": 16910299549440.0, + "grad_norm": 10.683763552679613, + "language_loss": 0.71157241, + "learning_rate": 3.613995253844061e-06, + "loss": 0.73380768, + "num_input_tokens_seen": 39817795, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.23168945, + "step": 1872, + "time_per_iteration": 2.771677017211914 + }, + { + "auxiliary_loss_clip": 0.01170981, + "auxiliary_loss_mlp": 0.01046795, + "balance_loss_clip": 1.05236292, + "balance_loss_mlp": 1.02365685, + "epoch": 0.22521493416701738, + "flos": 24681368292480.0, + "grad_norm": 2.661090743927626, + "language_loss": 0.80869794, + "learning_rate": 3.6135351060828414e-06, + "loss": 0.83087575, + "num_input_tokens_seen": 39838270, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.23132324, + "step": 1873, + "time_per_iteration": 2.8249831199645996 + }, + { + "auxiliary_loss_clip": 0.01179561, + "auxiliary_loss_mlp": 0.01051792, + "balance_loss_clip": 1.05609012, + "balance_loss_mlp": 1.0288924, + "epoch": 0.22533517705765646, + "flos": 17821963664640.0, + "grad_norm": 2.6454923881458106, + "language_loss": 0.69315124, + "learning_rate": 3.6130747135496285e-06, + "loss": 0.71546477, + "num_input_tokens_seen": 39857270, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.22900391, + "step": 1874, + "time_per_iteration": 2.771263837814331 + }, + { + "auxiliary_loss_clip": 0.01174281, + "auxiliary_loss_mlp": 0.01048402, + "balance_loss_clip": 1.05637646, + "balance_loss_mlp": 1.02611017, + "epoch": 0.22545541994829554, + "flos": 33691390899840.0, + "grad_norm": 1.937531762080876, + "language_loss": 0.65842158, + "learning_rate": 3.6126140763142646e-06, + "loss": 0.68064845, + "num_input_tokens_seen": 39882300, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.22290039, + "step": 1875, + "time_per_iteration": 2.862835168838501 + }, + { + "auxiliary_loss_clip": 0.01178748, + "auxiliary_loss_mlp": 0.01048885, + "balance_loss_clip": 1.05803621, + "balance_loss_mlp": 1.02629495, + "epoch": 0.22557566283893465, + "flos": 19171594310400.0, + "grad_norm": 3.2393349370151996, + "language_loss": 0.86225092, + "learning_rate": 3.6121531944466275e-06, + "loss": 0.88452721, + "num_input_tokens_seen": 39899625, + "router_z_loss_clip": 1.20605469, + "router_z_loss_mlp": 0.22583008, + "step": 1876, + "time_per_iteration": 2.889181137084961 + }, + { + "auxiliary_loss_clip": 0.01174171, + "auxiliary_loss_mlp": 0.01049033, + "balance_loss_clip": 1.05740607, + "balance_loss_mlp": 1.02716982, + "epoch": 0.22569590572957374, + "flos": 20773281669120.0, + "grad_norm": 2.402433090680985, + "language_loss": 0.78235281, + "learning_rate": 3.611692068016633e-06, + "loss": 0.80458486, + "num_input_tokens_seen": 39915955, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.21875, + "step": 1877, + "time_per_iteration": 2.98159122467041 + }, + { + "auxiliary_loss_clip": 0.01174545, + "auxiliary_loss_mlp": 0.01058217, + "balance_loss_clip": 1.05282223, + "balance_loss_mlp": 1.03594911, + "epoch": 0.22581614862021282, + "flos": 18442715529600.0, + "grad_norm": 2.551926641701302, + "language_loss": 0.75040042, + "learning_rate": 3.611230697094233e-06, + "loss": 0.77272809, + "num_input_tokens_seen": 39932655, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.22265625, + "step": 1878, + "time_per_iteration": 2.8842551708221436 + }, + { + "auxiliary_loss_clip": 0.01173778, + "auxiliary_loss_mlp": 0.01054657, + "balance_loss_clip": 1.05705738, + "balance_loss_mlp": 1.03166139, + "epoch": 0.22593639151085193, + "flos": 20048389297920.0, + "grad_norm": 2.5915973418939195, + "language_loss": 0.87557286, + "learning_rate": 3.6107690817494173e-06, + "loss": 0.89785725, + "num_input_tokens_seen": 39952875, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.23010254, + "step": 1879, + "time_per_iteration": 2.8589823246002197 + }, + { + "auxiliary_loss_clip": 0.0117569, + "auxiliary_loss_mlp": 0.01057211, + "balance_loss_clip": 1.05541658, + "balance_loss_mlp": 1.03426361, + "epoch": 0.226056634401491, + "flos": 13115116350720.0, + "grad_norm": 3.2465084313788983, + "language_loss": 0.70367885, + "learning_rate": 3.6103072220522117e-06, + "loss": 0.72600788, + "num_input_tokens_seen": 39968405, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.22949219, + "step": 1880, + "time_per_iteration": 2.938310384750366 + }, + { + "auxiliary_loss_clip": 0.01174754, + "auxiliary_loss_mlp": 0.01053392, + "balance_loss_clip": 1.05744731, + "balance_loss_mlp": 1.03115964, + "epoch": 0.2261768772921301, + "flos": 18988378012800.0, + "grad_norm": 1.9589978074903092, + "language_loss": 0.91508865, + "learning_rate": 3.609845118072682e-06, + "loss": 0.93737018, + "num_input_tokens_seen": 39987075, + "router_z_loss_clip": 1.17285156, + "router_z_loss_mlp": 0.22241211, + "step": 1881, + "time_per_iteration": 2.8806803226470947 + }, + { + "auxiliary_loss_clip": 0.01174949, + "auxiliary_loss_mlp": 0.01048841, + "balance_loss_clip": 1.05580997, + "balance_loss_mlp": 1.0260129, + "epoch": 0.2262971201827692, + "flos": 19974054101760.0, + "grad_norm": 1.8209192193831714, + "language_loss": 0.79860139, + "learning_rate": 3.6093827698809276e-06, + "loss": 0.82083929, + "num_input_tokens_seen": 40006175, + "router_z_loss_clip": 1.18945312, + "router_z_loss_mlp": 0.22814941, + "step": 1882, + "time_per_iteration": 2.8383419513702393 + }, + { + "auxiliary_loss_clip": 0.01170568, + "auxiliary_loss_mlp": 0.01047203, + "balance_loss_clip": 1.05444956, + "balance_loss_mlp": 1.0263418, + "epoch": 0.2264173630734083, + "flos": 16654543735680.0, + "grad_norm": 2.350164089302661, + "language_loss": 0.84079617, + "learning_rate": 3.6089201775470864e-06, + "loss": 0.86297393, + "num_input_tokens_seen": 40021630, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.20849609, + "step": 1883, + "time_per_iteration": 2.8043525218963623 + }, + { + "auxiliary_loss_clip": 0.01173257, + "auxiliary_loss_mlp": 0.01053036, + "balance_loss_clip": 1.05432796, + "balance_loss_mlp": 1.03043389, + "epoch": 0.22653760596404737, + "flos": 24389809597440.0, + "grad_norm": 2.040233526906506, + "language_loss": 0.77451026, + "learning_rate": 3.6084573411413334e-06, + "loss": 0.79677314, + "num_input_tokens_seen": 40041025, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.22595215, + "step": 1884, + "time_per_iteration": 2.8415820598602295 + }, + { + "auxiliary_loss_clip": 0.01184114, + "auxiliary_loss_mlp": 0.01055023, + "balance_loss_clip": 1.06020916, + "balance_loss_mlp": 1.0319438, + "epoch": 0.22665784885468646, + "flos": 18332541538560.0, + "grad_norm": 2.1406638156650075, + "language_loss": 0.81000501, + "learning_rate": 3.607994260733881e-06, + "loss": 0.83239639, + "num_input_tokens_seen": 40060265, + "router_z_loss_clip": 1.23828125, + "router_z_loss_mlp": 0.23095703, + "step": 1885, + "time_per_iteration": 2.7835566997528076 + }, + { + "auxiliary_loss_clip": 0.01172741, + "auxiliary_loss_mlp": 0.01042337, + "balance_loss_clip": 1.05647159, + "balance_loss_mlp": 1.02244139, + "epoch": 0.22677809174532557, + "flos": 24058102475520.0, + "grad_norm": 2.088945590042571, + "language_loss": 0.74425918, + "learning_rate": 3.6075309363949776e-06, + "loss": 0.76640993, + "num_input_tokens_seen": 40079435, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.19885254, + "step": 1886, + "time_per_iteration": 2.7995424270629883 + }, + { + "auxiliary_loss_clip": 0.01177014, + "auxiliary_loss_mlp": 0.01057447, + "balance_loss_clip": 1.05677712, + "balance_loss_mlp": 1.0354054, + "epoch": 0.22689833463596465, + "flos": 20374242503040.0, + "grad_norm": 2.2172587041520373, + "language_loss": 0.81488347, + "learning_rate": 3.6070673681949094e-06, + "loss": 0.83722806, + "num_input_tokens_seen": 40097800, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.22045898, + "step": 1887, + "time_per_iteration": 2.8820600509643555 + }, + { + "auxiliary_loss_clip": 0.01175937, + "auxiliary_loss_mlp": 0.01057511, + "balance_loss_clip": 1.05880499, + "balance_loss_mlp": 1.03530288, + "epoch": 0.22701857752660373, + "flos": 30120398438400.0, + "grad_norm": 2.1506767607051787, + "language_loss": 0.81177169, + "learning_rate": 3.606603556203999e-06, + "loss": 0.83410615, + "num_input_tokens_seen": 40122745, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.22216797, + "step": 1888, + "time_per_iteration": 2.8385605812072754 + }, + { + "auxiliary_loss_clip": 0.0117131, + "auxiliary_loss_mlp": 0.01055838, + "balance_loss_clip": 1.05409682, + "balance_loss_mlp": 1.03528666, + "epoch": 0.22713882041724284, + "flos": 22492182084480.0, + "grad_norm": 1.8079811625568083, + "language_loss": 0.83507365, + "learning_rate": 3.6061395004926066e-06, + "loss": 0.8573451, + "num_input_tokens_seen": 40141680, + "router_z_loss_clip": 1.17382812, + "router_z_loss_mlp": 0.20556641, + "step": 1889, + "time_per_iteration": 2.7850749492645264 + }, + { + "auxiliary_loss_clip": 0.01175578, + "auxiliary_loss_mlp": 0.01049455, + "balance_loss_clip": 1.05431294, + "balance_loss_mlp": 1.02701998, + "epoch": 0.22725906330788193, + "flos": 20521548178560.0, + "grad_norm": 2.474577331576255, + "language_loss": 0.85176063, + "learning_rate": 3.605675201131129e-06, + "loss": 0.87401092, + "num_input_tokens_seen": 40160140, + "router_z_loss_clip": 1.2109375, + "router_z_loss_mlp": 0.2244873, + "step": 1890, + "time_per_iteration": 2.838308334350586 + }, + { + "auxiliary_loss_clip": 0.01179346, + "auxiliary_loss_mlp": 0.0105629, + "balance_loss_clip": 1.06000876, + "balance_loss_mlp": 1.03529763, + "epoch": 0.227379306198521, + "flos": 18989922297600.0, + "grad_norm": 2.5257653310890102, + "language_loss": 0.78695858, + "learning_rate": 3.60521065819e-06, + "loss": 0.80931491, + "num_input_tokens_seen": 40177450, + "router_z_loss_clip": 1.19335938, + "router_z_loss_mlp": 0.20996094, + "step": 1891, + "time_per_iteration": 2.8285107612609863 + }, + { + "auxiliary_loss_clip": 0.01176564, + "auxiliary_loss_mlp": 0.0106281, + "balance_loss_clip": 1.05715036, + "balance_loss_mlp": 1.0409472, + "epoch": 0.2274995490891601, + "flos": 21798351999360.0, + "grad_norm": 2.724331156597194, + "language_loss": 0.87581843, + "learning_rate": 3.60474587173969e-06, + "loss": 0.89821219, + "num_input_tokens_seen": 40195935, + "router_z_loss_clip": 1.19433594, + "router_z_loss_mlp": 0.21875, + "step": 1892, + "time_per_iteration": 2.8345000743865967 + }, + { + "auxiliary_loss_clip": 0.01175664, + "auxiliary_loss_mlp": 0.01050783, + "balance_loss_clip": 1.05646312, + "balance_loss_mlp": 1.02919436, + "epoch": 0.2276197919797992, + "flos": 19058654972160.0, + "grad_norm": 16.602829086479826, + "language_loss": 0.84235585, + "learning_rate": 3.6042808418507084e-06, + "loss": 0.86462033, + "num_input_tokens_seen": 40213620, + "router_z_loss_clip": 1.19140625, + "router_z_loss_mlp": 0.21582031, + "step": 1893, + "time_per_iteration": 2.8289382457733154 + }, + { + "auxiliary_loss_clip": 0.01182352, + "auxiliary_loss_mlp": 0.0105509, + "balance_loss_clip": 1.0607717, + "balance_loss_mlp": 1.03164101, + "epoch": 0.22774003487043828, + "flos": 18806777827200.0, + "grad_norm": 2.495617530543809, + "language_loss": 0.77169257, + "learning_rate": 3.6038155685935976e-06, + "loss": 0.79406703, + "num_input_tokens_seen": 40230190, + "router_z_loss_clip": 1.21679688, + "router_z_loss_mlp": 0.23461914, + "step": 1894, + "time_per_iteration": 3.7752878665924072 + }, + { + "auxiliary_loss_clip": 0.01173746, + "auxiliary_loss_mlp": 0.01052549, + "balance_loss_clip": 1.05423987, + "balance_loss_mlp": 1.02942193, + "epoch": 0.22786027776107737, + "flos": 23002544476800.0, + "grad_norm": 2.7505589726005115, + "language_loss": 0.7016685, + "learning_rate": 3.6033500520389404e-06, + "loss": 0.72393143, + "num_input_tokens_seen": 40246860, + "router_z_loss_clip": 1.19433594, + "router_z_loss_mlp": 0.23120117, + "step": 1895, + "time_per_iteration": 2.8403310775756836 + }, + { + "auxiliary_loss_clip": 0.01096619, + "auxiliary_loss_mlp": 0.01018565, + "balance_loss_clip": 1.04943395, + "balance_loss_mlp": 1.01554894, + "epoch": 0.22798052065171648, + "flos": 66706872600960.0, + "grad_norm": 0.8648985352060348, + "language_loss": 0.64769608, + "learning_rate": 3.6028842922573553e-06, + "loss": 0.66884792, + "num_input_tokens_seen": 40311005, + "router_z_loss_clip": 0.47265625, + "router_z_loss_mlp": 0.03015137, + "step": 1896, + "time_per_iteration": 4.210516452789307 + }, + { + "auxiliary_loss_clip": 0.01093362, + "auxiliary_loss_mlp": 0.01016913, + "balance_loss_clip": 1.04594362, + "balance_loss_mlp": 1.01374209, + "epoch": 0.22810076354235556, + "flos": 62080896758400.0, + "grad_norm": 0.8542247558819865, + "language_loss": 0.62921226, + "learning_rate": 3.602418289319497e-06, + "loss": 0.65031505, + "num_input_tokens_seen": 40369560, + "router_z_loss_clip": 0.47460938, + "router_z_loss_mlp": 0.03173828, + "step": 1897, + "time_per_iteration": 4.120081424713135 + }, + { + "auxiliary_loss_clip": 0.01177176, + "auxiliary_loss_mlp": 0.01058545, + "balance_loss_clip": 1.05706882, + "balance_loss_mlp": 1.03713477, + "epoch": 0.22822100643299464, + "flos": 23876358635520.0, + "grad_norm": 1.8911255605237283, + "language_loss": 0.73343396, + "learning_rate": 3.601952043296059e-06, + "loss": 0.75579119, + "num_input_tokens_seen": 40389555, + "router_z_loss_clip": 1.20019531, + "router_z_loss_mlp": 0.2142334, + "step": 1898, + "time_per_iteration": 2.8699138164520264 + }, + { + "auxiliary_loss_clip": 0.01170736, + "auxiliary_loss_mlp": 0.01048384, + "balance_loss_clip": 1.05217028, + "balance_loss_mlp": 1.02687824, + "epoch": 0.22834124932363373, + "flos": 20991331180800.0, + "grad_norm": 2.238218514213315, + "language_loss": 0.80629653, + "learning_rate": 3.6014855542577696e-06, + "loss": 0.82848769, + "num_input_tokens_seen": 40406765, + "router_z_loss_clip": 1.18554688, + "router_z_loss_mlp": 0.21496582, + "step": 1899, + "time_per_iteration": 2.827383279800415 + }, + { + "auxiliary_loss_clip": 0.01173184, + "auxiliary_loss_mlp": 0.01048834, + "balance_loss_clip": 1.05564904, + "balance_loss_mlp": 1.02719712, + "epoch": 0.22846149221427284, + "flos": 24901572620160.0, + "grad_norm": 2.2872821262503846, + "language_loss": 0.84319723, + "learning_rate": 3.6010188222753943e-06, + "loss": 0.86541736, + "num_input_tokens_seen": 40427535, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.21630859, + "step": 1900, + "time_per_iteration": 2.8706350326538086 + }, + { + "auxiliary_loss_clip": 0.01078906, + "auxiliary_loss_mlp": 0.01004099, + "balance_loss_clip": 1.03103983, + "balance_loss_mlp": 1.00073707, + "epoch": 0.22858173510491192, + "flos": 56132294319360.0, + "grad_norm": 0.9021689357453276, + "language_loss": 0.6417399, + "learning_rate": 3.6005518474197372e-06, + "loss": 0.66256994, + "num_input_tokens_seen": 40479580, + "router_z_loss_clip": 0.48046875, + "router_z_loss_mlp": 0.03369141, + "step": 1901, + "time_per_iteration": 3.2227816581726074 + }, + { + "auxiliary_loss_clip": 0.01172292, + "auxiliary_loss_mlp": 0.0105037, + "balance_loss_clip": 1.05328918, + "balance_loss_mlp": 1.02843499, + "epoch": 0.228701977995551, + "flos": 24170826332160.0, + "grad_norm": 2.243786776105577, + "language_loss": 0.78029555, + "learning_rate": 3.6000846297616373e-06, + "loss": 0.80252218, + "num_input_tokens_seen": 40497880, + "router_z_loss_clip": 1.18945312, + "router_z_loss_mlp": 0.21936035, + "step": 1902, + "time_per_iteration": 2.8705546855926514 + }, + { + "auxiliary_loss_clip": 0.01176037, + "auxiliary_loss_mlp": 0.01055127, + "balance_loss_clip": 1.05515015, + "balance_loss_mlp": 1.03422999, + "epoch": 0.22882222088619011, + "flos": 21387892308480.0, + "grad_norm": 2.334911108411992, + "language_loss": 0.72689629, + "learning_rate": 3.5996171693719717e-06, + "loss": 0.74920785, + "num_input_tokens_seen": 40513975, + "router_z_loss_clip": 1.20898438, + "router_z_loss_mlp": 0.20898438, + "step": 1903, + "time_per_iteration": 3.0684316158294678 + }, + { + "auxiliary_loss_clip": 0.01075546, + "auxiliary_loss_mlp": 0.01007967, + "balance_loss_clip": 1.02695322, + "balance_loss_mlp": 1.00474787, + "epoch": 0.2289424637768292, + "flos": 64589615377920.0, + "grad_norm": 0.8780034540998384, + "language_loss": 0.64761519, + "learning_rate": 3.5991494663216528e-06, + "loss": 0.66845036, + "num_input_tokens_seen": 40576960, + "router_z_loss_clip": 0.484375, + "router_z_loss_mlp": 0.03222656, + "step": 1904, + "time_per_iteration": 3.4392893314361572 + }, + { + "auxiliary_loss_clip": 0.01173462, + "auxiliary_loss_mlp": 0.010559, + "balance_loss_clip": 1.05424452, + "balance_loss_mlp": 1.03342938, + "epoch": 0.22906270666746828, + "flos": 22163419877760.0, + "grad_norm": 2.9741906923975923, + "language_loss": 0.8713789, + "learning_rate": 3.5986815206816314e-06, + "loss": 0.89367247, + "num_input_tokens_seen": 40595780, + "router_z_loss_clip": 1.19042969, + "router_z_loss_mlp": 0.22485352, + "step": 1905, + "time_per_iteration": 2.825711727142334 + }, + { + "auxiliary_loss_clip": 0.0117438, + "auxiliary_loss_mlp": 0.01049519, + "balance_loss_clip": 1.05403566, + "balance_loss_mlp": 1.02759624, + "epoch": 0.2291829495581074, + "flos": 25772334122880.0, + "grad_norm": 2.0629296323982302, + "language_loss": 0.74239707, + "learning_rate": 3.598213332522895e-06, + "loss": 0.76463604, + "num_input_tokens_seen": 40615810, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.21923828, + "step": 1906, + "time_per_iteration": 2.8404431343078613 + }, + { + "auxiliary_loss_clip": 0.01165981, + "auxiliary_loss_mlp": 0.01053653, + "balance_loss_clip": 1.05249536, + "balance_loss_mlp": 1.03257656, + "epoch": 0.22930319244874647, + "flos": 31172760126720.0, + "grad_norm": 2.169569859995641, + "language_loss": 0.77528965, + "learning_rate": 3.597744901916466e-06, + "loss": 0.79748595, + "num_input_tokens_seen": 40637095, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.21069336, + "step": 1907, + "time_per_iteration": 2.8913767337799072 + }, + { + "auxiliary_loss_clip": 0.01176027, + "auxiliary_loss_mlp": 0.01050448, + "balance_loss_clip": 1.0530808, + "balance_loss_mlp": 1.0268209, + "epoch": 0.22942343533938556, + "flos": 23254098399360.0, + "grad_norm": 52.180446973862125, + "language_loss": 0.76872098, + "learning_rate": 3.5972762289334058e-06, + "loss": 0.7909857, + "num_input_tokens_seen": 40656725, + "router_z_loss_clip": 1.22851562, + "router_z_loss_mlp": 0.23632812, + "step": 1908, + "time_per_iteration": 2.798828601837158 + }, + { + "auxiliary_loss_clip": 0.01172199, + "auxiliary_loss_mlp": 0.010516, + "balance_loss_clip": 1.05491257, + "balance_loss_mlp": 1.02894998, + "epoch": 0.22954367823002464, + "flos": 14610903436800.0, + "grad_norm": 2.635291759356968, + "language_loss": 0.84510738, + "learning_rate": 3.5968073136448116e-06, + "loss": 0.86734533, + "num_input_tokens_seen": 40674745, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.22668457, + "step": 1909, + "time_per_iteration": 2.7852773666381836 + }, + { + "auxiliary_loss_clip": 0.01172922, + "auxiliary_loss_mlp": 0.01051439, + "balance_loss_clip": 1.05443978, + "balance_loss_mlp": 1.0302794, + "epoch": 0.22966392112066375, + "flos": 16763604405120.0, + "grad_norm": 2.0649596626829516, + "language_loss": 0.91436994, + "learning_rate": 3.596338156121818e-06, + "loss": 0.93661362, + "num_input_tokens_seen": 40693630, + "router_z_loss_clip": 1.18359375, + "router_z_loss_mlp": 0.21154785, + "step": 1910, + "time_per_iteration": 2.8052165508270264 + }, + { + "auxiliary_loss_clip": 0.01087655, + "auxiliary_loss_mlp": 0.01014666, + "balance_loss_clip": 1.0381043, + "balance_loss_mlp": 1.01159048, + "epoch": 0.22978416401130283, + "flos": 67474247783040.0, + "grad_norm": 0.7475879285745185, + "language_loss": 0.593458, + "learning_rate": 3.595868756435595e-06, + "loss": 0.61448121, + "num_input_tokens_seen": 40761310, + "router_z_loss_clip": 0.49609375, + "router_z_loss_mlp": 0.03076172, + "step": 1911, + "time_per_iteration": 3.4973769187927246 + }, + { + "auxiliary_loss_clip": 0.0117576, + "auxiliary_loss_mlp": 0.01053763, + "balance_loss_clip": 1.05789101, + "balance_loss_mlp": 1.03093481, + "epoch": 0.22990440690194192, + "flos": 19865137086720.0, + "grad_norm": 2.8689886165819622, + "language_loss": 0.80394721, + "learning_rate": 3.5953991146573504e-06, + "loss": 0.82624245, + "num_input_tokens_seen": 40779955, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.22827148, + "step": 1912, + "time_per_iteration": 2.851123809814453 + }, + { + "auxiliary_loss_clip": 0.01172891, + "auxiliary_loss_mlp": 0.01052452, + "balance_loss_clip": 1.05243921, + "balance_loss_mlp": 1.02802563, + "epoch": 0.23002464979258103, + "flos": 13289246507520.0, + "grad_norm": 3.054301264547081, + "language_loss": 0.83482945, + "learning_rate": 3.5949292308583294e-06, + "loss": 0.85708284, + "num_input_tokens_seen": 40793200, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.24414062, + "step": 1913, + "time_per_iteration": 2.7388646602630615 + }, + { + "auxiliary_loss_clip": 0.0117078, + "auxiliary_loss_mlp": 0.01054595, + "balance_loss_clip": 1.05294442, + "balance_loss_mlp": 1.03070521, + "epoch": 0.2301448926832201, + "flos": 22163779013760.0, + "grad_norm": 2.6184364131862052, + "language_loss": 0.80853987, + "learning_rate": 3.594459105109811e-06, + "loss": 0.83079362, + "num_input_tokens_seen": 40812380, + "router_z_loss_clip": 1.17773438, + "router_z_loss_mlp": 0.23876953, + "step": 1914, + "time_per_iteration": 2.813915729522705 + }, + { + "auxiliary_loss_clip": 0.01171292, + "auxiliary_loss_mlp": 0.01054823, + "balance_loss_clip": 1.0551523, + "balance_loss_mlp": 1.03341353, + "epoch": 0.2302651355738592, + "flos": 20704477167360.0, + "grad_norm": 2.316239590527094, + "language_loss": 0.8139919, + "learning_rate": 3.593988737483115e-06, + "loss": 0.83625317, + "num_input_tokens_seen": 40832320, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.2142334, + "step": 1915, + "time_per_iteration": 2.88862943649292 + }, + { + "auxiliary_loss_clip": 0.011712, + "auxiliary_loss_mlp": 0.01052201, + "balance_loss_clip": 1.05497324, + "balance_loss_mlp": 1.02897882, + "epoch": 0.23038537846449827, + "flos": 18588943797120.0, + "grad_norm": 2.4684245200273387, + "language_loss": 0.78001368, + "learning_rate": 3.5935181280495947e-06, + "loss": 0.80224776, + "num_input_tokens_seen": 40850900, + "router_z_loss_clip": 1.16113281, + "router_z_loss_mlp": 0.2322998, + "step": 1916, + "time_per_iteration": 2.927558422088623 + }, + { + "auxiliary_loss_clip": 0.01084034, + "auxiliary_loss_mlp": 0.01003257, + "balance_loss_clip": 1.03504419, + "balance_loss_mlp": 1.00039577, + "epoch": 0.23050562135513739, + "flos": 64224260190720.0, + "grad_norm": 0.8019298136556428, + "language_loss": 0.5428412, + "learning_rate": 3.5930472768806412e-06, + "loss": 0.56371415, + "num_input_tokens_seen": 40909570, + "router_z_loss_clip": 0.49023438, + "router_z_loss_mlp": 0.02856445, + "step": 1917, + "time_per_iteration": 3.349562168121338 + }, + { + "auxiliary_loss_clip": 0.0117076, + "auxiliary_loss_mlp": 0.01056123, + "balance_loss_clip": 1.05396092, + "balance_loss_mlp": 1.03346109, + "epoch": 0.23062586424577647, + "flos": 17313396952320.0, + "grad_norm": 2.1120323686193183, + "language_loss": 0.77268368, + "learning_rate": 3.5925761840476826e-06, + "loss": 0.79495251, + "num_input_tokens_seen": 40928180, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.2265625, + "step": 1918, + "time_per_iteration": 2.8319334983825684 + }, + { + "auxiliary_loss_clip": 0.01171909, + "auxiliary_loss_mlp": 0.0104556, + "balance_loss_clip": 1.05565178, + "balance_loss_mlp": 1.02500772, + "epoch": 0.23074610713641555, + "flos": 27855979194240.0, + "grad_norm": 2.9738053041724744, + "language_loss": 0.81934565, + "learning_rate": 3.592104849622183e-06, + "loss": 0.84152037, + "num_input_tokens_seen": 40950435, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.20532227, + "step": 1919, + "time_per_iteration": 2.8989017009735107 + }, + { + "auxiliary_loss_clip": 0.01171417, + "auxiliary_loss_mlp": 0.0104818, + "balance_loss_clip": 1.05403066, + "balance_loss_mlp": 1.02599561, + "epoch": 0.23086635002705466, + "flos": 28841798937600.0, + "grad_norm": 2.1872026290740525, + "language_loss": 0.73646814, + "learning_rate": 3.591633273675644e-06, + "loss": 0.75866413, + "num_input_tokens_seen": 40972670, + "router_z_loss_clip": 1.17382812, + "router_z_loss_mlp": 0.22167969, + "step": 1920, + "time_per_iteration": 2.9071462154388428 + }, + { + "auxiliary_loss_clip": 0.01078629, + "auxiliary_loss_mlp": 0.01003049, + "balance_loss_clip": 1.03023088, + "balance_loss_mlp": 1.00028324, + "epoch": 0.23098659291769374, + "flos": 62923681566720.0, + "grad_norm": 0.9061440680297178, + "language_loss": 0.58097649, + "learning_rate": 3.591161456279602e-06, + "loss": 0.60179323, + "num_input_tokens_seen": 41018215, + "router_z_loss_clip": 0.484375, + "router_z_loss_mlp": 0.02770996, + "step": 1921, + "time_per_iteration": 3.237536668777466 + }, + { + "auxiliary_loss_clip": 0.0117498, + "auxiliary_loss_mlp": 0.01049868, + "balance_loss_clip": 1.05445373, + "balance_loss_mlp": 1.02712309, + "epoch": 0.23110683580833283, + "flos": 23476816679040.0, + "grad_norm": 3.626035368800257, + "language_loss": 0.80410767, + "learning_rate": 3.590689397505633e-06, + "loss": 0.82635617, + "num_input_tokens_seen": 41039125, + "router_z_loss_clip": 1.20507812, + "router_z_loss_mlp": 0.22729492, + "step": 1922, + "time_per_iteration": 2.8428471088409424 + }, + { + "auxiliary_loss_clip": 0.01169285, + "auxiliary_loss_mlp": 0.01050522, + "balance_loss_clip": 1.05437922, + "balance_loss_mlp": 1.03030396, + "epoch": 0.2312270786989719, + "flos": 27271066124160.0, + "grad_norm": 1.9196910577748443, + "language_loss": 0.86843705, + "learning_rate": 3.590217097425347e-06, + "loss": 0.89063519, + "num_input_tokens_seen": 41059025, + "router_z_loss_clip": 1.15039062, + "router_z_loss_mlp": 0.20214844, + "step": 1923, + "time_per_iteration": 3.7714221477508545 + }, + { + "auxiliary_loss_clip": 0.01171887, + "auxiliary_loss_mlp": 0.01056312, + "balance_loss_clip": 1.05407906, + "balance_loss_mlp": 1.03334022, + "epoch": 0.23134732158961102, + "flos": 13261344618240.0, + "grad_norm": 2.4028439360763123, + "language_loss": 0.71146822, + "learning_rate": 3.589744556110391e-06, + "loss": 0.73375022, + "num_input_tokens_seen": 41077015, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.2298584, + "step": 1924, + "time_per_iteration": 2.748828172683716 + }, + { + "auxiliary_loss_clip": 0.01169968, + "auxiliary_loss_mlp": 0.01047105, + "balance_loss_clip": 1.05113137, + "balance_loss_mlp": 1.02576685, + "epoch": 0.2314675644802501, + "flos": 36977648250240.0, + "grad_norm": 4.416560512435205, + "language_loss": 0.84462297, + "learning_rate": 3.58927177363245e-06, + "loss": 0.86679369, + "num_input_tokens_seen": 41099840, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.21337891, + "step": 1925, + "time_per_iteration": 3.7185428142547607 + }, + { + "auxiliary_loss_clip": 0.01175404, + "auxiliary_loss_mlp": 0.01049424, + "balance_loss_clip": 1.05543065, + "balance_loss_mlp": 1.02743006, + "epoch": 0.2315878073708892, + "flos": 23842207779840.0, + "grad_norm": 2.224020962779199, + "language_loss": 0.72937167, + "learning_rate": 3.5887987500632447e-06, + "loss": 0.75161994, + "num_input_tokens_seen": 41117845, + "router_z_loss_clip": 1.19824219, + "router_z_loss_mlp": 0.22009277, + "step": 1926, + "time_per_iteration": 3.631274938583374 + }, + { + "auxiliary_loss_clip": 0.0116814, + "auxiliary_loss_mlp": 0.01047097, + "balance_loss_clip": 1.0561738, + "balance_loss_mlp": 1.02618754, + "epoch": 0.2317080502615283, + "flos": 23039424766080.0, + "grad_norm": 3.26986698914077, + "language_loss": 0.84385264, + "learning_rate": 3.5883254854745325e-06, + "loss": 0.866005, + "num_input_tokens_seen": 41136235, + "router_z_loss_clip": 1.11914062, + "router_z_loss_mlp": 0.20898438, + "step": 1927, + "time_per_iteration": 2.9066827297210693 + }, + { + "auxiliary_loss_clip": 0.01172177, + "auxiliary_loss_mlp": 0.01051983, + "balance_loss_clip": 1.05151415, + "balance_loss_mlp": 1.03013217, + "epoch": 0.23182829315216738, + "flos": 11254656435840.0, + "grad_norm": 3.115826531836039, + "language_loss": 0.75187743, + "learning_rate": 3.587851979938107e-06, + "loss": 0.77411908, + "num_input_tokens_seen": 41153125, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.21850586, + "step": 1928, + "time_per_iteration": 2.809048652648926 + }, + { + "auxiliary_loss_clip": 0.01172647, + "auxiliary_loss_mlp": 0.01060474, + "balance_loss_clip": 1.05295861, + "balance_loss_mlp": 1.03716898, + "epoch": 0.23194853604280646, + "flos": 19828939155840.0, + "grad_norm": 2.772205212019126, + "language_loss": 0.7769044, + "learning_rate": 3.5873782335257985e-06, + "loss": 0.79923558, + "num_input_tokens_seen": 41171290, + "router_z_loss_clip": 1.19628906, + "router_z_loss_mlp": 0.2331543, + "step": 1929, + "time_per_iteration": 2.7871816158294678 + }, + { + "auxiliary_loss_clip": 0.01174974, + "auxiliary_loss_mlp": 0.01053355, + "balance_loss_clip": 1.05635571, + "balance_loss_mlp": 1.03103876, + "epoch": 0.23206877893344555, + "flos": 15305020830720.0, + "grad_norm": 2.1985725120122273, + "language_loss": 0.78187716, + "learning_rate": 3.5869042463094744e-06, + "loss": 0.80416042, + "num_input_tokens_seen": 41189005, + "router_z_loss_clip": 1.18457031, + "router_z_loss_mlp": 0.22314453, + "step": 1930, + "time_per_iteration": 2.7776732444763184 + }, + { + "auxiliary_loss_clip": 0.01171405, + "auxiliary_loss_mlp": 0.01056181, + "balance_loss_clip": 1.05224597, + "balance_loss_mlp": 1.03425813, + "epoch": 0.23218902182408466, + "flos": 22711488572160.0, + "grad_norm": 1.8104628997970282, + "language_loss": 0.76988268, + "learning_rate": 3.586430018361038e-06, + "loss": 0.79215848, + "num_input_tokens_seen": 41208775, + "router_z_loss_clip": 1.19042969, + "router_z_loss_mlp": 0.21923828, + "step": 1931, + "time_per_iteration": 2.7894725799560547 + }, + { + "auxiliary_loss_clip": 0.0117285, + "auxiliary_loss_mlp": 0.01051983, + "balance_loss_clip": 1.05464554, + "balance_loss_mlp": 1.03067994, + "epoch": 0.23230926471472374, + "flos": 22710734386560.0, + "grad_norm": 3.4104142291018857, + "language_loss": 0.76210487, + "learning_rate": 3.5859555497524283e-06, + "loss": 0.78435314, + "num_input_tokens_seen": 41226010, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.21313477, + "step": 1932, + "time_per_iteration": 2.793818950653076 + }, + { + "auxiliary_loss_clip": 0.01167588, + "auxiliary_loss_mlp": 0.01052204, + "balance_loss_clip": 1.05432808, + "balance_loss_mlp": 1.03171158, + "epoch": 0.23242950760536282, + "flos": 20375499479040.0, + "grad_norm": 2.128618462756887, + "language_loss": 0.92662424, + "learning_rate": 3.5854808405556237e-06, + "loss": 0.94882214, + "num_input_tokens_seen": 41245245, + "router_z_loss_clip": 1.13183594, + "router_z_loss_mlp": 0.20495605, + "step": 1933, + "time_per_iteration": 2.9417221546173096 + }, + { + "auxiliary_loss_clip": 0.01167171, + "auxiliary_loss_mlp": 0.0104186, + "balance_loss_clip": 1.05110347, + "balance_loss_mlp": 1.02224982, + "epoch": 0.23254975049600193, + "flos": 16908324301440.0, + "grad_norm": 2.4798190769956885, + "language_loss": 0.74692261, + "learning_rate": 3.5850058908426355e-06, + "loss": 0.76901287, + "num_input_tokens_seen": 41263795, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.19604492, + "step": 1934, + "time_per_iteration": 2.9456658363342285 + }, + { + "auxiliary_loss_clip": 0.01171117, + "auxiliary_loss_mlp": 0.01045694, + "balance_loss_clip": 1.05267143, + "balance_loss_mlp": 1.02580965, + "epoch": 0.23266999338664102, + "flos": 23294821443840.0, + "grad_norm": 3.699532797715833, + "language_loss": 0.85943317, + "learning_rate": 3.584530700685514e-06, + "loss": 0.88160127, + "num_input_tokens_seen": 41284055, + "router_z_loss_clip": 1.18359375, + "router_z_loss_mlp": 0.19885254, + "step": 1935, + "time_per_iteration": 2.8435683250427246 + }, + { + "auxiliary_loss_clip": 0.01168472, + "auxiliary_loss_mlp": 0.01046733, + "balance_loss_clip": 1.0529108, + "balance_loss_mlp": 1.0248816, + "epoch": 0.2327902362772801, + "flos": 19569987031680.0, + "grad_norm": 2.438605717326273, + "language_loss": 0.88824087, + "learning_rate": 3.5840552701563448e-06, + "loss": 0.91039288, + "num_input_tokens_seen": 41300255, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.21850586, + "step": 1936, + "time_per_iteration": 2.797530174255371 + }, + { + "auxiliary_loss_clip": 0.0116556, + "auxiliary_loss_mlp": 0.01054411, + "balance_loss_clip": 1.05068302, + "balance_loss_mlp": 1.03298926, + "epoch": 0.2329104791679192, + "flos": 16727514215040.0, + "grad_norm": 3.052854798176574, + "language_loss": 0.81956708, + "learning_rate": 3.5835795993272513e-06, + "loss": 0.84176683, + "num_input_tokens_seen": 41318540, + "router_z_loss_clip": 1.14746094, + "router_z_loss_mlp": 0.21411133, + "step": 1937, + "time_per_iteration": 2.8212108612060547 + }, + { + "auxiliary_loss_clip": 0.01169584, + "auxiliary_loss_mlp": 0.0105484, + "balance_loss_clip": 1.0535059, + "balance_loss_mlp": 1.03407335, + "epoch": 0.2330307220585583, + "flos": 22163743100160.0, + "grad_norm": 1.9940283043569191, + "language_loss": 0.71932113, + "learning_rate": 3.583103688270391e-06, + "loss": 0.74156535, + "num_input_tokens_seen": 41338320, + "router_z_loss_clip": 1.16113281, + "router_z_loss_mlp": 0.20751953, + "step": 1938, + "time_per_iteration": 2.8490421772003174 + }, + { + "auxiliary_loss_clip": 0.01172032, + "auxiliary_loss_mlp": 0.01049517, + "balance_loss_clip": 1.05444241, + "balance_loss_mlp": 1.02912045, + "epoch": 0.23315096494919738, + "flos": 19317319787520.0, + "grad_norm": 2.5700999844208674, + "language_loss": 0.89145005, + "learning_rate": 3.58262753705796e-06, + "loss": 0.91366553, + "num_input_tokens_seen": 41353210, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.20397949, + "step": 1939, + "time_per_iteration": 2.833771228790283 + }, + { + "auxiliary_loss_clip": 0.01078863, + "auxiliary_loss_mlp": 0.00999663, + "balance_loss_clip": 1.03205144, + "balance_loss_mlp": 0.99713606, + "epoch": 0.23327120783983646, + "flos": 53031048946560.0, + "grad_norm": 0.7513703074956741, + "language_loss": 0.55470622, + "learning_rate": 3.5821511457621902e-06, + "loss": 0.57549149, + "num_input_tokens_seen": 41410510, + "router_z_loss_clip": 0.46875, + "router_z_loss_mlp": 0.02526855, + "step": 1940, + "time_per_iteration": 3.348432779312134 + }, + { + "auxiliary_loss_clip": 0.01174198, + "auxiliary_loss_mlp": 0.01063168, + "balance_loss_clip": 1.05271459, + "balance_loss_mlp": 1.03818214, + "epoch": 0.23339145073047557, + "flos": 17126984344320.0, + "grad_norm": 5.682750184815239, + "language_loss": 0.81018198, + "learning_rate": 3.5816745144553497e-06, + "loss": 0.83255565, + "num_input_tokens_seen": 41425830, + "router_z_loss_clip": 1.21582031, + "router_z_loss_mlp": 0.25, + "step": 1941, + "time_per_iteration": 2.792844772338867 + }, + { + "auxiliary_loss_clip": 0.01167307, + "auxiliary_loss_mlp": 0.01039945, + "balance_loss_clip": 1.05133224, + "balance_loss_mlp": 1.01895261, + "epoch": 0.23351169362111465, + "flos": 13078918419840.0, + "grad_norm": 2.1198154439084926, + "language_loss": 0.74945819, + "learning_rate": 3.5811976432097424e-06, + "loss": 0.77153075, + "num_input_tokens_seen": 41443500, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.21008301, + "step": 1942, + "time_per_iteration": 2.8382863998413086 + }, + { + "auxiliary_loss_clip": 0.01172718, + "auxiliary_loss_mlp": 0.01055706, + "balance_loss_clip": 1.05510855, + "balance_loss_mlp": 1.03498745, + "epoch": 0.23363193651175373, + "flos": 15851257931520.0, + "grad_norm": 2.1916449401236764, + "language_loss": 0.84488434, + "learning_rate": 3.58072053209771e-06, + "loss": 0.86716861, + "num_input_tokens_seen": 41460055, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.20739746, + "step": 1943, + "time_per_iteration": 2.8241922855377197 + }, + { + "auxiliary_loss_clip": 0.01172325, + "auxiliary_loss_mlp": 0.01057068, + "balance_loss_clip": 1.05402136, + "balance_loss_mlp": 1.03618288, + "epoch": 0.23375217940239285, + "flos": 21025769345280.0, + "grad_norm": 1.9253448300219047, + "language_loss": 0.78714406, + "learning_rate": 3.5802431811916296e-06, + "loss": 0.80943799, + "num_input_tokens_seen": 41476665, + "router_z_loss_clip": 1.18164062, + "router_z_loss_mlp": 0.2088623, + "step": 1944, + "time_per_iteration": 2.764141798019409 + }, + { + "auxiliary_loss_clip": 0.01168347, + "auxiliary_loss_mlp": 0.01047637, + "balance_loss_clip": 1.05069816, + "balance_loss_mlp": 1.02786064, + "epoch": 0.23387242229303193, + "flos": 20594698225920.0, + "grad_norm": 4.263500458416322, + "language_loss": 0.80623794, + "learning_rate": 3.579765590563916e-06, + "loss": 0.82839781, + "num_input_tokens_seen": 41496065, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.19775391, + "step": 1945, + "time_per_iteration": 2.7706527709960938 + }, + { + "auxiliary_loss_clip": 0.01171346, + "auxiliary_loss_mlp": 0.01050504, + "balance_loss_clip": 1.05437636, + "balance_loss_mlp": 1.02970207, + "epoch": 0.233992665183671, + "flos": 24279491952000.0, + "grad_norm": 2.005611844758672, + "language_loss": 0.81553102, + "learning_rate": 3.579287760287017e-06, + "loss": 0.83774948, + "num_input_tokens_seen": 41516815, + "router_z_loss_clip": 1.16894531, + "router_z_loss_mlp": 0.20812988, + "step": 1946, + "time_per_iteration": 2.782912015914917 + }, + { + "auxiliary_loss_clip": 0.01169342, + "auxiliary_loss_mlp": 0.01052758, + "balance_loss_clip": 1.05427837, + "balance_loss_mlp": 1.03327966, + "epoch": 0.2341129080743101, + "flos": 30154621121280.0, + "grad_norm": 8.629884703932996, + "language_loss": 0.72934163, + "learning_rate": 3.578809690433421e-06, + "loss": 0.7515626, + "num_input_tokens_seen": 41538525, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.19482422, + "step": 1947, + "time_per_iteration": 2.8453876972198486 + }, + { + "auxiliary_loss_clip": 0.01177307, + "auxiliary_loss_mlp": 0.01056145, + "balance_loss_clip": 1.05478621, + "balance_loss_mlp": 1.03375793, + "epoch": 0.2342331509649492, + "flos": 22784135829120.0, + "grad_norm": 2.6040405198126906, + "language_loss": 0.81221116, + "learning_rate": 3.578331381075651e-06, + "loss": 0.83454573, + "num_input_tokens_seen": 41559025, + "router_z_loss_clip": 1.22558594, + "router_z_loss_mlp": 0.22375488, + "step": 1948, + "time_per_iteration": 3.039860725402832 + }, + { + "auxiliary_loss_clip": 0.01174874, + "auxiliary_loss_mlp": 0.01069948, + "balance_loss_clip": 1.0547626, + "balance_loss_mlp": 1.04853833, + "epoch": 0.2343533938555883, + "flos": 23623152687360.0, + "grad_norm": 2.8039722386307497, + "language_loss": 0.69861639, + "learning_rate": 3.5778528322862646e-06, + "loss": 0.72106463, + "num_input_tokens_seen": 41577845, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.2142334, + "step": 1949, + "time_per_iteration": 2.83626389503479 + }, + { + "auxiliary_loss_clip": 0.01171181, + "auxiliary_loss_mlp": 0.01054185, + "balance_loss_clip": 1.05394566, + "balance_loss_mlp": 1.03469419, + "epoch": 0.23447363674622737, + "flos": 24570332375040.0, + "grad_norm": 1.6409107990905012, + "language_loss": 0.86623216, + "learning_rate": 3.5773740441378585e-06, + "loss": 0.88848591, + "num_input_tokens_seen": 41598600, + "router_z_loss_clip": 1.17285156, + "router_z_loss_mlp": 0.19482422, + "step": 1950, + "time_per_iteration": 2.82725191116333 + }, + { + "auxiliary_loss_clip": 0.01170991, + "auxiliary_loss_mlp": 0.01050521, + "balance_loss_clip": 1.0529108, + "balance_loss_mlp": 1.02990997, + "epoch": 0.23459387963686648, + "flos": 53140322119680.0, + "grad_norm": 2.3089481621065993, + "language_loss": 0.73120934, + "learning_rate": 3.5768950167030633e-06, + "loss": 0.75342441, + "num_input_tokens_seen": 41623300, + "router_z_loss_clip": 1.18066406, + "router_z_loss_mlp": 0.20617676, + "step": 1951, + "time_per_iteration": 3.100315570831299 + }, + { + "auxiliary_loss_clip": 0.01171946, + "auxiliary_loss_mlp": 0.01060958, + "balance_loss_clip": 1.05445802, + "balance_loss_mlp": 1.03921413, + "epoch": 0.23471412252750556, + "flos": 23951412103680.0, + "grad_norm": 2.799296191468158, + "language_loss": 0.78442889, + "learning_rate": 3.576415750054548e-06, + "loss": 0.80675793, + "num_input_tokens_seen": 41643420, + "router_z_loss_clip": 1.17382812, + "router_z_loss_mlp": 0.2175293, + "step": 1952, + "time_per_iteration": 4.561337232589722 + }, + { + "auxiliary_loss_clip": 0.01171375, + "auxiliary_loss_mlp": 0.01055647, + "balance_loss_clip": 1.05356717, + "balance_loss_mlp": 1.03526199, + "epoch": 0.23483436541814465, + "flos": 15706573948800.0, + "grad_norm": 2.1194617327918954, + "language_loss": 0.85840464, + "learning_rate": 3.5759362442650172e-06, + "loss": 0.88067496, + "num_input_tokens_seen": 41660170, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.20385742, + "step": 1953, + "time_per_iteration": 2.78930926322937 + }, + { + "auxiliary_loss_clip": 0.01172823, + "auxiliary_loss_mlp": 0.01052969, + "balance_loss_clip": 1.05664551, + "balance_loss_mlp": 1.03406262, + "epoch": 0.23495460830878373, + "flos": 24936262179840.0, + "grad_norm": 5.483785363305457, + "language_loss": 0.85270321, + "learning_rate": 3.5754564994072113e-06, + "loss": 0.87496108, + "num_input_tokens_seen": 41679010, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.18908691, + "step": 1954, + "time_per_iteration": 3.672957181930542 + }, + { + "auxiliary_loss_clip": 0.01176905, + "auxiliary_loss_mlp": 0.0105248, + "balance_loss_clip": 1.05937016, + "balance_loss_mlp": 1.03250039, + "epoch": 0.23507485119942284, + "flos": 30482665056000.0, + "grad_norm": 2.594867927585465, + "language_loss": 0.59914398, + "learning_rate": 3.5749765155539067e-06, + "loss": 0.62143779, + "num_input_tokens_seen": 41699495, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.1998291, + "step": 1955, + "time_per_iteration": 3.72770357131958 + }, + { + "auxiliary_loss_clip": 0.01177627, + "auxiliary_loss_mlp": 0.01057429, + "balance_loss_clip": 1.054533, + "balance_loss_mlp": 1.03634095, + "epoch": 0.23519509409006192, + "flos": 18329129746560.0, + "grad_norm": 2.282114214326101, + "language_loss": 0.9185586, + "learning_rate": 3.574496292777917e-06, + "loss": 0.94090915, + "num_input_tokens_seen": 41717705, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.21081543, + "step": 1956, + "time_per_iteration": 2.8316311836242676 + }, + { + "auxiliary_loss_clip": 0.01179586, + "auxiliary_loss_mlp": 0.01054402, + "balance_loss_clip": 1.05563974, + "balance_loss_mlp": 1.03270578, + "epoch": 0.235315336980701, + "flos": 29643217234560.0, + "grad_norm": 2.2951841120597085, + "language_loss": 0.71777725, + "learning_rate": 3.574015831152092e-06, + "loss": 0.74011719, + "num_input_tokens_seen": 41738120, + "router_z_loss_clip": 1.23730469, + "router_z_loss_mlp": 0.21691895, + "step": 1957, + "time_per_iteration": 2.849946975708008 + }, + { + "auxiliary_loss_clip": 0.01175635, + "auxiliary_loss_mlp": 0.01064536, + "balance_loss_clip": 1.06157494, + "balance_loss_mlp": 1.04571319, + "epoch": 0.23543557987134012, + "flos": 18551704371840.0, + "grad_norm": 3.6500856834739315, + "language_loss": 0.83194888, + "learning_rate": 3.573535130749316e-06, + "loss": 0.85435057, + "num_input_tokens_seen": 41756070, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.18823242, + "step": 1958, + "time_per_iteration": 2.9102377891540527 + }, + { + "auxiliary_loss_clip": 0.01176495, + "auxiliary_loss_mlp": 0.01058066, + "balance_loss_clip": 1.06083155, + "balance_loss_mlp": 1.0375855, + "epoch": 0.2355558227619792, + "flos": 24679033908480.0, + "grad_norm": 2.3328171855930275, + "language_loss": 0.73975599, + "learning_rate": 3.5730541916425127e-06, + "loss": 0.76210153, + "num_input_tokens_seen": 41777550, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.20471191, + "step": 1959, + "time_per_iteration": 2.864535331726074 + }, + { + "auxiliary_loss_clip": 0.01178142, + "auxiliary_loss_mlp": 0.01052262, + "balance_loss_clip": 1.06002879, + "balance_loss_mlp": 1.03198457, + "epoch": 0.23567606565261828, + "flos": 21944795748480.0, + "grad_norm": 2.2340211794699623, + "language_loss": 0.8625108, + "learning_rate": 3.572573013904639e-06, + "loss": 0.8848148, + "num_input_tokens_seen": 41797460, + "router_z_loss_clip": 1.18164062, + "router_z_loss_mlp": 0.20275879, + "step": 1960, + "time_per_iteration": 2.920585870742798 + }, + { + "auxiliary_loss_clip": 0.01176586, + "auxiliary_loss_mlp": 0.01046817, + "balance_loss_clip": 1.06028199, + "balance_loss_mlp": 1.02714717, + "epoch": 0.2357963085432574, + "flos": 13589352639360.0, + "grad_norm": 2.699720937476364, + "language_loss": 0.91693091, + "learning_rate": 3.572091597608689e-06, + "loss": 0.93916488, + "num_input_tokens_seen": 41815585, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.19689941, + "step": 1961, + "time_per_iteration": 2.8403828144073486 + }, + { + "auxiliary_loss_clip": 0.01184341, + "auxiliary_loss_mlp": 0.01053719, + "balance_loss_clip": 1.06084037, + "balance_loss_mlp": 1.03188038, + "epoch": 0.23591655143389648, + "flos": 22088689632000.0, + "grad_norm": 2.8403766060073194, + "language_loss": 0.74057281, + "learning_rate": 3.571609942827694e-06, + "loss": 0.7629534, + "num_input_tokens_seen": 41834700, + "router_z_loss_clip": 1.234375, + "router_z_loss_mlp": 0.21862793, + "step": 1962, + "time_per_iteration": 2.8136632442474365 + }, + { + "auxiliary_loss_clip": 0.01177319, + "auxiliary_loss_mlp": 0.01049108, + "balance_loss_clip": 1.06045544, + "balance_loss_mlp": 1.02824628, + "epoch": 0.23603679432453556, + "flos": 17017349057280.0, + "grad_norm": 2.364426964337369, + "language_loss": 0.88521135, + "learning_rate": 3.57112804963472e-06, + "loss": 0.90747565, + "num_input_tokens_seen": 41852915, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.20861816, + "step": 1963, + "time_per_iteration": 2.8520636558532715 + }, + { + "auxiliary_loss_clip": 0.01174714, + "auxiliary_loss_mlp": 0.01049995, + "balance_loss_clip": 1.05977798, + "balance_loss_mlp": 1.02964652, + "epoch": 0.23615703721517464, + "flos": 19171307001600.0, + "grad_norm": 4.119533706304387, + "language_loss": 0.76443321, + "learning_rate": 3.57064591810287e-06, + "loss": 0.78668022, + "num_input_tokens_seen": 41870415, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.20349121, + "step": 1964, + "time_per_iteration": 2.9703915119171143 + }, + { + "auxiliary_loss_clip": 0.01174767, + "auxiliary_loss_mlp": 0.01049324, + "balance_loss_clip": 1.0595758, + "balance_loss_mlp": 1.02857018, + "epoch": 0.23627728010581375, + "flos": 19098803399040.0, + "grad_norm": 2.9175061146488748, + "language_loss": 0.80334127, + "learning_rate": 3.570163548305284e-06, + "loss": 0.82558221, + "num_input_tokens_seen": 41889345, + "router_z_loss_clip": 1.15234375, + "router_z_loss_mlp": 0.20751953, + "step": 1965, + "time_per_iteration": 2.8994274139404297 + }, + { + "auxiliary_loss_clip": 0.0118133, + "auxiliary_loss_mlp": 0.01048286, + "balance_loss_clip": 1.06208146, + "balance_loss_mlp": 1.02689946, + "epoch": 0.23639752299645284, + "flos": 14282213057280.0, + "grad_norm": 3.1739262846905016, + "language_loss": 0.69674373, + "learning_rate": 3.569680940315135e-06, + "loss": 0.71903992, + "num_input_tokens_seen": 41905745, + "router_z_loss_clip": 1.19335938, + "router_z_loss_mlp": 0.21386719, + "step": 1966, + "time_per_iteration": 2.7670438289642334 + }, + { + "auxiliary_loss_clip": 0.01185329, + "auxiliary_loss_mlp": 0.01048585, + "balance_loss_clip": 1.06322885, + "balance_loss_mlp": 1.02750885, + "epoch": 0.23651776588709192, + "flos": 22893411980160.0, + "grad_norm": 1.9708294675004454, + "language_loss": 0.82153511, + "learning_rate": 3.5691980942056356e-06, + "loss": 0.84387422, + "num_input_tokens_seen": 41925115, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.21069336, + "step": 1967, + "time_per_iteration": 2.798788070678711 + }, + { + "auxiliary_loss_clip": 0.01180158, + "auxiliary_loss_mlp": 0.01047427, + "balance_loss_clip": 1.06209111, + "balance_loss_mlp": 1.02666116, + "epoch": 0.23663800877773103, + "flos": 18624531196800.0, + "grad_norm": 3.2963280502525203, + "language_loss": 0.79402721, + "learning_rate": 3.5687150100500332e-06, + "loss": 0.81630301, + "num_input_tokens_seen": 41944815, + "router_z_loss_clip": 1.18164062, + "router_z_loss_mlp": 0.20751953, + "step": 1968, + "time_per_iteration": 2.781592607498169 + }, + { + "auxiliary_loss_clip": 0.01176617, + "auxiliary_loss_mlp": 0.01056147, + "balance_loss_clip": 1.05766904, + "balance_loss_mlp": 1.03483248, + "epoch": 0.2367582516683701, + "flos": 25555828896000.0, + "grad_norm": 2.2437624303547126, + "language_loss": 0.74475372, + "learning_rate": 3.568231687921611e-06, + "loss": 0.76708138, + "num_input_tokens_seen": 41964990, + "router_z_loss_clip": 1.18945312, + "router_z_loss_mlp": 0.21337891, + "step": 1969, + "time_per_iteration": 2.8944571018218994 + }, + { + "auxiliary_loss_clip": 0.011759, + "auxiliary_loss_mlp": 0.0104171, + "balance_loss_clip": 1.05983818, + "balance_loss_mlp": 1.02232683, + "epoch": 0.2368784945590092, + "flos": 23295072839040.0, + "grad_norm": 1.778320442372548, + "language_loss": 0.80621141, + "learning_rate": 3.5677481278936883e-06, + "loss": 0.82838756, + "num_input_tokens_seen": 41984570, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.19396973, + "step": 1970, + "time_per_iteration": 2.7854113578796387 + }, + { + "auxiliary_loss_clip": 0.01080659, + "auxiliary_loss_mlp": 0.01012492, + "balance_loss_clip": 1.03399062, + "balance_loss_mlp": 1.00970232, + "epoch": 0.23699873744964828, + "flos": 69859291875840.0, + "grad_norm": 0.8411683233654562, + "language_loss": 0.57810575, + "learning_rate": 3.5672643300396214e-06, + "loss": 0.59903729, + "num_input_tokens_seen": 42053715, + "router_z_loss_clip": 0.46679688, + "router_z_loss_mlp": 0.0279541, + "step": 1971, + "time_per_iteration": 3.39105224609375 + }, + { + "auxiliary_loss_clip": 0.01174704, + "auxiliary_loss_mlp": 0.01043111, + "balance_loss_clip": 1.05814672, + "balance_loss_mlp": 1.02302384, + "epoch": 0.2371189803402874, + "flos": 21835052720640.0, + "grad_norm": 2.6566132292711773, + "language_loss": 0.67597443, + "learning_rate": 3.566780294432802e-06, + "loss": 0.69815254, + "num_input_tokens_seen": 42070890, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.20092773, + "step": 1972, + "time_per_iteration": 2.84063982963562 + }, + { + "auxiliary_loss_clip": 0.01176507, + "auxiliary_loss_mlp": 0.01053869, + "balance_loss_clip": 1.05795813, + "balance_loss_mlp": 1.03160071, + "epoch": 0.23723922323092647, + "flos": 21908490076800.0, + "grad_norm": 2.431663880425218, + "language_loss": 0.74277055, + "learning_rate": 3.566296021146657e-06, + "loss": 0.76507425, + "num_input_tokens_seen": 42090270, + "router_z_loss_clip": 1.18457031, + "router_z_loss_mlp": 0.22241211, + "step": 1973, + "time_per_iteration": 2.8737235069274902 + }, + { + "auxiliary_loss_clip": 0.01174902, + "auxiliary_loss_mlp": 0.01048822, + "balance_loss_clip": 1.05622971, + "balance_loss_mlp": 1.02819848, + "epoch": 0.23735946612156555, + "flos": 32708803380480.0, + "grad_norm": 1.9475501441877339, + "language_loss": 0.73153579, + "learning_rate": 3.565811510254652e-06, + "loss": 0.75377303, + "num_input_tokens_seen": 42111150, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.2064209, + "step": 1974, + "time_per_iteration": 2.934173822402954 + }, + { + "auxiliary_loss_clip": 0.01074476, + "auxiliary_loss_mlp": 0.01002344, + "balance_loss_clip": 1.02811694, + "balance_loss_mlp": 0.99973327, + "epoch": 0.23747970901220466, + "flos": 70546944821760.0, + "grad_norm": 0.8386552123737607, + "language_loss": 0.58239657, + "learning_rate": 3.5653267618302845e-06, + "loss": 0.60316473, + "num_input_tokens_seen": 42178730, + "router_z_loss_clip": 0.46484375, + "router_z_loss_mlp": 0.02612305, + "step": 1975, + "time_per_iteration": 3.4192442893981934 + }, + { + "auxiliary_loss_clip": 0.01175222, + "auxiliary_loss_mlp": 0.01046667, + "balance_loss_clip": 1.05504227, + "balance_loss_mlp": 1.02432704, + "epoch": 0.23759995190284375, + "flos": 20849807594880.0, + "grad_norm": 1.9144506046320022, + "language_loss": 0.85760176, + "learning_rate": 3.564841775947093e-06, + "loss": 0.87982059, + "num_input_tokens_seen": 42199620, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.22338867, + "step": 1976, + "time_per_iteration": 2.899510622024536 + }, + { + "auxiliary_loss_clip": 0.0117025, + "auxiliary_loss_mlp": 0.01053345, + "balance_loss_clip": 1.05336559, + "balance_loss_mlp": 1.03297222, + "epoch": 0.23772019479348283, + "flos": 32921645420160.0, + "grad_norm": 7.171574966644063, + "language_loss": 0.76396537, + "learning_rate": 3.5643565526786475e-06, + "loss": 0.78620136, + "num_input_tokens_seen": 42219560, + "router_z_loss_clip": 1.16894531, + "router_z_loss_mlp": 0.20385742, + "step": 1977, + "time_per_iteration": 2.892665386199951 + }, + { + "auxiliary_loss_clip": 0.01174334, + "auxiliary_loss_mlp": 0.01058582, + "balance_loss_clip": 1.05537724, + "balance_loss_mlp": 1.03825641, + "epoch": 0.2378404376841219, + "flos": 32342765834880.0, + "grad_norm": 2.2331692950578312, + "language_loss": 0.77232271, + "learning_rate": 3.5638710920985574e-06, + "loss": 0.79465187, + "num_input_tokens_seen": 42241020, + "router_z_loss_clip": 1.19042969, + "router_z_loss_mlp": 0.20324707, + "step": 1978, + "time_per_iteration": 2.9086620807647705 + }, + { + "auxiliary_loss_clip": 0.01178047, + "auxiliary_loss_mlp": 0.01048598, + "balance_loss_clip": 1.05511832, + "balance_loss_mlp": 1.02742672, + "epoch": 0.23796068057476102, + "flos": 22997624313600.0, + "grad_norm": 1.9849816694495421, + "language_loss": 0.81953955, + "learning_rate": 3.5633853942804655e-06, + "loss": 0.84180605, + "num_input_tokens_seen": 42259345, + "router_z_loss_clip": 1.22949219, + "router_z_loss_mlp": 0.21166992, + "step": 1979, + "time_per_iteration": 2.867863178253174 + }, + { + "auxiliary_loss_clip": 0.01177972, + "auxiliary_loss_mlp": 0.01060299, + "balance_loss_clip": 1.05573618, + "balance_loss_mlp": 1.03854322, + "epoch": 0.2380809234654001, + "flos": 13480938414720.0, + "grad_norm": 3.3423237036525566, + "language_loss": 0.76473236, + "learning_rate": 3.5628994592980527e-06, + "loss": 0.78711504, + "num_input_tokens_seen": 42277250, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.21777344, + "step": 1980, + "time_per_iteration": 2.8656070232391357 + }, + { + "auxiliary_loss_clip": 0.01173692, + "auxiliary_loss_mlp": 0.01057334, + "balance_loss_clip": 1.0534966, + "balance_loss_mlp": 1.03585219, + "epoch": 0.2382011663560392, + "flos": 16871803148160.0, + "grad_norm": 1.823328717294973, + "language_loss": 0.70123398, + "learning_rate": 3.562413287225034e-06, + "loss": 0.7235443, + "num_input_tokens_seen": 42295360, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.21484375, + "step": 1981, + "time_per_iteration": 3.7155990600585938 + }, + { + "auxiliary_loss_clip": 0.01178416, + "auxiliary_loss_mlp": 0.0104672, + "balance_loss_clip": 1.05792165, + "balance_loss_mlp": 1.0260371, + "epoch": 0.2383214092466783, + "flos": 18441135331200.0, + "grad_norm": 2.46806673396834, + "language_loss": 0.89414036, + "learning_rate": 3.5619268781351623e-06, + "loss": 0.91639173, + "num_input_tokens_seen": 42313430, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.20678711, + "step": 1982, + "time_per_iteration": 2.7639336585998535 + }, + { + "auxiliary_loss_clip": 0.01171276, + "auxiliary_loss_mlp": 0.01049388, + "balance_loss_clip": 1.05300069, + "balance_loss_mlp": 1.02967119, + "epoch": 0.23844165213731738, + "flos": 19755717281280.0, + "grad_norm": 4.107635168554737, + "language_loss": 0.76777011, + "learning_rate": 3.5614402321022256e-06, + "loss": 0.78997678, + "num_input_tokens_seen": 42331260, + "router_z_loss_clip": 1.18164062, + "router_z_loss_mlp": 0.19714355, + "step": 1983, + "time_per_iteration": 3.6035211086273193 + }, + { + "auxiliary_loss_clip": 0.01168336, + "auxiliary_loss_mlp": 0.01060145, + "balance_loss_clip": 1.05293119, + "balance_loss_mlp": 1.03978395, + "epoch": 0.23856189502795647, + "flos": 23367360960000.0, + "grad_norm": 2.051807157663801, + "language_loss": 0.87419844, + "learning_rate": 3.5609533492000463e-06, + "loss": 0.89648324, + "num_input_tokens_seen": 42350150, + "router_z_loss_clip": 1.15429688, + "router_z_loss_mlp": 0.20373535, + "step": 1984, + "time_per_iteration": 3.6240265369415283 + }, + { + "auxiliary_loss_clip": 0.01172427, + "auxiliary_loss_mlp": 0.0104706, + "balance_loss_clip": 1.05494666, + "balance_loss_mlp": 1.02646017, + "epoch": 0.23868213791859555, + "flos": 23475056912640.0, + "grad_norm": 2.3857618610756015, + "language_loss": 0.78803921, + "learning_rate": 3.560466229502485e-06, + "loss": 0.81023407, + "num_input_tokens_seen": 42369495, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.20593262, + "step": 1985, + "time_per_iteration": 2.796420097351074 + }, + { + "auxiliary_loss_clip": 0.01180146, + "auxiliary_loss_mlp": 0.01053533, + "balance_loss_clip": 1.05853653, + "balance_loss_mlp": 1.03286171, + "epoch": 0.23880238080923466, + "flos": 16617340224000.0, + "grad_norm": 3.0845826188783843, + "language_loss": 0.89647782, + "learning_rate": 3.5599788730834384e-06, + "loss": 0.91881466, + "num_input_tokens_seen": 42387455, + "router_z_loss_clip": 1.21679688, + "router_z_loss_mlp": 0.20678711, + "step": 1986, + "time_per_iteration": 2.7911887168884277 + }, + { + "auxiliary_loss_clip": 0.01177623, + "auxiliary_loss_mlp": 0.01056026, + "balance_loss_clip": 1.05630684, + "balance_loss_mlp": 1.03515208, + "epoch": 0.23892262369987374, + "flos": 17348409734400.0, + "grad_norm": 3.771623668447242, + "language_loss": 0.78664672, + "learning_rate": 3.559491280016836e-06, + "loss": 0.80898327, + "num_input_tokens_seen": 42405400, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.2088623, + "step": 1987, + "time_per_iteration": 2.830937623977661 + }, + { + "auxiliary_loss_clip": 0.0118147, + "auxiliary_loss_mlp": 0.0105392, + "balance_loss_clip": 1.05677307, + "balance_loss_mlp": 1.03141379, + "epoch": 0.23904286659051283, + "flos": 22309899540480.0, + "grad_norm": 2.711534410870519, + "language_loss": 0.70978129, + "learning_rate": 3.5590034503766465e-06, + "loss": 0.73213518, + "num_input_tokens_seen": 42425065, + "router_z_loss_clip": 1.24511719, + "router_z_loss_mlp": 0.22497559, + "step": 1988, + "time_per_iteration": 2.8338193893432617 + }, + { + "auxiliary_loss_clip": 0.01175628, + "auxiliary_loss_mlp": 0.01055208, + "balance_loss_clip": 1.05580223, + "balance_loss_mlp": 1.03381014, + "epoch": 0.23916310948115194, + "flos": 21178246579200.0, + "grad_norm": 2.355746301208337, + "language_loss": 0.81349081, + "learning_rate": 3.558515384236874e-06, + "loss": 0.83579916, + "num_input_tokens_seen": 42442495, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.21386719, + "step": 1989, + "time_per_iteration": 2.8380746841430664 + }, + { + "auxiliary_loss_clip": 0.01182622, + "auxiliary_loss_mlp": 0.01073139, + "balance_loss_clip": 1.0606637, + "balance_loss_mlp": 1.05305231, + "epoch": 0.23928335237179102, + "flos": 14137349506560.0, + "grad_norm": 2.549627593430768, + "language_loss": 0.83947802, + "learning_rate": 3.558027081671556e-06, + "loss": 0.86203563, + "num_input_tokens_seen": 42459480, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.2010498, + "step": 1990, + "time_per_iteration": 2.816450834274292 + }, + { + "auxiliary_loss_clip": 0.01179179, + "auxiliary_loss_mlp": 0.01056564, + "balance_loss_clip": 1.05655456, + "balance_loss_mlp": 1.03439105, + "epoch": 0.2394035952624301, + "flos": 23769596436480.0, + "grad_norm": 3.4793731992410493, + "language_loss": 0.68857801, + "learning_rate": 3.557538542754769e-06, + "loss": 0.71093547, + "num_input_tokens_seen": 42479175, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.22167969, + "step": 1991, + "time_per_iteration": 2.953366279602051 + }, + { + "auxiliary_loss_clip": 0.01179545, + "auxiliary_loss_mlp": 0.01049949, + "balance_loss_clip": 1.05919576, + "balance_loss_mlp": 1.02893209, + "epoch": 0.2395238381530692, + "flos": 24206198250240.0, + "grad_norm": 2.2732656329991396, + "language_loss": 0.66790164, + "learning_rate": 3.557049767560623e-06, + "loss": 0.69019657, + "num_input_tokens_seen": 42498090, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.21008301, + "step": 1992, + "time_per_iteration": 2.838585376739502 + }, + { + "auxiliary_loss_clip": 0.01177449, + "auxiliary_loss_mlp": 0.01053066, + "balance_loss_clip": 1.05784166, + "balance_loss_mlp": 1.03179908, + "epoch": 0.2396440810437083, + "flos": 25295763450240.0, + "grad_norm": 2.325906888496302, + "language_loss": 0.86106801, + "learning_rate": 3.5565607561632655e-06, + "loss": 0.88337314, + "num_input_tokens_seen": 42516930, + "router_z_loss_clip": 1.19433594, + "router_z_loss_mlp": 0.21264648, + "step": 1993, + "time_per_iteration": 2.8493332862854004 + }, + { + "auxiliary_loss_clip": 0.01178631, + "auxiliary_loss_mlp": 0.01061713, + "balance_loss_clip": 1.05866051, + "balance_loss_mlp": 1.04004049, + "epoch": 0.23976432393434738, + "flos": 28543093436160.0, + "grad_norm": 2.6560006052467626, + "language_loss": 0.79596567, + "learning_rate": 3.5560715086368787e-06, + "loss": 0.81836909, + "num_input_tokens_seen": 42534800, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.21691895, + "step": 1994, + "time_per_iteration": 2.8931636810302734 + }, + { + "auxiliary_loss_clip": 0.01174835, + "auxiliary_loss_mlp": 0.01049889, + "balance_loss_clip": 1.05513418, + "balance_loss_mlp": 1.02871716, + "epoch": 0.23988456682498646, + "flos": 19494358945920.0, + "grad_norm": 1.9533717128958197, + "language_loss": 0.82081771, + "learning_rate": 3.5555820250556816e-06, + "loss": 0.84306502, + "num_input_tokens_seen": 42552000, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.21166992, + "step": 1995, + "time_per_iteration": 2.787091016769409 + }, + { + "auxiliary_loss_clip": 0.01179744, + "auxiliary_loss_mlp": 0.01050776, + "balance_loss_clip": 1.05792511, + "balance_loss_mlp": 1.02898479, + "epoch": 0.24000480971562557, + "flos": 20266331068800.0, + "grad_norm": 2.709575146016677, + "language_loss": 0.69701356, + "learning_rate": 3.5550923054939278e-06, + "loss": 0.71931875, + "num_input_tokens_seen": 42571455, + "router_z_loss_clip": 1.21777344, + "router_z_loss_mlp": 0.21801758, + "step": 1996, + "time_per_iteration": 2.802769184112549 + }, + { + "auxiliary_loss_clip": 0.01176715, + "auxiliary_loss_mlp": 0.01045027, + "balance_loss_clip": 1.0561254, + "balance_loss_mlp": 1.02277088, + "epoch": 0.24012505260626466, + "flos": 25443176866560.0, + "grad_norm": 2.1984742412103517, + "language_loss": 0.74716818, + "learning_rate": 3.5546023500259083e-06, + "loss": 0.76938558, + "num_input_tokens_seen": 42592550, + "router_z_loss_clip": 1.20605469, + "router_z_loss_mlp": 0.22253418, + "step": 1997, + "time_per_iteration": 2.843620538711548 + }, + { + "auxiliary_loss_clip": 0.01178465, + "auxiliary_loss_mlp": 0.01052865, + "balance_loss_clip": 1.05894971, + "balance_loss_mlp": 1.02898705, + "epoch": 0.24024529549690374, + "flos": 15553342529280.0, + "grad_norm": 2.5821772722776517, + "language_loss": 0.808424, + "learning_rate": 3.5541121587259477e-06, + "loss": 0.83073735, + "num_input_tokens_seen": 42610385, + "router_z_loss_clip": 1.19433594, + "router_z_loss_mlp": 0.23864746, + "step": 1998, + "time_per_iteration": 2.810551166534424 + }, + { + "auxiliary_loss_clip": 0.01079736, + "auxiliary_loss_mlp": 0.01031423, + "balance_loss_clip": 1.03370142, + "balance_loss_mlp": 1.02832341, + "epoch": 0.24036553838754285, + "flos": 57122351867520.0, + "grad_norm": 0.853433044975118, + "language_loss": 0.57859665, + "learning_rate": 3.553621731668408e-06, + "loss": 0.59970826, + "num_input_tokens_seen": 42673595, + "router_z_loss_clip": 0.4609375, + "router_z_loss_mlp": 0.03100586, + "step": 1999, + "time_per_iteration": 3.39955735206604 + }, + { + "auxiliary_loss_clip": 0.01175734, + "auxiliary_loss_mlp": 0.0105009, + "balance_loss_clip": 1.05446386, + "balance_loss_mlp": 1.02759528, + "epoch": 0.24048578127818193, + "flos": 24969946158720.0, + "grad_norm": 2.069150873011158, + "language_loss": 0.83203286, + "learning_rate": 3.553131068927688e-06, + "loss": 0.85429108, + "num_input_tokens_seen": 42692000, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.22497559, + "step": 2000, + "time_per_iteration": 3.0365517139434814 + }, + { + "auxiliary_loss_clip": 0.0116911, + "auxiliary_loss_mlp": 0.01050063, + "balance_loss_clip": 1.05405951, + "balance_loss_mlp": 1.02876067, + "epoch": 0.24060602416882101, + "flos": 23330947547520.0, + "grad_norm": 1.8756754200323633, + "language_loss": 0.8027727, + "learning_rate": 3.552640170578219e-06, + "loss": 0.8249644, + "num_input_tokens_seen": 42712250, + "router_z_loss_clip": 1.15087891, + "router_z_loss_mlp": 0.21313477, + "step": 2001, + "time_per_iteration": 2.937767744064331 + }, + { + "auxiliary_loss_clip": 0.0117259, + "auxiliary_loss_mlp": 0.01050476, + "balance_loss_clip": 1.05372286, + "balance_loss_mlp": 1.0307107, + "epoch": 0.2407262670594601, + "flos": 14173260128640.0, + "grad_norm": 2.4048112689137695, + "language_loss": 0.77598727, + "learning_rate": 3.5521490366944703e-06, + "loss": 0.79821789, + "num_input_tokens_seen": 42729900, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.19775391, + "step": 2002, + "time_per_iteration": 2.7788898944854736 + }, + { + "auxiliary_loss_clip": 0.01174114, + "auxiliary_loss_mlp": 0.01056188, + "balance_loss_clip": 1.05612242, + "balance_loss_mlp": 1.03346658, + "epoch": 0.2408465099500992, + "flos": 13663113217920.0, + "grad_norm": 3.4299569284012277, + "language_loss": 0.80157304, + "learning_rate": 3.5516576673509474e-06, + "loss": 0.82387596, + "num_input_tokens_seen": 42747900, + "router_z_loss_clip": 1.1796875, + "router_z_loss_mlp": 0.22729492, + "step": 2003, + "time_per_iteration": 2.7949390411376953 + }, + { + "auxiliary_loss_clip": 0.01173504, + "auxiliary_loss_mlp": 0.01045217, + "balance_loss_clip": 1.05320787, + "balance_loss_mlp": 1.023247, + "epoch": 0.2409667528407383, + "flos": 31248029076480.0, + "grad_norm": 2.150466357159812, + "language_loss": 0.8630029, + "learning_rate": 3.5511660626221896e-06, + "loss": 0.88519001, + "num_input_tokens_seen": 42768540, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.21960449, + "step": 2004, + "time_per_iteration": 2.8353593349456787 + }, + { + "auxiliary_loss_clip": 0.01178341, + "auxiliary_loss_mlp": 0.01054989, + "balance_loss_clip": 1.05624008, + "balance_loss_mlp": 1.03356719, + "epoch": 0.24108699573137737, + "flos": 22199941031040.0, + "grad_norm": 2.526831995735345, + "language_loss": 0.89096969, + "learning_rate": 3.5506742225827744e-06, + "loss": 0.91330302, + "num_input_tokens_seen": 42785395, + "router_z_loss_clip": 1.22070312, + "router_z_loss_mlp": 0.2142334, + "step": 2005, + "time_per_iteration": 2.7845799922943115 + }, + { + "auxiliary_loss_clip": 0.01177107, + "auxiliary_loss_mlp": 0.01058054, + "balance_loss_clip": 1.05626202, + "balance_loss_mlp": 1.0364536, + "epoch": 0.24120723862201648, + "flos": 26103035664000.0, + "grad_norm": 2.613684469535447, + "language_loss": 0.90073812, + "learning_rate": 3.5501821473073116e-06, + "loss": 0.92308974, + "num_input_tokens_seen": 42801980, + "router_z_loss_clip": 1.20898438, + "router_z_loss_mlp": 0.21606445, + "step": 2006, + "time_per_iteration": 2.802783966064453 + }, + { + "auxiliary_loss_clip": 0.01179104, + "auxiliary_loss_mlp": 0.01049096, + "balance_loss_clip": 1.05553031, + "balance_loss_mlp": 1.02699494, + "epoch": 0.24132748151265557, + "flos": 18624926246400.0, + "grad_norm": 2.1310264014056592, + "language_loss": 0.87044394, + "learning_rate": 3.54968983687045e-06, + "loss": 0.89272594, + "num_input_tokens_seen": 42818850, + "router_z_loss_clip": 1.23535156, + "router_z_loss_mlp": 0.22106934, + "step": 2007, + "time_per_iteration": 2.7714195251464844 + }, + { + "auxiliary_loss_clip": 0.01174897, + "auxiliary_loss_mlp": 0.01051718, + "balance_loss_clip": 1.05487454, + "balance_loss_mlp": 1.02949739, + "epoch": 0.24144772440329465, + "flos": 15267673664640.0, + "grad_norm": 2.5319299878807695, + "language_loss": 0.89211696, + "learning_rate": 3.549197291346872e-06, + "loss": 0.91438317, + "num_input_tokens_seen": 42835375, + "router_z_loss_clip": 1.20117188, + "router_z_loss_mlp": 0.22229004, + "step": 2008, + "time_per_iteration": 2.7608482837677 + }, + { + "auxiliary_loss_clip": 0.01171468, + "auxiliary_loss_mlp": 0.01053932, + "balance_loss_clip": 1.05506504, + "balance_loss_mlp": 1.03142524, + "epoch": 0.24156796729393373, + "flos": 24024274842240.0, + "grad_norm": 2.4796879868018626, + "language_loss": 0.7963919, + "learning_rate": 3.548704510811297e-06, + "loss": 0.81864589, + "num_input_tokens_seen": 42854570, + "router_z_loss_clip": 1.1640625, + "router_z_loss_mlp": 0.22509766, + "step": 2009, + "time_per_iteration": 3.0029726028442383 + }, + { + "auxiliary_loss_clip": 0.01174686, + "auxiliary_loss_mlp": 0.01046513, + "balance_loss_clip": 1.05239677, + "balance_loss_mlp": 1.02436376, + "epoch": 0.24168821018457284, + "flos": 26286790665600.0, + "grad_norm": 2.646378171279224, + "language_loss": 0.75043333, + "learning_rate": 3.5482114953384787e-06, + "loss": 0.77264524, + "num_input_tokens_seen": 42873800, + "router_z_loss_clip": 1.22167969, + "router_z_loss_mlp": 0.22131348, + "step": 2010, + "time_per_iteration": 3.7363364696502686 + }, + { + "auxiliary_loss_clip": 0.01170335, + "auxiliary_loss_mlp": 0.01052211, + "balance_loss_clip": 1.05317974, + "balance_loss_mlp": 1.0308373, + "epoch": 0.24180845307521193, + "flos": 18223193560320.0, + "grad_norm": 2.012164518593375, + "language_loss": 0.84181541, + "learning_rate": 3.5477182450032077e-06, + "loss": 0.86404085, + "num_input_tokens_seen": 42892400, + "router_z_loss_clip": 1.16992188, + "router_z_loss_mlp": 0.21398926, + "step": 2011, + "time_per_iteration": 2.9227194786071777 + }, + { + "auxiliary_loss_clip": 0.01174028, + "auxiliary_loss_mlp": 0.01049232, + "balance_loss_clip": 1.05526233, + "balance_loss_mlp": 1.0273329, + "epoch": 0.241928695965851, + "flos": 20449260057600.0, + "grad_norm": 2.376523072890232, + "language_loss": 0.83344084, + "learning_rate": 3.5472247598803097e-06, + "loss": 0.85567343, + "num_input_tokens_seen": 42911745, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.21875, + "step": 2012, + "time_per_iteration": 3.7237401008605957 + }, + { + "auxiliary_loss_clip": 0.01174537, + "auxiliary_loss_mlp": 0.01053091, + "balance_loss_clip": 1.05421019, + "balance_loss_mlp": 1.03021526, + "epoch": 0.24204893885649012, + "flos": 25556475340800.0, + "grad_norm": 2.834697015653854, + "language_loss": 0.8519038, + "learning_rate": 3.546731040044645e-06, + "loss": 0.87418008, + "num_input_tokens_seen": 42926915, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.22900391, + "step": 2013, + "time_per_iteration": 3.6806836128234863 + }, + { + "auxiliary_loss_clip": 0.01176055, + "auxiliary_loss_mlp": 0.01050352, + "balance_loss_clip": 1.0553782, + "balance_loss_mlp": 1.02927577, + "epoch": 0.2421691817471292, + "flos": 30660207004800.0, + "grad_norm": 1.9059130567309321, + "language_loss": 0.74868852, + "learning_rate": 3.546237085571112e-06, + "loss": 0.77095264, + "num_input_tokens_seen": 42945350, + "router_z_loss_clip": 1.20800781, + "router_z_loss_mlp": 0.21081543, + "step": 2014, + "time_per_iteration": 2.8458521366119385 + }, + { + "auxiliary_loss_clip": 0.01175238, + "auxiliary_loss_mlp": 0.01051604, + "balance_loss_clip": 1.05573857, + "balance_loss_mlp": 1.03008676, + "epoch": 0.24228942463776829, + "flos": 21945011230080.0, + "grad_norm": 2.7837274911110717, + "language_loss": 0.72090185, + "learning_rate": 3.5457428965346425e-06, + "loss": 0.74317026, + "num_input_tokens_seen": 42964290, + "router_z_loss_clip": 1.19433594, + "router_z_loss_mlp": 0.21533203, + "step": 2015, + "time_per_iteration": 2.8611013889312744 + }, + { + "auxiliary_loss_clip": 0.0116875, + "auxiliary_loss_mlp": 0.01048916, + "balance_loss_clip": 1.05387616, + "balance_loss_mlp": 1.02742231, + "epoch": 0.2424096675284074, + "flos": 33984493879680.0, + "grad_norm": 2.018152478389661, + "language_loss": 0.74663496, + "learning_rate": 3.545248473010205e-06, + "loss": 0.7688117, + "num_input_tokens_seen": 42987095, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.21496582, + "step": 2016, + "time_per_iteration": 2.9528982639312744 + }, + { + "auxiliary_loss_clip": 0.0118089, + "auxiliary_loss_mlp": 0.01061418, + "balance_loss_clip": 1.05685353, + "balance_loss_mlp": 1.03774309, + "epoch": 0.24252991041904648, + "flos": 21653416621440.0, + "grad_norm": 1.817060195484082, + "language_loss": 0.87791175, + "learning_rate": 3.544753815072802e-06, + "loss": 0.90033484, + "num_input_tokens_seen": 43005750, + "router_z_loss_clip": 1.24023438, + "router_z_loss_mlp": 0.23693848, + "step": 2017, + "time_per_iteration": 2.8402938842773438 + }, + { + "auxiliary_loss_clip": 0.01174555, + "auxiliary_loss_mlp": 0.01054844, + "balance_loss_clip": 1.05453777, + "balance_loss_mlp": 1.03412557, + "epoch": 0.24265015330968556, + "flos": 21870065502720.0, + "grad_norm": 2.0005375175335094, + "language_loss": 0.8816216, + "learning_rate": 3.544258922797474e-06, + "loss": 0.90391564, + "num_input_tokens_seen": 43023870, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.20727539, + "step": 2018, + "time_per_iteration": 2.839236259460449 + }, + { + "auxiliary_loss_clip": 0.01169979, + "auxiliary_loss_mlp": 0.01051562, + "balance_loss_clip": 1.05351603, + "balance_loss_mlp": 1.03053379, + "epoch": 0.24277039620032465, + "flos": 25628260671360.0, + "grad_norm": 4.137838257279796, + "language_loss": 0.78229332, + "learning_rate": 3.543763796259295e-06, + "loss": 0.80450869, + "num_input_tokens_seen": 43043825, + "router_z_loss_clip": 1.1640625, + "router_z_loss_mlp": 0.21032715, + "step": 2019, + "time_per_iteration": 2.8673081398010254 + }, + { + "auxiliary_loss_clip": 0.01174844, + "auxiliary_loss_mlp": 0.01050651, + "balance_loss_clip": 1.05236745, + "balance_loss_mlp": 1.02832317, + "epoch": 0.24289063909096376, + "flos": 26286575184000.0, + "grad_norm": 1.9575276429068718, + "language_loss": 0.90998769, + "learning_rate": 3.5432684355333754e-06, + "loss": 0.93224263, + "num_input_tokens_seen": 43062480, + "router_z_loss_clip": 1.22558594, + "router_z_loss_mlp": 0.22314453, + "step": 2020, + "time_per_iteration": 3.003074884414673 + }, + { + "auxiliary_loss_clip": 0.01171726, + "auxiliary_loss_mlp": 0.01052856, + "balance_loss_clip": 1.05421484, + "balance_loss_mlp": 1.03117144, + "epoch": 0.24301088198160284, + "flos": 25075056332160.0, + "grad_norm": 1.941993528162839, + "language_loss": 0.7663911, + "learning_rate": 3.5427728406948613e-06, + "loss": 0.78863692, + "num_input_tokens_seen": 43081595, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.21691895, + "step": 2021, + "time_per_iteration": 2.7908785343170166 + }, + { + "auxiliary_loss_clip": 0.01073035, + "auxiliary_loss_mlp": 0.01012779, + "balance_loss_clip": 1.02498925, + "balance_loss_mlp": 1.00965559, + "epoch": 0.24313112487224192, + "flos": 69900948673920.0, + "grad_norm": 0.7530813538955227, + "language_loss": 0.57938874, + "learning_rate": 3.542277011818934e-06, + "loss": 0.60024685, + "num_input_tokens_seen": 43145430, + "router_z_loss_clip": 0.48046875, + "router_z_loss_mlp": 0.03112793, + "step": 2022, + "time_per_iteration": 3.4342336654663086 + }, + { + "auxiliary_loss_clip": 0.01169647, + "auxiliary_loss_mlp": 0.01057353, + "balance_loss_clip": 1.05349422, + "balance_loss_mlp": 1.03664625, + "epoch": 0.24325136776288103, + "flos": 40662334235520.0, + "grad_norm": 2.715737304097329, + "language_loss": 0.74162608, + "learning_rate": 3.5417809489808104e-06, + "loss": 0.76389611, + "num_input_tokens_seen": 43167040, + "router_z_loss_clip": 1.16113281, + "router_z_loss_mlp": 0.20715332, + "step": 2023, + "time_per_iteration": 3.0468854904174805 + }, + { + "auxiliary_loss_clip": 0.01167253, + "auxiliary_loss_mlp": 0.01049577, + "balance_loss_clip": 1.05266786, + "balance_loss_mlp": 1.02980042, + "epoch": 0.24337161065352012, + "flos": 25046400257280.0, + "grad_norm": 2.2073068785079926, + "language_loss": 0.72541559, + "learning_rate": 3.5412846522557422e-06, + "loss": 0.74758387, + "num_input_tokens_seen": 43187930, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.19775391, + "step": 2024, + "time_per_iteration": 2.8368473052978516 + }, + { + "auxiliary_loss_clip": 0.01172128, + "auxiliary_loss_mlp": 0.01051213, + "balance_loss_clip": 1.05368102, + "balance_loss_mlp": 1.02952874, + "epoch": 0.2434918535441592, + "flos": 18661160090880.0, + "grad_norm": 2.2625054468030017, + "language_loss": 0.74077046, + "learning_rate": 3.540788121719018e-06, + "loss": 0.76300395, + "num_input_tokens_seen": 43206350, + "router_z_loss_clip": 1.18457031, + "router_z_loss_mlp": 0.21691895, + "step": 2025, + "time_per_iteration": 2.7816805839538574 + }, + { + "auxiliary_loss_clip": 0.01169605, + "auxiliary_loss_mlp": 0.01049025, + "balance_loss_clip": 1.0513463, + "balance_loss_mlp": 1.02761531, + "epoch": 0.24361209643479828, + "flos": 23915142345600.0, + "grad_norm": 1.9511359635621122, + "language_loss": 0.82157463, + "learning_rate": 3.5402913574459604e-06, + "loss": 0.84376097, + "num_input_tokens_seen": 43226255, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.21411133, + "step": 2026, + "time_per_iteration": 2.787785530090332 + }, + { + "auxiliary_loss_clip": 0.01166744, + "auxiliary_loss_mlp": 0.01053738, + "balance_loss_clip": 1.05106902, + "balance_loss_mlp": 1.03350806, + "epoch": 0.2437323393254374, + "flos": 28657505232000.0, + "grad_norm": 2.3844849273727635, + "language_loss": 0.85418981, + "learning_rate": 3.5397943595119297e-06, + "loss": 0.87639463, + "num_input_tokens_seen": 43247675, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.20227051, + "step": 2027, + "time_per_iteration": 2.7866578102111816 + }, + { + "auxiliary_loss_clip": 0.0117186, + "auxiliary_loss_mlp": 0.01052739, + "balance_loss_clip": 1.05185068, + "balance_loss_mlp": 1.03193688, + "epoch": 0.24385258221607647, + "flos": 23550325862400.0, + "grad_norm": 2.857481299201782, + "language_loss": 0.77213454, + "learning_rate": 3.5392971279923177e-06, + "loss": 0.79438055, + "num_input_tokens_seen": 43265895, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.20788574, + "step": 2028, + "time_per_iteration": 2.763505458831787 + }, + { + "auxiliary_loss_clip": 0.01175291, + "auxiliary_loss_mlp": 0.01050679, + "balance_loss_clip": 1.052966, + "balance_loss_mlp": 1.02932882, + "epoch": 0.24397282510671556, + "flos": 25336091445120.0, + "grad_norm": 2.23850643766789, + "language_loss": 0.82938164, + "learning_rate": 3.5387996629625557e-06, + "loss": 0.85164142, + "num_input_tokens_seen": 43283485, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.21350098, + "step": 2029, + "time_per_iteration": 2.8115346431732178 + }, + { + "auxiliary_loss_clip": 0.01071026, + "auxiliary_loss_mlp": 0.01005202, + "balance_loss_clip": 1.02389657, + "balance_loss_mlp": 1.00244844, + "epoch": 0.24409306799735467, + "flos": 65187421430400.0, + "grad_norm": 0.8093023927513465, + "language_loss": 0.54988849, + "learning_rate": 3.5383019644981083e-06, + "loss": 0.57065082, + "num_input_tokens_seen": 43347180, + "router_z_loss_clip": 0.47070312, + "router_z_loss_mlp": 0.02758789, + "step": 2030, + "time_per_iteration": 3.298229932785034 + }, + { + "auxiliary_loss_clip": 0.01175649, + "auxiliary_loss_mlp": 0.01054455, + "balance_loss_clip": 1.05439675, + "balance_loss_mlp": 1.03277051, + "epoch": 0.24421331088799375, + "flos": 19537093152000.0, + "grad_norm": 2.645530754442532, + "language_loss": 0.72815925, + "learning_rate": 3.5378040326744763e-06, + "loss": 0.75046027, + "num_input_tokens_seen": 43366665, + "router_z_loss_clip": 1.21191406, + "router_z_loss_mlp": 0.21679688, + "step": 2031, + "time_per_iteration": 2.9121768474578857 + }, + { + "auxiliary_loss_clip": 0.01170955, + "auxiliary_loss_mlp": 0.01051061, + "balance_loss_clip": 1.0557853, + "balance_loss_mlp": 1.03251219, + "epoch": 0.24433355377863283, + "flos": 21068575378560.0, + "grad_norm": 2.9521311294085932, + "language_loss": 0.85706872, + "learning_rate": 3.5373058675671946e-06, + "loss": 0.87928891, + "num_input_tokens_seen": 43384670, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.1854248, + "step": 2032, + "time_per_iteration": 2.807405710220337 + }, + { + "auxiliary_loss_clip": 0.01174293, + "auxiliary_loss_mlp": 0.01047393, + "balance_loss_clip": 1.05352783, + "balance_loss_mlp": 1.02713966, + "epoch": 0.24445379666927192, + "flos": 22637189289600.0, + "grad_norm": 4.837203830224538, + "language_loss": 0.72580791, + "learning_rate": 3.536807469251836e-06, + "loss": 0.74802482, + "num_input_tokens_seen": 43403825, + "router_z_loss_clip": 1.20800781, + "router_z_loss_mlp": 0.20263672, + "step": 2033, + "time_per_iteration": 2.900555372238159 + }, + { + "auxiliary_loss_clip": 0.01170881, + "auxiliary_loss_mlp": 0.010503, + "balance_loss_clip": 1.05247867, + "balance_loss_mlp": 1.02949834, + "epoch": 0.24457403955991103, + "flos": 21251612108160.0, + "grad_norm": 2.1357472021108093, + "language_loss": 0.82925344, + "learning_rate": 3.5363088378040055e-06, + "loss": 0.85146523, + "num_input_tokens_seen": 43422715, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.20800781, + "step": 2034, + "time_per_iteration": 2.8244950771331787 + }, + { + "auxiliary_loss_clip": 0.01069889, + "auxiliary_loss_mlp": 0.01000769, + "balance_loss_clip": 1.02332914, + "balance_loss_mlp": 0.99775308, + "epoch": 0.2446942824505501, + "flos": 66997820764800.0, + "grad_norm": 0.7629903016416916, + "language_loss": 0.64313388, + "learning_rate": 3.5358099732993463e-06, + "loss": 0.66384053, + "num_input_tokens_seen": 43481825, + "router_z_loss_clip": 0.46484375, + "router_z_loss_mlp": 0.03015137, + "step": 2035, + "time_per_iteration": 3.295337677001953 + }, + { + "auxiliary_loss_clip": 0.01170774, + "auxiliary_loss_mlp": 0.01045529, + "balance_loss_clip": 1.05501938, + "balance_loss_mlp": 1.02549028, + "epoch": 0.2448145253411892, + "flos": 20411122792320.0, + "grad_norm": 3.1590779103365225, + "language_loss": 0.89585698, + "learning_rate": 3.535310875813535e-06, + "loss": 0.91802001, + "num_input_tokens_seen": 43500220, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.20031738, + "step": 2036, + "time_per_iteration": 2.938042163848877 + }, + { + "auxiliary_loss_clip": 0.01171446, + "auxiliary_loss_mlp": 0.01064731, + "balance_loss_clip": 1.05168915, + "balance_loss_mlp": 1.04345226, + "epoch": 0.2449347682318283, + "flos": 28804739080320.0, + "grad_norm": 2.450608365136224, + "language_loss": 0.8164537, + "learning_rate": 3.5348115454222843e-06, + "loss": 0.83881551, + "num_input_tokens_seen": 43522805, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.21264648, + "step": 2037, + "time_per_iteration": 2.950237274169922 + }, + { + "auxiliary_loss_clip": 0.01171793, + "auxiliary_loss_mlp": 0.01050346, + "balance_loss_clip": 1.05178428, + "balance_loss_mlp": 1.03004479, + "epoch": 0.2450550111224674, + "flos": 22528990546560.0, + "grad_norm": 2.018673344717236, + "language_loss": 0.85942292, + "learning_rate": 3.5343119822013425e-06, + "loss": 0.88164437, + "num_input_tokens_seen": 43541915, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.20288086, + "step": 2038, + "time_per_iteration": 2.8593297004699707 + }, + { + "auxiliary_loss_clip": 0.01182706, + "auxiliary_loss_mlp": 0.01052933, + "balance_loss_clip": 1.05969381, + "balance_loss_mlp": 1.03189206, + "epoch": 0.24517525401310647, + "flos": 21759137326080.0, + "grad_norm": 3.516888771082052, + "language_loss": 0.77026814, + "learning_rate": 3.533812186226493e-06, + "loss": 0.79262447, + "num_input_tokens_seen": 43562625, + "router_z_loss_clip": 1.22949219, + "router_z_loss_mlp": 0.21032715, + "step": 2039, + "time_per_iteration": 3.6872081756591797 + }, + { + "auxiliary_loss_clip": 0.01168864, + "auxiliary_loss_mlp": 0.01043196, + "balance_loss_clip": 1.05381942, + "balance_loss_mlp": 1.02328837, + "epoch": 0.24529549690374555, + "flos": 25043311687680.0, + "grad_norm": 2.0717044649168597, + "language_loss": 0.75900722, + "learning_rate": 3.5333121575735545e-06, + "loss": 0.78112781, + "num_input_tokens_seen": 43582265, + "router_z_loss_clip": 1.15039062, + "router_z_loss_mlp": 0.19885254, + "step": 2040, + "time_per_iteration": 3.6413586139678955 + }, + { + "auxiliary_loss_clip": 0.01167691, + "auxiliary_loss_mlp": 0.01047261, + "balance_loss_clip": 1.05161881, + "balance_loss_mlp": 1.02715087, + "epoch": 0.24541573979438466, + "flos": 32123638915200.0, + "grad_norm": 1.9123957328768597, + "language_loss": 0.75543475, + "learning_rate": 3.532811896318381e-06, + "loss": 0.77758425, + "num_input_tokens_seen": 43604335, + "router_z_loss_clip": 1.16113281, + "router_z_loss_mlp": 0.2010498, + "step": 2041, + "time_per_iteration": 2.8728044033050537 + }, + { + "auxiliary_loss_clip": 0.01174973, + "auxiliary_loss_mlp": 0.01045611, + "balance_loss_clip": 1.05602002, + "balance_loss_mlp": 1.02581024, + "epoch": 0.24553598268502375, + "flos": 31357556622720.0, + "grad_norm": 5.171112400306932, + "language_loss": 0.8183552, + "learning_rate": 3.5323114025368615e-06, + "loss": 0.84056103, + "num_input_tokens_seen": 43619400, + "router_z_loss_clip": 1.19140625, + "router_z_loss_mlp": 0.19799805, + "step": 2042, + "time_per_iteration": 3.6605844497680664 + }, + { + "auxiliary_loss_clip": 0.01167165, + "auxiliary_loss_mlp": 0.01039032, + "balance_loss_clip": 1.04984999, + "balance_loss_mlp": 1.01840878, + "epoch": 0.24565622557566283, + "flos": 14027462824320.0, + "grad_norm": 5.59188506359502, + "language_loss": 0.82109278, + "learning_rate": 3.53181067630492e-06, + "loss": 0.84315473, + "num_input_tokens_seen": 43636870, + "router_z_loss_clip": 1.17382812, + "router_z_loss_mlp": 0.20617676, + "step": 2043, + "time_per_iteration": 2.822396755218506 + }, + { + "auxiliary_loss_clip": 0.0117366, + "auxiliary_loss_mlp": 0.0104799, + "balance_loss_clip": 1.05357027, + "balance_loss_mlp": 1.02797508, + "epoch": 0.24577646846630194, + "flos": 16581465515520.0, + "grad_norm": 2.08257384297932, + "language_loss": 0.76107264, + "learning_rate": 3.5313097176985175e-06, + "loss": 0.78328907, + "num_input_tokens_seen": 43655180, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.20019531, + "step": 2044, + "time_per_iteration": 2.7862720489501953 + }, + { + "auxiliary_loss_clip": 0.01172004, + "auxiliary_loss_mlp": 0.01048767, + "balance_loss_clip": 1.05340528, + "balance_loss_mlp": 1.02941918, + "epoch": 0.24589671135694102, + "flos": 18807424272000.0, + "grad_norm": 3.627194148193776, + "language_loss": 0.81363636, + "learning_rate": 3.5308085267936482e-06, + "loss": 0.8358441, + "num_input_tokens_seen": 43672895, + "router_z_loss_clip": 1.18457031, + "router_z_loss_mlp": 0.19360352, + "step": 2045, + "time_per_iteration": 2.7592504024505615 + }, + { + "auxiliary_loss_clip": 0.01166452, + "auxiliary_loss_mlp": 0.01041179, + "balance_loss_clip": 1.05281138, + "balance_loss_mlp": 1.02230787, + "epoch": 0.2460169542475801, + "flos": 19938538529280.0, + "grad_norm": 1.7228182093622653, + "language_loss": 0.89705223, + "learning_rate": 3.530307103666342e-06, + "loss": 0.91912854, + "num_input_tokens_seen": 43691975, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.18859863, + "step": 2046, + "time_per_iteration": 2.875683546066284 + }, + { + "auxiliary_loss_clip": 0.01170586, + "auxiliary_loss_mlp": 0.01042915, + "balance_loss_clip": 1.05420208, + "balance_loss_mlp": 1.02212453, + "epoch": 0.24613719713821922, + "flos": 24171221381760.0, + "grad_norm": 2.0686627039734233, + "language_loss": 0.80218601, + "learning_rate": 3.5298054483926658e-06, + "loss": 0.82432097, + "num_input_tokens_seen": 43712670, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.20800781, + "step": 2047, + "time_per_iteration": 2.8940746784210205 + }, + { + "auxiliary_loss_clip": 0.01175891, + "auxiliary_loss_mlp": 0.0105671, + "balance_loss_clip": 1.05667651, + "balance_loss_mlp": 1.03634858, + "epoch": 0.2462574400288583, + "flos": 30221055325440.0, + "grad_norm": 2.0395277338627325, + "language_loss": 0.82914257, + "learning_rate": 3.5293035610487187e-06, + "loss": 0.85146856, + "num_input_tokens_seen": 43732035, + "router_z_loss_clip": 1.19140625, + "router_z_loss_mlp": 0.20361328, + "step": 2048, + "time_per_iteration": 2.846271276473999 + }, + { + "auxiliary_loss_clip": 0.01069274, + "auxiliary_loss_mlp": 0.01005262, + "balance_loss_clip": 1.02428269, + "balance_loss_mlp": 1.00209069, + "epoch": 0.24637768291949738, + "flos": 68943030819840.0, + "grad_norm": 0.7389842691821334, + "language_loss": 0.6197204, + "learning_rate": 3.5288014417106374e-06, + "loss": 0.64046574, + "num_input_tokens_seen": 43798055, + "router_z_loss_clip": 0.44921875, + "router_z_loss_mlp": 0.03173828, + "step": 2049, + "time_per_iteration": 3.343125343322754 + }, + { + "auxiliary_loss_clip": 0.01170745, + "auxiliary_loss_mlp": 0.01045624, + "balance_loss_clip": 1.05374956, + "balance_loss_mlp": 1.02327204, + "epoch": 0.24649792581013646, + "flos": 34383999922560.0, + "grad_norm": 2.1098998565943288, + "language_loss": 0.75217021, + "learning_rate": 3.528299090454593e-06, + "loss": 0.77433383, + "num_input_tokens_seen": 43818590, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.22363281, + "step": 2050, + "time_per_iteration": 2.900407314300537 + }, + { + "auxiliary_loss_clip": 0.01173764, + "auxiliary_loss_mlp": 0.01047307, + "balance_loss_clip": 1.05538368, + "balance_loss_mlp": 1.02700603, + "epoch": 0.24661816870077558, + "flos": 19680448331520.0, + "grad_norm": 3.9383976113007315, + "language_loss": 0.8280977, + "learning_rate": 3.527796507356792e-06, + "loss": 0.85030836, + "num_input_tokens_seen": 43832480, + "router_z_loss_clip": 1.18164062, + "router_z_loss_mlp": 0.20275879, + "step": 2051, + "time_per_iteration": 2.9893293380737305 + }, + { + "auxiliary_loss_clip": 0.01172926, + "auxiliary_loss_mlp": 0.01048944, + "balance_loss_clip": 1.05235243, + "balance_loss_mlp": 1.02852321, + "epoch": 0.24673841159141466, + "flos": 20002279213440.0, + "grad_norm": 2.9214938978599454, + "language_loss": 0.9008109, + "learning_rate": 3.527293692493475e-06, + "loss": 0.9230296, + "num_input_tokens_seen": 43848345, + "router_z_loss_clip": 1.20507812, + "router_z_loss_mlp": 0.20410156, + "step": 2052, + "time_per_iteration": 2.9244701862335205 + }, + { + "auxiliary_loss_clip": 0.01177198, + "auxiliary_loss_mlp": 0.01046725, + "balance_loss_clip": 1.05531359, + "balance_loss_mlp": 1.02490926, + "epoch": 0.24685865448205374, + "flos": 21646593037440.0, + "grad_norm": 4.1976519831875825, + "language_loss": 0.73409677, + "learning_rate": 3.52679064594092e-06, + "loss": 0.75633597, + "num_input_tokens_seen": 43865685, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.21813965, + "step": 2053, + "time_per_iteration": 2.8543307781219482 + }, + { + "auxiliary_loss_clip": 0.01169062, + "auxiliary_loss_mlp": 0.01047223, + "balance_loss_clip": 1.05291867, + "balance_loss_mlp": 1.02773178, + "epoch": 0.24697889737269285, + "flos": 17960470508160.0, + "grad_norm": 2.3111665665383714, + "language_loss": 0.74972928, + "learning_rate": 3.5262873677754375e-06, + "loss": 0.77189207, + "num_input_tokens_seen": 43883690, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.19506836, + "step": 2054, + "time_per_iteration": 2.838538408279419 + }, + { + "auxiliary_loss_clip": 0.01167932, + "auxiliary_loss_mlp": 0.01044558, + "balance_loss_clip": 1.05278993, + "balance_loss_mlp": 1.02579403, + "epoch": 0.24709914026333193, + "flos": 27344611221120.0, + "grad_norm": 1.8063908847422903, + "language_loss": 0.80538726, + "learning_rate": 3.5257838580733745e-06, + "loss": 0.82751215, + "num_input_tokens_seen": 43903295, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.18762207, + "step": 2055, + "time_per_iteration": 2.8299624919891357 + }, + { + "auxiliary_loss_clip": 0.01170612, + "auxiliary_loss_mlp": 0.01044565, + "balance_loss_clip": 1.05393326, + "balance_loss_mlp": 1.02552736, + "epoch": 0.24721938315397102, + "flos": 19275519335040.0, + "grad_norm": 2.9091373827836553, + "language_loss": 0.8750844, + "learning_rate": 3.5252801169111138e-06, + "loss": 0.89723623, + "num_input_tokens_seen": 43920960, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.19042969, + "step": 2056, + "time_per_iteration": 2.805734395980835 + }, + { + "auxiliary_loss_clip": 0.01170459, + "auxiliary_loss_mlp": 0.01047874, + "balance_loss_clip": 1.05475616, + "balance_loss_mlp": 1.02863336, + "epoch": 0.2473396260446101, + "flos": 23185796688000.0, + "grad_norm": 1.9326831389215604, + "language_loss": 0.80005473, + "learning_rate": 3.524776144365072e-06, + "loss": 0.82223809, + "num_input_tokens_seen": 43939415, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.19226074, + "step": 2057, + "time_per_iteration": 2.791738271713257 + }, + { + "auxiliary_loss_clip": 0.01171076, + "auxiliary_loss_mlp": 0.01052752, + "balance_loss_clip": 1.05698848, + "balance_loss_mlp": 1.0322485, + "epoch": 0.2474598689352492, + "flos": 21142443697920.0, + "grad_norm": 1.679045673102671, + "language_loss": 0.79303294, + "learning_rate": 3.5242719405117016e-06, + "loss": 0.81527126, + "num_input_tokens_seen": 43959220, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.20495605, + "step": 2058, + "time_per_iteration": 2.8408114910125732 + }, + { + "auxiliary_loss_clip": 0.01172468, + "auxiliary_loss_mlp": 0.01054291, + "balance_loss_clip": 1.05671978, + "balance_loss_mlp": 1.03348935, + "epoch": 0.2475801118258883, + "flos": 21648352803840.0, + "grad_norm": 4.485431692994157, + "language_loss": 0.74970502, + "learning_rate": 3.5237675054274893e-06, + "loss": 0.77197266, + "num_input_tokens_seen": 43978420, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.20800781, + "step": 2059, + "time_per_iteration": 2.896700859069824 + }, + { + "auxiliary_loss_clip": 0.01172948, + "auxiliary_loss_mlp": 0.0104803, + "balance_loss_clip": 1.05663753, + "balance_loss_mlp": 1.02687001, + "epoch": 0.24770035471652738, + "flos": 22674500542080.0, + "grad_norm": 2.1778154785151096, + "language_loss": 0.80017668, + "learning_rate": 3.5232628391889584e-06, + "loss": 0.8223865, + "num_input_tokens_seen": 43996710, + "router_z_loss_clip": 1.1640625, + "router_z_loss_mlp": 0.21166992, + "step": 2060, + "time_per_iteration": 2.8216440677642822 + }, + { + "auxiliary_loss_clip": 0.01170349, + "auxiliary_loss_mlp": 0.01045169, + "balance_loss_clip": 1.05532682, + "balance_loss_mlp": 1.02498746, + "epoch": 0.2478205976071665, + "flos": 22163814927360.0, + "grad_norm": 2.4589207494918672, + "language_loss": 0.6404174, + "learning_rate": 3.522757941872666e-06, + "loss": 0.66257256, + "num_input_tokens_seen": 44014865, + "router_z_loss_clip": 1.15039062, + "router_z_loss_mlp": 0.2019043, + "step": 2061, + "time_per_iteration": 2.8176801204681396 + }, + { + "auxiliary_loss_clip": 0.01174512, + "auxiliary_loss_mlp": 0.01049147, + "balance_loss_clip": 1.05700672, + "balance_loss_mlp": 1.0292871, + "epoch": 0.24794084049780557, + "flos": 24973106555520.0, + "grad_norm": 1.7463720009792532, + "language_loss": 0.82486916, + "learning_rate": 3.5222528135552042e-06, + "loss": 0.8471058, + "num_input_tokens_seen": 44036325, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.19873047, + "step": 2062, + "time_per_iteration": 2.8562469482421875 + }, + { + "auxiliary_loss_clip": 0.01174252, + "auxiliary_loss_mlp": 0.01047931, + "balance_loss_clip": 1.05841351, + "balance_loss_mlp": 1.02699828, + "epoch": 0.24806108338844465, + "flos": 18296379521280.0, + "grad_norm": 2.9312693948885675, + "language_loss": 0.8006736, + "learning_rate": 3.521747454313201e-06, + "loss": 0.82289547, + "num_input_tokens_seen": 44055005, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.20947266, + "step": 2063, + "time_per_iteration": 2.749258279800415 + }, + { + "auxiliary_loss_clip": 0.01173718, + "auxiliary_loss_mlp": 0.01055872, + "balance_loss_clip": 1.05557847, + "balance_loss_mlp": 1.0350337, + "epoch": 0.24818132627908374, + "flos": 19282163351040.0, + "grad_norm": 2.8876936471609977, + "language_loss": 0.66670179, + "learning_rate": 3.521241864223319e-06, + "loss": 0.68899775, + "num_input_tokens_seen": 44073965, + "router_z_loss_clip": 1.18066406, + "router_z_loss_mlp": 0.20825195, + "step": 2064, + "time_per_iteration": 2.7733712196350098 + }, + { + "auxiliary_loss_clip": 0.01071081, + "auxiliary_loss_mlp": 0.01023798, + "balance_loss_clip": 1.02604938, + "balance_loss_mlp": 1.02055538, + "epoch": 0.24830156916972285, + "flos": 70285837881600.0, + "grad_norm": 0.8032660355162841, + "language_loss": 0.61950827, + "learning_rate": 3.5207360433622552e-06, + "loss": 0.64045715, + "num_input_tokens_seen": 44135965, + "router_z_loss_clip": 0.44921875, + "router_z_loss_mlp": 0.0324707, + "step": 2065, + "time_per_iteration": 3.287520170211792 + }, + { + "auxiliary_loss_clip": 0.01168881, + "auxiliary_loss_mlp": 0.01059547, + "balance_loss_clip": 1.05651462, + "balance_loss_mlp": 1.039258, + "epoch": 0.24842181206036193, + "flos": 40409128287360.0, + "grad_norm": 6.8447084256705, + "language_loss": 0.74595648, + "learning_rate": 3.5202299918067437e-06, + "loss": 0.76824081, + "num_input_tokens_seen": 44159560, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.20300293, + "step": 2066, + "time_per_iteration": 2.9440014362335205 + }, + { + "auxiliary_loss_clip": 0.01166276, + "auxiliary_loss_mlp": 0.01049219, + "balance_loss_clip": 1.05287015, + "balance_loss_mlp": 1.02976394, + "epoch": 0.248542054951001, + "flos": 20082432412800.0, + "grad_norm": 2.7057733254307057, + "language_loss": 0.69508231, + "learning_rate": 3.519723709633551e-06, + "loss": 0.71723723, + "num_input_tokens_seen": 44178320, + "router_z_loss_clip": 1.13574219, + "router_z_loss_mlp": 0.19470215, + "step": 2067, + "time_per_iteration": 2.813337564468384 + }, + { + "auxiliary_loss_clip": 0.01169519, + "auxiliary_loss_mlp": 0.01056495, + "balance_loss_clip": 1.05442727, + "balance_loss_mlp": 1.03596735, + "epoch": 0.24866229784164012, + "flos": 23513948363520.0, + "grad_norm": 2.199309880396974, + "language_loss": 0.83694994, + "learning_rate": 3.519217196919479e-06, + "loss": 0.85921013, + "num_input_tokens_seen": 44197305, + "router_z_loss_clip": 1.15039062, + "router_z_loss_mlp": 0.20544434, + "step": 2068, + "time_per_iteration": 4.431660890579224 + }, + { + "auxiliary_loss_clip": 0.01167816, + "auxiliary_loss_mlp": 0.01050574, + "balance_loss_clip": 1.05459559, + "balance_loss_mlp": 1.03167951, + "epoch": 0.2487825407322792, + "flos": 19865101173120.0, + "grad_norm": 8.103262880617002, + "language_loss": 0.73414785, + "learning_rate": 3.518710453741367e-06, + "loss": 0.7563318, + "num_input_tokens_seen": 44216505, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.1887207, + "step": 2069, + "time_per_iteration": 3.657946825027466 + }, + { + "auxiliary_loss_clip": 0.01169802, + "auxiliary_loss_mlp": 0.01044159, + "balance_loss_clip": 1.05290842, + "balance_loss_mlp": 1.02386999, + "epoch": 0.2489027836229183, + "flos": 22017622573440.0, + "grad_norm": 3.9798243649585077, + "language_loss": 0.67818248, + "learning_rate": 3.518203480176086e-06, + "loss": 0.70032215, + "num_input_tokens_seen": 44235435, + "router_z_loss_clip": 1.16894531, + "router_z_loss_mlp": 0.20300293, + "step": 2070, + "time_per_iteration": 2.8071587085723877 + }, + { + "auxiliary_loss_clip": 0.01169376, + "auxiliary_loss_mlp": 0.01051792, + "balance_loss_clip": 1.05261588, + "balance_loss_mlp": 1.0311451, + "epoch": 0.2490230265135574, + "flos": 23294354567040.0, + "grad_norm": 1.9301077060277156, + "language_loss": 0.80894697, + "learning_rate": 3.517696276300545e-06, + "loss": 0.83115864, + "num_input_tokens_seen": 44256975, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.20654297, + "step": 2071, + "time_per_iteration": 3.7725188732147217 + }, + { + "auxiliary_loss_clip": 0.01177811, + "auxiliary_loss_mlp": 0.01053584, + "balance_loss_clip": 1.05808246, + "balance_loss_mlp": 1.03244853, + "epoch": 0.24914326940419648, + "flos": 19826784339840.0, + "grad_norm": 2.954190889644693, + "language_loss": 0.68698478, + "learning_rate": 3.517188842191685e-06, + "loss": 0.70929873, + "num_input_tokens_seen": 44275125, + "router_z_loss_clip": 1.19628906, + "router_z_loss_mlp": 0.21130371, + "step": 2072, + "time_per_iteration": 2.88447904586792 + }, + { + "auxiliary_loss_clip": 0.0117225, + "auxiliary_loss_mlp": 0.01043496, + "balance_loss_clip": 1.05360126, + "balance_loss_mlp": 1.02120352, + "epoch": 0.24926351229483557, + "flos": 20229271211520.0, + "grad_norm": 1.7788637008925654, + "language_loss": 0.73852617, + "learning_rate": 3.5166811779264837e-06, + "loss": 0.7606836, + "num_input_tokens_seen": 44295445, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.22290039, + "step": 2073, + "time_per_iteration": 2.8599770069122314 + }, + { + "auxiliary_loss_clip": 0.01171356, + "auxiliary_loss_mlp": 0.01047813, + "balance_loss_clip": 1.05292344, + "balance_loss_mlp": 1.02653384, + "epoch": 0.24938375518547465, + "flos": 23294570048640.0, + "grad_norm": 2.007878644153299, + "language_loss": 0.77930069, + "learning_rate": 3.5161732835819545e-06, + "loss": 0.80149245, + "num_input_tokens_seen": 44314755, + "router_z_loss_clip": 1.18554688, + "router_z_loss_mlp": 0.21276855, + "step": 2074, + "time_per_iteration": 2.837536096572876 + }, + { + "auxiliary_loss_clip": 0.01170386, + "auxiliary_loss_mlp": 0.01049327, + "balance_loss_clip": 1.05466437, + "balance_loss_mlp": 1.02889466, + "epoch": 0.24950399807611376, + "flos": 17311673099520.0, + "grad_norm": 1.943090474590334, + "language_loss": 0.83448386, + "learning_rate": 3.515665159235143e-06, + "loss": 0.85668093, + "num_input_tokens_seen": 44333640, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.2043457, + "step": 2075, + "time_per_iteration": 2.8547158241271973 + }, + { + "auxiliary_loss_clip": 0.01166343, + "auxiliary_loss_mlp": 0.01046536, + "balance_loss_clip": 1.0531106, + "balance_loss_mlp": 1.02573431, + "epoch": 0.24962424096675284, + "flos": 19024863252480.0, + "grad_norm": 1.7514748976952708, + "language_loss": 0.75007588, + "learning_rate": 3.5151568049631318e-06, + "loss": 0.77220464, + "num_input_tokens_seen": 44352355, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.20800781, + "step": 2076, + "time_per_iteration": 2.842897653579712 + }, + { + "auxiliary_loss_clip": 0.01169796, + "auxiliary_loss_mlp": 0.01050168, + "balance_loss_clip": 1.0505445, + "balance_loss_mlp": 1.02879333, + "epoch": 0.24974448385739192, + "flos": 33398790710400.0, + "grad_norm": 2.736052724306278, + "language_loss": 0.80745399, + "learning_rate": 3.5146482208430385e-06, + "loss": 0.82965368, + "num_input_tokens_seen": 44374185, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.21362305, + "step": 2077, + "time_per_iteration": 2.876075029373169 + }, + { + "auxiliary_loss_clip": 0.0117593, + "auxiliary_loss_mlp": 0.01048144, + "balance_loss_clip": 1.05355895, + "balance_loss_mlp": 1.02598321, + "epoch": 0.24986472674803104, + "flos": 30007279532160.0, + "grad_norm": 2.378497777622374, + "language_loss": 0.68133068, + "learning_rate": 3.514139406952014e-06, + "loss": 0.70357144, + "num_input_tokens_seen": 44396210, + "router_z_loss_clip": 1.22363281, + "router_z_loss_mlp": 0.22131348, + "step": 2078, + "time_per_iteration": 2.879794120788574 + }, + { + "auxiliary_loss_clip": 0.01166465, + "auxiliary_loss_mlp": 0.01046787, + "balance_loss_clip": 1.0518508, + "balance_loss_mlp": 1.0266645, + "epoch": 0.24998496963867012, + "flos": 26613074833920.0, + "grad_norm": 2.4080400721142867, + "language_loss": 0.837641, + "learning_rate": 3.5136303633672454e-06, + "loss": 0.85977352, + "num_input_tokens_seen": 44416340, + "router_z_loss_clip": 1.14648438, + "router_z_loss_mlp": 0.20129395, + "step": 2079, + "time_per_iteration": 2.9238953590393066 + }, + { + "auxiliary_loss_clip": 0.01169339, + "auxiliary_loss_mlp": 0.01054983, + "balance_loss_clip": 1.05225086, + "balance_loss_mlp": 1.03358495, + "epoch": 0.25010521252930923, + "flos": 23553989049600.0, + "grad_norm": 2.460984044543067, + "language_loss": 0.74752259, + "learning_rate": 3.5131210901659544e-06, + "loss": 0.76976579, + "num_input_tokens_seen": 44438095, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.21386719, + "step": 2080, + "time_per_iteration": 2.7943406105041504 + }, + { + "auxiliary_loss_clip": 0.01168085, + "auxiliary_loss_mlp": 0.01050441, + "balance_loss_clip": 1.05089808, + "balance_loss_mlp": 1.02867329, + "epoch": 0.2502254554199483, + "flos": 23441193365760.0, + "grad_norm": 4.0199513779930705, + "language_loss": 0.81986856, + "learning_rate": 3.5126115874253967e-06, + "loss": 0.84205383, + "num_input_tokens_seen": 44457650, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.21789551, + "step": 2081, + "time_per_iteration": 2.809267044067383 + }, + { + "auxiliary_loss_clip": 0.01169045, + "auxiliary_loss_mlp": 0.01044769, + "balance_loss_clip": 1.05197549, + "balance_loss_mlp": 1.02360928, + "epoch": 0.2503456983105874, + "flos": 28761681651840.0, + "grad_norm": 24.39189636006544, + "language_loss": 0.80876869, + "learning_rate": 3.5121018552228644e-06, + "loss": 0.83090681, + "num_input_tokens_seen": 44476155, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.21179199, + "step": 2082, + "time_per_iteration": 2.8943285942077637 + }, + { + "auxiliary_loss_clip": 0.01166502, + "auxiliary_loss_mlp": 0.01044096, + "balance_loss_clip": 1.05005479, + "balance_loss_mlp": 1.02282894, + "epoch": 0.2504659412012265, + "flos": 18770256673920.0, + "grad_norm": 2.0836673857016477, + "language_loss": 0.76126581, + "learning_rate": 3.5115918936356827e-06, + "loss": 0.78337181, + "num_input_tokens_seen": 44492910, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.21252441, + "step": 2083, + "time_per_iteration": 2.837947130203247 + }, + { + "auxiliary_loss_clip": 0.01169245, + "auxiliary_loss_mlp": 0.01050096, + "balance_loss_clip": 1.05325902, + "balance_loss_mlp": 1.03036666, + "epoch": 0.25058618409186556, + "flos": 16873383346560.0, + "grad_norm": 2.075837681487411, + "language_loss": 0.78656423, + "learning_rate": 3.5110817027412123e-06, + "loss": 0.80875766, + "num_input_tokens_seen": 44512000, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.19714355, + "step": 2084, + "time_per_iteration": 2.8085832595825195 + }, + { + "auxiliary_loss_clip": 0.01163853, + "auxiliary_loss_mlp": 0.01044382, + "balance_loss_clip": 1.05034494, + "balance_loss_mlp": 1.02332973, + "epoch": 0.25070642698250467, + "flos": 24425540651520.0, + "grad_norm": 5.810482421505792, + "language_loss": 0.6898458, + "learning_rate": 3.5105712826168493e-06, + "loss": 0.71192819, + "num_input_tokens_seen": 44531650, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.21057129, + "step": 2085, + "time_per_iteration": 2.791367769241333 + }, + { + "auxiliary_loss_clip": 0.01164255, + "auxiliary_loss_mlp": 0.01048261, + "balance_loss_clip": 1.05051863, + "balance_loss_mlp": 1.02726865, + "epoch": 0.2508266698731437, + "flos": 20260944028800.0, + "grad_norm": 2.0795147111943586, + "language_loss": 0.71045208, + "learning_rate": 3.5100606333400235e-06, + "loss": 0.7325772, + "num_input_tokens_seen": 44548785, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.20996094, + "step": 2086, + "time_per_iteration": 2.8134665489196777 + }, + { + "auxiliary_loss_clip": 0.01175109, + "auxiliary_loss_mlp": 0.01055833, + "balance_loss_clip": 1.05207896, + "balance_loss_mlp": 1.03348112, + "epoch": 0.25094691276378284, + "flos": 19245318975360.0, + "grad_norm": 2.2870376745124186, + "language_loss": 0.76892161, + "learning_rate": 3.5095497549882006e-06, + "loss": 0.79123104, + "num_input_tokens_seen": 44567230, + "router_z_loss_clip": 1.22949219, + "router_z_loss_mlp": 0.22351074, + "step": 2087, + "time_per_iteration": 2.8130943775177 + }, + { + "auxiliary_loss_clip": 0.01170173, + "auxiliary_loss_mlp": 0.01052674, + "balance_loss_clip": 1.05422115, + "balance_loss_mlp": 1.03293312, + "epoch": 0.25106715565442195, + "flos": 26943237671040.0, + "grad_norm": 2.1995441999230767, + "language_loss": 0.72652465, + "learning_rate": 3.50903864763888e-06, + "loss": 0.74875313, + "num_input_tokens_seen": 44588020, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.19726562, + "step": 2088, + "time_per_iteration": 2.8177199363708496 + }, + { + "auxiliary_loss_clip": 0.01170564, + "auxiliary_loss_mlp": 0.01050036, + "balance_loss_clip": 1.05253482, + "balance_loss_mlp": 1.02891183, + "epoch": 0.251187398545061, + "flos": 48359570572800.0, + "grad_norm": 2.2198212551525227, + "language_loss": 0.76114404, + "learning_rate": 3.5085273113695965e-06, + "loss": 0.78335005, + "num_input_tokens_seen": 44612590, + "router_z_loss_clip": 1.18066406, + "router_z_loss_mlp": 0.21118164, + "step": 2089, + "time_per_iteration": 3.032055139541626 + }, + { + "auxiliary_loss_clip": 0.01169337, + "auxiliary_loss_mlp": 0.0104789, + "balance_loss_clip": 1.051162, + "balance_loss_mlp": 1.02650356, + "epoch": 0.2513076414357001, + "flos": 27016100409600.0, + "grad_norm": 3.7355582088030275, + "language_loss": 0.78637832, + "learning_rate": 3.508015746257919e-06, + "loss": 0.8085506, + "num_input_tokens_seen": 44631630, + "router_z_loss_clip": 1.18066406, + "router_z_loss_mlp": 0.21374512, + "step": 2090, + "time_per_iteration": 2.8465068340301514 + }, + { + "auxiliary_loss_clip": 0.01165298, + "auxiliary_loss_mlp": 0.01046614, + "balance_loss_clip": 1.05141091, + "balance_loss_mlp": 1.02695644, + "epoch": 0.2514278843263392, + "flos": 19463619882240.0, + "grad_norm": 10.60678260960377, + "language_loss": 0.8308599, + "learning_rate": 3.5075039523814518e-06, + "loss": 0.852979, + "num_input_tokens_seen": 44650820, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.19665527, + "step": 2091, + "time_per_iteration": 2.8387022018432617 + }, + { + "auxiliary_loss_clip": 0.01168118, + "auxiliary_loss_mlp": 0.01054398, + "balance_loss_clip": 1.04981732, + "balance_loss_mlp": 1.03443074, + "epoch": 0.2515481272169783, + "flos": 16866092885760.0, + "grad_norm": 2.3405488002178916, + "language_loss": 0.81572121, + "learning_rate": 3.506991929817834e-06, + "loss": 0.8379463, + "num_input_tokens_seen": 44667540, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.19970703, + "step": 2092, + "time_per_iteration": 2.8174006938934326 + }, + { + "auxiliary_loss_clip": 0.01162375, + "auxiliary_loss_mlp": 0.0105199, + "balance_loss_clip": 1.05023122, + "balance_loss_mlp": 1.03186774, + "epoch": 0.2516683701076174, + "flos": 23732464752000.0, + "grad_norm": 2.5921230204145504, + "language_loss": 0.82758689, + "learning_rate": 3.506479678644738e-06, + "loss": 0.84973049, + "num_input_tokens_seen": 44687935, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.20129395, + "step": 2093, + "time_per_iteration": 2.9336934089660645 + }, + { + "auxiliary_loss_clip": 0.0116654, + "auxiliary_loss_mlp": 0.0105329, + "balance_loss_clip": 1.0499177, + "balance_loss_mlp": 1.0316174, + "epoch": 0.2517886129982565, + "flos": 27635954434560.0, + "grad_norm": 2.7830602285683836, + "language_loss": 0.74439842, + "learning_rate": 3.505967198939873e-06, + "loss": 0.76659667, + "num_input_tokens_seen": 44704975, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.21679688, + "step": 2094, + "time_per_iteration": 2.8447179794311523 + }, + { + "auxiliary_loss_clip": 0.01165733, + "auxiliary_loss_mlp": 0.01038671, + "balance_loss_clip": 1.05037665, + "balance_loss_mlp": 1.01870322, + "epoch": 0.25190885588889556, + "flos": 38104596529920.0, + "grad_norm": 2.741255863487288, + "language_loss": 0.78306252, + "learning_rate": 3.5054544907809813e-06, + "loss": 0.80510658, + "num_input_tokens_seen": 44725475, + "router_z_loss_clip": 1.15429688, + "router_z_loss_mlp": 0.19958496, + "step": 2095, + "time_per_iteration": 2.9676690101623535 + }, + { + "auxiliary_loss_clip": 0.01168343, + "auxiliary_loss_mlp": 0.01047698, + "balance_loss_clip": 1.05074501, + "balance_loss_mlp": 1.02720618, + "epoch": 0.25202909877953467, + "flos": 22269894768000.0, + "grad_norm": 2.4748276832304836, + "language_loss": 0.80394244, + "learning_rate": 3.50494155424584e-06, + "loss": 0.82610285, + "num_input_tokens_seen": 44744380, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.20495605, + "step": 2096, + "time_per_iteration": 4.666969060897827 + }, + { + "auxiliary_loss_clip": 0.01166964, + "auxiliary_loss_mlp": 0.01048295, + "balance_loss_clip": 1.05106056, + "balance_loss_mlp": 1.02715957, + "epoch": 0.2521493416701738, + "flos": 21761759018880.0, + "grad_norm": 1.834182302199211, + "language_loss": 0.83080328, + "learning_rate": 3.504428389412262e-06, + "loss": 0.85295588, + "num_input_tokens_seen": 44765190, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.21118164, + "step": 2097, + "time_per_iteration": 2.9239304065704346 + }, + { + "auxiliary_loss_clip": 0.0116519, + "auxiliary_loss_mlp": 0.01056119, + "balance_loss_clip": 1.0496974, + "balance_loss_mlp": 1.03572237, + "epoch": 0.25226958456081283, + "flos": 27746738956800.0, + "grad_norm": 3.9030188279149995, + "language_loss": 0.72881246, + "learning_rate": 3.5039149963580927e-06, + "loss": 0.7510255, + "num_input_tokens_seen": 44785210, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.20397949, + "step": 2098, + "time_per_iteration": 3.6357991695404053 + }, + { + "auxiliary_loss_clip": 0.01165594, + "auxiliary_loss_mlp": 0.01041578, + "balance_loss_clip": 1.05075848, + "balance_loss_mlp": 1.02184844, + "epoch": 0.25238982745145194, + "flos": 30732171903360.0, + "grad_norm": 3.3907761972460935, + "language_loss": 0.70444465, + "learning_rate": 3.503401375161215e-06, + "loss": 0.72651643, + "num_input_tokens_seen": 44804955, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.19726562, + "step": 2099, + "time_per_iteration": 2.853342294692993 + }, + { + "auxiliary_loss_clip": 0.01163256, + "auxiliary_loss_mlp": 0.01042833, + "balance_loss_clip": 1.04985952, + "balance_loss_mlp": 1.02354538, + "epoch": 0.252510070342091, + "flos": 20266331068800.0, + "grad_norm": 1.6244738606329643, + "language_loss": 0.83841085, + "learning_rate": 3.502887525899544e-06, + "loss": 0.86047173, + "num_input_tokens_seen": 44823935, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.19274902, + "step": 2100, + "time_per_iteration": 3.6494388580322266 + }, + { + "auxiliary_loss_clip": 0.01167322, + "auxiliary_loss_mlp": 0.01053663, + "balance_loss_clip": 1.05176997, + "balance_loss_mlp": 1.03320634, + "epoch": 0.2526303132327301, + "flos": 22747399194240.0, + "grad_norm": 5.224495911102198, + "language_loss": 0.82893169, + "learning_rate": 3.50237344865103e-06, + "loss": 0.85114157, + "num_input_tokens_seen": 44844935, + "router_z_loss_clip": 1.15429688, + "router_z_loss_mlp": 0.20458984, + "step": 2101, + "time_per_iteration": 2.805668354034424 + }, + { + "auxiliary_loss_clip": 0.01170887, + "auxiliary_loss_mlp": 0.0104952, + "balance_loss_clip": 1.05082679, + "balance_loss_mlp": 1.02926636, + "epoch": 0.2527505561233692, + "flos": 30263466309120.0, + "grad_norm": 7.700923399755513, + "language_loss": 0.76263404, + "learning_rate": 3.501859143493658e-06, + "loss": 0.7848382, + "num_input_tokens_seen": 44865565, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.20239258, + "step": 2102, + "time_per_iteration": 2.9311025142669678 + }, + { + "auxiliary_loss_clip": 0.01113085, + "auxiliary_loss_mlp": 0.01011783, + "balance_loss_clip": 1.06232691, + "balance_loss_mlp": 1.00682354, + "epoch": 0.2528707990140083, + "flos": 58492917164160.0, + "grad_norm": 0.9245011507361834, + "language_loss": 0.60461175, + "learning_rate": 3.5013446105054488e-06, + "loss": 0.62586045, + "num_input_tokens_seen": 44918485, + "router_z_loss_clip": 0.5078125, + "router_z_loss_mlp": 0.04956055, + "step": 2103, + "time_per_iteration": 3.134974956512451 + }, + { + "auxiliary_loss_clip": 0.01162896, + "auxiliary_loss_mlp": 0.01043475, + "balance_loss_clip": 1.04906511, + "balance_loss_mlp": 1.02381766, + "epoch": 0.2529910419046474, + "flos": 24645134448000.0, + "grad_norm": 2.0865516447449486, + "language_loss": 0.75070637, + "learning_rate": 3.5008298497644555e-06, + "loss": 0.77277011, + "num_input_tokens_seen": 44937530, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.1965332, + "step": 2104, + "time_per_iteration": 2.812361001968384 + }, + { + "auxiliary_loss_clip": 0.01165425, + "auxiliary_loss_mlp": 0.01039269, + "balance_loss_clip": 1.04989207, + "balance_loss_mlp": 1.02056503, + "epoch": 0.2531112847952865, + "flos": 23842135952640.0, + "grad_norm": 2.242508796063886, + "language_loss": 0.87934089, + "learning_rate": 3.500314861348767e-06, + "loss": 0.90138781, + "num_input_tokens_seen": 44958165, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.18701172, + "step": 2105, + "time_per_iteration": 2.7967798709869385 + }, + { + "auxiliary_loss_clip": 0.01167714, + "auxiliary_loss_mlp": 0.01050378, + "balance_loss_clip": 1.05167627, + "balance_loss_mlp": 1.03143597, + "epoch": 0.25323152768592555, + "flos": 16143822207360.0, + "grad_norm": 2.8956136816182148, + "language_loss": 0.77824068, + "learning_rate": 3.499799645336507e-06, + "loss": 0.8004216, + "num_input_tokens_seen": 44975060, + "router_z_loss_clip": 1.16015625, + "router_z_loss_mlp": 0.18945312, + "step": 2106, + "time_per_iteration": 2.78997802734375 + }, + { + "auxiliary_loss_clip": 0.01164043, + "auxiliary_loss_mlp": 0.01039663, + "balance_loss_clip": 1.0518024, + "balance_loss_mlp": 1.02145982, + "epoch": 0.25335177057656466, + "flos": 28405161210240.0, + "grad_norm": 10.602589394385513, + "language_loss": 0.87106383, + "learning_rate": 3.4992842018058336e-06, + "loss": 0.89310086, + "num_input_tokens_seen": 44997960, + "router_z_loss_clip": 1.12304688, + "router_z_loss_mlp": 0.18212891, + "step": 2107, + "time_per_iteration": 2.9083404541015625 + }, + { + "auxiliary_loss_clip": 0.01167064, + "auxiliary_loss_mlp": 0.0105138, + "balance_loss_clip": 1.04958248, + "balance_loss_mlp": 1.0316509, + "epoch": 0.25347201346720377, + "flos": 18799666934400.0, + "grad_norm": 2.45687140598534, + "language_loss": 0.88787544, + "learning_rate": 3.4987685308349384e-06, + "loss": 0.91005987, + "num_input_tokens_seen": 45015690, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.1973877, + "step": 2108, + "time_per_iteration": 2.8141672611236572 + }, + { + "auxiliary_loss_clip": 0.01163531, + "auxiliary_loss_mlp": 0.01046598, + "balance_loss_clip": 1.04778039, + "balance_loss_mlp": 1.02689219, + "epoch": 0.2535922563578428, + "flos": 15815490963840.0, + "grad_norm": 3.4282610307301136, + "language_loss": 0.61422008, + "learning_rate": 3.4982526325020497e-06, + "loss": 0.63632137, + "num_input_tokens_seen": 45032660, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.19702148, + "step": 2109, + "time_per_iteration": 2.866650104522705 + }, + { + "auxiliary_loss_clip": 0.01169874, + "auxiliary_loss_mlp": 0.01044147, + "balance_loss_clip": 1.05199814, + "balance_loss_mlp": 1.0247395, + "epoch": 0.25371249924848194, + "flos": 16318922031360.0, + "grad_norm": 2.7975141313915457, + "language_loss": 0.82840669, + "learning_rate": 3.4977365068854273e-06, + "loss": 0.8505469, + "num_input_tokens_seen": 45048280, + "router_z_loss_clip": 1.1796875, + "router_z_loss_mlp": 0.19396973, + "step": 2110, + "time_per_iteration": 2.7706050872802734 + }, + { + "auxiliary_loss_clip": 0.01168492, + "auxiliary_loss_mlp": 0.01041942, + "balance_loss_clip": 1.05298042, + "balance_loss_mlp": 1.02289224, + "epoch": 0.25383274213912105, + "flos": 21761615364480.0, + "grad_norm": 3.796299266121932, + "language_loss": 0.73342252, + "learning_rate": 3.4972201540633676e-06, + "loss": 0.75552684, + "num_input_tokens_seen": 45067635, + "router_z_loss_clip": 1.15429688, + "router_z_loss_mlp": 0.19042969, + "step": 2111, + "time_per_iteration": 2.831325054168701 + }, + { + "auxiliary_loss_clip": 0.01168061, + "auxiliary_loss_mlp": 0.0104483, + "balance_loss_clip": 1.05247569, + "balance_loss_mlp": 1.025172, + "epoch": 0.2539529850297601, + "flos": 21396870708480.0, + "grad_norm": 1.9205012919415292, + "language_loss": 0.85661578, + "learning_rate": 3.4967035741142008e-06, + "loss": 0.8787446, + "num_input_tokens_seen": 45086455, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.19665527, + "step": 2112, + "time_per_iteration": 2.8490262031555176 + }, + { + "auxiliary_loss_clip": 0.01161986, + "auxiliary_loss_mlp": 0.01043782, + "balance_loss_clip": 1.0537858, + "balance_loss_mlp": 1.02576971, + "epoch": 0.2540732279203992, + "flos": 25228467319680.0, + "grad_norm": 4.600927165216276, + "language_loss": 0.82038963, + "learning_rate": 3.4961867671162917e-06, + "loss": 0.84244728, + "num_input_tokens_seen": 45106385, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.18005371, + "step": 2113, + "time_per_iteration": 2.8976194858551025 + }, + { + "auxiliary_loss_clip": 0.01165945, + "auxiliary_loss_mlp": 0.01047893, + "balance_loss_clip": 1.04879439, + "balance_loss_mlp": 1.02799749, + "epoch": 0.2541934708110383, + "flos": 19427386037760.0, + "grad_norm": 6.454970140774916, + "language_loss": 0.77284259, + "learning_rate": 3.4956697331480402e-06, + "loss": 0.794981, + "num_input_tokens_seen": 45124955, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.19897461, + "step": 2114, + "time_per_iteration": 2.8134286403656006 + }, + { + "auxiliary_loss_clip": 0.01169147, + "auxiliary_loss_mlp": 0.01047942, + "balance_loss_clip": 1.05239129, + "balance_loss_mlp": 1.0278548, + "epoch": 0.2543137137016774, + "flos": 23949436855680.0, + "grad_norm": 1.9557184000763381, + "language_loss": 0.80241323, + "learning_rate": 3.495152472287879e-06, + "loss": 0.82458413, + "num_input_tokens_seen": 45145665, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.20080566, + "step": 2115, + "time_per_iteration": 2.8726141452789307 + }, + { + "auxiliary_loss_clip": 0.0116898, + "auxiliary_loss_mlp": 0.01048636, + "balance_loss_clip": 1.05246222, + "balance_loss_mlp": 1.02792895, + "epoch": 0.2544339565923165, + "flos": 25593283802880.0, + "grad_norm": 2.1464151033345633, + "language_loss": 0.73876363, + "learning_rate": 3.4946349846142766e-06, + "loss": 0.76093984, + "num_input_tokens_seen": 45164805, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.20715332, + "step": 2116, + "time_per_iteration": 2.861666440963745 + }, + { + "auxiliary_loss_clip": 0.01168134, + "auxiliary_loss_mlp": 0.01041494, + "balance_loss_clip": 1.05133605, + "balance_loss_mlp": 1.02236116, + "epoch": 0.25455419948295555, + "flos": 21689470897920.0, + "grad_norm": 2.5030022352585783, + "language_loss": 0.7573806, + "learning_rate": 3.4941172702057353e-06, + "loss": 0.77947688, + "num_input_tokens_seen": 45184865, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.19116211, + "step": 2117, + "time_per_iteration": 2.9166314601898193 + }, + { + "auxiliary_loss_clip": 0.01168713, + "auxiliary_loss_mlp": 0.01047444, + "balance_loss_clip": 1.05472326, + "balance_loss_mlp": 1.0281918, + "epoch": 0.25467444237359466, + "flos": 26250341339520.0, + "grad_norm": 2.2858613741223754, + "language_loss": 0.80257523, + "learning_rate": 3.4935993291407924e-06, + "loss": 0.82473677, + "num_input_tokens_seen": 45203690, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.19250488, + "step": 2118, + "time_per_iteration": 2.860205888748169 + }, + { + "auxiliary_loss_clip": 0.01168953, + "auxiliary_loss_mlp": 0.01051673, + "balance_loss_clip": 1.05164385, + "balance_loss_mlp": 1.03076339, + "epoch": 0.25479468526423377, + "flos": 26979686997120.0, + "grad_norm": 2.6606916774509632, + "language_loss": 0.71513408, + "learning_rate": 3.4930811614980183e-06, + "loss": 0.73734033, + "num_input_tokens_seen": 45225385, + "router_z_loss_clip": 1.17382812, + "router_z_loss_mlp": 0.20898438, + "step": 2119, + "time_per_iteration": 2.8815231323242188 + }, + { + "auxiliary_loss_clip": 0.01167305, + "auxiliary_loss_mlp": 0.01048352, + "balance_loss_clip": 1.05034637, + "balance_loss_mlp": 1.02949333, + "epoch": 0.2549149281548728, + "flos": 23475811098240.0, + "grad_norm": 2.7510823564593894, + "language_loss": 0.79246974, + "learning_rate": 3.4925627673560198e-06, + "loss": 0.81462628, + "num_input_tokens_seen": 45246045, + "router_z_loss_clip": 1.16992188, + "router_z_loss_mlp": 0.18859863, + "step": 2120, + "time_per_iteration": 2.812680959701538 + }, + { + "auxiliary_loss_clip": 0.01165612, + "auxiliary_loss_mlp": 0.01046202, + "balance_loss_clip": 1.0494982, + "balance_loss_mlp": 1.02659237, + "epoch": 0.25503517104551193, + "flos": 25812302981760.0, + "grad_norm": 1.9639546825118213, + "language_loss": 0.88316882, + "learning_rate": 3.4920441467934357e-06, + "loss": 0.90528691, + "num_input_tokens_seen": 45266560, + "router_z_loss_clip": 1.16015625, + "router_z_loss_mlp": 0.19592285, + "step": 2121, + "time_per_iteration": 2.818363904953003 + }, + { + "auxiliary_loss_clip": 0.01164149, + "auxiliary_loss_mlp": 0.01039649, + "balance_loss_clip": 1.05055392, + "balance_loss_mlp": 1.02123129, + "epoch": 0.25515541393615104, + "flos": 26645106787200.0, + "grad_norm": 2.654588123796889, + "language_loss": 0.8261044, + "learning_rate": 3.491525299888941e-06, + "loss": 0.84814239, + "num_input_tokens_seen": 45285405, + "router_z_loss_clip": 1.13574219, + "router_z_loss_mlp": 0.18432617, + "step": 2122, + "time_per_iteration": 2.8015613555908203 + }, + { + "auxiliary_loss_clip": 0.0112914, + "auxiliary_loss_mlp": 0.01021741, + "balance_loss_clip": 1.06694186, + "balance_loss_mlp": 1.01601911, + "epoch": 0.2552756568267901, + "flos": 65955945847680.0, + "grad_norm": 0.890265310854058, + "language_loss": 0.62591696, + "learning_rate": 3.491006226721244e-06, + "loss": 0.64742577, + "num_input_tokens_seen": 45349615, + "router_z_loss_clip": 0.625, + "router_z_loss_mlp": 0.05712891, + "step": 2123, + "time_per_iteration": 3.4146158695220947 + }, + { + "auxiliary_loss_clip": 0.01169601, + "auxiliary_loss_mlp": 0.01045918, + "balance_loss_clip": 1.05563974, + "balance_loss_mlp": 1.02677345, + "epoch": 0.2553958997174292, + "flos": 17931096161280.0, + "grad_norm": 3.669825638548016, + "language_loss": 0.77857161, + "learning_rate": 3.4904869273690882e-06, + "loss": 0.80072683, + "num_input_tokens_seen": 45367505, + "router_z_loss_clip": 1.13916016, + "router_z_loss_mlp": 0.19128418, + "step": 2124, + "time_per_iteration": 2.9077086448669434 + }, + { + "auxiliary_loss_clip": 0.01167624, + "auxiliary_loss_mlp": 0.010437, + "balance_loss_clip": 1.05197203, + "balance_loss_mlp": 1.02451956, + "epoch": 0.2555161426080683, + "flos": 23367791923200.0, + "grad_norm": 3.2318947621058913, + "language_loss": 0.8922528, + "learning_rate": 3.489967401911251e-06, + "loss": 0.91436607, + "num_input_tokens_seen": 45386805, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.19165039, + "step": 2125, + "time_per_iteration": 3.6410441398620605 + }, + { + "auxiliary_loss_clip": 0.01173656, + "auxiliary_loss_mlp": 0.01041433, + "balance_loss_clip": 1.05269873, + "balance_loss_mlp": 1.0196656, + "epoch": 0.2556363854987074, + "flos": 40625130723840.0, + "grad_norm": 1.8602632222793574, + "language_loss": 0.69487762, + "learning_rate": 3.4894476504265428e-06, + "loss": 0.7170285, + "num_input_tokens_seen": 45411045, + "router_z_loss_clip": 1.20898438, + "router_z_loss_mlp": 0.21765137, + "step": 2126, + "time_per_iteration": 2.9121391773223877 + }, + { + "auxiliary_loss_clip": 0.01111917, + "auxiliary_loss_mlp": 0.01010593, + "balance_loss_clip": 1.05546236, + "balance_loss_mlp": 1.00632501, + "epoch": 0.2557566283893465, + "flos": 68019443389440.0, + "grad_norm": 0.7845291209121382, + "language_loss": 0.54437882, + "learning_rate": 3.4889276729938104e-06, + "loss": 0.56560391, + "num_input_tokens_seen": 45469575, + "router_z_loss_clip": 0.56640625, + "router_z_loss_mlp": 0.04272461, + "step": 2127, + "time_per_iteration": 3.9919941425323486 + }, + { + "auxiliary_loss_clip": 0.01169776, + "auxiliary_loss_mlp": 0.010473, + "balance_loss_clip": 1.05256152, + "balance_loss_mlp": 1.02761841, + "epoch": 0.2558768712799856, + "flos": 22635645004800.0, + "grad_norm": 2.0244227731341176, + "language_loss": 0.80635858, + "learning_rate": 3.488407469691934e-06, + "loss": 0.82852936, + "num_input_tokens_seen": 45490270, + "router_z_loss_clip": 1.17285156, + "router_z_loss_mlp": 0.19689941, + "step": 2128, + "time_per_iteration": 3.671844482421875 + }, + { + "auxiliary_loss_clip": 0.01170789, + "auxiliary_loss_mlp": 0.01044239, + "balance_loss_clip": 1.05305195, + "balance_loss_mlp": 1.02409244, + "epoch": 0.25599711417062465, + "flos": 26396354125440.0, + "grad_norm": 5.869225615835312, + "language_loss": 0.80594379, + "learning_rate": 3.487887040599828e-06, + "loss": 0.82809407, + "num_input_tokens_seen": 45510070, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.20153809, + "step": 2129, + "time_per_iteration": 2.8841323852539062 + }, + { + "auxiliary_loss_clip": 0.01171989, + "auxiliary_loss_mlp": 0.01068828, + "balance_loss_clip": 1.05201316, + "balance_loss_mlp": 1.04896772, + "epoch": 0.25611735706126376, + "flos": 22852042490880.0, + "grad_norm": 5.3412667531876, + "language_loss": 0.76109052, + "learning_rate": 3.4873663857964407e-06, + "loss": 0.78349864, + "num_input_tokens_seen": 45527285, + "router_z_loss_clip": 1.19824219, + "router_z_loss_mlp": 0.1986084, + "step": 2130, + "time_per_iteration": 2.9235002994537354 + }, + { + "auxiliary_loss_clip": 0.01169654, + "auxiliary_loss_mlp": 0.010428, + "balance_loss_clip": 1.05274093, + "balance_loss_mlp": 1.02366734, + "epoch": 0.2562375999519028, + "flos": 23367863750400.0, + "grad_norm": 2.229533996449346, + "language_loss": 0.66663575, + "learning_rate": 3.4868455053607556e-06, + "loss": 0.68876028, + "num_input_tokens_seen": 45546900, + "router_z_loss_clip": 1.16894531, + "router_z_loss_mlp": 0.19140625, + "step": 2131, + "time_per_iteration": 2.911198616027832 + }, + { + "auxiliary_loss_clip": 0.01173287, + "auxiliary_loss_mlp": 0.01059664, + "balance_loss_clip": 1.05310631, + "balance_loss_mlp": 1.03561974, + "epoch": 0.2563578428425419, + "flos": 22856962654080.0, + "grad_norm": 3.3477889739672784, + "language_loss": 0.71690458, + "learning_rate": 3.486324399371789e-06, + "loss": 0.73923409, + "num_input_tokens_seen": 45566200, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.24035645, + "step": 2132, + "time_per_iteration": 2.866629123687744 + }, + { + "auxiliary_loss_clip": 0.01167337, + "auxiliary_loss_mlp": 0.01047375, + "balance_loss_clip": 1.05505109, + "balance_loss_mlp": 1.02793205, + "epoch": 0.25647808573318104, + "flos": 21653883498240.0, + "grad_norm": 2.7603005012996205, + "language_loss": 0.78304255, + "learning_rate": 3.485803067908593e-06, + "loss": 0.80518961, + "num_input_tokens_seen": 45585710, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.19445801, + "step": 2133, + "time_per_iteration": 2.9226789474487305 + }, + { + "auxiliary_loss_clip": 0.01167085, + "auxiliary_loss_mlp": 0.01057251, + "balance_loss_clip": 1.05068541, + "balance_loss_mlp": 1.0375334, + "epoch": 0.2565983286238201, + "flos": 33730569659520.0, + "grad_norm": 2.352750075582313, + "language_loss": 0.79855615, + "learning_rate": 3.485281511050253e-06, + "loss": 0.82079947, + "num_input_tokens_seen": 45607845, + "router_z_loss_clip": 1.1640625, + "router_z_loss_mlp": 0.19714355, + "step": 2134, + "time_per_iteration": 2.969654083251953 + }, + { + "auxiliary_loss_clip": 0.01166365, + "auxiliary_loss_mlp": 0.01046452, + "balance_loss_clip": 1.05165851, + "balance_loss_mlp": 1.02657974, + "epoch": 0.2567185715144592, + "flos": 16216002587520.0, + "grad_norm": 4.218983987489628, + "language_loss": 0.90075183, + "learning_rate": 3.484759728875889e-06, + "loss": 0.92287999, + "num_input_tokens_seen": 45623210, + "router_z_loss_clip": 1.14746094, + "router_z_loss_mlp": 0.19873047, + "step": 2135, + "time_per_iteration": 2.804131269454956 + }, + { + "auxiliary_loss_clip": 0.0116197, + "auxiliary_loss_mlp": 0.0104388, + "balance_loss_clip": 1.04929888, + "balance_loss_mlp": 1.0255816, + "epoch": 0.2568388144050983, + "flos": 17458475984640.0, + "grad_norm": 1.927376764604161, + "language_loss": 0.80738413, + "learning_rate": 3.4842377214646543e-06, + "loss": 0.82944268, + "num_input_tokens_seen": 45641505, + "router_z_loss_clip": 1.12597656, + "router_z_loss_mlp": 0.18322754, + "step": 2136, + "time_per_iteration": 2.8619046211242676 + }, + { + "auxiliary_loss_clip": 0.01165866, + "auxiliary_loss_mlp": 0.01043815, + "balance_loss_clip": 1.05041897, + "balance_loss_mlp": 1.02320409, + "epoch": 0.25695905729573737, + "flos": 20887442069760.0, + "grad_norm": 4.482716855281818, + "language_loss": 0.6705963, + "learning_rate": 3.483715488895737e-06, + "loss": 0.69269311, + "num_input_tokens_seen": 45661835, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.20617676, + "step": 2137, + "time_per_iteration": 2.843546152114868 + }, + { + "auxiliary_loss_clip": 0.01164773, + "auxiliary_loss_mlp": 0.01043909, + "balance_loss_clip": 1.04953432, + "balance_loss_mlp": 1.02408433, + "epoch": 0.2570793001863765, + "flos": 24717278914560.0, + "grad_norm": 3.048685757884719, + "language_loss": 0.78614897, + "learning_rate": 3.48319303124836e-06, + "loss": 0.80823576, + "num_input_tokens_seen": 45682215, + "router_z_loss_clip": 1.15234375, + "router_z_loss_mlp": 0.19824219, + "step": 2138, + "time_per_iteration": 2.8593335151672363 + }, + { + "auxiliary_loss_clip": 0.01169191, + "auxiliary_loss_mlp": 0.01046208, + "balance_loss_clip": 1.05285048, + "balance_loss_mlp": 1.02540576, + "epoch": 0.2571995430770156, + "flos": 26906896085760.0, + "grad_norm": 3.550198005411262, + "language_loss": 0.67575407, + "learning_rate": 3.4826703486017798e-06, + "loss": 0.69790804, + "num_input_tokens_seen": 45701840, + "router_z_loss_clip": 1.16357422, + "router_z_loss_mlp": 0.20800781, + "step": 2139, + "time_per_iteration": 2.7880241870880127 + }, + { + "auxiliary_loss_clip": 0.01165167, + "auxiliary_loss_mlp": 0.01052152, + "balance_loss_clip": 1.05158162, + "balance_loss_mlp": 1.03163576, + "epoch": 0.25731978596765465, + "flos": 19792561656960.0, + "grad_norm": 2.071773688351407, + "language_loss": 0.76998931, + "learning_rate": 3.4821474410352867e-06, + "loss": 0.79216254, + "num_input_tokens_seen": 45720500, + "router_z_loss_clip": 1.13476562, + "router_z_loss_mlp": 0.20495605, + "step": 2140, + "time_per_iteration": 2.916618585586548 + }, + { + "auxiliary_loss_clip": 0.01095086, + "auxiliary_loss_mlp": 0.01004437, + "balance_loss_clip": 1.04521859, + "balance_loss_mlp": 1.00154018, + "epoch": 0.25744002885829376, + "flos": 70564970471040.0, + "grad_norm": 0.8986515677931547, + "language_loss": 0.6269992, + "learning_rate": 3.481624308628205e-06, + "loss": 0.64799446, + "num_input_tokens_seen": 45781870, + "router_z_loss_clip": 0.49804688, + "router_z_loss_mlp": 0.02893066, + "step": 2141, + "time_per_iteration": 3.4893171787261963 + }, + { + "auxiliary_loss_clip": 0.01169506, + "auxiliary_loss_mlp": 0.01057605, + "balance_loss_clip": 1.05348778, + "balance_loss_mlp": 1.03648078, + "epoch": 0.25756027174893287, + "flos": 18038181582720.0, + "grad_norm": 5.671120675747671, + "language_loss": 1.00268888, + "learning_rate": 3.481100951459893e-06, + "loss": 1.02495992, + "num_input_tokens_seen": 45794890, + "router_z_loss_clip": 1.16015625, + "router_z_loss_mlp": 0.21130371, + "step": 2142, + "time_per_iteration": 2.81850528717041 + }, + { + "auxiliary_loss_clip": 0.01163065, + "auxiliary_loss_mlp": 0.01041078, + "balance_loss_clip": 1.05087256, + "balance_loss_mlp": 1.02019262, + "epoch": 0.2576805146395719, + "flos": 22674069578880.0, + "grad_norm": 1.8243588859286664, + "language_loss": 0.78629565, + "learning_rate": 3.4805773696097453e-06, + "loss": 0.80833697, + "num_input_tokens_seen": 45815780, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.2088623, + "step": 2143, + "time_per_iteration": 2.889768600463867 + }, + { + "auxiliary_loss_clip": 0.01166499, + "auxiliary_loss_mlp": 0.01049778, + "balance_loss_clip": 1.0527432, + "balance_loss_mlp": 1.02920246, + "epoch": 0.25780075753021103, + "flos": 16472225278080.0, + "grad_norm": 2.958292953155253, + "language_loss": 0.88089687, + "learning_rate": 3.4800535631571874e-06, + "loss": 0.9030596, + "num_input_tokens_seen": 45831310, + "router_z_loss_clip": 1.13574219, + "router_z_loss_mlp": 0.20581055, + "step": 2144, + "time_per_iteration": 2.860522985458374 + }, + { + "auxiliary_loss_clip": 0.0116997, + "auxiliary_loss_mlp": 0.0105176, + "balance_loss_clip": 1.05279684, + "balance_loss_mlp": 1.0302906, + "epoch": 0.25792100042085014, + "flos": 22820297846400.0, + "grad_norm": 2.328756327375953, + "language_loss": 0.76800638, + "learning_rate": 3.4795295321816804e-06, + "loss": 0.79022372, + "num_input_tokens_seen": 45850135, + "router_z_loss_clip": 1.17285156, + "router_z_loss_mlp": 0.21447754, + "step": 2145, + "time_per_iteration": 2.8199570178985596 + }, + { + "auxiliary_loss_clip": 0.01164149, + "auxiliary_loss_mlp": 0.01050472, + "balance_loss_clip": 1.05003667, + "balance_loss_mlp": 1.03095698, + "epoch": 0.2580412433114892, + "flos": 18697286194560.0, + "grad_norm": 3.178573765512394, + "language_loss": 0.9109087, + "learning_rate": 3.47900527676272e-06, + "loss": 0.93305486, + "num_input_tokens_seen": 45868470, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.1953125, + "step": 2146, + "time_per_iteration": 2.819976568222046 + }, + { + "auxiliary_loss_clip": 0.01171502, + "auxiliary_loss_mlp": 0.01055147, + "balance_loss_clip": 1.0547986, + "balance_loss_mlp": 1.03496504, + "epoch": 0.2581614862021283, + "flos": 14283146810880.0, + "grad_norm": 10.154124840581002, + "language_loss": 0.88513207, + "learning_rate": 3.478480796979835e-06, + "loss": 0.90739852, + "num_input_tokens_seen": 45886355, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.20178223, + "step": 2147, + "time_per_iteration": 2.7916412353515625 + }, + { + "auxiliary_loss_clip": 0.01162001, + "auxiliary_loss_mlp": 0.01052141, + "balance_loss_clip": 1.04946804, + "balance_loss_mlp": 1.03115988, + "epoch": 0.25828172909276736, + "flos": 29498281856640.0, + "grad_norm": 1.6389492895663946, + "language_loss": 0.78089625, + "learning_rate": 3.4779560929125894e-06, + "loss": 0.80303764, + "num_input_tokens_seen": 45907900, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.20983887, + "step": 2148, + "time_per_iteration": 2.8841428756713867 + }, + { + "auxiliary_loss_clip": 0.01086627, + "auxiliary_loss_mlp": 0.01009955, + "balance_loss_clip": 1.03778434, + "balance_loss_mlp": 1.00728452, + "epoch": 0.2584019719834065, + "flos": 67114387376640.0, + "grad_norm": 0.669329858792011, + "language_loss": 0.56894445, + "learning_rate": 3.4774311646405783e-06, + "loss": 0.58991027, + "num_input_tokens_seen": 45977805, + "router_z_loss_clip": 0.48828125, + "router_z_loss_mlp": 0.0267334, + "step": 2149, + "time_per_iteration": 3.4582693576812744 + }, + { + "auxiliary_loss_clip": 0.01166285, + "auxiliary_loss_mlp": 0.01048486, + "balance_loss_clip": 1.05240703, + "balance_loss_mlp": 1.02843475, + "epoch": 0.2585222148740456, + "flos": 22893555634560.0, + "grad_norm": 2.2967095571568152, + "language_loss": 0.83172274, + "learning_rate": 3.476906012243435e-06, + "loss": 0.85387045, + "num_input_tokens_seen": 45996715, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.20056152, + "step": 2150, + "time_per_iteration": 2.8410449028015137 + }, + { + "auxiliary_loss_clip": 0.01167142, + "auxiliary_loss_mlp": 0.01050454, + "balance_loss_clip": 1.05314302, + "balance_loss_mlp": 1.02992618, + "epoch": 0.25864245776468464, + "flos": 28909202808960.0, + "grad_norm": 1.8658489170768187, + "language_loss": 0.81165689, + "learning_rate": 3.476380635800824e-06, + "loss": 0.83383286, + "num_input_tokens_seen": 46017915, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.2052002, + "step": 2151, + "time_per_iteration": 2.8255341053009033 + }, + { + "auxiliary_loss_clip": 0.01167156, + "auxiliary_loss_mlp": 0.01051644, + "balance_loss_clip": 1.05277538, + "balance_loss_mlp": 1.03068709, + "epoch": 0.25876270065532375, + "flos": 14793185980800.0, + "grad_norm": 2.3447081257759823, + "language_loss": 0.85922766, + "learning_rate": 3.475855035392444e-06, + "loss": 0.88141561, + "num_input_tokens_seen": 46033235, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.20935059, + "step": 2152, + "time_per_iteration": 2.8279240131378174 + }, + { + "auxiliary_loss_clip": 0.01163945, + "auxiliary_loss_mlp": 0.01048046, + "balance_loss_clip": 1.04983497, + "balance_loss_mlp": 1.02769721, + "epoch": 0.25888294354596286, + "flos": 60467821810560.0, + "grad_norm": 2.6928261535570557, + "language_loss": 0.71361458, + "learning_rate": 3.475329211098029e-06, + "loss": 0.73573458, + "num_input_tokens_seen": 46056390, + "router_z_loss_clip": 1.14160156, + "router_z_loss_mlp": 0.20336914, + "step": 2153, + "time_per_iteration": 3.1714296340942383 + }, + { + "auxiliary_loss_clip": 0.01165404, + "auxiliary_loss_mlp": 0.0104948, + "balance_loss_clip": 1.05198348, + "balance_loss_mlp": 1.02939343, + "epoch": 0.2590031864366019, + "flos": 27851166771840.0, + "grad_norm": 1.8787762923364637, + "language_loss": 0.82546395, + "learning_rate": 3.4748031629973453e-06, + "loss": 0.84761286, + "num_input_tokens_seen": 46077120, + "router_z_loss_clip": 1.13476562, + "router_z_loss_mlp": 0.20092773, + "step": 2154, + "time_per_iteration": 4.68913722038269 + }, + { + "auxiliary_loss_clip": 0.01078738, + "auxiliary_loss_mlp": 0.01004898, + "balance_loss_clip": 1.03108907, + "balance_loss_mlp": 1.00207281, + "epoch": 0.25912342932724103, + "flos": 62422444206720.0, + "grad_norm": 0.9193069202012524, + "language_loss": 0.56527793, + "learning_rate": 3.4742768911701944e-06, + "loss": 0.58611429, + "num_input_tokens_seen": 46139815, + "router_z_loss_clip": 0.4765625, + "router_z_loss_mlp": 0.02819824, + "step": 2155, + "time_per_iteration": 3.420815944671631 + }, + { + "auxiliary_loss_clip": 0.01170842, + "auxiliary_loss_mlp": 0.01061321, + "balance_loss_clip": 1.05073512, + "balance_loss_mlp": 1.0376935, + "epoch": 0.25924367221788014, + "flos": 12378839368320.0, + "grad_norm": 3.1877199214512992, + "language_loss": 0.70414126, + "learning_rate": 3.4737503956964113e-06, + "loss": 0.7264629, + "num_input_tokens_seen": 46152120, + "router_z_loss_clip": 1.20117188, + "router_z_loss_mlp": 0.23632812, + "step": 2156, + "time_per_iteration": 3.6225576400756836 + }, + { + "auxiliary_loss_clip": 0.01174488, + "auxiliary_loss_mlp": 0.01053583, + "balance_loss_clip": 1.05218327, + "balance_loss_mlp": 1.03077817, + "epoch": 0.2593639151085192, + "flos": 14575208296320.0, + "grad_norm": 2.5331487661062115, + "language_loss": 0.67697048, + "learning_rate": 3.473223676655865e-06, + "loss": 0.69925117, + "num_input_tokens_seen": 46170120, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.22790527, + "step": 2157, + "time_per_iteration": 3.6534910202026367 + }, + { + "auxiliary_loss_clip": 0.01166944, + "auxiliary_loss_mlp": 0.01047367, + "balance_loss_clip": 1.05149043, + "balance_loss_mlp": 1.02613628, + "epoch": 0.2594841579991583, + "flos": 15230937029760.0, + "grad_norm": 1.843500981405152, + "language_loss": 0.79734075, + "learning_rate": 3.472696734128459e-06, + "loss": 0.81948388, + "num_input_tokens_seen": 46187985, + "router_z_loss_clip": 1.15429688, + "router_z_loss_mlp": 0.21228027, + "step": 2158, + "time_per_iteration": 2.795354127883911 + }, + { + "auxiliary_loss_clip": 0.01166188, + "auxiliary_loss_mlp": 0.01052666, + "balance_loss_clip": 1.04928243, + "balance_loss_mlp": 1.03117275, + "epoch": 0.2596044008897974, + "flos": 23623583650560.0, + "grad_norm": 1.7145419245763873, + "language_loss": 0.7558459, + "learning_rate": 3.4721695681941286e-06, + "loss": 0.77803439, + "num_input_tokens_seen": 46207025, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.21484375, + "step": 2159, + "time_per_iteration": 2.8184921741485596 + }, + { + "auxiliary_loss_clip": 0.01162656, + "auxiliary_loss_mlp": 0.01046771, + "balance_loss_clip": 1.04894185, + "balance_loss_mlp": 1.02571869, + "epoch": 0.25972464378043647, + "flos": 13772281628160.0, + "grad_norm": 2.449207520761529, + "language_loss": 0.82510412, + "learning_rate": 3.471642178932845e-06, + "loss": 0.84719843, + "num_input_tokens_seen": 46225670, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.21057129, + "step": 2160, + "time_per_iteration": 2.7283413410186768 + }, + { + "auxiliary_loss_clip": 0.01164528, + "auxiliary_loss_mlp": 0.01047321, + "balance_loss_clip": 1.0476563, + "balance_loss_mlp": 1.02713895, + "epoch": 0.2598448866710756, + "flos": 19573578391680.0, + "grad_norm": 2.428795175464026, + "language_loss": 0.89331019, + "learning_rate": 3.471114566424613e-06, + "loss": 0.9154287, + "num_input_tokens_seen": 46244130, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.20178223, + "step": 2161, + "time_per_iteration": 2.7444732189178467 + }, + { + "auxiliary_loss_clip": 0.01160861, + "auxiliary_loss_mlp": 0.01049719, + "balance_loss_clip": 1.048123, + "balance_loss_mlp": 1.0292387, + "epoch": 0.25996512956171464, + "flos": 21653237053440.0, + "grad_norm": 2.374321166592448, + "language_loss": 0.7547847, + "learning_rate": 3.4705867307494715e-06, + "loss": 0.77689052, + "num_input_tokens_seen": 46263200, + "router_z_loss_clip": 1.12792969, + "router_z_loss_mlp": 0.20495605, + "step": 2162, + "time_per_iteration": 2.7870242595672607 + }, + { + "auxiliary_loss_clip": 0.01163653, + "auxiliary_loss_mlp": 0.01054692, + "balance_loss_clip": 1.04910541, + "balance_loss_mlp": 1.03389001, + "epoch": 0.26008537245235375, + "flos": 18223480869120.0, + "grad_norm": 2.96047717291831, + "language_loss": 0.8433094, + "learning_rate": 3.470058671987492e-06, + "loss": 0.86549294, + "num_input_tokens_seen": 46281465, + "router_z_loss_clip": 1.14453125, + "router_z_loss_mlp": 0.20800781, + "step": 2163, + "time_per_iteration": 2.722247838973999 + }, + { + "auxiliary_loss_clip": 0.01169136, + "auxiliary_loss_mlp": 0.01054171, + "balance_loss_clip": 1.05179214, + "balance_loss_mlp": 1.03239179, + "epoch": 0.26020561534299286, + "flos": 24645385843200.0, + "grad_norm": 2.0103615675894178, + "language_loss": 0.84328991, + "learning_rate": 3.4695303902187805e-06, + "loss": 0.86552298, + "num_input_tokens_seen": 46301020, + "router_z_loss_clip": 1.17285156, + "router_z_loss_mlp": 0.21789551, + "step": 2164, + "time_per_iteration": 2.8606607913970947 + }, + { + "auxiliary_loss_clip": 0.01166823, + "auxiliary_loss_mlp": 0.01045921, + "balance_loss_clip": 1.05099213, + "balance_loss_mlp": 1.0240221, + "epoch": 0.2603258582336319, + "flos": 25773662926080.0, + "grad_norm": 4.781348155604089, + "language_loss": 0.78515112, + "learning_rate": 3.469001885523478e-06, + "loss": 0.80727863, + "num_input_tokens_seen": 46321740, + "router_z_loss_clip": 1.15820312, + "router_z_loss_mlp": 0.21911621, + "step": 2165, + "time_per_iteration": 2.77176833152771 + }, + { + "auxiliary_loss_clip": 0.01160899, + "auxiliary_loss_mlp": 0.0104912, + "balance_loss_clip": 1.04731083, + "balance_loss_mlp": 1.0292238, + "epoch": 0.260446101124271, + "flos": 28766314506240.0, + "grad_norm": 1.9364071239430447, + "language_loss": 0.81133938, + "learning_rate": 3.4684731579817568e-06, + "loss": 0.83343959, + "num_input_tokens_seen": 46342730, + "router_z_loss_clip": 1.13574219, + "router_z_loss_mlp": 0.19909668, + "step": 2166, + "time_per_iteration": 2.9189651012420654 + }, + { + "auxiliary_loss_clip": 0.0116511, + "auxiliary_loss_mlp": 0.01058241, + "balance_loss_clip": 1.05160427, + "balance_loss_mlp": 1.03797591, + "epoch": 0.26056634401491013, + "flos": 25666757072640.0, + "grad_norm": 2.3361420249849454, + "language_loss": 0.76744276, + "learning_rate": 3.4679442076738247e-06, + "loss": 0.78967631, + "num_input_tokens_seen": 46362445, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.20263672, + "step": 2167, + "time_per_iteration": 2.9625654220581055 + }, + { + "auxiliary_loss_clip": 0.01167889, + "auxiliary_loss_mlp": 0.01049584, + "balance_loss_clip": 1.05035555, + "balance_loss_mlp": 1.02842402, + "epoch": 0.2606865869055492, + "flos": 27052765217280.0, + "grad_norm": 4.5740500253027285, + "language_loss": 0.83203268, + "learning_rate": 3.4674150346799245e-06, + "loss": 0.8542074, + "num_input_tokens_seen": 46382145, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.21179199, + "step": 2168, + "time_per_iteration": 2.834047794342041 + }, + { + "auxiliary_loss_clip": 0.01163357, + "auxiliary_loss_mlp": 0.0104762, + "balance_loss_clip": 1.04715538, + "balance_loss_mlp": 1.02673483, + "epoch": 0.2608068297961883, + "flos": 17712615686400.0, + "grad_norm": 2.414441526111037, + "language_loss": 0.79970658, + "learning_rate": 3.4668856390803295e-06, + "loss": 0.82181633, + "num_input_tokens_seen": 46400025, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.20910645, + "step": 2169, + "time_per_iteration": 2.7950804233551025 + }, + { + "auxiliary_loss_clip": 0.01157321, + "auxiliary_loss_mlp": 0.01048012, + "balance_loss_clip": 1.04610729, + "balance_loss_mlp": 1.02834272, + "epoch": 0.2609270726868274, + "flos": 18551632544640.0, + "grad_norm": 2.3886096086336206, + "language_loss": 0.89906168, + "learning_rate": 3.4663560209553495e-06, + "loss": 0.92111492, + "num_input_tokens_seen": 46418090, + "router_z_loss_clip": 1.11083984, + "router_z_loss_mlp": 0.19677734, + "step": 2170, + "time_per_iteration": 2.81624174118042 + }, + { + "auxiliary_loss_clip": 0.01161686, + "auxiliary_loss_mlp": 0.01043482, + "balance_loss_clip": 1.04795265, + "balance_loss_mlp": 1.02231085, + "epoch": 0.26104731557746647, + "flos": 21835699165440.0, + "grad_norm": 2.2094379795808963, + "language_loss": 0.79686922, + "learning_rate": 3.4658261803853267e-06, + "loss": 0.81892085, + "num_input_tokens_seen": 46436015, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.21166992, + "step": 2171, + "time_per_iteration": 2.8695521354675293 + }, + { + "auxiliary_loss_clip": 0.01165077, + "auxiliary_loss_mlp": 0.01048206, + "balance_loss_clip": 1.05174398, + "balance_loss_mlp": 1.02741575, + "epoch": 0.2611675584681056, + "flos": 21689650465920.0, + "grad_norm": 2.565156443368233, + "language_loss": 0.81149554, + "learning_rate": 3.4652961174506383e-06, + "loss": 0.8336283, + "num_input_tokens_seen": 46455885, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.20788574, + "step": 2172, + "time_per_iteration": 2.8169023990631104 + }, + { + "auxiliary_loss_clip": 0.01071233, + "auxiliary_loss_mlp": 0.01000174, + "balance_loss_clip": 1.02367163, + "balance_loss_mlp": 0.99775451, + "epoch": 0.2612878013587447, + "flos": 71862101389440.0, + "grad_norm": 0.9816343915057909, + "language_loss": 0.5813418, + "learning_rate": 3.464765832231694e-06, + "loss": 0.60205585, + "num_input_tokens_seen": 46510050, + "router_z_loss_clip": 0.4765625, + "router_z_loss_mlp": 0.02416992, + "step": 2173, + "time_per_iteration": 3.334486484527588 + }, + { + "auxiliary_loss_clip": 0.01165503, + "auxiliary_loss_mlp": 0.01043427, + "balance_loss_clip": 1.05072403, + "balance_loss_mlp": 1.0240196, + "epoch": 0.26140804424938374, + "flos": 20227511445120.0, + "grad_norm": 2.312489677887186, + "language_loss": 0.70780396, + "learning_rate": 3.4642353248089373e-06, + "loss": 0.72989333, + "num_input_tokens_seen": 46528810, + "router_z_loss_clip": 1.14746094, + "router_z_loss_mlp": 0.1940918, + "step": 2174, + "time_per_iteration": 2.84260630607605 + }, + { + "auxiliary_loss_clip": 0.01165145, + "auxiliary_loss_mlp": 0.01052529, + "balance_loss_clip": 1.04934263, + "balance_loss_mlp": 1.03091645, + "epoch": 0.26152828714002285, + "flos": 25557085872000.0, + "grad_norm": 1.873373738934301, + "language_loss": 0.80494773, + "learning_rate": 3.463704595262846e-06, + "loss": 0.82712442, + "num_input_tokens_seen": 46549690, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.21594238, + "step": 2175, + "time_per_iteration": 2.9001269340515137 + }, + { + "auxiliary_loss_clip": 0.01164028, + "auxiliary_loss_mlp": 0.01057442, + "balance_loss_clip": 1.04928839, + "balance_loss_mlp": 1.03841591, + "epoch": 0.26164853003066196, + "flos": 25446516831360.0, + "grad_norm": 2.2582026188767625, + "language_loss": 0.70561111, + "learning_rate": 3.463173643673931e-06, + "loss": 0.72782576, + "num_input_tokens_seen": 46572215, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.19018555, + "step": 2176, + "time_per_iteration": 2.8835785388946533 + }, + { + "auxiliary_loss_clip": 0.01067008, + "auxiliary_loss_mlp": 0.0100979, + "balance_loss_clip": 1.02140427, + "balance_loss_mlp": 1.00726306, + "epoch": 0.261768772921301, + "flos": 53944580568960.0, + "grad_norm": 0.900166436763274, + "language_loss": 0.63463658, + "learning_rate": 3.4626424701227387e-06, + "loss": 0.65540457, + "num_input_tokens_seen": 46627275, + "router_z_loss_clip": 0.45703125, + "router_z_loss_mlp": 0.02526855, + "step": 2177, + "time_per_iteration": 3.205667495727539 + }, + { + "auxiliary_loss_clip": 0.01066579, + "auxiliary_loss_mlp": 0.0100849, + "balance_loss_clip": 1.02079415, + "balance_loss_mlp": 1.00599861, + "epoch": 0.26188901581194013, + "flos": 70687606481280.0, + "grad_norm": 0.8239743053499987, + "language_loss": 0.55746102, + "learning_rate": 3.4621110746898452e-06, + "loss": 0.57821172, + "num_input_tokens_seen": 46695135, + "router_z_loss_clip": 0.45703125, + "router_z_loss_mlp": 0.02490234, + "step": 2178, + "time_per_iteration": 3.4204275608062744 + }, + { + "auxiliary_loss_clip": 0.01161459, + "auxiliary_loss_mlp": 0.01051157, + "balance_loss_clip": 1.04858804, + "balance_loss_mlp": 1.03087997, + "epoch": 0.2620092587025792, + "flos": 21069580959360.0, + "grad_norm": 2.435045981118973, + "language_loss": 0.74537373, + "learning_rate": 3.4615794574558654e-06, + "loss": 0.76749986, + "num_input_tokens_seen": 46714145, + "router_z_loss_clip": 1.12890625, + "router_z_loss_mlp": 0.20275879, + "step": 2179, + "time_per_iteration": 2.8069839477539062 + }, + { + "auxiliary_loss_clip": 0.01160897, + "auxiliary_loss_mlp": 0.01050656, + "balance_loss_clip": 1.04856741, + "balance_loss_mlp": 1.02905583, + "epoch": 0.2621295015932183, + "flos": 18369601395840.0, + "grad_norm": 2.610725709354384, + "language_loss": 0.84423041, + "learning_rate": 3.4610476185014436e-06, + "loss": 0.866346, + "num_input_tokens_seen": 46731405, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.21594238, + "step": 2180, + "time_per_iteration": 2.730729579925537 + }, + { + "auxiliary_loss_clip": 0.01165227, + "auxiliary_loss_mlp": 0.01058322, + "balance_loss_clip": 1.04739141, + "balance_loss_mlp": 1.03666127, + "epoch": 0.2622497444838574, + "flos": 23659997063040.0, + "grad_norm": 2.647859291807562, + "language_loss": 0.79020363, + "learning_rate": 3.4605155579072597e-06, + "loss": 0.81243914, + "num_input_tokens_seen": 46751260, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.2166748, + "step": 2181, + "time_per_iteration": 2.8235514163970947 + }, + { + "auxiliary_loss_clip": 0.01163543, + "auxiliary_loss_mlp": 0.01048498, + "balance_loss_clip": 1.04882276, + "balance_loss_mlp": 1.02872097, + "epoch": 0.26236998737449646, + "flos": 22123810154880.0, + "grad_norm": 3.7901558868673293, + "language_loss": 0.71184576, + "learning_rate": 3.459983275754027e-06, + "loss": 0.73396623, + "num_input_tokens_seen": 46770155, + "router_z_loss_clip": 1.14746094, + "router_z_loss_mlp": 0.19763184, + "step": 2182, + "time_per_iteration": 2.7622244358062744 + }, + { + "auxiliary_loss_clip": 0.01165084, + "auxiliary_loss_mlp": 0.01055569, + "balance_loss_clip": 1.04895365, + "balance_loss_mlp": 1.0345881, + "epoch": 0.26249023026513557, + "flos": 17895185539200.0, + "grad_norm": 9.50576643798678, + "language_loss": 0.80309135, + "learning_rate": 3.4594507721224918e-06, + "loss": 0.82529789, + "num_input_tokens_seen": 46788805, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.20996094, + "step": 2183, + "time_per_iteration": 3.569676637649536 + }, + { + "auxiliary_loss_clip": 0.01162181, + "auxiliary_loss_mlp": 0.01054162, + "balance_loss_clip": 1.05025971, + "balance_loss_mlp": 1.0346595, + "epoch": 0.2626104731557747, + "flos": 18332936588160.0, + "grad_norm": 1.956370417224276, + "language_loss": 0.82285053, + "learning_rate": 3.4589180470934353e-06, + "loss": 0.84501398, + "num_input_tokens_seen": 46808670, + "router_z_loss_clip": 1.11962891, + "router_z_loss_mlp": 0.19494629, + "step": 2184, + "time_per_iteration": 2.9098987579345703 + }, + { + "auxiliary_loss_clip": 0.01166598, + "auxiliary_loss_mlp": 0.0104717, + "balance_loss_clip": 1.04805553, + "balance_loss_mlp": 1.02523565, + "epoch": 0.26273071604641374, + "flos": 19317714837120.0, + "grad_norm": 2.0034651617958934, + "language_loss": 0.76413167, + "learning_rate": 3.4583851007476713e-06, + "loss": 0.78626931, + "num_input_tokens_seen": 46827140, + "router_z_loss_clip": 1.18554688, + "router_z_loss_mlp": 0.21960449, + "step": 2185, + "time_per_iteration": 3.7260942459106445 + }, + { + "auxiliary_loss_clip": 0.01166703, + "auxiliary_loss_mlp": 0.01056279, + "balance_loss_clip": 1.05077398, + "balance_loss_mlp": 1.03377283, + "epoch": 0.26285095893705285, + "flos": 18327477720960.0, + "grad_norm": 2.8382413090362673, + "language_loss": 0.68354112, + "learning_rate": 3.4578519331660464e-06, + "loss": 0.70577097, + "num_input_tokens_seen": 46844135, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.22509766, + "step": 2186, + "time_per_iteration": 3.627814531326294 + }, + { + "auxiliary_loss_clip": 0.01163124, + "auxiliary_loss_mlp": 0.0105292, + "balance_loss_clip": 1.04959226, + "balance_loss_mlp": 1.03400135, + "epoch": 0.26297120182769196, + "flos": 20193827466240.0, + "grad_norm": 3.2715721726583946, + "language_loss": 0.82265031, + "learning_rate": 3.4573185444294426e-06, + "loss": 0.84481072, + "num_input_tokens_seen": 46862500, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.18920898, + "step": 2187, + "time_per_iteration": 2.7708189487457275 + }, + { + "auxiliary_loss_clip": 0.01167137, + "auxiliary_loss_mlp": 0.01071072, + "balance_loss_clip": 1.05172908, + "balance_loss_mlp": 1.04939926, + "epoch": 0.263091444718331, + "flos": 22418421505920.0, + "grad_norm": 2.9908368631135174, + "language_loss": 0.78713465, + "learning_rate": 3.456784934618774e-06, + "loss": 0.80951673, + "num_input_tokens_seen": 46883665, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.21679688, + "step": 2188, + "time_per_iteration": 2.796652317047119 + }, + { + "auxiliary_loss_clip": 0.01159906, + "auxiliary_loss_mlp": 0.01050031, + "balance_loss_clip": 1.04626632, + "balance_loss_mlp": 1.03117251, + "epoch": 0.2632116876089701, + "flos": 19024827338880.0, + "grad_norm": 2.7097530820724094, + "language_loss": 0.80147326, + "learning_rate": 3.4562511038149897e-06, + "loss": 0.82357264, + "num_input_tokens_seen": 46899160, + "router_z_loss_clip": 1.13574219, + "router_z_loss_mlp": 0.18859863, + "step": 2189, + "time_per_iteration": 2.8183958530426025 + }, + { + "auxiliary_loss_clip": 0.01063122, + "auxiliary_loss_mlp": 0.01018317, + "balance_loss_clip": 1.01960635, + "balance_loss_mlp": 1.01586175, + "epoch": 0.26333193049960923, + "flos": 67308054531840.0, + "grad_norm": 0.8562909622865034, + "language_loss": 0.57679272, + "learning_rate": 3.4557170520990705e-06, + "loss": 0.59760714, + "num_input_tokens_seen": 46959835, + "router_z_loss_clip": 0.43554688, + "router_z_loss_mlp": 0.02453613, + "step": 2190, + "time_per_iteration": 3.3898255825042725 + }, + { + "auxiliary_loss_clip": 0.01162125, + "auxiliary_loss_mlp": 0.01055589, + "balance_loss_clip": 1.04921567, + "balance_loss_mlp": 1.03620601, + "epoch": 0.2634521733902483, + "flos": 25048806468480.0, + "grad_norm": 1.6825694786293175, + "language_loss": 0.86554849, + "learning_rate": 3.4551827795520324e-06, + "loss": 0.88772559, + "num_input_tokens_seen": 46982720, + "router_z_loss_clip": 1.12792969, + "router_z_loss_mlp": 0.19384766, + "step": 2191, + "time_per_iteration": 2.8601067066192627 + }, + { + "auxiliary_loss_clip": 0.01158899, + "auxiliary_loss_mlp": 0.01043461, + "balance_loss_clip": 1.04860711, + "balance_loss_mlp": 1.02461386, + "epoch": 0.2635724162808874, + "flos": 20594985534720.0, + "grad_norm": 2.763111389970877, + "language_loss": 0.84976292, + "learning_rate": 3.4546482862549226e-06, + "loss": 0.87178653, + "num_input_tokens_seen": 47003035, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.18847656, + "step": 2192, + "time_per_iteration": 2.8183438777923584 + }, + { + "auxiliary_loss_clip": 0.01168217, + "auxiliary_loss_mlp": 0.01050136, + "balance_loss_clip": 1.05293107, + "balance_loss_mlp": 1.03034747, + "epoch": 0.2636926591715265, + "flos": 19244636616960.0, + "grad_norm": 2.5312617919775646, + "language_loss": 0.78673065, + "learning_rate": 3.4541135722888253e-06, + "loss": 0.80891418, + "num_input_tokens_seen": 47019625, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.19787598, + "step": 2193, + "time_per_iteration": 2.7765135765075684 + }, + { + "auxiliary_loss_clip": 0.01162077, + "auxiliary_loss_mlp": 0.01038604, + "balance_loss_clip": 1.04722643, + "balance_loss_mlp": 1.01805282, + "epoch": 0.26381290206216557, + "flos": 28804882734720.0, + "grad_norm": 1.9498339866974688, + "language_loss": 0.8027277, + "learning_rate": 3.453578637734854e-06, + "loss": 0.82473457, + "num_input_tokens_seen": 47040815, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.20544434, + "step": 2194, + "time_per_iteration": 2.8481228351593018 + }, + { + "auxiliary_loss_clip": 0.01166961, + "auxiliary_loss_mlp": 0.01054427, + "balance_loss_clip": 1.05282712, + "balance_loss_mlp": 1.0341022, + "epoch": 0.2639331449528047, + "flos": 25008909436800.0, + "grad_norm": 4.4592679682436165, + "language_loss": 0.78211582, + "learning_rate": 3.4530434826741605e-06, + "loss": 0.80432969, + "num_input_tokens_seen": 47061755, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.20336914, + "step": 2195, + "time_per_iteration": 2.8891444206237793 + }, + { + "auxiliary_loss_clip": 0.01160446, + "auxiliary_loss_mlp": 0.01044687, + "balance_loss_clip": 1.04880023, + "balance_loss_mlp": 1.02432585, + "epoch": 0.26405338784344373, + "flos": 46535775465600.0, + "grad_norm": 2.2573493859744733, + "language_loss": 0.68835121, + "learning_rate": 3.452508107187926e-06, + "loss": 0.71040261, + "num_input_tokens_seen": 47085130, + "router_z_loss_clip": 1.11621094, + "router_z_loss_mlp": 0.20349121, + "step": 2196, + "time_per_iteration": 2.984874963760376 + }, + { + "auxiliary_loss_clip": 0.01168504, + "auxiliary_loss_mlp": 0.01047882, + "balance_loss_clip": 1.05102658, + "balance_loss_mlp": 1.02582777, + "epoch": 0.26417363073408284, + "flos": 21179467641600.0, + "grad_norm": 3.1572037614311688, + "language_loss": 0.76725829, + "learning_rate": 3.451972511357366e-06, + "loss": 0.78942215, + "num_input_tokens_seen": 47104675, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.22045898, + "step": 2197, + "time_per_iteration": 2.7844743728637695 + }, + { + "auxiliary_loss_clip": 0.01159119, + "auxiliary_loss_mlp": 0.01050547, + "balance_loss_clip": 1.04851186, + "balance_loss_mlp": 1.03147399, + "epoch": 0.26429387362472195, + "flos": 22674751937280.0, + "grad_norm": 10.861885100476426, + "language_loss": 0.84853029, + "learning_rate": 3.45143669526373e-06, + "loss": 0.87062699, + "num_input_tokens_seen": 47124435, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.19091797, + "step": 2198, + "time_per_iteration": 2.782986879348755 + }, + { + "auxiliary_loss_clip": 0.0106013, + "auxiliary_loss_mlp": 0.01000992, + "balance_loss_clip": 1.01682162, + "balance_loss_mlp": 0.99834579, + "epoch": 0.264414116515361, + "flos": 67180534272000.0, + "grad_norm": 0.7915833073381799, + "language_loss": 0.63184983, + "learning_rate": 3.450900658988302e-06, + "loss": 0.65246105, + "num_input_tokens_seen": 47185985, + "router_z_loss_clip": 0.43359375, + "router_z_loss_mlp": 0.02648926, + "step": 2199, + "time_per_iteration": 3.2200167179107666 + }, + { + "auxiliary_loss_clip": 0.01165822, + "auxiliary_loss_mlp": 0.01047479, + "balance_loss_clip": 1.05100441, + "balance_loss_mlp": 1.02665305, + "epoch": 0.2645343594060001, + "flos": 25664709997440.0, + "grad_norm": 3.302078370503861, + "language_loss": 0.77912617, + "learning_rate": 3.450364402612397e-06, + "loss": 0.80125922, + "num_input_tokens_seen": 47203140, + "router_z_loss_clip": 1.14746094, + "router_z_loss_mlp": 0.20837402, + "step": 2200, + "time_per_iteration": 2.777402877807617 + }, + { + "auxiliary_loss_clip": 0.0116157, + "auxiliary_loss_mlp": 0.01046183, + "balance_loss_clip": 1.04497194, + "balance_loss_mlp": 1.0257144, + "epoch": 0.26465460229663923, + "flos": 22491822948480.0, + "grad_norm": 3.9110536466877592, + "language_loss": 0.83989048, + "learning_rate": 3.449827926217366e-06, + "loss": 0.86196804, + "num_input_tokens_seen": 47222575, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.20458984, + "step": 2201, + "time_per_iteration": 2.798797130584717 + }, + { + "auxiliary_loss_clip": 0.01160468, + "auxiliary_loss_mlp": 0.01056, + "balance_loss_clip": 1.04522586, + "balance_loss_mlp": 1.03569818, + "epoch": 0.2647748451872783, + "flos": 29388036038400.0, + "grad_norm": 2.1633719742895656, + "language_loss": 0.80549502, + "learning_rate": 3.449291229884591e-06, + "loss": 0.82765967, + "num_input_tokens_seen": 47243815, + "router_z_loss_clip": 1.15234375, + "router_z_loss_mlp": 0.20300293, + "step": 2202, + "time_per_iteration": 2.928480625152588 + }, + { + "auxiliary_loss_clip": 0.0116125, + "auxiliary_loss_mlp": 0.01046342, + "balance_loss_clip": 1.04789174, + "balance_loss_mlp": 1.02600503, + "epoch": 0.2648950880779174, + "flos": 26797799502720.0, + "grad_norm": 9.948698851843059, + "language_loss": 0.86826146, + "learning_rate": 3.4487543136954887e-06, + "loss": 0.89033735, + "num_input_tokens_seen": 47263435, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.20349121, + "step": 2203, + "time_per_iteration": 2.78493070602417 + }, + { + "auxiliary_loss_clip": 0.01162072, + "auxiliary_loss_mlp": 0.0104565, + "balance_loss_clip": 1.04804397, + "balance_loss_mlp": 1.02590895, + "epoch": 0.2650153309685565, + "flos": 28841008838400.0, + "grad_norm": 1.8963050330174813, + "language_loss": 0.91294849, + "learning_rate": 3.448217177731509e-06, + "loss": 0.93502569, + "num_input_tokens_seen": 47283920, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.19726562, + "step": 2204, + "time_per_iteration": 2.8061811923980713 + }, + { + "auxiliary_loss_clip": 0.01159476, + "auxiliary_loss_mlp": 0.010419, + "balance_loss_clip": 1.0494988, + "balance_loss_mlp": 1.02319622, + "epoch": 0.26513557385919556, + "flos": 20303247271680.0, + "grad_norm": 2.6521631705311295, + "language_loss": 0.77855229, + "learning_rate": 3.4476798220741348e-06, + "loss": 0.80056608, + "num_input_tokens_seen": 47302800, + "router_z_loss_clip": 1.09863281, + "router_z_loss_mlp": 0.18701172, + "step": 2205, + "time_per_iteration": 2.777268171310425 + }, + { + "auxiliary_loss_clip": 0.01163643, + "auxiliary_loss_mlp": 0.0105116, + "balance_loss_clip": 1.05148244, + "balance_loss_mlp": 1.0316571, + "epoch": 0.26525581674983467, + "flos": 17676274101120.0, + "grad_norm": 1.8356796741546528, + "language_loss": 0.78612661, + "learning_rate": 3.4471422468048826e-06, + "loss": 0.80827469, + "num_input_tokens_seen": 47321525, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.19506836, + "step": 2206, + "time_per_iteration": 2.7756950855255127 + }, + { + "auxiliary_loss_clip": 0.01170089, + "auxiliary_loss_mlp": 0.010451, + "balance_loss_clip": 1.0522455, + "balance_loss_mlp": 1.02547789, + "epoch": 0.2653760596404738, + "flos": 26833746038400.0, + "grad_norm": 8.965607497285685, + "language_loss": 0.73090988, + "learning_rate": 3.4466044520053022e-06, + "loss": 0.75306177, + "num_input_tokens_seen": 47340530, + "router_z_loss_clip": 1.17773438, + "router_z_loss_mlp": 0.19604492, + "step": 2207, + "time_per_iteration": 2.824033260345459 + }, + { + "auxiliary_loss_clip": 0.01161756, + "auxiliary_loss_mlp": 0.010479, + "balance_loss_clip": 1.04876566, + "balance_loss_mlp": 1.02831411, + "epoch": 0.26549630253111284, + "flos": 22782160581120.0, + "grad_norm": 1.9476214671597156, + "language_loss": 0.60380018, + "learning_rate": 3.446066437756977e-06, + "loss": 0.62589675, + "num_input_tokens_seen": 47359735, + "router_z_loss_clip": 1.12890625, + "router_z_loss_mlp": 0.19592285, + "step": 2208, + "time_per_iteration": 2.8080127239227295 + }, + { + "auxiliary_loss_clip": 0.01158959, + "auxiliary_loss_mlp": 0.01045414, + "balance_loss_clip": 1.04697919, + "balance_loss_mlp": 1.02568495, + "epoch": 0.26561654542175195, + "flos": 23550002640000.0, + "grad_norm": 3.6726519143367815, + "language_loss": 0.75210679, + "learning_rate": 3.4455282041415224e-06, + "loss": 0.77415049, + "num_input_tokens_seen": 47378945, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.19714355, + "step": 2209, + "time_per_iteration": 2.8298451900482178 + }, + { + "auxiliary_loss_clip": 0.01159031, + "auxiliary_loss_mlp": 0.01045459, + "balance_loss_clip": 1.04677224, + "balance_loss_mlp": 1.02483594, + "epoch": 0.265736788312391, + "flos": 26906680604160.0, + "grad_norm": 2.7478126827465568, + "language_loss": 0.87417644, + "learning_rate": 3.4449897512405894e-06, + "loss": 0.89622134, + "num_input_tokens_seen": 47398095, + "router_z_loss_clip": 1.12304688, + "router_z_loss_mlp": 0.20629883, + "step": 2210, + "time_per_iteration": 2.8001787662506104 + }, + { + "auxiliary_loss_clip": 0.01158171, + "auxiliary_loss_mlp": 0.01042738, + "balance_loss_clip": 1.0479877, + "balance_loss_mlp": 1.02392626, + "epoch": 0.2658570312030301, + "flos": 23477139901440.0, + "grad_norm": 2.9213460296038223, + "language_loss": 0.75145298, + "learning_rate": 3.444451079135859e-06, + "loss": 0.77346206, + "num_input_tokens_seen": 47417605, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.18798828, + "step": 2211, + "time_per_iteration": 2.9979407787323 + }, + { + "auxiliary_loss_clip": 0.0116034, + "auxiliary_loss_mlp": 0.01051869, + "balance_loss_clip": 1.04609144, + "balance_loss_mlp": 1.03299773, + "epoch": 0.2659772740936692, + "flos": 21866402315520.0, + "grad_norm": 2.2543758116023205, + "language_loss": 0.74247962, + "learning_rate": 3.4439121879090493e-06, + "loss": 0.76460171, + "num_input_tokens_seen": 47435385, + "router_z_loss_clip": 1.14257812, + "router_z_loss_mlp": 0.18859863, + "step": 2212, + "time_per_iteration": 3.622969627380371 + }, + { + "auxiliary_loss_clip": 0.01165904, + "auxiliary_loss_mlp": 0.01056051, + "balance_loss_clip": 1.05271995, + "balance_loss_mlp": 1.03608298, + "epoch": 0.2660975169843083, + "flos": 19793100360960.0, + "grad_norm": 3.2186724266499525, + "language_loss": 0.83536947, + "learning_rate": 3.4433730776419082e-06, + "loss": 0.85758901, + "num_input_tokens_seen": 47454310, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.19958496, + "step": 2213, + "time_per_iteration": 2.898404836654663 + }, + { + "auxiliary_loss_clip": 0.01165587, + "auxiliary_loss_mlp": 0.01057125, + "balance_loss_clip": 1.05041015, + "balance_loss_mlp": 1.03548884, + "epoch": 0.2662177598749474, + "flos": 29018981750400.0, + "grad_norm": 2.422696123518626, + "language_loss": 0.80500025, + "learning_rate": 3.4428337484162183e-06, + "loss": 0.82722735, + "num_input_tokens_seen": 47475120, + "router_z_loss_clip": 1.15234375, + "router_z_loss_mlp": 0.21630859, + "step": 2214, + "time_per_iteration": 3.7277028560638428 + }, + { + "auxiliary_loss_clip": 0.01161723, + "auxiliary_loss_mlp": 0.0104287, + "balance_loss_clip": 1.05207634, + "balance_loss_mlp": 1.02378488, + "epoch": 0.2663380027655865, + "flos": 21762549118080.0, + "grad_norm": 2.4691397842110394, + "language_loss": 0.84242618, + "learning_rate": 3.442294200313797e-06, + "loss": 0.86447215, + "num_input_tokens_seen": 47493150, + "router_z_loss_clip": 1.09570312, + "router_z_loss_mlp": 0.1907959, + "step": 2215, + "time_per_iteration": 3.785566806793213 + }, + { + "auxiliary_loss_clip": 0.01070353, + "auxiliary_loss_mlp": 0.01020537, + "balance_loss_clip": 1.02689457, + "balance_loss_mlp": 1.01771152, + "epoch": 0.26645824565622556, + "flos": 66980333819520.0, + "grad_norm": 0.8306818011711526, + "language_loss": 0.52739823, + "learning_rate": 3.4417544334164916e-06, + "loss": 0.54830706, + "num_input_tokens_seen": 47557295, + "router_z_loss_clip": 0.43359375, + "router_z_loss_mlp": 0.02819824, + "step": 2216, + "time_per_iteration": 3.3531558513641357 + }, + { + "auxiliary_loss_clip": 0.0116213, + "auxiliary_loss_mlp": 0.0104439, + "balance_loss_clip": 1.05048406, + "balance_loss_mlp": 1.02542377, + "epoch": 0.26657848854686467, + "flos": 25264198373760.0, + "grad_norm": 1.7018727427854266, + "language_loss": 0.7758522, + "learning_rate": 3.4412144478061854e-06, + "loss": 0.79791737, + "num_input_tokens_seen": 47579705, + "router_z_loss_clip": 1.11621094, + "router_z_loss_mlp": 0.18969727, + "step": 2217, + "time_per_iteration": 2.8096182346343994 + }, + { + "auxiliary_loss_clip": 0.01166906, + "auxiliary_loss_mlp": 0.01050402, + "balance_loss_clip": 1.04999757, + "balance_loss_mlp": 1.02969575, + "epoch": 0.2666987314375038, + "flos": 23696769611520.0, + "grad_norm": 1.9536803510046883, + "language_loss": 0.75940901, + "learning_rate": 3.4406742435647925e-06, + "loss": 0.78158212, + "num_input_tokens_seen": 47599770, + "router_z_loss_clip": 1.16894531, + "router_z_loss_mlp": 0.20715332, + "step": 2218, + "time_per_iteration": 2.7781713008880615 + }, + { + "auxiliary_loss_clip": 0.01158779, + "auxiliary_loss_mlp": 0.01053169, + "balance_loss_clip": 1.04940033, + "balance_loss_mlp": 1.03359437, + "epoch": 0.26681897432814283, + "flos": 27048958375680.0, + "grad_norm": 1.9162460820567968, + "language_loss": 0.78736907, + "learning_rate": 3.440133820774263e-06, + "loss": 0.80948853, + "num_input_tokens_seen": 47619580, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.19592285, + "step": 2219, + "time_per_iteration": 2.789865255355835 + }, + { + "auxiliary_loss_clip": 0.01167947, + "auxiliary_loss_mlp": 0.01054919, + "balance_loss_clip": 1.05142617, + "balance_loss_mlp": 1.03365231, + "epoch": 0.26693921721878194, + "flos": 28985944216320.0, + "grad_norm": 2.346308012495523, + "language_loss": 0.818847, + "learning_rate": 3.439593179516578e-06, + "loss": 0.8410756, + "num_input_tokens_seen": 47639490, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.21276855, + "step": 2220, + "time_per_iteration": 2.801064968109131 + }, + { + "auxiliary_loss_clip": 0.01156047, + "auxiliary_loss_mlp": 0.01040594, + "balance_loss_clip": 1.0464766, + "balance_loss_mlp": 1.02093649, + "epoch": 0.26705946010942105, + "flos": 21507834798720.0, + "grad_norm": 1.9222059622732053, + "language_loss": 0.81017983, + "learning_rate": 3.4390523198737524e-06, + "loss": 0.83214629, + "num_input_tokens_seen": 47658650, + "router_z_loss_clip": 1.09570312, + "router_z_loss_mlp": 0.19665527, + "step": 2221, + "time_per_iteration": 2.787710428237915 + }, + { + "auxiliary_loss_clip": 0.01165468, + "auxiliary_loss_mlp": 0.01043987, + "balance_loss_clip": 1.05175066, + "balance_loss_mlp": 1.02261233, + "epoch": 0.2671797030000601, + "flos": 21471277731840.0, + "grad_norm": 2.007640188802436, + "language_loss": 0.73687613, + "learning_rate": 3.4385112419278333e-06, + "loss": 0.75897068, + "num_input_tokens_seen": 47679875, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.21374512, + "step": 2222, + "time_per_iteration": 2.756486177444458 + }, + { + "auxiliary_loss_clip": 0.0106236, + "auxiliary_loss_mlp": 0.00997776, + "balance_loss_clip": 1.01972771, + "balance_loss_mlp": 0.99541515, + "epoch": 0.2672999458906992, + "flos": 64189929767040.0, + "grad_norm": 0.7916433516700712, + "language_loss": 0.6475445, + "learning_rate": 3.4379699457609033e-06, + "loss": 0.6681459, + "num_input_tokens_seen": 47737700, + "router_z_loss_clip": 0.42578125, + "router_z_loss_mlp": 0.02355957, + "step": 2223, + "time_per_iteration": 3.1674916744232178 + }, + { + "auxiliary_loss_clip": 0.0116224, + "auxiliary_loss_mlp": 0.01049804, + "balance_loss_clip": 1.04816413, + "balance_loss_mlp": 1.0303731, + "epoch": 0.26742018878133833, + "flos": 16909042573440.0, + "grad_norm": 2.53075311802201, + "language_loss": 0.90010452, + "learning_rate": 3.4374284314550755e-06, + "loss": 0.922225, + "num_input_tokens_seen": 47756740, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.19445801, + "step": 2224, + "time_per_iteration": 2.770906448364258 + }, + { + "auxiliary_loss_clip": 0.0116017, + "auxiliary_loss_mlp": 0.01043949, + "balance_loss_clip": 1.04852855, + "balance_loss_mlp": 1.02381468, + "epoch": 0.2675404316719774, + "flos": 20667560964480.0, + "grad_norm": 2.190723272717572, + "language_loss": 0.80460548, + "learning_rate": 3.436886699092498e-06, + "loss": 0.82664663, + "num_input_tokens_seen": 47775255, + "router_z_loss_clip": 1.11621094, + "router_z_loss_mlp": 0.20141602, + "step": 2225, + "time_per_iteration": 2.796630382537842 + }, + { + "auxiliary_loss_clip": 0.01169269, + "auxiliary_loss_mlp": 0.010525, + "balance_loss_clip": 1.05031526, + "balance_loss_mlp": 1.0310781, + "epoch": 0.2676606745626165, + "flos": 17485013157120.0, + "grad_norm": 2.7663530489394517, + "language_loss": 0.71422374, + "learning_rate": 3.4363447487553502e-06, + "loss": 0.73644137, + "num_input_tokens_seen": 47788570, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.2142334, + "step": 2226, + "time_per_iteration": 2.875930070877075 + }, + { + "auxiliary_loss_clip": 0.0115992, + "auxiliary_loss_mlp": 0.01041778, + "balance_loss_clip": 1.04807973, + "balance_loss_mlp": 1.02349091, + "epoch": 0.26778091745325555, + "flos": 27852675143040.0, + "grad_norm": 1.9707419207762586, + "language_loss": 0.78097296, + "learning_rate": 3.4358025805258455e-06, + "loss": 0.80298996, + "num_input_tokens_seen": 47808275, + "router_z_loss_clip": 1.11914062, + "router_z_loss_mlp": 0.1829834, + "step": 2227, + "time_per_iteration": 2.8958048820495605 + }, + { + "auxiliary_loss_clip": 0.01163917, + "auxiliary_loss_mlp": 0.01045871, + "balance_loss_clip": 1.05011141, + "balance_loss_mlp": 1.02536738, + "epoch": 0.26790116034389466, + "flos": 20955995176320.0, + "grad_norm": 5.267276918705914, + "language_loss": 0.83230591, + "learning_rate": 3.435260194486232e-06, + "loss": 0.85440379, + "num_input_tokens_seen": 47826245, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.20507812, + "step": 2228, + "time_per_iteration": 3.06656813621521 + }, + { + "auxiliary_loss_clip": 0.0116666, + "auxiliary_loss_mlp": 0.01053825, + "balance_loss_clip": 1.04987812, + "balance_loss_mlp": 1.03360724, + "epoch": 0.2680214032345338, + "flos": 18040659621120.0, + "grad_norm": 2.4237326365827516, + "language_loss": 0.82468212, + "learning_rate": 3.4347175907187875e-06, + "loss": 0.84688693, + "num_input_tokens_seen": 47843235, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.20227051, + "step": 2229, + "time_per_iteration": 2.9140679836273193 + }, + { + "auxiliary_loss_clip": 0.01160427, + "auxiliary_loss_mlp": 0.01051586, + "balance_loss_clip": 1.04877043, + "balance_loss_mlp": 1.03101063, + "epoch": 0.26814164612517283, + "flos": 22419427086720.0, + "grad_norm": 2.026139233255873, + "language_loss": 0.87710172, + "learning_rate": 3.4341747693058254e-06, + "loss": 0.89922184, + "num_input_tokens_seen": 47861710, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.20568848, + "step": 2230, + "time_per_iteration": 2.8303394317626953 + }, + { + "auxiliary_loss_clip": 0.01157909, + "auxiliary_loss_mlp": 0.01058659, + "balance_loss_clip": 1.04723966, + "balance_loss_mlp": 1.03934705, + "epoch": 0.26826188901581194, + "flos": 35627371159680.0, + "grad_norm": 1.855588052385764, + "language_loss": 0.77310961, + "learning_rate": 3.4336317303296916e-06, + "loss": 0.79527533, + "num_input_tokens_seen": 47882685, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.19311523, + "step": 2231, + "time_per_iteration": 2.945338010787964 + }, + { + "auxiliary_loss_clip": 0.01159495, + "auxiliary_loss_mlp": 0.01044181, + "balance_loss_clip": 1.04897571, + "balance_loss_mlp": 1.02531052, + "epoch": 0.26838213190645105, + "flos": 17639788861440.0, + "grad_norm": 2.13850797336482, + "language_loss": 0.75224268, + "learning_rate": 3.4330884738727635e-06, + "loss": 0.77427948, + "num_input_tokens_seen": 47900860, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.1887207, + "step": 2232, + "time_per_iteration": 2.8054141998291016 + }, + { + "auxiliary_loss_clip": 0.01162963, + "auxiliary_loss_mlp": 0.01054219, + "balance_loss_clip": 1.04778075, + "balance_loss_mlp": 1.03447747, + "epoch": 0.2685023747970901, + "flos": 22674823764480.0, + "grad_norm": 3.2948029705101716, + "language_loss": 0.70529711, + "learning_rate": 3.4325450000174535e-06, + "loss": 0.72746897, + "num_input_tokens_seen": 47917500, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.1973877, + "step": 2233, + "time_per_iteration": 2.8305540084838867 + }, + { + "auxiliary_loss_clip": 0.01165312, + "auxiliary_loss_mlp": 0.01056815, + "balance_loss_clip": 1.0495832, + "balance_loss_mlp": 1.0351193, + "epoch": 0.2686226176877292, + "flos": 20120533764480.0, + "grad_norm": 2.5018838908216257, + "language_loss": 0.74302387, + "learning_rate": 3.4320013088462067e-06, + "loss": 0.7652452, + "num_input_tokens_seen": 47934860, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.21704102, + "step": 2234, + "time_per_iteration": 2.8176681995391846 + }, + { + "auxiliary_loss_clip": 0.01160332, + "auxiliary_loss_mlp": 0.01055927, + "balance_loss_clip": 1.04899764, + "balance_loss_mlp": 1.03603137, + "epoch": 0.2687428605783683, + "flos": 21872040750720.0, + "grad_norm": 1.6108537113707877, + "language_loss": 0.81879574, + "learning_rate": 3.431457400441499e-06, + "loss": 0.84095836, + "num_input_tokens_seen": 47955255, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.19873047, + "step": 2235, + "time_per_iteration": 2.8818325996398926 + }, + { + "auxiliary_loss_clip": 0.01063936, + "auxiliary_loss_mlp": 0.01015602, + "balance_loss_clip": 1.02151108, + "balance_loss_mlp": 1.01321745, + "epoch": 0.2688631034690074, + "flos": 69943320766080.0, + "grad_norm": 0.9689656405933823, + "language_loss": 0.60768878, + "learning_rate": 3.4309132748858424e-06, + "loss": 0.62848425, + "num_input_tokens_seen": 48016245, + "router_z_loss_clip": 0.42578125, + "router_z_loss_mlp": 0.02380371, + "step": 2236, + "time_per_iteration": 3.4158878326416016 + }, + { + "auxiliary_loss_clip": 0.01159163, + "auxiliary_loss_mlp": 0.01045676, + "balance_loss_clip": 1.05001998, + "balance_loss_mlp": 1.02618551, + "epoch": 0.2689833463596465, + "flos": 22856639431680.0, + "grad_norm": 2.293019523697912, + "language_loss": 0.83728206, + "learning_rate": 3.430368932261779e-06, + "loss": 0.85933048, + "num_input_tokens_seen": 48036600, + "router_z_loss_clip": 1.09179688, + "router_z_loss_mlp": 0.19494629, + "step": 2237, + "time_per_iteration": 2.871858835220337 + }, + { + "auxiliary_loss_clip": 0.01163621, + "auxiliary_loss_mlp": 0.01056122, + "balance_loss_clip": 1.05114639, + "balance_loss_mlp": 1.03533173, + "epoch": 0.2691035892502856, + "flos": 17200242132480.0, + "grad_norm": 3.1133235566250996, + "language_loss": 0.75099659, + "learning_rate": 3.429824372651886e-06, + "loss": 0.77319396, + "num_input_tokens_seen": 48054750, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.20800781, + "step": 2238, + "time_per_iteration": 2.863173723220825 + }, + { + "auxiliary_loss_clip": 0.01164046, + "auxiliary_loss_mlp": 0.01057109, + "balance_loss_clip": 1.04913521, + "balance_loss_mlp": 1.03671217, + "epoch": 0.26922383214092466, + "flos": 17747484814080.0, + "grad_norm": 2.361968405095722, + "language_loss": 0.83738905, + "learning_rate": 3.4292795961387732e-06, + "loss": 0.8596006, + "num_input_tokens_seen": 48072650, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.20385742, + "step": 2239, + "time_per_iteration": 2.853914260864258 + }, + { + "auxiliary_loss_clip": 0.01162449, + "auxiliary_loss_mlp": 0.01042065, + "balance_loss_clip": 1.0490495, + "balance_loss_mlp": 1.02181089, + "epoch": 0.26934407503156377, + "flos": 16173376122240.0, + "grad_norm": 2.625156123160949, + "language_loss": 0.87719011, + "learning_rate": 3.4287346028050818e-06, + "loss": 0.89923525, + "num_input_tokens_seen": 48088720, + "router_z_loss_clip": 1.13183594, + "router_z_loss_mlp": 0.20263672, + "step": 2240, + "time_per_iteration": 3.613858461380005 + }, + { + "auxiliary_loss_clip": 0.01155241, + "auxiliary_loss_mlp": 0.01049275, + "balance_loss_clip": 1.0475688, + "balance_loss_mlp": 1.03010654, + "epoch": 0.2694643179222028, + "flos": 23732895715200.0, + "grad_norm": 1.9485153437313447, + "language_loss": 0.79776973, + "learning_rate": 3.4281893927334866e-06, + "loss": 0.81981486, + "num_input_tokens_seen": 48108630, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.19165039, + "step": 2241, + "time_per_iteration": 2.820492744445801 + }, + { + "auxiliary_loss_clip": 0.01159485, + "auxiliary_loss_mlp": 0.0104662, + "balance_loss_clip": 1.0483923, + "balance_loss_mlp": 1.02739191, + "epoch": 0.26958456081284193, + "flos": 24718140840960.0, + "grad_norm": 2.0176333031014204, + "language_loss": 0.75468427, + "learning_rate": 3.4276439660066963e-06, + "loss": 0.77674532, + "num_input_tokens_seen": 48128330, + "router_z_loss_clip": 1.11132812, + "router_z_loss_mlp": 0.19226074, + "step": 2242, + "time_per_iteration": 2.811419725418091 + }, + { + "auxiliary_loss_clip": 0.01163616, + "auxiliary_loss_mlp": 0.01052025, + "balance_loss_clip": 1.05234385, + "balance_loss_mlp": 1.03231955, + "epoch": 0.26970480370348104, + "flos": 18112588606080.0, + "grad_norm": 2.413379015358738, + "language_loss": 0.84126544, + "learning_rate": 3.427098322707452e-06, + "loss": 0.8634218, + "num_input_tokens_seen": 48144295, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.19714355, + "step": 2243, + "time_per_iteration": 3.6535279750823975 + }, + { + "auxiliary_loss_clip": 0.01168744, + "auxiliary_loss_mlp": 0.0103845, + "balance_loss_clip": 1.05578136, + "balance_loss_mlp": 1.0184468, + "epoch": 0.2698250465941201, + "flos": 10816546250880.0, + "grad_norm": 2.7741657267479094, + "language_loss": 0.89597821, + "learning_rate": 3.426552462918526e-06, + "loss": 0.91805017, + "num_input_tokens_seen": 48162230, + "router_z_loss_clip": 1.12988281, + "router_z_loss_mlp": 0.19995117, + "step": 2244, + "time_per_iteration": 3.6920177936553955 + }, + { + "auxiliary_loss_clip": 0.01160986, + "auxiliary_loss_mlp": 0.01046849, + "balance_loss_clip": 1.05127347, + "balance_loss_mlp": 1.02731025, + "epoch": 0.2699452894847592, + "flos": 17308117653120.0, + "grad_norm": 4.2888719999245115, + "language_loss": 0.73321629, + "learning_rate": 3.426006386722726e-06, + "loss": 0.75529456, + "num_input_tokens_seen": 48180290, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.19543457, + "step": 2245, + "time_per_iteration": 2.7900986671447754 + }, + { + "auxiliary_loss_clip": 0.01167084, + "auxiliary_loss_mlp": 0.01047413, + "balance_loss_clip": 1.05458033, + "balance_loss_mlp": 1.02823246, + "epoch": 0.2700655323753983, + "flos": 18078150441600.0, + "grad_norm": 1.9974145530798457, + "language_loss": 0.92495686, + "learning_rate": 3.4254600942028914e-06, + "loss": 0.94710183, + "num_input_tokens_seen": 48198165, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.19189453, + "step": 2246, + "time_per_iteration": 2.87109637260437 + }, + { + "auxiliary_loss_clip": 0.01162264, + "auxiliary_loss_mlp": 0.01046212, + "balance_loss_clip": 1.05227733, + "balance_loss_mlp": 1.02736509, + "epoch": 0.2701857752660374, + "flos": 18186636493440.0, + "grad_norm": 2.389811532475167, + "language_loss": 0.82819068, + "learning_rate": 3.424913585441893e-06, + "loss": 0.8502754, + "num_input_tokens_seen": 48216000, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.18847656, + "step": 2247, + "time_per_iteration": 2.8077199459075928 + }, + { + "auxiliary_loss_clip": 0.01160271, + "auxiliary_loss_mlp": 0.01047099, + "balance_loss_clip": 1.0511272, + "balance_loss_mlp": 1.02615356, + "epoch": 0.2703060181566765, + "flos": 16319496648960.0, + "grad_norm": 2.0342978019580915, + "language_loss": 0.8748256, + "learning_rate": 3.4243668605226374e-06, + "loss": 0.89689934, + "num_input_tokens_seen": 48233025, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.20947266, + "step": 2248, + "time_per_iteration": 2.8515639305114746 + }, + { + "auxiliary_loss_clip": 0.01168685, + "auxiliary_loss_mlp": 0.01048251, + "balance_loss_clip": 1.05039513, + "balance_loss_mlp": 1.0281285, + "epoch": 0.2704262610473156, + "flos": 19572357329280.0, + "grad_norm": 2.6163650378136354, + "language_loss": 0.82698989, + "learning_rate": 3.423819919528061e-06, + "loss": 0.8491593, + "num_input_tokens_seen": 48251110, + "router_z_loss_clip": 1.18164062, + "router_z_loss_mlp": 0.20141602, + "step": 2249, + "time_per_iteration": 2.8239879608154297 + }, + { + "auxiliary_loss_clip": 0.01158248, + "auxiliary_loss_mlp": 0.01050495, + "balance_loss_clip": 1.04816234, + "balance_loss_mlp": 1.03155243, + "epoch": 0.27054650393795465, + "flos": 20740746925440.0, + "grad_norm": 1.9688125314866844, + "language_loss": 0.78218859, + "learning_rate": 3.4232727625411355e-06, + "loss": 0.80427599, + "num_input_tokens_seen": 48270215, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.18933105, + "step": 2250, + "time_per_iteration": 2.8722879886627197 + }, + { + "auxiliary_loss_clip": 0.01157993, + "auxiliary_loss_mlp": 0.01043374, + "balance_loss_clip": 1.04773569, + "balance_loss_mlp": 1.0246464, + "epoch": 0.27066674682859376, + "flos": 18658322916480.0, + "grad_norm": 2.0751422101419905, + "language_loss": 0.86447668, + "learning_rate": 3.4227253896448626e-06, + "loss": 0.88649035, + "num_input_tokens_seen": 48288075, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.18713379, + "step": 2251, + "time_per_iteration": 2.866230010986328 + }, + { + "auxiliary_loss_clip": 0.01157647, + "auxiliary_loss_mlp": 0.01038244, + "balance_loss_clip": 1.04643703, + "balance_loss_mlp": 1.02057683, + "epoch": 0.2707869897192329, + "flos": 23002759958400.0, + "grad_norm": 2.978191316882876, + "language_loss": 0.81762528, + "learning_rate": 3.42217780092228e-06, + "loss": 0.83958423, + "num_input_tokens_seen": 48306415, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.17663574, + "step": 2252, + "time_per_iteration": 2.8183140754699707 + }, + { + "auxiliary_loss_clip": 0.01071363, + "auxiliary_loss_mlp": 0.00999998, + "balance_loss_clip": 1.02882957, + "balance_loss_mlp": 0.99778044, + "epoch": 0.27090723260987193, + "flos": 58323240293760.0, + "grad_norm": 0.7946316370113667, + "language_loss": 0.60332823, + "learning_rate": 3.421629996456456e-06, + "loss": 0.62404191, + "num_input_tokens_seen": 48365035, + "router_z_loss_clip": 0.42578125, + "router_z_loss_mlp": 0.0222168, + "step": 2253, + "time_per_iteration": 3.2583885192871094 + }, + { + "auxiliary_loss_clip": 0.01163723, + "auxiliary_loss_mlp": 0.01046775, + "balance_loss_clip": 1.05063772, + "balance_loss_mlp": 1.02616322, + "epoch": 0.27102747550051104, + "flos": 11984540797440.0, + "grad_norm": 2.2033671321107624, + "language_loss": 0.82511413, + "learning_rate": 3.421081976330491e-06, + "loss": 0.84721911, + "num_input_tokens_seen": 48383550, + "router_z_loss_clip": 1.13183594, + "router_z_loss_mlp": 0.20617676, + "step": 2254, + "time_per_iteration": 2.812826156616211 + }, + { + "auxiliary_loss_clip": 0.01160083, + "auxiliary_loss_mlp": 0.01044449, + "balance_loss_clip": 1.0498867, + "balance_loss_mlp": 1.02506566, + "epoch": 0.27114771839115015, + "flos": 19900401264000.0, + "grad_norm": 1.9922835249082942, + "language_loss": 0.88013375, + "learning_rate": 3.4205337406275207e-06, + "loss": 0.90217906, + "num_input_tokens_seen": 48403670, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.19384766, + "step": 2255, + "time_per_iteration": 2.9929938316345215 + }, + { + "auxiliary_loss_clip": 0.01157091, + "auxiliary_loss_mlp": 0.010466, + "balance_loss_clip": 1.04685462, + "balance_loss_mlp": 1.02687073, + "epoch": 0.2712679612817892, + "flos": 18331966920960.0, + "grad_norm": 3.1298105367454805, + "language_loss": 0.75704038, + "learning_rate": 3.4199852894307114e-06, + "loss": 0.77907729, + "num_input_tokens_seen": 48420420, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.19726562, + "step": 2256, + "time_per_iteration": 2.8084614276885986 + }, + { + "auxiliary_loss_clip": 0.01162767, + "auxiliary_loss_mlp": 0.01043854, + "balance_loss_clip": 1.0486747, + "balance_loss_mlp": 1.02408886, + "epoch": 0.2713882041724283, + "flos": 24460302038400.0, + "grad_norm": 2.592090845222943, + "language_loss": 0.78507149, + "learning_rate": 3.419436622823262e-06, + "loss": 0.80713767, + "num_input_tokens_seen": 48441140, + "router_z_loss_clip": 1.14160156, + "router_z_loss_mlp": 0.19763184, + "step": 2257, + "time_per_iteration": 2.8777143955230713 + }, + { + "auxiliary_loss_clip": 0.01156518, + "auxiliary_loss_mlp": 0.01044582, + "balance_loss_clip": 1.04899597, + "balance_loss_mlp": 1.0277735, + "epoch": 0.27150844706306737, + "flos": 23039317025280.0, + "grad_norm": 2.08716277554851, + "language_loss": 0.745049, + "learning_rate": 3.4188877408884063e-06, + "loss": 0.76706004, + "num_input_tokens_seen": 48461845, + "router_z_loss_clip": 1.07470703, + "router_z_loss_mlp": 0.16809082, + "step": 2258, + "time_per_iteration": 2.861936092376709 + }, + { + "auxiliary_loss_clip": 0.01162407, + "auxiliary_loss_mlp": 0.01043942, + "balance_loss_clip": 1.05060756, + "balance_loss_mlp": 1.02432013, + "epoch": 0.2716286899537065, + "flos": 22563644192640.0, + "grad_norm": 9.04459991965312, + "language_loss": 0.6503706, + "learning_rate": 3.4183386437094088e-06, + "loss": 0.67243409, + "num_input_tokens_seen": 48478510, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.19628906, + "step": 2259, + "time_per_iteration": 2.8062829971313477 + }, + { + "auxiliary_loss_clip": 0.011618, + "auxiliary_loss_mlp": 0.01049742, + "balance_loss_clip": 1.0492382, + "balance_loss_mlp": 1.02954841, + "epoch": 0.2717489328443456, + "flos": 13115044523520.0, + "grad_norm": 2.338968492427291, + "language_loss": 0.82532579, + "learning_rate": 3.417789331369565e-06, + "loss": 0.8474412, + "num_input_tokens_seen": 48494300, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.2019043, + "step": 2260, + "time_per_iteration": 2.8673152923583984 + }, + { + "auxiliary_loss_clip": 0.01165345, + "auxiliary_loss_mlp": 0.01051214, + "balance_loss_clip": 1.05170524, + "balance_loss_mlp": 1.02944648, + "epoch": 0.27186917573498465, + "flos": 29278688060160.0, + "grad_norm": 2.539022139769072, + "language_loss": 0.914922, + "learning_rate": 3.4172398039522088e-06, + "loss": 0.9370876, + "num_input_tokens_seen": 48515585, + "router_z_loss_clip": 1.13574219, + "router_z_loss_mlp": 0.21789551, + "step": 2261, + "time_per_iteration": 2.8486013412475586 + }, + { + "auxiliary_loss_clip": 0.01165859, + "auxiliary_loss_mlp": 0.01054448, + "balance_loss_clip": 1.04945493, + "balance_loss_mlp": 1.03364575, + "epoch": 0.27198941862562376, + "flos": 26032220000640.0, + "grad_norm": 1.8136105235400977, + "language_loss": 0.79589391, + "learning_rate": 3.4166900615407e-06, + "loss": 0.818097, + "num_input_tokens_seen": 48533500, + "router_z_loss_clip": 1.1640625, + "router_z_loss_mlp": 0.20800781, + "step": 2262, + "time_per_iteration": 2.8623569011688232 + }, + { + "auxiliary_loss_clip": 0.01161239, + "auxiliary_loss_mlp": 0.01045549, + "balance_loss_clip": 1.05040383, + "balance_loss_mlp": 1.02670217, + "epoch": 0.27210966151626287, + "flos": 32780983760640.0, + "grad_norm": 1.914485744297861, + "language_loss": 0.7504909, + "learning_rate": 3.416140104218436e-06, + "loss": 0.77255881, + "num_input_tokens_seen": 48552865, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.18835449, + "step": 2263, + "time_per_iteration": 2.889545202255249 + }, + { + "auxiliary_loss_clip": 0.01060852, + "auxiliary_loss_mlp": 0.01030827, + "balance_loss_clip": 1.0181942, + "balance_loss_mlp": 1.02835894, + "epoch": 0.2722299044069019, + "flos": 65471043219840.0, + "grad_norm": 0.84811366291223, + "language_loss": 0.69660008, + "learning_rate": 3.4155899320688437e-06, + "loss": 0.7175169, + "num_input_tokens_seen": 48618940, + "router_z_loss_clip": 0.42578125, + "router_z_loss_mlp": 0.0246582, + "step": 2264, + "time_per_iteration": 3.344194173812866 + }, + { + "auxiliary_loss_clip": 0.01157229, + "auxiliary_loss_mlp": 0.01043483, + "balance_loss_clip": 1.0466367, + "balance_loss_mlp": 1.02349138, + "epoch": 0.27235014729754103, + "flos": 15334143782400.0, + "grad_norm": 2.2949158793321303, + "language_loss": 0.73704994, + "learning_rate": 3.415039545175384e-06, + "loss": 0.75905704, + "num_input_tokens_seen": 48634665, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.19995117, + "step": 2265, + "time_per_iteration": 2.7260642051696777 + }, + { + "auxiliary_loss_clip": 0.0115952, + "auxiliary_loss_mlp": 0.01045858, + "balance_loss_clip": 1.04862285, + "balance_loss_mlp": 1.02586699, + "epoch": 0.27247039018818014, + "flos": 21872363973120.0, + "grad_norm": 3.1306109665738227, + "language_loss": 0.65377951, + "learning_rate": 3.414488943621551e-06, + "loss": 0.67583328, + "num_input_tokens_seen": 48653330, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.1998291, + "step": 2266, + "time_per_iteration": 2.828847885131836 + }, + { + "auxiliary_loss_clip": 0.01158619, + "auxiliary_loss_mlp": 0.01044023, + "balance_loss_clip": 1.0484513, + "balance_loss_mlp": 1.02416301, + "epoch": 0.2725906330788192, + "flos": 18695490514560.0, + "grad_norm": 2.1614691228930396, + "language_loss": 0.73461163, + "learning_rate": 3.41393812749087e-06, + "loss": 0.75663805, + "num_input_tokens_seen": 48671375, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.1986084, + "step": 2267, + "time_per_iteration": 2.815187454223633 + }, + { + "auxiliary_loss_clip": 0.01162032, + "auxiliary_loss_mlp": 0.01047929, + "balance_loss_clip": 1.04991698, + "balance_loss_mlp": 1.02843833, + "epoch": 0.2727108759694583, + "flos": 17886099398400.0, + "grad_norm": 3.0042878768156904, + "language_loss": 0.72095543, + "learning_rate": 3.4133870968668984e-06, + "loss": 0.74305511, + "num_input_tokens_seen": 48686175, + "router_z_loss_clip": 1.12304688, + "router_z_loss_mlp": 0.19494629, + "step": 2268, + "time_per_iteration": 2.89790940284729 + }, + { + "auxiliary_loss_clip": 0.01165404, + "auxiliary_loss_mlp": 0.01061318, + "balance_loss_clip": 1.05326712, + "balance_loss_mlp": 1.04008663, + "epoch": 0.2728311188600974, + "flos": 24461666755200.0, + "grad_norm": 3.3316945104800753, + "language_loss": 0.78575701, + "learning_rate": 3.412835851833229e-06, + "loss": 0.80802417, + "num_input_tokens_seen": 48708370, + "router_z_loss_clip": 1.12304688, + "router_z_loss_mlp": 0.21228027, + "step": 2269, + "time_per_iteration": 4.543592691421509 + }, + { + "auxiliary_loss_clip": 0.01159992, + "auxiliary_loss_mlp": 0.01051477, + "balance_loss_clip": 1.04952037, + "balance_loss_mlp": 1.03138995, + "epoch": 0.2729513617507365, + "flos": 30993314757120.0, + "grad_norm": 2.2196552578853583, + "language_loss": 0.77890915, + "learning_rate": 3.4122843924734834e-06, + "loss": 0.80102384, + "num_input_tokens_seen": 48730670, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.20092773, + "step": 2270, + "time_per_iteration": 2.8794922828674316 + }, + { + "auxiliary_loss_clip": 0.01162184, + "auxiliary_loss_mlp": 0.01045163, + "balance_loss_clip": 1.04984403, + "balance_loss_mlp": 1.02405095, + "epoch": 0.2730716046413756, + "flos": 19094637421440.0, + "grad_norm": 2.4762197718363947, + "language_loss": 0.87518406, + "learning_rate": 3.411732718871319e-06, + "loss": 0.89725751, + "num_input_tokens_seen": 48746510, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.21105957, + "step": 2271, + "time_per_iteration": 2.876312494277954 + }, + { + "auxiliary_loss_clip": 0.01155169, + "auxiliary_loss_mlp": 0.01050021, + "balance_loss_clip": 1.0499475, + "balance_loss_mlp": 1.03162658, + "epoch": 0.27319184753201464, + "flos": 26944566474240.0, + "grad_norm": 1.6813284906829329, + "language_loss": 0.78865147, + "learning_rate": 3.4111808311104227e-06, + "loss": 0.81070334, + "num_input_tokens_seen": 48768825, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.18395996, + "step": 2272, + "time_per_iteration": 4.7021167278289795 + }, + { + "auxiliary_loss_clip": 0.01161488, + "auxiliary_loss_mlp": 0.0105729, + "balance_loss_clip": 1.04951406, + "balance_loss_mlp": 1.03653526, + "epoch": 0.27331209042265375, + "flos": 31759828012800.0, + "grad_norm": 2.115131489402967, + "language_loss": 0.69695073, + "learning_rate": 3.410628729274517e-06, + "loss": 0.7191385, + "num_input_tokens_seen": 48790345, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.20751953, + "step": 2273, + "time_per_iteration": 2.9032492637634277 + }, + { + "auxiliary_loss_clip": 0.01159913, + "auxiliary_loss_mlp": 0.01048003, + "balance_loss_clip": 1.04693222, + "balance_loss_mlp": 1.02653301, + "epoch": 0.27343233331329286, + "flos": 25739081107200.0, + "grad_norm": 2.2188868881512143, + "language_loss": 0.82892811, + "learning_rate": 3.4100764134473546e-06, + "loss": 0.85100728, + "num_input_tokens_seen": 48809630, + "router_z_loss_clip": 1.12890625, + "router_z_loss_mlp": 0.21472168, + "step": 2274, + "time_per_iteration": 2.9499757289886475 + }, + { + "auxiliary_loss_clip": 0.01161046, + "auxiliary_loss_mlp": 0.01041588, + "balance_loss_clip": 1.05345452, + "balance_loss_mlp": 1.02291977, + "epoch": 0.2735525762039319, + "flos": 24389414547840.0, + "grad_norm": 2.472386303895963, + "language_loss": 0.85258836, + "learning_rate": 3.4095238837127215e-06, + "loss": 0.87461472, + "num_input_tokens_seen": 48828770, + "router_z_loss_clip": 1.07568359, + "router_z_loss_mlp": 0.18688965, + "step": 2275, + "time_per_iteration": 2.7802538871765137 + }, + { + "auxiliary_loss_clip": 0.01162938, + "auxiliary_loss_mlp": 0.01038217, + "balance_loss_clip": 1.04944491, + "balance_loss_mlp": 1.02077699, + "epoch": 0.27367281909457103, + "flos": 14465357527680.0, + "grad_norm": 2.2727695618770882, + "language_loss": 0.79435217, + "learning_rate": 3.4089711401544355e-06, + "loss": 0.81636363, + "num_input_tokens_seen": 48846365, + "router_z_loss_clip": 1.13574219, + "router_z_loss_mlp": 0.17431641, + "step": 2276, + "time_per_iteration": 2.774263381958008 + }, + { + "auxiliary_loss_clip": 0.01159399, + "auxiliary_loss_mlp": 0.01049417, + "balance_loss_clip": 1.04931414, + "balance_loss_mlp": 1.02892518, + "epoch": 0.27379306198521014, + "flos": 23476996247040.0, + "grad_norm": 2.562114529649229, + "language_loss": 0.67540467, + "learning_rate": 3.4084181828563486e-06, + "loss": 0.69749284, + "num_input_tokens_seen": 48863085, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.20483398, + "step": 2277, + "time_per_iteration": 2.834092140197754 + }, + { + "auxiliary_loss_clip": 0.01159464, + "auxiliary_loss_mlp": 0.01053436, + "balance_loss_clip": 1.05127597, + "balance_loss_mlp": 1.03498197, + "epoch": 0.2739133048758492, + "flos": 17458152762240.0, + "grad_norm": 2.0534617560067616, + "language_loss": 0.70958775, + "learning_rate": 3.4078650119023428e-06, + "loss": 0.73171675, + "num_input_tokens_seen": 48881400, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.18457031, + "step": 2278, + "time_per_iteration": 2.7792983055114746 + }, + { + "auxiliary_loss_clip": 0.01166487, + "auxiliary_loss_mlp": 0.01050367, + "balance_loss_clip": 1.05087852, + "balance_loss_mlp": 1.02813482, + "epoch": 0.2740335477664883, + "flos": 19273113123840.0, + "grad_norm": 2.7104689718046973, + "language_loss": 0.74632359, + "learning_rate": 3.4073116273763337e-06, + "loss": 0.76849222, + "num_input_tokens_seen": 48895845, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.22229004, + "step": 2279, + "time_per_iteration": 2.7292113304138184 + }, + { + "auxiliary_loss_clip": 0.01161842, + "auxiliary_loss_mlp": 0.010547, + "balance_loss_clip": 1.04854941, + "balance_loss_mlp": 1.03354001, + "epoch": 0.2741537906571274, + "flos": 26104723603200.0, + "grad_norm": 1.832456102576481, + "language_loss": 0.81108457, + "learning_rate": 3.40675802936227e-06, + "loss": 0.83325005, + "num_input_tokens_seen": 48916630, + "router_z_loss_clip": 1.13476562, + "router_z_loss_mlp": 0.21154785, + "step": 2280, + "time_per_iteration": 2.795543909072876 + }, + { + "auxiliary_loss_clip": 0.01164762, + "auxiliary_loss_mlp": 0.01050799, + "balance_loss_clip": 1.04965639, + "balance_loss_mlp": 1.02873373, + "epoch": 0.27427403354776647, + "flos": 34164190644480.0, + "grad_norm": 2.095294069077166, + "language_loss": 0.7211597, + "learning_rate": 3.4062042179441318e-06, + "loss": 0.74331534, + "num_input_tokens_seen": 48937100, + "router_z_loss_clip": 1.15039062, + "router_z_loss_mlp": 0.22058105, + "step": 2281, + "time_per_iteration": 2.861757755279541 + }, + { + "auxiliary_loss_clip": 0.01155153, + "auxiliary_loss_mlp": 0.01045428, + "balance_loss_clip": 1.0485816, + "balance_loss_mlp": 1.02685511, + "epoch": 0.2743942764384056, + "flos": 18766988536320.0, + "grad_norm": 2.4510344418387113, + "language_loss": 0.80881226, + "learning_rate": 3.4056501932059314e-06, + "loss": 0.83081806, + "num_input_tokens_seen": 48955175, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.18566895, + "step": 2282, + "time_per_iteration": 2.7743091583251953 + }, + { + "auxiliary_loss_clip": 0.01066101, + "auxiliary_loss_mlp": 0.01002883, + "balance_loss_clip": 1.02276564, + "balance_loss_mlp": 1.00023699, + "epoch": 0.2745145193290447, + "flos": 64904048058240.0, + "grad_norm": 0.7732864104518917, + "language_loss": 0.58091372, + "learning_rate": 3.405095955231715e-06, + "loss": 0.60160357, + "num_input_tokens_seen": 49006830, + "router_z_loss_clip": 0.43359375, + "router_z_loss_mlp": 0.02648926, + "step": 2283, + "time_per_iteration": 3.2336323261260986 + }, + { + "auxiliary_loss_clip": 0.01161746, + "auxiliary_loss_mlp": 0.01044521, + "balance_loss_clip": 1.04917598, + "balance_loss_mlp": 1.02394581, + "epoch": 0.27463476221968375, + "flos": 16136926796160.0, + "grad_norm": 2.847192308809256, + "language_loss": 0.94541085, + "learning_rate": 3.4045415041055585e-06, + "loss": 0.96747351, + "num_input_tokens_seen": 49022470, + "router_z_loss_clip": 1.12597656, + "router_z_loss_mlp": 0.20568848, + "step": 2284, + "time_per_iteration": 2.755280017852783 + }, + { + "auxiliary_loss_clip": 0.01162189, + "auxiliary_loss_mlp": 0.01049104, + "balance_loss_clip": 1.04887879, + "balance_loss_mlp": 1.02734816, + "epoch": 0.27475500511032286, + "flos": 10376712213120.0, + "grad_norm": 2.279005040429952, + "language_loss": 0.78296816, + "learning_rate": 3.4039868399115728e-06, + "loss": 0.80508107, + "num_input_tokens_seen": 49037110, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.2175293, + "step": 2285, + "time_per_iteration": 2.838261842727661 + }, + { + "auxiliary_loss_clip": 0.01165086, + "auxiliary_loss_mlp": 0.01055546, + "balance_loss_clip": 1.05365312, + "balance_loss_mlp": 1.03528011, + "epoch": 0.27487524800096197, + "flos": 17311062568320.0, + "grad_norm": 2.020087270487789, + "language_loss": 0.80157089, + "learning_rate": 3.4034319627339003e-06, + "loss": 0.8237772, + "num_input_tokens_seen": 49053975, + "router_z_loss_clip": 1.11425781, + "router_z_loss_mlp": 0.20263672, + "step": 2286, + "time_per_iteration": 2.9653220176696777 + }, + { + "auxiliary_loss_clip": 0.0116195, + "auxiliary_loss_mlp": 0.01045865, + "balance_loss_clip": 1.05090797, + "balance_loss_mlp": 1.02483702, + "epoch": 0.274995490891601, + "flos": 27120205002240.0, + "grad_norm": 2.810900577161689, + "language_loss": 0.69365144, + "learning_rate": 3.402876872656715e-06, + "loss": 0.71572959, + "num_input_tokens_seen": 49072295, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.21044922, + "step": 2287, + "time_per_iteration": 2.91658878326416 + }, + { + "auxiliary_loss_clip": 0.01164228, + "auxiliary_loss_mlp": 0.01049024, + "balance_loss_clip": 1.05115843, + "balance_loss_mlp": 1.02955759, + "epoch": 0.27511573378224013, + "flos": 23436093634560.0, + "grad_norm": 2.5534988597252566, + "language_loss": 0.89745057, + "learning_rate": 3.402321569764223e-06, + "loss": 0.91958308, + "num_input_tokens_seen": 49091600, + "router_z_loss_clip": 1.13183594, + "router_z_loss_mlp": 0.19470215, + "step": 2288, + "time_per_iteration": 2.9104843139648438 + }, + { + "auxiliary_loss_clip": 0.01164794, + "auxiliary_loss_mlp": 0.01051184, + "balance_loss_clip": 1.04948008, + "balance_loss_mlp": 1.02899957, + "epoch": 0.2752359766728792, + "flos": 16722019434240.0, + "grad_norm": 2.344855716837447, + "language_loss": 0.833637, + "learning_rate": 3.4017660541406635e-06, + "loss": 0.85579681, + "num_input_tokens_seen": 49107665, + "router_z_loss_clip": 1.15429688, + "router_z_loss_mlp": 0.22180176, + "step": 2289, + "time_per_iteration": 2.792217493057251 + }, + { + "auxiliary_loss_clip": 0.01156148, + "auxiliary_loss_mlp": 0.01047178, + "balance_loss_clip": 1.04685616, + "balance_loss_mlp": 1.02685332, + "epoch": 0.2753562195635183, + "flos": 25297738698240.0, + "grad_norm": 2.0064826221534093, + "language_loss": 0.7449398, + "learning_rate": 3.4012103258703092e-06, + "loss": 0.76697302, + "num_input_tokens_seen": 49126420, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.203125, + "step": 2290, + "time_per_iteration": 2.8663222789764404 + }, + { + "auxiliary_loss_clip": 0.01161151, + "auxiliary_loss_mlp": 0.01049251, + "balance_loss_clip": 1.05151272, + "balance_loss_mlp": 1.02980757, + "epoch": 0.2754764624541574, + "flos": 27338972785920.0, + "grad_norm": 2.0087482773144414, + "language_loss": 0.83108819, + "learning_rate": 3.4006543850374616e-06, + "loss": 0.85319221, + "num_input_tokens_seen": 49141470, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.19433594, + "step": 2291, + "time_per_iteration": 2.858900547027588 + }, + { + "auxiliary_loss_clip": 0.01157539, + "auxiliary_loss_mlp": 0.01047801, + "balance_loss_clip": 1.04705739, + "balance_loss_mlp": 1.02941906, + "epoch": 0.27559670534479647, + "flos": 17238379397760.0, + "grad_norm": 2.1195820255716225, + "language_loss": 0.7508682, + "learning_rate": 3.400098231726458e-06, + "loss": 0.77292162, + "num_input_tokens_seen": 49158570, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.18359375, + "step": 2292, + "time_per_iteration": 2.8367273807525635 + }, + { + "auxiliary_loss_clip": 0.01163364, + "auxiliary_loss_mlp": 0.01046471, + "balance_loss_clip": 1.04876006, + "balance_loss_mlp": 1.02479863, + "epoch": 0.2757169482354356, + "flos": 21939085486080.0, + "grad_norm": 2.079228373087781, + "language_loss": 0.87051815, + "learning_rate": 3.3995418660216657e-06, + "loss": 0.89261651, + "num_input_tokens_seen": 49176025, + "router_z_loss_clip": 1.14746094, + "router_z_loss_mlp": 0.2166748, + "step": 2293, + "time_per_iteration": 2.8250889778137207 + }, + { + "auxiliary_loss_clip": 0.01166894, + "auxiliary_loss_mlp": 0.01052576, + "balance_loss_clip": 1.0492934, + "balance_loss_mlp": 1.03014064, + "epoch": 0.2758371911260747, + "flos": 20850669521280.0, + "grad_norm": 2.7423499352547913, + "language_loss": 0.80555427, + "learning_rate": 3.3989852880074848e-06, + "loss": 0.82774901, + "num_input_tokens_seen": 49197455, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.22436523, + "step": 2294, + "time_per_iteration": 2.897082567214966 + }, + { + "auxiliary_loss_clip": 0.01065522, + "auxiliary_loss_mlp": 0.01002396, + "balance_loss_clip": 1.0222466, + "balance_loss_mlp": 0.9997614, + "epoch": 0.27595743401671374, + "flos": 69269063592960.0, + "grad_norm": 0.7536475350778399, + "language_loss": 0.60602802, + "learning_rate": 3.398428497768348e-06, + "loss": 0.62670726, + "num_input_tokens_seen": 49262625, + "router_z_loss_clip": 0.43359375, + "router_z_loss_mlp": 0.02636719, + "step": 2295, + "time_per_iteration": 3.430445909500122 + }, + { + "auxiliary_loss_clip": 0.01160826, + "auxiliary_loss_mlp": 0.01043923, + "balance_loss_clip": 1.0485549, + "balance_loss_mlp": 1.02413416, + "epoch": 0.27607767690735285, + "flos": 21215019127680.0, + "grad_norm": 1.956291523129879, + "language_loss": 0.71909899, + "learning_rate": 3.3978714953887205e-06, + "loss": 0.74114645, + "num_input_tokens_seen": 49282380, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.19799805, + "step": 2296, + "time_per_iteration": 2.7635409832000732 + }, + { + "auxiliary_loss_clip": 0.01166104, + "auxiliary_loss_mlp": 0.01052401, + "balance_loss_clip": 1.05110204, + "balance_loss_mlp": 1.03162277, + "epoch": 0.27619791979799196, + "flos": 24825334003200.0, + "grad_norm": 2.008245280344532, + "language_loss": 0.86150157, + "learning_rate": 3.397314280953098e-06, + "loss": 0.88368666, + "num_input_tokens_seen": 49303205, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.20788574, + "step": 2297, + "time_per_iteration": 2.849729061126709 + }, + { + "auxiliary_loss_clip": 0.01159186, + "auxiliary_loss_mlp": 0.01053443, + "balance_loss_clip": 1.04905558, + "balance_loss_mlp": 1.0326407, + "epoch": 0.276318162688631, + "flos": 24753548672640.0, + "grad_norm": 2.826614174412914, + "language_loss": 0.79818308, + "learning_rate": 3.3967568545460108e-06, + "loss": 0.8203094, + "num_input_tokens_seen": 49322745, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.20800781, + "step": 2298, + "time_per_iteration": 3.8707833290100098 + }, + { + "auxiliary_loss_clip": 0.01155982, + "auxiliary_loss_mlp": 0.01044742, + "balance_loss_clip": 1.04541016, + "balance_loss_mlp": 1.02613306, + "epoch": 0.27643840557927013, + "flos": 18150007599360.0, + "grad_norm": 5.965945101771596, + "language_loss": 0.80585337, + "learning_rate": 3.3961992162520185e-06, + "loss": 0.82786059, + "num_input_tokens_seen": 49341370, + "router_z_loss_clip": 1.10693359, + "router_z_loss_mlp": 0.18615723, + "step": 2299, + "time_per_iteration": 2.8819756507873535 + }, + { + "auxiliary_loss_clip": 0.01164832, + "auxiliary_loss_mlp": 0.01048693, + "balance_loss_clip": 1.04890418, + "balance_loss_mlp": 1.02843976, + "epoch": 0.27655864846990924, + "flos": 24823933372800.0, + "grad_norm": 2.6415882158373623, + "language_loss": 0.71668375, + "learning_rate": 3.3956413661557156e-06, + "loss": 0.738819, + "num_input_tokens_seen": 49361545, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.20239258, + "step": 2300, + "time_per_iteration": 2.8399057388305664 + }, + { + "auxiliary_loss_clip": 0.0116339, + "auxiliary_loss_mlp": 0.01050035, + "balance_loss_clip": 1.04981184, + "balance_loss_mlp": 1.02795792, + "epoch": 0.2766788913605483, + "flos": 20266582464000.0, + "grad_norm": 2.6632978226332154, + "language_loss": 0.66669637, + "learning_rate": 3.3950833043417273e-06, + "loss": 0.68883061, + "num_input_tokens_seen": 49379690, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.22094727, + "step": 2301, + "time_per_iteration": 4.437417507171631 + }, + { + "auxiliary_loss_clip": 0.01165382, + "auxiliary_loss_mlp": 0.01052516, + "balance_loss_clip": 1.05186176, + "balance_loss_mlp": 1.0317142, + "epoch": 0.2767991342511874, + "flos": 21470272151040.0, + "grad_norm": 2.7539851931484867, + "language_loss": 0.7340337, + "learning_rate": 3.3945250308947105e-06, + "loss": 0.75621271, + "num_input_tokens_seen": 49395995, + "router_z_loss_clip": 1.13476562, + "router_z_loss_mlp": 0.20812988, + "step": 2302, + "time_per_iteration": 2.8647968769073486 + }, + { + "auxiliary_loss_clip": 0.01067872, + "auxiliary_loss_mlp": 0.0101477, + "balance_loss_clip": 1.02369869, + "balance_loss_mlp": 1.01217175, + "epoch": 0.2769193771418265, + "flos": 66002627571840.0, + "grad_norm": 1.2683946483016928, + "language_loss": 0.68358648, + "learning_rate": 3.3939665458993556e-06, + "loss": 0.70441282, + "num_input_tokens_seen": 49450415, + "router_z_loss_clip": 0.44140625, + "router_z_loss_mlp": 0.02600098, + "step": 2303, + "time_per_iteration": 3.2694649696350098 + }, + { + "auxiliary_loss_clip": 0.01163327, + "auxiliary_loss_mlp": 0.01050495, + "balance_loss_clip": 1.04909182, + "balance_loss_mlp": 1.03125501, + "epoch": 0.27703962003246557, + "flos": 20704441253760.0, + "grad_norm": 2.099554139768896, + "language_loss": 0.76828146, + "learning_rate": 3.3934078494403843e-06, + "loss": 0.7904197, + "num_input_tokens_seen": 49469990, + "router_z_loss_clip": 1.14257812, + "router_z_loss_mlp": 0.19238281, + "step": 2304, + "time_per_iteration": 2.7970306873321533 + }, + { + "auxiliary_loss_clip": 0.01168101, + "auxiliary_loss_mlp": 0.01048867, + "balance_loss_clip": 1.05121744, + "balance_loss_mlp": 1.02770722, + "epoch": 0.2771598629231047, + "flos": 22929897219840.0, + "grad_norm": 1.751406093753892, + "language_loss": 0.81512225, + "learning_rate": 3.3928489416025495e-06, + "loss": 0.83729196, + "num_input_tokens_seen": 49490835, + "router_z_loss_clip": 1.16894531, + "router_z_loss_mlp": 0.21166992, + "step": 2305, + "time_per_iteration": 2.780808687210083 + }, + { + "auxiliary_loss_clip": 0.01167433, + "auxiliary_loss_mlp": 0.01051071, + "balance_loss_clip": 1.04952312, + "balance_loss_mlp": 1.02992344, + "epoch": 0.27728010581374374, + "flos": 18369457741440.0, + "grad_norm": 2.2217221926167325, + "language_loss": 0.78729624, + "learning_rate": 3.392289822470638e-06, + "loss": 0.80948126, + "num_input_tokens_seen": 49508815, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.21142578, + "step": 2306, + "time_per_iteration": 2.7814595699310303 + }, + { + "auxiliary_loss_clip": 0.01165437, + "auxiliary_loss_mlp": 0.01051799, + "balance_loss_clip": 1.05114627, + "balance_loss_mlp": 1.03048468, + "epoch": 0.27740034870438285, + "flos": 19427637432960.0, + "grad_norm": 2.6538241373567133, + "language_loss": 0.76075268, + "learning_rate": 3.3917304921294674e-06, + "loss": 0.78292513, + "num_input_tokens_seen": 49526980, + "router_z_loss_clip": 1.14257812, + "router_z_loss_mlp": 0.21325684, + "step": 2307, + "time_per_iteration": 2.8004379272460938 + }, + { + "auxiliary_loss_clip": 0.01168448, + "auxiliary_loss_mlp": 0.01046847, + "balance_loss_clip": 1.05055773, + "balance_loss_mlp": 1.02703476, + "epoch": 0.27752059159502196, + "flos": 21614776565760.0, + "grad_norm": 3.0041455757316284, + "language_loss": 0.80558395, + "learning_rate": 3.3911709506638876e-06, + "loss": 0.82773691, + "num_input_tokens_seen": 49546290, + "router_z_loss_clip": 1.1796875, + "router_z_loss_mlp": 0.19824219, + "step": 2308, + "time_per_iteration": 2.8521101474761963 + }, + { + "auxiliary_loss_clip": 0.01167467, + "auxiliary_loss_mlp": 0.01047635, + "balance_loss_clip": 1.05042958, + "balance_loss_mlp": 1.02720284, + "epoch": 0.277640834485661, + "flos": 26608011016320.0, + "grad_norm": 3.2345611526660374, + "language_loss": 0.81602299, + "learning_rate": 3.390611198158781e-06, + "loss": 0.83817405, + "num_input_tokens_seen": 49564165, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.20446777, + "step": 2309, + "time_per_iteration": 2.8540241718292236 + }, + { + "auxiliary_loss_clip": 0.01165879, + "auxiliary_loss_mlp": 0.01046627, + "balance_loss_clip": 1.04978013, + "balance_loss_mlp": 1.02532411, + "epoch": 0.2777610773763001, + "flos": 19492814661120.0, + "grad_norm": 2.745516192953782, + "language_loss": 0.8989563, + "learning_rate": 3.3900512346990612e-06, + "loss": 0.92108142, + "num_input_tokens_seen": 49580155, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.21289062, + "step": 2310, + "time_per_iteration": 2.8290586471557617 + }, + { + "auxiliary_loss_clip": 0.01166027, + "auxiliary_loss_mlp": 0.01051067, + "balance_loss_clip": 1.04836273, + "balance_loss_mlp": 1.03043211, + "epoch": 0.27788132026693924, + "flos": 38290650001920.0, + "grad_norm": 1.854700454694756, + "language_loss": 0.65425205, + "learning_rate": 3.389491060369674e-06, + "loss": 0.67642307, + "num_input_tokens_seen": 49605830, + "router_z_loss_clip": 1.17578125, + "router_z_loss_mlp": 0.20654297, + "step": 2311, + "time_per_iteration": 2.9332282543182373 + }, + { + "auxiliary_loss_clip": 0.01162662, + "auxiliary_loss_mlp": 0.01040729, + "balance_loss_clip": 1.04957879, + "balance_loss_mlp": 1.0211308, + "epoch": 0.2780015631575783, + "flos": 22382546797440.0, + "grad_norm": 2.4523958442171208, + "language_loss": 0.8975755, + "learning_rate": 3.388930675255598e-06, + "loss": 0.91960937, + "num_input_tokens_seen": 49625680, + "router_z_loss_clip": 1.12988281, + "router_z_loss_mlp": 0.19580078, + "step": 2312, + "time_per_iteration": 2.7886173725128174 + }, + { + "auxiliary_loss_clip": 0.01162617, + "auxiliary_loss_mlp": 0.0105069, + "balance_loss_clip": 1.0481894, + "balance_loss_mlp": 1.02913713, + "epoch": 0.2781218060482174, + "flos": 12203200840320.0, + "grad_norm": 2.511490071532468, + "language_loss": 0.7901665, + "learning_rate": 3.388370079441843e-06, + "loss": 0.81229955, + "num_input_tokens_seen": 49641195, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.21533203, + "step": 2313, + "time_per_iteration": 2.8644096851348877 + }, + { + "auxiliary_loss_clip": 0.01163605, + "auxiliary_loss_mlp": 0.01059249, + "balance_loss_clip": 1.05046022, + "balance_loss_mlp": 1.03824449, + "epoch": 0.2782420489388565, + "flos": 18107632529280.0, + "grad_norm": 2.1076771648227384, + "language_loss": 0.92740762, + "learning_rate": 3.3878092730134505e-06, + "loss": 0.94963616, + "num_input_tokens_seen": 49659180, + "router_z_loss_clip": 1.13183594, + "router_z_loss_mlp": 0.21008301, + "step": 2314, + "time_per_iteration": 2.828862190246582 + }, + { + "auxiliary_loss_clip": 0.01165335, + "auxiliary_loss_mlp": 0.01054323, + "balance_loss_clip": 1.05020785, + "balance_loss_mlp": 1.03371167, + "epoch": 0.27836229182949557, + "flos": 18514752255360.0, + "grad_norm": 1.8405199710171674, + "language_loss": 0.80779231, + "learning_rate": 3.3872482560554947e-06, + "loss": 0.8299889, + "num_input_tokens_seen": 49677955, + "router_z_loss_clip": 1.15039062, + "router_z_loss_mlp": 0.20617676, + "step": 2315, + "time_per_iteration": 2.7796359062194824 + }, + { + "auxiliary_loss_clip": 0.01063659, + "auxiliary_loss_mlp": 0.01004876, + "balance_loss_clip": 1.02017856, + "balance_loss_mlp": 1.00222969, + "epoch": 0.2784825347201347, + "flos": 67079230940160.0, + "grad_norm": 0.9519580141043124, + "language_loss": 0.56871122, + "learning_rate": 3.386687028653082e-06, + "loss": 0.58939654, + "num_input_tokens_seen": 49740800, + "router_z_loss_clip": 0.43359375, + "router_z_loss_mlp": 0.02648926, + "step": 2316, + "time_per_iteration": 3.4169089794158936 + }, + { + "auxiliary_loss_clip": 0.01160799, + "auxiliary_loss_mlp": 0.01049938, + "balance_loss_clip": 1.04787695, + "balance_loss_mlp": 1.02845633, + "epoch": 0.2786027776107738, + "flos": 22631119891200.0, + "grad_norm": 1.8248025782720843, + "language_loss": 0.85401225, + "learning_rate": 3.386125590891349e-06, + "loss": 0.87611955, + "num_input_tokens_seen": 49757675, + "router_z_loss_clip": 1.12890625, + "router_z_loss_mlp": 0.21484375, + "step": 2317, + "time_per_iteration": 2.773153781890869 + }, + { + "auxiliary_loss_clip": 0.01161999, + "auxiliary_loss_mlp": 0.01043445, + "balance_loss_clip": 1.04999602, + "balance_loss_mlp": 1.02391839, + "epoch": 0.27872302050141284, + "flos": 15778826156160.0, + "grad_norm": 2.176960057762141, + "language_loss": 0.8318392, + "learning_rate": 3.3855639428554657e-06, + "loss": 0.8538937, + "num_input_tokens_seen": 49775205, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.19543457, + "step": 2318, + "time_per_iteration": 2.7323009967803955 + }, + { + "auxiliary_loss_clip": 0.01161758, + "auxiliary_loss_mlp": 0.0104757, + "balance_loss_clip": 1.0482614, + "balance_loss_mlp": 1.02680349, + "epoch": 0.27884326339205195, + "flos": 22126970551680.0, + "grad_norm": 1.8456468506016928, + "language_loss": 0.80576324, + "learning_rate": 3.385002084630635e-06, + "loss": 0.82785648, + "num_input_tokens_seen": 49794175, + "router_z_loss_clip": 1.13574219, + "router_z_loss_mlp": 0.2076416, + "step": 2319, + "time_per_iteration": 2.8071982860565186 + }, + { + "auxiliary_loss_clip": 0.01161044, + "auxiliary_loss_mlp": 0.01050043, + "balance_loss_clip": 1.04915643, + "balance_loss_mlp": 1.02931213, + "epoch": 0.278963506282691, + "flos": 20558715776640.0, + "grad_norm": 3.882713841393461, + "language_loss": 0.85123944, + "learning_rate": 3.384440016302088e-06, + "loss": 0.87335032, + "num_input_tokens_seen": 49812850, + "router_z_loss_clip": 1.11914062, + "router_z_loss_mlp": 0.20739746, + "step": 2320, + "time_per_iteration": 2.7636067867279053 + }, + { + "auxiliary_loss_clip": 0.01162734, + "auxiliary_loss_mlp": 0.01051796, + "balance_loss_clip": 1.04948139, + "balance_loss_mlp": 1.029814, + "epoch": 0.2790837491733301, + "flos": 21942928241280.0, + "grad_norm": 2.02680895285977, + "language_loss": 0.62867975, + "learning_rate": 3.3838777379550923e-06, + "loss": 0.65082502, + "num_input_tokens_seen": 49832295, + "router_z_loss_clip": 1.13183594, + "router_z_loss_mlp": 0.21972656, + "step": 2321, + "time_per_iteration": 2.7889420986175537 + }, + { + "auxiliary_loss_clip": 0.01160235, + "auxiliary_loss_mlp": 0.01046483, + "balance_loss_clip": 1.0490942, + "balance_loss_mlp": 1.0257045, + "epoch": 0.27920399206396923, + "flos": 26286790665600.0, + "grad_norm": 2.454703252166854, + "language_loss": 0.78645861, + "learning_rate": 3.383315249674944e-06, + "loss": 0.8085258, + "num_input_tokens_seen": 49850860, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.20788574, + "step": 2322, + "time_per_iteration": 2.832435369491577 + }, + { + "auxiliary_loss_clip": 0.01163071, + "auxiliary_loss_mlp": 0.01048261, + "balance_loss_clip": 1.04983127, + "balance_loss_mlp": 1.02751827, + "epoch": 0.2793242349546083, + "flos": 25400981364480.0, + "grad_norm": 2.1600869739114965, + "language_loss": 0.85771269, + "learning_rate": 3.3827525515469715e-06, + "loss": 0.87982601, + "num_input_tokens_seen": 49865765, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.20739746, + "step": 2323, + "time_per_iteration": 2.8418703079223633 + }, + { + "auxiliary_loss_clip": 0.0116329, + "auxiliary_loss_mlp": 0.01049544, + "balance_loss_clip": 1.04900122, + "balance_loss_mlp": 1.02840829, + "epoch": 0.2794444778452474, + "flos": 20850346298880.0, + "grad_norm": 2.4432661649095224, + "language_loss": 0.70907938, + "learning_rate": 3.3821896436565367e-06, + "loss": 0.73120773, + "num_input_tokens_seen": 49885425, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.21142578, + "step": 2324, + "time_per_iteration": 2.7793304920196533 + }, + { + "auxiliary_loss_clip": 0.01163678, + "auxiliary_loss_mlp": 0.01045112, + "balance_loss_clip": 1.05177987, + "balance_loss_mlp": 1.02270019, + "epoch": 0.2795647207358865, + "flos": 21576244250880.0, + "grad_norm": 2.4762629293661336, + "language_loss": 0.70581317, + "learning_rate": 3.381626526089032e-06, + "loss": 0.7279011, + "num_input_tokens_seen": 49904990, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.22399902, + "step": 2325, + "time_per_iteration": 2.7566823959350586 + }, + { + "auxiliary_loss_clip": 0.01162177, + "auxiliary_loss_mlp": 0.01055216, + "balance_loss_clip": 1.04848385, + "balance_loss_mlp": 1.03309047, + "epoch": 0.27968496362652556, + "flos": 21471744608640.0, + "grad_norm": 2.1003047752771025, + "language_loss": 0.79191917, + "learning_rate": 3.3810631989298815e-06, + "loss": 0.81409311, + "num_input_tokens_seen": 49924600, + "router_z_loss_clip": 1.13769531, + "router_z_loss_mlp": 0.22119141, + "step": 2326, + "time_per_iteration": 2.7829091548919678 + }, + { + "auxiliary_loss_clip": 0.01169849, + "auxiliary_loss_mlp": 0.01060684, + "balance_loss_clip": 1.05281472, + "balance_loss_mlp": 1.03806984, + "epoch": 0.2798052065171647, + "flos": 23258695340160.0, + "grad_norm": 13.594926274424576, + "language_loss": 0.84304738, + "learning_rate": 3.3804996622645423e-06, + "loss": 0.86535269, + "num_input_tokens_seen": 49942600, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.22595215, + "step": 2327, + "time_per_iteration": 3.670245885848999 + }, + { + "auxiliary_loss_clip": 0.01159839, + "auxiliary_loss_mlp": 0.01052226, + "balance_loss_clip": 1.04770088, + "balance_loss_mlp": 1.03172183, + "epoch": 0.2799254494078038, + "flos": 21539328048000.0, + "grad_norm": 2.0351746012593033, + "language_loss": 0.89327204, + "learning_rate": 3.3799359161785015e-06, + "loss": 0.91539276, + "num_input_tokens_seen": 49962250, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.20507812, + "step": 2328, + "time_per_iteration": 2.8277475833892822 + }, + { + "auxiliary_loss_clip": 0.01164245, + "auxiliary_loss_mlp": 0.01049974, + "balance_loss_clip": 1.04882145, + "balance_loss_mlp": 1.02865922, + "epoch": 0.28004569229844284, + "flos": 26393912000640.0, + "grad_norm": 1.7782078701750321, + "language_loss": 0.8565979, + "learning_rate": 3.3793719607572798e-06, + "loss": 0.87874013, + "num_input_tokens_seen": 49983215, + "router_z_loss_clip": 1.15332031, + "router_z_loss_mlp": 0.21325684, + "step": 2329, + "time_per_iteration": 2.8552629947662354 + }, + { + "auxiliary_loss_clip": 0.01164204, + "auxiliary_loss_mlp": 0.01050883, + "balance_loss_clip": 1.05064368, + "balance_loss_mlp": 1.02954483, + "epoch": 0.28016593518908195, + "flos": 33547676584320.0, + "grad_norm": 1.9664033710054214, + "language_loss": 0.77433777, + "learning_rate": 3.378807796086428e-06, + "loss": 0.7964887, + "num_input_tokens_seen": 50006075, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.21337891, + "step": 2330, + "time_per_iteration": 4.572862386703491 + }, + { + "auxiliary_loss_clip": 0.01168777, + "auxiliary_loss_mlp": 0.01046973, + "balance_loss_clip": 1.05273438, + "balance_loss_mlp": 1.02743411, + "epoch": 0.28028617807972106, + "flos": 15340823712000.0, + "grad_norm": 2.2920994326368063, + "language_loss": 0.76834357, + "learning_rate": 3.37824342225153e-06, + "loss": 0.790501, + "num_input_tokens_seen": 50022495, + "router_z_loss_clip": 1.16015625, + "router_z_loss_mlp": 0.19543457, + "step": 2331, + "time_per_iteration": 2.7959799766540527 + }, + { + "auxiliary_loss_clip": 0.01161658, + "auxiliary_loss_mlp": 0.01044954, + "balance_loss_clip": 1.05122316, + "balance_loss_mlp": 1.02560616, + "epoch": 0.2804064209703601, + "flos": 25520277409920.0, + "grad_norm": 2.17575433882527, + "language_loss": 0.78009009, + "learning_rate": 3.3776788393382006e-06, + "loss": 0.80215621, + "num_input_tokens_seen": 50041975, + "router_z_loss_clip": 1.10498047, + "router_z_loss_mlp": 0.19348145, + "step": 2332, + "time_per_iteration": 2.815453052520752 + }, + { + "auxiliary_loss_clip": 0.01163469, + "auxiliary_loss_mlp": 0.01052182, + "balance_loss_clip": 1.0485785, + "balance_loss_mlp": 1.02987838, + "epoch": 0.2805266638609992, + "flos": 29351766280320.0, + "grad_norm": 3.2612960064779477, + "language_loss": 0.76844424, + "learning_rate": 3.3771140474320872e-06, + "loss": 0.79060078, + "num_input_tokens_seen": 50061925, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.22314453, + "step": 2333, + "time_per_iteration": 2.8958687782287598 + }, + { + "auxiliary_loss_clip": 0.01162738, + "auxiliary_loss_mlp": 0.01048478, + "balance_loss_clip": 1.05026984, + "balance_loss_mlp": 1.02661514, + "epoch": 0.28064690675163834, + "flos": 21463735875840.0, + "grad_norm": 2.2868700132272304, + "language_loss": 0.79987413, + "learning_rate": 3.3765490466188664e-06, + "loss": 0.82198626, + "num_input_tokens_seen": 50079325, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.21850586, + "step": 2334, + "time_per_iteration": 2.890676975250244 + }, + { + "auxiliary_loss_clip": 0.01161965, + "auxiliary_loss_mlp": 0.01053173, + "balance_loss_clip": 1.04859948, + "balance_loss_mlp": 1.03059459, + "epoch": 0.2807671496422774, + "flos": 20995640812800.0, + "grad_norm": 5.2584005434907075, + "language_loss": 0.74228007, + "learning_rate": 3.3759838369842508e-06, + "loss": 0.76443148, + "num_input_tokens_seen": 50097400, + "router_z_loss_clip": 1.13476562, + "router_z_loss_mlp": 0.22583008, + "step": 2335, + "time_per_iteration": 2.7848429679870605 + }, + { + "auxiliary_loss_clip": 0.0116333, + "auxiliary_loss_mlp": 0.01054266, + "balance_loss_clip": 1.04790044, + "balance_loss_mlp": 1.03273642, + "epoch": 0.2808873925329165, + "flos": 21506577822720.0, + "grad_norm": 2.960191256088889, + "language_loss": 0.73460758, + "learning_rate": 3.375418418613981e-06, + "loss": 0.75678355, + "num_input_tokens_seen": 50116425, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.21520996, + "step": 2336, + "time_per_iteration": 2.7903003692626953 + }, + { + "auxiliary_loss_clip": 0.01162584, + "auxiliary_loss_mlp": 0.01046308, + "balance_loss_clip": 1.04791164, + "balance_loss_mlp": 1.02551818, + "epoch": 0.28100763542355556, + "flos": 16070815814400.0, + "grad_norm": 4.194905630495661, + "language_loss": 0.83719933, + "learning_rate": 3.374852791593831e-06, + "loss": 0.85928822, + "num_input_tokens_seen": 50132625, + "router_z_loss_clip": 1.14648438, + "router_z_loss_mlp": 0.20788574, + "step": 2337, + "time_per_iteration": 2.7484967708587646 + }, + { + "auxiliary_loss_clip": 0.01168516, + "auxiliary_loss_mlp": 0.01048508, + "balance_loss_clip": 1.05013227, + "balance_loss_mlp": 1.02702606, + "epoch": 0.28112787831419467, + "flos": 19062605468160.0, + "grad_norm": 3.414627305834382, + "language_loss": 0.54032052, + "learning_rate": 3.374286956009605e-06, + "loss": 0.56249076, + "num_input_tokens_seen": 50151190, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.21472168, + "step": 2338, + "time_per_iteration": 2.7533864974975586 + }, + { + "auxiliary_loss_clip": 0.01165777, + "auxiliary_loss_mlp": 0.01047667, + "balance_loss_clip": 1.05326152, + "balance_loss_mlp": 1.02637601, + "epoch": 0.2812481212048338, + "flos": 12823629482880.0, + "grad_norm": 2.1765183442221723, + "language_loss": 0.75165313, + "learning_rate": 3.3737209119471405e-06, + "loss": 0.77378762, + "num_input_tokens_seen": 50167700, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.21289062, + "step": 2339, + "time_per_iteration": 2.779247760772705 + }, + { + "auxiliary_loss_clip": 0.01162972, + "auxiliary_loss_mlp": 0.01056026, + "balance_loss_clip": 1.0483005, + "balance_loss_mlp": 1.03432977, + "epoch": 0.28136836409547283, + "flos": 15633064765440.0, + "grad_norm": 3.481531599887071, + "language_loss": 0.63697809, + "learning_rate": 3.373154659492306e-06, + "loss": 0.65916806, + "num_input_tokens_seen": 50185840, + "router_z_loss_clip": 1.14648438, + "router_z_loss_mlp": 0.21679688, + "step": 2340, + "time_per_iteration": 2.809563159942627 + }, + { + "auxiliary_loss_clip": 0.01159426, + "auxiliary_loss_mlp": 0.01042994, + "balance_loss_clip": 1.04787874, + "balance_loss_mlp": 1.0226686, + "epoch": 0.28148860698611194, + "flos": 19933726106880.0, + "grad_norm": 2.2667164428185833, + "language_loss": 0.85657793, + "learning_rate": 3.3725881987310016e-06, + "loss": 0.87860215, + "num_input_tokens_seen": 50203375, + "router_z_loss_clip": 1.11523438, + "router_z_loss_mlp": 0.203125, + "step": 2341, + "time_per_iteration": 2.809852361679077 + }, + { + "auxiliary_loss_clip": 0.0115744, + "auxiliary_loss_mlp": 0.01048237, + "balance_loss_clip": 1.04502249, + "balance_loss_mlp": 1.02822208, + "epoch": 0.28160884987675106, + "flos": 17457219008640.0, + "grad_norm": 2.1147385294702516, + "language_loss": 0.87622702, + "learning_rate": 3.372021529749159e-06, + "loss": 0.89828384, + "num_input_tokens_seen": 50222435, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.20031738, + "step": 2342, + "time_per_iteration": 2.7951622009277344 + }, + { + "auxiliary_loss_clip": 0.01160677, + "auxiliary_loss_mlp": 0.01052437, + "balance_loss_clip": 1.04932117, + "balance_loss_mlp": 1.03171849, + "epoch": 0.2817290927673901, + "flos": 16834743290880.0, + "grad_norm": 2.3048993706075738, + "language_loss": 0.92232656, + "learning_rate": 3.3714546526327405e-06, + "loss": 0.94445771, + "num_input_tokens_seen": 50240435, + "router_z_loss_clip": 1.11523438, + "router_z_loss_mlp": 0.20727539, + "step": 2343, + "time_per_iteration": 2.8885996341705322 + }, + { + "auxiliary_loss_clip": 0.01162087, + "auxiliary_loss_mlp": 0.01045069, + "balance_loss_clip": 1.04740644, + "balance_loss_mlp": 1.02501798, + "epoch": 0.2818493356580292, + "flos": 15414081500160.0, + "grad_norm": 2.47132339333745, + "language_loss": 0.88108176, + "learning_rate": 3.3708875674677423e-06, + "loss": 0.9031533, + "num_input_tokens_seen": 50258410, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.20056152, + "step": 2344, + "time_per_iteration": 2.7981905937194824 + }, + { + "auxiliary_loss_clip": 0.01163059, + "auxiliary_loss_mlp": 0.01047322, + "balance_loss_clip": 1.04871416, + "balance_loss_mlp": 1.02721167, + "epoch": 0.28196957854866833, + "flos": 20412451595520.0, + "grad_norm": 2.1414327494412935, + "language_loss": 0.83821875, + "learning_rate": 3.37032027434019e-06, + "loss": 0.86032259, + "num_input_tokens_seen": 50277930, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.20092773, + "step": 2345, + "time_per_iteration": 2.8109986782073975 + }, + { + "auxiliary_loss_clip": 0.0116871, + "auxiliary_loss_mlp": 0.01056685, + "balance_loss_clip": 1.05002677, + "balance_loss_mlp": 1.03438044, + "epoch": 0.2820898214393074, + "flos": 19973120348160.0, + "grad_norm": 2.1079035495729745, + "language_loss": 0.82994497, + "learning_rate": 3.369752773336141e-06, + "loss": 0.8521989, + "num_input_tokens_seen": 50297410, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.22302246, + "step": 2346, + "time_per_iteration": 2.872889518737793 + }, + { + "auxiliary_loss_clip": 0.01165073, + "auxiliary_loss_mlp": 0.01050055, + "balance_loss_clip": 1.05086625, + "balance_loss_mlp": 1.02705944, + "epoch": 0.2822100643299465, + "flos": 22528308188160.0, + "grad_norm": 1.747179142187967, + "language_loss": 0.78683579, + "learning_rate": 3.3691850645416864e-06, + "loss": 0.80898702, + "num_input_tokens_seen": 50317120, + "router_z_loss_clip": 1.14257812, + "router_z_loss_mlp": 0.2298584, + "step": 2347, + "time_per_iteration": 2.846203565597534 + }, + { + "auxiliary_loss_clip": 0.01162201, + "auxiliary_loss_mlp": 0.01056841, + "balance_loss_clip": 1.04904628, + "balance_loss_mlp": 1.03587234, + "epoch": 0.2823303072205856, + "flos": 11546682007680.0, + "grad_norm": 2.2474775892567855, + "language_loss": 0.83286357, + "learning_rate": 3.368617148042945e-06, + "loss": 0.85505402, + "num_input_tokens_seen": 50334790, + "router_z_loss_clip": 1.13183594, + "router_z_loss_mlp": 0.20959473, + "step": 2348, + "time_per_iteration": 2.815697431564331 + }, + { + "auxiliary_loss_clip": 0.01165288, + "auxiliary_loss_mlp": 0.01049986, + "balance_loss_clip": 1.0510906, + "balance_loss_mlp": 1.02793252, + "epoch": 0.28245055011122466, + "flos": 18259894281600.0, + "grad_norm": 3.672151806131867, + "language_loss": 0.8464995, + "learning_rate": 3.368049023926071e-06, + "loss": 0.86865228, + "num_input_tokens_seen": 50353785, + "router_z_loss_clip": 1.14160156, + "router_z_loss_mlp": 0.22058105, + "step": 2349, + "time_per_iteration": 2.80198335647583 + }, + { + "auxiliary_loss_clip": 0.01154009, + "auxiliary_loss_mlp": 0.01046517, + "balance_loss_clip": 1.0470891, + "balance_loss_mlp": 1.02770543, + "epoch": 0.2825707930018638, + "flos": 24608110504320.0, + "grad_norm": 2.3369256233645896, + "language_loss": 0.83324939, + "learning_rate": 3.3674806922772476e-06, + "loss": 0.85525465, + "num_input_tokens_seen": 50374670, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.18811035, + "step": 2350, + "time_per_iteration": 2.8177387714385986 + }, + { + "auxiliary_loss_clip": 0.01163688, + "auxiliary_loss_mlp": 0.01055522, + "balance_loss_clip": 1.05068207, + "balance_loss_mlp": 1.03355217, + "epoch": 0.28269103589250283, + "flos": 25226994862080.0, + "grad_norm": 10.570994800993109, + "language_loss": 0.75047874, + "learning_rate": 3.3669121531826904e-06, + "loss": 0.77267081, + "num_input_tokens_seen": 50395650, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.21948242, + "step": 2351, + "time_per_iteration": 2.9014089107513428 + }, + { + "auxiliary_loss_clip": 0.0116426, + "auxiliary_loss_mlp": 0.0105069, + "balance_loss_clip": 1.0508337, + "balance_loss_mlp": 1.02906573, + "epoch": 0.28281127878314194, + "flos": 19281552819840.0, + "grad_norm": 4.181351702915689, + "language_loss": 0.82914329, + "learning_rate": 3.366343406728647e-06, + "loss": 0.85129273, + "num_input_tokens_seen": 50415100, + "router_z_loss_clip": 1.13476562, + "router_z_loss_mlp": 0.21618652, + "step": 2352, + "time_per_iteration": 2.848524332046509 + }, + { + "auxiliary_loss_clip": 0.01160878, + "auxiliary_loss_mlp": 0.01054089, + "balance_loss_clip": 1.04712558, + "balance_loss_mlp": 1.03300118, + "epoch": 0.28293152167378105, + "flos": 23878405710720.0, + "grad_norm": 2.3135974231803687, + "language_loss": 0.68939251, + "learning_rate": 3.3657744530013946e-06, + "loss": 0.71154213, + "num_input_tokens_seen": 50434335, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.21118164, + "step": 2353, + "time_per_iteration": 2.8033199310302734 + }, + { + "auxiliary_loss_clip": 0.01163477, + "auxiliary_loss_mlp": 0.01050056, + "balance_loss_clip": 1.0489552, + "balance_loss_mlp": 1.02973104, + "epoch": 0.2830517645644201, + "flos": 43866965928960.0, + "grad_norm": 2.5566168191213783, + "language_loss": 0.71809661, + "learning_rate": 3.3652052920872437e-06, + "loss": 0.74023187, + "num_input_tokens_seen": 50457200, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.20324707, + "step": 2354, + "time_per_iteration": 3.009580612182617 + }, + { + "auxiliary_loss_clip": 0.01166546, + "auxiliary_loss_mlp": 0.01059362, + "balance_loss_clip": 1.05210733, + "balance_loss_mlp": 1.03698635, + "epoch": 0.2831720074550592, + "flos": 26651750803200.0, + "grad_norm": 2.244346411623719, + "language_loss": 0.85875756, + "learning_rate": 3.3646359240725355e-06, + "loss": 0.88101661, + "num_input_tokens_seen": 50476390, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.22375488, + "step": 2355, + "time_per_iteration": 2.8311002254486084 + }, + { + "auxiliary_loss_clip": 0.01165851, + "auxiliary_loss_mlp": 0.01050446, + "balance_loss_clip": 1.05098999, + "balance_loss_mlp": 1.02951336, + "epoch": 0.2832922503456983, + "flos": 31029979564800.0, + "grad_norm": 3.0879410171693147, + "language_loss": 0.67643678, + "learning_rate": 3.364066349043643e-06, + "loss": 0.69859982, + "num_input_tokens_seen": 50497595, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.20922852, + "step": 2356, + "time_per_iteration": 4.500417947769165 + }, + { + "auxiliary_loss_clip": 0.01159886, + "auxiliary_loss_mlp": 0.01045515, + "balance_loss_clip": 1.049052, + "balance_loss_mlp": 1.02629888, + "epoch": 0.2834124932363374, + "flos": 20405699838720.0, + "grad_norm": 2.0450174007082356, + "language_loss": 0.82369709, + "learning_rate": 3.363496567086969e-06, + "loss": 0.84575111, + "num_input_tokens_seen": 50514690, + "router_z_loss_clip": 1.10839844, + "router_z_loss_mlp": 0.19226074, + "step": 2357, + "time_per_iteration": 2.8330421447753906 + }, + { + "auxiliary_loss_clip": 0.01162172, + "auxiliary_loss_mlp": 0.01049145, + "balance_loss_clip": 1.04912579, + "balance_loss_mlp": 1.02767491, + "epoch": 0.2835327361269765, + "flos": 39384848056320.0, + "grad_norm": 2.3353364154620233, + "language_loss": 0.75512421, + "learning_rate": 3.3629265782889506e-06, + "loss": 0.7772373, + "num_input_tokens_seen": 50536515, + "router_z_loss_clip": 1.12988281, + "router_z_loss_mlp": 0.21472168, + "step": 2358, + "time_per_iteration": 2.9135260581970215 + }, + { + "auxiliary_loss_clip": 0.01164857, + "auxiliary_loss_mlp": 0.0105256, + "balance_loss_clip": 1.04871392, + "balance_loss_mlp": 1.03283095, + "epoch": 0.2836529790176156, + "flos": 30261598801920.0, + "grad_norm": 2.0852214805415494, + "language_loss": 0.71970415, + "learning_rate": 3.362356382736054e-06, + "loss": 0.74187827, + "num_input_tokens_seen": 50557120, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.1973877, + "step": 2359, + "time_per_iteration": 4.488087892532349 + }, + { + "auxiliary_loss_clip": 0.01157937, + "auxiliary_loss_mlp": 0.01057502, + "balance_loss_clip": 1.04755569, + "balance_loss_mlp": 1.03708196, + "epoch": 0.28377322190825466, + "flos": 12677796264960.0, + "grad_norm": 3.9030429016402755, + "language_loss": 0.90839863, + "learning_rate": 3.361785980514777e-06, + "loss": 0.93055308, + "num_input_tokens_seen": 50573320, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.20410156, + "step": 2360, + "time_per_iteration": 2.918626070022583 + }, + { + "auxiliary_loss_clip": 0.01162642, + "auxiliary_loss_mlp": 0.01058269, + "balance_loss_clip": 1.04889786, + "balance_loss_mlp": 1.0366559, + "epoch": 0.28389346479889377, + "flos": 18296666830080.0, + "grad_norm": 2.7863944251873938, + "language_loss": 0.76818341, + "learning_rate": 3.361215371711649e-06, + "loss": 0.79039252, + "num_input_tokens_seen": 50592415, + "router_z_loss_clip": 1.13769531, + "router_z_loss_mlp": 0.21618652, + "step": 2361, + "time_per_iteration": 2.8169713020324707 + }, + { + "auxiliary_loss_clip": 0.01154211, + "auxiliary_loss_mlp": 0.01047149, + "balance_loss_clip": 1.04627442, + "balance_loss_mlp": 1.02627587, + "epoch": 0.2840137076895329, + "flos": 20406992728320.0, + "grad_norm": 2.0536127894990845, + "language_loss": 0.83842909, + "learning_rate": 3.3606445564132326e-06, + "loss": 0.86044264, + "num_input_tokens_seen": 50609710, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.20874023, + "step": 2362, + "time_per_iteration": 2.8694236278533936 + }, + { + "auxiliary_loss_clip": 0.0116111, + "auxiliary_loss_mlp": 0.01048802, + "balance_loss_clip": 1.04872835, + "balance_loss_mlp": 1.02753544, + "epoch": 0.28413395058017193, + "flos": 20048030161920.0, + "grad_norm": 2.163858165512162, + "language_loss": 0.82272309, + "learning_rate": 3.360073534706118e-06, + "loss": 0.84482223, + "num_input_tokens_seen": 50626865, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.21276855, + "step": 2363, + "time_per_iteration": 2.779758930206299 + }, + { + "auxiliary_loss_clip": 0.01165946, + "auxiliary_loss_mlp": 0.01056357, + "balance_loss_clip": 1.05067229, + "balance_loss_mlp": 1.03464925, + "epoch": 0.28425419347081105, + "flos": 37663613256960.0, + "grad_norm": 2.3419924488812334, + "language_loss": 0.76051348, + "learning_rate": 3.35950230667693e-06, + "loss": 0.78273654, + "num_input_tokens_seen": 50648560, + "router_z_loss_clip": 1.15332031, + "router_z_loss_mlp": 0.21716309, + "step": 2364, + "time_per_iteration": 2.923630952835083 + }, + { + "auxiliary_loss_clip": 0.01164133, + "auxiliary_loss_mlp": 0.01052059, + "balance_loss_clip": 1.05097258, + "balance_loss_mlp": 1.02990961, + "epoch": 0.28437443636145016, + "flos": 13845072539520.0, + "grad_norm": 2.5720456115055312, + "language_loss": 0.8579849, + "learning_rate": 3.358930872412323e-06, + "loss": 0.88014686, + "num_input_tokens_seen": 50665725, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.22131348, + "step": 2365, + "time_per_iteration": 2.8085687160491943 + }, + { + "auxiliary_loss_clip": 0.01161819, + "auxiliary_loss_mlp": 0.01053683, + "balance_loss_clip": 1.05049491, + "balance_loss_mlp": 1.03298879, + "epoch": 0.2844946792520892, + "flos": 22747794243840.0, + "grad_norm": 1.7299776830150289, + "language_loss": 0.80995321, + "learning_rate": 3.3583592319989825e-06, + "loss": 0.83210826, + "num_input_tokens_seen": 50685095, + "router_z_loss_clip": 1.11425781, + "router_z_loss_mlp": 0.20690918, + "step": 2366, + "time_per_iteration": 2.824124336242676 + }, + { + "auxiliary_loss_clip": 0.01171597, + "auxiliary_loss_mlp": 0.01058285, + "balance_loss_clip": 1.05416369, + "balance_loss_mlp": 1.03619599, + "epoch": 0.2846149221427283, + "flos": 32415987709440.0, + "grad_norm": 4.799942072901274, + "language_loss": 0.68980157, + "learning_rate": 3.357787385523627e-06, + "loss": 0.71210039, + "num_input_tokens_seen": 50706500, + "router_z_loss_clip": 1.17382812, + "router_z_loss_mlp": 0.2208252, + "step": 2367, + "time_per_iteration": 2.839233160018921 + }, + { + "auxiliary_loss_clip": 0.0116715, + "auxiliary_loss_mlp": 0.01066303, + "balance_loss_clip": 1.04969645, + "balance_loss_mlp": 1.04358125, + "epoch": 0.2847351650333674, + "flos": 28475976873600.0, + "grad_norm": 2.004725253671857, + "language_loss": 0.82485247, + "learning_rate": 3.3572153330730048e-06, + "loss": 0.84718692, + "num_input_tokens_seen": 50727595, + "router_z_loss_clip": 1.17382812, + "router_z_loss_mlp": 0.22692871, + "step": 2368, + "time_per_iteration": 2.8366777896881104 + }, + { + "auxiliary_loss_clip": 0.010699, + "auxiliary_loss_mlp": 0.01037198, + "balance_loss_clip": 1.02817762, + "balance_loss_mlp": 1.03456306, + "epoch": 0.2848554079240065, + "flos": 55753399704960.0, + "grad_norm": 0.837664715095127, + "language_loss": 0.64722955, + "learning_rate": 3.3566430747338956e-06, + "loss": 0.66830057, + "num_input_tokens_seen": 50782800, + "router_z_loss_clip": 0.41796875, + "router_z_loss_mlp": 0.02636719, + "step": 2369, + "time_per_iteration": 3.218353509902954 + }, + { + "auxiliary_loss_clip": 0.01162455, + "auxiliary_loss_mlp": 0.01047653, + "balance_loss_clip": 1.04865289, + "balance_loss_mlp": 1.02718496, + "epoch": 0.2849756508146456, + "flos": 11836875985920.0, + "grad_norm": 3.025239496273534, + "language_loss": 0.86749309, + "learning_rate": 3.35607061059311e-06, + "loss": 0.8895942, + "num_input_tokens_seen": 50797730, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.20458984, + "step": 2370, + "time_per_iteration": 2.8549046516418457 + }, + { + "auxiliary_loss_clip": 0.0115861, + "auxiliary_loss_mlp": 0.01047885, + "balance_loss_clip": 1.04861736, + "balance_loss_mlp": 1.0273335, + "epoch": 0.28509589370528465, + "flos": 25155209531520.0, + "grad_norm": 2.4816667392545724, + "language_loss": 0.74997836, + "learning_rate": 3.3554979407374917e-06, + "loss": 0.77204335, + "num_input_tokens_seen": 50819840, + "router_z_loss_clip": 1.09960938, + "router_z_loss_mlp": 0.20556641, + "step": 2371, + "time_per_iteration": 2.9447765350341797 + }, + { + "auxiliary_loss_clip": 0.01159083, + "auxiliary_loss_mlp": 0.01046406, + "balance_loss_clip": 1.04861307, + "balance_loss_mlp": 1.02597404, + "epoch": 0.28521613659592376, + "flos": 19974808287360.0, + "grad_norm": 1.7812840293620906, + "language_loss": 0.73387456, + "learning_rate": 3.3549250652539134e-06, + "loss": 0.75592947, + "num_input_tokens_seen": 50838935, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.20446777, + "step": 2372, + "time_per_iteration": 2.9183826446533203 + }, + { + "auxiliary_loss_clip": 0.01165049, + "auxiliary_loss_mlp": 0.0105957, + "balance_loss_clip": 1.04811287, + "balance_loss_mlp": 1.03587115, + "epoch": 0.2853363794865629, + "flos": 23367971491200.0, + "grad_norm": 3.1411848880820177, + "language_loss": 0.81769377, + "learning_rate": 3.3543519842292794e-06, + "loss": 0.83993995, + "num_input_tokens_seen": 50858590, + "router_z_loss_clip": 1.16992188, + "router_z_loss_mlp": 0.23693848, + "step": 2373, + "time_per_iteration": 2.937264919281006 + }, + { + "auxiliary_loss_clip": 0.01159226, + "auxiliary_loss_mlp": 0.01053081, + "balance_loss_clip": 1.04891288, + "balance_loss_mlp": 1.03208804, + "epoch": 0.28545662237720193, + "flos": 19861940776320.0, + "grad_norm": 2.916396293094937, + "language_loss": 0.83738196, + "learning_rate": 3.353778697750527e-06, + "loss": 0.85950506, + "num_input_tokens_seen": 50876995, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.21008301, + "step": 2374, + "time_per_iteration": 2.8803765773773193 + }, + { + "auxiliary_loss_clip": 0.01160895, + "auxiliary_loss_mlp": 0.0105044, + "balance_loss_clip": 1.04946017, + "balance_loss_mlp": 1.02994752, + "epoch": 0.28557686526784104, + "flos": 23879016241920.0, + "grad_norm": 2.185632510719986, + "language_loss": 0.89716721, + "learning_rate": 3.353205205904622e-06, + "loss": 0.91928053, + "num_input_tokens_seen": 50896105, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.20495605, + "step": 2375, + "time_per_iteration": 2.8426198959350586 + }, + { + "auxiliary_loss_clip": 0.01168619, + "auxiliary_loss_mlp": 0.0105911, + "balance_loss_clip": 1.05264211, + "balance_loss_mlp": 1.03674591, + "epoch": 0.28569710815848015, + "flos": 44890384233600.0, + "grad_norm": 2.1429347823199127, + "language_loss": 0.72155905, + "learning_rate": 3.3526315087785637e-06, + "loss": 0.74383628, + "num_input_tokens_seen": 50917220, + "router_z_loss_clip": 1.16113281, + "router_z_loss_mlp": 0.22351074, + "step": 2376, + "time_per_iteration": 3.028784990310669 + }, + { + "auxiliary_loss_clip": 0.01161633, + "auxiliary_loss_mlp": 0.01051395, + "balance_loss_clip": 1.05033433, + "balance_loss_mlp": 1.03120065, + "epoch": 0.2858173510491192, + "flos": 26829759628800.0, + "grad_norm": 1.8908683954471623, + "language_loss": 0.80974424, + "learning_rate": 3.3520576064593805e-06, + "loss": 0.83187449, + "num_input_tokens_seen": 50937175, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.2019043, + "step": 2377, + "time_per_iteration": 2.879324436187744 + }, + { + "auxiliary_loss_clip": 0.01164458, + "auxiliary_loss_mlp": 0.01050864, + "balance_loss_clip": 1.05069256, + "balance_loss_mlp": 1.0303961, + "epoch": 0.2859375939397583, + "flos": 23148916398720.0, + "grad_norm": 1.6810942052901292, + "language_loss": 0.82176375, + "learning_rate": 3.3514834990341337e-06, + "loss": 0.84391701, + "num_input_tokens_seen": 50957500, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.20471191, + "step": 2378, + "time_per_iteration": 2.8019819259643555 + }, + { + "auxiliary_loss_clip": 0.01160276, + "auxiliary_loss_mlp": 0.01046101, + "balance_loss_clip": 1.05081677, + "balance_loss_mlp": 1.02624106, + "epoch": 0.2860578368303974, + "flos": 12129799397760.0, + "grad_norm": 2.648030056364485, + "language_loss": 0.92479062, + "learning_rate": 3.3509091865899144e-06, + "loss": 0.94685435, + "num_input_tokens_seen": 50972690, + "router_z_loss_clip": 1.09570312, + "router_z_loss_mlp": 0.19836426, + "step": 2379, + "time_per_iteration": 2.78969669342041 + }, + { + "auxiliary_loss_clip": 0.01165258, + "auxiliary_loss_mlp": 0.01048502, + "balance_loss_clip": 1.05259538, + "balance_loss_mlp": 1.02775931, + "epoch": 0.2861780797210365, + "flos": 19938035738880.0, + "grad_norm": 2.318031666703901, + "language_loss": 0.70557189, + "learning_rate": 3.350334669213846e-06, + "loss": 0.72770947, + "num_input_tokens_seen": 50990095, + "router_z_loss_clip": 1.12792969, + "router_z_loss_mlp": 0.20739746, + "step": 2380, + "time_per_iteration": 2.7585134506225586 + }, + { + "auxiliary_loss_clip": 0.01162331, + "auxiliary_loss_mlp": 0.01041687, + "balance_loss_clip": 1.05245662, + "balance_loss_mlp": 1.02144539, + "epoch": 0.2862983226116756, + "flos": 27563127609600.0, + "grad_norm": 3.272648123633749, + "language_loss": 0.75845242, + "learning_rate": 3.3497599469930816e-06, + "loss": 0.78049254, + "num_input_tokens_seen": 51008305, + "router_z_loss_clip": 1.09960938, + "router_z_loss_mlp": 0.20239258, + "step": 2381, + "time_per_iteration": 2.888105869293213 + }, + { + "auxiliary_loss_clip": 0.01162164, + "auxiliary_loss_mlp": 0.01045833, + "balance_loss_clip": 1.0491004, + "balance_loss_mlp": 1.02597284, + "epoch": 0.28641856550231465, + "flos": 22053964158720.0, + "grad_norm": 2.4394513694060715, + "language_loss": 0.8339923, + "learning_rate": 3.349185020014807e-06, + "loss": 0.85607231, + "num_input_tokens_seen": 51025570, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.1986084, + "step": 2382, + "time_per_iteration": 2.8619132041931152 + }, + { + "auxiliary_loss_clip": 0.01167891, + "auxiliary_loss_mlp": 0.01046676, + "balance_loss_clip": 1.0522449, + "balance_loss_mlp": 1.02563596, + "epoch": 0.28653880839295376, + "flos": 22378775869440.0, + "grad_norm": 2.2299391746167725, + "language_loss": 0.7437892, + "learning_rate": 3.348609888366237e-06, + "loss": 0.76593488, + "num_input_tokens_seen": 51044585, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.21057129, + "step": 2383, + "time_per_iteration": 2.7846972942352295 + }, + { + "auxiliary_loss_clip": 0.01161122, + "auxiliary_loss_mlp": 0.01046396, + "balance_loss_clip": 1.05087805, + "balance_loss_mlp": 1.02596378, + "epoch": 0.28665905128359287, + "flos": 23367971491200.0, + "grad_norm": 6.579375035649397, + "language_loss": 0.63035518, + "learning_rate": 3.348034552134619e-06, + "loss": 0.65243036, + "num_input_tokens_seen": 51063990, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.2043457, + "step": 2384, + "time_per_iteration": 3.6513512134552 + }, + { + "auxiliary_loss_clip": 0.01161828, + "auxiliary_loss_mlp": 0.01050678, + "balance_loss_clip": 1.04983664, + "balance_loss_mlp": 1.02917266, + "epoch": 0.2867792941742319, + "flos": 20881695893760.0, + "grad_norm": 2.2919902564221712, + "language_loss": 0.84675783, + "learning_rate": 3.3474590114072316e-06, + "loss": 0.86888289, + "num_input_tokens_seen": 51081990, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.21508789, + "step": 2385, + "time_per_iteration": 3.74157452583313 + }, + { + "auxiliary_loss_clip": 0.01168934, + "auxiliary_loss_mlp": 0.01052485, + "balance_loss_clip": 1.05407763, + "balance_loss_mlp": 1.03041911, + "epoch": 0.28689953706487104, + "flos": 20664005518080.0, + "grad_norm": 2.2238621619151577, + "language_loss": 0.83158064, + "learning_rate": 3.3468832662713836e-06, + "loss": 0.85379481, + "num_input_tokens_seen": 51100235, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.22045898, + "step": 2386, + "time_per_iteration": 2.8949215412139893 + }, + { + "auxiliary_loss_clip": 0.01163197, + "auxiliary_loss_mlp": 0.01046717, + "balance_loss_clip": 1.05251098, + "balance_loss_mlp": 1.0276556, + "epoch": 0.28701977995551015, + "flos": 12675533708160.0, + "grad_norm": 2.2111395286622897, + "language_loss": 0.84042555, + "learning_rate": 3.346307316814415e-06, + "loss": 0.86252469, + "num_input_tokens_seen": 51115405, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.1907959, + "step": 2387, + "time_per_iteration": 3.6708271503448486 + }, + { + "auxiliary_loss_clip": 0.01166124, + "auxiliary_loss_mlp": 0.01055266, + "balance_loss_clip": 1.05225086, + "balance_loss_mlp": 1.03308153, + "epoch": 0.2871400228461492, + "flos": 21252366293760.0, + "grad_norm": 2.6908121583185607, + "language_loss": 0.75547588, + "learning_rate": 3.3457311631236965e-06, + "loss": 0.77768981, + "num_input_tokens_seen": 51136390, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.22167969, + "step": 2388, + "time_per_iteration": 3.6505484580993652 + }, + { + "auxiliary_loss_clip": 0.01164191, + "auxiliary_loss_mlp": 0.01051649, + "balance_loss_clip": 1.05012906, + "balance_loss_mlp": 1.029881, + "epoch": 0.2872602657367883, + "flos": 25119262995840.0, + "grad_norm": 2.3219713378015983, + "language_loss": 0.84702474, + "learning_rate": 3.345154805286631e-06, + "loss": 0.86918312, + "num_input_tokens_seen": 51156650, + "router_z_loss_clip": 1.14160156, + "router_z_loss_mlp": 0.21777344, + "step": 2389, + "time_per_iteration": 2.8246076107025146 + }, + { + "auxiliary_loss_clip": 0.01163884, + "auxiliary_loss_mlp": 0.01053081, + "balance_loss_clip": 1.05053782, + "balance_loss_mlp": 1.03230286, + "epoch": 0.2873805086274274, + "flos": 16646606830080.0, + "grad_norm": 3.3084658660926283, + "language_loss": 0.76617801, + "learning_rate": 3.344578243390651e-06, + "loss": 0.7883476, + "num_input_tokens_seen": 51172210, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.2076416, + "step": 2390, + "time_per_iteration": 2.7695860862731934 + }, + { + "auxiliary_loss_clip": 0.01160696, + "auxiliary_loss_mlp": 0.0105079, + "balance_loss_clip": 1.05034614, + "balance_loss_mlp": 1.02970183, + "epoch": 0.2875007515180665, + "flos": 17420123237760.0, + "grad_norm": 2.52110613690536, + "language_loss": 0.78933799, + "learning_rate": 3.3440014775232206e-06, + "loss": 0.81145287, + "num_input_tokens_seen": 51190265, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.21081543, + "step": 2391, + "time_per_iteration": 2.8446216583251953 + }, + { + "auxiliary_loss_clip": 0.01160267, + "auxiliary_loss_mlp": 0.01053702, + "balance_loss_clip": 1.05140424, + "balance_loss_mlp": 1.0319345, + "epoch": 0.2876209944087056, + "flos": 23434190213760.0, + "grad_norm": 3.4908591678730785, + "language_loss": 0.71270943, + "learning_rate": 3.343424507771834e-06, + "loss": 0.7348491, + "num_input_tokens_seen": 51208475, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.21765137, + "step": 2392, + "time_per_iteration": 2.892949104309082 + }, + { + "auxiliary_loss_clip": 0.01160529, + "auxiliary_loss_mlp": 0.01057311, + "balance_loss_clip": 1.05104768, + "balance_loss_mlp": 1.036044, + "epoch": 0.2877412372993447, + "flos": 13735509079680.0, + "grad_norm": 2.3386868830170573, + "language_loss": 0.86370313, + "learning_rate": 3.342847334224018e-06, + "loss": 0.88588154, + "num_input_tokens_seen": 51225875, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.21264648, + "step": 2393, + "time_per_iteration": 2.817596673965454 + }, + { + "auxiliary_loss_clip": 0.01081065, + "auxiliary_loss_mlp": 0.01006035, + "balance_loss_clip": 1.03856516, + "balance_loss_mlp": 1.00340009, + "epoch": 0.28786148018998375, + "flos": 58079695104000.0, + "grad_norm": 0.9611248076810783, + "language_loss": 0.62376046, + "learning_rate": 3.342269956967329e-06, + "loss": 0.64463151, + "num_input_tokens_seen": 51287780, + "router_z_loss_clip": 0.42578125, + "router_z_loss_mlp": 0.02636719, + "step": 2394, + "time_per_iteration": 3.416278839111328 + }, + { + "auxiliary_loss_clip": 0.01169597, + "auxiliary_loss_mlp": 0.01050881, + "balance_loss_clip": 1.05031312, + "balance_loss_mlp": 1.02782559, + "epoch": 0.28798172308062286, + "flos": 23435052140160.0, + "grad_norm": 4.182940327516579, + "language_loss": 0.71614492, + "learning_rate": 3.341692376089355e-06, + "loss": 0.73834974, + "num_input_tokens_seen": 51303335, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.23034668, + "step": 2395, + "time_per_iteration": 2.8213088512420654 + }, + { + "auxiliary_loss_clip": 0.01163293, + "auxiliary_loss_mlp": 0.01056297, + "balance_loss_clip": 1.05120301, + "balance_loss_mlp": 1.03456533, + "epoch": 0.288101965971262, + "flos": 25110033200640.0, + "grad_norm": 4.271224777033802, + "language_loss": 0.84744459, + "learning_rate": 3.3411145916777146e-06, + "loss": 0.86964047, + "num_input_tokens_seen": 51317495, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.21740723, + "step": 2396, + "time_per_iteration": 2.8268661499023438 + }, + { + "auxiliary_loss_clip": 0.01165627, + "auxiliary_loss_mlp": 0.01056325, + "balance_loss_clip": 1.05215025, + "balance_loss_mlp": 1.03571391, + "epoch": 0.28822220886190103, + "flos": 16252559654400.0, + "grad_norm": 4.102959458540964, + "language_loss": 0.91372222, + "learning_rate": 3.3405366038200566e-06, + "loss": 0.93594182, + "num_input_tokens_seen": 51336430, + "router_z_loss_clip": 1.13476562, + "router_z_loss_mlp": 0.20617676, + "step": 2397, + "time_per_iteration": 2.8284380435943604 + }, + { + "auxiliary_loss_clip": 0.01174615, + "auxiliary_loss_mlp": 0.01061412, + "balance_loss_clip": 1.05700207, + "balance_loss_mlp": 1.03989482, + "epoch": 0.28834245175254014, + "flos": 24535642815360.0, + "grad_norm": 2.58469291103458, + "language_loss": 0.85066044, + "learning_rate": 3.3399584126040617e-06, + "loss": 0.87302071, + "num_input_tokens_seen": 51355930, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.21520996, + "step": 2398, + "time_per_iteration": 2.8074114322662354 + }, + { + "auxiliary_loss_clip": 0.01162395, + "auxiliary_loss_mlp": 0.01049401, + "balance_loss_clip": 1.04940832, + "balance_loss_mlp": 1.02863479, + "epoch": 0.2884626946431792, + "flos": 24571445696640.0, + "grad_norm": 2.261646420877118, + "language_loss": 0.91036904, + "learning_rate": 3.339380018117441e-06, + "loss": 0.93248701, + "num_input_tokens_seen": 51376765, + "router_z_loss_clip": 1.12890625, + "router_z_loss_mlp": 0.20788574, + "step": 2399, + "time_per_iteration": 2.7950263023376465 + }, + { + "auxiliary_loss_clip": 0.01160703, + "auxiliary_loss_mlp": 0.01051455, + "balance_loss_clip": 1.0505904, + "balance_loss_mlp": 1.0314517, + "epoch": 0.2885829375338183, + "flos": 16544657053440.0, + "grad_norm": 2.79981469691504, + "language_loss": 0.7763207, + "learning_rate": 3.3388014204479366e-06, + "loss": 0.79844224, + "num_input_tokens_seen": 51394570, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.20007324, + "step": 2400, + "time_per_iteration": 2.740173578262329 + }, + { + "auxiliary_loss_clip": 0.01169808, + "auxiliary_loss_mlp": 0.01052291, + "balance_loss_clip": 1.05467343, + "balance_loss_mlp": 1.03169155, + "epoch": 0.2887031804244574, + "flos": 24061226958720.0, + "grad_norm": 2.987780663323479, + "language_loss": 0.91670454, + "learning_rate": 3.338222619683321e-06, + "loss": 0.9389255, + "num_input_tokens_seen": 51414535, + "router_z_loss_clip": 1.15332031, + "router_z_loss_mlp": 0.20593262, + "step": 2401, + "time_per_iteration": 2.7702064514160156 + }, + { + "auxiliary_loss_clip": 0.01173197, + "auxiliary_loss_mlp": 0.0105127, + "balance_loss_clip": 1.0546813, + "balance_loss_mlp": 1.02747619, + "epoch": 0.2888234233150965, + "flos": 23330696152320.0, + "grad_norm": 5.772050532324918, + "language_loss": 0.7364558, + "learning_rate": 3.337643615911398e-06, + "loss": 0.75870049, + "num_input_tokens_seen": 51434160, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.23815918, + "step": 2402, + "time_per_iteration": 2.8530304431915283 + }, + { + "auxiliary_loss_clip": 0.0116564, + "auxiliary_loss_mlp": 0.01055406, + "balance_loss_clip": 1.05152893, + "balance_loss_mlp": 1.03373349, + "epoch": 0.2889436662057356, + "flos": 22272767856000.0, + "grad_norm": 3.261575769748471, + "language_loss": 0.79149008, + "learning_rate": 3.3370644092200026e-06, + "loss": 0.81370056, + "num_input_tokens_seen": 51451435, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.2166748, + "step": 2403, + "time_per_iteration": 2.7567877769470215 + }, + { + "auxiliary_loss_clip": 0.01165418, + "auxiliary_loss_mlp": 0.01050889, + "balance_loss_clip": 1.0546999, + "balance_loss_mlp": 1.02930045, + "epoch": 0.2890639090963747, + "flos": 21616931381760.0, + "grad_norm": 4.5650601942225935, + "language_loss": 0.78622079, + "learning_rate": 3.3364849996969985e-06, + "loss": 0.80838382, + "num_input_tokens_seen": 51471455, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.21594238, + "step": 2404, + "time_per_iteration": 2.786649703979492 + }, + { + "auxiliary_loss_clip": 0.01164537, + "auxiliary_loss_mlp": 0.01057869, + "balance_loss_clip": 1.05118358, + "balance_loss_mlp": 1.03724575, + "epoch": 0.28918415198701375, + "flos": 28585540333440.0, + "grad_norm": 37.601415927961646, + "language_loss": 0.85383677, + "learning_rate": 3.335905387430283e-06, + "loss": 0.87606078, + "num_input_tokens_seen": 51492890, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.20654297, + "step": 2405, + "time_per_iteration": 2.8550028800964355 + }, + { + "auxiliary_loss_clip": 0.01161847, + "auxiliary_loss_mlp": 0.01055676, + "balance_loss_clip": 1.04921126, + "balance_loss_mlp": 1.03277647, + "epoch": 0.28930439487765286, + "flos": 21944688007680.0, + "grad_norm": 4.088275038451351, + "language_loss": 0.83105612, + "learning_rate": 3.335325572507782e-06, + "loss": 0.85323143, + "num_input_tokens_seen": 51513390, + "router_z_loss_clip": 1.12744141, + "router_z_loss_mlp": 0.22888184, + "step": 2406, + "time_per_iteration": 2.814511299133301 + }, + { + "auxiliary_loss_clip": 0.01163013, + "auxiliary_loss_mlp": 0.01047629, + "balance_loss_clip": 1.05273902, + "balance_loss_mlp": 1.02673125, + "epoch": 0.28942463776829197, + "flos": 19281911955840.0, + "grad_norm": 2.141758769965927, + "language_loss": 0.73949295, + "learning_rate": 3.3347455550174537e-06, + "loss": 0.7615993, + "num_input_tokens_seen": 51532730, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.2088623, + "step": 2407, + "time_per_iteration": 2.767486572265625 + }, + { + "auxiliary_loss_clip": 0.01164873, + "auxiliary_loss_mlp": 0.01046736, + "balance_loss_clip": 1.05051243, + "balance_loss_mlp": 1.02446735, + "epoch": 0.289544880658931, + "flos": 14645700737280.0, + "grad_norm": 2.646950425293465, + "language_loss": 0.67911196, + "learning_rate": 3.3341653350472864e-06, + "loss": 0.70122808, + "num_input_tokens_seen": 51549560, + "router_z_loss_clip": 1.14160156, + "router_z_loss_mlp": 0.22277832, + "step": 2408, + "time_per_iteration": 2.7335264682769775 + }, + { + "auxiliary_loss_clip": 0.0117842, + "auxiliary_loss_mlp": 0.01062809, + "balance_loss_clip": 1.05514741, + "balance_loss_mlp": 1.03754842, + "epoch": 0.28966512354957014, + "flos": 28621881918720.0, + "grad_norm": 4.530994946472679, + "language_loss": 0.6953671, + "learning_rate": 3.333584912685298e-06, + "loss": 0.7177794, + "num_input_tokens_seen": 51568180, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.25292969, + "step": 2409, + "time_per_iteration": 2.8397765159606934 + }, + { + "auxiliary_loss_clip": 0.01065537, + "auxiliary_loss_mlp": 0.01003041, + "balance_loss_clip": 1.0238533, + "balance_loss_mlp": 1.00015593, + "epoch": 0.28978536644020925, + "flos": 64711784511360.0, + "grad_norm": 0.8914851026067411, + "language_loss": 0.55507219, + "learning_rate": 3.3330042880195385e-06, + "loss": 0.57575786, + "num_input_tokens_seen": 51622530, + "router_z_loss_clip": 0.41796875, + "router_z_loss_mlp": 0.02880859, + "step": 2410, + "time_per_iteration": 3.3278257846832275 + }, + { + "auxiliary_loss_clip": 0.01161553, + "auxiliary_loss_mlp": 0.01051768, + "balance_loss_clip": 1.0509305, + "balance_loss_mlp": 1.03159809, + "epoch": 0.2899056093308483, + "flos": 18624638937600.0, + "grad_norm": 3.036849501714377, + "language_loss": 0.78472638, + "learning_rate": 3.3324234611380888e-06, + "loss": 0.80685961, + "num_input_tokens_seen": 51641260, + "router_z_loss_clip": 1.10791016, + "router_z_loss_mlp": 0.20178223, + "step": 2411, + "time_per_iteration": 2.782633066177368 + }, + { + "auxiliary_loss_clip": 0.01164845, + "auxiliary_loss_mlp": 0.01050862, + "balance_loss_clip": 1.05507708, + "balance_loss_mlp": 1.03104973, + "epoch": 0.2900258522214874, + "flos": 22893735202560.0, + "grad_norm": 2.9300277513157327, + "language_loss": 0.81938243, + "learning_rate": 3.3318424321290596e-06, + "loss": 0.8415395, + "num_input_tokens_seen": 51660975, + "router_z_loss_clip": 1.09765625, + "router_z_loss_mlp": 0.19812012, + "step": 2412, + "time_per_iteration": 2.8315324783325195 + }, + { + "auxiliary_loss_clip": 0.01065541, + "auxiliary_loss_mlp": 0.01001152, + "balance_loss_clip": 1.02417159, + "balance_loss_mlp": 0.99842167, + "epoch": 0.2901460951121265, + "flos": 71106036013440.0, + "grad_norm": 0.8387117033618697, + "language_loss": 0.59908479, + "learning_rate": 3.3312612010805917e-06, + "loss": 0.61975169, + "num_input_tokens_seen": 51720550, + "router_z_loss_clip": 0.4140625, + "router_z_loss_mlp": 0.02734375, + "step": 2413, + "time_per_iteration": 4.178373098373413 + }, + { + "auxiliary_loss_clip": 0.01168495, + "auxiliary_loss_mlp": 0.01053583, + "balance_loss_clip": 1.05343223, + "balance_loss_mlp": 1.03245914, + "epoch": 0.2902663380027656, + "flos": 32160986081280.0, + "grad_norm": 3.5814248327073144, + "language_loss": 0.70309246, + "learning_rate": 3.330679768080858e-06, + "loss": 0.72531319, + "num_input_tokens_seen": 51744435, + "router_z_loss_clip": 1.15039062, + "router_z_loss_mlp": 0.21105957, + "step": 2414, + "time_per_iteration": 3.7118844985961914 + }, + { + "auxiliary_loss_clip": 0.01162855, + "auxiliary_loss_mlp": 0.01062003, + "balance_loss_clip": 1.05218077, + "balance_loss_mlp": 1.04196382, + "epoch": 0.2903865808934047, + "flos": 29351658539520.0, + "grad_norm": 2.24615834965022, + "language_loss": 0.83766806, + "learning_rate": 3.3300981332180627e-06, + "loss": 0.85991663, + "num_input_tokens_seen": 51763640, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.20043945, + "step": 2415, + "time_per_iteration": 2.8372232913970947 + }, + { + "auxiliary_loss_clip": 0.01164117, + "auxiliary_loss_mlp": 0.01054326, + "balance_loss_clip": 1.05156577, + "balance_loss_mlp": 1.0325228, + "epoch": 0.29050682378404374, + "flos": 17089026647040.0, + "grad_norm": 2.5068486429829293, + "language_loss": 0.80079782, + "learning_rate": 3.3295162965804373e-06, + "loss": 0.82298231, + "num_input_tokens_seen": 51782135, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.21789551, + "step": 2416, + "time_per_iteration": 3.6110646724700928 + }, + { + "auxiliary_loss_clip": 0.01158735, + "auxiliary_loss_mlp": 0.0105168, + "balance_loss_clip": 1.05020761, + "balance_loss_mlp": 1.03204644, + "epoch": 0.29062706667468285, + "flos": 17858233422720.0, + "grad_norm": 2.876318502189502, + "language_loss": 0.7850101, + "learning_rate": 3.328934258256247e-06, + "loss": 0.80711418, + "num_input_tokens_seen": 51800200, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.19616699, + "step": 2417, + "time_per_iteration": 3.6171324253082275 + }, + { + "auxiliary_loss_clip": 0.01163737, + "auxiliary_loss_mlp": 0.01050772, + "balance_loss_clip": 1.05044758, + "balance_loss_mlp": 1.0296483, + "epoch": 0.29074730956532197, + "flos": 24279815174400.0, + "grad_norm": 2.3619824912010383, + "language_loss": 0.67128336, + "learning_rate": 3.3283520183337856e-06, + "loss": 0.69342846, + "num_input_tokens_seen": 51819905, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.21142578, + "step": 2418, + "time_per_iteration": 2.8628714084625244 + }, + { + "auxiliary_loss_clip": 0.01161602, + "auxiliary_loss_mlp": 0.01047837, + "balance_loss_clip": 1.05285156, + "balance_loss_mlp": 1.02813208, + "epoch": 0.290867552455961, + "flos": 22340961826560.0, + "grad_norm": 3.686483823246278, + "language_loss": 0.69112217, + "learning_rate": 3.3277695769013797e-06, + "loss": 0.7132166, + "num_input_tokens_seen": 51839350, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.19702148, + "step": 2419, + "time_per_iteration": 2.7474539279937744 + }, + { + "auxiliary_loss_clip": 0.0116169, + "auxiliary_loss_mlp": 0.01065111, + "balance_loss_clip": 1.05038631, + "balance_loss_mlp": 1.04415441, + "epoch": 0.29098779534660013, + "flos": 23186155824000.0, + "grad_norm": 2.6711336073374166, + "language_loss": 0.77634168, + "learning_rate": 3.327186934047385e-06, + "loss": 0.79860973, + "num_input_tokens_seen": 51858045, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.20935059, + "step": 2420, + "time_per_iteration": 2.7913334369659424 + }, + { + "auxiliary_loss_clip": 0.01163012, + "auxiliary_loss_mlp": 0.01050812, + "balance_loss_clip": 1.05220008, + "balance_loss_mlp": 1.0299263, + "epoch": 0.29110803823723924, + "flos": 15304194817920.0, + "grad_norm": 3.226002447864789, + "language_loss": 0.65754819, + "learning_rate": 3.3266040898601877e-06, + "loss": 0.67968637, + "num_input_tokens_seen": 51875880, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.20898438, + "step": 2421, + "time_per_iteration": 2.821685552597046 + }, + { + "auxiliary_loss_clip": 0.01163131, + "auxiliary_loss_mlp": 0.01061123, + "balance_loss_clip": 1.05457282, + "balance_loss_mlp": 1.04096437, + "epoch": 0.2912282811278783, + "flos": 22595352923520.0, + "grad_norm": 2.15996004178, + "language_loss": 0.77877247, + "learning_rate": 3.3260210444282045e-06, + "loss": 0.80101502, + "num_input_tokens_seen": 51893835, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.20141602, + "step": 2422, + "time_per_iteration": 2.8291282653808594 + }, + { + "auxiliary_loss_clip": 0.01159241, + "auxiliary_loss_mlp": 0.01043854, + "balance_loss_clip": 1.04973674, + "balance_loss_mlp": 1.02329075, + "epoch": 0.2913485240185174, + "flos": 24497900599680.0, + "grad_norm": 2.2486197597661994, + "language_loss": 0.72918969, + "learning_rate": 3.325437797839883e-06, + "loss": 0.7512207, + "num_input_tokens_seen": 51912205, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.20568848, + "step": 2423, + "time_per_iteration": 2.7990915775299072 + }, + { + "auxiliary_loss_clip": 0.01167699, + "auxiliary_loss_mlp": 0.01046468, + "balance_loss_clip": 1.05442858, + "balance_loss_mlp": 1.02591622, + "epoch": 0.2914687669091565, + "flos": 17931024334080.0, + "grad_norm": 3.1885673798200838, + "language_loss": 0.74951589, + "learning_rate": 3.3248543501837015e-06, + "loss": 0.77165747, + "num_input_tokens_seen": 51929410, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.20556641, + "step": 2424, + "time_per_iteration": 2.807889223098755 + }, + { + "auxiliary_loss_clip": 0.01162825, + "auxiliary_loss_mlp": 0.01054233, + "balance_loss_clip": 1.05321872, + "balance_loss_mlp": 1.03399181, + "epoch": 0.2915890097997956, + "flos": 22529313768960.0, + "grad_norm": 1.9515566870457077, + "language_loss": 0.77053642, + "learning_rate": 3.3242707015481684e-06, + "loss": 0.79270697, + "num_input_tokens_seen": 51949345, + "router_z_loss_clip": 1.09521484, + "router_z_loss_mlp": 0.20263672, + "step": 2425, + "time_per_iteration": 2.990375518798828 + }, + { + "auxiliary_loss_clip": 0.01160735, + "auxiliary_loss_mlp": 0.01046338, + "balance_loss_clip": 1.05119276, + "balance_loss_mlp": 1.0273602, + "epoch": 0.2917092526904347, + "flos": 13845216193920.0, + "grad_norm": 2.459950643189074, + "language_loss": 0.81091034, + "learning_rate": 3.323686852021823e-06, + "loss": 0.83298111, + "num_input_tokens_seen": 51966855, + "router_z_loss_clip": 1.09570312, + "router_z_loss_mlp": 0.18969727, + "step": 2426, + "time_per_iteration": 2.863750696182251 + }, + { + "auxiliary_loss_clip": 0.01160243, + "auxiliary_loss_mlp": 0.01046642, + "balance_loss_clip": 1.05091512, + "balance_loss_mlp": 1.02736604, + "epoch": 0.2918294955810738, + "flos": 22674859678080.0, + "grad_norm": 2.1560666167642, + "language_loss": 0.7995373, + "learning_rate": 3.323102801693235e-06, + "loss": 0.82160616, + "num_input_tokens_seen": 51985620, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.19287109, + "step": 2427, + "time_per_iteration": 2.818253993988037 + }, + { + "auxiliary_loss_clip": 0.01165113, + "auxiliary_loss_mlp": 0.01055401, + "balance_loss_clip": 1.05169272, + "balance_loss_mlp": 1.03469419, + "epoch": 0.29194973847171285, + "flos": 23438284364160.0, + "grad_norm": 2.6618659223403243, + "language_loss": 0.80502045, + "learning_rate": 3.322518550651003e-06, + "loss": 0.82722569, + "num_input_tokens_seen": 52004930, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.20703125, + "step": 2428, + "time_per_iteration": 2.8217828273773193 + }, + { + "auxiliary_loss_clip": 0.01164156, + "auxiliary_loss_mlp": 0.01056257, + "balance_loss_clip": 1.05226827, + "balance_loss_mlp": 1.0358243, + "epoch": 0.29206998136235196, + "flos": 21909064694400.0, + "grad_norm": 3.6561465129843818, + "language_loss": 0.81094229, + "learning_rate": 3.3219340989837586e-06, + "loss": 0.83314645, + "num_input_tokens_seen": 52024920, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.2043457, + "step": 2429, + "time_per_iteration": 2.791708469390869 + }, + { + "auxiliary_loss_clip": 0.01163382, + "auxiliary_loss_mlp": 0.0104434, + "balance_loss_clip": 1.05222774, + "balance_loss_mlp": 1.02529061, + "epoch": 0.292190224252991, + "flos": 23215925220480.0, + "grad_norm": 1.911342246974181, + "language_loss": 0.80397272, + "learning_rate": 3.3213494467801625e-06, + "loss": 0.82604992, + "num_input_tokens_seen": 52044095, + "router_z_loss_clip": 1.11132812, + "router_z_loss_mlp": 0.19042969, + "step": 2430, + "time_per_iteration": 2.8084630966186523 + }, + { + "auxiliary_loss_clip": 0.01163056, + "auxiliary_loss_mlp": 0.01048209, + "balance_loss_clip": 1.05052209, + "balance_loss_mlp": 1.02768099, + "epoch": 0.2923104671436301, + "flos": 20740818752640.0, + "grad_norm": 2.3992627575765777, + "language_loss": 0.71246278, + "learning_rate": 3.3207645941289063e-06, + "loss": 0.73457545, + "num_input_tokens_seen": 52062440, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.20532227, + "step": 2431, + "time_per_iteration": 2.9530293941497803 + }, + { + "auxiliary_loss_clip": 0.01161281, + "auxiliary_loss_mlp": 0.01051154, + "balance_loss_clip": 1.04953241, + "balance_loss_mlp": 1.03159189, + "epoch": 0.29243071003426924, + "flos": 35809114999680.0, + "grad_norm": 2.125601159904304, + "language_loss": 0.80236477, + "learning_rate": 3.320179541118711e-06, + "loss": 0.82448912, + "num_input_tokens_seen": 52084940, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.19567871, + "step": 2432, + "time_per_iteration": 2.9349160194396973 + }, + { + "auxiliary_loss_clip": 0.01064884, + "auxiliary_loss_mlp": 0.01011564, + "balance_loss_clip": 1.02361584, + "balance_loss_mlp": 1.00871503, + "epoch": 0.2925509529249083, + "flos": 58081598524800.0, + "grad_norm": 0.9948448327000361, + "language_loss": 0.60277212, + "learning_rate": 3.3195942878383293e-06, + "loss": 0.62353659, + "num_input_tokens_seen": 52141040, + "router_z_loss_clip": 0.4140625, + "router_z_loss_mlp": 0.02844238, + "step": 2433, + "time_per_iteration": 3.449693441390991 + }, + { + "auxiliary_loss_clip": 0.01169341, + "auxiliary_loss_mlp": 0.01045683, + "balance_loss_clip": 1.05306959, + "balance_loss_mlp": 1.02471399, + "epoch": 0.2926711958155474, + "flos": 21397122103680.0, + "grad_norm": 2.33140646574313, + "language_loss": 0.78041863, + "learning_rate": 3.319008834376543e-06, + "loss": 0.80256891, + "num_input_tokens_seen": 52160730, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.2097168, + "step": 2434, + "time_per_iteration": 2.793564558029175 + }, + { + "auxiliary_loss_clip": 0.01165577, + "auxiliary_loss_mlp": 0.01048126, + "balance_loss_clip": 1.05091119, + "balance_loss_mlp": 1.02570271, + "epoch": 0.2927914387061865, + "flos": 23185796688000.0, + "grad_norm": 3.1559758685562698, + "language_loss": 0.88625211, + "learning_rate": 3.3184231808221654e-06, + "loss": 0.90838909, + "num_input_tokens_seen": 52175055, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.22436523, + "step": 2435, + "time_per_iteration": 2.815814971923828 + }, + { + "auxiliary_loss_clip": 0.01164839, + "auxiliary_loss_mlp": 0.01047153, + "balance_loss_clip": 1.05251908, + "balance_loss_mlp": 1.02452672, + "epoch": 0.29291168159682557, + "flos": 22455553190400.0, + "grad_norm": 2.8174990427585658, + "language_loss": 0.63110894, + "learning_rate": 3.3178373272640394e-06, + "loss": 0.65322888, + "num_input_tokens_seen": 52194150, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.22644043, + "step": 2436, + "time_per_iteration": 2.802737236022949 + }, + { + "auxiliary_loss_clip": 0.01158122, + "auxiliary_loss_mlp": 0.0104638, + "balance_loss_clip": 1.05020821, + "balance_loss_mlp": 1.02708042, + "epoch": 0.2930319244874647, + "flos": 21170632896000.0, + "grad_norm": 3.028765168428729, + "language_loss": 0.85836053, + "learning_rate": 3.3172512737910387e-06, + "loss": 0.88040555, + "num_input_tokens_seen": 52211660, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.19311523, + "step": 2437, + "time_per_iteration": 2.7881858348846436 + }, + { + "auxiliary_loss_clip": 0.0116114, + "auxiliary_loss_mlp": 0.01042554, + "balance_loss_clip": 1.05174136, + "balance_loss_mlp": 1.02176356, + "epoch": 0.2931521673781038, + "flos": 31357843931520.0, + "grad_norm": 2.573654822235149, + "language_loss": 0.88545769, + "learning_rate": 3.3166650204920674e-06, + "loss": 0.90749466, + "num_input_tokens_seen": 52232830, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.20788574, + "step": 2438, + "time_per_iteration": 2.8425180912017822 + }, + { + "auxiliary_loss_clip": 0.01165798, + "auxiliary_loss_mlp": 0.01055616, + "balance_loss_clip": 1.05232549, + "balance_loss_mlp": 1.03436077, + "epoch": 0.29327241026874284, + "flos": 24200990778240.0, + "grad_norm": 2.647196530111255, + "language_loss": 0.81586254, + "learning_rate": 3.316078567456059e-06, + "loss": 0.83807671, + "num_input_tokens_seen": 52250670, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.21264648, + "step": 2439, + "time_per_iteration": 2.7829253673553467 + }, + { + "auxiliary_loss_clip": 0.01166453, + "auxiliary_loss_mlp": 0.01049282, + "balance_loss_clip": 1.05431771, + "balance_loss_mlp": 1.02780032, + "epoch": 0.29339265315938196, + "flos": 24242611662720.0, + "grad_norm": 3.632372069179374, + "language_loss": 0.76018411, + "learning_rate": 3.3154919147719786e-06, + "loss": 0.78234148, + "num_input_tokens_seen": 52271685, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.21472168, + "step": 2440, + "time_per_iteration": 2.781290054321289 + }, + { + "auxiliary_loss_clip": 0.01160438, + "auxiliary_loss_mlp": 0.01052757, + "balance_loss_clip": 1.05030346, + "balance_loss_mlp": 1.03157306, + "epoch": 0.29351289605002107, + "flos": 16946641134720.0, + "grad_norm": 2.578531411046639, + "language_loss": 0.87054408, + "learning_rate": 3.31490506252882e-06, + "loss": 0.89267606, + "num_input_tokens_seen": 52291065, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.21203613, + "step": 2441, + "time_per_iteration": 2.737992286682129 + }, + { + "auxiliary_loss_clip": 0.01158311, + "auxiliary_loss_mlp": 0.01041147, + "balance_loss_clip": 1.05134249, + "balance_loss_mlp": 1.02215743, + "epoch": 0.2936331389406601, + "flos": 19829082810240.0, + "grad_norm": 1.939923923221693, + "language_loss": 0.84123355, + "learning_rate": 3.31431801081561e-06, + "loss": 0.86322808, + "num_input_tokens_seen": 52310000, + "router_z_loss_clip": 1.06835938, + "router_z_loss_mlp": 0.18994141, + "step": 2442, + "time_per_iteration": 3.6549174785614014 + }, + { + "auxiliary_loss_clip": 0.01063177, + "auxiliary_loss_mlp": 0.01000239, + "balance_loss_clip": 1.02246332, + "balance_loss_mlp": 0.99749738, + "epoch": 0.29375338183129923, + "flos": 71416844398080.0, + "grad_norm": 0.9006920011921677, + "language_loss": 0.67945337, + "learning_rate": 3.313730759721402e-06, + "loss": 0.70008755, + "num_input_tokens_seen": 52372930, + "router_z_loss_clip": 0.40625, + "router_z_loss_mlp": 0.02746582, + "step": 2443, + "time_per_iteration": 4.251621961593628 + }, + { + "auxiliary_loss_clip": 0.01158025, + "auxiliary_loss_mlp": 0.0105253, + "balance_loss_clip": 1.05050611, + "balance_loss_mlp": 1.03281307, + "epoch": 0.29387362472193834, + "flos": 22054502862720.0, + "grad_norm": 2.3867045240133655, + "language_loss": 0.86763191, + "learning_rate": 3.313143309335282e-06, + "loss": 0.88973749, + "num_input_tokens_seen": 52391420, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.19726562, + "step": 2444, + "time_per_iteration": 2.8602492809295654 + }, + { + "auxiliary_loss_clip": 0.0115938, + "auxiliary_loss_mlp": 0.01048353, + "balance_loss_clip": 1.05101871, + "balance_loss_mlp": 1.02947068, + "epoch": 0.2939938676125774, + "flos": 22966418373120.0, + "grad_norm": 2.5685724817946145, + "language_loss": 0.84637117, + "learning_rate": 3.3125556597463665e-06, + "loss": 0.8684485, + "num_input_tokens_seen": 52410725, + "router_z_loss_clip": 1.08398438, + "router_z_loss_mlp": 0.18884277, + "step": 2445, + "time_per_iteration": 3.655137538909912 + }, + { + "auxiliary_loss_clip": 0.01159927, + "auxiliary_loss_mlp": 0.01048654, + "balance_loss_clip": 1.05203402, + "balance_loss_mlp": 1.02898431, + "epoch": 0.2941141105032165, + "flos": 31358705857920.0, + "grad_norm": 1.8760116156539068, + "language_loss": 0.66440731, + "learning_rate": 3.311967811043801e-06, + "loss": 0.68649304, + "num_input_tokens_seen": 52432645, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.19665527, + "step": 2446, + "time_per_iteration": 3.729869842529297 + }, + { + "auxiliary_loss_clip": 0.01162264, + "auxiliary_loss_mlp": 0.0105318, + "balance_loss_clip": 1.05391037, + "balance_loss_mlp": 1.03352284, + "epoch": 0.29423435339385556, + "flos": 23222138273280.0, + "grad_norm": 2.484549341064524, + "language_loss": 0.81858784, + "learning_rate": 3.3113797633167617e-06, + "loss": 0.84074229, + "num_input_tokens_seen": 52450940, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.19665527, + "step": 2447, + "time_per_iteration": 2.786855459213257 + }, + { + "auxiliary_loss_clip": 0.01163598, + "auxiliary_loss_mlp": 0.0104467, + "balance_loss_clip": 1.05319571, + "balance_loss_mlp": 1.0247618, + "epoch": 0.2943545962844947, + "flos": 26864054138880.0, + "grad_norm": 2.64217198024314, + "language_loss": 0.69612926, + "learning_rate": 3.310791516654455e-06, + "loss": 0.71821195, + "num_input_tokens_seen": 52468000, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.19909668, + "step": 2448, + "time_per_iteration": 2.8114798069000244 + }, + { + "auxiliary_loss_clip": 0.01164641, + "auxiliary_loss_mlp": 0.01056859, + "balance_loss_clip": 1.05357456, + "balance_loss_mlp": 1.03428066, + "epoch": 0.2944748391751338, + "flos": 20231677422720.0, + "grad_norm": 2.10825984471415, + "language_loss": 0.79043657, + "learning_rate": 3.3102030711461177e-06, + "loss": 0.81265152, + "num_input_tokens_seen": 52487575, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.22583008, + "step": 2449, + "time_per_iteration": 2.806628465652466 + }, + { + "auxiliary_loss_clip": 0.01162134, + "auxiliary_loss_mlp": 0.01043758, + "balance_loss_clip": 1.05161166, + "balance_loss_mlp": 1.02454162, + "epoch": 0.29459508206577284, + "flos": 15960965045760.0, + "grad_norm": 3.007552033199387, + "language_loss": 0.68004417, + "learning_rate": 3.3096144268810156e-06, + "loss": 0.70210308, + "num_input_tokens_seen": 52506335, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.19213867, + "step": 2450, + "time_per_iteration": 2.780954360961914 + }, + { + "auxiliary_loss_clip": 0.01163734, + "auxiliary_loss_mlp": 0.01048546, + "balance_loss_clip": 1.05285883, + "balance_loss_mlp": 1.02771974, + "epoch": 0.29471532495641195, + "flos": 20412882558720.0, + "grad_norm": 2.5107671964473934, + "language_loss": 0.73083067, + "learning_rate": 3.3090255839484462e-06, + "loss": 0.75295347, + "num_input_tokens_seen": 52524330, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.20825195, + "step": 2451, + "time_per_iteration": 2.7909786701202393 + }, + { + "auxiliary_loss_clip": 0.0115964, + "auxiliary_loss_mlp": 0.01039654, + "balance_loss_clip": 1.04750454, + "balance_loss_mlp": 1.01931643, + "epoch": 0.29483556784705106, + "flos": 20376576887040.0, + "grad_norm": 2.304531842653115, + "language_loss": 0.84994376, + "learning_rate": 3.3084365424377366e-06, + "loss": 0.87193668, + "num_input_tokens_seen": 52543095, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.20336914, + "step": 2452, + "time_per_iteration": 2.807689905166626 + }, + { + "auxiliary_loss_clip": 0.01064971, + "auxiliary_loss_mlp": 0.01028726, + "balance_loss_clip": 1.02373552, + "balance_loss_mlp": 1.02599609, + "epoch": 0.2949558107376901, + "flos": 68555660595840.0, + "grad_norm": 0.7312782800203139, + "language_loss": 0.55936265, + "learning_rate": 3.307847302438245e-06, + "loss": 0.58029962, + "num_input_tokens_seen": 52597075, + "router_z_loss_clip": 0.41210938, + "router_z_loss_mlp": 0.02734375, + "step": 2453, + "time_per_iteration": 3.241541862487793 + }, + { + "auxiliary_loss_clip": 0.01162678, + "auxiliary_loss_mlp": 0.01052281, + "balance_loss_clip": 1.05120254, + "balance_loss_mlp": 1.03175282, + "epoch": 0.2950760536283292, + "flos": 16107085572480.0, + "grad_norm": 2.832577643203349, + "language_loss": 0.78423846, + "learning_rate": 3.3072578640393562e-06, + "loss": 0.80638808, + "num_input_tokens_seen": 52614410, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.20532227, + "step": 2454, + "time_per_iteration": 2.7472786903381348 + }, + { + "auxiliary_loss_clip": 0.01160557, + "auxiliary_loss_mlp": 0.01055904, + "balance_loss_clip": 1.05060256, + "balance_loss_mlp": 1.03568625, + "epoch": 0.29519629651896834, + "flos": 20483626394880.0, + "grad_norm": 2.1218148712293563, + "language_loss": 0.7973882, + "learning_rate": 3.3066682273304886e-06, + "loss": 0.8195529, + "num_input_tokens_seen": 52632055, + "router_z_loss_clip": 1.09960938, + "router_z_loss_mlp": 0.20214844, + "step": 2455, + "time_per_iteration": 2.795018434524536 + }, + { + "auxiliary_loss_clip": 0.0116394, + "auxiliary_loss_mlp": 0.01042075, + "balance_loss_clip": 1.04912877, + "balance_loss_mlp": 1.0217731, + "epoch": 0.2953165394096074, + "flos": 18916484941440.0, + "grad_norm": 1.9528285766913818, + "language_loss": 0.79158306, + "learning_rate": 3.3060783924010904e-06, + "loss": 0.81364322, + "num_input_tokens_seen": 52649980, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.20288086, + "step": 2456, + "time_per_iteration": 2.840266466140747 + }, + { + "auxiliary_loss_clip": 0.01157967, + "auxiliary_loss_mlp": 0.01051171, + "balance_loss_clip": 1.04734588, + "balance_loss_mlp": 1.03016639, + "epoch": 0.2954367823002465, + "flos": 20624467622400.0, + "grad_norm": 3.254998875820413, + "language_loss": 0.84837103, + "learning_rate": 3.3054883593406387e-06, + "loss": 0.87046242, + "num_input_tokens_seen": 52664730, + "router_z_loss_clip": 1.10839844, + "router_z_loss_mlp": 0.20996094, + "step": 2457, + "time_per_iteration": 2.838801860809326 + }, + { + "auxiliary_loss_clip": 0.0115806, + "auxiliary_loss_mlp": 0.01044229, + "balance_loss_clip": 1.05007029, + "balance_loss_mlp": 1.02523923, + "epoch": 0.2955570251908856, + "flos": 31175525473920.0, + "grad_norm": 2.217454217857596, + "language_loss": 0.65655583, + "learning_rate": 3.3048981282386404e-06, + "loss": 0.67857873, + "num_input_tokens_seen": 52686040, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.18994141, + "step": 2458, + "time_per_iteration": 2.8624558448791504 + }, + { + "auxiliary_loss_clip": 0.0116304, + "auxiliary_loss_mlp": 0.01040511, + "balance_loss_clip": 1.05195796, + "balance_loss_mlp": 1.02072191, + "epoch": 0.29567726808152467, + "flos": 21650328051840.0, + "grad_norm": 5.5459510770133384, + "language_loss": 0.82243586, + "learning_rate": 3.304307699184634e-06, + "loss": 0.84447134, + "num_input_tokens_seen": 52704630, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.19787598, + "step": 2459, + "time_per_iteration": 2.787844181060791 + }, + { + "auxiliary_loss_clip": 0.01158256, + "auxiliary_loss_mlp": 0.01053505, + "balance_loss_clip": 1.0526185, + "balance_loss_mlp": 1.0338707, + "epoch": 0.2957975109721638, + "flos": 24243868638720.0, + "grad_norm": 3.742771479990072, + "language_loss": 0.78962386, + "learning_rate": 3.3037170722681866e-06, + "loss": 0.81174147, + "num_input_tokens_seen": 52725465, + "router_z_loss_clip": 1.05615234, + "router_z_loss_mlp": 0.19628906, + "step": 2460, + "time_per_iteration": 2.7995080947875977 + }, + { + "auxiliary_loss_clip": 0.01158042, + "auxiliary_loss_mlp": 0.01041297, + "balance_loss_clip": 1.05020547, + "balance_loss_mlp": 1.02076864, + "epoch": 0.29591775386280283, + "flos": 13479717352320.0, + "grad_norm": 2.6105744243990694, + "language_loss": 0.68070841, + "learning_rate": 3.3031262475788956e-06, + "loss": 0.70270181, + "num_input_tokens_seen": 52742405, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.20532227, + "step": 2461, + "time_per_iteration": 2.7958786487579346 + }, + { + "auxiliary_loss_clip": 0.01160084, + "auxiliary_loss_mlp": 0.01047964, + "balance_loss_clip": 1.05002582, + "balance_loss_mlp": 1.02810335, + "epoch": 0.29603799675344195, + "flos": 17749783284480.0, + "grad_norm": 2.015522169825606, + "language_loss": 0.72838312, + "learning_rate": 3.3025352252063897e-06, + "loss": 0.7504636, + "num_input_tokens_seen": 52761100, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.19848633, + "step": 2462, + "time_per_iteration": 2.829789638519287 + }, + { + "auxiliary_loss_clip": 0.01159569, + "auxiliary_loss_mlp": 0.01049852, + "balance_loss_clip": 1.05056131, + "balance_loss_mlp": 1.02994394, + "epoch": 0.29615823964408106, + "flos": 22783920347520.0, + "grad_norm": 2.472866705597736, + "language_loss": 0.75518316, + "learning_rate": 3.3019440052403252e-06, + "loss": 0.77727735, + "num_input_tokens_seen": 52780965, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.19934082, + "step": 2463, + "time_per_iteration": 2.7917776107788086 + }, + { + "auxiliary_loss_clip": 0.01160602, + "auxiliary_loss_mlp": 0.01041529, + "balance_loss_clip": 1.05251813, + "balance_loss_mlp": 1.02150202, + "epoch": 0.2962784825347201, + "flos": 23514199758720.0, + "grad_norm": 3.105455090905263, + "language_loss": 0.70896262, + "learning_rate": 3.30135258777039e-06, + "loss": 0.73098397, + "num_input_tokens_seen": 52800335, + "router_z_loss_clip": 1.08105469, + "router_z_loss_mlp": 0.20019531, + "step": 2464, + "time_per_iteration": 2.8637797832489014 + }, + { + "auxiliary_loss_clip": 0.0115907, + "auxiliary_loss_mlp": 0.0104687, + "balance_loss_clip": 1.047297, + "balance_loss_mlp": 1.02510214, + "epoch": 0.2963987254253592, + "flos": 16362769559040.0, + "grad_norm": 2.3596303708806867, + "language_loss": 0.70468831, + "learning_rate": 3.3007609728863024e-06, + "loss": 0.72674763, + "num_input_tokens_seen": 52818425, + "router_z_loss_clip": 1.11914062, + "router_z_loss_mlp": 0.21765137, + "step": 2465, + "time_per_iteration": 2.781097888946533 + }, + { + "auxiliary_loss_clip": 0.01157044, + "auxiliary_loss_mlp": 0.01041991, + "balance_loss_clip": 1.05006003, + "balance_loss_mlp": 1.02174926, + "epoch": 0.29651896831599833, + "flos": 33472263980160.0, + "grad_norm": 2.870326575458695, + "language_loss": 0.73193002, + "learning_rate": 3.300169160677809e-06, + "loss": 0.75392032, + "num_input_tokens_seen": 52842340, + "router_z_loss_clip": 1.06982422, + "router_z_loss_mlp": 0.20239258, + "step": 2466, + "time_per_iteration": 2.9233415126800537 + }, + { + "auxiliary_loss_clip": 0.01162645, + "auxiliary_loss_mlp": 0.01051154, + "balance_loss_clip": 1.0501616, + "balance_loss_mlp": 1.0311265, + "epoch": 0.2966392112066374, + "flos": 23805363404160.0, + "grad_norm": 2.4891001755760462, + "language_loss": 0.77787375, + "learning_rate": 3.2995771512346878e-06, + "loss": 0.80001175, + "num_input_tokens_seen": 52860690, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.20019531, + "step": 2467, + "time_per_iteration": 2.807429552078247 + }, + { + "auxiliary_loss_clip": 0.01166082, + "auxiliary_loss_mlp": 0.01050826, + "balance_loss_clip": 1.05248892, + "balance_loss_mlp": 1.02946413, + "epoch": 0.2967594540972765, + "flos": 19938466702080.0, + "grad_norm": 3.8579465627802993, + "language_loss": 0.73010772, + "learning_rate": 3.298984944646746e-06, + "loss": 0.75227684, + "num_input_tokens_seen": 52879370, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.21350098, + "step": 2468, + "time_per_iteration": 2.8159310817718506 + }, + { + "auxiliary_loss_clip": 0.01158285, + "auxiliary_loss_mlp": 0.01046771, + "balance_loss_clip": 1.05193388, + "balance_loss_mlp": 1.0268271, + "epoch": 0.2968796969879156, + "flos": 23732823888000.0, + "grad_norm": 2.6451729121269123, + "language_loss": 0.81512576, + "learning_rate": 3.298392541003822e-06, + "loss": 0.83717632, + "num_input_tokens_seen": 52898775, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.19946289, + "step": 2469, + "time_per_iteration": 2.9273886680603027 + }, + { + "auxiliary_loss_clip": 0.01161274, + "auxiliary_loss_mlp": 0.0104256, + "balance_loss_clip": 1.05106282, + "balance_loss_mlp": 1.02280724, + "epoch": 0.29699993987855466, + "flos": 22893699288960.0, + "grad_norm": 1.7527179503220132, + "language_loss": 0.89759827, + "learning_rate": 3.2977999403957806e-06, + "loss": 0.91963661, + "num_input_tokens_seen": 52917535, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.19750977, + "step": 2470, + "time_per_iteration": 3.1418375968933105 + }, + { + "auxiliary_loss_clip": 0.0116121, + "auxiliary_loss_mlp": 0.0104273, + "balance_loss_clip": 1.05199564, + "balance_loss_mlp": 1.02127206, + "epoch": 0.2971201827691938, + "flos": 33832555349760.0, + "grad_norm": 2.128506435860511, + "language_loss": 0.6702798, + "learning_rate": 3.2972071429125207e-06, + "loss": 0.69231915, + "num_input_tokens_seen": 52938755, + "router_z_loss_clip": 1.09179688, + "router_z_loss_mlp": 0.21459961, + "step": 2471, + "time_per_iteration": 3.7169289588928223 + }, + { + "auxiliary_loss_clip": 0.0116091, + "auxiliary_loss_mlp": 0.01049411, + "balance_loss_clip": 1.04998052, + "balance_loss_mlp": 1.02847826, + "epoch": 0.2972404256598329, + "flos": 22054359208320.0, + "grad_norm": 3.1133991202553553, + "language_loss": 0.88635063, + "learning_rate": 3.2966141486439682e-06, + "loss": 0.90845382, + "num_input_tokens_seen": 52957945, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.20935059, + "step": 2472, + "time_per_iteration": 3.686732530593872 + }, + { + "auxiliary_loss_clip": 0.01164636, + "auxiliary_loss_mlp": 0.01048851, + "balance_loss_clip": 1.05205035, + "balance_loss_mlp": 1.02864492, + "epoch": 0.29736066855047194, + "flos": 31978595796480.0, + "grad_norm": 2.2487116420374247, + "language_loss": 0.64451796, + "learning_rate": 3.29602095768008e-06, + "loss": 0.66665286, + "num_input_tokens_seen": 52978460, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.20214844, + "step": 2473, + "time_per_iteration": 2.9289138317108154 + }, + { + "auxiliary_loss_clip": 0.01157607, + "auxiliary_loss_mlp": 0.01049771, + "balance_loss_clip": 1.04907155, + "balance_loss_mlp": 1.03025651, + "epoch": 0.29748091144111105, + "flos": 33510401245440.0, + "grad_norm": 2.996018389600922, + "language_loss": 0.63659918, + "learning_rate": 3.2954275701108437e-06, + "loss": 0.65867299, + "num_input_tokens_seen": 52999640, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.19519043, + "step": 2474, + "time_per_iteration": 4.674702882766724 + }, + { + "auxiliary_loss_clip": 0.01159294, + "auxiliary_loss_mlp": 0.01049592, + "balance_loss_clip": 1.04894781, + "balance_loss_mlp": 1.02859974, + "epoch": 0.29760115433175016, + "flos": 41283373409280.0, + "grad_norm": 2.691077460235241, + "language_loss": 0.68476707, + "learning_rate": 3.294833986026275e-06, + "loss": 0.70685589, + "num_input_tokens_seen": 53022880, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.21008301, + "step": 2475, + "time_per_iteration": 3.0334582328796387 + }, + { + "auxiliary_loss_clip": 0.01160295, + "auxiliary_loss_mlp": 0.0104467, + "balance_loss_clip": 1.05161858, + "balance_loss_mlp": 1.02477407, + "epoch": 0.2977213972223892, + "flos": 24493339572480.0, + "grad_norm": 2.2876979973841047, + "language_loss": 0.85205293, + "learning_rate": 3.29424020551642e-06, + "loss": 0.87410259, + "num_input_tokens_seen": 53041515, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.19885254, + "step": 2476, + "time_per_iteration": 2.8095970153808594 + }, + { + "auxiliary_loss_clip": 0.01171106, + "auxiliary_loss_mlp": 0.01059666, + "balance_loss_clip": 1.05441165, + "balance_loss_mlp": 1.03801727, + "epoch": 0.2978416401130283, + "flos": 21285116519040.0, + "grad_norm": 2.3059726647139374, + "language_loss": 0.7245096, + "learning_rate": 3.2936462286713546e-06, + "loss": 0.74681735, + "num_input_tokens_seen": 53059865, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.21643066, + "step": 2477, + "time_per_iteration": 2.7661638259887695 + }, + { + "auxiliary_loss_clip": 0.0116175, + "auxiliary_loss_mlp": 0.01046673, + "balance_loss_clip": 1.05153275, + "balance_loss_mlp": 1.02666998, + "epoch": 0.2979618830036674, + "flos": 25772154554880.0, + "grad_norm": 2.206778881638946, + "language_loss": 0.77602255, + "learning_rate": 3.2930520555811846e-06, + "loss": 0.79810679, + "num_input_tokens_seen": 53079490, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.19995117, + "step": 2478, + "time_per_iteration": 2.8118538856506348 + }, + { + "auxiliary_loss_clip": 0.01161921, + "auxiliary_loss_mlp": 0.01047789, + "balance_loss_clip": 1.05130017, + "balance_loss_mlp": 1.0274632, + "epoch": 0.2980821258943065, + "flos": 23476996247040.0, + "grad_norm": 1.9034360635598937, + "language_loss": 0.80384898, + "learning_rate": 3.292457686336046e-06, + "loss": 0.82594609, + "num_input_tokens_seen": 53098810, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.203125, + "step": 2479, + "time_per_iteration": 2.764591693878174 + }, + { + "auxiliary_loss_clip": 0.0107574, + "auxiliary_loss_mlp": 0.01005238, + "balance_loss_clip": 1.0336206, + "balance_loss_mlp": 1.00216246, + "epoch": 0.2982023687849456, + "flos": 69752314195200.0, + "grad_norm": 0.9004926373726785, + "language_loss": 0.61282778, + "learning_rate": 3.291863121026105e-06, + "loss": 0.63363755, + "num_input_tokens_seen": 53162590, + "router_z_loss_clip": 0.421875, + "router_z_loss_mlp": 0.03063965, + "step": 2480, + "time_per_iteration": 3.3945553302764893 + }, + { + "auxiliary_loss_clip": 0.01166059, + "auxiliary_loss_mlp": 0.01050845, + "balance_loss_clip": 1.0511601, + "balance_loss_mlp": 1.03097248, + "epoch": 0.29832261167558466, + "flos": 29825930741760.0, + "grad_norm": 2.122793161308596, + "language_loss": 0.76888251, + "learning_rate": 3.2912683597415547e-06, + "loss": 0.79105151, + "num_input_tokens_seen": 53186675, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.19848633, + "step": 2481, + "time_per_iteration": 2.964918613433838 + }, + { + "auxiliary_loss_clip": 0.0116217, + "auxiliary_loss_mlp": 0.01048768, + "balance_loss_clip": 1.0524807, + "balance_loss_mlp": 1.02814507, + "epoch": 0.29844285456622377, + "flos": 33910158683520.0, + "grad_norm": 2.22841822507349, + "language_loss": 0.78223026, + "learning_rate": 3.2906734025726213e-06, + "loss": 0.80433965, + "num_input_tokens_seen": 53205940, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.20605469, + "step": 2482, + "time_per_iteration": 2.8700122833251953 + }, + { + "auxiliary_loss_clip": 0.01167438, + "auxiliary_loss_mlp": 0.01061311, + "balance_loss_clip": 1.05608153, + "balance_loss_mlp": 1.04000795, + "epoch": 0.2985630974568629, + "flos": 23876933253120.0, + "grad_norm": 2.6423707027940604, + "language_loss": 0.88157707, + "learning_rate": 3.290078249609559e-06, + "loss": 0.9038645, + "num_input_tokens_seen": 53225360, + "router_z_loss_clip": 1.11376953, + "router_z_loss_mlp": 0.21289062, + "step": 2483, + "time_per_iteration": 2.89361310005188 + }, + { + "auxiliary_loss_clip": 0.01164362, + "auxiliary_loss_mlp": 0.01045961, + "balance_loss_clip": 1.05462122, + "balance_loss_mlp": 1.02518284, + "epoch": 0.29868334034750194, + "flos": 21799106184960.0, + "grad_norm": 3.5936735017890364, + "language_loss": 0.88119715, + "learning_rate": 3.2894829009426514e-06, + "loss": 0.90330034, + "num_input_tokens_seen": 53243195, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.20788574, + "step": 2484, + "time_per_iteration": 2.9721150398254395 + }, + { + "auxiliary_loss_clip": 0.01160168, + "auxiliary_loss_mlp": 0.01041883, + "balance_loss_clip": 1.05108559, + "balance_loss_mlp": 1.02159369, + "epoch": 0.29880358323814105, + "flos": 25666649331840.0, + "grad_norm": 2.356556736038836, + "language_loss": 0.77514219, + "learning_rate": 3.288887356662213e-06, + "loss": 0.79716271, + "num_input_tokens_seen": 53264530, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.20275879, + "step": 2485, + "time_per_iteration": 2.8601644039154053 + }, + { + "auxiliary_loss_clip": 0.01071791, + "auxiliary_loss_mlp": 0.01005886, + "balance_loss_clip": 1.03011203, + "balance_loss_mlp": 1.00292993, + "epoch": 0.29892382612878016, + "flos": 71005846003200.0, + "grad_norm": 0.7747648778267253, + "language_loss": 0.59657252, + "learning_rate": 3.288291616858588e-06, + "loss": 0.61734933, + "num_input_tokens_seen": 53319920, + "router_z_loss_clip": 0.41601562, + "router_z_loss_mlp": 0.02954102, + "step": 2486, + "time_per_iteration": 3.2868330478668213 + }, + { + "auxiliary_loss_clip": 0.01162617, + "auxiliary_loss_mlp": 0.01052759, + "balance_loss_clip": 1.05242455, + "balance_loss_mlp": 1.03264856, + "epoch": 0.2990440690194192, + "flos": 25481134563840.0, + "grad_norm": 1.7819994938847952, + "language_loss": 0.76514763, + "learning_rate": 3.287695681622149e-06, + "loss": 0.78730142, + "num_input_tokens_seen": 53339270, + "router_z_loss_clip": 1.10107422, + "router_z_loss_mlp": 0.20092773, + "step": 2487, + "time_per_iteration": 2.8363494873046875 + }, + { + "auxiliary_loss_clip": 0.01158594, + "auxiliary_loss_mlp": 0.0105553, + "balance_loss_clip": 1.04854369, + "balance_loss_mlp": 1.03497887, + "epoch": 0.2991643119100583, + "flos": 23732357011200.0, + "grad_norm": 2.34383810624731, + "language_loss": 0.81095934, + "learning_rate": 3.2870995510432982e-06, + "loss": 0.83310062, + "num_input_tokens_seen": 53357750, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.20544434, + "step": 2488, + "time_per_iteration": 2.916283369064331 + }, + { + "auxiliary_loss_clip": 0.01155012, + "auxiliary_loss_mlp": 0.0104903, + "balance_loss_clip": 1.04860759, + "balance_loss_mlp": 1.02999258, + "epoch": 0.29928455480069743, + "flos": 27417545786880.0, + "grad_norm": 2.224187437917821, + "language_loss": 0.77266532, + "learning_rate": 3.2865032252124697e-06, + "loss": 0.79470575, + "num_input_tokens_seen": 53378265, + "router_z_loss_clip": 1.06347656, + "router_z_loss_mlp": 0.19042969, + "step": 2489, + "time_per_iteration": 2.8631725311279297 + }, + { + "auxiliary_loss_clip": 0.01156152, + "auxiliary_loss_mlp": 0.01045598, + "balance_loss_clip": 1.05031395, + "balance_loss_mlp": 1.02769315, + "epoch": 0.2994047976913365, + "flos": 33692935184640.0, + "grad_norm": 1.526618917392017, + "language_loss": 0.77562654, + "learning_rate": 3.2859067042201243e-06, + "loss": 0.79764402, + "num_input_tokens_seen": 53400305, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.17907715, + "step": 2490, + "time_per_iteration": 2.8960471153259277 + }, + { + "auxiliary_loss_clip": 0.01160491, + "auxiliary_loss_mlp": 0.01042108, + "balance_loss_clip": 1.05294073, + "balance_loss_mlp": 1.02308273, + "epoch": 0.2995250405819756, + "flos": 16763963541120.0, + "grad_norm": 2.1041255847018254, + "language_loss": 0.78195512, + "learning_rate": 3.2853099881567544e-06, + "loss": 0.80398107, + "num_input_tokens_seen": 53418705, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.19030762, + "step": 2491, + "time_per_iteration": 2.8338067531585693 + }, + { + "auxiliary_loss_clip": 0.01155716, + "auxiliary_loss_mlp": 0.01048825, + "balance_loss_clip": 1.05118632, + "balance_loss_mlp": 1.03066945, + "epoch": 0.29964528347261465, + "flos": 22963976248320.0, + "grad_norm": 1.997649657171625, + "language_loss": 0.79546762, + "learning_rate": 3.284713077112881e-06, + "loss": 0.81751305, + "num_input_tokens_seen": 53438135, + "router_z_loss_clip": 1.04541016, + "router_z_loss_mlp": 0.18164062, + "step": 2492, + "time_per_iteration": 2.9083080291748047 + }, + { + "auxiliary_loss_clip": 0.01165416, + "auxiliary_loss_mlp": 0.01052179, + "balance_loss_clip": 1.05330706, + "balance_loss_mlp": 1.02957702, + "epoch": 0.29976552636325376, + "flos": 16938021870720.0, + "grad_norm": 3.2051189062335306, + "language_loss": 0.86534166, + "learning_rate": 3.284115971179056e-06, + "loss": 0.88751757, + "num_input_tokens_seen": 53452165, + "router_z_loss_clip": 1.12255859, + "router_z_loss_mlp": 0.22607422, + "step": 2493, + "time_per_iteration": 2.9332196712493896 + }, + { + "auxiliary_loss_clip": 0.01157015, + "auxiliary_loss_mlp": 0.01048396, + "balance_loss_clip": 1.05198872, + "balance_loss_mlp": 1.02765369, + "epoch": 0.2998857692538929, + "flos": 17056455989760.0, + "grad_norm": 1.962878195265838, + "language_loss": 0.7899617, + "learning_rate": 3.283518670445859e-06, + "loss": 0.81201583, + "num_input_tokens_seen": 53470075, + "router_z_loss_clip": 1.04980469, + "router_z_loss_mlp": 0.20739746, + "step": 2494, + "time_per_iteration": 2.8257691860198975 + }, + { + "auxiliary_loss_clip": 0.0106947, + "auxiliary_loss_mlp": 0.01003966, + "balance_loss_clip": 1.02758908, + "balance_loss_mlp": 1.00077081, + "epoch": 0.30000601214453193, + "flos": 68831528025600.0, + "grad_norm": 0.7188815552608067, + "language_loss": 0.54271299, + "learning_rate": 3.2829211750038995e-06, + "loss": 0.5634473, + "num_input_tokens_seen": 53538705, + "router_z_loss_clip": 0.41796875, + "router_z_loss_mlp": 0.03198242, + "step": 2495, + "time_per_iteration": 3.381174325942993 + }, + { + "auxiliary_loss_clip": 0.01165713, + "auxiliary_loss_mlp": 0.01048162, + "balance_loss_clip": 1.05286348, + "balance_loss_mlp": 1.02787232, + "epoch": 0.30012625503517104, + "flos": 17603267708160.0, + "grad_norm": 2.3244013038119085, + "language_loss": 0.89453125, + "learning_rate": 3.2823234849438183e-06, + "loss": 0.91666996, + "num_input_tokens_seen": 53556740, + "router_z_loss_clip": 1.12792969, + "router_z_loss_mlp": 0.20288086, + "step": 2496, + "time_per_iteration": 2.83461594581604 + }, + { + "auxiliary_loss_clip": 0.01157555, + "auxiliary_loss_mlp": 0.01045904, + "balance_loss_clip": 1.05024457, + "balance_loss_mlp": 1.02578187, + "epoch": 0.30024649792581015, + "flos": 21252581775360.0, + "grad_norm": 4.1838537777502935, + "language_loss": 0.75948298, + "learning_rate": 3.2817256003562836e-06, + "loss": 0.78151751, + "num_input_tokens_seen": 53577115, + "router_z_loss_clip": 1.07373047, + "router_z_loss_mlp": 0.2010498, + "step": 2497, + "time_per_iteration": 2.814941883087158 + }, + { + "auxiliary_loss_clip": 0.01162976, + "auxiliary_loss_mlp": 0.01052937, + "balance_loss_clip": 1.05104983, + "balance_loss_mlp": 1.033113, + "epoch": 0.3003667408164492, + "flos": 23003262748800.0, + "grad_norm": 1.8941367069490331, + "language_loss": 0.66025674, + "learning_rate": 3.281127521331995e-06, + "loss": 0.68241584, + "num_input_tokens_seen": 53598295, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.19836426, + "step": 2498, + "time_per_iteration": 2.8164303302764893 + }, + { + "auxiliary_loss_clip": 0.01067952, + "auxiliary_loss_mlp": 0.00999119, + "balance_loss_clip": 1.02630782, + "balance_loss_mlp": 0.99590033, + "epoch": 0.3004869837070883, + "flos": 64232340750720.0, + "grad_norm": 0.8803552864721091, + "language_loss": 0.60641742, + "learning_rate": 3.2805292479616798e-06, + "loss": 0.62708813, + "num_input_tokens_seen": 53657160, + "router_z_loss_clip": 0.41601562, + "router_z_loss_mlp": 0.03222656, + "step": 2499, + "time_per_iteration": 3.219322681427002 + }, + { + "auxiliary_loss_clip": 0.0116244, + "auxiliary_loss_mlp": 0.01045448, + "balance_loss_clip": 1.05127954, + "balance_loss_mlp": 1.02513456, + "epoch": 0.30060722659772743, + "flos": 26248653400320.0, + "grad_norm": 2.461471084416216, + "language_loss": 0.91737878, + "learning_rate": 3.2799307803360955e-06, + "loss": 0.9394576, + "num_input_tokens_seen": 53673090, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.203125, + "step": 2500, + "time_per_iteration": 4.505375146865845 + }, + { + "auxiliary_loss_clip": 0.01155039, + "auxiliary_loss_mlp": 0.01045787, + "balance_loss_clip": 1.04885006, + "balance_loss_mlp": 1.02813244, + "epoch": 0.3007274694883665, + "flos": 24970879912320.0, + "grad_norm": 1.7573211780005868, + "language_loss": 0.81692868, + "learning_rate": 3.27933211854603e-06, + "loss": 0.83893692, + "num_input_tokens_seen": 53692145, + "router_z_loss_clip": 1.06103516, + "router_z_loss_mlp": 0.17663574, + "step": 2501, + "time_per_iteration": 2.8091866970062256 + }, + { + "auxiliary_loss_clip": 0.01159864, + "auxiliary_loss_mlp": 0.01046724, + "balance_loss_clip": 1.05068445, + "balance_loss_mlp": 1.02643442, + "epoch": 0.3008477123790056, + "flos": 17055845458560.0, + "grad_norm": 1.696591920865091, + "language_loss": 0.86925781, + "learning_rate": 3.278733262682299e-06, + "loss": 0.89132369, + "num_input_tokens_seen": 53710000, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.20275879, + "step": 2502, + "time_per_iteration": 2.805448055267334 + }, + { + "auxiliary_loss_clip": 0.01158458, + "auxiliary_loss_mlp": 0.01045428, + "balance_loss_clip": 1.04829443, + "balance_loss_mlp": 1.02505505, + "epoch": 0.3009679552696447, + "flos": 21506398254720.0, + "grad_norm": 2.897430009316064, + "language_loss": 0.82974857, + "learning_rate": 3.2781342128357484e-06, + "loss": 0.85178745, + "num_input_tokens_seen": 53729355, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.20397949, + "step": 2503, + "time_per_iteration": 4.4753358364105225 + }, + { + "auxiliary_loss_clip": 0.01160773, + "auxiliary_loss_mlp": 0.01044716, + "balance_loss_clip": 1.05369878, + "balance_loss_mlp": 1.02595282, + "epoch": 0.30108819816028376, + "flos": 21134004001920.0, + "grad_norm": 3.166661863502155, + "language_loss": 0.80738539, + "learning_rate": 3.2775349690972547e-06, + "loss": 0.82944036, + "num_input_tokens_seen": 53743505, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.18762207, + "step": 2504, + "time_per_iteration": 2.924466371536255 + }, + { + "auxiliary_loss_clip": 0.01068642, + "auxiliary_loss_mlp": 0.01007952, + "balance_loss_clip": 1.02749324, + "balance_loss_mlp": 1.00487614, + "epoch": 0.30120844105092287, + "flos": 71126434938240.0, + "grad_norm": 0.7694885357559538, + "language_loss": 0.51804018, + "learning_rate": 3.276935531557722e-06, + "loss": 0.53880608, + "num_input_tokens_seen": 53808725, + "router_z_loss_clip": 0.41015625, + "router_z_loss_mlp": 0.03063965, + "step": 2505, + "time_per_iteration": 3.4544830322265625 + }, + { + "auxiliary_loss_clip": 0.01163206, + "auxiliary_loss_mlp": 0.01055456, + "balance_loss_clip": 1.04908752, + "balance_loss_mlp": 1.03418875, + "epoch": 0.301328683941562, + "flos": 20264571302400.0, + "grad_norm": 2.4860757370342443, + "language_loss": 0.7940582, + "learning_rate": 3.2763359003080837e-06, + "loss": 0.81624484, + "num_input_tokens_seen": 53825680, + "router_z_loss_clip": 1.14160156, + "router_z_loss_mlp": 0.21276855, + "step": 2506, + "time_per_iteration": 2.817474842071533 + }, + { + "auxiliary_loss_clip": 0.0106986, + "auxiliary_loss_mlp": 0.0100965, + "balance_loss_clip": 1.02840734, + "balance_loss_mlp": 1.00643158, + "epoch": 0.30144892683220104, + "flos": 70648212240000.0, + "grad_norm": 0.863649906554659, + "language_loss": 0.62519693, + "learning_rate": 3.2757360754393047e-06, + "loss": 0.64599204, + "num_input_tokens_seen": 53889750, + "router_z_loss_clip": 0.4140625, + "router_z_loss_mlp": 0.03222656, + "step": 2507, + "time_per_iteration": 3.4191598892211914 + }, + { + "auxiliary_loss_clip": 0.01159942, + "auxiliary_loss_mlp": 0.01047949, + "balance_loss_clip": 1.05095363, + "balance_loss_mlp": 1.02746916, + "epoch": 0.30156916972284015, + "flos": 22820549241600.0, + "grad_norm": 3.7301917167489895, + "language_loss": 0.64036095, + "learning_rate": 3.2751360570423767e-06, + "loss": 0.66243994, + "num_input_tokens_seen": 53908135, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.20483398, + "step": 2508, + "time_per_iteration": 2.8284943103790283 + }, + { + "auxiliary_loss_clip": 0.01162994, + "auxiliary_loss_mlp": 0.01048644, + "balance_loss_clip": 1.04990256, + "balance_loss_mlp": 1.02967823, + "epoch": 0.3016894126134792, + "flos": 29899188529920.0, + "grad_norm": 2.7935737961584746, + "language_loss": 0.76195908, + "learning_rate": 3.2745358452083236e-06, + "loss": 0.7840755, + "num_input_tokens_seen": 53931035, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.18969727, + "step": 2509, + "time_per_iteration": 2.902963161468506 + }, + { + "auxiliary_loss_clip": 0.01152324, + "auxiliary_loss_mlp": 0.0103894, + "balance_loss_clip": 1.04932463, + "balance_loss_mlp": 1.02108228, + "epoch": 0.3018096555041183, + "flos": 21546331200000.0, + "grad_norm": 1.5137969890409289, + "language_loss": 0.82492232, + "learning_rate": 3.2739354400281955e-06, + "loss": 0.8468349, + "num_input_tokens_seen": 53952255, + "router_z_loss_clip": 1.03027344, + "router_z_loss_mlp": 0.1784668, + "step": 2510, + "time_per_iteration": 2.898411512374878 + }, + { + "auxiliary_loss_clip": 0.01066929, + "auxiliary_loss_mlp": 0.00999563, + "balance_loss_clip": 1.02607846, + "balance_loss_mlp": 0.99654692, + "epoch": 0.3019298983947574, + "flos": 59136294597120.0, + "grad_norm": 0.8655093703774883, + "language_loss": 0.63702261, + "learning_rate": 3.2733348415930744e-06, + "loss": 0.65768754, + "num_input_tokens_seen": 54014125, + "router_z_loss_clip": 0.41015625, + "router_z_loss_mlp": 0.03015137, + "step": 2511, + "time_per_iteration": 3.379276990890503 + }, + { + "auxiliary_loss_clip": 0.01156921, + "auxiliary_loss_mlp": 0.01044871, + "balance_loss_clip": 1.04671645, + "balance_loss_mlp": 1.02635837, + "epoch": 0.3020501412853965, + "flos": 34423070941440.0, + "grad_norm": 3.1688015870619757, + "language_loss": 0.80803549, + "learning_rate": 3.27273404999407e-06, + "loss": 0.83005345, + "num_input_tokens_seen": 54036345, + "router_z_loss_clip": 1.10107422, + "router_z_loss_mlp": 0.18530273, + "step": 2512, + "time_per_iteration": 3.049391269683838 + }, + { + "auxiliary_loss_clip": 0.01065625, + "auxiliary_loss_mlp": 0.01000284, + "balance_loss_clip": 1.02457762, + "balance_loss_mlp": 0.99723226, + "epoch": 0.3021703841760356, + "flos": 71008288128000.0, + "grad_norm": 0.8103341998936319, + "language_loss": 0.60398281, + "learning_rate": 3.272133065322322e-06, + "loss": 0.6246419, + "num_input_tokens_seen": 54094615, + "router_z_loss_clip": 0.41015625, + "router_z_loss_mlp": 0.03051758, + "step": 2513, + "time_per_iteration": 3.3579602241516113 + }, + { + "auxiliary_loss_clip": 0.01154646, + "auxiliary_loss_mlp": 0.01045595, + "balance_loss_clip": 1.04680967, + "balance_loss_mlp": 1.02597296, + "epoch": 0.3022906270666747, + "flos": 21510528318720.0, + "grad_norm": 1.8452694855328045, + "language_loss": 0.79683191, + "learning_rate": 3.271531887669e-06, + "loss": 0.81883436, + "num_input_tokens_seen": 54114675, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.19616699, + "step": 2514, + "time_per_iteration": 2.851320505142212 + }, + { + "auxiliary_loss_clip": 0.01165041, + "auxiliary_loss_mlp": 0.01051328, + "balance_loss_clip": 1.05221665, + "balance_loss_mlp": 1.0299778, + "epoch": 0.30241086995731375, + "flos": 31132001168640.0, + "grad_norm": 2.4869993442108176, + "language_loss": 0.64231241, + "learning_rate": 3.2709305171253015e-06, + "loss": 0.66447616, + "num_input_tokens_seen": 54134795, + "router_z_loss_clip": 1.12792969, + "router_z_loss_mlp": 0.21337891, + "step": 2515, + "time_per_iteration": 2.9342079162597656 + }, + { + "auxiliary_loss_clip": 0.01160907, + "auxiliary_loss_mlp": 0.01049267, + "balance_loss_clip": 1.05130994, + "balance_loss_mlp": 1.031147, + "epoch": 0.30253111284795287, + "flos": 23511542152320.0, + "grad_norm": 1.9742740855744445, + "language_loss": 0.77761769, + "learning_rate": 3.2703289537824536e-06, + "loss": 0.79971945, + "num_input_tokens_seen": 54154595, + "router_z_loss_clip": 1.09570312, + "router_z_loss_mlp": 0.18127441, + "step": 2516, + "time_per_iteration": 2.862293004989624 + }, + { + "auxiliary_loss_clip": 0.01160836, + "auxiliary_loss_mlp": 0.01053509, + "balance_loss_clip": 1.05086327, + "balance_loss_mlp": 1.03295732, + "epoch": 0.302651355738592, + "flos": 18725367651840.0, + "grad_norm": 3.4152938135662176, + "language_loss": 0.78954333, + "learning_rate": 3.269727197731714e-06, + "loss": 0.81168675, + "num_input_tokens_seen": 54167360, + "router_z_loss_clip": 1.09960938, + "router_z_loss_mlp": 0.20556641, + "step": 2517, + "time_per_iteration": 2.7968649864196777 + }, + { + "auxiliary_loss_clip": 0.01155394, + "auxiliary_loss_mlp": 0.01054245, + "balance_loss_clip": 1.04668081, + "balance_loss_mlp": 1.03439641, + "epoch": 0.30277159862923103, + "flos": 22418888382720.0, + "grad_norm": 2.7217881803708375, + "language_loss": 0.78119206, + "learning_rate": 3.269125249064367e-06, + "loss": 0.80328846, + "num_input_tokens_seen": 54187055, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.1986084, + "step": 2518, + "time_per_iteration": 2.8444197177886963 + }, + { + "auxiliary_loss_clip": 0.01158942, + "auxiliary_loss_mlp": 0.0105125, + "balance_loss_clip": 1.04754353, + "balance_loss_mlp": 1.03032899, + "epoch": 0.30289184151987014, + "flos": 22273126992000.0, + "grad_norm": 1.9047052738554737, + "language_loss": 0.83351105, + "learning_rate": 3.2685231078717297e-06, + "loss": 0.85561299, + "num_input_tokens_seen": 54207245, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.20922852, + "step": 2519, + "time_per_iteration": 2.8183999061584473 + }, + { + "auxiliary_loss_clip": 0.0116351, + "auxiliary_loss_mlp": 0.01052452, + "balance_loss_clip": 1.0501219, + "balance_loss_mlp": 1.03310382, + "epoch": 0.30301208441050925, + "flos": 25225594231680.0, + "grad_norm": 2.0683785495906526, + "language_loss": 0.7531507, + "learning_rate": 3.267920774245145e-06, + "loss": 0.77531034, + "num_input_tokens_seen": 54226650, + "router_z_loss_clip": 1.13330078, + "router_z_loss_mlp": 0.19335938, + "step": 2520, + "time_per_iteration": 2.8006861209869385 + }, + { + "auxiliary_loss_clip": 0.01165434, + "auxiliary_loss_mlp": 0.01051094, + "balance_loss_clip": 1.05118275, + "balance_loss_mlp": 1.02920711, + "epoch": 0.3031323273011483, + "flos": 23039245198080.0, + "grad_norm": 2.1709254945685066, + "language_loss": 0.84406197, + "learning_rate": 3.2673182482759876e-06, + "loss": 0.86622727, + "num_input_tokens_seen": 54245765, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.21887207, + "step": 2521, + "time_per_iteration": 2.7745814323425293 + }, + { + "auxiliary_loss_clip": 0.0115911, + "auxiliary_loss_mlp": 0.01050702, + "balance_loss_clip": 1.04947293, + "balance_loss_mlp": 1.03063953, + "epoch": 0.3032525701917874, + "flos": 18876695650560.0, + "grad_norm": 2.2497351881235343, + "language_loss": 0.66300583, + "learning_rate": 3.266715530055659e-06, + "loss": 0.68510401, + "num_input_tokens_seen": 54263915, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.20056152, + "step": 2522, + "time_per_iteration": 2.7663381099700928 + }, + { + "auxiliary_loss_clip": 0.01159502, + "auxiliary_loss_mlp": 0.01050463, + "balance_loss_clip": 1.04878044, + "balance_loss_mlp": 1.02957773, + "epoch": 0.30337281308242653, + "flos": 17782641250560.0, + "grad_norm": 1.9476587495098043, + "language_loss": 0.80514044, + "learning_rate": 3.2661126196755927e-06, + "loss": 0.82724011, + "num_input_tokens_seen": 54283025, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.20898438, + "step": 2523, + "time_per_iteration": 2.792771100997925 + }, + { + "auxiliary_loss_clip": 0.01063083, + "auxiliary_loss_mlp": 0.01016359, + "balance_loss_clip": 1.02261806, + "balance_loss_mlp": 1.0134865, + "epoch": 0.3034930559730656, + "flos": 57824298426240.0, + "grad_norm": 0.7814406684323696, + "language_loss": 0.55856353, + "learning_rate": 3.265509517227248e-06, + "loss": 0.57935798, + "num_input_tokens_seen": 54339840, + "router_z_loss_clip": 0.40625, + "router_z_loss_mlp": 0.02868652, + "step": 2524, + "time_per_iteration": 3.3219523429870605 + }, + { + "auxiliary_loss_clip": 0.01161123, + "auxiliary_loss_mlp": 0.01046729, + "balance_loss_clip": 1.04976535, + "balance_loss_mlp": 1.02614141, + "epoch": 0.3036132988637047, + "flos": 14755587419520.0, + "grad_norm": 1.9146363227421843, + "language_loss": 0.81481028, + "learning_rate": 3.264906222802115e-06, + "loss": 0.83688879, + "num_input_tokens_seen": 54357690, + "router_z_loss_clip": 1.11425781, + "router_z_loss_mlp": 0.20593262, + "step": 2525, + "time_per_iteration": 2.9030978679656982 + }, + { + "auxiliary_loss_clip": 0.01159541, + "auxiliary_loss_mlp": 0.01051203, + "balance_loss_clip": 1.04742563, + "balance_loss_mlp": 1.03013873, + "epoch": 0.30373354175434375, + "flos": 21033203460480.0, + "grad_norm": 7.401155753083322, + "language_loss": 0.7850644, + "learning_rate": 3.264302736491715e-06, + "loss": 0.80717182, + "num_input_tokens_seen": 54377810, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.21069336, + "step": 2526, + "time_per_iteration": 2.9935083389282227 + }, + { + "auxiliary_loss_clip": 0.01154987, + "auxiliary_loss_mlp": 0.01046921, + "balance_loss_clip": 1.04901743, + "balance_loss_mlp": 1.0275259, + "epoch": 0.30385378464498286, + "flos": 21143233797120.0, + "grad_norm": 1.893156394488065, + "language_loss": 0.87774205, + "learning_rate": 3.263699058387594e-06, + "loss": 0.89976114, + "num_input_tokens_seen": 54395245, + "router_z_loss_clip": 1.05908203, + "router_z_loss_mlp": 0.19384766, + "step": 2527, + "time_per_iteration": 2.8668124675750732 + }, + { + "auxiliary_loss_clip": 0.01157624, + "auxiliary_loss_mlp": 0.01046194, + "balance_loss_clip": 1.0475961, + "balance_loss_mlp": 1.02557111, + "epoch": 0.30397402753562197, + "flos": 20629244131200.0, + "grad_norm": 5.705851242291091, + "language_loss": 0.90692091, + "learning_rate": 3.2630951885813315e-06, + "loss": 0.92895913, + "num_input_tokens_seen": 54412640, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.20617676, + "step": 2528, + "time_per_iteration": 2.8681588172912598 + }, + { + "auxiliary_loss_clip": 0.01155017, + "auxiliary_loss_mlp": 0.01045648, + "balance_loss_clip": 1.04625571, + "balance_loss_mlp": 1.02628875, + "epoch": 0.304094270426261, + "flos": 15085678429440.0, + "grad_norm": 2.460460475644694, + "language_loss": 0.78543371, + "learning_rate": 3.262491127164533e-06, + "loss": 0.80744028, + "num_input_tokens_seen": 54431455, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.19348145, + "step": 2529, + "time_per_iteration": 4.525727272033691 + }, + { + "auxiliary_loss_clip": 0.01161993, + "auxiliary_loss_mlp": 0.01052936, + "balance_loss_clip": 1.04602647, + "balance_loss_mlp": 1.03077495, + "epoch": 0.30421451331690014, + "flos": 13845216193920.0, + "grad_norm": 2.4190921434801385, + "language_loss": 0.80229676, + "learning_rate": 3.2618868742288337e-06, + "loss": 0.82444602, + "num_input_tokens_seen": 54448380, + "router_z_loss_clip": 1.16015625, + "router_z_loss_mlp": 0.22167969, + "step": 2530, + "time_per_iteration": 2.811798095703125 + }, + { + "auxiliary_loss_clip": 0.01160684, + "auxiliary_loss_mlp": 0.01037247, + "balance_loss_clip": 1.0512135, + "balance_loss_mlp": 1.01781583, + "epoch": 0.30433475620753925, + "flos": 17384212615680.0, + "grad_norm": 1.9019958196457356, + "language_loss": 0.72169125, + "learning_rate": 3.261282429865899e-06, + "loss": 0.74367052, + "num_input_tokens_seen": 54466385, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.19433594, + "step": 2531, + "time_per_iteration": 2.943873643875122 + }, + { + "auxiliary_loss_clip": 0.01154486, + "auxiliary_loss_mlp": 0.01052696, + "balance_loss_clip": 1.05141127, + "balance_loss_mlp": 1.03243053, + "epoch": 0.3044549990981783, + "flos": 18916951818240.0, + "grad_norm": 2.871134586452433, + "language_loss": 0.72514951, + "learning_rate": 3.2606777941674225e-06, + "loss": 0.74722135, + "num_input_tokens_seen": 54485040, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.20251465, + "step": 2532, + "time_per_iteration": 4.513144254684448 + }, + { + "auxiliary_loss_clip": 0.01160684, + "auxiliary_loss_mlp": 0.01045753, + "balance_loss_clip": 1.04789829, + "balance_loss_mlp": 1.02584529, + "epoch": 0.3045752419888174, + "flos": 21068431724160.0, + "grad_norm": 2.253943339247882, + "language_loss": 0.84425187, + "learning_rate": 3.2600729672251276e-06, + "loss": 0.8663162, + "num_input_tokens_seen": 54502755, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.19897461, + "step": 2533, + "time_per_iteration": 2.7687747478485107 + }, + { + "auxiliary_loss_clip": 0.01156754, + "auxiliary_loss_mlp": 0.01044544, + "balance_loss_clip": 1.04907084, + "balance_loss_mlp": 1.02493405, + "epoch": 0.3046954848794565, + "flos": 29096405516160.0, + "grad_norm": 3.0411019253699503, + "language_loss": 0.65288234, + "learning_rate": 3.259467949130765e-06, + "loss": 0.67489529, + "num_input_tokens_seen": 54524165, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.19616699, + "step": 2534, + "time_per_iteration": 2.8666679859161377 + }, + { + "auxiliary_loss_clip": 0.01159377, + "auxiliary_loss_mlp": 0.01046088, + "balance_loss_clip": 1.04887176, + "balance_loss_mlp": 1.02734852, + "epoch": 0.3048157277700956, + "flos": 20295346279680.0, + "grad_norm": 3.754737379601503, + "language_loss": 0.82964766, + "learning_rate": 3.2588627399761164e-06, + "loss": 0.85170233, + "num_input_tokens_seen": 54540160, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.18737793, + "step": 2535, + "time_per_iteration": 2.775446653366089 + }, + { + "auxiliary_loss_clip": 0.01159552, + "auxiliary_loss_mlp": 0.01045619, + "balance_loss_clip": 1.04939008, + "balance_loss_mlp": 1.02517414, + "epoch": 0.3049359706607347, + "flos": 22739929165440.0, + "grad_norm": 2.850058806310698, + "language_loss": 0.71071833, + "learning_rate": 3.2582573398529903e-06, + "loss": 0.73276997, + "num_input_tokens_seen": 54557515, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.20446777, + "step": 2536, + "time_per_iteration": 2.8346686363220215 + }, + { + "auxiliary_loss_clip": 0.01159834, + "auxiliary_loss_mlp": 0.01055089, + "balance_loss_clip": 1.05013394, + "balance_loss_mlp": 1.03602791, + "epoch": 0.3050562135513738, + "flos": 18434634969600.0, + "grad_norm": 2.304262242636849, + "language_loss": 0.74149895, + "learning_rate": 3.2576517488532265e-06, + "loss": 0.76364821, + "num_input_tokens_seen": 54573865, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.19067383, + "step": 2537, + "time_per_iteration": 2.7895267009735107 + }, + { + "auxiliary_loss_clip": 0.01152856, + "auxiliary_loss_mlp": 0.01040778, + "balance_loss_clip": 1.04789674, + "balance_loss_mlp": 1.02183533, + "epoch": 0.30517645644201286, + "flos": 20370327920640.0, + "grad_norm": 2.0237384964861986, + "language_loss": 0.87529999, + "learning_rate": 3.257045967068692e-06, + "loss": 0.89723635, + "num_input_tokens_seen": 54593120, + "router_z_loss_clip": 1.04882812, + "router_z_loss_mlp": 0.18945312, + "step": 2538, + "time_per_iteration": 2.788417339324951 + }, + { + "auxiliary_loss_clip": 0.01161381, + "auxiliary_loss_mlp": 0.01050035, + "balance_loss_clip": 1.05026186, + "balance_loss_mlp": 1.03036511, + "epoch": 0.30529669933265197, + "flos": 21945118970880.0, + "grad_norm": 1.6656136349114805, + "language_loss": 0.82115412, + "learning_rate": 3.2564399945912848e-06, + "loss": 0.84326828, + "num_input_tokens_seen": 54612910, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.19665527, + "step": 2539, + "time_per_iteration": 2.8770909309387207 + }, + { + "auxiliary_loss_clip": 0.01153387, + "auxiliary_loss_mlp": 0.01041412, + "balance_loss_clip": 1.04858625, + "balance_loss_mlp": 1.02311289, + "epoch": 0.305416942223291, + "flos": 21835411856640.0, + "grad_norm": 2.4947197793063127, + "language_loss": 0.82212484, + "learning_rate": 3.2558338315129287e-06, + "loss": 0.84407276, + "num_input_tokens_seen": 54631055, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.18310547, + "step": 2540, + "time_per_iteration": 2.949561595916748 + }, + { + "auxiliary_loss_clip": 0.01154868, + "auxiliary_loss_mlp": 0.01044746, + "balance_loss_clip": 1.05028713, + "balance_loss_mlp": 1.02417088, + "epoch": 0.30553718511393013, + "flos": 33911810709120.0, + "grad_norm": 1.94710893363972, + "language_loss": 0.76329756, + "learning_rate": 3.2552274779255785e-06, + "loss": 0.7852937, + "num_input_tokens_seen": 54651985, + "router_z_loss_clip": 1.04541016, + "router_z_loss_mlp": 0.20556641, + "step": 2541, + "time_per_iteration": 2.907055377960205 + }, + { + "auxiliary_loss_clip": 0.01158376, + "auxiliary_loss_mlp": 0.01042531, + "balance_loss_clip": 1.04965043, + "balance_loss_mlp": 1.02327871, + "epoch": 0.30565742800456924, + "flos": 22268530051200.0, + "grad_norm": 2.3901571626609006, + "language_loss": 0.77238828, + "learning_rate": 3.2546209339212184e-06, + "loss": 0.79439735, + "num_input_tokens_seen": 54671005, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.19250488, + "step": 2542, + "time_per_iteration": 2.8114025592803955 + }, + { + "auxiliary_loss_clip": 0.01155394, + "auxiliary_loss_mlp": 0.01043687, + "balance_loss_clip": 1.0496726, + "balance_loss_mlp": 1.02399313, + "epoch": 0.3057776708952083, + "flos": 22565044823040.0, + "grad_norm": 1.659945786222727, + "language_loss": 0.77684605, + "learning_rate": 3.25401419959186e-06, + "loss": 0.79883689, + "num_input_tokens_seen": 54691615, + "router_z_loss_clip": 1.05712891, + "router_z_loss_mlp": 0.19689941, + "step": 2543, + "time_per_iteration": 2.8142080307006836 + }, + { + "auxiliary_loss_clip": 0.01162461, + "auxiliary_loss_mlp": 0.01051197, + "balance_loss_clip": 1.05159807, + "balance_loss_mlp": 1.03233838, + "epoch": 0.3058979137858474, + "flos": 21799213925760.0, + "grad_norm": 1.9802677034568064, + "language_loss": 0.75861597, + "learning_rate": 3.253407275029545e-06, + "loss": 0.78075254, + "num_input_tokens_seen": 54710520, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.1887207, + "step": 2544, + "time_per_iteration": 2.813467025756836 + }, + { + "auxiliary_loss_clip": 0.01165909, + "auxiliary_loss_mlp": 0.01053427, + "balance_loss_clip": 1.05336559, + "balance_loss_mlp": 1.0334239, + "epoch": 0.3060181566764865, + "flos": 26979435601920.0, + "grad_norm": 1.793859171710969, + "language_loss": 0.79989481, + "learning_rate": 3.2528001603263425e-06, + "loss": 0.82208818, + "num_input_tokens_seen": 54732590, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.19995117, + "step": 2545, + "time_per_iteration": 2.901625633239746 + }, + { + "auxiliary_loss_clip": 0.01161664, + "auxiliary_loss_mlp": 0.01043596, + "balance_loss_clip": 1.05159223, + "balance_loss_mlp": 1.02380753, + "epoch": 0.3061383995671256, + "flos": 19865101173120.0, + "grad_norm": 2.1598965690387795, + "language_loss": 0.81709123, + "learning_rate": 3.2521928555743514e-06, + "loss": 0.83914375, + "num_input_tokens_seen": 54749935, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.19775391, + "step": 2546, + "time_per_iteration": 2.811014175415039 + }, + { + "auxiliary_loss_clip": 0.0116025, + "auxiliary_loss_mlp": 0.01050223, + "balance_loss_clip": 1.05014849, + "balance_loss_mlp": 1.02783561, + "epoch": 0.3062586424577647, + "flos": 22127509255680.0, + "grad_norm": 1.936123406166945, + "language_loss": 0.68228185, + "learning_rate": 3.2515853608657e-06, + "loss": 0.70438659, + "num_input_tokens_seen": 54767935, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.22387695, + "step": 2547, + "time_per_iteration": 2.8568649291992188 + }, + { + "auxiliary_loss_clip": 0.01159803, + "auxiliary_loss_mlp": 0.01046994, + "balance_loss_clip": 1.05123305, + "balance_loss_mlp": 1.02689552, + "epoch": 0.3063788853484038, + "flos": 20845497962880.0, + "grad_norm": 2.260407662933, + "language_loss": 0.74844784, + "learning_rate": 3.250977676292545e-06, + "loss": 0.7705158, + "num_input_tokens_seen": 54786175, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.2010498, + "step": 2548, + "time_per_iteration": 2.799617052078247 + }, + { + "auxiliary_loss_clip": 0.01156821, + "auxiliary_loss_mlp": 0.01044495, + "balance_loss_clip": 1.04983568, + "balance_loss_mlp": 1.02377689, + "epoch": 0.30649912823904285, + "flos": 16209717707520.0, + "grad_norm": 3.100518303768853, + "language_loss": 0.79325819, + "learning_rate": 3.2503698019470712e-06, + "loss": 0.81527138, + "num_input_tokens_seen": 54801945, + "router_z_loss_clip": 1.06835938, + "router_z_loss_mlp": 0.20715332, + "step": 2549, + "time_per_iteration": 2.845599889755249 + }, + { + "auxiliary_loss_clip": 0.01159577, + "auxiliary_loss_mlp": 0.01042154, + "balance_loss_clip": 1.05050421, + "balance_loss_mlp": 1.02197146, + "epoch": 0.30661937112968196, + "flos": 18617815353600.0, + "grad_norm": 2.161655615519267, + "language_loss": 0.78087407, + "learning_rate": 3.249761737921492e-06, + "loss": 0.80289137, + "num_input_tokens_seen": 54818475, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.20166016, + "step": 2550, + "time_per_iteration": 2.7942986488342285 + }, + { + "auxiliary_loss_clip": 0.01155462, + "auxiliary_loss_mlp": 0.01044179, + "balance_loss_clip": 1.04844582, + "balance_loss_mlp": 1.02535582, + "epoch": 0.30673961402032107, + "flos": 31390809638400.0, + "grad_norm": 2.2700946452992867, + "language_loss": 0.74468881, + "learning_rate": 3.249153484308051e-06, + "loss": 0.76668525, + "num_input_tokens_seen": 54837090, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.18823242, + "step": 2551, + "time_per_iteration": 2.904930830001831 + }, + { + "auxiliary_loss_clip": 0.01157495, + "auxiliary_loss_mlp": 0.0104431, + "balance_loss_clip": 1.04963946, + "balance_loss_mlp": 1.02433062, + "epoch": 0.3068598569109601, + "flos": 20229809915520.0, + "grad_norm": 2.143269990252073, + "language_loss": 0.78186697, + "learning_rate": 3.2485450411990194e-06, + "loss": 0.80388504, + "num_input_tokens_seen": 54856445, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.19958496, + "step": 2552, + "time_per_iteration": 2.9297356605529785 + }, + { + "auxiliary_loss_clip": 0.011608, + "auxiliary_loss_mlp": 0.01050622, + "balance_loss_clip": 1.0489068, + "balance_loss_mlp": 1.03126216, + "epoch": 0.30698009980159924, + "flos": 29601991399680.0, + "grad_norm": 1.9121137588595494, + "language_loss": 0.82532144, + "learning_rate": 3.2479364086866983e-06, + "loss": 0.84743571, + "num_input_tokens_seen": 54876700, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.19348145, + "step": 2553, + "time_per_iteration": 3.008894681930542 + }, + { + "auxiliary_loss_clip": 0.01161588, + "auxiliary_loss_mlp": 0.01046812, + "balance_loss_clip": 1.05385518, + "balance_loss_mlp": 1.02671337, + "epoch": 0.30710034269223835, + "flos": 23842423261440.0, + "grad_norm": 1.8506644629507791, + "language_loss": 0.8180722, + "learning_rate": 3.247327586863416e-06, + "loss": 0.84015614, + "num_input_tokens_seen": 54897580, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.2010498, + "step": 2554, + "time_per_iteration": 2.9140734672546387 + }, + { + "auxiliary_loss_clip": 0.01159552, + "auxiliary_loss_mlp": 0.01051869, + "balance_loss_clip": 1.0488925, + "balance_loss_mlp": 1.03168654, + "epoch": 0.3072205855828774, + "flos": 25884986152320.0, + "grad_norm": 2.122299203303495, + "language_loss": 0.77468324, + "learning_rate": 3.2467185758215304e-06, + "loss": 0.79679751, + "num_input_tokens_seen": 54917320, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.2019043, + "step": 2555, + "time_per_iteration": 2.875635862350464 + }, + { + "auxiliary_loss_clip": 0.01156992, + "auxiliary_loss_mlp": 0.01044448, + "balance_loss_clip": 1.04999423, + "balance_loss_mlp": 1.02582705, + "epoch": 0.3073408284735165, + "flos": 22236390357120.0, + "grad_norm": 4.346457342630078, + "language_loss": 0.85863703, + "learning_rate": 3.246109375653428e-06, + "loss": 0.88065147, + "num_input_tokens_seen": 54934085, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.1862793, + "step": 2556, + "time_per_iteration": 2.925072431564331 + }, + { + "auxiliary_loss_clip": 0.01157208, + "auxiliary_loss_mlp": 0.01049594, + "balance_loss_clip": 1.04836309, + "balance_loss_mlp": 1.02959037, + "epoch": 0.30746107136415557, + "flos": 19500284689920.0, + "grad_norm": 2.4289266367867532, + "language_loss": 0.78627992, + "learning_rate": 3.2454999864515243e-06, + "loss": 0.80834794, + "num_input_tokens_seen": 54953460, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.20007324, + "step": 2557, + "time_per_iteration": 2.792776584625244 + }, + { + "auxiliary_loss_clip": 0.01158487, + "auxiliary_loss_mlp": 0.01046046, + "balance_loss_clip": 1.04970133, + "balance_loss_mlp": 1.0245285, + "epoch": 0.3075813142547947, + "flos": 21724806902400.0, + "grad_norm": 1.8759054647056828, + "language_loss": 0.69576579, + "learning_rate": 3.244890408308263e-06, + "loss": 0.71781111, + "num_input_tokens_seen": 54974165, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.21520996, + "step": 2558, + "time_per_iteration": 4.418939828872681 + }, + { + "auxiliary_loss_clip": 0.01157039, + "auxiliary_loss_mlp": 0.01049535, + "balance_loss_clip": 1.0497961, + "balance_loss_mlp": 1.02965081, + "epoch": 0.3077015571454338, + "flos": 24097963593600.0, + "grad_norm": 2.3711451660562664, + "language_loss": 0.61130488, + "learning_rate": 3.2442806413161165e-06, + "loss": 0.63337064, + "num_input_tokens_seen": 54993810, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.19885254, + "step": 2559, + "time_per_iteration": 2.789189577102661 + }, + { + "auxiliary_loss_clip": 0.01164008, + "auxiliary_loss_mlp": 0.01043927, + "balance_loss_clip": 1.05036747, + "balance_loss_mlp": 1.023435, + "epoch": 0.30782180003607285, + "flos": 18405476104320.0, + "grad_norm": 2.2186454607155146, + "language_loss": 0.75608921, + "learning_rate": 3.243670685567586e-06, + "loss": 0.77816856, + "num_input_tokens_seen": 55011210, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.20507812, + "step": 2560, + "time_per_iteration": 2.800323724746704 + }, + { + "auxiliary_loss_clip": 0.01154559, + "auxiliary_loss_mlp": 0.01051886, + "balance_loss_clip": 1.04755902, + "balance_loss_mlp": 1.03233528, + "epoch": 0.30794204292671196, + "flos": 23878549365120.0, + "grad_norm": 2.1592659081033965, + "language_loss": 0.80275029, + "learning_rate": 3.2430605411552012e-06, + "loss": 0.82481474, + "num_input_tokens_seen": 55031325, + "router_z_loss_clip": 1.06787109, + "router_z_loss_mlp": 0.19555664, + "step": 2561, + "time_per_iteration": 4.674056529998779 + }, + { + "auxiliary_loss_clip": 0.01065551, + "auxiliary_loss_mlp": 0.01013318, + "balance_loss_clip": 1.02591646, + "balance_loss_mlp": 1.01026654, + "epoch": 0.30806228581735107, + "flos": 67927800816000.0, + "grad_norm": 0.8963262162871999, + "language_loss": 0.70617735, + "learning_rate": 3.2424502081715205e-06, + "loss": 0.72696614, + "num_input_tokens_seen": 55094440, + "router_z_loss_clip": 0.39648438, + "router_z_loss_mlp": 0.03051758, + "step": 2562, + "time_per_iteration": 3.3968710899353027 + }, + { + "auxiliary_loss_clip": 0.011595, + "auxiliary_loss_mlp": 0.01044684, + "balance_loss_clip": 1.04994726, + "balance_loss_mlp": 1.02448988, + "epoch": 0.3081825287079901, + "flos": 23843213360640.0, + "grad_norm": 2.0808722340779764, + "language_loss": 0.78397131, + "learning_rate": 3.241839686709132e-06, + "loss": 0.80601317, + "num_input_tokens_seen": 55115375, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.20178223, + "step": 2563, + "time_per_iteration": 2.976498603820801 + }, + { + "auxiliary_loss_clip": 0.01163717, + "auxiliary_loss_mlp": 0.01050407, + "balance_loss_clip": 1.0516746, + "balance_loss_mlp": 1.02946222, + "epoch": 0.30830277159862923, + "flos": 16209969102720.0, + "grad_norm": 3.257349384715355, + "language_loss": 0.82564688, + "learning_rate": 3.2412289768606495e-06, + "loss": 0.8477881, + "num_input_tokens_seen": 55131945, + "router_z_loss_clip": 1.11914062, + "router_z_loss_mlp": 0.20947266, + "step": 2564, + "time_per_iteration": 2.889604091644287 + }, + { + "auxiliary_loss_clip": 0.0116161, + "auxiliary_loss_mlp": 0.01055075, + "balance_loss_clip": 1.05253983, + "balance_loss_mlp": 1.03395081, + "epoch": 0.30842301448926834, + "flos": 29349503723520.0, + "grad_norm": 9.405881558541966, + "language_loss": 0.82907289, + "learning_rate": 3.240618078718718e-06, + "loss": 0.85123968, + "num_input_tokens_seen": 55153405, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.21105957, + "step": 2565, + "time_per_iteration": 2.986713409423828 + }, + { + "auxiliary_loss_clip": 0.01166527, + "auxiliary_loss_mlp": 0.01048773, + "balance_loss_clip": 1.05302596, + "balance_loss_mlp": 1.02936554, + "epoch": 0.3085432573799074, + "flos": 21945190798080.0, + "grad_norm": 2.208444568017617, + "language_loss": 0.74631745, + "learning_rate": 3.240006992376011e-06, + "loss": 0.76847041, + "num_input_tokens_seen": 55173030, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.1940918, + "step": 2566, + "time_per_iteration": 2.822091579437256 + }, + { + "auxiliary_loss_clip": 0.01161259, + "auxiliary_loss_mlp": 0.01050659, + "balance_loss_clip": 1.05321646, + "balance_loss_mlp": 1.03197885, + "epoch": 0.3086635002705465, + "flos": 22054718344320.0, + "grad_norm": 2.662977864273441, + "language_loss": 0.75988865, + "learning_rate": 3.2393957179252284e-06, + "loss": 0.78200781, + "num_input_tokens_seen": 55189565, + "router_z_loss_clip": 1.08056641, + "router_z_loss_mlp": 0.18701172, + "step": 2567, + "time_per_iteration": 2.897238254547119 + }, + { + "auxiliary_loss_clip": 0.01165618, + "auxiliary_loss_mlp": 0.01049006, + "balance_loss_clip": 1.05448043, + "balance_loss_mlp": 1.02919292, + "epoch": 0.3087837431611856, + "flos": 32665925520000.0, + "grad_norm": 17.452200391949845, + "language_loss": 0.80709797, + "learning_rate": 3.2387842554591016e-06, + "loss": 0.82924426, + "num_input_tokens_seen": 55210380, + "router_z_loss_clip": 1.11083984, + "router_z_loss_mlp": 0.19824219, + "step": 2568, + "time_per_iteration": 2.9238440990448 + }, + { + "auxiliary_loss_clip": 0.0116473, + "auxiliary_loss_mlp": 0.0105519, + "balance_loss_clip": 1.0533551, + "balance_loss_mlp": 1.0343405, + "epoch": 0.3089039860518247, + "flos": 17599245384960.0, + "grad_norm": 2.9480138285831625, + "language_loss": 0.88046002, + "learning_rate": 3.238172605070388e-06, + "loss": 0.90265918, + "num_input_tokens_seen": 55225795, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.20837402, + "step": 2569, + "time_per_iteration": 2.855623483657837 + }, + { + "auxiliary_loss_clip": 0.01163856, + "auxiliary_loss_mlp": 0.01045591, + "balance_loss_clip": 1.05112767, + "balance_loss_mlp": 1.0245626, + "epoch": 0.3090242289424638, + "flos": 14383839611520.0, + "grad_norm": 2.6939481451582465, + "language_loss": 0.78774941, + "learning_rate": 3.2375607668518745e-06, + "loss": 0.8098439, + "num_input_tokens_seen": 55238830, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.21044922, + "step": 2570, + "time_per_iteration": 2.9194626808166504 + }, + { + "auxiliary_loss_clip": 0.01160989, + "auxiliary_loss_mlp": 0.01042524, + "balance_loss_clip": 1.05172193, + "balance_loss_mlp": 1.02303314, + "epoch": 0.30914447183310284, + "flos": 16068625084800.0, + "grad_norm": 2.808273104783963, + "language_loss": 0.90204376, + "learning_rate": 3.236948740896377e-06, + "loss": 0.92407888, + "num_input_tokens_seen": 55253630, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.19494629, + "step": 2571, + "time_per_iteration": 2.8538379669189453 + }, + { + "auxiliary_loss_clip": 0.01161314, + "auxiliary_loss_mlp": 0.01048873, + "balance_loss_clip": 1.05038106, + "balance_loss_mlp": 1.02996612, + "epoch": 0.30926471472374195, + "flos": 32230221546240.0, + "grad_norm": 1.6457983223692894, + "language_loss": 0.84430999, + "learning_rate": 3.2363365272967384e-06, + "loss": 0.86641181, + "num_input_tokens_seen": 55276200, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.18908691, + "step": 2572, + "time_per_iteration": 2.883613348007202 + }, + { + "auxiliary_loss_clip": 0.0116294, + "auxiliary_loss_mlp": 0.0104767, + "balance_loss_clip": 1.05284929, + "balance_loss_mlp": 1.02850151, + "epoch": 0.30938495761438106, + "flos": 20370722970240.0, + "grad_norm": 3.208850029417624, + "language_loss": 0.81945145, + "learning_rate": 3.235724126145832e-06, + "loss": 0.8415575, + "num_input_tokens_seen": 55292235, + "router_z_loss_clip": 1.09960938, + "router_z_loss_mlp": 0.19165039, + "step": 2573, + "time_per_iteration": 2.776961088180542 + }, + { + "auxiliary_loss_clip": 0.01157151, + "auxiliary_loss_mlp": 0.01043672, + "balance_loss_clip": 1.0487715, + "balance_loss_mlp": 1.02378774, + "epoch": 0.3095052005050201, + "flos": 24061155131520.0, + "grad_norm": 1.5868179899548989, + "language_loss": 0.77702147, + "learning_rate": 3.235111537536558e-06, + "loss": 0.79902971, + "num_input_tokens_seen": 55313050, + "router_z_loss_clip": 1.08349609, + "router_z_loss_mlp": 0.19885254, + "step": 2574, + "time_per_iteration": 2.8607330322265625 + }, + { + "auxiliary_loss_clip": 0.01156915, + "auxiliary_loss_mlp": 0.01041904, + "balance_loss_clip": 1.0487144, + "balance_loss_mlp": 1.02354634, + "epoch": 0.30962544339565923, + "flos": 23401547729280.0, + "grad_norm": 3.0894245438499115, + "language_loss": 0.82794559, + "learning_rate": 3.2344987615618456e-06, + "loss": 0.84993386, + "num_input_tokens_seen": 55332885, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.18334961, + "step": 2575, + "time_per_iteration": 2.8379437923431396 + }, + { + "auxiliary_loss_clip": 0.01155061, + "auxiliary_loss_mlp": 0.01039041, + "balance_loss_clip": 1.05090392, + "balance_loss_mlp": 1.0205282, + "epoch": 0.30974568628629834, + "flos": 33799984692480.0, + "grad_norm": 3.2914223764662753, + "language_loss": 0.79011256, + "learning_rate": 3.2338857983146533e-06, + "loss": 0.81205356, + "num_input_tokens_seen": 55354385, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.18505859, + "step": 2576, + "time_per_iteration": 3.0308303833007812 + }, + { + "auxiliary_loss_clip": 0.01162455, + "auxiliary_loss_mlp": 0.01045801, + "balance_loss_clip": 1.05129385, + "balance_loss_mlp": 1.02516532, + "epoch": 0.3098659291769374, + "flos": 20229594433920.0, + "grad_norm": 2.091109268435138, + "language_loss": 0.76586092, + "learning_rate": 3.233272647887966e-06, + "loss": 0.78794348, + "num_input_tokens_seen": 55373275, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.2064209, + "step": 2577, + "time_per_iteration": 2.7903342247009277 + }, + { + "auxiliary_loss_clip": 0.01164188, + "auxiliary_loss_mlp": 0.01046082, + "balance_loss_clip": 1.05368006, + "balance_loss_mlp": 1.02564955, + "epoch": 0.3099861720675765, + "flos": 24748556682240.0, + "grad_norm": 3.8791384856889617, + "language_loss": 0.90294385, + "learning_rate": 3.2326593103747985e-06, + "loss": 0.9250465, + "num_input_tokens_seen": 55392290, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.20446777, + "step": 2578, + "time_per_iteration": 2.841557025909424 + }, + { + "auxiliary_loss_clip": 0.01162897, + "auxiliary_loss_mlp": 0.01042778, + "balance_loss_clip": 1.0538373, + "balance_loss_mlp": 1.02296519, + "epoch": 0.3101064149582156, + "flos": 11765485704960.0, + "grad_norm": 2.1924535532533236, + "language_loss": 0.85123628, + "learning_rate": 3.2320457858681936e-06, + "loss": 0.87329304, + "num_input_tokens_seen": 55410680, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.19812012, + "step": 2579, + "time_per_iteration": 2.7869057655334473 + }, + { + "auxiliary_loss_clip": 0.01157782, + "auxiliary_loss_mlp": 0.01048172, + "balance_loss_clip": 1.05028355, + "balance_loss_mlp": 1.02962303, + "epoch": 0.31022665784885467, + "flos": 23033247626880.0, + "grad_norm": 2.5777933861168525, + "language_loss": 0.85428143, + "learning_rate": 3.2314320744612228e-06, + "loss": 0.87634099, + "num_input_tokens_seen": 55425980, + "router_z_loss_clip": 1.07617188, + "router_z_loss_mlp": 0.18530273, + "step": 2580, + "time_per_iteration": 2.7914798259735107 + }, + { + "auxiliary_loss_clip": 0.01155589, + "auxiliary_loss_mlp": 0.01046125, + "balance_loss_clip": 1.04903376, + "balance_loss_mlp": 1.0266341, + "epoch": 0.3103469007394938, + "flos": 16289188548480.0, + "grad_norm": 1.5934144260106144, + "language_loss": 0.76091486, + "learning_rate": 3.2308181762469854e-06, + "loss": 0.78293204, + "num_input_tokens_seen": 55443925, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.19506836, + "step": 2581, + "time_per_iteration": 2.8348870277404785 + }, + { + "auxiliary_loss_clip": 0.01168279, + "auxiliary_loss_mlp": 0.01044708, + "balance_loss_clip": 1.05227232, + "balance_loss_mlp": 1.02437115, + "epoch": 0.3104671436301329, + "flos": 30515271626880.0, + "grad_norm": 3.8150562477151286, + "language_loss": 0.78640848, + "learning_rate": 3.230204091318609e-06, + "loss": 0.80853832, + "num_input_tokens_seen": 55464465, + "router_z_loss_clip": 1.16015625, + "router_z_loss_mlp": 0.20349121, + "step": 2582, + "time_per_iteration": 2.841193437576294 + }, + { + "auxiliary_loss_clip": 0.01162447, + "auxiliary_loss_mlp": 0.01044945, + "balance_loss_clip": 1.05173373, + "balance_loss_mlp": 1.02628875, + "epoch": 0.31058738652077195, + "flos": 20047240062720.0, + "grad_norm": 1.9090471387307355, + "language_loss": 0.84871298, + "learning_rate": 3.2295898197692503e-06, + "loss": 0.87078691, + "num_input_tokens_seen": 55483425, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.18664551, + "step": 2583, + "time_per_iteration": 2.8683342933654785 + }, + { + "auxiliary_loss_clip": 0.01161961, + "auxiliary_loss_mlp": 0.01048831, + "balance_loss_clip": 1.05042386, + "balance_loss_mlp": 1.02894652, + "epoch": 0.31070762941141106, + "flos": 28074639237120.0, + "grad_norm": 3.0046646894834264, + "language_loss": 0.79567361, + "learning_rate": 3.228975361692094e-06, + "loss": 0.81778145, + "num_input_tokens_seen": 55504445, + "router_z_loss_clip": 1.11425781, + "router_z_loss_mlp": 0.19873047, + "step": 2584, + "time_per_iteration": 2.884227991104126 + }, + { + "auxiliary_loss_clip": 0.01160121, + "auxiliary_loss_mlp": 0.01048997, + "balance_loss_clip": 1.04822528, + "balance_loss_mlp": 1.02814746, + "epoch": 0.31082787230205017, + "flos": 20521907314560.0, + "grad_norm": 2.2626001600099452, + "language_loss": 0.80445504, + "learning_rate": 3.228360717180352e-06, + "loss": 0.82654619, + "num_input_tokens_seen": 55521970, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.20849609, + "step": 2585, + "time_per_iteration": 2.852613687515259 + }, + { + "auxiliary_loss_clip": 0.01070397, + "auxiliary_loss_mlp": 0.01000918, + "balance_loss_clip": 1.03111601, + "balance_loss_mlp": 0.99804479, + "epoch": 0.3109481151926892, + "flos": 62445928723200.0, + "grad_norm": 0.8343765906387725, + "language_loss": 0.59343922, + "learning_rate": 3.227745886327266e-06, + "loss": 0.61415237, + "num_input_tokens_seen": 55580665, + "router_z_loss_clip": 0.39257812, + "router_z_loss_mlp": 0.02868652, + "step": 2586, + "time_per_iteration": 4.0961058139801025 + }, + { + "auxiliary_loss_clip": 0.01070372, + "auxiliary_loss_mlp": 0.00998621, + "balance_loss_clip": 1.03125143, + "balance_loss_mlp": 0.99598616, + "epoch": 0.31106835808332833, + "flos": 44746744723200.0, + "grad_norm": 0.8103146962454459, + "language_loss": 0.55878699, + "learning_rate": 3.227130869226105e-06, + "loss": 0.57947695, + "num_input_tokens_seen": 55637825, + "router_z_loss_clip": 0.390625, + "router_z_loss_mlp": 0.02636719, + "step": 2587, + "time_per_iteration": 4.203127145767212 + }, + { + "auxiliary_loss_clip": 0.01162788, + "auxiliary_loss_mlp": 0.01048762, + "balance_loss_clip": 1.0524708, + "balance_loss_mlp": 1.02936661, + "epoch": 0.3111886009739674, + "flos": 23403056100480.0, + "grad_norm": 2.562597258601231, + "language_loss": 0.82557976, + "learning_rate": 3.226515665970167e-06, + "loss": 0.84769523, + "num_input_tokens_seen": 55655365, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.19384766, + "step": 2588, + "time_per_iteration": 2.8407974243164062 + }, + { + "auxiliary_loss_clip": 0.01159132, + "auxiliary_loss_mlp": 0.0105262, + "balance_loss_clip": 1.0482651, + "balance_loss_mlp": 1.03172278, + "epoch": 0.3113088438646065, + "flos": 17530728192000.0, + "grad_norm": 2.767142505790245, + "language_loss": 0.86252081, + "learning_rate": 3.225900276652777e-06, + "loss": 0.88463831, + "num_input_tokens_seen": 55672140, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.20910645, + "step": 2589, + "time_per_iteration": 2.8615329265594482 + }, + { + "auxiliary_loss_clip": 0.01159072, + "auxiliary_loss_mlp": 0.01061719, + "balance_loss_clip": 1.05033898, + "balance_loss_mlp": 1.04333687, + "epoch": 0.3114290867552456, + "flos": 28365802882560.0, + "grad_norm": 1.7121173764058646, + "language_loss": 0.75526983, + "learning_rate": 3.2252847013672906e-06, + "loss": 0.77747774, + "num_input_tokens_seen": 55694800, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.18371582, + "step": 2590, + "time_per_iteration": 4.589345932006836 + }, + { + "auxiliary_loss_clip": 0.0115738, + "auxiliary_loss_mlp": 0.01053376, + "balance_loss_clip": 1.04929543, + "balance_loss_mlp": 1.03430247, + "epoch": 0.31154932964588467, + "flos": 27379157126400.0, + "grad_norm": 2.6375529293313873, + "language_loss": 0.75940263, + "learning_rate": 3.224668940207089e-06, + "loss": 0.78151023, + "num_input_tokens_seen": 55713785, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.1907959, + "step": 2591, + "time_per_iteration": 2.8300187587738037 + }, + { + "auxiliary_loss_clip": 0.01164878, + "auxiliary_loss_mlp": 0.01056352, + "balance_loss_clip": 1.0529269, + "balance_loss_mlp": 1.03725493, + "epoch": 0.3116695725365238, + "flos": 26541864120960.0, + "grad_norm": 2.0911461460484806, + "language_loss": 0.87277758, + "learning_rate": 3.2240529932655828e-06, + "loss": 0.89498991, + "num_input_tokens_seen": 55733050, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.19104004, + "step": 2592, + "time_per_iteration": 2.8311939239501953 + }, + { + "auxiliary_loss_clip": 0.0116047, + "auxiliary_loss_mlp": 0.0105693, + "balance_loss_clip": 1.05229068, + "balance_loss_mlp": 1.03745103, + "epoch": 0.3117898154271629, + "flos": 21177600134400.0, + "grad_norm": 4.207203137987135, + "language_loss": 0.88726854, + "learning_rate": 3.223436860636211e-06, + "loss": 0.90944254, + "num_input_tokens_seen": 55748685, + "router_z_loss_clip": 1.08105469, + "router_z_loss_mlp": 0.19482422, + "step": 2593, + "time_per_iteration": 2.8021562099456787 + }, + { + "auxiliary_loss_clip": 0.01159712, + "auxiliary_loss_mlp": 0.01061712, + "balance_loss_clip": 1.04983115, + "balance_loss_mlp": 1.04125547, + "epoch": 0.31191005831780194, + "flos": 27272430840960.0, + "grad_norm": 1.8116831957796433, + "language_loss": 0.74136239, + "learning_rate": 3.2228205424124403e-06, + "loss": 0.76357663, + "num_input_tokens_seen": 55771840, + "router_z_loss_clip": 1.09863281, + "router_z_loss_mlp": 0.20446777, + "step": 2594, + "time_per_iteration": 2.862091302871704 + }, + { + "auxiliary_loss_clip": 0.01158529, + "auxiliary_loss_mlp": 0.01068479, + "balance_loss_clip": 1.05007911, + "balance_loss_mlp": 1.04814148, + "epoch": 0.31203030120844105, + "flos": 12963501043200.0, + "grad_norm": 6.1602202611468755, + "language_loss": 0.74871492, + "learning_rate": 3.222204038687765e-06, + "loss": 0.77098495, + "num_input_tokens_seen": 55784975, + "router_z_loss_clip": 1.08398438, + "router_z_loss_mlp": 0.20336914, + "step": 2595, + "time_per_iteration": 2.8085789680480957 + }, + { + "auxiliary_loss_clip": 0.01154999, + "auxiliary_loss_mlp": 0.01056235, + "balance_loss_clip": 1.04916739, + "balance_loss_mlp": 1.03790069, + "epoch": 0.31215054409908016, + "flos": 27562014288000.0, + "grad_norm": 1.7918171402948255, + "language_loss": 0.88117278, + "learning_rate": 3.221587349555709e-06, + "loss": 0.90328509, + "num_input_tokens_seen": 55805235, + "router_z_loss_clip": 1.06005859, + "router_z_loss_mlp": 0.18334961, + "step": 2596, + "time_per_iteration": 2.8394744396209717 + }, + { + "auxiliary_loss_clip": 0.01156475, + "auxiliary_loss_mlp": 0.01059531, + "balance_loss_clip": 1.04921448, + "balance_loss_mlp": 1.04031491, + "epoch": 0.3122707869897192, + "flos": 21506326427520.0, + "grad_norm": 2.0757369875815117, + "language_loss": 0.69388986, + "learning_rate": 3.2209704751098236e-06, + "loss": 0.71604985, + "num_input_tokens_seen": 55824265, + "router_z_loss_clip": 1.07373047, + "router_z_loss_mlp": 0.19226074, + "step": 2597, + "time_per_iteration": 2.888291358947754 + }, + { + "auxiliary_loss_clip": 0.01165039, + "auxiliary_loss_mlp": 0.0107137, + "balance_loss_clip": 1.05310941, + "balance_loss_mlp": 1.05108058, + "epoch": 0.31239102988035833, + "flos": 15187017674880.0, + "grad_norm": 2.6994958509553424, + "language_loss": 0.82996464, + "learning_rate": 3.2203534154436875e-06, + "loss": 0.85232878, + "num_input_tokens_seen": 55838620, + "router_z_loss_clip": 1.11914062, + "router_z_loss_mlp": 0.20275879, + "step": 2598, + "time_per_iteration": 2.8134474754333496 + }, + { + "auxiliary_loss_clip": 0.01158049, + "auxiliary_loss_mlp": 0.01060237, + "balance_loss_clip": 1.04952919, + "balance_loss_mlp": 1.04041219, + "epoch": 0.31251127277099744, + "flos": 22053712763520.0, + "grad_norm": 2.1476436108779584, + "language_loss": 0.75626695, + "learning_rate": 3.2197361706509084e-06, + "loss": 0.77844977, + "num_input_tokens_seen": 55859375, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.19848633, + "step": 2599, + "time_per_iteration": 3.028909683227539 + }, + { + "auxiliary_loss_clip": 0.0116841, + "auxiliary_loss_mlp": 0.01062356, + "balance_loss_clip": 1.05321217, + "balance_loss_mlp": 1.04008746, + "epoch": 0.3126315156616365, + "flos": 15193984913280.0, + "grad_norm": 3.3391675611330647, + "language_loss": 0.840056, + "learning_rate": 3.2191187408251228e-06, + "loss": 0.8623637, + "num_input_tokens_seen": 55876535, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.22253418, + "step": 2600, + "time_per_iteration": 2.956977367401123 + }, + { + "auxiliary_loss_clip": 0.01163918, + "auxiliary_loss_mlp": 0.01060725, + "balance_loss_clip": 1.04964185, + "balance_loss_mlp": 1.03981543, + "epoch": 0.3127517585522756, + "flos": 18145338831360.0, + "grad_norm": 2.411539276240448, + "language_loss": 0.78748715, + "learning_rate": 3.218501126059993e-06, + "loss": 0.80973363, + "num_input_tokens_seen": 55891930, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.20910645, + "step": 2601, + "time_per_iteration": 2.795820474624634 + }, + { + "auxiliary_loss_clip": 0.01156146, + "auxiliary_loss_mlp": 0.01055114, + "balance_loss_clip": 1.04782987, + "balance_loss_mlp": 1.03561139, + "epoch": 0.31287200144291466, + "flos": 21908633731200.0, + "grad_norm": 1.9491189268769167, + "language_loss": 0.81374848, + "learning_rate": 3.2178833264492116e-06, + "loss": 0.83586109, + "num_input_tokens_seen": 55910635, + "router_z_loss_clip": 1.08447266, + "router_z_loss_mlp": 0.19494629, + "step": 2602, + "time_per_iteration": 2.808051586151123 + }, + { + "auxiliary_loss_clip": 0.01165335, + "auxiliary_loss_mlp": 0.0104911, + "balance_loss_clip": 1.05343318, + "balance_loss_mlp": 1.02802157, + "epoch": 0.31299224433355377, + "flos": 29896997800320.0, + "grad_norm": 2.565481813953422, + "language_loss": 0.75827664, + "learning_rate": 3.217265342086498e-06, + "loss": 0.78042114, + "num_input_tokens_seen": 55931125, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.2109375, + "step": 2603, + "time_per_iteration": 2.948176622390747 + }, + { + "auxiliary_loss_clip": 0.01162999, + "auxiliary_loss_mlp": 0.01047623, + "balance_loss_clip": 1.05011594, + "balance_loss_mlp": 1.02764344, + "epoch": 0.3131124872241929, + "flos": 11655886331520.0, + "grad_norm": 8.1343875560714, + "language_loss": 0.73745126, + "learning_rate": 3.216647173065599e-06, + "loss": 0.75955749, + "num_input_tokens_seen": 55946590, + "router_z_loss_clip": 1.12890625, + "router_z_loss_mlp": 0.1998291, + "step": 2604, + "time_per_iteration": 2.8842039108276367 + }, + { + "auxiliary_loss_clip": 0.01161278, + "auxiliary_loss_mlp": 0.01048777, + "balance_loss_clip": 1.05256104, + "balance_loss_mlp": 1.02902389, + "epoch": 0.31323273011483194, + "flos": 49848785470080.0, + "grad_norm": 2.026225176854911, + "language_loss": 0.74147278, + "learning_rate": 3.216028819480292e-06, + "loss": 0.76357335, + "num_input_tokens_seen": 55967930, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.1973877, + "step": 2605, + "time_per_iteration": 3.2712080478668213 + }, + { + "auxiliary_loss_clip": 0.01158161, + "auxiliary_loss_mlp": 0.01042429, + "balance_loss_clip": 1.05041456, + "balance_loss_mlp": 1.02302194, + "epoch": 0.31335297300547105, + "flos": 22601278667520.0, + "grad_norm": 4.606080661633224, + "language_loss": 0.75536734, + "learning_rate": 3.2154102814243793e-06, + "loss": 0.77737319, + "num_input_tokens_seen": 55987070, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.19396973, + "step": 2606, + "time_per_iteration": 2.906282663345337 + }, + { + "auxiliary_loss_clip": 0.01163101, + "auxiliary_loss_mlp": 0.01052169, + "balance_loss_clip": 1.05390167, + "balance_loss_mlp": 1.03359663, + "epoch": 0.31347321589611016, + "flos": 34710858708480.0, + "grad_norm": 2.947332170949834, + "language_loss": 0.67173958, + "learning_rate": 3.2147915589916937e-06, + "loss": 0.69389224, + "num_input_tokens_seen": 56008630, + "router_z_loss_clip": 1.09179688, + "router_z_loss_mlp": 0.18591309, + "step": 2607, + "time_per_iteration": 2.9445242881774902 + }, + { + "auxiliary_loss_clip": 0.01158516, + "auxiliary_loss_mlp": 0.01044643, + "balance_loss_clip": 1.04809999, + "balance_loss_mlp": 1.02527153, + "epoch": 0.3135934587867492, + "flos": 19755789108480.0, + "grad_norm": 1.9315020158002112, + "language_loss": 0.83047533, + "learning_rate": 3.2141726522760938e-06, + "loss": 0.85250688, + "num_input_tokens_seen": 56026690, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.19372559, + "step": 2608, + "time_per_iteration": 2.8419029712677 + }, + { + "auxiliary_loss_clip": 0.01062672, + "auxiliary_loss_mlp": 0.01035426, + "balance_loss_clip": 1.02428508, + "balance_loss_mlp": 1.03308916, + "epoch": 0.3137137016773883, + "flos": 65815535583360.0, + "grad_norm": 0.7266617014862238, + "language_loss": 0.52582228, + "learning_rate": 3.213553561371469e-06, + "loss": 0.5468033, + "num_input_tokens_seen": 56090425, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.02331543, + "step": 2609, + "time_per_iteration": 3.43835186958313 + }, + { + "auxiliary_loss_clip": 0.01158863, + "auxiliary_loss_mlp": 0.0104637, + "balance_loss_clip": 1.05180728, + "balance_loss_mlp": 1.02726078, + "epoch": 0.31383394456802743, + "flos": 16252739222400.0, + "grad_norm": 2.8574879871238887, + "language_loss": 0.95708859, + "learning_rate": 3.212934286371733e-06, + "loss": 0.979141, + "num_input_tokens_seen": 56107135, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.19104004, + "step": 2610, + "time_per_iteration": 2.9418129920959473 + }, + { + "auxiliary_loss_clip": 0.01166026, + "auxiliary_loss_mlp": 0.01044809, + "balance_loss_clip": 1.05259538, + "balance_loss_mlp": 1.02404261, + "epoch": 0.3139541874586665, + "flos": 38795517613440.0, + "grad_norm": 2.307683410374017, + "language_loss": 0.83125317, + "learning_rate": 3.2123148273708304e-06, + "loss": 0.85336149, + "num_input_tokens_seen": 56127325, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.2076416, + "step": 2611, + "time_per_iteration": 2.9980530738830566 + }, + { + "auxiliary_loss_clip": 0.01162757, + "auxiliary_loss_mlp": 0.01046452, + "balance_loss_clip": 1.05390668, + "balance_loss_mlp": 1.02680635, + "epoch": 0.3140744303493056, + "flos": 25046328430080.0, + "grad_norm": 2.0435303971223298, + "language_loss": 0.76713991, + "learning_rate": 3.211695184462733e-06, + "loss": 0.78923202, + "num_input_tokens_seen": 56148500, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.19665527, + "step": 2612, + "time_per_iteration": 2.811310291290283 + }, + { + "auxiliary_loss_clip": 0.01061082, + "auxiliary_loss_mlp": 0.01001398, + "balance_loss_clip": 1.02221656, + "balance_loss_mlp": 0.99906141, + "epoch": 0.3141946732399447, + "flos": 72504254782080.0, + "grad_norm": 0.8916994546642667, + "language_loss": 0.60482377, + "learning_rate": 3.2110753577414383e-06, + "loss": 0.62544858, + "num_input_tokens_seen": 56210080, + "router_z_loss_clip": 0.38867188, + "router_z_loss_mlp": 0.02331543, + "step": 2613, + "time_per_iteration": 3.2958719730377197 + }, + { + "auxiliary_loss_clip": 0.01163443, + "auxiliary_loss_mlp": 0.01049549, + "balance_loss_clip": 1.05134511, + "balance_loss_mlp": 1.02928317, + "epoch": 0.31431491613058377, + "flos": 19239788280960.0, + "grad_norm": 2.1198101782617025, + "language_loss": 0.79538751, + "learning_rate": 3.2104553473009757e-06, + "loss": 0.81751746, + "num_input_tokens_seen": 56228200, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.20275879, + "step": 2614, + "time_per_iteration": 2.8298583030700684 + }, + { + "auxiliary_loss_clip": 0.01161417, + "auxiliary_loss_mlp": 0.01052418, + "balance_loss_clip": 1.05082273, + "balance_loss_mlp": 1.03054345, + "epoch": 0.3144351590212229, + "flos": 36210596290560.0, + "grad_norm": 2.671173337374186, + "language_loss": 0.67910194, + "learning_rate": 3.209835153235399e-06, + "loss": 0.7012403, + "num_input_tokens_seen": 56249755, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.21850586, + "step": 2615, + "time_per_iteration": 3.7319912910461426 + }, + { + "auxiliary_loss_clip": 0.01159734, + "auxiliary_loss_mlp": 0.01042884, + "balance_loss_clip": 1.05454135, + "balance_loss_mlp": 1.02316654, + "epoch": 0.314555401911862, + "flos": 18551740285440.0, + "grad_norm": 2.1530029707718845, + "language_loss": 0.67598188, + "learning_rate": 3.2092147756387916e-06, + "loss": 0.69800806, + "num_input_tokens_seen": 56270080, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.19714355, + "step": 2616, + "time_per_iteration": 3.667949914932251 + }, + { + "auxiliary_loss_clip": 0.01168206, + "auxiliary_loss_mlp": 0.0105177, + "balance_loss_clip": 1.05632591, + "balance_loss_mlp": 1.03092062, + "epoch": 0.31467564480250104, + "flos": 16362877299840.0, + "grad_norm": 2.4100197148695215, + "language_loss": 0.83853471, + "learning_rate": 3.208594214605264e-06, + "loss": 0.86073446, + "num_input_tokens_seen": 56288625, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.20861816, + "step": 2617, + "time_per_iteration": 2.786581039428711 + }, + { + "auxiliary_loss_clip": 0.01157699, + "auxiliary_loss_mlp": 0.01051542, + "balance_loss_clip": 1.05048132, + "balance_loss_mlp": 1.03161049, + "epoch": 0.31479588769314015, + "flos": 21652375127040.0, + "grad_norm": 3.3244966853492555, + "language_loss": 0.77367914, + "learning_rate": 3.2079734702289553e-06, + "loss": 0.79577154, + "num_input_tokens_seen": 56307520, + "router_z_loss_clip": 1.07226562, + "router_z_loss_mlp": 0.19934082, + "step": 2618, + "time_per_iteration": 3.6863272190093994 + }, + { + "auxiliary_loss_clip": 0.01061225, + "auxiliary_loss_mlp": 0.01005628, + "balance_loss_clip": 1.02272522, + "balance_loss_mlp": 1.00338733, + "epoch": 0.3149161305837792, + "flos": 66051072040320.0, + "grad_norm": 0.7984667894101345, + "language_loss": 0.60340333, + "learning_rate": 3.207352542604031e-06, + "loss": 0.62407184, + "num_input_tokens_seen": 56369855, + "router_z_loss_clip": 0.38476562, + "router_z_loss_mlp": 0.02246094, + "step": 2619, + "time_per_iteration": 4.207778215408325 + }, + { + "auxiliary_loss_clip": 0.01158723, + "auxiliary_loss_mlp": 0.01042612, + "balance_loss_clip": 1.05205321, + "balance_loss_mlp": 1.02487326, + "epoch": 0.3150363734744183, + "flos": 28987201192320.0, + "grad_norm": 1.7815426016278602, + "language_loss": 0.78473538, + "learning_rate": 3.2067314318246864e-06, + "loss": 0.80674875, + "num_input_tokens_seen": 56390570, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.17749023, + "step": 2620, + "time_per_iteration": 2.902693748474121 + }, + { + "auxiliary_loss_clip": 0.01167792, + "auxiliary_loss_mlp": 0.01049194, + "balance_loss_clip": 1.05509734, + "balance_loss_mlp": 1.02957237, + "epoch": 0.31515661636505743, + "flos": 27636600879360.0, + "grad_norm": 1.7403552731349543, + "language_loss": 0.77657807, + "learning_rate": 3.206110137985143e-06, + "loss": 0.79874802, + "num_input_tokens_seen": 56410775, + "router_z_loss_clip": 1.12597656, + "router_z_loss_mlp": 0.19641113, + "step": 2621, + "time_per_iteration": 2.93721866607666 + }, + { + "auxiliary_loss_clip": 0.01165781, + "auxiliary_loss_mlp": 0.01056571, + "balance_loss_clip": 1.05404162, + "balance_loss_mlp": 1.03514957, + "epoch": 0.3152768592556965, + "flos": 24605632465920.0, + "grad_norm": 3.9241428939921423, + "language_loss": 0.92156422, + "learning_rate": 3.2054886611796505e-06, + "loss": 0.94378781, + "num_input_tokens_seen": 56429770, + "router_z_loss_clip": 1.11621094, + "router_z_loss_mlp": 0.21435547, + "step": 2622, + "time_per_iteration": 2.8824269771575928 + }, + { + "auxiliary_loss_clip": 0.01061432, + "auxiliary_loss_mlp": 0.01009303, + "balance_loss_clip": 1.02263904, + "balance_loss_mlp": 1.00702655, + "epoch": 0.3153971021463356, + "flos": 68476908026880.0, + "grad_norm": 0.8945263113331491, + "language_loss": 0.63549042, + "learning_rate": 3.204867001502487e-06, + "loss": 0.65619779, + "num_input_tokens_seen": 56488425, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.02282715, + "step": 2623, + "time_per_iteration": 3.3435683250427246 + }, + { + "auxiliary_loss_clip": 0.01165714, + "auxiliary_loss_mlp": 0.01050846, + "balance_loss_clip": 1.05506516, + "balance_loss_mlp": 1.02980554, + "epoch": 0.3155173450369747, + "flos": 25593714766080.0, + "grad_norm": 2.604942060377477, + "language_loss": 0.81061202, + "learning_rate": 3.2042451590479567e-06, + "loss": 0.83277762, + "num_input_tokens_seen": 56508940, + "router_z_loss_clip": 1.10595703, + "router_z_loss_mlp": 0.21032715, + "step": 2624, + "time_per_iteration": 2.916790008544922 + }, + { + "auxiliary_loss_clip": 0.01160269, + "auxiliary_loss_mlp": 0.01042599, + "balance_loss_clip": 1.05280602, + "balance_loss_mlp": 1.02358544, + "epoch": 0.31563758792761376, + "flos": 24309333175680.0, + "grad_norm": 2.39546238162592, + "language_loss": 0.86729991, + "learning_rate": 3.203623133910394e-06, + "loss": 0.88932854, + "num_input_tokens_seen": 56527245, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.19006348, + "step": 2625, + "time_per_iteration": 2.856755018234253 + }, + { + "auxiliary_loss_clip": 0.01161592, + "auxiliary_loss_mlp": 0.01042852, + "balance_loss_clip": 1.05134153, + "balance_loss_mlp": 1.02355218, + "epoch": 0.31575783081825287, + "flos": 31903865550720.0, + "grad_norm": 3.5513575944955145, + "language_loss": 0.77265227, + "learning_rate": 3.203000926184158e-06, + "loss": 0.79469657, + "num_input_tokens_seen": 56546170, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.19299316, + "step": 2626, + "time_per_iteration": 2.8749210834503174 + }, + { + "auxiliary_loss_clip": 0.01161906, + "auxiliary_loss_mlp": 0.0104173, + "balance_loss_clip": 1.05234456, + "balance_loss_mlp": 1.02284682, + "epoch": 0.315878073708892, + "flos": 30810960385920.0, + "grad_norm": 6.311059891744055, + "language_loss": 0.77775192, + "learning_rate": 3.202378535963639e-06, + "loss": 0.7997883, + "num_input_tokens_seen": 56567085, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.18884277, + "step": 2627, + "time_per_iteration": 2.846869468688965 + }, + { + "auxiliary_loss_clip": 0.01166898, + "auxiliary_loss_mlp": 0.01050508, + "balance_loss_clip": 1.05201912, + "balance_loss_mlp": 1.02891898, + "epoch": 0.31599831659953104, + "flos": 22200264253440.0, + "grad_norm": 2.414396719706563, + "language_loss": 0.84297919, + "learning_rate": 3.2017559633432516e-06, + "loss": 0.86515331, + "num_input_tokens_seen": 56586715, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.21594238, + "step": 2628, + "time_per_iteration": 2.878490686416626 + }, + { + "auxiliary_loss_clip": 0.01163599, + "auxiliary_loss_mlp": 0.01056983, + "balance_loss_clip": 1.05288196, + "balance_loss_mlp": 1.0362885, + "epoch": 0.31611855949017015, + "flos": 25593463370880.0, + "grad_norm": 2.2370890802581602, + "language_loss": 0.66614056, + "learning_rate": 3.2011332084174398e-06, + "loss": 0.68834639, + "num_input_tokens_seen": 56607585, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.20666504, + "step": 2629, + "time_per_iteration": 2.8588855266571045 + }, + { + "auxiliary_loss_clip": 0.01165399, + "auxiliary_loss_mlp": 0.01045583, + "balance_loss_clip": 1.05388331, + "balance_loss_mlp": 1.02469754, + "epoch": 0.31623880238080926, + "flos": 20594087694720.0, + "grad_norm": 1.7132710986238022, + "language_loss": 0.89558083, + "learning_rate": 3.2005102712806756e-06, + "loss": 0.91769063, + "num_input_tokens_seen": 56626415, + "router_z_loss_clip": 1.11474609, + "router_z_loss_mlp": 0.2088623, + "step": 2630, + "time_per_iteration": 2.8186206817626953 + }, + { + "auxiliary_loss_clip": 0.01166722, + "auxiliary_loss_mlp": 0.01056788, + "balance_loss_clip": 1.05540431, + "balance_loss_mlp": 1.03558016, + "epoch": 0.3163590452714483, + "flos": 12784917600000.0, + "grad_norm": 3.1596345687941056, + "language_loss": 0.73215806, + "learning_rate": 3.1998871520274575e-06, + "loss": 0.7543931, + "num_input_tokens_seen": 56641750, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.21203613, + "step": 2631, + "time_per_iteration": 2.776728391647339 + }, + { + "auxiliary_loss_clip": 0.01164304, + "auxiliary_loss_mlp": 0.01044185, + "balance_loss_clip": 1.05366898, + "balance_loss_mlp": 1.02340651, + "epoch": 0.3164792881620874, + "flos": 23041292273280.0, + "grad_norm": 13.628707866919058, + "language_loss": 0.85246527, + "learning_rate": 3.199263850752312e-06, + "loss": 0.8745501, + "num_input_tokens_seen": 56662585, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.20788574, + "step": 2632, + "time_per_iteration": 2.796959161758423 + }, + { + "auxiliary_loss_clip": 0.01166651, + "auxiliary_loss_mlp": 0.01050816, + "balance_loss_clip": 1.05391431, + "balance_loss_mlp": 1.03022838, + "epoch": 0.31659953105272653, + "flos": 18296271780480.0, + "grad_norm": 2.4397254272994155, + "language_loss": 0.85759842, + "learning_rate": 3.198640367549795e-06, + "loss": 0.87977308, + "num_input_tokens_seen": 56681480, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.20581055, + "step": 2633, + "time_per_iteration": 2.815997838973999 + }, + { + "auxiliary_loss_clip": 0.0115873, + "auxiliary_loss_mlp": 0.01049022, + "balance_loss_clip": 1.05009651, + "balance_loss_mlp": 1.02750516, + "epoch": 0.3167197739433656, + "flos": 25703421880320.0, + "grad_norm": 1.824580548530921, + "language_loss": 0.8592, + "learning_rate": 3.198016702514487e-06, + "loss": 0.8812775, + "num_input_tokens_seen": 56701760, + "router_z_loss_clip": 1.08544922, + "router_z_loss_mlp": 0.21520996, + "step": 2634, + "time_per_iteration": 2.8204216957092285 + }, + { + "auxiliary_loss_clip": 0.01160479, + "auxiliary_loss_mlp": 0.01047313, + "balance_loss_clip": 1.05287433, + "balance_loss_mlp": 1.02822733, + "epoch": 0.3168400168340047, + "flos": 23546016230400.0, + "grad_norm": 1.6717493923352036, + "language_loss": 0.84663093, + "learning_rate": 3.1973928557409972e-06, + "loss": 0.86870885, + "num_input_tokens_seen": 56719800, + "router_z_loss_clip": 1.07617188, + "router_z_loss_mlp": 0.19104004, + "step": 2635, + "time_per_iteration": 2.7936792373657227 + }, + { + "auxiliary_loss_clip": 0.01160092, + "auxiliary_loss_mlp": 0.01045019, + "balance_loss_clip": 1.05403495, + "balance_loss_mlp": 1.02657735, + "epoch": 0.31696025972464376, + "flos": 28366449327360.0, + "grad_norm": 1.8570570177593893, + "language_loss": 0.71037304, + "learning_rate": 3.1967688273239636e-06, + "loss": 0.73242414, + "num_input_tokens_seen": 56739605, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.18432617, + "step": 2636, + "time_per_iteration": 2.8529133796691895 + }, + { + "auxiliary_loss_clip": 0.01160927, + "auxiliary_loss_mlp": 0.01042834, + "balance_loss_clip": 1.05146754, + "balance_loss_mlp": 1.02240181, + "epoch": 0.31708050261528287, + "flos": 16399111144320.0, + "grad_norm": 1.7985473079759784, + "language_loss": 0.82465768, + "learning_rate": 3.1961446173580503e-06, + "loss": 0.8466953, + "num_input_tokens_seen": 56756545, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.2043457, + "step": 2637, + "time_per_iteration": 2.8421576023101807 + }, + { + "auxiliary_loss_clip": 0.01164663, + "auxiliary_loss_mlp": 0.01051487, + "balance_loss_clip": 1.0555234, + "balance_loss_mlp": 1.03148389, + "epoch": 0.317200745505922, + "flos": 26212347728640.0, + "grad_norm": 64.9238032169311, + "language_loss": 0.77418667, + "learning_rate": 3.1955202259379502e-06, + "loss": 0.79634821, + "num_input_tokens_seen": 56778275, + "router_z_loss_clip": 1.09277344, + "router_z_loss_mlp": 0.19995117, + "step": 2638, + "time_per_iteration": 2.8213560581207275 + }, + { + "auxiliary_loss_clip": 0.01165348, + "auxiliary_loss_mlp": 0.01042465, + "balance_loss_clip": 1.0520786, + "balance_loss_mlp": 1.02129328, + "epoch": 0.31732098839656103, + "flos": 31350876693120.0, + "grad_norm": 1.9936575024857102, + "language_loss": 0.83447975, + "learning_rate": 3.194895653158381e-06, + "loss": 0.85655785, + "num_input_tokens_seen": 56797215, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.21154785, + "step": 2639, + "time_per_iteration": 2.8844265937805176 + }, + { + "auxiliary_loss_clip": 0.01063239, + "auxiliary_loss_mlp": 0.01015927, + "balance_loss_clip": 1.02499294, + "balance_loss_mlp": 1.01379287, + "epoch": 0.31744123128720014, + "flos": 58989024835200.0, + "grad_norm": 0.7745812688639285, + "language_loss": 0.5547834, + "learning_rate": 3.194270899114093e-06, + "loss": 0.57557505, + "num_input_tokens_seen": 56863010, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.0213623, + "step": 2640, + "time_per_iteration": 3.4360527992248535 + }, + { + "auxiliary_loss_clip": 0.01165775, + "auxiliary_loss_mlp": 0.01046995, + "balance_loss_clip": 1.05293703, + "balance_loss_mlp": 1.02655053, + "epoch": 0.31756147417783925, + "flos": 17417573372160.0, + "grad_norm": 2.229434271014282, + "language_loss": 0.82212806, + "learning_rate": 3.193645963899858e-06, + "loss": 0.84425569, + "num_input_tokens_seen": 56880625, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.20458984, + "step": 2641, + "time_per_iteration": 2.851776599884033 + }, + { + "auxiliary_loss_clip": 0.01167205, + "auxiliary_loss_mlp": 0.01060086, + "balance_loss_clip": 1.05535722, + "balance_loss_mlp": 1.03879571, + "epoch": 0.3176817170684783, + "flos": 25481673267840.0, + "grad_norm": 2.392829157169532, + "language_loss": 0.84030986, + "learning_rate": 3.193020847610479e-06, + "loss": 0.8625828, + "num_input_tokens_seen": 56900945, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.21289062, + "step": 2642, + "time_per_iteration": 2.8749890327453613 + }, + { + "auxiliary_loss_clip": 0.01164824, + "auxiliary_loss_mlp": 0.01047082, + "balance_loss_clip": 1.05422592, + "balance_loss_mlp": 1.02529049, + "epoch": 0.3178019599591174, + "flos": 24972603765120.0, + "grad_norm": 2.1359800274765903, + "language_loss": 0.71274626, + "learning_rate": 3.192395550340787e-06, + "loss": 0.73486531, + "num_input_tokens_seen": 56918895, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.21777344, + "step": 2643, + "time_per_iteration": 3.04107403755188 + }, + { + "auxiliary_loss_clip": 0.01166088, + "auxiliary_loss_mlp": 0.01048834, + "balance_loss_clip": 1.05501759, + "balance_loss_mlp": 1.02838922, + "epoch": 0.31792220284975653, + "flos": 12422220019200.0, + "grad_norm": 2.0623987784865436, + "language_loss": 0.769225, + "learning_rate": 3.191770072185638e-06, + "loss": 0.79137421, + "num_input_tokens_seen": 56935890, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.20446777, + "step": 2644, + "time_per_iteration": 3.6185989379882812 + }, + { + "auxiliary_loss_clip": 0.01166572, + "auxiliary_loss_mlp": 0.01050838, + "balance_loss_clip": 1.05571449, + "balance_loss_mlp": 1.03044081, + "epoch": 0.3180424457403956, + "flos": 15485759089920.0, + "grad_norm": 21.983580903870646, + "language_loss": 0.73072803, + "learning_rate": 3.191144413239916e-06, + "loss": 0.75290215, + "num_input_tokens_seen": 56952460, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.20397949, + "step": 2645, + "time_per_iteration": 3.7525391578674316 + }, + { + "auxiliary_loss_clip": 0.01166442, + "auxiliary_loss_mlp": 0.01052704, + "balance_loss_clip": 1.0533576, + "balance_loss_mlp": 1.03186619, + "epoch": 0.3181626886310347, + "flos": 26174964648960.0, + "grad_norm": 2.262617546187266, + "language_loss": 0.88420409, + "learning_rate": 3.190518573598534e-06, + "loss": 0.90639555, + "num_input_tokens_seen": 56969065, + "router_z_loss_clip": 1.13134766, + "router_z_loss_mlp": 0.20837402, + "step": 2646, + "time_per_iteration": 2.937620162963867 + }, + { + "auxiliary_loss_clip": 0.011658, + "auxiliary_loss_mlp": 0.01049462, + "balance_loss_clip": 1.0550561, + "balance_loss_mlp": 1.02821875, + "epoch": 0.3182829315216738, + "flos": 25483109811840.0, + "grad_norm": 8.337169410296724, + "language_loss": 0.77613699, + "learning_rate": 3.1898925533564308e-06, + "loss": 0.79828966, + "num_input_tokens_seen": 56990535, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.21240234, + "step": 2647, + "time_per_iteration": 3.730829954147339 + }, + { + "auxiliary_loss_clip": 0.01164192, + "auxiliary_loss_mlp": 0.01041706, + "balance_loss_clip": 1.05504215, + "balance_loss_mlp": 1.02172613, + "epoch": 0.31840317441231286, + "flos": 18113701927680.0, + "grad_norm": 4.889457541593614, + "language_loss": 0.64284194, + "learning_rate": 3.1892663526085733e-06, + "loss": 0.6649009, + "num_input_tokens_seen": 57008910, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.19970703, + "step": 2648, + "time_per_iteration": 3.7583506107330322 + }, + { + "auxiliary_loss_clip": 0.01061428, + "auxiliary_loss_mlp": 0.01008598, + "balance_loss_clip": 1.023435, + "balance_loss_mlp": 1.00638068, + "epoch": 0.31852341730295197, + "flos": 64741948957440.0, + "grad_norm": 0.7557966240601064, + "language_loss": 0.56894493, + "learning_rate": 3.188639971449956e-06, + "loss": 0.58964527, + "num_input_tokens_seen": 57074960, + "router_z_loss_clip": 0.37890625, + "router_z_loss_mlp": 0.0222168, + "step": 2649, + "time_per_iteration": 3.3313241004943848 + }, + { + "auxiliary_loss_clip": 0.01168098, + "auxiliary_loss_mlp": 0.01050524, + "balance_loss_clip": 1.05642939, + "balance_loss_mlp": 1.03127182, + "epoch": 0.318643660193591, + "flos": 20668135582080.0, + "grad_norm": 4.705662300840546, + "language_loss": 0.72623575, + "learning_rate": 3.1880134099756e-06, + "loss": 0.74842197, + "num_input_tokens_seen": 57094595, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.19250488, + "step": 2650, + "time_per_iteration": 2.816274642944336 + }, + { + "auxiliary_loss_clip": 0.01160725, + "auxiliary_loss_mlp": 0.01043957, + "balance_loss_clip": 1.05340111, + "balance_loss_mlp": 1.02396595, + "epoch": 0.31876390308423014, + "flos": 26943345411840.0, + "grad_norm": 2.055981415647691, + "language_loss": 0.69757581, + "learning_rate": 3.1873866682805535e-06, + "loss": 0.71962267, + "num_input_tokens_seen": 57115290, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.1998291, + "step": 2651, + "time_per_iteration": 2.8571550846099854 + }, + { + "auxiliary_loss_clip": 0.01164538, + "auxiliary_loss_mlp": 0.01054835, + "balance_loss_clip": 1.05368233, + "balance_loss_mlp": 1.03399706, + "epoch": 0.31888414597486925, + "flos": 18041916597120.0, + "grad_norm": 2.0069217827604056, + "language_loss": 0.88574642, + "learning_rate": 3.186759746459894e-06, + "loss": 0.90794009, + "num_input_tokens_seen": 57134400, + "router_z_loss_clip": 1.10839844, + "router_z_loss_mlp": 0.20825195, + "step": 2652, + "time_per_iteration": 2.7917163372039795 + }, + { + "auxiliary_loss_clip": 0.01157605, + "auxiliary_loss_mlp": 0.01047012, + "balance_loss_clip": 1.05002713, + "balance_loss_mlp": 1.0282129, + "epoch": 0.3190043888655083, + "flos": 25149319701120.0, + "grad_norm": 2.176787382066096, + "language_loss": 0.79879856, + "learning_rate": 3.1861326446087246e-06, + "loss": 0.82084465, + "num_input_tokens_seen": 57153140, + "router_z_loss_clip": 1.07617188, + "router_z_loss_mlp": 0.18811035, + "step": 2653, + "time_per_iteration": 2.8038482666015625 + }, + { + "auxiliary_loss_clip": 0.01164145, + "auxiliary_loss_mlp": 0.01035038, + "balance_loss_clip": 1.05394125, + "balance_loss_mlp": 1.01408148, + "epoch": 0.3191246317561474, + "flos": 22053892331520.0, + "grad_norm": 2.2574491508838386, + "language_loss": 0.71988946, + "learning_rate": 3.1855053628221763e-06, + "loss": 0.74188125, + "num_input_tokens_seen": 57172395, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.20983887, + "step": 2654, + "time_per_iteration": 2.8112072944641113 + }, + { + "auxiliary_loss_clip": 0.01164323, + "auxiliary_loss_mlp": 0.01055258, + "balance_loss_clip": 1.051682, + "balance_loss_mlp": 1.03447998, + "epoch": 0.3192448746467865, + "flos": 14901815687040.0, + "grad_norm": 2.821310756139791, + "language_loss": 0.89637661, + "learning_rate": 3.184877901195407e-06, + "loss": 0.91857243, + "num_input_tokens_seen": 57189090, + "router_z_loss_clip": 1.12597656, + "router_z_loss_mlp": 0.20788574, + "step": 2655, + "time_per_iteration": 2.803666353225708 + }, + { + "auxiliary_loss_clip": 0.01057955, + "auxiliary_loss_mlp": 0.01003317, + "balance_loss_clip": 1.01973522, + "balance_loss_mlp": 1.00117135, + "epoch": 0.3193651175374256, + "flos": 67234832657280.0, + "grad_norm": 0.7958563729075065, + "language_loss": 0.62841296, + "learning_rate": 3.184250259823602e-06, + "loss": 0.64902568, + "num_input_tokens_seen": 57251620, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.02148438, + "step": 2656, + "time_per_iteration": 3.4058127403259277 + }, + { + "auxiliary_loss_clip": 0.011661, + "auxiliary_loss_mlp": 0.01046679, + "balance_loss_clip": 1.0526278, + "balance_loss_mlp": 1.02652025, + "epoch": 0.3194853604280647, + "flos": 12233077977600.0, + "grad_norm": 2.592473334652962, + "language_loss": 0.81401026, + "learning_rate": 3.183622438801974e-06, + "loss": 0.83613813, + "num_input_tokens_seen": 57266910, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.20178223, + "step": 2657, + "time_per_iteration": 2.893134832382202 + }, + { + "auxiliary_loss_clip": 0.01165391, + "auxiliary_loss_mlp": 0.01051558, + "balance_loss_clip": 1.05698967, + "balance_loss_mlp": 1.03323555, + "epoch": 0.3196056033187038, + "flos": 14939917038720.0, + "grad_norm": 1.9267228438749335, + "language_loss": 0.74990797, + "learning_rate": 3.1829944382257637e-06, + "loss": 0.77207744, + "num_input_tokens_seen": 57285040, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.18322754, + "step": 2658, + "time_per_iteration": 2.8878543376922607 + }, + { + "auxiliary_loss_clip": 0.01165839, + "auxiliary_loss_mlp": 0.0105618, + "balance_loss_clip": 1.05378532, + "balance_loss_mlp": 1.03761923, + "epoch": 0.31972584620934286, + "flos": 23768878164480.0, + "grad_norm": 3.0133128061135444, + "language_loss": 0.81496954, + "learning_rate": 3.1823662581902373e-06, + "loss": 0.83718979, + "num_input_tokens_seen": 57302725, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.18566895, + "step": 2659, + "time_per_iteration": 2.871274709701538 + }, + { + "auxiliary_loss_clip": 0.01169731, + "auxiliary_loss_mlp": 0.01048393, + "balance_loss_clip": 1.05549705, + "balance_loss_mlp": 1.02840173, + "epoch": 0.31984608909998197, + "flos": 21251540280960.0, + "grad_norm": 2.5711881798893317, + "language_loss": 0.74720788, + "learning_rate": 3.1817378987906896e-06, + "loss": 0.76938915, + "num_input_tokens_seen": 57322230, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.19970703, + "step": 2660, + "time_per_iteration": 2.838412046432495 + }, + { + "auxiliary_loss_clip": 0.01165976, + "auxiliary_loss_mlp": 0.01049669, + "balance_loss_clip": 1.05458641, + "balance_loss_mlp": 1.02859259, + "epoch": 0.3199663319906211, + "flos": 18296235866880.0, + "grad_norm": 2.499781505134651, + "language_loss": 0.80047929, + "learning_rate": 3.181109360122442e-06, + "loss": 0.82263577, + "num_input_tokens_seen": 57339820, + "router_z_loss_clip": 1.11425781, + "router_z_loss_mlp": 0.21069336, + "step": 2661, + "time_per_iteration": 2.8331265449523926 + }, + { + "auxiliary_loss_clip": 0.0116605, + "auxiliary_loss_mlp": 0.01049135, + "balance_loss_clip": 1.05364656, + "balance_loss_mlp": 1.0277487, + "epoch": 0.32008657488126013, + "flos": 18733627779840.0, + "grad_norm": 2.1443590763996396, + "language_loss": 0.78809094, + "learning_rate": 3.1804806422808445e-06, + "loss": 0.81024277, + "num_input_tokens_seen": 57356955, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.21386719, + "step": 2662, + "time_per_iteration": 2.8477232456207275 + }, + { + "auxiliary_loss_clip": 0.01167681, + "auxiliary_loss_mlp": 0.01046949, + "balance_loss_clip": 1.05640745, + "balance_loss_mlp": 1.02708912, + "epoch": 0.32020681777189924, + "flos": 20595344670720.0, + "grad_norm": 1.9544438960500021, + "language_loss": 0.72877026, + "learning_rate": 3.1798517453612714e-06, + "loss": 0.75091654, + "num_input_tokens_seen": 57376760, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.1986084, + "step": 2663, + "time_per_iteration": 2.852553367614746 + }, + { + "auxiliary_loss_clip": 0.01162432, + "auxiliary_loss_mlp": 0.01050293, + "balance_loss_clip": 1.0522486, + "balance_loss_mlp": 1.02931213, + "epoch": 0.32032706066253835, + "flos": 35261692750080.0, + "grad_norm": 2.448489761114681, + "language_loss": 0.75316405, + "learning_rate": 3.1792226694591265e-06, + "loss": 0.77529132, + "num_input_tokens_seen": 57398145, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.2097168, + "step": 2664, + "time_per_iteration": 2.9262375831604004 + }, + { + "auxiliary_loss_clip": 0.011625, + "auxiliary_loss_mlp": 0.01049315, + "balance_loss_clip": 1.05411577, + "balance_loss_mlp": 1.03106403, + "epoch": 0.3204473035531774, + "flos": 15304230731520.0, + "grad_norm": 2.2196687943650297, + "language_loss": 0.80892092, + "learning_rate": 3.178593414669841e-06, + "loss": 0.83103907, + "num_input_tokens_seen": 57416730, + "router_z_loss_clip": 1.08398438, + "router_z_loss_mlp": 0.18261719, + "step": 2665, + "time_per_iteration": 2.8637776374816895 + }, + { + "auxiliary_loss_clip": 0.01168834, + "auxiliary_loss_mlp": 0.01055723, + "balance_loss_clip": 1.05263162, + "balance_loss_mlp": 1.03359759, + "epoch": 0.3205675464438165, + "flos": 24462564595200.0, + "grad_norm": 3.9491392031113506, + "language_loss": 0.70623654, + "learning_rate": 3.1779639810888707e-06, + "loss": 0.72848213, + "num_input_tokens_seen": 57436325, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.22119141, + "step": 2666, + "time_per_iteration": 2.888683795928955 + }, + { + "auxiliary_loss_clip": 0.01166539, + "auxiliary_loss_mlp": 0.01050504, + "balance_loss_clip": 1.0542686, + "balance_loss_mlp": 1.03045285, + "epoch": 0.3206877893344556, + "flos": 22456235548800.0, + "grad_norm": 2.3110293237882837, + "language_loss": 0.75957274, + "learning_rate": 3.1773343688117013e-06, + "loss": 0.78174311, + "num_input_tokens_seen": 57457235, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.20080566, + "step": 2667, + "time_per_iteration": 2.917419910430908 + }, + { + "auxiliary_loss_clip": 0.01168391, + "auxiliary_loss_mlp": 0.01050931, + "balance_loss_clip": 1.05631769, + "balance_loss_mlp": 1.03130925, + "epoch": 0.3208080322250947, + "flos": 20412236113920.0, + "grad_norm": 2.603948147095736, + "language_loss": 0.84454989, + "learning_rate": 3.1767045779338445e-06, + "loss": 0.86674309, + "num_input_tokens_seen": 57474895, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.19604492, + "step": 2668, + "time_per_iteration": 2.830366373062134 + }, + { + "auxiliary_loss_clip": 0.01160899, + "auxiliary_loss_mlp": 0.01049929, + "balance_loss_clip": 1.0511173, + "balance_loss_mlp": 1.02953291, + "epoch": 0.3209282751157338, + "flos": 21762118154880.0, + "grad_norm": 6.467607064471329, + "language_loss": 0.91355288, + "learning_rate": 3.176074608550839e-06, + "loss": 0.9356612, + "num_input_tokens_seen": 57490715, + "router_z_loss_clip": 1.09814453, + "router_z_loss_mlp": 0.20397949, + "step": 2669, + "time_per_iteration": 2.8433825969696045 + }, + { + "auxiliary_loss_clip": 0.01167551, + "auxiliary_loss_mlp": 0.01058086, + "balance_loss_clip": 1.05498791, + "balance_loss_mlp": 1.03699827, + "epoch": 0.32104851800637285, + "flos": 22055041566720.0, + "grad_norm": 2.801727827988656, + "language_loss": 0.81958544, + "learning_rate": 3.17544446075825e-06, + "loss": 0.84184188, + "num_input_tokens_seen": 57509880, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.21105957, + "step": 2670, + "time_per_iteration": 2.8566746711730957 + }, + { + "auxiliary_loss_clip": 0.01158412, + "auxiliary_loss_mlp": 0.01045266, + "balance_loss_clip": 1.05096328, + "balance_loss_mlp": 1.02713418, + "epoch": 0.32116876089701196, + "flos": 37012301896320.0, + "grad_norm": 1.7008624123810718, + "language_loss": 0.70930767, + "learning_rate": 3.174814134651671e-06, + "loss": 0.73134446, + "num_input_tokens_seen": 57532430, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.18127441, + "step": 2671, + "time_per_iteration": 2.8676116466522217 + }, + { + "auxiliary_loss_clip": 0.01158009, + "auxiliary_loss_mlp": 0.0104687, + "balance_loss_clip": 1.05122995, + "balance_loss_mlp": 1.02870202, + "epoch": 0.3212890037876511, + "flos": 21979233912960.0, + "grad_norm": 2.102721798160791, + "language_loss": 0.80722153, + "learning_rate": 3.1741836303267215e-06, + "loss": 0.82927036, + "num_input_tokens_seen": 57551965, + "router_z_loss_clip": 1.06738281, + "router_z_loss_mlp": 0.1817627, + "step": 2672, + "time_per_iteration": 2.8645694255828857 + }, + { + "auxiliary_loss_clip": 0.01160559, + "auxiliary_loss_mlp": 0.01053272, + "balance_loss_clip": 1.05397391, + "balance_loss_mlp": 1.03446054, + "epoch": 0.32140924667829013, + "flos": 10342345875840.0, + "grad_norm": 2.1306636614404515, + "language_loss": 0.75411701, + "learning_rate": 3.1735529478790496e-06, + "loss": 0.77625531, + "num_input_tokens_seen": 57569955, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.18811035, + "step": 2673, + "time_per_iteration": 3.5994255542755127 + }, + { + "auxiliary_loss_clip": 0.01162968, + "auxiliary_loss_mlp": 0.01056354, + "balance_loss_clip": 1.05198693, + "balance_loss_mlp": 1.03527784, + "epoch": 0.32152948956892924, + "flos": 50798910072960.0, + "grad_norm": 2.6485109469444135, + "language_loss": 0.79301685, + "learning_rate": 3.172922087404328e-06, + "loss": 0.8152101, + "num_input_tokens_seen": 57592215, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.2109375, + "step": 2674, + "time_per_iteration": 3.7971601486206055 + }, + { + "auxiliary_loss_clip": 0.01062822, + "auxiliary_loss_mlp": 0.01012393, + "balance_loss_clip": 1.02493608, + "balance_loss_mlp": 1.01031864, + "epoch": 0.32164973245956835, + "flos": 63863250549120.0, + "grad_norm": 0.8444910933636731, + "language_loss": 0.55173522, + "learning_rate": 3.1722910489982586e-06, + "loss": 0.57248741, + "num_input_tokens_seen": 57652575, + "router_z_loss_clip": 0.37890625, + "router_z_loss_mlp": 0.02075195, + "step": 2675, + "time_per_iteration": 3.443359375 + }, + { + "auxiliary_loss_clip": 0.01166674, + "auxiliary_loss_mlp": 0.01045721, + "balance_loss_clip": 1.05570054, + "balance_loss_mlp": 1.02661216, + "epoch": 0.3217699753502074, + "flos": 23513948363520.0, + "grad_norm": 1.604305225910993, + "language_loss": 0.79842234, + "learning_rate": 3.1716598327565694e-06, + "loss": 0.82054627, + "num_input_tokens_seen": 57672215, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.19116211, + "step": 2676, + "time_per_iteration": 3.65341854095459 + }, + { + "auxiliary_loss_clip": 0.01160491, + "auxiliary_loss_mlp": 0.01052886, + "balance_loss_clip": 1.05279303, + "balance_loss_mlp": 1.03378844, + "epoch": 0.3218902182408465, + "flos": 19062533640960.0, + "grad_norm": 1.7489611666001992, + "language_loss": 0.84495914, + "learning_rate": 3.171028438775015e-06, + "loss": 0.86709285, + "num_input_tokens_seen": 57691410, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.19091797, + "step": 2677, + "time_per_iteration": 3.739231824874878 + }, + { + "auxiliary_loss_clip": 0.01162451, + "auxiliary_loss_mlp": 0.01042908, + "balance_loss_clip": 1.05460751, + "balance_loss_mlp": 1.02264237, + "epoch": 0.3220104611314856, + "flos": 20375571306240.0, + "grad_norm": 4.174207089092724, + "language_loss": 0.84217638, + "learning_rate": 3.170396867149377e-06, + "loss": 0.86422998, + "num_input_tokens_seen": 57709415, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.20263672, + "step": 2678, + "time_per_iteration": 2.8816635608673096 + }, + { + "auxiliary_loss_clip": 0.01161721, + "auxiliary_loss_mlp": 0.01052517, + "balance_loss_clip": 1.05251479, + "balance_loss_mlp": 1.03287089, + "epoch": 0.3221307040221247, + "flos": 20117014231680.0, + "grad_norm": 1.8755869985612335, + "language_loss": 0.86376989, + "learning_rate": 3.1697651179754653e-06, + "loss": 0.88591224, + "num_input_tokens_seen": 57728075, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.1965332, + "step": 2679, + "time_per_iteration": 2.89076828956604 + }, + { + "auxiliary_loss_clip": 0.0116167, + "auxiliary_loss_mlp": 0.01048818, + "balance_loss_clip": 1.05462205, + "balance_loss_mlp": 1.03023314, + "epoch": 0.3222509469127638, + "flos": 23987789602560.0, + "grad_norm": 2.2061434221648226, + "language_loss": 0.73605704, + "learning_rate": 3.1691331913491153e-06, + "loss": 0.7581619, + "num_input_tokens_seen": 57750645, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.18591309, + "step": 2680, + "time_per_iteration": 2.946627378463745 + }, + { + "auxiliary_loss_clip": 0.01159137, + "auxiliary_loss_mlp": 0.01040552, + "balance_loss_clip": 1.05165958, + "balance_loss_mlp": 1.01898718, + "epoch": 0.32237118980340285, + "flos": 17675735397120.0, + "grad_norm": 2.197217276475613, + "language_loss": 0.85468888, + "learning_rate": 3.1685010873661898e-06, + "loss": 0.8766858, + "num_input_tokens_seen": 57769820, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.21557617, + "step": 2681, + "time_per_iteration": 2.866533041000366 + }, + { + "auxiliary_loss_clip": 0.01165168, + "auxiliary_loss_mlp": 0.01042237, + "balance_loss_clip": 1.0544579, + "balance_loss_mlp": 1.02311563, + "epoch": 0.32249143269404196, + "flos": 23147982645120.0, + "grad_norm": 10.539201411741423, + "language_loss": 0.79977882, + "learning_rate": 3.167868806122578e-06, + "loss": 0.82185286, + "num_input_tokens_seen": 57788870, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.19140625, + "step": 2682, + "time_per_iteration": 2.84830904006958 + }, + { + "auxiliary_loss_clip": 0.01165656, + "auxiliary_loss_mlp": 0.01052823, + "balance_loss_clip": 1.05376256, + "balance_loss_mlp": 1.03148413, + "epoch": 0.32261167558468107, + "flos": 24422308427520.0, + "grad_norm": 1.8352035556206177, + "language_loss": 0.66524392, + "learning_rate": 3.1672363477141968e-06, + "loss": 0.68742877, + "num_input_tokens_seen": 57808165, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.21350098, + "step": 2683, + "time_per_iteration": 2.8324553966522217 + }, + { + "auxiliary_loss_clip": 0.01162825, + "auxiliary_loss_mlp": 0.01052383, + "balance_loss_clip": 1.0513134, + "balance_loss_mlp": 1.03149772, + "epoch": 0.3227319184753201, + "flos": 30367175852160.0, + "grad_norm": 4.5087087180531045, + "language_loss": 0.85111737, + "learning_rate": 3.1666037122369903e-06, + "loss": 0.87326944, + "num_input_tokens_seen": 57828825, + "router_z_loss_clip": 1.11572266, + "router_z_loss_mlp": 0.2088623, + "step": 2684, + "time_per_iteration": 2.885322332382202 + }, + { + "auxiliary_loss_clip": 0.01157029, + "auxiliary_loss_mlp": 0.01048887, + "balance_loss_clip": 1.04874444, + "balance_loss_mlp": 1.02852654, + "epoch": 0.32285216136595923, + "flos": 16946174257920.0, + "grad_norm": 2.5037321046149437, + "language_loss": 0.86673498, + "learning_rate": 3.165970899786928e-06, + "loss": 0.88879418, + "num_input_tokens_seen": 57846740, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.20373535, + "step": 2685, + "time_per_iteration": 2.819999933242798 + }, + { + "auxiliary_loss_clip": 0.01159527, + "auxiliary_loss_mlp": 0.01038954, + "balance_loss_clip": 1.04971099, + "balance_loss_mlp": 1.01853359, + "epoch": 0.32297240425659834, + "flos": 21981532383360.0, + "grad_norm": 1.943066034579148, + "language_loss": 0.75419837, + "learning_rate": 3.1653379104600067e-06, + "loss": 0.77618313, + "num_input_tokens_seen": 57866885, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.20397949, + "step": 2686, + "time_per_iteration": 3.0645101070404053 + }, + { + "auxiliary_loss_clip": 0.01163188, + "auxiliary_loss_mlp": 0.01048766, + "balance_loss_clip": 1.0528208, + "balance_loss_mlp": 1.02809477, + "epoch": 0.3230926471472374, + "flos": 22748045639040.0, + "grad_norm": 1.6717132100140297, + "language_loss": 0.69612801, + "learning_rate": 3.164704744352251e-06, + "loss": 0.71824759, + "num_input_tokens_seen": 57887690, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.20666504, + "step": 2687, + "time_per_iteration": 2.8588662147521973 + }, + { + "auxiliary_loss_clip": 0.01160106, + "auxiliary_loss_mlp": 0.0104375, + "balance_loss_clip": 1.05009282, + "balance_loss_mlp": 1.02353215, + "epoch": 0.3232128900378765, + "flos": 16942977947520.0, + "grad_norm": 1.8915082276038018, + "language_loss": 0.80878013, + "learning_rate": 3.164071401559713e-06, + "loss": 0.83081871, + "num_input_tokens_seen": 57905090, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.20227051, + "step": 2688, + "time_per_iteration": 2.803910255432129 + }, + { + "auxiliary_loss_clip": 0.01162899, + "auxiliary_loss_mlp": 0.01055825, + "balance_loss_clip": 1.05287766, + "balance_loss_mlp": 1.03560674, + "epoch": 0.3233331329285156, + "flos": 24023736138240.0, + "grad_norm": 2.0131191568841613, + "language_loss": 0.71077859, + "learning_rate": 3.1634378821784674e-06, + "loss": 0.73296583, + "num_input_tokens_seen": 57925305, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.20202637, + "step": 2689, + "time_per_iteration": 2.8266773223876953 + }, + { + "auxiliary_loss_clip": 0.01164594, + "auxiliary_loss_mlp": 0.01051817, + "balance_loss_clip": 1.05353749, + "balance_loss_mlp": 1.03018022, + "epoch": 0.3234533758191547, + "flos": 18113845582080.0, + "grad_norm": 2.800684659561374, + "language_loss": 0.74269152, + "learning_rate": 3.1628041863046208e-06, + "loss": 0.76485568, + "num_input_tokens_seen": 57942720, + "router_z_loss_clip": 1.11132812, + "router_z_loss_mlp": 0.21643066, + "step": 2690, + "time_per_iteration": 2.799245834350586 + }, + { + "auxiliary_loss_clip": 0.01165102, + "auxiliary_loss_mlp": 0.0105514, + "balance_loss_clip": 1.05253196, + "balance_loss_mlp": 1.03301501, + "epoch": 0.3235736187097938, + "flos": 16946138344320.0, + "grad_norm": 3.05569706740952, + "language_loss": 0.91455829, + "learning_rate": 3.162170314034304e-06, + "loss": 0.93676066, + "num_input_tokens_seen": 57960135, + "router_z_loss_clip": 1.12597656, + "router_z_loss_mlp": 0.22131348, + "step": 2691, + "time_per_iteration": 2.847445011138916 + }, + { + "auxiliary_loss_clip": 0.01161406, + "auxiliary_loss_mlp": 0.01050726, + "balance_loss_clip": 1.05174732, + "balance_loss_mlp": 1.03079402, + "epoch": 0.3236938616004329, + "flos": 22127150119680.0, + "grad_norm": 1.8891307184255912, + "language_loss": 0.8100282, + "learning_rate": 3.1615362654636738e-06, + "loss": 0.83214951, + "num_input_tokens_seen": 57980875, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.19934082, + "step": 2692, + "time_per_iteration": 3.0305044651031494 + }, + { + "auxiliary_loss_clip": 0.01159376, + "auxiliary_loss_mlp": 0.01042844, + "balance_loss_clip": 1.05369115, + "balance_loss_mlp": 1.02428329, + "epoch": 0.32381410449107195, + "flos": 17164618819200.0, + "grad_norm": 1.878141794896613, + "language_loss": 0.86973214, + "learning_rate": 3.1609020406889163e-06, + "loss": 0.89175427, + "num_input_tokens_seen": 57998310, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.18554688, + "step": 2693, + "time_per_iteration": 2.8867878913879395 + }, + { + "auxiliary_loss_clip": 0.01161013, + "auxiliary_loss_mlp": 0.01054168, + "balance_loss_clip": 1.05125475, + "balance_loss_mlp": 1.03418875, + "epoch": 0.32393434738171106, + "flos": 16578125550720.0, + "grad_norm": 2.179043576958532, + "language_loss": 0.84907979, + "learning_rate": 3.1602676398062416e-06, + "loss": 0.87123168, + "num_input_tokens_seen": 58017220, + "router_z_loss_clip": 1.09765625, + "router_z_loss_mlp": 0.1998291, + "step": 2694, + "time_per_iteration": 2.8521618843078613 + }, + { + "auxiliary_loss_clip": 0.01164178, + "auxiliary_loss_mlp": 0.01050438, + "balance_loss_clip": 1.05264676, + "balance_loss_mlp": 1.0305537, + "epoch": 0.3240545902723502, + "flos": 25483612602240.0, + "grad_norm": 3.4864270028760314, + "language_loss": 0.61944103, + "learning_rate": 3.1596330629118886e-06, + "loss": 0.64158726, + "num_input_tokens_seen": 58037190, + "router_z_loss_clip": 1.11523438, + "router_z_loss_mlp": 0.19885254, + "step": 2695, + "time_per_iteration": 2.8591885566711426 + }, + { + "auxiliary_loss_clip": 0.0115978, + "auxiliary_loss_mlp": 0.01061026, + "balance_loss_clip": 1.05137706, + "balance_loss_mlp": 1.04054642, + "epoch": 0.32417483316298923, + "flos": 35845851634560.0, + "grad_norm": 2.5856791003598385, + "language_loss": 0.73247671, + "learning_rate": 3.1589983101021223e-06, + "loss": 0.75468481, + "num_input_tokens_seen": 58055820, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.20458984, + "step": 2696, + "time_per_iteration": 2.8724091053009033 + }, + { + "auxiliary_loss_clip": 0.01160002, + "auxiliary_loss_mlp": 0.01049409, + "balance_loss_clip": 1.04988956, + "balance_loss_mlp": 1.02975178, + "epoch": 0.32429507605362834, + "flos": 30080501406720.0, + "grad_norm": 2.1812525433256837, + "language_loss": 0.84801942, + "learning_rate": 3.1583633814732337e-06, + "loss": 0.87011355, + "num_input_tokens_seen": 58075340, + "router_z_loss_clip": 1.10107422, + "router_z_loss_mlp": 0.1965332, + "step": 2697, + "time_per_iteration": 2.8370978832244873 + }, + { + "auxiliary_loss_clip": 0.01160852, + "auxiliary_loss_mlp": 0.01044524, + "balance_loss_clip": 1.05081511, + "balance_loss_mlp": 1.02564156, + "epoch": 0.3244153189442674, + "flos": 18223265387520.0, + "grad_norm": 2.5186616531293695, + "language_loss": 0.72102457, + "learning_rate": 3.157728277121541e-06, + "loss": 0.74307841, + "num_input_tokens_seen": 58093515, + "router_z_loss_clip": 1.09912109, + "router_z_loss_mlp": 0.18908691, + "step": 2698, + "time_per_iteration": 2.9194467067718506 + }, + { + "auxiliary_loss_clip": 0.01160268, + "auxiliary_loss_mlp": 0.0105376, + "balance_loss_clip": 1.05017996, + "balance_loss_mlp": 1.03214788, + "epoch": 0.3245355618349065, + "flos": 17710317216000.0, + "grad_norm": 4.216585175988132, + "language_loss": 0.78234446, + "learning_rate": 3.1570929971433897e-06, + "loss": 0.80448472, + "num_input_tokens_seen": 58109300, + "router_z_loss_clip": 1.10107422, + "router_z_loss_mlp": 0.21630859, + "step": 2699, + "time_per_iteration": 2.85978364944458 + }, + { + "auxiliary_loss_clip": 0.01165572, + "auxiliary_loss_mlp": 0.01056582, + "balance_loss_clip": 1.0518887, + "balance_loss_mlp": 1.03562546, + "epoch": 0.3246558047255456, + "flos": 23440798316160.0, + "grad_norm": 4.03091781511512, + "language_loss": 0.83679843, + "learning_rate": 3.1564575416351504e-06, + "loss": 0.85901988, + "num_input_tokens_seen": 58128000, + "router_z_loss_clip": 1.13623047, + "router_z_loss_mlp": 0.20947266, + "step": 2700, + "time_per_iteration": 2.874837875366211 + }, + { + "auxiliary_loss_clip": 0.01157506, + "auxiliary_loss_mlp": 0.01046984, + "balance_loss_clip": 1.04985714, + "balance_loss_mlp": 1.02683747, + "epoch": 0.32477604761618467, + "flos": 21760861178880.0, + "grad_norm": 2.4687207887348106, + "language_loss": 0.74461424, + "learning_rate": 3.155821910693221e-06, + "loss": 0.76665914, + "num_input_tokens_seen": 58147415, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.20129395, + "step": 2701, + "time_per_iteration": 3.6637260913848877 + }, + { + "auxiliary_loss_clip": 0.0115987, + "auxiliary_loss_mlp": 0.01044802, + "balance_loss_clip": 1.0504986, + "balance_loss_mlp": 1.02355862, + "epoch": 0.3248962905068238, + "flos": 19828328624640.0, + "grad_norm": 2.0351856398378816, + "language_loss": 0.8604157, + "learning_rate": 3.1551861044140275e-06, + "loss": 0.88246244, + "num_input_tokens_seen": 58167050, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.21240234, + "step": 2702, + "time_per_iteration": 2.834097385406494 + }, + { + "auxiliary_loss_clip": 0.01159392, + "auxiliary_loss_mlp": 0.01051983, + "balance_loss_clip": 1.05212104, + "balance_loss_mlp": 1.03246832, + "epoch": 0.3250165333974629, + "flos": 23948215793280.0, + "grad_norm": 2.0142202409238603, + "language_loss": 0.77258569, + "learning_rate": 3.15455012289402e-06, + "loss": 0.79469943, + "num_input_tokens_seen": 58186695, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.19519043, + "step": 2703, + "time_per_iteration": 3.708564043045044 + }, + { + "auxiliary_loss_clip": 0.01163161, + "auxiliary_loss_mlp": 0.01059822, + "balance_loss_clip": 1.05210102, + "balance_loss_mlp": 1.03918719, + "epoch": 0.32513677628810195, + "flos": 23989333887360.0, + "grad_norm": 1.90495930741056, + "language_loss": 0.84340441, + "learning_rate": 3.153913966229677e-06, + "loss": 0.86563426, + "num_input_tokens_seen": 58205815, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.2064209, + "step": 2704, + "time_per_iteration": 2.89207124710083 + }, + { + "auxiliary_loss_clip": 0.01068654, + "auxiliary_loss_mlp": 0.01002022, + "balance_loss_clip": 1.03049135, + "balance_loss_mlp": 0.99970931, + "epoch": 0.32525701917874106, + "flos": 70655790009600.0, + "grad_norm": 0.6447338579141394, + "language_loss": 0.50248355, + "learning_rate": 3.1532776345175027e-06, + "loss": 0.52319026, + "num_input_tokens_seen": 58270960, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.02307129, + "step": 2705, + "time_per_iteration": 4.193142414093018 + }, + { + "auxiliary_loss_clip": 0.01155832, + "auxiliary_loss_mlp": 0.0105135, + "balance_loss_clip": 1.04924059, + "balance_loss_mlp": 1.03077507, + "epoch": 0.32537726206938017, + "flos": 19682639061120.0, + "grad_norm": 1.9398681409480363, + "language_loss": 0.78623438, + "learning_rate": 3.1526411278540285e-06, + "loss": 0.80830628, + "num_input_tokens_seen": 58289390, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.20544434, + "step": 2706, + "time_per_iteration": 3.638245105743408 + }, + { + "auxiliary_loss_clip": 0.01162696, + "auxiliary_loss_mlp": 0.01048541, + "balance_loss_clip": 1.05185676, + "balance_loss_mlp": 1.02708316, + "epoch": 0.3254975049600192, + "flos": 28760999293440.0, + "grad_norm": 4.278009390507763, + "language_loss": 0.8115744, + "learning_rate": 3.1520044463358116e-06, + "loss": 0.83368671, + "num_input_tokens_seen": 58306120, + "router_z_loss_clip": 1.10839844, + "router_z_loss_mlp": 0.21459961, + "step": 2707, + "time_per_iteration": 2.92402982711792 + }, + { + "auxiliary_loss_clip": 0.01156953, + "auxiliary_loss_mlp": 0.010514, + "balance_loss_clip": 1.0496304, + "balance_loss_mlp": 1.03143215, + "epoch": 0.32561774785065833, + "flos": 18877378008960.0, + "grad_norm": 1.5979639287868075, + "language_loss": 0.80367458, + "learning_rate": 3.151367590059436e-06, + "loss": 0.8257581, + "num_input_tokens_seen": 58324545, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.19958496, + "step": 2708, + "time_per_iteration": 2.780592441558838 + }, + { + "auxiliary_loss_clip": 0.01162378, + "auxiliary_loss_mlp": 0.01050195, + "balance_loss_clip": 1.0514667, + "balance_loss_mlp": 1.02916646, + "epoch": 0.32573799074129745, + "flos": 23112107936640.0, + "grad_norm": 2.123597643047324, + "language_loss": 0.87136954, + "learning_rate": 3.1507305591215117e-06, + "loss": 0.89349532, + "num_input_tokens_seen": 58342455, + "router_z_loss_clip": 1.10986328, + "router_z_loss_mlp": 0.21020508, + "step": 2709, + "time_per_iteration": 2.84161114692688 + }, + { + "auxiliary_loss_clip": 0.01064685, + "auxiliary_loss_mlp": 0.010008, + "balance_loss_clip": 1.02702165, + "balance_loss_mlp": 0.99857122, + "epoch": 0.3258582336319365, + "flos": 71237650423680.0, + "grad_norm": 0.735303213534989, + "language_loss": 0.55689567, + "learning_rate": 3.150093353618677e-06, + "loss": 0.57755047, + "num_input_tokens_seen": 58407185, + "router_z_loss_clip": 0.37695312, + "router_z_loss_mlp": 0.02233887, + "step": 2710, + "time_per_iteration": 3.374141216278076 + }, + { + "auxiliary_loss_clip": 0.01164111, + "auxiliary_loss_mlp": 0.0105214, + "balance_loss_clip": 1.05118704, + "balance_loss_mlp": 1.02989531, + "epoch": 0.3259784765225756, + "flos": 22456020067200.0, + "grad_norm": 2.2817329858256308, + "language_loss": 0.88293707, + "learning_rate": 3.149455973647596e-06, + "loss": 0.90509957, + "num_input_tokens_seen": 58425245, + "router_z_loss_clip": 1.12890625, + "router_z_loss_mlp": 0.22229004, + "step": 2711, + "time_per_iteration": 2.785245418548584 + }, + { + "auxiliary_loss_clip": 0.01162612, + "auxiliary_loss_mlp": 0.01052372, + "balance_loss_clip": 1.0507406, + "balance_loss_mlp": 1.030581, + "epoch": 0.32609871941321467, + "flos": 20484811543680.0, + "grad_norm": 1.974925229440968, + "language_loss": 0.76822329, + "learning_rate": 3.1488184193049563e-06, + "loss": 0.79037309, + "num_input_tokens_seen": 58444780, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.21789551, + "step": 2712, + "time_per_iteration": 2.8040263652801514 + }, + { + "auxiliary_loss_clip": 0.01154563, + "auxiliary_loss_mlp": 0.01050709, + "balance_loss_clip": 1.04904532, + "balance_loss_mlp": 1.0307889, + "epoch": 0.3262189623038538, + "flos": 22416805393920.0, + "grad_norm": 2.2220497093807245, + "language_loss": 0.72365558, + "learning_rate": 3.1481806906874767e-06, + "loss": 0.74570829, + "num_input_tokens_seen": 58466090, + "router_z_loss_clip": 1.05273438, + "router_z_loss_mlp": 0.19921875, + "step": 2713, + "time_per_iteration": 2.8149125576019287 + }, + { + "auxiliary_loss_clip": 0.01155997, + "auxiliary_loss_mlp": 0.0104587, + "balance_loss_clip": 1.0502907, + "balance_loss_mlp": 1.0262363, + "epoch": 0.3263392051944929, + "flos": 20923496346240.0, + "grad_norm": 1.8246699615644388, + "language_loss": 0.87954867, + "learning_rate": 3.147542787891899e-06, + "loss": 0.90156734, + "num_input_tokens_seen": 58485435, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.19641113, + "step": 2714, + "time_per_iteration": 2.7856271266937256 + }, + { + "auxiliary_loss_clip": 0.01161578, + "auxiliary_loss_mlp": 0.01053361, + "balance_loss_clip": 1.0530833, + "balance_loss_mlp": 1.03252351, + "epoch": 0.32645944808513194, + "flos": 24025172682240.0, + "grad_norm": 2.2012799418695024, + "language_loss": 0.75636673, + "learning_rate": 3.1469047110149926e-06, + "loss": 0.77851617, + "num_input_tokens_seen": 58504175, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.20812988, + "step": 2715, + "time_per_iteration": 2.8161866664886475 + }, + { + "auxiliary_loss_clip": 0.01163638, + "auxiliary_loss_mlp": 0.01052573, + "balance_loss_clip": 1.05216432, + "balance_loss_mlp": 1.03125882, + "epoch": 0.32657969097577105, + "flos": 21032413361280.0, + "grad_norm": 2.938984023835437, + "language_loss": 0.85240114, + "learning_rate": 3.146266460153554e-06, + "loss": 0.87456322, + "num_input_tokens_seen": 58523885, + "router_z_loss_clip": 1.11425781, + "router_z_loss_mlp": 0.21313477, + "step": 2716, + "time_per_iteration": 2.86057186126709 + }, + { + "auxiliary_loss_clip": 0.01164622, + "auxiliary_loss_mlp": 0.01053616, + "balance_loss_clip": 1.05482984, + "balance_loss_mlp": 1.03181243, + "epoch": 0.32669993386641016, + "flos": 22710267509760.0, + "grad_norm": 1.6419929729778853, + "language_loss": 0.79691559, + "learning_rate": 3.145628035404404e-06, + "loss": 0.819098, + "num_input_tokens_seen": 58543085, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.21813965, + "step": 2717, + "time_per_iteration": 2.8005974292755127 + }, + { + "auxiliary_loss_clip": 0.01065155, + "auxiliary_loss_mlp": 0.01008429, + "balance_loss_clip": 1.02831817, + "balance_loss_mlp": 1.00634313, + "epoch": 0.3268201767570492, + "flos": 72105718406400.0, + "grad_norm": 0.9040796421691212, + "language_loss": 0.57471502, + "learning_rate": 3.1449894368643922e-06, + "loss": 0.59545076, + "num_input_tokens_seen": 58605400, + "router_z_loss_clip": 0.3671875, + "router_z_loss_mlp": 0.02087402, + "step": 2718, + "time_per_iteration": 3.424151659011841 + }, + { + "auxiliary_loss_clip": 0.01156832, + "auxiliary_loss_mlp": 0.01045652, + "balance_loss_clip": 1.05298364, + "balance_loss_mlp": 1.02558899, + "epoch": 0.32694041964768833, + "flos": 24535175938560.0, + "grad_norm": 2.6287713780796134, + "language_loss": 0.71614575, + "learning_rate": 3.1443506646303934e-06, + "loss": 0.73817062, + "num_input_tokens_seen": 58626700, + "router_z_loss_clip": 1.03857422, + "router_z_loss_mlp": 0.20043945, + "step": 2719, + "time_per_iteration": 2.830782890319824 + }, + { + "auxiliary_loss_clip": 0.01159639, + "auxiliary_loss_mlp": 0.01049075, + "balance_loss_clip": 1.05033088, + "balance_loss_mlp": 1.02809393, + "epoch": 0.32706066253832744, + "flos": 33183003755520.0, + "grad_norm": 3.1726572186250372, + "language_loss": 0.67096138, + "learning_rate": 3.1437117187993086e-06, + "loss": 0.69304848, + "num_input_tokens_seen": 58649020, + "router_z_loss_clip": 1.09326172, + "router_z_loss_mlp": 0.20983887, + "step": 2720, + "time_per_iteration": 2.930161237716675 + }, + { + "auxiliary_loss_clip": 0.01161246, + "auxiliary_loss_mlp": 0.0105234, + "balance_loss_clip": 1.05135655, + "balance_loss_mlp": 1.03112102, + "epoch": 0.3271809054289665, + "flos": 24061622008320.0, + "grad_norm": 2.0154281179542175, + "language_loss": 0.79884267, + "learning_rate": 3.143072599468065e-06, + "loss": 0.82097852, + "num_input_tokens_seen": 58668845, + "router_z_loss_clip": 1.09863281, + "router_z_loss_mlp": 0.21228027, + "step": 2721, + "time_per_iteration": 2.871373176574707 + }, + { + "auxiliary_loss_clip": 0.01161635, + "auxiliary_loss_mlp": 0.01050272, + "balance_loss_clip": 1.0531317, + "balance_loss_mlp": 1.02972043, + "epoch": 0.3273011483196056, + "flos": 38253769712640.0, + "grad_norm": 6.291264994664551, + "language_loss": 0.75927818, + "learning_rate": 3.1424333067336174e-06, + "loss": 0.78139722, + "num_input_tokens_seen": 58691610, + "router_z_loss_clip": 1.08398438, + "router_z_loss_mlp": 0.20544434, + "step": 2722, + "time_per_iteration": 2.9553840160369873 + }, + { + "auxiliary_loss_clip": 0.01163552, + "auxiliary_loss_mlp": 0.01042436, + "balance_loss_clip": 1.05118585, + "balance_loss_mlp": 1.02132356, + "epoch": 0.3274213912102447, + "flos": 29054389582080.0, + "grad_norm": 1.7062036350276324, + "language_loss": 0.7799533, + "learning_rate": 3.141793840692945e-06, + "loss": 0.80201316, + "num_input_tokens_seen": 58712360, + "router_z_loss_clip": 1.12304688, + "router_z_loss_mlp": 0.21105957, + "step": 2723, + "time_per_iteration": 2.8649981021881104 + }, + { + "auxiliary_loss_clip": 0.01160092, + "auxiliary_loss_mlp": 0.01050334, + "balance_loss_clip": 1.0487206, + "balance_loss_mlp": 1.02917409, + "epoch": 0.32754163410088377, + "flos": 29133249891840.0, + "grad_norm": 2.17487767487497, + "language_loss": 0.6134094, + "learning_rate": 3.1411542014430553e-06, + "loss": 0.63551366, + "num_input_tokens_seen": 58733440, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.21166992, + "step": 2724, + "time_per_iteration": 2.8473148345947266 + }, + { + "auxiliary_loss_clip": 0.01156214, + "auxiliary_loss_mlp": 0.01048991, + "balance_loss_clip": 1.04829288, + "balance_loss_mlp": 1.02975082, + "epoch": 0.3276618769915229, + "flos": 20631075724800.0, + "grad_norm": 1.937493305389102, + "language_loss": 0.8219142, + "learning_rate": 3.1405143890809804e-06, + "loss": 0.84396625, + "num_input_tokens_seen": 58752735, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.19238281, + "step": 2725, + "time_per_iteration": 2.7756080627441406 + }, + { + "auxiliary_loss_clip": 0.01158076, + "auxiliary_loss_mlp": 0.01054481, + "balance_loss_clip": 1.05375075, + "balance_loss_mlp": 1.03361964, + "epoch": 0.327782119882162, + "flos": 18657425076480.0, + "grad_norm": 2.0882589523872093, + "language_loss": 0.69836515, + "learning_rate": 3.1398744037037796e-06, + "loss": 0.72049075, + "num_input_tokens_seen": 58772070, + "router_z_loss_clip": 1.04345703, + "router_z_loss_mlp": 0.20849609, + "step": 2726, + "time_per_iteration": 2.8291823863983154 + }, + { + "auxiliary_loss_clip": 0.01161058, + "auxiliary_loss_mlp": 0.01049512, + "balance_loss_clip": 1.0494864, + "balance_loss_mlp": 1.02795887, + "epoch": 0.32790236277280105, + "flos": 21795802133760.0, + "grad_norm": 22.39795959653222, + "language_loss": 0.84245872, + "learning_rate": 3.139234245408538e-06, + "loss": 0.86456442, + "num_input_tokens_seen": 58790950, + "router_z_loss_clip": 1.11621094, + "router_z_loss_mlp": 0.21569824, + "step": 2727, + "time_per_iteration": 2.7874245643615723 + }, + { + "auxiliary_loss_clip": 0.0115786, + "auxiliary_loss_mlp": 0.01042759, + "balance_loss_clip": 1.05164862, + "balance_loss_mlp": 1.02435303, + "epoch": 0.32802260566344016, + "flos": 23331414424320.0, + "grad_norm": 1.3872213835875078, + "language_loss": 0.76091778, + "learning_rate": 3.1385939142923666e-06, + "loss": 0.78292394, + "num_input_tokens_seen": 58813340, + "router_z_loss_clip": 1.06201172, + "router_z_loss_mlp": 0.18408203, + "step": 2728, + "time_per_iteration": 2.8645410537719727 + }, + { + "auxiliary_loss_clip": 0.01164031, + "auxiliary_loss_mlp": 0.01050563, + "balance_loss_clip": 1.05229974, + "balance_loss_mlp": 1.02736473, + "epoch": 0.3281428485540792, + "flos": 24206988349440.0, + "grad_norm": 2.009837005563715, + "language_loss": 0.78386337, + "learning_rate": 3.137953410452405e-06, + "loss": 0.80600929, + "num_input_tokens_seen": 58833610, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.23193359, + "step": 2729, + "time_per_iteration": 2.836888074874878 + }, + { + "auxiliary_loss_clip": 0.01161117, + "auxiliary_loss_mlp": 0.01055118, + "balance_loss_clip": 1.04947603, + "balance_loss_mlp": 1.03490055, + "epoch": 0.3282630914447183, + "flos": 34128962380800.0, + "grad_norm": 1.829616310213314, + "language_loss": 0.74845147, + "learning_rate": 3.1373127339858146e-06, + "loss": 0.77061379, + "num_input_tokens_seen": 58856210, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.20214844, + "step": 2730, + "time_per_iteration": 3.8786699771881104 + }, + { + "auxiliary_loss_clip": 0.01156045, + "auxiliary_loss_mlp": 0.01048246, + "balance_loss_clip": 1.04827404, + "balance_loss_mlp": 1.02899337, + "epoch": 0.32838333433535744, + "flos": 27600726170880.0, + "grad_norm": 2.010038631418887, + "language_loss": 0.7463451, + "learning_rate": 3.136671884989787e-06, + "loss": 0.76838803, + "num_input_tokens_seen": 58876120, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.19250488, + "step": 2731, + "time_per_iteration": 2.8202075958251953 + }, + { + "auxiliary_loss_clip": 0.01166954, + "auxiliary_loss_mlp": 0.01047803, + "balance_loss_clip": 1.05390215, + "balance_loss_mlp": 1.02729917, + "epoch": 0.3285035772259965, + "flos": 12349500935040.0, + "grad_norm": 3.2483133225195573, + "language_loss": 0.87253392, + "learning_rate": 3.1360308635615383e-06, + "loss": 0.89468145, + "num_input_tokens_seen": 58894660, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.2052002, + "step": 2732, + "time_per_iteration": 3.5664021968841553 + }, + { + "auxiliary_loss_clip": 0.01166629, + "auxiliary_loss_mlp": 0.01049229, + "balance_loss_clip": 1.05163133, + "balance_loss_mlp": 1.02536321, + "epoch": 0.3286238201166356, + "flos": 24316084932480.0, + "grad_norm": 4.5918864371408405, + "language_loss": 0.78707445, + "learning_rate": 3.135389669798311e-06, + "loss": 0.80923307, + "num_input_tokens_seen": 58912720, + "router_z_loss_clip": 1.15136719, + "router_z_loss_mlp": 0.23840332, + "step": 2733, + "time_per_iteration": 2.7638044357299805 + }, + { + "auxiliary_loss_clip": 0.01159989, + "auxiliary_loss_mlp": 0.01049485, + "balance_loss_clip": 1.05175757, + "balance_loss_mlp": 1.02954102, + "epoch": 0.3287440630072747, + "flos": 21392812471680.0, + "grad_norm": 2.1673883326039007, + "language_loss": 0.79958224, + "learning_rate": 3.134748303797373e-06, + "loss": 0.82167691, + "num_input_tokens_seen": 58930090, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.19958496, + "step": 2734, + "time_per_iteration": 3.583374500274658 + }, + { + "auxiliary_loss_clip": 0.01169511, + "auxiliary_loss_mlp": 0.01054107, + "balance_loss_clip": 1.05231786, + "balance_loss_mlp": 1.03045583, + "epoch": 0.32886430589791377, + "flos": 23732536579200.0, + "grad_norm": 2.22772330734606, + "language_loss": 0.81691802, + "learning_rate": 3.1341067656560203e-06, + "loss": 0.83915424, + "num_input_tokens_seen": 58947935, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.23681641, + "step": 2735, + "time_per_iteration": 3.662132501602173 + }, + { + "auxiliary_loss_clip": 0.01162597, + "auxiliary_loss_mlp": 0.01046715, + "balance_loss_clip": 1.05050731, + "balance_loss_mlp": 1.02504313, + "epoch": 0.3289845487885529, + "flos": 22418708814720.0, + "grad_norm": 2.112383954917711, + "language_loss": 0.86426085, + "learning_rate": 3.133465055471572e-06, + "loss": 0.88635397, + "num_input_tokens_seen": 58967720, + "router_z_loss_clip": 1.12060547, + "router_z_loss_mlp": 0.2166748, + "step": 2736, + "time_per_iteration": 2.81125807762146 + }, + { + "auxiliary_loss_clip": 0.01162681, + "auxiliary_loss_mlp": 0.01054956, + "balance_loss_clip": 1.05103397, + "balance_loss_mlp": 1.03243744, + "epoch": 0.329104791679192, + "flos": 19682603147520.0, + "grad_norm": 2.226324753658153, + "language_loss": 0.67089832, + "learning_rate": 3.1328231733413767e-06, + "loss": 0.6930747, + "num_input_tokens_seen": 58984360, + "router_z_loss_clip": 1.11621094, + "router_z_loss_mlp": 0.22521973, + "step": 2737, + "time_per_iteration": 2.8113653659820557 + }, + { + "auxiliary_loss_clip": 0.01164612, + "auxiliary_loss_mlp": 0.01049695, + "balance_loss_clip": 1.05233002, + "balance_loss_mlp": 1.02915502, + "epoch": 0.32922503456983104, + "flos": 15997234803840.0, + "grad_norm": 2.281075476402058, + "language_loss": 0.90671039, + "learning_rate": 3.1321811193628067e-06, + "loss": 0.92885351, + "num_input_tokens_seen": 59002505, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.20544434, + "step": 2738, + "time_per_iteration": 2.8133156299591064 + }, + { + "auxiliary_loss_clip": 0.01162126, + "auxiliary_loss_mlp": 0.01050533, + "balance_loss_clip": 1.04959607, + "balance_loss_mlp": 1.02741861, + "epoch": 0.32934527746047015, + "flos": 26834069260800.0, + "grad_norm": 2.2652514352519413, + "language_loss": 0.70329201, + "learning_rate": 3.131538893633261e-06, + "loss": 0.72541863, + "num_input_tokens_seen": 59022065, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.23120117, + "step": 2739, + "time_per_iteration": 2.8617396354675293 + }, + { + "auxiliary_loss_clip": 0.01160156, + "auxiliary_loss_mlp": 0.01051222, + "balance_loss_clip": 1.05084252, + "balance_loss_mlp": 1.03093243, + "epoch": 0.32946552035110926, + "flos": 23403774372480.0, + "grad_norm": 2.4725089908174125, + "language_loss": 0.78305024, + "learning_rate": 3.130896496250165e-06, + "loss": 0.80516398, + "num_input_tokens_seen": 59041890, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.20300293, + "step": 2740, + "time_per_iteration": 2.857022523880005 + }, + { + "auxiliary_loss_clip": 0.01161465, + "auxiliary_loss_mlp": 0.01046723, + "balance_loss_clip": 1.05061805, + "balance_loss_mlp": 1.02608788, + "epoch": 0.3295857632417483, + "flos": 14172470029440.0, + "grad_norm": 2.0887730543611576, + "language_loss": 0.87142909, + "learning_rate": 3.1302539273109693e-06, + "loss": 0.89351094, + "num_input_tokens_seen": 59058715, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.2064209, + "step": 2741, + "time_per_iteration": 2.8470215797424316 + }, + { + "auxiliary_loss_clip": 0.01167163, + "auxiliary_loss_mlp": 0.01050467, + "balance_loss_clip": 1.05266261, + "balance_loss_mlp": 1.02812767, + "epoch": 0.32970600613238743, + "flos": 22196708807040.0, + "grad_norm": 1.6067473945338981, + "language_loss": 0.80636138, + "learning_rate": 3.1296111869131513e-06, + "loss": 0.8285377, + "num_input_tokens_seen": 59076140, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.22351074, + "step": 2742, + "time_per_iteration": 2.906796455383301 + }, + { + "auxiliary_loss_clip": 0.01163075, + "auxiliary_loss_mlp": 0.01048271, + "balance_loss_clip": 1.05288339, + "balance_loss_mlp": 1.02645588, + "epoch": 0.32982624902302654, + "flos": 22053784590720.0, + "grad_norm": 2.2038036451874277, + "language_loss": 0.86017066, + "learning_rate": 3.1289682751542153e-06, + "loss": 0.88228416, + "num_input_tokens_seen": 59095700, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.21813965, + "step": 2743, + "time_per_iteration": 2.8334848880767822 + }, + { + "auxiliary_loss_clip": 0.01161617, + "auxiliary_loss_mlp": 0.01048119, + "balance_loss_clip": 1.05148649, + "balance_loss_mlp": 1.02762651, + "epoch": 0.3299464919136656, + "flos": 18661626967680.0, + "grad_norm": 3.1076274588151667, + "language_loss": 0.71494126, + "learning_rate": 3.1283251921316883e-06, + "loss": 0.73703861, + "num_input_tokens_seen": 59113445, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.20495605, + "step": 2744, + "time_per_iteration": 2.788161516189575 + }, + { + "auxiliary_loss_clip": 0.01160743, + "auxiliary_loss_mlp": 0.0104632, + "balance_loss_clip": 1.05251098, + "balance_loss_mlp": 1.02725804, + "epoch": 0.3300667348043047, + "flos": 13407357404160.0, + "grad_norm": 2.0531562096608926, + "language_loss": 0.80939007, + "learning_rate": 3.1276819379431277e-06, + "loss": 0.83146071, + "num_input_tokens_seen": 59131535, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.19055176, + "step": 2745, + "time_per_iteration": 2.825308084487915 + }, + { + "auxiliary_loss_clip": 0.01166912, + "auxiliary_loss_mlp": 0.01044575, + "balance_loss_clip": 1.05353761, + "balance_loss_mlp": 1.02445316, + "epoch": 0.33018697769494376, + "flos": 15742556398080.0, + "grad_norm": 3.7659248552684637, + "language_loss": 0.74968421, + "learning_rate": 3.1270385126861134e-06, + "loss": 0.77179909, + "num_input_tokens_seen": 59149520, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.20141602, + "step": 2746, + "time_per_iteration": 2.750408172607422 + }, + { + "auxiliary_loss_clip": 0.01161786, + "auxiliary_loss_mlp": 0.01048954, + "balance_loss_clip": 1.05249405, + "balance_loss_mlp": 1.02780676, + "epoch": 0.3303072205855829, + "flos": 18258601392000.0, + "grad_norm": 2.3290990142458385, + "language_loss": 0.82353783, + "learning_rate": 3.1263949164582533e-06, + "loss": 0.84564525, + "num_input_tokens_seen": 59169170, + "router_z_loss_clip": 1.09277344, + "router_z_loss_mlp": 0.21154785, + "step": 2747, + "time_per_iteration": 2.8103461265563965 + }, + { + "auxiliary_loss_clip": 0.01163822, + "auxiliary_loss_mlp": 0.01048102, + "balance_loss_clip": 1.05362868, + "balance_loss_mlp": 1.02679968, + "epoch": 0.330427463476222, + "flos": 17749424148480.0, + "grad_norm": 1.9554730740160513, + "language_loss": 0.78416711, + "learning_rate": 3.1257511493571797e-06, + "loss": 0.80628633, + "num_input_tokens_seen": 59187675, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.2130127, + "step": 2748, + "time_per_iteration": 2.791475772857666 + }, + { + "auxiliary_loss_clip": 0.01161162, + "auxiliary_loss_mlp": 0.01052168, + "balance_loss_clip": 1.05383241, + "balance_loss_mlp": 1.03286839, + "epoch": 0.33054770636686104, + "flos": 27162580072320.0, + "grad_norm": 2.5025711046189643, + "language_loss": 0.78589165, + "learning_rate": 3.125107211480552e-06, + "loss": 0.808025, + "num_input_tokens_seen": 59207610, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.19311523, + "step": 2749, + "time_per_iteration": 2.887641668319702 + }, + { + "auxiliary_loss_clip": 0.01160829, + "auxiliary_loss_mlp": 0.01048089, + "balance_loss_clip": 1.05236506, + "balance_loss_mlp": 1.02770472, + "epoch": 0.33066794925750015, + "flos": 20117193799680.0, + "grad_norm": 1.862503835418366, + "language_loss": 0.7987498, + "learning_rate": 3.124463102926054e-06, + "loss": 0.82083899, + "num_input_tokens_seen": 59226945, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.20373535, + "step": 2750, + "time_per_iteration": 2.842149496078491 + }, + { + "auxiliary_loss_clip": 0.01071851, + "auxiliary_loss_mlp": 0.0100324, + "balance_loss_clip": 1.03572226, + "balance_loss_mlp": 1.00085545, + "epoch": 0.33078819214813926, + "flos": 70642609718400.0, + "grad_norm": 0.7683957560601191, + "language_loss": 0.61517555, + "learning_rate": 3.1238188237913984e-06, + "loss": 0.63592649, + "num_input_tokens_seen": 59291485, + "router_z_loss_clip": 0.36132812, + "router_z_loss_mlp": 0.02380371, + "step": 2751, + "time_per_iteration": 3.451223373413086 + }, + { + "auxiliary_loss_clip": 0.01167722, + "auxiliary_loss_mlp": 0.01055077, + "balance_loss_clip": 1.05374575, + "balance_loss_mlp": 1.03285623, + "epoch": 0.3309084350387783, + "flos": 21141940907520.0, + "grad_norm": 2.080061211972801, + "language_loss": 0.76518989, + "learning_rate": 3.1231743741743202e-06, + "loss": 0.78741783, + "num_input_tokens_seen": 59310990, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.22229004, + "step": 2752, + "time_per_iteration": 2.857203960418701 + }, + { + "auxiliary_loss_clip": 0.01159009, + "auxiliary_loss_mlp": 0.01043625, + "balance_loss_clip": 1.05106425, + "balance_loss_mlp": 1.02288246, + "epoch": 0.3310286779294174, + "flos": 14209350318720.0, + "grad_norm": 2.1145704130768963, + "language_loss": 0.83573222, + "learning_rate": 3.122529754172582e-06, + "loss": 0.85775864, + "num_input_tokens_seen": 59327875, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.20739746, + "step": 2753, + "time_per_iteration": 2.774350881576538 + }, + { + "auxiliary_loss_clip": 0.0116295, + "auxiliary_loss_mlp": 0.01054082, + "balance_loss_clip": 1.0538168, + "balance_loss_mlp": 1.03332722, + "epoch": 0.33114892082005654, + "flos": 20778130005120.0, + "grad_norm": 1.8398874010658754, + "language_loss": 0.72681659, + "learning_rate": 3.1218849638839736e-06, + "loss": 0.74898696, + "num_input_tokens_seen": 59347135, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.2076416, + "step": 2754, + "time_per_iteration": 2.810025930404663 + }, + { + "auxiliary_loss_clip": 0.01167381, + "auxiliary_loss_mlp": 0.0105505, + "balance_loss_clip": 1.0534941, + "balance_loss_mlp": 1.03240025, + "epoch": 0.3312691637106956, + "flos": 17090750499840.0, + "grad_norm": 2.248793648413503, + "language_loss": 0.78648502, + "learning_rate": 3.121240003406307e-06, + "loss": 0.80870926, + "num_input_tokens_seen": 59365985, + "router_z_loss_clip": 1.14013672, + "router_z_loss_mlp": 0.22668457, + "step": 2755, + "time_per_iteration": 2.806783676147461 + }, + { + "auxiliary_loss_clip": 0.01165201, + "auxiliary_loss_mlp": 0.01050502, + "balance_loss_clip": 1.05334687, + "balance_loss_mlp": 1.03022408, + "epoch": 0.3313894066013347, + "flos": 29456230008960.0, + "grad_norm": 2.094322561525272, + "language_loss": 0.72923964, + "learning_rate": 3.120594872837425e-06, + "loss": 0.75139666, + "num_input_tokens_seen": 59384655, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.20275879, + "step": 2756, + "time_per_iteration": 2.887106418609619 + }, + { + "auxiliary_loss_clip": 0.01067064, + "auxiliary_loss_mlp": 0.01003575, + "balance_loss_clip": 1.03110147, + "balance_loss_mlp": 1.00132227, + "epoch": 0.3315096494919738, + "flos": 61419242280960.0, + "grad_norm": 0.8363109818346269, + "language_loss": 0.62372935, + "learning_rate": 3.1199495722751906e-06, + "loss": 0.64443576, + "num_input_tokens_seen": 59444185, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.02258301, + "step": 2757, + "time_per_iteration": 3.3800206184387207 + }, + { + "auxiliary_loss_clip": 0.01165272, + "auxiliary_loss_mlp": 0.01054869, + "balance_loss_clip": 1.05519128, + "balance_loss_mlp": 1.0336138, + "epoch": 0.33162989238261287, + "flos": 21653057485440.0, + "grad_norm": 1.8132434267347317, + "language_loss": 0.83850884, + "learning_rate": 3.1193041018174972e-06, + "loss": 0.86071026, + "num_input_tokens_seen": 59464900, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.21264648, + "step": 2758, + "time_per_iteration": 2.8281795978546143 + }, + { + "auxiliary_loss_clip": 0.01169531, + "auxiliary_loss_mlp": 0.01048346, + "balance_loss_clip": 1.0551064, + "balance_loss_mlp": 1.02721071, + "epoch": 0.331750135273252, + "flos": 22674787850880.0, + "grad_norm": 2.48160381648453, + "language_loss": 0.94839144, + "learning_rate": 3.118658461562261e-06, + "loss": 0.97057033, + "num_input_tokens_seen": 59481000, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.21118164, + "step": 2759, + "time_per_iteration": 3.7667794227600098 + }, + { + "auxiliary_loss_clip": 0.01161357, + "auxiliary_loss_mlp": 0.0105918, + "balance_loss_clip": 1.05397439, + "balance_loss_mlp": 1.03937936, + "epoch": 0.33187037816389103, + "flos": 22746896403840.0, + "grad_norm": 1.9675798928322539, + "language_loss": 0.85016072, + "learning_rate": 3.118012651607426e-06, + "loss": 0.87236607, + "num_input_tokens_seen": 59502605, + "router_z_loss_clip": 1.07226562, + "router_z_loss_mlp": 0.19812012, + "step": 2760, + "time_per_iteration": 2.8170106410980225 + }, + { + "auxiliary_loss_clip": 0.01165005, + "auxiliary_loss_mlp": 0.01043807, + "balance_loss_clip": 1.05364728, + "balance_loss_mlp": 1.02262354, + "epoch": 0.33199062105453014, + "flos": 19203769918080.0, + "grad_norm": 2.1923291122305324, + "language_loss": 0.83748901, + "learning_rate": 3.1173666720509603e-06, + "loss": 0.85957712, + "num_input_tokens_seen": 59519540, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.21191406, + "step": 2761, + "time_per_iteration": 3.7379307746887207 + }, + { + "auxiliary_loss_clip": 0.01165785, + "auxiliary_loss_mlp": 0.01044189, + "balance_loss_clip": 1.05620217, + "balance_loss_mlp": 1.02356601, + "epoch": 0.33211086394516925, + "flos": 31577006764800.0, + "grad_norm": 1.9445704624725428, + "language_loss": 0.68107253, + "learning_rate": 3.116720522990859e-06, + "loss": 0.70317227, + "num_input_tokens_seen": 59540415, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.20629883, + "step": 2762, + "time_per_iteration": 2.9091248512268066 + }, + { + "auxiliary_loss_clip": 0.01164911, + "auxiliary_loss_mlp": 0.01054558, + "balance_loss_clip": 1.05339003, + "balance_loss_mlp": 1.0342567, + "epoch": 0.3322311068358083, + "flos": 17932496791680.0, + "grad_norm": 2.1640574745096, + "language_loss": 0.62470269, + "learning_rate": 3.116074204525142e-06, + "loss": 0.64689732, + "num_input_tokens_seen": 59558590, + "router_z_loss_clip": 1.11425781, + "router_z_loss_mlp": 0.20288086, + "step": 2763, + "time_per_iteration": 3.723804235458374 + }, + { + "auxiliary_loss_clip": 0.01164592, + "auxiliary_loss_mlp": 0.01048526, + "balance_loss_clip": 1.05424368, + "balance_loss_mlp": 1.02797401, + "epoch": 0.3323513497264474, + "flos": 32269831269120.0, + "grad_norm": 2.3975914071027944, + "language_loss": 0.83939278, + "learning_rate": 3.1154277167518553e-06, + "loss": 0.86152399, + "num_input_tokens_seen": 59580205, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.20568848, + "step": 2764, + "time_per_iteration": 3.856931209564209 + }, + { + "auxiliary_loss_clip": 0.01062296, + "auxiliary_loss_mlp": 0.01001868, + "balance_loss_clip": 1.0262568, + "balance_loss_mlp": 0.99962729, + "epoch": 0.33247159261708653, + "flos": 52668674588160.0, + "grad_norm": 0.7786501644713224, + "language_loss": 0.59425795, + "learning_rate": 3.114781059769072e-06, + "loss": 0.61489964, + "num_input_tokens_seen": 59631530, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.02246094, + "step": 2765, + "time_per_iteration": 3.200737476348877 + }, + { + "auxiliary_loss_clip": 0.01164745, + "auxiliary_loss_mlp": 0.01049635, + "balance_loss_clip": 1.05423534, + "balance_loss_mlp": 1.02803433, + "epoch": 0.3325918355077256, + "flos": 27125232906240.0, + "grad_norm": 2.9998579387683053, + "language_loss": 0.68412375, + "learning_rate": 3.1141342336748874e-06, + "loss": 0.70626754, + "num_input_tokens_seen": 59651090, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.21606445, + "step": 2766, + "time_per_iteration": 2.8167247772216797 + }, + { + "auxiliary_loss_clip": 0.0115891, + "auxiliary_loss_mlp": 0.01054191, + "balance_loss_clip": 1.05421317, + "balance_loss_mlp": 1.03415179, + "epoch": 0.3327120783983647, + "flos": 23664414435840.0, + "grad_norm": 2.0447884213559413, + "language_loss": 0.82200658, + "learning_rate": 3.1134872385674253e-06, + "loss": 0.84413755, + "num_input_tokens_seen": 59675245, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.20031738, + "step": 2767, + "time_per_iteration": 2.825535535812378 + }, + { + "auxiliary_loss_clip": 0.01157461, + "auxiliary_loss_mlp": 0.0105433, + "balance_loss_clip": 1.05018044, + "balance_loss_mlp": 1.03250289, + "epoch": 0.3328323212890038, + "flos": 19171378828800.0, + "grad_norm": 1.7899258231310293, + "language_loss": 0.85968935, + "learning_rate": 3.1128400745448353e-06, + "loss": 0.88180733, + "num_input_tokens_seen": 59694625, + "router_z_loss_clip": 1.07226562, + "router_z_loss_mlp": 0.21826172, + "step": 2768, + "time_per_iteration": 2.7832157611846924 + }, + { + "auxiliary_loss_clip": 0.01166711, + "auxiliary_loss_mlp": 0.01058585, + "balance_loss_clip": 1.05585861, + "balance_loss_mlp": 1.03843856, + "epoch": 0.33295256417964286, + "flos": 37706347463040.0, + "grad_norm": 2.210202579975115, + "language_loss": 0.63361406, + "learning_rate": 3.11219274170529e-06, + "loss": 0.65586704, + "num_input_tokens_seen": 59716435, + "router_z_loss_clip": 1.10839844, + "router_z_loss_mlp": 0.20141602, + "step": 2769, + "time_per_iteration": 2.9411306381225586 + }, + { + "auxiliary_loss_clip": 0.01156872, + "auxiliary_loss_mlp": 0.01052274, + "balance_loss_clip": 1.0508256, + "balance_loss_mlp": 1.03310478, + "epoch": 0.333072807070282, + "flos": 26505989412480.0, + "grad_norm": 2.3479568608073347, + "language_loss": 0.82212901, + "learning_rate": 3.1115452401469903e-06, + "loss": 0.84422046, + "num_input_tokens_seen": 59736835, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.19165039, + "step": 2770, + "time_per_iteration": 2.8110604286193848 + }, + { + "auxiliary_loss_clip": 0.01159209, + "auxiliary_loss_mlp": 0.01050007, + "balance_loss_clip": 1.05387306, + "balance_loss_mlp": 1.03091002, + "epoch": 0.3331930499609211, + "flos": 21430913823360.0, + "grad_norm": 2.171559741741788, + "language_loss": 0.86630201, + "learning_rate": 3.1108975699681613e-06, + "loss": 0.88839424, + "num_input_tokens_seen": 59754230, + "router_z_loss_clip": 1.05371094, + "router_z_loss_mlp": 0.19104004, + "step": 2771, + "time_per_iteration": 2.810519218444824 + }, + { + "auxiliary_loss_clip": 0.0115434, + "auxiliary_loss_mlp": 0.01064052, + "balance_loss_clip": 1.05100894, + "balance_loss_mlp": 1.04515719, + "epoch": 0.33331329285156014, + "flos": 20659947281280.0, + "grad_norm": 2.5575460420450966, + "language_loss": 0.72076249, + "learning_rate": 3.1102497312670542e-06, + "loss": 0.74294639, + "num_input_tokens_seen": 59772235, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.18896484, + "step": 2772, + "time_per_iteration": 2.8045156002044678 + }, + { + "auxiliary_loss_clip": 0.01155477, + "auxiliary_loss_mlp": 0.01053117, + "balance_loss_clip": 1.04896283, + "balance_loss_mlp": 1.03299487, + "epoch": 0.33343353574219925, + "flos": 28001596930560.0, + "grad_norm": 1.88129610594865, + "language_loss": 0.80689776, + "learning_rate": 3.109601724141946e-06, + "loss": 0.82898366, + "num_input_tokens_seen": 59791230, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.20117188, + "step": 2773, + "time_per_iteration": 3.011890411376953 + }, + { + "auxiliary_loss_clip": 0.0115755, + "auxiliary_loss_mlp": 0.0105475, + "balance_loss_clip": 1.05118966, + "balance_loss_mlp": 1.03467524, + "epoch": 0.33355377863283836, + "flos": 23764963582080.0, + "grad_norm": 3.485399222300317, + "language_loss": 0.6817131, + "learning_rate": 3.108953548691138e-06, + "loss": 0.70383608, + "num_input_tokens_seen": 59811315, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.20068359, + "step": 2774, + "time_per_iteration": 2.830836057662964 + }, + { + "auxiliary_loss_clip": 0.01162111, + "auxiliary_loss_mlp": 0.01047069, + "balance_loss_clip": 1.05118489, + "balance_loss_mlp": 1.0255518, + "epoch": 0.3336740215234774, + "flos": 37779677078400.0, + "grad_norm": 2.559569246164907, + "language_loss": 0.72286451, + "learning_rate": 3.108305205012959e-06, + "loss": 0.74495631, + "num_input_tokens_seen": 59832010, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.21508789, + "step": 2775, + "time_per_iteration": 2.955833673477173 + }, + { + "auxiliary_loss_clip": 0.01156776, + "auxiliary_loss_mlp": 0.01052082, + "balance_loss_clip": 1.05056834, + "balance_loss_mlp": 1.03197181, + "epoch": 0.3337942644141165, + "flos": 25519056347520.0, + "grad_norm": 1.9995162645507891, + "language_loss": 0.87603372, + "learning_rate": 3.107656693205761e-06, + "loss": 0.89812231, + "num_input_tokens_seen": 59851450, + "router_z_loss_clip": 1.06152344, + "router_z_loss_mlp": 0.20117188, + "step": 2776, + "time_per_iteration": 2.923081636428833 + }, + { + "auxiliary_loss_clip": 0.01166766, + "auxiliary_loss_mlp": 0.01052814, + "balance_loss_clip": 1.05196047, + "balance_loss_mlp": 1.02973545, + "epoch": 0.3339145073047556, + "flos": 25989844930560.0, + "grad_norm": 2.5670409295765197, + "language_loss": 0.70355093, + "learning_rate": 3.107008013367924e-06, + "loss": 0.72574681, + "num_input_tokens_seen": 59870245, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.23071289, + "step": 2777, + "time_per_iteration": 2.8650593757629395 + }, + { + "auxiliary_loss_clip": 0.01159025, + "auxiliary_loss_mlp": 0.01048318, + "balance_loss_clip": 1.05113292, + "balance_loss_mlp": 1.02782547, + "epoch": 0.3340347501953947, + "flos": 19062569554560.0, + "grad_norm": 2.1658244087303036, + "language_loss": 0.86494291, + "learning_rate": 3.1063591655978507e-06, + "loss": 0.8870163, + "num_input_tokens_seen": 59886195, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.20495605, + "step": 2778, + "time_per_iteration": 2.861833095550537 + }, + { + "auxiliary_loss_clip": 0.01154648, + "auxiliary_loss_mlp": 0.01049625, + "balance_loss_clip": 1.048576, + "balance_loss_mlp": 1.0296216, + "epoch": 0.3341549930860338, + "flos": 18109715518080.0, + "grad_norm": 1.8769228964148663, + "language_loss": 0.79980218, + "learning_rate": 3.105710149993972e-06, + "loss": 0.82184494, + "num_input_tokens_seen": 59905525, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.20007324, + "step": 2779, + "time_per_iteration": 3.024447441101074 + }, + { + "auxiliary_loss_clip": 0.01162318, + "auxiliary_loss_mlp": 0.01047283, + "balance_loss_clip": 1.05152154, + "balance_loss_mlp": 1.02658832, + "epoch": 0.33427523597667286, + "flos": 22674967418880.0, + "grad_norm": 7.203754989082763, + "language_loss": 0.85654819, + "learning_rate": 3.1050609666547427e-06, + "loss": 0.87864423, + "num_input_tokens_seen": 59925085, + "router_z_loss_clip": 1.10742188, + "router_z_loss_mlp": 0.20678711, + "step": 2780, + "time_per_iteration": 2.8500962257385254 + }, + { + "auxiliary_loss_clip": 0.01158831, + "auxiliary_loss_mlp": 0.01053737, + "balance_loss_clip": 1.05062246, + "balance_loss_mlp": 1.03365016, + "epoch": 0.33439547886731197, + "flos": 22638338524800.0, + "grad_norm": 1.9144788190150042, + "language_loss": 0.77763081, + "learning_rate": 3.104411615678644e-06, + "loss": 0.79975647, + "num_input_tokens_seen": 59943935, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.2010498, + "step": 2781, + "time_per_iteration": 2.8176960945129395 + }, + { + "auxiliary_loss_clip": 0.01161483, + "auxiliary_loss_mlp": 0.01053501, + "balance_loss_clip": 1.05142879, + "balance_loss_mlp": 1.03249657, + "epoch": 0.3345157217579511, + "flos": 24096383395200.0, + "grad_norm": 2.6817853608783953, + "language_loss": 0.73857212, + "learning_rate": 3.1037620971641803e-06, + "loss": 0.76072198, + "num_input_tokens_seen": 59963725, + "router_z_loss_clip": 1.09863281, + "router_z_loss_mlp": 0.21008301, + "step": 2782, + "time_per_iteration": 2.8236734867095947 + }, + { + "auxiliary_loss_clip": 0.01161638, + "auxiliary_loss_mlp": 0.01046047, + "balance_loss_clip": 1.05056882, + "balance_loss_mlp": 1.02472043, + "epoch": 0.33463596464859013, + "flos": 18989491334400.0, + "grad_norm": 2.552767069831178, + "language_loss": 0.6548413, + "learning_rate": 3.1031124112098844e-06, + "loss": 0.67691815, + "num_input_tokens_seen": 59981935, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.21350098, + "step": 2783, + "time_per_iteration": 2.8825769424438477 + }, + { + "auxiliary_loss_clip": 0.01159175, + "auxiliary_loss_mlp": 0.01047802, + "balance_loss_clip": 1.05247116, + "balance_loss_mlp": 1.0286932, + "epoch": 0.33475620753922924, + "flos": 20375607219840.0, + "grad_norm": 3.1267310257176857, + "language_loss": 0.72098219, + "learning_rate": 3.1024625579143127e-06, + "loss": 0.74305195, + "num_input_tokens_seen": 59999455, + "router_z_loss_clip": 1.06835938, + "router_z_loss_mlp": 0.19104004, + "step": 2784, + "time_per_iteration": 2.8659842014312744 + }, + { + "auxiliary_loss_clip": 0.0116112, + "auxiliary_loss_mlp": 0.010514, + "balance_loss_clip": 1.05379748, + "balance_loss_mlp": 1.03174233, + "epoch": 0.33487645042986836, + "flos": 18182578256640.0, + "grad_norm": 1.9646723102804726, + "language_loss": 0.73187351, + "learning_rate": 3.101812537376048e-06, + "loss": 0.75399882, + "num_input_tokens_seen": 60018475, + "router_z_loss_clip": 1.07373047, + "router_z_loss_mlp": 0.19665527, + "step": 2785, + "time_per_iteration": 2.8731513023376465 + }, + { + "auxiliary_loss_clip": 0.01161016, + "auxiliary_loss_mlp": 0.01048166, + "balance_loss_clip": 1.05246282, + "balance_loss_mlp": 1.02837777, + "epoch": 0.3349966933205074, + "flos": 25848824135040.0, + "grad_norm": 2.257449093873185, + "language_loss": 0.84202188, + "learning_rate": 3.1011623496936973e-06, + "loss": 0.86411369, + "num_input_tokens_seen": 60036770, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.19775391, + "step": 2786, + "time_per_iteration": 2.8385727405548096 + }, + { + "auxiliary_loss_clip": 0.01159406, + "auxiliary_loss_mlp": 0.01046323, + "balance_loss_clip": 1.05230689, + "balance_loss_mlp": 1.02708292, + "epoch": 0.3351169362111465, + "flos": 28111447699200.0, + "grad_norm": 2.8822514146222398, + "language_loss": 0.69464988, + "learning_rate": 3.100511994965893e-06, + "loss": 0.71670717, + "num_input_tokens_seen": 60056725, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.19250488, + "step": 2787, + "time_per_iteration": 2.819659948348999 + }, + { + "auxiliary_loss_clip": 0.01158268, + "auxiliary_loss_mlp": 0.01056769, + "balance_loss_clip": 1.05226731, + "balance_loss_mlp": 1.03739715, + "epoch": 0.33523717910178563, + "flos": 22673315393280.0, + "grad_norm": 1.737928351661573, + "language_loss": 0.845007, + "learning_rate": 3.0998614732912947e-06, + "loss": 0.86715734, + "num_input_tokens_seen": 60076100, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.19384766, + "step": 2788, + "time_per_iteration": 3.750786781311035 + }, + { + "auxiliary_loss_clip": 0.01160235, + "auxiliary_loss_mlp": 0.0105793, + "balance_loss_clip": 1.05456805, + "balance_loss_mlp": 1.0382849, + "epoch": 0.3353574219924247, + "flos": 15669801400320.0, + "grad_norm": 1.9461453995699587, + "language_loss": 0.67893267, + "learning_rate": 3.0992107847685855e-06, + "loss": 0.7011143, + "num_input_tokens_seen": 60093815, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.19641113, + "step": 2789, + "time_per_iteration": 2.7974750995635986 + }, + { + "auxiliary_loss_clip": 0.01166959, + "auxiliary_loss_mlp": 0.01062883, + "balance_loss_clip": 1.05592322, + "balance_loss_mlp": 1.04020965, + "epoch": 0.3354776648830638, + "flos": 24790644443520.0, + "grad_norm": 1.6426485948470715, + "language_loss": 0.79598099, + "learning_rate": 3.0985599294964736e-06, + "loss": 0.81827939, + "num_input_tokens_seen": 60113370, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.22692871, + "step": 2790, + "time_per_iteration": 3.680264472961426 + }, + { + "auxiliary_loss_clip": 0.01165646, + "auxiliary_loss_mlp": 0.01059151, + "balance_loss_clip": 1.0547483, + "balance_loss_mlp": 1.03863525, + "epoch": 0.33559790777370285, + "flos": 28694852398080.0, + "grad_norm": 2.058258820242392, + "language_loss": 0.70298111, + "learning_rate": 3.097908907573695e-06, + "loss": 0.72522908, + "num_input_tokens_seen": 60131350, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.20507812, + "step": 2791, + "time_per_iteration": 2.806042194366455 + }, + { + "auxiliary_loss_clip": 0.01160611, + "auxiliary_loss_mlp": 0.01051141, + "balance_loss_clip": 1.05322683, + "balance_loss_mlp": 1.03074467, + "epoch": 0.33571815066434196, + "flos": 22235779825920.0, + "grad_norm": 2.7370025999494714, + "language_loss": 0.89804596, + "learning_rate": 3.0972577190990067e-06, + "loss": 0.92016351, + "num_input_tokens_seen": 60149830, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.20397949, + "step": 2792, + "time_per_iteration": 3.651289463043213 + }, + { + "auxiliary_loss_clip": 0.01159716, + "auxiliary_loss_mlp": 0.01048987, + "balance_loss_clip": 1.05179346, + "balance_loss_mlp": 1.03030694, + "epoch": 0.3358383935549811, + "flos": 23842279607040.0, + "grad_norm": 1.9614406945323697, + "language_loss": 0.80538201, + "learning_rate": 3.096606364171196e-06, + "loss": 0.82746899, + "num_input_tokens_seen": 60169620, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.18664551, + "step": 2793, + "time_per_iteration": 3.6481285095214844 + }, + { + "auxiliary_loss_clip": 0.01161203, + "auxiliary_loss_mlp": 0.01051468, + "balance_loss_clip": 1.05483139, + "balance_loss_mlp": 1.03303862, + "epoch": 0.33595863644562013, + "flos": 22267308988800.0, + "grad_norm": 3.3148716211919824, + "language_loss": 0.85222733, + "learning_rate": 3.0959548428890703e-06, + "loss": 0.87435406, + "num_input_tokens_seen": 60188490, + "router_z_loss_clip": 1.06347656, + "router_z_loss_mlp": 0.18432617, + "step": 2794, + "time_per_iteration": 2.839277744293213 + }, + { + "auxiliary_loss_clip": 0.01160304, + "auxiliary_loss_mlp": 0.01058627, + "balance_loss_clip": 1.05373549, + "balance_loss_mlp": 1.03858757, + "epoch": 0.33607887933625924, + "flos": 20119779578880.0, + "grad_norm": 1.6686982687656702, + "language_loss": 0.84550309, + "learning_rate": 3.095303155351468e-06, + "loss": 0.86769241, + "num_input_tokens_seen": 60208695, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.20043945, + "step": 2795, + "time_per_iteration": 2.8432776927948 + }, + { + "auxiliary_loss_clip": 0.01161318, + "auxiliary_loss_mlp": 0.01051442, + "balance_loss_clip": 1.05398917, + "balance_loss_mlp": 1.0318203, + "epoch": 0.33619912222689835, + "flos": 19318109886720.0, + "grad_norm": 2.4671039723365276, + "language_loss": 0.79555267, + "learning_rate": 3.0946513016572464e-06, + "loss": 0.8176803, + "num_input_tokens_seen": 60227600, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.19628906, + "step": 2796, + "time_per_iteration": 2.873316764831543 + }, + { + "auxiliary_loss_clip": 0.01165814, + "auxiliary_loss_mlp": 0.01043022, + "balance_loss_clip": 1.05123127, + "balance_loss_mlp": 1.02230382, + "epoch": 0.3363193651175374, + "flos": 16800664262400.0, + "grad_norm": 1.9223991762169022, + "language_loss": 0.77163565, + "learning_rate": 3.0939992819052938e-06, + "loss": 0.79372406, + "num_input_tokens_seen": 60245110, + "router_z_loss_clip": 1.14746094, + "router_z_loss_mlp": 0.20727539, + "step": 2797, + "time_per_iteration": 2.8302903175354004 + }, + { + "auxiliary_loss_clip": 0.01163908, + "auxiliary_loss_mlp": 0.01059249, + "balance_loss_clip": 1.05461311, + "balance_loss_mlp": 1.03947258, + "epoch": 0.3364396080081765, + "flos": 23550289948800.0, + "grad_norm": 15.83517519625304, + "language_loss": 0.81337941, + "learning_rate": 3.0933470961945193e-06, + "loss": 0.83561099, + "num_input_tokens_seen": 60263405, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.19763184, + "step": 2798, + "time_per_iteration": 2.8224079608917236 + }, + { + "auxiliary_loss_clip": 0.01159889, + "auxiliary_loss_mlp": 0.01055457, + "balance_loss_clip": 1.05426836, + "balance_loss_mlp": 1.0360502, + "epoch": 0.3365598508988156, + "flos": 28037902602240.0, + "grad_norm": 1.807916043823043, + "language_loss": 0.68383145, + "learning_rate": 3.0926947446238597e-06, + "loss": 0.70598489, + "num_input_tokens_seen": 60282975, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.19396973, + "step": 2799, + "time_per_iteration": 2.94942569732666 + }, + { + "auxiliary_loss_clip": 0.01161945, + "auxiliary_loss_mlp": 0.01052105, + "balance_loss_clip": 1.04965949, + "balance_loss_mlp": 1.0311358, + "epoch": 0.3366800937894547, + "flos": 16982767238400.0, + "grad_norm": 2.3474422864376523, + "language_loss": 0.82294273, + "learning_rate": 3.092042227292276e-06, + "loss": 0.84508324, + "num_input_tokens_seen": 60299810, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.2097168, + "step": 2800, + "time_per_iteration": 2.9691390991210938 + }, + { + "auxiliary_loss_clip": 0.01157519, + "auxiliary_loss_mlp": 0.0105487, + "balance_loss_clip": 1.05139315, + "balance_loss_mlp": 1.03593993, + "epoch": 0.3368003366800938, + "flos": 23915321913600.0, + "grad_norm": 1.6805757448477971, + "language_loss": 0.88300568, + "learning_rate": 3.0913895442987557e-06, + "loss": 0.90512955, + "num_input_tokens_seen": 60320775, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.18945312, + "step": 2801, + "time_per_iteration": 2.864330768585205 + }, + { + "auxiliary_loss_clip": 0.01162842, + "auxiliary_loss_mlp": 0.01047395, + "balance_loss_clip": 1.05363846, + "balance_loss_mlp": 1.02769005, + "epoch": 0.3369205795707329, + "flos": 24791219061120.0, + "grad_norm": 2.4690819793169494, + "language_loss": 0.86255592, + "learning_rate": 3.090736695742308e-06, + "loss": 0.88465834, + "num_input_tokens_seen": 60341905, + "router_z_loss_clip": 1.09277344, + "router_z_loss_mlp": 0.19702148, + "step": 2802, + "time_per_iteration": 2.886939525604248 + }, + { + "auxiliary_loss_clip": 0.01159782, + "auxiliary_loss_mlp": 0.01050279, + "balance_loss_clip": 1.05314791, + "balance_loss_mlp": 1.03164697, + "epoch": 0.33704082246137196, + "flos": 17931096161280.0, + "grad_norm": 4.064774198533521, + "language_loss": 0.52303541, + "learning_rate": 3.0900836817219713e-06, + "loss": 0.54513597, + "num_input_tokens_seen": 60358335, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.18640137, + "step": 2803, + "time_per_iteration": 2.8938779830932617 + }, + { + "auxiliary_loss_clip": 0.011602, + "auxiliary_loss_mlp": 0.01052283, + "balance_loss_clip": 1.0522759, + "balance_loss_mlp": 1.03275621, + "epoch": 0.33716106535201107, + "flos": 21286517149440.0, + "grad_norm": 1.762863745504328, + "language_loss": 0.83922744, + "learning_rate": 3.089430502336807e-06, + "loss": 0.86135226, + "num_input_tokens_seen": 60378305, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.19519043, + "step": 2804, + "time_per_iteration": 2.8886966705322266 + }, + { + "auxiliary_loss_clip": 0.01165906, + "auxiliary_loss_mlp": 0.01048877, + "balance_loss_clip": 1.05443859, + "balance_loss_mlp": 1.02814686, + "epoch": 0.3372813082426502, + "flos": 18402962152320.0, + "grad_norm": 2.279503279105009, + "language_loss": 0.90616912, + "learning_rate": 3.088777157685902e-06, + "loss": 0.92831695, + "num_input_tokens_seen": 60393895, + "router_z_loss_clip": 1.11425781, + "router_z_loss_mlp": 0.20727539, + "step": 2805, + "time_per_iteration": 2.812617063522339 + }, + { + "auxiliary_loss_clip": 0.01160909, + "auxiliary_loss_mlp": 0.01044706, + "balance_loss_clip": 1.05539441, + "balance_loss_mlp": 1.02497649, + "epoch": 0.33740155113328923, + "flos": 17201391367680.0, + "grad_norm": 3.849201657477278, + "language_loss": 0.86294407, + "learning_rate": 3.088123647868367e-06, + "loss": 0.88500017, + "num_input_tokens_seen": 60410445, + "router_z_loss_clip": 1.05566406, + "router_z_loss_mlp": 0.19726562, + "step": 2806, + "time_per_iteration": 2.791928768157959 + }, + { + "auxiliary_loss_clip": 0.01162542, + "auxiliary_loss_mlp": 0.01052498, + "balance_loss_clip": 1.0521667, + "balance_loss_mlp": 1.03281677, + "epoch": 0.33752179402392835, + "flos": 29058950609280.0, + "grad_norm": 2.7427235388592726, + "language_loss": 0.81193233, + "learning_rate": 3.0874699729833405e-06, + "loss": 0.83408272, + "num_input_tokens_seen": 60431815, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.19677734, + "step": 2807, + "time_per_iteration": 2.844236135482788 + }, + { + "auxiliary_loss_clip": 0.01158935, + "auxiliary_loss_mlp": 0.01048227, + "balance_loss_clip": 1.05062568, + "balance_loss_mlp": 1.02766347, + "epoch": 0.3376420369145674, + "flos": 25080730680960.0, + "grad_norm": 1.6917404661379987, + "language_loss": 0.80118018, + "learning_rate": 3.086816133129983e-06, + "loss": 0.82325184, + "num_input_tokens_seen": 60452075, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.20556641, + "step": 2808, + "time_per_iteration": 2.8449954986572266 + }, + { + "auxiliary_loss_clip": 0.01163156, + "auxiliary_loss_mlp": 0.01054658, + "balance_loss_clip": 1.05635464, + "balance_loss_mlp": 1.03519106, + "epoch": 0.3377622798052065, + "flos": 27490624007040.0, + "grad_norm": 2.0287838147976887, + "language_loss": 0.7620095, + "learning_rate": 3.0861621284074826e-06, + "loss": 0.78418767, + "num_input_tokens_seen": 60472600, + "router_z_loss_clip": 1.06835938, + "router_z_loss_mlp": 0.19482422, + "step": 2809, + "time_per_iteration": 2.841181755065918 + }, + { + "auxiliary_loss_clip": 0.0115838, + "auxiliary_loss_mlp": 0.01048867, + "balance_loss_clip": 1.05324495, + "balance_loss_mlp": 1.02948403, + "epoch": 0.3378825226958456, + "flos": 21975211589760.0, + "grad_norm": 1.5047155217504709, + "language_loss": 0.731686, + "learning_rate": 3.085507958915051e-06, + "loss": 0.75375843, + "num_input_tokens_seen": 60491030, + "router_z_loss_clip": 1.05322266, + "router_z_loss_mlp": 0.19384766, + "step": 2810, + "time_per_iteration": 2.841174364089966 + }, + { + "auxiliary_loss_clip": 0.01161803, + "auxiliary_loss_mlp": 0.01054689, + "balance_loss_clip": 1.05025077, + "balance_loss_mlp": 1.03332686, + "epoch": 0.3380027655864847, + "flos": 42523189200000.0, + "grad_norm": 1.9711485679347358, + "language_loss": 0.71468323, + "learning_rate": 3.084853624751925e-06, + "loss": 0.73684812, + "num_input_tokens_seen": 60512615, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.21374512, + "step": 2811, + "time_per_iteration": 3.0380377769470215 + }, + { + "auxiliary_loss_clip": 0.01161661, + "auxiliary_loss_mlp": 0.01049441, + "balance_loss_clip": 1.05365574, + "balance_loss_mlp": 1.03003347, + "epoch": 0.3381230084771238, + "flos": 26725080418560.0, + "grad_norm": 1.8414056270399852, + "language_loss": 0.86132097, + "learning_rate": 3.0841991260173668e-06, + "loss": 0.88343197, + "num_input_tokens_seen": 60532520, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.1940918, + "step": 2812, + "time_per_iteration": 2.869371175765991 + }, + { + "auxiliary_loss_clip": 0.01161167, + "auxiliary_loss_mlp": 0.01057328, + "balance_loss_clip": 1.05215764, + "balance_loss_mlp": 1.03677678, + "epoch": 0.3382432513677629, + "flos": 22710375250560.0, + "grad_norm": 2.706022020095574, + "language_loss": 0.80667436, + "learning_rate": 3.0835444628106634e-06, + "loss": 0.82885933, + "num_input_tokens_seen": 60551500, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.20556641, + "step": 2813, + "time_per_iteration": 2.8293042182922363 + }, + { + "auxiliary_loss_clip": 0.01160513, + "auxiliary_loss_mlp": 0.01053314, + "balance_loss_clip": 1.05293798, + "balance_loss_mlp": 1.03301263, + "epoch": 0.33836349425840195, + "flos": 22122409524480.0, + "grad_norm": 1.9099482552067422, + "language_loss": 0.83240354, + "learning_rate": 3.082889635231126e-06, + "loss": 0.85454184, + "num_input_tokens_seen": 60570160, + "router_z_loss_clip": 1.07617188, + "router_z_loss_mlp": 0.20300293, + "step": 2814, + "time_per_iteration": 2.798189401626587 + }, + { + "auxiliary_loss_clip": 0.01156926, + "auxiliary_loss_mlp": 0.01059453, + "balance_loss_clip": 1.04832935, + "balance_loss_mlp": 1.03960514, + "epoch": 0.33848373714904106, + "flos": 27308090067840.0, + "grad_norm": 3.7493985153540508, + "language_loss": 0.77101672, + "learning_rate": 3.0822346433780925e-06, + "loss": 0.79318053, + "num_input_tokens_seen": 60590885, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.1986084, + "step": 2815, + "time_per_iteration": 2.836205244064331 + }, + { + "auxiliary_loss_clip": 0.0116127, + "auxiliary_loss_mlp": 0.01042195, + "balance_loss_clip": 1.05103636, + "balance_loss_mlp": 1.02159572, + "epoch": 0.3386039800396802, + "flos": 25848716394240.0, + "grad_norm": 1.8876872797167976, + "language_loss": 0.8732568, + "learning_rate": 3.0815794873509237e-06, + "loss": 0.89529145, + "num_input_tokens_seen": 60609170, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.20605469, + "step": 2816, + "time_per_iteration": 3.729950189590454 + }, + { + "auxiliary_loss_clip": 0.01156827, + "auxiliary_loss_mlp": 0.01047613, + "balance_loss_clip": 1.05056822, + "balance_loss_mlp": 1.02805078, + "epoch": 0.33872422293031923, + "flos": 18880646146560.0, + "grad_norm": 1.8725806385945556, + "language_loss": 0.72847086, + "learning_rate": 3.0809241672490066e-06, + "loss": 0.75051528, + "num_input_tokens_seen": 60627340, + "router_z_loss_clip": 1.06347656, + "router_z_loss_mlp": 0.19555664, + "step": 2817, + "time_per_iteration": 2.948397636413574 + }, + { + "auxiliary_loss_clip": 0.01154135, + "auxiliary_loss_mlp": 0.01043063, + "balance_loss_clip": 1.04965699, + "balance_loss_mlp": 1.02390623, + "epoch": 0.33884446582095834, + "flos": 23146977064320.0, + "grad_norm": 1.6797743974980406, + "language_loss": 0.84986544, + "learning_rate": 3.080268683171753e-06, + "loss": 0.87183744, + "num_input_tokens_seen": 60647630, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.19152832, + "step": 2818, + "time_per_iteration": 2.8885371685028076 + }, + { + "auxiliary_loss_clip": 0.01152375, + "auxiliary_loss_mlp": 0.01051147, + "balance_loss_clip": 1.04696429, + "balance_loss_mlp": 1.0321337, + "epoch": 0.33896470871159745, + "flos": 15997342544640.0, + "grad_norm": 2.4529222021039057, + "language_loss": 0.89519101, + "learning_rate": 3.0796130352185985e-06, + "loss": 0.91722625, + "num_input_tokens_seen": 60664485, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.19006348, + "step": 2819, + "time_per_iteration": 3.638969898223877 + }, + { + "auxiliary_loss_clip": 0.01156448, + "auxiliary_loss_mlp": 0.01045767, + "balance_loss_clip": 1.04724324, + "balance_loss_mlp": 1.02568007, + "epoch": 0.3390849516022365, + "flos": 34495754112000.0, + "grad_norm": 1.797116669630948, + "language_loss": 0.6646179, + "learning_rate": 3.0789572234890057e-06, + "loss": 0.68664002, + "num_input_tokens_seen": 60686125, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.20092773, + "step": 2820, + "time_per_iteration": 2.9358975887298584 + }, + { + "auxiliary_loss_clip": 0.01156564, + "auxiliary_loss_mlp": 0.01054537, + "balance_loss_clip": 1.04846406, + "balance_loss_mlp": 1.03392625, + "epoch": 0.3392051944928756, + "flos": 16180307447040.0, + "grad_norm": 1.5315966737425648, + "language_loss": 0.773655, + "learning_rate": 3.0783012480824596e-06, + "loss": 0.795766, + "num_input_tokens_seen": 60705270, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.20605469, + "step": 2821, + "time_per_iteration": 4.511235952377319 + }, + { + "auxiliary_loss_clip": 0.01160861, + "auxiliary_loss_mlp": 0.01053932, + "balance_loss_clip": 1.05154502, + "balance_loss_mlp": 1.03382146, + "epoch": 0.33932543738351467, + "flos": 17086656349440.0, + "grad_norm": 2.383514278289759, + "language_loss": 0.74209034, + "learning_rate": 3.077645109098471e-06, + "loss": 0.7642383, + "num_input_tokens_seen": 60721540, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.2010498, + "step": 2822, + "time_per_iteration": 3.039076328277588 + }, + { + "auxiliary_loss_clip": 0.01156339, + "auxiliary_loss_mlp": 0.01040171, + "balance_loss_clip": 1.04951036, + "balance_loss_mlp": 1.02128816, + "epoch": 0.3394456802741538, + "flos": 22126970551680.0, + "grad_norm": 1.9271363625818718, + "language_loss": 0.72234952, + "learning_rate": 3.076988806636577e-06, + "loss": 0.74431467, + "num_input_tokens_seen": 60739300, + "router_z_loss_clip": 1.06884766, + "router_z_loss_mlp": 0.18884277, + "step": 2823, + "time_per_iteration": 2.8852438926696777 + }, + { + "auxiliary_loss_clip": 0.01159923, + "auxiliary_loss_mlp": 0.01053932, + "balance_loss_clip": 1.04909277, + "balance_loss_mlp": 1.03228378, + "epoch": 0.3395659231647929, + "flos": 25226887121280.0, + "grad_norm": 2.016026246412083, + "language_loss": 0.88561928, + "learning_rate": 3.0763323407963377e-06, + "loss": 0.90775776, + "num_input_tokens_seen": 60758910, + "router_z_loss_clip": 1.10839844, + "router_z_loss_mlp": 0.21630859, + "step": 2824, + "time_per_iteration": 2.8693370819091797 + }, + { + "auxiliary_loss_clip": 0.0116021, + "auxiliary_loss_mlp": 0.01051601, + "balance_loss_clip": 1.05127656, + "balance_loss_mlp": 1.03158545, + "epoch": 0.33968616605543195, + "flos": 29096477343360.0, + "grad_norm": 2.108447019702995, + "language_loss": 0.80386245, + "learning_rate": 3.075675711677337e-06, + "loss": 0.82598054, + "num_input_tokens_seen": 60779005, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.20007324, + "step": 2825, + "time_per_iteration": 2.804859161376953 + }, + { + "auxiliary_loss_clip": 0.01158268, + "auxiliary_loss_mlp": 0.0104936, + "balance_loss_clip": 1.05082583, + "balance_loss_mlp": 1.02983332, + "epoch": 0.33980640894607106, + "flos": 21433966479360.0, + "grad_norm": 2.199333834112002, + "language_loss": 0.78387165, + "learning_rate": 3.0750189193791865e-06, + "loss": 0.8059479, + "num_input_tokens_seen": 60798590, + "router_z_loss_clip": 1.07470703, + "router_z_loss_mlp": 0.1953125, + "step": 2826, + "time_per_iteration": 2.819753646850586 + }, + { + "auxiliary_loss_clip": 0.0115546, + "auxiliary_loss_mlp": 0.01047359, + "balance_loss_clip": 1.0489707, + "balance_loss_mlp": 1.02768946, + "epoch": 0.33992665183671017, + "flos": 32490035596800.0, + "grad_norm": 2.6789998846752794, + "language_loss": 0.70513815, + "learning_rate": 3.0743619640015203e-06, + "loss": 0.7271663, + "num_input_tokens_seen": 60818840, + "router_z_loss_clip": 1.06347656, + "router_z_loss_mlp": 0.19665527, + "step": 2827, + "time_per_iteration": 2.821016788482666 + }, + { + "auxiliary_loss_clip": 0.01159208, + "auxiliary_loss_mlp": 0.01047638, + "balance_loss_clip": 1.05150986, + "balance_loss_mlp": 1.02778959, + "epoch": 0.3400468947273492, + "flos": 17055414495360.0, + "grad_norm": 1.918614655647517, + "language_loss": 0.92708373, + "learning_rate": 3.073704845643999e-06, + "loss": 0.94915211, + "num_input_tokens_seen": 60835965, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.19836426, + "step": 2828, + "time_per_iteration": 2.7922933101654053 + }, + { + "auxiliary_loss_clip": 0.01161322, + "auxiliary_loss_mlp": 0.01051528, + "balance_loss_clip": 1.05083609, + "balance_loss_mlp": 1.0299387, + "epoch": 0.34016713761798834, + "flos": 16872988296960.0, + "grad_norm": 5.279849498768275, + "language_loss": 0.7766726, + "learning_rate": 3.0730475644063063e-06, + "loss": 0.79880106, + "num_input_tokens_seen": 60851065, + "router_z_loss_clip": 1.10498047, + "router_z_loss_mlp": 0.21594238, + "step": 2829, + "time_per_iteration": 2.774953603744507 + }, + { + "auxiliary_loss_clip": 0.01155539, + "auxiliary_loss_mlp": 0.01047017, + "balance_loss_clip": 1.04956055, + "balance_loss_mlp": 1.02669215, + "epoch": 0.34028738050862745, + "flos": 21907161273600.0, + "grad_norm": 1.821252610035794, + "language_loss": 0.65112221, + "learning_rate": 3.072390120388151e-06, + "loss": 0.6731478, + "num_input_tokens_seen": 60869390, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.203125, + "step": 2830, + "time_per_iteration": 2.785775899887085 + }, + { + "auxiliary_loss_clip": 0.01162939, + "auxiliary_loss_mlp": 0.01059532, + "balance_loss_clip": 1.05265236, + "balance_loss_mlp": 1.03766894, + "epoch": 0.3404076233992665, + "flos": 22746034477440.0, + "grad_norm": 2.621825772356102, + "language_loss": 0.71500432, + "learning_rate": 3.071732513689267e-06, + "loss": 0.73722911, + "num_input_tokens_seen": 60887925, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.21838379, + "step": 2831, + "time_per_iteration": 2.880887985229492 + }, + { + "auxiliary_loss_clip": 0.01164408, + "auxiliary_loss_mlp": 0.01047216, + "balance_loss_clip": 1.05255914, + "balance_loss_mlp": 1.02662873, + "epoch": 0.3405278662899056, + "flos": 17052361839360.0, + "grad_norm": 3.0113162374836295, + "language_loss": 0.67688167, + "learning_rate": 3.0710747444094134e-06, + "loss": 0.69899797, + "num_input_tokens_seen": 60905955, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.20581055, + "step": 2832, + "time_per_iteration": 2.7722792625427246 + }, + { + "auxiliary_loss_clip": 0.0116578, + "auxiliary_loss_mlp": 0.0104989, + "balance_loss_clip": 1.05431724, + "balance_loss_mlp": 1.02972007, + "epoch": 0.3406481091805447, + "flos": 42813131783040.0, + "grad_norm": 8.044310388125668, + "language_loss": 0.64995044, + "learning_rate": 3.070416812648372e-06, + "loss": 0.67210716, + "num_input_tokens_seen": 60929405, + "router_z_loss_clip": 1.11474609, + "router_z_loss_mlp": 0.20166016, + "step": 2833, + "time_per_iteration": 3.1754469871520996 + }, + { + "auxiliary_loss_clip": 0.01159682, + "auxiliary_loss_mlp": 0.01050622, + "balance_loss_clip": 1.05271435, + "balance_loss_mlp": 1.0318706, + "epoch": 0.3407683520711838, + "flos": 26761457917440.0, + "grad_norm": 2.4087360388706864, + "language_loss": 0.65274835, + "learning_rate": 3.069758718505951e-06, + "loss": 0.67485136, + "num_input_tokens_seen": 60951145, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.18762207, + "step": 2834, + "time_per_iteration": 2.8408544063568115 + }, + { + "auxiliary_loss_clip": 0.01160659, + "auxiliary_loss_mlp": 0.01055771, + "balance_loss_clip": 1.05335498, + "balance_loss_mlp": 1.035851, + "epoch": 0.3408885949618229, + "flos": 28767643309440.0, + "grad_norm": 2.834492054837866, + "language_loss": 0.80352437, + "learning_rate": 3.0691004620819836e-06, + "loss": 0.8256886, + "num_input_tokens_seen": 60971275, + "router_z_loss_clip": 1.07226562, + "router_z_loss_mlp": 0.19921875, + "step": 2835, + "time_per_iteration": 2.9552297592163086 + }, + { + "auxiliary_loss_clip": 0.01072263, + "auxiliary_loss_mlp": 0.01013179, + "balance_loss_clip": 1.03389561, + "balance_loss_mlp": 1.01071191, + "epoch": 0.341008837852462, + "flos": 63576252881280.0, + "grad_norm": 0.8140140409825254, + "language_loss": 0.60248351, + "learning_rate": 3.0684420434763254e-06, + "loss": 0.62333792, + "num_input_tokens_seen": 61037460, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.0246582, + "step": 2836, + "time_per_iteration": 3.4129860401153564 + }, + { + "auxiliary_loss_clip": 0.01159633, + "auxiliary_loss_mlp": 0.01049568, + "balance_loss_clip": 1.05317259, + "balance_loss_mlp": 1.03099501, + "epoch": 0.34112908074310105, + "flos": 20812173120000.0, + "grad_norm": 2.1266470985245087, + "language_loss": 0.76860738, + "learning_rate": 3.06778346278886e-06, + "loss": 0.79069936, + "num_input_tokens_seen": 61056295, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.18591309, + "step": 2837, + "time_per_iteration": 2.8654839992523193 + }, + { + "auxiliary_loss_clip": 0.0116528, + "auxiliary_loss_mlp": 0.01049669, + "balance_loss_clip": 1.05546653, + "balance_loss_mlp": 1.02837813, + "epoch": 0.34124932363374016, + "flos": 24976446520320.0, + "grad_norm": 1.7183141926151433, + "language_loss": 0.79096437, + "learning_rate": 3.0671247201194906e-06, + "loss": 0.81311387, + "num_input_tokens_seen": 61078430, + "router_z_loss_clip": 1.09863281, + "router_z_loss_mlp": 0.21313477, + "step": 2838, + "time_per_iteration": 2.8675382137298584 + }, + { + "auxiliary_loss_clip": 0.01164522, + "auxiliary_loss_mlp": 0.01059056, + "balance_loss_clip": 1.05453014, + "balance_loss_mlp": 1.03818274, + "epoch": 0.3413695665243792, + "flos": 28402970480640.0, + "grad_norm": 3.9009512850308696, + "language_loss": 0.75467396, + "learning_rate": 3.066465815568151e-06, + "loss": 0.77690971, + "num_input_tokens_seen": 61099260, + "router_z_loss_clip": 1.09863281, + "router_z_loss_mlp": 0.2088623, + "step": 2839, + "time_per_iteration": 2.869605302810669 + }, + { + "auxiliary_loss_clip": 0.011635, + "auxiliary_loss_mlp": 0.01049883, + "balance_loss_clip": 1.05524719, + "balance_loss_mlp": 1.03011799, + "epoch": 0.34148980941501833, + "flos": 25302012416640.0, + "grad_norm": 2.370256896998356, + "language_loss": 0.69454986, + "learning_rate": 3.0658067492347947e-06, + "loss": 0.71668363, + "num_input_tokens_seen": 61121900, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.19763184, + "step": 2840, + "time_per_iteration": 2.909069299697876 + }, + { + "auxiliary_loss_clip": 0.01164967, + "auxiliary_loss_mlp": 0.01053773, + "balance_loss_clip": 1.05751109, + "balance_loss_mlp": 1.03360319, + "epoch": 0.34161005230565744, + "flos": 17530081747200.0, + "grad_norm": 1.9658781484557686, + "language_loss": 0.67576659, + "learning_rate": 3.065147521219402e-06, + "loss": 0.69795394, + "num_input_tokens_seen": 61141155, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.20178223, + "step": 2841, + "time_per_iteration": 2.9081099033355713 + }, + { + "auxiliary_loss_clip": 0.01157389, + "auxiliary_loss_mlp": 0.01045963, + "balance_loss_clip": 1.05352485, + "balance_loss_mlp": 1.02725935, + "epoch": 0.3417302951962965, + "flos": 43650101566080.0, + "grad_norm": 3.706312311823876, + "language_loss": 0.74570978, + "learning_rate": 3.064488131621977e-06, + "loss": 0.76774335, + "num_input_tokens_seen": 61164480, + "router_z_loss_clip": 1.03808594, + "router_z_loss_mlp": 0.18701172, + "step": 2842, + "time_per_iteration": 2.991408109664917 + }, + { + "auxiliary_loss_clip": 0.01157586, + "auxiliary_loss_mlp": 0.01044367, + "balance_loss_clip": 1.05152106, + "balance_loss_mlp": 1.02414894, + "epoch": 0.3418505380869356, + "flos": 30882207012480.0, + "grad_norm": 1.8764833510597703, + "language_loss": 0.73944122, + "learning_rate": 3.063828580542549e-06, + "loss": 0.76146066, + "num_input_tokens_seen": 61185675, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.20227051, + "step": 2843, + "time_per_iteration": 2.925513744354248 + }, + { + "auxiliary_loss_clip": 0.01161311, + "auxiliary_loss_mlp": 0.01046947, + "balance_loss_clip": 1.05711257, + "balance_loss_mlp": 1.02666998, + "epoch": 0.3419707809775747, + "flos": 19463871277440.0, + "grad_norm": 1.7807022709131841, + "language_loss": 0.73784947, + "learning_rate": 3.0631688680811706e-06, + "loss": 0.75993204, + "num_input_tokens_seen": 61205300, + "router_z_loss_clip": 1.04150391, + "router_z_loss_mlp": 0.20275879, + "step": 2844, + "time_per_iteration": 2.8516757488250732 + }, + { + "auxiliary_loss_clip": 0.01165567, + "auxiliary_loss_mlp": 0.01047581, + "balance_loss_clip": 1.05456471, + "balance_loss_mlp": 1.02822196, + "epoch": 0.3420910238682138, + "flos": 28727818104960.0, + "grad_norm": 2.0652762130374644, + "language_loss": 0.75618696, + "learning_rate": 3.062508994337921e-06, + "loss": 0.77831841, + "num_input_tokens_seen": 61224905, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.19348145, + "step": 2845, + "time_per_iteration": 3.768738269805908 + }, + { + "auxiliary_loss_clip": 0.01163371, + "auxiliary_loss_mlp": 0.0104708, + "balance_loss_clip": 1.05329835, + "balance_loss_mlp": 1.02620649, + "epoch": 0.3422112667588529, + "flos": 21397265758080.0, + "grad_norm": 2.048797128567509, + "language_loss": 0.79458797, + "learning_rate": 3.0618489594129013e-06, + "loss": 0.81669247, + "num_input_tokens_seen": 61243045, + "router_z_loss_clip": 1.10058594, + "router_z_loss_mlp": 0.2088623, + "step": 2846, + "time_per_iteration": 2.8772363662719727 + }, + { + "auxiliary_loss_clip": 0.01163133, + "auxiliary_loss_mlp": 0.01048136, + "balance_loss_clip": 1.05528474, + "balance_loss_mlp": 1.02776337, + "epoch": 0.342331509649492, + "flos": 13881450038400.0, + "grad_norm": 2.5902609609280405, + "language_loss": 0.7148298, + "learning_rate": 3.061188763406239e-06, + "loss": 0.73694253, + "num_input_tokens_seen": 61259190, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.20385742, + "step": 2847, + "time_per_iteration": 3.7667829990386963 + }, + { + "auxiliary_loss_clip": 0.01165793, + "auxiliary_loss_mlp": 0.01050462, + "balance_loss_clip": 1.0568912, + "balance_loss_mlp": 1.02993369, + "epoch": 0.34245175254013105, + "flos": 28621450955520.0, + "grad_norm": 3.294332490039168, + "language_loss": 0.82600367, + "learning_rate": 3.060528406418085e-06, + "loss": 0.84816617, + "num_input_tokens_seen": 61279040, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.20507812, + "step": 2848, + "time_per_iteration": 2.876596212387085 + }, + { + "auxiliary_loss_clip": 0.01164189, + "auxiliary_loss_mlp": 0.01042686, + "balance_loss_clip": 1.05602419, + "balance_loss_mlp": 1.02314782, + "epoch": 0.34257199543077016, + "flos": 34127058960000.0, + "grad_norm": 1.6527108186716044, + "language_loss": 0.62018293, + "learning_rate": 3.0598678885486145e-06, + "loss": 0.64225173, + "num_input_tokens_seen": 61301580, + "router_z_loss_clip": 1.08251953, + "router_z_loss_mlp": 0.19555664, + "step": 2849, + "time_per_iteration": 3.7152395248413086 + }, + { + "auxiliary_loss_clip": 0.01162034, + "auxiliary_loss_mlp": 0.01047468, + "balance_loss_clip": 1.0533905, + "balance_loss_mlp": 1.02826381, + "epoch": 0.34269223832140927, + "flos": 19974018188160.0, + "grad_norm": 1.756707021893195, + "language_loss": 0.74423933, + "learning_rate": 3.0592072098980282e-06, + "loss": 0.76633435, + "num_input_tokens_seen": 61321240, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.1920166, + "step": 2850, + "time_per_iteration": 3.6493945121765137 + }, + { + "auxiliary_loss_clip": 0.01167236, + "auxiliary_loss_mlp": 0.01055548, + "balance_loss_clip": 1.0577842, + "balance_loss_mlp": 1.03555679, + "epoch": 0.3428124812120483, + "flos": 27235658292480.0, + "grad_norm": 2.3798754711856893, + "language_loss": 0.73256409, + "learning_rate": 3.0585463705665514e-06, + "loss": 0.75479186, + "num_input_tokens_seen": 61341615, + "router_z_loss_clip": 1.09423828, + "router_z_loss_mlp": 0.19970703, + "step": 2851, + "time_per_iteration": 2.8902480602264404 + }, + { + "auxiliary_loss_clip": 0.011626, + "auxiliary_loss_mlp": 0.01054733, + "balance_loss_clip": 1.05434275, + "balance_loss_mlp": 1.03455055, + "epoch": 0.34293272410268744, + "flos": 24570871079040.0, + "grad_norm": 2.15565909734258, + "language_loss": 0.70521671, + "learning_rate": 3.0578853706544304e-06, + "loss": 0.72738999, + "num_input_tokens_seen": 61359005, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.2019043, + "step": 2852, + "time_per_iteration": 2.793511390686035 + }, + { + "auxiliary_loss_clip": 0.01164989, + "auxiliary_loss_mlp": 0.01056131, + "balance_loss_clip": 1.05620718, + "balance_loss_mlp": 1.03561485, + "epoch": 0.34305296699332655, + "flos": 21506865131520.0, + "grad_norm": 2.360246316225549, + "language_loss": 0.65585661, + "learning_rate": 3.0572242102619404e-06, + "loss": 0.67806786, + "num_input_tokens_seen": 61376160, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.20507812, + "step": 2853, + "time_per_iteration": 2.853296995162964 + }, + { + "auxiliary_loss_clip": 0.01159857, + "auxiliary_loss_mlp": 0.01053977, + "balance_loss_clip": 1.0541985, + "balance_loss_mlp": 1.03461719, + "epoch": 0.3431732098839656, + "flos": 24056665931520.0, + "grad_norm": 2.8301281766800646, + "language_loss": 0.80505997, + "learning_rate": 3.0565628894893784e-06, + "loss": 0.82719827, + "num_input_tokens_seen": 61396795, + "router_z_loss_clip": 1.05810547, + "router_z_loss_mlp": 0.19372559, + "step": 2854, + "time_per_iteration": 2.873253583908081 + }, + { + "auxiliary_loss_clip": 0.01166397, + "auxiliary_loss_mlp": 0.01054056, + "balance_loss_clip": 1.0573374, + "balance_loss_mlp": 1.03340912, + "epoch": 0.3432934527746047, + "flos": 16800879744000.0, + "grad_norm": 1.8777206972876386, + "language_loss": 0.74775493, + "learning_rate": 3.0559014084370655e-06, + "loss": 0.76995945, + "num_input_tokens_seen": 61415320, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.20654297, + "step": 2855, + "time_per_iteration": 2.83597731590271 + }, + { + "auxiliary_loss_clip": 0.01166614, + "auxiliary_loss_mlp": 0.01058651, + "balance_loss_clip": 1.05384183, + "balance_loss_mlp": 1.03731227, + "epoch": 0.34341369566524377, + "flos": 23439720908160.0, + "grad_norm": 1.7008922249047418, + "language_loss": 0.78940839, + "learning_rate": 3.055239767205349e-06, + "loss": 0.81166106, + "num_input_tokens_seen": 61437070, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.21337891, + "step": 2856, + "time_per_iteration": 2.8693387508392334 + }, + { + "auxiliary_loss_clip": 0.01160929, + "auxiliary_loss_mlp": 0.01048136, + "balance_loss_clip": 1.05884361, + "balance_loss_mlp": 1.02868092, + "epoch": 0.3435339385558829, + "flos": 17267466435840.0, + "grad_norm": 1.9310162510813143, + "language_loss": 0.78658575, + "learning_rate": 3.054577965894599e-06, + "loss": 0.80867636, + "num_input_tokens_seen": 61453215, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.19458008, + "step": 2857, + "time_per_iteration": 2.834770679473877 + }, + { + "auxiliary_loss_clip": 0.01166557, + "auxiliary_loss_mlp": 0.01051014, + "balance_loss_clip": 1.05515027, + "balance_loss_mlp": 1.03111827, + "epoch": 0.343654181446522, + "flos": 22199366413440.0, + "grad_norm": 1.9504541219224607, + "language_loss": 0.70200133, + "learning_rate": 3.0539160046052094e-06, + "loss": 0.724177, + "num_input_tokens_seen": 61472915, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.19897461, + "step": 2858, + "time_per_iteration": 2.937624931335449 + }, + { + "auxiliary_loss_clip": 0.01167397, + "auxiliary_loss_mlp": 0.01043055, + "balance_loss_clip": 1.05423033, + "balance_loss_mlp": 1.02088201, + "epoch": 0.34377442433716104, + "flos": 19901801894400.0, + "grad_norm": 5.908223634205888, + "language_loss": 0.70511639, + "learning_rate": 3.0532538834376003e-06, + "loss": 0.72722089, + "num_input_tokens_seen": 61492475, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.22167969, + "step": 2859, + "time_per_iteration": 2.9625377655029297 + }, + { + "auxiliary_loss_clip": 0.01166851, + "auxiliary_loss_mlp": 0.01046093, + "balance_loss_clip": 1.05710793, + "balance_loss_mlp": 1.02564883, + "epoch": 0.34389466722780015, + "flos": 22197678474240.0, + "grad_norm": 1.8145267984464062, + "language_loss": 0.78116935, + "learning_rate": 3.0525916024922143e-06, + "loss": 0.80329883, + "num_input_tokens_seen": 61511660, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.2043457, + "step": 2860, + "time_per_iteration": 2.944481134414673 + }, + { + "auxiliary_loss_clip": 0.01168083, + "auxiliary_loss_mlp": 0.01046332, + "balance_loss_clip": 1.05844522, + "balance_loss_mlp": 1.02694869, + "epoch": 0.34401491011843927, + "flos": 18624567110400.0, + "grad_norm": 2.71210455023386, + "language_loss": 0.8448078, + "learning_rate": 3.0519291618695193e-06, + "loss": 0.86695194, + "num_input_tokens_seen": 61529060, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.19396973, + "step": 2861, + "time_per_iteration": 3.1212053298950195 + }, + { + "auxiliary_loss_clip": 0.01161061, + "auxiliary_loss_mlp": 0.01051426, + "balance_loss_clip": 1.05344081, + "balance_loss_mlp": 1.03199482, + "epoch": 0.3441351530090783, + "flos": 17858197509120.0, + "grad_norm": 1.6124163241091245, + "language_loss": 0.75628787, + "learning_rate": 3.0512665616700065e-06, + "loss": 0.77841276, + "num_input_tokens_seen": 61548125, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.19421387, + "step": 2862, + "time_per_iteration": 2.8634419441223145 + }, + { + "auxiliary_loss_clip": 0.0116095, + "auxiliary_loss_mlp": 0.01053455, + "balance_loss_clip": 1.05355787, + "balance_loss_mlp": 1.03397596, + "epoch": 0.34425539589971743, + "flos": 23112754381440.0, + "grad_norm": 2.272425706808826, + "language_loss": 0.89011145, + "learning_rate": 3.0506038019941933e-06, + "loss": 0.91225553, + "num_input_tokens_seen": 61568135, + "router_z_loss_clip": 1.07373047, + "router_z_loss_mlp": 0.19482422, + "step": 2863, + "time_per_iteration": 2.9017977714538574 + }, + { + "auxiliary_loss_clip": 0.01165352, + "auxiliary_loss_mlp": 0.01055885, + "balance_loss_clip": 1.05481136, + "balance_loss_mlp": 1.03423643, + "epoch": 0.34437563879035654, + "flos": 21907699977600.0, + "grad_norm": 2.2493969396098796, + "language_loss": 0.67391777, + "learning_rate": 3.049940882942617e-06, + "loss": 0.6961301, + "num_input_tokens_seen": 61586920, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.21630859, + "step": 2864, + "time_per_iteration": 2.8680996894836426 + }, + { + "auxiliary_loss_clip": 0.0116552, + "auxiliary_loss_mlp": 0.01055354, + "balance_loss_clip": 1.0547111, + "balance_loss_mlp": 1.03578031, + "epoch": 0.3444958816809956, + "flos": 23076915586560.0, + "grad_norm": 2.2849819368842845, + "language_loss": 0.8078351, + "learning_rate": 3.0492778046158448e-06, + "loss": 0.83004391, + "num_input_tokens_seen": 61608340, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.19580078, + "step": 2865, + "time_per_iteration": 2.8686697483062744 + }, + { + "auxiliary_loss_clip": 0.01162295, + "auxiliary_loss_mlp": 0.01048776, + "balance_loss_clip": 1.05813289, + "balance_loss_mlp": 1.02944016, + "epoch": 0.3446161245716347, + "flos": 21908633731200.0, + "grad_norm": 2.200690175134382, + "language_loss": 0.7735126, + "learning_rate": 3.0486145671144633e-06, + "loss": 0.7956233, + "num_input_tokens_seen": 61628130, + "router_z_loss_clip": 1.04150391, + "router_z_loss_mlp": 0.1932373, + "step": 2866, + "time_per_iteration": 2.8763697147369385 + }, + { + "auxiliary_loss_clip": 0.01167638, + "auxiliary_loss_mlp": 0.01055506, + "balance_loss_clip": 1.05702007, + "balance_loss_mlp": 1.03529978, + "epoch": 0.3447363674622738, + "flos": 25112834461440.0, + "grad_norm": 2.6082296622266274, + "language_loss": 0.76692766, + "learning_rate": 3.047951170539086e-06, + "loss": 0.78915906, + "num_input_tokens_seen": 61647755, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.20202637, + "step": 2867, + "time_per_iteration": 2.8261220455169678 + }, + { + "auxiliary_loss_clip": 0.01154813, + "auxiliary_loss_mlp": 0.01052109, + "balance_loss_clip": 1.05320334, + "balance_loss_mlp": 1.03488302, + "epoch": 0.3448566103529129, + "flos": 11984684451840.0, + "grad_norm": 1.8693720872870705, + "language_loss": 0.84256577, + "learning_rate": 3.047287614990349e-06, + "loss": 0.86463499, + "num_input_tokens_seen": 61665675, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.17236328, + "step": 2868, + "time_per_iteration": 2.7757561206817627 + }, + { + "auxiliary_loss_clip": 0.01167958, + "auxiliary_loss_mlp": 0.01063227, + "balance_loss_clip": 1.05445802, + "balance_loss_mlp": 1.04052901, + "epoch": 0.344976853243552, + "flos": 40187882465280.0, + "grad_norm": 3.126503780535251, + "language_loss": 0.61666059, + "learning_rate": 3.046623900568914e-06, + "loss": 0.6389724, + "num_input_tokens_seen": 61688240, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.22692871, + "step": 2869, + "time_per_iteration": 2.9291436672210693 + }, + { + "auxiliary_loss_clip": 0.01168425, + "auxiliary_loss_mlp": 0.01052326, + "balance_loss_clip": 1.0568943, + "balance_loss_mlp": 1.03151226, + "epoch": 0.34509709613419104, + "flos": 28723652127360.0, + "grad_norm": 3.162982249926584, + "language_loss": 0.69920778, + "learning_rate": 3.045960027375465e-06, + "loss": 0.72141528, + "num_input_tokens_seen": 61706075, + "router_z_loss_clip": 1.11523438, + "router_z_loss_mlp": 0.20812988, + "step": 2870, + "time_per_iteration": 2.848400115966797 + }, + { + "auxiliary_loss_clip": 0.01166525, + "auxiliary_loss_mlp": 0.01050862, + "balance_loss_clip": 1.05541921, + "balance_loss_mlp": 1.02842736, + "epoch": 0.34521733902483015, + "flos": 29967597982080.0, + "grad_norm": 2.438444099681594, + "language_loss": 0.82551408, + "learning_rate": 3.045295995510711e-06, + "loss": 0.84768796, + "num_input_tokens_seen": 61723045, + "router_z_loss_clip": 1.11132812, + "router_z_loss_mlp": 0.22436523, + "step": 2871, + "time_per_iteration": 2.919675827026367 + }, + { + "auxiliary_loss_clip": 0.01159543, + "auxiliary_loss_mlp": 0.01045656, + "balance_loss_clip": 1.05528212, + "balance_loss_mlp": 1.02741683, + "epoch": 0.34533758191546926, + "flos": 27923059843200.0, + "grad_norm": 1.9810760120760975, + "language_loss": 0.73756993, + "learning_rate": 3.0446318050753865e-06, + "loss": 0.75962192, + "num_input_tokens_seen": 61743525, + "router_z_loss_clip": 1.04150391, + "router_z_loss_mlp": 0.18237305, + "step": 2872, + "time_per_iteration": 2.8486714363098145 + }, + { + "auxiliary_loss_clip": 0.01165196, + "auxiliary_loss_mlp": 0.01054049, + "balance_loss_clip": 1.0580616, + "balance_loss_mlp": 1.03474903, + "epoch": 0.3454578248061083, + "flos": 27125879351040.0, + "grad_norm": 1.8894065472368187, + "language_loss": 0.7814073, + "learning_rate": 3.0439674561702474e-06, + "loss": 0.80359972, + "num_input_tokens_seen": 61763025, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.19287109, + "step": 2873, + "time_per_iteration": 2.823256731033325 + }, + { + "auxiliary_loss_clip": 0.01162378, + "auxiliary_loss_mlp": 0.01049319, + "balance_loss_clip": 1.05745077, + "balance_loss_mlp": 1.02961349, + "epoch": 0.3455780676967474, + "flos": 19024899166080.0, + "grad_norm": 2.201030485113851, + "language_loss": 0.88423342, + "learning_rate": 3.043302948896076e-06, + "loss": 0.90635043, + "num_input_tokens_seen": 61781630, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.19702148, + "step": 2874, + "time_per_iteration": 3.72731876373291 + }, + { + "auxiliary_loss_clip": 0.01168399, + "auxiliary_loss_mlp": 0.01047954, + "balance_loss_clip": 1.05889153, + "balance_loss_mlp": 1.02777171, + "epoch": 0.34569831058738654, + "flos": 34496005507200.0, + "grad_norm": 4.368267538541099, + "language_loss": 0.60522789, + "learning_rate": 3.0426382833536756e-06, + "loss": 0.62739146, + "num_input_tokens_seen": 61804985, + "router_z_loss_clip": 1.09716797, + "router_z_loss_mlp": 0.2019043, + "step": 2875, + "time_per_iteration": 2.986825704574585 + }, + { + "auxiliary_loss_clip": 0.01161747, + "auxiliary_loss_mlp": 0.0104444, + "balance_loss_clip": 1.05431855, + "balance_loss_mlp": 1.02541447, + "epoch": 0.3458185534780256, + "flos": 31138681098240.0, + "grad_norm": 2.3465723649093926, + "language_loss": 0.78161371, + "learning_rate": 3.041973459643877e-06, + "loss": 0.80367553, + "num_input_tokens_seen": 61824440, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.19018555, + "step": 2876, + "time_per_iteration": 3.856266736984253 + }, + { + "auxiliary_loss_clip": 0.01164927, + "auxiliary_loss_mlp": 0.01052698, + "balance_loss_clip": 1.05754781, + "balance_loss_mlp": 1.03293264, + "epoch": 0.3459387963686647, + "flos": 32452508862720.0, + "grad_norm": 2.0801098805253924, + "language_loss": 0.66648406, + "learning_rate": 3.0413084778675334e-06, + "loss": 0.68866032, + "num_input_tokens_seen": 61845690, + "router_z_loss_clip": 1.07373047, + "router_z_loss_mlp": 0.19775391, + "step": 2877, + "time_per_iteration": 2.9556150436401367 + }, + { + "auxiliary_loss_clip": 0.0116126, + "auxiliary_loss_mlp": 0.0104763, + "balance_loss_clip": 1.05577016, + "balance_loss_mlp": 1.02840137, + "epoch": 0.3460590392593038, + "flos": 24675658030080.0, + "grad_norm": 1.9152938354861133, + "language_loss": 0.84116006, + "learning_rate": 3.0406433381255214e-06, + "loss": 0.863249, + "num_input_tokens_seen": 61863725, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.19238281, + "step": 2878, + "time_per_iteration": 3.6560330390930176 + }, + { + "auxiliary_loss_clip": 0.01168098, + "auxiliary_loss_mlp": 0.0104315, + "balance_loss_clip": 1.06092215, + "balance_loss_mlp": 1.02412379, + "epoch": 0.34617928214994287, + "flos": 18807316531200.0, + "grad_norm": 3.170934417764683, + "language_loss": 0.82654178, + "learning_rate": 3.0399780405187425e-06, + "loss": 0.84865427, + "num_input_tokens_seen": 61882720, + "router_z_loss_clip": 1.06982422, + "router_z_loss_mlp": 0.19018555, + "step": 2879, + "time_per_iteration": 3.7323429584503174 + }, + { + "auxiliary_loss_clip": 0.01160458, + "auxiliary_loss_mlp": 0.01051465, + "balance_loss_clip": 1.0568397, + "balance_loss_mlp": 1.03265345, + "epoch": 0.346299525040582, + "flos": 24857653265280.0, + "grad_norm": 1.9433408173700861, + "language_loss": 0.78582609, + "learning_rate": 3.0393125851481216e-06, + "loss": 0.80794537, + "num_input_tokens_seen": 61902595, + "router_z_loss_clip": 1.03613281, + "router_z_loss_mlp": 0.18798828, + "step": 2880, + "time_per_iteration": 2.8388898372650146 + }, + { + "auxiliary_loss_clip": 0.01160479, + "auxiliary_loss_mlp": 0.01039939, + "balance_loss_clip": 1.0552516, + "balance_loss_mlp": 1.01942277, + "epoch": 0.3464197679312211, + "flos": 16434914025600.0, + "grad_norm": 2.595099218583388, + "language_loss": 0.86827397, + "learning_rate": 3.038646972114608e-06, + "loss": 0.89027816, + "num_input_tokens_seen": 61918920, + "router_z_loss_clip": 1.04980469, + "router_z_loss_mlp": 0.2052002, + "step": 2881, + "time_per_iteration": 2.924863815307617 + }, + { + "auxiliary_loss_clip": 0.01161414, + "auxiliary_loss_mlp": 0.01044001, + "balance_loss_clip": 1.05552542, + "balance_loss_mlp": 1.02414131, + "epoch": 0.34654001082186014, + "flos": 22382474970240.0, + "grad_norm": 1.7489848193894806, + "language_loss": 0.67323148, + "learning_rate": 3.037981201519174e-06, + "loss": 0.69528562, + "num_input_tokens_seen": 61939520, + "router_z_loss_clip": 1.05859375, + "router_z_loss_mlp": 0.1986084, + "step": 2882, + "time_per_iteration": 2.9019267559051514 + }, + { + "auxiliary_loss_clip": 0.01161851, + "auxiliary_loss_mlp": 0.01047221, + "balance_loss_clip": 1.05757725, + "balance_loss_mlp": 1.02811158, + "epoch": 0.34666025371249926, + "flos": 19573901614080.0, + "grad_norm": 3.884159527832834, + "language_loss": 0.71636975, + "learning_rate": 3.0373152734628175e-06, + "loss": 0.73846042, + "num_input_tokens_seen": 61957800, + "router_z_loss_clip": 1.04394531, + "router_z_loss_mlp": 0.19104004, + "step": 2883, + "time_per_iteration": 2.83429217338562 + }, + { + "auxiliary_loss_clip": 0.01162165, + "auxiliary_loss_mlp": 0.01054182, + "balance_loss_clip": 1.05337906, + "balance_loss_mlp": 1.03358293, + "epoch": 0.34678049660313837, + "flos": 15267637751040.0, + "grad_norm": 2.5190332551734844, + "language_loss": 0.76169324, + "learning_rate": 3.0366491880465584e-06, + "loss": 0.78385675, + "num_input_tokens_seen": 61975820, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.20581055, + "step": 2884, + "time_per_iteration": 2.7956299781799316 + }, + { + "auxiliary_loss_clip": 0.01164831, + "auxiliary_loss_mlp": 0.01056997, + "balance_loss_clip": 1.05557752, + "balance_loss_mlp": 1.03702927, + "epoch": 0.3469007394937774, + "flos": 21181550630400.0, + "grad_norm": 2.0341577216160482, + "language_loss": 0.81976819, + "learning_rate": 3.035982945371443e-06, + "loss": 0.84198654, + "num_input_tokens_seen": 61997515, + "router_z_loss_clip": 1.09179688, + "router_z_loss_mlp": 0.19970703, + "step": 2885, + "time_per_iteration": 2.8457298278808594 + }, + { + "auxiliary_loss_clip": 0.01157673, + "auxiliary_loss_mlp": 0.01048787, + "balance_loss_clip": 1.05273604, + "balance_loss_mlp": 1.03042877, + "epoch": 0.34702098238441653, + "flos": 22375471818240.0, + "grad_norm": 2.3537652915366323, + "language_loss": 0.85402453, + "learning_rate": 3.035316545538537e-06, + "loss": 0.87608922, + "num_input_tokens_seen": 62016310, + "router_z_loss_clip": 1.04882812, + "router_z_loss_mlp": 0.18371582, + "step": 2886, + "time_per_iteration": 2.8030848503112793 + }, + { + "auxiliary_loss_clip": 0.01164869, + "auxiliary_loss_mlp": 0.01045287, + "balance_loss_clip": 1.05819726, + "balance_loss_mlp": 1.02702439, + "epoch": 0.3471412252750556, + "flos": 22929430343040.0, + "grad_norm": 1.964012181890642, + "language_loss": 0.79220819, + "learning_rate": 3.034649988648935e-06, + "loss": 0.81430972, + "num_input_tokens_seen": 62036075, + "router_z_loss_clip": 1.06738281, + "router_z_loss_mlp": 0.18261719, + "step": 2887, + "time_per_iteration": 2.8038127422332764 + }, + { + "auxiliary_loss_clip": 0.0115994, + "auxiliary_loss_mlp": 0.01047254, + "balance_loss_clip": 1.05451512, + "balance_loss_mlp": 1.02708411, + "epoch": 0.3472614681656947, + "flos": 21324259365120.0, + "grad_norm": 1.7287849535541817, + "language_loss": 0.81043839, + "learning_rate": 3.033983274803752e-06, + "loss": 0.83251035, + "num_input_tokens_seen": 62055865, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.20153809, + "step": 2888, + "time_per_iteration": 2.782012462615967 + }, + { + "auxiliary_loss_clip": 0.01158286, + "auxiliary_loss_mlp": 0.01048012, + "balance_loss_clip": 1.05486858, + "balance_loss_mlp": 1.0296061, + "epoch": 0.3473817110563338, + "flos": 23475739271040.0, + "grad_norm": 2.0811567701890223, + "language_loss": 0.72557241, + "learning_rate": 3.0333164041041283e-06, + "loss": 0.74763542, + "num_input_tokens_seen": 62072180, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.18408203, + "step": 2889, + "time_per_iteration": 2.8474769592285156 + }, + { + "auxiliary_loss_clip": 0.01162995, + "auxiliary_loss_mlp": 0.01054996, + "balance_loss_clip": 1.05808258, + "balance_loss_mlp": 1.03544557, + "epoch": 0.34750195394697286, + "flos": 22346025644160.0, + "grad_norm": 1.998005692077991, + "language_loss": 0.71596634, + "learning_rate": 3.032649376651228e-06, + "loss": 0.73814619, + "num_input_tokens_seen": 62091600, + "router_z_loss_clip": 1.04833984, + "router_z_loss_mlp": 0.19543457, + "step": 2890, + "time_per_iteration": 2.8294503688812256 + }, + { + "auxiliary_loss_clip": 0.0116609, + "auxiliary_loss_mlp": 0.01043695, + "balance_loss_clip": 1.05491614, + "balance_loss_mlp": 1.02425206, + "epoch": 0.347622196837612, + "flos": 29095004885760.0, + "grad_norm": 1.7486246479856116, + "language_loss": 0.76217747, + "learning_rate": 3.031982192546238e-06, + "loss": 0.78427529, + "num_input_tokens_seen": 62114695, + "router_z_loss_clip": 1.109375, + "router_z_loss_mlp": 0.19433594, + "step": 2891, + "time_per_iteration": 2.8835971355438232 + }, + { + "auxiliary_loss_clip": 0.01158812, + "auxiliary_loss_mlp": 0.01047192, + "balance_loss_clip": 1.05248654, + "balance_loss_mlp": 1.02780867, + "epoch": 0.3477424397282511, + "flos": 22455732758400.0, + "grad_norm": 3.089562861574744, + "language_loss": 0.94511521, + "learning_rate": 3.0313148518903696e-06, + "loss": 0.9671753, + "num_input_tokens_seen": 62134520, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.19372559, + "step": 2892, + "time_per_iteration": 2.897604465484619 + }, + { + "auxiliary_loss_clip": 0.01160152, + "auxiliary_loss_mlp": 0.01048254, + "balance_loss_clip": 1.05420625, + "balance_loss_mlp": 1.03016949, + "epoch": 0.34786268261889014, + "flos": 15778790242560.0, + "grad_norm": 1.9939751359342937, + "language_loss": 0.81429029, + "learning_rate": 3.030647354784859e-06, + "loss": 0.83637434, + "num_input_tokens_seen": 62151560, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.1809082, + "step": 2893, + "time_per_iteration": 2.814730644226074 + }, + { + "auxiliary_loss_clip": 0.01154188, + "auxiliary_loss_mlp": 0.0104513, + "balance_loss_clip": 1.04946542, + "balance_loss_mlp": 1.02627087, + "epoch": 0.34798292550952925, + "flos": 20777627214720.0, + "grad_norm": 1.9710930532082858, + "language_loss": 0.7773481, + "learning_rate": 3.029979701330964e-06, + "loss": 0.79934132, + "num_input_tokens_seen": 62170985, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.1887207, + "step": 2894, + "time_per_iteration": 2.9176058769226074 + }, + { + "auxiliary_loss_clip": 0.01158591, + "auxiliary_loss_mlp": 0.01049947, + "balance_loss_clip": 1.0542872, + "balance_loss_mlp": 1.03080225, + "epoch": 0.34810316840016836, + "flos": 19937820257280.0, + "grad_norm": 2.1313386324828407, + "language_loss": 0.80081856, + "learning_rate": 3.029311891629966e-06, + "loss": 0.82290393, + "num_input_tokens_seen": 62189440, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.19152832, + "step": 2895, + "time_per_iteration": 2.965172290802002 + }, + { + "auxiliary_loss_clip": 0.01157232, + "auxiliary_loss_mlp": 0.01046706, + "balance_loss_clip": 1.05298805, + "balance_loss_mlp": 1.02838373, + "epoch": 0.3482234112908074, + "flos": 23623296341760.0, + "grad_norm": 3.441423348826301, + "language_loss": 0.74141264, + "learning_rate": 3.0286439257831744e-06, + "loss": 0.76345205, + "num_input_tokens_seen": 62208910, + "router_z_loss_clip": 1.04248047, + "router_z_loss_mlp": 0.18322754, + "step": 2896, + "time_per_iteration": 2.817810535430908 + }, + { + "auxiliary_loss_clip": 0.01168505, + "auxiliary_loss_mlp": 0.01061554, + "balance_loss_clip": 1.05277765, + "balance_loss_mlp": 1.04014444, + "epoch": 0.3483436541814465, + "flos": 23986712194560.0, + "grad_norm": 2.9315328251379356, + "language_loss": 0.72030842, + "learning_rate": 3.0279758038919156e-06, + "loss": 0.74260896, + "num_input_tokens_seen": 62227135, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.2142334, + "step": 2897, + "time_per_iteration": 2.8605222702026367 + }, + { + "auxiliary_loss_clip": 0.01159522, + "auxiliary_loss_mlp": 0.01065437, + "balance_loss_clip": 1.05288827, + "balance_loss_mlp": 1.04583943, + "epoch": 0.34846389707208564, + "flos": 22638338524800.0, + "grad_norm": 1.9158907784329058, + "language_loss": 0.78750688, + "learning_rate": 3.0273075260575455e-06, + "loss": 0.80975652, + "num_input_tokens_seen": 62246035, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.19604492, + "step": 2898, + "time_per_iteration": 2.78863263130188 + }, + { + "auxiliary_loss_clip": 0.01165272, + "auxiliary_loss_mlp": 0.01050996, + "balance_loss_clip": 1.0535816, + "balance_loss_mlp": 1.03069484, + "epoch": 0.3485841399627247, + "flos": 21792857218560.0, + "grad_norm": 1.9521393419506576, + "language_loss": 0.80684054, + "learning_rate": 3.0266390923814396e-06, + "loss": 0.82900321, + "num_input_tokens_seen": 62264095, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.20300293, + "step": 2899, + "time_per_iteration": 2.7838263511657715 + }, + { + "auxiliary_loss_clip": 0.01166713, + "auxiliary_loss_mlp": 0.01053974, + "balance_loss_clip": 1.05683517, + "balance_loss_mlp": 1.03462601, + "epoch": 0.3487043828533638, + "flos": 17019036996480.0, + "grad_norm": 1.861806507226419, + "language_loss": 0.82100356, + "learning_rate": 3.0259705029650008e-06, + "loss": 0.84321046, + "num_input_tokens_seen": 62282025, + "router_z_loss_clip": 1.09960938, + "router_z_loss_mlp": 0.19348145, + "step": 2900, + "time_per_iteration": 2.8082222938537598 + }, + { + "auxiliary_loss_clip": 0.01159768, + "auxiliary_loss_mlp": 0.01048803, + "balance_loss_clip": 1.05325127, + "balance_loss_mlp": 1.03002799, + "epoch": 0.34882462574400286, + "flos": 22601135013120.0, + "grad_norm": 2.0521298624001716, + "language_loss": 0.7298733, + "learning_rate": 3.025301757909652e-06, + "loss": 0.75195903, + "num_input_tokens_seen": 62302220, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.18762207, + "step": 2901, + "time_per_iteration": 2.7890381813049316 + }, + { + "auxiliary_loss_clip": 0.01162434, + "auxiliary_loss_mlp": 0.01058708, + "balance_loss_clip": 1.05277395, + "balance_loss_mlp": 1.03819191, + "epoch": 0.34894486863464197, + "flos": 29861518141440.0, + "grad_norm": 1.6492035738310624, + "language_loss": 0.80744064, + "learning_rate": 3.024632857316842e-06, + "loss": 0.82965207, + "num_input_tokens_seen": 62323535, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.2052002, + "step": 2902, + "time_per_iteration": 3.7694809436798096 + }, + { + "auxiliary_loss_clip": 0.01162735, + "auxiliary_loss_mlp": 0.01047394, + "balance_loss_clip": 1.05380821, + "balance_loss_mlp": 1.02834415, + "epoch": 0.3490651115252811, + "flos": 22122265870080.0, + "grad_norm": 2.0181038965105227, + "language_loss": 0.77580452, + "learning_rate": 3.0239638012880412e-06, + "loss": 0.7979058, + "num_input_tokens_seen": 62343430, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.19042969, + "step": 2903, + "time_per_iteration": 2.977776288986206 + }, + { + "auxiliary_loss_clip": 0.0116396, + "auxiliary_loss_mlp": 0.01055691, + "balance_loss_clip": 1.05075276, + "balance_loss_mlp": 1.03566384, + "epoch": 0.34918535441592014, + "flos": 12676682943360.0, + "grad_norm": 4.8939107195985025, + "language_loss": 0.81892526, + "learning_rate": 3.0232945899247466e-06, + "loss": 0.84112173, + "num_input_tokens_seen": 62360365, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.20031738, + "step": 2904, + "time_per_iteration": 2.8015787601470947 + }, + { + "auxiliary_loss_clip": 0.01163976, + "auxiliary_loss_mlp": 0.01050707, + "balance_loss_clip": 1.05348539, + "balance_loss_mlp": 1.02955973, + "epoch": 0.34930559730655925, + "flos": 23185617120000.0, + "grad_norm": 2.061176669750402, + "language_loss": 0.77541935, + "learning_rate": 3.022625223328476e-06, + "loss": 0.79756618, + "num_input_tokens_seen": 62382105, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.21142578, + "step": 2905, + "time_per_iteration": 3.6743319034576416 + }, + { + "auxiliary_loss_clip": 0.01163899, + "auxiliary_loss_mlp": 0.01048862, + "balance_loss_clip": 1.05249357, + "balance_loss_mlp": 1.02813113, + "epoch": 0.34942584019719836, + "flos": 22855023319680.0, + "grad_norm": 1.4047277387827575, + "language_loss": 0.6918537, + "learning_rate": 3.0219557016007723e-06, + "loss": 0.71398133, + "num_input_tokens_seen": 62402235, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.20739746, + "step": 2906, + "time_per_iteration": 2.836503267288208 + }, + { + "auxiliary_loss_clip": 0.01155024, + "auxiliary_loss_mlp": 0.01048529, + "balance_loss_clip": 1.04975879, + "balance_loss_mlp": 1.02903807, + "epoch": 0.3495460830878374, + "flos": 24426043441920.0, + "grad_norm": 1.8654686833707679, + "language_loss": 0.70087057, + "learning_rate": 3.021286024843202e-06, + "loss": 0.72290611, + "num_input_tokens_seen": 62420430, + "router_z_loss_clip": 1.05224609, + "router_z_loss_mlp": 0.19494629, + "step": 2907, + "time_per_iteration": 3.6682252883911133 + }, + { + "auxiliary_loss_clip": 0.01064573, + "auxiliary_loss_mlp": 0.0099944, + "balance_loss_clip": 1.02675509, + "balance_loss_mlp": 0.99685305, + "epoch": 0.3496663259784765, + "flos": 70008749389440.0, + "grad_norm": 1.0752077693373607, + "language_loss": 0.64894015, + "learning_rate": 3.0206161931573526e-06, + "loss": 0.66958028, + "num_input_tokens_seen": 62472980, + "router_z_loss_clip": 0.37890625, + "router_z_loss_mlp": 0.02587891, + "step": 2908, + "time_per_iteration": 4.006621599197388 + }, + { + "auxiliary_loss_clip": 0.01154311, + "auxiliary_loss_mlp": 0.01042122, + "balance_loss_clip": 1.05033278, + "balance_loss_mlp": 1.02368021, + "epoch": 0.34978656886911563, + "flos": 28692805322880.0, + "grad_norm": 2.167361030442436, + "language_loss": 0.92902189, + "learning_rate": 3.0199462066448388e-06, + "loss": 0.95098615, + "num_input_tokens_seen": 62495175, + "router_z_loss_clip": 1.04052734, + "router_z_loss_mlp": 0.18432617, + "step": 2909, + "time_per_iteration": 2.8714029788970947 + }, + { + "auxiliary_loss_clip": 0.01163231, + "auxiliary_loss_mlp": 0.01047299, + "balance_loss_clip": 1.0531615, + "balance_loss_mlp": 1.02654457, + "epoch": 0.3499068117597547, + "flos": 21142156389120.0, + "grad_norm": 2.107972391568534, + "language_loss": 0.69401264, + "learning_rate": 3.019276065407296e-06, + "loss": 0.71611798, + "num_input_tokens_seen": 62514295, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.20751953, + "step": 2910, + "time_per_iteration": 2.8254191875457764 + }, + { + "auxiliary_loss_clip": 0.01163396, + "auxiliary_loss_mlp": 0.01049029, + "balance_loss_clip": 1.05303431, + "balance_loss_mlp": 1.02803648, + "epoch": 0.3500270546503938, + "flos": 22782699285120.0, + "grad_norm": 1.922391705906047, + "language_loss": 0.80674314, + "learning_rate": 3.018605769546385e-06, + "loss": 0.82886744, + "num_input_tokens_seen": 62534850, + "router_z_loss_clip": 1.10400391, + "router_z_loss_mlp": 0.20983887, + "step": 2911, + "time_per_iteration": 2.804060697555542 + }, + { + "auxiliary_loss_clip": 0.01163852, + "auxiliary_loss_mlp": 0.01051195, + "balance_loss_clip": 1.05254877, + "balance_loss_mlp": 1.03097677, + "epoch": 0.3501472975410329, + "flos": 22894058424960.0, + "grad_norm": 2.53747478889275, + "language_loss": 0.79692948, + "learning_rate": 3.017935319163788e-06, + "loss": 0.81907994, + "num_input_tokens_seen": 62553810, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.20214844, + "step": 2912, + "time_per_iteration": 2.806488275527954 + }, + { + "auxiliary_loss_clip": 0.01166277, + "auxiliary_loss_mlp": 0.01049416, + "balance_loss_clip": 1.05302811, + "balance_loss_mlp": 1.02625346, + "epoch": 0.35026754043167196, + "flos": 25446588658560.0, + "grad_norm": 1.7698991294340292, + "language_loss": 0.70672488, + "learning_rate": 3.017264714361213e-06, + "loss": 0.72888172, + "num_input_tokens_seen": 62573460, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.23156738, + "step": 2913, + "time_per_iteration": 2.838819980621338 + }, + { + "auxiliary_loss_clip": 0.01162925, + "auxiliary_loss_mlp": 0.01049582, + "balance_loss_clip": 1.05165553, + "balance_loss_mlp": 1.02800536, + "epoch": 0.3503877833223111, + "flos": 19573757959680.0, + "grad_norm": 3.677171235868394, + "language_loss": 0.82665503, + "learning_rate": 3.016593955240389e-06, + "loss": 0.84878016, + "num_input_tokens_seen": 62592150, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.21569824, + "step": 2914, + "time_per_iteration": 2.8078627586364746 + }, + { + "auxiliary_loss_clip": 0.01059126, + "auxiliary_loss_mlp": 0.01003915, + "balance_loss_clip": 1.02160525, + "balance_loss_mlp": 1.00144708, + "epoch": 0.3505080262129502, + "flos": 65072075880960.0, + "grad_norm": 0.8436203251375978, + "language_loss": 0.63753664, + "learning_rate": 3.015923041903071e-06, + "loss": 0.65816712, + "num_input_tokens_seen": 62658275, + "router_z_loss_clip": 0.375, + "router_z_loss_mlp": 0.0246582, + "step": 2915, + "time_per_iteration": 3.3808836936950684 + }, + { + "auxiliary_loss_clip": 0.01161368, + "auxiliary_loss_mlp": 0.01048846, + "balance_loss_clip": 1.05235434, + "balance_loss_mlp": 1.02836633, + "epoch": 0.35062826910358924, + "flos": 29314562768640.0, + "grad_norm": 2.343728504366022, + "language_loss": 0.83910692, + "learning_rate": 3.0152519744510347e-06, + "loss": 0.86120915, + "num_input_tokens_seen": 62678075, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.20471191, + "step": 2916, + "time_per_iteration": 2.8556017875671387 + }, + { + "auxiliary_loss_clip": 0.01159845, + "auxiliary_loss_mlp": 0.01050183, + "balance_loss_clip": 1.04943788, + "balance_loss_mlp": 1.02986979, + "epoch": 0.35074851199422835, + "flos": 23987717775360.0, + "grad_norm": 1.7587371277841493, + "language_loss": 0.83071828, + "learning_rate": 3.014580752986081e-06, + "loss": 0.85281861, + "num_input_tokens_seen": 62696950, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.20324707, + "step": 2917, + "time_per_iteration": 2.877533435821533 + }, + { + "auxiliary_loss_clip": 0.01157731, + "auxiliary_loss_mlp": 0.01046774, + "balance_loss_clip": 1.05167055, + "balance_loss_mlp": 1.02662802, + "epoch": 0.3508687548848674, + "flos": 15224436668160.0, + "grad_norm": 1.960173849038579, + "language_loss": 0.79208452, + "learning_rate": 3.0139093776100345e-06, + "loss": 0.81412959, + "num_input_tokens_seen": 62713540, + "router_z_loss_clip": 1.06152344, + "router_z_loss_mlp": 0.20141602, + "step": 2918, + "time_per_iteration": 2.8905060291290283 + }, + { + "auxiliary_loss_clip": 0.01156752, + "auxiliary_loss_mlp": 0.01049962, + "balance_loss_clip": 1.05046606, + "balance_loss_mlp": 1.03128242, + "epoch": 0.3509889977755065, + "flos": 21361750185600.0, + "grad_norm": 1.710118050243266, + "language_loss": 0.76043379, + "learning_rate": 3.013237848424741e-06, + "loss": 0.78250092, + "num_input_tokens_seen": 62732925, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.18688965, + "step": 2919, + "time_per_iteration": 2.8464324474334717 + }, + { + "auxiliary_loss_clip": 0.0116167, + "auxiliary_loss_mlp": 0.01040352, + "balance_loss_clip": 1.0532198, + "balance_loss_mlp": 1.02101624, + "epoch": 0.35110924066614563, + "flos": 19135360465920.0, + "grad_norm": 1.9276274605093524, + "language_loss": 0.75495923, + "learning_rate": 3.012566165532072e-06, + "loss": 0.77697945, + "num_input_tokens_seen": 62751715, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.19348145, + "step": 2920, + "time_per_iteration": 2.804335117340088 + }, + { + "auxiliary_loss_clip": 0.01161679, + "auxiliary_loss_mlp": 0.01050947, + "balance_loss_clip": 1.0520947, + "balance_loss_mlp": 1.03064549, + "epoch": 0.3512294835567847, + "flos": 21980885938560.0, + "grad_norm": 3.3672489002258974, + "language_loss": 0.76813298, + "learning_rate": 3.0118943290339207e-06, + "loss": 0.79025924, + "num_input_tokens_seen": 62771925, + "router_z_loss_clip": 1.09716797, + "router_z_loss_mlp": 0.20300293, + "step": 2921, + "time_per_iteration": 2.8280789852142334 + }, + { + "auxiliary_loss_clip": 0.01162975, + "auxiliary_loss_mlp": 0.01051764, + "balance_loss_clip": 1.05103433, + "balance_loss_mlp": 1.03053236, + "epoch": 0.3513497264474238, + "flos": 17817294896640.0, + "grad_norm": 1.872375126038177, + "language_loss": 0.6832149, + "learning_rate": 3.011222339032204e-06, + "loss": 0.70536232, + "num_input_tokens_seen": 62790075, + "router_z_loss_clip": 1.12060547, + "router_z_loss_mlp": 0.21228027, + "step": 2922, + "time_per_iteration": 2.7983975410461426 + }, + { + "auxiliary_loss_clip": 0.0115733, + "auxiliary_loss_mlp": 0.0105782, + "balance_loss_clip": 1.05169916, + "balance_loss_mlp": 1.03747106, + "epoch": 0.3514699693380629, + "flos": 26943417239040.0, + "grad_norm": 2.0401833633242, + "language_loss": 0.69736779, + "learning_rate": 3.0105501956288626e-06, + "loss": 0.71951926, + "num_input_tokens_seen": 62810545, + "router_z_loss_clip": 1.05566406, + "router_z_loss_mlp": 0.20349121, + "step": 2923, + "time_per_iteration": 2.953918695449829 + }, + { + "auxiliary_loss_clip": 0.01163053, + "auxiliary_loss_mlp": 0.0105315, + "balance_loss_clip": 1.05064797, + "balance_loss_mlp": 1.03206193, + "epoch": 0.35159021222870196, + "flos": 15267565923840.0, + "grad_norm": 2.0873664279339326, + "language_loss": 0.72394943, + "learning_rate": 3.0098778989258602e-06, + "loss": 0.74611145, + "num_input_tokens_seen": 62829155, + "router_z_loss_clip": 1.12304688, + "router_z_loss_mlp": 0.2109375, + "step": 2924, + "time_per_iteration": 2.746685028076172 + }, + { + "auxiliary_loss_clip": 0.01162066, + "auxiliary_loss_mlp": 0.01053076, + "balance_loss_clip": 1.05450797, + "balance_loss_mlp": 1.03190446, + "epoch": 0.35171045511934107, + "flos": 13984154000640.0, + "grad_norm": 2.18525136746053, + "language_loss": 0.88418484, + "learning_rate": 3.009205449025183e-06, + "loss": 0.90633631, + "num_input_tokens_seen": 62845350, + "router_z_loss_clip": 1.07617188, + "router_z_loss_mlp": 0.21166992, + "step": 2925, + "time_per_iteration": 2.8291590213775635 + }, + { + "auxiliary_loss_clip": 0.01153615, + "auxiliary_loss_mlp": 0.0104537, + "balance_loss_clip": 1.04902554, + "balance_loss_mlp": 1.0261538, + "epoch": 0.3518306980099802, + "flos": 14283434119680.0, + "grad_norm": 3.7426271219170593, + "language_loss": 0.63582289, + "learning_rate": 3.008532846028842e-06, + "loss": 0.65781271, + "num_input_tokens_seen": 62862110, + "router_z_loss_clip": 1.04638672, + "router_z_loss_mlp": 0.19226074, + "step": 2926, + "time_per_iteration": 2.81304669380188 + }, + { + "auxiliary_loss_clip": 0.01165789, + "auxiliary_loss_mlp": 0.01047083, + "balance_loss_clip": 1.05443811, + "balance_loss_mlp": 1.02582788, + "epoch": 0.35195094090061924, + "flos": 27052872958080.0, + "grad_norm": 2.4070962307769483, + "language_loss": 0.71894616, + "learning_rate": 3.0078600900388694e-06, + "loss": 0.74107486, + "num_input_tokens_seen": 62882415, + "router_z_loss_clip": 1.11230469, + "router_z_loss_mlp": 0.21252441, + "step": 2927, + "time_per_iteration": 2.7916383743286133 + }, + { + "auxiliary_loss_clip": 0.01157103, + "auxiliary_loss_mlp": 0.01046257, + "balance_loss_clip": 1.04901862, + "balance_loss_mlp": 1.02711248, + "epoch": 0.35207118379125835, + "flos": 25629266252160.0, + "grad_norm": 1.813211421685196, + "language_loss": 0.73761892, + "learning_rate": 3.007187181157323e-06, + "loss": 0.7596525, + "num_input_tokens_seen": 62902425, + "router_z_loss_clip": 1.08105469, + "router_z_loss_mlp": 0.19152832, + "step": 2928, + "time_per_iteration": 2.8579046726226807 + }, + { + "auxiliary_loss_clip": 0.01162303, + "auxiliary_loss_mlp": 0.0104863, + "balance_loss_clip": 1.05271912, + "balance_loss_mlp": 1.02899599, + "epoch": 0.35219142668189746, + "flos": 18004713085440.0, + "grad_norm": 2.7503328865728482, + "language_loss": 0.68837035, + "learning_rate": 3.006514119486282e-06, + "loss": 0.71047968, + "num_input_tokens_seen": 62919255, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.1965332, + "step": 2929, + "time_per_iteration": 2.768784761428833 + }, + { + "auxiliary_loss_clip": 0.01160949, + "auxiliary_loss_mlp": 0.01045362, + "balance_loss_clip": 1.05095053, + "balance_loss_mlp": 1.02591908, + "epoch": 0.3523116695725365, + "flos": 14028109269120.0, + "grad_norm": 1.7083464293450341, + "language_loss": 0.70146912, + "learning_rate": 3.005840905127849e-06, + "loss": 0.7235322, + "num_input_tokens_seen": 62936160, + "router_z_loss_clip": 1.09765625, + "router_z_loss_mlp": 0.19433594, + "step": 2930, + "time_per_iteration": 2.744628667831421 + }, + { + "auxiliary_loss_clip": 0.01156707, + "auxiliary_loss_mlp": 0.01053517, + "balance_loss_clip": 1.05279815, + "balance_loss_mlp": 1.03395438, + "epoch": 0.3524319124631756, + "flos": 21433966479360.0, + "grad_norm": 2.2695429024125504, + "language_loss": 0.87015069, + "learning_rate": 3.0051675381841516e-06, + "loss": 0.89225298, + "num_input_tokens_seen": 62953470, + "router_z_loss_clip": 1.03857422, + "router_z_loss_mlp": 0.19555664, + "step": 2931, + "time_per_iteration": 3.657498836517334 + }, + { + "auxiliary_loss_clip": 0.01162904, + "auxiliary_loss_mlp": 0.01053458, + "balance_loss_clip": 1.05438709, + "balance_loss_mlp": 1.03354967, + "epoch": 0.3525521553538147, + "flos": 26322773114880.0, + "grad_norm": 1.5617040159422968, + "language_loss": 0.76989812, + "learning_rate": 3.0044940187573363e-06, + "loss": 0.79206175, + "num_input_tokens_seen": 62974480, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.19897461, + "step": 2932, + "time_per_iteration": 2.824876070022583 + }, + { + "auxiliary_loss_clip": 0.01160753, + "auxiliary_loss_mlp": 0.01053361, + "balance_loss_clip": 1.0526377, + "balance_loss_mlp": 1.03419185, + "epoch": 0.3526723982444538, + "flos": 21543314457600.0, + "grad_norm": 2.398114924626184, + "language_loss": 0.64893353, + "learning_rate": 3.003820346949578e-06, + "loss": 0.67107475, + "num_input_tokens_seen": 62992560, + "router_z_loss_clip": 1.08056641, + "router_z_loss_mlp": 0.19165039, + "step": 2933, + "time_per_iteration": 2.7577321529388428 + }, + { + "auxiliary_loss_clip": 0.01160028, + "auxiliary_loss_mlp": 0.01052785, + "balance_loss_clip": 1.04987526, + "balance_loss_mlp": 1.03242373, + "epoch": 0.3527926411350929, + "flos": 23733649900800.0, + "grad_norm": 2.522031177450135, + "language_loss": 0.79623497, + "learning_rate": 3.003146522863071e-06, + "loss": 0.81836313, + "num_input_tokens_seen": 63013445, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.20361328, + "step": 2934, + "time_per_iteration": 3.669893980026245 + }, + { + "auxiliary_loss_clip": 0.01160179, + "auxiliary_loss_mlp": 0.01053399, + "balance_loss_clip": 1.05167747, + "balance_loss_mlp": 1.03318167, + "epoch": 0.35291288402573195, + "flos": 30445461544320.0, + "grad_norm": 3.1573956368593006, + "language_loss": 0.86556482, + "learning_rate": 3.0024725466000345e-06, + "loss": 0.88770062, + "num_input_tokens_seen": 63033400, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.20214844, + "step": 2935, + "time_per_iteration": 3.011857748031616 + }, + { + "auxiliary_loss_clip": 0.0116116, + "auxiliary_loss_mlp": 0.01050142, + "balance_loss_clip": 1.05486166, + "balance_loss_mlp": 1.0307591, + "epoch": 0.35303312691637107, + "flos": 23112179763840.0, + "grad_norm": 2.9948587244079667, + "language_loss": 0.79018593, + "learning_rate": 3.0017984182627087e-06, + "loss": 0.81229889, + "num_input_tokens_seen": 63052725, + "router_z_loss_clip": 1.06396484, + "router_z_loss_mlp": 0.19396973, + "step": 2936, + "time_per_iteration": 3.6420884132385254 + }, + { + "auxiliary_loss_clip": 0.01160972, + "auxiliary_loss_mlp": 0.01049646, + "balance_loss_clip": 1.05046773, + "balance_loss_mlp": 1.0293448, + "epoch": 0.3531533698070102, + "flos": 21835699165440.0, + "grad_norm": 1.875171868673218, + "language_loss": 0.82462871, + "learning_rate": 3.00112413795336e-06, + "loss": 0.84673488, + "num_input_tokens_seen": 63072560, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.20300293, + "step": 2937, + "time_per_iteration": 3.723470687866211 + }, + { + "auxiliary_loss_clip": 0.01157318, + "auxiliary_loss_mlp": 0.01054511, + "balance_loss_clip": 1.0501008, + "balance_loss_mlp": 1.03456736, + "epoch": 0.35327361269764923, + "flos": 15778969810560.0, + "grad_norm": 2.1155960192141436, + "language_loss": 0.79533982, + "learning_rate": 3.000449705774275e-06, + "loss": 0.81745815, + "num_input_tokens_seen": 63090800, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.19958496, + "step": 2938, + "time_per_iteration": 2.817741870880127 + }, + { + "auxiliary_loss_clip": 0.01158741, + "auxiliary_loss_mlp": 0.01051401, + "balance_loss_clip": 1.05165577, + "balance_loss_mlp": 1.03237534, + "epoch": 0.35339385558828834, + "flos": 22090413484800.0, + "grad_norm": 2.1457716823772763, + "language_loss": 0.71941483, + "learning_rate": 2.9997751218277654e-06, + "loss": 0.74151623, + "num_input_tokens_seen": 63108955, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.19018555, + "step": 2939, + "time_per_iteration": 2.780855417251587 + }, + { + "auxiliary_loss_clip": 0.01161171, + "auxiliary_loss_mlp": 0.01056352, + "balance_loss_clip": 1.0528096, + "balance_loss_mlp": 1.03571725, + "epoch": 0.35351409847892745, + "flos": 24165008328960.0, + "grad_norm": 1.8861944528147694, + "language_loss": 0.77846539, + "learning_rate": 2.999100386216166e-06, + "loss": 0.80064064, + "num_input_tokens_seen": 63127895, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.2064209, + "step": 2940, + "time_per_iteration": 2.812732696533203 + }, + { + "auxiliary_loss_clip": 0.01163075, + "auxiliary_loss_mlp": 0.01045372, + "balance_loss_clip": 1.05343509, + "balance_loss_mlp": 1.02610743, + "epoch": 0.3536343413695665, + "flos": 27052298340480.0, + "grad_norm": 1.775500875302751, + "language_loss": 0.74602103, + "learning_rate": 2.998425499041831e-06, + "loss": 0.76810551, + "num_input_tokens_seen": 63148410, + "router_z_loss_clip": 1.09570312, + "router_z_loss_mlp": 0.19262695, + "step": 2941, + "time_per_iteration": 2.853816032409668 + }, + { + "auxiliary_loss_clip": 0.01054409, + "auxiliary_loss_mlp": 0.01024797, + "balance_loss_clip": 1.01746666, + "balance_loss_mlp": 1.02253246, + "epoch": 0.3537545842602056, + "flos": 65991066370560.0, + "grad_norm": 1.2944564481795702, + "language_loss": 0.64583552, + "learning_rate": 2.997750460407142e-06, + "loss": 0.66662753, + "num_input_tokens_seen": 63209765, + "router_z_loss_clip": 0.36914062, + "router_z_loss_mlp": 0.02270508, + "step": 2942, + "time_per_iteration": 3.450477361679077 + }, + { + "auxiliary_loss_clip": 0.01161239, + "auxiliary_loss_mlp": 0.01045471, + "balance_loss_clip": 1.04964852, + "balance_loss_mlp": 1.02371538, + "epoch": 0.35387482715084473, + "flos": 18436897526400.0, + "grad_norm": 2.3457428740670596, + "language_loss": 0.7010783, + "learning_rate": 2.997075270414501e-06, + "loss": 0.72314537, + "num_input_tokens_seen": 63226980, + "router_z_loss_clip": 1.11572266, + "router_z_loss_mlp": 0.21728516, + "step": 2943, + "time_per_iteration": 2.806431531906128 + }, + { + "auxiliary_loss_clip": 0.01053583, + "auxiliary_loss_mlp": 0.01013518, + "balance_loss_clip": 1.01648986, + "balance_loss_mlp": 1.01124144, + "epoch": 0.3539950700414838, + "flos": 65588579498880.0, + "grad_norm": 0.7042186009350584, + "language_loss": 0.57690507, + "learning_rate": 2.9963999291663347e-06, + "loss": 0.59757608, + "num_input_tokens_seen": 63292760, + "router_z_loss_clip": 0.37109375, + "router_z_loss_mlp": 0.02282715, + "step": 2944, + "time_per_iteration": 3.4394726753234863 + }, + { + "auxiliary_loss_clip": 0.0116295, + "auxiliary_loss_mlp": 0.01045621, + "balance_loss_clip": 1.0524013, + "balance_loss_mlp": 1.02597511, + "epoch": 0.3541153129321229, + "flos": 20521655919360.0, + "grad_norm": 2.230133475882705, + "language_loss": 0.7411567, + "learning_rate": 2.9957244367650915e-06, + "loss": 0.76324236, + "num_input_tokens_seen": 63309005, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.19641113, + "step": 2945, + "time_per_iteration": 2.9157323837280273 + }, + { + "auxiliary_loss_clip": 0.011571, + "auxiliary_loss_mlp": 0.0105194, + "balance_loss_clip": 1.05162501, + "balance_loss_mlp": 1.03191304, + "epoch": 0.354235555822762, + "flos": 19573578391680.0, + "grad_norm": 1.7855008350647497, + "language_loss": 0.84311843, + "learning_rate": 2.9950487933132425e-06, + "loss": 0.86520886, + "num_input_tokens_seen": 63326420, + "router_z_loss_clip": 1.05517578, + "router_z_loss_mlp": 0.20031738, + "step": 2946, + "time_per_iteration": 2.8149099349975586 + }, + { + "auxiliary_loss_clip": 0.01162774, + "auxiliary_loss_mlp": 0.01047735, + "balance_loss_clip": 1.05310214, + "balance_loss_mlp": 1.02620578, + "epoch": 0.35435579871340106, + "flos": 20777268078720.0, + "grad_norm": 2.7334781123139056, + "language_loss": 0.71452081, + "learning_rate": 2.994372998913283e-06, + "loss": 0.73662591, + "num_input_tokens_seen": 63344925, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.21533203, + "step": 2947, + "time_per_iteration": 2.7927005290985107 + }, + { + "auxiliary_loss_clip": 0.01159474, + "auxiliary_loss_mlp": 0.01049264, + "balance_loss_clip": 1.05097818, + "balance_loss_mlp": 1.02892733, + "epoch": 0.35447604160404017, + "flos": 23951807153280.0, + "grad_norm": 2.45646305434815, + "language_loss": 0.62543589, + "learning_rate": 2.99369705366773e-06, + "loss": 0.64752322, + "num_input_tokens_seen": 63365170, + "router_z_loss_clip": 1.08544922, + "router_z_loss_mlp": 0.20349121, + "step": 2948, + "time_per_iteration": 2.827056646347046 + }, + { + "auxiliary_loss_clip": 0.01158607, + "auxiliary_loss_mlp": 0.01041849, + "balance_loss_clip": 1.05109739, + "balance_loss_mlp": 1.02244198, + "epoch": 0.3545962844946792, + "flos": 23435662671360.0, + "grad_norm": 2.082778188009158, + "language_loss": 0.82308626, + "learning_rate": 2.9930209576791244e-06, + "loss": 0.84509081, + "num_input_tokens_seen": 63383645, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.1940918, + "step": 2949, + "time_per_iteration": 2.8285670280456543 + }, + { + "auxiliary_loss_clip": 0.01156684, + "auxiliary_loss_mlp": 0.01039985, + "balance_loss_clip": 1.0495677, + "balance_loss_mlp": 1.02140021, + "epoch": 0.35471652738531834, + "flos": 22085134185600.0, + "grad_norm": 2.8007920426099258, + "language_loss": 0.63592505, + "learning_rate": 2.9923447110500285e-06, + "loss": 0.65789175, + "num_input_tokens_seen": 63402390, + "router_z_loss_clip": 1.07226562, + "router_z_loss_mlp": 0.18579102, + "step": 2950, + "time_per_iteration": 2.79181170463562 + }, + { + "auxiliary_loss_clip": 0.01157639, + "auxiliary_loss_mlp": 0.010516, + "balance_loss_clip": 1.05194378, + "balance_loss_mlp": 1.03252649, + "epoch": 0.35483677027595745, + "flos": 27341881787520.0, + "grad_norm": 1.6945790375562966, + "language_loss": 0.75617123, + "learning_rate": 2.9916683138830295e-06, + "loss": 0.77826357, + "num_input_tokens_seen": 63423055, + "router_z_loss_clip": 1.05566406, + "router_z_loss_mlp": 0.19067383, + "step": 2951, + "time_per_iteration": 2.812717914581299 + }, + { + "auxiliary_loss_clip": 0.0115881, + "auxiliary_loss_mlp": 0.01050706, + "balance_loss_clip": 1.05228734, + "balance_loss_mlp": 1.03115582, + "epoch": 0.3549570131665965, + "flos": 13516166678400.0, + "grad_norm": 2.915367129553238, + "language_loss": 0.81086522, + "learning_rate": 2.9909917662807353e-06, + "loss": 0.83296037, + "num_input_tokens_seen": 63440855, + "router_z_loss_clip": 1.06494141, + "router_z_loss_mlp": 0.19555664, + "step": 2952, + "time_per_iteration": 2.7801008224487305 + }, + { + "auxiliary_loss_clip": 0.01158703, + "auxiliary_loss_mlp": 0.01047336, + "balance_loss_clip": 1.04982829, + "balance_loss_mlp": 1.02753532, + "epoch": 0.3550772560572356, + "flos": 20887549810560.0, + "grad_norm": 2.3951324152375726, + "language_loss": 0.69392544, + "learning_rate": 2.9903150683457783e-06, + "loss": 0.71598577, + "num_input_tokens_seen": 63459400, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.19812012, + "step": 2953, + "time_per_iteration": 2.849862813949585 + }, + { + "auxiliary_loss_clip": 0.01156883, + "auxiliary_loss_mlp": 0.01053341, + "balance_loss_clip": 1.04876268, + "balance_loss_mlp": 1.03406501, + "epoch": 0.3551974989478747, + "flos": 20194042947840.0, + "grad_norm": 1.74398602069332, + "language_loss": 0.6518634, + "learning_rate": 2.9896382201808126e-06, + "loss": 0.67396563, + "num_input_tokens_seen": 63476800, + "router_z_loss_clip": 1.08105469, + "router_z_loss_mlp": 0.19274902, + "step": 2954, + "time_per_iteration": 2.9982175827026367 + }, + { + "auxiliary_loss_clip": 0.01156318, + "auxiliary_loss_mlp": 0.0105421, + "balance_loss_clip": 1.04808962, + "balance_loss_mlp": 1.0348388, + "epoch": 0.3553177418385138, + "flos": 19828831415040.0, + "grad_norm": 2.4607338728053514, + "language_loss": 0.8111797, + "learning_rate": 2.988961221888516e-06, + "loss": 0.83328497, + "num_input_tokens_seen": 63493475, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.19372559, + "step": 2955, + "time_per_iteration": 2.869553804397583 + }, + { + "auxiliary_loss_clip": 0.01159533, + "auxiliary_loss_mlp": 0.01043103, + "balance_loss_clip": 1.05024123, + "balance_loss_mlp": 1.02436376, + "epoch": 0.3554379847291529, + "flos": 14829132516480.0, + "grad_norm": 2.3216521973930626, + "language_loss": 0.79538822, + "learning_rate": 2.988284073571589e-06, + "loss": 0.81741458, + "num_input_tokens_seen": 63509560, + "router_z_loss_clip": 1.09130859, + "router_z_loss_mlp": 0.18762207, + "step": 2956, + "time_per_iteration": 2.76911997795105 + }, + { + "auxiliary_loss_clip": 0.01155107, + "auxiliary_loss_mlp": 0.01056756, + "balance_loss_clip": 1.0499506, + "balance_loss_mlp": 1.03641891, + "epoch": 0.355558227619792, + "flos": 20485350247680.0, + "grad_norm": 2.8936259765657866, + "language_loss": 0.73002946, + "learning_rate": 2.9876067753327528e-06, + "loss": 0.75214809, + "num_input_tokens_seen": 63527290, + "router_z_loss_clip": 1.05078125, + "router_z_loss_mlp": 0.20336914, + "step": 2957, + "time_per_iteration": 2.781731605529785 + }, + { + "auxiliary_loss_clip": 0.01160537, + "auxiliary_loss_mlp": 0.01058345, + "balance_loss_clip": 1.05046344, + "balance_loss_mlp": 1.0377214, + "epoch": 0.35567847051043106, + "flos": 37663613256960.0, + "grad_norm": 1.8769064794110728, + "language_loss": 0.80518407, + "learning_rate": 2.986929327274754e-06, + "loss": 0.82737291, + "num_input_tokens_seen": 63547870, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.20617676, + "step": 2958, + "time_per_iteration": 2.912008285522461 + }, + { + "auxiliary_loss_clip": 0.01157038, + "auxiliary_loss_mlp": 0.01055825, + "balance_loss_clip": 1.05080283, + "balance_loss_mlp": 1.0373354, + "epoch": 0.35579871340107017, + "flos": 26943058103040.0, + "grad_norm": 2.2063054310620918, + "language_loss": 0.78877854, + "learning_rate": 2.9862517295003617e-06, + "loss": 0.81090724, + "num_input_tokens_seen": 63568285, + "router_z_loss_clip": 1.06201172, + "router_z_loss_mlp": 0.18505859, + "step": 2959, + "time_per_iteration": 2.837682008743286 + }, + { + "auxiliary_loss_clip": 0.01155356, + "auxiliary_loss_mlp": 0.01050045, + "balance_loss_clip": 1.04735577, + "balance_loss_mlp": 1.03043461, + "epoch": 0.3559189562917093, + "flos": 28293335193600.0, + "grad_norm": 1.7356032538797674, + "language_loss": 0.72541863, + "learning_rate": 2.9855739821123654e-06, + "loss": 0.74747264, + "num_input_tokens_seen": 63589865, + "router_z_loss_clip": 1.08105469, + "router_z_loss_mlp": 0.19592285, + "step": 2960, + "time_per_iteration": 3.6785988807678223 + }, + { + "auxiliary_loss_clip": 0.01155881, + "auxiliary_loss_mlp": 0.01049228, + "balance_loss_clip": 1.05051446, + "balance_loss_mlp": 1.03100061, + "epoch": 0.35603919918234833, + "flos": 25664063552640.0, + "grad_norm": 2.004611380962796, + "language_loss": 0.82459545, + "learning_rate": 2.98489608521358e-06, + "loss": 0.84664649, + "num_input_tokens_seen": 63609805, + "router_z_loss_clip": 1.05371094, + "router_z_loss_mlp": 0.18225098, + "step": 2961, + "time_per_iteration": 2.826773166656494 + }, + { + "auxiliary_loss_clip": 0.01156756, + "auxiliary_loss_mlp": 0.0105418, + "balance_loss_clip": 1.05000281, + "balance_loss_mlp": 1.03595269, + "epoch": 0.35615944207298744, + "flos": 23000856537600.0, + "grad_norm": 2.2017145587167373, + "language_loss": 0.79831266, + "learning_rate": 2.9842180389068425e-06, + "loss": 0.82042205, + "num_input_tokens_seen": 63627115, + "router_z_loss_clip": 1.06738281, + "router_z_loss_mlp": 0.18212891, + "step": 2962, + "time_per_iteration": 2.8444714546203613 + }, + { + "auxiliary_loss_clip": 0.01050425, + "auxiliary_loss_mlp": 0.01007068, + "balance_loss_clip": 1.01406634, + "balance_loss_mlp": 1.00454068, + "epoch": 0.35627968496362655, + "flos": 68251283723520.0, + "grad_norm": 0.7645856086111364, + "language_loss": 0.59186804, + "learning_rate": 2.98353984329501e-06, + "loss": 0.61244297, + "num_input_tokens_seen": 63691460, + "router_z_loss_clip": 0.36328125, + "router_z_loss_mlp": 0.02526855, + "step": 2963, + "time_per_iteration": 4.16358208656311 + }, + { + "auxiliary_loss_clip": 0.01159214, + "auxiliary_loss_mlp": 0.01048845, + "balance_loss_clip": 1.05060995, + "balance_loss_mlp": 1.02894878, + "epoch": 0.3563999278542656, + "flos": 22641714403200.0, + "grad_norm": 1.7469446994978903, + "language_loss": 0.70648777, + "learning_rate": 2.982861498480965e-06, + "loss": 0.72856832, + "num_input_tokens_seen": 63713840, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.19885254, + "step": 2964, + "time_per_iteration": 2.840385913848877 + }, + { + "auxiliary_loss_clip": 0.01152771, + "auxiliary_loss_mlp": 0.01054096, + "balance_loss_clip": 1.04822278, + "balance_loss_mlp": 1.03628576, + "epoch": 0.3565201707449047, + "flos": 25952533678080.0, + "grad_norm": 1.735374008950274, + "language_loss": 0.83028358, + "learning_rate": 2.9821830045676122e-06, + "loss": 0.85235226, + "num_input_tokens_seen": 63733540, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.17834473, + "step": 2965, + "time_per_iteration": 3.711486577987671 + }, + { + "auxiliary_loss_clip": 0.01160011, + "auxiliary_loss_mlp": 0.01043663, + "balance_loss_clip": 1.05188358, + "balance_loss_mlp": 1.02474403, + "epoch": 0.3566404136355438, + "flos": 28475725478400.0, + "grad_norm": 1.886151242357947, + "language_loss": 0.73147452, + "learning_rate": 2.9815043616578793e-06, + "loss": 0.75351125, + "num_input_tokens_seen": 63754335, + "router_z_loss_clip": 1.07958984, + "router_z_loss_mlp": 0.18920898, + "step": 2966, + "time_per_iteration": 3.7139124870300293 + }, + { + "auxiliary_loss_clip": 0.0115804, + "auxiliary_loss_mlp": 0.0104413, + "balance_loss_clip": 1.04953253, + "balance_loss_mlp": 1.02510452, + "epoch": 0.3567606565261829, + "flos": 38363117690880.0, + "grad_norm": 1.8011601918656235, + "language_loss": 0.76975071, + "learning_rate": 2.9808255698547145e-06, + "loss": 0.79177237, + "num_input_tokens_seen": 63777135, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.19030762, + "step": 2967, + "time_per_iteration": 2.9049644470214844 + }, + { + "auxiliary_loss_clip": 0.01155624, + "auxiliary_loss_mlp": 0.01049379, + "balance_loss_clip": 1.05028915, + "balance_loss_mlp": 1.0312469, + "epoch": 0.356880899416822, + "flos": 21981029592960.0, + "grad_norm": 2.219311944550765, + "language_loss": 0.80298024, + "learning_rate": 2.9801466292610913e-06, + "loss": 0.82503027, + "num_input_tokens_seen": 63797020, + "router_z_loss_clip": 1.05273438, + "router_z_loss_mlp": 0.18139648, + "step": 2968, + "time_per_iteration": 2.8963606357574463 + }, + { + "auxiliary_loss_clip": 0.01155424, + "auxiliary_loss_mlp": 0.01049437, + "balance_loss_clip": 1.04966509, + "balance_loss_mlp": 1.02987432, + "epoch": 0.35700114230746105, + "flos": 18989132198400.0, + "grad_norm": 2.538938523782415, + "language_loss": 0.81041902, + "learning_rate": 2.979467539980003e-06, + "loss": 0.83246768, + "num_input_tokens_seen": 63813810, + "router_z_loss_clip": 1.05908203, + "router_z_loss_mlp": 0.19567871, + "step": 2969, + "time_per_iteration": 2.8904294967651367 + }, + { + "auxiliary_loss_clip": 0.01160462, + "auxiliary_loss_mlp": 0.01051304, + "balance_loss_clip": 1.05148113, + "balance_loss_mlp": 1.03255248, + "epoch": 0.35712138519810016, + "flos": 19756112330880.0, + "grad_norm": 2.146791659814868, + "language_loss": 0.77162832, + "learning_rate": 2.978788302114468e-06, + "loss": 0.79374599, + "num_input_tokens_seen": 63830925, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.18762207, + "step": 2970, + "time_per_iteration": 2.7882802486419678 + }, + { + "auxiliary_loss_clip": 0.01159573, + "auxiliary_loss_mlp": 0.01046969, + "balance_loss_clip": 1.04964566, + "balance_loss_mlp": 1.0275023, + "epoch": 0.35724162808873927, + "flos": 35183012008320.0, + "grad_norm": 2.081297409682585, + "language_loss": 0.83282924, + "learning_rate": 2.9781089157675255e-06, + "loss": 0.85489464, + "num_input_tokens_seen": 63849385, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.19470215, + "step": 2971, + "time_per_iteration": 2.919337034225464 + }, + { + "auxiliary_loss_clip": 0.01156131, + "auxiliary_loss_mlp": 0.01048839, + "balance_loss_clip": 1.04933751, + "balance_loss_mlp": 1.03007519, + "epoch": 0.3573618709793783, + "flos": 25556726736000.0, + "grad_norm": 1.5465963364618418, + "language_loss": 0.8842932, + "learning_rate": 2.977429381042238e-06, + "loss": 0.90634286, + "num_input_tokens_seen": 63870060, + "router_z_loss_clip": 1.06835938, + "router_z_loss_mlp": 0.1875, + "step": 2972, + "time_per_iteration": 2.83266544342041 + }, + { + "auxiliary_loss_clip": 0.01153476, + "auxiliary_loss_mlp": 0.01042534, + "balance_loss_clip": 1.05072713, + "balance_loss_mlp": 1.02481961, + "epoch": 0.35748211387001744, + "flos": 29132352051840.0, + "grad_norm": 2.222622505175643, + "language_loss": 0.89185065, + "learning_rate": 2.9767496980416913e-06, + "loss": 0.91381073, + "num_input_tokens_seen": 63889355, + "router_z_loss_clip": 1.02734375, + "router_z_loss_mlp": 0.17712402, + "step": 2973, + "time_per_iteration": 2.8845059871673584 + }, + { + "auxiliary_loss_clip": 0.01156705, + "auxiliary_loss_mlp": 0.01048973, + "balance_loss_clip": 1.05031192, + "balance_loss_mlp": 1.03010249, + "epoch": 0.35760235676065655, + "flos": 13954169122560.0, + "grad_norm": 3.110441579532494, + "language_loss": 0.81434494, + "learning_rate": 2.9760698668689914e-06, + "loss": 0.8364017, + "num_input_tokens_seen": 63905580, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.18859863, + "step": 2974, + "time_per_iteration": 2.943310260772705 + }, + { + "auxiliary_loss_clip": 0.01154645, + "auxiliary_loss_mlp": 0.01051236, + "balance_loss_clip": 1.04910588, + "balance_loss_mlp": 1.03242445, + "epoch": 0.3577225996512956, + "flos": 44018688977280.0, + "grad_norm": 1.9539563799493935, + "language_loss": 0.71497357, + "learning_rate": 2.975389887627269e-06, + "loss": 0.73703235, + "num_input_tokens_seen": 63928180, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.18798828, + "step": 2975, + "time_per_iteration": 2.977255344390869 + }, + { + "auxiliary_loss_clip": 0.01154438, + "auxiliary_loss_mlp": 0.01053582, + "balance_loss_clip": 1.04827571, + "balance_loss_mlp": 1.03584361, + "epoch": 0.3578428425419347, + "flos": 17055199013760.0, + "grad_norm": 3.6349052331102367, + "language_loss": 0.90051961, + "learning_rate": 2.9747097604196764e-06, + "loss": 0.92259979, + "num_input_tokens_seen": 63944825, + "router_z_loss_clip": 1.06201172, + "router_z_loss_mlp": 0.17736816, + "step": 2976, + "time_per_iteration": 2.8012003898620605 + }, + { + "auxiliary_loss_clip": 0.01049903, + "auxiliary_loss_mlp": 0.01013163, + "balance_loss_clip": 1.01412606, + "balance_loss_mlp": 1.01076651, + "epoch": 0.3579630854325738, + "flos": 71676550707840.0, + "grad_norm": 0.6758778084010951, + "language_loss": 0.56599432, + "learning_rate": 2.9740294853493875e-06, + "loss": 0.58662498, + "num_input_tokens_seen": 64016385, + "router_z_loss_clip": 0.35742188, + "router_z_loss_mlp": 0.02392578, + "step": 2977, + "time_per_iteration": 3.581214666366577 + }, + { + "auxiliary_loss_clip": 0.0115803, + "auxiliary_loss_mlp": 0.01045561, + "balance_loss_clip": 1.05040956, + "balance_loss_mlp": 1.02627277, + "epoch": 0.3580833283232129, + "flos": 25046651652480.0, + "grad_norm": 2.1539739299768996, + "language_loss": 0.67352951, + "learning_rate": 2.9733490625196008e-06, + "loss": 0.69556546, + "num_input_tokens_seen": 64036245, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.19287109, + "step": 2978, + "time_per_iteration": 2.8727831840515137 + }, + { + "auxiliary_loss_clip": 0.01153971, + "auxiliary_loss_mlp": 0.01050517, + "balance_loss_clip": 1.04864383, + "balance_loss_mlp": 1.03230155, + "epoch": 0.358203571213852, + "flos": 13953127628160.0, + "grad_norm": 4.089206112827422, + "language_loss": 0.76016158, + "learning_rate": 2.9726684920335353e-06, + "loss": 0.78220648, + "num_input_tokens_seen": 64054110, + "router_z_loss_clip": 1.05371094, + "router_z_loss_mlp": 0.18206787, + "step": 2979, + "time_per_iteration": 2.8176448345184326 + }, + { + "auxiliary_loss_clip": 0.01159255, + "auxiliary_loss_mlp": 0.0105373, + "balance_loss_clip": 1.04933929, + "balance_loss_mlp": 1.03438258, + "epoch": 0.35832381410449105, + "flos": 20302457172480.0, + "grad_norm": 2.185304133233755, + "language_loss": 0.82015622, + "learning_rate": 2.971987773994432e-06, + "loss": 0.84228611, + "num_input_tokens_seen": 64070295, + "router_z_loss_clip": 1.09863281, + "router_z_loss_mlp": 0.19348145, + "step": 2980, + "time_per_iteration": 2.820756435394287 + }, + { + "auxiliary_loss_clip": 0.01152677, + "auxiliary_loss_mlp": 0.0104693, + "balance_loss_clip": 1.04793334, + "balance_loss_mlp": 1.02798808, + "epoch": 0.35844405699513016, + "flos": 16983234115200.0, + "grad_norm": 1.9698568850816895, + "language_loss": 0.83583117, + "learning_rate": 2.9713069085055566e-06, + "loss": 0.85782719, + "num_input_tokens_seen": 64088605, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.18933105, + "step": 2981, + "time_per_iteration": 2.8466973304748535 + }, + { + "auxiliary_loss_clip": 0.01155607, + "auxiliary_loss_mlp": 0.01045199, + "balance_loss_clip": 1.05089867, + "balance_loss_mlp": 1.02721035, + "epoch": 0.35856429988576927, + "flos": 23216858974080.0, + "grad_norm": 1.6574566728477667, + "language_loss": 0.78854573, + "learning_rate": 2.9706258956701958e-06, + "loss": 0.81055379, + "num_input_tokens_seen": 64108595, + "router_z_loss_clip": 1.04638672, + "router_z_loss_mlp": 0.17993164, + "step": 2982, + "time_per_iteration": 2.7929203510284424 + }, + { + "auxiliary_loss_clip": 0.01156956, + "auxiliary_loss_mlp": 0.01045839, + "balance_loss_clip": 1.04861593, + "balance_loss_mlp": 1.02544248, + "epoch": 0.3586845427764083, + "flos": 23034576430080.0, + "grad_norm": 2.6491574778642293, + "language_loss": 0.77291882, + "learning_rate": 2.9699447355916575e-06, + "loss": 0.79494679, + "num_input_tokens_seen": 64127405, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.20385742, + "step": 2983, + "time_per_iteration": 2.7528538703918457 + }, + { + "auxiliary_loss_clip": 0.01155066, + "auxiliary_loss_mlp": 0.01043711, + "balance_loss_clip": 1.04929185, + "balance_loss_mlp": 1.02596056, + "epoch": 0.35880478566704743, + "flos": 20010682995840.0, + "grad_norm": 2.573395761410388, + "language_loss": 0.73973882, + "learning_rate": 2.969263428373275e-06, + "loss": 0.76172662, + "num_input_tokens_seen": 64145755, + "router_z_loss_clip": 1.05859375, + "router_z_loss_mlp": 0.1776123, + "step": 2984, + "time_per_iteration": 2.778132677078247 + }, + { + "auxiliary_loss_clip": 0.01158018, + "auxiliary_loss_mlp": 0.01050914, + "balance_loss_clip": 1.05134201, + "balance_loss_mlp": 1.0326153, + "epoch": 0.35892502855768654, + "flos": 13699095667200.0, + "grad_norm": 2.3957511716864452, + "language_loss": 0.79645658, + "learning_rate": 2.9685819741184007e-06, + "loss": 0.81854594, + "num_input_tokens_seen": 64164195, + "router_z_loss_clip": 1.06738281, + "router_z_loss_mlp": 0.1829834, + "step": 2985, + "time_per_iteration": 2.844984292984009 + }, + { + "auxiliary_loss_clip": 0.01150731, + "auxiliary_loss_mlp": 0.01041602, + "balance_loss_clip": 1.05045438, + "balance_loss_mlp": 1.02281475, + "epoch": 0.3590452714483256, + "flos": 18114096977280.0, + "grad_norm": 2.1908900236510642, + "language_loss": 0.68779618, + "learning_rate": 2.967900372930411e-06, + "loss": 0.70971948, + "num_input_tokens_seen": 64182705, + "router_z_loss_clip": 1.00195312, + "router_z_loss_mlp": 0.18786621, + "step": 2986, + "time_per_iteration": 2.7612557411193848 + }, + { + "auxiliary_loss_clip": 0.01160339, + "auxiliary_loss_mlp": 0.01051609, + "balance_loss_clip": 1.05116534, + "balance_loss_mlp": 1.03129578, + "epoch": 0.3591655143389647, + "flos": 17749352321280.0, + "grad_norm": 2.2119905923920076, + "language_loss": 0.7921049, + "learning_rate": 2.9672186249127046e-06, + "loss": 0.81422436, + "num_input_tokens_seen": 64202170, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.20324707, + "step": 2987, + "time_per_iteration": 2.7464492321014404 + }, + { + "auxiliary_loss_clip": 0.01153311, + "auxiliary_loss_mlp": 0.01041756, + "balance_loss_clip": 1.05000925, + "balance_loss_mlp": 1.02368438, + "epoch": 0.3592857572296038, + "flos": 25224409082880.0, + "grad_norm": 1.9843359450510956, + "language_loss": 0.79322612, + "learning_rate": 2.9665367301687014e-06, + "loss": 0.81517673, + "num_input_tokens_seen": 64220415, + "router_z_loss_clip": 1.03320312, + "router_z_loss_mlp": 0.18066406, + "step": 2988, + "time_per_iteration": 2.797333002090454 + }, + { + "auxiliary_loss_clip": 0.01157258, + "auxiliary_loss_mlp": 0.01042247, + "balance_loss_clip": 1.05101359, + "balance_loss_mlp": 1.02251816, + "epoch": 0.3594060001202429, + "flos": 29384408764800.0, + "grad_norm": 1.7922173800001913, + "language_loss": 0.76938367, + "learning_rate": 2.965854688801845e-06, + "loss": 0.79137874, + "num_input_tokens_seen": 64242475, + "router_z_loss_clip": 1.06152344, + "router_z_loss_mlp": 0.1973877, + "step": 2989, + "time_per_iteration": 3.7271652221679688 + }, + { + "auxiliary_loss_clip": 0.01149793, + "auxiliary_loss_mlp": 0.01043156, + "balance_loss_clip": 1.04575634, + "balance_loss_mlp": 1.02464271, + "epoch": 0.359526243010882, + "flos": 17052900543360.0, + "grad_norm": 1.888085612880936, + "language_loss": 0.76664615, + "learning_rate": 2.9651725009156005e-06, + "loss": 0.78857565, + "num_input_tokens_seen": 64260220, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.18518066, + "step": 2990, + "time_per_iteration": 2.844815492630005 + }, + { + "auxiliary_loss_clip": 0.01155136, + "auxiliary_loss_mlp": 0.0104501, + "balance_loss_clip": 1.04890811, + "balance_loss_mlp": 1.02579308, + "epoch": 0.3596464859015211, + "flos": 22965089569920.0, + "grad_norm": 1.7354170643078073, + "language_loss": 0.74398267, + "learning_rate": 2.964490166613454e-06, + "loss": 0.76598406, + "num_input_tokens_seen": 64280145, + "router_z_loss_clip": 1.06152344, + "router_z_loss_mlp": 0.19213867, + "step": 2991, + "time_per_iteration": 2.809068441390991 + }, + { + "auxiliary_loss_clip": 0.01052979, + "auxiliary_loss_mlp": 0.01006376, + "balance_loss_clip": 1.01722455, + "balance_loss_mlp": 1.00419414, + "epoch": 0.35976672879216015, + "flos": 54739462590720.0, + "grad_norm": 0.7566238208192352, + "language_loss": 0.57681692, + "learning_rate": 2.963807685998917e-06, + "loss": 0.59741056, + "num_input_tokens_seen": 64336010, + "router_z_loss_clip": 0.35546875, + "router_z_loss_mlp": 0.02185059, + "step": 2992, + "time_per_iteration": 3.9800755977630615 + }, + { + "auxiliary_loss_clip": 0.01154285, + "auxiliary_loss_mlp": 0.01047236, + "balance_loss_clip": 1.0494523, + "balance_loss_mlp": 1.02950978, + "epoch": 0.35988697168279926, + "flos": 43139020901760.0, + "grad_norm": 1.5863898789650563, + "language_loss": 0.78451478, + "learning_rate": 2.9631250591755196e-06, + "loss": 0.80653006, + "num_input_tokens_seen": 64358725, + "router_z_loss_clip": 1.04980469, + "router_z_loss_mlp": 0.17724609, + "step": 2993, + "time_per_iteration": 2.9785895347595215 + }, + { + "auxiliary_loss_clip": 0.01155008, + "auxiliary_loss_mlp": 0.01055265, + "balance_loss_clip": 1.04993224, + "balance_loss_mlp": 1.0359056, + "epoch": 0.36000721457343837, + "flos": 35845600239360.0, + "grad_norm": 1.651683101048606, + "language_loss": 0.57698995, + "learning_rate": 2.962442286246817e-06, + "loss": 0.59909266, + "num_input_tokens_seen": 64381555, + "router_z_loss_clip": 1.05126953, + "router_z_loss_mlp": 0.19372559, + "step": 2994, + "time_per_iteration": 3.747358560562134 + }, + { + "auxiliary_loss_clip": 0.0115693, + "auxiliary_loss_mlp": 0.01043189, + "balance_loss_clip": 1.05123472, + "balance_loss_mlp": 1.02458072, + "epoch": 0.3601274574640774, + "flos": 18291100222080.0, + "grad_norm": 1.6538408551289963, + "language_loss": 0.70212984, + "learning_rate": 2.9617593673163853e-06, + "loss": 0.72413105, + "num_input_tokens_seen": 64400375, + "router_z_loss_clip": 1.05615234, + "router_z_loss_mlp": 0.18591309, + "step": 2995, + "time_per_iteration": 3.745753049850464 + }, + { + "auxiliary_loss_clip": 0.01153791, + "auxiliary_loss_mlp": 0.01044685, + "balance_loss_clip": 1.05006707, + "balance_loss_mlp": 1.02675581, + "epoch": 0.36024770035471654, + "flos": 13333955961600.0, + "grad_norm": 2.323560597009243, + "language_loss": 0.77256256, + "learning_rate": 2.9610763024878216e-06, + "loss": 0.79454732, + "num_input_tokens_seen": 64415880, + "router_z_loss_clip": 1.03759766, + "router_z_loss_mlp": 0.17907715, + "step": 2996, + "time_per_iteration": 2.7944369316101074 + }, + { + "auxiliary_loss_clip": 0.01154526, + "auxiliary_loss_mlp": 0.0104645, + "balance_loss_clip": 1.05001521, + "balance_loss_mlp": 1.0272572, + "epoch": 0.3603679432453556, + "flos": 20267013427200.0, + "grad_norm": 2.7288942517071395, + "language_loss": 0.91927934, + "learning_rate": 2.960393091864747e-06, + "loss": 0.94128919, + "num_input_tokens_seen": 64434260, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.19189453, + "step": 2997, + "time_per_iteration": 2.970285415649414 + }, + { + "auxiliary_loss_clip": 0.01154242, + "auxiliary_loss_mlp": 0.01041549, + "balance_loss_clip": 1.04980683, + "balance_loss_mlp": 1.02383447, + "epoch": 0.3604881861359947, + "flos": 22451135817600.0, + "grad_norm": 1.9503153290366793, + "language_loss": 0.75273788, + "learning_rate": 2.959709735550804e-06, + "loss": 0.77469581, + "num_input_tokens_seen": 64453855, + "router_z_loss_clip": 1.04394531, + "router_z_loss_mlp": 0.17700195, + "step": 2998, + "time_per_iteration": 2.86889386177063 + }, + { + "auxiliary_loss_clip": 0.01157917, + "auxiliary_loss_mlp": 0.01045529, + "balance_loss_clip": 1.05142617, + "balance_loss_mlp": 1.02695656, + "epoch": 0.3606084290266338, + "flos": 22054251467520.0, + "grad_norm": 2.0382235666222046, + "language_loss": 0.76232898, + "learning_rate": 2.9590262336496575e-06, + "loss": 0.78436351, + "num_input_tokens_seen": 64473585, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.18554688, + "step": 2999, + "time_per_iteration": 2.8260293006896973 + }, + { + "auxiliary_loss_clip": 0.01158972, + "auxiliary_loss_mlp": 0.01047801, + "balance_loss_clip": 1.0542531, + "balance_loss_mlp": 1.02720153, + "epoch": 0.36072867191727287, + "flos": 15632921111040.0, + "grad_norm": 1.985298212477775, + "language_loss": 0.85554981, + "learning_rate": 2.9583425862649936e-06, + "loss": 0.87761754, + "num_input_tokens_seen": 64491720, + "router_z_loss_clip": 1.04736328, + "router_z_loss_mlp": 0.20605469, + "step": 3000, + "time_per_iteration": 2.889068841934204 + }, + { + "auxiliary_loss_clip": 0.01160469, + "auxiliary_loss_mlp": 0.01046829, + "balance_loss_clip": 1.05152094, + "balance_loss_mlp": 1.02712345, + "epoch": 0.360848914807912, + "flos": 19677000625920.0, + "grad_norm": 2.093406694809659, + "language_loss": 0.74229848, + "learning_rate": 2.9576587935005215e-06, + "loss": 0.7643714, + "num_input_tokens_seen": 64509800, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.19702148, + "step": 3001, + "time_per_iteration": 2.8452930450439453 + }, + { + "auxiliary_loss_clip": 0.01159031, + "auxiliary_loss_mlp": 0.01046219, + "balance_loss_clip": 1.05095649, + "balance_loss_mlp": 1.02527428, + "epoch": 0.3609691576985511, + "flos": 18877808972160.0, + "grad_norm": 2.3283688837289978, + "language_loss": 0.72485149, + "learning_rate": 2.9569748554599713e-06, + "loss": 0.74690402, + "num_input_tokens_seen": 64525410, + "router_z_loss_clip": 1.08203125, + "router_z_loss_mlp": 0.20947266, + "step": 3002, + "time_per_iteration": 2.8518455028533936 + }, + { + "auxiliary_loss_clip": 0.01154214, + "auxiliary_loss_mlp": 0.01053085, + "balance_loss_clip": 1.05263186, + "balance_loss_mlp": 1.03637147, + "epoch": 0.36108940058919015, + "flos": 42224088648960.0, + "grad_norm": 2.0382975720912184, + "language_loss": 0.7320267, + "learning_rate": 2.956290772247097e-06, + "loss": 0.75409967, + "num_input_tokens_seen": 64544085, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.16699219, + "step": 3003, + "time_per_iteration": 2.985738754272461 + }, + { + "auxiliary_loss_clip": 0.01157259, + "auxiliary_loss_mlp": 0.01043302, + "balance_loss_clip": 1.05175483, + "balance_loss_mlp": 1.02618372, + "epoch": 0.36120964347982926, + "flos": 23185150243200.0, + "grad_norm": 8.892222354057553, + "language_loss": 0.73441511, + "learning_rate": 2.9556065439656724e-06, + "loss": 0.75642073, + "num_input_tokens_seen": 64563135, + "router_z_loss_clip": 1.05371094, + "router_z_loss_mlp": 0.17102051, + "step": 3004, + "time_per_iteration": 2.8887457847595215 + }, + { + "auxiliary_loss_clip": 0.01153536, + "auxiliary_loss_mlp": 0.01045591, + "balance_loss_clip": 1.04954553, + "balance_loss_mlp": 1.02722049, + "epoch": 0.36132988637046837, + "flos": 18113055482880.0, + "grad_norm": 1.712776388770158, + "language_loss": 0.81834561, + "learning_rate": 2.9549221707194952e-06, + "loss": 0.84033692, + "num_input_tokens_seen": 64581985, + "router_z_loss_clip": 1.04052734, + "router_z_loss_mlp": 0.18383789, + "step": 3005, + "time_per_iteration": 2.8083345890045166 + }, + { + "auxiliary_loss_clip": 0.0115621, + "auxiliary_loss_mlp": 0.01046019, + "balance_loss_clip": 1.05003405, + "balance_loss_mlp": 1.02599239, + "epoch": 0.3614501292611074, + "flos": 27813101333760.0, + "grad_norm": 3.0312116304931425, + "language_loss": 0.72847438, + "learning_rate": 2.954237652612384e-06, + "loss": 0.75049663, + "num_input_tokens_seen": 64601035, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.20031738, + "step": 3006, + "time_per_iteration": 2.783280611038208 + }, + { + "auxiliary_loss_clip": 0.01154096, + "auxiliary_loss_mlp": 0.0104799, + "balance_loss_clip": 1.05145192, + "balance_loss_mlp": 1.02926266, + "epoch": 0.36157037215174653, + "flos": 22634926732800.0, + "grad_norm": 2.841812806305505, + "language_loss": 0.84928983, + "learning_rate": 2.9535529897481796e-06, + "loss": 0.87131071, + "num_input_tokens_seen": 64618580, + "router_z_loss_clip": 1.02685547, + "router_z_loss_mlp": 0.18737793, + "step": 3007, + "time_per_iteration": 2.796879529953003 + }, + { + "auxiliary_loss_clip": 0.01160297, + "auxiliary_loss_mlp": 0.0104764, + "balance_loss_clip": 1.05323267, + "balance_loss_mlp": 1.02731454, + "epoch": 0.36169061504238564, + "flos": 12600839376000.0, + "grad_norm": 2.3125256987511564, + "language_loss": 0.7760179, + "learning_rate": 2.9528681822307446e-06, + "loss": 0.79809731, + "num_input_tokens_seen": 64635430, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.20336914, + "step": 3008, + "time_per_iteration": 2.774979829788208 + }, + { + "auxiliary_loss_clip": 0.01156132, + "auxiliary_loss_mlp": 0.01046356, + "balance_loss_clip": 1.0523212, + "balance_loss_mlp": 1.02766371, + "epoch": 0.3618108579330247, + "flos": 26684644682880.0, + "grad_norm": 2.4457469553270257, + "language_loss": 0.82365143, + "learning_rate": 2.952183230163964e-06, + "loss": 0.84567636, + "num_input_tokens_seen": 64655005, + "router_z_loss_clip": 1.03710938, + "router_z_loss_mlp": 0.18701172, + "step": 3009, + "time_per_iteration": 2.8009557723999023 + }, + { + "auxiliary_loss_clip": 0.0115517, + "auxiliary_loss_mlp": 0.01040667, + "balance_loss_clip": 1.05121744, + "balance_loss_mlp": 1.02202296, + "epoch": 0.3619311008236638, + "flos": 22817029708800.0, + "grad_norm": 1.8135693441892378, + "language_loss": 0.7286129, + "learning_rate": 2.9514981336517448e-06, + "loss": 0.75057125, + "num_input_tokens_seen": 64674775, + "router_z_loss_clip": 1.04003906, + "router_z_loss_mlp": 0.18652344, + "step": 3010, + "time_per_iteration": 2.788109540939331 + }, + { + "auxiliary_loss_clip": 0.01158808, + "auxiliary_loss_mlp": 0.01049337, + "balance_loss_clip": 1.05232787, + "balance_loss_mlp": 1.02880883, + "epoch": 0.36205134371430286, + "flos": 25919603884800.0, + "grad_norm": 2.406152419252583, + "language_loss": 0.81526279, + "learning_rate": 2.950812892798015e-06, + "loss": 0.83734423, + "num_input_tokens_seen": 64695670, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.20532227, + "step": 3011, + "time_per_iteration": 2.8255555629730225 + }, + { + "auxiliary_loss_clip": 0.01154567, + "auxiliary_loss_mlp": 0.0103951, + "balance_loss_clip": 1.05116761, + "balance_loss_mlp": 1.02139044, + "epoch": 0.362171586604942, + "flos": 26139592730880.0, + "grad_norm": 1.8717265728119594, + "language_loss": 0.87626433, + "learning_rate": 2.9501275077067256e-06, + "loss": 0.89820504, + "num_input_tokens_seen": 64716290, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.18103027, + "step": 3012, + "time_per_iteration": 2.8692386150360107 + }, + { + "auxiliary_loss_clip": 0.01151845, + "auxiliary_loss_mlp": 0.01040782, + "balance_loss_clip": 1.05051851, + "balance_loss_mlp": 1.02325845, + "epoch": 0.3622918294955811, + "flos": 28074208273920.0, + "grad_norm": 1.521350608183489, + "language_loss": 0.88614297, + "learning_rate": 2.949441978481848e-06, + "loss": 0.90806925, + "num_input_tokens_seen": 64737190, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.17529297, + "step": 3013, + "time_per_iteration": 3.022573232650757 + }, + { + "auxiliary_loss_clip": 0.01163295, + "auxiliary_loss_mlp": 0.01053241, + "balance_loss_clip": 1.0531193, + "balance_loss_mlp": 1.03381014, + "epoch": 0.36241207238622014, + "flos": 19828005402240.0, + "grad_norm": 1.8503722301219048, + "language_loss": 0.80208313, + "learning_rate": 2.9487563052273778e-06, + "loss": 0.82424855, + "num_input_tokens_seen": 64753950, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.19421387, + "step": 3014, + "time_per_iteration": 2.8718135356903076 + }, + { + "auxiliary_loss_clip": 0.01155337, + "auxiliary_loss_mlp": 0.01045698, + "balance_loss_clip": 1.05100787, + "balance_loss_mlp": 1.02719641, + "epoch": 0.36253231527685925, + "flos": 21397158017280.0, + "grad_norm": 1.7786843648325963, + "language_loss": 0.85992455, + "learning_rate": 2.94807048804733e-06, + "loss": 0.88193488, + "num_input_tokens_seen": 64773570, + "router_z_loss_clip": 1.04345703, + "router_z_loss_mlp": 0.18493652, + "step": 3015, + "time_per_iteration": 2.812272310256958 + }, + { + "auxiliary_loss_clip": 0.01163344, + "auxiliary_loss_mlp": 0.01054411, + "balance_loss_clip": 1.05659699, + "balance_loss_mlp": 1.03401411, + "epoch": 0.36265255816749836, + "flos": 18362885552640.0, + "grad_norm": 1.7243895076932765, + "language_loss": 0.90451998, + "learning_rate": 2.9473845270457434e-06, + "loss": 0.92669749, + "num_input_tokens_seen": 64790385, + "router_z_loss_clip": 1.06738281, + "router_z_loss_mlp": 0.20397949, + "step": 3016, + "time_per_iteration": 2.8923590183258057 + }, + { + "auxiliary_loss_clip": 0.01156041, + "auxiliary_loss_mlp": 0.01046508, + "balance_loss_clip": 1.05405378, + "balance_loss_mlp": 1.0277679, + "epoch": 0.3627728010581374, + "flos": 18660046769280.0, + "grad_norm": 2.774230142239914, + "language_loss": 0.70375121, + "learning_rate": 2.946698422326677e-06, + "loss": 0.72577667, + "num_input_tokens_seen": 64807845, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.18737793, + "step": 3017, + "time_per_iteration": 2.8360204696655273 + }, + { + "auxiliary_loss_clip": 0.01159174, + "auxiliary_loss_mlp": 0.01046179, + "balance_loss_clip": 1.05176246, + "balance_loss_mlp": 1.02648604, + "epoch": 0.36289304394877653, + "flos": 27524272072320.0, + "grad_norm": 2.4532755594985494, + "language_loss": 0.80050695, + "learning_rate": 2.946012173994213e-06, + "loss": 0.82256049, + "num_input_tokens_seen": 64827630, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.19702148, + "step": 3018, + "time_per_iteration": 3.723841905593872 + }, + { + "auxiliary_loss_clip": 0.01151595, + "auxiliary_loss_mlp": 0.01054316, + "balance_loss_clip": 1.05098915, + "balance_loss_mlp": 1.03453922, + "epoch": 0.36301328683941564, + "flos": 34533244932480.0, + "grad_norm": 1.4873412712725025, + "language_loss": 0.68044555, + "learning_rate": 2.945325782152454e-06, + "loss": 0.70250469, + "num_input_tokens_seen": 64850665, + "router_z_loss_clip": 1.00830078, + "router_z_loss_mlp": 0.19763184, + "step": 3019, + "time_per_iteration": 2.9309515953063965 + }, + { + "auxiliary_loss_clip": 0.01153727, + "auxiliary_loss_mlp": 0.01047356, + "balance_loss_clip": 1.05186653, + "balance_loss_mlp": 1.02846169, + "epoch": 0.3631335297300547, + "flos": 19025976574080.0, + "grad_norm": 2.16244922478406, + "language_loss": 0.79211414, + "learning_rate": 2.9446392469055257e-06, + "loss": 0.81412494, + "num_input_tokens_seen": 64868700, + "router_z_loss_clip": 1.01757812, + "router_z_loss_mlp": 0.18896484, + "step": 3020, + "time_per_iteration": 3.680457592010498 + }, + { + "auxiliary_loss_clip": 0.01153319, + "auxiliary_loss_mlp": 0.01049361, + "balance_loss_clip": 1.05230665, + "balance_loss_mlp": 1.0297507, + "epoch": 0.3632537726206938, + "flos": 19536769929600.0, + "grad_norm": 3.0199352757448152, + "language_loss": 0.79844159, + "learning_rate": 2.9439525683575745e-06, + "loss": 0.82046843, + "num_input_tokens_seen": 64887620, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.19616699, + "step": 3021, + "time_per_iteration": 2.8889737129211426 + }, + { + "auxiliary_loss_clip": 0.01160667, + "auxiliary_loss_mlp": 0.01049769, + "balance_loss_clip": 1.05438399, + "balance_loss_mlp": 1.03080273, + "epoch": 0.3633740155113329, + "flos": 21068611292160.0, + "grad_norm": 2.04246572678323, + "language_loss": 0.74942076, + "learning_rate": 2.9432657466127694e-06, + "loss": 0.77152514, + "num_input_tokens_seen": 64907190, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.1895752, + "step": 3022, + "time_per_iteration": 2.8308467864990234 + }, + { + "auxiliary_loss_clip": 0.0115643, + "auxiliary_loss_mlp": 0.01051727, + "balance_loss_clip": 1.05303669, + "balance_loss_mlp": 1.03305864, + "epoch": 0.36349425840197197, + "flos": 20298722158080.0, + "grad_norm": 1.7654378748602113, + "language_loss": 0.76695478, + "learning_rate": 2.9425787817753007e-06, + "loss": 0.78903633, + "num_input_tokens_seen": 64925850, + "router_z_loss_clip": 1.03466797, + "router_z_loss_mlp": 0.18664551, + "step": 3023, + "time_per_iteration": 3.667832851409912 + }, + { + "auxiliary_loss_clip": 0.01161325, + "auxiliary_loss_mlp": 0.01050423, + "balance_loss_clip": 1.05387592, + "balance_loss_mlp": 1.03016949, + "epoch": 0.3636145012926111, + "flos": 29716762331520.0, + "grad_norm": 1.4667712820130454, + "language_loss": 0.71690297, + "learning_rate": 2.94189167394938e-06, + "loss": 0.73902047, + "num_input_tokens_seen": 64948285, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.20263672, + "step": 3024, + "time_per_iteration": 3.7021701335906982 + }, + { + "auxiliary_loss_clip": 0.01160459, + "auxiliary_loss_mlp": 0.01058307, + "balance_loss_clip": 1.0554167, + "balance_loss_mlp": 1.03924608, + "epoch": 0.3637347441832502, + "flos": 21431847576960.0, + "grad_norm": 1.8452863631338563, + "language_loss": 0.81026167, + "learning_rate": 2.941204423239241e-06, + "loss": 0.83244932, + "num_input_tokens_seen": 64967160, + "router_z_loss_clip": 1.05078125, + "router_z_loss_mlp": 0.19055176, + "step": 3025, + "time_per_iteration": 2.920652389526367 + }, + { + "auxiliary_loss_clip": 0.01161056, + "auxiliary_loss_mlp": 0.01052962, + "balance_loss_clip": 1.05386031, + "balance_loss_mlp": 1.03140926, + "epoch": 0.36385498707388925, + "flos": 29533941083520.0, + "grad_norm": 2.4890102325208527, + "language_loss": 0.75959575, + "learning_rate": 2.9405170297491395e-06, + "loss": 0.78173596, + "num_input_tokens_seen": 64987155, + "router_z_loss_clip": 1.07226562, + "router_z_loss_mlp": 0.21569824, + "step": 3026, + "time_per_iteration": 2.8514952659606934 + }, + { + "auxiliary_loss_clip": 0.01161264, + "auxiliary_loss_mlp": 0.01052368, + "balance_loss_clip": 1.05717111, + "balance_loss_mlp": 1.03272247, + "epoch": 0.36397522996452836, + "flos": 22236569925120.0, + "grad_norm": 2.474756909228108, + "language_loss": 0.80487907, + "learning_rate": 2.939829493583353e-06, + "loss": 0.8270154, + "num_input_tokens_seen": 65003800, + "router_z_loss_clip": 1.04101562, + "router_z_loss_mlp": 0.19641113, + "step": 3027, + "time_per_iteration": 2.7600150108337402 + }, + { + "auxiliary_loss_clip": 0.01158587, + "auxiliary_loss_mlp": 0.01044258, + "balance_loss_clip": 1.05268049, + "balance_loss_mlp": 1.02479088, + "epoch": 0.3640954728551674, + "flos": 21506505995520.0, + "grad_norm": 2.6394472815539376, + "language_loss": 0.83451432, + "learning_rate": 2.939141814846179e-06, + "loss": 0.85654283, + "num_input_tokens_seen": 65021215, + "router_z_loss_clip": 1.05908203, + "router_z_loss_mlp": 0.19458008, + "step": 3028, + "time_per_iteration": 2.8132708072662354 + }, + { + "auxiliary_loss_clip": 0.01158663, + "auxiliary_loss_mlp": 0.01044125, + "balance_loss_clip": 1.05451703, + "balance_loss_mlp": 1.02489614, + "epoch": 0.3642157157458065, + "flos": 17712867081600.0, + "grad_norm": 1.6159296956251779, + "language_loss": 0.82534742, + "learning_rate": 2.938453993641938e-06, + "loss": 0.84737527, + "num_input_tokens_seen": 65039590, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.19238281, + "step": 3029, + "time_per_iteration": 2.7583043575286865 + }, + { + "auxiliary_loss_clip": 0.01156667, + "auxiliary_loss_mlp": 0.01049899, + "balance_loss_clip": 1.05451894, + "balance_loss_mlp": 1.03043175, + "epoch": 0.36433595863644563, + "flos": 17639537466240.0, + "grad_norm": 2.6932143293684065, + "language_loss": 0.70885968, + "learning_rate": 2.937766030074973e-06, + "loss": 0.73092532, + "num_input_tokens_seen": 65056845, + "router_z_loss_clip": 1.0234375, + "router_z_loss_mlp": 0.19458008, + "step": 3030, + "time_per_iteration": 2.77759051322937 + }, + { + "auxiliary_loss_clip": 0.01160396, + "auxiliary_loss_mlp": 0.01057719, + "balance_loss_clip": 1.05369282, + "balance_loss_mlp": 1.03719115, + "epoch": 0.3644562015270847, + "flos": 26833279161600.0, + "grad_norm": 1.6713059488433342, + "language_loss": 0.82724869, + "learning_rate": 2.937077924249646e-06, + "loss": 0.84942979, + "num_input_tokens_seen": 65079435, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.20532227, + "step": 3031, + "time_per_iteration": 2.8230984210968018 + }, + { + "auxiliary_loss_clip": 0.01162266, + "auxiliary_loss_mlp": 0.01048227, + "balance_loss_clip": 1.0554924, + "balance_loss_mlp": 1.02858138, + "epoch": 0.3645764444177238, + "flos": 14282715847680.0, + "grad_norm": 2.2343953438624475, + "language_loss": 0.75911117, + "learning_rate": 2.9363896762703443e-06, + "loss": 0.78121614, + "num_input_tokens_seen": 65096500, + "router_z_loss_clip": 1.06787109, + "router_z_loss_mlp": 0.19641113, + "step": 3032, + "time_per_iteration": 2.814385414123535 + }, + { + "auxiliary_loss_clip": 0.01161723, + "auxiliary_loss_mlp": 0.01051628, + "balance_loss_clip": 1.05463648, + "balance_loss_mlp": 1.0314225, + "epoch": 0.3646966873083629, + "flos": 20667489137280.0, + "grad_norm": 1.7447170940484407, + "language_loss": 0.842893, + "learning_rate": 2.9357012862414725e-06, + "loss": 0.86502653, + "num_input_tokens_seen": 65115860, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.20202637, + "step": 3033, + "time_per_iteration": 2.845245599746704 + }, + { + "auxiliary_loss_clip": 0.01162137, + "auxiliary_loss_mlp": 0.01043503, + "balance_loss_clip": 1.05757689, + "balance_loss_mlp": 1.02458417, + "epoch": 0.36481693019900197, + "flos": 27782613665280.0, + "grad_norm": 1.8464021305224896, + "language_loss": 0.71721601, + "learning_rate": 2.9350127542674593e-06, + "loss": 0.73927236, + "num_input_tokens_seen": 65138070, + "router_z_loss_clip": 1.04443359, + "router_z_loss_mlp": 0.18920898, + "step": 3034, + "time_per_iteration": 2.829596757888794 + }, + { + "auxiliary_loss_clip": 0.01163489, + "auxiliary_loss_mlp": 0.01046773, + "balance_loss_clip": 1.05543947, + "balance_loss_mlp": 1.02734184, + "epoch": 0.3649371730896411, + "flos": 19712588025600.0, + "grad_norm": 1.9572527229882184, + "language_loss": 0.76785624, + "learning_rate": 2.934324080452755e-06, + "loss": 0.78995883, + "num_input_tokens_seen": 65155860, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.19445801, + "step": 3035, + "time_per_iteration": 2.7747855186462402 + }, + { + "auxiliary_loss_clip": 0.01161165, + "auxiliary_loss_mlp": 0.01045921, + "balance_loss_clip": 1.05276346, + "balance_loss_mlp": 1.02482057, + "epoch": 0.3650574159802802, + "flos": 24750496016640.0, + "grad_norm": 1.6432496862398276, + "language_loss": 0.78232169, + "learning_rate": 2.9336352649018307e-06, + "loss": 0.80439258, + "num_input_tokens_seen": 65175930, + "router_z_loss_clip": 1.08447266, + "router_z_loss_mlp": 0.2109375, + "step": 3036, + "time_per_iteration": 2.947193145751953 + }, + { + "auxiliary_loss_clip": 0.01165044, + "auxiliary_loss_mlp": 0.01047119, + "balance_loss_clip": 1.05858982, + "balance_loss_mlp": 1.02564955, + "epoch": 0.36517765887091924, + "flos": 32853487363200.0, + "grad_norm": 1.7972982825698802, + "language_loss": 0.70390618, + "learning_rate": 2.9329463077191783e-06, + "loss": 0.72602785, + "num_input_tokens_seen": 65199305, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.21472168, + "step": 3037, + "time_per_iteration": 2.977081775665283 + }, + { + "auxiliary_loss_clip": 0.01159965, + "auxiliary_loss_mlp": 0.01049064, + "balance_loss_clip": 1.0547626, + "balance_loss_mlp": 1.02775002, + "epoch": 0.36529790176155835, + "flos": 20120318282880.0, + "grad_norm": 1.9642632443242083, + "language_loss": 0.6438154, + "learning_rate": 2.9322572090093135e-06, + "loss": 0.66590571, + "num_input_tokens_seen": 65218010, + "router_z_loss_clip": 1.05224609, + "router_z_loss_mlp": 0.21289062, + "step": 3038, + "time_per_iteration": 2.834026336669922 + }, + { + "auxiliary_loss_clip": 0.01161582, + "auxiliary_loss_mlp": 0.01050483, + "balance_loss_clip": 1.05406141, + "balance_loss_mlp": 1.02959728, + "epoch": 0.36541814465219746, + "flos": 17639573379840.0, + "grad_norm": 2.6407670382913633, + "language_loss": 0.76982319, + "learning_rate": 2.9315679688767713e-06, + "loss": 0.79194379, + "num_input_tokens_seen": 65236020, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.20898438, + "step": 3039, + "time_per_iteration": 2.82463002204895 + }, + { + "auxiliary_loss_clip": 0.0115972, + "auxiliary_loss_mlp": 0.01050671, + "balance_loss_clip": 1.05565405, + "balance_loss_mlp": 1.03133512, + "epoch": 0.3655383875428365, + "flos": 22674356887680.0, + "grad_norm": 1.552221592139191, + "language_loss": 0.66509271, + "learning_rate": 2.9308785874261085e-06, + "loss": 0.68719655, + "num_input_tokens_seen": 65256210, + "router_z_loss_clip": 1.04101562, + "router_z_loss_mlp": 0.19348145, + "step": 3040, + "time_per_iteration": 2.864924669265747 + }, + { + "auxiliary_loss_clip": 0.01161473, + "auxiliary_loss_mlp": 0.01050243, + "balance_loss_clip": 1.05537105, + "balance_loss_mlp": 1.03009677, + "epoch": 0.36565863043347563, + "flos": 21981173247360.0, + "grad_norm": 2.1509056884440696, + "language_loss": 0.81790626, + "learning_rate": 2.9301890647619045e-06, + "loss": 0.84002346, + "num_input_tokens_seen": 65275505, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.20129395, + "step": 3041, + "time_per_iteration": 2.8063347339630127 + }, + { + "auxiliary_loss_clip": 0.01161752, + "auxiliary_loss_mlp": 0.0105307, + "balance_loss_clip": 1.05325246, + "balance_loss_mlp": 1.03244662, + "epoch": 0.36577887332411474, + "flos": 24827632473600.0, + "grad_norm": 3.100088885173903, + "language_loss": 0.80616182, + "learning_rate": 2.929499400988759e-06, + "loss": 0.82831007, + "num_input_tokens_seen": 65296665, + "router_z_loss_clip": 1.08398438, + "router_z_loss_mlp": 0.20629883, + "step": 3042, + "time_per_iteration": 3.012219190597534 + }, + { + "auxiliary_loss_clip": 0.01161175, + "auxiliary_loss_mlp": 0.01059905, + "balance_loss_clip": 1.0517478, + "balance_loss_mlp": 1.03979445, + "epoch": 0.3658991162147538, + "flos": 28293191539200.0, + "grad_norm": 1.7415059353023383, + "language_loss": 0.65426862, + "learning_rate": 2.9288095962112927e-06, + "loss": 0.67647946, + "num_input_tokens_seen": 65317370, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.20092773, + "step": 3043, + "time_per_iteration": 2.8006138801574707 + }, + { + "auxiliary_loss_clip": 0.0115832, + "auxiliary_loss_mlp": 0.01045736, + "balance_loss_clip": 1.0529933, + "balance_loss_mlp": 1.02516019, + "epoch": 0.3660193591053929, + "flos": 17785550252160.0, + "grad_norm": 1.8377124626366041, + "language_loss": 0.85410941, + "learning_rate": 2.9281196505341503e-06, + "loss": 0.87614989, + "num_input_tokens_seen": 65334540, + "router_z_loss_clip": 1.05273438, + "router_z_loss_mlp": 0.20568848, + "step": 3044, + "time_per_iteration": 2.7928972244262695 + }, + { + "auxiliary_loss_clip": 0.01155181, + "auxiliary_loss_mlp": 0.01050724, + "balance_loss_clip": 1.05140758, + "balance_loss_mlp": 1.03062582, + "epoch": 0.36613960199603196, + "flos": 10342776839040.0, + "grad_norm": 3.6212317235096285, + "language_loss": 0.79044664, + "learning_rate": 2.9274295640619946e-06, + "loss": 0.81250572, + "num_input_tokens_seen": 65351670, + "router_z_loss_clip": 1.03857422, + "router_z_loss_mlp": 0.20092773, + "step": 3045, + "time_per_iteration": 2.8190536499023438 + }, + { + "auxiliary_loss_clip": 0.01160165, + "auxiliary_loss_mlp": 0.01046727, + "balance_loss_clip": 1.05397332, + "balance_loss_mlp": 1.02706909, + "epoch": 0.36625984488667107, + "flos": 19755609540480.0, + "grad_norm": 2.347850911437205, + "language_loss": 0.78460753, + "learning_rate": 2.9267393368995103e-06, + "loss": 0.80667639, + "num_input_tokens_seen": 65370900, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.1965332, + "step": 3046, + "time_per_iteration": 2.8224964141845703 + }, + { + "auxiliary_loss_clip": 0.01157377, + "auxiliary_loss_mlp": 0.01057815, + "balance_loss_clip": 1.05268097, + "balance_loss_mlp": 1.03728747, + "epoch": 0.3663800877773102, + "flos": 17674262939520.0, + "grad_norm": 2.1809370695391537, + "language_loss": 0.74675411, + "learning_rate": 2.926048969151407e-06, + "loss": 0.768906, + "num_input_tokens_seen": 65388185, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.20556641, + "step": 3047, + "time_per_iteration": 3.7409205436706543 + }, + { + "auxiliary_loss_clip": 0.01157231, + "auxiliary_loss_mlp": 0.01048996, + "balance_loss_clip": 1.05195189, + "balance_loss_mlp": 1.0296011, + "epoch": 0.36650033066794924, + "flos": 20303606407680.0, + "grad_norm": 2.0318747762273044, + "language_loss": 0.68035352, + "learning_rate": 2.92535846092241e-06, + "loss": 0.70241582, + "num_input_tokens_seen": 65407200, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.1940918, + "step": 3048, + "time_per_iteration": 2.8105242252349854 + }, + { + "auxiliary_loss_clip": 0.01162635, + "auxiliary_loss_mlp": 0.0105941, + "balance_loss_clip": 1.05399525, + "balance_loss_mlp": 1.03808379, + "epoch": 0.36662057355858835, + "flos": 24716237420160.0, + "grad_norm": 2.151139700305502, + "language_loss": 0.83035994, + "learning_rate": 2.9246678123172704e-06, + "loss": 0.85258037, + "num_input_tokens_seen": 65427290, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.21325684, + "step": 3049, + "time_per_iteration": 3.689627170562744 + }, + { + "auxiliary_loss_clip": 0.01160968, + "auxiliary_loss_mlp": 0.01051315, + "balance_loss_clip": 1.05309057, + "balance_loss_mlp": 1.03102565, + "epoch": 0.36674081644922746, + "flos": 12385267902720.0, + "grad_norm": 2.6227300714048316, + "language_loss": 0.74209446, + "learning_rate": 2.9239770234407596e-06, + "loss": 0.76421726, + "num_input_tokens_seen": 65445595, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.20300293, + "step": 3050, + "time_per_iteration": 2.814467430114746 + }, + { + "auxiliary_loss_clip": 0.01159929, + "auxiliary_loss_mlp": 0.0104726, + "balance_loss_clip": 1.05171204, + "balance_loss_mlp": 1.02586186, + "epoch": 0.3668610593398665, + "flos": 21105922544640.0, + "grad_norm": 1.6029937746492762, + "language_loss": 0.68579227, + "learning_rate": 2.9232860943976686e-06, + "loss": 0.70786417, + "num_input_tokens_seen": 65466330, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.21398926, + "step": 3051, + "time_per_iteration": 2.8117783069610596 + }, + { + "auxiliary_loss_clip": 0.01156867, + "auxiliary_loss_mlp": 0.01045371, + "balance_loss_clip": 1.05492771, + "balance_loss_mlp": 1.02603531, + "epoch": 0.3669813022305056, + "flos": 26758082039040.0, + "grad_norm": 1.6246642969607448, + "language_loss": 0.84247291, + "learning_rate": 2.9225950252928115e-06, + "loss": 0.86449528, + "num_input_tokens_seen": 65487180, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.19348145, + "step": 3052, + "time_per_iteration": 3.6942238807678223 + }, + { + "auxiliary_loss_clip": 0.01160018, + "auxiliary_loss_mlp": 0.01046157, + "balance_loss_clip": 1.05213821, + "balance_loss_mlp": 1.02595127, + "epoch": 0.36710154512114473, + "flos": 19099521671040.0, + "grad_norm": 2.7383853454814506, + "language_loss": 0.81774497, + "learning_rate": 2.9219038162310217e-06, + "loss": 0.8398068, + "num_input_tokens_seen": 65505380, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.20202637, + "step": 3053, + "time_per_iteration": 3.673693895339966 + }, + { + "auxiliary_loss_clip": 0.01158813, + "auxiliary_loss_mlp": 0.01054325, + "balance_loss_clip": 1.05008149, + "balance_loss_mlp": 1.03397572, + "epoch": 0.3672217880117838, + "flos": 20812029465600.0, + "grad_norm": 1.9817764184517161, + "language_loss": 0.82606363, + "learning_rate": 2.921212467317157e-06, + "loss": 0.84819502, + "num_input_tokens_seen": 65524825, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.20349121, + "step": 3054, + "time_per_iteration": 2.8078670501708984 + }, + { + "auxiliary_loss_clip": 0.01158155, + "auxiliary_loss_mlp": 0.01053024, + "balance_loss_clip": 1.04948854, + "balance_loss_mlp": 1.03058875, + "epoch": 0.3673420309024229, + "flos": 13590394133760.0, + "grad_norm": 2.0808696918398595, + "language_loss": 0.80270433, + "learning_rate": 2.920520978656093e-06, + "loss": 0.82481611, + "num_input_tokens_seen": 65541790, + "router_z_loss_clip": 1.08740234, + "router_z_loss_mlp": 0.22436523, + "step": 3055, + "time_per_iteration": 2.8651294708251953 + }, + { + "auxiliary_loss_clip": 0.01156844, + "auxiliary_loss_mlp": 0.01056736, + "balance_loss_clip": 1.05349469, + "balance_loss_mlp": 1.03632748, + "epoch": 0.367462273793062, + "flos": 28986877969920.0, + "grad_norm": 2.8720158705190237, + "language_loss": 0.76889074, + "learning_rate": 2.919829350352729e-06, + "loss": 0.79102659, + "num_input_tokens_seen": 65563395, + "router_z_loss_clip": 1.03222656, + "router_z_loss_mlp": 0.20410156, + "step": 3056, + "time_per_iteration": 2.9689836502075195 + }, + { + "auxiliary_loss_clip": 0.010553, + "auxiliary_loss_mlp": 0.01003669, + "balance_loss_clip": 1.01935554, + "balance_loss_mlp": 1.00120187, + "epoch": 0.36758251668370107, + "flos": 62643148346880.0, + "grad_norm": 0.7577475061052658, + "language_loss": 0.59931988, + "learning_rate": 2.919137582511983e-06, + "loss": 0.61990958, + "num_input_tokens_seen": 65619835, + "router_z_loss_clip": 0.359375, + "router_z_loss_mlp": 0.0246582, + "step": 3057, + "time_per_iteration": 3.3906021118164062 + }, + { + "auxiliary_loss_clip": 0.01158928, + "auxiliary_loss_mlp": 0.01047025, + "balance_loss_clip": 1.05166578, + "balance_loss_mlp": 1.02754641, + "epoch": 0.3677027595743402, + "flos": 12713886455040.0, + "grad_norm": 1.8588920395133761, + "language_loss": 0.63864613, + "learning_rate": 2.918445675238797e-06, + "loss": 0.66070569, + "num_input_tokens_seen": 65636760, + "router_z_loss_clip": 1.07275391, + "router_z_loss_mlp": 0.19482422, + "step": 3058, + "time_per_iteration": 2.8520147800445557 + }, + { + "auxiliary_loss_clip": 0.01157463, + "auxiliary_loss_mlp": 0.01058754, + "balance_loss_clip": 1.0493542, + "balance_loss_mlp": 1.03884649, + "epoch": 0.36782300246497923, + "flos": 25046579825280.0, + "grad_norm": 1.7841048657080036, + "language_loss": 0.69928622, + "learning_rate": 2.917753628638132e-06, + "loss": 0.7214483, + "num_input_tokens_seen": 65657065, + "router_z_loss_clip": 1.08105469, + "router_z_loss_mlp": 0.19909668, + "step": 3059, + "time_per_iteration": 2.9482061862945557 + }, + { + "auxiliary_loss_clip": 0.01157846, + "auxiliary_loss_mlp": 0.01049739, + "balance_loss_clip": 1.05145299, + "balance_loss_mlp": 1.02983069, + "epoch": 0.36794324535561834, + "flos": 17419512706560.0, + "grad_norm": 2.1164119502508623, + "language_loss": 0.70374751, + "learning_rate": 2.9170614428149716e-06, + "loss": 0.7258234, + "num_input_tokens_seen": 65675400, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.19921875, + "step": 3060, + "time_per_iteration": 2.9200260639190674 + }, + { + "auxiliary_loss_clip": 0.01161669, + "auxiliary_loss_mlp": 0.01053286, + "balance_loss_clip": 1.05406106, + "balance_loss_mlp": 1.03265119, + "epoch": 0.36806348824625745, + "flos": 24089128848000.0, + "grad_norm": 2.4774790194044325, + "language_loss": 0.87103117, + "learning_rate": 2.9163691178743195e-06, + "loss": 0.89318073, + "num_input_tokens_seen": 65694050, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.2064209, + "step": 3061, + "time_per_iteration": 2.854644298553467 + }, + { + "auxiliary_loss_clip": 0.01154227, + "auxiliary_loss_mlp": 0.01053386, + "balance_loss_clip": 1.0502255, + "balance_loss_mlp": 1.03495657, + "epoch": 0.3681837311368965, + "flos": 20521871400960.0, + "grad_norm": 2.0549827684500785, + "language_loss": 0.77731264, + "learning_rate": 2.9156766539212006e-06, + "loss": 0.79938877, + "num_input_tokens_seen": 65711695, + "router_z_loss_clip": 1.04101562, + "router_z_loss_mlp": 0.18444824, + "step": 3062, + "time_per_iteration": 2.7980988025665283 + }, + { + "auxiliary_loss_clip": 0.01154115, + "auxiliary_loss_mlp": 0.01042494, + "balance_loss_clip": 1.04867435, + "balance_loss_mlp": 1.02332473, + "epoch": 0.3683039740275356, + "flos": 21466644877440.0, + "grad_norm": 3.896419248577915, + "language_loss": 0.72112244, + "learning_rate": 2.9149840510606614e-06, + "loss": 0.74308854, + "num_input_tokens_seen": 65730350, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.19177246, + "step": 3063, + "time_per_iteration": 2.795701026916504 + }, + { + "auxiliary_loss_clip": 0.0105442, + "auxiliary_loss_mlp": 0.01006384, + "balance_loss_clip": 1.01915503, + "balance_loss_mlp": 1.00424981, + "epoch": 0.36842421691817473, + "flos": 70380999987840.0, + "grad_norm": 1.0352884499669575, + "language_loss": 0.64256561, + "learning_rate": 2.914291309397769e-06, + "loss": 0.66317368, + "num_input_tokens_seen": 65787820, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.0213623, + "step": 3064, + "time_per_iteration": 3.393280267715454 + }, + { + "auxiliary_loss_clip": 0.0115368, + "auxiliary_loss_mlp": 0.01049733, + "balance_loss_clip": 1.04749477, + "balance_loss_mlp": 1.02888322, + "epoch": 0.3685444598088138, + "flos": 23331378510720.0, + "grad_norm": 2.041230705365664, + "language_loss": 0.79216939, + "learning_rate": 2.9135984290376117e-06, + "loss": 0.8142035, + "num_input_tokens_seen": 65806685, + "router_z_loss_clip": 1.06103516, + "router_z_loss_mlp": 0.20849609, + "step": 3065, + "time_per_iteration": 2.8123273849487305 + }, + { + "auxiliary_loss_clip": 0.0115528, + "auxiliary_loss_mlp": 0.0104766, + "balance_loss_clip": 1.05209279, + "balance_loss_mlp": 1.02796638, + "epoch": 0.3686647026994529, + "flos": 23070271570560.0, + "grad_norm": 1.816896527472302, + "language_loss": 0.83179784, + "learning_rate": 2.9129054100853e-06, + "loss": 0.85382724, + "num_input_tokens_seen": 65825525, + "router_z_loss_clip": 1.03320312, + "router_z_loss_mlp": 0.19714355, + "step": 3066, + "time_per_iteration": 2.7646703720092773 + }, + { + "auxiliary_loss_clip": 0.01161482, + "auxiliary_loss_mlp": 0.01051397, + "balance_loss_clip": 1.05131459, + "balance_loss_mlp": 1.03091741, + "epoch": 0.368784945590092, + "flos": 25119909440640.0, + "grad_norm": 1.8717504168128627, + "language_loss": 0.76242888, + "learning_rate": 2.912212252645963e-06, + "loss": 0.78455764, + "num_input_tokens_seen": 65848110, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.20483398, + "step": 3067, + "time_per_iteration": 2.888335943222046 + }, + { + "auxiliary_loss_clip": 0.01162329, + "auxiliary_loss_mlp": 0.01055798, + "balance_loss_clip": 1.05167842, + "balance_loss_mlp": 1.03499603, + "epoch": 0.36890518848073106, + "flos": 18442284566400.0, + "grad_norm": 2.182393991527316, + "language_loss": 0.75977153, + "learning_rate": 2.9115189568247523e-06, + "loss": 0.7819528, + "num_input_tokens_seen": 65865670, + "router_z_loss_clip": 1.10839844, + "router_z_loss_mlp": 0.20825195, + "step": 3068, + "time_per_iteration": 2.7730178833007812 + }, + { + "auxiliary_loss_clip": 0.0115537, + "auxiliary_loss_mlp": 0.01046228, + "balance_loss_clip": 1.05173278, + "balance_loss_mlp": 1.02659452, + "epoch": 0.36902543137137017, + "flos": 16362446336640.0, + "grad_norm": 2.0634900050051566, + "language_loss": 0.92171282, + "learning_rate": 2.910825522726841e-06, + "loss": 0.9437288, + "num_input_tokens_seen": 65883195, + "router_z_loss_clip": 1.03613281, + "router_z_loss_mlp": 0.19641113, + "step": 3069, + "time_per_iteration": 2.8402459621429443 + }, + { + "auxiliary_loss_clip": 0.01154686, + "auxiliary_loss_mlp": 0.01052437, + "balance_loss_clip": 1.05018675, + "balance_loss_mlp": 1.03249335, + "epoch": 0.3691456742620093, + "flos": 12275596702080.0, + "grad_norm": 2.126968868430298, + "language_loss": 0.77293593, + "learning_rate": 2.9101319504574215e-06, + "loss": 0.79500711, + "num_input_tokens_seen": 65899635, + "router_z_loss_clip": 1.04345703, + "router_z_loss_mlp": 0.19946289, + "step": 3070, + "time_per_iteration": 2.789525032043457 + }, + { + "auxiliary_loss_clip": 0.01156251, + "auxiliary_loss_mlp": 0.01048148, + "balance_loss_clip": 1.04990149, + "balance_loss_mlp": 1.02807307, + "epoch": 0.36926591715264834, + "flos": 17786412178560.0, + "grad_norm": 2.0184148921217444, + "language_loss": 0.76322293, + "learning_rate": 2.909438240121709e-06, + "loss": 0.78526694, + "num_input_tokens_seen": 65919910, + "router_z_loss_clip": 1.06347656, + "router_z_loss_mlp": 0.20080566, + "step": 3071, + "time_per_iteration": 2.8343665599823 + }, + { + "auxiliary_loss_clip": 0.01153777, + "auxiliary_loss_mlp": 0.01046192, + "balance_loss_clip": 1.05150497, + "balance_loss_mlp": 1.0273087, + "epoch": 0.36938616004328745, + "flos": 28948309741440.0, + "grad_norm": 1.997848407844899, + "language_loss": 0.70293784, + "learning_rate": 2.908744391824939e-06, + "loss": 0.72493756, + "num_input_tokens_seen": 65940930, + "router_z_loss_clip": 1.02246094, + "router_z_loss_mlp": 0.18884277, + "step": 3072, + "time_per_iteration": 2.903947353363037 + }, + { + "auxiliary_loss_clip": 0.01156083, + "auxiliary_loss_mlp": 0.01045579, + "balance_loss_clip": 1.04756689, + "balance_loss_mlp": 1.02611208, + "epoch": 0.36950640293392656, + "flos": 29205394358400.0, + "grad_norm": 1.9776801575976046, + "language_loss": 0.79206097, + "learning_rate": 2.908050405672367e-06, + "loss": 0.81407762, + "num_input_tokens_seen": 65960475, + "router_z_loss_clip": 1.08398438, + "router_z_loss_mlp": 0.19470215, + "step": 3073, + "time_per_iteration": 2.932896614074707 + }, + { + "auxiliary_loss_clip": 0.01155023, + "auxiliary_loss_mlp": 0.01051221, + "balance_loss_clip": 1.04933071, + "balance_loss_mlp": 1.03089619, + "epoch": 0.3696266458245656, + "flos": 24827776128000.0, + "grad_norm": 1.77464239642954, + "language_loss": 0.79394436, + "learning_rate": 2.9073562817692703e-06, + "loss": 0.81600678, + "num_input_tokens_seen": 65979160, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.203125, + "step": 3074, + "time_per_iteration": 2.919365167617798 + }, + { + "auxiliary_loss_clip": 0.01052645, + "auxiliary_loss_mlp": 0.01007438, + "balance_loss_clip": 1.01729667, + "balance_loss_mlp": 1.00524449, + "epoch": 0.3697468887152047, + "flos": 59887257264000.0, + "grad_norm": 0.7212180588281834, + "language_loss": 0.56545109, + "learning_rate": 2.9066620202209468e-06, + "loss": 0.58605188, + "num_input_tokens_seen": 66041650, + "router_z_loss_clip": 0.35351562, + "router_z_loss_mlp": 0.02197266, + "step": 3075, + "time_per_iteration": 3.3137102127075195 + }, + { + "auxiliary_loss_clip": 0.01151151, + "auxiliary_loss_mlp": 0.01049446, + "balance_loss_clip": 1.04887867, + "balance_loss_mlp": 1.02982426, + "epoch": 0.3698671316058438, + "flos": 26137581569280.0, + "grad_norm": 3.1765298249028646, + "language_loss": 0.77827227, + "learning_rate": 2.905967621132716e-06, + "loss": 0.80027825, + "num_input_tokens_seen": 66059260, + "router_z_loss_clip": 1.02197266, + "router_z_loss_mlp": 0.19604492, + "step": 3076, + "time_per_iteration": 3.7279717922210693 + }, + { + "auxiliary_loss_clip": 0.01158545, + "auxiliary_loss_mlp": 0.01048576, + "balance_loss_clip": 1.04993463, + "balance_loss_mlp": 1.02750015, + "epoch": 0.3699873744964829, + "flos": 24607464059520.0, + "grad_norm": 1.8879342349923496, + "language_loss": 0.75137937, + "learning_rate": 2.9052730846099172e-06, + "loss": 0.77345061, + "num_input_tokens_seen": 66080605, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.21069336, + "step": 3077, + "time_per_iteration": 2.9016151428222656 + }, + { + "auxiliary_loss_clip": 0.01052147, + "auxiliary_loss_mlp": 0.01004911, + "balance_loss_clip": 1.01670694, + "balance_loss_mlp": 1.0028609, + "epoch": 0.370107617387122, + "flos": 64885340050560.0, + "grad_norm": 0.8634851630020677, + "language_loss": 0.60940212, + "learning_rate": 2.9045784107579123e-06, + "loss": 0.62997264, + "num_input_tokens_seen": 66140710, + "router_z_loss_clip": 0.35546875, + "router_z_loss_mlp": 0.02050781, + "step": 3078, + "time_per_iteration": 4.185152053833008 + }, + { + "auxiliary_loss_clip": 0.01155809, + "auxiliary_loss_mlp": 0.0104461, + "balance_loss_clip": 1.04994917, + "balance_loss_mlp": 1.02466607, + "epoch": 0.37022786027776106, + "flos": 15961683317760.0, + "grad_norm": 2.023014539344151, + "language_loss": 0.66754568, + "learning_rate": 2.9038835996820807e-06, + "loss": 0.68954986, + "num_input_tokens_seen": 66158320, + "router_z_loss_clip": 1.05810547, + "router_z_loss_mlp": 0.19958496, + "step": 3079, + "time_per_iteration": 2.888181209564209 + }, + { + "auxiliary_loss_clip": 0.0115789, + "auxiliary_loss_mlp": 0.01048269, + "balance_loss_clip": 1.05340588, + "balance_loss_mlp": 1.03064966, + "epoch": 0.37034810316840017, + "flos": 18546927863040.0, + "grad_norm": 2.0501822517608432, + "language_loss": 0.79579425, + "learning_rate": 2.903188651487826e-06, + "loss": 0.81785583, + "num_input_tokens_seen": 66176875, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.17614746, + "step": 3080, + "time_per_iteration": 2.824934959411621 + }, + { + "auxiliary_loss_clip": 0.01154577, + "auxiliary_loss_mlp": 0.01046398, + "balance_loss_clip": 1.05180335, + "balance_loss_mlp": 1.02668118, + "epoch": 0.3704683460590393, + "flos": 17821927751040.0, + "grad_norm": 2.0077129816774186, + "language_loss": 0.86861366, + "learning_rate": 2.902493566280571e-06, + "loss": 0.89062345, + "num_input_tokens_seen": 66194980, + "router_z_loss_clip": 1.02685547, + "router_z_loss_mlp": 0.19726562, + "step": 3081, + "time_per_iteration": 3.5960984230041504 + }, + { + "auxiliary_loss_clip": 0.01161786, + "auxiliary_loss_mlp": 0.01050494, + "balance_loss_clip": 1.05139613, + "balance_loss_mlp": 1.03088355, + "epoch": 0.37058858894967833, + "flos": 14134081368960.0, + "grad_norm": 2.048943085225998, + "language_loss": 0.81530857, + "learning_rate": 2.9017983441657595e-06, + "loss": 0.83743143, + "num_input_tokens_seen": 66212310, + "router_z_loss_clip": 1.10205078, + "router_z_loss_mlp": 0.19592285, + "step": 3082, + "time_per_iteration": 3.601691722869873 + }, + { + "auxiliary_loss_clip": 0.01154573, + "auxiliary_loss_mlp": 0.01052321, + "balance_loss_clip": 1.05123127, + "balance_loss_mlp": 1.03194785, + "epoch": 0.37070883184031744, + "flos": 13954492344960.0, + "grad_norm": 3.053916424700847, + "language_loss": 0.75311655, + "learning_rate": 2.9011029852488564e-06, + "loss": 0.77518553, + "num_input_tokens_seen": 66229545, + "router_z_loss_clip": 1.03320312, + "router_z_loss_mlp": 0.20373535, + "step": 3083, + "time_per_iteration": 2.8125693798065186 + }, + { + "auxiliary_loss_clip": 0.01050549, + "auxiliary_loss_mlp": 0.01003163, + "balance_loss_clip": 1.01550269, + "balance_loss_mlp": 1.0011481, + "epoch": 0.37082907473095655, + "flos": 52315419306240.0, + "grad_norm": 0.9961468355240134, + "language_loss": 0.62486702, + "learning_rate": 2.9004074896353465e-06, + "loss": 0.64540416, + "num_input_tokens_seen": 66283545, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.0201416, + "step": 3084, + "time_per_iteration": 3.2603721618652344 + }, + { + "auxiliary_loss_clip": 0.01151826, + "auxiliary_loss_mlp": 0.01042996, + "balance_loss_clip": 1.05163002, + "balance_loss_mlp": 1.02526999, + "epoch": 0.3709493176215956, + "flos": 15998096730240.0, + "grad_norm": 1.8457623358596387, + "language_loss": 0.81784016, + "learning_rate": 2.8997118574307362e-06, + "loss": 0.83978838, + "num_input_tokens_seen": 66300500, + "router_z_loss_clip": 1.00195312, + "router_z_loss_mlp": 0.17736816, + "step": 3085, + "time_per_iteration": 2.8885114192962646 + }, + { + "auxiliary_loss_clip": 0.01158474, + "auxiliary_loss_mlp": 0.01053505, + "balance_loss_clip": 1.05156553, + "balance_loss_mlp": 1.0337168, + "epoch": 0.3710695605122347, + "flos": 20959837931520.0, + "grad_norm": 2.181445271435654, + "language_loss": 0.74982983, + "learning_rate": 2.899016088740553e-06, + "loss": 0.77194965, + "num_input_tokens_seen": 66318610, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.19787598, + "step": 3086, + "time_per_iteration": 2.8016598224639893 + }, + { + "auxiliary_loss_clip": 0.01153174, + "auxiliary_loss_mlp": 0.01042694, + "balance_loss_clip": 1.04841399, + "balance_loss_mlp": 1.02388287, + "epoch": 0.37118980340287383, + "flos": 14355578586240.0, + "grad_norm": 2.030137914318691, + "language_loss": 0.79771292, + "learning_rate": 2.898320183670344e-06, + "loss": 0.81967163, + "num_input_tokens_seen": 66336025, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.18811035, + "step": 3087, + "time_per_iteration": 2.8517355918884277 + }, + { + "auxiliary_loss_clip": 0.01158137, + "auxiliary_loss_mlp": 0.01047335, + "balance_loss_clip": 1.05085027, + "balance_loss_mlp": 1.02742743, + "epoch": 0.3713100462935129, + "flos": 25885381201920.0, + "grad_norm": 1.7919596112560516, + "language_loss": 0.89042228, + "learning_rate": 2.8976241423256767e-06, + "loss": 0.91247702, + "num_input_tokens_seen": 66356120, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.19921875, + "step": 3088, + "time_per_iteration": 2.8521268367767334 + }, + { + "auxiliary_loss_clip": 0.01148979, + "auxiliary_loss_mlp": 0.01050874, + "balance_loss_clip": 1.04927564, + "balance_loss_mlp": 1.03250408, + "epoch": 0.371430289184152, + "flos": 30518934814080.0, + "grad_norm": 2.3334354555716295, + "language_loss": 0.68328309, + "learning_rate": 2.896927964812142e-06, + "loss": 0.70528156, + "num_input_tokens_seen": 66376685, + "router_z_loss_clip": 0.99755859, + "router_z_loss_mlp": 0.18359375, + "step": 3089, + "time_per_iteration": 2.8552968502044678 + }, + { + "auxiliary_loss_clip": 0.01155476, + "auxiliary_loss_mlp": 0.01055035, + "balance_loss_clip": 1.05167019, + "balance_loss_mlp": 1.03554416, + "epoch": 0.37155053207479105, + "flos": 15742233175680.0, + "grad_norm": 2.519327359676987, + "language_loss": 0.75053036, + "learning_rate": 2.8962316512353465e-06, + "loss": 0.77263546, + "num_input_tokens_seen": 66394230, + "router_z_loss_clip": 1.03857422, + "router_z_loss_mlp": 0.19482422, + "step": 3090, + "time_per_iteration": 2.8561925888061523 + }, + { + "auxiliary_loss_clip": 0.01153804, + "auxiliary_loss_mlp": 0.01047689, + "balance_loss_clip": 1.05006373, + "balance_loss_mlp": 1.028198, + "epoch": 0.37167077496543016, + "flos": 23404061681280.0, + "grad_norm": 1.5359814833702592, + "language_loss": 0.75259966, + "learning_rate": 2.8955352017009233e-06, + "loss": 0.77461457, + "num_input_tokens_seen": 66413475, + "router_z_loss_clip": 1.03759766, + "router_z_loss_mlp": 0.19494629, + "step": 3091, + "time_per_iteration": 2.855987310409546 + }, + { + "auxiliary_loss_clip": 0.01157089, + "auxiliary_loss_mlp": 0.01059594, + "balance_loss_clip": 1.051898, + "balance_loss_mlp": 1.03944802, + "epoch": 0.3717910178560693, + "flos": 22088653718400.0, + "grad_norm": 2.050906495566384, + "language_loss": 0.77545983, + "learning_rate": 2.8948386163145212e-06, + "loss": 0.79762667, + "num_input_tokens_seen": 66432685, + "router_z_loss_clip": 1.05322266, + "router_z_loss_mlp": 0.20129395, + "step": 3092, + "time_per_iteration": 2.812791585922241 + }, + { + "auxiliary_loss_clip": 0.01152133, + "auxiliary_loss_mlp": 0.01048273, + "balance_loss_clip": 1.0496881, + "balance_loss_mlp": 1.02999866, + "epoch": 0.3719112607467083, + "flos": 26939969533440.0, + "grad_norm": 1.7790118822577288, + "language_loss": 0.79442656, + "learning_rate": 2.8941418951818135e-06, + "loss": 0.81643057, + "num_input_tokens_seen": 66452245, + "router_z_loss_clip": 1.02587891, + "router_z_loss_mlp": 0.18286133, + "step": 3093, + "time_per_iteration": 2.8229596614837646 + }, + { + "auxiliary_loss_clip": 0.0115446, + "auxiliary_loss_mlp": 0.01056703, + "balance_loss_clip": 1.04938293, + "balance_loss_mlp": 1.03739142, + "epoch": 0.37203150363734744, + "flos": 12166500119040.0, + "grad_norm": 2.314333091528386, + "language_loss": 0.71131432, + "learning_rate": 2.8934450384084903e-06, + "loss": 0.73342597, + "num_input_tokens_seen": 66469760, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.19311523, + "step": 3094, + "time_per_iteration": 2.809091806411743 + }, + { + "auxiliary_loss_clip": 0.01159733, + "auxiliary_loss_mlp": 0.01055783, + "balance_loss_clip": 1.05121267, + "balance_loss_mlp": 1.03670967, + "epoch": 0.37215174652798655, + "flos": 23697595624320.0, + "grad_norm": 2.0049544495776526, + "language_loss": 0.69607317, + "learning_rate": 2.8927480461002653e-06, + "loss": 0.71822834, + "num_input_tokens_seen": 66489730, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.19067383, + "step": 3095, + "time_per_iteration": 2.8678197860717773 + }, + { + "auxiliary_loss_clip": 0.01159613, + "auxiliary_loss_mlp": 0.01057549, + "balance_loss_clip": 1.05086482, + "balance_loss_mlp": 1.03743863, + "epoch": 0.3722719894186256, + "flos": 17887751424000.0, + "grad_norm": 2.402523863598717, + "language_loss": 0.85968232, + "learning_rate": 2.892050918362872e-06, + "loss": 0.88185394, + "num_input_tokens_seen": 66504785, + "router_z_loss_clip": 1.08837891, + "router_z_loss_mlp": 0.20117188, + "step": 3096, + "time_per_iteration": 2.9046051502227783 + }, + { + "auxiliary_loss_clip": 0.01050288, + "auxiliary_loss_mlp": 0.01007822, + "balance_loss_clip": 1.01546144, + "balance_loss_mlp": 1.00565243, + "epoch": 0.3723922323092647, + "flos": 62419891363200.0, + "grad_norm": 0.850952970343582, + "language_loss": 0.5598371, + "learning_rate": 2.8913536553020626e-06, + "loss": 0.58041823, + "num_input_tokens_seen": 66558840, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.02172852, + "step": 3097, + "time_per_iteration": 3.3703842163085938 + }, + { + "auxiliary_loss_clip": 0.01156841, + "auxiliary_loss_mlp": 0.01054808, + "balance_loss_clip": 1.04957151, + "balance_loss_mlp": 1.03543639, + "epoch": 0.3725124751999038, + "flos": 23039747988480.0, + "grad_norm": 1.8630253774156804, + "language_loss": 0.85034382, + "learning_rate": 2.8906562570236137e-06, + "loss": 0.87246037, + "num_input_tokens_seen": 66576750, + "router_z_loss_clip": 1.07226562, + "router_z_loss_mlp": 0.19372559, + "step": 3098, + "time_per_iteration": 2.8820762634277344 + }, + { + "auxiliary_loss_clip": 0.01152615, + "auxiliary_loss_mlp": 0.01050213, + "balance_loss_clip": 1.049716, + "balance_loss_mlp": 1.03154469, + "epoch": 0.3726327180905429, + "flos": 20920551431040.0, + "grad_norm": 1.5351239319446497, + "language_loss": 0.7656287, + "learning_rate": 2.889958723633318e-06, + "loss": 0.78765696, + "num_input_tokens_seen": 66595690, + "router_z_loss_clip": 1.02832031, + "router_z_loss_mlp": 0.18664551, + "step": 3099, + "time_per_iteration": 2.8316094875335693 + }, + { + "auxiliary_loss_clip": 0.01155842, + "auxiliary_loss_mlp": 0.01052811, + "balance_loss_clip": 1.05010605, + "balance_loss_mlp": 1.0342859, + "epoch": 0.372752960981182, + "flos": 30592156688640.0, + "grad_norm": 1.5985493762810947, + "language_loss": 0.7447018, + "learning_rate": 2.889261055236992e-06, + "loss": 0.76678836, + "num_input_tokens_seen": 66617905, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.18530273, + "step": 3100, + "time_per_iteration": 3.076826333999634 + }, + { + "auxiliary_loss_clip": 0.01159149, + "auxiliary_loss_mlp": 0.01042994, + "balance_loss_clip": 1.05392981, + "balance_loss_mlp": 1.02449274, + "epoch": 0.3728732038718211, + "flos": 25116749043840.0, + "grad_norm": 1.982952040793673, + "language_loss": 0.82694077, + "learning_rate": 2.8885632519404704e-06, + "loss": 0.84896225, + "num_input_tokens_seen": 66638175, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.18505859, + "step": 3101, + "time_per_iteration": 2.9067163467407227 + }, + { + "auxiliary_loss_clip": 0.01155802, + "auxiliary_loss_mlp": 0.01043342, + "balance_loss_clip": 1.04914212, + "balance_loss_mlp": 1.02482891, + "epoch": 0.37299344676246016, + "flos": 25302048330240.0, + "grad_norm": 1.8267340342781133, + "language_loss": 0.75900578, + "learning_rate": 2.8878653138496107e-06, + "loss": 0.78099728, + "num_input_tokens_seen": 66658670, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.18518066, + "step": 3102, + "time_per_iteration": 2.8370134830474854 + }, + { + "auxiliary_loss_clip": 0.01157804, + "auxiliary_loss_mlp": 0.01049606, + "balance_loss_clip": 1.05168581, + "balance_loss_mlp": 1.02997196, + "epoch": 0.37311368965309927, + "flos": 23842531002240.0, + "grad_norm": 60.24265514433279, + "language_loss": 0.76745361, + "learning_rate": 2.8871672410702878e-06, + "loss": 0.78952771, + "num_input_tokens_seen": 66676030, + "router_z_loss_clip": 1.06152344, + "router_z_loss_mlp": 0.19641113, + "step": 3103, + "time_per_iteration": 2.8244192600250244 + }, + { + "auxiliary_loss_clip": 0.01159546, + "auxiliary_loss_mlp": 0.01042257, + "balance_loss_clip": 1.05005133, + "balance_loss_mlp": 1.022802, + "epoch": 0.3732339325437384, + "flos": 25811943845760.0, + "grad_norm": 1.7488722425955663, + "language_loss": 0.82552207, + "learning_rate": 2.8864690337084008e-06, + "loss": 0.84754008, + "num_input_tokens_seen": 66695305, + "router_z_loss_clip": 1.09521484, + "router_z_loss_mlp": 0.19445801, + "step": 3104, + "time_per_iteration": 3.669962167739868 + }, + { + "auxiliary_loss_clip": 0.01157193, + "auxiliary_loss_mlp": 0.01048337, + "balance_loss_clip": 1.05259895, + "balance_loss_mlp": 1.02931094, + "epoch": 0.37335417543437743, + "flos": 26208433146240.0, + "grad_norm": 1.7466352830898006, + "language_loss": 0.78254795, + "learning_rate": 2.885770691869866e-06, + "loss": 0.80460322, + "num_input_tokens_seen": 66716185, + "router_z_loss_clip": 1.04443359, + "router_z_loss_mlp": 0.19030762, + "step": 3105, + "time_per_iteration": 2.8463943004608154 + }, + { + "auxiliary_loss_clip": 0.01148941, + "auxiliary_loss_mlp": 0.01045136, + "balance_loss_clip": 1.04810417, + "balance_loss_mlp": 1.02726686, + "epoch": 0.37347441832501654, + "flos": 24023879792640.0, + "grad_norm": 2.4023085501794013, + "language_loss": 0.74940217, + "learning_rate": 2.8850722156606207e-06, + "loss": 0.77134293, + "num_input_tokens_seen": 66734575, + "router_z_loss_clip": 1.00830078, + "router_z_loss_mlp": 0.17871094, + "step": 3106, + "time_per_iteration": 2.801231861114502 + }, + { + "auxiliary_loss_clip": 0.01152344, + "auxiliary_loss_mlp": 0.0104578, + "balance_loss_clip": 1.04987347, + "balance_loss_mlp": 1.02713537, + "epoch": 0.3735946612156556, + "flos": 19714922409600.0, + "grad_norm": 1.8059585067631256, + "language_loss": 0.67265958, + "learning_rate": 2.8843736051866252e-06, + "loss": 0.69464082, + "num_input_tokens_seen": 66753500, + "router_z_loss_clip": 1.0234375, + "router_z_loss_mlp": 0.1862793, + "step": 3107, + "time_per_iteration": 3.655876398086548 + }, + { + "auxiliary_loss_clip": 0.01154732, + "auxiliary_loss_mlp": 0.01052067, + "balance_loss_clip": 1.04925275, + "balance_loss_mlp": 1.0335536, + "epoch": 0.3737149041062947, + "flos": 23039604334080.0, + "grad_norm": 1.8665287316940349, + "language_loss": 0.69434404, + "learning_rate": 2.8836748605538557e-06, + "loss": 0.71641207, + "num_input_tokens_seen": 66775140, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.18518066, + "step": 3108, + "time_per_iteration": 2.9389383792877197 + }, + { + "auxiliary_loss_clip": 0.01158105, + "auxiliary_loss_mlp": 0.0104102, + "balance_loss_clip": 1.05059361, + "balance_loss_mlp": 1.02233994, + "epoch": 0.3738351469969338, + "flos": 34678108483200.0, + "grad_norm": 2.322341716648102, + "language_loss": 0.63969839, + "learning_rate": 2.882975981868313e-06, + "loss": 0.66168964, + "num_input_tokens_seen": 66795525, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.18688965, + "step": 3109, + "time_per_iteration": 2.9730567932128906 + }, + { + "auxiliary_loss_clip": 0.01162413, + "auxiliary_loss_mlp": 0.01046275, + "balance_loss_clip": 1.05300283, + "balance_loss_mlp": 1.02661753, + "epoch": 0.3739553898875729, + "flos": 43507967448960.0, + "grad_norm": 1.9587476657854404, + "language_loss": 0.6862703, + "learning_rate": 2.882276969236016e-06, + "loss": 0.70835716, + "num_input_tokens_seen": 66816885, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.19665527, + "step": 3110, + "time_per_iteration": 3.8991549015045166 + }, + { + "auxiliary_loss_clip": 0.0115321, + "auxiliary_loss_mlp": 0.01047964, + "balance_loss_clip": 1.04889739, + "balance_loss_mlp": 1.02797294, + "epoch": 0.374075632778212, + "flos": 12856487448960.0, + "grad_norm": 1.9663005810542258, + "language_loss": 0.76800478, + "learning_rate": 2.881577822763005e-06, + "loss": 0.79001653, + "num_input_tokens_seen": 66834835, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.19995117, + "step": 3111, + "time_per_iteration": 3.6783525943756104 + }, + { + "auxiliary_loss_clip": 0.01156324, + "auxiliary_loss_mlp": 0.01043146, + "balance_loss_clip": 1.05081904, + "balance_loss_mlp": 1.02491879, + "epoch": 0.3741958756688511, + "flos": 26024031699840.0, + "grad_norm": 1.8130988631137968, + "language_loss": 0.87148082, + "learning_rate": 2.880878542555338e-06, + "loss": 0.89347559, + "num_input_tokens_seen": 66852600, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.18237305, + "step": 3112, + "time_per_iteration": 2.8300673961639404 + }, + { + "auxiliary_loss_clip": 0.01157846, + "auxiliary_loss_mlp": 0.01048091, + "balance_loss_clip": 1.05122316, + "balance_loss_mlp": 1.02750373, + "epoch": 0.37431611855949015, + "flos": 21433894652160.0, + "grad_norm": 2.0508904034842805, + "language_loss": 0.80590558, + "learning_rate": 2.8801791287190976e-06, + "loss": 0.82796496, + "num_input_tokens_seen": 66870595, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.20581055, + "step": 3113, + "time_per_iteration": 2.899945020675659 + }, + { + "auxiliary_loss_clip": 0.01158348, + "auxiliary_loss_mlp": 0.01043323, + "balance_loss_clip": 1.05055022, + "balance_loss_mlp": 1.02377295, + "epoch": 0.37443636145012926, + "flos": 24207096090240.0, + "grad_norm": 3.0114576473162233, + "language_loss": 0.86316574, + "learning_rate": 2.8794795813603817e-06, + "loss": 0.8851825, + "num_input_tokens_seen": 66886060, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.19555664, + "step": 3114, + "time_per_iteration": 2.865755796432495 + }, + { + "auxiliary_loss_clip": 0.01154841, + "auxiliary_loss_mlp": 0.01049667, + "balance_loss_clip": 1.0497694, + "balance_loss_mlp": 1.03029561, + "epoch": 0.3745566043407684, + "flos": 15378601841280.0, + "grad_norm": 1.7796427744048413, + "language_loss": 0.81846339, + "learning_rate": 2.878779900585314e-06, + "loss": 0.84050846, + "num_input_tokens_seen": 66903900, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.19372559, + "step": 3115, + "time_per_iteration": 2.8449079990386963 + }, + { + "auxiliary_loss_clip": 0.01152162, + "auxiliary_loss_mlp": 0.01044969, + "balance_loss_clip": 1.05021441, + "balance_loss_mlp": 1.02577686, + "epoch": 0.37467684723140743, + "flos": 24608218245120.0, + "grad_norm": 1.5003451818873774, + "language_loss": 0.75399613, + "learning_rate": 2.8780800865000336e-06, + "loss": 0.77596748, + "num_input_tokens_seen": 66925210, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.19189453, + "step": 3116, + "time_per_iteration": 2.8831584453582764 + }, + { + "auxiliary_loss_clip": 0.0105378, + "auxiliary_loss_mlp": 0.01001989, + "balance_loss_clip": 1.01893902, + "balance_loss_mlp": 0.99979585, + "epoch": 0.37479709012204654, + "flos": 64377491610240.0, + "grad_norm": 0.980821365808295, + "language_loss": 0.59236312, + "learning_rate": 2.877380139210702e-06, + "loss": 0.61292088, + "num_input_tokens_seen": 66983880, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.02197266, + "step": 3117, + "time_per_iteration": 3.3121399879455566 + }, + { + "auxiliary_loss_clip": 0.01159593, + "auxiliary_loss_mlp": 0.01043114, + "balance_loss_clip": 1.05025887, + "balance_loss_mlp": 1.02339721, + "epoch": 0.37491733301268565, + "flos": 23803962773760.0, + "grad_norm": 1.7790007900909466, + "language_loss": 0.76656228, + "learning_rate": 2.876680058823501e-06, + "loss": 0.78858936, + "num_input_tokens_seen": 67004280, + "router_z_loss_clip": 1.09277344, + "router_z_loss_mlp": 0.19702148, + "step": 3118, + "time_per_iteration": 2.9059932231903076 + }, + { + "auxiliary_loss_clip": 0.01158105, + "auxiliary_loss_mlp": 0.01043096, + "balance_loss_clip": 1.05013084, + "balance_loss_mlp": 1.02249646, + "epoch": 0.3750375759033247, + "flos": 32160950167680.0, + "grad_norm": 1.9472501544428427, + "language_loss": 0.6600492, + "learning_rate": 2.8759798454446314e-06, + "loss": 0.6820612, + "num_input_tokens_seen": 67027445, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.20581055, + "step": 3119, + "time_per_iteration": 2.9050204753875732 + }, + { + "auxiliary_loss_clip": 0.01154864, + "auxiliary_loss_mlp": 0.01061533, + "balance_loss_clip": 1.05144215, + "balance_loss_mlp": 1.04244781, + "epoch": 0.3751578187939638, + "flos": 23367791923200.0, + "grad_norm": 1.8185787107562008, + "language_loss": 0.81523615, + "learning_rate": 2.8752794991803173e-06, + "loss": 0.83740014, + "num_input_tokens_seen": 67045130, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.1907959, + "step": 3120, + "time_per_iteration": 2.906144142150879 + }, + { + "auxiliary_loss_clip": 0.01154751, + "auxiliary_loss_mlp": 0.0104875, + "balance_loss_clip": 1.0516516, + "balance_loss_mlp": 1.03113079, + "epoch": 0.37527806168460287, + "flos": 14605731878400.0, + "grad_norm": 2.024709936979224, + "language_loss": 0.75568974, + "learning_rate": 2.8745790201367976e-06, + "loss": 0.77772468, + "num_input_tokens_seen": 67060885, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.1763916, + "step": 3121, + "time_per_iteration": 2.898149251937866 + }, + { + "auxiliary_loss_clip": 0.01157041, + "auxiliary_loss_mlp": 0.01052604, + "balance_loss_clip": 1.04797435, + "balance_loss_mlp": 1.03375709, + "epoch": 0.375398304575242, + "flos": 26390823431040.0, + "grad_norm": 1.978032004401412, + "language_loss": 0.84737247, + "learning_rate": 2.8738784084203373e-06, + "loss": 0.86946893, + "num_input_tokens_seen": 67080960, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.18847656, + "step": 3122, + "time_per_iteration": 2.8654563426971436 + }, + { + "auxiliary_loss_clip": 0.01153737, + "auxiliary_loss_mlp": 0.01043034, + "balance_loss_clip": 1.05148983, + "balance_loss_mlp": 1.02585554, + "epoch": 0.3755185474658811, + "flos": 22236605838720.0, + "grad_norm": 1.7500329976791782, + "language_loss": 0.79262209, + "learning_rate": 2.873177664137216e-06, + "loss": 0.81458974, + "num_input_tokens_seen": 67101890, + "router_z_loss_clip": 1.02294922, + "router_z_loss_mlp": 0.17175293, + "step": 3123, + "time_per_iteration": 2.8239405155181885 + }, + { + "auxiliary_loss_clip": 0.01152501, + "auxiliary_loss_mlp": 0.01042837, + "balance_loss_clip": 1.0499289, + "balance_loss_mlp": 1.02435923, + "epoch": 0.37563879035652015, + "flos": 30812935633920.0, + "grad_norm": 1.7633421961722822, + "language_loss": 0.69668877, + "learning_rate": 2.8724767873937384e-06, + "loss": 0.71864212, + "num_input_tokens_seen": 67126010, + "router_z_loss_clip": 1.0234375, + "router_z_loss_mlp": 0.18493652, + "step": 3124, + "time_per_iteration": 2.8728809356689453 + }, + { + "auxiliary_loss_clip": 0.01155863, + "auxiliary_loss_mlp": 0.01050576, + "balance_loss_clip": 1.0518682, + "balance_loss_mlp": 1.03286123, + "epoch": 0.37575903324715926, + "flos": 20773533064320.0, + "grad_norm": 2.0566863982856183, + "language_loss": 0.87052393, + "learning_rate": 2.871775778296225e-06, + "loss": 0.89258826, + "num_input_tokens_seen": 67143100, + "router_z_loss_clip": 1.04101562, + "router_z_loss_mlp": 0.17736816, + "step": 3125, + "time_per_iteration": 2.801846981048584 + }, + { + "auxiliary_loss_clip": 0.01160124, + "auxiliary_loss_mlp": 0.01050472, + "balance_loss_clip": 1.05074048, + "balance_loss_mlp": 1.03040934, + "epoch": 0.37587927613779837, + "flos": 18697681244160.0, + "grad_norm": 2.196024810454186, + "language_loss": 0.78513002, + "learning_rate": 2.8710746369510196e-06, + "loss": 0.80723602, + "num_input_tokens_seen": 67161085, + "router_z_loss_clip": 1.09423828, + "router_z_loss_mlp": 0.20068359, + "step": 3126, + "time_per_iteration": 2.7875280380249023 + }, + { + "auxiliary_loss_clip": 0.01160174, + "auxiliary_loss_mlp": 0.010448, + "balance_loss_clip": 1.05320334, + "balance_loss_mlp": 1.02529764, + "epoch": 0.3759995190284374, + "flos": 13624796384640.0, + "grad_norm": 2.490203918353052, + "language_loss": 0.83469665, + "learning_rate": 2.8703733634644846e-06, + "loss": 0.85674632, + "num_input_tokens_seen": 67175840, + "router_z_loss_clip": 1.06884766, + "router_z_loss_mlp": 0.19506836, + "step": 3127, + "time_per_iteration": 2.7908935546875 + }, + { + "auxiliary_loss_clip": 0.01152581, + "auxiliary_loss_mlp": 0.01049215, + "balance_loss_clip": 1.05101466, + "balance_loss_mlp": 1.03173876, + "epoch": 0.37611976191907653, + "flos": 20484847457280.0, + "grad_norm": 2.0058934161193878, + "language_loss": 0.79840803, + "learning_rate": 2.869671957943002e-06, + "loss": 0.82042599, + "num_input_tokens_seen": 67194995, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.17480469, + "step": 3128, + "time_per_iteration": 2.8793752193450928 + }, + { + "auxiliary_loss_clip": 0.01155439, + "auxiliary_loss_mlp": 0.01052726, + "balance_loss_clip": 1.05189848, + "balance_loss_mlp": 1.03287804, + "epoch": 0.37624000480971564, + "flos": 21141797253120.0, + "grad_norm": 1.8484018581387722, + "language_loss": 0.74541974, + "learning_rate": 2.8689704204929747e-06, + "loss": 0.76750135, + "num_input_tokens_seen": 67214175, + "router_z_loss_clip": 1.03564453, + "router_z_loss_mlp": 0.19836426, + "step": 3129, + "time_per_iteration": 2.882272720336914 + }, + { + "auxiliary_loss_clip": 0.01153569, + "auxiliary_loss_mlp": 0.01050645, + "balance_loss_clip": 1.04950333, + "balance_loss_mlp": 1.03233504, + "epoch": 0.3763602477003547, + "flos": 22564470205440.0, + "grad_norm": 1.8466505897646688, + "language_loss": 0.81393296, + "learning_rate": 2.8682687512208253e-06, + "loss": 0.83597505, + "num_input_tokens_seen": 67233185, + "router_z_loss_clip": 1.03955078, + "router_z_loss_mlp": 0.18322754, + "step": 3130, + "time_per_iteration": 2.8578948974609375 + }, + { + "auxiliary_loss_clip": 0.01158484, + "auxiliary_loss_mlp": 0.01050897, + "balance_loss_clip": 1.04968321, + "balance_loss_mlp": 1.03137064, + "epoch": 0.3764804905909938, + "flos": 27526857851520.0, + "grad_norm": 1.9380923360104803, + "language_loss": 0.80481613, + "learning_rate": 2.8675669502329972e-06, + "loss": 0.8269099, + "num_input_tokens_seen": 67254715, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.1953125, + "step": 3131, + "time_per_iteration": 2.8824033737182617 + }, + { + "auxiliary_loss_clip": 0.0115623, + "auxiliary_loss_mlp": 0.01048488, + "balance_loss_clip": 1.05078661, + "balance_loss_mlp": 1.02816319, + "epoch": 0.3766007334816329, + "flos": 22528092706560.0, + "grad_norm": 2.661785911818193, + "language_loss": 0.86064464, + "learning_rate": 2.866865017635952e-06, + "loss": 0.8826918, + "num_input_tokens_seen": 67272535, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.20324707, + "step": 3132, + "time_per_iteration": 2.9146029949188232 + }, + { + "auxiliary_loss_clip": 0.0115842, + "auxiliary_loss_mlp": 0.01046512, + "balance_loss_clip": 1.05264044, + "balance_loss_mlp": 1.02646148, + "epoch": 0.376720976372272, + "flos": 25957166532480.0, + "grad_norm": 1.897207115929313, + "language_loss": 0.79474747, + "learning_rate": 2.866162953536174e-06, + "loss": 0.81679678, + "num_input_tokens_seen": 67293505, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.20056152, + "step": 3133, + "time_per_iteration": 3.7333192825317383 + }, + { + "auxiliary_loss_clip": 0.011557, + "auxiliary_loss_mlp": 0.01055403, + "balance_loss_clip": 1.05200052, + "balance_loss_mlp": 1.03628218, + "epoch": 0.3768412192629111, + "flos": 18041162411520.0, + "grad_norm": 1.8404636498891027, + "language_loss": 0.75056589, + "learning_rate": 2.8654607580401634e-06, + "loss": 0.77267689, + "num_input_tokens_seen": 67313240, + "router_z_loss_clip": 1.03613281, + "router_z_loss_mlp": 0.19128418, + "step": 3134, + "time_per_iteration": 2.7978968620300293 + }, + { + "auxiliary_loss_clip": 0.01054893, + "auxiliary_loss_mlp": 0.0100576, + "balance_loss_clip": 1.0202831, + "balance_loss_mlp": 1.00371003, + "epoch": 0.3769614621535502, + "flos": 62989472304000.0, + "grad_norm": 0.8888469101001498, + "language_loss": 0.65211344, + "learning_rate": 2.8647584312544446e-06, + "loss": 0.67271996, + "num_input_tokens_seen": 67378445, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.02050781, + "step": 3135, + "time_per_iteration": 3.3481855392456055 + }, + { + "auxiliary_loss_clip": 0.01152804, + "auxiliary_loss_mlp": 0.01050384, + "balance_loss_clip": 1.05033958, + "balance_loss_mlp": 1.03163218, + "epoch": 0.37708170504418925, + "flos": 23661685002240.0, + "grad_norm": 1.3892544533109692, + "language_loss": 0.85442919, + "learning_rate": 2.864055973285559e-06, + "loss": 0.87646103, + "num_input_tokens_seen": 67400445, + "router_z_loss_clip": 1.02539062, + "router_z_loss_mlp": 0.18762207, + "step": 3136, + "time_per_iteration": 3.6844003200531006 + }, + { + "auxiliary_loss_clip": 0.01147738, + "auxiliary_loss_mlp": 0.01051408, + "balance_loss_clip": 1.04963291, + "balance_loss_mlp": 1.03425407, + "epoch": 0.37720194793482836, + "flos": 24423170353920.0, + "grad_norm": 1.7654746900490503, + "language_loss": 0.86958683, + "learning_rate": 2.8633533842400698e-06, + "loss": 0.8915782, + "num_input_tokens_seen": 67420645, + "router_z_loss_clip": 0.97998047, + "router_z_loss_mlp": 0.17150879, + "step": 3137, + "time_per_iteration": 2.862570285797119 + }, + { + "auxiliary_loss_clip": 0.01163515, + "auxiliary_loss_mlp": 0.01052528, + "balance_loss_clip": 1.05463171, + "balance_loss_mlp": 1.03227437, + "epoch": 0.3773221908254674, + "flos": 20996502739200.0, + "grad_norm": 1.8468426970264682, + "language_loss": 0.77233028, + "learning_rate": 2.862650664224558e-06, + "loss": 0.79449069, + "num_input_tokens_seen": 67439495, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.20263672, + "step": 3138, + "time_per_iteration": 2.891730785369873 + }, + { + "auxiliary_loss_clip": 0.01156679, + "auxiliary_loss_mlp": 0.01038869, + "balance_loss_clip": 1.0539279, + "balance_loss_mlp": 1.02170289, + "epoch": 0.37744243371610653, + "flos": 37631724958080.0, + "grad_norm": 2.1539420253474866, + "language_loss": 0.6979084, + "learning_rate": 2.861947813345627e-06, + "loss": 0.71986389, + "num_input_tokens_seen": 67462195, + "router_z_loss_clip": 1.02636719, + "router_z_loss_mlp": 0.17150879, + "step": 3139, + "time_per_iteration": 4.57797646522522 + }, + { + "auxiliary_loss_clip": 0.01160916, + "auxiliary_loss_mlp": 0.01047277, + "balance_loss_clip": 1.05613112, + "balance_loss_mlp": 1.0280838, + "epoch": 0.37756267660674564, + "flos": 26140526484480.0, + "grad_norm": 2.1708581026384732, + "language_loss": 0.7267434, + "learning_rate": 2.8612448317098974e-06, + "loss": 0.74882531, + "num_input_tokens_seen": 67482530, + "router_z_loss_clip": 1.04833984, + "router_z_loss_mlp": 0.1920166, + "step": 3140, + "time_per_iteration": 2.8034231662750244 + }, + { + "auxiliary_loss_clip": 0.011568, + "auxiliary_loss_mlp": 0.01048476, + "balance_loss_clip": 1.05079484, + "balance_loss_mlp": 1.02857971, + "epoch": 0.3776829194973847, + "flos": 19427888828160.0, + "grad_norm": 2.0335080456841395, + "language_loss": 0.83306193, + "learning_rate": 2.8605417194240114e-06, + "loss": 0.8551147, + "num_input_tokens_seen": 67500890, + "router_z_loss_clip": 1.06005859, + "router_z_loss_mlp": 0.19897461, + "step": 3141, + "time_per_iteration": 2.844558000564575 + }, + { + "auxiliary_loss_clip": 0.01152784, + "auxiliary_loss_mlp": 0.01043176, + "balance_loss_clip": 1.0508436, + "balance_loss_mlp": 1.02518737, + "epoch": 0.3778031623880238, + "flos": 17382309194880.0, + "grad_norm": 1.8558665766784757, + "language_loss": 0.78927994, + "learning_rate": 2.8598384765946315e-06, + "loss": 0.81123948, + "num_input_tokens_seen": 67519545, + "router_z_loss_clip": 1.01806641, + "router_z_loss_mlp": 0.17980957, + "step": 3142, + "time_per_iteration": 2.8325085639953613 + }, + { + "auxiliary_loss_clip": 0.01154162, + "auxiliary_loss_mlp": 0.01043923, + "balance_loss_clip": 1.049016, + "balance_loss_mlp": 1.02560055, + "epoch": 0.3779234052786629, + "flos": 27125843437440.0, + "grad_norm": 1.8665368437585055, + "language_loss": 0.71691835, + "learning_rate": 2.8591351033284377e-06, + "loss": 0.73889923, + "num_input_tokens_seen": 67539275, + "router_z_loss_clip": 1.05224609, + "router_z_loss_mlp": 0.18310547, + "step": 3143, + "time_per_iteration": 3.0369908809661865 + }, + { + "auxiliary_loss_clip": 0.01159311, + "auxiliary_loss_mlp": 0.01049572, + "balance_loss_clip": 1.05494165, + "balance_loss_mlp": 1.03087997, + "epoch": 0.37804364816930197, + "flos": 19682639061120.0, + "grad_norm": 2.3572344451722906, + "language_loss": 0.83579981, + "learning_rate": 2.8584315997321325e-06, + "loss": 0.8578887, + "num_input_tokens_seen": 67558280, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.18701172, + "step": 3144, + "time_per_iteration": 2.926461935043335 + }, + { + "auxiliary_loss_clip": 0.01156441, + "auxiliary_loss_mlp": 0.01040255, + "balance_loss_clip": 1.05253601, + "balance_loss_mlp": 1.02134824, + "epoch": 0.3781638910599411, + "flos": 22702905221760.0, + "grad_norm": 2.378318830601884, + "language_loss": 0.77593386, + "learning_rate": 2.8577279659124356e-06, + "loss": 0.79790086, + "num_input_tokens_seen": 67575955, + "router_z_loss_clip": 1.03808594, + "router_z_loss_mlp": 0.18908691, + "step": 3145, + "time_per_iteration": 2.7977542877197266 + }, + { + "auxiliary_loss_clip": 0.0115204, + "auxiliary_loss_mlp": 0.01040707, + "balance_loss_clip": 1.05163622, + "balance_loss_mlp": 1.02240801, + "epoch": 0.3782841339505802, + "flos": 14647604158080.0, + "grad_norm": 1.9087260776741448, + "language_loss": 0.83681095, + "learning_rate": 2.857024201976089e-06, + "loss": 0.85873842, + "num_input_tokens_seen": 67593515, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.1829834, + "step": 3146, + "time_per_iteration": 2.7749853134155273 + }, + { + "auxiliary_loss_clip": 0.01160028, + "auxiliary_loss_mlp": 0.01043565, + "balance_loss_clip": 1.05202317, + "balance_loss_mlp": 1.02325153, + "epoch": 0.37840437684121925, + "flos": 32818223185920.0, + "grad_norm": 2.2883983500611516, + "language_loss": 0.73513299, + "learning_rate": 2.8563203080298516e-06, + "loss": 0.75716889, + "num_input_tokens_seen": 67614290, + "router_z_loss_clip": 1.08154297, + "router_z_loss_mlp": 0.203125, + "step": 3147, + "time_per_iteration": 2.8467049598693848 + }, + { + "auxiliary_loss_clip": 0.01160327, + "auxiliary_loss_mlp": 0.01039745, + "balance_loss_clip": 1.05289125, + "balance_loss_mlp": 1.02110124, + "epoch": 0.37852461973185836, + "flos": 18369206346240.0, + "grad_norm": 2.111313428873068, + "language_loss": 0.89827955, + "learning_rate": 2.855616284180505e-06, + "loss": 0.92028034, + "num_input_tokens_seen": 67631340, + "router_z_loss_clip": 1.07470703, + "router_z_loss_mlp": 0.18652344, + "step": 3148, + "time_per_iteration": 2.903573513031006 + }, + { + "auxiliary_loss_clip": 0.0105591, + "auxiliary_loss_mlp": 0.01000714, + "balance_loss_clip": 1.02154756, + "balance_loss_mlp": 0.99848455, + "epoch": 0.37864486262249747, + "flos": 59500680117120.0, + "grad_norm": 0.8762053688962728, + "language_loss": 0.66161871, + "learning_rate": 2.8549121305348477e-06, + "loss": 0.68218493, + "num_input_tokens_seen": 67691125, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.02233887, + "step": 3149, + "time_per_iteration": 3.318030834197998 + }, + { + "auxiliary_loss_clip": 0.01153867, + "auxiliary_loss_mlp": 0.01043438, + "balance_loss_clip": 1.05244529, + "balance_loss_mlp": 1.02634323, + "epoch": 0.3787651055131365, + "flos": 23363015414400.0, + "grad_norm": 2.263784577198049, + "language_loss": 0.83512378, + "learning_rate": 2.8542078471997006e-06, + "loss": 0.85709679, + "num_input_tokens_seen": 67708740, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.17089844, + "step": 3150, + "time_per_iteration": 2.8625197410583496 + }, + { + "auxiliary_loss_clip": 0.01159648, + "auxiliary_loss_mlp": 0.01045082, + "balance_loss_clip": 1.05562186, + "balance_loss_mlp": 1.02733135, + "epoch": 0.37888534840377563, + "flos": 24601394661120.0, + "grad_norm": 1.7672842893375622, + "language_loss": 0.75551116, + "learning_rate": 2.8535034342819013e-06, + "loss": 0.77755845, + "num_input_tokens_seen": 67726150, + "router_z_loss_clip": 1.03955078, + "router_z_loss_mlp": 0.17736816, + "step": 3151, + "time_per_iteration": 2.8258004188537598 + }, + { + "auxiliary_loss_clip": 0.01153239, + "auxiliary_loss_mlp": 0.01049487, + "balance_loss_clip": 1.05205727, + "balance_loss_mlp": 1.03098619, + "epoch": 0.37900559129441475, + "flos": 23986891762560.0, + "grad_norm": 1.5392612902472638, + "language_loss": 0.7293812, + "learning_rate": 2.85279889188831e-06, + "loss": 0.75140846, + "num_input_tokens_seen": 67746525, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.18493652, + "step": 3152, + "time_per_iteration": 2.8288567066192627 + }, + { + "auxiliary_loss_clip": 0.011574, + "auxiliary_loss_mlp": 0.01057047, + "balance_loss_clip": 1.05150878, + "balance_loss_mlp": 1.03812838, + "epoch": 0.3791258341850538, + "flos": 24644667571200.0, + "grad_norm": 1.7379889071003611, + "language_loss": 0.81171858, + "learning_rate": 2.852094220125805e-06, + "loss": 0.83386302, + "num_input_tokens_seen": 67766035, + "router_z_loss_clip": 1.05810547, + "router_z_loss_mlp": 0.18920898, + "step": 3153, + "time_per_iteration": 2.85214900970459 + }, + { + "auxiliary_loss_clip": 0.01160305, + "auxiliary_loss_mlp": 0.01053326, + "balance_loss_clip": 1.05560851, + "balance_loss_mlp": 1.0337522, + "epoch": 0.3792460770756929, + "flos": 17420841509760.0, + "grad_norm": 2.264925654559665, + "language_loss": 0.71070713, + "learning_rate": 2.8513894191012846e-06, + "loss": 0.73284346, + "num_input_tokens_seen": 67785015, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.19567871, + "step": 3154, + "time_per_iteration": 2.8103857040405273 + }, + { + "auxiliary_loss_clip": 0.01157109, + "auxiliary_loss_mlp": 0.01042976, + "balance_loss_clip": 1.05159354, + "balance_loss_mlp": 1.02535653, + "epoch": 0.37936631996633197, + "flos": 24206557386240.0, + "grad_norm": 1.802274432725097, + "language_loss": 0.78861868, + "learning_rate": 2.8506844889216664e-06, + "loss": 0.81061947, + "num_input_tokens_seen": 67804400, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.17626953, + "step": 3155, + "time_per_iteration": 2.8625247478485107 + }, + { + "auxiliary_loss_clip": 0.01053495, + "auxiliary_loss_mlp": 0.01005259, + "balance_loss_clip": 1.01893115, + "balance_loss_mlp": 1.00316131, + "epoch": 0.3794865628569711, + "flos": 70297114752000.0, + "grad_norm": 0.8658613368500028, + "language_loss": 0.62899613, + "learning_rate": 2.849979429693887e-06, + "loss": 0.6495837, + "num_input_tokens_seen": 67865385, + "router_z_loss_clip": 0.34570312, + "router_z_loss_mlp": 0.02099609, + "step": 3156, + "time_per_iteration": 3.4722836017608643 + }, + { + "auxiliary_loss_clip": 0.01156136, + "auxiliary_loss_mlp": 0.01049642, + "balance_loss_clip": 1.05247927, + "balance_loss_mlp": 1.02914953, + "epoch": 0.3796068057476102, + "flos": 15779364860160.0, + "grad_norm": 2.0110454690248276, + "language_loss": 0.74207985, + "learning_rate": 2.8492742415249042e-06, + "loss": 0.76413763, + "num_input_tokens_seen": 67883030, + "router_z_loss_clip": 1.03710938, + "router_z_loss_mlp": 0.20507812, + "step": 3157, + "time_per_iteration": 2.8785758018493652 + }, + { + "auxiliary_loss_clip": 0.01157582, + "auxiliary_loss_mlp": 0.01047888, + "balance_loss_clip": 1.05236197, + "balance_loss_mlp": 1.02906466, + "epoch": 0.37972704863824924, + "flos": 25191694771200.0, + "grad_norm": 1.6868811368179322, + "language_loss": 0.76764286, + "learning_rate": 2.848568924521694e-06, + "loss": 0.78969753, + "num_input_tokens_seen": 67903810, + "router_z_loss_clip": 1.05029297, + "router_z_loss_mlp": 0.18823242, + "step": 3158, + "time_per_iteration": 2.9034276008605957 + }, + { + "auxiliary_loss_clip": 0.01155089, + "auxiliary_loss_mlp": 0.01058219, + "balance_loss_clip": 1.05065656, + "balance_loss_mlp": 1.03911018, + "epoch": 0.37984729152888835, + "flos": 26210372480640.0, + "grad_norm": 1.862292340230467, + "language_loss": 0.73730576, + "learning_rate": 2.8478634787912526e-06, + "loss": 0.75943881, + "num_input_tokens_seen": 67921865, + "router_z_loss_clip": 1.04394531, + "router_z_loss_mlp": 0.19104004, + "step": 3159, + "time_per_iteration": 2.9383294582366943 + }, + { + "auxiliary_loss_clip": 0.01155237, + "auxiliary_loss_mlp": 0.01054416, + "balance_loss_clip": 1.05157065, + "balance_loss_mlp": 1.03466272, + "epoch": 0.37996753441952746, + "flos": 25629302165760.0, + "grad_norm": 2.889206819634193, + "language_loss": 0.76867497, + "learning_rate": 2.847157904440596e-06, + "loss": 0.79077148, + "num_input_tokens_seen": 67941595, + "router_z_loss_clip": 1.03515625, + "router_z_loss_mlp": 0.19750977, + "step": 3160, + "time_per_iteration": 2.8296713829040527 + }, + { + "auxiliary_loss_clip": 0.01155012, + "auxiliary_loss_mlp": 0.01058056, + "balance_loss_clip": 1.05269408, + "balance_loss_mlp": 1.039626, + "epoch": 0.3800877773101665, + "flos": 20118414862080.0, + "grad_norm": 1.5162258729559126, + "language_loss": 0.73803473, + "learning_rate": 2.846452201576759e-06, + "loss": 0.76016545, + "num_input_tokens_seen": 67960970, + "router_z_loss_clip": 1.02246094, + "router_z_loss_mlp": 0.1842041, + "step": 3161, + "time_per_iteration": 2.7951159477233887 + }, + { + "auxiliary_loss_clip": 0.01051297, + "auxiliary_loss_mlp": 0.01002389, + "balance_loss_clip": 1.0169884, + "balance_loss_mlp": 1.00032651, + "epoch": 0.38020802020080563, + "flos": 63053608037760.0, + "grad_norm": 0.8528376187841257, + "language_loss": 0.62757462, + "learning_rate": 2.845746370306795e-06, + "loss": 0.64811146, + "num_input_tokens_seen": 68026160, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.02062988, + "step": 3162, + "time_per_iteration": 4.2411041259765625 + }, + { + "auxiliary_loss_clip": 0.01156483, + "auxiliary_loss_mlp": 0.0105147, + "balance_loss_clip": 1.0517323, + "balance_loss_mlp": 1.0317055, + "epoch": 0.38032826309144474, + "flos": 21288420570240.0, + "grad_norm": 2.1328380700083938, + "language_loss": 0.78948343, + "learning_rate": 2.84504041073778e-06, + "loss": 0.81156301, + "num_input_tokens_seen": 68044575, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.19750977, + "step": 3163, + "time_per_iteration": 2.800259828567505 + }, + { + "auxiliary_loss_clip": 0.01154914, + "auxiliary_loss_mlp": 0.0106264, + "balance_loss_clip": 1.05136359, + "balance_loss_mlp": 1.04333973, + "epoch": 0.3804485059820838, + "flos": 18954119416320.0, + "grad_norm": 1.9633222794196086, + "language_loss": 0.79735196, + "learning_rate": 2.844334322976806e-06, + "loss": 0.81952751, + "num_input_tokens_seen": 68064790, + "router_z_loss_clip": 1.03466797, + "router_z_loss_mlp": 0.19274902, + "step": 3164, + "time_per_iteration": 2.817784547805786 + }, + { + "auxiliary_loss_clip": 0.0116172, + "auxiliary_loss_mlp": 0.01059221, + "balance_loss_clip": 1.05060196, + "balance_loss_mlp": 1.03813267, + "epoch": 0.3805687488727229, + "flos": 21833759831040.0, + "grad_norm": 2.023209760522217, + "language_loss": 0.83363903, + "learning_rate": 2.8436281071309866e-06, + "loss": 0.85584837, + "num_input_tokens_seen": 68083330, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.21081543, + "step": 3165, + "time_per_iteration": 3.6417524814605713 + }, + { + "auxiliary_loss_clip": 0.01049484, + "auxiliary_loss_mlp": 0.01004086, + "balance_loss_clip": 1.01531696, + "balance_loss_mlp": 1.0018326, + "epoch": 0.380688991763362, + "flos": 58546209968640.0, + "grad_norm": 0.7270685012804989, + "language_loss": 0.53025293, + "learning_rate": 2.842921763307455e-06, + "loss": 0.55078864, + "num_input_tokens_seen": 68146140, + "router_z_loss_clip": 0.34179688, + "router_z_loss_mlp": 0.02258301, + "step": 3166, + "time_per_iteration": 3.2513694763183594 + }, + { + "auxiliary_loss_clip": 0.01158844, + "auxiliary_loss_mlp": 0.01052872, + "balance_loss_clip": 1.05540419, + "balance_loss_mlp": 1.03458548, + "epoch": 0.38080923465400107, + "flos": 23799509487360.0, + "grad_norm": 1.8037214769762755, + "language_loss": 0.82697499, + "learning_rate": 2.842215291613361e-06, + "loss": 0.84909219, + "num_input_tokens_seen": 68164520, + "router_z_loss_clip": 1.03515625, + "router_z_loss_mlp": 0.1829834, + "step": 3167, + "time_per_iteration": 2.8932836055755615 + }, + { + "auxiliary_loss_clip": 0.01048929, + "auxiliary_loss_mlp": 0.01003363, + "balance_loss_clip": 1.01462281, + "balance_loss_mlp": 1.00135982, + "epoch": 0.3809294775446402, + "flos": 54969866380800.0, + "grad_norm": 0.828390341851758, + "language_loss": 0.59237492, + "learning_rate": 2.8415086921558774e-06, + "loss": 0.61289787, + "num_input_tokens_seen": 68227945, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.02001953, + "step": 3168, + "time_per_iteration": 5.012714862823486 + }, + { + "auxiliary_loss_clip": 0.01149993, + "auxiliary_loss_mlp": 0.01044431, + "balance_loss_clip": 1.04628015, + "balance_loss_mlp": 1.02577448, + "epoch": 0.38104972043527924, + "flos": 24643697904000.0, + "grad_norm": 1.5686786123161964, + "language_loss": 0.78853303, + "learning_rate": 2.840801965042194e-06, + "loss": 0.81047726, + "num_input_tokens_seen": 68247405, + "router_z_loss_clip": 1.03808594, + "router_z_loss_mlp": 0.18664551, + "step": 3169, + "time_per_iteration": 2.835089921951294 + }, + { + "auxiliary_loss_clip": 0.01154095, + "auxiliary_loss_mlp": 0.01045042, + "balance_loss_clip": 1.05066383, + "balance_loss_mlp": 1.02568293, + "epoch": 0.38116996332591835, + "flos": 22856783086080.0, + "grad_norm": 1.7888951240415105, + "language_loss": 0.83833212, + "learning_rate": 2.840095110379521e-06, + "loss": 0.86032349, + "num_input_tokens_seen": 68266925, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.19360352, + "step": 3170, + "time_per_iteration": 2.844447374343872 + }, + { + "auxiliary_loss_clip": 0.01047016, + "auxiliary_loss_mlp": 0.01001677, + "balance_loss_clip": 1.01248026, + "balance_loss_mlp": 0.99963874, + "epoch": 0.38129020621655746, + "flos": 60836160804480.0, + "grad_norm": 0.7286883522066622, + "language_loss": 0.53946674, + "learning_rate": 2.8393881282750884e-06, + "loss": 0.55995369, + "num_input_tokens_seen": 68329755, + "router_z_loss_clip": 0.34570312, + "router_z_loss_mlp": 0.02038574, + "step": 3171, + "time_per_iteration": 3.3506717681884766 + }, + { + "auxiliary_loss_clip": 0.01155776, + "auxiliary_loss_mlp": 0.01047118, + "balance_loss_clip": 1.0505023, + "balance_loss_mlp": 1.0270189, + "epoch": 0.3814104491071965, + "flos": 21648101408640.0, + "grad_norm": 1.8233471764379232, + "language_loss": 0.79140711, + "learning_rate": 2.838681018836144e-06, + "loss": 0.81343603, + "num_input_tokens_seen": 68347075, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.20117188, + "step": 3172, + "time_per_iteration": 2.862185478210449 + }, + { + "auxiliary_loss_clip": 0.01154288, + "auxiliary_loss_mlp": 0.0104708, + "balance_loss_clip": 1.04918408, + "balance_loss_mlp": 1.02913916, + "epoch": 0.3815306919978356, + "flos": 19099090707840.0, + "grad_norm": 2.1833390995857274, + "language_loss": 0.78226399, + "learning_rate": 2.837973782169955e-06, + "loss": 0.80427772, + "num_input_tokens_seen": 68365450, + "router_z_loss_clip": 1.05273438, + "router_z_loss_mlp": 0.17944336, + "step": 3173, + "time_per_iteration": 2.870602607727051 + }, + { + "auxiliary_loss_clip": 0.0104502, + "auxiliary_loss_mlp": 0.01006077, + "balance_loss_clip": 1.01090717, + "balance_loss_mlp": 1.00401437, + "epoch": 0.38165093488847474, + "flos": 67067918156160.0, + "grad_norm": 0.8050586768623741, + "language_loss": 0.59146011, + "learning_rate": 2.8372664183838096e-06, + "loss": 0.61197102, + "num_input_tokens_seen": 68428470, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.02062988, + "step": 3174, + "time_per_iteration": 3.397462844848633 + }, + { + "auxiliary_loss_clip": 0.01153067, + "auxiliary_loss_mlp": 0.01057504, + "balance_loss_clip": 1.04962277, + "balance_loss_mlp": 1.03914571, + "epoch": 0.3817711777791138, + "flos": 22341105480960.0, + "grad_norm": 3.1864205746756182, + "language_loss": 0.68396121, + "learning_rate": 2.836558927585015e-06, + "loss": 0.70606691, + "num_input_tokens_seen": 68445440, + "router_z_loss_clip": 1.03515625, + "router_z_loss_mlp": 0.18371582, + "step": 3175, + "time_per_iteration": 2.885432720184326 + }, + { + "auxiliary_loss_clip": 0.01157676, + "auxiliary_loss_mlp": 0.01046425, + "balance_loss_clip": 1.05206776, + "balance_loss_mlp": 1.02761436, + "epoch": 0.3818914206697529, + "flos": 22820621068800.0, + "grad_norm": 1.8174085046948465, + "language_loss": 0.82572246, + "learning_rate": 2.8358513098808957e-06, + "loss": 0.84776348, + "num_input_tokens_seen": 68465755, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.18823242, + "step": 3176, + "time_per_iteration": 2.8363723754882812 + }, + { + "auxiliary_loss_clip": 0.01158767, + "auxiliary_loss_mlp": 0.01053701, + "balance_loss_clip": 1.05422425, + "balance_loss_mlp": 1.03223169, + "epoch": 0.382011663560392, + "flos": 24386074583040.0, + "grad_norm": 1.9445479953907834, + "language_loss": 0.76738757, + "learning_rate": 2.835143565378798e-06, + "loss": 0.78951228, + "num_input_tokens_seen": 68486220, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.21459961, + "step": 3177, + "time_per_iteration": 2.8811471462249756 + }, + { + "auxiliary_loss_clip": 0.01154957, + "auxiliary_loss_mlp": 0.01052727, + "balance_loss_clip": 1.0494262, + "balance_loss_mlp": 1.03243732, + "epoch": 0.38213190645103107, + "flos": 21981568296960.0, + "grad_norm": 2.3350524215631543, + "language_loss": 0.78447795, + "learning_rate": 2.8344356941860847e-06, + "loss": 0.80655473, + "num_input_tokens_seen": 68505850, + "router_z_loss_clip": 1.05566406, + "router_z_loss_mlp": 0.20288086, + "step": 3178, + "time_per_iteration": 2.9015090465545654 + }, + { + "auxiliary_loss_clip": 0.01154085, + "auxiliary_loss_mlp": 0.01053947, + "balance_loss_clip": 1.05197978, + "balance_loss_mlp": 1.03433669, + "epoch": 0.3822521493416702, + "flos": 35516945773440.0, + "grad_norm": 2.0217535535495017, + "language_loss": 0.66762292, + "learning_rate": 2.8337276964101403e-06, + "loss": 0.68970329, + "num_input_tokens_seen": 68526290, + "router_z_loss_clip": 1.02148438, + "router_z_loss_mlp": 0.19604492, + "step": 3179, + "time_per_iteration": 2.967412233352661 + }, + { + "auxiliary_loss_clip": 0.01157922, + "auxiliary_loss_mlp": 0.01044298, + "balance_loss_clip": 1.0522666, + "balance_loss_mlp": 1.02406824, + "epoch": 0.3823723922323093, + "flos": 21069904181760.0, + "grad_norm": 1.9167503604939242, + "language_loss": 0.7653324, + "learning_rate": 2.833019572158367e-06, + "loss": 0.78735459, + "num_input_tokens_seen": 68544725, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.20227051, + "step": 3180, + "time_per_iteration": 2.8775084018707275 + }, + { + "auxiliary_loss_clip": 0.01152669, + "auxiliary_loss_mlp": 0.01049203, + "balance_loss_clip": 1.04871869, + "balance_loss_mlp": 1.03083336, + "epoch": 0.38249263512294834, + "flos": 19789149864960.0, + "grad_norm": 1.78982571077818, + "language_loss": 0.80290222, + "learning_rate": 2.8323113215381872e-06, + "loss": 0.82492089, + "num_input_tokens_seen": 68563070, + "router_z_loss_clip": 1.04052734, + "router_z_loss_mlp": 0.18383789, + "step": 3181, + "time_per_iteration": 2.8211913108825684 + }, + { + "auxiliary_loss_clip": 0.01158695, + "auxiliary_loss_mlp": 0.01057751, + "balance_loss_clip": 1.0493567, + "balance_loss_mlp": 1.03662777, + "epoch": 0.38261287801358745, + "flos": 21433930565760.0, + "grad_norm": 1.9376119876324824, + "language_loss": 0.76279253, + "learning_rate": 2.831602944657042e-06, + "loss": 0.78495699, + "num_input_tokens_seen": 68581150, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.21130371, + "step": 3182, + "time_per_iteration": 2.8085031509399414 + }, + { + "auxiliary_loss_clip": 0.01155064, + "auxiliary_loss_mlp": 0.01050259, + "balance_loss_clip": 1.04968476, + "balance_loss_mlp": 1.03063667, + "epoch": 0.38273312090422656, + "flos": 21981568296960.0, + "grad_norm": 2.666871261204332, + "language_loss": 0.7459476, + "learning_rate": 2.830894441622391e-06, + "loss": 0.76800078, + "num_input_tokens_seen": 68597800, + "router_z_loss_clip": 1.05273438, + "router_z_loss_mlp": 0.19616699, + "step": 3183, + "time_per_iteration": 2.812415599822998 + }, + { + "auxiliary_loss_clip": 0.01161639, + "auxiliary_loss_mlp": 0.01050706, + "balance_loss_clip": 1.05186272, + "balance_loss_mlp": 1.0312866, + "epoch": 0.3828533637948656, + "flos": 24790895838720.0, + "grad_norm": 1.8175098449092089, + "language_loss": 0.80431342, + "learning_rate": 2.8301858125417134e-06, + "loss": 0.82643688, + "num_input_tokens_seen": 68617640, + "router_z_loss_clip": 1.09716797, + "router_z_loss_mlp": 0.1940918, + "step": 3184, + "time_per_iteration": 2.875023126602173 + }, + { + "auxiliary_loss_clip": 0.01155022, + "auxiliary_loss_mlp": 0.01043268, + "balance_loss_clip": 1.0522759, + "balance_loss_mlp": 1.02501678, + "epoch": 0.38297360668550473, + "flos": 22455445449600.0, + "grad_norm": 1.939359058769351, + "language_loss": 0.74151087, + "learning_rate": 2.8294770575225082e-06, + "loss": 0.76349378, + "num_input_tokens_seen": 68637770, + "router_z_loss_clip": 1.02832031, + "router_z_loss_mlp": 0.18249512, + "step": 3185, + "time_per_iteration": 2.8330180644989014 + }, + { + "auxiliary_loss_clip": 0.01158196, + "auxiliary_loss_mlp": 0.01050444, + "balance_loss_clip": 1.05032587, + "balance_loss_mlp": 1.03035712, + "epoch": 0.3830938495761438, + "flos": 24896903852160.0, + "grad_norm": 1.8033619504776421, + "language_loss": 0.84069598, + "learning_rate": 2.828768176672293e-06, + "loss": 0.86278236, + "num_input_tokens_seen": 68656885, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.2010498, + "step": 3186, + "time_per_iteration": 2.976266622543335 + }, + { + "auxiliary_loss_clip": 0.01160725, + "auxiliary_loss_mlp": 0.01046581, + "balance_loss_clip": 1.05131054, + "balance_loss_mlp": 1.02580285, + "epoch": 0.3832140924667829, + "flos": 33036236784000.0, + "grad_norm": 1.7781985651038317, + "language_loss": 0.72173244, + "learning_rate": 2.8280591700986044e-06, + "loss": 0.74380553, + "num_input_tokens_seen": 68678750, + "router_z_loss_clip": 1.09179688, + "router_z_loss_mlp": 0.20788574, + "step": 3187, + "time_per_iteration": 3.110980272293091 + }, + { + "auxiliary_loss_clip": 0.01152652, + "auxiliary_loss_mlp": 0.01044195, + "balance_loss_clip": 1.04744947, + "balance_loss_mlp": 1.02410793, + "epoch": 0.383334335357422, + "flos": 31903721896320.0, + "grad_norm": 2.1136403920532363, + "language_loss": 0.75116032, + "learning_rate": 2.827350037908999e-06, + "loss": 0.77312875, + "num_input_tokens_seen": 68698190, + "router_z_loss_clip": 1.05126953, + "router_z_loss_mlp": 0.20092773, + "step": 3188, + "time_per_iteration": 2.913052558898926 + }, + { + "auxiliary_loss_clip": 0.01154383, + "auxiliary_loss_mlp": 0.01049515, + "balance_loss_clip": 1.04833806, + "balance_loss_mlp": 1.02957118, + "epoch": 0.38345457824806106, + "flos": 19791915212160.0, + "grad_norm": 2.2786948646033167, + "language_loss": 0.79329914, + "learning_rate": 2.8266407802110496e-06, + "loss": 0.81533813, + "num_input_tokens_seen": 68716445, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.19934082, + "step": 3189, + "time_per_iteration": 2.899223804473877 + }, + { + "auxiliary_loss_clip": 0.01160608, + "auxiliary_loss_mlp": 0.01046538, + "balance_loss_clip": 1.05130315, + "balance_loss_mlp": 1.02567637, + "epoch": 0.3835748211387002, + "flos": 22419391173120.0, + "grad_norm": 1.7931373372166792, + "language_loss": 0.75850791, + "learning_rate": 2.8259313971123515e-06, + "loss": 0.78057933, + "num_input_tokens_seen": 68737565, + "router_z_loss_clip": 1.09277344, + "router_z_loss_mlp": 0.20861816, + "step": 3190, + "time_per_iteration": 2.8741958141326904 + }, + { + "auxiliary_loss_clip": 0.01151663, + "auxiliary_loss_mlp": 0.01044022, + "balance_loss_clip": 1.05118585, + "balance_loss_mlp": 1.02726114, + "epoch": 0.3836950640293393, + "flos": 25118436983040.0, + "grad_norm": 1.561527691214058, + "language_loss": 0.7876097, + "learning_rate": 2.8252218887205166e-06, + "loss": 0.80956656, + "num_input_tokens_seen": 68758255, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.16760254, + "step": 3191, + "time_per_iteration": 3.7551445960998535 + }, + { + "auxiliary_loss_clip": 0.01155964, + "auxiliary_loss_mlp": 0.01057692, + "balance_loss_clip": 1.0508498, + "balance_loss_mlp": 1.03830874, + "epoch": 0.38381530691997834, + "flos": 21799213925760.0, + "grad_norm": 1.7985999986671564, + "language_loss": 0.80970061, + "learning_rate": 2.824512255143178e-06, + "loss": 0.83183724, + "num_input_tokens_seen": 68777490, + "router_z_loss_clip": 1.05029297, + "router_z_loss_mlp": 0.19372559, + "step": 3192, + "time_per_iteration": 2.8562002182006836 + }, + { + "auxiliary_loss_clip": 0.01155329, + "auxiliary_loss_mlp": 0.01044448, + "balance_loss_clip": 1.05062604, + "balance_loss_mlp": 1.02601826, + "epoch": 0.38393554981061745, + "flos": 21252689516160.0, + "grad_norm": 4.496623282214337, + "language_loss": 0.79189765, + "learning_rate": 2.8238024964879855e-06, + "loss": 0.81389534, + "num_input_tokens_seen": 68798385, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.1842041, + "step": 3193, + "time_per_iteration": 3.6879334449768066 + }, + { + "auxiliary_loss_clip": 0.01160566, + "auxiliary_loss_mlp": 0.01045718, + "balance_loss_clip": 1.05146968, + "balance_loss_mlp": 1.02563119, + "epoch": 0.38405579270125656, + "flos": 17019360218880.0, + "grad_norm": 3.32732241062708, + "language_loss": 0.77006435, + "learning_rate": 2.8230926128626095e-06, + "loss": 0.79212719, + "num_input_tokens_seen": 68816880, + "router_z_loss_clip": 1.08984375, + "router_z_loss_mlp": 0.20080566, + "step": 3194, + "time_per_iteration": 2.87721586227417 + }, + { + "auxiliary_loss_clip": 0.0115651, + "auxiliary_loss_mlp": 0.01042453, + "balance_loss_clip": 1.05093884, + "balance_loss_mlp": 1.0235939, + "epoch": 0.3841760355918956, + "flos": 21835375943040.0, + "grad_norm": 4.4654179789583734, + "language_loss": 0.79276848, + "learning_rate": 2.822382604374738e-06, + "loss": 0.81475812, + "num_input_tokens_seen": 68835805, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.1887207, + "step": 3195, + "time_per_iteration": 2.8451898097991943 + }, + { + "auxiliary_loss_clip": 0.01154008, + "auxiliary_loss_mlp": 0.0105842, + "balance_loss_clip": 1.05054367, + "balance_loss_mlp": 1.03792775, + "epoch": 0.3842962784825347, + "flos": 25915114684800.0, + "grad_norm": 2.2224847309476146, + "language_loss": 0.6588639, + "learning_rate": 2.8216724711320793e-06, + "loss": 0.68098819, + "num_input_tokens_seen": 68854930, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.20483398, + "step": 3196, + "time_per_iteration": 2.8372464179992676 + }, + { + "auxiliary_loss_clip": 0.01151326, + "auxiliary_loss_mlp": 0.01040012, + "balance_loss_clip": 1.04831719, + "balance_loss_mlp": 1.0214752, + "epoch": 0.38441652137317384, + "flos": 25337492075520.0, + "grad_norm": 1.5618088292899752, + "language_loss": 0.79694057, + "learning_rate": 2.820962213242361e-06, + "loss": 0.81885391, + "num_input_tokens_seen": 68874260, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.18530273, + "step": 3197, + "time_per_iteration": 4.428937911987305 + }, + { + "auxiliary_loss_clip": 0.01154698, + "auxiliary_loss_mlp": 0.01051043, + "balance_loss_clip": 1.0497539, + "balance_loss_mlp": 1.03195775, + "epoch": 0.3845367642638129, + "flos": 18113486446080.0, + "grad_norm": 2.1287304611985878, + "language_loss": 0.84527707, + "learning_rate": 2.8202518308133264e-06, + "loss": 0.86733449, + "num_input_tokens_seen": 68891535, + "router_z_loss_clip": 1.04980469, + "router_z_loss_mlp": 0.1907959, + "step": 3198, + "time_per_iteration": 2.8027994632720947 + }, + { + "auxiliary_loss_clip": 0.01156564, + "auxiliary_loss_mlp": 0.01053085, + "balance_loss_clip": 1.04894948, + "balance_loss_mlp": 1.03287888, + "epoch": 0.384657007154452, + "flos": 25228395492480.0, + "grad_norm": 2.250397652863761, + "language_loss": 0.73600656, + "learning_rate": 2.8195413239527426e-06, + "loss": 0.75810301, + "num_input_tokens_seen": 68911275, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.20214844, + "step": 3199, + "time_per_iteration": 2.8350183963775635 + }, + { + "auxiliary_loss_clip": 0.01154713, + "auxiliary_loss_mlp": 0.01047894, + "balance_loss_clip": 1.0489645, + "balance_loss_mlp": 1.02878499, + "epoch": 0.38477725004509106, + "flos": 19865855358720.0, + "grad_norm": 1.7528340527239024, + "language_loss": 0.80845225, + "learning_rate": 2.8188306927683906e-06, + "loss": 0.83047831, + "num_input_tokens_seen": 68930745, + "router_z_loss_clip": 1.05566406, + "router_z_loss_mlp": 0.19116211, + "step": 3200, + "time_per_iteration": 2.7981622219085693 + }, + { + "auxiliary_loss_clip": 0.01155096, + "auxiliary_loss_mlp": 0.01050734, + "balance_loss_clip": 1.05046368, + "balance_loss_mlp": 1.03034937, + "epoch": 0.38489749293573017, + "flos": 18259391491200.0, + "grad_norm": 2.0476443464104808, + "language_loss": 0.75006688, + "learning_rate": 2.818119937368074e-06, + "loss": 0.77212512, + "num_input_tokens_seen": 68949380, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.20385742, + "step": 3201, + "time_per_iteration": 2.8626506328582764 + }, + { + "auxiliary_loss_clip": 0.0115729, + "auxiliary_loss_mlp": 0.01047504, + "balance_loss_clip": 1.04927182, + "balance_loss_mlp": 1.02711916, + "epoch": 0.3850177358263693, + "flos": 24389163152640.0, + "grad_norm": 1.8084210407483046, + "language_loss": 0.65732735, + "learning_rate": 2.817409057859613e-06, + "loss": 0.67937529, + "num_input_tokens_seen": 68968370, + "router_z_loss_clip": 1.08105469, + "router_z_loss_mlp": 0.20397949, + "step": 3202, + "time_per_iteration": 2.872136116027832 + }, + { + "auxiliary_loss_clip": 0.01159372, + "auxiliary_loss_mlp": 0.01053621, + "balance_loss_clip": 1.05096412, + "balance_loss_mlp": 1.03255653, + "epoch": 0.38513797871700833, + "flos": 17671533505920.0, + "grad_norm": 2.3354958076762276, + "language_loss": 0.79334491, + "learning_rate": 2.8166980543508482e-06, + "loss": 0.81547487, + "num_input_tokens_seen": 68984260, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.21069336, + "step": 3203, + "time_per_iteration": 2.895209312438965 + }, + { + "auxiliary_loss_clip": 0.01154776, + "auxiliary_loss_mlp": 0.01044639, + "balance_loss_clip": 1.05134439, + "balance_loss_mlp": 1.02639985, + "epoch": 0.38525822160764744, + "flos": 25739583897600.0, + "grad_norm": 2.697219350612931, + "language_loss": 0.7947979, + "learning_rate": 2.815986926949638e-06, + "loss": 0.81679201, + "num_input_tokens_seen": 69002760, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.18237305, + "step": 3204, + "time_per_iteration": 2.8233132362365723 + }, + { + "auxiliary_loss_clip": 0.01156066, + "auxiliary_loss_mlp": 0.01045291, + "balance_loss_clip": 1.05157399, + "balance_loss_mlp": 1.02665913, + "epoch": 0.38537846449828655, + "flos": 20193647898240.0, + "grad_norm": 1.7696942124806083, + "language_loss": 0.80397308, + "learning_rate": 2.8152756757638597e-06, + "loss": 0.82598662, + "num_input_tokens_seen": 69021260, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.18615723, + "step": 3205, + "time_per_iteration": 2.7505245208740234 + }, + { + "auxiliary_loss_clip": 0.01157379, + "auxiliary_loss_mlp": 0.01050412, + "balance_loss_clip": 1.05120146, + "balance_loss_mlp": 1.031672, + "epoch": 0.3854987073889256, + "flos": 23039352938880.0, + "grad_norm": 1.9728717220838605, + "language_loss": 0.84573603, + "learning_rate": 2.8145643009014093e-06, + "loss": 0.86781394, + "num_input_tokens_seen": 69039755, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.1875, + "step": 3206, + "time_per_iteration": 2.819286346435547 + }, + { + "auxiliary_loss_clip": 0.01146696, + "auxiliary_loss_mlp": 0.01044329, + "balance_loss_clip": 1.04837394, + "balance_loss_mlp": 1.02707946, + "epoch": 0.3856189502795647, + "flos": 20190631155840.0, + "grad_norm": 1.9638226918560266, + "language_loss": 0.79060102, + "learning_rate": 2.813852802470202e-06, + "loss": 0.81251132, + "num_input_tokens_seen": 69057650, + "router_z_loss_clip": 0.98242188, + "router_z_loss_mlp": 0.17248535, + "step": 3207, + "time_per_iteration": 2.808326482772827 + }, + { + "auxiliary_loss_clip": 0.0115907, + "auxiliary_loss_mlp": 0.01042246, + "balance_loss_clip": 1.05265582, + "balance_loss_mlp": 1.02373266, + "epoch": 0.38573919317020383, + "flos": 25702631781120.0, + "grad_norm": 3.707841056598383, + "language_loss": 0.72785854, + "learning_rate": 2.8131411805781717e-06, + "loss": 0.74987173, + "num_input_tokens_seen": 69077775, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.18518066, + "step": 3208, + "time_per_iteration": 2.9142754077911377 + }, + { + "auxiliary_loss_clip": 0.01158086, + "auxiliary_loss_mlp": 0.0105327, + "balance_loss_clip": 1.05208182, + "balance_loss_mlp": 1.03346968, + "epoch": 0.3858594360608429, + "flos": 29821405628160.0, + "grad_norm": 2.3781572800640025, + "language_loss": 0.64404428, + "learning_rate": 2.8124294353332707e-06, + "loss": 0.6661579, + "num_input_tokens_seen": 69096450, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.19799805, + "step": 3209, + "time_per_iteration": 2.985002279281616 + }, + { + "auxiliary_loss_clip": 0.01151227, + "auxiliary_loss_mlp": 0.01051912, + "balance_loss_clip": 1.04936862, + "balance_loss_mlp": 1.03430521, + "epoch": 0.385979678951482, + "flos": 24790428961920.0, + "grad_norm": 2.4857285924198025, + "language_loss": 0.77570617, + "learning_rate": 2.8117175668434713e-06, + "loss": 0.79773754, + "num_input_tokens_seen": 69116110, + "router_z_loss_clip": 1.01855469, + "router_z_loss_mlp": 0.17590332, + "step": 3210, + "time_per_iteration": 2.80971097946167 + }, + { + "auxiliary_loss_clip": 0.01158539, + "auxiliary_loss_mlp": 0.01038033, + "balance_loss_clip": 1.05081463, + "balance_loss_mlp": 1.01872146, + "epoch": 0.3860999218421211, + "flos": 21287881866240.0, + "grad_norm": 2.171366837445265, + "language_loss": 0.69931197, + "learning_rate": 2.811005575216762e-06, + "loss": 0.72127765, + "num_input_tokens_seen": 69134825, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.1932373, + "step": 3211, + "time_per_iteration": 2.822209596633911 + }, + { + "auxiliary_loss_clip": 0.01155917, + "auxiliary_loss_mlp": 0.01044914, + "balance_loss_clip": 1.0489912, + "balance_loss_mlp": 1.0263648, + "epoch": 0.38622016473276016, + "flos": 24536720223360.0, + "grad_norm": 1.5247227554658882, + "language_loss": 0.7897839, + "learning_rate": 2.8102934605611513e-06, + "loss": 0.81179225, + "num_input_tokens_seen": 69156460, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.18566895, + "step": 3212, + "time_per_iteration": 2.9521684646606445 + }, + { + "auxiliary_loss_clip": 0.01159385, + "auxiliary_loss_mlp": 0.01047473, + "balance_loss_clip": 1.05152261, + "balance_loss_mlp": 1.02730274, + "epoch": 0.3863404076233993, + "flos": 20558212986240.0, + "grad_norm": 2.637609459757323, + "language_loss": 0.67663199, + "learning_rate": 2.8095812229846665e-06, + "loss": 0.69870055, + "num_input_tokens_seen": 69176420, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.20166016, + "step": 3213, + "time_per_iteration": 2.855531692504883 + }, + { + "auxiliary_loss_clip": 0.0115485, + "auxiliary_loss_mlp": 0.01056642, + "balance_loss_clip": 1.04976177, + "balance_loss_mlp": 1.03630447, + "epoch": 0.3864606505140384, + "flos": 22346277039360.0, + "grad_norm": 2.532649815965384, + "language_loss": 0.69176245, + "learning_rate": 2.808868862595355e-06, + "loss": 0.71387738, + "num_input_tokens_seen": 69196665, + "router_z_loss_clip": 1.05078125, + "router_z_loss_mlp": 0.20349121, + "step": 3214, + "time_per_iteration": 2.8124444484710693 + }, + { + "auxiliary_loss_clip": 0.01157376, + "auxiliary_loss_mlp": 0.01044388, + "balance_loss_clip": 1.05312109, + "balance_loss_mlp": 1.02649426, + "epoch": 0.38658089340467744, + "flos": 25703601448320.0, + "grad_norm": 2.7263323370812564, + "language_loss": 0.79541117, + "learning_rate": 2.8081563795012795e-06, + "loss": 0.81742883, + "num_input_tokens_seen": 69216290, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.17895508, + "step": 3215, + "time_per_iteration": 2.985532283782959 + }, + { + "auxiliary_loss_clip": 0.01153144, + "auxiliary_loss_mlp": 0.01043564, + "balance_loss_clip": 1.04968131, + "balance_loss_mlp": 1.02440763, + "epoch": 0.38670113629531655, + "flos": 33802534558080.0, + "grad_norm": 1.7953544846924605, + "language_loss": 0.74126625, + "learning_rate": 2.807443773810524e-06, + "loss": 0.76323336, + "num_input_tokens_seen": 69237550, + "router_z_loss_clip": 1.03466797, + "router_z_loss_mlp": 0.19128418, + "step": 3216, + "time_per_iteration": 2.9827916622161865 + }, + { + "auxiliary_loss_clip": 0.01158848, + "auxiliary_loss_mlp": 0.01047252, + "balance_loss_clip": 1.05275393, + "balance_loss_mlp": 1.02748692, + "epoch": 0.3868213791859556, + "flos": 23331522165120.0, + "grad_norm": 2.1996183876885413, + "language_loss": 0.89915383, + "learning_rate": 2.80673104563119e-06, + "loss": 0.92121482, + "num_input_tokens_seen": 69258175, + "router_z_loss_clip": 1.06152344, + "router_z_loss_mlp": 0.1973877, + "step": 3217, + "time_per_iteration": 2.7615232467651367 + }, + { + "auxiliary_loss_clip": 0.01152405, + "auxiliary_loss_mlp": 0.01044425, + "balance_loss_clip": 1.0506649, + "balance_loss_mlp": 1.02597165, + "epoch": 0.3869416220765947, + "flos": 18441530380800.0, + "grad_norm": 1.9667338660239453, + "language_loss": 0.78852546, + "learning_rate": 2.8060181950713976e-06, + "loss": 0.81049371, + "num_input_tokens_seen": 69274965, + "router_z_loss_clip": 1.01757812, + "router_z_loss_mlp": 0.18444824, + "step": 3218, + "time_per_iteration": 2.7297284603118896 + }, + { + "auxiliary_loss_clip": 0.0115882, + "auxiliary_loss_mlp": 0.01057298, + "balance_loss_clip": 1.05335045, + "balance_loss_mlp": 1.03879714, + "epoch": 0.3870618649672338, + "flos": 15632992938240.0, + "grad_norm": 3.0646783751052156, + "language_loss": 0.81205481, + "learning_rate": 2.805305222239286e-06, + "loss": 0.834216, + "num_input_tokens_seen": 69292220, + "router_z_loss_clip": 1.05371094, + "router_z_loss_mlp": 0.18493652, + "step": 3219, + "time_per_iteration": 3.490581750869751 + }, + { + "auxiliary_loss_clip": 0.01154457, + "auxiliary_loss_mlp": 0.01046446, + "balance_loss_clip": 1.05127001, + "balance_loss_mlp": 1.02838588, + "epoch": 0.3871821078578729, + "flos": 23513804709120.0, + "grad_norm": 2.0421283030898634, + "language_loss": 0.73940182, + "learning_rate": 2.8045921272430118e-06, + "loss": 0.76141089, + "num_input_tokens_seen": 69311900, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.18078613, + "step": 3220, + "time_per_iteration": 2.7907893657684326 + }, + { + "auxiliary_loss_clip": 0.01160131, + "auxiliary_loss_mlp": 0.01059538, + "balance_loss_clip": 1.05219722, + "balance_loss_mlp": 1.03783047, + "epoch": 0.387302350748512, + "flos": 17778259791360.0, + "grad_norm": 1.9995741657336297, + "language_loss": 0.76886129, + "learning_rate": 2.803878910190753e-06, + "loss": 0.79105806, + "num_input_tokens_seen": 69328820, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.21691895, + "step": 3221, + "time_per_iteration": 2.73378324508667 + }, + { + "auxiliary_loss_clip": 0.01157592, + "auxiliary_loss_mlp": 0.01047119, + "balance_loss_clip": 1.05421829, + "balance_loss_mlp": 1.02904713, + "epoch": 0.3874225936391511, + "flos": 11503409097600.0, + "grad_norm": 2.253626980822248, + "language_loss": 0.81670493, + "learning_rate": 2.8031655711907017e-06, + "loss": 0.83875203, + "num_input_tokens_seen": 69342525, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.18054199, + "step": 3222, + "time_per_iteration": 3.5147862434387207 + }, + { + "auxiliary_loss_clip": 0.0115949, + "auxiliary_loss_mlp": 0.01042533, + "balance_loss_clip": 1.05324674, + "balance_loss_mlp": 1.02397192, + "epoch": 0.38754283652979016, + "flos": 21945154884480.0, + "grad_norm": 2.194983366735363, + "language_loss": 0.80992293, + "learning_rate": 2.8024521103510723e-06, + "loss": 0.83194315, + "num_input_tokens_seen": 69359295, + "router_z_loss_clip": 1.06347656, + "router_z_loss_mlp": 0.18566895, + "step": 3223, + "time_per_iteration": 2.7567479610443115 + }, + { + "auxiliary_loss_clip": 0.01157671, + "auxiliary_loss_mlp": 0.01048953, + "balance_loss_clip": 1.0517813, + "balance_loss_mlp": 1.03074956, + "epoch": 0.38766307942042927, + "flos": 21175984022400.0, + "grad_norm": 1.7004802587555277, + "language_loss": 0.75430858, + "learning_rate": 2.8017385277800952e-06, + "loss": 0.77637482, + "num_input_tokens_seen": 69377650, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.18225098, + "step": 3224, + "time_per_iteration": 2.7082722187042236 + }, + { + "auxiliary_loss_clip": 0.01160321, + "auxiliary_loss_mlp": 0.01048486, + "balance_loss_clip": 1.05367231, + "balance_loss_mlp": 1.02880454, + "epoch": 0.3877833223110684, + "flos": 27417294391680.0, + "grad_norm": 2.3897753060268454, + "language_loss": 0.75095463, + "learning_rate": 2.8010248235860213e-06, + "loss": 0.77304268, + "num_input_tokens_seen": 69397765, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.19689941, + "step": 3225, + "time_per_iteration": 2.764660358428955 + }, + { + "auxiliary_loss_clip": 0.01058167, + "auxiliary_loss_mlp": 0.01001577, + "balance_loss_clip": 1.02431941, + "balance_loss_mlp": 0.99946654, + "epoch": 0.38790356520170743, + "flos": 64500019879680.0, + "grad_norm": 0.835889891555949, + "language_loss": 0.62791467, + "learning_rate": 2.8003109978771192e-06, + "loss": 0.64851213, + "num_input_tokens_seen": 69458930, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.02111816, + "step": 3226, + "time_per_iteration": 4.969139575958252 + }, + { + "auxiliary_loss_clip": 0.0115532, + "auxiliary_loss_mlp": 0.01045706, + "balance_loss_clip": 1.05069578, + "balance_loss_mlp": 1.02610826, + "epoch": 0.38802380809234654, + "flos": 22345415112960.0, + "grad_norm": 2.2991826889200437, + "language_loss": 0.79297781, + "learning_rate": 2.799597050761674e-06, + "loss": 0.81498802, + "num_input_tokens_seen": 69475135, + "router_z_loss_clip": 1.04541016, + "router_z_loss_mlp": 0.19604492, + "step": 3227, + "time_per_iteration": 2.8274292945861816 + }, + { + "auxiliary_loss_clip": 0.01161619, + "auxiliary_loss_mlp": 0.01047502, + "balance_loss_clip": 1.05311811, + "balance_loss_mlp": 1.02760625, + "epoch": 0.38814405098298566, + "flos": 25261361199360.0, + "grad_norm": 1.9117234893463912, + "language_loss": 0.79322934, + "learning_rate": 2.7988829823479924e-06, + "loss": 0.81532061, + "num_input_tokens_seen": 69493525, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.19897461, + "step": 3228, + "time_per_iteration": 2.817681074142456 + }, + { + "auxiliary_loss_clip": 0.0116034, + "auxiliary_loss_mlp": 0.0105776, + "balance_loss_clip": 1.05251276, + "balance_loss_mlp": 1.03713667, + "epoch": 0.3882642938736247, + "flos": 18841180078080.0, + "grad_norm": 1.872584069398883, + "language_loss": 0.64458585, + "learning_rate": 2.7981687927443976e-06, + "loss": 0.66676688, + "num_input_tokens_seen": 69510325, + "router_z_loss_clip": 1.07861328, + "router_z_loss_mlp": 0.2064209, + "step": 3229, + "time_per_iteration": 2.8016445636749268 + }, + { + "auxiliary_loss_clip": 0.01154818, + "auxiliary_loss_mlp": 0.01041025, + "balance_loss_clip": 1.04838324, + "balance_loss_mlp": 1.02311993, + "epoch": 0.3883845367642638, + "flos": 21652806090240.0, + "grad_norm": 2.1121813713469257, + "language_loss": 0.85456145, + "learning_rate": 2.797454482059231e-06, + "loss": 0.87651992, + "num_input_tokens_seen": 69530480, + "router_z_loss_clip": 1.06347656, + "router_z_loss_mlp": 0.17919922, + "step": 3230, + "time_per_iteration": 3.026716470718384 + }, + { + "auxiliary_loss_clip": 0.01154161, + "auxiliary_loss_mlp": 0.01043909, + "balance_loss_clip": 1.05015981, + "balance_loss_mlp": 1.02549148, + "epoch": 0.3885047796549029, + "flos": 20557530627840.0, + "grad_norm": 2.111092841555308, + "language_loss": 0.84435868, + "learning_rate": 2.7967400504008537e-06, + "loss": 0.86633933, + "num_input_tokens_seen": 69549780, + "router_z_loss_clip": 1.04003906, + "router_z_loss_mlp": 0.1842041, + "step": 3231, + "time_per_iteration": 2.792360782623291 + }, + { + "auxiliary_loss_clip": 0.01054439, + "auxiliary_loss_mlp": 0.01006597, + "balance_loss_clip": 1.02063751, + "balance_loss_mlp": 1.00441551, + "epoch": 0.388625022545542, + "flos": 64325491695360.0, + "grad_norm": 0.7917820600436613, + "language_loss": 0.57442063, + "learning_rate": 2.7960254978776456e-06, + "loss": 0.59503102, + "num_input_tokens_seen": 69611870, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.02185059, + "step": 3232, + "time_per_iteration": 3.2860071659088135 + }, + { + "auxiliary_loss_clip": 0.01155574, + "auxiliary_loss_mlp": 0.01052082, + "balance_loss_clip": 1.05004644, + "balance_loss_mlp": 1.03337812, + "epoch": 0.3887452654361811, + "flos": 18113881495680.0, + "grad_norm": 2.035488864600934, + "language_loss": 0.81400919, + "learning_rate": 2.7953108245980006e-06, + "loss": 0.8360858, + "num_input_tokens_seen": 69630385, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.18701172, + "step": 3233, + "time_per_iteration": 2.78045916557312 + }, + { + "auxiliary_loss_clip": 0.01150465, + "auxiliary_loss_mlp": 0.01052561, + "balance_loss_clip": 1.04897332, + "balance_loss_mlp": 1.03605008, + "epoch": 0.38886550832682015, + "flos": 24975261371520.0, + "grad_norm": 2.010113319927281, + "language_loss": 0.73594761, + "learning_rate": 2.7945960306703365e-06, + "loss": 0.75797784, + "num_input_tokens_seen": 69653370, + "router_z_loss_clip": 1.01464844, + "router_z_loss_mlp": 0.16516113, + "step": 3234, + "time_per_iteration": 2.856684923171997 + }, + { + "auxiliary_loss_clip": 0.011495, + "auxiliary_loss_mlp": 0.01050404, + "balance_loss_clip": 1.04797149, + "balance_loss_mlp": 1.03184342, + "epoch": 0.38898575121745926, + "flos": 27199496275200.0, + "grad_norm": 1.7448471727828687, + "language_loss": 0.65907562, + "learning_rate": 2.7938811162030865e-06, + "loss": 0.68107468, + "num_input_tokens_seen": 69673635, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.18554688, + "step": 3235, + "time_per_iteration": 2.7979094982147217 + }, + { + "auxiliary_loss_clip": 0.01149048, + "auxiliary_loss_mlp": 0.0104332, + "balance_loss_clip": 1.04679227, + "balance_loss_mlp": 1.02596343, + "epoch": 0.3891059941080984, + "flos": 28763728727040.0, + "grad_norm": 1.6796740897940594, + "language_loss": 0.82507944, + "learning_rate": 2.793166081304702e-06, + "loss": 0.8470031, + "num_input_tokens_seen": 69694130, + "router_z_loss_clip": 1.02148438, + "router_z_loss_mlp": 0.17370605, + "step": 3236, + "time_per_iteration": 2.871753692626953 + }, + { + "auxiliary_loss_clip": 0.01158898, + "auxiliary_loss_mlp": 0.01046658, + "balance_loss_clip": 1.05178332, + "balance_loss_mlp": 1.02743006, + "epoch": 0.38922623699873743, + "flos": 22893447893760.0, + "grad_norm": 1.8761933229911893, + "language_loss": 0.82544017, + "learning_rate": 2.7924509260836543e-06, + "loss": 0.84749579, + "num_input_tokens_seen": 69713255, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.19226074, + "step": 3237, + "time_per_iteration": 2.7556774616241455 + }, + { + "auxiliary_loss_clip": 0.01155205, + "auxiliary_loss_mlp": 0.01051368, + "balance_loss_clip": 1.05156183, + "balance_loss_mlp": 1.03242612, + "epoch": 0.38934647988937654, + "flos": 19792418002560.0, + "grad_norm": 1.5430184350240996, + "language_loss": 0.68457901, + "learning_rate": 2.791735650648431e-06, + "loss": 0.70664477, + "num_input_tokens_seen": 69732375, + "router_z_loss_clip": 1.03662109, + "router_z_loss_mlp": 0.18945312, + "step": 3238, + "time_per_iteration": 2.754970073699951 + }, + { + "auxiliary_loss_clip": 0.01151505, + "auxiliary_loss_mlp": 0.01047355, + "balance_loss_clip": 1.04649425, + "balance_loss_mlp": 1.02804267, + "epoch": 0.38946672278001565, + "flos": 19202081978880.0, + "grad_norm": 2.883621283121745, + "language_loss": 0.74546015, + "learning_rate": 2.791020255107538e-06, + "loss": 0.76744878, + "num_input_tokens_seen": 69749745, + "router_z_loss_clip": 1.04882812, + "router_z_loss_mlp": 0.1932373, + "step": 3239, + "time_per_iteration": 2.7362146377563477 + }, + { + "auxiliary_loss_clip": 0.01154197, + "auxiliary_loss_mlp": 0.01053235, + "balance_loss_clip": 1.04934311, + "balance_loss_mlp": 1.03267097, + "epoch": 0.3895869656706547, + "flos": 24936477661440.0, + "grad_norm": 1.514554089279579, + "language_loss": 0.80522531, + "learning_rate": 2.7903047395695023e-06, + "loss": 0.82729971, + "num_input_tokens_seen": 69769645, + "router_z_loss_clip": 1.04882812, + "router_z_loss_mlp": 0.20556641, + "step": 3240, + "time_per_iteration": 2.807199716567993 + }, + { + "auxiliary_loss_clip": 0.01154476, + "auxiliary_loss_mlp": 0.01053066, + "balance_loss_clip": 1.04860961, + "balance_loss_mlp": 1.03277671, + "epoch": 0.3897072085612938, + "flos": 24133622820480.0, + "grad_norm": 2.2170018292684683, + "language_loss": 0.90366113, + "learning_rate": 2.789589104142865e-06, + "loss": 0.92573655, + "num_input_tokens_seen": 69787270, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.20300293, + "step": 3241, + "time_per_iteration": 2.8053207397460938 + }, + { + "auxiliary_loss_clip": 0.01151025, + "auxiliary_loss_mlp": 0.0104673, + "balance_loss_clip": 1.04906678, + "balance_loss_mlp": 1.02808571, + "epoch": 0.3898274514519329, + "flos": 17166342672000.0, + "grad_norm": 1.80500683898098, + "language_loss": 0.76919562, + "learning_rate": 2.7888733489361895e-06, + "loss": 0.79117316, + "num_input_tokens_seen": 69805685, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.18640137, + "step": 3242, + "time_per_iteration": 2.7207987308502197 + }, + { + "auxiliary_loss_clip": 0.01047778, + "auxiliary_loss_mlp": 0.01002748, + "balance_loss_clip": 1.01406038, + "balance_loss_mlp": 1.00066209, + "epoch": 0.389947694342572, + "flos": 66074807952000.0, + "grad_norm": 0.7336748796672097, + "language_loss": 0.58713317, + "learning_rate": 2.788157474058054e-06, + "loss": 0.60763836, + "num_input_tokens_seen": 69867960, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.02087402, + "step": 3243, + "time_per_iteration": 3.450604200363159 + }, + { + "auxiliary_loss_clip": 0.01149021, + "auxiliary_loss_mlp": 0.01038858, + "balance_loss_clip": 1.04870367, + "balance_loss_mlp": 1.0209527, + "epoch": 0.3900679372332111, + "flos": 25740912700800.0, + "grad_norm": 1.5145830008200158, + "language_loss": 0.70458865, + "learning_rate": 2.7874414796170555e-06, + "loss": 0.72646743, + "num_input_tokens_seen": 69889450, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.17895508, + "step": 3244, + "time_per_iteration": 2.807706356048584 + }, + { + "auxiliary_loss_clip": 0.01152308, + "auxiliary_loss_mlp": 0.01043473, + "balance_loss_clip": 1.04715514, + "balance_loss_mlp": 1.0247215, + "epoch": 0.3901881801238502, + "flos": 11801611808640.0, + "grad_norm": 2.2929948152190534, + "language_loss": 0.83933944, + "learning_rate": 2.7867253657218113e-06, + "loss": 0.86129725, + "num_input_tokens_seen": 69903340, + "router_z_loss_clip": 1.05224609, + "router_z_loss_mlp": 0.18737793, + "step": 3245, + "time_per_iteration": 2.7493131160736084 + }, + { + "auxiliary_loss_clip": 0.01153027, + "auxiliary_loss_mlp": 0.01048223, + "balance_loss_clip": 1.04838753, + "balance_loss_mlp": 1.02814865, + "epoch": 0.39030842301448926, + "flos": 27308951994240.0, + "grad_norm": 1.8249915275562378, + "language_loss": 0.72884506, + "learning_rate": 2.7860091324809544e-06, + "loss": 0.75085759, + "num_input_tokens_seen": 69924400, + "router_z_loss_clip": 1.04833984, + "router_z_loss_mlp": 0.20080566, + "step": 3246, + "time_per_iteration": 2.7688496112823486 + }, + { + "auxiliary_loss_clip": 0.01152954, + "auxiliary_loss_mlp": 0.01047279, + "balance_loss_clip": 1.04964173, + "balance_loss_mlp": 1.02858675, + "epoch": 0.39042866590512837, + "flos": 27163334257920.0, + "grad_norm": 2.3319155257392934, + "language_loss": 0.81420851, + "learning_rate": 2.7852927800031377e-06, + "loss": 0.83621079, + "num_input_tokens_seen": 69944565, + "router_z_loss_clip": 1.03222656, + "router_z_loss_mlp": 0.18676758, + "step": 3247, + "time_per_iteration": 2.779653310775757 + }, + { + "auxiliary_loss_clip": 0.01154222, + "auxiliary_loss_mlp": 0.01049724, + "balance_loss_clip": 1.0494206, + "balance_loss_mlp": 1.03080595, + "epoch": 0.3905489087957674, + "flos": 29716115886720.0, + "grad_norm": 2.2011675991122903, + "language_loss": 0.83250403, + "learning_rate": 2.7845763083970298e-06, + "loss": 0.85454345, + "num_input_tokens_seen": 69964965, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.18920898, + "step": 3248, + "time_per_iteration": 3.585625648498535 + }, + { + "auxiliary_loss_clip": 0.01153798, + "auxiliary_loss_mlp": 0.01043669, + "balance_loss_clip": 1.04802823, + "balance_loss_mlp": 1.02351081, + "epoch": 0.39066915168640653, + "flos": 24498618871680.0, + "grad_norm": 1.8090933762838517, + "language_loss": 0.82262659, + "learning_rate": 2.7838597177713205e-06, + "loss": 0.84460127, + "num_input_tokens_seen": 69986055, + "router_z_loss_clip": 1.05712891, + "router_z_loss_mlp": 0.20178223, + "step": 3249, + "time_per_iteration": 2.752065896987915 + }, + { + "auxiliary_loss_clip": 0.01154374, + "auxiliary_loss_mlp": 0.01044788, + "balance_loss_clip": 1.04852951, + "balance_loss_mlp": 1.02641749, + "epoch": 0.39078939457704565, + "flos": 20558572122240.0, + "grad_norm": 1.8729997229635635, + "language_loss": 0.73598737, + "learning_rate": 2.7831430082347143e-06, + "loss": 0.75797898, + "num_input_tokens_seen": 70005260, + "router_z_loss_clip": 1.05810547, + "router_z_loss_mlp": 0.18371582, + "step": 3250, + "time_per_iteration": 2.7775566577911377 + }, + { + "auxiliary_loss_clip": 0.01152981, + "auxiliary_loss_mlp": 0.01041141, + "balance_loss_clip": 1.05161726, + "balance_loss_mlp": 1.0227828, + "epoch": 0.3909096374676847, + "flos": 22783417557120.0, + "grad_norm": 2.013570697286482, + "language_loss": 0.82288468, + "learning_rate": 2.7824261798959373e-06, + "loss": 0.84482586, + "num_input_tokens_seen": 70023440, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.18371582, + "step": 3251, + "time_per_iteration": 3.6616315841674805 + }, + { + "auxiliary_loss_clip": 0.01150058, + "auxiliary_loss_mlp": 0.01051637, + "balance_loss_clip": 1.04656005, + "balance_loss_mlp": 1.03283799, + "epoch": 0.3910298803583238, + "flos": 23003119094400.0, + "grad_norm": 1.8109229473665909, + "language_loss": 0.79879272, + "learning_rate": 2.78170923286373e-06, + "loss": 0.82080972, + "num_input_tokens_seen": 70043040, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.18786621, + "step": 3252, + "time_per_iteration": 2.737475872039795 + }, + { + "auxiliary_loss_clip": 0.01153409, + "auxiliary_loss_mlp": 0.0105186, + "balance_loss_clip": 1.05088854, + "balance_loss_mlp": 1.03279805, + "epoch": 0.3911501232489629, + "flos": 24316264500480.0, + "grad_norm": 2.5064344845059137, + "language_loss": 0.83786654, + "learning_rate": 2.780992167246854e-06, + "loss": 0.85991919, + "num_input_tokens_seen": 70060565, + "router_z_loss_clip": 1.02539062, + "router_z_loss_mlp": 0.19055176, + "step": 3253, + "time_per_iteration": 2.7959563732147217 + }, + { + "auxiliary_loss_clip": 0.01049217, + "auxiliary_loss_mlp": 0.01003758, + "balance_loss_clip": 1.01490784, + "balance_loss_mlp": 1.00157666, + "epoch": 0.391270366139602, + "flos": 60869054684160.0, + "grad_norm": 0.977220338272514, + "language_loss": 0.7210148, + "learning_rate": 2.7802749831540883e-06, + "loss": 0.74154454, + "num_input_tokens_seen": 70119465, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.02185059, + "step": 3254, + "time_per_iteration": 3.2999870777130127 + }, + { + "auxiliary_loss_clip": 0.01144933, + "auxiliary_loss_mlp": 0.01047525, + "balance_loss_clip": 1.04727232, + "balance_loss_mlp": 1.03118145, + "epoch": 0.3913906090302411, + "flos": 21543494025600.0, + "grad_norm": 2.051469838985422, + "language_loss": 0.8209064, + "learning_rate": 2.7795576806942268e-06, + "loss": 0.8428309, + "num_input_tokens_seen": 70138270, + "router_z_loss_clip": 0.9765625, + "router_z_loss_mlp": 0.16357422, + "step": 3255, + "time_per_iteration": 4.3266661167144775 + }, + { + "auxiliary_loss_clip": 0.01048874, + "auxiliary_loss_mlp": 0.01002667, + "balance_loss_clip": 1.014714, + "balance_loss_mlp": 1.00055695, + "epoch": 0.3915108519208802, + "flos": 49839953702400.0, + "grad_norm": 0.7615878588286443, + "language_loss": 0.54874307, + "learning_rate": 2.778840259976085e-06, + "loss": 0.56925845, + "num_input_tokens_seen": 70193500, + "router_z_loss_clip": 0.34179688, + "router_z_loss_mlp": 0.02111816, + "step": 3256, + "time_per_iteration": 3.235029458999634 + }, + { + "auxiliary_loss_clip": 0.01153777, + "auxiliary_loss_mlp": 0.0104612, + "balance_loss_clip": 1.04881096, + "balance_loss_mlp": 1.0263555, + "epoch": 0.39163109481151925, + "flos": 16506447960960.0, + "grad_norm": 3.1008979526688316, + "language_loss": 0.76731545, + "learning_rate": 2.778122721108495e-06, + "loss": 0.78931439, + "num_input_tokens_seen": 70211730, + "router_z_loss_clip": 1.04980469, + "router_z_loss_mlp": 0.19750977, + "step": 3257, + "time_per_iteration": 2.7418127059936523 + }, + { + "auxiliary_loss_clip": 0.01148585, + "auxiliary_loss_mlp": 0.01053594, + "balance_loss_clip": 1.04767549, + "balance_loss_mlp": 1.03489017, + "epoch": 0.39175133770215836, + "flos": 26067484177920.0, + "grad_norm": 1.8693265632621794, + "language_loss": 0.88257438, + "learning_rate": 2.7774050642003076e-06, + "loss": 0.90459621, + "num_input_tokens_seen": 70232540, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.18701172, + "step": 3258, + "time_per_iteration": 2.7752678394317627 + }, + { + "auxiliary_loss_clip": 0.01158277, + "auxiliary_loss_mlp": 0.01050356, + "balance_loss_clip": 1.05134511, + "balance_loss_mlp": 1.03067446, + "epoch": 0.3918715805927975, + "flos": 21872076664320.0, + "grad_norm": 1.8230096603932682, + "language_loss": 0.93653655, + "learning_rate": 2.7766872893603896e-06, + "loss": 0.95862281, + "num_input_tokens_seen": 70252515, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.19677734, + "step": 3259, + "time_per_iteration": 2.846618890762329 + }, + { + "auxiliary_loss_clip": 0.01150872, + "auxiliary_loss_mlp": 0.01046223, + "balance_loss_clip": 1.04920053, + "balance_loss_mlp": 1.02862775, + "epoch": 0.39199182348343653, + "flos": 20376181837440.0, + "grad_norm": 1.8275456568648436, + "language_loss": 0.73437613, + "learning_rate": 2.7759693966976275e-06, + "loss": 0.75634706, + "num_input_tokens_seen": 70271020, + "router_z_loss_clip": 1.01708984, + "router_z_loss_mlp": 0.17578125, + "step": 3260, + "time_per_iteration": 2.7229857444763184 + }, + { + "auxiliary_loss_clip": 0.01154224, + "auxiliary_loss_mlp": 0.01044936, + "balance_loss_clip": 1.04664052, + "balance_loss_mlp": 1.02576733, + "epoch": 0.39211206637407564, + "flos": 21683545153920.0, + "grad_norm": 1.9679327920397771, + "language_loss": 0.8554827, + "learning_rate": 2.7752513863209242e-06, + "loss": 0.87747431, + "num_input_tokens_seen": 70289600, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.19177246, + "step": 3261, + "time_per_iteration": 2.801593542098999 + }, + { + "auxiliary_loss_clip": 0.01146985, + "auxiliary_loss_mlp": 0.01046985, + "balance_loss_clip": 1.04776716, + "balance_loss_mlp": 1.03057003, + "epoch": 0.39223230926471475, + "flos": 21066276908160.0, + "grad_norm": 1.7377916500669321, + "language_loss": 0.8440429, + "learning_rate": 2.774533258339203e-06, + "loss": 0.86598265, + "num_input_tokens_seen": 70307060, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.16418457, + "step": 3262, + "time_per_iteration": 2.7696523666381836 + }, + { + "auxiliary_loss_clip": 0.01153597, + "auxiliary_loss_mlp": 0.01052496, + "balance_loss_clip": 1.04807711, + "balance_loss_mlp": 1.03323221, + "epoch": 0.3923525521553538, + "flos": 17603016312960.0, + "grad_norm": 2.096453820524763, + "language_loss": 0.79536241, + "learning_rate": 2.7738150128614014e-06, + "loss": 0.81742334, + "num_input_tokens_seen": 70324465, + "router_z_loss_clip": 1.05517578, + "router_z_loss_mlp": 0.19274902, + "step": 3263, + "time_per_iteration": 2.7465643882751465 + }, + { + "auxiliary_loss_clip": 0.01149823, + "auxiliary_loss_mlp": 0.0105219, + "balance_loss_clip": 1.04710627, + "balance_loss_mlp": 1.03378367, + "epoch": 0.3924727950459929, + "flos": 20558284813440.0, + "grad_norm": 1.8334052746803409, + "language_loss": 0.89949989, + "learning_rate": 2.7730966499964777e-06, + "loss": 0.92151999, + "num_input_tokens_seen": 70341415, + "router_z_loss_clip": 1.02734375, + "router_z_loss_mlp": 0.18395996, + "step": 3264, + "time_per_iteration": 2.744309186935425 + }, + { + "auxiliary_loss_clip": 0.01152301, + "auxiliary_loss_mlp": 0.01047943, + "balance_loss_clip": 1.04600739, + "balance_loss_mlp": 1.02816629, + "epoch": 0.39259303793663197, + "flos": 16216110328320.0, + "grad_norm": 2.6073675726774574, + "language_loss": 0.81075835, + "learning_rate": 2.772378169853408e-06, + "loss": 0.83276081, + "num_input_tokens_seen": 70358985, + "router_z_loss_clip": 1.06103516, + "router_z_loss_mlp": 0.19775391, + "step": 3265, + "time_per_iteration": 2.9005324840545654 + }, + { + "auxiliary_loss_clip": 0.0115242, + "auxiliary_loss_mlp": 0.01042801, + "balance_loss_clip": 1.05107903, + "balance_loss_mlp": 1.02465725, + "epoch": 0.3927132808272711, + "flos": 16797001075200.0, + "grad_norm": 1.90277468774072, + "language_loss": 0.74226731, + "learning_rate": 2.771659572541183e-06, + "loss": 0.76421952, + "num_input_tokens_seen": 70376915, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.18139648, + "step": 3266, + "time_per_iteration": 2.7536911964416504 + }, + { + "auxiliary_loss_clip": 0.01153028, + "auxiliary_loss_mlp": 0.01048702, + "balance_loss_clip": 1.05118668, + "balance_loss_mlp": 1.03028381, + "epoch": 0.3928335237179102, + "flos": 20267228908800.0, + "grad_norm": 1.9422176229757262, + "language_loss": 0.86971378, + "learning_rate": 2.7709408581688143e-06, + "loss": 0.89173108, + "num_input_tokens_seen": 70396900, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.1842041, + "step": 3267, + "time_per_iteration": 2.7639124393463135 + }, + { + "auxiliary_loss_clip": 0.01150282, + "auxiliary_loss_mlp": 0.01051458, + "balance_loss_clip": 1.04997277, + "balance_loss_mlp": 1.03448272, + "epoch": 0.39295376660854925, + "flos": 24973250209920.0, + "grad_norm": 1.6645410744602676, + "language_loss": 0.88154775, + "learning_rate": 2.7702220268453307e-06, + "loss": 0.90356517, + "num_input_tokens_seen": 70417260, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.16992188, + "step": 3268, + "time_per_iteration": 2.7692461013793945 + }, + { + "auxiliary_loss_clip": 0.01157646, + "auxiliary_loss_mlp": 0.01045083, + "balance_loss_clip": 1.04917169, + "balance_loss_mlp": 1.02651024, + "epoch": 0.39307400949918836, + "flos": 18697788984960.0, + "grad_norm": 3.672383011052719, + "language_loss": 0.84874749, + "learning_rate": 2.7695030786797785e-06, + "loss": 0.87077481, + "num_input_tokens_seen": 70433155, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.18566895, + "step": 3269, + "time_per_iteration": 2.8095428943634033 + }, + { + "auxiliary_loss_clip": 0.01150793, + "auxiliary_loss_mlp": 0.01043939, + "balance_loss_clip": 1.04877245, + "balance_loss_mlp": 1.02441263, + "epoch": 0.39319425238982747, + "flos": 22415476590720.0, + "grad_norm": 2.5011367645535945, + "language_loss": 0.74809831, + "learning_rate": 2.7687840137812206e-06, + "loss": 0.77004564, + "num_input_tokens_seen": 70451240, + "router_z_loss_clip": 1.02050781, + "router_z_loss_mlp": 0.19519043, + "step": 3270, + "time_per_iteration": 2.7784318923950195 + }, + { + "auxiliary_loss_clip": 0.01048918, + "auxiliary_loss_mlp": 0.01010417, + "balance_loss_clip": 1.01527882, + "balance_loss_mlp": 1.00817573, + "epoch": 0.3933144952804665, + "flos": 66192954762240.0, + "grad_norm": 0.7997775859537276, + "language_loss": 0.62046647, + "learning_rate": 2.7680648322587395e-06, + "loss": 0.64105982, + "num_input_tokens_seen": 70516115, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.02246094, + "step": 3271, + "time_per_iteration": 3.331458568572998 + }, + { + "auxiliary_loss_clip": 0.01148951, + "auxiliary_loss_mlp": 0.01045362, + "balance_loss_clip": 1.04794121, + "balance_loss_mlp": 1.02720606, + "epoch": 0.39343473817110564, + "flos": 15487159720320.0, + "grad_norm": 1.7610303423787208, + "language_loss": 0.80860454, + "learning_rate": 2.7673455342214334e-06, + "loss": 0.83054769, + "num_input_tokens_seen": 70533105, + "router_z_loss_clip": 1.00927734, + "router_z_loss_mlp": 0.18151855, + "step": 3272, + "time_per_iteration": 2.8237128257751465 + }, + { + "auxiliary_loss_clip": 0.01150705, + "auxiliary_loss_mlp": 0.01044223, + "balance_loss_clip": 1.05056334, + "balance_loss_mlp": 1.02671146, + "epoch": 0.39355498106174475, + "flos": 21324905809920.0, + "grad_norm": 1.9796362007477397, + "language_loss": 0.7627793, + "learning_rate": 2.7666261197784198e-06, + "loss": 0.78472853, + "num_input_tokens_seen": 70551920, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.17504883, + "step": 3273, + "time_per_iteration": 2.723811149597168 + }, + { + "auxiliary_loss_clip": 0.0114995, + "auxiliary_loss_mlp": 0.01055169, + "balance_loss_clip": 1.04872656, + "balance_loss_mlp": 1.03708553, + "epoch": 0.3936752239523838, + "flos": 13296357400320.0, + "grad_norm": 2.15309710642202, + "language_loss": 0.76844484, + "learning_rate": 2.7659065890388336e-06, + "loss": 0.79049599, + "num_input_tokens_seen": 70567920, + "router_z_loss_clip": 1.01171875, + "router_z_loss_mlp": 0.18066406, + "step": 3274, + "time_per_iteration": 2.7756309509277344 + }, + { + "auxiliary_loss_clip": 0.01147758, + "auxiliary_loss_mlp": 0.01046097, + "balance_loss_clip": 1.04715252, + "balance_loss_mlp": 1.02740526, + "epoch": 0.3937954668430229, + "flos": 16800161472000.0, + "grad_norm": 1.8201438648118384, + "language_loss": 0.85207379, + "learning_rate": 2.7651869421118266e-06, + "loss": 0.87401235, + "num_input_tokens_seen": 70584530, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.18701172, + "step": 3275, + "time_per_iteration": 3.0542197227478027 + }, + { + "auxiliary_loss_clip": 0.01152996, + "auxiliary_loss_mlp": 0.01044309, + "balance_loss_clip": 1.05250263, + "balance_loss_mlp": 1.02622485, + "epoch": 0.393915709733662, + "flos": 21064229832960.0, + "grad_norm": 1.6715641472499352, + "language_loss": 0.82825112, + "learning_rate": 2.76446717910657e-06, + "loss": 0.85022414, + "num_input_tokens_seen": 70605235, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.18078613, + "step": 3276, + "time_per_iteration": 2.744682788848877 + }, + { + "auxiliary_loss_clip": 0.01150972, + "auxiliary_loss_mlp": 0.01047034, + "balance_loss_clip": 1.04997158, + "balance_loss_mlp": 1.02844918, + "epoch": 0.3940359526243011, + "flos": 17165265264000.0, + "grad_norm": 2.5885960737566256, + "language_loss": 0.77290452, + "learning_rate": 2.763747300132249e-06, + "loss": 0.79488456, + "num_input_tokens_seen": 70622675, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.18579102, + "step": 3277, + "time_per_iteration": 3.522928476333618 + }, + { + "auxiliary_loss_clip": 0.01150511, + "auxiliary_loss_mlp": 0.01047867, + "balance_loss_clip": 1.04902494, + "balance_loss_mlp": 1.02955604, + "epoch": 0.3941561955149402, + "flos": 20995856294400.0, + "grad_norm": 1.599665622351703, + "language_loss": 0.86334354, + "learning_rate": 2.7630273052980704e-06, + "loss": 0.88532734, + "num_input_tokens_seen": 70643265, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.18310547, + "step": 3278, + "time_per_iteration": 2.735008716583252 + }, + { + "auxiliary_loss_clip": 0.01151795, + "auxiliary_loss_mlp": 0.01047344, + "balance_loss_clip": 1.04997373, + "balance_loss_mlp": 1.02810383, + "epoch": 0.39427643840557924, + "flos": 18843406721280.0, + "grad_norm": 2.065897833044294, + "language_loss": 0.67357349, + "learning_rate": 2.7623071947132554e-06, + "loss": 0.69556493, + "num_input_tokens_seen": 70660295, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.19238281, + "step": 3279, + "time_per_iteration": 2.7708163261413574 + }, + { + "auxiliary_loss_clip": 0.01155436, + "auxiliary_loss_mlp": 0.01045757, + "balance_loss_clip": 1.04912233, + "balance_loss_mlp": 1.02692151, + "epoch": 0.39439668129621835, + "flos": 23258659426560.0, + "grad_norm": 3.217902840619215, + "language_loss": 0.7897948, + "learning_rate": 2.7615869684870458e-06, + "loss": 0.81180668, + "num_input_tokens_seen": 70679605, + "router_z_loss_clip": 1.06396484, + "router_z_loss_mlp": 0.18823242, + "step": 3280, + "time_per_iteration": 3.641554355621338 + }, + { + "auxiliary_loss_clip": 0.01152029, + "auxiliary_loss_mlp": 0.01047442, + "balance_loss_clip": 1.0497663, + "balance_loss_mlp": 1.02909613, + "epoch": 0.39451692418685746, + "flos": 26652289507200.0, + "grad_norm": 1.7255273133719427, + "language_loss": 0.85042232, + "learning_rate": 2.7608666267286986e-06, + "loss": 0.87241703, + "num_input_tokens_seen": 70699835, + "router_z_loss_clip": 1.02099609, + "router_z_loss_mlp": 0.18334961, + "step": 3281, + "time_per_iteration": 2.776294708251953 + }, + { + "auxiliary_loss_clip": 0.01151656, + "auxiliary_loss_mlp": 0.01048227, + "balance_loss_clip": 1.04880476, + "balance_loss_mlp": 1.03019071, + "epoch": 0.3946371670774965, + "flos": 18258709132800.0, + "grad_norm": 2.277771650893221, + "language_loss": 0.86557734, + "learning_rate": 2.760146169547489e-06, + "loss": 0.88757616, + "num_input_tokens_seen": 70716600, + "router_z_loss_clip": 1.02783203, + "router_z_loss_mlp": 0.18011475, + "step": 3282, + "time_per_iteration": 2.7717387676239014 + }, + { + "auxiliary_loss_clip": 0.01152586, + "auxiliary_loss_mlp": 0.01059496, + "balance_loss_clip": 1.05054808, + "balance_loss_mlp": 1.04147172, + "epoch": 0.39475740996813563, + "flos": 24206126423040.0, + "grad_norm": 1.497454127022061, + "language_loss": 0.76667523, + "learning_rate": 2.75942559705271e-06, + "loss": 0.78879607, + "num_input_tokens_seen": 70736335, + "router_z_loss_clip": 1.02050781, + "router_z_loss_mlp": 0.18041992, + "step": 3283, + "time_per_iteration": 3.5566139221191406 + }, + { + "auxiliary_loss_clip": 0.01150206, + "auxiliary_loss_mlp": 0.01044817, + "balance_loss_clip": 1.04690897, + "balance_loss_mlp": 1.02604175, + "epoch": 0.39487765285877474, + "flos": 19317858491520.0, + "grad_norm": 1.965187093394609, + "language_loss": 0.89621073, + "learning_rate": 2.7587049093536713e-06, + "loss": 0.91816097, + "num_input_tokens_seen": 70752665, + "router_z_loss_clip": 1.03222656, + "router_z_loss_mlp": 0.18774414, + "step": 3284, + "time_per_iteration": 3.5787930488586426 + }, + { + "auxiliary_loss_clip": 0.01150799, + "auxiliary_loss_mlp": 0.01045694, + "balance_loss_clip": 1.04918814, + "balance_loss_mlp": 1.02842104, + "epoch": 0.3949978957494138, + "flos": 17311744926720.0, + "grad_norm": 1.898161306664852, + "language_loss": 0.80860162, + "learning_rate": 2.757984106559701e-06, + "loss": 0.83056659, + "num_input_tokens_seen": 70771650, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.17285156, + "step": 3285, + "time_per_iteration": 2.791996479034424 + }, + { + "auxiliary_loss_clip": 0.01149391, + "auxiliary_loss_mlp": 0.01050729, + "balance_loss_clip": 1.05076385, + "balance_loss_mlp": 1.03326464, + "epoch": 0.3951181386400529, + "flos": 36317861280000.0, + "grad_norm": 2.317685025561878, + "language_loss": 0.71382725, + "learning_rate": 2.7572631887801446e-06, + "loss": 0.73582846, + "num_input_tokens_seen": 70793275, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.17468262, + "step": 3286, + "time_per_iteration": 2.8976569175720215 + }, + { + "auxiliary_loss_clip": 0.01150828, + "auxiliary_loss_mlp": 0.01039079, + "balance_loss_clip": 1.04679728, + "balance_loss_mlp": 1.01980281, + "epoch": 0.395238381530692, + "flos": 23110348170240.0, + "grad_norm": 1.7098260636362748, + "language_loss": 0.76750672, + "learning_rate": 2.7565421561243654e-06, + "loss": 0.78940576, + "num_input_tokens_seen": 70811440, + "router_z_loss_clip": 1.04101562, + "router_z_loss_mlp": 0.19262695, + "step": 3287, + "time_per_iteration": 2.802684783935547 + }, + { + "auxiliary_loss_clip": 0.01147286, + "auxiliary_loss_mlp": 0.01043207, + "balance_loss_clip": 1.04749966, + "balance_loss_mlp": 1.02669692, + "epoch": 0.3953586244213311, + "flos": 24347614095360.0, + "grad_norm": 1.98792111820388, + "language_loss": 0.82462436, + "learning_rate": 2.7558210087017413e-06, + "loss": 0.84652936, + "num_input_tokens_seen": 70831375, + "router_z_loss_clip": 0.99804688, + "router_z_loss_mlp": 0.16516113, + "step": 3288, + "time_per_iteration": 2.7503390312194824 + }, + { + "auxiliary_loss_clip": 0.01151899, + "auxiliary_loss_mlp": 0.01047283, + "balance_loss_clip": 1.04919624, + "balance_loss_mlp": 1.02766109, + "epoch": 0.3954788673119702, + "flos": 23440080044160.0, + "grad_norm": 1.913626861132055, + "language_loss": 0.73939919, + "learning_rate": 2.7550997466216724e-06, + "loss": 0.76139104, + "num_input_tokens_seen": 70849170, + "router_z_loss_clip": 1.02783203, + "router_z_loss_mlp": 0.19616699, + "step": 3289, + "time_per_iteration": 2.7438743114471436 + }, + { + "auxiliary_loss_clip": 0.01149578, + "auxiliary_loss_mlp": 0.01048697, + "balance_loss_clip": 1.05035198, + "balance_loss_mlp": 1.03093517, + "epoch": 0.3955991102026093, + "flos": 17494063384320.0, + "grad_norm": 1.9369651750986565, + "language_loss": 0.81599349, + "learning_rate": 2.7543783699935714e-06, + "loss": 0.83797628, + "num_input_tokens_seen": 70867200, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.1776123, + "step": 3290, + "time_per_iteration": 2.7491986751556396 + }, + { + "auxiliary_loss_clip": 0.01146356, + "auxiliary_loss_mlp": 0.01048767, + "balance_loss_clip": 1.04778183, + "balance_loss_mlp": 1.03115988, + "epoch": 0.39571935309324835, + "flos": 18221326053120.0, + "grad_norm": 3.033464197590667, + "language_loss": 0.86399651, + "learning_rate": 2.753656878926872e-06, + "loss": 0.8859477, + "num_input_tokens_seen": 70883080, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.17614746, + "step": 3291, + "time_per_iteration": 2.771389961242676 + }, + { + "auxiliary_loss_clip": 0.01147296, + "auxiliary_loss_mlp": 0.01042394, + "balance_loss_clip": 1.04479229, + "balance_loss_mlp": 1.02345192, + "epoch": 0.39583959598388746, + "flos": 17748813617280.0, + "grad_norm": 22.80187180440075, + "language_loss": 0.74322909, + "learning_rate": 2.752935273531023e-06, + "loss": 0.76512599, + "num_input_tokens_seen": 70901230, + "router_z_loss_clip": 1.02490234, + "router_z_loss_mlp": 0.18945312, + "step": 3292, + "time_per_iteration": 2.743710994720459 + }, + { + "auxiliary_loss_clip": 0.01153292, + "auxiliary_loss_mlp": 0.01042964, + "balance_loss_clip": 1.04732609, + "balance_loss_mlp": 1.02210271, + "epoch": 0.39595983887452657, + "flos": 19352368483200.0, + "grad_norm": 1.8139540427067078, + "language_loss": 0.78785098, + "learning_rate": 2.752213553915492e-06, + "loss": 0.80981356, + "num_input_tokens_seen": 70919585, + "router_z_loss_clip": 1.05859375, + "router_z_loss_mlp": 0.20874023, + "step": 3293, + "time_per_iteration": 2.698622465133667 + }, + { + "auxiliary_loss_clip": 0.01046627, + "auxiliary_loss_mlp": 0.01002953, + "balance_loss_clip": 1.01303363, + "balance_loss_mlp": 1.00009179, + "epoch": 0.3960800817651656, + "flos": 60682282940160.0, + "grad_norm": 0.9153231539001255, + "language_loss": 0.66071439, + "learning_rate": 2.751491720189762e-06, + "loss": 0.68121016, + "num_input_tokens_seen": 70977695, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.02856445, + "step": 3294, + "time_per_iteration": 3.2200722694396973 + }, + { + "auxiliary_loss_clip": 0.01149908, + "auxiliary_loss_mlp": 0.01042792, + "balance_loss_clip": 1.04966021, + "balance_loss_mlp": 1.02598405, + "epoch": 0.39620032465580474, + "flos": 16836718538880.0, + "grad_norm": 2.123930975760452, + "language_loss": 0.92197227, + "learning_rate": 2.7507697724633364e-06, + "loss": 0.94389927, + "num_input_tokens_seen": 70994455, + "router_z_loss_clip": 1.00292969, + "router_z_loss_mlp": 0.16809082, + "step": 3295, + "time_per_iteration": 2.699450969696045 + }, + { + "auxiliary_loss_clip": 0.01045147, + "auxiliary_loss_mlp": 0.01004766, + "balance_loss_clip": 1.0115999, + "balance_loss_mlp": 1.00211966, + "epoch": 0.3963205675464438, + "flos": 69071445941760.0, + "grad_norm": 0.7990676227944918, + "language_loss": 0.54597288, + "learning_rate": 2.7500477108457327e-06, + "loss": 0.56647205, + "num_input_tokens_seen": 71046465, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.02648926, + "step": 3296, + "time_per_iteration": 3.1045684814453125 + }, + { + "auxiliary_loss_clip": 0.01152311, + "auxiliary_loss_mlp": 0.010563, + "balance_loss_clip": 1.04922915, + "balance_loss_mlp": 1.03518748, + "epoch": 0.3964408104370829, + "flos": 25667439431040.0, + "grad_norm": 1.9921989251392902, + "language_loss": 0.80841827, + "learning_rate": 2.7493255354464877e-06, + "loss": 0.83050442, + "num_input_tokens_seen": 71064275, + "router_z_loss_clip": 1.03076172, + "router_z_loss_mlp": 0.21099854, + "step": 3297, + "time_per_iteration": 2.827580213546753 + }, + { + "auxiliary_loss_clip": 0.01149211, + "auxiliary_loss_mlp": 0.01045982, + "balance_loss_clip": 1.04859269, + "balance_loss_mlp": 1.02925658, + "epoch": 0.396561053327722, + "flos": 24277480790400.0, + "grad_norm": 2.376279077794355, + "language_loss": 0.75892627, + "learning_rate": 2.748603246375156e-06, + "loss": 0.78087825, + "num_input_tokens_seen": 71082290, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.1673584, + "step": 3298, + "time_per_iteration": 2.798851490020752 + }, + { + "auxiliary_loss_clip": 0.01150756, + "auxiliary_loss_mlp": 0.0104526, + "balance_loss_clip": 1.04988396, + "balance_loss_mlp": 1.02654397, + "epoch": 0.39668129621836107, + "flos": 20522302364160.0, + "grad_norm": 2.0211097722689244, + "language_loss": 0.69293404, + "learning_rate": 2.7478808437413055e-06, + "loss": 0.71489418, + "num_input_tokens_seen": 71101700, + "router_z_loss_clip": 1.00878906, + "router_z_loss_mlp": 0.18713379, + "step": 3299, + "time_per_iteration": 2.8181042671203613 + }, + { + "auxiliary_loss_clip": 0.01155732, + "auxiliary_loss_mlp": 0.01047585, + "balance_loss_clip": 1.04984808, + "balance_loss_mlp": 1.0292263, + "epoch": 0.3968015391090002, + "flos": 27052585649280.0, + "grad_norm": 1.6680930567077306, + "language_loss": 0.6578033, + "learning_rate": 2.7471583276545263e-06, + "loss": 0.67983651, + "num_input_tokens_seen": 71122360, + "router_z_loss_clip": 1.06005859, + "router_z_loss_mlp": 0.18359375, + "step": 3300, + "time_per_iteration": 2.757815361022949 + }, + { + "auxiliary_loss_clip": 0.01150447, + "auxiliary_loss_mlp": 0.01049438, + "balance_loss_clip": 1.04661405, + "balance_loss_mlp": 1.03098488, + "epoch": 0.3969217819996393, + "flos": 12531819392640.0, + "grad_norm": 1.8811491338793813, + "language_loss": 0.7058295, + "learning_rate": 2.7464356982244224e-06, + "loss": 0.72782838, + "num_input_tokens_seen": 71140360, + "router_z_loss_clip": 1.03808594, + "router_z_loss_mlp": 0.18457031, + "step": 3301, + "time_per_iteration": 2.712476968765259 + }, + { + "auxiliary_loss_clip": 0.01044653, + "auxiliary_loss_mlp": 0.0100228, + "balance_loss_clip": 1.0110935, + "balance_loss_mlp": 0.99968129, + "epoch": 0.39704202489027834, + "flos": 66241399230720.0, + "grad_norm": 0.7725178774516546, + "language_loss": 0.61689317, + "learning_rate": 2.745712955560617e-06, + "loss": 0.63736254, + "num_input_tokens_seen": 71196565, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.02600098, + "step": 3302, + "time_per_iteration": 3.2037410736083984 + }, + { + "auxiliary_loss_clip": 0.01153104, + "auxiliary_loss_mlp": 0.01051801, + "balance_loss_clip": 1.04933691, + "balance_loss_mlp": 1.0330615, + "epoch": 0.39716226778091746, + "flos": 16982982720000.0, + "grad_norm": 2.812295683692583, + "language_loss": 0.76936489, + "learning_rate": 2.7449900997727496e-06, + "loss": 0.7914139, + "num_input_tokens_seen": 71214675, + "router_z_loss_clip": 1.03857422, + "router_z_loss_mlp": 0.18737793, + "step": 3303, + "time_per_iteration": 2.7988109588623047 + }, + { + "auxiliary_loss_clip": 0.01146702, + "auxiliary_loss_mlp": 0.01036469, + "balance_loss_clip": 1.04808354, + "balance_loss_mlp": 1.02060246, + "epoch": 0.39728251067155657, + "flos": 23477139901440.0, + "grad_norm": 1.6744967199314524, + "language_loss": 0.84153736, + "learning_rate": 2.744267130970476e-06, + "loss": 0.86336905, + "num_input_tokens_seen": 71234400, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.15881348, + "step": 3304, + "time_per_iteration": 2.7981343269348145 + }, + { + "auxiliary_loss_clip": 0.01152711, + "auxiliary_loss_mlp": 0.01049952, + "balance_loss_clip": 1.0512526, + "balance_loss_mlp": 1.03328633, + "epoch": 0.3974027535621956, + "flos": 20704441253760.0, + "grad_norm": 1.7669208948663793, + "language_loss": 0.772856, + "learning_rate": 2.7435440492634697e-06, + "loss": 0.79488266, + "num_input_tokens_seen": 71253725, + "router_z_loss_clip": 1.01464844, + "router_z_loss_mlp": 0.16662598, + "step": 3305, + "time_per_iteration": 2.7541399002075195 + }, + { + "auxiliary_loss_clip": 0.01156378, + "auxiliary_loss_mlp": 0.01054172, + "balance_loss_clip": 1.04751897, + "balance_loss_mlp": 1.03371572, + "epoch": 0.39752299645283473, + "flos": 21543278544000.0, + "grad_norm": 2.1849955208471177, + "language_loss": 0.67255652, + "learning_rate": 2.7428208547614228e-06, + "loss": 0.69466197, + "num_input_tokens_seen": 71273220, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.20458984, + "step": 3306, + "time_per_iteration": 3.55966854095459 + }, + { + "auxiliary_loss_clip": 0.0115364, + "auxiliary_loss_mlp": 0.0104749, + "balance_loss_clip": 1.04914021, + "balance_loss_mlp": 1.02790403, + "epoch": 0.39764323934347384, + "flos": 19208295031680.0, + "grad_norm": 1.9279833223702527, + "language_loss": 0.77226269, + "learning_rate": 2.742097547574043e-06, + "loss": 0.79427397, + "num_input_tokens_seen": 71291445, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.19592285, + "step": 3307, + "time_per_iteration": 2.742703914642334 + }, + { + "auxiliary_loss_clip": 0.01153553, + "auxiliary_loss_mlp": 0.01049923, + "balance_loss_clip": 1.04993725, + "balance_loss_mlp": 1.02999139, + "epoch": 0.3977634822341129, + "flos": 20850202644480.0, + "grad_norm": 1.7295225554492533, + "language_loss": 0.78021491, + "learning_rate": 2.7413741278110544e-06, + "loss": 0.80224967, + "num_input_tokens_seen": 71310135, + "router_z_loss_clip": 1.03613281, + "router_z_loss_mlp": 0.19934082, + "step": 3308, + "time_per_iteration": 2.748256206512451 + }, + { + "auxiliary_loss_clip": 0.01156329, + "auxiliary_loss_mlp": 0.01048608, + "balance_loss_clip": 1.05088234, + "balance_loss_mlp": 1.02945137, + "epoch": 0.397883725124752, + "flos": 39786042038400.0, + "grad_norm": 2.216116965717415, + "language_loss": 0.69399393, + "learning_rate": 2.7406505955822016e-06, + "loss": 0.71604335, + "num_input_tokens_seen": 71331160, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.19152832, + "step": 3309, + "time_per_iteration": 3.734182357788086 + }, + { + "auxiliary_loss_clip": 0.01152913, + "auxiliary_loss_mlp": 0.01056789, + "balance_loss_clip": 1.0490396, + "balance_loss_mlp": 1.03939688, + "epoch": 0.39800396801539106, + "flos": 17379507934080.0, + "grad_norm": 18.843823593195435, + "language_loss": 0.66121572, + "learning_rate": 2.7399269509972415e-06, + "loss": 0.68331277, + "num_input_tokens_seen": 71345315, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.1739502, + "step": 3310, + "time_per_iteration": 2.8878352642059326 + }, + { + "auxiliary_loss_clip": 0.01156923, + "auxiliary_loss_mlp": 0.010474, + "balance_loss_clip": 1.04825974, + "balance_loss_mlp": 1.02755153, + "epoch": 0.3981242109060302, + "flos": 19202764337280.0, + "grad_norm": 2.8749242334078704, + "language_loss": 0.85169542, + "learning_rate": 2.7392031941659514e-06, + "loss": 0.87373865, + "num_input_tokens_seen": 71363160, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.1986084, + "step": 3311, + "time_per_iteration": 2.7619659900665283 + }, + { + "auxiliary_loss_clip": 0.011543, + "auxiliary_loss_mlp": 0.01053894, + "balance_loss_clip": 1.05192518, + "balance_loss_mlp": 1.03610766, + "epoch": 0.3982444537966693, + "flos": 24565124903040.0, + "grad_norm": 2.2066626109237126, + "language_loss": 0.86189997, + "learning_rate": 2.7384793251981244e-06, + "loss": 0.88398188, + "num_input_tokens_seen": 71382145, + "router_z_loss_clip": 1.02392578, + "router_z_loss_mlp": 0.17785645, + "step": 3312, + "time_per_iteration": 3.4930758476257324 + }, + { + "auxiliary_loss_clip": 0.01156187, + "auxiliary_loss_mlp": 0.01048988, + "balance_loss_clip": 1.05126262, + "balance_loss_mlp": 1.02925837, + "epoch": 0.39836469668730834, + "flos": 26213856099840.0, + "grad_norm": 1.6882259081862308, + "language_loss": 0.8056742, + "learning_rate": 2.737755344203571e-06, + "loss": 0.82772595, + "num_input_tokens_seen": 71402095, + "router_z_loss_clip": 1.04882812, + "router_z_loss_mlp": 0.1973877, + "step": 3313, + "time_per_iteration": 3.7027313709259033 + }, + { + "auxiliary_loss_clip": 0.01148792, + "auxiliary_loss_mlp": 0.01049303, + "balance_loss_clip": 1.04866588, + "balance_loss_mlp": 1.03143322, + "epoch": 0.39848493957794745, + "flos": 27636134002560.0, + "grad_norm": 1.5784943758070944, + "language_loss": 0.80006468, + "learning_rate": 2.7370312512921186e-06, + "loss": 0.82204568, + "num_input_tokens_seen": 71423875, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.17871094, + "step": 3314, + "time_per_iteration": 2.7724690437316895 + }, + { + "auxiliary_loss_clip": 0.01153103, + "auxiliary_loss_mlp": 0.01047612, + "balance_loss_clip": 1.04746222, + "balance_loss_mlp": 1.02967072, + "epoch": 0.39860518246858656, + "flos": 12239326944000.0, + "grad_norm": 2.575644554625546, + "language_loss": 0.7657426, + "learning_rate": 2.736307046573611e-06, + "loss": 0.78774977, + "num_input_tokens_seen": 71439745, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.17944336, + "step": 3315, + "time_per_iteration": 2.686791181564331 + }, + { + "auxiliary_loss_clip": 0.01151067, + "auxiliary_loss_mlp": 0.01051253, + "balance_loss_clip": 1.0483135, + "balance_loss_mlp": 1.03366959, + "epoch": 0.3987254253592256, + "flos": 22379135005440.0, + "grad_norm": 3.7762273572172407, + "language_loss": 0.8182317, + "learning_rate": 2.73558273015791e-06, + "loss": 0.8402549, + "num_input_tokens_seen": 71459575, + "router_z_loss_clip": 1.02636719, + "router_z_loss_mlp": 0.17590332, + "step": 3316, + "time_per_iteration": 2.7371551990509033 + }, + { + "auxiliary_loss_clip": 0.01152284, + "auxiliary_loss_mlp": 0.0105432, + "balance_loss_clip": 1.04930806, + "balance_loss_mlp": 1.0345192, + "epoch": 0.3988456682498647, + "flos": 23514020190720.0, + "grad_norm": 2.128036015023896, + "language_loss": 0.70526958, + "learning_rate": 2.734858302154894e-06, + "loss": 0.72733563, + "num_input_tokens_seen": 71481075, + "router_z_loss_clip": 1.02929688, + "router_z_loss_mlp": 0.19812012, + "step": 3317, + "time_per_iteration": 2.7014336585998535 + }, + { + "auxiliary_loss_clip": 0.01150831, + "auxiliary_loss_mlp": 0.01049078, + "balance_loss_clip": 1.04760849, + "balance_loss_mlp": 1.0299567, + "epoch": 0.39896591114050384, + "flos": 19208761908480.0, + "grad_norm": 2.3825067852060604, + "language_loss": 0.7630347, + "learning_rate": 2.734133762674457e-06, + "loss": 0.78503382, + "num_input_tokens_seen": 71500665, + "router_z_loss_clip": 1.03222656, + "router_z_loss_mlp": 0.19104004, + "step": 3318, + "time_per_iteration": 2.6909282207489014 + }, + { + "auxiliary_loss_clip": 0.01153017, + "auxiliary_loss_mlp": 0.01048026, + "balance_loss_clip": 1.04790163, + "balance_loss_mlp": 1.02934599, + "epoch": 0.3990861540311429, + "flos": 28401031146240.0, + "grad_norm": 2.223693499307636, + "language_loss": 0.71079874, + "learning_rate": 2.7334091118265124e-06, + "loss": 0.73280919, + "num_input_tokens_seen": 71522560, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.18701172, + "step": 3319, + "time_per_iteration": 2.8285651206970215 + }, + { + "auxiliary_loss_clip": 0.01048063, + "auxiliary_loss_mlp": 0.01002665, + "balance_loss_clip": 1.01440358, + "balance_loss_mlp": 1.00038779, + "epoch": 0.399206396921782, + "flos": 61758563086080.0, + "grad_norm": 0.6848301111768731, + "language_loss": 0.57803082, + "learning_rate": 2.732684349720989e-06, + "loss": 0.5985381, + "num_input_tokens_seen": 71590520, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.02282715, + "step": 3320, + "time_per_iteration": 3.3044302463531494 + }, + { + "auxiliary_loss_clip": 0.01156908, + "auxiliary_loss_mlp": 0.01049198, + "balance_loss_clip": 1.04969931, + "balance_loss_mlp": 1.03117394, + "epoch": 0.3993266398124211, + "flos": 28074567409920.0, + "grad_norm": 1.6846502946037871, + "language_loss": 0.75573158, + "learning_rate": 2.7319594764678318e-06, + "loss": 0.77779263, + "num_input_tokens_seen": 71612620, + "router_z_loss_clip": 1.07177734, + "router_z_loss_mlp": 0.18029785, + "step": 3321, + "time_per_iteration": 2.7604992389678955 + }, + { + "auxiliary_loss_clip": 0.01151671, + "auxiliary_loss_mlp": 0.01048078, + "balance_loss_clip": 1.04626095, + "balance_loss_mlp": 1.02895665, + "epoch": 0.39944688270306017, + "flos": 23225083188480.0, + "grad_norm": 2.8195473978039307, + "language_loss": 0.8354919, + "learning_rate": 2.7312344921770044e-06, + "loss": 0.85748935, + "num_input_tokens_seen": 71634320, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.19116211, + "step": 3322, + "time_per_iteration": 2.7698495388031006 + }, + { + "auxiliary_loss_clip": 0.01151079, + "auxiliary_loss_mlp": 0.0104101, + "balance_loss_clip": 1.04829216, + "balance_loss_mlp": 1.02279532, + "epoch": 0.3995671255936993, + "flos": 19390433921280.0, + "grad_norm": 1.9646083668702188, + "language_loss": 0.78495938, + "learning_rate": 2.7305093969584857e-06, + "loss": 0.80688024, + "num_input_tokens_seen": 71653145, + "router_z_loss_clip": 1.02832031, + "router_z_loss_mlp": 0.18200684, + "step": 3323, + "time_per_iteration": 2.752204179763794 + }, + { + "auxiliary_loss_clip": 0.011512, + "auxiliary_loss_mlp": 0.01045886, + "balance_loss_clip": 1.04758751, + "balance_loss_mlp": 1.02703857, + "epoch": 0.3996873684843384, + "flos": 23842638743040.0, + "grad_norm": 2.33427680285535, + "language_loss": 0.80180728, + "learning_rate": 2.729784190922272e-06, + "loss": 0.82377815, + "num_input_tokens_seen": 71674580, + "router_z_loss_clip": 1.03515625, + "router_z_loss_mlp": 0.18835449, + "step": 3324, + "time_per_iteration": 2.824469804763794 + }, + { + "auxiliary_loss_clip": 0.01049063, + "auxiliary_loss_mlp": 0.01004003, + "balance_loss_clip": 1.01523876, + "balance_loss_mlp": 1.00178576, + "epoch": 0.39980761137497745, + "flos": 66576877280640.0, + "grad_norm": 0.9420498330278071, + "language_loss": 0.57152843, + "learning_rate": 2.729058874178378e-06, + "loss": 0.59205914, + "num_input_tokens_seen": 71745260, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.0222168, + "step": 3325, + "time_per_iteration": 3.410088539123535 + }, + { + "auxiliary_loss_clip": 0.01154818, + "auxiliary_loss_mlp": 0.01043207, + "balance_loss_clip": 1.04976046, + "balance_loss_mlp": 1.02455032, + "epoch": 0.39992785426561656, + "flos": 28549162834560.0, + "grad_norm": 1.747300881713442, + "language_loss": 0.69462806, + "learning_rate": 2.7283334468368315e-06, + "loss": 0.71660829, + "num_input_tokens_seen": 71766540, + "router_z_loss_clip": 1.05126953, + "router_z_loss_mlp": 0.18652344, + "step": 3326, + "time_per_iteration": 2.8962526321411133 + }, + { + "auxiliary_loss_clip": 0.01155934, + "auxiliary_loss_mlp": 0.0104898, + "balance_loss_clip": 1.04955041, + "balance_loss_mlp": 1.0300734, + "epoch": 0.4000480971562556, + "flos": 15049408671360.0, + "grad_norm": 2.316047367053236, + "language_loss": 0.73331267, + "learning_rate": 2.72760790900768e-06, + "loss": 0.75536191, + "num_input_tokens_seen": 71783125, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.18920898, + "step": 3327, + "time_per_iteration": 2.7000584602355957 + }, + { + "auxiliary_loss_clip": 0.01156363, + "auxiliary_loss_mlp": 0.01053067, + "balance_loss_clip": 1.0507201, + "balance_loss_mlp": 1.03489947, + "epoch": 0.4001683400468947, + "flos": 23915609222400.0, + "grad_norm": 1.7929348063438832, + "language_loss": 0.78979594, + "learning_rate": 2.7268822608009875e-06, + "loss": 0.81189024, + "num_input_tokens_seen": 71802500, + "router_z_loss_clip": 1.05566406, + "router_z_loss_mlp": 0.18164062, + "step": 3328, + "time_per_iteration": 2.763673782348633 + }, + { + "auxiliary_loss_clip": 0.01157199, + "auxiliary_loss_mlp": 0.01053127, + "balance_loss_clip": 1.0505383, + "balance_loss_mlp": 1.03354144, + "epoch": 0.40028858293753383, + "flos": 24352677912960.0, + "grad_norm": 3.091591925229758, + "language_loss": 0.78440523, + "learning_rate": 2.726156502326834e-06, + "loss": 0.80650848, + "num_input_tokens_seen": 71823800, + "router_z_loss_clip": 1.06738281, + "router_z_loss_mlp": 0.19580078, + "step": 3329, + "time_per_iteration": 3.0388643741607666 + }, + { + "auxiliary_loss_clip": 0.01046156, + "auxiliary_loss_mlp": 0.01002881, + "balance_loss_clip": 1.01230884, + "balance_loss_mlp": 1.00050855, + "epoch": 0.4004088258281729, + "flos": 66787025800320.0, + "grad_norm": 0.7015214628370896, + "language_loss": 0.60270715, + "learning_rate": 2.725430633695316e-06, + "loss": 0.62319744, + "num_input_tokens_seen": 71886880, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.02368164, + "step": 3330, + "time_per_iteration": 3.395540237426758 + }, + { + "auxiliary_loss_clip": 0.01046738, + "auxiliary_loss_mlp": 0.01004387, + "balance_loss_clip": 1.01290679, + "balance_loss_mlp": 1.00225341, + "epoch": 0.400529068718812, + "flos": 58598386473600.0, + "grad_norm": 0.8897672710433326, + "language_loss": 0.57936883, + "learning_rate": 2.7247046550165485e-06, + "loss": 0.5998801, + "num_input_tokens_seen": 71939005, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.0213623, + "step": 3331, + "time_per_iteration": 3.144261598587036 + }, + { + "auxiliary_loss_clip": 0.01157698, + "auxiliary_loss_mlp": 0.01058883, + "balance_loss_clip": 1.05165553, + "balance_loss_mlp": 1.04090655, + "epoch": 0.4006493116094511, + "flos": 25377460934400.0, + "grad_norm": 1.4549624200450821, + "language_loss": 0.7588799, + "learning_rate": 2.7239785664006606e-06, + "loss": 0.78104568, + "num_input_tokens_seen": 71962545, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.1796875, + "step": 3332, + "time_per_iteration": 2.868840217590332 + }, + { + "auxiliary_loss_clip": 0.01045236, + "auxiliary_loss_mlp": 0.01005783, + "balance_loss_clip": 1.01157713, + "balance_loss_mlp": 1.00367284, + "epoch": 0.40076955450009016, + "flos": 60280729822080.0, + "grad_norm": 0.7770568158588249, + "language_loss": 0.61782205, + "learning_rate": 2.7232523679578002e-06, + "loss": 0.63833225, + "num_input_tokens_seen": 72025625, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.02111816, + "step": 3333, + "time_per_iteration": 3.3573241233825684 + }, + { + "auxiliary_loss_clip": 0.01151814, + "auxiliary_loss_mlp": 0.01045071, + "balance_loss_clip": 1.04985416, + "balance_loss_mlp": 1.02710605, + "epoch": 0.4008897973907293, + "flos": 16617268396800.0, + "grad_norm": 2.4610710940560927, + "language_loss": 0.7891382, + "learning_rate": 2.7225260597981295e-06, + "loss": 0.81110704, + "num_input_tokens_seen": 72043330, + "router_z_loss_clip": 1.01855469, + "router_z_loss_mlp": 0.17980957, + "step": 3334, + "time_per_iteration": 3.5253334045410156 + }, + { + "auxiliary_loss_clip": 0.01153702, + "auxiliary_loss_mlp": 0.01050642, + "balance_loss_clip": 1.04803658, + "balance_loss_mlp": 1.03161621, + "epoch": 0.4010100402813684, + "flos": 15377344865280.0, + "grad_norm": 2.729005049530467, + "language_loss": 0.78708518, + "learning_rate": 2.721799642031831e-06, + "loss": 0.80912864, + "num_input_tokens_seen": 72059500, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.19042969, + "step": 3335, + "time_per_iteration": 2.881889820098877 + }, + { + "auxiliary_loss_clip": 0.01154536, + "auxiliary_loss_mlp": 0.01055709, + "balance_loss_clip": 1.0505687, + "balance_loss_mlp": 1.03797102, + "epoch": 0.40113028317200744, + "flos": 13298835438720.0, + "grad_norm": 1.8881811998448998, + "language_loss": 0.77444434, + "learning_rate": 2.721073114769101e-06, + "loss": 0.79654682, + "num_input_tokens_seen": 72077175, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.17749023, + "step": 3336, + "time_per_iteration": 2.905184268951416 + }, + { + "auxiliary_loss_clip": 0.01149623, + "auxiliary_loss_mlp": 0.01046441, + "balance_loss_clip": 1.04909229, + "balance_loss_mlp": 1.02745116, + "epoch": 0.40125052606264655, + "flos": 20668027841280.0, + "grad_norm": 2.0589458831236196, + "language_loss": 0.75229508, + "learning_rate": 2.7203464781201523e-06, + "loss": 0.77425575, + "num_input_tokens_seen": 72096490, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.19006348, + "step": 3337, + "time_per_iteration": 3.7676093578338623 + }, + { + "auxiliary_loss_clip": 0.01155064, + "auxiliary_loss_mlp": 0.01043976, + "balance_loss_clip": 1.05082977, + "balance_loss_mlp": 1.0261898, + "epoch": 0.40137076895328566, + "flos": 24607679541120.0, + "grad_norm": 1.8661850248113159, + "language_loss": 0.78282815, + "learning_rate": 2.719619732195215e-06, + "loss": 0.80481851, + "num_input_tokens_seen": 72118130, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.17797852, + "step": 3338, + "time_per_iteration": 3.0571370124816895 + }, + { + "auxiliary_loss_clip": 0.01154541, + "auxiliary_loss_mlp": 0.01045281, + "balance_loss_clip": 1.04929113, + "balance_loss_mlp": 1.02726865, + "epoch": 0.4014910118439247, + "flos": 24206593299840.0, + "grad_norm": 1.480833723871309, + "language_loss": 0.72777849, + "learning_rate": 2.7188928771045377e-06, + "loss": 0.74977672, + "num_input_tokens_seen": 72139450, + "router_z_loss_clip": 1.05273438, + "router_z_loss_mlp": 0.18017578, + "step": 3339, + "time_per_iteration": 2.959488868713379 + }, + { + "auxiliary_loss_clip": 0.01152917, + "auxiliary_loss_mlp": 0.01044427, + "balance_loss_clip": 1.04768109, + "balance_loss_mlp": 1.0261879, + "epoch": 0.4016112547345638, + "flos": 26725080418560.0, + "grad_norm": 1.6310024202649922, + "language_loss": 0.8024286, + "learning_rate": 2.7181659129583815e-06, + "loss": 0.82440209, + "num_input_tokens_seen": 72159040, + "router_z_loss_clip": 1.05273438, + "router_z_loss_mlp": 0.18237305, + "step": 3340, + "time_per_iteration": 2.8457303047180176 + }, + { + "auxiliary_loss_clip": 0.01153806, + "auxiliary_loss_mlp": 0.01043501, + "balance_loss_clip": 1.0499475, + "balance_loss_mlp": 1.02491605, + "epoch": 0.4017314976252029, + "flos": 21288025520640.0, + "grad_norm": 2.044421073947847, + "language_loss": 0.75880265, + "learning_rate": 2.7174388398670276e-06, + "loss": 0.78077567, + "num_input_tokens_seen": 72178220, + "router_z_loss_clip": 1.03759766, + "router_z_loss_mlp": 0.18591309, + "step": 3341, + "time_per_iteration": 4.387508153915405 + }, + { + "auxiliary_loss_clip": 0.01154528, + "auxiliary_loss_mlp": 0.01042145, + "balance_loss_clip": 1.04886508, + "balance_loss_mlp": 1.02293992, + "epoch": 0.401851740515842, + "flos": 25484690010240.0, + "grad_norm": 1.8188381191579992, + "language_loss": 0.9251824, + "learning_rate": 2.716711657940773e-06, + "loss": 0.9471491, + "num_input_tokens_seen": 72199230, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.1920166, + "step": 3342, + "time_per_iteration": 2.8409857749938965 + }, + { + "auxiliary_loss_clip": 0.0104534, + "auxiliary_loss_mlp": 0.0099912, + "balance_loss_clip": 1.01195633, + "balance_loss_mlp": 0.9971056, + "epoch": 0.4019719834064811, + "flos": 55395334978560.0, + "grad_norm": 0.8166732638733154, + "language_loss": 0.5656116, + "learning_rate": 2.7159843672899284e-06, + "loss": 0.58605623, + "num_input_tokens_seen": 72263430, + "router_z_loss_clip": 0.33203125, + "router_z_loss_mlp": 0.0201416, + "step": 3343, + "time_per_iteration": 3.4218931198120117 + }, + { + "auxiliary_loss_clip": 0.01154752, + "auxiliary_loss_mlp": 0.01042953, + "balance_loss_clip": 1.05060685, + "balance_loss_mlp": 1.02351046, + "epoch": 0.40209222629712016, + "flos": 18180100218240.0, + "grad_norm": 2.00540217616149, + "language_loss": 0.81342399, + "learning_rate": 2.715256968024825e-06, + "loss": 0.835401, + "num_input_tokens_seen": 72280505, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.19445801, + "step": 3344, + "time_per_iteration": 2.8121883869171143 + }, + { + "auxiliary_loss_clip": 0.01151702, + "auxiliary_loss_mlp": 0.01045743, + "balance_loss_clip": 1.04861283, + "balance_loss_mlp": 1.02788544, + "epoch": 0.40221246918775927, + "flos": 25961009287680.0, + "grad_norm": 1.5018575365466558, + "language_loss": 0.82258451, + "learning_rate": 2.7145294602558083e-06, + "loss": 0.84455895, + "num_input_tokens_seen": 72301215, + "router_z_loss_clip": 1.03027344, + "router_z_loss_mlp": 0.17858887, + "step": 3345, + "time_per_iteration": 2.7990081310272217 + }, + { + "auxiliary_loss_clip": 0.01155092, + "auxiliary_loss_mlp": 0.0104188, + "balance_loss_clip": 1.0481559, + "balance_loss_mlp": 1.02359343, + "epoch": 0.4023327120783984, + "flos": 33838912056960.0, + "grad_norm": 2.424907650290101, + "language_loss": 0.70928842, + "learning_rate": 2.713801844093241e-06, + "loss": 0.73125815, + "num_input_tokens_seen": 72322365, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.1829834, + "step": 3346, + "time_per_iteration": 2.8607895374298096 + }, + { + "auxiliary_loss_clip": 0.01155106, + "auxiliary_loss_mlp": 0.01046034, + "balance_loss_clip": 1.05299783, + "balance_loss_mlp": 1.0278306, + "epoch": 0.40245295496903744, + "flos": 26900252069760.0, + "grad_norm": 1.9790178012542334, + "language_loss": 0.88316917, + "learning_rate": 2.7130741196475014e-06, + "loss": 0.90518057, + "num_input_tokens_seen": 72340495, + "router_z_loss_clip": 1.02148438, + "router_z_loss_mlp": 0.18212891, + "step": 3347, + "time_per_iteration": 2.834693431854248 + }, + { + "auxiliary_loss_clip": 0.01160406, + "auxiliary_loss_mlp": 0.01048924, + "balance_loss_clip": 1.04874253, + "balance_loss_mlp": 1.02911198, + "epoch": 0.40257319785967655, + "flos": 36902738436480.0, + "grad_norm": 1.7686288583204894, + "language_loss": 0.78999054, + "learning_rate": 2.7123462870289848e-06, + "loss": 0.81208384, + "num_input_tokens_seen": 72360545, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.19812012, + "step": 3348, + "time_per_iteration": 2.876436233520508 + }, + { + "auxiliary_loss_clip": 0.01153176, + "auxiliary_loss_mlp": 0.0104099, + "balance_loss_clip": 1.04838347, + "balance_loss_mlp": 1.02242947, + "epoch": 0.40269344075031566, + "flos": 24353180703360.0, + "grad_norm": 1.5293549679075127, + "language_loss": 0.81448841, + "learning_rate": 2.711618346348102e-06, + "loss": 0.83643007, + "num_input_tokens_seen": 72381070, + "router_z_loss_clip": 1.04882812, + "router_z_loss_mlp": 0.18554688, + "step": 3349, + "time_per_iteration": 2.8568716049194336 + }, + { + "auxiliary_loss_clip": 0.01149715, + "auxiliary_loss_mlp": 0.0105294, + "balance_loss_clip": 1.04984474, + "balance_loss_mlp": 1.03580987, + "epoch": 0.4028136836409547, + "flos": 14389657614720.0, + "grad_norm": 1.6571117283315369, + "language_loss": 0.63633084, + "learning_rate": 2.7108902977152825e-06, + "loss": 0.65835738, + "num_input_tokens_seen": 72398970, + "router_z_loss_clip": 0.99902344, + "router_z_loss_mlp": 0.17126465, + "step": 3350, + "time_per_iteration": 2.7545721530914307 + }, + { + "auxiliary_loss_clip": 0.01153475, + "auxiliary_loss_mlp": 0.01045402, + "balance_loss_clip": 1.05187511, + "balance_loss_mlp": 1.02707934, + "epoch": 0.4029339265315938, + "flos": 26136037284480.0, + "grad_norm": 2.383567138496257, + "language_loss": 0.75188154, + "learning_rate": 2.7101621412409704e-06, + "loss": 0.77387035, + "num_input_tokens_seen": 72418455, + "router_z_loss_clip": 1.01757812, + "router_z_loss_mlp": 0.18322754, + "step": 3351, + "time_per_iteration": 2.8300461769104004 + }, + { + "auxiliary_loss_clip": 0.01153926, + "auxiliary_loss_mlp": 0.0105357, + "balance_loss_clip": 1.04844403, + "balance_loss_mlp": 1.03593922, + "epoch": 0.40305416942223293, + "flos": 23256325042560.0, + "grad_norm": 2.009653195233429, + "language_loss": 0.85972136, + "learning_rate": 2.7094338770356256e-06, + "loss": 0.8817963, + "num_input_tokens_seen": 72437540, + "router_z_loss_clip": 1.05419922, + "router_z_loss_mlp": 0.17626953, + "step": 3352, + "time_per_iteration": 2.8572146892547607 + }, + { + "auxiliary_loss_clip": 0.01151547, + "auxiliary_loss_mlp": 0.01049885, + "balance_loss_clip": 1.04878521, + "balance_loss_mlp": 1.03056157, + "epoch": 0.403174412312872, + "flos": 27089645506560.0, + "grad_norm": 2.1751446472448612, + "language_loss": 0.64433491, + "learning_rate": 2.708705505209726e-06, + "loss": 0.66634929, + "num_input_tokens_seen": 72458315, + "router_z_loss_clip": 1.02685547, + "router_z_loss_mlp": 0.19311523, + "step": 3353, + "time_per_iteration": 2.8579201698303223 + }, + { + "auxiliary_loss_clip": 0.01154925, + "auxiliary_loss_mlp": 0.01045182, + "balance_loss_clip": 1.05132675, + "balance_loss_mlp": 1.02685928, + "epoch": 0.4032946552035111, + "flos": 21756336065280.0, + "grad_norm": 1.8532907725377867, + "language_loss": 0.91485089, + "learning_rate": 2.7079770258737646e-06, + "loss": 0.93685192, + "num_input_tokens_seen": 72476225, + "router_z_loss_clip": 1.03564453, + "router_z_loss_mlp": 0.18334961, + "step": 3354, + "time_per_iteration": 2.82714581489563 + }, + { + "auxiliary_loss_clip": 0.011531, + "auxiliary_loss_mlp": 0.01054013, + "balance_loss_clip": 1.04897761, + "balance_loss_mlp": 1.03428411, + "epoch": 0.4034148980941502, + "flos": 17343956448000.0, + "grad_norm": 2.0565516810191307, + "language_loss": 0.74988419, + "learning_rate": 2.707248439138251e-06, + "loss": 0.77195525, + "num_input_tokens_seen": 72492460, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.19714355, + "step": 3355, + "time_per_iteration": 2.791820526123047 + }, + { + "auxiliary_loss_clip": 0.01153266, + "auxiliary_loss_mlp": 0.01048673, + "balance_loss_clip": 1.05066967, + "balance_loss_mlp": 1.03100586, + "epoch": 0.40353514098478926, + "flos": 22017838055040.0, + "grad_norm": 2.1033831767011746, + "language_loss": 0.6563015, + "learning_rate": 2.7065197451137114e-06, + "loss": 0.67832083, + "num_input_tokens_seen": 72513840, + "router_z_loss_clip": 1.02539062, + "router_z_loss_mlp": 0.17651367, + "step": 3356, + "time_per_iteration": 2.7335751056671143 + }, + { + "auxiliary_loss_clip": 0.01152561, + "auxiliary_loss_mlp": 0.01046005, + "balance_loss_clip": 1.04713178, + "balance_loss_mlp": 1.02724195, + "epoch": 0.4036553838754284, + "flos": 14246446089600.0, + "grad_norm": 2.194750664745993, + "language_loss": 0.67177439, + "learning_rate": 2.7057909439106894e-06, + "loss": 0.69376004, + "num_input_tokens_seen": 72531695, + "router_z_loss_clip": 1.05419922, + "router_z_loss_mlp": 0.18774414, + "step": 3357, + "time_per_iteration": 2.85661244392395 + }, + { + "auxiliary_loss_clip": 0.01156879, + "auxiliary_loss_mlp": 0.01038966, + "balance_loss_clip": 1.05266416, + "balance_loss_mlp": 1.01964188, + "epoch": 0.40377562676606743, + "flos": 24790644443520.0, + "grad_norm": 1.7847056644946497, + "language_loss": 0.78822875, + "learning_rate": 2.7050620356397417e-06, + "loss": 0.81018722, + "num_input_tokens_seen": 72550645, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.1932373, + "step": 3358, + "time_per_iteration": 2.802541732788086 + }, + { + "auxiliary_loss_clip": 0.01148295, + "auxiliary_loss_mlp": 0.01039978, + "balance_loss_clip": 1.04949141, + "balance_loss_mlp": 1.02299094, + "epoch": 0.40389586965670654, + "flos": 24061226958720.0, + "grad_norm": 1.8220289120565052, + "language_loss": 0.72424603, + "learning_rate": 2.7043330204114437e-06, + "loss": 0.74612874, + "num_input_tokens_seen": 72569355, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.16992188, + "step": 3359, + "time_per_iteration": 2.759000301361084 + }, + { + "auxiliary_loss_clip": 0.0114928, + "auxiliary_loss_mlp": 0.01047608, + "balance_loss_clip": 1.04865217, + "balance_loss_mlp": 1.03020382, + "epoch": 0.40401611254734565, + "flos": 16399613934720.0, + "grad_norm": 2.0304145484850693, + "language_loss": 0.86344206, + "learning_rate": 2.7036038983363862e-06, + "loss": 0.88541096, + "num_input_tokens_seen": 72585960, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.1739502, + "step": 3360, + "time_per_iteration": 2.7885711193084717 + }, + { + "auxiliary_loss_clip": 0.01149654, + "auxiliary_loss_mlp": 0.01055707, + "balance_loss_clip": 1.04960489, + "balance_loss_mlp": 1.03740799, + "epoch": 0.4041363554379847, + "flos": 23988220565760.0, + "grad_norm": 1.7199944811186823, + "language_loss": 0.84543645, + "learning_rate": 2.702874669525177e-06, + "loss": 0.86749005, + "num_input_tokens_seen": 72604440, + "router_z_loss_clip": 1.00048828, + "router_z_loss_mlp": 0.1829834, + "step": 3361, + "time_per_iteration": 3.1248481273651123 + }, + { + "auxiliary_loss_clip": 0.01149439, + "auxiliary_loss_mlp": 0.01051215, + "balance_loss_clip": 1.04909968, + "balance_loss_mlp": 1.03271365, + "epoch": 0.4042565983286238, + "flos": 28401964899840.0, + "grad_norm": 1.9783010372275756, + "language_loss": 0.69886363, + "learning_rate": 2.7021453340884394e-06, + "loss": 0.72087014, + "num_input_tokens_seen": 72622165, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.18505859, + "step": 3362, + "time_per_iteration": 2.906416416168213 + }, + { + "auxiliary_loss_clip": 0.01150225, + "auxiliary_loss_mlp": 0.01050535, + "balance_loss_clip": 1.04933655, + "balance_loss_mlp": 1.03272533, + "epoch": 0.40437684121926293, + "flos": 17710963660800.0, + "grad_norm": 2.978313830754555, + "language_loss": 0.73117071, + "learning_rate": 2.7014158921368125e-06, + "loss": 0.7531783, + "num_input_tokens_seen": 72640490, + "router_z_loss_clip": 1.00878906, + "router_z_loss_mlp": 0.17822266, + "step": 3363, + "time_per_iteration": 3.591325044631958 + }, + { + "auxiliary_loss_clip": 0.01153674, + "auxiliary_loss_mlp": 0.01042134, + "balance_loss_clip": 1.05117261, + "balance_loss_mlp": 1.02418113, + "epoch": 0.404497084109902, + "flos": 24018959629440.0, + "grad_norm": 2.5365357735265297, + "language_loss": 0.85883778, + "learning_rate": 2.700686343780953e-06, + "loss": 0.88079584, + "num_input_tokens_seen": 72660360, + "router_z_loss_clip": 1.02441406, + "router_z_loss_mlp": 0.17956543, + "step": 3364, + "time_per_iteration": 3.0236642360687256 + }, + { + "auxiliary_loss_clip": 0.01152241, + "auxiliary_loss_mlp": 0.01047753, + "balance_loss_clip": 1.04881871, + "balance_loss_mlp": 1.02984738, + "epoch": 0.4046173270005411, + "flos": 22929861306240.0, + "grad_norm": 1.7736780448707699, + "language_loss": 0.88683271, + "learning_rate": 2.699956689131532e-06, + "loss": 0.90883267, + "num_input_tokens_seen": 72680345, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.17895508, + "step": 3365, + "time_per_iteration": 2.8501474857330322 + }, + { + "auxiliary_loss_clip": 0.01151202, + "auxiliary_loss_mlp": 0.01042729, + "balance_loss_clip": 1.04769373, + "balance_loss_mlp": 1.02363181, + "epoch": 0.4047375698911802, + "flos": 20668135582080.0, + "grad_norm": 1.932023433692183, + "language_loss": 0.85490477, + "learning_rate": 2.699226928299238e-06, + "loss": 0.87684417, + "num_input_tokens_seen": 72698365, + "router_z_loss_clip": 1.03515625, + "router_z_loss_mlp": 0.19116211, + "step": 3366, + "time_per_iteration": 3.694854736328125 + }, + { + "auxiliary_loss_clip": 0.01153096, + "auxiliary_loss_mlp": 0.01041466, + "balance_loss_clip": 1.04984653, + "balance_loss_mlp": 1.02285743, + "epoch": 0.40485781278181926, + "flos": 28912865996160.0, + "grad_norm": 2.3102681313880864, + "language_loss": 0.7863552, + "learning_rate": 2.698497061394774e-06, + "loss": 0.80830085, + "num_input_tokens_seen": 72716850, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.18615723, + "step": 3367, + "time_per_iteration": 2.92144513130188 + }, + { + "auxiliary_loss_clip": 0.01157476, + "auxiliary_loss_mlp": 0.01047075, + "balance_loss_clip": 1.05194783, + "balance_loss_mlp": 1.0279181, + "epoch": 0.40497805567245837, + "flos": 23148377694720.0, + "grad_norm": 1.5335397292853483, + "language_loss": 0.80764723, + "learning_rate": 2.6977670885288627e-06, + "loss": 0.82969272, + "num_input_tokens_seen": 72738250, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.19152832, + "step": 3368, + "time_per_iteration": 2.9830052852630615 + }, + { + "auxiliary_loss_clip": 0.01153106, + "auxiliary_loss_mlp": 0.01035524, + "balance_loss_clip": 1.04883862, + "balance_loss_mlp": 1.01782119, + "epoch": 0.4050982985630975, + "flos": 16289404030080.0, + "grad_norm": 2.5320510965831287, + "language_loss": 0.75726688, + "learning_rate": 2.6970370098122378e-06, + "loss": 0.77915317, + "num_input_tokens_seen": 72755235, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.17700195, + "step": 3369, + "time_per_iteration": 3.6078240871429443 + }, + { + "auxiliary_loss_clip": 0.0115252, + "auxiliary_loss_mlp": 0.01043435, + "balance_loss_clip": 1.05048633, + "balance_loss_mlp": 1.02499318, + "epoch": 0.40521854145373654, + "flos": 34459484353920.0, + "grad_norm": 1.720075489448786, + "language_loss": 0.86719131, + "learning_rate": 2.6963068253556535e-06, + "loss": 0.88915086, + "num_input_tokens_seen": 72776620, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.18444824, + "step": 3370, + "time_per_iteration": 3.699099540710449 + }, + { + "auxiliary_loss_clip": 0.01156613, + "auxiliary_loss_mlp": 0.0104979, + "balance_loss_clip": 1.04902673, + "balance_loss_mlp": 1.03025126, + "epoch": 0.40533878434437565, + "flos": 25331099454720.0, + "grad_norm": 2.320183838523711, + "language_loss": 0.8572883, + "learning_rate": 2.6955765352698763e-06, + "loss": 0.87935233, + "num_input_tokens_seen": 72796765, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.19543457, + "step": 3371, + "time_per_iteration": 2.7613372802734375 + }, + { + "auxiliary_loss_clip": 0.01157833, + "auxiliary_loss_mlp": 0.01055359, + "balance_loss_clip": 1.05114067, + "balance_loss_mlp": 1.03621411, + "epoch": 0.40545902723501476, + "flos": 15012061505280.0, + "grad_norm": 2.093061071931514, + "language_loss": 0.731327, + "learning_rate": 2.6948461396656923e-06, + "loss": 0.75345892, + "num_input_tokens_seen": 72814175, + "router_z_loss_clip": 1.06787109, + "router_z_loss_mlp": 0.19152832, + "step": 3372, + "time_per_iteration": 2.7305855751037598 + }, + { + "auxiliary_loss_clip": 0.01158748, + "auxiliary_loss_mlp": 0.01051383, + "balance_loss_clip": 1.05492723, + "balance_loss_mlp": 1.03186846, + "epoch": 0.4055792701256538, + "flos": 25521103422720.0, + "grad_norm": 2.3046027229425103, + "language_loss": 0.74485481, + "learning_rate": 2.6941156386539013e-06, + "loss": 0.76695615, + "num_input_tokens_seen": 72834125, + "router_z_loss_clip": 1.03808594, + "router_z_loss_mlp": 0.19506836, + "step": 3373, + "time_per_iteration": 2.7726759910583496 + }, + { + "auxiliary_loss_clip": 0.01154718, + "auxiliary_loss_mlp": 0.01061444, + "balance_loss_clip": 1.05235243, + "balance_loss_mlp": 1.04259682, + "epoch": 0.4056995130162929, + "flos": 19574583972480.0, + "grad_norm": 2.283513592308261, + "language_loss": 0.81086266, + "learning_rate": 2.6933850323453203e-06, + "loss": 0.83302426, + "num_input_tokens_seen": 72852570, + "router_z_loss_clip": 1.02392578, + "router_z_loss_mlp": 0.18859863, + "step": 3374, + "time_per_iteration": 2.7264814376831055 + }, + { + "auxiliary_loss_clip": 0.01155403, + "auxiliary_loss_mlp": 0.01043422, + "balance_loss_clip": 1.05208564, + "balance_loss_mlp": 1.02579141, + "epoch": 0.405819755906932, + "flos": 15413794191360.0, + "grad_norm": 1.7184233715483552, + "language_loss": 0.7493571, + "learning_rate": 2.6926543208507806e-06, + "loss": 0.77134538, + "num_input_tokens_seen": 72871250, + "router_z_loss_clip": 1.03271484, + "router_z_loss_mlp": 0.1763916, + "step": 3375, + "time_per_iteration": 2.7516090869903564 + }, + { + "auxiliary_loss_clip": 0.01155467, + "auxiliary_loss_mlp": 0.01045374, + "balance_loss_clip": 1.05060172, + "balance_loss_mlp": 1.02621698, + "epoch": 0.4059399987975711, + "flos": 21433930565760.0, + "grad_norm": 2.1649830568043704, + "language_loss": 0.80204219, + "learning_rate": 2.6919235042811316e-06, + "loss": 0.82405055, + "num_input_tokens_seen": 72890035, + "router_z_loss_clip": 1.04833984, + "router_z_loss_mlp": 0.19152832, + "step": 3376, + "time_per_iteration": 2.7580742835998535 + }, + { + "auxiliary_loss_clip": 0.01158045, + "auxiliary_loss_mlp": 0.01049295, + "balance_loss_clip": 1.0512569, + "balance_loss_mlp": 1.028386, + "epoch": 0.4060602416882102, + "flos": 25556942217600.0, + "grad_norm": 1.8369741033032483, + "language_loss": 0.76186717, + "learning_rate": 2.691192582747237e-06, + "loss": 0.78394061, + "num_input_tokens_seen": 72909665, + "router_z_loss_clip": 1.06787109, + "router_z_loss_mlp": 0.20922852, + "step": 3377, + "time_per_iteration": 2.779615879058838 + }, + { + "auxiliary_loss_clip": 0.01154135, + "auxiliary_loss_mlp": 0.01055994, + "balance_loss_clip": 1.04932201, + "balance_loss_mlp": 1.0355258, + "epoch": 0.40618048457884925, + "flos": 23766759262080.0, + "grad_norm": 1.6634679706285702, + "language_loss": 0.74098909, + "learning_rate": 2.6904615563599765e-06, + "loss": 0.76309037, + "num_input_tokens_seen": 72929465, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.20471191, + "step": 3378, + "time_per_iteration": 2.9066948890686035 + }, + { + "auxiliary_loss_clip": 0.0115455, + "auxiliary_loss_mlp": 0.01047989, + "balance_loss_clip": 1.0512569, + "balance_loss_mlp": 1.0291779, + "epoch": 0.40630072746948837, + "flos": 17639681120640.0, + "grad_norm": 2.402990846457433, + "language_loss": 0.83620143, + "learning_rate": 2.6897304252302477e-06, + "loss": 0.8582269, + "num_input_tokens_seen": 72946785, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.18798828, + "step": 3379, + "time_per_iteration": 2.772552013397217 + }, + { + "auxiliary_loss_clip": 0.01048507, + "auxiliary_loss_mlp": 0.01016107, + "balance_loss_clip": 1.01466036, + "balance_loss_mlp": 1.01416349, + "epoch": 0.4064209703601275, + "flos": 60836053063680.0, + "grad_norm": 0.8053819100350683, + "language_loss": 0.54770792, + "learning_rate": 2.688999189468962e-06, + "loss": 0.56835401, + "num_input_tokens_seen": 73003215, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.01940918, + "step": 3380, + "time_per_iteration": 3.2025654315948486 + }, + { + "auxiliary_loss_clip": 0.01152043, + "auxiliary_loss_mlp": 0.01043649, + "balance_loss_clip": 1.0500083, + "balance_loss_mlp": 1.02525508, + "epoch": 0.40654121325076653, + "flos": 24024346669440.0, + "grad_norm": 2.1085171228541113, + "language_loss": 0.75716203, + "learning_rate": 2.6882678491870464e-06, + "loss": 0.77911896, + "num_input_tokens_seen": 73023650, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.18408203, + "step": 3381, + "time_per_iteration": 2.937013864517212 + }, + { + "auxiliary_loss_clip": 0.01156222, + "auxiliary_loss_mlp": 0.01046989, + "balance_loss_clip": 1.05127871, + "balance_loss_mlp": 1.02936947, + "epoch": 0.40666145614140564, + "flos": 27344252085120.0, + "grad_norm": 1.7145966915541515, + "language_loss": 0.71457684, + "learning_rate": 2.6875364044954453e-06, + "loss": 0.73660898, + "num_input_tokens_seen": 73043880, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.17626953, + "step": 3382, + "time_per_iteration": 3.02828311920166 + }, + { + "auxiliary_loss_clip": 0.01148728, + "auxiliary_loss_mlp": 0.01048566, + "balance_loss_clip": 1.04837275, + "balance_loss_mlp": 1.0313282, + "epoch": 0.40678169903204475, + "flos": 26176724415360.0, + "grad_norm": 1.5439040184018658, + "language_loss": 0.8213383, + "learning_rate": 2.6868048555051185e-06, + "loss": 0.84331131, + "num_input_tokens_seen": 73065410, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.17236328, + "step": 3383, + "time_per_iteration": 2.776616096496582 + }, + { + "auxiliary_loss_clip": 0.01155741, + "auxiliary_loss_mlp": 0.01041717, + "balance_loss_clip": 1.04893231, + "balance_loss_mlp": 1.02252388, + "epoch": 0.4069019419226838, + "flos": 28622420622720.0, + "grad_norm": 2.7004177454937546, + "language_loss": 0.85887098, + "learning_rate": 2.686073202327041e-06, + "loss": 0.88084555, + "num_input_tokens_seen": 73084410, + "router_z_loss_clip": 1.06835938, + "router_z_loss_mlp": 0.19177246, + "step": 3384, + "time_per_iteration": 2.8106844425201416 + }, + { + "auxiliary_loss_clip": 0.0115064, + "auxiliary_loss_mlp": 0.01053493, + "balance_loss_clip": 1.04798424, + "balance_loss_mlp": 1.03586197, + "epoch": 0.4070221848133229, + "flos": 25229006023680.0, + "grad_norm": 2.2830539598303004, + "language_loss": 0.73674399, + "learning_rate": 2.6853414450722043e-06, + "loss": 0.75878531, + "num_input_tokens_seen": 73104075, + "router_z_loss_clip": 1.02636719, + "router_z_loss_mlp": 0.1763916, + "step": 3385, + "time_per_iteration": 2.811302900314331 + }, + { + "auxiliary_loss_clip": 0.01152086, + "auxiliary_loss_mlp": 0.01042439, + "balance_loss_clip": 1.04936624, + "balance_loss_mlp": 1.02434325, + "epoch": 0.40714242770396203, + "flos": 18405224709120.0, + "grad_norm": 1.701162985870459, + "language_loss": 0.85395992, + "learning_rate": 2.684609583851616e-06, + "loss": 0.87590516, + "num_input_tokens_seen": 73122250, + "router_z_loss_clip": 1.02636719, + "router_z_loss_mlp": 0.18103027, + "step": 3386, + "time_per_iteration": 2.8032960891723633 + }, + { + "auxiliary_loss_clip": 0.01150729, + "auxiliary_loss_mlp": 0.01047888, + "balance_loss_clip": 1.04719043, + "balance_loss_mlp": 1.02825463, + "epoch": 0.4072626705946011, + "flos": 30228920403840.0, + "grad_norm": 1.5359146691144763, + "language_loss": 0.80596304, + "learning_rate": 2.683877618776297e-06, + "loss": 0.82794929, + "num_input_tokens_seen": 73144505, + "router_z_loss_clip": 1.03515625, + "router_z_loss_mlp": 0.19628906, + "step": 3387, + "time_per_iteration": 2.9101457595825195 + }, + { + "auxiliary_loss_clip": 0.01154401, + "auxiliary_loss_mlp": 0.01044905, + "balance_loss_clip": 1.04843903, + "balance_loss_mlp": 1.02524757, + "epoch": 0.4073829134852402, + "flos": 21834549930240.0, + "grad_norm": 2.2395632596877735, + "language_loss": 0.73953581, + "learning_rate": 2.6831455499572876e-06, + "loss": 0.76152891, + "num_input_tokens_seen": 73162440, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.19641113, + "step": 3388, + "time_per_iteration": 2.8750388622283936 + }, + { + "auxiliary_loss_clip": 0.01150256, + "auxiliary_loss_mlp": 0.01046004, + "balance_loss_clip": 1.04738998, + "balance_loss_mlp": 1.02608466, + "epoch": 0.40750315637587925, + "flos": 25260211964160.0, + "grad_norm": 2.5527191169017827, + "language_loss": 0.78032506, + "learning_rate": 2.682413377505641e-06, + "loss": 0.8022877, + "num_input_tokens_seen": 73181245, + "router_z_loss_clip": 1.02783203, + "router_z_loss_mlp": 0.19934082, + "step": 3389, + "time_per_iteration": 2.9623122215270996 + }, + { + "auxiliary_loss_clip": 0.01152129, + "auxiliary_loss_mlp": 0.01047287, + "balance_loss_clip": 1.0478152, + "balance_loss_mlp": 1.02646089, + "epoch": 0.40762339926651836, + "flos": 19712767593600.0, + "grad_norm": 1.8223256693342589, + "language_loss": 0.76568002, + "learning_rate": 2.6816811015324284e-06, + "loss": 0.78767419, + "num_input_tokens_seen": 73199295, + "router_z_loss_clip": 1.04248047, + "router_z_loss_mlp": 0.20812988, + "step": 3390, + "time_per_iteration": 2.805234909057617 + }, + { + "auxiliary_loss_clip": 0.01047285, + "auxiliary_loss_mlp": 0.01002027, + "balance_loss_clip": 1.01309621, + "balance_loss_mlp": 1.0000838, + "epoch": 0.40774364215715747, + "flos": 71449307314560.0, + "grad_norm": 0.7252000118649196, + "language_loss": 0.56653798, + "learning_rate": 2.6809487221487343e-06, + "loss": 0.58703107, + "num_input_tokens_seen": 73258780, + "router_z_loss_clip": 0.34179688, + "router_z_loss_mlp": 0.01940918, + "step": 3391, + "time_per_iteration": 3.307724952697754 + }, + { + "auxiliary_loss_clip": 0.0115677, + "auxiliary_loss_mlp": 0.01057368, + "balance_loss_clip": 1.05078387, + "balance_loss_mlp": 1.03802013, + "epoch": 0.4078638850477965, + "flos": 15084134144640.0, + "grad_norm": 2.660632226598685, + "language_loss": 0.82202077, + "learning_rate": 2.6802162394656605e-06, + "loss": 0.84416217, + "num_input_tokens_seen": 73275490, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.19348145, + "step": 3392, + "time_per_iteration": 3.656200647354126 + }, + { + "auxiliary_loss_clip": 0.01150603, + "auxiliary_loss_mlp": 0.0104652, + "balance_loss_clip": 1.04680896, + "balance_loss_mlp": 1.02698183, + "epoch": 0.40798412793843564, + "flos": 23842890138240.0, + "grad_norm": 1.6748047818277085, + "language_loss": 0.72339869, + "learning_rate": 2.679483653594324e-06, + "loss": 0.74536997, + "num_input_tokens_seen": 73297260, + "router_z_loss_clip": 1.03808594, + "router_z_loss_mlp": 0.19543457, + "step": 3393, + "time_per_iteration": 3.1809141635894775 + }, + { + "auxiliary_loss_clip": 0.01154649, + "auxiliary_loss_mlp": 0.0104306, + "balance_loss_clip": 1.05002129, + "balance_loss_mlp": 1.02386737, + "epoch": 0.40810437082907475, + "flos": 21065774117760.0, + "grad_norm": 2.1008154894915947, + "language_loss": 0.76694119, + "learning_rate": 2.678750964645857e-06, + "loss": 0.78891826, + "num_input_tokens_seen": 73316340, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.19189453, + "step": 3394, + "time_per_iteration": 3.004035711288452 + }, + { + "auxiliary_loss_clip": 0.01157936, + "auxiliary_loss_mlp": 0.01043724, + "balance_loss_clip": 1.05228567, + "balance_loss_mlp": 1.02448368, + "epoch": 0.4082246137197138, + "flos": 11321377948800.0, + "grad_norm": 2.617434136720845, + "language_loss": 0.83372498, + "learning_rate": 2.6780181727314094e-06, + "loss": 0.8557415, + "num_input_tokens_seen": 73331245, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.19238281, + "step": 3395, + "time_per_iteration": 3.732410430908203 + }, + { + "auxiliary_loss_clip": 0.01152964, + "auxiliary_loss_mlp": 0.01053792, + "balance_loss_clip": 1.05222058, + "balance_loss_mlp": 1.03562438, + "epoch": 0.4083448566103529, + "flos": 19062569554560.0, + "grad_norm": 1.8417309011374572, + "language_loss": 0.77754796, + "learning_rate": 2.6772852779621435e-06, + "loss": 0.7996155, + "num_input_tokens_seen": 73349105, + "router_z_loss_clip": 1.00634766, + "router_z_loss_mlp": 0.18188477, + "step": 3396, + "time_per_iteration": 2.937243700027466 + }, + { + "auxiliary_loss_clip": 0.01147871, + "auxiliary_loss_mlp": 0.01042768, + "balance_loss_clip": 1.05046797, + "balance_loss_mlp": 1.02541137, + "epoch": 0.408465099500992, + "flos": 23550254035200.0, + "grad_norm": 1.884402104579088, + "language_loss": 0.86412179, + "learning_rate": 2.676552280449239e-06, + "loss": 0.88602817, + "num_input_tokens_seen": 73368990, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.17346191, + "step": 3397, + "time_per_iteration": 3.178464651107788 + }, + { + "auxiliary_loss_clip": 0.01150343, + "auxiliary_loss_mlp": 0.01045267, + "balance_loss_clip": 1.04830587, + "balance_loss_mlp": 1.02718258, + "epoch": 0.4085853423916311, + "flos": 12750012558720.0, + "grad_norm": 2.348040081007658, + "language_loss": 0.75799233, + "learning_rate": 2.6758191803038917e-06, + "loss": 0.77994847, + "num_input_tokens_seen": 73387485, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.18103027, + "step": 3398, + "time_per_iteration": 3.6140477657318115 + }, + { + "auxiliary_loss_clip": 0.0115861, + "auxiliary_loss_mlp": 0.0104305, + "balance_loss_clip": 1.05265689, + "balance_loss_mlp": 1.0231899, + "epoch": 0.4087055852822702, + "flos": 24353072962560.0, + "grad_norm": 1.6285238354366522, + "language_loss": 0.82984722, + "learning_rate": 2.6750859776373125e-06, + "loss": 0.8518638, + "num_input_tokens_seen": 73406940, + "router_z_loss_clip": 1.05859375, + "router_z_loss_mlp": 0.19848633, + "step": 3399, + "time_per_iteration": 3.6950299739837646 + }, + { + "auxiliary_loss_clip": 0.0104659, + "auxiliary_loss_mlp": 0.01001283, + "balance_loss_clip": 1.01271987, + "balance_loss_mlp": 0.99930459, + "epoch": 0.4088258281729093, + "flos": 66387950720640.0, + "grad_norm": 0.765981597510387, + "language_loss": 0.60344267, + "learning_rate": 2.674352672560727e-06, + "loss": 0.62392139, + "num_input_tokens_seen": 73468385, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.01977539, + "step": 3400, + "time_per_iteration": 3.3866074085235596 + }, + { + "auxiliary_loss_clip": 0.01154279, + "auxiliary_loss_mlp": 0.01053289, + "balance_loss_clip": 1.05150437, + "balance_loss_mlp": 1.03411984, + "epoch": 0.40894607106354836, + "flos": 20449260057600.0, + "grad_norm": 2.219066220343597, + "language_loss": 0.77134359, + "learning_rate": 2.673619265185377e-06, + "loss": 0.79341924, + "num_input_tokens_seen": 73488225, + "router_z_loss_clip": 1.02685547, + "router_z_loss_mlp": 0.19165039, + "step": 3401, + "time_per_iteration": 2.842529535293579 + }, + { + "auxiliary_loss_clip": 0.01152044, + "auxiliary_loss_mlp": 0.01043576, + "balance_loss_clip": 1.04908454, + "balance_loss_mlp": 1.0237639, + "epoch": 0.40906631395418747, + "flos": 27053627143680.0, + "grad_norm": 1.835240084331055, + "language_loss": 0.78115505, + "learning_rate": 2.672885755622521e-06, + "loss": 0.80311126, + "num_input_tokens_seen": 73510640, + "router_z_loss_clip": 1.02880859, + "router_z_loss_mlp": 0.19824219, + "step": 3402, + "time_per_iteration": 2.787135124206543 + }, + { + "auxiliary_loss_clip": 0.01146939, + "auxiliary_loss_mlp": 0.01041761, + "balance_loss_clip": 1.04604316, + "balance_loss_mlp": 1.02320015, + "epoch": 0.4091865568448266, + "flos": 25484151306240.0, + "grad_norm": 2.686090089455145, + "language_loss": 0.70397162, + "learning_rate": 2.67215214398343e-06, + "loss": 0.72585863, + "num_input_tokens_seen": 73530655, + "router_z_loss_clip": 1.00878906, + "router_z_loss_mlp": 0.18554688, + "step": 3403, + "time_per_iteration": 2.953359603881836 + }, + { + "auxiliary_loss_clip": 0.01155182, + "auxiliary_loss_mlp": 0.01044875, + "balance_loss_clip": 1.05053592, + "balance_loss_mlp": 1.02612329, + "epoch": 0.40930679973546563, + "flos": 28657864368000.0, + "grad_norm": 2.4149767732036915, + "language_loss": 0.79114735, + "learning_rate": 2.671418430379393e-06, + "loss": 0.8131479, + "num_input_tokens_seen": 73549340, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.1875, + "step": 3404, + "time_per_iteration": 2.8574576377868652 + }, + { + "auxiliary_loss_clip": 0.01151257, + "auxiliary_loss_mlp": 0.0103815, + "balance_loss_clip": 1.0476191, + "balance_loss_mlp": 1.0203042, + "epoch": 0.40942704262610474, + "flos": 20886292834560.0, + "grad_norm": 1.870555393012599, + "language_loss": 0.83438277, + "learning_rate": 2.670684614921715e-06, + "loss": 0.85627681, + "num_input_tokens_seen": 73568315, + "router_z_loss_clip": 1.03613281, + "router_z_loss_mlp": 0.1784668, + "step": 3405, + "time_per_iteration": 2.9611833095550537 + }, + { + "auxiliary_loss_clip": 0.01156374, + "auxiliary_loss_mlp": 0.01047578, + "balance_loss_clip": 1.0523572, + "balance_loss_mlp": 1.02889812, + "epoch": 0.4095472855167438, + "flos": 21618080616960.0, + "grad_norm": 2.5220498449704447, + "language_loss": 0.69413936, + "learning_rate": 2.6699506977217128e-06, + "loss": 0.71617889, + "num_input_tokens_seen": 73588490, + "router_z_loss_clip": 1.04003906, + "router_z_loss_mlp": 0.18701172, + "step": 3406, + "time_per_iteration": 2.885054349899292 + }, + { + "auxiliary_loss_clip": 0.01150614, + "auxiliary_loss_mlp": 0.01047354, + "balance_loss_clip": 1.04778993, + "balance_loss_mlp": 1.02890038, + "epoch": 0.4096675284073829, + "flos": 27926112499200.0, + "grad_norm": 2.056688132686968, + "language_loss": 0.7040233, + "learning_rate": 2.6692166788907233e-06, + "loss": 0.72600293, + "num_input_tokens_seen": 73608685, + "router_z_loss_clip": 1.02832031, + "router_z_loss_mlp": 0.18469238, + "step": 3407, + "time_per_iteration": 2.822791576385498 + }, + { + "auxiliary_loss_clip": 0.01148953, + "auxiliary_loss_mlp": 0.01045268, + "balance_loss_clip": 1.04739785, + "balance_loss_mlp": 1.02621818, + "epoch": 0.409787771298022, + "flos": 19206607092480.0, + "grad_norm": 1.7374267669599421, + "language_loss": 0.77734852, + "learning_rate": 2.6684825585400957e-06, + "loss": 0.79929078, + "num_input_tokens_seen": 73627630, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.19055176, + "step": 3408, + "time_per_iteration": 2.7952678203582764 + }, + { + "auxiliary_loss_clip": 0.01046321, + "auxiliary_loss_mlp": 0.01007123, + "balance_loss_clip": 1.01286316, + "balance_loss_mlp": 1.00517964, + "epoch": 0.4099080141886611, + "flos": 59269234832640.0, + "grad_norm": 0.8078921424983415, + "language_loss": 0.65126908, + "learning_rate": 2.6677483367811947e-06, + "loss": 0.67180353, + "num_input_tokens_seen": 73687670, + "router_z_loss_clip": 0.33398438, + "router_z_loss_mlp": 0.01940918, + "step": 3409, + "time_per_iteration": 3.40513277053833 + }, + { + "auxiliary_loss_clip": 0.01151827, + "auxiliary_loss_mlp": 0.01045909, + "balance_loss_clip": 1.04705834, + "balance_loss_mlp": 1.028409, + "epoch": 0.4100282570793002, + "flos": 21906443001600.0, + "grad_norm": 1.7169662599245314, + "language_loss": 0.75794137, + "learning_rate": 2.6670140137254028e-06, + "loss": 0.77991879, + "num_input_tokens_seen": 73707145, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.17492676, + "step": 3410, + "time_per_iteration": 2.808669328689575 + }, + { + "auxiliary_loss_clip": 0.01149866, + "auxiliary_loss_mlp": 0.01040475, + "balance_loss_clip": 1.0472244, + "balance_loss_mlp": 1.02156806, + "epoch": 0.4101484999699393, + "flos": 18551596631040.0, + "grad_norm": 2.473317684825972, + "language_loss": 0.89882898, + "learning_rate": 2.666279589484115e-06, + "loss": 0.92073238, + "num_input_tokens_seen": 73725045, + "router_z_loss_clip": 1.02587891, + "router_z_loss_mlp": 0.18920898, + "step": 3411, + "time_per_iteration": 2.779616594314575 + }, + { + "auxiliary_loss_clip": 0.01147718, + "auxiliary_loss_mlp": 0.01050781, + "balance_loss_clip": 1.0462898, + "balance_loss_mlp": 1.03294706, + "epoch": 0.41026874286057835, + "flos": 19094529680640.0, + "grad_norm": 2.2808041669233057, + "language_loss": 0.81105685, + "learning_rate": 2.6655450641687435e-06, + "loss": 0.83304185, + "num_input_tokens_seen": 73742610, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.17834473, + "step": 3412, + "time_per_iteration": 2.759669780731201 + }, + { + "auxiliary_loss_clip": 0.01147951, + "auxiliary_loss_mlp": 0.01046744, + "balance_loss_clip": 1.04762542, + "balance_loss_mlp": 1.02939928, + "epoch": 0.41038898575121746, + "flos": 31209568588800.0, + "grad_norm": 1.6890549243358017, + "language_loss": 0.69158912, + "learning_rate": 2.664810437890715e-06, + "loss": 0.71353608, + "num_input_tokens_seen": 73764280, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.17358398, + "step": 3413, + "time_per_iteration": 2.806520700454712 + }, + { + "auxiliary_loss_clip": 0.011501, + "auxiliary_loss_mlp": 0.01048044, + "balance_loss_clip": 1.05125356, + "balance_loss_mlp": 1.03066397, + "epoch": 0.41050922864185657, + "flos": 14355865895040.0, + "grad_norm": 1.8582457358552158, + "language_loss": 0.79777122, + "learning_rate": 2.6640757107614714e-06, + "loss": 0.81975269, + "num_input_tokens_seen": 73782375, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.17370605, + "step": 3414, + "time_per_iteration": 2.7223854064941406 + }, + { + "auxiliary_loss_clip": 0.01152958, + "auxiliary_loss_mlp": 0.01044074, + "balance_loss_clip": 1.04936361, + "balance_loss_mlp": 1.0250001, + "epoch": 0.4106294715324956, + "flos": 30956290813440.0, + "grad_norm": 2.3660675362829124, + "language_loss": 0.69473022, + "learning_rate": 2.6633408828924697e-06, + "loss": 0.71670049, + "num_input_tokens_seen": 73801240, + "router_z_loss_clip": 1.03662109, + "router_z_loss_mlp": 0.19091797, + "step": 3415, + "time_per_iteration": 2.9329168796539307 + }, + { + "auxiliary_loss_clip": 0.01149742, + "auxiliary_loss_mlp": 0.01046995, + "balance_loss_clip": 1.05001867, + "balance_loss_mlp": 1.02956665, + "epoch": 0.41074971442313474, + "flos": 24457321209600.0, + "grad_norm": 1.5567152782093496, + "language_loss": 0.69997835, + "learning_rate": 2.662605954395185e-06, + "loss": 0.72194564, + "num_input_tokens_seen": 73821200, + "router_z_loss_clip": 0.99853516, + "router_z_loss_mlp": 0.17431641, + "step": 3416, + "time_per_iteration": 2.7169320583343506 + }, + { + "auxiliary_loss_clip": 0.01148334, + "auxiliary_loss_mlp": 0.01044185, + "balance_loss_clip": 1.04764032, + "balance_loss_mlp": 1.0266608, + "epoch": 0.41086995731377385, + "flos": 21542991235200.0, + "grad_norm": 2.028817491409459, + "language_loss": 0.83918649, + "learning_rate": 2.6618709253811027e-06, + "loss": 0.86111164, + "num_input_tokens_seen": 73840655, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.1751709, + "step": 3417, + "time_per_iteration": 2.7646899223327637 + }, + { + "auxiliary_loss_clip": 0.01143319, + "auxiliary_loss_mlp": 0.01037639, + "balance_loss_clip": 1.04615521, + "balance_loss_mlp": 1.02110422, + "epoch": 0.4109902002044129, + "flos": 20702753314560.0, + "grad_norm": 1.678220265432693, + "language_loss": 0.87569058, + "learning_rate": 2.6611357959617277e-06, + "loss": 0.89750016, + "num_input_tokens_seen": 73860275, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.16552734, + "step": 3418, + "time_per_iteration": 2.7298498153686523 + }, + { + "auxiliary_loss_clip": 0.01147918, + "auxiliary_loss_mlp": 0.01050338, + "balance_loss_clip": 1.04759169, + "balance_loss_mlp": 1.03238487, + "epoch": 0.411110443095052, + "flos": 18179992477440.0, + "grad_norm": 1.8667002861026076, + "language_loss": 0.9132297, + "learning_rate": 2.660400566248578e-06, + "loss": 0.93521231, + "num_input_tokens_seen": 73878400, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.17956543, + "step": 3419, + "time_per_iteration": 2.7812585830688477 + }, + { + "auxiliary_loss_clip": 0.01151089, + "auxiliary_loss_mlp": 0.01054655, + "balance_loss_clip": 1.04757738, + "balance_loss_mlp": 1.03552175, + "epoch": 0.41123068598569107, + "flos": 14575244209920.0, + "grad_norm": 2.302308502354129, + "language_loss": 0.66453385, + "learning_rate": 2.6596652363531876e-06, + "loss": 0.68659127, + "num_input_tokens_seen": 73894275, + "router_z_loss_clip": 1.03808594, + "router_z_loss_mlp": 0.19116211, + "step": 3420, + "time_per_iteration": 2.7513811588287354 + }, + { + "auxiliary_loss_clip": 0.01146498, + "auxiliary_loss_mlp": 0.01052186, + "balance_loss_clip": 1.04773498, + "balance_loss_mlp": 1.03424525, + "epoch": 0.4113509288763302, + "flos": 21177995184000.0, + "grad_norm": 1.4683786853115033, + "language_loss": 0.78046715, + "learning_rate": 2.6589298063871055e-06, + "loss": 0.80245399, + "num_input_tokens_seen": 73914450, + "router_z_loss_clip": 0.98730469, + "router_z_loss_mlp": 0.17944336, + "step": 3421, + "time_per_iteration": 3.5739924907684326 + }, + { + "auxiliary_loss_clip": 0.01150254, + "auxiliary_loss_mlp": 0.01053041, + "balance_loss_clip": 1.04960787, + "balance_loss_mlp": 1.03507638, + "epoch": 0.4114711717669693, + "flos": 18442212739200.0, + "grad_norm": 1.9776937736022473, + "language_loss": 0.70029569, + "learning_rate": 2.658194276461895e-06, + "loss": 0.72232866, + "num_input_tokens_seen": 73932375, + "router_z_loss_clip": 1.00634766, + "router_z_loss_mlp": 0.17956543, + "step": 3422, + "time_per_iteration": 2.760216236114502 + }, + { + "auxiliary_loss_clip": 0.01155269, + "auxiliary_loss_mlp": 0.01056537, + "balance_loss_clip": 1.04777527, + "balance_loss_mlp": 1.03598523, + "epoch": 0.41159141465760835, + "flos": 27233395735680.0, + "grad_norm": 1.782752176544637, + "language_loss": 0.66940475, + "learning_rate": 2.6574586466891368e-06, + "loss": 0.69152284, + "num_input_tokens_seen": 73952850, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.20556641, + "step": 3423, + "time_per_iteration": 2.8988304138183594 + }, + { + "auxiliary_loss_clip": 0.01144473, + "auxiliary_loss_mlp": 0.01042074, + "balance_loss_clip": 1.04600596, + "balance_loss_mlp": 1.02439523, + "epoch": 0.41171165754824746, + "flos": 20006876154240.0, + "grad_norm": 3.10134460317772, + "language_loss": 0.64873701, + "learning_rate": 2.6567229171804247e-06, + "loss": 0.6706025, + "num_input_tokens_seen": 73970735, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.17700195, + "step": 3424, + "time_per_iteration": 3.6419544219970703 + }, + { + "auxiliary_loss_clip": 0.0115277, + "auxiliary_loss_mlp": 0.01044197, + "balance_loss_clip": 1.04637957, + "balance_loss_mlp": 1.02496839, + "epoch": 0.41183190043888657, + "flos": 18004318035840.0, + "grad_norm": 3.4997120481925417, + "language_loss": 0.88065243, + "learning_rate": 2.655987088047368e-06, + "loss": 0.9026221, + "num_input_tokens_seen": 73989080, + "router_z_loss_clip": 1.06347656, + "router_z_loss_mlp": 0.19238281, + "step": 3425, + "time_per_iteration": 2.817669630050659 + }, + { + "auxiliary_loss_clip": 0.01151226, + "auxiliary_loss_mlp": 0.01042571, + "balance_loss_clip": 1.04610062, + "balance_loss_mlp": 1.02400994, + "epoch": 0.4119521433295256, + "flos": 27163370171520.0, + "grad_norm": 1.8347069212032645, + "language_loss": 0.78718168, + "learning_rate": 2.6552511594015912e-06, + "loss": 0.80911964, + "num_input_tokens_seen": 74009470, + "router_z_loss_clip": 1.05126953, + "router_z_loss_mlp": 0.18554688, + "step": 3426, + "time_per_iteration": 2.7564315795898438 + }, + { + "auxiliary_loss_clip": 0.01153712, + "auxiliary_loss_mlp": 0.01057022, + "balance_loss_clip": 1.0470351, + "balance_loss_mlp": 1.03770995, + "epoch": 0.41207238622016473, + "flos": 15122020014720.0, + "grad_norm": 2.365585836393452, + "language_loss": 0.85591722, + "learning_rate": 2.654515131354735e-06, + "loss": 0.87802458, + "num_input_tokens_seen": 74027735, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.1932373, + "step": 3427, + "time_per_iteration": 3.531024694442749 + }, + { + "auxiliary_loss_clip": 0.01150407, + "auxiliary_loss_mlp": 0.01047812, + "balance_loss_clip": 1.04952383, + "balance_loss_mlp": 1.02963305, + "epoch": 0.41219262911080384, + "flos": 27052872958080.0, + "grad_norm": 1.9033101758416013, + "language_loss": 0.84463072, + "learning_rate": 2.653779004018453e-06, + "loss": 0.86661291, + "num_input_tokens_seen": 74048300, + "router_z_loss_clip": 1.00878906, + "router_z_loss_mlp": 0.18188477, + "step": 3428, + "time_per_iteration": 3.5666213035583496 + }, + { + "auxiliary_loss_clip": 0.01148821, + "auxiliary_loss_mlp": 0.01047803, + "balance_loss_clip": 1.0480516, + "balance_loss_mlp": 1.03061295, + "epoch": 0.4123128720014429, + "flos": 24686360282880.0, + "grad_norm": 3.069460935029344, + "language_loss": 0.82485139, + "learning_rate": 2.653042777504417e-06, + "loss": 0.84681761, + "num_input_tokens_seen": 74070890, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.17199707, + "step": 3429, + "time_per_iteration": 2.819185256958008 + }, + { + "auxiliary_loss_clip": 0.01153817, + "auxiliary_loss_mlp": 0.01051432, + "balance_loss_clip": 1.04766417, + "balance_loss_mlp": 1.03091586, + "epoch": 0.412433114892082, + "flos": 26244774731520.0, + "grad_norm": 3.264017701865099, + "language_loss": 0.80162889, + "learning_rate": 2.6523064519243105e-06, + "loss": 0.82368135, + "num_input_tokens_seen": 74090460, + "router_z_loss_clip": 1.06347656, + "router_z_loss_mlp": 0.2052002, + "step": 3430, + "time_per_iteration": 2.766301393508911 + }, + { + "auxiliary_loss_clip": 0.01151093, + "auxiliary_loss_mlp": 0.01049315, + "balance_loss_clip": 1.04831052, + "balance_loss_mlp": 1.02842951, + "epoch": 0.4125533577827211, + "flos": 21361031913600.0, + "grad_norm": 2.1861817447411895, + "language_loss": 0.7947821, + "learning_rate": 2.6515700273898333e-06, + "loss": 0.81678617, + "num_input_tokens_seen": 74108335, + "router_z_loss_clip": 1.02734375, + "router_z_loss_mlp": 0.20861816, + "step": 3431, + "time_per_iteration": 2.718616247177124 + }, + { + "auxiliary_loss_clip": 0.01152041, + "auxiliary_loss_mlp": 0.01049427, + "balance_loss_clip": 1.05012536, + "balance_loss_mlp": 1.03155732, + "epoch": 0.4126736006733602, + "flos": 26067556005120.0, + "grad_norm": 2.4789565091453984, + "language_loss": 0.69214261, + "learning_rate": 2.6508335040127018e-06, + "loss": 0.71415734, + "num_input_tokens_seen": 74128030, + "router_z_loss_clip": 1.01757812, + "router_z_loss_mlp": 0.17871094, + "step": 3432, + "time_per_iteration": 2.768075704574585 + }, + { + "auxiliary_loss_clip": 0.01149577, + "auxiliary_loss_mlp": 0.01047866, + "balance_loss_clip": 1.04831171, + "balance_loss_mlp": 1.02969873, + "epoch": 0.4127938435639993, + "flos": 25666146541440.0, + "grad_norm": 1.4471909377152847, + "language_loss": 0.77242112, + "learning_rate": 2.6500968819046446e-06, + "loss": 0.79439557, + "num_input_tokens_seen": 74148330, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.18164062, + "step": 3433, + "time_per_iteration": 2.759157657623291 + }, + { + "auxiliary_loss_clip": 0.01145347, + "auxiliary_loss_mlp": 0.01042928, + "balance_loss_clip": 1.04641342, + "balance_loss_mlp": 1.02529681, + "epoch": 0.4129140864546384, + "flos": 17995914253440.0, + "grad_norm": 2.449081697521854, + "language_loss": 0.59622562, + "learning_rate": 2.649360161177408e-06, + "loss": 0.61810839, + "num_input_tokens_seen": 74163390, + "router_z_loss_clip": 0.99023438, + "router_z_loss_mlp": 0.1763916, + "step": 3434, + "time_per_iteration": 2.7107527256011963 + }, + { + "auxiliary_loss_clip": 0.01153109, + "auxiliary_loss_mlp": 0.01056539, + "balance_loss_clip": 1.0478586, + "balance_loss_mlp": 1.03675056, + "epoch": 0.41303432934527745, + "flos": 23732895715200.0, + "grad_norm": 1.6743668663294249, + "language_loss": 0.73449868, + "learning_rate": 2.6486233419427504e-06, + "loss": 0.75659513, + "num_input_tokens_seen": 74183205, + "router_z_loss_clip": 1.05273438, + "router_z_loss_mlp": 0.19775391, + "step": 3435, + "time_per_iteration": 2.7569735050201416 + }, + { + "auxiliary_loss_clip": 0.01157783, + "auxiliary_loss_mlp": 0.01042639, + "balance_loss_clip": 1.04985309, + "balance_loss_mlp": 1.02435207, + "epoch": 0.41315457223591656, + "flos": 19755286318080.0, + "grad_norm": 2.307975190969445, + "language_loss": 0.75160551, + "learning_rate": 2.6478864243124484e-06, + "loss": 0.77360976, + "num_input_tokens_seen": 74202870, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.18286133, + "step": 3436, + "time_per_iteration": 2.7380800247192383 + }, + { + "auxiliary_loss_clip": 0.01149604, + "auxiliary_loss_mlp": 0.01040358, + "balance_loss_clip": 1.04867923, + "balance_loss_mlp": 1.02313256, + "epoch": 0.4132748151265556, + "flos": 20923316778240.0, + "grad_norm": 1.801699029136447, + "language_loss": 0.85519606, + "learning_rate": 2.6471494083982903e-06, + "loss": 0.8770957, + "num_input_tokens_seen": 74222255, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.17236328, + "step": 3437, + "time_per_iteration": 2.7297439575195312 + }, + { + "auxiliary_loss_clip": 0.01151914, + "auxiliary_loss_mlp": 0.01047355, + "balance_loss_clip": 1.05042613, + "balance_loss_mlp": 1.02824569, + "epoch": 0.4133950580171947, + "flos": 32232520016640.0, + "grad_norm": 2.2386097732845753, + "language_loss": 0.74798048, + "learning_rate": 2.6464122943120818e-06, + "loss": 0.76997316, + "num_input_tokens_seen": 74242480, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.19116211, + "step": 3438, + "time_per_iteration": 2.7883665561676025 + }, + { + "auxiliary_loss_clip": 0.01152162, + "auxiliary_loss_mlp": 0.01049578, + "balance_loss_clip": 1.0483067, + "balance_loss_mlp": 1.0309099, + "epoch": 0.41351530090783384, + "flos": 23292487059840.0, + "grad_norm": 3.057344139285756, + "language_loss": 0.82134694, + "learning_rate": 2.645675082165642e-06, + "loss": 0.84336436, + "num_input_tokens_seen": 74258690, + "router_z_loss_clip": 1.03710938, + "router_z_loss_mlp": 0.18652344, + "step": 3439, + "time_per_iteration": 2.764751672744751 + }, + { + "auxiliary_loss_clip": 0.01155343, + "auxiliary_loss_mlp": 0.01043491, + "balance_loss_clip": 1.04865575, + "balance_loss_mlp": 1.02421474, + "epoch": 0.4136355437984729, + "flos": 25593571111680.0, + "grad_norm": 2.253919887304796, + "language_loss": 0.75671077, + "learning_rate": 2.644937772070806e-06, + "loss": 0.77869916, + "num_input_tokens_seen": 74277135, + "router_z_loss_clip": 1.06738281, + "router_z_loss_mlp": 0.19274902, + "step": 3440, + "time_per_iteration": 2.7655978202819824 + }, + { + "auxiliary_loss_clip": 0.01146938, + "auxiliary_loss_mlp": 0.01040159, + "balance_loss_clip": 1.04770267, + "balance_loss_mlp": 1.02366054, + "epoch": 0.413755786689112, + "flos": 19828615933440.0, + "grad_norm": 2.9780338293312836, + "language_loss": 0.83666646, + "learning_rate": 2.6442003641394225e-06, + "loss": 0.85853744, + "num_input_tokens_seen": 74294730, + "router_z_loss_clip": 0.99072266, + "router_z_loss_mlp": 0.16503906, + "step": 3441, + "time_per_iteration": 2.946991443634033 + }, + { + "auxiliary_loss_clip": 0.01145956, + "auxiliary_loss_mlp": 0.01040653, + "balance_loss_clip": 1.04675639, + "balance_loss_mlp": 1.02401125, + "epoch": 0.4138760295797511, + "flos": 26870446759680.0, + "grad_norm": 1.703167423605858, + "language_loss": 0.84028518, + "learning_rate": 2.643462858483356e-06, + "loss": 0.86215127, + "num_input_tokens_seen": 74315015, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.16638184, + "step": 3442, + "time_per_iteration": 2.748046398162842 + }, + { + "auxiliary_loss_clip": 0.01152213, + "auxiliary_loss_mlp": 0.01047833, + "balance_loss_clip": 1.04831004, + "balance_loss_mlp": 1.02868772, + "epoch": 0.41399627247039017, + "flos": 16399254798720.0, + "grad_norm": 2.5673984823264626, + "language_loss": 0.72721493, + "learning_rate": 2.6427252552144856e-06, + "loss": 0.74921542, + "num_input_tokens_seen": 74333665, + "router_z_loss_clip": 1.03857422, + "router_z_loss_mlp": 0.19152832, + "step": 3443, + "time_per_iteration": 2.734104633331299 + }, + { + "auxiliary_loss_clip": 0.01150492, + "auxiliary_loss_mlp": 0.01042189, + "balance_loss_clip": 1.05018353, + "balance_loss_mlp": 1.02312791, + "epoch": 0.4141165153610293, + "flos": 22930220442240.0, + "grad_norm": 1.9372545137194215, + "language_loss": 0.7485404, + "learning_rate": 2.6419875544447044e-06, + "loss": 0.77046728, + "num_input_tokens_seen": 74355065, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.19055176, + "step": 3444, + "time_per_iteration": 2.7496516704559326 + }, + { + "auxiliary_loss_clip": 0.01152306, + "auxiliary_loss_mlp": 0.01044694, + "balance_loss_clip": 1.04824376, + "balance_loss_mlp": 1.02603769, + "epoch": 0.4142367582516684, + "flos": 25192556697600.0, + "grad_norm": 1.7630261127021194, + "language_loss": 0.71658695, + "learning_rate": 2.6412497562859218e-06, + "loss": 0.73855698, + "num_input_tokens_seen": 74376345, + "router_z_loss_clip": 1.03955078, + "router_z_loss_mlp": 0.18652344, + "step": 3445, + "time_per_iteration": 2.824213743209839 + }, + { + "auxiliary_loss_clip": 0.01148345, + "auxiliary_loss_mlp": 0.01050846, + "balance_loss_clip": 1.04598093, + "balance_loss_mlp": 1.03226078, + "epoch": 0.41435700114230745, + "flos": 21690476478720.0, + "grad_norm": 3.14317571018206, + "language_loss": 0.76309323, + "learning_rate": 2.6405118608500617e-06, + "loss": 0.78508508, + "num_input_tokens_seen": 74395170, + "router_z_loss_clip": 1.02246094, + "router_z_loss_mlp": 0.18579102, + "step": 3446, + "time_per_iteration": 2.8068203926086426 + }, + { + "auxiliary_loss_clip": 0.01149602, + "auxiliary_loss_mlp": 0.01056956, + "balance_loss_clip": 1.05044937, + "balance_loss_mlp": 1.03900313, + "epoch": 0.41447724403294656, + "flos": 25995160143360.0, + "grad_norm": 1.7426159594662778, + "language_loss": 0.816046, + "learning_rate": 2.6397738682490613e-06, + "loss": 0.83811152, + "num_input_tokens_seen": 74416070, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.17956543, + "step": 3447, + "time_per_iteration": 2.8574862480163574 + }, + { + "auxiliary_loss_clip": 0.01151586, + "auxiliary_loss_mlp": 0.01041584, + "balance_loss_clip": 1.05009508, + "balance_loss_mlp": 1.0244422, + "epoch": 0.41459748692358567, + "flos": 18259678800000.0, + "grad_norm": 1.8003005913529435, + "language_loss": 0.75210357, + "learning_rate": 2.6390357785948734e-06, + "loss": 0.77403533, + "num_input_tokens_seen": 74433185, + "router_z_loss_clip": 1.01660156, + "router_z_loss_mlp": 0.17138672, + "step": 3448, + "time_per_iteration": 2.7706191539764404 + }, + { + "auxiliary_loss_clip": 0.01155259, + "auxiliary_loss_mlp": 0.0104807, + "balance_loss_clip": 1.05189133, + "balance_loss_mlp": 1.0281024, + "epoch": 0.4147177298142247, + "flos": 24168456034560.0, + "grad_norm": 1.7289448198039374, + "language_loss": 0.80180192, + "learning_rate": 2.6382975919994667e-06, + "loss": 0.82383519, + "num_input_tokens_seen": 74453760, + "router_z_loss_clip": 1.03417969, + "router_z_loss_mlp": 0.19970703, + "step": 3449, + "time_per_iteration": 2.729520320892334 + }, + { + "auxiliary_loss_clip": 0.0114758, + "auxiliary_loss_mlp": 0.01038542, + "balance_loss_clip": 1.04947293, + "balance_loss_mlp": 1.02236485, + "epoch": 0.41483797270486383, + "flos": 20084659056000.0, + "grad_norm": 1.7330638001204413, + "language_loss": 0.73060232, + "learning_rate": 2.637559308574822e-06, + "loss": 0.75246358, + "num_input_tokens_seen": 74473505, + "router_z_loss_clip": 0.98242188, + "router_z_loss_mlp": 0.16174316, + "step": 3450, + "time_per_iteration": 3.5335962772369385 + }, + { + "auxiliary_loss_clip": 0.01152733, + "auxiliary_loss_mlp": 0.01054421, + "balance_loss_clip": 1.0497191, + "balance_loss_mlp": 1.03563404, + "epoch": 0.4149582155955029, + "flos": 30081040110720.0, + "grad_norm": 2.0441887724529093, + "language_loss": 0.7180869, + "learning_rate": 2.6368209284329376e-06, + "loss": 0.74015844, + "num_input_tokens_seen": 74494135, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.18774414, + "step": 3451, + "time_per_iteration": 2.8674397468566895 + }, + { + "auxiliary_loss_clip": 0.01152462, + "auxiliary_loss_mlp": 0.0104572, + "balance_loss_clip": 1.04963017, + "balance_loss_mlp": 1.02692044, + "epoch": 0.415078458486142, + "flos": 16764394504320.0, + "grad_norm": 2.0942489668752726, + "language_loss": 0.75875843, + "learning_rate": 2.636082451685825e-06, + "loss": 0.78074026, + "num_input_tokens_seen": 74512335, + "router_z_loss_clip": 1.02734375, + "router_z_loss_mlp": 0.18786621, + "step": 3452, + "time_per_iteration": 3.539066791534424 + }, + { + "auxiliary_loss_clip": 0.01154667, + "auxiliary_loss_mlp": 0.01048262, + "balance_loss_clip": 1.05124557, + "balance_loss_mlp": 1.02972472, + "epoch": 0.4151987013767811, + "flos": 26033692458240.0, + "grad_norm": 1.5073331500148497, + "language_loss": 0.86620927, + "learning_rate": 2.6353438784455094e-06, + "loss": 0.88823855, + "num_input_tokens_seen": 74535620, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.18554688, + "step": 3453, + "time_per_iteration": 2.8760061264038086 + }, + { + "auxiliary_loss_clip": 0.01154821, + "auxiliary_loss_mlp": 0.01047977, + "balance_loss_clip": 1.05270946, + "balance_loss_mlp": 1.02789021, + "epoch": 0.41531894426742016, + "flos": 24608002763520.0, + "grad_norm": 2.6395811522821693, + "language_loss": 0.71516454, + "learning_rate": 2.6346052088240326e-06, + "loss": 0.73719251, + "num_input_tokens_seen": 74555140, + "router_z_loss_clip": 1.02099609, + "router_z_loss_mlp": 0.20092773, + "step": 3454, + "time_per_iteration": 2.728196144104004 + }, + { + "auxiliary_loss_clip": 0.01153172, + "auxiliary_loss_mlp": 0.0104532, + "balance_loss_clip": 1.0506804, + "balance_loss_mlp": 1.02643776, + "epoch": 0.4154391871580593, + "flos": 14975791747200.0, + "grad_norm": 1.9852642360587343, + "language_loss": 0.77401733, + "learning_rate": 2.63386644293345e-06, + "loss": 0.79600227, + "num_input_tokens_seen": 74571485, + "router_z_loss_clip": 1.02441406, + "router_z_loss_mlp": 0.18884277, + "step": 3455, + "time_per_iteration": 2.77968692779541 + }, + { + "auxiliary_loss_clip": 0.01149364, + "auxiliary_loss_mlp": 0.01049162, + "balance_loss_clip": 1.04870009, + "balance_loss_mlp": 1.03057766, + "epoch": 0.4155594300486984, + "flos": 14647173194880.0, + "grad_norm": 2.3167851310457968, + "language_loss": 0.82957047, + "learning_rate": 2.633127580885833e-06, + "loss": 0.85155571, + "num_input_tokens_seen": 74585985, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.18591309, + "step": 3456, + "time_per_iteration": 3.4409356117248535 + }, + { + "auxiliary_loss_clip": 0.01150801, + "auxiliary_loss_mlp": 0.01049555, + "balance_loss_clip": 1.05094898, + "balance_loss_mlp": 1.03149462, + "epoch": 0.41567967293933744, + "flos": 29497276275840.0, + "grad_norm": 1.9213896950189984, + "language_loss": 0.65156841, + "learning_rate": 2.632388622793265e-06, + "loss": 0.67357194, + "num_input_tokens_seen": 74605140, + "router_z_loss_clip": 0.99951172, + "router_z_loss_mlp": 0.18048096, + "step": 3457, + "time_per_iteration": 2.7783615589141846 + }, + { + "auxiliary_loss_clip": 0.0115619, + "auxiliary_loss_mlp": 0.01048914, + "balance_loss_clip": 1.05303454, + "balance_loss_mlp": 1.02992415, + "epoch": 0.41579991582997655, + "flos": 19238387650560.0, + "grad_norm": 1.7623302874537203, + "language_loss": 0.68236768, + "learning_rate": 2.6316495687678457e-06, + "loss": 0.70441872, + "num_input_tokens_seen": 74623790, + "router_z_loss_clip": 1.03076172, + "router_z_loss_mlp": 0.18981934, + "step": 3458, + "time_per_iteration": 3.5367162227630615 + }, + { + "auxiliary_loss_clip": 0.01149324, + "auxiliary_loss_mlp": 0.01046379, + "balance_loss_clip": 1.0486927, + "balance_loss_mlp": 1.02715015, + "epoch": 0.41592015872061566, + "flos": 24462061804800.0, + "grad_norm": 3.573967823816589, + "language_loss": 0.76425451, + "learning_rate": 2.6309104189216887e-06, + "loss": 0.78621161, + "num_input_tokens_seen": 74641355, + "router_z_loss_clip": 1.00537109, + "router_z_loss_mlp": 0.19226074, + "step": 3459, + "time_per_iteration": 2.7891037464141846 + }, + { + "auxiliary_loss_clip": 0.01152302, + "auxiliary_loss_mlp": 0.01053527, + "balance_loss_clip": 1.04761374, + "balance_loss_mlp": 1.03546667, + "epoch": 0.4160404016112547, + "flos": 20775651966720.0, + "grad_norm": 2.5225618285140277, + "language_loss": 0.74919611, + "learning_rate": 2.630171173366923e-06, + "loss": 0.77125442, + "num_input_tokens_seen": 74657155, + "router_z_loss_clip": 1.04638672, + "router_z_loss_mlp": 0.18041992, + "step": 3460, + "time_per_iteration": 2.7794880867004395 + }, + { + "auxiliary_loss_clip": 0.01152752, + "auxiliary_loss_mlp": 0.01047159, + "balance_loss_clip": 1.0506984, + "balance_loss_mlp": 1.02857399, + "epoch": 0.41616064450189383, + "flos": 13916462820480.0, + "grad_norm": 4.378815062031885, + "language_loss": 0.74504358, + "learning_rate": 2.629431832215691e-06, + "loss": 0.7670427, + "num_input_tokens_seen": 74671960, + "router_z_loss_clip": 1.01953125, + "router_z_loss_mlp": 0.18579102, + "step": 3461, + "time_per_iteration": 2.743213176727295 + }, + { + "auxiliary_loss_clip": 0.01149759, + "auxiliary_loss_mlp": 0.01040742, + "balance_loss_clip": 1.04737496, + "balance_loss_mlp": 1.02272952, + "epoch": 0.41628088739253294, + "flos": 20010826650240.0, + "grad_norm": 1.6997572554098055, + "language_loss": 0.87269211, + "learning_rate": 2.628692395580151e-06, + "loss": 0.89459711, + "num_input_tokens_seen": 74692050, + "router_z_loss_clip": 1.0234375, + "router_z_loss_mlp": 0.18017578, + "step": 3462, + "time_per_iteration": 2.7436623573303223 + }, + { + "auxiliary_loss_clip": 0.01155989, + "auxiliary_loss_mlp": 0.01046684, + "balance_loss_clip": 1.05198443, + "balance_loss_mlp": 1.02788508, + "epoch": 0.416401130283172, + "flos": 29168801377920.0, + "grad_norm": 1.7852441825057146, + "language_loss": 0.79601812, + "learning_rate": 2.6279528635724747e-06, + "loss": 0.81804484, + "num_input_tokens_seen": 74712205, + "router_z_loss_clip": 1.03857422, + "router_z_loss_mlp": 0.18798828, + "step": 3463, + "time_per_iteration": 2.790863037109375 + }, + { + "auxiliary_loss_clip": 0.01158414, + "auxiliary_loss_mlp": 0.01042756, + "balance_loss_clip": 1.05272162, + "balance_loss_mlp": 1.02281237, + "epoch": 0.4165213731738111, + "flos": 16246813478400.0, + "grad_norm": 2.4766516808900447, + "language_loss": 0.78458488, + "learning_rate": 2.627213236304848e-06, + "loss": 0.80659664, + "num_input_tokens_seen": 74729005, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.19946289, + "step": 3464, + "time_per_iteration": 2.6963253021240234 + }, + { + "auxiliary_loss_clip": 0.01149271, + "auxiliary_loss_mlp": 0.01049419, + "balance_loss_clip": 1.04946923, + "balance_loss_mlp": 1.03195441, + "epoch": 0.4166416160644502, + "flos": 33765438787200.0, + "grad_norm": 1.8721274396439807, + "language_loss": 0.70744324, + "learning_rate": 2.626473513889472e-06, + "loss": 0.7294302, + "num_input_tokens_seen": 74751385, + "router_z_loss_clip": 0.99951172, + "router_z_loss_mlp": 0.17468262, + "step": 3465, + "time_per_iteration": 2.888650417327881 + }, + { + "auxiliary_loss_clip": 0.011495, + "auxiliary_loss_mlp": 0.01044398, + "balance_loss_clip": 1.04900599, + "balance_loss_mlp": 1.02521706, + "epoch": 0.41676185895508927, + "flos": 20917498775040.0, + "grad_norm": 2.0281186184104314, + "language_loss": 0.83277416, + "learning_rate": 2.625733696438562e-06, + "loss": 0.85471308, + "num_input_tokens_seen": 74768890, + "router_z_loss_clip": 1.00439453, + "router_z_loss_mlp": 0.19177246, + "step": 3466, + "time_per_iteration": 2.7341063022613525 + }, + { + "auxiliary_loss_clip": 0.01152689, + "auxiliary_loss_mlp": 0.01056405, + "balance_loss_clip": 1.05078077, + "balance_loss_mlp": 1.03827322, + "epoch": 0.4168821018457284, + "flos": 18406122549120.0, + "grad_norm": 3.3483603345590742, + "language_loss": 0.7533946, + "learning_rate": 2.6249937840643476e-06, + "loss": 0.77548552, + "num_input_tokens_seen": 74787195, + "router_z_loss_clip": 1.01855469, + "router_z_loss_mlp": 0.18127441, + "step": 3467, + "time_per_iteration": 2.8058431148529053 + }, + { + "auxiliary_loss_clip": 0.0115037, + "auxiliary_loss_mlp": 0.01052045, + "balance_loss_clip": 1.04941559, + "balance_loss_mlp": 1.03220892, + "epoch": 0.41700234473636744, + "flos": 18698399516160.0, + "grad_norm": 1.6742027357757203, + "language_loss": 0.67058098, + "learning_rate": 2.6242537768790733e-06, + "loss": 0.69260514, + "num_input_tokens_seen": 74806350, + "router_z_loss_clip": 1.00830078, + "router_z_loss_mlp": 0.19848633, + "step": 3468, + "time_per_iteration": 2.707057237625122 + }, + { + "auxiliary_loss_clip": 0.0115711, + "auxiliary_loss_mlp": 0.01052819, + "balance_loss_clip": 1.05054617, + "balance_loss_mlp": 1.03263724, + "epoch": 0.41712258762700655, + "flos": 31033283616000.0, + "grad_norm": 1.7996492814716225, + "language_loss": 0.68606806, + "learning_rate": 2.6235136749949975e-06, + "loss": 0.70816731, + "num_input_tokens_seen": 74829800, + "router_z_loss_clip": 1.06738281, + "router_z_loss_mlp": 0.2019043, + "step": 3469, + "time_per_iteration": 2.827141046524048 + }, + { + "auxiliary_loss_clip": 0.01152846, + "auxiliary_loss_mlp": 0.01049508, + "balance_loss_clip": 1.04933751, + "balance_loss_mlp": 1.0303396, + "epoch": 0.41724283051764566, + "flos": 35914763877120.0, + "grad_norm": 1.9748341964831995, + "language_loss": 0.61254299, + "learning_rate": 2.6227734785243924e-06, + "loss": 0.63456661, + "num_input_tokens_seen": 74849760, + "router_z_loss_clip": 1.03515625, + "router_z_loss_mlp": 0.19177246, + "step": 3470, + "time_per_iteration": 2.8143723011016846 + }, + { + "auxiliary_loss_clip": 0.01151297, + "auxiliary_loss_mlp": 0.01041303, + "balance_loss_clip": 1.04679799, + "balance_loss_mlp": 1.02351713, + "epoch": 0.4173630734082847, + "flos": 25333649320320.0, + "grad_norm": 1.8013217508960992, + "language_loss": 0.79224575, + "learning_rate": 2.6220331875795466e-06, + "loss": 0.81417179, + "num_input_tokens_seen": 74869110, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.1776123, + "step": 3471, + "time_per_iteration": 2.7411253452301025 + }, + { + "auxiliary_loss_clip": 0.01151549, + "auxiliary_loss_mlp": 0.01054506, + "balance_loss_clip": 1.04939151, + "balance_loss_mlp": 1.03487217, + "epoch": 0.4174833162989238, + "flos": 26685398868480.0, + "grad_norm": 1.5764737215260278, + "language_loss": 0.75271267, + "learning_rate": 2.62129280227276e-06, + "loss": 0.77477324, + "num_input_tokens_seen": 74889110, + "router_z_loss_clip": 1.02197266, + "router_z_loss_mlp": 0.19641113, + "step": 3472, + "time_per_iteration": 2.7762503623962402 + }, + { + "auxiliary_loss_clip": 0.01154082, + "auxiliary_loss_mlp": 0.01048891, + "balance_loss_clip": 1.0490644, + "balance_loss_mlp": 1.02939975, + "epoch": 0.41760355918956293, + "flos": 74739584010240.0, + "grad_norm": 2.0295600154965228, + "language_loss": 0.68764472, + "learning_rate": 2.62055232271635e-06, + "loss": 0.70967448, + "num_input_tokens_seen": 74916260, + "router_z_loss_clip": 1.05029297, + "router_z_loss_mlp": 0.19506836, + "step": 3473, + "time_per_iteration": 3.1430649757385254 + }, + { + "auxiliary_loss_clip": 0.01150213, + "auxiliary_loss_mlp": 0.01048617, + "balance_loss_clip": 1.04922998, + "balance_loss_mlp": 1.02866125, + "epoch": 0.417723802080202, + "flos": 14317513148160.0, + "grad_norm": 2.0306046061407304, + "language_loss": 0.88230741, + "learning_rate": 2.619811749022646e-06, + "loss": 0.90429574, + "num_input_tokens_seen": 74931570, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.19958496, + "step": 3474, + "time_per_iteration": 2.724061965942383 + }, + { + "auxiliary_loss_clip": 0.01152345, + "auxiliary_loss_mlp": 0.01056681, + "balance_loss_clip": 1.04683352, + "balance_loss_mlp": 1.03645098, + "epoch": 0.4178440449708411, + "flos": 14643797316480.0, + "grad_norm": 2.0712824127883924, + "language_loss": 0.7115196, + "learning_rate": 2.6190710813039917e-06, + "loss": 0.73360986, + "num_input_tokens_seen": 74944695, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.20214844, + "step": 3475, + "time_per_iteration": 2.7127761840820312 + }, + { + "auxiliary_loss_clip": 0.01151329, + "auxiliary_loss_mlp": 0.01050333, + "balance_loss_clip": 1.04630172, + "balance_loss_mlp": 1.03050828, + "epoch": 0.4179642878614802, + "flos": 21507296094720.0, + "grad_norm": 2.154823752812767, + "language_loss": 0.84025127, + "learning_rate": 2.618330319672747e-06, + "loss": 0.86226791, + "num_input_tokens_seen": 74964115, + "router_z_loss_clip": 1.05078125, + "router_z_loss_mlp": 0.19836426, + "step": 3476, + "time_per_iteration": 2.7384533882141113 + }, + { + "auxiliary_loss_clip": 0.01150371, + "auxiliary_loss_mlp": 0.01048795, + "balance_loss_clip": 1.0475775, + "balance_loss_mlp": 1.0301981, + "epoch": 0.41808453075211927, + "flos": 18441997257600.0, + "grad_norm": 1.9744902299470983, + "language_loss": 0.92352879, + "learning_rate": 2.617589464241284e-06, + "loss": 0.94552052, + "num_input_tokens_seen": 74978515, + "router_z_loss_clip": 1.02783203, + "router_z_loss_mlp": 0.18603516, + "step": 3477, + "time_per_iteration": 2.687514066696167 + }, + { + "auxiliary_loss_clip": 0.0114706, + "auxiliary_loss_mlp": 0.01045734, + "balance_loss_clip": 1.04671788, + "balance_loss_mlp": 1.02683949, + "epoch": 0.4182047736427584, + "flos": 20301020628480.0, + "grad_norm": 1.8854897405227569, + "language_loss": 0.74638218, + "learning_rate": 2.6168485151219914e-06, + "loss": 0.76831013, + "num_input_tokens_seen": 74998135, + "router_z_loss_clip": 1.00292969, + "router_z_loss_mlp": 0.18896484, + "step": 3478, + "time_per_iteration": 2.786137580871582 + }, + { + "auxiliary_loss_clip": 0.01154189, + "auxiliary_loss_mlp": 0.01044486, + "balance_loss_clip": 1.04921353, + "balance_loss_mlp": 1.02424383, + "epoch": 0.4183250165333975, + "flos": 18876623823360.0, + "grad_norm": 2.3989365521816093, + "language_loss": 0.71261001, + "learning_rate": 2.616107472427269e-06, + "loss": 0.73459673, + "num_input_tokens_seen": 75012830, + "router_z_loss_clip": 1.04931641, + "router_z_loss_mlp": 0.20239258, + "step": 3479, + "time_per_iteration": 3.49631667137146 + }, + { + "auxiliary_loss_clip": 0.01152859, + "auxiliary_loss_mlp": 0.01049359, + "balance_loss_clip": 1.04840469, + "balance_loss_mlp": 1.0294627, + "epoch": 0.41844525942403654, + "flos": 17740050698880.0, + "grad_norm": 2.3462459216329012, + "language_loss": 0.76237381, + "learning_rate": 2.615366336269533e-06, + "loss": 0.78439593, + "num_input_tokens_seen": 75026495, + "router_z_loss_clip": 1.04394531, + "router_z_loss_mlp": 0.19909668, + "step": 3480, + "time_per_iteration": 2.815981388092041 + }, + { + "auxiliary_loss_clip": 0.0115491, + "auxiliary_loss_mlp": 0.01053928, + "balance_loss_clip": 1.04747248, + "balance_loss_mlp": 1.03372216, + "epoch": 0.41856550231467565, + "flos": 18361377181440.0, + "grad_norm": 2.4222511547076095, + "language_loss": 0.80706888, + "learning_rate": 2.6146251067612126e-06, + "loss": 0.82915729, + "num_input_tokens_seen": 75041970, + "router_z_loss_clip": 1.07421875, + "router_z_loss_mlp": 0.2019043, + "step": 3481, + "time_per_iteration": 3.665694236755371 + }, + { + "auxiliary_loss_clip": 0.01155603, + "auxiliary_loss_mlp": 0.01048468, + "balance_loss_clip": 1.0516268, + "balance_loss_mlp": 1.03043175, + "epoch": 0.41868574520531476, + "flos": 22781801445120.0, + "grad_norm": 1.5475088300910445, + "language_loss": 0.82663476, + "learning_rate": 2.6138837840147525e-06, + "loss": 0.84867549, + "num_input_tokens_seen": 75061005, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.18029785, + "step": 3482, + "time_per_iteration": 2.79738450050354 + }, + { + "auxiliary_loss_clip": 0.01150882, + "auxiliary_loss_mlp": 0.01041676, + "balance_loss_clip": 1.04838204, + "balance_loss_mlp": 1.02326965, + "epoch": 0.4188059880959538, + "flos": 13699167494400.0, + "grad_norm": 3.574496023386193, + "language_loss": 0.75950354, + "learning_rate": 2.6131423681426103e-06, + "loss": 0.78142917, + "num_input_tokens_seen": 75076920, + "router_z_loss_clip": 1.02490234, + "router_z_loss_mlp": 0.18408203, + "step": 3483, + "time_per_iteration": 3.0533077716827393 + }, + { + "auxiliary_loss_clip": 0.01144539, + "auxiliary_loss_mlp": 0.01044981, + "balance_loss_clip": 1.04439557, + "balance_loss_mlp": 1.02777958, + "epoch": 0.41892623098659293, + "flos": 37818281220480.0, + "grad_norm": 1.7321092042545705, + "language_loss": 0.73024619, + "learning_rate": 2.6124008592572587e-06, + "loss": 0.75214148, + "num_input_tokens_seen": 75100905, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.17211914, + "step": 3484, + "time_per_iteration": 3.0765223503112793 + }, + { + "auxiliary_loss_clip": 0.01152172, + "auxiliary_loss_mlp": 0.01048571, + "balance_loss_clip": 1.04721034, + "balance_loss_mlp": 1.02855611, + "epoch": 0.419046473877232, + "flos": 23258874908160.0, + "grad_norm": 1.9541824814236446, + "language_loss": 0.81706262, + "learning_rate": 2.6116592574711835e-06, + "loss": 0.83907008, + "num_input_tokens_seen": 75119205, + "router_z_loss_clip": 1.04931641, + "router_z_loss_mlp": 0.20007324, + "step": 3485, + "time_per_iteration": 3.620633125305176 + }, + { + "auxiliary_loss_clip": 0.0115187, + "auxiliary_loss_mlp": 0.01042877, + "balance_loss_clip": 1.04716086, + "balance_loss_mlp": 1.02380395, + "epoch": 0.4191667167678711, + "flos": 20741034234240.0, + "grad_norm": 1.6996242334736913, + "language_loss": 0.84148639, + "learning_rate": 2.6109175628968853e-06, + "loss": 0.8634339, + "num_input_tokens_seen": 75138970, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.1907959, + "step": 3486, + "time_per_iteration": 4.102076292037964 + }, + { + "auxiliary_loss_clip": 0.01146423, + "auxiliary_loss_mlp": 0.01048983, + "balance_loss_clip": 1.04637349, + "balance_loss_mlp": 1.0306958, + "epoch": 0.4192869596585102, + "flos": 23586416052480.0, + "grad_norm": 1.9747966016076244, + "language_loss": 0.82892954, + "learning_rate": 2.610175775646878e-06, + "loss": 0.8508836, + "num_input_tokens_seen": 75157550, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.18273926, + "step": 3487, + "time_per_iteration": 3.030121326446533 + }, + { + "auxiliary_loss_clip": 0.01152447, + "auxiliary_loss_mlp": 0.01051638, + "balance_loss_clip": 1.04705441, + "balance_loss_mlp": 1.03208721, + "epoch": 0.41940720254914926, + "flos": 25081269384960.0, + "grad_norm": 1.9372401041227163, + "language_loss": 0.73358011, + "learning_rate": 2.6094338958336907e-06, + "loss": 0.75562096, + "num_input_tokens_seen": 75176220, + "router_z_loss_clip": 1.05566406, + "router_z_loss_mlp": 0.19543457, + "step": 3488, + "time_per_iteration": 2.9406723976135254 + }, + { + "auxiliary_loss_clip": 0.0114488, + "auxiliary_loss_mlp": 0.01045456, + "balance_loss_clip": 1.04626966, + "balance_loss_mlp": 1.02647734, + "epoch": 0.41952744543978837, + "flos": 15554132628480.0, + "grad_norm": 2.052402206643981, + "language_loss": 0.82662928, + "learning_rate": 2.608691923569867e-06, + "loss": 0.84853268, + "num_input_tokens_seen": 75193095, + "router_z_loss_clip": 0.98583984, + "router_z_loss_mlp": 0.18969727, + "step": 3489, + "time_per_iteration": 3.0485832691192627 + }, + { + "auxiliary_loss_clip": 0.01151199, + "auxiliary_loss_mlp": 0.010488, + "balance_loss_clip": 1.04962444, + "balance_loss_mlp": 1.03037035, + "epoch": 0.4196476883304275, + "flos": 24644775312000.0, + "grad_norm": 1.6221339885056705, + "language_loss": 0.7635321, + "learning_rate": 2.6079498589679616e-06, + "loss": 0.78553212, + "num_input_tokens_seen": 75214185, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.18432617, + "step": 3490, + "time_per_iteration": 3.0095155239105225 + }, + { + "auxiliary_loss_clip": 0.01155431, + "auxiliary_loss_mlp": 0.01049258, + "balance_loss_clip": 1.04836154, + "balance_loss_mlp": 1.02954102, + "epoch": 0.41976793122106654, + "flos": 24531333183360.0, + "grad_norm": 1.9071360402583206, + "language_loss": 0.76141655, + "learning_rate": 2.6072077021405465e-06, + "loss": 0.78346348, + "num_input_tokens_seen": 75233020, + "router_z_loss_clip": 1.0703125, + "router_z_loss_mlp": 0.19702148, + "step": 3491, + "time_per_iteration": 2.8490071296691895 + }, + { + "auxiliary_loss_clip": 0.01148843, + "auxiliary_loss_mlp": 0.0105427, + "balance_loss_clip": 1.04840362, + "balance_loss_mlp": 1.03560233, + "epoch": 0.41988817411170565, + "flos": 21175301664000.0, + "grad_norm": 1.7504824680296418, + "language_loss": 0.69342709, + "learning_rate": 2.6064654532002054e-06, + "loss": 0.71545821, + "num_input_tokens_seen": 75252030, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.18664551, + "step": 3492, + "time_per_iteration": 2.7934207916259766 + }, + { + "auxiliary_loss_clip": 0.01150194, + "auxiliary_loss_mlp": 0.01047314, + "balance_loss_clip": 1.04876757, + "balance_loss_mlp": 1.02976608, + "epoch": 0.42000841700234476, + "flos": 31649402626560.0, + "grad_norm": 1.4793401313812427, + "language_loss": 0.76147878, + "learning_rate": 2.6057231122595375e-06, + "loss": 0.78345382, + "num_input_tokens_seen": 75273340, + "router_z_loss_clip": 1.01416016, + "router_z_loss_mlp": 0.17541504, + "step": 3493, + "time_per_iteration": 2.842104911804199 + }, + { + "auxiliary_loss_clip": 0.01149099, + "auxiliary_loss_mlp": 0.01053366, + "balance_loss_clip": 1.04633093, + "balance_loss_mlp": 1.03443575, + "epoch": 0.4201286598929838, + "flos": 21281525159040.0, + "grad_norm": 1.7400864301707808, + "language_loss": 0.72500694, + "learning_rate": 2.604980679431154e-06, + "loss": 0.74703163, + "num_input_tokens_seen": 75291580, + "router_z_loss_clip": 1.02734375, + "router_z_loss_mlp": 0.18945312, + "step": 3494, + "time_per_iteration": 2.7794482707977295 + }, + { + "auxiliary_loss_clip": 0.01151139, + "auxiliary_loss_mlp": 0.01046524, + "balance_loss_clip": 1.04933095, + "balance_loss_mlp": 1.02826095, + "epoch": 0.4202489027836229, + "flos": 18546532813440.0, + "grad_norm": 2.1070640819566977, + "language_loss": 0.74827027, + "learning_rate": 2.604238154827684e-06, + "loss": 0.77024698, + "num_input_tokens_seen": 75308205, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.18273926, + "step": 3495, + "time_per_iteration": 2.7542102336883545 + }, + { + "auxiliary_loss_clip": 0.01149354, + "auxiliary_loss_mlp": 0.01033041, + "balance_loss_clip": 1.0494473, + "balance_loss_mlp": 1.01592231, + "epoch": 0.42036914567426203, + "flos": 19317643009920.0, + "grad_norm": 1.8427652026436612, + "language_loss": 0.72908276, + "learning_rate": 2.6034955385617656e-06, + "loss": 0.75090671, + "num_input_tokens_seen": 75326535, + "router_z_loss_clip": 0.99951172, + "router_z_loss_mlp": 0.17114258, + "step": 3496, + "time_per_iteration": 2.7670326232910156 + }, + { + "auxiliary_loss_clip": 0.01058918, + "auxiliary_loss_mlp": 0.01007812, + "balance_loss_clip": 1.02503204, + "balance_loss_mlp": 1.00565386, + "epoch": 0.4204893885649011, + "flos": 67842942935040.0, + "grad_norm": 0.7284513813361393, + "language_loss": 0.61597437, + "learning_rate": 2.6027528307460544e-06, + "loss": 0.63664168, + "num_input_tokens_seen": 75390540, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.02160645, + "step": 3497, + "time_per_iteration": 3.3611536026000977 + }, + { + "auxiliary_loss_clip": 0.01151841, + "auxiliary_loss_mlp": 0.01052584, + "balance_loss_clip": 1.04810703, + "balance_loss_mlp": 1.03448784, + "epoch": 0.4206096314555402, + "flos": 21908777385600.0, + "grad_norm": 1.7062662168904197, + "language_loss": 0.86830944, + "learning_rate": 2.602010031493217e-06, + "loss": 0.89035368, + "num_input_tokens_seen": 75408770, + "router_z_loss_clip": 1.03710938, + "router_z_loss_mlp": 0.1809082, + "step": 3498, + "time_per_iteration": 2.737769842147827 + }, + { + "auxiliary_loss_clip": 0.01147063, + "auxiliary_loss_mlp": 0.01048855, + "balance_loss_clip": 1.04480207, + "balance_loss_mlp": 1.02947199, + "epoch": 0.42072987434617926, + "flos": 29278185269760.0, + "grad_norm": 1.8958988058905355, + "language_loss": 0.86752081, + "learning_rate": 2.6012671409159367e-06, + "loss": 0.88947999, + "num_input_tokens_seen": 75430105, + "router_z_loss_clip": 1.02441406, + "router_z_loss_mlp": 0.19396973, + "step": 3499, + "time_per_iteration": 2.828317165374756 + }, + { + "auxiliary_loss_clip": 0.01155146, + "auxiliary_loss_mlp": 0.0104955, + "balance_loss_clip": 1.05002856, + "balance_loss_mlp": 1.03017855, + "epoch": 0.42085011723681837, + "flos": 27600726170880.0, + "grad_norm": 1.7262679092255884, + "language_loss": 0.81994313, + "learning_rate": 2.6005241591269097e-06, + "loss": 0.84199005, + "num_input_tokens_seen": 75449475, + "router_z_loss_clip": 1.05078125, + "router_z_loss_mlp": 0.19360352, + "step": 3500, + "time_per_iteration": 2.972203493118286 + }, + { + "auxiliary_loss_clip": 0.01148269, + "auxiliary_loss_mlp": 0.01049501, + "balance_loss_clip": 1.04964805, + "balance_loss_mlp": 1.03223956, + "epoch": 0.4209703601274575, + "flos": 27818632028160.0, + "grad_norm": 1.654813750353017, + "language_loss": 0.79884446, + "learning_rate": 2.5997810862388454e-06, + "loss": 0.82082212, + "num_input_tokens_seen": 75469315, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.17260742, + "step": 3501, + "time_per_iteration": 2.9067893028259277 + }, + { + "auxiliary_loss_clip": 0.01151177, + "auxiliary_loss_mlp": 0.01048611, + "balance_loss_clip": 1.04793465, + "balance_loss_mlp": 1.02844095, + "epoch": 0.42109060301809653, + "flos": 27525529048320.0, + "grad_norm": 1.9619975802863878, + "language_loss": 0.75389683, + "learning_rate": 2.599037922364467e-06, + "loss": 0.77589476, + "num_input_tokens_seen": 75488215, + "router_z_loss_clip": 1.03369141, + "router_z_loss_mlp": 0.20166016, + "step": 3502, + "time_per_iteration": 2.7715909481048584 + }, + { + "auxiliary_loss_clip": 0.01148225, + "auxiliary_loss_mlp": 0.01046622, + "balance_loss_clip": 1.04629064, + "balance_loss_mlp": 1.02802491, + "epoch": 0.42121084590873564, + "flos": 29314275459840.0, + "grad_norm": 2.3362772341304883, + "language_loss": 0.75139886, + "learning_rate": 2.5982946676165112e-06, + "loss": 0.77334732, + "num_input_tokens_seen": 75507985, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.18603516, + "step": 3503, + "time_per_iteration": 2.8875105381011963 + }, + { + "auxiliary_loss_clip": 0.01054429, + "auxiliary_loss_mlp": 0.01015915, + "balance_loss_clip": 1.02067399, + "balance_loss_mlp": 1.0138762, + "epoch": 0.42133108879937475, + "flos": 67398835178880.0, + "grad_norm": 0.7260363076764779, + "language_loss": 0.57593226, + "learning_rate": 2.5975513221077313e-06, + "loss": 0.5966357, + "num_input_tokens_seen": 75571955, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.02038574, + "step": 3504, + "time_per_iteration": 3.4104926586151123 + }, + { + "auxiliary_loss_clip": 0.01149095, + "auxiliary_loss_mlp": 0.01047336, + "balance_loss_clip": 1.04713941, + "balance_loss_mlp": 1.02928829, + "epoch": 0.4214513316900138, + "flos": 23106038538240.0, + "grad_norm": 2.5357043835501067, + "language_loss": 0.88455355, + "learning_rate": 2.5968078859508897e-06, + "loss": 0.90651786, + "num_input_tokens_seen": 75589155, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.18041992, + "step": 3505, + "time_per_iteration": 2.7523748874664307 + }, + { + "auxiliary_loss_clip": 0.01148489, + "auxiliary_loss_mlp": 0.01044677, + "balance_loss_clip": 1.0464555, + "balance_loss_mlp": 1.02600884, + "epoch": 0.4215715745806529, + "flos": 15336190857600.0, + "grad_norm": 1.8515857569613343, + "language_loss": 0.80300581, + "learning_rate": 2.5960643592587673e-06, + "loss": 0.82493746, + "num_input_tokens_seen": 75606565, + "router_z_loss_clip": 1.02148438, + "router_z_loss_mlp": 0.18652344, + "step": 3506, + "time_per_iteration": 2.753185510635376 + }, + { + "auxiliary_loss_clip": 0.01148341, + "auxiliary_loss_mlp": 0.010452, + "balance_loss_clip": 1.04674196, + "balance_loss_mlp": 1.02578044, + "epoch": 0.42169181747129203, + "flos": 22127257860480.0, + "grad_norm": 2.2818843324433273, + "language_loss": 0.81716001, + "learning_rate": 2.5953207421441553e-06, + "loss": 0.83909535, + "num_input_tokens_seen": 75625165, + "router_z_loss_clip": 1.01757812, + "router_z_loss_mlp": 0.19421387, + "step": 3507, + "time_per_iteration": 2.9557912349700928 + }, + { + "auxiliary_loss_clip": 0.01146903, + "auxiliary_loss_mlp": 0.01042765, + "balance_loss_clip": 1.04852557, + "balance_loss_mlp": 1.02454948, + "epoch": 0.4218120603619311, + "flos": 22630724841600.0, + "grad_norm": 3.0664760721125983, + "language_loss": 0.75734323, + "learning_rate": 2.5945770347198603e-06, + "loss": 0.77923995, + "num_input_tokens_seen": 75643320, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.18212891, + "step": 3508, + "time_per_iteration": 3.5449910163879395 + }, + { + "auxiliary_loss_clip": 0.01146555, + "auxiliary_loss_mlp": 0.01044975, + "balance_loss_clip": 1.04627132, + "balance_loss_mlp": 1.02591372, + "epoch": 0.4219323032525702, + "flos": 19682818629120.0, + "grad_norm": 1.712843843641413, + "language_loss": 0.82779956, + "learning_rate": 2.593833237098701e-06, + "loss": 0.84971488, + "num_input_tokens_seen": 75660920, + "router_z_loss_clip": 1.00292969, + "router_z_loss_mlp": 0.19055176, + "step": 3509, + "time_per_iteration": 2.7745132446289062 + }, + { + "auxiliary_loss_clip": 0.01154034, + "auxiliary_loss_mlp": 0.01049518, + "balance_loss_clip": 1.04811931, + "balance_loss_mlp": 1.0290854, + "epoch": 0.4220525461432093, + "flos": 30190747224960.0, + "grad_norm": 2.4414053463753516, + "language_loss": 0.62364244, + "learning_rate": 2.593089349393512e-06, + "loss": 0.64567798, + "num_input_tokens_seen": 75681410, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.2043457, + "step": 3510, + "time_per_iteration": 3.8078773021698 + }, + { + "auxiliary_loss_clip": 0.01153871, + "auxiliary_loss_mlp": 0.01049893, + "balance_loss_clip": 1.04920077, + "balance_loss_mlp": 1.03201103, + "epoch": 0.42217278903384836, + "flos": 24315941278080.0, + "grad_norm": 2.5998275051910356, + "language_loss": 0.84051192, + "learning_rate": 2.592345371717141e-06, + "loss": 0.86254954, + "num_input_tokens_seen": 75700940, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.17883301, + "step": 3511, + "time_per_iteration": 2.776923894882202 + }, + { + "auxiliary_loss_clip": 0.01155585, + "auxiliary_loss_mlp": 0.01060616, + "balance_loss_clip": 1.05257583, + "balance_loss_mlp": 1.04101801, + "epoch": 0.42229303192448747, + "flos": 17092474352640.0, + "grad_norm": 2.2839776345159075, + "language_loss": 0.71953166, + "learning_rate": 2.591601304182448e-06, + "loss": 0.74169362, + "num_input_tokens_seen": 75718910, + "router_z_loss_clip": 1.02880859, + "router_z_loss_mlp": 0.19592285, + "step": 3512, + "time_per_iteration": 2.8532915115356445 + }, + { + "auxiliary_loss_clip": 0.01149317, + "auxiliary_loss_mlp": 0.01051009, + "balance_loss_clip": 1.05019414, + "balance_loss_mlp": 1.03390265, + "epoch": 0.4224132748151266, + "flos": 22784530878720.0, + "grad_norm": 1.81579504774344, + "language_loss": 0.79528064, + "learning_rate": 2.5908571469023067e-06, + "loss": 0.81728399, + "num_input_tokens_seen": 75738395, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.17102051, + "step": 3513, + "time_per_iteration": 2.8054094314575195 + }, + { + "auxiliary_loss_clip": 0.01148999, + "auxiliary_loss_mlp": 0.01043857, + "balance_loss_clip": 1.04799616, + "balance_loss_mlp": 1.02491474, + "epoch": 0.42253351770576564, + "flos": 17819090576640.0, + "grad_norm": 2.1910229149425557, + "language_loss": 0.75643986, + "learning_rate": 2.5901128999896067e-06, + "loss": 0.77836841, + "num_input_tokens_seen": 75753825, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.18933105, + "step": 3514, + "time_per_iteration": 3.5348470211029053 + }, + { + "auxiliary_loss_clip": 0.0115001, + "auxiliary_loss_mlp": 0.01043373, + "balance_loss_clip": 1.04832315, + "balance_loss_mlp": 1.02500272, + "epoch": 0.42265376059640475, + "flos": 28512390286080.0, + "grad_norm": 1.9115836767138765, + "language_loss": 0.68470025, + "learning_rate": 2.5893685635572487e-06, + "loss": 0.70663404, + "num_input_tokens_seen": 75774675, + "router_z_loss_clip": 1.01660156, + "router_z_loss_mlp": 0.18371582, + "step": 3515, + "time_per_iteration": 3.582247734069824 + }, + { + "auxiliary_loss_clip": 0.01148684, + "auxiliary_loss_mlp": 0.01047694, + "balance_loss_clip": 1.04726744, + "balance_loss_mlp": 1.02900195, + "epoch": 0.4227740034870438, + "flos": 16253349753600.0, + "grad_norm": 4.660885334509196, + "language_loss": 0.69381344, + "learning_rate": 2.5886241377181483e-06, + "loss": 0.71577728, + "num_input_tokens_seen": 75793545, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.18688965, + "step": 3516, + "time_per_iteration": 2.7206993103027344 + }, + { + "auxiliary_loss_clip": 0.0115531, + "auxiliary_loss_mlp": 0.01042849, + "balance_loss_clip": 1.04869461, + "balance_loss_mlp": 1.0232271, + "epoch": 0.4228942463776829, + "flos": 25295691623040.0, + "grad_norm": 1.9025452296378864, + "language_loss": 0.81567061, + "learning_rate": 2.587879622585234e-06, + "loss": 0.83765221, + "num_input_tokens_seen": 75812145, + "router_z_loss_clip": 1.06542969, + "router_z_loss_mlp": 0.19616699, + "step": 3517, + "time_per_iteration": 2.8093984127044678 + }, + { + "auxiliary_loss_clip": 0.01150679, + "auxiliary_loss_mlp": 0.01043537, + "balance_loss_clip": 1.04989767, + "balance_loss_mlp": 1.0250001, + "epoch": 0.423014489268322, + "flos": 26395779507840.0, + "grad_norm": 2.001336960103719, + "language_loss": 0.75561368, + "learning_rate": 2.5871350182714486e-06, + "loss": 0.77755582, + "num_input_tokens_seen": 75833025, + "router_z_loss_clip": 1.00878906, + "router_z_loss_mlp": 0.1854248, + "step": 3518, + "time_per_iteration": 2.742021083831787 + }, + { + "auxiliary_loss_clip": 0.01150634, + "auxiliary_loss_mlp": 0.01047746, + "balance_loss_clip": 1.05036783, + "balance_loss_mlp": 1.02903068, + "epoch": 0.4231347321589611, + "flos": 17274002711040.0, + "grad_norm": 1.8935376613863732, + "language_loss": 0.8034088, + "learning_rate": 2.586390324889748e-06, + "loss": 0.8253926, + "num_input_tokens_seen": 75848925, + "router_z_loss_clip": 1.00244141, + "router_z_loss_mlp": 0.18713379, + "step": 3519, + "time_per_iteration": 2.7724876403808594 + }, + { + "auxiliary_loss_clip": 0.01148428, + "auxiliary_loss_mlp": 0.01042887, + "balance_loss_clip": 1.04881215, + "balance_loss_mlp": 1.02389681, + "epoch": 0.4232549750496002, + "flos": 22999635475200.0, + "grad_norm": 1.8701008875147203, + "language_loss": 0.67515719, + "learning_rate": 2.5856455425531003e-06, + "loss": 0.69707036, + "num_input_tokens_seen": 75870400, + "router_z_loss_clip": 0.99804688, + "router_z_loss_mlp": 0.18981934, + "step": 3520, + "time_per_iteration": 2.778054714202881 + }, + { + "auxiliary_loss_clip": 0.01144716, + "auxiliary_loss_mlp": 0.01049324, + "balance_loss_clip": 1.04732573, + "balance_loss_mlp": 1.03096628, + "epoch": 0.4233752179402393, + "flos": 21248343970560.0, + "grad_norm": 3.3510959146761334, + "language_loss": 0.8104372, + "learning_rate": 2.5849006713744902e-06, + "loss": 0.83237767, + "num_input_tokens_seen": 75889195, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.18359375, + "step": 3521, + "time_per_iteration": 2.749762773513794 + }, + { + "auxiliary_loss_clip": 0.01150257, + "auxiliary_loss_mlp": 0.01047262, + "balance_loss_clip": 1.05011332, + "balance_loss_mlp": 1.0288918, + "epoch": 0.42349546083087836, + "flos": 20704297599360.0, + "grad_norm": 2.1592805236431976, + "language_loss": 0.7299698, + "learning_rate": 2.5841557114669135e-06, + "loss": 0.75194496, + "num_input_tokens_seen": 75906055, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.18371582, + "step": 3522, + "time_per_iteration": 2.750459909439087 + }, + { + "auxiliary_loss_clip": 0.01154784, + "auxiliary_loss_mlp": 0.01047269, + "balance_loss_clip": 1.04839134, + "balance_loss_mlp": 1.02752829, + "epoch": 0.42361570372151747, + "flos": 18585065128320.0, + "grad_norm": 4.011796830170223, + "language_loss": 0.67880571, + "learning_rate": 2.58341066294338e-06, + "loss": 0.70082623, + "num_input_tokens_seen": 75922720, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.19750977, + "step": 3523, + "time_per_iteration": 2.803241014480591 + }, + { + "auxiliary_loss_clip": 0.01150749, + "auxiliary_loss_mlp": 0.01039919, + "balance_loss_clip": 1.04910707, + "balance_loss_mlp": 1.02154922, + "epoch": 0.4237359466121566, + "flos": 20959478795520.0, + "grad_norm": 2.1341519308295926, + "language_loss": 0.85824835, + "learning_rate": 2.5826655259169124e-06, + "loss": 0.88015497, + "num_input_tokens_seen": 75941375, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.18383789, + "step": 3524, + "time_per_iteration": 3.098142147064209 + }, + { + "auxiliary_loss_clip": 0.01152924, + "auxiliary_loss_mlp": 0.01047181, + "balance_loss_clip": 1.0523684, + "balance_loss_mlp": 1.02838206, + "epoch": 0.42385618950279563, + "flos": 18038181582720.0, + "grad_norm": 2.150973004465935, + "language_loss": 0.90554512, + "learning_rate": 2.5819203005005475e-06, + "loss": 0.92754614, + "num_input_tokens_seen": 75958710, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.18811035, + "step": 3525, + "time_per_iteration": 2.8508617877960205 + }, + { + "auxiliary_loss_clip": 0.01148055, + "auxiliary_loss_mlp": 0.01050011, + "balance_loss_clip": 1.04776716, + "balance_loss_mlp": 1.03105652, + "epoch": 0.42397643239343474, + "flos": 23769129559680.0, + "grad_norm": 1.5675973345343286, + "language_loss": 0.78747499, + "learning_rate": 2.581174986807336e-06, + "loss": 0.80945563, + "num_input_tokens_seen": 75978945, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.1895752, + "step": 3526, + "time_per_iteration": 2.9282612800598145 + }, + { + "auxiliary_loss_clip": 0.01149605, + "auxiliary_loss_mlp": 0.01047751, + "balance_loss_clip": 1.04874921, + "balance_loss_mlp": 1.02963138, + "epoch": 0.42409667528407385, + "flos": 16545088016640.0, + "grad_norm": 2.5265087505359056, + "language_loss": 0.90878332, + "learning_rate": 2.580429584950341e-06, + "loss": 0.93075681, + "num_input_tokens_seen": 75994695, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.18139648, + "step": 3527, + "time_per_iteration": 2.996028423309326 + }, + { + "auxiliary_loss_clip": 0.01153843, + "auxiliary_loss_mlp": 0.01051977, + "balance_loss_clip": 1.05105209, + "balance_loss_mlp": 1.03185475, + "epoch": 0.4242169181747129, + "flos": 16034186920320.0, + "grad_norm": 3.0174538588098563, + "language_loss": 0.66698027, + "learning_rate": 2.5796840950426397e-06, + "loss": 0.68903852, + "num_input_tokens_seen": 76011780, + "router_z_loss_clip": 1.02783203, + "router_z_loss_mlp": 0.20129395, + "step": 3528, + "time_per_iteration": 2.91678786277771 + }, + { + "auxiliary_loss_clip": 0.01147756, + "auxiliary_loss_mlp": 0.01042627, + "balance_loss_clip": 1.04802418, + "balance_loss_mlp": 1.0243876, + "epoch": 0.424337161065352, + "flos": 20084012611200.0, + "grad_norm": 1.9146844826913734, + "language_loss": 0.65649933, + "learning_rate": 2.578938517197322e-06, + "loss": 0.67840314, + "num_input_tokens_seen": 76029875, + "router_z_loss_clip": 0.99853516, + "router_z_loss_mlp": 0.18212891, + "step": 3529, + "time_per_iteration": 3.1610991954803467 + }, + { + "auxiliary_loss_clip": 0.01149043, + "auxiliary_loss_mlp": 0.01051657, + "balance_loss_clip": 1.05022693, + "balance_loss_mlp": 1.03197539, + "epoch": 0.4244574039559911, + "flos": 23878369797120.0, + "grad_norm": 2.2940440440751617, + "language_loss": 0.62389898, + "learning_rate": 2.5781928515274916e-06, + "loss": 0.64590597, + "num_input_tokens_seen": 76048595, + "router_z_loss_clip": 0.98730469, + "router_z_loss_mlp": 0.19689941, + "step": 3530, + "time_per_iteration": 3.2018837928771973 + }, + { + "auxiliary_loss_clip": 0.01151495, + "auxiliary_loss_mlp": 0.0104343, + "balance_loss_clip": 1.04918706, + "balance_loss_mlp": 1.02448726, + "epoch": 0.4245776468466302, + "flos": 17565920542080.0, + "grad_norm": 2.2615862884102893, + "language_loss": 0.67878747, + "learning_rate": 2.577447098146265e-06, + "loss": 0.70073676, + "num_input_tokens_seen": 76065770, + "router_z_loss_clip": 1.02294922, + "router_z_loss_mlp": 0.18945312, + "step": 3531, + "time_per_iteration": 3.019273042678833 + }, + { + "auxiliary_loss_clip": 0.01151809, + "auxiliary_loss_mlp": 0.01048516, + "balance_loss_clip": 1.05050445, + "balance_loss_mlp": 1.0303483, + "epoch": 0.4246978897372693, + "flos": 27776256958080.0, + "grad_norm": 1.710233419764686, + "language_loss": 0.79228538, + "learning_rate": 2.5767012571667724e-06, + "loss": 0.81428862, + "num_input_tokens_seen": 76085250, + "router_z_loss_clip": 1.01220703, + "router_z_loss_mlp": 0.18164062, + "step": 3532, + "time_per_iteration": 2.999483823776245 + }, + { + "auxiliary_loss_clip": 0.01151613, + "auxiliary_loss_mlp": 0.01048212, + "balance_loss_clip": 1.04870677, + "balance_loss_mlp": 1.02797031, + "epoch": 0.42481813262790835, + "flos": 15596615439360.0, + "grad_norm": 1.993116572918325, + "language_loss": 0.68306649, + "learning_rate": 2.5759553287021587e-06, + "loss": 0.70506477, + "num_input_tokens_seen": 76103580, + "router_z_loss_clip": 1.02978516, + "router_z_loss_mlp": 0.20227051, + "step": 3533, + "time_per_iteration": 4.070947647094727 + }, + { + "auxiliary_loss_clip": 0.01152794, + "auxiliary_loss_mlp": 0.01046933, + "balance_loss_clip": 1.05101776, + "balance_loss_mlp": 1.02852714, + "epoch": 0.42493837551854746, + "flos": 23951088881280.0, + "grad_norm": 1.9716555838072967, + "language_loss": 0.77949142, + "learning_rate": 2.5752093128655786e-06, + "loss": 0.80148876, + "num_input_tokens_seen": 76121825, + "router_z_loss_clip": 1.01806641, + "router_z_loss_mlp": 0.1842041, + "step": 3534, + "time_per_iteration": 5.724841594696045 + }, + { + "auxiliary_loss_clip": 0.01150888, + "auxiliary_loss_mlp": 0.01038412, + "balance_loss_clip": 1.04739094, + "balance_loss_mlp": 1.01962471, + "epoch": 0.4250586184091866, + "flos": 20813466009600.0, + "grad_norm": 1.8233349926475246, + "language_loss": 0.73858166, + "learning_rate": 2.574463209770204e-06, + "loss": 0.76047468, + "num_input_tokens_seen": 76141140, + "router_z_loss_clip": 1.03466797, + "router_z_loss_mlp": 0.18774414, + "step": 3535, + "time_per_iteration": 2.8478782176971436 + }, + { + "auxiliary_loss_clip": 0.01150598, + "auxiliary_loss_mlp": 0.01043825, + "balance_loss_clip": 1.04880846, + "balance_loss_mlp": 1.02452528, + "epoch": 0.42517886129982563, + "flos": 30371018607360.0, + "grad_norm": 1.969812194514969, + "language_loss": 0.79687417, + "learning_rate": 2.5737170195292165e-06, + "loss": 0.81881839, + "num_input_tokens_seen": 76164475, + "router_z_loss_clip": 1.01757812, + "router_z_loss_mlp": 0.19311523, + "step": 3536, + "time_per_iteration": 3.797090530395508 + }, + { + "auxiliary_loss_clip": 0.01150699, + "auxiliary_loss_mlp": 0.01040427, + "balance_loss_clip": 1.04755831, + "balance_loss_mlp": 1.02222347, + "epoch": 0.42529910419046474, + "flos": 20080636732800.0, + "grad_norm": 1.7259245859797863, + "language_loss": 0.78151625, + "learning_rate": 2.572970742255814e-06, + "loss": 0.80342752, + "num_input_tokens_seen": 76182965, + "router_z_loss_clip": 1.03076172, + "router_z_loss_mlp": 0.18200684, + "step": 3537, + "time_per_iteration": 3.048774242401123 + }, + { + "auxiliary_loss_clip": 0.01147894, + "auxiliary_loss_mlp": 0.01048233, + "balance_loss_clip": 1.04873323, + "balance_loss_mlp": 1.03020859, + "epoch": 0.42541934708110385, + "flos": 22632448694400.0, + "grad_norm": 1.692626419081598, + "language_loss": 0.81574887, + "learning_rate": 2.5722243780632046e-06, + "loss": 0.83771014, + "num_input_tokens_seen": 76201230, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.18017578, + "step": 3538, + "time_per_iteration": 2.9190218448638916 + }, + { + "auxiliary_loss_clip": 0.01050559, + "auxiliary_loss_mlp": 0.01002351, + "balance_loss_clip": 1.01748323, + "balance_loss_mlp": 1.00039566, + "epoch": 0.4255395899717429, + "flos": 66200676186240.0, + "grad_norm": 0.7660170842257249, + "language_loss": 0.60499108, + "learning_rate": 2.5714779270646125e-06, + "loss": 0.62552023, + "num_input_tokens_seen": 76262000, + "router_z_loss_clip": 0.33007812, + "router_z_loss_mlp": 0.01953125, + "step": 3539, + "time_per_iteration": 4.266170024871826 + }, + { + "auxiliary_loss_clip": 0.01153195, + "auxiliary_loss_mlp": 0.01052131, + "balance_loss_clip": 1.05009913, + "balance_loss_mlp": 1.03200829, + "epoch": 0.425659832862382, + "flos": 17931814433280.0, + "grad_norm": 2.22020805886227, + "language_loss": 0.7805267, + "learning_rate": 2.5707313893732735e-06, + "loss": 0.80257988, + "num_input_tokens_seen": 76280540, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.20117188, + "step": 3540, + "time_per_iteration": 2.7372756004333496 + }, + { + "auxiliary_loss_clip": 0.01151405, + "auxiliary_loss_mlp": 0.01041688, + "balance_loss_clip": 1.04772389, + "balance_loss_mlp": 1.02256703, + "epoch": 0.4257800757530211, + "flos": 24022550989440.0, + "grad_norm": 1.6805620941197943, + "language_loss": 0.77093959, + "learning_rate": 2.5699847651024364e-06, + "loss": 0.79287052, + "num_input_tokens_seen": 76301180, + "router_z_loss_clip": 1.03759766, + "router_z_loss_mlp": 0.19128418, + "step": 3541, + "time_per_iteration": 2.8234610557556152 + }, + { + "auxiliary_loss_clip": 0.01148033, + "auxiliary_loss_mlp": 0.01054255, + "balance_loss_clip": 1.05037069, + "balance_loss_mlp": 1.03596783, + "epoch": 0.4259003186436602, + "flos": 23696015425920.0, + "grad_norm": 2.3496528076858296, + "language_loss": 0.76841521, + "learning_rate": 2.5692380543653627e-06, + "loss": 0.79043806, + "num_input_tokens_seen": 76319335, + "router_z_loss_clip": 0.97802734, + "router_z_loss_mlp": 0.1829834, + "step": 3542, + "time_per_iteration": 3.6903536319732666 + }, + { + "auxiliary_loss_clip": 0.01149782, + "auxiliary_loss_mlp": 0.01052152, + "balance_loss_clip": 1.04786253, + "balance_loss_mlp": 1.03133798, + "epoch": 0.4260205615342993, + "flos": 15259772672640.0, + "grad_norm": 2.837507075921648, + "language_loss": 0.69860226, + "learning_rate": 2.5684912572753293e-06, + "loss": 0.72062165, + "num_input_tokens_seen": 76335010, + "router_z_loss_clip": 1.02099609, + "router_z_loss_mlp": 0.20800781, + "step": 3543, + "time_per_iteration": 2.7555978298187256 + }, + { + "auxiliary_loss_clip": 0.01143776, + "auxiliary_loss_mlp": 0.0104217, + "balance_loss_clip": 1.04725313, + "balance_loss_mlp": 1.02502751, + "epoch": 0.4261408044249384, + "flos": 30665306736000.0, + "grad_norm": 1.8929826513137027, + "language_loss": 0.83858329, + "learning_rate": 2.5677443739456245e-06, + "loss": 0.86044276, + "num_input_tokens_seen": 76356670, + "router_z_loss_clip": 0.96533203, + "router_z_loss_mlp": 0.17150879, + "step": 3544, + "time_per_iteration": 3.619194984436035 + }, + { + "auxiliary_loss_clip": 0.01147001, + "auxiliary_loss_mlp": 0.01049574, + "balance_loss_clip": 1.04711485, + "balance_loss_mlp": 1.03141797, + "epoch": 0.42626104731557746, + "flos": 23257905240960.0, + "grad_norm": 2.284450694757134, + "language_loss": 0.79478586, + "learning_rate": 2.5669974044895495e-06, + "loss": 0.81675154, + "num_input_tokens_seen": 76373065, + "router_z_loss_clip": 0.99951172, + "router_z_loss_mlp": 0.18151855, + "step": 3545, + "time_per_iteration": 2.9695467948913574 + }, + { + "auxiliary_loss_clip": 0.01147179, + "auxiliary_loss_mlp": 0.01041508, + "balance_loss_clip": 1.04757428, + "balance_loss_mlp": 1.02320981, + "epoch": 0.42638129020621657, + "flos": 25884770670720.0, + "grad_norm": 1.7756430296646113, + "language_loss": 0.7952776, + "learning_rate": 2.5662503490204187e-06, + "loss": 0.81716442, + "num_input_tokens_seen": 76393230, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.18310547, + "step": 3546, + "time_per_iteration": 3.1278817653656006 + }, + { + "auxiliary_loss_clip": 0.01147362, + "auxiliary_loss_mlp": 0.01040986, + "balance_loss_clip": 1.0457406, + "balance_loss_mlp": 1.0229733, + "epoch": 0.4265015330968556, + "flos": 26502362138880.0, + "grad_norm": 2.0018405304096123, + "language_loss": 0.75888956, + "learning_rate": 2.5655032076515603e-06, + "loss": 0.78077298, + "num_input_tokens_seen": 76412555, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.18005371, + "step": 3547, + "time_per_iteration": 3.0799338817596436 + }, + { + "auxiliary_loss_clip": 0.01146909, + "auxiliary_loss_mlp": 0.01048652, + "balance_loss_clip": 1.04729056, + "balance_loss_mlp": 1.02989984, + "epoch": 0.42662177598749473, + "flos": 24389522288640.0, + "grad_norm": 2.294000409529217, + "language_loss": 0.82444066, + "learning_rate": 2.5647559804963155e-06, + "loss": 0.84639627, + "num_input_tokens_seen": 76432485, + "router_z_loss_clip": 0.99609375, + "router_z_loss_mlp": 0.1875, + "step": 3548, + "time_per_iteration": 3.1523563861846924 + }, + { + "auxiliary_loss_clip": 0.01150326, + "auxiliary_loss_mlp": 0.01052637, + "balance_loss_clip": 1.04870975, + "balance_loss_mlp": 1.03454125, + "epoch": 0.42674201887813384, + "flos": 23148629089920.0, + "grad_norm": 1.9435035354386951, + "language_loss": 0.79123187, + "learning_rate": 2.5640086676680364e-06, + "loss": 0.81326151, + "num_input_tokens_seen": 76453980, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.18103027, + "step": 3549, + "time_per_iteration": 3.050180673599243 + }, + { + "auxiliary_loss_clip": 0.01149425, + "auxiliary_loss_mlp": 0.0104599, + "balance_loss_clip": 1.04886162, + "balance_loss_mlp": 1.02732134, + "epoch": 0.4268622617687729, + "flos": 21689614552320.0, + "grad_norm": 5.182787856192115, + "language_loss": 0.8084932, + "learning_rate": 2.5632612692800923e-06, + "loss": 0.83044732, + "num_input_tokens_seen": 76473045, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.18664551, + "step": 3550, + "time_per_iteration": 2.987736940383911 + }, + { + "auxiliary_loss_clip": 0.01158679, + "auxiliary_loss_mlp": 0.01058658, + "balance_loss_clip": 1.05238807, + "balance_loss_mlp": 1.03737903, + "epoch": 0.426982504659412, + "flos": 23440151871360.0, + "grad_norm": 2.162794480430706, + "language_loss": 0.74993593, + "learning_rate": 2.5625137854458603e-06, + "loss": 0.77210927, + "num_input_tokens_seen": 76492060, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.21276855, + "step": 3551, + "time_per_iteration": 2.83634090423584 + }, + { + "auxiliary_loss_clip": 0.01141719, + "auxiliary_loss_mlp": 0.01041865, + "balance_loss_clip": 1.04527497, + "balance_loss_mlp": 1.02416253, + "epoch": 0.4271027475500511, + "flos": 18916556768640.0, + "grad_norm": 1.8354253052419458, + "language_loss": 0.80378687, + "learning_rate": 2.561766216278735e-06, + "loss": 0.82562274, + "num_input_tokens_seen": 76509655, + "router_z_loss_clip": 0.96435547, + "router_z_loss_mlp": 0.17687988, + "step": 3552, + "time_per_iteration": 2.870065212249756 + }, + { + "auxiliary_loss_clip": 0.01147018, + "auxiliary_loss_mlp": 0.01052969, + "balance_loss_clip": 1.04707217, + "balance_loss_mlp": 1.03416955, + "epoch": 0.4272229904406902, + "flos": 26870554500480.0, + "grad_norm": 1.8200767860818432, + "language_loss": 0.81265855, + "learning_rate": 2.561018561892121e-06, + "loss": 0.8346585, + "num_input_tokens_seen": 76528795, + "router_z_loss_clip": 0.99804688, + "router_z_loss_mlp": 0.18811035, + "step": 3553, + "time_per_iteration": 2.9842734336853027 + }, + { + "auxiliary_loss_clip": 0.01146587, + "auxiliary_loss_mlp": 0.0104608, + "balance_loss_clip": 1.04602385, + "balance_loss_mlp": 1.02761436, + "epoch": 0.4273432333313293, + "flos": 23951376190080.0, + "grad_norm": 1.6138097374571945, + "language_loss": 0.7694034, + "learning_rate": 2.5602708223994363e-06, + "loss": 0.7913301, + "num_input_tokens_seen": 76550660, + "router_z_loss_clip": 1.00634766, + "router_z_loss_mlp": 0.18469238, + "step": 3554, + "time_per_iteration": 2.8953073024749756 + }, + { + "auxiliary_loss_clip": 0.01148196, + "auxiliary_loss_mlp": 0.01045789, + "balance_loss_clip": 1.04768276, + "balance_loss_mlp": 1.02743077, + "epoch": 0.4274634762219684, + "flos": 29570354496000.0, + "grad_norm": 2.125262193887397, + "language_loss": 0.68257087, + "learning_rate": 2.559522997914115e-06, + "loss": 0.70451075, + "num_input_tokens_seen": 76570240, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.18371582, + "step": 3555, + "time_per_iteration": 3.052739143371582 + }, + { + "auxiliary_loss_clip": 0.0114805, + "auxiliary_loss_mlp": 0.010439, + "balance_loss_clip": 1.04947209, + "balance_loss_mlp": 1.02613819, + "epoch": 0.42758371911260745, + "flos": 21434146047360.0, + "grad_norm": 1.959771114438203, + "language_loss": 0.84973288, + "learning_rate": 2.558775088549599e-06, + "loss": 0.87165236, + "num_input_tokens_seen": 76589820, + "router_z_loss_clip": 0.98681641, + "router_z_loss_mlp": 0.17785645, + "step": 3556, + "time_per_iteration": 2.792985439300537 + }, + { + "auxiliary_loss_clip": 0.01152341, + "auxiliary_loss_mlp": 0.01048244, + "balance_loss_clip": 1.04848409, + "balance_loss_mlp": 1.0279541, + "epoch": 0.42770396200324656, + "flos": 14752822072320.0, + "grad_norm": 2.6671200343454755, + "language_loss": 0.66677773, + "learning_rate": 2.5580270944193467e-06, + "loss": 0.68878365, + "num_input_tokens_seen": 76606640, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.20263672, + "step": 3557, + "time_per_iteration": 2.9482827186584473 + }, + { + "auxiliary_loss_clip": 0.0105314, + "auxiliary_loss_mlp": 0.01005386, + "balance_loss_clip": 1.01946902, + "balance_loss_mlp": 1.00320458, + "epoch": 0.4278242048938857, + "flos": 70654712601600.0, + "grad_norm": 0.7532796434028362, + "language_loss": 0.55478865, + "learning_rate": 2.557279015636827e-06, + "loss": 0.57537395, + "num_input_tokens_seen": 76667050, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.02185059, + "step": 3558, + "time_per_iteration": 3.4837536811828613 + }, + { + "auxiliary_loss_clip": 0.01051492, + "auxiliary_loss_mlp": 0.01003938, + "balance_loss_clip": 1.01773024, + "balance_loss_mlp": 1.00175619, + "epoch": 0.42794444778452473, + "flos": 69366165033600.0, + "grad_norm": 0.7694421608524971, + "language_loss": 0.61261332, + "learning_rate": 2.5565308523155245e-06, + "loss": 0.63316762, + "num_input_tokens_seen": 76726650, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.02185059, + "step": 3559, + "time_per_iteration": 3.3197357654571533 + }, + { + "auxiliary_loss_clip": 0.01145994, + "auxiliary_loss_mlp": 0.01054112, + "balance_loss_clip": 1.04890132, + "balance_loss_mlp": 1.03459716, + "epoch": 0.42806469067516384, + "flos": 18215328481920.0, + "grad_norm": 2.9880635162723364, + "language_loss": 0.82591093, + "learning_rate": 2.5557826045689336e-06, + "loss": 0.84791195, + "num_input_tokens_seen": 76742890, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.1953125, + "step": 3560, + "time_per_iteration": 2.835254430770874 + }, + { + "auxiliary_loss_clip": 0.01051315, + "auxiliary_loss_mlp": 0.01003125, + "balance_loss_clip": 1.01755047, + "balance_loss_mlp": 1.00097942, + "epoch": 0.4281849335658029, + "flos": 54535814432640.0, + "grad_norm": 0.8327904649325678, + "language_loss": 0.58755821, + "learning_rate": 2.5550342725105643e-06, + "loss": 0.60810256, + "num_input_tokens_seen": 76801055, + "router_z_loss_clip": 0.33789062, + "router_z_loss_mlp": 0.02148438, + "step": 3561, + "time_per_iteration": 3.300941228866577 + }, + { + "auxiliary_loss_clip": 0.01152491, + "auxiliary_loss_mlp": 0.01053625, + "balance_loss_clip": 1.0504992, + "balance_loss_mlp": 1.03387189, + "epoch": 0.428305176456442, + "flos": 17274828723840.0, + "grad_norm": 1.883578491497875, + "language_loss": 0.80898464, + "learning_rate": 2.554285856253937e-06, + "loss": 0.83104581, + "num_input_tokens_seen": 76819890, + "router_z_loss_clip": 1.02099609, + "router_z_loss_mlp": 0.19763184, + "step": 3562, + "time_per_iteration": 2.841547727584839 + }, + { + "auxiliary_loss_clip": 0.01145614, + "auxiliary_loss_mlp": 0.01051567, + "balance_loss_clip": 1.04819667, + "balance_loss_mlp": 1.03438854, + "epoch": 0.4284254193470811, + "flos": 26359509749760.0, + "grad_norm": 2.3362143018883574, + "language_loss": 0.77419323, + "learning_rate": 2.5535373559125855e-06, + "loss": 0.79616505, + "num_input_tokens_seen": 76840255, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.17175293, + "step": 3563, + "time_per_iteration": 2.930449962615967 + }, + { + "auxiliary_loss_clip": 0.01151318, + "auxiliary_loss_mlp": 0.01047516, + "balance_loss_clip": 1.04778016, + "balance_loss_mlp": 1.02772772, + "epoch": 0.42854566223772017, + "flos": 29714248379520.0, + "grad_norm": 1.6187917090325434, + "language_loss": 0.82355356, + "learning_rate": 2.552788771600057e-06, + "loss": 0.84554189, + "num_input_tokens_seen": 76860565, + "router_z_loss_clip": 1.03613281, + "router_z_loss_mlp": 0.19799805, + "step": 3564, + "time_per_iteration": 2.889035940170288 + }, + { + "auxiliary_loss_clip": 0.0115372, + "auxiliary_loss_mlp": 0.01043551, + "balance_loss_clip": 1.0511477, + "balance_loss_mlp": 1.02572894, + "epoch": 0.4286659051283593, + "flos": 22018161277440.0, + "grad_norm": 3.1322900909462597, + "language_loss": 0.82338375, + "learning_rate": 2.5520401034299118e-06, + "loss": 0.84535646, + "num_input_tokens_seen": 76878325, + "router_z_loss_clip": 1.02587891, + "router_z_loss_mlp": 0.17822266, + "step": 3565, + "time_per_iteration": 3.8707079887390137 + }, + { + "auxiliary_loss_clip": 0.01155631, + "auxiliary_loss_mlp": 0.01055043, + "balance_loss_clip": 1.04858923, + "balance_loss_mlp": 1.03500366, + "epoch": 0.4287861480189984, + "flos": 13334422838400.0, + "grad_norm": 2.073410044281034, + "language_loss": 0.88451314, + "learning_rate": 2.551291351515722e-06, + "loss": 0.90661991, + "num_input_tokens_seen": 76895340, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.20056152, + "step": 3566, + "time_per_iteration": 2.9389419555664062 + }, + { + "auxiliary_loss_clip": 0.01143775, + "auxiliary_loss_mlp": 0.01056297, + "balance_loss_clip": 1.04362273, + "balance_loss_mlp": 1.0374856, + "epoch": 0.42890639090963745, + "flos": 26651535321600.0, + "grad_norm": 1.6745992761973787, + "language_loss": 0.85828131, + "learning_rate": 2.5505425159710726e-06, + "loss": 0.88028204, + "num_input_tokens_seen": 76915150, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.18823242, + "step": 3567, + "time_per_iteration": 2.964754104614258 + }, + { + "auxiliary_loss_clip": 0.01152833, + "auxiliary_loss_mlp": 0.01048915, + "balance_loss_clip": 1.04996979, + "balance_loss_mlp": 1.03016329, + "epoch": 0.42902663380027656, + "flos": 24055768091520.0, + "grad_norm": 2.335221266307286, + "language_loss": 0.83111346, + "learning_rate": 2.549793596909561e-06, + "loss": 0.85313094, + "num_input_tokens_seen": 76933770, + "router_z_loss_clip": 1.02832031, + "router_z_loss_mlp": 0.18762207, + "step": 3568, + "time_per_iteration": 3.758463144302368 + }, + { + "auxiliary_loss_clip": 0.01147597, + "auxiliary_loss_mlp": 0.010455, + "balance_loss_clip": 1.04622817, + "balance_loss_mlp": 1.02615225, + "epoch": 0.42914687669091567, + "flos": 15632561975040.0, + "grad_norm": 1.9067951269110193, + "language_loss": 0.65950596, + "learning_rate": 2.5490445944447976e-06, + "loss": 0.6814369, + "num_input_tokens_seen": 76952265, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.19372559, + "step": 3569, + "time_per_iteration": 2.862255811691284 + }, + { + "auxiliary_loss_clip": 0.01147101, + "auxiliary_loss_mlp": 0.01051336, + "balance_loss_clip": 1.0476681, + "balance_loss_mlp": 1.03260863, + "epoch": 0.4292671195815547, + "flos": 31467802440960.0, + "grad_norm": 2.094039582031906, + "language_loss": 0.65354621, + "learning_rate": 2.548295508690406e-06, + "loss": 0.67553055, + "num_input_tokens_seen": 76973560, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.18737793, + "step": 3570, + "time_per_iteration": 2.910816192626953 + }, + { + "auxiliary_loss_clip": 0.01150629, + "auxiliary_loss_mlp": 0.01058405, + "balance_loss_clip": 1.04852819, + "balance_loss_mlp": 1.03954589, + "epoch": 0.42938736247219383, + "flos": 30257756046720.0, + "grad_norm": 7.17496575756128, + "language_loss": 0.76501036, + "learning_rate": 2.5475463397600217e-06, + "loss": 0.78710067, + "num_input_tokens_seen": 76993640, + "router_z_loss_clip": 1.02197266, + "router_z_loss_mlp": 0.18859863, + "step": 3571, + "time_per_iteration": 3.7917122840881348 + }, + { + "auxiliary_loss_clip": 0.011505, + "auxiliary_loss_mlp": 0.01056746, + "balance_loss_clip": 1.0475353, + "balance_loss_mlp": 1.03584814, + "epoch": 0.42950760536283294, + "flos": 29349683291520.0, + "grad_norm": 2.157206825841722, + "language_loss": 0.76977921, + "learning_rate": 2.546797087767293e-06, + "loss": 0.79185164, + "num_input_tokens_seen": 77013765, + "router_z_loss_clip": 1.02929688, + "router_z_loss_mlp": 0.2088623, + "step": 3572, + "time_per_iteration": 2.8355135917663574 + }, + { + "auxiliary_loss_clip": 0.01146543, + "auxiliary_loss_mlp": 0.0104791, + "balance_loss_clip": 1.04755139, + "balance_loss_mlp": 1.02920604, + "epoch": 0.429627848253472, + "flos": 26869943969280.0, + "grad_norm": 1.6848329317751316, + "language_loss": 0.87300158, + "learning_rate": 2.546047752825881e-06, + "loss": 0.8949461, + "num_input_tokens_seen": 77034370, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.18725586, + "step": 3573, + "time_per_iteration": 3.7640719413757324 + }, + { + "auxiliary_loss_clip": 0.0114419, + "auxiliary_loss_mlp": 0.01052501, + "balance_loss_clip": 1.04455185, + "balance_loss_mlp": 1.03359413, + "epoch": 0.4297480911441111, + "flos": 13881270470400.0, + "grad_norm": 2.265677523058224, + "language_loss": 0.93524861, + "learning_rate": 2.5452983350494595e-06, + "loss": 0.95721549, + "num_input_tokens_seen": 77049925, + "router_z_loss_clip": 0.99658203, + "router_z_loss_mlp": 0.18920898, + "step": 3574, + "time_per_iteration": 2.8111088275909424 + }, + { + "auxiliary_loss_clip": 0.01153012, + "auxiliary_loss_mlp": 0.01042496, + "balance_loss_clip": 1.05182254, + "balance_loss_mlp": 1.02306473, + "epoch": 0.4298683340347502, + "flos": 20741141975040.0, + "grad_norm": 2.1739904364579523, + "language_loss": 0.65141821, + "learning_rate": 2.544548834551713e-06, + "loss": 0.67337328, + "num_input_tokens_seen": 77068930, + "router_z_loss_clip": 1.01220703, + "router_z_loss_mlp": 0.19433594, + "step": 3575, + "time_per_iteration": 2.8540995121002197 + }, + { + "auxiliary_loss_clip": 0.01146913, + "auxiliary_loss_mlp": 0.01043677, + "balance_loss_clip": 1.04720223, + "balance_loss_mlp": 1.02425826, + "epoch": 0.4299885769253893, + "flos": 20882126856960.0, + "grad_norm": 2.5733890816312606, + "language_loss": 0.95137036, + "learning_rate": 2.5437992514463424e-06, + "loss": 0.97327626, + "num_input_tokens_seen": 77082255, + "router_z_loss_clip": 0.99707031, + "router_z_loss_mlp": 0.1940918, + "step": 3576, + "time_per_iteration": 2.788933038711548 + }, + { + "auxiliary_loss_clip": 0.01152195, + "auxiliary_loss_mlp": 0.01052291, + "balance_loss_clip": 1.04857469, + "balance_loss_mlp": 1.03289533, + "epoch": 0.4301088198160284, + "flos": 25484618183040.0, + "grad_norm": 1.7317578898203114, + "language_loss": 0.88129163, + "learning_rate": 2.5430495858470565e-06, + "loss": 0.90333652, + "num_input_tokens_seen": 77101725, + "router_z_loss_clip": 1.03613281, + "router_z_loss_mlp": 0.19396973, + "step": 3577, + "time_per_iteration": 2.814013957977295 + }, + { + "auxiliary_loss_clip": 0.01152893, + "auxiliary_loss_mlp": 0.01044002, + "balance_loss_clip": 1.05160666, + "balance_loss_mlp": 1.0250237, + "epoch": 0.43022906270666744, + "flos": 18259427404800.0, + "grad_norm": 3.2638374751841908, + "language_loss": 0.77533913, + "learning_rate": 2.54229983786758e-06, + "loss": 0.79730809, + "num_input_tokens_seen": 77119670, + "router_z_loss_clip": 1.01220703, + "router_z_loss_mlp": 0.18994141, + "step": 3578, + "time_per_iteration": 2.7893285751342773 + }, + { + "auxiliary_loss_clip": 0.01149881, + "auxiliary_loss_mlp": 0.01051651, + "balance_loss_clip": 1.04807591, + "balance_loss_mlp": 1.03196943, + "epoch": 0.43034930559730655, + "flos": 23399536567680.0, + "grad_norm": 1.738892608392659, + "language_loss": 0.8525492, + "learning_rate": 2.541550007621651e-06, + "loss": 0.87456447, + "num_input_tokens_seen": 77138160, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.19702148, + "step": 3579, + "time_per_iteration": 2.7779815196990967 + }, + { + "auxiliary_loss_clip": 0.01147419, + "auxiliary_loss_mlp": 0.01045206, + "balance_loss_clip": 1.04912138, + "balance_loss_mlp": 1.02589393, + "epoch": 0.43046954848794566, + "flos": 28184382264960.0, + "grad_norm": 2.0216001281451748, + "language_loss": 0.80316836, + "learning_rate": 2.5408000952230156e-06, + "loss": 0.82509458, + "num_input_tokens_seen": 77156950, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.19311523, + "step": 3580, + "time_per_iteration": 2.843174695968628 + }, + { + "auxiliary_loss_clip": 0.01153376, + "auxiliary_loss_mlp": 0.01050724, + "balance_loss_clip": 1.04947388, + "balance_loss_mlp": 1.0309236, + "epoch": 0.4305897913785847, + "flos": 28580476515840.0, + "grad_norm": 2.066888378484179, + "language_loss": 0.90799803, + "learning_rate": 2.5400501007854357e-06, + "loss": 0.93003899, + "num_input_tokens_seen": 77176395, + "router_z_loss_clip": 1.04003906, + "router_z_loss_mlp": 0.19799805, + "step": 3581, + "time_per_iteration": 2.8683371543884277 + }, + { + "auxiliary_loss_clip": 0.01147401, + "auxiliary_loss_mlp": 0.01050897, + "balance_loss_clip": 1.04727471, + "balance_loss_mlp": 1.03141832, + "epoch": 0.43071003426922383, + "flos": 20448721353600.0, + "grad_norm": 1.8804990158055377, + "language_loss": 0.75721711, + "learning_rate": 2.539300024422685e-06, + "loss": 0.77920008, + "num_input_tokens_seen": 77194340, + "router_z_loss_clip": 1.00195312, + "router_z_loss_mlp": 0.19482422, + "step": 3582, + "time_per_iteration": 2.8726558685302734 + }, + { + "auxiliary_loss_clip": 0.01045935, + "auxiliary_loss_mlp": 0.01025523, + "balance_loss_clip": 1.01279354, + "balance_loss_mlp": 1.02347279, + "epoch": 0.43083027715986294, + "flos": 51997969883520.0, + "grad_norm": 0.7958111590790847, + "language_loss": 0.60851473, + "learning_rate": 2.538549866248549e-06, + "loss": 0.62922931, + "num_input_tokens_seen": 77249320, + "router_z_loss_clip": 0.33203125, + "router_z_loss_mlp": 0.02050781, + "step": 3583, + "time_per_iteration": 3.279599905014038 + }, + { + "auxiliary_loss_clip": 0.01154523, + "auxiliary_loss_mlp": 0.01044803, + "balance_loss_clip": 1.05022836, + "balance_loss_mlp": 1.0257895, + "epoch": 0.430950520050502, + "flos": 16690885320960.0, + "grad_norm": 2.087946469897322, + "language_loss": 0.81111336, + "learning_rate": 2.5377996263768274e-06, + "loss": 0.8331067, + "num_input_tokens_seen": 77267400, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.19018555, + "step": 3584, + "time_per_iteration": 2.845665693283081 + }, + { + "auxiliary_loss_clip": 0.01148709, + "auxiliary_loss_mlp": 0.01046162, + "balance_loss_clip": 1.04872322, + "balance_loss_mlp": 1.02631378, + "epoch": 0.4310707629411411, + "flos": 24608433726720.0, + "grad_norm": 1.865043261292025, + "language_loss": 0.68775475, + "learning_rate": 2.5370493049213293e-06, + "loss": 0.70970345, + "num_input_tokens_seen": 77287045, + "router_z_loss_clip": 0.99902344, + "router_z_loss_mlp": 0.19848633, + "step": 3585, + "time_per_iteration": 2.8498916625976562 + }, + { + "auxiliary_loss_clip": 0.01153872, + "auxiliary_loss_mlp": 0.01051507, + "balance_loss_clip": 1.04814172, + "balance_loss_mlp": 1.03080046, + "epoch": 0.4311910058317802, + "flos": 26432983019520.0, + "grad_norm": 1.9524573656385045, + "language_loss": 0.8016274, + "learning_rate": 2.536298901995878e-06, + "loss": 0.82368112, + "num_input_tokens_seen": 77306255, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.20703125, + "step": 3586, + "time_per_iteration": 2.8723766803741455 + }, + { + "auxiliary_loss_clip": 0.01149104, + "auxiliary_loss_mlp": 0.01050655, + "balance_loss_clip": 1.04749537, + "balance_loss_mlp": 1.0303421, + "epoch": 0.43131124872241927, + "flos": 25155891889920.0, + "grad_norm": 1.7505937509560798, + "language_loss": 0.79985631, + "learning_rate": 2.535548417714311e-06, + "loss": 0.82185388, + "num_input_tokens_seen": 77325555, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.203125, + "step": 3587, + "time_per_iteration": 2.8102049827575684 + }, + { + "auxiliary_loss_clip": 0.01148591, + "auxiliary_loss_mlp": 0.01050593, + "balance_loss_clip": 1.04681873, + "balance_loss_mlp": 1.03061342, + "epoch": 0.4314314916130584, + "flos": 21614812479360.0, + "grad_norm": 3.2999683477603003, + "language_loss": 0.87311029, + "learning_rate": 2.534797852190474e-06, + "loss": 0.89510208, + "num_input_tokens_seen": 77345735, + "router_z_loss_clip": 1.01855469, + "router_z_loss_mlp": 0.1998291, + "step": 3588, + "time_per_iteration": 2.8400614261627197 + }, + { + "auxiliary_loss_clip": 0.01149904, + "auxiliary_loss_mlp": 0.01062225, + "balance_loss_clip": 1.0485363, + "balance_loss_mlp": 1.04213822, + "epoch": 0.4315517345036975, + "flos": 19275016544640.0, + "grad_norm": 1.9804569138228145, + "language_loss": 0.81696337, + "learning_rate": 2.5340472055382283e-06, + "loss": 0.83908463, + "num_input_tokens_seen": 77361765, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.20080566, + "step": 3589, + "time_per_iteration": 2.7913804054260254 + }, + { + "auxiliary_loss_clip": 0.01146644, + "auxiliary_loss_mlp": 0.01045788, + "balance_loss_clip": 1.04753995, + "balance_loss_mlp": 1.02636909, + "epoch": 0.43167197739433655, + "flos": 24273853516800.0, + "grad_norm": 1.9174022280466991, + "language_loss": 0.80734199, + "learning_rate": 2.5332964778714468e-06, + "loss": 0.82926631, + "num_input_tokens_seen": 77378950, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.1940918, + "step": 3590, + "time_per_iteration": 2.807464599609375 + }, + { + "auxiliary_loss_clip": 0.01151013, + "auxiliary_loss_mlp": 0.01052124, + "balance_loss_clip": 1.04865193, + "balance_loss_mlp": 1.03288412, + "epoch": 0.43179222028497566, + "flos": 16867816738560.0, + "grad_norm": 1.6530282767418194, + "language_loss": 0.66782761, + "learning_rate": 2.5325456693040123e-06, + "loss": 0.68985897, + "num_input_tokens_seen": 77396145, + "router_z_loss_clip": 1.0234375, + "router_z_loss_mlp": 0.19238281, + "step": 3591, + "time_per_iteration": 2.770920753479004 + }, + { + "auxiliary_loss_clip": 0.01151996, + "auxiliary_loss_mlp": 0.01048406, + "balance_loss_clip": 1.04855847, + "balance_loss_mlp": 1.02690041, + "epoch": 0.43191246317561477, + "flos": 17639214243840.0, + "grad_norm": 2.6426151872507315, + "language_loss": 0.75343084, + "learning_rate": 2.531794779949824e-06, + "loss": 0.77543485, + "num_input_tokens_seen": 77414045, + "router_z_loss_clip": 1.03515625, + "router_z_loss_mlp": 0.21508789, + "step": 3592, + "time_per_iteration": 2.7784652709960938 + }, + { + "auxiliary_loss_clip": 0.01147724, + "auxiliary_loss_mlp": 0.01042193, + "balance_loss_clip": 1.04889154, + "balance_loss_mlp": 1.02439475, + "epoch": 0.4320327060662538, + "flos": 23878800760320.0, + "grad_norm": 1.9893562571027918, + "language_loss": 0.88038093, + "learning_rate": 2.5310438099227903e-06, + "loss": 0.90228009, + "num_input_tokens_seen": 77431310, + "router_z_loss_clip": 0.98779297, + "router_z_loss_mlp": 0.17797852, + "step": 3593, + "time_per_iteration": 2.8047525882720947 + }, + { + "auxiliary_loss_clip": 0.01045999, + "auxiliary_loss_mlp": 0.01004195, + "balance_loss_clip": 1.01316297, + "balance_loss_mlp": 1.00231135, + "epoch": 0.43215294895689293, + "flos": 66394917959040.0, + "grad_norm": 0.8037972142697705, + "language_loss": 0.5340783, + "learning_rate": 2.530292759336833e-06, + "loss": 0.55458015, + "num_input_tokens_seen": 77492045, + "router_z_loss_clip": 0.328125, + "router_z_loss_mlp": 0.01879883, + "step": 3594, + "time_per_iteration": 4.165318727493286 + }, + { + "auxiliary_loss_clip": 0.01151914, + "auxiliary_loss_mlp": 0.01046009, + "balance_loss_clip": 1.04676008, + "balance_loss_mlp": 1.02552843, + "epoch": 0.432273191847532, + "flos": 20594267262720.0, + "grad_norm": 1.9719126890873964, + "language_loss": 0.70245004, + "learning_rate": 2.5295416283058855e-06, + "loss": 0.72442925, + "num_input_tokens_seen": 77510910, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.20483398, + "step": 3595, + "time_per_iteration": 2.8178536891937256 + }, + { + "auxiliary_loss_clip": 0.01147904, + "auxiliary_loss_mlp": 0.01043881, + "balance_loss_clip": 1.04839933, + "balance_loss_mlp": 1.0259285, + "epoch": 0.4323934347381711, + "flos": 19282127437440.0, + "grad_norm": 1.652699644163366, + "language_loss": 0.66413969, + "learning_rate": 2.5287904169438943e-06, + "loss": 0.68605757, + "num_input_tokens_seen": 77530115, + "router_z_loss_clip": 0.99658203, + "router_z_loss_mlp": 0.17956543, + "step": 3596, + "time_per_iteration": 2.768131732940674 + }, + { + "auxiliary_loss_clip": 0.01157819, + "auxiliary_loss_mlp": 0.01051212, + "balance_loss_clip": 1.05159783, + "balance_loss_mlp": 1.03091025, + "epoch": 0.4325136776288102, + "flos": 21726315273600.0, + "grad_norm": 2.1313617848200854, + "language_loss": 0.63706851, + "learning_rate": 2.528039125364817e-06, + "loss": 0.65915877, + "num_input_tokens_seen": 77548920, + "router_z_loss_clip": 1.06298828, + "router_z_loss_mlp": 0.20300293, + "step": 3597, + "time_per_iteration": 3.598252058029175 + }, + { + "auxiliary_loss_clip": 0.01147851, + "auxiliary_loss_mlp": 0.01043801, + "balance_loss_clip": 1.04510725, + "balance_loss_mlp": 1.0226891, + "epoch": 0.43263392051944927, + "flos": 22340746344960.0, + "grad_norm": 2.228374379089444, + "language_loss": 0.76282829, + "learning_rate": 2.5272877536826246e-06, + "loss": 0.78474486, + "num_input_tokens_seen": 77567715, + "router_z_loss_clip": 1.02783203, + "router_z_loss_mlp": 0.21118164, + "step": 3598, + "time_per_iteration": 2.8835582733154297 + }, + { + "auxiliary_loss_clip": 0.01148158, + "auxiliary_loss_mlp": 0.01047648, + "balance_loss_clip": 1.04521251, + "balance_loss_mlp": 1.02836013, + "epoch": 0.4327541634100884, + "flos": 29168406328320.0, + "grad_norm": 2.8074753434052093, + "language_loss": 0.70907301, + "learning_rate": 2.5265363020112986e-06, + "loss": 0.73103106, + "num_input_tokens_seen": 77588035, + "router_z_loss_clip": 1.03076172, + "router_z_loss_mlp": 0.19287109, + "step": 3599, + "time_per_iteration": 2.8120229244232178 + }, + { + "auxiliary_loss_clip": 0.01150452, + "auxiliary_loss_mlp": 0.01043953, + "balance_loss_clip": 1.04887354, + "balance_loss_mlp": 1.02539241, + "epoch": 0.4328744063007275, + "flos": 26067448264320.0, + "grad_norm": 1.7923004611837272, + "language_loss": 0.83902335, + "learning_rate": 2.5257847704648344e-06, + "loss": 0.8609674, + "num_input_tokens_seen": 77609265, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.18566895, + "step": 3600, + "time_per_iteration": 3.6295766830444336 + }, + { + "auxiliary_loss_clip": 0.01145244, + "auxiliary_loss_mlp": 0.01042954, + "balance_loss_clip": 1.04620552, + "balance_loss_mlp": 1.02442896, + "epoch": 0.43299464919136654, + "flos": 16581357774720.0, + "grad_norm": 1.7235961298897513, + "language_loss": 0.75709832, + "learning_rate": 2.525033159157239e-06, + "loss": 0.77898037, + "num_input_tokens_seen": 77625580, + "router_z_loss_clip": 0.98974609, + "router_z_loss_mlp": 0.18518066, + "step": 3601, + "time_per_iteration": 2.8744664192199707 + }, + { + "auxiliary_loss_clip": 0.01153451, + "auxiliary_loss_mlp": 0.01046802, + "balance_loss_clip": 1.04760313, + "balance_loss_mlp": 1.02448642, + "epoch": 0.43311489208200565, + "flos": 16107265140480.0, + "grad_norm": 1.69917386346086, + "language_loss": 0.77052265, + "learning_rate": 2.52428146820253e-06, + "loss": 0.79252523, + "num_input_tokens_seen": 77643835, + "router_z_loss_clip": 1.05908203, + "router_z_loss_mlp": 0.22314453, + "step": 3602, + "time_per_iteration": 3.6335580348968506 + }, + { + "auxiliary_loss_clip": 0.01152813, + "auxiliary_loss_mlp": 0.01046005, + "balance_loss_clip": 1.04712975, + "balance_loss_mlp": 1.02618027, + "epoch": 0.43323513497264476, + "flos": 22930220442240.0, + "grad_norm": 1.7970976010821096, + "language_loss": 0.8194803, + "learning_rate": 2.52352969771474e-06, + "loss": 0.84146845, + "num_input_tokens_seen": 77663060, + "router_z_loss_clip": 1.05859375, + "router_z_loss_mlp": 0.19836426, + "step": 3603, + "time_per_iteration": 2.880255937576294 + }, + { + "auxiliary_loss_clip": 0.0114567, + "auxiliary_loss_mlp": 0.01041811, + "balance_loss_clip": 1.04674017, + "balance_loss_mlp": 1.02341688, + "epoch": 0.4333553778632838, + "flos": 25299031587840.0, + "grad_norm": 2.0944643873302904, + "language_loss": 0.88246858, + "learning_rate": 2.5227778478079106e-06, + "loss": 0.90434343, + "num_input_tokens_seen": 77682470, + "router_z_loss_clip": 0.98925781, + "router_z_loss_mlp": 0.18395996, + "step": 3604, + "time_per_iteration": 2.828904390335083 + }, + { + "auxiliary_loss_clip": 0.01141411, + "auxiliary_loss_mlp": 0.01045937, + "balance_loss_clip": 1.04618728, + "balance_loss_mlp": 1.02906942, + "epoch": 0.43347562075392293, + "flos": 19387165783680.0, + "grad_norm": 1.624383319379795, + "language_loss": 0.76998758, + "learning_rate": 2.522025918596098e-06, + "loss": 0.79186106, + "num_input_tokens_seen": 77700770, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.16870117, + "step": 3605, + "time_per_iteration": 2.7993104457855225 + }, + { + "auxiliary_loss_clip": 0.01140737, + "auxiliary_loss_mlp": 0.01043013, + "balance_loss_clip": 1.04564869, + "balance_loss_mlp": 1.02517939, + "epoch": 0.43359586364456204, + "flos": 26325969425280.0, + "grad_norm": 1.5465018983753702, + "language_loss": 0.65533197, + "learning_rate": 2.521273910193368e-06, + "loss": 0.67716944, + "num_input_tokens_seen": 77723950, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.17834473, + "step": 3606, + "time_per_iteration": 2.869337320327759 + }, + { + "auxiliary_loss_clip": 0.01153347, + "auxiliary_loss_mlp": 0.01051828, + "balance_loss_clip": 1.04942048, + "balance_loss_mlp": 1.03352976, + "epoch": 0.4337161065352011, + "flos": 15989261984640.0, + "grad_norm": 2.0268588098622224, + "language_loss": 0.87005836, + "learning_rate": 2.5205218227138006e-06, + "loss": 0.89211011, + "num_input_tokens_seen": 77736905, + "router_z_loss_clip": 1.03808594, + "router_z_loss_mlp": 0.18286133, + "step": 3607, + "time_per_iteration": 2.790456533432007 + }, + { + "auxiliary_loss_clip": 0.01149272, + "auxiliary_loss_mlp": 0.01048341, + "balance_loss_clip": 1.04588413, + "balance_loss_mlp": 1.02947044, + "epoch": 0.4338363494258402, + "flos": 20224710184320.0, + "grad_norm": 2.0465929583808555, + "language_loss": 0.79329753, + "learning_rate": 2.519769656271486e-06, + "loss": 0.81527364, + "num_input_tokens_seen": 77754325, + "router_z_loss_clip": 1.03466797, + "router_z_loss_mlp": 0.1887207, + "step": 3608, + "time_per_iteration": 2.859682083129883 + }, + { + "auxiliary_loss_clip": 0.01146824, + "auxiliary_loss_mlp": 0.01044993, + "balance_loss_clip": 1.04613304, + "balance_loss_mlp": 1.02681327, + "epoch": 0.43395659231647926, + "flos": 20083904870400.0, + "grad_norm": 1.8064326236496338, + "language_loss": 0.67548984, + "learning_rate": 2.5190174109805285e-06, + "loss": 0.69740802, + "num_input_tokens_seen": 77774150, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.1817627, + "step": 3609, + "time_per_iteration": 2.790541648864746 + }, + { + "auxiliary_loss_clip": 0.01143352, + "auxiliary_loss_mlp": 0.0105005, + "balance_loss_clip": 1.04436839, + "balance_loss_mlp": 1.03115571, + "epoch": 0.43407683520711837, + "flos": 19901801894400.0, + "grad_norm": 2.178992789718469, + "language_loss": 0.64398372, + "learning_rate": 2.518265086955042e-06, + "loss": 0.66591775, + "num_input_tokens_seen": 77791870, + "router_z_loss_clip": 0.98974609, + "router_z_loss_mlp": 0.18896484, + "step": 3610, + "time_per_iteration": 2.9249792098999023 + }, + { + "auxiliary_loss_clip": 0.01149398, + "auxiliary_loss_mlp": 0.0105579, + "balance_loss_clip": 1.04778826, + "balance_loss_mlp": 1.03651404, + "epoch": 0.4341970780977575, + "flos": 23108732058240.0, + "grad_norm": 1.9083308370423422, + "language_loss": 0.83682954, + "learning_rate": 2.5175126843091534e-06, + "loss": 0.85888141, + "num_input_tokens_seen": 77811240, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.19274902, + "step": 3611, + "time_per_iteration": 2.772371530532837 + }, + { + "auxiliary_loss_clip": 0.01147335, + "auxiliary_loss_mlp": 0.01045741, + "balance_loss_clip": 1.04813933, + "balance_loss_mlp": 1.02720439, + "epoch": 0.43431732098839654, + "flos": 37408288406400.0, + "grad_norm": 2.0319268469226133, + "language_loss": 0.75496876, + "learning_rate": 2.5167602031570034e-06, + "loss": 0.77689946, + "num_input_tokens_seen": 77831425, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.18530273, + "step": 3612, + "time_per_iteration": 2.887120246887207 + }, + { + "auxiliary_loss_clip": 0.01149192, + "auxiliary_loss_mlp": 0.01044901, + "balance_loss_clip": 1.04817855, + "balance_loss_mlp": 1.02734113, + "epoch": 0.43443756387903565, + "flos": 31868206323840.0, + "grad_norm": 1.7221520486988158, + "language_loss": 0.7319141, + "learning_rate": 2.51600764361274e-06, + "loss": 0.75385499, + "num_input_tokens_seen": 77852950, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.17553711, + "step": 3613, + "time_per_iteration": 2.9111783504486084 + }, + { + "auxiliary_loss_clip": 0.01144823, + "auxiliary_loss_mlp": 0.01042449, + "balance_loss_clip": 1.04613495, + "balance_loss_mlp": 1.02432907, + "epoch": 0.43455780676967476, + "flos": 23477139901440.0, + "grad_norm": 2.337808337671138, + "language_loss": 0.79056346, + "learning_rate": 2.5152550057905283e-06, + "loss": 0.81243622, + "num_input_tokens_seen": 77872840, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.18127441, + "step": 3614, + "time_per_iteration": 3.1552820205688477 + }, + { + "auxiliary_loss_clip": 0.01152377, + "auxiliary_loss_mlp": 0.01046975, + "balance_loss_clip": 1.04854643, + "balance_loss_mlp": 1.02875996, + "epoch": 0.4346780496603138, + "flos": 24207060176640.0, + "grad_norm": 5.1486056670337454, + "language_loss": 0.76754975, + "learning_rate": 2.5145022898045415e-06, + "loss": 0.78954327, + "num_input_tokens_seen": 77892025, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.18225098, + "step": 3615, + "time_per_iteration": 3.142740488052368 + }, + { + "auxiliary_loss_clip": 0.01153908, + "auxiliary_loss_mlp": 0.01050717, + "balance_loss_clip": 1.04917192, + "balance_loss_mlp": 1.03134596, + "epoch": 0.4347982925509529, + "flos": 17092366611840.0, + "grad_norm": 3.582354576491511, + "language_loss": 0.89730543, + "learning_rate": 2.5137494957689664e-06, + "loss": 0.91935164, + "num_input_tokens_seen": 77907635, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.19360352, + "step": 3616, + "time_per_iteration": 2.843812942504883 + }, + { + "auxiliary_loss_clip": 0.01044232, + "auxiliary_loss_mlp": 0.0102238, + "balance_loss_clip": 1.01195049, + "balance_loss_mlp": 1.02049661, + "epoch": 0.43491853544159204, + "flos": 60945544696320.0, + "grad_norm": 0.7680751574034079, + "language_loss": 0.57290065, + "learning_rate": 2.5129966237980016e-06, + "loss": 0.59356678, + "num_input_tokens_seen": 77970630, + "router_z_loss_clip": 0.32421875, + "router_z_loss_mlp": 0.01879883, + "step": 3617, + "time_per_iteration": 3.412804126739502 + }, + { + "auxiliary_loss_clip": 0.0114407, + "auxiliary_loss_mlp": 0.01042752, + "balance_loss_clip": 1.04501748, + "balance_loss_mlp": 1.02429843, + "epoch": 0.4350387783322311, + "flos": 21944652094080.0, + "grad_norm": 1.7975530677100195, + "language_loss": 0.78551441, + "learning_rate": 2.512243674005857e-06, + "loss": 0.80738258, + "num_input_tokens_seen": 77989995, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.18457031, + "step": 3618, + "time_per_iteration": 2.9483487606048584 + }, + { + "auxiliary_loss_clip": 0.01149054, + "auxiliary_loss_mlp": 0.01041299, + "balance_loss_clip": 1.04837763, + "balance_loss_mlp": 1.02332222, + "epoch": 0.4351590212228702, + "flos": 25082705928960.0, + "grad_norm": 2.461705352484294, + "language_loss": 0.85892677, + "learning_rate": 2.5114906465067537e-06, + "loss": 0.88083035, + "num_input_tokens_seen": 78010980, + "router_z_loss_clip": 1.00634766, + "router_z_loss_mlp": 0.17980957, + "step": 3619, + "time_per_iteration": 2.879912853240967 + }, + { + "auxiliary_loss_clip": 0.01144445, + "auxiliary_loss_mlp": 0.01045618, + "balance_loss_clip": 1.04461765, + "balance_loss_mlp": 1.02746272, + "epoch": 0.4352792641135093, + "flos": 21506541909120.0, + "grad_norm": 1.7893250309303352, + "language_loss": 0.75314808, + "learning_rate": 2.5107375414149264e-06, + "loss": 0.77504873, + "num_input_tokens_seen": 78030225, + "router_z_loss_clip": 0.99755859, + "router_z_loss_mlp": 0.18151855, + "step": 3620, + "time_per_iteration": 2.8765933513641357 + }, + { + "auxiliary_loss_clip": 0.01146242, + "auxiliary_loss_mlp": 0.01048597, + "balance_loss_clip": 1.04483628, + "balance_loss_mlp": 1.03038204, + "epoch": 0.43539950700414837, + "flos": 16253457494400.0, + "grad_norm": 1.9849889594782213, + "language_loss": 0.71566832, + "learning_rate": 2.5099843588446197e-06, + "loss": 0.73761678, + "num_input_tokens_seen": 78048545, + "router_z_loss_clip": 1.01464844, + "router_z_loss_mlp": 0.18200684, + "step": 3621, + "time_per_iteration": 2.886343479156494 + }, + { + "auxiliary_loss_clip": 0.01147506, + "auxiliary_loss_mlp": 0.01046796, + "balance_loss_clip": 1.04805684, + "balance_loss_mlp": 1.02848577, + "epoch": 0.4355197498947875, + "flos": 16691819074560.0, + "grad_norm": 1.5886885071670924, + "language_loss": 0.61431015, + "learning_rate": 2.509231098910091e-06, + "loss": 0.63625312, + "num_input_tokens_seen": 78068415, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.18322754, + "step": 3622, + "time_per_iteration": 2.827289342880249 + }, + { + "auxiliary_loss_clip": 0.01145541, + "auxiliary_loss_mlp": 0.01044215, + "balance_loss_clip": 1.04660153, + "balance_loss_mlp": 1.02423573, + "epoch": 0.4356399927854266, + "flos": 16362733645440.0, + "grad_norm": 2.150097218099309, + "language_loss": 0.75066233, + "learning_rate": 2.508477761725611e-06, + "loss": 0.77255988, + "num_input_tokens_seen": 78086690, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.19958496, + "step": 3623, + "time_per_iteration": 3.647566318511963 + }, + { + "auxiliary_loss_clip": 0.01148007, + "auxiliary_loss_mlp": 0.01049327, + "balance_loss_clip": 1.0476346, + "balance_loss_mlp": 1.03048003, + "epoch": 0.43576023567606564, + "flos": 17202037812480.0, + "grad_norm": 1.725388028651848, + "language_loss": 0.81014407, + "learning_rate": 2.507724347405458e-06, + "loss": 0.83211738, + "num_input_tokens_seen": 78104640, + "router_z_loss_clip": 1.00341797, + "router_z_loss_mlp": 0.18823242, + "step": 3624, + "time_per_iteration": 3.0972647666931152 + }, + { + "auxiliary_loss_clip": 0.01148248, + "auxiliary_loss_mlp": 0.0104328, + "balance_loss_clip": 1.04829562, + "balance_loss_mlp": 1.02557743, + "epoch": 0.43588047856670475, + "flos": 15917656222080.0, + "grad_norm": 2.247108758255963, + "language_loss": 0.8222816, + "learning_rate": 2.5069708560639243e-06, + "loss": 0.84419692, + "num_input_tokens_seen": 78122550, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.17700195, + "step": 3625, + "time_per_iteration": 3.2409396171569824 + }, + { + "auxiliary_loss_clip": 0.01146994, + "auxiliary_loss_mlp": 0.01047281, + "balance_loss_clip": 1.0462296, + "balance_loss_mlp": 1.02848148, + "epoch": 0.4360007214573438, + "flos": 23659566099840.0, + "grad_norm": 2.2004179608633514, + "language_loss": 0.61803067, + "learning_rate": 2.5062172878153158e-06, + "loss": 0.6399734, + "num_input_tokens_seen": 78141825, + "router_z_loss_clip": 1.00830078, + "router_z_loss_mlp": 0.18798828, + "step": 3626, + "time_per_iteration": 4.229516267776489 + }, + { + "auxiliary_loss_clip": 0.01144931, + "auxiliary_loss_mlp": 0.01042764, + "balance_loss_clip": 1.0446862, + "balance_loss_mlp": 1.02497804, + "epoch": 0.4361209643479829, + "flos": 21978767036160.0, + "grad_norm": 2.167021631374331, + "language_loss": 0.87562662, + "learning_rate": 2.505463642773947e-06, + "loss": 0.8975035, + "num_input_tokens_seen": 78161790, + "router_z_loss_clip": 1.00195312, + "router_z_loss_mlp": 0.17773438, + "step": 3627, + "time_per_iteration": 2.9706192016601562 + }, + { + "auxiliary_loss_clip": 0.01146585, + "auxiliary_loss_mlp": 0.01049144, + "balance_loss_clip": 1.04564214, + "balance_loss_mlp": 1.03060699, + "epoch": 0.43624120723862203, + "flos": 17420159151360.0, + "grad_norm": 2.2868808805783223, + "language_loss": 0.74919504, + "learning_rate": 2.504709921054146e-06, + "loss": 0.77115238, + "num_input_tokens_seen": 78178605, + "router_z_loss_clip": 1.00927734, + "router_z_loss_mlp": 0.18518066, + "step": 3628, + "time_per_iteration": 2.896054744720459 + }, + { + "auxiliary_loss_clip": 0.0114697, + "auxiliary_loss_mlp": 0.01045406, + "balance_loss_clip": 1.04766142, + "balance_loss_mlp": 1.0271908, + "epoch": 0.4363614501292611, + "flos": 17895293280000.0, + "grad_norm": 2.5893665867064244, + "language_loss": 0.84085321, + "learning_rate": 2.50395612277025e-06, + "loss": 0.86277699, + "num_input_tokens_seen": 78194460, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.18212891, + "step": 3629, + "time_per_iteration": 3.6617395877838135 + }, + { + "auxiliary_loss_clip": 0.01145744, + "auxiliary_loss_mlp": 0.01040686, + "balance_loss_clip": 1.04631937, + "balance_loss_mlp": 1.02117181, + "epoch": 0.4364816930199002, + "flos": 20302888135680.0, + "grad_norm": 2.1097406423537084, + "language_loss": 0.72874451, + "learning_rate": 2.503202248036612e-06, + "loss": 0.75060886, + "num_input_tokens_seen": 78213315, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.19506836, + "step": 3630, + "time_per_iteration": 2.9039134979248047 + }, + { + "auxiliary_loss_clip": 0.01144991, + "auxiliary_loss_mlp": 0.01045303, + "balance_loss_clip": 1.04522896, + "balance_loss_mlp": 1.02706397, + "epoch": 0.4366019359105393, + "flos": 24061334699520.0, + "grad_norm": 2.077165572208598, + "language_loss": 0.73781836, + "learning_rate": 2.5024482969675927e-06, + "loss": 0.75972128, + "num_input_tokens_seen": 78233270, + "router_z_loss_clip": 0.99755859, + "router_z_loss_mlp": 0.18249512, + "step": 3631, + "time_per_iteration": 3.776489734649658 + }, + { + "auxiliary_loss_clip": 0.01141475, + "auxiliary_loss_mlp": 0.0104313, + "balance_loss_clip": 1.0440762, + "balance_loss_mlp": 1.02452123, + "epoch": 0.43672217880117836, + "flos": 21754109422080.0, + "grad_norm": 2.3698138995048903, + "language_loss": 0.84872252, + "learning_rate": 2.501694269677566e-06, + "loss": 0.87056851, + "num_input_tokens_seen": 78251040, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.18615723, + "step": 3632, + "time_per_iteration": 2.863166570663452 + }, + { + "auxiliary_loss_clip": 0.01147703, + "auxiliary_loss_mlp": 0.01039234, + "balance_loss_clip": 1.04617131, + "balance_loss_mlp": 1.02051854, + "epoch": 0.4368424216918175, + "flos": 18035200753920.0, + "grad_norm": 2.0039703557020925, + "language_loss": 0.8045885, + "learning_rate": 2.500940166280918e-06, + "loss": 0.82645786, + "num_input_tokens_seen": 78269470, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.18713379, + "step": 3633, + "time_per_iteration": 2.8458080291748047 + }, + { + "auxiliary_loss_clip": 0.01142493, + "auxiliary_loss_mlp": 0.01041082, + "balance_loss_clip": 1.04316223, + "balance_loss_mlp": 1.02278304, + "epoch": 0.4369626645824566, + "flos": 25447127362560.0, + "grad_norm": 1.9971044729136096, + "language_loss": 0.79043484, + "learning_rate": 2.500185986892045e-06, + "loss": 0.81227058, + "num_input_tokens_seen": 78288955, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.1829834, + "step": 3634, + "time_per_iteration": 3.1153717041015625 + }, + { + "auxiliary_loss_clip": 0.01147833, + "auxiliary_loss_mlp": 0.01047415, + "balance_loss_clip": 1.04634345, + "balance_loss_mlp": 1.02849638, + "epoch": 0.43708290747309564, + "flos": 25302694775040.0, + "grad_norm": 2.1078834060291562, + "language_loss": 0.77427644, + "learning_rate": 2.499431731625355e-06, + "loss": 0.79622889, + "num_input_tokens_seen": 78307980, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.18933105, + "step": 3635, + "time_per_iteration": 3.075068235397339 + }, + { + "auxiliary_loss_clip": 0.0114859, + "auxiliary_loss_mlp": 0.01042136, + "balance_loss_clip": 1.0459528, + "balance_loss_mlp": 1.02393246, + "epoch": 0.43720315036373475, + "flos": 31575103344000.0, + "grad_norm": 1.9089462265092614, + "language_loss": 0.79841757, + "learning_rate": 2.4986774005952686e-06, + "loss": 0.82032478, + "num_input_tokens_seen": 78330355, + "router_z_loss_clip": 1.02783203, + "router_z_loss_mlp": 0.18200684, + "step": 3636, + "time_per_iteration": 3.085268020629883 + }, + { + "auxiliary_loss_clip": 0.01145622, + "auxiliary_loss_mlp": 0.0103402, + "balance_loss_clip": 1.0462842, + "balance_loss_mlp": 1.01606679, + "epoch": 0.43732339325437386, + "flos": 23112000195840.0, + "grad_norm": 2.04134655857564, + "language_loss": 0.84710139, + "learning_rate": 2.4979229939162166e-06, + "loss": 0.8688978, + "num_input_tokens_seen": 78349135, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.17956543, + "step": 3637, + "time_per_iteration": 3.1848227977752686 + }, + { + "auxiliary_loss_clip": 0.0114744, + "auxiliary_loss_mlp": 0.01048229, + "balance_loss_clip": 1.04813075, + "balance_loss_mlp": 1.02984691, + "epoch": 0.4374436361450129, + "flos": 27746272080000.0, + "grad_norm": 1.972132422908748, + "language_loss": 0.80566061, + "learning_rate": 2.4971685117026433e-06, + "loss": 0.82761729, + "num_input_tokens_seen": 78368900, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.18383789, + "step": 3638, + "time_per_iteration": 2.9377002716064453 + }, + { + "auxiliary_loss_clip": 0.01145962, + "auxiliary_loss_mlp": 0.0104774, + "balance_loss_clip": 1.04703891, + "balance_loss_mlp": 1.0294416, + "epoch": 0.437563879035652, + "flos": 24172370616960.0, + "grad_norm": 1.3965524560712388, + "language_loss": 0.76425064, + "learning_rate": 2.4964139540690018e-06, + "loss": 0.78618765, + "num_input_tokens_seen": 78392235, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.1829834, + "step": 3639, + "time_per_iteration": 2.936401128768921 + }, + { + "auxiliary_loss_clip": 0.01146177, + "auxiliary_loss_mlp": 0.01043559, + "balance_loss_clip": 1.04620528, + "balance_loss_mlp": 1.02372277, + "epoch": 0.4376841219262911, + "flos": 23477211728640.0, + "grad_norm": 2.1151954472459735, + "language_loss": 0.72669601, + "learning_rate": 2.495659321129758e-06, + "loss": 0.74859345, + "num_input_tokens_seen": 78409980, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.19836426, + "step": 3640, + "time_per_iteration": 2.8818037509918213 + }, + { + "auxiliary_loss_clip": 0.01148032, + "auxiliary_loss_mlp": 0.01045199, + "balance_loss_clip": 1.04687107, + "balance_loss_mlp": 1.02824783, + "epoch": 0.4378043648169302, + "flos": 25447809720960.0, + "grad_norm": 1.7644068636253172, + "language_loss": 0.75672418, + "learning_rate": 2.494904612999389e-06, + "loss": 0.77865648, + "num_input_tokens_seen": 78428690, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.16943359, + "step": 3641, + "time_per_iteration": 2.8570168018341064 + }, + { + "auxiliary_loss_clip": 0.01043624, + "auxiliary_loss_mlp": 0.01005882, + "balance_loss_clip": 1.01151514, + "balance_loss_mlp": 1.00415373, + "epoch": 0.4379246077075693, + "flos": 53914056986880.0, + "grad_norm": 0.7465397806424459, + "language_loss": 0.56531549, + "learning_rate": 2.4941498297923843e-06, + "loss": 0.58581054, + "num_input_tokens_seen": 78489260, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.01733398, + "step": 3642, + "time_per_iteration": 3.419975757598877 + }, + { + "auxiliary_loss_clip": 0.01146968, + "auxiliary_loss_mlp": 0.01044525, + "balance_loss_clip": 1.04828405, + "balance_loss_mlp": 1.02660811, + "epoch": 0.43804485059820836, + "flos": 20588305605120.0, + "grad_norm": 1.735274011300017, + "language_loss": 0.69383854, + "learning_rate": 2.4933949716232424e-06, + "loss": 0.7157535, + "num_input_tokens_seen": 78506785, + "router_z_loss_clip": 0.98681641, + "router_z_loss_mlp": 0.17919922, + "step": 3643, + "time_per_iteration": 2.9538841247558594 + }, + { + "auxiliary_loss_clip": 0.01145198, + "auxiliary_loss_mlp": 0.01046403, + "balance_loss_clip": 1.04731488, + "balance_loss_mlp": 1.02868867, + "epoch": 0.43816509348884747, + "flos": 23876214981120.0, + "grad_norm": 5.04015306011754, + "language_loss": 0.73592204, + "learning_rate": 2.492640038606476e-06, + "loss": 0.75783807, + "num_input_tokens_seen": 78525150, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.17712402, + "step": 3644, + "time_per_iteration": 3.1136326789855957 + }, + { + "auxiliary_loss_clip": 0.01147427, + "auxiliary_loss_mlp": 0.01042405, + "balance_loss_clip": 1.04618108, + "balance_loss_mlp": 1.02215171, + "epoch": 0.4382853363794866, + "flos": 14684448533760.0, + "grad_norm": 1.8057615647298928, + "language_loss": 0.78589684, + "learning_rate": 2.491885030856608e-06, + "loss": 0.80779517, + "num_input_tokens_seen": 78543245, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.20239258, + "step": 3645, + "time_per_iteration": 2.9952380657196045 + }, + { + "auxiliary_loss_clip": 0.01145371, + "auxiliary_loss_mlp": 0.01046586, + "balance_loss_clip": 1.04634881, + "balance_loss_mlp": 1.02807319, + "epoch": 0.43840557927012563, + "flos": 17165301177600.0, + "grad_norm": 2.212242144928504, + "language_loss": 0.82611239, + "learning_rate": 2.4911299484881713e-06, + "loss": 0.84803194, + "num_input_tokens_seen": 78560775, + "router_z_loss_clip": 0.99023438, + "router_z_loss_mlp": 0.18518066, + "step": 3646, + "time_per_iteration": 3.038118839263916 + }, + { + "auxiliary_loss_clip": 0.01144611, + "auxiliary_loss_mlp": 0.01045704, + "balance_loss_clip": 1.04617381, + "balance_loss_mlp": 1.02801335, + "epoch": 0.43852582216076474, + "flos": 19390685316480.0, + "grad_norm": 1.5475428550370598, + "language_loss": 0.81242669, + "learning_rate": 2.490374791615712e-06, + "loss": 0.83432984, + "num_input_tokens_seen": 78580800, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.17687988, + "step": 3647, + "time_per_iteration": 3.0552945137023926 + }, + { + "auxiliary_loss_clip": 0.01151758, + "auxiliary_loss_mlp": 0.01051286, + "balance_loss_clip": 1.04808831, + "balance_loss_mlp": 1.03251076, + "epoch": 0.43864606505140386, + "flos": 18075133699200.0, + "grad_norm": 3.34669258990518, + "language_loss": 0.7758255, + "learning_rate": 2.4896195603537867e-06, + "loss": 0.79785591, + "num_input_tokens_seen": 78595410, + "router_z_loss_clip": 1.03613281, + "router_z_loss_mlp": 0.18774414, + "step": 3648, + "time_per_iteration": 2.9613211154937744 + }, + { + "auxiliary_loss_clip": 0.01148019, + "auxiliary_loss_mlp": 0.01047182, + "balance_loss_clip": 1.04863167, + "balance_loss_mlp": 1.02975345, + "epoch": 0.4387663079420429, + "flos": 19644896845440.0, + "grad_norm": 1.9887012160788387, + "language_loss": 0.74016821, + "learning_rate": 2.488864254816964e-06, + "loss": 0.76212025, + "num_input_tokens_seen": 78614100, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.17431641, + "step": 3649, + "time_per_iteration": 3.02598237991333 + }, + { + "auxiliary_loss_clip": 0.01150472, + "auxiliary_loss_mlp": 0.01041609, + "balance_loss_clip": 1.04739904, + "balance_loss_mlp": 1.02259517, + "epoch": 0.438886550832682, + "flos": 19719339782400.0, + "grad_norm": 6.535894112466148, + "language_loss": 0.68399805, + "learning_rate": 2.4881088751198218e-06, + "loss": 0.70591885, + "num_input_tokens_seen": 78632260, + "router_z_loss_clip": 1.03076172, + "router_z_loss_mlp": 0.19018555, + "step": 3650, + "time_per_iteration": 3.0893073081970215 + }, + { + "auxiliary_loss_clip": 0.01150392, + "auxiliary_loss_mlp": 0.01053739, + "balance_loss_clip": 1.04660785, + "balance_loss_mlp": 1.03516626, + "epoch": 0.43900679372332113, + "flos": 14536675981440.0, + "grad_norm": 2.55389997328439, + "language_loss": 0.64478266, + "learning_rate": 2.4873534213769517e-06, + "loss": 0.66682398, + "num_input_tokens_seen": 78647490, + "router_z_loss_clip": 1.03808594, + "router_z_loss_mlp": 0.18579102, + "step": 3651, + "time_per_iteration": 2.9332385063171387 + }, + { + "auxiliary_loss_clip": 0.01144787, + "auxiliary_loss_mlp": 0.01044924, + "balance_loss_clip": 1.04704332, + "balance_loss_mlp": 1.02778196, + "epoch": 0.4391270366139602, + "flos": 24056234968320.0, + "grad_norm": 1.7490759961528541, + "language_loss": 0.71937531, + "learning_rate": 2.4865978937029547e-06, + "loss": 0.74127245, + "num_input_tokens_seen": 78666470, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.17126465, + "step": 3652, + "time_per_iteration": 3.716845750808716 + }, + { + "auxiliary_loss_clip": 0.01151644, + "auxiliary_loss_mlp": 0.01043702, + "balance_loss_clip": 1.05186522, + "balance_loss_mlp": 1.0253557, + "epoch": 0.4392472795045993, + "flos": 31538510363520.0, + "grad_norm": 1.5849861526068165, + "language_loss": 0.66231132, + "learning_rate": 2.485842292212445e-06, + "loss": 0.68426478, + "num_input_tokens_seen": 78687685, + "router_z_loss_clip": 0.99755859, + "router_z_loss_mlp": 0.18347168, + "step": 3653, + "time_per_iteration": 3.0762851238250732 + }, + { + "auxiliary_loss_clip": 0.01153009, + "auxiliary_loss_mlp": 0.01053837, + "balance_loss_clip": 1.04975581, + "balance_loss_mlp": 1.03527653, + "epoch": 0.4393675223952384, + "flos": 14866300114560.0, + "grad_norm": 2.77335352494325, + "language_loss": 0.80510539, + "learning_rate": 2.485086617020045e-06, + "loss": 0.82717383, + "num_input_tokens_seen": 78706180, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.18554688, + "step": 3654, + "time_per_iteration": 3.062852382659912 + }, + { + "auxiliary_loss_clip": 0.0115171, + "auxiliary_loss_mlp": 0.0104687, + "balance_loss_clip": 1.04756784, + "balance_loss_mlp": 1.02582943, + "epoch": 0.43948776528587746, + "flos": 14825900292480.0, + "grad_norm": 2.044440258273211, + "language_loss": 0.81818533, + "learning_rate": 2.4843308682403903e-06, + "loss": 0.84017122, + "num_input_tokens_seen": 78723095, + "router_z_loss_clip": 1.04150391, + "router_z_loss_mlp": 0.21032715, + "step": 3655, + "time_per_iteration": 4.218359470367432 + }, + { + "auxiliary_loss_clip": 0.01148001, + "auxiliary_loss_mlp": 0.01042337, + "balance_loss_clip": 1.04751539, + "balance_loss_mlp": 1.02500367, + "epoch": 0.4396080081765166, + "flos": 13914523486080.0, + "grad_norm": 1.7912934235765017, + "language_loss": 0.83248532, + "learning_rate": 2.4835750459881294e-06, + "loss": 0.85438871, + "num_input_tokens_seen": 78739720, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.17333984, + "step": 3656, + "time_per_iteration": 2.943389654159546 + }, + { + "auxiliary_loss_clip": 0.01148595, + "auxiliary_loss_mlp": 0.01046732, + "balance_loss_clip": 1.04777038, + "balance_loss_mlp": 1.02983999, + "epoch": 0.43972825106715563, + "flos": 18222978078720.0, + "grad_norm": 1.7206848612249355, + "language_loss": 0.81830508, + "learning_rate": 2.4828191503779177e-06, + "loss": 0.84025836, + "num_input_tokens_seen": 78757820, + "router_z_loss_clip": 1.0078125, + "router_z_loss_mlp": 0.16882324, + "step": 3657, + "time_per_iteration": 3.7403407096862793 + }, + { + "auxiliary_loss_clip": 0.01150578, + "auxiliary_loss_mlp": 0.01042957, + "balance_loss_clip": 1.04917455, + "balance_loss_mlp": 1.02484941, + "epoch": 0.43984849395779474, + "flos": 16873239692160.0, + "grad_norm": 2.5457185459206966, + "language_loss": 0.89612651, + "learning_rate": 2.482063181524425e-06, + "loss": 0.91806197, + "num_input_tokens_seen": 78773720, + "router_z_loss_clip": 1.01416016, + "router_z_loss_mlp": 0.18115234, + "step": 3658, + "time_per_iteration": 2.934845447540283 + }, + { + "auxiliary_loss_clip": 0.01150712, + "auxiliary_loss_mlp": 0.01050296, + "balance_loss_clip": 1.04908955, + "balance_loss_mlp": 1.03200912, + "epoch": 0.43996873684843385, + "flos": 18691504104960.0, + "grad_norm": 2.5079891814807507, + "language_loss": 0.81830657, + "learning_rate": 2.4813071395423307e-06, + "loss": 0.84031665, + "num_input_tokens_seen": 78791285, + "router_z_loss_clip": 1.01513672, + "router_z_loss_mlp": 0.18286133, + "step": 3659, + "time_per_iteration": 2.9666638374328613 + }, + { + "auxiliary_loss_clip": 0.0114992, + "auxiliary_loss_mlp": 0.01047227, + "balance_loss_clip": 1.04845285, + "balance_loss_mlp": 1.02858293, + "epoch": 0.4400889797390729, + "flos": 23653460787840.0, + "grad_norm": 1.7182665614117383, + "language_loss": 0.64710855, + "learning_rate": 2.4805510245463263e-06, + "loss": 0.66908002, + "num_input_tokens_seen": 78811440, + "router_z_loss_clip": 1.01464844, + "router_z_loss_mlp": 0.18652344, + "step": 3660, + "time_per_iteration": 3.714660167694092 + }, + { + "auxiliary_loss_clip": 0.01148165, + "auxiliary_loss_mlp": 0.01042012, + "balance_loss_clip": 1.04877329, + "balance_loss_mlp": 1.02465558, + "epoch": 0.440209222629712, + "flos": 23149203707520.0, + "grad_norm": 1.9635026774384876, + "language_loss": 0.60360193, + "learning_rate": 2.4797948366511137e-06, + "loss": 0.62550366, + "num_input_tokens_seen": 78831150, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.17358398, + "step": 3661, + "time_per_iteration": 2.949721574783325 + }, + { + "auxiliary_loss_clip": 0.01149474, + "auxiliary_loss_mlp": 0.01043278, + "balance_loss_clip": 1.04803348, + "balance_loss_mlp": 1.02446699, + "epoch": 0.4403294655203511, + "flos": 24823394668800.0, + "grad_norm": 2.01813602330231, + "language_loss": 0.76407599, + "learning_rate": 2.4790385759714055e-06, + "loss": 0.78600347, + "num_input_tokens_seen": 78850215, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.18823242, + "step": 3662, + "time_per_iteration": 2.845949172973633 + }, + { + "auxiliary_loss_clip": 0.01147081, + "auxiliary_loss_mlp": 0.01043327, + "balance_loss_clip": 1.04717875, + "balance_loss_mlp": 1.0252434, + "epoch": 0.4404497084109902, + "flos": 22565080736640.0, + "grad_norm": 1.8048057607033487, + "language_loss": 0.7116679, + "learning_rate": 2.478282242621926e-06, + "loss": 0.73357201, + "num_input_tokens_seen": 78870675, + "router_z_loss_clip": 0.99853516, + "router_z_loss_mlp": 0.1809082, + "step": 3663, + "time_per_iteration": 3.0725181102752686 + }, + { + "auxiliary_loss_clip": 0.01047258, + "auxiliary_loss_mlp": 0.01001133, + "balance_loss_clip": 1.01468396, + "balance_loss_mlp": 0.99921334, + "epoch": 0.4405699513016293, + "flos": 64967073448320.0, + "grad_norm": 0.8377882076890881, + "language_loss": 0.5963065, + "learning_rate": 2.477525836717411e-06, + "loss": 0.61679047, + "num_input_tokens_seen": 78938440, + "router_z_loss_clip": 0.32617188, + "router_z_loss_mlp": 0.01916504, + "step": 3664, + "time_per_iteration": 3.4371249675750732 + }, + { + "auxiliary_loss_clip": 0.01150183, + "auxiliary_loss_mlp": 0.01052185, + "balance_loss_clip": 1.04751337, + "balance_loss_mlp": 1.03352845, + "epoch": 0.4406901941922684, + "flos": 35661952978560.0, + "grad_norm": 2.3622357143716273, + "language_loss": 0.79603946, + "learning_rate": 2.476769358372606e-06, + "loss": 0.81806314, + "num_input_tokens_seen": 78960090, + "router_z_loss_clip": 1.02685547, + "router_z_loss_mlp": 0.18664551, + "step": 3665, + "time_per_iteration": 2.893372058868408 + }, + { + "auxiliary_loss_clip": 0.01148708, + "auxiliary_loss_mlp": 0.01051302, + "balance_loss_clip": 1.05124831, + "balance_loss_mlp": 1.03395724, + "epoch": 0.44081043708290746, + "flos": 18040767361920.0, + "grad_norm": 2.0749700808158864, + "language_loss": 0.74230552, + "learning_rate": 2.4760128077022683e-06, + "loss": 0.76430559, + "num_input_tokens_seen": 78978225, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.17358398, + "step": 3666, + "time_per_iteration": 2.7991199493408203 + }, + { + "auxiliary_loss_clip": 0.01147092, + "auxiliary_loss_mlp": 0.01047645, + "balance_loss_clip": 1.05065262, + "balance_loss_mlp": 1.03026462, + "epoch": 0.44093067997354657, + "flos": 30153507799680.0, + "grad_norm": 1.8717163052837509, + "language_loss": 0.68723333, + "learning_rate": 2.4752561848211672e-06, + "loss": 0.70918071, + "num_input_tokens_seen": 79000625, + "router_z_loss_clip": 0.96533203, + "router_z_loss_mlp": 0.17382812, + "step": 3667, + "time_per_iteration": 3.0474181175231934 + }, + { + "auxiliary_loss_clip": 0.01144168, + "auxiliary_loss_mlp": 0.01045255, + "balance_loss_clip": 1.04882658, + "balance_loss_mlp": 1.02776742, + "epoch": 0.4410509228641857, + "flos": 23255068066560.0, + "grad_norm": 1.833729227497649, + "language_loss": 0.71377778, + "learning_rate": 2.4744994898440797e-06, + "loss": 0.735672, + "num_input_tokens_seen": 79019415, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.17492676, + "step": 3668, + "time_per_iteration": 3.0602283477783203 + }, + { + "auxiliary_loss_clip": 0.0114857, + "auxiliary_loss_mlp": 0.01045444, + "balance_loss_clip": 1.04742777, + "balance_loss_mlp": 1.02810478, + "epoch": 0.44117116575482473, + "flos": 19500571998720.0, + "grad_norm": 1.9462254574424502, + "language_loss": 0.84162629, + "learning_rate": 2.473742722885797e-06, + "loss": 0.86356646, + "num_input_tokens_seen": 79038435, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.17340088, + "step": 3669, + "time_per_iteration": 3.0611300468444824 + }, + { + "auxiliary_loss_clip": 0.01152622, + "auxiliary_loss_mlp": 0.01045777, + "balance_loss_clip": 1.0518229, + "balance_loss_mlp": 1.02729905, + "epoch": 0.44129140864546385, + "flos": 27053124353280.0, + "grad_norm": 3.0258495405497428, + "language_loss": 0.65546095, + "learning_rate": 2.4729858840611197e-06, + "loss": 0.67744493, + "num_input_tokens_seen": 79057345, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.18481445, + "step": 3670, + "time_per_iteration": 3.0677008628845215 + }, + { + "auxiliary_loss_clip": 0.01145664, + "auxiliary_loss_mlp": 0.01050273, + "balance_loss_clip": 1.0487361, + "balance_loss_mlp": 1.03242719, + "epoch": 0.4414116515361029, + "flos": 26102101910400.0, + "grad_norm": 2.0057347772631395, + "language_loss": 0.72779948, + "learning_rate": 2.4722289734848605e-06, + "loss": 0.7497589, + "num_input_tokens_seen": 79077810, + "router_z_loss_clip": 0.96777344, + "router_z_loss_mlp": 0.17858887, + "step": 3671, + "time_per_iteration": 3.054137945175171 + }, + { + "auxiliary_loss_clip": 0.01147714, + "auxiliary_loss_mlp": 0.01049282, + "balance_loss_clip": 1.0504837, + "balance_loss_mlp": 1.0312103, + "epoch": 0.441531894426742, + "flos": 21906083865600.0, + "grad_norm": 2.4869559881871535, + "language_loss": 0.78090721, + "learning_rate": 2.471471991271841e-06, + "loss": 0.80287719, + "num_input_tokens_seen": 79094935, + "router_z_loss_clip": 0.97265625, + "router_z_loss_mlp": 0.18078613, + "step": 3672, + "time_per_iteration": 3.087783098220825 + }, + { + "auxiliary_loss_clip": 0.01147221, + "auxiliary_loss_mlp": 0.01049284, + "balance_loss_clip": 1.04784441, + "balance_loss_mlp": 1.03150988, + "epoch": 0.4416521373173811, + "flos": 23437099215360.0, + "grad_norm": 3.192726474354295, + "language_loss": 0.79333925, + "learning_rate": 2.470714937536896e-06, + "loss": 0.81530428, + "num_input_tokens_seen": 79113660, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.1776123, + "step": 3673, + "time_per_iteration": 3.0163395404815674 + }, + { + "auxiliary_loss_clip": 0.0115129, + "auxiliary_loss_mlp": 0.01046971, + "balance_loss_clip": 1.0500381, + "balance_loss_mlp": 1.02887487, + "epoch": 0.4417723802080202, + "flos": 20334345471360.0, + "grad_norm": 1.8209729469000238, + "language_loss": 0.70367277, + "learning_rate": 2.469957812394868e-06, + "loss": 0.72565544, + "num_input_tokens_seen": 79132470, + "router_z_loss_clip": 1.01416016, + "router_z_loss_mlp": 0.1809082, + "step": 3674, + "time_per_iteration": 2.8060057163238525 + }, + { + "auxiliary_loss_clip": 0.0115148, + "auxiliary_loss_mlp": 0.01043127, + "balance_loss_clip": 1.0522368, + "balance_loss_mlp": 1.02557921, + "epoch": 0.4418926230986593, + "flos": 18880682060160.0, + "grad_norm": 1.8594190964606743, + "language_loss": 0.76026887, + "learning_rate": 2.4692006159606148e-06, + "loss": 0.78221488, + "num_input_tokens_seen": 79150000, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.17553711, + "step": 3675, + "time_per_iteration": 2.8749048709869385 + }, + { + "auxiliary_loss_clip": 0.01149566, + "auxiliary_loss_mlp": 0.01050099, + "balance_loss_clip": 1.05055106, + "balance_loss_mlp": 1.03139496, + "epoch": 0.4420128659892984, + "flos": 19464409981440.0, + "grad_norm": 1.7059567800027002, + "language_loss": 0.78956735, + "learning_rate": 2.468443348349e-06, + "loss": 0.81156397, + "num_input_tokens_seen": 79167875, + "router_z_loss_clip": 0.99121094, + "router_z_loss_mlp": 0.18713379, + "step": 3676, + "time_per_iteration": 2.8998188972473145 + }, + { + "auxiliary_loss_clip": 0.01151485, + "auxiliary_loss_mlp": 0.01042772, + "balance_loss_clip": 1.04981387, + "balance_loss_mlp": 1.02433038, + "epoch": 0.44213310887993745, + "flos": 17894359526400.0, + "grad_norm": 3.430275560297835, + "language_loss": 0.82270169, + "learning_rate": 2.467686009674902e-06, + "loss": 0.84464431, + "num_input_tokens_seen": 79182325, + "router_z_loss_clip": 1.01806641, + "router_z_loss_mlp": 0.18444824, + "step": 3677, + "time_per_iteration": 2.851060628890991 + }, + { + "auxiliary_loss_clip": 0.01151621, + "auxiliary_loss_mlp": 0.0105338, + "balance_loss_clip": 1.05008221, + "balance_loss_mlp": 1.03521264, + "epoch": 0.44225335177057656, + "flos": 19204667758080.0, + "grad_norm": 2.0857986759396345, + "language_loss": 0.85418808, + "learning_rate": 2.466928600053209e-06, + "loss": 0.87623811, + "num_input_tokens_seen": 79197630, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.18164062, + "step": 3678, + "time_per_iteration": 2.7915127277374268 + }, + { + "auxiliary_loss_clip": 0.01146845, + "auxiliary_loss_mlp": 0.01048511, + "balance_loss_clip": 1.04721224, + "balance_loss_mlp": 1.03054619, + "epoch": 0.4423735946612157, + "flos": 23471321898240.0, + "grad_norm": 1.8096907938856022, + "language_loss": 0.71138507, + "learning_rate": 2.466171119598818e-06, + "loss": 0.73333865, + "num_input_tokens_seen": 79217600, + "router_z_loss_clip": 0.99511719, + "router_z_loss_mlp": 0.1796875, + "step": 3679, + "time_per_iteration": 2.8447883129119873 + }, + { + "auxiliary_loss_clip": 0.01154446, + "auxiliary_loss_mlp": 0.01057238, + "balance_loss_clip": 1.0490135, + "balance_loss_mlp": 1.03702033, + "epoch": 0.44249383755185473, + "flos": 26685398868480.0, + "grad_norm": 2.1544477486281455, + "language_loss": 0.77509856, + "learning_rate": 2.465413568426639e-06, + "loss": 0.7972154, + "num_input_tokens_seen": 79238550, + "router_z_loss_clip": 1.05419922, + "router_z_loss_mlp": 0.20227051, + "step": 3680, + "time_per_iteration": 2.8221280574798584 + }, + { + "auxiliary_loss_clip": 0.01144425, + "auxiliary_loss_mlp": 0.01051612, + "balance_loss_clip": 1.04657245, + "balance_loss_mlp": 1.03429091, + "epoch": 0.44261408044249384, + "flos": 23147659422720.0, + "grad_norm": 1.8554548173047893, + "language_loss": 0.81291962, + "learning_rate": 2.464655946651591e-06, + "loss": 0.83487999, + "num_input_tokens_seen": 79257555, + "router_z_loss_clip": 0.97851562, + "router_z_loss_mlp": 0.1730957, + "step": 3681, + "time_per_iteration": 3.7704484462738037 + }, + { + "auxiliary_loss_clip": 0.01153127, + "auxiliary_loss_mlp": 0.0104494, + "balance_loss_clip": 1.05162787, + "balance_loss_mlp": 1.02630782, + "epoch": 0.44273432333313295, + "flos": 24462564595200.0, + "grad_norm": 1.8729780074641338, + "language_loss": 0.8088479, + "learning_rate": 2.4638982543886065e-06, + "loss": 0.83082855, + "num_input_tokens_seen": 79277595, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.1862793, + "step": 3682, + "time_per_iteration": 2.833420991897583 + }, + { + "auxiliary_loss_clip": 0.01152452, + "auxiliary_loss_mlp": 0.01050708, + "balance_loss_clip": 1.05011225, + "balance_loss_mlp": 1.03256476, + "epoch": 0.442854566223772, + "flos": 17528932512000.0, + "grad_norm": 2.6852268091382876, + "language_loss": 0.87445962, + "learning_rate": 2.4631404917526254e-06, + "loss": 0.89649117, + "num_input_tokens_seen": 79294550, + "router_z_loss_clip": 1.0234375, + "router_z_loss_mlp": 0.18151855, + "step": 3683, + "time_per_iteration": 3.8017961978912354 + }, + { + "auxiliary_loss_clip": 0.01145945, + "auxiliary_loss_mlp": 0.01050482, + "balance_loss_clip": 1.04819059, + "balance_loss_mlp": 1.03341174, + "epoch": 0.4429748091144111, + "flos": 24896293320960.0, + "grad_norm": 1.566322345869812, + "language_loss": 0.79006916, + "learning_rate": 2.4623826588586e-06, + "loss": 0.81203341, + "num_input_tokens_seen": 79314820, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.1706543, + "step": 3684, + "time_per_iteration": 2.9200799465179443 + }, + { + "auxiliary_loss_clip": 0.0114851, + "auxiliary_loss_mlp": 0.0104678, + "balance_loss_clip": 1.04731512, + "balance_loss_mlp": 1.02928019, + "epoch": 0.4430950520050502, + "flos": 21614704738560.0, + "grad_norm": 1.4979889240497308, + "language_loss": 0.82572365, + "learning_rate": 2.461624755821492e-06, + "loss": 0.84767658, + "num_input_tokens_seen": 79334300, + "router_z_loss_clip": 1.01171875, + "router_z_loss_mlp": 0.17504883, + "step": 3685, + "time_per_iteration": 2.7927091121673584 + }, + { + "auxiliary_loss_clip": 0.01146221, + "auxiliary_loss_mlp": 0.01042649, + "balance_loss_clip": 1.04656827, + "balance_loss_mlp": 1.02489841, + "epoch": 0.4432152948956893, + "flos": 24572271709440.0, + "grad_norm": 2.3276726323236416, + "language_loss": 0.76715982, + "learning_rate": 2.4608667827562763e-06, + "loss": 0.78904855, + "num_input_tokens_seen": 79353630, + "router_z_loss_clip": 0.99755859, + "router_z_loss_mlp": 0.1774292, + "step": 3686, + "time_per_iteration": 3.7302865982055664 + }, + { + "auxiliary_loss_clip": 0.0115138, + "auxiliary_loss_mlp": 0.0105053, + "balance_loss_clip": 1.04815137, + "balance_loss_mlp": 1.03174293, + "epoch": 0.4433355377863284, + "flos": 21762261809280.0, + "grad_norm": 1.9082715074157157, + "language_loss": 0.89778543, + "learning_rate": 2.460108739777936e-06, + "loss": 0.91980451, + "num_input_tokens_seen": 79372765, + "router_z_loss_clip": 1.03271484, + "router_z_loss_mlp": 0.18786621, + "step": 3687, + "time_per_iteration": 2.7911887168884277 + }, + { + "auxiliary_loss_clip": 0.01149158, + "auxiliary_loss_mlp": 0.01049901, + "balance_loss_clip": 1.05027723, + "balance_loss_mlp": 1.03281879, + "epoch": 0.44345578067696745, + "flos": 20084479488000.0, + "grad_norm": 1.6125553177746987, + "language_loss": 0.76661658, + "learning_rate": 2.4593506270014656e-06, + "loss": 0.78860718, + "num_input_tokens_seen": 79391735, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.17089844, + "step": 3688, + "time_per_iteration": 2.9321675300598145 + }, + { + "auxiliary_loss_clip": 0.01147342, + "auxiliary_loss_mlp": 0.01043905, + "balance_loss_clip": 1.04603219, + "balance_loss_mlp": 1.02619076, + "epoch": 0.44357602356760656, + "flos": 24169497528960.0, + "grad_norm": 1.600817406601671, + "language_loss": 0.82248271, + "learning_rate": 2.45859244454187e-06, + "loss": 0.84439516, + "num_input_tokens_seen": 79411525, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.17712402, + "step": 3689, + "time_per_iteration": 3.83186411857605 + }, + { + "auxiliary_loss_clip": 0.01147061, + "auxiliary_loss_mlp": 0.01037048, + "balance_loss_clip": 1.04784083, + "balance_loss_mlp": 1.02003694, + "epoch": 0.44369626645824567, + "flos": 22707717644160.0, + "grad_norm": 1.5579369965917282, + "language_loss": 0.65996492, + "learning_rate": 2.4578341925141655e-06, + "loss": 0.68180597, + "num_input_tokens_seen": 79430740, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.17004395, + "step": 3690, + "time_per_iteration": 2.989271402359009 + }, + { + "auxiliary_loss_clip": 0.01151346, + "auxiliary_loss_mlp": 0.01049707, + "balance_loss_clip": 1.04877639, + "balance_loss_mlp": 1.03040695, + "epoch": 0.4438165093488847, + "flos": 38030225420160.0, + "grad_norm": 2.086966952701807, + "language_loss": 0.72807974, + "learning_rate": 2.457075871033378e-06, + "loss": 0.75009018, + "num_input_tokens_seen": 79452615, + "router_z_loss_clip": 1.02587891, + "router_z_loss_mlp": 0.19311523, + "step": 3691, + "time_per_iteration": 2.853440046310425 + }, + { + "auxiliary_loss_clip": 0.01143511, + "auxiliary_loss_mlp": 0.01041814, + "balance_loss_clip": 1.04668021, + "balance_loss_mlp": 1.02297914, + "epoch": 0.44393675223952384, + "flos": 15523213996800.0, + "grad_norm": 2.411973138112001, + "language_loss": 0.88748157, + "learning_rate": 2.4563174802145445e-06, + "loss": 0.90933478, + "num_input_tokens_seen": 79469865, + "router_z_loss_clip": 0.96826172, + "router_z_loss_mlp": 0.18859863, + "step": 3692, + "time_per_iteration": 2.8320939540863037 + }, + { + "auxiliary_loss_clip": 0.01045104, + "auxiliary_loss_mlp": 0.01005616, + "balance_loss_clip": 1.01270962, + "balance_loss_mlp": 1.00373268, + "epoch": 0.44405699513016295, + "flos": 64574893779840.0, + "grad_norm": 0.6355431917052805, + "language_loss": 0.48550251, + "learning_rate": 2.455559020172712e-06, + "loss": 0.5060097, + "num_input_tokens_seen": 79537220, + "router_z_loss_clip": 0.32421875, + "router_z_loss_mlp": 0.01879883, + "step": 3693, + "time_per_iteration": 3.4196550846099854 + }, + { + "auxiliary_loss_clip": 0.01148522, + "auxiliary_loss_mlp": 0.0104273, + "balance_loss_clip": 1.05011761, + "balance_loss_mlp": 1.02269089, + "epoch": 0.444177238020802, + "flos": 23987394552960.0, + "grad_norm": 1.7451813714336537, + "language_loss": 0.89836609, + "learning_rate": 2.4548004910229385e-06, + "loss": 0.92027861, + "num_input_tokens_seen": 79554795, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.20043945, + "step": 3694, + "time_per_iteration": 2.7821576595306396 + }, + { + "auxiliary_loss_clip": 0.0114716, + "auxiliary_loss_mlp": 0.01048123, + "balance_loss_clip": 1.04708588, + "balance_loss_mlp": 1.03049231, + "epoch": 0.4442974809114411, + "flos": 22563069575040.0, + "grad_norm": 1.7085478020607205, + "language_loss": 0.87070978, + "learning_rate": 2.4540418928802913e-06, + "loss": 0.89266258, + "num_input_tokens_seen": 79573530, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.17626953, + "step": 3695, + "time_per_iteration": 2.7216901779174805 + }, + { + "auxiliary_loss_clip": 0.01152878, + "auxiliary_loss_mlp": 0.01050836, + "balance_loss_clip": 1.05001926, + "balance_loss_mlp": 1.03201318, + "epoch": 0.4444177238020802, + "flos": 17675699483520.0, + "grad_norm": 2.0823076862896936, + "language_loss": 0.66146177, + "learning_rate": 2.4532832258598506e-06, + "loss": 0.68349892, + "num_input_tokens_seen": 79591360, + "router_z_loss_clip": 1.02734375, + "router_z_loss_mlp": 0.18823242, + "step": 3696, + "time_per_iteration": 3.2069251537323 + }, + { + "auxiliary_loss_clip": 0.01144517, + "auxiliary_loss_mlp": 0.01039455, + "balance_loss_clip": 1.04772782, + "balance_loss_mlp": 1.02083492, + "epoch": 0.4445379666927193, + "flos": 28621594609920.0, + "grad_norm": 1.8237670617521107, + "language_loss": 0.80987585, + "learning_rate": 2.4525244900767047e-06, + "loss": 0.83171558, + "num_input_tokens_seen": 79612175, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.1862793, + "step": 3697, + "time_per_iteration": 3.1095008850097656 + }, + { + "auxiliary_loss_clip": 0.01045629, + "auxiliary_loss_mlp": 0.01003271, + "balance_loss_clip": 1.01339602, + "balance_loss_mlp": 1.00123239, + "epoch": 0.4446582095833584, + "flos": 70487370115200.0, + "grad_norm": 0.7768827855744848, + "language_loss": 0.60489368, + "learning_rate": 2.4517656856459536e-06, + "loss": 0.62538266, + "num_input_tokens_seen": 79678020, + "router_z_loss_clip": 0.32226562, + "router_z_loss_mlp": 0.02038574, + "step": 3698, + "time_per_iteration": 3.3807785511016846 + }, + { + "auxiliary_loss_clip": 0.01149099, + "auxiliary_loss_mlp": 0.01043198, + "balance_loss_clip": 1.04874206, + "balance_loss_mlp": 1.02374339, + "epoch": 0.4447784524739975, + "flos": 26505199313280.0, + "grad_norm": 1.6696939844536556, + "language_loss": 0.68357456, + "learning_rate": 2.4510068126827073e-06, + "loss": 0.7054975, + "num_input_tokens_seen": 79699020, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.19458008, + "step": 3699, + "time_per_iteration": 2.7971930503845215 + }, + { + "auxiliary_loss_clip": 0.01148555, + "auxiliary_loss_mlp": 0.01047478, + "balance_loss_clip": 1.04950607, + "balance_loss_mlp": 1.02940607, + "epoch": 0.44489869536463655, + "flos": 11656209553920.0, + "grad_norm": 2.288800952937147, + "language_loss": 0.82225668, + "learning_rate": 2.450247871302086e-06, + "loss": 0.844217, + "num_input_tokens_seen": 79716795, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.18054199, + "step": 3700, + "time_per_iteration": 2.757943630218506 + }, + { + "auxiliary_loss_clip": 0.01147534, + "auxiliary_loss_mlp": 0.01046386, + "balance_loss_clip": 1.04724836, + "balance_loss_mlp": 1.02991092, + "epoch": 0.44501893825527566, + "flos": 20448469958400.0, + "grad_norm": 2.9092387407749123, + "language_loss": 0.83455402, + "learning_rate": 2.44948886161922e-06, + "loss": 0.85649323, + "num_input_tokens_seen": 79735810, + "router_z_loss_clip": 1.00341797, + "router_z_loss_mlp": 0.16467285, + "step": 3701, + "time_per_iteration": 2.8051891326904297 + }, + { + "auxiliary_loss_clip": 0.01147486, + "auxiliary_loss_mlp": 0.0104335, + "balance_loss_clip": 1.04867852, + "balance_loss_mlp": 1.02598071, + "epoch": 0.4451391811459148, + "flos": 18261079430400.0, + "grad_norm": 1.5800032338089363, + "language_loss": 0.84785515, + "learning_rate": 2.4487297837492524e-06, + "loss": 0.86976349, + "num_input_tokens_seen": 79754975, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.17370605, + "step": 3702, + "time_per_iteration": 2.7504217624664307 + }, + { + "auxiliary_loss_clip": 0.01152385, + "auxiliary_loss_mlp": 0.01043111, + "balance_loss_clip": 1.04957068, + "balance_loss_mlp": 1.02376366, + "epoch": 0.44525942403655383, + "flos": 16910155895040.0, + "grad_norm": 2.032079287809234, + "language_loss": 0.62012017, + "learning_rate": 2.4479706378073323e-06, + "loss": 0.64207512, + "num_input_tokens_seen": 79773515, + "router_z_loss_clip": 1.02880859, + "router_z_loss_mlp": 0.19348145, + "step": 3703, + "time_per_iteration": 2.7150449752807617 + }, + { + "auxiliary_loss_clip": 0.0114366, + "auxiliary_loss_mlp": 0.01043941, + "balance_loss_clip": 1.04711103, + "balance_loss_mlp": 1.02725184, + "epoch": 0.44537966692719294, + "flos": 23258838994560.0, + "grad_norm": 1.596439289034757, + "language_loss": 0.83698815, + "learning_rate": 2.447211423908623e-06, + "loss": 0.85886419, + "num_input_tokens_seen": 79793560, + "router_z_loss_clip": 0.96435547, + "router_z_loss_mlp": 0.16687012, + "step": 3704, + "time_per_iteration": 2.978435754776001 + }, + { + "auxiliary_loss_clip": 0.01147081, + "auxiliary_loss_mlp": 0.01045874, + "balance_loss_clip": 1.04797661, + "balance_loss_mlp": 1.02727723, + "epoch": 0.445499909817832, + "flos": 21724160457600.0, + "grad_norm": 2.01557392402029, + "language_loss": 0.75169003, + "learning_rate": 2.4464521421682966e-06, + "loss": 0.77361959, + "num_input_tokens_seen": 79811150, + "router_z_loss_clip": 0.99023438, + "router_z_loss_mlp": 0.18591309, + "step": 3705, + "time_per_iteration": 2.7224581241607666 + }, + { + "auxiliary_loss_clip": 0.01144046, + "auxiliary_loss_mlp": 0.01035948, + "balance_loss_clip": 1.04768324, + "balance_loss_mlp": 1.01819801, + "epoch": 0.4456201527084711, + "flos": 23987969170560.0, + "grad_norm": 1.3708480583652685, + "language_loss": 0.87938786, + "learning_rate": 2.4456927927015345e-06, + "loss": 0.90118778, + "num_input_tokens_seen": 79832190, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.17736816, + "step": 3706, + "time_per_iteration": 2.7693357467651367 + }, + { + "auxiliary_loss_clip": 0.01152956, + "auxiliary_loss_mlp": 0.01049686, + "balance_loss_clip": 1.04984033, + "balance_loss_mlp": 1.02942038, + "epoch": 0.4457403955991102, + "flos": 18807065136000.0, + "grad_norm": 2.1073715266124853, + "language_loss": 0.76544452, + "learning_rate": 2.4449333756235307e-06, + "loss": 0.78747094, + "num_input_tokens_seen": 79848905, + "router_z_loss_clip": 1.03027344, + "router_z_loss_mlp": 0.20251465, + "step": 3707, + "time_per_iteration": 2.7152535915374756 + }, + { + "auxiliary_loss_clip": 0.01146265, + "auxiliary_loss_mlp": 0.01052522, + "balance_loss_clip": 1.04838467, + "balance_loss_mlp": 1.03431869, + "epoch": 0.4458606384897493, + "flos": 19207756327680.0, + "grad_norm": 2.1296528835571458, + "language_loss": 0.79007256, + "learning_rate": 2.4441738910494876e-06, + "loss": 0.81206036, + "num_input_tokens_seen": 79863640, + "router_z_loss_clip": 0.97949219, + "router_z_loss_mlp": 0.18200684, + "step": 3708, + "time_per_iteration": 2.7209596633911133 + }, + { + "auxiliary_loss_clip": 0.0114984, + "auxiliary_loss_mlp": 0.01041455, + "balance_loss_clip": 1.04900455, + "balance_loss_mlp": 1.02322769, + "epoch": 0.4459808813803884, + "flos": 21361283308800.0, + "grad_norm": 1.7611104162045732, + "language_loss": 0.82156831, + "learning_rate": 2.4434143390946176e-06, + "loss": 0.8434813, + "num_input_tokens_seen": 79882450, + "router_z_loss_clip": 1.0078125, + "router_z_loss_mlp": 0.18237305, + "step": 3709, + "time_per_iteration": 2.7394418716430664 + }, + { + "auxiliary_loss_clip": 0.01147086, + "auxiliary_loss_mlp": 0.01042569, + "balance_loss_clip": 1.04744089, + "balance_loss_mlp": 1.02442503, + "epoch": 0.4461011242710275, + "flos": 23288967527040.0, + "grad_norm": 2.1197294623167213, + "language_loss": 0.85342789, + "learning_rate": 2.4426547198741457e-06, + "loss": 0.87532449, + "num_input_tokens_seen": 79900655, + "router_z_loss_clip": 0.99658203, + "router_z_loss_mlp": 0.18139648, + "step": 3710, + "time_per_iteration": 3.5905768871307373 + }, + { + "auxiliary_loss_clip": 0.01153467, + "auxiliary_loss_mlp": 0.01050398, + "balance_loss_clip": 1.05133736, + "balance_loss_mlp": 1.03171754, + "epoch": 0.44622136716166655, + "flos": 20193001453440.0, + "grad_norm": 2.38578693202245, + "language_loss": 0.74799502, + "learning_rate": 2.441895033503305e-06, + "loss": 0.77003372, + "num_input_tokens_seen": 79918575, + "router_z_loss_clip": 1.02099609, + "router_z_loss_mlp": 0.18664551, + "step": 3711, + "time_per_iteration": 2.7381303310394287 + }, + { + "auxiliary_loss_clip": 0.01147395, + "auxiliary_loss_mlp": 0.01058018, + "balance_loss_clip": 1.04571462, + "balance_loss_mlp": 1.03708506, + "epoch": 0.44634161005230566, + "flos": 21283033530240.0, + "grad_norm": 1.692156453689229, + "language_loss": 0.82337844, + "learning_rate": 2.4411352800973375e-06, + "loss": 0.84543252, + "num_input_tokens_seen": 79937010, + "router_z_loss_clip": 1.01757812, + "router_z_loss_mlp": 0.20922852, + "step": 3712, + "time_per_iteration": 3.600703716278076 + }, + { + "auxiliary_loss_clip": 0.01148424, + "auxiliary_loss_mlp": 0.01047431, + "balance_loss_clip": 1.04707003, + "balance_loss_mlp": 1.02813077, + "epoch": 0.44646185294294477, + "flos": 22929358515840.0, + "grad_norm": 2.4549683770551356, + "language_loss": 0.75200844, + "learning_rate": 2.4403754597715005e-06, + "loss": 0.77396703, + "num_input_tokens_seen": 79956455, + "router_z_loss_clip": 1.01220703, + "router_z_loss_mlp": 0.19299316, + "step": 3713, + "time_per_iteration": 2.730149269104004 + }, + { + "auxiliary_loss_clip": 0.01147745, + "auxiliary_loss_mlp": 0.01052803, + "balance_loss_clip": 1.04588175, + "balance_loss_mlp": 1.03375363, + "epoch": 0.4465820958335838, + "flos": 22637692080000.0, + "grad_norm": 2.659502941465813, + "language_loss": 0.93114364, + "learning_rate": 2.4396155726410553e-06, + "loss": 0.95314908, + "num_input_tokens_seen": 79975065, + "router_z_loss_clip": 1.01904297, + "router_z_loss_mlp": 0.19055176, + "step": 3714, + "time_per_iteration": 2.813930034637451 + }, + { + "auxiliary_loss_clip": 0.01149843, + "auxiliary_loss_mlp": 0.01045226, + "balance_loss_clip": 1.04728878, + "balance_loss_mlp": 1.02620006, + "epoch": 0.44670233872422294, + "flos": 22672525294080.0, + "grad_norm": 2.5417640894939977, + "language_loss": 0.90835541, + "learning_rate": 2.438855618821278e-06, + "loss": 0.93030608, + "num_input_tokens_seen": 79990865, + "router_z_loss_clip": 1.02441406, + "router_z_loss_mlp": 0.19030762, + "step": 3715, + "time_per_iteration": 3.5006706714630127 + }, + { + "auxiliary_loss_clip": 0.01144571, + "auxiliary_loss_mlp": 0.01044119, + "balance_loss_clip": 1.04544997, + "balance_loss_mlp": 1.0254271, + "epoch": 0.44682258161486205, + "flos": 23582178247680.0, + "grad_norm": 2.050534028587985, + "language_loss": 0.67719156, + "learning_rate": 2.4380955984274517e-06, + "loss": 0.69907844, + "num_input_tokens_seen": 80009520, + "router_z_loss_clip": 0.99072266, + "router_z_loss_mlp": 0.18676758, + "step": 3716, + "time_per_iteration": 2.7608988285064697 + }, + { + "auxiliary_loss_clip": 0.01149079, + "auxiliary_loss_mlp": 0.01051671, + "balance_loss_clip": 1.04716945, + "balance_loss_mlp": 1.03346801, + "epoch": 0.4469428245055011, + "flos": 26501356558080.0, + "grad_norm": 2.0246239535626147, + "language_loss": 0.76404941, + "learning_rate": 2.4373355115748716e-06, + "loss": 0.78605688, + "num_input_tokens_seen": 80030350, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.18212891, + "step": 3717, + "time_per_iteration": 3.5609560012817383 + }, + { + "auxiliary_loss_clip": 0.01147751, + "auxiliary_loss_mlp": 0.01049678, + "balance_loss_clip": 1.04829228, + "balance_loss_mlp": 1.03120041, + "epoch": 0.4470630673961402, + "flos": 21504925797120.0, + "grad_norm": 1.609442395616211, + "language_loss": 0.72028208, + "learning_rate": 2.436575358378842e-06, + "loss": 0.7422564, + "num_input_tokens_seen": 80049840, + "router_z_loss_clip": 0.99511719, + "router_z_loss_mlp": 0.18469238, + "step": 3718, + "time_per_iteration": 2.7184829711914062 + }, + { + "auxiliary_loss_clip": 0.01153894, + "auxiliary_loss_mlp": 0.01045685, + "balance_loss_clip": 1.04951191, + "balance_loss_mlp": 1.02611041, + "epoch": 0.44718331028677927, + "flos": 16173986653440.0, + "grad_norm": 2.6666405899102315, + "language_loss": 0.83194083, + "learning_rate": 2.4358151389546782e-06, + "loss": 0.85393661, + "num_input_tokens_seen": 80066525, + "router_z_loss_clip": 1.04394531, + "router_z_loss_mlp": 0.19592285, + "step": 3719, + "time_per_iteration": 2.7384016513824463 + }, + { + "auxiliary_loss_clip": 0.01149797, + "auxiliary_loss_mlp": 0.01044209, + "balance_loss_clip": 1.0497998, + "balance_loss_mlp": 1.02654171, + "epoch": 0.4473035531774184, + "flos": 19681238430720.0, + "grad_norm": 2.2477342087335845, + "language_loss": 0.75683481, + "learning_rate": 2.4350548534177035e-06, + "loss": 0.7787748, + "num_input_tokens_seen": 80083355, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.17663574, + "step": 3720, + "time_per_iteration": 2.7477877140045166 + }, + { + "auxiliary_loss_clip": 0.01142485, + "auxiliary_loss_mlp": 0.01045104, + "balance_loss_clip": 1.04815292, + "balance_loss_mlp": 1.02794957, + "epoch": 0.4474237960680575, + "flos": 41427590515200.0, + "grad_norm": 1.6615615015502705, + "language_loss": 0.66878724, + "learning_rate": 2.434294501883254e-06, + "loss": 0.69066316, + "num_input_tokens_seen": 80106450, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.17163086, + "step": 3721, + "time_per_iteration": 2.908134937286377 + }, + { + "auxiliary_loss_clip": 0.01144378, + "auxiliary_loss_mlp": 0.01044534, + "balance_loss_clip": 1.0477196, + "balance_loss_mlp": 1.02506709, + "epoch": 0.44754403895869654, + "flos": 22891328991360.0, + "grad_norm": 1.9047513447941609, + "language_loss": 0.6595751, + "learning_rate": 2.433534084466674e-06, + "loss": 0.6814642, + "num_input_tokens_seen": 80125670, + "router_z_loss_clip": 0.96679688, + "router_z_loss_mlp": 0.19470215, + "step": 3722, + "time_per_iteration": 2.7349045276641846 + }, + { + "auxiliary_loss_clip": 0.0114188, + "auxiliary_loss_mlp": 0.010446, + "balance_loss_clip": 1.04601693, + "balance_loss_mlp": 1.02774346, + "epoch": 0.44766428184933565, + "flos": 25630271832960.0, + "grad_norm": 1.4486967088975904, + "language_loss": 0.7078433, + "learning_rate": 2.4327736012833178e-06, + "loss": 0.72970814, + "num_input_tokens_seen": 80147390, + "router_z_loss_clip": 0.95898438, + "router_z_loss_mlp": 0.1685791, + "step": 3723, + "time_per_iteration": 2.816476583480835 + }, + { + "auxiliary_loss_clip": 0.01147018, + "auxiliary_loss_mlp": 0.0104199, + "balance_loss_clip": 1.04579246, + "balance_loss_mlp": 1.02316642, + "epoch": 0.44778452473997477, + "flos": 20448972748800.0, + "grad_norm": 2.0704861737432982, + "language_loss": 0.7652576, + "learning_rate": 2.4320130524485506e-06, + "loss": 0.78714764, + "num_input_tokens_seen": 80166185, + "router_z_loss_clip": 1.01171875, + "router_z_loss_mlp": 0.18835449, + "step": 3724, + "time_per_iteration": 2.724637985229492 + }, + { + "auxiliary_loss_clip": 0.01143532, + "auxiliary_loss_mlp": 0.01044794, + "balance_loss_clip": 1.04769337, + "balance_loss_mlp": 1.02805662, + "epoch": 0.4479047676306138, + "flos": 21975462984960.0, + "grad_norm": 1.542431508404079, + "language_loss": 0.79491258, + "learning_rate": 2.431252438077746e-06, + "loss": 0.81679583, + "num_input_tokens_seen": 80185685, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.16748047, + "step": 3725, + "time_per_iteration": 2.8078691959381104 + }, + { + "auxiliary_loss_clip": 0.01145339, + "auxiliary_loss_mlp": 0.01049288, + "balance_loss_clip": 1.04476357, + "balance_loss_mlp": 1.03083432, + "epoch": 0.44802501052125293, + "flos": 21467219495040.0, + "grad_norm": 2.422150112701802, + "language_loss": 0.77571189, + "learning_rate": 2.4304917582862906e-06, + "loss": 0.79765821, + "num_input_tokens_seen": 80204865, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.18469238, + "step": 3726, + "time_per_iteration": 2.7648565769195557 + }, + { + "auxiliary_loss_clip": 0.01145048, + "auxiliary_loss_mlp": 0.01046663, + "balance_loss_clip": 1.04622889, + "balance_loss_mlp": 1.02838862, + "epoch": 0.44814525341189204, + "flos": 22126970551680.0, + "grad_norm": 2.5927981995443377, + "language_loss": 0.88476586, + "learning_rate": 2.4297310131895774e-06, + "loss": 0.90668297, + "num_input_tokens_seen": 80223410, + "router_z_loss_clip": 0.98779297, + "router_z_loss_mlp": 0.18273926, + "step": 3727, + "time_per_iteration": 2.7237775325775146 + }, + { + "auxiliary_loss_clip": 0.01147946, + "auxiliary_loss_mlp": 0.01039708, + "balance_loss_clip": 1.04642463, + "balance_loss_mlp": 1.02151704, + "epoch": 0.4482654963025311, + "flos": 16653933204480.0, + "grad_norm": 2.0372276894008925, + "language_loss": 0.74454999, + "learning_rate": 2.4289702029030113e-06, + "loss": 0.76642644, + "num_input_tokens_seen": 80240880, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.18200684, + "step": 3728, + "time_per_iteration": 2.7320034503936768 + }, + { + "auxiliary_loss_clip": 0.01151544, + "auxiliary_loss_mlp": 0.0104101, + "balance_loss_clip": 1.05185282, + "balance_loss_mlp": 1.02321196, + "epoch": 0.4483857391931702, + "flos": 18841251905280.0, + "grad_norm": 1.829368588845361, + "language_loss": 0.83379126, + "learning_rate": 2.4282093275420057e-06, + "loss": 0.85571676, + "num_input_tokens_seen": 80259910, + "router_z_loss_clip": 0.99707031, + "router_z_loss_mlp": 0.17797852, + "step": 3729, + "time_per_iteration": 2.724243640899658 + }, + { + "auxiliary_loss_clip": 0.01144241, + "auxiliary_loss_mlp": 0.01050428, + "balance_loss_clip": 1.04747677, + "balance_loss_mlp": 1.03253508, + "epoch": 0.4485059820838093, + "flos": 20372590477440.0, + "grad_norm": 2.240569372684756, + "language_loss": 0.70785958, + "learning_rate": 2.4274483872219863e-06, + "loss": 0.7298063, + "num_input_tokens_seen": 80277270, + "router_z_loss_clip": 0.96826172, + "router_z_loss_mlp": 0.17901611, + "step": 3730, + "time_per_iteration": 2.7528655529022217 + }, + { + "auxiliary_loss_clip": 0.01142448, + "auxiliary_loss_mlp": 0.01046394, + "balance_loss_clip": 1.04706705, + "balance_loss_mlp": 1.0278573, + "epoch": 0.4486262249744484, + "flos": 20047742853120.0, + "grad_norm": 1.7937053757340704, + "language_loss": 0.93986714, + "learning_rate": 2.426687382058386e-06, + "loss": 0.96175551, + "num_input_tokens_seen": 80295550, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.18530273, + "step": 3731, + "time_per_iteration": 2.726205825805664 + }, + { + "auxiliary_loss_clip": 0.01045257, + "auxiliary_loss_mlp": 0.01019667, + "balance_loss_clip": 1.01357579, + "balance_loss_mlp": 1.01808202, + "epoch": 0.4487464678650875, + "flos": 64595684776320.0, + "grad_norm": 0.8661321237679925, + "language_loss": 0.59758312, + "learning_rate": 2.425926312166649e-06, + "loss": 0.61823237, + "num_input_tokens_seen": 80348425, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01586914, + "step": 3732, + "time_per_iteration": 3.135453939437866 + }, + { + "auxiliary_loss_clip": 0.01150645, + "auxiliary_loss_mlp": 0.01047134, + "balance_loss_clip": 1.04871678, + "balance_loss_mlp": 1.02794147, + "epoch": 0.4488667107557266, + "flos": 20769798049920.0, + "grad_norm": 2.551272123601317, + "language_loss": 0.73618639, + "learning_rate": 2.42516517766223e-06, + "loss": 0.75816417, + "num_input_tokens_seen": 80366505, + "router_z_loss_clip": 1.01806641, + "router_z_loss_mlp": 0.19177246, + "step": 3733, + "time_per_iteration": 2.7393336296081543 + }, + { + "auxiliary_loss_clip": 0.01144878, + "auxiliary_loss_mlp": 0.01045127, + "balance_loss_clip": 1.04797196, + "balance_loss_mlp": 1.02749586, + "epoch": 0.44898695364636565, + "flos": 23951735326080.0, + "grad_norm": 1.8893357901733292, + "language_loss": 0.67881334, + "learning_rate": 2.4244039786605907e-06, + "loss": 0.7007134, + "num_input_tokens_seen": 80387510, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.17626953, + "step": 3734, + "time_per_iteration": 2.807798385620117 + }, + { + "auxiliary_loss_clip": 0.0114338, + "auxiliary_loss_mlp": 0.01042224, + "balance_loss_clip": 1.04528165, + "balance_loss_mlp": 1.02485526, + "epoch": 0.44910719653700476, + "flos": 18624351628800.0, + "grad_norm": 7.793144338756141, + "language_loss": 0.82411784, + "learning_rate": 2.4236427152772055e-06, + "loss": 0.84597385, + "num_input_tokens_seen": 80405915, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.17346191, + "step": 3735, + "time_per_iteration": 2.709167242050171 + }, + { + "auxiliary_loss_clip": 0.01047467, + "auxiliary_loss_mlp": 0.01003381, + "balance_loss_clip": 1.01540685, + "balance_loss_mlp": 1.001688, + "epoch": 0.4492274394276438, + "flos": 57033435749760.0, + "grad_norm": 0.8327444595521712, + "language_loss": 0.5739907, + "learning_rate": 2.422881387627557e-06, + "loss": 0.59449911, + "num_input_tokens_seen": 80458365, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.01696777, + "step": 3736, + "time_per_iteration": 3.0240466594696045 + }, + { + "auxiliary_loss_clip": 0.0114824, + "auxiliary_loss_mlp": 0.01044009, + "balance_loss_clip": 1.04791367, + "balance_loss_mlp": 1.02719998, + "epoch": 0.4493476823182829, + "flos": 23254888498560.0, + "grad_norm": 2.5529376068822422, + "language_loss": 0.7736342, + "learning_rate": 2.422119995827139e-06, + "loss": 0.79555666, + "num_input_tokens_seen": 80478490, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.16809082, + "step": 3737, + "time_per_iteration": 2.7478420734405518 + }, + { + "auxiliary_loss_clip": 0.01142969, + "auxiliary_loss_mlp": 0.01046388, + "balance_loss_clip": 1.04608595, + "balance_loss_mlp": 1.02708769, + "epoch": 0.44946792520892204, + "flos": 15815131827840.0, + "grad_norm": 2.4556578652389334, + "language_loss": 0.7391094, + "learning_rate": 2.4213585399914528e-06, + "loss": 0.76100302, + "num_input_tokens_seen": 80495695, + "router_z_loss_clip": 0.96826172, + "router_z_loss_mlp": 0.19287109, + "step": 3738, + "time_per_iteration": 2.692499876022339 + }, + { + "auxiliary_loss_clip": 0.01143047, + "auxiliary_loss_mlp": 0.01043301, + "balance_loss_clip": 1.04709518, + "balance_loss_mlp": 1.02515745, + "epoch": 0.4495881680995611, + "flos": 19610063631360.0, + "grad_norm": 1.7676131833033084, + "language_loss": 0.85389215, + "learning_rate": 2.4205970202360113e-06, + "loss": 0.87575561, + "num_input_tokens_seen": 80515260, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.18127441, + "step": 3739, + "time_per_iteration": 3.5628013610839844 + }, + { + "auxiliary_loss_clip": 0.01144675, + "auxiliary_loss_mlp": 0.01049899, + "balance_loss_clip": 1.04708266, + "balance_loss_mlp": 1.03288782, + "epoch": 0.4497084109902002, + "flos": 26031465815040.0, + "grad_norm": 1.8726658601175705, + "language_loss": 0.78101814, + "learning_rate": 2.4198354366763354e-06, + "loss": 0.80296385, + "num_input_tokens_seen": 80533900, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.17016602, + "step": 3740, + "time_per_iteration": 2.7694289684295654 + }, + { + "auxiliary_loss_clip": 0.01147216, + "auxiliary_loss_mlp": 0.0104086, + "balance_loss_clip": 1.04787767, + "balance_loss_mlp": 1.02372956, + "epoch": 0.4498286538808393, + "flos": 14793688771200.0, + "grad_norm": 1.9103872624735367, + "language_loss": 0.78702182, + "learning_rate": 2.4190737894279587e-06, + "loss": 0.80890256, + "num_input_tokens_seen": 80551270, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.17138672, + "step": 3741, + "time_per_iteration": 3.5761466026306152 + }, + { + "auxiliary_loss_clip": 0.01142794, + "auxiliary_loss_mlp": 0.01042667, + "balance_loss_clip": 1.04642487, + "balance_loss_mlp": 1.02567935, + "epoch": 0.44994889677147837, + "flos": 15450171690240.0, + "grad_norm": 2.2293353242294978, + "language_loss": 0.80648124, + "learning_rate": 2.4183120786064203e-06, + "loss": 0.82833588, + "num_input_tokens_seen": 80568145, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.16992188, + "step": 3742, + "time_per_iteration": 2.7564165592193604 + }, + { + "auxiliary_loss_clip": 0.01149351, + "auxiliary_loss_mlp": 0.01050467, + "balance_loss_clip": 1.04971015, + "balance_loss_mlp": 1.03214407, + "epoch": 0.4500691396621175, + "flos": 21798316085760.0, + "grad_norm": 3.8649056053628428, + "language_loss": 0.85724455, + "learning_rate": 2.417550304327273e-06, + "loss": 0.87924272, + "num_input_tokens_seen": 80586185, + "router_z_loss_clip": 0.99658203, + "router_z_loss_mlp": 0.18334961, + "step": 3743, + "time_per_iteration": 2.708282470703125 + }, + { + "auxiliary_loss_clip": 0.0115255, + "auxiliary_loss_mlp": 0.01047146, + "balance_loss_clip": 1.05014944, + "balance_loss_mlp": 1.02721429, + "epoch": 0.4501893825527566, + "flos": 32382016421760.0, + "grad_norm": 1.7373842790080367, + "language_loss": 0.75671566, + "learning_rate": 2.4167884667060763e-06, + "loss": 0.77871263, + "num_input_tokens_seen": 80608895, + "router_z_loss_clip": 1.02441406, + "router_z_loss_mlp": 0.19921875, + "step": 3744, + "time_per_iteration": 3.576185941696167 + }, + { + "auxiliary_loss_clip": 0.01147646, + "auxiliary_loss_mlp": 0.01046264, + "balance_loss_clip": 1.04683924, + "balance_loss_mlp": 1.0276556, + "epoch": 0.45030962544339564, + "flos": 16544944362240.0, + "grad_norm": 2.10105511697928, + "language_loss": 0.87165427, + "learning_rate": 2.4160265658584e-06, + "loss": 0.89359337, + "num_input_tokens_seen": 80623785, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.18603516, + "step": 3745, + "time_per_iteration": 2.7064881324768066 + }, + { + "auxiliary_loss_clip": 0.01142881, + "auxiliary_loss_mlp": 0.0104519, + "balance_loss_clip": 1.04554248, + "balance_loss_mlp": 1.02498364, + "epoch": 0.45042986833403476, + "flos": 19573039687680.0, + "grad_norm": 1.878788457239436, + "language_loss": 0.68738914, + "learning_rate": 2.4152646018998253e-06, + "loss": 0.70926988, + "num_input_tokens_seen": 80642735, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.20214844, + "step": 3746, + "time_per_iteration": 3.5322327613830566 + }, + { + "auxiliary_loss_clip": 0.01148303, + "auxiliary_loss_mlp": 0.01044731, + "balance_loss_clip": 1.04827809, + "balance_loss_mlp": 1.02744555, + "epoch": 0.45055011122467387, + "flos": 23112467072640.0, + "grad_norm": 1.8445980944121805, + "language_loss": 0.71811551, + "learning_rate": 2.4145025749459403e-06, + "loss": 0.74004579, + "num_input_tokens_seen": 80663760, + "router_z_loss_clip": 0.99902344, + "router_z_loss_mlp": 0.17272949, + "step": 3747, + "time_per_iteration": 2.755575656890869 + }, + { + "auxiliary_loss_clip": 0.01149879, + "auxiliary_loss_mlp": 0.01046435, + "balance_loss_clip": 1.04924035, + "balance_loss_mlp": 1.02854192, + "epoch": 0.4506703541153129, + "flos": 19934623946880.0, + "grad_norm": 2.600745570397522, + "language_loss": 0.7016784, + "learning_rate": 2.413740485112344e-06, + "loss": 0.72364151, + "num_input_tokens_seen": 80682100, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.17883301, + "step": 3748, + "time_per_iteration": 2.698871374130249 + }, + { + "auxiliary_loss_clip": 0.01139645, + "auxiliary_loss_mlp": 0.01039954, + "balance_loss_clip": 1.04482329, + "balance_loss_mlp": 1.02321744, + "epoch": 0.45079059700595203, + "flos": 19499530504320.0, + "grad_norm": 1.561218608421276, + "language_loss": 0.82458198, + "learning_rate": 2.412978332514646e-06, + "loss": 0.84637797, + "num_input_tokens_seen": 80700880, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.16748047, + "step": 3749, + "time_per_iteration": 2.8279170989990234 + }, + { + "auxiliary_loss_clip": 0.0114881, + "auxiliary_loss_mlp": 0.01047971, + "balance_loss_clip": 1.04956174, + "balance_loss_mlp": 1.03020835, + "epoch": 0.4509108398965911, + "flos": 27636313570560.0, + "grad_norm": 1.9715986016859688, + "language_loss": 0.72300792, + "learning_rate": 2.4122161172684623e-06, + "loss": 0.74497575, + "num_input_tokens_seen": 80721675, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.17773438, + "step": 3750, + "time_per_iteration": 2.81021785736084 + }, + { + "auxiliary_loss_clip": 0.0114944, + "auxiliary_loss_mlp": 0.01048922, + "balance_loss_clip": 1.05045843, + "balance_loss_mlp": 1.02996731, + "epoch": 0.4510310827872302, + "flos": 20995712640000.0, + "grad_norm": 2.084198800230426, + "language_loss": 0.84751529, + "learning_rate": 2.4114538394894216e-06, + "loss": 0.86949891, + "num_input_tokens_seen": 80739315, + "router_z_loss_clip": 0.98925781, + "router_z_loss_mlp": 0.18945312, + "step": 3751, + "time_per_iteration": 2.7261059284210205 + }, + { + "auxiliary_loss_clip": 0.01142738, + "auxiliary_loss_mlp": 0.01047672, + "balance_loss_clip": 1.04710603, + "balance_loss_mlp": 1.03039896, + "epoch": 0.4511513256778693, + "flos": 16216684945920.0, + "grad_norm": 2.027520041327888, + "language_loss": 0.83520597, + "learning_rate": 2.410691499293161e-06, + "loss": 0.85711002, + "num_input_tokens_seen": 80757470, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.17272949, + "step": 3752, + "time_per_iteration": 2.7878148555755615 + }, + { + "auxiliary_loss_clip": 0.01144322, + "auxiliary_loss_mlp": 0.01050834, + "balance_loss_clip": 1.0464406, + "balance_loss_mlp": 1.03302407, + "epoch": 0.45127156856850836, + "flos": 25186702780800.0, + "grad_norm": 1.533044164632311, + "language_loss": 0.74118221, + "learning_rate": 2.409929096795326e-06, + "loss": 0.76313376, + "num_input_tokens_seen": 80777840, + "router_z_loss_clip": 0.97949219, + "router_z_loss_mlp": 0.17797852, + "step": 3753, + "time_per_iteration": 2.779783248901367 + }, + { + "auxiliary_loss_clip": 0.01148812, + "auxiliary_loss_mlp": 0.01043645, + "balance_loss_clip": 1.04916525, + "balance_loss_mlp": 1.02496445, + "epoch": 0.4513918114591475, + "flos": 20412523422720.0, + "grad_norm": 1.8199649130572273, + "language_loss": 0.79245663, + "learning_rate": 2.409166632111573e-06, + "loss": 0.81438124, + "num_input_tokens_seen": 80795975, + "router_z_loss_clip": 0.99609375, + "router_z_loss_mlp": 0.18701172, + "step": 3754, + "time_per_iteration": 2.7417666912078857 + }, + { + "auxiliary_loss_clip": 0.01151133, + "auxiliary_loss_mlp": 0.01039077, + "balance_loss_clip": 1.04799342, + "balance_loss_mlp": 1.02083814, + "epoch": 0.4515120543497866, + "flos": 26648482665600.0, + "grad_norm": 2.885403558690869, + "language_loss": 0.8089484, + "learning_rate": 2.4084041053575674e-06, + "loss": 0.83085048, + "num_input_tokens_seen": 80815395, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.18249512, + "step": 3755, + "time_per_iteration": 2.745462656021118 + }, + { + "auxiliary_loss_clip": 0.0114644, + "auxiliary_loss_mlp": 0.01042529, + "balance_loss_clip": 1.04819798, + "balance_loss_mlp": 1.0257082, + "epoch": 0.45163229724042564, + "flos": 20595093275520.0, + "grad_norm": 2.0016545324374633, + "language_loss": 0.72286212, + "learning_rate": 2.4076415166489834e-06, + "loss": 0.74475187, + "num_input_tokens_seen": 80834805, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.16809082, + "step": 3756, + "time_per_iteration": 2.7936315536499023 + }, + { + "auxiliary_loss_clip": 0.01145374, + "auxiliary_loss_mlp": 0.01041137, + "balance_loss_clip": 1.04690325, + "balance_loss_mlp": 1.02435279, + "epoch": 0.45175254013106475, + "flos": 21689004021120.0, + "grad_norm": 1.5910206583493394, + "language_loss": 0.78834867, + "learning_rate": 2.406878866101506e-06, + "loss": 0.8102138, + "num_input_tokens_seen": 80853770, + "router_z_loss_clip": 0.98583984, + "router_z_loss_mlp": 0.16809082, + "step": 3757, + "time_per_iteration": 2.723982810974121 + }, + { + "auxiliary_loss_clip": 0.01145875, + "auxiliary_loss_mlp": 0.0104389, + "balance_loss_clip": 1.0503391, + "balance_loss_mlp": 1.02646148, + "epoch": 0.45187278302170386, + "flos": 18878850466560.0, + "grad_norm": 2.0824041366984467, + "language_loss": 0.78261662, + "learning_rate": 2.4061161538308273e-06, + "loss": 0.80451429, + "num_input_tokens_seen": 80870615, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.17431641, + "step": 3758, + "time_per_iteration": 2.7332863807678223 + }, + { + "auxiliary_loss_clip": 0.01147253, + "auxiliary_loss_mlp": 0.01040827, + "balance_loss_clip": 1.04994082, + "balance_loss_mlp": 1.02186048, + "epoch": 0.4519930259123429, + "flos": 18582479349120.0, + "grad_norm": 1.7853515907264306, + "language_loss": 0.89508319, + "learning_rate": 2.4053533799526523e-06, + "loss": 0.91696399, + "num_input_tokens_seen": 80886335, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.18969727, + "step": 3759, + "time_per_iteration": 2.7280218601226807 + }, + { + "auxiliary_loss_clip": 0.01139549, + "auxiliary_loss_mlp": 0.01036153, + "balance_loss_clip": 1.04520774, + "balance_loss_mlp": 1.01887941, + "epoch": 0.452113268802982, + "flos": 25192377129600.0, + "grad_norm": 1.6212237613489529, + "language_loss": 0.86157459, + "learning_rate": 2.404590544582691e-06, + "loss": 0.88333154, + "num_input_tokens_seen": 80904570, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.17285156, + "step": 3760, + "time_per_iteration": 2.736999988555908 + }, + { + "auxiliary_loss_clip": 0.01144391, + "auxiliary_loss_mlp": 0.01044705, + "balance_loss_clip": 1.04789948, + "balance_loss_mlp": 1.02594137, + "epoch": 0.45223351169362114, + "flos": 39378922312320.0, + "grad_norm": 1.638814011782926, + "language_loss": 0.808496, + "learning_rate": 2.403827647836666e-06, + "loss": 0.830387, + "num_input_tokens_seen": 80925125, + "router_z_loss_clip": 0.96582031, + "router_z_loss_mlp": 0.18762207, + "step": 3761, + "time_per_iteration": 2.902538299560547 + }, + { + "auxiliary_loss_clip": 0.01143563, + "auxiliary_loss_mlp": 0.0104546, + "balance_loss_clip": 1.04462218, + "balance_loss_mlp": 1.02704179, + "epoch": 0.4523537545842602, + "flos": 21582169994880.0, + "grad_norm": 1.7998101910980737, + "language_loss": 0.69233018, + "learning_rate": 2.4030646898303075e-06, + "loss": 0.7142204, + "num_input_tokens_seen": 80946615, + "router_z_loss_clip": 0.98974609, + "router_z_loss_mlp": 0.18408203, + "step": 3762, + "time_per_iteration": 2.7265167236328125 + }, + { + "auxiliary_loss_clip": 0.0114509, + "auxiliary_loss_mlp": 0.01052771, + "balance_loss_clip": 1.04649699, + "balance_loss_mlp": 1.03468704, + "epoch": 0.4524739974748993, + "flos": 28439527547520.0, + "grad_norm": 2.4221874786356157, + "language_loss": 0.82185715, + "learning_rate": 2.4023016706793566e-06, + "loss": 0.84383577, + "num_input_tokens_seen": 80966410, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.18078613, + "step": 3763, + "time_per_iteration": 2.813951253890991 + }, + { + "auxiliary_loss_clip": 0.01043974, + "auxiliary_loss_mlp": 0.01004712, + "balance_loss_clip": 1.01233089, + "balance_loss_mlp": 1.0027926, + "epoch": 0.4525942403655384, + "flos": 61556492148480.0, + "grad_norm": 0.7628486697437369, + "language_loss": 0.56860858, + "learning_rate": 2.401538590499561e-06, + "loss": 0.58909547, + "num_input_tokens_seen": 81026865, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01916504, + "step": 3764, + "time_per_iteration": 3.2788445949554443 + }, + { + "auxiliary_loss_clip": 0.01147854, + "auxiliary_loss_mlp": 0.01041939, + "balance_loss_clip": 1.05009913, + "balance_loss_mlp": 1.02418828, + "epoch": 0.45271448325617747, + "flos": 27529838680320.0, + "grad_norm": 1.9550054416256344, + "language_loss": 0.72381878, + "learning_rate": 2.400775449406682e-06, + "loss": 0.74571663, + "num_input_tokens_seen": 81050060, + "router_z_loss_clip": 0.97851562, + "router_z_loss_mlp": 0.17736816, + "step": 3765, + "time_per_iteration": 2.7919280529022217 + }, + { + "auxiliary_loss_clip": 0.01142223, + "auxiliary_loss_mlp": 0.01041135, + "balance_loss_clip": 1.0464884, + "balance_loss_mlp": 1.02381349, + "epoch": 0.4528347261468166, + "flos": 22452608275200.0, + "grad_norm": 1.996337443384312, + "language_loss": 0.73069173, + "learning_rate": 2.400012247516485e-06, + "loss": 0.75252533, + "num_input_tokens_seen": 81070625, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.17321777, + "step": 3766, + "time_per_iteration": 2.852689743041992 + }, + { + "auxiliary_loss_clip": 0.01144807, + "auxiliary_loss_mlp": 0.01046575, + "balance_loss_clip": 1.04798818, + "balance_loss_mlp": 1.0295701, + "epoch": 0.45295496903745563, + "flos": 21103875469440.0, + "grad_norm": 1.8377360839531596, + "language_loss": 0.90377498, + "learning_rate": 2.3992489849447484e-06, + "loss": 0.92568886, + "num_input_tokens_seen": 81089080, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.17004395, + "step": 3767, + "time_per_iteration": 3.5810165405273438 + }, + { + "auxiliary_loss_clip": 0.01146861, + "auxiliary_loss_mlp": 0.0104639, + "balance_loss_clip": 1.04844391, + "balance_loss_mlp": 1.02861619, + "epoch": 0.45307521192809475, + "flos": 23221168606080.0, + "grad_norm": 1.5998335837494, + "language_loss": 0.78800428, + "learning_rate": 2.3984856618072584e-06, + "loss": 0.80993682, + "num_input_tokens_seen": 81109115, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.17785645, + "step": 3768, + "time_per_iteration": 2.7621231079101562 + }, + { + "auxiliary_loss_clip": 0.01144558, + "auxiliary_loss_mlp": 0.0105756, + "balance_loss_clip": 1.0476284, + "balance_loss_mlp": 1.04088306, + "epoch": 0.45319545481873386, + "flos": 15560094286080.0, + "grad_norm": 1.8255308709411031, + "language_loss": 0.73653233, + "learning_rate": 2.3977222782198098e-06, + "loss": 0.7585535, + "num_input_tokens_seen": 81127750, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.16662598, + "step": 3769, + "time_per_iteration": 2.7013492584228516 + }, + { + "auxiliary_loss_clip": 0.01147423, + "auxiliary_loss_mlp": 0.01045204, + "balance_loss_clip": 1.04752207, + "balance_loss_mlp": 1.02620173, + "epoch": 0.4533156977093729, + "flos": 21944759834880.0, + "grad_norm": 1.6428592136355311, + "language_loss": 0.75435477, + "learning_rate": 2.3969588342982077e-06, + "loss": 0.77628112, + "num_input_tokens_seen": 81147125, + "router_z_loss_clip": 0.99853516, + "router_z_loss_mlp": 0.19006348, + "step": 3770, + "time_per_iteration": 3.6069176197052 + }, + { + "auxiliary_loss_clip": 0.01148299, + "auxiliary_loss_mlp": 0.01047231, + "balance_loss_clip": 1.04929113, + "balance_loss_mlp": 1.02943265, + "epoch": 0.453435940600012, + "flos": 24242180699520.0, + "grad_norm": 1.4948083182549057, + "language_loss": 0.72608328, + "learning_rate": 2.396195330158267e-06, + "loss": 0.74803853, + "num_input_tokens_seen": 81167015, + "router_z_loss_clip": 0.99072266, + "router_z_loss_mlp": 0.17785645, + "step": 3771, + "time_per_iteration": 2.7537407875061035 + }, + { + "auxiliary_loss_clip": 0.01149468, + "auxiliary_loss_mlp": 0.0105266, + "balance_loss_clip": 1.04936481, + "balance_loss_mlp": 1.03312135, + "epoch": 0.45355618349065113, + "flos": 23440367352960.0, + "grad_norm": 1.749554511254241, + "language_loss": 0.79735696, + "learning_rate": 2.3954317659158094e-06, + "loss": 0.81937826, + "num_input_tokens_seen": 81187350, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.1953125, + "step": 3772, + "time_per_iteration": 2.736994504928589 + }, + { + "auxiliary_loss_clip": 0.01042841, + "auxiliary_loss_mlp": 0.01003624, + "balance_loss_clip": 1.01106763, + "balance_loss_mlp": 1.00178838, + "epoch": 0.4536764263812902, + "flos": 66903161448960.0, + "grad_norm": 0.9232920632329542, + "language_loss": 0.56886554, + "learning_rate": 2.394668141686667e-06, + "loss": 0.5893302, + "num_input_tokens_seen": 81249315, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01831055, + "step": 3773, + "time_per_iteration": 4.0772693157196045 + }, + { + "auxiliary_loss_clip": 0.01144512, + "auxiliary_loss_mlp": 0.01040973, + "balance_loss_clip": 1.04751158, + "balance_loss_mlp": 1.02436721, + "epoch": 0.4537966692719293, + "flos": 42739766254080.0, + "grad_norm": 2.782290191143624, + "language_loss": 0.69632655, + "learning_rate": 2.3939044575866813e-06, + "loss": 0.71818137, + "num_input_tokens_seen": 81272065, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.16589355, + "step": 3774, + "time_per_iteration": 2.924004554748535 + }, + { + "auxiliary_loss_clip": 0.0114306, + "auxiliary_loss_mlp": 0.01043309, + "balance_loss_clip": 1.04606009, + "balance_loss_mlp": 1.02570224, + "epoch": 0.4539169121625684, + "flos": 35549480517120.0, + "grad_norm": 2.0075951716406144, + "language_loss": 0.75575066, + "learning_rate": 2.3931407137317024e-06, + "loss": 0.77761436, + "num_input_tokens_seen": 81292220, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.17602539, + "step": 3775, + "time_per_iteration": 3.6429543495178223 + }, + { + "auxiliary_loss_clip": 0.01142861, + "auxiliary_loss_mlp": 0.01049235, + "balance_loss_clip": 1.04645491, + "balance_loss_mlp": 1.03091216, + "epoch": 0.45403715505320746, + "flos": 18514716341760.0, + "grad_norm": 1.9442872652547074, + "language_loss": 0.85063404, + "learning_rate": 2.3923769102375907e-06, + "loss": 0.87255502, + "num_input_tokens_seen": 81311085, + "router_z_loss_clip": 0.96435547, + "router_z_loss_mlp": 0.18322754, + "step": 3776, + "time_per_iteration": 2.7330875396728516 + }, + { + "auxiliary_loss_clip": 0.0114822, + "auxiliary_loss_mlp": 0.01047522, + "balance_loss_clip": 1.04811382, + "balance_loss_mlp": 1.03002262, + "epoch": 0.4541573979438466, + "flos": 25045825639680.0, + "grad_norm": 2.0815818749834265, + "language_loss": 0.78741455, + "learning_rate": 2.391613047220213e-06, + "loss": 0.80937195, + "num_input_tokens_seen": 81330985, + "router_z_loss_clip": 1.00146484, + "router_z_loss_mlp": 0.17504883, + "step": 3777, + "time_per_iteration": 2.800687074661255 + }, + { + "auxiliary_loss_clip": 0.01146336, + "auxiliary_loss_mlp": 0.0104411, + "balance_loss_clip": 1.04767585, + "balance_loss_mlp": 1.02492976, + "epoch": 0.4542776408344857, + "flos": 18332397884160.0, + "grad_norm": 1.914741411673628, + "language_loss": 0.79482126, + "learning_rate": 2.390849124795447e-06, + "loss": 0.81672567, + "num_input_tokens_seen": 81346985, + "router_z_loss_clip": 0.98730469, + "router_z_loss_mlp": 0.19177246, + "step": 3778, + "time_per_iteration": 2.712034225463867 + }, + { + "auxiliary_loss_clip": 0.01146226, + "auxiliary_loss_mlp": 0.01039624, + "balance_loss_clip": 1.0478915, + "balance_loss_mlp": 1.0229404, + "epoch": 0.45439788372512474, + "flos": 20701173116160.0, + "grad_norm": 5.057290777253049, + "language_loss": 0.8399024, + "learning_rate": 2.3900851430791804e-06, + "loss": 0.86176091, + "num_input_tokens_seen": 81365005, + "router_z_loss_clip": 0.98388672, + "router_z_loss_mlp": 0.16680908, + "step": 3779, + "time_per_iteration": 2.7147738933563232 + }, + { + "auxiliary_loss_clip": 0.01148391, + "auxiliary_loss_mlp": 0.01050228, + "balance_loss_clip": 1.04741633, + "balance_loss_mlp": 1.03121376, + "epoch": 0.45451812661576385, + "flos": 22309432663680.0, + "grad_norm": 2.1759839706501554, + "language_loss": 0.84931678, + "learning_rate": 2.389321102187307e-06, + "loss": 0.87130296, + "num_input_tokens_seen": 81383785, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.19018555, + "step": 3780, + "time_per_iteration": 2.7290475368499756 + }, + { + "auxiliary_loss_clip": 0.01148318, + "auxiliary_loss_mlp": 0.01058252, + "balance_loss_clip": 1.04857063, + "balance_loss_mlp": 1.03861773, + "epoch": 0.4546383695064029, + "flos": 21763303303680.0, + "grad_norm": 1.7134827576010774, + "language_loss": 0.81708539, + "learning_rate": 2.3885570022357326e-06, + "loss": 0.83915114, + "num_input_tokens_seen": 81402915, + "router_z_loss_clip": 0.99755859, + "router_z_loss_mlp": 0.19616699, + "step": 3781, + "time_per_iteration": 2.7938854694366455 + }, + { + "auxiliary_loss_clip": 0.01042766, + "auxiliary_loss_mlp": 0.01001796, + "balance_loss_clip": 1.01117265, + "balance_loss_mlp": 0.99991268, + "epoch": 0.454758612397042, + "flos": 64242755694720.0, + "grad_norm": 0.8067211739777304, + "language_loss": 0.60895491, + "learning_rate": 2.38779284334037e-06, + "loss": 0.62940055, + "num_input_tokens_seen": 81467890, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01879883, + "step": 3782, + "time_per_iteration": 3.3893120288848877 + }, + { + "auxiliary_loss_clip": 0.01148046, + "auxiliary_loss_mlp": 0.01042306, + "balance_loss_clip": 1.04818738, + "balance_loss_mlp": 1.0249846, + "epoch": 0.4548788552876811, + "flos": 27304175485440.0, + "grad_norm": 1.8484570995518412, + "language_loss": 0.78789246, + "learning_rate": 2.387028625617141e-06, + "loss": 0.80979598, + "num_input_tokens_seen": 81487105, + "router_z_loss_clip": 0.99902344, + "router_z_loss_mlp": 0.17321777, + "step": 3783, + "time_per_iteration": 2.791990280151367 + }, + { + "auxiliary_loss_clip": 0.01143523, + "auxiliary_loss_mlp": 0.01043486, + "balance_loss_clip": 1.04715395, + "balance_loss_mlp": 1.02569962, + "epoch": 0.4549990981783202, + "flos": 22857142222080.0, + "grad_norm": 1.9189094333951682, + "language_loss": 0.84292424, + "learning_rate": 2.3862643491819766e-06, + "loss": 0.86479437, + "num_input_tokens_seen": 81505670, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.17773438, + "step": 3784, + "time_per_iteration": 2.7330024242401123 + }, + { + "auxiliary_loss_clip": 0.01144321, + "auxiliary_loss_mlp": 0.01041441, + "balance_loss_clip": 1.04525125, + "balance_loss_mlp": 1.02359533, + "epoch": 0.4551193410689593, + "flos": 23258587599360.0, + "grad_norm": 1.9880022779381017, + "language_loss": 0.84275496, + "learning_rate": 2.3855000141508186e-06, + "loss": 0.86461258, + "num_input_tokens_seen": 81525825, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.17834473, + "step": 3785, + "time_per_iteration": 2.8157522678375244 + }, + { + "auxiliary_loss_clip": 0.0114696, + "auxiliary_loss_mlp": 0.01044143, + "balance_loss_clip": 1.04754734, + "balance_loss_mlp": 1.02467608, + "epoch": 0.4552395839595984, + "flos": 20777519473920.0, + "grad_norm": 1.9853620995039596, + "language_loss": 0.83976936, + "learning_rate": 2.3847356206396143e-06, + "loss": 0.86168039, + "num_input_tokens_seen": 81543135, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.19458008, + "step": 3786, + "time_per_iteration": 2.7643585205078125 + }, + { + "auxiliary_loss_clip": 0.01147837, + "auxiliary_loss_mlp": 0.0104263, + "balance_loss_clip": 1.05024922, + "balance_loss_mlp": 1.02482009, + "epoch": 0.45535982685023746, + "flos": 23257510191360.0, + "grad_norm": 1.48521067267826, + "language_loss": 0.78751093, + "learning_rate": 2.3839711687643227e-06, + "loss": 0.80941558, + "num_input_tokens_seen": 81564360, + "router_z_loss_clip": 0.97558594, + "router_z_loss_mlp": 0.17810059, + "step": 3787, + "time_per_iteration": 2.760923147201538 + }, + { + "auxiliary_loss_clip": 0.01148896, + "auxiliary_loss_mlp": 0.01046385, + "balance_loss_clip": 1.04689193, + "balance_loss_mlp": 1.02732301, + "epoch": 0.45548006974087657, + "flos": 19646117907840.0, + "grad_norm": 1.9416230485026833, + "language_loss": 0.74463648, + "learning_rate": 2.38320665864091e-06, + "loss": 0.76658928, + "num_input_tokens_seen": 81583710, + "router_z_loss_clip": 1.02001953, + "router_z_loss_mlp": 0.19067383, + "step": 3788, + "time_per_iteration": 2.8035712242126465 + }, + { + "auxiliary_loss_clip": 0.01145817, + "auxiliary_loss_mlp": 0.01039902, + "balance_loss_clip": 1.04580152, + "balance_loss_mlp": 1.02231872, + "epoch": 0.4556003126315157, + "flos": 20047778766720.0, + "grad_norm": 1.7485943537496202, + "language_loss": 0.81819177, + "learning_rate": 2.3824420903853516e-06, + "loss": 0.84004891, + "num_input_tokens_seen": 81602175, + "router_z_loss_clip": 0.99853516, + "router_z_loss_mlp": 0.17590332, + "step": 3789, + "time_per_iteration": 2.70440411567688 + }, + { + "auxiliary_loss_clip": 0.01146371, + "auxiliary_loss_mlp": 0.01042987, + "balance_loss_clip": 1.04707694, + "balance_loss_mlp": 1.02452183, + "epoch": 0.45572055552215474, + "flos": 22959738443520.0, + "grad_norm": 2.0867688290589537, + "language_loss": 0.82157826, + "learning_rate": 2.3816774641136324e-06, + "loss": 0.84347177, + "num_input_tokens_seen": 81619430, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.18444824, + "step": 3790, + "time_per_iteration": 2.743150472640991 + }, + { + "auxiliary_loss_clip": 0.01145828, + "auxiliary_loss_mlp": 0.0104234, + "balance_loss_clip": 1.04709637, + "balance_loss_mlp": 1.02457809, + "epoch": 0.45584079841279385, + "flos": 33109925535360.0, + "grad_norm": 1.9410516186904052, + "language_loss": 0.7125783, + "learning_rate": 2.380912779941745e-06, + "loss": 0.73445994, + "num_input_tokens_seen": 81642550, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.1776123, + "step": 3791, + "time_per_iteration": 2.7959768772125244 + }, + { + "auxiliary_loss_clip": 0.01149431, + "auxiliary_loss_mlp": 0.01046248, + "balance_loss_clip": 1.04615808, + "balance_loss_mlp": 1.02527928, + "epoch": 0.45596104130343296, + "flos": 27272179445760.0, + "grad_norm": 2.130077157907538, + "language_loss": 0.83416051, + "learning_rate": 2.3801480379856918e-06, + "loss": 0.85611725, + "num_input_tokens_seen": 81664260, + "router_z_loss_clip": 1.03320312, + "router_z_loss_mlp": 0.2097168, + "step": 3792, + "time_per_iteration": 2.871685028076172 + }, + { + "auxiliary_loss_clip": 0.01148551, + "auxiliary_loss_mlp": 0.01046225, + "balance_loss_clip": 1.04924953, + "balance_loss_mlp": 1.02864122, + "epoch": 0.456081284194072, + "flos": 21579799697280.0, + "grad_norm": 1.6768243813389918, + "language_loss": 0.83746547, + "learning_rate": 2.379383238361484e-06, + "loss": 0.85941321, + "num_input_tokens_seen": 81683620, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.17578125, + "step": 3793, + "time_per_iteration": 2.79056978225708 + }, + { + "auxiliary_loss_clip": 0.01144049, + "auxiliary_loss_mlp": 0.01038271, + "balance_loss_clip": 1.04564822, + "balance_loss_mlp": 1.02108109, + "epoch": 0.4562015270847111, + "flos": 35918822113920.0, + "grad_norm": 2.0989569167002764, + "language_loss": 0.80119926, + "learning_rate": 2.3786183811851407e-06, + "loss": 0.82302248, + "num_input_tokens_seen": 81704325, + "router_z_loss_clip": 0.98388672, + "router_z_loss_mlp": 0.171875, + "step": 3794, + "time_per_iteration": 2.952359676361084 + }, + { + "auxiliary_loss_clip": 0.0115053, + "auxiliary_loss_mlp": 0.01045254, + "balance_loss_clip": 1.04947114, + "balance_loss_mlp": 1.02622843, + "epoch": 0.45632176997535023, + "flos": 13589783602560.0, + "grad_norm": 1.7926333354959507, + "language_loss": 0.80160189, + "learning_rate": 2.3778534665726892e-06, + "loss": 0.82355976, + "num_input_tokens_seen": 81721155, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.19030762, + "step": 3795, + "time_per_iteration": 2.773383617401123 + }, + { + "auxiliary_loss_clip": 0.0114449, + "auxiliary_loss_mlp": 0.01048411, + "balance_loss_clip": 1.04816699, + "balance_loss_mlp": 1.03153121, + "epoch": 0.4564420128659893, + "flos": 32635401937920.0, + "grad_norm": 1.783499614489578, + "language_loss": 0.72867054, + "learning_rate": 2.377088494640168e-06, + "loss": 0.7505995, + "num_input_tokens_seen": 81742905, + "router_z_loss_clip": 0.96240234, + "router_z_loss_mlp": 0.16870117, + "step": 3796, + "time_per_iteration": 3.75380277633667 + }, + { + "auxiliary_loss_clip": 0.01145479, + "auxiliary_loss_mlp": 0.01043534, + "balance_loss_clip": 1.04711282, + "balance_loss_mlp": 1.02583122, + "epoch": 0.4565622557566284, + "flos": 20377690208640.0, + "grad_norm": 1.7960417704663052, + "language_loss": 0.78017807, + "learning_rate": 2.3763234655036216e-06, + "loss": 0.80206817, + "num_input_tokens_seen": 81762105, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.17687988, + "step": 3797, + "time_per_iteration": 2.7672815322875977 + }, + { + "auxiliary_loss_clip": 0.01144484, + "auxiliary_loss_mlp": 0.01045443, + "balance_loss_clip": 1.04676306, + "balance_loss_mlp": 1.02771688, + "epoch": 0.45668249864726745, + "flos": 25374372364800.0, + "grad_norm": 1.8083771834716638, + "language_loss": 0.86948955, + "learning_rate": 2.3755583792791046e-06, + "loss": 0.89138877, + "num_input_tokens_seen": 81781975, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.17736816, + "step": 3798, + "time_per_iteration": 2.773197889328003 + }, + { + "auxiliary_loss_clip": 0.01148932, + "auxiliary_loss_mlp": 0.01046262, + "balance_loss_clip": 1.04814744, + "balance_loss_mlp": 1.02724767, + "epoch": 0.45680274153790656, + "flos": 15559806977280.0, + "grad_norm": 1.9124654875867444, + "language_loss": 0.75063384, + "learning_rate": 2.3747932360826803e-06, + "loss": 0.77258581, + "num_input_tokens_seen": 81798905, + "router_z_loss_clip": 1.00878906, + "router_z_loss_mlp": 0.19006348, + "step": 3799, + "time_per_iteration": 3.7198264598846436 + }, + { + "auxiliary_loss_clip": 0.01151183, + "auxiliary_loss_mlp": 0.0104666, + "balance_loss_clip": 1.04908407, + "balance_loss_mlp": 1.02709746, + "epoch": 0.4569229844285457, + "flos": 19792884879360.0, + "grad_norm": 1.8258198627294318, + "language_loss": 0.82684243, + "learning_rate": 2.3740280360304205e-06, + "loss": 0.84882087, + "num_input_tokens_seen": 81816630, + "router_z_loss_clip": 1.02099609, + "router_z_loss_mlp": 0.19567871, + "step": 3800, + "time_per_iteration": 2.7494657039642334 + }, + { + "auxiliary_loss_clip": 0.0114825, + "auxiliary_loss_mlp": 0.01046323, + "balance_loss_clip": 1.04911089, + "balance_loss_mlp": 1.0284059, + "epoch": 0.45704322731918473, + "flos": 24093941270400.0, + "grad_norm": 1.5996418951021387, + "language_loss": 0.68210351, + "learning_rate": 2.3732627792384038e-06, + "loss": 0.70404929, + "num_input_tokens_seen": 81837700, + "router_z_loss_clip": 0.99121094, + "router_z_loss_mlp": 0.17907715, + "step": 3801, + "time_per_iteration": 2.836209297180176 + }, + { + "auxiliary_loss_clip": 0.01143968, + "auxiliary_loss_mlp": 0.01039957, + "balance_loss_clip": 1.04581451, + "balance_loss_mlp": 1.02144408, + "epoch": 0.45716347020982384, + "flos": 31317803245440.0, + "grad_norm": 1.8934909286769466, + "language_loss": 0.75227571, + "learning_rate": 2.3724974658227207e-06, + "loss": 0.77411497, + "num_input_tokens_seen": 81858490, + "router_z_loss_clip": 0.98144531, + "router_z_loss_mlp": 0.18530273, + "step": 3802, + "time_per_iteration": 3.8187239170074463 + }, + { + "auxiliary_loss_clip": 0.01145954, + "auxiliary_loss_mlp": 0.01045752, + "balance_loss_clip": 1.04680872, + "balance_loss_mlp": 1.02614236, + "epoch": 0.45728371310046295, + "flos": 26501392471680.0, + "grad_norm": 1.7967250783322084, + "language_loss": 0.71122491, + "learning_rate": 2.3717320958994687e-06, + "loss": 0.73314196, + "num_input_tokens_seen": 81876050, + "router_z_loss_clip": 0.99121094, + "router_z_loss_mlp": 0.19628906, + "step": 3803, + "time_per_iteration": 2.9036004543304443 + }, + { + "auxiliary_loss_clip": 0.01142549, + "auxiliary_loss_mlp": 0.01043593, + "balance_loss_clip": 1.04575086, + "balance_loss_mlp": 1.02573562, + "epoch": 0.457403955991102, + "flos": 17929408222080.0, + "grad_norm": 1.811240311471289, + "language_loss": 0.70605421, + "learning_rate": 2.3709666695847534e-06, + "loss": 0.72791564, + "num_input_tokens_seen": 81894230, + "router_z_loss_clip": 0.96582031, + "router_z_loss_mlp": 0.17834473, + "step": 3804, + "time_per_iteration": 3.6971170902252197 + }, + { + "auxiliary_loss_clip": 0.01143331, + "auxiliary_loss_mlp": 0.01038047, + "balance_loss_clip": 1.04706717, + "balance_loss_mlp": 1.02082133, + "epoch": 0.4575241988817411, + "flos": 42230660837760.0, + "grad_norm": 1.6905879177295438, + "language_loss": 0.70299709, + "learning_rate": 2.370201186994689e-06, + "loss": 0.72481084, + "num_input_tokens_seen": 81917915, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.17236328, + "step": 3805, + "time_per_iteration": 2.965444326400757 + }, + { + "auxiliary_loss_clip": 0.01146281, + "auxiliary_loss_mlp": 0.01041813, + "balance_loss_clip": 1.04875112, + "balance_loss_mlp": 1.02389574, + "epoch": 0.45764444177238023, + "flos": 30117309868800.0, + "grad_norm": 1.791947423289706, + "language_loss": 0.70032191, + "learning_rate": 2.369435648245399e-06, + "loss": 0.72220284, + "num_input_tokens_seen": 81938130, + "router_z_loss_clip": 0.97509766, + "router_z_loss_mlp": 0.17907715, + "step": 3806, + "time_per_iteration": 2.814826250076294 + }, + { + "auxiliary_loss_clip": 0.01147022, + "auxiliary_loss_mlp": 0.0105183, + "balance_loss_clip": 1.04904449, + "balance_loss_mlp": 1.03356671, + "epoch": 0.4577646846630193, + "flos": 24060293205120.0, + "grad_norm": 1.814799754417524, + "language_loss": 0.85381532, + "learning_rate": 2.368670053453015e-06, + "loss": 0.87580383, + "num_input_tokens_seen": 81959820, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.18273926, + "step": 3807, + "time_per_iteration": 3.250906229019165 + }, + { + "auxiliary_loss_clip": 0.01151053, + "auxiliary_loss_mlp": 0.01055408, + "balance_loss_clip": 1.05096459, + "balance_loss_mlp": 1.03701425, + "epoch": 0.4578849275536584, + "flos": 17418578952960.0, + "grad_norm": 2.750050370927329, + "language_loss": 0.74743068, + "learning_rate": 2.3679044027336757e-06, + "loss": 0.76949537, + "num_input_tokens_seen": 81975710, + "router_z_loss_clip": 1.00146484, + "router_z_loss_mlp": 0.18383789, + "step": 3808, + "time_per_iteration": 5.677364826202393 + }, + { + "auxiliary_loss_clip": 0.01148865, + "auxiliary_loss_mlp": 0.01042396, + "balance_loss_clip": 1.04841995, + "balance_loss_mlp": 1.02422881, + "epoch": 0.4580051704442975, + "flos": 13510169107200.0, + "grad_norm": 2.656274735243589, + "language_loss": 0.69663429, + "learning_rate": 2.3671386962035326e-06, + "loss": 0.71854693, + "num_input_tokens_seen": 81993180, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.18164062, + "step": 3809, + "time_per_iteration": 3.893296241760254 + }, + { + "auxiliary_loss_clip": 0.0114718, + "auxiliary_loss_mlp": 0.01046295, + "balance_loss_clip": 1.0468061, + "balance_loss_mlp": 1.02710199, + "epoch": 0.45812541333493656, + "flos": 18037606965120.0, + "grad_norm": 2.0443973101529824, + "language_loss": 0.68248057, + "learning_rate": 2.3663729339787405e-06, + "loss": 0.70441538, + "num_input_tokens_seen": 82010115, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.19177246, + "step": 3810, + "time_per_iteration": 2.89198899269104 + }, + { + "auxiliary_loss_clip": 0.01149058, + "auxiliary_loss_mlp": 0.01045062, + "balance_loss_clip": 1.04975796, + "balance_loss_mlp": 1.02635837, + "epoch": 0.45824565622557567, + "flos": 20222196232320.0, + "grad_norm": 3.3925243201099478, + "language_loss": 0.73490572, + "learning_rate": 2.365607116175466e-06, + "loss": 0.7568469, + "num_input_tokens_seen": 82025540, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.18701172, + "step": 3811, + "time_per_iteration": 2.7765445709228516 + }, + { + "auxiliary_loss_clip": 0.01144139, + "auxiliary_loss_mlp": 0.01041584, + "balance_loss_clip": 1.04545736, + "balance_loss_mlp": 1.0239048, + "epoch": 0.4583658991162148, + "flos": 19864885691520.0, + "grad_norm": 2.331490187335771, + "language_loss": 0.67334539, + "learning_rate": 2.3648412429098825e-06, + "loss": 0.69520259, + "num_input_tokens_seen": 82043890, + "router_z_loss_clip": 0.98730469, + "router_z_loss_mlp": 0.17675781, + "step": 3812, + "time_per_iteration": 2.7752890586853027 + }, + { + "auxiliary_loss_clip": 0.01153718, + "auxiliary_loss_mlp": 0.01041119, + "balance_loss_clip": 1.04932857, + "balance_loss_mlp": 1.02099705, + "epoch": 0.45848614200685384, + "flos": 21029935322880.0, + "grad_norm": 1.9156480846451631, + "language_loss": 0.81862819, + "learning_rate": 2.364075314298172e-06, + "loss": 0.84057665, + "num_input_tokens_seen": 82061345, + "router_z_loss_clip": 1.04443359, + "router_z_loss_mlp": 0.20129395, + "step": 3813, + "time_per_iteration": 2.86529278755188 + }, + { + "auxiliary_loss_clip": 0.01146408, + "auxiliary_loss_mlp": 0.01048575, + "balance_loss_clip": 1.04726076, + "balance_loss_mlp": 1.02909648, + "epoch": 0.45860638489749295, + "flos": 21069293650560.0, + "grad_norm": 1.8432189774080139, + "language_loss": 0.70179617, + "learning_rate": 2.3633093304565267e-06, + "loss": 0.72374606, + "num_input_tokens_seen": 82080400, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.19494629, + "step": 3814, + "time_per_iteration": 3.0403318405151367 + }, + { + "auxiliary_loss_clip": 0.01152861, + "auxiliary_loss_mlp": 0.01048361, + "balance_loss_clip": 1.05213118, + "balance_loss_mlp": 1.02896523, + "epoch": 0.458726627788132, + "flos": 26833889692800.0, + "grad_norm": 1.7326452082914767, + "language_loss": 0.62824953, + "learning_rate": 2.3625432915011443e-06, + "loss": 0.65026176, + "num_input_tokens_seen": 82102310, + "router_z_loss_clip": 1.0078125, + "router_z_loss_mlp": 0.1940918, + "step": 3815, + "time_per_iteration": 2.7825839519500732 + }, + { + "auxiliary_loss_clip": 0.01149588, + "auxiliary_loss_mlp": 0.01055891, + "balance_loss_clip": 1.05048084, + "balance_loss_mlp": 1.03685355, + "epoch": 0.4588468706787711, + "flos": 24097927680000.0, + "grad_norm": 1.6343986507565416, + "language_loss": 0.654531, + "learning_rate": 2.3617771975482334e-06, + "loss": 0.67658579, + "num_input_tokens_seen": 82121140, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.19030762, + "step": 3816, + "time_per_iteration": 2.7763566970825195 + }, + { + "auxiliary_loss_clip": 0.01145567, + "auxiliary_loss_mlp": 0.01044597, + "balance_loss_clip": 1.04892981, + "balance_loss_mlp": 1.02737117, + "epoch": 0.4589671135694102, + "flos": 17889331622400.0, + "grad_norm": 1.6249683861681246, + "language_loss": 0.74908012, + "learning_rate": 2.3610110487140083e-06, + "loss": 0.77098179, + "num_input_tokens_seen": 82139575, + "router_z_loss_clip": 0.96582031, + "router_z_loss_mlp": 0.17236328, + "step": 3817, + "time_per_iteration": 2.810652732849121 + }, + { + "auxiliary_loss_clip": 0.01143629, + "auxiliary_loss_mlp": 0.01041025, + "balance_loss_clip": 1.04639685, + "balance_loss_mlp": 1.02297664, + "epoch": 0.4590873564600493, + "flos": 25626967781760.0, + "grad_norm": 1.6924716884099134, + "language_loss": 0.8044312, + "learning_rate": 2.360244845114695e-06, + "loss": 0.82627773, + "num_input_tokens_seen": 82159195, + "router_z_loss_clip": 0.97265625, + "router_z_loss_mlp": 0.18041992, + "step": 3818, + "time_per_iteration": 2.823293447494507 + }, + { + "auxiliary_loss_clip": 0.01143883, + "auxiliary_loss_mlp": 0.01045464, + "balance_loss_clip": 1.04657841, + "balance_loss_mlp": 1.02796412, + "epoch": 0.4592075993506884, + "flos": 18514788168960.0, + "grad_norm": 2.06010582048155, + "language_loss": 0.68287838, + "learning_rate": 2.3594785868665245e-06, + "loss": 0.70477188, + "num_input_tokens_seen": 82175500, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.17492676, + "step": 3819, + "time_per_iteration": 2.730419874191284 + }, + { + "auxiliary_loss_clip": 0.01150666, + "auxiliary_loss_mlp": 0.01047793, + "balance_loss_clip": 1.04924881, + "balance_loss_mlp": 1.02892208, + "epoch": 0.4593278422413275, + "flos": 20631111638400.0, + "grad_norm": 2.233958570713951, + "language_loss": 0.80556148, + "learning_rate": 2.3587122740857386e-06, + "loss": 0.82754612, + "num_input_tokens_seen": 82192600, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.1887207, + "step": 3820, + "time_per_iteration": 2.8839006423950195 + }, + { + "auxiliary_loss_clip": 0.01141369, + "auxiliary_loss_mlp": 0.01037542, + "balance_loss_clip": 1.04685545, + "balance_loss_mlp": 1.02104306, + "epoch": 0.45944808513196655, + "flos": 21358517961600.0, + "grad_norm": 1.5163419028490495, + "language_loss": 0.78253746, + "learning_rate": 2.357945906888586e-06, + "loss": 0.80432659, + "num_input_tokens_seen": 82212040, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.16497803, + "step": 3821, + "time_per_iteration": 2.8912301063537598 + }, + { + "auxiliary_loss_clip": 0.01151768, + "auxiliary_loss_mlp": 0.01045755, + "balance_loss_clip": 1.04994011, + "balance_loss_mlp": 1.0266459, + "epoch": 0.45956832802260567, + "flos": 21427789340160.0, + "grad_norm": 3.3828709053676316, + "language_loss": 0.80261505, + "learning_rate": 2.357179485391324e-06, + "loss": 0.82459033, + "num_input_tokens_seen": 82229895, + "router_z_loss_clip": 1.01953125, + "router_z_loss_mlp": 0.19116211, + "step": 3822, + "time_per_iteration": 2.839224338531494 + }, + { + "auxiliary_loss_clip": 0.01146286, + "auxiliary_loss_mlp": 0.01048372, + "balance_loss_clip": 1.05000615, + "balance_loss_mlp": 1.03057456, + "epoch": 0.4596885709132448, + "flos": 22382654538240.0, + "grad_norm": 1.720215222631914, + "language_loss": 0.86048019, + "learning_rate": 2.3564130097102173e-06, + "loss": 0.88242674, + "num_input_tokens_seen": 82249550, + "router_z_loss_clip": 0.96240234, + "router_z_loss_mlp": 0.17797852, + "step": 3823, + "time_per_iteration": 2.869719982147217 + }, + { + "auxiliary_loss_clip": 0.01148803, + "auxiliary_loss_mlp": 0.01041671, + "balance_loss_clip": 1.0506103, + "balance_loss_mlp": 1.02400422, + "epoch": 0.45980881380388383, + "flos": 28981957806720.0, + "grad_norm": 3.1681689209686574, + "language_loss": 0.75356942, + "learning_rate": 2.355646479961541e-06, + "loss": 0.77547419, + "num_input_tokens_seen": 82268860, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.17663574, + "step": 3824, + "time_per_iteration": 2.8223485946655273 + }, + { + "auxiliary_loss_clip": 0.01148271, + "auxiliary_loss_mlp": 0.01049878, + "balance_loss_clip": 1.05010927, + "balance_loss_mlp": 1.03216386, + "epoch": 0.45992905669452294, + "flos": 33396599980800.0, + "grad_norm": 1.7073194773895672, + "language_loss": 0.71457607, + "learning_rate": 2.354879896261576e-06, + "loss": 0.73655748, + "num_input_tokens_seen": 82289070, + "router_z_loss_clip": 0.98193359, + "router_z_loss_mlp": 0.17712402, + "step": 3825, + "time_per_iteration": 3.657130718231201 + }, + { + "auxiliary_loss_clip": 0.01143622, + "auxiliary_loss_mlp": 0.01049733, + "balance_loss_clip": 1.04588902, + "balance_loss_mlp": 1.0311718, + "epoch": 0.46004929958516205, + "flos": 36318184502400.0, + "grad_norm": 2.362770916406472, + "language_loss": 0.57017934, + "learning_rate": 2.3541132587266133e-06, + "loss": 0.5921129, + "num_input_tokens_seen": 82311790, + "router_z_loss_clip": 0.97705078, + "router_z_loss_mlp": 0.18579102, + "step": 3826, + "time_per_iteration": 2.8577418327331543 + }, + { + "auxiliary_loss_clip": 0.0115135, + "auxiliary_loss_mlp": 0.01049803, + "balance_loss_clip": 1.05058813, + "balance_loss_mlp": 1.03149223, + "epoch": 0.4601695424758011, + "flos": 17238451224960.0, + "grad_norm": 1.9986832773198793, + "language_loss": 0.6929788, + "learning_rate": 2.3533465674729515e-06, + "loss": 0.71499032, + "num_input_tokens_seen": 82329020, + "router_z_loss_clip": 1.00683594, + "router_z_loss_mlp": 0.18310547, + "step": 3827, + "time_per_iteration": 2.8394522666931152 + }, + { + "auxiliary_loss_clip": 0.01149157, + "auxiliary_loss_mlp": 0.01041516, + "balance_loss_clip": 1.04986608, + "balance_loss_mlp": 1.02265668, + "epoch": 0.4602897853664402, + "flos": 15888425529600.0, + "grad_norm": 1.9755614111013666, + "language_loss": 0.72831947, + "learning_rate": 2.352579822616895e-06, + "loss": 0.7502262, + "num_input_tokens_seen": 82346455, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.18847656, + "step": 3828, + "time_per_iteration": 3.7206830978393555 + }, + { + "auxiliary_loss_clip": 0.01145058, + "auxiliary_loss_mlp": 0.01042327, + "balance_loss_clip": 1.04870141, + "balance_loss_mlp": 1.02463675, + "epoch": 0.4604100282570793, + "flos": 25412617370880.0, + "grad_norm": 1.7404764211496726, + "language_loss": 0.77522236, + "learning_rate": 2.351813024274761e-06, + "loss": 0.79709613, + "num_input_tokens_seen": 82367810, + "router_z_loss_clip": 0.96289062, + "router_z_loss_mlp": 0.17687988, + "step": 3829, + "time_per_iteration": 2.8837666511535645 + }, + { + "auxiliary_loss_clip": 0.01148369, + "auxiliary_loss_mlp": 0.0104886, + "balance_loss_clip": 1.05185843, + "balance_loss_mlp": 1.03027487, + "epoch": 0.4605302711477184, + "flos": 27630711048960.0, + "grad_norm": 1.7199562931324062, + "language_loss": 0.73561138, + "learning_rate": 2.3510461725628693e-06, + "loss": 0.75758362, + "num_input_tokens_seen": 82388275, + "router_z_loss_clip": 0.96435547, + "router_z_loss_mlp": 0.18579102, + "step": 3830, + "time_per_iteration": 2.9150657653808594 + }, + { + "auxiliary_loss_clip": 0.01150031, + "auxiliary_loss_mlp": 0.0104061, + "balance_loss_clip": 1.05191398, + "balance_loss_mlp": 1.02295542, + "epoch": 0.4606505140383575, + "flos": 23839657914240.0, + "grad_norm": 2.179004923972338, + "language_loss": 0.7118355, + "learning_rate": 2.350279267597554e-06, + "loss": 0.73374194, + "num_input_tokens_seen": 82408915, + "router_z_loss_clip": 0.98144531, + "router_z_loss_mlp": 0.17663574, + "step": 3831, + "time_per_iteration": 3.828686237335205 + }, + { + "auxiliary_loss_clip": 0.01149687, + "auxiliary_loss_mlp": 0.01042372, + "balance_loss_clip": 1.04982615, + "balance_loss_mlp": 1.02317929, + "epoch": 0.46077075692899655, + "flos": 16107013745280.0, + "grad_norm": 2.001005231014844, + "language_loss": 0.82735872, + "learning_rate": 2.3495123094951515e-06, + "loss": 0.84927928, + "num_input_tokens_seen": 82427260, + "router_z_loss_clip": 0.99951172, + "router_z_loss_mlp": 0.19189453, + "step": 3832, + "time_per_iteration": 3.151870012283325 + }, + { + "auxiliary_loss_clip": 0.01148496, + "auxiliary_loss_mlp": 0.01045178, + "balance_loss_clip": 1.05083954, + "balance_loss_mlp": 1.02676058, + "epoch": 0.46089099981963566, + "flos": 48798147634560.0, + "grad_norm": 2.1701694241403726, + "language_loss": 0.76130658, + "learning_rate": 2.34874529837201e-06, + "loss": 0.7832433, + "num_input_tokens_seen": 82450805, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.1842041, + "step": 3833, + "time_per_iteration": 4.227497100830078 + }, + { + "auxiliary_loss_clip": 0.01143699, + "auxiliary_loss_mlp": 0.0104348, + "balance_loss_clip": 1.04756188, + "balance_loss_mlp": 1.02555132, + "epoch": 0.46101124271027477, + "flos": 19099234362240.0, + "grad_norm": 1.92099970925205, + "language_loss": 0.79204166, + "learning_rate": 2.347978234344483e-06, + "loss": 0.81391346, + "num_input_tokens_seen": 82467010, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.17932129, + "step": 3834, + "time_per_iteration": 2.838125228881836 + }, + { + "auxiliary_loss_clip": 0.01147196, + "auxiliary_loss_mlp": 0.01044175, + "balance_loss_clip": 1.04763412, + "balance_loss_mlp": 1.02469015, + "epoch": 0.4611314856009138, + "flos": 39347931853440.0, + "grad_norm": 1.9948324862854532, + "language_loss": 0.69186515, + "learning_rate": 2.347211117528935e-06, + "loss": 0.71377885, + "num_input_tokens_seen": 82489310, + "router_z_loss_clip": 0.99707031, + "router_z_loss_mlp": 0.19482422, + "step": 3835, + "time_per_iteration": 2.9166224002838135 + }, + { + "auxiliary_loss_clip": 0.01149184, + "auxiliary_loss_mlp": 0.01041895, + "balance_loss_clip": 1.0514946, + "balance_loss_mlp": 1.02320313, + "epoch": 0.46125172849155294, + "flos": 20810772489600.0, + "grad_norm": 1.5007533669872448, + "language_loss": 0.7159071, + "learning_rate": 2.3464439480417374e-06, + "loss": 0.73781788, + "num_input_tokens_seen": 82508830, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.18688965, + "step": 3836, + "time_per_iteration": 2.8475334644317627 + }, + { + "auxiliary_loss_clip": 0.01153066, + "auxiliary_loss_mlp": 0.01042835, + "balance_loss_clip": 1.05091381, + "balance_loss_mlp": 1.02311802, + "epoch": 0.46137197138219205, + "flos": 17930808852480.0, + "grad_norm": 2.385575760556568, + "language_loss": 0.7741698, + "learning_rate": 2.3456767259992676e-06, + "loss": 0.79612875, + "num_input_tokens_seen": 82526475, + "router_z_loss_clip": 1.02197266, + "router_z_loss_mlp": 0.19714355, + "step": 3837, + "time_per_iteration": 2.800668478012085 + }, + { + "auxiliary_loss_clip": 0.01145968, + "auxiliary_loss_mlp": 0.01046677, + "balance_loss_clip": 1.04679942, + "balance_loss_mlp": 1.02954698, + "epoch": 0.4614922142728311, + "flos": 16836610798080.0, + "grad_norm": 2.2197599479417613, + "language_loss": 0.88800311, + "learning_rate": 2.3449094515179135e-06, + "loss": 0.90992951, + "num_input_tokens_seen": 82543935, + "router_z_loss_clip": 0.99121094, + "router_z_loss_mlp": 0.17138672, + "step": 3838, + "time_per_iteration": 2.7833056449890137 + }, + { + "auxiliary_loss_clip": 0.01146443, + "auxiliary_loss_mlp": 0.01041551, + "balance_loss_clip": 1.05024803, + "balance_loss_mlp": 1.024755, + "epoch": 0.4616124571634702, + "flos": 26614906427520.0, + "grad_norm": 1.6804827095206738, + "language_loss": 0.81972921, + "learning_rate": 2.34414212471407e-06, + "loss": 0.84160918, + "num_input_tokens_seen": 82563730, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.16796875, + "step": 3839, + "time_per_iteration": 2.967202663421631 + }, + { + "auxiliary_loss_clip": 0.01150681, + "auxiliary_loss_mlp": 0.0104965, + "balance_loss_clip": 1.04868424, + "balance_loss_mlp": 1.03098226, + "epoch": 0.4617327000541093, + "flos": 20340127560960.0, + "grad_norm": 2.0046240225629575, + "language_loss": 0.72890943, + "learning_rate": 2.3433747457041394e-06, + "loss": 0.75091279, + "num_input_tokens_seen": 82582435, + "router_z_loss_clip": 1.02099609, + "router_z_loss_mlp": 0.18664551, + "step": 3840, + "time_per_iteration": 2.862423896789551 + }, + { + "auxiliary_loss_clip": 0.01148141, + "auxiliary_loss_mlp": 0.01045594, + "balance_loss_clip": 1.04776573, + "balance_loss_mlp": 1.02628195, + "epoch": 0.4618529429447484, + "flos": 29570749545600.0, + "grad_norm": 1.9536807488962056, + "language_loss": 0.84932935, + "learning_rate": 2.342607314604533e-06, + "loss": 0.87126672, + "num_input_tokens_seen": 82602185, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.1932373, + "step": 3841, + "time_per_iteration": 3.0021369457244873 + }, + { + "auxiliary_loss_clip": 0.01145218, + "auxiliary_loss_mlp": 0.0104479, + "balance_loss_clip": 1.04666078, + "balance_loss_mlp": 1.02717018, + "epoch": 0.4619731858353875, + "flos": 19787030962560.0, + "grad_norm": 1.7487082699069612, + "language_loss": 0.84171605, + "learning_rate": 2.3418398315316694e-06, + "loss": 0.86361611, + "num_input_tokens_seen": 82620005, + "router_z_loss_clip": 0.98681641, + "router_z_loss_mlp": 0.17626953, + "step": 3842, + "time_per_iteration": 2.78041934967041 + }, + { + "auxiliary_loss_clip": 0.01143674, + "auxiliary_loss_mlp": 0.01045666, + "balance_loss_clip": 1.04613543, + "balance_loss_mlp": 1.02729619, + "epoch": 0.4620934287260266, + "flos": 18951138587520.0, + "grad_norm": 2.804060141107342, + "language_loss": 0.78209907, + "learning_rate": 2.3410722966019755e-06, + "loss": 0.80399245, + "num_input_tokens_seen": 82635120, + "router_z_loss_clip": 0.97509766, + "router_z_loss_mlp": 0.18371582, + "step": 3843, + "time_per_iteration": 2.741818904876709 + }, + { + "auxiliary_loss_clip": 0.01140747, + "auxiliary_loss_mlp": 0.01047883, + "balance_loss_clip": 1.04363191, + "balance_loss_mlp": 1.03034782, + "epoch": 0.46221367161666566, + "flos": 37341674634240.0, + "grad_norm": 2.0449691295379004, + "language_loss": 0.65987694, + "learning_rate": 2.3403047099318848e-06, + "loss": 0.68176323, + "num_input_tokens_seen": 82659190, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.1751709, + "step": 3844, + "time_per_iteration": 2.860865354537964 + }, + { + "auxiliary_loss_clip": 0.0114466, + "auxiliary_loss_mlp": 0.01042732, + "balance_loss_clip": 1.04716039, + "balance_loss_mlp": 1.02584004, + "epoch": 0.46233391450730477, + "flos": 14428549065600.0, + "grad_norm": 2.6272539585194052, + "language_loss": 0.75345147, + "learning_rate": 2.3395370716378405e-06, + "loss": 0.77532542, + "num_input_tokens_seen": 82676635, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.16894531, + "step": 3845, + "time_per_iteration": 2.765519857406616 + }, + { + "auxiliary_loss_clip": 0.01144466, + "auxiliary_loss_mlp": 0.01050203, + "balance_loss_clip": 1.04520905, + "balance_loss_mlp": 1.03225029, + "epoch": 0.4624541573979438, + "flos": 22493044010880.0, + "grad_norm": 2.2586865564021537, + "language_loss": 0.72338593, + "learning_rate": 2.338769381836292e-06, + "loss": 0.7453326, + "num_input_tokens_seen": 82696245, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.17956543, + "step": 3846, + "time_per_iteration": 2.7884669303894043 + }, + { + "auxiliary_loss_clip": 0.01145408, + "auxiliary_loss_mlp": 0.01053096, + "balance_loss_clip": 1.0453018, + "balance_loss_mlp": 1.03390336, + "epoch": 0.46257440028858293, + "flos": 14465070218880.0, + "grad_norm": 1.9692204863170446, + "language_loss": 0.73493314, + "learning_rate": 2.3380016406436984e-06, + "loss": 0.75691819, + "num_input_tokens_seen": 82713725, + "router_z_loss_clip": 1.00048828, + "router_z_loss_mlp": 0.1920166, + "step": 3847, + "time_per_iteration": 2.795746326446533 + }, + { + "auxiliary_loss_clip": 0.0114851, + "auxiliary_loss_mlp": 0.01055455, + "balance_loss_clip": 1.04687548, + "balance_loss_mlp": 1.03573751, + "epoch": 0.46269464317922204, + "flos": 23332204523520.0, + "grad_norm": 1.874681115535585, + "language_loss": 0.81649697, + "learning_rate": 2.337233848176524e-06, + "loss": 0.83853662, + "num_input_tokens_seen": 82731495, + "router_z_loss_clip": 1.01708984, + "router_z_loss_mlp": 0.19702148, + "step": 3848, + "time_per_iteration": 2.8068699836730957 + }, + { + "auxiliary_loss_clip": 0.01147363, + "auxiliary_loss_mlp": 0.01047417, + "balance_loss_clip": 1.04841566, + "balance_loss_mlp": 1.02969074, + "epoch": 0.4628148860698611, + "flos": 18552027594240.0, + "grad_norm": 1.9003393720562007, + "language_loss": 0.8356244, + "learning_rate": 2.3364660045512435e-06, + "loss": 0.8575722, + "num_input_tokens_seen": 82750255, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.17749023, + "step": 3849, + "time_per_iteration": 2.805140495300293 + }, + { + "auxiliary_loss_clip": 0.01043602, + "auxiliary_loss_mlp": 0.01017482, + "balance_loss_clip": 1.01161706, + "balance_loss_mlp": 1.01561022, + "epoch": 0.4629351289605002, + "flos": 70667569670400.0, + "grad_norm": 0.741459549084169, + "language_loss": 0.58199358, + "learning_rate": 2.335698109884337e-06, + "loss": 0.60260439, + "num_input_tokens_seen": 82815460, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.01867676, + "step": 3850, + "time_per_iteration": 3.5400946140289307 + }, + { + "auxiliary_loss_clip": 0.01043458, + "auxiliary_loss_mlp": 0.01013259, + "balance_loss_clip": 1.01143312, + "balance_loss_mlp": 1.01170969, + "epoch": 0.4630553718511393, + "flos": 59687200465920.0, + "grad_norm": 0.7884944249061436, + "language_loss": 0.5985955, + "learning_rate": 2.334930164292294e-06, + "loss": 0.61916268, + "num_input_tokens_seen": 82878010, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.01544189, + "step": 3851, + "time_per_iteration": 3.412550210952759 + }, + { + "auxiliary_loss_clip": 0.0114333, + "auxiliary_loss_mlp": 0.01047723, + "balance_loss_clip": 1.04723418, + "balance_loss_mlp": 1.03074765, + "epoch": 0.4631756147417784, + "flos": 15960605909760.0, + "grad_norm": 2.0690698118171746, + "language_loss": 0.79812264, + "learning_rate": 2.334162167891612e-06, + "loss": 0.82003319, + "num_input_tokens_seen": 82895275, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.1697998, + "step": 3852, + "time_per_iteration": 2.849977970123291 + }, + { + "auxiliary_loss_clip": 0.01146655, + "auxiliary_loss_mlp": 0.01051182, + "balance_loss_clip": 1.04670179, + "balance_loss_mlp": 1.03312147, + "epoch": 0.4632958576324175, + "flos": 16472907636480.0, + "grad_norm": 2.1862600885858914, + "language_loss": 0.75306201, + "learning_rate": 2.333394120798795e-06, + "loss": 0.77504039, + "num_input_tokens_seen": 82914010, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.18054199, + "step": 3853, + "time_per_iteration": 2.8364028930664062 + }, + { + "auxiliary_loss_clip": 0.01140221, + "auxiliary_loss_mlp": 0.01039666, + "balance_loss_clip": 1.04312158, + "balance_loss_mlp": 1.02232122, + "epoch": 0.4634161005230566, + "flos": 22346492520960.0, + "grad_norm": 2.113268230773699, + "language_loss": 0.71705794, + "learning_rate": 2.3326260231303545e-06, + "loss": 0.73885685, + "num_input_tokens_seen": 82932610, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.17346191, + "step": 3854, + "time_per_iteration": 3.552584648132324 + }, + { + "auxiliary_loss_clip": 0.01142892, + "auxiliary_loss_mlp": 0.01040797, + "balance_loss_clip": 1.04712892, + "balance_loss_mlp": 1.02547812, + "epoch": 0.46353634341369565, + "flos": 15742233175680.0, + "grad_norm": 1.600218286475412, + "language_loss": 0.86535466, + "learning_rate": 2.331857875002811e-06, + "loss": 0.88719153, + "num_input_tokens_seen": 82951210, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.15319824, + "step": 3855, + "time_per_iteration": 2.7715296745300293 + }, + { + "auxiliary_loss_clip": 0.01144651, + "auxiliary_loss_mlp": 0.01045421, + "balance_loss_clip": 1.04802334, + "balance_loss_mlp": 1.02721751, + "epoch": 0.46365658630433476, + "flos": 28329820433280.0, + "grad_norm": 1.6129319064288536, + "language_loss": 0.76480329, + "learning_rate": 2.3310896765326916e-06, + "loss": 0.78670406, + "num_input_tokens_seen": 82972210, + "router_z_loss_clip": 0.96679688, + "router_z_loss_mlp": 0.18212891, + "step": 3856, + "time_per_iteration": 3.6418330669403076 + }, + { + "auxiliary_loss_clip": 0.0114684, + "auxiliary_loss_mlp": 0.01043616, + "balance_loss_clip": 1.04717362, + "balance_loss_mlp": 1.02459025, + "epoch": 0.46377682919497387, + "flos": 24608074590720.0, + "grad_norm": 1.5402007900665238, + "language_loss": 0.83953249, + "learning_rate": 2.330321427836531e-06, + "loss": 0.86143702, + "num_input_tokens_seen": 82994080, + "router_z_loss_clip": 0.99609375, + "router_z_loss_mlp": 0.19018555, + "step": 3857, + "time_per_iteration": 2.828720808029175 + }, + { + "auxiliary_loss_clip": 0.01142979, + "auxiliary_loss_mlp": 0.01040854, + "balance_loss_clip": 1.04743457, + "balance_loss_mlp": 1.02305627, + "epoch": 0.4638970720856129, + "flos": 19060953442560.0, + "grad_norm": 2.5088288704300874, + "language_loss": 0.83159733, + "learning_rate": 2.3295531290308733e-06, + "loss": 0.8534357, + "num_input_tokens_seen": 83012230, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.17797852, + "step": 3858, + "time_per_iteration": 2.7446796894073486 + }, + { + "auxiliary_loss_clip": 0.01146871, + "auxiliary_loss_mlp": 0.01045205, + "balance_loss_clip": 1.04800761, + "balance_loss_mlp": 1.02638197, + "epoch": 0.46401731497625204, + "flos": 18471012468480.0, + "grad_norm": 5.559569144129457, + "language_loss": 0.75729585, + "learning_rate": 2.3287847802322678e-06, + "loss": 0.77921659, + "num_input_tokens_seen": 83027800, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.18835449, + "step": 3859, + "time_per_iteration": 2.820150375366211 + }, + { + "auxiliary_loss_clip": 0.01153565, + "auxiliary_loss_mlp": 0.01053634, + "balance_loss_clip": 1.04815686, + "balance_loss_mlp": 1.03414285, + "epoch": 0.4641375578668911, + "flos": 26067053214720.0, + "grad_norm": 1.7156478534254378, + "language_loss": 0.83960468, + "learning_rate": 2.3280163815572723e-06, + "loss": 0.86167669, + "num_input_tokens_seen": 83048395, + "router_z_loss_clip": 1.0546875, + "router_z_loss_mlp": 0.19506836, + "step": 3860, + "time_per_iteration": 3.6032042503356934 + }, + { + "auxiliary_loss_clip": 0.01142978, + "auxiliary_loss_mlp": 0.01044747, + "balance_loss_clip": 1.04459512, + "balance_loss_mlp": 1.02650845, + "epoch": 0.4642578007575302, + "flos": 19570382081280.0, + "grad_norm": 2.7338656518488604, + "language_loss": 0.77131295, + "learning_rate": 2.3272479331224522e-06, + "loss": 0.79319024, + "num_input_tokens_seen": 83065825, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.18225098, + "step": 3861, + "time_per_iteration": 2.736168146133423 + }, + { + "auxiliary_loss_clip": 0.01143011, + "auxiliary_loss_mlp": 0.01041226, + "balance_loss_clip": 1.04461038, + "balance_loss_mlp": 1.02317739, + "epoch": 0.4643780436481693, + "flos": 28186249772160.0, + "grad_norm": 1.6897935147021927, + "language_loss": 0.78141415, + "learning_rate": 2.3264794350443817e-06, + "loss": 0.80325651, + "num_input_tokens_seen": 83087920, + "router_z_loss_clip": 0.98388672, + "router_z_loss_mlp": 0.18029785, + "step": 3862, + "time_per_iteration": 3.6989927291870117 + }, + { + "auxiliary_loss_clip": 0.01147274, + "auxiliary_loss_mlp": 0.01043677, + "balance_loss_clip": 1.04822922, + "balance_loss_mlp": 1.02580762, + "epoch": 0.46449828653880837, + "flos": 25375270204800.0, + "grad_norm": 1.8430552337910409, + "language_loss": 0.788396, + "learning_rate": 2.3257108874396396e-06, + "loss": 0.8103056, + "num_input_tokens_seen": 83109015, + "router_z_loss_clip": 0.99072266, + "router_z_loss_mlp": 0.17871094, + "step": 3863, + "time_per_iteration": 2.7866568565368652 + }, + { + "auxiliary_loss_clip": 0.01145384, + "auxiliary_loss_mlp": 0.01049748, + "balance_loss_clip": 1.04835844, + "balance_loss_mlp": 1.03190267, + "epoch": 0.4646185294294475, + "flos": 16034330574720.0, + "grad_norm": 1.9469884246077973, + "language_loss": 0.73778534, + "learning_rate": 2.3249422904248152e-06, + "loss": 0.75973666, + "num_input_tokens_seen": 83127450, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.17858887, + "step": 3864, + "time_per_iteration": 2.7497105598449707 + }, + { + "auxiliary_loss_clip": 0.01142277, + "auxiliary_loss_mlp": 0.01039682, + "balance_loss_clip": 1.04728937, + "balance_loss_mlp": 1.0233022, + "epoch": 0.4647387723200866, + "flos": 26363101109760.0, + "grad_norm": 1.3940520205752516, + "language_loss": 0.87321627, + "learning_rate": 2.324173644116504e-06, + "loss": 0.89503586, + "num_input_tokens_seen": 83150300, + "router_z_loss_clip": 0.94970703, + "router_z_loss_mlp": 0.16381836, + "step": 3865, + "time_per_iteration": 2.789936065673828 + }, + { + "auxiliary_loss_clip": 0.01148996, + "auxiliary_loss_mlp": 0.01053558, + "balance_loss_clip": 1.04804993, + "balance_loss_mlp": 1.03381729, + "epoch": 0.46485901521072565, + "flos": 27160209774720.0, + "grad_norm": 2.1220001889836317, + "language_loss": 0.81944054, + "learning_rate": 2.3234049486313087e-06, + "loss": 0.84146607, + "num_input_tokens_seen": 83171750, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.1973877, + "step": 3866, + "time_per_iteration": 2.7666780948638916 + }, + { + "auxiliary_loss_clip": 0.01143106, + "auxiliary_loss_mlp": 0.01039824, + "balance_loss_clip": 1.04739976, + "balance_loss_mlp": 1.02363586, + "epoch": 0.46497925810136476, + "flos": 24279851088000.0, + "grad_norm": 1.9042457033769624, + "language_loss": 0.75866979, + "learning_rate": 2.322636204085839e-06, + "loss": 0.78049904, + "num_input_tokens_seen": 83191820, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.1619873, + "step": 3867, + "time_per_iteration": 2.7608635425567627 + }, + { + "auxiliary_loss_clip": 0.0114595, + "auxiliary_loss_mlp": 0.0104416, + "balance_loss_clip": 1.0491159, + "balance_loss_mlp": 1.02695787, + "epoch": 0.46509950099200387, + "flos": 16253134272000.0, + "grad_norm": 2.0710650975417386, + "language_loss": 0.78469652, + "learning_rate": 2.3218674105967143e-06, + "loss": 0.80659759, + "num_input_tokens_seen": 83210085, + "router_z_loss_clip": 0.96826172, + "router_z_loss_mlp": 0.17199707, + "step": 3868, + "time_per_iteration": 2.7157387733459473 + }, + { + "auxiliary_loss_clip": 0.01144112, + "auxiliary_loss_mlp": 0.01040234, + "balance_loss_clip": 1.04552364, + "balance_loss_mlp": 1.0221498, + "epoch": 0.4652197438826429, + "flos": 23442270773760.0, + "grad_norm": 1.4978563692995241, + "language_loss": 0.83686513, + "learning_rate": 2.3210985682805593e-06, + "loss": 0.85870862, + "num_input_tokens_seen": 83231865, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.1809082, + "step": 3869, + "time_per_iteration": 2.802506685256958 + }, + { + "auxiliary_loss_clip": 0.0114826, + "auxiliary_loss_mlp": 0.01036264, + "balance_loss_clip": 1.04881859, + "balance_loss_mlp": 1.0187881, + "epoch": 0.46533998677328203, + "flos": 16216397637120.0, + "grad_norm": 2.5761238177571846, + "language_loss": 0.68213648, + "learning_rate": 2.320329677254007e-06, + "loss": 0.7039817, + "num_input_tokens_seen": 83249195, + "router_z_loss_clip": 0.99609375, + "router_z_loss_mlp": 0.17468262, + "step": 3870, + "time_per_iteration": 2.741270065307617 + }, + { + "auxiliary_loss_clip": 0.01145249, + "auxiliary_loss_mlp": 0.01046995, + "balance_loss_clip": 1.04714119, + "balance_loss_mlp": 1.0277667, + "epoch": 0.46546022966392114, + "flos": 21141869080320.0, + "grad_norm": 3.08894022985412, + "language_loss": 0.72971427, + "learning_rate": 2.319560737633697e-06, + "loss": 0.75163674, + "num_input_tokens_seen": 83267915, + "router_z_loss_clip": 0.98095703, + "router_z_loss_mlp": 0.19250488, + "step": 3871, + "time_per_iteration": 2.7681360244750977 + }, + { + "auxiliary_loss_clip": 0.01151133, + "auxiliary_loss_mlp": 0.01046064, + "balance_loss_clip": 1.04699421, + "balance_loss_mlp": 1.02749085, + "epoch": 0.4655804725545602, + "flos": 41171942442240.0, + "grad_norm": 1.6538818313377694, + "language_loss": 0.6798687, + "learning_rate": 2.3187917495362775e-06, + "loss": 0.70184064, + "num_input_tokens_seen": 83292325, + "router_z_loss_clip": 1.04248047, + "router_z_loss_mlp": 0.18566895, + "step": 3872, + "time_per_iteration": 2.96982741355896 + }, + { + "auxiliary_loss_clip": 0.01145809, + "auxiliary_loss_mlp": 0.01045247, + "balance_loss_clip": 1.04802537, + "balance_loss_mlp": 1.02859354, + "epoch": 0.4657007154451993, + "flos": 19570956698880.0, + "grad_norm": 2.3216982268827238, + "language_loss": 0.76652509, + "learning_rate": 2.318022713078403e-06, + "loss": 0.7884357, + "num_input_tokens_seen": 83306905, + "router_z_loss_clip": 0.9765625, + "router_z_loss_mlp": 0.16662598, + "step": 3873, + "time_per_iteration": 2.8874518871307373 + }, + { + "auxiliary_loss_clip": 0.01143519, + "auxiliary_loss_mlp": 0.01042367, + "balance_loss_clip": 1.04519522, + "balance_loss_mlp": 1.02374625, + "epoch": 0.4658209583358384, + "flos": 15517826956800.0, + "grad_norm": 2.1684272485721574, + "language_loss": 0.85330093, + "learning_rate": 2.3172536283767354e-06, + "loss": 0.8751598, + "num_input_tokens_seen": 83320665, + "router_z_loss_clip": 0.98486328, + "router_z_loss_mlp": 0.18603516, + "step": 3874, + "time_per_iteration": 3.015719175338745 + }, + { + "auxiliary_loss_clip": 0.01146005, + "auxiliary_loss_mlp": 0.01049216, + "balance_loss_clip": 1.0491035, + "balance_loss_mlp": 1.03107214, + "epoch": 0.4659412012264775, + "flos": 14903180403840.0, + "grad_norm": 1.8712199007814847, + "language_loss": 0.81130648, + "learning_rate": 2.3164844955479447e-06, + "loss": 0.83325869, + "num_input_tokens_seen": 83336475, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.18139648, + "step": 3875, + "time_per_iteration": 2.8990001678466797 + }, + { + "auxiliary_loss_clip": 0.01145521, + "auxiliary_loss_mlp": 0.01042381, + "balance_loss_clip": 1.0460515, + "balance_loss_mlp": 1.0245707, + "epoch": 0.4660614441171166, + "flos": 24425612478720.0, + "grad_norm": 1.867578852725877, + "language_loss": 0.70587349, + "learning_rate": 2.3157153147087082e-06, + "loss": 0.72775251, + "num_input_tokens_seen": 83358365, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.17797852, + "step": 3876, + "time_per_iteration": 2.842142343521118 + }, + { + "auxiliary_loss_clip": 0.01148139, + "auxiliary_loss_mlp": 0.01048618, + "balance_loss_clip": 1.04967117, + "balance_loss_mlp": 1.0303905, + "epoch": 0.46618168700775564, + "flos": 22091095843200.0, + "grad_norm": 1.9375443137994788, + "language_loss": 0.83325124, + "learning_rate": 2.314946085975709e-06, + "loss": 0.85521877, + "num_input_tokens_seen": 83377345, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.18225098, + "step": 3877, + "time_per_iteration": 2.7476999759674072 + }, + { + "auxiliary_loss_clip": 0.01144681, + "auxiliary_loss_mlp": 0.01048977, + "balance_loss_clip": 1.04803741, + "balance_loss_mlp": 1.03144169, + "epoch": 0.46630192989839475, + "flos": 26176975810560.0, + "grad_norm": 1.7924551875331916, + "language_loss": 0.82304054, + "learning_rate": 2.3141768094656393e-06, + "loss": 0.8449772, + "num_input_tokens_seen": 83395920, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.17529297, + "step": 3878, + "time_per_iteration": 2.9483327865600586 + }, + { + "auxiliary_loss_clip": 0.01144771, + "auxiliary_loss_mlp": 0.01046323, + "balance_loss_clip": 1.04720891, + "balance_loss_mlp": 1.02856112, + "epoch": 0.46642217278903386, + "flos": 11509622150400.0, + "grad_norm": 2.877336034109289, + "language_loss": 0.83406574, + "learning_rate": 2.3134074852951966e-06, + "loss": 0.85597658, + "num_input_tokens_seen": 83412510, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.1776123, + "step": 3879, + "time_per_iteration": 2.8815112113952637 + }, + { + "auxiliary_loss_clip": 0.01147031, + "auxiliary_loss_mlp": 0.01042736, + "balance_loss_clip": 1.05106604, + "balance_loss_mlp": 1.0259161, + "epoch": 0.4665424156796729, + "flos": 32306819299200.0, + "grad_norm": 1.6330535711565901, + "language_loss": 0.77699, + "learning_rate": 2.312638113581088e-06, + "loss": 0.79888761, + "num_input_tokens_seen": 83432995, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.16809082, + "step": 3880, + "time_per_iteration": 2.971759557723999 + }, + { + "auxiliary_loss_clip": 0.01144359, + "auxiliary_loss_mlp": 0.0104933, + "balance_loss_clip": 1.0471921, + "balance_loss_mlp": 1.0319016, + "epoch": 0.46666265857031203, + "flos": 18436179254400.0, + "grad_norm": 2.55140759138608, + "language_loss": 0.78457463, + "learning_rate": 2.311868694440027e-06, + "loss": 0.80651152, + "num_input_tokens_seen": 83447415, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.17431641, + "step": 3881, + "time_per_iteration": 3.2862396240234375 + }, + { + "auxiliary_loss_clip": 0.01044121, + "auxiliary_loss_mlp": 0.01002367, + "balance_loss_clip": 1.01232862, + "balance_loss_mlp": 1.00064993, + "epoch": 0.46678290146095114, + "flos": 68438989221120.0, + "grad_norm": 0.7410270671173504, + "language_loss": 0.62450397, + "learning_rate": 2.3110992279887323e-06, + "loss": 0.64496887, + "num_input_tokens_seen": 83519340, + "router_z_loss_clip": 0.31835938, + "router_z_loss_mlp": 0.01721191, + "step": 3882, + "time_per_iteration": 5.937715291976929 + }, + { + "auxiliary_loss_clip": 0.0114788, + "auxiliary_loss_mlp": 0.01050095, + "balance_loss_clip": 1.04878402, + "balance_loss_mlp": 1.03131962, + "epoch": 0.4669031443515902, + "flos": 17712507945600.0, + "grad_norm": 5.105794048727674, + "language_loss": 0.85090905, + "learning_rate": 2.310329714343932e-06, + "loss": 0.87288874, + "num_input_tokens_seen": 83535490, + "router_z_loss_clip": 0.99072266, + "router_z_loss_mlp": 0.18762207, + "step": 3883, + "time_per_iteration": 4.119075298309326 + }, + { + "auxiliary_loss_clip": 0.01147112, + "auxiliary_loss_mlp": 0.01042901, + "balance_loss_clip": 1.04944611, + "balance_loss_mlp": 1.0256753, + "epoch": 0.4670233872422293, + "flos": 23947748916480.0, + "grad_norm": 2.176730739983436, + "language_loss": 0.82269049, + "learning_rate": 2.309560153622361e-06, + "loss": 0.84459054, + "num_input_tokens_seen": 83552400, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.17211914, + "step": 3884, + "time_per_iteration": 2.8973236083984375 + }, + { + "auxiliary_loss_clip": 0.01148336, + "auxiliary_loss_mlp": 0.01046836, + "balance_loss_clip": 1.04897797, + "balance_loss_mlp": 1.02883565, + "epoch": 0.4671436301328684, + "flos": 28111268131200.0, + "grad_norm": 2.0399815417722382, + "language_loss": 0.74816489, + "learning_rate": 2.3087905459407602e-06, + "loss": 0.77011657, + "num_input_tokens_seen": 83571340, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.18005371, + "step": 3885, + "time_per_iteration": 4.097559690475464 + }, + { + "auxiliary_loss_clip": 0.01042981, + "auxiliary_loss_mlp": 0.01000418, + "balance_loss_clip": 1.01173413, + "balance_loss_mlp": 0.99880904, + "epoch": 0.46726387302350747, + "flos": 69369684566400.0, + "grad_norm": 0.8098217808885787, + "language_loss": 0.62923324, + "learning_rate": 2.3080208914158795e-06, + "loss": 0.6496672, + "num_input_tokens_seen": 83634340, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01611328, + "step": 3886, + "time_per_iteration": 3.4674813747406006 + }, + { + "auxiliary_loss_clip": 0.01146152, + "auxiliary_loss_mlp": 0.01042077, + "balance_loss_clip": 1.04863763, + "balance_loss_mlp": 1.02445757, + "epoch": 0.4673841159141466, + "flos": 25519666878720.0, + "grad_norm": 2.414822367510611, + "language_loss": 0.7258743, + "learning_rate": 2.3072511901644753e-06, + "loss": 0.7477566, + "num_input_tokens_seen": 83653410, + "router_z_loss_clip": 0.97558594, + "router_z_loss_mlp": 0.17602539, + "step": 3887, + "time_per_iteration": 3.2474231719970703 + }, + { + "auxiliary_loss_clip": 0.01144232, + "auxiliary_loss_mlp": 0.01047162, + "balance_loss_clip": 1.04957974, + "balance_loss_mlp": 1.03055584, + "epoch": 0.4675043588047857, + "flos": 24499265316480.0, + "grad_norm": 3.737556357060655, + "language_loss": 0.81484568, + "learning_rate": 2.306481442303309e-06, + "loss": 0.83675957, + "num_input_tokens_seen": 83672985, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.16601562, + "step": 3888, + "time_per_iteration": 3.0553996562957764 + }, + { + "auxiliary_loss_clip": 0.01143808, + "auxiliary_loss_mlp": 0.01050987, + "balance_loss_clip": 1.04548609, + "balance_loss_mlp": 1.0332253, + "epoch": 0.46762460169542475, + "flos": 20960771685120.0, + "grad_norm": 1.7836894792797027, + "language_loss": 0.73770833, + "learning_rate": 2.3057116479491515e-06, + "loss": 0.75965631, + "num_input_tokens_seen": 83692395, + "router_z_loss_clip": 0.98291016, + "router_z_loss_mlp": 0.17749023, + "step": 3889, + "time_per_iteration": 4.031542778015137 + }, + { + "auxiliary_loss_clip": 0.01143405, + "auxiliary_loss_mlp": 0.01043743, + "balance_loss_clip": 1.04650486, + "balance_loss_mlp": 1.02538443, + "epoch": 0.46774484458606386, + "flos": 19171666137600.0, + "grad_norm": 1.8283403858163096, + "language_loss": 0.75848866, + "learning_rate": 2.30494180721878e-06, + "loss": 0.7803601, + "num_input_tokens_seen": 83709735, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.18359375, + "step": 3890, + "time_per_iteration": 4.440445423126221 + }, + { + "auxiliary_loss_clip": 0.0114143, + "auxiliary_loss_mlp": 0.01049558, + "balance_loss_clip": 1.04708123, + "balance_loss_mlp": 1.03320265, + "epoch": 0.4678650874767029, + "flos": 17967689141760.0, + "grad_norm": 2.921792829693075, + "language_loss": 0.90385729, + "learning_rate": 2.3041719202289794e-06, + "loss": 0.92576718, + "num_input_tokens_seen": 83725910, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.16345215, + "step": 3891, + "time_per_iteration": 5.767359495162964 + }, + { + "auxiliary_loss_clip": 0.01141495, + "auxiliary_loss_mlp": 0.01048272, + "balance_loss_clip": 1.04717362, + "balance_loss_mlp": 1.03178525, + "epoch": 0.467985330367342, + "flos": 21360816432000.0, + "grad_norm": 2.352801139002886, + "language_loss": 0.80621767, + "learning_rate": 2.30340198709654e-06, + "loss": 0.82811534, + "num_input_tokens_seen": 83745745, + "router_z_loss_clip": 0.94238281, + "router_z_loss_mlp": 0.16479492, + "step": 3892, + "time_per_iteration": 2.9873533248901367 + }, + { + "auxiliary_loss_clip": 0.01144665, + "auxiliary_loss_mlp": 0.0105569, + "balance_loss_clip": 1.04820871, + "balance_loss_mlp": 1.03859568, + "epoch": 0.46810557325798113, + "flos": 20521835487360.0, + "grad_norm": 2.058569269938891, + "language_loss": 0.74516898, + "learning_rate": 2.3026320079382605e-06, + "loss": 0.76717257, + "num_input_tokens_seen": 83762680, + "router_z_loss_clip": 0.96289062, + "router_z_loss_mlp": 0.17102051, + "step": 3893, + "time_per_iteration": 2.9909603595733643 + }, + { + "auxiliary_loss_clip": 0.01138321, + "auxiliary_loss_mlp": 0.01041917, + "balance_loss_clip": 1.04455209, + "balance_loss_mlp": 1.02591896, + "epoch": 0.4682258161486202, + "flos": 30117848572800.0, + "grad_norm": 2.17185973273675, + "language_loss": 0.76955259, + "learning_rate": 2.3018619828709454e-06, + "loss": 0.79135489, + "num_input_tokens_seen": 83784220, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.15991211, + "step": 3894, + "time_per_iteration": 3.0205776691436768 + }, + { + "auxiliary_loss_clip": 0.01143541, + "auxiliary_loss_mlp": 0.01048972, + "balance_loss_clip": 1.04652715, + "balance_loss_mlp": 1.03144848, + "epoch": 0.4683460590392593, + "flos": 25293357239040.0, + "grad_norm": 1.8359721596275347, + "language_loss": 0.82371539, + "learning_rate": 2.3010919120114084e-06, + "loss": 0.84564054, + "num_input_tokens_seen": 83800750, + "router_z_loss_clip": 0.96826172, + "router_z_loss_mlp": 0.17529297, + "step": 3895, + "time_per_iteration": 2.910698413848877 + }, + { + "auxiliary_loss_clip": 0.01143144, + "auxiliary_loss_mlp": 0.01046447, + "balance_loss_clip": 1.04446411, + "balance_loss_mlp": 1.02662277, + "epoch": 0.4684663019298984, + "flos": 15368330551680.0, + "grad_norm": 2.198130375475361, + "language_loss": 0.6553489, + "learning_rate": 2.3003217954764672e-06, + "loss": 0.67724478, + "num_input_tokens_seen": 83815455, + "router_z_loss_clip": 0.98779297, + "router_z_loss_mlp": 0.19799805, + "step": 3896, + "time_per_iteration": 3.0045547485351562 + }, + { + "auxiliary_loss_clip": 0.0114286, + "auxiliary_loss_mlp": 0.01052582, + "balance_loss_clip": 1.0447247, + "balance_loss_mlp": 1.03344893, + "epoch": 0.46858654482053747, + "flos": 27778842737280.0, + "grad_norm": 1.6640928816777965, + "language_loss": 0.79384166, + "learning_rate": 2.299551633382949e-06, + "loss": 0.81579602, + "num_input_tokens_seen": 83835765, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.19128418, + "step": 3897, + "time_per_iteration": 3.0236127376556396 + }, + { + "auxiliary_loss_clip": 0.01143315, + "auxiliary_loss_mlp": 0.01047034, + "balance_loss_clip": 1.04585922, + "balance_loss_mlp": 1.02954626, + "epoch": 0.4687067877111766, + "flos": 18040623707520.0, + "grad_norm": 1.8935850560447318, + "language_loss": 0.85996217, + "learning_rate": 2.2987814258476854e-06, + "loss": 0.88186562, + "num_input_tokens_seen": 83853565, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.17480469, + "step": 3898, + "time_per_iteration": 2.861144542694092 + }, + { + "auxiliary_loss_clip": 0.01147876, + "auxiliary_loss_mlp": 0.01044019, + "balance_loss_clip": 1.0462693, + "balance_loss_mlp": 1.02654314, + "epoch": 0.4688270306018157, + "flos": 16977380198400.0, + "grad_norm": 2.3761860304957674, + "language_loss": 0.68074358, + "learning_rate": 2.2980111729875177e-06, + "loss": 0.70266259, + "num_input_tokens_seen": 83869815, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.17468262, + "step": 3899, + "time_per_iteration": 2.8457417488098145 + }, + { + "auxiliary_loss_clip": 0.01141962, + "auxiliary_loss_mlp": 0.0105126, + "balance_loss_clip": 1.04510272, + "balance_loss_mlp": 1.03429675, + "epoch": 0.46894727349245474, + "flos": 17821640442240.0, + "grad_norm": 1.7028558269472043, + "language_loss": 0.82406557, + "learning_rate": 2.2972408749192917e-06, + "loss": 0.84599781, + "num_input_tokens_seen": 83887545, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.1697998, + "step": 3900, + "time_per_iteration": 2.7744104862213135 + }, + { + "auxiliary_loss_clip": 0.01142179, + "auxiliary_loss_mlp": 0.01048794, + "balance_loss_clip": 1.04704642, + "balance_loss_mlp": 1.0314728, + "epoch": 0.46906751638309385, + "flos": 21471349559040.0, + "grad_norm": 1.689448755800631, + "language_loss": 0.66923338, + "learning_rate": 2.296470531759861e-06, + "loss": 0.6911431, + "num_input_tokens_seen": 83905645, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.17333984, + "step": 3901, + "time_per_iteration": 2.8269736766815186 + }, + { + "auxiliary_loss_clip": 0.01144299, + "auxiliary_loss_mlp": 0.01044049, + "balance_loss_clip": 1.04501843, + "balance_loss_mlp": 1.0260725, + "epoch": 0.46918775927373296, + "flos": 20337829090560.0, + "grad_norm": 2.145030712994108, + "language_loss": 0.79238158, + "learning_rate": 2.2957001436260866e-06, + "loss": 0.81426507, + "num_input_tokens_seen": 83922705, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.17993164, + "step": 3902, + "time_per_iteration": 2.703505277633667 + }, + { + "auxiliary_loss_clip": 0.01146547, + "auxiliary_loss_mlp": 0.01044482, + "balance_loss_clip": 1.04871452, + "balance_loss_mlp": 1.02704132, + "epoch": 0.469308002164372, + "flos": 18403249461120.0, + "grad_norm": 1.6249625154120648, + "language_loss": 0.73202264, + "learning_rate": 2.294929710634836e-06, + "loss": 0.75393295, + "num_input_tokens_seen": 83940795, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.17443848, + "step": 3903, + "time_per_iteration": 2.786932945251465 + }, + { + "auxiliary_loss_clip": 0.01145636, + "auxiliary_loss_mlp": 0.01047803, + "balance_loss_clip": 1.04618788, + "balance_loss_mlp": 1.02987349, + "epoch": 0.46942824505501113, + "flos": 37962067363200.0, + "grad_norm": 1.8091392156537274, + "language_loss": 0.61301601, + "learning_rate": 2.2941592329029823e-06, + "loss": 0.6349504, + "num_input_tokens_seen": 83961900, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.17932129, + "step": 3904, + "time_per_iteration": 2.8768022060394287 + }, + { + "auxiliary_loss_clip": 0.01143199, + "auxiliary_loss_mlp": 0.01046844, + "balance_loss_clip": 1.04553866, + "balance_loss_mlp": 1.02904606, + "epoch": 0.46954848794565024, + "flos": 21872507627520.0, + "grad_norm": 1.8252112717970677, + "language_loss": 0.79188967, + "learning_rate": 2.2933887105474067e-06, + "loss": 0.81379008, + "num_input_tokens_seen": 83980075, + "router_z_loss_clip": 0.9765625, + "router_z_loss_mlp": 0.17797852, + "step": 3905, + "time_per_iteration": 2.8419606685638428 + }, + { + "auxiliary_loss_clip": 0.01139733, + "auxiliary_loss_mlp": 0.01042446, + "balance_loss_clip": 1.04661083, + "balance_loss_mlp": 1.02616191, + "epoch": 0.4696687308362893, + "flos": 22016545165440.0, + "grad_norm": 1.635822115863819, + "language_loss": 0.81501985, + "learning_rate": 2.2926181436849974e-06, + "loss": 0.83684164, + "num_input_tokens_seen": 83999430, + "router_z_loss_clip": 0.93017578, + "router_z_loss_mlp": 0.16296387, + "step": 3906, + "time_per_iteration": 2.7626302242279053 + }, + { + "auxiliary_loss_clip": 0.01143537, + "auxiliary_loss_mlp": 0.01046589, + "balance_loss_clip": 1.04617214, + "balance_loss_mlp": 1.02778959, + "epoch": 0.4697889737269284, + "flos": 21613663244160.0, + "grad_norm": 1.6585442352036994, + "language_loss": 0.72844183, + "learning_rate": 2.2918475324326478e-06, + "loss": 0.75034308, + "num_input_tokens_seen": 84019150, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.18786621, + "step": 3907, + "time_per_iteration": 2.775891065597534 + }, + { + "auxiliary_loss_clip": 0.01147323, + "auxiliary_loss_mlp": 0.0105325, + "balance_loss_clip": 1.04676294, + "balance_loss_mlp": 1.03385496, + "epoch": 0.46990921661756746, + "flos": 25228323665280.0, + "grad_norm": 1.9972248721834986, + "language_loss": 0.91511273, + "learning_rate": 2.2910768769072603e-06, + "loss": 0.93711847, + "num_input_tokens_seen": 84037930, + "router_z_loss_clip": 1.00585938, + "router_z_loss_mlp": 0.19384766, + "step": 3908, + "time_per_iteration": 2.852013349533081 + }, + { + "auxiliary_loss_clip": 0.01143339, + "auxiliary_loss_mlp": 0.01043213, + "balance_loss_clip": 1.0474788, + "balance_loss_mlp": 1.02659512, + "epoch": 0.47002945950820657, + "flos": 13844031045120.0, + "grad_norm": 2.012592049739869, + "language_loss": 0.76070809, + "learning_rate": 2.2903061772257417e-06, + "loss": 0.78257358, + "num_input_tokens_seen": 84055915, + "router_z_loss_clip": 0.95849609, + "router_z_loss_mlp": 0.1661377, + "step": 3909, + "time_per_iteration": 2.73793625831604 + }, + { + "auxiliary_loss_clip": 0.01140876, + "auxiliary_loss_mlp": 0.01039701, + "balance_loss_clip": 1.04538572, + "balance_loss_mlp": 1.02210557, + "epoch": 0.4701497023988457, + "flos": 26247001374720.0, + "grad_norm": 1.4028120540746052, + "language_loss": 0.7884596, + "learning_rate": 2.289535433505007e-06, + "loss": 0.81026536, + "num_input_tokens_seen": 84077270, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.17602539, + "step": 3910, + "time_per_iteration": 2.8097474575042725 + }, + { + "auxiliary_loss_clip": 0.01142637, + "auxiliary_loss_mlp": 0.01036146, + "balance_loss_clip": 1.04679012, + "balance_loss_mlp": 1.01923013, + "epoch": 0.47026994528948474, + "flos": 25629517647360.0, + "grad_norm": 1.8739940408307116, + "language_loss": 0.63652974, + "learning_rate": 2.2887646458619767e-06, + "loss": 0.65831757, + "num_input_tokens_seen": 84098635, + "router_z_loss_clip": 0.95849609, + "router_z_loss_mlp": 0.16906738, + "step": 3911, + "time_per_iteration": 2.8514444828033447 + }, + { + "auxiliary_loss_clip": 0.01147843, + "auxiliary_loss_mlp": 0.01047308, + "balance_loss_clip": 1.04830956, + "balance_loss_mlp": 1.02881885, + "epoch": 0.47039018818012385, + "flos": 20554406144640.0, + "grad_norm": 1.7822415302580417, + "language_loss": 0.76761615, + "learning_rate": 2.2879938144135797e-06, + "loss": 0.78956771, + "num_input_tokens_seen": 84114740, + "router_z_loss_clip": 0.99414062, + "router_z_loss_mlp": 0.18505859, + "step": 3912, + "time_per_iteration": 3.5834953784942627 + }, + { + "auxiliary_loss_clip": 0.01142348, + "auxiliary_loss_mlp": 0.01045764, + "balance_loss_clip": 1.04652381, + "balance_loss_mlp": 1.02914631, + "epoch": 0.47051043107076296, + "flos": 21577249831680.0, + "grad_norm": 1.6922571741952057, + "language_loss": 0.75421941, + "learning_rate": 2.2872229392767496e-06, + "loss": 0.77610052, + "num_input_tokens_seen": 84134845, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.1661377, + "step": 3913, + "time_per_iteration": 2.7986176013946533 + }, + { + "auxiliary_loss_clip": 0.01144641, + "auxiliary_loss_mlp": 0.01042689, + "balance_loss_clip": 1.04846907, + "balance_loss_mlp": 1.02558219, + "epoch": 0.470630673961402, + "flos": 18953185662720.0, + "grad_norm": 1.6180707251408768, + "language_loss": 0.74851894, + "learning_rate": 2.286452020568428e-06, + "loss": 0.7703923, + "num_input_tokens_seen": 84152920, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.17114258, + "step": 3914, + "time_per_iteration": 3.833387613296509 + }, + { + "auxiliary_loss_clip": 0.01150107, + "auxiliary_loss_mlp": 0.01051418, + "balance_loss_clip": 1.04730105, + "balance_loss_mlp": 1.03348863, + "epoch": 0.4707509168520411, + "flos": 19938969492480.0, + "grad_norm": 1.829631084440289, + "language_loss": 0.73060721, + "learning_rate": 2.2856810584055637e-06, + "loss": 0.75262249, + "num_input_tokens_seen": 84170455, + "router_z_loss_clip": 1.02929688, + "router_z_loss_mlp": 0.17919922, + "step": 3915, + "time_per_iteration": 2.890778064727783 + }, + { + "auxiliary_loss_clip": 0.01143555, + "auxiliary_loss_mlp": 0.01044569, + "balance_loss_clip": 1.04600477, + "balance_loss_mlp": 1.02646136, + "epoch": 0.47087115974268023, + "flos": 40118754741120.0, + "grad_norm": 1.583346460398666, + "language_loss": 0.68173552, + "learning_rate": 2.2849100529051085e-06, + "loss": 0.7036168, + "num_input_tokens_seen": 84197390, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.18115234, + "step": 3916, + "time_per_iteration": 2.9661550521850586 + }, + { + "auxiliary_loss_clip": 0.01144154, + "auxiliary_loss_mlp": 0.01045125, + "balance_loss_clip": 1.04884934, + "balance_loss_mlp": 1.02783918, + "epoch": 0.4709914026333193, + "flos": 13552723745280.0, + "grad_norm": 2.4073356684546834, + "language_loss": 0.80045706, + "learning_rate": 2.284139004184026e-06, + "loss": 0.82234979, + "num_input_tokens_seen": 84214620, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.17285156, + "step": 3917, + "time_per_iteration": 2.904863119125366 + }, + { + "auxiliary_loss_clip": 0.01145046, + "auxiliary_loss_mlp": 0.01044988, + "balance_loss_clip": 1.04667711, + "balance_loss_mlp": 1.02665341, + "epoch": 0.4711116455239584, + "flos": 19974628719360.0, + "grad_norm": 2.0937418869177282, + "language_loss": 0.74156332, + "learning_rate": 2.2833679123592814e-06, + "loss": 0.76346362, + "num_input_tokens_seen": 84231880, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.18359375, + "step": 3918, + "time_per_iteration": 3.9823243618011475 + }, + { + "auxiliary_loss_clip": 0.01147098, + "auxiliary_loss_mlp": 0.01045194, + "balance_loss_clip": 1.04921246, + "balance_loss_mlp": 1.02751577, + "epoch": 0.4712318884145975, + "flos": 32124824064000.0, + "grad_norm": 1.699224376093902, + "language_loss": 0.63588881, + "learning_rate": 2.2825967775478508e-06, + "loss": 0.65781176, + "num_input_tokens_seen": 84252980, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.17675781, + "step": 3919, + "time_per_iteration": 3.7560133934020996 + }, + { + "auxiliary_loss_clip": 0.01142739, + "auxiliary_loss_mlp": 0.01043838, + "balance_loss_clip": 1.04491436, + "balance_loss_mlp": 1.02514613, + "epoch": 0.47135213130523657, + "flos": 20047850593920.0, + "grad_norm": 2.1611873625808116, + "language_loss": 0.83673334, + "learning_rate": 2.2818255998667135e-06, + "loss": 0.85859907, + "num_input_tokens_seen": 84271490, + "router_z_loss_clip": 0.97705078, + "router_z_loss_mlp": 0.18676758, + "step": 3920, + "time_per_iteration": 2.791175365447998 + }, + { + "auxiliary_loss_clip": 0.01143881, + "auxiliary_loss_mlp": 0.01040424, + "balance_loss_clip": 1.04666495, + "balance_loss_mlp": 1.0238539, + "epoch": 0.4714723741958757, + "flos": 19426990988160.0, + "grad_norm": 1.5738442689629177, + "language_loss": 0.79396474, + "learning_rate": 2.2810543794328566e-06, + "loss": 0.81580782, + "num_input_tokens_seen": 84290525, + "router_z_loss_clip": 0.97119141, + "router_z_loss_mlp": 0.16577148, + "step": 3921, + "time_per_iteration": 2.8764753341674805 + }, + { + "auxiliary_loss_clip": 0.01146981, + "auxiliary_loss_mlp": 0.01048776, + "balance_loss_clip": 1.04794502, + "balance_loss_mlp": 1.03033447, + "epoch": 0.4715926170865148, + "flos": 20373883367040.0, + "grad_norm": 1.8062490228301062, + "language_loss": 0.82045853, + "learning_rate": 2.2802831163632735e-06, + "loss": 0.84241617, + "num_input_tokens_seen": 84309245, + "router_z_loss_clip": 0.99072266, + "router_z_loss_mlp": 0.1842041, + "step": 3922, + "time_per_iteration": 2.7675294876098633 + }, + { + "auxiliary_loss_clip": 0.01150515, + "auxiliary_loss_mlp": 0.01047816, + "balance_loss_clip": 1.04896832, + "balance_loss_mlp": 1.02796781, + "epoch": 0.47171285997715384, + "flos": 22672884430080.0, + "grad_norm": 1.8517507979390686, + "language_loss": 0.74516326, + "learning_rate": 2.279511810774965e-06, + "loss": 0.76714659, + "num_input_tokens_seen": 84330775, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.19836426, + "step": 3923, + "time_per_iteration": 2.72759747505188 + }, + { + "auxiliary_loss_clip": 0.01144323, + "auxiliary_loss_mlp": 0.01046189, + "balance_loss_clip": 1.04716468, + "balance_loss_mlp": 1.02829599, + "epoch": 0.47183310286779295, + "flos": 21105419754240.0, + "grad_norm": 1.8665433807382428, + "language_loss": 0.71429658, + "learning_rate": 2.2787404627849364e-06, + "loss": 0.73620176, + "num_input_tokens_seen": 84349985, + "router_z_loss_clip": 0.97021484, + "router_z_loss_mlp": 0.17895508, + "step": 3924, + "time_per_iteration": 2.828887462615967 + }, + { + "auxiliary_loss_clip": 0.01145517, + "auxiliary_loss_mlp": 0.01040035, + "balance_loss_clip": 1.04746199, + "balance_loss_mlp": 1.02205849, + "epoch": 0.471953345758432, + "flos": 21726566668800.0, + "grad_norm": 1.6965173681718868, + "language_loss": 0.78764641, + "learning_rate": 2.277969072510202e-06, + "loss": 0.80950189, + "num_input_tokens_seen": 84368965, + "router_z_loss_clip": 0.97998047, + "router_z_loss_mlp": 0.1796875, + "step": 3925, + "time_per_iteration": 2.772273063659668 + }, + { + "auxiliary_loss_clip": 0.01142153, + "auxiliary_loss_mlp": 0.01046359, + "balance_loss_clip": 1.04770076, + "balance_loss_mlp": 1.02975273, + "epoch": 0.4720735886490711, + "flos": 19861078849920.0, + "grad_norm": 1.7751918657859962, + "language_loss": 0.81293434, + "learning_rate": 2.2771976400677803e-06, + "loss": 0.83481944, + "num_input_tokens_seen": 84387795, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.16601562, + "step": 3926, + "time_per_iteration": 2.780669689178467 + }, + { + "auxiliary_loss_clip": 0.01142367, + "auxiliary_loss_mlp": 0.01036701, + "balance_loss_clip": 1.04697001, + "balance_loss_mlp": 1.02013087, + "epoch": 0.47219383153971023, + "flos": 19171809792000.0, + "grad_norm": 1.7267684662901412, + "language_loss": 0.78874856, + "learning_rate": 2.2764261655746965e-06, + "loss": 0.81053925, + "num_input_tokens_seen": 84405290, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.16564941, + "step": 3927, + "time_per_iteration": 2.98571515083313 + }, + { + "auxiliary_loss_clip": 0.01146334, + "auxiliary_loss_mlp": 0.01044618, + "balance_loss_clip": 1.04786539, + "balance_loss_mlp": 1.02649832, + "epoch": 0.4723140744303493, + "flos": 23224005780480.0, + "grad_norm": 1.65859983409246, + "language_loss": 0.75604963, + "learning_rate": 2.2756546491479832e-06, + "loss": 0.77795911, + "num_input_tokens_seen": 84426205, + "router_z_loss_clip": 0.98388672, + "router_z_loss_mlp": 0.18115234, + "step": 3928, + "time_per_iteration": 3.0110554695129395 + }, + { + "auxiliary_loss_clip": 0.01146381, + "auxiliary_loss_mlp": 0.01041736, + "balance_loss_clip": 1.04693747, + "balance_loss_mlp": 1.02405763, + "epoch": 0.4724343173209884, + "flos": 18223265387520.0, + "grad_norm": 2.5830926649061703, + "language_loss": 0.79712653, + "learning_rate": 2.274883090904679e-06, + "loss": 0.81900775, + "num_input_tokens_seen": 84443970, + "router_z_loss_clip": 0.99511719, + "router_z_loss_mlp": 0.17675781, + "step": 3929, + "time_per_iteration": 2.9140210151672363 + }, + { + "auxiliary_loss_clip": 0.0114724, + "auxiliary_loss_mlp": 0.0104609, + "balance_loss_clip": 1.04891253, + "balance_loss_mlp": 1.02823246, + "epoch": 0.4725545602116275, + "flos": 21251037490560.0, + "grad_norm": 2.4918208816406198, + "language_loss": 0.67424226, + "learning_rate": 2.2741114909618283e-06, + "loss": 0.69617552, + "num_input_tokens_seen": 84459865, + "router_z_loss_clip": 0.98388672, + "router_z_loss_mlp": 0.17858887, + "step": 3930, + "time_per_iteration": 3.330570936203003 + }, + { + "auxiliary_loss_clip": 0.01144144, + "auxiliary_loss_mlp": 0.01039358, + "balance_loss_clip": 1.04789591, + "balance_loss_mlp": 1.02262139, + "epoch": 0.47267480310226656, + "flos": 21434002392960.0, + "grad_norm": 1.9063750455508481, + "language_loss": 0.72194034, + "learning_rate": 2.2733398494364828e-06, + "loss": 0.74377537, + "num_input_tokens_seen": 84479110, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.1673584, + "step": 3931, + "time_per_iteration": 5.266872406005859 + }, + { + "auxiliary_loss_clip": 0.01142153, + "auxiliary_loss_mlp": 0.01040649, + "balance_loss_clip": 1.04642248, + "balance_loss_mlp": 1.02391243, + "epoch": 0.47279504599290567, + "flos": 18770508069120.0, + "grad_norm": 1.9053159609417998, + "language_loss": 0.84172446, + "learning_rate": 2.272568166445699e-06, + "loss": 0.86355245, + "num_input_tokens_seen": 84497675, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.1673584, + "step": 3932, + "time_per_iteration": 3.6893880367279053 + }, + { + "auxiliary_loss_clip": 0.01144797, + "auxiliary_loss_mlp": 0.01050602, + "balance_loss_clip": 1.04907751, + "balance_loss_mlp": 1.03229141, + "epoch": 0.4729152888835448, + "flos": 21105742976640.0, + "grad_norm": 2.133052878515923, + "language_loss": 0.64309496, + "learning_rate": 2.271796442106541e-06, + "loss": 0.66504896, + "num_input_tokens_seen": 84517030, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.1829834, + "step": 3933, + "time_per_iteration": 2.8276796340942383 + }, + { + "auxiliary_loss_clip": 0.01051353, + "auxiliary_loss_mlp": 0.01007711, + "balance_loss_clip": 1.01959705, + "balance_loss_mlp": 1.00601792, + "epoch": 0.47303553177418384, + "flos": 70201877840640.0, + "grad_norm": 0.8084014570283139, + "language_loss": 0.56560338, + "learning_rate": 2.271024676536079e-06, + "loss": 0.58619404, + "num_input_tokens_seen": 84577290, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01696777, + "step": 3934, + "time_per_iteration": 3.281643867492676 + }, + { + "auxiliary_loss_clip": 0.01152579, + "auxiliary_loss_mlp": 0.0105339, + "balance_loss_clip": 1.05187225, + "balance_loss_mlp": 1.03461504, + "epoch": 0.47315577466482295, + "flos": 22455122227200.0, + "grad_norm": 2.0092218625559584, + "language_loss": 0.7301783, + "learning_rate": 2.2702528698513894e-06, + "loss": 0.75223798, + "num_input_tokens_seen": 84598415, + "router_z_loss_clip": 1.0078125, + "router_z_loss_mlp": 0.18762207, + "step": 3935, + "time_per_iteration": 2.8642454147338867 + }, + { + "auxiliary_loss_clip": 0.01145715, + "auxiliary_loss_mlp": 0.01045868, + "balance_loss_clip": 1.04549122, + "balance_loss_mlp": 1.02649689, + "epoch": 0.47327601755546206, + "flos": 24352857480960.0, + "grad_norm": 1.8783548254695723, + "language_loss": 0.78584194, + "learning_rate": 2.269481022169554e-06, + "loss": 0.80775774, + "num_input_tokens_seen": 84617010, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.19372559, + "step": 3936, + "time_per_iteration": 2.8996541500091553 + }, + { + "auxiliary_loss_clip": 0.01147175, + "auxiliary_loss_mlp": 0.01044229, + "balance_loss_clip": 1.04744291, + "balance_loss_mlp": 1.0248332, + "epoch": 0.4733962604461011, + "flos": 22926772736640.0, + "grad_norm": 1.7480841386704626, + "language_loss": 0.80379933, + "learning_rate": 2.2687091336076614e-06, + "loss": 0.82571334, + "num_input_tokens_seen": 84636350, + "router_z_loss_clip": 0.99853516, + "router_z_loss_mlp": 0.19396973, + "step": 3937, + "time_per_iteration": 2.869170904159546 + }, + { + "auxiliary_loss_clip": 0.01144064, + "auxiliary_loss_mlp": 0.01040504, + "balance_loss_clip": 1.0478431, + "balance_loss_mlp": 1.02299237, + "epoch": 0.4735165033367402, + "flos": 18327369980160.0, + "grad_norm": 2.3237791707791198, + "language_loss": 0.79893577, + "learning_rate": 2.267937204282807e-06, + "loss": 0.82078147, + "num_input_tokens_seen": 84653490, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.17504883, + "step": 3938, + "time_per_iteration": 2.804755926132202 + }, + { + "auxiliary_loss_clip": 0.01153322, + "auxiliary_loss_mlp": 0.01046343, + "balance_loss_clip": 1.04994416, + "balance_loss_mlp": 1.02693534, + "epoch": 0.4736367462273793, + "flos": 23037018554880.0, + "grad_norm": 3.0021165888999324, + "language_loss": 0.78940564, + "learning_rate": 2.2671652343120926e-06, + "loss": 0.81140232, + "num_input_tokens_seen": 84673965, + "router_z_loss_clip": 1.03271484, + "router_z_loss_mlp": 0.1940918, + "step": 3939, + "time_per_iteration": 2.8070068359375 + }, + { + "auxiliary_loss_clip": 0.0114461, + "auxiliary_loss_mlp": 0.01044084, + "balance_loss_clip": 1.04788446, + "balance_loss_mlp": 1.02665544, + "epoch": 0.4737569891180184, + "flos": 25374336451200.0, + "grad_norm": 1.7383143937643943, + "language_loss": 0.80781859, + "learning_rate": 2.2663932238126236e-06, + "loss": 0.82970554, + "num_input_tokens_seen": 84692525, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.17431641, + "step": 3940, + "time_per_iteration": 4.359499931335449 + }, + { + "auxiliary_loss_clip": 0.01145006, + "auxiliary_loss_mlp": 0.01041573, + "balance_loss_clip": 1.04611158, + "balance_loss_mlp": 1.0237987, + "epoch": 0.4738772320086575, + "flos": 25849326925440.0, + "grad_norm": 1.4026845546948479, + "language_loss": 0.80392754, + "learning_rate": 2.265621172901515e-06, + "loss": 0.82579333, + "num_input_tokens_seen": 84715640, + "router_z_loss_clip": 0.98779297, + "router_z_loss_mlp": 0.1776123, + "step": 3941, + "time_per_iteration": 5.740253686904907 + }, + { + "auxiliary_loss_clip": 0.01149643, + "auxiliary_loss_mlp": 0.01050211, + "balance_loss_clip": 1.04995823, + "balance_loss_mlp": 1.03170991, + "epoch": 0.47399747489929656, + "flos": 27564420499200.0, + "grad_norm": 2.0077644631846505, + "language_loss": 0.71115863, + "learning_rate": 2.2648490816958854e-06, + "loss": 0.73315716, + "num_input_tokens_seen": 84736635, + "router_z_loss_clip": 0.99804688, + "router_z_loss_mlp": 0.18505859, + "step": 3942, + "time_per_iteration": 2.9819107055664062 + }, + { + "auxiliary_loss_clip": 0.01146307, + "auxiliary_loss_mlp": 0.01049829, + "balance_loss_clip": 1.04603887, + "balance_loss_mlp": 1.02993286, + "epoch": 0.47411771778993567, + "flos": 24863650836480.0, + "grad_norm": 3.998814010927575, + "language_loss": 0.73674774, + "learning_rate": 2.264076950312861e-06, + "loss": 0.75870907, + "num_input_tokens_seen": 84755445, + "router_z_loss_clip": 1.00292969, + "router_z_loss_mlp": 0.19897461, + "step": 3943, + "time_per_iteration": 3.7642431259155273 + }, + { + "auxiliary_loss_clip": 0.01148031, + "auxiliary_loss_mlp": 0.01045637, + "balance_loss_clip": 1.04528403, + "balance_loss_mlp": 1.02702856, + "epoch": 0.4742379606805748, + "flos": 22748009725440.0, + "grad_norm": 1.9462226906329412, + "language_loss": 0.81914145, + "learning_rate": 2.2633047788695727e-06, + "loss": 0.84107816, + "num_input_tokens_seen": 84775750, + "router_z_loss_clip": 1.02783203, + "router_z_loss_mlp": 0.18591309, + "step": 3944, + "time_per_iteration": 2.812069892883301 + }, + { + "auxiliary_loss_clip": 0.01138984, + "auxiliary_loss_mlp": 0.01043298, + "balance_loss_clip": 1.04511511, + "balance_loss_mlp": 1.02571464, + "epoch": 0.47435820357121383, + "flos": 19681130689920.0, + "grad_norm": 1.7397398170360272, + "language_loss": 0.64374793, + "learning_rate": 2.262532567483159e-06, + "loss": 0.66557074, + "num_input_tokens_seen": 84794310, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.17578125, + "step": 3945, + "time_per_iteration": 2.8225438594818115 + }, + { + "auxiliary_loss_clip": 0.0114514, + "auxiliary_loss_mlp": 0.01041085, + "balance_loss_clip": 1.04719949, + "balance_loss_mlp": 1.02291739, + "epoch": 0.47447844646185294, + "flos": 25228718714880.0, + "grad_norm": 1.9344893593320907, + "language_loss": 0.80570376, + "learning_rate": 2.2617603162707635e-06, + "loss": 0.82756603, + "num_input_tokens_seen": 84814720, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.1817627, + "step": 3946, + "time_per_iteration": 3.592487096786499 + }, + { + "auxiliary_loss_clip": 0.0114116, + "auxiliary_loss_mlp": 0.01047942, + "balance_loss_clip": 1.04550624, + "balance_loss_mlp": 1.03058481, + "epoch": 0.47459868935249205, + "flos": 24570619683840.0, + "grad_norm": 1.7162788923062973, + "language_loss": 0.82962966, + "learning_rate": 2.2609880253495363e-06, + "loss": 0.85152066, + "num_input_tokens_seen": 84834355, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.17333984, + "step": 3947, + "time_per_iteration": 2.837602376937866 + }, + { + "auxiliary_loss_clip": 0.01144652, + "auxiliary_loss_mlp": 0.01045076, + "balance_loss_clip": 1.04841709, + "balance_loss_mlp": 1.02643132, + "epoch": 0.4747189322431311, + "flos": 20558500295040.0, + "grad_norm": 1.7601581800083301, + "language_loss": 0.86496121, + "learning_rate": 2.260215694836633e-06, + "loss": 0.88685846, + "num_input_tokens_seen": 84853530, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.1862793, + "step": 3948, + "time_per_iteration": 3.72823429107666 + }, + { + "auxiliary_loss_clip": 0.01143866, + "auxiliary_loss_mlp": 0.01042407, + "balance_loss_clip": 1.04688549, + "balance_loss_mlp": 1.02490675, + "epoch": 0.4748391751337702, + "flos": 25995231970560.0, + "grad_norm": 2.892554970008226, + "language_loss": 0.64857966, + "learning_rate": 2.2594433248492157e-06, + "loss": 0.67044234, + "num_input_tokens_seen": 84872505, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.17504883, + "step": 3949, + "time_per_iteration": 2.792318344116211 + }, + { + "auxiliary_loss_clip": 0.01145412, + "auxiliary_loss_mlp": 0.01044112, + "balance_loss_clip": 1.04644167, + "balance_loss_mlp": 1.02639794, + "epoch": 0.47495941802440933, + "flos": 22821052032000.0, + "grad_norm": 5.971381383481204, + "language_loss": 0.8006832, + "learning_rate": 2.2586709155044527e-06, + "loss": 0.82257843, + "num_input_tokens_seen": 84893105, + "router_z_loss_clip": 0.98974609, + "router_z_loss_mlp": 0.17712402, + "step": 3950, + "time_per_iteration": 2.8513400554656982 + }, + { + "auxiliary_loss_clip": 0.01146299, + "auxiliary_loss_mlp": 0.01046856, + "balance_loss_clip": 1.0472554, + "balance_loss_mlp": 1.02892745, + "epoch": 0.4750796609150484, + "flos": 27891782075520.0, + "grad_norm": 1.5048243167765845, + "language_loss": 0.76185322, + "learning_rate": 2.2578984669195167e-06, + "loss": 0.78378481, + "num_input_tokens_seen": 84914070, + "router_z_loss_clip": 0.99121094, + "router_z_loss_mlp": 0.17907715, + "step": 3951, + "time_per_iteration": 2.811321496963501 + }, + { + "auxiliary_loss_clip": 0.01141785, + "auxiliary_loss_mlp": 0.01042706, + "balance_loss_clip": 1.04708242, + "balance_loss_mlp": 1.02569485, + "epoch": 0.4751999038056875, + "flos": 35660085471360.0, + "grad_norm": 1.8940457320573467, + "language_loss": 0.67826724, + "learning_rate": 2.2571259792115887e-06, + "loss": 0.7001121, + "num_input_tokens_seen": 84935290, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.17004395, + "step": 3952, + "time_per_iteration": 2.852480411529541 + }, + { + "auxiliary_loss_clip": 0.01141125, + "auxiliary_loss_mlp": 0.0104322, + "balance_loss_clip": 1.04719758, + "balance_loss_mlp": 1.02657855, + "epoch": 0.4753201466963266, + "flos": 22090880361600.0, + "grad_norm": 1.944967712545687, + "language_loss": 0.79477799, + "learning_rate": 2.2563534524978544e-06, + "loss": 0.81662142, + "num_input_tokens_seen": 84952760, + "router_z_loss_clip": 0.93798828, + "router_z_loss_mlp": 0.16650391, + "step": 3953, + "time_per_iteration": 2.7222092151641846 + }, + { + "auxiliary_loss_clip": 0.01138859, + "auxiliary_loss_mlp": 0.0104438, + "balance_loss_clip": 1.0447582, + "balance_loss_mlp": 1.0265224, + "epoch": 0.47544038958696566, + "flos": 30190854965760.0, + "grad_norm": 1.7032058873513782, + "language_loss": 0.70620281, + "learning_rate": 2.2555808868955052e-06, + "loss": 0.72803521, + "num_input_tokens_seen": 84974890, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.17858887, + "step": 3954, + "time_per_iteration": 2.8147480487823486 + }, + { + "auxiliary_loss_clip": 0.01146544, + "auxiliary_loss_mlp": 0.01051736, + "balance_loss_clip": 1.04730237, + "balance_loss_mlp": 1.03286469, + "epoch": 0.47556063247760477, + "flos": 23472219738240.0, + "grad_norm": 3.1307956658697442, + "language_loss": 0.73521364, + "learning_rate": 2.254808282521738e-06, + "loss": 0.75719643, + "num_input_tokens_seen": 84993640, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.18884277, + "step": 3955, + "time_per_iteration": 2.936657667160034 + }, + { + "auxiliary_loss_clip": 0.01143706, + "auxiliary_loss_mlp": 0.01044591, + "balance_loss_clip": 1.04825187, + "balance_loss_mlp": 1.02645898, + "epoch": 0.4756808753682438, + "flos": 25155209531520.0, + "grad_norm": 1.9444570786974813, + "language_loss": 0.81354386, + "learning_rate": 2.2540356394937573e-06, + "loss": 0.83542681, + "num_input_tokens_seen": 85012340, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.18127441, + "step": 3956, + "time_per_iteration": 3.066904306411743 + }, + { + "auxiliary_loss_clip": 0.01148034, + "auxiliary_loss_mlp": 0.010462, + "balance_loss_clip": 1.04828501, + "balance_loss_mlp": 1.02692378, + "epoch": 0.47580111825888294, + "flos": 15669729573120.0, + "grad_norm": 2.0319389441564297, + "language_loss": 0.84060889, + "learning_rate": 2.253262957928772e-06, + "loss": 0.86255121, + "num_input_tokens_seen": 85029225, + "router_z_loss_clip": 0.99853516, + "router_z_loss_mlp": 0.19287109, + "step": 3957, + "time_per_iteration": 2.779871940612793 + }, + { + "auxiliary_loss_clip": 0.01145314, + "auxiliary_loss_mlp": 0.01040854, + "balance_loss_clip": 1.04745507, + "balance_loss_mlp": 1.02280617, + "epoch": 0.47592136114952205, + "flos": 17636556637440.0, + "grad_norm": 1.6150139527658214, + "language_loss": 0.72197318, + "learning_rate": 2.2524902379439976e-06, + "loss": 0.74383485, + "num_input_tokens_seen": 85047895, + "router_z_loss_clip": 0.97851562, + "router_z_loss_mlp": 0.18029785, + "step": 3958, + "time_per_iteration": 2.7644758224487305 + }, + { + "auxiliary_loss_clip": 0.01046648, + "auxiliary_loss_mlp": 0.01011078, + "balance_loss_clip": 1.01487982, + "balance_loss_mlp": 1.00921845, + "epoch": 0.4760416040401611, + "flos": 61417159292160.0, + "grad_norm": 0.8014012378792157, + "language_loss": 0.6364603, + "learning_rate": 2.251717479656655e-06, + "loss": 0.65703756, + "num_input_tokens_seen": 85112690, + "router_z_loss_clip": 0.31835938, + "router_z_loss_mlp": 0.01855469, + "step": 3959, + "time_per_iteration": 3.3095221519470215 + }, + { + "auxiliary_loss_clip": 0.01144797, + "auxiliary_loss_mlp": 0.01046301, + "balance_loss_clip": 1.04583907, + "balance_loss_mlp": 1.02867031, + "epoch": 0.4761618469308002, + "flos": 18405871153920.0, + "grad_norm": 1.794654157827839, + "language_loss": 0.76264966, + "learning_rate": 2.2509446831839704e-06, + "loss": 0.78456062, + "num_input_tokens_seen": 85132130, + "router_z_loss_clip": 0.98925781, + "router_z_loss_mlp": 0.17626953, + "step": 3960, + "time_per_iteration": 2.766965866088867 + }, + { + "auxiliary_loss_clip": 0.01145366, + "auxiliary_loss_mlp": 0.01045436, + "balance_loss_clip": 1.04607582, + "balance_loss_mlp": 1.02704167, + "epoch": 0.4762820898214393, + "flos": 18040911016320.0, + "grad_norm": 2.2007019498654516, + "language_loss": 0.82141352, + "learning_rate": 2.250171848643177e-06, + "loss": 0.8433215, + "num_input_tokens_seen": 85149420, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.18383789, + "step": 3961, + "time_per_iteration": 2.7263758182525635 + }, + { + "auxiliary_loss_clip": 0.01139452, + "auxiliary_loss_mlp": 0.01042333, + "balance_loss_clip": 1.04450607, + "balance_loss_mlp": 1.02484548, + "epoch": 0.4764023327120784, + "flos": 19318253541120.0, + "grad_norm": 1.7619589426140074, + "language_loss": 0.8598125, + "learning_rate": 2.249398976151513e-06, + "loss": 0.88163036, + "num_input_tokens_seen": 85166970, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.17492676, + "step": 3962, + "time_per_iteration": 2.8292689323425293 + }, + { + "auxiliary_loss_clip": 0.0113894, + "auxiliary_loss_mlp": 0.0104188, + "balance_loss_clip": 1.0434972, + "balance_loss_mlp": 1.02519083, + "epoch": 0.4765225756027175, + "flos": 22747255539840.0, + "grad_norm": 2.1612991033091653, + "language_loss": 0.78750992, + "learning_rate": 2.248626065826223e-06, + "loss": 0.80931818, + "num_input_tokens_seen": 85185175, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.16699219, + "step": 3963, + "time_per_iteration": 2.759223461151123 + }, + { + "auxiliary_loss_clip": 0.01045157, + "auxiliary_loss_mlp": 0.01004127, + "balance_loss_clip": 1.01355827, + "balance_loss_mlp": 1.00252974, + "epoch": 0.4766428184933566, + "flos": 65933392106880.0, + "grad_norm": 0.7626027844450071, + "language_loss": 0.62568438, + "learning_rate": 2.2478531177845564e-06, + "loss": 0.64617729, + "num_input_tokens_seen": 85246170, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.01599121, + "step": 3964, + "time_per_iteration": 3.281100034713745 + }, + { + "auxiliary_loss_clip": 0.01145605, + "auxiliary_loss_mlp": 0.01048582, + "balance_loss_clip": 1.04718912, + "balance_loss_mlp": 1.03223825, + "epoch": 0.47676306138399566, + "flos": 24136495908480.0, + "grad_norm": 1.7866253157124294, + "language_loss": 0.85020268, + "learning_rate": 2.247080132143769e-06, + "loss": 0.87214452, + "num_input_tokens_seen": 85268525, + "router_z_loss_clip": 0.98388672, + "router_z_loss_mlp": 0.16345215, + "step": 3965, + "time_per_iteration": 2.750849962234497 + }, + { + "auxiliary_loss_clip": 0.01144842, + "auxiliary_loss_mlp": 0.01051978, + "balance_loss_clip": 1.04390752, + "balance_loss_mlp": 1.03336966, + "epoch": 0.47688330427463477, + "flos": 12604322995200.0, + "grad_norm": 2.2710424263215754, + "language_loss": 0.69218099, + "learning_rate": 2.246307109021121e-06, + "loss": 0.71414924, + "num_input_tokens_seen": 85285930, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.18603516, + "step": 3966, + "time_per_iteration": 2.747091770172119 + }, + { + "auxiliary_loss_clip": 0.01139755, + "auxiliary_loss_mlp": 0.01041268, + "balance_loss_clip": 1.04554784, + "balance_loss_mlp": 1.02428102, + "epoch": 0.4770035471652739, + "flos": 21390585828480.0, + "grad_norm": 1.6568349793060613, + "language_loss": 0.82469958, + "learning_rate": 2.2455340485338817e-06, + "loss": 0.84650981, + "num_input_tokens_seen": 85303565, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.16992188, + "step": 3967, + "time_per_iteration": 2.7981832027435303 + }, + { + "auxiliary_loss_clip": 0.01142811, + "auxiliary_loss_mlp": 0.01043657, + "balance_loss_clip": 1.0476706, + "balance_loss_mlp": 1.02609718, + "epoch": 0.47712379005591293, + "flos": 25156251025920.0, + "grad_norm": 2.4388195238906354, + "language_loss": 0.67926788, + "learning_rate": 2.244760950799322e-06, + "loss": 0.70113254, + "num_input_tokens_seen": 85321835, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.17565918, + "step": 3968, + "time_per_iteration": 2.7870121002197266 + }, + { + "auxiliary_loss_clip": 0.01140562, + "auxiliary_loss_mlp": 0.01040666, + "balance_loss_clip": 1.04598784, + "balance_loss_mlp": 1.0236665, + "epoch": 0.47724403294655204, + "flos": 22054323294720.0, + "grad_norm": 2.1904254743861444, + "language_loss": 0.72327965, + "learning_rate": 2.2439878159347203e-06, + "loss": 0.74509192, + "num_input_tokens_seen": 85341260, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.17004395, + "step": 3969, + "time_per_iteration": 3.5996909141540527 + }, + { + "auxiliary_loss_clip": 0.01045324, + "auxiliary_loss_mlp": 0.01006079, + "balance_loss_clip": 1.01375008, + "balance_loss_mlp": 1.00451696, + "epoch": 0.4773642758371911, + "flos": 70229387658240.0, + "grad_norm": 0.7261657143570586, + "language_loss": 0.5523507, + "learning_rate": 2.2432146440573616e-06, + "loss": 0.57286471, + "num_input_tokens_seen": 85407220, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.015625, + "step": 3970, + "time_per_iteration": 3.370938539505005 + }, + { + "auxiliary_loss_clip": 0.01143053, + "auxiliary_loss_mlp": 0.01038356, + "balance_loss_clip": 1.04701352, + "balance_loss_mlp": 1.02164257, + "epoch": 0.4774845187278302, + "flos": 23548602009600.0, + "grad_norm": 2.091339843712022, + "language_loss": 0.66673326, + "learning_rate": 2.242441435284534e-06, + "loss": 0.68854731, + "num_input_tokens_seen": 85426095, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.16699219, + "step": 3971, + "time_per_iteration": 2.797416925430298 + }, + { + "auxiliary_loss_clip": 0.01144743, + "auxiliary_loss_mlp": 0.01051233, + "balance_loss_clip": 1.04566932, + "balance_loss_mlp": 1.03331578, + "epoch": 0.4776047616184693, + "flos": 23075371301760.0, + "grad_norm": 2.2979616010778923, + "language_loss": 0.85338956, + "learning_rate": 2.2416681897335337e-06, + "loss": 0.87534934, + "num_input_tokens_seen": 85444245, + "router_z_loss_clip": 0.99121094, + "router_z_loss_mlp": 0.17907715, + "step": 3972, + "time_per_iteration": 3.639035224914551 + }, + { + "auxiliary_loss_clip": 0.01145102, + "auxiliary_loss_mlp": 0.01042068, + "balance_loss_clip": 1.044855, + "balance_loss_mlp": 1.02395988, + "epoch": 0.4777250045091084, + "flos": 31898119374720.0, + "grad_norm": 1.7546205448867167, + "language_loss": 0.6685766, + "learning_rate": 2.240894907521661e-06, + "loss": 0.69044834, + "num_input_tokens_seen": 85463325, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.18127441, + "step": 3973, + "time_per_iteration": 2.791508674621582 + }, + { + "auxiliary_loss_clip": 0.0114505, + "auxiliary_loss_mlp": 0.01041989, + "balance_loss_clip": 1.04779744, + "balance_loss_mlp": 1.02465653, + "epoch": 0.4778452473997475, + "flos": 24278163148800.0, + "grad_norm": 1.8241089477797365, + "language_loss": 0.64089078, + "learning_rate": 2.240121588766223e-06, + "loss": 0.66276121, + "num_input_tokens_seen": 85483375, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.17346191, + "step": 3974, + "time_per_iteration": 2.7981200218200684 + }, + { + "auxiliary_loss_clip": 0.01141227, + "auxiliary_loss_mlp": 0.01040034, + "balance_loss_clip": 1.04698765, + "balance_loss_mlp": 1.02388132, + "epoch": 0.4779654902903866, + "flos": 31575031516800.0, + "grad_norm": 2.064245355856819, + "language_loss": 0.71614456, + "learning_rate": 2.239348233584531e-06, + "loss": 0.73795724, + "num_input_tokens_seen": 85504230, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.16149902, + "step": 3975, + "time_per_iteration": 3.6204662322998047 + }, + { + "auxiliary_loss_clip": 0.01143914, + "auxiliary_loss_mlp": 0.01045036, + "balance_loss_clip": 1.04732871, + "balance_loss_mlp": 1.02888322, + "epoch": 0.47808573318102565, + "flos": 19500428344320.0, + "grad_norm": 1.7669648496573265, + "language_loss": 0.80781376, + "learning_rate": 2.2385748420939013e-06, + "loss": 0.82970333, + "num_input_tokens_seen": 85523425, + "router_z_loss_clip": 0.96484375, + "router_z_loss_mlp": 0.16174316, + "step": 3976, + "time_per_iteration": 2.726898670196533 + }, + { + "auxiliary_loss_clip": 0.01141441, + "auxiliary_loss_mlp": 0.01038161, + "balance_loss_clip": 1.04812598, + "balance_loss_mlp": 1.02241945, + "epoch": 0.47820597607166476, + "flos": 22601135013120.0, + "grad_norm": 1.9247704691264003, + "language_loss": 0.72299862, + "learning_rate": 2.2378014144116583e-06, + "loss": 0.74479467, + "num_input_tokens_seen": 85542235, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.1572876, + "step": 3977, + "time_per_iteration": 3.620479106903076 + }, + { + "auxiliary_loss_clip": 0.01145825, + "auxiliary_loss_mlp": 0.0104136, + "balance_loss_clip": 1.04700446, + "balance_loss_mlp": 1.02430141, + "epoch": 0.4783262189623039, + "flos": 23003011353600.0, + "grad_norm": 2.345283398018291, + "language_loss": 0.79629159, + "learning_rate": 2.23702795065513e-06, + "loss": 0.81816339, + "num_input_tokens_seen": 85561815, + "router_z_loss_clip": 0.98779297, + "router_z_loss_mlp": 0.1706543, + "step": 3978, + "time_per_iteration": 2.7280447483062744 + }, + { + "auxiliary_loss_clip": 0.01046416, + "auxiliary_loss_mlp": 0.01001737, + "balance_loss_clip": 1.01483583, + "balance_loss_mlp": 1.00005662, + "epoch": 0.47844646185294293, + "flos": 49772801226240.0, + "grad_norm": 0.9876339104100722, + "language_loss": 0.67473495, + "learning_rate": 2.2362544509416493e-06, + "loss": 0.69521654, + "num_input_tokens_seen": 85613930, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.0168457, + "step": 3979, + "time_per_iteration": 3.1491658687591553 + }, + { + "auxiliary_loss_clip": 0.01136123, + "auxiliary_loss_mlp": 0.01040712, + "balance_loss_clip": 1.04229963, + "balance_loss_mlp": 1.02283096, + "epoch": 0.47856670474358204, + "flos": 20229558520320.0, + "grad_norm": 2.5609035351497025, + "language_loss": 0.83002067, + "learning_rate": 2.2354809153885572e-06, + "loss": 0.851789, + "num_input_tokens_seen": 85631000, + "router_z_loss_clip": 0.93945312, + "router_z_loss_mlp": 0.17883301, + "step": 3980, + "time_per_iteration": 2.765143632888794 + }, + { + "auxiliary_loss_clip": 0.01142506, + "auxiliary_loss_mlp": 0.01042624, + "balance_loss_clip": 1.04552209, + "balance_loss_mlp": 1.02539802, + "epoch": 0.47868694763422115, + "flos": 20990936131200.0, + "grad_norm": 2.0967945919131106, + "language_loss": 0.83198917, + "learning_rate": 2.234707344113197e-06, + "loss": 0.85384041, + "num_input_tokens_seen": 85649095, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.17224121, + "step": 3981, + "time_per_iteration": 2.7172346115112305 + }, + { + "auxiliary_loss_clip": 0.0114103, + "auxiliary_loss_mlp": 0.01044516, + "balance_loss_clip": 1.04590094, + "balance_loss_mlp": 1.02723074, + "epoch": 0.4788071905248602, + "flos": 19026551191680.0, + "grad_norm": 1.7714615504757718, + "language_loss": 0.77828246, + "learning_rate": 2.233933737232919e-06, + "loss": 0.80013788, + "num_input_tokens_seen": 85666875, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.17272949, + "step": 3982, + "time_per_iteration": 2.7345657348632812 + }, + { + "auxiliary_loss_clip": 0.01141763, + "auxiliary_loss_mlp": 0.01044484, + "balance_loss_clip": 1.04491138, + "balance_loss_mlp": 1.02874887, + "epoch": 0.4789274334154993, + "flos": 23002221254400.0, + "grad_norm": 2.1749473926902025, + "language_loss": 0.78098488, + "learning_rate": 2.2331600948650793e-06, + "loss": 0.80284739, + "num_input_tokens_seen": 85687020, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.1572876, + "step": 3983, + "time_per_iteration": 2.7428760528564453 + }, + { + "auxiliary_loss_clip": 0.01142992, + "auxiliary_loss_mlp": 0.01040058, + "balance_loss_clip": 1.04542708, + "balance_loss_mlp": 1.02112734, + "epoch": 0.4790476763061384, + "flos": 23075586783360.0, + "grad_norm": 1.5069616655940157, + "language_loss": 0.80068278, + "learning_rate": 2.2323864171270386e-06, + "loss": 0.82251328, + "num_input_tokens_seen": 85708290, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.18933105, + "step": 3984, + "time_per_iteration": 2.766169309616089 + }, + { + "auxiliary_loss_clip": 0.01143347, + "auxiliary_loss_mlp": 0.01052703, + "balance_loss_clip": 1.04443085, + "balance_loss_mlp": 1.03454757, + "epoch": 0.4791679191967775, + "flos": 21179288073600.0, + "grad_norm": 2.1114888461582444, + "language_loss": 0.72934175, + "learning_rate": 2.231612704136164e-06, + "loss": 0.75130218, + "num_input_tokens_seen": 85728660, + "router_z_loss_clip": 0.98974609, + "router_z_loss_mlp": 0.18164062, + "step": 3985, + "time_per_iteration": 2.729971170425415 + }, + { + "auxiliary_loss_clip": 0.01143994, + "auxiliary_loss_mlp": 0.010405, + "balance_loss_clip": 1.04578185, + "balance_loss_mlp": 1.02265453, + "epoch": 0.4792881620874166, + "flos": 22301495758080.0, + "grad_norm": 2.5695511812255756, + "language_loss": 0.75851035, + "learning_rate": 2.2308389560098253e-06, + "loss": 0.78035527, + "num_input_tokens_seen": 85745035, + "router_z_loss_clip": 0.98291016, + "router_z_loss_mlp": 0.1784668, + "step": 3986, + "time_per_iteration": 2.7318263053894043 + }, + { + "auxiliary_loss_clip": 0.01142903, + "auxiliary_loss_mlp": 0.01043623, + "balance_loss_clip": 1.04562974, + "balance_loss_mlp": 1.02654004, + "epoch": 0.47940840497805565, + "flos": 17420877423360.0, + "grad_norm": 2.081789644060216, + "language_loss": 0.77064806, + "learning_rate": 2.2300651728654008e-06, + "loss": 0.79251325, + "num_input_tokens_seen": 85760295, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.17089844, + "step": 3987, + "time_per_iteration": 2.6909780502319336 + }, + { + "auxiliary_loss_clip": 0.01041557, + "auxiliary_loss_mlp": 0.01002093, + "balance_loss_clip": 1.0101676, + "balance_loss_mlp": 1.00050735, + "epoch": 0.47952864786869476, + "flos": 65358175708800.0, + "grad_norm": 0.7276043749717473, + "language_loss": 0.60169578, + "learning_rate": 2.229291354820272e-06, + "loss": 0.62213224, + "num_input_tokens_seen": 85821305, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01586914, + "step": 3988, + "time_per_iteration": 3.3200645446777344 + }, + { + "auxiliary_loss_clip": 0.01145018, + "auxiliary_loss_mlp": 0.01041864, + "balance_loss_clip": 1.04885149, + "balance_loss_mlp": 1.02344632, + "epoch": 0.47964889075933387, + "flos": 16799802336000.0, + "grad_norm": 1.906264745689653, + "language_loss": 0.76427495, + "learning_rate": 2.228517501991828e-06, + "loss": 0.78614378, + "num_input_tokens_seen": 85840105, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.1842041, + "step": 3989, + "time_per_iteration": 2.7460596561431885 + }, + { + "auxiliary_loss_clip": 0.01040771, + "auxiliary_loss_mlp": 0.01001139, + "balance_loss_clip": 1.00955582, + "balance_loss_mlp": 0.9995535, + "epoch": 0.4797691336499729, + "flos": 70079244808320.0, + "grad_norm": 0.8140206003331746, + "language_loss": 0.61063015, + "learning_rate": 2.22774361449746e-06, + "loss": 0.63104928, + "num_input_tokens_seen": 85896585, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01586914, + "step": 3990, + "time_per_iteration": 3.2947165966033936 + }, + { + "auxiliary_loss_clip": 0.01145815, + "auxiliary_loss_mlp": 0.01047339, + "balance_loss_clip": 1.04615247, + "balance_loss_mlp": 1.03038788, + "epoch": 0.47988937654061203, + "flos": 18953329317120.0, + "grad_norm": 2.5815069993745134, + "language_loss": 0.70869112, + "learning_rate": 2.2269696924545668e-06, + "loss": 0.73062265, + "num_input_tokens_seen": 85914415, + "router_z_loss_clip": 0.99609375, + "router_z_loss_mlp": 0.16955566, + "step": 3991, + "time_per_iteration": 2.779500722885132 + }, + { + "auxiliary_loss_clip": 0.01141182, + "auxiliary_loss_mlp": 0.01051132, + "balance_loss_clip": 1.04683614, + "balance_loss_mlp": 1.03369141, + "epoch": 0.48000961943125114, + "flos": 14461981649280.0, + "grad_norm": 2.362485586810773, + "language_loss": 0.77809, + "learning_rate": 2.2261957359805523e-06, + "loss": 0.80001318, + "num_input_tokens_seen": 85931650, + "router_z_loss_clip": 0.94384766, + "router_z_loss_mlp": 0.17456055, + "step": 3992, + "time_per_iteration": 2.7282626628875732 + }, + { + "auxiliary_loss_clip": 0.01142336, + "auxiliary_loss_mlp": 0.01050247, + "balance_loss_clip": 1.04591513, + "balance_loss_mlp": 1.03389144, + "epoch": 0.4801298623218902, + "flos": 27051149105280.0, + "grad_norm": 2.061244320067995, + "language_loss": 0.73982638, + "learning_rate": 2.225421745192823e-06, + "loss": 0.76175225, + "num_input_tokens_seen": 85951805, + "router_z_loss_clip": 0.96484375, + "router_z_loss_mlp": 0.16357422, + "step": 3993, + "time_per_iteration": 2.8069050312042236 + }, + { + "auxiliary_loss_clip": 0.0114169, + "auxiliary_loss_mlp": 0.010467, + "balance_loss_clip": 1.04357076, + "balance_loss_mlp": 1.02796078, + "epoch": 0.4802501052125293, + "flos": 26355236031360.0, + "grad_norm": 2.393312662101336, + "language_loss": 0.77844805, + "learning_rate": 2.2246477202087955e-06, + "loss": 0.80033195, + "num_input_tokens_seen": 85972485, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.1875, + "step": 3994, + "time_per_iteration": 2.772495985031128 + }, + { + "auxiliary_loss_clip": 0.01138219, + "auxiliary_loss_mlp": 0.0103792, + "balance_loss_clip": 1.0449152, + "balance_loss_mlp": 1.02208924, + "epoch": 0.4803703481031684, + "flos": 20993916960000.0, + "grad_norm": 1.5945842428083243, + "language_loss": 0.83013612, + "learning_rate": 2.223873661145887e-06, + "loss": 0.85189748, + "num_input_tokens_seen": 85992540, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.15820312, + "step": 3995, + "time_per_iteration": 2.7660341262817383 + }, + { + "auxiliary_loss_clip": 0.01141154, + "auxiliary_loss_mlp": 0.01043306, + "balance_loss_clip": 1.04653215, + "balance_loss_mlp": 1.02714109, + "epoch": 0.4804905909938075, + "flos": 20703722981760.0, + "grad_norm": 2.1571955992290905, + "language_loss": 0.71524268, + "learning_rate": 2.2230995681215226e-06, + "loss": 0.73708731, + "num_input_tokens_seen": 86012065, + "router_z_loss_clip": 0.94580078, + "router_z_loss_mlp": 0.16162109, + "step": 3996, + "time_per_iteration": 2.799928665161133 + }, + { + "auxiliary_loss_clip": 0.01143753, + "auxiliary_loss_mlp": 0.01050121, + "balance_loss_clip": 1.04538298, + "balance_loss_mlp": 1.03139293, + "epoch": 0.4806108338844466, + "flos": 16654831044480.0, + "grad_norm": 3.2333740116834675, + "language_loss": 0.78326356, + "learning_rate": 2.2223254412531305e-06, + "loss": 0.80520225, + "num_input_tokens_seen": 86029435, + "router_z_loss_clip": 0.98486328, + "router_z_loss_mlp": 0.18725586, + "step": 3997, + "time_per_iteration": 2.721482515335083 + }, + { + "auxiliary_loss_clip": 0.01136299, + "auxiliary_loss_mlp": 0.01040967, + "balance_loss_clip": 1.04271913, + "balance_loss_mlp": 1.02494574, + "epoch": 0.4807310767750857, + "flos": 20011329440640.0, + "grad_norm": 1.8417852780236261, + "language_loss": 0.8202163, + "learning_rate": 2.221551280658146e-06, + "loss": 0.84198892, + "num_input_tokens_seen": 86048495, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16015625, + "step": 3998, + "time_per_iteration": 3.652951955795288 + }, + { + "auxiliary_loss_clip": 0.01140717, + "auxiliary_loss_mlp": 0.01042534, + "balance_loss_clip": 1.04721642, + "balance_loss_mlp": 1.0262264, + "epoch": 0.48085131966572475, + "flos": 23185257984000.0, + "grad_norm": 1.8602461408253117, + "language_loss": 0.73931134, + "learning_rate": 2.2207770864540085e-06, + "loss": 0.76114392, + "num_input_tokens_seen": 86067470, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.16296387, + "step": 3999, + "time_per_iteration": 2.72749400138855 + }, + { + "auxiliary_loss_clip": 0.01144557, + "auxiliary_loss_mlp": 0.01041232, + "balance_loss_clip": 1.04723573, + "balance_loss_mlp": 1.02398229, + "epoch": 0.48097156255636386, + "flos": 20558643949440.0, + "grad_norm": 2.1243369841755557, + "language_loss": 0.7307632, + "learning_rate": 2.220002858758162e-06, + "loss": 0.75262105, + "num_input_tokens_seen": 86085460, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.17248535, + "step": 4000, + "time_per_iteration": 2.820812225341797 + }, + { + "auxiliary_loss_clip": 0.01039118, + "auxiliary_loss_mlp": 0.01002958, + "balance_loss_clip": 1.00845075, + "balance_loss_mlp": 1.0014205, + "epoch": 0.481091805447003, + "flos": 70511608817280.0, + "grad_norm": 0.876289360233716, + "language_loss": 0.60870928, + "learning_rate": 2.2192285976880573e-06, + "loss": 0.62913001, + "num_input_tokens_seen": 86149715, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01538086, + "step": 4001, + "time_per_iteration": 4.161239147186279 + }, + { + "auxiliary_loss_clip": 0.01138125, + "auxiliary_loss_mlp": 0.01047644, + "balance_loss_clip": 1.04427063, + "balance_loss_mlp": 1.03133607, + "epoch": 0.48121204833764203, + "flos": 36428214839040.0, + "grad_norm": 1.6069207477161231, + "language_loss": 0.81005991, + "learning_rate": 2.2184543033611485e-06, + "loss": 0.83191764, + "num_input_tokens_seen": 86170795, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.16308594, + "step": 4002, + "time_per_iteration": 2.827188014984131 + }, + { + "auxiliary_loss_clip": 0.01138731, + "auxiliary_loss_mlp": 0.01047576, + "balance_loss_clip": 1.04361272, + "balance_loss_mlp": 1.03045738, + "epoch": 0.48133229122828114, + "flos": 27490264871040.0, + "grad_norm": 2.2103978621805336, + "language_loss": 0.81746668, + "learning_rate": 2.2176799758948957e-06, + "loss": 0.83932972, + "num_input_tokens_seen": 86190955, + "router_z_loss_clip": 0.95019531, + "router_z_loss_mlp": 0.17114258, + "step": 4003, + "time_per_iteration": 2.7708635330200195 + }, + { + "auxiliary_loss_clip": 0.01146387, + "auxiliary_loss_mlp": 0.01054433, + "balance_loss_clip": 1.04646277, + "balance_loss_mlp": 1.03626513, + "epoch": 0.4814525341189202, + "flos": 43072802179200.0, + "grad_norm": 1.6974953550222274, + "language_loss": 0.72767609, + "learning_rate": 2.2169056154067635e-06, + "loss": 0.74968433, + "num_input_tokens_seen": 86214875, + "router_z_loss_clip": 0.99804688, + "router_z_loss_mlp": 0.1817627, + "step": 4004, + "time_per_iteration": 3.684919834136963 + }, + { + "auxiliary_loss_clip": 0.01144923, + "auxiliary_loss_mlp": 0.01039421, + "balance_loss_clip": 1.04682577, + "balance_loss_mlp": 1.02198088, + "epoch": 0.4815727770095593, + "flos": 24236901400320.0, + "grad_norm": 2.2746471047301835, + "language_loss": 0.8253606, + "learning_rate": 2.216131222014222e-06, + "loss": 0.84720403, + "num_input_tokens_seen": 86232950, + "router_z_loss_clip": 0.98095703, + "router_z_loss_mlp": 0.17431641, + "step": 4005, + "time_per_iteration": 2.728283166885376 + }, + { + "auxiliary_loss_clip": 0.0114331, + "auxiliary_loss_mlp": 0.01046255, + "balance_loss_clip": 1.04681945, + "balance_loss_mlp": 1.02869534, + "epoch": 0.4816930199001984, + "flos": 18113630100480.0, + "grad_norm": 2.530282716269809, + "language_loss": 0.8042956, + "learning_rate": 2.2153567958347455e-06, + "loss": 0.82619131, + "num_input_tokens_seen": 86249160, + "router_z_loss_clip": 0.96435547, + "router_z_loss_mlp": 0.17553711, + "step": 4006, + "time_per_iteration": 3.5842010974884033 + }, + { + "auxiliary_loss_clip": 0.01142249, + "auxiliary_loss_mlp": 0.01044424, + "balance_loss_clip": 1.04575276, + "balance_loss_mlp": 1.02580297, + "epoch": 0.48181326279083747, + "flos": 17274720983040.0, + "grad_norm": 2.2084527313781015, + "language_loss": 0.80136317, + "learning_rate": 2.214582336985815e-06, + "loss": 0.82322985, + "num_input_tokens_seen": 86267060, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.18615723, + "step": 4007, + "time_per_iteration": 2.755403518676758 + }, + { + "auxiliary_loss_clip": 0.01144192, + "auxiliary_loss_mlp": 0.01042571, + "balance_loss_clip": 1.04608965, + "balance_loss_mlp": 1.02470183, + "epoch": 0.4819335056814766, + "flos": 14903252231040.0, + "grad_norm": 2.5978841914404356, + "language_loss": 0.66272414, + "learning_rate": 2.2138078455849142e-06, + "loss": 0.68459177, + "num_input_tokens_seen": 86285055, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.17871094, + "step": 4008, + "time_per_iteration": 2.78377628326416 + }, + { + "auxiliary_loss_clip": 0.01145133, + "auxiliary_loss_mlp": 0.01045287, + "balance_loss_clip": 1.04799044, + "balance_loss_mlp": 1.02824008, + "epoch": 0.4820537485721157, + "flos": 19244888012160.0, + "grad_norm": 1.9933236496320976, + "language_loss": 0.78638911, + "learning_rate": 2.2130333217495334e-06, + "loss": 0.80829334, + "num_input_tokens_seen": 86304225, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.17053223, + "step": 4009, + "time_per_iteration": 2.7993900775909424 + }, + { + "auxiliary_loss_clip": 0.01145717, + "auxiliary_loss_mlp": 0.01042268, + "balance_loss_clip": 1.04567719, + "balance_loss_mlp": 1.02575755, + "epoch": 0.48217399146275475, + "flos": 16033791870720.0, + "grad_norm": 2.483925066379292, + "language_loss": 0.67816567, + "learning_rate": 2.2122587655971665e-06, + "loss": 0.70004553, + "num_input_tokens_seen": 86319170, + "router_z_loss_clip": 1.00048828, + "router_z_loss_mlp": 0.16516113, + "step": 4010, + "time_per_iteration": 2.718102216720581 + }, + { + "auxiliary_loss_clip": 0.01140016, + "auxiliary_loss_mlp": 0.01040685, + "balance_loss_clip": 1.04529619, + "balance_loss_mlp": 1.02345943, + "epoch": 0.48229423435339386, + "flos": 24134197438080.0, + "grad_norm": 1.5632635483142694, + "language_loss": 0.63999999, + "learning_rate": 2.211484177245314e-06, + "loss": 0.66180694, + "num_input_tokens_seen": 86338760, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.17224121, + "step": 4011, + "time_per_iteration": 2.80464768409729 + }, + { + "auxiliary_loss_clip": 0.01143719, + "auxiliary_loss_mlp": 0.01050138, + "balance_loss_clip": 1.0467186, + "balance_loss_mlp": 1.03169656, + "epoch": 0.48241447724403297, + "flos": 23805435231360.0, + "grad_norm": 7.860940529013375, + "language_loss": 0.72227168, + "learning_rate": 2.21070955681148e-06, + "loss": 0.74421024, + "num_input_tokens_seen": 86357865, + "router_z_loss_clip": 0.96972656, + "router_z_loss_mlp": 0.18432617, + "step": 4012, + "time_per_iteration": 2.76330828666687 + }, + { + "auxiliary_loss_clip": 0.0114082, + "auxiliary_loss_mlp": 0.01043594, + "balance_loss_clip": 1.04553461, + "balance_loss_mlp": 1.02603459, + "epoch": 0.482534720134672, + "flos": 23110312256640.0, + "grad_norm": 1.6027881814131197, + "language_loss": 0.78593838, + "learning_rate": 2.209934904413174e-06, + "loss": 0.80778253, + "num_input_tokens_seen": 86379470, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.17553711, + "step": 4013, + "time_per_iteration": 2.7886154651641846 + }, + { + "auxiliary_loss_clip": 0.0114579, + "auxiliary_loss_mlp": 0.01040914, + "balance_loss_clip": 1.0476898, + "balance_loss_mlp": 1.02285409, + "epoch": 0.48265496302531113, + "flos": 20923819568640.0, + "grad_norm": 2.2272876700855297, + "language_loss": 0.7178117, + "learning_rate": 2.2091602201679095e-06, + "loss": 0.73967874, + "num_input_tokens_seen": 86399080, + "router_z_loss_clip": 0.97998047, + "router_z_loss_mlp": 0.18054199, + "step": 4014, + "time_per_iteration": 2.7423255443573 + }, + { + "auxiliary_loss_clip": 0.01143201, + "auxiliary_loss_mlp": 0.01036806, + "balance_loss_clip": 1.04743302, + "balance_loss_mlp": 1.01981914, + "epoch": 0.48277520591595025, + "flos": 15231152511360.0, + "grad_norm": 2.003702767906707, + "language_loss": 0.83227015, + "learning_rate": 2.208385504193206e-06, + "loss": 0.85407019, + "num_input_tokens_seen": 86416580, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.1697998, + "step": 4015, + "time_per_iteration": 2.7746694087982178 + }, + { + "auxiliary_loss_clip": 0.01141722, + "auxiliary_loss_mlp": 0.01044756, + "balance_loss_clip": 1.04405856, + "balance_loss_mlp": 1.02807248, + "epoch": 0.4828954488065893, + "flos": 17858664385920.0, + "grad_norm": 2.856930246976285, + "language_loss": 0.80886066, + "learning_rate": 2.2076107566065873e-06, + "loss": 0.83072543, + "num_input_tokens_seen": 86434365, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.16680908, + "step": 4016, + "time_per_iteration": 2.75494647026062 + }, + { + "auxiliary_loss_clip": 0.01141871, + "auxiliary_loss_mlp": 0.01043757, + "balance_loss_clip": 1.04620898, + "balance_loss_mlp": 1.02760422, + "epoch": 0.4830156916972284, + "flos": 32087405070720.0, + "grad_norm": 2.2833198219158626, + "language_loss": 0.75616586, + "learning_rate": 2.2068359775255816e-06, + "loss": 0.77802217, + "num_input_tokens_seen": 86452675, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.16149902, + "step": 4017, + "time_per_iteration": 2.8174736499786377 + }, + { + "auxiliary_loss_clip": 0.0114278, + "auxiliary_loss_mlp": 0.01049813, + "balance_loss_clip": 1.04527903, + "balance_loss_mlp": 1.03340995, + "epoch": 0.48313593458786747, + "flos": 21871717528320.0, + "grad_norm": 4.294133638317413, + "language_loss": 0.78841805, + "learning_rate": 2.206061167067723e-06, + "loss": 0.81034398, + "num_input_tokens_seen": 86470785, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.1640625, + "step": 4018, + "time_per_iteration": 2.7979445457458496 + }, + { + "auxiliary_loss_clip": 0.01143898, + "auxiliary_loss_mlp": 0.01044306, + "balance_loss_clip": 1.04644823, + "balance_loss_mlp": 1.02691293, + "epoch": 0.4832561774785066, + "flos": 22601206840320.0, + "grad_norm": 2.40876930459021, + "language_loss": 0.79334521, + "learning_rate": 2.205286325350549e-06, + "loss": 0.81522721, + "num_input_tokens_seen": 86489850, + "router_z_loss_clip": 0.97509766, + "router_z_loss_mlp": 0.17407227, + "step": 4019, + "time_per_iteration": 2.7370288372039795 + }, + { + "auxiliary_loss_clip": 0.01140973, + "auxiliary_loss_mlp": 0.01043981, + "balance_loss_clip": 1.04513693, + "balance_loss_mlp": 1.02737546, + "epoch": 0.4833764203691457, + "flos": 13437342282240.0, + "grad_norm": 2.0635212102450424, + "language_loss": 0.72664154, + "learning_rate": 2.204511452491603e-06, + "loss": 0.74849105, + "num_input_tokens_seen": 86506475, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.16601562, + "step": 4020, + "time_per_iteration": 2.724219560623169 + }, + { + "auxiliary_loss_clip": 0.0113696, + "auxiliary_loss_mlp": 0.01040267, + "balance_loss_clip": 1.04471207, + "balance_loss_mlp": 1.02418542, + "epoch": 0.48349666325978474, + "flos": 44128036955520.0, + "grad_norm": 1.8019239044432003, + "language_loss": 0.75334591, + "learning_rate": 2.2037365486084316e-06, + "loss": 0.77511817, + "num_input_tokens_seen": 86529715, + "router_z_loss_clip": 0.92236328, + "router_z_loss_mlp": 0.16088867, + "step": 4021, + "time_per_iteration": 2.9483489990234375 + }, + { + "auxiliary_loss_clip": 0.01143041, + "auxiliary_loss_mlp": 0.01037608, + "balance_loss_clip": 1.04573905, + "balance_loss_mlp": 1.01967931, + "epoch": 0.48361690615042385, + "flos": 26028377245440.0, + "grad_norm": 1.9017290746536837, + "language_loss": 0.78043222, + "learning_rate": 2.2029616138185886e-06, + "loss": 0.8022387, + "num_input_tokens_seen": 86548715, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.17944336, + "step": 4022, + "time_per_iteration": 2.7669970989227295 + }, + { + "auxiliary_loss_clip": 0.01141152, + "auxiliary_loss_mlp": 0.0104954, + "balance_loss_clip": 1.04491115, + "balance_loss_mlp": 1.03273177, + "epoch": 0.48373714904106296, + "flos": 22273306560000.0, + "grad_norm": 1.6261122784234785, + "language_loss": 0.82848859, + "learning_rate": 2.202186648239629e-06, + "loss": 0.85039556, + "num_input_tokens_seen": 86568650, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.16796875, + "step": 4023, + "time_per_iteration": 2.7988908290863037 + }, + { + "auxiliary_loss_clip": 0.01140307, + "auxiliary_loss_mlp": 0.01037295, + "balance_loss_clip": 1.04679811, + "balance_loss_mlp": 1.0218339, + "epoch": 0.483857391931702, + "flos": 28292293699200.0, + "grad_norm": 1.7587899438326804, + "language_loss": 0.71575987, + "learning_rate": 2.201411651989117e-06, + "loss": 0.73753595, + "num_input_tokens_seen": 86590630, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.15466309, + "step": 4024, + "time_per_iteration": 2.8149020671844482 + }, + { + "auxiliary_loss_clip": 0.01139605, + "auxiliary_loss_mlp": 0.01045937, + "balance_loss_clip": 1.04469144, + "balance_loss_mlp": 1.02900958, + "epoch": 0.48397763482234113, + "flos": 27418048577280.0, + "grad_norm": 1.9543837329924505, + "language_loss": 0.78104907, + "learning_rate": 2.2006366251846167e-06, + "loss": 0.80290449, + "num_input_tokens_seen": 86611270, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.16943359, + "step": 4025, + "time_per_iteration": 2.772007465362549 + }, + { + "auxiliary_loss_clip": 0.01140045, + "auxiliary_loss_mlp": 0.01043509, + "balance_loss_clip": 1.04812431, + "balance_loss_mlp": 1.02624726, + "epoch": 0.48409787771298024, + "flos": 16797252470400.0, + "grad_norm": 1.8644365428890721, + "language_loss": 0.7533347, + "learning_rate": 2.1998615679436997e-06, + "loss": 0.77517027, + "num_input_tokens_seen": 86628810, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.17260742, + "step": 4026, + "time_per_iteration": 2.707026720046997 + }, + { + "auxiliary_loss_clip": 0.01145014, + "auxiliary_loss_mlp": 0.01040948, + "balance_loss_clip": 1.04564977, + "balance_loss_mlp": 1.02250671, + "epoch": 0.4842181206036193, + "flos": 25083496028160.0, + "grad_norm": 3.1771075209593636, + "language_loss": 0.77071929, + "learning_rate": 2.199086480383942e-06, + "loss": 0.79257888, + "num_input_tokens_seen": 86648185, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.18444824, + "step": 4027, + "time_per_iteration": 3.5542151927948 + }, + { + "auxiliary_loss_clip": 0.01152516, + "auxiliary_loss_mlp": 0.01053488, + "balance_loss_clip": 1.04984212, + "balance_loss_mlp": 1.03442609, + "epoch": 0.4843383634942584, + "flos": 30372311496960.0, + "grad_norm": 4.465066216963799, + "language_loss": 0.68469381, + "learning_rate": 2.1983113626229234e-06, + "loss": 0.70675385, + "num_input_tokens_seen": 86667435, + "router_z_loss_clip": 1.02734375, + "router_z_loss_mlp": 0.19067383, + "step": 4028, + "time_per_iteration": 2.8105556964874268 + }, + { + "auxiliary_loss_clip": 0.01144238, + "auxiliary_loss_mlp": 0.01048279, + "balance_loss_clip": 1.04795384, + "balance_loss_mlp": 1.0309341, + "epoch": 0.4844586063848975, + "flos": 20413564917120.0, + "grad_norm": 1.6824947865108595, + "language_loss": 0.78691638, + "learning_rate": 2.1975362147782293e-06, + "loss": 0.80884153, + "num_input_tokens_seen": 86686630, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.17346191, + "step": 4029, + "time_per_iteration": 3.7406952381134033 + }, + { + "auxiliary_loss_clip": 0.01040479, + "auxiliary_loss_mlp": 0.01002151, + "balance_loss_clip": 1.00930345, + "balance_loss_mlp": 1.00038719, + "epoch": 0.48457884927553657, + "flos": 70303722854400.0, + "grad_norm": 0.687829489675912, + "language_loss": 0.54074299, + "learning_rate": 2.196761036967448e-06, + "loss": 0.56116927, + "num_input_tokens_seen": 86754595, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.0177002, + "step": 4030, + "time_per_iteration": 3.5207059383392334 + }, + { + "auxiliary_loss_clip": 0.01134287, + "auxiliary_loss_mlp": 0.01034873, + "balance_loss_clip": 1.04288292, + "balance_loss_mlp": 1.01947069, + "epoch": 0.4846990921661757, + "flos": 19934516206080.0, + "grad_norm": 1.9350042856012508, + "language_loss": 0.77698475, + "learning_rate": 2.1959858293081743e-06, + "loss": 0.79867637, + "num_input_tokens_seen": 86773730, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.15405273, + "step": 4031, + "time_per_iteration": 2.787144899368286 + }, + { + "auxiliary_loss_clip": 0.01141357, + "auxiliary_loss_mlp": 0.01039665, + "balance_loss_clip": 1.04734731, + "balance_loss_mlp": 1.02333379, + "epoch": 0.4848193350568148, + "flos": 23075945919360.0, + "grad_norm": 1.843782024642564, + "language_loss": 0.76254117, + "learning_rate": 2.1952105919180056e-06, + "loss": 0.78435135, + "num_input_tokens_seen": 86792985, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.16333008, + "step": 4032, + "time_per_iteration": 2.918116331100464 + }, + { + "auxiliary_loss_clip": 0.01145624, + "auxiliary_loss_mlp": 0.01044033, + "balance_loss_clip": 1.04733813, + "balance_loss_mlp": 1.02641356, + "epoch": 0.48493957794745385, + "flos": 22455481363200.0, + "grad_norm": 4.24576655265346, + "language_loss": 0.68582159, + "learning_rate": 2.1944353249145456e-06, + "loss": 0.70771813, + "num_input_tokens_seen": 86812095, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.17614746, + "step": 4033, + "time_per_iteration": 3.742097854614258 + }, + { + "auxiliary_loss_clip": 0.01139903, + "auxiliary_loss_mlp": 0.01042947, + "balance_loss_clip": 1.04832125, + "balance_loss_mlp": 1.0273422, + "epoch": 0.48505982083809296, + "flos": 25046112948480.0, + "grad_norm": 1.6546830103929056, + "language_loss": 0.74979413, + "learning_rate": 2.193660028415401e-06, + "loss": 0.77162266, + "num_input_tokens_seen": 86832875, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.15594482, + "step": 4034, + "time_per_iteration": 2.75783371925354 + }, + { + "auxiliary_loss_clip": 0.01143169, + "auxiliary_loss_mlp": 0.01041972, + "balance_loss_clip": 1.04578972, + "balance_loss_mlp": 1.02486551, + "epoch": 0.485180063728732, + "flos": 26761386090240.0, + "grad_norm": 1.6809864887540094, + "language_loss": 0.8220287, + "learning_rate": 2.1928847025381852e-06, + "loss": 0.84388006, + "num_input_tokens_seen": 86853480, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.17102051, + "step": 4035, + "time_per_iteration": 4.058286666870117 + }, + { + "auxiliary_loss_clip": 0.01143033, + "auxiliary_loss_mlp": 0.01050272, + "balance_loss_clip": 1.04411709, + "balance_loss_mlp": 1.03283226, + "epoch": 0.4853003066193711, + "flos": 24059143969920.0, + "grad_norm": 1.650584385238471, + "language_loss": 0.84176308, + "learning_rate": 2.192109347400512e-06, + "loss": 0.8636961, + "num_input_tokens_seen": 86873695, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.17456055, + "step": 4036, + "time_per_iteration": 5.574532747268677 + }, + { + "auxiliary_loss_clip": 0.01144425, + "auxiliary_loss_mlp": 0.01045272, + "balance_loss_clip": 1.0467453, + "balance_loss_mlp": 1.02724791, + "epoch": 0.48542054951001024, + "flos": 23076376882560.0, + "grad_norm": 2.029306819624383, + "language_loss": 0.79172051, + "learning_rate": 2.191333963120004e-06, + "loss": 0.81361747, + "num_input_tokens_seen": 86892675, + "router_z_loss_clip": 0.9765625, + "router_z_loss_mlp": 0.18029785, + "step": 4037, + "time_per_iteration": 6.152212381362915 + }, + { + "auxiliary_loss_clip": 0.01144004, + "auxiliary_loss_mlp": 0.01048568, + "balance_loss_clip": 1.04551721, + "balance_loss_mlp": 1.0312711, + "epoch": 0.4855407924006493, + "flos": 25664889565440.0, + "grad_norm": 2.4639398627336733, + "language_loss": 0.70294487, + "learning_rate": 2.190558549814286e-06, + "loss": 0.72487068, + "num_input_tokens_seen": 86912835, + "router_z_loss_clip": 0.98486328, + "router_z_loss_mlp": 0.17297363, + "step": 4038, + "time_per_iteration": 3.078603506088257 + }, + { + "auxiliary_loss_clip": 0.01143465, + "auxiliary_loss_mlp": 0.01044631, + "balance_loss_clip": 1.04708052, + "balance_loss_mlp": 1.02741671, + "epoch": 0.4856610352912884, + "flos": 23987933256960.0, + "grad_norm": 1.7443230497173865, + "language_loss": 0.7955032, + "learning_rate": 2.1897831076009872e-06, + "loss": 0.81738412, + "num_input_tokens_seen": 86932475, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.17211914, + "step": 4039, + "time_per_iteration": 3.0485804080963135 + }, + { + "auxiliary_loss_clip": 0.01144543, + "auxiliary_loss_mlp": 0.01043966, + "balance_loss_clip": 1.04517961, + "balance_loss_mlp": 1.02589417, + "epoch": 0.4857812781819275, + "flos": 24096814358400.0, + "grad_norm": 1.925273811001216, + "language_loss": 0.79704177, + "learning_rate": 2.1890076365977426e-06, + "loss": 0.81892687, + "num_input_tokens_seen": 86952300, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.18066406, + "step": 4040, + "time_per_iteration": 3.082892894744873 + }, + { + "auxiliary_loss_clip": 0.01041286, + "auxiliary_loss_mlp": 0.01000643, + "balance_loss_clip": 1.00985909, + "balance_loss_mlp": 0.99897426, + "epoch": 0.48590152107256657, + "flos": 56266635185280.0, + "grad_norm": 0.8531000961131346, + "language_loss": 0.52747828, + "learning_rate": 2.188232136922189e-06, + "loss": 0.54789758, + "num_input_tokens_seen": 87010420, + "router_z_loss_clip": 0.31445312, + "router_z_loss_mlp": 0.01672363, + "step": 4041, + "time_per_iteration": 3.417309284210205 + }, + { + "auxiliary_loss_clip": 0.01142699, + "auxiliary_loss_mlp": 0.01042023, + "balance_loss_clip": 1.04449654, + "balance_loss_mlp": 1.02292609, + "epoch": 0.4860217639632057, + "flos": 20046988667520.0, + "grad_norm": 3.456126490737233, + "language_loss": 0.76033294, + "learning_rate": 2.187456608691971e-06, + "loss": 0.78218019, + "num_input_tokens_seen": 87029295, + "router_z_loss_clip": 0.98144531, + "router_z_loss_mlp": 0.19104004, + "step": 4042, + "time_per_iteration": 3.050574779510498 + }, + { + "auxiliary_loss_clip": 0.01145707, + "auxiliary_loss_mlp": 0.01041233, + "balance_loss_clip": 1.04880333, + "balance_loss_mlp": 1.02403069, + "epoch": 0.4861420068538448, + "flos": 17822143232640.0, + "grad_norm": 1.915586801444543, + "language_loss": 0.87612951, + "learning_rate": 2.1866810520247334e-06, + "loss": 0.89799887, + "num_input_tokens_seen": 87048165, + "router_z_loss_clip": 0.96972656, + "router_z_loss_mlp": 0.17199707, + "step": 4043, + "time_per_iteration": 2.8681697845458984 + }, + { + "auxiliary_loss_clip": 0.01147387, + "auxiliary_loss_mlp": 0.01042789, + "balance_loss_clip": 1.04714346, + "balance_loss_mlp": 1.02437091, + "epoch": 0.48626224974448384, + "flos": 26250125857920.0, + "grad_norm": 2.7073751443363094, + "language_loss": 0.64723158, + "learning_rate": 2.185905467038129e-06, + "loss": 0.66913331, + "num_input_tokens_seen": 87067070, + "router_z_loss_clip": 1.00292969, + "router_z_loss_mlp": 0.1842041, + "step": 4044, + "time_per_iteration": 2.877547025680542 + }, + { + "auxiliary_loss_clip": 0.01141602, + "auxiliary_loss_mlp": 0.01048907, + "balance_loss_clip": 1.04739499, + "balance_loss_mlp": 1.0330646, + "epoch": 0.48638249263512295, + "flos": 22054502862720.0, + "grad_norm": 1.6759652343071574, + "language_loss": 0.78001201, + "learning_rate": 2.1851298538498127e-06, + "loss": 0.80191714, + "num_input_tokens_seen": 87086785, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.15844727, + "step": 4045, + "time_per_iteration": 2.8187952041625977 + }, + { + "auxiliary_loss_clip": 0.01148991, + "auxiliary_loss_mlp": 0.01045816, + "balance_loss_clip": 1.04703808, + "balance_loss_mlp": 1.02751684, + "epoch": 0.48650273552576206, + "flos": 25119945354240.0, + "grad_norm": 2.159073585813052, + "language_loss": 0.80069864, + "learning_rate": 2.184354212577446e-06, + "loss": 0.82264668, + "num_input_tokens_seen": 87107090, + "router_z_loss_clip": 1.01855469, + "router_z_loss_mlp": 0.1829834, + "step": 4046, + "time_per_iteration": 2.8120386600494385 + }, + { + "auxiliary_loss_clip": 0.01145228, + "auxiliary_loss_mlp": 0.01049952, + "balance_loss_clip": 1.04569864, + "balance_loss_mlp": 1.03198695, + "epoch": 0.4866229784164011, + "flos": 17456931699840.0, + "grad_norm": 3.71397055931124, + "language_loss": 0.62734985, + "learning_rate": 2.1835785433386907e-06, + "loss": 0.64930165, + "num_input_tokens_seen": 87125905, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.1796875, + "step": 4047, + "time_per_iteration": 2.849628210067749 + }, + { + "auxiliary_loss_clip": 0.01145045, + "auxiliary_loss_mlp": 0.01050143, + "balance_loss_clip": 1.04652381, + "balance_loss_mlp": 1.03092623, + "epoch": 0.48674322130704023, + "flos": 23331127115520.0, + "grad_norm": 1.926247606694261, + "language_loss": 0.65619206, + "learning_rate": 2.182802846251216e-06, + "loss": 0.67814398, + "num_input_tokens_seen": 87146175, + "router_z_loss_clip": 0.98486328, + "router_z_loss_mlp": 0.19226074, + "step": 4048, + "time_per_iteration": 3.242312431335449 + }, + { + "auxiliary_loss_clip": 0.01140997, + "auxiliary_loss_mlp": 0.01046713, + "balance_loss_clip": 1.0456543, + "balance_loss_mlp": 1.02871275, + "epoch": 0.4868634641976793, + "flos": 28804344030720.0, + "grad_norm": 1.8538957507599334, + "language_loss": 0.72457731, + "learning_rate": 2.182027121432696e-06, + "loss": 0.74645436, + "num_input_tokens_seen": 87166800, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.18005371, + "step": 4049, + "time_per_iteration": 3.155515670776367 + }, + { + "auxiliary_loss_clip": 0.01147329, + "auxiliary_loss_mlp": 0.01043943, + "balance_loss_clip": 1.04696572, + "balance_loss_mlp": 1.02472675, + "epoch": 0.4869837070883184, + "flos": 19025976574080.0, + "grad_norm": 1.6479409115790407, + "language_loss": 0.82344371, + "learning_rate": 2.1812513690008054e-06, + "loss": 0.84535646, + "num_input_tokens_seen": 87185920, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.19213867, + "step": 4050, + "time_per_iteration": 3.018611192703247 + }, + { + "auxiliary_loss_clip": 0.01147164, + "auxiliary_loss_mlp": 0.01048896, + "balance_loss_clip": 1.04877543, + "balance_loss_mlp": 1.03069234, + "epoch": 0.4871039499789575, + "flos": 15121409483520.0, + "grad_norm": 4.484314837817247, + "language_loss": 0.80325192, + "learning_rate": 2.180475589073227e-06, + "loss": 0.82521248, + "num_input_tokens_seen": 87203620, + "router_z_loss_clip": 0.98388672, + "router_z_loss_mlp": 0.18200684, + "step": 4051, + "time_per_iteration": 3.1185083389282227 + }, + { + "auxiliary_loss_clip": 0.01140548, + "auxiliary_loss_mlp": 0.0104342, + "balance_loss_clip": 1.0457902, + "balance_loss_mlp": 1.02723134, + "epoch": 0.48722419286959656, + "flos": 26174066808960.0, + "grad_norm": 1.7809396566573426, + "language_loss": 0.73465967, + "learning_rate": 2.1796997817676456e-06, + "loss": 0.75649935, + "num_input_tokens_seen": 87224630, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.1619873, + "step": 4052, + "time_per_iteration": 3.0885932445526123 + }, + { + "auxiliary_loss_clip": 0.01138548, + "auxiliary_loss_mlp": 0.01047708, + "balance_loss_clip": 1.04550874, + "balance_loss_mlp": 1.03122115, + "epoch": 0.4873444357602357, + "flos": 24026142349440.0, + "grad_norm": 2.6584287820668733, + "language_loss": 0.67276883, + "learning_rate": 2.1789239472017494e-06, + "loss": 0.69463146, + "num_input_tokens_seen": 87246280, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.16479492, + "step": 4053, + "time_per_iteration": 3.0862481594085693 + }, + { + "auxiliary_loss_clip": 0.01141149, + "auxiliary_loss_mlp": 0.01042221, + "balance_loss_clip": 1.04614925, + "balance_loss_mlp": 1.02467382, + "epoch": 0.4874646786508748, + "flos": 22820441500800.0, + "grad_norm": 2.2347649558040614, + "language_loss": 0.73005593, + "learning_rate": 2.1781480854932326e-06, + "loss": 0.75188959, + "num_input_tokens_seen": 87266045, + "router_z_loss_clip": 0.95019531, + "router_z_loss_mlp": 0.17541504, + "step": 4054, + "time_per_iteration": 3.039907693862915 + }, + { + "auxiliary_loss_clip": 0.01139699, + "auxiliary_loss_mlp": 0.01045229, + "balance_loss_clip": 1.04457712, + "balance_loss_mlp": 1.02786064, + "epoch": 0.48758492154151384, + "flos": 21287594557440.0, + "grad_norm": 1.8158017911961901, + "language_loss": 0.79410499, + "learning_rate": 2.1773721967597933e-06, + "loss": 0.81595433, + "num_input_tokens_seen": 87284495, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.17358398, + "step": 4055, + "time_per_iteration": 2.9589362144470215 + }, + { + "auxiliary_loss_clip": 0.01041267, + "auxiliary_loss_mlp": 0.01002338, + "balance_loss_clip": 1.00987577, + "balance_loss_mlp": 1.00059736, + "epoch": 0.48770516443215295, + "flos": 62244109180800.0, + "grad_norm": 0.8471020003596793, + "language_loss": 0.57368183, + "learning_rate": 2.1765962811191322e-06, + "loss": 0.59411788, + "num_input_tokens_seen": 87338960, + "router_z_loss_clip": 0.31445312, + "router_z_loss_mlp": 0.01745605, + "step": 4056, + "time_per_iteration": 4.050853490829468 + }, + { + "auxiliary_loss_clip": 0.01041623, + "auxiliary_loss_mlp": 0.01004998, + "balance_loss_clip": 1.01012301, + "balance_loss_mlp": 1.00331736, + "epoch": 0.48782540732279206, + "flos": 66133451882880.0, + "grad_norm": 0.827555341679873, + "language_loss": 0.61930382, + "learning_rate": 2.1758203386889566e-06, + "loss": 0.63977003, + "num_input_tokens_seen": 87401730, + "router_z_loss_clip": 0.31445312, + "router_z_loss_mlp": 0.0168457, + "step": 4057, + "time_per_iteration": 3.339552879333496 + }, + { + "auxiliary_loss_clip": 0.01145226, + "auxiliary_loss_mlp": 0.01044645, + "balance_loss_clip": 1.04599452, + "balance_loss_mlp": 1.02691829, + "epoch": 0.4879456502134311, + "flos": 14607922608000.0, + "grad_norm": 2.061054810154448, + "language_loss": 0.84318691, + "learning_rate": 2.1750443695869746e-06, + "loss": 0.86508566, + "num_input_tokens_seen": 87417300, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.17712402, + "step": 4058, + "time_per_iteration": 3.6722676753997803 + }, + { + "auxiliary_loss_clip": 0.01145891, + "auxiliary_loss_mlp": 0.01045198, + "balance_loss_clip": 1.04709876, + "balance_loss_mlp": 1.02782941, + "epoch": 0.4880658931040702, + "flos": 19500464257920.0, + "grad_norm": 1.8148048451912155, + "language_loss": 0.85903287, + "learning_rate": 2.174268373930901e-06, + "loss": 0.88094378, + "num_input_tokens_seen": 87434815, + "router_z_loss_clip": 0.98730469, + "router_z_loss_mlp": 0.17370605, + "step": 4059, + "time_per_iteration": 2.784787178039551 + }, + { + "auxiliary_loss_clip": 0.01145728, + "auxiliary_loss_mlp": 0.01040356, + "balance_loss_clip": 1.04739928, + "balance_loss_mlp": 1.02250993, + "epoch": 0.48818613599470934, + "flos": 16723060928640.0, + "grad_norm": 5.857895449477289, + "language_loss": 0.7969991, + "learning_rate": 2.1734923518384537e-06, + "loss": 0.81885999, + "num_input_tokens_seen": 87451420, + "router_z_loss_clip": 0.98291016, + "router_z_loss_mlp": 0.17822266, + "step": 4060, + "time_per_iteration": 2.8436079025268555 + }, + { + "auxiliary_loss_clip": 0.01141335, + "auxiliary_loss_mlp": 0.01041633, + "balance_loss_clip": 1.04658413, + "balance_loss_mlp": 1.02496743, + "epoch": 0.4883063788853484, + "flos": 26756932803840.0, + "grad_norm": 1.7770216812685278, + "language_loss": 0.82708859, + "learning_rate": 2.1727163034273547e-06, + "loss": 0.84891826, + "num_input_tokens_seen": 87469585, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.16662598, + "step": 4061, + "time_per_iteration": 2.9449663162231445 + }, + { + "auxiliary_loss_clip": 0.01140801, + "auxiliary_loss_mlp": 0.01048767, + "balance_loss_clip": 1.04511833, + "balance_loss_mlp": 1.0314101, + "epoch": 0.4884266217759875, + "flos": 16763388923520.0, + "grad_norm": 2.296030037154617, + "language_loss": 0.78873456, + "learning_rate": 2.17194022881533e-06, + "loss": 0.8106302, + "num_input_tokens_seen": 87485675, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.17346191, + "step": 4062, + "time_per_iteration": 3.5907089710235596 + }, + { + "auxiliary_loss_clip": 0.011472, + "auxiliary_loss_mlp": 0.01046797, + "balance_loss_clip": 1.04669821, + "balance_loss_mlp": 1.02812886, + "epoch": 0.4885468646666266, + "flos": 24207132003840.0, + "grad_norm": 2.0907070272485604, + "language_loss": 0.67673802, + "learning_rate": 2.1711641281201092e-06, + "loss": 0.69867802, + "num_input_tokens_seen": 87505605, + "router_z_loss_clip": 1.00537109, + "router_z_loss_mlp": 0.18676758, + "step": 4063, + "time_per_iteration": 2.8098864555358887 + }, + { + "auxiliary_loss_clip": 0.01142312, + "auxiliary_loss_mlp": 0.01043504, + "balance_loss_clip": 1.04770756, + "balance_loss_mlp": 1.02598071, + "epoch": 0.48866710755726567, + "flos": 14610795696000.0, + "grad_norm": 3.2058053650821288, + "language_loss": 0.79739708, + "learning_rate": 2.1703880014594264e-06, + "loss": 0.81925529, + "num_input_tokens_seen": 87523195, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.17529297, + "step": 4064, + "time_per_iteration": 3.5455005168914795 + }, + { + "auxiliary_loss_clip": 0.01142241, + "auxiliary_loss_mlp": 0.01038577, + "balance_loss_clip": 1.04674947, + "balance_loss_mlp": 1.02046871, + "epoch": 0.4887873504479048, + "flos": 28804451771520.0, + "grad_norm": 2.281501008768241, + "language_loss": 0.73774946, + "learning_rate": 2.1696118489510182e-06, + "loss": 0.7595576, + "num_input_tokens_seen": 87544125, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.1809082, + "step": 4065, + "time_per_iteration": 3.8011221885681152 + }, + { + "auxiliary_loss_clip": 0.01147407, + "auxiliary_loss_mlp": 0.01050078, + "balance_loss_clip": 1.0496434, + "balance_loss_mlp": 1.03279257, + "epoch": 0.48890759333854383, + "flos": 22784387224320.0, + "grad_norm": 2.375088836996472, + "language_loss": 0.72453272, + "learning_rate": 2.1688356707126286e-06, + "loss": 0.74650764, + "num_input_tokens_seen": 87563745, + "router_z_loss_clip": 0.97705078, + "router_z_loss_mlp": 0.17297363, + "step": 4066, + "time_per_iteration": 6.46759033203125 + }, + { + "auxiliary_loss_clip": 0.01144802, + "auxiliary_loss_mlp": 0.0105054, + "balance_loss_clip": 1.04555988, + "balance_loss_mlp": 1.03259921, + "epoch": 0.48902783622918294, + "flos": 17786088956160.0, + "grad_norm": 2.1933067743820187, + "language_loss": 0.69891638, + "learning_rate": 2.168059466862001e-06, + "loss": 0.72086978, + "num_input_tokens_seen": 87581895, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.1796875, + "step": 4067, + "time_per_iteration": 3.4719438552856445 + }, + { + "auxiliary_loss_clip": 0.01140543, + "auxiliary_loss_mlp": 0.01036419, + "balance_loss_clip": 1.04454851, + "balance_loss_mlp": 1.01926458, + "epoch": 0.48914807911982205, + "flos": 22310294590080.0, + "grad_norm": 1.9297704208691269, + "language_loss": 0.82221276, + "learning_rate": 2.167283237516887e-06, + "loss": 0.84398234, + "num_input_tokens_seen": 87600170, + "router_z_loss_clip": 0.95898438, + "router_z_loss_mlp": 0.17138672, + "step": 4068, + "time_per_iteration": 2.922734498977661 + }, + { + "auxiliary_loss_clip": 0.011414, + "auxiliary_loss_mlp": 0.01046739, + "balance_loss_clip": 1.04346538, + "balance_loss_mlp": 1.02791631, + "epoch": 0.4892683220104611, + "flos": 16363020954240.0, + "grad_norm": 3.0073221999537045, + "language_loss": 0.74624974, + "learning_rate": 2.1665069827950383e-06, + "loss": 0.76813114, + "num_input_tokens_seen": 87617455, + "router_z_loss_clip": 0.97851562, + "router_z_loss_mlp": 0.18811035, + "step": 4069, + "time_per_iteration": 2.826946258544922 + }, + { + "auxiliary_loss_clip": 0.0113857, + "auxiliary_loss_mlp": 0.01040979, + "balance_loss_clip": 1.04452538, + "balance_loss_mlp": 1.02480245, + "epoch": 0.4893885649011002, + "flos": 15739144606080.0, + "grad_norm": 1.8043974600442112, + "language_loss": 0.8663004, + "learning_rate": 2.1657307028142126e-06, + "loss": 0.88809586, + "num_input_tokens_seen": 87634995, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.1618042, + "step": 4070, + "time_per_iteration": 2.888664722442627 + }, + { + "auxiliary_loss_clip": 0.01144675, + "auxiliary_loss_mlp": 0.01043074, + "balance_loss_clip": 1.04517353, + "balance_loss_mlp": 1.02526379, + "epoch": 0.48950880779173933, + "flos": 28581984887040.0, + "grad_norm": 1.9597879477770832, + "language_loss": 0.67723739, + "learning_rate": 2.164954397692171e-06, + "loss": 0.69911492, + "num_input_tokens_seen": 87654420, + "router_z_loss_clip": 0.99511719, + "router_z_loss_mlp": 0.17810059, + "step": 4071, + "time_per_iteration": 2.8575563430786133 + }, + { + "auxiliary_loss_clip": 0.01040087, + "auxiliary_loss_mlp": 0.01002607, + "balance_loss_clip": 1.0088222, + "balance_loss_mlp": 1.00096214, + "epoch": 0.4896290506823784, + "flos": 66186310746240.0, + "grad_norm": 1.0942138208639067, + "language_loss": 0.7720654, + "learning_rate": 2.164178067546678e-06, + "loss": 0.79249239, + "num_input_tokens_seen": 87713585, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.01647949, + "step": 4072, + "time_per_iteration": 3.65279221534729 + }, + { + "auxiliary_loss_clip": 0.0114498, + "auxiliary_loss_mlp": 0.01044274, + "balance_loss_clip": 1.04702246, + "balance_loss_mlp": 1.0269053, + "epoch": 0.4897492935730175, + "flos": 12531065207040.0, + "grad_norm": 2.521557629646397, + "language_loss": 0.91007113, + "learning_rate": 2.163401712495504e-06, + "loss": 0.93196368, + "num_input_tokens_seen": 87731280, + "router_z_loss_clip": 0.97900391, + "router_z_loss_mlp": 0.17370605, + "step": 4073, + "time_per_iteration": 2.9148147106170654 + }, + { + "auxiliary_loss_clip": 0.01145107, + "auxiliary_loss_mlp": 0.01049663, + "balance_loss_clip": 1.04630911, + "balance_loss_mlp": 1.03143644, + "epoch": 0.4898695364636566, + "flos": 23476816679040.0, + "grad_norm": 1.560851127438421, + "language_loss": 0.79363734, + "learning_rate": 2.1626253326564194e-06, + "loss": 0.81558502, + "num_input_tokens_seen": 87750230, + "router_z_loss_clip": 0.98681641, + "router_z_loss_mlp": 0.18237305, + "step": 4074, + "time_per_iteration": 2.829737663269043 + }, + { + "auxiliary_loss_clip": 0.01144959, + "auxiliary_loss_mlp": 0.01049544, + "balance_loss_clip": 1.04567575, + "balance_loss_mlp": 1.03185368, + "epoch": 0.48998977935429566, + "flos": 27160209774720.0, + "grad_norm": 1.6133778733811737, + "language_loss": 0.76719928, + "learning_rate": 2.161848928147201e-06, + "loss": 0.78914434, + "num_input_tokens_seen": 87770500, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.17687988, + "step": 4075, + "time_per_iteration": 2.942197799682617 + }, + { + "auxiliary_loss_clip": 0.01144651, + "auxiliary_loss_mlp": 0.01052676, + "balance_loss_clip": 1.04615831, + "balance_loss_mlp": 1.03491402, + "epoch": 0.4901100222449348, + "flos": 20339588856960.0, + "grad_norm": 3.0164109774525034, + "language_loss": 0.80803388, + "learning_rate": 2.161072499085629e-06, + "loss": 0.83000708, + "num_input_tokens_seen": 87789495, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.1776123, + "step": 4076, + "time_per_iteration": 3.0927271842956543 + }, + { + "auxiliary_loss_clip": 0.01147003, + "auxiliary_loss_mlp": 0.01043545, + "balance_loss_clip": 1.04948521, + "balance_loss_mlp": 1.02610517, + "epoch": 0.4902302651355739, + "flos": 30446359384320.0, + "grad_norm": 1.5998807742119276, + "language_loss": 0.83235991, + "learning_rate": 2.160296045589487e-06, + "loss": 0.85426539, + "num_input_tokens_seen": 87812955, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.17443848, + "step": 4077, + "time_per_iteration": 2.887317180633545 + }, + { + "auxiliary_loss_clip": 0.01143738, + "auxiliary_loss_mlp": 0.01045707, + "balance_loss_clip": 1.04468822, + "balance_loss_mlp": 1.02806449, + "epoch": 0.49035050802621294, + "flos": 19174180089600.0, + "grad_norm": 1.7602215173973883, + "language_loss": 0.70055872, + "learning_rate": 2.159519567776562e-06, + "loss": 0.72245324, + "num_input_tokens_seen": 87832605, + "router_z_loss_clip": 0.99023438, + "router_z_loss_mlp": 0.17663574, + "step": 4078, + "time_per_iteration": 2.7874631881713867 + }, + { + "auxiliary_loss_clip": 0.01143979, + "auxiliary_loss_mlp": 0.01040396, + "balance_loss_clip": 1.04533744, + "balance_loss_mlp": 1.02265751, + "epoch": 0.49047075091685205, + "flos": 22228489365120.0, + "grad_norm": 4.33877017935142, + "language_loss": 0.70540655, + "learning_rate": 2.1587430657646463e-06, + "loss": 0.72725034, + "num_input_tokens_seen": 87846040, + "router_z_loss_clip": 0.98681641, + "router_z_loss_mlp": 0.17749023, + "step": 4079, + "time_per_iteration": 2.76802921295166 + }, + { + "auxiliary_loss_clip": 0.01140559, + "auxiliary_loss_mlp": 0.0104276, + "balance_loss_clip": 1.04534316, + "balance_loss_mlp": 1.02464044, + "epoch": 0.4905909938074911, + "flos": 20156516213760.0, + "grad_norm": 1.7912234553106123, + "language_loss": 0.7770682, + "learning_rate": 2.157966539671533e-06, + "loss": 0.79890144, + "num_input_tokens_seen": 87865680, + "router_z_loss_clip": 0.95263672, + "router_z_loss_mlp": 0.18115234, + "step": 4080, + "time_per_iteration": 2.9051785469055176 + }, + { + "auxiliary_loss_clip": 0.01139627, + "auxiliary_loss_mlp": 0.01041548, + "balance_loss_clip": 1.04532826, + "balance_loss_mlp": 1.02464437, + "epoch": 0.4907112366981302, + "flos": 17202217380480.0, + "grad_norm": 1.790208001761146, + "language_loss": 0.67054027, + "learning_rate": 2.157189989615021e-06, + "loss": 0.69235206, + "num_input_tokens_seen": 87884270, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.16894531, + "step": 4081, + "time_per_iteration": 2.750100612640381 + }, + { + "auxiliary_loss_clip": 0.01152232, + "auxiliary_loss_mlp": 0.01041712, + "balance_loss_clip": 1.04991889, + "balance_loss_mlp": 1.02299643, + "epoch": 0.4908314795887693, + "flos": 21688968107520.0, + "grad_norm": 1.6615089069667048, + "language_loss": 0.75209218, + "learning_rate": 2.156413415712913e-06, + "loss": 0.77403158, + "num_input_tokens_seen": 87906320, + "router_z_loss_clip": 1.02539062, + "router_z_loss_mlp": 0.18701172, + "step": 4082, + "time_per_iteration": 2.776262044906616 + }, + { + "auxiliary_loss_clip": 0.01146964, + "auxiliary_loss_mlp": 0.01048804, + "balance_loss_clip": 1.04775977, + "balance_loss_mlp": 1.02980173, + "epoch": 0.4909517224794084, + "flos": 26213676531840.0, + "grad_norm": 1.600345127031657, + "language_loss": 0.78786516, + "learning_rate": 2.155636818083014e-06, + "loss": 0.8098228, + "num_input_tokens_seen": 87927690, + "router_z_loss_clip": 0.99023438, + "router_z_loss_mlp": 0.19006348, + "step": 4083, + "time_per_iteration": 3.082341432571411 + }, + { + "auxiliary_loss_clip": 0.01140858, + "auxiliary_loss_mlp": 0.01053318, + "balance_loss_clip": 1.04512, + "balance_loss_mlp": 1.03604472, + "epoch": 0.4910719653700475, + "flos": 23148377694720.0, + "grad_norm": 1.9998060889249825, + "language_loss": 0.84285784, + "learning_rate": 2.154860196843134e-06, + "loss": 0.86479962, + "num_input_tokens_seen": 87946885, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.17272949, + "step": 4084, + "time_per_iteration": 2.9766995906829834 + }, + { + "auxiliary_loss_clip": 0.01142904, + "auxiliary_loss_mlp": 0.01048627, + "balance_loss_clip": 1.04571152, + "balance_loss_mlp": 1.02989924, + "epoch": 0.4911922082606866, + "flos": 23331845387520.0, + "grad_norm": 3.3222913386977933, + "language_loss": 0.76691723, + "learning_rate": 2.154083552111085e-06, + "loss": 0.78883255, + "num_input_tokens_seen": 87966055, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.18737793, + "step": 4085, + "time_per_iteration": 4.356683254241943 + }, + { + "auxiliary_loss_clip": 0.01142882, + "auxiliary_loss_mlp": 0.01046432, + "balance_loss_clip": 1.04339123, + "balance_loss_mlp": 1.02915859, + "epoch": 0.49131245115132566, + "flos": 29203239542400.0, + "grad_norm": 1.8613648241474425, + "language_loss": 0.81809294, + "learning_rate": 2.1533068840046834e-06, + "loss": 0.83998609, + "num_input_tokens_seen": 87986320, + "router_z_loss_clip": 0.99462891, + "router_z_loss_mlp": 0.17272949, + "step": 4086, + "time_per_iteration": 2.9567723274230957 + }, + { + "auxiliary_loss_clip": 0.0114544, + "auxiliary_loss_mlp": 0.01046865, + "balance_loss_clip": 1.04770076, + "balance_loss_mlp": 1.02791119, + "epoch": 0.49143269404196477, + "flos": 20147465986560.0, + "grad_norm": 2.3046010277196594, + "language_loss": 0.61438674, + "learning_rate": 2.152530192641749e-06, + "loss": 0.6363098, + "num_input_tokens_seen": 88001230, + "router_z_loss_clip": 0.97558594, + "router_z_loss_mlp": 0.18933105, + "step": 4087, + "time_per_iteration": 3.713771104812622 + }, + { + "auxiliary_loss_clip": 0.01141823, + "auxiliary_loss_mlp": 0.01042831, + "balance_loss_clip": 1.04626858, + "balance_loss_mlp": 1.02615333, + "epoch": 0.4915529369326039, + "flos": 24389809597440.0, + "grad_norm": 2.1751083055577345, + "language_loss": 0.72846997, + "learning_rate": 2.1517534781401068e-06, + "loss": 0.7503165, + "num_input_tokens_seen": 88019110, + "router_z_loss_clip": 0.95507812, + "router_z_loss_mlp": 0.16687012, + "step": 4088, + "time_per_iteration": 2.8636906147003174 + }, + { + "auxiliary_loss_clip": 0.01143099, + "auxiliary_loss_mlp": 0.01041172, + "balance_loss_clip": 1.04648149, + "balance_loss_mlp": 1.02383947, + "epoch": 0.49167317982324293, + "flos": 10524305197440.0, + "grad_norm": 2.126995940295044, + "language_loss": 0.6920374, + "learning_rate": 2.150976740617581e-06, + "loss": 0.71388006, + "num_input_tokens_seen": 88035670, + "router_z_loss_clip": 0.96435547, + "router_z_loss_mlp": 0.17346191, + "step": 4089, + "time_per_iteration": 2.8612852096557617 + }, + { + "auxiliary_loss_clip": 0.01145992, + "auxiliary_loss_mlp": 0.01047822, + "balance_loss_clip": 1.04489231, + "balance_loss_mlp": 1.02867651, + "epoch": 0.49179342271388204, + "flos": 25593427457280.0, + "grad_norm": 2.4490541828450065, + "language_loss": 0.71432608, + "learning_rate": 2.150199980192006e-06, + "loss": 0.73626423, + "num_input_tokens_seen": 88054790, + "router_z_loss_clip": 1.01171875, + "router_z_loss_mlp": 0.19152832, + "step": 4090, + "time_per_iteration": 3.6779427528381348 + }, + { + "auxiliary_loss_clip": 0.01140286, + "auxiliary_loss_mlp": 0.01041956, + "balance_loss_clip": 1.04523945, + "balance_loss_mlp": 1.02538633, + "epoch": 0.49191366560452116, + "flos": 21102043875840.0, + "grad_norm": 7.839649787494465, + "language_loss": 0.80885172, + "learning_rate": 2.1494231969812114e-06, + "loss": 0.83067411, + "num_input_tokens_seen": 88073780, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.16564941, + "step": 4091, + "time_per_iteration": 2.786547899246216 + }, + { + "auxiliary_loss_clip": 0.01142337, + "auxiliary_loss_mlp": 0.01041207, + "balance_loss_clip": 1.04692495, + "balance_loss_mlp": 1.02448153, + "epoch": 0.4920339084951602, + "flos": 26067520091520.0, + "grad_norm": 2.748441039877881, + "language_loss": 0.81027961, + "learning_rate": 2.1486463911030372e-06, + "loss": 0.83211505, + "num_input_tokens_seen": 88094430, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.16711426, + "step": 4092, + "time_per_iteration": 2.878166675567627 + }, + { + "auxiliary_loss_clip": 0.011407, + "auxiliary_loss_mlp": 0.01045812, + "balance_loss_clip": 1.04344165, + "balance_loss_mlp": 1.02734709, + "epoch": 0.4921541513857993, + "flos": 25081269384960.0, + "grad_norm": 1.7078130529411262, + "language_loss": 0.74145043, + "learning_rate": 2.147869562675324e-06, + "loss": 0.76331556, + "num_input_tokens_seen": 88113400, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.18457031, + "step": 4093, + "time_per_iteration": 3.6837923526763916 + }, + { + "auxiliary_loss_clip": 0.01146984, + "auxiliary_loss_mlp": 0.01041207, + "balance_loss_clip": 1.0462631, + "balance_loss_mlp": 1.02253866, + "epoch": 0.49227439427643843, + "flos": 24389809597440.0, + "grad_norm": 2.048667037906498, + "language_loss": 0.7260834, + "learning_rate": 2.147092711815915e-06, + "loss": 0.74796534, + "num_input_tokens_seen": 88132750, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.18676758, + "step": 4094, + "time_per_iteration": 2.8516197204589844 + }, + { + "auxiliary_loss_clip": 0.01144322, + "auxiliary_loss_mlp": 0.01045018, + "balance_loss_clip": 1.04625165, + "balance_loss_mlp": 1.0267427, + "epoch": 0.4923946371670775, + "flos": 11363753018880.0, + "grad_norm": 2.448478256888965, + "language_loss": 0.86473715, + "learning_rate": 2.1463158386426593e-06, + "loss": 0.88663054, + "num_input_tokens_seen": 88150560, + "router_z_loss_clip": 0.98095703, + "router_z_loss_mlp": 0.18261719, + "step": 4095, + "time_per_iteration": 2.8134243488311768 + }, + { + "auxiliary_loss_clip": 0.01147335, + "auxiliary_loss_mlp": 0.01045136, + "balance_loss_clip": 1.04718041, + "balance_loss_mlp": 1.02510881, + "epoch": 0.4925148800577166, + "flos": 30445964334720.0, + "grad_norm": 2.7188634315939586, + "language_loss": 0.772407, + "learning_rate": 2.145538943273407e-06, + "loss": 0.79433179, + "num_input_tokens_seen": 88170835, + "router_z_loss_clip": 1.00195312, + "router_z_loss_mlp": 0.20043945, + "step": 4096, + "time_per_iteration": 2.9090769290924072 + }, + { + "auxiliary_loss_clip": 0.01147228, + "auxiliary_loss_mlp": 0.01047498, + "balance_loss_clip": 1.04901433, + "balance_loss_mlp": 1.02885365, + "epoch": 0.49263512294835565, + "flos": 20850454039680.0, + "grad_norm": 2.290936715189709, + "language_loss": 0.72254997, + "learning_rate": 2.144762025826013e-06, + "loss": 0.7444973, + "num_input_tokens_seen": 88189925, + "router_z_loss_clip": 0.98242188, + "router_z_loss_mlp": 0.18640137, + "step": 4097, + "time_per_iteration": 2.792440891265869 + }, + { + "auxiliary_loss_clip": 0.01144786, + "auxiliary_loss_mlp": 0.01045173, + "balance_loss_clip": 1.04630554, + "balance_loss_mlp": 1.02621865, + "epoch": 0.49275536583899476, + "flos": 23767477534080.0, + "grad_norm": 2.1820608247009377, + "language_loss": 0.86748898, + "learning_rate": 2.143985086418334e-06, + "loss": 0.88938856, + "num_input_tokens_seen": 88205105, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.1895752, + "step": 4098, + "time_per_iteration": 2.8417885303497314 + }, + { + "auxiliary_loss_clip": 0.01140887, + "auxiliary_loss_mlp": 0.01046855, + "balance_loss_clip": 1.04591548, + "balance_loss_mlp": 1.02943873, + "epoch": 0.4928756087296339, + "flos": 22273522041600.0, + "grad_norm": 1.431072864672277, + "language_loss": 0.76571262, + "learning_rate": 2.1432081251682324e-06, + "loss": 0.78759003, + "num_input_tokens_seen": 88225475, + "router_z_loss_clip": 0.94921875, + "router_z_loss_mlp": 0.17431641, + "step": 4099, + "time_per_iteration": 2.84444522857666 + }, + { + "auxiliary_loss_clip": 0.01143021, + "auxiliary_loss_mlp": 0.01041305, + "balance_loss_clip": 1.04649258, + "balance_loss_mlp": 1.02394819, + "epoch": 0.49299585162027293, + "flos": 19645471463040.0, + "grad_norm": 1.6942274263560122, + "language_loss": 0.87175703, + "learning_rate": 2.142431142193572e-06, + "loss": 0.89360023, + "num_input_tokens_seen": 88243255, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.17370605, + "step": 4100, + "time_per_iteration": 2.7842512130737305 + }, + { + "auxiliary_loss_clip": 0.01140438, + "auxiliary_loss_mlp": 0.01046551, + "balance_loss_clip": 1.04696357, + "balance_loss_mlp": 1.0290513, + "epoch": 0.49311609451091204, + "flos": 38837138497920.0, + "grad_norm": 1.9224481129714142, + "language_loss": 0.71490943, + "learning_rate": 2.1416541376122207e-06, + "loss": 0.73677933, + "num_input_tokens_seen": 88263435, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.1751709, + "step": 4101, + "time_per_iteration": 2.9154253005981445 + }, + { + "auxiliary_loss_clip": 0.01140034, + "auxiliary_loss_mlp": 0.01040229, + "balance_loss_clip": 1.04452753, + "balance_loss_mlp": 1.02261019, + "epoch": 0.49323633740155115, + "flos": 28329102161280.0, + "grad_norm": 1.7594662145414377, + "language_loss": 0.72836053, + "learning_rate": 2.1408771115420496e-06, + "loss": 0.7501632, + "num_input_tokens_seen": 88283295, + "router_z_loss_clip": 0.95507812, + "router_z_loss_mlp": 0.17651367, + "step": 4102, + "time_per_iteration": 2.772557258605957 + }, + { + "auxiliary_loss_clip": 0.0114373, + "auxiliary_loss_mlp": 0.01041176, + "balance_loss_clip": 1.04821348, + "balance_loss_mlp": 1.02465367, + "epoch": 0.4933565802921902, + "flos": 21135584200320.0, + "grad_norm": 2.061814620819499, + "language_loss": 0.64782441, + "learning_rate": 2.140100064100932e-06, + "loss": 0.66967344, + "num_input_tokens_seen": 88299270, + "router_z_loss_clip": 0.95507812, + "router_z_loss_mlp": 0.16491699, + "step": 4103, + "time_per_iteration": 2.8320207595825195 + }, + { + "auxiliary_loss_clip": 0.01141162, + "auxiliary_loss_mlp": 0.01043382, + "balance_loss_clip": 1.04382348, + "balance_loss_mlp": 1.02546453, + "epoch": 0.4934768231828293, + "flos": 18039007595520.0, + "grad_norm": 1.8853666336519053, + "language_loss": 0.75951993, + "learning_rate": 2.139322995406746e-06, + "loss": 0.78136539, + "num_input_tokens_seen": 88316905, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.17919922, + "step": 4104, + "time_per_iteration": 2.705317497253418 + }, + { + "auxiliary_loss_clip": 0.01142269, + "auxiliary_loss_mlp": 0.01047307, + "balance_loss_clip": 1.04634833, + "balance_loss_mlp": 1.02871037, + "epoch": 0.4935970660734684, + "flos": 23469957181440.0, + "grad_norm": 1.8107044948742532, + "language_loss": 0.79896522, + "learning_rate": 2.1385459055773727e-06, + "loss": 0.82086098, + "num_input_tokens_seen": 88335095, + "router_z_loss_clip": 0.95898438, + "router_z_loss_mlp": 0.18579102, + "step": 4105, + "time_per_iteration": 2.769869804382324 + }, + { + "auxiliary_loss_clip": 0.0113864, + "auxiliary_loss_mlp": 0.01041324, + "balance_loss_clip": 1.0457356, + "balance_loss_mlp": 1.02472973, + "epoch": 0.4937173089641075, + "flos": 64479258840960.0, + "grad_norm": 1.9017950712405611, + "language_loss": 0.73590064, + "learning_rate": 2.137768794730696e-06, + "loss": 0.7577002, + "num_input_tokens_seen": 88358545, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.16589355, + "step": 4106, + "time_per_iteration": 3.111095666885376 + }, + { + "auxiliary_loss_clip": 0.01150134, + "auxiliary_loss_mlp": 0.01047462, + "balance_loss_clip": 1.04896903, + "balance_loss_mlp": 1.02816248, + "epoch": 0.4938375518547466, + "flos": 22346025644160.0, + "grad_norm": 1.8806407091962485, + "language_loss": 0.80206943, + "learning_rate": 2.1369916629846026e-06, + "loss": 0.82404542, + "num_input_tokens_seen": 88378295, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.19299316, + "step": 4107, + "time_per_iteration": 2.8046295642852783 + }, + { + "auxiliary_loss_clip": 0.01142922, + "auxiliary_loss_mlp": 0.01045059, + "balance_loss_clip": 1.04626143, + "balance_loss_mlp": 1.02738023, + "epoch": 0.4939577947453857, + "flos": 17858700299520.0, + "grad_norm": 2.159627337357814, + "language_loss": 0.74910367, + "learning_rate": 2.136214510456983e-06, + "loss": 0.77098358, + "num_input_tokens_seen": 88396750, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.17675781, + "step": 4108, + "time_per_iteration": 2.775721311569214 + }, + { + "auxiliary_loss_clip": 0.01039876, + "auxiliary_loss_mlp": 0.01001121, + "balance_loss_clip": 1.00931644, + "balance_loss_mlp": 0.9995122, + "epoch": 0.49407803763602476, + "flos": 70066746875520.0, + "grad_norm": 0.9097309596881217, + "language_loss": 0.63183743, + "learning_rate": 2.1354373372657296e-06, + "loss": 0.65224743, + "num_input_tokens_seen": 88455190, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01611328, + "step": 4109, + "time_per_iteration": 3.2728383541107178 + }, + { + "auxiliary_loss_clip": 0.01144394, + "auxiliary_loss_mlp": 0.01049749, + "balance_loss_clip": 1.04925084, + "balance_loss_mlp": 1.03295207, + "epoch": 0.49419828052666387, + "flos": 24317485562880.0, + "grad_norm": 1.5701042518660893, + "language_loss": 0.71265233, + "learning_rate": 2.1346601435287404e-06, + "loss": 0.73459375, + "num_input_tokens_seen": 88477460, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.16796875, + "step": 4110, + "time_per_iteration": 2.784078598022461 + }, + { + "auxiliary_loss_clip": 0.01143449, + "auxiliary_loss_mlp": 0.01045643, + "balance_loss_clip": 1.0463419, + "balance_loss_mlp": 1.02820289, + "epoch": 0.494318523417303, + "flos": 29386060790400.0, + "grad_norm": 1.8358271193820102, + "language_loss": 0.80383301, + "learning_rate": 2.1338829293639144e-06, + "loss": 0.82572401, + "num_input_tokens_seen": 88497820, + "router_z_loss_clip": 0.97021484, + "router_z_loss_mlp": 0.17456055, + "step": 4111, + "time_per_iteration": 2.8126211166381836 + }, + { + "auxiliary_loss_clip": 0.01142591, + "auxiliary_loss_mlp": 0.01044332, + "balance_loss_clip": 1.04551363, + "balance_loss_mlp": 1.0262959, + "epoch": 0.49443876630794203, + "flos": 15268284195840.0, + "grad_norm": 2.0305181287615763, + "language_loss": 0.82965398, + "learning_rate": 2.1331056948891547e-06, + "loss": 0.85152322, + "num_input_tokens_seen": 88514920, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.18029785, + "step": 4112, + "time_per_iteration": 2.802947759628296 + }, + { + "auxiliary_loss_clip": 0.01140316, + "auxiliary_loss_mlp": 0.01037455, + "balance_loss_clip": 1.04409516, + "balance_loss_mlp": 1.02084947, + "epoch": 0.49455900919858115, + "flos": 12347453859840.0, + "grad_norm": 2.2351707926249516, + "language_loss": 0.76335812, + "learning_rate": 2.1323284402223666e-06, + "loss": 0.78513587, + "num_input_tokens_seen": 88530910, + "router_z_loss_clip": 0.96240234, + "router_z_loss_mlp": 0.16601562, + "step": 4113, + "time_per_iteration": 2.7324345111846924 + }, + { + "auxiliary_loss_clip": 0.01141574, + "auxiliary_loss_mlp": 0.01045291, + "balance_loss_clip": 1.05030763, + "balance_loss_mlp": 1.02819669, + "epoch": 0.4946792520892202, + "flos": 22779610715520.0, + "grad_norm": 1.9004836197717814, + "language_loss": 0.88454592, + "learning_rate": 2.1315511654814597e-06, + "loss": 0.90641451, + "num_input_tokens_seen": 88549320, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.17089844, + "step": 4114, + "time_per_iteration": 3.61517071723938 + }, + { + "auxiliary_loss_clip": 0.01135842, + "auxiliary_loss_mlp": 0.01033657, + "balance_loss_clip": 1.04494667, + "balance_loss_mlp": 1.01793349, + "epoch": 0.4947994949798593, + "flos": 23148126299520.0, + "grad_norm": 1.8355946169483914, + "language_loss": 0.78473127, + "learning_rate": 2.1307738707843456e-06, + "loss": 0.80642629, + "num_input_tokens_seen": 88568985, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.15722656, + "step": 4115, + "time_per_iteration": 2.772493362426758 + }, + { + "auxiliary_loss_clip": 0.01146059, + "auxiliary_loss_mlp": 0.01049765, + "balance_loss_clip": 1.04659986, + "balance_loss_mlp": 1.03011894, + "epoch": 0.4949197378704984, + "flos": 23659997063040.0, + "grad_norm": 2.832133619220279, + "language_loss": 0.68835789, + "learning_rate": 2.1299965562489385e-06, + "loss": 0.71031612, + "num_input_tokens_seen": 88588790, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.1965332, + "step": 4116, + "time_per_iteration": 3.640820026397705 + }, + { + "auxiliary_loss_clip": 0.01142262, + "auxiliary_loss_mlp": 0.01046109, + "balance_loss_clip": 1.04526067, + "balance_loss_mlp": 1.02835846, + "epoch": 0.4950399807611375, + "flos": 26911493026560.0, + "grad_norm": 3.103827709968514, + "language_loss": 0.78825313, + "learning_rate": 2.129219221993158e-06, + "loss": 0.81013685, + "num_input_tokens_seen": 88613575, + "router_z_loss_clip": 0.96972656, + "router_z_loss_mlp": 0.17749023, + "step": 4117, + "time_per_iteration": 2.8573293685913086 + }, + { + "auxiliary_loss_clip": 0.01038626, + "auxiliary_loss_mlp": 0.01002302, + "balance_loss_clip": 1.00793695, + "balance_loss_mlp": 1.00083017, + "epoch": 0.4951602236517766, + "flos": 67315270187520.0, + "grad_norm": 0.787175866135567, + "language_loss": 0.59910333, + "learning_rate": 2.128441868134924e-06, + "loss": 0.61951262, + "num_input_tokens_seen": 88675510, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01470947, + "step": 4118, + "time_per_iteration": 3.329374074935913 + }, + { + "auxiliary_loss_clip": 0.01140687, + "auxiliary_loss_mlp": 0.01042021, + "balance_loss_clip": 1.04600382, + "balance_loss_mlp": 1.02399623, + "epoch": 0.4952804665424157, + "flos": 19901442758400.0, + "grad_norm": 2.242749803446263, + "language_loss": 0.83083844, + "learning_rate": 2.1276644947921606e-06, + "loss": 0.85266554, + "num_input_tokens_seen": 88694425, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.18017578, + "step": 4119, + "time_per_iteration": 3.533881425857544 + }, + { + "auxiliary_loss_clip": 0.0114647, + "auxiliary_loss_mlp": 0.01048525, + "balance_loss_clip": 1.04788458, + "balance_loss_mlp": 1.03011894, + "epoch": 0.49540070943305475, + "flos": 18806813740800.0, + "grad_norm": 2.3003199146928814, + "language_loss": 0.82507002, + "learning_rate": 2.126887102082795e-06, + "loss": 0.84701997, + "num_input_tokens_seen": 88714450, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.18408203, + "step": 4120, + "time_per_iteration": 2.7570419311523438 + }, + { + "auxiliary_loss_clip": 0.01142796, + "auxiliary_loss_mlp": 0.01047317, + "balance_loss_clip": 1.0480175, + "balance_loss_mlp": 1.03042507, + "epoch": 0.49552095232369386, + "flos": 24934179191040.0, + "grad_norm": 1.983903673811977, + "language_loss": 0.7007817, + "learning_rate": 2.126109690124757e-06, + "loss": 0.72268283, + "num_input_tokens_seen": 88735265, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.16894531, + "step": 4121, + "time_per_iteration": 2.7975142002105713 + }, + { + "auxiliary_loss_clip": 0.01141757, + "auxiliary_loss_mlp": 0.01046057, + "balance_loss_clip": 1.04577541, + "balance_loss_mlp": 1.02813971, + "epoch": 0.495641195214333, + "flos": 22857249962880.0, + "grad_norm": 1.7386083956417921, + "language_loss": 0.71264684, + "learning_rate": 2.1253322590359786e-06, + "loss": 0.73452497, + "num_input_tokens_seen": 88754600, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.17919922, + "step": 4122, + "time_per_iteration": 3.5740394592285156 + }, + { + "auxiliary_loss_clip": 0.01145183, + "auxiliary_loss_mlp": 0.01038093, + "balance_loss_clip": 1.04827023, + "balance_loss_mlp": 1.01885271, + "epoch": 0.49576143810497203, + "flos": 25769748343680.0, + "grad_norm": 1.5752554534824004, + "language_loss": 0.74358654, + "learning_rate": 2.124554808934397e-06, + "loss": 0.7654193, + "num_input_tokens_seen": 88775180, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.19238281, + "step": 4123, + "time_per_iteration": 2.854541778564453 + }, + { + "auxiliary_loss_clip": 0.01140291, + "auxiliary_loss_mlp": 0.01039824, + "balance_loss_clip": 1.04625118, + "balance_loss_mlp": 1.02235973, + "epoch": 0.49588168099561114, + "flos": 22128838058880.0, + "grad_norm": 1.79132049986298, + "language_loss": 0.72946721, + "learning_rate": 2.1237773399379496e-06, + "loss": 0.75126839, + "num_input_tokens_seen": 88796145, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.17468262, + "step": 4124, + "time_per_iteration": 2.802147150039673 + }, + { + "auxiliary_loss_clip": 0.01142212, + "auxiliary_loss_mlp": 0.01044639, + "balance_loss_clip": 1.04333258, + "balance_loss_mlp": 1.02595913, + "epoch": 0.49600192388625025, + "flos": 24387331559040.0, + "grad_norm": 1.6671391511929985, + "language_loss": 0.87109923, + "learning_rate": 2.122999852164578e-06, + "loss": 0.89296782, + "num_input_tokens_seen": 88816765, + "router_z_loss_clip": 0.98925781, + "router_z_loss_mlp": 0.18688965, + "step": 4125, + "time_per_iteration": 2.812713146209717 + }, + { + "auxiliary_loss_clip": 0.01146508, + "auxiliary_loss_mlp": 0.01043265, + "balance_loss_clip": 1.04892218, + "balance_loss_mlp": 1.02617002, + "epoch": 0.4961221667768893, + "flos": 22857429530880.0, + "grad_norm": 2.0180977984916977, + "language_loss": 0.59244716, + "learning_rate": 2.122222345732227e-06, + "loss": 0.61434489, + "num_input_tokens_seen": 88836680, + "router_z_loss_clip": 0.97558594, + "router_z_loss_mlp": 0.17089844, + "step": 4126, + "time_per_iteration": 2.9826385974884033 + }, + { + "auxiliary_loss_clip": 0.01147836, + "auxiliary_loss_mlp": 0.01046466, + "balance_loss_clip": 1.04987979, + "balance_loss_mlp": 1.02848935, + "epoch": 0.4962424096675284, + "flos": 17858089768320.0, + "grad_norm": 1.9863911626681823, + "language_loss": 0.83135515, + "learning_rate": 2.121444820758843e-06, + "loss": 0.85329819, + "num_input_tokens_seen": 88855320, + "router_z_loss_clip": 0.97998047, + "router_z_loss_mlp": 0.17980957, + "step": 4127, + "time_per_iteration": 2.8101112842559814 + }, + { + "auxiliary_loss_clip": 0.01151982, + "auxiliary_loss_mlp": 0.01049167, + "balance_loss_clip": 1.04831839, + "balance_loss_mlp": 1.02934265, + "epoch": 0.49636265255816747, + "flos": 21793611404160.0, + "grad_norm": 2.0634126411460603, + "language_loss": 0.78466225, + "learning_rate": 2.120667277362376e-06, + "loss": 0.80667377, + "num_input_tokens_seen": 88874035, + "router_z_loss_clip": 1.03613281, + "router_z_loss_mlp": 0.19812012, + "step": 4128, + "time_per_iteration": 2.756730079650879 + }, + { + "auxiliary_loss_clip": 0.01143083, + "auxiliary_loss_mlp": 0.01046248, + "balance_loss_clip": 1.04721808, + "balance_loss_mlp": 1.02778256, + "epoch": 0.4964828954488066, + "flos": 16358603581440.0, + "grad_norm": 2.297556207534718, + "language_loss": 0.84818459, + "learning_rate": 2.1198897156607796e-06, + "loss": 0.87007791, + "num_input_tokens_seen": 88891390, + "router_z_loss_clip": 0.95898438, + "router_z_loss_mlp": 0.18444824, + "step": 4129, + "time_per_iteration": 2.716475486755371 + }, + { + "auxiliary_loss_clip": 0.01144892, + "auxiliary_loss_mlp": 0.01041236, + "balance_loss_clip": 1.04611444, + "balance_loss_mlp": 1.02262735, + "epoch": 0.4966031383394457, + "flos": 24711101775360.0, + "grad_norm": 1.9438304882359874, + "language_loss": 0.74133104, + "learning_rate": 2.1191121357720085e-06, + "loss": 0.7631923, + "num_input_tokens_seen": 88909450, + "router_z_loss_clip": 0.98779297, + "router_z_loss_mlp": 0.18603516, + "step": 4130, + "time_per_iteration": 2.7663629055023193 + }, + { + "auxiliary_loss_clip": 0.01142184, + "auxiliary_loss_mlp": 0.01045937, + "balance_loss_clip": 1.04571342, + "balance_loss_mlp": 1.02892578, + "epoch": 0.49672338123008475, + "flos": 22930615491840.0, + "grad_norm": 1.6819420560462692, + "language_loss": 0.74607599, + "learning_rate": 2.1183345378140206e-06, + "loss": 0.76795721, + "num_input_tokens_seen": 88929195, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.17004395, + "step": 4131, + "time_per_iteration": 2.7855138778686523 + }, + { + "auxiliary_loss_clip": 0.01039964, + "auxiliary_loss_mlp": 0.01004099, + "balance_loss_clip": 1.00921333, + "balance_loss_mlp": 1.00244164, + "epoch": 0.49684362412072386, + "flos": 65976736844160.0, + "grad_norm": 0.8586939368657337, + "language_loss": 0.61965019, + "learning_rate": 2.1175569219047783e-06, + "loss": 0.64009082, + "num_input_tokens_seen": 88990635, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01660156, + "step": 4132, + "time_per_iteration": 3.4765050411224365 + }, + { + "auxiliary_loss_clip": 0.01143211, + "auxiliary_loss_mlp": 0.010424, + "balance_loss_clip": 1.04656267, + "balance_loss_mlp": 1.02571106, + "epoch": 0.49696386701136297, + "flos": 19971288754560.0, + "grad_norm": 1.7218698639119547, + "language_loss": 0.73201752, + "learning_rate": 2.1167792881622437e-06, + "loss": 0.75387359, + "num_input_tokens_seen": 89009655, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.16687012, + "step": 4133, + "time_per_iteration": 3.036219358444214 + }, + { + "auxiliary_loss_clip": 0.01142299, + "auxiliary_loss_mlp": 0.01040817, + "balance_loss_clip": 1.04534853, + "balance_loss_mlp": 1.02377045, + "epoch": 0.497084109902002, + "flos": 24750819239040.0, + "grad_norm": 1.7350955186299808, + "language_loss": 0.8096869, + "learning_rate": 2.116001636704384e-06, + "loss": 0.83151805, + "num_input_tokens_seen": 89030040, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.17041016, + "step": 4134, + "time_per_iteration": 2.9545114040374756 + }, + { + "auxiliary_loss_clip": 0.01146096, + "auxiliary_loss_mlp": 0.01037988, + "balance_loss_clip": 1.04981375, + "balance_loss_mlp": 1.0199281, + "epoch": 0.49720435279264114, + "flos": 21871825269120.0, + "grad_norm": 1.8794802094185992, + "language_loss": 0.80157888, + "learning_rate": 2.1152239676491685e-06, + "loss": 0.82341969, + "num_input_tokens_seen": 89048145, + "router_z_loss_clip": 0.95996094, + "router_z_loss_mlp": 0.18078613, + "step": 4135, + "time_per_iteration": 3.014815330505371 + }, + { + "auxiliary_loss_clip": 0.01141126, + "auxiliary_loss_mlp": 0.01044036, + "balance_loss_clip": 1.04593849, + "balance_loss_mlp": 1.02728677, + "epoch": 0.49732459568328025, + "flos": 23805794367360.0, + "grad_norm": 1.7298753014367814, + "language_loss": 0.73850536, + "learning_rate": 2.114446281114569e-06, + "loss": 0.76035702, + "num_input_tokens_seen": 89067165, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.16748047, + "step": 4136, + "time_per_iteration": 3.049966812133789 + }, + { + "auxiliary_loss_clip": 0.0113969, + "auxiliary_loss_mlp": 0.01042754, + "balance_loss_clip": 1.0471468, + "balance_loss_mlp": 1.02581406, + "epoch": 0.4974448385739193, + "flos": 20047742853120.0, + "grad_norm": 1.7186199862001836, + "language_loss": 0.75916266, + "learning_rate": 2.1136685772185587e-06, + "loss": 0.78098708, + "num_input_tokens_seen": 89086190, + "router_z_loss_clip": 0.92431641, + "router_z_loss_mlp": 0.16955566, + "step": 4137, + "time_per_iteration": 2.8639066219329834 + }, + { + "auxiliary_loss_clip": 0.01144694, + "auxiliary_loss_mlp": 0.0104377, + "balance_loss_clip": 1.04657328, + "balance_loss_mlp": 1.02675867, + "epoch": 0.4975650814645584, + "flos": 24821347593600.0, + "grad_norm": 1.6050952041888182, + "language_loss": 0.78118455, + "learning_rate": 2.1128908560791163e-06, + "loss": 0.80306923, + "num_input_tokens_seen": 89106020, + "router_z_loss_clip": 0.98095703, + "router_z_loss_mlp": 0.17004395, + "step": 4138, + "time_per_iteration": 2.9572575092315674 + }, + { + "auxiliary_loss_clip": 0.01142762, + "auxiliary_loss_mlp": 0.01046568, + "balance_loss_clip": 1.04651475, + "balance_loss_mlp": 1.02917552, + "epoch": 0.4976853243551975, + "flos": 19829477859840.0, + "grad_norm": 1.9344071051953715, + "language_loss": 0.78532511, + "learning_rate": 2.1121131178142203e-06, + "loss": 0.80721843, + "num_input_tokens_seen": 89125385, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.17382812, + "step": 4139, + "time_per_iteration": 3.251339912414551 + }, + { + "auxiliary_loss_clip": 0.01141181, + "auxiliary_loss_mlp": 0.01039345, + "balance_loss_clip": 1.04375184, + "balance_loss_mlp": 1.02236986, + "epoch": 0.4978055672458366, + "flos": 23142990654720.0, + "grad_norm": 1.5304193929135765, + "language_loss": 0.82640994, + "learning_rate": 2.1113353625418544e-06, + "loss": 0.84821522, + "num_input_tokens_seen": 89143935, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.1697998, + "step": 4140, + "time_per_iteration": 2.960012674331665 + }, + { + "auxiliary_loss_clip": 0.01138581, + "auxiliary_loss_mlp": 0.01038178, + "balance_loss_clip": 1.04791307, + "balance_loss_mlp": 1.02254963, + "epoch": 0.4979258101364757, + "flos": 15559914718080.0, + "grad_norm": 1.5443543925473744, + "language_loss": 0.78784025, + "learning_rate": 2.1105575903800017e-06, + "loss": 0.80960786, + "num_input_tokens_seen": 89162655, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.15625, + "step": 4141, + "time_per_iteration": 2.9820101261138916 + }, + { + "auxiliary_loss_clip": 0.01143768, + "auxiliary_loss_mlp": 0.01040597, + "balance_loss_clip": 1.04455793, + "balance_loss_mlp": 1.02179813, + "epoch": 0.4980460530271148, + "flos": 26356169784960.0, + "grad_norm": 1.662443684675289, + "language_loss": 0.85284615, + "learning_rate": 2.1097798014466502e-06, + "loss": 0.87468988, + "num_input_tokens_seen": 89182255, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.18798828, + "step": 4142, + "time_per_iteration": 2.8474605083465576 + }, + { + "auxiliary_loss_clip": 0.01146909, + "auxiliary_loss_mlp": 0.010471, + "balance_loss_clip": 1.04641557, + "balance_loss_mlp": 1.02815771, + "epoch": 0.49816629591775385, + "flos": 17274541415040.0, + "grad_norm": 2.0841005809220943, + "language_loss": 0.58927345, + "learning_rate": 2.109001995859791e-06, + "loss": 0.61121356, + "num_input_tokens_seen": 89201155, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.18945312, + "step": 4143, + "time_per_iteration": 3.608224868774414 + }, + { + "auxiliary_loss_clip": 0.01039731, + "auxiliary_loss_mlp": 0.01001196, + "balance_loss_clip": 1.00902617, + "balance_loss_mlp": 0.99957514, + "epoch": 0.49828653880839296, + "flos": 64930947344640.0, + "grad_norm": 0.7881064017618419, + "language_loss": 0.60059893, + "learning_rate": 2.108224173737415e-06, + "loss": 0.62100828, + "num_input_tokens_seen": 89264455, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01623535, + "step": 4144, + "time_per_iteration": 3.307079315185547 + }, + { + "auxiliary_loss_clip": 0.01142495, + "auxiliary_loss_mlp": 0.01041703, + "balance_loss_clip": 1.04517293, + "balance_loss_mlp": 1.02273726, + "epoch": 0.498406781699032, + "flos": 27484806003840.0, + "grad_norm": 1.7741757253770367, + "language_loss": 0.76201916, + "learning_rate": 2.1074463351975183e-06, + "loss": 0.78386122, + "num_input_tokens_seen": 89283340, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.18969727, + "step": 4145, + "time_per_iteration": 3.7361741065979004 + }, + { + "auxiliary_loss_clip": 0.01138927, + "auxiliary_loss_mlp": 0.01042731, + "balance_loss_clip": 1.04455161, + "balance_loss_mlp": 1.02655435, + "epoch": 0.49852702458967113, + "flos": 31499870307840.0, + "grad_norm": 1.8260894305712583, + "language_loss": 0.71472132, + "learning_rate": 2.106668480358098e-06, + "loss": 0.73653793, + "num_input_tokens_seen": 89303565, + "router_z_loss_clip": 0.94384766, + "router_z_loss_mlp": 0.16162109, + "step": 4146, + "time_per_iteration": 2.8486056327819824 + }, + { + "auxiliary_loss_clip": 0.01148579, + "auxiliary_loss_mlp": 0.01049675, + "balance_loss_clip": 1.0489229, + "balance_loss_mlp": 1.02938604, + "epoch": 0.49864726748031024, + "flos": 22852868503680.0, + "grad_norm": 1.6075790604834284, + "language_loss": 0.70943117, + "learning_rate": 2.105890609337154e-06, + "loss": 0.73141372, + "num_input_tokens_seen": 89322080, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.20263672, + "step": 4147, + "time_per_iteration": 2.9000399112701416 + }, + { + "auxiliary_loss_clip": 0.01039022, + "auxiliary_loss_mlp": 0.01000847, + "balance_loss_clip": 1.0083406, + "balance_loss_mlp": 0.9992134, + "epoch": 0.4987675103709493, + "flos": 70405708544640.0, + "grad_norm": 0.6859178316478853, + "language_loss": 0.63789117, + "learning_rate": 2.1051127222526883e-06, + "loss": 0.65828985, + "num_input_tokens_seen": 89394195, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01635742, + "step": 4148, + "time_per_iteration": 4.316951274871826 + }, + { + "auxiliary_loss_clip": 0.0114334, + "auxiliary_loss_mlp": 0.01040957, + "balance_loss_clip": 1.04925096, + "balance_loss_mlp": 1.02366042, + "epoch": 0.4988877532615884, + "flos": 28767571482240.0, + "grad_norm": 1.5208443501631086, + "language_loss": 0.80614328, + "learning_rate": 2.1043348192227067e-06, + "loss": 0.82798624, + "num_input_tokens_seen": 89414565, + "router_z_loss_clip": 0.93945312, + "router_z_loss_mlp": 0.17297363, + "step": 4149, + "time_per_iteration": 2.853306531906128 + }, + { + "auxiliary_loss_clip": 0.01144443, + "auxiliary_loss_mlp": 0.01052899, + "balance_loss_clip": 1.04908013, + "balance_loss_mlp": 1.03712726, + "epoch": 0.4990079961522275, + "flos": 16872700988160.0, + "grad_norm": 1.6580830536709001, + "language_loss": 0.61735463, + "learning_rate": 2.1035569003652156e-06, + "loss": 0.639328, + "num_input_tokens_seen": 89433195, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.15783691, + "step": 4150, + "time_per_iteration": 2.779487133026123 + }, + { + "auxiliary_loss_clip": 0.01150909, + "auxiliary_loss_mlp": 0.01048389, + "balance_loss_clip": 1.04824114, + "balance_loss_mlp": 1.02782536, + "epoch": 0.4991282390428666, + "flos": 13291042187520.0, + "grad_norm": 1.8502810024842513, + "language_loss": 0.819498, + "learning_rate": 2.1027789657982255e-06, + "loss": 0.84149098, + "num_input_tokens_seen": 89447410, + "router_z_loss_clip": 1.02783203, + "router_z_loss_mlp": 0.20581055, + "step": 4151, + "time_per_iteration": 3.750615119934082 + }, + { + "auxiliary_loss_clip": 0.01142214, + "auxiliary_loss_mlp": 0.01044723, + "balance_loss_clip": 1.04504406, + "balance_loss_mlp": 1.02644837, + "epoch": 0.4992484819335057, + "flos": 21537496454400.0, + "grad_norm": 1.936384326866548, + "language_loss": 0.77289975, + "learning_rate": 2.1020010156397482e-06, + "loss": 0.79476905, + "num_input_tokens_seen": 89464630, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.18261719, + "step": 4152, + "time_per_iteration": 2.8140463829040527 + }, + { + "auxiliary_loss_clip": 0.01139243, + "auxiliary_loss_mlp": 0.01041176, + "balance_loss_clip": 1.04716706, + "balance_loss_mlp": 1.02495158, + "epoch": 0.4993687248241448, + "flos": 24860095390080.0, + "grad_norm": 1.6019933779792752, + "language_loss": 0.77120185, + "learning_rate": 2.101223050007797e-06, + "loss": 0.79300606, + "num_input_tokens_seen": 89483180, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.16235352, + "step": 4153, + "time_per_iteration": 2.974942684173584 + }, + { + "auxiliary_loss_clip": 0.01038961, + "auxiliary_loss_mlp": 0.01000084, + "balance_loss_clip": 1.00800705, + "balance_loss_mlp": 0.99847454, + "epoch": 0.49948896771478385, + "flos": 62941602453120.0, + "grad_norm": 0.8104617287771506, + "language_loss": 0.53797174, + "learning_rate": 2.1004450690203904e-06, + "loss": 0.55836219, + "num_input_tokens_seen": 89539260, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01611328, + "step": 4154, + "time_per_iteration": 3.3714635372161865 + }, + { + "auxiliary_loss_clip": 0.01038273, + "auxiliary_loss_mlp": 0.01001922, + "balance_loss_clip": 1.0075295, + "balance_loss_mlp": 1.00025272, + "epoch": 0.49960921060542296, + "flos": 68284213516800.0, + "grad_norm": 0.8528740013753637, + "language_loss": 0.6334759, + "learning_rate": 2.099667072795546e-06, + "loss": 0.65387785, + "num_input_tokens_seen": 89601380, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01672363, + "step": 4155, + "time_per_iteration": 3.3685736656188965 + }, + { + "auxiliary_loss_clip": 0.01145235, + "auxiliary_loss_mlp": 0.01040756, + "balance_loss_clip": 1.04609501, + "balance_loss_mlp": 1.02367318, + "epoch": 0.49972945349606207, + "flos": 23659350618240.0, + "grad_norm": 1.7393669868015464, + "language_loss": 0.79756486, + "learning_rate": 2.0988890614512864e-06, + "loss": 0.81942475, + "num_input_tokens_seen": 89621270, + "router_z_loss_clip": 0.99072266, + "router_z_loss_mlp": 0.17089844, + "step": 4156, + "time_per_iteration": 2.8081564903259277 + }, + { + "auxiliary_loss_clip": 0.01144867, + "auxiliary_loss_mlp": 0.01047035, + "balance_loss_clip": 1.04771233, + "balance_loss_mlp": 1.02845025, + "epoch": 0.4998496963867011, + "flos": 19755825022080.0, + "grad_norm": 1.9509460437127202, + "language_loss": 0.84099197, + "learning_rate": 2.098111035105635e-06, + "loss": 0.86291099, + "num_input_tokens_seen": 89639695, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.18579102, + "step": 4157, + "time_per_iteration": 2.8656344413757324 + }, + { + "auxiliary_loss_clip": 0.0114573, + "auxiliary_loss_mlp": 0.01051876, + "balance_loss_clip": 1.04819918, + "balance_loss_mlp": 1.03337491, + "epoch": 0.49996993927734024, + "flos": 22265728790400.0, + "grad_norm": 1.7777103906179037, + "language_loss": 0.7276448, + "learning_rate": 2.0973329938766176e-06, + "loss": 0.74962085, + "num_input_tokens_seen": 89657125, + "router_z_loss_clip": 0.97509766, + "router_z_loss_mlp": 0.18493652, + "step": 4158, + "time_per_iteration": 2.8056247234344482 + }, + { + "auxiliary_loss_clip": 0.01149472, + "auxiliary_loss_mlp": 0.01048299, + "balance_loss_clip": 1.0486908, + "balance_loss_mlp": 1.03032267, + "epoch": 0.5000901821679793, + "flos": 23327212533120.0, + "grad_norm": 2.1353265044140564, + "language_loss": 0.79163206, + "learning_rate": 2.0965549378822618e-06, + "loss": 0.81360972, + "num_input_tokens_seen": 89678415, + "router_z_loss_clip": 1.00634766, + "router_z_loss_mlp": 0.17993164, + "step": 4159, + "time_per_iteration": 2.862997055053711 + }, + { + "auxiliary_loss_clip": 0.01146131, + "auxiliary_loss_mlp": 0.01048021, + "balance_loss_clip": 1.04918051, + "balance_loss_mlp": 1.02980554, + "epoch": 0.5002104250586185, + "flos": 20339014239360.0, + "grad_norm": 1.947349416216437, + "language_loss": 0.83935815, + "learning_rate": 2.095776867240599e-06, + "loss": 0.86129963, + "num_input_tokens_seen": 89695405, + "router_z_loss_clip": 0.97021484, + "router_z_loss_mlp": 0.18212891, + "step": 4160, + "time_per_iteration": 2.7715861797332764 + }, + { + "auxiliary_loss_clip": 0.01144294, + "auxiliary_loss_mlp": 0.01046631, + "balance_loss_clip": 1.0474782, + "balance_loss_mlp": 1.0294528, + "epoch": 0.5003306679492575, + "flos": 13991372634240.0, + "grad_norm": 2.281480167982304, + "language_loss": 0.82718927, + "learning_rate": 2.094998782069661e-06, + "loss": 0.8490985, + "num_input_tokens_seen": 89713110, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.17163086, + "step": 4161, + "time_per_iteration": 2.853410243988037 + }, + { + "auxiliary_loss_clip": 0.01141042, + "auxiliary_loss_mlp": 0.01047324, + "balance_loss_clip": 1.04462576, + "balance_loss_mlp": 1.0289185, + "epoch": 0.5004509108398966, + "flos": 27672762896640.0, + "grad_norm": 1.8238691265646103, + "language_loss": 0.75987446, + "learning_rate": 2.0942206824874845e-06, + "loss": 0.78175819, + "num_input_tokens_seen": 89735885, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.18395996, + "step": 4162, + "time_per_iteration": 2.772826671600342 + }, + { + "auxiliary_loss_clip": 0.01144192, + "auxiliary_loss_mlp": 0.01042672, + "balance_loss_clip": 1.04728997, + "balance_loss_mlp": 1.02462339, + "epoch": 0.5005711537305357, + "flos": 14976186796800.0, + "grad_norm": 2.1914215473395475, + "language_loss": 0.79196572, + "learning_rate": 2.093442568612105e-06, + "loss": 0.81383431, + "num_input_tokens_seen": 89753690, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.18054199, + "step": 4163, + "time_per_iteration": 2.7354538440704346 + }, + { + "auxiliary_loss_clip": 0.01149247, + "auxiliary_loss_mlp": 0.01043906, + "balance_loss_clip": 1.04817247, + "balance_loss_mlp": 1.02564299, + "epoch": 0.5006913966211748, + "flos": 26503259978880.0, + "grad_norm": 1.6481510181825532, + "language_loss": 0.85063899, + "learning_rate": 2.0926644405615613e-06, + "loss": 0.87257051, + "num_input_tokens_seen": 89774590, + "router_z_loss_clip": 1.01171875, + "router_z_loss_mlp": 0.18261719, + "step": 4164, + "time_per_iteration": 2.7972545623779297 + }, + { + "auxiliary_loss_clip": 0.01140826, + "auxiliary_loss_mlp": 0.01036878, + "balance_loss_clip": 1.04472589, + "balance_loss_mlp": 1.01937795, + "epoch": 0.5008116395118138, + "flos": 20449295971200.0, + "grad_norm": 2.526085816153438, + "language_loss": 0.81155807, + "learning_rate": 2.091886298453897e-06, + "loss": 0.83333504, + "num_input_tokens_seen": 89792775, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.17480469, + "step": 4165, + "time_per_iteration": 2.710691452026367 + }, + { + "auxiliary_loss_clip": 0.01141571, + "auxiliary_loss_mlp": 0.01041648, + "balance_loss_clip": 1.04870439, + "balance_loss_mlp": 1.02486372, + "epoch": 0.500931882402453, + "flos": 21579871524480.0, + "grad_norm": 1.8669811153522093, + "language_loss": 0.7337473, + "learning_rate": 2.091108142407153e-06, + "loss": 0.75557947, + "num_input_tokens_seen": 89811515, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.16766357, + "step": 4166, + "time_per_iteration": 2.790278911590576 + }, + { + "auxiliary_loss_clip": 0.01039011, + "auxiliary_loss_mlp": 0.010095, + "balance_loss_clip": 1.0082761, + "balance_loss_mlp": 1.00807571, + "epoch": 0.5010521252930921, + "flos": 57785011925760.0, + "grad_norm": 0.8373586028152256, + "language_loss": 0.6242305, + "learning_rate": 2.090329972539377e-06, + "loss": 0.64471561, + "num_input_tokens_seen": 89870080, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01422119, + "step": 4167, + "time_per_iteration": 3.2934741973876953 + }, + { + "auxiliary_loss_clip": 0.01146703, + "auxiliary_loss_mlp": 0.01042144, + "balance_loss_clip": 1.04906368, + "balance_loss_mlp": 1.02564561, + "epoch": 0.5011723681837311, + "flos": 18625500864000.0, + "grad_norm": 2.2538186033361423, + "language_loss": 0.68163365, + "learning_rate": 2.089551788968616e-06, + "loss": 0.70352209, + "num_input_tokens_seen": 89888045, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.16503906, + "step": 4168, + "time_per_iteration": 2.80336332321167 + }, + { + "auxiliary_loss_clip": 0.0103872, + "auxiliary_loss_mlp": 0.01010002, + "balance_loss_clip": 1.00801075, + "balance_loss_mlp": 1.0085057, + "epoch": 0.5012926110743702, + "flos": 55883146608000.0, + "grad_norm": 0.8305625323122151, + "language_loss": 0.60717988, + "learning_rate": 2.08877359181292e-06, + "loss": 0.62766707, + "num_input_tokens_seen": 89944610, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01495361, + "step": 4169, + "time_per_iteration": 3.2573020458221436 + }, + { + "auxiliary_loss_clip": 0.01142944, + "auxiliary_loss_mlp": 0.01045954, + "balance_loss_clip": 1.04537892, + "balance_loss_mlp": 1.02883506, + "epoch": 0.5014128539650093, + "flos": 24238266117120.0, + "grad_norm": 3.6676135293285728, + "language_loss": 0.85439765, + "learning_rate": 2.0879953811903396e-06, + "loss": 0.87628663, + "num_input_tokens_seen": 89959495, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.17126465, + "step": 4170, + "time_per_iteration": 2.8855223655700684 + }, + { + "auxiliary_loss_clip": 0.01144666, + "auxiliary_loss_mlp": 0.01048082, + "balance_loss_clip": 1.04615569, + "balance_loss_mlp": 1.02990222, + "epoch": 0.5015330968556484, + "flos": 27527468382720.0, + "grad_norm": 1.8189968127562273, + "language_loss": 0.78749168, + "learning_rate": 2.08721715721893e-06, + "loss": 0.80941916, + "num_input_tokens_seen": 89978820, + "router_z_loss_clip": 0.98486328, + "router_z_loss_mlp": 0.1817627, + "step": 4171, + "time_per_iteration": 2.859424591064453 + }, + { + "auxiliary_loss_clip": 0.01144084, + "auxiliary_loss_mlp": 0.01040526, + "balance_loss_clip": 1.04630947, + "balance_loss_mlp": 1.02281189, + "epoch": 0.5016533397462875, + "flos": 23800802376960.0, + "grad_norm": 2.2307815005485283, + "language_loss": 0.77145827, + "learning_rate": 2.0864389200167477e-06, + "loss": 0.79330432, + "num_input_tokens_seen": 89997075, + "router_z_loss_clip": 0.9765625, + "router_z_loss_mlp": 0.17700195, + "step": 4172, + "time_per_iteration": 3.712879180908203 + }, + { + "auxiliary_loss_clip": 0.01142789, + "auxiliary_loss_mlp": 0.01040517, + "balance_loss_clip": 1.04725897, + "balance_loss_mlp": 1.02293932, + "epoch": 0.5017735826369266, + "flos": 25295009264640.0, + "grad_norm": 2.2823312025881215, + "language_loss": 0.78863269, + "learning_rate": 2.0856606697018504e-06, + "loss": 0.81046575, + "num_input_tokens_seen": 90015085, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.17584229, + "step": 4173, + "time_per_iteration": 2.8269426822662354 + }, + { + "auxiliary_loss_clip": 0.01141316, + "auxiliary_loss_mlp": 0.01043238, + "balance_loss_clip": 1.04473495, + "balance_loss_mlp": 1.0239259, + "epoch": 0.5018938255275657, + "flos": 16873203778560.0, + "grad_norm": 2.0961361567058345, + "language_loss": 0.73598891, + "learning_rate": 2.084882406392297e-06, + "loss": 0.75783443, + "num_input_tokens_seen": 90033045, + "router_z_loss_clip": 0.96679688, + "router_z_loss_mlp": 0.1932373, + "step": 4174, + "time_per_iteration": 3.6492059230804443 + }, + { + "auxiliary_loss_clip": 0.01142223, + "auxiliary_loss_mlp": 0.01035893, + "balance_loss_clip": 1.04655957, + "balance_loss_mlp": 1.01836896, + "epoch": 0.5020140684182047, + "flos": 25515429073920.0, + "grad_norm": 2.909221130183072, + "language_loss": 0.7145524, + "learning_rate": 2.0841041302061496e-06, + "loss": 0.73633361, + "num_input_tokens_seen": 90052505, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.17529297, + "step": 4175, + "time_per_iteration": 2.779601812362671 + }, + { + "auxiliary_loss_clip": 0.01144907, + "auxiliary_loss_mlp": 0.01041698, + "balance_loss_clip": 1.0481596, + "balance_loss_mlp": 1.02322042, + "epoch": 0.5021343113088439, + "flos": 23659278791040.0, + "grad_norm": 1.941674440804943, + "language_loss": 0.7536425, + "learning_rate": 2.083325841261473e-06, + "loss": 0.77550852, + "num_input_tokens_seen": 90071565, + "router_z_loss_clip": 0.96777344, + "router_z_loss_mlp": 0.18481445, + "step": 4176, + "time_per_iteration": 2.915605068206787 + }, + { + "auxiliary_loss_clip": 0.01142501, + "auxiliary_loss_mlp": 0.01043277, + "balance_loss_clip": 1.04566419, + "balance_loss_mlp": 1.0242517, + "epoch": 0.502254554199483, + "flos": 24534673148160.0, + "grad_norm": 1.8827845196164628, + "language_loss": 0.66135228, + "learning_rate": 2.0825475396763322e-06, + "loss": 0.68321013, + "num_input_tokens_seen": 90092215, + "router_z_loss_clip": 0.96777344, + "router_z_loss_mlp": 0.19018555, + "step": 4177, + "time_per_iteration": 3.6586074829101562 + }, + { + "auxiliary_loss_clip": 0.01142411, + "auxiliary_loss_mlp": 0.01040625, + "balance_loss_clip": 1.04746246, + "balance_loss_mlp": 1.02326787, + "epoch": 0.502374797090122, + "flos": 34240285607040.0, + "grad_norm": 1.6589715110032972, + "language_loss": 0.65618312, + "learning_rate": 2.081769225568796e-06, + "loss": 0.6780135, + "num_input_tokens_seen": 90114665, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.17346191, + "step": 4178, + "time_per_iteration": 2.9212398529052734 + }, + { + "auxiliary_loss_clip": 0.01144846, + "auxiliary_loss_mlp": 0.01045304, + "balance_loss_clip": 1.04697084, + "balance_loss_mlp": 1.02549136, + "epoch": 0.5024950399807612, + "flos": 26031106679040.0, + "grad_norm": 1.5666044318576862, + "language_loss": 0.76078141, + "learning_rate": 2.0809908990569327e-06, + "loss": 0.7826829, + "num_input_tokens_seen": 90136445, + "router_z_loss_clip": 0.97705078, + "router_z_loss_mlp": 0.19812012, + "step": 4179, + "time_per_iteration": 2.8383636474609375 + }, + { + "auxiliary_loss_clip": 0.01144639, + "auxiliary_loss_mlp": 0.01043798, + "balance_loss_clip": 1.0486902, + "balance_loss_mlp": 1.0264647, + "epoch": 0.5026152828714002, + "flos": 21252438120960.0, + "grad_norm": 1.7667940955598005, + "language_loss": 0.7954545, + "learning_rate": 2.0802125602588146e-06, + "loss": 0.81733894, + "num_input_tokens_seen": 90155710, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.17333984, + "step": 4180, + "time_per_iteration": 3.583071708679199 + }, + { + "auxiliary_loss_clip": 0.01142076, + "auxiliary_loss_mlp": 0.01038809, + "balance_loss_clip": 1.04628587, + "balance_loss_mlp": 1.02054596, + "epoch": 0.5027355257620393, + "flos": 30956111245440.0, + "grad_norm": 2.10322695336384, + "language_loss": 0.66341048, + "learning_rate": 2.0794342092925146e-06, + "loss": 0.68521929, + "num_input_tokens_seen": 90176845, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.18261719, + "step": 4181, + "time_per_iteration": 2.8224315643310547 + }, + { + "auxiliary_loss_clip": 0.01139015, + "auxiliary_loss_mlp": 0.01038582, + "balance_loss_clip": 1.04685152, + "balance_loss_mlp": 1.02197647, + "epoch": 0.5028557686526784, + "flos": 24791147233920.0, + "grad_norm": 1.8707561092744722, + "language_loss": 0.67960203, + "learning_rate": 2.078655846276108e-06, + "loss": 0.70137799, + "num_input_tokens_seen": 90197175, + "router_z_loss_clip": 0.92089844, + "router_z_loss_mlp": 0.16589355, + "step": 4182, + "time_per_iteration": 2.8093864917755127 + }, + { + "auxiliary_loss_clip": 0.01146216, + "auxiliary_loss_mlp": 0.01044944, + "balance_loss_clip": 1.04841745, + "balance_loss_mlp": 1.02489281, + "epoch": 0.5029760115433175, + "flos": 22966992990720.0, + "grad_norm": 2.1893764799664512, + "language_loss": 0.68936598, + "learning_rate": 2.0778774713276727e-06, + "loss": 0.71127754, + "num_input_tokens_seen": 90216650, + "router_z_loss_clip": 0.97705078, + "router_z_loss_mlp": 0.20056152, + "step": 4183, + "time_per_iteration": 2.7426230907440186 + }, + { + "auxiliary_loss_clip": 0.01147185, + "auxiliary_loss_mlp": 0.01044772, + "balance_loss_clip": 1.04670906, + "balance_loss_mlp": 1.02549589, + "epoch": 0.5030962544339566, + "flos": 15305164485120.0, + "grad_norm": 2.1323175987162353, + "language_loss": 0.67680895, + "learning_rate": 2.077099084565287e-06, + "loss": 0.69872844, + "num_input_tokens_seen": 90234055, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.19274902, + "step": 4184, + "time_per_iteration": 2.746516466140747 + }, + { + "auxiliary_loss_clip": 0.01144119, + "auxiliary_loss_mlp": 0.01040165, + "balance_loss_clip": 1.04595363, + "balance_loss_mlp": 1.02219987, + "epoch": 0.5032164973245957, + "flos": 24494847943680.0, + "grad_norm": 2.366541526843958, + "language_loss": 0.65354228, + "learning_rate": 2.0763206861070313e-06, + "loss": 0.67538512, + "num_input_tokens_seen": 90253115, + "router_z_loss_clip": 0.98144531, + "router_z_loss_mlp": 0.17980957, + "step": 4185, + "time_per_iteration": 2.745213508605957 + }, + { + "auxiliary_loss_clip": 0.01142618, + "auxiliary_loss_mlp": 0.01042956, + "balance_loss_clip": 1.04523253, + "balance_loss_mlp": 1.02389407, + "epoch": 0.5033367402152348, + "flos": 16213452721920.0, + "grad_norm": 2.2521493339125387, + "language_loss": 0.7537666, + "learning_rate": 2.0755422760709876e-06, + "loss": 0.77562237, + "num_input_tokens_seen": 90270515, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.19055176, + "step": 4186, + "time_per_iteration": 2.785057544708252 + }, + { + "auxiliary_loss_clip": 0.01145223, + "auxiliary_loss_mlp": 0.01045573, + "balance_loss_clip": 1.04789567, + "balance_loss_mlp": 1.02969444, + "epoch": 0.5034569831058738, + "flos": 21391375927680.0, + "grad_norm": 2.7309741500433886, + "language_loss": 0.76926589, + "learning_rate": 2.0747638545752417e-06, + "loss": 0.79117382, + "num_input_tokens_seen": 90289075, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.15881348, + "step": 4187, + "time_per_iteration": 2.7237205505371094 + }, + { + "auxiliary_loss_clip": 0.01141493, + "auxiliary_loss_mlp": 0.01047519, + "balance_loss_clip": 1.04646683, + "balance_loss_mlp": 1.02879095, + "epoch": 0.503577225996513, + "flos": 20558751690240.0, + "grad_norm": 1.8791179669555016, + "language_loss": 0.83201307, + "learning_rate": 2.073985421737878e-06, + "loss": 0.85390317, + "num_input_tokens_seen": 90306385, + "router_z_loss_clip": 0.95019531, + "router_z_loss_mlp": 0.18725586, + "step": 4188, + "time_per_iteration": 2.7591230869293213 + }, + { + "auxiliary_loss_clip": 0.01147377, + "auxiliary_loss_mlp": 0.01052508, + "balance_loss_clip": 1.04814827, + "balance_loss_mlp": 1.03319597, + "epoch": 0.5036974688871521, + "flos": 27229157930880.0, + "grad_norm": 6.770340019587365, + "language_loss": 0.74053496, + "learning_rate": 2.0732069776769844e-06, + "loss": 0.76253378, + "num_input_tokens_seen": 90323795, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.1932373, + "step": 4189, + "time_per_iteration": 2.751171827316284 + }, + { + "auxiliary_loss_clip": 0.01146077, + "auxiliary_loss_mlp": 0.01053381, + "balance_loss_clip": 1.04940295, + "balance_loss_mlp": 1.03644145, + "epoch": 0.5038177117777911, + "flos": 20412164286720.0, + "grad_norm": 1.9264411209190557, + "language_loss": 0.73162746, + "learning_rate": 2.072428522510651e-06, + "loss": 0.75362206, + "num_input_tokens_seen": 90340360, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.16931152, + "step": 4190, + "time_per_iteration": 2.752700090408325 + }, + { + "auxiliary_loss_clip": 0.0114491, + "auxiliary_loss_mlp": 0.01042197, + "balance_loss_clip": 1.047997, + "balance_loss_mlp": 1.02566266, + "epoch": 0.5039379546684303, + "flos": 21907987286400.0, + "grad_norm": 2.2356272682964207, + "language_loss": 0.76602614, + "learning_rate": 2.071650056356968e-06, + "loss": 0.78789717, + "num_input_tokens_seen": 90357900, + "router_z_loss_clip": 0.96972656, + "router_z_loss_mlp": 0.16522217, + "step": 4191, + "time_per_iteration": 2.7708353996276855 + }, + { + "auxiliary_loss_clip": 0.01144635, + "auxiliary_loss_mlp": 0.01038388, + "balance_loss_clip": 1.0486865, + "balance_loss_mlp": 1.02099514, + "epoch": 0.5040581975590693, + "flos": 20010718909440.0, + "grad_norm": 2.0673007226289646, + "language_loss": 0.79869395, + "learning_rate": 2.070871579334028e-06, + "loss": 0.82052416, + "num_input_tokens_seen": 90377010, + "router_z_loss_clip": 0.95996094, + "router_z_loss_mlp": 0.17382812, + "step": 4192, + "time_per_iteration": 2.7394776344299316 + }, + { + "auxiliary_loss_clip": 0.01143155, + "auxiliary_loss_mlp": 0.01043391, + "balance_loss_clip": 1.04601169, + "balance_loss_mlp": 1.02574754, + "epoch": 0.5041784404497084, + "flos": 20959837931520.0, + "grad_norm": 1.5763160048481957, + "language_loss": 0.71702564, + "learning_rate": 2.0700930915599264e-06, + "loss": 0.73889112, + "num_input_tokens_seen": 90396740, + "router_z_loss_clip": 0.97265625, + "router_z_loss_mlp": 0.17663574, + "step": 4193, + "time_per_iteration": 2.810023546218872 + }, + { + "auxiliary_loss_clip": 0.01143791, + "auxiliary_loss_mlp": 0.01040341, + "balance_loss_clip": 1.04782557, + "balance_loss_mlp": 1.0231272, + "epoch": 0.5042986833403476, + "flos": 12495082757760.0, + "grad_norm": 2.2647252755413567, + "language_loss": 0.77917016, + "learning_rate": 2.0693145931527583e-06, + "loss": 0.8010115, + "num_input_tokens_seen": 90413220, + "router_z_loss_clip": 0.95898438, + "router_z_loss_mlp": 0.17199707, + "step": 4194, + "time_per_iteration": 2.7543482780456543 + }, + { + "auxiliary_loss_clip": 0.01138174, + "auxiliary_loss_mlp": 0.01041226, + "balance_loss_clip": 1.04366589, + "balance_loss_mlp": 1.02429819, + "epoch": 0.5044189262309866, + "flos": 29202305788800.0, + "grad_norm": 2.1772881198098704, + "language_loss": 0.77924025, + "learning_rate": 2.068536084230622e-06, + "loss": 0.80103421, + "num_input_tokens_seen": 90435085, + "router_z_loss_clip": 0.94580078, + "router_z_loss_mlp": 0.16918945, + "step": 4195, + "time_per_iteration": 2.860031843185425 + }, + { + "auxiliary_loss_clip": 0.01143458, + "auxiliary_loss_mlp": 0.01043614, + "balance_loss_clip": 1.04540694, + "balance_loss_mlp": 1.02563715, + "epoch": 0.5045391691216257, + "flos": 23873198238720.0, + "grad_norm": 1.9843499236031026, + "language_loss": 0.88714004, + "learning_rate": 2.067757564911616e-06, + "loss": 0.90901077, + "num_input_tokens_seen": 90453660, + "router_z_loss_clip": 0.98144531, + "router_z_loss_mlp": 0.17980957, + "step": 4196, + "time_per_iteration": 2.747272253036499 + }, + { + "auxiliary_loss_clip": 0.01147173, + "auxiliary_loss_mlp": 0.0104659, + "balance_loss_clip": 1.04909444, + "balance_loss_mlp": 1.02926922, + "epoch": 0.5046594120122648, + "flos": 24644990793600.0, + "grad_norm": 1.8392079526810847, + "language_loss": 0.92977935, + "learning_rate": 2.0669790353138407e-06, + "loss": 0.95171696, + "num_input_tokens_seen": 90472625, + "router_z_loss_clip": 0.98095703, + "router_z_loss_mlp": 0.1730957, + "step": 4197, + "time_per_iteration": 2.801553964614868 + }, + { + "auxiliary_loss_clip": 0.01143631, + "auxiliary_loss_mlp": 0.01043383, + "balance_loss_clip": 1.0456816, + "balance_loss_mlp": 1.02552581, + "epoch": 0.5047796549029039, + "flos": 23362835846400.0, + "grad_norm": 3.378248843770173, + "language_loss": 0.73480892, + "learning_rate": 2.0662004955553995e-06, + "loss": 0.75667912, + "num_input_tokens_seen": 90492325, + "router_z_loss_clip": 0.97998047, + "router_z_loss_mlp": 0.17858887, + "step": 4198, + "time_per_iteration": 2.7606215476989746 + }, + { + "auxiliary_loss_clip": 0.01142045, + "auxiliary_loss_mlp": 0.01041225, + "balance_loss_clip": 1.04801488, + "balance_loss_mlp": 1.02584732, + "epoch": 0.5048998977935429, + "flos": 17304095329920.0, + "grad_norm": 1.923497685992243, + "language_loss": 0.76977086, + "learning_rate": 2.065421945754395e-06, + "loss": 0.79160357, + "num_input_tokens_seen": 90510055, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.15368652, + "step": 4199, + "time_per_iteration": 2.896580457687378 + }, + { + "auxiliary_loss_clip": 0.01143638, + "auxiliary_loss_mlp": 0.01047314, + "balance_loss_clip": 1.04702282, + "balance_loss_mlp": 1.02924228, + "epoch": 0.505020140684182, + "flos": 34856979235200.0, + "grad_norm": 1.80935113135683, + "language_loss": 0.7799986, + "learning_rate": 2.0646433860289344e-06, + "loss": 0.80190814, + "num_input_tokens_seen": 90528980, + "router_z_loss_clip": 0.96435547, + "router_z_loss_mlp": 0.18066406, + "step": 4200, + "time_per_iteration": 3.7460880279541016 + }, + { + "auxiliary_loss_clip": 0.01144477, + "auxiliary_loss_mlp": 0.01045473, + "balance_loss_clip": 1.04603219, + "balance_loss_mlp": 1.02641177, + "epoch": 0.5051403835748212, + "flos": 24863974058880.0, + "grad_norm": 2.092166752827619, + "language_loss": 0.83043909, + "learning_rate": 2.0638648164971233e-06, + "loss": 0.85233855, + "num_input_tokens_seen": 90547445, + "router_z_loss_clip": 0.98486328, + "router_z_loss_mlp": 0.19055176, + "step": 4201, + "time_per_iteration": 2.850191354751587 + }, + { + "auxiliary_loss_clip": 0.01140632, + "auxiliary_loss_mlp": 0.01044886, + "balance_loss_clip": 1.04655886, + "balance_loss_mlp": 1.02941871, + "epoch": 0.5052606264654602, + "flos": 20959694277120.0, + "grad_norm": 2.4780540580118893, + "language_loss": 0.88009381, + "learning_rate": 2.06308623727707e-06, + "loss": 0.90194893, + "num_input_tokens_seen": 90567545, + "router_z_loss_clip": 0.93945312, + "router_z_loss_mlp": 0.15454102, + "step": 4202, + "time_per_iteration": 2.8037655353546143 + }, + { + "auxiliary_loss_clip": 0.01146257, + "auxiliary_loss_mlp": 0.01048876, + "balance_loss_clip": 1.04759192, + "balance_loss_mlp": 1.03021991, + "epoch": 0.5053808693560993, + "flos": 19642382893440.0, + "grad_norm": 2.470153091320498, + "language_loss": 0.76047951, + "learning_rate": 2.0623076484868846e-06, + "loss": 0.78243083, + "num_input_tokens_seen": 90585000, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.18652344, + "step": 4203, + "time_per_iteration": 3.5967557430267334 + }, + { + "auxiliary_loss_clip": 0.01042264, + "auxiliary_loss_mlp": 0.0100332, + "balance_loss_clip": 1.01122546, + "balance_loss_mlp": 1.00175881, + "epoch": 0.5055011122467384, + "flos": 67504915019520.0, + "grad_norm": 0.8403588008862887, + "language_loss": 0.60724282, + "learning_rate": 2.061529050244679e-06, + "loss": 0.62769866, + "num_input_tokens_seen": 90644745, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.015625, + "step": 4204, + "time_per_iteration": 3.2337634563446045 + }, + { + "auxiliary_loss_clip": 0.01142349, + "auxiliary_loss_mlp": 0.0105142, + "balance_loss_clip": 1.04500389, + "balance_loss_mlp": 1.03477836, + "epoch": 0.5056213551373775, + "flos": 16872952383360.0, + "grad_norm": 2.034827344207924, + "language_loss": 0.73926109, + "learning_rate": 2.060750442668565e-06, + "loss": 0.76119876, + "num_input_tokens_seen": 90662500, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.16638184, + "step": 4205, + "time_per_iteration": 2.7911226749420166 + }, + { + "auxiliary_loss_clip": 0.01142476, + "auxiliary_loss_mlp": 0.01042218, + "balance_loss_clip": 1.04750609, + "balance_loss_mlp": 1.02490878, + "epoch": 0.5057415980280165, + "flos": 15334179696000.0, + "grad_norm": 2.474971785669662, + "language_loss": 0.64299607, + "learning_rate": 2.059971825876657e-06, + "loss": 0.66484302, + "num_input_tokens_seen": 90677010, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.17297363, + "step": 4206, + "time_per_iteration": 3.635878324508667 + }, + { + "auxiliary_loss_clip": 0.01142833, + "auxiliary_loss_mlp": 0.01046804, + "balance_loss_clip": 1.04579091, + "balance_loss_mlp": 1.03012621, + "epoch": 0.5058618409186557, + "flos": 19025976574080.0, + "grad_norm": 1.8171861529957887, + "language_loss": 0.76434839, + "learning_rate": 2.0591931999870713e-06, + "loss": 0.78624475, + "num_input_tokens_seen": 90695935, + "router_z_loss_clip": 0.97119141, + "router_z_loss_mlp": 0.16687012, + "step": 4207, + "time_per_iteration": 2.7451839447021484 + }, + { + "auxiliary_loss_clip": 0.01041006, + "auxiliary_loss_mlp": 0.0100138, + "balance_loss_clip": 1.01013148, + "balance_loss_mlp": 0.99988359, + "epoch": 0.5059820838092948, + "flos": 63453114080640.0, + "grad_norm": 0.8950543719734172, + "language_loss": 0.57542115, + "learning_rate": 2.0584145651179234e-06, + "loss": 0.59584504, + "num_input_tokens_seen": 90751645, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01495361, + "step": 4208, + "time_per_iteration": 3.2937569618225098 + }, + { + "auxiliary_loss_clip": 0.01146595, + "auxiliary_loss_mlp": 0.01043248, + "balance_loss_clip": 1.04901719, + "balance_loss_mlp": 1.02517581, + "epoch": 0.5061023266999338, + "flos": 15441803821440.0, + "grad_norm": 2.2299750784381174, + "language_loss": 0.79889727, + "learning_rate": 2.0576359213873327e-06, + "loss": 0.82079571, + "num_input_tokens_seen": 90766795, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.18078613, + "step": 4209, + "time_per_iteration": 3.713393211364746 + }, + { + "auxiliary_loss_clip": 0.01149837, + "auxiliary_loss_mlp": 0.01041152, + "balance_loss_clip": 1.04611993, + "balance_loss_mlp": 1.02244806, + "epoch": 0.506222569590573, + "flos": 22451063990400.0, + "grad_norm": 2.691804342297987, + "language_loss": 0.70734048, + "learning_rate": 2.056857268913419e-06, + "loss": 0.72925043, + "num_input_tokens_seen": 90786845, + "router_z_loss_clip": 1.03710938, + "router_z_loss_mlp": 0.18701172, + "step": 4210, + "time_per_iteration": 2.7352726459503174 + }, + { + "auxiliary_loss_clip": 0.01142603, + "auxiliary_loss_mlp": 0.01050405, + "balance_loss_clip": 1.04613674, + "balance_loss_mlp": 1.03261948, + "epoch": 0.506342812481212, + "flos": 17558665994880.0, + "grad_norm": 2.341498929961407, + "language_loss": 0.84141904, + "learning_rate": 2.056078607814303e-06, + "loss": 0.86334914, + "num_input_tokens_seen": 90802630, + "router_z_loss_clip": 0.96582031, + "router_z_loss_mlp": 0.17797852, + "step": 4211, + "time_per_iteration": 2.766835927963257 + }, + { + "auxiliary_loss_clip": 0.01142007, + "auxiliary_loss_mlp": 0.01038994, + "balance_loss_clip": 1.04667115, + "balance_loss_mlp": 1.02144635, + "epoch": 0.5064630553718511, + "flos": 23402050519680.0, + "grad_norm": 1.928796192164899, + "language_loss": 0.78545713, + "learning_rate": 2.055299938208106e-06, + "loss": 0.80726713, + "num_input_tokens_seen": 90823620, + "router_z_loss_clip": 0.95263672, + "router_z_loss_mlp": 0.17529297, + "step": 4212, + "time_per_iteration": 2.759392738342285 + }, + { + "auxiliary_loss_clip": 0.01145928, + "auxiliary_loss_mlp": 0.0105051, + "balance_loss_clip": 1.04789412, + "balance_loss_mlp": 1.03251004, + "epoch": 0.5065832982624903, + "flos": 23987035416960.0, + "grad_norm": 1.5608705263281843, + "language_loss": 0.8623693, + "learning_rate": 2.0545212602129526e-06, + "loss": 0.88433373, + "num_input_tokens_seen": 90843475, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.17980957, + "step": 4213, + "time_per_iteration": 2.8060779571533203 + }, + { + "auxiliary_loss_clip": 0.01143744, + "auxiliary_loss_mlp": 0.01046644, + "balance_loss_clip": 1.04630613, + "balance_loss_mlp": 1.02852428, + "epoch": 0.5067035411531293, + "flos": 21503058289920.0, + "grad_norm": 1.9359500351928678, + "language_loss": 0.66584688, + "learning_rate": 2.0537425739469673e-06, + "loss": 0.68775082, + "num_input_tokens_seen": 90862410, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.18127441, + "step": 4214, + "time_per_iteration": 2.8827879428863525 + }, + { + "auxiliary_loss_clip": 0.01039923, + "auxiliary_loss_mlp": 0.01010114, + "balance_loss_clip": 1.00898361, + "balance_loss_mlp": 1.008564, + "epoch": 0.5068237840437684, + "flos": 65934397687680.0, + "grad_norm": 0.8401039502935705, + "language_loss": 0.59454751, + "learning_rate": 2.052963879528276e-06, + "loss": 0.61504781, + "num_input_tokens_seen": 90922280, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01544189, + "step": 4215, + "time_per_iteration": 3.285416603088379 + }, + { + "auxiliary_loss_clip": 0.01143687, + "auxiliary_loss_mlp": 0.01048627, + "balance_loss_clip": 1.04639959, + "balance_loss_mlp": 1.03165209, + "epoch": 0.5069440269344075, + "flos": 27264206626560.0, + "grad_norm": 2.147180823673953, + "language_loss": 0.764382, + "learning_rate": 2.052185177075007e-06, + "loss": 0.78630519, + "num_input_tokens_seen": 90941850, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.1697998, + "step": 4216, + "time_per_iteration": 2.824406385421753 + }, + { + "auxiliary_loss_clip": 0.0114714, + "auxiliary_loss_mlp": 0.01049609, + "balance_loss_clip": 1.04846442, + "balance_loss_mlp": 1.03228807, + "epoch": 0.5070642698250466, + "flos": 23366319465600.0, + "grad_norm": 1.9868114221410047, + "language_loss": 0.83168221, + "learning_rate": 2.051406466705288e-06, + "loss": 0.85364968, + "num_input_tokens_seen": 90961390, + "router_z_loss_clip": 0.98583984, + "router_z_loss_mlp": 0.17321777, + "step": 4217, + "time_per_iteration": 2.7677125930786133 + }, + { + "auxiliary_loss_clip": 0.01142128, + "auxiliary_loss_mlp": 0.01047182, + "balance_loss_clip": 1.04601312, + "balance_loss_mlp": 1.02983713, + "epoch": 0.5071845127156857, + "flos": 20340127560960.0, + "grad_norm": 2.0970798470876986, + "language_loss": 0.81495678, + "learning_rate": 2.0506277485372486e-06, + "loss": 0.83684993, + "num_input_tokens_seen": 90980215, + "router_z_loss_clip": 0.96044922, + "router_z_loss_mlp": 0.17333984, + "step": 4218, + "time_per_iteration": 2.835763454437256 + }, + { + "auxiliary_loss_clip": 0.01146109, + "auxiliary_loss_mlp": 0.0104115, + "balance_loss_clip": 1.04721689, + "balance_loss_mlp": 1.02313685, + "epoch": 0.5073047556063248, + "flos": 12092955022080.0, + "grad_norm": 2.1758885926621048, + "language_loss": 0.67220616, + "learning_rate": 2.04984902268902e-06, + "loss": 0.69407874, + "num_input_tokens_seen": 90997415, + "router_z_loss_clip": 0.98925781, + "router_z_loss_mlp": 0.18005371, + "step": 4219, + "time_per_iteration": 2.853145122528076 + }, + { + "auxiliary_loss_clip": 0.01145338, + "auxiliary_loss_mlp": 0.01050641, + "balance_loss_clip": 1.0441016, + "balance_loss_mlp": 1.03241396, + "epoch": 0.5074249984969639, + "flos": 19682854542720.0, + "grad_norm": 2.6693229254295567, + "language_loss": 0.75380784, + "learning_rate": 2.0490702892787345e-06, + "loss": 0.77576768, + "num_input_tokens_seen": 91016475, + "router_z_loss_clip": 1.01220703, + "router_z_loss_mlp": 0.18237305, + "step": 4220, + "time_per_iteration": 2.864762783050537 + }, + { + "auxiliary_loss_clip": 0.01135273, + "auxiliary_loss_mlp": 0.0104163, + "balance_loss_clip": 1.04300165, + "balance_loss_mlp": 1.02498817, + "epoch": 0.5075452413876029, + "flos": 28765703975040.0, + "grad_norm": 1.6539388426852748, + "language_loss": 0.62407285, + "learning_rate": 2.0482915484245246e-06, + "loss": 0.64584184, + "num_input_tokens_seen": 91038095, + "router_z_loss_clip": 0.92431641, + "router_z_loss_mlp": 0.16638184, + "step": 4221, + "time_per_iteration": 2.947110652923584 + }, + { + "auxiliary_loss_clip": 0.01143965, + "auxiliary_loss_mlp": 0.0104468, + "balance_loss_clip": 1.04533267, + "balance_loss_mlp": 1.02703702, + "epoch": 0.5076654842782421, + "flos": 20339445202560.0, + "grad_norm": 2.3971878476985826, + "language_loss": 0.83938932, + "learning_rate": 2.047512800244526e-06, + "loss": 0.86127579, + "num_input_tokens_seen": 91053360, + "router_z_loss_clip": 0.98681641, + "router_z_loss_mlp": 0.1763916, + "step": 4222, + "time_per_iteration": 2.8315768241882324 + }, + { + "auxiliary_loss_clip": 0.01143771, + "auxiliary_loss_mlp": 0.0104526, + "balance_loss_clip": 1.04583418, + "balance_loss_mlp": 1.02773643, + "epoch": 0.5077857271688812, + "flos": 26359653404160.0, + "grad_norm": 1.8060513554982967, + "language_loss": 0.79277289, + "learning_rate": 2.046734044856873e-06, + "loss": 0.81466317, + "num_input_tokens_seen": 91072770, + "router_z_loss_clip": 0.97949219, + "router_z_loss_mlp": 0.1751709, + "step": 4223, + "time_per_iteration": 2.7855143547058105 + }, + { + "auxiliary_loss_clip": 0.01141891, + "auxiliary_loss_mlp": 0.01043433, + "balance_loss_clip": 1.04535437, + "balance_loss_mlp": 1.02574182, + "epoch": 0.5079059700595202, + "flos": 21798962530560.0, + "grad_norm": 2.036988164276839, + "language_loss": 0.81674439, + "learning_rate": 2.045955282379702e-06, + "loss": 0.8385976, + "num_input_tokens_seen": 91091430, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.17700195, + "step": 4224, + "time_per_iteration": 2.749028444290161 + }, + { + "auxiliary_loss_clip": 0.01143193, + "auxiliary_loss_mlp": 0.01046886, + "balance_loss_clip": 1.04549789, + "balance_loss_mlp": 1.0285877, + "epoch": 0.5080262129501594, + "flos": 13187943175680.0, + "grad_norm": 2.35658034973089, + "language_loss": 0.76086485, + "learning_rate": 2.045176512931152e-06, + "loss": 0.78276563, + "num_input_tokens_seen": 91106060, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.1829834, + "step": 4225, + "time_per_iteration": 2.745056629180908 + }, + { + "auxiliary_loss_clip": 0.01138674, + "auxiliary_loss_mlp": 0.0104524, + "balance_loss_clip": 1.04667354, + "balance_loss_mlp": 1.0290755, + "epoch": 0.5081464558407984, + "flos": 25301473712640.0, + "grad_norm": 1.9589163174625557, + "language_loss": 0.76277721, + "learning_rate": 2.0443977366293604e-06, + "loss": 0.78461635, + "num_input_tokens_seen": 91124100, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.16149902, + "step": 4226, + "time_per_iteration": 2.757150650024414 + }, + { + "auxiliary_loss_clip": 0.01148017, + "auxiliary_loss_mlp": 0.01047284, + "balance_loss_clip": 1.04775155, + "balance_loss_mlp": 1.02871156, + "epoch": 0.5082666987314375, + "flos": 30951226995840.0, + "grad_norm": 1.855485014839873, + "language_loss": 0.76753455, + "learning_rate": 2.043618953592468e-06, + "loss": 0.7894876, + "num_input_tokens_seen": 91146555, + "router_z_loss_clip": 1.00341797, + "router_z_loss_mlp": 0.18579102, + "step": 4227, + "time_per_iteration": 2.8936612606048584 + }, + { + "auxiliary_loss_clip": 0.01143418, + "auxiliary_loss_mlp": 0.01050222, + "balance_loss_clip": 1.0461477, + "balance_loss_mlp": 1.03189945, + "epoch": 0.5083869416220766, + "flos": 19682495406720.0, + "grad_norm": 1.5248591545018604, + "language_loss": 0.81265986, + "learning_rate": 2.0428401639386144e-06, + "loss": 0.83459628, + "num_input_tokens_seen": 91167120, + "router_z_loss_clip": 0.97265625, + "router_z_loss_mlp": 0.18310547, + "step": 4228, + "time_per_iteration": 2.760181427001953 + }, + { + "auxiliary_loss_clip": 0.01039291, + "auxiliary_loss_mlp": 0.01000514, + "balance_loss_clip": 1.00844884, + "balance_loss_mlp": 0.99896377, + "epoch": 0.5085071845127157, + "flos": 71817535589760.0, + "grad_norm": 0.8223030207189908, + "language_loss": 0.58057106, + "learning_rate": 2.042061367785943e-06, + "loss": 0.60096908, + "num_input_tokens_seen": 91220260, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01550293, + "step": 4229, + "time_per_iteration": 4.082841157913208 + }, + { + "auxiliary_loss_clip": 0.01143592, + "auxiliary_loss_mlp": 0.01042498, + "balance_loss_clip": 1.04545271, + "balance_loss_mlp": 1.02497411, + "epoch": 0.5086274274033548, + "flos": 35951608252800.0, + "grad_norm": 4.768370525822188, + "language_loss": 0.75460434, + "learning_rate": 2.041282565252594e-06, + "loss": 0.77646518, + "num_input_tokens_seen": 91240425, + "router_z_loss_clip": 0.98095703, + "router_z_loss_mlp": 0.1751709, + "step": 4230, + "time_per_iteration": 2.9603259563446045 + }, + { + "auxiliary_loss_clip": 0.01140483, + "auxiliary_loss_mlp": 0.01043401, + "balance_loss_clip": 1.04537463, + "balance_loss_mlp": 1.0251981, + "epoch": 0.5087476702939938, + "flos": 23513732881920.0, + "grad_norm": 1.6993572566721975, + "language_loss": 0.7713052, + "learning_rate": 2.040503756456714e-06, + "loss": 0.79314405, + "num_input_tokens_seen": 91259635, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.18212891, + "step": 4231, + "time_per_iteration": 2.819554328918457 + }, + { + "auxiliary_loss_clip": 0.01139532, + "auxiliary_loss_mlp": 0.01040917, + "balance_loss_clip": 1.04546845, + "balance_loss_mlp": 1.02364326, + "epoch": 0.508867913184633, + "flos": 15122091841920.0, + "grad_norm": 1.795064930065432, + "language_loss": 0.78539574, + "learning_rate": 2.0397249415164456e-06, + "loss": 0.80720019, + "num_input_tokens_seen": 91276990, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.17272949, + "step": 4232, + "time_per_iteration": 3.6121678352355957 + }, + { + "auxiliary_loss_clip": 0.01142613, + "auxiliary_loss_mlp": 0.01042937, + "balance_loss_clip": 1.04641366, + "balance_loss_mlp": 1.02568758, + "epoch": 0.508988156075272, + "flos": 25885309374720.0, + "grad_norm": 1.648909330767022, + "language_loss": 0.79860413, + "learning_rate": 2.0389461205499354e-06, + "loss": 0.82045966, + "num_input_tokens_seen": 91296125, + "router_z_loss_clip": 0.96240234, + "router_z_loss_mlp": 0.17260742, + "step": 4233, + "time_per_iteration": 2.946282386779785 + }, + { + "auxiliary_loss_clip": 0.0113948, + "auxiliary_loss_mlp": 0.01037938, + "balance_loss_clip": 1.04521096, + "balance_loss_mlp": 1.01984191, + "epoch": 0.5091083989659111, + "flos": 13844857057920.0, + "grad_norm": 2.1867046608634237, + "language_loss": 0.73985291, + "learning_rate": 2.03816729367533e-06, + "loss": 0.76162708, + "num_input_tokens_seen": 91314280, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.1809082, + "step": 4234, + "time_per_iteration": 2.8205645084381104 + }, + { + "auxiliary_loss_clip": 0.01145253, + "auxiliary_loss_mlp": 0.01041157, + "balance_loss_clip": 1.04702902, + "balance_loss_mlp": 1.02363372, + "epoch": 0.5092286418565503, + "flos": 21104881050240.0, + "grad_norm": 3.0513787363786737, + "language_loss": 0.71682465, + "learning_rate": 2.0373884610107765e-06, + "loss": 0.73868877, + "num_input_tokens_seen": 91334595, + "router_z_loss_clip": 0.98242188, + "router_z_loss_mlp": 0.17529297, + "step": 4235, + "time_per_iteration": 3.586386203765869 + }, + { + "auxiliary_loss_clip": 0.01137717, + "auxiliary_loss_mlp": 0.01037117, + "balance_loss_clip": 1.04282022, + "balance_loss_mlp": 1.02110696, + "epoch": 0.5093488847471893, + "flos": 18621298972800.0, + "grad_norm": 3.1092728422131732, + "language_loss": 0.69797933, + "learning_rate": 2.0366096226744225e-06, + "loss": 0.71972769, + "num_input_tokens_seen": 91349790, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.16015625, + "step": 4236, + "time_per_iteration": 2.8328733444213867 + }, + { + "auxiliary_loss_clip": 0.01140092, + "auxiliary_loss_mlp": 0.01040641, + "balance_loss_clip": 1.04480195, + "balance_loss_mlp": 1.02399969, + "epoch": 0.5094691276378284, + "flos": 23803783205760.0, + "grad_norm": 2.2309173366862405, + "language_loss": 0.76570743, + "learning_rate": 2.035830778784418e-06, + "loss": 0.78751481, + "num_input_tokens_seen": 91370465, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.16650391, + "step": 4237, + "time_per_iteration": 2.8604633808135986 + }, + { + "auxiliary_loss_clip": 0.01143735, + "auxiliary_loss_mlp": 0.01043374, + "balance_loss_clip": 1.0472188, + "balance_loss_mlp": 1.02548099, + "epoch": 0.5095893705284675, + "flos": 17420410546560.0, + "grad_norm": 1.8120462313452208, + "language_loss": 0.80067301, + "learning_rate": 2.0350519294589134e-06, + "loss": 0.8225441, + "num_input_tokens_seen": 91388505, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.17895508, + "step": 4238, + "time_per_iteration": 3.5825226306915283 + }, + { + "auxiliary_loss_clip": 0.01143255, + "auxiliary_loss_mlp": 0.01049792, + "balance_loss_clip": 1.04393899, + "balance_loss_mlp": 1.03158903, + "epoch": 0.5097096134191066, + "flos": 25849362839040.0, + "grad_norm": 1.6442492638702235, + "language_loss": 0.82572782, + "learning_rate": 2.0342730748160588e-06, + "loss": 0.84765828, + "num_input_tokens_seen": 91408970, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.18200684, + "step": 4239, + "time_per_iteration": 2.7518343925476074 + }, + { + "auxiliary_loss_clip": 0.01141037, + "auxiliary_loss_mlp": 0.01044827, + "balance_loss_clip": 1.04393673, + "balance_loss_mlp": 1.02629018, + "epoch": 0.5098298563097456, + "flos": 27745122844800.0, + "grad_norm": 2.1595536719230086, + "language_loss": 0.70562643, + "learning_rate": 2.033494214974006e-06, + "loss": 0.72748506, + "num_input_tokens_seen": 91430115, + "router_z_loss_clip": 0.97119141, + "router_z_loss_mlp": 0.18530273, + "step": 4240, + "time_per_iteration": 2.826105833053589 + }, + { + "auxiliary_loss_clip": 0.01137375, + "auxiliary_loss_mlp": 0.01043336, + "balance_loss_clip": 1.04594815, + "balance_loss_mlp": 1.02662325, + "epoch": 0.5099500992003848, + "flos": 21358913011200.0, + "grad_norm": 1.7232247896492263, + "language_loss": 0.83828658, + "learning_rate": 2.0327153500509067e-06, + "loss": 0.86009371, + "num_input_tokens_seen": 91449140, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.16699219, + "step": 4241, + "time_per_iteration": 2.7541868686676025 + }, + { + "auxiliary_loss_clip": 0.01144051, + "auxiliary_loss_mlp": 0.01044716, + "balance_loss_clip": 1.04444551, + "balance_loss_mlp": 1.02774096, + "epoch": 0.5100703420910239, + "flos": 19865999013120.0, + "grad_norm": 1.9027595271939117, + "language_loss": 0.84777766, + "learning_rate": 2.031936480164916e-06, + "loss": 0.86966532, + "num_input_tokens_seen": 91466880, + "router_z_loss_clip": 0.99658203, + "router_z_loss_mlp": 0.16955566, + "step": 4242, + "time_per_iteration": 2.7561111450195312 + }, + { + "auxiliary_loss_clip": 0.01142337, + "auxiliary_loss_mlp": 0.01037861, + "balance_loss_clip": 1.04702759, + "balance_loss_mlp": 1.02152932, + "epoch": 0.5101905849816629, + "flos": 24648797635200.0, + "grad_norm": 1.9173076986970468, + "language_loss": 0.80023545, + "learning_rate": 2.0311576054341857e-06, + "loss": 0.82203746, + "num_input_tokens_seen": 91487495, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.16333008, + "step": 4243, + "time_per_iteration": 2.83341908454895 + }, + { + "auxiliary_loss_clip": 0.01144718, + "auxiliary_loss_mlp": 0.01041548, + "balance_loss_clip": 1.04825509, + "balance_loss_mlp": 1.02433431, + "epoch": 0.5103108278723021, + "flos": 22930076787840.0, + "grad_norm": 1.8817149465283658, + "language_loss": 0.6256355, + "learning_rate": 2.0303787259768715e-06, + "loss": 0.64749819, + "num_input_tokens_seen": 91508395, + "router_z_loss_clip": 0.96533203, + "router_z_loss_mlp": 0.17199707, + "step": 4244, + "time_per_iteration": 2.743553876876831 + }, + { + "auxiliary_loss_clip": 0.01140686, + "auxiliary_loss_mlp": 0.01042944, + "balance_loss_clip": 1.04524851, + "balance_loss_mlp": 1.02509797, + "epoch": 0.5104310707629411, + "flos": 21506613736320.0, + "grad_norm": 2.0091263596674453, + "language_loss": 0.68936014, + "learning_rate": 2.0295998419111294e-06, + "loss": 0.71119642, + "num_input_tokens_seen": 91525685, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.1784668, + "step": 4245, + "time_per_iteration": 2.736391067504883 + }, + { + "auxiliary_loss_clip": 0.01140793, + "auxiliary_loss_mlp": 0.01040036, + "balance_loss_clip": 1.04733992, + "balance_loss_mlp": 1.02340639, + "epoch": 0.5105513136535802, + "flos": 14903180403840.0, + "grad_norm": 3.6314015477327994, + "language_loss": 0.74198306, + "learning_rate": 2.028820953355115e-06, + "loss": 0.76379138, + "num_input_tokens_seen": 91543785, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.16625977, + "step": 4246, + "time_per_iteration": 2.7080023288726807 + }, + { + "auxiliary_loss_clip": 0.01144651, + "auxiliary_loss_mlp": 0.01044004, + "balance_loss_clip": 1.0458411, + "balance_loss_mlp": 1.02692163, + "epoch": 0.5106715565442194, + "flos": 22602212421120.0, + "grad_norm": 2.02173807737404, + "language_loss": 0.78423178, + "learning_rate": 2.0280420604269834e-06, + "loss": 0.80611837, + "num_input_tokens_seen": 91563325, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.17077637, + "step": 4247, + "time_per_iteration": 2.767570734024048 + }, + { + "auxiliary_loss_clip": 0.01042494, + "auxiliary_loss_mlp": 0.01004672, + "balance_loss_clip": 1.01157975, + "balance_loss_mlp": 1.00318766, + "epoch": 0.5107917994348584, + "flos": 71027645558400.0, + "grad_norm": 0.7059810632396306, + "language_loss": 0.58873975, + "learning_rate": 2.027263163244895e-06, + "loss": 0.60921133, + "num_input_tokens_seen": 91632450, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01483154, + "step": 4248, + "time_per_iteration": 3.415182113647461 + }, + { + "auxiliary_loss_clip": 0.01138384, + "auxiliary_loss_mlp": 0.01038337, + "balance_loss_clip": 1.04549563, + "balance_loss_mlp": 1.02006245, + "epoch": 0.5109120423254975, + "flos": 24827416992000.0, + "grad_norm": 1.7044536290121148, + "language_loss": 0.74366671, + "learning_rate": 2.026484261927005e-06, + "loss": 0.76543391, + "num_input_tokens_seen": 91651945, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.18273926, + "step": 4249, + "time_per_iteration": 2.773347854614258 + }, + { + "auxiliary_loss_clip": 0.01147578, + "auxiliary_loss_mlp": 0.01051437, + "balance_loss_clip": 1.04885244, + "balance_loss_mlp": 1.03336489, + "epoch": 0.5110322852161366, + "flos": 21247661612160.0, + "grad_norm": 2.484516332092672, + "language_loss": 0.74044502, + "learning_rate": 2.025705356591475e-06, + "loss": 0.7624352, + "num_input_tokens_seen": 91669635, + "router_z_loss_clip": 0.98583984, + "router_z_loss_mlp": 0.18078613, + "step": 4250, + "time_per_iteration": 2.7693912982940674 + }, + { + "auxiliary_loss_clip": 0.01042398, + "auxiliary_loss_mlp": 0.00999641, + "balance_loss_clip": 1.01169324, + "balance_loss_mlp": 0.99817485, + "epoch": 0.5111525281067757, + "flos": 66457114358400.0, + "grad_norm": 0.7631298380470013, + "language_loss": 0.57983875, + "learning_rate": 2.024926447356462e-06, + "loss": 0.60025907, + "num_input_tokens_seen": 91731920, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01464844, + "step": 4251, + "time_per_iteration": 3.1995368003845215 + }, + { + "auxiliary_loss_clip": 0.01146463, + "auxiliary_loss_mlp": 0.01041071, + "balance_loss_clip": 1.04726744, + "balance_loss_mlp": 1.02180707, + "epoch": 0.5112727709974147, + "flos": 14866731077760.0, + "grad_norm": 1.9799128734694234, + "language_loss": 0.78756404, + "learning_rate": 2.024147534340127e-06, + "loss": 0.80943942, + "num_input_tokens_seen": 91749780, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.19274902, + "step": 4252, + "time_per_iteration": 2.758159637451172 + }, + { + "auxiliary_loss_clip": 0.01137526, + "auxiliary_loss_mlp": 0.01041926, + "balance_loss_clip": 1.04192519, + "balance_loss_mlp": 1.02481985, + "epoch": 0.5113930138880539, + "flos": 21177600134400.0, + "grad_norm": 1.6795355711015578, + "language_loss": 0.7967205, + "learning_rate": 2.02336861766063e-06, + "loss": 0.81851506, + "num_input_tokens_seen": 91768840, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.17102051, + "step": 4253, + "time_per_iteration": 2.7338502407073975 + }, + { + "auxiliary_loss_clip": 0.01144274, + "auxiliary_loss_mlp": 0.01046015, + "balance_loss_clip": 1.04672003, + "balance_loss_mlp": 1.02830029, + "epoch": 0.511513256778693, + "flos": 20409111630720.0, + "grad_norm": 1.8527927643920463, + "language_loss": 0.7888695, + "learning_rate": 2.0225896974361327e-06, + "loss": 0.81077236, + "num_input_tokens_seen": 91788945, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.17724609, + "step": 4254, + "time_per_iteration": 2.767188310623169 + }, + { + "auxiliary_loss_clip": 0.01041816, + "auxiliary_loss_mlp": 0.01003933, + "balance_loss_clip": 1.01120615, + "balance_loss_mlp": 1.002455, + "epoch": 0.511633499669332, + "flos": 69879975131520.0, + "grad_norm": 0.8563173531016105, + "language_loss": 0.59932792, + "learning_rate": 2.0218107737847962e-06, + "loss": 0.61978543, + "num_input_tokens_seen": 91850990, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01477051, + "step": 4255, + "time_per_iteration": 3.3382673263549805 + }, + { + "auxiliary_loss_clip": 0.01139412, + "auxiliary_loss_mlp": 0.01046964, + "balance_loss_clip": 1.04436314, + "balance_loss_mlp": 1.0293802, + "epoch": 0.5117537425599712, + "flos": 24097855852800.0, + "grad_norm": 1.847037595157824, + "language_loss": 0.74388099, + "learning_rate": 2.0210318468247826e-06, + "loss": 0.76574475, + "num_input_tokens_seen": 91869960, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.17565918, + "step": 4256, + "time_per_iteration": 2.764817953109741 + }, + { + "auxiliary_loss_clip": 0.0113592, + "auxiliary_loss_mlp": 0.01039103, + "balance_loss_clip": 1.04315794, + "balance_loss_mlp": 1.02280676, + "epoch": 0.5118739854506102, + "flos": 20959550622720.0, + "grad_norm": 1.7301915707689344, + "language_loss": 0.82132775, + "learning_rate": 2.020252916674255e-06, + "loss": 0.84307796, + "num_input_tokens_seen": 91889075, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.16296387, + "step": 4257, + "time_per_iteration": 2.75191068649292 + }, + { + "auxiliary_loss_clip": 0.01140557, + "auxiliary_loss_mlp": 0.01045239, + "balance_loss_clip": 1.04490638, + "balance_loss_mlp": 1.02850151, + "epoch": 0.5119942283412493, + "flos": 17457326749440.0, + "grad_norm": 1.7101502674383544, + "language_loss": 0.81290102, + "learning_rate": 2.019473983451375e-06, + "loss": 0.834759, + "num_input_tokens_seen": 91907495, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.16711426, + "step": 4258, + "time_per_iteration": 3.710331916809082 + }, + { + "auxiliary_loss_clip": 0.01142983, + "auxiliary_loss_mlp": 0.01044769, + "balance_loss_clip": 1.04478526, + "balance_loss_mlp": 1.02784109, + "epoch": 0.5121144712318885, + "flos": 21066743784960.0, + "grad_norm": 1.798473929034673, + "language_loss": 0.71662688, + "learning_rate": 2.0186950472743076e-06, + "loss": 0.73850435, + "num_input_tokens_seen": 91927400, + "router_z_loss_clip": 0.98242188, + "router_z_loss_mlp": 0.16918945, + "step": 4259, + "time_per_iteration": 2.7744836807250977 + }, + { + "auxiliary_loss_clip": 0.01139727, + "auxiliary_loss_mlp": 0.01044852, + "balance_loss_clip": 1.04347372, + "balance_loss_mlp": 1.02800727, + "epoch": 0.5122347141225275, + "flos": 19860791541120.0, + "grad_norm": 1.83829861194476, + "language_loss": 0.73938203, + "learning_rate": 2.0179161082612162e-06, + "loss": 0.76122785, + "num_input_tokens_seen": 91946790, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.16845703, + "step": 4260, + "time_per_iteration": 2.7259445190429688 + }, + { + "auxiliary_loss_clip": 0.01139431, + "auxiliary_loss_mlp": 0.01036508, + "balance_loss_clip": 1.04322183, + "balance_loss_mlp": 1.01909196, + "epoch": 0.5123549570131666, + "flos": 22528487756160.0, + "grad_norm": 1.791349317431679, + "language_loss": 0.72910619, + "learning_rate": 2.017137166530266e-06, + "loss": 0.75086558, + "num_input_tokens_seen": 91966325, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.17419434, + "step": 4261, + "time_per_iteration": 3.6240808963775635 + }, + { + "auxiliary_loss_clip": 0.01140699, + "auxiliary_loss_mlp": 0.01044586, + "balance_loss_clip": 1.04631996, + "balance_loss_mlp": 1.02796793, + "epoch": 0.5124751999038056, + "flos": 20333375804160.0, + "grad_norm": 1.9886991468418669, + "language_loss": 0.80354357, + "learning_rate": 2.0163582221996213e-06, + "loss": 0.82539642, + "num_input_tokens_seen": 91984700, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.16625977, + "step": 4262, + "time_per_iteration": 2.7528834342956543 + }, + { + "auxiliary_loss_clip": 0.01140994, + "auxiliary_loss_mlp": 0.01039241, + "balance_loss_clip": 1.04440022, + "balance_loss_mlp": 1.0223968, + "epoch": 0.5125954427944448, + "flos": 39785970211200.0, + "grad_norm": 1.867972730330241, + "language_loss": 0.67804611, + "learning_rate": 2.015579275387446e-06, + "loss": 0.69984853, + "num_input_tokens_seen": 92010020, + "router_z_loss_clip": 0.96582031, + "router_z_loss_mlp": 0.16845703, + "step": 4263, + "time_per_iteration": 2.8795645236968994 + }, + { + "auxiliary_loss_clip": 0.01142681, + "auxiliary_loss_mlp": 0.01038856, + "balance_loss_clip": 1.04780912, + "balance_loss_mlp": 1.02189302, + "epoch": 0.5127156856850839, + "flos": 29205394358400.0, + "grad_norm": 1.910074928370249, + "language_loss": 0.68377692, + "learning_rate": 2.0148003262119085e-06, + "loss": 0.70559227, + "num_input_tokens_seen": 92030990, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.16955566, + "step": 4264, + "time_per_iteration": 3.565133571624756 + }, + { + "auxiliary_loss_clip": 0.01144151, + "auxiliary_loss_mlp": 0.01042891, + "balance_loss_clip": 1.04610252, + "balance_loss_mlp": 1.0240798, + "epoch": 0.5128359285757229, + "flos": 13553693412480.0, + "grad_norm": 1.8097338043611708, + "language_loss": 0.76330012, + "learning_rate": 2.0140213747911728e-06, + "loss": 0.78517056, + "num_input_tokens_seen": 92049525, + "router_z_loss_clip": 0.98095703, + "router_z_loss_mlp": 0.18798828, + "step": 4265, + "time_per_iteration": 2.843653678894043 + }, + { + "auxiliary_loss_clip": 0.01145001, + "auxiliary_loss_mlp": 0.0104755, + "balance_loss_clip": 1.04696369, + "balance_loss_mlp": 1.02978802, + "epoch": 0.5129561714663621, + "flos": 25192089820800.0, + "grad_norm": 2.0040515459632267, + "language_loss": 0.80428278, + "learning_rate": 2.013242421243406e-06, + "loss": 0.82620823, + "num_input_tokens_seen": 92068430, + "router_z_loss_clip": 0.98242188, + "router_z_loss_mlp": 0.17749023, + "step": 4266, + "time_per_iteration": 2.735218048095703 + }, + { + "auxiliary_loss_clip": 0.0113873, + "auxiliary_loss_mlp": 0.01040263, + "balance_loss_clip": 1.04471648, + "balance_loss_mlp": 1.02458692, + "epoch": 0.5130764143570011, + "flos": 18150223080960.0, + "grad_norm": 1.5602111888882888, + "language_loss": 0.79118836, + "learning_rate": 2.012463465686774e-06, + "loss": 0.81297827, + "num_input_tokens_seen": 92088180, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.15673828, + "step": 4267, + "time_per_iteration": 3.5742530822753906 + }, + { + "auxiliary_loss_clip": 0.01039811, + "auxiliary_loss_mlp": 0.01002527, + "balance_loss_clip": 1.00895238, + "balance_loss_mlp": 1.00082195, + "epoch": 0.5131966572476402, + "flos": 59794896418560.0, + "grad_norm": 0.7692899858150016, + "language_loss": 0.54776424, + "learning_rate": 2.0116845082394446e-06, + "loss": 0.56818759, + "num_input_tokens_seen": 92153015, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01708984, + "step": 4268, + "time_per_iteration": 3.26865553855896 + }, + { + "auxiliary_loss_clip": 0.01144074, + "auxiliary_loss_mlp": 0.01046572, + "balance_loss_clip": 1.04670596, + "balance_loss_mlp": 1.02889371, + "epoch": 0.5133169001382794, + "flos": 18515219132160.0, + "grad_norm": 1.862313526971093, + "language_loss": 0.78745848, + "learning_rate": 2.0109055490195836e-06, + "loss": 0.80936497, + "num_input_tokens_seen": 92171470, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.17687988, + "step": 4269, + "time_per_iteration": 2.735539197921753 + }, + { + "auxiliary_loss_clip": 0.01139763, + "auxiliary_loss_mlp": 0.01040701, + "balance_loss_clip": 1.04406393, + "balance_loss_mlp": 1.02299893, + "epoch": 0.5134371430289184, + "flos": 15523537219200.0, + "grad_norm": 1.9470051853041683, + "language_loss": 0.64546537, + "learning_rate": 2.0101265881453605e-06, + "loss": 0.66727, + "num_input_tokens_seen": 92189945, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.17712402, + "step": 4270, + "time_per_iteration": 2.7618601322174072 + }, + { + "auxiliary_loss_clip": 0.01142145, + "auxiliary_loss_mlp": 0.0104713, + "balance_loss_clip": 1.05036426, + "balance_loss_mlp": 1.0315975, + "epoch": 0.5135573859195575, + "flos": 21433786911360.0, + "grad_norm": 1.9059425474790603, + "language_loss": 0.78200853, + "learning_rate": 2.009347625734941e-06, + "loss": 0.80390131, + "num_input_tokens_seen": 92209855, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.15527344, + "step": 4271, + "time_per_iteration": 2.753939390182495 + }, + { + "auxiliary_loss_clip": 0.01142637, + "auxiliary_loss_mlp": 0.0104311, + "balance_loss_clip": 1.04739821, + "balance_loss_mlp": 1.02438271, + "epoch": 0.5136776288101966, + "flos": 17712651600000.0, + "grad_norm": 2.4684046347761597, + "language_loss": 0.75656402, + "learning_rate": 2.0085686619064954e-06, + "loss": 0.77842152, + "num_input_tokens_seen": 92226295, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.18725586, + "step": 4272, + "time_per_iteration": 2.756491184234619 + }, + { + "auxiliary_loss_clip": 0.01142231, + "auxiliary_loss_mlp": 0.01041034, + "balance_loss_clip": 1.04739904, + "balance_loss_mlp": 1.02484524, + "epoch": 0.5137978717008357, + "flos": 16581680997120.0, + "grad_norm": 1.922397757541375, + "language_loss": 0.82754588, + "learning_rate": 2.00778969677819e-06, + "loss": 0.84937853, + "num_input_tokens_seen": 92243330, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.16186523, + "step": 4273, + "time_per_iteration": 2.73415470123291 + }, + { + "auxiliary_loss_clip": 0.01141018, + "auxiliary_loss_mlp": 0.01043319, + "balance_loss_clip": 1.04698253, + "balance_loss_mlp": 1.02536654, + "epoch": 0.5139181145914747, + "flos": 20668243322880.0, + "grad_norm": 1.8506018141389788, + "language_loss": 0.63663954, + "learning_rate": 2.0070107304681934e-06, + "loss": 0.65848291, + "num_input_tokens_seen": 92262285, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.17956543, + "step": 4274, + "time_per_iteration": 2.7857978343963623 + }, + { + "auxiliary_loss_clip": 0.0114409, + "auxiliary_loss_mlp": 0.01037444, + "balance_loss_clip": 1.04992402, + "balance_loss_mlp": 1.0208267, + "epoch": 0.5140383574821139, + "flos": 32926996546560.0, + "grad_norm": 1.598776708504596, + "language_loss": 0.78005517, + "learning_rate": 2.006231763094675e-06, + "loss": 0.80187047, + "num_input_tokens_seen": 92283305, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.16625977, + "step": 4275, + "time_per_iteration": 2.8463268280029297 + }, + { + "auxiliary_loss_clip": 0.01138787, + "auxiliary_loss_mlp": 0.01041402, + "balance_loss_clip": 1.04593039, + "balance_loss_mlp": 1.02466536, + "epoch": 0.514158600372753, + "flos": 19537093152000.0, + "grad_norm": 2.0254693551744616, + "language_loss": 0.8768084, + "learning_rate": 2.0054527947758027e-06, + "loss": 0.89861029, + "num_input_tokens_seen": 92302105, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.1673584, + "step": 4276, + "time_per_iteration": 2.7848124504089355 + }, + { + "auxiliary_loss_clip": 0.01042077, + "auxiliary_loss_mlp": 0.01001681, + "balance_loss_clip": 1.01111448, + "balance_loss_mlp": 1.00004816, + "epoch": 0.514278843263392, + "flos": 62523855279360.0, + "grad_norm": 0.7209702584582783, + "language_loss": 0.55875385, + "learning_rate": 2.004673825629746e-06, + "loss": 0.57919139, + "num_input_tokens_seen": 92362885, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.01635742, + "step": 4277, + "time_per_iteration": 3.254748582839966 + }, + { + "auxiliary_loss_clip": 0.01140847, + "auxiliary_loss_mlp": 0.01041582, + "balance_loss_clip": 1.04573822, + "balance_loss_mlp": 1.02402234, + "epoch": 0.5143990861540312, + "flos": 25882328545920.0, + "grad_norm": 1.53648258249121, + "language_loss": 0.72771639, + "learning_rate": 2.0038948557746744e-06, + "loss": 0.74954069, + "num_input_tokens_seen": 92384740, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.17565918, + "step": 4278, + "time_per_iteration": 2.7876739501953125 + }, + { + "auxiliary_loss_clip": 0.01140716, + "auxiliary_loss_mlp": 0.01040566, + "balance_loss_clip": 1.04541874, + "balance_loss_mlp": 1.0240314, + "epoch": 0.5145193290446702, + "flos": 23330660238720.0, + "grad_norm": 3.334068303541238, + "language_loss": 0.75198984, + "learning_rate": 2.0031158853287558e-06, + "loss": 0.77380264, + "num_input_tokens_seen": 92405175, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.16540527, + "step": 4279, + "time_per_iteration": 2.8629541397094727 + }, + { + "auxiliary_loss_clip": 0.01141514, + "auxiliary_loss_mlp": 0.01034366, + "balance_loss_clip": 1.04610598, + "balance_loss_mlp": 1.01783192, + "epoch": 0.5146395719353093, + "flos": 22856603518080.0, + "grad_norm": 2.1212966952697347, + "language_loss": 0.70391983, + "learning_rate": 2.0023369144101593e-06, + "loss": 0.72567862, + "num_input_tokens_seen": 92423345, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.1652832, + "step": 4280, + "time_per_iteration": 2.7740514278411865 + }, + { + "auxiliary_loss_clip": 0.01139188, + "auxiliary_loss_mlp": 0.01038657, + "balance_loss_clip": 1.04531705, + "balance_loss_mlp": 1.02200294, + "epoch": 0.5147598148259485, + "flos": 26391577616640.0, + "grad_norm": 1.791761010486061, + "language_loss": 0.767223, + "learning_rate": 2.0015579431370555e-06, + "loss": 0.78900146, + "num_input_tokens_seen": 92445025, + "router_z_loss_clip": 0.93847656, + "router_z_loss_mlp": 0.16650391, + "step": 4281, + "time_per_iteration": 2.8417060375213623 + }, + { + "auxiliary_loss_clip": 0.01139935, + "auxiliary_loss_mlp": 0.01045259, + "balance_loss_clip": 1.04658794, + "balance_loss_mlp": 1.02837849, + "epoch": 0.5148800577165875, + "flos": 29965694561280.0, + "grad_norm": 2.0220474233663293, + "language_loss": 0.70169032, + "learning_rate": 2.000778971627612e-06, + "loss": 0.72354227, + "num_input_tokens_seen": 92464490, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.16870117, + "step": 4282, + "time_per_iteration": 2.8411269187927246 + }, + { + "auxiliary_loss_clip": 0.01144135, + "auxiliary_loss_mlp": 0.01042103, + "balance_loss_clip": 1.04935634, + "balance_loss_mlp": 1.02702284, + "epoch": 0.5150003006072266, + "flos": 17931383470080.0, + "grad_norm": 1.981107166860643, + "language_loss": 0.90339446, + "learning_rate": 2e-06, + "loss": 0.92525685, + "num_input_tokens_seen": 92482085, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.1506958, + "step": 4283, + "time_per_iteration": 2.776017665863037 + }, + { + "auxiliary_loss_clip": 0.01140975, + "auxiliary_loss_mlp": 0.01038783, + "balance_loss_clip": 1.04596996, + "balance_loss_mlp": 1.02336931, + "epoch": 0.5151205434978657, + "flos": 18478733892480.0, + "grad_norm": 1.6469419144677124, + "language_loss": 0.85539293, + "learning_rate": 1.9992210283723878e-06, + "loss": 0.87719047, + "num_input_tokens_seen": 92499325, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.15405273, + "step": 4284, + "time_per_iteration": 2.730699062347412 + }, + { + "auxiliary_loss_clip": 0.01139269, + "auxiliary_loss_mlp": 0.01031797, + "balance_loss_clip": 1.04565144, + "balance_loss_mlp": 1.01640725, + "epoch": 0.5152407863885048, + "flos": 25341263003520.0, + "grad_norm": 1.5006283555533602, + "language_loss": 0.79329032, + "learning_rate": 1.9984420568629448e-06, + "loss": 0.81500101, + "num_input_tokens_seen": 92522090, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.15393066, + "step": 4285, + "time_per_iteration": 2.758031129837036 + }, + { + "auxiliary_loss_clip": 0.01142086, + "auxiliary_loss_mlp": 0.01038859, + "balance_loss_clip": 1.04917181, + "balance_loss_mlp": 1.02313542, + "epoch": 0.5153610292791438, + "flos": 18329740277760.0, + "grad_norm": 1.9163826199928162, + "language_loss": 0.78590906, + "learning_rate": 1.9976630855898405e-06, + "loss": 0.80771846, + "num_input_tokens_seen": 92539845, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.15698242, + "step": 4286, + "time_per_iteration": 2.74130916595459 + }, + { + "auxiliary_loss_clip": 0.01135202, + "auxiliary_loss_mlp": 0.01038483, + "balance_loss_clip": 1.0444622, + "balance_loss_mlp": 1.02299786, + "epoch": 0.515481272169783, + "flos": 30409945971840.0, + "grad_norm": 2.208438017198295, + "language_loss": 0.74481529, + "learning_rate": 1.9968841146712445e-06, + "loss": 0.76655215, + "num_input_tokens_seen": 92559460, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.1550293, + "step": 4287, + "time_per_iteration": 3.5858421325683594 + }, + { + "auxiliary_loss_clip": 0.01142934, + "auxiliary_loss_mlp": 0.01037292, + "balance_loss_clip": 1.047014, + "balance_loss_mlp": 1.02107954, + "epoch": 0.5156015150604221, + "flos": 23037305863680.0, + "grad_norm": 1.5203974109495035, + "language_loss": 0.71329623, + "learning_rate": 1.996105144225326e-06, + "loss": 0.73509848, + "num_input_tokens_seen": 92579695, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.16223145, + "step": 4288, + "time_per_iteration": 2.7699248790740967 + }, + { + "auxiliary_loss_clip": 0.01138292, + "auxiliary_loss_mlp": 0.01039869, + "balance_loss_clip": 1.04537725, + "balance_loss_mlp": 1.02477658, + "epoch": 0.5157217579510611, + "flos": 17858556645120.0, + "grad_norm": 2.0305587580659696, + "language_loss": 0.78920072, + "learning_rate": 1.995326174370254e-06, + "loss": 0.81098235, + "num_input_tokens_seen": 92598795, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.15075684, + "step": 4289, + "time_per_iteration": 2.7157435417175293 + }, + { + "auxiliary_loss_clip": 0.01136017, + "auxiliary_loss_mlp": 0.01037793, + "balance_loss_clip": 1.04503202, + "balance_loss_mlp": 1.02292728, + "epoch": 0.5158420008417003, + "flos": 19171486569600.0, + "grad_norm": 1.9204328943986042, + "language_loss": 0.73077178, + "learning_rate": 1.994547205224197e-06, + "loss": 0.75250989, + "num_input_tokens_seen": 92617700, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.14868164, + "step": 4290, + "time_per_iteration": 3.5643739700317383 + }, + { + "auxiliary_loss_clip": 0.01139556, + "auxiliary_loss_mlp": 0.01041414, + "balance_loss_clip": 1.045946, + "balance_loss_mlp": 1.02458119, + "epoch": 0.5159622437323393, + "flos": 22419534827520.0, + "grad_norm": 2.079787265822208, + "language_loss": 0.67857218, + "learning_rate": 1.993768236905325e-06, + "loss": 0.70038193, + "num_input_tokens_seen": 92638370, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.16833496, + "step": 4291, + "time_per_iteration": 2.8301637172698975 + }, + { + "auxiliary_loss_clip": 0.01139185, + "auxiliary_loss_mlp": 0.01038738, + "balance_loss_clip": 1.04355872, + "balance_loss_mlp": 1.02065432, + "epoch": 0.5160824866229784, + "flos": 24603010773120.0, + "grad_norm": 2.116294012439165, + "language_loss": 0.65928596, + "learning_rate": 1.992989269531807e-06, + "loss": 0.6810652, + "num_input_tokens_seen": 92657180, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.1809082, + "step": 4292, + "time_per_iteration": 2.831695079803467 + }, + { + "auxiliary_loss_clip": 0.01144799, + "auxiliary_loss_mlp": 0.01040574, + "balance_loss_clip": 1.04835474, + "balance_loss_mlp": 1.02295506, + "epoch": 0.5162027295136175, + "flos": 18002737837440.0, + "grad_norm": 2.585644744217261, + "language_loss": 0.68387896, + "learning_rate": 1.99221030322181e-06, + "loss": 0.7057327, + "num_input_tokens_seen": 92673985, + "router_z_loss_clip": 0.96484375, + "router_z_loss_mlp": 0.17614746, + "step": 4293, + "time_per_iteration": 3.5499439239501953 + }, + { + "auxiliary_loss_clip": 0.01138936, + "auxiliary_loss_mlp": 0.01037281, + "balance_loss_clip": 1.04594922, + "balance_loss_mlp": 1.0209136, + "epoch": 0.5163229724042566, + "flos": 27344611221120.0, + "grad_norm": 1.5058877967773792, + "language_loss": 0.80837852, + "learning_rate": 1.991431338093505e-06, + "loss": 0.83014071, + "num_input_tokens_seen": 92696340, + "router_z_loss_clip": 0.93017578, + "router_z_loss_mlp": 0.16357422, + "step": 4294, + "time_per_iteration": 2.811652421951294 + }, + { + "auxiliary_loss_clip": 0.01139933, + "auxiliary_loss_mlp": 0.01044403, + "balance_loss_clip": 1.04677272, + "balance_loss_mlp": 1.02893019, + "epoch": 0.5164432152948957, + "flos": 21762764599680.0, + "grad_norm": 1.7061813325180786, + "language_loss": 0.79534721, + "learning_rate": 1.9906523742650587e-06, + "loss": 0.81719053, + "num_input_tokens_seen": 92715200, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.15478516, + "step": 4295, + "time_per_iteration": 2.8465890884399414 + }, + { + "auxiliary_loss_clip": 0.01143065, + "auxiliary_loss_mlp": 0.01041591, + "balance_loss_clip": 1.04619217, + "balance_loss_mlp": 1.02390075, + "epoch": 0.5165634581855347, + "flos": 25550334115200.0, + "grad_norm": 1.8245838502217084, + "language_loss": 0.77102113, + "learning_rate": 1.9898734118546397e-06, + "loss": 0.79286766, + "num_input_tokens_seen": 92735150, + "router_z_loss_clip": 0.96777344, + "router_z_loss_mlp": 0.17687988, + "step": 4296, + "time_per_iteration": 3.6249969005584717 + }, + { + "auxiliary_loss_clip": 0.01142492, + "auxiliary_loss_mlp": 0.01041727, + "balance_loss_clip": 1.04796386, + "balance_loss_mlp": 1.02417898, + "epoch": 0.5166837010761739, + "flos": 19901191363200.0, + "grad_norm": 1.579698471955536, + "language_loss": 0.80612659, + "learning_rate": 1.989094450980416e-06, + "loss": 0.82796872, + "num_input_tokens_seen": 92755250, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.17541504, + "step": 4297, + "time_per_iteration": 2.7516074180603027 + }, + { + "auxiliary_loss_clip": 0.01139073, + "auxiliary_loss_mlp": 0.01041413, + "balance_loss_clip": 1.04435408, + "balance_loss_mlp": 1.02547455, + "epoch": 0.516803943966813, + "flos": 26646076454400.0, + "grad_norm": 2.0149272561164637, + "language_loss": 0.765342, + "learning_rate": 1.9883154917605556e-06, + "loss": 0.78714693, + "num_input_tokens_seen": 92774460, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.15942383, + "step": 4298, + "time_per_iteration": 2.7790541648864746 + }, + { + "auxiliary_loss_clip": 0.01140367, + "auxiliary_loss_mlp": 0.0104094, + "balance_loss_clip": 1.04548216, + "balance_loss_mlp": 1.02401245, + "epoch": 0.516924186857452, + "flos": 19682854542720.0, + "grad_norm": 1.8057379066152353, + "language_loss": 0.83253932, + "learning_rate": 1.9875365343132262e-06, + "loss": 0.85435236, + "num_input_tokens_seen": 92791580, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.16931152, + "step": 4299, + "time_per_iteration": 2.86397123336792 + }, + { + "auxiliary_loss_clip": 0.01142575, + "auxiliary_loss_mlp": 0.01042604, + "balance_loss_clip": 1.04782665, + "balance_loss_mlp": 1.02603412, + "epoch": 0.5170444297480912, + "flos": 15956583586560.0, + "grad_norm": 2.2088899812992753, + "language_loss": 0.85050899, + "learning_rate": 1.9867575787565946e-06, + "loss": 0.87236071, + "num_input_tokens_seen": 92806240, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.16577148, + "step": 4300, + "time_per_iteration": 2.734372138977051 + }, + { + "auxiliary_loss_clip": 0.01142671, + "auxiliary_loss_mlp": 0.01044089, + "balance_loss_clip": 1.04743981, + "balance_loss_mlp": 1.02602828, + "epoch": 0.5171646726387302, + "flos": 14174157968640.0, + "grad_norm": 1.8862840750753687, + "language_loss": 0.85867429, + "learning_rate": 1.9859786252088275e-06, + "loss": 0.88054192, + "num_input_tokens_seen": 92823420, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.18066406, + "step": 4301, + "time_per_iteration": 2.7387490272521973 + }, + { + "auxiliary_loss_clip": 0.0114301, + "auxiliary_loss_mlp": 0.01038105, + "balance_loss_clip": 1.04585183, + "balance_loss_mlp": 1.02132022, + "epoch": 0.5172849155293693, + "flos": 23578550974080.0, + "grad_norm": 2.598392704664245, + "language_loss": 0.6693871, + "learning_rate": 1.9851996737880914e-06, + "loss": 0.69119829, + "num_input_tokens_seen": 92838605, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.16796875, + "step": 4302, + "time_per_iteration": 2.716881275177002 + }, + { + "auxiliary_loss_clip": 0.01144225, + "auxiliary_loss_mlp": 0.01049365, + "balance_loss_clip": 1.04708755, + "balance_loss_mlp": 1.03134036, + "epoch": 0.5174051584200084, + "flos": 14283541860480.0, + "grad_norm": 1.9707645172948243, + "language_loss": 0.74509835, + "learning_rate": 1.9844207246125537e-06, + "loss": 0.76703429, + "num_input_tokens_seen": 92855185, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.18029785, + "step": 4303, + "time_per_iteration": 2.9575462341308594 + }, + { + "auxiliary_loss_clip": 0.01137325, + "auxiliary_loss_mlp": 0.01046528, + "balance_loss_clip": 1.04672039, + "balance_loss_mlp": 1.03141248, + "epoch": 0.5175254013106475, + "flos": 37889384192640.0, + "grad_norm": 1.7455903101135843, + "language_loss": 0.68124139, + "learning_rate": 1.983641777800379e-06, + "loss": 0.70307994, + "num_input_tokens_seen": 92877830, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.15112305, + "step": 4304, + "time_per_iteration": 2.867806911468506 + }, + { + "auxiliary_loss_clip": 0.01041428, + "auxiliary_loss_mlp": 0.00999639, + "balance_loss_clip": 1.01118445, + "balance_loss_mlp": 0.99828643, + "epoch": 0.5176456442012866, + "flos": 68549737829760.0, + "grad_norm": 0.7471548364524718, + "language_loss": 0.58769828, + "learning_rate": 1.9828628334697343e-06, + "loss": 0.608109, + "num_input_tokens_seen": 92945040, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.0135498, + "step": 4305, + "time_per_iteration": 3.412794589996338 + }, + { + "auxiliary_loss_clip": 0.01041461, + "auxiliary_loss_mlp": 0.0100097, + "balance_loss_clip": 1.01112258, + "balance_loss_mlp": 0.99950331, + "epoch": 0.5177658870919257, + "flos": 64084137235200.0, + "grad_norm": 0.7607996749786642, + "language_loss": 0.54610908, + "learning_rate": 1.982083891738784e-06, + "loss": 0.56653339, + "num_input_tokens_seen": 93005910, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01464844, + "step": 4306, + "time_per_iteration": 3.283078193664551 + }, + { + "auxiliary_loss_clip": 0.01135332, + "auxiliary_loss_mlp": 0.01034554, + "balance_loss_clip": 1.04501891, + "balance_loss_mlp": 1.01947379, + "epoch": 0.5178861299825648, + "flos": 26651248012800.0, + "grad_norm": 1.5905033483947584, + "language_loss": 0.83041966, + "learning_rate": 1.9813049527256923e-06, + "loss": 0.85211855, + "num_input_tokens_seen": 93026305, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.15087891, + "step": 4307, + "time_per_iteration": 2.756605386734009 + }, + { + "auxiliary_loss_clip": 0.01140908, + "auxiliary_loss_mlp": 0.01039632, + "balance_loss_clip": 1.04641831, + "balance_loss_mlp": 1.02341962, + "epoch": 0.5180063728732038, + "flos": 17931886260480.0, + "grad_norm": 8.827706193445554, + "language_loss": 0.82130933, + "learning_rate": 1.9805260165486252e-06, + "loss": 0.84311473, + "num_input_tokens_seen": 93045675, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.16223145, + "step": 4308, + "time_per_iteration": 2.77290415763855 + }, + { + "auxiliary_loss_clip": 0.01139825, + "auxiliary_loss_mlp": 0.01037468, + "balance_loss_clip": 1.04439259, + "balance_loss_mlp": 1.02052808, + "epoch": 0.518126615763843, + "flos": 19500895221120.0, + "grad_norm": 1.8833522351125291, + "language_loss": 0.86445493, + "learning_rate": 1.9797470833257457e-06, + "loss": 0.88622785, + "num_input_tokens_seen": 93065375, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.16943359, + "step": 4309, + "time_per_iteration": 2.752635955810547 + }, + { + "auxiliary_loss_clip": 0.01144991, + "auxiliary_loss_mlp": 0.01052429, + "balance_loss_clip": 1.04914844, + "balance_loss_mlp": 1.03553104, + "epoch": 0.5182468586544821, + "flos": 20704082117760.0, + "grad_norm": 1.9490161877305723, + "language_loss": 0.77618825, + "learning_rate": 1.9789681531752177e-06, + "loss": 0.79816252, + "num_input_tokens_seen": 93085595, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.16900635, + "step": 4310, + "time_per_iteration": 2.835860252380371 + }, + { + "auxiliary_loss_clip": 0.0113846, + "auxiliary_loss_mlp": 0.01044538, + "balance_loss_clip": 1.04558861, + "balance_loss_mlp": 1.02861142, + "epoch": 0.5183671015451211, + "flos": 23112107936640.0, + "grad_norm": 1.614062705156778, + "language_loss": 0.72627276, + "learning_rate": 1.978189226215204e-06, + "loss": 0.74810272, + "num_input_tokens_seen": 93106140, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.15917969, + "step": 4311, + "time_per_iteration": 2.7566401958465576 + }, + { + "auxiliary_loss_clip": 0.01140914, + "auxiliary_loss_mlp": 0.01045866, + "balance_loss_clip": 1.04617548, + "balance_loss_mlp": 1.02887845, + "epoch": 0.5184873444357603, + "flos": 17597090568960.0, + "grad_norm": 1.7409406726335512, + "language_loss": 0.76610988, + "learning_rate": 1.9774103025638675e-06, + "loss": 0.7879777, + "num_input_tokens_seen": 93124265, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.16992188, + "step": 4312, + "time_per_iteration": 2.723270893096924 + }, + { + "auxiliary_loss_clip": 0.01144256, + "auxiliary_loss_mlp": 0.01055082, + "balance_loss_clip": 1.04808748, + "balance_loss_mlp": 1.03696251, + "epoch": 0.5186075873263993, + "flos": 24936800883840.0, + "grad_norm": 1.5288431231237831, + "language_loss": 0.76646852, + "learning_rate": 1.9766313823393696e-06, + "loss": 0.78846192, + "num_input_tokens_seen": 93145130, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.18127441, + "step": 4313, + "time_per_iteration": 2.804319143295288 + }, + { + "auxiliary_loss_clip": 0.01139327, + "auxiliary_loss_mlp": 0.01043558, + "balance_loss_clip": 1.04466856, + "balance_loss_mlp": 1.02546203, + "epoch": 0.5187278302170384, + "flos": 15190106244480.0, + "grad_norm": 2.114057403158985, + "language_loss": 0.69245839, + "learning_rate": 1.975852465659873e-06, + "loss": 0.71428728, + "num_input_tokens_seen": 93161110, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.1809082, + "step": 4314, + "time_per_iteration": 2.7318975925445557 + }, + { + "auxiliary_loss_clip": 0.01149369, + "auxiliary_loss_mlp": 0.01042483, + "balance_loss_clip": 1.04824066, + "balance_loss_mlp": 1.02573383, + "epoch": 0.5188480731076776, + "flos": 25009412227200.0, + "grad_norm": 2.1028450045234406, + "language_loss": 0.70741475, + "learning_rate": 1.9750735526435377e-06, + "loss": 0.72933328, + "num_input_tokens_seen": 93178055, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.16748047, + "step": 4315, + "time_per_iteration": 2.7862720489501953 + }, + { + "auxiliary_loss_clip": 0.01138198, + "auxiliary_loss_mlp": 0.01043781, + "balance_loss_clip": 1.04280913, + "balance_loss_mlp": 1.02644813, + "epoch": 0.5189683159983166, + "flos": 24790141653120.0, + "grad_norm": 2.138722548133385, + "language_loss": 0.79694301, + "learning_rate": 1.974294643408525e-06, + "loss": 0.81876278, + "num_input_tokens_seen": 93195850, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.17333984, + "step": 4316, + "time_per_iteration": 3.577179431915283 + }, + { + "auxiliary_loss_clip": 0.01141455, + "auxiliary_loss_mlp": 0.01048494, + "balance_loss_clip": 1.04435205, + "balance_loss_mlp": 1.03076732, + "epoch": 0.5190885588889557, + "flos": 24754266944640.0, + "grad_norm": 2.8418193769995175, + "language_loss": 0.67408383, + "learning_rate": 1.9735157380729947e-06, + "loss": 0.69598335, + "num_input_tokens_seen": 93216260, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.17736816, + "step": 4317, + "time_per_iteration": 2.8671317100524902 + }, + { + "auxiliary_loss_clip": 0.01137007, + "auxiliary_loss_mlp": 0.01038745, + "balance_loss_clip": 1.04446197, + "balance_loss_mlp": 1.0215317, + "epoch": 0.5192088017795948, + "flos": 24712646060160.0, + "grad_norm": 1.8649616331027326, + "language_loss": 0.84349263, + "learning_rate": 1.9727368367551053e-06, + "loss": 0.86525017, + "num_input_tokens_seen": 93234810, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.17199707, + "step": 4318, + "time_per_iteration": 3.6536524295806885 + }, + { + "auxiliary_loss_clip": 0.01140484, + "auxiliary_loss_mlp": 0.01045458, + "balance_loss_clip": 1.04457486, + "balance_loss_mlp": 1.02844632, + "epoch": 0.5193290446702339, + "flos": 27229588894080.0, + "grad_norm": 2.4173492075348064, + "language_loss": 0.68938363, + "learning_rate": 1.9719579395730164e-06, + "loss": 0.71124303, + "num_input_tokens_seen": 93254185, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.17004395, + "step": 4319, + "time_per_iteration": 2.7975258827209473 + }, + { + "auxiliary_loss_clip": 0.01143294, + "auxiliary_loss_mlp": 0.01048075, + "balance_loss_clip": 1.0481956, + "balance_loss_mlp": 1.03109956, + "epoch": 0.5194492875608729, + "flos": 11473352392320.0, + "grad_norm": 2.0919322366456714, + "language_loss": 0.93444693, + "learning_rate": 1.9711790466448854e-06, + "loss": 0.95636058, + "num_input_tokens_seen": 93268205, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.16955566, + "step": 4320, + "time_per_iteration": 2.747187614440918 + }, + { + "auxiliary_loss_clip": 0.01143532, + "auxiliary_loss_mlp": 0.01043166, + "balance_loss_clip": 1.04640174, + "balance_loss_mlp": 1.02597618, + "epoch": 0.5195695304515121, + "flos": 20338906498560.0, + "grad_norm": 2.15756648637811, + "language_loss": 0.7158559, + "learning_rate": 1.9704001580888704e-06, + "loss": 0.73772287, + "num_input_tokens_seen": 93286945, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.17211914, + "step": 4321, + "time_per_iteration": 2.74922251701355 + }, + { + "auxiliary_loss_clip": 0.01138695, + "auxiliary_loss_mlp": 0.01046771, + "balance_loss_clip": 1.0455184, + "balance_loss_mlp": 1.02996254, + "epoch": 0.5196897733421512, + "flos": 20048317470720.0, + "grad_norm": 1.9402803052011923, + "language_loss": 0.86815917, + "learning_rate": 1.9696212740231283e-06, + "loss": 0.89001387, + "num_input_tokens_seen": 93305595, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.16784668, + "step": 4322, + "time_per_iteration": 3.5392160415649414 + }, + { + "auxiliary_loss_clip": 0.01145074, + "auxiliary_loss_mlp": 0.0104626, + "balance_loss_clip": 1.04637361, + "balance_loss_mlp": 1.02676952, + "epoch": 0.5198100162327902, + "flos": 23805507058560.0, + "grad_norm": 3.3767149424981704, + "language_loss": 0.82430363, + "learning_rate": 1.9688423945658146e-06, + "loss": 0.84621692, + "num_input_tokens_seen": 93326460, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.19506836, + "step": 4323, + "time_per_iteration": 2.853201389312744 + }, + { + "auxiliary_loss_clip": 0.01143309, + "auxiliary_loss_mlp": 0.01041908, + "balance_loss_clip": 1.04739809, + "balance_loss_mlp": 1.02436078, + "epoch": 0.5199302591234293, + "flos": 24023951619840.0, + "grad_norm": 2.417719243566563, + "language_loss": 0.72063559, + "learning_rate": 1.9680635198350845e-06, + "loss": 0.74248779, + "num_input_tokens_seen": 93346170, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.17553711, + "step": 4324, + "time_per_iteration": 2.9381103515625 + }, + { + "auxiliary_loss_clip": 0.01146626, + "auxiliary_loss_mlp": 0.01041193, + "balance_loss_clip": 1.04814768, + "balance_loss_mlp": 1.02370453, + "epoch": 0.5200505020140684, + "flos": 26359366095360.0, + "grad_norm": 1.9068142892016482, + "language_loss": 0.72488749, + "learning_rate": 1.967284649949093e-06, + "loss": 0.74676561, + "num_input_tokens_seen": 93365380, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.17492676, + "step": 4325, + "time_per_iteration": 3.6728744506835938 + }, + { + "auxiliary_loss_clip": 0.01137652, + "auxiliary_loss_mlp": 0.01041352, + "balance_loss_clip": 1.04371512, + "balance_loss_mlp": 1.02528214, + "epoch": 0.5201707449047075, + "flos": 39604262284800.0, + "grad_norm": 1.9477924652669956, + "language_loss": 0.72483724, + "learning_rate": 1.966505785025994e-06, + "loss": 0.74662733, + "num_input_tokens_seen": 93387285, + "router_z_loss_clip": 0.93847656, + "router_z_loss_mlp": 0.16064453, + "step": 4326, + "time_per_iteration": 2.9684131145477295 + }, + { + "auxiliary_loss_clip": 0.01144303, + "auxiliary_loss_mlp": 0.01040447, + "balance_loss_clip": 1.04838848, + "balance_loss_mlp": 1.02303064, + "epoch": 0.5202909877953465, + "flos": 53682788292480.0, + "grad_norm": 1.672968994298373, + "language_loss": 0.76117778, + "learning_rate": 1.965726925183941e-06, + "loss": 0.78302526, + "num_input_tokens_seen": 93410390, + "router_z_loss_clip": 0.96044922, + "router_z_loss_mlp": 0.17407227, + "step": 4327, + "time_per_iteration": 3.1261801719665527 + }, + { + "auxiliary_loss_clip": 0.01137679, + "auxiliary_loss_mlp": 0.01044535, + "balance_loss_clip": 1.04442596, + "balance_loss_mlp": 1.02685618, + "epoch": 0.5204112306859857, + "flos": 19537021324800.0, + "grad_norm": 2.953548905308902, + "language_loss": 0.84538072, + "learning_rate": 1.964948070541087e-06, + "loss": 0.86720288, + "num_input_tokens_seen": 93429050, + "router_z_loss_clip": 0.93164062, + "router_z_loss_mlp": 0.17675781, + "step": 4328, + "time_per_iteration": 2.9598491191864014 + }, + { + "auxiliary_loss_clip": 0.01139811, + "auxiliary_loss_mlp": 0.01041303, + "balance_loss_clip": 1.0455327, + "balance_loss_mlp": 1.02454185, + "epoch": 0.5205314735766248, + "flos": 15304697608320.0, + "grad_norm": 2.176539609058087, + "language_loss": 0.69823933, + "learning_rate": 1.9641692212155816e-06, + "loss": 0.72005045, + "num_input_tokens_seen": 93446815, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.16760254, + "step": 4329, + "time_per_iteration": 2.947180986404419 + }, + { + "auxiliary_loss_clip": 0.01142275, + "auxiliary_loss_mlp": 0.01043194, + "balance_loss_clip": 1.04794669, + "balance_loss_mlp": 1.02706456, + "epoch": 0.5206517164672638, + "flos": 59263701160320.0, + "grad_norm": 1.8076786391281585, + "language_loss": 0.72956759, + "learning_rate": 1.9633903773255777e-06, + "loss": 0.75142229, + "num_input_tokens_seen": 93469130, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.16137695, + "step": 4330, + "time_per_iteration": 3.1704936027526855 + }, + { + "auxiliary_loss_clip": 0.01140258, + "auxiliary_loss_mlp": 0.01037216, + "balance_loss_clip": 1.04518461, + "balance_loss_mlp": 1.02072954, + "epoch": 0.520771959357903, + "flos": 26871129118080.0, + "grad_norm": 1.766769767186246, + "language_loss": 0.75141168, + "learning_rate": 1.9626115389892237e-06, + "loss": 0.77318639, + "num_input_tokens_seen": 93489920, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.16503906, + "step": 4331, + "time_per_iteration": 3.0051841735839844 + }, + { + "auxiliary_loss_clip": 0.01141817, + "auxiliary_loss_mlp": 0.01039708, + "balance_loss_clip": 1.04778218, + "balance_loss_mlp": 1.02174294, + "epoch": 0.520892202248542, + "flos": 26907075653760.0, + "grad_norm": 2.2479812771878467, + "language_loss": 0.85211313, + "learning_rate": 1.96183270632467e-06, + "loss": 0.87392837, + "num_input_tokens_seen": 93509770, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.1796875, + "step": 4332, + "time_per_iteration": 2.9622416496276855 + }, + { + "auxiliary_loss_clip": 0.01140643, + "auxiliary_loss_mlp": 0.01042908, + "balance_loss_clip": 1.04449129, + "balance_loss_mlp": 1.02522922, + "epoch": 0.5210124451391811, + "flos": 25849434666240.0, + "grad_norm": 2.853735473807865, + "language_loss": 0.79013944, + "learning_rate": 1.9610538794500644e-06, + "loss": 0.811975, + "num_input_tokens_seen": 93529320, + "router_z_loss_clip": 0.96044922, + "router_z_loss_mlp": 0.17663574, + "step": 4333, + "time_per_iteration": 2.859299898147583 + }, + { + "auxiliary_loss_clip": 0.01040898, + "auxiliary_loss_mlp": 0.01004813, + "balance_loss_clip": 1.01072311, + "balance_loss_mlp": 1.00336504, + "epoch": 0.5211326880298203, + "flos": 70553804319360.0, + "grad_norm": 0.788868157758144, + "language_loss": 0.5941304, + "learning_rate": 1.9602750584835542e-06, + "loss": 0.61458755, + "num_input_tokens_seen": 93595255, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01446533, + "step": 4334, + "time_per_iteration": 3.4237358570098877 + }, + { + "auxiliary_loss_clip": 0.01135695, + "auxiliary_loss_mlp": 0.01031544, + "balance_loss_clip": 1.04295659, + "balance_loss_mlp": 1.0146997, + "epoch": 0.5212529309204593, + "flos": 15628898787840.0, + "grad_norm": 1.9549765685103746, + "language_loss": 0.82568431, + "learning_rate": 1.959496243543286e-06, + "loss": 0.84735668, + "num_input_tokens_seen": 93613135, + "router_z_loss_clip": 0.92773438, + "router_z_loss_mlp": 0.16833496, + "step": 4335, + "time_per_iteration": 2.780775547027588 + }, + { + "auxiliary_loss_clip": 0.01149298, + "auxiliary_loss_mlp": 0.01058056, + "balance_loss_clip": 1.05168033, + "balance_loss_mlp": 1.04165244, + "epoch": 0.5213731738110984, + "flos": 26242655829120.0, + "grad_norm": 1.985210537351043, + "language_loss": 0.79258358, + "learning_rate": 1.9587174347474057e-06, + "loss": 0.81465709, + "num_input_tokens_seen": 93629645, + "router_z_loss_clip": 0.97802734, + "router_z_loss_mlp": 0.16418457, + "step": 4336, + "time_per_iteration": 2.8307511806488037 + }, + { + "auxiliary_loss_clip": 0.01146729, + "auxiliary_loss_mlp": 0.01050027, + "balance_loss_clip": 1.04771566, + "balance_loss_mlp": 1.03185987, + "epoch": 0.5214934167017375, + "flos": 19418407637760.0, + "grad_norm": 3.257841292761412, + "language_loss": 0.81898487, + "learning_rate": 1.9579386322140574e-06, + "loss": 0.8409524, + "num_input_tokens_seen": 93645325, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.18164062, + "step": 4337, + "time_per_iteration": 2.768226146697998 + }, + { + "auxiliary_loss_clip": 0.01143012, + "auxiliary_loss_mlp": 0.01046277, + "balance_loss_clip": 1.04603493, + "balance_loss_mlp": 1.02837205, + "epoch": 0.5216136595923766, + "flos": 30955788023040.0, + "grad_norm": 1.7497694248968987, + "language_loss": 0.80773997, + "learning_rate": 1.9571598360613854e-06, + "loss": 0.82963288, + "num_input_tokens_seen": 93668200, + "router_z_loss_clip": 0.96972656, + "router_z_loss_mlp": 0.17895508, + "step": 4338, + "time_per_iteration": 3.051753282546997 + }, + { + "auxiliary_loss_clip": 0.01140557, + "auxiliary_loss_mlp": 0.01040145, + "balance_loss_clip": 1.04353833, + "balance_loss_mlp": 1.02259684, + "epoch": 0.5217339024830157, + "flos": 21945047143680.0, + "grad_norm": 2.0378181543965326, + "language_loss": 0.69611311, + "learning_rate": 1.956381046407532e-06, + "loss": 0.71792012, + "num_input_tokens_seen": 93688495, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.17541504, + "step": 4339, + "time_per_iteration": 2.808058738708496 + }, + { + "auxiliary_loss_clip": 0.01140337, + "auxiliary_loss_mlp": 0.01038952, + "balance_loss_clip": 1.0461241, + "balance_loss_mlp": 1.02245355, + "epoch": 0.5218541453736548, + "flos": 20923209037440.0, + "grad_norm": 1.7061233947882553, + "language_loss": 0.86183214, + "learning_rate": 1.9556022633706394e-06, + "loss": 0.88362503, + "num_input_tokens_seen": 93707285, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.16503906, + "step": 4340, + "time_per_iteration": 2.831923484802246 + }, + { + "auxiliary_loss_clip": 0.0114095, + "auxiliary_loss_mlp": 0.01038442, + "balance_loss_clip": 1.04670429, + "balance_loss_mlp": 1.0215981, + "epoch": 0.5219743882642939, + "flos": 23951663498880.0, + "grad_norm": 1.9323374600565022, + "language_loss": 0.79833049, + "learning_rate": 1.954823487068848e-06, + "loss": 0.82012439, + "num_input_tokens_seen": 93727495, + "router_z_loss_clip": 0.94238281, + "router_z_loss_mlp": 0.1685791, + "step": 4341, + "time_per_iteration": 2.7884325981140137 + }, + { + "auxiliary_loss_clip": 0.01141798, + "auxiliary_loss_mlp": 0.01038404, + "balance_loss_clip": 1.04770756, + "balance_loss_mlp": 1.02189398, + "epoch": 0.5220946311549329, + "flos": 28799280213120.0, + "grad_norm": 1.6557842503757263, + "language_loss": 0.81146181, + "learning_rate": 1.9540447176202976e-06, + "loss": 0.83326381, + "num_input_tokens_seen": 93748740, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.16516113, + "step": 4342, + "time_per_iteration": 3.2424917221069336 + }, + { + "auxiliary_loss_clip": 0.01040426, + "auxiliary_loss_mlp": 0.01002183, + "balance_loss_clip": 1.0097878, + "balance_loss_mlp": 1.0006218, + "epoch": 0.5222148740455721, + "flos": 67189369017600.0, + "grad_norm": 0.9075753610700444, + "language_loss": 0.60730207, + "learning_rate": 1.9532659551431272e-06, + "loss": 0.6277281, + "num_input_tokens_seen": 93815770, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.01556396, + "step": 4343, + "time_per_iteration": 3.480175733566284 + }, + { + "auxiliary_loss_clip": 0.01135656, + "auxiliary_loss_mlp": 0.01033935, + "balance_loss_clip": 1.04426908, + "balance_loss_mlp": 1.01675725, + "epoch": 0.5223351169362112, + "flos": 61856164339200.0, + "grad_norm": 1.5591808487622758, + "language_loss": 0.67875767, + "learning_rate": 1.9524871997554744e-06, + "loss": 0.70045364, + "num_input_tokens_seen": 93843530, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.17175293, + "step": 4344, + "time_per_iteration": 3.1764159202575684 + }, + { + "auxiliary_loss_clip": 0.01142911, + "auxiliary_loss_mlp": 0.01045067, + "balance_loss_clip": 1.04744959, + "balance_loss_mlp": 1.02716184, + "epoch": 0.5224553598268502, + "flos": 14647388676480.0, + "grad_norm": 2.0548759553855738, + "language_loss": 0.80933881, + "learning_rate": 1.951708451575475e-06, + "loss": 0.8312186, + "num_input_tokens_seen": 93860595, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.17907715, + "step": 4345, + "time_per_iteration": 3.6110615730285645 + }, + { + "auxiliary_loss_clip": 0.01140684, + "auxiliary_loss_mlp": 0.01047001, + "balance_loss_clip": 1.04681098, + "balance_loss_mlp": 1.02908361, + "epoch": 0.5225756027174894, + "flos": 14826043946880.0, + "grad_norm": 1.8911051186980548, + "language_loss": 0.82083029, + "learning_rate": 1.9509297107212657e-06, + "loss": 0.84270716, + "num_input_tokens_seen": 93877365, + "router_z_loss_clip": 0.93896484, + "router_z_loss_mlp": 0.17907715, + "step": 4346, + "time_per_iteration": 2.841273784637451 + }, + { + "auxiliary_loss_clip": 0.01143883, + "auxiliary_loss_mlp": 0.01045463, + "balance_loss_clip": 1.04950869, + "balance_loss_mlp": 1.02933455, + "epoch": 0.5226958456081284, + "flos": 23512009029120.0, + "grad_norm": 1.6855595535207315, + "language_loss": 0.79094017, + "learning_rate": 1.95015097731098e-06, + "loss": 0.81283367, + "num_input_tokens_seen": 93896855, + "router_z_loss_clip": 0.94335938, + "router_z_loss_mlp": 0.16137695, + "step": 4347, + "time_per_iteration": 3.6168391704559326 + }, + { + "auxiliary_loss_clip": 0.01138694, + "auxiliary_loss_mlp": 0.01045246, + "balance_loss_clip": 1.04385972, + "balance_loss_mlp": 1.02792478, + "epoch": 0.5228160884987675, + "flos": 19062928690560.0, + "grad_norm": 2.1194571567045832, + "language_loss": 0.82062817, + "learning_rate": 1.949372251462751e-06, + "loss": 0.84246755, + "num_input_tokens_seen": 93914270, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.17321777, + "step": 4348, + "time_per_iteration": 2.768862724304199 + }, + { + "auxiliary_loss_clip": 0.01139877, + "auxiliary_loss_mlp": 0.01044005, + "balance_loss_clip": 1.04725218, + "balance_loss_mlp": 1.02766132, + "epoch": 0.5229363313894067, + "flos": 21063224252160.0, + "grad_norm": 2.394156259801348, + "language_loss": 0.82661915, + "learning_rate": 1.9485935332947124e-06, + "loss": 0.84845793, + "num_input_tokens_seen": 93932180, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.16333008, + "step": 4349, + "time_per_iteration": 2.747844934463501 + }, + { + "auxiliary_loss_clip": 0.01136914, + "auxiliary_loss_mlp": 0.01040639, + "balance_loss_clip": 1.04474926, + "balance_loss_mlp": 1.02389002, + "epoch": 0.5230565742800457, + "flos": 14830389492480.0, + "grad_norm": 2.353664673922617, + "language_loss": 0.83596009, + "learning_rate": 1.947814822924993e-06, + "loss": 0.85773557, + "num_input_tokens_seen": 93949690, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.16748047, + "step": 4350, + "time_per_iteration": 2.739199638366699 + }, + { + "auxiliary_loss_clip": 0.01137773, + "auxiliary_loss_mlp": 0.01036573, + "balance_loss_clip": 1.04702425, + "balance_loss_mlp": 1.02099192, + "epoch": 0.5231768171706848, + "flos": 25813021253760.0, + "grad_norm": 1.8070325043233093, + "language_loss": 0.83162987, + "learning_rate": 1.9470361204717236e-06, + "loss": 0.85337329, + "num_input_tokens_seen": 93968830, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.15576172, + "step": 4351, + "time_per_iteration": 3.553258180618286 + }, + { + "auxiliary_loss_clip": 0.0114393, + "auxiliary_loss_mlp": 0.01045524, + "balance_loss_clip": 1.04911399, + "balance_loss_mlp": 1.0285008, + "epoch": 0.5232970600613239, + "flos": 22743807834240.0, + "grad_norm": 1.5568924626253215, + "language_loss": 0.80625308, + "learning_rate": 1.9462574260530326e-06, + "loss": 0.82814771, + "num_input_tokens_seen": 93989110, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.17028809, + "step": 4352, + "time_per_iteration": 2.7273545265197754 + }, + { + "auxiliary_loss_clip": 0.01140302, + "auxiliary_loss_mlp": 0.01041866, + "balance_loss_clip": 1.04653871, + "balance_loss_mlp": 1.02446127, + "epoch": 0.523417302951963, + "flos": 17310703432320.0, + "grad_norm": 1.7329682417942314, + "language_loss": 0.80885768, + "learning_rate": 1.9454787397870472e-06, + "loss": 0.83067942, + "num_input_tokens_seen": 94006430, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.17407227, + "step": 4353, + "time_per_iteration": 2.830486536026001 + }, + { + "auxiliary_loss_clip": 0.01140256, + "auxiliary_loss_mlp": 0.01045519, + "balance_loss_clip": 1.04602098, + "balance_loss_mlp": 1.02867496, + "epoch": 0.523537545842602, + "flos": 18551740285440.0, + "grad_norm": 1.847357954536432, + "language_loss": 0.71716386, + "learning_rate": 1.944700061791894e-06, + "loss": 0.73902166, + "num_input_tokens_seen": 94024825, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.16833496, + "step": 4354, + "time_per_iteration": 3.571681499481201 + }, + { + "auxiliary_loss_clip": 0.01140106, + "auxiliary_loss_mlp": 0.01039565, + "balance_loss_clip": 1.04709172, + "balance_loss_mlp": 1.0237335, + "epoch": 0.5236577887332411, + "flos": 19719267955200.0, + "grad_norm": 2.142586736998612, + "language_loss": 0.65595913, + "learning_rate": 1.943921392185698e-06, + "loss": 0.67775583, + "num_input_tokens_seen": 94043450, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.1583252, + "step": 4355, + "time_per_iteration": 2.743727922439575 + }, + { + "auxiliary_loss_clip": 0.01140202, + "auxiliary_loss_mlp": 0.0104603, + "balance_loss_clip": 1.04604292, + "balance_loss_mlp": 1.02984095, + "epoch": 0.5237780316238803, + "flos": 23550218121600.0, + "grad_norm": 1.9223581924462154, + "language_loss": 0.7706579, + "learning_rate": 1.9431427310865814e-06, + "loss": 0.79252023, + "num_input_tokens_seen": 94063055, + "router_z_loss_clip": 0.94238281, + "router_z_loss_mlp": 0.16186523, + "step": 4356, + "time_per_iteration": 2.7769932746887207 + }, + { + "auxiliary_loss_clip": 0.01132987, + "auxiliary_loss_mlp": 0.0103309, + "balance_loss_clip": 1.04392838, + "balance_loss_mlp": 1.01739013, + "epoch": 0.5238982745145193, + "flos": 22491894775680.0, + "grad_norm": 2.016819344334222, + "language_loss": 0.78701389, + "learning_rate": 1.942364078612667e-06, + "loss": 0.80867469, + "num_input_tokens_seen": 94081785, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.15698242, + "step": 4357, + "time_per_iteration": 2.732783794403076 + }, + { + "auxiliary_loss_clip": 0.01143914, + "auxiliary_loss_mlp": 0.01045834, + "balance_loss_clip": 1.04754496, + "balance_loss_mlp": 1.02724981, + "epoch": 0.5240185174051584, + "flos": 27088927234560.0, + "grad_norm": 1.7992666262151262, + "language_loss": 0.75423181, + "learning_rate": 1.9415854348820765e-06, + "loss": 0.77612925, + "num_input_tokens_seen": 94101635, + "router_z_loss_clip": 0.96289062, + "router_z_loss_mlp": 0.18579102, + "step": 4358, + "time_per_iteration": 2.7739782333374023 + }, + { + "auxiliary_loss_clip": 0.01143715, + "auxiliary_loss_mlp": 0.0104945, + "balance_loss_clip": 1.04645681, + "balance_loss_mlp": 1.03211689, + "epoch": 0.5241387602957975, + "flos": 22674680110080.0, + "grad_norm": 2.1024182959072606, + "language_loss": 0.68431342, + "learning_rate": 1.940806800012929e-06, + "loss": 0.70624506, + "num_input_tokens_seen": 94121705, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.17346191, + "step": 4359, + "time_per_iteration": 2.753737688064575 + }, + { + "auxiliary_loss_clip": 0.01144757, + "auxiliary_loss_mlp": 0.01054527, + "balance_loss_clip": 1.04840851, + "balance_loss_mlp": 1.03600168, + "epoch": 0.5242590031864366, + "flos": 40553453134080.0, + "grad_norm": 2.398378170222604, + "language_loss": 0.63569784, + "learning_rate": 1.9400281741233432e-06, + "loss": 0.6576907, + "num_input_tokens_seen": 94146595, + "router_z_loss_clip": 0.96240234, + "router_z_loss_mlp": 0.18530273, + "step": 4360, + "time_per_iteration": 2.9014992713928223 + }, + { + "auxiliary_loss_clip": 0.01039712, + "auxiliary_loss_mlp": 0.01004605, + "balance_loss_clip": 1.00983715, + "balance_loss_mlp": 1.0030551, + "epoch": 0.5243792460770756, + "flos": 66676313105280.0, + "grad_norm": 0.7317998420276385, + "language_loss": 0.52524054, + "learning_rate": 1.939249557331435e-06, + "loss": 0.54568374, + "num_input_tokens_seen": 94212410, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.01550293, + "step": 4361, + "time_per_iteration": 3.314089298248291 + }, + { + "auxiliary_loss_clip": 0.01142707, + "auxiliary_loss_mlp": 0.01040861, + "balance_loss_clip": 1.04753137, + "balance_loss_mlp": 1.02407598, + "epoch": 0.5244994889677148, + "flos": 28183663992960.0, + "grad_norm": 1.963486543069152, + "language_loss": 0.72907519, + "learning_rate": 1.938470949755321e-06, + "loss": 0.75091088, + "num_input_tokens_seen": 94232290, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.16772461, + "step": 4362, + "time_per_iteration": 2.798227310180664 + }, + { + "auxiliary_loss_clip": 0.01040624, + "auxiliary_loss_mlp": 0.01004733, + "balance_loss_clip": 1.01075065, + "balance_loss_mlp": 1.00326109, + "epoch": 0.5246197318583539, + "flos": 65950379239680.0, + "grad_norm": 0.8071443990170697, + "language_loss": 0.55692065, + "learning_rate": 1.937692351513115e-06, + "loss": 0.57737422, + "num_input_tokens_seen": 94291285, + "router_z_loss_clip": 0.29882812, + "router_z_loss_mlp": 0.01470947, + "step": 4363, + "time_per_iteration": 3.2722747325897217 + }, + { + "auxiliary_loss_clip": 0.01141121, + "auxiliary_loss_mlp": 0.01039545, + "balance_loss_clip": 1.04488003, + "balance_loss_mlp": 1.0230819, + "epoch": 0.5247399747489929, + "flos": 21033490769280.0, + "grad_norm": 1.6533407434528429, + "language_loss": 0.80733919, + "learning_rate": 1.9369137627229297e-06, + "loss": 0.82914591, + "num_input_tokens_seen": 94309685, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.16467285, + "step": 4364, + "time_per_iteration": 2.8337152004241943 + }, + { + "auxiliary_loss_clip": 0.01143412, + "auxiliary_loss_mlp": 0.01041116, + "balance_loss_clip": 1.04956484, + "balance_loss_mlp": 1.02378285, + "epoch": 0.5248602176396321, + "flos": 19025940660480.0, + "grad_norm": 1.9846984746607406, + "language_loss": 0.8816489, + "learning_rate": 1.936135183502877e-06, + "loss": 0.90349418, + "num_input_tokens_seen": 94326985, + "router_z_loss_clip": 0.93798828, + "router_z_loss_mlp": 0.17333984, + "step": 4365, + "time_per_iteration": 2.843992233276367 + }, + { + "auxiliary_loss_clip": 0.01140443, + "auxiliary_loss_mlp": 0.0104877, + "balance_loss_clip": 1.04509437, + "balance_loss_mlp": 1.03091252, + "epoch": 0.5249804605302711, + "flos": 22200084685440.0, + "grad_norm": 2.37516627159083, + "language_loss": 0.80179489, + "learning_rate": 1.935356613971066e-06, + "loss": 0.82368696, + "num_input_tokens_seen": 94347645, + "router_z_loss_clip": 0.95263672, + "router_z_loss_mlp": 0.17858887, + "step": 4366, + "time_per_iteration": 2.806452751159668 + }, + { + "auxiliary_loss_clip": 0.01137352, + "auxiliary_loss_mlp": 0.01047676, + "balance_loss_clip": 1.04358232, + "balance_loss_mlp": 1.03035522, + "epoch": 0.5251007034209102, + "flos": 23805686626560.0, + "grad_norm": 1.7035590089070058, + "language_loss": 0.76644206, + "learning_rate": 1.9345780542456047e-06, + "loss": 0.78829229, + "num_input_tokens_seen": 94367020, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.17321777, + "step": 4367, + "time_per_iteration": 2.8768491744995117 + }, + { + "auxiliary_loss_clip": 0.01138898, + "auxiliary_loss_mlp": 0.01039392, + "balance_loss_clip": 1.0460968, + "balance_loss_mlp": 1.02276254, + "epoch": 0.5252209463115494, + "flos": 23294605962240.0, + "grad_norm": 1.8029791528307955, + "language_loss": 0.71790826, + "learning_rate": 1.9337995044446007e-06, + "loss": 0.7396912, + "num_input_tokens_seen": 94385860, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.16625977, + "step": 4368, + "time_per_iteration": 2.7847096920013428 + }, + { + "auxiliary_loss_clip": 0.01139848, + "auxiliary_loss_mlp": 0.01043096, + "balance_loss_clip": 1.0451684, + "balance_loss_mlp": 1.02569163, + "epoch": 0.5253411892021884, + "flos": 19828687760640.0, + "grad_norm": 2.520744884216176, + "language_loss": 0.80008721, + "learning_rate": 1.9330209646861596e-06, + "loss": 0.8219167, + "num_input_tokens_seen": 94405010, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.1739502, + "step": 4369, + "time_per_iteration": 2.751195192337036 + }, + { + "auxiliary_loss_clip": 0.01143713, + "auxiliary_loss_mlp": 0.01041944, + "balance_loss_clip": 1.04749691, + "balance_loss_mlp": 1.02507591, + "epoch": 0.5254614320928275, + "flos": 24133730561280.0, + "grad_norm": 1.5805053686774395, + "language_loss": 0.77584678, + "learning_rate": 1.9322424350883843e-06, + "loss": 0.79770339, + "num_input_tokens_seen": 94426845, + "router_z_loss_clip": 0.96240234, + "router_z_loss_mlp": 0.1685791, + "step": 4370, + "time_per_iteration": 2.7831766605377197 + }, + { + "auxiliary_loss_clip": 0.01138815, + "auxiliary_loss_mlp": 0.01036787, + "balance_loss_clip": 1.0457921, + "balance_loss_mlp": 1.01984763, + "epoch": 0.5255816749834666, + "flos": 24644954880000.0, + "grad_norm": 1.6709213047867548, + "language_loss": 0.78892463, + "learning_rate": 1.931463915769379e-06, + "loss": 0.81068063, + "num_input_tokens_seen": 94446960, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.16943359, + "step": 4371, + "time_per_iteration": 2.7665841579437256 + }, + { + "auxiliary_loss_clip": 0.01142622, + "auxiliary_loss_mlp": 0.01040333, + "balance_loss_clip": 1.04881489, + "balance_loss_mlp": 1.02362001, + "epoch": 0.5257019178741057, + "flos": 14136595320960.0, + "grad_norm": 2.550975308178701, + "language_loss": 0.74157929, + "learning_rate": 1.930685406847242e-06, + "loss": 0.76340884, + "num_input_tokens_seen": 94461535, + "router_z_loss_clip": 0.93847656, + "router_z_loss_mlp": 0.16711426, + "step": 4372, + "time_per_iteration": 2.727658271789551 + }, + { + "auxiliary_loss_clip": 0.01138574, + "auxiliary_loss_mlp": 0.01039126, + "balance_loss_clip": 1.04667222, + "balance_loss_mlp": 1.02267516, + "epoch": 0.5258221607647448, + "flos": 23548961145600.0, + "grad_norm": 1.4500879266354223, + "language_loss": 0.81660688, + "learning_rate": 1.9299069084400734e-06, + "loss": 0.83838391, + "num_input_tokens_seen": 94482395, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.16455078, + "step": 4373, + "time_per_iteration": 3.559375524520874 + }, + { + "auxiliary_loss_clip": 0.01143248, + "auxiliary_loss_mlp": 0.01042088, + "balance_loss_clip": 1.04703331, + "balance_loss_mlp": 1.02359819, + "epoch": 0.5259424036553839, + "flos": 24966103403520.0, + "grad_norm": 2.391523889361201, + "language_loss": 0.6950593, + "learning_rate": 1.9291284206659717e-06, + "loss": 0.71691263, + "num_input_tokens_seen": 94500580, + "router_z_loss_clip": 0.96289062, + "router_z_loss_mlp": 0.18505859, + "step": 4374, + "time_per_iteration": 2.7599895000457764 + }, + { + "auxiliary_loss_clip": 0.01141873, + "auxiliary_loss_mlp": 0.01044873, + "balance_loss_clip": 1.04774654, + "balance_loss_mlp": 1.02702749, + "epoch": 0.526062646546023, + "flos": 28763908295040.0, + "grad_norm": 2.3735053353923434, + "language_loss": 0.7176404, + "learning_rate": 1.928349943643032e-06, + "loss": 0.73950791, + "num_input_tokens_seen": 94519680, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.17858887, + "step": 4375, + "time_per_iteration": 2.833733558654785 + }, + { + "auxiliary_loss_clip": 0.01140763, + "auxiliary_loss_mlp": 0.01038955, + "balance_loss_clip": 1.0484066, + "balance_loss_mlp": 1.02307606, + "epoch": 0.526182889436662, + "flos": 22821375254400.0, + "grad_norm": 1.6445300527207867, + "language_loss": 0.81839114, + "learning_rate": 1.9275714774893493e-06, + "loss": 0.84018826, + "num_input_tokens_seen": 94539135, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.15881348, + "step": 4376, + "time_per_iteration": 3.5776374340057373 + }, + { + "auxiliary_loss_clip": 0.01140013, + "auxiliary_loss_mlp": 0.01055146, + "balance_loss_clip": 1.04521275, + "balance_loss_mlp": 1.03718138, + "epoch": 0.5263031323273012, + "flos": 22929466256640.0, + "grad_norm": 3.3701683498319754, + "language_loss": 0.72833943, + "learning_rate": 1.9267930223230154e-06, + "loss": 0.75029099, + "num_input_tokens_seen": 94557610, + "router_z_loss_clip": 0.94921875, + "router_z_loss_mlp": 0.17956543, + "step": 4377, + "time_per_iteration": 2.757199287414551 + }, + { + "auxiliary_loss_clip": 0.0114179, + "auxiliary_loss_mlp": 0.01042303, + "balance_loss_clip": 1.04850566, + "balance_loss_mlp": 1.02395689, + "epoch": 0.5264233752179402, + "flos": 17748634049280.0, + "grad_norm": 2.033472363584416, + "language_loss": 0.78330457, + "learning_rate": 1.9260145782621224e-06, + "loss": 0.8051455, + "num_input_tokens_seen": 94575390, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.18359375, + "step": 4378, + "time_per_iteration": 2.800668478012085 + }, + { + "auxiliary_loss_clip": 0.01142061, + "auxiliary_loss_mlp": 0.01041335, + "balance_loss_clip": 1.04707861, + "balance_loss_mlp": 1.02463377, + "epoch": 0.5265436181085793, + "flos": 24421626069120.0, + "grad_norm": 1.9859210866434749, + "language_loss": 0.88415062, + "learning_rate": 1.925236145424758e-06, + "loss": 0.90598458, + "num_input_tokens_seen": 94594210, + "router_z_loss_clip": 0.94970703, + "router_z_loss_mlp": 0.16711426, + "step": 4379, + "time_per_iteration": 2.8431625366210938 + }, + { + "auxiliary_loss_clip": 0.01041512, + "auxiliary_loss_mlp": 0.01010429, + "balance_loss_clip": 1.01115417, + "balance_loss_mlp": 1.00898063, + "epoch": 0.5266638609992185, + "flos": 69207298156800.0, + "grad_norm": 0.6992016481865029, + "language_loss": 0.57549608, + "learning_rate": 1.924457723929012e-06, + "loss": 0.59601551, + "num_input_tokens_seen": 94665020, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.01446533, + "step": 4380, + "time_per_iteration": 4.22179388999939 + }, + { + "auxiliary_loss_clip": 0.01142147, + "auxiliary_loss_mlp": 0.01036392, + "balance_loss_clip": 1.04681492, + "balance_loss_mlp": 1.01898789, + "epoch": 0.5267841038898575, + "flos": 20738699850240.0, + "grad_norm": 1.4772873736866754, + "language_loss": 0.82840085, + "learning_rate": 1.9236793138929685e-06, + "loss": 0.85018623, + "num_input_tokens_seen": 94684290, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.17419434, + "step": 4381, + "time_per_iteration": 2.7369539737701416 + }, + { + "auxiliary_loss_clip": 0.01142762, + "auxiliary_loss_mlp": 0.01046423, + "balance_loss_clip": 1.04692626, + "balance_loss_mlp": 1.02941155, + "epoch": 0.5269043467804966, + "flos": 17234392988160.0, + "grad_norm": 2.158277878454217, + "language_loss": 0.81076032, + "learning_rate": 1.9229009154347133e-06, + "loss": 0.83265221, + "num_input_tokens_seen": 94701880, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.16992188, + "step": 4382, + "time_per_iteration": 3.5458362102508545 + }, + { + "auxiliary_loss_clip": 0.01138503, + "auxiliary_loss_mlp": 0.01047246, + "balance_loss_clip": 1.04603279, + "balance_loss_mlp": 1.03036642, + "epoch": 0.5270245896711357, + "flos": 18223157646720.0, + "grad_norm": 4.153885990625012, + "language_loss": 0.80803347, + "learning_rate": 1.922122528672327e-06, + "loss": 0.82989097, + "num_input_tokens_seen": 94720545, + "router_z_loss_clip": 0.92382812, + "router_z_loss_mlp": 0.16882324, + "step": 4383, + "time_per_iteration": 2.7755308151245117 + }, + { + "auxiliary_loss_clip": 0.01136227, + "auxiliary_loss_mlp": 0.01038543, + "balance_loss_clip": 1.0453285, + "balance_loss_mlp": 1.02259254, + "epoch": 0.5271448325617748, + "flos": 21287558643840.0, + "grad_norm": 2.377607391129202, + "language_loss": 0.78217018, + "learning_rate": 1.9213441537238914e-06, + "loss": 0.80391788, + "num_input_tokens_seen": 94737420, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.1595459, + "step": 4384, + "time_per_iteration": 2.7460787296295166 + }, + { + "auxiliary_loss_clip": 0.01041228, + "auxiliary_loss_mlp": 0.01003129, + "balance_loss_clip": 1.01087654, + "balance_loss_mlp": 1.0014838, + "epoch": 0.5272650754524139, + "flos": 65495497403520.0, + "grad_norm": 0.8345176627101144, + "language_loss": 0.57299012, + "learning_rate": 1.920565790707485e-06, + "loss": 0.59343368, + "num_input_tokens_seen": 94802810, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.01647949, + "step": 4385, + "time_per_iteration": 3.4783449172973633 + }, + { + "auxiliary_loss_clip": 0.0114366, + "auxiliary_loss_mlp": 0.01047824, + "balance_loss_clip": 1.04661155, + "balance_loss_mlp": 1.03016865, + "epoch": 0.527385318343053, + "flos": 19676426008320.0, + "grad_norm": 2.574625963880629, + "language_loss": 0.65907276, + "learning_rate": 1.9197874397411853e-06, + "loss": 0.6809876, + "num_input_tokens_seen": 94819440, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.17651367, + "step": 4386, + "time_per_iteration": 2.7915143966674805 + }, + { + "auxiliary_loss_clip": 0.01139633, + "auxiliary_loss_mlp": 0.01041379, + "balance_loss_clip": 1.04334331, + "balance_loss_mlp": 1.02484429, + "epoch": 0.5275055612336921, + "flos": 12712018947840.0, + "grad_norm": 2.761458042055519, + "language_loss": 0.66874141, + "learning_rate": 1.919009100943067e-06, + "loss": 0.69055152, + "num_input_tokens_seen": 94835130, + "router_z_loss_clip": 0.96289062, + "router_z_loss_mlp": 0.1652832, + "step": 4387, + "time_per_iteration": 2.7191262245178223 + }, + { + "auxiliary_loss_clip": 0.01143961, + "auxiliary_loss_mlp": 0.01052812, + "balance_loss_clip": 1.04622173, + "balance_loss_mlp": 1.03509712, + "epoch": 0.5276258041243311, + "flos": 17749029098880.0, + "grad_norm": 2.0740310644285325, + "language_loss": 0.65434182, + "learning_rate": 1.9182307744312043e-06, + "loss": 0.67630959, + "num_input_tokens_seen": 94852235, + "router_z_loss_clip": 0.9765625, + "router_z_loss_mlp": 0.17712402, + "step": 4388, + "time_per_iteration": 2.7618601322174072 + }, + { + "auxiliary_loss_clip": 0.01140106, + "auxiliary_loss_mlp": 0.01039529, + "balance_loss_clip": 1.04677296, + "balance_loss_mlp": 1.02086115, + "epoch": 0.5277460470149702, + "flos": 22710447077760.0, + "grad_norm": 1.8970646400344446, + "language_loss": 0.7651149, + "learning_rate": 1.9174524603236676e-06, + "loss": 0.78691125, + "num_input_tokens_seen": 94871185, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.18652344, + "step": 4389, + "time_per_iteration": 2.734769821166992 + }, + { + "auxiliary_loss_clip": 0.01143435, + "auxiliary_loss_mlp": 0.01046938, + "balance_loss_clip": 1.04478145, + "balance_loss_mlp": 1.02894974, + "epoch": 0.5278662899056094, + "flos": 19902699734400.0, + "grad_norm": 2.051276265474118, + "language_loss": 0.76885158, + "learning_rate": 1.916674158738527e-06, + "loss": 0.79075527, + "num_input_tokens_seen": 94890090, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.17980957, + "step": 4390, + "time_per_iteration": 2.8276209831237793 + }, + { + "auxiliary_loss_clip": 0.01143049, + "auxiliary_loss_mlp": 0.01044748, + "balance_loss_clip": 1.04601431, + "balance_loss_mlp": 1.02568626, + "epoch": 0.5279865327962484, + "flos": 18005215875840.0, + "grad_norm": 2.183991981890627, + "language_loss": 0.60219187, + "learning_rate": 1.9158958697938506e-06, + "loss": 0.62406981, + "num_input_tokens_seen": 94908470, + "router_z_loss_clip": 0.97021484, + "router_z_loss_mlp": 0.19055176, + "step": 4391, + "time_per_iteration": 2.873156785964966 + }, + { + "auxiliary_loss_clip": 0.01139253, + "auxiliary_loss_mlp": 0.01046162, + "balance_loss_clip": 1.04422688, + "balance_loss_mlp": 1.02884054, + "epoch": 0.5281067756868875, + "flos": 15924443892480.0, + "grad_norm": 2.1684840535193306, + "language_loss": 0.85985196, + "learning_rate": 1.9151175936077032e-06, + "loss": 0.88170606, + "num_input_tokens_seen": 94923440, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.1730957, + "step": 4392, + "time_per_iteration": 2.7611074447631836 + }, + { + "auxiliary_loss_clip": 0.01138002, + "auxiliary_loss_mlp": 0.01045156, + "balance_loss_clip": 1.04495609, + "balance_loss_mlp": 1.02837086, + "epoch": 0.5282270185775266, + "flos": 19426488197760.0, + "grad_norm": 1.5581667465493028, + "language_loss": 0.7925728, + "learning_rate": 1.9143393302981507e-06, + "loss": 0.81440437, + "num_input_tokens_seen": 94941125, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.16784668, + "step": 4393, + "time_per_iteration": 2.760646343231201 + }, + { + "auxiliary_loss_clip": 0.01141095, + "auxiliary_loss_mlp": 0.01042231, + "balance_loss_clip": 1.04525518, + "balance_loss_mlp": 1.02597094, + "epoch": 0.5283472614681657, + "flos": 16399613934720.0, + "grad_norm": 2.080152358787955, + "language_loss": 0.83026779, + "learning_rate": 1.913561079983252e-06, + "loss": 0.85210109, + "num_input_tokens_seen": 94959950, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.16259766, + "step": 4394, + "time_per_iteration": 2.7324395179748535 + }, + { + "auxiliary_loss_clip": 0.01148201, + "auxiliary_loss_mlp": 0.01051061, + "balance_loss_clip": 1.04819012, + "balance_loss_mlp": 1.03215456, + "epoch": 0.5284675043588047, + "flos": 26760524163840.0, + "grad_norm": 2.395781207180484, + "language_loss": 0.74929678, + "learning_rate": 1.9127828427810693e-06, + "loss": 0.77128935, + "num_input_tokens_seen": 94980515, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.18896484, + "step": 4395, + "time_per_iteration": 2.791203498840332 + }, + { + "auxiliary_loss_clip": 0.0114282, + "auxiliary_loss_mlp": 0.01037673, + "balance_loss_clip": 1.04550004, + "balance_loss_mlp": 1.02057803, + "epoch": 0.5285877472494439, + "flos": 19899898473600.0, + "grad_norm": 2.100219427381942, + "language_loss": 0.81007743, + "learning_rate": 1.9120046188096607e-06, + "loss": 0.83188236, + "num_input_tokens_seen": 94998560, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.17089844, + "step": 4396, + "time_per_iteration": 2.8174452781677246 + }, + { + "auxiliary_loss_clip": 0.01142398, + "auxiliary_loss_mlp": 0.01043762, + "balance_loss_clip": 1.04761839, + "balance_loss_mlp": 1.02570152, + "epoch": 0.528707990140083, + "flos": 20011257613440.0, + "grad_norm": 1.8219661484066398, + "language_loss": 0.74194658, + "learning_rate": 1.9112264081870804e-06, + "loss": 0.76380813, + "num_input_tokens_seen": 95016950, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.18054199, + "step": 4397, + "time_per_iteration": 2.7423152923583984 + }, + { + "auxiliary_loss_clip": 0.01144232, + "auxiliary_loss_mlp": 0.01041532, + "balance_loss_clip": 1.04459989, + "balance_loss_mlp": 1.02266097, + "epoch": 0.528828233030722, + "flos": 20667956014080.0, + "grad_norm": 2.3362272765896686, + "language_loss": 0.76110584, + "learning_rate": 1.9104482110313843e-06, + "loss": 0.78296351, + "num_input_tokens_seen": 95036540, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.1887207, + "step": 4398, + "time_per_iteration": 2.8271470069885254 + }, + { + "auxiliary_loss_clip": 0.01137847, + "auxiliary_loss_mlp": 0.01042735, + "balance_loss_clip": 1.04470789, + "balance_loss_mlp": 1.02490139, + "epoch": 0.5289484759213612, + "flos": 25192448956800.0, + "grad_norm": 1.6683044246035672, + "language_loss": 0.73866373, + "learning_rate": 1.909670027460623e-06, + "loss": 0.76046956, + "num_input_tokens_seen": 95053840, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.1784668, + "step": 4399, + "time_per_iteration": 2.8218979835510254 + }, + { + "auxiliary_loss_clip": 0.01141295, + "auxiliary_loss_mlp": 0.01041607, + "balance_loss_clip": 1.0473938, + "balance_loss_mlp": 1.02584779, + "epoch": 0.5290687188120002, + "flos": 31139255715840.0, + "grad_norm": 1.824411638357081, + "language_loss": 0.71930164, + "learning_rate": 1.908891857592847e-06, + "loss": 0.74113065, + "num_input_tokens_seen": 95074910, + "router_z_loss_clip": 0.93896484, + "router_z_loss_mlp": 0.1574707, + "step": 4400, + "time_per_iteration": 2.8504786491394043 + }, + { + "auxiliary_loss_clip": 0.0114104, + "auxiliary_loss_mlp": 0.01042069, + "balance_loss_clip": 1.04574049, + "balance_loss_mlp": 1.02548647, + "epoch": 0.5291889617026393, + "flos": 20119851406080.0, + "grad_norm": 2.5974249705579457, + "language_loss": 0.90435302, + "learning_rate": 1.9081137015461034e-06, + "loss": 0.92618412, + "num_input_tokens_seen": 95090985, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.16577148, + "step": 4401, + "time_per_iteration": 2.7375411987304688 + }, + { + "auxiliary_loss_clip": 0.01146344, + "auxiliary_loss_mlp": 0.01040513, + "balance_loss_clip": 1.05005646, + "balance_loss_mlp": 1.02370405, + "epoch": 0.5293092045932785, + "flos": 19643747610240.0, + "grad_norm": 1.802930138176083, + "language_loss": 0.91008592, + "learning_rate": 1.9073355594384383e-06, + "loss": 0.9319545, + "num_input_tokens_seen": 95109225, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.16821289, + "step": 4402, + "time_per_iteration": 3.6208202838897705 + }, + { + "auxiliary_loss_clip": 0.01141753, + "auxiliary_loss_mlp": 0.01044775, + "balance_loss_clip": 1.04756927, + "balance_loss_mlp": 1.02739394, + "epoch": 0.5294294474839175, + "flos": 24317736958080.0, + "grad_norm": 1.8397883890619764, + "language_loss": 0.8068068, + "learning_rate": 1.906557431387895e-06, + "loss": 0.82867205, + "num_input_tokens_seen": 95128215, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.17382812, + "step": 4403, + "time_per_iteration": 2.8149077892303467 + }, + { + "auxiliary_loss_clip": 0.01140779, + "auxiliary_loss_mlp": 0.01046061, + "balance_loss_clip": 1.04668224, + "balance_loss_mlp": 1.0280726, + "epoch": 0.5295496903745566, + "flos": 18875941464960.0, + "grad_norm": 1.8853499652529195, + "language_loss": 0.78747773, + "learning_rate": 1.905779317512516e-06, + "loss": 0.80934608, + "num_input_tokens_seen": 95145760, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.17987061, + "step": 4404, + "time_per_iteration": 2.818023443222046 + }, + { + "auxiliary_loss_clip": 0.01141671, + "auxiliary_loss_mlp": 0.0104639, + "balance_loss_clip": 1.04637218, + "balance_loss_mlp": 1.02896166, + "epoch": 0.5296699332651957, + "flos": 20923101296640.0, + "grad_norm": 2.4083953210462243, + "language_loss": 0.80508494, + "learning_rate": 1.9050012179303385e-06, + "loss": 0.82696557, + "num_input_tokens_seen": 95164270, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.17419434, + "step": 4405, + "time_per_iteration": 3.6406610012054443 + }, + { + "auxiliary_loss_clip": 0.01144, + "auxiliary_loss_mlp": 0.01043718, + "balance_loss_clip": 1.04453909, + "balance_loss_mlp": 1.02614605, + "epoch": 0.5297901761558348, + "flos": 22046745525120.0, + "grad_norm": 2.2363710499403195, + "language_loss": 0.69438738, + "learning_rate": 1.904223132759401e-06, + "loss": 0.71626461, + "num_input_tokens_seen": 95182870, + "router_z_loss_clip": 0.99511719, + "router_z_loss_mlp": 0.175354, + "step": 4406, + "time_per_iteration": 2.7977020740509033 + }, + { + "auxiliary_loss_clip": 0.01142202, + "auxiliary_loss_mlp": 0.01037604, + "balance_loss_clip": 1.04596853, + "balance_loss_mlp": 1.02031875, + "epoch": 0.5299104190464738, + "flos": 21798495653760.0, + "grad_norm": 2.348469537491469, + "language_loss": 0.69232291, + "learning_rate": 1.9034450621177383e-06, + "loss": 0.71412098, + "num_input_tokens_seen": 95201190, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.17285156, + "step": 4407, + "time_per_iteration": 2.7692887783050537 + }, + { + "auxiliary_loss_clip": 0.0114657, + "auxiliary_loss_mlp": 0.01052381, + "balance_loss_clip": 1.04865587, + "balance_loss_mlp": 1.03401065, + "epoch": 0.530030661937113, + "flos": 14720790119040.0, + "grad_norm": 2.104178252575755, + "language_loss": 0.70602328, + "learning_rate": 1.9026670061233824e-06, + "loss": 0.7280128, + "num_input_tokens_seen": 95218625, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.18371582, + "step": 4408, + "time_per_iteration": 2.726025342941284 + }, + { + "auxiliary_loss_clip": 0.01139524, + "auxiliary_loss_mlp": 0.01035046, + "balance_loss_clip": 1.04668832, + "balance_loss_mlp": 1.01873255, + "epoch": 0.5301509048277521, + "flos": 21251504367360.0, + "grad_norm": 1.6127584808832405, + "language_loss": 0.80354607, + "learning_rate": 1.901888964894365e-06, + "loss": 0.82529175, + "num_input_tokens_seen": 95237665, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.16314697, + "step": 4409, + "time_per_iteration": 3.524644613265991 + }, + { + "auxiliary_loss_clip": 0.01143033, + "auxiliary_loss_mlp": 0.01039866, + "balance_loss_clip": 1.04546726, + "balance_loss_mlp": 1.02239037, + "epoch": 0.5302711477183911, + "flos": 25957058791680.0, + "grad_norm": 2.0575990954332943, + "language_loss": 0.67725003, + "learning_rate": 1.9011109385487134e-06, + "loss": 0.69907898, + "num_input_tokens_seen": 95258915, + "router_z_loss_clip": 0.97558594, + "router_z_loss_mlp": 0.17480469, + "step": 4410, + "time_per_iteration": 2.7588417530059814 + }, + { + "auxiliary_loss_clip": 0.01141895, + "auxiliary_loss_mlp": 0.01042771, + "balance_loss_clip": 1.04491043, + "balance_loss_mlp": 1.02440119, + "epoch": 0.5303913906090303, + "flos": 22273126992000.0, + "grad_norm": 2.343972154349563, + "language_loss": 0.66165477, + "learning_rate": 1.900332927204454e-06, + "loss": 0.68350148, + "num_input_tokens_seen": 95277365, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.18371582, + "step": 4411, + "time_per_iteration": 3.530435562133789 + }, + { + "auxiliary_loss_clip": 0.01143532, + "auxiliary_loss_mlp": 0.01049454, + "balance_loss_clip": 1.04485071, + "balance_loss_mlp": 1.03104782, + "epoch": 0.5305116334996693, + "flos": 24936010784640.0, + "grad_norm": 1.7772156631482727, + "language_loss": 0.76914799, + "learning_rate": 1.8995549309796097e-06, + "loss": 0.79107785, + "num_input_tokens_seen": 95296670, + "router_z_loss_clip": 0.98583984, + "router_z_loss_mlp": 0.18408203, + "step": 4412, + "time_per_iteration": 2.7752788066864014 + }, + { + "auxiliary_loss_clip": 0.01143521, + "auxiliary_loss_mlp": 0.01050043, + "balance_loss_clip": 1.04856944, + "balance_loss_mlp": 1.0310173, + "epoch": 0.5306318763903084, + "flos": 20189338266240.0, + "grad_norm": 1.5973216451456567, + "language_loss": 0.76434088, + "learning_rate": 1.8987769499922028e-06, + "loss": 0.78627658, + "num_input_tokens_seen": 95315640, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.19042969, + "step": 4413, + "time_per_iteration": 2.7240302562713623 + }, + { + "auxiliary_loss_clip": 0.01139576, + "auxiliary_loss_mlp": 0.01037845, + "balance_loss_clip": 1.04617369, + "balance_loss_mlp": 1.02084589, + "epoch": 0.5307521192809476, + "flos": 20266366982400.0, + "grad_norm": 2.138963126949252, + "language_loss": 0.70477825, + "learning_rate": 1.897998984360252e-06, + "loss": 0.72655249, + "num_input_tokens_seen": 95334610, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.16992188, + "step": 4414, + "time_per_iteration": 2.7545769214630127 + }, + { + "auxiliary_loss_clip": 0.01138357, + "auxiliary_loss_mlp": 0.01039973, + "balance_loss_clip": 1.04641294, + "balance_loss_mlp": 1.02389169, + "epoch": 0.5308723621715866, + "flos": 28844276976000.0, + "grad_norm": 1.3309645977175044, + "language_loss": 0.78441346, + "learning_rate": 1.897221034201775e-06, + "loss": 0.80619681, + "num_input_tokens_seen": 95358350, + "router_z_loss_clip": 0.91943359, + "router_z_loss_mlp": 0.1607666, + "step": 4415, + "time_per_iteration": 2.779332399368286 + }, + { + "auxiliary_loss_clip": 0.01137987, + "auxiliary_loss_mlp": 0.01035774, + "balance_loss_clip": 1.04668379, + "balance_loss_mlp": 1.01970506, + "epoch": 0.5309926050622257, + "flos": 27457766040960.0, + "grad_norm": 1.4925237563445994, + "language_loss": 0.66637242, + "learning_rate": 1.8964430996347842e-06, + "loss": 0.68811005, + "num_input_tokens_seen": 95379900, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.16064453, + "step": 4416, + "time_per_iteration": 2.8076250553131104 + }, + { + "auxiliary_loss_clip": 0.01144127, + "auxiliary_loss_mlp": 0.01039951, + "balance_loss_clip": 1.04730511, + "balance_loss_mlp": 1.0219388, + "epoch": 0.5311128479528648, + "flos": 20514545026560.0, + "grad_norm": 1.5778223525178954, + "language_loss": 0.82486975, + "learning_rate": 1.8956651807772931e-06, + "loss": 0.84671044, + "num_input_tokens_seen": 95397935, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.17993164, + "step": 4417, + "time_per_iteration": 2.774080276489258 + }, + { + "auxiliary_loss_clip": 0.01138075, + "auxiliary_loss_mlp": 0.01042596, + "balance_loss_clip": 1.04587102, + "balance_loss_mlp": 1.02683663, + "epoch": 0.5312330908435039, + "flos": 21397660807680.0, + "grad_norm": 1.5531772793466088, + "language_loss": 0.84004521, + "learning_rate": 1.8948872777473115e-06, + "loss": 0.86185193, + "num_input_tokens_seen": 95415890, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.15771484, + "step": 4418, + "time_per_iteration": 2.741978168487549 + }, + { + "auxiliary_loss_clip": 0.01144837, + "auxiliary_loss_mlp": 0.01044514, + "balance_loss_clip": 1.04937553, + "balance_loss_mlp": 1.02608407, + "epoch": 0.531353333734143, + "flos": 24717350741760.0, + "grad_norm": 1.7857005083228046, + "language_loss": 0.63853109, + "learning_rate": 1.8941093906628458e-06, + "loss": 0.66042459, + "num_input_tokens_seen": 95433675, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.18432617, + "step": 4419, + "time_per_iteration": 2.7888128757476807 + }, + { + "auxiliary_loss_clip": 0.0114006, + "auxiliary_loss_mlp": 0.01040502, + "balance_loss_clip": 1.04445601, + "balance_loss_mlp": 1.02306187, + "epoch": 0.531473576624782, + "flos": 30480689808000.0, + "grad_norm": 1.8322962096176043, + "language_loss": 0.70924067, + "learning_rate": 1.893331519641902e-06, + "loss": 0.73104632, + "num_input_tokens_seen": 95455820, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.17443848, + "step": 4420, + "time_per_iteration": 2.8097102642059326 + }, + { + "auxiliary_loss_clip": 0.01141201, + "auxiliary_loss_mlp": 0.01040016, + "balance_loss_clip": 1.04680181, + "balance_loss_mlp": 1.02274215, + "epoch": 0.5315938195154212, + "flos": 23002975440000.0, + "grad_norm": 2.490591157619427, + "language_loss": 0.74095351, + "learning_rate": 1.8925536648024815e-06, + "loss": 0.76276571, + "num_input_tokens_seen": 95473240, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.17272949, + "step": 4421, + "time_per_iteration": 2.814432144165039 + }, + { + "auxiliary_loss_clip": 0.01141945, + "auxiliary_loss_mlp": 0.01038137, + "balance_loss_clip": 1.04612184, + "balance_loss_mlp": 1.0217098, + "epoch": 0.5317140624060602, + "flos": 22748584343040.0, + "grad_norm": 2.0757650226345383, + "language_loss": 0.76257777, + "learning_rate": 1.8917758262625849e-06, + "loss": 0.78437859, + "num_input_tokens_seen": 95493480, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.16442871, + "step": 4422, + "time_per_iteration": 2.773806571960449 + }, + { + "auxiliary_loss_clip": 0.0113873, + "auxiliary_loss_mlp": 0.01041109, + "balance_loss_clip": 1.04529929, + "balance_loss_mlp": 1.0243001, + "epoch": 0.5318343052966993, + "flos": 22821087945600.0, + "grad_norm": 1.6887503908901091, + "language_loss": 0.80635953, + "learning_rate": 1.8909980041402089e-06, + "loss": 0.82815796, + "num_input_tokens_seen": 95512075, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16821289, + "step": 4423, + "time_per_iteration": 2.793212413787842 + }, + { + "auxiliary_loss_clip": 0.01141286, + "auxiliary_loss_mlp": 0.01043327, + "balance_loss_clip": 1.04787445, + "balance_loss_mlp": 1.02641153, + "epoch": 0.5319545481873384, + "flos": 13626089274240.0, + "grad_norm": 2.1588117368442683, + "language_loss": 0.6621387, + "learning_rate": 1.8902201985533494e-06, + "loss": 0.68398488, + "num_input_tokens_seen": 95529340, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.16906738, + "step": 4424, + "time_per_iteration": 2.741164207458496 + }, + { + "auxiliary_loss_clip": 0.01138872, + "auxiliary_loss_mlp": 0.01037306, + "balance_loss_clip": 1.04515362, + "balance_loss_mlp": 1.02023506, + "epoch": 0.5320747910779775, + "flos": 22162522037760.0, + "grad_norm": 1.8091813861464698, + "language_loss": 0.74808532, + "learning_rate": 1.8894424096199983e-06, + "loss": 0.7698471, + "num_input_tokens_seen": 95548545, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.17077637, + "step": 4425, + "time_per_iteration": 2.790761947631836 + }, + { + "auxiliary_loss_clip": 0.01144919, + "auxiliary_loss_mlp": 0.01044665, + "balance_loss_clip": 1.04926562, + "balance_loss_mlp": 1.02661633, + "epoch": 0.5321950339686166, + "flos": 18588081870720.0, + "grad_norm": 2.033403970849386, + "language_loss": 0.85737777, + "learning_rate": 1.8886646374581463e-06, + "loss": 0.87927359, + "num_input_tokens_seen": 95567770, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.18029785, + "step": 4426, + "time_per_iteration": 2.7627382278442383 + }, + { + "auxiliary_loss_clip": 0.01143987, + "auxiliary_loss_mlp": 0.01050514, + "balance_loss_clip": 1.04773045, + "balance_loss_mlp": 1.03110695, + "epoch": 0.5323152768592557, + "flos": 22856818999680.0, + "grad_norm": 1.730179026778341, + "language_loss": 0.71322024, + "learning_rate": 1.8878868821857795e-06, + "loss": 0.73516524, + "num_input_tokens_seen": 95587420, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.19396973, + "step": 4427, + "time_per_iteration": 2.7504191398620605 + }, + { + "auxiliary_loss_clip": 0.01142621, + "auxiliary_loss_mlp": 0.01049415, + "balance_loss_clip": 1.04656029, + "balance_loss_mlp": 1.03135514, + "epoch": 0.5324355197498948, + "flos": 33948690998400.0, + "grad_norm": 2.1112290627449757, + "language_loss": 0.75301588, + "learning_rate": 1.8871091439208838e-06, + "loss": 0.7749362, + "num_input_tokens_seen": 95609030, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.18054199, + "step": 4428, + "time_per_iteration": 2.8683547973632812 + }, + { + "auxiliary_loss_clip": 0.01147489, + "auxiliary_loss_mlp": 0.01051894, + "balance_loss_clip": 1.0492537, + "balance_loss_mlp": 1.0330826, + "epoch": 0.5325557626405338, + "flos": 23256720092160.0, + "grad_norm": 1.9615028805822339, + "language_loss": 0.77552354, + "learning_rate": 1.8863314227814414e-06, + "loss": 0.79751742, + "num_input_tokens_seen": 95627340, + "router_z_loss_clip": 0.98193359, + "router_z_loss_mlp": 0.18823242, + "step": 4429, + "time_per_iteration": 2.710477590560913 + }, + { + "auxiliary_loss_clip": 0.01143185, + "auxiliary_loss_mlp": 0.01040314, + "balance_loss_clip": 1.04589224, + "balance_loss_mlp": 1.02296913, + "epoch": 0.532676005531173, + "flos": 26718687797760.0, + "grad_norm": 2.1977877851228604, + "language_loss": 0.48740932, + "learning_rate": 1.8855537188854313e-06, + "loss": 0.50924432, + "num_input_tokens_seen": 95646315, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.17321777, + "step": 4430, + "time_per_iteration": 2.790631055831909 + }, + { + "auxiliary_loss_clip": 0.01139968, + "auxiliary_loss_mlp": 0.01044907, + "balance_loss_clip": 1.04488611, + "balance_loss_mlp": 1.02764535, + "epoch": 0.5327962484218121, + "flos": 17894610921600.0, + "grad_norm": 1.9367744967755445, + "language_loss": 0.78394741, + "learning_rate": 1.8847760323508315e-06, + "loss": 0.80579615, + "num_input_tokens_seen": 95665220, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.17272949, + "step": 4431, + "time_per_iteration": 3.504056692123413 + }, + { + "auxiliary_loss_clip": 0.01136518, + "auxiliary_loss_mlp": 0.01045812, + "balance_loss_clip": 1.0456388, + "balance_loss_mlp": 1.02871704, + "epoch": 0.5329164913124511, + "flos": 17925385898880.0, + "grad_norm": 1.648625334830258, + "language_loss": 0.75470054, + "learning_rate": 1.883998363295616e-06, + "loss": 0.77652383, + "num_input_tokens_seen": 95682700, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.17102051, + "step": 4432, + "time_per_iteration": 2.726950168609619 + }, + { + "auxiliary_loss_clip": 0.01041949, + "auxiliary_loss_mlp": 0.01004175, + "balance_loss_clip": 1.0114851, + "balance_loss_mlp": 1.0024699, + "epoch": 0.5330367342030903, + "flos": 57254178781440.0, + "grad_norm": 0.8758000915960151, + "language_loss": 0.62565148, + "learning_rate": 1.8832207118377565e-06, + "loss": 0.64611268, + "num_input_tokens_seen": 95738070, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.01708984, + "step": 4433, + "time_per_iteration": 3.1753103733062744 + }, + { + "auxiliary_loss_clip": 0.0113858, + "auxiliary_loss_mlp": 0.01042596, + "balance_loss_clip": 1.0457015, + "balance_loss_mlp": 1.02609789, + "epoch": 0.5331569770937293, + "flos": 17420518287360.0, + "grad_norm": 1.8947514133314498, + "language_loss": 0.69973975, + "learning_rate": 1.882443078095222e-06, + "loss": 0.72155154, + "num_input_tokens_seen": 95756950, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.16503906, + "step": 4434, + "time_per_iteration": 3.5013082027435303 + }, + { + "auxiliary_loss_clip": 0.01041381, + "auxiliary_loss_mlp": 0.01000105, + "balance_loss_clip": 1.01117921, + "balance_loss_mlp": 0.99853152, + "epoch": 0.5332772199843684, + "flos": 56750783627520.0, + "grad_norm": 0.869290210426392, + "language_loss": 0.6687851, + "learning_rate": 1.8816654621859794e-06, + "loss": 0.68919992, + "num_input_tokens_seen": 95816615, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.01574707, + "step": 4435, + "time_per_iteration": 3.1657495498657227 + }, + { + "auxiliary_loss_clip": 0.01137301, + "auxiliary_loss_mlp": 0.01043657, + "balance_loss_clip": 1.04526317, + "balance_loss_mlp": 1.02697921, + "epoch": 0.5333974628750076, + "flos": 18697753071360.0, + "grad_norm": 2.1026824249140077, + "language_loss": 0.72178352, + "learning_rate": 1.8808878642279915e-06, + "loss": 0.74359316, + "num_input_tokens_seen": 95832020, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.16662598, + "step": 4436, + "time_per_iteration": 2.8776164054870605 + }, + { + "auxiliary_loss_clip": 0.0114808, + "auxiliary_loss_mlp": 0.01051099, + "balance_loss_clip": 1.04871416, + "balance_loss_mlp": 1.03265774, + "epoch": 0.5335177057656466, + "flos": 23805507058560.0, + "grad_norm": 2.1678874129575134, + "language_loss": 0.65533578, + "learning_rate": 1.8801102843392209e-06, + "loss": 0.67732763, + "num_input_tokens_seen": 95851425, + "router_z_loss_clip": 0.99462891, + "router_z_loss_mlp": 0.18444824, + "step": 4437, + "time_per_iteration": 2.797117233276367 + }, + { + "auxiliary_loss_clip": 0.01140347, + "auxiliary_loss_mlp": 0.01041735, + "balance_loss_clip": 1.04561234, + "balance_loss_mlp": 1.02419949, + "epoch": 0.5336379486562857, + "flos": 25078683605760.0, + "grad_norm": 1.5730673605278531, + "language_loss": 0.8518877, + "learning_rate": 1.8793327226376238e-06, + "loss": 0.87370861, + "num_input_tokens_seen": 95870745, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.17529297, + "step": 4438, + "time_per_iteration": 3.6697182655334473 + }, + { + "auxiliary_loss_clip": 0.01143583, + "auxiliary_loss_mlp": 0.01046494, + "balance_loss_clip": 1.04817545, + "balance_loss_mlp": 1.02836263, + "epoch": 0.5337581915469248, + "flos": 21396691140480.0, + "grad_norm": 1.8056137279276945, + "language_loss": 0.80337304, + "learning_rate": 1.8785551792411569e-06, + "loss": 0.82527381, + "num_input_tokens_seen": 95889755, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.18127441, + "step": 4439, + "time_per_iteration": 2.824958086013794 + }, + { + "auxiliary_loss_clip": 0.01139432, + "auxiliary_loss_mlp": 0.01044644, + "balance_loss_clip": 1.04604316, + "balance_loss_mlp": 1.02772856, + "epoch": 0.5338784344375639, + "flos": 14865905064960.0, + "grad_norm": 2.8003766990218275, + "language_loss": 0.83204973, + "learning_rate": 1.8777776542677733e-06, + "loss": 0.85389054, + "num_input_tokens_seen": 95907805, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.16918945, + "step": 4440, + "time_per_iteration": 3.569411516189575 + }, + { + "auxiliary_loss_clip": 0.01141498, + "auxiliary_loss_mlp": 0.01042129, + "balance_loss_clip": 1.04388738, + "balance_loss_mlp": 1.02349615, + "epoch": 0.5339986773282029, + "flos": 20813501923200.0, + "grad_norm": 2.1965062846593906, + "language_loss": 0.73247039, + "learning_rate": 1.8770001478354216e-06, + "loss": 0.75430667, + "num_input_tokens_seen": 95927480, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.1862793, + "step": 4441, + "time_per_iteration": 2.7861130237579346 + }, + { + "auxiliary_loss_clip": 0.01145168, + "auxiliary_loss_mlp": 0.01051475, + "balance_loss_clip": 1.04795194, + "balance_loss_mlp": 1.03305697, + "epoch": 0.5341189202188421, + "flos": 17969089772160.0, + "grad_norm": 2.4259596465806554, + "language_loss": 0.84025323, + "learning_rate": 1.8762226600620504e-06, + "loss": 0.86221969, + "num_input_tokens_seen": 95946095, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.18408203, + "step": 4442, + "time_per_iteration": 2.8326501846313477 + }, + { + "auxiliary_loss_clip": 0.01147638, + "auxiliary_loss_mlp": 0.01047116, + "balance_loss_clip": 1.04759312, + "balance_loss_mlp": 1.02760112, + "epoch": 0.5342391631094812, + "flos": 11031866328960.0, + "grad_norm": 2.4101501036396074, + "language_loss": 0.59304821, + "learning_rate": 1.8754451910656031e-06, + "loss": 0.61499572, + "num_input_tokens_seen": 95959995, + "router_z_loss_clip": 1.00048828, + "router_z_loss_mlp": 0.19506836, + "step": 4443, + "time_per_iteration": 2.7562930583953857 + }, + { + "auxiliary_loss_clip": 0.01143903, + "auxiliary_loss_mlp": 0.01042725, + "balance_loss_clip": 1.04653096, + "balance_loss_mlp": 1.02564216, + "epoch": 0.5343594060001202, + "flos": 15339135772800.0, + "grad_norm": 1.9323966843121105, + "language_loss": 0.8265233, + "learning_rate": 1.8746677409640212e-06, + "loss": 0.84838963, + "num_input_tokens_seen": 95977095, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.17089844, + "step": 4444, + "time_per_iteration": 2.7408804893493652 + }, + { + "auxiliary_loss_clip": 0.011422, + "auxiliary_loss_mlp": 0.01047815, + "balance_loss_clip": 1.04656076, + "balance_loss_mlp": 1.03024328, + "epoch": 0.5344796488907594, + "flos": 26900898514560.0, + "grad_norm": 1.7269499597399487, + "language_loss": 0.84944963, + "learning_rate": 1.8738903098752432e-06, + "loss": 0.87134969, + "num_input_tokens_seen": 95996225, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.17565918, + "step": 4445, + "time_per_iteration": 2.7746524810791016 + }, + { + "auxiliary_loss_clip": 0.01145756, + "auxiliary_loss_mlp": 0.01042143, + "balance_loss_clip": 1.04916167, + "balance_loss_mlp": 1.02473795, + "epoch": 0.5345998917813984, + "flos": 25411216740480.0, + "grad_norm": 1.9998930278284344, + "language_loss": 0.73541629, + "learning_rate": 1.8731128979172052e-06, + "loss": 0.75729525, + "num_input_tokens_seen": 96015425, + "router_z_loss_clip": 0.96582031, + "router_z_loss_mlp": 0.17407227, + "step": 4446, + "time_per_iteration": 2.762523651123047 + }, + { + "auxiliary_loss_clip": 0.01139937, + "auxiliary_loss_mlp": 0.01038661, + "balance_loss_clip": 1.04469275, + "balance_loss_mlp": 1.02148342, + "epoch": 0.5347201346720375, + "flos": 32853379622400.0, + "grad_norm": 2.1309772344033355, + "language_loss": 0.67287564, + "learning_rate": 1.8723355052078394e-06, + "loss": 0.69466162, + "num_input_tokens_seen": 96035460, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.171875, + "step": 4447, + "time_per_iteration": 2.8254082202911377 + }, + { + "auxiliary_loss_clip": 0.01147653, + "auxiliary_loss_mlp": 0.01049531, + "balance_loss_clip": 1.04659939, + "balance_loss_mlp": 1.03064775, + "epoch": 0.5348403775626767, + "flos": 17967940536960.0, + "grad_norm": 2.376457702162526, + "language_loss": 0.77742326, + "learning_rate": 1.8715581318650765e-06, + "loss": 0.79939508, + "num_input_tokens_seen": 96054515, + "router_z_loss_clip": 1.00878906, + "router_z_loss_mlp": 0.1887207, + "step": 4448, + "time_per_iteration": 2.7404582500457764 + }, + { + "auxiliary_loss_clip": 0.01149276, + "auxiliary_loss_mlp": 0.01041102, + "balance_loss_clip": 1.04941702, + "balance_loss_mlp": 1.02264881, + "epoch": 0.5349606204533157, + "flos": 17603339535360.0, + "grad_norm": 2.4406416845258914, + "language_loss": 0.81662339, + "learning_rate": 1.8707807780068422e-06, + "loss": 0.83852714, + "num_input_tokens_seen": 96072330, + "router_z_loss_clip": 0.99951172, + "router_z_loss_mlp": 0.18457031, + "step": 4449, + "time_per_iteration": 2.7794909477233887 + }, + { + "auxiliary_loss_clip": 0.01142858, + "auxiliary_loss_mlp": 0.01049202, + "balance_loss_clip": 1.0469321, + "balance_loss_mlp": 1.03198779, + "epoch": 0.5350808633439548, + "flos": 29167831710720.0, + "grad_norm": 1.835925971078625, + "language_loss": 0.66349185, + "learning_rate": 1.8700034437510611e-06, + "loss": 0.68541241, + "num_input_tokens_seen": 96092425, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.17211914, + "step": 4450, + "time_per_iteration": 2.8367254734039307 + }, + { + "auxiliary_loss_clip": 0.01143114, + "auxiliary_loss_mlp": 0.01047926, + "balance_loss_clip": 1.04825616, + "balance_loss_mlp": 1.0299257, + "epoch": 0.5352011062345938, + "flos": 19499997381120.0, + "grad_norm": 2.235254168089322, + "language_loss": 0.81906116, + "learning_rate": 1.8692261292156549e-06, + "loss": 0.84097159, + "num_input_tokens_seen": 96111660, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.17993164, + "step": 4451, + "time_per_iteration": 2.7922067642211914 + }, + { + "auxiliary_loss_clip": 0.01141073, + "auxiliary_loss_mlp": 0.01037094, + "balance_loss_clip": 1.05010986, + "balance_loss_mlp": 1.02150166, + "epoch": 0.535321349125233, + "flos": 23477642691840.0, + "grad_norm": 2.0820263297229937, + "language_loss": 0.81189632, + "learning_rate": 1.8684488345185401e-06, + "loss": 0.83367801, + "num_input_tokens_seen": 96131835, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.15576172, + "step": 4452, + "time_per_iteration": 2.831122398376465 + }, + { + "auxiliary_loss_clip": 0.01144035, + "auxiliary_loss_mlp": 0.01044311, + "balance_loss_clip": 1.04898322, + "balance_loss_mlp": 1.02628636, + "epoch": 0.535441592015872, + "flos": 20478059786880.0, + "grad_norm": 2.4745836389736837, + "language_loss": 0.78551447, + "learning_rate": 1.8676715597776332e-06, + "loss": 0.80739796, + "num_input_tokens_seen": 96150180, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.18029785, + "step": 4453, + "time_per_iteration": 2.71706223487854 + }, + { + "auxiliary_loss_clip": 0.01140284, + "auxiliary_loss_mlp": 0.01046692, + "balance_loss_clip": 1.04749787, + "balance_loss_mlp": 1.03040242, + "epoch": 0.5355618349065111, + "flos": 19573147428480.0, + "grad_norm": 1.7975700153638943, + "language_loss": 0.76455361, + "learning_rate": 1.8668943051108455e-06, + "loss": 0.78642333, + "num_input_tokens_seen": 96167485, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.16290283, + "step": 4454, + "time_per_iteration": 2.72511887550354 + }, + { + "auxiliary_loss_clip": 0.01144165, + "auxiliary_loss_mlp": 0.0104655, + "balance_loss_clip": 1.04779744, + "balance_loss_mlp": 1.02895451, + "epoch": 0.5356820777971503, + "flos": 24024633978240.0, + "grad_norm": 1.6973211187956851, + "language_loss": 0.76348835, + "learning_rate": 1.8661170706360856e-06, + "loss": 0.78539544, + "num_input_tokens_seen": 96186650, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.17602539, + "step": 4455, + "time_per_iteration": 2.8338799476623535 + }, + { + "auxiliary_loss_clip": 0.01142063, + "auxiliary_loss_mlp": 0.01041841, + "balance_loss_clip": 1.04818058, + "balance_loss_mlp": 1.02633238, + "epoch": 0.5358023206877893, + "flos": 20884676722560.0, + "grad_norm": 1.5994442744476285, + "language_loss": 0.81687367, + "learning_rate": 1.8653398564712594e-06, + "loss": 0.83871269, + "num_input_tokens_seen": 96205595, + "router_z_loss_clip": 0.93945312, + "router_z_loss_mlp": 0.15527344, + "step": 4456, + "time_per_iteration": 2.735661268234253 + }, + { + "auxiliary_loss_clip": 0.01139716, + "auxiliary_loss_mlp": 0.01043989, + "balance_loss_clip": 1.04753113, + "balance_loss_mlp": 1.02787149, + "epoch": 0.5359225635784284, + "flos": 22418996123520.0, + "grad_norm": 1.5365574897811236, + "language_loss": 0.8207441, + "learning_rate": 1.8645626627342704e-06, + "loss": 0.84258115, + "num_input_tokens_seen": 96226360, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.16137695, + "step": 4457, + "time_per_iteration": 2.7701611518859863 + }, + { + "auxiliary_loss_clip": 0.0114304, + "auxiliary_loss_mlp": 0.01051473, + "balance_loss_clip": 1.04750955, + "balance_loss_mlp": 1.0345695, + "epoch": 0.5360428064690675, + "flos": 24097784025600.0, + "grad_norm": 2.128144653292693, + "language_loss": 0.80890769, + "learning_rate": 1.8637854895430172e-06, + "loss": 0.83085287, + "num_input_tokens_seen": 96245625, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.16918945, + "step": 4458, + "time_per_iteration": 2.7457761764526367 + }, + { + "auxiliary_loss_clip": 0.01144634, + "auxiliary_loss_mlp": 0.01042858, + "balance_loss_clip": 1.04716539, + "balance_loss_mlp": 1.0237484, + "epoch": 0.5361630493597066, + "flos": 21434505183360.0, + "grad_norm": 1.9810812489179108, + "language_loss": 0.69803739, + "learning_rate": 1.8630083370153978e-06, + "loss": 0.71991241, + "num_input_tokens_seen": 96265265, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.19128418, + "step": 4459, + "time_per_iteration": 2.7913455963134766 + }, + { + "auxiliary_loss_clip": 0.01043369, + "auxiliary_loss_mlp": 0.01013167, + "balance_loss_clip": 1.01299167, + "balance_loss_mlp": 1.01116455, + "epoch": 0.5362832922503457, + "flos": 68888696520960.0, + "grad_norm": 0.793562685761555, + "language_loss": 0.55407274, + "learning_rate": 1.8622312052693041e-06, + "loss": 0.57463807, + "num_input_tokens_seen": 96326445, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.02001953, + "step": 4460, + "time_per_iteration": 4.272305965423584 + }, + { + "auxiliary_loss_clip": 0.01136549, + "auxiliary_loss_mlp": 0.01039578, + "balance_loss_clip": 1.04315996, + "balance_loss_mlp": 1.02245998, + "epoch": 0.5364035351409848, + "flos": 9793702563840.0, + "grad_norm": 2.3560842592035063, + "language_loss": 0.71811795, + "learning_rate": 1.8614540944226267e-06, + "loss": 0.73987925, + "num_input_tokens_seen": 96343115, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.17126465, + "step": 4461, + "time_per_iteration": 2.742640495300293 + }, + { + "auxiliary_loss_clip": 0.01140429, + "auxiliary_loss_mlp": 0.0104256, + "balance_loss_clip": 1.04936481, + "balance_loss_mlp": 1.02540612, + "epoch": 0.5365237780316239, + "flos": 23290080848640.0, + "grad_norm": 1.893780153214865, + "language_loss": 0.68194628, + "learning_rate": 1.8606770045932537e-06, + "loss": 0.70377618, + "num_input_tokens_seen": 96362230, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.17144775, + "step": 4462, + "time_per_iteration": 2.9349122047424316 + }, + { + "auxiliary_loss_clip": 0.01145726, + "auxiliary_loss_mlp": 0.01039212, + "balance_loss_clip": 1.04990554, + "balance_loss_mlp": 1.02302361, + "epoch": 0.5366440209222629, + "flos": 26578133879040.0, + "grad_norm": 1.8426083879714774, + "language_loss": 0.81559551, + "learning_rate": 1.859899935899068e-06, + "loss": 0.8374449, + "num_input_tokens_seen": 96382085, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.16186523, + "step": 4463, + "time_per_iteration": 3.6486401557922363 + }, + { + "auxiliary_loss_clip": 0.01138888, + "auxiliary_loss_mlp": 0.01040752, + "balance_loss_clip": 1.04412675, + "balance_loss_mlp": 1.02352595, + "epoch": 0.5367642638129021, + "flos": 19608052469760.0, + "grad_norm": 1.4540403030909872, + "language_loss": 0.78827131, + "learning_rate": 1.8591228884579506e-06, + "loss": 0.81006765, + "num_input_tokens_seen": 96400580, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.17211914, + "step": 4464, + "time_per_iteration": 2.772902250289917 + }, + { + "auxiliary_loss_clip": 0.0114461, + "auxiliary_loss_mlp": 0.01044112, + "balance_loss_clip": 1.04775882, + "balance_loss_mlp": 1.02596807, + "epoch": 0.5368845067035412, + "flos": 23915214172800.0, + "grad_norm": 3.215263227083211, + "language_loss": 0.82019997, + "learning_rate": 1.8583458623877795e-06, + "loss": 0.84208721, + "num_input_tokens_seen": 96419680, + "router_z_loss_clip": 0.96728516, + "router_z_loss_mlp": 0.18151855, + "step": 4465, + "time_per_iteration": 2.7380802631378174 + }, + { + "auxiliary_loss_clip": 0.01144207, + "auxiliary_loss_mlp": 0.01037573, + "balance_loss_clip": 1.04793227, + "balance_loss_mlp": 1.02009737, + "epoch": 0.5370047495941802, + "flos": 16873131951360.0, + "grad_norm": 1.9641559226964864, + "language_loss": 0.74026775, + "learning_rate": 1.8575688578064281e-06, + "loss": 0.76208556, + "num_input_tokens_seen": 96437805, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.17456055, + "step": 4466, + "time_per_iteration": 2.7440946102142334 + }, + { + "auxiliary_loss_clip": 0.01141304, + "auxiliary_loss_mlp": 0.01041247, + "balance_loss_clip": 1.04702246, + "balance_loss_mlp": 1.02379489, + "epoch": 0.5371249924848194, + "flos": 20740926493440.0, + "grad_norm": 2.0057400248313324, + "language_loss": 0.76798987, + "learning_rate": 1.8567918748317674e-06, + "loss": 0.78981543, + "num_input_tokens_seen": 96457155, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.17468262, + "step": 4467, + "time_per_iteration": 3.563091278076172 + }, + { + "auxiliary_loss_clip": 0.01142647, + "auxiliary_loss_mlp": 0.01036651, + "balance_loss_clip": 1.04629982, + "balance_loss_mlp": 1.01819718, + "epoch": 0.5372452353754584, + "flos": 17968120104960.0, + "grad_norm": 2.033207350454452, + "language_loss": 0.83072031, + "learning_rate": 1.8560149135816659e-06, + "loss": 0.85251325, + "num_input_tokens_seen": 96473990, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.18457031, + "step": 4468, + "time_per_iteration": 2.691394805908203 + }, + { + "auxiliary_loss_clip": 0.01138985, + "auxiliary_loss_mlp": 0.01039009, + "balance_loss_clip": 1.04581523, + "balance_loss_mlp": 1.02226043, + "epoch": 0.5373654782660975, + "flos": 15377021642880.0, + "grad_norm": 2.1569863713935558, + "language_loss": 0.83767968, + "learning_rate": 1.8552379741739873e-06, + "loss": 0.85945964, + "num_input_tokens_seen": 96491335, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.1673584, + "step": 4469, + "time_per_iteration": 3.536656141281128 + }, + { + "auxiliary_loss_clip": 0.01042091, + "auxiliary_loss_mlp": 0.01005635, + "balance_loss_clip": 1.01205802, + "balance_loss_mlp": 1.00378752, + "epoch": 0.5374857211567367, + "flos": 69000091574400.0, + "grad_norm": 0.89284343563955, + "language_loss": 0.55619335, + "learning_rate": 1.8544610567265935e-06, + "loss": 0.57667065, + "num_input_tokens_seen": 96545275, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01843262, + "step": 4470, + "time_per_iteration": 3.2009122371673584 + }, + { + "auxiliary_loss_clip": 0.01142402, + "auxiliary_loss_mlp": 0.01041707, + "balance_loss_clip": 1.04804921, + "balance_loss_mlp": 1.02567291, + "epoch": 0.5376059640473757, + "flos": 15085355207040.0, + "grad_norm": 1.8661548972201747, + "language_loss": 0.83306909, + "learning_rate": 1.853684161357341e-06, + "loss": 0.85491014, + "num_input_tokens_seen": 96562935, + "router_z_loss_clip": 0.94384766, + "router_z_loss_mlp": 0.16040039, + "step": 4471, + "time_per_iteration": 2.7151966094970703 + }, + { + "auxiliary_loss_clip": 0.01138423, + "auxiliary_loss_mlp": 0.01042263, + "balance_loss_clip": 1.04471254, + "balance_loss_mlp": 1.0254426, + "epoch": 0.5377262069380148, + "flos": 19792597570560.0, + "grad_norm": 3.564064419410491, + "language_loss": 0.76706487, + "learning_rate": 1.852907288184085e-06, + "loss": 0.78887177, + "num_input_tokens_seen": 96581820, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.16821289, + "step": 4472, + "time_per_iteration": 2.7342188358306885 + }, + { + "auxiliary_loss_clip": 0.01146869, + "auxiliary_loss_mlp": 0.01043347, + "balance_loss_clip": 1.04681611, + "balance_loss_mlp": 1.02473855, + "epoch": 0.5378464498286539, + "flos": 30003077640960.0, + "grad_norm": 1.7136599615313697, + "language_loss": 0.70245534, + "learning_rate": 1.8521304373246762e-06, + "loss": 0.72435749, + "num_input_tokens_seen": 96602865, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.18603516, + "step": 4473, + "time_per_iteration": 2.7686333656311035 + }, + { + "auxiliary_loss_clip": 0.01146109, + "auxiliary_loss_mlp": 0.01044339, + "balance_loss_clip": 1.04625034, + "balance_loss_mlp": 1.02538419, + "epoch": 0.537966692719293, + "flos": 21251217058560.0, + "grad_norm": 2.749533192456487, + "language_loss": 0.88515347, + "learning_rate": 1.8513536088969626e-06, + "loss": 0.90705788, + "num_input_tokens_seen": 96620530, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.1895752, + "step": 4474, + "time_per_iteration": 2.752720832824707 + }, + { + "auxiliary_loss_clip": 0.01147654, + "auxiliary_loss_mlp": 0.01043729, + "balance_loss_clip": 1.04899406, + "balance_loss_mlp": 1.02533472, + "epoch": 0.538086935609932, + "flos": 21543170803200.0, + "grad_norm": 1.8232242417987508, + "language_loss": 0.80413461, + "learning_rate": 1.8505768030187884e-06, + "loss": 0.82604837, + "num_input_tokens_seen": 96640660, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.18395996, + "step": 4475, + "time_per_iteration": 2.7707061767578125 + }, + { + "auxiliary_loss_clip": 0.01140713, + "auxiliary_loss_mlp": 0.0104184, + "balance_loss_clip": 1.04834366, + "balance_loss_mlp": 1.02584255, + "epoch": 0.5382071785005712, + "flos": 22747219626240.0, + "grad_norm": 1.4933451290057813, + "language_loss": 0.79723537, + "learning_rate": 1.849800019807995e-06, + "loss": 0.81906086, + "num_input_tokens_seen": 96661885, + "router_z_loss_clip": 0.92382812, + "router_z_loss_mlp": 0.15991211, + "step": 4476, + "time_per_iteration": 2.872626543045044 + }, + { + "auxiliary_loss_clip": 0.01146359, + "auxiliary_loss_mlp": 0.01038436, + "balance_loss_clip": 1.04921699, + "balance_loss_mlp": 1.02085304, + "epoch": 0.5383274213912103, + "flos": 24934574240640.0, + "grad_norm": 2.0451339570528573, + "language_loss": 0.70938474, + "learning_rate": 1.8490232593824186e-06, + "loss": 0.7312327, + "num_input_tokens_seen": 96678340, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.17590332, + "step": 4477, + "time_per_iteration": 2.755483627319336 + }, + { + "auxiliary_loss_clip": 0.01136444, + "auxiliary_loss_mlp": 0.01035415, + "balance_loss_clip": 1.04434538, + "balance_loss_mlp": 1.01922679, + "epoch": 0.5384476642818493, + "flos": 22310186849280.0, + "grad_norm": 1.5147390490204669, + "language_loss": 0.84720844, + "learning_rate": 1.8482465218598935e-06, + "loss": 0.86892706, + "num_input_tokens_seen": 96698285, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.16186523, + "step": 4478, + "time_per_iteration": 2.745237112045288 + }, + { + "auxiliary_loss_clip": 0.01146832, + "auxiliary_loss_mlp": 0.01048687, + "balance_loss_clip": 1.04906154, + "balance_loss_mlp": 1.03072238, + "epoch": 0.5385679071724885, + "flos": 22711021695360.0, + "grad_norm": 1.8469869926940143, + "language_loss": 0.83334172, + "learning_rate": 1.8474698073582508e-06, + "loss": 0.85529691, + "num_input_tokens_seen": 96719655, + "router_z_loss_clip": 0.97558594, + "router_z_loss_mlp": 0.1796875, + "step": 4479, + "time_per_iteration": 2.789491653442383 + }, + { + "auxiliary_loss_clip": 0.01145473, + "auxiliary_loss_mlp": 0.01055194, + "balance_loss_clip": 1.04931974, + "balance_loss_mlp": 1.03697896, + "epoch": 0.5386881500631275, + "flos": 15953746412160.0, + "grad_norm": 3.7954594012196767, + "language_loss": 0.87077641, + "learning_rate": 1.8466931159953166e-06, + "loss": 0.89278305, + "num_input_tokens_seen": 96736290, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.18225098, + "step": 4480, + "time_per_iteration": 2.7632343769073486 + }, + { + "auxiliary_loss_clip": 0.01144804, + "auxiliary_loss_mlp": 0.01044009, + "balance_loss_clip": 1.04924774, + "balance_loss_mlp": 1.02613926, + "epoch": 0.5388083929537666, + "flos": 24060041809920.0, + "grad_norm": 1.753386028657013, + "language_loss": 0.84219068, + "learning_rate": 1.8459164478889158e-06, + "loss": 0.86407876, + "num_input_tokens_seen": 96757685, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.17871094, + "step": 4481, + "time_per_iteration": 2.8457393646240234 + }, + { + "auxiliary_loss_clip": 0.01140095, + "auxiliary_loss_mlp": 0.01039572, + "balance_loss_clip": 1.04671216, + "balance_loss_mlp": 1.02308559, + "epoch": 0.5389286358444056, + "flos": 22236893147520.0, + "grad_norm": 1.6711752724542042, + "language_loss": 0.75681901, + "learning_rate": 1.8451398031568663e-06, + "loss": 0.77861565, + "num_input_tokens_seen": 96777310, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.16479492, + "step": 4482, + "time_per_iteration": 2.7316598892211914 + }, + { + "auxiliary_loss_clip": 0.01144636, + "auxiliary_loss_mlp": 0.01050635, + "balance_loss_clip": 1.04892206, + "balance_loss_mlp": 1.03231287, + "epoch": 0.5390488787350448, + "flos": 24281718595200.0, + "grad_norm": 1.590081245351071, + "language_loss": 0.74790108, + "learning_rate": 1.844363181916986e-06, + "loss": 0.76985377, + "num_input_tokens_seen": 96798035, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.18334961, + "step": 4483, + "time_per_iteration": 2.7848494052886963 + }, + { + "auxiliary_loss_clip": 0.01144999, + "auxiliary_loss_mlp": 0.01046079, + "balance_loss_clip": 1.04690814, + "balance_loss_mlp": 1.02722049, + "epoch": 0.5391691216256839, + "flos": 16581393688320.0, + "grad_norm": 3.1391570589490083, + "language_loss": 0.83133924, + "learning_rate": 1.8435865842870868e-06, + "loss": 0.85325003, + "num_input_tokens_seen": 96815975, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.18859863, + "step": 4484, + "time_per_iteration": 2.6883466243743896 + }, + { + "auxiliary_loss_clip": 0.0113741, + "auxiliary_loss_mlp": 0.01047925, + "balance_loss_clip": 1.0447346, + "balance_loss_mlp": 1.03157008, + "epoch": 0.5392893645163229, + "flos": 23330049707520.0, + "grad_norm": 1.8231649862826451, + "language_loss": 0.71803355, + "learning_rate": 1.8428100103849787e-06, + "loss": 0.73988688, + "num_input_tokens_seen": 96835770, + "router_z_loss_clip": 0.92675781, + "router_z_loss_mlp": 0.16357422, + "step": 4485, + "time_per_iteration": 2.7411842346191406 + }, + { + "auxiliary_loss_clip": 0.01142578, + "auxiliary_loss_mlp": 0.01040264, + "balance_loss_clip": 1.04858685, + "balance_loss_mlp": 1.02320504, + "epoch": 0.5394096074069621, + "flos": 15669801400320.0, + "grad_norm": 2.4721721615332033, + "language_loss": 0.73754811, + "learning_rate": 1.842033460328467e-06, + "loss": 0.75937653, + "num_input_tokens_seen": 96854490, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.17041016, + "step": 4486, + "time_per_iteration": 2.699174165725708 + }, + { + "auxiliary_loss_clip": 0.01139985, + "auxiliary_loss_mlp": 0.01042763, + "balance_loss_clip": 1.04739439, + "balance_loss_mlp": 1.02634811, + "epoch": 0.5395298502976011, + "flos": 22893447893760.0, + "grad_norm": 1.87989380585213, + "language_loss": 0.75161099, + "learning_rate": 1.8412569342353541e-06, + "loss": 0.77343851, + "num_input_tokens_seen": 96874645, + "router_z_loss_clip": 0.92675781, + "router_z_loss_mlp": 0.16418457, + "step": 4487, + "time_per_iteration": 2.7648262977600098 + }, + { + "auxiliary_loss_clip": 0.01150049, + "auxiliary_loss_mlp": 0.01054895, + "balance_loss_clip": 1.04882288, + "balance_loss_mlp": 1.03625083, + "epoch": 0.5396500931882402, + "flos": 23842135952640.0, + "grad_norm": 2.2226658471568945, + "language_loss": 0.84843349, + "learning_rate": 1.840480432223438e-06, + "loss": 0.87048292, + "num_input_tokens_seen": 96893650, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.18652344, + "step": 4488, + "time_per_iteration": 2.768881320953369 + }, + { + "auxiliary_loss_clip": 0.01138858, + "auxiliary_loss_mlp": 0.01043727, + "balance_loss_clip": 1.04485178, + "balance_loss_mlp": 1.02747893, + "epoch": 0.5397703360788794, + "flos": 26322988596480.0, + "grad_norm": 2.120201721917014, + "language_loss": 0.78169072, + "learning_rate": 1.8397039544105131e-06, + "loss": 0.80351663, + "num_input_tokens_seen": 96912735, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.16247559, + "step": 4489, + "time_per_iteration": 3.610238790512085 + }, + { + "auxiliary_loss_clip": 0.01143191, + "auxiliary_loss_mlp": 0.01040745, + "balance_loss_clip": 1.04581451, + "balance_loss_mlp": 1.022506, + "epoch": 0.5398905789695184, + "flos": 21214588164480.0, + "grad_norm": 1.9984315264283974, + "language_loss": 0.69300067, + "learning_rate": 1.8389275009143711e-06, + "loss": 0.71483999, + "num_input_tokens_seen": 96932475, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.18225098, + "step": 4490, + "time_per_iteration": 2.7389087677001953 + }, + { + "auxiliary_loss_clip": 0.01136285, + "auxiliary_loss_mlp": 0.01038385, + "balance_loss_clip": 1.0439024, + "balance_loss_mlp": 1.02202976, + "epoch": 0.5400108218601575, + "flos": 25080335631360.0, + "grad_norm": 1.8597713566841934, + "language_loss": 0.73714304, + "learning_rate": 1.8381510718527988e-06, + "loss": 0.75888973, + "num_input_tokens_seen": 96952085, + "router_z_loss_clip": 0.92333984, + "router_z_loss_mlp": 0.16357422, + "step": 4491, + "time_per_iteration": 2.7639336585998535 + }, + { + "auxiliary_loss_clip": 0.01141965, + "auxiliary_loss_mlp": 0.0104832, + "balance_loss_clip": 1.04617381, + "balance_loss_mlp": 1.02989054, + "epoch": 0.5401310647507966, + "flos": 26357498588160.0, + "grad_norm": 1.8800027261575243, + "language_loss": 0.63759351, + "learning_rate": 1.8373746673435812e-06, + "loss": 0.65949637, + "num_input_tokens_seen": 96973110, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.18432617, + "step": 4492, + "time_per_iteration": 3.6338510513305664 + }, + { + "auxiliary_loss_clip": 0.01143972, + "auxiliary_loss_mlp": 0.01045701, + "balance_loss_clip": 1.04746819, + "balance_loss_mlp": 1.02777207, + "epoch": 0.5402513076414357, + "flos": 27855332749440.0, + "grad_norm": 1.7226738572072036, + "language_loss": 0.79145783, + "learning_rate": 1.8365982875044964e-06, + "loss": 0.81335455, + "num_input_tokens_seen": 96993420, + "router_z_loss_clip": 0.96484375, + "router_z_loss_mlp": 0.17932129, + "step": 4493, + "time_per_iteration": 2.8033194541931152 + }, + { + "auxiliary_loss_clip": 0.01142219, + "auxiliary_loss_mlp": 0.01043562, + "balance_loss_clip": 1.04496169, + "balance_loss_mlp": 1.02470279, + "epoch": 0.5403715505320748, + "flos": 22893771116160.0, + "grad_norm": 2.2635099873709654, + "language_loss": 0.7592473, + "learning_rate": 1.8358219324533217e-06, + "loss": 0.7811051, + "num_input_tokens_seen": 97013685, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.1887207, + "step": 4494, + "time_per_iteration": 2.7514095306396484 + }, + { + "auxiliary_loss_clip": 0.01132509, + "auxiliary_loss_mlp": 0.01036639, + "balance_loss_clip": 1.04406381, + "balance_loss_mlp": 1.02099848, + "epoch": 0.5404917934227139, + "flos": 30224143895040.0, + "grad_norm": 1.6563679949168268, + "language_loss": 0.70517612, + "learning_rate": 1.8350456023078292e-06, + "loss": 0.72686768, + "num_input_tokens_seen": 97036060, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.15625, + "step": 4495, + "time_per_iteration": 3.678438901901245 + }, + { + "auxiliary_loss_clip": 0.01145591, + "auxiliary_loss_mlp": 0.01047653, + "balance_loss_clip": 1.04708767, + "balance_loss_mlp": 1.02856791, + "epoch": 0.540612036313353, + "flos": 19938502615680.0, + "grad_norm": 2.1639305316926816, + "language_loss": 0.78174353, + "learning_rate": 1.8342692971857874e-06, + "loss": 0.80367595, + "num_input_tokens_seen": 97055260, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.1907959, + "step": 4496, + "time_per_iteration": 2.742377758026123 + }, + { + "auxiliary_loss_clip": 0.01139659, + "auxiliary_loss_mlp": 0.01037322, + "balance_loss_clip": 1.04548979, + "balance_loss_mlp": 1.02143168, + "epoch": 0.540732279203992, + "flos": 24279599692800.0, + "grad_norm": 2.2948563028578364, + "language_loss": 0.71352726, + "learning_rate": 1.833493017204962e-06, + "loss": 0.73529708, + "num_input_tokens_seen": 97075365, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.15893555, + "step": 4497, + "time_per_iteration": 2.7286183834075928 + }, + { + "auxiliary_loss_clip": 0.01140651, + "auxiliary_loss_mlp": 0.01047061, + "balance_loss_clip": 1.04546261, + "balance_loss_mlp": 1.02931106, + "epoch": 0.5408525220946312, + "flos": 20193216935040.0, + "grad_norm": 2.302079592730057, + "language_loss": 0.78123587, + "learning_rate": 1.8327167624831134e-06, + "loss": 0.80311298, + "num_input_tokens_seen": 97093095, + "router_z_loss_clip": 0.95263672, + "router_z_loss_mlp": 0.1776123, + "step": 4498, + "time_per_iteration": 3.5170671939849854 + }, + { + "auxiliary_loss_clip": 0.01141006, + "auxiliary_loss_mlp": 0.0103827, + "balance_loss_clip": 1.04766381, + "balance_loss_mlp": 1.02041245, + "epoch": 0.5409727649852702, + "flos": 24134448833280.0, + "grad_norm": 1.6446611093499457, + "language_loss": 0.70782983, + "learning_rate": 1.831940533137999e-06, + "loss": 0.72962266, + "num_input_tokens_seen": 97112000, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.1784668, + "step": 4499, + "time_per_iteration": 2.7527074813842773 + }, + { + "auxiliary_loss_clip": 0.01137705, + "auxiliary_loss_mlp": 0.01044598, + "balance_loss_clip": 1.04547465, + "balance_loss_mlp": 1.02824283, + "epoch": 0.5410930078759093, + "flos": 23912700220800.0, + "grad_norm": 1.6830551091498915, + "language_loss": 0.72314972, + "learning_rate": 1.8311643292873718e-06, + "loss": 0.74497277, + "num_input_tokens_seen": 97130820, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.16345215, + "step": 4500, + "time_per_iteration": 2.7624900341033936 + }, + { + "auxiliary_loss_clip": 0.0113549, + "auxiliary_loss_mlp": 0.01043868, + "balance_loss_clip": 1.04446793, + "balance_loss_mlp": 1.0285852, + "epoch": 0.5412132507665485, + "flos": 21105132445440.0, + "grad_norm": 1.7797161192887547, + "language_loss": 0.87902033, + "learning_rate": 1.8303881510489818e-06, + "loss": 0.90081394, + "num_input_tokens_seen": 97149210, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.15270996, + "step": 4501, + "time_per_iteration": 2.762895107269287 + }, + { + "auxiliary_loss_clip": 0.01142087, + "auxiliary_loss_mlp": 0.01042215, + "balance_loss_clip": 1.04462874, + "balance_loss_mlp": 1.02522779, + "epoch": 0.5413334936571875, + "flos": 30227340205440.0, + "grad_norm": 1.8088573765467382, + "language_loss": 0.69020879, + "learning_rate": 1.829611998540574e-06, + "loss": 0.71205175, + "num_input_tokens_seen": 97170415, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.16992188, + "step": 4502, + "time_per_iteration": 2.8017725944519043 + }, + { + "auxiliary_loss_clip": 0.01141052, + "auxiliary_loss_mlp": 0.01037624, + "balance_loss_clip": 1.04326427, + "balance_loss_mlp": 1.02024364, + "epoch": 0.5414537365478266, + "flos": 24279635606400.0, + "grad_norm": 1.6984172553176557, + "language_loss": 0.7984134, + "learning_rate": 1.8288358718798914e-06, + "loss": 0.8202002, + "num_input_tokens_seen": 97189605, + "router_z_loss_clip": 0.97949219, + "router_z_loss_mlp": 0.17370605, + "step": 4503, + "time_per_iteration": 2.7729063034057617 + }, + { + "auxiliary_loss_clip": 0.01140619, + "auxiliary_loss_mlp": 0.01046886, + "balance_loss_clip": 1.0469923, + "balance_loss_mlp": 1.03025603, + "epoch": 0.5415739794384657, + "flos": 16654543735680.0, + "grad_norm": 1.7437339078770717, + "language_loss": 0.72493815, + "learning_rate": 1.8280597711846703e-06, + "loss": 0.74681324, + "num_input_tokens_seen": 97207845, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.16625977, + "step": 4504, + "time_per_iteration": 2.7133967876434326 + }, + { + "auxiliary_loss_clip": 0.01140346, + "auxiliary_loss_mlp": 0.01040418, + "balance_loss_clip": 1.04611218, + "balance_loss_mlp": 1.02431262, + "epoch": 0.5416942223291048, + "flos": 23185724860800.0, + "grad_norm": 1.8023238872997112, + "language_loss": 0.83366489, + "learning_rate": 1.8272836965726455e-06, + "loss": 0.85547256, + "num_input_tokens_seen": 97226780, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.16101074, + "step": 4505, + "time_per_iteration": 2.752340316772461 + }, + { + "auxiliary_loss_clip": 0.01140566, + "auxiliary_loss_mlp": 0.01042827, + "balance_loss_clip": 1.04610789, + "balance_loss_mlp": 1.02554131, + "epoch": 0.5418144652197439, + "flos": 20303247271680.0, + "grad_norm": 1.76666554915054, + "language_loss": 0.78431857, + "learning_rate": 1.8265076481615461e-06, + "loss": 0.80615252, + "num_input_tokens_seen": 97246695, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.17285156, + "step": 4506, + "time_per_iteration": 2.7941694259643555 + }, + { + "auxiliary_loss_clip": 0.01146857, + "auxiliary_loss_mlp": 0.01042497, + "balance_loss_clip": 1.04964066, + "balance_loss_mlp": 1.02512813, + "epoch": 0.541934708110383, + "flos": 12458633431680.0, + "grad_norm": 1.9448288399651286, + "language_loss": 0.87084103, + "learning_rate": 1.8257316260690987e-06, + "loss": 0.89273459, + "num_input_tokens_seen": 97264480, + "router_z_loss_clip": 0.97119141, + "router_z_loss_mlp": 0.17370605, + "step": 4507, + "time_per_iteration": 2.714604139328003 + }, + { + "auxiliary_loss_clip": 0.01135126, + "auxiliary_loss_mlp": 0.01039917, + "balance_loss_clip": 1.04375684, + "balance_loss_mlp": 1.02379966, + "epoch": 0.5420549510010221, + "flos": 21253802837760.0, + "grad_norm": 1.409032656188517, + "language_loss": 0.7608161, + "learning_rate": 1.8249556304130254e-06, + "loss": 0.78256655, + "num_input_tokens_seen": 97285760, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.16113281, + "step": 4508, + "time_per_iteration": 2.730403184890747 + }, + { + "auxiliary_loss_clip": 0.01140716, + "auxiliary_loss_mlp": 0.01042535, + "balance_loss_clip": 1.04648292, + "balance_loss_mlp": 1.02565479, + "epoch": 0.5421751938916611, + "flos": 29490524519040.0, + "grad_norm": 2.0873502769747665, + "language_loss": 0.68820429, + "learning_rate": 1.824179661311044e-06, + "loss": 0.71003675, + "num_input_tokens_seen": 97304510, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.16882324, + "step": 4509, + "time_per_iteration": 2.8071136474609375 + }, + { + "auxiliary_loss_clip": 0.01139445, + "auxiliary_loss_mlp": 0.01039388, + "balance_loss_clip": 1.04538333, + "balance_loss_mlp": 1.0219357, + "epoch": 0.5422954367823003, + "flos": 18734238311040.0, + "grad_norm": 2.620400289920031, + "language_loss": 0.80248833, + "learning_rate": 1.823403718880868e-06, + "loss": 0.82427663, + "num_input_tokens_seen": 97323270, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.17443848, + "step": 4510, + "time_per_iteration": 2.696157455444336 + }, + { + "auxiliary_loss_clip": 0.01143376, + "auxiliary_loss_mlp": 0.01040845, + "balance_loss_clip": 1.04790509, + "balance_loss_mlp": 1.02203417, + "epoch": 0.5424156796729394, + "flos": 39969006940800.0, + "grad_norm": 1.692380363379129, + "language_loss": 0.66717601, + "learning_rate": 1.822627803240207e-06, + "loss": 0.68901825, + "num_input_tokens_seen": 97345600, + "router_z_loss_clip": 0.95507812, + "router_z_loss_mlp": 0.18823242, + "step": 4511, + "time_per_iteration": 2.9448492527008057 + }, + { + "auxiliary_loss_clip": 0.01141377, + "auxiliary_loss_mlp": 0.01037741, + "balance_loss_clip": 1.04660618, + "balance_loss_mlp": 1.0210278, + "epoch": 0.5425359225635784, + "flos": 11546538353280.0, + "grad_norm": 2.16405700928169, + "language_loss": 0.85436487, + "learning_rate": 1.8218519145067675e-06, + "loss": 0.87615609, + "num_input_tokens_seen": 97361220, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.16705322, + "step": 4512, + "time_per_iteration": 2.8592939376831055 + }, + { + "auxiliary_loss_clip": 0.01139638, + "auxiliary_loss_mlp": 0.01041872, + "balance_loss_clip": 1.04617262, + "balance_loss_mlp": 1.02364528, + "epoch": 0.5426561654542175, + "flos": 20229702174720.0, + "grad_norm": 1.7491746634879732, + "language_loss": 0.8956486, + "learning_rate": 1.8210760527982508e-06, + "loss": 0.91746372, + "num_input_tokens_seen": 97381505, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.18225098, + "step": 4513, + "time_per_iteration": 2.7755210399627686 + }, + { + "auxiliary_loss_clip": 0.01138486, + "auxiliary_loss_mlp": 0.01037536, + "balance_loss_clip": 1.04685116, + "balance_loss_mlp": 1.02125263, + "epoch": 0.5427764083448566, + "flos": 21871681614720.0, + "grad_norm": 1.7502405500979832, + "language_loss": 0.7539705, + "learning_rate": 1.8203002182323552e-06, + "loss": 0.77573073, + "num_input_tokens_seen": 97399060, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.1628418, + "step": 4514, + "time_per_iteration": 2.877472162246704 + }, + { + "auxiliary_loss_clip": 0.01138194, + "auxiliary_loss_mlp": 0.0104906, + "balance_loss_clip": 1.04353774, + "balance_loss_mlp": 1.03117824, + "epoch": 0.5428966512354957, + "flos": 19640946349440.0, + "grad_norm": 1.8846825913978995, + "language_loss": 0.75391823, + "learning_rate": 1.819524410926773e-06, + "loss": 0.77579081, + "num_input_tokens_seen": 97416740, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.17871094, + "step": 4515, + "time_per_iteration": 2.815617561340332 + }, + { + "auxiliary_loss_clip": 0.01140649, + "auxiliary_loss_mlp": 0.01045289, + "balance_loss_clip": 1.04691696, + "balance_loss_mlp": 1.02644241, + "epoch": 0.5430168941261347, + "flos": 22382187661440.0, + "grad_norm": 1.6234464858488122, + "language_loss": 0.76977378, + "learning_rate": 1.8187486309991944e-06, + "loss": 0.79163319, + "num_input_tokens_seen": 97437620, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.18859863, + "step": 4516, + "time_per_iteration": 2.841890811920166 + }, + { + "auxiliary_loss_clip": 0.01141415, + "auxiliary_loss_mlp": 0.01038272, + "balance_loss_clip": 1.04737484, + "balance_loss_mlp": 1.0222261, + "epoch": 0.5431371370167739, + "flos": 18764187275520.0, + "grad_norm": 1.8508448654100689, + "language_loss": 0.77623487, + "learning_rate": 1.817972878567304e-06, + "loss": 0.79803175, + "num_input_tokens_seen": 97456275, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.16052246, + "step": 4517, + "time_per_iteration": 3.0473599433898926 + }, + { + "auxiliary_loss_clip": 0.01135783, + "auxiliary_loss_mlp": 0.01043832, + "balance_loss_clip": 1.04456139, + "balance_loss_mlp": 1.02776265, + "epoch": 0.543257379907413, + "flos": 18806023641600.0, + "grad_norm": 1.7279237451702192, + "language_loss": 0.76720858, + "learning_rate": 1.8171971537487834e-06, + "loss": 0.78900468, + "num_input_tokens_seen": 97474925, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.1607666, + "step": 4518, + "time_per_iteration": 3.7817001342773438 + }, + { + "auxiliary_loss_clip": 0.01136981, + "auxiliary_loss_mlp": 0.01039077, + "balance_loss_clip": 1.04241753, + "balance_loss_mlp": 1.02194643, + "epoch": 0.543377622798052, + "flos": 17493381025920.0, + "grad_norm": 1.8589974894748797, + "language_loss": 0.80959618, + "learning_rate": 1.8164214566613093e-06, + "loss": 0.83135676, + "num_input_tokens_seen": 97493550, + "router_z_loss_clip": 0.9453125, + "router_z_loss_mlp": 0.17138672, + "step": 4519, + "time_per_iteration": 2.8345298767089844 + }, + { + "auxiliary_loss_clip": 0.01137355, + "auxiliary_loss_mlp": 0.01039408, + "balance_loss_clip": 1.04426694, + "balance_loss_mlp": 1.02244484, + "epoch": 0.5434978656886912, + "flos": 18989311766400.0, + "grad_norm": 2.9065619627887176, + "language_loss": 0.66262531, + "learning_rate": 1.8156457874225547e-06, + "loss": 0.68439293, + "num_input_tokens_seen": 97512010, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.16955566, + "step": 4520, + "time_per_iteration": 3.0962789058685303 + }, + { + "auxiliary_loss_clip": 0.01136847, + "auxiliary_loss_mlp": 0.01034991, + "balance_loss_clip": 1.04402709, + "balance_loss_mlp": 1.01710999, + "epoch": 0.5436181085793302, + "flos": 17274936464640.0, + "grad_norm": 1.7222874811193127, + "language_loss": 0.80839455, + "learning_rate": 1.814870146150187e-06, + "loss": 0.83011293, + "num_input_tokens_seen": 97530120, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.17883301, + "step": 4521, + "time_per_iteration": 3.6586732864379883 + }, + { + "auxiliary_loss_clip": 0.01139629, + "auxiliary_loss_mlp": 0.01042493, + "balance_loss_clip": 1.04501343, + "balance_loss_mlp": 1.02386105, + "epoch": 0.5437383514699693, + "flos": 19098587917440.0, + "grad_norm": 2.0658451300589595, + "language_loss": 0.78494728, + "learning_rate": 1.814094532961871e-06, + "loss": 0.80676854, + "num_input_tokens_seen": 97548695, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.18640137, + "step": 4522, + "time_per_iteration": 2.8415563106536865 + }, + { + "auxiliary_loss_clip": 0.01140532, + "auxiliary_loss_mlp": 0.01043007, + "balance_loss_clip": 1.0446384, + "balance_loss_mlp": 1.02462447, + "epoch": 0.5438585943606085, + "flos": 22602715211520.0, + "grad_norm": 1.8273053491407996, + "language_loss": 0.8325057, + "learning_rate": 1.8133189479752666e-06, + "loss": 0.85434103, + "num_input_tokens_seen": 97567625, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.18408203, + "step": 4523, + "time_per_iteration": 2.7686867713928223 + }, + { + "auxiliary_loss_clip": 0.01137516, + "auxiliary_loss_mlp": 0.01046328, + "balance_loss_clip": 1.04435325, + "balance_loss_mlp": 1.03106868, + "epoch": 0.5439788372512475, + "flos": 21798495653760.0, + "grad_norm": 1.8467157097082736, + "language_loss": 0.81763923, + "learning_rate": 1.8125433913080292e-06, + "loss": 0.8394776, + "num_input_tokens_seen": 97585325, + "router_z_loss_clip": 0.93164062, + "router_z_loss_mlp": 0.15258789, + "step": 4524, + "time_per_iteration": 3.625858783721924 + }, + { + "auxiliary_loss_clip": 0.01134719, + "auxiliary_loss_mlp": 0.01034662, + "balance_loss_clip": 1.04399931, + "balance_loss_mlp": 1.0194149, + "epoch": 0.5440990801418866, + "flos": 16399362539520.0, + "grad_norm": 1.8357211063868968, + "language_loss": 0.82638752, + "learning_rate": 1.811767863077811e-06, + "loss": 0.84808135, + "num_input_tokens_seen": 97604275, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.15240479, + "step": 4525, + "time_per_iteration": 2.8443949222564697 + }, + { + "auxiliary_loss_clip": 0.01132861, + "auxiliary_loss_mlp": 0.01043583, + "balance_loss_clip": 1.04315448, + "balance_loss_mlp": 1.02812099, + "epoch": 0.5442193230325257, + "flos": 21615638492160.0, + "grad_norm": 1.6060523716035695, + "language_loss": 0.7868889, + "learning_rate": 1.8109923634022577e-06, + "loss": 0.80865335, + "num_input_tokens_seen": 97624300, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.15466309, + "step": 4526, + "time_per_iteration": 2.812434196472168 + }, + { + "auxiliary_loss_clip": 0.0114077, + "auxiliary_loss_mlp": 0.0104628, + "balance_loss_clip": 1.04384613, + "balance_loss_mlp": 1.0287447, + "epoch": 0.5443395659231648, + "flos": 15481198062720.0, + "grad_norm": 2.732914516164683, + "language_loss": 0.86602801, + "learning_rate": 1.8102168923990128e-06, + "loss": 0.88789856, + "num_input_tokens_seen": 97637845, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.17529297, + "step": 4527, + "time_per_iteration": 3.648050308227539 + }, + { + "auxiliary_loss_clip": 0.01138363, + "auxiliary_loss_mlp": 0.01045664, + "balance_loss_clip": 1.04508042, + "balance_loss_mlp": 1.02893877, + "epoch": 0.5444598088138038, + "flos": 18770436241920.0, + "grad_norm": 2.1965062590586135, + "language_loss": 0.79895103, + "learning_rate": 1.809441450185714e-06, + "loss": 0.8207913, + "num_input_tokens_seen": 97656330, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.16723633, + "step": 4528, + "time_per_iteration": 2.7394206523895264 + }, + { + "auxiliary_loss_clip": 0.01140512, + "auxiliary_loss_mlp": 0.01043603, + "balance_loss_clip": 1.04521894, + "balance_loss_mlp": 1.02792692, + "epoch": 0.544580051704443, + "flos": 21142335957120.0, + "grad_norm": 2.2473577538712153, + "language_loss": 0.73509121, + "learning_rate": 1.8086660368799958e-06, + "loss": 0.75693238, + "num_input_tokens_seen": 97674380, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.15661621, + "step": 4529, + "time_per_iteration": 2.7709262371063232 + }, + { + "auxiliary_loss_clip": 0.01142355, + "auxiliary_loss_mlp": 0.010457, + "balance_loss_clip": 1.04696822, + "balance_loss_mlp": 1.02743697, + "epoch": 0.5447002945950821, + "flos": 32491508054400.0, + "grad_norm": 1.7550833936022776, + "language_loss": 0.7719298, + "learning_rate": 1.807890652599488e-06, + "loss": 0.79381037, + "num_input_tokens_seen": 97698765, + "router_z_loss_clip": 0.95361328, + "router_z_loss_mlp": 0.18273926, + "step": 4530, + "time_per_iteration": 2.852628707885742 + }, + { + "auxiliary_loss_clip": 0.01132581, + "auxiliary_loss_mlp": 0.01046468, + "balance_loss_clip": 1.04352689, + "balance_loss_mlp": 1.03077459, + "epoch": 0.5448205374857211, + "flos": 11798307757440.0, + "grad_norm": 1.9837063910699206, + "language_loss": 0.82799089, + "learning_rate": 1.8071152974618156e-06, + "loss": 0.84978133, + "num_input_tokens_seen": 97716565, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.15704346, + "step": 4531, + "time_per_iteration": 2.7578907012939453 + }, + { + "auxiliary_loss_clip": 0.01133826, + "auxiliary_loss_mlp": 0.01031959, + "balance_loss_clip": 1.04210734, + "balance_loss_mlp": 1.01602101, + "epoch": 0.5449407803763603, + "flos": 24133766474880.0, + "grad_norm": 2.491614686791982, + "language_loss": 0.7845329, + "learning_rate": 1.806339971584599e-06, + "loss": 0.80619079, + "num_input_tokens_seen": 97733225, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.15917969, + "step": 4532, + "time_per_iteration": 2.776837110519409 + }, + { + "auxiliary_loss_clip": 0.01138563, + "auxiliary_loss_mlp": 0.0103981, + "balance_loss_clip": 1.04442775, + "balance_loss_mlp": 1.02339458, + "epoch": 0.5450610232669993, + "flos": 23258551685760.0, + "grad_norm": 1.7161610303945443, + "language_loss": 0.85384494, + "learning_rate": 1.8055646750854546e-06, + "loss": 0.87562865, + "num_input_tokens_seen": 97752735, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.1640625, + "step": 4533, + "time_per_iteration": 2.812966823577881 + }, + { + "auxiliary_loss_clip": 0.01138756, + "auxiliary_loss_mlp": 0.01038602, + "balance_loss_clip": 1.04572654, + "balance_loss_mlp": 1.02153099, + "epoch": 0.5451812661576384, + "flos": 17785083375360.0, + "grad_norm": 2.489170550457156, + "language_loss": 0.81769204, + "learning_rate": 1.8047894080819945e-06, + "loss": 0.83946562, + "num_input_tokens_seen": 97769985, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.1706543, + "step": 4534, + "time_per_iteration": 2.744192361831665 + }, + { + "auxiliary_loss_clip": 0.01043094, + "auxiliary_loss_mlp": 0.01015455, + "balance_loss_clip": 1.01341105, + "balance_loss_mlp": 1.01383364, + "epoch": 0.5453015090482776, + "flos": 71062586513280.0, + "grad_norm": 0.7369243311402859, + "language_loss": 0.63144493, + "learning_rate": 1.8040141706918258e-06, + "loss": 0.65203047, + "num_input_tokens_seen": 97831225, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.01623535, + "step": 4535, + "time_per_iteration": 3.402890920639038 + }, + { + "auxiliary_loss_clip": 0.0113809, + "auxiliary_loss_mlp": 0.01042699, + "balance_loss_clip": 1.04543865, + "balance_loss_mlp": 1.0265938, + "epoch": 0.5454217519389166, + "flos": 25552201622400.0, + "grad_norm": 1.6973581501440391, + "language_loss": 0.7712214, + "learning_rate": 1.8032389630325525e-06, + "loss": 0.79302931, + "num_input_tokens_seen": 97849975, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.16113281, + "step": 4536, + "time_per_iteration": 2.7870869636535645 + }, + { + "auxiliary_loss_clip": 0.01135851, + "auxiliary_loss_mlp": 0.01036175, + "balance_loss_clip": 1.04394329, + "balance_loss_mlp": 1.02051127, + "epoch": 0.5455419948295557, + "flos": 23658345037440.0, + "grad_norm": 1.6987064840475008, + "language_loss": 0.75794518, + "learning_rate": 1.8024637852217707e-06, + "loss": 0.77966547, + "num_input_tokens_seen": 97869700, + "router_z_loss_clip": 0.91894531, + "router_z_loss_mlp": 0.15661621, + "step": 4537, + "time_per_iteration": 2.965208053588867 + }, + { + "auxiliary_loss_clip": 0.01138743, + "auxiliary_loss_mlp": 0.01043486, + "balance_loss_clip": 1.04683638, + "balance_loss_mlp": 1.02798927, + "epoch": 0.5456622377201948, + "flos": 23403989854080.0, + "grad_norm": 1.7554234414264018, + "language_loss": 0.84710944, + "learning_rate": 1.8016886373770766e-06, + "loss": 0.86893177, + "num_input_tokens_seen": 97888215, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.1550293, + "step": 4538, + "time_per_iteration": 2.743114471435547 + }, + { + "auxiliary_loss_clip": 0.01137607, + "auxiliary_loss_mlp": 0.01040293, + "balance_loss_clip": 1.04483163, + "balance_loss_mlp": 1.02464032, + "epoch": 0.5457824806108339, + "flos": 23988040997760.0, + "grad_norm": 1.7128942790255488, + "language_loss": 0.78557873, + "learning_rate": 1.8009135196160579e-06, + "loss": 0.80735779, + "num_input_tokens_seen": 97907090, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.15637207, + "step": 4539, + "time_per_iteration": 2.7721545696258545 + }, + { + "auxiliary_loss_clip": 0.01135465, + "auxiliary_loss_mlp": 0.01042855, + "balance_loss_clip": 1.04494262, + "balance_loss_mlp": 1.02717876, + "epoch": 0.545902723501473, + "flos": 22565870835840.0, + "grad_norm": 1.6258768792642941, + "language_loss": 0.84590709, + "learning_rate": 1.8001384320563e-06, + "loss": 0.86769027, + "num_input_tokens_seen": 97927345, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.15649414, + "step": 4540, + "time_per_iteration": 2.766979217529297 + }, + { + "auxiliary_loss_clip": 0.01042347, + "auxiliary_loss_mlp": 0.01003831, + "balance_loss_clip": 1.01286173, + "balance_loss_mlp": 1.00212657, + "epoch": 0.5460229663921121, + "flos": 55198399685760.0, + "grad_norm": 0.7783112287923023, + "language_loss": 0.577824, + "learning_rate": 1.7993633748153833e-06, + "loss": 0.59828579, + "num_input_tokens_seen": 97981950, + "router_z_loss_clip": 0.29492188, + "router_z_loss_mlp": 0.01708984, + "step": 4541, + "time_per_iteration": 3.159097194671631 + }, + { + "auxiliary_loss_clip": 0.01139241, + "auxiliary_loss_mlp": 0.01050529, + "balance_loss_clip": 1.04521155, + "balance_loss_mlp": 1.03281426, + "epoch": 0.5461432092827512, + "flos": 15413866018560.0, + "grad_norm": 2.0538635014409183, + "language_loss": 0.72637612, + "learning_rate": 1.7985883480108834e-06, + "loss": 0.74827385, + "num_input_tokens_seen": 97999585, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.17712402, + "step": 4542, + "time_per_iteration": 2.783237934112549 + }, + { + "auxiliary_loss_clip": 0.01138228, + "auxiliary_loss_mlp": 0.01038342, + "balance_loss_clip": 1.04462194, + "balance_loss_mlp": 1.02205825, + "epoch": 0.5462634521733902, + "flos": 24024921287040.0, + "grad_norm": 1.5387960384459938, + "language_loss": 0.722224, + "learning_rate": 1.797813351760371e-06, + "loss": 0.74398971, + "num_input_tokens_seen": 98021290, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16296387, + "step": 4543, + "time_per_iteration": 2.8567631244659424 + }, + { + "auxiliary_loss_clip": 0.01140042, + "auxiliary_loss_mlp": 0.01041932, + "balance_loss_clip": 1.04390407, + "balance_loss_mlp": 1.02359748, + "epoch": 0.5463836950640293, + "flos": 22820944291200.0, + "grad_norm": 1.805525383042685, + "language_loss": 0.78306442, + "learning_rate": 1.7970383861814116e-06, + "loss": 0.8048842, + "num_input_tokens_seen": 98041060, + "router_z_loss_clip": 0.96240234, + "router_z_loss_mlp": 0.18347168, + "step": 4544, + "time_per_iteration": 2.78690767288208 + }, + { + "auxiliary_loss_clip": 0.01140455, + "auxiliary_loss_mlp": 0.01041094, + "balance_loss_clip": 1.04635811, + "balance_loss_mlp": 1.02358222, + "epoch": 0.5465039379546685, + "flos": 20448290390400.0, + "grad_norm": 1.8773983918550692, + "language_loss": 0.74207318, + "learning_rate": 1.7962634513915684e-06, + "loss": 0.7638886, + "num_input_tokens_seen": 98058410, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.17504883, + "step": 4545, + "time_per_iteration": 2.751849889755249 + }, + { + "auxiliary_loss_clip": 0.01135018, + "auxiliary_loss_mlp": 0.01039877, + "balance_loss_clip": 1.04304934, + "balance_loss_mlp": 1.023736, + "epoch": 0.5466241808453075, + "flos": 17343310003200.0, + "grad_norm": 1.607035912764564, + "language_loss": 0.79518282, + "learning_rate": 1.7954885475083969e-06, + "loss": 0.81693172, + "num_input_tokens_seen": 98076080, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.16137695, + "step": 4546, + "time_per_iteration": 2.7734298706054688 + }, + { + "auxiliary_loss_clip": 0.01141429, + "auxiliary_loss_mlp": 0.01041844, + "balance_loss_clip": 1.04551828, + "balance_loss_mlp": 1.02397418, + "epoch": 0.5467444237359466, + "flos": 21617039122560.0, + "grad_norm": 2.083789781055006, + "language_loss": 0.7299912, + "learning_rate": 1.7947136746494513e-06, + "loss": 0.7518239, + "num_input_tokens_seen": 98096995, + "router_z_loss_clip": 0.96044922, + "router_z_loss_mlp": 0.17871094, + "step": 4547, + "time_per_iteration": 3.641616106033325 + }, + { + "auxiliary_loss_clip": 0.01137382, + "auxiliary_loss_mlp": 0.0103966, + "balance_loss_clip": 1.04329824, + "balance_loss_mlp": 1.02258897, + "epoch": 0.5468646666265857, + "flos": 24170467196160.0, + "grad_norm": 1.970940410069323, + "language_loss": 0.87687624, + "learning_rate": 1.793938832932277e-06, + "loss": 0.89864665, + "num_input_tokens_seen": 98115105, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.17053223, + "step": 4548, + "time_per_iteration": 2.810883045196533 + }, + { + "auxiliary_loss_clip": 0.01137529, + "auxiliary_loss_mlp": 0.01045199, + "balance_loss_clip": 1.04408264, + "balance_loss_mlp": 1.02814066, + "epoch": 0.5469849095172248, + "flos": 27527001505920.0, + "grad_norm": 1.7338621105483671, + "language_loss": 0.70107126, + "learning_rate": 1.7931640224744185e-06, + "loss": 0.72289854, + "num_input_tokens_seen": 98135655, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.17053223, + "step": 4549, + "time_per_iteration": 2.8030879497528076 + }, + { + "auxiliary_loss_clip": 0.01134223, + "auxiliary_loss_mlp": 0.01037604, + "balance_loss_clip": 1.04173028, + "balance_loss_mlp": 1.02081895, + "epoch": 0.5471051524078638, + "flos": 27964680727680.0, + "grad_norm": 1.5962962721982161, + "language_loss": 0.73697501, + "learning_rate": 1.7923892433934127e-06, + "loss": 0.75869328, + "num_input_tokens_seen": 98156730, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.16784668, + "step": 4550, + "time_per_iteration": 3.55478835105896 + }, + { + "auxiliary_loss_clip": 0.01142255, + "auxiliary_loss_mlp": 0.01047936, + "balance_loss_clip": 1.04719949, + "balance_loss_mlp": 1.02981567, + "epoch": 0.547225395298503, + "flos": 18150510389760.0, + "grad_norm": 1.8904406990370806, + "language_loss": 0.78460646, + "learning_rate": 1.7916144958067939e-06, + "loss": 0.80650836, + "num_input_tokens_seen": 98174590, + "router_z_loss_clip": 0.94970703, + "router_z_loss_mlp": 0.18127441, + "step": 4551, + "time_per_iteration": 2.7430379390716553 + }, + { + "auxiliary_loss_clip": 0.0113824, + "auxiliary_loss_mlp": 0.01047604, + "balance_loss_clip": 1.0446111, + "balance_loss_mlp": 1.03152251, + "epoch": 0.5473456381891421, + "flos": 21361498790400.0, + "grad_norm": 1.8212740093972537, + "language_loss": 0.79092956, + "learning_rate": 1.7908397798320905e-06, + "loss": 0.81278789, + "num_input_tokens_seen": 98194325, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16088867, + "step": 4552, + "time_per_iteration": 2.739323139190674 + }, + { + "auxiliary_loss_clip": 0.01140078, + "auxiliary_loss_mlp": 0.01034582, + "balance_loss_clip": 1.04561126, + "balance_loss_mlp": 1.01754737, + "epoch": 0.5474658810797811, + "flos": 19932145908480.0, + "grad_norm": 1.8682449270421135, + "language_loss": 0.74620378, + "learning_rate": 1.7900650955868265e-06, + "loss": 0.76795036, + "num_input_tokens_seen": 98213970, + "router_z_loss_clip": 0.94335938, + "router_z_loss_mlp": 0.17016602, + "step": 4553, + "time_per_iteration": 3.627305746078491 + }, + { + "auxiliary_loss_clip": 0.01138659, + "auxiliary_loss_mlp": 0.01039181, + "balance_loss_clip": 1.04579675, + "balance_loss_mlp": 1.02297997, + "epoch": 0.5475861239704203, + "flos": 50476217264640.0, + "grad_norm": 1.435062483549656, + "language_loss": 0.76329118, + "learning_rate": 1.7892904431885202e-06, + "loss": 0.78506958, + "num_input_tokens_seen": 98241145, + "router_z_loss_clip": 0.92773438, + "router_z_loss_mlp": 0.1619873, + "step": 4554, + "time_per_iteration": 3.015918254852295 + }, + { + "auxiliary_loss_clip": 0.01131746, + "auxiliary_loss_mlp": 0.01041277, + "balance_loss_clip": 1.04228687, + "balance_loss_mlp": 1.02601814, + "epoch": 0.5477063668610593, + "flos": 20705123612160.0, + "grad_norm": 1.7402030802804085, + "language_loss": 0.75033587, + "learning_rate": 1.788515822754686e-06, + "loss": 0.77206612, + "num_input_tokens_seen": 98261565, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.15264893, + "step": 4555, + "time_per_iteration": 2.765852451324463 + }, + { + "auxiliary_loss_clip": 0.01141297, + "auxiliary_loss_mlp": 0.01039735, + "balance_loss_clip": 1.04658675, + "balance_loss_mlp": 1.0229857, + "epoch": 0.5478266097516984, + "flos": 19609740408960.0, + "grad_norm": 1.96531152788997, + "language_loss": 0.78283513, + "learning_rate": 1.7877412344028335e-06, + "loss": 0.80464542, + "num_input_tokens_seen": 98281370, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.16760254, + "step": 4556, + "time_per_iteration": 3.5895750522613525 + }, + { + "auxiliary_loss_clip": 0.01139866, + "auxiliary_loss_mlp": 0.01044097, + "balance_loss_clip": 1.04643035, + "balance_loss_mlp": 1.02684724, + "epoch": 0.5479468526423376, + "flos": 12896599962240.0, + "grad_norm": 2.098574116454142, + "language_loss": 0.77453035, + "learning_rate": 1.7869666782504668e-06, + "loss": 0.79636997, + "num_input_tokens_seen": 98297950, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.17248535, + "step": 4557, + "time_per_iteration": 2.7328546047210693 + }, + { + "auxiliary_loss_clip": 0.0113427, + "auxiliary_loss_mlp": 0.01037654, + "balance_loss_clip": 1.04543972, + "balance_loss_mlp": 1.02239501, + "epoch": 0.5480670955329766, + "flos": 18588800142720.0, + "grad_norm": 1.7473699002741403, + "language_loss": 0.6864866, + "learning_rate": 1.7861921544150867e-06, + "loss": 0.70820582, + "num_input_tokens_seen": 98316800, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.15258789, + "step": 4558, + "time_per_iteration": 2.8007099628448486 + }, + { + "auxiliary_loss_clip": 0.0114006, + "auxiliary_loss_mlp": 0.01049758, + "balance_loss_clip": 1.04644346, + "balance_loss_mlp": 1.03296161, + "epoch": 0.5481873384236157, + "flos": 15954608338560.0, + "grad_norm": 1.753592010091821, + "language_loss": 0.76661372, + "learning_rate": 1.7854176630141856e-06, + "loss": 0.78851187, + "num_input_tokens_seen": 98333935, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16796875, + "step": 4559, + "time_per_iteration": 2.80157470703125 + }, + { + "auxiliary_loss_clip": 0.01143914, + "auxiliary_loss_mlp": 0.01044128, + "balance_loss_clip": 1.04790568, + "balance_loss_mlp": 1.02584171, + "epoch": 0.5483075813142548, + "flos": 22783812606720.0, + "grad_norm": 2.998633494918665, + "language_loss": 0.84295481, + "learning_rate": 1.784643204165255e-06, + "loss": 0.86483526, + "num_input_tokens_seen": 98353255, + "router_z_loss_clip": 0.96044922, + "router_z_loss_mlp": 0.18286133, + "step": 4560, + "time_per_iteration": 2.798532485961914 + }, + { + "auxiliary_loss_clip": 0.01136949, + "auxiliary_loss_mlp": 0.01036403, + "balance_loss_clip": 1.04586494, + "balance_loss_mlp": 1.01958239, + "epoch": 0.5484278242048939, + "flos": 19317212046720.0, + "grad_norm": 2.2406981378718043, + "language_loss": 0.77235091, + "learning_rate": 1.7838687779857783e-06, + "loss": 0.79408443, + "num_input_tokens_seen": 98371130, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.16809082, + "step": 4561, + "time_per_iteration": 2.7624754905700684 + }, + { + "auxiliary_loss_clip": 0.01137285, + "auxiliary_loss_mlp": 0.01045333, + "balance_loss_clip": 1.04581094, + "balance_loss_mlp": 1.02898347, + "epoch": 0.5485480670955329, + "flos": 22816024128000.0, + "grad_norm": 1.7859203428241728, + "language_loss": 0.64104342, + "learning_rate": 1.7830943845932366e-06, + "loss": 0.66286957, + "num_input_tokens_seen": 98390455, + "router_z_loss_clip": 0.91357422, + "router_z_loss_mlp": 0.16351318, + "step": 4562, + "time_per_iteration": 2.7841908931732178 + }, + { + "auxiliary_loss_clip": 0.01137797, + "auxiliary_loss_mlp": 0.01043346, + "balance_loss_clip": 1.04637718, + "balance_loss_mlp": 1.0273596, + "epoch": 0.5486683099861721, + "flos": 22671304231680.0, + "grad_norm": 1.5443212653837928, + "language_loss": 0.75206625, + "learning_rate": 1.7823200241051044e-06, + "loss": 0.77387774, + "num_input_tokens_seen": 98409370, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.15991211, + "step": 4563, + "time_per_iteration": 2.7819998264312744 + }, + { + "auxiliary_loss_clip": 0.01141227, + "auxiliary_loss_mlp": 0.01046495, + "balance_loss_clip": 1.0492022, + "balance_loss_mlp": 1.03033006, + "epoch": 0.5487885528768112, + "flos": 23149383275520.0, + "grad_norm": 1.8541475778390253, + "language_loss": 0.79997993, + "learning_rate": 1.7815456966388513e-06, + "loss": 0.82185709, + "num_input_tokens_seen": 98428465, + "router_z_loss_clip": 0.91943359, + "router_z_loss_mlp": 0.16174316, + "step": 4564, + "time_per_iteration": 2.8289248943328857 + }, + { + "auxiliary_loss_clip": 0.01140875, + "auxiliary_loss_mlp": 0.01042496, + "balance_loss_clip": 1.04553938, + "balance_loss_mlp": 1.02482891, + "epoch": 0.5489087957674502, + "flos": 22053928245120.0, + "grad_norm": 3.5665234393311342, + "language_loss": 0.8135832, + "learning_rate": 1.780771402311943e-06, + "loss": 0.83541691, + "num_input_tokens_seen": 98447300, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.17663574, + "step": 4565, + "time_per_iteration": 2.8174936771392822 + }, + { + "auxiliary_loss_clip": 0.01139916, + "auxiliary_loss_mlp": 0.0104561, + "balance_loss_clip": 1.04671073, + "balance_loss_mlp": 1.02789569, + "epoch": 0.5490290386580894, + "flos": 24315977191680.0, + "grad_norm": 2.2617411008146524, + "language_loss": 0.78747797, + "learning_rate": 1.7799971412418374e-06, + "loss": 0.80933321, + "num_input_tokens_seen": 98468695, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.17736816, + "step": 4566, + "time_per_iteration": 2.831333637237549 + }, + { + "auxiliary_loss_clip": 0.01143311, + "auxiliary_loss_mlp": 0.01052202, + "balance_loss_clip": 1.04647899, + "balance_loss_mlp": 1.03495216, + "epoch": 0.5491492815487284, + "flos": 18294942977280.0, + "grad_norm": 1.9297114395390025, + "language_loss": 0.74135041, + "learning_rate": 1.7792229135459918e-06, + "loss": 0.7633056, + "num_input_tokens_seen": 98485345, + "router_z_loss_clip": 0.96826172, + "router_z_loss_mlp": 0.17248535, + "step": 4567, + "time_per_iteration": 2.7442963123321533 + }, + { + "auxiliary_loss_clip": 0.01045971, + "auxiliary_loss_mlp": 0.01005156, + "balance_loss_clip": 1.01590323, + "balance_loss_mlp": 1.00347543, + "epoch": 0.5492695244393675, + "flos": 64550257050240.0, + "grad_norm": 0.7326325317590507, + "language_loss": 0.61575472, + "learning_rate": 1.7784487193418538e-06, + "loss": 0.63626599, + "num_input_tokens_seen": 98543195, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.0168457, + "step": 4568, + "time_per_iteration": 3.1905293464660645 + }, + { + "auxiliary_loss_clip": 0.01141823, + "auxiliary_loss_mlp": 0.01040832, + "balance_loss_clip": 1.04386044, + "balance_loss_mlp": 1.02302241, + "epoch": 0.5493897673300067, + "flos": 17379579761280.0, + "grad_norm": 1.9263964030966712, + "language_loss": 0.6120221, + "learning_rate": 1.7776745587468698e-06, + "loss": 0.63384867, + "num_input_tokens_seen": 98560620, + "router_z_loss_clip": 0.97851562, + "router_z_loss_mlp": 0.17810059, + "step": 4569, + "time_per_iteration": 2.747422933578491 + }, + { + "auxiliary_loss_clip": 0.01138893, + "auxiliary_loss_mlp": 0.01034622, + "balance_loss_clip": 1.04518843, + "balance_loss_mlp": 1.01770663, + "epoch": 0.5495100102206457, + "flos": 19901765980800.0, + "grad_norm": 2.3755057856702404, + "language_loss": 0.81586707, + "learning_rate": 1.7769004318784776e-06, + "loss": 0.83760226, + "num_input_tokens_seen": 98578265, + "router_z_loss_clip": 0.93652344, + "router_z_loss_mlp": 0.16918945, + "step": 4570, + "time_per_iteration": 2.784999132156372 + }, + { + "auxiliary_loss_clip": 0.0113806, + "auxiliary_loss_mlp": 0.01037928, + "balance_loss_clip": 1.04472899, + "balance_loss_mlp": 1.02160859, + "epoch": 0.5496302531112848, + "flos": 16727190992640.0, + "grad_norm": 1.917914887533705, + "language_loss": 0.80703288, + "learning_rate": 1.776126338854113e-06, + "loss": 0.82879281, + "num_input_tokens_seen": 98596055, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.16320801, + "step": 4571, + "time_per_iteration": 2.7248153686523438 + }, + { + "auxiliary_loss_clip": 0.01134067, + "auxiliary_loss_mlp": 0.01038306, + "balance_loss_clip": 1.04335284, + "balance_loss_mlp": 1.02273738, + "epoch": 0.5497504960019239, + "flos": 24572343536640.0, + "grad_norm": 1.5095435012722813, + "language_loss": 0.84323657, + "learning_rate": 1.7753522797912044e-06, + "loss": 0.86496031, + "num_input_tokens_seen": 98616140, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.15557861, + "step": 4572, + "time_per_iteration": 2.8551993370056152 + }, + { + "auxiliary_loss_clip": 0.01138812, + "auxiliary_loss_mlp": 0.01048346, + "balance_loss_clip": 1.04502416, + "balance_loss_mlp": 1.03228831, + "epoch": 0.549870738892563, + "flos": 15450494912640.0, + "grad_norm": 2.050444521990237, + "language_loss": 0.69801551, + "learning_rate": 1.7745782548071765e-06, + "loss": 0.71988702, + "num_input_tokens_seen": 98633035, + "router_z_loss_clip": 0.93847656, + "router_z_loss_mlp": 0.16052246, + "step": 4573, + "time_per_iteration": 2.7459378242492676 + }, + { + "auxiliary_loss_clip": 0.01135529, + "auxiliary_loss_mlp": 0.01038833, + "balance_loss_clip": 1.0454278, + "balance_loss_mlp": 1.02258432, + "epoch": 0.549990981783202, + "flos": 21069114082560.0, + "grad_norm": 1.702537131244411, + "language_loss": 0.74732846, + "learning_rate": 1.7738042640194482e-06, + "loss": 0.76907206, + "num_input_tokens_seen": 98652700, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.16271973, + "step": 4574, + "time_per_iteration": 2.728822708129883 + }, + { + "auxiliary_loss_clip": 0.01139641, + "auxiliary_loss_mlp": 0.01044155, + "balance_loss_clip": 1.04617453, + "balance_loss_mlp": 1.02617872, + "epoch": 0.5501112246738411, + "flos": 21395901041280.0, + "grad_norm": 1.5590608831503225, + "language_loss": 0.70603204, + "learning_rate": 1.7730303075454335e-06, + "loss": 0.72786999, + "num_input_tokens_seen": 98671590, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.17993164, + "step": 4575, + "time_per_iteration": 2.781721591949463 + }, + { + "auxiliary_loss_clip": 0.01141997, + "auxiliary_loss_mlp": 0.01046276, + "balance_loss_clip": 1.04627061, + "balance_loss_mlp": 1.02939582, + "epoch": 0.5502314675644803, + "flos": 17456931699840.0, + "grad_norm": 1.742111475523213, + "language_loss": 0.84912592, + "learning_rate": 1.7722563855025402e-06, + "loss": 0.87100863, + "num_input_tokens_seen": 98689620, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.16882324, + "step": 4576, + "time_per_iteration": 3.5922470092773438 + }, + { + "auxiliary_loss_clip": 0.01140672, + "auxiliary_loss_mlp": 0.01037986, + "balance_loss_clip": 1.04504609, + "balance_loss_mlp": 1.0210588, + "epoch": 0.5503517104551193, + "flos": 24310410583680.0, + "grad_norm": 2.2356313696629484, + "language_loss": 0.71028984, + "learning_rate": 1.7714824980081721e-06, + "loss": 0.73207641, + "num_input_tokens_seen": 98708915, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.16918945, + "step": 4577, + "time_per_iteration": 2.77351450920105 + }, + { + "auxiliary_loss_clip": 0.01137434, + "auxiliary_loss_mlp": 0.01036073, + "balance_loss_clip": 1.04703784, + "balance_loss_mlp": 1.01977754, + "epoch": 0.5504719533457584, + "flos": 22419427086720.0, + "grad_norm": 1.636321241064812, + "language_loss": 0.74124253, + "learning_rate": 1.7707086451797276e-06, + "loss": 0.7629776, + "num_input_tokens_seen": 98729790, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.16296387, + "step": 4578, + "time_per_iteration": 2.7725844383239746 + }, + { + "auxiliary_loss_clip": 0.01045907, + "auxiliary_loss_mlp": 0.01001993, + "balance_loss_clip": 1.01592541, + "balance_loss_mlp": 1.00026417, + "epoch": 0.5505921962363975, + "flos": 67294155968640.0, + "grad_norm": 0.7036731890589599, + "language_loss": 0.523211, + "learning_rate": 1.7699348271345993e-06, + "loss": 0.54368997, + "num_input_tokens_seen": 98792415, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01733398, + "step": 4579, + "time_per_iteration": 4.0563318729400635 + }, + { + "auxiliary_loss_clip": 0.01045973, + "auxiliary_loss_mlp": 0.01002052, + "balance_loss_clip": 1.01578617, + "balance_loss_mlp": 1.00033534, + "epoch": 0.5507124391270366, + "flos": 45685125578880.0, + "grad_norm": 0.7067059652278678, + "language_loss": 0.54459935, + "learning_rate": 1.7691610439901753e-06, + "loss": 0.56507957, + "num_input_tokens_seen": 98855350, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.01721191, + "step": 4580, + "time_per_iteration": 3.3381917476654053 + }, + { + "auxiliary_loss_clip": 0.01136367, + "auxiliary_loss_mlp": 0.01040023, + "balance_loss_clip": 1.04362643, + "balance_loss_mlp": 1.02338743, + "epoch": 0.5508326820176757, + "flos": 22273845264000.0, + "grad_norm": 1.6524566659277773, + "language_loss": 0.7525481, + "learning_rate": 1.7683872958638367e-06, + "loss": 0.77431196, + "num_input_tokens_seen": 98874230, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.16644287, + "step": 4581, + "time_per_iteration": 2.7722978591918945 + }, + { + "auxiliary_loss_clip": 0.01138149, + "auxiliary_loss_mlp": 0.01038925, + "balance_loss_clip": 1.04463387, + "balance_loss_mlp": 1.02217579, + "epoch": 0.5509529249083148, + "flos": 20012442762240.0, + "grad_norm": 3.7141019431753026, + "language_loss": 0.84403574, + "learning_rate": 1.7676135828729614e-06, + "loss": 0.86580646, + "num_input_tokens_seen": 98893940, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.16748047, + "step": 4582, + "time_per_iteration": 3.542605400085449 + }, + { + "auxiliary_loss_clip": 0.01140936, + "auxiliary_loss_mlp": 0.01037674, + "balance_loss_clip": 1.04671156, + "balance_loss_mlp": 1.02190232, + "epoch": 0.5510731677989539, + "flos": 21834801325440.0, + "grad_norm": 1.9982332398315743, + "language_loss": 0.82700348, + "learning_rate": 1.7668399051349205e-06, + "loss": 0.84878957, + "num_input_tokens_seen": 98913620, + "router_z_loss_clip": 0.94238281, + "router_z_loss_mlp": 0.15771484, + "step": 4583, + "time_per_iteration": 2.7834107875823975 + }, + { + "auxiliary_loss_clip": 0.01141184, + "auxiliary_loss_mlp": 0.01050314, + "balance_loss_clip": 1.04631734, + "balance_loss_mlp": 1.03337479, + "epoch": 0.5511934106895929, + "flos": 21467901853440.0, + "grad_norm": 2.065404040114269, + "language_loss": 0.83388996, + "learning_rate": 1.766066262767081e-06, + "loss": 0.85580492, + "num_input_tokens_seen": 98931460, + "router_z_loss_clip": 0.94921875, + "router_z_loss_mlp": 0.16943359, + "step": 4584, + "time_per_iteration": 2.732332468032837 + }, + { + "auxiliary_loss_clip": 0.01135711, + "auxiliary_loss_mlp": 0.01041926, + "balance_loss_clip": 1.0446012, + "balance_loss_mlp": 1.02589226, + "epoch": 0.5513136535802321, + "flos": 21068934514560.0, + "grad_norm": 2.107945623431397, + "language_loss": 0.77326024, + "learning_rate": 1.765292655886803e-06, + "loss": 0.79503667, + "num_input_tokens_seen": 98950105, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.16027832, + "step": 4585, + "time_per_iteration": 3.59134578704834 + }, + { + "auxiliary_loss_clip": 0.01137438, + "auxiliary_loss_mlp": 0.01046235, + "balance_loss_clip": 1.04484427, + "balance_loss_mlp": 1.02931881, + "epoch": 0.5514338964708712, + "flos": 27815004754560.0, + "grad_norm": 2.159851038330378, + "language_loss": 0.71036494, + "learning_rate": 1.764519084611443e-06, + "loss": 0.7322017, + "num_input_tokens_seen": 98970560, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.16918945, + "step": 4586, + "time_per_iteration": 2.7454278469085693 + }, + { + "auxiliary_loss_clip": 0.01143868, + "auxiliary_loss_mlp": 0.01044827, + "balance_loss_clip": 1.046803, + "balance_loss_mlp": 1.02736318, + "epoch": 0.5515541393615102, + "flos": 21908525990400.0, + "grad_norm": 1.6918540500205987, + "language_loss": 0.78041494, + "learning_rate": 1.7637455490583505e-06, + "loss": 0.80230188, + "num_input_tokens_seen": 98989885, + "router_z_loss_clip": 0.97119141, + "router_z_loss_mlp": 0.17456055, + "step": 4587, + "time_per_iteration": 2.72405743598938 + }, + { + "auxiliary_loss_clip": 0.01135114, + "auxiliary_loss_mlp": 0.01043345, + "balance_loss_clip": 1.0450356, + "balance_loss_mlp": 1.02791297, + "epoch": 0.5516743822521494, + "flos": 20485422074880.0, + "grad_norm": 2.019576017417932, + "language_loss": 0.77324909, + "learning_rate": 1.7629720493448701e-06, + "loss": 0.79503369, + "num_input_tokens_seen": 99007180, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.15423584, + "step": 4588, + "time_per_iteration": 2.7275164127349854 + }, + { + "auxiliary_loss_clip": 0.0113922, + "auxiliary_loss_mlp": 0.01042054, + "balance_loss_clip": 1.04553199, + "balance_loss_mlp": 1.02374375, + "epoch": 0.5517946251427884, + "flos": 14940383915520.0, + "grad_norm": 1.724908766074257, + "language_loss": 0.85237938, + "learning_rate": 1.7621985855883418e-06, + "loss": 0.87419212, + "num_input_tokens_seen": 99023880, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.1829834, + "step": 4589, + "time_per_iteration": 2.683948278427124 + }, + { + "auxiliary_loss_clip": 0.01134819, + "auxiliary_loss_mlp": 0.01035851, + "balance_loss_clip": 1.04413271, + "balance_loss_mlp": 1.01972198, + "epoch": 0.5519148680334275, + "flos": 18404865573120.0, + "grad_norm": 1.8019042332051467, + "language_loss": 0.7249136, + "learning_rate": 1.7614251579060983e-06, + "loss": 0.74662024, + "num_input_tokens_seen": 99042475, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.16125488, + "step": 4590, + "time_per_iteration": 2.7331254482269287 + }, + { + "auxiliary_loss_clip": 0.01137503, + "auxiliary_loss_mlp": 0.01041878, + "balance_loss_clip": 1.04301071, + "balance_loss_mlp": 1.02542734, + "epoch": 0.5520351109240667, + "flos": 25113337251840.0, + "grad_norm": 1.8136544441078106, + "language_loss": 0.84745145, + "learning_rate": 1.76065176641547e-06, + "loss": 0.86924517, + "num_input_tokens_seen": 99065185, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.16430664, + "step": 4591, + "time_per_iteration": 2.809624195098877 + }, + { + "auxiliary_loss_clip": 0.01135619, + "auxiliary_loss_mlp": 0.01043915, + "balance_loss_clip": 1.04341125, + "balance_loss_mlp": 1.02845371, + "epoch": 0.5521553538147057, + "flos": 21069545045760.0, + "grad_norm": 2.100348218523541, + "language_loss": 0.77991486, + "learning_rate": 1.759878411233777e-06, + "loss": 0.80171019, + "num_input_tokens_seen": 99083645, + "router_z_loss_clip": 0.92138672, + "router_z_loss_mlp": 0.15460205, + "step": 4592, + "time_per_iteration": 2.7943320274353027 + }, + { + "auxiliary_loss_clip": 0.01134542, + "auxiliary_loss_mlp": 0.01046541, + "balance_loss_clip": 1.04472184, + "balance_loss_mlp": 1.03014946, + "epoch": 0.5522755967053448, + "flos": 18879999701760.0, + "grad_norm": 2.069212022399048, + "language_loss": 0.75971186, + "learning_rate": 1.7591050924783388e-06, + "loss": 0.78152275, + "num_input_tokens_seen": 99100835, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.16394043, + "step": 4593, + "time_per_iteration": 2.7188849449157715 + }, + { + "auxiliary_loss_clip": 0.01044204, + "auxiliary_loss_mlp": 0.0100275, + "balance_loss_clip": 1.01504922, + "balance_loss_mlp": 1.00127816, + "epoch": 0.5523958395959839, + "flos": 64675622494080.0, + "grad_norm": 0.8299141915674257, + "language_loss": 0.57883227, + "learning_rate": 1.7583318102664661e-06, + "loss": 0.59930182, + "num_input_tokens_seen": 99168400, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01470947, + "step": 4594, + "time_per_iteration": 3.3138184547424316 + }, + { + "auxiliary_loss_clip": 0.01136248, + "auxiliary_loss_mlp": 0.01039985, + "balance_loss_clip": 1.04300189, + "balance_loss_mlp": 1.02408206, + "epoch": 0.552516082486623, + "flos": 10889732211840.0, + "grad_norm": 1.740165812419504, + "language_loss": 0.7916767, + "learning_rate": 1.757558564715466e-06, + "loss": 0.81343901, + "num_input_tokens_seen": 99186475, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.15905762, + "step": 4595, + "time_per_iteration": 2.8296616077423096 + }, + { + "auxiliary_loss_clip": 0.01140596, + "auxiliary_loss_mlp": 0.0104191, + "balance_loss_clip": 1.04232073, + "balance_loss_mlp": 1.02497005, + "epoch": 0.552636325377262, + "flos": 22199797376640.0, + "grad_norm": 2.5838667653874303, + "language_loss": 0.74173838, + "learning_rate": 1.7567853559426386e-06, + "loss": 0.76356345, + "num_input_tokens_seen": 99203525, + "router_z_loss_clip": 0.98193359, + "router_z_loss_mlp": 0.16943359, + "step": 4596, + "time_per_iteration": 2.8233203887939453 + }, + { + "auxiliary_loss_clip": 0.0113848, + "auxiliary_loss_mlp": 0.01039356, + "balance_loss_clip": 1.04599917, + "balance_loss_mlp": 1.02367973, + "epoch": 0.5527565682679012, + "flos": 23988184652160.0, + "grad_norm": 2.473225663250515, + "language_loss": 0.75673687, + "learning_rate": 1.7560121840652797e-06, + "loss": 0.77851522, + "num_input_tokens_seen": 99222910, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.15686035, + "step": 4597, + "time_per_iteration": 2.856552839279175 + }, + { + "auxiliary_loss_clip": 0.0113803, + "auxiliary_loss_mlp": 0.01041985, + "balance_loss_clip": 1.04611874, + "balance_loss_mlp": 1.02599907, + "epoch": 0.5528768111585403, + "flos": 19719267955200.0, + "grad_norm": 3.355970684347729, + "language_loss": 0.69069481, + "learning_rate": 1.7552390492006782e-06, + "loss": 0.71249503, + "num_input_tokens_seen": 99241230, + "router_z_loss_clip": 0.91943359, + "router_z_loss_mlp": 0.15979004, + "step": 4598, + "time_per_iteration": 2.9110188484191895 + }, + { + "auxiliary_loss_clip": 0.01138571, + "auxiliary_loss_mlp": 0.01042025, + "balance_loss_clip": 1.04530931, + "balance_loss_mlp": 1.02470398, + "epoch": 0.5529970540491793, + "flos": 26215975002240.0, + "grad_norm": 3.391681104520554, + "language_loss": 0.65085387, + "learning_rate": 1.7544659514661184e-06, + "loss": 0.67265987, + "num_input_tokens_seen": 99264320, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.17321777, + "step": 4599, + "time_per_iteration": 2.853896379470825 + }, + { + "auxiliary_loss_clip": 0.01137966, + "auxiliary_loss_mlp": 0.01037264, + "balance_loss_clip": 1.04455161, + "balance_loss_mlp": 1.02169538, + "epoch": 0.5531172969398185, + "flos": 24425971614720.0, + "grad_norm": 2.1684829376705514, + "language_loss": 0.79987156, + "learning_rate": 1.7536928909788786e-06, + "loss": 0.82162386, + "num_input_tokens_seen": 99283625, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.15576172, + "step": 4600, + "time_per_iteration": 2.897334098815918 + }, + { + "auxiliary_loss_clip": 0.01043265, + "auxiliary_loss_mlp": 0.01002815, + "balance_loss_clip": 1.01376295, + "balance_loss_mlp": 1.00119352, + "epoch": 0.5532375398304575, + "flos": 64907316195840.0, + "grad_norm": 0.8784224715619077, + "language_loss": 0.61956131, + "learning_rate": 1.752919867856231e-06, + "loss": 0.6400221, + "num_input_tokens_seen": 99335270, + "router_z_loss_clip": 0.29492188, + "router_z_loss_mlp": 0.01623535, + "step": 4601, + "time_per_iteration": 3.256814956665039 + }, + { + "auxiliary_loss_clip": 0.01130794, + "auxiliary_loss_mlp": 0.01040274, + "balance_loss_clip": 1.04075742, + "balance_loss_mlp": 1.02457356, + "epoch": 0.5533577827210966, + "flos": 19683105937920.0, + "grad_norm": 2.8996207812211683, + "language_loss": 0.78725231, + "learning_rate": 1.7521468822154436e-06, + "loss": 0.80896294, + "num_input_tokens_seen": 99354185, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.15686035, + "step": 4602, + "time_per_iteration": 2.9929487705230713 + }, + { + "auxiliary_loss_clip": 0.01130589, + "auxiliary_loss_mlp": 0.01040093, + "balance_loss_clip": 1.04191899, + "balance_loss_mlp": 1.02396417, + "epoch": 0.5534780256117358, + "flos": 32306496076800.0, + "grad_norm": 2.411229378306766, + "language_loss": 0.75588453, + "learning_rate": 1.751373934173777e-06, + "loss": 0.77759135, + "num_input_tokens_seen": 99376930, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.16125488, + "step": 4603, + "time_per_iteration": 3.1991593837738037 + }, + { + "auxiliary_loss_clip": 0.01139173, + "auxiliary_loss_mlp": 0.01038847, + "balance_loss_clip": 1.04473853, + "balance_loss_mlp": 1.02281332, + "epoch": 0.5535982685023748, + "flos": 23222425582080.0, + "grad_norm": 1.5787041153192283, + "language_loss": 0.73320615, + "learning_rate": 1.750601023848487e-06, + "loss": 0.75498629, + "num_input_tokens_seen": 99397655, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.16040039, + "step": 4604, + "time_per_iteration": 2.8498151302337646 + }, + { + "auxiliary_loss_clip": 0.01135882, + "auxiliary_loss_mlp": 0.0103748, + "balance_loss_clip": 1.04468226, + "balance_loss_mlp": 1.0218991, + "epoch": 0.5537185113930139, + "flos": 24352534258560.0, + "grad_norm": 2.051232239984975, + "language_loss": 0.74041283, + "learning_rate": 1.749828151356823e-06, + "loss": 0.76214641, + "num_input_tokens_seen": 99417850, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.15563965, + "step": 4605, + "time_per_iteration": 3.7400906085968018 + }, + { + "auxiliary_loss_clip": 0.01134386, + "auxiliary_loss_mlp": 0.01037905, + "balance_loss_clip": 1.04328978, + "balance_loss_mlp": 1.02218127, + "epoch": 0.553838754283653, + "flos": 23549068886400.0, + "grad_norm": 1.5319692353568213, + "language_loss": 0.75649476, + "learning_rate": 1.7490553168160297e-06, + "loss": 0.77821767, + "num_input_tokens_seen": 99438920, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.15710449, + "step": 4606, + "time_per_iteration": 2.8771252632141113 + }, + { + "auxiliary_loss_clip": 0.01136675, + "auxiliary_loss_mlp": 0.01039558, + "balance_loss_clip": 1.04507875, + "balance_loss_mlp": 1.02384675, + "epoch": 0.5539589971742921, + "flos": 17275044205440.0, + "grad_norm": 2.0076186146503217, + "language_loss": 0.76780522, + "learning_rate": 1.748282520343345e-06, + "loss": 0.78956753, + "num_input_tokens_seen": 99457950, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.15710449, + "step": 4607, + "time_per_iteration": 2.98523211479187 + }, + { + "auxiliary_loss_clip": 0.01140437, + "auxiliary_loss_mlp": 0.01037889, + "balance_loss_clip": 1.04541075, + "balance_loss_mlp": 1.02140248, + "epoch": 0.5540792400649311, + "flos": 27564169104000.0, + "grad_norm": 2.035670612560311, + "language_loss": 0.78787875, + "learning_rate": 1.7475097620560023e-06, + "loss": 0.80966198, + "num_input_tokens_seen": 99478015, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.16491699, + "step": 4608, + "time_per_iteration": 3.7110633850097656 + }, + { + "auxiliary_loss_clip": 0.01136039, + "auxiliary_loss_mlp": 0.01040821, + "balance_loss_clip": 1.04533958, + "balance_loss_mlp": 1.02560961, + "epoch": 0.5541994829555702, + "flos": 23878657105920.0, + "grad_norm": 1.6832657188538265, + "language_loss": 0.71374679, + "learning_rate": 1.746737042071228e-06, + "loss": 0.73551536, + "num_input_tokens_seen": 99496520, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.15209961, + "step": 4609, + "time_per_iteration": 2.7830092906951904 + }, + { + "auxiliary_loss_clip": 0.01133042, + "auxiliary_loss_mlp": 0.01037274, + "balance_loss_clip": 1.04367471, + "balance_loss_mlp": 1.02103734, + "epoch": 0.5543197258462094, + "flos": 20115721342080.0, + "grad_norm": 1.8422439181834722, + "language_loss": 0.789702, + "learning_rate": 1.7459643605062424e-06, + "loss": 0.81140518, + "num_input_tokens_seen": 99513780, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.16223145, + "step": 4610, + "time_per_iteration": 2.8430609703063965 + }, + { + "auxiliary_loss_clip": 0.01135918, + "auxiliary_loss_mlp": 0.01044548, + "balance_loss_clip": 1.04424703, + "balance_loss_mlp": 1.02712011, + "epoch": 0.5544399687368484, + "flos": 20916565021440.0, + "grad_norm": 1.8076660848791237, + "language_loss": 0.80847633, + "learning_rate": 1.745191717478262e-06, + "loss": 0.8302809, + "num_input_tokens_seen": 99532360, + "router_z_loss_clip": 0.91601562, + "router_z_loss_mlp": 0.17431641, + "step": 4611, + "time_per_iteration": 3.5754780769348145 + }, + { + "auxiliary_loss_clip": 0.01134194, + "auxiliary_loss_mlp": 0.01038585, + "balance_loss_clip": 1.04477656, + "balance_loss_mlp": 1.02341008, + "epoch": 0.5545602116274875, + "flos": 25518661297920.0, + "grad_norm": 1.7489078548506283, + "language_loss": 0.79696798, + "learning_rate": 1.7444191131044948e-06, + "loss": 0.81869578, + "num_input_tokens_seen": 99552635, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.15161133, + "step": 4612, + "time_per_iteration": 2.8007452487945557 + }, + { + "auxiliary_loss_clip": 0.01140502, + "auxiliary_loss_mlp": 0.01044972, + "balance_loss_clip": 1.04695988, + "balance_loss_mlp": 1.02890253, + "epoch": 0.5546804545181266, + "flos": 20995568985600.0, + "grad_norm": 1.7193009365905902, + "language_loss": 0.73086315, + "learning_rate": 1.7436465475021456e-06, + "loss": 0.75271791, + "num_input_tokens_seen": 99572685, + "router_z_loss_clip": 0.93505859, + "router_z_loss_mlp": 0.1607666, + "step": 4613, + "time_per_iteration": 2.7576606273651123 + }, + { + "auxiliary_loss_clip": 0.01135122, + "auxiliary_loss_mlp": 0.01044554, + "balance_loss_clip": 1.04511428, + "balance_loss_mlp": 1.02866328, + "epoch": 0.5548006974087657, + "flos": 26833638297600.0, + "grad_norm": 2.171760805526864, + "language_loss": 0.71467197, + "learning_rate": 1.7428740207884111e-06, + "loss": 0.73646867, + "num_input_tokens_seen": 99593565, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.15899658, + "step": 4614, + "time_per_iteration": 3.6516010761260986 + }, + { + "auxiliary_loss_clip": 0.01138711, + "auxiliary_loss_mlp": 0.01042975, + "balance_loss_clip": 1.04455006, + "balance_loss_mlp": 1.02697742, + "epoch": 0.5549209402994048, + "flos": 33656414031360.0, + "grad_norm": 1.7137134583363527, + "language_loss": 0.61051714, + "learning_rate": 1.7421015330804833e-06, + "loss": 0.63233399, + "num_input_tokens_seen": 99613485, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.15991211, + "step": 4615, + "time_per_iteration": 2.893162965774536 + }, + { + "auxiliary_loss_clip": 0.01137902, + "auxiliary_loss_mlp": 0.01042383, + "balance_loss_clip": 1.04497826, + "balance_loss_mlp": 1.02508545, + "epoch": 0.5550411831900439, + "flos": 23769524609280.0, + "grad_norm": 1.8035235704937564, + "language_loss": 0.72251797, + "learning_rate": 1.7413290844955475e-06, + "loss": 0.74432087, + "num_input_tokens_seen": 99633515, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.17297363, + "step": 4616, + "time_per_iteration": 2.8531670570373535 + }, + { + "auxiliary_loss_clip": 0.01137766, + "auxiliary_loss_mlp": 0.01044329, + "balance_loss_clip": 1.04763663, + "balance_loss_mlp": 1.02802122, + "epoch": 0.555161426080683, + "flos": 21651189978240.0, + "grad_norm": 1.8070780678166416, + "language_loss": 0.78197122, + "learning_rate": 1.7405566751507843e-06, + "loss": 0.80379224, + "num_input_tokens_seen": 99651560, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.16308594, + "step": 4617, + "time_per_iteration": 2.7878801822662354 + }, + { + "auxiliary_loss_clip": 0.0113466, + "auxiliary_loss_mlp": 0.01039651, + "balance_loss_clip": 1.0443449, + "balance_loss_mlp": 1.02358174, + "epoch": 0.555281668971322, + "flos": 49563116605440.0, + "grad_norm": 1.4523752289688396, + "language_loss": 0.67601311, + "learning_rate": 1.7397843051633668e-06, + "loss": 0.69775629, + "num_input_tokens_seen": 99674255, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.16064453, + "step": 4618, + "time_per_iteration": 3.0844836235046387 + }, + { + "auxiliary_loss_clip": 0.01137225, + "auxiliary_loss_mlp": 0.01037444, + "balance_loss_clip": 1.04481721, + "balance_loss_mlp": 1.02033758, + "epoch": 0.5554019118619612, + "flos": 20741608851840.0, + "grad_norm": 1.6328480017419735, + "language_loss": 0.71434307, + "learning_rate": 1.739011974650464e-06, + "loss": 0.73608977, + "num_input_tokens_seen": 99693585, + "router_z_loss_clip": 0.92382812, + "router_z_loss_mlp": 0.17102051, + "step": 4619, + "time_per_iteration": 2.904815435409546 + }, + { + "auxiliary_loss_clip": 0.0114242, + "auxiliary_loss_mlp": 0.01042612, + "balance_loss_clip": 1.04447126, + "balance_loss_mlp": 1.02400374, + "epoch": 0.5555221547526003, + "flos": 25483217552640.0, + "grad_norm": 2.0369802255398914, + "language_loss": 0.76761949, + "learning_rate": 1.7382396837292365e-06, + "loss": 0.78946984, + "num_input_tokens_seen": 99714045, + "router_z_loss_clip": 0.97851562, + "router_z_loss_mlp": 0.1862793, + "step": 4620, + "time_per_iteration": 2.9130914211273193 + }, + { + "auxiliary_loss_clip": 0.01137084, + "auxiliary_loss_mlp": 0.010429, + "balance_loss_clip": 1.04532456, + "balance_loss_mlp": 1.0256629, + "epoch": 0.5556423976432393, + "flos": 21762513204480.0, + "grad_norm": 1.6060890616177057, + "language_loss": 0.73251474, + "learning_rate": 1.737467432516841e-06, + "loss": 0.7543146, + "num_input_tokens_seen": 99734145, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.17236328, + "step": 4621, + "time_per_iteration": 2.816301107406616 + }, + { + "auxiliary_loss_clip": 0.01136966, + "auxiliary_loss_mlp": 0.01035959, + "balance_loss_clip": 1.04319787, + "balance_loss_mlp": 1.01937747, + "epoch": 0.5557626405338785, + "flos": 24900171989760.0, + "grad_norm": 2.371520553596783, + "language_loss": 0.73938894, + "learning_rate": 1.7366952211304274e-06, + "loss": 0.76111817, + "num_input_tokens_seen": 99751990, + "router_z_loss_clip": 0.93896484, + "router_z_loss_mlp": 0.16601562, + "step": 4622, + "time_per_iteration": 2.8394055366516113 + }, + { + "auxiliary_loss_clip": 0.01139561, + "auxiliary_loss_mlp": 0.01043373, + "balance_loss_clip": 1.04659438, + "balance_loss_mlp": 1.02819753, + "epoch": 0.5558828834245175, + "flos": 18697501676160.0, + "grad_norm": 2.124425405985386, + "language_loss": 0.83454788, + "learning_rate": 1.735923049687139e-06, + "loss": 0.85637724, + "num_input_tokens_seen": 99768565, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.15185547, + "step": 4623, + "time_per_iteration": 2.7989633083343506 + }, + { + "auxiliary_loss_clip": 0.01133923, + "auxiliary_loss_mlp": 0.01038334, + "balance_loss_clip": 1.04421782, + "balance_loss_mlp": 1.0220027, + "epoch": 0.5560031263151566, + "flos": 27272179445760.0, + "grad_norm": 1.461303130292067, + "language_loss": 0.73821616, + "learning_rate": 1.7351509183041144e-06, + "loss": 0.75993872, + "num_input_tokens_seen": 99788895, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.16320801, + "step": 4624, + "time_per_iteration": 2.8341565132141113 + }, + { + "auxiliary_loss_clip": 0.01136612, + "auxiliary_loss_mlp": 0.01041224, + "balance_loss_clip": 1.04436505, + "balance_loss_mlp": 1.02502918, + "epoch": 0.5561233692057957, + "flos": 23403738458880.0, + "grad_norm": 1.6766478854648383, + "language_loss": 0.71831441, + "learning_rate": 1.7343788270984852e-06, + "loss": 0.74009275, + "num_input_tokens_seen": 99808035, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.16192627, + "step": 4625, + "time_per_iteration": 2.9880850315093994 + }, + { + "auxiliary_loss_clip": 0.01137542, + "auxiliary_loss_mlp": 0.01049693, + "balance_loss_clip": 1.04404175, + "balance_loss_mlp": 1.03233576, + "epoch": 0.5562436120964348, + "flos": 37670867804160.0, + "grad_norm": 2.0562860539638805, + "language_loss": 0.74639964, + "learning_rate": 1.7336067761873764e-06, + "loss": 0.76827198, + "num_input_tokens_seen": 99830460, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.17358398, + "step": 4626, + "time_per_iteration": 2.9474942684173584 + }, + { + "auxiliary_loss_clip": 0.01139754, + "auxiliary_loss_mlp": 0.010421, + "balance_loss_clip": 1.0433507, + "balance_loss_mlp": 1.02496958, + "epoch": 0.5563638549870739, + "flos": 25155245445120.0, + "grad_norm": 2.0946334691027935, + "language_loss": 0.76398605, + "learning_rate": 1.7328347656879076e-06, + "loss": 0.78580463, + "num_input_tokens_seen": 99850320, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.17114258, + "step": 4627, + "time_per_iteration": 2.8480606079101562 + }, + { + "auxiliary_loss_clip": 0.01141105, + "auxiliary_loss_mlp": 0.010432, + "balance_loss_clip": 1.04699278, + "balance_loss_mlp": 1.02673697, + "epoch": 0.556484097877713, + "flos": 13581810783360.0, + "grad_norm": 2.188175091550685, + "language_loss": 0.68804985, + "learning_rate": 1.7320627957171927e-06, + "loss": 0.70989287, + "num_input_tokens_seen": 99864980, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.16467285, + "step": 4628, + "time_per_iteration": 2.734287977218628 + }, + { + "auxiliary_loss_clip": 0.01137276, + "auxiliary_loss_mlp": 0.01045337, + "balance_loss_clip": 1.04547858, + "balance_loss_mlp": 1.02856994, + "epoch": 0.5566043407683521, + "flos": 24681368292480.0, + "grad_norm": 1.8929932585765106, + "language_loss": 0.81660521, + "learning_rate": 1.7312908663923382e-06, + "loss": 0.83843136, + "num_input_tokens_seen": 99881155, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.16766357, + "step": 4629, + "time_per_iteration": 2.8525357246398926 + }, + { + "auxiliary_loss_clip": 0.0113832, + "auxiliary_loss_mlp": 0.01042783, + "balance_loss_clip": 1.04469776, + "balance_loss_mlp": 1.024472, + "epoch": 0.5567245836589911, + "flos": 20588161950720.0, + "grad_norm": 2.403312623479095, + "language_loss": 0.67084187, + "learning_rate": 1.7305189778304463e-06, + "loss": 0.69265294, + "num_input_tokens_seen": 99899330, + "router_z_loss_clip": 0.93652344, + "router_z_loss_mlp": 0.18310547, + "step": 4630, + "time_per_iteration": 2.8141260147094727 + }, + { + "auxiliary_loss_clip": 0.01137503, + "auxiliary_loss_mlp": 0.01035313, + "balance_loss_clip": 1.04595613, + "balance_loss_mlp": 1.02031636, + "epoch": 0.5568448265496303, + "flos": 20704189858560.0, + "grad_norm": 1.8557438247994105, + "language_loss": 0.79961181, + "learning_rate": 1.729747130148611e-06, + "loss": 0.82133996, + "num_input_tokens_seen": 99918525, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.14996338, + "step": 4631, + "time_per_iteration": 2.7836625576019287 + }, + { + "auxiliary_loss_clip": 0.0114079, + "auxiliary_loss_mlp": 0.0104198, + "balance_loss_clip": 1.0438087, + "balance_loss_mlp": 1.02427733, + "epoch": 0.5569650694402694, + "flos": 25302910256640.0, + "grad_norm": 2.0838039890906046, + "language_loss": 0.76663363, + "learning_rate": 1.7289753234639208e-06, + "loss": 0.78846133, + "num_input_tokens_seen": 99937500, + "router_z_loss_clip": 0.97021484, + "router_z_loss_mlp": 0.17681885, + "step": 4632, + "time_per_iteration": 3.0165183544158936 + }, + { + "auxiliary_loss_clip": 0.01139328, + "auxiliary_loss_mlp": 0.01045086, + "balance_loss_clip": 1.04465961, + "balance_loss_mlp": 1.02808714, + "epoch": 0.5570853123309084, + "flos": 19712623939200.0, + "grad_norm": 1.7378087804143896, + "language_loss": 0.76639283, + "learning_rate": 1.7282035578934592e-06, + "loss": 0.78823698, + "num_input_tokens_seen": 99955665, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.17004395, + "step": 4633, + "time_per_iteration": 2.908928394317627 + }, + { + "auxiliary_loss_clip": 0.01138472, + "auxiliary_loss_mlp": 0.01045021, + "balance_loss_clip": 1.04665685, + "balance_loss_mlp": 1.02805793, + "epoch": 0.5572055552215476, + "flos": 16108091153280.0, + "grad_norm": 1.6690200965976236, + "language_loss": 0.78604567, + "learning_rate": 1.727431833554301e-06, + "loss": 0.80788064, + "num_input_tokens_seen": 99974140, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.1697998, + "step": 4634, + "time_per_iteration": 3.815157651901245 + }, + { + "auxiliary_loss_clip": 0.01139438, + "auxiliary_loss_mlp": 0.01041638, + "balance_loss_clip": 1.04582524, + "balance_loss_mlp": 1.024019, + "epoch": 0.5573257981121866, + "flos": 17128815937920.0, + "grad_norm": 1.8302348493108176, + "language_loss": 0.77184224, + "learning_rate": 1.7266601505635175e-06, + "loss": 0.79365301, + "num_input_tokens_seen": 99991480, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.17626953, + "step": 4635, + "time_per_iteration": 2.802077054977417 + }, + { + "auxiliary_loss_clip": 0.01139952, + "auxiliary_loss_mlp": 0.01038701, + "balance_loss_clip": 1.04554057, + "balance_loss_mlp": 1.02178514, + "epoch": 0.5574460410028257, + "flos": 18807029222400.0, + "grad_norm": 2.039740365338987, + "language_loss": 0.75713944, + "learning_rate": 1.7258885090381717e-06, + "loss": 0.77892596, + "num_input_tokens_seen": 100009520, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.16918945, + "step": 4636, + "time_per_iteration": 2.855135679244995 + }, + { + "auxiliary_loss_clip": 0.01136263, + "auxiliary_loss_mlp": 0.01045618, + "balance_loss_clip": 1.04538834, + "balance_loss_mlp": 1.02960205, + "epoch": 0.5575662838934649, + "flos": 29642678530560.0, + "grad_norm": 1.7901264402478863, + "language_loss": 0.78544462, + "learning_rate": 1.7251169090953213e-06, + "loss": 0.80726337, + "num_input_tokens_seen": 100029995, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.16009521, + "step": 4637, + "time_per_iteration": 4.542562961578369 + }, + { + "auxiliary_loss_clip": 0.01141729, + "auxiliary_loss_mlp": 0.01048594, + "balance_loss_clip": 1.04622674, + "balance_loss_mlp": 1.03120089, + "epoch": 0.5576865267841039, + "flos": 22054466949120.0, + "grad_norm": 2.56857670922154, + "language_loss": 0.76620018, + "learning_rate": 1.7243453508520168e-06, + "loss": 0.78810334, + "num_input_tokens_seen": 100046980, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.1739502, + "step": 4638, + "time_per_iteration": 7.333468198776245 + }, + { + "auxiliary_loss_clip": 0.01140712, + "auxiliary_loss_mlp": 0.01037203, + "balance_loss_clip": 1.04561043, + "balance_loss_mlp": 1.02089572, + "epoch": 0.557806769674743, + "flos": 17196040241280.0, + "grad_norm": 1.9311565663045296, + "language_loss": 0.84266913, + "learning_rate": 1.7235738344253038e-06, + "loss": 0.86444831, + "num_input_tokens_seen": 100060610, + "router_z_loss_clip": 0.95019531, + "router_z_loss_mlp": 0.16308594, + "step": 4639, + "time_per_iteration": 3.6492910385131836 + }, + { + "auxiliary_loss_clip": 0.0114054, + "auxiliary_loss_mlp": 0.01044057, + "balance_loss_clip": 1.04579902, + "balance_loss_mlp": 1.02678323, + "epoch": 0.557927012565382, + "flos": 24712717887360.0, + "grad_norm": 1.740110851112513, + "language_loss": 0.82443678, + "learning_rate": 1.72280235993222e-06, + "loss": 0.84628272, + "num_input_tokens_seen": 100078915, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.17260742, + "step": 4640, + "time_per_iteration": 3.914973258972168 + }, + { + "auxiliary_loss_clip": 0.01140907, + "auxiliary_loss_mlp": 0.01043375, + "balance_loss_clip": 1.04548383, + "balance_loss_mlp": 1.02501655, + "epoch": 0.5580472554560212, + "flos": 16983090460800.0, + "grad_norm": 2.165256955933871, + "language_loss": 0.69613028, + "learning_rate": 1.722030927489798e-06, + "loss": 0.71797311, + "num_input_tokens_seen": 100096195, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.18347168, + "step": 4641, + "time_per_iteration": 3.1037659645080566 + }, + { + "auxiliary_loss_clip": 0.01137573, + "auxiliary_loss_mlp": 0.01042356, + "balance_loss_clip": 1.0453403, + "balance_loss_mlp": 1.02563143, + "epoch": 0.5581674983466602, + "flos": 23509100027520.0, + "grad_norm": 1.6260980804568574, + "language_loss": 0.74430954, + "learning_rate": 1.7212595372150634e-06, + "loss": 0.76610887, + "num_input_tokens_seen": 100116175, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.16711426, + "step": 4642, + "time_per_iteration": 3.7678067684173584 + }, + { + "auxiliary_loss_clip": 0.01137999, + "auxiliary_loss_mlp": 0.01041951, + "balance_loss_clip": 1.04636168, + "balance_loss_mlp": 1.02466595, + "epoch": 0.5582877412372993, + "flos": 13480291969920.0, + "grad_norm": 2.233392985618583, + "language_loss": 0.72795463, + "learning_rate": 1.720488189225035e-06, + "loss": 0.74975407, + "num_input_tokens_seen": 100133875, + "router_z_loss_clip": 0.91650391, + "router_z_loss_mlp": 0.17297363, + "step": 4643, + "time_per_iteration": 3.0243170261383057 + }, + { + "auxiliary_loss_clip": 0.01140605, + "auxiliary_loss_mlp": 0.01041815, + "balance_loss_clip": 1.04510403, + "balance_loss_mlp": 1.02439833, + "epoch": 0.5584079841279385, + "flos": 21903605827200.0, + "grad_norm": 2.514554445332564, + "language_loss": 0.79720736, + "learning_rate": 1.7197168836367265e-06, + "loss": 0.81903148, + "num_input_tokens_seen": 100150685, + "router_z_loss_clip": 0.95507812, + "router_z_loss_mlp": 0.17419434, + "step": 4644, + "time_per_iteration": 2.9244327545166016 + }, + { + "auxiliary_loss_clip": 0.01136463, + "auxiliary_loss_mlp": 0.01038595, + "balance_loss_clip": 1.04508185, + "balance_loss_mlp": 1.02272844, + "epoch": 0.5585282270185775, + "flos": 18843550375680.0, + "grad_norm": 1.8406642713266934, + "language_loss": 0.82046735, + "learning_rate": 1.7189456205671433e-06, + "loss": 0.84221792, + "num_input_tokens_seen": 100169530, + "router_z_loss_clip": 0.91357422, + "router_z_loss_mlp": 0.15881348, + "step": 4645, + "time_per_iteration": 2.857154607772827 + }, + { + "auxiliary_loss_clip": 0.01142618, + "auxiliary_loss_mlp": 0.01045294, + "balance_loss_clip": 1.04776263, + "balance_loss_mlp": 1.02722228, + "epoch": 0.5586484699092166, + "flos": 21868449390720.0, + "grad_norm": 2.20349134998783, + "language_loss": 0.82663655, + "learning_rate": 1.7181744001332866e-06, + "loss": 0.84851575, + "num_input_tokens_seen": 100188140, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.18078613, + "step": 4646, + "time_per_iteration": 2.99674391746521 + }, + { + "auxiliary_loss_clip": 0.01138575, + "auxiliary_loss_mlp": 0.01039225, + "balance_loss_clip": 1.04691982, + "balance_loss_mlp": 1.02385902, + "epoch": 0.5587687127998557, + "flos": 22893232412160.0, + "grad_norm": 2.0273311132859693, + "language_loss": 0.63330054, + "learning_rate": 1.7174032224521493e-06, + "loss": 0.65507853, + "num_input_tokens_seen": 100206850, + "router_z_loss_clip": 0.91552734, + "router_z_loss_mlp": 0.15380859, + "step": 4647, + "time_per_iteration": 2.7762112617492676 + }, + { + "auxiliary_loss_clip": 0.0113384, + "auxiliary_loss_mlp": 0.01038156, + "balance_loss_clip": 1.04468393, + "balance_loss_mlp": 1.02137733, + "epoch": 0.5588889556904948, + "flos": 20303067703680.0, + "grad_norm": 1.6102884556382264, + "language_loss": 0.69944978, + "learning_rate": 1.7166320876407184e-06, + "loss": 0.72116983, + "num_input_tokens_seen": 100226270, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.1675415, + "step": 4648, + "time_per_iteration": 2.792240619659424 + }, + { + "auxiliary_loss_clip": 0.01142961, + "auxiliary_loss_mlp": 0.01044842, + "balance_loss_clip": 1.04788482, + "balance_loss_mlp": 1.02749729, + "epoch": 0.5590091985811338, + "flos": 16472153450880.0, + "grad_norm": 2.102679189186136, + "language_loss": 0.67446887, + "learning_rate": 1.7158609958159742e-06, + "loss": 0.69634688, + "num_input_tokens_seen": 100243675, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.17321777, + "step": 4649, + "time_per_iteration": 2.9693660736083984 + }, + { + "auxiliary_loss_clip": 0.01143171, + "auxiliary_loss_mlp": 0.01043095, + "balance_loss_clip": 1.04754782, + "balance_loss_mlp": 1.02501142, + "epoch": 0.559129441471773, + "flos": 14532186781440.0, + "grad_norm": 1.9827535696133987, + "language_loss": 0.78442359, + "learning_rate": 1.7150899470948911e-06, + "loss": 0.80628622, + "num_input_tokens_seen": 100258940, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.1809082, + "step": 4650, + "time_per_iteration": 2.9400806427001953 + }, + { + "auxiliary_loss_clip": 0.01045301, + "auxiliary_loss_mlp": 0.01002729, + "balance_loss_clip": 1.01609623, + "balance_loss_mlp": 1.00084555, + "epoch": 0.5592496843624121, + "flos": 60521009852160.0, + "grad_norm": 0.8041474377196924, + "language_loss": 0.56580961, + "learning_rate": 1.7143189415944365e-06, + "loss": 0.58628988, + "num_input_tokens_seen": 100323400, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01879883, + "step": 4651, + "time_per_iteration": 3.3603904247283936 + }, + { + "auxiliary_loss_clip": 0.01138987, + "auxiliary_loss_mlp": 0.0104301, + "balance_loss_clip": 1.04693985, + "balance_loss_mlp": 1.02596283, + "epoch": 0.5593699272530511, + "flos": 20886256920960.0, + "grad_norm": 1.7344718493316895, + "language_loss": 0.76548135, + "learning_rate": 1.7135479794315714e-06, + "loss": 0.78730142, + "num_input_tokens_seen": 100340355, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.17041016, + "step": 4652, + "time_per_iteration": 2.9014384746551514 + }, + { + "auxiliary_loss_clip": 0.01136495, + "auxiliary_loss_mlp": 0.01037707, + "balance_loss_clip": 1.04476976, + "balance_loss_mlp": 1.02079105, + "epoch": 0.5594901701436903, + "flos": 12896743616640.0, + "grad_norm": 3.057817461163595, + "language_loss": 0.78719026, + "learning_rate": 1.7127770607232502e-06, + "loss": 0.8089323, + "num_input_tokens_seen": 100358900, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.16918945, + "step": 4653, + "time_per_iteration": 2.8593955039978027 + }, + { + "auxiliary_loss_clip": 0.01138141, + "auxiliary_loss_mlp": 0.01046678, + "balance_loss_clip": 1.04511309, + "balance_loss_mlp": 1.02988148, + "epoch": 0.5596104130343293, + "flos": 23112107936640.0, + "grad_norm": 1.974468098899662, + "language_loss": 0.79863179, + "learning_rate": 1.7120061855864204e-06, + "loss": 0.82047993, + "num_input_tokens_seen": 100378910, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.16784668, + "step": 4654, + "time_per_iteration": 2.9594473838806152 + }, + { + "auxiliary_loss_clip": 0.01140597, + "auxiliary_loss_mlp": 0.01046168, + "balance_loss_clip": 1.04573131, + "balance_loss_mlp": 1.02820289, + "epoch": 0.5597306559249684, + "flos": 25957812977280.0, + "grad_norm": 3.9409405502114305, + "language_loss": 0.7141428, + "learning_rate": 1.7112353541380233e-06, + "loss": 0.73601049, + "num_input_tokens_seen": 100398770, + "router_z_loss_clip": 0.94921875, + "router_z_loss_mlp": 0.1796875, + "step": 4655, + "time_per_iteration": 2.9204182624816895 + }, + { + "auxiliary_loss_clip": 0.01137883, + "auxiliary_loss_mlp": 0.0105181, + "balance_loss_clip": 1.04759157, + "balance_loss_mlp": 1.03359473, + "epoch": 0.5598508988156076, + "flos": 22492289825280.0, + "grad_norm": 1.5294085484267603, + "language_loss": 0.72149915, + "learning_rate": 1.7104645664949931e-06, + "loss": 0.74339604, + "num_input_tokens_seen": 100421240, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.18237305, + "step": 4656, + "time_per_iteration": 3.065232753753662 + }, + { + "auxiliary_loss_clip": 0.01140334, + "auxiliary_loss_mlp": 0.01038848, + "balance_loss_clip": 1.04686427, + "balance_loss_mlp": 1.02280283, + "epoch": 0.5599711417062466, + "flos": 23112538899840.0, + "grad_norm": 1.8044933474035474, + "language_loss": 0.71536005, + "learning_rate": 1.7096938227742584e-06, + "loss": 0.73715186, + "num_input_tokens_seen": 100442370, + "router_z_loss_clip": 0.93505859, + "router_z_loss_mlp": 0.16052246, + "step": 4657, + "time_per_iteration": 3.0176267623901367 + }, + { + "auxiliary_loss_clip": 0.01137341, + "auxiliary_loss_mlp": 0.010436, + "balance_loss_clip": 1.04316092, + "balance_loss_mlp": 1.02589762, + "epoch": 0.5600913845968857, + "flos": 22339345714560.0, + "grad_norm": 8.828284412311717, + "language_loss": 0.84378779, + "learning_rate": 1.70892312309274e-06, + "loss": 0.86559719, + "num_input_tokens_seen": 100460260, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.17700195, + "step": 4658, + "time_per_iteration": 2.9252045154571533 + }, + { + "auxiliary_loss_clip": 0.01139494, + "auxiliary_loss_mlp": 0.01046393, + "balance_loss_clip": 1.04538131, + "balance_loss_mlp": 1.02830863, + "epoch": 0.5602116274875248, + "flos": 17633791290240.0, + "grad_norm": 1.9227786603363475, + "language_loss": 0.68285871, + "learning_rate": 1.7081524675673523e-06, + "loss": 0.70471764, + "num_input_tokens_seen": 100475750, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.1809082, + "step": 4659, + "time_per_iteration": 2.8171751499176025 + }, + { + "auxiliary_loss_clip": 0.01044136, + "auxiliary_loss_mlp": 0.01001709, + "balance_loss_clip": 1.01482844, + "balance_loss_mlp": 0.99992049, + "epoch": 0.5603318703781639, + "flos": 70115945529600.0, + "grad_norm": 0.7681904419180438, + "language_loss": 0.59632474, + "learning_rate": 1.7073818563150026e-06, + "loss": 0.6167832, + "num_input_tokens_seen": 100537830, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01782227, + "step": 4660, + "time_per_iteration": 3.419017791748047 + }, + { + "auxiliary_loss_clip": 0.01139945, + "auxiliary_loss_mlp": 0.01038872, + "balance_loss_clip": 1.04517639, + "balance_loss_mlp": 1.02176571, + "epoch": 0.560452113268803, + "flos": 18545850455040.0, + "grad_norm": 2.8652716450387685, + "language_loss": 0.86783922, + "learning_rate": 1.7066112894525935e-06, + "loss": 0.8896274, + "num_input_tokens_seen": 100555910, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.17089844, + "step": 4661, + "time_per_iteration": 2.950057029724121 + }, + { + "auxiliary_loss_clip": 0.01136572, + "auxiliary_loss_mlp": 0.01042206, + "balance_loss_clip": 1.04393828, + "balance_loss_mlp": 1.02357352, + "epoch": 0.5605723561594421, + "flos": 25264665250560.0, + "grad_norm": 2.2627929026194264, + "language_loss": 0.72565186, + "learning_rate": 1.7058407670970177e-06, + "loss": 0.74743962, + "num_input_tokens_seen": 100577385, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.1862793, + "step": 4662, + "time_per_iteration": 2.865309476852417 + }, + { + "auxiliary_loss_clip": 0.01143225, + "auxiliary_loss_mlp": 0.01047898, + "balance_loss_clip": 1.04638433, + "balance_loss_mlp": 1.02793002, + "epoch": 0.5606925990500812, + "flos": 20594949621120.0, + "grad_norm": 1.6571869349116117, + "language_loss": 0.61460841, + "learning_rate": 1.7050702893651643e-06, + "loss": 0.63651961, + "num_input_tokens_seen": 100596965, + "router_z_loss_clip": 0.96777344, + "router_z_loss_mlp": 0.1998291, + "step": 4663, + "time_per_iteration": 3.7466838359832764 + }, + { + "auxiliary_loss_clip": 0.01144501, + "auxiliary_loss_mlp": 0.01051619, + "balance_loss_clip": 1.04769266, + "balance_loss_mlp": 1.03379726, + "epoch": 0.5608128419407202, + "flos": 35006044677120.0, + "grad_norm": 2.085083593809708, + "language_loss": 0.75950706, + "learning_rate": 1.7042998563739134e-06, + "loss": 0.78146827, + "num_input_tokens_seen": 100615315, + "router_z_loss_clip": 0.96728516, + "router_z_loss_mlp": 0.17822266, + "step": 4664, + "time_per_iteration": 2.9649507999420166 + }, + { + "auxiliary_loss_clip": 0.01144487, + "auxiliary_loss_mlp": 0.0104058, + "balance_loss_clip": 1.04650164, + "balance_loss_mlp": 1.0229249, + "epoch": 0.5609330848313594, + "flos": 24639819235200.0, + "grad_norm": 1.7846516849929022, + "language_loss": 0.71915329, + "learning_rate": 1.703529468240139e-06, + "loss": 0.74100399, + "num_input_tokens_seen": 100634185, + "router_z_loss_clip": 0.97998047, + "router_z_loss_mlp": 0.17651367, + "step": 4665, + "time_per_iteration": 2.8269596099853516 + }, + { + "auxiliary_loss_clip": 0.01141223, + "auxiliary_loss_mlp": 0.01044219, + "balance_loss_clip": 1.04676318, + "balance_loss_mlp": 1.02842391, + "epoch": 0.5610533277219985, + "flos": 18762894385920.0, + "grad_norm": 2.429929883367336, + "language_loss": 0.73551607, + "learning_rate": 1.7027591250807088e-06, + "loss": 0.75737059, + "num_input_tokens_seen": 100651360, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.15795898, + "step": 4666, + "time_per_iteration": 3.738309860229492 + }, + { + "auxiliary_loss_clip": 0.0114373, + "auxiliary_loss_mlp": 0.01039665, + "balance_loss_clip": 1.04758358, + "balance_loss_mlp": 1.02218854, + "epoch": 0.5611735706126375, + "flos": 15012384727680.0, + "grad_norm": 2.5011035846178715, + "language_loss": 0.84634703, + "learning_rate": 1.7019888270124825e-06, + "loss": 0.86818099, + "num_input_tokens_seen": 100668525, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.17468262, + "step": 4667, + "time_per_iteration": 2.784528970718384 + }, + { + "auxiliary_loss_clip": 0.01139828, + "auxiliary_loss_mlp": 0.01046417, + "balance_loss_clip": 1.04733348, + "balance_loss_mlp": 1.02956057, + "epoch": 0.5612938135032767, + "flos": 16468167041280.0, + "grad_norm": 1.7793316605859135, + "language_loss": 0.81942952, + "learning_rate": 1.7012185741523147e-06, + "loss": 0.84129202, + "num_input_tokens_seen": 100684850, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.16870117, + "step": 4668, + "time_per_iteration": 2.745352029800415 + }, + { + "auxiliary_loss_clip": 0.01139191, + "auxiliary_loss_mlp": 0.01052251, + "balance_loss_clip": 1.04646969, + "balance_loss_mlp": 1.03504944, + "epoch": 0.5614140563939157, + "flos": 25666433850240.0, + "grad_norm": 1.857576420453753, + "language_loss": 0.62746024, + "learning_rate": 1.7004483666170514e-06, + "loss": 0.6493746, + "num_input_tokens_seen": 100705345, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.17199707, + "step": 4669, + "time_per_iteration": 3.600411891937256 + }, + { + "auxiliary_loss_clip": 0.01135964, + "auxiliary_loss_mlp": 0.01044022, + "balance_loss_clip": 1.04371619, + "balance_loss_mlp": 1.02695107, + "epoch": 0.5615342992845548, + "flos": 24717566223360.0, + "grad_norm": 1.9427151058249716, + "language_loss": 0.80510753, + "learning_rate": 1.699678204523533e-06, + "loss": 0.8269074, + "num_input_tokens_seen": 100725210, + "router_z_loss_clip": 0.92089844, + "router_z_loss_mlp": 0.1706543, + "step": 4670, + "time_per_iteration": 2.7686071395874023 + }, + { + "auxiliary_loss_clip": 0.01144508, + "auxiliary_loss_mlp": 0.01044317, + "balance_loss_clip": 1.04842758, + "balance_loss_mlp": 1.02701998, + "epoch": 0.5616545421751938, + "flos": 22015934634240.0, + "grad_norm": 2.8189251217250035, + "language_loss": 0.68930739, + "learning_rate": 1.6989080879885918e-06, + "loss": 0.71119565, + "num_input_tokens_seen": 100743070, + "router_z_loss_clip": 0.95996094, + "router_z_loss_mlp": 0.17297363, + "step": 4671, + "time_per_iteration": 3.565725803375244 + }, + { + "auxiliary_loss_clip": 0.01041556, + "auxiliary_loss_mlp": 0.01002827, + "balance_loss_clip": 1.01252139, + "balance_loss_mlp": 1.0012058, + "epoch": 0.561774785065833, + "flos": 53760358690560.0, + "grad_norm": 0.905324320447234, + "language_loss": 0.60992146, + "learning_rate": 1.6981380171290544e-06, + "loss": 0.63036525, + "num_input_tokens_seen": 100804095, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01623535, + "step": 4672, + "time_per_iteration": 3.2616770267486572 + }, + { + "auxiliary_loss_clip": 0.01137157, + "auxiliary_loss_mlp": 0.0104214, + "balance_loss_clip": 1.04418159, + "balance_loss_mlp": 1.02523565, + "epoch": 0.5618950279564721, + "flos": 19750007018880.0, + "grad_norm": 1.743612209556019, + "language_loss": 0.7402699, + "learning_rate": 1.6973679920617396e-06, + "loss": 0.76206291, + "num_input_tokens_seen": 100821630, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.16894531, + "step": 4673, + "time_per_iteration": 2.740492582321167 + }, + { + "auxiliary_loss_clip": 0.01140322, + "auxiliary_loss_mlp": 0.01045469, + "balance_loss_clip": 1.04529381, + "balance_loss_mlp": 1.02743316, + "epoch": 0.5620152708471111, + "flos": 16800592435200.0, + "grad_norm": 1.9515227625513039, + "language_loss": 0.85120332, + "learning_rate": 1.6965980129034603e-06, + "loss": 0.87306118, + "num_input_tokens_seen": 100839015, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.18017578, + "step": 4674, + "time_per_iteration": 2.7524161338806152 + }, + { + "auxiliary_loss_clip": 0.01140568, + "auxiliary_loss_mlp": 0.01047736, + "balance_loss_clip": 1.04769826, + "balance_loss_mlp": 1.03058195, + "epoch": 0.5621355137377503, + "flos": 26797799502720.0, + "grad_norm": 1.561977078459976, + "language_loss": 0.76296246, + "learning_rate": 1.6958280797710209e-06, + "loss": 0.78484553, + "num_input_tokens_seen": 100860940, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.17175293, + "step": 4675, + "time_per_iteration": 2.825059413909912 + }, + { + "auxiliary_loss_clip": 0.01041193, + "auxiliary_loss_mlp": 0.01001882, + "balance_loss_clip": 1.01215172, + "balance_loss_mlp": 1.00022471, + "epoch": 0.5622557566283893, + "flos": 61207046686080.0, + "grad_norm": 0.7112587307666216, + "language_loss": 0.54717481, + "learning_rate": 1.6950581927812198e-06, + "loss": 0.56760556, + "num_input_tokens_seen": 100920510, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01660156, + "step": 4676, + "time_per_iteration": 3.1694819927215576 + }, + { + "auxiliary_loss_clip": 0.01141296, + "auxiliary_loss_mlp": 0.01042072, + "balance_loss_clip": 1.04759884, + "balance_loss_mlp": 1.02535844, + "epoch": 0.5623759995190284, + "flos": 26468534505600.0, + "grad_norm": 2.307962758345171, + "language_loss": 0.78837556, + "learning_rate": 1.6942883520508486e-06, + "loss": 0.81020927, + "num_input_tokens_seen": 100939245, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16699219, + "step": 4677, + "time_per_iteration": 2.8156206607818604 + }, + { + "auxiliary_loss_clip": 0.01137558, + "auxiliary_loss_mlp": 0.01040138, + "balance_loss_clip": 1.04578781, + "balance_loss_mlp": 1.02387238, + "epoch": 0.5624962424096676, + "flos": 19390900798080.0, + "grad_norm": 2.084555665082333, + "language_loss": 0.77320731, + "learning_rate": 1.693518557696691e-06, + "loss": 0.79498422, + "num_input_tokens_seen": 100958385, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.16278076, + "step": 4678, + "time_per_iteration": 2.7998697757720947 + }, + { + "auxiliary_loss_clip": 0.01136522, + "auxiliary_loss_mlp": 0.01043033, + "balance_loss_clip": 1.04452574, + "balance_loss_mlp": 1.02562809, + "epoch": 0.5626164853003066, + "flos": 20667345482880.0, + "grad_norm": 2.0779770643942794, + "language_loss": 0.88876528, + "learning_rate": 1.6927488098355252e-06, + "loss": 0.91056085, + "num_input_tokens_seen": 100976015, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.17407227, + "step": 4679, + "time_per_iteration": 2.755451202392578 + }, + { + "auxiliary_loss_clip": 0.01039935, + "auxiliary_loss_mlp": 0.00999311, + "balance_loss_clip": 1.01083219, + "balance_loss_mlp": 0.99774951, + "epoch": 0.5627367281909457, + "flos": 62766071665920.0, + "grad_norm": 0.893088246998193, + "language_loss": 0.6315434, + "learning_rate": 1.6919791085841201e-06, + "loss": 0.65193594, + "num_input_tokens_seen": 101033425, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.015625, + "step": 4680, + "time_per_iteration": 3.2025604248046875 + }, + { + "auxiliary_loss_clip": 0.01137115, + "auxiliary_loss_mlp": 0.0104812, + "balance_loss_clip": 1.04350209, + "balance_loss_mlp": 1.03052533, + "epoch": 0.5628569710815848, + "flos": 12787144243200.0, + "grad_norm": 2.060920039249858, + "language_loss": 0.78764755, + "learning_rate": 1.6912094540592396e-06, + "loss": 0.80949986, + "num_input_tokens_seen": 101048945, + "router_z_loss_clip": 0.93652344, + "router_z_loss_mlp": 0.17578125, + "step": 4681, + "time_per_iteration": 2.7433676719665527 + }, + { + "auxiliary_loss_clip": 0.01141952, + "auxiliary_loss_mlp": 0.01052012, + "balance_loss_clip": 1.04580247, + "balance_loss_mlp": 1.03397596, + "epoch": 0.5629772139722239, + "flos": 13762082165760.0, + "grad_norm": 2.86057045463064, + "language_loss": 0.81512916, + "learning_rate": 1.6904398463776393e-06, + "loss": 0.8370688, + "num_input_tokens_seen": 101062745, + "router_z_loss_clip": 0.9609375, + "router_z_loss_mlp": 0.18029785, + "step": 4682, + "time_per_iteration": 2.726402521133423 + }, + { + "auxiliary_loss_clip": 0.01136541, + "auxiliary_loss_mlp": 0.01038587, + "balance_loss_clip": 1.04475379, + "balance_loss_mlp": 1.02216053, + "epoch": 0.5630974568628629, + "flos": 21467830026240.0, + "grad_norm": 1.6444566192783543, + "language_loss": 0.72608662, + "learning_rate": 1.6896702856560683e-06, + "loss": 0.7478379, + "num_input_tokens_seen": 101081840, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.16430664, + "step": 4683, + "time_per_iteration": 2.859259843826294 + }, + { + "auxiliary_loss_clip": 0.01138609, + "auxiliary_loss_mlp": 0.01039941, + "balance_loss_clip": 1.04450619, + "balance_loss_mlp": 1.02351427, + "epoch": 0.5632176997535021, + "flos": 14245907385600.0, + "grad_norm": 3.7852188073148256, + "language_loss": 0.69253635, + "learning_rate": 1.6889007720112677e-06, + "loss": 0.71432185, + "num_input_tokens_seen": 101099585, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.16430664, + "step": 4684, + "time_per_iteration": 2.74452543258667 + }, + { + "auxiliary_loss_clip": 0.01141572, + "auxiliary_loss_mlp": 0.0104007, + "balance_loss_clip": 1.04798698, + "balance_loss_mlp": 1.02304721, + "epoch": 0.5633379426441412, + "flos": 20812244947200.0, + "grad_norm": 1.6146594035747706, + "language_loss": 0.77282465, + "learning_rate": 1.6881313055599734e-06, + "loss": 0.79464102, + "num_input_tokens_seen": 101119515, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.17004395, + "step": 4685, + "time_per_iteration": 2.827558755874634 + }, + { + "auxiliary_loss_clip": 0.01145949, + "auxiliary_loss_mlp": 0.01049269, + "balance_loss_clip": 1.0472244, + "balance_loss_mlp": 1.03087473, + "epoch": 0.5634581855347802, + "flos": 22600883617920.0, + "grad_norm": 2.2434987976019087, + "language_loss": 0.82153565, + "learning_rate": 1.6873618864189117e-06, + "loss": 0.8434878, + "num_input_tokens_seen": 101135285, + "router_z_loss_clip": 0.98583984, + "router_z_loss_mlp": 0.18395996, + "step": 4686, + "time_per_iteration": 2.759052038192749 + }, + { + "auxiliary_loss_clip": 0.01138132, + "auxiliary_loss_mlp": 0.01037968, + "balance_loss_clip": 1.04292536, + "balance_loss_mlp": 1.02082562, + "epoch": 0.5635784284254194, + "flos": 21506972872320.0, + "grad_norm": 2.580042832864431, + "language_loss": 0.7767427, + "learning_rate": 1.686592514704803e-06, + "loss": 0.79850364, + "num_input_tokens_seen": 101152680, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.17138672, + "step": 4687, + "time_per_iteration": 2.7182977199554443 + }, + { + "auxiliary_loss_clip": 0.01138559, + "auxiliary_loss_mlp": 0.01038838, + "balance_loss_clip": 1.04612112, + "balance_loss_mlp": 1.02301848, + "epoch": 0.5636986713160584, + "flos": 19827466698240.0, + "grad_norm": 2.6033765420354307, + "language_loss": 0.7047146, + "learning_rate": 1.685823190534361e-06, + "loss": 0.72648853, + "num_input_tokens_seen": 101170920, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.15808105, + "step": 4688, + "time_per_iteration": 2.7298343181610107 + }, + { + "auxiliary_loss_clip": 0.01143177, + "auxiliary_loss_mlp": 0.01037974, + "balance_loss_clip": 1.04582691, + "balance_loss_mlp": 1.02006865, + "epoch": 0.5638189142066975, + "flos": 19792453916160.0, + "grad_norm": 1.7951322302990076, + "language_loss": 0.84351122, + "learning_rate": 1.6850539140242907e-06, + "loss": 0.86532271, + "num_input_tokens_seen": 101190180, + "router_z_loss_clip": 0.97363281, + "router_z_loss_mlp": 0.17919922, + "step": 4689, + "time_per_iteration": 2.7552268505096436 + }, + { + "auxiliary_loss_clip": 0.01138101, + "auxiliary_loss_mlp": 0.01042755, + "balance_loss_clip": 1.04486716, + "balance_loss_mlp": 1.02551699, + "epoch": 0.5639391570973367, + "flos": 22893771116160.0, + "grad_norm": 1.6711692977040096, + "language_loss": 0.82316631, + "learning_rate": 1.684284685291292e-06, + "loss": 0.84497488, + "num_input_tokens_seen": 101211825, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.17236328, + "step": 4690, + "time_per_iteration": 2.8385136127471924 + }, + { + "auxiliary_loss_clip": 0.01138584, + "auxiliary_loss_mlp": 0.01043406, + "balance_loss_clip": 1.04499316, + "balance_loss_mlp": 1.02565575, + "epoch": 0.5640593999879757, + "flos": 23727077712000.0, + "grad_norm": 1.8876874568231101, + "language_loss": 0.81174815, + "learning_rate": 1.683515504452055e-06, + "loss": 0.83356804, + "num_input_tokens_seen": 101229200, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.17749023, + "step": 4691, + "time_per_iteration": 2.7683348655700684 + }, + { + "auxiliary_loss_clip": 0.01138398, + "auxiliary_loss_mlp": 0.01046949, + "balance_loss_clip": 1.04303026, + "balance_loss_mlp": 1.02962768, + "epoch": 0.5641796428786148, + "flos": 22710123855360.0, + "grad_norm": 1.6386371383774707, + "language_loss": 0.66446823, + "learning_rate": 1.6827463716232648e-06, + "loss": 0.68632168, + "num_input_tokens_seen": 101249860, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.17321777, + "step": 4692, + "time_per_iteration": 3.6530444622039795 + }, + { + "auxiliary_loss_clip": 0.01143437, + "auxiliary_loss_mlp": 0.01041318, + "balance_loss_clip": 1.04717648, + "balance_loss_mlp": 1.02460527, + "epoch": 0.5642998857692539, + "flos": 19791987039360.0, + "grad_norm": 1.672530698918591, + "language_loss": 0.75533617, + "learning_rate": 1.6819772869215972e-06, + "loss": 0.77718377, + "num_input_tokens_seen": 101268940, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.16711426, + "step": 4693, + "time_per_iteration": 2.8052539825439453 + }, + { + "auxiliary_loss_clip": 0.0113853, + "auxiliary_loss_mlp": 0.01040688, + "balance_loss_clip": 1.04482937, + "balance_loss_mlp": 1.0240823, + "epoch": 0.564420128659893, + "flos": 23185904428800.0, + "grad_norm": 2.135357519755585, + "language_loss": 0.82085323, + "learning_rate": 1.6812082504637228e-06, + "loss": 0.84264541, + "num_input_tokens_seen": 101290260, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.16625977, + "step": 4694, + "time_per_iteration": 2.7440552711486816 + }, + { + "auxiliary_loss_clip": 0.01137247, + "auxiliary_loss_mlp": 0.01042212, + "balance_loss_clip": 1.04490614, + "balance_loss_mlp": 1.02516556, + "epoch": 0.564540371550532, + "flos": 23258264376960.0, + "grad_norm": 1.4514326663696584, + "language_loss": 0.74374586, + "learning_rate": 1.6804392623663025e-06, + "loss": 0.76554048, + "num_input_tokens_seen": 101311465, + "router_z_loss_clip": 0.92382812, + "router_z_loss_mlp": 0.17059326, + "step": 4695, + "time_per_iteration": 3.6215314865112305 + }, + { + "auxiliary_loss_clip": 0.01133553, + "auxiliary_loss_mlp": 0.01034439, + "balance_loss_clip": 1.04368937, + "balance_loss_mlp": 1.0187633, + "epoch": 0.5646606144411712, + "flos": 25010058672000.0, + "grad_norm": 1.6332019109782692, + "language_loss": 0.78311861, + "learning_rate": 1.6796703227459935e-06, + "loss": 0.80479848, + "num_input_tokens_seen": 101329420, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.15673828, + "step": 4696, + "time_per_iteration": 2.967226505279541 + }, + { + "auxiliary_loss_clip": 0.01138186, + "auxiliary_loss_mlp": 0.01040519, + "balance_loss_clip": 1.04377759, + "balance_loss_mlp": 1.02437854, + "epoch": 0.5647808573318103, + "flos": 36539645806080.0, + "grad_norm": 1.7730338927473794, + "language_loss": 0.76142097, + "learning_rate": 1.6789014317194407e-06, + "loss": 0.78320801, + "num_input_tokens_seen": 101350900, + "router_z_loss_clip": 0.94335938, + "router_z_loss_mlp": 0.16137695, + "step": 4697, + "time_per_iteration": 2.9098174571990967 + }, + { + "auxiliary_loss_clip": 0.0114297, + "auxiliary_loss_mlp": 0.01051575, + "balance_loss_clip": 1.04682803, + "balance_loss_mlp": 1.03378892, + "epoch": 0.5649011002224493, + "flos": 22528451842560.0, + "grad_norm": 2.1446954182704476, + "language_loss": 0.73200828, + "learning_rate": 1.6781325894032853e-06, + "loss": 0.7539537, + "num_input_tokens_seen": 101369860, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.17785645, + "step": 4698, + "time_per_iteration": 3.581192970275879 + }, + { + "auxiliary_loss_clip": 0.011385, + "auxiliary_loss_mlp": 0.01045572, + "balance_loss_clip": 1.04567289, + "balance_loss_mlp": 1.02866864, + "epoch": 0.5650213431130885, + "flos": 18515147304960.0, + "grad_norm": 1.9703027653385266, + "language_loss": 0.918383, + "learning_rate": 1.6773637959141608e-06, + "loss": 0.94022369, + "num_input_tokens_seen": 101386835, + "router_z_loss_clip": 0.92773438, + "router_z_loss_mlp": 0.16918945, + "step": 4699, + "time_per_iteration": 2.826880693435669 + }, + { + "auxiliary_loss_clip": 0.01135677, + "auxiliary_loss_mlp": 0.01044912, + "balance_loss_clip": 1.04596949, + "balance_loss_mlp": 1.02812719, + "epoch": 0.5651415860037275, + "flos": 17526310819200.0, + "grad_norm": 2.0061260732794217, + "language_loss": 0.66227949, + "learning_rate": 1.6765950513686915e-06, + "loss": 0.68408537, + "num_input_tokens_seen": 101404945, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.16784668, + "step": 4700, + "time_per_iteration": 3.7134039402008057 + }, + { + "auxiliary_loss_clip": 0.01139675, + "auxiliary_loss_mlp": 0.01045018, + "balance_loss_clip": 1.04631257, + "balance_loss_mlp": 1.02749443, + "epoch": 0.5652618288943666, + "flos": 25520026014720.0, + "grad_norm": 2.0129701394627775, + "language_loss": 0.76269668, + "learning_rate": 1.675826355883496e-06, + "loss": 0.78454363, + "num_input_tokens_seen": 101424160, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.1751709, + "step": 4701, + "time_per_iteration": 2.8772733211517334 + }, + { + "auxiliary_loss_clip": 0.01138164, + "auxiliary_loss_mlp": 0.01048743, + "balance_loss_clip": 1.04646218, + "balance_loss_mlp": 1.03179133, + "epoch": 0.5653820717850057, + "flos": 19683105937920.0, + "grad_norm": 2.3824863593214287, + "language_loss": 0.79105949, + "learning_rate": 1.6750577095751848e-06, + "loss": 0.81292856, + "num_input_tokens_seen": 101443270, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.16955566, + "step": 4702, + "time_per_iteration": 2.8630330562591553 + }, + { + "auxiliary_loss_clip": 0.01133807, + "auxiliary_loss_mlp": 0.01037586, + "balance_loss_clip": 1.04372299, + "balance_loss_mlp": 1.0215044, + "epoch": 0.5655023146756448, + "flos": 26979722910720.0, + "grad_norm": 1.5989541607835724, + "language_loss": 0.72969997, + "learning_rate": 1.6742891125603605e-06, + "loss": 0.75141394, + "num_input_tokens_seen": 101464175, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.16088867, + "step": 4703, + "time_per_iteration": 2.9409468173980713 + }, + { + "auxiliary_loss_clip": 0.01140358, + "auxiliary_loss_mlp": 0.0104075, + "balance_loss_clip": 1.04564714, + "balance_loss_mlp": 1.02234423, + "epoch": 0.5656225575662839, + "flos": 27669351104640.0, + "grad_norm": 1.8907752613177713, + "language_loss": 0.71865904, + "learning_rate": 1.6735205649556185e-06, + "loss": 0.74047011, + "num_input_tokens_seen": 101484045, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.1842041, + "step": 4704, + "time_per_iteration": 3.00182843208313 + }, + { + "auxiliary_loss_clip": 0.01136838, + "auxiliary_loss_mlp": 0.01051927, + "balance_loss_clip": 1.04336345, + "balance_loss_mlp": 1.034284, + "epoch": 0.5657428004569229, + "flos": 24349732997760.0, + "grad_norm": 1.613972206816568, + "language_loss": 0.8465122, + "learning_rate": 1.6727520668775476e-06, + "loss": 0.86839986, + "num_input_tokens_seen": 101504330, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.17626953, + "step": 4705, + "time_per_iteration": 2.9790115356445312 + }, + { + "auxiliary_loss_clip": 0.01139715, + "auxiliary_loss_mlp": 0.01045355, + "balance_loss_clip": 1.04426169, + "balance_loss_mlp": 1.02778363, + "epoch": 0.5658630433475621, + "flos": 21944041562880.0, + "grad_norm": 1.578015859429524, + "language_loss": 0.75393969, + "learning_rate": 1.6719836184427275e-06, + "loss": 0.77579039, + "num_input_tokens_seen": 101524635, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.17578125, + "step": 4706, + "time_per_iteration": 2.9358975887298584 + }, + { + "auxiliary_loss_clip": 0.01134655, + "auxiliary_loss_mlp": 0.01043622, + "balance_loss_clip": 1.0417788, + "balance_loss_mlp": 1.0272305, + "epoch": 0.5659832862382012, + "flos": 30409012218240.0, + "grad_norm": 1.8188662355274239, + "language_loss": 0.64447343, + "learning_rate": 1.671215219767733e-06, + "loss": 0.66625619, + "num_input_tokens_seen": 101544095, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.16381836, + "step": 4707, + "time_per_iteration": 2.835181713104248 + }, + { + "auxiliary_loss_clip": 0.01139648, + "auxiliary_loss_mlp": 0.01049423, + "balance_loss_clip": 1.04565215, + "balance_loss_mlp": 1.03266203, + "epoch": 0.5661035291288402, + "flos": 13188194570880.0, + "grad_norm": 1.9744224689013887, + "language_loss": 0.75801438, + "learning_rate": 1.670446870969127e-06, + "loss": 0.77990508, + "num_input_tokens_seen": 101561760, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.16760254, + "step": 4708, + "time_per_iteration": 2.818385362625122 + }, + { + "auxiliary_loss_clip": 0.01134059, + "auxiliary_loss_mlp": 0.01047949, + "balance_loss_clip": 1.04211617, + "balance_loss_mlp": 1.0308429, + "epoch": 0.5662237720194794, + "flos": 16143032108160.0, + "grad_norm": 1.9667886062657591, + "language_loss": 0.8024714, + "learning_rate": 1.6696785721634685e-06, + "loss": 0.82429147, + "num_input_tokens_seen": 101576245, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.17114258, + "step": 4709, + "time_per_iteration": 2.750884532928467 + }, + { + "auxiliary_loss_clip": 0.01139771, + "auxiliary_loss_mlp": 0.01041707, + "balance_loss_clip": 1.04577303, + "balance_loss_mlp": 1.0245055, + "epoch": 0.5663440149101184, + "flos": 17676848718720.0, + "grad_norm": 1.8466912493193404, + "language_loss": 0.74039549, + "learning_rate": 1.6689103234673086e-06, + "loss": 0.76221025, + "num_input_tokens_seen": 101594565, + "router_z_loss_clip": 0.93896484, + "router_z_loss_mlp": 0.17199707, + "step": 4710, + "time_per_iteration": 3.2067582607269287 + }, + { + "auxiliary_loss_clip": 0.01142103, + "auxiliary_loss_mlp": 0.01040187, + "balance_loss_clip": 1.04647112, + "balance_loss_mlp": 1.02310443, + "epoch": 0.5664642578007575, + "flos": 23368330627200.0, + "grad_norm": 3.3635952426784597, + "language_loss": 0.77242965, + "learning_rate": 1.668142124997189e-06, + "loss": 0.79425251, + "num_input_tokens_seen": 101614225, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.17077637, + "step": 4711, + "time_per_iteration": 2.8746371269226074 + }, + { + "auxiliary_loss_clip": 0.0103993, + "auxiliary_loss_mlp": 0.01003023, + "balance_loss_clip": 1.0108844, + "balance_loss_mlp": 1.00138974, + "epoch": 0.5665845006913967, + "flos": 65516470945920.0, + "grad_norm": 0.7562369407969652, + "language_loss": 0.59787649, + "learning_rate": 1.6673739768696453e-06, + "loss": 0.61830604, + "num_input_tokens_seen": 101680795, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01635742, + "step": 4712, + "time_per_iteration": 3.3145699501037598 + }, + { + "auxiliary_loss_clip": 0.01137139, + "auxiliary_loss_mlp": 0.01044456, + "balance_loss_clip": 1.0430932, + "balance_loss_mlp": 1.027004, + "epoch": 0.5667047435820357, + "flos": 26140885620480.0, + "grad_norm": 1.724229497602459, + "language_loss": 0.77209198, + "learning_rate": 1.6666058792012052e-06, + "loss": 0.793908, + "num_input_tokens_seen": 101701680, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.17443848, + "step": 4713, + "time_per_iteration": 2.873013734817505 + }, + { + "auxiliary_loss_clip": 0.01039563, + "auxiliary_loss_mlp": 0.0100395, + "balance_loss_clip": 1.01055527, + "balance_loss_mlp": 1.00241208, + "epoch": 0.5668249864726748, + "flos": 71866949725440.0, + "grad_norm": 0.8946734122349705, + "language_loss": 0.68725848, + "learning_rate": 1.6658378321083878e-06, + "loss": 0.7076937, + "num_input_tokens_seen": 101766010, + "router_z_loss_clip": 0.2890625, + "router_z_loss_mlp": 0.01538086, + "step": 4714, + "time_per_iteration": 3.389514684677124 + }, + { + "auxiliary_loss_clip": 0.01133474, + "auxiliary_loss_mlp": 0.01041145, + "balance_loss_clip": 1.0418433, + "balance_loss_mlp": 1.02477765, + "epoch": 0.5669452293633139, + "flos": 22195667312640.0, + "grad_norm": 1.8779299500184643, + "language_loss": 0.82524455, + "learning_rate": 1.6650698357077055e-06, + "loss": 0.84699082, + "num_input_tokens_seen": 101783055, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.16345215, + "step": 4715, + "time_per_iteration": 2.931103229522705 + }, + { + "auxiliary_loss_clip": 0.01137651, + "auxiliary_loss_mlp": 0.01045526, + "balance_loss_clip": 1.04282343, + "balance_loss_mlp": 1.02825284, + "epoch": 0.567065472253953, + "flos": 18223193560320.0, + "grad_norm": 2.9752809871925305, + "language_loss": 0.80946863, + "learning_rate": 1.6643018901156632e-06, + "loss": 0.83130038, + "num_input_tokens_seen": 101802150, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.17285156, + "step": 4716, + "time_per_iteration": 2.9268767833709717 + }, + { + "auxiliary_loss_clip": 0.01136278, + "auxiliary_loss_mlp": 0.01041174, + "balance_loss_clip": 1.04296565, + "balance_loss_mlp": 1.02438939, + "epoch": 0.567185715144592, + "flos": 20371548983040.0, + "grad_norm": 6.6877547607376595, + "language_loss": 0.79088116, + "learning_rate": 1.6635339954487566e-06, + "loss": 0.81265569, + "num_input_tokens_seen": 101818025, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.16772461, + "step": 4717, + "time_per_iteration": 2.7847025394439697 + }, + { + "auxiliary_loss_clip": 0.01141627, + "auxiliary_loss_mlp": 0.01044076, + "balance_loss_clip": 1.04537523, + "balance_loss_mlp": 1.02599227, + "epoch": 0.5673059580352312, + "flos": 23221348174080.0, + "grad_norm": 1.6301056714874702, + "language_loss": 0.81943023, + "learning_rate": 1.6627661518234765e-06, + "loss": 0.84128726, + "num_input_tokens_seen": 101837280, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.18103027, + "step": 4718, + "time_per_iteration": 2.9056499004364014 + }, + { + "auxiliary_loss_clip": 0.01140483, + "auxiliary_loss_mlp": 0.01050366, + "balance_loss_clip": 1.0450356, + "balance_loss_mlp": 1.03341413, + "epoch": 0.5674262009258703, + "flos": 21719599430400.0, + "grad_norm": 1.7588636218255351, + "language_loss": 0.85558951, + "learning_rate": 1.661998359356302e-06, + "loss": 0.87749803, + "num_input_tokens_seen": 101856310, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.16955566, + "step": 4719, + "time_per_iteration": 2.8135828971862793 + }, + { + "auxiliary_loss_clip": 0.01039197, + "auxiliary_loss_mlp": 0.01002083, + "balance_loss_clip": 1.00996327, + "balance_loss_mlp": 1.00037861, + "epoch": 0.5675464438165093, + "flos": 67470369114240.0, + "grad_norm": 0.7483985765047007, + "language_loss": 0.5572691, + "learning_rate": 1.6612306181637077e-06, + "loss": 0.5776819, + "num_input_tokens_seen": 101915635, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.01708984, + "step": 4720, + "time_per_iteration": 3.311418056488037 + }, + { + "auxiliary_loss_clip": 0.01137954, + "auxiliary_loss_mlp": 0.01043622, + "balance_loss_clip": 1.04548097, + "balance_loss_mlp": 1.02746964, + "epoch": 0.5676666867071485, + "flos": 18879173688960.0, + "grad_norm": 2.102452239025963, + "language_loss": 0.6603756, + "learning_rate": 1.6604629283621598e-06, + "loss": 0.68219131, + "num_input_tokens_seen": 101933565, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.16149902, + "step": 4721, + "time_per_iteration": 3.7243592739105225 + }, + { + "auxiliary_loss_clip": 0.0114068, + "auxiliary_loss_mlp": 0.01045571, + "balance_loss_clip": 1.04608035, + "balance_loss_mlp": 1.02703357, + "epoch": 0.5677869295977875, + "flos": 33546778744320.0, + "grad_norm": 5.927015394763545, + "language_loss": 0.74412417, + "learning_rate": 1.6596952900681152e-06, + "loss": 0.76598668, + "num_input_tokens_seen": 101954325, + "router_z_loss_clip": 0.9453125, + "router_z_loss_mlp": 0.18530273, + "step": 4722, + "time_per_iteration": 2.888916015625 + }, + { + "auxiliary_loss_clip": 0.01138453, + "auxiliary_loss_mlp": 0.01041242, + "balance_loss_clip": 1.04499543, + "balance_loss_mlp": 1.0234319, + "epoch": 0.5679071724884266, + "flos": 28037256157440.0, + "grad_norm": 2.0998768906553353, + "language_loss": 0.82307339, + "learning_rate": 1.658927703398025e-06, + "loss": 0.84487033, + "num_input_tokens_seen": 101974390, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.17822266, + "step": 4723, + "time_per_iteration": 3.7532567977905273 + }, + { + "auxiliary_loss_clip": 0.01137016, + "auxiliary_loss_mlp": 0.01041111, + "balance_loss_clip": 1.04383945, + "balance_loss_mlp": 1.02448153, + "epoch": 0.5680274153790658, + "flos": 23550110380800.0, + "grad_norm": 2.2221589890194346, + "language_loss": 0.77492887, + "learning_rate": 1.6581601684683309e-06, + "loss": 0.79671013, + "num_input_tokens_seen": 101994815, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.1661377, + "step": 4724, + "time_per_iteration": 2.8927011489868164 + }, + { + "auxiliary_loss_clip": 0.01139668, + "auxiliary_loss_mlp": 0.0104559, + "balance_loss_clip": 1.04415679, + "balance_loss_mlp": 1.02812564, + "epoch": 0.5681476582697048, + "flos": 22455158140800.0, + "grad_norm": 2.7232908313334425, + "language_loss": 0.68184519, + "learning_rate": 1.6573926853954674e-06, + "loss": 0.7036978, + "num_input_tokens_seen": 102012400, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.17492676, + "step": 4725, + "time_per_iteration": 2.975015878677368 + }, + { + "auxiliary_loss_clip": 0.01131369, + "auxiliary_loss_mlp": 0.01039609, + "balance_loss_clip": 1.04308105, + "balance_loss_mlp": 1.02424264, + "epoch": 0.5682679011603439, + "flos": 19536913584000.0, + "grad_norm": 1.786079433257181, + "language_loss": 0.83169794, + "learning_rate": 1.6566252542958608e-06, + "loss": 0.85340774, + "num_input_tokens_seen": 102031900, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15380859, + "step": 4726, + "time_per_iteration": 3.0136606693267822 + }, + { + "auxiliary_loss_clip": 0.01141392, + "auxiliary_loss_mlp": 0.01049274, + "balance_loss_clip": 1.04581642, + "balance_loss_mlp": 1.03233457, + "epoch": 0.568388144050983, + "flos": 28765488493440.0, + "grad_norm": 2.029375432115838, + "language_loss": 0.78239334, + "learning_rate": 1.6558578752859305e-06, + "loss": 0.80429995, + "num_input_tokens_seen": 102050860, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.16925049, + "step": 4727, + "time_per_iteration": 3.7393157482147217 + }, + { + "auxiliary_loss_clip": 0.01136845, + "auxiliary_loss_mlp": 0.010391, + "balance_loss_clip": 1.04393089, + "balance_loss_mlp": 1.02149248, + "epoch": 0.5685083869416221, + "flos": 21209452519680.0, + "grad_norm": 2.1188612402599807, + "language_loss": 0.78901064, + "learning_rate": 1.6550905484820865e-06, + "loss": 0.81077015, + "num_input_tokens_seen": 102069320, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.17602539, + "step": 4728, + "time_per_iteration": 2.855534076690674 + }, + { + "auxiliary_loss_clip": 0.01139289, + "auxiliary_loss_mlp": 0.01043192, + "balance_loss_clip": 1.04343772, + "balance_loss_mlp": 1.02564454, + "epoch": 0.5686286298322611, + "flos": 24827021942400.0, + "grad_norm": 3.141015650841629, + "language_loss": 0.7896806, + "learning_rate": 1.6543232740007328e-06, + "loss": 0.81150544, + "num_input_tokens_seen": 102086435, + "router_z_loss_clip": 0.95703125, + "router_z_loss_mlp": 0.17541504, + "step": 4729, + "time_per_iteration": 3.6835904121398926 + }, + { + "auxiliary_loss_clip": 0.0113881, + "auxiliary_loss_mlp": 0.01040924, + "balance_loss_clip": 1.04490006, + "balance_loss_mlp": 1.02270913, + "epoch": 0.5687488727229003, + "flos": 26615121909120.0, + "grad_norm": 2.582935832340911, + "language_loss": 0.66795927, + "learning_rate": 1.653556051958263e-06, + "loss": 0.68975657, + "num_input_tokens_seen": 102106115, + "router_z_loss_clip": 0.93798828, + "router_z_loss_mlp": 0.18225098, + "step": 4730, + "time_per_iteration": 2.825136661529541 + }, + { + "auxiliary_loss_clip": 0.01138629, + "auxiliary_loss_mlp": 0.01048178, + "balance_loss_clip": 1.04541755, + "balance_loss_mlp": 1.03179908, + "epoch": 0.5688691156135394, + "flos": 20808725414400.0, + "grad_norm": 1.6995481539865365, + "language_loss": 0.7372843, + "learning_rate": 1.6527888824710642e-06, + "loss": 0.75915235, + "num_input_tokens_seen": 102125715, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.16369629, + "step": 4731, + "time_per_iteration": 2.861140251159668 + }, + { + "auxiliary_loss_clip": 0.01141123, + "auxiliary_loss_mlp": 0.01038076, + "balance_loss_clip": 1.0447638, + "balance_loss_mlp": 1.02044547, + "epoch": 0.5689893585041784, + "flos": 25880963829120.0, + "grad_norm": 2.141958091541528, + "language_loss": 0.77069181, + "learning_rate": 1.6520217656555166e-06, + "loss": 0.79248381, + "num_input_tokens_seen": 102145005, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.17626953, + "step": 4732, + "time_per_iteration": 3.0397849082946777 + }, + { + "auxiliary_loss_clip": 0.01138397, + "auxiliary_loss_mlp": 0.01037166, + "balance_loss_clip": 1.04755974, + "balance_loss_mlp": 1.02144253, + "epoch": 0.5691096013948175, + "flos": 23477463123840.0, + "grad_norm": 1.4645366245772502, + "language_loss": 0.70620441, + "learning_rate": 1.65125470162799e-06, + "loss": 0.72796005, + "num_input_tokens_seen": 102165360, + "router_z_loss_clip": 0.90771484, + "router_z_loss_mlp": 0.1574707, + "step": 4733, + "time_per_iteration": 2.909390449523926 + }, + { + "auxiliary_loss_clip": 0.01139416, + "auxiliary_loss_mlp": 0.01042467, + "balance_loss_clip": 1.04400063, + "balance_loss_mlp": 1.02613544, + "epoch": 0.5692298442854566, + "flos": 18075600576000.0, + "grad_norm": 1.9880732916749624, + "language_loss": 0.69711208, + "learning_rate": 1.6504876905048485e-06, + "loss": 0.71893096, + "num_input_tokens_seen": 102182320, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.16333008, + "step": 4734, + "time_per_iteration": 2.858142137527466 + }, + { + "auxiliary_loss_clip": 0.01134418, + "auxiliary_loss_mlp": 0.01038755, + "balance_loss_clip": 1.04385805, + "balance_loss_mlp": 1.02269745, + "epoch": 0.5693500871760957, + "flos": 23039317025280.0, + "grad_norm": 1.5828013518970074, + "language_loss": 0.72229838, + "learning_rate": 1.6497207324024464e-06, + "loss": 0.74403012, + "num_input_tokens_seen": 102201220, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.16052246, + "step": 4735, + "time_per_iteration": 2.8999297618865967 + }, + { + "auxiliary_loss_clip": 0.0113512, + "auxiliary_loss_mlp": 0.01044415, + "balance_loss_clip": 1.04211617, + "balance_loss_mlp": 1.02690351, + "epoch": 0.5694703300667348, + "flos": 18989670902400.0, + "grad_norm": 1.844605691508268, + "language_loss": 0.8298251, + "learning_rate": 1.6489538274371305e-06, + "loss": 0.85162044, + "num_input_tokens_seen": 102219825, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.1751709, + "step": 4736, + "time_per_iteration": 2.9608030319213867 + }, + { + "auxiliary_loss_clip": 0.01133868, + "auxiliary_loss_mlp": 0.0103689, + "balance_loss_clip": 1.0441308, + "balance_loss_mlp": 1.02086878, + "epoch": 0.5695905729573739, + "flos": 21908705558400.0, + "grad_norm": 2.3033538381289405, + "language_loss": 0.83345789, + "learning_rate": 1.6481869757252396e-06, + "loss": 0.85516548, + "num_input_tokens_seen": 102238160, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.16040039, + "step": 4737, + "time_per_iteration": 2.985391855239868 + }, + { + "auxiliary_loss_clip": 0.01134791, + "auxiliary_loss_mlp": 0.01041366, + "balance_loss_clip": 1.0444808, + "balance_loss_mlp": 1.02533269, + "epoch": 0.569710815848013, + "flos": 28476659232000.0, + "grad_norm": 1.8736763033865935, + "language_loss": 0.71757203, + "learning_rate": 1.647420177383105e-06, + "loss": 0.73933363, + "num_input_tokens_seen": 102261030, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.16015625, + "step": 4738, + "time_per_iteration": 2.8923330307006836 + }, + { + "auxiliary_loss_clip": 0.01133792, + "auxiliary_loss_mlp": 0.01051647, + "balance_loss_clip": 1.04476154, + "balance_loss_mlp": 1.03525603, + "epoch": 0.569831058738652, + "flos": 28366162018560.0, + "grad_norm": 1.871774390314375, + "language_loss": 0.72433591, + "learning_rate": 1.646653432527049e-06, + "loss": 0.74619031, + "num_input_tokens_seen": 102281670, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.1640625, + "step": 4739, + "time_per_iteration": 2.9037160873413086 + }, + { + "auxiliary_loss_clip": 0.01139261, + "auxiliary_loss_mlp": 0.01040464, + "balance_loss_clip": 1.04648471, + "balance_loss_mlp": 1.02220106, + "epoch": 0.5699513016292912, + "flos": 25849973370240.0, + "grad_norm": 1.7975516165116279, + "language_loss": 0.74406993, + "learning_rate": 1.645886741273387e-06, + "loss": 0.76586717, + "num_input_tokens_seen": 102303485, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.18273926, + "step": 4740, + "time_per_iteration": 3.4495022296905518 + }, + { + "auxiliary_loss_clip": 0.01136874, + "auxiliary_loss_mlp": 0.01040249, + "balance_loss_clip": 1.04338765, + "balance_loss_mlp": 1.02340472, + "epoch": 0.5700715445199303, + "flos": 18037858360320.0, + "grad_norm": 1.8229442224972878, + "language_loss": 0.73596078, + "learning_rate": 1.645120103738424e-06, + "loss": 0.75773203, + "num_input_tokens_seen": 102320995, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.16845703, + "step": 4741, + "time_per_iteration": 2.8914568424224854 + }, + { + "auxiliary_loss_clip": 0.01131626, + "auxiliary_loss_mlp": 0.010377, + "balance_loss_clip": 1.04299712, + "balance_loss_mlp": 1.02209508, + "epoch": 0.5701917874105693, + "flos": 11473352392320.0, + "grad_norm": 2.868757736019231, + "language_loss": 0.83902836, + "learning_rate": 1.6443535200384591e-06, + "loss": 0.86072159, + "num_input_tokens_seen": 102339170, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.15625, + "step": 4742, + "time_per_iteration": 2.827578067779541 + }, + { + "auxiliary_loss_clip": 0.01138362, + "auxiliary_loss_mlp": 0.01050479, + "balance_loss_clip": 1.04626274, + "balance_loss_mlp": 1.03235948, + "epoch": 0.5703120303012085, + "flos": 21761759018880.0, + "grad_norm": 1.6496232914177111, + "language_loss": 0.70631588, + "learning_rate": 1.6435869902897827e-06, + "loss": 0.72820431, + "num_input_tokens_seen": 102357750, + "router_z_loss_clip": 0.92089844, + "router_z_loss_mlp": 0.18127441, + "step": 4743, + "time_per_iteration": 2.874056339263916 + }, + { + "auxiliary_loss_clip": 0.01038641, + "auxiliary_loss_mlp": 0.00999388, + "balance_loss_clip": 1.01001441, + "balance_loss_mlp": 0.99788046, + "epoch": 0.5704322731918475, + "flos": 56746258513920.0, + "grad_norm": 0.7944807838571766, + "language_loss": 0.61993492, + "learning_rate": 1.6428205146086764e-06, + "loss": 0.64031518, + "num_input_tokens_seen": 102419730, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01507568, + "step": 4744, + "time_per_iteration": 3.3961145877838135 + }, + { + "auxiliary_loss_clip": 0.01138294, + "auxiliary_loss_mlp": 0.01036604, + "balance_loss_clip": 1.04504752, + "balance_loss_mlp": 1.0193907, + "epoch": 0.5705525160824866, + "flos": 20741141975040.0, + "grad_norm": 1.5273723659011695, + "language_loss": 0.70892489, + "learning_rate": 1.6420540931114142e-06, + "loss": 0.73067385, + "num_input_tokens_seen": 102440320, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.17211914, + "step": 4745, + "time_per_iteration": 2.8108818531036377 + }, + { + "auxiliary_loss_clip": 0.01138039, + "auxiliary_loss_mlp": 0.01048066, + "balance_loss_clip": 1.04333568, + "balance_loss_mlp": 1.03016102, + "epoch": 0.5706727589731257, + "flos": 18771262254720.0, + "grad_norm": 1.6836961929907572, + "language_loss": 0.79335785, + "learning_rate": 1.6412877259142616e-06, + "loss": 0.81521893, + "num_input_tokens_seen": 102460240, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.17907715, + "step": 4746, + "time_per_iteration": 2.861081600189209 + }, + { + "auxiliary_loss_clip": 0.01142962, + "auxiliary_loss_mlp": 0.01045908, + "balance_loss_clip": 1.04628074, + "balance_loss_mlp": 1.02834868, + "epoch": 0.5707930018637648, + "flos": 27634733372160.0, + "grad_norm": 1.8310821790456844, + "language_loss": 0.73766148, + "learning_rate": 1.6405214131334757e-06, + "loss": 0.75955021, + "num_input_tokens_seen": 102478765, + "router_z_loss_clip": 0.96777344, + "router_z_loss_mlp": 0.17578125, + "step": 4747, + "time_per_iteration": 2.7913758754730225 + }, + { + "auxiliary_loss_clip": 0.01134359, + "auxiliary_loss_mlp": 0.01039506, + "balance_loss_clip": 1.0447576, + "balance_loss_mlp": 1.0236156, + "epoch": 0.5709132447544039, + "flos": 27597673514880.0, + "grad_norm": 2.123409739923179, + "language_loss": 0.79375052, + "learning_rate": 1.6397551548853052e-06, + "loss": 0.81548917, + "num_input_tokens_seen": 102496930, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.15869141, + "step": 4748, + "time_per_iteration": 2.796562671661377 + }, + { + "auxiliary_loss_clip": 0.01143862, + "auxiliary_loss_mlp": 0.01048231, + "balance_loss_clip": 1.0468843, + "balance_loss_mlp": 1.0291574, + "epoch": 0.571033487645043, + "flos": 21686095019520.0, + "grad_norm": 2.0326791781460356, + "language_loss": 0.70870805, + "learning_rate": 1.6389889512859917e-06, + "loss": 0.73062897, + "num_input_tokens_seen": 102516590, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.1907959, + "step": 4749, + "time_per_iteration": 2.8185441493988037 + }, + { + "auxiliary_loss_clip": 0.01039028, + "auxiliary_loss_mlp": 0.01000775, + "balance_loss_clip": 1.00999928, + "balance_loss_mlp": 0.99936259, + "epoch": 0.5711537305356821, + "flos": 70181445980160.0, + "grad_norm": 0.8149318102381596, + "language_loss": 0.6034261, + "learning_rate": 1.638222802451767e-06, + "loss": 0.62382412, + "num_input_tokens_seen": 102578070, + "router_z_loss_clip": 0.29101562, + "router_z_loss_mlp": 0.01409912, + "step": 4750, + "time_per_iteration": 4.14683198928833 + }, + { + "auxiliary_loss_clip": 0.01131494, + "auxiliary_loss_mlp": 0.01038144, + "balance_loss_clip": 1.04294562, + "balance_loss_mlp": 1.02202678, + "epoch": 0.5712739734263211, + "flos": 24717494396160.0, + "grad_norm": 1.6451307545012182, + "language_loss": 0.75462633, + "learning_rate": 1.6374567084988561e-06, + "loss": 0.77632266, + "num_input_tokens_seen": 102599255, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16125488, + "step": 4751, + "time_per_iteration": 2.7659196853637695 + }, + { + "auxiliary_loss_clip": 0.01141986, + "auxiliary_loss_mlp": 0.01041777, + "balance_loss_clip": 1.04681277, + "balance_loss_mlp": 1.02407479, + "epoch": 0.5713942163169603, + "flos": 26578169792640.0, + "grad_norm": 1.772876939591605, + "language_loss": 0.76404905, + "learning_rate": 1.6366906695434738e-06, + "loss": 0.7858867, + "num_input_tokens_seen": 102621775, + "router_z_loss_clip": 0.95263672, + "router_z_loss_mlp": 0.17687988, + "step": 4752, + "time_per_iteration": 3.698432207107544 + }, + { + "auxiliary_loss_clip": 0.01135906, + "auxiliary_loss_mlp": 0.01043666, + "balance_loss_clip": 1.04390395, + "balance_loss_mlp": 1.02689362, + "epoch": 0.5715144592075994, + "flos": 21142443697920.0, + "grad_norm": 1.851498148213371, + "language_loss": 0.85842454, + "learning_rate": 1.6359246857018275e-06, + "loss": 0.88022029, + "num_input_tokens_seen": 102639305, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.16772461, + "step": 4753, + "time_per_iteration": 2.766000747680664 + }, + { + "auxiliary_loss_clip": 0.01135927, + "auxiliary_loss_mlp": 0.01039203, + "balance_loss_clip": 1.0438329, + "balance_loss_mlp": 1.02195406, + "epoch": 0.5716347020982384, + "flos": 23330265189120.0, + "grad_norm": 1.9173750482467642, + "language_loss": 0.78063715, + "learning_rate": 1.6351587570901178e-06, + "loss": 0.80238849, + "num_input_tokens_seen": 102659430, + "router_z_loss_clip": 0.92089844, + "router_z_loss_mlp": 0.17260742, + "step": 4754, + "time_per_iteration": 2.7624378204345703 + }, + { + "auxiliary_loss_clip": 0.0113799, + "auxiliary_loss_mlp": 0.01040058, + "balance_loss_clip": 1.04509592, + "balance_loss_mlp": 1.02286792, + "epoch": 0.5717549449888776, + "flos": 17009555806080.0, + "grad_norm": 2.3591176900380035, + "language_loss": 0.76060104, + "learning_rate": 1.634392883824534e-06, + "loss": 0.78238153, + "num_input_tokens_seen": 102671430, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.17175293, + "step": 4755, + "time_per_iteration": 3.5420010089874268 + }, + { + "auxiliary_loss_clip": 0.01140791, + "auxiliary_loss_mlp": 0.0104003, + "balance_loss_clip": 1.04596281, + "balance_loss_mlp": 1.02269709, + "epoch": 0.5718751878795166, + "flos": 35518130922240.0, + "grad_norm": 1.7264255231288277, + "language_loss": 0.67767483, + "learning_rate": 1.6336270660212595e-06, + "loss": 0.69948304, + "num_input_tokens_seen": 102693025, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.17333984, + "step": 4756, + "time_per_iteration": 3.004591226577759 + }, + { + "auxiliary_loss_clip": 0.01138493, + "auxiliary_loss_mlp": 0.01047155, + "balance_loss_clip": 1.04375279, + "balance_loss_mlp": 1.0291549, + "epoch": 0.5719954307701557, + "flos": 38613989255040.0, + "grad_norm": 2.130721081655933, + "language_loss": 0.65637505, + "learning_rate": 1.6328613037964676e-06, + "loss": 0.6782316, + "num_input_tokens_seen": 102716090, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.18005371, + "step": 4757, + "time_per_iteration": 2.9223053455352783 + }, + { + "auxiliary_loss_clip": 0.01132791, + "auxiliary_loss_mlp": 0.01037268, + "balance_loss_clip": 1.04250836, + "balance_loss_mlp": 1.02016139, + "epoch": 0.5721156736607949, + "flos": 20631111638400.0, + "grad_norm": 1.9674686851607677, + "language_loss": 0.68272305, + "learning_rate": 1.6320955972663241e-06, + "loss": 0.70442361, + "num_input_tokens_seen": 102735685, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.17102051, + "step": 4758, + "time_per_iteration": 3.613499879837036 + }, + { + "auxiliary_loss_clip": 0.01136654, + "auxiliary_loss_mlp": 0.01038226, + "balance_loss_clip": 1.04377556, + "balance_loss_mlp": 1.02159667, + "epoch": 0.5722359165514339, + "flos": 37415076076800.0, + "grad_norm": 1.647889961147353, + "language_loss": 0.65510875, + "learning_rate": 1.6313299465469857e-06, + "loss": 0.67685759, + "num_input_tokens_seen": 102758415, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.16638184, + "step": 4759, + "time_per_iteration": 2.8749840259552 + }, + { + "auxiliary_loss_clip": 0.01140538, + "auxiliary_loss_mlp": 0.01047271, + "balance_loss_clip": 1.04547918, + "balance_loss_mlp": 1.02815008, + "epoch": 0.572356159442073, + "flos": 21972877205760.0, + "grad_norm": 2.4649012823090843, + "language_loss": 0.79544842, + "learning_rate": 1.6305643517546014e-06, + "loss": 0.81732649, + "num_input_tokens_seen": 102773795, + "router_z_loss_clip": 0.95068359, + "router_z_loss_mlp": 0.19128418, + "step": 4760, + "time_per_iteration": 2.8120152950286865 + }, + { + "auxiliary_loss_clip": 0.01133155, + "auxiliary_loss_mlp": 0.01041863, + "balance_loss_clip": 1.04440188, + "balance_loss_mlp": 1.02491164, + "epoch": 0.5724764023327121, + "flos": 19135540033920.0, + "grad_norm": 1.9583081609681485, + "language_loss": 0.85083258, + "learning_rate": 1.629798813005311e-06, + "loss": 0.87258279, + "num_input_tokens_seen": 102793515, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.16967773, + "step": 4761, + "time_per_iteration": 2.7458465099334717 + }, + { + "auxiliary_loss_clip": 0.01134094, + "auxiliary_loss_mlp": 0.01044566, + "balance_loss_clip": 1.04307938, + "balance_loss_mlp": 1.02695882, + "epoch": 0.5725966452233512, + "flos": 22819759142400.0, + "grad_norm": 1.8743846153413304, + "language_loss": 0.7090857, + "learning_rate": 1.6290333304152473e-06, + "loss": 0.73087227, + "num_input_tokens_seen": 102813390, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.17626953, + "step": 4762, + "time_per_iteration": 2.769575834274292 + }, + { + "auxiliary_loss_clip": 0.01133704, + "auxiliary_loss_mlp": 0.01043982, + "balance_loss_clip": 1.0431571, + "balance_loss_mlp": 1.02749527, + "epoch": 0.5727168881139902, + "flos": 41496610498560.0, + "grad_norm": 1.74971879466376, + "language_loss": 0.56937027, + "learning_rate": 1.6282679041005314e-06, + "loss": 0.59114707, + "num_input_tokens_seen": 102838980, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.16473389, + "step": 4763, + "time_per_iteration": 2.9179768562316895 + }, + { + "auxiliary_loss_clip": 0.01134169, + "auxiliary_loss_mlp": 0.01034466, + "balance_loss_clip": 1.04076862, + "balance_loss_mlp": 1.01781297, + "epoch": 0.5728371310046293, + "flos": 14647675985280.0, + "grad_norm": 2.2608723392207013, + "language_loss": 0.87738568, + "learning_rate": 1.6275025341772789e-06, + "loss": 0.89907199, + "num_input_tokens_seen": 102855285, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16662598, + "step": 4764, + "time_per_iteration": 2.7506051063537598 + }, + { + "auxiliary_loss_clip": 0.01138243, + "auxiliary_loss_mlp": 0.01048004, + "balance_loss_clip": 1.04301512, + "balance_loss_mlp": 1.0301584, + "epoch": 0.5729573738952685, + "flos": 21506613736320.0, + "grad_norm": 3.045197157304236, + "language_loss": 0.82402813, + "learning_rate": 1.626737220761596e-06, + "loss": 0.84589064, + "num_input_tokens_seen": 102872750, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.17834473, + "step": 4765, + "time_per_iteration": 2.7057831287384033 + }, + { + "auxiliary_loss_clip": 0.01138713, + "auxiliary_loss_mlp": 0.01042795, + "balance_loss_clip": 1.04500008, + "balance_loss_mlp": 1.0263561, + "epoch": 0.5730776167859075, + "flos": 23621680229760.0, + "grad_norm": 2.8702346224194386, + "language_loss": 0.78618646, + "learning_rate": 1.62597196396958e-06, + "loss": 0.80800152, + "num_input_tokens_seen": 102890920, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.16430664, + "step": 4766, + "time_per_iteration": 2.749053955078125 + }, + { + "auxiliary_loss_clip": 0.01135543, + "auxiliary_loss_mlp": 0.01051037, + "balance_loss_clip": 1.04233801, + "balance_loss_mlp": 1.03377557, + "epoch": 0.5731978596765466, + "flos": 25739224761600.0, + "grad_norm": 1.8136619048908513, + "language_loss": 0.85878897, + "learning_rate": 1.6252067639173197e-06, + "loss": 0.88065475, + "num_input_tokens_seen": 102912830, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.17248535, + "step": 4767, + "time_per_iteration": 2.876558542251587 + }, + { + "auxiliary_loss_clip": 0.01135467, + "auxiliary_loss_mlp": 0.01040744, + "balance_loss_clip": 1.04310095, + "balance_loss_mlp": 1.02406645, + "epoch": 0.5733181025671857, + "flos": 26359509749760.0, + "grad_norm": 1.8095381462541509, + "language_loss": 0.69716632, + "learning_rate": 1.6244416207208956e-06, + "loss": 0.7189284, + "num_input_tokens_seen": 102933765, + "router_z_loss_clip": 0.92382812, + "router_z_loss_mlp": 0.16699219, + "step": 4768, + "time_per_iteration": 2.771425485610962 + }, + { + "auxiliary_loss_clip": 0.01139066, + "auxiliary_loss_mlp": 0.01042687, + "balance_loss_clip": 1.04491782, + "balance_loss_mlp": 1.02561665, + "epoch": 0.5734383454578248, + "flos": 29423874833280.0, + "grad_norm": 1.7052748380665688, + "language_loss": 0.73482555, + "learning_rate": 1.6236765344963787e-06, + "loss": 0.75664306, + "num_input_tokens_seen": 102955025, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.1706543, + "step": 4769, + "time_per_iteration": 2.840270757675171 + }, + { + "auxiliary_loss_clip": 0.01137064, + "auxiliary_loss_mlp": 0.01045607, + "balance_loss_clip": 1.04294276, + "balance_loss_mlp": 1.02895308, + "epoch": 0.5735585883484638, + "flos": 34969954487040.0, + "grad_norm": 1.9730046400600718, + "language_loss": 0.69152653, + "learning_rate": 1.6229115053598322e-06, + "loss": 0.71335322, + "num_input_tokens_seen": 102976780, + "router_z_loss_clip": 0.94238281, + "router_z_loss_mlp": 0.16638184, + "step": 4770, + "time_per_iteration": 2.8958470821380615 + }, + { + "auxiliary_loss_clip": 0.01140009, + "auxiliary_loss_mlp": 0.01043034, + "balance_loss_clip": 1.046363, + "balance_loss_mlp": 1.02616632, + "epoch": 0.573678831239103, + "flos": 18770759464320.0, + "grad_norm": 1.8296566593617654, + "language_loss": 0.72192752, + "learning_rate": 1.6221465334273108e-06, + "loss": 0.74375796, + "num_input_tokens_seen": 102995990, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.16882324, + "step": 4771, + "time_per_iteration": 2.7865231037139893 + }, + { + "auxiliary_loss_clip": 0.01136205, + "auxiliary_loss_mlp": 0.01047193, + "balance_loss_clip": 1.04268813, + "balance_loss_mlp": 1.02947855, + "epoch": 0.5737990741297421, + "flos": 25702883176320.0, + "grad_norm": 1.9964559743725188, + "language_loss": 0.61654878, + "learning_rate": 1.6213816188148593e-06, + "loss": 0.63838273, + "num_input_tokens_seen": 103014695, + "router_z_loss_clip": 0.93505859, + "router_z_loss_mlp": 0.17712402, + "step": 4772, + "time_per_iteration": 2.7856507301330566 + }, + { + "auxiliary_loss_clip": 0.0113256, + "auxiliary_loss_mlp": 0.0103836, + "balance_loss_clip": 1.04408479, + "balance_loss_mlp": 1.02348256, + "epoch": 0.5739193170203811, + "flos": 27269234530560.0, + "grad_norm": 1.882887589383934, + "language_loss": 0.77322578, + "learning_rate": 1.6206167616385162e-06, + "loss": 0.79493499, + "num_input_tokens_seen": 103035760, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.14874268, + "step": 4773, + "time_per_iteration": 2.910708427429199 + }, + { + "auxiliary_loss_clip": 0.01142063, + "auxiliary_loss_mlp": 0.01043501, + "balance_loss_clip": 1.04665124, + "balance_loss_mlp": 1.02619171, + "epoch": 0.5740395599110203, + "flos": 12239721993600.0, + "grad_norm": 1.8736553570556724, + "language_loss": 0.73440707, + "learning_rate": 1.6198519620143078e-06, + "loss": 0.75626272, + "num_input_tokens_seen": 103052915, + "router_z_loss_clip": 0.95458984, + "router_z_loss_mlp": 0.17297363, + "step": 4774, + "time_per_iteration": 2.794790267944336 + }, + { + "auxiliary_loss_clip": 0.01137286, + "auxiliary_loss_mlp": 0.01048779, + "balance_loss_clip": 1.04609919, + "balance_loss_mlp": 1.03179193, + "epoch": 0.5741598028016593, + "flos": 25921399564800.0, + "grad_norm": 1.4783145842700334, + "language_loss": 0.77966899, + "learning_rate": 1.6190872200582546e-06, + "loss": 0.80152965, + "num_input_tokens_seen": 103074655, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.1697998, + "step": 4775, + "time_per_iteration": 2.8238768577575684 + }, + { + "auxiliary_loss_clip": 0.01132815, + "auxiliary_loss_mlp": 0.01044125, + "balance_loss_clip": 1.04247546, + "balance_loss_mlp": 1.02672005, + "epoch": 0.5742800456922984, + "flos": 19244133826560.0, + "grad_norm": 2.2193381646462025, + "language_loss": 0.78379124, + "learning_rate": 1.6183225358863676e-06, + "loss": 0.80556059, + "num_input_tokens_seen": 103091550, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.1739502, + "step": 4776, + "time_per_iteration": 2.8263070583343506 + }, + { + "auxiliary_loss_clip": 0.01136836, + "auxiliary_loss_mlp": 0.0104061, + "balance_loss_clip": 1.04321182, + "balance_loss_mlp": 1.02386141, + "epoch": 0.5744002885829376, + "flos": 30920487932160.0, + "grad_norm": 2.283914044322851, + "language_loss": 0.71571225, + "learning_rate": 1.617557909614648e-06, + "loss": 0.73748672, + "num_input_tokens_seen": 103110985, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.16748047, + "step": 4777, + "time_per_iteration": 2.9250850677490234 + }, + { + "auxiliary_loss_clip": 0.01133947, + "auxiliary_loss_mlp": 0.01041106, + "balance_loss_clip": 1.04265642, + "balance_loss_mlp": 1.02512026, + "epoch": 0.5745205314735766, + "flos": 23840017050240.0, + "grad_norm": 1.8145643789596775, + "language_loss": 0.85932845, + "learning_rate": 1.6167933413590899e-06, + "loss": 0.88107896, + "num_input_tokens_seen": 103129890, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.15979004, + "step": 4778, + "time_per_iteration": 3.7941834926605225 + }, + { + "auxiliary_loss_clip": 0.01136877, + "auxiliary_loss_mlp": 0.01044318, + "balance_loss_clip": 1.04576588, + "balance_loss_mlp": 1.02756941, + "epoch": 0.5746407743642157, + "flos": 12311902373760.0, + "grad_norm": 2.5245462955486806, + "language_loss": 0.90381289, + "learning_rate": 1.6160288312356773e-06, + "loss": 0.92562479, + "num_input_tokens_seen": 103147020, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.16748047, + "step": 4779, + "time_per_iteration": 3.0669403076171875 + }, + { + "auxiliary_loss_clip": 0.01135988, + "auxiliary_loss_mlp": 0.0103984, + "balance_loss_clip": 1.04356718, + "balance_loss_mlp": 1.02304399, + "epoch": 0.5747610172548548, + "flos": 24133658734080.0, + "grad_norm": 1.5301392081338334, + "language_loss": 0.8165347, + "learning_rate": 1.6152643793603857e-06, + "loss": 0.83829296, + "num_input_tokens_seen": 103167370, + "router_z_loss_clip": 0.92431641, + "router_z_loss_mlp": 0.16809082, + "step": 4780, + "time_per_iteration": 2.8331377506256104 + }, + { + "auxiliary_loss_clip": 0.01137898, + "auxiliary_loss_mlp": 0.01043381, + "balance_loss_clip": 1.04641438, + "balance_loss_mlp": 1.02663207, + "epoch": 0.5748812601454939, + "flos": 25408451393280.0, + "grad_norm": 1.7612120088888177, + "language_loss": 0.87984329, + "learning_rate": 1.6144999858491815e-06, + "loss": 0.90165615, + "num_input_tokens_seen": 103186000, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.16760254, + "step": 4781, + "time_per_iteration": 3.787085771560669 + }, + { + "auxiliary_loss_clip": 0.01139462, + "auxiliary_loss_mlp": 0.01043067, + "balance_loss_clip": 1.04464948, + "balance_loss_mlp": 1.02559125, + "epoch": 0.575001503036133, + "flos": 30624942827520.0, + "grad_norm": 1.7118589317621458, + "language_loss": 0.8575837, + "learning_rate": 1.6137356508180232e-06, + "loss": 0.87940896, + "num_input_tokens_seen": 103207710, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.17480469, + "step": 4782, + "time_per_iteration": 2.8847391605377197 + }, + { + "auxiliary_loss_clip": 0.01138277, + "auxiliary_loss_mlp": 0.01041016, + "balance_loss_clip": 1.04468882, + "balance_loss_mlp": 1.02368331, + "epoch": 0.5751217459267721, + "flos": 21726566668800.0, + "grad_norm": 1.6726358307022253, + "language_loss": 0.81457144, + "learning_rate": 1.6129713743828593e-06, + "loss": 0.83636439, + "num_input_tokens_seen": 103226720, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.17333984, + "step": 4783, + "time_per_iteration": 2.8092079162597656 + }, + { + "auxiliary_loss_clip": 0.011343, + "auxiliary_loss_mlp": 0.01047664, + "balance_loss_clip": 1.04426384, + "balance_loss_mlp": 1.03080797, + "epoch": 0.5752419888174112, + "flos": 21651620941440.0, + "grad_norm": 1.3743288257558102, + "language_loss": 0.75436866, + "learning_rate": 1.6122071566596306e-06, + "loss": 0.77618831, + "num_input_tokens_seen": 103246995, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.16845703, + "step": 4784, + "time_per_iteration": 3.614638090133667 + }, + { + "auxiliary_loss_clip": 0.01138545, + "auxiliary_loss_mlp": 0.01034636, + "balance_loss_clip": 1.04445505, + "balance_loss_mlp": 1.01836419, + "epoch": 0.5753622317080502, + "flos": 17775997234560.0, + "grad_norm": 2.202566585706195, + "language_loss": 0.83329439, + "learning_rate": 1.6114429977642674e-06, + "loss": 0.85502625, + "num_input_tokens_seen": 103261500, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.1628418, + "step": 4785, + "time_per_iteration": 2.7795443534851074 + }, + { + "auxiliary_loss_clip": 0.01139321, + "auxiliary_loss_mlp": 0.01048934, + "balance_loss_clip": 1.04680586, + "balance_loss_mlp": 1.03265023, + "epoch": 0.5754824745986894, + "flos": 19789616741760.0, + "grad_norm": 1.665176538446171, + "language_loss": 0.73921716, + "learning_rate": 1.6106788978126926e-06, + "loss": 0.7610997, + "num_input_tokens_seen": 103280475, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.16271973, + "step": 4786, + "time_per_iteration": 2.8286921977996826 + }, + { + "auxiliary_loss_clip": 0.01137024, + "auxiliary_loss_mlp": 0.01039906, + "balance_loss_clip": 1.04326069, + "balance_loss_mlp": 1.02275229, + "epoch": 0.5756027174893285, + "flos": 30985665160320.0, + "grad_norm": 2.553902385156381, + "language_loss": 0.79018557, + "learning_rate": 1.6099148569208196e-06, + "loss": 0.81195486, + "num_input_tokens_seen": 103297695, + "router_z_loss_clip": 0.93847656, + "router_z_loss_mlp": 0.17138672, + "step": 4787, + "time_per_iteration": 3.7959160804748535 + }, + { + "auxiliary_loss_clip": 0.01145329, + "auxiliary_loss_mlp": 0.01036728, + "balance_loss_clip": 1.04870093, + "balance_loss_mlp": 1.0184052, + "epoch": 0.5757229603799675, + "flos": 28546864364160.0, + "grad_norm": 1.8215901370786627, + "language_loss": 0.6339556, + "learning_rate": 1.6091508752045523e-06, + "loss": 0.65577614, + "num_input_tokens_seen": 103318575, + "router_z_loss_clip": 0.96533203, + "router_z_loss_mlp": 0.1831665, + "step": 4788, + "time_per_iteration": 2.7952723503112793 + }, + { + "auxiliary_loss_clip": 0.01135067, + "auxiliary_loss_mlp": 0.01046004, + "balance_loss_clip": 1.04373479, + "balance_loss_mlp": 1.02925491, + "epoch": 0.5758432032706067, + "flos": 22999024944000.0, + "grad_norm": 1.566101485478083, + "language_loss": 0.86428714, + "learning_rate": 1.608386952779787e-06, + "loss": 0.88609785, + "num_input_tokens_seen": 103337945, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.16772461, + "step": 4789, + "time_per_iteration": 2.787581205368042 + }, + { + "auxiliary_loss_clip": 0.01134115, + "auxiliary_loss_mlp": 0.01034984, + "balance_loss_clip": 1.0434866, + "balance_loss_mlp": 1.01723385, + "epoch": 0.5759634461612457, + "flos": 25739727552000.0, + "grad_norm": 3.579300658642808, + "language_loss": 0.74467933, + "learning_rate": 1.6076230897624098e-06, + "loss": 0.76637036, + "num_input_tokens_seen": 103360150, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.17749023, + "step": 4790, + "time_per_iteration": 2.766087770462036 + }, + { + "auxiliary_loss_clip": 0.01139712, + "auxiliary_loss_mlp": 0.01040827, + "balance_loss_clip": 1.04605103, + "balance_loss_mlp": 1.02377963, + "epoch": 0.5760836890518848, + "flos": 30591761639040.0, + "grad_norm": 1.8907558059528178, + "language_loss": 0.78358138, + "learning_rate": 1.6068592862682974e-06, + "loss": 0.80538678, + "num_input_tokens_seen": 103378305, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.17041016, + "step": 4791, + "time_per_iteration": 2.855712890625 + }, + { + "auxiliary_loss_clip": 0.01137315, + "auxiliary_loss_mlp": 0.01044782, + "balance_loss_clip": 1.04429018, + "balance_loss_mlp": 1.02744913, + "epoch": 0.576203931942524, + "flos": 36538963447680.0, + "grad_norm": 1.8918922899043784, + "language_loss": 0.73379803, + "learning_rate": 1.6060955424133187e-06, + "loss": 0.75561899, + "num_input_tokens_seen": 103399230, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.17346191, + "step": 4792, + "time_per_iteration": 2.945664882659912 + }, + { + "auxiliary_loss_clip": 0.01139317, + "auxiliary_loss_mlp": 0.01039567, + "balance_loss_clip": 1.04499316, + "balance_loss_mlp": 1.02279425, + "epoch": 0.576324174833163, + "flos": 25516937445120.0, + "grad_norm": 1.7746595459653896, + "language_loss": 0.89757907, + "learning_rate": 1.6053318583133332e-06, + "loss": 0.91936797, + "num_input_tokens_seen": 103420100, + "router_z_loss_clip": 0.94238281, + "router_z_loss_mlp": 0.16772461, + "step": 4793, + "time_per_iteration": 2.7781143188476562 + }, + { + "auxiliary_loss_clip": 0.01135583, + "auxiliary_loss_mlp": 0.01038801, + "balance_loss_clip": 1.0449121, + "balance_loss_mlp": 1.02118158, + "epoch": 0.5764444177238021, + "flos": 25119262995840.0, + "grad_norm": 5.306992310646581, + "language_loss": 0.746858, + "learning_rate": 1.6045682340841907e-06, + "loss": 0.76860183, + "num_input_tokens_seen": 103439025, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.17626953, + "step": 4794, + "time_per_iteration": 2.828756332397461 + }, + { + "auxiliary_loss_clip": 0.0103937, + "auxiliary_loss_mlp": 0.01005839, + "balance_loss_clip": 1.01066351, + "balance_loss_mlp": 1.00437832, + "epoch": 0.5765646606144411, + "flos": 62212687758720.0, + "grad_norm": 0.7551483907979669, + "language_loss": 0.58007598, + "learning_rate": 1.6038046698417336e-06, + "loss": 0.600528, + "num_input_tokens_seen": 103499920, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.0145874, + "step": 4795, + "time_per_iteration": 3.31396222114563 + }, + { + "auxiliary_loss_clip": 0.01136369, + "auxiliary_loss_mlp": 0.01039733, + "balance_loss_clip": 1.04392481, + "balance_loss_mlp": 1.02312756, + "epoch": 0.5766849035050803, + "flos": 25118760205440.0, + "grad_norm": 2.501491134617885, + "language_loss": 0.68888116, + "learning_rate": 1.6030411657017919e-06, + "loss": 0.71064216, + "num_input_tokens_seen": 103519575, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.16601562, + "step": 4796, + "time_per_iteration": 2.8836560249328613 + }, + { + "auxiliary_loss_clip": 0.01137112, + "auxiliary_loss_mlp": 0.01039384, + "balance_loss_clip": 1.04520202, + "balance_loss_mlp": 1.0231061, + "epoch": 0.5768051463957193, + "flos": 15991093578240.0, + "grad_norm": 2.9308263755693154, + "language_loss": 0.84687006, + "learning_rate": 1.6022777217801903e-06, + "loss": 0.868635, + "num_input_tokens_seen": 103536530, + "router_z_loss_clip": 0.91943359, + "router_z_loss_mlp": 0.16278076, + "step": 4797, + "time_per_iteration": 2.822007417678833 + }, + { + "auxiliary_loss_clip": 0.01142006, + "auxiliary_loss_mlp": 0.0104339, + "balance_loss_clip": 1.0480752, + "balance_loss_mlp": 1.02577078, + "epoch": 0.5769253892863584, + "flos": 22163635359360.0, + "grad_norm": 7.225329460692968, + "language_loss": 0.73512316, + "learning_rate": 1.601514338192742e-06, + "loss": 0.75697708, + "num_input_tokens_seen": 103556460, + "router_z_loss_clip": 0.93945312, + "router_z_loss_mlp": 0.17602539, + "step": 4798, + "time_per_iteration": 2.852173089981079 + }, + { + "auxiliary_loss_clip": 0.01131615, + "auxiliary_loss_mlp": 0.01033988, + "balance_loss_clip": 1.04296184, + "balance_loss_mlp": 1.01839566, + "epoch": 0.5770456321769976, + "flos": 22856388036480.0, + "grad_norm": 2.576356276604563, + "language_loss": 0.71778142, + "learning_rate": 1.6007510150552514e-06, + "loss": 0.73943746, + "num_input_tokens_seen": 103574520, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.15588379, + "step": 4799, + "time_per_iteration": 2.7919530868530273 + }, + { + "auxiliary_loss_clip": 0.01143159, + "auxiliary_loss_mlp": 0.01053311, + "balance_loss_clip": 1.04689932, + "balance_loss_mlp": 1.03566861, + "epoch": 0.5771658750676366, + "flos": 46353672489600.0, + "grad_norm": 1.5065072989407335, + "language_loss": 0.61975324, + "learning_rate": 1.599987752483515e-06, + "loss": 0.64171797, + "num_input_tokens_seen": 103598965, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.17651367, + "step": 4800, + "time_per_iteration": 2.9804253578186035 + }, + { + "auxiliary_loss_clip": 0.01137508, + "auxiliary_loss_mlp": 0.01041403, + "balance_loss_clip": 1.04422379, + "balance_loss_mlp": 1.02461219, + "epoch": 0.5772861179582757, + "flos": 22159972172160.0, + "grad_norm": 1.6749983992515571, + "language_loss": 0.67981672, + "learning_rate": 1.5992245505933184e-06, + "loss": 0.7016058, + "num_input_tokens_seen": 103618665, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.16778564, + "step": 4801, + "time_per_iteration": 2.82232403755188 + }, + { + "auxiliary_loss_clip": 0.01137851, + "auxiliary_loss_mlp": 0.01033809, + "balance_loss_clip": 1.04588366, + "balance_loss_mlp": 1.01742983, + "epoch": 0.5774063608489148, + "flos": 31248926916480.0, + "grad_norm": 2.0363779446809156, + "language_loss": 0.70754743, + "learning_rate": 1.5984614095004388e-06, + "loss": 0.72926408, + "num_input_tokens_seen": 103639800, + "router_z_loss_clip": 0.91943359, + "router_z_loss_mlp": 0.16394043, + "step": 4802, + "time_per_iteration": 2.858146905899048 + }, + { + "auxiliary_loss_clip": 0.01141446, + "auxiliary_loss_mlp": 0.01044205, + "balance_loss_clip": 1.04636502, + "balance_loss_mlp": 1.02732539, + "epoch": 0.5775266037395539, + "flos": 22527123039360.0, + "grad_norm": 3.043749935901093, + "language_loss": 0.80542231, + "learning_rate": 1.5976983293206438e-06, + "loss": 0.82727879, + "num_input_tokens_seen": 103655605, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.16882324, + "step": 4803, + "time_per_iteration": 2.8922688961029053 + }, + { + "auxiliary_loss_clip": 0.01135871, + "auxiliary_loss_mlp": 0.01034992, + "balance_loss_clip": 1.04527116, + "balance_loss_mlp": 1.01762331, + "epoch": 0.577646846630193, + "flos": 21068790860160.0, + "grad_norm": 1.629950778769214, + "language_loss": 0.71306503, + "learning_rate": 1.5969353101696928e-06, + "loss": 0.73477364, + "num_input_tokens_seen": 103674045, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.1739502, + "step": 4804, + "time_per_iteration": 2.8181886672973633 + }, + { + "auxiliary_loss_clip": 0.01133419, + "auxiliary_loss_mlp": 0.01035425, + "balance_loss_clip": 1.04524922, + "balance_loss_mlp": 1.01955175, + "epoch": 0.5777670895208321, + "flos": 29714284293120.0, + "grad_norm": 1.8821361136011108, + "language_loss": 0.79747874, + "learning_rate": 1.5961723521633341e-06, + "loss": 0.8191672, + "num_input_tokens_seen": 103695285, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15875244, + "step": 4805, + "time_per_iteration": 2.818580389022827 + }, + { + "auxiliary_loss_clip": 0.01137105, + "auxiliary_loss_mlp": 0.01041307, + "balance_loss_clip": 1.04517913, + "balance_loss_mlp": 1.02496386, + "epoch": 0.5778873324114712, + "flos": 19500428344320.0, + "grad_norm": 2.443088962321616, + "language_loss": 0.9132905, + "learning_rate": 1.5954094554173097e-06, + "loss": 0.93507469, + "num_input_tokens_seen": 103713275, + "router_z_loss_clip": 0.91943359, + "router_z_loss_mlp": 0.16345215, + "step": 4806, + "time_per_iteration": 2.775017261505127 + }, + { + "auxiliary_loss_clip": 0.01137109, + "auxiliary_loss_mlp": 0.01039398, + "balance_loss_clip": 1.04615068, + "balance_loss_mlp": 1.02243495, + "epoch": 0.5780075753021102, + "flos": 14136846716160.0, + "grad_norm": 2.050745145835001, + "language_loss": 0.79441524, + "learning_rate": 1.5946466200473482e-06, + "loss": 0.81618035, + "num_input_tokens_seen": 103731185, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.16955566, + "step": 4807, + "time_per_iteration": 3.5495033264160156 + }, + { + "auxiliary_loss_clip": 0.01136397, + "auxiliary_loss_mlp": 0.01040095, + "balance_loss_clip": 1.04531336, + "balance_loss_mlp": 1.02365649, + "epoch": 0.5781278181927494, + "flos": 15262178883840.0, + "grad_norm": 1.7313493316698958, + "language_loss": 0.83504915, + "learning_rate": 1.5938838461691723e-06, + "loss": 0.85681415, + "num_input_tokens_seen": 103748095, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.16430664, + "step": 4808, + "time_per_iteration": 2.811674118041992 + }, + { + "auxiliary_loss_clip": 0.0114019, + "auxiliary_loss_mlp": 0.01043905, + "balance_loss_clip": 1.04759908, + "balance_loss_mlp": 1.02592802, + "epoch": 0.5782480610833884, + "flos": 16726831856640.0, + "grad_norm": 2.269892121304707, + "language_loss": 0.83195424, + "learning_rate": 1.593121133898494e-06, + "loss": 0.85379523, + "num_input_tokens_seen": 103765300, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.1796875, + "step": 4809, + "time_per_iteration": 2.7832326889038086 + }, + { + "auxiliary_loss_clip": 0.01140251, + "auxiliary_loss_mlp": 0.01040262, + "balance_loss_clip": 1.04632211, + "balance_loss_mlp": 1.02395391, + "epoch": 0.5783683039740275, + "flos": 25482140144640.0, + "grad_norm": 2.3667626051202815, + "language_loss": 0.79711068, + "learning_rate": 1.592358483351016e-06, + "loss": 0.81891578, + "num_input_tokens_seen": 103785475, + "router_z_loss_clip": 0.93896484, + "router_z_loss_mlp": 0.16308594, + "step": 4810, + "time_per_iteration": 3.6890993118286133 + }, + { + "auxiliary_loss_clip": 0.01132933, + "auxiliary_loss_mlp": 0.01041877, + "balance_loss_clip": 1.04343486, + "balance_loss_mlp": 1.02641571, + "epoch": 0.5784885468646667, + "flos": 18405835240320.0, + "grad_norm": 2.2313697518578612, + "language_loss": 0.72458076, + "learning_rate": 1.5915958946424326e-06, + "loss": 0.74632883, + "num_input_tokens_seen": 103804160, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.15466309, + "step": 4811, + "time_per_iteration": 2.8367135524749756 + }, + { + "auxiliary_loss_clip": 0.01138581, + "auxiliary_loss_mlp": 0.01046791, + "balance_loss_clip": 1.04423153, + "balance_loss_mlp": 1.0293622, + "epoch": 0.5786087897553057, + "flos": 46100717936640.0, + "grad_norm": 1.5582628754046226, + "language_loss": 0.74249065, + "learning_rate": 1.5908333678884271e-06, + "loss": 0.76434433, + "num_input_tokens_seen": 103830580, + "router_z_loss_clip": 0.94335938, + "router_z_loss_mlp": 0.17431641, + "step": 4812, + "time_per_iteration": 2.9520769119262695 + }, + { + "auxiliary_loss_clip": 0.01135497, + "auxiliary_loss_mlp": 0.01040887, + "balance_loss_clip": 1.04523182, + "balance_loss_mlp": 1.02307689, + "epoch": 0.5787290326459448, + "flos": 12385950261120.0, + "grad_norm": 2.2100665609541243, + "language_loss": 0.73809469, + "learning_rate": 1.5900709032046743e-06, + "loss": 0.75985855, + "num_input_tokens_seen": 103848655, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.17834473, + "step": 4813, + "time_per_iteration": 3.7391300201416016 + }, + { + "auxiliary_loss_clip": 0.01138172, + "auxiliary_loss_mlp": 0.01044189, + "balance_loss_clip": 1.04759574, + "balance_loss_mlp": 1.02807188, + "epoch": 0.5788492755365839, + "flos": 23290332243840.0, + "grad_norm": 1.9598121121118766, + "language_loss": 0.78301197, + "learning_rate": 1.5893085007068391e-06, + "loss": 0.80483562, + "num_input_tokens_seen": 103866215, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.16113281, + "step": 4814, + "time_per_iteration": 2.936100482940674 + }, + { + "auxiliary_loss_clip": 0.01137778, + "auxiliary_loss_mlp": 0.01043296, + "balance_loss_clip": 1.04616523, + "balance_loss_mlp": 1.02721465, + "epoch": 0.578969518427223, + "flos": 24061047390720.0, + "grad_norm": 1.8894772964947915, + "language_loss": 0.70811111, + "learning_rate": 1.5885461605105786e-06, + "loss": 0.72992182, + "num_input_tokens_seen": 103887815, + "router_z_loss_clip": 0.91650391, + "router_z_loss_mlp": 0.1607666, + "step": 4815, + "time_per_iteration": 2.791208028793335 + }, + { + "auxiliary_loss_clip": 0.01138457, + "auxiliary_loss_mlp": 0.01043696, + "balance_loss_clip": 1.0450604, + "balance_loss_mlp": 1.02626789, + "epoch": 0.579089761317862, + "flos": 21871825269120.0, + "grad_norm": 1.800509575201947, + "language_loss": 0.7659502, + "learning_rate": 1.5877838827315375e-06, + "loss": 0.78777182, + "num_input_tokens_seen": 103906360, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.17431641, + "step": 4816, + "time_per_iteration": 3.637126922607422 + }, + { + "auxiliary_loss_clip": 0.01140544, + "auxiliary_loss_mlp": 0.01038857, + "balance_loss_clip": 1.04757726, + "balance_loss_mlp": 1.02247763, + "epoch": 0.5792100042085012, + "flos": 22929681738240.0, + "grad_norm": 1.7970163472884044, + "language_loss": 0.70273393, + "learning_rate": 1.587021667485355e-06, + "loss": 0.72452796, + "num_input_tokens_seen": 103925730, + "router_z_loss_clip": 0.93017578, + "router_z_loss_mlp": 0.16381836, + "step": 4817, + "time_per_iteration": 2.8925061225891113 + }, + { + "auxiliary_loss_clip": 0.01137947, + "auxiliary_loss_mlp": 0.01040239, + "balance_loss_clip": 1.04556262, + "balance_loss_mlp": 1.02482474, + "epoch": 0.5793302470991403, + "flos": 21470056669440.0, + "grad_norm": 1.634903644868507, + "language_loss": 0.7839247, + "learning_rate": 1.5862595148876559e-06, + "loss": 0.80570662, + "num_input_tokens_seen": 103945835, + "router_z_loss_clip": 0.92431641, + "router_z_loss_mlp": 0.15411377, + "step": 4818, + "time_per_iteration": 2.8744094371795654 + }, + { + "auxiliary_loss_clip": 0.0113876, + "auxiliary_loss_mlp": 0.01048315, + "balance_loss_clip": 1.0455333, + "balance_loss_mlp": 1.03071928, + "epoch": 0.5794504899897793, + "flos": 12711013367040.0, + "grad_norm": 2.185447808681897, + "language_loss": 0.76224899, + "learning_rate": 1.58549742505406e-06, + "loss": 0.78411973, + "num_input_tokens_seen": 103960580, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.17602539, + "step": 4819, + "time_per_iteration": 2.8773977756500244 + }, + { + "auxiliary_loss_clip": 0.0113796, + "auxiliary_loss_mlp": 0.01041472, + "balance_loss_clip": 1.04549265, + "balance_loss_mlp": 1.02487803, + "epoch": 0.5795707328804185, + "flos": 14867054300160.0, + "grad_norm": 2.1180306042168353, + "language_loss": 0.7534703, + "learning_rate": 1.5847353981001747e-06, + "loss": 0.77526462, + "num_input_tokens_seen": 103977760, + "router_z_loss_clip": 0.92529297, + "router_z_loss_mlp": 0.16589355, + "step": 4820, + "time_per_iteration": 2.781712770462036 + }, + { + "auxiliary_loss_clip": 0.01134559, + "auxiliary_loss_mlp": 0.01043589, + "balance_loss_clip": 1.04336917, + "balance_loss_mlp": 1.02684021, + "epoch": 0.5796909757710575, + "flos": 36430046432640.0, + "grad_norm": 1.6641700155211088, + "language_loss": 0.70197326, + "learning_rate": 1.5839734341415993e-06, + "loss": 0.72375476, + "num_input_tokens_seen": 103999960, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.16748047, + "step": 4821, + "time_per_iteration": 2.9392220973968506 + }, + { + "auxiliary_loss_clip": 0.01133111, + "auxiliary_loss_mlp": 0.01041512, + "balance_loss_clip": 1.04587328, + "balance_loss_mlp": 1.02679014, + "epoch": 0.5798112186616966, + "flos": 23039891642880.0, + "grad_norm": 1.6549568737983469, + "language_loss": 0.7640726, + "learning_rate": 1.5832115332939238e-06, + "loss": 0.78581882, + "num_input_tokens_seen": 104018400, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.1473999, + "step": 4822, + "time_per_iteration": 2.8063929080963135 + }, + { + "auxiliary_loss_clip": 0.01134847, + "auxiliary_loss_mlp": 0.01047736, + "balance_loss_clip": 1.0450207, + "balance_loss_mlp": 1.03079677, + "epoch": 0.5799314615523358, + "flos": 16652604401280.0, + "grad_norm": 1.609360143557301, + "language_loss": 0.74588168, + "learning_rate": 1.5824496956727272e-06, + "loss": 0.76770753, + "num_input_tokens_seen": 104035605, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.16943359, + "step": 4823, + "time_per_iteration": 2.800855875015259 + }, + { + "auxiliary_loss_clip": 0.01131756, + "auxiliary_loss_mlp": 0.01039475, + "balance_loss_clip": 1.0435791, + "balance_loss_mlp": 1.02387047, + "epoch": 0.5800517044429748, + "flos": 20485673470080.0, + "grad_norm": 1.6563526909255943, + "language_loss": 0.73460567, + "learning_rate": 1.5816879213935797e-06, + "loss": 0.75631797, + "num_input_tokens_seen": 104054415, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.15612793, + "step": 4824, + "time_per_iteration": 2.7644078731536865 + }, + { + "auxiliary_loss_clip": 0.01134129, + "auxiliary_loss_mlp": 0.01040966, + "balance_loss_clip": 1.04544091, + "balance_loss_mlp": 1.02602863, + "epoch": 0.5801719473336139, + "flos": 31538258968320.0, + "grad_norm": 1.5515374465631728, + "language_loss": 0.79791504, + "learning_rate": 1.5809262105720416e-06, + "loss": 0.81966597, + "num_input_tokens_seen": 104075455, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.14953613, + "step": 4825, + "time_per_iteration": 2.8404836654663086 + }, + { + "auxiliary_loss_clip": 0.0113159, + "auxiliary_loss_mlp": 0.01040814, + "balance_loss_clip": 1.04213119, + "balance_loss_mlp": 1.02544808, + "epoch": 0.580292190224253, + "flos": 20375966355840.0, + "grad_norm": 1.477794455850753, + "language_loss": 0.79619795, + "learning_rate": 1.5801645633236644e-06, + "loss": 0.817922, + "num_input_tokens_seen": 104096440, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.15380859, + "step": 4826, + "time_per_iteration": 2.7909464836120605 + }, + { + "auxiliary_loss_clip": 0.01132755, + "auxiliary_loss_mlp": 0.01043826, + "balance_loss_clip": 1.04373598, + "balance_loss_mlp": 1.02669549, + "epoch": 0.5804124331148921, + "flos": 26615373304320.0, + "grad_norm": 2.177741101142878, + "language_loss": 0.77417171, + "learning_rate": 1.579402979763989e-06, + "loss": 0.79593754, + "num_input_tokens_seen": 104116775, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.17114258, + "step": 4827, + "time_per_iteration": 2.7878360748291016 + }, + { + "auxiliary_loss_clip": 0.01141546, + "auxiliary_loss_mlp": 0.01038277, + "balance_loss_clip": 1.04678226, + "balance_loss_mlp": 1.02206469, + "epoch": 0.5805326760055312, + "flos": 13478496289920.0, + "grad_norm": 2.570547424225575, + "language_loss": 0.81298602, + "learning_rate": 1.578641460008548e-06, + "loss": 0.83478421, + "num_input_tokens_seen": 104134510, + "router_z_loss_clip": 0.94775391, + "router_z_loss_mlp": 0.16210938, + "step": 4828, + "time_per_iteration": 2.75469708442688 + }, + { + "auxiliary_loss_clip": 0.01130879, + "auxiliary_loss_mlp": 0.01031763, + "balance_loss_clip": 1.04245448, + "balance_loss_mlp": 1.01705313, + "epoch": 0.5806529188961702, + "flos": 12091374823680.0, + "grad_norm": 1.8851998360718982, + "language_loss": 0.67919362, + "learning_rate": 1.5778800041728613e-06, + "loss": 0.70081997, + "num_input_tokens_seen": 104150800, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.14709473, + "step": 4829, + "time_per_iteration": 2.793870449066162 + }, + { + "auxiliary_loss_clip": 0.01131987, + "auxiliary_loss_mlp": 0.0104303, + "balance_loss_clip": 1.04334188, + "balance_loss_mlp": 1.02622128, + "epoch": 0.5807731617868094, + "flos": 26214107495040.0, + "grad_norm": 1.3674305342914013, + "language_loss": 0.66038203, + "learning_rate": 1.577118612372443e-06, + "loss": 0.68213218, + "num_input_tokens_seen": 104172640, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.16790771, + "step": 4830, + "time_per_iteration": 2.8183932304382324 + }, + { + "auxiliary_loss_clip": 0.01138674, + "auxiliary_loss_mlp": 0.01038563, + "balance_loss_clip": 1.04551756, + "balance_loss_mlp": 1.0212543, + "epoch": 0.5808934046774484, + "flos": 37962139190400.0, + "grad_norm": 1.7073240385767834, + "language_loss": 0.70290852, + "learning_rate": 1.5763572847227943e-06, + "loss": 0.7246809, + "num_input_tokens_seen": 104193525, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.17297363, + "step": 4831, + "time_per_iteration": 2.890962600708008 + }, + { + "auxiliary_loss_clip": 0.01131702, + "auxiliary_loss_mlp": 0.01040466, + "balance_loss_clip": 1.04287624, + "balance_loss_mlp": 1.02411008, + "epoch": 0.5810136475680875, + "flos": 20485853038080.0, + "grad_norm": 1.8559498004615957, + "language_loss": 0.81669199, + "learning_rate": 1.5755960213394091e-06, + "loss": 0.83841366, + "num_input_tokens_seen": 104210625, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.16345215, + "step": 4832, + "time_per_iteration": 2.8665950298309326 + }, + { + "auxiliary_loss_clip": 0.01133872, + "auxiliary_loss_mlp": 0.0103875, + "balance_loss_clip": 1.04452538, + "balance_loss_mlp": 1.0234437, + "epoch": 0.5811338904587267, + "flos": 17530153574400.0, + "grad_norm": 1.805110018205744, + "language_loss": 0.78321111, + "learning_rate": 1.5748348223377703e-06, + "loss": 0.80493736, + "num_input_tokens_seen": 104228180, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.15313721, + "step": 4833, + "time_per_iteration": 2.870999813079834 + }, + { + "auxiliary_loss_clip": 0.01132888, + "auxiliary_loss_mlp": 0.01044575, + "balance_loss_clip": 1.04422188, + "balance_loss_mlp": 1.02837431, + "epoch": 0.5812541333493657, + "flos": 19458017360640.0, + "grad_norm": 1.6282895845808958, + "language_loss": 0.77932942, + "learning_rate": 1.5740736878333507e-06, + "loss": 0.80110407, + "num_input_tokens_seen": 104246020, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.16204834, + "step": 4834, + "time_per_iteration": 2.8644180297851562 + }, + { + "auxiliary_loss_clip": 0.01140124, + "auxiliary_loss_mlp": 0.01045044, + "balance_loss_clip": 1.04643559, + "balance_loss_mlp": 1.0278182, + "epoch": 0.5813743762400048, + "flos": 20594949621120.0, + "grad_norm": 2.444030243041237, + "language_loss": 0.78002131, + "learning_rate": 1.5733126179416143e-06, + "loss": 0.80187297, + "num_input_tokens_seen": 104260505, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.17211914, + "step": 4835, + "time_per_iteration": 2.8189327716827393 + }, + { + "auxiliary_loss_clip": 0.0113731, + "auxiliary_loss_mlp": 0.01046398, + "balance_loss_clip": 1.04568219, + "balance_loss_mlp": 1.02902913, + "epoch": 0.5814946191306439, + "flos": 33178227246720.0, + "grad_norm": 2.0832954785912485, + "language_loss": 0.72739345, + "learning_rate": 1.5725516127780137e-06, + "loss": 0.7492305, + "num_input_tokens_seen": 104282640, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.17358398, + "step": 4836, + "time_per_iteration": 3.790411949157715 + }, + { + "auxiliary_loss_clip": 0.01143875, + "auxiliary_loss_mlp": 0.01040995, + "balance_loss_clip": 1.04750502, + "balance_loss_mlp": 1.0235908, + "epoch": 0.581614862021283, + "flos": 16143283503360.0, + "grad_norm": 2.0684166953229384, + "language_loss": 0.88365984, + "learning_rate": 1.5717906724579943e-06, + "loss": 0.90550852, + "num_input_tokens_seen": 104299700, + "router_z_loss_clip": 0.96435547, + "router_z_loss_mlp": 0.1739502, + "step": 4837, + "time_per_iteration": 2.794466733932495 + }, + { + "auxiliary_loss_clip": 0.01139202, + "auxiliary_loss_mlp": 0.01041735, + "balance_loss_clip": 1.04745984, + "balance_loss_mlp": 1.0261184, + "epoch": 0.581735104911922, + "flos": 33802642298880.0, + "grad_norm": 2.318130578922023, + "language_loss": 0.67534769, + "learning_rate": 1.571029797096989e-06, + "loss": 0.69715697, + "num_input_tokens_seen": 104320805, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.15612793, + "step": 4838, + "time_per_iteration": 2.908766508102417 + }, + { + "auxiliary_loss_clip": 0.01132761, + "auxiliary_loss_mlp": 0.01037074, + "balance_loss_clip": 1.04289091, + "balance_loss_mlp": 1.02201819, + "epoch": 0.5818553478025612, + "flos": 23331163029120.0, + "grad_norm": 1.7955088740042662, + "language_loss": 0.78752184, + "learning_rate": 1.570268986810423e-06, + "loss": 0.80922019, + "num_input_tokens_seen": 104340700, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.15075684, + "step": 4839, + "time_per_iteration": 3.6965107917785645 + }, + { + "auxiliary_loss_clip": 0.01132792, + "auxiliary_loss_mlp": 0.01037275, + "balance_loss_clip": 1.04398787, + "balance_loss_mlp": 1.02209938, + "epoch": 0.5819755906932003, + "flos": 20996143603200.0, + "grad_norm": 1.786114846375465, + "language_loss": 0.74977124, + "learning_rate": 1.5695082417137096e-06, + "loss": 0.77147186, + "num_input_tokens_seen": 104358575, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.1517334, + "step": 4840, + "time_per_iteration": 2.810420274734497 + }, + { + "auxiliary_loss_clip": 0.01136314, + "auxiliary_loss_mlp": 0.0104523, + "balance_loss_clip": 1.04608023, + "balance_loss_mlp": 1.02883911, + "epoch": 0.5820958335838393, + "flos": 21431668008960.0, + "grad_norm": 1.7394562445688364, + "language_loss": 0.75190073, + "learning_rate": 1.5687475619222539e-06, + "loss": 0.77371615, + "num_input_tokens_seen": 104378530, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.16394043, + "step": 4841, + "time_per_iteration": 2.8138763904571533 + }, + { + "auxiliary_loss_clip": 0.01137874, + "auxiliary_loss_mlp": 0.01037307, + "balance_loss_clip": 1.0461657, + "balance_loss_mlp": 1.02090991, + "epoch": 0.5822160764744785, + "flos": 17967473660160.0, + "grad_norm": 2.5008155694232674, + "language_loss": 0.73140991, + "learning_rate": 1.5679869475514496e-06, + "loss": 0.75316173, + "num_input_tokens_seen": 104395465, + "router_z_loss_clip": 0.91650391, + "router_z_loss_mlp": 0.16400146, + "step": 4842, + "time_per_iteration": 3.6463887691497803 + }, + { + "auxiliary_loss_clip": 0.01137307, + "auxiliary_loss_mlp": 0.01036131, + "balance_loss_clip": 1.04388976, + "balance_loss_mlp": 1.01948977, + "epoch": 0.5823363193651175, + "flos": 23033858158080.0, + "grad_norm": 2.373042947350222, + "language_loss": 0.80963588, + "learning_rate": 1.567226398716682e-06, + "loss": 0.83137023, + "num_input_tokens_seen": 104415380, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.16650391, + "step": 4843, + "time_per_iteration": 2.791374444961548 + }, + { + "auxiliary_loss_clip": 0.01137025, + "auxiliary_loss_mlp": 0.01045333, + "balance_loss_clip": 1.04210949, + "balance_loss_mlp": 1.02810717, + "epoch": 0.5824565622557566, + "flos": 32891840110080.0, + "grad_norm": 1.666169527646356, + "language_loss": 0.61773014, + "learning_rate": 1.566465915533326e-06, + "loss": 0.63955367, + "num_input_tokens_seen": 104437410, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.17248535, + "step": 4844, + "time_per_iteration": 2.873623847961426 + }, + { + "auxiliary_loss_clip": 0.01134891, + "auxiliary_loss_mlp": 0.01037938, + "balance_loss_clip": 1.04486895, + "balance_loss_mlp": 1.02112961, + "epoch": 0.5825768051463958, + "flos": 22229674513920.0, + "grad_norm": 1.821180617696229, + "language_loss": 0.88403606, + "learning_rate": 1.5657054981167458e-06, + "loss": 0.9057644, + "num_input_tokens_seen": 104456305, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.16809082, + "step": 4845, + "time_per_iteration": 3.6917383670806885 + }, + { + "auxiliary_loss_clip": 0.01130612, + "auxiliary_loss_mlp": 0.01040682, + "balance_loss_clip": 1.04400301, + "balance_loss_mlp": 1.02578044, + "epoch": 0.5826970480370348, + "flos": 28001561016960.0, + "grad_norm": 1.7678216000091913, + "language_loss": 0.67654896, + "learning_rate": 1.5649451465822965e-06, + "loss": 0.69826192, + "num_input_tokens_seen": 104477695, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.14880371, + "step": 4846, + "time_per_iteration": 2.9253084659576416 + }, + { + "auxiliary_loss_clip": 0.01135337, + "auxiliary_loss_mlp": 0.01039839, + "balance_loss_clip": 1.04607582, + "balance_loss_mlp": 1.02471089, + "epoch": 0.5828172909276739, + "flos": 17858053854720.0, + "grad_norm": 1.6451871409300387, + "language_loss": 0.83740449, + "learning_rate": 1.5641848610453218e-06, + "loss": 0.85915625, + "num_input_tokens_seen": 104496355, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.15112305, + "step": 4847, + "time_per_iteration": 2.7780659198760986 + }, + { + "auxiliary_loss_clip": 0.01135151, + "auxiliary_loss_mlp": 0.01038523, + "balance_loss_clip": 1.04350543, + "balance_loss_mlp": 1.02208412, + "epoch": 0.582937533818313, + "flos": 19865244827520.0, + "grad_norm": 1.924756014277764, + "language_loss": 0.85861921, + "learning_rate": 1.563424641621158e-06, + "loss": 0.88035595, + "num_input_tokens_seen": 104515535, + "router_z_loss_clip": 0.91552734, + "router_z_loss_mlp": 0.16430664, + "step": 4848, + "time_per_iteration": 2.815993547439575 + }, + { + "auxiliary_loss_clip": 0.01137699, + "auxiliary_loss_mlp": 0.01044386, + "balance_loss_clip": 1.0463202, + "balance_loss_mlp": 1.02800608, + "epoch": 0.5830577767089521, + "flos": 26870734068480.0, + "grad_norm": 2.230375696404139, + "language_loss": 0.70347041, + "learning_rate": 1.5626644884251282e-06, + "loss": 0.72529119, + "num_input_tokens_seen": 104535055, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.16369629, + "step": 4849, + "time_per_iteration": 2.8281891345977783 + }, + { + "auxiliary_loss_clip": 0.01130364, + "auxiliary_loss_mlp": 0.01035458, + "balance_loss_clip": 1.04215646, + "balance_loss_mlp": 1.0197227, + "epoch": 0.5831780195995911, + "flos": 25298205575040.0, + "grad_norm": 1.5756175400660786, + "language_loss": 0.88244534, + "learning_rate": 1.5619044015725488e-06, + "loss": 0.90410358, + "num_input_tokens_seen": 104554745, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15734863, + "step": 4850, + "time_per_iteration": 2.8140645027160645 + }, + { + "auxiliary_loss_clip": 0.01143786, + "auxiliary_loss_mlp": 0.01045548, + "balance_loss_clip": 1.04814577, + "balance_loss_mlp": 1.02909684, + "epoch": 0.5832982624902303, + "flos": 14756988049920.0, + "grad_norm": 2.338707397589051, + "language_loss": 0.86983573, + "learning_rate": 1.5611443811787224e-06, + "loss": 0.89172906, + "num_input_tokens_seen": 104568870, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.16442871, + "step": 4851, + "time_per_iteration": 2.744896173477173 + }, + { + "auxiliary_loss_clip": 0.01133157, + "auxiliary_loss_mlp": 0.01043531, + "balance_loss_clip": 1.04462552, + "balance_loss_mlp": 1.02854061, + "epoch": 0.5834185053808694, + "flos": 20444555376000.0, + "grad_norm": 2.1021807888060784, + "language_loss": 0.69040537, + "learning_rate": 1.560384427358945e-06, + "loss": 0.71217227, + "num_input_tokens_seen": 104588415, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.14990234, + "step": 4852, + "time_per_iteration": 2.8151562213897705 + }, + { + "auxiliary_loss_clip": 0.01136494, + "auxiliary_loss_mlp": 0.01038844, + "balance_loss_clip": 1.04364634, + "balance_loss_mlp": 1.02241731, + "epoch": 0.5835387482715084, + "flos": 27200394115200.0, + "grad_norm": 1.5114851714831474, + "language_loss": 0.73250771, + "learning_rate": 1.5596245402284998e-06, + "loss": 0.75426108, + "num_input_tokens_seen": 104611940, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.16430664, + "step": 4853, + "time_per_iteration": 2.8425076007843018 + }, + { + "auxiliary_loss_clip": 0.01137683, + "auxiliary_loss_mlp": 0.0104671, + "balance_loss_clip": 1.04612327, + "balance_loss_mlp": 1.03134346, + "epoch": 0.5836589911621476, + "flos": 16654615562880.0, + "grad_norm": 1.6448948929138436, + "language_loss": 0.82127613, + "learning_rate": 1.5588647199026619e-06, + "loss": 0.8431201, + "num_input_tokens_seen": 104629675, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.15356445, + "step": 4854, + "time_per_iteration": 2.774468421936035 + }, + { + "auxiliary_loss_clip": 0.01139134, + "auxiliary_loss_mlp": 0.01041844, + "balance_loss_clip": 1.04656148, + "balance_loss_mlp": 1.02587032, + "epoch": 0.5837792340527866, + "flos": 20446817932800.0, + "grad_norm": 2.224630303842542, + "language_loss": 0.87650478, + "learning_rate": 1.5581049664966956e-06, + "loss": 0.8983146, + "num_input_tokens_seen": 104647435, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.15979004, + "step": 4855, + "time_per_iteration": 2.8056859970092773 + }, + { + "auxiliary_loss_clip": 0.0103896, + "auxiliary_loss_mlp": 0.01003796, + "balance_loss_clip": 1.01025581, + "balance_loss_mlp": 1.00222254, + "epoch": 0.5838994769434257, + "flos": 65995480765440.0, + "grad_norm": 1.0087749784792202, + "language_loss": 0.65067303, + "learning_rate": 1.5573452801258545e-06, + "loss": 0.67110056, + "num_input_tokens_seen": 104694605, + "router_z_loss_clip": 0.28710938, + "router_z_loss_mlp": 0.01574707, + "step": 4856, + "time_per_iteration": 3.0843307971954346 + }, + { + "auxiliary_loss_clip": 0.01139783, + "auxiliary_loss_mlp": 0.01043138, + "balance_loss_clip": 1.04576027, + "balance_loss_mlp": 1.02644312, + "epoch": 0.5840197198340649, + "flos": 21470523546240.0, + "grad_norm": 1.8265510187178138, + "language_loss": 0.63811243, + "learning_rate": 1.5565856609053824e-06, + "loss": 0.65994161, + "num_input_tokens_seen": 104713400, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.16680908, + "step": 4857, + "time_per_iteration": 2.866022825241089 + }, + { + "auxiliary_loss_clip": 0.01137886, + "auxiliary_loss_mlp": 0.01039713, + "balance_loss_clip": 1.04615164, + "balance_loss_mlp": 1.02286279, + "epoch": 0.5841399627247039, + "flos": 19135144984320.0, + "grad_norm": 1.8063106237692415, + "language_loss": 0.80009413, + "learning_rate": 1.5558261089505127e-06, + "loss": 0.82187015, + "num_input_tokens_seen": 104732130, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.16864014, + "step": 4858, + "time_per_iteration": 2.9102723598480225 + }, + { + "auxiliary_loss_clip": 0.01139026, + "auxiliary_loss_mlp": 0.01042414, + "balance_loss_clip": 1.04646218, + "balance_loss_mlp": 1.02566457, + "epoch": 0.584260205615343, + "flos": 26425692558720.0, + "grad_norm": 1.8571836132176904, + "language_loss": 0.8009662, + "learning_rate": 1.5550666243764697e-06, + "loss": 0.82278061, + "num_input_tokens_seen": 104750290, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.16748047, + "step": 4859, + "time_per_iteration": 2.8135182857513428 + }, + { + "auxiliary_loss_clip": 0.01137522, + "auxiliary_loss_mlp": 0.01036111, + "balance_loss_clip": 1.04436982, + "balance_loss_mlp": 1.02005327, + "epoch": 0.584380448505982, + "flos": 13881809174400.0, + "grad_norm": 2.174356195053295, + "language_loss": 0.76903939, + "learning_rate": 1.554307207298465e-06, + "loss": 0.79077578, + "num_input_tokens_seen": 104768550, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.16064453, + "step": 4860, + "time_per_iteration": 2.866809606552124 + }, + { + "auxiliary_loss_clip": 0.01139732, + "auxiliary_loss_mlp": 0.01040184, + "balance_loss_clip": 1.04608095, + "balance_loss_mlp": 1.02415609, + "epoch": 0.5845006913966212, + "flos": 21543709507200.0, + "grad_norm": 1.780657876066803, + "language_loss": 0.78654933, + "learning_rate": 1.553547857831704e-06, + "loss": 0.80834854, + "num_input_tokens_seen": 104785060, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.16027832, + "step": 4861, + "time_per_iteration": 2.869387149810791 + }, + { + "auxiliary_loss_clip": 0.01037869, + "auxiliary_loss_mlp": 0.0100349, + "balance_loss_clip": 1.0094943, + "balance_loss_mlp": 1.00177372, + "epoch": 0.5846209342872603, + "flos": 58375452712320.0, + "grad_norm": 0.8935075748080147, + "language_loss": 0.64221942, + "learning_rate": 1.5527885760913771e-06, + "loss": 0.66263306, + "num_input_tokens_seen": 104834950, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01721191, + "step": 4862, + "time_per_iteration": 3.1202259063720703 + }, + { + "auxiliary_loss_clip": 0.01132566, + "auxiliary_loss_mlp": 0.01032552, + "balance_loss_clip": 1.04556298, + "balance_loss_mlp": 1.01776981, + "epoch": 0.5847411771778993, + "flos": 18588045957120.0, + "grad_norm": 1.5922054714942333, + "language_loss": 0.76534826, + "learning_rate": 1.552029362192668e-06, + "loss": 0.78699946, + "num_input_tokens_seen": 104854210, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.14782715, + "step": 4863, + "time_per_iteration": 2.744220018386841 + }, + { + "auxiliary_loss_clip": 0.0113635, + "auxiliary_loss_mlp": 0.01043253, + "balance_loss_clip": 1.04389405, + "balance_loss_mlp": 1.0263015, + "epoch": 0.5848614200685385, + "flos": 24240780069120.0, + "grad_norm": 2.2371672464220658, + "language_loss": 0.72563648, + "learning_rate": 1.5512702162507478e-06, + "loss": 0.74743247, + "num_input_tokens_seen": 104874525, + "router_z_loss_clip": 0.92431641, + "router_z_loss_mlp": 0.16955566, + "step": 4864, + "time_per_iteration": 2.7688441276550293 + }, + { + "auxiliary_loss_clip": 0.01037276, + "auxiliary_loss_mlp": 0.01000565, + "balance_loss_clip": 1.00874281, + "balance_loss_mlp": 0.99895537, + "epoch": 0.5849816629591775, + "flos": 71660245933440.0, + "grad_norm": 1.137262159122269, + "language_loss": 0.55774045, + "learning_rate": 1.5505111383807792e-06, + "loss": 0.5781188, + "num_input_tokens_seen": 104937195, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01611328, + "step": 4865, + "time_per_iteration": 4.170589923858643 + }, + { + "auxiliary_loss_clip": 0.01138101, + "auxiliary_loss_mlp": 0.01038702, + "balance_loss_clip": 1.0458504, + "balance_loss_mlp": 1.02242959, + "epoch": 0.5851019058498166, + "flos": 23802095266560.0, + "grad_norm": 1.6752244917243153, + "language_loss": 0.80828249, + "learning_rate": 1.5497521286979138e-06, + "loss": 0.83005047, + "num_input_tokens_seen": 104957435, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.16271973, + "step": 4866, + "time_per_iteration": 2.788407564163208 + }, + { + "auxiliary_loss_clip": 0.01139112, + "auxiliary_loss_mlp": 0.01042626, + "balance_loss_clip": 1.0434804, + "balance_loss_mlp": 1.02453005, + "epoch": 0.5852221487404557, + "flos": 24388516707840.0, + "grad_norm": 1.8430266516388933, + "language_loss": 0.74331963, + "learning_rate": 1.5489931873172927e-06, + "loss": 0.76513696, + "num_input_tokens_seen": 104978755, + "router_z_loss_clip": 0.95556641, + "router_z_loss_mlp": 0.1809082, + "step": 4867, + "time_per_iteration": 2.804291248321533 + }, + { + "auxiliary_loss_clip": 0.01133584, + "auxiliary_loss_mlp": 0.01038096, + "balance_loss_clip": 1.04376841, + "balance_loss_mlp": 1.02144289, + "epoch": 0.5853423916310948, + "flos": 27271425260160.0, + "grad_norm": 1.7761504446262444, + "language_loss": 0.79120064, + "learning_rate": 1.5482343143540467e-06, + "loss": 0.81291747, + "num_input_tokens_seen": 105000020, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.16650391, + "step": 4868, + "time_per_iteration": 3.7284255027770996 + }, + { + "auxiliary_loss_clip": 0.01136543, + "auxiliary_loss_mlp": 0.01035546, + "balance_loss_clip": 1.0441556, + "balance_loss_mlp": 1.01926172, + "epoch": 0.5854626345217339, + "flos": 11983786611840.0, + "grad_norm": 1.7748554615431997, + "language_loss": 0.82542747, + "learning_rate": 1.547475509923295e-06, + "loss": 0.84714842, + "num_input_tokens_seen": 105017060, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.16271973, + "step": 4869, + "time_per_iteration": 2.782388925552368 + }, + { + "auxiliary_loss_clip": 0.01036621, + "auxiliary_loss_mlp": 0.00999515, + "balance_loss_clip": 1.00811386, + "balance_loss_mlp": 0.99781054, + "epoch": 0.585582877412373, + "flos": 64342335173760.0, + "grad_norm": 0.7249315684061067, + "language_loss": 0.55992472, + "learning_rate": 1.5467167741401495e-06, + "loss": 0.58028603, + "num_input_tokens_seen": 105078540, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01708984, + "step": 4870, + "time_per_iteration": 3.5102906227111816 + }, + { + "auxiliary_loss_clip": 0.01138378, + "auxiliary_loss_mlp": 0.01037274, + "balance_loss_clip": 1.04691315, + "balance_loss_mlp": 1.02073956, + "epoch": 0.5857031203030121, + "flos": 17011926103680.0, + "grad_norm": 2.1713797990516404, + "language_loss": 0.71198547, + "learning_rate": 1.5459581071197083e-06, + "loss": 0.733742, + "num_input_tokens_seen": 105094200, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.16516113, + "step": 4871, + "time_per_iteration": 3.913904905319214 + }, + { + "auxiliary_loss_clip": 0.01143294, + "auxiliary_loss_mlp": 0.01042037, + "balance_loss_clip": 1.04836106, + "balance_loss_mlp": 1.02523994, + "epoch": 0.5858233631936511, + "flos": 20885682303360.0, + "grad_norm": 1.9931444282140673, + "language_loss": 0.83619869, + "learning_rate": 1.5451995089770624e-06, + "loss": 0.85805202, + "num_input_tokens_seen": 105113985, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.16784668, + "step": 4872, + "time_per_iteration": 2.814525842666626 + }, + { + "auxiliary_loss_clip": 0.01133443, + "auxiliary_loss_mlp": 0.01030521, + "balance_loss_clip": 1.0441606, + "balance_loss_mlp": 1.01529789, + "epoch": 0.5859436060842903, + "flos": 23191902000000.0, + "grad_norm": 1.2846736039451694, + "language_loss": 0.71892571, + "learning_rate": 1.5444409798272885e-06, + "loss": 0.74056542, + "num_input_tokens_seen": 105138075, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.15228271, + "step": 4873, + "time_per_iteration": 2.892634391784668 + }, + { + "auxiliary_loss_clip": 0.01138972, + "auxiliary_loss_mlp": 0.01050111, + "balance_loss_clip": 1.04484844, + "balance_loss_mlp": 1.03210998, + "epoch": 0.5860638489749294, + "flos": 22492648961280.0, + "grad_norm": 2.089256266139294, + "language_loss": 0.80741107, + "learning_rate": 1.543682519785456e-06, + "loss": 0.82930189, + "num_input_tokens_seen": 105156555, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.18005371, + "step": 4874, + "time_per_iteration": 3.9530248641967773 + }, + { + "auxiliary_loss_clip": 0.01134396, + "auxiliary_loss_mlp": 0.01040609, + "balance_loss_clip": 1.04514623, + "balance_loss_mlp": 1.02346659, + "epoch": 0.5861840918655684, + "flos": 17566243764480.0, + "grad_norm": 2.5049704648886144, + "language_loss": 0.80567098, + "learning_rate": 1.5429241289666219e-06, + "loss": 0.82742095, + "num_input_tokens_seen": 105174055, + "router_z_loss_clip": 0.89208984, + "router_z_loss_mlp": 0.17138672, + "step": 4875, + "time_per_iteration": 3.4703898429870605 + }, + { + "auxiliary_loss_clip": 0.01129957, + "auxiliary_loss_mlp": 0.01036749, + "balance_loss_clip": 1.04265881, + "balance_loss_mlp": 1.02106094, + "epoch": 0.5863043347562076, + "flos": 25556152118400.0, + "grad_norm": 2.126960247706983, + "language_loss": 0.69914764, + "learning_rate": 1.5421658074858342e-06, + "loss": 0.7208147, + "num_input_tokens_seen": 105192160, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15698242, + "step": 4876, + "time_per_iteration": 3.665245771408081 + }, + { + "auxiliary_loss_clip": 0.01140244, + "auxiliary_loss_mlp": 0.0104522, + "balance_loss_clip": 1.04659438, + "balance_loss_mlp": 1.02901971, + "epoch": 0.5864245776468466, + "flos": 20667525050880.0, + "grad_norm": 2.5089211233536615, + "language_loss": 0.66445768, + "learning_rate": 1.5414075554581298e-06, + "loss": 0.68631232, + "num_input_tokens_seen": 105210205, + "router_z_loss_clip": 0.93652344, + "router_z_loss_mlp": 0.1619873, + "step": 4877, + "time_per_iteration": 6.213597536087036 + }, + { + "auxiliary_loss_clip": 0.01137274, + "auxiliary_loss_mlp": 0.01044365, + "balance_loss_clip": 1.04459214, + "balance_loss_mlp": 1.02762818, + "epoch": 0.5865448205374857, + "flos": 28913907490560.0, + "grad_norm": 2.1700930706937185, + "language_loss": 0.78112108, + "learning_rate": 1.5406493729985348e-06, + "loss": 0.80293745, + "num_input_tokens_seen": 105229400, + "router_z_loss_clip": 0.92675781, + "router_z_loss_mlp": 0.1673584, + "step": 4878, + "time_per_iteration": 6.847816228866577 + }, + { + "auxiliary_loss_clip": 0.01138109, + "auxiliary_loss_mlp": 0.01044413, + "balance_loss_clip": 1.04613757, + "balance_loss_mlp": 1.02772403, + "epoch": 0.5866650634281249, + "flos": 25842575168640.0, + "grad_norm": 1.8850664131126482, + "language_loss": 0.72129714, + "learning_rate": 1.5398912602220644e-06, + "loss": 0.74312234, + "num_input_tokens_seen": 105248675, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.16662598, + "step": 4879, + "time_per_iteration": 2.9312193393707275 + }, + { + "auxiliary_loss_clip": 0.01138363, + "auxiliary_loss_mlp": 0.0104207, + "balance_loss_clip": 1.04615712, + "balance_loss_mlp": 1.02572632, + "epoch": 0.5867853063187639, + "flos": 17052325925760.0, + "grad_norm": 2.050122713487466, + "language_loss": 0.7878902, + "learning_rate": 1.539133217243724e-06, + "loss": 0.80969453, + "num_input_tokens_seen": 105265695, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.16345215, + "step": 4880, + "time_per_iteration": 3.0549347400665283 + }, + { + "auxiliary_loss_clip": 0.01142218, + "auxiliary_loss_mlp": 0.01052907, + "balance_loss_clip": 1.04682279, + "balance_loss_mlp": 1.03389299, + "epoch": 0.586905549209403, + "flos": 24645026707200.0, + "grad_norm": 2.0339337596600036, + "language_loss": 0.76183009, + "learning_rate": 1.5383752441785081e-06, + "loss": 0.78378129, + "num_input_tokens_seen": 105284920, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.19018555, + "step": 4881, + "time_per_iteration": 3.0717272758483887 + }, + { + "auxiliary_loss_clip": 0.01141095, + "auxiliary_loss_mlp": 0.01048207, + "balance_loss_clip": 1.04657173, + "balance_loss_mlp": 1.03056455, + "epoch": 0.5870257921000421, + "flos": 14720538723840.0, + "grad_norm": 2.166046830355669, + "language_loss": 0.85866058, + "learning_rate": 1.5376173411414003e-06, + "loss": 0.8805536, + "num_input_tokens_seen": 105302960, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.17626953, + "step": 4882, + "time_per_iteration": 3.211118459701538 + }, + { + "auxiliary_loss_clip": 0.0113366, + "auxiliary_loss_mlp": 0.01045467, + "balance_loss_clip": 1.0441854, + "balance_loss_mlp": 1.03025615, + "epoch": 0.5871460349906812, + "flos": 23914998691200.0, + "grad_norm": 1.9721674197155734, + "language_loss": 0.78886306, + "learning_rate": 1.5368595082473753e-06, + "loss": 0.81065428, + "num_input_tokens_seen": 105321260, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.15209961, + "step": 4883, + "time_per_iteration": 3.0612916946411133 + }, + { + "auxiliary_loss_clip": 0.01131462, + "auxiliary_loss_mlp": 0.01039369, + "balance_loss_clip": 1.04199803, + "balance_loss_mlp": 1.02282238, + "epoch": 0.5872662778813202, + "flos": 22164174063360.0, + "grad_norm": 1.5763754747261645, + "language_loss": 0.77946514, + "learning_rate": 1.5361017456113935e-06, + "loss": 0.80117345, + "num_input_tokens_seen": 105341610, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.16540527, + "step": 4884, + "time_per_iteration": 2.9143083095550537 + }, + { + "auxiliary_loss_clip": 0.01136254, + "auxiliary_loss_mlp": 0.01038286, + "balance_loss_clip": 1.04457462, + "balance_loss_mlp": 1.02181149, + "epoch": 0.5873865207719594, + "flos": 18441925430400.0, + "grad_norm": 2.078638559068359, + "language_loss": 0.85985744, + "learning_rate": 1.5353440533484085e-06, + "loss": 0.88160288, + "num_input_tokens_seen": 105360465, + "router_z_loss_clip": 0.91552734, + "router_z_loss_mlp": 0.16467285, + "step": 4885, + "time_per_iteration": 2.96683931350708 + }, + { + "auxiliary_loss_clip": 0.01138385, + "auxiliary_loss_mlp": 0.01057279, + "balance_loss_clip": 1.04541099, + "balance_loss_mlp": 1.03927827, + "epoch": 0.5875067636625985, + "flos": 54015321427200.0, + "grad_norm": 2.2387853563148132, + "language_loss": 0.66234505, + "learning_rate": 1.534586431573361e-06, + "loss": 0.68430167, + "num_input_tokens_seen": 105385405, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.18005371, + "step": 4886, + "time_per_iteration": 3.4164483547210693 + }, + { + "auxiliary_loss_clip": 0.01137999, + "auxiliary_loss_mlp": 0.0104892, + "balance_loss_clip": 1.0446527, + "balance_loss_mlp": 1.03161144, + "epoch": 0.5876270065532375, + "flos": 27995707100160.0, + "grad_norm": 1.7794630929447437, + "language_loss": 0.79501688, + "learning_rate": 1.5338288804011817e-06, + "loss": 0.81688607, + "num_input_tokens_seen": 105404905, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.1730957, + "step": 4887, + "time_per_iteration": 3.0633344650268555 + }, + { + "auxiliary_loss_clip": 0.01136437, + "auxiliary_loss_mlp": 0.01035359, + "balance_loss_clip": 1.04240298, + "balance_loss_mlp": 1.01899147, + "epoch": 0.5877472494438767, + "flos": 21361462876800.0, + "grad_norm": 3.483824935159532, + "language_loss": 0.71336126, + "learning_rate": 1.533071399946791e-06, + "loss": 0.73507929, + "num_input_tokens_seen": 105423650, + "router_z_loss_clip": 0.93896484, + "router_z_loss_mlp": 0.16357422, + "step": 4888, + "time_per_iteration": 3.025606870651245 + }, + { + "auxiliary_loss_clip": 0.01135403, + "auxiliary_loss_mlp": 0.01039312, + "balance_loss_clip": 1.04319358, + "balance_loss_mlp": 1.02275419, + "epoch": 0.5878674923345157, + "flos": 22383013674240.0, + "grad_norm": 1.9980484069198488, + "language_loss": 0.57459468, + "learning_rate": 1.5323139903250977e-06, + "loss": 0.59634179, + "num_input_tokens_seen": 105444255, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.16552734, + "step": 4889, + "time_per_iteration": 3.221350908279419 + }, + { + "auxiliary_loss_clip": 0.01139243, + "auxiliary_loss_mlp": 0.01040727, + "balance_loss_clip": 1.04779768, + "balance_loss_mlp": 1.02444315, + "epoch": 0.5879877352251548, + "flos": 21868664872320.0, + "grad_norm": 1.5662986463236217, + "language_loss": 0.76984453, + "learning_rate": 1.5315566516510002e-06, + "loss": 0.79164422, + "num_input_tokens_seen": 105462425, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.16296387, + "step": 4890, + "time_per_iteration": 3.136352300643921 + }, + { + "auxiliary_loss_clip": 0.01134976, + "auxiliary_loss_mlp": 0.01048407, + "balance_loss_clip": 1.04492903, + "balance_loss_mlp": 1.03257585, + "epoch": 0.5881079781157939, + "flos": 17493811989120.0, + "grad_norm": 1.77942995699452, + "language_loss": 0.67485875, + "learning_rate": 1.5307993840393857e-06, + "loss": 0.69669259, + "num_input_tokens_seen": 105480505, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.1583252, + "step": 4891, + "time_per_iteration": 3.1112446784973145 + }, + { + "auxiliary_loss_clip": 0.01133831, + "auxiliary_loss_mlp": 0.01036867, + "balance_loss_clip": 1.04291701, + "balance_loss_mlp": 1.02160788, + "epoch": 0.588228221006433, + "flos": 22601853285120.0, + "grad_norm": 1.8152613986471766, + "language_loss": 0.80646646, + "learning_rate": 1.530042187605132e-06, + "loss": 0.8281734, + "num_input_tokens_seen": 105499760, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.15258789, + "step": 4892, + "time_per_iteration": 2.906442880630493 + }, + { + "auxiliary_loss_clip": 0.01133208, + "auxiliary_loss_mlp": 0.0104161, + "balance_loss_clip": 1.04468215, + "balance_loss_mlp": 1.02657747, + "epoch": 0.5883484638970721, + "flos": 26176939896960.0, + "grad_norm": 1.4247160708209312, + "language_loss": 0.84245741, + "learning_rate": 1.5292850624631044e-06, + "loss": 0.86420554, + "num_input_tokens_seen": 105521955, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15026855, + "step": 4893, + "time_per_iteration": 3.0290706157684326 + }, + { + "auxiliary_loss_clip": 0.01134432, + "auxiliary_loss_mlp": 0.01054867, + "balance_loss_clip": 1.04499733, + "balance_loss_mlp": 1.03801107, + "epoch": 0.5884687067877111, + "flos": 30443737691520.0, + "grad_norm": 1.8257828632588262, + "language_loss": 0.79790056, + "learning_rate": 1.5285280087281593e-06, + "loss": 0.81979346, + "num_input_tokens_seen": 105542685, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.16851807, + "step": 4894, + "time_per_iteration": 3.8969085216522217 + }, + { + "auxiliary_loss_clip": 0.01037193, + "auxiliary_loss_mlp": 0.0101263, + "balance_loss_clip": 1.00868142, + "balance_loss_mlp": 1.01091337, + "epoch": 0.5885889496783503, + "flos": 70507550580480.0, + "grad_norm": 0.6427000819573111, + "language_loss": 0.56570083, + "learning_rate": 1.5277710265151398e-06, + "loss": 0.58619899, + "num_input_tokens_seen": 105612165, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01721191, + "step": 4895, + "time_per_iteration": 3.524178981781006 + }, + { + "auxiliary_loss_clip": 0.01142296, + "auxiliary_loss_mlp": 0.0104866, + "balance_loss_clip": 1.04718804, + "balance_loss_mlp": 1.03051591, + "epoch": 0.5887091925689893, + "flos": 19098767485440.0, + "grad_norm": 2.7177214298782024, + "language_loss": 0.77112257, + "learning_rate": 1.5270141159388803e-06, + "loss": 0.79303211, + "num_input_tokens_seen": 105629185, + "router_z_loss_clip": 0.94970703, + "router_z_loss_mlp": 0.18151855, + "step": 4896, + "time_per_iteration": 2.89795184135437 + }, + { + "auxiliary_loss_clip": 0.01133689, + "auxiliary_loss_mlp": 0.01041074, + "balance_loss_clip": 1.04094911, + "balance_loss_mlp": 1.02504015, + "epoch": 0.5888294354596284, + "flos": 23294282739840.0, + "grad_norm": 1.8135830592850417, + "language_loss": 0.80429763, + "learning_rate": 1.526257277114203e-06, + "loss": 0.82604527, + "num_input_tokens_seen": 105650260, + "router_z_loss_clip": 0.92675781, + "router_z_loss_mlp": 0.16040039, + "step": 4897, + "time_per_iteration": 3.800086259841919 + }, + { + "auxiliary_loss_clip": 0.01135793, + "auxiliary_loss_mlp": 0.01036983, + "balance_loss_clip": 1.04490972, + "balance_loss_mlp": 1.02079403, + "epoch": 0.5889496783502676, + "flos": 21981532383360.0, + "grad_norm": 1.9895426857779361, + "language_loss": 0.79671049, + "learning_rate": 1.5255005101559201e-06, + "loss": 0.81843829, + "num_input_tokens_seen": 105667870, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.16186523, + "step": 4898, + "time_per_iteration": 2.9451587200164795 + }, + { + "auxiliary_loss_clip": 0.01133439, + "auxiliary_loss_mlp": 0.01035798, + "balance_loss_clip": 1.04332912, + "balance_loss_mlp": 1.02053285, + "epoch": 0.5890699212409066, + "flos": 21685233093120.0, + "grad_norm": 2.3420387393324797, + "language_loss": 0.77017093, + "learning_rate": 1.524743815178833e-06, + "loss": 0.79186326, + "num_input_tokens_seen": 105685830, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.15264893, + "step": 4899, + "time_per_iteration": 2.9170315265655518 + }, + { + "auxiliary_loss_clip": 0.01133005, + "auxiliary_loss_mlp": 0.01042166, + "balance_loss_clip": 1.04331803, + "balance_loss_mlp": 1.02629912, + "epoch": 0.5891901641315457, + "flos": 19464553635840.0, + "grad_norm": 1.669079040257566, + "language_loss": 0.80519617, + "learning_rate": 1.5239871922977315e-06, + "loss": 0.82694787, + "num_input_tokens_seen": 105705745, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.15856934, + "step": 4900, + "time_per_iteration": 3.680656671524048 + }, + { + "auxiliary_loss_clip": 0.01131824, + "auxiliary_loss_mlp": 0.01040781, + "balance_loss_clip": 1.04333591, + "balance_loss_mlp": 1.02418661, + "epoch": 0.5893104070221848, + "flos": 19609884063360.0, + "grad_norm": 1.7169480084057833, + "language_loss": 0.90062898, + "learning_rate": 1.523230641627394e-06, + "loss": 0.92235506, + "num_input_tokens_seen": 105724730, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.16589355, + "step": 4901, + "time_per_iteration": 2.9722158908843994 + }, + { + "auxiliary_loss_clip": 0.01130411, + "auxiliary_loss_mlp": 0.01040263, + "balance_loss_clip": 1.0427264, + "balance_loss_mlp": 1.02378893, + "epoch": 0.5894306499128239, + "flos": 29060063930880.0, + "grad_norm": 2.667366384886204, + "language_loss": 0.73158115, + "learning_rate": 1.5224741632825888e-06, + "loss": 0.75328791, + "num_input_tokens_seen": 105744920, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.16491699, + "step": 4902, + "time_per_iteration": 2.969869375228882 + }, + { + "auxiliary_loss_clip": 0.01140227, + "auxiliary_loss_mlp": 0.0104825, + "balance_loss_clip": 1.0464716, + "balance_loss_mlp": 1.03047585, + "epoch": 0.589550892803463, + "flos": 42298890721920.0, + "grad_norm": 1.74411699912689, + "language_loss": 0.69471747, + "learning_rate": 1.521717757378074e-06, + "loss": 0.71660227, + "num_input_tokens_seen": 105765465, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.17785645, + "step": 4903, + "time_per_iteration": 4.120761394500732 + }, + { + "auxiliary_loss_clip": 0.01137744, + "auxiliary_loss_mlp": 0.0103729, + "balance_loss_clip": 1.04434299, + "balance_loss_mlp": 1.02006412, + "epoch": 0.5896711356941021, + "flos": 14137062197760.0, + "grad_norm": 2.0363628674283696, + "language_loss": 0.69295937, + "learning_rate": 1.5209614240285943e-06, + "loss": 0.71470964, + "num_input_tokens_seen": 105783120, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.17224121, + "step": 4904, + "time_per_iteration": 2.8056132793426514 + }, + { + "auxiliary_loss_clip": 0.01133309, + "auxiliary_loss_mlp": 0.01031955, + "balance_loss_clip": 1.0418309, + "balance_loss_mlp": 1.01495588, + "epoch": 0.5897913785847412, + "flos": 17201355454080.0, + "grad_norm": 2.1303748116253547, + "language_loss": 0.8486923, + "learning_rate": 1.520205163348887e-06, + "loss": 0.87034488, + "num_input_tokens_seen": 105801055, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.16992188, + "step": 4905, + "time_per_iteration": 2.8911635875701904 + }, + { + "auxiliary_loss_clip": 0.01038023, + "auxiliary_loss_mlp": 0.01000911, + "balance_loss_clip": 1.00969374, + "balance_loss_mlp": 0.99927759, + "epoch": 0.5899116214753802, + "flos": 48794164202880.0, + "grad_norm": 0.7241967394581638, + "language_loss": 0.56969786, + "learning_rate": 1.519448975453674e-06, + "loss": 0.59008718, + "num_input_tokens_seen": 105856155, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01635742, + "step": 4906, + "time_per_iteration": 3.2626819610595703 + }, + { + "auxiliary_loss_clip": 0.01138051, + "auxiliary_loss_mlp": 0.01039508, + "balance_loss_clip": 1.04379535, + "balance_loss_mlp": 1.02281904, + "epoch": 0.5900318643660194, + "flos": 21103659987840.0, + "grad_norm": 2.0359002474453867, + "language_loss": 0.7621882, + "learning_rate": 1.5186928604576696e-06, + "loss": 0.7839638, + "num_input_tokens_seen": 105873350, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.16699219, + "step": 4907, + "time_per_iteration": 2.973019599914551 + }, + { + "auxiliary_loss_clip": 0.01135122, + "auxiliary_loss_mlp": 0.01037661, + "balance_loss_clip": 1.04422104, + "balance_loss_mlp": 1.02074564, + "epoch": 0.5901521072566585, + "flos": 21178390233600.0, + "grad_norm": 2.0249204732681365, + "language_loss": 0.77176392, + "learning_rate": 1.5179368184755752e-06, + "loss": 0.79349172, + "num_input_tokens_seen": 105891435, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.16918945, + "step": 4908, + "time_per_iteration": 2.935953378677368 + }, + { + "auxiliary_loss_clip": 0.01134847, + "auxiliary_loss_mlp": 0.01036305, + "balance_loss_clip": 1.04393899, + "balance_loss_mlp": 1.01982391, + "epoch": 0.5902723501472975, + "flos": 20225967160320.0, + "grad_norm": 1.4384807578430956, + "language_loss": 0.83020294, + "learning_rate": 1.5171808496220821e-06, + "loss": 0.85191441, + "num_input_tokens_seen": 105910190, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.16473389, + "step": 4909, + "time_per_iteration": 2.890632390975952 + }, + { + "auxiliary_loss_clip": 0.01136476, + "auxiliary_loss_mlp": 0.01039463, + "balance_loss_clip": 1.04444933, + "balance_loss_mlp": 1.02329814, + "epoch": 0.5903925930379367, + "flos": 22964407211520.0, + "grad_norm": 1.6614012441832042, + "language_loss": 0.81490207, + "learning_rate": 1.5164249540118708e-06, + "loss": 0.83666146, + "num_input_tokens_seen": 105929315, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.16162109, + "step": 4910, + "time_per_iteration": 2.8070647716522217 + }, + { + "auxiliary_loss_clip": 0.01136739, + "auxiliary_loss_mlp": 0.01035088, + "balance_loss_clip": 1.04620838, + "balance_loss_mlp": 1.01916111, + "epoch": 0.5905128359285757, + "flos": 23367720096000.0, + "grad_norm": 1.7730037715130447, + "language_loss": 0.83041292, + "learning_rate": 1.5156691317596093e-06, + "loss": 0.85213119, + "num_input_tokens_seen": 105950740, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.15917969, + "step": 4911, + "time_per_iteration": 3.1320207118988037 + }, + { + "auxiliary_loss_clip": 0.01134676, + "auxiliary_loss_mlp": 0.01044806, + "balance_loss_clip": 1.04430795, + "balance_loss_mlp": 1.02778292, + "epoch": 0.5906330788192148, + "flos": 28032335994240.0, + "grad_norm": 2.0566118184264646, + "language_loss": 0.6646471, + "learning_rate": 1.5149133829799556e-06, + "loss": 0.6864419, + "num_input_tokens_seen": 105968735, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.17016602, + "step": 4912, + "time_per_iteration": 3.0270464420318604 + }, + { + "auxiliary_loss_clip": 0.01139501, + "auxiliary_loss_mlp": 0.01042334, + "balance_loss_clip": 1.04699397, + "balance_loss_mlp": 1.02675295, + "epoch": 0.590753321709854, + "flos": 18477943793280.0, + "grad_norm": 1.8007018005687723, + "language_loss": 0.80658388, + "learning_rate": 1.5141577077875556e-06, + "loss": 0.82840222, + "num_input_tokens_seen": 105986060, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.15588379, + "step": 4913, + "time_per_iteration": 3.0666251182556152 + }, + { + "auxiliary_loss_clip": 0.01135891, + "auxiliary_loss_mlp": 0.01043346, + "balance_loss_clip": 1.0456996, + "balance_loss_mlp": 1.02671611, + "epoch": 0.590873564600493, + "flos": 16873706568960.0, + "grad_norm": 1.7802672836889972, + "language_loss": 0.72187269, + "learning_rate": 1.5134021062970451e-06, + "loss": 0.7436651, + "num_input_tokens_seen": 106004440, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.16625977, + "step": 4914, + "time_per_iteration": 2.9422607421875 + }, + { + "auxiliary_loss_clip": 0.01132457, + "auxiliary_loss_mlp": 0.01040377, + "balance_loss_clip": 1.04359066, + "balance_loss_mlp": 1.02522552, + "epoch": 0.5909938074911321, + "flos": 13516166678400.0, + "grad_norm": 1.6863645941289498, + "language_loss": 0.80965877, + "learning_rate": 1.5126465786230483e-06, + "loss": 0.83138716, + "num_input_tokens_seen": 106021215, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.15148926, + "step": 4915, + "time_per_iteration": 2.983372688293457 + }, + { + "auxiliary_loss_clip": 0.01134219, + "auxiliary_loss_mlp": 0.01043735, + "balance_loss_clip": 1.04249716, + "balance_loss_mlp": 1.0277611, + "epoch": 0.5911140503817712, + "flos": 26024067613440.0, + "grad_norm": 1.8031757233400334, + "language_loss": 0.82191145, + "learning_rate": 1.5118911248801787e-06, + "loss": 0.84369099, + "num_input_tokens_seen": 106039225, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.15979004, + "step": 4916, + "time_per_iteration": 2.951812505722046 + }, + { + "auxiliary_loss_clip": 0.01132237, + "auxiliary_loss_mlp": 0.01038526, + "balance_loss_clip": 1.04339361, + "balance_loss_mlp": 1.02294552, + "epoch": 0.5912342932724103, + "flos": 23258731253760.0, + "grad_norm": 2.0203093821494047, + "language_loss": 0.79996467, + "learning_rate": 1.5111357451830364e-06, + "loss": 0.82167232, + "num_input_tokens_seen": 106057920, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.15588379, + "step": 4917, + "time_per_iteration": 3.383416175842285 + }, + { + "auxiliary_loss_clip": 0.01131789, + "auxiliary_loss_mlp": 0.01044284, + "balance_loss_clip": 1.04285812, + "balance_loss_mlp": 1.02729678, + "epoch": 0.5913545361630493, + "flos": 19573039687680.0, + "grad_norm": 1.747176903805866, + "language_loss": 0.71057552, + "learning_rate": 1.5103804396462131e-06, + "loss": 0.73233628, + "num_input_tokens_seen": 106077855, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.16992188, + "step": 4918, + "time_per_iteration": 3.578402519226074 + }, + { + "auxiliary_loss_clip": 0.01137717, + "auxiliary_loss_mlp": 0.01046847, + "balance_loss_clip": 1.04266393, + "balance_loss_mlp": 1.02871549, + "epoch": 0.5914747790536885, + "flos": 26213532877440.0, + "grad_norm": 2.060779145102591, + "language_loss": 0.79931563, + "learning_rate": 1.5096252083842877e-06, + "loss": 0.82116127, + "num_input_tokens_seen": 106097065, + "router_z_loss_clip": 0.94921875, + "router_z_loss_mlp": 0.18115234, + "step": 4919, + "time_per_iteration": 5.662804365158081 + }, + { + "auxiliary_loss_clip": 0.01137147, + "auxiliary_loss_mlp": 0.01040674, + "balance_loss_clip": 1.04307437, + "balance_loss_mlp": 1.02239871, + "epoch": 0.5915950219443276, + "flos": 27417545786880.0, + "grad_norm": 1.8929239584231656, + "language_loss": 0.84940898, + "learning_rate": 1.5088700515118285e-06, + "loss": 0.87118715, + "num_input_tokens_seen": 106116385, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.18286133, + "step": 4920, + "time_per_iteration": 9.255596160888672 + }, + { + "auxiliary_loss_clip": 0.01138937, + "auxiliary_loss_mlp": 0.01039699, + "balance_loss_clip": 1.04625201, + "balance_loss_mlp": 1.02335572, + "epoch": 0.5917152648349666, + "flos": 21907879545600.0, + "grad_norm": 2.1052252595623937, + "language_loss": 0.66864425, + "learning_rate": 1.508114969143392e-06, + "loss": 0.69043058, + "num_input_tokens_seen": 106136370, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.16339111, + "step": 4921, + "time_per_iteration": 2.975538969039917 + }, + { + "auxiliary_loss_clip": 0.0112955, + "auxiliary_loss_mlp": 0.01035554, + "balance_loss_clip": 1.0429883, + "balance_loss_mlp": 1.01977086, + "epoch": 0.5918355077256057, + "flos": 28109185142400.0, + "grad_norm": 1.508293586270763, + "language_loss": 0.77638322, + "learning_rate": 1.5073599613935238e-06, + "loss": 0.79803431, + "num_input_tokens_seen": 106158490, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.15795898, + "step": 4922, + "time_per_iteration": 2.959967613220215 + }, + { + "auxiliary_loss_clip": 0.0113509, + "auxiliary_loss_mlp": 0.01041082, + "balance_loss_clip": 1.04448545, + "balance_loss_mlp": 1.02410722, + "epoch": 0.5919557506162448, + "flos": 28183807647360.0, + "grad_norm": 1.742599568246027, + "language_loss": 0.57723683, + "learning_rate": 1.5066050283767574e-06, + "loss": 0.59899855, + "num_input_tokens_seen": 106179170, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.16967773, + "step": 4923, + "time_per_iteration": 4.18498682975769 + }, + { + "auxiliary_loss_clip": 0.01132997, + "auxiliary_loss_mlp": 0.01047835, + "balance_loss_clip": 1.0454582, + "balance_loss_mlp": 1.03304684, + "epoch": 0.5920759935068839, + "flos": 12094355652480.0, + "grad_norm": 1.9703127254550636, + "language_loss": 0.82988501, + "learning_rate": 1.505850170207616e-06, + "loss": 0.85169333, + "num_input_tokens_seen": 106196035, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.14776611, + "step": 4924, + "time_per_iteration": 3.1465086936950684 + }, + { + "auxiliary_loss_clip": 0.01132489, + "auxiliary_loss_mlp": 0.01040101, + "balance_loss_clip": 1.04303908, + "balance_loss_mlp": 1.02415061, + "epoch": 0.592196236397523, + "flos": 29424772673280.0, + "grad_norm": 2.0070066699678084, + "language_loss": 0.78242123, + "learning_rate": 1.505095387000611e-06, + "loss": 0.80414712, + "num_input_tokens_seen": 106218335, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.15942383, + "step": 4925, + "time_per_iteration": 3.8602914810180664 + }, + { + "auxiliary_loss_clip": 0.01136419, + "auxiliary_loss_mlp": 0.01047709, + "balance_loss_clip": 1.04602361, + "balance_loss_mlp": 1.03104401, + "epoch": 0.5923164792881621, + "flos": 24384709866240.0, + "grad_norm": 1.9525631976246538, + "language_loss": 0.74246144, + "learning_rate": 1.504340678870242e-06, + "loss": 0.76430273, + "num_input_tokens_seen": 106236550, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.16674805, + "step": 4926, + "time_per_iteration": 2.8772196769714355 + }, + { + "auxiliary_loss_clip": 0.01133556, + "auxiliary_loss_mlp": 0.01041535, + "balance_loss_clip": 1.0422796, + "balance_loss_mlp": 1.0250479, + "epoch": 0.5924367221788012, + "flos": 24024238928640.0, + "grad_norm": 2.095691311306696, + "language_loss": 0.89769542, + "learning_rate": 1.5035860459309989e-06, + "loss": 0.91944629, + "num_input_tokens_seen": 106254265, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.16491699, + "step": 4927, + "time_per_iteration": 2.833862781524658 + }, + { + "auxiliary_loss_clip": 0.01137809, + "auxiliary_loss_mlp": 0.01044085, + "balance_loss_clip": 1.04473245, + "balance_loss_mlp": 1.02608395, + "epoch": 0.5925569650694402, + "flos": 26870590414080.0, + "grad_norm": 1.7785026326354425, + "language_loss": 0.63464189, + "learning_rate": 1.5028314882973568e-06, + "loss": 0.65646088, + "num_input_tokens_seen": 106274670, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.17993164, + "step": 4928, + "time_per_iteration": 3.030632972717285 + }, + { + "auxiliary_loss_clip": 0.01136764, + "auxiliary_loss_mlp": 0.01046119, + "balance_loss_clip": 1.04417694, + "balance_loss_mlp": 1.02859509, + "epoch": 0.5926772079600794, + "flos": 22302788647680.0, + "grad_norm": 1.8650836081449835, + "language_loss": 0.8515141, + "learning_rate": 1.502077006083783e-06, + "loss": 0.87334293, + "num_input_tokens_seen": 106293330, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.17504883, + "step": 4929, + "time_per_iteration": 3.819194793701172 + }, + { + "auxiliary_loss_clip": 0.0113659, + "auxiliary_loss_mlp": 0.0104407, + "balance_loss_clip": 1.0452801, + "balance_loss_mlp": 1.02778554, + "epoch": 0.5927974508507184, + "flos": 19865244827520.0, + "grad_norm": 1.6823341466135877, + "language_loss": 0.76662469, + "learning_rate": 1.5013225994047315e-06, + "loss": 0.78843129, + "num_input_tokens_seen": 106310960, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.16296387, + "step": 4930, + "time_per_iteration": 2.915741205215454 + }, + { + "auxiliary_loss_clip": 0.0113679, + "auxiliary_loss_mlp": 0.01041225, + "balance_loss_clip": 1.04607201, + "balance_loss_mlp": 1.02469051, + "epoch": 0.5929176937413575, + "flos": 15776743167360.0, + "grad_norm": 1.7857280123635468, + "language_loss": 0.80679554, + "learning_rate": 1.5005682683746452e-06, + "loss": 0.82857573, + "num_input_tokens_seen": 106329475, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.1652832, + "step": 4931, + "time_per_iteration": 3.8061301708221436 + }, + { + "auxiliary_loss_clip": 0.01138846, + "auxiliary_loss_mlp": 0.01038976, + "balance_loss_clip": 1.046152, + "balance_loss_mlp": 1.02317429, + "epoch": 0.5930379366319967, + "flos": 17601472028160.0, + "grad_norm": 2.0208422306771885, + "language_loss": 0.73042703, + "learning_rate": 1.4998140131079553e-06, + "loss": 0.75220525, + "num_input_tokens_seen": 106345565, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.15802002, + "step": 4932, + "time_per_iteration": 2.8446881771087646 + }, + { + "auxiliary_loss_clip": 0.01133402, + "auxiliary_loss_mlp": 0.01036163, + "balance_loss_clip": 1.04418886, + "balance_loss_mlp": 1.0206542, + "epoch": 0.5931581795226357, + "flos": 17704283731200.0, + "grad_norm": 10.999950569281452, + "language_loss": 0.73600608, + "learning_rate": 1.4990598337190821e-06, + "loss": 0.75770169, + "num_input_tokens_seen": 106361920, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.1550293, + "step": 4933, + "time_per_iteration": 2.7968246936798096 + }, + { + "auxiliary_loss_clip": 0.01133224, + "auxiliary_loss_mlp": 0.01035847, + "balance_loss_clip": 1.04356396, + "balance_loss_mlp": 1.02112508, + "epoch": 0.5932784224132748, + "flos": 24280102483200.0, + "grad_norm": 2.217119087036368, + "language_loss": 0.68239069, + "learning_rate": 1.4983057303224338e-06, + "loss": 0.70408142, + "num_input_tokens_seen": 106381735, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.14703369, + "step": 4934, + "time_per_iteration": 2.874817371368408 + }, + { + "auxiliary_loss_clip": 0.01136798, + "auxiliary_loss_mlp": 0.01045228, + "balance_loss_clip": 1.04480314, + "balance_loss_mlp": 1.02856231, + "epoch": 0.5933986653039139, + "flos": 22926700909440.0, + "grad_norm": 1.954863054319968, + "language_loss": 0.876508, + "learning_rate": 1.4975517030324072e-06, + "loss": 0.8983283, + "num_input_tokens_seen": 106399745, + "router_z_loss_clip": 0.92089844, + "router_z_loss_mlp": 0.16662598, + "step": 4935, + "time_per_iteration": 2.818798065185547 + }, + { + "auxiliary_loss_clip": 0.010379, + "auxiliary_loss_mlp": 0.01002809, + "balance_loss_clip": 1.00958776, + "balance_loss_mlp": 1.00128293, + "epoch": 0.593518908194553, + "flos": 71121730256640.0, + "grad_norm": 1.109972357667154, + "language_loss": 0.61767268, + "learning_rate": 1.4967977519633882e-06, + "loss": 0.63807976, + "num_input_tokens_seen": 106457205, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01525879, + "step": 4936, + "time_per_iteration": 3.541314125061035 + }, + { + "auxiliary_loss_clip": 0.01136389, + "auxiliary_loss_mlp": 0.01034959, + "balance_loss_clip": 1.04307079, + "balance_loss_mlp": 1.01938426, + "epoch": 0.593639151085192, + "flos": 20448649526400.0, + "grad_norm": 2.017690023421364, + "language_loss": 0.78526938, + "learning_rate": 1.4960438772297494e-06, + "loss": 0.80698282, + "num_input_tokens_seen": 106474250, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.15582275, + "step": 4937, + "time_per_iteration": 2.9142308235168457 + }, + { + "auxiliary_loss_clip": 0.01138941, + "auxiliary_loss_mlp": 0.01038642, + "balance_loss_clip": 1.04595721, + "balance_loss_mlp": 1.0233711, + "epoch": 0.5937593939758312, + "flos": 30883428074880.0, + "grad_norm": 2.062791204632167, + "language_loss": 0.73833925, + "learning_rate": 1.495290078945855e-06, + "loss": 0.76011509, + "num_input_tokens_seen": 106494015, + "router_z_loss_clip": 0.93017578, + "router_z_loss_mlp": 0.15270996, + "step": 4938, + "time_per_iteration": 2.9190468788146973 + }, + { + "auxiliary_loss_clip": 0.0113392, + "auxiliary_loss_mlp": 0.01038769, + "balance_loss_clip": 1.04490221, + "balance_loss_mlp": 1.02341509, + "epoch": 0.5938796368664703, + "flos": 36898069668480.0, + "grad_norm": 2.1021176552378154, + "language_loss": 0.74648738, + "learning_rate": 1.4945363572260529e-06, + "loss": 0.76821429, + "num_input_tokens_seen": 106515010, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.15356445, + "step": 4939, + "time_per_iteration": 3.1624879837036133 + }, + { + "auxiliary_loss_clip": 0.01135945, + "auxiliary_loss_mlp": 0.01034773, + "balance_loss_clip": 1.04339004, + "balance_loss_mlp": 1.017905, + "epoch": 0.5939998797571093, + "flos": 23842926051840.0, + "grad_norm": 1.8377055158614866, + "language_loss": 0.6852504, + "learning_rate": 1.4937827121846845e-06, + "loss": 0.70695758, + "num_input_tokens_seen": 106535265, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.16870117, + "step": 4940, + "time_per_iteration": 3.0036299228668213 + }, + { + "auxiliary_loss_clip": 0.01134992, + "auxiliary_loss_mlp": 0.01043577, + "balance_loss_clip": 1.04538751, + "balance_loss_mlp": 1.02724552, + "epoch": 0.5941201226477485, + "flos": 25191407462400.0, + "grad_norm": 1.480255755139372, + "language_loss": 0.73213005, + "learning_rate": 1.4930291439360755e-06, + "loss": 0.75391573, + "num_input_tokens_seen": 106557830, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.16333008, + "step": 4941, + "time_per_iteration": 2.85945200920105 + }, + { + "auxiliary_loss_clip": 0.01138967, + "auxiliary_loss_mlp": 0.01047814, + "balance_loss_clip": 1.04497766, + "balance_loss_mlp": 1.03054047, + "epoch": 0.5942403655383875, + "flos": 22418996123520.0, + "grad_norm": 1.8901373008224671, + "language_loss": 0.78856468, + "learning_rate": 1.4922756525945427e-06, + "loss": 0.81043243, + "num_input_tokens_seen": 106577140, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.17260742, + "step": 4942, + "time_per_iteration": 2.8152856826782227 + }, + { + "auxiliary_loss_clip": 0.01040019, + "auxiliary_loss_mlp": 0.01000539, + "balance_loss_clip": 1.01161909, + "balance_loss_mlp": 0.998918, + "epoch": 0.5943606084290266, + "flos": 67629310796160.0, + "grad_norm": 0.9813061347681844, + "language_loss": 0.5955441, + "learning_rate": 1.4915222382743894e-06, + "loss": 0.61594963, + "num_input_tokens_seen": 106635975, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01623535, + "step": 4943, + "time_per_iteration": 3.423936367034912 + }, + { + "auxiliary_loss_clip": 0.01138435, + "auxiliary_loss_mlp": 0.01039672, + "balance_loss_clip": 1.04744267, + "balance_loss_mlp": 1.02248216, + "epoch": 0.5944808513196658, + "flos": 18223157646720.0, + "grad_norm": 1.9391734799326936, + "language_loss": 0.72265005, + "learning_rate": 1.4907689010899085e-06, + "loss": 0.74443114, + "num_input_tokens_seen": 106653555, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.17199707, + "step": 4944, + "time_per_iteration": 2.821082353591919 + }, + { + "auxiliary_loss_clip": 0.01135013, + "auxiliary_loss_mlp": 0.0103539, + "balance_loss_clip": 1.04328752, + "balance_loss_mlp": 1.01878452, + "epoch": 0.5946010942103048, + "flos": 24790824011520.0, + "grad_norm": 1.8871228323596363, + "language_loss": 0.62694609, + "learning_rate": 1.4900156411553804e-06, + "loss": 0.64865017, + "num_input_tokens_seen": 106673385, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.16601562, + "step": 4945, + "time_per_iteration": 2.9287893772125244 + }, + { + "auxiliary_loss_clip": 0.01134452, + "auxiliary_loss_mlp": 0.01039536, + "balance_loss_clip": 1.04490757, + "balance_loss_mlp": 1.02326405, + "epoch": 0.5947213371009439, + "flos": 15231619388160.0, + "grad_norm": 2.6978344216681394, + "language_loss": 0.85370648, + "learning_rate": 1.4892624585850739e-06, + "loss": 0.87544644, + "num_input_tokens_seen": 106691740, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.16259766, + "step": 4946, + "time_per_iteration": 2.7627618312835693 + }, + { + "auxiliary_loss_clip": 0.0113774, + "auxiliary_loss_mlp": 0.01040186, + "balance_loss_clip": 1.04473305, + "balance_loss_mlp": 1.0231154, + "epoch": 0.594841579991583, + "flos": 25848069949440.0, + "grad_norm": 2.0532712273543408, + "language_loss": 0.8009516, + "learning_rate": 1.4885093534932465e-06, + "loss": 0.8227309, + "num_input_tokens_seen": 106709705, + "router_z_loss_clip": 0.93017578, + "router_z_loss_mlp": 0.17089844, + "step": 4947, + "time_per_iteration": 2.937176465988159 + }, + { + "auxiliary_loss_clip": 0.0113667, + "auxiliary_loss_mlp": 0.01047199, + "balance_loss_clip": 1.04524219, + "balance_loss_mlp": 1.03042614, + "epoch": 0.5949618228822221, + "flos": 23981109672960.0, + "grad_norm": 1.9123496724300404, + "language_loss": 0.71405101, + "learning_rate": 1.4877563259941433e-06, + "loss": 0.73588967, + "num_input_tokens_seen": 106727560, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.16760254, + "step": 4948, + "time_per_iteration": 2.904881477355957 + }, + { + "auxiliary_loss_clip": 0.01138895, + "auxiliary_loss_mlp": 0.01037998, + "balance_loss_clip": 1.04634523, + "balance_loss_mlp": 1.02227426, + "epoch": 0.5950820657728612, + "flos": 40547491476480.0, + "grad_norm": 1.7357803154764697, + "language_loss": 0.67517227, + "learning_rate": 1.4870033762019988e-06, + "loss": 0.69694126, + "num_input_tokens_seen": 106747725, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.15722656, + "step": 4949, + "time_per_iteration": 2.9413554668426514 + }, + { + "auxiliary_loss_clip": 0.01138575, + "auxiliary_loss_mlp": 0.01044789, + "balance_loss_clip": 1.04609776, + "balance_loss_mlp": 1.02676439, + "epoch": 0.5952023086635003, + "flos": 23184467884800.0, + "grad_norm": 1.8287523503342094, + "language_loss": 0.73383427, + "learning_rate": 1.4862505042310334e-06, + "loss": 0.75566792, + "num_input_tokens_seen": 106767010, + "router_z_loss_clip": 0.92675781, + "router_z_loss_mlp": 0.18005371, + "step": 4950, + "time_per_iteration": 2.8149476051330566 + }, + { + "auxiliary_loss_clip": 0.01132795, + "auxiliary_loss_mlp": 0.01039438, + "balance_loss_clip": 1.04552007, + "balance_loss_mlp": 1.02456093, + "epoch": 0.5953225515541394, + "flos": 33653289548160.0, + "grad_norm": 1.5676194297440207, + "language_loss": 0.69500691, + "learning_rate": 1.4854977101954587e-06, + "loss": 0.71672928, + "num_input_tokens_seen": 106789230, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.14880371, + "step": 4951, + "time_per_iteration": 2.9946861267089844 + }, + { + "auxiliary_loss_clip": 0.0113526, + "auxiliary_loss_mlp": 0.01039342, + "balance_loss_clip": 1.04322505, + "balance_loss_mlp": 1.0233326, + "epoch": 0.5954427944447784, + "flos": 24459619680000.0, + "grad_norm": 1.84753469604762, + "language_loss": 0.86131072, + "learning_rate": 1.4847449942094716e-06, + "loss": 0.8830567, + "num_input_tokens_seen": 106808110, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.16015625, + "step": 4952, + "time_per_iteration": 3.8547422885894775 + }, + { + "auxiliary_loss_clip": 0.01133715, + "auxiliary_loss_mlp": 0.01043927, + "balance_loss_clip": 1.04383826, + "balance_loss_mlp": 1.02791739, + "epoch": 0.5955630373354175, + "flos": 18551848026240.0, + "grad_norm": 1.920299792761761, + "language_loss": 0.86226082, + "learning_rate": 1.4839923563872598e-06, + "loss": 0.8840372, + "num_input_tokens_seen": 106826650, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.16003418, + "step": 4953, + "time_per_iteration": 2.9893529415130615 + }, + { + "auxiliary_loss_clip": 0.01135262, + "auxiliary_loss_mlp": 0.01040943, + "balance_loss_clip": 1.04600465, + "balance_loss_mlp": 1.02444446, + "epoch": 0.5956832802260567, + "flos": 19791699730560.0, + "grad_norm": 1.8366073920494812, + "language_loss": 0.75827754, + "learning_rate": 1.483239796842997e-06, + "loss": 0.78003955, + "num_input_tokens_seen": 106844680, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.16503906, + "step": 4954, + "time_per_iteration": 3.844616413116455 + }, + { + "auxiliary_loss_clip": 0.01131411, + "auxiliary_loss_mlp": 0.01046614, + "balance_loss_clip": 1.043715, + "balance_loss_mlp": 1.03024673, + "epoch": 0.5958035231166957, + "flos": 19750868945280.0, + "grad_norm": 1.6243911769042103, + "language_loss": 0.84068048, + "learning_rate": 1.4824873156908462e-06, + "loss": 0.86246073, + "num_input_tokens_seen": 106862605, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16369629, + "step": 4955, + "time_per_iteration": 2.998015880584717 + }, + { + "auxiliary_loss_clip": 0.01139472, + "auxiliary_loss_mlp": 0.01040912, + "balance_loss_clip": 1.04560471, + "balance_loss_mlp": 1.02249396, + "epoch": 0.5959237660073348, + "flos": 21652806090240.0, + "grad_norm": 1.6017703245136055, + "language_loss": 0.75657535, + "learning_rate": 1.4817349130449584e-06, + "loss": 0.7783792, + "num_input_tokens_seen": 106882325, + "router_z_loss_clip": 0.93896484, + "router_z_loss_mlp": 0.18395996, + "step": 4956, + "time_per_iteration": 2.939399003982544 + }, + { + "auxiliary_loss_clip": 0.01133388, + "auxiliary_loss_mlp": 0.01041335, + "balance_loss_clip": 1.04433298, + "balance_loss_mlp": 1.02450299, + "epoch": 0.5960440088979739, + "flos": 21171207513600.0, + "grad_norm": 1.827410804143494, + "language_loss": 0.82556528, + "learning_rate": 1.4809825890194717e-06, + "loss": 0.84731251, + "num_input_tokens_seen": 106900995, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.16833496, + "step": 4957, + "time_per_iteration": 3.7585978507995605 + }, + { + "auxiliary_loss_clip": 0.01132922, + "auxiliary_loss_mlp": 0.01042924, + "balance_loss_clip": 1.04530942, + "balance_loss_mlp": 1.02749252, + "epoch": 0.596164251788613, + "flos": 14757526753920.0, + "grad_norm": 1.7016973108473579, + "language_loss": 0.76973474, + "learning_rate": 1.4802303437285139e-06, + "loss": 0.79149324, + "num_input_tokens_seen": 106918265, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.15423584, + "step": 4958, + "time_per_iteration": 3.2440598011016846 + }, + { + "auxiliary_loss_clip": 0.01135557, + "auxiliary_loss_mlp": 0.01038318, + "balance_loss_clip": 1.04458237, + "balance_loss_mlp": 1.02192616, + "epoch": 0.596284494679252, + "flos": 20485924865280.0, + "grad_norm": 2.2337560415908375, + "language_loss": 0.80496383, + "learning_rate": 1.4794781772861994e-06, + "loss": 0.82670254, + "num_input_tokens_seen": 106934760, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.16381836, + "step": 4959, + "time_per_iteration": 2.999368906021118 + }, + { + "auxiliary_loss_clip": 0.01132593, + "auxiliary_loss_mlp": 0.01042516, + "balance_loss_clip": 1.04477167, + "balance_loss_mlp": 1.0269711, + "epoch": 0.5964047375698912, + "flos": 31212262108800.0, + "grad_norm": 1.856377669914003, + "language_loss": 0.66877222, + "learning_rate": 1.4787260898066324e-06, + "loss": 0.69052333, + "num_input_tokens_seen": 106954760, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15533447, + "step": 4960, + "time_per_iteration": 4.016515493392944 + }, + { + "auxiliary_loss_clip": 0.01130331, + "auxiliary_loss_mlp": 0.0103738, + "balance_loss_clip": 1.04254484, + "balance_loss_mlp": 1.02005887, + "epoch": 0.5965249804605303, + "flos": 27483620855040.0, + "grad_norm": 2.0154472966150583, + "language_loss": 0.86022753, + "learning_rate": 1.4779740814039023e-06, + "loss": 0.88190466, + "num_input_tokens_seen": 106974845, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.17303467, + "step": 4961, + "time_per_iteration": 2.9996275901794434 + }, + { + "auxiliary_loss_clip": 0.01136908, + "auxiliary_loss_mlp": 0.01041496, + "balance_loss_clip": 1.04540622, + "balance_loss_mlp": 1.02475262, + "epoch": 0.5966452233511693, + "flos": 30773936442240.0, + "grad_norm": 1.7474801091468632, + "language_loss": 0.68394387, + "learning_rate": 1.4772221521920894e-06, + "loss": 0.70572793, + "num_input_tokens_seen": 106994870, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.16741943, + "step": 4962, + "time_per_iteration": 2.8890535831451416 + }, + { + "auxiliary_loss_clip": 0.01136582, + "auxiliary_loss_mlp": 0.01040893, + "balance_loss_clip": 1.04645562, + "balance_loss_mlp": 1.02518153, + "epoch": 0.5967654662418085, + "flos": 25481170477440.0, + "grad_norm": 2.1542657504347824, + "language_loss": 0.7421841, + "learning_rate": 1.4764703022852598e-06, + "loss": 0.76395881, + "num_input_tokens_seen": 107015390, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.15698242, + "step": 4963, + "time_per_iteration": 2.9055662155151367 + }, + { + "auxiliary_loss_clip": 0.01129571, + "auxiliary_loss_mlp": 0.01036213, + "balance_loss_clip": 1.04206634, + "balance_loss_mlp": 1.02139533, + "epoch": 0.5968857091324475, + "flos": 19099126621440.0, + "grad_norm": 1.7395412151468286, + "language_loss": 0.76949948, + "learning_rate": 1.4757185317974696e-06, + "loss": 0.79115736, + "num_input_tokens_seen": 107033775, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.14819336, + "step": 4964, + "time_per_iteration": 2.8614964485168457 + }, + { + "auxiliary_loss_clip": 0.01140023, + "auxiliary_loss_mlp": 0.01044695, + "balance_loss_clip": 1.0455972, + "balance_loss_mlp": 1.02781522, + "epoch": 0.5970059520230866, + "flos": 23692711374720.0, + "grad_norm": 1.9799169215287096, + "language_loss": 0.71169364, + "learning_rate": 1.474966840842761e-06, + "loss": 0.73354089, + "num_input_tokens_seen": 107053355, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.16870117, + "step": 4965, + "time_per_iteration": 2.9066624641418457 + }, + { + "auxiliary_loss_clip": 0.01133408, + "auxiliary_loss_mlp": 0.01038476, + "balance_loss_clip": 1.04395115, + "balance_loss_mlp": 1.02229905, + "epoch": 0.5971261949137258, + "flos": 23185545292800.0, + "grad_norm": 1.757207575113481, + "language_loss": 0.86925459, + "learning_rate": 1.4742152295351655e-06, + "loss": 0.89097345, + "num_input_tokens_seen": 107072510, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.16174316, + "step": 4966, + "time_per_iteration": 3.025297164916992 + }, + { + "auxiliary_loss_clip": 0.01137831, + "auxiliary_loss_mlp": 0.0105482, + "balance_loss_clip": 1.04326701, + "balance_loss_mlp": 1.03710508, + "epoch": 0.5972464378043648, + "flos": 20557710195840.0, + "grad_norm": 2.412530105564447, + "language_loss": 0.63974941, + "learning_rate": 1.4734636979887016e-06, + "loss": 0.66167593, + "num_input_tokens_seen": 107089970, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.17700195, + "step": 4967, + "time_per_iteration": 2.9166100025177 + }, + { + "auxiliary_loss_clip": 0.01136956, + "auxiliary_loss_mlp": 0.01038607, + "balance_loss_clip": 1.04443192, + "balance_loss_mlp": 1.02231157, + "epoch": 0.5973666806950039, + "flos": 29387030457600.0, + "grad_norm": 2.083765137988435, + "language_loss": 0.90285295, + "learning_rate": 1.4727122463173755e-06, + "loss": 0.92460859, + "num_input_tokens_seen": 107108500, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.1628418, + "step": 4968, + "time_per_iteration": 2.954906463623047 + }, + { + "auxiliary_loss_clip": 0.01135877, + "auxiliary_loss_mlp": 0.01040855, + "balance_loss_clip": 1.04563951, + "balance_loss_mlp": 1.02474976, + "epoch": 0.597486923585643, + "flos": 22273522041600.0, + "grad_norm": 1.658178790956572, + "language_loss": 0.64479786, + "learning_rate": 1.471960874635183e-06, + "loss": 0.66656518, + "num_input_tokens_seen": 107128060, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.16101074, + "step": 4969, + "time_per_iteration": 2.7795517444610596 + }, + { + "auxiliary_loss_clip": 0.01139648, + "auxiliary_loss_mlp": 0.01040216, + "balance_loss_clip": 1.04580975, + "balance_loss_mlp": 1.02368176, + "epoch": 0.5976071664762821, + "flos": 13772461196160.0, + "grad_norm": 2.278659358127917, + "language_loss": 0.7059294, + "learning_rate": 1.4712095830561055e-06, + "loss": 0.72772813, + "num_input_tokens_seen": 107146550, + "router_z_loss_clip": 0.93896484, + "router_z_loss_mlp": 0.16540527, + "step": 4970, + "time_per_iteration": 2.860175132751465 + }, + { + "auxiliary_loss_clip": 0.01132763, + "auxiliary_loss_mlp": 0.01044231, + "balance_loss_clip": 1.0440979, + "balance_loss_mlp": 1.02919888, + "epoch": 0.5977274093669211, + "flos": 19098623831040.0, + "grad_norm": 1.881180181167979, + "language_loss": 0.80780005, + "learning_rate": 1.4704583716941147e-06, + "loss": 0.82957006, + "num_input_tokens_seen": 107165415, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.15026855, + "step": 4971, + "time_per_iteration": 2.8353917598724365 + }, + { + "auxiliary_loss_clip": 0.01137521, + "auxiliary_loss_mlp": 0.01043535, + "balance_loss_clip": 1.04770815, + "balance_loss_mlp": 1.02719116, + "epoch": 0.5978476522575603, + "flos": 20376002269440.0, + "grad_norm": 1.769075809926621, + "language_loss": 0.72707182, + "learning_rate": 1.4697072406631672e-06, + "loss": 0.74888241, + "num_input_tokens_seen": 107185320, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.16333008, + "step": 4972, + "time_per_iteration": 2.8299801349639893 + }, + { + "auxiliary_loss_clip": 0.01140339, + "auxiliary_loss_mlp": 0.01046657, + "balance_loss_clip": 1.04686666, + "balance_loss_mlp": 1.02901387, + "epoch": 0.5979678951481994, + "flos": 29023147728000.0, + "grad_norm": 2.1601654437101248, + "language_loss": 0.72560191, + "learning_rate": 1.4689561900772097e-06, + "loss": 0.74747181, + "num_input_tokens_seen": 107205380, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.1763916, + "step": 4973, + "time_per_iteration": 2.919118881225586 + }, + { + "auxiliary_loss_clip": 0.01134011, + "auxiliary_loss_mlp": 0.01045087, + "balance_loss_clip": 1.04312885, + "balance_loss_mlp": 1.02836168, + "epoch": 0.5980881380388384, + "flos": 17967689141760.0, + "grad_norm": 2.2182229750307094, + "language_loss": 0.72861254, + "learning_rate": 1.4682052200501758e-06, + "loss": 0.75040352, + "num_input_tokens_seen": 107222585, + "router_z_loss_clip": 0.90771484, + "router_z_loss_mlp": 0.1673584, + "step": 4974, + "time_per_iteration": 2.878241777420044 + }, + { + "auxiliary_loss_clip": 0.01132622, + "auxiliary_loss_mlp": 0.01040456, + "balance_loss_clip": 1.04260325, + "balance_loss_mlp": 1.0243628, + "epoch": 0.5982083809294776, + "flos": 22962827013120.0, + "grad_norm": 1.7775939630320474, + "language_loss": 0.80217838, + "learning_rate": 1.4674543306959876e-06, + "loss": 0.82390922, + "num_input_tokens_seen": 107242055, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.16088867, + "step": 4975, + "time_per_iteration": 2.9043076038360596 + }, + { + "auxiliary_loss_clip": 0.0113867, + "auxiliary_loss_mlp": 0.01041877, + "balance_loss_clip": 1.04436457, + "balance_loss_mlp": 1.02541375, + "epoch": 0.5983286238201166, + "flos": 20991941712000.0, + "grad_norm": 1.9213708064939425, + "language_loss": 0.84197116, + "learning_rate": 1.4667035221285535e-06, + "loss": 0.86377662, + "num_input_tokens_seen": 107259695, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.16479492, + "step": 4976, + "time_per_iteration": 2.77340030670166 + }, + { + "auxiliary_loss_clip": 0.01131779, + "auxiliary_loss_mlp": 0.01039196, + "balance_loss_clip": 1.04122949, + "balance_loss_mlp": 1.02255416, + "epoch": 0.5984488667107557, + "flos": 28183448511360.0, + "grad_norm": 1.7189570604264703, + "language_loss": 0.74205768, + "learning_rate": 1.4659527944617715e-06, + "loss": 0.76376742, + "num_input_tokens_seen": 107279640, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.16650391, + "step": 4977, + "time_per_iteration": 2.779132604598999 + }, + { + "auxiliary_loss_clip": 0.01134627, + "auxiliary_loss_mlp": 0.01046415, + "balance_loss_clip": 1.04344726, + "balance_loss_mlp": 1.0303936, + "epoch": 0.5985691096013949, + "flos": 16471794314880.0, + "grad_norm": 1.9450607901625732, + "language_loss": 0.76031297, + "learning_rate": 1.465202147809526e-06, + "loss": 0.78212345, + "num_input_tokens_seen": 107298135, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.16027832, + "step": 4978, + "time_per_iteration": 2.7482736110687256 + }, + { + "auxiliary_loss_clip": 0.01135761, + "auxiliary_loss_mlp": 0.01038298, + "balance_loss_clip": 1.04464352, + "balance_loss_mlp": 1.02371323, + "epoch": 0.5986893524920339, + "flos": 26719046933760.0, + "grad_norm": 1.8226267262653912, + "language_loss": 0.76168919, + "learning_rate": 1.4644515822856888e-06, + "loss": 0.7834298, + "num_input_tokens_seen": 107316570, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.14587402, + "step": 4979, + "time_per_iteration": 2.891387701034546 + }, + { + "auxiliary_loss_clip": 0.0103906, + "auxiliary_loss_mlp": 0.01009489, + "balance_loss_clip": 1.01052737, + "balance_loss_mlp": 1.00791502, + "epoch": 0.598809595382673, + "flos": 61608061100160.0, + "grad_norm": 0.7624634739638207, + "language_loss": 0.56500763, + "learning_rate": 1.4637010980041215e-06, + "loss": 0.58549309, + "num_input_tokens_seen": 107378680, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01574707, + "step": 4980, + "time_per_iteration": 3.303760051727295 + }, + { + "auxiliary_loss_clip": 0.01134959, + "auxiliary_loss_mlp": 0.01044652, + "balance_loss_clip": 1.04384673, + "balance_loss_mlp": 1.02765846, + "epoch": 0.5989298382733121, + "flos": 11801719549440.0, + "grad_norm": 3.012703960863451, + "language_loss": 0.89797354, + "learning_rate": 1.4629506950786707e-06, + "loss": 0.91976959, + "num_input_tokens_seen": 107394860, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.16986084, + "step": 4981, + "time_per_iteration": 3.6360678672790527 + }, + { + "auxiliary_loss_clip": 0.01038909, + "auxiliary_loss_mlp": 0.01002378, + "balance_loss_clip": 1.01042616, + "balance_loss_mlp": 1.00073326, + "epoch": 0.5990500811639512, + "flos": 60025800021120.0, + "grad_norm": 0.8181858030657574, + "language_loss": 0.56005996, + "learning_rate": 1.4622003736231733e-06, + "loss": 0.58047283, + "num_input_tokens_seen": 107453850, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01647949, + "step": 4982, + "time_per_iteration": 3.385507345199585 + }, + { + "auxiliary_loss_clip": 0.01136796, + "auxiliary_loss_mlp": 0.01043721, + "balance_loss_clip": 1.0437336, + "balance_loss_mlp": 1.02775931, + "epoch": 0.5991703240545903, + "flos": 18222726683520.0, + "grad_norm": 2.2074195126058687, + "language_loss": 0.80928439, + "learning_rate": 1.461450133751451e-06, + "loss": 0.83108962, + "num_input_tokens_seen": 107471920, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.1595459, + "step": 4983, + "time_per_iteration": 3.763664960861206 + }, + { + "auxiliary_loss_clip": 0.01132561, + "auxiliary_loss_mlp": 0.01038203, + "balance_loss_clip": 1.04354155, + "balance_loss_mlp": 1.02238417, + "epoch": 0.5992905669452293, + "flos": 27709894581120.0, + "grad_norm": 2.2630448385963176, + "language_loss": 0.75896865, + "learning_rate": 1.4606999755773153e-06, + "loss": 0.78067625, + "num_input_tokens_seen": 107493125, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.1583252, + "step": 4984, + "time_per_iteration": 2.78082013130188 + }, + { + "auxiliary_loss_clip": 0.01130039, + "auxiliary_loss_mlp": 0.01036678, + "balance_loss_clip": 1.04268312, + "balance_loss_mlp": 1.02077591, + "epoch": 0.5994108098358685, + "flos": 20449008662400.0, + "grad_norm": 1.5876838975515195, + "language_loss": 0.822752, + "learning_rate": 1.4599498992145643e-06, + "loss": 0.84441918, + "num_input_tokens_seen": 107513150, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15887451, + "step": 4985, + "time_per_iteration": 2.756690263748169 + }, + { + "auxiliary_loss_clip": 0.01134682, + "auxiliary_loss_mlp": 0.01038355, + "balance_loss_clip": 1.04450369, + "balance_loss_mlp": 1.02240491, + "epoch": 0.5995310527265075, + "flos": 22269966595200.0, + "grad_norm": 1.7369668879778388, + "language_loss": 0.70975149, + "learning_rate": 1.4591999047769846e-06, + "loss": 0.73148179, + "num_input_tokens_seen": 107532005, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.1595459, + "step": 4986, + "time_per_iteration": 3.5640039443969727 + }, + { + "auxiliary_loss_clip": 0.01137246, + "auxiliary_loss_mlp": 0.0103977, + "balance_loss_clip": 1.04585671, + "balance_loss_mlp": 1.02240109, + "epoch": 0.5996512956171466, + "flos": 18916951818240.0, + "grad_norm": 1.6356840229122442, + "language_loss": 0.75356221, + "learning_rate": 1.4584499923783486e-06, + "loss": 0.77533239, + "num_input_tokens_seen": 107550585, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.17382812, + "step": 4987, + "time_per_iteration": 2.712158203125 + }, + { + "auxiliary_loss_clip": 0.01130692, + "auxiliary_loss_mlp": 0.01036947, + "balance_loss_clip": 1.04195046, + "balance_loss_mlp": 1.0209496, + "epoch": 0.5997715385077858, + "flos": 15370916330880.0, + "grad_norm": 2.5515346640276664, + "language_loss": 0.76234698, + "learning_rate": 1.457700162132419e-06, + "loss": 0.7840234, + "num_input_tokens_seen": 107567575, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.15979004, + "step": 4988, + "time_per_iteration": 2.7044811248779297 + }, + { + "auxiliary_loss_clip": 0.01133977, + "auxiliary_loss_mlp": 0.01033358, + "balance_loss_clip": 1.04369211, + "balance_loss_mlp": 1.01706219, + "epoch": 0.5998917813984248, + "flos": 25264844818560.0, + "grad_norm": 2.6038699414873245, + "language_loss": 0.72413468, + "learning_rate": 1.4569504141529433e-06, + "loss": 0.74580801, + "num_input_tokens_seen": 107585410, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.16296387, + "step": 4989, + "time_per_iteration": 3.525120496749878 + }, + { + "auxiliary_loss_clip": 0.0113597, + "auxiliary_loss_mlp": 0.01040031, + "balance_loss_clip": 1.0429709, + "balance_loss_mlp": 1.0215776, + "epoch": 0.6000120242890639, + "flos": 22054502862720.0, + "grad_norm": 2.846294297080712, + "language_loss": 0.72058105, + "learning_rate": 1.456200748553658e-06, + "loss": 0.74234104, + "num_input_tokens_seen": 107603405, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.18432617, + "step": 4990, + "time_per_iteration": 2.7280921936035156 + }, + { + "auxiliary_loss_clip": 0.0113491, + "auxiliary_loss_mlp": 0.01043919, + "balance_loss_clip": 1.04439187, + "balance_loss_mlp": 1.02800465, + "epoch": 0.600132267179703, + "flos": 29863421562240.0, + "grad_norm": 1.476140997451629, + "language_loss": 0.78713095, + "learning_rate": 1.455451165448287e-06, + "loss": 0.80891931, + "num_input_tokens_seen": 107626060, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.15905762, + "step": 4991, + "time_per_iteration": 2.797158718109131 + }, + { + "auxiliary_loss_clip": 0.01135311, + "auxiliary_loss_mlp": 0.01038913, + "balance_loss_clip": 1.04265618, + "balance_loss_mlp": 1.02249837, + "epoch": 0.6002525100703421, + "flos": 25045358762880.0, + "grad_norm": 2.9104931656393127, + "language_loss": 0.73492503, + "learning_rate": 1.4547016649505407e-06, + "loss": 0.75666732, + "num_input_tokens_seen": 107644070, + "router_z_loss_clip": 0.92675781, + "router_z_loss_mlp": 0.16430664, + "step": 4992, + "time_per_iteration": 2.7339279651641846 + }, + { + "auxiliary_loss_clip": 0.0113409, + "auxiliary_loss_mlp": 0.01036001, + "balance_loss_clip": 1.04264832, + "balance_loss_mlp": 1.01929975, + "epoch": 0.6003727529609811, + "flos": 20849592113280.0, + "grad_norm": 1.9892949315665798, + "language_loss": 0.84884405, + "learning_rate": 1.4539522471741193e-06, + "loss": 0.87054503, + "num_input_tokens_seen": 107661495, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.16711426, + "step": 4993, + "time_per_iteration": 2.7437777519226074 + }, + { + "auxiliary_loss_clip": 0.01138125, + "auxiliary_loss_mlp": 0.01045079, + "balance_loss_clip": 1.04388165, + "balance_loss_mlp": 1.02630353, + "epoch": 0.6004929958516203, + "flos": 15594604277760.0, + "grad_norm": 2.215880861984186, + "language_loss": 0.70708978, + "learning_rate": 1.4532029122327067e-06, + "loss": 0.72892183, + "num_input_tokens_seen": 107678280, + "router_z_loss_clip": 0.94238281, + "router_z_loss_mlp": 0.18774414, + "step": 4994, + "time_per_iteration": 2.742553472518921 + }, + { + "auxiliary_loss_clip": 0.01135291, + "auxiliary_loss_mlp": 0.01036915, + "balance_loss_clip": 1.04510999, + "balance_loss_mlp": 1.02120304, + "epoch": 0.6006132387422594, + "flos": 21763267390080.0, + "grad_norm": 1.8281389050536256, + "language_loss": 0.75653279, + "learning_rate": 1.4524536602399783e-06, + "loss": 0.77825487, + "num_input_tokens_seen": 107697370, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.15710449, + "step": 4995, + "time_per_iteration": 2.7744386196136475 + }, + { + "auxiliary_loss_clip": 0.01130906, + "auxiliary_loss_mlp": 0.01038362, + "balance_loss_clip": 1.04442859, + "balance_loss_mlp": 1.02266192, + "epoch": 0.6007334816328984, + "flos": 22858542852480.0, + "grad_norm": 1.4910902582652525, + "language_loss": 0.77507085, + "learning_rate": 1.4517044913095938e-06, + "loss": 0.79676348, + "num_input_tokens_seen": 107717790, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.15686035, + "step": 4996, + "time_per_iteration": 2.832836151123047 + }, + { + "auxiliary_loss_clip": 0.01134951, + "auxiliary_loss_mlp": 0.01040406, + "balance_loss_clip": 1.04315257, + "balance_loss_mlp": 1.02334714, + "epoch": 0.6008537245235376, + "flos": 28324577047680.0, + "grad_norm": 1.7138861393239229, + "language_loss": 0.81474179, + "learning_rate": 1.4509554055552022e-06, + "loss": 0.8364954, + "num_input_tokens_seen": 107738020, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.17053223, + "step": 4997, + "time_per_iteration": 2.780186653137207 + }, + { + "auxiliary_loss_clip": 0.01135566, + "auxiliary_loss_mlp": 0.01041157, + "balance_loss_clip": 1.0438509, + "balance_loss_mlp": 1.02383542, + "epoch": 0.6009739674141766, + "flos": 20886113266560.0, + "grad_norm": 2.153045001007507, + "language_loss": 0.84258282, + "learning_rate": 1.450206403090439e-06, + "loss": 0.86435008, + "num_input_tokens_seen": 107756215, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.17333984, + "step": 4998, + "time_per_iteration": 2.7552475929260254 + }, + { + "auxiliary_loss_clip": 0.0113325, + "auxiliary_loss_mlp": 0.01033067, + "balance_loss_clip": 1.04414737, + "balance_loss_mlp": 1.01817775, + "epoch": 0.6010942103048157, + "flos": 20481004702080.0, + "grad_norm": 1.9716506674540117, + "language_loss": 0.86411035, + "learning_rate": 1.4494574840289274e-06, + "loss": 0.88577354, + "num_input_tokens_seen": 107773330, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.14874268, + "step": 4999, + "time_per_iteration": 2.7650513648986816 + }, + { + "auxiliary_loss_clip": 0.01140956, + "auxiliary_loss_mlp": 0.01041681, + "balance_loss_clip": 1.04453552, + "balance_loss_mlp": 1.02391899, + "epoch": 0.6012144531954549, + "flos": 23805973935360.0, + "grad_norm": 1.5308225373819018, + "language_loss": 0.73726159, + "learning_rate": 1.4487086484842782e-06, + "loss": 0.75908792, + "num_input_tokens_seen": 107791975, + "router_z_loss_clip": 0.96435547, + "router_z_loss_mlp": 0.17749023, + "step": 5000, + "time_per_iteration": 2.858448028564453 + }, + { + "auxiliary_loss_clip": 0.0113192, + "auxiliary_loss_mlp": 0.01041804, + "balance_loss_clip": 1.04349673, + "balance_loss_mlp": 1.02592492, + "epoch": 0.6013346960860939, + "flos": 18988378012800.0, + "grad_norm": 1.9742885190290533, + "language_loss": 0.60671711, + "learning_rate": 1.4479598965700878e-06, + "loss": 0.62845433, + "num_input_tokens_seen": 107809240, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.15869141, + "step": 5001, + "time_per_iteration": 2.8594398498535156 + }, + { + "auxiliary_loss_clip": 0.01132886, + "auxiliary_loss_mlp": 0.01041513, + "balance_loss_clip": 1.04449058, + "balance_loss_mlp": 1.02466846, + "epoch": 0.601454938976733, + "flos": 24025316336640.0, + "grad_norm": 2.0873635038027145, + "language_loss": 0.69078422, + "learning_rate": 1.4472112283999427e-06, + "loss": 0.71252823, + "num_input_tokens_seen": 107827895, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.16833496, + "step": 5002, + "time_per_iteration": 2.7910196781158447 + }, + { + "auxiliary_loss_clip": 0.01130239, + "auxiliary_loss_mlp": 0.01036135, + "balance_loss_clip": 1.04241657, + "balance_loss_mlp": 1.02050698, + "epoch": 0.6015751818673721, + "flos": 26427129102720.0, + "grad_norm": 2.0179167203138655, + "language_loss": 0.69460428, + "learning_rate": 1.4464626440874143e-06, + "loss": 0.716268, + "num_input_tokens_seen": 107847010, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.15625, + "step": 5003, + "time_per_iteration": 2.8789007663726807 + }, + { + "auxiliary_loss_clip": 0.01141706, + "auxiliary_loss_mlp": 0.01039712, + "balance_loss_clip": 1.04713845, + "balance_loss_mlp": 1.02217603, + "epoch": 0.6016954247580112, + "flos": 13115260005120.0, + "grad_norm": 2.3756512751028644, + "language_loss": 0.74447978, + "learning_rate": 1.4457141437460636e-06, + "loss": 0.766294, + "num_input_tokens_seen": 107864235, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.17529297, + "step": 5004, + "time_per_iteration": 3.3056840896606445 + }, + { + "auxiliary_loss_clip": 0.01136477, + "auxiliary_loss_mlp": 0.0103911, + "balance_loss_clip": 1.04196763, + "balance_loss_mlp": 1.02065659, + "epoch": 0.6018156676486502, + "flos": 23768447201280.0, + "grad_norm": 1.8077219316637512, + "language_loss": 0.73307264, + "learning_rate": 1.444965727489436e-06, + "loss": 0.75482845, + "num_input_tokens_seen": 107883680, + "router_z_loss_clip": 0.94482422, + "router_z_loss_mlp": 0.18457031, + "step": 5005, + "time_per_iteration": 3.16636061668396 + }, + { + "auxiliary_loss_clip": 0.01135143, + "auxiliary_loss_mlp": 0.01042004, + "balance_loss_clip": 1.04457188, + "balance_loss_mlp": 1.02561331, + "epoch": 0.6019359105392894, + "flos": 26469360518400.0, + "grad_norm": 1.7064693165592766, + "language_loss": 0.63297898, + "learning_rate": 1.444217395431066e-06, + "loss": 0.65475047, + "num_input_tokens_seen": 107906220, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.16394043, + "step": 5006, + "time_per_iteration": 3.087413787841797 + }, + { + "auxiliary_loss_clip": 0.01039102, + "auxiliary_loss_mlp": 0.010004, + "balance_loss_clip": 1.01050222, + "balance_loss_mlp": 0.9986949, + "epoch": 0.6020561534299285, + "flos": 69190849728000.0, + "grad_norm": 0.8149860002269246, + "language_loss": 0.55746138, + "learning_rate": 1.4434691476844755e-06, + "loss": 0.57785642, + "num_input_tokens_seen": 107967195, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.01708984, + "step": 5007, + "time_per_iteration": 3.480309247970581 + }, + { + "auxiliary_loss_clip": 0.01131453, + "auxiliary_loss_mlp": 0.01045428, + "balance_loss_clip": 1.04466236, + "balance_loss_mlp": 1.02897716, + "epoch": 0.6021763963205675, + "flos": 21835304115840.0, + "grad_norm": 2.300640050214169, + "language_loss": 0.66717166, + "learning_rate": 1.4427209843631729e-06, + "loss": 0.68894047, + "num_input_tokens_seen": 107984245, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.16455078, + "step": 5008, + "time_per_iteration": 3.08060622215271 + }, + { + "auxiliary_loss_clip": 0.01132588, + "auxiliary_loss_mlp": 0.01041121, + "balance_loss_clip": 1.04422641, + "balance_loss_mlp": 1.02496815, + "epoch": 0.6022966392112067, + "flos": 26578636669440.0, + "grad_norm": 1.8448420232485483, + "language_loss": 0.81604487, + "learning_rate": 1.4419729055806534e-06, + "loss": 0.83778203, + "num_input_tokens_seen": 108003680, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.16137695, + "step": 5009, + "time_per_iteration": 3.0785458087921143 + }, + { + "auxiliary_loss_clip": 0.01130205, + "auxiliary_loss_mlp": 0.01041835, + "balance_loss_clip": 1.04384708, + "balance_loss_mlp": 1.02648127, + "epoch": 0.6024168821018457, + "flos": 20703722981760.0, + "grad_norm": 1.706565632819733, + "language_loss": 0.82043839, + "learning_rate": 1.441224911450401e-06, + "loss": 0.84215873, + "num_input_tokens_seen": 108019635, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.15368652, + "step": 5010, + "time_per_iteration": 4.479416370391846 + }, + { + "auxiliary_loss_clip": 0.01135525, + "auxiliary_loss_mlp": 0.01043184, + "balance_loss_clip": 1.04540181, + "balance_loss_mlp": 1.02660155, + "epoch": 0.6025371249924848, + "flos": 24680973242880.0, + "grad_norm": 1.650704297115411, + "language_loss": 0.82639229, + "learning_rate": 1.4404770020858851e-06, + "loss": 0.8481794, + "num_input_tokens_seen": 108039120, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.16577148, + "step": 5011, + "time_per_iteration": 3.118704080581665 + }, + { + "auxiliary_loss_clip": 0.0113211, + "auxiliary_loss_mlp": 0.01045837, + "balance_loss_clip": 1.0427897, + "balance_loss_mlp": 1.02920771, + "epoch": 0.602657367883124, + "flos": 25955801815680.0, + "grad_norm": 1.5751845805794338, + "language_loss": 0.86241078, + "learning_rate": 1.439729177600563e-06, + "loss": 0.88419026, + "num_input_tokens_seen": 108059615, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.16625977, + "step": 5012, + "time_per_iteration": 3.9597394466400146 + }, + { + "auxiliary_loss_clip": 0.01139499, + "auxiliary_loss_mlp": 0.01043355, + "balance_loss_clip": 1.04864943, + "balance_loss_mlp": 1.0268445, + "epoch": 0.602777610773763, + "flos": 16690633925760.0, + "grad_norm": 2.394523946787821, + "language_loss": 0.7310313, + "learning_rate": 1.4389814381078793e-06, + "loss": 0.75285989, + "num_input_tokens_seen": 108078855, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.16491699, + "step": 5013, + "time_per_iteration": 3.185319423675537 + }, + { + "auxiliary_loss_clip": 0.0113445, + "auxiliary_loss_mlp": 0.01038556, + "balance_loss_clip": 1.04522133, + "balance_loss_mlp": 1.02178955, + "epoch": 0.6028978536644021, + "flos": 13334243270400.0, + "grad_norm": 1.9087663020978418, + "language_loss": 0.7997039, + "learning_rate": 1.438233783721265e-06, + "loss": 0.8214339, + "num_input_tokens_seen": 108095020, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.16766357, + "step": 5014, + "time_per_iteration": 3.0704383850097656 + }, + { + "auxiliary_loss_clip": 0.01132157, + "auxiliary_loss_mlp": 0.01038153, + "balance_loss_clip": 1.04391599, + "balance_loss_mlp": 1.02086794, + "epoch": 0.6030180965550412, + "flos": 19644825018240.0, + "grad_norm": 2.029208768116478, + "language_loss": 0.77661306, + "learning_rate": 1.43748621455414e-06, + "loss": 0.79831612, + "num_input_tokens_seen": 108111455, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.17285156, + "step": 5015, + "time_per_iteration": 3.7612578868865967 + }, + { + "auxiliary_loss_clip": 0.01132628, + "auxiliary_loss_mlp": 0.01036826, + "balance_loss_clip": 1.04126668, + "balance_loss_mlp": 1.02030408, + "epoch": 0.6031383394456803, + "flos": 14458390289280.0, + "grad_norm": 2.7786489306480484, + "language_loss": 0.806117, + "learning_rate": 1.4367387307199082e-06, + "loss": 0.8278116, + "num_input_tokens_seen": 108128305, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.16503906, + "step": 5016, + "time_per_iteration": 3.145933151245117 + }, + { + "auxiliary_loss_clip": 0.01135332, + "auxiliary_loss_mlp": 0.01041623, + "balance_loss_clip": 1.04291511, + "balance_loss_mlp": 1.02507699, + "epoch": 0.6032585823363193, + "flos": 13917791623680.0, + "grad_norm": 2.006745943468261, + "language_loss": 0.82337755, + "learning_rate": 1.4359913323319632e-06, + "loss": 0.84514713, + "num_input_tokens_seen": 108145475, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.16540527, + "step": 5017, + "time_per_iteration": 3.1745145320892334 + }, + { + "auxiliary_loss_clip": 0.01131319, + "auxiliary_loss_mlp": 0.01047941, + "balance_loss_clip": 1.04303229, + "balance_loss_mlp": 1.03145456, + "epoch": 0.6033788252269584, + "flos": 24353252530560.0, + "grad_norm": 1.7477372633054953, + "language_loss": 0.77836579, + "learning_rate": 1.4352440195036847e-06, + "loss": 0.80015838, + "num_input_tokens_seen": 108165650, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.16491699, + "step": 5018, + "time_per_iteration": 3.8896090984344482 + }, + { + "auxiliary_loss_clip": 0.01133293, + "auxiliary_loss_mlp": 0.01048762, + "balance_loss_clip": 1.04243505, + "balance_loss_mlp": 1.03278816, + "epoch": 0.6034990681175976, + "flos": 25521247077120.0, + "grad_norm": 2.3373231768785327, + "language_loss": 0.80061334, + "learning_rate": 1.4344967923484395e-06, + "loss": 0.82243395, + "num_input_tokens_seen": 108187620, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.15979004, + "step": 5019, + "time_per_iteration": 3.0336527824401855 + }, + { + "auxiliary_loss_clip": 0.01136365, + "auxiliary_loss_mlp": 0.0104731, + "balance_loss_clip": 1.04501903, + "balance_loss_mlp": 1.03028655, + "epoch": 0.6036193110082366, + "flos": 25958387594880.0, + "grad_norm": 1.9955969208138629, + "language_loss": 0.71947551, + "learning_rate": 1.433749650979581e-06, + "loss": 0.74131227, + "num_input_tokens_seen": 108207605, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.17028809, + "step": 5020, + "time_per_iteration": 3.016148090362549 + }, + { + "auxiliary_loss_clip": 0.01131783, + "auxiliary_loss_mlp": 0.01041162, + "balance_loss_clip": 1.04156744, + "balance_loss_mlp": 1.02511621, + "epoch": 0.6037395538988757, + "flos": 25593427457280.0, + "grad_norm": 1.944076992181752, + "language_loss": 0.67895323, + "learning_rate": 1.433002595510451e-06, + "loss": 0.7006827, + "num_input_tokens_seen": 108226385, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.16052246, + "step": 5021, + "time_per_iteration": 3.0444934368133545 + }, + { + "auxiliary_loss_clip": 0.01133462, + "auxiliary_loss_mlp": 0.01038591, + "balance_loss_clip": 1.04264843, + "balance_loss_mlp": 1.02233696, + "epoch": 0.6038597967895148, + "flos": 17816253402240.0, + "grad_norm": 1.806185098061126, + "language_loss": 0.72192711, + "learning_rate": 1.4322556260543757e-06, + "loss": 0.74364763, + "num_input_tokens_seen": 108242960, + "router_z_loss_clip": 0.90771484, + "router_z_loss_mlp": 0.16265869, + "step": 5022, + "time_per_iteration": 3.049997091293335 + }, + { + "auxiliary_loss_clip": 0.01037785, + "auxiliary_loss_mlp": 0.01009061, + "balance_loss_clip": 1.00953054, + "balance_loss_mlp": 1.00747538, + "epoch": 0.6039800396801539, + "flos": 65169213235200.0, + "grad_norm": 0.8975714996521893, + "language_loss": 0.62726545, + "learning_rate": 1.4315087427246703e-06, + "loss": 0.64773393, + "num_input_tokens_seen": 108296785, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01586914, + "step": 5023, + "time_per_iteration": 3.3753631114959717 + }, + { + "auxiliary_loss_clip": 0.01037888, + "auxiliary_loss_mlp": 0.01006019, + "balance_loss_clip": 1.00959361, + "balance_loss_mlp": 1.00451744, + "epoch": 0.604100282570793, + "flos": 67386409073280.0, + "grad_norm": 0.8685845607007404, + "language_loss": 0.58466136, + "learning_rate": 1.4307619456346372e-06, + "loss": 0.60510045, + "num_input_tokens_seen": 108341090, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01501465, + "step": 5024, + "time_per_iteration": 3.176234722137451 + }, + { + "auxiliary_loss_clip": 0.01135139, + "auxiliary_loss_mlp": 0.01048378, + "balance_loss_clip": 1.04407215, + "balance_loss_mlp": 1.03227329, + "epoch": 0.6042205254614321, + "flos": 35297495631360.0, + "grad_norm": 2.077188452269335, + "language_loss": 0.74132931, + "learning_rate": 1.430015234897564e-06, + "loss": 0.76316446, + "num_input_tokens_seen": 108364370, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.16101074, + "step": 5025, + "time_per_iteration": 3.115583896636963 + }, + { + "auxiliary_loss_clip": 0.01132555, + "auxiliary_loss_mlp": 0.01039183, + "balance_loss_clip": 1.04183412, + "balance_loss_mlp": 1.02299452, + "epoch": 0.6043407683520712, + "flos": 45658262206080.0, + "grad_norm": 2.133245724672993, + "language_loss": 0.66654593, + "learning_rate": 1.4292686106267274e-06, + "loss": 0.6882633, + "num_input_tokens_seen": 108387220, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.16186523, + "step": 5026, + "time_per_iteration": 3.1685869693756104 + }, + { + "auxiliary_loss_clip": 0.01134259, + "auxiliary_loss_mlp": 0.01039536, + "balance_loss_clip": 1.04426146, + "balance_loss_mlp": 1.02376485, + "epoch": 0.6044610112427102, + "flos": 16180020138240.0, + "grad_norm": 1.578431607157526, + "language_loss": 0.76972032, + "learning_rate": 1.4285220729353876e-06, + "loss": 0.79145825, + "num_input_tokens_seen": 108405760, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.15771484, + "step": 5027, + "time_per_iteration": 3.0459539890289307 + }, + { + "auxiliary_loss_clip": 0.01133154, + "auxiliary_loss_mlp": 0.010388, + "balance_loss_clip": 1.04412198, + "balance_loss_mlp": 1.02275407, + "epoch": 0.6045812541333494, + "flos": 13804062186240.0, + "grad_norm": 1.8260190359440076, + "language_loss": 0.77919173, + "learning_rate": 1.4277756219367957e-06, + "loss": 0.80091125, + "num_input_tokens_seen": 108422785, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.16040039, + "step": 5028, + "time_per_iteration": 3.1215813159942627 + }, + { + "auxiliary_loss_clip": 0.01137353, + "auxiliary_loss_mlp": 0.01040761, + "balance_loss_clip": 1.04440165, + "balance_loss_mlp": 1.02326107, + "epoch": 0.6047014970239885, + "flos": 19975059682560.0, + "grad_norm": 2.4383374091088372, + "language_loss": 0.79544973, + "learning_rate": 1.4270292577441864e-06, + "loss": 0.81723088, + "num_input_tokens_seen": 108442290, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.17492676, + "step": 5029, + "time_per_iteration": 3.1186230182647705 + }, + { + "auxiliary_loss_clip": 0.01133138, + "auxiliary_loss_mlp": 0.01039178, + "balance_loss_clip": 1.04222012, + "balance_loss_mlp": 1.02291811, + "epoch": 0.6048217399146275, + "flos": 25337097025920.0, + "grad_norm": 1.7414823937826809, + "language_loss": 0.72026575, + "learning_rate": 1.4262829804707836e-06, + "loss": 0.7419889, + "num_input_tokens_seen": 108464280, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.16247559, + "step": 5030, + "time_per_iteration": 3.048773765563965 + }, + { + "auxiliary_loss_clip": 0.01135703, + "auxiliary_loss_mlp": 0.01040981, + "balance_loss_clip": 1.04318523, + "balance_loss_mlp": 1.02429175, + "epoch": 0.6049419828052667, + "flos": 26030819370240.0, + "grad_norm": 1.4201869661854225, + "language_loss": 0.69956195, + "learning_rate": 1.4255367902297958e-06, + "loss": 0.72132879, + "num_input_tokens_seen": 108485610, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.16674805, + "step": 5031, + "time_per_iteration": 3.074254035949707 + }, + { + "auxiliary_loss_clip": 0.01129921, + "auxiliary_loss_mlp": 0.01035724, + "balance_loss_clip": 1.04130602, + "balance_loss_mlp": 1.02017927, + "epoch": 0.6050622256959057, + "flos": 14648106948480.0, + "grad_norm": 2.0107909930498447, + "language_loss": 0.78513902, + "learning_rate": 1.4247906871344215e-06, + "loss": 0.80679548, + "num_input_tokens_seen": 108501005, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.15533447, + "step": 5032, + "time_per_iteration": 3.0359933376312256 + }, + { + "auxiliary_loss_clip": 0.01131006, + "auxiliary_loss_mlp": 0.01030548, + "balance_loss_clip": 1.04224408, + "balance_loss_mlp": 1.01456201, + "epoch": 0.6051824685865448, + "flos": 23331450337920.0, + "grad_norm": 1.9632677983972189, + "language_loss": 0.75483972, + "learning_rate": 1.4240446712978415e-06, + "loss": 0.77645528, + "num_input_tokens_seen": 108519990, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.15991211, + "step": 5033, + "time_per_iteration": 3.1125543117523193 + }, + { + "auxiliary_loss_clip": 0.01133769, + "auxiliary_loss_mlp": 0.01044956, + "balance_loss_clip": 1.04209018, + "balance_loss_mlp": 1.02682471, + "epoch": 0.605302711477184, + "flos": 27563307177600.0, + "grad_norm": 1.9519210078760374, + "language_loss": 0.74530745, + "learning_rate": 1.423298742833227e-06, + "loss": 0.76709467, + "num_input_tokens_seen": 108538650, + "router_z_loss_clip": 0.91601562, + "router_z_loss_mlp": 0.18127441, + "step": 5034, + "time_per_iteration": 2.978844165802002 + }, + { + "auxiliary_loss_clip": 0.01133584, + "auxiliary_loss_mlp": 0.01043496, + "balance_loss_clip": 1.04204762, + "balance_loss_mlp": 1.02747452, + "epoch": 0.605422954367823, + "flos": 15154698412800.0, + "grad_norm": 2.4300328099696067, + "language_loss": 0.71795332, + "learning_rate": 1.4225529018537352e-06, + "loss": 0.7397241, + "num_input_tokens_seen": 108554155, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.16003418, + "step": 5035, + "time_per_iteration": 3.0074551105499268 + }, + { + "auxiliary_loss_clip": 0.01132842, + "auxiliary_loss_mlp": 0.01040467, + "balance_loss_clip": 1.04351377, + "balance_loss_mlp": 1.02349198, + "epoch": 0.6055431972584621, + "flos": 27673912131840.0, + "grad_norm": 1.4792400840415538, + "language_loss": 0.77868658, + "learning_rate": 1.4218071484725082e-06, + "loss": 0.80041969, + "num_input_tokens_seen": 108576275, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.1697998, + "step": 5036, + "time_per_iteration": 3.057016372680664 + }, + { + "auxiliary_loss_clip": 0.01136363, + "auxiliary_loss_mlp": 0.01040238, + "balance_loss_clip": 1.04470968, + "balance_loss_mlp": 1.02396572, + "epoch": 0.6056634401491012, + "flos": 19387489006080.0, + "grad_norm": 1.8571401532128078, + "language_loss": 0.76409662, + "learning_rate": 1.4210614828026786e-06, + "loss": 0.78586257, + "num_input_tokens_seen": 108594125, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.16271973, + "step": 5037, + "time_per_iteration": 2.9766900539398193 + }, + { + "auxiliary_loss_clip": 0.01133159, + "auxiliary_loss_mlp": 0.01039092, + "balance_loss_clip": 1.04305351, + "balance_loss_mlp": 1.02249777, + "epoch": 0.6057836830397403, + "flos": 24789459294720.0, + "grad_norm": 1.515510386617841, + "language_loss": 0.74219024, + "learning_rate": 1.4203159049573605e-06, + "loss": 0.76391274, + "num_input_tokens_seen": 108615360, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.1661377, + "step": 5038, + "time_per_iteration": 3.001847505569458 + }, + { + "auxiliary_loss_clip": 0.01133124, + "auxiliary_loss_mlp": 0.01040811, + "balance_loss_clip": 1.04197311, + "balance_loss_mlp": 1.02437186, + "epoch": 0.6059039259303793, + "flos": 20558248899840.0, + "grad_norm": 2.0269997575578746, + "language_loss": 0.86961085, + "learning_rate": 1.4195704150496593e-06, + "loss": 0.89135015, + "num_input_tokens_seen": 108633075, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.16442871, + "step": 5039, + "time_per_iteration": 4.2656779289245605 + }, + { + "auxiliary_loss_clip": 0.01134781, + "auxiliary_loss_mlp": 0.01035096, + "balance_loss_clip": 1.04396629, + "balance_loss_mlp": 1.01852596, + "epoch": 0.6060241688210185, + "flos": 21069724613760.0, + "grad_norm": 1.6005434271234091, + "language_loss": 0.73730689, + "learning_rate": 1.4188250131926639e-06, + "loss": 0.75900561, + "num_input_tokens_seen": 108651875, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.16564941, + "step": 5040, + "time_per_iteration": 3.1248703002929688 + }, + { + "auxiliary_loss_clip": 0.0113699, + "auxiliary_loss_mlp": 0.01038804, + "balance_loss_clip": 1.04388261, + "balance_loss_mlp": 1.02144718, + "epoch": 0.6061444117116576, + "flos": 16361081619840.0, + "grad_norm": 1.7903287082295167, + "language_loss": 0.80623817, + "learning_rate": 1.4180796994994525e-06, + "loss": 0.82799613, + "num_input_tokens_seen": 108669290, + "router_z_loss_clip": 0.93017578, + "router_z_loss_mlp": 0.17346191, + "step": 5041, + "time_per_iteration": 3.8917887210845947 + }, + { + "auxiliary_loss_clip": 0.01132188, + "auxiliary_loss_mlp": 0.01042277, + "balance_loss_clip": 1.04267526, + "balance_loss_mlp": 1.02568328, + "epoch": 0.6062646546022966, + "flos": 21507296094720.0, + "grad_norm": 1.7029490030710062, + "language_loss": 0.71669245, + "learning_rate": 1.4173344740830877e-06, + "loss": 0.73843706, + "num_input_tokens_seen": 108688420, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.16589355, + "step": 5042, + "time_per_iteration": 3.1166677474975586 + }, + { + "auxiliary_loss_clip": 0.01136647, + "auxiliary_loss_mlp": 0.01041191, + "balance_loss_clip": 1.04742932, + "balance_loss_mlp": 1.02549088, + "epoch": 0.6063848974929358, + "flos": 38983151283840.0, + "grad_norm": 1.4866365141864972, + "language_loss": 0.70986885, + "learning_rate": 1.4165893370566206e-06, + "loss": 0.73164719, + "num_input_tokens_seen": 108712175, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.15698242, + "step": 5043, + "time_per_iteration": 3.1173808574676514 + }, + { + "auxiliary_loss_clip": 0.01133543, + "auxiliary_loss_mlp": 0.01045428, + "balance_loss_clip": 1.04294848, + "balance_loss_mlp": 1.02757049, + "epoch": 0.6065051403835748, + "flos": 19646584784640.0, + "grad_norm": 1.5288202919739409, + "language_loss": 0.77443242, + "learning_rate": 1.4158442885330865e-06, + "loss": 0.79622209, + "num_input_tokens_seen": 108730745, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.17858887, + "step": 5044, + "time_per_iteration": 3.795877456665039 + }, + { + "auxiliary_loss_clip": 0.0112992, + "auxiliary_loss_mlp": 0.01039152, + "balance_loss_clip": 1.04149604, + "balance_loss_mlp": 1.02345276, + "epoch": 0.6066253832742139, + "flos": 23513086437120.0, + "grad_norm": 1.7762866779664532, + "language_loss": 0.79012233, + "learning_rate": 1.4150993286255094e-06, + "loss": 0.81181306, + "num_input_tokens_seen": 108749995, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15698242, + "step": 5045, + "time_per_iteration": 3.054905891418457 + }, + { + "auxiliary_loss_clip": 0.01130763, + "auxiliary_loss_mlp": 0.01039594, + "balance_loss_clip": 1.04228163, + "balance_loss_mlp": 1.0236553, + "epoch": 0.6067456261648531, + "flos": 19133708440320.0, + "grad_norm": 2.1877338889088955, + "language_loss": 0.79902637, + "learning_rate": 1.4143544574468993e-06, + "loss": 0.82072985, + "num_input_tokens_seen": 108768355, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15942383, + "step": 5046, + "time_per_iteration": 2.9136087894439697 + }, + { + "auxiliary_loss_clip": 0.01131172, + "auxiliary_loss_mlp": 0.01037594, + "balance_loss_clip": 1.04219484, + "balance_loss_mlp": 1.02157235, + "epoch": 0.6068658690554921, + "flos": 20520614424960.0, + "grad_norm": 1.58782928871999, + "language_loss": 0.82368374, + "learning_rate": 1.4136096751102523e-06, + "loss": 0.84537143, + "num_input_tokens_seen": 108786685, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.16027832, + "step": 5047, + "time_per_iteration": 3.955263376235962 + }, + { + "auxiliary_loss_clip": 0.01132465, + "auxiliary_loss_mlp": 0.01042498, + "balance_loss_clip": 1.04333949, + "balance_loss_mlp": 1.0264405, + "epoch": 0.6069861119461312, + "flos": 27374560185600.0, + "grad_norm": 1.9797401185250385, + "language_loss": 0.83093709, + "learning_rate": 1.4128649817285516e-06, + "loss": 0.8526867, + "num_input_tokens_seen": 108804820, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.16064453, + "step": 5048, + "time_per_iteration": 3.0232105255126953 + }, + { + "auxiliary_loss_clip": 0.01136467, + "auxiliary_loss_mlp": 0.01039266, + "balance_loss_clip": 1.04676139, + "balance_loss_mlp": 1.02331603, + "epoch": 0.6071063548367702, + "flos": 25626500904960.0, + "grad_norm": 1.6868953746612918, + "language_loss": 0.62966996, + "learning_rate": 1.412120377414766e-06, + "loss": 0.65142733, + "num_input_tokens_seen": 108825010, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.1595459, + "step": 5049, + "time_per_iteration": 3.0028576850891113 + }, + { + "auxiliary_loss_clip": 0.01130404, + "auxiliary_loss_mlp": 0.01042989, + "balance_loss_clip": 1.04240382, + "balance_loss_mlp": 1.02724147, + "epoch": 0.6072265977274094, + "flos": 24460517520000.0, + "grad_norm": 1.6798445047632709, + "language_loss": 0.71337259, + "learning_rate": 1.4113758622818522e-06, + "loss": 0.73510647, + "num_input_tokens_seen": 108845075, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.1574707, + "step": 5050, + "time_per_iteration": 3.0897228717803955 + }, + { + "auxiliary_loss_clip": 0.01131937, + "auxiliary_loss_mlp": 0.01044453, + "balance_loss_clip": 1.04400468, + "balance_loss_mlp": 1.02750158, + "epoch": 0.6073468406180484, + "flos": 18149253413760.0, + "grad_norm": 1.7424759781540453, + "language_loss": 0.83293831, + "learning_rate": 1.410631436442751e-06, + "loss": 0.85470217, + "num_input_tokens_seen": 108863870, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.16943359, + "step": 5051, + "time_per_iteration": 2.9585375785827637 + }, + { + "auxiliary_loss_clip": 0.01135025, + "auxiliary_loss_mlp": 0.01039595, + "balance_loss_clip": 1.04277956, + "balance_loss_mlp": 1.02234554, + "epoch": 0.6074670835086875, + "flos": 20697617669760.0, + "grad_norm": 1.9586722020058311, + "language_loss": 0.86791855, + "learning_rate": 1.4098871000103936e-06, + "loss": 0.88966477, + "num_input_tokens_seen": 108882470, + "router_z_loss_clip": 0.92236328, + "router_z_loss_mlp": 0.17236328, + "step": 5052, + "time_per_iteration": 2.9157650470733643 + }, + { + "auxiliary_loss_clip": 0.01132346, + "auxiliary_loss_mlp": 0.01037431, + "balance_loss_clip": 1.04500842, + "balance_loss_mlp": 1.02190995, + "epoch": 0.6075873263993267, + "flos": 23769955572480.0, + "grad_norm": 1.6706947789388291, + "language_loss": 0.82645023, + "learning_rate": 1.409142853097693e-06, + "loss": 0.84814799, + "num_input_tokens_seen": 108902710, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15539551, + "step": 5053, + "time_per_iteration": 2.887025833129883 + }, + { + "auxiliary_loss_clip": 0.01135141, + "auxiliary_loss_mlp": 0.01035169, + "balance_loss_clip": 1.04364324, + "balance_loss_mlp": 1.01853967, + "epoch": 0.6077075692899657, + "flos": 24454484035200.0, + "grad_norm": 2.013004326887731, + "language_loss": 0.79469991, + "learning_rate": 1.408398695817553e-06, + "loss": 0.81640303, + "num_input_tokens_seen": 108919935, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.1661377, + "step": 5054, + "time_per_iteration": 2.8967785835266113 + }, + { + "auxiliary_loss_clip": 0.01140561, + "auxiliary_loss_mlp": 0.01043027, + "balance_loss_clip": 1.04559422, + "balance_loss_mlp": 1.02480006, + "epoch": 0.6078278121806048, + "flos": 27382102041600.0, + "grad_norm": 1.8830983372397647, + "language_loss": 0.70413965, + "learning_rate": 1.4076546282828593e-06, + "loss": 0.72597551, + "num_input_tokens_seen": 108942790, + "router_z_loss_clip": 0.94970703, + "router_z_loss_mlp": 0.18212891, + "step": 5055, + "time_per_iteration": 2.88547420501709 + }, + { + "auxiliary_loss_clip": 0.01135928, + "auxiliary_loss_mlp": 0.01041325, + "balance_loss_clip": 1.04295421, + "balance_loss_mlp": 1.02493382, + "epoch": 0.6079480550712439, + "flos": 38436447306240.0, + "grad_norm": 2.3248158845895697, + "language_loss": 0.66104817, + "learning_rate": 1.4069106506064874e-06, + "loss": 0.68282074, + "num_input_tokens_seen": 108964215, + "router_z_loss_clip": 0.92919922, + "router_z_loss_mlp": 0.16381836, + "step": 5056, + "time_per_iteration": 2.8739373683929443 + }, + { + "auxiliary_loss_clip": 0.01131381, + "auxiliary_loss_mlp": 0.01040239, + "balance_loss_clip": 1.04466808, + "balance_loss_mlp": 1.02445579, + "epoch": 0.608068297961883, + "flos": 25336271013120.0, + "grad_norm": 1.7294323200703605, + "language_loss": 0.78480327, + "learning_rate": 1.4061667629012989e-06, + "loss": 0.80651945, + "num_input_tokens_seen": 108984885, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15783691, + "step": 5057, + "time_per_iteration": 2.8322508335113525 + }, + { + "auxiliary_loss_clip": 0.01135007, + "auxiliary_loss_mlp": 0.01037344, + "balance_loss_clip": 1.04508841, + "balance_loss_mlp": 1.02045798, + "epoch": 0.608188540852522, + "flos": 24202463235840.0, + "grad_norm": 1.5192666387932015, + "language_loss": 0.8330518, + "learning_rate": 1.40542296528014e-06, + "loss": 0.85477531, + "num_input_tokens_seen": 109004545, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.16882324, + "step": 5058, + "time_per_iteration": 2.8390722274780273 + }, + { + "auxiliary_loss_clip": 0.01135709, + "auxiliary_loss_mlp": 0.01045328, + "balance_loss_clip": 1.04329622, + "balance_loss_mlp": 1.02933002, + "epoch": 0.6083087837431612, + "flos": 21284146851840.0, + "grad_norm": 1.7741009356759625, + "language_loss": 0.76422316, + "learning_rate": 1.4046792578558452e-06, + "loss": 0.78603351, + "num_input_tokens_seen": 109022440, + "router_z_loss_clip": 0.92333984, + "router_z_loss_mlp": 0.16003418, + "step": 5059, + "time_per_iteration": 2.8392083644866943 + }, + { + "auxiliary_loss_clip": 0.01133904, + "auxiliary_loss_mlp": 0.01041352, + "balance_loss_clip": 1.04411709, + "balance_loss_mlp": 1.02537775, + "epoch": 0.6084290266338003, + "flos": 16471435178880.0, + "grad_norm": 2.049315420095538, + "language_loss": 0.75613046, + "learning_rate": 1.4039356407412325e-06, + "loss": 0.77788305, + "num_input_tokens_seen": 109035680, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.15991211, + "step": 5060, + "time_per_iteration": 2.901491165161133 + }, + { + "auxiliary_loss_clip": 0.01037909, + "auxiliary_loss_mlp": 0.00998872, + "balance_loss_clip": 1.01009178, + "balance_loss_mlp": 0.99741757, + "epoch": 0.6085492695244393, + "flos": 66443574931200.0, + "grad_norm": 0.7838158959491139, + "language_loss": 0.5706166, + "learning_rate": 1.40319211404911e-06, + "loss": 0.59098446, + "num_input_tokens_seen": 109090680, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01452637, + "step": 5061, + "time_per_iteration": 3.2520463466644287 + }, + { + "auxiliary_loss_clip": 0.0113457, + "auxiliary_loss_mlp": 0.0104626, + "balance_loss_clip": 1.04408693, + "balance_loss_mlp": 1.02951097, + "epoch": 0.6086695124150785, + "flos": 23618986709760.0, + "grad_norm": 1.7519546538673285, + "language_loss": 0.90752417, + "learning_rate": 1.4024486778922691e-06, + "loss": 0.92933244, + "num_input_tokens_seen": 109108995, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.1673584, + "step": 5062, + "time_per_iteration": 2.9655892848968506 + }, + { + "auxiliary_loss_clip": 0.01131533, + "auxiliary_loss_mlp": 0.01043457, + "balance_loss_clip": 1.0429132, + "balance_loss_mlp": 1.02731657, + "epoch": 0.6087897553057176, + "flos": 20157054917760.0, + "grad_norm": 2.0042665938313116, + "language_loss": 0.77817118, + "learning_rate": 1.4017053323834884e-06, + "loss": 0.79992115, + "num_input_tokens_seen": 109128825, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.16137695, + "step": 5063, + "time_per_iteration": 2.9374566078186035 + }, + { + "auxiliary_loss_clip": 0.01131493, + "auxiliary_loss_mlp": 0.0104124, + "balance_loss_clip": 1.04127288, + "balance_loss_mlp": 1.02618372, + "epoch": 0.6089099981963566, + "flos": 25482535194240.0, + "grad_norm": 2.890921786098508, + "language_loss": 0.75950587, + "learning_rate": 1.4009620776355333e-06, + "loss": 0.78123319, + "num_input_tokens_seen": 109150425, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.1505127, + "step": 5064, + "time_per_iteration": 2.872081756591797 + }, + { + "auxiliary_loss_clip": 0.01132657, + "auxiliary_loss_mlp": 0.01043209, + "balance_loss_clip": 1.04264212, + "balance_loss_mlp": 1.02715158, + "epoch": 0.6090302410869958, + "flos": 25332895134720.0, + "grad_norm": 1.6652299520551814, + "language_loss": 0.7949875, + "learning_rate": 1.4002189137611553e-06, + "loss": 0.81674618, + "num_input_tokens_seen": 109169765, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.16052246, + "step": 5065, + "time_per_iteration": 2.957066774368286 + }, + { + "auxiliary_loss_clip": 0.01134468, + "auxiliary_loss_mlp": 0.01039144, + "balance_loss_clip": 1.04491544, + "balance_loss_mlp": 1.02413583, + "epoch": 0.6091504839776348, + "flos": 23987358639360.0, + "grad_norm": 1.7077373912448595, + "language_loss": 0.69879985, + "learning_rate": 1.3994758408730901e-06, + "loss": 0.72053593, + "num_input_tokens_seen": 109188950, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.15014648, + "step": 5066, + "time_per_iteration": 3.0013625621795654 + }, + { + "auxiliary_loss_clip": 0.01137714, + "auxiliary_loss_mlp": 0.01039935, + "balance_loss_clip": 1.0441196, + "balance_loss_mlp": 1.02356791, + "epoch": 0.6092707268682739, + "flos": 29643037666560.0, + "grad_norm": 1.9616663759549169, + "language_loss": 0.76256859, + "learning_rate": 1.3987328590840629e-06, + "loss": 0.78434509, + "num_input_tokens_seen": 109209895, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16357422, + "step": 5067, + "time_per_iteration": 3.060563802719116 + }, + { + "auxiliary_loss_clip": 0.01134761, + "auxiliary_loss_mlp": 0.01048652, + "balance_loss_clip": 1.0429213, + "balance_loss_mlp": 1.03230882, + "epoch": 0.609390969758913, + "flos": 24024957200640.0, + "grad_norm": 1.8296201817768805, + "language_loss": 0.8594594, + "learning_rate": 1.397989968506783e-06, + "loss": 0.88129354, + "num_input_tokens_seen": 109228905, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.16357422, + "step": 5068, + "time_per_iteration": 3.883507251739502 + }, + { + "auxiliary_loss_clip": 0.01136067, + "auxiliary_loss_mlp": 0.01050535, + "balance_loss_clip": 1.04372835, + "balance_loss_mlp": 1.03457332, + "epoch": 0.6095112126495521, + "flos": 11102143288320.0, + "grad_norm": 1.9600624087609213, + "language_loss": 0.71615553, + "learning_rate": 1.3972471692539458e-06, + "loss": 0.73802155, + "num_input_tokens_seen": 109243620, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.1595459, + "step": 5069, + "time_per_iteration": 2.9168858528137207 + }, + { + "auxiliary_loss_clip": 0.01133735, + "auxiliary_loss_mlp": 0.01048465, + "balance_loss_clip": 1.044981, + "balance_loss_mlp": 1.03355229, + "epoch": 0.6096314555401912, + "flos": 17265491187840.0, + "grad_norm": 1.999559989834814, + "language_loss": 0.75465012, + "learning_rate": 1.3965044614382348e-06, + "loss": 0.77647215, + "num_input_tokens_seen": 109259070, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.14904785, + "step": 5070, + "time_per_iteration": 3.8242452144622803 + }, + { + "auxiliary_loss_clip": 0.01139047, + "auxiliary_loss_mlp": 0.01049583, + "balance_loss_clip": 1.04489708, + "balance_loss_mlp": 1.03058124, + "epoch": 0.6097516984308303, + "flos": 21645910679040.0, + "grad_norm": 2.3499884644116484, + "language_loss": 0.7531215, + "learning_rate": 1.3957618451723162e-06, + "loss": 0.77500772, + "num_input_tokens_seen": 109275100, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.19006348, + "step": 5071, + "time_per_iteration": 2.991431951522827 + }, + { + "auxiliary_loss_clip": 0.01133745, + "auxiliary_loss_mlp": 0.01045225, + "balance_loss_clip": 1.0431236, + "balance_loss_mlp": 1.02832115, + "epoch": 0.6098719413214694, + "flos": 27199208966400.0, + "grad_norm": 1.7736077624031805, + "language_loss": 0.71061337, + "learning_rate": 1.3950193205688457e-06, + "loss": 0.7324031, + "num_input_tokens_seen": 109294825, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.16894531, + "step": 5072, + "time_per_iteration": 2.9842276573181152 + }, + { + "auxiliary_loss_clip": 0.01133393, + "auxiliary_loss_mlp": 0.01038373, + "balance_loss_clip": 1.04315782, + "balance_loss_mlp": 1.02248216, + "epoch": 0.6099921842121084, + "flos": 20412954385920.0, + "grad_norm": 2.0788920517290395, + "language_loss": 0.83939791, + "learning_rate": 1.3942768877404627e-06, + "loss": 0.86111557, + "num_input_tokens_seen": 109313790, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.15893555, + "step": 5073, + "time_per_iteration": 3.8304409980773926 + }, + { + "auxiliary_loss_clip": 0.0113238, + "auxiliary_loss_mlp": 0.01040642, + "balance_loss_clip": 1.04164267, + "balance_loss_mlp": 1.0262351, + "epoch": 0.6101124271027476, + "flos": 23366139897600.0, + "grad_norm": 1.5957915661419664, + "language_loss": 0.74022019, + "learning_rate": 1.393534546799795e-06, + "loss": 0.76195043, + "num_input_tokens_seen": 109333490, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.14434814, + "step": 5074, + "time_per_iteration": 3.0244638919830322 + }, + { + "auxiliary_loss_clip": 0.01134456, + "auxiliary_loss_mlp": 0.0105132, + "balance_loss_clip": 1.04446602, + "balance_loss_mlp": 1.03521526, + "epoch": 0.6102326699933867, + "flos": 26687840993280.0, + "grad_norm": 1.8160645831442757, + "language_loss": 0.67774796, + "learning_rate": 1.3927922978594536e-06, + "loss": 0.6996057, + "num_input_tokens_seen": 109354575, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.16101074, + "step": 5075, + "time_per_iteration": 2.9346275329589844 + }, + { + "auxiliary_loss_clip": 0.0103745, + "auxiliary_loss_mlp": 0.01005644, + "balance_loss_clip": 1.00955594, + "balance_loss_mlp": 1.00404632, + "epoch": 0.6103529128840257, + "flos": 60644612551680.0, + "grad_norm": 0.7753208321862993, + "language_loss": 0.57374167, + "learning_rate": 1.3920501410320387e-06, + "loss": 0.5941726, + "num_input_tokens_seen": 109410690, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01599121, + "step": 5076, + "time_per_iteration": 4.172625303268433 + }, + { + "auxiliary_loss_clip": 0.01133818, + "auxiliary_loss_mlp": 0.01039104, + "balance_loss_clip": 1.04227901, + "balance_loss_mlp": 1.02330887, + "epoch": 0.6104731557746649, + "flos": 19021307806080.0, + "grad_norm": 2.04502676740648, + "language_loss": 0.76050174, + "learning_rate": 1.3913080764301333e-06, + "loss": 0.78223091, + "num_input_tokens_seen": 109427650, + "router_z_loss_clip": 0.91601562, + "router_z_loss_mlp": 0.15783691, + "step": 5077, + "time_per_iteration": 2.8273069858551025 + }, + { + "auxiliary_loss_clip": 0.01132249, + "auxiliary_loss_mlp": 0.01040711, + "balance_loss_clip": 1.04204512, + "balance_loss_mlp": 1.02352047, + "epoch": 0.6105933986653039, + "flos": 23366894083200.0, + "grad_norm": 1.6915248277302835, + "language_loss": 0.71315849, + "learning_rate": 1.3905661041663085e-06, + "loss": 0.73488808, + "num_input_tokens_seen": 109448835, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.171875, + "step": 5078, + "time_per_iteration": 2.757114887237549 + }, + { + "auxiliary_loss_clip": 0.01136843, + "auxiliary_loss_mlp": 0.01041432, + "balance_loss_clip": 1.04404032, + "balance_loss_mlp": 1.02359879, + "epoch": 0.610713641555943, + "flos": 34637565006720.0, + "grad_norm": 1.9964282014682069, + "language_loss": 0.65660554, + "learning_rate": 1.389824224353122e-06, + "loss": 0.6783883, + "num_input_tokens_seen": 109470425, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.17834473, + "step": 5079, + "time_per_iteration": 2.8781540393829346 + }, + { + "auxiliary_loss_clip": 0.01134863, + "auxiliary_loss_mlp": 0.01042676, + "balance_loss_clip": 1.04369116, + "balance_loss_mlp": 1.02549815, + "epoch": 0.610833884446582, + "flos": 26646471504000.0, + "grad_norm": 1.5259235485049212, + "language_loss": 0.77009726, + "learning_rate": 1.389082437103115e-06, + "loss": 0.79187268, + "num_input_tokens_seen": 109489695, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.17175293, + "step": 5080, + "time_per_iteration": 2.756483316421509 + }, + { + "auxiliary_loss_clip": 0.01136506, + "auxiliary_loss_mlp": 0.01043879, + "balance_loss_clip": 1.04435623, + "balance_loss_mlp": 1.02715385, + "epoch": 0.6109541273372212, + "flos": 21215126868480.0, + "grad_norm": 2.0233121784940593, + "language_loss": 0.77994561, + "learning_rate": 1.3883407425288172e-06, + "loss": 0.80174947, + "num_input_tokens_seen": 109510030, + "router_z_loss_clip": 0.92138672, + "router_z_loss_mlp": 0.16723633, + "step": 5081, + "time_per_iteration": 2.8372011184692383 + }, + { + "auxiliary_loss_clip": 0.01133298, + "auxiliary_loss_mlp": 0.01043403, + "balance_loss_clip": 1.04187143, + "balance_loss_mlp": 1.02622449, + "epoch": 0.6110743702278603, + "flos": 20084084438400.0, + "grad_norm": 2.011988575512997, + "language_loss": 0.79917216, + "learning_rate": 1.3875991407427417e-06, + "loss": 0.82093918, + "num_input_tokens_seen": 109528255, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.17175293, + "step": 5082, + "time_per_iteration": 2.7486207485198975 + }, + { + "auxiliary_loss_clip": 0.01037821, + "auxiliary_loss_mlp": 0.01004887, + "balance_loss_clip": 1.00985527, + "balance_loss_mlp": 1.00332582, + "epoch": 0.6111946131184993, + "flos": 68302957438080.0, + "grad_norm": 0.770357580225887, + "language_loss": 0.58151394, + "learning_rate": 1.38685763185739e-06, + "loss": 0.60194105, + "num_input_tokens_seen": 109581915, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01556396, + "step": 5083, + "time_per_iteration": 3.3556723594665527 + }, + { + "auxiliary_loss_clip": 0.0113261, + "auxiliary_loss_mlp": 0.01033596, + "balance_loss_clip": 1.04239011, + "balance_loss_mlp": 1.01728785, + "epoch": 0.6113148560091385, + "flos": 19937676602880.0, + "grad_norm": 2.723150049281598, + "language_loss": 0.67494106, + "learning_rate": 1.3861162159852476e-06, + "loss": 0.69660312, + "num_input_tokens_seen": 109600050, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.16320801, + "step": 5084, + "time_per_iteration": 2.825706720352173 + }, + { + "auxiliary_loss_clip": 0.01135955, + "auxiliary_loss_mlp": 0.01050422, + "balance_loss_clip": 1.04460907, + "balance_loss_mlp": 1.03414989, + "epoch": 0.6114350988997775, + "flos": 23731854220800.0, + "grad_norm": 2.486953567346174, + "language_loss": 0.79447985, + "learning_rate": 1.3853748932387875e-06, + "loss": 0.81634367, + "num_input_tokens_seen": 109620690, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.16247559, + "step": 5085, + "time_per_iteration": 2.806833028793335 + }, + { + "auxiliary_loss_clip": 0.01134041, + "auxiliary_loss_mlp": 0.01037089, + "balance_loss_clip": 1.04082608, + "balance_loss_mlp": 1.02014315, + "epoch": 0.6115553417904166, + "flos": 24023700224640.0, + "grad_norm": 2.1300394076549303, + "language_loss": 0.75285602, + "learning_rate": 1.3846336637304671e-06, + "loss": 0.77456725, + "num_input_tokens_seen": 109638960, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.16937256, + "step": 5086, + "time_per_iteration": 2.8598031997680664 + }, + { + "auxiliary_loss_clip": 0.01129864, + "auxiliary_loss_mlp": 0.01034093, + "balance_loss_clip": 1.0404911, + "balance_loss_mlp": 1.01860738, + "epoch": 0.6116755846810558, + "flos": 23733542160000.0, + "grad_norm": 1.9089270451787994, + "language_loss": 0.83364338, + "learning_rate": 1.3838925275727316e-06, + "loss": 0.85528296, + "num_input_tokens_seen": 109659700, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.15490723, + "step": 5087, + "time_per_iteration": 2.795067310333252 + }, + { + "auxiliary_loss_clip": 0.01136078, + "auxiliary_loss_mlp": 0.01043651, + "balance_loss_clip": 1.04580092, + "balance_loss_mlp": 1.02785563, + "epoch": 0.6117958275716948, + "flos": 18661626967680.0, + "grad_norm": 1.7505443619397276, + "language_loss": 0.79013979, + "learning_rate": 1.3831514848780089e-06, + "loss": 0.81193709, + "num_input_tokens_seen": 109679275, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.15795898, + "step": 5088, + "time_per_iteration": 2.8481669425964355 + }, + { + "auxiliary_loss_clip": 0.01132603, + "auxiliary_loss_mlp": 0.01040385, + "balance_loss_clip": 1.04430318, + "balance_loss_mlp": 1.02412486, + "epoch": 0.6119160704623339, + "flos": 16471183783680.0, + "grad_norm": 2.1154440827604675, + "language_loss": 0.91751611, + "learning_rate": 1.3824105357587152e-06, + "loss": 0.939246, + "num_input_tokens_seen": 109696380, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16271973, + "step": 5089, + "time_per_iteration": 2.8150033950805664 + }, + { + "auxiliary_loss_clip": 0.01131489, + "auxiliary_loss_mlp": 0.01034608, + "balance_loss_clip": 1.04068041, + "balance_loss_mlp": 1.01864564, + "epoch": 0.612036313352973, + "flos": 23915465568000.0, + "grad_norm": 1.5174715225556146, + "language_loss": 0.8250581, + "learning_rate": 1.381669680327253e-06, + "loss": 0.84671915, + "num_input_tokens_seen": 109718060, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.15966797, + "step": 5090, + "time_per_iteration": 3.131606101989746 + }, + { + "auxiliary_loss_clip": 0.01133045, + "auxiliary_loss_mlp": 0.01040418, + "balance_loss_clip": 1.04466939, + "balance_loss_mlp": 1.02461123, + "epoch": 0.6121565562436121, + "flos": 26974766833920.0, + "grad_norm": 1.9943046524113275, + "language_loss": 0.71058756, + "learning_rate": 1.380928918696008e-06, + "loss": 0.73232222, + "num_input_tokens_seen": 109736830, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.15820312, + "step": 5091, + "time_per_iteration": 2.888105869293213 + }, + { + "auxiliary_loss_clip": 0.01131781, + "auxiliary_loss_mlp": 0.01036391, + "balance_loss_clip": 1.04328847, + "balance_loss_mlp": 1.01949906, + "epoch": 0.6122767991342511, + "flos": 15668867646720.0, + "grad_norm": 2.3733818487980765, + "language_loss": 0.71725053, + "learning_rate": 1.3801882509773548e-06, + "loss": 0.73893225, + "num_input_tokens_seen": 109754690, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.16906738, + "step": 5092, + "time_per_iteration": 2.9271633625030518 + }, + { + "auxiliary_loss_clip": 0.01133501, + "auxiliary_loss_mlp": 0.01043384, + "balance_loss_clip": 1.04231763, + "balance_loss_mlp": 1.02674258, + "epoch": 0.6123970420248903, + "flos": 27964321591680.0, + "grad_norm": 1.7363130598472094, + "language_loss": 0.81924677, + "learning_rate": 1.3794476772836503e-06, + "loss": 0.84101564, + "num_input_tokens_seen": 109775790, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.16650391, + "step": 5093, + "time_per_iteration": 2.994633197784424 + }, + { + "auxiliary_loss_clip": 0.01134128, + "auxiliary_loss_mlp": 0.01038238, + "balance_loss_clip": 1.04188704, + "balance_loss_mlp": 1.02067876, + "epoch": 0.6125172849155294, + "flos": 21468727866240.0, + "grad_norm": 1.6053225169066492, + "language_loss": 0.84778309, + "learning_rate": 1.3787071977272402e-06, + "loss": 0.86950672, + "num_input_tokens_seen": 109795050, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.17553711, + "step": 5094, + "time_per_iteration": 2.8319051265716553 + }, + { + "auxiliary_loss_clip": 0.0113724, + "auxiliary_loss_mlp": 0.01043817, + "balance_loss_clip": 1.04297984, + "balance_loss_mlp": 1.02682948, + "epoch": 0.6126375278061684, + "flos": 16248321849600.0, + "grad_norm": 2.8381052477787825, + "language_loss": 0.72148395, + "learning_rate": 1.3779668124204535e-06, + "loss": 0.7432946, + "num_input_tokens_seen": 109811465, + "router_z_loss_clip": 0.94238281, + "router_z_loss_mlp": 0.17004395, + "step": 5095, + "time_per_iteration": 2.896059036254883 + }, + { + "auxiliary_loss_clip": 0.0112761, + "auxiliary_loss_mlp": 0.01039969, + "balance_loss_clip": 1.04127479, + "balance_loss_mlp": 1.02404249, + "epoch": 0.6127577706968076, + "flos": 20448865008000.0, + "grad_norm": 1.7771297307467553, + "language_loss": 0.81088382, + "learning_rate": 1.3772265214756074e-06, + "loss": 0.83255959, + "num_input_tokens_seen": 109831225, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15924072, + "step": 5096, + "time_per_iteration": 3.772524356842041 + }, + { + "auxiliary_loss_clip": 0.01138422, + "auxiliary_loss_mlp": 0.0104376, + "balance_loss_clip": 1.04378819, + "balance_loss_mlp": 1.0257473, + "epoch": 0.6128780135874466, + "flos": 18260397072000.0, + "grad_norm": 1.965204125982134, + "language_loss": 0.75131917, + "learning_rate": 1.3764863250050025e-06, + "loss": 0.77314091, + "num_input_tokens_seen": 109849465, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.18017578, + "step": 5097, + "time_per_iteration": 2.841400146484375 + }, + { + "auxiliary_loss_clip": 0.01132372, + "auxiliary_loss_mlp": 0.01041378, + "balance_loss_clip": 1.04396617, + "balance_loss_mlp": 1.02471232, + "epoch": 0.6129982564780857, + "flos": 24937088192640.0, + "grad_norm": 1.7575377069880826, + "language_loss": 0.8057375, + "learning_rate": 1.3757462231209272e-06, + "loss": 0.82747501, + "num_input_tokens_seen": 109869770, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.16662598, + "step": 5098, + "time_per_iteration": 2.9459431171417236 + }, + { + "auxiliary_loss_clip": 0.01131727, + "auxiliary_loss_mlp": 0.01044445, + "balance_loss_clip": 1.04198456, + "balance_loss_mlp": 1.02792239, + "epoch": 0.6131184993687249, + "flos": 22492038430080.0, + "grad_norm": 1.8772365797246613, + "language_loss": 0.88838613, + "learning_rate": 1.3750062159356525e-06, + "loss": 0.91014779, + "num_input_tokens_seen": 109889120, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.1652832, + "step": 5099, + "time_per_iteration": 3.7443149089813232 + }, + { + "auxiliary_loss_clip": 0.01131154, + "auxiliary_loss_mlp": 0.01040904, + "balance_loss_clip": 1.04394102, + "balance_loss_mlp": 1.02472711, + "epoch": 0.6132387422593639, + "flos": 15885839750400.0, + "grad_norm": 1.6683837804528538, + "language_loss": 0.82948363, + "learning_rate": 1.3742663035614382e-06, + "loss": 0.85120416, + "num_input_tokens_seen": 109906490, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.1618042, + "step": 5100, + "time_per_iteration": 2.815321683883667 + }, + { + "auxiliary_loss_clip": 0.01135218, + "auxiliary_loss_mlp": 0.01045995, + "balance_loss_clip": 1.04407942, + "balance_loss_mlp": 1.03052139, + "epoch": 0.613358985150003, + "flos": 25411539962880.0, + "grad_norm": 1.747997636071377, + "language_loss": 0.79697484, + "learning_rate": 1.3735264861105283e-06, + "loss": 0.81878698, + "num_input_tokens_seen": 109927130, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.15490723, + "step": 5101, + "time_per_iteration": 2.887636423110962 + }, + { + "auxiliary_loss_clip": 0.01130886, + "auxiliary_loss_mlp": 0.01037503, + "balance_loss_clip": 1.0416975, + "balance_loss_mlp": 1.02181482, + "epoch": 0.6134792280406421, + "flos": 21361283308800.0, + "grad_norm": 2.0868833402484888, + "language_loss": 0.78812671, + "learning_rate": 1.372786763695152e-06, + "loss": 0.80981064, + "num_input_tokens_seen": 109945890, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.15673828, + "step": 5102, + "time_per_iteration": 3.846395492553711 + }, + { + "auxiliary_loss_clip": 0.01136507, + "auxiliary_loss_mlp": 0.01043905, + "balance_loss_clip": 1.04491103, + "balance_loss_mlp": 1.02666712, + "epoch": 0.6135994709312812, + "flos": 21211248199680.0, + "grad_norm": 1.7926146298682486, + "language_loss": 0.76994574, + "learning_rate": 1.3720471364275257e-06, + "loss": 0.79174984, + "num_input_tokens_seen": 109965535, + "router_z_loss_clip": 0.91552734, + "router_z_loss_mlp": 0.17236328, + "step": 5103, + "time_per_iteration": 2.885091543197632 + }, + { + "auxiliary_loss_clip": 0.01135433, + "auxiliary_loss_mlp": 0.01040217, + "balance_loss_clip": 1.04265213, + "balance_loss_mlp": 1.02368212, + "epoch": 0.6137197138219203, + "flos": 14794047907200.0, + "grad_norm": 2.656497068328678, + "language_loss": 0.77889997, + "learning_rate": 1.3713076044198486e-06, + "loss": 0.80065644, + "num_input_tokens_seen": 109982345, + "router_z_loss_clip": 0.92773438, + "router_z_loss_mlp": 0.16540527, + "step": 5104, + "time_per_iteration": 2.8852169513702393 + }, + { + "auxiliary_loss_clip": 0.01132508, + "auxiliary_loss_mlp": 0.01038839, + "balance_loss_clip": 1.04259169, + "balance_loss_mlp": 1.02226913, + "epoch": 0.6138399567125594, + "flos": 20084515401600.0, + "grad_norm": 1.9502508658106277, + "language_loss": 0.80925655, + "learning_rate": 1.3705681677843086e-06, + "loss": 0.83097005, + "num_input_tokens_seen": 110000940, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.16552734, + "step": 5105, + "time_per_iteration": 3.718649387359619 + }, + { + "auxiliary_loss_clip": 0.01038251, + "auxiliary_loss_mlp": 0.00998233, + "balance_loss_clip": 1.01029587, + "balance_loss_mlp": 0.99655169, + "epoch": 0.6139601996031985, + "flos": 60123838193280.0, + "grad_norm": 0.7816274535033075, + "language_loss": 0.6053189, + "learning_rate": 1.3698288266330768e-06, + "loss": 0.62568378, + "num_input_tokens_seen": 110061565, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.0168457, + "step": 5106, + "time_per_iteration": 3.477403163909912 + }, + { + "auxiliary_loss_clip": 0.01130646, + "auxiliary_loss_mlp": 0.01041175, + "balance_loss_clip": 1.04300749, + "balance_loss_mlp": 1.02558231, + "epoch": 0.6140804424938375, + "flos": 23586703361280.0, + "grad_norm": 2.1948203315149235, + "language_loss": 0.72825783, + "learning_rate": 1.3690895810783113e-06, + "loss": 0.74997604, + "num_input_tokens_seen": 110080360, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.15606689, + "step": 5107, + "time_per_iteration": 2.896306037902832 + }, + { + "auxiliary_loss_clip": 0.01133816, + "auxiliary_loss_mlp": 0.01038192, + "balance_loss_clip": 1.04197502, + "balance_loss_mlp": 1.02194428, + "epoch": 0.6142006853844767, + "flos": 21398199511680.0, + "grad_norm": 2.1701388325259257, + "language_loss": 0.71453631, + "learning_rate": 1.3683504312321543e-06, + "loss": 0.73625636, + "num_input_tokens_seen": 110100695, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.16247559, + "step": 5108, + "time_per_iteration": 2.9060873985290527 + }, + { + "auxiliary_loss_clip": 0.01135171, + "auxiliary_loss_mlp": 0.01039897, + "balance_loss_clip": 1.04303765, + "balance_loss_mlp": 1.02361274, + "epoch": 0.6143209282751158, + "flos": 12057367622400.0, + "grad_norm": 2.0250260277390812, + "language_loss": 0.80343801, + "learning_rate": 1.3676113772067355e-06, + "loss": 0.8251887, + "num_input_tokens_seen": 110117750, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.16290283, + "step": 5109, + "time_per_iteration": 2.9865849018096924 + }, + { + "auxiliary_loss_clip": 0.01134923, + "auxiliary_loss_mlp": 0.01046868, + "balance_loss_clip": 1.0429976, + "balance_loss_mlp": 1.02930856, + "epoch": 0.6144411711657548, + "flos": 25082274965760.0, + "grad_norm": 1.838114837649115, + "language_loss": 0.72907847, + "learning_rate": 1.3668724191141671e-06, + "loss": 0.75089645, + "num_input_tokens_seen": 110137020, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.17541504, + "step": 5110, + "time_per_iteration": 2.960648775100708 + }, + { + "auxiliary_loss_clip": 0.01133854, + "auxiliary_loss_mlp": 0.01041141, + "balance_loss_clip": 1.04489851, + "balance_loss_mlp": 1.02551293, + "epoch": 0.6145614140563939, + "flos": 20114069316480.0, + "grad_norm": 3.9828645056005296, + "language_loss": 0.66418988, + "learning_rate": 1.3661335570665493e-06, + "loss": 0.68593979, + "num_input_tokens_seen": 110154930, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.15625, + "step": 5111, + "time_per_iteration": 2.838186502456665 + }, + { + "auxiliary_loss_clip": 0.0113457, + "auxiliary_loss_mlp": 0.01037248, + "balance_loss_clip": 1.04460371, + "balance_loss_mlp": 1.02045155, + "epoch": 0.614681656947033, + "flos": 16800376953600.0, + "grad_norm": 2.345357782678735, + "language_loss": 0.70153761, + "learning_rate": 1.3653947911759676e-06, + "loss": 0.72325575, + "num_input_tokens_seen": 110172480, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.16809082, + "step": 5112, + "time_per_iteration": 2.9503509998321533 + }, + { + "auxiliary_loss_clip": 0.01136437, + "auxiliary_loss_mlp": 0.01043909, + "balance_loss_clip": 1.04328239, + "balance_loss_mlp": 1.02824485, + "epoch": 0.6148018998376721, + "flos": 38801587011840.0, + "grad_norm": 1.5133339398164143, + "language_loss": 0.74420798, + "learning_rate": 1.3646561215544904e-06, + "loss": 0.76601148, + "num_input_tokens_seen": 110197120, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.15661621, + "step": 5113, + "time_per_iteration": 2.9504506587982178 + }, + { + "auxiliary_loss_clip": 0.0113492, + "auxiliary_loss_mlp": 0.01041841, + "balance_loss_clip": 1.04301178, + "balance_loss_mlp": 1.0245558, + "epoch": 0.6149221427283111, + "flos": 23327032965120.0, + "grad_norm": 2.0828291452305367, + "language_loss": 0.79657757, + "learning_rate": 1.363917548314176e-06, + "loss": 0.81834519, + "num_input_tokens_seen": 110216385, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.17285156, + "step": 5114, + "time_per_iteration": 2.830963134765625 + }, + { + "auxiliary_loss_clip": 0.01134206, + "auxiliary_loss_mlp": 0.01045552, + "balance_loss_clip": 1.04200888, + "balance_loss_mlp": 1.02809978, + "epoch": 0.6150423856189503, + "flos": 22379494141440.0, + "grad_norm": 1.6171716216450127, + "language_loss": 0.73046827, + "learning_rate": 1.3631790715670626e-06, + "loss": 0.75226587, + "num_input_tokens_seen": 110234790, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.17468262, + "step": 5115, + "time_per_iteration": 2.7679035663604736 + }, + { + "auxiliary_loss_clip": 0.01128117, + "auxiliary_loss_mlp": 0.01041919, + "balance_loss_clip": 1.04182363, + "balance_loss_mlp": 1.02732801, + "epoch": 0.6151626285095894, + "flos": 18692078722560.0, + "grad_norm": 1.8940488637917796, + "language_loss": 0.85847867, + "learning_rate": 1.3624406914251783e-06, + "loss": 0.88017893, + "num_input_tokens_seen": 110251910, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.14587402, + "step": 5116, + "time_per_iteration": 2.788045883178711 + }, + { + "auxiliary_loss_clip": 0.01129767, + "auxiliary_loss_mlp": 0.01042569, + "balance_loss_clip": 1.04138279, + "balance_loss_mlp": 1.02676225, + "epoch": 0.6152828714002284, + "flos": 15851688894720.0, + "grad_norm": 1.9488516348802536, + "language_loss": 0.88173735, + "learning_rate": 1.3617024080005335e-06, + "loss": 0.90346074, + "num_input_tokens_seen": 110268810, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.15808105, + "step": 5117, + "time_per_iteration": 2.740309715270996 + }, + { + "auxiliary_loss_clip": 0.01133878, + "auxiliary_loss_mlp": 0.010372, + "balance_loss_clip": 1.04236984, + "balance_loss_mlp": 1.02046299, + "epoch": 0.6154031142908676, + "flos": 24869792062080.0, + "grad_norm": 1.5227506214434217, + "language_loss": 0.74149263, + "learning_rate": 1.3609642214051266e-06, + "loss": 0.76320338, + "num_input_tokens_seen": 110293035, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.16748047, + "step": 5118, + "time_per_iteration": 2.857987403869629 + }, + { + "auxiliary_loss_clip": 0.01134223, + "auxiliary_loss_mlp": 0.01037171, + "balance_loss_clip": 1.04384983, + "balance_loss_mlp": 1.02036273, + "epoch": 0.6155233571815066, + "flos": 19244744357760.0, + "grad_norm": 1.7727788906887423, + "language_loss": 0.65634269, + "learning_rate": 1.3602261317509385e-06, + "loss": 0.67805666, + "num_input_tokens_seen": 110309695, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.16796875, + "step": 5119, + "time_per_iteration": 2.8229265213012695 + }, + { + "auxiliary_loss_clip": 0.01136782, + "auxiliary_loss_mlp": 0.01036176, + "balance_loss_clip": 1.04300821, + "balance_loss_mlp": 1.0191772, + "epoch": 0.6156436000721457, + "flos": 18770077105920.0, + "grad_norm": 2.33345703183763, + "language_loss": 0.82778668, + "learning_rate": 1.3594881391499387e-06, + "loss": 0.84951627, + "num_input_tokens_seen": 110328610, + "router_z_loss_clip": 0.93798828, + "router_z_loss_mlp": 0.17004395, + "step": 5120, + "time_per_iteration": 2.807830810546875 + }, + { + "auxiliary_loss_clip": 0.01131697, + "auxiliary_loss_mlp": 0.01047638, + "balance_loss_clip": 1.04416704, + "balance_loss_mlp": 1.03098452, + "epoch": 0.6157638429627849, + "flos": 18041198325120.0, + "grad_norm": 2.762184833256465, + "language_loss": 0.79096764, + "learning_rate": 1.3587502437140778e-06, + "loss": 0.81276095, + "num_input_tokens_seen": 110346775, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.16662598, + "step": 5121, + "time_per_iteration": 2.9322712421417236 + }, + { + "auxiliary_loss_clip": 0.01134609, + "auxiliary_loss_mlp": 0.01041872, + "balance_loss_clip": 1.04311037, + "balance_loss_mlp": 1.02484858, + "epoch": 0.6158840858534239, + "flos": 25556726736000.0, + "grad_norm": 2.015510494244596, + "language_loss": 0.85517442, + "learning_rate": 1.3580124455552952e-06, + "loss": 0.87693918, + "num_input_tokens_seen": 110366140, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.17016602, + "step": 5122, + "time_per_iteration": 2.889434576034546 + }, + { + "auxiliary_loss_clip": 0.01134134, + "auxiliary_loss_mlp": 0.01039484, + "balance_loss_clip": 1.04468441, + "balance_loss_mlp": 1.02210367, + "epoch": 0.616004328744063, + "flos": 24640788902400.0, + "grad_norm": 1.842321658774495, + "language_loss": 0.87923139, + "learning_rate": 1.3572747447855148e-06, + "loss": 0.90096754, + "num_input_tokens_seen": 110386550, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.17382812, + "step": 5123, + "time_per_iteration": 3.002927780151367 + }, + { + "auxiliary_loss_clip": 0.01136697, + "auxiliary_loss_mlp": 0.01042959, + "balance_loss_clip": 1.04359651, + "balance_loss_mlp": 1.02607858, + "epoch": 0.6161245716347021, + "flos": 21689686379520.0, + "grad_norm": 2.497138886855247, + "language_loss": 0.69032496, + "learning_rate": 1.356537141516644e-06, + "loss": 0.71212149, + "num_input_tokens_seen": 110403970, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.16882324, + "step": 5124, + "time_per_iteration": 2.936495065689087 + }, + { + "auxiliary_loss_clip": 0.01136109, + "auxiliary_loss_mlp": 0.01035174, + "balance_loss_clip": 1.04454613, + "balance_loss_mlp": 1.01917624, + "epoch": 0.6162448145253412, + "flos": 35189225061120.0, + "grad_norm": 1.679058317723026, + "language_loss": 0.62157595, + "learning_rate": 1.3557996358605775e-06, + "loss": 0.64328873, + "num_input_tokens_seen": 110423890, + "router_z_loss_clip": 0.91552734, + "router_z_loss_mlp": 0.16003418, + "step": 5125, + "time_per_iteration": 3.71360445022583 + }, + { + "auxiliary_loss_clip": 0.01130727, + "auxiliary_loss_mlp": 0.01040351, + "balance_loss_clip": 1.04215217, + "balance_loss_mlp": 1.02570021, + "epoch": 0.6163650574159802, + "flos": 21615279356160.0, + "grad_norm": 3.3617841125459105, + "language_loss": 0.70743024, + "learning_rate": 1.3550622279291941e-06, + "loss": 0.729141, + "num_input_tokens_seen": 110442035, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.14648438, + "step": 5126, + "time_per_iteration": 2.8964688777923584 + }, + { + "auxiliary_loss_clip": 0.01132253, + "auxiliary_loss_mlp": 0.01042663, + "balance_loss_clip": 1.04261601, + "balance_loss_mlp": 1.02707016, + "epoch": 0.6164853003066194, + "flos": 24572163968640.0, + "grad_norm": 1.374230208521247, + "language_loss": 0.8318882, + "learning_rate": 1.354324917834358e-06, + "loss": 0.85363734, + "num_input_tokens_seen": 110463280, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.15600586, + "step": 5127, + "time_per_iteration": 3.7034342288970947 + }, + { + "auxiliary_loss_clip": 0.01129269, + "auxiliary_loss_mlp": 0.01038863, + "balance_loss_clip": 1.04087567, + "balance_loss_mlp": 1.02241254, + "epoch": 0.6166055431972585, + "flos": 21835986474240.0, + "grad_norm": 1.6325966925620365, + "language_loss": 0.77148926, + "learning_rate": 1.353587705687918e-06, + "loss": 0.79317063, + "num_input_tokens_seen": 110481455, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.16455078, + "step": 5128, + "time_per_iteration": 2.8785758018493652 + }, + { + "auxiliary_loss_clip": 0.01138673, + "auxiliary_loss_mlp": 0.01045343, + "balance_loss_clip": 1.04324079, + "balance_loss_mlp": 1.02704453, + "epoch": 0.6167257860878975, + "flos": 17785262943360.0, + "grad_norm": 2.7133890376434913, + "language_loss": 0.72131747, + "learning_rate": 1.3528505916017096e-06, + "loss": 0.74315763, + "num_input_tokens_seen": 110499155, + "router_z_loss_clip": 0.95507812, + "router_z_loss_mlp": 0.18286133, + "step": 5129, + "time_per_iteration": 2.9325757026672363 + }, + { + "auxiliary_loss_clip": 0.0113864, + "auxiliary_loss_mlp": 0.01037962, + "balance_loss_clip": 1.04461992, + "balance_loss_mlp": 1.02164233, + "epoch": 0.6168460289785367, + "flos": 23214811898880.0, + "grad_norm": 2.2875804802754467, + "language_loss": 0.88514161, + "learning_rate": 1.3521135756875514e-06, + "loss": 0.90690762, + "num_input_tokens_seen": 110515470, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.16320801, + "step": 5130, + "time_per_iteration": 3.7216365337371826 + }, + { + "auxiliary_loss_clip": 0.01130229, + "auxiliary_loss_mlp": 0.01041673, + "balance_loss_clip": 1.04353082, + "balance_loss_mlp": 1.02677155, + "epoch": 0.6169662718691757, + "flos": 26213281482240.0, + "grad_norm": 1.458857274415815, + "language_loss": 0.86024898, + "learning_rate": 1.3513766580572496e-06, + "loss": 0.88196796, + "num_input_tokens_seen": 110538290, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.14904785, + "step": 5131, + "time_per_iteration": 2.97330379486084 + }, + { + "auxiliary_loss_clip": 0.01132786, + "auxiliary_loss_mlp": 0.01044843, + "balance_loss_clip": 1.04385185, + "balance_loss_mlp": 1.0286541, + "epoch": 0.6170865147598148, + "flos": 19026120228480.0, + "grad_norm": 2.7226864267416495, + "language_loss": 0.77434677, + "learning_rate": 1.3506398388225924e-06, + "loss": 0.79612303, + "num_input_tokens_seen": 110555610, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.1619873, + "step": 5132, + "time_per_iteration": 2.8852128982543945 + }, + { + "auxiliary_loss_clip": 0.01133621, + "auxiliary_loss_mlp": 0.01040534, + "balance_loss_clip": 1.04510045, + "balance_loss_mlp": 1.02382112, + "epoch": 0.617206757650454, + "flos": 18260361158400.0, + "grad_norm": 1.711280228571385, + "language_loss": 0.71866745, + "learning_rate": 1.349903118095355e-06, + "loss": 0.74040896, + "num_input_tokens_seen": 110574745, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.16711426, + "step": 5133, + "time_per_iteration": 2.993798017501831 + }, + { + "auxiliary_loss_clip": 0.01133629, + "auxiliary_loss_mlp": 0.01035601, + "balance_loss_clip": 1.04250932, + "balance_loss_mlp": 1.01924562, + "epoch": 0.617327000541093, + "flos": 18186959715840.0, + "grad_norm": 1.804792208315338, + "language_loss": 0.73611742, + "learning_rate": 1.349166495987298e-06, + "loss": 0.75780964, + "num_input_tokens_seen": 110593310, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.16369629, + "step": 5134, + "time_per_iteration": 3.7827308177948 + }, + { + "auxiliary_loss_clip": 0.01039081, + "auxiliary_loss_mlp": 0.01006956, + "balance_loss_clip": 1.01089835, + "balance_loss_mlp": 1.00539482, + "epoch": 0.6174472434317321, + "flos": 61833796122240.0, + "grad_norm": 0.8229296914940615, + "language_loss": 0.60846674, + "learning_rate": 1.348429972610166e-06, + "loss": 0.62892711, + "num_input_tokens_seen": 110657615, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.015625, + "step": 5135, + "time_per_iteration": 3.3693904876708984 + }, + { + "auxiliary_loss_clip": 0.01039231, + "auxiliary_loss_mlp": 0.01004324, + "balance_loss_clip": 1.01101446, + "balance_loss_mlp": 1.00282192, + "epoch": 0.6175674863223712, + "flos": 71230970494080.0, + "grad_norm": 0.8459580847556034, + "language_loss": 0.57832086, + "learning_rate": 1.3476935480756897e-06, + "loss": 0.59875643, + "num_input_tokens_seen": 110714365, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01501465, + "step": 5136, + "time_per_iteration": 3.1880009174346924 + }, + { + "auxiliary_loss_clip": 0.01132682, + "auxiliary_loss_mlp": 0.01041439, + "balance_loss_clip": 1.04317534, + "balance_loss_mlp": 1.02411151, + "epoch": 0.6176877292130103, + "flos": 21835447770240.0, + "grad_norm": 2.01709262389916, + "language_loss": 0.75647223, + "learning_rate": 1.346957222495583e-06, + "loss": 0.7782135, + "num_input_tokens_seen": 110732160, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.17327881, + "step": 5137, + "time_per_iteration": 2.938725471496582 + }, + { + "auxiliary_loss_clip": 0.01138701, + "auxiliary_loss_mlp": 0.01042866, + "balance_loss_clip": 1.04624188, + "balance_loss_mlp": 1.02546144, + "epoch": 0.6178079721036493, + "flos": 17741738638080.0, + "grad_norm": 2.7509257697858414, + "language_loss": 0.70955002, + "learning_rate": 1.3462209959815466e-06, + "loss": 0.73136568, + "num_input_tokens_seen": 110746900, + "router_z_loss_clip": 0.92529297, + "router_z_loss_mlp": 0.1739502, + "step": 5138, + "time_per_iteration": 2.8571791648864746 + }, + { + "auxiliary_loss_clip": 0.0113298, + "auxiliary_loss_mlp": 0.0103744, + "balance_loss_clip": 1.04421163, + "balance_loss_mlp": 1.02156091, + "epoch": 0.6179282149942885, + "flos": 22633131052800.0, + "grad_norm": 1.7787251454275839, + "language_loss": 0.74750692, + "learning_rate": 1.345484868645265e-06, + "loss": 0.76921111, + "num_input_tokens_seen": 110765710, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.15881348, + "step": 5139, + "time_per_iteration": 2.84961199760437 + }, + { + "auxiliary_loss_clip": 0.01133591, + "auxiliary_loss_mlp": 0.01035043, + "balance_loss_clip": 1.04343247, + "balance_loss_mlp": 1.01798368, + "epoch": 0.6180484578849276, + "flos": 22310330503680.0, + "grad_norm": 1.7530754374700188, + "language_loss": 0.78089607, + "learning_rate": 1.3447488405984088e-06, + "loss": 0.80258238, + "num_input_tokens_seen": 110783970, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.1706543, + "step": 5140, + "time_per_iteration": 2.8875396251678467 + }, + { + "auxiliary_loss_clip": 0.01134703, + "auxiliary_loss_mlp": 0.01048032, + "balance_loss_clip": 1.04284227, + "balance_loss_mlp": 1.03098488, + "epoch": 0.6181687007755666, + "flos": 35225458905600.0, + "grad_norm": 2.2466197012300895, + "language_loss": 0.70069504, + "learning_rate": 1.3440129119526322e-06, + "loss": 0.72252238, + "num_input_tokens_seen": 110806395, + "router_z_loss_clip": 0.91894531, + "router_z_loss_mlp": 0.17041016, + "step": 5141, + "time_per_iteration": 2.9201691150665283 + }, + { + "auxiliary_loss_clip": 0.01038123, + "auxiliary_loss_mlp": 0.0100373, + "balance_loss_clip": 1.00980449, + "balance_loss_mlp": 1.00228179, + "epoch": 0.6182889436662057, + "flos": 61547370094080.0, + "grad_norm": 0.8387603665898647, + "language_loss": 0.51180643, + "learning_rate": 1.3432770828195762e-06, + "loss": 0.53222495, + "num_input_tokens_seen": 110867380, + "router_z_loss_clip": 0.28320312, + "router_z_loss_mlp": 0.01446533, + "step": 5142, + "time_per_iteration": 3.433838367462158 + }, + { + "auxiliary_loss_clip": 0.01132292, + "auxiliary_loss_mlp": 0.01037055, + "balance_loss_clip": 1.04036129, + "balance_loss_mlp": 1.02018738, + "epoch": 0.6184091865568448, + "flos": 19609991804160.0, + "grad_norm": 2.3059753881798994, + "language_loss": 0.69966859, + "learning_rate": 1.3425413533108635e-06, + "loss": 0.72136211, + "num_input_tokens_seen": 110885980, + "router_z_loss_clip": 0.91943359, + "router_z_loss_mlp": 0.16870117, + "step": 5143, + "time_per_iteration": 2.925354242324829 + }, + { + "auxiliary_loss_clip": 0.01135905, + "auxiliary_loss_mlp": 0.01045971, + "balance_loss_clip": 1.04664803, + "balance_loss_mlp": 1.02880478, + "epoch": 0.6185294294474839, + "flos": 23586882929280.0, + "grad_norm": 2.071167275054689, + "language_loss": 0.70380676, + "learning_rate": 1.341805723538105e-06, + "loss": 0.72562551, + "num_input_tokens_seen": 110906085, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.17163086, + "step": 5144, + "time_per_iteration": 2.8630402088165283 + }, + { + "auxiliary_loss_clip": 0.01136762, + "auxiliary_loss_mlp": 0.01042893, + "balance_loss_clip": 1.04440558, + "balance_loss_mlp": 1.02690744, + "epoch": 0.618649672338123, + "flos": 26762032535040.0, + "grad_norm": 1.4945624526188996, + "language_loss": 0.7714045, + "learning_rate": 1.3410701936128948e-06, + "loss": 0.79320103, + "num_input_tokens_seen": 110928865, + "router_z_loss_clip": 0.92382812, + "router_z_loss_mlp": 0.15966797, + "step": 5145, + "time_per_iteration": 3.0217933654785156 + }, + { + "auxiliary_loss_clip": 0.01137094, + "auxiliary_loss_mlp": 0.01048802, + "balance_loss_clip": 1.04532611, + "balance_loss_mlp": 1.03251791, + "epoch": 0.6187699152287621, + "flos": 14456630522880.0, + "grad_norm": 2.4601588394504597, + "language_loss": 0.85141265, + "learning_rate": 1.340334763646812e-06, + "loss": 0.87327158, + "num_input_tokens_seen": 110943000, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.16296387, + "step": 5146, + "time_per_iteration": 3.0043654441833496 + }, + { + "auxiliary_loss_clip": 0.01135617, + "auxiliary_loss_mlp": 0.01044461, + "balance_loss_clip": 1.04267812, + "balance_loss_mlp": 1.02740169, + "epoch": 0.6188901581194012, + "flos": 20084766796800.0, + "grad_norm": 1.5858168165466118, + "language_loss": 0.74198121, + "learning_rate": 1.3395994337514218e-06, + "loss": 0.76378202, + "num_input_tokens_seen": 110963170, + "router_z_loss_clip": 0.93017578, + "router_z_loss_mlp": 0.1706543, + "step": 5147, + "time_per_iteration": 2.8641955852508545 + }, + { + "auxiliary_loss_clip": 0.01130319, + "auxiliary_loss_mlp": 0.01038609, + "balance_loss_clip": 1.04134429, + "balance_loss_mlp": 1.02376795, + "epoch": 0.6190104010100402, + "flos": 25700728360320.0, + "grad_norm": 1.5549177766192082, + "language_loss": 0.78737605, + "learning_rate": 1.3388642040382725e-06, + "loss": 0.80906528, + "num_input_tokens_seen": 110983595, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.1484375, + "step": 5148, + "time_per_iteration": 2.8885886669158936 + }, + { + "auxiliary_loss_clip": 0.01132962, + "auxiliary_loss_mlp": 0.01039089, + "balance_loss_clip": 1.04409981, + "balance_loss_mlp": 1.02197075, + "epoch": 0.6191306439006794, + "flos": 30442372974720.0, + "grad_norm": 1.7407772430946489, + "language_loss": 0.84435785, + "learning_rate": 1.3381290746188975e-06, + "loss": 0.86607838, + "num_input_tokens_seen": 111002965, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.17138672, + "step": 5149, + "time_per_iteration": 3.0515761375427246 + }, + { + "auxiliary_loss_clip": 0.01136121, + "auxiliary_loss_mlp": 0.01042016, + "balance_loss_clip": 1.04451609, + "balance_loss_mlp": 1.02629209, + "epoch": 0.6192508867913185, + "flos": 26685793918080.0, + "grad_norm": 1.6952165505299703, + "language_loss": 0.67205358, + "learning_rate": 1.3373940456048152e-06, + "loss": 0.6938349, + "num_input_tokens_seen": 111022990, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.15734863, + "step": 5150, + "time_per_iteration": 2.9764316082000732 + }, + { + "auxiliary_loss_clip": 0.01132374, + "auxiliary_loss_mlp": 0.01039167, + "balance_loss_clip": 1.0441395, + "balance_loss_mlp": 1.02381253, + "epoch": 0.6193711296819575, + "flos": 36722036090880.0, + "grad_norm": 1.7559299020302255, + "language_loss": 0.59418702, + "learning_rate": 1.3366591171075299e-06, + "loss": 0.61590242, + "num_input_tokens_seen": 111046495, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15356445, + "step": 5151, + "time_per_iteration": 2.9519991874694824 + }, + { + "auxiliary_loss_clip": 0.01132449, + "auxiliary_loss_mlp": 0.0103808, + "balance_loss_clip": 1.04293883, + "balance_loss_mlp": 1.02264857, + "epoch": 0.6194913725725967, + "flos": 25192556697600.0, + "grad_norm": 2.4417703698736077, + "language_loss": 0.90964705, + "learning_rate": 1.335924289238529e-06, + "loss": 0.93135238, + "num_input_tokens_seen": 111065705, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.15429688, + "step": 5152, + "time_per_iteration": 2.8848085403442383 + }, + { + "auxiliary_loss_clip": 0.01134454, + "auxiliary_loss_mlp": 0.01046857, + "balance_loss_clip": 1.04526925, + "balance_loss_mlp": 1.02969134, + "epoch": 0.6196116154632357, + "flos": 21178821196800.0, + "grad_norm": 1.8097353722344514, + "language_loss": 0.76461983, + "learning_rate": 1.3351895621092859e-06, + "loss": 0.78643298, + "num_input_tokens_seen": 111086050, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.17175293, + "step": 5153, + "time_per_iteration": 2.9225339889526367 + }, + { + "auxiliary_loss_clip": 0.01128297, + "auxiliary_loss_mlp": 0.0104005, + "balance_loss_clip": 1.04181385, + "balance_loss_mlp": 1.02423072, + "epoch": 0.6197318583538748, + "flos": 16253744803200.0, + "grad_norm": 1.9182700396886228, + "language_loss": 0.76434022, + "learning_rate": 1.3344549358312567e-06, + "loss": 0.78602368, + "num_input_tokens_seen": 111104450, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15820312, + "step": 5154, + "time_per_iteration": 3.639493942260742 + }, + { + "auxiliary_loss_clip": 0.01138883, + "auxiliary_loss_mlp": 0.0103607, + "balance_loss_clip": 1.04508066, + "balance_loss_mlp": 1.01944017, + "epoch": 0.619852101244514, + "flos": 24425612478720.0, + "grad_norm": 2.0053021476495094, + "language_loss": 0.78230369, + "learning_rate": 1.3337204105158852e-06, + "loss": 0.80405319, + "num_input_tokens_seen": 111123320, + "router_z_loss_clip": 0.93798828, + "router_z_loss_mlp": 0.16638184, + "step": 5155, + "time_per_iteration": 2.9540114402770996 + }, + { + "auxiliary_loss_clip": 0.01133393, + "auxiliary_loss_mlp": 0.01039241, + "balance_loss_clip": 1.04311681, + "balance_loss_mlp": 1.02331448, + "epoch": 0.619972344135153, + "flos": 16727298733440.0, + "grad_norm": 2.9420348413979553, + "language_loss": 0.72859347, + "learning_rate": 1.332985986274597e-06, + "loss": 0.75031984, + "num_input_tokens_seen": 111140950, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.15917969, + "step": 5156, + "time_per_iteration": 3.682438850402832 + }, + { + "auxiliary_loss_clip": 0.01129765, + "auxiliary_loss_mlp": 0.01039902, + "balance_loss_clip": 1.04439688, + "balance_loss_mlp": 1.02389765, + "epoch": 0.6200925870257921, + "flos": 12495190498560.0, + "grad_norm": 2.5245763227962827, + "language_loss": 0.75415063, + "learning_rate": 1.3322516632188047e-06, + "loss": 0.77584726, + "num_input_tokens_seen": 111157845, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.16009521, + "step": 5157, + "time_per_iteration": 2.9023146629333496 + }, + { + "auxiliary_loss_clip": 0.01136434, + "auxiliary_loss_mlp": 0.01041013, + "balance_loss_clip": 1.04398584, + "balance_loss_mlp": 1.02327514, + "epoch": 0.6202128299164312, + "flos": 26539350168960.0, + "grad_norm": 2.031973948848299, + "language_loss": 0.66684955, + "learning_rate": 1.3315174414599045e-06, + "loss": 0.68862402, + "num_input_tokens_seen": 111179165, + "router_z_loss_clip": 0.92529297, + "router_z_loss_mlp": 0.17724609, + "step": 5158, + "time_per_iteration": 2.91772723197937 + }, + { + "auxiliary_loss_clip": 0.01133973, + "auxiliary_loss_mlp": 0.01040387, + "balance_loss_clip": 1.0433383, + "balance_loss_mlp": 1.02366197, + "epoch": 0.6203330728070703, + "flos": 18770508069120.0, + "grad_norm": 1.7230225678317592, + "language_loss": 0.75369239, + "learning_rate": 1.3307833211092768e-06, + "loss": 0.77543604, + "num_input_tokens_seen": 111197830, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.16723633, + "step": 5159, + "time_per_iteration": 3.666417121887207 + }, + { + "auxiliary_loss_clip": 0.01131123, + "auxiliary_loss_mlp": 0.01041877, + "balance_loss_clip": 1.04295421, + "balance_loss_mlp": 1.02567673, + "epoch": 0.6204533156977093, + "flos": 20629782835200.0, + "grad_norm": 1.538669262353589, + "language_loss": 0.75101161, + "learning_rate": 1.3300493022782873e-06, + "loss": 0.77274156, + "num_input_tokens_seen": 111218400, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.16186523, + "step": 5160, + "time_per_iteration": 2.985206127166748 + }, + { + "auxiliary_loss_clip": 0.01130178, + "auxiliary_loss_mlp": 0.01046461, + "balance_loss_clip": 1.0413518, + "balance_loss_mlp": 1.03033781, + "epoch": 0.6205735585883485, + "flos": 17348050598400.0, + "grad_norm": 1.813328755603711, + "language_loss": 0.72618455, + "learning_rate": 1.3293153850782855e-06, + "loss": 0.74795091, + "num_input_tokens_seen": 111236720, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.16119385, + "step": 5161, + "time_per_iteration": 2.9628329277038574 + }, + { + "auxiliary_loss_clip": 0.01137577, + "auxiliary_loss_mlp": 0.01048696, + "balance_loss_clip": 1.04299283, + "balance_loss_mlp": 1.03100562, + "epoch": 0.6206938014789876, + "flos": 22965017742720.0, + "grad_norm": 1.9861391927011565, + "language_loss": 0.71169448, + "learning_rate": 1.3285815696206069e-06, + "loss": 0.73355722, + "num_input_tokens_seen": 111258265, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.17687988, + "step": 5162, + "time_per_iteration": 2.868576765060425 + }, + { + "auxiliary_loss_clip": 0.01134789, + "auxiliary_loss_mlp": 0.01039422, + "balance_loss_clip": 1.04086399, + "balance_loss_mlp": 1.02262497, + "epoch": 0.6208140443696266, + "flos": 23983192661760.0, + "grad_norm": 1.8818469685456185, + "language_loss": 0.76897848, + "learning_rate": 1.32784785601657e-06, + "loss": 0.79072064, + "num_input_tokens_seen": 111277675, + "router_z_loss_clip": 0.93945312, + "router_z_loss_mlp": 0.16809082, + "step": 5163, + "time_per_iteration": 3.8378241062164307 + }, + { + "auxiliary_loss_clip": 0.0113043, + "auxiliary_loss_mlp": 0.01043271, + "balance_loss_clip": 1.04199278, + "balance_loss_mlp": 1.02651644, + "epoch": 0.6209342872602658, + "flos": 35077291303680.0, + "grad_norm": 2.370262438960613, + "language_loss": 0.74028897, + "learning_rate": 1.3271142443774798e-06, + "loss": 0.76202601, + "num_input_tokens_seen": 111299910, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.16760254, + "step": 5164, + "time_per_iteration": 2.932523250579834 + }, + { + "auxiliary_loss_clip": 0.01131862, + "auxiliary_loss_mlp": 0.01029882, + "balance_loss_clip": 1.04273856, + "balance_loss_mlp": 1.01525545, + "epoch": 0.6210545301509048, + "flos": 26979327861120.0, + "grad_norm": 1.8260031694070087, + "language_loss": 0.81572807, + "learning_rate": 1.3263807348146228e-06, + "loss": 0.83734548, + "num_input_tokens_seen": 111319765, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.1463623, + "step": 5165, + "time_per_iteration": 2.87570858001709 + }, + { + "auxiliary_loss_clip": 0.01135381, + "auxiliary_loss_mlp": 0.0104178, + "balance_loss_clip": 1.04310846, + "balance_loss_mlp": 1.02511501, + "epoch": 0.6211747730415439, + "flos": 33618240852480.0, + "grad_norm": 1.9666815569481702, + "language_loss": 0.73333406, + "learning_rate": 1.3256473274392733e-06, + "loss": 0.75510561, + "num_input_tokens_seen": 111341110, + "router_z_loss_clip": 0.92333984, + "router_z_loss_mlp": 0.16674805, + "step": 5166, + "time_per_iteration": 2.9920687675476074 + }, + { + "auxiliary_loss_clip": 0.01132974, + "auxiliary_loss_mlp": 0.0103805, + "balance_loss_clip": 1.04306579, + "balance_loss_mlp": 1.02155101, + "epoch": 0.6212950159321831, + "flos": 34167099646080.0, + "grad_norm": 1.786486460406995, + "language_loss": 0.70230591, + "learning_rate": 1.3249140223626873e-06, + "loss": 0.72401619, + "num_input_tokens_seen": 111362730, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.16503906, + "step": 5167, + "time_per_iteration": 2.967971086502075 + }, + { + "auxiliary_loss_clip": 0.01131484, + "auxiliary_loss_mlp": 0.01054693, + "balance_loss_clip": 1.04365337, + "balance_loss_mlp": 1.03813505, + "epoch": 0.6214152588228221, + "flos": 27965758135680.0, + "grad_norm": 2.7578758551038836, + "language_loss": 0.75804234, + "learning_rate": 1.3241808196961077e-06, + "loss": 0.77990407, + "num_input_tokens_seen": 111383855, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.16552734, + "step": 5168, + "time_per_iteration": 3.053150177001953 + }, + { + "auxiliary_loss_clip": 0.01128354, + "auxiliary_loss_mlp": 0.01037565, + "balance_loss_clip": 1.04167247, + "balance_loss_mlp": 1.02181768, + "epoch": 0.6215355017134612, + "flos": 20230204965120.0, + "grad_norm": 1.6415082020159415, + "language_loss": 0.70990813, + "learning_rate": 1.3234477195507608e-06, + "loss": 0.73156726, + "num_input_tokens_seen": 111402685, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.1574707, + "step": 5169, + "time_per_iteration": 2.9913878440856934 + }, + { + "auxiliary_loss_clip": 0.01133246, + "auxiliary_loss_mlp": 0.01041381, + "balance_loss_clip": 1.04267037, + "balance_loss_mlp": 1.02425075, + "epoch": 0.6216557446041003, + "flos": 41428129219200.0, + "grad_norm": 2.0367486229961536, + "language_loss": 0.62363935, + "learning_rate": 1.322714722037857e-06, + "loss": 0.64538562, + "num_input_tokens_seen": 111424130, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.17114258, + "step": 5170, + "time_per_iteration": 2.970590591430664 + }, + { + "auxiliary_loss_clip": 0.01138626, + "auxiliary_loss_mlp": 0.01053198, + "balance_loss_clip": 1.04398739, + "balance_loss_mlp": 1.03551936, + "epoch": 0.6217759874947394, + "flos": 27928770105600.0, + "grad_norm": 2.0674561271137764, + "language_loss": 0.77417862, + "learning_rate": 1.321981827268591e-06, + "loss": 0.79609692, + "num_input_tokens_seen": 111444785, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.17675781, + "step": 5171, + "time_per_iteration": 2.984441041946411 + }, + { + "auxiliary_loss_clip": 0.01130344, + "auxiliary_loss_mlp": 0.01034995, + "balance_loss_clip": 1.0408535, + "balance_loss_mlp": 1.01931906, + "epoch": 0.6218962303853784, + "flos": 21765673601280.0, + "grad_norm": 1.8907318479683901, + "language_loss": 0.81450713, + "learning_rate": 1.3212490353541426e-06, + "loss": 0.83616054, + "num_input_tokens_seen": 111467045, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.15686035, + "step": 5172, + "time_per_iteration": 2.9256861209869385 + }, + { + "auxiliary_loss_clip": 0.01134736, + "auxiliary_loss_mlp": 0.01044348, + "balance_loss_clip": 1.04392242, + "balance_loss_mlp": 1.02776551, + "epoch": 0.6220164732760175, + "flos": 21246260981760.0, + "grad_norm": 2.1428839009306837, + "language_loss": 0.80523992, + "learning_rate": 1.3205163464056762e-06, + "loss": 0.82703078, + "num_input_tokens_seen": 111483650, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.16577148, + "step": 5173, + "time_per_iteration": 3.0813639163970947 + }, + { + "auxiliary_loss_clip": 0.01131663, + "auxiliary_loss_mlp": 0.01038008, + "balance_loss_clip": 1.04081917, + "balance_loss_mlp": 1.02181959, + "epoch": 0.6221367161666567, + "flos": 26136360506880.0, + "grad_norm": 2.602724704738717, + "language_loss": 0.72743565, + "learning_rate": 1.319783760534339e-06, + "loss": 0.74913239, + "num_input_tokens_seen": 111502895, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.1619873, + "step": 5174, + "time_per_iteration": 2.8877429962158203 + }, + { + "auxiliary_loss_clip": 0.01133597, + "auxiliary_loss_mlp": 0.0104059, + "balance_loss_clip": 1.04270744, + "balance_loss_mlp": 1.02349544, + "epoch": 0.6222569590572957, + "flos": 16284196558080.0, + "grad_norm": 2.023232434352128, + "language_loss": 0.75731051, + "learning_rate": 1.319051277851266e-06, + "loss": 0.77905238, + "num_input_tokens_seen": 111519180, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.17114258, + "step": 5175, + "time_per_iteration": 2.8596677780151367 + }, + { + "auxiliary_loss_clip": 0.01131798, + "auxiliary_loss_mlp": 0.01036944, + "balance_loss_clip": 1.0432179, + "balance_loss_mlp": 1.02089894, + "epoch": 0.6223772019479348, + "flos": 18223840005120.0, + "grad_norm": 2.0692798686588185, + "language_loss": 0.84313911, + "learning_rate": 1.3183188984675716e-06, + "loss": 0.86482656, + "num_input_tokens_seen": 111537545, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.16027832, + "step": 5176, + "time_per_iteration": 2.8524444103240967 + }, + { + "auxiliary_loss_clip": 0.0113424, + "auxiliary_loss_mlp": 0.01040849, + "balance_loss_clip": 1.04384577, + "balance_loss_mlp": 1.02407646, + "epoch": 0.6224974448385739, + "flos": 27489797994240.0, + "grad_norm": 2.1226295290488357, + "language_loss": 0.71002471, + "learning_rate": 1.3175866224943586e-06, + "loss": 0.73177564, + "num_input_tokens_seen": 111556265, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.16772461, + "step": 5177, + "time_per_iteration": 2.8625221252441406 + }, + { + "auxiliary_loss_clip": 0.01137373, + "auxiliary_loss_mlp": 0.01041077, + "balance_loss_clip": 1.04510593, + "balance_loss_mlp": 1.02430391, + "epoch": 0.622617687729213, + "flos": 19791951125760.0, + "grad_norm": 2.1087836106734614, + "language_loss": 0.73376018, + "learning_rate": 1.316854450042712e-06, + "loss": 0.75554466, + "num_input_tokens_seen": 111574205, + "router_z_loss_clip": 0.92333984, + "router_z_loss_mlp": 0.16784668, + "step": 5178, + "time_per_iteration": 2.87040114402771 + }, + { + "auxiliary_loss_clip": 0.01137199, + "auxiliary_loss_mlp": 0.01041343, + "balance_loss_clip": 1.04474998, + "balance_loss_mlp": 1.02578568, + "epoch": 0.622737930619852, + "flos": 23038886062080.0, + "grad_norm": 1.9619190816499827, + "language_loss": 0.74490708, + "learning_rate": 1.3161223812237024e-06, + "loss": 0.76669246, + "num_input_tokens_seen": 111593560, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.15563965, + "step": 5179, + "time_per_iteration": 2.92568302154541 + }, + { + "auxiliary_loss_clip": 0.01132879, + "auxiliary_loss_mlp": 0.01047206, + "balance_loss_clip": 1.04216671, + "balance_loss_mlp": 1.0310055, + "epoch": 0.6228581735104912, + "flos": 12634271959680.0, + "grad_norm": 2.7089189895815036, + "language_loss": 0.85392314, + "learning_rate": 1.3153904161483842e-06, + "loss": 0.87572396, + "num_input_tokens_seen": 111608860, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.16186523, + "step": 5180, + "time_per_iteration": 3.027026653289795 + }, + { + "auxiliary_loss_clip": 0.01136748, + "auxiliary_loss_mlp": 0.01043378, + "balance_loss_clip": 1.04213107, + "balance_loss_mlp": 1.02518678, + "epoch": 0.6229784164011303, + "flos": 23802813538560.0, + "grad_norm": 2.186524955623491, + "language_loss": 0.85362047, + "learning_rate": 1.3146585549277953e-06, + "loss": 0.8754217, + "num_input_tokens_seen": 111627500, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.1817627, + "step": 5181, + "time_per_iteration": 3.0316362380981445 + }, + { + "auxiliary_loss_clip": 0.01136361, + "auxiliary_loss_mlp": 0.01039927, + "balance_loss_clip": 1.04542923, + "balance_loss_mlp": 1.02308285, + "epoch": 0.6230986592917693, + "flos": 22414219614720.0, + "grad_norm": 2.4743330890384576, + "language_loss": 0.78578293, + "learning_rate": 1.3139267976729591e-06, + "loss": 0.80754584, + "num_input_tokens_seen": 111647690, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.16845703, + "step": 5182, + "time_per_iteration": 2.9342644214630127 + }, + { + "auxiliary_loss_clip": 0.01133842, + "auxiliary_loss_mlp": 0.01043662, + "balance_loss_clip": 1.04389703, + "balance_loss_mlp": 1.02650821, + "epoch": 0.6232189021824085, + "flos": 34528217028480.0, + "grad_norm": 1.754968815301863, + "language_loss": 0.71807933, + "learning_rate": 1.3131951444948815e-06, + "loss": 0.7398544, + "num_input_tokens_seen": 111667090, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.17150879, + "step": 5183, + "time_per_iteration": 3.7942299842834473 + }, + { + "auxiliary_loss_clip": 0.01136064, + "auxiliary_loss_mlp": 0.01041634, + "balance_loss_clip": 1.0461936, + "balance_loss_mlp": 1.02338302, + "epoch": 0.6233391450730476, + "flos": 22237000888320.0, + "grad_norm": 1.8584636095717082, + "language_loss": 0.76227772, + "learning_rate": 1.3124635955045546e-06, + "loss": 0.78405464, + "num_input_tokens_seen": 111686905, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.18249512, + "step": 5184, + "time_per_iteration": 2.892822504043579 + }, + { + "auxiliary_loss_clip": 0.01131014, + "auxiliary_loss_mlp": 0.01039871, + "balance_loss_clip": 1.04163742, + "balance_loss_mlp": 1.0237062, + "epoch": 0.6234593879636866, + "flos": 20332693445760.0, + "grad_norm": 1.8671146467152184, + "language_loss": 0.84220493, + "learning_rate": 1.3117321508129537e-06, + "loss": 0.86391377, + "num_input_tokens_seen": 111704985, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.16174316, + "step": 5185, + "time_per_iteration": 2.9523544311523438 + }, + { + "auxiliary_loss_clip": 0.01136203, + "auxiliary_loss_mlp": 0.01037839, + "balance_loss_clip": 1.04595232, + "balance_loss_mlp": 1.02103066, + "epoch": 0.6235796308543258, + "flos": 20664903358080.0, + "grad_norm": 1.6338911071727655, + "language_loss": 0.76602244, + "learning_rate": 1.3110008105310388e-06, + "loss": 0.78776288, + "num_input_tokens_seen": 111724805, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.16796875, + "step": 5186, + "time_per_iteration": 3.7847163677215576 + }, + { + "auxiliary_loss_clip": 0.01135067, + "auxiliary_loss_mlp": 0.0103573, + "balance_loss_clip": 1.04296029, + "balance_loss_mlp": 1.01863551, + "epoch": 0.6236998737449648, + "flos": 26618641441920.0, + "grad_norm": 1.6196928796661934, + "language_loss": 0.78139704, + "learning_rate": 1.3102695747697526e-06, + "loss": 0.803105, + "num_input_tokens_seen": 111747675, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.17077637, + "step": 5187, + "time_per_iteration": 2.909142017364502 + }, + { + "auxiliary_loss_clip": 0.01136207, + "auxiliary_loss_mlp": 0.0104126, + "balance_loss_clip": 1.04458904, + "balance_loss_mlp": 1.02429628, + "epoch": 0.6238201166356039, + "flos": 12674599954560.0, + "grad_norm": 2.205207231899486, + "language_loss": 0.90487802, + "learning_rate": 1.3095384436400237e-06, + "loss": 0.92665267, + "num_input_tokens_seen": 111759205, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.16967773, + "step": 5188, + "time_per_iteration": 3.7774100303649902 + }, + { + "auxiliary_loss_clip": 0.01137658, + "auxiliary_loss_mlp": 0.01040802, + "balance_loss_clip": 1.04657221, + "balance_loss_mlp": 1.02462518, + "epoch": 0.623940359526243, + "flos": 10452160730880.0, + "grad_norm": 2.0239336977765814, + "language_loss": 0.82665908, + "learning_rate": 1.3088074172527633e-06, + "loss": 0.84844369, + "num_input_tokens_seen": 111776335, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.16186523, + "step": 5189, + "time_per_iteration": 2.9320006370544434 + }, + { + "auxiliary_loss_clip": 0.01133958, + "auxiliary_loss_mlp": 0.01034039, + "balance_loss_clip": 1.04135728, + "balance_loss_mlp": 1.01836896, + "epoch": 0.6240606024168821, + "flos": 29059525226880.0, + "grad_norm": 2.026505027543054, + "language_loss": 0.71744025, + "learning_rate": 1.3080764957188684e-06, + "loss": 0.73912024, + "num_input_tokens_seen": 111796580, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.15679932, + "step": 5190, + "time_per_iteration": 2.816060781478882 + }, + { + "auxiliary_loss_clip": 0.01134589, + "auxiliary_loss_mlp": 0.01040204, + "balance_loss_clip": 1.04348838, + "balance_loss_mlp": 1.02328837, + "epoch": 0.6241808453075212, + "flos": 22018089450240.0, + "grad_norm": 1.729476655250802, + "language_loss": 0.7095843, + "learning_rate": 1.3073456791492192e-06, + "loss": 0.73133224, + "num_input_tokens_seen": 111816290, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.16918945, + "step": 5191, + "time_per_iteration": 3.7675087451934814 + }, + { + "auxiliary_loss_clip": 0.01133601, + "auxiliary_loss_mlp": 0.01047617, + "balance_loss_clip": 1.04323077, + "balance_loss_mlp": 1.03170252, + "epoch": 0.6243010881981603, + "flos": 21138708683520.0, + "grad_norm": 1.8565835638299175, + "language_loss": 0.7806679, + "learning_rate": 1.3066149676546801e-06, + "loss": 0.8024801, + "num_input_tokens_seen": 111834470, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.15917969, + "step": 5192, + "time_per_iteration": 2.9095852375030518 + }, + { + "auxiliary_loss_clip": 0.0113308, + "auxiliary_loss_mlp": 0.01042328, + "balance_loss_clip": 1.0436852, + "balance_loss_mlp": 1.02787983, + "epoch": 0.6244213310887994, + "flos": 22344948236160.0, + "grad_norm": 1.8170624635855668, + "language_loss": 0.65497249, + "learning_rate": 1.3058843613460985e-06, + "loss": 0.67672658, + "num_input_tokens_seen": 111852410, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.14453125, + "step": 5193, + "time_per_iteration": 2.814678192138672 + }, + { + "auxiliary_loss_clip": 0.01135048, + "auxiliary_loss_mlp": 0.01037978, + "balance_loss_clip": 1.04249334, + "balance_loss_mlp": 1.02025151, + "epoch": 0.6245415739794384, + "flos": 15231978524160.0, + "grad_norm": 2.563186617286095, + "language_loss": 0.74469554, + "learning_rate": 1.3051538603343075e-06, + "loss": 0.76642579, + "num_input_tokens_seen": 111870340, + "router_z_loss_clip": 0.92529297, + "router_z_loss_mlp": 0.17736816, + "step": 5194, + "time_per_iteration": 2.802870273590088 + }, + { + "auxiliary_loss_clip": 0.01134801, + "auxiliary_loss_mlp": 0.01037192, + "balance_loss_clip": 1.0460403, + "balance_loss_mlp": 1.02158761, + "epoch": 0.6246618168700776, + "flos": 18879891960960.0, + "grad_norm": 2.1948543501987983, + "language_loss": 0.6776427, + "learning_rate": 1.3044234647301235e-06, + "loss": 0.69936258, + "num_input_tokens_seen": 111888365, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.15600586, + "step": 5195, + "time_per_iteration": 2.7270991802215576 + }, + { + "auxiliary_loss_clip": 0.01129236, + "auxiliary_loss_mlp": 0.01049866, + "balance_loss_clip": 1.04177558, + "balance_loss_mlp": 1.03453577, + "epoch": 0.6247820597607167, + "flos": 14319201087360.0, + "grad_norm": 4.6225112155542245, + "language_loss": 0.72209448, + "learning_rate": 1.303693174644347e-06, + "loss": 0.74388552, + "num_input_tokens_seen": 111905840, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15319824, + "step": 5196, + "time_per_iteration": 2.8271396160125732 + }, + { + "auxiliary_loss_clip": 0.01133513, + "auxiliary_loss_mlp": 0.01045492, + "balance_loss_clip": 1.04260612, + "balance_loss_mlp": 1.0296495, + "epoch": 0.6249023026513557, + "flos": 22637979388800.0, + "grad_norm": 4.931196036707496, + "language_loss": 0.80756623, + "learning_rate": 1.3029629901877625e-06, + "loss": 0.82935631, + "num_input_tokens_seen": 111925215, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.15844727, + "step": 5197, + "time_per_iteration": 2.7610466480255127 + }, + { + "auxiliary_loss_clip": 0.01136346, + "auxiliary_loss_mlp": 0.01043501, + "balance_loss_clip": 1.04342937, + "balance_loss_mlp": 1.02732444, + "epoch": 0.6250225455419949, + "flos": 20266690204800.0, + "grad_norm": 3.7132005906952266, + "language_loss": 0.77613187, + "learning_rate": 1.3022329114711376e-06, + "loss": 0.79793036, + "num_input_tokens_seen": 111943925, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.16168213, + "step": 5198, + "time_per_iteration": 2.7854323387145996 + }, + { + "auxiliary_loss_clip": 0.01131412, + "auxiliary_loss_mlp": 0.0105072, + "balance_loss_clip": 1.04220605, + "balance_loss_mlp": 1.03535366, + "epoch": 0.6251427884326339, + "flos": 23437853400960.0, + "grad_norm": 1.9582641972463775, + "language_loss": 0.69741458, + "learning_rate": 1.3015029386052256e-06, + "loss": 0.7192359, + "num_input_tokens_seen": 111964095, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.15368652, + "step": 5199, + "time_per_iteration": 2.865936517715454 + }, + { + "auxiliary_loss_clip": 0.0113456, + "auxiliary_loss_mlp": 0.01043245, + "balance_loss_clip": 1.04324889, + "balance_loss_mlp": 1.02704465, + "epoch": 0.625263031323273, + "flos": 31723055464320.0, + "grad_norm": 1.7814116521692709, + "language_loss": 0.73207355, + "learning_rate": 1.3007730717007622e-06, + "loss": 0.75385165, + "num_input_tokens_seen": 111984910, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.1619873, + "step": 5200, + "time_per_iteration": 2.821319580078125 + }, + { + "auxiliary_loss_clip": 0.01139496, + "auxiliary_loss_mlp": 0.01039925, + "balance_loss_clip": 1.04729056, + "balance_loss_mlp": 1.02247238, + "epoch": 0.6253832742139122, + "flos": 24134341092480.0, + "grad_norm": 1.741686790720651, + "language_loss": 0.75770104, + "learning_rate": 1.3000433108684676e-06, + "loss": 0.77949524, + "num_input_tokens_seen": 112005410, + "router_z_loss_clip": 0.92236328, + "router_z_loss_mlp": 0.17456055, + "step": 5201, + "time_per_iteration": 2.833005428314209 + }, + { + "auxiliary_loss_clip": 0.0113081, + "auxiliary_loss_mlp": 0.01040857, + "balance_loss_clip": 1.04107261, + "balance_loss_mlp": 1.02468038, + "epoch": 0.6255035171045512, + "flos": 27668812400640.0, + "grad_norm": 2.2977267609641223, + "language_loss": 0.79998738, + "learning_rate": 1.2993136562190467e-06, + "loss": 0.82170403, + "num_input_tokens_seen": 112024530, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.16174316, + "step": 5202, + "time_per_iteration": 2.829746723175049 + }, + { + "auxiliary_loss_clip": 0.01132961, + "auxiliary_loss_mlp": 0.01046542, + "balance_loss_clip": 1.04308903, + "balance_loss_mlp": 1.03010261, + "epoch": 0.6256237599951903, + "flos": 20227798753920.0, + "grad_norm": 1.5668725241847687, + "language_loss": 0.70437515, + "learning_rate": 1.2985841078631871e-06, + "loss": 0.72617018, + "num_input_tokens_seen": 112043850, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.16442871, + "step": 5203, + "time_per_iteration": 2.9311540126800537 + }, + { + "auxiliary_loss_clip": 0.0113272, + "auxiliary_loss_mlp": 0.0104026, + "balance_loss_clip": 1.04177701, + "balance_loss_mlp": 1.02330875, + "epoch": 0.6257440028858293, + "flos": 24170574936960.0, + "grad_norm": 1.7466609635996264, + "language_loss": 0.78095609, + "learning_rate": 1.2978546659115608e-06, + "loss": 0.80268592, + "num_input_tokens_seen": 112061930, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.16943359, + "step": 5204, + "time_per_iteration": 2.877986192703247 + }, + { + "auxiliary_loss_clip": 0.01130578, + "auxiliary_loss_mlp": 0.01043584, + "balance_loss_clip": 1.04216731, + "balance_loss_mlp": 1.02737117, + "epoch": 0.6258642457764685, + "flos": 15851940289920.0, + "grad_norm": 1.830834449623495, + "language_loss": 0.85490036, + "learning_rate": 1.2971253304748228e-06, + "loss": 0.87664199, + "num_input_tokens_seen": 112079645, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.1619873, + "step": 5205, + "time_per_iteration": 2.8128662109375 + }, + { + "auxiliary_loss_clip": 0.01140876, + "auxiliary_loss_mlp": 0.01050666, + "balance_loss_clip": 1.04537916, + "balance_loss_mlp": 1.0342803, + "epoch": 0.6259844886671075, + "flos": 11911354836480.0, + "grad_norm": 1.6233765128549815, + "language_loss": 0.74899858, + "learning_rate": 1.296396101663614e-06, + "loss": 0.77091402, + "num_input_tokens_seen": 112096205, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.16387939, + "step": 5206, + "time_per_iteration": 2.773069381713867 + }, + { + "auxiliary_loss_clip": 0.01131033, + "auxiliary_loss_mlp": 0.01046681, + "balance_loss_clip": 1.04360199, + "balance_loss_mlp": 1.03062367, + "epoch": 0.6261047315577466, + "flos": 15887958652800.0, + "grad_norm": 2.40867762824463, + "language_loss": 0.84549439, + "learning_rate": 1.2956669795885565e-06, + "loss": 0.86727154, + "num_input_tokens_seen": 112112835, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.16040039, + "step": 5207, + "time_per_iteration": 2.848963499069214 + }, + { + "auxiliary_loss_clip": 0.01135549, + "auxiliary_loss_mlp": 0.01047778, + "balance_loss_clip": 1.04548573, + "balance_loss_mlp": 1.03102887, + "epoch": 0.6262249744483858, + "flos": 31248926916480.0, + "grad_norm": 1.8101503757644821, + "language_loss": 0.67959559, + "learning_rate": 1.294937964360259e-06, + "loss": 0.70142889, + "num_input_tokens_seen": 112133105, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.16748047, + "step": 5208, + "time_per_iteration": 2.9013257026672363 + }, + { + "auxiliary_loss_clip": 0.01132932, + "auxiliary_loss_mlp": 0.01057438, + "balance_loss_clip": 1.0418067, + "balance_loss_mlp": 1.04070091, + "epoch": 0.6263452173390248, + "flos": 27198598435200.0, + "grad_norm": 2.1402344997201803, + "language_loss": 0.71438497, + "learning_rate": 1.2942090560893108e-06, + "loss": 0.73628867, + "num_input_tokens_seen": 112152510, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.16760254, + "step": 5209, + "time_per_iteration": 2.7941653728485107 + }, + { + "auxiliary_loss_clip": 0.01130433, + "auxiliary_loss_mlp": 0.01040715, + "balance_loss_clip": 1.04276776, + "balance_loss_mlp": 1.02563548, + "epoch": 0.6264654602296639, + "flos": 37342069683840.0, + "grad_norm": 1.6734168621552539, + "language_loss": 0.60388011, + "learning_rate": 1.2934802548862882e-06, + "loss": 0.62559164, + "num_input_tokens_seen": 112175295, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.15087891, + "step": 5210, + "time_per_iteration": 2.9389195442199707 + }, + { + "auxiliary_loss_clip": 0.01130119, + "auxiliary_loss_mlp": 0.01040207, + "balance_loss_clip": 1.0404743, + "balance_loss_mlp": 1.02431607, + "epoch": 0.626585703120303, + "flos": 14756952136320.0, + "grad_norm": 1.9609041104959857, + "language_loss": 0.82931161, + "learning_rate": 1.292751560861749e-06, + "loss": 0.85101479, + "num_input_tokens_seen": 112190200, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.15893555, + "step": 5211, + "time_per_iteration": 2.7258124351501465 + }, + { + "auxiliary_loss_clip": 0.01134853, + "auxiliary_loss_mlp": 0.01044888, + "balance_loss_clip": 1.04340601, + "balance_loss_mlp": 1.0281868, + "epoch": 0.6267059460109421, + "flos": 22347318533760.0, + "grad_norm": 1.8219737834189493, + "language_loss": 0.79802501, + "learning_rate": 1.2920229741262354e-06, + "loss": 0.81982243, + "num_input_tokens_seen": 112208205, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.16711426, + "step": 5212, + "time_per_iteration": 3.5845279693603516 + }, + { + "auxiliary_loss_clip": 0.01131831, + "auxiliary_loss_mlp": 0.0104206, + "balance_loss_clip": 1.04317653, + "balance_loss_mlp": 1.02584791, + "epoch": 0.6268261889015811, + "flos": 17748813617280.0, + "grad_norm": 2.1612154051716983, + "language_loss": 0.75022066, + "learning_rate": 1.2912944947902739e-06, + "loss": 0.77195948, + "num_input_tokens_seen": 112224690, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.16217041, + "step": 5213, + "time_per_iteration": 2.9073660373687744 + }, + { + "auxiliary_loss_clip": 0.01136697, + "auxiliary_loss_mlp": 0.0104065, + "balance_loss_clip": 1.04438901, + "balance_loss_mlp": 1.02278042, + "epoch": 0.6269464317922203, + "flos": 32846484211200.0, + "grad_norm": 2.5288426756691575, + "language_loss": 0.71968114, + "learning_rate": 1.2905661229643742e-06, + "loss": 0.74145454, + "num_input_tokens_seen": 112244450, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.17883301, + "step": 5214, + "time_per_iteration": 3.9077258110046387 + }, + { + "auxiliary_loss_clip": 0.0113343, + "auxiliary_loss_mlp": 0.01041752, + "balance_loss_clip": 1.04270649, + "balance_loss_mlp": 1.02427578, + "epoch": 0.6270666746828594, + "flos": 17929192740480.0, + "grad_norm": 2.08331205296658, + "language_loss": 0.83798146, + "learning_rate": 1.2898378587590299e-06, + "loss": 0.85973328, + "num_input_tokens_seen": 112261050, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.17468262, + "step": 5215, + "time_per_iteration": 2.9317572116851807 + }, + { + "auxiliary_loss_clip": 0.0113058, + "auxiliary_loss_mlp": 0.01039369, + "balance_loss_clip": 1.04244363, + "balance_loss_mlp": 1.02393126, + "epoch": 0.6271869175734984, + "flos": 17457326749440.0, + "grad_norm": 1.7285635232716434, + "language_loss": 0.87307638, + "learning_rate": 1.2891097022847173e-06, + "loss": 0.89477581, + "num_input_tokens_seen": 112278395, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.15429688, + "step": 5216, + "time_per_iteration": 2.982792854309082 + }, + { + "auxiliary_loss_clip": 0.01140269, + "auxiliary_loss_mlp": 0.0104426, + "balance_loss_clip": 1.04459071, + "balance_loss_mlp": 1.02697444, + "epoch": 0.6273071604641376, + "flos": 26868615166080.0, + "grad_norm": 1.5752602416212997, + "language_loss": 0.6655634, + "learning_rate": 1.2883816536518978e-06, + "loss": 0.68740869, + "num_input_tokens_seen": 112299535, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.17272949, + "step": 5217, + "time_per_iteration": 3.681894063949585 + }, + { + "auxiliary_loss_clip": 0.01127263, + "auxiliary_loss_mlp": 0.01040185, + "balance_loss_clip": 1.0411799, + "balance_loss_mlp": 1.02430582, + "epoch": 0.6274274033547766, + "flos": 26062384446720.0, + "grad_norm": 2.238668589239514, + "language_loss": 0.81952286, + "learning_rate": 1.2876537129710155e-06, + "loss": 0.84119731, + "num_input_tokens_seen": 112317265, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15869141, + "step": 5218, + "time_per_iteration": 2.8946001529693604 + }, + { + "auxiliary_loss_clip": 0.01138489, + "auxiliary_loss_mlp": 0.01040829, + "balance_loss_clip": 1.04587722, + "balance_loss_mlp": 1.02325773, + "epoch": 0.6275476462454157, + "flos": 20266259241600.0, + "grad_norm": 2.0805869420718404, + "language_loss": 0.75837362, + "learning_rate": 1.286925880352499e-06, + "loss": 0.78016686, + "num_input_tokens_seen": 112336125, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.17565918, + "step": 5219, + "time_per_iteration": 3.0218920707702637 + }, + { + "auxiliary_loss_clip": 0.01132564, + "auxiliary_loss_mlp": 0.01034907, + "balance_loss_clip": 1.0431397, + "balance_loss_mlp": 1.01865911, + "epoch": 0.6276678891360549, + "flos": 26320402817280.0, + "grad_norm": 1.670439512487817, + "language_loss": 0.71351272, + "learning_rate": 1.2861981559067592e-06, + "loss": 0.73518741, + "num_input_tokens_seen": 112356730, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.16247559, + "step": 5220, + "time_per_iteration": 3.8284287452697754 + }, + { + "auxiliary_loss_clip": 0.0112898, + "auxiliary_loss_mlp": 0.01042533, + "balance_loss_clip": 1.04212952, + "balance_loss_mlp": 1.02611804, + "epoch": 0.6277881320266939, + "flos": 13912512324480.0, + "grad_norm": 1.9041918222251282, + "language_loss": 0.80073398, + "learning_rate": 1.2854705397441917e-06, + "loss": 0.82244915, + "num_input_tokens_seen": 112372270, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.1640625, + "step": 5221, + "time_per_iteration": 3.001209259033203 + }, + { + "auxiliary_loss_clip": 0.01132255, + "auxiliary_loss_mlp": 0.01040487, + "balance_loss_clip": 1.04208279, + "balance_loss_mlp": 1.02378631, + "epoch": 0.627908374917333, + "flos": 27048922462080.0, + "grad_norm": 2.4642213549233536, + "language_loss": 0.77156335, + "learning_rate": 1.2847430319751747e-06, + "loss": 0.79329073, + "num_input_tokens_seen": 112390365, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.16711426, + "step": 5222, + "time_per_iteration": 2.9581711292266846 + }, + { + "auxiliary_loss_clip": 0.01132214, + "auxiliary_loss_mlp": 0.01047165, + "balance_loss_clip": 1.04392636, + "balance_loss_mlp": 1.03172755, + "epoch": 0.6280286178079721, + "flos": 23769201386880.0, + "grad_norm": 2.423529745571755, + "language_loss": 0.67590332, + "learning_rate": 1.2840156327100712e-06, + "loss": 0.6976971, + "num_input_tokens_seen": 112407490, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15423584, + "step": 5223, + "time_per_iteration": 2.9076995849609375 + }, + { + "auxiliary_loss_clip": 0.01132785, + "auxiliary_loss_mlp": 0.01032822, + "balance_loss_clip": 1.04536366, + "balance_loss_mlp": 1.01766419, + "epoch": 0.6281488606986112, + "flos": 26359150613760.0, + "grad_norm": 1.891107982541501, + "language_loss": 0.72435141, + "learning_rate": 1.2832883420592272e-06, + "loss": 0.74600744, + "num_input_tokens_seen": 112426385, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15148926, + "step": 5224, + "time_per_iteration": 3.067115545272827 + }, + { + "auxiliary_loss_clip": 0.01135465, + "auxiliary_loss_mlp": 0.0104186, + "balance_loss_clip": 1.04587305, + "balance_loss_mlp": 1.02625549, + "epoch": 0.6282691035892503, + "flos": 36137194848000.0, + "grad_norm": 2.0056216613572047, + "language_loss": 0.64531094, + "learning_rate": 1.282561160132972e-06, + "loss": 0.66708422, + "num_input_tokens_seen": 112446905, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.15612793, + "step": 5225, + "time_per_iteration": 2.957789182662964 + }, + { + "auxiliary_loss_clip": 0.0113392, + "auxiliary_loss_mlp": 0.01045327, + "balance_loss_clip": 1.04206836, + "balance_loss_mlp": 1.02891231, + "epoch": 0.6283893464798894, + "flos": 26537231266560.0, + "grad_norm": 1.6199164913482131, + "language_loss": 0.80909264, + "learning_rate": 1.2818340870416186e-06, + "loss": 0.83088505, + "num_input_tokens_seen": 112468040, + "router_z_loss_clip": 0.91894531, + "router_z_loss_mlp": 0.16418457, + "step": 5226, + "time_per_iteration": 2.984201192855835 + }, + { + "auxiliary_loss_clip": 0.01136201, + "auxiliary_loss_mlp": 0.01041825, + "balance_loss_clip": 1.0426836, + "balance_loss_mlp": 1.02463508, + "epoch": 0.6285095893705285, + "flos": 22237216369920.0, + "grad_norm": 1.9115502051627202, + "language_loss": 0.75754523, + "learning_rate": 1.2811071228954626e-06, + "loss": 0.77932549, + "num_input_tokens_seen": 112486675, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.17199707, + "step": 5227, + "time_per_iteration": 3.0679867267608643 + }, + { + "auxiliary_loss_clip": 0.01133008, + "auxiliary_loss_mlp": 0.01038514, + "balance_loss_clip": 1.04361212, + "balance_loss_mlp": 1.02329063, + "epoch": 0.6286298322611675, + "flos": 26542259170560.0, + "grad_norm": 2.0294180855907165, + "language_loss": 0.81040722, + "learning_rate": 1.2803802678047846e-06, + "loss": 0.83212245, + "num_input_tokens_seen": 112506825, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.15222168, + "step": 5228, + "time_per_iteration": 2.903367519378662 + }, + { + "auxiliary_loss_clip": 0.01138438, + "auxiliary_loss_mlp": 0.01043913, + "balance_loss_clip": 1.04507816, + "balance_loss_mlp": 1.02690196, + "epoch": 0.6287500751518067, + "flos": 21795227516160.0, + "grad_norm": 1.725408764336484, + "language_loss": 0.73967135, + "learning_rate": 1.279653521879848e-06, + "loss": 0.76149487, + "num_input_tokens_seen": 112526890, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.17028809, + "step": 5229, + "time_per_iteration": 2.947983741760254 + }, + { + "auxiliary_loss_clip": 0.01128927, + "auxiliary_loss_mlp": 0.01039373, + "balance_loss_clip": 1.04390538, + "balance_loss_mlp": 1.02416158, + "epoch": 0.6288703180424458, + "flos": 20009605587840.0, + "grad_norm": 2.252906318762197, + "language_loss": 0.83871955, + "learning_rate": 1.2789268852308997e-06, + "loss": 0.86040258, + "num_input_tokens_seen": 112542100, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.15209961, + "step": 5230, + "time_per_iteration": 2.9415926933288574 + }, + { + "auxiliary_loss_clip": 0.01133663, + "auxiliary_loss_mlp": 0.01034991, + "balance_loss_clip": 1.04461718, + "balance_loss_mlp": 1.01917195, + "epoch": 0.6289905609330848, + "flos": 22124923476480.0, + "grad_norm": 2.2826243588429755, + "language_loss": 0.70944434, + "learning_rate": 1.2782003579681688e-06, + "loss": 0.7311309, + "num_input_tokens_seen": 112561630, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.15820312, + "step": 5231, + "time_per_iteration": 2.9729785919189453 + }, + { + "auxiliary_loss_clip": 0.01134894, + "auxiliary_loss_mlp": 0.01039797, + "balance_loss_clip": 1.04365301, + "balance_loss_mlp": 1.02388215, + "epoch": 0.629110803823724, + "flos": 25518481729920.0, + "grad_norm": 2.138849284437859, + "language_loss": 0.7442131, + "learning_rate": 1.2774739402018701e-06, + "loss": 0.76596004, + "num_input_tokens_seen": 112582465, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.15917969, + "step": 5232, + "time_per_iteration": 2.916980504989624 + }, + { + "auxiliary_loss_clip": 0.0113607, + "auxiliary_loss_mlp": 0.0104244, + "balance_loss_clip": 1.04593587, + "balance_loss_mlp": 1.02499914, + "epoch": 0.629231046714363, + "flos": 20886616056960.0, + "grad_norm": 2.05475784743399, + "language_loss": 0.7264598, + "learning_rate": 1.2767476320422002e-06, + "loss": 0.74824488, + "num_input_tokens_seen": 112602390, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.17431641, + "step": 5233, + "time_per_iteration": 3.0262227058410645 + }, + { + "auxiliary_loss_clip": 0.01040605, + "auxiliary_loss_mlp": 0.01002689, + "balance_loss_clip": 1.0125072, + "balance_loss_mlp": 1.00113893, + "epoch": 0.6293512896050021, + "flos": 65050027908480.0, + "grad_norm": 0.6897676784628526, + "language_loss": 0.57169998, + "learning_rate": 1.2760214335993392e-06, + "loss": 0.59213281, + "num_input_tokens_seen": 112669035, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01550293, + "step": 5234, + "time_per_iteration": 3.5043721199035645 + }, + { + "auxiliary_loss_clip": 0.01130408, + "auxiliary_loss_mlp": 0.010343, + "balance_loss_clip": 1.04381645, + "balance_loss_mlp": 1.01927996, + "epoch": 0.6294715324956413, + "flos": 34677857088000.0, + "grad_norm": 2.406814027764307, + "language_loss": 0.59042132, + "learning_rate": 1.2752953449834514e-06, + "loss": 0.61206841, + "num_input_tokens_seen": 112691485, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.15026855, + "step": 5235, + "time_per_iteration": 3.0448009967803955 + }, + { + "auxiliary_loss_clip": 0.01132622, + "auxiliary_loss_mlp": 0.01040449, + "balance_loss_clip": 1.04311836, + "balance_loss_mlp": 1.02514267, + "epoch": 0.6295917753862803, + "flos": 22784207656320.0, + "grad_norm": 1.6758044360657727, + "language_loss": 0.802472, + "learning_rate": 1.2745693663046836e-06, + "loss": 0.82420272, + "num_input_tokens_seen": 112710555, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.15307617, + "step": 5236, + "time_per_iteration": 2.9841041564941406 + }, + { + "auxiliary_loss_clip": 0.01128507, + "auxiliary_loss_mlp": 0.01038921, + "balance_loss_clip": 1.04156637, + "balance_loss_mlp": 1.02385283, + "epoch": 0.6297120182769194, + "flos": 20850454039680.0, + "grad_norm": 1.9028114807605558, + "language_loss": 0.81343055, + "learning_rate": 1.2738434976731662e-06, + "loss": 0.83510482, + "num_input_tokens_seen": 112728740, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.1506958, + "step": 5237, + "time_per_iteration": 2.975870132446289 + }, + { + "auxiliary_loss_clip": 0.01138825, + "auxiliary_loss_mlp": 0.01043386, + "balance_loss_clip": 1.04712439, + "balance_loss_mlp": 1.02803159, + "epoch": 0.6298322611675584, + "flos": 19497662997120.0, + "grad_norm": 1.5796952277099727, + "language_loss": 0.75366879, + "learning_rate": 1.2731177391990125e-06, + "loss": 0.77549082, + "num_input_tokens_seen": 112748665, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.15356445, + "step": 5238, + "time_per_iteration": 2.9981117248535156 + }, + { + "auxiliary_loss_clip": 0.01132299, + "auxiliary_loss_mlp": 0.01038006, + "balance_loss_clip": 1.04220045, + "balance_loss_mlp": 1.02073264, + "epoch": 0.6299525040581976, + "flos": 12604466649600.0, + "grad_norm": 1.9717689759303274, + "language_loss": 0.82098597, + "learning_rate": 1.2723920909923203e-06, + "loss": 0.84268904, + "num_input_tokens_seen": 112764410, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.17285156, + "step": 5239, + "time_per_iteration": 3.046318769454956 + }, + { + "auxiliary_loss_clip": 0.01040912, + "auxiliary_loss_mlp": 0.01004369, + "balance_loss_clip": 1.01292872, + "balance_loss_mlp": 1.00278306, + "epoch": 0.6300727469488366, + "flos": 57725685636480.0, + "grad_norm": 0.8659131002474963, + "language_loss": 0.60448319, + "learning_rate": 1.2716665531631688e-06, + "loss": 0.62493593, + "num_input_tokens_seen": 112818695, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01586914, + "step": 5240, + "time_per_iteration": 3.363511323928833 + }, + { + "auxiliary_loss_clip": 0.01133958, + "auxiliary_loss_mlp": 0.01039376, + "balance_loss_clip": 1.04296446, + "balance_loss_mlp": 1.02247226, + "epoch": 0.6301929898394757, + "flos": 22527302607360.0, + "grad_norm": 1.6694115724228098, + "language_loss": 0.7713747, + "learning_rate": 1.270941125821623e-06, + "loss": 0.79310805, + "num_input_tokens_seen": 112839120, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.16894531, + "step": 5241, + "time_per_iteration": 3.8287739753723145 + }, + { + "auxiliary_loss_clip": 0.01130206, + "auxiliary_loss_mlp": 0.01043219, + "balance_loss_clip": 1.0421505, + "balance_loss_mlp": 1.02788889, + "epoch": 0.6303132327301149, + "flos": 28293550675200.0, + "grad_norm": 1.5746404149761004, + "language_loss": 0.75153083, + "learning_rate": 1.2702158090777278e-06, + "loss": 0.77326512, + "num_input_tokens_seen": 112860210, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.15356445, + "step": 5242, + "time_per_iteration": 3.0449159145355225 + }, + { + "auxiliary_loss_clip": 0.01129483, + "auxiliary_loss_mlp": 0.01048337, + "balance_loss_clip": 1.04279315, + "balance_loss_mlp": 1.03238726, + "epoch": 0.6304334756207539, + "flos": 25264521596160.0, + "grad_norm": 1.7813587084179532, + "language_loss": 0.74659729, + "learning_rate": 1.2694906030415148e-06, + "loss": 0.76837546, + "num_input_tokens_seen": 112877955, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15942383, + "step": 5243, + "time_per_iteration": 3.9342522621154785 + }, + { + "auxiliary_loss_clip": 0.01137926, + "auxiliary_loss_mlp": 0.01042215, + "balance_loss_clip": 1.04384017, + "balance_loss_mlp": 1.02457201, + "epoch": 0.630553718511393, + "flos": 18033548728320.0, + "grad_norm": 3.0191215607742015, + "language_loss": 0.82319367, + "learning_rate": 1.2687655078229958e-06, + "loss": 0.84499508, + "num_input_tokens_seen": 112892285, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.17651367, + "step": 5244, + "time_per_iteration": 2.9456632137298584 + }, + { + "auxiliary_loss_clip": 0.01135101, + "auxiliary_loss_mlp": 0.01038059, + "balance_loss_clip": 1.0450871, + "balance_loss_mlp": 1.02148938, + "epoch": 0.6306739614020321, + "flos": 27304103658240.0, + "grad_norm": 2.045498519918156, + "language_loss": 0.69288957, + "learning_rate": 1.2680405235321678e-06, + "loss": 0.71462119, + "num_input_tokens_seen": 112913620, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.16577148, + "step": 5245, + "time_per_iteration": 2.9781432151794434 + }, + { + "auxiliary_loss_clip": 0.01137295, + "auxiliary_loss_mlp": 0.01043286, + "balance_loss_clip": 1.04462302, + "balance_loss_mlp": 1.02726412, + "epoch": 0.6307942042926712, + "flos": 15341434243200.0, + "grad_norm": 2.046738799462239, + "language_loss": 0.78603363, + "learning_rate": 1.267315650279011e-06, + "loss": 0.80783945, + "num_input_tokens_seen": 112932090, + "router_z_loss_clip": 0.92675781, + "router_z_loss_mlp": 0.16015625, + "step": 5246, + "time_per_iteration": 3.6737561225891113 + }, + { + "auxiliary_loss_clip": 0.01136771, + "auxiliary_loss_mlp": 0.01036403, + "balance_loss_clip": 1.0431664, + "balance_loss_mlp": 1.02008283, + "epoch": 0.6309144471833102, + "flos": 19606400444160.0, + "grad_norm": 1.7989567946534175, + "language_loss": 0.74361515, + "learning_rate": 1.2665908881734874e-06, + "loss": 0.76534688, + "num_input_tokens_seen": 112950925, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.16308594, + "step": 5247, + "time_per_iteration": 2.9850876331329346 + }, + { + "auxiliary_loss_clip": 0.0113123, + "auxiliary_loss_mlp": 0.01035625, + "balance_loss_clip": 1.04306245, + "balance_loss_mlp": 1.01935244, + "epoch": 0.6310346900739494, + "flos": 17493345112320.0, + "grad_norm": 2.0173161471498195, + "language_loss": 0.84765375, + "learning_rate": 1.2658662373255432e-06, + "loss": 0.86932224, + "num_input_tokens_seen": 112969315, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.16259766, + "step": 5248, + "time_per_iteration": 2.9212374687194824 + }, + { + "auxiliary_loss_clip": 0.01038111, + "auxiliary_loss_mlp": 0.01006331, + "balance_loss_clip": 1.01018906, + "balance_loss_mlp": 1.00485885, + "epoch": 0.6311549329645885, + "flos": 55070164131840.0, + "grad_norm": 6.709189648277266, + "language_loss": 0.52215141, + "learning_rate": 1.2651416978451063e-06, + "loss": 0.5425958, + "num_input_tokens_seen": 113034700, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01470947, + "step": 5249, + "time_per_iteration": 4.34483790397644 + }, + { + "auxiliary_loss_clip": 0.01139422, + "auxiliary_loss_mlp": 0.01041196, + "balance_loss_clip": 1.04650974, + "balance_loss_mlp": 1.02391076, + "epoch": 0.6312751758552275, + "flos": 41902545075840.0, + "grad_norm": 1.7829128232985754, + "language_loss": 0.65042639, + "learning_rate": 1.2644172698420903e-06, + "loss": 0.67223257, + "num_input_tokens_seen": 113056805, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.17297363, + "step": 5250, + "time_per_iteration": 3.0594000816345215 + }, + { + "auxiliary_loss_clip": 0.01131976, + "auxiliary_loss_mlp": 0.01038595, + "balance_loss_clip": 1.04207754, + "balance_loss_mlp": 1.02260876, + "epoch": 0.6313954187458667, + "flos": 19646800266240.0, + "grad_norm": 1.708258048465472, + "language_loss": 0.84485203, + "learning_rate": 1.2636929534263892e-06, + "loss": 0.86655772, + "num_input_tokens_seen": 113075790, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.15979004, + "step": 5251, + "time_per_iteration": 2.8942747116088867 + }, + { + "auxiliary_loss_clip": 0.01133239, + "auxiliary_loss_mlp": 0.01036854, + "balance_loss_clip": 1.04354358, + "balance_loss_mlp": 1.02062964, + "epoch": 0.6315156616365057, + "flos": 22894273906560.0, + "grad_norm": 1.7766625243478746, + "language_loss": 0.7765038, + "learning_rate": 1.2629687487078821e-06, + "loss": 0.79820466, + "num_input_tokens_seen": 113094600, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.16247559, + "step": 5252, + "time_per_iteration": 2.8473031520843506 + }, + { + "auxiliary_loss_clip": 0.01134104, + "auxiliary_loss_mlp": 0.01042334, + "balance_loss_clip": 1.04276323, + "balance_loss_mlp": 1.02560925, + "epoch": 0.6316359045271448, + "flos": 23726251699200.0, + "grad_norm": 2.0056944644545425, + "language_loss": 0.76482564, + "learning_rate": 1.2622446557964293e-06, + "loss": 0.7865901, + "num_input_tokens_seen": 113112605, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.1673584, + "step": 5253, + "time_per_iteration": 2.9597790241241455 + }, + { + "auxiliary_loss_clip": 0.01129868, + "auxiliary_loss_mlp": 0.01042107, + "balance_loss_clip": 1.0413456, + "balance_loss_mlp": 1.02501225, + "epoch": 0.631756147417784, + "flos": 33108417164160.0, + "grad_norm": 1.7782840417200654, + "language_loss": 0.7143997, + "learning_rate": 1.261520674801876e-06, + "loss": 0.73611951, + "num_input_tokens_seen": 113133200, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.17089844, + "step": 5254, + "time_per_iteration": 2.9810078144073486 + }, + { + "auxiliary_loss_clip": 0.01137815, + "auxiliary_loss_mlp": 0.01037582, + "balance_loss_clip": 1.04529798, + "balance_loss_mlp": 1.02072549, + "epoch": 0.631876390308423, + "flos": 31248424126080.0, + "grad_norm": 2.3640253258526043, + "language_loss": 0.72122371, + "learning_rate": 1.2607968058340488e-06, + "loss": 0.74297768, + "num_input_tokens_seen": 113152895, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.16845703, + "step": 5255, + "time_per_iteration": 2.971315622329712 + }, + { + "auxiliary_loss_clip": 0.01129316, + "auxiliary_loss_mlp": 0.01044912, + "balance_loss_clip": 1.04165637, + "balance_loss_mlp": 1.02922428, + "epoch": 0.6319966331990621, + "flos": 24681152810880.0, + "grad_norm": 1.6820420521423698, + "language_loss": 0.73151857, + "learning_rate": 1.2600730490027583e-06, + "loss": 0.75326085, + "num_input_tokens_seen": 113173135, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.15686035, + "step": 5256, + "time_per_iteration": 2.9291532039642334 + }, + { + "auxiliary_loss_clip": 0.01127913, + "auxiliary_loss_mlp": 0.01039269, + "balance_loss_clip": 1.0403235, + "balance_loss_mlp": 1.023247, + "epoch": 0.6321168760897012, + "flos": 17491764913920.0, + "grad_norm": 1.5946669488371952, + "language_loss": 0.80391586, + "learning_rate": 1.2593494044177984e-06, + "loss": 0.82558769, + "num_input_tokens_seen": 113191440, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.16027832, + "step": 5257, + "time_per_iteration": 2.9723169803619385 + }, + { + "auxiliary_loss_clip": 0.01136356, + "auxiliary_loss_mlp": 0.01042071, + "balance_loss_clip": 1.0422281, + "balance_loss_mlp": 1.02453542, + "epoch": 0.6322371189803403, + "flos": 18295373940480.0, + "grad_norm": 2.5089659839275504, + "language_loss": 0.80965614, + "learning_rate": 1.2586258721889448e-06, + "loss": 0.83144045, + "num_input_tokens_seen": 113208790, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.1751709, + "step": 5258, + "time_per_iteration": 2.9487228393554688 + }, + { + "auxiliary_loss_clip": 0.01132884, + "auxiliary_loss_mlp": 0.01038465, + "balance_loss_clip": 1.04355836, + "balance_loss_mlp": 1.02183533, + "epoch": 0.6323573618709794, + "flos": 20157270399360.0, + "grad_norm": 1.842721733452871, + "language_loss": 0.81890959, + "learning_rate": 1.2579024524259573e-06, + "loss": 0.84062308, + "num_input_tokens_seen": 113225050, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.16625977, + "step": 5259, + "time_per_iteration": 2.9001529216766357 + }, + { + "auxiliary_loss_clip": 0.01131352, + "auxiliary_loss_mlp": 0.01041026, + "balance_loss_clip": 1.04209805, + "balance_loss_mlp": 1.0233829, + "epoch": 0.6324776047616185, + "flos": 20042391726720.0, + "grad_norm": 1.891086374588962, + "language_loss": 0.91477525, + "learning_rate": 1.2571791452385768e-06, + "loss": 0.93649906, + "num_input_tokens_seen": 113242315, + "router_z_loss_clip": 0.89208984, + "router_z_loss_mlp": 0.17626953, + "step": 5260, + "time_per_iteration": 2.918055295944214 + }, + { + "auxiliary_loss_clip": 0.01126447, + "auxiliary_loss_mlp": 0.01039857, + "balance_loss_clip": 1.04174089, + "balance_loss_mlp": 1.02387071, + "epoch": 0.6325978476522576, + "flos": 30848235724800.0, + "grad_norm": 2.248511260544462, + "language_loss": 0.77586108, + "learning_rate": 1.2564559507365301e-06, + "loss": 0.79752409, + "num_input_tokens_seen": 113264720, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.15991211, + "step": 5261, + "time_per_iteration": 3.043747663497925 + }, + { + "auxiliary_loss_clip": 0.01134274, + "auxiliary_loss_mlp": 0.01037957, + "balance_loss_clip": 1.04313135, + "balance_loss_mlp": 1.0211482, + "epoch": 0.6327180905428966, + "flos": 24535104111360.0, + "grad_norm": 2.124056970312301, + "language_loss": 0.79339468, + "learning_rate": 1.2557328690295244e-06, + "loss": 0.815117, + "num_input_tokens_seen": 113282910, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.16809082, + "step": 5262, + "time_per_iteration": 2.9282021522521973 + }, + { + "auxiliary_loss_clip": 0.01129854, + "auxiliary_loss_mlp": 0.01051256, + "balance_loss_clip": 1.04177403, + "balance_loss_mlp": 1.03542447, + "epoch": 0.6328383334335358, + "flos": 21575274583680.0, + "grad_norm": 1.6772067747337798, + "language_loss": 0.75747377, + "learning_rate": 1.255009900227251e-06, + "loss": 0.77928489, + "num_input_tokens_seen": 113301935, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.15820312, + "step": 5263, + "time_per_iteration": 3.0902907848358154 + }, + { + "auxiliary_loss_clip": 0.01130841, + "auxiliary_loss_mlp": 0.01036644, + "balance_loss_clip": 1.0436995, + "balance_loss_mlp": 1.02003813, + "epoch": 0.6329585763241748, + "flos": 22929861306240.0, + "grad_norm": 1.7744606293925134, + "language_loss": 0.79307401, + "learning_rate": 1.254287044439383e-06, + "loss": 0.81474888, + "num_input_tokens_seen": 113321540, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.1661377, + "step": 5264, + "time_per_iteration": 2.996769428253174 + }, + { + "auxiliary_loss_clip": 0.01037261, + "auxiliary_loss_mlp": 0.01009038, + "balance_loss_clip": 1.00928009, + "balance_loss_mlp": 1.00745237, + "epoch": 0.6330788192148139, + "flos": 70936897847040.0, + "grad_norm": 0.7843127151500887, + "language_loss": 0.54462045, + "learning_rate": 1.2535643017755776e-06, + "loss": 0.56508344, + "num_input_tokens_seen": 113383730, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01586914, + "step": 5265, + "time_per_iteration": 3.459745168685913 + }, + { + "auxiliary_loss_clip": 0.0113145, + "auxiliary_loss_mlp": 0.01042914, + "balance_loss_clip": 1.04221749, + "balance_loss_mlp": 1.02699983, + "epoch": 0.6331990621054531, + "flos": 21244501215360.0, + "grad_norm": 2.2674926881404556, + "language_loss": 0.71873689, + "learning_rate": 1.2528416723454737e-06, + "loss": 0.74048054, + "num_input_tokens_seen": 113400400, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.15917969, + "step": 5266, + "time_per_iteration": 2.8905794620513916 + }, + { + "auxiliary_loss_clip": 0.01127578, + "auxiliary_loss_mlp": 0.01041048, + "balance_loss_clip": 1.04223132, + "balance_loss_mlp": 1.02598023, + "epoch": 0.6333193049960921, + "flos": 34459412526720.0, + "grad_norm": 1.5275874824352105, + "language_loss": 0.7101832, + "learning_rate": 1.2521191562586945e-06, + "loss": 0.73186946, + "num_input_tokens_seen": 113424050, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.15063477, + "step": 5267, + "time_per_iteration": 2.878828763961792 + }, + { + "auxiliary_loss_clip": 0.01131181, + "auxiliary_loss_mlp": 0.01039266, + "balance_loss_clip": 1.04261804, + "balance_loss_mlp": 1.0234586, + "epoch": 0.6334395478867312, + "flos": 18329883932160.0, + "grad_norm": 1.9639319114760152, + "language_loss": 0.76444125, + "learning_rate": 1.2513967536248445e-06, + "loss": 0.78614569, + "num_input_tokens_seen": 113440370, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15808105, + "step": 5268, + "time_per_iteration": 2.8346009254455566 + }, + { + "auxiliary_loss_clip": 0.01129736, + "auxiliary_loss_mlp": 0.01042771, + "balance_loss_clip": 1.04068649, + "balance_loss_mlp": 1.0262363, + "epoch": 0.6335597907773702, + "flos": 23623152687360.0, + "grad_norm": 1.6974460141887457, + "language_loss": 0.81437492, + "learning_rate": 1.2506744645535117e-06, + "loss": 0.83609998, + "num_input_tokens_seen": 113460800, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.1652832, + "step": 5269, + "time_per_iteration": 2.8413379192352295 + }, + { + "auxiliary_loss_clip": 0.01130147, + "auxiliary_loss_mlp": 0.01040235, + "balance_loss_clip": 1.04143488, + "balance_loss_mlp": 1.0232358, + "epoch": 0.6336800336680094, + "flos": 22710913954560.0, + "grad_norm": 2.2903902124123334, + "language_loss": 0.59899998, + "learning_rate": 1.249952289154267e-06, + "loss": 0.62070376, + "num_input_tokens_seen": 113480840, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.16992188, + "step": 5270, + "time_per_iteration": 3.6332812309265137 + }, + { + "auxiliary_loss_clip": 0.01130124, + "auxiliary_loss_mlp": 0.01038852, + "balance_loss_clip": 1.04322147, + "balance_loss_mlp": 1.02415943, + "epoch": 0.6338002765586485, + "flos": 23622757637760.0, + "grad_norm": 1.624421919968014, + "language_loss": 0.76661015, + "learning_rate": 1.2492302275366635e-06, + "loss": 0.78829992, + "num_input_tokens_seen": 113500515, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.14678955, + "step": 5271, + "time_per_iteration": 2.8307747840881348 + }, + { + "auxiliary_loss_clip": 0.01135729, + "auxiliary_loss_mlp": 0.01041728, + "balance_loss_clip": 1.04325485, + "balance_loss_mlp": 1.02415633, + "epoch": 0.6339205194492875, + "flos": 26505450708480.0, + "grad_norm": 2.204120687691986, + "language_loss": 0.66178048, + "learning_rate": 1.2485082798102377e-06, + "loss": 0.68355507, + "num_input_tokens_seen": 113520930, + "router_z_loss_clip": 0.92382812, + "router_z_loss_mlp": 0.17565918, + "step": 5272, + "time_per_iteration": 3.630255937576294 + }, + { + "auxiliary_loss_clip": 0.01135556, + "auxiliary_loss_mlp": 0.01044073, + "balance_loss_clip": 1.0452745, + "balance_loss_mlp": 1.02712119, + "epoch": 0.6340407623399267, + "flos": 18544306170240.0, + "grad_norm": 2.3944814821654585, + "language_loss": 0.68564528, + "learning_rate": 1.2477864460845084e-06, + "loss": 0.70744157, + "num_input_tokens_seen": 113537330, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.16943359, + "step": 5273, + "time_per_iteration": 2.877415418624878 + }, + { + "auxiliary_loss_clip": 0.01138929, + "auxiliary_loss_mlp": 0.01044699, + "balance_loss_clip": 1.04377294, + "balance_loss_mlp": 1.02620935, + "epoch": 0.6341610052305657, + "flos": 17712579772800.0, + "grad_norm": 2.2446335778181923, + "language_loss": 0.73705512, + "learning_rate": 1.2470647264689776e-06, + "loss": 0.7588914, + "num_input_tokens_seen": 113555810, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.18505859, + "step": 5274, + "time_per_iteration": 3.831514835357666 + }, + { + "auxiliary_loss_clip": 0.01131335, + "auxiliary_loss_mlp": 0.01038799, + "balance_loss_clip": 1.04121542, + "balance_loss_mlp": 1.02153802, + "epoch": 0.6342812481212048, + "flos": 23587026583680.0, + "grad_norm": 1.987995926363299, + "language_loss": 0.71265423, + "learning_rate": 1.2463431210731282e-06, + "loss": 0.73435557, + "num_input_tokens_seen": 113575395, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.17260742, + "step": 5275, + "time_per_iteration": 3.0036959648132324 + }, + { + "auxiliary_loss_clip": 0.01135331, + "auxiliary_loss_mlp": 0.01043224, + "balance_loss_clip": 1.04363441, + "balance_loss_mlp": 1.02606988, + "epoch": 0.634401491011844, + "flos": 17821927751040.0, + "grad_norm": 2.2662274770311552, + "language_loss": 0.7637955, + "learning_rate": 1.2456216300064289e-06, + "loss": 0.78558105, + "num_input_tokens_seen": 113592945, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.17150879, + "step": 5276, + "time_per_iteration": 2.926821231842041 + }, + { + "auxiliary_loss_clip": 0.01136295, + "auxiliary_loss_mlp": 0.01037585, + "balance_loss_clip": 1.04436421, + "balance_loss_mlp": 1.02051425, + "epoch": 0.634521733902483, + "flos": 21358158825600.0, + "grad_norm": 1.6055812343383757, + "language_loss": 0.78788221, + "learning_rate": 1.244900253378328e-06, + "loss": 0.80962098, + "num_input_tokens_seen": 113613000, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.17089844, + "step": 5277, + "time_per_iteration": 3.008172035217285 + }, + { + "auxiliary_loss_clip": 0.01130733, + "auxiliary_loss_mlp": 0.01039086, + "balance_loss_clip": 1.04111588, + "balance_loss_mlp": 1.02382159, + "epoch": 0.6346419767931221, + "flos": 16545052103040.0, + "grad_norm": 1.974409083948787, + "language_loss": 0.69762582, + "learning_rate": 1.2441789912982583e-06, + "loss": 0.71932405, + "num_input_tokens_seen": 113630085, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.15264893, + "step": 5278, + "time_per_iteration": 3.729595899581909 + }, + { + "auxiliary_loss_clip": 0.01136923, + "auxiliary_loss_mlp": 0.01038571, + "balance_loss_clip": 1.04474032, + "balance_loss_mlp": 1.02151179, + "epoch": 0.6347622196837612, + "flos": 24350989973760.0, + "grad_norm": 1.812001562012135, + "language_loss": 0.64700294, + "learning_rate": 1.2434578438756346e-06, + "loss": 0.6687578, + "num_input_tokens_seen": 113650515, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.17053223, + "step": 5279, + "time_per_iteration": 2.952547311782837 + }, + { + "auxiliary_loss_clip": 0.01127881, + "auxiliary_loss_mlp": 0.01040281, + "balance_loss_clip": 1.03993595, + "balance_loss_mlp": 1.0241282, + "epoch": 0.6348824625744003, + "flos": 64523178195840.0, + "grad_norm": 2.1570140160836737, + "language_loss": 0.77789605, + "learning_rate": 1.242736811219855e-06, + "loss": 0.79957765, + "num_input_tokens_seen": 113676475, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.16149902, + "step": 5280, + "time_per_iteration": 3.327235460281372 + }, + { + "auxiliary_loss_clip": 0.01129977, + "auxiliary_loss_mlp": 0.01035108, + "balance_loss_clip": 1.04170442, + "balance_loss_mlp": 1.01944423, + "epoch": 0.6350027054650393, + "flos": 28622133313920.0, + "grad_norm": 1.851579842967795, + "language_loss": 0.82153058, + "learning_rate": 1.2420158934402988e-06, + "loss": 0.84318143, + "num_input_tokens_seen": 113697090, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.15655518, + "step": 5281, + "time_per_iteration": 2.9315178394317627 + }, + { + "auxiliary_loss_clip": 0.01131235, + "auxiliary_loss_mlp": 0.01040886, + "balance_loss_clip": 1.04064012, + "balance_loss_mlp": 1.02437544, + "epoch": 0.6351229483556785, + "flos": 23002544476800.0, + "grad_norm": 1.8294122720908612, + "language_loss": 0.84983921, + "learning_rate": 1.2412950906463286e-06, + "loss": 0.87156045, + "num_input_tokens_seen": 113714395, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.16503906, + "step": 5282, + "time_per_iteration": 2.969001293182373 + }, + { + "auxiliary_loss_clip": 0.01129424, + "auxiliary_loss_mlp": 0.01038399, + "balance_loss_clip": 1.04227901, + "balance_loss_mlp": 1.02235317, + "epoch": 0.6352431912463176, + "flos": 21939300967680.0, + "grad_norm": 1.7988940775817677, + "language_loss": 0.90031958, + "learning_rate": 1.2405744029472902e-06, + "loss": 0.92199779, + "num_input_tokens_seen": 113733880, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.16040039, + "step": 5283, + "time_per_iteration": 2.886976957321167 + }, + { + "auxiliary_loss_clip": 0.01132077, + "auxiliary_loss_mlp": 0.01042088, + "balance_loss_clip": 1.04245353, + "balance_loss_mlp": 1.02605462, + "epoch": 0.6353634341369566, + "flos": 13735257684480.0, + "grad_norm": 2.079452493549729, + "language_loss": 0.76509875, + "learning_rate": 1.2398538304525108e-06, + "loss": 0.78684044, + "num_input_tokens_seen": 113752505, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.16027832, + "step": 5284, + "time_per_iteration": 2.907226324081421 + }, + { + "auxiliary_loss_clip": 0.01138564, + "auxiliary_loss_mlp": 0.01044358, + "balance_loss_clip": 1.04561305, + "balance_loss_mlp": 1.02627373, + "epoch": 0.6354836770275958, + "flos": 19316170552320.0, + "grad_norm": 2.3459038346019008, + "language_loss": 0.75879049, + "learning_rate": 1.2391333732713016e-06, + "loss": 0.78061974, + "num_input_tokens_seen": 113770310, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.1809082, + "step": 5285, + "time_per_iteration": 2.898625373840332 + }, + { + "auxiliary_loss_clip": 0.01135669, + "auxiliary_loss_mlp": 0.01045388, + "balance_loss_clip": 1.04329419, + "balance_loss_mlp": 1.02807879, + "epoch": 0.6356039199182348, + "flos": 21613375935360.0, + "grad_norm": 3.0486139079904073, + "language_loss": 0.78454328, + "learning_rate": 1.2384130315129543e-06, + "loss": 0.80635393, + "num_input_tokens_seen": 113788635, + "router_z_loss_clip": 0.92333984, + "router_z_loss_mlp": 0.17321777, + "step": 5286, + "time_per_iteration": 2.8076343536376953 + }, + { + "auxiliary_loss_clip": 0.01135733, + "auxiliary_loss_mlp": 0.01042901, + "balance_loss_clip": 1.04222703, + "balance_loss_mlp": 1.02505493, + "epoch": 0.6357241628088739, + "flos": 18111978074880.0, + "grad_norm": 2.0788650265099116, + "language_loss": 0.73772395, + "learning_rate": 1.2376928052867447e-06, + "loss": 0.75951022, + "num_input_tokens_seen": 113807755, + "router_z_loss_clip": 0.93652344, + "router_z_loss_mlp": 0.17858887, + "step": 5287, + "time_per_iteration": 2.8217437267303467 + }, + { + "auxiliary_loss_clip": 0.01135975, + "auxiliary_loss_mlp": 0.01041625, + "balance_loss_clip": 1.04433465, + "balance_loss_mlp": 1.02591372, + "epoch": 0.6358444056995131, + "flos": 24935256599040.0, + "grad_norm": 2.249101239521016, + "language_loss": 0.77514255, + "learning_rate": 1.2369726947019299e-06, + "loss": 0.79691851, + "num_input_tokens_seen": 113828230, + "router_z_loss_clip": 0.91552734, + "router_z_loss_mlp": 0.15710449, + "step": 5288, + "time_per_iteration": 2.8634798526763916 + }, + { + "auxiliary_loss_clip": 0.01129338, + "auxiliary_loss_mlp": 0.01034428, + "balance_loss_clip": 1.04312634, + "balance_loss_mlp": 1.01924062, + "epoch": 0.6359646485901521, + "flos": 23293348986240.0, + "grad_norm": 2.8007288322205306, + "language_loss": 0.66960013, + "learning_rate": 1.2362526998677511e-06, + "loss": 0.69123781, + "num_input_tokens_seen": 113844595, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.1517334, + "step": 5289, + "time_per_iteration": 2.8367817401885986 + }, + { + "auxiliary_loss_clip": 0.01128945, + "auxiliary_loss_mlp": 0.01045857, + "balance_loss_clip": 1.04280114, + "balance_loss_mlp": 1.03020489, + "epoch": 0.6360848914807912, + "flos": 20887442069760.0, + "grad_norm": 1.7190276119767613, + "language_loss": 0.84340358, + "learning_rate": 1.2355328208934301e-06, + "loss": 0.86515158, + "num_input_tokens_seen": 113863470, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15637207, + "step": 5290, + "time_per_iteration": 2.8003509044647217 + }, + { + "auxiliary_loss_clip": 0.01128796, + "auxiliary_loss_mlp": 0.01038095, + "balance_loss_clip": 1.03906071, + "balance_loss_mlp": 1.02198958, + "epoch": 0.6362051343714303, + "flos": 18479775386880.0, + "grad_norm": 1.6964341785185266, + "language_loss": 0.723607, + "learning_rate": 1.2348130578881728e-06, + "loss": 0.74527586, + "num_input_tokens_seen": 113881690, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.16101074, + "step": 5291, + "time_per_iteration": 2.819157361984253 + }, + { + "auxiliary_loss_clip": 0.01133118, + "auxiliary_loss_mlp": 0.01038589, + "balance_loss_clip": 1.04171836, + "balance_loss_mlp": 1.02149487, + "epoch": 0.6363253772620694, + "flos": 24389594115840.0, + "grad_norm": 2.1098629819797203, + "language_loss": 0.76154613, + "learning_rate": 1.2340934109611664e-06, + "loss": 0.78326321, + "num_input_tokens_seen": 113902450, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.17114258, + "step": 5292, + "time_per_iteration": 2.838290214538574 + }, + { + "auxiliary_loss_clip": 0.0114173, + "auxiliary_loss_mlp": 0.01045519, + "balance_loss_clip": 1.04472339, + "balance_loss_mlp": 1.02691007, + "epoch": 0.6364456201527084, + "flos": 25958243940480.0, + "grad_norm": 2.2931627818906954, + "language_loss": 0.69101119, + "learning_rate": 1.2333738802215798e-06, + "loss": 0.71288371, + "num_input_tokens_seen": 113922670, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.18603516, + "step": 5293, + "time_per_iteration": 2.908700942993164 + }, + { + "auxiliary_loss_clip": 0.0113233, + "auxiliary_loss_mlp": 0.01035549, + "balance_loss_clip": 1.04409444, + "balance_loss_mlp": 1.0201714, + "epoch": 0.6365658630433476, + "flos": 20740711011840.0, + "grad_norm": 1.8734432698074828, + "language_loss": 0.81187755, + "learning_rate": 1.2326544657785668e-06, + "loss": 0.83355635, + "num_input_tokens_seen": 113942360, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.15368652, + "step": 5294, + "time_per_iteration": 3.0161750316619873 + }, + { + "auxiliary_loss_clip": 0.01133589, + "auxiliary_loss_mlp": 0.01044898, + "balance_loss_clip": 1.04512274, + "balance_loss_mlp": 1.02881658, + "epoch": 0.6366861059339867, + "flos": 21434146047360.0, + "grad_norm": 2.49288438515531, + "language_loss": 0.74502629, + "learning_rate": 1.2319351677412608e-06, + "loss": 0.76681113, + "num_input_tokens_seen": 113959405, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16088867, + "step": 5295, + "time_per_iteration": 2.885956048965454 + }, + { + "auxiliary_loss_clip": 0.01134946, + "auxiliary_loss_mlp": 0.01041479, + "balance_loss_clip": 1.0432905, + "balance_loss_mlp": 1.02301347, + "epoch": 0.6368063488246257, + "flos": 22267093507200.0, + "grad_norm": 1.9588380392356473, + "language_loss": 0.74408931, + "learning_rate": 1.2312159862187796e-06, + "loss": 0.76585358, + "num_input_tokens_seen": 113977815, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.18469238, + "step": 5296, + "time_per_iteration": 2.8581504821777344 + }, + { + "auxiliary_loss_clip": 0.01135242, + "auxiliary_loss_mlp": 0.01039865, + "balance_loss_clip": 1.04365551, + "balance_loss_mlp": 1.02269912, + "epoch": 0.6369265917152649, + "flos": 22420719976320.0, + "grad_norm": 1.5779143966896145, + "language_loss": 0.76098198, + "learning_rate": 1.2304969213202217e-06, + "loss": 0.78273302, + "num_input_tokens_seen": 113999075, + "router_z_loss_clip": 0.91552734, + "router_z_loss_mlp": 0.17175293, + "step": 5297, + "time_per_iteration": 2.812826633453369 + }, + { + "auxiliary_loss_clip": 0.01131853, + "auxiliary_loss_mlp": 0.01034518, + "balance_loss_clip": 1.04372668, + "balance_loss_mlp": 1.01984358, + "epoch": 0.6370468346059039, + "flos": 24718176754560.0, + "grad_norm": 2.7301406455900654, + "language_loss": 0.79294682, + "learning_rate": 1.2297779731546692e-06, + "loss": 0.81461048, + "num_input_tokens_seen": 114018170, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.14672852, + "step": 5298, + "time_per_iteration": 2.847414970397949 + }, + { + "auxiliary_loss_clip": 0.01133758, + "auxiliary_loss_mlp": 0.01039634, + "balance_loss_clip": 1.04472816, + "balance_loss_mlp": 1.02314699, + "epoch": 0.637167077496543, + "flos": 25296589463040.0, + "grad_norm": 2.0992099478141433, + "language_loss": 0.77553827, + "learning_rate": 1.2290591418311853e-06, + "loss": 0.79727221, + "num_input_tokens_seen": 114035565, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.16491699, + "step": 5299, + "time_per_iteration": 3.6476943492889404 + }, + { + "auxiliary_loss_clip": 0.01136071, + "auxiliary_loss_mlp": 0.0103877, + "balance_loss_clip": 1.04400194, + "balance_loss_mlp": 1.02242661, + "epoch": 0.637287320387182, + "flos": 27671110871040.0, + "grad_norm": 1.707720622659557, + "language_loss": 0.72272861, + "learning_rate": 1.2283404274588172e-06, + "loss": 0.74447703, + "num_input_tokens_seen": 114054510, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.16333008, + "step": 5300, + "time_per_iteration": 2.7886033058166504 + }, + { + "auxiliary_loss_clip": 0.01037395, + "auxiliary_loss_mlp": 0.01000642, + "balance_loss_clip": 1.0094378, + "balance_loss_mlp": 0.99917001, + "epoch": 0.6374075632778212, + "flos": 63173406873600.0, + "grad_norm": 0.7374200190928295, + "language_loss": 0.52747178, + "learning_rate": 1.227621830146592e-06, + "loss": 0.54785216, + "num_input_tokens_seen": 114109875, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01470947, + "step": 5301, + "time_per_iteration": 4.038040637969971 + }, + { + "auxiliary_loss_clip": 0.01136244, + "auxiliary_loss_mlp": 0.01041125, + "balance_loss_clip": 1.04490638, + "balance_loss_mlp": 1.02391136, + "epoch": 0.6375278061684603, + "flos": 25558127366400.0, + "grad_norm": 2.0030342984740392, + "language_loss": 0.79423344, + "learning_rate": 1.2269033500035217e-06, + "loss": 0.8160072, + "num_input_tokens_seen": 114130010, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.17211914, + "step": 5302, + "time_per_iteration": 2.8569467067718506 + }, + { + "auxiliary_loss_clip": 0.01132935, + "auxiliary_loss_mlp": 0.01039238, + "balance_loss_clip": 1.04382312, + "balance_loss_mlp": 1.02353764, + "epoch": 0.6376480490590993, + "flos": 25666362023040.0, + "grad_norm": 1.7796113496043686, + "language_loss": 0.73653388, + "learning_rate": 1.2261849871385988e-06, + "loss": 0.7582556, + "num_input_tokens_seen": 114151115, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.15722656, + "step": 5303, + "time_per_iteration": 3.612734317779541 + }, + { + "auxiliary_loss_clip": 0.01132584, + "auxiliary_loss_mlp": 0.01040839, + "balance_loss_clip": 1.04157805, + "balance_loss_mlp": 1.02299333, + "epoch": 0.6377682919497385, + "flos": 31537684350720.0, + "grad_norm": 2.042425843544886, + "language_loss": 0.62327182, + "learning_rate": 1.2254667416607972e-06, + "loss": 0.64500606, + "num_input_tokens_seen": 114172715, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.17834473, + "step": 5304, + "time_per_iteration": 2.893362522125244 + }, + { + "auxiliary_loss_clip": 0.0113518, + "auxiliary_loss_mlp": 0.01041688, + "balance_loss_clip": 1.04425788, + "balance_loss_mlp": 1.02572572, + "epoch": 0.6378885348403776, + "flos": 23039209284480.0, + "grad_norm": 2.339888978360287, + "language_loss": 0.83071607, + "learning_rate": 1.2247486136790756e-06, + "loss": 0.85248476, + "num_input_tokens_seen": 114192195, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.1595459, + "step": 5305, + "time_per_iteration": 2.781625509262085 + }, + { + "auxiliary_loss_clip": 0.01132494, + "auxiliary_loss_mlp": 0.01040168, + "balance_loss_clip": 1.04379404, + "balance_loss_mlp": 1.02388442, + "epoch": 0.6380087777310166, + "flos": 18697070712960.0, + "grad_norm": 1.9604571175287904, + "language_loss": 0.80556035, + "learning_rate": 1.2240306033023726e-06, + "loss": 0.82728696, + "num_input_tokens_seen": 114210020, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.1628418, + "step": 5306, + "time_per_iteration": 2.838982343673706 + }, + { + "auxiliary_loss_clip": 0.01128835, + "auxiliary_loss_mlp": 0.01039383, + "balance_loss_clip": 1.0406003, + "balance_loss_mlp": 1.02345645, + "epoch": 0.6381290206216558, + "flos": 23331558078720.0, + "grad_norm": 1.6171278119187564, + "language_loss": 0.72064036, + "learning_rate": 1.223312710639611e-06, + "loss": 0.7423225, + "num_input_tokens_seen": 114228740, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15930176, + "step": 5307, + "time_per_iteration": 3.8207268714904785 + }, + { + "auxiliary_loss_clip": 0.01136311, + "auxiliary_loss_mlp": 0.01043226, + "balance_loss_clip": 1.04629874, + "balance_loss_mlp": 1.02538049, + "epoch": 0.6382492635122948, + "flos": 18880466578560.0, + "grad_norm": 3.768581077245472, + "language_loss": 0.86844623, + "learning_rate": 1.2225949357996928e-06, + "loss": 0.89024162, + "num_input_tokens_seen": 114246865, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.17834473, + "step": 5308, + "time_per_iteration": 2.861259937286377 + }, + { + "auxiliary_loss_clip": 0.01131612, + "auxiliary_loss_mlp": 0.01032466, + "balance_loss_clip": 1.04330778, + "balance_loss_mlp": 1.01674259, + "epoch": 0.6383695064029339, + "flos": 27819134818560.0, + "grad_norm": 1.5632196402069691, + "language_loss": 0.80172813, + "learning_rate": 1.221877278891505e-06, + "loss": 0.82336891, + "num_input_tokens_seen": 114266120, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.15722656, + "step": 5309, + "time_per_iteration": 2.875786781311035 + }, + { + "auxiliary_loss_clip": 0.01136884, + "auxiliary_loss_mlp": 0.01042872, + "balance_loss_clip": 1.04307413, + "balance_loss_mlp": 1.02454948, + "epoch": 0.638489749293573, + "flos": 26395635853440.0, + "grad_norm": 1.9759085395402278, + "language_loss": 0.71667469, + "learning_rate": 1.221159740023915e-06, + "loss": 0.73847222, + "num_input_tokens_seen": 114285950, + "router_z_loss_clip": 0.93847656, + "router_z_loss_mlp": 0.18310547, + "step": 5310, + "time_per_iteration": 2.856896162033081 + }, + { + "auxiliary_loss_clip": 0.01137853, + "auxiliary_loss_mlp": 0.01047472, + "balance_loss_clip": 1.04492092, + "balance_loss_mlp": 1.03061628, + "epoch": 0.6386099921842121, + "flos": 23988328306560.0, + "grad_norm": 2.0761708307805007, + "language_loss": 0.72569567, + "learning_rate": 1.2204423193057735e-06, + "loss": 0.74754894, + "num_input_tokens_seen": 114304780, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.16845703, + "step": 5311, + "time_per_iteration": 2.8054091930389404 + }, + { + "auxiliary_loss_clip": 0.01037796, + "auxiliary_loss_mlp": 0.01004451, + "balance_loss_clip": 1.00998187, + "balance_loss_mlp": 1.00291276, + "epoch": 0.6387302350748512, + "flos": 71731169337600.0, + "grad_norm": 0.8523769540649622, + "language_loss": 0.63379496, + "learning_rate": 1.2197250168459122e-06, + "loss": 0.65421742, + "num_input_tokens_seen": 114361180, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01538086, + "step": 5312, + "time_per_iteration": 3.401499032974243 + }, + { + "auxiliary_loss_clip": 0.01129722, + "auxiliary_loss_mlp": 0.01043094, + "balance_loss_clip": 1.04274821, + "balance_loss_mlp": 1.02669072, + "epoch": 0.6388504779654903, + "flos": 14535778141440.0, + "grad_norm": 1.7947106662564527, + "language_loss": 0.74284297, + "learning_rate": 1.2190078327531454e-06, + "loss": 0.76457107, + "num_input_tokens_seen": 114377425, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.16418457, + "step": 5313, + "time_per_iteration": 2.8055953979492188 + }, + { + "auxiliary_loss_clip": 0.01132707, + "auxiliary_loss_mlp": 0.0104321, + "balance_loss_clip": 1.04414916, + "balance_loss_mlp": 1.02652061, + "epoch": 0.6389707208561294, + "flos": 22346133384960.0, + "grad_norm": 1.530781482943438, + "language_loss": 0.72686946, + "learning_rate": 1.2182907671362697e-06, + "loss": 0.74862862, + "num_input_tokens_seen": 114398120, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.16699219, + "step": 5314, + "time_per_iteration": 2.9621949195861816 + }, + { + "auxiliary_loss_clip": 0.01135099, + "auxiliary_loss_mlp": 0.01044142, + "balance_loss_clip": 1.04393971, + "balance_loss_mlp": 1.02772701, + "epoch": 0.6390909637467684, + "flos": 19426883247360.0, + "grad_norm": 1.7767021898185187, + "language_loss": 0.78630441, + "learning_rate": 1.2175738201040626e-06, + "loss": 0.80809677, + "num_input_tokens_seen": 114415160, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.1640625, + "step": 5315, + "time_per_iteration": 2.8203251361846924 + }, + { + "auxiliary_loss_clip": 0.01134443, + "auxiliary_loss_mlp": 0.01044409, + "balance_loss_clip": 1.04279089, + "balance_loss_mlp": 1.02700496, + "epoch": 0.6392112066374076, + "flos": 24090852700800.0, + "grad_norm": 1.8298494420666305, + "language_loss": 0.78427392, + "learning_rate": 1.2168569917652855e-06, + "loss": 0.80606234, + "num_input_tokens_seen": 114435015, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.17419434, + "step": 5316, + "time_per_iteration": 2.8575735092163086 + }, + { + "auxiliary_loss_clip": 0.01132878, + "auxiliary_loss_mlp": 0.01044427, + "balance_loss_clip": 1.04321814, + "balance_loss_mlp": 1.02769041, + "epoch": 0.6393314495280467, + "flos": 26795141896320.0, + "grad_norm": 1.4143953572738917, + "language_loss": 0.63887119, + "learning_rate": 1.2161402822286797e-06, + "loss": 0.66064417, + "num_input_tokens_seen": 114455700, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.16748047, + "step": 5317, + "time_per_iteration": 2.900620460510254 + }, + { + "auxiliary_loss_clip": 0.01134375, + "auxiliary_loss_mlp": 0.01041142, + "balance_loss_clip": 1.04355311, + "balance_loss_mlp": 1.02301073, + "epoch": 0.6394516924186857, + "flos": 20260692633600.0, + "grad_norm": 1.9265176442401195, + "language_loss": 0.79192609, + "learning_rate": 1.2154236916029703e-06, + "loss": 0.81368124, + "num_input_tokens_seen": 114473675, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.18127441, + "step": 5318, + "time_per_iteration": 2.8480944633483887 + }, + { + "auxiliary_loss_clip": 0.01129188, + "auxiliary_loss_mlp": 0.01045661, + "balance_loss_clip": 1.04079127, + "balance_loss_mlp": 1.02924573, + "epoch": 0.6395719353093249, + "flos": 18368847210240.0, + "grad_norm": 2.7042506109911963, + "language_loss": 0.73651707, + "learning_rate": 1.2147072199968627e-06, + "loss": 0.7582655, + "num_input_tokens_seen": 114492310, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.16418457, + "step": 5319, + "time_per_iteration": 2.9971017837524414 + }, + { + "auxiliary_loss_clip": 0.01128732, + "auxiliary_loss_mlp": 0.01039097, + "balance_loss_clip": 1.04216111, + "balance_loss_mlp": 1.02294445, + "epoch": 0.6396921781999639, + "flos": 17566315591680.0, + "grad_norm": 1.7824094818713692, + "language_loss": 0.71911716, + "learning_rate": 1.2139908675190454e-06, + "loss": 0.74079543, + "num_input_tokens_seen": 114511520, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.16149902, + "step": 5320, + "time_per_iteration": 2.90279221534729 + }, + { + "auxiliary_loss_clip": 0.01132561, + "auxiliary_loss_mlp": 0.01040294, + "balance_loss_clip": 1.04440439, + "balance_loss_mlp": 1.02323472, + "epoch": 0.639812421090603, + "flos": 21251252972160.0, + "grad_norm": 1.8019905557743419, + "language_loss": 0.75077373, + "learning_rate": 1.2132746342781883e-06, + "loss": 0.7725023, + "num_input_tokens_seen": 114532680, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.17041016, + "step": 5321, + "time_per_iteration": 2.9395992755889893 + }, + { + "auxiliary_loss_clip": 0.01133179, + "auxiliary_loss_mlp": 0.01037147, + "balance_loss_clip": 1.04242611, + "balance_loss_mlp": 1.02013564, + "epoch": 0.6399326639812422, + "flos": 11180967684480.0, + "grad_norm": 2.4662318770146303, + "language_loss": 0.79779208, + "learning_rate": 1.2125585203829442e-06, + "loss": 0.81949532, + "num_input_tokens_seen": 114548320, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.17028809, + "step": 5322, + "time_per_iteration": 3.0242834091186523 + }, + { + "auxiliary_loss_clip": 0.01134823, + "auxiliary_loss_mlp": 0.01036707, + "balance_loss_clip": 1.04336357, + "balance_loss_mlp": 1.0204587, + "epoch": 0.6400529068718812, + "flos": 23911048195200.0, + "grad_norm": 1.7922356226111593, + "language_loss": 0.7423867, + "learning_rate": 1.211842525941946e-06, + "loss": 0.76410204, + "num_input_tokens_seen": 114568115, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.16235352, + "step": 5323, + "time_per_iteration": 2.966813325881958 + }, + { + "auxiliary_loss_clip": 0.01129177, + "auxiliary_loss_mlp": 0.01038175, + "balance_loss_clip": 1.04175019, + "balance_loss_mlp": 1.02127123, + "epoch": 0.6401731497625203, + "flos": 44018724890880.0, + "grad_norm": 1.8005955843710513, + "language_loss": 0.78944218, + "learning_rate": 1.2111266510638105e-06, + "loss": 0.81111574, + "num_input_tokens_seen": 114591040, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.16912842, + "step": 5324, + "time_per_iteration": 3.072371244430542 + }, + { + "auxiliary_loss_clip": 0.01134215, + "auxiliary_loss_mlp": 0.01043349, + "balance_loss_clip": 1.04452896, + "balance_loss_mlp": 1.02705336, + "epoch": 0.6402933926531594, + "flos": 20662209838080.0, + "grad_norm": 3.928309147185363, + "language_loss": 0.79949826, + "learning_rate": 1.2104108958571346e-06, + "loss": 0.82127392, + "num_input_tokens_seen": 114609310, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.1628418, + "step": 5325, + "time_per_iteration": 2.778566837310791 + }, + { + "auxiliary_loss_clip": 0.01128423, + "auxiliary_loss_mlp": 0.01044791, + "balance_loss_clip": 1.04124427, + "balance_loss_mlp": 1.02886415, + "epoch": 0.6404136355437985, + "flos": 24863327614080.0, + "grad_norm": 1.4658882186355653, + "language_loss": 0.75629169, + "learning_rate": 1.2096952604304975e-06, + "loss": 0.77802378, + "num_input_tokens_seen": 114629740, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15899658, + "step": 5326, + "time_per_iteration": 2.822505235671997 + }, + { + "auxiliary_loss_clip": 0.01133554, + "auxiliary_loss_mlp": 0.0104194, + "balance_loss_clip": 1.04127717, + "balance_loss_mlp": 1.02415383, + "epoch": 0.6405338784344375, + "flos": 40479548901120.0, + "grad_norm": 1.8617558833011922, + "language_loss": 0.70457327, + "learning_rate": 1.2089797448924616e-06, + "loss": 0.72632813, + "num_input_tokens_seen": 114653615, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.17785645, + "step": 5327, + "time_per_iteration": 3.789191484451294 + }, + { + "auxiliary_loss_clip": 0.01131683, + "auxiliary_loss_mlp": 0.01040597, + "balance_loss_clip": 1.04124212, + "balance_loss_mlp": 1.02446771, + "epoch": 0.6406541213250767, + "flos": 20886041439360.0, + "grad_norm": 1.987962599562607, + "language_loss": 0.66145927, + "learning_rate": 1.2082643493515692e-06, + "loss": 0.68318206, + "num_input_tokens_seen": 114671935, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.16137695, + "step": 5328, + "time_per_iteration": 2.7719883918762207 + }, + { + "auxiliary_loss_clip": 0.0113259, + "auxiliary_loss_mlp": 0.01036529, + "balance_loss_clip": 1.04395103, + "balance_loss_mlp": 1.01990533, + "epoch": 0.6407743642157158, + "flos": 23295970679040.0, + "grad_norm": 1.8406312109805238, + "language_loss": 0.81899822, + "learning_rate": 1.207549073916346e-06, + "loss": 0.84068936, + "num_input_tokens_seen": 114692870, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.16619873, + "step": 5329, + "time_per_iteration": 2.8793482780456543 + }, + { + "auxiliary_loss_clip": 0.01135836, + "auxiliary_loss_mlp": 0.01047977, + "balance_loss_clip": 1.04529917, + "balance_loss_mlp": 1.0313834, + "epoch": 0.6408946071063548, + "flos": 15012636122880.0, + "grad_norm": 1.9437467243987772, + "language_loss": 0.77853554, + "learning_rate": 1.2068339186952976e-06, + "loss": 0.80037367, + "num_input_tokens_seen": 114710410, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.16601562, + "step": 5330, + "time_per_iteration": 3.6996662616729736 + }, + { + "auxiliary_loss_clip": 0.01136159, + "auxiliary_loss_mlp": 0.01041336, + "balance_loss_clip": 1.0453701, + "balance_loss_mlp": 1.02524281, + "epoch": 0.6410148499969939, + "flos": 22528595496960.0, + "grad_norm": 1.6520670470001857, + "language_loss": 0.73107111, + "learning_rate": 1.2061188837969136e-06, + "loss": 0.75284612, + "num_input_tokens_seen": 114730020, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.16101074, + "step": 5331, + "time_per_iteration": 2.799574613571167 + }, + { + "auxiliary_loss_clip": 0.01136404, + "auxiliary_loss_mlp": 0.01039796, + "balance_loss_clip": 1.04281569, + "balance_loss_mlp": 1.02242696, + "epoch": 0.641135092887633, + "flos": 12422004537600.0, + "grad_norm": 2.428695396155329, + "language_loss": 0.84696907, + "learning_rate": 1.2054039693296631e-06, + "loss": 0.86873114, + "num_input_tokens_seen": 114748015, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.17370605, + "step": 5332, + "time_per_iteration": 3.736879348754883 + }, + { + "auxiliary_loss_clip": 0.0113009, + "auxiliary_loss_mlp": 0.01040586, + "balance_loss_clip": 1.04256034, + "balance_loss_mlp": 1.02561951, + "epoch": 0.6412553357782721, + "flos": 22127329687680.0, + "grad_norm": 1.7498869051118218, + "language_loss": 0.81323785, + "learning_rate": 1.2046891754019992e-06, + "loss": 0.83494461, + "num_input_tokens_seen": 114768625, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.14953613, + "step": 5333, + "time_per_iteration": 2.877394676208496 + }, + { + "auxiliary_loss_clip": 0.01131083, + "auxiliary_loss_mlp": 0.0104244, + "balance_loss_clip": 1.04347968, + "balance_loss_mlp": 1.02584577, + "epoch": 0.6413755786689112, + "flos": 15888605097600.0, + "grad_norm": 1.824928703955403, + "language_loss": 0.82653463, + "learning_rate": 1.2039745021223548e-06, + "loss": 0.84826982, + "num_input_tokens_seen": 114786045, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16595459, + "step": 5334, + "time_per_iteration": 2.949204921722412 + }, + { + "auxiliary_loss_clip": 0.01040023, + "auxiliary_loss_mlp": 0.0100043, + "balance_loss_clip": 1.01178145, + "balance_loss_mlp": 0.99895823, + "epoch": 0.6414958215595503, + "flos": 68039159955840.0, + "grad_norm": 0.7838213741054587, + "language_loss": 0.570198, + "learning_rate": 1.2032599495991456e-06, + "loss": 0.59060252, + "num_input_tokens_seen": 114850785, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01470947, + "step": 5335, + "time_per_iteration": 3.45573091506958 + }, + { + "auxiliary_loss_clip": 0.01137252, + "auxiliary_loss_mlp": 0.01043197, + "balance_loss_clip": 1.04453278, + "balance_loss_mlp": 1.02624583, + "epoch": 0.6416160644501894, + "flos": 44091300320640.0, + "grad_norm": 2.0038550638993717, + "language_loss": 0.69486135, + "learning_rate": 1.2025455179407685e-06, + "loss": 0.71666586, + "num_input_tokens_seen": 114871945, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.16943359, + "step": 5336, + "time_per_iteration": 3.93263578414917 + }, + { + "auxiliary_loss_clip": 0.01133041, + "auxiliary_loss_mlp": 0.01043379, + "balance_loss_clip": 1.04189086, + "balance_loss_mlp": 1.02555704, + "epoch": 0.6417363073408284, + "flos": 20959837931520.0, + "grad_norm": 5.529019497200021, + "language_loss": 0.73788464, + "learning_rate": 1.2018312072556022e-06, + "loss": 0.7596488, + "num_input_tokens_seen": 114890445, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.17822266, + "step": 5337, + "time_per_iteration": 2.845290422439575 + }, + { + "auxiliary_loss_clip": 0.01128488, + "auxiliary_loss_mlp": 0.01047144, + "balance_loss_clip": 1.04041529, + "balance_loss_mlp": 1.02914369, + "epoch": 0.6418565502314676, + "flos": 22455122227200.0, + "grad_norm": 2.5003645906441885, + "language_loss": 0.74366736, + "learning_rate": 1.2011170176520077e-06, + "loss": 0.76542366, + "num_input_tokens_seen": 114911360, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.17980957, + "step": 5338, + "time_per_iteration": 2.9010097980499268 + }, + { + "auxiliary_loss_clip": 0.01126032, + "auxiliary_loss_mlp": 0.01036756, + "balance_loss_clip": 1.04096913, + "balance_loss_mlp": 1.02088952, + "epoch": 0.6419767931221066, + "flos": 25045502417280.0, + "grad_norm": 1.5985017185367691, + "language_loss": 0.81486768, + "learning_rate": 1.2004029492383256e-06, + "loss": 0.83649552, + "num_input_tokens_seen": 114932700, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.15869141, + "step": 5339, + "time_per_iteration": 2.8676035404205322 + }, + { + "auxiliary_loss_clip": 0.01130816, + "auxiliary_loss_mlp": 0.01040392, + "balance_loss_clip": 1.0428102, + "balance_loss_mlp": 1.02383375, + "epoch": 0.6420970360127457, + "flos": 19463691709440.0, + "grad_norm": 1.7873416271738554, + "language_loss": 0.73593575, + "learning_rate": 1.1996890021228814e-06, + "loss": 0.75764781, + "num_input_tokens_seen": 114949475, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.16564941, + "step": 5340, + "time_per_iteration": 2.918879747390747 + }, + { + "auxiliary_loss_clip": 0.011307, + "auxiliary_loss_mlp": 0.01045506, + "balance_loss_clip": 1.04212189, + "balance_loss_mlp": 1.0293417, + "epoch": 0.6422172789033849, + "flos": 40406147458560.0, + "grad_norm": 1.5359712612373198, + "language_loss": 0.7032041, + "learning_rate": 1.1989751764139785e-06, + "loss": 0.72496617, + "num_input_tokens_seen": 114973125, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.16149902, + "step": 5341, + "time_per_iteration": 3.0006628036499023 + }, + { + "auxiliary_loss_clip": 0.01133291, + "auxiliary_loss_mlp": 0.01039812, + "balance_loss_clip": 1.04237533, + "balance_loss_mlp": 1.0243926, + "epoch": 0.6423375217940239, + "flos": 27672870637440.0, + "grad_norm": 1.5749049997437838, + "language_loss": 0.83060193, + "learning_rate": 1.1982614722199044e-06, + "loss": 0.85233295, + "num_input_tokens_seen": 114994300, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.15405273, + "step": 5342, + "time_per_iteration": 2.875176191329956 + }, + { + "auxiliary_loss_clip": 0.01130601, + "auxiliary_loss_mlp": 0.01039509, + "balance_loss_clip": 1.04285288, + "balance_loss_mlp": 1.02354717, + "epoch": 0.642457764684663, + "flos": 18369242259840.0, + "grad_norm": 1.861779306844643, + "language_loss": 0.77520847, + "learning_rate": 1.1975478896489276e-06, + "loss": 0.79690957, + "num_input_tokens_seen": 115012135, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.15966797, + "step": 5343, + "time_per_iteration": 2.9416427612304688 + }, + { + "auxiliary_loss_clip": 0.01128234, + "auxiliary_loss_mlp": 0.01042022, + "balance_loss_clip": 1.04077756, + "balance_loss_mlp": 1.02669203, + "epoch": 0.6425780075753021, + "flos": 19750509809280.0, + "grad_norm": 1.9614946812440865, + "language_loss": 0.76763743, + "learning_rate": 1.1968344288092981e-06, + "loss": 0.78934002, + "num_input_tokens_seen": 115028715, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.15319824, + "step": 5344, + "time_per_iteration": 2.9030487537384033 + }, + { + "auxiliary_loss_clip": 0.01130889, + "auxiliary_loss_mlp": 0.0103694, + "balance_loss_clip": 1.04199004, + "balance_loss_mlp": 1.02149022, + "epoch": 0.6426982504659412, + "flos": 20558536208640.0, + "grad_norm": 1.866469901145357, + "language_loss": 0.64629447, + "learning_rate": 1.1961210898092468e-06, + "loss": 0.66797268, + "num_input_tokens_seen": 115047665, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.15454102, + "step": 5345, + "time_per_iteration": 2.8281760215759277 + }, + { + "auxiliary_loss_clip": 0.01136289, + "auxiliary_loss_mlp": 0.01037679, + "balance_loss_clip": 1.04415429, + "balance_loss_mlp": 1.02104902, + "epoch": 0.6428184933565803, + "flos": 17851984456320.0, + "grad_norm": 2.0375446241239366, + "language_loss": 0.79262614, + "learning_rate": 1.1954078727569874e-06, + "loss": 0.8143658, + "num_input_tokens_seen": 115064965, + "router_z_loss_clip": 0.92089844, + "router_z_loss_mlp": 0.16625977, + "step": 5346, + "time_per_iteration": 2.7652204036712646 + }, + { + "auxiliary_loss_clip": 0.01132665, + "auxiliary_loss_mlp": 0.01036817, + "balance_loss_clip": 1.04274273, + "balance_loss_mlp": 1.02050912, + "epoch": 0.6429387362472194, + "flos": 22456953820800.0, + "grad_norm": 1.6128619188093523, + "language_loss": 0.77700871, + "learning_rate": 1.1946947777607141e-06, + "loss": 0.79870355, + "num_input_tokens_seen": 115086100, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.16308594, + "step": 5347, + "time_per_iteration": 2.876812219619751 + }, + { + "auxiliary_loss_clip": 0.01134266, + "auxiliary_loss_mlp": 0.01046469, + "balance_loss_clip": 1.04211307, + "balance_loss_mlp": 1.02877855, + "epoch": 0.6430589791378585, + "flos": 24752579005440.0, + "grad_norm": 2.1558257596834944, + "language_loss": 0.8012076, + "learning_rate": 1.1939818049286024e-06, + "loss": 0.82301497, + "num_input_tokens_seen": 115104260, + "router_z_loss_clip": 0.92138672, + "router_z_loss_mlp": 0.17687988, + "step": 5348, + "time_per_iteration": 2.856502056121826 + }, + { + "auxiliary_loss_clip": 0.01131853, + "auxiliary_loss_mlp": 0.01047848, + "balance_loss_clip": 1.04162514, + "balance_loss_mlp": 1.03095627, + "epoch": 0.6431792220284975, + "flos": 24901249397760.0, + "grad_norm": 1.5682715545961383, + "language_loss": 0.754556, + "learning_rate": 1.1932689543688101e-06, + "loss": 0.776353, + "num_input_tokens_seen": 115125365, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.16894531, + "step": 5349, + "time_per_iteration": 2.823418617248535 + }, + { + "auxiliary_loss_clip": 0.01134141, + "auxiliary_loss_mlp": 0.01041385, + "balance_loss_clip": 1.04250312, + "balance_loss_mlp": 1.0239445, + "epoch": 0.6432994649191367, + "flos": 21032305620480.0, + "grad_norm": 1.9057007152122523, + "language_loss": 0.72647882, + "learning_rate": 1.1925562261894756e-06, + "loss": 0.74823409, + "num_input_tokens_seen": 115144445, + "router_z_loss_clip": 0.91699219, + "router_z_loss_mlp": 0.17443848, + "step": 5350, + "time_per_iteration": 2.8595123291015625 + }, + { + "auxiliary_loss_clip": 0.01130068, + "auxiliary_loss_mlp": 0.01031366, + "balance_loss_clip": 1.04169893, + "balance_loss_mlp": 1.01529646, + "epoch": 0.6434197078097758, + "flos": 30884433655680.0, + "grad_norm": 1.769138796407044, + "language_loss": 0.77468079, + "learning_rate": 1.1918436204987207e-06, + "loss": 0.79629517, + "num_input_tokens_seen": 115166305, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.16088867, + "step": 5351, + "time_per_iteration": 2.914436101913452 + }, + { + "auxiliary_loss_clip": 0.01129979, + "auxiliary_loss_mlp": 0.01038592, + "balance_loss_clip": 1.0417285, + "balance_loss_mlp": 1.02340436, + "epoch": 0.6435399507004148, + "flos": 15012492468480.0, + "grad_norm": 2.0432650236100884, + "language_loss": 0.81751609, + "learning_rate": 1.191131137404645e-06, + "loss": 0.83920181, + "num_input_tokens_seen": 115183045, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.1517334, + "step": 5352, + "time_per_iteration": 2.8828909397125244 + }, + { + "auxiliary_loss_clip": 0.01134679, + "auxiliary_loss_mlp": 0.01042899, + "balance_loss_clip": 1.04333758, + "balance_loss_mlp": 1.02688313, + "epoch": 0.643660193591054, + "flos": 19901981462400.0, + "grad_norm": 2.0335267791611082, + "language_loss": 0.77405441, + "learning_rate": 1.190418777015333e-06, + "loss": 0.79583013, + "num_input_tokens_seen": 115201955, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.15997314, + "step": 5353, + "time_per_iteration": 2.865530490875244 + }, + { + "auxiliary_loss_clip": 0.01130009, + "auxiliary_loss_mlp": 0.01036589, + "balance_loss_clip": 1.04100621, + "balance_loss_mlp": 1.02113914, + "epoch": 0.643780436481693, + "flos": 24133622820480.0, + "grad_norm": 1.4353979144012146, + "language_loss": 0.73692095, + "learning_rate": 1.1897065394388487e-06, + "loss": 0.75858688, + "num_input_tokens_seen": 115222395, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.15441895, + "step": 5354, + "time_per_iteration": 2.917651653289795 + }, + { + "auxiliary_loss_clip": 0.01131072, + "auxiliary_loss_mlp": 0.01042752, + "balance_loss_clip": 1.04408407, + "balance_loss_mlp": 1.02723086, + "epoch": 0.6439006793723321, + "flos": 23148808657920.0, + "grad_norm": 1.5323193867956542, + "language_loss": 0.76815319, + "learning_rate": 1.1889944247832385e-06, + "loss": 0.78989136, + "num_input_tokens_seen": 115242635, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15527344, + "step": 5355, + "time_per_iteration": 3.031383752822876 + }, + { + "auxiliary_loss_clip": 0.01130091, + "auxiliary_loss_mlp": 0.01040628, + "balance_loss_clip": 1.04114604, + "balance_loss_mlp": 1.02470207, + "epoch": 0.6440209222629713, + "flos": 23617909301760.0, + "grad_norm": 3.0520513827486786, + "language_loss": 0.70903552, + "learning_rate": 1.1882824331565283e-06, + "loss": 0.73074275, + "num_input_tokens_seen": 115262095, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.15911865, + "step": 5356, + "time_per_iteration": 3.6916515827178955 + }, + { + "auxiliary_loss_clip": 0.01127019, + "auxiliary_loss_mlp": 0.01040405, + "balance_loss_clip": 1.04104161, + "balance_loss_mlp": 1.02496731, + "epoch": 0.6441411651536103, + "flos": 16544872535040.0, + "grad_norm": 2.011941008655845, + "language_loss": 0.89140528, + "learning_rate": 1.1875705646667287e-06, + "loss": 0.9130795, + "num_input_tokens_seen": 115279985, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15429688, + "step": 5357, + "time_per_iteration": 2.8575384616851807 + }, + { + "auxiliary_loss_clip": 0.01128763, + "auxiliary_loss_mlp": 0.01035198, + "balance_loss_clip": 1.04019439, + "balance_loss_mlp": 1.0194025, + "epoch": 0.6442614080442494, + "flos": 25410965345280.0, + "grad_norm": 2.3594021791696966, + "language_loss": 0.75281578, + "learning_rate": 1.1868588194218282e-06, + "loss": 0.77445543, + "num_input_tokens_seen": 115300365, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.15795898, + "step": 5358, + "time_per_iteration": 2.8638274669647217 + }, + { + "auxiliary_loss_clip": 0.01132936, + "auxiliary_loss_mlp": 0.01034809, + "balance_loss_clip": 1.04240191, + "balance_loss_mlp": 1.01897836, + "epoch": 0.6443816509348885, + "flos": 28294017552000.0, + "grad_norm": 1.5461776831733416, + "language_loss": 0.74196827, + "learning_rate": 1.1861471975297979e-06, + "loss": 0.76364577, + "num_input_tokens_seen": 115322060, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.15838623, + "step": 5359, + "time_per_iteration": 3.602487564086914 + }, + { + "auxiliary_loss_clip": 0.01135412, + "auxiliary_loss_mlp": 0.01042232, + "balance_loss_clip": 1.04232979, + "balance_loss_mlp": 1.02493477, + "epoch": 0.6445018938255276, + "flos": 36690075964800.0, + "grad_norm": 1.522303555064249, + "language_loss": 0.7096827, + "learning_rate": 1.185435699098591e-06, + "loss": 0.73145914, + "num_input_tokens_seen": 115348255, + "router_z_loss_clip": 0.93164062, + "router_z_loss_mlp": 0.17285156, + "step": 5360, + "time_per_iteration": 2.9966108798980713 + }, + { + "auxiliary_loss_clip": 0.01135111, + "auxiliary_loss_mlp": 0.0103975, + "balance_loss_clip": 1.04405928, + "balance_loss_mlp": 1.02298903, + "epoch": 0.6446221367161666, + "flos": 14501411804160.0, + "grad_norm": 2.1760721311445934, + "language_loss": 0.7782802, + "learning_rate": 1.1847243242361403e-06, + "loss": 0.8000288, + "num_input_tokens_seen": 115366845, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.16772461, + "step": 5361, + "time_per_iteration": 3.619285821914673 + }, + { + "auxiliary_loss_clip": 0.01131932, + "auxiliary_loss_mlp": 0.01040812, + "balance_loss_clip": 1.04141665, + "balance_loss_mlp": 1.02418208, + "epoch": 0.6447423796068057, + "flos": 24609367480320.0, + "grad_norm": 2.013315089820579, + "language_loss": 0.7825563, + "learning_rate": 1.1840130730503624e-06, + "loss": 0.80428362, + "num_input_tokens_seen": 115388125, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.1661377, + "step": 5362, + "time_per_iteration": 2.8282885551452637 + }, + { + "auxiliary_loss_clip": 0.01128971, + "auxiliary_loss_mlp": 0.01036717, + "balance_loss_clip": 1.04016113, + "balance_loss_mlp": 1.0219593, + "epoch": 0.6448626224974449, + "flos": 25047298097280.0, + "grad_norm": 1.7401652269148684, + "language_loss": 0.74887329, + "learning_rate": 1.1833019456491518e-06, + "loss": 0.7705301, + "num_input_tokens_seen": 115409655, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.14746094, + "step": 5363, + "time_per_iteration": 2.920673131942749 + }, + { + "auxiliary_loss_clip": 0.01130841, + "auxiliary_loss_mlp": 0.01042999, + "balance_loss_clip": 1.04107428, + "balance_loss_mlp": 1.02653587, + "epoch": 0.6449828653880839, + "flos": 22530355263360.0, + "grad_norm": 2.0385522349695067, + "language_loss": 0.79684782, + "learning_rate": 1.1825909421403871e-06, + "loss": 0.81858623, + "num_input_tokens_seen": 115428750, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.16455078, + "step": 5364, + "time_per_iteration": 2.878269910812378 + }, + { + "auxiliary_loss_clip": 0.01130843, + "auxiliary_loss_mlp": 0.01037137, + "balance_loss_clip": 1.04075789, + "balance_loss_mlp": 1.02132356, + "epoch": 0.645103108278723, + "flos": 25695736369920.0, + "grad_norm": 1.6707427610607357, + "language_loss": 0.76210403, + "learning_rate": 1.181880062631926e-06, + "loss": 0.78378391, + "num_input_tokens_seen": 115448085, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.15802002, + "step": 5365, + "time_per_iteration": 3.8379266262054443 + }, + { + "auxiliary_loss_clip": 0.01137106, + "auxiliary_loss_mlp": 0.01042659, + "balance_loss_clip": 1.04665971, + "balance_loss_mlp": 1.02561188, + "epoch": 0.6452233511693621, + "flos": 27450331925760.0, + "grad_norm": 2.057848910909326, + "language_loss": 0.84874141, + "learning_rate": 1.1811693072316093e-06, + "loss": 0.87053901, + "num_input_tokens_seen": 115465765, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.17053223, + "step": 5366, + "time_per_iteration": 2.9165594577789307 + }, + { + "auxiliary_loss_clip": 0.01129333, + "auxiliary_loss_mlp": 0.01040092, + "balance_loss_clip": 1.03964186, + "balance_loss_mlp": 1.02364099, + "epoch": 0.6453435940600012, + "flos": 19208618254080.0, + "grad_norm": 2.3950109926716894, + "language_loss": 0.83832705, + "learning_rate": 1.1804586760472574e-06, + "loss": 0.86002129, + "num_input_tokens_seen": 115482230, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.16442871, + "step": 5367, + "time_per_iteration": 2.8758270740509033 + }, + { + "auxiliary_loss_clip": 0.01132517, + "auxiliary_loss_mlp": 0.01038606, + "balance_loss_clip": 1.04402804, + "balance_loss_mlp": 1.02324009, + "epoch": 0.6454638369506402, + "flos": 25737680476800.0, + "grad_norm": 3.29149585082439, + "language_loss": 0.79989994, + "learning_rate": 1.1797481691866736e-06, + "loss": 0.82161117, + "num_input_tokens_seen": 115499455, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.15356445, + "step": 5368, + "time_per_iteration": 2.9078009128570557 + }, + { + "auxiliary_loss_clip": 0.01127962, + "auxiliary_loss_mlp": 0.01042022, + "balance_loss_clip": 1.04266632, + "balance_loss_mlp": 1.02722824, + "epoch": 0.6455840798412794, + "flos": 20989176364800.0, + "grad_norm": 2.012679951302416, + "language_loss": 0.83022052, + "learning_rate": 1.1790377867576393e-06, + "loss": 0.85192037, + "num_input_tokens_seen": 115517205, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.14776611, + "step": 5369, + "time_per_iteration": 2.791853904724121 + }, + { + "auxiliary_loss_clip": 0.01130905, + "auxiliary_loss_mlp": 0.01040148, + "balance_loss_clip": 1.04197049, + "balance_loss_mlp": 1.02497232, + "epoch": 0.6457043227319185, + "flos": 26067556005120.0, + "grad_norm": 1.7053233203652323, + "language_loss": 0.76310492, + "learning_rate": 1.1783275288679203e-06, + "loss": 0.78481543, + "num_input_tokens_seen": 115534370, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.1517334, + "step": 5370, + "time_per_iteration": 2.838085412979126 + }, + { + "auxiliary_loss_clip": 0.01036871, + "auxiliary_loss_mlp": 0.01001226, + "balance_loss_clip": 1.00910091, + "balance_loss_mlp": 0.9998194, + "epoch": 0.6458245656225575, + "flos": 60370831088640.0, + "grad_norm": 0.8465258621048389, + "language_loss": 0.57097864, + "learning_rate": 1.177617395625262e-06, + "loss": 0.59135962, + "num_input_tokens_seen": 115592345, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01403809, + "step": 5371, + "time_per_iteration": 3.350160598754883 + }, + { + "auxiliary_loss_clip": 0.01131376, + "auxiliary_loss_mlp": 0.0104108, + "balance_loss_clip": 1.04160357, + "balance_loss_mlp": 1.02609539, + "epoch": 0.6459448085131967, + "flos": 23076771932160.0, + "grad_norm": 1.9087872634383944, + "language_loss": 0.75330734, + "learning_rate": 1.1769073871373908e-06, + "loss": 0.77503192, + "num_input_tokens_seen": 115612550, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.14978027, + "step": 5372, + "time_per_iteration": 2.8546268939971924 + }, + { + "auxiliary_loss_clip": 0.01129893, + "auxiliary_loss_mlp": 0.01041477, + "balance_loss_clip": 1.04216135, + "balance_loss_mlp": 1.02528858, + "epoch": 0.6460650514038357, + "flos": 22598190097920.0, + "grad_norm": 1.7648093253477268, + "language_loss": 0.83566839, + "learning_rate": 1.176197503512015e-06, + "loss": 0.85738206, + "num_input_tokens_seen": 115632265, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.16186523, + "step": 5373, + "time_per_iteration": 2.910771369934082 + }, + { + "auxiliary_loss_clip": 0.01130213, + "auxiliary_loss_mlp": 0.01043365, + "balance_loss_clip": 1.04366076, + "balance_loss_mlp": 1.02907205, + "epoch": 0.6461852942944748, + "flos": 20266726118400.0, + "grad_norm": 2.080106445937724, + "language_loss": 0.83231241, + "learning_rate": 1.1754877448568223e-06, + "loss": 0.85404819, + "num_input_tokens_seen": 115651720, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.14282227, + "step": 5374, + "time_per_iteration": 2.855806350708008 + }, + { + "auxiliary_loss_clip": 0.01129685, + "auxiliary_loss_mlp": 0.01036666, + "balance_loss_clip": 1.04169464, + "balance_loss_mlp": 1.02128792, + "epoch": 0.646305537185114, + "flos": 23367109564800.0, + "grad_norm": 2.5629970042541674, + "language_loss": 0.89909708, + "learning_rate": 1.1747781112794837e-06, + "loss": 0.92076063, + "num_input_tokens_seen": 115668215, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.15368652, + "step": 5375, + "time_per_iteration": 2.9271528720855713 + }, + { + "auxiliary_loss_clip": 0.01133092, + "auxiliary_loss_mlp": 0.01039527, + "balance_loss_clip": 1.0460093, + "balance_loss_mlp": 1.02439904, + "epoch": 0.646425780075753, + "flos": 24277480790400.0, + "grad_norm": 1.6718170447605172, + "language_loss": 0.83112264, + "learning_rate": 1.1740686028876487e-06, + "loss": 0.85284883, + "num_input_tokens_seen": 115687080, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15124512, + "step": 5376, + "time_per_iteration": 2.8351433277130127 + }, + { + "auxiliary_loss_clip": 0.01129784, + "auxiliary_loss_mlp": 0.01035936, + "balance_loss_clip": 1.04118991, + "balance_loss_mlp": 1.02077889, + "epoch": 0.6465460229663921, + "flos": 20813968800000.0, + "grad_norm": 2.4599572503555915, + "language_loss": 0.74902153, + "learning_rate": 1.1733592197889507e-06, + "loss": 0.77067864, + "num_input_tokens_seen": 115703990, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.15167236, + "step": 5377, + "time_per_iteration": 2.89418888092041 + }, + { + "auxiliary_loss_clip": 0.01128742, + "auxiliary_loss_mlp": 0.01032121, + "balance_loss_clip": 1.04329658, + "balance_loss_mlp": 1.01720774, + "epoch": 0.6466662658570312, + "flos": 22853299466880.0, + "grad_norm": 1.8695357124556171, + "language_loss": 0.72521114, + "learning_rate": 1.1726499620910014e-06, + "loss": 0.74681973, + "num_input_tokens_seen": 115724270, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.14892578, + "step": 5378, + "time_per_iteration": 2.9247305393218994 + }, + { + "auxiliary_loss_clip": 0.01132885, + "auxiliary_loss_mlp": 0.01042503, + "balance_loss_clip": 1.04299629, + "balance_loss_mlp": 1.02644587, + "epoch": 0.6467865087476703, + "flos": 15304553953920.0, + "grad_norm": 1.9602571006683653, + "language_loss": 0.77997422, + "learning_rate": 1.1719408299013955e-06, + "loss": 0.80172813, + "num_input_tokens_seen": 115742995, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.16040039, + "step": 5379, + "time_per_iteration": 3.0124151706695557 + }, + { + "auxiliary_loss_clip": 0.01127417, + "auxiliary_loss_mlp": 0.01039081, + "balance_loss_clip": 1.04022503, + "balance_loss_mlp": 1.02464509, + "epoch": 0.6469067516383094, + "flos": 19573650218880.0, + "grad_norm": 2.376122597169494, + "language_loss": 0.75901878, + "learning_rate": 1.1712318233277067e-06, + "loss": 0.78068376, + "num_input_tokens_seen": 115762015, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.14440918, + "step": 5380, + "time_per_iteration": 2.98856258392334 + }, + { + "auxiliary_loss_clip": 0.01036565, + "auxiliary_loss_mlp": 0.01001116, + "balance_loss_clip": 1.00880814, + "balance_loss_mlp": 0.99966115, + "epoch": 0.6470269945289485, + "flos": 65098002522240.0, + "grad_norm": 0.760126997599945, + "language_loss": 0.57867002, + "learning_rate": 1.1705229424774916e-06, + "loss": 0.59904683, + "num_input_tokens_seen": 115816285, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01452637, + "step": 5381, + "time_per_iteration": 3.221957206726074 + }, + { + "auxiliary_loss_clip": 0.01129902, + "auxiliary_loss_mlp": 0.01039679, + "balance_loss_clip": 1.04089499, + "balance_loss_mlp": 1.02366877, + "epoch": 0.6471472374195876, + "flos": 30696943639680.0, + "grad_norm": 1.7651247604026552, + "language_loss": 0.64269799, + "learning_rate": 1.1698141874582867e-06, + "loss": 0.66439378, + "num_input_tokens_seen": 115837330, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.16003418, + "step": 5382, + "time_per_iteration": 2.8193953037261963 + }, + { + "auxiliary_loss_clip": 0.01128643, + "auxiliary_loss_mlp": 0.01041994, + "balance_loss_clip": 1.04195166, + "balance_loss_mlp": 1.02631807, + "epoch": 0.6472674803102266, + "flos": 20521835487360.0, + "grad_norm": 2.169686484826593, + "language_loss": 0.72300732, + "learning_rate": 1.169105558377609e-06, + "loss": 0.74471366, + "num_input_tokens_seen": 115857420, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15673828, + "step": 5383, + "time_per_iteration": 2.803338050842285 + }, + { + "auxiliary_loss_clip": 0.01126872, + "auxiliary_loss_mlp": 0.01042898, + "balance_loss_clip": 1.04274619, + "balance_loss_mlp": 1.02775288, + "epoch": 0.6473877232008658, + "flos": 24715447320960.0, + "grad_norm": 1.9790119917456348, + "language_loss": 0.78570777, + "learning_rate": 1.1683970553429587e-06, + "loss": 0.80740547, + "num_input_tokens_seen": 115878875, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.15142822, + "step": 5384, + "time_per_iteration": 2.7947998046875 + }, + { + "auxiliary_loss_clip": 0.01129666, + "auxiliary_loss_mlp": 0.01040191, + "balance_loss_clip": 1.04391849, + "balance_loss_mlp": 1.02505159, + "epoch": 0.6475079660915048, + "flos": 15885552441600.0, + "grad_norm": 2.424320913673857, + "language_loss": 0.82333452, + "learning_rate": 1.1676886784618128e-06, + "loss": 0.84503305, + "num_input_tokens_seen": 115895540, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15124512, + "step": 5385, + "time_per_iteration": 3.7155091762542725 + }, + { + "auxiliary_loss_clip": 0.01131995, + "auxiliary_loss_mlp": 0.01040534, + "balance_loss_clip": 1.04159749, + "balance_loss_mlp": 1.02428579, + "epoch": 0.6476282089821439, + "flos": 17381590922880.0, + "grad_norm": 2.1177197197019924, + "language_loss": 0.8397758, + "learning_rate": 1.1669804278416332e-06, + "loss": 0.8615011, + "num_input_tokens_seen": 115910265, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.16235352, + "step": 5386, + "time_per_iteration": 2.7497730255126953 + }, + { + "auxiliary_loss_clip": 0.01131239, + "auxiliary_loss_mlp": 0.01035035, + "balance_loss_clip": 1.04063725, + "balance_loss_mlp": 1.01844072, + "epoch": 0.6477484518727831, + "flos": 20194078861440.0, + "grad_norm": 1.9800337584446184, + "language_loss": 0.71506715, + "learning_rate": 1.1662723035898602e-06, + "loss": 0.73672986, + "num_input_tokens_seen": 115930025, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.16577148, + "step": 5387, + "time_per_iteration": 2.821110963821411 + }, + { + "auxiliary_loss_clip": 0.0113035, + "auxiliary_loss_mlp": 0.01036258, + "balance_loss_clip": 1.04054821, + "balance_loss_mlp": 1.01902008, + "epoch": 0.6478686947634221, + "flos": 25410426641280.0, + "grad_norm": 1.762814226117796, + "language_loss": 0.81987989, + "learning_rate": 1.165564305813915e-06, + "loss": 0.841546, + "num_input_tokens_seen": 115949025, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.17260742, + "step": 5388, + "time_per_iteration": 3.6930532455444336 + }, + { + "auxiliary_loss_clip": 0.01123836, + "auxiliary_loss_mlp": 0.01038827, + "balance_loss_clip": 1.04103065, + "balance_loss_mlp": 1.02380681, + "epoch": 0.6479889376540612, + "flos": 20083581648000.0, + "grad_norm": 1.7063230881107216, + "language_loss": 0.81251347, + "learning_rate": 1.1648564346212019e-06, + "loss": 0.83414006, + "num_input_tokens_seen": 115968145, + "router_z_loss_clip": 0.82763672, + "router_z_loss_mlp": 0.15039062, + "step": 5389, + "time_per_iteration": 2.7557790279388428 + }, + { + "auxiliary_loss_clip": 0.01126131, + "auxiliary_loss_mlp": 0.01037904, + "balance_loss_clip": 1.04098129, + "balance_loss_mlp": 1.02296662, + "epoch": 0.6481091805447003, + "flos": 26758082039040.0, + "grad_norm": 1.6989796941980042, + "language_loss": 0.75972354, + "learning_rate": 1.164148690119104e-06, + "loss": 0.78136396, + "num_input_tokens_seen": 115989425, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.14935303, + "step": 5390, + "time_per_iteration": 3.709031343460083 + }, + { + "auxiliary_loss_clip": 0.01128057, + "auxiliary_loss_mlp": 0.01039023, + "balance_loss_clip": 1.04254019, + "balance_loss_mlp": 1.02427638, + "epoch": 0.6482294234353394, + "flos": 23952094462080.0, + "grad_norm": 1.741177527251241, + "language_loss": 0.74138516, + "learning_rate": 1.163441072414985e-06, + "loss": 0.76305598, + "num_input_tokens_seen": 116009630, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.14746094, + "step": 5391, + "time_per_iteration": 2.972323417663574 + }, + { + "auxiliary_loss_clip": 0.01128019, + "auxiliary_loss_mlp": 0.01031504, + "balance_loss_clip": 1.04329598, + "balance_loss_mlp": 1.01583946, + "epoch": 0.6483496663259785, + "flos": 26209833776640.0, + "grad_norm": 1.849154964829531, + "language_loss": 0.69788229, + "learning_rate": 1.16273358161619e-06, + "loss": 0.71947759, + "num_input_tokens_seen": 116029965, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.15673828, + "step": 5392, + "time_per_iteration": 2.968703508377075 + }, + { + "auxiliary_loss_clip": 0.01135532, + "auxiliary_loss_mlp": 0.01040289, + "balance_loss_clip": 1.04394937, + "balance_loss_mlp": 1.02450585, + "epoch": 0.6484699092166175, + "flos": 20922239370240.0, + "grad_norm": 2.04169175216554, + "language_loss": 0.83539498, + "learning_rate": 1.1620262178300446e-06, + "loss": 0.85715324, + "num_input_tokens_seen": 116048580, + "router_z_loss_clip": 0.91650391, + "router_z_loss_mlp": 0.15795898, + "step": 5393, + "time_per_iteration": 2.9982035160064697 + }, + { + "auxiliary_loss_clip": 0.01130302, + "auxiliary_loss_mlp": 0.01033281, + "balance_loss_clip": 1.04357934, + "balance_loss_mlp": 1.0179863, + "epoch": 0.6485901521072567, + "flos": 33072865678080.0, + "grad_norm": 1.8333468145924825, + "language_loss": 0.75664747, + "learning_rate": 1.1613189811638563e-06, + "loss": 0.7782833, + "num_input_tokens_seen": 116070305, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15283203, + "step": 5394, + "time_per_iteration": 3.9250648021698 + }, + { + "auxiliary_loss_clip": 0.01130124, + "auxiliary_loss_mlp": 0.01040464, + "balance_loss_clip": 1.04408431, + "balance_loss_mlp": 1.02546716, + "epoch": 0.6487103949978957, + "flos": 22274060745600.0, + "grad_norm": 1.5942547470970232, + "language_loss": 0.78056216, + "learning_rate": 1.1606118717249117e-06, + "loss": 0.80226803, + "num_input_tokens_seen": 116090405, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.14990234, + "step": 5395, + "time_per_iteration": 2.928858518600464 + }, + { + "auxiliary_loss_clip": 0.01132049, + "auxiliary_loss_mlp": 0.01043907, + "balance_loss_clip": 1.04098129, + "balance_loss_mlp": 1.02728903, + "epoch": 0.6488306378885348, + "flos": 22930400010240.0, + "grad_norm": 1.6236563219962, + "language_loss": 0.67900705, + "learning_rate": 1.1599048896204787e-06, + "loss": 0.70076656, + "num_input_tokens_seen": 116110285, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.1661377, + "step": 5396, + "time_per_iteration": 2.9055235385894775 + }, + { + "auxiliary_loss_clip": 0.01128216, + "auxiliary_loss_mlp": 0.01047187, + "balance_loss_clip": 1.03971243, + "balance_loss_mlp": 1.03139174, + "epoch": 0.648950880779174, + "flos": 20376110010240.0, + "grad_norm": 1.8204161105888945, + "language_loss": 0.80942667, + "learning_rate": 1.1591980349578061e-06, + "loss": 0.83118069, + "num_input_tokens_seen": 116128955, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.15771484, + "step": 5397, + "time_per_iteration": 2.856904983520508 + }, + { + "auxiliary_loss_clip": 0.01037092, + "auxiliary_loss_mlp": 0.01003334, + "balance_loss_clip": 1.00905156, + "balance_loss_mlp": 1.00172484, + "epoch": 0.649071123669813, + "flos": 59930889310080.0, + "grad_norm": 0.7352976131432156, + "language_loss": 0.54285729, + "learning_rate": 1.158491307844123e-06, + "loss": 0.56326163, + "num_input_tokens_seen": 116188875, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01611328, + "step": 5398, + "time_per_iteration": 3.3036835193634033 + }, + { + "auxiliary_loss_clip": 0.0112825, + "auxiliary_loss_mlp": 0.01036022, + "balance_loss_clip": 1.04265726, + "balance_loss_mlp": 1.01983309, + "epoch": 0.6491913665604521, + "flos": 20446566537600.0, + "grad_norm": 1.8515646972627402, + "language_loss": 0.8399927, + "learning_rate": 1.1577847083866387e-06, + "loss": 0.86163545, + "num_input_tokens_seen": 116207910, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.16174316, + "step": 5399, + "time_per_iteration": 2.8799989223480225 + }, + { + "auxiliary_loss_clip": 0.01131936, + "auxiliary_loss_mlp": 0.01034614, + "balance_loss_clip": 1.04199338, + "balance_loss_mlp": 1.01781726, + "epoch": 0.6493116094510912, + "flos": 16946820702720.0, + "grad_norm": 1.6949703566263843, + "language_loss": 0.71974945, + "learning_rate": 1.1570782366925453e-06, + "loss": 0.7414149, + "num_input_tokens_seen": 116226425, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.16796875, + "step": 5400, + "time_per_iteration": 2.813997268676758 + }, + { + "auxiliary_loss_clip": 0.01129479, + "auxiliary_loss_mlp": 0.01036263, + "balance_loss_clip": 1.03901815, + "balance_loss_mlp": 1.02068186, + "epoch": 0.6494318523417303, + "flos": 18802935072000.0, + "grad_norm": 2.3423972557220667, + "language_loss": 0.75867581, + "learning_rate": 1.1563718928690132e-06, + "loss": 0.78033328, + "num_input_tokens_seen": 116243860, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.15588379, + "step": 5401, + "time_per_iteration": 2.8719289302825928 + }, + { + "auxiliary_loss_clip": 0.01130871, + "auxiliary_loss_mlp": 0.01047585, + "balance_loss_clip": 1.04189813, + "balance_loss_mlp": 1.03124142, + "epoch": 0.6495520952323693, + "flos": 18982847318400.0, + "grad_norm": 1.8966278617307073, + "language_loss": 0.7150023, + "learning_rate": 1.1556656770231942e-06, + "loss": 0.73678684, + "num_input_tokens_seen": 116260055, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.16333008, + "step": 5402, + "time_per_iteration": 2.965179920196533 + }, + { + "auxiliary_loss_clip": 0.01125641, + "auxiliary_loss_mlp": 0.01041138, + "balance_loss_clip": 1.04035759, + "balance_loss_mlp": 1.02605796, + "epoch": 0.6496723381230085, + "flos": 22745388032640.0, + "grad_norm": 2.184682911723615, + "language_loss": 0.76261133, + "learning_rate": 1.1549595892622207e-06, + "loss": 0.78427911, + "num_input_tokens_seen": 116278825, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.1506958, + "step": 5403, + "time_per_iteration": 2.9334561824798584 + }, + { + "auxiliary_loss_clip": 0.01036804, + "auxiliary_loss_mlp": 0.00999177, + "balance_loss_clip": 1.00884962, + "balance_loss_mlp": 0.99751949, + "epoch": 0.6497925810136476, + "flos": 62145283887360.0, + "grad_norm": 0.8187295226044442, + "language_loss": 0.58975637, + "learning_rate": 1.1542536296932047e-06, + "loss": 0.61011624, + "num_input_tokens_seen": 116342360, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01660156, + "step": 5404, + "time_per_iteration": 3.3512887954711914 + }, + { + "auxiliary_loss_clip": 0.01134559, + "auxiliary_loss_mlp": 0.01041408, + "balance_loss_clip": 1.04345334, + "balance_loss_mlp": 1.02483845, + "epoch": 0.6499128239042866, + "flos": 20156731695360.0, + "grad_norm": 1.60263623049662, + "language_loss": 0.70167857, + "learning_rate": 1.1535477984232414e-06, + "loss": 0.72343826, + "num_input_tokens_seen": 116362235, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.16577148, + "step": 5405, + "time_per_iteration": 2.9209868907928467 + }, + { + "auxiliary_loss_clip": 0.01125779, + "auxiliary_loss_mlp": 0.01038514, + "balance_loss_clip": 1.03953242, + "balance_loss_mlp": 1.02225411, + "epoch": 0.6500330667949258, + "flos": 24462420940800.0, + "grad_norm": 1.652903867693156, + "language_loss": 0.76748812, + "learning_rate": 1.152842095559404e-06, + "loss": 0.78913105, + "num_input_tokens_seen": 116382895, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.16271973, + "step": 5406, + "time_per_iteration": 2.9675512313842773 + }, + { + "auxiliary_loss_clip": 0.01126246, + "auxiliary_loss_mlp": 0.01036506, + "balance_loss_clip": 1.04142773, + "balance_loss_mlp": 1.02053225, + "epoch": 0.6501533096855648, + "flos": 25477399549440.0, + "grad_norm": 1.8074933811561944, + "language_loss": 0.77226847, + "learning_rate": 1.1521365212087474e-06, + "loss": 0.79389602, + "num_input_tokens_seen": 116402880, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15966797, + "step": 5407, + "time_per_iteration": 2.9039082527160645 + }, + { + "auxiliary_loss_clip": 0.0113328, + "auxiliary_loss_mlp": 0.01041949, + "balance_loss_clip": 1.0428015, + "balance_loss_mlp": 1.02556956, + "epoch": 0.6502735525762039, + "flos": 44819245347840.0, + "grad_norm": 1.714852395369914, + "language_loss": 0.70735896, + "learning_rate": 1.1514310754783062e-06, + "loss": 0.72911125, + "num_input_tokens_seen": 116425830, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.16369629, + "step": 5408, + "time_per_iteration": 3.0338940620422363 + }, + { + "auxiliary_loss_clip": 0.0112921, + "auxiliary_loss_mlp": 0.01042854, + "balance_loss_clip": 1.04341936, + "balance_loss_mlp": 1.02560425, + "epoch": 0.6503937954668431, + "flos": 28658546726400.0, + "grad_norm": 2.527460133138139, + "language_loss": 0.73424184, + "learning_rate": 1.1507257584750964e-06, + "loss": 0.75596249, + "num_input_tokens_seen": 116446010, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.17260742, + "step": 5409, + "time_per_iteration": 2.90696120262146 + }, + { + "auxiliary_loss_clip": 0.01130456, + "auxiliary_loss_mlp": 0.01047485, + "balance_loss_clip": 1.04286289, + "balance_loss_mlp": 1.03150535, + "epoch": 0.6505140383574821, + "flos": 20922562592640.0, + "grad_norm": 1.9562467902696856, + "language_loss": 0.77681184, + "learning_rate": 1.150020570306113e-06, + "loss": 0.79859126, + "num_input_tokens_seen": 116465150, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.159729, + "step": 5410, + "time_per_iteration": 2.9065358638763428 + }, + { + "auxiliary_loss_clip": 0.01133383, + "auxiliary_loss_mlp": 0.01041291, + "balance_loss_clip": 1.0421145, + "balance_loss_mlp": 1.02458334, + "epoch": 0.6506342812481212, + "flos": 20595236929920.0, + "grad_norm": 1.7094007508845168, + "language_loss": 0.74611074, + "learning_rate": 1.1493155110783338e-06, + "loss": 0.76785749, + "num_input_tokens_seen": 116483675, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.16699219, + "step": 5411, + "time_per_iteration": 2.912835121154785 + }, + { + "auxiliary_loss_clip": 0.01132238, + "auxiliary_loss_mlp": 0.01033023, + "balance_loss_clip": 1.04235029, + "balance_loss_mlp": 1.01703739, + "epoch": 0.6507545241387603, + "flos": 30226478279040.0, + "grad_norm": 1.8829925322434675, + "language_loss": 0.70487607, + "learning_rate": 1.1486105808987155e-06, + "loss": 0.7265287, + "num_input_tokens_seen": 116505165, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.15966797, + "step": 5412, + "time_per_iteration": 2.8883655071258545 + }, + { + "auxiliary_loss_clip": 0.01132387, + "auxiliary_loss_mlp": 0.01039212, + "balance_loss_clip": 1.0421586, + "balance_loss_mlp": 1.02238011, + "epoch": 0.6508747670293994, + "flos": 17128241320320.0, + "grad_norm": 1.8644851988357036, + "language_loss": 0.81155741, + "learning_rate": 1.1479057798741947e-06, + "loss": 0.83327341, + "num_input_tokens_seen": 116523220, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.168396, + "step": 5413, + "time_per_iteration": 3.6514456272125244 + }, + { + "auxiliary_loss_clip": 0.01036011, + "auxiliary_loss_mlp": 0.01001382, + "balance_loss_clip": 1.00803781, + "balance_loss_mlp": 0.99986821, + "epoch": 0.6509950099200384, + "flos": 68559826573440.0, + "grad_norm": 0.7851543691058347, + "language_loss": 0.53345209, + "learning_rate": 1.14720110811169e-06, + "loss": 0.55382603, + "num_input_tokens_seen": 116580450, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01513672, + "step": 5414, + "time_per_iteration": 3.40820050239563 + }, + { + "auxiliary_loss_clip": 0.01133316, + "auxiliary_loss_mlp": 0.01047338, + "balance_loss_clip": 1.04270911, + "balance_loss_mlp": 1.03032708, + "epoch": 0.6511152528106776, + "flos": 22347462188160.0, + "grad_norm": 1.7844946434417275, + "language_loss": 0.76794553, + "learning_rate": 1.146496565718098e-06, + "loss": 0.78975207, + "num_input_tokens_seen": 116601020, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.17016602, + "step": 5415, + "time_per_iteration": 3.2070813179016113 + }, + { + "auxiliary_loss_clip": 0.01131322, + "auxiliary_loss_mlp": 0.01035718, + "balance_loss_clip": 1.04100466, + "balance_loss_mlp": 1.01909971, + "epoch": 0.6512354957013167, + "flos": 20522158709760.0, + "grad_norm": 2.906564504724292, + "language_loss": 0.75570947, + "learning_rate": 1.1457921528002996e-06, + "loss": 0.77737987, + "num_input_tokens_seen": 116619455, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.1661377, + "step": 5416, + "time_per_iteration": 2.8760921955108643 + }, + { + "auxiliary_loss_clip": 0.01131034, + "auxiliary_loss_mlp": 0.01042751, + "balance_loss_clip": 1.04266512, + "balance_loss_mlp": 1.02639604, + "epoch": 0.6513557385919557, + "flos": 32337342881280.0, + "grad_norm": 2.2957402403072336, + "language_loss": 0.72125852, + "learning_rate": 1.1450878694651522e-06, + "loss": 0.74299634, + "num_input_tokens_seen": 116640020, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.16345215, + "step": 5417, + "time_per_iteration": 3.6205997467041016 + }, + { + "auxiliary_loss_clip": 0.01129366, + "auxiliary_loss_mlp": 0.01033144, + "balance_loss_clip": 1.0403223, + "balance_loss_mlp": 1.01740861, + "epoch": 0.6514759814825949, + "flos": 12093206417280.0, + "grad_norm": 2.368490878118356, + "language_loss": 0.63334048, + "learning_rate": 1.1443837158194954e-06, + "loss": 0.65496564, + "num_input_tokens_seen": 116655165, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.15734863, + "step": 5418, + "time_per_iteration": 2.7611711025238037 + }, + { + "auxiliary_loss_clip": 0.0113195, + "auxiliary_loss_mlp": 0.01039008, + "balance_loss_clip": 1.04243159, + "balance_loss_mlp": 1.02260447, + "epoch": 0.651596224373234, + "flos": 22526907557760.0, + "grad_norm": 1.5254292051625624, + "language_loss": 0.74301547, + "learning_rate": 1.1436796919701484e-06, + "loss": 0.76472509, + "num_input_tokens_seen": 116673880, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.16430664, + "step": 5419, + "time_per_iteration": 3.626802682876587 + }, + { + "auxiliary_loss_clip": 0.01130371, + "auxiliary_loss_mlp": 0.01038222, + "balance_loss_clip": 1.04104614, + "balance_loss_mlp": 1.02140117, + "epoch": 0.651716467263873, + "flos": 27818955250560.0, + "grad_norm": 2.070201541787434, + "language_loss": 0.61978328, + "learning_rate": 1.1429757980239115e-06, + "loss": 0.64146924, + "num_input_tokens_seen": 116694305, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.16809082, + "step": 5420, + "time_per_iteration": 2.7640206813812256 + }, + { + "auxiliary_loss_clip": 0.01134666, + "auxiliary_loss_mlp": 0.01048685, + "balance_loss_clip": 1.04100418, + "balance_loss_mlp": 1.03071988, + "epoch": 0.6518367101545122, + "flos": 24316300414080.0, + "grad_norm": 2.20886466360558, + "language_loss": 0.82023185, + "learning_rate": 1.1422720340875636e-06, + "loss": 0.84206533, + "num_input_tokens_seen": 116713055, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.17944336, + "step": 5421, + "time_per_iteration": 2.8110859394073486 + }, + { + "auxiliary_loss_clip": 0.01133688, + "auxiliary_loss_mlp": 0.01040596, + "balance_loss_clip": 1.04226196, + "balance_loss_mlp": 1.02414465, + "epoch": 0.6519569530451512, + "flos": 20011939971840.0, + "grad_norm": 2.186691515609219, + "language_loss": 0.79254586, + "learning_rate": 1.1415684002678671e-06, + "loss": 0.81428874, + "num_input_tokens_seen": 116731815, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.16442871, + "step": 5422, + "time_per_iteration": 2.725600004196167 + }, + { + "auxiliary_loss_clip": 0.01135973, + "auxiliary_loss_mlp": 0.01048078, + "balance_loss_clip": 1.04418814, + "balance_loss_mlp": 1.03056669, + "epoch": 0.6520771959357903, + "flos": 21576064682880.0, + "grad_norm": 2.4958582007188395, + "language_loss": 0.77444476, + "learning_rate": 1.1408648966715617e-06, + "loss": 0.79628527, + "num_input_tokens_seen": 116749335, + "router_z_loss_clip": 0.91845703, + "router_z_loss_mlp": 0.17504883, + "step": 5423, + "time_per_iteration": 3.6340668201446533 + }, + { + "auxiliary_loss_clip": 0.01136081, + "auxiliary_loss_mlp": 0.01042882, + "balance_loss_clip": 1.04232335, + "balance_loss_mlp": 1.02637172, + "epoch": 0.6521974388264293, + "flos": 22711021695360.0, + "grad_norm": 1.8404443197368185, + "language_loss": 0.72202194, + "learning_rate": 1.1401615234053683e-06, + "loss": 0.74381149, + "num_input_tokens_seen": 116768155, + "router_z_loss_clip": 0.93701172, + "router_z_loss_mlp": 0.1652832, + "step": 5424, + "time_per_iteration": 2.8197786808013916 + }, + { + "auxiliary_loss_clip": 0.01130978, + "auxiliary_loss_mlp": 0.01044673, + "balance_loss_clip": 1.04054844, + "balance_loss_mlp": 1.02794778, + "epoch": 0.6523176817170685, + "flos": 23002939526400.0, + "grad_norm": 1.8189234036950974, + "language_loss": 0.75820553, + "learning_rate": 1.1394582805759885e-06, + "loss": 0.77996206, + "num_input_tokens_seen": 116787435, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.1673584, + "step": 5425, + "time_per_iteration": 2.786639928817749 + }, + { + "auxiliary_loss_clip": 0.01133253, + "auxiliary_loss_mlp": 0.01041726, + "balance_loss_clip": 1.04283381, + "balance_loss_mlp": 1.02578735, + "epoch": 0.6524379246077076, + "flos": 21688249835520.0, + "grad_norm": 1.839446932457074, + "language_loss": 0.75802249, + "learning_rate": 1.1387551682901022e-06, + "loss": 0.77977228, + "num_input_tokens_seen": 116808040, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.15942383, + "step": 5426, + "time_per_iteration": 2.8172245025634766 + }, + { + "auxiliary_loss_clip": 0.01131595, + "auxiliary_loss_mlp": 0.01040506, + "balance_loss_clip": 1.04225445, + "balance_loss_mlp": 1.02348244, + "epoch": 0.6525581674983466, + "flos": 19390936711680.0, + "grad_norm": 2.6491502998627183, + "language_loss": 0.70449603, + "learning_rate": 1.138052186654373e-06, + "loss": 0.72621709, + "num_input_tokens_seen": 116825510, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.17016602, + "step": 5427, + "time_per_iteration": 2.7269375324249268 + }, + { + "auxiliary_loss_clip": 0.0113633, + "auxiliary_loss_mlp": 0.01044513, + "balance_loss_clip": 1.04416144, + "balance_loss_mlp": 1.02678633, + "epoch": 0.6526784103889858, + "flos": 17165444832000.0, + "grad_norm": 2.041218849779682, + "language_loss": 0.88217378, + "learning_rate": 1.1373493357754417e-06, + "loss": 0.90398222, + "num_input_tokens_seen": 116844415, + "router_z_loss_clip": 0.92138672, + "router_z_loss_mlp": 0.17724609, + "step": 5428, + "time_per_iteration": 2.7583930492401123 + }, + { + "auxiliary_loss_clip": 0.01128678, + "auxiliary_loss_mlp": 0.01039922, + "balance_loss_clip": 1.04139388, + "balance_loss_mlp": 1.02419233, + "epoch": 0.6527986532796248, + "flos": 18989168112000.0, + "grad_norm": 2.3519773687921703, + "language_loss": 0.77677298, + "learning_rate": 1.1366466157599303e-06, + "loss": 0.79845893, + "num_input_tokens_seen": 116863690, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15740967, + "step": 5429, + "time_per_iteration": 2.8132236003875732 + }, + { + "auxiliary_loss_clip": 0.01133787, + "auxiliary_loss_mlp": 0.01038637, + "balance_loss_clip": 1.04299152, + "balance_loss_mlp": 1.02197123, + "epoch": 0.6529188961702639, + "flos": 14238581011200.0, + "grad_norm": 2.514288158230323, + "language_loss": 0.76149893, + "learning_rate": 1.1359440267144412e-06, + "loss": 0.78322315, + "num_input_tokens_seen": 116881145, + "router_z_loss_clip": 0.90771484, + "router_z_loss_mlp": 0.16662598, + "step": 5430, + "time_per_iteration": 2.819675922393799 + }, + { + "auxiliary_loss_clip": 0.01130233, + "auxiliary_loss_mlp": 0.01042298, + "balance_loss_clip": 1.04243207, + "balance_loss_mlp": 1.02590632, + "epoch": 0.653039139060903, + "flos": 36682929158400.0, + "grad_norm": 2.0058954003553686, + "language_loss": 0.74678481, + "learning_rate": 1.1352415687455556e-06, + "loss": 0.7685101, + "num_input_tokens_seen": 116902405, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.16381836, + "step": 5431, + "time_per_iteration": 3.024214506149292 + }, + { + "auxiliary_loss_clip": 0.01131995, + "auxiliary_loss_mlp": 0.01040418, + "balance_loss_clip": 1.04345202, + "balance_loss_mlp": 1.02316892, + "epoch": 0.6531593819515421, + "flos": 25376275785600.0, + "grad_norm": 2.156971554794135, + "language_loss": 0.64020777, + "learning_rate": 1.1345392419598362e-06, + "loss": 0.66193187, + "num_input_tokens_seen": 116921285, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.17248535, + "step": 5432, + "time_per_iteration": 2.8463642597198486 + }, + { + "auxiliary_loss_clip": 0.01129299, + "auxiliary_loss_mlp": 0.01039316, + "balance_loss_clip": 1.04103518, + "balance_loss_mlp": 1.02313972, + "epoch": 0.6532796248421812, + "flos": 21178533888000.0, + "grad_norm": 1.6305748693217617, + "language_loss": 0.71867347, + "learning_rate": 1.1338370464638263e-06, + "loss": 0.74035966, + "num_input_tokens_seen": 116940685, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.16186523, + "step": 5433, + "time_per_iteration": 2.8692679405212402 + }, + { + "auxiliary_loss_clip": 0.01128205, + "auxiliary_loss_mlp": 0.01040055, + "balance_loss_clip": 1.04025054, + "balance_loss_mlp": 1.0231626, + "epoch": 0.6533998677328203, + "flos": 17675950878720.0, + "grad_norm": 2.1280999322991216, + "language_loss": 0.64114732, + "learning_rate": 1.1331349823640474e-06, + "loss": 0.66282988, + "num_input_tokens_seen": 116958115, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.16870117, + "step": 5434, + "time_per_iteration": 2.8269028663635254 + }, + { + "auxiliary_loss_clip": 0.01129771, + "auxiliary_loss_mlp": 0.010367, + "balance_loss_clip": 1.04280758, + "balance_loss_mlp": 1.02107167, + "epoch": 0.6535201106234594, + "flos": 28400384701440.0, + "grad_norm": 8.383060526822726, + "language_loss": 0.78188926, + "learning_rate": 1.132433049767003e-06, + "loss": 0.80355394, + "num_input_tokens_seen": 116976030, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15625, + "step": 5435, + "time_per_iteration": 2.8883981704711914 + }, + { + "auxiliary_loss_clip": 0.01127168, + "auxiliary_loss_mlp": 0.01043118, + "balance_loss_clip": 1.04101181, + "balance_loss_mlp": 1.02684569, + "epoch": 0.6536403535140984, + "flos": 23586667447680.0, + "grad_norm": 1.7711436399897946, + "language_loss": 0.81227916, + "learning_rate": 1.1317312487791748e-06, + "loss": 0.83398205, + "num_input_tokens_seen": 116997680, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.16259766, + "step": 5436, + "time_per_iteration": 3.0030517578125 + }, + { + "auxiliary_loss_clip": 0.01128641, + "auxiliary_loss_mlp": 0.01033106, + "balance_loss_clip": 1.04070401, + "balance_loss_mlp": 1.01763248, + "epoch": 0.6537605964047376, + "flos": 21579476474880.0, + "grad_norm": 2.173873099673333, + "language_loss": 0.73140275, + "learning_rate": 1.1310295795070253e-06, + "loss": 0.75302023, + "num_input_tokens_seen": 117017620, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.15478516, + "step": 5437, + "time_per_iteration": 2.921504020690918 + }, + { + "auxiliary_loss_clip": 0.01129257, + "auxiliary_loss_mlp": 0.01040236, + "balance_loss_clip": 1.04340458, + "balance_loss_mlp": 1.02420211, + "epoch": 0.6538808392953767, + "flos": 26833997433600.0, + "grad_norm": 1.6238383419746614, + "language_loss": 0.81040436, + "learning_rate": 1.1303280420569982e-06, + "loss": 0.83209932, + "num_input_tokens_seen": 117039505, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.16027832, + "step": 5438, + "time_per_iteration": 3.0677993297576904 + }, + { + "auxiliary_loss_clip": 0.01130162, + "auxiliary_loss_mlp": 0.01043695, + "balance_loss_clip": 1.04200196, + "balance_loss_mlp": 1.02751219, + "epoch": 0.6540010821860157, + "flos": 30738241301760.0, + "grad_norm": 1.8479773425756978, + "language_loss": 0.77382767, + "learning_rate": 1.1296266365355158e-06, + "loss": 0.79556632, + "num_input_tokens_seen": 117062890, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.16186523, + "step": 5439, + "time_per_iteration": 2.98883318901062 + }, + { + "auxiliary_loss_clip": 0.01134176, + "auxiliary_loss_mlp": 0.01043401, + "balance_loss_clip": 1.04284906, + "balance_loss_mlp": 1.0264138, + "epoch": 0.6541213250766549, + "flos": 26907147480960.0, + "grad_norm": 2.0866507076630323, + "language_loss": 0.73859799, + "learning_rate": 1.1289253630489806e-06, + "loss": 0.76037377, + "num_input_tokens_seen": 117083940, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.16992188, + "step": 5440, + "time_per_iteration": 2.8367981910705566 + }, + { + "auxiliary_loss_clip": 0.01136207, + "auxiliary_loss_mlp": 0.01044414, + "balance_loss_clip": 1.04227662, + "balance_loss_mlp": 1.02685404, + "epoch": 0.6542415679672939, + "flos": 19172384409600.0, + "grad_norm": 2.4816914376273487, + "language_loss": 0.71987784, + "learning_rate": 1.1282242217037753e-06, + "loss": 0.74168408, + "num_input_tokens_seen": 117101440, + "router_z_loss_clip": 0.93945312, + "router_z_loss_mlp": 0.17553711, + "step": 5441, + "time_per_iteration": 2.8100616931915283 + }, + { + "auxiliary_loss_clip": 0.01137087, + "auxiliary_loss_mlp": 0.01036627, + "balance_loss_clip": 1.04424691, + "balance_loss_mlp": 1.01937723, + "epoch": 0.654361810857933, + "flos": 48173517100800.0, + "grad_norm": 1.9499792213205043, + "language_loss": 0.6207298, + "learning_rate": 1.127523212606262e-06, + "loss": 0.64246696, + "num_input_tokens_seen": 117124265, + "router_z_loss_clip": 0.92773438, + "router_z_loss_mlp": 0.17260742, + "step": 5442, + "time_per_iteration": 3.7909562587738037 + }, + { + "auxiliary_loss_clip": 0.01127896, + "auxiliary_loss_mlp": 0.0103648, + "balance_loss_clip": 1.04154134, + "balance_loss_mlp": 1.02167392, + "epoch": 0.6544820537485722, + "flos": 26943165843840.0, + "grad_norm": 1.8178954179025335, + "language_loss": 0.73184645, + "learning_rate": 1.1268223358627835e-06, + "loss": 0.75349015, + "num_input_tokens_seen": 117146755, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.14788818, + "step": 5443, + "time_per_iteration": 2.873321056365967 + }, + { + "auxiliary_loss_clip": 0.01130269, + "auxiliary_loss_mlp": 0.0104612, + "balance_loss_clip": 1.04135787, + "balance_loss_mlp": 1.03039622, + "epoch": 0.6546022966392112, + "flos": 20886328748160.0, + "grad_norm": 1.741963035901988, + "language_loss": 0.71989888, + "learning_rate": 1.126121591579663e-06, + "loss": 0.74166274, + "num_input_tokens_seen": 117165960, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.15734863, + "step": 5444, + "time_per_iteration": 2.781933069229126 + }, + { + "auxiliary_loss_clip": 0.01125428, + "auxiliary_loss_mlp": 0.01043071, + "balance_loss_clip": 1.04172182, + "balance_loss_mlp": 1.02706099, + "epoch": 0.6547225395298503, + "flos": 24936693143040.0, + "grad_norm": 1.506562465414639, + "language_loss": 0.69127899, + "learning_rate": 1.1254209798632018e-06, + "loss": 0.71296394, + "num_input_tokens_seen": 117186980, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.16015625, + "step": 5445, + "time_per_iteration": 2.840428352355957 + }, + { + "auxiliary_loss_clip": 0.01126665, + "auxiliary_loss_mlp": 0.01042784, + "balance_loss_clip": 1.04095888, + "balance_loss_mlp": 1.02636874, + "epoch": 0.6548427824204894, + "flos": 22565942663040.0, + "grad_norm": 1.7710052456864742, + "language_loss": 0.84736657, + "learning_rate": 1.124720500819683e-06, + "loss": 0.86906105, + "num_input_tokens_seen": 117205135, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.16418457, + "step": 5446, + "time_per_iteration": 3.693460702896118 + }, + { + "auxiliary_loss_clip": 0.01130904, + "auxiliary_loss_mlp": 0.01038672, + "balance_loss_clip": 1.04298925, + "balance_loss_mlp": 1.02228069, + "epoch": 0.6549630253111285, + "flos": 18442500048000.0, + "grad_norm": 1.912285110423039, + "language_loss": 0.82225376, + "learning_rate": 1.1240201545553682e-06, + "loss": 0.84394956, + "num_input_tokens_seen": 117222935, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.16394043, + "step": 5447, + "time_per_iteration": 2.747741222381592 + }, + { + "auxiliary_loss_clip": 0.01132455, + "auxiliary_loss_mlp": 0.01043449, + "balance_loss_clip": 1.04276037, + "balance_loss_mlp": 1.02683115, + "epoch": 0.6550832682017675, + "flos": 25187313312000.0, + "grad_norm": 1.7532426152185137, + "language_loss": 0.73185158, + "learning_rate": 1.1233199411764987e-06, + "loss": 0.75361067, + "num_input_tokens_seen": 117242370, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.16625977, + "step": 5448, + "time_per_iteration": 3.644970417022705 + }, + { + "auxiliary_loss_clip": 0.01127669, + "auxiliary_loss_mlp": 0.01037555, + "balance_loss_clip": 1.04044867, + "balance_loss_mlp": 1.02174807, + "epoch": 0.6552035110924067, + "flos": 22748153379840.0, + "grad_norm": 3.0818983190326152, + "language_loss": 0.68864518, + "learning_rate": 1.1226198607892978e-06, + "loss": 0.71029747, + "num_input_tokens_seen": 117262930, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.15795898, + "step": 5449, + "time_per_iteration": 2.9893851280212402 + }, + { + "auxiliary_loss_clip": 0.01130724, + "auxiliary_loss_mlp": 0.01037768, + "balance_loss_clip": 1.04109716, + "balance_loss_mlp": 1.02143645, + "epoch": 0.6553237539830458, + "flos": 21799178012160.0, + "grad_norm": 1.925844231657784, + "language_loss": 0.79648519, + "learning_rate": 1.1219199134999664e-06, + "loss": 0.81817007, + "num_input_tokens_seen": 117281430, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.16345215, + "step": 5450, + "time_per_iteration": 2.9744012355804443 + }, + { + "auxiliary_loss_clip": 0.01131336, + "auxiliary_loss_mlp": 0.01042966, + "balance_loss_clip": 1.04157531, + "balance_loss_mlp": 1.02317691, + "epoch": 0.6554439968736848, + "flos": 20887226588160.0, + "grad_norm": 2.2330529591652275, + "language_loss": 0.79188311, + "learning_rate": 1.1212200994146863e-06, + "loss": 0.81362611, + "num_input_tokens_seen": 117299185, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.19775391, + "step": 5451, + "time_per_iteration": 3.0415897369384766 + }, + { + "auxiliary_loss_clip": 0.01126519, + "auxiliary_loss_mlp": 0.0103722, + "balance_loss_clip": 1.03926182, + "balance_loss_mlp": 1.02142477, + "epoch": 0.655564239764324, + "flos": 16139045698560.0, + "grad_norm": 2.6975166298656, + "language_loss": 0.75580215, + "learning_rate": 1.120520418639618e-06, + "loss": 0.77743953, + "num_input_tokens_seen": 117317720, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15795898, + "step": 5452, + "time_per_iteration": 3.8713464736938477 + }, + { + "auxiliary_loss_clip": 0.01129701, + "auxiliary_loss_mlp": 0.01042218, + "balance_loss_clip": 1.04197836, + "balance_loss_mlp": 1.02699542, + "epoch": 0.655684482654963, + "flos": 29570354496000.0, + "grad_norm": 1.8641697158973014, + "language_loss": 0.83462954, + "learning_rate": 1.119820871280903e-06, + "loss": 0.85634875, + "num_input_tokens_seen": 117338795, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.15234375, + "step": 5453, + "time_per_iteration": 2.898715019226074 + }, + { + "auxiliary_loss_clip": 0.01128286, + "auxiliary_loss_mlp": 0.01040286, + "balance_loss_clip": 1.04160023, + "balance_loss_mlp": 1.02445483, + "epoch": 0.6558047255456021, + "flos": 29789409588480.0, + "grad_norm": 1.811032123230717, + "language_loss": 0.73505914, + "learning_rate": 1.1191214574446614e-06, + "loss": 0.7567448, + "num_input_tokens_seen": 117359040, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.15820312, + "step": 5454, + "time_per_iteration": 2.951930046081543 + }, + { + "auxiliary_loss_clip": 0.01130492, + "auxiliary_loss_mlp": 0.01037771, + "balance_loss_clip": 1.04080892, + "balance_loss_mlp": 1.02071238, + "epoch": 0.6559249684362413, + "flos": 29059166090880.0, + "grad_norm": 1.5253184359328311, + "language_loss": 0.80001169, + "learning_rate": 1.118422177236995e-06, + "loss": 0.82169437, + "num_input_tokens_seen": 117380865, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.17053223, + "step": 5455, + "time_per_iteration": 2.8746848106384277 + }, + { + "auxiliary_loss_clip": 0.01128322, + "auxiliary_loss_mlp": 0.01042528, + "balance_loss_clip": 1.04104531, + "balance_loss_mlp": 1.02509952, + "epoch": 0.6560452113268803, + "flos": 20225464369920.0, + "grad_norm": 2.0751057775258017, + "language_loss": 0.85795295, + "learning_rate": 1.1177230307639835e-06, + "loss": 0.87966144, + "num_input_tokens_seen": 117398405, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.17419434, + "step": 5456, + "time_per_iteration": 2.911611557006836 + }, + { + "auxiliary_loss_clip": 0.01132361, + "auxiliary_loss_mlp": 0.01046437, + "balance_loss_clip": 1.04320741, + "balance_loss_mlp": 1.02950907, + "epoch": 0.6561654542175194, + "flos": 25045538330880.0, + "grad_norm": 1.6498289211150552, + "language_loss": 0.79076958, + "learning_rate": 1.1170240181316865e-06, + "loss": 0.81255758, + "num_input_tokens_seen": 117419850, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.16912842, + "step": 5457, + "time_per_iteration": 2.8530468940734863 + }, + { + "auxiliary_loss_clip": 0.01130147, + "auxiliary_loss_mlp": 0.01044169, + "balance_loss_clip": 1.03999162, + "balance_loss_mlp": 1.02749133, + "epoch": 0.6562856971081584, + "flos": 22856711258880.0, + "grad_norm": 2.0774577566898826, + "language_loss": 0.79183471, + "learning_rate": 1.1163251394461442e-06, + "loss": 0.81357777, + "num_input_tokens_seen": 117438330, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.16674805, + "step": 5458, + "time_per_iteration": 2.8398239612579346 + }, + { + "auxiliary_loss_clip": 0.01131436, + "auxiliary_loss_mlp": 0.01040895, + "balance_loss_clip": 1.0428009, + "balance_loss_mlp": 1.02474213, + "epoch": 0.6564059399987976, + "flos": 18872565586560.0, + "grad_norm": 2.143151732019039, + "language_loss": 0.8238796, + "learning_rate": 1.1156263948133746e-06, + "loss": 0.84560287, + "num_input_tokens_seen": 117454985, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.16137695, + "step": 5459, + "time_per_iteration": 2.848119020462036 + }, + { + "auxiliary_loss_clip": 0.01130722, + "auxiliary_loss_mlp": 0.01042268, + "balance_loss_clip": 1.04176784, + "balance_loss_mlp": 1.02545989, + "epoch": 0.6565261828894366, + "flos": 25484187219840.0, + "grad_norm": 1.6673435957751306, + "language_loss": 0.77342725, + "learning_rate": 1.1149277843393787e-06, + "loss": 0.79515713, + "num_input_tokens_seen": 117476145, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.16809082, + "step": 5460, + "time_per_iteration": 2.809736967086792 + }, + { + "auxiliary_loss_clip": 0.01129378, + "auxiliary_loss_mlp": 0.01040161, + "balance_loss_clip": 1.04171979, + "balance_loss_mlp": 1.02399659, + "epoch": 0.6566464257800757, + "flos": 19683500987520.0, + "grad_norm": 2.4019189378262156, + "language_loss": 0.6333698, + "learning_rate": 1.1142293081301342e-06, + "loss": 0.65506518, + "num_input_tokens_seen": 117494025, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16174316, + "step": 5461, + "time_per_iteration": 2.817500352859497 + }, + { + "auxiliary_loss_clip": 0.01125603, + "auxiliary_loss_mlp": 0.01036621, + "balance_loss_clip": 1.03991139, + "balance_loss_mlp": 1.02232742, + "epoch": 0.6567666686707149, + "flos": 23514127931520.0, + "grad_norm": 1.7788979528861337, + "language_loss": 0.6800828, + "learning_rate": 1.1135309662915995e-06, + "loss": 0.70170504, + "num_input_tokens_seen": 117514190, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.14300537, + "step": 5462, + "time_per_iteration": 2.8734092712402344 + }, + { + "auxiliary_loss_clip": 0.01127598, + "auxiliary_loss_mlp": 0.01033924, + "balance_loss_clip": 1.03944755, + "balance_loss_mlp": 1.01741338, + "epoch": 0.6568869115613539, + "flos": 32781342896640.0, + "grad_norm": 1.996723375351301, + "language_loss": 0.60407293, + "learning_rate": 1.112832758929712e-06, + "loss": 0.62568814, + "num_input_tokens_seen": 117536800, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.16503906, + "step": 5463, + "time_per_iteration": 2.976567506790161 + }, + { + "auxiliary_loss_clip": 0.01128621, + "auxiliary_loss_mlp": 0.01033923, + "balance_loss_clip": 1.04056716, + "balance_loss_mlp": 1.01763892, + "epoch": 0.657007154451993, + "flos": 18442428220800.0, + "grad_norm": 1.5776139119362786, + "language_loss": 0.74721235, + "learning_rate": 1.11213468615039e-06, + "loss": 0.76883775, + "num_input_tokens_seen": 117556230, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.1628418, + "step": 5464, + "time_per_iteration": 2.7766430377960205 + }, + { + "auxiliary_loss_clip": 0.01123369, + "auxiliary_loss_mlp": 0.01034474, + "balance_loss_clip": 1.04101646, + "balance_loss_mlp": 1.01968014, + "epoch": 0.6571273973426321, + "flos": 25156717902720.0, + "grad_norm": 1.5315991750606435, + "language_loss": 0.75484025, + "learning_rate": 1.1114367480595292e-06, + "loss": 0.77641869, + "num_input_tokens_seen": 117577310, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.14801025, + "step": 5465, + "time_per_iteration": 2.862489700317383 + }, + { + "auxiliary_loss_clip": 0.0113439, + "auxiliary_loss_mlp": 0.01045217, + "balance_loss_clip": 1.04158711, + "balance_loss_mlp": 1.0270735, + "epoch": 0.6572476402332712, + "flos": 17529830352000.0, + "grad_norm": 1.8584844906272746, + "language_loss": 0.81031454, + "learning_rate": 1.1107389447630086e-06, + "loss": 0.83211064, + "num_input_tokens_seen": 117596010, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.18139648, + "step": 5466, + "time_per_iteration": 2.79082989692688 + }, + { + "auxiliary_loss_clip": 0.01129129, + "auxiliary_loss_mlp": 0.01037518, + "balance_loss_clip": 1.04110479, + "balance_loss_mlp": 1.02092385, + "epoch": 0.6573678831239103, + "flos": 17014260487680.0, + "grad_norm": 2.2212309124523695, + "language_loss": 0.7859993, + "learning_rate": 1.1100412763666818e-06, + "loss": 0.80766571, + "num_input_tokens_seen": 117611270, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.16589355, + "step": 5467, + "time_per_iteration": 2.826096773147583 + }, + { + "auxiliary_loss_clip": 0.01128379, + "auxiliary_loss_mlp": 0.01041353, + "balance_loss_clip": 1.04170442, + "balance_loss_mlp": 1.02616572, + "epoch": 0.6574881260145494, + "flos": 23910078528000.0, + "grad_norm": 1.6364204473302923, + "language_loss": 0.800834, + "learning_rate": 1.1093437429763865e-06, + "loss": 0.82253134, + "num_input_tokens_seen": 117631535, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.15197754, + "step": 5468, + "time_per_iteration": 2.8440425395965576 + }, + { + "auxiliary_loss_clip": 0.01129457, + "auxiliary_loss_mlp": 0.01042801, + "balance_loss_clip": 1.04403639, + "balance_loss_mlp": 1.02564681, + "epoch": 0.6576083689051885, + "flos": 11218458504960.0, + "grad_norm": 2.208312701085008, + "language_loss": 0.73626769, + "learning_rate": 1.1086463446979361e-06, + "loss": 0.75799024, + "num_input_tokens_seen": 117649885, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.17150879, + "step": 5469, + "time_per_iteration": 2.820996046066284 + }, + { + "auxiliary_loss_clip": 0.0113312, + "auxiliary_loss_mlp": 0.01043382, + "balance_loss_clip": 1.04497695, + "balance_loss_mlp": 1.02728927, + "epoch": 0.6577286117958275, + "flos": 22455553190400.0, + "grad_norm": 1.7807363803240415, + "language_loss": 0.7735616, + "learning_rate": 1.1079490816371277e-06, + "loss": 0.79532665, + "num_input_tokens_seen": 117669650, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.1607666, + "step": 5470, + "time_per_iteration": 2.90639591217041 + }, + { + "auxiliary_loss_clip": 0.01128938, + "auxiliary_loss_mlp": 0.01035555, + "balance_loss_clip": 1.03906488, + "balance_loss_mlp": 1.01886582, + "epoch": 0.6578488546864667, + "flos": 21872184405120.0, + "grad_norm": 2.2161110653532172, + "language_loss": 0.74674213, + "learning_rate": 1.1072519538997352e-06, + "loss": 0.76838708, + "num_input_tokens_seen": 117688790, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.16699219, + "step": 5471, + "time_per_iteration": 3.5670816898345947 + }, + { + "auxiliary_loss_clip": 0.01129193, + "auxiliary_loss_mlp": 0.01038028, + "balance_loss_clip": 1.04073024, + "balance_loss_mlp": 1.0213511, + "epoch": 0.6579690975771058, + "flos": 23543753673600.0, + "grad_norm": 1.609941245577014, + "language_loss": 0.81950402, + "learning_rate": 1.1065549615915095e-06, + "loss": 0.84117627, + "num_input_tokens_seen": 117708620, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.16662598, + "step": 5472, + "time_per_iteration": 2.834395170211792 + }, + { + "auxiliary_loss_clip": 0.01134042, + "auxiliary_loss_mlp": 0.01041016, + "balance_loss_clip": 1.045789, + "balance_loss_mlp": 1.02462459, + "epoch": 0.6580893404677448, + "flos": 32743995730560.0, + "grad_norm": 2.25622482551731, + "language_loss": 0.78910196, + "learning_rate": 1.105858104818187e-06, + "loss": 0.81085265, + "num_input_tokens_seen": 117729775, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.16394043, + "step": 5473, + "time_per_iteration": 2.9057235717773438 + }, + { + "auxiliary_loss_clip": 0.0113303, + "auxiliary_loss_mlp": 0.01041288, + "balance_loss_clip": 1.04313087, + "balance_loss_mlp": 1.02387166, + "epoch": 0.658209583358384, + "flos": 15888138220800.0, + "grad_norm": 2.4470262242268817, + "language_loss": 0.75304532, + "learning_rate": 1.105161383685478e-06, + "loss": 0.7747885, + "num_input_tokens_seen": 117746160, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.17443848, + "step": 5474, + "time_per_iteration": 2.900550127029419 + }, + { + "auxiliary_loss_clip": 0.01037246, + "auxiliary_loss_mlp": 0.01000332, + "balance_loss_clip": 1.00968194, + "balance_loss_mlp": 0.99871063, + "epoch": 0.658329826249023, + "flos": 62695902447360.0, + "grad_norm": 0.7265972751375439, + "language_loss": 0.56292003, + "learning_rate": 1.1044647982990771e-06, + "loss": 0.58329582, + "num_input_tokens_seen": 117808045, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01623535, + "step": 5475, + "time_per_iteration": 4.146405220031738 + }, + { + "auxiliary_loss_clip": 0.01132602, + "auxiliary_loss_mlp": 0.01044073, + "balance_loss_clip": 1.04157913, + "balance_loss_mlp": 1.02733564, + "epoch": 0.6584500691396621, + "flos": 31722624501120.0, + "grad_norm": 2.4684074669352416, + "language_loss": 0.64698756, + "learning_rate": 1.1037683487646536e-06, + "loss": 0.66875428, + "num_input_tokens_seen": 117828330, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.1673584, + "step": 5476, + "time_per_iteration": 2.92642879486084 + }, + { + "auxiliary_loss_clip": 0.01126982, + "auxiliary_loss_mlp": 0.01037415, + "balance_loss_clip": 1.04151165, + "balance_loss_mlp": 1.02126217, + "epoch": 0.6585703120303013, + "flos": 18406086635520.0, + "grad_norm": 1.7509058334825673, + "language_loss": 0.77245426, + "learning_rate": 1.1030720351878583e-06, + "loss": 0.79409826, + "num_input_tokens_seen": 117846450, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.16137695, + "step": 5477, + "time_per_iteration": 3.6351194381713867 + }, + { + "auxiliary_loss_clip": 0.01036318, + "auxiliary_loss_mlp": 0.01004725, + "balance_loss_clip": 1.00864947, + "balance_loss_mlp": 1.00304413, + "epoch": 0.6586905549209403, + "flos": 58309880434560.0, + "grad_norm": 0.802205578973495, + "language_loss": 0.57588255, + "learning_rate": 1.102375857674323e-06, + "loss": 0.59629297, + "num_input_tokens_seen": 117908365, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.0168457, + "step": 5478, + "time_per_iteration": 3.313328266143799 + }, + { + "auxiliary_loss_clip": 0.01128414, + "auxiliary_loss_mlp": 0.01035108, + "balance_loss_clip": 1.04153717, + "balance_loss_mlp": 1.02015901, + "epoch": 0.6588107978115794, + "flos": 22782627457920.0, + "grad_norm": 1.6652765076431801, + "language_loss": 0.90363556, + "learning_rate": 1.1016798163296561e-06, + "loss": 0.9252708, + "num_input_tokens_seen": 117927565, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.1496582, + "step": 5479, + "time_per_iteration": 2.94150447845459 + }, + { + "auxiliary_loss_clip": 0.0112836, + "auxiliary_loss_mlp": 0.01037305, + "balance_loss_clip": 1.04163313, + "balance_loss_mlp": 1.02066374, + "epoch": 0.6589310407022185, + "flos": 20667525050880.0, + "grad_norm": 1.865168606081476, + "language_loss": 0.66074157, + "learning_rate": 1.1009839112594471e-06, + "loss": 0.6823982, + "num_input_tokens_seen": 117945590, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.16650391, + "step": 5480, + "time_per_iteration": 3.7687792778015137 + }, + { + "auxiliary_loss_clip": 0.01128566, + "auxiliary_loss_mlp": 0.01041066, + "balance_loss_clip": 1.04122627, + "balance_loss_mlp": 1.02468622, + "epoch": 0.6590512835928576, + "flos": 25630595055360.0, + "grad_norm": 2.4310078492894127, + "language_loss": 0.71792859, + "learning_rate": 1.1002881425692638e-06, + "loss": 0.73962486, + "num_input_tokens_seen": 117966020, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.16394043, + "step": 5481, + "time_per_iteration": 2.8398215770721436 + }, + { + "auxiliary_loss_clip": 0.01130518, + "auxiliary_loss_mlp": 0.01040511, + "balance_loss_clip": 1.04116654, + "balance_loss_mlp": 1.02435851, + "epoch": 0.6591715264834966, + "flos": 23726108044800.0, + "grad_norm": 1.7366455028540855, + "language_loss": 0.75546122, + "learning_rate": 1.0995925103646532e-06, + "loss": 0.77717149, + "num_input_tokens_seen": 117984620, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.16162109, + "step": 5482, + "time_per_iteration": 2.849484443664551 + }, + { + "auxiliary_loss_clip": 0.01130711, + "auxiliary_loss_mlp": 0.01041194, + "balance_loss_clip": 1.04346371, + "balance_loss_mlp": 1.02401042, + "epoch": 0.6592917693741358, + "flos": 35773850822400.0, + "grad_norm": 1.440578805776147, + "language_loss": 0.66684115, + "learning_rate": 1.0988970147511437e-06, + "loss": 0.68856019, + "num_input_tokens_seen": 118006500, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.17181396, + "step": 5483, + "time_per_iteration": 2.9405534267425537 + }, + { + "auxiliary_loss_clip": 0.01132215, + "auxiliary_loss_mlp": 0.01041345, + "balance_loss_clip": 1.04362941, + "balance_loss_mlp": 1.02529955, + "epoch": 0.6594120122647749, + "flos": 21396834794880.0, + "grad_norm": 2.084258206524778, + "language_loss": 0.80602193, + "learning_rate": 1.0982016558342405e-06, + "loss": 0.82775748, + "num_input_tokens_seen": 118025470, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.16046143, + "step": 5484, + "time_per_iteration": 2.8389365673065186 + }, + { + "auxiliary_loss_clip": 0.01131192, + "auxiliary_loss_mlp": 0.01043696, + "balance_loss_clip": 1.0430243, + "balance_loss_mlp": 1.02751899, + "epoch": 0.6595322551554139, + "flos": 19351829779200.0, + "grad_norm": 1.795559389248273, + "language_loss": 0.7130565, + "learning_rate": 1.0975064337194291e-06, + "loss": 0.73480541, + "num_input_tokens_seen": 118043515, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.1618042, + "step": 5485, + "time_per_iteration": 2.8192343711853027 + }, + { + "auxiliary_loss_clip": 0.01131808, + "auxiliary_loss_mlp": 0.0104772, + "balance_loss_clip": 1.04303563, + "balance_loss_mlp": 1.0315913, + "epoch": 0.6596524980460531, + "flos": 16837113588480.0, + "grad_norm": 1.5274205521328104, + "language_loss": 0.70133168, + "learning_rate": 1.0968113485121743e-06, + "loss": 0.72312695, + "num_input_tokens_seen": 118063105, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.16125488, + "step": 5486, + "time_per_iteration": 2.8849263191223145 + }, + { + "auxiliary_loss_clip": 0.0112828, + "auxiliary_loss_mlp": 0.01036781, + "balance_loss_clip": 1.0400455, + "balance_loss_mlp": 1.02048564, + "epoch": 0.6597727409366921, + "flos": 21798567480960.0, + "grad_norm": 1.7495198362976794, + "language_loss": 0.80151874, + "learning_rate": 1.0961164003179185e-06, + "loss": 0.82316941, + "num_input_tokens_seen": 118081615, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.1628418, + "step": 5487, + "time_per_iteration": 3.0063185691833496 + }, + { + "auxiliary_loss_clip": 0.01131539, + "auxiliary_loss_mlp": 0.01038457, + "balance_loss_clip": 1.04230952, + "balance_loss_mlp": 1.0221014, + "epoch": 0.6598929838273312, + "flos": 23730704985600.0, + "grad_norm": 1.7519483828677005, + "language_loss": 0.84490538, + "learning_rate": 1.0954215892420884e-06, + "loss": 0.8666054, + "num_input_tokens_seen": 118102315, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.16357422, + "step": 5488, + "time_per_iteration": 2.806857109069824 + }, + { + "auxiliary_loss_clip": 0.01132817, + "auxiliary_loss_mlp": 0.01040289, + "balance_loss_clip": 1.04253078, + "balance_loss_mlp": 1.02418375, + "epoch": 0.6600132267179702, + "flos": 19974520978560.0, + "grad_norm": 1.6287939080550646, + "language_loss": 0.70340872, + "learning_rate": 1.094726915390082e-06, + "loss": 0.72513974, + "num_input_tokens_seen": 118120650, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.16088867, + "step": 5489, + "time_per_iteration": 2.877051830291748 + }, + { + "auxiliary_loss_clip": 0.01131405, + "auxiliary_loss_mlp": 0.01038341, + "balance_loss_clip": 1.04237247, + "balance_loss_mlp": 1.02196157, + "epoch": 0.6601334696086094, + "flos": 22342649765760.0, + "grad_norm": 1.9850775669791112, + "language_loss": 0.69668883, + "learning_rate": 1.0940323788672836e-06, + "loss": 0.71838629, + "num_input_tokens_seen": 118139825, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.16381836, + "step": 5490, + "time_per_iteration": 2.854743003845215 + }, + { + "auxiliary_loss_clip": 0.01125743, + "auxiliary_loss_mlp": 0.01032404, + "balance_loss_clip": 1.0411365, + "balance_loss_mlp": 1.01777661, + "epoch": 0.6602537124992485, + "flos": 25703098657920.0, + "grad_norm": 1.5143661238323702, + "language_loss": 0.73870349, + "learning_rate": 1.0933379797790522e-06, + "loss": 0.7602849, + "num_input_tokens_seen": 118159240, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.14624023, + "step": 5491, + "time_per_iteration": 2.894660234451294 + }, + { + "auxiliary_loss_clip": 0.01128745, + "auxiliary_loss_mlp": 0.01043784, + "balance_loss_clip": 1.0416162, + "balance_loss_mlp": 1.02820301, + "epoch": 0.6603739553898875, + "flos": 25848572739840.0, + "grad_norm": 4.387427156435987, + "language_loss": 0.71553826, + "learning_rate": 1.0926437182307293e-06, + "loss": 0.73726356, + "num_input_tokens_seen": 118178050, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15600586, + "step": 5492, + "time_per_iteration": 2.858466863632202 + }, + { + "auxiliary_loss_clip": 0.0112961, + "auxiliary_loss_mlp": 0.01044067, + "balance_loss_clip": 1.04170847, + "balance_loss_mlp": 1.02758026, + "epoch": 0.6604941982805267, + "flos": 24570296461440.0, + "grad_norm": 1.6472117519124907, + "language_loss": 0.77775645, + "learning_rate": 1.0919495943276338e-06, + "loss": 0.79949319, + "num_input_tokens_seen": 118199070, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.16479492, + "step": 5493, + "time_per_iteration": 3.026715040206909 + }, + { + "auxiliary_loss_clip": 0.01135343, + "auxiliary_loss_mlp": 0.01041765, + "balance_loss_clip": 1.04442465, + "balance_loss_mlp": 1.02513552, + "epoch": 0.6606144411711657, + "flos": 13261775581440.0, + "grad_norm": 2.865608674815674, + "language_loss": 0.76531291, + "learning_rate": 1.0912556081750611e-06, + "loss": 0.78708398, + "num_input_tokens_seen": 118217000, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.16625977, + "step": 5494, + "time_per_iteration": 2.973588705062866 + }, + { + "auxiliary_loss_clip": 0.01129748, + "auxiliary_loss_mlp": 0.01035857, + "balance_loss_clip": 1.04218078, + "balance_loss_mlp": 1.01976395, + "epoch": 0.6607346840618048, + "flos": 25155281358720.0, + "grad_norm": 3.302900695162531, + "language_loss": 0.76472998, + "learning_rate": 1.0905617598782909e-06, + "loss": 0.78638607, + "num_input_tokens_seen": 118237205, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.16088867, + "step": 5495, + "time_per_iteration": 2.971911668777466 + }, + { + "auxiliary_loss_clip": 0.01128365, + "auxiliary_loss_mlp": 0.01040606, + "balance_loss_clip": 1.04186773, + "balance_loss_mlp": 1.02471566, + "epoch": 0.660854926952444, + "flos": 17638029095040.0, + "grad_norm": 1.9571814499662383, + "language_loss": 0.81120646, + "learning_rate": 1.0898680495425775e-06, + "loss": 0.83289617, + "num_input_tokens_seen": 118255495, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15893555, + "step": 5496, + "time_per_iteration": 2.8483543395996094 + }, + { + "auxiliary_loss_clip": 0.0112873, + "auxiliary_loss_mlp": 0.01043508, + "balance_loss_clip": 1.03994071, + "balance_loss_mlp": 1.02664042, + "epoch": 0.660975169843083, + "flos": 16836000266880.0, + "grad_norm": 1.6711485232103132, + "language_loss": 0.8027882, + "learning_rate": 1.0891744772731594e-06, + "loss": 0.82451063, + "num_input_tokens_seen": 118273310, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.16851807, + "step": 5497, + "time_per_iteration": 2.8875842094421387 + }, + { + "auxiliary_loss_clip": 0.01127041, + "auxiliary_loss_mlp": 0.01042148, + "balance_loss_clip": 1.04113865, + "balance_loss_mlp": 1.02721167, + "epoch": 0.6610954127337221, + "flos": 26870410846080.0, + "grad_norm": 1.7993997997361708, + "language_loss": 0.65964228, + "learning_rate": 1.088481043175248e-06, + "loss": 0.68133414, + "num_input_tokens_seen": 118293880, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.14941406, + "step": 5498, + "time_per_iteration": 2.9919471740722656 + }, + { + "auxiliary_loss_clip": 0.0112887, + "auxiliary_loss_mlp": 0.01043316, + "balance_loss_clip": 1.04219317, + "balance_loss_mlp": 1.02769911, + "epoch": 0.6612156556243612, + "flos": 26465697331200.0, + "grad_norm": 1.8930942392747836, + "language_loss": 0.75576687, + "learning_rate": 1.0877877473540368e-06, + "loss": 0.77748871, + "num_input_tokens_seen": 118314465, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.15612793, + "step": 5499, + "time_per_iteration": 2.949469566345215 + }, + { + "auxiliary_loss_clip": 0.01131971, + "auxiliary_loss_mlp": 0.01041667, + "balance_loss_clip": 1.04259026, + "balance_loss_mlp": 1.02640843, + "epoch": 0.6613358985150003, + "flos": 19791915212160.0, + "grad_norm": 2.026295142943729, + "language_loss": 0.72798371, + "learning_rate": 1.0870945899147002e-06, + "loss": 0.74972016, + "num_input_tokens_seen": 118331110, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.15258789, + "step": 5500, + "time_per_iteration": 3.473834991455078 + }, + { + "auxiliary_loss_clip": 0.01128111, + "auxiliary_loss_mlp": 0.0103936, + "balance_loss_clip": 1.04294562, + "balance_loss_mlp": 1.02413654, + "epoch": 0.6614561414056394, + "flos": 26831627136000.0, + "grad_norm": 1.7614329752972537, + "language_loss": 0.75926518, + "learning_rate": 1.0864015709623879e-06, + "loss": 0.78093988, + "num_input_tokens_seen": 118351980, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.15234375, + "step": 5501, + "time_per_iteration": 2.859311580657959 + }, + { + "auxiliary_loss_clip": 0.0113123, + "auxiliary_loss_mlp": 0.01041155, + "balance_loss_clip": 1.04110479, + "balance_loss_mlp": 1.02501392, + "epoch": 0.6615763842962785, + "flos": 22894597128960.0, + "grad_norm": 2.218296955391349, + "language_loss": 0.8067739, + "learning_rate": 1.0857086906022313e-06, + "loss": 0.82849777, + "num_input_tokens_seen": 118370315, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.16137695, + "step": 5502, + "time_per_iteration": 2.879662036895752 + }, + { + "auxiliary_loss_clip": 0.01128422, + "auxiliary_loss_mlp": 0.01040041, + "balance_loss_clip": 1.04062331, + "balance_loss_mlp": 1.02342367, + "epoch": 0.6616966271869176, + "flos": 24790321221120.0, + "grad_norm": 1.8922027282243798, + "language_loss": 0.73244369, + "learning_rate": 1.0850159489393388e-06, + "loss": 0.75412834, + "num_input_tokens_seen": 118389575, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.16601562, + "step": 5503, + "time_per_iteration": 2.8095154762268066 + }, + { + "auxiliary_loss_clip": 0.01128639, + "auxiliary_loss_mlp": 0.01039471, + "balance_loss_clip": 1.04035091, + "balance_loss_mlp": 1.02399755, + "epoch": 0.6618168700775566, + "flos": 17202109639680.0, + "grad_norm": 1.8838682424010254, + "language_loss": 0.82412791, + "learning_rate": 1.0843233460787992e-06, + "loss": 0.84580904, + "num_input_tokens_seen": 118406790, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15490723, + "step": 5504, + "time_per_iteration": 3.7551536560058594 + }, + { + "auxiliary_loss_clip": 0.01126412, + "auxiliary_loss_mlp": 0.01038786, + "balance_loss_clip": 1.04156089, + "balance_loss_mlp": 1.02334833, + "epoch": 0.6619371129681958, + "flos": 25447091448960.0, + "grad_norm": 1.8610838503961928, + "language_loss": 0.78232437, + "learning_rate": 1.0836308821256805e-06, + "loss": 0.80397636, + "num_input_tokens_seen": 118427590, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.15429688, + "step": 5505, + "time_per_iteration": 2.8615927696228027 + }, + { + "auxiliary_loss_clip": 0.01129109, + "auxiliary_loss_mlp": 0.01042859, + "balance_loss_clip": 1.0427568, + "balance_loss_mlp": 1.02717137, + "epoch": 0.6620573558588349, + "flos": 18040444139520.0, + "grad_norm": 1.947123426340136, + "language_loss": 0.77494538, + "learning_rate": 1.0829385571850282e-06, + "loss": 0.79666507, + "num_input_tokens_seen": 118444570, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.15673828, + "step": 5506, + "time_per_iteration": 3.6021206378936768 + }, + { + "auxiliary_loss_clip": 0.01135412, + "auxiliary_loss_mlp": 0.0104653, + "balance_loss_clip": 1.04246521, + "balance_loss_mlp": 1.02879179, + "epoch": 0.6621775987494739, + "flos": 17785586165760.0, + "grad_norm": 2.5755547967600663, + "language_loss": 0.83532774, + "learning_rate": 1.0822463713618679e-06, + "loss": 0.85714722, + "num_input_tokens_seen": 118461425, + "router_z_loss_clip": 0.93066406, + "router_z_loss_mlp": 0.17736816, + "step": 5507, + "time_per_iteration": 2.8225901126861572 + }, + { + "auxiliary_loss_clip": 0.01129421, + "auxiliary_loss_mlp": 0.01046041, + "balance_loss_clip": 1.04122114, + "balance_loss_mlp": 1.03012645, + "epoch": 0.6622978416401131, + "flos": 17492590926720.0, + "grad_norm": 2.0387049879614625, + "language_loss": 0.85123384, + "learning_rate": 1.0815543247612034e-06, + "loss": 0.87298846, + "num_input_tokens_seen": 118478495, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15917969, + "step": 5508, + "time_per_iteration": 2.871896743774414 + }, + { + "auxiliary_loss_clip": 0.0112816, + "auxiliary_loss_mlp": 0.01039598, + "balance_loss_clip": 1.04012465, + "balance_loss_mlp": 1.02331388, + "epoch": 0.6624180845307521, + "flos": 21648352803840.0, + "grad_norm": 1.9055046300542526, + "language_loss": 0.8296268, + "learning_rate": 1.0808624174880168e-06, + "loss": 0.85130435, + "num_input_tokens_seen": 118499145, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.16271973, + "step": 5509, + "time_per_iteration": 3.882478713989258 + }, + { + "auxiliary_loss_clip": 0.01126711, + "auxiliary_loss_mlp": 0.01033737, + "balance_loss_clip": 1.04052091, + "balance_loss_mlp": 1.01801324, + "epoch": 0.6625383274213912, + "flos": 23805902108160.0, + "grad_norm": 2.9500211500818976, + "language_loss": 0.79944783, + "learning_rate": 1.080170649647272e-06, + "loss": 0.82105231, + "num_input_tokens_seen": 118518950, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15734863, + "step": 5510, + "time_per_iteration": 3.030850410461426 + }, + { + "auxiliary_loss_clip": 0.01128449, + "auxiliary_loss_mlp": 0.0103905, + "balance_loss_clip": 1.04029608, + "balance_loss_mlp": 1.0230701, + "epoch": 0.6626585703120303, + "flos": 33262941473280.0, + "grad_norm": 1.6482388362398492, + "language_loss": 0.6740433, + "learning_rate": 1.0794790213439068e-06, + "loss": 0.69571829, + "num_input_tokens_seen": 118545850, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.159729, + "step": 5511, + "time_per_iteration": 3.0210442543029785 + }, + { + "auxiliary_loss_clip": 0.01132942, + "auxiliary_loss_mlp": 0.01046676, + "balance_loss_clip": 1.04240823, + "balance_loss_mlp": 1.02943826, + "epoch": 0.6627788132026694, + "flos": 22085780630400.0, + "grad_norm": 1.936906558406778, + "language_loss": 0.78444624, + "learning_rate": 1.078787532682843e-06, + "loss": 0.80624235, + "num_input_tokens_seen": 118563325, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.17248535, + "step": 5512, + "time_per_iteration": 2.9444010257720947 + }, + { + "auxiliary_loss_clip": 0.01131154, + "auxiliary_loss_mlp": 0.01035963, + "balance_loss_clip": 1.04181099, + "balance_loss_mlp": 1.01924992, + "epoch": 0.6628990560933085, + "flos": 36173608260480.0, + "grad_norm": 2.1525283494574783, + "language_loss": 0.76208794, + "learning_rate": 1.0780961837689773e-06, + "loss": 0.78375912, + "num_input_tokens_seen": 118582835, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.16723633, + "step": 5513, + "time_per_iteration": 2.950026512145996 + }, + { + "auxiliary_loss_clip": 0.01127363, + "auxiliary_loss_mlp": 0.01036912, + "balance_loss_clip": 1.04217815, + "balance_loss_mlp": 1.02077127, + "epoch": 0.6630192989839476, + "flos": 18513567106560.0, + "grad_norm": 1.4818113601012624, + "language_loss": 0.69735497, + "learning_rate": 1.0774049747071883e-06, + "loss": 0.71899766, + "num_input_tokens_seen": 118600715, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.16149902, + "step": 5514, + "time_per_iteration": 2.88492488861084 + }, + { + "auxiliary_loss_clip": 0.01128687, + "auxiliary_loss_mlp": 0.01041855, + "balance_loss_clip": 1.04323888, + "balance_loss_mlp": 1.02563119, + "epoch": 0.6631395418745867, + "flos": 35809510049280.0, + "grad_norm": 4.602777351147472, + "language_loss": 0.68060625, + "learning_rate": 1.076713905602332e-06, + "loss": 0.70231164, + "num_input_tokens_seen": 118621290, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.16235352, + "step": 5515, + "time_per_iteration": 3.089379072189331 + }, + { + "auxiliary_loss_clip": 0.01133548, + "auxiliary_loss_mlp": 0.01037226, + "balance_loss_clip": 1.04465818, + "balance_loss_mlp": 1.02176476, + "epoch": 0.6632597847652257, + "flos": 20047742853120.0, + "grad_norm": 1.8493510513001055, + "language_loss": 0.80657661, + "learning_rate": 1.07602297655924e-06, + "loss": 0.82828438, + "num_input_tokens_seen": 118639610, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.15466309, + "step": 5516, + "time_per_iteration": 2.960905075073242 + }, + { + "auxiliary_loss_clip": 0.01129497, + "auxiliary_loss_mlp": 0.01036302, + "balance_loss_clip": 1.0420239, + "balance_loss_mlp": 1.02129304, + "epoch": 0.6633800276558649, + "flos": 21214480423680.0, + "grad_norm": 1.6474603990662038, + "language_loss": 0.81320339, + "learning_rate": 1.0753321876827292e-06, + "loss": 0.8348614, + "num_input_tokens_seen": 118658895, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.15014648, + "step": 5517, + "time_per_iteration": 2.8960530757904053 + }, + { + "auxiliary_loss_clip": 0.01130948, + "auxiliary_loss_mlp": 0.01034636, + "balance_loss_clip": 1.04126334, + "balance_loss_mlp": 1.01857865, + "epoch": 0.663500270546504, + "flos": 23987753688960.0, + "grad_norm": 2.461189217041742, + "language_loss": 0.73731351, + "learning_rate": 1.0746415390775893e-06, + "loss": 0.75896937, + "num_input_tokens_seen": 118677025, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.16052246, + "step": 5518, + "time_per_iteration": 2.9110031127929688 + }, + { + "auxiliary_loss_clip": 0.01127955, + "auxiliary_loss_mlp": 0.01036792, + "balance_loss_clip": 1.04305673, + "balance_loss_mlp": 1.02167594, + "epoch": 0.663620513437143, + "flos": 17932389050880.0, + "grad_norm": 2.148109666498624, + "language_loss": 0.76609176, + "learning_rate": 1.0739510308485939e-06, + "loss": 0.78773916, + "num_input_tokens_seen": 118694240, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.15124512, + "step": 5519, + "time_per_iteration": 2.7893574237823486 + }, + { + "auxiliary_loss_clip": 0.01036578, + "auxiliary_loss_mlp": 0.01002676, + "balance_loss_clip": 1.00860262, + "balance_loss_mlp": 1.0009594, + "epoch": 0.6637407563277821, + "flos": 57840241086720.0, + "grad_norm": 0.8100165796685581, + "language_loss": 0.62491447, + "learning_rate": 1.07326066310049e-06, + "loss": 0.645307, + "num_input_tokens_seen": 118758365, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01721191, + "step": 5520, + "time_per_iteration": 3.3058128356933594 + }, + { + "auxiliary_loss_clip": 0.01133531, + "auxiliary_loss_mlp": 0.01045285, + "balance_loss_clip": 1.04298651, + "balance_loss_mlp": 1.02835751, + "epoch": 0.6638609992184212, + "flos": 27306007079040.0, + "grad_norm": 1.817230986333432, + "language_loss": 0.79437011, + "learning_rate": 1.0725704359380059e-06, + "loss": 0.81615829, + "num_input_tokens_seen": 118778220, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.16931152, + "step": 5521, + "time_per_iteration": 2.8136465549468994 + }, + { + "auxiliary_loss_clip": 0.01129661, + "auxiliary_loss_mlp": 0.01033175, + "balance_loss_clip": 1.04169011, + "balance_loss_mlp": 1.01819026, + "epoch": 0.6639812421090603, + "flos": 18624854419200.0, + "grad_norm": 1.6825393549921124, + "language_loss": 0.7228812, + "learning_rate": 1.0718803494658497e-06, + "loss": 0.74450958, + "num_input_tokens_seen": 118797110, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.15002441, + "step": 5522, + "time_per_iteration": 2.8493640422821045 + }, + { + "auxiliary_loss_clip": 0.01127778, + "auxiliary_loss_mlp": 0.01041223, + "balance_loss_clip": 1.0416292, + "balance_loss_mlp": 1.02542782, + "epoch": 0.6641014849996993, + "flos": 15924479806080.0, + "grad_norm": 2.2179672183636727, + "language_loss": 0.84104133, + "learning_rate": 1.071190403788707e-06, + "loss": 0.86273134, + "num_input_tokens_seen": 118812415, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.15783691, + "step": 5523, + "time_per_iteration": 2.8347816467285156 + }, + { + "auxiliary_loss_clip": 0.01135945, + "auxiliary_loss_mlp": 0.01039912, + "balance_loss_clip": 1.04392338, + "balance_loss_mlp": 1.02205443, + "epoch": 0.6642217278903385, + "flos": 26505486622080.0, + "grad_norm": 1.8865399781935828, + "language_loss": 0.75886345, + "learning_rate": 1.0705005990112415e-06, + "loss": 0.78062201, + "num_input_tokens_seen": 118832195, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.17834473, + "step": 5524, + "time_per_iteration": 2.8034446239471436 + }, + { + "auxiliary_loss_clip": 0.0112745, + "auxiliary_loss_mlp": 0.01042016, + "balance_loss_clip": 1.04028773, + "balance_loss_mlp": 1.026757, + "epoch": 0.6643419707809776, + "flos": 15377308951680.0, + "grad_norm": 3.0296678521329703, + "language_loss": 0.74444467, + "learning_rate": 1.0698109352380957e-06, + "loss": 0.76613933, + "num_input_tokens_seen": 118849795, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.15258789, + "step": 5525, + "time_per_iteration": 2.7520265579223633 + }, + { + "auxiliary_loss_clip": 0.01132627, + "auxiliary_loss_mlp": 0.0103545, + "balance_loss_clip": 1.04231226, + "balance_loss_mlp": 1.01913023, + "epoch": 0.6644622136716166, + "flos": 25117610970240.0, + "grad_norm": 1.846034467729809, + "language_loss": 0.78094733, + "learning_rate": 1.0691214125738909e-06, + "loss": 0.80262816, + "num_input_tokens_seen": 118870000, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.16320801, + "step": 5526, + "time_per_iteration": 2.855921506881714 + }, + { + "auxiliary_loss_clip": 0.01036144, + "auxiliary_loss_mlp": 0.01000098, + "balance_loss_clip": 1.00819945, + "balance_loss_mlp": 0.99839288, + "epoch": 0.6645824565622558, + "flos": 66201717680640.0, + "grad_norm": 0.8405109143821343, + "language_loss": 0.57470739, + "learning_rate": 1.0684320311232287e-06, + "loss": 0.59506977, + "num_input_tokens_seen": 118932905, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01708984, + "step": 5527, + "time_per_iteration": 3.386329174041748 + }, + { + "auxiliary_loss_clip": 0.01131762, + "auxiliary_loss_mlp": 0.01039358, + "balance_loss_clip": 1.04049826, + "balance_loss_mlp": 1.02271008, + "epoch": 0.6647026994528948, + "flos": 25082131311360.0, + "grad_norm": 1.97939035569776, + "language_loss": 0.81303489, + "learning_rate": 1.0677427909906865e-06, + "loss": 0.83474612, + "num_input_tokens_seen": 118953355, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.16644287, + "step": 5528, + "time_per_iteration": 2.9276397228240967 + }, + { + "auxiliary_loss_clip": 0.01133419, + "auxiliary_loss_mlp": 0.01041143, + "balance_loss_clip": 1.04254794, + "balance_loss_mlp": 1.02512097, + "epoch": 0.6648229423435339, + "flos": 18222187979520.0, + "grad_norm": 1.7906597637798305, + "language_loss": 0.72218543, + "learning_rate": 1.0670536922808216e-06, + "loss": 0.74393106, + "num_input_tokens_seen": 118973480, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.16027832, + "step": 5529, + "time_per_iteration": 3.684854507446289 + }, + { + "auxiliary_loss_clip": 0.01126375, + "auxiliary_loss_mlp": 0.01042535, + "balance_loss_clip": 1.04044354, + "balance_loss_mlp": 1.02732396, + "epoch": 0.6649431852341731, + "flos": 18296882311680.0, + "grad_norm": 1.9257713429953367, + "language_loss": 0.72003472, + "learning_rate": 1.06636473509817e-06, + "loss": 0.74172378, + "num_input_tokens_seen": 118989860, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15222168, + "step": 5530, + "time_per_iteration": 2.861320734024048 + }, + { + "auxiliary_loss_clip": 0.01130461, + "auxiliary_loss_mlp": 0.01036257, + "balance_loss_clip": 1.04222703, + "balance_loss_mlp": 1.02064037, + "epoch": 0.6650634281248121, + "flos": 17019575700480.0, + "grad_norm": 1.8820657285573403, + "language_loss": 0.80682766, + "learning_rate": 1.0656759195472447e-06, + "loss": 0.82849479, + "num_input_tokens_seen": 119007150, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15625, + "step": 5531, + "time_per_iteration": 2.849996566772461 + }, + { + "auxiliary_loss_clip": 0.01036161, + "auxiliary_loss_mlp": 0.01002784, + "balance_loss_clip": 1.00859666, + "balance_loss_mlp": 1.00115132, + "epoch": 0.6651836710154512, + "flos": 69294810666240.0, + "grad_norm": 0.774649147598826, + "language_loss": 0.59775925, + "learning_rate": 1.0649872457325414e-06, + "loss": 0.61814868, + "num_input_tokens_seen": 119068435, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01635742, + "step": 5532, + "time_per_iteration": 3.2878501415252686 + }, + { + "auxiliary_loss_clip": 0.01036674, + "auxiliary_loss_mlp": 0.01002142, + "balance_loss_clip": 1.00884914, + "balance_loss_mlp": 1.00037813, + "epoch": 0.6653039139060903, + "flos": 66883444882560.0, + "grad_norm": 0.8601889732654755, + "language_loss": 0.55087161, + "learning_rate": 1.0642987137585278e-06, + "loss": 0.57125986, + "num_input_tokens_seen": 119127960, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.0177002, + "step": 5533, + "time_per_iteration": 4.036995887756348 + }, + { + "auxiliary_loss_clip": 0.01128939, + "auxiliary_loss_mlp": 0.01035443, + "balance_loss_clip": 1.04304004, + "balance_loss_mlp": 1.01907516, + "epoch": 0.6654241567967294, + "flos": 21470056669440.0, + "grad_norm": 1.6375944011003902, + "language_loss": 0.82506472, + "learning_rate": 1.0636103237296561e-06, + "loss": 0.84670854, + "num_input_tokens_seen": 119146885, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.16357422, + "step": 5534, + "time_per_iteration": 2.9268813133239746 + }, + { + "auxiliary_loss_clip": 0.01125911, + "auxiliary_loss_mlp": 0.01033621, + "balance_loss_clip": 1.04265523, + "balance_loss_mlp": 1.01941729, + "epoch": 0.6655443996873684, + "flos": 25119514391040.0, + "grad_norm": 1.7481586159702511, + "language_loss": 0.84366727, + "learning_rate": 1.062922075750353e-06, + "loss": 0.86526251, + "num_input_tokens_seen": 119166900, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.14215088, + "step": 5535, + "time_per_iteration": 3.6309800148010254 + }, + { + "auxiliary_loss_clip": 0.01126419, + "auxiliary_loss_mlp": 0.0103941, + "balance_loss_clip": 1.0412935, + "balance_loss_mlp": 1.02480125, + "epoch": 0.6656646425780076, + "flos": 17457326749440.0, + "grad_norm": 1.8656744158331395, + "language_loss": 0.72259581, + "learning_rate": 1.0622339699250267e-06, + "loss": 0.74425411, + "num_input_tokens_seen": 119184820, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.14605713, + "step": 5536, + "time_per_iteration": 3.012507200241089 + }, + { + "auxiliary_loss_clip": 0.01128797, + "auxiliary_loss_mlp": 0.01040931, + "balance_loss_clip": 1.0422256, + "balance_loss_mlp": 1.02574396, + "epoch": 0.6657848854686467, + "flos": 23434190213760.0, + "grad_norm": 1.7265893554154985, + "language_loss": 0.79381841, + "learning_rate": 1.0615460063580624e-06, + "loss": 0.8155157, + "num_input_tokens_seen": 119203295, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.1517334, + "step": 5537, + "time_per_iteration": 3.0701699256896973 + }, + { + "auxiliary_loss_clip": 0.01127173, + "auxiliary_loss_mlp": 0.01035878, + "balance_loss_clip": 1.04045701, + "balance_loss_mlp": 1.02044082, + "epoch": 0.6659051283592857, + "flos": 11509909459200.0, + "grad_norm": 1.9356728131553547, + "language_loss": 0.72653013, + "learning_rate": 1.060858185153821e-06, + "loss": 0.74816066, + "num_input_tokens_seen": 119221395, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15441895, + "step": 5538, + "time_per_iteration": 3.8134686946868896 + }, + { + "auxiliary_loss_clip": 0.01135139, + "auxiliary_loss_mlp": 0.01041138, + "balance_loss_clip": 1.04320502, + "balance_loss_mlp": 1.02350688, + "epoch": 0.6660253712499249, + "flos": 20594554571520.0, + "grad_norm": 2.367905119177273, + "language_loss": 0.75861406, + "learning_rate": 1.0601705064166474e-06, + "loss": 0.78037691, + "num_input_tokens_seen": 119239790, + "router_z_loss_clip": 0.91894531, + "router_z_loss_mlp": 0.1763916, + "step": 5539, + "time_per_iteration": 2.907818555831909 + }, + { + "auxiliary_loss_clip": 0.01125797, + "auxiliary_loss_mlp": 0.01039466, + "balance_loss_clip": 1.03974617, + "balance_loss_mlp": 1.02381372, + "epoch": 0.666145614140564, + "flos": 21251504367360.0, + "grad_norm": 2.075738837239584, + "language_loss": 0.73606128, + "learning_rate": 1.0594829702508596e-06, + "loss": 0.75771391, + "num_input_tokens_seen": 119257505, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15649414, + "step": 5540, + "time_per_iteration": 2.884732723236084 + }, + { + "auxiliary_loss_clip": 0.01127298, + "auxiliary_loss_mlp": 0.01039675, + "balance_loss_clip": 1.0408814, + "balance_loss_mlp": 1.02349806, + "epoch": 0.666265857031203, + "flos": 33726188200320.0, + "grad_norm": 1.6790710758241871, + "language_loss": 0.54835856, + "learning_rate": 1.0587955767607592e-06, + "loss": 0.57002825, + "num_input_tokens_seen": 119279365, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.16186523, + "step": 5541, + "time_per_iteration": 2.995058059692383 + }, + { + "auxiliary_loss_clip": 0.01129967, + "auxiliary_loss_mlp": 0.01034136, + "balance_loss_clip": 1.04186296, + "balance_loss_mlp": 1.01810241, + "epoch": 0.6663860999218422, + "flos": 17456644391040.0, + "grad_norm": 6.37931090345369, + "language_loss": 0.77673054, + "learning_rate": 1.0581083260506206e-06, + "loss": 0.79837155, + "num_input_tokens_seen": 119296150, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.16052246, + "step": 5542, + "time_per_iteration": 2.877211093902588 + }, + { + "auxiliary_loss_clip": 0.01125989, + "auxiliary_loss_mlp": 0.01037241, + "balance_loss_clip": 1.04035878, + "balance_loss_mlp": 1.02192259, + "epoch": 0.6665063428124812, + "flos": 17676740977920.0, + "grad_norm": 2.3990956437179345, + "language_loss": 0.77008688, + "learning_rate": 1.0574212182246993e-06, + "loss": 0.79171914, + "num_input_tokens_seen": 119314845, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15319824, + "step": 5543, + "time_per_iteration": 2.9125688076019287 + }, + { + "auxiliary_loss_clip": 0.01132669, + "auxiliary_loss_mlp": 0.01041435, + "balance_loss_clip": 1.04047275, + "balance_loss_mlp": 1.02401817, + "epoch": 0.6666265857031203, + "flos": 27673265687040.0, + "grad_norm": 2.3534880782155474, + "language_loss": 0.7567963, + "learning_rate": 1.0567342533872303e-06, + "loss": 0.77853733, + "num_input_tokens_seen": 119334875, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.17419434, + "step": 5544, + "time_per_iteration": 3.034538745880127 + }, + { + "auxiliary_loss_clip": 0.01132312, + "auxiliary_loss_mlp": 0.01046661, + "balance_loss_clip": 1.04336119, + "balance_loss_mlp": 1.03109217, + "epoch": 0.6667468285937594, + "flos": 25046831220480.0, + "grad_norm": 1.6885941590954239, + "language_loss": 0.81267905, + "learning_rate": 1.0560474316424255e-06, + "loss": 0.83446872, + "num_input_tokens_seen": 119354635, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.15588379, + "step": 5545, + "time_per_iteration": 3.0603439807891846 + }, + { + "auxiliary_loss_clip": 0.01132158, + "auxiliary_loss_mlp": 0.01046001, + "balance_loss_clip": 1.04161906, + "balance_loss_mlp": 1.02751207, + "epoch": 0.6668670714843985, + "flos": 22780472641920.0, + "grad_norm": 2.2250294067018177, + "language_loss": 0.73592156, + "learning_rate": 1.0553607530944746e-06, + "loss": 0.75770313, + "num_input_tokens_seen": 119372690, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.18469238, + "step": 5546, + "time_per_iteration": 3.006242513656616 + }, + { + "auxiliary_loss_clip": 0.01130771, + "auxiliary_loss_mlp": 0.01044067, + "balance_loss_clip": 1.04294968, + "balance_loss_mlp": 1.02846289, + "epoch": 0.6669873143750376, + "flos": 22163886754560.0, + "grad_norm": 1.9079867529866337, + "language_loss": 0.89771396, + "learning_rate": 1.0546742178475463e-06, + "loss": 0.91946232, + "num_input_tokens_seen": 119391685, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.15600586, + "step": 5547, + "time_per_iteration": 3.025380849838257 + }, + { + "auxiliary_loss_clip": 0.01125335, + "auxiliary_loss_mlp": 0.01034397, + "balance_loss_clip": 1.04169178, + "balance_loss_mlp": 1.02106929, + "epoch": 0.6671075572656767, + "flos": 20514832335360.0, + "grad_norm": 1.8637448628426518, + "language_loss": 0.86637866, + "learning_rate": 1.0539878260057868e-06, + "loss": 0.88797593, + "num_input_tokens_seen": 119410725, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.13354492, + "step": 5548, + "time_per_iteration": 3.067307472229004 + }, + { + "auxiliary_loss_clip": 0.01136904, + "auxiliary_loss_mlp": 0.01044441, + "balance_loss_clip": 1.04294467, + "balance_loss_mlp": 1.02803755, + "epoch": 0.6672278001563158, + "flos": 17931203902080.0, + "grad_norm": 2.689638357659768, + "language_loss": 0.68086624, + "learning_rate": 1.0533015776733226e-06, + "loss": 0.70267975, + "num_input_tokens_seen": 119426875, + "router_z_loss_clip": 0.93945312, + "router_z_loss_mlp": 0.1640625, + "step": 5549, + "time_per_iteration": 3.068145990371704 + }, + { + "auxiliary_loss_clip": 0.01131175, + "auxiliary_loss_mlp": 0.01048698, + "balance_loss_clip": 1.04387343, + "balance_loss_mlp": 1.03124022, + "epoch": 0.6673480430469548, + "flos": 22342146975360.0, + "grad_norm": 2.2933993675010313, + "language_loss": 0.78618491, + "learning_rate": 1.0526154729542566e-06, + "loss": 0.8079837, + "num_input_tokens_seen": 119446935, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.17449951, + "step": 5550, + "time_per_iteration": 3.0048608779907227 + }, + { + "auxiliary_loss_clip": 0.01132965, + "auxiliary_loss_mlp": 0.01044509, + "balance_loss_clip": 1.04562402, + "balance_loss_mlp": 1.02777255, + "epoch": 0.6674682859375939, + "flos": 20703830722560.0, + "grad_norm": 2.370584583872468, + "language_loss": 0.79732728, + "learning_rate": 1.0519295119526699e-06, + "loss": 0.81910205, + "num_input_tokens_seen": 119463240, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.16723633, + "step": 5551, + "time_per_iteration": 3.051328182220459 + }, + { + "auxiliary_loss_clip": 0.01131517, + "auxiliary_loss_mlp": 0.01045075, + "balance_loss_clip": 1.04354596, + "balance_loss_mlp": 1.02979207, + "epoch": 0.667588528828233, + "flos": 26206673379840.0, + "grad_norm": 1.719898582476846, + "language_loss": 0.82794648, + "learning_rate": 1.0512436947726227e-06, + "loss": 0.84971243, + "num_input_tokens_seen": 119484655, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.15283203, + "step": 5552, + "time_per_iteration": 3.011469841003418 + }, + { + "auxiliary_loss_clip": 0.01132924, + "auxiliary_loss_mlp": 0.01039027, + "balance_loss_clip": 1.04089141, + "balance_loss_mlp": 1.02206349, + "epoch": 0.6677087717188721, + "flos": 23071025756160.0, + "grad_norm": 2.5263558074253094, + "language_loss": 0.65384924, + "learning_rate": 1.0505580215181517e-06, + "loss": 0.67556882, + "num_input_tokens_seen": 119502895, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.16967773, + "step": 5553, + "time_per_iteration": 3.007939577102661 + }, + { + "auxiliary_loss_clip": 0.01035904, + "auxiliary_loss_mlp": 0.01004044, + "balance_loss_clip": 1.00814295, + "balance_loss_mlp": 1.00244617, + "epoch": 0.6678290146095112, + "flos": 70941315219840.0, + "grad_norm": 0.7727663883556251, + "language_loss": 0.56541491, + "learning_rate": 1.0498724922932753e-06, + "loss": 0.58581442, + "num_input_tokens_seen": 119561010, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01599121, + "step": 5554, + "time_per_iteration": 3.4175710678100586 + }, + { + "auxiliary_loss_clip": 0.01135013, + "auxiliary_loss_mlp": 0.01037802, + "balance_loss_clip": 1.04457593, + "balance_loss_mlp": 1.01918173, + "epoch": 0.6679492575001503, + "flos": 18661088263680.0, + "grad_norm": 1.9270997239622663, + "language_loss": 0.86704314, + "learning_rate": 1.0491871072019851e-06, + "loss": 0.88877124, + "num_input_tokens_seen": 119578900, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.1862793, + "step": 5555, + "time_per_iteration": 2.9801530838012695 + }, + { + "auxiliary_loss_clip": 0.01129487, + "auxiliary_loss_mlp": 0.01036135, + "balance_loss_clip": 1.04113042, + "balance_loss_mlp": 1.02006507, + "epoch": 0.6680695003907894, + "flos": 29711985822720.0, + "grad_norm": 1.881779134337189, + "language_loss": 0.64259398, + "learning_rate": 1.0485018663482555e-06, + "loss": 0.66425014, + "num_input_tokens_seen": 119598920, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16052246, + "step": 5556, + "time_per_iteration": 3.1337804794311523 + }, + { + "auxiliary_loss_clip": 0.01131636, + "auxiliary_loss_mlp": 0.01039192, + "balance_loss_clip": 1.0430634, + "balance_loss_mlp": 1.02145326, + "epoch": 0.6681897432814284, + "flos": 28218964083840.0, + "grad_norm": 2.3826603930140235, + "language_loss": 0.70084631, + "learning_rate": 1.0478167698360354e-06, + "loss": 0.72255456, + "num_input_tokens_seen": 119618220, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.17736816, + "step": 5557, + "time_per_iteration": 2.994680166244507 + }, + { + "auxiliary_loss_clip": 0.01128648, + "auxiliary_loss_mlp": 0.01045467, + "balance_loss_clip": 1.04213679, + "balance_loss_mlp": 1.02989852, + "epoch": 0.6683099861720676, + "flos": 25046543911680.0, + "grad_norm": 2.0488594564476936, + "language_loss": 0.69871157, + "learning_rate": 1.0471318177692556e-06, + "loss": 0.72045267, + "num_input_tokens_seen": 119638520, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15570068, + "step": 5558, + "time_per_iteration": 3.689847230911255 + }, + { + "auxiliary_loss_clip": 0.01129534, + "auxiliary_loss_mlp": 0.01041176, + "balance_loss_clip": 1.04013133, + "balance_loss_mlp": 1.02473116, + "epoch": 0.6684302290627067, + "flos": 22996977868800.0, + "grad_norm": 2.3741661761514514, + "language_loss": 0.75952816, + "learning_rate": 1.046447010251821e-06, + "loss": 0.78123522, + "num_input_tokens_seen": 119655850, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.16442871, + "step": 5559, + "time_per_iteration": 2.881779432296753 + }, + { + "auxiliary_loss_clip": 0.01127443, + "auxiliary_loss_mlp": 0.01034786, + "balance_loss_clip": 1.0412215, + "balance_loss_mlp": 1.0193485, + "epoch": 0.6685504719533457, + "flos": 26573824247040.0, + "grad_norm": 1.637492818726023, + "language_loss": 0.75581747, + "learning_rate": 1.0457623473876157e-06, + "loss": 0.77743971, + "num_input_tokens_seen": 119675355, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15429688, + "step": 5560, + "time_per_iteration": 2.963331937789917 + }, + { + "auxiliary_loss_clip": 0.0112667, + "auxiliary_loss_mlp": 0.01042816, + "balance_loss_clip": 1.04081285, + "balance_loss_mlp": 1.02714026, + "epoch": 0.6686707148439849, + "flos": 28986087870720.0, + "grad_norm": 1.7262203999744232, + "language_loss": 0.70926541, + "learning_rate": 1.0450778292805046e-06, + "loss": 0.73096031, + "num_input_tokens_seen": 119695340, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.15673828, + "step": 5561, + "time_per_iteration": 2.9975342750549316 + }, + { + "auxiliary_loss_clip": 0.01133727, + "auxiliary_loss_mlp": 0.01035783, + "balance_loss_clip": 1.0461092, + "balance_loss_mlp": 1.02065516, + "epoch": 0.6687909577346239, + "flos": 23623152687360.0, + "grad_norm": 1.5876079551629791, + "language_loss": 0.78606057, + "learning_rate": 1.0443934560343267e-06, + "loss": 0.80775559, + "num_input_tokens_seen": 119716750, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15136719, + "step": 5562, + "time_per_iteration": 3.6949257850646973 + }, + { + "auxiliary_loss_clip": 0.01127653, + "auxiliary_loss_mlp": 0.01039494, + "balance_loss_clip": 1.04127216, + "balance_loss_mlp": 1.02350807, + "epoch": 0.668911200625263, + "flos": 23148593176320.0, + "grad_norm": 2.190076740983249, + "language_loss": 0.7805987, + "learning_rate": 1.0437092277529034e-06, + "loss": 0.80227017, + "num_input_tokens_seen": 119736005, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15991211, + "step": 5563, + "time_per_iteration": 3.614830493927002 + }, + { + "auxiliary_loss_clip": 0.01125948, + "auxiliary_loss_mlp": 0.01036163, + "balance_loss_clip": 1.04022908, + "balance_loss_mlp": 1.01970053, + "epoch": 0.6690314435159022, + "flos": 18551919853440.0, + "grad_norm": 2.27774539099414, + "language_loss": 0.73902917, + "learning_rate": 1.0430251445400292e-06, + "loss": 0.76065028, + "num_input_tokens_seen": 119754050, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.16455078, + "step": 5564, + "time_per_iteration": 2.8708086013793945 + }, + { + "auxiliary_loss_clip": 0.01131405, + "auxiliary_loss_mlp": 0.01039926, + "balance_loss_clip": 1.04295361, + "balance_loss_mlp": 1.02396369, + "epoch": 0.6691516864065412, + "flos": 31759540704000.0, + "grad_norm": 2.0226667041021087, + "language_loss": 0.62262893, + "learning_rate": 1.0423412064994787e-06, + "loss": 0.64434224, + "num_input_tokens_seen": 119774820, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15966797, + "step": 5565, + "time_per_iteration": 2.9576363563537598 + }, + { + "auxiliary_loss_clip": 0.01124401, + "auxiliary_loss_mlp": 0.01037974, + "balance_loss_clip": 1.03930974, + "balance_loss_mlp": 1.02312064, + "epoch": 0.6692719292971803, + "flos": 34933864296960.0, + "grad_norm": 1.8158481656134347, + "language_loss": 0.73958576, + "learning_rate": 1.0416574137350064e-06, + "loss": 0.76120955, + "num_input_tokens_seen": 119795525, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.14868164, + "step": 5566, + "time_per_iteration": 3.0295536518096924 + }, + { + "auxiliary_loss_clip": 0.01132512, + "auxiliary_loss_mlp": 0.01035226, + "balance_loss_clip": 1.0437243, + "balance_loss_mlp": 1.02084923, + "epoch": 0.6693921721878194, + "flos": 20449188230400.0, + "grad_norm": 2.6922028369883924, + "language_loss": 0.80641097, + "learning_rate": 1.0409737663503428e-06, + "loss": 0.8280884, + "num_input_tokens_seen": 119813905, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.14379883, + "step": 5567, + "time_per_iteration": 4.0145485401153564 + }, + { + "auxiliary_loss_clip": 0.01130051, + "auxiliary_loss_mlp": 0.01036567, + "balance_loss_clip": 1.04103351, + "balance_loss_mlp": 1.01948452, + "epoch": 0.6695124150784585, + "flos": 16614538963200.0, + "grad_norm": 1.9292860941141976, + "language_loss": 0.83021122, + "learning_rate": 1.040290264449196e-06, + "loss": 0.85187733, + "num_input_tokens_seen": 119832010, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.17089844, + "step": 5568, + "time_per_iteration": 2.9654202461242676 + }, + { + "auxiliary_loss_clip": 0.01124931, + "auxiliary_loss_mlp": 0.01032187, + "balance_loss_clip": 1.0413518, + "balance_loss_mlp": 1.01672554, + "epoch": 0.6696326579690975, + "flos": 26652145852800.0, + "grad_norm": 1.9110525561014868, + "language_loss": 0.64095026, + "learning_rate": 1.0396069081352532e-06, + "loss": 0.66252148, + "num_input_tokens_seen": 119851165, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.15466309, + "step": 5569, + "time_per_iteration": 3.0746448040008545 + }, + { + "auxiliary_loss_clip": 0.01036057, + "auxiliary_loss_mlp": 0.01002226, + "balance_loss_clip": 1.00860751, + "balance_loss_mlp": 1.00058067, + "epoch": 0.6697529008597367, + "flos": 66964603662720.0, + "grad_norm": 0.7749759078124447, + "language_loss": 0.56036955, + "learning_rate": 1.0389236975121782e-06, + "loss": 0.58075237, + "num_input_tokens_seen": 119906015, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01647949, + "step": 5570, + "time_per_iteration": 3.3903512954711914 + }, + { + "auxiliary_loss_clip": 0.01133019, + "auxiliary_loss_mlp": 0.0104509, + "balance_loss_clip": 1.04297149, + "balance_loss_mlp": 1.02841234, + "epoch": 0.6698731437503758, + "flos": 20886939279360.0, + "grad_norm": 2.015832979376423, + "language_loss": 0.71508503, + "learning_rate": 1.0382406326836147e-06, + "loss": 0.73686612, + "num_input_tokens_seen": 119925160, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.16674805, + "step": 5571, + "time_per_iteration": 3.0361599922180176 + }, + { + "auxiliary_loss_clip": 0.01132069, + "auxiliary_loss_mlp": 0.01043168, + "balance_loss_clip": 1.0415262, + "balance_loss_mlp": 1.02597833, + "epoch": 0.6699933866410148, + "flos": 20409470766720.0, + "grad_norm": 1.8710095739625638, + "language_loss": 0.75408709, + "learning_rate": 1.0375577137531828e-06, + "loss": 0.77583945, + "num_input_tokens_seen": 119943720, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.17175293, + "step": 5572, + "time_per_iteration": 3.0536327362060547 + }, + { + "auxiliary_loss_clip": 0.01131226, + "auxiliary_loss_mlp": 0.01037535, + "balance_loss_clip": 1.04174995, + "balance_loss_mlp": 1.02090538, + "epoch": 0.670113629531654, + "flos": 29023075900800.0, + "grad_norm": 1.4702588803280434, + "language_loss": 0.72399062, + "learning_rate": 1.0368749408244802e-06, + "loss": 0.74567819, + "num_input_tokens_seen": 119966640, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.16625977, + "step": 5573, + "time_per_iteration": 3.1378371715545654 + }, + { + "auxiliary_loss_clip": 0.01128459, + "auxiliary_loss_mlp": 0.01042814, + "balance_loss_clip": 1.0419457, + "balance_loss_mlp": 1.02681613, + "epoch": 0.670233872422293, + "flos": 19791699730560.0, + "grad_norm": 1.7848573679635493, + "language_loss": 0.78756106, + "learning_rate": 1.0361923140010836e-06, + "loss": 0.80927378, + "num_input_tokens_seen": 119985125, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.16003418, + "step": 5574, + "time_per_iteration": 2.995030403137207 + }, + { + "auxiliary_loss_clip": 0.01132854, + "auxiliary_loss_mlp": 0.01036533, + "balance_loss_clip": 1.04314518, + "balance_loss_mlp": 1.02059484, + "epoch": 0.6703541153129321, + "flos": 24243689070720.0, + "grad_norm": 2.0088061744230714, + "language_loss": 0.6349926, + "learning_rate": 1.0355098333865455e-06, + "loss": 0.65668648, + "num_input_tokens_seen": 120004355, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.15942383, + "step": 5575, + "time_per_iteration": 2.9560797214508057 + }, + { + "auxiliary_loss_clip": 0.01128364, + "auxiliary_loss_mlp": 0.01035953, + "balance_loss_clip": 1.04264641, + "balance_loss_mlp": 1.02062809, + "epoch": 0.6704743582035713, + "flos": 26688523351680.0, + "grad_norm": 1.832689391395416, + "language_loss": 0.69053745, + "learning_rate": 1.0348274990844006e-06, + "loss": 0.71218055, + "num_input_tokens_seen": 120027115, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15325928, + "step": 5576, + "time_per_iteration": 2.9326443672180176 + }, + { + "auxiliary_loss_clip": 0.01127823, + "auxiliary_loss_mlp": 0.01035326, + "balance_loss_clip": 1.04037106, + "balance_loss_mlp": 1.01997185, + "epoch": 0.6705946010942103, + "flos": 23514379326720.0, + "grad_norm": 1.6898536425453061, + "language_loss": 0.72835135, + "learning_rate": 1.034145311198155e-06, + "loss": 0.74998289, + "num_input_tokens_seen": 120047130, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.15356445, + "step": 5577, + "time_per_iteration": 2.9757027626037598 + }, + { + "auxiliary_loss_clip": 0.01126842, + "auxiliary_loss_mlp": 0.01042336, + "balance_loss_clip": 1.04113173, + "balance_loss_mlp": 1.02708876, + "epoch": 0.6707148439848494, + "flos": 24061011477120.0, + "grad_norm": 1.6677186432841085, + "language_loss": 0.63582939, + "learning_rate": 1.0334632698312989e-06, + "loss": 0.65752113, + "num_input_tokens_seen": 120067925, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15246582, + "step": 5578, + "time_per_iteration": 3.0741326808929443 + }, + { + "auxiliary_loss_clip": 0.01128108, + "auxiliary_loss_mlp": 0.01035925, + "balance_loss_clip": 1.04115808, + "balance_loss_mlp": 1.01966441, + "epoch": 0.6708350868754885, + "flos": 22528667324160.0, + "grad_norm": 1.9222087777092318, + "language_loss": 0.75219643, + "learning_rate": 1.032781375087295e-06, + "loss": 0.77383679, + "num_input_tokens_seen": 120087825, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.16271973, + "step": 5579, + "time_per_iteration": 3.147650718688965 + }, + { + "auxiliary_loss_clip": 0.01130539, + "auxiliary_loss_mlp": 0.01042319, + "balance_loss_clip": 1.04441607, + "balance_loss_mlp": 1.02655983, + "epoch": 0.6709553297661276, + "flos": 25227749047680.0, + "grad_norm": 1.3927849471166112, + "language_loss": 0.67322969, + "learning_rate": 1.0320996270695891e-06, + "loss": 0.69495827, + "num_input_tokens_seen": 120108895, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15759277, + "step": 5580, + "time_per_iteration": 2.8836662769317627 + }, + { + "auxiliary_loss_clip": 0.01128621, + "auxiliary_loss_mlp": 0.01040731, + "balance_loss_clip": 1.04233074, + "balance_loss_mlp": 1.02544832, + "epoch": 0.6710755726567667, + "flos": 20448757267200.0, + "grad_norm": 1.8536525338905914, + "language_loss": 0.73590547, + "learning_rate": 1.0314180258815998e-06, + "loss": 0.757599, + "num_input_tokens_seen": 120127535, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.1529541, + "step": 5581, + "time_per_iteration": 2.784912347793579 + }, + { + "auxiliary_loss_clip": 0.01125217, + "auxiliary_loss_mlp": 0.0103771, + "balance_loss_clip": 1.04169989, + "balance_loss_mlp": 1.02317834, + "epoch": 0.6711958155474057, + "flos": 25995411538560.0, + "grad_norm": 1.746658421515713, + "language_loss": 0.7413643, + "learning_rate": 1.0307365716267247e-06, + "loss": 0.76299357, + "num_input_tokens_seen": 120147980, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14532471, + "step": 5582, + "time_per_iteration": 2.8453805446624756 + }, + { + "auxiliary_loss_clip": 0.01129714, + "auxiliary_loss_mlp": 0.01038046, + "balance_loss_clip": 1.04034066, + "balance_loss_mlp": 1.02305532, + "epoch": 0.6713160584380449, + "flos": 19937712516480.0, + "grad_norm": 2.422147244692785, + "language_loss": 0.78332138, + "learning_rate": 1.0300552644083423e-06, + "loss": 0.80499899, + "num_input_tokens_seen": 120166905, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.14984131, + "step": 5583, + "time_per_iteration": 2.7447662353515625 + }, + { + "auxiliary_loss_clip": 0.01134422, + "auxiliary_loss_mlp": 0.01049199, + "balance_loss_clip": 1.04215407, + "balance_loss_mlp": 1.0321641, + "epoch": 0.6714363013286839, + "flos": 18223373128320.0, + "grad_norm": 2.192060278491259, + "language_loss": 0.72419947, + "learning_rate": 1.0293741043298036e-06, + "loss": 0.7460357, + "num_input_tokens_seen": 120185255, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.17053223, + "step": 5584, + "time_per_iteration": 2.7457118034362793 + }, + { + "auxiliary_loss_clip": 0.01134455, + "auxiliary_loss_mlp": 0.01041974, + "balance_loss_clip": 1.04385805, + "balance_loss_mlp": 1.02706051, + "epoch": 0.671556544219323, + "flos": 25812374808960.0, + "grad_norm": 3.239360005167856, + "language_loss": 0.7146464, + "learning_rate": 1.0286930914944436e-06, + "loss": 0.73641068, + "num_input_tokens_seen": 120205070, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.14904785, + "step": 5585, + "time_per_iteration": 2.829847574234009 + }, + { + "auxiliary_loss_clip": 0.0112965, + "auxiliary_loss_mlp": 0.01035661, + "balance_loss_clip": 1.03970289, + "balance_loss_mlp": 1.02031922, + "epoch": 0.6716767871099621, + "flos": 15850431918720.0, + "grad_norm": 2.8913246576658946, + "language_loss": 0.77041608, + "learning_rate": 1.0280122260055684e-06, + "loss": 0.7920692, + "num_input_tokens_seen": 120220780, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.15344238, + "step": 5586, + "time_per_iteration": 2.760706901550293 + }, + { + "auxiliary_loss_clip": 0.01128597, + "auxiliary_loss_mlp": 0.01037877, + "balance_loss_clip": 1.04113781, + "balance_loss_mlp": 1.02161694, + "epoch": 0.6717970300006012, + "flos": 19756112330880.0, + "grad_norm": 2.272909306577757, + "language_loss": 0.82488859, + "learning_rate": 1.0273315079664652e-06, + "loss": 0.84655333, + "num_input_tokens_seen": 120238735, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.16247559, + "step": 5587, + "time_per_iteration": 3.54780650138855 + }, + { + "auxiliary_loss_clip": 0.01131118, + "auxiliary_loss_mlp": 0.0103769, + "balance_loss_clip": 1.04339325, + "balance_loss_mlp": 1.0231998, + "epoch": 0.6719172728912403, + "flos": 25485049146240.0, + "grad_norm": 2.1252388500426247, + "language_loss": 0.74144673, + "learning_rate": 1.0266509374803992e-06, + "loss": 0.76313484, + "num_input_tokens_seen": 120259895, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.14483643, + "step": 5588, + "time_per_iteration": 2.8497188091278076 + }, + { + "auxiliary_loss_clip": 0.01131856, + "auxiliary_loss_mlp": 0.01045292, + "balance_loss_clip": 1.04231226, + "balance_loss_mlp": 1.02885282, + "epoch": 0.6720375157818794, + "flos": 15880344969600.0, + "grad_norm": 2.4268145146997555, + "language_loss": 0.84078169, + "learning_rate": 1.0259705146506123e-06, + "loss": 0.86255312, + "num_input_tokens_seen": 120274790, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.16430664, + "step": 5589, + "time_per_iteration": 2.811723470687866 + }, + { + "auxiliary_loss_clip": 0.01128135, + "auxiliary_loss_mlp": 0.0104227, + "balance_loss_clip": 1.04117107, + "balance_loss_mlp": 1.02586627, + "epoch": 0.6721577586725185, + "flos": 32010843231360.0, + "grad_norm": 2.090823933305028, + "language_loss": 0.77680212, + "learning_rate": 1.025290239580324e-06, + "loss": 0.7985062, + "num_input_tokens_seen": 120295460, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.1640625, + "step": 5590, + "time_per_iteration": 2.8609213829040527 + }, + { + "auxiliary_loss_clip": 0.01129767, + "auxiliary_loss_mlp": 0.01039741, + "balance_loss_clip": 1.04171562, + "balance_loss_mlp": 1.02222896, + "epoch": 0.6722780015631575, + "flos": 20737873837440.0, + "grad_norm": 1.776926809911922, + "language_loss": 0.75380504, + "learning_rate": 1.0246101123727313e-06, + "loss": 0.77550012, + "num_input_tokens_seen": 120314440, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.17492676, + "step": 5591, + "time_per_iteration": 3.736415147781372 + }, + { + "auxiliary_loss_clip": 0.01126028, + "auxiliary_loss_mlp": 0.0103646, + "balance_loss_clip": 1.04128957, + "balance_loss_mlp": 1.02146375, + "epoch": 0.6723982444537967, + "flos": 16909617191040.0, + "grad_norm": 1.9502967183567657, + "language_loss": 0.78457886, + "learning_rate": 1.0239301331310085e-06, + "loss": 0.80620372, + "num_input_tokens_seen": 120332060, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.14990234, + "step": 5592, + "time_per_iteration": 3.703773021697998 + }, + { + "auxiliary_loss_clip": 0.01123766, + "auxiliary_loss_mlp": 0.01035637, + "balance_loss_clip": 1.03980243, + "balance_loss_mlp": 1.02075934, + "epoch": 0.6725184873444358, + "flos": 20667812359680.0, + "grad_norm": 1.591661750360688, + "language_loss": 0.88535857, + "learning_rate": 1.0232503019583088e-06, + "loss": 0.90695262, + "num_input_tokens_seen": 120351670, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.14886475, + "step": 5593, + "time_per_iteration": 2.9328880310058594 + }, + { + "auxiliary_loss_clip": 0.01126237, + "auxiliary_loss_mlp": 0.01046478, + "balance_loss_clip": 1.03891909, + "balance_loss_mlp": 1.02863848, + "epoch": 0.6726387302350748, + "flos": 23727616416000.0, + "grad_norm": 1.8393136297157895, + "language_loss": 0.69925797, + "learning_rate": 1.0225706189577619e-06, + "loss": 0.72098517, + "num_input_tokens_seen": 120370195, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.17852783, + "step": 5594, + "time_per_iteration": 2.873246669769287 + }, + { + "auxiliary_loss_clip": 0.01128927, + "auxiliary_loss_mlp": 0.01043308, + "balance_loss_clip": 1.04131913, + "balance_loss_mlp": 1.02699983, + "epoch": 0.672758973125714, + "flos": 15188274650880.0, + "grad_norm": 1.8894888108253407, + "language_loss": 0.74851757, + "learning_rate": 1.021891084232475e-06, + "loss": 0.77023989, + "num_input_tokens_seen": 120388130, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16308594, + "step": 5595, + "time_per_iteration": 2.8996737003326416 + }, + { + "auxiliary_loss_clip": 0.01129745, + "auxiliary_loss_mlp": 0.01040513, + "balance_loss_clip": 1.04118443, + "balance_loss_mlp": 1.02481329, + "epoch": 0.672879216016353, + "flos": 18077252601600.0, + "grad_norm": 2.019574245631008, + "language_loss": 0.79608023, + "learning_rate": 1.0212116978855325e-06, + "loss": 0.81778276, + "num_input_tokens_seen": 120406145, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.15698242, + "step": 5596, + "time_per_iteration": 3.8224635124206543 + }, + { + "auxiliary_loss_clip": 0.01125321, + "auxiliary_loss_mlp": 0.01044504, + "balance_loss_clip": 1.0388931, + "balance_loss_mlp": 1.0282433, + "epoch": 0.6729994589069921, + "flos": 23476349802240.0, + "grad_norm": 1.6327628564362537, + "language_loss": 0.7861408, + "learning_rate": 1.020532460019997e-06, + "loss": 0.80783904, + "num_input_tokens_seen": 120425395, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.16259766, + "step": 5597, + "time_per_iteration": 2.9854815006256104 + }, + { + "auxiliary_loss_clip": 0.01131763, + "auxiliary_loss_mlp": 0.01036409, + "balance_loss_clip": 1.04154718, + "balance_loss_mlp": 1.02063751, + "epoch": 0.6731197017976313, + "flos": 26322018929280.0, + "grad_norm": 1.8755877243320571, + "language_loss": 0.71242595, + "learning_rate": 1.0198533707389096e-06, + "loss": 0.73410761, + "num_input_tokens_seen": 120446270, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.15759277, + "step": 5598, + "time_per_iteration": 2.8953640460968018 + }, + { + "auxiliary_loss_clip": 0.01129143, + "auxiliary_loss_mlp": 0.01041979, + "balance_loss_clip": 1.04142964, + "balance_loss_mlp": 1.0259099, + "epoch": 0.6732399446882703, + "flos": 21616428591360.0, + "grad_norm": 1.6671773882494274, + "language_loss": 0.7315861, + "learning_rate": 1.0191744301452853e-06, + "loss": 0.75329733, + "num_input_tokens_seen": 120465570, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.1607666, + "step": 5599, + "time_per_iteration": 2.9325900077819824 + }, + { + "auxiliary_loss_clip": 0.01128904, + "auxiliary_loss_mlp": 0.01036702, + "balance_loss_clip": 1.04046917, + "balance_loss_mlp": 1.02081132, + "epoch": 0.6733601875789094, + "flos": 25880173729920.0, + "grad_norm": 3.164225128271482, + "language_loss": 0.70426631, + "learning_rate": 1.0184956383421208e-06, + "loss": 0.72592235, + "num_input_tokens_seen": 120484220, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.15905762, + "step": 5600, + "time_per_iteration": 2.855674982070923 + }, + { + "auxiliary_loss_clip": 0.01129222, + "auxiliary_loss_mlp": 0.01046569, + "balance_loss_clip": 1.0436337, + "balance_loss_mlp": 1.02971232, + "epoch": 0.6734804304695485, + "flos": 22929573997440.0, + "grad_norm": 2.1583288155088476, + "language_loss": 0.6556766, + "learning_rate": 1.017816995432387e-06, + "loss": 0.6774345, + "num_input_tokens_seen": 120503320, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.16845703, + "step": 5601, + "time_per_iteration": 2.8997113704681396 + }, + { + "auxiliary_loss_clip": 0.01128098, + "auxiliary_loss_mlp": 0.01046652, + "balance_loss_clip": 1.04163778, + "balance_loss_mlp": 1.03045177, + "epoch": 0.6736006733601876, + "flos": 18697968552960.0, + "grad_norm": 2.0778017590396423, + "language_loss": 0.74539578, + "learning_rate": 1.0171385015190353e-06, + "loss": 0.76714325, + "num_input_tokens_seen": 120523180, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.16210938, + "step": 5602, + "time_per_iteration": 2.8812804222106934 + }, + { + "auxiliary_loss_clip": 0.01125902, + "auxiliary_loss_mlp": 0.01037781, + "balance_loss_clip": 1.04025114, + "balance_loss_mlp": 1.02233195, + "epoch": 0.6737209162508266, + "flos": 19427745173760.0, + "grad_norm": 1.7902735325336125, + "language_loss": 0.72871602, + "learning_rate": 1.0164601567049908e-06, + "loss": 0.75035286, + "num_input_tokens_seen": 120541710, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15429688, + "step": 5603, + "time_per_iteration": 2.9460177421569824 + }, + { + "auxiliary_loss_clip": 0.01133042, + "auxiliary_loss_mlp": 0.01045423, + "balance_loss_clip": 1.04272842, + "balance_loss_mlp": 1.02851951, + "epoch": 0.6738411591414658, + "flos": 20158060498560.0, + "grad_norm": 1.6795555212025797, + "language_loss": 0.80291331, + "learning_rate": 1.015781961093158e-06, + "loss": 0.82469797, + "num_input_tokens_seen": 120561030, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.16900635, + "step": 5604, + "time_per_iteration": 2.9397084712982178 + }, + { + "auxiliary_loss_clip": 0.01123759, + "auxiliary_loss_mlp": 0.01039746, + "balance_loss_clip": 1.03831351, + "balance_loss_mlp": 1.02480888, + "epoch": 0.6739614020321049, + "flos": 21653847584640.0, + "grad_norm": 1.6323930521273724, + "language_loss": 0.7701757, + "learning_rate": 1.0151039147864197e-06, + "loss": 0.79181069, + "num_input_tokens_seen": 120581005, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.14953613, + "step": 5605, + "time_per_iteration": 2.8510184288024902 + }, + { + "auxiliary_loss_clip": 0.01132929, + "auxiliary_loss_mlp": 0.01043944, + "balance_loss_clip": 1.04219413, + "balance_loss_mlp": 1.02708805, + "epoch": 0.6740816449227439, + "flos": 19171702051200.0, + "grad_norm": 1.9078003298542807, + "language_loss": 0.65615231, + "learning_rate": 1.0144260178876336e-06, + "loss": 0.67792106, + "num_input_tokens_seen": 120600350, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.1685791, + "step": 5606, + "time_per_iteration": 2.908107280731201 + }, + { + "auxiliary_loss_clip": 0.01126594, + "auxiliary_loss_mlp": 0.01036353, + "balance_loss_clip": 1.03955615, + "balance_loss_mlp": 1.0199734, + "epoch": 0.6742018878133831, + "flos": 21097015971840.0, + "grad_norm": 2.221507629899264, + "language_loss": 0.67704123, + "learning_rate": 1.0137482704996388e-06, + "loss": 0.69867074, + "num_input_tokens_seen": 120614700, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.16381836, + "step": 5607, + "time_per_iteration": 2.9046499729156494 + }, + { + "auxiliary_loss_clip": 0.01131822, + "auxiliary_loss_mlp": 0.01039807, + "balance_loss_clip": 1.04290199, + "balance_loss_mlp": 1.02324939, + "epoch": 0.6743221307040221, + "flos": 23549966726400.0, + "grad_norm": 1.9046486893835415, + "language_loss": 0.79095906, + "learning_rate": 1.0130706727252461e-06, + "loss": 0.81267536, + "num_input_tokens_seen": 120631755, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.16540527, + "step": 5608, + "time_per_iteration": 3.0779802799224854 + }, + { + "auxiliary_loss_clip": 0.01127782, + "auxiliary_loss_mlp": 0.01038084, + "balance_loss_clip": 1.04081631, + "balance_loss_mlp": 1.02293253, + "epoch": 0.6744423735946612, + "flos": 16249542912000.0, + "grad_norm": 2.4197137197968206, + "language_loss": 0.68105721, + "learning_rate": 1.0123932246672468e-06, + "loss": 0.70271587, + "num_input_tokens_seen": 120645900, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15161133, + "step": 5609, + "time_per_iteration": 2.8711965084075928 + }, + { + "auxiliary_loss_clip": 0.01034381, + "auxiliary_loss_mlp": 0.01003006, + "balance_loss_clip": 1.00715041, + "balance_loss_mlp": 1.00154579, + "epoch": 0.6745626164853004, + "flos": 57843257829120.0, + "grad_norm": 0.7493672855825882, + "language_loss": 0.5577231, + "learning_rate": 1.0117159264284114e-06, + "loss": 0.57809699, + "num_input_tokens_seen": 120709070, + "router_z_loss_clip": 0.27148438, + "router_z_loss_mlp": 0.0145874, + "step": 5610, + "time_per_iteration": 3.3513870239257812 + }, + { + "auxiliary_loss_clip": 0.01126329, + "auxiliary_loss_mlp": 0.01041121, + "balance_loss_clip": 1.04099035, + "balance_loss_mlp": 1.024652, + "epoch": 0.6746828593759394, + "flos": 20485027025280.0, + "grad_norm": 1.7225988731505035, + "language_loss": 0.76830536, + "learning_rate": 1.0110387781114837e-06, + "loss": 0.78997988, + "num_input_tokens_seen": 120727685, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.16467285, + "step": 5611, + "time_per_iteration": 2.915933847427368 + }, + { + "auxiliary_loss_clip": 0.01127842, + "auxiliary_loss_mlp": 0.01036055, + "balance_loss_clip": 1.04065609, + "balance_loss_mlp": 1.02067709, + "epoch": 0.6748031022665785, + "flos": 19208223204480.0, + "grad_norm": 2.0298266576677864, + "language_loss": 0.76873136, + "learning_rate": 1.0103617798191872e-06, + "loss": 0.79037035, + "num_input_tokens_seen": 120747160, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15386963, + "step": 5612, + "time_per_iteration": 2.876068592071533 + }, + { + "auxiliary_loss_clip": 0.01132447, + "auxiliary_loss_mlp": 0.01038594, + "balance_loss_clip": 1.04232407, + "balance_loss_mlp": 1.02229786, + "epoch": 0.6749233451572175, + "flos": 15195026407680.0, + "grad_norm": 2.258705941239121, + "language_loss": 0.82552886, + "learning_rate": 1.0096849316542217e-06, + "loss": 0.84723926, + "num_input_tokens_seen": 120763710, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.16308594, + "step": 5613, + "time_per_iteration": 2.9202253818511963 + }, + { + "auxiliary_loss_clip": 0.01130239, + "auxiliary_loss_mlp": 0.0104185, + "balance_loss_clip": 1.04063606, + "balance_loss_mlp": 1.0262754, + "epoch": 0.6750435880478567, + "flos": 26499489050880.0, + "grad_norm": 2.615088303345951, + "language_loss": 0.74683917, + "learning_rate": 1.0090082337192643e-06, + "loss": 0.76856005, + "num_input_tokens_seen": 120783355, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.15570068, + "step": 5614, + "time_per_iteration": 2.884828805923462 + }, + { + "auxiliary_loss_clip": 0.01124495, + "auxiliary_loss_mlp": 0.01036491, + "balance_loss_clip": 1.03875351, + "balance_loss_mlp": 1.02076697, + "epoch": 0.6751638309384957, + "flos": 23404313076480.0, + "grad_norm": 2.011351103962171, + "language_loss": 0.78361225, + "learning_rate": 1.0083316861169705e-06, + "loss": 0.80522215, + "num_input_tokens_seen": 120802090, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15734863, + "step": 5615, + "time_per_iteration": 2.922410726547241 + }, + { + "auxiliary_loss_clip": 0.01129818, + "auxiliary_loss_mlp": 0.01039074, + "balance_loss_clip": 1.03961563, + "balance_loss_mlp": 1.02263498, + "epoch": 0.6752840738291348, + "flos": 23441408847360.0, + "grad_norm": 2.1058120417395934, + "language_loss": 0.72275251, + "learning_rate": 1.0076552889499713e-06, + "loss": 0.74444139, + "num_input_tokens_seen": 120822855, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.16442871, + "step": 5616, + "time_per_iteration": 3.6154532432556152 + }, + { + "auxiliary_loss_clip": 0.01127823, + "auxiliary_loss_mlp": 0.01039603, + "balance_loss_clip": 1.04110813, + "balance_loss_mlp": 1.02417755, + "epoch": 0.675404316719774, + "flos": 30335826257280.0, + "grad_norm": 1.829862539950188, + "language_loss": 0.74009562, + "learning_rate": 1.006979042320876e-06, + "loss": 0.76176989, + "num_input_tokens_seen": 120843070, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.1541748, + "step": 5617, + "time_per_iteration": 2.9004855155944824 + }, + { + "auxiliary_loss_clip": 0.0112584, + "auxiliary_loss_mlp": 0.01035798, + "balance_loss_clip": 1.03923631, + "balance_loss_mlp": 1.02016926, + "epoch": 0.675524559610413, + "flos": 23622613983360.0, + "grad_norm": 2.00514717263811, + "language_loss": 0.6291014, + "learning_rate": 1.0063029463322702e-06, + "loss": 0.65071774, + "num_input_tokens_seen": 120863345, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.15625, + "step": 5618, + "time_per_iteration": 2.9484939575195312 + }, + { + "auxiliary_loss_clip": 0.01129279, + "auxiliary_loss_mlp": 0.01035697, + "balance_loss_clip": 1.04046035, + "balance_loss_mlp": 1.01990807, + "epoch": 0.6756448025010521, + "flos": 21248631279360.0, + "grad_norm": 2.298340181547425, + "language_loss": 0.75230527, + "learning_rate": 1.0056270010867164e-06, + "loss": 0.77395499, + "num_input_tokens_seen": 120880915, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.15789795, + "step": 5619, + "time_per_iteration": 3.5943069458007812 + }, + { + "auxiliary_loss_clip": 0.01134008, + "auxiliary_loss_mlp": 0.01041232, + "balance_loss_clip": 1.04212391, + "balance_loss_mlp": 1.02467418, + "epoch": 0.6757650453916912, + "flos": 21646521210240.0, + "grad_norm": 2.4403637198655486, + "language_loss": 0.7861526, + "learning_rate": 1.004951206686758e-06, + "loss": 0.80790496, + "num_input_tokens_seen": 120899190, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.16564941, + "step": 5620, + "time_per_iteration": 2.8197312355041504 + }, + { + "auxiliary_loss_clip": 0.0112848, + "auxiliary_loss_mlp": 0.01033874, + "balance_loss_clip": 1.04178965, + "balance_loss_mlp": 1.01723218, + "epoch": 0.6758852882823303, + "flos": 21795658479360.0, + "grad_norm": 1.9686798766304916, + "language_loss": 0.71404266, + "learning_rate": 1.0042755632349087e-06, + "loss": 0.73566622, + "num_input_tokens_seen": 120916080, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.16638184, + "step": 5621, + "time_per_iteration": 3.646320104598999 + }, + { + "auxiliary_loss_clip": 0.01128233, + "auxiliary_loss_mlp": 0.01035542, + "balance_loss_clip": 1.04069185, + "balance_loss_mlp": 1.01856673, + "epoch": 0.6760055311729694, + "flos": 27088783580160.0, + "grad_norm": 2.0804329604709686, + "language_loss": 0.62614489, + "learning_rate": 1.0036000708336653e-06, + "loss": 0.64778268, + "num_input_tokens_seen": 120935210, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.16992188, + "step": 5622, + "time_per_iteration": 2.9524829387664795 + }, + { + "auxiliary_loss_clip": 0.01129792, + "auxiliary_loss_mlp": 0.01034332, + "balance_loss_clip": 1.04275942, + "balance_loss_mlp": 1.01935971, + "epoch": 0.6761257740636085, + "flos": 17999792922240.0, + "grad_norm": 2.525562348072416, + "language_loss": 0.79475462, + "learning_rate": 1.0029247295854984e-06, + "loss": 0.81639588, + "num_input_tokens_seen": 120951830, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.14990234, + "step": 5623, + "time_per_iteration": 2.954080104827881 + }, + { + "auxiliary_loss_clip": 0.01127886, + "auxiliary_loss_mlp": 0.01038085, + "balance_loss_clip": 1.04321992, + "balance_loss_mlp": 1.0225935, + "epoch": 0.6762460169542476, + "flos": 15121912273920.0, + "grad_norm": 1.7371769612872139, + "language_loss": 0.71789813, + "learning_rate": 1.0022495395928588e-06, + "loss": 0.73955786, + "num_input_tokens_seen": 120970310, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15484619, + "step": 5624, + "time_per_iteration": 2.874311685562134 + }, + { + "auxiliary_loss_clip": 0.01034531, + "auxiliary_loss_mlp": 0.01001168, + "balance_loss_clip": 1.00711274, + "balance_loss_mlp": 0.99973756, + "epoch": 0.6763662598448866, + "flos": 67886970030720.0, + "grad_norm": 0.7946844266028686, + "language_loss": 0.62320274, + "learning_rate": 1.0015745009581697e-06, + "loss": 0.64355969, + "num_input_tokens_seen": 121031915, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01428223, + "step": 5625, + "time_per_iteration": 4.291182994842529 + }, + { + "auxiliary_loss_clip": 0.01131559, + "auxiliary_loss_mlp": 0.01038922, + "balance_loss_clip": 1.04223549, + "balance_loss_mlp": 1.02307355, + "epoch": 0.6764865027355258, + "flos": 20631829910400.0, + "grad_norm": 1.876169267775588, + "language_loss": 0.66567171, + "learning_rate": 1.0008996137838343e-06, + "loss": 0.6873765, + "num_input_tokens_seen": 121050890, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.1585083, + "step": 5626, + "time_per_iteration": 2.91865611076355 + }, + { + "auxiliary_loss_clip": 0.0113335, + "auxiliary_loss_mlp": 0.01036091, + "balance_loss_clip": 1.04133177, + "balance_loss_mlp": 1.01915157, + "epoch": 0.6766067456261649, + "flos": 21215809226880.0, + "grad_norm": 2.1355440922893507, + "language_loss": 0.79798073, + "learning_rate": 1.000224878172234e-06, + "loss": 0.81967515, + "num_input_tokens_seen": 121070015, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.16931152, + "step": 5627, + "time_per_iteration": 2.840796947479248 + }, + { + "auxiliary_loss_clip": 0.01126344, + "auxiliary_loss_mlp": 0.01032976, + "balance_loss_clip": 1.04088879, + "balance_loss_mlp": 1.01777697, + "epoch": 0.6767269885168039, + "flos": 19938251220480.0, + "grad_norm": 1.9772951586541105, + "language_loss": 0.72924811, + "learning_rate": 9.99550294225724e-07, + "loss": 0.75084126, + "num_input_tokens_seen": 121089170, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15167236, + "step": 5628, + "time_per_iteration": 3.0858683586120605 + }, + { + "auxiliary_loss_clip": 0.01129053, + "auxiliary_loss_mlp": 0.01038158, + "balance_loss_clip": 1.04118514, + "balance_loss_mlp": 1.02270842, + "epoch": 0.6768472314074431, + "flos": 20814076540800.0, + "grad_norm": 2.269184871173939, + "language_loss": 0.73033738, + "learning_rate": 9.988758620466402e-07, + "loss": 0.75200951, + "num_input_tokens_seen": 121108040, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.15454102, + "step": 5629, + "time_per_iteration": 2.91021990776062 + }, + { + "auxiliary_loss_clip": 0.01123249, + "auxiliary_loss_mlp": 0.01036085, + "balance_loss_clip": 1.03965485, + "balance_loss_mlp": 1.02174997, + "epoch": 0.6769674742980821, + "flos": 23186012169600.0, + "grad_norm": 1.561466420374459, + "language_loss": 0.7623992, + "learning_rate": 9.982015817372917e-07, + "loss": 0.78399253, + "num_input_tokens_seen": 121128480, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14337158, + "step": 5630, + "time_per_iteration": 2.8085274696350098 + }, + { + "auxiliary_loss_clip": 0.0112815, + "auxiliary_loss_mlp": 0.01036683, + "balance_loss_clip": 1.04014301, + "balance_loss_mlp": 1.02074468, + "epoch": 0.6770877171887212, + "flos": 24242934885120.0, + "grad_norm": 2.589387610225703, + "language_loss": 0.82498813, + "learning_rate": 9.975274533999657e-07, + "loss": 0.84663641, + "num_input_tokens_seen": 121148010, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.15930176, + "step": 5631, + "time_per_iteration": 2.853027582168579 + }, + { + "auxiliary_loss_clip": 0.01131745, + "auxiliary_loss_mlp": 0.01038505, + "balance_loss_clip": 1.04072654, + "balance_loss_mlp": 1.02299631, + "epoch": 0.6772079600793603, + "flos": 18141567903360.0, + "grad_norm": 2.380504860639277, + "language_loss": 0.83958757, + "learning_rate": 9.96853477136929e-07, + "loss": 0.8612901, + "num_input_tokens_seen": 121162755, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.1550293, + "step": 5632, + "time_per_iteration": 2.8226876258850098 + }, + { + "auxiliary_loss_clip": 0.01128748, + "auxiliary_loss_mlp": 0.01040833, + "balance_loss_clip": 1.04157162, + "balance_loss_mlp": 1.02482939, + "epoch": 0.6773282029699994, + "flos": 22452069571200.0, + "grad_norm": 2.352905841853048, + "language_loss": 0.74696869, + "learning_rate": 9.96179653050422e-07, + "loss": 0.76866448, + "num_input_tokens_seen": 121182915, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.15991211, + "step": 5633, + "time_per_iteration": 2.823483943939209 + }, + { + "auxiliary_loss_clip": 0.01131041, + "auxiliary_loss_mlp": 0.01038245, + "balance_loss_clip": 1.04137754, + "balance_loss_mlp": 1.02173448, + "epoch": 0.6774484458606385, + "flos": 18693730748160.0, + "grad_norm": 2.427522561626198, + "language_loss": 0.74193251, + "learning_rate": 9.955059812426635e-07, + "loss": 0.76362538, + "num_input_tokens_seen": 121200445, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.16516113, + "step": 5634, + "time_per_iteration": 2.8130576610565186 + }, + { + "auxiliary_loss_clip": 0.0112948, + "auxiliary_loss_mlp": 0.0104013, + "balance_loss_clip": 1.0434804, + "balance_loss_mlp": 1.02476406, + "epoch": 0.6775686887512776, + "flos": 25994046821760.0, + "grad_norm": 1.830580013499677, + "language_loss": 0.83448988, + "learning_rate": 9.948324618158493e-07, + "loss": 0.85618597, + "num_input_tokens_seen": 121220785, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15344238, + "step": 5635, + "time_per_iteration": 2.9138457775115967 + }, + { + "auxiliary_loss_clip": 0.01132801, + "auxiliary_loss_mlp": 0.01039525, + "balance_loss_clip": 1.04138947, + "balance_loss_mlp": 1.02233481, + "epoch": 0.6776889316419167, + "flos": 13587987922560.0, + "grad_norm": 2.0565290191729, + "language_loss": 0.77324307, + "learning_rate": 9.941590948721502e-07, + "loss": 0.79496634, + "num_input_tokens_seen": 121237985, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.171875, + "step": 5636, + "time_per_iteration": 2.8517415523529053 + }, + { + "auxiliary_loss_clip": 0.01128282, + "auxiliary_loss_mlp": 0.01029921, + "balance_loss_clip": 1.04112494, + "balance_loss_mlp": 1.01454902, + "epoch": 0.6778091745325557, + "flos": 27601121220480.0, + "grad_norm": 1.7038456156948274, + "language_loss": 0.76148844, + "learning_rate": 9.934858805137188e-07, + "loss": 0.7830705, + "num_input_tokens_seen": 121258635, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.15362549, + "step": 5637, + "time_per_iteration": 2.944246530532837 + }, + { + "auxiliary_loss_clip": 0.01125682, + "auxiliary_loss_mlp": 0.01041618, + "balance_loss_clip": 1.04106402, + "balance_loss_mlp": 1.02690768, + "epoch": 0.6779294174231949, + "flos": 18734058743040.0, + "grad_norm": 1.5795566786517108, + "language_loss": 0.80610406, + "learning_rate": 9.92812818842677e-07, + "loss": 0.82777703, + "num_input_tokens_seen": 121277810, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.14709473, + "step": 5638, + "time_per_iteration": 2.9820008277893066 + }, + { + "auxiliary_loss_clip": 0.01127306, + "auxiliary_loss_mlp": 0.01039727, + "balance_loss_clip": 1.04033113, + "balance_loss_mlp": 1.02383649, + "epoch": 0.678049660313834, + "flos": 45873797765760.0, + "grad_norm": 1.7056315172392293, + "language_loss": 0.64182806, + "learning_rate": 9.921399099611306e-07, + "loss": 0.66349834, + "num_input_tokens_seen": 121298975, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15887451, + "step": 5639, + "time_per_iteration": 3.04471755027771 + }, + { + "auxiliary_loss_clip": 0.01126623, + "auxiliary_loss_mlp": 0.01038102, + "balance_loss_clip": 1.04067206, + "balance_loss_mlp": 1.0227834, + "epoch": 0.678169903204473, + "flos": 19974556892160.0, + "grad_norm": 1.8030915875515858, + "language_loss": 0.6894567, + "learning_rate": 9.914671539711588e-07, + "loss": 0.71110392, + "num_input_tokens_seen": 121318495, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.15313721, + "step": 5640, + "time_per_iteration": 3.020815134048462 + }, + { + "auxiliary_loss_clip": 0.01129244, + "auxiliary_loss_mlp": 0.01039717, + "balance_loss_clip": 1.04019427, + "balance_loss_mlp": 1.02252674, + "epoch": 0.6782901460951122, + "flos": 21395613732480.0, + "grad_norm": 2.0346522238643807, + "language_loss": 0.78290808, + "learning_rate": 9.90794550974817e-07, + "loss": 0.80459774, + "num_input_tokens_seen": 121338890, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.17211914, + "step": 5641, + "time_per_iteration": 2.9153847694396973 + }, + { + "auxiliary_loss_clip": 0.01130764, + "auxiliary_loss_mlp": 0.01038907, + "balance_loss_clip": 1.04181886, + "balance_loss_mlp": 1.02332616, + "epoch": 0.6784103889857512, + "flos": 21434002392960.0, + "grad_norm": 1.957021103529582, + "language_loss": 0.81464422, + "learning_rate": 9.901221010741407e-07, + "loss": 0.8363409, + "num_input_tokens_seen": 121358210, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.15588379, + "step": 5642, + "time_per_iteration": 2.9336302280426025 + }, + { + "auxiliary_loss_clip": 0.01129851, + "auxiliary_loss_mlp": 0.01042621, + "balance_loss_clip": 1.04236722, + "balance_loss_mlp": 1.02711761, + "epoch": 0.6785306318763903, + "flos": 32671923091200.0, + "grad_norm": 2.1552043521694304, + "language_loss": 0.74876243, + "learning_rate": 9.894498043711375e-07, + "loss": 0.77048719, + "num_input_tokens_seen": 121379955, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.15496826, + "step": 5643, + "time_per_iteration": 2.9999866485595703 + }, + { + "auxiliary_loss_clip": 0.01125605, + "auxiliary_loss_mlp": 0.01038897, + "balance_loss_clip": 1.04012334, + "balance_loss_mlp": 1.0229286, + "epoch": 0.6786508747670293, + "flos": 25632139340160.0, + "grad_norm": 1.835984964628557, + "language_loss": 0.69408739, + "learning_rate": 9.887776609677962e-07, + "loss": 0.71573234, + "num_input_tokens_seen": 121401325, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.159729, + "step": 5644, + "time_per_iteration": 3.001326084136963 + }, + { + "auxiliary_loss_clip": 0.01126925, + "auxiliary_loss_mlp": 0.01041269, + "balance_loss_clip": 1.04080725, + "balance_loss_mlp": 1.02623081, + "epoch": 0.6787711176576685, + "flos": 19171881619200.0, + "grad_norm": 1.6152140523825402, + "language_loss": 0.72115231, + "learning_rate": 9.88105670966079e-07, + "loss": 0.74283421, + "num_input_tokens_seen": 121419785, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.15045166, + "step": 5645, + "time_per_iteration": 3.6427316665649414 + }, + { + "auxiliary_loss_clip": 0.01125068, + "auxiliary_loss_mlp": 0.01038021, + "balance_loss_clip": 1.0399909, + "balance_loss_mlp": 1.02263117, + "epoch": 0.6788913605483076, + "flos": 13985159581440.0, + "grad_norm": 1.966807945252756, + "language_loss": 0.79158235, + "learning_rate": 9.874338344679283e-07, + "loss": 0.81321329, + "num_input_tokens_seen": 121435630, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15405273, + "step": 5646, + "time_per_iteration": 2.8933866024017334 + }, + { + "auxiliary_loss_clip": 0.01125734, + "auxiliary_loss_mlp": 0.0103751, + "balance_loss_clip": 1.04114914, + "balance_loss_mlp": 1.02331233, + "epoch": 0.6790116034389466, + "flos": 22017586659840.0, + "grad_norm": 1.7089038606182436, + "language_loss": 0.73897797, + "learning_rate": 9.86762151575259e-07, + "loss": 0.7606104, + "num_input_tokens_seen": 121455625, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.14190674, + "step": 5647, + "time_per_iteration": 2.9461658000946045 + }, + { + "auxiliary_loss_clip": 0.01122576, + "auxiliary_loss_mlp": 0.0103875, + "balance_loss_clip": 1.04053426, + "balance_loss_mlp": 1.02482641, + "epoch": 0.6791318463295858, + "flos": 20922454851840.0, + "grad_norm": 1.444621314552804, + "language_loss": 0.80286849, + "learning_rate": 9.860906223899651e-07, + "loss": 0.82448173, + "num_input_tokens_seen": 121475020, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.13922119, + "step": 5648, + "time_per_iteration": 3.801664113998413 + }, + { + "auxiliary_loss_clip": 0.01126866, + "auxiliary_loss_mlp": 0.01039655, + "balance_loss_clip": 1.04040623, + "balance_loss_mlp": 1.02486157, + "epoch": 0.6792520892202248, + "flos": 28512749422080.0, + "grad_norm": 1.5336843366367034, + "language_loss": 0.75541693, + "learning_rate": 9.854192470139184e-07, + "loss": 0.77708215, + "num_input_tokens_seen": 121496500, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.14788818, + "step": 5649, + "time_per_iteration": 3.043079137802124 + }, + { + "auxiliary_loss_clip": 0.01130836, + "auxiliary_loss_mlp": 0.01040333, + "balance_loss_clip": 1.04349458, + "balance_loss_mlp": 1.02465653, + "epoch": 0.6793723321108639, + "flos": 20011904058240.0, + "grad_norm": 4.88784757088703, + "language_loss": 0.72282374, + "learning_rate": 9.847480255489645e-07, + "loss": 0.74453533, + "num_input_tokens_seen": 121515525, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15686035, + "step": 5650, + "time_per_iteration": 3.825018882751465 + }, + { + "auxiliary_loss_clip": 0.01129048, + "auxiliary_loss_mlp": 0.0104514, + "balance_loss_clip": 1.04131174, + "balance_loss_mlp": 1.02858233, + "epoch": 0.6794925750015031, + "flos": 26649488246400.0, + "grad_norm": 1.7676009979948852, + "language_loss": 0.6879251, + "learning_rate": 9.840769580969295e-07, + "loss": 0.70966697, + "num_input_tokens_seen": 121535965, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.16564941, + "step": 5651, + "time_per_iteration": 2.911681890487671 + }, + { + "auxiliary_loss_clip": 0.01128943, + "auxiliary_loss_mlp": 0.01038026, + "balance_loss_clip": 1.04196298, + "balance_loss_mlp": 1.02158713, + "epoch": 0.6796128178921421, + "flos": 21580374314880.0, + "grad_norm": 2.167892758855469, + "language_loss": 0.79661715, + "learning_rate": 9.834060447596114e-07, + "loss": 0.8182869, + "num_input_tokens_seen": 121555235, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.16430664, + "step": 5652, + "time_per_iteration": 2.881915807723999 + }, + { + "auxiliary_loss_clip": 0.01128727, + "auxiliary_loss_mlp": 0.01036399, + "balance_loss_clip": 1.04050398, + "balance_loss_mlp": 1.02046084, + "epoch": 0.6797330607827812, + "flos": 22492002516480.0, + "grad_norm": 1.9394834898766213, + "language_loss": 0.78265512, + "learning_rate": 9.827352856387868e-07, + "loss": 0.80430639, + "num_input_tokens_seen": 121574945, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15930176, + "step": 5653, + "time_per_iteration": 2.874063730239868 + }, + { + "auxiliary_loss_clip": 0.01035005, + "auxiliary_loss_mlp": 0.01005434, + "balance_loss_clip": 1.00740075, + "balance_loss_mlp": 1.0039916, + "epoch": 0.6798533036734203, + "flos": 66306648286080.0, + "grad_norm": 0.7850956482147469, + "language_loss": 0.64261234, + "learning_rate": 9.820646808362118e-07, + "loss": 0.6630168, + "num_input_tokens_seen": 121641200, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.0144043, + "step": 5654, + "time_per_iteration": 4.308752059936523 + }, + { + "auxiliary_loss_clip": 0.01127465, + "auxiliary_loss_mlp": 0.01042831, + "balance_loss_clip": 1.0424149, + "balance_loss_mlp": 1.02623749, + "epoch": 0.6799735465640594, + "flos": 16180163792640.0, + "grad_norm": 2.2236730220514445, + "language_loss": 0.7291671, + "learning_rate": 9.813942304536154e-07, + "loss": 0.75087011, + "num_input_tokens_seen": 121659170, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.16601562, + "step": 5655, + "time_per_iteration": 2.9340903759002686 + }, + { + "auxiliary_loss_clip": 0.01128751, + "auxiliary_loss_mlp": 0.01037111, + "balance_loss_clip": 1.04154634, + "balance_loss_mlp": 1.02195358, + "epoch": 0.6800937894546984, + "flos": 22125749489280.0, + "grad_norm": 1.6277494979587812, + "language_loss": 0.63667625, + "learning_rate": 9.807239345927043e-07, + "loss": 0.65833485, + "num_input_tokens_seen": 121679180, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.15167236, + "step": 5656, + "time_per_iteration": 2.9038379192352295 + }, + { + "auxiliary_loss_clip": 0.01126349, + "auxiliary_loss_mlp": 0.01038317, + "balance_loss_clip": 1.03895891, + "balance_loss_mlp": 1.02379751, + "epoch": 0.6802140323453376, + "flos": 31612953300480.0, + "grad_norm": 2.108499330422323, + "language_loss": 0.722287, + "learning_rate": 9.80053793355162e-07, + "loss": 0.74393368, + "num_input_tokens_seen": 121697875, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.14550781, + "step": 5657, + "time_per_iteration": 2.8918118476867676 + }, + { + "auxiliary_loss_clip": 0.01129079, + "auxiliary_loss_mlp": 0.0103939, + "balance_loss_clip": 1.04083037, + "balance_loss_mlp": 1.0239768, + "epoch": 0.6803342752359767, + "flos": 17712938908800.0, + "grad_norm": 2.164234922533599, + "language_loss": 0.75053471, + "learning_rate": 9.793838068426472e-07, + "loss": 0.77221942, + "num_input_tokens_seen": 121715570, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15405273, + "step": 5658, + "time_per_iteration": 2.9044082164764404 + }, + { + "auxiliary_loss_clip": 0.01126822, + "auxiliary_loss_mlp": 0.0103566, + "balance_loss_clip": 1.04053354, + "balance_loss_mlp": 1.0191499, + "epoch": 0.6804545181266157, + "flos": 11326800902400.0, + "grad_norm": 2.5162608373325908, + "language_loss": 0.61264241, + "learning_rate": 9.78713975156799e-07, + "loss": 0.63426721, + "num_input_tokens_seen": 121731435, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.16516113, + "step": 5659, + "time_per_iteration": 2.897101402282715 + }, + { + "auxiliary_loss_clip": 0.01132205, + "auxiliary_loss_mlp": 0.0104267, + "balance_loss_clip": 1.04314542, + "balance_loss_mlp": 1.02670836, + "epoch": 0.6805747610172549, + "flos": 29350976181120.0, + "grad_norm": 1.695217361922263, + "language_loss": 0.71783102, + "learning_rate": 9.780442983992273e-07, + "loss": 0.73957974, + "num_input_tokens_seen": 121749950, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.1595459, + "step": 5660, + "time_per_iteration": 2.9183356761932373 + }, + { + "auxiliary_loss_clip": 0.01132116, + "auxiliary_loss_mlp": 0.01037222, + "balance_loss_clip": 1.04303694, + "balance_loss_mlp": 1.02097416, + "epoch": 0.680695003907894, + "flos": 37631868612480.0, + "grad_norm": 1.7521610082745946, + "language_loss": 0.71742398, + "learning_rate": 9.773747766715238e-07, + "loss": 0.73911738, + "num_input_tokens_seen": 121770770, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.16247559, + "step": 5661, + "time_per_iteration": 3.062041759490967 + }, + { + "auxiliary_loss_clip": 0.01130657, + "auxiliary_loss_mlp": 0.01037742, + "balance_loss_clip": 1.04253662, + "balance_loss_mlp": 1.02089739, + "epoch": 0.680815246798533, + "flos": 22127365601280.0, + "grad_norm": 2.173982404286858, + "language_loss": 0.80139494, + "learning_rate": 9.767054100752536e-07, + "loss": 0.82307887, + "num_input_tokens_seen": 121790720, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.16845703, + "step": 5662, + "time_per_iteration": 2.937814712524414 + }, + { + "auxiliary_loss_clip": 0.01130525, + "auxiliary_loss_mlp": 0.0103866, + "balance_loss_clip": 1.04280365, + "balance_loss_mlp": 1.02228069, + "epoch": 0.6809354896891722, + "flos": 17201822330880.0, + "grad_norm": 2.548842012922472, + "language_loss": 0.81563926, + "learning_rate": 9.760361987119584e-07, + "loss": 0.83733112, + "num_input_tokens_seen": 121808455, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.16369629, + "step": 5663, + "time_per_iteration": 2.868295431137085 + }, + { + "auxiliary_loss_clip": 0.01128025, + "auxiliary_loss_mlp": 0.01041973, + "balance_loss_clip": 1.03952837, + "balance_loss_mlp": 1.02494955, + "epoch": 0.6810557325798112, + "flos": 12458166554880.0, + "grad_norm": 2.2379818880528597, + "language_loss": 0.68157178, + "learning_rate": 9.753671426831592e-07, + "loss": 0.70327181, + "num_input_tokens_seen": 121824470, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.17028809, + "step": 5664, + "time_per_iteration": 2.8854753971099854 + }, + { + "auxiliary_loss_clip": 0.01126142, + "auxiliary_loss_mlp": 0.01039966, + "balance_loss_clip": 1.03989625, + "balance_loss_mlp": 1.02356243, + "epoch": 0.6811759754704503, + "flos": 22156165330560.0, + "grad_norm": 4.730228648223378, + "language_loss": 0.79764479, + "learning_rate": 9.746982420903483e-07, + "loss": 0.8193059, + "num_input_tokens_seen": 121842665, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.16381836, + "step": 5665, + "time_per_iteration": 2.9951183795928955 + }, + { + "auxiliary_loss_clip": 0.01128127, + "auxiliary_loss_mlp": 0.01036982, + "balance_loss_clip": 1.04158187, + "balance_loss_mlp": 1.0220567, + "epoch": 0.6812962183610894, + "flos": 17525377065600.0, + "grad_norm": 1.6256238168966157, + "language_loss": 0.74604207, + "learning_rate": 9.740294970349993e-07, + "loss": 0.76769316, + "num_input_tokens_seen": 121859080, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.14929199, + "step": 5666, + "time_per_iteration": 2.9587907791137695 + }, + { + "auxiliary_loss_clip": 0.01035914, + "auxiliary_loss_mlp": 0.01002666, + "balance_loss_clip": 1.00835693, + "balance_loss_mlp": 1.00122392, + "epoch": 0.6814164612517285, + "flos": 60274480855680.0, + "grad_norm": 0.8978825519795967, + "language_loss": 0.60944939, + "learning_rate": 9.733609076185594e-07, + "loss": 0.62983519, + "num_input_tokens_seen": 121915485, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.0144043, + "step": 5667, + "time_per_iteration": 3.2325363159179688 + }, + { + "auxiliary_loss_clip": 0.01131132, + "auxiliary_loss_mlp": 0.01039903, + "balance_loss_clip": 1.04288697, + "balance_loss_mlp": 1.02459693, + "epoch": 0.6815367041423676, + "flos": 19317750750720.0, + "grad_norm": 1.9413691165738065, + "language_loss": 0.84088671, + "learning_rate": 9.72692473942455e-07, + "loss": 0.86259711, + "num_input_tokens_seen": 121932710, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.15307617, + "step": 5668, + "time_per_iteration": 3.0056302547454834 + }, + { + "auxiliary_loss_clip": 0.01134703, + "auxiliary_loss_mlp": 0.01039835, + "balance_loss_clip": 1.04378867, + "balance_loss_mlp": 1.02328837, + "epoch": 0.6816569470330067, + "flos": 22161696024960.0, + "grad_norm": 1.5145567498842825, + "language_loss": 0.77325207, + "learning_rate": 9.720241961080849e-07, + "loss": 0.79499745, + "num_input_tokens_seen": 121952025, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.16540527, + "step": 5669, + "time_per_iteration": 2.9054930210113525 + }, + { + "auxiliary_loss_clip": 0.01126904, + "auxiliary_loss_mlp": 0.01033622, + "balance_loss_clip": 1.04017758, + "balance_loss_mlp": 1.01867294, + "epoch": 0.6817771899236458, + "flos": 41463501137280.0, + "grad_norm": 1.8799850998889498, + "language_loss": 0.73104542, + "learning_rate": 9.713560742168259e-07, + "loss": 0.75265068, + "num_input_tokens_seen": 121974650, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.1496582, + "step": 5670, + "time_per_iteration": 3.0246036052703857 + }, + { + "auxiliary_loss_clip": 0.01125772, + "auxiliary_loss_mlp": 0.01037527, + "balance_loss_clip": 1.04059243, + "balance_loss_mlp": 1.02224445, + "epoch": 0.6818974328142848, + "flos": 21106138026240.0, + "grad_norm": 1.8490046395503856, + "language_loss": 0.71378583, + "learning_rate": 9.706881083700333e-07, + "loss": 0.7354188, + "num_input_tokens_seen": 121994335, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.1529541, + "step": 5671, + "time_per_iteration": 2.9562840461730957 + }, + { + "auxiliary_loss_clip": 0.01132405, + "auxiliary_loss_mlp": 0.01042945, + "balance_loss_clip": 1.04339528, + "balance_loss_mlp": 1.02688742, + "epoch": 0.682017675704924, + "flos": 20441897769600.0, + "grad_norm": 2.273487676477525, + "language_loss": 0.82332575, + "learning_rate": 9.700202986690357e-07, + "loss": 0.8450793, + "num_input_tokens_seen": 122012635, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.16040039, + "step": 5672, + "time_per_iteration": 2.8575854301452637 + }, + { + "auxiliary_loss_clip": 0.01129176, + "auxiliary_loss_mlp": 0.01045485, + "balance_loss_clip": 1.04096317, + "balance_loss_mlp": 1.02706718, + "epoch": 0.682137918595563, + "flos": 20044438801920.0, + "grad_norm": 1.7828685210772257, + "language_loss": 0.66263807, + "learning_rate": 9.693526452151413e-07, + "loss": 0.68438458, + "num_input_tokens_seen": 122031685, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.18444824, + "step": 5673, + "time_per_iteration": 3.6324615478515625 + }, + { + "auxiliary_loss_clip": 0.01133108, + "auxiliary_loss_mlp": 0.0103974, + "balance_loss_clip": 1.04225338, + "balance_loss_mlp": 1.02291942, + "epoch": 0.6822581614862021, + "flos": 31684559063040.0, + "grad_norm": 3.1773102026589304, + "language_loss": 0.75660574, + "learning_rate": 9.686851481096305e-07, + "loss": 0.77833432, + "num_input_tokens_seen": 122052995, + "router_z_loss_clip": 0.90771484, + "router_z_loss_mlp": 0.16821289, + "step": 5674, + "time_per_iteration": 2.938783645629883 + }, + { + "auxiliary_loss_clip": 0.01127234, + "auxiliary_loss_mlp": 0.01035086, + "balance_loss_clip": 1.04002619, + "balance_loss_mlp": 1.01906419, + "epoch": 0.6823784043768413, + "flos": 23477570864640.0, + "grad_norm": 2.145337048017663, + "language_loss": 0.71994436, + "learning_rate": 9.68017807453762e-07, + "loss": 0.74156755, + "num_input_tokens_seen": 122071740, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.16003418, + "step": 5675, + "time_per_iteration": 2.8968923091888428 + }, + { + "auxiliary_loss_clip": 0.01128898, + "auxiliary_loss_mlp": 0.01036785, + "balance_loss_clip": 1.04252076, + "balance_loss_mlp": 1.02163386, + "epoch": 0.6824986472674803, + "flos": 14137134024960.0, + "grad_norm": 1.6271621615344223, + "language_loss": 0.73272121, + "learning_rate": 9.673506233487721e-07, + "loss": 0.75437808, + "num_input_tokens_seen": 122089705, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15161133, + "step": 5676, + "time_per_iteration": 2.8091788291931152 + }, + { + "auxiliary_loss_clip": 0.01125096, + "auxiliary_loss_mlp": 0.01029058, + "balance_loss_clip": 1.04188037, + "balance_loss_mlp": 1.01456809, + "epoch": 0.6826188901581194, + "flos": 21504997624320.0, + "grad_norm": 1.848651759101762, + "language_loss": 0.86171043, + "learning_rate": 9.666835958958717e-07, + "loss": 0.88325202, + "num_input_tokens_seen": 122109025, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.14483643, + "step": 5677, + "time_per_iteration": 3.7036144733428955 + }, + { + "auxiliary_loss_clip": 0.01128033, + "auxiliary_loss_mlp": 0.01038908, + "balance_loss_clip": 1.04225898, + "balance_loss_mlp": 1.02320862, + "epoch": 0.6827391330487584, + "flos": 20810126044800.0, + "grad_norm": 3.6410504658304412, + "language_loss": 0.81149906, + "learning_rate": 9.660167251962484e-07, + "loss": 0.83316851, + "num_input_tokens_seen": 122127385, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15710449, + "step": 5678, + "time_per_iteration": 2.8251147270202637 + }, + { + "auxiliary_loss_clip": 0.01125875, + "auxiliary_loss_mlp": 0.01039051, + "balance_loss_clip": 1.04195714, + "balance_loss_mlp": 1.02417374, + "epoch": 0.6828593759393976, + "flos": 21688788539520.0, + "grad_norm": 1.5437905581851088, + "language_loss": 0.77838427, + "learning_rate": 9.653500113510654e-07, + "loss": 0.80003357, + "num_input_tokens_seen": 122146500, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.14886475, + "step": 5679, + "time_per_iteration": 3.678316116333008 + }, + { + "auxiliary_loss_clip": 0.01128043, + "auxiliary_loss_mlp": 0.0104665, + "balance_loss_clip": 1.0411768, + "balance_loss_mlp": 1.02987695, + "epoch": 0.6829796188300367, + "flos": 25337707557120.0, + "grad_norm": 2.4032025449529906, + "language_loss": 0.67254531, + "learning_rate": 9.646834544614627e-07, + "loss": 0.69429225, + "num_input_tokens_seen": 122167000, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.16772461, + "step": 5680, + "time_per_iteration": 2.8751754760742188 + }, + { + "auxiliary_loss_clip": 0.01128362, + "auxiliary_loss_mlp": 0.01036907, + "balance_loss_clip": 1.04200995, + "balance_loss_mlp": 1.02017021, + "epoch": 0.6830998617206757, + "flos": 20704800389760.0, + "grad_norm": 1.8308546841995936, + "language_loss": 0.76209062, + "learning_rate": 9.64017054628558e-07, + "loss": 0.78374326, + "num_input_tokens_seen": 122185825, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.1673584, + "step": 5681, + "time_per_iteration": 2.921802043914795 + }, + { + "auxiliary_loss_clip": 0.01128113, + "auxiliary_loss_mlp": 0.01039087, + "balance_loss_clip": 1.03916085, + "balance_loss_mlp": 1.0235064, + "epoch": 0.6832201046113149, + "flos": 21726638496000.0, + "grad_norm": 1.6209654362478298, + "language_loss": 0.79329646, + "learning_rate": 9.63350811953441e-07, + "loss": 0.81496835, + "num_input_tokens_seen": 122206200, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.15563965, + "step": 5682, + "time_per_iteration": 2.882430076599121 + }, + { + "auxiliary_loss_clip": 0.01128246, + "auxiliary_loss_mlp": 0.01035476, + "balance_loss_clip": 1.04038262, + "balance_loss_mlp": 1.01919174, + "epoch": 0.6833403475019539, + "flos": 19536554448000.0, + "grad_norm": 6.776739574508274, + "language_loss": 0.70816123, + "learning_rate": 9.626847265371826e-07, + "loss": 0.72979844, + "num_input_tokens_seen": 122225520, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.16296387, + "step": 5683, + "time_per_iteration": 3.7584447860717773 + }, + { + "auxiliary_loss_clip": 0.01127959, + "auxiliary_loss_mlp": 0.01041408, + "balance_loss_clip": 1.0406847, + "balance_loss_mlp": 1.02579117, + "epoch": 0.683460590392593, + "flos": 19352153001600.0, + "grad_norm": 1.8578744506594005, + "language_loss": 0.78872728, + "learning_rate": 9.620187984808262e-07, + "loss": 0.81042099, + "num_input_tokens_seen": 122244320, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.15612793, + "step": 5684, + "time_per_iteration": 2.858337879180908 + }, + { + "auxiliary_loss_clip": 0.01128424, + "auxiliary_loss_mlp": 0.0103872, + "balance_loss_clip": 1.04129159, + "balance_loss_mlp": 1.02319896, + "epoch": 0.6835808332832322, + "flos": 23288500650240.0, + "grad_norm": 2.3344370474942227, + "language_loss": 0.86168206, + "learning_rate": 9.613530278853919e-07, + "loss": 0.88335347, + "num_input_tokens_seen": 122264295, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.15527344, + "step": 5685, + "time_per_iteration": 2.969317674636841 + }, + { + "auxiliary_loss_clip": 0.01128397, + "auxiliary_loss_mlp": 0.01039166, + "balance_loss_clip": 1.04068005, + "balance_loss_mlp": 1.02366829, + "epoch": 0.6837010761738712, + "flos": 21653416621440.0, + "grad_norm": 1.6661680176637528, + "language_loss": 0.74046797, + "learning_rate": 9.60687414851879e-07, + "loss": 0.76214361, + "num_input_tokens_seen": 122285300, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.15490723, + "step": 5686, + "time_per_iteration": 2.844420909881592 + }, + { + "auxiliary_loss_clip": 0.01129973, + "auxiliary_loss_mlp": 0.01043877, + "balance_loss_clip": 1.04183602, + "balance_loss_mlp": 1.02868354, + "epoch": 0.6838213190645103, + "flos": 17566387418880.0, + "grad_norm": 2.3406848089921377, + "language_loss": 0.77788466, + "learning_rate": 9.600219594812575e-07, + "loss": 0.79962313, + "num_input_tokens_seen": 122303240, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.15209961, + "step": 5687, + "time_per_iteration": 2.923407793045044 + }, + { + "auxiliary_loss_clip": 0.01124663, + "auxiliary_loss_mlp": 0.01032289, + "balance_loss_clip": 1.04050779, + "balance_loss_mlp": 1.01831806, + "epoch": 0.6839415619551494, + "flos": 23112538899840.0, + "grad_norm": 1.5706536765116839, + "language_loss": 0.72849953, + "learning_rate": 9.593566618744786e-07, + "loss": 0.75006908, + "num_input_tokens_seen": 122323390, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.13977051, + "step": 5688, + "time_per_iteration": 2.9130470752716064 + }, + { + "auxiliary_loss_clip": 0.01128902, + "auxiliary_loss_mlp": 0.01037562, + "balance_loss_clip": 1.04227126, + "balance_loss_mlp": 1.02242255, + "epoch": 0.6840618048457885, + "flos": 22127868391680.0, + "grad_norm": 1.7209853118705047, + "language_loss": 0.73727822, + "learning_rate": 9.58691522132466e-07, + "loss": 0.75894284, + "num_input_tokens_seen": 122342200, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.15148926, + "step": 5689, + "time_per_iteration": 2.864476203918457 + }, + { + "auxiliary_loss_clip": 0.01132241, + "auxiliary_loss_mlp": 0.01038524, + "balance_loss_clip": 1.04184151, + "balance_loss_mlp": 1.02278852, + "epoch": 0.6841820477364275, + "flos": 22015898720640.0, + "grad_norm": 1.9933102245221432, + "language_loss": 0.84884846, + "learning_rate": 9.58026540356123e-07, + "loss": 0.87055618, + "num_input_tokens_seen": 122360465, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.1574707, + "step": 5690, + "time_per_iteration": 2.8472042083740234 + }, + { + "auxiliary_loss_clip": 0.01130565, + "auxiliary_loss_mlp": 0.01040627, + "balance_loss_clip": 1.04337239, + "balance_loss_mlp": 1.02484965, + "epoch": 0.6843022906270667, + "flos": 24900531125760.0, + "grad_norm": 1.8139552474661507, + "language_loss": 0.86588693, + "learning_rate": 9.573617166463246e-07, + "loss": 0.88759887, + "num_input_tokens_seen": 122381680, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.15789795, + "step": 5691, + "time_per_iteration": 2.8321619033813477 + }, + { + "auxiliary_loss_clip": 0.01128613, + "auxiliary_loss_mlp": 0.01037638, + "balance_loss_clip": 1.04173195, + "balance_loss_mlp": 1.02159262, + "epoch": 0.6844225335177058, + "flos": 19969924037760.0, + "grad_norm": 2.7299820153275642, + "language_loss": 0.6013574, + "learning_rate": 9.56697051103924e-07, + "loss": 0.62301993, + "num_input_tokens_seen": 122399120, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.16052246, + "step": 5692, + "time_per_iteration": 2.825563907623291 + }, + { + "auxiliary_loss_clip": 0.01124073, + "auxiliary_loss_mlp": 0.0103909, + "balance_loss_clip": 1.03994095, + "balance_loss_mlp": 1.02403343, + "epoch": 0.6845427764083448, + "flos": 25883334126720.0, + "grad_norm": 1.8934273592674955, + "language_loss": 0.81117964, + "learning_rate": 9.560325438297522e-07, + "loss": 0.83281124, + "num_input_tokens_seen": 122417430, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.1505127, + "step": 5693, + "time_per_iteration": 2.751969337463379 + }, + { + "auxiliary_loss_clip": 0.01131384, + "auxiliary_loss_mlp": 0.01039743, + "balance_loss_clip": 1.04304707, + "balance_loss_mlp": 1.02356637, + "epoch": 0.684663019298984, + "flos": 18880143356160.0, + "grad_norm": 1.9032907074460197, + "language_loss": 0.86510587, + "learning_rate": 9.553681949246127e-07, + "loss": 0.88681716, + "num_input_tokens_seen": 122435055, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16174316, + "step": 5694, + "time_per_iteration": 2.796839952468872 + }, + { + "auxiliary_loss_clip": 0.01134516, + "auxiliary_loss_mlp": 0.01043345, + "balance_loss_clip": 1.04350495, + "balance_loss_mlp": 1.02688193, + "epoch": 0.684783262189623, + "flos": 54193725302400.0, + "grad_norm": 1.89569335184651, + "language_loss": 0.75652802, + "learning_rate": 9.547040044892886e-07, + "loss": 0.7783066, + "num_input_tokens_seen": 122462570, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.16455078, + "step": 5695, + "time_per_iteration": 3.035703420639038 + }, + { + "auxiliary_loss_clip": 0.01036033, + "auxiliary_loss_mlp": 0.01001203, + "balance_loss_clip": 1.0085094, + "balance_loss_mlp": 0.99970073, + "epoch": 0.6849035050802621, + "flos": 63970264143360.0, + "grad_norm": 1.291827906903767, + "language_loss": 0.60270411, + "learning_rate": 9.540399726245354e-07, + "loss": 0.6230765, + "num_input_tokens_seen": 122519275, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01501465, + "step": 5696, + "time_per_iteration": 3.1769163608551025 + }, + { + "auxiliary_loss_clip": 0.01132734, + "auxiliary_loss_mlp": 0.01037167, + "balance_loss_clip": 1.04225516, + "balance_loss_mlp": 1.02131259, + "epoch": 0.6850237479709013, + "flos": 25224121774080.0, + "grad_norm": 1.7600788264243385, + "language_loss": 0.68804741, + "learning_rate": 9.533760994310859e-07, + "loss": 0.70974642, + "num_input_tokens_seen": 122539675, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.15844727, + "step": 5697, + "time_per_iteration": 2.8700194358825684 + }, + { + "auxiliary_loss_clip": 0.01130863, + "auxiliary_loss_mlp": 0.01036057, + "balance_loss_clip": 1.04366493, + "balance_loss_mlp": 1.02029717, + "epoch": 0.6851439908615403, + "flos": 19354128249600.0, + "grad_norm": 1.9779602710647504, + "language_loss": 0.75187522, + "learning_rate": 9.527123850096508e-07, + "loss": 0.77354443, + "num_input_tokens_seen": 122558035, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15771484, + "step": 5698, + "time_per_iteration": 2.727508068084717 + }, + { + "auxiliary_loss_clip": 0.01127495, + "auxiliary_loss_mlp": 0.0103527, + "balance_loss_clip": 1.04034996, + "balance_loss_mlp": 1.02034497, + "epoch": 0.6852642337521794, + "flos": 23182133500800.0, + "grad_norm": 1.8065178546549538, + "language_loss": 0.71703947, + "learning_rate": 9.520488294609142e-07, + "loss": 0.73866707, + "num_input_tokens_seen": 122576815, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.14916992, + "step": 5699, + "time_per_iteration": 2.8582894802093506 + }, + { + "auxiliary_loss_clip": 0.01035864, + "auxiliary_loss_mlp": 0.01001255, + "balance_loss_clip": 1.00839365, + "balance_loss_mlp": 0.99978846, + "epoch": 0.6853844766428185, + "flos": 62647206583680.0, + "grad_norm": 0.741726610367073, + "language_loss": 0.5383395, + "learning_rate": 9.513854328855368e-07, + "loss": 0.55871069, + "num_input_tokens_seen": 122634690, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01464844, + "step": 5700, + "time_per_iteration": 3.2592790126800537 + }, + { + "auxiliary_loss_clip": 0.01123691, + "auxiliary_loss_mlp": 0.0103922, + "balance_loss_clip": 1.04020894, + "balance_loss_mlp": 1.02362156, + "epoch": 0.6855047195334576, + "flos": 23437242869760.0, + "grad_norm": 1.7914242321232499, + "language_loss": 0.81285268, + "learning_rate": 9.507221953841558e-07, + "loss": 0.83448184, + "num_input_tokens_seen": 122652320, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.15594482, + "step": 5701, + "time_per_iteration": 2.7988831996917725 + }, + { + "auxiliary_loss_clip": 0.01133843, + "auxiliary_loss_mlp": 0.01038844, + "balance_loss_clip": 1.04375219, + "balance_loss_mlp": 1.02284622, + "epoch": 0.6856249624240967, + "flos": 20664831530880.0, + "grad_norm": 1.5243853726906376, + "language_loss": 0.7810415, + "learning_rate": 9.500591170573824e-07, + "loss": 0.80276835, + "num_input_tokens_seen": 122672340, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.16015625, + "step": 5702, + "time_per_iteration": 3.5469276905059814 + }, + { + "auxiliary_loss_clip": 0.01128387, + "auxiliary_loss_mlp": 0.01033572, + "balance_loss_clip": 1.04213607, + "balance_loss_mlp": 1.0179323, + "epoch": 0.6857452053147358, + "flos": 17087302794240.0, + "grad_norm": 1.8264790758612008, + "language_loss": 0.74263823, + "learning_rate": 9.493961980058078e-07, + "loss": 0.76425779, + "num_input_tokens_seen": 122689935, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15637207, + "step": 5703, + "time_per_iteration": 2.966479778289795 + }, + { + "auxiliary_loss_clip": 0.01126636, + "auxiliary_loss_mlp": 0.01036585, + "balance_loss_clip": 1.04130709, + "balance_loss_mlp": 1.02146327, + "epoch": 0.6858654482053749, + "flos": 30847266057600.0, + "grad_norm": 2.963949491698132, + "language_loss": 0.6770981, + "learning_rate": 9.48733438329993e-07, + "loss": 0.69873029, + "num_input_tokens_seen": 122710200, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.15118408, + "step": 5704, + "time_per_iteration": 2.9128987789154053 + }, + { + "auxiliary_loss_clip": 0.01125578, + "auxiliary_loss_mlp": 0.01039541, + "balance_loss_clip": 1.04210913, + "balance_loss_mlp": 1.02452707, + "epoch": 0.6859856910960139, + "flos": 28877314510080.0, + "grad_norm": 1.6281965574625545, + "language_loss": 0.7452414, + "learning_rate": 9.480708381304807e-07, + "loss": 0.76689255, + "num_input_tokens_seen": 122731495, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.15008545, + "step": 5705, + "time_per_iteration": 2.85927677154541 + }, + { + "auxiliary_loss_clip": 0.01128859, + "auxiliary_loss_mlp": 0.01034787, + "balance_loss_clip": 1.04252267, + "balance_loss_mlp": 1.01843131, + "epoch": 0.6861059339866531, + "flos": 19354523299200.0, + "grad_norm": 2.073014945647265, + "language_loss": 0.83683389, + "learning_rate": 9.474083975077858e-07, + "loss": 0.85847038, + "num_input_tokens_seen": 122748620, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.16345215, + "step": 5706, + "time_per_iteration": 3.5785579681396484 + }, + { + "auxiliary_loss_clip": 0.01129496, + "auxiliary_loss_mlp": 0.01038636, + "balance_loss_clip": 1.04316628, + "balance_loss_mlp": 1.02304327, + "epoch": 0.6862261768772921, + "flos": 22199976944640.0, + "grad_norm": 1.9390386825144361, + "language_loss": 0.79984105, + "learning_rate": 9.467461165623994e-07, + "loss": 0.82152236, + "num_input_tokens_seen": 122767670, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.15576172, + "step": 5707, + "time_per_iteration": 2.824981689453125 + }, + { + "auxiliary_loss_clip": 0.01129936, + "auxiliary_loss_mlp": 0.01038845, + "balance_loss_clip": 1.04322433, + "balance_loss_mlp": 1.02449846, + "epoch": 0.6863464197679312, + "flos": 26285677344000.0, + "grad_norm": 1.8381142858582658, + "language_loss": 0.79478729, + "learning_rate": 9.46083995394791e-07, + "loss": 0.81647515, + "num_input_tokens_seen": 122785480, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.14349365, + "step": 5708, + "time_per_iteration": 3.6994593143463135 + }, + { + "auxiliary_loss_clip": 0.01125982, + "auxiliary_loss_mlp": 0.01035806, + "balance_loss_clip": 1.04073822, + "balance_loss_mlp": 1.02028513, + "epoch": 0.6864666626585703, + "flos": 37815228564480.0, + "grad_norm": 2.0715904689131515, + "language_loss": 0.63342106, + "learning_rate": 9.454220341054012e-07, + "loss": 0.65503895, + "num_input_tokens_seen": 122810265, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.15515137, + "step": 5709, + "time_per_iteration": 3.1056416034698486 + }, + { + "auxiliary_loss_clip": 0.01128778, + "auxiliary_loss_mlp": 0.0103793, + "balance_loss_clip": 1.04177737, + "balance_loss_mlp": 1.02195597, + "epoch": 0.6865869055492094, + "flos": 19391152193280.0, + "grad_norm": 2.1667853531786245, + "language_loss": 0.80206043, + "learning_rate": 9.447602327946512e-07, + "loss": 0.82372749, + "num_input_tokens_seen": 122828905, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.1595459, + "step": 5710, + "time_per_iteration": 2.8850173950195312 + }, + { + "auxiliary_loss_clip": 0.01133124, + "auxiliary_loss_mlp": 0.0103989, + "balance_loss_clip": 1.04364181, + "balance_loss_mlp": 1.02414227, + "epoch": 0.6867071484398485, + "flos": 20375966355840.0, + "grad_norm": 1.7890871717021475, + "language_loss": 0.76658988, + "learning_rate": 9.440985915629338e-07, + "loss": 0.78831995, + "num_input_tokens_seen": 122846235, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.1574707, + "step": 5711, + "time_per_iteration": 2.7757389545440674 + }, + { + "auxiliary_loss_clip": 0.01126673, + "auxiliary_loss_mlp": 0.01034571, + "balance_loss_clip": 1.04261565, + "balance_loss_mlp": 1.02079046, + "epoch": 0.6868273913304875, + "flos": 15889143801600.0, + "grad_norm": 1.8872971712045035, + "language_loss": 0.73099983, + "learning_rate": 9.434371105106223e-07, + "loss": 0.75261223, + "num_input_tokens_seen": 122863835, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.13787842, + "step": 5712, + "time_per_iteration": 3.59787917137146 + }, + { + "auxiliary_loss_clip": 0.01128968, + "auxiliary_loss_mlp": 0.01035658, + "balance_loss_clip": 1.04039204, + "balance_loss_mlp": 1.01999342, + "epoch": 0.6869476342211267, + "flos": 24462492768000.0, + "grad_norm": 1.7461119915355898, + "language_loss": 0.71020222, + "learning_rate": 9.427757897380602e-07, + "loss": 0.73184848, + "num_input_tokens_seen": 122883235, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.15661621, + "step": 5713, + "time_per_iteration": 2.797116994857788 + }, + { + "auxiliary_loss_clip": 0.01129369, + "auxiliary_loss_mlp": 0.01036574, + "balance_loss_clip": 1.04106176, + "balance_loss_mlp": 1.02095723, + "epoch": 0.6870678771117658, + "flos": 18442571875200.0, + "grad_norm": 2.0406789392781057, + "language_loss": 0.84647787, + "learning_rate": 9.421146293455695e-07, + "loss": 0.86813736, + "num_input_tokens_seen": 122898975, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.15606689, + "step": 5714, + "time_per_iteration": 2.7400991916656494 + }, + { + "auxiliary_loss_clip": 0.0112764, + "auxiliary_loss_mlp": 0.0104149, + "balance_loss_clip": 1.04262424, + "balance_loss_mlp": 1.02598071, + "epoch": 0.6871881200024048, + "flos": 22200371994240.0, + "grad_norm": 1.6820804990570049, + "language_loss": 0.67756277, + "learning_rate": 9.414536294334489e-07, + "loss": 0.69925404, + "num_input_tokens_seen": 122918995, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.1550293, + "step": 5715, + "time_per_iteration": 2.8151280879974365 + }, + { + "auxiliary_loss_clip": 0.01128549, + "auxiliary_loss_mlp": 0.01036531, + "balance_loss_clip": 1.04049039, + "balance_loss_mlp": 1.02089095, + "epoch": 0.687308362893044, + "flos": 22127724737280.0, + "grad_norm": 2.117445235322953, + "language_loss": 0.69570333, + "learning_rate": 9.407927901019708e-07, + "loss": 0.71735418, + "num_input_tokens_seen": 122938125, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15643311, + "step": 5716, + "time_per_iteration": 2.7972917556762695 + }, + { + "auxiliary_loss_clip": 0.01126029, + "auxiliary_loss_mlp": 0.01040907, + "balance_loss_clip": 1.04070759, + "balance_loss_mlp": 1.0252434, + "epoch": 0.687428605783683, + "flos": 25040546340480.0, + "grad_norm": 1.8959882225698301, + "language_loss": 0.76740015, + "learning_rate": 9.401321114513854e-07, + "loss": 0.78906953, + "num_input_tokens_seen": 122957020, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.15655518, + "step": 5717, + "time_per_iteration": 2.8129677772521973 + }, + { + "auxiliary_loss_clip": 0.01129548, + "auxiliary_loss_mlp": 0.01039505, + "balance_loss_clip": 1.04210758, + "balance_loss_mlp": 1.02283943, + "epoch": 0.6875488486743221, + "flos": 23770063313280.0, + "grad_norm": 1.6291254975955645, + "language_loss": 0.75016975, + "learning_rate": 9.394715935819155e-07, + "loss": 0.77186024, + "num_input_tokens_seen": 122977410, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.16662598, + "step": 5718, + "time_per_iteration": 2.815274953842163 + }, + { + "auxiliary_loss_clip": 0.01129181, + "auxiliary_loss_mlp": 0.01044462, + "balance_loss_clip": 1.04170263, + "balance_loss_mlp": 1.02845216, + "epoch": 0.6876690915649613, + "flos": 25516937445120.0, + "grad_norm": 1.7979393463479028, + "language_loss": 0.62533987, + "learning_rate": 9.388112365937608e-07, + "loss": 0.64707631, + "num_input_tokens_seen": 122996875, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.16003418, + "step": 5719, + "time_per_iteration": 2.893394947052002 + }, + { + "auxiliary_loss_clip": 0.01132741, + "auxiliary_loss_mlp": 0.01043718, + "balance_loss_clip": 1.04276562, + "balance_loss_mlp": 1.028018, + "epoch": 0.6877893344556003, + "flos": 19427996568960.0, + "grad_norm": 2.1383077031268343, + "language_loss": 0.82842994, + "learning_rate": 9.381510405870985e-07, + "loss": 0.85019457, + "num_input_tokens_seen": 123015890, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.15698242, + "step": 5720, + "time_per_iteration": 2.8689887523651123 + }, + { + "auxiliary_loss_clip": 0.01135074, + "auxiliary_loss_mlp": 0.01041397, + "balance_loss_clip": 1.04574156, + "balance_loss_mlp": 1.02474356, + "epoch": 0.6879095773462394, + "flos": 18661303745280.0, + "grad_norm": 4.1809728779745985, + "language_loss": 0.7690984, + "learning_rate": 9.374910056620791e-07, + "loss": 0.79086304, + "num_input_tokens_seen": 123034955, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.16650391, + "step": 5721, + "time_per_iteration": 2.870836019515991 + }, + { + "auxiliary_loss_clip": 0.01132568, + "auxiliary_loss_mlp": 0.01039083, + "balance_loss_clip": 1.04279399, + "balance_loss_mlp": 1.02308524, + "epoch": 0.6880298202368785, + "flos": 20883132437760.0, + "grad_norm": 1.6701276038726383, + "language_loss": 0.80841637, + "learning_rate": 9.368311319188293e-07, + "loss": 0.83013284, + "num_input_tokens_seen": 123052770, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.15991211, + "step": 5722, + "time_per_iteration": 2.8958237171173096 + }, + { + "auxiliary_loss_clip": 0.01127378, + "auxiliary_loss_mlp": 0.01040442, + "balance_loss_clip": 1.04037452, + "balance_loss_mlp": 1.02561212, + "epoch": 0.6881500631275176, + "flos": 30153292318080.0, + "grad_norm": 1.6430042578688484, + "language_loss": 0.79104424, + "learning_rate": 9.361714194574515e-07, + "loss": 0.81272238, + "num_input_tokens_seen": 123075105, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.1484375, + "step": 5723, + "time_per_iteration": 2.9211981296539307 + }, + { + "auxiliary_loss_clip": 0.01033704, + "auxiliary_loss_mlp": 0.01003487, + "balance_loss_clip": 1.00630426, + "balance_loss_mlp": 1.00188947, + "epoch": 0.6882703060181566, + "flos": 66181537215360.0, + "grad_norm": 0.7411989446095564, + "language_loss": 0.58255291, + "learning_rate": 9.355118683780228e-07, + "loss": 0.60292476, + "num_input_tokens_seen": 123145175, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01599121, + "step": 5724, + "time_per_iteration": 3.452463150024414 + }, + { + "auxiliary_loss_clip": 0.0112744, + "auxiliary_loss_mlp": 0.01038327, + "balance_loss_clip": 1.04035258, + "balance_loss_mlp": 1.02279401, + "epoch": 0.6883905489087958, + "flos": 18214646123520.0, + "grad_norm": 2.2176204492098743, + "language_loss": 0.78878582, + "learning_rate": 9.348524787805987e-07, + "loss": 0.81044352, + "num_input_tokens_seen": 123160365, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.15509033, + "step": 5725, + "time_per_iteration": 2.840313196182251 + }, + { + "auxiliary_loss_clip": 0.01127561, + "auxiliary_loss_mlp": 0.01038429, + "balance_loss_clip": 1.03884768, + "balance_loss_mlp": 1.02196598, + "epoch": 0.6885107917994349, + "flos": 14056262553600.0, + "grad_norm": 2.7694386215991083, + "language_loss": 0.85361004, + "learning_rate": 9.341932507652053e-07, + "loss": 0.87526989, + "num_input_tokens_seen": 123174855, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.16455078, + "step": 5726, + "time_per_iteration": 2.801845073699951 + }, + { + "auxiliary_loss_clip": 0.01131092, + "auxiliary_loss_mlp": 0.01037648, + "balance_loss_clip": 1.04046583, + "balance_loss_mlp": 1.02199602, + "epoch": 0.6886310346900739, + "flos": 28690722334080.0, + "grad_norm": 1.7542586700212477, + "language_loss": 0.78597957, + "learning_rate": 9.335341844318489e-07, + "loss": 0.80766696, + "num_input_tokens_seen": 123194995, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.15661621, + "step": 5727, + "time_per_iteration": 2.901625871658325 + }, + { + "auxiliary_loss_clip": 0.01127503, + "auxiliary_loss_mlp": 0.01038655, + "balance_loss_clip": 1.04037213, + "balance_loss_mlp": 1.02305079, + "epoch": 0.6887512775807131, + "flos": 24535319592960.0, + "grad_norm": 1.7042209645631519, + "language_loss": 0.73054183, + "learning_rate": 9.328752798805091e-07, + "loss": 0.75220346, + "num_input_tokens_seen": 123213465, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15600586, + "step": 5728, + "time_per_iteration": 2.827347755432129 + }, + { + "auxiliary_loss_clip": 0.0113196, + "auxiliary_loss_mlp": 0.01043484, + "balance_loss_clip": 1.04210734, + "balance_loss_mlp": 1.0276829, + "epoch": 0.6888715204713521, + "flos": 22414363269120.0, + "grad_norm": 1.8933826910122549, + "language_loss": 0.75828201, + "learning_rate": 9.322165372111399e-07, + "loss": 0.78003645, + "num_input_tokens_seen": 123231610, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.15802002, + "step": 5729, + "time_per_iteration": 2.836357355117798 + }, + { + "auxiliary_loss_clip": 0.01130923, + "auxiliary_loss_mlp": 0.01042727, + "balance_loss_clip": 1.04246151, + "balance_loss_mlp": 1.0274446, + "epoch": 0.6889917633619912, + "flos": 22054323294720.0, + "grad_norm": 1.9929061714216982, + "language_loss": 0.75972402, + "learning_rate": 9.315579565236747e-07, + "loss": 0.78146052, + "num_input_tokens_seen": 123250715, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15270996, + "step": 5730, + "time_per_iteration": 2.967447519302368 + }, + { + "auxiliary_loss_clip": 0.01128435, + "auxiliary_loss_mlp": 0.01043966, + "balance_loss_clip": 1.04090786, + "balance_loss_mlp": 1.0278728, + "epoch": 0.6891120062526304, + "flos": 23949724164480.0, + "grad_norm": 1.6519037427779726, + "language_loss": 0.74306715, + "learning_rate": 9.308995379180162e-07, + "loss": 0.76479113, + "num_input_tokens_seen": 123270270, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.16101074, + "step": 5731, + "time_per_iteration": 3.7435426712036133 + }, + { + "auxiliary_loss_clip": 0.0103401, + "auxiliary_loss_mlp": 0.01002327, + "balance_loss_clip": 1.00661099, + "balance_loss_mlp": 1.00061059, + "epoch": 0.6892322491432694, + "flos": 64117354337280.0, + "grad_norm": 0.7716355793590424, + "language_loss": 0.59448922, + "learning_rate": 9.302412814940488e-07, + "loss": 0.61485267, + "num_input_tokens_seen": 123333045, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01721191, + "step": 5732, + "time_per_iteration": 3.4268414974212646 + }, + { + "auxiliary_loss_clip": 0.01132088, + "auxiliary_loss_mlp": 0.01040891, + "balance_loss_clip": 1.04378033, + "balance_loss_mlp": 1.02460682, + "epoch": 0.6893524920339085, + "flos": 23002436736000.0, + "grad_norm": 2.0017974974967805, + "language_loss": 0.71015108, + "learning_rate": 9.295831873516276e-07, + "loss": 0.7318809, + "num_input_tokens_seen": 123352320, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.16290283, + "step": 5733, + "time_per_iteration": 2.8960421085357666 + }, + { + "auxiliary_loss_clip": 0.01128029, + "auxiliary_loss_mlp": 0.01035935, + "balance_loss_clip": 1.04226518, + "balance_loss_mlp": 1.0218327, + "epoch": 0.6894727349245476, + "flos": 21396260177280.0, + "grad_norm": 1.586054388225491, + "language_loss": 0.76075792, + "learning_rate": 9.289252555905873e-07, + "loss": 0.78239751, + "num_input_tokens_seen": 123372400, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.14099121, + "step": 5734, + "time_per_iteration": 2.8721466064453125 + }, + { + "auxiliary_loss_clip": 0.0113065, + "auxiliary_loss_mlp": 0.01043857, + "balance_loss_clip": 1.04257727, + "balance_loss_mlp": 1.02758455, + "epoch": 0.6895929778151867, + "flos": 19865316654720.0, + "grad_norm": 1.8930461465276611, + "language_loss": 0.76218331, + "learning_rate": 9.282674863107334e-07, + "loss": 0.78392839, + "num_input_tokens_seen": 123390215, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.16271973, + "step": 5735, + "time_per_iteration": 3.5989716053009033 + }, + { + "auxiliary_loss_clip": 0.01131037, + "auxiliary_loss_mlp": 0.01035124, + "balance_loss_clip": 1.04268074, + "balance_loss_mlp": 1.01914954, + "epoch": 0.6897132207058257, + "flos": 18179166464640.0, + "grad_norm": 1.9978614039293308, + "language_loss": 0.75619793, + "learning_rate": 9.276098796118488e-07, + "loss": 0.77785957, + "num_input_tokens_seen": 123406700, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15966797, + "step": 5736, + "time_per_iteration": 3.58882737159729 + }, + { + "auxiliary_loss_clip": 0.01127738, + "auxiliary_loss_mlp": 0.01036331, + "balance_loss_clip": 1.04094982, + "balance_loss_mlp": 1.02103627, + "epoch": 0.6898334635964649, + "flos": 32561641359360.0, + "grad_norm": 1.6870660778532203, + "language_loss": 0.66306639, + "learning_rate": 9.269524355936938e-07, + "loss": 0.68470705, + "num_input_tokens_seen": 123429880, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15307617, + "step": 5737, + "time_per_iteration": 2.8748373985290527 + }, + { + "auxiliary_loss_clip": 0.01125725, + "auxiliary_loss_mlp": 0.01043515, + "balance_loss_clip": 1.04201269, + "balance_loss_mlp": 1.02813125, + "epoch": 0.689953706487104, + "flos": 22819004956800.0, + "grad_norm": 1.937210418047497, + "language_loss": 0.85144353, + "learning_rate": 9.262951543560002e-07, + "loss": 0.87313592, + "num_input_tokens_seen": 123449105, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.1539917, + "step": 5738, + "time_per_iteration": 2.852273464202881 + }, + { + "auxiliary_loss_clip": 0.01133461, + "auxiliary_loss_mlp": 0.0103685, + "balance_loss_clip": 1.04353642, + "balance_loss_mlp": 1.02099538, + "epoch": 0.690073949377743, + "flos": 18515362786560.0, + "grad_norm": 2.8351441798722097, + "language_loss": 0.86086041, + "learning_rate": 9.256380359984795e-07, + "loss": 0.88256353, + "num_input_tokens_seen": 123466215, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.15856934, + "step": 5739, + "time_per_iteration": 2.835541248321533 + }, + { + "auxiliary_loss_clip": 0.0113076, + "auxiliary_loss_mlp": 0.0103457, + "balance_loss_clip": 1.04202068, + "balance_loss_mlp": 1.01888204, + "epoch": 0.6901941922683821, + "flos": 34857194716800.0, + "grad_norm": 3.445004515988919, + "language_loss": 0.74567819, + "learning_rate": 9.249810806208139e-07, + "loss": 0.76733148, + "num_input_tokens_seen": 123485480, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.15673828, + "step": 5740, + "time_per_iteration": 2.9737370014190674 + }, + { + "auxiliary_loss_clip": 0.01127949, + "auxiliary_loss_mlp": 0.01040869, + "balance_loss_clip": 1.04296291, + "balance_loss_mlp": 1.02636755, + "epoch": 0.6903144351590212, + "flos": 16253672976000.0, + "grad_norm": 1.939327394762778, + "language_loss": 0.798536, + "learning_rate": 9.243242883226627e-07, + "loss": 0.82022417, + "num_input_tokens_seen": 123504575, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.1449585, + "step": 5741, + "time_per_iteration": 3.728024482727051 + }, + { + "auxiliary_loss_clip": 0.01129271, + "auxiliary_loss_mlp": 0.01042699, + "balance_loss_clip": 1.03912354, + "balance_loss_mlp": 1.02584279, + "epoch": 0.6904346780496603, + "flos": 28035137255040.0, + "grad_norm": 1.7912588644814216, + "language_loss": 0.69944417, + "learning_rate": 9.236676592036628e-07, + "loss": 0.72116387, + "num_input_tokens_seen": 123524250, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.16870117, + "step": 5742, + "time_per_iteration": 2.9130470752716064 + }, + { + "auxiliary_loss_clip": 0.01128145, + "auxiliary_loss_mlp": 0.01037423, + "balance_loss_clip": 1.0422523, + "balance_loss_mlp": 1.02162766, + "epoch": 0.6905549209402994, + "flos": 23624266008960.0, + "grad_norm": 2.199855355334897, + "language_loss": 0.73684502, + "learning_rate": 9.230111933634228e-07, + "loss": 0.7585007, + "num_input_tokens_seen": 123545845, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15808105, + "step": 5743, + "time_per_iteration": 2.9910056591033936 + }, + { + "auxiliary_loss_clip": 0.01131859, + "auxiliary_loss_mlp": 0.01049234, + "balance_loss_clip": 1.04438591, + "balance_loss_mlp": 1.03230619, + "epoch": 0.6906751638309385, + "flos": 23114945111040.0, + "grad_norm": 1.545449498249108, + "language_loss": 0.8069247, + "learning_rate": 9.223548909015288e-07, + "loss": 0.82873559, + "num_input_tokens_seen": 123567535, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.16931152, + "step": 5744, + "time_per_iteration": 2.9516265392303467 + }, + { + "auxiliary_loss_clip": 0.0112497, + "auxiliary_loss_mlp": 0.0103511, + "balance_loss_clip": 1.04032457, + "balance_loss_mlp": 1.02079332, + "epoch": 0.6907954067215776, + "flos": 27305468375040.0, + "grad_norm": 2.0205895279215613, + "language_loss": 0.72444606, + "learning_rate": 9.216987519175407e-07, + "loss": 0.74604684, + "num_input_tokens_seen": 123587710, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.14324951, + "step": 5745, + "time_per_iteration": 2.9204890727996826 + }, + { + "auxiliary_loss_clip": 0.01125198, + "auxiliary_loss_mlp": 0.01036253, + "balance_loss_clip": 1.04025531, + "balance_loss_mlp": 1.02070773, + "epoch": 0.6909156496122166, + "flos": 21689399070720.0, + "grad_norm": 1.786893184816474, + "language_loss": 0.68655801, + "learning_rate": 9.210427765109942e-07, + "loss": 0.70817256, + "num_input_tokens_seen": 123607385, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.15539551, + "step": 5746, + "time_per_iteration": 2.9096920490264893 + }, + { + "auxiliary_loss_clip": 0.01135753, + "auxiliary_loss_mlp": 0.01038667, + "balance_loss_clip": 1.04291058, + "balance_loss_mlp": 1.02112532, + "epoch": 0.6910358925028558, + "flos": 22561453463040.0, + "grad_norm": 2.4006162225295866, + "language_loss": 0.81343222, + "learning_rate": 9.20386964781402e-07, + "loss": 0.83517635, + "num_input_tokens_seen": 123625405, + "router_z_loss_clip": 0.92773438, + "router_z_loss_mlp": 0.17547607, + "step": 5747, + "time_per_iteration": 2.9266583919525146 + }, + { + "auxiliary_loss_clip": 0.01129624, + "auxiliary_loss_mlp": 0.01036922, + "balance_loss_clip": 1.04081047, + "balance_loss_mlp": 1.02111518, + "epoch": 0.6911561353934949, + "flos": 22054107813120.0, + "grad_norm": 2.450185025541962, + "language_loss": 0.84262145, + "learning_rate": 9.197313168282472e-07, + "loss": 0.8642869, + "num_input_tokens_seen": 123642850, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.15808105, + "step": 5748, + "time_per_iteration": 2.9165353775024414 + }, + { + "auxiliary_loss_clip": 0.01127978, + "auxiliary_loss_mlp": 0.01036086, + "balance_loss_clip": 1.0390265, + "balance_loss_mlp": 1.02031493, + "epoch": 0.6912763782841339, + "flos": 24206557386240.0, + "grad_norm": 1.952614132932125, + "language_loss": 0.72124863, + "learning_rate": 9.190758327509935e-07, + "loss": 0.74288929, + "num_input_tokens_seen": 123661595, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.15795898, + "step": 5749, + "time_per_iteration": 2.9597935676574707 + }, + { + "auxiliary_loss_clip": 0.01033759, + "auxiliary_loss_mlp": 0.00999953, + "balance_loss_clip": 1.00626552, + "balance_loss_mlp": 0.99841481, + "epoch": 0.6913966211747731, + "flos": 52329641091840.0, + "grad_norm": 0.9306034602287948, + "language_loss": 0.6443525, + "learning_rate": 9.184205126490767e-07, + "loss": 0.6646896, + "num_input_tokens_seen": 123710490, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01531982, + "step": 5750, + "time_per_iteration": 3.230325937271118 + }, + { + "auxiliary_loss_clip": 0.01033382, + "auxiliary_loss_mlp": 0.0100029, + "balance_loss_clip": 1.00581217, + "balance_loss_mlp": 0.99865705, + "epoch": 0.6915168640654121, + "flos": 66741274851840.0, + "grad_norm": 1.11860202772089, + "language_loss": 0.59638178, + "learning_rate": 9.177653566219075e-07, + "loss": 0.61671853, + "num_input_tokens_seen": 123765215, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01635742, + "step": 5751, + "time_per_iteration": 3.1954805850982666 + }, + { + "auxiliary_loss_clip": 0.01131577, + "auxiliary_loss_mlp": 0.01038685, + "balance_loss_clip": 1.04321861, + "balance_loss_mlp": 1.02253246, + "epoch": 0.6916371069560512, + "flos": 18296523175680.0, + "grad_norm": 5.866624741775249, + "language_loss": 0.76156616, + "learning_rate": 9.171103647688744e-07, + "loss": 0.78326875, + "num_input_tokens_seen": 123783955, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.16162109, + "step": 5752, + "time_per_iteration": 2.947014808654785 + }, + { + "auxiliary_loss_clip": 0.01123948, + "auxiliary_loss_mlp": 0.01052015, + "balance_loss_clip": 1.04072213, + "balance_loss_mlp": 1.03633881, + "epoch": 0.6917573498466904, + "flos": 19645794685440.0, + "grad_norm": 1.800852067310076, + "language_loss": 0.6902861, + "learning_rate": 9.164555371893367e-07, + "loss": 0.71204573, + "num_input_tokens_seen": 123803885, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.15686035, + "step": 5753, + "time_per_iteration": 2.9447038173675537 + }, + { + "auxiliary_loss_clip": 0.01126773, + "auxiliary_loss_mlp": 0.01037464, + "balance_loss_clip": 1.0408231, + "balance_loss_mlp": 1.02187121, + "epoch": 0.6918775927373294, + "flos": 14210319985920.0, + "grad_norm": 1.82456004251647, + "language_loss": 0.75427026, + "learning_rate": 9.158008739826333e-07, + "loss": 0.77591264, + "num_input_tokens_seen": 123821485, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.15576172, + "step": 5754, + "time_per_iteration": 2.902341365814209 + }, + { + "auxiliary_loss_clip": 0.01132485, + "auxiliary_loss_mlp": 0.01035604, + "balance_loss_clip": 1.04226756, + "balance_loss_mlp": 1.01917696, + "epoch": 0.6919978356279685, + "flos": 23985455218560.0, + "grad_norm": 9.16726832792721, + "language_loss": 0.86469537, + "learning_rate": 9.151463752480744e-07, + "loss": 0.88637626, + "num_input_tokens_seen": 123840215, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.16418457, + "step": 5755, + "time_per_iteration": 2.940119504928589 + }, + { + "auxiliary_loss_clip": 0.01122849, + "auxiliary_loss_mlp": 0.01035624, + "balance_loss_clip": 1.03897262, + "balance_loss_mlp": 1.02059162, + "epoch": 0.6921180785186076, + "flos": 23622937205760.0, + "grad_norm": 1.4885250270502932, + "language_loss": 0.80412376, + "learning_rate": 9.144920410849493e-07, + "loss": 0.82570839, + "num_input_tokens_seen": 123861450, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.15026855, + "step": 5756, + "time_per_iteration": 2.9279844760894775 + }, + { + "auxiliary_loss_clip": 0.01130906, + "auxiliary_loss_mlp": 0.01035679, + "balance_loss_clip": 1.04356039, + "balance_loss_mlp": 1.02083731, + "epoch": 0.6922383214092467, + "flos": 21142623265920.0, + "grad_norm": 1.8009967875924824, + "language_loss": 0.80658114, + "learning_rate": 9.138378715925176e-07, + "loss": 0.82824695, + "num_input_tokens_seen": 123880545, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.1484375, + "step": 5757, + "time_per_iteration": 2.848914384841919 + }, + { + "auxiliary_loss_clip": 0.01125898, + "auxiliary_loss_mlp": 0.01034319, + "balance_loss_clip": 1.03926754, + "balance_loss_mlp": 1.01927495, + "epoch": 0.6923585642998857, + "flos": 21470667200640.0, + "grad_norm": 1.706912447723893, + "language_loss": 0.81258726, + "learning_rate": 9.131838668700167e-07, + "loss": 0.83418947, + "num_input_tokens_seen": 123900615, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15039062, + "step": 5758, + "time_per_iteration": 2.8360774517059326 + }, + { + "auxiliary_loss_clip": 0.01128391, + "auxiliary_loss_mlp": 0.01035291, + "balance_loss_clip": 1.04141176, + "balance_loss_mlp": 1.01950729, + "epoch": 0.6924788071905249, + "flos": 21105204272640.0, + "grad_norm": 1.7684970262662185, + "language_loss": 0.86346126, + "learning_rate": 9.125300270166598e-07, + "loss": 0.8850981, + "num_input_tokens_seen": 123921220, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15777588, + "step": 5759, + "time_per_iteration": 2.8639824390411377 + }, + { + "auxiliary_loss_clip": 0.01126785, + "auxiliary_loss_mlp": 0.01037538, + "balance_loss_clip": 1.04047227, + "balance_loss_mlp": 1.02245808, + "epoch": 0.692599050081164, + "flos": 26250018117120.0, + "grad_norm": 1.7183475256722078, + "language_loss": 0.85966265, + "learning_rate": 9.118763521316324e-07, + "loss": 0.88130587, + "num_input_tokens_seen": 123941795, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15075684, + "step": 5760, + "time_per_iteration": 3.565528154373169 + }, + { + "auxiliary_loss_clip": 0.01127211, + "auxiliary_loss_mlp": 0.01036634, + "balance_loss_clip": 1.0403254, + "balance_loss_mlp": 1.02130365, + "epoch": 0.692719292971803, + "flos": 20885215426560.0, + "grad_norm": 2.452848399794362, + "language_loss": 0.76213056, + "learning_rate": 9.112228423140987e-07, + "loss": 0.78376901, + "num_input_tokens_seen": 123960715, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15319824, + "step": 5761, + "time_per_iteration": 2.868589162826538 + }, + { + "auxiliary_loss_clip": 0.01130243, + "auxiliary_loss_mlp": 0.01038958, + "balance_loss_clip": 1.04003501, + "balance_loss_mlp": 1.02311468, + "epoch": 0.6928395358624422, + "flos": 25921938268800.0, + "grad_norm": 3.489066234189587, + "language_loss": 0.86101377, + "learning_rate": 9.105694976631932e-07, + "loss": 0.88270581, + "num_input_tokens_seen": 123978625, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.1583252, + "step": 5762, + "time_per_iteration": 2.9041576385498047 + }, + { + "auxiliary_loss_clip": 0.01129349, + "auxiliary_loss_mlp": 0.01045909, + "balance_loss_clip": 1.0423789, + "balance_loss_mlp": 1.02890968, + "epoch": 0.6929597787530812, + "flos": 23586559706880.0, + "grad_norm": 1.9951601780679824, + "language_loss": 0.72543591, + "learning_rate": 9.099163182780283e-07, + "loss": 0.74718845, + "num_input_tokens_seen": 123996780, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.17004395, + "step": 5763, + "time_per_iteration": 2.9191601276397705 + }, + { + "auxiliary_loss_clip": 0.01127781, + "auxiliary_loss_mlp": 0.01033751, + "balance_loss_clip": 1.0405581, + "balance_loss_mlp": 1.01737213, + "epoch": 0.6930800216437203, + "flos": 18255656476800.0, + "grad_norm": 2.725500198672146, + "language_loss": 0.49291074, + "learning_rate": 9.092633042576916e-07, + "loss": 0.51452607, + "num_input_tokens_seen": 124014045, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.16357422, + "step": 5764, + "time_per_iteration": 3.7144594192504883 + }, + { + "auxiliary_loss_clip": 0.01128019, + "auxiliary_loss_mlp": 0.01035354, + "balance_loss_clip": 1.04096007, + "balance_loss_mlp": 1.01986814, + "epoch": 0.6932002645343595, + "flos": 29168621809920.0, + "grad_norm": 1.8303819392104768, + "language_loss": 0.56398702, + "learning_rate": 9.086104557012446e-07, + "loss": 0.58562076, + "num_input_tokens_seen": 124034615, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15490723, + "step": 5765, + "time_per_iteration": 3.782320261001587 + }, + { + "auxiliary_loss_clip": 0.01127091, + "auxiliary_loss_mlp": 0.01041491, + "balance_loss_clip": 1.04055405, + "balance_loss_mlp": 1.02759111, + "epoch": 0.6933205074249985, + "flos": 23842746483840.0, + "grad_norm": 2.317179964082007, + "language_loss": 0.65964186, + "learning_rate": 9.079577727077239e-07, + "loss": 0.6813277, + "num_input_tokens_seen": 124053445, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.13909912, + "step": 5766, + "time_per_iteration": 2.992018938064575 + }, + { + "auxiliary_loss_clip": 0.01127536, + "auxiliary_loss_mlp": 0.01042247, + "balance_loss_clip": 1.04103231, + "balance_loss_mlp": 1.02629638, + "epoch": 0.6934407503156376, + "flos": 24166696268160.0, + "grad_norm": 2.5974893641052454, + "language_loss": 0.71764356, + "learning_rate": 9.073052553761404e-07, + "loss": 0.73934138, + "num_input_tokens_seen": 124072810, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15942383, + "step": 5767, + "time_per_iteration": 2.923896074295044 + }, + { + "auxiliary_loss_clip": 0.01133396, + "auxiliary_loss_mlp": 0.01042949, + "balance_loss_clip": 1.04268563, + "balance_loss_mlp": 1.0257237, + "epoch": 0.6935609932062767, + "flos": 20631327120000.0, + "grad_norm": 1.6072284408014104, + "language_loss": 0.77916366, + "learning_rate": 9.066529038054805e-07, + "loss": 0.80092716, + "num_input_tokens_seen": 124092875, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.17224121, + "step": 5768, + "time_per_iteration": 2.9594075679779053 + }, + { + "auxiliary_loss_clip": 0.01128199, + "auxiliary_loss_mlp": 0.0103675, + "balance_loss_clip": 1.04236317, + "balance_loss_mlp": 1.02115726, + "epoch": 0.6936812360969158, + "flos": 18254184019200.0, + "grad_norm": 1.6771880830247043, + "language_loss": 0.7423023, + "learning_rate": 9.060007180947071e-07, + "loss": 0.76395178, + "num_input_tokens_seen": 124110930, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15600586, + "step": 5769, + "time_per_iteration": 2.8290154933929443 + }, + { + "auxiliary_loss_clip": 0.0113013, + "auxiliary_loss_mlp": 0.01046466, + "balance_loss_clip": 1.04020607, + "balance_loss_mlp": 1.02945495, + "epoch": 0.6938014789875548, + "flos": 31317336368640.0, + "grad_norm": 1.8008075977247213, + "language_loss": 0.73255241, + "learning_rate": 9.053486983427534e-07, + "loss": 0.75431836, + "num_input_tokens_seen": 124132180, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.17028809, + "step": 5770, + "time_per_iteration": 3.851329803466797 + }, + { + "auxiliary_loss_clip": 0.01128413, + "auxiliary_loss_mlp": 0.01042449, + "balance_loss_clip": 1.04075408, + "balance_loss_mlp": 1.02580702, + "epoch": 0.6939217218781939, + "flos": 17528429721600.0, + "grad_norm": 1.8182446766447693, + "language_loss": 0.70552087, + "learning_rate": 9.046968446485326e-07, + "loss": 0.72722948, + "num_input_tokens_seen": 124150585, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.16625977, + "step": 5771, + "time_per_iteration": 2.8125369548797607 + }, + { + "auxiliary_loss_clip": 0.01129577, + "auxiliary_loss_mlp": 0.01039201, + "balance_loss_clip": 1.04112864, + "balance_loss_mlp": 1.0222733, + "epoch": 0.6940419647688331, + "flos": 18551776199040.0, + "grad_norm": 2.036850850291078, + "language_loss": 0.70469081, + "learning_rate": 9.040451571109295e-07, + "loss": 0.72637856, + "num_input_tokens_seen": 124166205, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.16931152, + "step": 5772, + "time_per_iteration": 2.798056125640869 + }, + { + "auxiliary_loss_clip": 0.01034285, + "auxiliary_loss_mlp": 0.01004145, + "balance_loss_clip": 1.00696492, + "balance_loss_mlp": 1.00252342, + "epoch": 0.6941622076594721, + "flos": 66926286829440.0, + "grad_norm": 0.8345784478936623, + "language_loss": 0.6038273, + "learning_rate": 9.033936358288042e-07, + "loss": 0.62421155, + "num_input_tokens_seen": 124219940, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01623535, + "step": 5773, + "time_per_iteration": 3.194453001022339 + }, + { + "auxiliary_loss_clip": 0.01128935, + "auxiliary_loss_mlp": 0.01039505, + "balance_loss_clip": 1.04052722, + "balance_loss_mlp": 1.02393627, + "epoch": 0.6942824505501112, + "flos": 26578062051840.0, + "grad_norm": 1.7272785406651876, + "language_loss": 0.82656562, + "learning_rate": 9.027422809009937e-07, + "loss": 0.84825003, + "num_input_tokens_seen": 124239885, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15563965, + "step": 5774, + "time_per_iteration": 2.8189640045166016 + }, + { + "auxiliary_loss_clip": 0.01128199, + "auxiliary_loss_mlp": 0.01039175, + "balance_loss_clip": 1.04111099, + "balance_loss_mlp": 1.0233202, + "epoch": 0.6944026934407503, + "flos": 21248308056960.0, + "grad_norm": 1.5265180857803013, + "language_loss": 0.83454216, + "learning_rate": 9.020910924263054e-07, + "loss": 0.85621589, + "num_input_tokens_seen": 124258410, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15844727, + "step": 5775, + "time_per_iteration": 2.853489637374878 + }, + { + "auxiliary_loss_clip": 0.01034272, + "auxiliary_loss_mlp": 0.01005896, + "balance_loss_clip": 1.00688326, + "balance_loss_mlp": 1.00438213, + "epoch": 0.6945229363313894, + "flos": 70677191537280.0, + "grad_norm": 0.8080153915378049, + "language_loss": 0.58138758, + "learning_rate": 9.014400705035261e-07, + "loss": 0.60178924, + "num_input_tokens_seen": 124315315, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01513672, + "step": 5776, + "time_per_iteration": 3.368705987930298 + }, + { + "auxiliary_loss_clip": 0.01124109, + "auxiliary_loss_mlp": 0.01035822, + "balance_loss_clip": 1.04047012, + "balance_loss_mlp": 1.02083766, + "epoch": 0.6946431792220285, + "flos": 18952934267520.0, + "grad_norm": 2.001615072145894, + "language_loss": 0.76780891, + "learning_rate": 9.00789215231414e-07, + "loss": 0.78940821, + "num_input_tokens_seen": 124333710, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.14990234, + "step": 5777, + "time_per_iteration": 2.780301570892334 + }, + { + "auxiliary_loss_clip": 0.01131708, + "auxiliary_loss_mlp": 0.01042332, + "balance_loss_clip": 1.0422523, + "balance_loss_mlp": 1.02549958, + "epoch": 0.6947634221126676, + "flos": 20338834671360.0, + "grad_norm": 1.6591525425899227, + "language_loss": 0.82242906, + "learning_rate": 9.001385267087056e-07, + "loss": 0.8441695, + "num_input_tokens_seen": 124352855, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.1685791, + "step": 5778, + "time_per_iteration": 2.7829151153564453 + }, + { + "auxiliary_loss_clip": 0.01125953, + "auxiliary_loss_mlp": 0.0103984, + "balance_loss_clip": 1.03910732, + "balance_loss_mlp": 1.024176, + "epoch": 0.6948836650033067, + "flos": 21833723917440.0, + "grad_norm": 1.4962352410662716, + "language_loss": 0.70420635, + "learning_rate": 8.994880050341072e-07, + "loss": 0.72586429, + "num_input_tokens_seen": 124372960, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.15673828, + "step": 5779, + "time_per_iteration": 2.8277337551116943 + }, + { + "auxiliary_loss_clip": 0.01131529, + "auxiliary_loss_mlp": 0.01039023, + "balance_loss_clip": 1.04478335, + "balance_loss_mlp": 1.02272725, + "epoch": 0.6950039078939457, + "flos": 23657519024640.0, + "grad_norm": 1.869506035989219, + "language_loss": 0.77558422, + "learning_rate": 8.988376503063026e-07, + "loss": 0.79728985, + "num_input_tokens_seen": 124394220, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.16296387, + "step": 5780, + "time_per_iteration": 2.91894268989563 + }, + { + "auxiliary_loss_clip": 0.01130959, + "auxiliary_loss_mlp": 0.01041551, + "balance_loss_clip": 1.04109287, + "balance_loss_mlp": 1.02508783, + "epoch": 0.6951241507845849, + "flos": 21792462168960.0, + "grad_norm": 2.0467815627525594, + "language_loss": 0.81500924, + "learning_rate": 8.981874626239521e-07, + "loss": 0.83673441, + "num_input_tokens_seen": 124412795, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.16455078, + "step": 5781, + "time_per_iteration": 2.8521246910095215 + }, + { + "auxiliary_loss_clip": 0.0113072, + "auxiliary_loss_mlp": 0.01035322, + "balance_loss_clip": 1.04309106, + "balance_loss_mlp": 1.02016425, + "epoch": 0.695244393675224, + "flos": 14647568244480.0, + "grad_norm": 1.9057873038111748, + "language_loss": 0.88253498, + "learning_rate": 8.975374420856872e-07, + "loss": 0.90419531, + "num_input_tokens_seen": 124429690, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.15161133, + "step": 5782, + "time_per_iteration": 2.7880616188049316 + }, + { + "auxiliary_loss_clip": 0.01124977, + "auxiliary_loss_mlp": 0.01037863, + "balance_loss_clip": 1.03985381, + "balance_loss_mlp": 1.02297413, + "epoch": 0.695364636565863, + "flos": 16873203778560.0, + "grad_norm": 2.3149876666395746, + "language_loss": 0.72780073, + "learning_rate": 8.968875887901157e-07, + "loss": 0.74942911, + "num_input_tokens_seen": 124447070, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.14880371, + "step": 5783, + "time_per_iteration": 2.8070592880249023 + }, + { + "auxiliary_loss_clip": 0.01127072, + "auxiliary_loss_mlp": 0.0104042, + "balance_loss_clip": 1.04089653, + "balance_loss_mlp": 1.02553129, + "epoch": 0.6954848794565022, + "flos": 19354523299200.0, + "grad_norm": 2.0299057468849746, + "language_loss": 0.62684232, + "learning_rate": 8.9623790283582e-07, + "loss": 0.64851725, + "num_input_tokens_seen": 124464950, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.14892578, + "step": 5784, + "time_per_iteration": 2.763607978820801 + }, + { + "auxiliary_loss_clip": 0.01133382, + "auxiliary_loss_mlp": 0.01043907, + "balance_loss_clip": 1.04351974, + "balance_loss_mlp": 1.02713442, + "epoch": 0.6956051223471412, + "flos": 18990209606400.0, + "grad_norm": 2.38074063927358, + "language_loss": 0.76146138, + "learning_rate": 8.955883843213561e-07, + "loss": 0.78323424, + "num_input_tokens_seen": 124483965, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.16760254, + "step": 5785, + "time_per_iteration": 2.770319938659668 + }, + { + "auxiliary_loss_clip": 0.01132862, + "auxiliary_loss_mlp": 0.01042718, + "balance_loss_clip": 1.04191279, + "balance_loss_mlp": 1.02503908, + "epoch": 0.6957253652377803, + "flos": 16107229226880.0, + "grad_norm": 1.855910510292787, + "language_loss": 0.86842382, + "learning_rate": 8.949390333452569e-07, + "loss": 0.89017957, + "num_input_tokens_seen": 124501910, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.17675781, + "step": 5786, + "time_per_iteration": 2.8087997436523438 + }, + { + "auxiliary_loss_clip": 0.01125849, + "auxiliary_loss_mlp": 0.01040749, + "balance_loss_clip": 1.03938508, + "balance_loss_mlp": 1.02513218, + "epoch": 0.6958456081284194, + "flos": 29388646569600.0, + "grad_norm": 2.9195381197545585, + "language_loss": 0.67782861, + "learning_rate": 8.942898500060279e-07, + "loss": 0.6994946, + "num_input_tokens_seen": 124521625, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15625, + "step": 5787, + "time_per_iteration": 2.912367820739746 + }, + { + "auxiliary_loss_clip": 0.01133907, + "auxiliary_loss_mlp": 0.01046394, + "balance_loss_clip": 1.0431149, + "balance_loss_mlp": 1.03026497, + "epoch": 0.6959658510190585, + "flos": 25154850395520.0, + "grad_norm": 2.2694722346688376, + "language_loss": 0.71641099, + "learning_rate": 8.936408344021493e-07, + "loss": 0.73821402, + "num_input_tokens_seen": 124538540, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.16137695, + "step": 5788, + "time_per_iteration": 2.8566317558288574 + }, + { + "auxiliary_loss_clip": 0.01132879, + "auxiliary_loss_mlp": 0.01043734, + "balance_loss_clip": 1.04299235, + "balance_loss_mlp": 1.02682984, + "epoch": 0.6960860939096976, + "flos": 42814388759040.0, + "grad_norm": 1.9890371660473434, + "language_loss": 0.7148304, + "learning_rate": 8.929919866320765e-07, + "loss": 0.73659658, + "num_input_tokens_seen": 124559355, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.16918945, + "step": 5789, + "time_per_iteration": 3.824249267578125 + }, + { + "auxiliary_loss_clip": 0.01129216, + "auxiliary_loss_mlp": 0.01045484, + "balance_loss_clip": 1.0411936, + "balance_loss_mlp": 1.03002238, + "epoch": 0.6962063368003367, + "flos": 17566566986880.0, + "grad_norm": 1.8556369857271922, + "language_loss": 0.81690693, + "learning_rate": 8.923433067942385e-07, + "loss": 0.83865398, + "num_input_tokens_seen": 124577920, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.15466309, + "step": 5790, + "time_per_iteration": 2.8264882564544678 + }, + { + "auxiliary_loss_clip": 0.01128758, + "auxiliary_loss_mlp": 0.01037366, + "balance_loss_clip": 1.04334188, + "balance_loss_mlp": 1.02187502, + "epoch": 0.6963265796909758, + "flos": 21251648021760.0, + "grad_norm": 2.4284643737343554, + "language_loss": 0.68915832, + "learning_rate": 8.916947949870417e-07, + "loss": 0.7108196, + "num_input_tokens_seen": 124597585, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15484619, + "step": 5791, + "time_per_iteration": 2.9551727771759033 + }, + { + "auxiliary_loss_clip": 0.01033813, + "auxiliary_loss_mlp": 0.00999686, + "balance_loss_clip": 1.00652194, + "balance_loss_mlp": 0.99798125, + "epoch": 0.6964468225816148, + "flos": 68828295801600.0, + "grad_norm": 0.7466889921026812, + "language_loss": 0.5813306, + "learning_rate": 8.910464513088615e-07, + "loss": 0.60166556, + "num_input_tokens_seen": 124661625, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01708984, + "step": 5792, + "time_per_iteration": 3.4055449962615967 + }, + { + "auxiliary_loss_clip": 0.0112995, + "auxiliary_loss_mlp": 0.01037553, + "balance_loss_clip": 1.04091024, + "balance_loss_mlp": 1.02136445, + "epoch": 0.696567065472254, + "flos": 18950887192320.0, + "grad_norm": 1.8373123521938708, + "language_loss": 0.78527325, + "learning_rate": 8.903982758580542e-07, + "loss": 0.80694824, + "num_input_tokens_seen": 124680565, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.1618042, + "step": 5793, + "time_per_iteration": 3.6520376205444336 + }, + { + "auxiliary_loss_clip": 0.01131533, + "auxiliary_loss_mlp": 0.01039683, + "balance_loss_clip": 1.04164112, + "balance_loss_mlp": 1.02397132, + "epoch": 0.696687308362893, + "flos": 22856675345280.0, + "grad_norm": 1.7886665623650453, + "language_loss": 0.8031292, + "learning_rate": 8.897502687329457e-07, + "loss": 0.82484132, + "num_input_tokens_seen": 124700365, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.15722656, + "step": 5794, + "time_per_iteration": 3.7744359970092773 + }, + { + "auxiliary_loss_clip": 0.01127213, + "auxiliary_loss_mlp": 0.01039123, + "balance_loss_clip": 1.043082, + "balance_loss_mlp": 1.02337575, + "epoch": 0.6968075512535321, + "flos": 24972926987520.0, + "grad_norm": 3.284902932057809, + "language_loss": 0.7975173, + "learning_rate": 8.891024300318382e-07, + "loss": 0.81918061, + "num_input_tokens_seen": 124718935, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.15759277, + "step": 5795, + "time_per_iteration": 2.9179816246032715 + }, + { + "auxiliary_loss_clip": 0.01128163, + "auxiliary_loss_mlp": 0.01037736, + "balance_loss_clip": 1.04114413, + "balance_loss_mlp": 1.0226922, + "epoch": 0.6969277941441713, + "flos": 21030438113280.0, + "grad_norm": 1.4622855537745902, + "language_loss": 0.75716567, + "learning_rate": 8.884547598530103e-07, + "loss": 0.77882469, + "num_input_tokens_seen": 124739505, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.15039062, + "step": 5796, + "time_per_iteration": 2.900237798690796 + }, + { + "auxiliary_loss_clip": 0.01128794, + "auxiliary_loss_mlp": 0.01043794, + "balance_loss_clip": 1.04078639, + "balance_loss_mlp": 1.02663994, + "epoch": 0.6970480370348103, + "flos": 21579404647680.0, + "grad_norm": 1.7713138277330143, + "language_loss": 0.75227606, + "learning_rate": 8.8780725829471e-07, + "loss": 0.77400196, + "num_input_tokens_seen": 124757410, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.17138672, + "step": 5797, + "time_per_iteration": 2.785125255584717 + }, + { + "auxiliary_loss_clip": 0.01130224, + "auxiliary_loss_mlp": 0.01043728, + "balance_loss_clip": 1.04196119, + "balance_loss_mlp": 1.02771807, + "epoch": 0.6971682799254494, + "flos": 22419175691520.0, + "grad_norm": 1.939608233965579, + "language_loss": 0.77857882, + "learning_rate": 8.87159925455165e-07, + "loss": 0.80031836, + "num_input_tokens_seen": 124777240, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.16015625, + "step": 5798, + "time_per_iteration": 2.839512586593628 + }, + { + "auxiliary_loss_clip": 0.01125542, + "auxiliary_loss_mlp": 0.01038839, + "balance_loss_clip": 1.03936684, + "balance_loss_mlp": 1.02172065, + "epoch": 0.6972885228160886, + "flos": 20005834659840.0, + "grad_norm": 1.7784253767340894, + "language_loss": 0.73027122, + "learning_rate": 8.865127614325738e-07, + "loss": 0.75191498, + "num_input_tokens_seen": 124795670, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.17126465, + "step": 5799, + "time_per_iteration": 3.7279441356658936 + }, + { + "auxiliary_loss_clip": 0.01132424, + "auxiliary_loss_mlp": 0.01043399, + "balance_loss_clip": 1.04295909, + "balance_loss_mlp": 1.02561271, + "epoch": 0.6974087657067276, + "flos": 37853437656960.0, + "grad_norm": 1.9324677776027173, + "language_loss": 0.66600478, + "learning_rate": 8.85865766325113e-07, + "loss": 0.68776304, + "num_input_tokens_seen": 124819600, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.17785645, + "step": 5800, + "time_per_iteration": 3.0071308612823486 + }, + { + "auxiliary_loss_clip": 0.01131145, + "auxiliary_loss_mlp": 0.01041915, + "balance_loss_clip": 1.04171491, + "balance_loss_mlp": 1.0255599, + "epoch": 0.6975290085973667, + "flos": 29489267543040.0, + "grad_norm": 2.9599498119512595, + "language_loss": 0.72079331, + "learning_rate": 8.852189402309287e-07, + "loss": 0.74252385, + "num_input_tokens_seen": 124838785, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.16369629, + "step": 5801, + "time_per_iteration": 2.903792381286621 + }, + { + "auxiliary_loss_clip": 0.01132467, + "auxiliary_loss_mlp": 0.01045559, + "balance_loss_clip": 1.04210114, + "balance_loss_mlp": 1.02863109, + "epoch": 0.6976492514880057, + "flos": 12895630295040.0, + "grad_norm": 2.3974311020770687, + "language_loss": 0.74045861, + "learning_rate": 8.845722832481441e-07, + "loss": 0.76223892, + "num_input_tokens_seen": 124854215, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.16918945, + "step": 5802, + "time_per_iteration": 2.842214822769165 + }, + { + "auxiliary_loss_clip": 0.01127222, + "auxiliary_loss_mlp": 0.01038652, + "balance_loss_clip": 1.03989434, + "balance_loss_mlp": 1.02230787, + "epoch": 0.6977694943786449, + "flos": 24352929308160.0, + "grad_norm": 11.961252257235808, + "language_loss": 0.77848446, + "learning_rate": 8.83925795474858e-07, + "loss": 0.80014324, + "num_input_tokens_seen": 124874340, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.16333008, + "step": 5803, + "time_per_iteration": 2.884369373321533 + }, + { + "auxiliary_loss_clip": 0.01131773, + "auxiliary_loss_mlp": 0.01043125, + "balance_loss_clip": 1.04213941, + "balance_loss_mlp": 1.02581561, + "epoch": 0.6978897372692839, + "flos": 29898470257920.0, + "grad_norm": 2.4449465184924644, + "language_loss": 0.59528792, + "learning_rate": 8.832794770091414e-07, + "loss": 0.61703694, + "num_input_tokens_seen": 124895175, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.1730957, + "step": 5804, + "time_per_iteration": 2.927861452102661 + }, + { + "auxiliary_loss_clip": 0.01128214, + "auxiliary_loss_mlp": 0.01043456, + "balance_loss_clip": 1.03924239, + "balance_loss_mlp": 1.02695763, + "epoch": 0.698009980159923, + "flos": 21761579450880.0, + "grad_norm": 2.2115218063875774, + "language_loss": 0.82770932, + "learning_rate": 8.826333279490401e-07, + "loss": 0.84942603, + "num_input_tokens_seen": 124915810, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.16491699, + "step": 5805, + "time_per_iteration": 2.8529934883117676 + }, + { + "auxiliary_loss_clip": 0.01130394, + "auxiliary_loss_mlp": 0.01035551, + "balance_loss_clip": 1.04419041, + "balance_loss_mlp": 1.0200901, + "epoch": 0.6981302230505622, + "flos": 19857164267520.0, + "grad_norm": 2.006654749486011, + "language_loss": 0.68584991, + "learning_rate": 8.819873483925748e-07, + "loss": 0.7075094, + "num_input_tokens_seen": 124932930, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15466309, + "step": 5806, + "time_per_iteration": 2.8776769638061523 + }, + { + "auxiliary_loss_clip": 0.01129195, + "auxiliary_loss_mlp": 0.01038862, + "balance_loss_clip": 1.04160547, + "balance_loss_mlp": 1.02279222, + "epoch": 0.6982504659412012, + "flos": 22198648141440.0, + "grad_norm": 2.2987969663853245, + "language_loss": 0.74692768, + "learning_rate": 8.81341538437739e-07, + "loss": 0.76860821, + "num_input_tokens_seen": 124951220, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16064453, + "step": 5807, + "time_per_iteration": 2.9439547061920166 + }, + { + "auxiliary_loss_clip": 0.01126763, + "auxiliary_loss_mlp": 0.01033385, + "balance_loss_clip": 1.03969109, + "balance_loss_mlp": 1.01786399, + "epoch": 0.6983707088318403, + "flos": 35588479708800.0, + "grad_norm": 3.225754113570572, + "language_loss": 0.68084109, + "learning_rate": 8.80695898182503e-07, + "loss": 0.70244259, + "num_input_tokens_seen": 124972200, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.1550293, + "step": 5808, + "time_per_iteration": 3.0214195251464844 + }, + { + "auxiliary_loss_clip": 0.0103466, + "auxiliary_loss_mlp": 0.01005418, + "balance_loss_clip": 1.0074966, + "balance_loss_mlp": 1.00385666, + "epoch": 0.6984909517224794, + "flos": 65440052760960.0, + "grad_norm": 0.8403262249432787, + "language_loss": 0.65118623, + "learning_rate": 8.800504277248093e-07, + "loss": 0.67158699, + "num_input_tokens_seen": 125036950, + "router_z_loss_clip": 0.27148438, + "router_z_loss_mlp": 0.01556396, + "step": 5809, + "time_per_iteration": 3.3788673877716064 + }, + { + "auxiliary_loss_clip": 0.01130053, + "auxiliary_loss_mlp": 0.01042533, + "balance_loss_clip": 1.04182446, + "balance_loss_mlp": 1.02723813, + "epoch": 0.6986111946131185, + "flos": 18546927863040.0, + "grad_norm": 1.721816568504473, + "language_loss": 0.75328845, + "learning_rate": 8.794051271625753e-07, + "loss": 0.77501428, + "num_input_tokens_seen": 125054585, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.15283203, + "step": 5810, + "time_per_iteration": 2.815821647644043 + }, + { + "auxiliary_loss_clip": 0.01127838, + "auxiliary_loss_mlp": 0.01033461, + "balance_loss_clip": 1.04110324, + "balance_loss_mlp": 1.01875091, + "epoch": 0.6987314375037575, + "flos": 23039173370880.0, + "grad_norm": 1.5368519692990323, + "language_loss": 0.83238149, + "learning_rate": 8.787599965936925e-07, + "loss": 0.85399449, + "num_input_tokens_seen": 125075515, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.14709473, + "step": 5811, + "time_per_iteration": 2.977835178375244 + }, + { + "auxiliary_loss_clip": 0.01127848, + "auxiliary_loss_mlp": 0.01044363, + "balance_loss_clip": 1.04365194, + "balance_loss_mlp": 1.02905691, + "epoch": 0.6988516803943967, + "flos": 38400393029760.0, + "grad_norm": 1.7713191495622993, + "language_loss": 0.72431409, + "learning_rate": 8.781150361160261e-07, + "loss": 0.74603617, + "num_input_tokens_seen": 125097425, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.15289307, + "step": 5812, + "time_per_iteration": 2.9624650478363037 + }, + { + "auxiliary_loss_clip": 0.01129562, + "auxiliary_loss_mlp": 0.01037989, + "balance_loss_clip": 1.04296112, + "balance_loss_mlp": 1.02189529, + "epoch": 0.6989719232850358, + "flos": 24096993926400.0, + "grad_norm": 1.6744483063508047, + "language_loss": 0.73769456, + "learning_rate": 8.774702458274181e-07, + "loss": 0.75937009, + "num_input_tokens_seen": 125117830, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.16101074, + "step": 5813, + "time_per_iteration": 2.862886905670166 + }, + { + "auxiliary_loss_clip": 0.01128812, + "auxiliary_loss_mlp": 0.01038484, + "balance_loss_clip": 1.03844988, + "balance_loss_mlp": 1.02204525, + "epoch": 0.6990921661756748, + "flos": 14866838818560.0, + "grad_norm": 3.0250671220643164, + "language_loss": 0.70491385, + "learning_rate": 8.768256258256799e-07, + "loss": 0.72658682, + "num_input_tokens_seen": 125134455, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.16418457, + "step": 5814, + "time_per_iteration": 2.8358399868011475 + }, + { + "auxiliary_loss_clip": 0.0112857, + "auxiliary_loss_mlp": 0.01034693, + "balance_loss_clip": 1.04107046, + "balance_loss_mlp": 1.01973188, + "epoch": 0.699212409066314, + "flos": 20193719725440.0, + "grad_norm": 1.6434520991359256, + "language_loss": 0.7366156, + "learning_rate": 8.76181176208602e-07, + "loss": 0.75824821, + "num_input_tokens_seen": 125152555, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.14953613, + "step": 5815, + "time_per_iteration": 2.875298023223877 + }, + { + "auxiliary_loss_clip": 0.01128487, + "auxiliary_loss_mlp": 0.01040222, + "balance_loss_clip": 1.04000568, + "balance_loss_mlp": 1.02364028, + "epoch": 0.699332651956953, + "flos": 19427888828160.0, + "grad_norm": 1.779418016738155, + "language_loss": 0.73710525, + "learning_rate": 8.755368970739461e-07, + "loss": 0.7587924, + "num_input_tokens_seen": 125171915, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16589355, + "step": 5816, + "time_per_iteration": 2.8781208992004395 + }, + { + "auxiliary_loss_clip": 0.01129991, + "auxiliary_loss_mlp": 0.01039818, + "balance_loss_clip": 1.04031503, + "balance_loss_mlp": 1.02293813, + "epoch": 0.6994528948475921, + "flos": 16143714466560.0, + "grad_norm": 2.3015305458743733, + "language_loss": 0.61223507, + "learning_rate": 8.748927885194479e-07, + "loss": 0.63393319, + "num_input_tokens_seen": 125190220, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.16882324, + "step": 5817, + "time_per_iteration": 2.810865879058838 + }, + { + "auxiliary_loss_clip": 0.0103387, + "auxiliary_loss_mlp": 0.01000312, + "balance_loss_clip": 1.00663817, + "balance_loss_mlp": 0.99874991, + "epoch": 0.6995731377382313, + "flos": 64952420699520.0, + "grad_norm": 0.8046771079376042, + "language_loss": 0.57364404, + "learning_rate": 8.742488506428209e-07, + "loss": 0.5939858, + "num_input_tokens_seen": 125249310, + "router_z_loss_clip": 0.27148438, + "router_z_loss_mlp": 0.015625, + "step": 5818, + "time_per_iteration": 3.977159261703491 + }, + { + "auxiliary_loss_clip": 0.01125597, + "auxiliary_loss_mlp": 0.01045645, + "balance_loss_clip": 1.03964663, + "balance_loss_mlp": 1.03014731, + "epoch": 0.6996933806288703, + "flos": 24900136076160.0, + "grad_norm": 1.6796845889441894, + "language_loss": 0.77981657, + "learning_rate": 8.736050835417466e-07, + "loss": 0.80152899, + "num_input_tokens_seen": 125269350, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.1550293, + "step": 5819, + "time_per_iteration": 3.000011682510376 + }, + { + "auxiliary_loss_clip": 0.01130051, + "auxiliary_loss_mlp": 0.01037415, + "balance_loss_clip": 1.04194474, + "balance_loss_mlp": 1.02125037, + "epoch": 0.6998136235195094, + "flos": 20777806782720.0, + "grad_norm": 2.1698829500654564, + "language_loss": 0.60967392, + "learning_rate": 8.729614873138862e-07, + "loss": 0.63134849, + "num_input_tokens_seen": 125286985, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.16162109, + "step": 5820, + "time_per_iteration": 2.86002516746521 + }, + { + "auxiliary_loss_clip": 0.01132172, + "auxiliary_loss_mlp": 0.01048578, + "balance_loss_clip": 1.04136634, + "balance_loss_mlp": 1.03131664, + "epoch": 0.6999338664101485, + "flos": 23733470332800.0, + "grad_norm": 2.02397996834446, + "language_loss": 0.77613521, + "learning_rate": 8.723180620568716e-07, + "loss": 0.7979427, + "num_input_tokens_seen": 125306240, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.17266846, + "step": 5821, + "time_per_iteration": 2.95536470413208 + }, + { + "auxiliary_loss_clip": 0.01125956, + "auxiliary_loss_mlp": 0.01032278, + "balance_loss_clip": 1.04140759, + "balance_loss_mlp": 1.01775837, + "epoch": 0.7000541093007876, + "flos": 19864598382720.0, + "grad_norm": 1.7793825744230913, + "language_loss": 0.85103357, + "learning_rate": 8.716748078683116e-07, + "loss": 0.87261593, + "num_input_tokens_seen": 125323015, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14526367, + "step": 5822, + "time_per_iteration": 3.702554225921631 + }, + { + "auxiliary_loss_clip": 0.01131063, + "auxiliary_loss_mlp": 0.01042489, + "balance_loss_clip": 1.04266977, + "balance_loss_mlp": 1.02485824, + "epoch": 0.7001743521914267, + "flos": 29679056029440.0, + "grad_norm": 2.7148672118910135, + "language_loss": 0.68570423, + "learning_rate": 8.710317248457855e-07, + "loss": 0.70743978, + "num_input_tokens_seen": 125342630, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.1763916, + "step": 5823, + "time_per_iteration": 3.7289645671844482 + }, + { + "auxiliary_loss_clip": 0.01128278, + "auxiliary_loss_mlp": 0.01040455, + "balance_loss_clip": 1.04273224, + "balance_loss_mlp": 1.02436209, + "epoch": 0.7002945950820658, + "flos": 27489762080640.0, + "grad_norm": 1.736972036811543, + "language_loss": 0.72226942, + "learning_rate": 8.703888130868482e-07, + "loss": 0.7439568, + "num_input_tokens_seen": 125364480, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.16101074, + "step": 5824, + "time_per_iteration": 2.934648036956787 + }, + { + "auxiliary_loss_clip": 0.01127987, + "auxiliary_loss_mlp": 0.01040893, + "balance_loss_clip": 1.04120255, + "balance_loss_mlp": 1.02558637, + "epoch": 0.7004148379727049, + "flos": 22158463800960.0, + "grad_norm": 1.995536090644742, + "language_loss": 0.82246941, + "learning_rate": 8.697460726890307e-07, + "loss": 0.84415817, + "num_input_tokens_seen": 125381625, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.15319824, + "step": 5825, + "time_per_iteration": 2.931706666946411 + }, + { + "auxiliary_loss_clip": 0.0112795, + "auxiliary_loss_mlp": 0.0103769, + "balance_loss_clip": 1.0402534, + "balance_loss_mlp": 1.02265811, + "epoch": 0.7005350808633439, + "flos": 19423758764160.0, + "grad_norm": 2.285477458741891, + "language_loss": 0.90351093, + "learning_rate": 8.691035037498354e-07, + "loss": 0.92516732, + "num_input_tokens_seen": 125397615, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.15039062, + "step": 5826, + "time_per_iteration": 2.923409938812256 + }, + { + "auxiliary_loss_clip": 0.01126826, + "auxiliary_loss_mlp": 0.01037874, + "balance_loss_clip": 1.04113925, + "balance_loss_mlp": 1.02304435, + "epoch": 0.7006553237539831, + "flos": 23476708938240.0, + "grad_norm": 1.5963141896011162, + "language_loss": 0.72199893, + "learning_rate": 8.684611063667391e-07, + "loss": 0.74364591, + "num_input_tokens_seen": 125418080, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.1484375, + "step": 5827, + "time_per_iteration": 2.8980166912078857 + }, + { + "auxiliary_loss_clip": 0.01125391, + "auxiliary_loss_mlp": 0.01034263, + "balance_loss_clip": 1.0396111, + "balance_loss_mlp": 1.01961231, + "epoch": 0.7007755666446221, + "flos": 31212872640000.0, + "grad_norm": 1.772670555237092, + "language_loss": 0.7748493, + "learning_rate": 8.678188806371935e-07, + "loss": 0.79644579, + "num_input_tokens_seen": 125440115, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.14660645, + "step": 5828, + "time_per_iteration": 3.7774460315704346 + }, + { + "auxiliary_loss_clip": 0.01123009, + "auxiliary_loss_mlp": 0.01029067, + "balance_loss_clip": 1.0410701, + "balance_loss_mlp": 1.01507163, + "epoch": 0.7008958095352612, + "flos": 18149899858560.0, + "grad_norm": 1.7623014146565048, + "language_loss": 0.85363251, + "learning_rate": 8.671768266586228e-07, + "loss": 0.8751533, + "num_input_tokens_seen": 125458240, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.14007568, + "step": 5829, + "time_per_iteration": 2.7972114086151123 + }, + { + "auxiliary_loss_clip": 0.01132122, + "auxiliary_loss_mlp": 0.01033728, + "balance_loss_clip": 1.04494166, + "balance_loss_mlp": 1.01742041, + "epoch": 0.7010160524259004, + "flos": 27452307173760.0, + "grad_norm": 1.8495899215755185, + "language_loss": 0.78135419, + "learning_rate": 8.665349445284275e-07, + "loss": 0.80301267, + "num_input_tokens_seen": 125477980, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.16308594, + "step": 5830, + "time_per_iteration": 2.793647289276123 + }, + { + "auxiliary_loss_clip": 0.01126554, + "auxiliary_loss_mlp": 0.01043636, + "balance_loss_clip": 1.04066539, + "balance_loss_mlp": 1.02787697, + "epoch": 0.7011362953165394, + "flos": 23842064125440.0, + "grad_norm": 1.4439832317261099, + "language_loss": 0.80809784, + "learning_rate": 8.658932343439799e-07, + "loss": 0.82979977, + "num_input_tokens_seen": 125497765, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15771484, + "step": 5831, + "time_per_iteration": 2.835663318634033 + }, + { + "auxiliary_loss_clip": 0.01129723, + "auxiliary_loss_mlp": 0.01039861, + "balance_loss_clip": 1.04153204, + "balance_loss_mlp": 1.02279019, + "epoch": 0.7012565382071785, + "flos": 24823430582400.0, + "grad_norm": 1.9434640608354758, + "language_loss": 0.77784812, + "learning_rate": 8.65251696202627e-07, + "loss": 0.79954386, + "num_input_tokens_seen": 125514145, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.17077637, + "step": 5832, + "time_per_iteration": 2.828857421875 + }, + { + "auxiliary_loss_clip": 0.01128723, + "auxiliary_loss_mlp": 0.01036646, + "balance_loss_clip": 1.04166007, + "balance_loss_mlp": 1.02100551, + "epoch": 0.7013767810978175, + "flos": 21397445326080.0, + "grad_norm": 1.9029966585178024, + "language_loss": 0.87739617, + "learning_rate": 8.646103302016896e-07, + "loss": 0.89904982, + "num_input_tokens_seen": 125533115, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.15649414, + "step": 5833, + "time_per_iteration": 2.743225574493408 + }, + { + "auxiliary_loss_clip": 0.01131978, + "auxiliary_loss_mlp": 0.01036308, + "balance_loss_clip": 1.04244101, + "balance_loss_mlp": 1.01940382, + "epoch": 0.7014970239884567, + "flos": 16687150306560.0, + "grad_norm": 2.3233217281187843, + "language_loss": 0.88547254, + "learning_rate": 8.639691364384614e-07, + "loss": 0.90715539, + "num_input_tokens_seen": 125550740, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.16918945, + "step": 5834, + "time_per_iteration": 2.771934747695923 + }, + { + "auxiliary_loss_clip": 0.01131485, + "auxiliary_loss_mlp": 0.01048104, + "balance_loss_clip": 1.04228437, + "balance_loss_mlp": 1.0324167, + "epoch": 0.7016172668790958, + "flos": 12568268718720.0, + "grad_norm": 1.9286990340045962, + "language_loss": 0.72974747, + "learning_rate": 8.633281150102136e-07, + "loss": 0.7515434, + "num_input_tokens_seen": 125567590, + "router_z_loss_clip": 0.89208984, + "router_z_loss_mlp": 0.15698242, + "step": 5835, + "time_per_iteration": 2.7446281909942627 + }, + { + "auxiliary_loss_clip": 0.01128424, + "auxiliary_loss_mlp": 0.01038666, + "balance_loss_clip": 1.04193592, + "balance_loss_mlp": 1.02455139, + "epoch": 0.7017375097697348, + "flos": 17452729808640.0, + "grad_norm": 2.891557640990616, + "language_loss": 0.67503238, + "learning_rate": 8.626872660141855e-07, + "loss": 0.69670326, + "num_input_tokens_seen": 125585500, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.14123535, + "step": 5836, + "time_per_iteration": 2.751471996307373 + }, + { + "auxiliary_loss_clip": 0.011279, + "auxiliary_loss_mlp": 0.01032244, + "balance_loss_clip": 1.04200196, + "balance_loss_mlp": 1.01740289, + "epoch": 0.701857752660374, + "flos": 18513028402560.0, + "grad_norm": 3.1098604509934247, + "language_loss": 0.74670619, + "learning_rate": 8.620465895475957e-07, + "loss": 0.76830763, + "num_input_tokens_seen": 125603720, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.1484375, + "step": 5837, + "time_per_iteration": 2.8493683338165283 + }, + { + "auxiliary_loss_clip": 0.01128277, + "auxiliary_loss_mlp": 0.01037518, + "balance_loss_clip": 1.04242158, + "balance_loss_mlp": 1.02183044, + "epoch": 0.701977995551013, + "flos": 24425971614720.0, + "grad_norm": 1.5066621761207801, + "language_loss": 0.75500852, + "learning_rate": 8.614060857076333e-07, + "loss": 0.77666652, + "num_input_tokens_seen": 125624390, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15686035, + "step": 5838, + "time_per_iteration": 2.975538969039917 + }, + { + "auxiliary_loss_clip": 0.011278, + "auxiliary_loss_mlp": 0.01040246, + "balance_loss_clip": 1.0411582, + "balance_loss_mlp": 1.02479649, + "epoch": 0.7020982384416521, + "flos": 23002759958400.0, + "grad_norm": 1.8758488748038513, + "language_loss": 0.7464875, + "learning_rate": 8.60765754591462e-07, + "loss": 0.76816791, + "num_input_tokens_seen": 125644085, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15460205, + "step": 5839, + "time_per_iteration": 3.0471832752227783 + }, + { + "auxiliary_loss_clip": 0.0112748, + "auxiliary_loss_mlp": 0.0104216, + "balance_loss_clip": 1.0406692, + "balance_loss_mlp": 1.02656722, + "epoch": 0.7022184813322913, + "flos": 20449080489600.0, + "grad_norm": 1.9113492890075146, + "language_loss": 0.72909367, + "learning_rate": 8.601255962962211e-07, + "loss": 0.75079012, + "num_input_tokens_seen": 125663095, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15600586, + "step": 5840, + "time_per_iteration": 2.831897020339966 + }, + { + "auxiliary_loss_clip": 0.01136085, + "auxiliary_loss_mlp": 0.01047527, + "balance_loss_clip": 1.04345846, + "balance_loss_mlp": 1.03025365, + "epoch": 0.7023387242229303, + "flos": 19790514581760.0, + "grad_norm": 2.6145549612590466, + "language_loss": 0.71971667, + "learning_rate": 8.594856109190194e-07, + "loss": 0.74155277, + "num_input_tokens_seen": 125680125, + "router_z_loss_clip": 0.92529297, + "router_z_loss_mlp": 0.17285156, + "step": 5841, + "time_per_iteration": 2.884838581085205 + }, + { + "auxiliary_loss_clip": 0.01129673, + "auxiliary_loss_mlp": 0.01041263, + "balance_loss_clip": 1.04196167, + "balance_loss_mlp": 1.02514625, + "epoch": 0.7024589671135694, + "flos": 33259278286080.0, + "grad_norm": 1.7072271590888026, + "language_loss": 0.68880802, + "learning_rate": 8.588457985569446e-07, + "loss": 0.71051741, + "num_input_tokens_seen": 125703035, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.16113281, + "step": 5842, + "time_per_iteration": 2.9077610969543457 + }, + { + "auxiliary_loss_clip": 0.01131097, + "auxiliary_loss_mlp": 0.01046036, + "balance_loss_clip": 1.04197192, + "balance_loss_mlp": 1.02872705, + "epoch": 0.7025792100042085, + "flos": 19098982967040.0, + "grad_norm": 1.909456215419107, + "language_loss": 0.71549785, + "learning_rate": 8.582061593070542e-07, + "loss": 0.73726916, + "num_input_tokens_seen": 125723765, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.17297363, + "step": 5843, + "time_per_iteration": 2.817195415496826 + }, + { + "auxiliary_loss_clip": 0.01128981, + "auxiliary_loss_mlp": 0.01046032, + "balance_loss_clip": 1.04073083, + "balance_loss_mlp": 1.03015327, + "epoch": 0.7026994528948476, + "flos": 18952611045120.0, + "grad_norm": 2.1980247838697227, + "language_loss": 0.77251381, + "learning_rate": 8.57566693266383e-07, + "loss": 0.79426396, + "num_input_tokens_seen": 125741455, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.15881348, + "step": 5844, + "time_per_iteration": 2.823028326034546 + }, + { + "auxiliary_loss_clip": 0.01131693, + "auxiliary_loss_mlp": 0.0103827, + "balance_loss_clip": 1.04296386, + "balance_loss_mlp": 1.02252901, + "epoch": 0.7028196957854866, + "flos": 19536662188800.0, + "grad_norm": 2.060185193982046, + "language_loss": 0.68754673, + "learning_rate": 8.569274005319354e-07, + "loss": 0.7092464, + "num_input_tokens_seen": 125759855, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.15740967, + "step": 5845, + "time_per_iteration": 2.796751022338867 + }, + { + "auxiliary_loss_clip": 0.01128186, + "auxiliary_loss_mlp": 0.0103901, + "balance_loss_clip": 1.0415504, + "balance_loss_mlp": 1.02466321, + "epoch": 0.7029399386761258, + "flos": 20845318394880.0, + "grad_norm": 1.8223067833990696, + "language_loss": 0.79781473, + "learning_rate": 8.562882812006913e-07, + "loss": 0.81948662, + "num_input_tokens_seen": 125777345, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.14331055, + "step": 5846, + "time_per_iteration": 2.797842264175415 + }, + { + "auxiliary_loss_clip": 0.01127748, + "auxiliary_loss_mlp": 0.01034171, + "balance_loss_clip": 1.04117966, + "balance_loss_mlp": 1.01818502, + "epoch": 0.7030601815667649, + "flos": 22055005653120.0, + "grad_norm": 1.6640305646349123, + "language_loss": 0.77792275, + "learning_rate": 8.556493353696066e-07, + "loss": 0.79954183, + "num_input_tokens_seen": 125796345, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.15979004, + "step": 5847, + "time_per_iteration": 3.6711950302124023 + }, + { + "auxiliary_loss_clip": 0.01129645, + "auxiliary_loss_mlp": 0.01044872, + "balance_loss_clip": 1.04246759, + "balance_loss_mlp": 1.02781272, + "epoch": 0.7031804244574039, + "flos": 27198742089600.0, + "grad_norm": 2.1583001239119253, + "language_loss": 0.68794549, + "learning_rate": 8.550105631356077e-07, + "loss": 0.70969069, + "num_input_tokens_seen": 125816070, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.17053223, + "step": 5848, + "time_per_iteration": 2.8594207763671875 + }, + { + "auxiliary_loss_clip": 0.01127136, + "auxiliary_loss_mlp": 0.01037388, + "balance_loss_clip": 1.04127479, + "balance_loss_mlp": 1.02221274, + "epoch": 0.7033006673480431, + "flos": 22379853277440.0, + "grad_norm": 1.7847356365439648, + "language_loss": 0.77123535, + "learning_rate": 8.543719645955961e-07, + "loss": 0.79288054, + "num_input_tokens_seen": 125834400, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.1517334, + "step": 5849, + "time_per_iteration": 2.795356273651123 + }, + { + "auxiliary_loss_clip": 0.01129308, + "auxiliary_loss_mlp": 0.01040596, + "balance_loss_clip": 1.04054403, + "balance_loss_mlp": 1.02469373, + "epoch": 0.7034209102386821, + "flos": 24715986024960.0, + "grad_norm": 1.533918126908064, + "language_loss": 0.74471635, + "learning_rate": 8.537335398464467e-07, + "loss": 0.76641536, + "num_input_tokens_seen": 125854720, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.15917969, + "step": 5850, + "time_per_iteration": 2.9027578830718994 + }, + { + "auxiliary_loss_clip": 0.01130525, + "auxiliary_loss_mlp": 0.01043716, + "balance_loss_clip": 1.04096746, + "balance_loss_mlp": 1.02591753, + "epoch": 0.7035411531293212, + "flos": 22556174163840.0, + "grad_norm": 2.978142900931646, + "language_loss": 0.84806764, + "learning_rate": 8.53095288985007e-07, + "loss": 0.86980999, + "num_input_tokens_seen": 125868455, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.17797852, + "step": 5851, + "time_per_iteration": 3.573817491531372 + }, + { + "auxiliary_loss_clip": 0.01121741, + "auxiliary_loss_mlp": 0.01039212, + "balance_loss_clip": 1.03934073, + "balance_loss_mlp": 1.02411985, + "epoch": 0.7036613960199604, + "flos": 22674967418880.0, + "grad_norm": 1.605910169655524, + "language_loss": 0.82550782, + "learning_rate": 8.524572121081009e-07, + "loss": 0.8471173, + "num_input_tokens_seen": 125888555, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.15087891, + "step": 5852, + "time_per_iteration": 3.634896993637085 + }, + { + "auxiliary_loss_clip": 0.01133438, + "auxiliary_loss_mlp": 0.01037015, + "balance_loss_clip": 1.04295337, + "balance_loss_mlp": 1.02117229, + "epoch": 0.7037816389105994, + "flos": 22492146170880.0, + "grad_norm": 2.23277025143918, + "language_loss": 0.62142158, + "learning_rate": 8.518193093125232e-07, + "loss": 0.64312613, + "num_input_tokens_seen": 125907610, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.1583252, + "step": 5853, + "time_per_iteration": 2.8254611492156982 + }, + { + "auxiliary_loss_clip": 0.01128165, + "auxiliary_loss_mlp": 0.0104263, + "balance_loss_clip": 1.0420301, + "balance_loss_mlp": 1.02747869, + "epoch": 0.7039018818012385, + "flos": 27087490690560.0, + "grad_norm": 1.621850833025608, + "language_loss": 0.8087433, + "learning_rate": 8.511815806950436e-07, + "loss": 0.83045125, + "num_input_tokens_seen": 125928640, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.15161133, + "step": 5854, + "time_per_iteration": 2.9147653579711914 + }, + { + "auxiliary_loss_clip": 0.01124058, + "auxiliary_loss_mlp": 0.01038164, + "balance_loss_clip": 1.03873897, + "balance_loss_mlp": 1.02351332, + "epoch": 0.7040221246918776, + "flos": 17749819198080.0, + "grad_norm": 1.5781573060590233, + "language_loss": 0.78182244, + "learning_rate": 8.505440263524044e-07, + "loss": 0.80344462, + "num_input_tokens_seen": 125947485, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.14654541, + "step": 5855, + "time_per_iteration": 2.7727668285369873 + }, + { + "auxiliary_loss_clip": 0.01131465, + "auxiliary_loss_mlp": 0.01036628, + "balance_loss_clip": 1.0413264, + "balance_loss_mlp": 1.01871061, + "epoch": 0.7041423675825167, + "flos": 16279851012480.0, + "grad_norm": 2.9168571285251543, + "language_loss": 0.88008112, + "learning_rate": 8.49906646381322e-07, + "loss": 0.90176213, + "num_input_tokens_seen": 125960320, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.17919922, + "step": 5856, + "time_per_iteration": 2.790052890777588 + }, + { + "auxiliary_loss_clip": 0.01122717, + "auxiliary_loss_mlp": 0.01034219, + "balance_loss_clip": 1.03937447, + "balance_loss_mlp": 1.02016413, + "epoch": 0.7042626104731557, + "flos": 25483181639040.0, + "grad_norm": 1.7071591939353337, + "language_loss": 0.72287416, + "learning_rate": 8.492694408784884e-07, + "loss": 0.74444354, + "num_input_tokens_seen": 125980575, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.14056396, + "step": 5857, + "time_per_iteration": 3.655031681060791 + }, + { + "auxiliary_loss_clip": 0.01126561, + "auxiliary_loss_mlp": 0.01047561, + "balance_loss_clip": 1.03986287, + "balance_loss_mlp": 1.03119326, + "epoch": 0.7043828533637949, + "flos": 17857622891520.0, + "grad_norm": 2.373199091483834, + "language_loss": 0.62415695, + "learning_rate": 8.486324099405642e-07, + "loss": 0.6458981, + "num_input_tokens_seen": 125997420, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.16369629, + "step": 5858, + "time_per_iteration": 2.847883939743042 + }, + { + "auxiliary_loss_clip": 0.01124558, + "auxiliary_loss_mlp": 0.0103742, + "balance_loss_clip": 1.03850138, + "balance_loss_mlp": 1.02164829, + "epoch": 0.704503096254434, + "flos": 29494259533440.0, + "grad_norm": 1.643859949706157, + "language_loss": 0.74912888, + "learning_rate": 8.479955536641887e-07, + "loss": 0.77074867, + "num_input_tokens_seen": 126018915, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.15783691, + "step": 5859, + "time_per_iteration": 2.9174578189849854 + }, + { + "auxiliary_loss_clip": 0.01125167, + "auxiliary_loss_mlp": 0.01037338, + "balance_loss_clip": 1.04020178, + "balance_loss_mlp": 1.02085173, + "epoch": 0.704623339145073, + "flos": 30920739327360.0, + "grad_norm": 2.003059898391203, + "language_loss": 0.66485918, + "learning_rate": 8.473588721459716e-07, + "loss": 0.68648422, + "num_input_tokens_seen": 126038825, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.16479492, + "step": 5860, + "time_per_iteration": 2.9062554836273193 + }, + { + "auxiliary_loss_clip": 0.01132221, + "auxiliary_loss_mlp": 0.01040518, + "balance_loss_clip": 1.0413785, + "balance_loss_mlp": 1.023471, + "epoch": 0.7047435820357122, + "flos": 23914747296000.0, + "grad_norm": 2.332422449205699, + "language_loss": 0.70611608, + "learning_rate": 8.467223654824967e-07, + "loss": 0.72784346, + "num_input_tokens_seen": 126058280, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.17041016, + "step": 5861, + "time_per_iteration": 2.861285448074341 + }, + { + "auxiliary_loss_clip": 0.01128444, + "auxiliary_loss_mlp": 0.01032374, + "balance_loss_clip": 1.04124713, + "balance_loss_mlp": 1.01806307, + "epoch": 0.7048638249263512, + "flos": 46494010926720.0, + "grad_norm": 1.6679642919798148, + "language_loss": 0.62510931, + "learning_rate": 8.460860337703233e-07, + "loss": 0.64671749, + "num_input_tokens_seen": 126078885, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.14312744, + "step": 5862, + "time_per_iteration": 3.213026762008667 + }, + { + "auxiliary_loss_clip": 0.011314, + "auxiliary_loss_mlp": 0.01043522, + "balance_loss_clip": 1.04116631, + "balance_loss_mlp": 1.02593851, + "epoch": 0.7049840678169903, + "flos": 21689219502720.0, + "grad_norm": 1.760443508456548, + "language_loss": 0.70539999, + "learning_rate": 8.454498771059797e-07, + "loss": 0.72714925, + "num_input_tokens_seen": 126098260, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.17590332, + "step": 5863, + "time_per_iteration": 2.921999216079712 + }, + { + "auxiliary_loss_clip": 0.01128526, + "auxiliary_loss_mlp": 0.01038731, + "balance_loss_clip": 1.04056346, + "balance_loss_mlp": 1.02324557, + "epoch": 0.7051043107076294, + "flos": 18405081054720.0, + "grad_norm": 2.111179212277894, + "language_loss": 0.8329041, + "learning_rate": 8.448138955859725e-07, + "loss": 0.85457659, + "num_input_tokens_seen": 126114845, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.15496826, + "step": 5864, + "time_per_iteration": 2.862480401992798 + }, + { + "auxiliary_loss_clip": 0.01128467, + "auxiliary_loss_mlp": 0.01038454, + "balance_loss_clip": 1.04123449, + "balance_loss_mlp": 1.02248037, + "epoch": 0.7052245535982685, + "flos": 19319043640320.0, + "grad_norm": 1.8373132008437787, + "language_loss": 0.90013707, + "learning_rate": 8.44178089306778e-07, + "loss": 0.92180628, + "num_input_tokens_seen": 126132780, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.1595459, + "step": 5865, + "time_per_iteration": 2.838590383529663 + }, + { + "auxiliary_loss_clip": 0.01126797, + "auxiliary_loss_mlp": 0.01038403, + "balance_loss_clip": 1.0409466, + "balance_loss_mlp": 1.02288151, + "epoch": 0.7053447964889076, + "flos": 19062138591360.0, + "grad_norm": 1.758889305359608, + "language_loss": 0.7699995, + "learning_rate": 8.4354245836485e-07, + "loss": 0.79165149, + "num_input_tokens_seen": 126151225, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.1550293, + "step": 5866, + "time_per_iteration": 2.8844361305236816 + }, + { + "auxiliary_loss_clip": 0.01132343, + "auxiliary_loss_mlp": 0.01041528, + "balance_loss_clip": 1.04221773, + "balance_loss_mlp": 1.02563787, + "epoch": 0.7054650393795466, + "flos": 27379228953600.0, + "grad_norm": 2.2758105186687874, + "language_loss": 0.72944665, + "learning_rate": 8.429070028566108e-07, + "loss": 0.75118536, + "num_input_tokens_seen": 126172535, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.15905762, + "step": 5867, + "time_per_iteration": 2.9177613258361816 + }, + { + "auxiliary_loss_clip": 0.01127162, + "auxiliary_loss_mlp": 0.0104294, + "balance_loss_clip": 1.04211402, + "balance_loss_mlp": 1.02789617, + "epoch": 0.7055852822701858, + "flos": 16102201322880.0, + "grad_norm": 1.6847342517766022, + "language_loss": 0.74982923, + "learning_rate": 8.422717228784586e-07, + "loss": 0.77153027, + "num_input_tokens_seen": 126189410, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.15039062, + "step": 5868, + "time_per_iteration": 2.904006242752075 + }, + { + "auxiliary_loss_clip": 0.01132636, + "auxiliary_loss_mlp": 0.01039904, + "balance_loss_clip": 1.04380083, + "balance_loss_mlp": 1.02395391, + "epoch": 0.7057055251608249, + "flos": 11692299744000.0, + "grad_norm": 1.97127225079712, + "language_loss": 0.69161749, + "learning_rate": 8.416366185267663e-07, + "loss": 0.71334291, + "num_input_tokens_seen": 126206910, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.1595459, + "step": 5869, + "time_per_iteration": 2.9398608207702637 + }, + { + "auxiliary_loss_clip": 0.01122806, + "auxiliary_loss_mlp": 0.01036067, + "balance_loss_clip": 1.03952527, + "balance_loss_mlp": 1.02178597, + "epoch": 0.7058257680514639, + "flos": 22711560399360.0, + "grad_norm": 1.6703151778973184, + "language_loss": 0.77967405, + "learning_rate": 8.410016898978778e-07, + "loss": 0.80126286, + "num_input_tokens_seen": 126224385, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.14276123, + "step": 5870, + "time_per_iteration": 2.8627326488494873 + }, + { + "auxiliary_loss_clip": 0.01126869, + "auxiliary_loss_mlp": 0.01034055, + "balance_loss_clip": 1.04081655, + "balance_loss_mlp": 1.0189631, + "epoch": 0.7059460109421031, + "flos": 17529543043200.0, + "grad_norm": 1.9194103642980436, + "language_loss": 0.78997362, + "learning_rate": 8.403669370881115e-07, + "loss": 0.8115828, + "num_input_tokens_seen": 126243120, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15093994, + "step": 5871, + "time_per_iteration": 2.9213099479675293 + }, + { + "auxiliary_loss_clip": 0.01126388, + "auxiliary_loss_mlp": 0.01032857, + "balance_loss_clip": 1.04103994, + "balance_loss_mlp": 1.01750314, + "epoch": 0.7060662538327421, + "flos": 23544687427200.0, + "grad_norm": 1.607847541045821, + "language_loss": 0.78637826, + "learning_rate": 8.397323601937587e-07, + "loss": 0.8079707, + "num_input_tokens_seen": 126263020, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.15344238, + "step": 5872, + "time_per_iteration": 2.877021312713623 + }, + { + "auxiliary_loss_clip": 0.01127765, + "auxiliary_loss_mlp": 0.01039934, + "balance_loss_clip": 1.04043853, + "balance_loss_mlp": 1.02405477, + "epoch": 0.7061864967233812, + "flos": 30260736875520.0, + "grad_norm": 1.9185904555587465, + "language_loss": 0.77044594, + "learning_rate": 8.390979593110838e-07, + "loss": 0.7921229, + "num_input_tokens_seen": 126285150, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15869141, + "step": 5873, + "time_per_iteration": 2.9237685203552246 + }, + { + "auxiliary_loss_clip": 0.01131592, + "auxiliary_loss_mlp": 0.01040333, + "balance_loss_clip": 1.04178822, + "balance_loss_mlp": 1.02462113, + "epoch": 0.7063067396140204, + "flos": 20701460424960.0, + "grad_norm": 1.6267802223917287, + "language_loss": 0.81879663, + "learning_rate": 8.384637345363262e-07, + "loss": 0.84051585, + "num_input_tokens_seen": 126304340, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.15710449, + "step": 5874, + "time_per_iteration": 2.903459310531616 + }, + { + "auxiliary_loss_clip": 0.01127235, + "auxiliary_loss_mlp": 0.01032838, + "balance_loss_clip": 1.04058266, + "balance_loss_mlp": 1.01686442, + "epoch": 0.7064269825046594, + "flos": 32266168081920.0, + "grad_norm": 1.6249885610522918, + "language_loss": 0.76726335, + "learning_rate": 8.378296859656964e-07, + "loss": 0.78886408, + "num_input_tokens_seen": 126325495, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.1595459, + "step": 5875, + "time_per_iteration": 3.761852741241455 + }, + { + "auxiliary_loss_clip": 0.01126725, + "auxiliary_loss_mlp": 0.01033744, + "balance_loss_clip": 1.04226732, + "balance_loss_mlp": 1.01874757, + "epoch": 0.7065472253952985, + "flos": 30227124723840.0, + "grad_norm": 2.114114441432084, + "language_loss": 0.67924905, + "learning_rate": 8.371958136953792e-07, + "loss": 0.70085382, + "num_input_tokens_seen": 126345525, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.14996338, + "step": 5876, + "time_per_iteration": 2.874711275100708 + }, + { + "auxiliary_loss_clip": 0.01128577, + "auxiliary_loss_mlp": 0.01038158, + "balance_loss_clip": 1.04005396, + "balance_loss_mlp": 1.02179039, + "epoch": 0.7066674682859376, + "flos": 16216720859520.0, + "grad_norm": 3.5002649966282298, + "language_loss": 0.66511393, + "learning_rate": 8.365621178215326e-07, + "loss": 0.68678129, + "num_input_tokens_seen": 126361995, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.16369629, + "step": 5877, + "time_per_iteration": 2.811798334121704 + }, + { + "auxiliary_loss_clip": 0.01126735, + "auxiliary_loss_mlp": 0.01030485, + "balance_loss_clip": 1.04135191, + "balance_loss_mlp": 1.01592934, + "epoch": 0.7067877111765767, + "flos": 14830461319680.0, + "grad_norm": 1.8906099066375195, + "language_loss": 0.75104207, + "learning_rate": 8.359285984402871e-07, + "loss": 0.77261424, + "num_input_tokens_seen": 126379260, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.14562988, + "step": 5878, + "time_per_iteration": 2.8289968967437744 + }, + { + "auxiliary_loss_clip": 0.0112632, + "auxiliary_loss_mlp": 0.0103577, + "balance_loss_clip": 1.03981674, + "balance_loss_mlp": 1.01996899, + "epoch": 0.7069079540672157, + "flos": 25440196037760.0, + "grad_norm": 1.9491358062529136, + "language_loss": 0.74280024, + "learning_rate": 8.352952556477489e-07, + "loss": 0.76442111, + "num_input_tokens_seen": 126397170, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15808105, + "step": 5879, + "time_per_iteration": 2.9100048542022705 + }, + { + "auxiliary_loss_clip": 0.01127082, + "auxiliary_loss_mlp": 0.01035464, + "balance_loss_clip": 1.04054022, + "balance_loss_mlp": 1.019871, + "epoch": 0.7070281969578549, + "flos": 24607751368320.0, + "grad_norm": 1.7711807921774156, + "language_loss": 0.76848018, + "learning_rate": 8.34662089539993e-07, + "loss": 0.79010564, + "num_input_tokens_seen": 126416680, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.15588379, + "step": 5880, + "time_per_iteration": 4.214256763458252 + }, + { + "auxiliary_loss_clip": 0.01123428, + "auxiliary_loss_mlp": 0.01042032, + "balance_loss_clip": 1.03969204, + "balance_loss_mlp": 1.02662992, + "epoch": 0.707148439848494, + "flos": 26724469887360.0, + "grad_norm": 1.8879792322933904, + "language_loss": 0.79146421, + "learning_rate": 8.340291002130722e-07, + "loss": 0.81311882, + "num_input_tokens_seen": 126435870, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.15393066, + "step": 5881, + "time_per_iteration": 3.733433246612549 + }, + { + "auxiliary_loss_clip": 0.01134051, + "auxiliary_loss_mlp": 0.01040163, + "balance_loss_clip": 1.04364526, + "balance_loss_mlp": 1.02380705, + "epoch": 0.707268682739133, + "flos": 15085750256640.0, + "grad_norm": 2.5092759395320723, + "language_loss": 0.79622746, + "learning_rate": 8.3339628776301e-07, + "loss": 0.81796956, + "num_input_tokens_seen": 126454010, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.16345215, + "step": 5882, + "time_per_iteration": 3.0543954372406006 + }, + { + "auxiliary_loss_clip": 0.01122981, + "auxiliary_loss_mlp": 0.01035271, + "balance_loss_clip": 1.0388484, + "balance_loss_mlp": 1.01958323, + "epoch": 0.7073889256297722, + "flos": 34313148345600.0, + "grad_norm": 1.8758414404647192, + "language_loss": 0.56791735, + "learning_rate": 8.327636522858033e-07, + "loss": 0.58949989, + "num_input_tokens_seen": 126473615, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15698242, + "step": 5883, + "time_per_iteration": 2.981389045715332 + }, + { + "auxiliary_loss_clip": 0.01129495, + "auxiliary_loss_mlp": 0.01035211, + "balance_loss_clip": 1.04229319, + "balance_loss_mlp": 1.01860547, + "epoch": 0.7075091685204112, + "flos": 20083940784000.0, + "grad_norm": 1.802277535087184, + "language_loss": 0.77104914, + "learning_rate": 8.321311938774225e-07, + "loss": 0.79269618, + "num_input_tokens_seen": 126492705, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.16625977, + "step": 5884, + "time_per_iteration": 2.8583316802978516 + }, + { + "auxiliary_loss_clip": 0.01129616, + "auxiliary_loss_mlp": 0.01041004, + "balance_loss_clip": 1.0413897, + "balance_loss_mlp": 1.02514958, + "epoch": 0.7076294114110503, + "flos": 20777124424320.0, + "grad_norm": 2.0243081221784074, + "language_loss": 0.79518485, + "learning_rate": 8.314989126338104e-07, + "loss": 0.81689095, + "num_input_tokens_seen": 126512715, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15856934, + "step": 5885, + "time_per_iteration": 2.8837084770202637 + }, + { + "auxiliary_loss_clip": 0.01129327, + "auxiliary_loss_mlp": 0.01034867, + "balance_loss_clip": 1.04169881, + "balance_loss_mlp": 1.0181179, + "epoch": 0.7077496543016895, + "flos": 17967689141760.0, + "grad_norm": 1.7114279639768302, + "language_loss": 0.84317291, + "learning_rate": 8.308668086508847e-07, + "loss": 0.86481476, + "num_input_tokens_seen": 126530795, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16760254, + "step": 5886, + "time_per_iteration": 3.8911080360412598 + }, + { + "auxiliary_loss_clip": 0.01129205, + "auxiliary_loss_mlp": 0.01040175, + "balance_loss_clip": 1.03995156, + "balance_loss_mlp": 1.02339005, + "epoch": 0.7078698971923285, + "flos": 45478098564480.0, + "grad_norm": 1.9140288546065145, + "language_loss": 0.74047756, + "learning_rate": 8.302348820245342e-07, + "loss": 0.76217139, + "num_input_tokens_seen": 126553360, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.16772461, + "step": 5887, + "time_per_iteration": 3.037827491760254 + }, + { + "auxiliary_loss_clip": 0.01133445, + "auxiliary_loss_mlp": 0.01035985, + "balance_loss_clip": 1.04195988, + "balance_loss_mlp": 1.01837766, + "epoch": 0.7079901400829676, + "flos": 26943704547840.0, + "grad_norm": 2.4804779915203063, + "language_loss": 0.70079684, + "learning_rate": 8.296031328506232e-07, + "loss": 0.72249115, + "num_input_tokens_seen": 126573110, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.17626953, + "step": 5888, + "time_per_iteration": 2.869011878967285 + }, + { + "auxiliary_loss_clip": 0.01128023, + "auxiliary_loss_mlp": 0.01037846, + "balance_loss_clip": 1.04141438, + "balance_loss_mlp": 1.02193117, + "epoch": 0.7081103829736067, + "flos": 24423206267520.0, + "grad_norm": 1.8075352166610805, + "language_loss": 0.75452769, + "learning_rate": 8.289715612249857e-07, + "loss": 0.77618635, + "num_input_tokens_seen": 126593725, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15917969, + "step": 5889, + "time_per_iteration": 2.86964750289917 + }, + { + "auxiliary_loss_clip": 0.01128096, + "auxiliary_loss_mlp": 0.01042781, + "balance_loss_clip": 1.04137886, + "balance_loss_mlp": 1.02696204, + "epoch": 0.7082306258642458, + "flos": 18543300589440.0, + "grad_norm": 2.466600790717278, + "language_loss": 0.77587944, + "learning_rate": 8.283401672434305e-07, + "loss": 0.79758823, + "num_input_tokens_seen": 126608950, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.15808105, + "step": 5890, + "time_per_iteration": 2.7691736221313477 + }, + { + "auxiliary_loss_clip": 0.01125741, + "auxiliary_loss_mlp": 0.01041453, + "balance_loss_clip": 1.03977847, + "balance_loss_mlp": 1.02594376, + "epoch": 0.7083508687548848, + "flos": 23477534951040.0, + "grad_norm": 1.8234972715678508, + "language_loss": 0.70214176, + "learning_rate": 8.277089510017412e-07, + "loss": 0.72381365, + "num_input_tokens_seen": 126629755, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.1550293, + "step": 5891, + "time_per_iteration": 2.8110437393188477 + }, + { + "auxiliary_loss_clip": 0.01125991, + "auxiliary_loss_mlp": 0.01040446, + "balance_loss_clip": 1.04074895, + "balance_loss_mlp": 1.02578354, + "epoch": 0.708471111645524, + "flos": 22419463000320.0, + "grad_norm": 1.6583097162622584, + "language_loss": 0.82116169, + "learning_rate": 8.270779125956719e-07, + "loss": 0.84282607, + "num_input_tokens_seen": 126650135, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.14648438, + "step": 5892, + "time_per_iteration": 2.819200038909912 + }, + { + "auxiliary_loss_clip": 0.01128933, + "auxiliary_loss_mlp": 0.01041358, + "balance_loss_clip": 1.04211235, + "balance_loss_mlp": 1.02599216, + "epoch": 0.7085913545361631, + "flos": 20922885815040.0, + "grad_norm": 2.4869030955371456, + "language_loss": 0.79859996, + "learning_rate": 8.264470521209505e-07, + "loss": 0.82030278, + "num_input_tokens_seen": 126668500, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15368652, + "step": 5893, + "time_per_iteration": 2.8770363330841064 + }, + { + "auxiliary_loss_clip": 0.01126079, + "auxiliary_loss_mlp": 0.0104251, + "balance_loss_clip": 1.03865051, + "balance_loss_mlp": 1.02678657, + "epoch": 0.7087115974268021, + "flos": 15012384727680.0, + "grad_norm": 2.1171993462967524, + "language_loss": 0.7654686, + "learning_rate": 8.258163696732785e-07, + "loss": 0.78715444, + "num_input_tokens_seen": 126686090, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.1574707, + "step": 5894, + "time_per_iteration": 2.8426096439361572 + }, + { + "auxiliary_loss_clip": 0.01128923, + "auxiliary_loss_mlp": 0.01036476, + "balance_loss_clip": 1.04223323, + "balance_loss_mlp": 1.02179503, + "epoch": 0.7088318403174413, + "flos": 21539040739200.0, + "grad_norm": 1.790576889457177, + "language_loss": 0.76872951, + "learning_rate": 8.251858653483288e-07, + "loss": 0.79038352, + "num_input_tokens_seen": 126704255, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.14672852, + "step": 5895, + "time_per_iteration": 2.8038604259490967 + }, + { + "auxiliary_loss_clip": 0.0113137, + "auxiliary_loss_mlp": 0.01046968, + "balance_loss_clip": 1.04287434, + "balance_loss_mlp": 1.03149498, + "epoch": 0.7089520832080803, + "flos": 15516785462400.0, + "grad_norm": 2.115324454648993, + "language_loss": 0.86087477, + "learning_rate": 8.245555392417501e-07, + "loss": 0.88265818, + "num_input_tokens_seen": 126718910, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.15466309, + "step": 5896, + "time_per_iteration": 2.767282485961914 + }, + { + "auxiliary_loss_clip": 0.01125994, + "auxiliary_loss_mlp": 0.01035806, + "balance_loss_clip": 1.03977168, + "balance_loss_mlp": 1.0206542, + "epoch": 0.7090723260987194, + "flos": 20412667077120.0, + "grad_norm": 1.7961606957237222, + "language_loss": 0.78790367, + "learning_rate": 8.239253914491613e-07, + "loss": 0.80952162, + "num_input_tokens_seen": 126737235, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.15161133, + "step": 5897, + "time_per_iteration": 2.7862257957458496 + }, + { + "auxiliary_loss_clip": 0.01131848, + "auxiliary_loss_mlp": 0.01039998, + "balance_loss_clip": 1.04146361, + "balance_loss_mlp": 1.02328444, + "epoch": 0.7091925689893585, + "flos": 25668337271040.0, + "grad_norm": 1.829486925531743, + "language_loss": 0.75009954, + "learning_rate": 8.232954220661556e-07, + "loss": 0.77181798, + "num_input_tokens_seen": 126759970, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.16699219, + "step": 5898, + "time_per_iteration": 2.902553081512451 + }, + { + "auxiliary_loss_clip": 0.01127155, + "auxiliary_loss_mlp": 0.01041127, + "balance_loss_clip": 1.04215384, + "balance_loss_mlp": 1.02635705, + "epoch": 0.7093128118799976, + "flos": 24206629213440.0, + "grad_norm": 2.47064974747713, + "language_loss": 0.70701313, + "learning_rate": 8.226656311882989e-07, + "loss": 0.72869593, + "num_input_tokens_seen": 126779280, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.14758301, + "step": 5899, + "time_per_iteration": 2.8818397521972656 + }, + { + "auxiliary_loss_clip": 0.01124719, + "auxiliary_loss_mlp": 0.01041663, + "balance_loss_clip": 1.04114461, + "balance_loss_mlp": 1.02776277, + "epoch": 0.7094330547706367, + "flos": 16646786398080.0, + "grad_norm": 2.1705406875387663, + "language_loss": 0.76845384, + "learning_rate": 8.22036018911129e-07, + "loss": 0.79011768, + "num_input_tokens_seen": 126797310, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.13909912, + "step": 5900, + "time_per_iteration": 2.7870543003082275 + }, + { + "auxiliary_loss_clip": 0.01132988, + "auxiliary_loss_mlp": 0.01044641, + "balance_loss_clip": 1.04066658, + "balance_loss_mlp": 1.0277493, + "epoch": 0.7095532976612757, + "flos": 16283370545280.0, + "grad_norm": 2.0518931210304863, + "language_loss": 0.80330575, + "learning_rate": 8.214065853301599e-07, + "loss": 0.82508206, + "num_input_tokens_seen": 126812840, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.16906738, + "step": 5901, + "time_per_iteration": 2.816654682159424 + }, + { + "auxiliary_loss_clip": 0.01033825, + "auxiliary_loss_mlp": 0.01002336, + "balance_loss_clip": 1.00647426, + "balance_loss_mlp": 1.0008812, + "epoch": 0.7096735405519149, + "flos": 70722080559360.0, + "grad_norm": 0.8168929857103627, + "language_loss": 0.58238423, + "learning_rate": 8.207773305408734e-07, + "loss": 0.60274589, + "num_input_tokens_seen": 126880060, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01452637, + "step": 5902, + "time_per_iteration": 3.411292314529419 + }, + { + "auxiliary_loss_clip": 0.01131595, + "auxiliary_loss_mlp": 0.01035118, + "balance_loss_clip": 1.04229999, + "balance_loss_mlp": 1.01817811, + "epoch": 0.709793783442554, + "flos": 23621500661760.0, + "grad_norm": 2.8999136155676055, + "language_loss": 0.80203468, + "learning_rate": 8.201482546387288e-07, + "loss": 0.8237018, + "num_input_tokens_seen": 126899535, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.16931152, + "step": 5903, + "time_per_iteration": 2.8736345767974854 + }, + { + "auxiliary_loss_clip": 0.01128258, + "auxiliary_loss_mlp": 0.0103664, + "balance_loss_clip": 1.04219139, + "balance_loss_mlp": 1.02144122, + "epoch": 0.709914026333193, + "flos": 25993472204160.0, + "grad_norm": 1.7375791124115498, + "language_loss": 0.91951382, + "learning_rate": 8.195193577191553e-07, + "loss": 0.94116282, + "num_input_tokens_seen": 126921365, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.15209961, + "step": 5904, + "time_per_iteration": 3.6709609031677246 + }, + { + "auxiliary_loss_clip": 0.01130842, + "auxiliary_loss_mlp": 0.01032284, + "balance_loss_clip": 1.04021168, + "balance_loss_mlp": 1.01707244, + "epoch": 0.7100342692238322, + "flos": 24861531934080.0, + "grad_norm": 1.782530091245025, + "language_loss": 0.84607446, + "learning_rate": 8.188906398775579e-07, + "loss": 0.8677057, + "num_input_tokens_seen": 126941910, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.15197754, + "step": 5905, + "time_per_iteration": 2.787123441696167 + }, + { + "auxiliary_loss_clip": 0.01129868, + "auxiliary_loss_mlp": 0.01036164, + "balance_loss_clip": 1.04154801, + "balance_loss_mlp": 1.01997542, + "epoch": 0.7101545121144712, + "flos": 24932203943040.0, + "grad_norm": 2.280465697909395, + "language_loss": 0.68501079, + "learning_rate": 8.18262101209311e-07, + "loss": 0.70667106, + "num_input_tokens_seen": 126961120, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.1619873, + "step": 5906, + "time_per_iteration": 2.83288836479187 + }, + { + "auxiliary_loss_clip": 0.01126756, + "auxiliary_loss_mlp": 0.01039829, + "balance_loss_clip": 1.04076457, + "balance_loss_mlp": 1.02425396, + "epoch": 0.7102747550051103, + "flos": 23768842250880.0, + "grad_norm": 1.7420885187393023, + "language_loss": 0.70154732, + "learning_rate": 8.176337418097626e-07, + "loss": 0.72321314, + "num_input_tokens_seen": 126981590, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15557861, + "step": 5907, + "time_per_iteration": 2.846829414367676 + }, + { + "auxiliary_loss_clip": 0.01129072, + "auxiliary_loss_mlp": 0.01046918, + "balance_loss_clip": 1.04228497, + "balance_loss_mlp": 1.02990723, + "epoch": 0.7103949978957494, + "flos": 15303907509120.0, + "grad_norm": 2.106277525773937, + "language_loss": 0.79480135, + "learning_rate": 8.170055617742364e-07, + "loss": 0.81656122, + "num_input_tokens_seen": 126998870, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.16992188, + "step": 5908, + "time_per_iteration": 2.813659906387329 + }, + { + "auxiliary_loss_clip": 0.01129274, + "auxiliary_loss_mlp": 0.01043276, + "balance_loss_clip": 1.04129767, + "balance_loss_mlp": 1.02650356, + "epoch": 0.7105152407863885, + "flos": 22638805401600.0, + "grad_norm": 1.9003217890216957, + "language_loss": 0.70912445, + "learning_rate": 8.163775611980252e-07, + "loss": 0.73084998, + "num_input_tokens_seen": 127017980, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.16772461, + "step": 5909, + "time_per_iteration": 4.4286370277404785 + }, + { + "auxiliary_loss_clip": 0.01128749, + "auxiliary_loss_mlp": 0.01040167, + "balance_loss_clip": 1.04154515, + "balance_loss_mlp": 1.02435946, + "epoch": 0.7106354836770276, + "flos": 17238594879360.0, + "grad_norm": 1.5856492824871864, + "language_loss": 0.78522909, + "learning_rate": 8.157497401763982e-07, + "loss": 0.80691826, + "num_input_tokens_seen": 127035645, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.1583252, + "step": 5910, + "time_per_iteration": 2.9423367977142334 + }, + { + "auxiliary_loss_clip": 0.01127787, + "auxiliary_loss_mlp": 0.01038414, + "balance_loss_clip": 1.04061222, + "balance_loss_mlp": 1.02264285, + "epoch": 0.7107557265676667, + "flos": 20193647898240.0, + "grad_norm": 1.9622659032915384, + "language_loss": 0.77707529, + "learning_rate": 8.151220988045935e-07, + "loss": 0.79873729, + "num_input_tokens_seen": 127054900, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.15783691, + "step": 5911, + "time_per_iteration": 2.9189581871032715 + }, + { + "auxiliary_loss_clip": 0.0112614, + "auxiliary_loss_mlp": 0.01040446, + "balance_loss_clip": 1.04049158, + "balance_loss_mlp": 1.02572393, + "epoch": 0.7108759694583058, + "flos": 21507080613120.0, + "grad_norm": 1.701191207247149, + "language_loss": 0.82676351, + "learning_rate": 8.144946371778234e-07, + "loss": 0.84842944, + "num_input_tokens_seen": 127075010, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.14733887, + "step": 5912, + "time_per_iteration": 2.9144763946533203 + }, + { + "auxiliary_loss_clip": 0.01126669, + "auxiliary_loss_mlp": 0.01037285, + "balance_loss_clip": 1.04049635, + "balance_loss_mlp": 1.02177644, + "epoch": 0.7109962123489448, + "flos": 24061909317120.0, + "grad_norm": 1.8372776844600616, + "language_loss": 0.7826345, + "learning_rate": 8.138673553912751e-07, + "loss": 0.80427408, + "num_input_tokens_seen": 127095570, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.1550293, + "step": 5913, + "time_per_iteration": 2.8713550567626953 + }, + { + "auxiliary_loss_clip": 0.0113262, + "auxiliary_loss_mlp": 0.01049023, + "balance_loss_clip": 1.04313898, + "balance_loss_mlp": 1.03345454, + "epoch": 0.711116455239584, + "flos": 30480474326400.0, + "grad_norm": 2.776315079603336, + "language_loss": 0.57113218, + "learning_rate": 8.132402535401059e-07, + "loss": 0.59294868, + "num_input_tokens_seen": 127116825, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.15563965, + "step": 5914, + "time_per_iteration": 2.8750216960906982 + }, + { + "auxiliary_loss_clip": 0.0112689, + "auxiliary_loss_mlp": 0.01034399, + "balance_loss_clip": 1.04080653, + "balance_loss_mlp": 1.01905668, + "epoch": 0.711236698130223, + "flos": 25045610158080.0, + "grad_norm": 2.1948285492208677, + "language_loss": 0.74315745, + "learning_rate": 8.126133317194465e-07, + "loss": 0.76477039, + "num_input_tokens_seen": 127137015, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.15338135, + "step": 5915, + "time_per_iteration": 3.7818117141723633 + }, + { + "auxiliary_loss_clip": 0.01133069, + "auxiliary_loss_mlp": 0.01043622, + "balance_loss_clip": 1.04283261, + "balance_loss_mlp": 1.02584791, + "epoch": 0.7113569410208621, + "flos": 24206701040640.0, + "grad_norm": 1.7840293098608488, + "language_loss": 0.74218369, + "learning_rate": 8.11986590024401e-07, + "loss": 0.76395065, + "num_input_tokens_seen": 127156755, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.1776123, + "step": 5916, + "time_per_iteration": 2.8836662769317627 + }, + { + "auxiliary_loss_clip": 0.0113063, + "auxiliary_loss_mlp": 0.01039338, + "balance_loss_clip": 1.04229701, + "balance_loss_mlp": 1.02341175, + "epoch": 0.7114771839115013, + "flos": 35439306526080.0, + "grad_norm": 1.552078221378223, + "language_loss": 0.68946564, + "learning_rate": 8.113600285500442e-07, + "loss": 0.71116531, + "num_input_tokens_seen": 127176965, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15942383, + "step": 5917, + "time_per_iteration": 3.031179189682007 + }, + { + "auxiliary_loss_clip": 0.01128238, + "auxiliary_loss_mlp": 0.01038028, + "balance_loss_clip": 1.04100227, + "balance_loss_mlp": 1.02344871, + "epoch": 0.7115974268021403, + "flos": 21099458096640.0, + "grad_norm": 1.688758483630296, + "language_loss": 0.74297637, + "learning_rate": 8.107336473914268e-07, + "loss": 0.76463902, + "num_input_tokens_seen": 127195595, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.14575195, + "step": 5918, + "time_per_iteration": 2.9350709915161133 + }, + { + "auxiliary_loss_clip": 0.0103409, + "auxiliary_loss_mlp": 0.01000393, + "balance_loss_clip": 1.00673056, + "balance_loss_mlp": 0.99880713, + "epoch": 0.7117176696927794, + "flos": 56752866616320.0, + "grad_norm": 0.8093445041427834, + "language_loss": 0.55712491, + "learning_rate": 8.101074466435694e-07, + "loss": 0.57746977, + "num_input_tokens_seen": 127255070, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01586914, + "step": 5919, + "time_per_iteration": 3.3042287826538086 + }, + { + "auxiliary_loss_clip": 0.01129295, + "auxiliary_loss_mlp": 0.01044805, + "balance_loss_clip": 1.04202414, + "balance_loss_mlp": 1.02819908, + "epoch": 0.7118379125834186, + "flos": 15925269905280.0, + "grad_norm": 1.8337632440692866, + "language_loss": 0.67956936, + "learning_rate": 8.094814264014662e-07, + "loss": 0.7013104, + "num_input_tokens_seen": 127273825, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.1661377, + "step": 5920, + "time_per_iteration": 3.0041332244873047 + }, + { + "auxiliary_loss_clip": 0.01132268, + "auxiliary_loss_mlp": 0.01041427, + "balance_loss_clip": 1.04056334, + "balance_loss_mlp": 1.0235579, + "epoch": 0.7119581554740576, + "flos": 20193360589440.0, + "grad_norm": 2.5032773931856203, + "language_loss": 0.81635624, + "learning_rate": 8.088555867600844e-07, + "loss": 0.83809316, + "num_input_tokens_seen": 127289990, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.17871094, + "step": 5921, + "time_per_iteration": 3.009889602661133 + }, + { + "auxiliary_loss_clip": 0.01125177, + "auxiliary_loss_mlp": 0.01043011, + "balance_loss_clip": 1.04297805, + "balance_loss_mlp": 1.02923036, + "epoch": 0.7120783983646967, + "flos": 34715383822080.0, + "grad_norm": 1.7198332040370257, + "language_loss": 0.60559905, + "learning_rate": 8.08229927814362e-07, + "loss": 0.62728089, + "num_input_tokens_seen": 127312880, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.13787842, + "step": 5922, + "time_per_iteration": 3.096867561340332 + }, + { + "auxiliary_loss_clip": 0.01126119, + "auxiliary_loss_mlp": 0.01036527, + "balance_loss_clip": 1.04208922, + "balance_loss_mlp": 1.02187657, + "epoch": 0.7121986412553358, + "flos": 26359114700160.0, + "grad_norm": 1.8595598455848676, + "language_loss": 0.64890146, + "learning_rate": 8.076044496592134e-07, + "loss": 0.67052794, + "num_input_tokens_seen": 127334730, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.14660645, + "step": 5923, + "time_per_iteration": 3.1001827716827393 + }, + { + "auxiliary_loss_clip": 0.01128724, + "auxiliary_loss_mlp": 0.01041763, + "balance_loss_clip": 1.04172206, + "balance_loss_mlp": 1.02569413, + "epoch": 0.7123188841459749, + "flos": 11145344371200.0, + "grad_norm": 2.2149301866716398, + "language_loss": 0.7815336, + "learning_rate": 8.069791523895204e-07, + "loss": 0.80323839, + "num_input_tokens_seen": 127351180, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.16064453, + "step": 5924, + "time_per_iteration": 2.9578216075897217 + }, + { + "auxiliary_loss_clip": 0.01127862, + "auxiliary_loss_mlp": 0.0103504, + "balance_loss_clip": 1.04095674, + "balance_loss_mlp": 1.02003157, + "epoch": 0.7124391270366139, + "flos": 20811670329600.0, + "grad_norm": 1.8373153982174701, + "language_loss": 0.77458954, + "learning_rate": 8.063540361001422e-07, + "loss": 0.79621857, + "num_input_tokens_seen": 127369750, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.15014648, + "step": 5925, + "time_per_iteration": 3.014526128768921 + }, + { + "auxiliary_loss_clip": 0.01129446, + "auxiliary_loss_mlp": 0.01044499, + "balance_loss_clip": 1.04202139, + "balance_loss_mlp": 1.02873945, + "epoch": 0.7125593699272531, + "flos": 17603734584960.0, + "grad_norm": 2.101804334841545, + "language_loss": 0.7992698, + "learning_rate": 8.057291008859069e-07, + "loss": 0.82100928, + "num_input_tokens_seen": 127387910, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15759277, + "step": 5926, + "time_per_iteration": 2.9604203701019287 + }, + { + "auxiliary_loss_clip": 0.0112583, + "auxiliary_loss_mlp": 0.01033856, + "balance_loss_clip": 1.04077697, + "balance_loss_mlp": 1.01933622, + "epoch": 0.7126796128178922, + "flos": 28654057526400.0, + "grad_norm": 1.8863980383799, + "language_loss": 0.67856359, + "learning_rate": 8.051043468416187e-07, + "loss": 0.70016038, + "num_input_tokens_seen": 127409160, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.1451416, + "step": 5927, + "time_per_iteration": 2.954728841781616 + }, + { + "auxiliary_loss_clip": 0.01124778, + "auxiliary_loss_mlp": 0.01039652, + "balance_loss_clip": 1.04123735, + "balance_loss_mlp": 1.02517998, + "epoch": 0.7127998557085312, + "flos": 16034438315520.0, + "grad_norm": 1.9126329260423605, + "language_loss": 0.81948268, + "learning_rate": 8.044797740620506e-07, + "loss": 0.84112692, + "num_input_tokens_seen": 127427765, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.14483643, + "step": 5928, + "time_per_iteration": 2.8773179054260254 + }, + { + "auxiliary_loss_clip": 0.01128754, + "auxiliary_loss_mlp": 0.01036834, + "balance_loss_clip": 1.04264069, + "balance_loss_mlp": 1.0216291, + "epoch": 0.7129200985991703, + "flos": 23403271582080.0, + "grad_norm": 1.982636062627902, + "language_loss": 0.78646016, + "learning_rate": 8.038553826419494e-07, + "loss": 0.80811602, + "num_input_tokens_seen": 127446475, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.15216064, + "step": 5929, + "time_per_iteration": 2.8614442348480225 + }, + { + "auxiliary_loss_clip": 0.01127989, + "auxiliary_loss_mlp": 0.0103782, + "balance_loss_clip": 1.03989077, + "balance_loss_mlp": 1.02231729, + "epoch": 0.7130403414898094, + "flos": 21397445326080.0, + "grad_norm": 2.812651511824449, + "language_loss": 0.80726391, + "learning_rate": 8.032311726760364e-07, + "loss": 0.82892203, + "num_input_tokens_seen": 127467695, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.15509033, + "step": 5930, + "time_per_iteration": 2.7946550846099854 + }, + { + "auxiliary_loss_clip": 0.01129389, + "auxiliary_loss_mlp": 0.01042207, + "balance_loss_clip": 1.0411787, + "balance_loss_mlp": 1.02471924, + "epoch": 0.7131605843804485, + "flos": 74739045306240.0, + "grad_norm": 1.6359390931529658, + "language_loss": 0.68872029, + "learning_rate": 8.026071442590022e-07, + "loss": 0.71043622, + "num_input_tokens_seen": 127494590, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.17504883, + "step": 5931, + "time_per_iteration": 3.1486940383911133 + }, + { + "auxiliary_loss_clip": 0.0112578, + "auxiliary_loss_mlp": 0.01039797, + "balance_loss_clip": 1.04019749, + "balance_loss_mlp": 1.02504444, + "epoch": 0.7132808272710875, + "flos": 18368739469440.0, + "grad_norm": 1.8894697056431637, + "language_loss": 0.80210459, + "learning_rate": 8.019832974855134e-07, + "loss": 0.82376039, + "num_input_tokens_seen": 127512550, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.14752197, + "step": 5932, + "time_per_iteration": 2.765995979309082 + }, + { + "auxiliary_loss_clip": 0.0113133, + "auxiliary_loss_mlp": 0.01047445, + "balance_loss_clip": 1.042557, + "balance_loss_mlp": 1.03158998, + "epoch": 0.7134010701617267, + "flos": 23253380127360.0, + "grad_norm": 2.238531405186849, + "language_loss": 0.82436085, + "learning_rate": 8.013596324502052e-07, + "loss": 0.84614861, + "num_input_tokens_seen": 127531015, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.15856934, + "step": 5933, + "time_per_iteration": 2.8433570861816406 + }, + { + "auxiliary_loss_clip": 0.01123848, + "auxiliary_loss_mlp": 0.01037843, + "balance_loss_clip": 1.04089832, + "balance_loss_mlp": 1.02257252, + "epoch": 0.7135213130523658, + "flos": 23653137565440.0, + "grad_norm": 1.9495766373326398, + "language_loss": 0.78550017, + "learning_rate": 8.007361492476872e-07, + "loss": 0.8071171, + "num_input_tokens_seen": 127550340, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.15270996, + "step": 5934, + "time_per_iteration": 3.632214069366455 + }, + { + "auxiliary_loss_clip": 0.01128934, + "auxiliary_loss_mlp": 0.01044843, + "balance_loss_clip": 1.04102993, + "balance_loss_mlp": 1.02904761, + "epoch": 0.7136415559430048, + "flos": 24790644443520.0, + "grad_norm": 1.5996326916066368, + "language_loss": 0.78875637, + "learning_rate": 8.001128479725426e-07, + "loss": 0.81049418, + "num_input_tokens_seen": 127572245, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.15795898, + "step": 5935, + "time_per_iteration": 2.9089620113372803 + }, + { + "auxiliary_loss_clip": 0.01130379, + "auxiliary_loss_mlp": 0.01040098, + "balance_loss_clip": 1.0419414, + "balance_loss_mlp": 1.02449906, + "epoch": 0.713761798833644, + "flos": 18296954138880.0, + "grad_norm": 1.5590058141244167, + "language_loss": 0.81439918, + "learning_rate": 7.994897287193248e-07, + "loss": 0.83610392, + "num_input_tokens_seen": 127591625, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.15606689, + "step": 5936, + "time_per_iteration": 2.8098857402801514 + }, + { + "auxiliary_loss_clip": 0.01129805, + "auxiliary_loss_mlp": 0.01043709, + "balance_loss_clip": 1.04204655, + "balance_loss_mlp": 1.0278064, + "epoch": 0.713882041724283, + "flos": 15558262692480.0, + "grad_norm": 4.8019114512213275, + "language_loss": 0.83819413, + "learning_rate": 7.988667915825605e-07, + "loss": 0.85992932, + "num_input_tokens_seen": 127608690, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.15905762, + "step": 5937, + "time_per_iteration": 3.6425812244415283 + }, + { + "auxiliary_loss_clip": 0.01131351, + "auxiliary_loss_mlp": 0.01044527, + "balance_loss_clip": 1.04174304, + "balance_loss_mlp": 1.02885127, + "epoch": 0.7140022846149221, + "flos": 24061011477120.0, + "grad_norm": 1.9843478773974925, + "language_loss": 0.75436246, + "learning_rate": 7.982440366567491e-07, + "loss": 0.7761212, + "num_input_tokens_seen": 127627180, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.15686035, + "step": 5938, + "time_per_iteration": 3.69498348236084 + }, + { + "auxiliary_loss_clip": 0.01124642, + "auxiliary_loss_mlp": 0.01033978, + "balance_loss_clip": 1.04017091, + "balance_loss_mlp": 1.01952362, + "epoch": 0.7141225275055613, + "flos": 27891710248320.0, + "grad_norm": 1.6086674522916482, + "language_loss": 0.75294518, + "learning_rate": 7.97621464036361e-07, + "loss": 0.77453136, + "num_input_tokens_seen": 127648940, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.14459229, + "step": 5939, + "time_per_iteration": 2.9120125770568848 + }, + { + "auxiliary_loss_clip": 0.01127141, + "auxiliary_loss_mlp": 0.01042009, + "balance_loss_clip": 1.04014409, + "balance_loss_mlp": 1.02682137, + "epoch": 0.7142427703962003, + "flos": 19682603147520.0, + "grad_norm": 1.6183793699708686, + "language_loss": 0.67922539, + "learning_rate": 7.969990738158417e-07, + "loss": 0.70091689, + "num_input_tokens_seen": 127667350, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15185547, + "step": 5940, + "time_per_iteration": 2.902846336364746 + }, + { + "auxiliary_loss_clip": 0.01132868, + "auxiliary_loss_mlp": 0.01042787, + "balance_loss_clip": 1.04403639, + "balance_loss_mlp": 1.02603841, + "epoch": 0.7143630132868394, + "flos": 21032377447680.0, + "grad_norm": 1.8544489451408124, + "language_loss": 0.84643984, + "learning_rate": 7.963768660896062e-07, + "loss": 0.86819631, + "num_input_tokens_seen": 127685760, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.1673584, + "step": 5941, + "time_per_iteration": 2.851943016052246 + }, + { + "auxiliary_loss_clip": 0.01125114, + "auxiliary_loss_mlp": 0.010367, + "balance_loss_clip": 1.04046297, + "balance_loss_mlp": 1.02047586, + "epoch": 0.7144832561774785, + "flos": 24129923719680.0, + "grad_norm": 1.8513175733951122, + "language_loss": 0.82839537, + "learning_rate": 7.957548409520432e-07, + "loss": 0.85001349, + "num_input_tokens_seen": 127704985, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.16235352, + "step": 5942, + "time_per_iteration": 2.8600499629974365 + }, + { + "auxiliary_loss_clip": 0.01124279, + "auxiliary_loss_mlp": 0.01037592, + "balance_loss_clip": 1.03947616, + "balance_loss_mlp": 1.02302456, + "epoch": 0.7146034990681176, + "flos": 16325817442560.0, + "grad_norm": 2.1970779962934843, + "language_loss": 0.8394717, + "learning_rate": 7.951329984975135e-07, + "loss": 0.86109042, + "num_input_tokens_seen": 127721925, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.14581299, + "step": 5943, + "time_per_iteration": 2.899027109146118 + }, + { + "auxiliary_loss_clip": 0.01035187, + "auxiliary_loss_mlp": 0.01003054, + "balance_loss_clip": 1.00792122, + "balance_loss_mlp": 1.00152838, + "epoch": 0.7147237419587567, + "flos": 69627164232960.0, + "grad_norm": 0.7175163472494102, + "language_loss": 0.54307818, + "learning_rate": 7.94511338820349e-07, + "loss": 0.56346059, + "num_input_tokens_seen": 127784230, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01525879, + "step": 5944, + "time_per_iteration": 4.240453243255615 + }, + { + "auxiliary_loss_clip": 0.01134029, + "auxiliary_loss_mlp": 0.01040877, + "balance_loss_clip": 1.04128361, + "balance_loss_mlp": 1.02436614, + "epoch": 0.7148439848493958, + "flos": 22266806198400.0, + "grad_norm": 1.886821712206307, + "language_loss": 0.78517002, + "learning_rate": 7.938898620148575e-07, + "loss": 0.8069191, + "num_input_tokens_seen": 127801990, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.16516113, + "step": 5945, + "time_per_iteration": 2.945906162261963 + }, + { + "auxiliary_loss_clip": 0.01129275, + "auxiliary_loss_mlp": 0.01043533, + "balance_loss_clip": 1.04188728, + "balance_loss_mlp": 1.02823806, + "epoch": 0.7149642277400349, + "flos": 17931383470080.0, + "grad_norm": 2.1329959163169057, + "language_loss": 0.71274531, + "learning_rate": 7.932685681753135e-07, + "loss": 0.73447335, + "num_input_tokens_seen": 127819270, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.15301514, + "step": 5946, + "time_per_iteration": 2.928093910217285 + }, + { + "auxiliary_loss_clip": 0.01124531, + "auxiliary_loss_mlp": 0.01036408, + "balance_loss_clip": 1.04132533, + "balance_loss_mlp": 1.02229333, + "epoch": 0.7150844706306739, + "flos": 31681937370240.0, + "grad_norm": 1.865373745643973, + "language_loss": 0.62770712, + "learning_rate": 7.92647457395969e-07, + "loss": 0.64931649, + "num_input_tokens_seen": 127841095, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.14117432, + "step": 5947, + "time_per_iteration": 2.8647773265838623 + }, + { + "auxiliary_loss_clip": 0.0112798, + "auxiliary_loss_mlp": 0.01037877, + "balance_loss_clip": 1.0416466, + "balance_loss_mlp": 1.02156961, + "epoch": 0.7152047135213131, + "flos": 10926217451520.0, + "grad_norm": 2.3294856196534623, + "language_loss": 0.74289435, + "learning_rate": 7.920265297710444e-07, + "loss": 0.76455295, + "num_input_tokens_seen": 127858485, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.16290283, + "step": 5948, + "time_per_iteration": 2.812243700027466 + }, + { + "auxiliary_loss_clip": 0.01125018, + "auxiliary_loss_mlp": 0.0103608, + "balance_loss_clip": 1.03992915, + "balance_loss_mlp": 1.02079725, + "epoch": 0.7153249564119522, + "flos": 20995640812800.0, + "grad_norm": 1.8416003742993539, + "language_loss": 0.73508191, + "learning_rate": 7.914057853947363e-07, + "loss": 0.75669289, + "num_input_tokens_seen": 127877665, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.15264893, + "step": 5949, + "time_per_iteration": 2.782787561416626 + }, + { + "auxiliary_loss_clip": 0.01128649, + "auxiliary_loss_mlp": 0.010435, + "balance_loss_clip": 1.03942847, + "balance_loss_mlp": 1.02771688, + "epoch": 0.7154451993025912, + "flos": 24243114453120.0, + "grad_norm": 1.7037252357020316, + "language_loss": 0.62279499, + "learning_rate": 7.907852243612089e-07, + "loss": 0.64451647, + "num_input_tokens_seen": 127898070, + "router_z_loss_clip": 0.89208984, + "router_z_loss_mlp": 0.15783691, + "step": 5950, + "time_per_iteration": 2.8541054725646973 + }, + { + "auxiliary_loss_clip": 0.01123331, + "auxiliary_loss_mlp": 0.01040497, + "balance_loss_clip": 1.03930306, + "balance_loss_mlp": 1.02566791, + "epoch": 0.7155654421932304, + "flos": 23330947547520.0, + "grad_norm": 1.6343764370693843, + "language_loss": 0.7239697, + "learning_rate": 7.901648467646009e-07, + "loss": 0.74560797, + "num_input_tokens_seen": 127917010, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14813232, + "step": 5951, + "time_per_iteration": 2.80835223197937 + }, + { + "auxiliary_loss_clip": 0.01129952, + "auxiliary_loss_mlp": 0.0103709, + "balance_loss_clip": 1.04099965, + "balance_loss_mlp": 1.02109206, + "epoch": 0.7156856850838694, + "flos": 22711883621760.0, + "grad_norm": 1.626976001755788, + "language_loss": 0.72509563, + "learning_rate": 7.895446526990244e-07, + "loss": 0.74676609, + "num_input_tokens_seen": 127937025, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.15991211, + "step": 5952, + "time_per_iteration": 2.8154146671295166 + }, + { + "auxiliary_loss_clip": 0.01129146, + "auxiliary_loss_mlp": 0.01041543, + "balance_loss_clip": 1.04047787, + "balance_loss_mlp": 1.02573574, + "epoch": 0.7158059279745085, + "flos": 19865424395520.0, + "grad_norm": 1.6253452432234345, + "language_loss": 0.75544548, + "learning_rate": 7.889246422585609e-07, + "loss": 0.7771523, + "num_input_tokens_seen": 127956410, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.15795898, + "step": 5953, + "time_per_iteration": 2.749788999557495 + }, + { + "auxiliary_loss_clip": 0.01126806, + "auxiliary_loss_mlp": 0.01042291, + "balance_loss_clip": 1.03990412, + "balance_loss_mlp": 1.02682972, + "epoch": 0.7159261708651476, + "flos": 24134772055680.0, + "grad_norm": 1.9299603952412017, + "language_loss": 0.73551852, + "learning_rate": 7.883048155372675e-07, + "loss": 0.75720948, + "num_input_tokens_seen": 127974925, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15454102, + "step": 5954, + "time_per_iteration": 2.837574005126953 + }, + { + "auxiliary_loss_clip": 0.01131183, + "auxiliary_loss_mlp": 0.0103704, + "balance_loss_clip": 1.04347849, + "balance_loss_mlp": 1.02051759, + "epoch": 0.7160464137557867, + "flos": 16983198201600.0, + "grad_norm": 2.6493257571954074, + "language_loss": 0.71549684, + "learning_rate": 7.876851726291698e-07, + "loss": 0.73717904, + "num_input_tokens_seen": 127993225, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16516113, + "step": 5955, + "time_per_iteration": 2.7796318531036377 + }, + { + "auxiliary_loss_clip": 0.01131319, + "auxiliary_loss_mlp": 0.01043206, + "balance_loss_clip": 1.04490697, + "balance_loss_mlp": 1.02837622, + "epoch": 0.7161666566464258, + "flos": 25228251838080.0, + "grad_norm": 1.967222792470497, + "language_loss": 0.78655314, + "learning_rate": 7.870657136282666e-07, + "loss": 0.80829835, + "num_input_tokens_seen": 128012085, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.14807129, + "step": 5956, + "time_per_iteration": 2.812600612640381 + }, + { + "auxiliary_loss_clip": 0.01124842, + "auxiliary_loss_mlp": 0.01042771, + "balance_loss_clip": 1.0401032, + "balance_loss_mlp": 1.02679706, + "epoch": 0.7162868995370649, + "flos": 26468390851200.0, + "grad_norm": 1.9383118391704046, + "language_loss": 0.82079393, + "learning_rate": 7.86446438628531e-07, + "loss": 0.84247005, + "num_input_tokens_seen": 128033155, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15966797, + "step": 5957, + "time_per_iteration": 2.888664960861206 + }, + { + "auxiliary_loss_clip": 0.01034846, + "auxiliary_loss_mlp": 0.01000212, + "balance_loss_clip": 1.00734484, + "balance_loss_mlp": 0.99872172, + "epoch": 0.716407142427704, + "flos": 69998912040960.0, + "grad_norm": 0.7710070365210373, + "language_loss": 0.5687148, + "learning_rate": 7.858273477239059e-07, + "loss": 0.58906537, + "num_input_tokens_seen": 128101575, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01489258, + "step": 5958, + "time_per_iteration": 3.3188202381134033 + }, + { + "auxiliary_loss_clip": 0.01129998, + "auxiliary_loss_mlp": 0.01039809, + "balance_loss_clip": 1.04349613, + "balance_loss_mlp": 1.02506328, + "epoch": 0.716527385318343, + "flos": 20740459616640.0, + "grad_norm": 4.620711477532143, + "language_loss": 0.71366298, + "learning_rate": 7.852084410083067e-07, + "loss": 0.73536098, + "num_input_tokens_seen": 128120395, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.14746094, + "step": 5959, + "time_per_iteration": 2.8346457481384277 + }, + { + "auxiliary_loss_clip": 0.01121778, + "auxiliary_loss_mlp": 0.01036862, + "balance_loss_clip": 1.04039526, + "balance_loss_mlp": 1.02261043, + "epoch": 0.7166476282089821, + "flos": 25371966153600.0, + "grad_norm": 1.5492218586418678, + "language_loss": 0.6384356, + "learning_rate": 7.84589718575621e-07, + "loss": 0.66002202, + "num_input_tokens_seen": 128140840, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.14251709, + "step": 5960, + "time_per_iteration": 2.7972071170806885 + }, + { + "auxiliary_loss_clip": 0.01129107, + "auxiliary_loss_mlp": 0.01036164, + "balance_loss_clip": 1.03883028, + "balance_loss_mlp": 1.02069068, + "epoch": 0.7167678710996213, + "flos": 24133730561280.0, + "grad_norm": 1.9410934223602394, + "language_loss": 0.69170713, + "learning_rate": 7.83971180519708e-07, + "loss": 0.71335977, + "num_input_tokens_seen": 128159695, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.15478516, + "step": 5961, + "time_per_iteration": 2.7979836463928223 + }, + { + "auxiliary_loss_clip": 0.01127761, + "auxiliary_loss_mlp": 0.01035082, + "balance_loss_clip": 1.04083323, + "balance_loss_mlp": 1.01808274, + "epoch": 0.7168881139902603, + "flos": 30226586019840.0, + "grad_norm": 2.586862547895537, + "language_loss": 0.75567937, + "learning_rate": 7.833528269344008e-07, + "loss": 0.77730775, + "num_input_tokens_seen": 128179600, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.17004395, + "step": 5962, + "time_per_iteration": 3.6825931072235107 + }, + { + "auxiliary_loss_clip": 0.011313, + "auxiliary_loss_mlp": 0.01041254, + "balance_loss_clip": 1.04190326, + "balance_loss_mlp": 1.0250653, + "epoch": 0.7170083568808994, + "flos": 14606414236800.0, + "grad_norm": 3.2804533740818425, + "language_loss": 0.77528667, + "learning_rate": 7.827346579135023e-07, + "loss": 0.79701221, + "num_input_tokens_seen": 128196940, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.16174316, + "step": 5963, + "time_per_iteration": 2.7448699474334717 + }, + { + "auxiliary_loss_clip": 0.01127416, + "auxiliary_loss_mlp": 0.01038607, + "balance_loss_clip": 1.04031551, + "balance_loss_mlp": 1.02241886, + "epoch": 0.7171285997715385, + "flos": 23331091201920.0, + "grad_norm": 2.094313253497727, + "language_loss": 0.83142978, + "learning_rate": 7.821166735507885e-07, + "loss": 0.85308999, + "num_input_tokens_seen": 128215970, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.16186523, + "step": 5964, + "time_per_iteration": 2.8266401290893555 + }, + { + "auxiliary_loss_clip": 0.01123799, + "auxiliary_loss_mlp": 0.01038575, + "balance_loss_clip": 1.03872228, + "balance_loss_mlp": 1.02319717, + "epoch": 0.7172488426621776, + "flos": 16543543731840.0, + "grad_norm": 1.6018790324878682, + "language_loss": 0.68703598, + "learning_rate": 7.81498873940007e-07, + "loss": 0.70865977, + "num_input_tokens_seen": 128233185, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15368652, + "step": 5965, + "time_per_iteration": 2.791905403137207 + }, + { + "auxiliary_loss_clip": 0.01131407, + "auxiliary_loss_mlp": 0.01039403, + "balance_loss_clip": 1.04033637, + "balance_loss_mlp": 1.02327371, + "epoch": 0.7173690855528166, + "flos": 26541612725760.0, + "grad_norm": 5.563222496284378, + "language_loss": 0.7776469, + "learning_rate": 7.808812591748768e-07, + "loss": 0.79935503, + "num_input_tokens_seen": 128253565, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.16113281, + "step": 5966, + "time_per_iteration": 3.6489245891571045 + }, + { + "auxiliary_loss_clip": 0.01128368, + "auxiliary_loss_mlp": 0.01041138, + "balance_loss_clip": 1.04036498, + "balance_loss_mlp": 1.02448463, + "epoch": 0.7174893284434558, + "flos": 22784099915520.0, + "grad_norm": 2.005320807933065, + "language_loss": 0.64779657, + "learning_rate": 7.802638293490915e-07, + "loss": 0.66949165, + "num_input_tokens_seen": 128273210, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.16662598, + "step": 5967, + "time_per_iteration": 3.623067617416382 + }, + { + "auxiliary_loss_clip": 0.01130272, + "auxiliary_loss_mlp": 0.01044653, + "balance_loss_clip": 1.04496121, + "balance_loss_mlp": 1.02889299, + "epoch": 0.7176095713340949, + "flos": 23293564467840.0, + "grad_norm": 1.5614161055742908, + "language_loss": 0.77017128, + "learning_rate": 7.796465845563123e-07, + "loss": 0.79192048, + "num_input_tokens_seen": 128292085, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15759277, + "step": 5968, + "time_per_iteration": 2.808654308319092 + }, + { + "auxiliary_loss_clip": 0.01128148, + "auxiliary_loss_mlp": 0.01039174, + "balance_loss_clip": 1.04239738, + "balance_loss_mlp": 1.02360559, + "epoch": 0.7177298142247339, + "flos": 25591631777280.0, + "grad_norm": 1.8921857964766984, + "language_loss": 0.79152024, + "learning_rate": 7.790295248901766e-07, + "loss": 0.8131935, + "num_input_tokens_seen": 128313215, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.15551758, + "step": 5969, + "time_per_iteration": 3.0166072845458984 + }, + { + "auxiliary_loss_clip": 0.01127893, + "auxiliary_loss_mlp": 0.01039798, + "balance_loss_clip": 1.04209459, + "balance_loss_mlp": 1.02474189, + "epoch": 0.7178500571153731, + "flos": 31652778504960.0, + "grad_norm": 1.7457358143663508, + "language_loss": 0.62226689, + "learning_rate": 7.784126504442902e-07, + "loss": 0.64394385, + "num_input_tokens_seen": 128336445, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15063477, + "step": 5970, + "time_per_iteration": 3.0676076412200928 + }, + { + "auxiliary_loss_clip": 0.01126976, + "auxiliary_loss_mlp": 0.0103609, + "balance_loss_clip": 1.04027843, + "balance_loss_mlp": 1.02092671, + "epoch": 0.7179703000060121, + "flos": 19427242383360.0, + "grad_norm": 1.422950228982964, + "language_loss": 0.67719853, + "learning_rate": 7.777959613122351e-07, + "loss": 0.69882917, + "num_input_tokens_seen": 128356270, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.15148926, + "step": 5971, + "time_per_iteration": 2.934359312057495 + }, + { + "auxiliary_loss_clip": 0.0112298, + "auxiliary_loss_mlp": 0.01037291, + "balance_loss_clip": 1.03917122, + "balance_loss_mlp": 1.02291417, + "epoch": 0.7180905428966512, + "flos": 28839249072000.0, + "grad_norm": 1.5924621678076762, + "language_loss": 0.77695572, + "learning_rate": 7.771794575875604e-07, + "loss": 0.79855841, + "num_input_tokens_seen": 128378140, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.14367676, + "step": 5972, + "time_per_iteration": 2.829531669616699 + }, + { + "auxiliary_loss_clip": 0.01132697, + "auxiliary_loss_mlp": 0.01043987, + "balance_loss_clip": 1.04284096, + "balance_loss_mlp": 1.02560496, + "epoch": 0.7182107857872904, + "flos": 20047563285120.0, + "grad_norm": 2.1500302735932166, + "language_loss": 0.7769044, + "learning_rate": 7.765631393637888e-07, + "loss": 0.79867125, + "num_input_tokens_seen": 128396335, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.18383789, + "step": 5973, + "time_per_iteration": 3.650592088699341 + }, + { + "auxiliary_loss_clip": 0.01131119, + "auxiliary_loss_mlp": 0.01036805, + "balance_loss_clip": 1.04209042, + "balance_loss_mlp": 1.02061582, + "epoch": 0.7183310286779294, + "flos": 22747686503040.0, + "grad_norm": 5.49646995317298, + "language_loss": 0.48806179, + "learning_rate": 7.75947006734417e-07, + "loss": 0.50974107, + "num_input_tokens_seen": 128414115, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.16210938, + "step": 5974, + "time_per_iteration": 2.848252296447754 + }, + { + "auxiliary_loss_clip": 0.01127672, + "auxiliary_loss_mlp": 0.01040473, + "balance_loss_clip": 1.04011118, + "balance_loss_mlp": 1.02377188, + "epoch": 0.7184512715685685, + "flos": 17158262112000.0, + "grad_norm": 2.3769206383282344, + "language_loss": 0.82822168, + "learning_rate": 7.753310597929101e-07, + "loss": 0.84990323, + "num_input_tokens_seen": 128430755, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.16699219, + "step": 5975, + "time_per_iteration": 2.8109195232391357 + }, + { + "auxiliary_loss_clip": 0.0103571, + "auxiliary_loss_mlp": 0.01001288, + "balance_loss_clip": 1.00813055, + "balance_loss_mlp": 0.99984592, + "epoch": 0.7185715144592076, + "flos": 65509611448320.0, + "grad_norm": 0.7544335871917156, + "language_loss": 0.5510006, + "learning_rate": 7.747152986327095e-07, + "loss": 0.5713706, + "num_input_tokens_seen": 128491300, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.0144043, + "step": 5976, + "time_per_iteration": 3.2060976028442383 + }, + { + "auxiliary_loss_clip": 0.01127628, + "auxiliary_loss_mlp": 0.01040884, + "balance_loss_clip": 1.04171586, + "balance_loss_mlp": 1.02440977, + "epoch": 0.7186917573498467, + "flos": 16180522928640.0, + "grad_norm": 1.6409544122613113, + "language_loss": 0.67876095, + "learning_rate": 7.740997233472228e-07, + "loss": 0.70044613, + "num_input_tokens_seen": 128508920, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.16503906, + "step": 5977, + "time_per_iteration": 2.8679933547973633 + }, + { + "auxiliary_loss_clip": 0.01122632, + "auxiliary_loss_mlp": 0.01032863, + "balance_loss_clip": 1.03915286, + "balance_loss_mlp": 1.01836109, + "epoch": 0.7188120002404857, + "flos": 29242274647680.0, + "grad_norm": 2.057776897155618, + "language_loss": 0.70693481, + "learning_rate": 7.734843340298329e-07, + "loss": 0.72848976, + "num_input_tokens_seen": 128528745, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.14508057, + "step": 5978, + "time_per_iteration": 2.891627550125122 + }, + { + "auxiliary_loss_clip": 0.01132456, + "auxiliary_loss_mlp": 0.01036423, + "balance_loss_clip": 1.04210246, + "balance_loss_mlp": 1.02079511, + "epoch": 0.7189322431311249, + "flos": 33401161008000.0, + "grad_norm": 4.040895294153145, + "language_loss": 0.74905312, + "learning_rate": 7.72869130773895e-07, + "loss": 0.77074194, + "num_input_tokens_seen": 128549345, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.15618896, + "step": 5979, + "time_per_iteration": 2.973703622817993 + }, + { + "auxiliary_loss_clip": 0.01035539, + "auxiliary_loss_mlp": 0.00999871, + "balance_loss_clip": 1.00819695, + "balance_loss_mlp": 0.99842244, + "epoch": 0.719052486021764, + "flos": 61351263792000.0, + "grad_norm": 0.7881289918775671, + "language_loss": 0.59377992, + "learning_rate": 7.722541136727343e-07, + "loss": 0.61413407, + "num_input_tokens_seen": 128605360, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01446533, + "step": 5980, + "time_per_iteration": 3.2142906188964844 + }, + { + "auxiliary_loss_clip": 0.01126404, + "auxiliary_loss_mlp": 0.01036888, + "balance_loss_clip": 1.04017258, + "balance_loss_mlp": 1.02133691, + "epoch": 0.719172728912403, + "flos": 15596795007360.0, + "grad_norm": 1.8238454742323849, + "language_loss": 0.80797207, + "learning_rate": 7.716392828196483e-07, + "loss": 0.82960498, + "num_input_tokens_seen": 128623160, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15557861, + "step": 5981, + "time_per_iteration": 2.868818998336792 + }, + { + "auxiliary_loss_clip": 0.01127635, + "auxiliary_loss_mlp": 0.01037298, + "balance_loss_clip": 1.04044342, + "balance_loss_mlp": 1.02119267, + "epoch": 0.7192929718030422, + "flos": 15553162961280.0, + "grad_norm": 2.2052420124666807, + "language_loss": 0.77596283, + "learning_rate": 7.710246383079064e-07, + "loss": 0.79761213, + "num_input_tokens_seen": 128638545, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.16101074, + "step": 5982, + "time_per_iteration": 2.8939766883850098 + }, + { + "auxiliary_loss_clip": 0.01125058, + "auxiliary_loss_mlp": 0.01037993, + "balance_loss_clip": 1.03920722, + "balance_loss_mlp": 1.02302659, + "epoch": 0.7194132146936812, + "flos": 21862487733120.0, + "grad_norm": 2.309470250950531, + "language_loss": 0.91488314, + "learning_rate": 7.704101802307492e-07, + "loss": 0.93651372, + "num_input_tokens_seen": 128650845, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.14959717, + "step": 5983, + "time_per_iteration": 2.865612268447876 + }, + { + "auxiliary_loss_clip": 0.01129828, + "auxiliary_loss_mlp": 0.01032575, + "balance_loss_clip": 1.043033, + "balance_loss_mlp": 1.0170424, + "epoch": 0.7195334575843203, + "flos": 27338900958720.0, + "grad_norm": 2.0101749648800644, + "language_loss": 0.87171257, + "learning_rate": 7.697959086813912e-07, + "loss": 0.89333665, + "num_input_tokens_seen": 128667010, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.15527344, + "step": 5984, + "time_per_iteration": 2.983144760131836 + }, + { + "auxiliary_loss_clip": 0.01125261, + "auxiliary_loss_mlp": 0.01035304, + "balance_loss_clip": 1.04109037, + "balance_loss_mlp": 1.01977062, + "epoch": 0.7196537004749595, + "flos": 18770615809920.0, + "grad_norm": 1.9119831400340384, + "language_loss": 0.80043149, + "learning_rate": 7.691818237530145e-07, + "loss": 0.8220371, + "num_input_tokens_seen": 128685870, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.15527344, + "step": 5985, + "time_per_iteration": 2.9091455936431885 + }, + { + "auxiliary_loss_clip": 0.01131218, + "auxiliary_loss_mlp": 0.01035442, + "balance_loss_clip": 1.04388297, + "balance_loss_mlp": 1.02043343, + "epoch": 0.7197739433655985, + "flos": 24531009960960.0, + "grad_norm": 1.9770635057963437, + "language_loss": 0.77598345, + "learning_rate": 7.685679255387774e-07, + "loss": 0.79765004, + "num_input_tokens_seen": 128704185, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.15014648, + "step": 5986, + "time_per_iteration": 2.8460092544555664 + }, + { + "auxiliary_loss_clip": 0.01126625, + "auxiliary_loss_mlp": 0.01033781, + "balance_loss_clip": 1.04024422, + "balance_loss_mlp": 1.01826024, + "epoch": 0.7198941862562376, + "flos": 18040587793920.0, + "grad_norm": 1.8608121314796728, + "language_loss": 0.76611924, + "learning_rate": 7.679542141318065e-07, + "loss": 0.7877233, + "num_input_tokens_seen": 128721290, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.1552124, + "step": 5987, + "time_per_iteration": 2.806286096572876 + }, + { + "auxiliary_loss_clip": 0.01127003, + "auxiliary_loss_mlp": 0.01039153, + "balance_loss_clip": 1.04156709, + "balance_loss_mlp": 1.02307177, + "epoch": 0.7200144291468767, + "flos": 29022393542400.0, + "grad_norm": 1.7694451286409338, + "language_loss": 0.76345932, + "learning_rate": 7.673406896252013e-07, + "loss": 0.7851209, + "num_input_tokens_seen": 128742665, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.16088867, + "step": 5988, + "time_per_iteration": 2.83132004737854 + }, + { + "auxiliary_loss_clip": 0.01129412, + "auxiliary_loss_mlp": 0.01049115, + "balance_loss_clip": 1.04121542, + "balance_loss_mlp": 1.03146005, + "epoch": 0.7201346720375158, + "flos": 25374264624000.0, + "grad_norm": 1.548537267527806, + "language_loss": 0.78462553, + "learning_rate": 7.667273521120347e-07, + "loss": 0.80641085, + "num_input_tokens_seen": 128762225, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.17675781, + "step": 5989, + "time_per_iteration": 2.8045315742492676 + }, + { + "auxiliary_loss_clip": 0.01130401, + "auxiliary_loss_mlp": 0.01040959, + "balance_loss_clip": 1.04192364, + "balance_loss_mlp": 1.02490187, + "epoch": 0.7202549149281549, + "flos": 14355614499840.0, + "grad_norm": 2.0678053159035454, + "language_loss": 0.79739559, + "learning_rate": 7.661142016853468e-07, + "loss": 0.81910926, + "num_input_tokens_seen": 128779585, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.1605835, + "step": 5990, + "time_per_iteration": 2.75866961479187 + }, + { + "auxiliary_loss_clip": 0.01125401, + "auxiliary_loss_mlp": 0.01037795, + "balance_loss_clip": 1.04027081, + "balance_loss_mlp": 1.02245271, + "epoch": 0.7203751578187939, + "flos": 23001682550400.0, + "grad_norm": 1.7305028060909882, + "language_loss": 0.74877548, + "learning_rate": 7.655012384381543e-07, + "loss": 0.77040744, + "num_input_tokens_seen": 128799070, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15356445, + "step": 5991, + "time_per_iteration": 3.545517921447754 + }, + { + "auxiliary_loss_clip": 0.01127243, + "auxiliary_loss_mlp": 0.01046236, + "balance_loss_clip": 1.04061937, + "balance_loss_mlp": 1.03069091, + "epoch": 0.7204954007094331, + "flos": 23692424065920.0, + "grad_norm": 1.8033741944512323, + "language_loss": 0.82137549, + "learning_rate": 7.648884624634415e-07, + "loss": 0.84311032, + "num_input_tokens_seen": 128817620, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15527344, + "step": 5992, + "time_per_iteration": 2.817272663116455 + }, + { + "auxiliary_loss_clip": 0.01129124, + "auxiliary_loss_mlp": 0.01036646, + "balance_loss_clip": 1.04191887, + "balance_loss_mlp": 1.02169681, + "epoch": 0.7206156436000721, + "flos": 16253026531200.0, + "grad_norm": 1.847486992613768, + "language_loss": 0.89103514, + "learning_rate": 7.642758738541683e-07, + "loss": 0.91269284, + "num_input_tokens_seen": 128834200, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.14941406, + "step": 5993, + "time_per_iteration": 2.7914702892303467 + }, + { + "auxiliary_loss_clip": 0.01035503, + "auxiliary_loss_mlp": 0.01000686, + "balance_loss_clip": 1.00784802, + "balance_loss_mlp": 0.9992916, + "epoch": 0.7207358864907112, + "flos": 54377806504320.0, + "grad_norm": 0.763873666713765, + "language_loss": 0.60730612, + "learning_rate": 7.636634727032621e-07, + "loss": 0.62766802, + "num_input_tokens_seen": 128891305, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01397705, + "step": 5994, + "time_per_iteration": 3.232517957687378 + }, + { + "auxiliary_loss_clip": 0.01131967, + "auxiliary_loss_mlp": 0.01049875, + "balance_loss_clip": 1.0421896, + "balance_loss_mlp": 1.03329253, + "epoch": 0.7208561293813504, + "flos": 19135540033920.0, + "grad_norm": 1.9565868564370605, + "language_loss": 0.78933793, + "learning_rate": 7.630512591036231e-07, + "loss": 0.81115639, + "num_input_tokens_seen": 128910615, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.16601562, + "step": 5995, + "time_per_iteration": 3.784881114959717 + }, + { + "auxiliary_loss_clip": 0.01127155, + "auxiliary_loss_mlp": 0.01049907, + "balance_loss_clip": 1.04052103, + "balance_loss_mlp": 1.03470814, + "epoch": 0.7209763722719894, + "flos": 17748526308480.0, + "grad_norm": 2.117189912576532, + "language_loss": 0.642986, + "learning_rate": 7.624392331481255e-07, + "loss": 0.66475666, + "num_input_tokens_seen": 128928270, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15209961, + "step": 5996, + "time_per_iteration": 3.6864757537841797 + }, + { + "auxiliary_loss_clip": 0.01035474, + "auxiliary_loss_mlp": 0.01003077, + "balance_loss_clip": 1.00774884, + "balance_loss_mlp": 1.00176609, + "epoch": 0.7210966151626285, + "flos": 66819488716800.0, + "grad_norm": 0.7396076411940034, + "language_loss": 0.51851422, + "learning_rate": 7.618273949296115e-07, + "loss": 0.53889972, + "num_input_tokens_seen": 128987780, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01312256, + "step": 5997, + "time_per_iteration": 3.2103989124298096 + }, + { + "auxiliary_loss_clip": 0.01125359, + "auxiliary_loss_mlp": 0.01039833, + "balance_loss_clip": 1.03779769, + "balance_loss_mlp": 1.02409744, + "epoch": 0.7212168580532676, + "flos": 21141869080320.0, + "grad_norm": 1.9604008907808486, + "language_loss": 0.68667746, + "learning_rate": 7.612157445408987e-07, + "loss": 0.70832938, + "num_input_tokens_seen": 129005590, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.15734863, + "step": 5998, + "time_per_iteration": 2.8797237873077393 + }, + { + "auxiliary_loss_clip": 0.01130341, + "auxiliary_loss_mlp": 0.01036776, + "balance_loss_clip": 1.04230797, + "balance_loss_mlp": 1.02088547, + "epoch": 0.7213371009439067, + "flos": 22345738335360.0, + "grad_norm": 2.9359053528614503, + "language_loss": 0.74782473, + "learning_rate": 7.606042820747716e-07, + "loss": 0.7694959, + "num_input_tokens_seen": 129021995, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.15869141, + "step": 5999, + "time_per_iteration": 2.8298118114471436 + }, + { + "auxiliary_loss_clip": 0.01131499, + "auxiliary_loss_mlp": 0.01036399, + "balance_loss_clip": 1.04371119, + "balance_loss_mlp": 1.02121139, + "epoch": 0.7214573438345457, + "flos": 18515901490560.0, + "grad_norm": 1.6698103613248099, + "language_loss": 0.85411203, + "learning_rate": 7.599930076239889e-07, + "loss": 0.87579095, + "num_input_tokens_seen": 129039280, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.15185547, + "step": 6000, + "time_per_iteration": 2.8506014347076416 + }, + { + "auxiliary_loss_clip": 0.01129486, + "auxiliary_loss_mlp": 0.01043832, + "balance_loss_clip": 1.04183459, + "balance_loss_mlp": 1.02673793, + "epoch": 0.7215775867251849, + "flos": 35736108606720.0, + "grad_norm": 1.8783332537625483, + "language_loss": 0.70277655, + "learning_rate": 7.593819212812818e-07, + "loss": 0.72450972, + "num_input_tokens_seen": 129060860, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.17102051, + "step": 6001, + "time_per_iteration": 2.910001516342163 + }, + { + "auxiliary_loss_clip": 0.01125904, + "auxiliary_loss_mlp": 0.01037087, + "balance_loss_clip": 1.04015529, + "balance_loss_mlp": 1.02198291, + "epoch": 0.721697829615824, + "flos": 20372410909440.0, + "grad_norm": 1.7394591221744726, + "language_loss": 0.71845961, + "learning_rate": 7.587710231393508e-07, + "loss": 0.74008948, + "num_input_tokens_seen": 129079215, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.15100098, + "step": 6002, + "time_per_iteration": 3.7328219413757324 + }, + { + "auxiliary_loss_clip": 0.01127179, + "auxiliary_loss_mlp": 0.01036584, + "balance_loss_clip": 1.03878129, + "balance_loss_mlp": 1.01984668, + "epoch": 0.721818072506463, + "flos": 20229809915520.0, + "grad_norm": 2.1154933127371724, + "language_loss": 0.83773464, + "learning_rate": 7.581603132908685e-07, + "loss": 0.85937226, + "num_input_tokens_seen": 129097185, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16748047, + "step": 6003, + "time_per_iteration": 2.8711466789245605 + }, + { + "auxiliary_loss_clip": 0.01130093, + "auxiliary_loss_mlp": 0.01040793, + "balance_loss_clip": 1.04110658, + "balance_loss_mlp": 1.02468824, + "epoch": 0.7219383153971022, + "flos": 18186887888640.0, + "grad_norm": 1.8055907418230288, + "language_loss": 0.78502864, + "learning_rate": 7.575497918284795e-07, + "loss": 0.80673754, + "num_input_tokens_seen": 129114730, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.16101074, + "step": 6004, + "time_per_iteration": 2.8406364917755127 + }, + { + "auxiliary_loss_clip": 0.01130267, + "auxiliary_loss_mlp": 0.01043604, + "balance_loss_clip": 1.0397166, + "balance_loss_mlp": 1.02711749, + "epoch": 0.7220585582877412, + "flos": 17342124854400.0, + "grad_norm": 2.1468728892186535, + "language_loss": 0.74103075, + "learning_rate": 7.569394588447984e-07, + "loss": 0.76276946, + "num_input_tokens_seen": 129131745, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.16503906, + "step": 6005, + "time_per_iteration": 2.911843776702881 + }, + { + "auxiliary_loss_clip": 0.01127056, + "auxiliary_loss_mlp": 0.01040288, + "balance_loss_clip": 1.0407176, + "balance_loss_mlp": 1.025172, + "epoch": 0.7221788011783803, + "flos": 16976338704000.0, + "grad_norm": 2.240685667365878, + "language_loss": 0.77919137, + "learning_rate": 7.563293144324146e-07, + "loss": 0.80086482, + "num_input_tokens_seen": 129147295, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.15112305, + "step": 6006, + "time_per_iteration": 2.790712594985962 + }, + { + "auxiliary_loss_clip": 0.01123647, + "auxiliary_loss_mlp": 0.0103534, + "balance_loss_clip": 1.04062963, + "balance_loss_mlp": 1.02139854, + "epoch": 0.7222990440690195, + "flos": 26286359702400.0, + "grad_norm": 1.8252583551991646, + "language_loss": 0.80462277, + "learning_rate": 7.557193586838834e-07, + "loss": 0.82621264, + "num_input_tokens_seen": 129162660, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.13928223, + "step": 6007, + "time_per_iteration": 2.860520362854004 + }, + { + "auxiliary_loss_clip": 0.01128506, + "auxiliary_loss_mlp": 0.01039377, + "balance_loss_clip": 1.03983045, + "balance_loss_mlp": 1.02445841, + "epoch": 0.7224192869596585, + "flos": 17601687509760.0, + "grad_norm": 2.2081651139246388, + "language_loss": 0.70720857, + "learning_rate": 7.551095916917371e-07, + "loss": 0.72888744, + "num_input_tokens_seen": 129179990, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.14923096, + "step": 6008, + "time_per_iteration": 2.7705507278442383 + }, + { + "auxiliary_loss_clip": 0.01136334, + "auxiliary_loss_mlp": 0.0104583, + "balance_loss_clip": 1.04273069, + "balance_loss_mlp": 1.02862859, + "epoch": 0.7225395298502976, + "flos": 12932331016320.0, + "grad_norm": 2.25030603173637, + "language_loss": 0.66380906, + "learning_rate": 7.545000135484758e-07, + "loss": 0.68563068, + "num_input_tokens_seen": 129197425, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.17199707, + "step": 6009, + "time_per_iteration": 2.7971696853637695 + }, + { + "auxiliary_loss_clip": 0.01130337, + "auxiliary_loss_mlp": 0.01047059, + "balance_loss_clip": 1.04250443, + "balance_loss_mlp": 1.03138328, + "epoch": 0.7226597727409367, + "flos": 29643899592960.0, + "grad_norm": 2.2367954585040786, + "language_loss": 0.62806624, + "learning_rate": 7.538906243465714e-07, + "loss": 0.64984018, + "num_input_tokens_seen": 129217560, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15686035, + "step": 6010, + "time_per_iteration": 2.884129524230957 + }, + { + "auxiliary_loss_clip": 0.01128861, + "auxiliary_loss_mlp": 0.01037616, + "balance_loss_clip": 1.04144955, + "balance_loss_mlp": 1.02178478, + "epoch": 0.7227800156315758, + "flos": 13771635183360.0, + "grad_norm": 4.024204031774477, + "language_loss": 0.78597766, + "learning_rate": 7.5328142417847e-07, + "loss": 0.80764246, + "num_input_tokens_seen": 129234325, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.15844727, + "step": 6011, + "time_per_iteration": 2.7307746410369873 + }, + { + "auxiliary_loss_clip": 0.01127945, + "auxiliary_loss_mlp": 0.01034385, + "balance_loss_clip": 1.04105949, + "balance_loss_mlp": 1.01922131, + "epoch": 0.7229002585222148, + "flos": 20301882554880.0, + "grad_norm": 1.783535555522393, + "language_loss": 0.69270074, + "learning_rate": 7.526724131365838e-07, + "loss": 0.71432412, + "num_input_tokens_seen": 129255280, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.15179443, + "step": 6012, + "time_per_iteration": 2.789933204650879 + }, + { + "auxiliary_loss_clip": 0.01132057, + "auxiliary_loss_mlp": 0.01044304, + "balance_loss_clip": 1.04363084, + "balance_loss_mlp": 1.02762699, + "epoch": 0.723020501412854, + "flos": 16581250033920.0, + "grad_norm": 1.91765330123076, + "language_loss": 0.70593017, + "learning_rate": 7.520635913133017e-07, + "loss": 0.7276938, + "num_input_tokens_seen": 129273910, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.16687012, + "step": 6013, + "time_per_iteration": 2.8755977153778076 + }, + { + "auxiliary_loss_clip": 0.01131108, + "auxiliary_loss_mlp": 0.01041565, + "balance_loss_clip": 1.04031134, + "balance_loss_mlp": 1.02452958, + "epoch": 0.7231407443034931, + "flos": 28548300908160.0, + "grad_norm": 1.8724007053057083, + "language_loss": 0.82526475, + "learning_rate": 7.514549588009798e-07, + "loss": 0.84699148, + "num_input_tokens_seen": 129294785, + "router_z_loss_clip": 0.90722656, + "router_z_loss_mlp": 0.17028809, + "step": 6014, + "time_per_iteration": 2.8976590633392334 + }, + { + "auxiliary_loss_clip": 0.01132502, + "auxiliary_loss_mlp": 0.01049064, + "balance_loss_clip": 1.0421226, + "balance_loss_mlp": 1.03348339, + "epoch": 0.7232609871941321, + "flos": 30008536508160.0, + "grad_norm": 2.0653371261775493, + "language_loss": 0.70769429, + "learning_rate": 7.508465156919492e-07, + "loss": 0.72951001, + "num_input_tokens_seen": 129318295, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.15576172, + "step": 6015, + "time_per_iteration": 2.7891266345977783 + }, + { + "auxiliary_loss_clip": 0.0112974, + "auxiliary_loss_mlp": 0.0103272, + "balance_loss_clip": 1.04213262, + "balance_loss_mlp": 1.01753259, + "epoch": 0.7233812300847713, + "flos": 16654005031680.0, + "grad_norm": 3.1708013537276933, + "language_loss": 0.6115818, + "learning_rate": 7.502382620785083e-07, + "loss": 0.63320637, + "num_input_tokens_seen": 129334845, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.1517334, + "step": 6016, + "time_per_iteration": 2.776707410812378 + }, + { + "auxiliary_loss_clip": 0.01035413, + "auxiliary_loss_mlp": 0.01003506, + "balance_loss_clip": 1.0078721, + "balance_loss_mlp": 1.00206971, + "epoch": 0.7235014729754103, + "flos": 67258784050560.0, + "grad_norm": 0.8145039479838256, + "language_loss": 0.62545174, + "learning_rate": 7.496301980529289e-07, + "loss": 0.64584094, + "num_input_tokens_seen": 129398055, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01434326, + "step": 6017, + "time_per_iteration": 3.317119836807251 + }, + { + "auxiliary_loss_clip": 0.01130063, + "auxiliary_loss_mlp": 0.01038121, + "balance_loss_clip": 1.04219556, + "balance_loss_mlp": 1.02254665, + "epoch": 0.7236217158660494, + "flos": 26943237671040.0, + "grad_norm": 2.092335247068273, + "language_loss": 0.74480271, + "learning_rate": 7.490223237074547e-07, + "loss": 0.76648462, + "num_input_tokens_seen": 129417765, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.15570068, + "step": 6018, + "time_per_iteration": 2.832697629928589 + }, + { + "auxiliary_loss_clip": 0.01131253, + "auxiliary_loss_mlp": 0.01037786, + "balance_loss_clip": 1.04282713, + "balance_loss_mlp": 1.02269435, + "epoch": 0.7237419587566886, + "flos": 29423372042880.0, + "grad_norm": 2.5656960090954457, + "language_loss": 0.66158664, + "learning_rate": 7.484146391342989e-07, + "loss": 0.68327701, + "num_input_tokens_seen": 129437560, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15112305, + "step": 6019, + "time_per_iteration": 2.8375790119171143 + }, + { + "auxiliary_loss_clip": 0.0112941, + "auxiliary_loss_mlp": 0.01032357, + "balance_loss_clip": 1.04014945, + "balance_loss_mlp": 1.01696682, + "epoch": 0.7238622016473276, + "flos": 17821496787840.0, + "grad_norm": 2.487507787276587, + "language_loss": 0.57155573, + "learning_rate": 7.478071444256484e-07, + "loss": 0.59317338, + "num_input_tokens_seen": 129455320, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.15380859, + "step": 6020, + "time_per_iteration": 3.5696914196014404 + }, + { + "auxiliary_loss_clip": 0.01129333, + "auxiliary_loss_mlp": 0.01038059, + "balance_loss_clip": 1.04102004, + "balance_loss_mlp": 1.02162039, + "epoch": 0.7239824445379667, + "flos": 25739117020800.0, + "grad_norm": 1.6995213332085306, + "language_loss": 0.78739178, + "learning_rate": 7.471998396736579e-07, + "loss": 0.8090657, + "num_input_tokens_seen": 129475700, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.16430664, + "step": 6021, + "time_per_iteration": 2.7936315536499023 + }, + { + "auxiliary_loss_clip": 0.0112716, + "auxiliary_loss_mlp": 0.01035844, + "balance_loss_clip": 1.04224229, + "balance_loss_mlp": 1.02135444, + "epoch": 0.7241026874286057, + "flos": 23148916398720.0, + "grad_norm": 1.7325155473193894, + "language_loss": 0.75891197, + "learning_rate": 7.465927249704549e-07, + "loss": 0.78054202, + "num_input_tokens_seen": 129493585, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.14483643, + "step": 6022, + "time_per_iteration": 2.8101277351379395 + }, + { + "auxiliary_loss_clip": 0.01126571, + "auxiliary_loss_mlp": 0.01037367, + "balance_loss_clip": 1.03986382, + "balance_loss_mlp": 1.02200055, + "epoch": 0.7242229303192449, + "flos": 20266905686400.0, + "grad_norm": 2.2131010782291702, + "language_loss": 0.77555388, + "learning_rate": 7.459858004081398e-07, + "loss": 0.79719329, + "num_input_tokens_seen": 129511555, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15356445, + "step": 6023, + "time_per_iteration": 2.8288514614105225 + }, + { + "auxiliary_loss_clip": 0.01035465, + "auxiliary_loss_mlp": 0.01001119, + "balance_loss_clip": 1.00785184, + "balance_loss_mlp": 0.99955714, + "epoch": 0.724343173209884, + "flos": 62311659684480.0, + "grad_norm": 0.654915964632024, + "language_loss": 0.5802924, + "learning_rate": 7.453790660787815e-07, + "loss": 0.60065824, + "num_input_tokens_seen": 129579650, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.015625, + "step": 6024, + "time_per_iteration": 5.391742944717407 + }, + { + "auxiliary_loss_clip": 0.01131338, + "auxiliary_loss_mlp": 0.01045299, + "balance_loss_clip": 1.04192996, + "balance_loss_mlp": 1.02825189, + "epoch": 0.724463416100523, + "flos": 35006403813120.0, + "grad_norm": 2.0609535639745467, + "language_loss": 0.63586879, + "learning_rate": 7.447725220744214e-07, + "loss": 0.65763509, + "num_input_tokens_seen": 129601895, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.17053223, + "step": 6025, + "time_per_iteration": 2.899104118347168 + }, + { + "auxiliary_loss_clip": 0.01129282, + "auxiliary_loss_mlp": 0.01033686, + "balance_loss_clip": 1.04112816, + "balance_loss_mlp": 1.01879668, + "epoch": 0.7245836589911622, + "flos": 21871968923520.0, + "grad_norm": 2.077008929570273, + "language_loss": 0.77333069, + "learning_rate": 7.441661684870717e-07, + "loss": 0.79496038, + "num_input_tokens_seen": 129622150, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.14886475, + "step": 6026, + "time_per_iteration": 2.7709052562713623 + }, + { + "auxiliary_loss_clip": 0.01126128, + "auxiliary_loss_mlp": 0.01040002, + "balance_loss_clip": 1.03978944, + "balance_loss_mlp": 1.02461171, + "epoch": 0.7247039018818012, + "flos": 23006494972800.0, + "grad_norm": 1.6176928492012292, + "language_loss": 0.81681192, + "learning_rate": 7.435600054087152e-07, + "loss": 0.83847314, + "num_input_tokens_seen": 129644315, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.15380859, + "step": 6027, + "time_per_iteration": 2.8119561672210693 + }, + { + "auxiliary_loss_clip": 0.01130239, + "auxiliary_loss_mlp": 0.01042083, + "balance_loss_clip": 1.04318762, + "balance_loss_mlp": 1.02534604, + "epoch": 0.7248241447724403, + "flos": 31722588587520.0, + "grad_norm": 2.138908570709671, + "language_loss": 0.74221015, + "learning_rate": 7.42954032931308e-07, + "loss": 0.76393336, + "num_input_tokens_seen": 129665355, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.16748047, + "step": 6028, + "time_per_iteration": 2.8045096397399902 + }, + { + "auxiliary_loss_clip": 0.01128904, + "auxiliary_loss_mlp": 0.01047017, + "balance_loss_clip": 1.04155064, + "balance_loss_mlp": 1.03092384, + "epoch": 0.7249443876630794, + "flos": 34896984007680.0, + "grad_norm": 1.8200769669331875, + "language_loss": 0.74821794, + "learning_rate": 7.423482511467733e-07, + "loss": 0.76997721, + "num_input_tokens_seen": 129686125, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.16088867, + "step": 6029, + "time_per_iteration": 2.8610153198242188 + }, + { + "auxiliary_loss_clip": 0.01132288, + "auxiliary_loss_mlp": 0.01032955, + "balance_loss_clip": 1.04432845, + "balance_loss_mlp": 1.01731527, + "epoch": 0.7250646305537185, + "flos": 26359294268160.0, + "grad_norm": 2.0912965560208594, + "language_loss": 0.64926505, + "learning_rate": 7.417426601470099e-07, + "loss": 0.67091751, + "num_input_tokens_seen": 129706485, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.15643311, + "step": 6030, + "time_per_iteration": 2.810692310333252 + }, + { + "auxiliary_loss_clip": 0.01131744, + "auxiliary_loss_mlp": 0.01031411, + "balance_loss_clip": 1.04112124, + "balance_loss_mlp": 1.01476955, + "epoch": 0.7251848734443576, + "flos": 30081614728320.0, + "grad_norm": 2.14930769620298, + "language_loss": 0.78384745, + "learning_rate": 7.411372600238841e-07, + "loss": 0.80547893, + "num_input_tokens_seen": 129727100, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.16650391, + "step": 6031, + "time_per_iteration": 3.626000165939331 + }, + { + "auxiliary_loss_clip": 0.01129177, + "auxiliary_loss_mlp": 0.01045476, + "balance_loss_clip": 1.04256964, + "balance_loss_mlp": 1.02904916, + "epoch": 0.7253051163349967, + "flos": 17785262943360.0, + "grad_norm": 2.0130179743238132, + "language_loss": 0.73724008, + "learning_rate": 7.405320508692346e-07, + "loss": 0.75898665, + "num_input_tokens_seen": 129745840, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.16418457, + "step": 6032, + "time_per_iteration": 2.739304304122925 + }, + { + "auxiliary_loss_clip": 0.01127363, + "auxiliary_loss_mlp": 0.01039754, + "balance_loss_clip": 1.04257298, + "balance_loss_mlp": 1.02471542, + "epoch": 0.7254253592256358, + "flos": 12641346938880.0, + "grad_norm": 1.9226758431668391, + "language_loss": 0.75513858, + "learning_rate": 7.399270327748727e-07, + "loss": 0.77680969, + "num_input_tokens_seen": 129763500, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15045166, + "step": 6033, + "time_per_iteration": 2.742570161819458 + }, + { + "auxiliary_loss_clip": 0.01124866, + "auxiliary_loss_mlp": 0.01041198, + "balance_loss_clip": 1.04145646, + "balance_loss_mlp": 1.02626157, + "epoch": 0.7255456021162748, + "flos": 27199208966400.0, + "grad_norm": 1.7899849134933434, + "language_loss": 0.74170011, + "learning_rate": 7.39322205832577e-07, + "loss": 0.7633608, + "num_input_tokens_seen": 129784390, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.14953613, + "step": 6034, + "time_per_iteration": 2.807234764099121 + }, + { + "auxiliary_loss_clip": 0.01126312, + "auxiliary_loss_mlp": 0.01043335, + "balance_loss_clip": 1.04046988, + "balance_loss_mlp": 1.02771807, + "epoch": 0.725665845006914, + "flos": 21288205088640.0, + "grad_norm": 1.9475859394374202, + "language_loss": 0.80934489, + "learning_rate": 7.387175701341009e-07, + "loss": 0.83104134, + "num_input_tokens_seen": 129803060, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.15625, + "step": 6035, + "time_per_iteration": 2.825390100479126 + }, + { + "auxiliary_loss_clip": 0.01129967, + "auxiliary_loss_mlp": 0.01039718, + "balance_loss_clip": 1.04102695, + "balance_loss_mlp": 1.02277827, + "epoch": 0.7257860878975531, + "flos": 16033684129920.0, + "grad_norm": 1.9219637955136175, + "language_loss": 0.72160196, + "learning_rate": 7.381131257711659e-07, + "loss": 0.74329877, + "num_input_tokens_seen": 129820165, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.16955566, + "step": 6036, + "time_per_iteration": 2.8445956707000732 + }, + { + "auxiliary_loss_clip": 0.01128398, + "auxiliary_loss_mlp": 0.01042394, + "balance_loss_clip": 1.04368389, + "balance_loss_mlp": 1.02732623, + "epoch": 0.7259063307881921, + "flos": 12129943052160.0, + "grad_norm": 1.9348248467734457, + "language_loss": 0.8351146, + "learning_rate": 7.375088728354677e-07, + "loss": 0.85682249, + "num_input_tokens_seen": 129835195, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15057373, + "step": 6037, + "time_per_iteration": 2.7650222778320312 + }, + { + "auxiliary_loss_clip": 0.011265, + "auxiliary_loss_mlp": 0.01038454, + "balance_loss_clip": 1.04127312, + "balance_loss_mlp": 1.02321863, + "epoch": 0.7260265736788313, + "flos": 30443845432320.0, + "grad_norm": 1.5515925949037406, + "language_loss": 0.67495644, + "learning_rate": 7.369048114186691e-07, + "loss": 0.69660592, + "num_input_tokens_seen": 129856240, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.15234375, + "step": 6038, + "time_per_iteration": 2.866265058517456 + }, + { + "auxiliary_loss_clip": 0.01125302, + "auxiliary_loss_mlp": 0.01035043, + "balance_loss_clip": 1.04312611, + "balance_loss_mlp": 1.02048743, + "epoch": 0.7261468165694703, + "flos": 21142264129920.0, + "grad_norm": 1.6748296211936151, + "language_loss": 0.83327401, + "learning_rate": 7.363009416124055e-07, + "loss": 0.85487753, + "num_input_tokens_seen": 129875565, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.14562988, + "step": 6039, + "time_per_iteration": 2.9041154384613037 + }, + { + "auxiliary_loss_clip": 0.01131212, + "auxiliary_loss_mlp": 0.01039499, + "balance_loss_clip": 1.04183102, + "balance_loss_mlp": 1.02373981, + "epoch": 0.7262670594601094, + "flos": 22306308180480.0, + "grad_norm": 2.133905580868444, + "language_loss": 0.62554705, + "learning_rate": 7.356972635082852e-07, + "loss": 0.64725411, + "num_input_tokens_seen": 129894420, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.15759277, + "step": 6040, + "time_per_iteration": 2.8519725799560547 + }, + { + "auxiliary_loss_clip": 0.01131044, + "auxiliary_loss_mlp": 0.01042345, + "balance_loss_clip": 1.04360712, + "balance_loss_mlp": 1.02597785, + "epoch": 0.7263873023507486, + "flos": 25335049950720.0, + "grad_norm": 1.6018953820310078, + "language_loss": 0.75441182, + "learning_rate": 7.35093777197884e-07, + "loss": 0.77614576, + "num_input_tokens_seen": 129914490, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.16369629, + "step": 6041, + "time_per_iteration": 2.924051523208618 + }, + { + "auxiliary_loss_clip": 0.01127064, + "auxiliary_loss_mlp": 0.01043383, + "balance_loss_clip": 1.041206, + "balance_loss_mlp": 1.02848184, + "epoch": 0.7265075452413876, + "flos": 23878621192320.0, + "grad_norm": 2.0034891320708486, + "language_loss": 0.85612237, + "learning_rate": 7.344904827727525e-07, + "loss": 0.87782681, + "num_input_tokens_seen": 129931670, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.14904785, + "step": 6042, + "time_per_iteration": 2.9230778217315674 + }, + { + "auxiliary_loss_clip": 0.01127185, + "auxiliary_loss_mlp": 0.01037548, + "balance_loss_clip": 1.04067791, + "balance_loss_mlp": 1.02307558, + "epoch": 0.7266277881320267, + "flos": 28724549967360.0, + "grad_norm": 3.172894608346546, + "language_loss": 0.73744607, + "learning_rate": 7.338873803244076e-07, + "loss": 0.7590934, + "num_input_tokens_seen": 129946905, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.14477539, + "step": 6043, + "time_per_iteration": 2.874594211578369 + }, + { + "auxiliary_loss_clip": 0.01125488, + "auxiliary_loss_mlp": 0.01037347, + "balance_loss_clip": 1.04131126, + "balance_loss_mlp": 1.0225656, + "epoch": 0.7267480310226658, + "flos": 24863507182080.0, + "grad_norm": 1.7096775950076533, + "language_loss": 0.80828625, + "learning_rate": 7.332844699443401e-07, + "loss": 0.82991457, + "num_input_tokens_seen": 129965505, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.14794922, + "step": 6044, + "time_per_iteration": 2.903043746948242 + }, + { + "auxiliary_loss_clip": 0.01126097, + "auxiliary_loss_mlp": 0.01044577, + "balance_loss_clip": 1.04049397, + "balance_loss_mlp": 1.0301702, + "epoch": 0.7268682739133049, + "flos": 27198490694400.0, + "grad_norm": 1.8172770111497387, + "language_loss": 0.75458437, + "learning_rate": 7.326817517240121e-07, + "loss": 0.77629113, + "num_input_tokens_seen": 129987210, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.14398193, + "step": 6045, + "time_per_iteration": 2.7956886291503906 + }, + { + "auxiliary_loss_clip": 0.01127268, + "auxiliary_loss_mlp": 0.01039462, + "balance_loss_clip": 1.04266655, + "balance_loss_mlp": 1.02601516, + "epoch": 0.7269885168039439, + "flos": 33508138688640.0, + "grad_norm": 1.765732809973517, + "language_loss": 0.83513892, + "learning_rate": 7.320792257548545e-07, + "loss": 0.85680616, + "num_input_tokens_seen": 130008385, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.13452148, + "step": 6046, + "time_per_iteration": 2.914937973022461 + }, + { + "auxiliary_loss_clip": 0.01128701, + "auxiliary_loss_mlp": 0.01047467, + "balance_loss_clip": 1.04130769, + "balance_loss_mlp": 1.03102756, + "epoch": 0.7271087596945831, + "flos": 24313750548480.0, + "grad_norm": 1.8754919568813966, + "language_loss": 0.76159155, + "learning_rate": 7.314768921282704e-07, + "loss": 0.78335321, + "num_input_tokens_seen": 130029040, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.16455078, + "step": 6047, + "time_per_iteration": 2.7761363983154297 + }, + { + "auxiliary_loss_clip": 0.01125874, + "auxiliary_loss_mlp": 0.0103698, + "balance_loss_clip": 1.04136884, + "balance_loss_mlp": 1.02255011, + "epoch": 0.7272290025852222, + "flos": 23805147922560.0, + "grad_norm": 2.3087705459467065, + "language_loss": 0.72367871, + "learning_rate": 7.30874750935633e-07, + "loss": 0.74530721, + "num_input_tokens_seen": 130048725, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14459229, + "step": 6048, + "time_per_iteration": 2.7967209815979004 + }, + { + "auxiliary_loss_clip": 0.01126422, + "auxiliary_loss_mlp": 0.01041965, + "balance_loss_clip": 1.04055834, + "balance_loss_mlp": 1.02663493, + "epoch": 0.7273492454758612, + "flos": 16720367408640.0, + "grad_norm": 1.832578067124417, + "language_loss": 0.79167765, + "learning_rate": 7.30272802268286e-07, + "loss": 0.81336153, + "num_input_tokens_seen": 130065720, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15332031, + "step": 6049, + "time_per_iteration": 3.518662214279175 + }, + { + "auxiliary_loss_clip": 0.01120833, + "auxiliary_loss_mlp": 0.01032195, + "balance_loss_clip": 1.04013729, + "balance_loss_mlp": 1.01897454, + "epoch": 0.7274694883665004, + "flos": 28031330413440.0, + "grad_norm": 1.8655727259450925, + "language_loss": 0.76310515, + "learning_rate": 7.29671046217547e-07, + "loss": 0.78463542, + "num_input_tokens_seen": 130084830, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.13226318, + "step": 6050, + "time_per_iteration": 2.8684678077697754 + }, + { + "auxiliary_loss_clip": 0.01126894, + "auxiliary_loss_mlp": 0.01038698, + "balance_loss_clip": 1.04057622, + "balance_loss_mlp": 1.02342772, + "epoch": 0.7275897312571394, + "flos": 30372706546560.0, + "grad_norm": 1.8715193079751888, + "language_loss": 0.81645513, + "learning_rate": 7.290694828746988e-07, + "loss": 0.83811098, + "num_input_tokens_seen": 130104495, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.15270996, + "step": 6051, + "time_per_iteration": 2.941760778427124 + }, + { + "auxiliary_loss_clip": 0.01125921, + "auxiliary_loss_mlp": 0.01038376, + "balance_loss_clip": 1.04230118, + "balance_loss_mlp": 1.0235821, + "epoch": 0.7277099741477785, + "flos": 19204775498880.0, + "grad_norm": 1.7475313525252627, + "language_loss": 0.85913026, + "learning_rate": 7.284681123310004e-07, + "loss": 0.88077319, + "num_input_tokens_seen": 130123210, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14788818, + "step": 6052, + "time_per_iteration": 2.7966809272766113 + }, + { + "auxiliary_loss_clip": 0.01131076, + "auxiliary_loss_mlp": 0.01037418, + "balance_loss_clip": 1.04071438, + "balance_loss_mlp": 1.02231407, + "epoch": 0.7278302170384175, + "flos": 20667884186880.0, + "grad_norm": 1.7149349887210503, + "language_loss": 0.79445708, + "learning_rate": 7.27866934677678e-07, + "loss": 0.81614202, + "num_input_tokens_seen": 130142880, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.15087891, + "step": 6053, + "time_per_iteration": 3.7276175022125244 + }, + { + "auxiliary_loss_clip": 0.01126571, + "auxiliary_loss_mlp": 0.01040922, + "balance_loss_clip": 1.04025948, + "balance_loss_mlp": 1.02490067, + "epoch": 0.7279504599290567, + "flos": 19093200877440.0, + "grad_norm": 2.4835356534427198, + "language_loss": 0.78613645, + "learning_rate": 7.272659500059297e-07, + "loss": 0.80781138, + "num_input_tokens_seen": 130160220, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.16040039, + "step": 6054, + "time_per_iteration": 2.898972272872925 + }, + { + "auxiliary_loss_clip": 0.01129619, + "auxiliary_loss_mlp": 0.0105038, + "balance_loss_clip": 1.04125714, + "balance_loss_mlp": 1.0330708, + "epoch": 0.7280707028196958, + "flos": 19062174504960.0, + "grad_norm": 2.009279103081734, + "language_loss": 0.80140579, + "learning_rate": 7.266651584069264e-07, + "loss": 0.82320583, + "num_input_tokens_seen": 130177885, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.1730957, + "step": 6055, + "time_per_iteration": 2.8406527042388916 + }, + { + "auxiliary_loss_clip": 0.01130355, + "auxiliary_loss_mlp": 0.01042486, + "balance_loss_clip": 1.04356837, + "balance_loss_mlp": 1.02703667, + "epoch": 0.7281909457103348, + "flos": 37196308293120.0, + "grad_norm": 1.5276866385581405, + "language_loss": 0.57102013, + "learning_rate": 7.260645599718045e-07, + "loss": 0.59274852, + "num_input_tokens_seen": 130204240, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.15441895, + "step": 6056, + "time_per_iteration": 2.9492955207824707 + }, + { + "auxiliary_loss_clip": 0.01133128, + "auxiliary_loss_mlp": 0.01045698, + "balance_loss_clip": 1.0421555, + "balance_loss_mlp": 1.02846074, + "epoch": 0.728311188600974, + "flos": 20667094087680.0, + "grad_norm": 2.223012714027188, + "language_loss": 0.67654097, + "learning_rate": 7.254641547916767e-07, + "loss": 0.69832927, + "num_input_tokens_seen": 130221735, + "router_z_loss_clip": 0.90966797, + "router_z_loss_mlp": 0.17248535, + "step": 6057, + "time_per_iteration": 3.002434730529785 + }, + { + "auxiliary_loss_clip": 0.01129217, + "auxiliary_loss_mlp": 0.01038201, + "balance_loss_clip": 1.04222465, + "balance_loss_mlp": 1.02140403, + "epoch": 0.728431431491613, + "flos": 28840685616000.0, + "grad_norm": 1.6801035566494285, + "language_loss": 0.69100988, + "learning_rate": 7.248639429576226e-07, + "loss": 0.71268404, + "num_input_tokens_seen": 130241190, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.16796875, + "step": 6058, + "time_per_iteration": 2.9379820823669434 + }, + { + "auxiliary_loss_clip": 0.0112831, + "auxiliary_loss_mlp": 0.01032792, + "balance_loss_clip": 1.04106915, + "balance_loss_mlp": 1.01781321, + "epoch": 0.7285516743822521, + "flos": 25991856092160.0, + "grad_norm": 1.6639002872762165, + "language_loss": 0.71988964, + "learning_rate": 7.242639245606959e-07, + "loss": 0.74150062, + "num_input_tokens_seen": 130260980, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.14984131, + "step": 6059, + "time_per_iteration": 3.734525203704834 + }, + { + "auxiliary_loss_clip": 0.01127717, + "auxiliary_loss_mlp": 0.01050605, + "balance_loss_clip": 1.04141247, + "balance_loss_mlp": 1.03536975, + "epoch": 0.7286719172728913, + "flos": 16399721675520.0, + "grad_norm": 1.9648087794603533, + "language_loss": 0.82226336, + "learning_rate": 7.236640996919168e-07, + "loss": 0.84404653, + "num_input_tokens_seen": 130280025, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.15234375, + "step": 6060, + "time_per_iteration": 2.8643124103546143 + }, + { + "auxiliary_loss_clip": 0.01125462, + "auxiliary_loss_mlp": 0.01035462, + "balance_loss_clip": 1.04132485, + "balance_loss_mlp": 1.02062011, + "epoch": 0.7287921601635303, + "flos": 22018161277440.0, + "grad_norm": 1.560381527302597, + "language_loss": 0.70391512, + "learning_rate": 7.230644684422782e-07, + "loss": 0.72552437, + "num_input_tokens_seen": 130300255, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14849854, + "step": 6061, + "time_per_iteration": 2.838026285171509 + }, + { + "auxiliary_loss_clip": 0.0113063, + "auxiliary_loss_mlp": 0.01040589, + "balance_loss_clip": 1.04163861, + "balance_loss_mlp": 1.02521086, + "epoch": 0.7289124030541694, + "flos": 24600927784320.0, + "grad_norm": 1.6836089197234503, + "language_loss": 0.81480068, + "learning_rate": 7.224650309027451e-07, + "loss": 0.83651286, + "num_input_tokens_seen": 130320005, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.15368652, + "step": 6062, + "time_per_iteration": 2.8813815116882324 + }, + { + "auxiliary_loss_clip": 0.01128713, + "auxiliary_loss_mlp": 0.0104153, + "balance_loss_clip": 1.04292607, + "balance_loss_mlp": 1.02625954, + "epoch": 0.7290326459448085, + "flos": 21393638484480.0, + "grad_norm": 1.6279127137863363, + "language_loss": 0.68882751, + "learning_rate": 7.218657871642506e-07, + "loss": 0.71053004, + "num_input_tokens_seen": 130338810, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15270996, + "step": 6063, + "time_per_iteration": 2.824270009994507 + }, + { + "auxiliary_loss_clip": 0.01129411, + "auxiliary_loss_mlp": 0.01044398, + "balance_loss_clip": 1.04099917, + "balance_loss_mlp": 1.02807808, + "epoch": 0.7291528888354476, + "flos": 18587686821120.0, + "grad_norm": 1.996052685122361, + "language_loss": 0.62102222, + "learning_rate": 7.212667373177012e-07, + "loss": 0.64276028, + "num_input_tokens_seen": 130353805, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.16320801, + "step": 6064, + "time_per_iteration": 2.926859140396118 + }, + { + "auxiliary_loss_clip": 0.01125056, + "auxiliary_loss_mlp": 0.01037769, + "balance_loss_clip": 1.04107654, + "balance_loss_mlp": 1.02308261, + "epoch": 0.7292731317260867, + "flos": 18951066760320.0, + "grad_norm": 1.830492406824833, + "language_loss": 0.7544167, + "learning_rate": 7.206678814539704e-07, + "loss": 0.77604496, + "num_input_tokens_seen": 130372105, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14697266, + "step": 6065, + "time_per_iteration": 2.877984046936035 + }, + { + "auxiliary_loss_clip": 0.01124517, + "auxiliary_loss_mlp": 0.01033262, + "balance_loss_clip": 1.04143989, + "balance_loss_mlp": 1.01903403, + "epoch": 0.7293933746167258, + "flos": 21067569797760.0, + "grad_norm": 1.4767730072123049, + "language_loss": 0.72904658, + "learning_rate": 7.20069219663904e-07, + "loss": 0.75062442, + "num_input_tokens_seen": 130391990, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.14227295, + "step": 6066, + "time_per_iteration": 2.811854362487793 + }, + { + "auxiliary_loss_clip": 0.01127708, + "auxiliary_loss_mlp": 0.01038647, + "balance_loss_clip": 1.03992558, + "balance_loss_mlp": 1.02306581, + "epoch": 0.7295136175073649, + "flos": 22453326547200.0, + "grad_norm": 1.8379290981627578, + "language_loss": 0.80017221, + "learning_rate": 7.1947075203832e-07, + "loss": 0.82183576, + "num_input_tokens_seen": 130411970, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.15576172, + "step": 6067, + "time_per_iteration": 2.803745746612549 + }, + { + "auxiliary_loss_clip": 0.01035673, + "auxiliary_loss_mlp": 0.01002173, + "balance_loss_clip": 1.00813746, + "balance_loss_mlp": 1.00071859, + "epoch": 0.7296338603980039, + "flos": 56125506648960.0, + "grad_norm": 0.8918288983563841, + "language_loss": 0.60153759, + "learning_rate": 7.188724786680049e-07, + "loss": 0.62191606, + "num_input_tokens_seen": 130472440, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01452637, + "step": 6068, + "time_per_iteration": 3.3472845554351807 + }, + { + "auxiliary_loss_clip": 0.0112415, + "auxiliary_loss_mlp": 0.01035429, + "balance_loss_clip": 1.04037118, + "balance_loss_mlp": 1.02040267, + "epoch": 0.7297541032886431, + "flos": 25228287751680.0, + "grad_norm": 1.7701119017824425, + "language_loss": 0.75586343, + "learning_rate": 7.182743996437162e-07, + "loss": 0.77745926, + "num_input_tokens_seen": 130491975, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.15032959, + "step": 6069, + "time_per_iteration": 2.8184056282043457 + }, + { + "auxiliary_loss_clip": 0.01130148, + "auxiliary_loss_mlp": 0.01039094, + "balance_loss_clip": 1.0412991, + "balance_loss_mlp": 1.02315569, + "epoch": 0.7298743461792822, + "flos": 26467600752000.0, + "grad_norm": 1.8960832653001822, + "language_loss": 0.68863058, + "learning_rate": 7.176765150561819e-07, + "loss": 0.71032304, + "num_input_tokens_seen": 130510580, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.15930176, + "step": 6070, + "time_per_iteration": 2.8192877769470215 + }, + { + "auxiliary_loss_clip": 0.01127113, + "auxiliary_loss_mlp": 0.01037103, + "balance_loss_clip": 1.03873873, + "balance_loss_mlp": 1.02114129, + "epoch": 0.7299945890699212, + "flos": 19569053278080.0, + "grad_norm": 3.0508701165253838, + "language_loss": 0.80042434, + "learning_rate": 7.170788249961002e-07, + "loss": 0.82206643, + "num_input_tokens_seen": 130529090, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.1595459, + "step": 6071, + "time_per_iteration": 2.8325936794281006 + }, + { + "auxiliary_loss_clip": 0.0112383, + "auxiliary_loss_mlp": 0.01038036, + "balance_loss_clip": 1.03893089, + "balance_loss_mlp": 1.02249146, + "epoch": 0.7301148319605604, + "flos": 22928963466240.0, + "grad_norm": 1.8796862365748614, + "language_loss": 0.88125443, + "learning_rate": 7.164813295541418e-07, + "loss": 0.9028731, + "num_input_tokens_seen": 130548655, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.15527344, + "step": 6072, + "time_per_iteration": 2.757918119430542 + }, + { + "auxiliary_loss_clip": 0.01126238, + "auxiliary_loss_mlp": 0.01038257, + "balance_loss_clip": 1.03984809, + "balance_loss_mlp": 1.0225687, + "epoch": 0.7302350748511994, + "flos": 25369703596800.0, + "grad_norm": 1.6834342183949942, + "language_loss": 0.70189291, + "learning_rate": 7.15884028820944e-07, + "loss": 0.7235378, + "num_input_tokens_seen": 130567710, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.15686035, + "step": 6073, + "time_per_iteration": 2.7853140830993652 + }, + { + "auxiliary_loss_clip": 0.01125276, + "auxiliary_loss_mlp": 0.01034604, + "balance_loss_clip": 1.03860044, + "balance_loss_mlp": 1.01987576, + "epoch": 0.7303553177418385, + "flos": 27819170732160.0, + "grad_norm": 1.9332607145644145, + "language_loss": 0.60262406, + "learning_rate": 7.152869228871185e-07, + "loss": 0.62422287, + "num_input_tokens_seen": 130590195, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.14715576, + "step": 6074, + "time_per_iteration": 2.82241153717041 + }, + { + "auxiliary_loss_clip": 0.01126026, + "auxiliary_loss_mlp": 0.01041363, + "balance_loss_clip": 1.04012311, + "balance_loss_mlp": 1.0249362, + "epoch": 0.7304755606324776, + "flos": 24426510318720.0, + "grad_norm": 2.0610880559662648, + "language_loss": 0.72050154, + "learning_rate": 7.146900118432457e-07, + "loss": 0.74217546, + "num_input_tokens_seen": 130609940, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.16418457, + "step": 6075, + "time_per_iteration": 2.935943603515625 + }, + { + "auxiliary_loss_clip": 0.01126379, + "auxiliary_loss_mlp": 0.01034191, + "balance_loss_clip": 1.0414232, + "balance_loss_mlp": 1.01976645, + "epoch": 0.7305958035231167, + "flos": 23840483927040.0, + "grad_norm": 1.6476909273863056, + "language_loss": 0.85809934, + "learning_rate": 7.140932957798753e-07, + "loss": 0.87970507, + "num_input_tokens_seen": 130628380, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.14422607, + "step": 6076, + "time_per_iteration": 2.871262550354004 + }, + { + "auxiliary_loss_clip": 0.01129427, + "auxiliary_loss_mlp": 0.01040506, + "balance_loss_clip": 1.04295278, + "balance_loss_mlp": 1.02465105, + "epoch": 0.7307160464137558, + "flos": 16726939597440.0, + "grad_norm": 1.8903077378302142, + "language_loss": 0.7116763, + "learning_rate": 7.134967747875309e-07, + "loss": 0.73337567, + "num_input_tokens_seen": 130646590, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.15869141, + "step": 6077, + "time_per_iteration": 2.9327306747436523 + }, + { + "auxiliary_loss_clip": 0.0112926, + "auxiliary_loss_mlp": 0.01038294, + "balance_loss_clip": 1.04256678, + "balance_loss_mlp": 1.02253461, + "epoch": 0.7308362893043949, + "flos": 21798280172160.0, + "grad_norm": 1.8047811234043407, + "language_loss": 0.81866825, + "learning_rate": 7.129004489567014e-07, + "loss": 0.84034383, + "num_input_tokens_seen": 130664070, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15759277, + "step": 6078, + "time_per_iteration": 3.674764394760132 + }, + { + "auxiliary_loss_clip": 0.01127662, + "auxiliary_loss_mlp": 0.01042148, + "balance_loss_clip": 1.04027224, + "balance_loss_mlp": 1.0267942, + "epoch": 0.730956532195034, + "flos": 10707377840640.0, + "grad_norm": 2.8653926039429822, + "language_loss": 0.78245139, + "learning_rate": 7.123043183778512e-07, + "loss": 0.80414951, + "num_input_tokens_seen": 130681400, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15368652, + "step": 6079, + "time_per_iteration": 2.8305201530456543 + }, + { + "auxiliary_loss_clip": 0.01131351, + "auxiliary_loss_mlp": 0.0103646, + "balance_loss_clip": 1.04233861, + "balance_loss_mlp": 1.02102256, + "epoch": 0.731076775085673, + "flos": 19791987039360.0, + "grad_norm": 1.4552151659686148, + "language_loss": 0.65077835, + "learning_rate": 7.117083831414114e-07, + "loss": 0.6724565, + "num_input_tokens_seen": 130700675, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.15466309, + "step": 6080, + "time_per_iteration": 2.8411331176757812 + }, + { + "auxiliary_loss_clip": 0.01124589, + "auxiliary_loss_mlp": 0.01036103, + "balance_loss_clip": 1.04055834, + "balance_loss_mlp": 1.02114773, + "epoch": 0.7311970179763122, + "flos": 20447033414400.0, + "grad_norm": 1.6301700281335365, + "language_loss": 0.69861066, + "learning_rate": 7.11112643337787e-07, + "loss": 0.72021759, + "num_input_tokens_seen": 130719720, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14959717, + "step": 6081, + "time_per_iteration": 2.853058338165283 + }, + { + "auxiliary_loss_clip": 0.01131324, + "auxiliary_loss_mlp": 0.01041043, + "balance_loss_clip": 1.0415827, + "balance_loss_mlp": 1.02531886, + "epoch": 0.7313172608669513, + "flos": 18513818501760.0, + "grad_norm": 2.313731201214145, + "language_loss": 0.76505423, + "learning_rate": 7.10517099057349e-07, + "loss": 0.78677797, + "num_input_tokens_seen": 130736670, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.15710449, + "step": 6082, + "time_per_iteration": 4.441192865371704 + }, + { + "auxiliary_loss_clip": 0.01130335, + "auxiliary_loss_mlp": 0.01039935, + "balance_loss_clip": 1.04212379, + "balance_loss_mlp": 1.02267373, + "epoch": 0.7314375037575903, + "flos": 16180738410240.0, + "grad_norm": 2.333696607464558, + "language_loss": 0.60965854, + "learning_rate": 7.099217503904411e-07, + "loss": 0.63136125, + "num_input_tokens_seen": 130754525, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.17260742, + "step": 6083, + "time_per_iteration": 2.789064884185791 + }, + { + "auxiliary_loss_clip": 0.01126682, + "auxiliary_loss_mlp": 0.01042356, + "balance_loss_clip": 1.04071593, + "balance_loss_mlp": 1.02777052, + "epoch": 0.7315577466482295, + "flos": 17967940536960.0, + "grad_norm": 2.068960274527048, + "language_loss": 0.90339196, + "learning_rate": 7.093265974273788e-07, + "loss": 0.92508233, + "num_input_tokens_seen": 130772420, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.14581299, + "step": 6084, + "time_per_iteration": 2.890570640563965 + }, + { + "auxiliary_loss_clip": 0.01124734, + "auxiliary_loss_mlp": 0.01033987, + "balance_loss_clip": 1.04019129, + "balance_loss_mlp": 1.02079082, + "epoch": 0.7316779895388685, + "flos": 18405440190720.0, + "grad_norm": 1.7011245085194038, + "language_loss": 0.71774328, + "learning_rate": 7.087316402584447e-07, + "loss": 0.73933047, + "num_input_tokens_seen": 130791245, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.13195801, + "step": 6085, + "time_per_iteration": 2.807756185531616 + }, + { + "auxiliary_loss_clip": 0.0112776, + "auxiliary_loss_mlp": 0.01041668, + "balance_loss_clip": 1.0408082, + "balance_loss_mlp": 1.02623606, + "epoch": 0.7317982324295076, + "flos": 17928294900480.0, + "grad_norm": 1.8261996931270943, + "language_loss": 0.86479509, + "learning_rate": 7.081368789738953e-07, + "loss": 0.88648939, + "num_input_tokens_seen": 130808445, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15435791, + "step": 6086, + "time_per_iteration": 2.86224102973938 + }, + { + "auxiliary_loss_clip": 0.01125298, + "auxiliary_loss_mlp": 0.0103898, + "balance_loss_clip": 1.04044032, + "balance_loss_mlp": 1.02306581, + "epoch": 0.7319184753201466, + "flos": 27229840289280.0, + "grad_norm": 2.067823853582044, + "language_loss": 0.77725703, + "learning_rate": 7.075423136639537e-07, + "loss": 0.79889983, + "num_input_tokens_seen": 130827700, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15930176, + "step": 6087, + "time_per_iteration": 2.9031734466552734 + }, + { + "auxiliary_loss_clip": 0.01127611, + "auxiliary_loss_mlp": 0.01043804, + "balance_loss_clip": 1.04051042, + "balance_loss_mlp": 1.02779424, + "epoch": 0.7320387182107858, + "flos": 37448544574080.0, + "grad_norm": 2.539524442552191, + "language_loss": 0.74784005, + "learning_rate": 7.069479444188149e-07, + "loss": 0.76955414, + "num_input_tokens_seen": 130848290, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.16015625, + "step": 6088, + "time_per_iteration": 3.8754043579101562 + }, + { + "auxiliary_loss_clip": 0.01127926, + "auxiliary_loss_mlp": 0.01037414, + "balance_loss_clip": 1.04288924, + "balance_loss_mlp": 1.02266741, + "epoch": 0.7321589611014249, + "flos": 17859023521920.0, + "grad_norm": 1.6863052362498272, + "language_loss": 0.82201171, + "learning_rate": 7.063537713286453e-07, + "loss": 0.84366512, + "num_input_tokens_seen": 130865970, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.14733887, + "step": 6089, + "time_per_iteration": 2.8563573360443115 + }, + { + "auxiliary_loss_clip": 0.01128243, + "auxiliary_loss_mlp": 0.01039232, + "balance_loss_clip": 1.03988814, + "balance_loss_mlp": 1.02361536, + "epoch": 0.7322792039920639, + "flos": 26100593539200.0, + "grad_norm": 2.1321675149862376, + "language_loss": 0.80999863, + "learning_rate": 7.057597944835803e-07, + "loss": 0.83167332, + "num_input_tokens_seen": 130885245, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.15625, + "step": 6090, + "time_per_iteration": 2.771782875061035 + }, + { + "auxiliary_loss_clip": 0.01125974, + "auxiliary_loss_mlp": 0.01039465, + "balance_loss_clip": 1.04081035, + "balance_loss_mlp": 1.02436078, + "epoch": 0.7323994468827031, + "flos": 25369093065600.0, + "grad_norm": 1.766749877363672, + "language_loss": 0.74921167, + "learning_rate": 7.051660139737253e-07, + "loss": 0.77086604, + "num_input_tokens_seen": 130903465, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15100098, + "step": 6091, + "time_per_iteration": 2.7963900566101074 + }, + { + "auxiliary_loss_clip": 0.01127498, + "auxiliary_loss_mlp": 0.01039848, + "balance_loss_clip": 1.03987241, + "balance_loss_mlp": 1.02351677, + "epoch": 0.7325196897733421, + "flos": 26907075653760.0, + "grad_norm": 1.8680327519106052, + "language_loss": 0.76540542, + "learning_rate": 7.045724298891565e-07, + "loss": 0.78707886, + "num_input_tokens_seen": 130922935, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.16320801, + "step": 6092, + "time_per_iteration": 2.776069402694702 + }, + { + "auxiliary_loss_clip": 0.01128068, + "auxiliary_loss_mlp": 0.01038389, + "balance_loss_clip": 1.04114985, + "balance_loss_mlp": 1.02209353, + "epoch": 0.7326399326639812, + "flos": 25775781828480.0, + "grad_norm": 1.8624973412168633, + "language_loss": 0.69119275, + "learning_rate": 7.039790423199192e-07, + "loss": 0.71285737, + "num_input_tokens_seen": 130942575, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.16308594, + "step": 6093, + "time_per_iteration": 2.7846810817718506 + }, + { + "auxiliary_loss_clip": 0.01131644, + "auxiliary_loss_mlp": 0.01039242, + "balance_loss_clip": 1.04213846, + "balance_loss_mlp": 1.02338696, + "epoch": 0.7327601755546204, + "flos": 21032269706880.0, + "grad_norm": 2.0787139642631245, + "language_loss": 0.78080904, + "learning_rate": 7.033858513560322e-07, + "loss": 0.80251789, + "num_input_tokens_seen": 130958870, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.15856934, + "step": 6094, + "time_per_iteration": 2.799281120300293 + }, + { + "auxiliary_loss_clip": 0.01124022, + "auxiliary_loss_mlp": 0.01039984, + "balance_loss_clip": 1.04070699, + "balance_loss_mlp": 1.02487993, + "epoch": 0.7328804184452594, + "flos": 16289224462080.0, + "grad_norm": 2.056487360180258, + "language_loss": 0.7579987, + "learning_rate": 7.027928570874794e-07, + "loss": 0.77963877, + "num_input_tokens_seen": 130977060, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.15100098, + "step": 6095, + "time_per_iteration": 2.736783742904663 + }, + { + "auxiliary_loss_clip": 0.01126804, + "auxiliary_loss_mlp": 0.01041993, + "balance_loss_clip": 1.04086232, + "balance_loss_mlp": 1.02693117, + "epoch": 0.7330006613358985, + "flos": 17858233422720.0, + "grad_norm": 1.8755854307315853, + "language_loss": 0.85448205, + "learning_rate": 7.022000596042194e-07, + "loss": 0.8761701, + "num_input_tokens_seen": 130994160, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.1506958, + "step": 6096, + "time_per_iteration": 2.7817585468292236 + }, + { + "auxiliary_loss_clip": 0.0112155, + "auxiliary_loss_mlp": 0.01035296, + "balance_loss_clip": 1.03813446, + "balance_loss_mlp": 1.02067494, + "epoch": 0.7331209042265376, + "flos": 22492074343680.0, + "grad_norm": 3.4379579825405076, + "language_loss": 0.82023621, + "learning_rate": 7.016074589961784e-07, + "loss": 0.84180468, + "num_input_tokens_seen": 131012725, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.1461792, + "step": 6097, + "time_per_iteration": 2.796980381011963 + }, + { + "auxiliary_loss_clip": 0.01124526, + "auxiliary_loss_mlp": 0.01036995, + "balance_loss_clip": 1.0408802, + "balance_loss_mlp": 1.02185547, + "epoch": 0.7332411471171767, + "flos": 33072757937280.0, + "grad_norm": 1.7258688169625653, + "language_loss": 0.66764581, + "learning_rate": 7.01015055353253e-07, + "loss": 0.68926102, + "num_input_tokens_seen": 131035150, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.15161133, + "step": 6098, + "time_per_iteration": 2.924449920654297 + }, + { + "auxiliary_loss_clip": 0.01131207, + "auxiliary_loss_mlp": 0.01040727, + "balance_loss_clip": 1.04294467, + "balance_loss_mlp": 1.02523005, + "epoch": 0.7333613900078157, + "flos": 22743017735040.0, + "grad_norm": 2.064579836555721, + "language_loss": 0.77826244, + "learning_rate": 7.004228487653123e-07, + "loss": 0.79998183, + "num_input_tokens_seen": 131055955, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.15490723, + "step": 6099, + "time_per_iteration": 2.8113300800323486 + }, + { + "auxiliary_loss_clip": 0.01126692, + "auxiliary_loss_mlp": 0.01038555, + "balance_loss_clip": 1.04021573, + "balance_loss_mlp": 1.02319479, + "epoch": 0.7334816328984549, + "flos": 22346133384960.0, + "grad_norm": 1.8658134457545497, + "language_loss": 0.78275782, + "learning_rate": 6.998308393221906e-07, + "loss": 0.80441028, + "num_input_tokens_seen": 131074360, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15350342, + "step": 6100, + "time_per_iteration": 2.7993452548980713 + }, + { + "auxiliary_loss_clip": 0.01130153, + "auxiliary_loss_mlp": 0.01035536, + "balance_loss_clip": 1.0431788, + "balance_loss_mlp": 1.01999068, + "epoch": 0.733601875789094, + "flos": 20736149984640.0, + "grad_norm": 2.073928132139127, + "language_loss": 0.7130959, + "learning_rate": 6.992390271136977e-07, + "loss": 0.73475277, + "num_input_tokens_seen": 131090070, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15545654, + "step": 6101, + "time_per_iteration": 2.915249824523926 + }, + { + "auxiliary_loss_clip": 0.01123136, + "auxiliary_loss_mlp": 0.01037678, + "balance_loss_clip": 1.03888333, + "balance_loss_mlp": 1.0209527, + "epoch": 0.733722118679733, + "flos": 22564362464640.0, + "grad_norm": 1.7115178099005186, + "language_loss": 0.85866487, + "learning_rate": 6.986474122296094e-07, + "loss": 0.88027298, + "num_input_tokens_seen": 131109185, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.16711426, + "step": 6102, + "time_per_iteration": 3.059943437576294 + }, + { + "auxiliary_loss_clip": 0.01131401, + "auxiliary_loss_mlp": 0.01044324, + "balance_loss_clip": 1.04321551, + "balance_loss_mlp": 1.02798033, + "epoch": 0.7338423615703722, + "flos": 20084192179200.0, + "grad_norm": 1.7958983077346526, + "language_loss": 0.72494328, + "learning_rate": 6.980559947596751e-07, + "loss": 0.74670053, + "num_input_tokens_seen": 131127725, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.16333008, + "step": 6103, + "time_per_iteration": 2.9758377075195312 + }, + { + "auxiliary_loss_clip": 0.01128214, + "auxiliary_loss_mlp": 0.01038137, + "balance_loss_clip": 1.04285789, + "balance_loss_mlp": 1.02202034, + "epoch": 0.7339626044610112, + "flos": 21687675217920.0, + "grad_norm": 1.75816943928647, + "language_loss": 0.75710404, + "learning_rate": 6.974647747936109e-07, + "loss": 0.77876759, + "num_input_tokens_seen": 131146110, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.16125488, + "step": 6104, + "time_per_iteration": 2.8515262603759766 + }, + { + "auxiliary_loss_clip": 0.01127813, + "auxiliary_loss_mlp": 0.01035065, + "balance_loss_clip": 1.04208946, + "balance_loss_mlp": 1.01981854, + "epoch": 0.7340828473516503, + "flos": 15268248282240.0, + "grad_norm": 2.0546649670092187, + "language_loss": 0.82253736, + "learning_rate": 6.968737524211039e-07, + "loss": 0.84416616, + "num_input_tokens_seen": 131162920, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15270996, + "step": 6105, + "time_per_iteration": 2.8651912212371826 + }, + { + "auxiliary_loss_clip": 0.01130836, + "auxiliary_loss_mlp": 0.01033276, + "balance_loss_clip": 1.04277909, + "balance_loss_mlp": 1.0176239, + "epoch": 0.7342030902422895, + "flos": 22930112701440.0, + "grad_norm": 1.8975285090009635, + "language_loss": 0.80217481, + "learning_rate": 6.962829277318132e-07, + "loss": 0.82381594, + "num_input_tokens_seen": 131182515, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.15661621, + "step": 6106, + "time_per_iteration": 2.7818105220794678 + }, + { + "auxiliary_loss_clip": 0.01128638, + "auxiliary_loss_mlp": 0.01042544, + "balance_loss_clip": 1.04273093, + "balance_loss_mlp": 1.02729166, + "epoch": 0.7343233331329285, + "flos": 25847890381440.0, + "grad_norm": 2.0896547347306575, + "language_loss": 0.83465791, + "learning_rate": 6.956923008153652e-07, + "loss": 0.85636967, + "num_input_tokens_seen": 131202280, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.15252686, + "step": 6107, + "time_per_iteration": 3.5382306575775146 + }, + { + "auxiliary_loss_clip": 0.01124647, + "auxiliary_loss_mlp": 0.01033712, + "balance_loss_clip": 1.04088271, + "balance_loss_mlp": 1.01904321, + "epoch": 0.7344435760235676, + "flos": 18478985287680.0, + "grad_norm": 2.501456186710032, + "language_loss": 0.84514999, + "learning_rate": 6.951018717613593e-07, + "loss": 0.86673355, + "num_input_tokens_seen": 131221295, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.14666748, + "step": 6108, + "time_per_iteration": 2.801377534866333 + }, + { + "auxiliary_loss_clip": 0.01127691, + "auxiliary_loss_mlp": 0.0103252, + "balance_loss_clip": 1.04163623, + "balance_loss_mlp": 1.01824498, + "epoch": 0.7345638189142067, + "flos": 17640040256640.0, + "grad_norm": 1.9107102075190754, + "language_loss": 0.78220809, + "learning_rate": 6.945116406593614e-07, + "loss": 0.80381018, + "num_input_tokens_seen": 131240150, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.14282227, + "step": 6109, + "time_per_iteration": 2.8450515270233154 + }, + { + "auxiliary_loss_clip": 0.01125859, + "auxiliary_loss_mlp": 0.01042435, + "balance_loss_clip": 1.0396148, + "balance_loss_mlp": 1.02687836, + "epoch": 0.7346840618048458, + "flos": 20260225756800.0, + "grad_norm": 2.5816836124566787, + "language_loss": 0.74519062, + "learning_rate": 6.939216075989089e-07, + "loss": 0.76687354, + "num_input_tokens_seen": 131258080, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.15539551, + "step": 6110, + "time_per_iteration": 2.769530773162842 + }, + { + "auxiliary_loss_clip": 0.01126709, + "auxiliary_loss_mlp": 0.01037082, + "balance_loss_clip": 1.04150867, + "balance_loss_mlp": 1.02104878, + "epoch": 0.7348043046954849, + "flos": 29023183641600.0, + "grad_norm": 1.674617647077264, + "language_loss": 0.66129911, + "learning_rate": 6.933317726695109e-07, + "loss": 0.68293703, + "num_input_tokens_seen": 131279310, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.16027832, + "step": 6111, + "time_per_iteration": 4.457306385040283 + }, + { + "auxiliary_loss_clip": 0.01125526, + "auxiliary_loss_mlp": 0.01033006, + "balance_loss_clip": 1.04084516, + "balance_loss_mlp": 1.01884401, + "epoch": 0.734924547586124, + "flos": 17931203902080.0, + "grad_norm": 2.462189118839094, + "language_loss": 0.79718864, + "learning_rate": 6.92742135960644e-07, + "loss": 0.81877393, + "num_input_tokens_seen": 131297010, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.14172363, + "step": 6112, + "time_per_iteration": 2.777839422225952 + }, + { + "auxiliary_loss_clip": 0.01035706, + "auxiliary_loss_mlp": 0.01002178, + "balance_loss_clip": 1.00817907, + "balance_loss_mlp": 1.00074148, + "epoch": 0.7350447904767631, + "flos": 63588319850880.0, + "grad_norm": 0.8200824929690679, + "language_loss": 0.55678916, + "learning_rate": 6.921526975617556e-07, + "loss": 0.57716811, + "num_input_tokens_seen": 131356470, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01434326, + "step": 6113, + "time_per_iteration": 3.394534111022949 + }, + { + "auxiliary_loss_clip": 0.01128345, + "auxiliary_loss_mlp": 0.01046552, + "balance_loss_clip": 1.04174197, + "balance_loss_mlp": 1.03022075, + "epoch": 0.7351650333674021, + "flos": 21580015178880.0, + "grad_norm": 1.9333307207478043, + "language_loss": 0.75133353, + "learning_rate": 6.915634575622631e-07, + "loss": 0.77308249, + "num_input_tokens_seen": 131374985, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.16357422, + "step": 6114, + "time_per_iteration": 2.9251325130462646 + }, + { + "auxiliary_loss_clip": 0.01127237, + "auxiliary_loss_mlp": 0.01036674, + "balance_loss_clip": 1.04150426, + "balance_loss_mlp": 1.02099764, + "epoch": 0.7352852762580413, + "flos": 18186349184640.0, + "grad_norm": 1.9424142684093295, + "language_loss": 0.70757651, + "learning_rate": 6.909744160515532e-07, + "loss": 0.72921562, + "num_input_tokens_seen": 131393125, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15673828, + "step": 6115, + "time_per_iteration": 2.865692138671875 + }, + { + "auxiliary_loss_clip": 0.01124342, + "auxiliary_loss_mlp": 0.01033748, + "balance_loss_clip": 1.0389626, + "balance_loss_mlp": 1.01704657, + "epoch": 0.7354055191486804, + "flos": 38910073063680.0, + "grad_norm": 1.6972956354966504, + "language_loss": 0.6926477, + "learning_rate": 6.903855731189849e-07, + "loss": 0.71422857, + "num_input_tokens_seen": 131415760, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.16699219, + "step": 6116, + "time_per_iteration": 3.0049219131469727 + }, + { + "auxiliary_loss_clip": 0.01131931, + "auxiliary_loss_mlp": 0.01040545, + "balance_loss_clip": 1.04135108, + "balance_loss_mlp": 1.02368844, + "epoch": 0.7355257620393194, + "flos": 16289978647680.0, + "grad_norm": 2.806729827045246, + "language_loss": 0.82050776, + "learning_rate": 6.897969288538825e-07, + "loss": 0.84223247, + "num_input_tokens_seen": 131433705, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.16845703, + "step": 6117, + "time_per_iteration": 3.770407199859619 + }, + { + "auxiliary_loss_clip": 0.0112509, + "auxiliary_loss_mlp": 0.01034502, + "balance_loss_clip": 1.03983855, + "balance_loss_mlp": 1.0192076, + "epoch": 0.7356460049299585, + "flos": 18114240631680.0, + "grad_norm": 2.2801522527802423, + "language_loss": 0.81573296, + "learning_rate": 6.892084833455452e-07, + "loss": 0.83732891, + "num_input_tokens_seen": 131453275, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.1529541, + "step": 6118, + "time_per_iteration": 2.8769326210021973 + }, + { + "auxiliary_loss_clip": 0.01125861, + "auxiliary_loss_mlp": 0.01034994, + "balance_loss_clip": 1.04095638, + "balance_loss_mlp": 1.01937783, + "epoch": 0.7357662478205976, + "flos": 21325193118720.0, + "grad_norm": 1.4765578077255785, + "language_loss": 0.83779418, + "learning_rate": 6.886202366832384e-07, + "loss": 0.85940272, + "num_input_tokens_seen": 131474960, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15606689, + "step": 6119, + "time_per_iteration": 2.8818392753601074 + }, + { + "auxiliary_loss_clip": 0.01127307, + "auxiliary_loss_mlp": 0.01038628, + "balance_loss_clip": 1.03987205, + "balance_loss_mlp": 1.02313125, + "epoch": 0.7358864907112367, + "flos": 14246841139200.0, + "grad_norm": 1.8464007037686379, + "language_loss": 0.73797548, + "learning_rate": 6.880321889561987e-07, + "loss": 0.75963485, + "num_input_tokens_seen": 131492935, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.1550293, + "step": 6120, + "time_per_iteration": 2.845226287841797 + }, + { + "auxiliary_loss_clip": 0.01134381, + "auxiliary_loss_mlp": 0.01046347, + "balance_loss_clip": 1.04102457, + "balance_loss_mlp": 1.02969372, + "epoch": 0.7360067336018757, + "flos": 22309684058880.0, + "grad_norm": 2.31995398982488, + "language_loss": 0.65451807, + "learning_rate": 6.874443402536338e-07, + "loss": 0.67632532, + "num_input_tokens_seen": 131512025, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.16662598, + "step": 6121, + "time_per_iteration": 2.8704023361206055 + }, + { + "auxiliary_loss_clip": 0.01128718, + "auxiliary_loss_mlp": 0.01038616, + "balance_loss_clip": 1.04290712, + "balance_loss_mlp": 1.02241576, + "epoch": 0.7361269764925149, + "flos": 25554607833600.0, + "grad_norm": 1.7632612246812487, + "language_loss": 0.80449033, + "learning_rate": 6.868566906647177e-07, + "loss": 0.82616371, + "num_input_tokens_seen": 131532975, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.1619873, + "step": 6122, + "time_per_iteration": 2.8596580028533936 + }, + { + "auxiliary_loss_clip": 0.01131579, + "auxiliary_loss_mlp": 0.01037722, + "balance_loss_clip": 1.04304385, + "balance_loss_mlp": 1.02221286, + "epoch": 0.736247219383154, + "flos": 20376505059840.0, + "grad_norm": 1.7963872597973134, + "language_loss": 0.83399713, + "learning_rate": 6.862692402785984e-07, + "loss": 0.85569012, + "num_input_tokens_seen": 131553225, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.1550293, + "step": 6123, + "time_per_iteration": 2.885282278060913 + }, + { + "auxiliary_loss_clip": 0.01035361, + "auxiliary_loss_mlp": 0.01000384, + "balance_loss_clip": 1.0079006, + "balance_loss_mlp": 0.99895334, + "epoch": 0.736367462273793, + "flos": 70339525735680.0, + "grad_norm": 0.6890248274102146, + "language_loss": 0.4959946, + "learning_rate": 6.856819891843899e-07, + "loss": 0.51635206, + "num_input_tokens_seen": 131617930, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01428223, + "step": 6124, + "time_per_iteration": 3.485323190689087 + }, + { + "auxiliary_loss_clip": 0.01130349, + "auxiliary_loss_mlp": 0.01040707, + "balance_loss_clip": 1.042665, + "balance_loss_mlp": 1.02474451, + "epoch": 0.7364877051644322, + "flos": 22412711243520.0, + "grad_norm": 1.8885543813859387, + "language_loss": 0.72281528, + "learning_rate": 6.8509493747118e-07, + "loss": 0.74452579, + "num_input_tokens_seen": 131636740, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.15966797, + "step": 6125, + "time_per_iteration": 2.8562426567077637 + }, + { + "auxiliary_loss_clip": 0.01129319, + "auxiliary_loss_mlp": 0.0104156, + "balance_loss_clip": 1.04283142, + "balance_loss_mlp": 1.02583599, + "epoch": 0.7366079480550712, + "flos": 12130266274560.0, + "grad_norm": 2.3116344805798725, + "language_loss": 0.88318574, + "learning_rate": 6.845080852280221e-07, + "loss": 0.90489447, + "num_input_tokens_seen": 131653810, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15722656, + "step": 6126, + "time_per_iteration": 2.782503843307495 + }, + { + "auxiliary_loss_clip": 0.01125925, + "auxiliary_loss_mlp": 0.01036984, + "balance_loss_clip": 1.04156041, + "balance_loss_mlp": 1.0221132, + "epoch": 0.7367281909457103, + "flos": 15049336844160.0, + "grad_norm": 1.8052355032301226, + "language_loss": 0.74261343, + "learning_rate": 6.839214325439409e-07, + "loss": 0.76424253, + "num_input_tokens_seen": 131671505, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14874268, + "step": 6127, + "time_per_iteration": 2.813814640045166 + }, + { + "auxiliary_loss_clip": 0.01123687, + "auxiliary_loss_mlp": 0.01040267, + "balance_loss_clip": 1.04062366, + "balance_loss_mlp": 1.02484179, + "epoch": 0.7368484338363495, + "flos": 23510752053120.0, + "grad_norm": 2.257754599043877, + "language_loss": 0.71598709, + "learning_rate": 6.833349795079327e-07, + "loss": 0.73762661, + "num_input_tokens_seen": 131690615, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.15441895, + "step": 6128, + "time_per_iteration": 2.876967191696167 + }, + { + "auxiliary_loss_clip": 0.01128204, + "auxiliary_loss_mlp": 0.01036889, + "balance_loss_clip": 1.04076731, + "balance_loss_mlp": 1.02212465, + "epoch": 0.7369686767269885, + "flos": 27417833095680.0, + "grad_norm": 1.7134656575533025, + "language_loss": 0.68782938, + "learning_rate": 6.827487262089613e-07, + "loss": 0.70948035, + "num_input_tokens_seen": 131711120, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.14746094, + "step": 6129, + "time_per_iteration": 2.931427240371704 + }, + { + "auxiliary_loss_clip": 0.01034648, + "auxiliary_loss_mlp": 0.00999616, + "balance_loss_clip": 1.00715542, + "balance_loss_mlp": 0.99819708, + "epoch": 0.7370889196176276, + "flos": 70293343824000.0, + "grad_norm": 0.8871290032727192, + "language_loss": 0.56768763, + "learning_rate": 6.821626727359606e-07, + "loss": 0.58803022, + "num_input_tokens_seen": 131776680, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01416016, + "step": 6130, + "time_per_iteration": 3.3635079860687256 + }, + { + "auxiliary_loss_clip": 0.01132519, + "auxiliary_loss_mlp": 0.01039972, + "balance_loss_clip": 1.04281747, + "balance_loss_mlp": 1.02374792, + "epoch": 0.7372091625082667, + "flos": 18040839189120.0, + "grad_norm": 2.176954870964674, + "language_loss": 0.76766109, + "learning_rate": 6.815768191778348e-07, + "loss": 0.78938603, + "num_input_tokens_seen": 131794760, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.16210938, + "step": 6131, + "time_per_iteration": 2.816373825073242 + }, + { + "auxiliary_loss_clip": 0.01129138, + "auxiliary_loss_mlp": 0.01038962, + "balance_loss_clip": 1.04212403, + "balance_loss_mlp": 1.02246368, + "epoch": 0.7373294053989058, + "flos": 33726331854720.0, + "grad_norm": 1.6874669030804623, + "language_loss": 0.73108846, + "learning_rate": 6.809911656234569e-07, + "loss": 0.75276947, + "num_input_tokens_seen": 131816735, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.16503906, + "step": 6132, + "time_per_iteration": 2.876492977142334 + }, + { + "auxiliary_loss_clip": 0.01126069, + "auxiliary_loss_mlp": 0.01035904, + "balance_loss_clip": 1.03966939, + "balance_loss_mlp": 1.02112174, + "epoch": 0.7374496482895448, + "flos": 21506326427520.0, + "grad_norm": 2.1108712047827103, + "language_loss": 0.78437865, + "learning_rate": 6.804057121616707e-07, + "loss": 0.80599838, + "num_input_tokens_seen": 131834940, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.14782715, + "step": 6133, + "time_per_iteration": 2.7825379371643066 + }, + { + "auxiliary_loss_clip": 0.01125629, + "auxiliary_loss_mlp": 0.010425, + "balance_loss_clip": 1.03971553, + "balance_loss_mlp": 1.0268954, + "epoch": 0.737569891180184, + "flos": 24936908624640.0, + "grad_norm": 1.7953337092392034, + "language_loss": 0.72099388, + "learning_rate": 6.798204588812888e-07, + "loss": 0.74267519, + "num_input_tokens_seen": 131854355, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.15600586, + "step": 6134, + "time_per_iteration": 2.7866854667663574 + }, + { + "auxiliary_loss_clip": 0.01125178, + "auxiliary_loss_mlp": 0.01035858, + "balance_loss_clip": 1.0402267, + "balance_loss_mlp": 1.02142143, + "epoch": 0.7376901340708231, + "flos": 20664544222080.0, + "grad_norm": 1.657551113395549, + "language_loss": 0.7548542, + "learning_rate": 6.792354058710937e-07, + "loss": 0.77646458, + "num_input_tokens_seen": 131871825, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.14428711, + "step": 6135, + "time_per_iteration": 2.8178646564483643 + }, + { + "auxiliary_loss_clip": 0.01119139, + "auxiliary_loss_mlp": 0.01029083, + "balance_loss_clip": 1.03874707, + "balance_loss_mlp": 1.01476622, + "epoch": 0.7378103769614621, + "flos": 23805794367360.0, + "grad_norm": 1.8887195566807093, + "language_loss": 0.65282542, + "learning_rate": 6.786505532198374e-07, + "loss": 0.6743077, + "num_input_tokens_seen": 131890770, + "router_z_loss_clip": 0.80419922, + "router_z_loss_mlp": 0.14331055, + "step": 6136, + "time_per_iteration": 3.6889514923095703 + }, + { + "auxiliary_loss_clip": 0.01126404, + "auxiliary_loss_mlp": 0.01040441, + "balance_loss_clip": 1.04123104, + "balance_loss_mlp": 1.0249083, + "epoch": 0.7379306198521013, + "flos": 22237216369920.0, + "grad_norm": 1.6430427539663055, + "language_loss": 0.8535881, + "learning_rate": 6.780659010162411e-07, + "loss": 0.87525654, + "num_input_tokens_seen": 131909720, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.15527344, + "step": 6137, + "time_per_iteration": 2.8771588802337646 + }, + { + "auxiliary_loss_clip": 0.01122308, + "auxiliary_loss_mlp": 0.01037745, + "balance_loss_clip": 1.03919184, + "balance_loss_mlp": 1.02422071, + "epoch": 0.7380508627427403, + "flos": 14903108576640.0, + "grad_norm": 1.6044807503111358, + "language_loss": 0.83305562, + "learning_rate": 6.774814493489975e-07, + "loss": 0.85465616, + "num_input_tokens_seen": 131927395, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.1350708, + "step": 6138, + "time_per_iteration": 2.8428080081939697 + }, + { + "auxiliary_loss_clip": 0.01123508, + "auxiliary_loss_mlp": 0.01036326, + "balance_loss_clip": 1.04030848, + "balance_loss_mlp": 1.02141309, + "epoch": 0.7381711056333794, + "flos": 21685843624320.0, + "grad_norm": 1.7284893010682063, + "language_loss": 0.66340262, + "learning_rate": 6.768971983067655e-07, + "loss": 0.68500096, + "num_input_tokens_seen": 131947725, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.14916992, + "step": 6139, + "time_per_iteration": 2.9015204906463623 + }, + { + "auxiliary_loss_clip": 0.01034716, + "auxiliary_loss_mlp": 0.00999896, + "balance_loss_clip": 1.00717402, + "balance_loss_mlp": 0.99848956, + "epoch": 0.7382913485240186, + "flos": 52404263596800.0, + "grad_norm": 1.011430843296247, + "language_loss": 0.67692786, + "learning_rate": 6.763131479781772e-07, + "loss": 0.69727403, + "num_input_tokens_seen": 131997485, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01403809, + "step": 6140, + "time_per_iteration": 4.722250938415527 + }, + { + "auxiliary_loss_clip": 0.01124335, + "auxiliary_loss_mlp": 0.01034682, + "balance_loss_clip": 1.03983915, + "balance_loss_mlp": 1.02051437, + "epoch": 0.7384115914146576, + "flos": 21798818876160.0, + "grad_norm": 1.7778992057228373, + "language_loss": 0.76251096, + "learning_rate": 6.757292984518316e-07, + "loss": 0.78410113, + "num_input_tokens_seen": 132016885, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.14160156, + "step": 6141, + "time_per_iteration": 2.8529865741729736 + }, + { + "auxiliary_loss_clip": 0.01034274, + "auxiliary_loss_mlp": 0.01000899, + "balance_loss_clip": 1.00670981, + "balance_loss_mlp": 0.99951571, + "epoch": 0.7385318343052967, + "flos": 61494331662720.0, + "grad_norm": 0.80528173212964, + "language_loss": 0.56394529, + "learning_rate": 6.751456498162981e-07, + "loss": 0.584297, + "num_input_tokens_seen": 132075920, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01385498, + "step": 6142, + "time_per_iteration": 3.257092237472534 + }, + { + "auxiliary_loss_clip": 0.01125758, + "auxiliary_loss_mlp": 0.01037687, + "balance_loss_clip": 1.03983545, + "balance_loss_mlp": 1.02313757, + "epoch": 0.7386520771959358, + "flos": 17013757697280.0, + "grad_norm": 2.4335901732743914, + "language_loss": 0.85551471, + "learning_rate": 6.745622021601174e-07, + "loss": 0.87714911, + "num_input_tokens_seen": 132092945, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.14544678, + "step": 6143, + "time_per_iteration": 2.872204065322876 + }, + { + "auxiliary_loss_clip": 0.01127237, + "auxiliary_loss_mlp": 0.01040225, + "balance_loss_clip": 1.03931355, + "balance_loss_mlp": 1.02483475, + "epoch": 0.7387723200865749, + "flos": 18770759464320.0, + "grad_norm": 1.9858600682282586, + "language_loss": 0.69637716, + "learning_rate": 6.739789555717954e-07, + "loss": 0.71805179, + "num_input_tokens_seen": 132109920, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.15405273, + "step": 6144, + "time_per_iteration": 2.887779951095581 + }, + { + "auxiliary_loss_clip": 0.0112395, + "auxiliary_loss_mlp": 0.01043608, + "balance_loss_clip": 1.03898191, + "balance_loss_mlp": 1.02859974, + "epoch": 0.738892562977214, + "flos": 22525542840960.0, + "grad_norm": 1.9944950356546327, + "language_loss": 0.77725911, + "learning_rate": 6.733959101398124e-07, + "loss": 0.79893464, + "num_input_tokens_seen": 132128050, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.15002441, + "step": 6145, + "time_per_iteration": 2.9639956951141357 + }, + { + "auxiliary_loss_clip": 0.01124379, + "auxiliary_loss_mlp": 0.01031158, + "balance_loss_clip": 1.03883624, + "balance_loss_mlp": 1.01722276, + "epoch": 0.7390128058678531, + "flos": 21501478091520.0, + "grad_norm": 1.6492224695823827, + "language_loss": 0.81559676, + "learning_rate": 6.728130659526143e-07, + "loss": 0.83715212, + "num_input_tokens_seen": 132145860, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.13934326, + "step": 6146, + "time_per_iteration": 3.6786580085754395 + }, + { + "auxiliary_loss_clip": 0.0112866, + "auxiliary_loss_mlp": 0.01036198, + "balance_loss_clip": 1.04046059, + "balance_loss_mlp": 1.02120101, + "epoch": 0.7391330487584922, + "flos": 25776176878080.0, + "grad_norm": 1.9671460933512024, + "language_loss": 0.70778847, + "learning_rate": 6.7223042309862e-07, + "loss": 0.72943699, + "num_input_tokens_seen": 132166060, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.15002441, + "step": 6147, + "time_per_iteration": 2.8372063636779785 + }, + { + "auxiliary_loss_clip": 0.01125053, + "auxiliary_loss_mlp": 0.01035367, + "balance_loss_clip": 1.04080844, + "balance_loss_mlp": 1.02007246, + "epoch": 0.7392532916491312, + "flos": 28366736636160.0, + "grad_norm": 2.0041605986281983, + "language_loss": 0.7337631, + "learning_rate": 6.716479816662144e-07, + "loss": 0.75536728, + "num_input_tokens_seen": 132187790, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.1529541, + "step": 6148, + "time_per_iteration": 2.9044206142425537 + }, + { + "auxiliary_loss_clip": 0.01125507, + "auxiliary_loss_mlp": 0.01040399, + "balance_loss_clip": 1.04072416, + "balance_loss_mlp": 1.02521229, + "epoch": 0.7393735345397703, + "flos": 23585877348480.0, + "grad_norm": 1.931901094276829, + "language_loss": 0.73383594, + "learning_rate": 6.710657417437531e-07, + "loss": 0.75549501, + "num_input_tokens_seen": 132207495, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.1517334, + "step": 6149, + "time_per_iteration": 2.915483236312866 + }, + { + "auxiliary_loss_clip": 0.01126025, + "auxiliary_loss_mlp": 0.0103832, + "balance_loss_clip": 1.04010499, + "balance_loss_mlp": 1.02337122, + "epoch": 0.7394937774304094, + "flos": 19974772373760.0, + "grad_norm": 3.2407547177346245, + "language_loss": 0.80438608, + "learning_rate": 6.704837034195628e-07, + "loss": 0.82602954, + "num_input_tokens_seen": 132225960, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.1494751, + "step": 6150, + "time_per_iteration": 2.797502279281616 + }, + { + "auxiliary_loss_clip": 0.01126679, + "auxiliary_loss_mlp": 0.01035157, + "balance_loss_clip": 1.03939319, + "balance_loss_mlp": 1.01961207, + "epoch": 0.7396140203210485, + "flos": 23478037741440.0, + "grad_norm": 1.730568153543914, + "language_loss": 0.85116243, + "learning_rate": 6.699018667819376e-07, + "loss": 0.8727808, + "num_input_tokens_seen": 132245360, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15551758, + "step": 6151, + "time_per_iteration": 2.7809839248657227 + }, + { + "auxiliary_loss_clip": 0.01127087, + "auxiliary_loss_mlp": 0.01038404, + "balance_loss_clip": 1.03994429, + "balance_loss_mlp": 1.02197683, + "epoch": 0.7397342632116876, + "flos": 25555433846400.0, + "grad_norm": 1.8291907454002145, + "language_loss": 0.72794867, + "learning_rate": 6.693202319191415e-07, + "loss": 0.74960357, + "num_input_tokens_seen": 132267095, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.16418457, + "step": 6152, + "time_per_iteration": 2.8002593517303467 + }, + { + "auxiliary_loss_clip": 0.01127088, + "auxiliary_loss_mlp": 0.01044154, + "balance_loss_clip": 1.04415822, + "balance_loss_mlp": 1.02956867, + "epoch": 0.7398545061023267, + "flos": 24755021130240.0, + "grad_norm": 2.0199290541496975, + "language_loss": 0.75303322, + "learning_rate": 6.687387989194084e-07, + "loss": 0.77474564, + "num_input_tokens_seen": 132286610, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.14593506, + "step": 6153, + "time_per_iteration": 2.827441692352295 + }, + { + "auxiliary_loss_clip": 0.01121273, + "auxiliary_loss_mlp": 0.01037317, + "balance_loss_clip": 1.03811121, + "balance_loss_mlp": 1.02280354, + "epoch": 0.7399747489929658, + "flos": 16508602776960.0, + "grad_norm": 1.8188472050456022, + "language_loss": 0.79519093, + "learning_rate": 6.681575678709404e-07, + "loss": 0.81677681, + "num_input_tokens_seen": 132305300, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.14520264, + "step": 6154, + "time_per_iteration": 2.847003221511841 + }, + { + "auxiliary_loss_clip": 0.0112518, + "auxiliary_loss_mlp": 0.0104124, + "balance_loss_clip": 1.03898144, + "balance_loss_mlp": 1.02570748, + "epoch": 0.7400949918836048, + "flos": 24097065753600.0, + "grad_norm": 1.9187963369457997, + "language_loss": 0.7059232, + "learning_rate": 6.67576538861911e-07, + "loss": 0.72758734, + "num_input_tokens_seen": 132323875, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.15539551, + "step": 6155, + "time_per_iteration": 2.8069891929626465 + }, + { + "auxiliary_loss_clip": 0.01127212, + "auxiliary_loss_mlp": 0.01036662, + "balance_loss_clip": 1.04229426, + "balance_loss_mlp": 1.02222025, + "epoch": 0.740215234774244, + "flos": 21802517976960.0, + "grad_norm": 1.569811837099138, + "language_loss": 0.82156116, + "learning_rate": 6.669957119804612e-07, + "loss": 0.84319991, + "num_input_tokens_seen": 132345510, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.14453125, + "step": 6156, + "time_per_iteration": 2.8810839653015137 + }, + { + "auxiliary_loss_clip": 0.01126557, + "auxiliary_loss_mlp": 0.01040938, + "balance_loss_clip": 1.03851295, + "balance_loss_mlp": 1.02473712, + "epoch": 0.7403354776648831, + "flos": 18733196816640.0, + "grad_norm": 2.7870287900543538, + "language_loss": 0.72729588, + "learning_rate": 6.66415087314702e-07, + "loss": 0.74897081, + "num_input_tokens_seen": 132360465, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.1619873, + "step": 6157, + "time_per_iteration": 2.7782182693481445 + }, + { + "auxiliary_loss_clip": 0.01125743, + "auxiliary_loss_mlp": 0.010369, + "balance_loss_clip": 1.04115069, + "balance_loss_mlp": 1.02086616, + "epoch": 0.7404557205555221, + "flos": 16909581277440.0, + "grad_norm": 2.0100987062533333, + "language_loss": 0.73307484, + "learning_rate": 6.65834664952714e-07, + "loss": 0.75470126, + "num_input_tokens_seen": 132377915, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.16027832, + "step": 6158, + "time_per_iteration": 2.824154853820801 + }, + { + "auxiliary_loss_clip": 0.01124491, + "auxiliary_loss_mlp": 0.01044336, + "balance_loss_clip": 1.0406487, + "balance_loss_mlp": 1.02958417, + "epoch": 0.7405759634461613, + "flos": 21214408596480.0, + "grad_norm": 1.5842479760095827, + "language_loss": 0.75901502, + "learning_rate": 6.652544449825457e-07, + "loss": 0.78070331, + "num_input_tokens_seen": 132398170, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14752197, + "step": 6159, + "time_per_iteration": 2.8487396240234375 + }, + { + "auxiliary_loss_clip": 0.01129081, + "auxiliary_loss_mlp": 0.01037422, + "balance_loss_clip": 1.0401721, + "balance_loss_mlp": 1.0209713, + "epoch": 0.7406962063368003, + "flos": 20480106862080.0, + "grad_norm": 1.54529602972746, + "language_loss": 0.76430488, + "learning_rate": 6.646744274922182e-07, + "loss": 0.78596997, + "num_input_tokens_seen": 132416615, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.16455078, + "step": 6160, + "time_per_iteration": 2.9012739658355713 + }, + { + "auxiliary_loss_clip": 0.01125219, + "auxiliary_loss_mlp": 0.01034947, + "balance_loss_clip": 1.04065776, + "balance_loss_mlp": 1.01991487, + "epoch": 0.7408164492274394, + "flos": 19791915212160.0, + "grad_norm": 2.72058153504397, + "language_loss": 0.75593996, + "learning_rate": 6.640946125697171e-07, + "loss": 0.77754164, + "num_input_tokens_seen": 132434145, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.15039062, + "step": 6161, + "time_per_iteration": 2.9131906032562256 + }, + { + "auxiliary_loss_clip": 0.01129714, + "auxiliary_loss_mlp": 0.01040456, + "balance_loss_clip": 1.04002392, + "balance_loss_mlp": 1.02389765, + "epoch": 0.7409366921180786, + "flos": 29204855654400.0, + "grad_norm": 1.8359463837668761, + "language_loss": 0.75261116, + "learning_rate": 6.635150003030017e-07, + "loss": 0.77431285, + "num_input_tokens_seen": 132452670, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.16577148, + "step": 6162, + "time_per_iteration": 2.814851760864258 + }, + { + "auxiliary_loss_clip": 0.01123704, + "auxiliary_loss_mlp": 0.0103899, + "balance_loss_clip": 1.03884315, + "balance_loss_mlp": 1.02413011, + "epoch": 0.7410569350087176, + "flos": 22930004960640.0, + "grad_norm": 1.9106045680574502, + "language_loss": 0.85530782, + "learning_rate": 6.629355907799981e-07, + "loss": 0.87693477, + "num_input_tokens_seen": 132472475, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.14862061, + "step": 6163, + "time_per_iteration": 2.805128574371338 + }, + { + "auxiliary_loss_clip": 0.01128462, + "auxiliary_loss_mlp": 0.01038227, + "balance_loss_clip": 1.03975153, + "balance_loss_mlp": 1.02327847, + "epoch": 0.7411771778993567, + "flos": 30440397726720.0, + "grad_norm": 1.7086378900838173, + "language_loss": 0.68968558, + "learning_rate": 6.623563840886015e-07, + "loss": 0.71135253, + "num_input_tokens_seen": 132493400, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.14953613, + "step": 6164, + "time_per_iteration": 2.8484489917755127 + }, + { + "auxiliary_loss_clip": 0.01123615, + "auxiliary_loss_mlp": 0.01034499, + "balance_loss_clip": 1.04067171, + "balance_loss_mlp": 1.02009845, + "epoch": 0.7412974207899958, + "flos": 20522050968960.0, + "grad_norm": 1.584680928056336, + "language_loss": 0.69617224, + "learning_rate": 6.617773803166795e-07, + "loss": 0.71775341, + "num_input_tokens_seen": 132511725, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.14416504, + "step": 6165, + "time_per_iteration": 3.4627909660339355 + }, + { + "auxiliary_loss_clip": 0.0112985, + "auxiliary_loss_mlp": 0.01048504, + "balance_loss_clip": 1.04117513, + "balance_loss_mlp": 1.03199315, + "epoch": 0.7414176636806349, + "flos": 22090700793600.0, + "grad_norm": 2.3263370221151063, + "language_loss": 0.8219474, + "learning_rate": 6.611985795520634e-07, + "loss": 0.84373093, + "num_input_tokens_seen": 132530270, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.16516113, + "step": 6166, + "time_per_iteration": 2.767756938934326 + }, + { + "auxiliary_loss_clip": 0.01135594, + "auxiliary_loss_mlp": 0.01038168, + "balance_loss_clip": 1.04442382, + "balance_loss_mlp": 1.02233696, + "epoch": 0.7415379065712739, + "flos": 25155245445120.0, + "grad_norm": 2.1352629179300404, + "language_loss": 0.77289736, + "learning_rate": 6.606199818825588e-07, + "loss": 0.79463506, + "num_input_tokens_seen": 132550725, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.1583252, + "step": 6167, + "time_per_iteration": 2.7730727195739746 + }, + { + "auxiliary_loss_clip": 0.01123357, + "auxiliary_loss_mlp": 0.01038651, + "balance_loss_clip": 1.03895712, + "balance_loss_mlp": 1.02259398, + "epoch": 0.7416581494619131, + "flos": 16871731320960.0, + "grad_norm": 2.0120060081301485, + "language_loss": 0.81983948, + "learning_rate": 6.600415873959377e-07, + "loss": 0.84145963, + "num_input_tokens_seen": 132568600, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.16052246, + "step": 6168, + "time_per_iteration": 2.7600393295288086 + }, + { + "auxiliary_loss_clip": 0.0112152, + "auxiliary_loss_mlp": 0.01037392, + "balance_loss_clip": 1.04014337, + "balance_loss_mlp": 1.02290201, + "epoch": 0.7417783923525522, + "flos": 28438881102720.0, + "grad_norm": 2.1842941460901777, + "language_loss": 0.64828843, + "learning_rate": 6.594633961799437e-07, + "loss": 0.66987753, + "num_input_tokens_seen": 132587640, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.14477539, + "step": 6169, + "time_per_iteration": 4.363194942474365 + }, + { + "auxiliary_loss_clip": 0.01126327, + "auxiliary_loss_mlp": 0.01042063, + "balance_loss_clip": 1.04048347, + "balance_loss_mlp": 1.02694714, + "epoch": 0.7418986352431912, + "flos": 20084299920000.0, + "grad_norm": 1.7238878243577933, + "language_loss": 0.81730229, + "learning_rate": 6.588854083222857e-07, + "loss": 0.83898616, + "num_input_tokens_seen": 132607075, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15124512, + "step": 6170, + "time_per_iteration": 2.88240385055542 + }, + { + "auxiliary_loss_clip": 0.011301, + "auxiliary_loss_mlp": 0.01042709, + "balance_loss_clip": 1.04108477, + "balance_loss_mlp": 1.02649617, + "epoch": 0.7420188781338304, + "flos": 18259571059200.0, + "grad_norm": 1.9741937315185951, + "language_loss": 0.80702567, + "learning_rate": 6.583076239106444e-07, + "loss": 0.82875383, + "num_input_tokens_seen": 132625580, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.16210938, + "step": 6171, + "time_per_iteration": 2.7981278896331787 + }, + { + "auxiliary_loss_clip": 0.01125972, + "auxiliary_loss_mlp": 0.01046762, + "balance_loss_clip": 1.0392766, + "balance_loss_mlp": 1.03079951, + "epoch": 0.7421391210244694, + "flos": 13771994319360.0, + "grad_norm": 2.48348580659612, + "language_loss": 0.75816071, + "learning_rate": 6.577300430326707e-07, + "loss": 0.77988803, + "num_input_tokens_seen": 132640525, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.1595459, + "step": 6172, + "time_per_iteration": 2.833080768585205 + }, + { + "auxiliary_loss_clip": 0.01125276, + "auxiliary_loss_mlp": 0.01043805, + "balance_loss_clip": 1.03981638, + "balance_loss_mlp": 1.02852833, + "epoch": 0.7422593639151085, + "flos": 15961683317760.0, + "grad_norm": 1.9752886089393014, + "language_loss": 0.71885431, + "learning_rate": 6.571526657759821e-07, + "loss": 0.74054515, + "num_input_tokens_seen": 132656265, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15289307, + "step": 6173, + "time_per_iteration": 2.8627617359161377 + }, + { + "auxiliary_loss_clip": 0.0112584, + "auxiliary_loss_mlp": 0.01036885, + "balance_loss_clip": 1.03920555, + "balance_loss_mlp": 1.02141118, + "epoch": 0.7423796068057477, + "flos": 30114400867200.0, + "grad_norm": 1.8211199958056687, + "language_loss": 0.70571196, + "learning_rate": 6.565754922281663e-07, + "loss": 0.72733927, + "num_input_tokens_seen": 132678510, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15478516, + "step": 6174, + "time_per_iteration": 2.827826976776123 + }, + { + "auxiliary_loss_clip": 0.01126126, + "auxiliary_loss_mlp": 0.01043992, + "balance_loss_clip": 1.03932643, + "balance_loss_mlp": 1.02825665, + "epoch": 0.7424998496963867, + "flos": 20521907314560.0, + "grad_norm": 2.67752599369446, + "language_loss": 0.78421247, + "learning_rate": 6.559985224767801e-07, + "loss": 0.80591369, + "num_input_tokens_seen": 132696385, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.1574707, + "step": 6175, + "time_per_iteration": 3.709279775619507 + }, + { + "auxiliary_loss_clip": 0.01127981, + "auxiliary_loss_mlp": 0.01042432, + "balance_loss_clip": 1.04295969, + "balance_loss_mlp": 1.02716172, + "epoch": 0.7426200925870258, + "flos": 21871573873920.0, + "grad_norm": 2.6936211108663284, + "language_loss": 0.75085026, + "learning_rate": 6.55421756609349e-07, + "loss": 0.77255434, + "num_input_tokens_seen": 132714640, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15270996, + "step": 6176, + "time_per_iteration": 2.8482563495635986 + }, + { + "auxiliary_loss_clip": 0.01127028, + "auxiliary_loss_mlp": 0.01042886, + "balance_loss_clip": 1.0397476, + "balance_loss_mlp": 1.02794862, + "epoch": 0.7427403354776649, + "flos": 26432049265920.0, + "grad_norm": 1.868452347135128, + "language_loss": 0.79321337, + "learning_rate": 6.54845194713369e-07, + "loss": 0.8149125, + "num_input_tokens_seen": 132735590, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.14929199, + "step": 6177, + "time_per_iteration": 2.8493990898132324 + }, + { + "auxiliary_loss_clip": 0.01130575, + "auxiliary_loss_mlp": 0.01042215, + "balance_loss_clip": 1.04353321, + "balance_loss_mlp": 1.02691424, + "epoch": 0.742860578368304, + "flos": 19898390102400.0, + "grad_norm": 3.0223757665315927, + "language_loss": 0.79890716, + "learning_rate": 6.542688368763034e-07, + "loss": 0.82063508, + "num_input_tokens_seen": 132753995, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15301514, + "step": 6178, + "time_per_iteration": 2.816563129425049 + }, + { + "auxiliary_loss_clip": 0.01125193, + "auxiliary_loss_mlp": 0.01036201, + "balance_loss_clip": 1.03971529, + "balance_loss_mlp": 1.02194977, + "epoch": 0.742980821258943, + "flos": 24827201510400.0, + "grad_norm": 1.5594709090972447, + "language_loss": 0.77314246, + "learning_rate": 6.536926831855854e-07, + "loss": 0.79475641, + "num_input_tokens_seen": 132773160, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.14257812, + "step": 6179, + "time_per_iteration": 2.817723035812378 + }, + { + "auxiliary_loss_clip": 0.01124801, + "auxiliary_loss_mlp": 0.01030304, + "balance_loss_clip": 1.04003596, + "balance_loss_mlp": 1.01590908, + "epoch": 0.7431010641495821, + "flos": 25228646887680.0, + "grad_norm": 2.1000659965885977, + "language_loss": 0.72867942, + "learning_rate": 6.531167337286165e-07, + "loss": 0.75023043, + "num_input_tokens_seen": 132793180, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14379883, + "step": 6180, + "time_per_iteration": 2.8779795169830322 + }, + { + "auxiliary_loss_clip": 0.01125073, + "auxiliary_loss_mlp": 0.01042293, + "balance_loss_clip": 1.04069304, + "balance_loss_mlp": 1.02609277, + "epoch": 0.7432213070402213, + "flos": 21762369550080.0, + "grad_norm": 1.4510532083252348, + "language_loss": 0.79495531, + "learning_rate": 6.52540988592768e-07, + "loss": 0.81662899, + "num_input_tokens_seen": 132814200, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.1619873, + "step": 6181, + "time_per_iteration": 2.8736701011657715 + }, + { + "auxiliary_loss_clip": 0.01123373, + "auxiliary_loss_mlp": 0.01040138, + "balance_loss_clip": 1.03929114, + "balance_loss_mlp": 1.02557635, + "epoch": 0.7433415499308603, + "flos": 14793832425600.0, + "grad_norm": 2.0145431609722726, + "language_loss": 0.83208072, + "learning_rate": 6.519654478653814e-07, + "loss": 0.85371584, + "num_input_tokens_seen": 132832565, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.14556885, + "step": 6182, + "time_per_iteration": 2.7770302295684814 + }, + { + "auxiliary_loss_clip": 0.0103566, + "auxiliary_loss_mlp": 0.01006674, + "balance_loss_clip": 1.00797391, + "balance_loss_mlp": 1.00530362, + "epoch": 0.7434617928214994, + "flos": 67155577297920.0, + "grad_norm": 0.7406854147691356, + "language_loss": 0.56036752, + "learning_rate": 6.51390111633763e-07, + "loss": 0.58079088, + "num_input_tokens_seen": 132897840, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01373291, + "step": 6183, + "time_per_iteration": 3.3844048976898193 + }, + { + "auxiliary_loss_clip": 0.0112553, + "auxiliary_loss_mlp": 0.01040511, + "balance_loss_clip": 1.04198503, + "balance_loss_mlp": 1.02488279, + "epoch": 0.7435820357121385, + "flos": 27377576928000.0, + "grad_norm": 1.6853634550223826, + "language_loss": 0.76057088, + "learning_rate": 6.508149799851932e-07, + "loss": 0.78223133, + "num_input_tokens_seen": 132919505, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.15625, + "step": 6184, + "time_per_iteration": 2.8288731575012207 + }, + { + "auxiliary_loss_clip": 0.01123514, + "auxiliary_loss_mlp": 0.01038488, + "balance_loss_clip": 1.03813684, + "balance_loss_mlp": 1.0242784, + "epoch": 0.7437022786027776, + "flos": 23987645948160.0, + "grad_norm": 1.9780544400630333, + "language_loss": 0.61369622, + "learning_rate": 6.502400530069183e-07, + "loss": 0.63531625, + "num_input_tokens_seen": 132939390, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.14202881, + "step": 6185, + "time_per_iteration": 2.848031520843506 + }, + { + "auxiliary_loss_clip": 0.01135183, + "auxiliary_loss_mlp": 0.01039264, + "balance_loss_clip": 1.04440153, + "balance_loss_mlp": 1.02308762, + "epoch": 0.7438225214934167, + "flos": 21866761451520.0, + "grad_norm": 1.6258047630671162, + "language_loss": 0.68177724, + "learning_rate": 6.496653307861535e-07, + "loss": 0.70352173, + "num_input_tokens_seen": 132960060, + "router_z_loss_clip": 0.90820312, + "router_z_loss_mlp": 0.1618042, + "step": 6186, + "time_per_iteration": 2.8308041095733643 + }, + { + "auxiliary_loss_clip": 0.01126734, + "auxiliary_loss_mlp": 0.01040634, + "balance_loss_clip": 1.04124355, + "balance_loss_mlp": 1.0240159, + "epoch": 0.7439427643840558, + "flos": 20230097224320.0, + "grad_norm": 1.7296132705942155, + "language_loss": 0.65601778, + "learning_rate": 6.490908134100857e-07, + "loss": 0.67769146, + "num_input_tokens_seen": 132978525, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.16625977, + "step": 6187, + "time_per_iteration": 2.816934823989868 + }, + { + "auxiliary_loss_clip": 0.01126359, + "auxiliary_loss_mlp": 0.01034725, + "balance_loss_clip": 1.04134679, + "balance_loss_mlp": 1.02028835, + "epoch": 0.7440630072746949, + "flos": 20849915335680.0, + "grad_norm": 2.086944439178613, + "language_loss": 0.69593483, + "learning_rate": 6.48516500965866e-07, + "loss": 0.71754569, + "num_input_tokens_seen": 132998460, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.14422607, + "step": 6188, + "time_per_iteration": 2.874202013015747 + }, + { + "auxiliary_loss_clip": 0.01126933, + "auxiliary_loss_mlp": 0.01033034, + "balance_loss_clip": 1.03924358, + "balance_loss_mlp": 1.01745296, + "epoch": 0.7441832501653339, + "flos": 26503762769280.0, + "grad_norm": 1.731366349770576, + "language_loss": 0.81813329, + "learning_rate": 6.479423935406192e-07, + "loss": 0.83973294, + "num_input_tokens_seen": 133018445, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.15588379, + "step": 6189, + "time_per_iteration": 2.8138010501861572 + }, + { + "auxiliary_loss_clip": 0.01035076, + "auxiliary_loss_mlp": 0.0100411, + "balance_loss_clip": 1.007303, + "balance_loss_mlp": 1.00260758, + "epoch": 0.7443034930559731, + "flos": 68602848088320.0, + "grad_norm": 0.8060172656584136, + "language_loss": 0.61971068, + "learning_rate": 6.473684912214357e-07, + "loss": 0.64010251, + "num_input_tokens_seen": 133082005, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01501465, + "step": 6190, + "time_per_iteration": 3.548266649246216 + }, + { + "auxiliary_loss_clip": 0.01131298, + "auxiliary_loss_mlp": 0.01040256, + "balance_loss_clip": 1.04411352, + "balance_loss_mlp": 1.02502155, + "epoch": 0.7444237359466122, + "flos": 18654982951680.0, + "grad_norm": 2.033256153094802, + "language_loss": 0.69952404, + "learning_rate": 6.467947940953778e-07, + "loss": 0.72123969, + "num_input_tokens_seen": 133100530, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15222168, + "step": 6191, + "time_per_iteration": 2.910865545272827 + }, + { + "auxiliary_loss_clip": 0.01123852, + "auxiliary_loss_mlp": 0.01035281, + "balance_loss_clip": 1.04069948, + "balance_loss_mlp": 1.02069557, + "epoch": 0.7445439788372512, + "flos": 22817604326400.0, + "grad_norm": 1.7986033020772325, + "language_loss": 0.72699374, + "learning_rate": 6.462213022494732e-07, + "loss": 0.74858505, + "num_input_tokens_seen": 133119775, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.14593506, + "step": 6192, + "time_per_iteration": 2.7791295051574707 + }, + { + "auxiliary_loss_clip": 0.01034931, + "auxiliary_loss_mlp": 0.01002454, + "balance_loss_clip": 1.0071547, + "balance_loss_mlp": 1.00099945, + "epoch": 0.7446642217278904, + "flos": 67045690615680.0, + "grad_norm": 0.7727447082128373, + "language_loss": 0.61008912, + "learning_rate": 6.456480157707201e-07, + "loss": 0.630463, + "num_input_tokens_seen": 133184550, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01452637, + "step": 6193, + "time_per_iteration": 3.256612539291382 + }, + { + "auxiliary_loss_clip": 0.01125365, + "auxiliary_loss_mlp": 0.010349, + "balance_loss_clip": 1.04224706, + "balance_loss_mlp": 1.0194509, + "epoch": 0.7447844646185294, + "flos": 17417465631360.0, + "grad_norm": 2.0801115619252983, + "language_loss": 0.8512466, + "learning_rate": 6.450749347460866e-07, + "loss": 0.87284923, + "num_input_tokens_seen": 133201525, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.15454102, + "step": 6194, + "time_per_iteration": 3.525965929031372 + }, + { + "auxiliary_loss_clip": 0.01130854, + "auxiliary_loss_mlp": 0.01038204, + "balance_loss_clip": 1.04249978, + "balance_loss_mlp": 1.02217007, + "epoch": 0.7449047075091685, + "flos": 26615876094720.0, + "grad_norm": 1.7714484666297137, + "language_loss": 0.79219258, + "learning_rate": 6.445020592625083e-07, + "loss": 0.81388319, + "num_input_tokens_seen": 133222175, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16027832, + "step": 6195, + "time_per_iteration": 2.848478078842163 + }, + { + "auxiliary_loss_clip": 0.0112571, + "auxiliary_loss_mlp": 0.01042017, + "balance_loss_clip": 1.04038227, + "balance_loss_mlp": 1.02654326, + "epoch": 0.7450249503998077, + "flos": 14170458867840.0, + "grad_norm": 2.149386588738232, + "language_loss": 0.80393672, + "learning_rate": 6.4392938940689e-07, + "loss": 0.82561398, + "num_input_tokens_seen": 133237590, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15490723, + "step": 6196, + "time_per_iteration": 2.8655970096588135 + }, + { + "auxiliary_loss_clip": 0.01124326, + "auxiliary_loss_mlp": 0.01035031, + "balance_loss_clip": 1.04044175, + "balance_loss_mlp": 1.01970053, + "epoch": 0.7451451932904467, + "flos": 19606687752960.0, + "grad_norm": 2.175126446513762, + "language_loss": 0.7161181, + "learning_rate": 6.433569252661049e-07, + "loss": 0.73771167, + "num_input_tokens_seen": 133255590, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.15332031, + "step": 6197, + "time_per_iteration": 3.624204158782959 + }, + { + "auxiliary_loss_clip": 0.01122899, + "auxiliary_loss_mlp": 0.01039469, + "balance_loss_clip": 1.04042518, + "balance_loss_mlp": 1.02559304, + "epoch": 0.7452654361810858, + "flos": 12495405980160.0, + "grad_norm": 1.7390039740726864, + "language_loss": 0.71220809, + "learning_rate": 6.427846669269952e-07, + "loss": 0.73383176, + "num_input_tokens_seen": 133273210, + "router_z_loss_clip": 0.82421875, + "router_z_loss_mlp": 0.13891602, + "step": 6198, + "time_per_iteration": 3.6166248321533203 + }, + { + "auxiliary_loss_clip": 0.0112822, + "auxiliary_loss_mlp": 0.01037551, + "balance_loss_clip": 1.04333746, + "balance_loss_mlp": 1.0227747, + "epoch": 0.7453856790717249, + "flos": 22127329687680.0, + "grad_norm": 2.0315171844448927, + "language_loss": 0.8270371, + "learning_rate": 6.422126144763729e-07, + "loss": 0.84869486, + "num_input_tokens_seen": 133292600, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.14752197, + "step": 6199, + "time_per_iteration": 2.8457093238830566 + }, + { + "auxiliary_loss_clip": 0.0112664, + "auxiliary_loss_mlp": 0.01041009, + "balance_loss_clip": 1.04038966, + "balance_loss_mlp": 1.02502275, + "epoch": 0.745505921962364, + "flos": 20010682995840.0, + "grad_norm": 2.033746659406455, + "language_loss": 0.76575744, + "learning_rate": 6.416407680010174e-07, + "loss": 0.78743392, + "num_input_tokens_seen": 133306960, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15979004, + "step": 6200, + "time_per_iteration": 2.8524186611175537 + }, + { + "auxiliary_loss_clip": 0.011284, + "auxiliary_loss_mlp": 0.01039509, + "balance_loss_clip": 1.04152524, + "balance_loss_mlp": 1.02450061, + "epoch": 0.745626164853003, + "flos": 24677884673280.0, + "grad_norm": 1.9005843180490722, + "language_loss": 0.81164235, + "learning_rate": 6.410691275876774e-07, + "loss": 0.83332145, + "num_input_tokens_seen": 133326380, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.15014648, + "step": 6201, + "time_per_iteration": 2.975825786590576 + }, + { + "auxiliary_loss_clip": 0.01124956, + "auxiliary_loss_mlp": 0.01033928, + "balance_loss_clip": 1.04049063, + "balance_loss_mlp": 1.01910424, + "epoch": 0.7457464077436422, + "flos": 14538830797440.0, + "grad_norm": 2.146685054368139, + "language_loss": 0.76576746, + "learning_rate": 6.404976933230704e-07, + "loss": 0.78735632, + "num_input_tokens_seen": 133342900, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.14825439, + "step": 6202, + "time_per_iteration": 2.8615260124206543 + }, + { + "auxiliary_loss_clip": 0.01132008, + "auxiliary_loss_mlp": 0.0103738, + "balance_loss_clip": 1.04319, + "balance_loss_mlp": 1.0204041, + "epoch": 0.7458666506342813, + "flos": 34021194600960.0, + "grad_norm": 1.8183287037976086, + "language_loss": 0.72355318, + "learning_rate": 6.399264652938813e-07, + "loss": 0.74524707, + "num_input_tokens_seen": 133363805, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.16955566, + "step": 6203, + "time_per_iteration": 2.904452323913574 + }, + { + "auxiliary_loss_clip": 0.01123578, + "auxiliary_loss_mlp": 0.01031425, + "balance_loss_clip": 1.03980255, + "balance_loss_mlp": 1.01620221, + "epoch": 0.7459868935249203, + "flos": 24279025075200.0, + "grad_norm": 1.821220597510871, + "language_loss": 0.74633133, + "learning_rate": 6.393554435867679e-07, + "loss": 0.76788139, + "num_input_tokens_seen": 133384655, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.15228271, + "step": 6204, + "time_per_iteration": 3.7086174488067627 + }, + { + "auxiliary_loss_clip": 0.01123682, + "auxiliary_loss_mlp": 0.01036036, + "balance_loss_clip": 1.03965843, + "balance_loss_mlp": 1.02174318, + "epoch": 0.7461071364155595, + "flos": 21908777385600.0, + "grad_norm": 2.0733778323506216, + "language_loss": 0.83675081, + "learning_rate": 6.387846282883502e-07, + "loss": 0.85834801, + "num_input_tokens_seen": 133401185, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14294434, + "step": 6205, + "time_per_iteration": 2.7800557613372803 + }, + { + "auxiliary_loss_clip": 0.01126187, + "auxiliary_loss_mlp": 0.01035403, + "balance_loss_clip": 1.04004812, + "balance_loss_mlp": 1.02065086, + "epoch": 0.7462273793061985, + "flos": 22889712879360.0, + "grad_norm": 1.8934316036048036, + "language_loss": 0.77229607, + "learning_rate": 6.38214019485223e-07, + "loss": 0.79391199, + "num_input_tokens_seen": 133420010, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.14758301, + "step": 6206, + "time_per_iteration": 2.8573172092437744 + }, + { + "auxiliary_loss_clip": 0.01124446, + "auxiliary_loss_mlp": 0.01033351, + "balance_loss_clip": 1.03937149, + "balance_loss_mlp": 1.01833081, + "epoch": 0.7463476221968376, + "flos": 19968451580160.0, + "grad_norm": 1.6764712575094, + "language_loss": 0.71479762, + "learning_rate": 6.376436172639461e-07, + "loss": 0.73637557, + "num_input_tokens_seen": 133437855, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.15002441, + "step": 6207, + "time_per_iteration": 2.8224129676818848 + }, + { + "auxiliary_loss_clip": 0.01129082, + "auxiliary_loss_mlp": 0.01043719, + "balance_loss_clip": 1.04116845, + "balance_loss_mlp": 1.02752995, + "epoch": 0.7464678650874768, + "flos": 16836610798080.0, + "grad_norm": 3.712288279436746, + "language_loss": 0.65115869, + "learning_rate": 6.370734217110487e-07, + "loss": 0.67288673, + "num_input_tokens_seen": 133456600, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.16174316, + "step": 6208, + "time_per_iteration": 2.9974045753479004 + }, + { + "auxiliary_loss_clip": 0.01132992, + "auxiliary_loss_mlp": 0.01043154, + "balance_loss_clip": 1.04263675, + "balance_loss_mlp": 1.02630329, + "epoch": 0.7465881079781158, + "flos": 48100869843840.0, + "grad_norm": 1.5486188794674147, + "language_loss": 0.64365661, + "learning_rate": 6.36503432913031e-07, + "loss": 0.66541815, + "num_input_tokens_seen": 133479745, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.16851807, + "step": 6209, + "time_per_iteration": 3.1242003440856934 + }, + { + "auxiliary_loss_clip": 0.01129233, + "auxiliary_loss_mlp": 0.01036241, + "balance_loss_clip": 1.04036582, + "balance_loss_mlp": 1.02052951, + "epoch": 0.7467083508687549, + "flos": 19677359761920.0, + "grad_norm": 22.58334435950203, + "language_loss": 0.69047797, + "learning_rate": 6.359336509563569e-07, + "loss": 0.71213269, + "num_input_tokens_seen": 133495765, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.15710449, + "step": 6210, + "time_per_iteration": 2.789799213409424 + }, + { + "auxiliary_loss_clip": 0.01124076, + "auxiliary_loss_mlp": 0.0103262, + "balance_loss_clip": 1.03861129, + "balance_loss_mlp": 1.01684904, + "epoch": 0.7468285937593939, + "flos": 17895436934400.0, + "grad_norm": 1.858826768734816, + "language_loss": 0.80781317, + "learning_rate": 6.353640759274641e-07, + "loss": 0.82938015, + "num_input_tokens_seen": 133514655, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15771484, + "step": 6211, + "time_per_iteration": 2.9967753887176514 + }, + { + "auxiliary_loss_clip": 0.0112269, + "auxiliary_loss_mlp": 0.01035713, + "balance_loss_clip": 1.03681898, + "balance_loss_mlp": 1.02109766, + "epoch": 0.7469488366500331, + "flos": 23141446369920.0, + "grad_norm": 2.3528649917158986, + "language_loss": 0.75393629, + "learning_rate": 6.347947079127556e-07, + "loss": 0.77552032, + "num_input_tokens_seen": 133532555, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.14599609, + "step": 6212, + "time_per_iteration": 2.868793487548828 + }, + { + "auxiliary_loss_clip": 0.01126258, + "auxiliary_loss_mlp": 0.01037129, + "balance_loss_clip": 1.04106843, + "balance_loss_mlp": 1.02245402, + "epoch": 0.7470690795406721, + "flos": 16690849407360.0, + "grad_norm": 2.0291201352472346, + "language_loss": 0.76775861, + "learning_rate": 6.342255469986053e-07, + "loss": 0.78939247, + "num_input_tokens_seen": 133551300, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.14672852, + "step": 6213, + "time_per_iteration": 2.8414618968963623 + }, + { + "auxiliary_loss_clip": 0.0112543, + "auxiliary_loss_mlp": 0.01034915, + "balance_loss_clip": 1.04068828, + "balance_loss_mlp": 1.01925111, + "epoch": 0.7471893224313112, + "flos": 25192700352000.0, + "grad_norm": 1.7618496752440456, + "language_loss": 0.76232243, + "learning_rate": 6.336565932713533e-07, + "loss": 0.78392589, + "num_input_tokens_seen": 133570725, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.15649414, + "step": 6214, + "time_per_iteration": 2.9915659427642822 + }, + { + "auxiliary_loss_clip": 0.01126057, + "auxiliary_loss_mlp": 0.01040467, + "balance_loss_clip": 1.03886366, + "balance_loss_mlp": 1.02525592, + "epoch": 0.7473095653219504, + "flos": 22526225199360.0, + "grad_norm": 1.7067020029494844, + "language_loss": 0.77951515, + "learning_rate": 6.330878468173088e-07, + "loss": 0.80118036, + "num_input_tokens_seen": 133590790, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.15209961, + "step": 6215, + "time_per_iteration": 3.0514070987701416 + }, + { + "auxiliary_loss_clip": 0.01125873, + "auxiliary_loss_mlp": 0.01034404, + "balance_loss_clip": 1.04058337, + "balance_loss_mlp": 1.01950264, + "epoch": 0.7474298082125894, + "flos": 18113989236480.0, + "grad_norm": 1.6110040000682175, + "language_loss": 0.72903848, + "learning_rate": 6.32519307722752e-07, + "loss": 0.75064129, + "num_input_tokens_seen": 133608685, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.14916992, + "step": 6216, + "time_per_iteration": 2.9329655170440674 + }, + { + "auxiliary_loss_clip": 0.01034683, + "auxiliary_loss_mlp": 0.00999851, + "balance_loss_clip": 1.00686109, + "balance_loss_mlp": 0.99854594, + "epoch": 0.7475500511032285, + "flos": 62086535193600.0, + "grad_norm": 0.8680332155014128, + "language_loss": 0.54984748, + "learning_rate": 6.31950976073929e-07, + "loss": 0.57019281, + "num_input_tokens_seen": 133662775, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01306152, + "step": 6217, + "time_per_iteration": 3.270677328109741 + }, + { + "auxiliary_loss_clip": 0.0112607, + "auxiliary_loss_mlp": 0.01038728, + "balance_loss_clip": 1.03943467, + "balance_loss_mlp": 1.02325523, + "epoch": 0.7476702939938676, + "flos": 17785586165760.0, + "grad_norm": 2.2937586363730764, + "language_loss": 0.81034642, + "learning_rate": 6.31382851957055e-07, + "loss": 0.83199441, + "num_input_tokens_seen": 133679595, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15466309, + "step": 6218, + "time_per_iteration": 2.799670696258545 + }, + { + "auxiliary_loss_clip": 0.01124241, + "auxiliary_loss_mlp": 0.01039835, + "balance_loss_clip": 1.04069567, + "balance_loss_mlp": 1.02455878, + "epoch": 0.7477905368845067, + "flos": 27927944092800.0, + "grad_norm": 1.8335715297206683, + "language_loss": 0.71464807, + "learning_rate": 6.308149354583143e-07, + "loss": 0.73628891, + "num_input_tokens_seen": 133699000, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.152771, + "step": 6219, + "time_per_iteration": 2.7852139472961426 + }, + { + "auxiliary_loss_clip": 0.01127966, + "auxiliary_loss_mlp": 0.0103691, + "balance_loss_clip": 1.04102468, + "balance_loss_mlp": 1.01971996, + "epoch": 0.7479107797751458, + "flos": 26870374932480.0, + "grad_norm": 1.9915731989364212, + "language_loss": 0.81672156, + "learning_rate": 6.302472266638586e-07, + "loss": 0.83837032, + "num_input_tokens_seen": 133719540, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.171875, + "step": 6220, + "time_per_iteration": 2.8775932788848877 + }, + { + "auxiliary_loss_clip": 0.01133782, + "auxiliary_loss_mlp": 0.01043834, + "balance_loss_clip": 1.04338837, + "balance_loss_mlp": 1.02641726, + "epoch": 0.7480310226657849, + "flos": 33943375785600.0, + "grad_norm": 2.1599943988029895, + "language_loss": 0.70372128, + "learning_rate": 6.296797256598101e-07, + "loss": 0.72549742, + "num_input_tokens_seen": 133741020, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.17419434, + "step": 6221, + "time_per_iteration": 2.982499837875366 + }, + { + "auxiliary_loss_clip": 0.01124793, + "auxiliary_loss_mlp": 0.01037106, + "balance_loss_clip": 1.04146457, + "balance_loss_mlp": 1.02240121, + "epoch": 0.748151265556424, + "flos": 24826555065600.0, + "grad_norm": 1.7380728134311891, + "language_loss": 0.81460613, + "learning_rate": 6.291124325322576e-07, + "loss": 0.83622515, + "num_input_tokens_seen": 133761145, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.14709473, + "step": 6222, + "time_per_iteration": 2.8304152488708496 + }, + { + "auxiliary_loss_clip": 0.01124176, + "auxiliary_loss_mlp": 0.01037083, + "balance_loss_clip": 1.04080355, + "balance_loss_mlp": 1.02231264, + "epoch": 0.748271508447063, + "flos": 38399351535360.0, + "grad_norm": 1.5367371544381074, + "language_loss": 0.62768632, + "learning_rate": 6.285453473672595e-07, + "loss": 0.64929891, + "num_input_tokens_seen": 133783715, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.14758301, + "step": 6223, + "time_per_iteration": 3.655604124069214 + }, + { + "auxiliary_loss_clip": 0.01125221, + "auxiliary_loss_mlp": 0.0103662, + "balance_loss_clip": 1.04091406, + "balance_loss_mlp": 1.02211261, + "epoch": 0.7483917513377022, + "flos": 21541842000000.0, + "grad_norm": 2.0051383268953415, + "language_loss": 0.75595975, + "learning_rate": 6.279784702508415e-07, + "loss": 0.77757812, + "num_input_tokens_seen": 133804465, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.1451416, + "step": 6224, + "time_per_iteration": 2.8195440769195557 + }, + { + "auxiliary_loss_clip": 0.01034921, + "auxiliary_loss_mlp": 0.00998895, + "balance_loss_clip": 1.00712407, + "balance_loss_mlp": 0.99750596, + "epoch": 0.7485119942283412, + "flos": 62314532772480.0, + "grad_norm": 0.788997960309904, + "language_loss": 0.58584315, + "learning_rate": 6.274118012689979e-07, + "loss": 0.60618132, + "num_input_tokens_seen": 133866365, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01391602, + "step": 6225, + "time_per_iteration": 3.3644113540649414 + }, + { + "auxiliary_loss_clip": 0.01123231, + "auxiliary_loss_mlp": 0.01035269, + "balance_loss_clip": 1.03889155, + "balance_loss_mlp": 1.02010524, + "epoch": 0.7486322371189803, + "flos": 29937613104000.0, + "grad_norm": 1.764344840889579, + "language_loss": 0.68381256, + "learning_rate": 6.268453405076943e-07, + "loss": 0.70539761, + "num_input_tokens_seen": 133888760, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.1517334, + "step": 6226, + "time_per_iteration": 3.5564398765563965 + }, + { + "auxiliary_loss_clip": 0.01123259, + "auxiliary_loss_mlp": 0.0103975, + "balance_loss_clip": 1.0391283, + "balance_loss_mlp": 1.02456307, + "epoch": 0.7487524800096195, + "flos": 18949414734720.0, + "grad_norm": 2.0701323477735114, + "language_loss": 0.82334256, + "learning_rate": 6.262790880528592e-07, + "loss": 0.84497267, + "num_input_tokens_seen": 133906380, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.15185547, + "step": 6227, + "time_per_iteration": 3.5729098320007324 + }, + { + "auxiliary_loss_clip": 0.01129718, + "auxiliary_loss_mlp": 0.01041466, + "balance_loss_clip": 1.04161298, + "balance_loss_mlp": 1.02477694, + "epoch": 0.7488727229002585, + "flos": 18697393935360.0, + "grad_norm": 2.712718720510977, + "language_loss": 0.79569244, + "learning_rate": 6.257130439903951e-07, + "loss": 0.81740427, + "num_input_tokens_seen": 133922875, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.16699219, + "step": 6228, + "time_per_iteration": 2.9357659816741943 + }, + { + "auxiliary_loss_clip": 0.01128617, + "auxiliary_loss_mlp": 0.0104435, + "balance_loss_clip": 1.04272079, + "balance_loss_mlp": 1.0285666, + "epoch": 0.7489929657908976, + "flos": 23623368168960.0, + "grad_norm": 4.496834284546773, + "language_loss": 0.81056404, + "learning_rate": 6.251472084061695e-07, + "loss": 0.83229375, + "num_input_tokens_seen": 133941795, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15783691, + "step": 6229, + "time_per_iteration": 2.809292793273926 + }, + { + "auxiliary_loss_clip": 0.01125273, + "auxiliary_loss_mlp": 0.01037594, + "balance_loss_clip": 1.03999484, + "balance_loss_mlp": 1.02306271, + "epoch": 0.7491132086815367, + "flos": 20551533056640.0, + "grad_norm": 1.961072854005142, + "language_loss": 0.88981545, + "learning_rate": 6.245815813860191e-07, + "loss": 0.91144419, + "num_input_tokens_seen": 133957305, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.14532471, + "step": 6230, + "time_per_iteration": 2.868697166442871 + }, + { + "auxiliary_loss_clip": 0.01130808, + "auxiliary_loss_mlp": 0.01036807, + "balance_loss_clip": 1.04037881, + "balance_loss_mlp": 1.0205946, + "epoch": 0.7492334515721758, + "flos": 23003011353600.0, + "grad_norm": 1.823126212913006, + "language_loss": 0.70018607, + "learning_rate": 6.240161630157495e-07, + "loss": 0.7218622, + "num_input_tokens_seen": 133976660, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.1619873, + "step": 6231, + "time_per_iteration": 2.9320273399353027 + }, + { + "auxiliary_loss_clip": 0.01128478, + "auxiliary_loss_mlp": 0.01038372, + "balance_loss_clip": 1.03976011, + "balance_loss_mlp": 1.02194512, + "epoch": 0.7493536944628149, + "flos": 16398823835520.0, + "grad_norm": 1.9589709939046058, + "language_loss": 0.69969845, + "learning_rate": 6.23450953381133e-07, + "loss": 0.72136694, + "num_input_tokens_seen": 133994750, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.16442871, + "step": 6232, + "time_per_iteration": 2.9685580730438232 + }, + { + "auxiliary_loss_clip": 0.01123013, + "auxiliary_loss_mlp": 0.01039809, + "balance_loss_clip": 1.03874302, + "balance_loss_mlp": 1.02516437, + "epoch": 0.749473937353454, + "flos": 15338561155200.0, + "grad_norm": 1.9572166505507467, + "language_loss": 0.68177038, + "learning_rate": 6.228859525679131e-07, + "loss": 0.70339859, + "num_input_tokens_seen": 134009165, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.14654541, + "step": 6233, + "time_per_iteration": 4.056352853775024 + }, + { + "auxiliary_loss_clip": 0.01124439, + "auxiliary_loss_mlp": 0.01037212, + "balance_loss_clip": 1.04029059, + "balance_loss_mlp": 1.02332962, + "epoch": 0.7495941802440931, + "flos": 18951138587520.0, + "grad_norm": 2.9429131832573003, + "language_loss": 0.79589319, + "learning_rate": 6.223211606617986e-07, + "loss": 0.81750965, + "num_input_tokens_seen": 134027585, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.13873291, + "step": 6234, + "time_per_iteration": 3.011748790740967 + }, + { + "auxiliary_loss_clip": 0.0112177, + "auxiliary_loss_mlp": 0.0102872, + "balance_loss_clip": 1.04299402, + "balance_loss_mlp": 1.01505303, + "epoch": 0.7497144231347321, + "flos": 22492469393280.0, + "grad_norm": 1.8680272418839912, + "language_loss": 0.84000671, + "learning_rate": 6.217565777484701e-07, + "loss": 0.86151159, + "num_input_tokens_seen": 134046680, + "router_z_loss_clip": 0.78808594, + "router_z_loss_mlp": 0.13677979, + "step": 6235, + "time_per_iteration": 3.021854877471924 + }, + { + "auxiliary_loss_clip": 0.01120029, + "auxiliary_loss_mlp": 0.01039025, + "balance_loss_clip": 1.03718853, + "balance_loss_mlp": 1.02494025, + "epoch": 0.7498346660253713, + "flos": 24243509502720.0, + "grad_norm": 1.928924894490424, + "language_loss": 0.80603838, + "learning_rate": 6.211922039135722e-07, + "loss": 0.82762891, + "num_input_tokens_seen": 134066825, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.14086914, + "step": 6236, + "time_per_iteration": 3.019259452819824 + }, + { + "auxiliary_loss_clip": 0.01125724, + "auxiliary_loss_mlp": 0.01037729, + "balance_loss_clip": 1.04034901, + "balance_loss_mlp": 1.02291167, + "epoch": 0.7499549089160104, + "flos": 24387080163840.0, + "grad_norm": 1.789097991333567, + "language_loss": 0.80983406, + "learning_rate": 6.206280392427201e-07, + "loss": 0.83146858, + "num_input_tokens_seen": 134086410, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.14813232, + "step": 6237, + "time_per_iteration": 3.012489080429077 + }, + { + "auxiliary_loss_clip": 0.0112182, + "auxiliary_loss_mlp": 0.01033625, + "balance_loss_clip": 1.03850675, + "balance_loss_mlp": 1.01943362, + "epoch": 0.7500751518066494, + "flos": 34057320704640.0, + "grad_norm": 2.007125910092058, + "language_loss": 0.73700571, + "learning_rate": 6.200640838214983e-07, + "loss": 0.75856018, + "num_input_tokens_seen": 134109185, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.14190674, + "step": 6238, + "time_per_iteration": 3.0600948333740234 + }, + { + "auxiliary_loss_clip": 0.01124619, + "auxiliary_loss_mlp": 0.01038627, + "balance_loss_clip": 1.03997803, + "balance_loss_mlp": 1.02411306, + "epoch": 0.7501953946972886, + "flos": 18843586289280.0, + "grad_norm": 2.00630883716818, + "language_loss": 0.66812462, + "learning_rate": 6.195003377354578e-07, + "loss": 0.68975705, + "num_input_tokens_seen": 134128455, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.14526367, + "step": 6239, + "time_per_iteration": 2.9420573711395264 + }, + { + "auxiliary_loss_clip": 0.01128238, + "auxiliary_loss_mlp": 0.01039323, + "balance_loss_clip": 1.04238892, + "balance_loss_mlp": 1.02414727, + "epoch": 0.7503156375879276, + "flos": 20257675891200.0, + "grad_norm": 4.829329303306663, + "language_loss": 0.72986799, + "learning_rate": 6.189368010701183e-07, + "loss": 0.75154364, + "num_input_tokens_seen": 134145515, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15161133, + "step": 6240, + "time_per_iteration": 2.9992706775665283 + }, + { + "auxiliary_loss_clip": 0.01127922, + "auxiliary_loss_mlp": 0.01036441, + "balance_loss_clip": 1.03974128, + "balance_loss_mlp": 1.02092552, + "epoch": 0.7504358804785667, + "flos": 13480040574720.0, + "grad_norm": 2.0737835437519, + "language_loss": 0.7632159, + "learning_rate": 6.183734739109683e-07, + "loss": 0.78485954, + "num_input_tokens_seen": 134163335, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.15509033, + "step": 6241, + "time_per_iteration": 3.0321125984191895 + }, + { + "auxiliary_loss_clip": 0.01133247, + "auxiliary_loss_mlp": 0.01042097, + "balance_loss_clip": 1.04307127, + "balance_loss_mlp": 1.02574205, + "epoch": 0.7505561233692057, + "flos": 29461042431360.0, + "grad_norm": 2.134874199748689, + "language_loss": 0.68415034, + "learning_rate": 6.178103563434629e-07, + "loss": 0.70590383, + "num_input_tokens_seen": 134182335, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.16357422, + "step": 6242, + "time_per_iteration": 3.068687677383423 + }, + { + "auxiliary_loss_clip": 0.0112708, + "auxiliary_loss_mlp": 0.01039793, + "balance_loss_clip": 1.04062605, + "balance_loss_mlp": 1.02380705, + "epoch": 0.7506763662598449, + "flos": 20302457172480.0, + "grad_norm": 1.776986811350903, + "language_loss": 0.83758104, + "learning_rate": 6.172474484530283e-07, + "loss": 0.85924977, + "num_input_tokens_seen": 134201070, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.15979004, + "step": 6243, + "time_per_iteration": 2.9887800216674805 + }, + { + "auxiliary_loss_clip": 0.01125453, + "auxiliary_loss_mlp": 0.01036241, + "balance_loss_clip": 1.03921747, + "balance_loss_mlp": 1.0209229, + "epoch": 0.750796609150484, + "flos": 37230961939200.0, + "grad_norm": 2.0836423251531175, + "language_loss": 0.7613641, + "learning_rate": 6.166847503250563e-07, + "loss": 0.78298104, + "num_input_tokens_seen": 134223310, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.1529541, + "step": 6244, + "time_per_iteration": 3.131631374359131 + }, + { + "auxiliary_loss_clip": 0.01123617, + "auxiliary_loss_mlp": 0.01036599, + "balance_loss_clip": 1.03938651, + "balance_loss_mlp": 1.0209707, + "epoch": 0.750916852041123, + "flos": 19609417186560.0, + "grad_norm": 2.2053562607101913, + "language_loss": 0.78717399, + "learning_rate": 6.161222620449078e-07, + "loss": 0.80877614, + "num_input_tokens_seen": 134242085, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.15637207, + "step": 6245, + "time_per_iteration": 2.960419178009033 + }, + { + "auxiliary_loss_clip": 0.01125384, + "auxiliary_loss_mlp": 0.01034502, + "balance_loss_clip": 1.0397954, + "balance_loss_mlp": 1.01918948, + "epoch": 0.7510370949317622, + "flos": 25112690807040.0, + "grad_norm": 2.318813165779833, + "language_loss": 0.80454195, + "learning_rate": 6.155599836979117e-07, + "loss": 0.82614088, + "num_input_tokens_seen": 134260770, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15313721, + "step": 6246, + "time_per_iteration": 2.9257779121398926 + }, + { + "auxiliary_loss_clip": 0.01128631, + "auxiliary_loss_mlp": 0.01047332, + "balance_loss_clip": 1.03966379, + "balance_loss_mlp": 1.0298264, + "epoch": 0.7511573378224012, + "flos": 19062282245760.0, + "grad_norm": 2.0458781078931185, + "language_loss": 0.81645036, + "learning_rate": 6.149979153693649e-07, + "loss": 0.83821005, + "num_input_tokens_seen": 134278025, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.17510986, + "step": 6247, + "time_per_iteration": 2.8824572563171387 + }, + { + "auxiliary_loss_clip": 0.0112872, + "auxiliary_loss_mlp": 0.01045318, + "balance_loss_clip": 1.04102254, + "balance_loss_mlp": 1.02976131, + "epoch": 0.7512775807130403, + "flos": 19937676602880.0, + "grad_norm": 2.0998954140859807, + "language_loss": 0.76888508, + "learning_rate": 6.144360571445343e-07, + "loss": 0.79062545, + "num_input_tokens_seen": 134297170, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.15563965, + "step": 6248, + "time_per_iteration": 2.9294686317443848 + }, + { + "auxiliary_loss_clip": 0.01124563, + "auxiliary_loss_mlp": 0.01038706, + "balance_loss_clip": 1.03981054, + "balance_loss_mlp": 1.02365005, + "epoch": 0.7513978236036795, + "flos": 20739920912640.0, + "grad_norm": 1.7374527755422697, + "language_loss": 0.80062336, + "learning_rate": 6.138744091086509e-07, + "loss": 0.82225609, + "num_input_tokens_seen": 134316755, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15063477, + "step": 6249, + "time_per_iteration": 2.9345827102661133 + }, + { + "auxiliary_loss_clip": 0.01128956, + "auxiliary_loss_mlp": 0.01040658, + "balance_loss_clip": 1.04245591, + "balance_loss_mlp": 1.0255065, + "epoch": 0.7515180664943185, + "flos": 27563163523200.0, + "grad_norm": 2.0486710797675007, + "language_loss": 0.72692961, + "learning_rate": 6.133129713469183e-07, + "loss": 0.74862576, + "num_input_tokens_seen": 134335960, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15148926, + "step": 6250, + "time_per_iteration": 2.8981778621673584 + }, + { + "auxiliary_loss_clip": 0.0112953, + "auxiliary_loss_mlp": 0.01039989, + "balance_loss_clip": 1.04116774, + "balance_loss_mlp": 1.0246948, + "epoch": 0.7516383093849576, + "flos": 33803181002880.0, + "grad_norm": 1.715879199147698, + "language_loss": 0.63886058, + "learning_rate": 6.127517439445053e-07, + "loss": 0.66055572, + "num_input_tokens_seen": 134356805, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.1529541, + "step": 6251, + "time_per_iteration": 3.6832120418548584 + }, + { + "auxiliary_loss_clip": 0.01121759, + "auxiliary_loss_mlp": 0.01039944, + "balance_loss_clip": 1.04063916, + "balance_loss_mlp": 1.02637768, + "epoch": 0.7517585522755967, + "flos": 29746172592000.0, + "grad_norm": 1.9945151046989198, + "language_loss": 0.82011527, + "learning_rate": 6.121907269865498e-07, + "loss": 0.84173226, + "num_input_tokens_seen": 134376295, + "router_z_loss_clip": 0.81103516, + "router_z_loss_mlp": 0.13568115, + "step": 6252, + "time_per_iteration": 3.024310350418091 + }, + { + "auxiliary_loss_clip": 0.01034364, + "auxiliary_loss_mlp": 0.01002921, + "balance_loss_clip": 1.00659442, + "balance_loss_mlp": 1.00141275, + "epoch": 0.7518787951662358, + "flos": 69807974319360.0, + "grad_norm": 0.9240857387438987, + "language_loss": 0.67204714, + "learning_rate": 6.116299205581577e-07, + "loss": 0.69242001, + "num_input_tokens_seen": 134431125, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01507568, + "step": 6253, + "time_per_iteration": 3.409914016723633 + }, + { + "auxiliary_loss_clip": 0.01133608, + "auxiliary_loss_mlp": 0.01039857, + "balance_loss_clip": 1.04222083, + "balance_loss_mlp": 1.02295351, + "epoch": 0.7519990380568748, + "flos": 34203225749760.0, + "grad_norm": 2.1665357171762247, + "language_loss": 0.68377471, + "learning_rate": 6.110693247444018e-07, + "loss": 0.70550942, + "num_input_tokens_seen": 134452960, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.16882324, + "step": 6254, + "time_per_iteration": 3.046931028366089 + }, + { + "auxiliary_loss_clip": 0.01125823, + "auxiliary_loss_mlp": 0.01037924, + "balance_loss_clip": 1.04173231, + "balance_loss_mlp": 1.02311838, + "epoch": 0.752119280947514, + "flos": 21725704742400.0, + "grad_norm": 1.990081399915252, + "language_loss": 0.82467502, + "learning_rate": 6.105089396303258e-07, + "loss": 0.84631246, + "num_input_tokens_seen": 134471350, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.14788818, + "step": 6255, + "time_per_iteration": 3.9115045070648193 + }, + { + "auxiliary_loss_clip": 0.01128265, + "auxiliary_loss_mlp": 0.01038556, + "balance_loss_clip": 1.0405854, + "balance_loss_mlp": 1.02232003, + "epoch": 0.7522395238381531, + "flos": 32742774668160.0, + "grad_norm": 1.8039626383045901, + "language_loss": 0.75973892, + "learning_rate": 6.099487653009383e-07, + "loss": 0.78140712, + "num_input_tokens_seen": 134490695, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16223145, + "step": 6256, + "time_per_iteration": 3.8464694023132324 + }, + { + "auxiliary_loss_clip": 0.01123276, + "auxiliary_loss_mlp": 0.01035577, + "balance_loss_clip": 1.04092371, + "balance_loss_mlp": 1.02247, + "epoch": 0.7523597667287921, + "flos": 23476026579840.0, + "grad_norm": 2.144736323175705, + "language_loss": 0.82883537, + "learning_rate": 6.093888018412192e-07, + "loss": 0.85042393, + "num_input_tokens_seen": 134506885, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.13110352, + "step": 6257, + "time_per_iteration": 3.1659417152404785 + }, + { + "auxiliary_loss_clip": 0.01033903, + "auxiliary_loss_mlp": 0.01002962, + "balance_loss_clip": 1.00601459, + "balance_loss_mlp": 1.00138807, + "epoch": 0.7524800096194313, + "flos": 67346730501120.0, + "grad_norm": 0.711732638447878, + "language_loss": 0.54662102, + "learning_rate": 6.088290493361125e-07, + "loss": 0.56698966, + "num_input_tokens_seen": 134571770, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01574707, + "step": 6258, + "time_per_iteration": 3.578463315963745 + }, + { + "auxiliary_loss_clip": 0.01127616, + "auxiliary_loss_mlp": 0.0103535, + "balance_loss_clip": 1.0402838, + "balance_loss_mlp": 1.02013934, + "epoch": 0.7526002525100703, + "flos": 13006055681280.0, + "grad_norm": 2.0907051548926776, + "language_loss": 0.71504343, + "learning_rate": 6.082695078705322e-07, + "loss": 0.73667312, + "num_input_tokens_seen": 134589250, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15203857, + "step": 6259, + "time_per_iteration": 2.9967479705810547 + }, + { + "auxiliary_loss_clip": 0.01127994, + "auxiliary_loss_mlp": 0.01039784, + "balance_loss_clip": 1.04065514, + "balance_loss_mlp": 1.02415586, + "epoch": 0.7527204954007094, + "flos": 21397229844480.0, + "grad_norm": 2.4388782957099266, + "language_loss": 0.68642545, + "learning_rate": 6.077101775293618e-07, + "loss": 0.70810318, + "num_input_tokens_seen": 134608075, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.15625, + "step": 6260, + "time_per_iteration": 2.996659517288208 + }, + { + "auxiliary_loss_clip": 0.01128429, + "auxiliary_loss_mlp": 0.0103809, + "balance_loss_clip": 1.04036617, + "balance_loss_mlp": 1.02097118, + "epoch": 0.7528407382913486, + "flos": 18947188091520.0, + "grad_norm": 3.8533401633085504, + "language_loss": 0.82943803, + "learning_rate": 6.071510583974504e-07, + "loss": 0.85110319, + "num_input_tokens_seen": 134623260, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.17126465, + "step": 6261, + "time_per_iteration": 3.801739454269409 + }, + { + "auxiliary_loss_clip": 0.01126897, + "auxiliary_loss_mlp": 0.01041273, + "balance_loss_clip": 1.03984976, + "balance_loss_mlp": 1.02636623, + "epoch": 0.7529609811819876, + "flos": 15231798956160.0, + "grad_norm": 1.9473597001114227, + "language_loss": 0.72018164, + "learning_rate": 6.065921505596161e-07, + "loss": 0.74186337, + "num_input_tokens_seen": 134641540, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.14898682, + "step": 6262, + "time_per_iteration": 2.972926378250122 + }, + { + "auxiliary_loss_clip": 0.01127481, + "auxiliary_loss_mlp": 0.01037901, + "balance_loss_clip": 1.04330468, + "balance_loss_mlp": 1.02313745, + "epoch": 0.7530812240726267, + "flos": 19354487385600.0, + "grad_norm": 1.5327987776869125, + "language_loss": 0.77028584, + "learning_rate": 6.060334541006445e-07, + "loss": 0.79193968, + "num_input_tokens_seen": 134660035, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.14746094, + "step": 6263, + "time_per_iteration": 2.8566887378692627 + }, + { + "auxiliary_loss_clip": 0.01122309, + "auxiliary_loss_mlp": 0.01037213, + "balance_loss_clip": 1.03818166, + "balance_loss_mlp": 1.02284837, + "epoch": 0.7532014669632658, + "flos": 27748247328000.0, + "grad_norm": 1.4823936870355918, + "language_loss": 0.69036138, + "learning_rate": 6.05474969105289e-07, + "loss": 0.71195656, + "num_input_tokens_seen": 134683025, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14355469, + "step": 6264, + "time_per_iteration": 2.9438681602478027 + }, + { + "auxiliary_loss_clip": 0.01130604, + "auxiliary_loss_mlp": 0.01037877, + "balance_loss_clip": 1.04164934, + "balance_loss_mlp": 1.02143776, + "epoch": 0.7533217098539049, + "flos": 14137421333760.0, + "grad_norm": 2.5780152865339048, + "language_loss": 0.73395908, + "learning_rate": 6.049166956582725e-07, + "loss": 0.75564396, + "num_input_tokens_seen": 134701290, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.16430664, + "step": 6265, + "time_per_iteration": 2.8554093837738037 + }, + { + "auxiliary_loss_clip": 0.01123449, + "auxiliary_loss_mlp": 0.01037213, + "balance_loss_clip": 1.03989887, + "balance_loss_mlp": 1.02257442, + "epoch": 0.753441952744544, + "flos": 26429068437120.0, + "grad_norm": 2.0226253022831724, + "language_loss": 0.87563187, + "learning_rate": 6.043586338442841e-07, + "loss": 0.89723849, + "num_input_tokens_seen": 134720345, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.1463623, + "step": 6266, + "time_per_iteration": 2.943117618560791 + }, + { + "auxiliary_loss_clip": 0.01122398, + "auxiliary_loss_mlp": 0.01036683, + "balance_loss_clip": 1.04172349, + "balance_loss_mlp": 1.02354026, + "epoch": 0.7535621956351831, + "flos": 23878621192320.0, + "grad_norm": 4.294385521747003, + "language_loss": 0.73167866, + "learning_rate": 6.038007837479815e-07, + "loss": 0.75326949, + "num_input_tokens_seen": 134741450, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.13128662, + "step": 6267, + "time_per_iteration": 3.0473225116729736 + }, + { + "auxiliary_loss_clip": 0.01124696, + "auxiliary_loss_mlp": 0.01031791, + "balance_loss_clip": 1.04175377, + "balance_loss_mlp": 1.01792729, + "epoch": 0.7536824385258222, + "flos": 21795873960960.0, + "grad_norm": 1.757684112263253, + "language_loss": 0.63996518, + "learning_rate": 6.032431454539897e-07, + "loss": 0.66153008, + "num_input_tokens_seen": 134760295, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.13867188, + "step": 6268, + "time_per_iteration": 3.063424825668335 + }, + { + "auxiliary_loss_clip": 0.01123779, + "auxiliary_loss_mlp": 0.01041434, + "balance_loss_clip": 1.03938222, + "balance_loss_mlp": 1.02584124, + "epoch": 0.7538026814164612, + "flos": 28911644933760.0, + "grad_norm": 1.8058161434317124, + "language_loss": 0.81782174, + "learning_rate": 6.026857190469014e-07, + "loss": 0.83947396, + "num_input_tokens_seen": 134782050, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.15600586, + "step": 6269, + "time_per_iteration": 2.9186089038848877 + }, + { + "auxiliary_loss_clip": 0.01128004, + "auxiliary_loss_mlp": 0.01042627, + "balance_loss_clip": 1.04152858, + "balance_loss_mlp": 1.02654576, + "epoch": 0.7539229243071004, + "flos": 21104701482240.0, + "grad_norm": 2.4787545051112163, + "language_loss": 0.73820317, + "learning_rate": 6.0212850461128e-07, + "loss": 0.75990951, + "num_input_tokens_seen": 134801170, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.16064453, + "step": 6270, + "time_per_iteration": 2.953707695007324 + }, + { + "auxiliary_loss_clip": 0.0112427, + "auxiliary_loss_mlp": 0.01036741, + "balance_loss_clip": 1.03831089, + "balance_loss_mlp": 1.02038574, + "epoch": 0.7540431671977395, + "flos": 15158469340800.0, + "grad_norm": 2.03320415985788, + "language_loss": 0.74316055, + "learning_rate": 6.015715022316516e-07, + "loss": 0.76477063, + "num_input_tokens_seen": 134819150, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.16357422, + "step": 6271, + "time_per_iteration": 2.947126626968384 + }, + { + "auxiliary_loss_clip": 0.01128978, + "auxiliary_loss_mlp": 0.01042318, + "balance_loss_clip": 1.03985834, + "balance_loss_mlp": 1.02611709, + "epoch": 0.7541634100883785, + "flos": 18770579896320.0, + "grad_norm": 2.5085039670897786, + "language_loss": 0.78097862, + "learning_rate": 6.010147119925154e-07, + "loss": 0.80269158, + "num_input_tokens_seen": 134836905, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.1619873, + "step": 6272, + "time_per_iteration": 2.91371488571167 + }, + { + "auxiliary_loss_clip": 0.01126838, + "auxiliary_loss_mlp": 0.01040757, + "balance_loss_clip": 1.04060292, + "balance_loss_mlp": 1.02485442, + "epoch": 0.7542836529790176, + "flos": 20594770053120.0, + "grad_norm": 1.934338040596507, + "language_loss": 0.66385734, + "learning_rate": 6.004581339783348e-07, + "loss": 0.68553329, + "num_input_tokens_seen": 134855225, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15893555, + "step": 6273, + "time_per_iteration": 3.0037591457366943 + }, + { + "auxiliary_loss_clip": 0.01129452, + "auxiliary_loss_mlp": 0.01044528, + "balance_loss_clip": 1.03940928, + "balance_loss_mlp": 1.02750516, + "epoch": 0.7544038958696567, + "flos": 19095104298240.0, + "grad_norm": 2.525598273578575, + "language_loss": 0.69303286, + "learning_rate": 5.999017682735425e-07, + "loss": 0.71477264, + "num_input_tokens_seen": 134871615, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.17016602, + "step": 6274, + "time_per_iteration": 3.1827399730682373 + }, + { + "auxiliary_loss_clip": 0.01125543, + "auxiliary_loss_mlp": 0.01043852, + "balance_loss_clip": 1.03970647, + "balance_loss_mlp": 1.02839088, + "epoch": 0.7545241387602958, + "flos": 31723306859520.0, + "grad_norm": 4.155165004506985, + "language_loss": 0.66521311, + "learning_rate": 5.993456149625387e-07, + "loss": 0.68690711, + "num_input_tokens_seen": 134892765, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.15454102, + "step": 6275, + "time_per_iteration": 3.1906774044036865 + }, + { + "auxiliary_loss_clip": 0.01123147, + "auxiliary_loss_mlp": 0.01035121, + "balance_loss_clip": 1.04065061, + "balance_loss_mlp": 1.02129316, + "epoch": 0.7546443816509348, + "flos": 20296495514880.0, + "grad_norm": 1.7668225369831105, + "language_loss": 0.82205546, + "learning_rate": 5.987896741296909e-07, + "loss": 0.84363818, + "num_input_tokens_seen": 134910505, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.13812256, + "step": 6276, + "time_per_iteration": 2.9638359546661377 + }, + { + "auxiliary_loss_clip": 0.01126633, + "auxiliary_loss_mlp": 0.01049323, + "balance_loss_clip": 1.04062939, + "balance_loss_mlp": 1.03399229, + "epoch": 0.754764624541574, + "flos": 23696159080320.0, + "grad_norm": 1.9580870540980386, + "language_loss": 0.78216279, + "learning_rate": 5.982339458593361e-07, + "loss": 0.80392241, + "num_input_tokens_seen": 134930445, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15332031, + "step": 6277, + "time_per_iteration": 2.830136299133301 + }, + { + "auxiliary_loss_clip": 0.01125403, + "auxiliary_loss_mlp": 0.01030578, + "balance_loss_clip": 1.04004991, + "balance_loss_mlp": 1.01623726, + "epoch": 0.7548848674322131, + "flos": 25337204766720.0, + "grad_norm": 1.590752246731304, + "language_loss": 0.84009922, + "learning_rate": 5.976784302357767e-07, + "loss": 0.86165899, + "num_input_tokens_seen": 134951010, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.14343262, + "step": 6278, + "time_per_iteration": 2.8329644203186035 + }, + { + "auxiliary_loss_clip": 0.01125239, + "auxiliary_loss_mlp": 0.01041743, + "balance_loss_clip": 1.04077864, + "balance_loss_mlp": 1.02623403, + "epoch": 0.7550051103228521, + "flos": 19573147428480.0, + "grad_norm": 1.9016752836882818, + "language_loss": 0.73394167, + "learning_rate": 5.971231273432855e-07, + "loss": 0.75561154, + "num_input_tokens_seen": 134970495, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.1550293, + "step": 6279, + "time_per_iteration": 2.8927102088928223 + }, + { + "auxiliary_loss_clip": 0.01034128, + "auxiliary_loss_mlp": 0.01000064, + "balance_loss_clip": 1.00640202, + "balance_loss_mlp": 0.99853784, + "epoch": 0.7551253532134913, + "flos": 64150068648960.0, + "grad_norm": 0.9014828033270268, + "language_loss": 0.54591507, + "learning_rate": 5.965680372661e-07, + "loss": 0.566257, + "num_input_tokens_seen": 135028060, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01525879, + "step": 6280, + "time_per_iteration": 4.037787437438965 + }, + { + "auxiliary_loss_clip": 0.0112218, + "auxiliary_loss_mlp": 0.01034537, + "balance_loss_clip": 1.04101431, + "balance_loss_mlp": 1.02128065, + "epoch": 0.7552455961041303, + "flos": 26067986968320.0, + "grad_norm": 1.7823721519024838, + "language_loss": 0.56251609, + "learning_rate": 5.960131600884266e-07, + "loss": 0.58408326, + "num_input_tokens_seen": 135047330, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.13269043, + "step": 6281, + "time_per_iteration": 2.812418222427368 + }, + { + "auxiliary_loss_clip": 0.01124096, + "auxiliary_loss_mlp": 0.01035704, + "balance_loss_clip": 1.04167986, + "balance_loss_mlp": 1.02123153, + "epoch": 0.7553658389947694, + "flos": 24498223822080.0, + "grad_norm": 1.7360196776881858, + "language_loss": 0.76009881, + "learning_rate": 5.954584958944413e-07, + "loss": 0.78169674, + "num_input_tokens_seen": 135065995, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.14465332, + "step": 6282, + "time_per_iteration": 2.8040659427642822 + }, + { + "auxiliary_loss_clip": 0.01124877, + "auxiliary_loss_mlp": 0.01037361, + "balance_loss_clip": 1.03920591, + "balance_loss_mlp": 1.02269268, + "epoch": 0.7554860818854086, + "flos": 21799465320960.0, + "grad_norm": 1.9942925153292108, + "language_loss": 0.81851804, + "learning_rate": 5.949040447682854e-07, + "loss": 0.84014046, + "num_input_tokens_seen": 135085820, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.14666748, + "step": 6283, + "time_per_iteration": 2.7747607231140137 + }, + { + "auxiliary_loss_clip": 0.01130594, + "auxiliary_loss_mlp": 0.0103994, + "balance_loss_clip": 1.04220915, + "balance_loss_mlp": 1.0232861, + "epoch": 0.7556063247760476, + "flos": 16362123114240.0, + "grad_norm": 2.0973208596012682, + "language_loss": 0.68267542, + "learning_rate": 5.943498067940686e-07, + "loss": 0.70438075, + "num_input_tokens_seen": 135102845, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.16662598, + "step": 6284, + "time_per_iteration": 4.33265233039856 + }, + { + "auxiliary_loss_clip": 0.01123133, + "auxiliary_loss_mlp": 0.01042833, + "balance_loss_clip": 1.04123807, + "balance_loss_mlp": 1.02691889, + "epoch": 0.7557265676666867, + "flos": 27235155502080.0, + "grad_norm": 1.7596846798549413, + "language_loss": 0.81384647, + "learning_rate": 5.937957820558686e-07, + "loss": 0.83550614, + "num_input_tokens_seen": 135122190, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.15905762, + "step": 6285, + "time_per_iteration": 2.809176445007324 + }, + { + "auxiliary_loss_clip": 0.01034442, + "auxiliary_loss_mlp": 0.0100224, + "balance_loss_clip": 1.00675964, + "balance_loss_mlp": 1.00064301, + "epoch": 0.7558468105573258, + "flos": 62189131415040.0, + "grad_norm": 0.9970619846141255, + "language_loss": 0.65329063, + "learning_rate": 5.932419706377296e-07, + "loss": 0.67365754, + "num_input_tokens_seen": 135180495, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01599121, + "step": 6286, + "time_per_iteration": 3.219494342803955 + }, + { + "auxiliary_loss_clip": 0.01126358, + "auxiliary_loss_mlp": 0.01044107, + "balance_loss_clip": 1.04156327, + "balance_loss_mlp": 1.02959323, + "epoch": 0.7559670534479649, + "flos": 33249078823680.0, + "grad_norm": 1.7736981972441535, + "language_loss": 0.7415911, + "learning_rate": 5.92688372623666e-07, + "loss": 0.76329565, + "num_input_tokens_seen": 135199200, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.14508057, + "step": 6287, + "time_per_iteration": 2.8267173767089844 + }, + { + "auxiliary_loss_clip": 0.01128583, + "auxiliary_loss_mlp": 0.01041121, + "balance_loss_clip": 1.04051566, + "balance_loss_mlp": 1.02496862, + "epoch": 0.7560872963386039, + "flos": 14064379027200.0, + "grad_norm": 2.0684848159044877, + "language_loss": 0.74052906, + "learning_rate": 5.921349880976574e-07, + "loss": 0.7622261, + "num_input_tokens_seen": 135217035, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.16143799, + "step": 6288, + "time_per_iteration": 2.741678476333618 + }, + { + "auxiliary_loss_clip": 0.0112861, + "auxiliary_loss_mlp": 0.01033431, + "balance_loss_clip": 1.04252768, + "balance_loss_mlp": 1.01811242, + "epoch": 0.7562075392292431, + "flos": 20412307941120.0, + "grad_norm": 2.206547842406275, + "language_loss": 0.81886804, + "learning_rate": 5.915818171436515e-07, + "loss": 0.84048843, + "num_input_tokens_seen": 135236370, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.15301514, + "step": 6289, + "time_per_iteration": 2.881133794784546 + }, + { + "auxiliary_loss_clip": 0.01125277, + "auxiliary_loss_mlp": 0.01040708, + "balance_loss_clip": 1.03913665, + "balance_loss_mlp": 1.02462709, + "epoch": 0.7563277821198822, + "flos": 20376792368640.0, + "grad_norm": 1.6554240434948218, + "language_loss": 0.74623442, + "learning_rate": 5.910288598455642e-07, + "loss": 0.76789427, + "num_input_tokens_seen": 135255720, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.1607666, + "step": 6290, + "time_per_iteration": 3.925175666809082 + }, + { + "auxiliary_loss_clip": 0.0113157, + "auxiliary_loss_mlp": 0.01044768, + "balance_loss_clip": 1.04210532, + "balance_loss_mlp": 1.02806735, + "epoch": 0.7564480250105212, + "flos": 18588261438720.0, + "grad_norm": 2.3480063528937607, + "language_loss": 0.74962211, + "learning_rate": 5.90476116287278e-07, + "loss": 0.77138555, + "num_input_tokens_seen": 135273320, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.16699219, + "step": 6291, + "time_per_iteration": 3.016775608062744 + }, + { + "auxiliary_loss_clip": 0.01125763, + "auxiliary_loss_mlp": 0.01035514, + "balance_loss_clip": 1.04019141, + "balance_loss_mlp": 1.02024341, + "epoch": 0.7565682679011604, + "flos": 21215521918080.0, + "grad_norm": 1.7970186299660826, + "language_loss": 0.67756587, + "learning_rate": 5.899235865526456e-07, + "loss": 0.6991787, + "num_input_tokens_seen": 135292615, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15270996, + "step": 6292, + "time_per_iteration": 3.015946388244629 + }, + { + "auxiliary_loss_clip": 0.01124192, + "auxiliary_loss_mlp": 0.01035016, + "balance_loss_clip": 1.04196858, + "balance_loss_mlp": 1.02155101, + "epoch": 0.7566885107917994, + "flos": 20449008662400.0, + "grad_norm": 1.6928545994382471, + "language_loss": 0.82071614, + "learning_rate": 5.893712707254825e-07, + "loss": 0.84230822, + "num_input_tokens_seen": 135310075, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.13482666, + "step": 6293, + "time_per_iteration": 2.9627490043640137 + }, + { + "auxiliary_loss_clip": 0.01127834, + "auxiliary_loss_mlp": 0.01036206, + "balance_loss_clip": 1.03918731, + "balance_loss_mlp": 1.01973128, + "epoch": 0.7568087536824385, + "flos": 19025832919680.0, + "grad_norm": 3.9919864835214223, + "language_loss": 0.6524331, + "learning_rate": 5.888191688895769e-07, + "loss": 0.67407346, + "num_input_tokens_seen": 135327335, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.16467285, + "step": 6294, + "time_per_iteration": 2.8817050457000732 + }, + { + "auxiliary_loss_clip": 0.01127436, + "auxiliary_loss_mlp": 0.01030814, + "balance_loss_clip": 1.03922653, + "balance_loss_mlp": 1.01542401, + "epoch": 0.7569289965730777, + "flos": 15225442248960.0, + "grad_norm": 2.052922233704521, + "language_loss": 0.62152779, + "learning_rate": 5.882672811286813e-07, + "loss": 0.64311028, + "num_input_tokens_seen": 135343615, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.15393066, + "step": 6295, + "time_per_iteration": 2.9633233547210693 + }, + { + "auxiliary_loss_clip": 0.01128776, + "auxiliary_loss_mlp": 0.01035511, + "balance_loss_clip": 1.04053831, + "balance_loss_mlp": 1.01938152, + "epoch": 0.7570492394637167, + "flos": 20769367086720.0, + "grad_norm": 2.0280285637308975, + "language_loss": 0.69061959, + "learning_rate": 5.877156075265166e-07, + "loss": 0.71226239, + "num_input_tokens_seen": 135359880, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.16125488, + "step": 6296, + "time_per_iteration": 2.9474434852600098 + }, + { + "auxiliary_loss_clip": 0.01126296, + "auxiliary_loss_mlp": 0.01036625, + "balance_loss_clip": 1.04008472, + "balance_loss_mlp": 1.01994729, + "epoch": 0.7571694823543558, + "flos": 15664091137920.0, + "grad_norm": 2.842717023190675, + "language_loss": 0.6985718, + "learning_rate": 5.871641481667715e-07, + "loss": 0.72020108, + "num_input_tokens_seen": 135374325, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.16674805, + "step": 6297, + "time_per_iteration": 2.943478584289551 + }, + { + "auxiliary_loss_clip": 0.01127439, + "auxiliary_loss_mlp": 0.0103659, + "balance_loss_clip": 1.03984523, + "balance_loss_mlp": 1.02084231, + "epoch": 0.7572897252449949, + "flos": 25409241492480.0, + "grad_norm": 1.6210821732811966, + "language_loss": 0.84366339, + "learning_rate": 5.866129031331011e-07, + "loss": 0.8653037, + "num_input_tokens_seen": 135393980, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.15740967, + "step": 6298, + "time_per_iteration": 3.0586884021759033 + }, + { + "auxiliary_loss_clip": 0.01125521, + "auxiliary_loss_mlp": 0.0102876, + "balance_loss_clip": 1.03967094, + "balance_loss_mlp": 1.01419806, + "epoch": 0.757409968135634, + "flos": 24279348297600.0, + "grad_norm": 2.0833141436905036, + "language_loss": 0.83575857, + "learning_rate": 5.8606187250913e-07, + "loss": 0.85730135, + "num_input_tokens_seen": 135412030, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.14556885, + "step": 6299, + "time_per_iteration": 3.004378080368042 + }, + { + "auxiliary_loss_clip": 0.01128241, + "auxiliary_loss_mlp": 0.01042467, + "balance_loss_clip": 1.04167128, + "balance_loss_mlp": 1.02592099, + "epoch": 0.757530211026273, + "flos": 24133766474880.0, + "grad_norm": 1.7250679614454845, + "language_loss": 0.84300804, + "learning_rate": 5.855110563784482e-07, + "loss": 0.8647151, + "num_input_tokens_seen": 135430565, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.16540527, + "step": 6300, + "time_per_iteration": 3.0485785007476807 + }, + { + "auxiliary_loss_clip": 0.01121772, + "auxiliary_loss_mlp": 0.01030458, + "balance_loss_clip": 1.03832364, + "balance_loss_mlp": 1.01679623, + "epoch": 0.7576504539169122, + "flos": 23951807153280.0, + "grad_norm": 1.6251878942265392, + "language_loss": 0.64287865, + "learning_rate": 5.849604548246156e-07, + "loss": 0.66440094, + "num_input_tokens_seen": 135451675, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.13671875, + "step": 6301, + "time_per_iteration": 3.030900716781616 + }, + { + "auxiliary_loss_clip": 0.011272, + "auxiliary_loss_mlp": 0.01035609, + "balance_loss_clip": 1.0403893, + "balance_loss_mlp": 1.02076733, + "epoch": 0.7577706968075513, + "flos": 21251360712960.0, + "grad_norm": 1.9572867089592785, + "language_loss": 0.80300188, + "learning_rate": 5.844100679311565e-07, + "loss": 0.82462996, + "num_input_tokens_seen": 135470635, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.14831543, + "step": 6302, + "time_per_iteration": 3.126751661300659 + }, + { + "auxiliary_loss_clip": 0.01127525, + "auxiliary_loss_mlp": 0.01043322, + "balance_loss_clip": 1.04012227, + "balance_loss_mlp": 1.02727699, + "epoch": 0.7578909396981903, + "flos": 18296595002880.0, + "grad_norm": 2.2893715934054977, + "language_loss": 0.76203704, + "learning_rate": 5.838598957815637e-07, + "loss": 0.78374553, + "num_input_tokens_seen": 135487865, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.16052246, + "step": 6303, + "time_per_iteration": 3.034721612930298 + }, + { + "auxiliary_loss_clip": 0.01122716, + "auxiliary_loss_mlp": 0.01033443, + "balance_loss_clip": 1.03909028, + "balance_loss_mlp": 1.01699781, + "epoch": 0.7580111825888295, + "flos": 25373869574400.0, + "grad_norm": 1.5905443430745714, + "language_loss": 0.8548432, + "learning_rate": 5.833099384592996e-07, + "loss": 0.87640476, + "num_input_tokens_seen": 135508440, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.16436768, + "step": 6304, + "time_per_iteration": 2.9322168827056885 + }, + { + "auxiliary_loss_clip": 0.01126928, + "auxiliary_loss_mlp": 0.01035291, + "balance_loss_clip": 1.04066372, + "balance_loss_mlp": 1.01999676, + "epoch": 0.7581314254794685, + "flos": 23768662682880.0, + "grad_norm": 3.1951973016369273, + "language_loss": 0.72043824, + "learning_rate": 5.827601960477913e-07, + "loss": 0.74206042, + "num_input_tokens_seen": 135526365, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.1529541, + "step": 6305, + "time_per_iteration": 2.9300482273101807 + }, + { + "auxiliary_loss_clip": 0.01124944, + "auxiliary_loss_mlp": 0.01035576, + "balance_loss_clip": 1.0406034, + "balance_loss_mlp": 1.0206151, + "epoch": 0.7582516683701076, + "flos": 22054610603520.0, + "grad_norm": 1.6845587145581746, + "language_loss": 0.70478487, + "learning_rate": 5.822106686304344e-07, + "loss": 0.72639012, + "num_input_tokens_seen": 135545655, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14959717, + "step": 6306, + "time_per_iteration": 2.951089859008789 + }, + { + "auxiliary_loss_clip": 0.01127315, + "auxiliary_loss_mlp": 0.01043361, + "balance_loss_clip": 1.0397718, + "balance_loss_mlp": 1.0273633, + "epoch": 0.7583719112607467, + "flos": 31649725848960.0, + "grad_norm": 2.157939358330647, + "language_loss": 0.58093917, + "learning_rate": 5.816613562905919e-07, + "loss": 0.60264587, + "num_input_tokens_seen": 135566840, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.16003418, + "step": 6307, + "time_per_iteration": 3.018648147583008 + }, + { + "auxiliary_loss_clip": 0.01126471, + "auxiliary_loss_mlp": 0.01039918, + "balance_loss_clip": 1.04276764, + "balance_loss_mlp": 1.02494538, + "epoch": 0.7584921541513858, + "flos": 33068376478080.0, + "grad_norm": 1.4706206394162418, + "language_loss": 0.69871867, + "learning_rate": 5.811122591115933e-07, + "loss": 0.72038257, + "num_input_tokens_seen": 135587825, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14978027, + "step": 6308, + "time_per_iteration": 2.932979106903076 + }, + { + "auxiliary_loss_clip": 0.01128251, + "auxiliary_loss_mlp": 0.01039235, + "balance_loss_clip": 1.04097462, + "balance_loss_mlp": 1.02353549, + "epoch": 0.7586123970420249, + "flos": 23326350606720.0, + "grad_norm": 2.394076035108682, + "language_loss": 0.71175307, + "learning_rate": 5.805633771767376e-07, + "loss": 0.73342788, + "num_input_tokens_seen": 135605220, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15686035, + "step": 6309, + "time_per_iteration": 3.734968662261963 + }, + { + "auxiliary_loss_clip": 0.01126032, + "auxiliary_loss_mlp": 0.01045168, + "balance_loss_clip": 1.04272962, + "balance_loss_mlp": 1.0314889, + "epoch": 0.7587326399326639, + "flos": 18334229477760.0, + "grad_norm": 1.685855802073008, + "language_loss": 0.77601415, + "learning_rate": 5.800147105692888e-07, + "loss": 0.79772615, + "num_input_tokens_seen": 135624795, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.13671875, + "step": 6310, + "time_per_iteration": 2.970266103744507 + }, + { + "auxiliary_loss_clip": 0.01125631, + "auxiliary_loss_mlp": 0.01039304, + "balance_loss_clip": 1.04020524, + "balance_loss_mlp": 1.02352047, + "epoch": 0.7588528828233031, + "flos": 17275080119040.0, + "grad_norm": 1.770849708747952, + "language_loss": 0.79293084, + "learning_rate": 5.794662593724795e-07, + "loss": 0.81458014, + "num_input_tokens_seen": 135643800, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.15783691, + "step": 6311, + "time_per_iteration": 2.816256523132324 + }, + { + "auxiliary_loss_clip": 0.01127127, + "auxiliary_loss_mlp": 0.01046734, + "balance_loss_clip": 1.04178977, + "balance_loss_mlp": 1.03190434, + "epoch": 0.7589731257139422, + "flos": 17713621267200.0, + "grad_norm": 2.1128094737971543, + "language_loss": 0.75296611, + "learning_rate": 5.789180236695091e-07, + "loss": 0.77470464, + "num_input_tokens_seen": 135660655, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.14819336, + "step": 6312, + "time_per_iteration": 2.8692591190338135 + }, + { + "auxiliary_loss_clip": 0.01121335, + "auxiliary_loss_mlp": 0.01030863, + "balance_loss_clip": 1.04065251, + "balance_loss_mlp": 1.01642728, + "epoch": 0.7590933686045812, + "flos": 15961072786560.0, + "grad_norm": 1.8368279581161966, + "language_loss": 0.84899706, + "learning_rate": 5.78370003543544e-07, + "loss": 0.87051904, + "num_input_tokens_seen": 135679410, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.14453125, + "step": 6313, + "time_per_iteration": 4.5144736766815186 + }, + { + "auxiliary_loss_clip": 0.01127742, + "auxiliary_loss_mlp": 0.01041678, + "balance_loss_clip": 1.03957319, + "balance_loss_mlp": 1.02653205, + "epoch": 0.7592136114952204, + "flos": 21068072588160.0, + "grad_norm": 1.9841334450982713, + "language_loss": 0.83819312, + "learning_rate": 5.778221990777203e-07, + "loss": 0.85988736, + "num_input_tokens_seen": 135697150, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15142822, + "step": 6314, + "time_per_iteration": 2.8788719177246094 + }, + { + "auxiliary_loss_clip": 0.01129203, + "auxiliary_loss_mlp": 0.01046151, + "balance_loss_clip": 1.04158902, + "balance_loss_mlp": 1.02959287, + "epoch": 0.7593338543858594, + "flos": 25297666871040.0, + "grad_norm": 2.0010519531929223, + "language_loss": 0.82971996, + "learning_rate": 5.772746103551372e-07, + "loss": 0.85147351, + "num_input_tokens_seen": 135712545, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.16546631, + "step": 6315, + "time_per_iteration": 2.860854148864746 + }, + { + "auxiliary_loss_clip": 0.01127783, + "auxiliary_loss_mlp": 0.01040094, + "balance_loss_clip": 1.04266453, + "balance_loss_mlp": 1.02408457, + "epoch": 0.7594540972764985, + "flos": 31832367528960.0, + "grad_norm": 1.681908796971934, + "language_loss": 0.71642423, + "learning_rate": 5.767272374588648e-07, + "loss": 0.73810303, + "num_input_tokens_seen": 135733950, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.16027832, + "step": 6316, + "time_per_iteration": 2.933716297149658 + }, + { + "auxiliary_loss_clip": 0.0112405, + "auxiliary_loss_mlp": 0.01041295, + "balance_loss_clip": 1.04018271, + "balance_loss_mlp": 1.02694845, + "epoch": 0.7595743401671377, + "flos": 37597250880000.0, + "grad_norm": 2.046052321998395, + "language_loss": 0.78310323, + "learning_rate": 5.76180080471939e-07, + "loss": 0.80475664, + "num_input_tokens_seen": 135757120, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14337158, + "step": 6317, + "time_per_iteration": 3.0215506553649902 + }, + { + "auxiliary_loss_clip": 0.01131606, + "auxiliary_loss_mlp": 0.01041677, + "balance_loss_clip": 1.04078937, + "balance_loss_mlp": 1.02554798, + "epoch": 0.7596945830577767, + "flos": 18287724343680.0, + "grad_norm": 1.989639284429418, + "language_loss": 0.72526777, + "learning_rate": 5.756331394773631e-07, + "loss": 0.74700058, + "num_input_tokens_seen": 135773335, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.16113281, + "step": 6318, + "time_per_iteration": 2.8805177211761475 + }, + { + "auxiliary_loss_clip": 0.01123389, + "auxiliary_loss_mlp": 0.01037369, + "balance_loss_clip": 1.03756332, + "balance_loss_mlp": 1.02200329, + "epoch": 0.7598148259484158, + "flos": 22233122219520.0, + "grad_norm": 1.7211952322249306, + "language_loss": 0.75997651, + "learning_rate": 5.750864145581071e-07, + "loss": 0.78158414, + "num_input_tokens_seen": 135792555, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15350342, + "step": 6319, + "time_per_iteration": 3.722022294998169 + }, + { + "auxiliary_loss_clip": 0.01124, + "auxiliary_loss_mlp": 0.01038302, + "balance_loss_clip": 1.04026937, + "balance_loss_mlp": 1.02360368, + "epoch": 0.7599350688390549, + "flos": 27161718145920.0, + "grad_norm": 2.26812537893533, + "language_loss": 0.86063409, + "learning_rate": 5.745399057971085e-07, + "loss": 0.8822571, + "num_input_tokens_seen": 135813690, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14709473, + "step": 6320, + "time_per_iteration": 2.872274160385132 + }, + { + "auxiliary_loss_clip": 0.01130055, + "auxiliary_loss_mlp": 0.01034191, + "balance_loss_clip": 1.04180408, + "balance_loss_mlp": 1.01852727, + "epoch": 0.760055311729694, + "flos": 15560704817280.0, + "grad_norm": 2.3208018290733046, + "language_loss": 0.75065958, + "learning_rate": 5.739936132772738e-07, + "loss": 0.77230203, + "num_input_tokens_seen": 135832255, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15661621, + "step": 6321, + "time_per_iteration": 2.857334613800049 + }, + { + "auxiliary_loss_clip": 0.01123452, + "auxiliary_loss_mlp": 0.01035752, + "balance_loss_clip": 1.03840387, + "balance_loss_mlp": 1.02100575, + "epoch": 0.760175554620333, + "flos": 25155496840320.0, + "grad_norm": 1.8992630833370905, + "language_loss": 0.74108273, + "learning_rate": 5.734475370814733e-07, + "loss": 0.76267481, + "num_input_tokens_seen": 135851935, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.14758301, + "step": 6322, + "time_per_iteration": 2.851320743560791 + }, + { + "auxiliary_loss_clip": 0.01127237, + "auxiliary_loss_mlp": 0.01038031, + "balance_loss_clip": 1.04057765, + "balance_loss_mlp": 1.02137733, + "epoch": 0.7602957975109722, + "flos": 24353791234560.0, + "grad_norm": 1.5475757096527867, + "language_loss": 0.7837202, + "learning_rate": 5.729016772925483e-07, + "loss": 0.80537283, + "num_input_tokens_seen": 135873510, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.16674805, + "step": 6323, + "time_per_iteration": 2.869370460510254 + }, + { + "auxiliary_loss_clip": 0.01125393, + "auxiliary_loss_mlp": 0.01039045, + "balance_loss_clip": 1.03923869, + "balance_loss_mlp": 1.02297509, + "epoch": 0.7604160404016113, + "flos": 25192664438400.0, + "grad_norm": 1.7872869116809955, + "language_loss": 0.70945191, + "learning_rate": 5.723560339933038e-07, + "loss": 0.73109627, + "num_input_tokens_seen": 135893845, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.16088867, + "step": 6324, + "time_per_iteration": 2.8187453746795654 + }, + { + "auxiliary_loss_clip": 0.01126412, + "auxiliary_loss_mlp": 0.01034607, + "balance_loss_clip": 1.03914797, + "balance_loss_mlp": 1.01856112, + "epoch": 0.7605362832922503, + "flos": 29861841363840.0, + "grad_norm": 1.9552899947652174, + "language_loss": 0.65112853, + "learning_rate": 5.71810607266513e-07, + "loss": 0.67273867, + "num_input_tokens_seen": 135912430, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.16040039, + "step": 6325, + "time_per_iteration": 2.8505618572235107 + }, + { + "auxiliary_loss_clip": 0.01121686, + "auxiliary_loss_mlp": 0.0103298, + "balance_loss_clip": 1.0380981, + "balance_loss_mlp": 1.01797175, + "epoch": 0.7606565261828895, + "flos": 13917935278080.0, + "grad_norm": 1.952325996649948, + "language_loss": 0.60147226, + "learning_rate": 5.712653971949184e-07, + "loss": 0.62301898, + "num_input_tokens_seen": 135930550, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14984131, + "step": 6326, + "time_per_iteration": 2.84039568901062 + }, + { + "auxiliary_loss_clip": 0.01126067, + "auxiliary_loss_mlp": 0.01037446, + "balance_loss_clip": 1.04096627, + "balance_loss_mlp": 1.02140021, + "epoch": 0.7607767690735285, + "flos": 18551273408640.0, + "grad_norm": 2.429408105463442, + "language_loss": 0.75439811, + "learning_rate": 5.707204038612268e-07, + "loss": 0.77603322, + "num_input_tokens_seen": 135947980, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.16040039, + "step": 6327, + "time_per_iteration": 2.8562252521514893 + }, + { + "auxiliary_loss_clip": 0.01134166, + "auxiliary_loss_mlp": 0.010475, + "balance_loss_clip": 1.04491937, + "balance_loss_mlp": 1.03031027, + "epoch": 0.7608970119641676, + "flos": 20922993555840.0, + "grad_norm": 2.38486311762415, + "language_loss": 0.73866081, + "learning_rate": 5.701756273481138e-07, + "loss": 0.76047748, + "num_input_tokens_seen": 135965400, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.171875, + "step": 6328, + "time_per_iteration": 2.8174400329589844 + }, + { + "auxiliary_loss_clip": 0.01123711, + "auxiliary_loss_mlp": 0.01036066, + "balance_loss_clip": 1.03947794, + "balance_loss_mlp": 1.02133179, + "epoch": 0.7610172548548068, + "flos": 23807302738560.0, + "grad_norm": 1.4488704467181925, + "language_loss": 0.73908842, + "learning_rate": 5.696310677382212e-07, + "loss": 0.76068622, + "num_input_tokens_seen": 135986795, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.1472168, + "step": 6329, + "time_per_iteration": 2.7790255546569824 + }, + { + "auxiliary_loss_clip": 0.01034658, + "auxiliary_loss_mlp": 0.01002546, + "balance_loss_clip": 1.00698054, + "balance_loss_mlp": 1.00105023, + "epoch": 0.7611374977454458, + "flos": 66496580426880.0, + "grad_norm": 0.8700728718402528, + "language_loss": 0.61721337, + "learning_rate": 5.690867251141576e-07, + "loss": 0.6375854, + "num_input_tokens_seen": 136053450, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01495361, + "step": 6330, + "time_per_iteration": 3.4376096725463867 + }, + { + "auxiliary_loss_clip": 0.01130219, + "auxiliary_loss_mlp": 0.01038818, + "balance_loss_clip": 1.04149151, + "balance_loss_mlp": 1.02348793, + "epoch": 0.7612577406360849, + "flos": 15633136592640.0, + "grad_norm": 2.629907683707683, + "language_loss": 0.9188329, + "learning_rate": 5.685425995585013e-07, + "loss": 0.94052333, + "num_input_tokens_seen": 136071375, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.15332031, + "step": 6331, + "time_per_iteration": 2.8583884239196777 + }, + { + "auxiliary_loss_clip": 0.01034475, + "auxiliary_loss_mlp": 0.00999496, + "balance_loss_clip": 1.0067389, + "balance_loss_mlp": 0.9979462, + "epoch": 0.761377983526724, + "flos": 60526253237760.0, + "grad_norm": 0.8215909525390866, + "language_loss": 0.59006, + "learning_rate": 5.679986911537935e-07, + "loss": 0.61039972, + "num_input_tokens_seen": 136138905, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01550293, + "step": 6332, + "time_per_iteration": 3.43660569190979 + }, + { + "auxiliary_loss_clip": 0.01123956, + "auxiliary_loss_mlp": 0.01042512, + "balance_loss_clip": 1.03896976, + "balance_loss_mlp": 1.02727723, + "epoch": 0.7614982264173631, + "flos": 35772522019200.0, + "grad_norm": 1.7872998480127105, + "language_loss": 0.66875362, + "learning_rate": 5.674549999825462e-07, + "loss": 0.69041836, + "num_input_tokens_seen": 136161720, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15246582, + "step": 6333, + "time_per_iteration": 2.8789541721343994 + }, + { + "auxiliary_loss_clip": 0.01034075, + "auxiliary_loss_mlp": 0.0099991, + "balance_loss_clip": 1.00627601, + "balance_loss_mlp": 0.99833626, + "epoch": 0.7616184693080021, + "flos": 67925502345600.0, + "grad_norm": 0.9187356544027608, + "language_loss": 0.71364087, + "learning_rate": 5.669115261272363e-07, + "loss": 0.73398072, + "num_input_tokens_seen": 136222040, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01574707, + "step": 6334, + "time_per_iteration": 3.282330274581909 + }, + { + "auxiliary_loss_clip": 0.01130496, + "auxiliary_loss_mlp": 0.01041177, + "balance_loss_clip": 1.0408715, + "balance_loss_mlp": 1.02480972, + "epoch": 0.7617387121986413, + "flos": 20521979141760.0, + "grad_norm": 2.0452321068476453, + "language_loss": 0.72227407, + "learning_rate": 5.663682696703081e-07, + "loss": 0.74399084, + "num_input_tokens_seen": 136240305, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.16369629, + "step": 6335, + "time_per_iteration": 2.767936944961548 + }, + { + "auxiliary_loss_clip": 0.01124207, + "auxiliary_loss_mlp": 0.01036451, + "balance_loss_clip": 1.04078817, + "balance_loss_mlp": 1.02182388, + "epoch": 0.7618589550892804, + "flos": 18624495283200.0, + "grad_norm": 1.7335625488651243, + "language_loss": 0.8227787, + "learning_rate": 5.658252306941746e-07, + "loss": 0.84438527, + "num_input_tokens_seen": 136259625, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.14648438, + "step": 6336, + "time_per_iteration": 2.7707161903381348 + }, + { + "auxiliary_loss_clip": 0.01128502, + "auxiliary_loss_mlp": 0.0104422, + "balance_loss_clip": 1.04051363, + "balance_loss_mlp": 1.02825785, + "epoch": 0.7619791979799194, + "flos": 17453735389440.0, + "grad_norm": 1.9988165594186054, + "language_loss": 0.75334287, + "learning_rate": 5.65282409281212e-07, + "loss": 0.77507001, + "num_input_tokens_seen": 136277090, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.15966797, + "step": 6337, + "time_per_iteration": 2.758957862854004 + }, + { + "auxiliary_loss_clip": 0.01127806, + "auxiliary_loss_mlp": 0.01035066, + "balance_loss_clip": 1.04212368, + "balance_loss_mlp": 1.01964056, + "epoch": 0.7620994408705585, + "flos": 14137421333760.0, + "grad_norm": 2.306724700784659, + "language_loss": 0.70082355, + "learning_rate": 5.64739805513768e-07, + "loss": 0.72245222, + "num_input_tokens_seen": 136294635, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15441895, + "step": 6338, + "time_per_iteration": 3.6588127613067627 + }, + { + "auxiliary_loss_clip": 0.01034778, + "auxiliary_loss_mlp": 0.01002654, + "balance_loss_clip": 1.00700879, + "balance_loss_mlp": 1.00127125, + "epoch": 0.7622196837611976, + "flos": 70708792527360.0, + "grad_norm": 0.7885630524405939, + "language_loss": 0.55719125, + "learning_rate": 5.641974194741541e-07, + "loss": 0.57756555, + "num_input_tokens_seen": 136350320, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01385498, + "step": 6339, + "time_per_iteration": 3.185441493988037 + }, + { + "auxiliary_loss_clip": 0.01034936, + "auxiliary_loss_mlp": 0.01003879, + "balance_loss_clip": 1.00708556, + "balance_loss_mlp": 1.00238276, + "epoch": 0.7623399266518367, + "flos": 60684150447360.0, + "grad_norm": 0.7799703502427994, + "language_loss": 0.63733923, + "learning_rate": 5.636552512446502e-07, + "loss": 0.65772736, + "num_input_tokens_seen": 136411375, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01495361, + "step": 6340, + "time_per_iteration": 3.2295665740966797 + }, + { + "auxiliary_loss_clip": 0.01128167, + "auxiliary_loss_mlp": 0.01039765, + "balance_loss_clip": 1.04093575, + "balance_loss_mlp": 1.02402902, + "epoch": 0.7624601695424758, + "flos": 26468893641600.0, + "grad_norm": 1.6595331967108506, + "language_loss": 0.77807963, + "learning_rate": 5.631133009075027e-07, + "loss": 0.79975903, + "num_input_tokens_seen": 136430560, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.15734863, + "step": 6341, + "time_per_iteration": 2.8606326580047607 + }, + { + "auxiliary_loss_clip": 0.01127176, + "auxiliary_loss_mlp": 0.01037668, + "balance_loss_clip": 1.04137182, + "balance_loss_mlp": 1.02330887, + "epoch": 0.7625804124331149, + "flos": 19135755515520.0, + "grad_norm": 1.7755219190404783, + "language_loss": 0.68493581, + "learning_rate": 5.625715685449242e-07, + "loss": 0.70658427, + "num_input_tokens_seen": 136448665, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.14355469, + "step": 6342, + "time_per_iteration": 4.361689329147339 + }, + { + "auxiliary_loss_clip": 0.01120811, + "auxiliary_loss_mlp": 0.01036431, + "balance_loss_clip": 1.04041886, + "balance_loss_mlp": 1.0214045, + "epoch": 0.762700655323754, + "flos": 26213101914240.0, + "grad_norm": 1.6226956579920278, + "language_loss": 0.72000563, + "learning_rate": 5.620300542390966e-07, + "loss": 0.7415781, + "num_input_tokens_seen": 136469710, + "router_z_loss_clip": 0.80371094, + "router_z_loss_mlp": 0.15008545, + "step": 6343, + "time_per_iteration": 2.7906885147094727 + }, + { + "auxiliary_loss_clip": 0.01121586, + "auxiliary_loss_mlp": 0.01032465, + "balance_loss_clip": 1.04005754, + "balance_loss_mlp": 1.0188632, + "epoch": 0.762820898214393, + "flos": 22382582711040.0, + "grad_norm": 1.8383647896578357, + "language_loss": 0.85495245, + "learning_rate": 5.614887580721659e-07, + "loss": 0.87649298, + "num_input_tokens_seen": 136489855, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.13598633, + "step": 6344, + "time_per_iteration": 2.7839365005493164 + }, + { + "auxiliary_loss_clip": 0.01125876, + "auxiliary_loss_mlp": 0.01039454, + "balance_loss_clip": 1.04071999, + "balance_loss_mlp": 1.02301455, + "epoch": 0.7629411411050322, + "flos": 15700504550400.0, + "grad_norm": 1.7956479817312352, + "language_loss": 0.73487931, + "learning_rate": 5.609476801262481e-07, + "loss": 0.75653255, + "num_input_tokens_seen": 136504715, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.16442871, + "step": 6345, + "time_per_iteration": 2.7297203540802 + }, + { + "auxiliary_loss_clip": 0.01129281, + "auxiliary_loss_mlp": 0.01038834, + "balance_loss_clip": 1.04158902, + "balance_loss_mlp": 1.02363467, + "epoch": 0.7630613839956712, + "flos": 13770342293760.0, + "grad_norm": 3.966236103137824, + "language_loss": 0.64140797, + "learning_rate": 5.604068204834223e-07, + "loss": 0.66308904, + "num_input_tokens_seen": 136521610, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15203857, + "step": 6346, + "time_per_iteration": 2.7651655673980713 + }, + { + "auxiliary_loss_clip": 0.01127027, + "auxiliary_loss_mlp": 0.01039807, + "balance_loss_clip": 1.03918612, + "balance_loss_mlp": 1.0241785, + "epoch": 0.7631816268863103, + "flos": 14569569861120.0, + "grad_norm": 2.356704685027799, + "language_loss": 0.75971603, + "learning_rate": 5.598661792257367e-07, + "loss": 0.78138441, + "num_input_tokens_seen": 136538655, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.15625, + "step": 6347, + "time_per_iteration": 2.714431047439575 + }, + { + "auxiliary_loss_clip": 0.01121085, + "auxiliary_loss_mlp": 0.0104547, + "balance_loss_clip": 1.03747356, + "balance_loss_mlp": 1.02992511, + "epoch": 0.7633018697769495, + "flos": 19062210418560.0, + "grad_norm": 1.921120802846761, + "language_loss": 0.76046062, + "learning_rate": 5.593257564352071e-07, + "loss": 0.78212619, + "num_input_tokens_seen": 136557095, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.15533447, + "step": 6348, + "time_per_iteration": 3.5762226581573486 + }, + { + "auxiliary_loss_clip": 0.01122926, + "auxiliary_loss_mlp": 0.01035322, + "balance_loss_clip": 1.04044247, + "balance_loss_mlp": 1.02012908, + "epoch": 0.7634221126675885, + "flos": 22052958577920.0, + "grad_norm": 1.471576748270097, + "language_loss": 0.75439185, + "learning_rate": 5.58785552193815e-07, + "loss": 0.77597439, + "num_input_tokens_seen": 136577340, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.15203857, + "step": 6349, + "time_per_iteration": 2.765272617340088 + }, + { + "auxiliary_loss_clip": 0.0112498, + "auxiliary_loss_mlp": 0.01037879, + "balance_loss_clip": 1.03984118, + "balance_loss_mlp": 1.02374673, + "epoch": 0.7635423555582276, + "flos": 29382720825600.0, + "grad_norm": 1.715891151898356, + "language_loss": 0.75576293, + "learning_rate": 5.582455665835086e-07, + "loss": 0.77739149, + "num_input_tokens_seen": 136597635, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.14135742, + "step": 6350, + "time_per_iteration": 2.840066432952881 + }, + { + "auxiliary_loss_clip": 0.01132844, + "auxiliary_loss_mlp": 0.01042987, + "balance_loss_clip": 1.04170823, + "balance_loss_mlp": 1.02609539, + "epoch": 0.7636625984488667, + "flos": 17784903807360.0, + "grad_norm": 3.098274298581204, + "language_loss": 0.73192793, + "learning_rate": 5.577057996862036e-07, + "loss": 0.75368619, + "num_input_tokens_seen": 136615260, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.16894531, + "step": 6351, + "time_per_iteration": 2.796692132949829 + }, + { + "auxiliary_loss_clip": 0.01123154, + "auxiliary_loss_mlp": 0.01029656, + "balance_loss_clip": 1.04123425, + "balance_loss_mlp": 1.01503468, + "epoch": 0.7637828413395058, + "flos": 23734583654400.0, + "grad_norm": 1.5798742544284887, + "language_loss": 0.76037288, + "learning_rate": 5.571662515837814e-07, + "loss": 0.781901, + "num_input_tokens_seen": 136637220, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.1461792, + "step": 6352, + "time_per_iteration": 2.7893784046173096 + }, + { + "auxiliary_loss_clip": 0.01123937, + "auxiliary_loss_mlp": 0.01035999, + "balance_loss_clip": 1.03973532, + "balance_loss_mlp": 1.02075243, + "epoch": 0.7639030842301449, + "flos": 36283279461120.0, + "grad_norm": 1.6558264542542274, + "language_loss": 0.83572775, + "learning_rate": 5.566269223580926e-07, + "loss": 0.8573271, + "num_input_tokens_seen": 136658930, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.15246582, + "step": 6353, + "time_per_iteration": 2.88224196434021 + }, + { + "auxiliary_loss_clip": 0.01123062, + "auxiliary_loss_mlp": 0.01039186, + "balance_loss_clip": 1.03907466, + "balance_loss_mlp": 1.02473748, + "epoch": 0.764023327120784, + "flos": 28878104609280.0, + "grad_norm": 1.5955455151438442, + "language_loss": 0.75193894, + "learning_rate": 5.560878120909511e-07, + "loss": 0.77356136, + "num_input_tokens_seen": 136681530, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.14447021, + "step": 6354, + "time_per_iteration": 2.828278064727783 + }, + { + "auxiliary_loss_clip": 0.01034722, + "auxiliary_loss_mlp": 0.00999388, + "balance_loss_clip": 1.00701332, + "balance_loss_mlp": 0.99793959, + "epoch": 0.7641435700114231, + "flos": 64789711067520.0, + "grad_norm": 1.014171766180817, + "language_loss": 0.58558857, + "learning_rate": 5.55548920864141e-07, + "loss": 0.60592967, + "num_input_tokens_seen": 136742185, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01446533, + "step": 6355, + "time_per_iteration": 3.3143742084503174 + }, + { + "auxiliary_loss_clip": 0.01125706, + "auxiliary_loss_mlp": 0.01037291, + "balance_loss_clip": 1.04138136, + "balance_loss_mlp": 1.02384377, + "epoch": 0.7642638129020621, + "flos": 16835784785280.0, + "grad_norm": 1.65038890462436, + "language_loss": 0.7832064, + "learning_rate": 5.550102487594113e-07, + "loss": 0.80483633, + "num_input_tokens_seen": 136760855, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.13439941, + "step": 6356, + "time_per_iteration": 2.8533172607421875 + }, + { + "auxiliary_loss_clip": 0.01125039, + "auxiliary_loss_mlp": 0.01036585, + "balance_loss_clip": 1.03864551, + "balance_loss_mlp": 1.02097416, + "epoch": 0.7643840557927013, + "flos": 30408940391040.0, + "grad_norm": 1.7716498758998527, + "language_loss": 0.71268046, + "learning_rate": 5.54471795858477e-07, + "loss": 0.73429668, + "num_input_tokens_seen": 136780925, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.15594482, + "step": 6357, + "time_per_iteration": 2.8386850357055664 + }, + { + "auxiliary_loss_clip": 0.01123984, + "auxiliary_loss_mlp": 0.01031395, + "balance_loss_clip": 1.04035294, + "balance_loss_mlp": 1.01633894, + "epoch": 0.7645042986833404, + "flos": 16983234115200.0, + "grad_norm": 2.047167814622349, + "language_loss": 0.82897764, + "learning_rate": 5.539335622430235e-07, + "loss": 0.85053146, + "num_input_tokens_seen": 136799545, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.1505127, + "step": 6358, + "time_per_iteration": 2.8342838287353516 + }, + { + "auxiliary_loss_clip": 0.01121839, + "auxiliary_loss_mlp": 0.0103304, + "balance_loss_clip": 1.03807378, + "balance_loss_mlp": 1.01876473, + "epoch": 0.7646245415739794, + "flos": 17311493531520.0, + "grad_norm": 2.1114855411059694, + "language_loss": 0.7441445, + "learning_rate": 5.533955479946975e-07, + "loss": 0.76569331, + "num_input_tokens_seen": 136818325, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14263916, + "step": 6359, + "time_per_iteration": 2.879333734512329 + }, + { + "auxiliary_loss_clip": 0.01034011, + "auxiliary_loss_mlp": 0.01001467, + "balance_loss_clip": 1.0063467, + "balance_loss_mlp": 0.99995297, + "epoch": 0.7647447844646186, + "flos": 70402332666240.0, + "grad_norm": 0.8612822017328657, + "language_loss": 0.65761375, + "learning_rate": 5.528577531951173e-07, + "loss": 0.6779685, + "num_input_tokens_seen": 136878730, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01513672, + "step": 6360, + "time_per_iteration": 3.2888009548187256 + }, + { + "auxiliary_loss_clip": 0.01124244, + "auxiliary_loss_mlp": 0.01031394, + "balance_loss_clip": 1.04061055, + "balance_loss_mlp": 1.01671922, + "epoch": 0.7648650273552576, + "flos": 17675914965120.0, + "grad_norm": 1.9668465355234404, + "language_loss": 0.73911572, + "learning_rate": 5.523201779258653e-07, + "loss": 0.76067209, + "num_input_tokens_seen": 136897705, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14685059, + "step": 6361, + "time_per_iteration": 2.915599822998047 + }, + { + "auxiliary_loss_clip": 0.01123302, + "auxiliary_loss_mlp": 0.01034308, + "balance_loss_clip": 1.0369091, + "balance_loss_mlp": 1.01835752, + "epoch": 0.7649852702458967, + "flos": 22162019247360.0, + "grad_norm": 1.9426041693260092, + "language_loss": 0.8409493, + "learning_rate": 5.517828222684912e-07, + "loss": 0.86252546, + "num_input_tokens_seen": 136918360, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.1595459, + "step": 6362, + "time_per_iteration": 2.8759286403656006 + }, + { + "auxiliary_loss_clip": 0.01034634, + "auxiliary_loss_mlp": 0.01001188, + "balance_loss_clip": 1.00679922, + "balance_loss_mlp": 0.99961418, + "epoch": 0.7651055131365359, + "flos": 69848338227840.0, + "grad_norm": 0.766579889507941, + "language_loss": 0.5904758, + "learning_rate": 5.512456863045117e-07, + "loss": 0.610834, + "num_input_tokens_seen": 136979050, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01574707, + "step": 6363, + "time_per_iteration": 3.2789266109466553 + }, + { + "auxiliary_loss_clip": 0.01125069, + "auxiliary_loss_mlp": 0.01032732, + "balance_loss_clip": 1.03895307, + "balance_loss_mlp": 1.01750934, + "epoch": 0.7652257560271749, + "flos": 19464014931840.0, + "grad_norm": 1.6993409102783694, + "language_loss": 0.74232042, + "learning_rate": 5.507087701154089e-07, + "loss": 0.76389837, + "num_input_tokens_seen": 136998970, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15222168, + "step": 6364, + "time_per_iteration": 2.7747206687927246 + }, + { + "auxiliary_loss_clip": 0.01126481, + "auxiliary_loss_mlp": 0.01036286, + "balance_loss_clip": 1.04140043, + "balance_loss_mlp": 1.02109861, + "epoch": 0.765345998917814, + "flos": 15961108700160.0, + "grad_norm": 1.841528801682232, + "language_loss": 0.75193596, + "learning_rate": 5.50172073782634e-07, + "loss": 0.77356368, + "num_input_tokens_seen": 137016950, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.15185547, + "step": 6365, + "time_per_iteration": 2.7772579193115234 + }, + { + "auxiliary_loss_clip": 0.01123185, + "auxiliary_loss_mlp": 0.01036443, + "balance_loss_clip": 1.04067862, + "balance_loss_mlp": 1.0218215, + "epoch": 0.7654662418084531, + "flos": 23659853408640.0, + "grad_norm": 1.776003189499515, + "language_loss": 0.8740027, + "learning_rate": 5.496355973876023e-07, + "loss": 0.89559889, + "num_input_tokens_seen": 137036205, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.1461792, + "step": 6366, + "time_per_iteration": 2.759105920791626 + }, + { + "auxiliary_loss_clip": 0.01132258, + "auxiliary_loss_mlp": 0.01036467, + "balance_loss_clip": 1.04137731, + "balance_loss_mlp": 1.02073097, + "epoch": 0.7655864846990922, + "flos": 41463608878080.0, + "grad_norm": 1.654720211982244, + "language_loss": 0.70951432, + "learning_rate": 5.490993410116984e-07, + "loss": 0.73120153, + "num_input_tokens_seen": 137059195, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.15722656, + "step": 6367, + "time_per_iteration": 3.681344747543335 + }, + { + "auxiliary_loss_clip": 0.01124726, + "auxiliary_loss_mlp": 0.01030109, + "balance_loss_clip": 1.04062128, + "balance_loss_mlp": 1.01463509, + "epoch": 0.7657067275897312, + "flos": 43142684088960.0, + "grad_norm": 1.6642117551043, + "language_loss": 0.69928628, + "learning_rate": 5.485633047362704e-07, + "loss": 0.72083455, + "num_input_tokens_seen": 137081200, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.15454102, + "step": 6368, + "time_per_iteration": 2.953404426574707 + }, + { + "auxiliary_loss_clip": 0.01133444, + "auxiliary_loss_mlp": 0.01040276, + "balance_loss_clip": 1.04488552, + "balance_loss_mlp": 1.02516019, + "epoch": 0.7658269704803703, + "flos": 17311780840320.0, + "grad_norm": 1.7980464209762361, + "language_loss": 0.78040302, + "learning_rate": 5.480274886426341e-07, + "loss": 0.80214024, + "num_input_tokens_seen": 137097840, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.15112305, + "step": 6369, + "time_per_iteration": 2.7547779083251953 + }, + { + "auxiliary_loss_clip": 0.01122873, + "auxiliary_loss_mlp": 0.01034009, + "balance_loss_clip": 1.03966653, + "balance_loss_mlp": 1.0198772, + "epoch": 0.7659472133710095, + "flos": 12568160977920.0, + "grad_norm": 4.809297682219353, + "language_loss": 0.77875793, + "learning_rate": 5.474918928120744e-07, + "loss": 0.8003267, + "num_input_tokens_seen": 137114335, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.14123535, + "step": 6370, + "time_per_iteration": 2.7435719966888428 + }, + { + "auxiliary_loss_clip": 0.01123273, + "auxiliary_loss_mlp": 0.01030184, + "balance_loss_clip": 1.04022551, + "balance_loss_mlp": 1.0162065, + "epoch": 0.7660674562616485, + "flos": 22707430335360.0, + "grad_norm": 1.616945113388171, + "language_loss": 0.87438107, + "learning_rate": 5.469565173258392e-07, + "loss": 0.89591575, + "num_input_tokens_seen": 137132850, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.13983154, + "step": 6371, + "time_per_iteration": 4.544707536697388 + }, + { + "auxiliary_loss_clip": 0.01129857, + "auxiliary_loss_mlp": 0.01042463, + "balance_loss_clip": 1.04043436, + "balance_loss_mlp": 1.02607214, + "epoch": 0.7661876991522876, + "flos": 17056455989760.0, + "grad_norm": 1.6113435957088584, + "language_loss": 0.63960254, + "learning_rate": 5.464213622651454e-07, + "loss": 0.66132575, + "num_input_tokens_seen": 137150665, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.16394043, + "step": 6372, + "time_per_iteration": 2.79270339012146 + }, + { + "auxiliary_loss_clip": 0.01123874, + "auxiliary_loss_mlp": 0.01040945, + "balance_loss_clip": 1.03850591, + "balance_loss_mlp": 1.02499509, + "epoch": 0.7663079420429267, + "flos": 20084228092800.0, + "grad_norm": 2.2801936979987802, + "language_loss": 0.8440721, + "learning_rate": 5.458864277111753e-07, + "loss": 0.86572027, + "num_input_tokens_seen": 137168500, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.15930176, + "step": 6373, + "time_per_iteration": 2.8677079677581787 + }, + { + "auxiliary_loss_clip": 0.01121414, + "auxiliary_loss_mlp": 0.01035786, + "balance_loss_clip": 1.03856897, + "balance_loss_mlp": 1.02075374, + "epoch": 0.7664281849335658, + "flos": 12677473042560.0, + "grad_norm": 3.68190456340032, + "language_loss": 0.69440663, + "learning_rate": 5.453517137450769e-07, + "loss": 0.71597862, + "num_input_tokens_seen": 137185075, + "router_z_loss_clip": 0.828125, + "router_z_loss_mlp": 0.15039062, + "step": 6374, + "time_per_iteration": 2.798283100128174 + }, + { + "auxiliary_loss_clip": 0.0112922, + "auxiliary_loss_mlp": 0.01034449, + "balance_loss_clip": 1.0417769, + "balance_loss_mlp": 1.01966667, + "epoch": 0.7665484278242048, + "flos": 22345271458560.0, + "grad_norm": 1.7367192755068503, + "language_loss": 0.76031828, + "learning_rate": 5.448172204479684e-07, + "loss": 0.78195494, + "num_input_tokens_seen": 137204355, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.14770508, + "step": 6375, + "time_per_iteration": 2.759815216064453 + }, + { + "auxiliary_loss_clip": 0.01122136, + "auxiliary_loss_mlp": 0.01042083, + "balance_loss_clip": 1.03835154, + "balance_loss_mlp": 1.02696776, + "epoch": 0.766668670714844, + "flos": 23617909301760.0, + "grad_norm": 1.682524239436276, + "language_loss": 0.74107456, + "learning_rate": 5.442829479009294e-07, + "loss": 0.76271677, + "num_input_tokens_seen": 137223135, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.15112305, + "step": 6376, + "time_per_iteration": 2.8218586444854736 + }, + { + "auxiliary_loss_clip": 0.01130461, + "auxiliary_loss_mlp": 0.01038725, + "balance_loss_clip": 1.04137373, + "balance_loss_mlp": 1.0218327, + "epoch": 0.7667889136054831, + "flos": 19427134642560.0, + "grad_norm": 2.1063427289823933, + "language_loss": 0.71784437, + "learning_rate": 5.437488961850103e-07, + "loss": 0.73953617, + "num_input_tokens_seen": 137242935, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.16882324, + "step": 6377, + "time_per_iteration": 3.6795101165771484 + }, + { + "auxiliary_loss_clip": 0.01122864, + "auxiliary_loss_mlp": 0.01032792, + "balance_loss_clip": 1.04035258, + "balance_loss_mlp": 1.01858854, + "epoch": 0.7669091564961221, + "flos": 26866352609280.0, + "grad_norm": 1.7215966215340754, + "language_loss": 0.75332129, + "learning_rate": 5.432150653812258e-07, + "loss": 0.77487791, + "num_input_tokens_seen": 137262970, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.14202881, + "step": 6378, + "time_per_iteration": 2.8243937492370605 + }, + { + "auxiliary_loss_clip": 0.01124527, + "auxiliary_loss_mlp": 0.01039258, + "balance_loss_clip": 1.03943849, + "balance_loss_mlp": 1.02271199, + "epoch": 0.7670293993867613, + "flos": 12385303816320.0, + "grad_norm": 2.3588510995272816, + "language_loss": 0.82905924, + "learning_rate": 5.42681455570557e-07, + "loss": 0.85069716, + "num_input_tokens_seen": 137279500, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.16558838, + "step": 6379, + "time_per_iteration": 2.760944128036499 + }, + { + "auxiliary_loss_clip": 0.01121567, + "auxiliary_loss_mlp": 0.01032137, + "balance_loss_clip": 1.03687012, + "balance_loss_mlp": 1.01666963, + "epoch": 0.7671496422774003, + "flos": 21762944167680.0, + "grad_norm": 1.653988407007615, + "language_loss": 0.64831167, + "learning_rate": 5.42148066833954e-07, + "loss": 0.66984868, + "num_input_tokens_seen": 137298745, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15460205, + "step": 6380, + "time_per_iteration": 2.7821242809295654 + }, + { + "auxiliary_loss_clip": 0.01123574, + "auxiliary_loss_mlp": 0.01037043, + "balance_loss_clip": 1.04032898, + "balance_loss_mlp": 1.02218938, + "epoch": 0.7672698851680394, + "flos": 21069221823360.0, + "grad_norm": 1.8569054416220958, + "language_loss": 0.7568332, + "learning_rate": 5.416148992523289e-07, + "loss": 0.77843928, + "num_input_tokens_seen": 137317320, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.1484375, + "step": 6381, + "time_per_iteration": 2.8752946853637695 + }, + { + "auxiliary_loss_clip": 0.01125675, + "auxiliary_loss_mlp": 0.01043458, + "balance_loss_clip": 1.04104114, + "balance_loss_mlp": 1.02840805, + "epoch": 0.7673901280586786, + "flos": 16976697840000.0, + "grad_norm": 1.6574089653144881, + "language_loss": 0.78431028, + "learning_rate": 5.410819529065644e-07, + "loss": 0.80600166, + "num_input_tokens_seen": 137335275, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.15045166, + "step": 6382, + "time_per_iteration": 2.9062745571136475 + }, + { + "auxiliary_loss_clip": 0.01122981, + "auxiliary_loss_mlp": 0.01034615, + "balance_loss_clip": 1.03898692, + "balance_loss_mlp": 1.01948762, + "epoch": 0.7675103709493176, + "flos": 29242669697280.0, + "grad_norm": 1.955775333145555, + "language_loss": 0.6522426, + "learning_rate": 5.405492278775079e-07, + "loss": 0.67381853, + "num_input_tokens_seen": 137355055, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.15112305, + "step": 6383, + "time_per_iteration": 2.844724178314209 + }, + { + "auxiliary_loss_clip": 0.01129694, + "auxiliary_loss_mlp": 0.01039256, + "balance_loss_clip": 1.04097044, + "balance_loss_mlp": 1.02303195, + "epoch": 0.7676306138399567, + "flos": 29023004073600.0, + "grad_norm": 2.0470069284854686, + "language_loss": 0.80031043, + "learning_rate": 5.400167242459732e-07, + "loss": 0.82199997, + "num_input_tokens_seen": 137374015, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.16210938, + "step": 6384, + "time_per_iteration": 2.8536603450775146 + }, + { + "auxiliary_loss_clip": 0.01124571, + "auxiliary_loss_mlp": 0.01038889, + "balance_loss_clip": 1.04034019, + "balance_loss_mlp": 1.02354717, + "epoch": 0.7677508567305958, + "flos": 22565116650240.0, + "grad_norm": 1.7006147544941794, + "language_loss": 0.80617034, + "learning_rate": 5.394844420927405e-07, + "loss": 0.82780492, + "num_input_tokens_seen": 137393625, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.15332031, + "step": 6385, + "time_per_iteration": 2.8369810581207275 + }, + { + "auxiliary_loss_clip": 0.01122636, + "auxiliary_loss_mlp": 0.01040512, + "balance_loss_clip": 1.03896713, + "balance_loss_mlp": 1.024472, + "epoch": 0.7678710996212349, + "flos": 25411432222080.0, + "grad_norm": 1.9687626800457823, + "language_loss": 0.73864067, + "learning_rate": 5.389523814985562e-07, + "loss": 0.76027215, + "num_input_tokens_seen": 137413045, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.16009521, + "step": 6386, + "time_per_iteration": 2.8504581451416016 + }, + { + "auxiliary_loss_clip": 0.01128658, + "auxiliary_loss_mlp": 0.01040065, + "balance_loss_clip": 1.04213619, + "balance_loss_mlp": 1.02360237, + "epoch": 0.767991342511874, + "flos": 26756825063040.0, + "grad_norm": 2.222711782511241, + "language_loss": 0.76034522, + "learning_rate": 5.384205425441344e-07, + "loss": 0.78203249, + "num_input_tokens_seen": 137433955, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.16467285, + "step": 6387, + "time_per_iteration": 2.836544990539551 + }, + { + "auxiliary_loss_clip": 0.01123947, + "auxiliary_loss_mlp": 0.01035997, + "balance_loss_clip": 1.0393312, + "balance_loss_mlp": 1.02113128, + "epoch": 0.7681115854025131, + "flos": 26359509749760.0, + "grad_norm": 1.5917314715398354, + "language_loss": 0.84190905, + "learning_rate": 5.378889253101537e-07, + "loss": 0.86350846, + "num_input_tokens_seen": 137454510, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.14855957, + "step": 6388, + "time_per_iteration": 2.8677992820739746 + }, + { + "auxiliary_loss_clip": 0.01121008, + "auxiliary_loss_mlp": 0.01041648, + "balance_loss_clip": 1.03901124, + "balance_loss_mlp": 1.02782583, + "epoch": 0.7682318282931522, + "flos": 23257043314560.0, + "grad_norm": 1.6249953420653516, + "language_loss": 0.8079266, + "learning_rate": 5.373575298772617e-07, + "loss": 0.82955319, + "num_input_tokens_seen": 137473630, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.13812256, + "step": 6389, + "time_per_iteration": 2.7590014934539795 + }, + { + "auxiliary_loss_clip": 0.01033754, + "auxiliary_loss_mlp": 0.01004935, + "balance_loss_clip": 1.00596714, + "balance_loss_mlp": 1.00339735, + "epoch": 0.7683520711837912, + "flos": 70072457137920.0, + "grad_norm": 0.7687224541736523, + "language_loss": 0.61348104, + "learning_rate": 5.368263563260689e-07, + "loss": 0.63386798, + "num_input_tokens_seen": 137538765, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01538086, + "step": 6390, + "time_per_iteration": 3.359769582748413 + }, + { + "auxiliary_loss_clip": 0.01129974, + "auxiliary_loss_mlp": 0.01040602, + "balance_loss_clip": 1.04272854, + "balance_loss_mlp": 1.02569497, + "epoch": 0.7684723140744304, + "flos": 18624890332800.0, + "grad_norm": 2.245255702316175, + "language_loss": 0.64092934, + "learning_rate": 5.362954047371537e-07, + "loss": 0.66263509, + "num_input_tokens_seen": 137557875, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.14898682, + "step": 6391, + "time_per_iteration": 2.7106077671051025 + }, + { + "auxiliary_loss_clip": 0.01135225, + "auxiliary_loss_mlp": 0.01047215, + "balance_loss_clip": 1.04433942, + "balance_loss_mlp": 1.03026378, + "epoch": 0.7685925569650695, + "flos": 27452989532160.0, + "grad_norm": 2.0276434675212274, + "language_loss": 0.72199571, + "learning_rate": 5.357646751910627e-07, + "loss": 0.74382019, + "num_input_tokens_seen": 137579055, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.16955566, + "step": 6392, + "time_per_iteration": 2.8225972652435303 + }, + { + "auxiliary_loss_clip": 0.01131359, + "auxiliary_loss_mlp": 0.01042674, + "balance_loss_clip": 1.04129577, + "balance_loss_mlp": 1.02576447, + "epoch": 0.7687127998557085, + "flos": 24535714642560.0, + "grad_norm": 2.1037326142884125, + "language_loss": 0.80054295, + "learning_rate": 5.352341677683061e-07, + "loss": 0.82228327, + "num_input_tokens_seen": 137600355, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.16912842, + "step": 6393, + "time_per_iteration": 2.8400461673736572 + }, + { + "auxiliary_loss_clip": 0.01125064, + "auxiliary_loss_mlp": 0.01036798, + "balance_loss_clip": 1.03905272, + "balance_loss_mlp": 1.02166438, + "epoch": 0.7688330427463477, + "flos": 25155963717120.0, + "grad_norm": 1.9248992886646115, + "language_loss": 0.79134774, + "learning_rate": 5.347038825493617e-07, + "loss": 0.81296635, + "num_input_tokens_seen": 137621885, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15136719, + "step": 6394, + "time_per_iteration": 2.755486249923706 + }, + { + "auxiliary_loss_clip": 0.01121254, + "auxiliary_loss_mlp": 0.01040891, + "balance_loss_clip": 1.03945827, + "balance_loss_mlp": 1.02690732, + "epoch": 0.7689532856369867, + "flos": 21211284113280.0, + "grad_norm": 1.8309144318780544, + "language_loss": 0.68607384, + "learning_rate": 5.341738196146732e-07, + "loss": 0.70769531, + "num_input_tokens_seen": 137640230, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.13983154, + "step": 6395, + "time_per_iteration": 2.785409927368164 + }, + { + "auxiliary_loss_clip": 0.01126864, + "auxiliary_loss_mlp": 0.01033913, + "balance_loss_clip": 1.03969622, + "balance_loss_mlp": 1.018332, + "epoch": 0.7690735285276258, + "flos": 25119083427840.0, + "grad_norm": 1.9459401409920418, + "language_loss": 0.73734498, + "learning_rate": 5.336439790446503e-07, + "loss": 0.7589528, + "num_input_tokens_seen": 137659330, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15563965, + "step": 6396, + "time_per_iteration": 3.57211971282959 + }, + { + "auxiliary_loss_clip": 0.011285, + "auxiliary_loss_mlp": 0.01042628, + "balance_loss_clip": 1.0427376, + "balance_loss_mlp": 1.02779889, + "epoch": 0.769193771418265, + "flos": 54744020640000.0, + "grad_norm": 1.7876750046196883, + "language_loss": 0.62634349, + "learning_rate": 5.331143609196711e-07, + "loss": 0.64805484, + "num_input_tokens_seen": 137683145, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.14831543, + "step": 6397, + "time_per_iteration": 3.058394432067871 + }, + { + "auxiliary_loss_clip": 0.01125425, + "auxiliary_loss_mlp": 0.01042779, + "balance_loss_clip": 1.03965759, + "balance_loss_mlp": 1.02762794, + "epoch": 0.769314014308904, + "flos": 37341890115840.0, + "grad_norm": 1.7684682116870785, + "language_loss": 0.77199757, + "learning_rate": 5.325849653200758e-07, + "loss": 0.79367959, + "num_input_tokens_seen": 137707095, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.15161133, + "step": 6398, + "time_per_iteration": 2.8648147583007812 + }, + { + "auxiliary_loss_clip": 0.01125891, + "auxiliary_loss_mlp": 0.01038978, + "balance_loss_clip": 1.04076207, + "balance_loss_mlp": 1.02349293, + "epoch": 0.7694342571995431, + "flos": 20631686256000.0, + "grad_norm": 1.805894558355587, + "language_loss": 0.76705503, + "learning_rate": 5.32055792326175e-07, + "loss": 0.78870368, + "num_input_tokens_seen": 137725520, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15478516, + "step": 6399, + "time_per_iteration": 2.8831753730773926 + }, + { + "auxiliary_loss_clip": 0.01129597, + "auxiliary_loss_mlp": 0.01037211, + "balance_loss_clip": 1.04227841, + "balance_loss_mlp": 1.02173138, + "epoch": 0.7695545000901821, + "flos": 24207706621440.0, + "grad_norm": 2.157799608561002, + "language_loss": 0.72990358, + "learning_rate": 5.315268420182437e-07, + "loss": 0.7515716, + "num_input_tokens_seen": 137744195, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15484619, + "step": 6400, + "time_per_iteration": 4.441914319992065 + }, + { + "auxiliary_loss_clip": 0.01127185, + "auxiliary_loss_mlp": 0.01040722, + "balance_loss_clip": 1.04015112, + "balance_loss_mlp": 1.02394938, + "epoch": 0.7696747429808213, + "flos": 28001273708160.0, + "grad_norm": 1.7292873519928893, + "language_loss": 0.7647751, + "learning_rate": 5.309981144765221e-07, + "loss": 0.78645414, + "num_input_tokens_seen": 137764340, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.16760254, + "step": 6401, + "time_per_iteration": 2.8103256225585938 + }, + { + "auxiliary_loss_clip": 0.01125527, + "auxiliary_loss_mlp": 0.0103956, + "balance_loss_clip": 1.04104519, + "balance_loss_mlp": 1.02451539, + "epoch": 0.7697949858714603, + "flos": 11509550323200.0, + "grad_norm": 2.3797688773876793, + "language_loss": 0.75724345, + "learning_rate": 5.304696097812196e-07, + "loss": 0.77889431, + "num_input_tokens_seen": 137780940, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.1505127, + "step": 6402, + "time_per_iteration": 2.783458948135376 + }, + { + "auxiliary_loss_clip": 0.01133428, + "auxiliary_loss_mlp": 0.01048676, + "balance_loss_clip": 1.04189205, + "balance_loss_mlp": 1.03077078, + "epoch": 0.7699152287620994, + "flos": 26688271956480.0, + "grad_norm": 2.8610700656690216, + "language_loss": 0.60887796, + "learning_rate": 5.299413280125078e-07, + "loss": 0.63069898, + "num_input_tokens_seen": 137799250, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.17907715, + "step": 6403, + "time_per_iteration": 2.836780548095703 + }, + { + "auxiliary_loss_clip": 0.01126344, + "auxiliary_loss_mlp": 0.01039468, + "balance_loss_clip": 1.0395273, + "balance_loss_mlp": 1.02497244, + "epoch": 0.7700354716527386, + "flos": 16544944362240.0, + "grad_norm": 3.2553115140121727, + "language_loss": 0.73274821, + "learning_rate": 5.294132692505284e-07, + "loss": 0.75440633, + "num_input_tokens_seen": 137817660, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.14483643, + "step": 6404, + "time_per_iteration": 2.7204105854034424 + }, + { + "auxiliary_loss_clip": 0.01123441, + "auxiliary_loss_mlp": 0.01035437, + "balance_loss_clip": 1.03927124, + "balance_loss_mlp": 1.02130449, + "epoch": 0.7701557145433776, + "flos": 19242733196160.0, + "grad_norm": 1.8791265091780236, + "language_loss": 0.79233015, + "learning_rate": 5.288854335753861e-07, + "loss": 0.81391895, + "num_input_tokens_seen": 137835920, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.14154053, + "step": 6405, + "time_per_iteration": 2.822817087173462 + }, + { + "auxiliary_loss_clip": 0.01123994, + "auxiliary_loss_mlp": 0.0104006, + "balance_loss_clip": 1.0389806, + "balance_loss_mlp": 1.02492023, + "epoch": 0.7702759574340167, + "flos": 31685744211840.0, + "grad_norm": 1.5525153083851448, + "language_loss": 0.75424302, + "learning_rate": 5.283578210671551e-07, + "loss": 0.77588356, + "num_input_tokens_seen": 137858160, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.15142822, + "step": 6406, + "time_per_iteration": 3.675597667694092 + }, + { + "auxiliary_loss_clip": 0.0112484, + "auxiliary_loss_mlp": 0.01041345, + "balance_loss_clip": 1.04118919, + "balance_loss_mlp": 1.02627718, + "epoch": 0.7703962003246558, + "flos": 16800089644800.0, + "grad_norm": 2.1328913572724306, + "language_loss": 0.76125425, + "learning_rate": 5.278304318058719e-07, + "loss": 0.78291613, + "num_input_tokens_seen": 137876015, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.15075684, + "step": 6407, + "time_per_iteration": 2.731497287750244 + }, + { + "auxiliary_loss_clip": 0.01124077, + "auxiliary_loss_mlp": 0.01035979, + "balance_loss_clip": 1.03821385, + "balance_loss_mlp": 1.02062511, + "epoch": 0.7705164432152949, + "flos": 35736072693120.0, + "grad_norm": 2.001563521070412, + "language_loss": 0.79086411, + "learning_rate": 5.273032658715411e-07, + "loss": 0.81246459, + "num_input_tokens_seen": 137898825, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15356445, + "step": 6408, + "time_per_iteration": 2.8777036666870117 + }, + { + "auxiliary_loss_clip": 0.01124115, + "auxiliary_loss_mlp": 0.01037591, + "balance_loss_clip": 1.03751719, + "balance_loss_mlp": 1.02305961, + "epoch": 0.7706366861059339, + "flos": 23365960329600.0, + "grad_norm": 2.007166834793315, + "language_loss": 0.77010119, + "learning_rate": 5.267763233441347e-07, + "loss": 0.79171824, + "num_input_tokens_seen": 137919455, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.14550781, + "step": 6409, + "time_per_iteration": 2.786112070083618 + }, + { + "auxiliary_loss_clip": 0.01133059, + "auxiliary_loss_mlp": 0.01044667, + "balance_loss_clip": 1.04264522, + "balance_loss_mlp": 1.02738154, + "epoch": 0.7707569289965731, + "flos": 22929897219840.0, + "grad_norm": 2.1349203639628427, + "language_loss": 0.69298291, + "learning_rate": 5.26249604303588e-07, + "loss": 0.71476018, + "num_input_tokens_seen": 137937960, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.1730957, + "step": 6410, + "time_per_iteration": 2.7563436031341553 + }, + { + "auxiliary_loss_clip": 0.01124918, + "auxiliary_loss_mlp": 0.01043978, + "balance_loss_clip": 1.04085612, + "balance_loss_mlp": 1.02875543, + "epoch": 0.7708771718872122, + "flos": 17420661941760.0, + "grad_norm": 2.265969766140575, + "language_loss": 0.78518605, + "learning_rate": 5.257231088298057e-07, + "loss": 0.80687499, + "num_input_tokens_seen": 137956370, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.15234375, + "step": 6411, + "time_per_iteration": 2.744600772857666 + }, + { + "auxiliary_loss_clip": 0.01033594, + "auxiliary_loss_mlp": 0.0099956, + "balance_loss_clip": 1.00582314, + "balance_loss_mlp": 0.9980405, + "epoch": 0.7709974147778512, + "flos": 72241316248320.0, + "grad_norm": 0.7939680599318549, + "language_loss": 0.5393728, + "learning_rate": 5.25196837002655e-07, + "loss": 0.5597043, + "num_input_tokens_seen": 138016080, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01519775, + "step": 6412, + "time_per_iteration": 3.346919059753418 + }, + { + "auxiliary_loss_clip": 0.01130886, + "auxiliary_loss_mlp": 0.01036471, + "balance_loss_clip": 1.04256964, + "balance_loss_mlp": 1.02134323, + "epoch": 0.7711176576684904, + "flos": 39859694876160.0, + "grad_norm": 2.0640311796179, + "language_loss": 0.68173158, + "learning_rate": 5.24670788901971e-07, + "loss": 0.70340514, + "num_input_tokens_seen": 138039170, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.15136719, + "step": 6413, + "time_per_iteration": 3.0194880962371826 + }, + { + "auxiliary_loss_clip": 0.01134037, + "auxiliary_loss_mlp": 0.01044227, + "balance_loss_clip": 1.04441047, + "balance_loss_mlp": 1.02538025, + "epoch": 0.7712379005591294, + "flos": 36976391274240.0, + "grad_norm": 2.2454102276767816, + "language_loss": 0.68550062, + "learning_rate": 5.241449646075557e-07, + "loss": 0.70728326, + "num_input_tokens_seen": 138062395, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.18835449, + "step": 6414, + "time_per_iteration": 2.9186978340148926 + }, + { + "auxiliary_loss_clip": 0.01133614, + "auxiliary_loss_mlp": 0.01042888, + "balance_loss_clip": 1.04515243, + "balance_loss_mlp": 1.02764094, + "epoch": 0.7713581434497685, + "flos": 22776773541120.0, + "grad_norm": 1.9251403404138971, + "language_loss": 0.7247746, + "learning_rate": 5.236193641991762e-07, + "loss": 0.74653965, + "num_input_tokens_seen": 138080325, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.15240479, + "step": 6415, + "time_per_iteration": 2.8428945541381836 + }, + { + "auxiliary_loss_clip": 0.01126558, + "auxiliary_loss_mlp": 0.01038812, + "balance_loss_clip": 1.04152882, + "balance_loss_mlp": 1.02417326, + "epoch": 0.7714783863404077, + "flos": 24097460803200.0, + "grad_norm": 2.0546326464638724, + "language_loss": 0.69877416, + "learning_rate": 5.23093987756565e-07, + "loss": 0.72042787, + "num_input_tokens_seen": 138099020, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.1463623, + "step": 6416, + "time_per_iteration": 2.75124454498291 + }, + { + "auxiliary_loss_clip": 0.01129387, + "auxiliary_loss_mlp": 0.01043779, + "balance_loss_clip": 1.03954697, + "balance_loss_mlp": 1.02493167, + "epoch": 0.7715986292310467, + "flos": 21063655215360.0, + "grad_norm": 2.0441545048962704, + "language_loss": 0.753447, + "learning_rate": 5.225688353594217e-07, + "loss": 0.77517867, + "num_input_tokens_seen": 138118650, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.18835449, + "step": 6417, + "time_per_iteration": 2.8069238662719727 + }, + { + "auxiliary_loss_clip": 0.01126147, + "auxiliary_loss_mlp": 0.01037009, + "balance_loss_clip": 1.04028893, + "balance_loss_mlp": 1.02192855, + "epoch": 0.7717188721216858, + "flos": 20594877793920.0, + "grad_norm": 2.2399271690635403, + "language_loss": 0.7752142, + "learning_rate": 5.220439070874108e-07, + "loss": 0.79684573, + "num_input_tokens_seen": 138137890, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15063477, + "step": 6418, + "time_per_iteration": 2.7520742416381836 + }, + { + "auxiliary_loss_clip": 0.01126807, + "auxiliary_loss_mlp": 0.0104829, + "balance_loss_clip": 1.04283571, + "balance_loss_mlp": 1.03412831, + "epoch": 0.7718391150123249, + "flos": 26250951870720.0, + "grad_norm": 1.5537532141307817, + "language_loss": 0.70914799, + "learning_rate": 5.215192030201652e-07, + "loss": 0.73089898, + "num_input_tokens_seen": 138158880, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14172363, + "step": 6419, + "time_per_iteration": 2.814612865447998 + }, + { + "auxiliary_loss_clip": 0.01125685, + "auxiliary_loss_mlp": 0.01038257, + "balance_loss_clip": 1.04010367, + "balance_loss_mlp": 1.0222888, + "epoch": 0.771959357902964, + "flos": 22049762267520.0, + "grad_norm": 2.8302452762860475, + "language_loss": 0.86267185, + "learning_rate": 5.209947232372798e-07, + "loss": 0.8843112, + "num_input_tokens_seen": 138176370, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15960693, + "step": 6420, + "time_per_iteration": 2.735192060470581 + }, + { + "auxiliary_loss_clip": 0.01127531, + "auxiliary_loss_mlp": 0.01040071, + "balance_loss_clip": 1.04069567, + "balance_loss_mlp": 1.02446604, + "epoch": 0.772079600793603, + "flos": 30446000248320.0, + "grad_norm": 1.6322256646282638, + "language_loss": 0.81294215, + "learning_rate": 5.204704678183196e-07, + "loss": 0.83461815, + "num_input_tokens_seen": 138195105, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.15612793, + "step": 6421, + "time_per_iteration": 2.812494993209839 + }, + { + "auxiliary_loss_clip": 0.01129284, + "auxiliary_loss_mlp": 0.01035072, + "balance_loss_clip": 1.04252613, + "balance_loss_mlp": 1.01890075, + "epoch": 0.7721998436842422, + "flos": 12969857750400.0, + "grad_norm": 1.8169198661449006, + "language_loss": 0.85011148, + "learning_rate": 5.19946436842813e-07, + "loss": 0.871755, + "num_input_tokens_seen": 138212235, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.16156006, + "step": 6422, + "time_per_iteration": 2.8263213634490967 + }, + { + "auxiliary_loss_clip": 0.01124152, + "auxiliary_loss_mlp": 0.01036287, + "balance_loss_clip": 1.0407263, + "balance_loss_mlp": 1.02167749, + "epoch": 0.7723200865748813, + "flos": 32635509678720.0, + "grad_norm": 1.8030380777871944, + "language_loss": 0.68479681, + "learning_rate": 5.194226303902546e-07, + "loss": 0.70640123, + "num_input_tokens_seen": 138231970, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.1461792, + "step": 6423, + "time_per_iteration": 2.7805187702178955 + }, + { + "auxiliary_loss_clip": 0.0112806, + "auxiliary_loss_mlp": 0.0104126, + "balance_loss_clip": 1.04054618, + "balance_loss_mlp": 1.02595329, + "epoch": 0.7724403294655203, + "flos": 21105707063040.0, + "grad_norm": 1.8938412343594748, + "language_loss": 0.70643455, + "learning_rate": 5.188990485401072e-07, + "loss": 0.72812784, + "num_input_tokens_seen": 138251175, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.15319824, + "step": 6424, + "time_per_iteration": 3.6181583404541016 + }, + { + "auxiliary_loss_clip": 0.01130062, + "auxiliary_loss_mlp": 0.01043236, + "balance_loss_clip": 1.0416081, + "balance_loss_mlp": 1.02777457, + "epoch": 0.7725605723561595, + "flos": 22090736707200.0, + "grad_norm": 1.8846473058259516, + "language_loss": 0.86187148, + "learning_rate": 5.183756913717954e-07, + "loss": 0.88360447, + "num_input_tokens_seen": 138270950, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15466309, + "step": 6425, + "time_per_iteration": 2.7454631328582764 + }, + { + "auxiliary_loss_clip": 0.01126646, + "auxiliary_loss_mlp": 0.01045197, + "balance_loss_clip": 1.04113805, + "balance_loss_mlp": 1.02823377, + "epoch": 0.7726808152467985, + "flos": 34495610457600.0, + "grad_norm": 1.72528734294243, + "language_loss": 0.73300076, + "learning_rate": 5.178525589647136e-07, + "loss": 0.75471926, + "num_input_tokens_seen": 138292590, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.1697998, + "step": 6426, + "time_per_iteration": 2.9228055477142334 + }, + { + "auxiliary_loss_clip": 0.01125468, + "auxiliary_loss_mlp": 0.01040927, + "balance_loss_clip": 1.04185772, + "balance_loss_mlp": 1.02560222, + "epoch": 0.7728010581374376, + "flos": 22306344094080.0, + "grad_norm": 1.933886142203562, + "language_loss": 0.78952265, + "learning_rate": 5.173296513982197e-07, + "loss": 0.81118655, + "num_input_tokens_seen": 138311115, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.15301514, + "step": 6427, + "time_per_iteration": 2.7385759353637695 + }, + { + "auxiliary_loss_clip": 0.01131738, + "auxiliary_loss_mlp": 0.01048994, + "balance_loss_clip": 1.04305124, + "balance_loss_mlp": 1.03151739, + "epoch": 0.7729213010280768, + "flos": 27126453968640.0, + "grad_norm": 2.033312919829764, + "language_loss": 0.65023708, + "learning_rate": 5.168069687516398e-07, + "loss": 0.6720444, + "num_input_tokens_seen": 138330885, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.17480469, + "step": 6428, + "time_per_iteration": 2.760526418685913 + }, + { + "auxiliary_loss_clip": 0.01129911, + "auxiliary_loss_mlp": 0.01040163, + "balance_loss_clip": 1.04333627, + "balance_loss_mlp": 1.02482057, + "epoch": 0.7730415439187158, + "flos": 18150223080960.0, + "grad_norm": 1.9738633818219027, + "language_loss": 0.71622562, + "learning_rate": 5.16284511104263e-07, + "loss": 0.73792642, + "num_input_tokens_seen": 138350020, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15338135, + "step": 6429, + "time_per_iteration": 4.482450723648071 + }, + { + "auxiliary_loss_clip": 0.01130957, + "auxiliary_loss_mlp": 0.0103922, + "balance_loss_clip": 1.04158044, + "balance_loss_mlp": 1.02340078, + "epoch": 0.7731617868093549, + "flos": 11947480940160.0, + "grad_norm": 2.518601639226869, + "language_loss": 0.80800152, + "learning_rate": 5.157622785353457e-07, + "loss": 0.82970333, + "num_input_tokens_seen": 138368135, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.15808105, + "step": 6430, + "time_per_iteration": 2.8757612705230713 + }, + { + "auxiliary_loss_clip": 0.01033248, + "auxiliary_loss_mlp": 0.01002492, + "balance_loss_clip": 1.00549006, + "balance_loss_mlp": 1.00102603, + "epoch": 0.7732820296999939, + "flos": 64201027069440.0, + "grad_norm": 0.6442475554798014, + "language_loss": 0.60349631, + "learning_rate": 5.152402711241113e-07, + "loss": 0.62385368, + "num_input_tokens_seen": 138436040, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01464844, + "step": 6431, + "time_per_iteration": 3.368699550628662 + }, + { + "auxiliary_loss_clip": 0.01123298, + "auxiliary_loss_mlp": 0.01037885, + "balance_loss_clip": 1.03968871, + "balance_loss_mlp": 1.02384758, + "epoch": 0.7734022725906331, + "flos": 25302191984640.0, + "grad_norm": 1.709738963118906, + "language_loss": 0.82622463, + "learning_rate": 5.147184889497465e-07, + "loss": 0.84783649, + "num_input_tokens_seen": 138455510, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14038086, + "step": 6432, + "time_per_iteration": 2.8023250102996826 + }, + { + "auxiliary_loss_clip": 0.01128378, + "auxiliary_loss_mlp": 0.01042209, + "balance_loss_clip": 1.04070902, + "balance_loss_mlp": 1.02584136, + "epoch": 0.7735225154812722, + "flos": 17347440067200.0, + "grad_norm": 1.9561291687250524, + "language_loss": 0.79681599, + "learning_rate": 5.141969320914072e-07, + "loss": 0.8185218, + "num_input_tokens_seen": 138473015, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16363525, + "step": 6433, + "time_per_iteration": 2.756469964981079 + }, + { + "auxiliary_loss_clip": 0.01133032, + "auxiliary_loss_mlp": 0.01044618, + "balance_loss_clip": 1.04256344, + "balance_loss_mlp": 1.02642632, + "epoch": 0.7736427583719112, + "flos": 32630086725120.0, + "grad_norm": 3.4678359502160006, + "language_loss": 0.62252223, + "learning_rate": 5.136756006282113e-07, + "loss": 0.64429873, + "num_input_tokens_seen": 138491680, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.18188477, + "step": 6434, + "time_per_iteration": 2.872629404067993 + }, + { + "auxiliary_loss_clip": 0.01129385, + "auxiliary_loss_mlp": 0.01047448, + "balance_loss_clip": 1.04099965, + "balance_loss_mlp": 1.0333997, + "epoch": 0.7737630012625504, + "flos": 19860073269120.0, + "grad_norm": 2.2726713993481127, + "language_loss": 0.84983087, + "learning_rate": 5.131544946392446e-07, + "loss": 0.87159914, + "num_input_tokens_seen": 138506960, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.14050293, + "step": 6435, + "time_per_iteration": 3.6744632720947266 + }, + { + "auxiliary_loss_clip": 0.01129765, + "auxiliary_loss_mlp": 0.01037414, + "balance_loss_clip": 1.04179585, + "balance_loss_mlp": 1.02020013, + "epoch": 0.7738832441531894, + "flos": 36022639397760.0, + "grad_norm": 2.1048646179730532, + "language_loss": 0.64501852, + "learning_rate": 5.126336142035592e-07, + "loss": 0.66669029, + "num_input_tokens_seen": 138526995, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.17224121, + "step": 6436, + "time_per_iteration": 2.9143083095550537 + }, + { + "auxiliary_loss_clip": 0.01126863, + "auxiliary_loss_mlp": 0.01033377, + "balance_loss_clip": 1.04053783, + "balance_loss_mlp": 1.01764119, + "epoch": 0.7740034870438285, + "flos": 13405274415360.0, + "grad_norm": 2.1756459446002223, + "language_loss": 0.71972483, + "learning_rate": 5.121129594001721e-07, + "loss": 0.74132723, + "num_input_tokens_seen": 138541260, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15734863, + "step": 6437, + "time_per_iteration": 2.8084821701049805 + }, + { + "auxiliary_loss_clip": 0.01128301, + "auxiliary_loss_mlp": 0.01041024, + "balance_loss_clip": 1.04243445, + "balance_loss_mlp": 1.02437067, + "epoch": 0.7741237299344677, + "flos": 22086714384000.0, + "grad_norm": 1.560310134097879, + "language_loss": 0.80937696, + "learning_rate": 5.115925303080661e-07, + "loss": 0.83107024, + "num_input_tokens_seen": 138560970, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.16650391, + "step": 6438, + "time_per_iteration": 2.791743040084839 + }, + { + "auxiliary_loss_clip": 0.01122502, + "auxiliary_loss_mlp": 0.01036489, + "balance_loss_clip": 1.03855324, + "balance_loss_mlp": 1.02129602, + "epoch": 0.7742439728251067, + "flos": 19864777950720.0, + "grad_norm": 1.9208117180465445, + "language_loss": 0.78968573, + "learning_rate": 5.110723270061899e-07, + "loss": 0.8112756, + "num_input_tokens_seen": 138577460, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.15185547, + "step": 6439, + "time_per_iteration": 2.8087680339813232 + }, + { + "auxiliary_loss_clip": 0.01127854, + "auxiliary_loss_mlp": 0.01037672, + "balance_loss_clip": 1.04104817, + "balance_loss_mlp": 1.02097678, + "epoch": 0.7743642157157458, + "flos": 16690167048960.0, + "grad_norm": 1.890601903035611, + "language_loss": 0.79358685, + "learning_rate": 5.105523495734572e-07, + "loss": 0.81524205, + "num_input_tokens_seen": 138594860, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.16680908, + "step": 6440, + "time_per_iteration": 2.764667510986328 + }, + { + "auxiliary_loss_clip": 0.01125906, + "auxiliary_loss_mlp": 0.01039863, + "balance_loss_clip": 1.0393796, + "balance_loss_mlp": 1.02503371, + "epoch": 0.7744844586063849, + "flos": 20304360593280.0, + "grad_norm": 1.5786633988146486, + "language_loss": 0.75449181, + "learning_rate": 5.100325980887499e-07, + "loss": 0.77614951, + "num_input_tokens_seen": 138614785, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.14819336, + "step": 6441, + "time_per_iteration": 2.8470382690429688 + }, + { + "auxiliary_loss_clip": 0.0112574, + "auxiliary_loss_mlp": 0.01043739, + "balance_loss_clip": 1.04028058, + "balance_loss_mlp": 1.02863526, + "epoch": 0.774604701497024, + "flos": 22966705681920.0, + "grad_norm": 1.9140807855411548, + "language_loss": 0.8339445, + "learning_rate": 5.095130726309116e-07, + "loss": 0.85563934, + "num_input_tokens_seen": 138634960, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15100098, + "step": 6442, + "time_per_iteration": 2.788609027862549 + }, + { + "auxiliary_loss_clip": 0.01032842, + "auxiliary_loss_mlp": 0.01000289, + "balance_loss_clip": 1.00507402, + "balance_loss_mlp": 0.99872702, + "epoch": 0.774724944387663, + "flos": 60288523073280.0, + "grad_norm": 0.796685539851808, + "language_loss": 0.58991015, + "learning_rate": 5.089937732787559e-07, + "loss": 0.61024141, + "num_input_tokens_seen": 138699520, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.015625, + "step": 6443, + "time_per_iteration": 3.300610303878784 + }, + { + "auxiliary_loss_clip": 0.01130318, + "auxiliary_loss_mlp": 0.01041445, + "balance_loss_clip": 1.04238188, + "balance_loss_mlp": 1.02508903, + "epoch": 0.7748451872783022, + "flos": 26761026954240.0, + "grad_norm": 2.3405131910529904, + "language_loss": 0.6640321, + "learning_rate": 5.084747001110592e-07, + "loss": 0.68574971, + "num_input_tokens_seen": 138719145, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.16369629, + "step": 6444, + "time_per_iteration": 2.9650020599365234 + }, + { + "auxiliary_loss_clip": 0.01123419, + "auxiliary_loss_mlp": 0.01035432, + "balance_loss_clip": 1.04014194, + "balance_loss_mlp": 1.02072775, + "epoch": 0.7749654301689413, + "flos": 30338627518080.0, + "grad_norm": 1.6217773417269776, + "language_loss": 0.70087683, + "learning_rate": 5.07955853206564e-07, + "loss": 0.7224654, + "num_input_tokens_seen": 138743850, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.14703369, + "step": 6445, + "time_per_iteration": 2.815507173538208 + }, + { + "auxiliary_loss_clip": 0.01126304, + "auxiliary_loss_mlp": 0.01044848, + "balance_loss_clip": 1.04085696, + "balance_loss_mlp": 1.02887368, + "epoch": 0.7750856730595803, + "flos": 43179851687040.0, + "grad_norm": 1.6083574510776548, + "language_loss": 0.70763445, + "learning_rate": 5.074372326439807e-07, + "loss": 0.72934592, + "num_input_tokens_seen": 138766860, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15979004, + "step": 6446, + "time_per_iteration": 2.9218411445617676 + }, + { + "auxiliary_loss_clip": 0.01127604, + "auxiliary_loss_mlp": 0.01037975, + "balance_loss_clip": 1.03864813, + "balance_loss_mlp": 1.02120245, + "epoch": 0.7752059159502195, + "flos": 17640040256640.0, + "grad_norm": 5.035901091393798, + "language_loss": 0.74031198, + "learning_rate": 5.069188385019814e-07, + "loss": 0.76196778, + "num_input_tokens_seen": 138784560, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.16772461, + "step": 6447, + "time_per_iteration": 2.839946985244751 + }, + { + "auxiliary_loss_clip": 0.01127931, + "auxiliary_loss_mlp": 0.01038916, + "balance_loss_clip": 1.04028082, + "balance_loss_mlp": 1.0226562, + "epoch": 0.7753261588408585, + "flos": 12677688524160.0, + "grad_norm": 2.613637064181446, + "language_loss": 0.61258209, + "learning_rate": 5.064006708592077e-07, + "loss": 0.63425052, + "num_input_tokens_seen": 138800805, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.16247559, + "step": 6448, + "time_per_iteration": 2.7733659744262695 + }, + { + "auxiliary_loss_clip": 0.01124936, + "auxiliary_loss_mlp": 0.01035531, + "balance_loss_clip": 1.04163384, + "balance_loss_mlp": 1.02146459, + "epoch": 0.7754464017314976, + "flos": 16690741666560.0, + "grad_norm": 2.1566063409644536, + "language_loss": 0.75854576, + "learning_rate": 5.058827297942641e-07, + "loss": 0.78015047, + "num_input_tokens_seen": 138815910, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.140625, + "step": 6449, + "time_per_iteration": 2.843580722808838 + }, + { + "auxiliary_loss_clip": 0.01122973, + "auxiliary_loss_mlp": 0.01040906, + "balance_loss_clip": 1.03892207, + "balance_loss_mlp": 1.02600503, + "epoch": 0.7755666446221368, + "flos": 19718944732800.0, + "grad_norm": 2.008528801414583, + "language_loss": 0.75393748, + "learning_rate": 5.053650153857237e-07, + "loss": 0.77557629, + "num_input_tokens_seen": 138834920, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14886475, + "step": 6450, + "time_per_iteration": 2.751950979232788 + }, + { + "auxiliary_loss_clip": 0.01125099, + "auxiliary_loss_mlp": 0.01043749, + "balance_loss_clip": 1.04076314, + "balance_loss_mlp": 1.02818012, + "epoch": 0.7756868875127758, + "flos": 18693623007360.0, + "grad_norm": 1.6105258287208248, + "language_loss": 0.697083, + "learning_rate": 5.048475277121214e-07, + "loss": 0.71877146, + "num_input_tokens_seen": 138852135, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.15551758, + "step": 6451, + "time_per_iteration": 2.7906110286712646 + }, + { + "auxiliary_loss_clip": 0.01127624, + "auxiliary_loss_mlp": 0.01035198, + "balance_loss_clip": 1.04065919, + "balance_loss_mlp": 1.01881909, + "epoch": 0.7758071304034149, + "flos": 28404191543040.0, + "grad_norm": 1.5611446655398462, + "language_loss": 0.77087605, + "learning_rate": 5.043302668519598e-07, + "loss": 0.79250425, + "num_input_tokens_seen": 138871470, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.16381836, + "step": 6452, + "time_per_iteration": 2.82314133644104 + }, + { + "auxiliary_loss_clip": 0.01123755, + "auxiliary_loss_mlp": 0.01031444, + "balance_loss_clip": 1.03829587, + "balance_loss_mlp": 1.01726985, + "epoch": 0.775927373294054, + "flos": 20595344670720.0, + "grad_norm": 1.7945346095280144, + "language_loss": 0.72028673, + "learning_rate": 5.038132328837079e-07, + "loss": 0.74183869, + "num_input_tokens_seen": 138889860, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.14172363, + "step": 6453, + "time_per_iteration": 3.5870871543884277 + }, + { + "auxiliary_loss_clip": 0.01125717, + "auxiliary_loss_mlp": 0.01033319, + "balance_loss_clip": 1.04096353, + "balance_loss_mlp": 1.01820302, + "epoch": 0.7760476161846931, + "flos": 22526368853760.0, + "grad_norm": 2.0373762727428253, + "language_loss": 0.74006462, + "learning_rate": 5.032964258857993e-07, + "loss": 0.76165491, + "num_input_tokens_seen": 138909955, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.15112305, + "step": 6454, + "time_per_iteration": 2.7699170112609863 + }, + { + "auxiliary_loss_clip": 0.01125119, + "auxiliary_loss_mlp": 0.01035047, + "balance_loss_clip": 1.03910422, + "balance_loss_mlp": 1.01953816, + "epoch": 0.7761678590753321, + "flos": 48651488403840.0, + "grad_norm": 1.4560012426617173, + "language_loss": 0.67988217, + "learning_rate": 5.027798459366329e-07, + "loss": 0.70148385, + "num_input_tokens_seen": 138935320, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15515137, + "step": 6455, + "time_per_iteration": 2.9906575679779053 + }, + { + "auxiliary_loss_clip": 0.01125713, + "auxiliary_loss_mlp": 0.01041584, + "balance_loss_clip": 1.04009092, + "balance_loss_mlp": 1.02514505, + "epoch": 0.7762881019659713, + "flos": 26177047637760.0, + "grad_norm": 1.4349961793259847, + "language_loss": 0.63589251, + "learning_rate": 5.02263493114573e-07, + "loss": 0.65756547, + "num_input_tokens_seen": 138957115, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.16442871, + "step": 6456, + "time_per_iteration": 2.8194997310638428 + }, + { + "auxiliary_loss_clip": 0.0112368, + "auxiliary_loss_mlp": 0.01043859, + "balance_loss_clip": 1.03960323, + "balance_loss_mlp": 1.02818882, + "epoch": 0.7764083448566104, + "flos": 20588341518720.0, + "grad_norm": 2.5705845405075016, + "language_loss": 0.76980591, + "learning_rate": 5.017473674979502e-07, + "loss": 0.79148126, + "num_input_tokens_seen": 138973140, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15667725, + "step": 6457, + "time_per_iteration": 3.592796564102173 + }, + { + "auxiliary_loss_clip": 0.01032425, + "auxiliary_loss_mlp": 0.00999049, + "balance_loss_clip": 1.00479269, + "balance_loss_mlp": 0.99741596, + "epoch": 0.7765285877472494, + "flos": 67293078560640.0, + "grad_norm": 0.7393029739274081, + "language_loss": 0.58282244, + "learning_rate": 5.01231469165061e-07, + "loss": 0.60313714, + "num_input_tokens_seen": 139028965, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01635742, + "step": 6458, + "time_per_iteration": 4.020467042922974 + }, + { + "auxiliary_loss_clip": 0.01032139, + "auxiliary_loss_mlp": 0.01002725, + "balance_loss_clip": 1.00448012, + "balance_loss_mlp": 1.00113916, + "epoch": 0.7766488306378886, + "flos": 61344476121600.0, + "grad_norm": 0.824686559225413, + "language_loss": 0.56940538, + "learning_rate": 5.007157981941663e-07, + "loss": 0.58975405, + "num_input_tokens_seen": 139094325, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01586914, + "step": 6459, + "time_per_iteration": 3.422297239303589 + }, + { + "auxiliary_loss_clip": 0.01032646, + "auxiliary_loss_mlp": 0.01004059, + "balance_loss_clip": 1.005018, + "balance_loss_mlp": 1.00243759, + "epoch": 0.7767690735285276, + "flos": 62946199393920.0, + "grad_norm": 0.8800587628810256, + "language_loss": 0.67423987, + "learning_rate": 5.002003546634928e-07, + "loss": 0.6946069, + "num_input_tokens_seen": 139150425, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01623535, + "step": 6460, + "time_per_iteration": 3.286604881286621 + }, + { + "auxiliary_loss_clip": 0.01124762, + "auxiliary_loss_mlp": 0.01037797, + "balance_loss_clip": 1.04176593, + "balance_loss_mlp": 1.02259779, + "epoch": 0.7768893164191667, + "flos": 20886400575360.0, + "grad_norm": 1.7889157128669864, + "language_loss": 0.76110142, + "learning_rate": 4.996851386512331e-07, + "loss": 0.782727, + "num_input_tokens_seen": 139169130, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.15216064, + "step": 6461, + "time_per_iteration": 2.813174247741699 + }, + { + "auxiliary_loss_clip": 0.01132072, + "auxiliary_loss_mlp": 0.01041767, + "balance_loss_clip": 1.04203212, + "balance_loss_mlp": 1.0248034, + "epoch": 0.7770095593098058, + "flos": 20704584908160.0, + "grad_norm": 1.7118838807871, + "language_loss": 0.83030671, + "learning_rate": 4.991701502355444e-07, + "loss": 0.85204506, + "num_input_tokens_seen": 139189595, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.16955566, + "step": 6462, + "time_per_iteration": 2.8088676929473877 + }, + { + "auxiliary_loss_clip": 0.01123819, + "auxiliary_loss_mlp": 0.01037126, + "balance_loss_clip": 1.03846681, + "balance_loss_mlp": 1.02254713, + "epoch": 0.7771298022004449, + "flos": 24717709877760.0, + "grad_norm": 1.5092735311025287, + "language_loss": 0.76124293, + "learning_rate": 4.986553894945518e-07, + "loss": 0.78285241, + "num_input_tokens_seen": 139210805, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.14575195, + "step": 6463, + "time_per_iteration": 2.777623414993286 + }, + { + "auxiliary_loss_clip": 0.01123851, + "auxiliary_loss_mlp": 0.01037464, + "balance_loss_clip": 1.03931475, + "balance_loss_mlp": 1.02246749, + "epoch": 0.777250045091084, + "flos": 25009232659200.0, + "grad_norm": 1.8745498679905588, + "language_loss": 0.85979187, + "learning_rate": 4.981408565063416e-07, + "loss": 0.881405, + "num_input_tokens_seen": 139230750, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14996338, + "step": 6464, + "time_per_iteration": 3.651339292526245 + }, + { + "auxiliary_loss_clip": 0.01129539, + "auxiliary_loss_mlp": 0.01040287, + "balance_loss_clip": 1.04179358, + "balance_loss_mlp": 1.02433681, + "epoch": 0.777370287981723, + "flos": 20119887319680.0, + "grad_norm": 1.9137739106466543, + "language_loss": 0.76200175, + "learning_rate": 4.976265513489701e-07, + "loss": 0.78370005, + "num_input_tokens_seen": 139250720, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15979004, + "step": 6465, + "time_per_iteration": 2.789855480194092 + }, + { + "auxiliary_loss_clip": 0.01125261, + "auxiliary_loss_mlp": 0.01038862, + "balance_loss_clip": 1.03912163, + "balance_loss_mlp": 1.02356744, + "epoch": 0.7774905308723622, + "flos": 21718809331200.0, + "grad_norm": 2.7408143738364568, + "language_loss": 0.80343699, + "learning_rate": 4.971124741004562e-07, + "loss": 0.82507825, + "num_input_tokens_seen": 139269720, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15283203, + "step": 6466, + "time_per_iteration": 2.777822732925415 + }, + { + "auxiliary_loss_clip": 0.01123824, + "auxiliary_loss_mlp": 0.01037492, + "balance_loss_clip": 1.03875017, + "balance_loss_mlp": 1.02163768, + "epoch": 0.7776107737630013, + "flos": 16034115093120.0, + "grad_norm": 1.9709404380398752, + "language_loss": 0.76580226, + "learning_rate": 4.965986248387846e-07, + "loss": 0.78741544, + "num_input_tokens_seen": 139288035, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.15856934, + "step": 6467, + "time_per_iteration": 2.7534618377685547 + }, + { + "auxiliary_loss_clip": 0.01125263, + "auxiliary_loss_mlp": 0.01039023, + "balance_loss_clip": 1.038975, + "balance_loss_mlp": 1.02468252, + "epoch": 0.7777310166536403, + "flos": 24790895838720.0, + "grad_norm": 1.49737630807132, + "language_loss": 0.77236795, + "learning_rate": 4.960850036419073e-07, + "loss": 0.79401088, + "num_input_tokens_seen": 139307135, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.14355469, + "step": 6468, + "time_per_iteration": 2.774273157119751 + }, + { + "auxiliary_loss_clip": 0.01127452, + "auxiliary_loss_mlp": 0.01036345, + "balance_loss_clip": 1.04009581, + "balance_loss_mlp": 1.02096725, + "epoch": 0.7778512595442795, + "flos": 17272530253440.0, + "grad_norm": 1.6960598454401152, + "language_loss": 0.78371799, + "learning_rate": 4.955716105877378e-07, + "loss": 0.80535597, + "num_input_tokens_seen": 139325905, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.15362549, + "step": 6469, + "time_per_iteration": 2.8395559787750244 + }, + { + "auxiliary_loss_clip": 0.01127203, + "auxiliary_loss_mlp": 0.01035167, + "balance_loss_clip": 1.04048419, + "balance_loss_mlp": 1.01976466, + "epoch": 0.7779715024349185, + "flos": 17748418567680.0, + "grad_norm": 1.6393485814846622, + "language_loss": 0.83177888, + "learning_rate": 4.950584457541598e-07, + "loss": 0.85340255, + "num_input_tokens_seen": 139344370, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15393066, + "step": 6470, + "time_per_iteration": 2.8453564643859863 + }, + { + "auxiliary_loss_clip": 0.01125329, + "auxiliary_loss_mlp": 0.01042789, + "balance_loss_clip": 1.0397495, + "balance_loss_mlp": 1.02714324, + "epoch": 0.7780917453255576, + "flos": 24316875031680.0, + "grad_norm": 1.4552358457141494, + "language_loss": 0.8190428, + "learning_rate": 4.945455092190183e-07, + "loss": 0.84072399, + "num_input_tokens_seen": 139365625, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15643311, + "step": 6471, + "time_per_iteration": 2.8144164085388184 + }, + { + "auxiliary_loss_clip": 0.01032766, + "auxiliary_loss_mlp": 0.01002891, + "balance_loss_clip": 1.00491095, + "balance_loss_mlp": 1.00137079, + "epoch": 0.7782119882161967, + "flos": 56364601530240.0, + "grad_norm": 0.6836656058886011, + "language_loss": 0.56006676, + "learning_rate": 4.940328010601271e-07, + "loss": 0.5804233, + "num_input_tokens_seen": 139430540, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01519775, + "step": 6472, + "time_per_iteration": 3.3358237743377686 + }, + { + "auxiliary_loss_clip": 0.01132466, + "auxiliary_loss_mlp": 0.01047148, + "balance_loss_clip": 1.04337764, + "balance_loss_mlp": 1.02989852, + "epoch": 0.7783322311068358, + "flos": 46789986994560.0, + "grad_norm": 5.698472063586486, + "language_loss": 0.76999485, + "learning_rate": 4.935203213552621e-07, + "loss": 0.79179096, + "num_input_tokens_seen": 139454280, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.17248535, + "step": 6473, + "time_per_iteration": 3.011540174484253 + }, + { + "auxiliary_loss_clip": 0.01127921, + "auxiliary_loss_mlp": 0.01036239, + "balance_loss_clip": 1.04063821, + "balance_loss_mlp": 1.01965714, + "epoch": 0.7784524739974749, + "flos": 19057864872960.0, + "grad_norm": 2.0955099276219906, + "language_loss": 0.66915822, + "learning_rate": 4.930080701821662e-07, + "loss": 0.69079983, + "num_input_tokens_seen": 139471745, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.16589355, + "step": 6474, + "time_per_iteration": 2.8401498794555664 + }, + { + "auxiliary_loss_clip": 0.01125135, + "auxiliary_loss_mlp": 0.01040123, + "balance_loss_clip": 1.03874683, + "balance_loss_mlp": 1.02519858, + "epoch": 0.778572716888114, + "flos": 24791111320320.0, + "grad_norm": 1.8970577547065044, + "language_loss": 0.77331305, + "learning_rate": 4.92496047618548e-07, + "loss": 0.79496562, + "num_input_tokens_seen": 139491505, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.14929199, + "step": 6475, + "time_per_iteration": 2.8945393562316895 + }, + { + "auxiliary_loss_clip": 0.01128849, + "auxiliary_loss_mlp": 0.01043714, + "balance_loss_clip": 1.04235506, + "balance_loss_mlp": 1.02707255, + "epoch": 0.7786929597787531, + "flos": 20078086867200.0, + "grad_norm": 1.6603765835341355, + "language_loss": 0.77920377, + "learning_rate": 4.919842537420811e-07, + "loss": 0.80092943, + "num_input_tokens_seen": 139508620, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.16650391, + "step": 6476, + "time_per_iteration": 2.8079440593719482 + }, + { + "auxiliary_loss_clip": 0.01123917, + "auxiliary_loss_mlp": 0.01037086, + "balance_loss_clip": 1.03991616, + "balance_loss_mlp": 1.02225637, + "epoch": 0.7788132026693921, + "flos": 21872220318720.0, + "grad_norm": 1.6167928677030645, + "language_loss": 0.79110432, + "learning_rate": 4.91472688630404e-07, + "loss": 0.81271434, + "num_input_tokens_seen": 139529360, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.14825439, + "step": 6477, + "time_per_iteration": 2.8625893592834473 + }, + { + "auxiliary_loss_clip": 0.01119476, + "auxiliary_loss_mlp": 0.01031555, + "balance_loss_clip": 1.03701615, + "balance_loss_mlp": 1.01653481, + "epoch": 0.7789334455600313, + "flos": 11181937351680.0, + "grad_norm": 1.6642148104502401, + "language_loss": 0.7416808, + "learning_rate": 4.909613523611202e-07, + "loss": 0.7631911, + "num_input_tokens_seen": 139546240, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.15026855, + "step": 6478, + "time_per_iteration": 2.753892183303833 + }, + { + "auxiliary_loss_clip": 0.01126363, + "auxiliary_loss_mlp": 0.01042568, + "balance_loss_clip": 1.03933442, + "balance_loss_mlp": 1.02596176, + "epoch": 0.7790536884506704, + "flos": 28695427015680.0, + "grad_norm": 2.4542046578209145, + "language_loss": 0.74726427, + "learning_rate": 4.904502450117991e-07, + "loss": 0.76895356, + "num_input_tokens_seen": 139567200, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.16601562, + "step": 6479, + "time_per_iteration": 2.834089517593384 + }, + { + "auxiliary_loss_clip": 0.0112655, + "auxiliary_loss_mlp": 0.01040285, + "balance_loss_clip": 1.03955865, + "balance_loss_mlp": 1.0249548, + "epoch": 0.7791739313413094, + "flos": 11072302064640.0, + "grad_norm": 2.498761706355337, + "language_loss": 0.72354186, + "learning_rate": 4.899393666599762e-07, + "loss": 0.74521023, + "num_input_tokens_seen": 139583775, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.15332031, + "step": 6480, + "time_per_iteration": 2.7922523021698 + }, + { + "auxiliary_loss_clip": 0.01122323, + "auxiliary_loss_mlp": 0.01037768, + "balance_loss_clip": 1.03765559, + "balance_loss_mlp": 1.02287304, + "epoch": 0.7792941742319486, + "flos": 14679276975360.0, + "grad_norm": 2.178566911933372, + "language_loss": 0.72581601, + "learning_rate": 4.894287173831506e-07, + "loss": 0.74741691, + "num_input_tokens_seen": 139599735, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14898682, + "step": 6481, + "time_per_iteration": 2.8095943927764893 + }, + { + "auxiliary_loss_clip": 0.01125593, + "auxiliary_loss_mlp": 0.01032614, + "balance_loss_clip": 1.03756618, + "balance_loss_mlp": 1.01669943, + "epoch": 0.7794144171225876, + "flos": 23258874908160.0, + "grad_norm": 1.9212381874737436, + "language_loss": 0.84542829, + "learning_rate": 4.889182972587877e-07, + "loss": 0.86701035, + "num_input_tokens_seen": 139619030, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15905762, + "step": 6482, + "time_per_iteration": 3.6065585613250732 + }, + { + "auxiliary_loss_clip": 0.01127702, + "auxiliary_loss_mlp": 0.01041045, + "balance_loss_clip": 1.04357362, + "balance_loss_mlp": 1.02577472, + "epoch": 0.7795346600132267, + "flos": 21507080613120.0, + "grad_norm": 1.8039724154965844, + "language_loss": 0.66489017, + "learning_rate": 4.884081063643177e-07, + "loss": 0.68657762, + "num_input_tokens_seen": 139637690, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.15270996, + "step": 6483, + "time_per_iteration": 2.7881221771240234 + }, + { + "auxiliary_loss_clip": 0.01032359, + "auxiliary_loss_mlp": 0.0100004, + "balance_loss_clip": 1.00480688, + "balance_loss_mlp": 0.99847871, + "epoch": 0.7796549029038659, + "flos": 70052273694720.0, + "grad_norm": 0.850977768053145, + "language_loss": 0.5248279, + "learning_rate": 4.878981447771353e-07, + "loss": 0.54515189, + "num_input_tokens_seen": 139692070, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.015625, + "step": 6484, + "time_per_iteration": 3.365339994430542 + }, + { + "auxiliary_loss_clip": 0.01133501, + "auxiliary_loss_mlp": 0.01042949, + "balance_loss_clip": 1.04424024, + "balance_loss_mlp": 1.02731442, + "epoch": 0.7797751457945049, + "flos": 23989405714560.0, + "grad_norm": 1.6962879796856085, + "language_loss": 0.73191977, + "learning_rate": 4.873884125746035e-07, + "loss": 0.75368416, + "num_input_tokens_seen": 139713745, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.15643311, + "step": 6485, + "time_per_iteration": 2.8142876625061035 + }, + { + "auxiliary_loss_clip": 0.01125242, + "auxiliary_loss_mlp": 0.01031999, + "balance_loss_clip": 1.04001331, + "balance_loss_mlp": 1.01657367, + "epoch": 0.779895388685144, + "flos": 22674751937280.0, + "grad_norm": 3.2982973054798066, + "language_loss": 0.71961617, + "learning_rate": 4.868789098340456e-07, + "loss": 0.74118859, + "num_input_tokens_seen": 139731650, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.15429688, + "step": 6486, + "time_per_iteration": 3.62709379196167 + }, + { + "auxiliary_loss_clip": 0.01129759, + "auxiliary_loss_mlp": 0.01040719, + "balance_loss_clip": 1.04187047, + "balance_loss_mlp": 1.02514982, + "epoch": 0.7800156315757831, + "flos": 23768698596480.0, + "grad_norm": 2.130211327427228, + "language_loss": 0.72892165, + "learning_rate": 4.863696366327543e-07, + "loss": 0.75062639, + "num_input_tokens_seen": 139750820, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.15563965, + "step": 6487, + "time_per_iteration": 3.529120683670044 + }, + { + "auxiliary_loss_clip": 0.01123357, + "auxiliary_loss_mlp": 0.01037699, + "balance_loss_clip": 1.03831363, + "balance_loss_mlp": 1.02138555, + "epoch": 0.7801358744664222, + "flos": 26429714881920.0, + "grad_norm": 1.644402402832771, + "language_loss": 0.78036177, + "learning_rate": 4.85860593047986e-07, + "loss": 0.80197239, + "num_input_tokens_seen": 139770885, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.16314697, + "step": 6488, + "time_per_iteration": 2.754500389099121 + }, + { + "auxiliary_loss_clip": 0.01124839, + "auxiliary_loss_mlp": 0.01037252, + "balance_loss_clip": 1.03957677, + "balance_loss_mlp": 1.02185059, + "epoch": 0.7802561173570612, + "flos": 26322162583680.0, + "grad_norm": 1.55693416436769, + "language_loss": 0.74770069, + "learning_rate": 4.853517791569613e-07, + "loss": 0.76932156, + "num_input_tokens_seen": 139793065, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.15380859, + "step": 6489, + "time_per_iteration": 2.798114538192749 + }, + { + "auxiliary_loss_clip": 0.01132164, + "auxiliary_loss_mlp": 0.0103617, + "balance_loss_clip": 1.04203033, + "balance_loss_mlp": 1.01974344, + "epoch": 0.7803763602477004, + "flos": 40333751596800.0, + "grad_norm": 1.9134949862917399, + "language_loss": 0.66089427, + "learning_rate": 4.848431950368684e-07, + "loss": 0.68257761, + "num_input_tokens_seen": 139815625, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.16430664, + "step": 6490, + "time_per_iteration": 2.876796245574951 + }, + { + "auxiliary_loss_clip": 0.01032383, + "auxiliary_loss_mlp": 0.01004168, + "balance_loss_clip": 1.00485897, + "balance_loss_mlp": 1.00259423, + "epoch": 0.7804966031383395, + "flos": 67001448038400.0, + "grad_norm": 0.7035902702521818, + "language_loss": 0.55694556, + "learning_rate": 4.843348407648569e-07, + "loss": 0.57731104, + "num_input_tokens_seen": 139876905, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01574707, + "step": 6491, + "time_per_iteration": 3.2533679008483887 + }, + { + "auxiliary_loss_clip": 0.01124879, + "auxiliary_loss_mlp": 0.01039898, + "balance_loss_clip": 1.03737664, + "balance_loss_mlp": 1.02345943, + "epoch": 0.7806168460289785, + "flos": 17740733057280.0, + "grad_norm": 2.4311713176916667, + "language_loss": 0.83132786, + "learning_rate": 4.838267164180457e-07, + "loss": 0.85297561, + "num_input_tokens_seen": 139892575, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.16455078, + "step": 6492, + "time_per_iteration": 2.767869710922241 + }, + { + "auxiliary_loss_clip": 0.0112838, + "auxiliary_loss_mlp": 0.01038618, + "balance_loss_clip": 1.03985524, + "balance_loss_mlp": 1.02267933, + "epoch": 0.7807370889196176, + "flos": 23946240545280.0, + "grad_norm": 2.113370630292613, + "language_loss": 0.8376978, + "learning_rate": 4.833188220735156e-07, + "loss": 0.85936785, + "num_input_tokens_seen": 139912245, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15936279, + "step": 6493, + "time_per_iteration": 3.7581217288970947 + }, + { + "auxiliary_loss_clip": 0.01121567, + "auxiliary_loss_mlp": 0.01031728, + "balance_loss_clip": 1.03890657, + "balance_loss_mlp": 1.01714921, + "epoch": 0.7808573318102567, + "flos": 18989024457600.0, + "grad_norm": 2.4440024564023597, + "language_loss": 0.74792516, + "learning_rate": 4.828111578083152e-07, + "loss": 0.76945817, + "num_input_tokens_seen": 139929150, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.14575195, + "step": 6494, + "time_per_iteration": 2.86617112159729 + }, + { + "auxiliary_loss_clip": 0.01127952, + "auxiliary_loss_mlp": 0.01038041, + "balance_loss_clip": 1.04194391, + "balance_loss_mlp": 1.02310967, + "epoch": 0.7809775747008958, + "flos": 23980750536960.0, + "grad_norm": 2.297524825097001, + "language_loss": 0.81367892, + "learning_rate": 4.823037236994556e-07, + "loss": 0.83533883, + "num_input_tokens_seen": 139947315, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.14935303, + "step": 6495, + "time_per_iteration": 2.919330596923828 + }, + { + "auxiliary_loss_clip": 0.01032171, + "auxiliary_loss_mlp": 0.01002325, + "balance_loss_clip": 1.00467861, + "balance_loss_mlp": 1.00088239, + "epoch": 0.7810978175915348, + "flos": 68535875180160.0, + "grad_norm": 0.7192724743278414, + "language_loss": 0.56339157, + "learning_rate": 4.817965198239136e-07, + "loss": 0.58373648, + "num_input_tokens_seen": 140013775, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.0144043, + "step": 6496, + "time_per_iteration": 3.2681350708007812 + }, + { + "auxiliary_loss_clip": 0.01127731, + "auxiliary_loss_mlp": 0.01041177, + "balance_loss_clip": 1.04070747, + "balance_loss_mlp": 1.02626419, + "epoch": 0.781218060482174, + "flos": 19642131498240.0, + "grad_norm": 2.1307697768015066, + "language_loss": 0.74590582, + "learning_rate": 4.812895462586331e-07, + "loss": 0.76759493, + "num_input_tokens_seen": 140031600, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.14880371, + "step": 6497, + "time_per_iteration": 2.7760887145996094 + }, + { + "auxiliary_loss_clip": 0.01124346, + "auxiliary_loss_mlp": 0.01037977, + "balance_loss_clip": 1.03952193, + "balance_loss_mlp": 1.02301669, + "epoch": 0.7813383033728131, + "flos": 25627865621760.0, + "grad_norm": 1.6329550491747982, + "language_loss": 0.81491911, + "learning_rate": 4.807828030805207e-07, + "loss": 0.83654237, + "num_input_tokens_seen": 140050590, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.14953613, + "step": 6498, + "time_per_iteration": 2.7711994647979736 + }, + { + "auxiliary_loss_clip": 0.01127108, + "auxiliary_loss_mlp": 0.0103741, + "balance_loss_clip": 1.03974545, + "balance_loss_mlp": 1.02236581, + "epoch": 0.7814585462634521, + "flos": 20485924865280.0, + "grad_norm": 2.3665193498193386, + "language_loss": 0.68042803, + "learning_rate": 4.802762903664495e-07, + "loss": 0.70207322, + "num_input_tokens_seen": 140069770, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.15039062, + "step": 6499, + "time_per_iteration": 2.8664004802703857 + }, + { + "auxiliary_loss_clip": 0.01130206, + "auxiliary_loss_mlp": 0.01042306, + "balance_loss_clip": 1.04067624, + "balance_loss_mlp": 1.02542651, + "epoch": 0.7815787891540913, + "flos": 22304297018880.0, + "grad_norm": 2.327897675937555, + "language_loss": 0.73808235, + "learning_rate": 4.797700081932565e-07, + "loss": 0.75980747, + "num_input_tokens_seen": 140087635, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.16894531, + "step": 6500, + "time_per_iteration": 2.786672592163086 + }, + { + "auxiliary_loss_clip": 0.01125609, + "auxiliary_loss_mlp": 0.01035524, + "balance_loss_clip": 1.0409106, + "balance_loss_mlp": 1.02015829, + "epoch": 0.7816990320447303, + "flos": 22600668136320.0, + "grad_norm": 2.362528445549591, + "language_loss": 0.82066828, + "learning_rate": 4.792639566377442e-07, + "loss": 0.84227955, + "num_input_tokens_seen": 140105045, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.15380859, + "step": 6501, + "time_per_iteration": 2.7918546199798584 + }, + { + "auxiliary_loss_clip": 0.01122557, + "auxiliary_loss_mlp": 0.01041709, + "balance_loss_clip": 1.03777623, + "balance_loss_mlp": 1.02727842, + "epoch": 0.7818192749353694, + "flos": 24935974871040.0, + "grad_norm": 1.7458196123201806, + "language_loss": 0.7737627, + "learning_rate": 4.78758135776681e-07, + "loss": 0.79540533, + "num_input_tokens_seen": 140124900, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14416504, + "step": 6502, + "time_per_iteration": 2.799616813659668 + }, + { + "auxiliary_loss_clip": 0.01126961, + "auxiliary_loss_mlp": 0.01045546, + "balance_loss_clip": 1.04017878, + "balance_loss_mlp": 1.02939296, + "epoch": 0.7819395178260086, + "flos": 23733039369600.0, + "grad_norm": 1.9377901323393087, + "language_loss": 0.78847575, + "learning_rate": 4.782525456867989e-07, + "loss": 0.81020069, + "num_input_tokens_seen": 140143755, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.16168213, + "step": 6503, + "time_per_iteration": 2.768378257751465 + }, + { + "auxiliary_loss_clip": 0.0113114, + "auxiliary_loss_mlp": 0.01043002, + "balance_loss_clip": 1.04155684, + "balance_loss_mlp": 1.02656317, + "epoch": 0.7820597607166476, + "flos": 23221671396480.0, + "grad_norm": 1.5585887489050247, + "language_loss": 0.83145368, + "learning_rate": 4.777471864447959e-07, + "loss": 0.85319507, + "num_input_tokens_seen": 140164495, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.16442871, + "step": 6504, + "time_per_iteration": 2.8330130577087402 + }, + { + "auxiliary_loss_clip": 0.01128952, + "auxiliary_loss_mlp": 0.01033328, + "balance_loss_clip": 1.04142344, + "balance_loss_mlp": 1.01760459, + "epoch": 0.7821800036072867, + "flos": 22309540404480.0, + "grad_norm": 2.2195406343017465, + "language_loss": 0.80720401, + "learning_rate": 4.772420581273344e-07, + "loss": 0.82882679, + "num_input_tokens_seen": 140181980, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15710449, + "step": 6505, + "time_per_iteration": 2.748624563217163 + }, + { + "auxiliary_loss_clip": 0.01125879, + "auxiliary_loss_mlp": 0.01038804, + "balance_loss_clip": 1.03918743, + "balance_loss_mlp": 1.02300906, + "epoch": 0.7823002464979258, + "flos": 21544176384000.0, + "grad_norm": 1.8410401445272617, + "language_loss": 0.76461118, + "learning_rate": 4.7673716081104134e-07, + "loss": 0.78625798, + "num_input_tokens_seen": 140202155, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15795898, + "step": 6506, + "time_per_iteration": 2.8430793285369873 + }, + { + "auxiliary_loss_clip": 0.01129562, + "auxiliary_loss_mlp": 0.01040106, + "balance_loss_clip": 1.04205227, + "balance_loss_mlp": 1.02529407, + "epoch": 0.7824204893885649, + "flos": 24535642815360.0, + "grad_norm": 1.6946838961023059, + "language_loss": 0.84361374, + "learning_rate": 4.762324945725109e-07, + "loss": 0.86531037, + "num_input_tokens_seen": 140221600, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.14788818, + "step": 6507, + "time_per_iteration": 2.780078411102295 + }, + { + "auxiliary_loss_clip": 0.01125575, + "auxiliary_loss_mlp": 0.01040458, + "balance_loss_clip": 1.04164743, + "balance_loss_mlp": 1.02510428, + "epoch": 0.782540732279204, + "flos": 27415211402880.0, + "grad_norm": 1.7576725222447316, + "language_loss": 0.75762272, + "learning_rate": 4.7572805948829844e-07, + "loss": 0.77928305, + "num_input_tokens_seen": 140241860, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.15368652, + "step": 6508, + "time_per_iteration": 2.841054677963257 + }, + { + "auxiliary_loss_clip": 0.01123171, + "auxiliary_loss_mlp": 0.01035514, + "balance_loss_clip": 1.03836679, + "balance_loss_mlp": 1.01998091, + "epoch": 0.7826609751698431, + "flos": 24353216616960.0, + "grad_norm": 1.569638447860816, + "language_loss": 0.70827985, + "learning_rate": 4.7522385563492795e-07, + "loss": 0.72986674, + "num_input_tokens_seen": 140262160, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.15515137, + "step": 6509, + "time_per_iteration": 2.769164562225342 + }, + { + "auxiliary_loss_clip": 0.01128808, + "auxiliary_loss_mlp": 0.0104005, + "balance_loss_clip": 1.04118347, + "balance_loss_mlp": 1.02476764, + "epoch": 0.7827812180604822, + "flos": 23988543788160.0, + "grad_norm": 2.2797270842803994, + "language_loss": 0.70295495, + "learning_rate": 4.747198830888863e-07, + "loss": 0.72464353, + "num_input_tokens_seen": 140282030, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.1529541, + "step": 6510, + "time_per_iteration": 2.795159339904785 + }, + { + "auxiliary_loss_clip": 0.01124782, + "auxiliary_loss_mlp": 0.01035536, + "balance_loss_clip": 1.04069746, + "balance_loss_mlp": 1.02115893, + "epoch": 0.7829014609511212, + "flos": 27454318335360.0, + "grad_norm": 2.5106196052639023, + "language_loss": 0.68690586, + "learning_rate": 4.742161419266251e-07, + "loss": 0.70850903, + "num_input_tokens_seen": 140301190, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.14379883, + "step": 6511, + "time_per_iteration": 3.6131105422973633 + }, + { + "auxiliary_loss_clip": 0.01134215, + "auxiliary_loss_mlp": 0.01038821, + "balance_loss_clip": 1.04490709, + "balance_loss_mlp": 1.02120197, + "epoch": 0.7830217038417604, + "flos": 29204532432000.0, + "grad_norm": 2.501441893371299, + "language_loss": 0.64699787, + "learning_rate": 4.7371263222456304e-07, + "loss": 0.66872823, + "num_input_tokens_seen": 140318510, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.17614746, + "step": 6512, + "time_per_iteration": 2.82926607131958 + }, + { + "auxiliary_loss_clip": 0.01032611, + "auxiliary_loss_mlp": 0.01002702, + "balance_loss_clip": 1.00513506, + "balance_loss_mlp": 1.00125396, + "epoch": 0.7831419467323995, + "flos": 60950895822720.0, + "grad_norm": 0.8126935313428837, + "language_loss": 0.6135388, + "learning_rate": 4.7320935405908004e-07, + "loss": 0.63389194, + "num_input_tokens_seen": 140379380, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01446533, + "step": 6513, + "time_per_iteration": 3.2648205757141113 + }, + { + "auxiliary_loss_clip": 0.01131524, + "auxiliary_loss_mlp": 0.01038873, + "balance_loss_clip": 1.04031992, + "balance_loss_mlp": 1.02269673, + "epoch": 0.7832621896230385, + "flos": 19682531320320.0, + "grad_norm": 2.1579535607353786, + "language_loss": 0.84370607, + "learning_rate": 4.7270630750652475e-07, + "loss": 0.86541009, + "num_input_tokens_seen": 140395335, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.16186523, + "step": 6514, + "time_per_iteration": 2.9847500324249268 + }, + { + "auxiliary_loss_clip": 0.01121334, + "auxiliary_loss_mlp": 0.0103497, + "balance_loss_clip": 1.03730774, + "balance_loss_mlp": 1.02034938, + "epoch": 0.7833824325136777, + "flos": 25009232659200.0, + "grad_norm": 1.6983036794860802, + "language_loss": 0.80145109, + "learning_rate": 4.7220349264320746e-07, + "loss": 0.82301414, + "num_input_tokens_seen": 140414420, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14630127, + "step": 6515, + "time_per_iteration": 3.54176664352417 + }, + { + "auxiliary_loss_clip": 0.01032848, + "auxiliary_loss_mlp": 0.00999836, + "balance_loss_clip": 1.00522876, + "balance_loss_mlp": 0.99836361, + "epoch": 0.7835026754043167, + "flos": 68800142517120.0, + "grad_norm": 0.7371908793408558, + "language_loss": 0.54874921, + "learning_rate": 4.71700909545407e-07, + "loss": 0.56907606, + "num_input_tokens_seen": 140477365, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01470947, + "step": 6516, + "time_per_iteration": 4.081447601318359 + }, + { + "auxiliary_loss_clip": 0.01124979, + "auxiliary_loss_mlp": 0.01036439, + "balance_loss_clip": 1.04105318, + "balance_loss_mlp": 1.02104926, + "epoch": 0.7836229182949558, + "flos": 19864598382720.0, + "grad_norm": 3.0481274991259375, + "language_loss": 0.76859546, + "learning_rate": 4.711985582893627e-07, + "loss": 0.79020965, + "num_input_tokens_seen": 140495885, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.15405273, + "step": 6517, + "time_per_iteration": 2.779414653778076 + }, + { + "auxiliary_loss_clip": 0.01126576, + "auxiliary_loss_mlp": 0.01038525, + "balance_loss_clip": 1.04005671, + "balance_loss_mlp": 1.02134061, + "epoch": 0.783743161185595, + "flos": 22965843755520.0, + "grad_norm": 1.6954606219390356, + "language_loss": 0.71334714, + "learning_rate": 4.706964389512811e-07, + "loss": 0.73499811, + "num_input_tokens_seen": 140515920, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.17181396, + "step": 6518, + "time_per_iteration": 2.808539390563965 + }, + { + "auxiliary_loss_clip": 0.01126995, + "auxiliary_loss_mlp": 0.01042857, + "balance_loss_clip": 1.04268146, + "balance_loss_mlp": 1.0273006, + "epoch": 0.783863404076234, + "flos": 12458489777280.0, + "grad_norm": 1.8391644009558972, + "language_loss": 0.87722957, + "learning_rate": 4.701945516073345e-07, + "loss": 0.89892817, + "num_input_tokens_seen": 140533395, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.15563965, + "step": 6519, + "time_per_iteration": 2.8795478343963623 + }, + { + "auxiliary_loss_clip": 0.01125844, + "auxiliary_loss_mlp": 0.01031834, + "balance_loss_clip": 1.03874767, + "balance_loss_mlp": 1.01671815, + "epoch": 0.7839836469668731, + "flos": 24243940465920.0, + "grad_norm": 1.793477841565663, + "language_loss": 0.75430018, + "learning_rate": 4.696928963336577e-07, + "loss": 0.77587694, + "num_input_tokens_seen": 140552825, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15100098, + "step": 6520, + "time_per_iteration": 2.7710156440734863 + }, + { + "auxiliary_loss_clip": 0.01033013, + "auxiliary_loss_mlp": 0.01002299, + "balance_loss_clip": 1.00533938, + "balance_loss_mlp": 1.00079656, + "epoch": 0.7841038898575122, + "flos": 62121978938880.0, + "grad_norm": 0.8488688673947471, + "language_loss": 0.60902154, + "learning_rate": 4.6919147320635224e-07, + "loss": 0.62937468, + "num_input_tokens_seen": 140615535, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01501465, + "step": 6521, + "time_per_iteration": 3.2471423149108887 + }, + { + "auxiliary_loss_clip": 0.0112642, + "auxiliary_loss_mlp": 0.01039593, + "balance_loss_clip": 1.04277134, + "balance_loss_mlp": 1.02333891, + "epoch": 0.7842241327481513, + "flos": 20193899293440.0, + "grad_norm": 2.0867718087572875, + "language_loss": 0.73389697, + "learning_rate": 4.6869028230148286e-07, + "loss": 0.75555712, + "num_input_tokens_seen": 140633330, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.16265869, + "step": 6522, + "time_per_iteration": 3.675873279571533 + }, + { + "auxiliary_loss_clip": 0.01126825, + "auxiliary_loss_mlp": 0.01041427, + "balance_loss_clip": 1.04069114, + "balance_loss_mlp": 1.02570319, + "epoch": 0.7843443756387903, + "flos": 28074531496320.0, + "grad_norm": 2.359039421935505, + "language_loss": 0.60126114, + "learning_rate": 4.6818932369507957e-07, + "loss": 0.62294364, + "num_input_tokens_seen": 140652830, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.15722656, + "step": 6523, + "time_per_iteration": 2.797682046890259 + }, + { + "auxiliary_loss_clip": 0.01125598, + "auxiliary_loss_mlp": 0.01043984, + "balance_loss_clip": 1.04014826, + "balance_loss_mlp": 1.02735424, + "epoch": 0.7844646185294295, + "flos": 21323397438720.0, + "grad_norm": 1.9629196057049907, + "language_loss": 0.88872987, + "learning_rate": 4.676885974631386e-07, + "loss": 0.91042578, + "num_input_tokens_seen": 140671190, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.1661377, + "step": 6524, + "time_per_iteration": 2.754202365875244 + }, + { + "auxiliary_loss_clip": 0.0112696, + "auxiliary_loss_mlp": 0.01039513, + "balance_loss_clip": 1.04079318, + "balance_loss_mlp": 1.0242722, + "epoch": 0.7845848614200686, + "flos": 23656585271040.0, + "grad_norm": 1.963751932814769, + "language_loss": 0.80860728, + "learning_rate": 4.67188103681619e-07, + "loss": 0.83027202, + "num_input_tokens_seen": 140690975, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15216064, + "step": 6525, + "time_per_iteration": 2.7575442790985107 + }, + { + "auxiliary_loss_clip": 0.0112296, + "auxiliary_loss_mlp": 0.01039402, + "balance_loss_clip": 1.04038596, + "balance_loss_mlp": 1.02366614, + "epoch": 0.7847051043107076, + "flos": 23402194174080.0, + "grad_norm": 2.2077034006634975, + "language_loss": 0.69238931, + "learning_rate": 4.666878424264453e-07, + "loss": 0.71401292, + "num_input_tokens_seen": 140710930, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.15734863, + "step": 6526, + "time_per_iteration": 2.838186025619507 + }, + { + "auxiliary_loss_clip": 0.01118376, + "auxiliary_loss_mlp": 0.01033729, + "balance_loss_clip": 1.04051292, + "balance_loss_mlp": 1.02036524, + "epoch": 0.7848253472013467, + "flos": 19022277473280.0, + "grad_norm": 1.5229810525978518, + "language_loss": 0.73526466, + "learning_rate": 4.661878137735069e-07, + "loss": 0.75678575, + "num_input_tokens_seen": 140729120, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.13378906, + "step": 6527, + "time_per_iteration": 2.734372138977051 + }, + { + "auxiliary_loss_clip": 0.01125943, + "auxiliary_loss_mlp": 0.01039911, + "balance_loss_clip": 1.04001403, + "balance_loss_mlp": 1.02508187, + "epoch": 0.7849455900919858, + "flos": 21179180332800.0, + "grad_norm": 1.9559712423349476, + "language_loss": 0.74811089, + "learning_rate": 4.656880177986571e-07, + "loss": 0.76976943, + "num_input_tokens_seen": 140747665, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.14831543, + "step": 6528, + "time_per_iteration": 2.753462791442871 + }, + { + "auxiliary_loss_clip": 0.01128681, + "auxiliary_loss_mlp": 0.0103544, + "balance_loss_clip": 1.03969479, + "balance_loss_mlp": 1.01941204, + "epoch": 0.7850658329826249, + "flos": 19536482620800.0, + "grad_norm": 7.646036036467385, + "language_loss": 0.81779921, + "learning_rate": 4.6518845457771607e-07, + "loss": 0.83944046, + "num_input_tokens_seen": 140766525, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.16009521, + "step": 6529, + "time_per_iteration": 2.822155237197876 + }, + { + "auxiliary_loss_clip": 0.01127292, + "auxiliary_loss_mlp": 0.01036073, + "balance_loss_clip": 1.04101121, + "balance_loss_mlp": 1.02099276, + "epoch": 0.7851860758732639, + "flos": 12495334152960.0, + "grad_norm": 1.9404123793325527, + "language_loss": 0.79313838, + "learning_rate": 4.646891241864652e-07, + "loss": 0.81477201, + "num_input_tokens_seen": 140785090, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.15075684, + "step": 6530, + "time_per_iteration": 2.821265935897827 + }, + { + "auxiliary_loss_clip": 0.01131844, + "auxiliary_loss_mlp": 0.01047442, + "balance_loss_clip": 1.04285705, + "balance_loss_mlp": 1.03134847, + "epoch": 0.7853063187639031, + "flos": 22960959505920.0, + "grad_norm": 2.1846617396673187, + "language_loss": 0.73153222, + "learning_rate": 4.6419002670065397e-07, + "loss": 0.75332505, + "num_input_tokens_seen": 140804670, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.16101074, + "step": 6531, + "time_per_iteration": 2.890953540802002 + }, + { + "auxiliary_loss_clip": 0.0112889, + "auxiliary_loss_mlp": 0.01039333, + "balance_loss_clip": 1.04203463, + "balance_loss_mlp": 1.02357388, + "epoch": 0.7854265616545422, + "flos": 17347260499200.0, + "grad_norm": 1.9464376924758986, + "language_loss": 0.86946547, + "learning_rate": 4.6369116219599445e-07, + "loss": 0.89114773, + "num_input_tokens_seen": 140820655, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.15759277, + "step": 6532, + "time_per_iteration": 2.8417749404907227 + }, + { + "auxiliary_loss_clip": 0.01124868, + "auxiliary_loss_mlp": 0.01040266, + "balance_loss_clip": 1.04003072, + "balance_loss_mlp": 1.02529335, + "epoch": 0.7855468045451812, + "flos": 23838293197440.0, + "grad_norm": 1.6670865666644108, + "language_loss": 0.79553235, + "learning_rate": 4.631925307481637e-07, + "loss": 0.81718361, + "num_input_tokens_seen": 140840470, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.14978027, + "step": 6533, + "time_per_iteration": 2.822235345840454 + }, + { + "auxiliary_loss_clip": 0.01123478, + "auxiliary_loss_mlp": 0.01032548, + "balance_loss_clip": 1.04126358, + "balance_loss_mlp": 1.01770604, + "epoch": 0.7856670474358204, + "flos": 25666792986240.0, + "grad_norm": 1.9570123567398998, + "language_loss": 0.75605476, + "learning_rate": 4.6269413243280533e-07, + "loss": 0.77761507, + "num_input_tokens_seen": 140859890, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.14831543, + "step": 6534, + "time_per_iteration": 2.7946197986602783 + }, + { + "auxiliary_loss_clip": 0.01132309, + "auxiliary_loss_mlp": 0.01043187, + "balance_loss_clip": 1.04120362, + "balance_loss_mlp": 1.02662933, + "epoch": 0.7857872903264594, + "flos": 18144656472960.0, + "grad_norm": 2.2673439593466598, + "language_loss": 0.741256, + "learning_rate": 4.621959673255236e-07, + "loss": 0.76301098, + "num_input_tokens_seen": 140876190, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.16564941, + "step": 6535, + "time_per_iteration": 2.747760057449341 + }, + { + "auxiliary_loss_clip": 0.01123983, + "auxiliary_loss_mlp": 0.01044499, + "balance_loss_clip": 1.04091263, + "balance_loss_mlp": 1.02921605, + "epoch": 0.7859075332170985, + "flos": 14386138081920.0, + "grad_norm": 2.2183075952019835, + "language_loss": 0.90346122, + "learning_rate": 4.6169803550189135e-07, + "loss": 0.92514604, + "num_input_tokens_seen": 140891885, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.152771, + "step": 6536, + "time_per_iteration": 2.7906391620635986 + }, + { + "auxiliary_loss_clip": 0.0112937, + "auxiliary_loss_mlp": 0.01039257, + "balance_loss_clip": 1.04025841, + "balance_loss_mlp": 1.02218664, + "epoch": 0.7860277761077377, + "flos": 19864059678720.0, + "grad_norm": 2.2064974704746447, + "language_loss": 0.77457392, + "learning_rate": 4.6120033703744355e-07, + "loss": 0.79626012, + "num_input_tokens_seen": 140910780, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.1706543, + "step": 6537, + "time_per_iteration": 2.909670352935791 + }, + { + "auxiliary_loss_clip": 0.01123838, + "auxiliary_loss_mlp": 0.01038948, + "balance_loss_clip": 1.04051065, + "balance_loss_mlp": 1.02347481, + "epoch": 0.7861480189983767, + "flos": 26396174557440.0, + "grad_norm": 1.8863446145302498, + "language_loss": 0.78464234, + "learning_rate": 4.607028720076822e-07, + "loss": 0.80627018, + "num_input_tokens_seen": 140927460, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.15490723, + "step": 6538, + "time_per_iteration": 2.8362889289855957 + }, + { + "auxiliary_loss_clip": 0.01126042, + "auxiliary_loss_mlp": 0.01039052, + "balance_loss_clip": 1.04066074, + "balance_loss_mlp": 1.02366197, + "epoch": 0.7862682618890158, + "flos": 24236578177920.0, + "grad_norm": 2.1495152588832465, + "language_loss": 0.73349351, + "learning_rate": 4.6020564048807074e-07, + "loss": 0.75514442, + "num_input_tokens_seen": 140945135, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.15393066, + "step": 6539, + "time_per_iteration": 3.6494150161743164 + }, + { + "auxiliary_loss_clip": 0.01124794, + "auxiliary_loss_mlp": 0.01037041, + "balance_loss_clip": 1.03838491, + "balance_loss_mlp": 1.02148414, + "epoch": 0.7863885047796549, + "flos": 47551508259840.0, + "grad_norm": 1.9956489777099884, + "language_loss": 0.71854615, + "learning_rate": 4.5970864255403883e-07, + "loss": 0.74016452, + "num_input_tokens_seen": 140966660, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.15563965, + "step": 6540, + "time_per_iteration": 2.9618337154388428 + }, + { + "auxiliary_loss_clip": 0.01123849, + "auxiliary_loss_mlp": 0.01031764, + "balance_loss_clip": 1.04149997, + "balance_loss_mlp": 1.01786458, + "epoch": 0.786508747670294, + "flos": 24389234979840.0, + "grad_norm": 1.8398059068688666, + "language_loss": 0.82058561, + "learning_rate": 4.59211878280982e-07, + "loss": 0.84214175, + "num_input_tokens_seen": 140986175, + "router_z_loss_clip": 0.82421875, + "router_z_loss_mlp": 0.13909912, + "step": 6541, + "time_per_iteration": 2.8157267570495605 + }, + { + "auxiliary_loss_clip": 0.01124709, + "auxiliary_loss_mlp": 0.01038412, + "balance_loss_clip": 1.03964365, + "balance_loss_mlp": 1.02297425, + "epoch": 0.786628990560933, + "flos": 18041234238720.0, + "grad_norm": 2.164280080483696, + "language_loss": 0.69703972, + "learning_rate": 4.587153477442578e-07, + "loss": 0.71867096, + "num_input_tokens_seen": 141002490, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15441895, + "step": 6542, + "time_per_iteration": 2.7262520790100098 + }, + { + "auxiliary_loss_clip": 0.01129275, + "auxiliary_loss_mlp": 0.0103675, + "balance_loss_clip": 1.04095864, + "balance_loss_mlp": 1.01930976, + "epoch": 0.7867492334515722, + "flos": 25848860048640.0, + "grad_norm": 2.290258520502589, + "language_loss": 0.81455243, + "learning_rate": 4.582190510191899e-07, + "loss": 0.83621275, + "num_input_tokens_seen": 141021150, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.17468262, + "step": 6543, + "time_per_iteration": 2.7959630489349365 + }, + { + "auxiliary_loss_clip": 0.01122746, + "auxiliary_loss_mlp": 0.01035342, + "balance_loss_clip": 1.03917646, + "balance_loss_mlp": 1.01974916, + "epoch": 0.7868694763422113, + "flos": 16580819070720.0, + "grad_norm": 1.8887924985702746, + "language_loss": 0.87088716, + "learning_rate": 4.5772298818106625e-07, + "loss": 0.89246809, + "num_input_tokens_seen": 141036940, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.15612793, + "step": 6544, + "time_per_iteration": 3.507848024368286 + }, + { + "auxiliary_loss_clip": 0.01130152, + "auxiliary_loss_mlp": 0.01041873, + "balance_loss_clip": 1.04177737, + "balance_loss_mlp": 1.02607167, + "epoch": 0.7869897192328503, + "flos": 29386276272000.0, + "grad_norm": 2.463026656233433, + "language_loss": 0.72123235, + "learning_rate": 4.572271593051384e-07, + "loss": 0.74295259, + "num_input_tokens_seen": 141054295, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.15789795, + "step": 6545, + "time_per_iteration": 3.6250500679016113 + }, + { + "auxiliary_loss_clip": 0.01128419, + "auxiliary_loss_mlp": 0.01036852, + "balance_loss_clip": 1.04104388, + "balance_loss_mlp": 1.02236795, + "epoch": 0.7871099621234895, + "flos": 17128923678720.0, + "grad_norm": 1.797109978788657, + "language_loss": 0.78251094, + "learning_rate": 4.567315644666245e-07, + "loss": 0.80416363, + "num_input_tokens_seen": 141073090, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.14489746, + "step": 6546, + "time_per_iteration": 2.7341644763946533 + }, + { + "auxiliary_loss_clip": 0.01123113, + "auxiliary_loss_mlp": 0.01041971, + "balance_loss_clip": 1.04133081, + "balance_loss_mlp": 1.02633703, + "epoch": 0.7872302050141285, + "flos": 23440187784960.0, + "grad_norm": 2.088603999955862, + "language_loss": 0.84573072, + "learning_rate": 4.5623620374070507e-07, + "loss": 0.86738157, + "num_input_tokens_seen": 141092405, + "router_z_loss_clip": 0.81689453, + "router_z_loss_mlp": 0.15631104, + "step": 6547, + "time_per_iteration": 2.7893786430358887 + }, + { + "auxiliary_loss_clip": 0.01032269, + "auxiliary_loss_mlp": 0.01002186, + "balance_loss_clip": 1.00473237, + "balance_loss_mlp": 1.00075519, + "epoch": 0.7873504479047676, + "flos": 65959752689280.0, + "grad_norm": 0.7634479683860511, + "language_loss": 0.58379209, + "learning_rate": 4.557410772025263e-07, + "loss": 0.60413659, + "num_input_tokens_seen": 141154355, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01428223, + "step": 6548, + "time_per_iteration": 3.3719441890716553 + }, + { + "auxiliary_loss_clip": 0.01126767, + "auxiliary_loss_mlp": 0.01036042, + "balance_loss_clip": 1.04133224, + "balance_loss_mlp": 1.01975787, + "epoch": 0.7874706907954068, + "flos": 23258336204160.0, + "grad_norm": 1.9252593610463729, + "language_loss": 0.66121298, + "learning_rate": 4.5524618492719803e-07, + "loss": 0.68284106, + "num_input_tokens_seen": 141173575, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.16259766, + "step": 6549, + "time_per_iteration": 2.7860004901885986 + }, + { + "auxiliary_loss_clip": 0.01123723, + "auxiliary_loss_mlp": 0.01032381, + "balance_loss_clip": 1.03857541, + "balance_loss_mlp": 1.01727152, + "epoch": 0.7875909336860458, + "flos": 28767786963840.0, + "grad_norm": 1.5607182280793832, + "language_loss": 0.78718102, + "learning_rate": 4.54751526989795e-07, + "loss": 0.80874205, + "num_input_tokens_seen": 141195415, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.15106201, + "step": 6550, + "time_per_iteration": 3.734398603439331 + }, + { + "auxiliary_loss_clip": 0.01126352, + "auxiliary_loss_mlp": 0.01038296, + "balance_loss_clip": 1.03906131, + "balance_loss_mlp": 1.02279913, + "epoch": 0.7877111765766849, + "flos": 18697286194560.0, + "grad_norm": 1.8876583254364239, + "language_loss": 0.79273951, + "learning_rate": 4.5425710346535775e-07, + "loss": 0.81438601, + "num_input_tokens_seen": 141213360, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15509033, + "step": 6551, + "time_per_iteration": 2.786989450454712 + }, + { + "auxiliary_loss_clip": 0.01125639, + "auxiliary_loss_mlp": 0.0103535, + "balance_loss_clip": 1.03871262, + "balance_loss_mlp": 1.01968598, + "epoch": 0.787831419467324, + "flos": 27592968833280.0, + "grad_norm": 3.5216580500740986, + "language_loss": 0.8165555, + "learning_rate": 4.537629144288877e-07, + "loss": 0.83816546, + "num_input_tokens_seen": 141230815, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15661621, + "step": 6552, + "time_per_iteration": 2.7947769165039062 + }, + { + "auxiliary_loss_clip": 0.01126418, + "auxiliary_loss_mlp": 0.0103857, + "balance_loss_clip": 1.0401392, + "balance_loss_mlp": 1.02290606, + "epoch": 0.7879516623579631, + "flos": 18150187167360.0, + "grad_norm": 16.908202583372525, + "language_loss": 0.75235081, + "learning_rate": 4.5326895995535477e-07, + "loss": 0.77400064, + "num_input_tokens_seen": 141249715, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15673828, + "step": 6553, + "time_per_iteration": 2.800645351409912 + }, + { + "auxiliary_loss_clip": 0.01124783, + "auxiliary_loss_mlp": 0.0104184, + "balance_loss_clip": 1.03873777, + "balance_loss_mlp": 1.02616394, + "epoch": 0.7880719052486022, + "flos": 20339193807360.0, + "grad_norm": 2.241253709806838, + "language_loss": 0.84509861, + "learning_rate": 4.527752401196907e-07, + "loss": 0.8667649, + "num_input_tokens_seen": 141267730, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15686035, + "step": 6554, + "time_per_iteration": 2.7709739208221436 + }, + { + "auxiliary_loss_clip": 0.01127561, + "auxiliary_loss_mlp": 0.0103647, + "balance_loss_clip": 1.04005694, + "balance_loss_mlp": 1.02119339, + "epoch": 0.7881921481392413, + "flos": 21653237053440.0, + "grad_norm": 1.7741419679651735, + "language_loss": 0.66479349, + "learning_rate": 4.5228175499679254e-07, + "loss": 0.68643379, + "num_input_tokens_seen": 141287315, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.15258789, + "step": 6555, + "time_per_iteration": 2.8759961128234863 + }, + { + "auxiliary_loss_clip": 0.01032296, + "auxiliary_loss_mlp": 0.01003321, + "balance_loss_clip": 1.00480819, + "balance_loss_mlp": 1.0019021, + "epoch": 0.7883123910298804, + "flos": 68565860058240.0, + "grad_norm": 0.8397120840810692, + "language_loss": 0.54523379, + "learning_rate": 4.5178850466152174e-07, + "loss": 0.56558996, + "num_input_tokens_seen": 141346145, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01416016, + "step": 6556, + "time_per_iteration": 3.336068868637085 + }, + { + "auxiliary_loss_clip": 0.0112354, + "auxiliary_loss_mlp": 0.01033085, + "balance_loss_clip": 1.03988361, + "balance_loss_mlp": 1.018255, + "epoch": 0.7884326339205194, + "flos": 19318217627520.0, + "grad_norm": 2.0310369063608085, + "language_loss": 0.82024944, + "learning_rate": 4.512954891887031e-07, + "loss": 0.84181565, + "num_input_tokens_seen": 141364445, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.1484375, + "step": 6557, + "time_per_iteration": 2.8061468601226807 + }, + { + "auxiliary_loss_clip": 0.01128152, + "auxiliary_loss_mlp": 0.01032797, + "balance_loss_clip": 1.0415287, + "balance_loss_mlp": 1.01662052, + "epoch": 0.7885528768111585, + "flos": 17784903807360.0, + "grad_norm": 2.354938825524672, + "language_loss": 0.83811915, + "learning_rate": 4.5080270865312806e-07, + "loss": 0.85972863, + "num_input_tokens_seen": 141381640, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.16186523, + "step": 6558, + "time_per_iteration": 2.9442403316497803 + }, + { + "auxiliary_loss_clip": 0.01126917, + "auxiliary_loss_mlp": 0.01037323, + "balance_loss_clip": 1.04037333, + "balance_loss_mlp": 1.02170622, + "epoch": 0.7886731197017977, + "flos": 18807639753600.0, + "grad_norm": 4.783790126131543, + "language_loss": 0.71450132, + "learning_rate": 4.5031016312954985e-07, + "loss": 0.73614371, + "num_input_tokens_seen": 141399955, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15612793, + "step": 6559, + "time_per_iteration": 2.7808189392089844 + }, + { + "auxiliary_loss_clip": 0.01133174, + "auxiliary_loss_mlp": 0.01043978, + "balance_loss_clip": 1.04369354, + "balance_loss_mlp": 1.02694297, + "epoch": 0.7887933625924367, + "flos": 33365358126720.0, + "grad_norm": 1.8779259450085732, + "language_loss": 0.74385726, + "learning_rate": 4.498178526926886e-07, + "loss": 0.76562881, + "num_input_tokens_seen": 141420820, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.17041016, + "step": 6560, + "time_per_iteration": 2.8438236713409424 + }, + { + "auxiliary_loss_clip": 0.01126306, + "auxiliary_loss_mlp": 0.01039617, + "balance_loss_clip": 1.04066861, + "balance_loss_mlp": 1.02546096, + "epoch": 0.7889136054830758, + "flos": 17019360218880.0, + "grad_norm": 2.2812094155132128, + "language_loss": 0.7277382, + "learning_rate": 4.4932577741722635e-07, + "loss": 0.74939746, + "num_input_tokens_seen": 141439350, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.14135742, + "step": 6561, + "time_per_iteration": 2.790872812271118 + }, + { + "auxiliary_loss_clip": 0.01127294, + "auxiliary_loss_mlp": 0.01037351, + "balance_loss_clip": 1.04067802, + "balance_loss_mlp": 1.02177072, + "epoch": 0.7890338483737149, + "flos": 29424629018880.0, + "grad_norm": 1.921986552122146, + "language_loss": 0.74206233, + "learning_rate": 4.4883393737780985e-07, + "loss": 0.76370883, + "num_input_tokens_seen": 141460300, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15588379, + "step": 6562, + "time_per_iteration": 2.7720706462860107 + }, + { + "auxiliary_loss_clip": 0.01123651, + "auxiliary_loss_mlp": 0.01035482, + "balance_loss_clip": 1.03900075, + "balance_loss_mlp": 1.02039051, + "epoch": 0.789154091264354, + "flos": 19971576063360.0, + "grad_norm": 1.8093221367573735, + "language_loss": 0.78314638, + "learning_rate": 4.4834233264905254e-07, + "loss": 0.80473769, + "num_input_tokens_seen": 141477315, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15087891, + "step": 6563, + "time_per_iteration": 2.7283685207366943 + }, + { + "auxiliary_loss_clip": 0.01128847, + "auxiliary_loss_mlp": 0.01038278, + "balance_loss_clip": 1.0427835, + "balance_loss_mlp": 1.02231622, + "epoch": 0.789274334154993, + "flos": 14537825216640.0, + "grad_norm": 2.3850651175438125, + "language_loss": 0.72087413, + "learning_rate": 4.478509633055294e-07, + "loss": 0.74254537, + "num_input_tokens_seen": 141495025, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.1595459, + "step": 6564, + "time_per_iteration": 2.7128896713256836 + }, + { + "auxiliary_loss_clip": 0.011298, + "auxiliary_loss_mlp": 0.01046522, + "balance_loss_clip": 1.0414114, + "balance_loss_mlp": 1.02998769, + "epoch": 0.7893945770456322, + "flos": 21827403123840.0, + "grad_norm": 2.8087890842850993, + "language_loss": 0.79679585, + "learning_rate": 4.473598294217813e-07, + "loss": 0.81855911, + "num_input_tokens_seen": 141510450, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.1652832, + "step": 6565, + "time_per_iteration": 2.764009952545166 + }, + { + "auxiliary_loss_clip": 0.01122021, + "auxiliary_loss_mlp": 0.01037845, + "balance_loss_clip": 1.03848743, + "balance_loss_mlp": 1.02427864, + "epoch": 0.7895148199362713, + "flos": 20740639184640.0, + "grad_norm": 2.270868032509319, + "language_loss": 0.72004998, + "learning_rate": 4.468689310723124e-07, + "loss": 0.74164861, + "num_input_tokens_seen": 141528265, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.13574219, + "step": 6566, + "time_per_iteration": 2.760493278503418 + }, + { + "auxiliary_loss_clip": 0.01124517, + "auxiliary_loss_mlp": 0.01038305, + "balance_loss_clip": 1.03992295, + "balance_loss_mlp": 1.02324867, + "epoch": 0.7896350628269103, + "flos": 16690669839360.0, + "grad_norm": 1.791668605288899, + "language_loss": 0.78768426, + "learning_rate": 4.463782683315913e-07, + "loss": 0.80931252, + "num_input_tokens_seen": 141547270, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.1505127, + "step": 6567, + "time_per_iteration": 2.766106128692627 + }, + { + "auxiliary_loss_clip": 0.01122962, + "auxiliary_loss_mlp": 0.01035383, + "balance_loss_clip": 1.03938258, + "balance_loss_mlp": 1.02082801, + "epoch": 0.7897553057175495, + "flos": 22638374438400.0, + "grad_norm": 1.6632198402418084, + "language_loss": 0.73629439, + "learning_rate": 4.458878412740523e-07, + "loss": 0.75787777, + "num_input_tokens_seen": 141566050, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14550781, + "step": 6568, + "time_per_iteration": 3.6489317417144775 + }, + { + "auxiliary_loss_clip": 0.01127391, + "auxiliary_loss_mlp": 0.01032374, + "balance_loss_clip": 1.04056025, + "balance_loss_mlp": 1.01692486, + "epoch": 0.7898755486081885, + "flos": 14537573821440.0, + "grad_norm": 2.378626474151565, + "language_loss": 0.78433156, + "learning_rate": 4.453976499740919e-07, + "loss": 0.80592924, + "num_input_tokens_seen": 141583695, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15441895, + "step": 6569, + "time_per_iteration": 2.751192569732666 + }, + { + "auxiliary_loss_clip": 0.01126987, + "auxiliary_loss_mlp": 0.01040907, + "balance_loss_clip": 1.04106462, + "balance_loss_mlp": 1.02604175, + "epoch": 0.7899957914988276, + "flos": 17238487138560.0, + "grad_norm": 1.8482985073720462, + "language_loss": 0.77591485, + "learning_rate": 4.4490769450607215e-07, + "loss": 0.79759377, + "num_input_tokens_seen": 141601320, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.14855957, + "step": 6570, + "time_per_iteration": 2.7957842350006104 + }, + { + "auxiliary_loss_clip": 0.01126063, + "auxiliary_loss_mlp": 0.01040356, + "balance_loss_clip": 1.04120088, + "balance_loss_mlp": 1.02483463, + "epoch": 0.7901160343894668, + "flos": 41279351086080.0, + "grad_norm": 1.7781748114056288, + "language_loss": 0.72493303, + "learning_rate": 4.4441797494431845e-07, + "loss": 0.74659717, + "num_input_tokens_seen": 141623125, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15515137, + "step": 6571, + "time_per_iteration": 2.8903677463531494 + }, + { + "auxiliary_loss_clip": 0.01125553, + "auxiliary_loss_mlp": 0.01039794, + "balance_loss_clip": 1.03980565, + "balance_loss_mlp": 1.02459478, + "epoch": 0.7902362772801058, + "flos": 16837005847680.0, + "grad_norm": 2.0371801141929127, + "language_loss": 0.7777527, + "learning_rate": 4.439284913631207e-07, + "loss": 0.79940617, + "num_input_tokens_seen": 141640335, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.15222168, + "step": 6572, + "time_per_iteration": 2.7369251251220703 + }, + { + "auxiliary_loss_clip": 0.01127815, + "auxiliary_loss_mlp": 0.01041606, + "balance_loss_clip": 1.04020321, + "balance_loss_mlp": 1.02457106, + "epoch": 0.7903565201707449, + "flos": 27125987091840.0, + "grad_norm": 1.986192444402602, + "language_loss": 0.83652234, + "learning_rate": 4.434392438367347e-07, + "loss": 0.85821652, + "num_input_tokens_seen": 141659760, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.17041016, + "step": 6573, + "time_per_iteration": 3.604550838470459 + }, + { + "auxiliary_loss_clip": 0.01127091, + "auxiliary_loss_mlp": 0.01037127, + "balance_loss_clip": 1.0409472, + "balance_loss_mlp": 1.02231514, + "epoch": 0.790476763061384, + "flos": 31025167142400.0, + "grad_norm": 1.775853077181255, + "language_loss": 0.7377274, + "learning_rate": 4.4295023243937677e-07, + "loss": 0.75936961, + "num_input_tokens_seen": 141679965, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.14813232, + "step": 6574, + "time_per_iteration": 3.6198413372039795 + }, + { + "auxiliary_loss_clip": 0.01134015, + "auxiliary_loss_mlp": 0.01044613, + "balance_loss_clip": 1.04293799, + "balance_loss_mlp": 1.02635074, + "epoch": 0.7905970059520231, + "flos": 22089084681600.0, + "grad_norm": 1.5962246109216227, + "language_loss": 0.80155313, + "learning_rate": 4.4246145724523123e-07, + "loss": 0.82333946, + "num_input_tokens_seen": 141697710, + "router_z_loss_clip": 0.91064453, + "router_z_loss_mlp": 0.18249512, + "step": 6575, + "time_per_iteration": 2.785829782485962 + }, + { + "auxiliary_loss_clip": 0.01126999, + "auxiliary_loss_mlp": 0.01039393, + "balance_loss_clip": 1.04142976, + "balance_loss_mlp": 1.02495635, + "epoch": 0.7907172488426621, + "flos": 20558141159040.0, + "grad_norm": 2.274323384182088, + "language_loss": 0.7805891, + "learning_rate": 4.41972918328444e-07, + "loss": 0.80225301, + "num_input_tokens_seen": 141715145, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.14428711, + "step": 6576, + "time_per_iteration": 2.7519168853759766 + }, + { + "auxiliary_loss_clip": 0.01125506, + "auxiliary_loss_mlp": 0.0103766, + "balance_loss_clip": 1.04139018, + "balance_loss_mlp": 1.02279496, + "epoch": 0.7908374917333013, + "flos": 30081542901120.0, + "grad_norm": 2.6044758570596236, + "language_loss": 0.77801955, + "learning_rate": 4.4148461576312646e-07, + "loss": 0.79965121, + "num_input_tokens_seen": 141734810, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14868164, + "step": 6577, + "time_per_iteration": 2.811289072036743 + }, + { + "auxiliary_loss_clip": 0.01121191, + "auxiliary_loss_mlp": 0.01030841, + "balance_loss_clip": 1.0414089, + "balance_loss_mlp": 1.01711977, + "epoch": 0.7909577346239404, + "flos": 20996359084800.0, + "grad_norm": 1.4428600567720136, + "language_loss": 0.74773753, + "learning_rate": 4.4099654962335343e-07, + "loss": 0.7692579, + "num_input_tokens_seen": 141755260, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.137146, + "step": 6578, + "time_per_iteration": 2.8140501976013184 + }, + { + "auxiliary_loss_clip": 0.01124585, + "auxiliary_loss_mlp": 0.01037593, + "balance_loss_clip": 1.04033017, + "balance_loss_mlp": 1.02117777, + "epoch": 0.7910779775145794, + "flos": 26247935128320.0, + "grad_norm": 1.789145111860497, + "language_loss": 0.74774182, + "learning_rate": 4.405087199831636e-07, + "loss": 0.76936352, + "num_input_tokens_seen": 141775500, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.1640625, + "step": 6579, + "time_per_iteration": 3.73026442527771 + }, + { + "auxiliary_loss_clip": 0.01124539, + "auxiliary_loss_mlp": 0.01038756, + "balance_loss_clip": 1.04022014, + "balance_loss_mlp": 1.02424812, + "epoch": 0.7911982204052186, + "flos": 22564434291840.0, + "grad_norm": 1.843757362703816, + "language_loss": 0.67228436, + "learning_rate": 4.400211269165619e-07, + "loss": 0.69391727, + "num_input_tokens_seen": 141791955, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.14501953, + "step": 6580, + "time_per_iteration": 2.8122477531433105 + }, + { + "auxiliary_loss_clip": 0.01128256, + "auxiliary_loss_mlp": 0.01034538, + "balance_loss_clip": 1.04410172, + "balance_loss_mlp": 1.02102017, + "epoch": 0.7913184632958576, + "flos": 23112538899840.0, + "grad_norm": 1.489163075566481, + "language_loss": 0.76596498, + "learning_rate": 4.3953377049751416e-07, + "loss": 0.78759301, + "num_input_tokens_seen": 141812380, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.13513184, + "step": 6581, + "time_per_iteration": 2.7898433208465576 + }, + { + "auxiliary_loss_clip": 0.01126845, + "auxiliary_loss_mlp": 0.01039541, + "balance_loss_clip": 1.04047823, + "balance_loss_mlp": 1.02486587, + "epoch": 0.7914387061864967, + "flos": 12311758719360.0, + "grad_norm": 2.3020284838655116, + "language_loss": 0.78383738, + "learning_rate": 4.390466507999537e-07, + "loss": 0.80550122, + "num_input_tokens_seen": 141828130, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.14685059, + "step": 6582, + "time_per_iteration": 2.7404165267944336 + }, + { + "auxiliary_loss_clip": 0.01129192, + "auxiliary_loss_mlp": 0.01039108, + "balance_loss_clip": 1.04136348, + "balance_loss_mlp": 1.02299142, + "epoch": 0.7915589490771359, + "flos": 17603267708160.0, + "grad_norm": 2.630287303753761, + "language_loss": 0.75890929, + "learning_rate": 4.385597678977748e-07, + "loss": 0.78059232, + "num_input_tokens_seen": 141846965, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.16125488, + "step": 6583, + "time_per_iteration": 2.7255640029907227 + }, + { + "auxiliary_loss_clip": 0.01127159, + "auxiliary_loss_mlp": 0.01038267, + "balance_loss_clip": 1.03859055, + "balance_loss_mlp": 1.02148271, + "epoch": 0.7916791919677749, + "flos": 25591272641280.0, + "grad_norm": 1.6247721399577333, + "language_loss": 0.75656885, + "learning_rate": 4.3807312186483726e-07, + "loss": 0.77822316, + "num_input_tokens_seen": 141867685, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.16796875, + "step": 6584, + "time_per_iteration": 2.8046302795410156 + }, + { + "auxiliary_loss_clip": 0.01127438, + "auxiliary_loss_mlp": 0.01035904, + "balance_loss_clip": 1.04294872, + "balance_loss_mlp": 1.02133632, + "epoch": 0.791799434858414, + "flos": 18844340474880.0, + "grad_norm": 1.8857809220992134, + "language_loss": 0.78752393, + "learning_rate": 4.375867127749655e-07, + "loss": 0.80915737, + "num_input_tokens_seen": 141885960, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.14562988, + "step": 6585, + "time_per_iteration": 2.736912488937378 + }, + { + "auxiliary_loss_clip": 0.01125859, + "auxiliary_loss_mlp": 0.01032454, + "balance_loss_clip": 1.04050732, + "balance_loss_mlp": 1.01651537, + "epoch": 0.7919196777490531, + "flos": 25812015672960.0, + "grad_norm": 2.005806538001701, + "language_loss": 0.66896975, + "learning_rate": 4.3710054070194744e-07, + "loss": 0.69055289, + "num_input_tokens_seen": 141905655, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.15942383, + "step": 6586, + "time_per_iteration": 2.80121111869812 + }, + { + "auxiliary_loss_clip": 0.01131012, + "auxiliary_loss_mlp": 0.01042684, + "balance_loss_clip": 1.04199612, + "balance_loss_mlp": 1.02536297, + "epoch": 0.7920399206396922, + "flos": 11947624594560.0, + "grad_norm": 2.653023118251306, + "language_loss": 0.66858423, + "learning_rate": 4.3661460571953455e-07, + "loss": 0.69032115, + "num_input_tokens_seen": 141922390, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.17321777, + "step": 6587, + "time_per_iteration": 2.7891781330108643 + }, + { + "auxiliary_loss_clip": 0.01124016, + "auxiliary_loss_mlp": 0.01028815, + "balance_loss_clip": 1.03911769, + "balance_loss_mlp": 1.01356816, + "epoch": 0.7921601635303313, + "flos": 21579907438080.0, + "grad_norm": 1.5954512382058395, + "language_loss": 0.68507081, + "learning_rate": 4.36128907901443e-07, + "loss": 0.70659912, + "num_input_tokens_seen": 141941985, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15258789, + "step": 6588, + "time_per_iteration": 2.751983642578125 + }, + { + "auxiliary_loss_clip": 0.01124217, + "auxiliary_loss_mlp": 0.01040138, + "balance_loss_clip": 1.04035163, + "balance_loss_mlp": 1.02449775, + "epoch": 0.7922804064209703, + "flos": 18113989236480.0, + "grad_norm": 4.142785473502636, + "language_loss": 0.72550631, + "learning_rate": 4.356434473213519e-07, + "loss": 0.74714988, + "num_input_tokens_seen": 141959435, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15649414, + "step": 6589, + "time_per_iteration": 2.796368360519409 + }, + { + "auxiliary_loss_clip": 0.01129059, + "auxiliary_loss_mlp": 0.01046739, + "balance_loss_clip": 1.04253089, + "balance_loss_mlp": 1.03223085, + "epoch": 0.7924006493116095, + "flos": 21652806090240.0, + "grad_norm": 1.5749084883842592, + "language_loss": 0.79499459, + "learning_rate": 4.351582240529068e-07, + "loss": 0.81675261, + "num_input_tokens_seen": 141980265, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.1451416, + "step": 6590, + "time_per_iteration": 2.7615888118743896 + }, + { + "auxiliary_loss_clip": 0.01031578, + "auxiliary_loss_mlp": 0.01002193, + "balance_loss_clip": 1.00413501, + "balance_loss_mlp": 1.00067329, + "epoch": 0.7925208922022485, + "flos": 64242755694720.0, + "grad_norm": 0.6811519721317164, + "language_loss": 0.58160579, + "learning_rate": 4.346732381697149e-07, + "loss": 0.60194349, + "num_input_tokens_seen": 142044395, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01519775, + "step": 6591, + "time_per_iteration": 3.352144479751587 + }, + { + "auxiliary_loss_clip": 0.01126362, + "auxiliary_loss_mlp": 0.01038123, + "balance_loss_clip": 1.03973365, + "balance_loss_mlp": 1.02318597, + "epoch": 0.7926411350928876, + "flos": 16941541403520.0, + "grad_norm": 2.115395637925792, + "language_loss": 0.80970424, + "learning_rate": 4.3418848974534825e-07, + "loss": 0.83134913, + "num_input_tokens_seen": 142061335, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.14941406, + "step": 6592, + "time_per_iteration": 2.7586419582366943 + }, + { + "auxiliary_loss_clip": 0.01126639, + "auxiliary_loss_mlp": 0.01038177, + "balance_loss_clip": 1.04193974, + "balance_loss_mlp": 1.0229187, + "epoch": 0.7927613779835267, + "flos": 34459987144320.0, + "grad_norm": 1.6453475891245164, + "language_loss": 0.68620622, + "learning_rate": 4.3370397885334276e-07, + "loss": 0.70785439, + "num_input_tokens_seen": 142081965, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15258789, + "step": 6593, + "time_per_iteration": 2.8428101539611816 + }, + { + "auxiliary_loss_clip": 0.01125367, + "auxiliary_loss_mlp": 0.01038253, + "balance_loss_clip": 1.04077554, + "balance_loss_mlp": 1.02288675, + "epoch": 0.7928816208741658, + "flos": 18951174501120.0, + "grad_norm": 1.8392665689542063, + "language_loss": 0.75420308, + "learning_rate": 4.3321970556719777e-07, + "loss": 0.77583933, + "num_input_tokens_seen": 142100260, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.15362549, + "step": 6594, + "time_per_iteration": 2.766184091567993 + }, + { + "auxiliary_loss_clip": 0.01128024, + "auxiliary_loss_mlp": 0.01035282, + "balance_loss_clip": 1.04151165, + "balance_loss_mlp": 1.01934409, + "epoch": 0.7930018637648049, + "flos": 18623022825600.0, + "grad_norm": 2.478335257261148, + "language_loss": 0.71910834, + "learning_rate": 4.3273566996037856e-07, + "loss": 0.74074137, + "num_input_tokens_seen": 142116955, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.15942383, + "step": 6595, + "time_per_iteration": 2.745453119277954 + }, + { + "auxiliary_loss_clip": 0.011199, + "auxiliary_loss_mlp": 0.01035769, + "balance_loss_clip": 1.03784442, + "balance_loss_mlp": 1.02133894, + "epoch": 0.793122106655444, + "flos": 24530650824960.0, + "grad_norm": 1.8748264493076259, + "language_loss": 0.80420971, + "learning_rate": 4.322518721063113e-07, + "loss": 0.82576644, + "num_input_tokens_seen": 142135505, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.14422607, + "step": 6596, + "time_per_iteration": 2.7921946048736572 + }, + { + "auxiliary_loss_clip": 0.01129438, + "auxiliary_loss_mlp": 0.01040287, + "balance_loss_clip": 1.04157746, + "balance_loss_mlp": 1.02544522, + "epoch": 0.7932423495460831, + "flos": 34421203434240.0, + "grad_norm": 2.0508149147842603, + "language_loss": 0.70416605, + "learning_rate": 4.3176831207838906e-07, + "loss": 0.72586334, + "num_input_tokens_seen": 142158915, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.14837646, + "step": 6597, + "time_per_iteration": 3.605211019515991 + }, + { + "auxiliary_loss_clip": 0.0112718, + "auxiliary_loss_mlp": 0.0104249, + "balance_loss_clip": 1.04205894, + "balance_loss_mlp": 1.02731478, + "epoch": 0.7933625924367221, + "flos": 26980333441920.0, + "grad_norm": 1.8243450945739776, + "language_loss": 0.74328697, + "learning_rate": 4.3128498994996685e-07, + "loss": 0.76498365, + "num_input_tokens_seen": 142178390, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.1517334, + "step": 6598, + "time_per_iteration": 2.8338327407836914 + }, + { + "auxiliary_loss_clip": 0.01129326, + "auxiliary_loss_mlp": 0.01038199, + "balance_loss_clip": 1.04062343, + "balance_loss_mlp": 1.02204633, + "epoch": 0.7934828353273613, + "flos": 29568630643200.0, + "grad_norm": 2.0516331159714056, + "language_loss": 0.71207631, + "learning_rate": 4.308019057943646e-07, + "loss": 0.73375154, + "num_input_tokens_seen": 142200115, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.16149902, + "step": 6599, + "time_per_iteration": 2.8193185329437256 + }, + { + "auxiliary_loss_clip": 0.01124044, + "auxiliary_loss_mlp": 0.01043452, + "balance_loss_clip": 1.0394721, + "balance_loss_mlp": 1.02846706, + "epoch": 0.7936030782180004, + "flos": 28615381557120.0, + "grad_norm": 1.6500654648596236, + "language_loss": 0.74500299, + "learning_rate": 4.3031905968486535e-07, + "loss": 0.76667792, + "num_input_tokens_seen": 142220945, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14978027, + "step": 6600, + "time_per_iteration": 2.802544116973877 + }, + { + "auxiliary_loss_clip": 0.0112714, + "auxiliary_loss_mlp": 0.01034469, + "balance_loss_clip": 1.04354632, + "balance_loss_mlp": 1.01887655, + "epoch": 0.7937233211086394, + "flos": 16392574869120.0, + "grad_norm": 2.307906772299454, + "language_loss": 0.69209349, + "learning_rate": 4.298364516947162e-07, + "loss": 0.71370959, + "num_input_tokens_seen": 142238175, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.15600586, + "step": 6601, + "time_per_iteration": 2.733649969100952 + }, + { + "auxiliary_loss_clip": 0.01127789, + "auxiliary_loss_mlp": 0.01037194, + "balance_loss_clip": 1.04173565, + "balance_loss_mlp": 1.02175665, + "epoch": 0.7938435639992786, + "flos": 22013420682240.0, + "grad_norm": 2.0136613974846456, + "language_loss": 0.65459132, + "learning_rate": 4.293540818971295e-07, + "loss": 0.67624116, + "num_input_tokens_seen": 142255980, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15429688, + "step": 6602, + "time_per_iteration": 3.6114578247070312 + }, + { + "auxiliary_loss_clip": 0.01127255, + "auxiliary_loss_mlp": 0.01031587, + "balance_loss_clip": 1.04111934, + "balance_loss_mlp": 1.01662672, + "epoch": 0.7939638068899176, + "flos": 22197032029440.0, + "grad_norm": 15.319248491001035, + "language_loss": 0.76678747, + "learning_rate": 4.2887195036527934e-07, + "loss": 0.78837591, + "num_input_tokens_seen": 142274785, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.14953613, + "step": 6603, + "time_per_iteration": 3.616276741027832 + }, + { + "auxiliary_loss_clip": 0.01127311, + "auxiliary_loss_mlp": 0.01036104, + "balance_loss_clip": 1.04077268, + "balance_loss_mlp": 1.02001083, + "epoch": 0.7940840497805567, + "flos": 17745186343680.0, + "grad_norm": 2.9213674615260863, + "language_loss": 0.73254514, + "learning_rate": 4.28390057172306e-07, + "loss": 0.75417924, + "num_input_tokens_seen": 142291290, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.16088867, + "step": 6604, + "time_per_iteration": 2.8141865730285645 + }, + { + "auxiliary_loss_clip": 0.01129196, + "auxiliary_loss_mlp": 0.010446, + "balance_loss_clip": 1.04082298, + "balance_loss_mlp": 1.02752876, + "epoch": 0.7942042926711959, + "flos": 23805435231360.0, + "grad_norm": 2.352317048438861, + "language_loss": 0.72189462, + "learning_rate": 4.279084023913111e-07, + "loss": 0.74363256, + "num_input_tokens_seen": 142309165, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.1706543, + "step": 6605, + "time_per_iteration": 2.784829616546631 + }, + { + "auxiliary_loss_clip": 0.01121952, + "auxiliary_loss_mlp": 0.01039634, + "balance_loss_clip": 1.03962541, + "balance_loss_mlp": 1.02495956, + "epoch": 0.7943245355618349, + "flos": 19244959839360.0, + "grad_norm": 1.8815017702947245, + "language_loss": 0.69602156, + "learning_rate": 4.2742698609536096e-07, + "loss": 0.71763742, + "num_input_tokens_seen": 142327475, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.14672852, + "step": 6606, + "time_per_iteration": 2.7673089504241943 + }, + { + "auxiliary_loss_clip": 0.01124336, + "auxiliary_loss_mlp": 0.01041784, + "balance_loss_clip": 1.03946853, + "balance_loss_mlp": 1.02572668, + "epoch": 0.794444778452474, + "flos": 25007616547200.0, + "grad_norm": 3.180108880273333, + "language_loss": 0.78883016, + "learning_rate": 4.2694580835748706e-07, + "loss": 0.81049132, + "num_input_tokens_seen": 142347335, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.16064453, + "step": 6607, + "time_per_iteration": 2.7700412273406982 + }, + { + "auxiliary_loss_clip": 0.01125067, + "auxiliary_loss_mlp": 0.01037811, + "balance_loss_clip": 1.03941202, + "balance_loss_mlp": 1.02310669, + "epoch": 0.7945650213431131, + "flos": 23221491828480.0, + "grad_norm": 2.0453302071139436, + "language_loss": 0.74131596, + "learning_rate": 4.264648692506836e-07, + "loss": 0.76294476, + "num_input_tokens_seen": 142366125, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.1472168, + "step": 6608, + "time_per_iteration": 3.6952357292175293 + }, + { + "auxiliary_loss_clip": 0.01129058, + "auxiliary_loss_mlp": 0.01045261, + "balance_loss_clip": 1.04214907, + "balance_loss_mlp": 1.02889347, + "epoch": 0.7946852642337522, + "flos": 26062887237120.0, + "grad_norm": 1.7029199131547679, + "language_loss": 0.72335958, + "learning_rate": 4.2598416884790824e-07, + "loss": 0.74510282, + "num_input_tokens_seen": 142385175, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.16369629, + "step": 6609, + "time_per_iteration": 2.8027780055999756 + }, + { + "auxiliary_loss_clip": 0.01127988, + "auxiliary_loss_mlp": 0.01040181, + "balance_loss_clip": 1.03953242, + "balance_loss_mlp": 1.02409959, + "epoch": 0.7948055071243912, + "flos": 23769704177280.0, + "grad_norm": 2.6646010604590407, + "language_loss": 0.80817419, + "learning_rate": 4.255037072220828e-07, + "loss": 0.82985592, + "num_input_tokens_seen": 142406545, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.1607666, + "step": 6610, + "time_per_iteration": 2.8572499752044678 + }, + { + "auxiliary_loss_clip": 0.01120796, + "auxiliary_loss_mlp": 0.01033519, + "balance_loss_clip": 1.03842223, + "balance_loss_mlp": 1.01842749, + "epoch": 0.7949257500150304, + "flos": 21980814111360.0, + "grad_norm": 1.9194612870655534, + "language_loss": 0.71916127, + "learning_rate": 4.2502348444609293e-07, + "loss": 0.74070442, + "num_input_tokens_seen": 142426165, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.15087891, + "step": 6611, + "time_per_iteration": 2.809532880783081 + }, + { + "auxiliary_loss_clip": 0.01125236, + "auxiliary_loss_mlp": 0.01035468, + "balance_loss_clip": 1.03971386, + "balance_loss_mlp": 1.02096677, + "epoch": 0.7950459929056695, + "flos": 25774129802880.0, + "grad_norm": 1.6947462628377352, + "language_loss": 0.69460076, + "learning_rate": 4.2454350059278844e-07, + "loss": 0.7162078, + "num_input_tokens_seen": 142447225, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.14501953, + "step": 6612, + "time_per_iteration": 2.828791618347168 + }, + { + "auxiliary_loss_clip": 0.01124174, + "auxiliary_loss_mlp": 0.01036043, + "balance_loss_clip": 1.03777957, + "balance_loss_mlp": 1.02197003, + "epoch": 0.7951662357963085, + "flos": 22158068751360.0, + "grad_norm": 1.751056882441943, + "language_loss": 0.84342343, + "learning_rate": 4.240637557349824e-07, + "loss": 0.86502558, + "num_input_tokens_seen": 142464440, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.14080811, + "step": 6613, + "time_per_iteration": 2.8232157230377197 + }, + { + "auxiliary_loss_clip": 0.01122858, + "auxiliary_loss_mlp": 0.01035712, + "balance_loss_clip": 1.03877819, + "balance_loss_mlp": 1.01923752, + "epoch": 0.7952864786869477, + "flos": 24641938137600.0, + "grad_norm": 1.829602730953036, + "language_loss": 0.6698122, + "learning_rate": 4.235842499454516e-07, + "loss": 0.69139791, + "num_input_tokens_seen": 142484355, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.16467285, + "step": 6614, + "time_per_iteration": 2.784461259841919 + }, + { + "auxiliary_loss_clip": 0.01124038, + "auxiliary_loss_mlp": 0.0104493, + "balance_loss_clip": 1.04095221, + "balance_loss_mlp": 1.03007102, + "epoch": 0.7954067215775867, + "flos": 21830922656640.0, + "grad_norm": 2.362888543087969, + "language_loss": 0.82914191, + "learning_rate": 4.2310498329693687e-07, + "loss": 0.85083163, + "num_input_tokens_seen": 142505255, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.14862061, + "step": 6615, + "time_per_iteration": 2.767361640930176 + }, + { + "auxiliary_loss_clip": 0.01127899, + "auxiliary_loss_mlp": 0.0104396, + "balance_loss_clip": 1.04015148, + "balance_loss_mlp": 1.02716339, + "epoch": 0.7955269644682258, + "flos": 24060652341120.0, + "grad_norm": 1.5724301776797742, + "language_loss": 0.80558479, + "learning_rate": 4.2262595586214164e-07, + "loss": 0.82730329, + "num_input_tokens_seen": 142526350, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16796875, + "step": 6616, + "time_per_iteration": 2.9519457817077637 + }, + { + "auxiliary_loss_clip": 0.0112622, + "auxiliary_loss_mlp": 0.01041672, + "balance_loss_clip": 1.04030168, + "balance_loss_mlp": 1.02610302, + "epoch": 0.795647207358865, + "flos": 25010741030400.0, + "grad_norm": 1.8094170548270627, + "language_loss": 0.76869452, + "learning_rate": 4.221471677137358e-07, + "loss": 0.79037344, + "num_input_tokens_seen": 142547165, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15551758, + "step": 6617, + "time_per_iteration": 2.858166217803955 + }, + { + "auxiliary_loss_clip": 0.01121864, + "auxiliary_loss_mlp": 0.01037774, + "balance_loss_clip": 1.03900266, + "balance_loss_mlp": 1.02357018, + "epoch": 0.795767450249504, + "flos": 14648358343680.0, + "grad_norm": 2.9345639649583246, + "language_loss": 0.70218527, + "learning_rate": 4.216686189243492e-07, + "loss": 0.7237817, + "num_input_tokens_seen": 142565955, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.14202881, + "step": 6618, + "time_per_iteration": 2.817455291748047 + }, + { + "auxiliary_loss_clip": 0.01126061, + "auxiliary_loss_mlp": 0.01041322, + "balance_loss_clip": 1.03949571, + "balance_loss_mlp": 1.02652264, + "epoch": 0.7958876931401431, + "flos": 18547897530240.0, + "grad_norm": 1.5632988395321379, + "language_loss": 0.72458553, + "learning_rate": 4.211903095665785e-07, + "loss": 0.74625945, + "num_input_tokens_seen": 142585340, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.14801025, + "step": 6619, + "time_per_iteration": 2.7373483180999756 + }, + { + "auxiliary_loss_clip": 0.01120108, + "auxiliary_loss_mlp": 0.01033801, + "balance_loss_clip": 1.03884971, + "balance_loss_mlp": 1.01868534, + "epoch": 0.7960079360307821, + "flos": 21543960902400.0, + "grad_norm": 1.7560811050185308, + "language_loss": 0.75476289, + "learning_rate": 4.2071223971298277e-07, + "loss": 0.77630198, + "num_input_tokens_seen": 142602525, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.15112305, + "step": 6620, + "time_per_iteration": 2.7469353675842285 + }, + { + "auxiliary_loss_clip": 0.01127986, + "auxiliary_loss_mlp": 0.01033932, + "balance_loss_clip": 1.04107332, + "balance_loss_mlp": 1.01808929, + "epoch": 0.7961281789214213, + "flos": 25481745095040.0, + "grad_norm": 3.6952580763331793, + "language_loss": 0.61088264, + "learning_rate": 4.2023440943608433e-07, + "loss": 0.63250184, + "num_input_tokens_seen": 142622490, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.1583252, + "step": 6621, + "time_per_iteration": 2.8068647384643555 + }, + { + "auxiliary_loss_clip": 0.01118407, + "auxiliary_loss_mlp": 0.01035308, + "balance_loss_clip": 1.03716874, + "balance_loss_mlp": 1.02119374, + "epoch": 0.7962484218120603, + "flos": 21944436612480.0, + "grad_norm": 1.6248687800193795, + "language_loss": 0.77835375, + "learning_rate": 4.1975681880837023e-07, + "loss": 0.79989088, + "num_input_tokens_seen": 142642495, + "router_z_loss_clip": 0.81201172, + "router_z_loss_mlp": 0.14111328, + "step": 6622, + "time_per_iteration": 2.731905460357666 + }, + { + "auxiliary_loss_clip": 0.0112822, + "auxiliary_loss_mlp": 0.01037471, + "balance_loss_clip": 1.04372668, + "balance_loss_mlp": 1.02319622, + "epoch": 0.7963686647026994, + "flos": 18876264687360.0, + "grad_norm": 1.813634898981423, + "language_loss": 0.82641405, + "learning_rate": 4.192794679022895e-07, + "loss": 0.84807098, + "num_input_tokens_seen": 142660820, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.1428833, + "step": 6623, + "time_per_iteration": 2.7463936805725098 + }, + { + "auxiliary_loss_clip": 0.01123632, + "auxiliary_loss_mlp": 0.01044224, + "balance_loss_clip": 1.04004765, + "balance_loss_mlp": 1.02783298, + "epoch": 0.7964889075933386, + "flos": 29716582763520.0, + "grad_norm": 1.9791804438724023, + "language_loss": 0.72166568, + "learning_rate": 4.1880235679025743e-07, + "loss": 0.74334431, + "num_input_tokens_seen": 142680915, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.16418457, + "step": 6624, + "time_per_iteration": 2.844233751296997 + }, + { + "auxiliary_loss_clip": 0.01127954, + "auxiliary_loss_mlp": 0.01039542, + "balance_loss_clip": 1.04105783, + "balance_loss_mlp": 1.02382398, + "epoch": 0.7966091504839776, + "flos": 29491458272640.0, + "grad_norm": 1.750813971525214, + "language_loss": 0.63546956, + "learning_rate": 4.1832548554464986e-07, + "loss": 0.65714455, + "num_input_tokens_seen": 142699210, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.1572876, + "step": 6625, + "time_per_iteration": 2.8233296871185303 + }, + { + "auxiliary_loss_clip": 0.01032686, + "auxiliary_loss_mlp": 0.01001439, + "balance_loss_clip": 1.00524497, + "balance_loss_mlp": 0.99987739, + "epoch": 0.7967293933746167, + "flos": 67288697101440.0, + "grad_norm": 0.7414270689587084, + "language_loss": 0.58688462, + "learning_rate": 4.178488542378098e-07, + "loss": 0.60722589, + "num_input_tokens_seen": 142756790, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01556396, + "step": 6626, + "time_per_iteration": 4.0211005210876465 + }, + { + "auxiliary_loss_clip": 0.0112641, + "auxiliary_loss_mlp": 0.01034166, + "balance_loss_clip": 1.03962898, + "balance_loss_mlp": 1.01794124, + "epoch": 0.7968496362652558, + "flos": 25554679660800.0, + "grad_norm": 1.6500887313014752, + "language_loss": 0.88930571, + "learning_rate": 4.173724629420401e-07, + "loss": 0.91091144, + "num_input_tokens_seen": 142778150, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.16223145, + "step": 6627, + "time_per_iteration": 2.771829128265381 + }, + { + "auxiliary_loss_clip": 0.01130357, + "auxiliary_loss_mlp": 0.01037847, + "balance_loss_clip": 1.04125249, + "balance_loss_mlp": 1.02164674, + "epoch": 0.7969698791558949, + "flos": 14501088581760.0, + "grad_norm": 2.327528371467339, + "language_loss": 0.68029654, + "learning_rate": 4.168963117296087e-07, + "loss": 0.70197856, + "num_input_tokens_seen": 142795485, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.1619873, + "step": 6628, + "time_per_iteration": 2.7993431091308594 + }, + { + "auxiliary_loss_clip": 0.0112816, + "auxiliary_loss_mlp": 0.01039994, + "balance_loss_clip": 1.04209614, + "balance_loss_mlp": 1.02398443, + "epoch": 0.797090122046534, + "flos": 22127545169280.0, + "grad_norm": 2.8586656888186783, + "language_loss": 0.76362717, + "learning_rate": 4.1642040067274876e-07, + "loss": 0.78530866, + "num_input_tokens_seen": 142815155, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.16003418, + "step": 6629, + "time_per_iteration": 2.858715772628784 + }, + { + "auxiliary_loss_clip": 0.01126034, + "auxiliary_loss_mlp": 0.01046851, + "balance_loss_clip": 1.04042006, + "balance_loss_mlp": 1.03109193, + "epoch": 0.7972103649371731, + "flos": 19897671830400.0, + "grad_norm": 1.6545439704859506, + "language_loss": 0.72452176, + "learning_rate": 4.1594472984365493e-07, + "loss": 0.74625063, + "num_input_tokens_seen": 142833840, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15765381, + "step": 6630, + "time_per_iteration": 2.855588674545288 + }, + { + "auxiliary_loss_clip": 0.01125402, + "auxiliary_loss_mlp": 0.01034893, + "balance_loss_clip": 1.03856134, + "balance_loss_mlp": 1.02004528, + "epoch": 0.7973306078278122, + "flos": 36058621847040.0, + "grad_norm": 1.720164500726766, + "language_loss": 0.7754072, + "learning_rate": 4.154692993144862e-07, + "loss": 0.79701018, + "num_input_tokens_seen": 142853610, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.14855957, + "step": 6631, + "time_per_iteration": 3.767094373703003 + }, + { + "auxiliary_loss_clip": 0.01125863, + "auxiliary_loss_mlp": 0.01035016, + "balance_loss_clip": 1.03974926, + "balance_loss_mlp": 1.02000737, + "epoch": 0.7974508507184512, + "flos": 21360600950400.0, + "grad_norm": 8.716648739961448, + "language_loss": 0.71131873, + "learning_rate": 4.1499410915736476e-07, + "loss": 0.73292744, + "num_input_tokens_seen": 142872540, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.14984131, + "step": 6632, + "time_per_iteration": 3.565337896347046 + }, + { + "auxiliary_loss_clip": 0.01032748, + "auxiliary_loss_mlp": 0.01001915, + "balance_loss_clip": 1.00530767, + "balance_loss_mlp": 1.00031805, + "epoch": 0.7975710936090904, + "flos": 68253115317120.0, + "grad_norm": 0.7998632783815366, + "language_loss": 0.64212084, + "learning_rate": 4.145191594443762e-07, + "loss": 0.66246748, + "num_input_tokens_seen": 142936895, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01599121, + "step": 6633, + "time_per_iteration": 3.434591770172119 + }, + { + "auxiliary_loss_clip": 0.01131935, + "auxiliary_loss_mlp": 0.01040096, + "balance_loss_clip": 1.04398561, + "balance_loss_mlp": 1.02427709, + "epoch": 0.7976913364997295, + "flos": 22492433479680.0, + "grad_norm": 1.8163033513280449, + "language_loss": 0.70540613, + "learning_rate": 4.140444502475713e-07, + "loss": 0.72712648, + "num_input_tokens_seen": 142956445, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.15820312, + "step": 6634, + "time_per_iteration": 2.8350822925567627 + }, + { + "auxiliary_loss_clip": 0.01127548, + "auxiliary_loss_mlp": 0.01037289, + "balance_loss_clip": 1.04094183, + "balance_loss_mlp": 1.02300167, + "epoch": 0.7978115793903685, + "flos": 15263220378240.0, + "grad_norm": 3.1253218201437236, + "language_loss": 0.69704056, + "learning_rate": 4.1356998163896216e-07, + "loss": 0.71868885, + "num_input_tokens_seen": 142973495, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.1428833, + "step": 6635, + "time_per_iteration": 2.7754621505737305 + }, + { + "auxiliary_loss_clip": 0.01126369, + "auxiliary_loss_mlp": 0.01038501, + "balance_loss_clip": 1.04011846, + "balance_loss_mlp": 1.0225513, + "epoch": 0.7979318222810077, + "flos": 19719232041600.0, + "grad_norm": 2.67703781077187, + "language_loss": 0.75127876, + "learning_rate": 4.130957536905255e-07, + "loss": 0.77292752, + "num_input_tokens_seen": 142991510, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15948486, + "step": 6636, + "time_per_iteration": 2.7749884128570557 + }, + { + "auxiliary_loss_clip": 0.01128414, + "auxiliary_loss_mlp": 0.01041037, + "balance_loss_clip": 1.04199886, + "balance_loss_mlp": 1.02431202, + "epoch": 0.7980520651716467, + "flos": 15560273854080.0, + "grad_norm": 2.1990363114555334, + "language_loss": 0.71045303, + "learning_rate": 4.1262176647420134e-07, + "loss": 0.73214757, + "num_input_tokens_seen": 143009675, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.16723633, + "step": 6637, + "time_per_iteration": 3.6040310859680176 + }, + { + "auxiliary_loss_clip": 0.01126658, + "auxiliary_loss_mlp": 0.01044059, + "balance_loss_clip": 1.03965676, + "balance_loss_mlp": 1.02822793, + "epoch": 0.7981723080622858, + "flos": 22309432663680.0, + "grad_norm": 2.0647478079238626, + "language_loss": 0.7978794, + "learning_rate": 4.121480200618923e-07, + "loss": 0.81958652, + "num_input_tokens_seen": 143029330, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.15820312, + "step": 6638, + "time_per_iteration": 2.7951161861419678 + }, + { + "auxiliary_loss_clip": 0.01123348, + "auxiliary_loss_mlp": 0.01038324, + "balance_loss_clip": 1.04008079, + "balance_loss_mlp": 1.02285039, + "epoch": 0.798292550952925, + "flos": 22929573997440.0, + "grad_norm": 1.6687638899148252, + "language_loss": 0.80100048, + "learning_rate": 4.116745145254674e-07, + "loss": 0.82261717, + "num_input_tokens_seen": 143048865, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.15478516, + "step": 6639, + "time_per_iteration": 2.7557387351989746 + }, + { + "auxiliary_loss_clip": 0.01032759, + "auxiliary_loss_mlp": 0.01002833, + "balance_loss_clip": 1.00520682, + "balance_loss_mlp": 1.00137222, + "epoch": 0.798412793843564, + "flos": 64497936890880.0, + "grad_norm": 0.7635886478065216, + "language_loss": 0.57886624, + "learning_rate": 4.1120124993675476e-07, + "loss": 0.59922218, + "num_input_tokens_seen": 143113295, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.0145874, + "step": 6640, + "time_per_iteration": 3.3067805767059326 + }, + { + "auxiliary_loss_clip": 0.01130535, + "auxiliary_loss_mlp": 0.01039149, + "balance_loss_clip": 1.04155231, + "balance_loss_mlp": 1.02309144, + "epoch": 0.7985330367342031, + "flos": 13586910514560.0, + "grad_norm": 1.9528832810562502, + "language_loss": 0.61858952, + "learning_rate": 4.107282263675498e-07, + "loss": 0.64028633, + "num_input_tokens_seen": 143130965, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.16064453, + "step": 6641, + "time_per_iteration": 2.7854597568511963 + }, + { + "auxiliary_loss_clip": 0.01032549, + "auxiliary_loss_mlp": 0.01003865, + "balance_loss_clip": 1.00514483, + "balance_loss_mlp": 1.00227976, + "epoch": 0.7986532796248422, + "flos": 67698797656320.0, + "grad_norm": 0.7649319366260777, + "language_loss": 0.5247798, + "learning_rate": 4.1025544388960907e-07, + "loss": 0.5451439, + "num_input_tokens_seen": 143192005, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01586914, + "step": 6642, + "time_per_iteration": 3.2558209896087646 + }, + { + "auxiliary_loss_clip": 0.01126352, + "auxiliary_loss_mlp": 0.01044781, + "balance_loss_clip": 1.04127336, + "balance_loss_mlp": 1.02933717, + "epoch": 0.7987735225154813, + "flos": 22455373622400.0, + "grad_norm": 1.8072353316039897, + "language_loss": 0.71707499, + "learning_rate": 4.097829025746538e-07, + "loss": 0.73878634, + "num_input_tokens_seen": 143213550, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.15441895, + "step": 6643, + "time_per_iteration": 2.7682833671569824 + }, + { + "auxiliary_loss_clip": 0.01031845, + "auxiliary_loss_mlp": 0.01002962, + "balance_loss_clip": 1.00449109, + "balance_loss_mlp": 1.00138843, + "epoch": 0.7988937654061203, + "flos": 68864098682880.0, + "grad_norm": 0.6580605823387718, + "language_loss": 0.6102621, + "learning_rate": 4.0931060249436757e-07, + "loss": 0.63061017, + "num_input_tokens_seen": 143277390, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01574707, + "step": 6644, + "time_per_iteration": 3.3188610076904297 + }, + { + "auxiliary_loss_clip": 0.01125157, + "auxiliary_loss_mlp": 0.01041299, + "balance_loss_clip": 1.04026175, + "balance_loss_mlp": 1.02648115, + "epoch": 0.7990140082967595, + "flos": 20806893820800.0, + "grad_norm": 2.09523582402868, + "language_loss": 0.69686311, + "learning_rate": 4.088385437203978e-07, + "loss": 0.71852767, + "num_input_tokens_seen": 143294400, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.14819336, + "step": 6645, + "time_per_iteration": 2.749750852584839 + }, + { + "auxiliary_loss_clip": 0.01125835, + "auxiliary_loss_mlp": 0.01043888, + "balance_loss_clip": 1.03925562, + "balance_loss_mlp": 1.02846253, + "epoch": 0.7991342511873986, + "flos": 18985289443200.0, + "grad_norm": 2.1507743350432866, + "language_loss": 0.77179003, + "learning_rate": 4.083667263243564e-07, + "loss": 0.79348719, + "num_input_tokens_seen": 143312745, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.15435791, + "step": 6646, + "time_per_iteration": 2.7606027126312256 + }, + { + "auxiliary_loss_clip": 0.01123668, + "auxiliary_loss_mlp": 0.01034522, + "balance_loss_clip": 1.03960061, + "balance_loss_mlp": 1.01906645, + "epoch": 0.7992544940780376, + "flos": 20816805974400.0, + "grad_norm": 1.5995713083812015, + "language_loss": 0.71685505, + "learning_rate": 4.0789515037781653e-07, + "loss": 0.73843694, + "num_input_tokens_seen": 143333470, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15460205, + "step": 6647, + "time_per_iteration": 2.8155057430267334 + }, + { + "auxiliary_loss_clip": 0.01126732, + "auxiliary_loss_mlp": 0.01039642, + "balance_loss_clip": 1.04115343, + "balance_loss_mlp": 1.02327454, + "epoch": 0.7993747369686768, + "flos": 12640772321280.0, + "grad_norm": 1.9570060046081006, + "language_loss": 0.82499921, + "learning_rate": 4.0742381595231755e-07, + "loss": 0.84666294, + "num_input_tokens_seen": 143350195, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.16381836, + "step": 6648, + "time_per_iteration": 2.754777193069458 + }, + { + "auxiliary_loss_clip": 0.01123702, + "auxiliary_loss_mlp": 0.01036466, + "balance_loss_clip": 1.03912735, + "balance_loss_mlp": 1.02254784, + "epoch": 0.7994949798593158, + "flos": 20078769225600.0, + "grad_norm": 2.060673684188266, + "language_loss": 0.78628027, + "learning_rate": 4.06952723119359e-07, + "loss": 0.80788195, + "num_input_tokens_seen": 143370070, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.13928223, + "step": 6649, + "time_per_iteration": 2.747620105743408 + }, + { + "auxiliary_loss_clip": 0.01124722, + "auxiliary_loss_mlp": 0.01039185, + "balance_loss_clip": 1.04059577, + "balance_loss_mlp": 1.0236218, + "epoch": 0.7996152227499549, + "flos": 38654209509120.0, + "grad_norm": 2.0270677584994585, + "language_loss": 0.67179877, + "learning_rate": 4.0648187195040504e-07, + "loss": 0.69343781, + "num_input_tokens_seen": 143392275, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15557861, + "step": 6650, + "time_per_iteration": 2.900252342224121 + }, + { + "auxiliary_loss_clip": 0.01031802, + "auxiliary_loss_mlp": 0.01000424, + "balance_loss_clip": 1.00438046, + "balance_loss_mlp": 0.99886286, + "epoch": 0.799735465640594, + "flos": 70243821947520.0, + "grad_norm": 1.0235138349262543, + "language_loss": 0.67615843, + "learning_rate": 4.060112625168848e-07, + "loss": 0.69648075, + "num_input_tokens_seen": 143457385, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01556396, + "step": 6651, + "time_per_iteration": 3.3929874897003174 + }, + { + "auxiliary_loss_clip": 0.01126164, + "auxiliary_loss_mlp": 0.01038826, + "balance_loss_clip": 1.04068315, + "balance_loss_mlp": 1.0232935, + "epoch": 0.7998557085312331, + "flos": 24240995550720.0, + "grad_norm": 1.7433868459838728, + "language_loss": 0.73909664, + "learning_rate": 4.055408948901886e-07, + "loss": 0.7607466, + "num_input_tokens_seen": 143478785, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15539551, + "step": 6652, + "time_per_iteration": 3.2013320922851562 + }, + { + "auxiliary_loss_clip": 0.011284, + "auxiliary_loss_mlp": 0.01040903, + "balance_loss_clip": 1.04194522, + "balance_loss_mlp": 1.02449942, + "epoch": 0.7999759514218722, + "flos": 27564025449600.0, + "grad_norm": 1.7075807040674653, + "language_loss": 0.71052063, + "learning_rate": 4.050707691416708e-07, + "loss": 0.73221374, + "num_input_tokens_seen": 143500095, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.1640625, + "step": 6653, + "time_per_iteration": 2.8790676593780518 + }, + { + "auxiliary_loss_clip": 0.01032207, + "auxiliary_loss_mlp": 0.01003724, + "balance_loss_clip": 1.00460327, + "balance_loss_mlp": 1.00229919, + "epoch": 0.8000961943125112, + "flos": 67337428878720.0, + "grad_norm": 0.6794392067745482, + "language_loss": 0.59744143, + "learning_rate": 4.046008853426495e-07, + "loss": 0.61780071, + "num_input_tokens_seen": 143563410, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01422119, + "step": 6654, + "time_per_iteration": 3.3795087337493896 + }, + { + "auxiliary_loss_clip": 0.01127755, + "auxiliary_loss_mlp": 0.01036665, + "balance_loss_clip": 1.03966129, + "balance_loss_mlp": 1.02017879, + "epoch": 0.8002164372031504, + "flos": 28733815676160.0, + "grad_norm": 1.6819812547561916, + "language_loss": 0.62120658, + "learning_rate": 4.0413124356440464e-07, + "loss": 0.64285076, + "num_input_tokens_seen": 143587455, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.16491699, + "step": 6655, + "time_per_iteration": 3.600586175918579 + }, + { + "auxiliary_loss_clip": 0.01128582, + "auxiliary_loss_mlp": 0.01039068, + "balance_loss_clip": 1.04046726, + "balance_loss_mlp": 1.02399409, + "epoch": 0.8003366800937894, + "flos": 17639429725440.0, + "grad_norm": 2.7627221356398834, + "language_loss": 0.82414585, + "learning_rate": 4.0366184387818223e-07, + "loss": 0.84582233, + "num_input_tokens_seen": 143605915, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.15081787, + "step": 6656, + "time_per_iteration": 2.7758302688598633 + }, + { + "auxiliary_loss_clip": 0.01132086, + "auxiliary_loss_mlp": 0.01039893, + "balance_loss_clip": 1.04170656, + "balance_loss_mlp": 1.02405024, + "epoch": 0.8004569229844285, + "flos": 25995303797760.0, + "grad_norm": 1.8544080462333765, + "language_loss": 0.8533901, + "learning_rate": 4.0319268635518797e-07, + "loss": 0.87510997, + "num_input_tokens_seen": 143626490, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.15844727, + "step": 6657, + "time_per_iteration": 2.995492935180664 + }, + { + "auxiliary_loss_clip": 0.01123474, + "auxiliary_loss_mlp": 0.0103129, + "balance_loss_clip": 1.04041564, + "balance_loss_mlp": 1.01699686, + "epoch": 0.8005771658750677, + "flos": 20812352688000.0, + "grad_norm": 1.59497027957134, + "language_loss": 0.75223416, + "learning_rate": 4.027237710665943e-07, + "loss": 0.77378178, + "num_input_tokens_seen": 143644955, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.14282227, + "step": 6658, + "time_per_iteration": 2.8719642162323 + }, + { + "auxiliary_loss_clip": 0.01125496, + "auxiliary_loss_mlp": 0.01039149, + "balance_loss_clip": 1.03889561, + "balance_loss_mlp": 1.02394938, + "epoch": 0.8006974087657067, + "flos": 25812626204160.0, + "grad_norm": 1.7766577110455521, + "language_loss": 0.69231701, + "learning_rate": 4.022550980835344e-07, + "loss": 0.71396339, + "num_input_tokens_seen": 143667200, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15203857, + "step": 6659, + "time_per_iteration": 3.698488235473633 + }, + { + "auxiliary_loss_clip": 0.01124554, + "auxiliary_loss_mlp": 0.0103659, + "balance_loss_clip": 1.03821969, + "balance_loss_mlp": 1.02181435, + "epoch": 0.8008176516563458, + "flos": 17164690646400.0, + "grad_norm": 2.4878624712055677, + "language_loss": 0.79067397, + "learning_rate": 4.017866674771051e-07, + "loss": 0.81228536, + "num_input_tokens_seen": 143684685, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.14776611, + "step": 6660, + "time_per_iteration": 3.7859153747558594 + }, + { + "auxiliary_loss_clip": 0.01128103, + "auxiliary_loss_mlp": 0.01037583, + "balance_loss_clip": 1.0410769, + "balance_loss_mlp": 1.02203774, + "epoch": 0.8009378945469849, + "flos": 24207311571840.0, + "grad_norm": 1.6728956562620623, + "language_loss": 0.74394512, + "learning_rate": 4.013184793183688e-07, + "loss": 0.76560199, + "num_input_tokens_seen": 143706780, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15533447, + "step": 6661, + "time_per_iteration": 2.9830265045166016 + }, + { + "auxiliary_loss_clip": 0.01123135, + "auxiliary_loss_mlp": 0.01040021, + "balance_loss_clip": 1.03812635, + "balance_loss_mlp": 1.02540016, + "epoch": 0.801058137437624, + "flos": 19787318271360.0, + "grad_norm": 1.7220931318816473, + "language_loss": 0.72852337, + "learning_rate": 4.008505336783472e-07, + "loss": 0.75015497, + "num_input_tokens_seen": 143724505, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.1461792, + "step": 6662, + "time_per_iteration": 2.9549546241760254 + }, + { + "auxiliary_loss_clip": 0.01117802, + "auxiliary_loss_mlp": 0.01030136, + "balance_loss_clip": 1.03845763, + "balance_loss_mlp": 1.01633191, + "epoch": 0.801178380328263, + "flos": 18659400324480.0, + "grad_norm": 1.9188636080357218, + "language_loss": 0.80753297, + "learning_rate": 4.003828306280284e-07, + "loss": 0.82901233, + "num_input_tokens_seen": 143742180, + "router_z_loss_clip": 0.79394531, + "router_z_loss_mlp": 0.13812256, + "step": 6663, + "time_per_iteration": 2.9915266036987305 + }, + { + "auxiliary_loss_clip": 0.0112292, + "auxiliary_loss_mlp": 0.01035305, + "balance_loss_clip": 1.0405339, + "balance_loss_mlp": 1.02021933, + "epoch": 0.8012986232189022, + "flos": 15706573948800.0, + "grad_norm": 1.6471835160205879, + "language_loss": 0.77888429, + "learning_rate": 3.999153702383626e-07, + "loss": 0.80046654, + "num_input_tokens_seen": 143760070, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.15087891, + "step": 6664, + "time_per_iteration": 2.9607222080230713 + }, + { + "auxiliary_loss_clip": 0.01127848, + "auxiliary_loss_mlp": 0.0103652, + "balance_loss_clip": 1.04074597, + "balance_loss_mlp": 1.02089119, + "epoch": 0.8014188661095413, + "flos": 28584139703040.0, + "grad_norm": 3.130624205739196, + "language_loss": 0.73720098, + "learning_rate": 3.9944815258026263e-07, + "loss": 0.75884461, + "num_input_tokens_seen": 143781890, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15625, + "step": 6665, + "time_per_iteration": 4.262771844863892 + }, + { + "auxiliary_loss_clip": 0.01126889, + "auxiliary_loss_mlp": 0.01037101, + "balance_loss_clip": 1.03909683, + "balance_loss_mlp": 1.02124643, + "epoch": 0.8015391090001803, + "flos": 29310360877440.0, + "grad_norm": 1.7051181794321328, + "language_loss": 0.82966173, + "learning_rate": 3.989811777246057e-07, + "loss": 0.85130155, + "num_input_tokens_seen": 143802060, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.15844727, + "step": 6666, + "time_per_iteration": 3.11431884765625 + }, + { + "auxiliary_loss_clip": 0.01031921, + "auxiliary_loss_mlp": 0.01003757, + "balance_loss_clip": 1.00419509, + "balance_loss_mlp": 1.0023917, + "epoch": 0.8016593518908195, + "flos": 70397340675840.0, + "grad_norm": 0.8490620862080129, + "language_loss": 0.66261542, + "learning_rate": 3.985144457422305e-07, + "loss": 0.68297219, + "num_input_tokens_seen": 143856345, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01367188, + "step": 6667, + "time_per_iteration": 3.4957797527313232 + }, + { + "auxiliary_loss_clip": 0.01125987, + "auxiliary_loss_mlp": 0.01042034, + "balance_loss_clip": 1.04080892, + "balance_loss_mlp": 1.02682292, + "epoch": 0.8017795947814585, + "flos": 26026114688640.0, + "grad_norm": 1.9771396594425115, + "language_loss": 0.76782429, + "learning_rate": 3.9804795670394096e-07, + "loss": 0.78950453, + "num_input_tokens_seen": 143876470, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15209961, + "step": 6668, + "time_per_iteration": 2.86645770072937 + }, + { + "auxiliary_loss_clip": 0.01125113, + "auxiliary_loss_mlp": 0.01038615, + "balance_loss_clip": 1.04059696, + "balance_loss_mlp": 1.02318931, + "epoch": 0.8018998376720976, + "flos": 22087181260800.0, + "grad_norm": 1.5298291959743828, + "language_loss": 0.70490944, + "learning_rate": 3.975817106805022e-07, + "loss": 0.72654676, + "num_input_tokens_seen": 143895170, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15429688, + "step": 6669, + "time_per_iteration": 2.7998580932617188 + }, + { + "auxiliary_loss_clip": 0.01128545, + "auxiliary_loss_mlp": 0.01043149, + "balance_loss_clip": 1.04235184, + "balance_loss_mlp": 1.02656698, + "epoch": 0.8020200805627368, + "flos": 34568545023360.0, + "grad_norm": 1.9221181154037807, + "language_loss": 0.65075457, + "learning_rate": 3.97115707742645e-07, + "loss": 0.67247152, + "num_input_tokens_seen": 143915845, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.16564941, + "step": 6670, + "time_per_iteration": 2.9026691913604736 + }, + { + "auxiliary_loss_clip": 0.01124822, + "auxiliary_loss_mlp": 0.01039884, + "balance_loss_clip": 1.04100537, + "balance_loss_mlp": 1.02399385, + "epoch": 0.8021403234533758, + "flos": 20120354196480.0, + "grad_norm": 1.8826104111056698, + "language_loss": 0.65348625, + "learning_rate": 3.966499479610599e-07, + "loss": 0.67513329, + "num_input_tokens_seen": 143933940, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.15869141, + "step": 6671, + "time_per_iteration": 2.865236282348633 + }, + { + "auxiliary_loss_clip": 0.01126078, + "auxiliary_loss_mlp": 0.01038966, + "balance_loss_clip": 1.04230976, + "balance_loss_mlp": 1.02414811, + "epoch": 0.8022605663440149, + "flos": 27746200252800.0, + "grad_norm": 2.692568629284159, + "language_loss": 0.65090275, + "learning_rate": 3.9618443140640225e-07, + "loss": 0.67255312, + "num_input_tokens_seen": 143952850, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14819336, + "step": 6672, + "time_per_iteration": 2.780243396759033 + }, + { + "auxiliary_loss_clip": 0.01031958, + "auxiliary_loss_mlp": 0.01000788, + "balance_loss_clip": 1.0044136, + "balance_loss_mlp": 0.99932808, + "epoch": 0.802380809234654, + "flos": 60244998768000.0, + "grad_norm": 0.6816696860512126, + "language_loss": 0.51301116, + "learning_rate": 3.957191581492918e-07, + "loss": 0.53333867, + "num_input_tokens_seen": 144013610, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.0145874, + "step": 6673, + "time_per_iteration": 3.3281044960021973 + }, + { + "auxiliary_loss_clip": 0.01128558, + "auxiliary_loss_mlp": 0.01035368, + "balance_loss_clip": 1.04186714, + "balance_loss_mlp": 1.01920283, + "epoch": 0.8025010521252931, + "flos": 15080722352640.0, + "grad_norm": 2.8873846052503285, + "language_loss": 0.70668977, + "learning_rate": 3.952541282603097e-07, + "loss": 0.72832906, + "num_input_tokens_seen": 144028715, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.16174316, + "step": 6674, + "time_per_iteration": 2.8050405979156494 + }, + { + "auxiliary_loss_clip": 0.0112724, + "auxiliary_loss_mlp": 0.01038148, + "balance_loss_clip": 1.04054117, + "balance_loss_mlp": 1.02225769, + "epoch": 0.8026212950159322, + "flos": 22163527618560.0, + "grad_norm": 1.788656521779844, + "language_loss": 0.83441985, + "learning_rate": 3.9478934181000013e-07, + "loss": 0.85607374, + "num_input_tokens_seen": 144048740, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15881348, + "step": 6675, + "time_per_iteration": 2.940396308898926 + }, + { + "auxiliary_loss_clip": 0.01130293, + "auxiliary_loss_mlp": 0.01036263, + "balance_loss_clip": 1.04061699, + "balance_loss_mlp": 1.02008617, + "epoch": 0.8027415379065713, + "flos": 17675986792320.0, + "grad_norm": 3.746259072547551, + "language_loss": 0.8442862, + "learning_rate": 3.943247988688714e-07, + "loss": 0.86595178, + "num_input_tokens_seen": 144067435, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.16162109, + "step": 6676, + "time_per_iteration": 2.756016254425049 + }, + { + "auxiliary_loss_clip": 0.01120858, + "auxiliary_loss_mlp": 0.01035148, + "balance_loss_clip": 1.03901768, + "balance_loss_mlp": 1.02024674, + "epoch": 0.8028617807972104, + "flos": 21979593048960.0, + "grad_norm": 1.6183323969287757, + "language_loss": 0.71697742, + "learning_rate": 3.938604995073933e-07, + "loss": 0.73853749, + "num_input_tokens_seen": 144085905, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.14916992, + "step": 6677, + "time_per_iteration": 2.743833065032959 + }, + { + "auxiliary_loss_clip": 0.01125567, + "auxiliary_loss_mlp": 0.01037873, + "balance_loss_clip": 1.04049325, + "balance_loss_mlp": 1.02161312, + "epoch": 0.8029820236878494, + "flos": 26428457905920.0, + "grad_norm": 2.1743552668254247, + "language_loss": 0.65451002, + "learning_rate": 3.9339644379600157e-07, + "loss": 0.67614448, + "num_input_tokens_seen": 144105735, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.16259766, + "step": 6678, + "time_per_iteration": 2.8162214756011963 + }, + { + "auxiliary_loss_clip": 0.01125372, + "auxiliary_loss_mlp": 0.01035788, + "balance_loss_clip": 1.04037023, + "balance_loss_mlp": 1.02063036, + "epoch": 0.8031022665784886, + "flos": 17676489582720.0, + "grad_norm": 1.7511754628571334, + "language_loss": 0.71459949, + "learning_rate": 3.929326318050907e-07, + "loss": 0.73621112, + "num_input_tokens_seen": 144123405, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15161133, + "step": 6679, + "time_per_iteration": 2.75408935546875 + }, + { + "auxiliary_loss_clip": 0.01120252, + "auxiliary_loss_mlp": 0.01038661, + "balance_loss_clip": 1.03693032, + "balance_loss_mlp": 1.02396262, + "epoch": 0.8032225094691277, + "flos": 15450279431040.0, + "grad_norm": 1.8783955681329922, + "language_loss": 0.78656167, + "learning_rate": 3.924690636050225e-07, + "loss": 0.80815083, + "num_input_tokens_seen": 144140815, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.14697266, + "step": 6680, + "time_per_iteration": 2.8784022331237793 + }, + { + "auxiliary_loss_clip": 0.01126497, + "auxiliary_loss_mlp": 0.01042329, + "balance_loss_clip": 1.04032779, + "balance_loss_mlp": 1.02608037, + "epoch": 0.8033427523597667, + "flos": 26179202453760.0, + "grad_norm": 2.040847117736461, + "language_loss": 0.72854906, + "learning_rate": 3.9200573926611915e-07, + "loss": 0.75023729, + "num_input_tokens_seen": 144162230, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.16247559, + "step": 6681, + "time_per_iteration": 2.811605215072632 + }, + { + "auxiliary_loss_clip": 0.011297, + "auxiliary_loss_mlp": 0.01041127, + "balance_loss_clip": 1.04278958, + "balance_loss_mlp": 1.02570152, + "epoch": 0.8034629952504058, + "flos": 21324905809920.0, + "grad_norm": 2.1912285836202954, + "language_loss": 0.72832394, + "learning_rate": 3.9154265885866613e-07, + "loss": 0.75003219, + "num_input_tokens_seen": 144181540, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.15441895, + "step": 6682, + "time_per_iteration": 2.857201099395752 + }, + { + "auxiliary_loss_clip": 0.01127678, + "auxiliary_loss_mlp": 0.01044074, + "balance_loss_clip": 1.0406636, + "balance_loss_mlp": 1.02798712, + "epoch": 0.8035832381410449, + "flos": 21651585027840.0, + "grad_norm": 3.9140651271163005, + "language_loss": 0.74949616, + "learning_rate": 3.9107982245291394e-07, + "loss": 0.77121365, + "num_input_tokens_seen": 144199665, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.16101074, + "step": 6683, + "time_per_iteration": 3.7621419429779053 + }, + { + "auxiliary_loss_clip": 0.0112967, + "auxiliary_loss_mlp": 0.01041178, + "balance_loss_clip": 1.04278445, + "balance_loss_mlp": 1.02548993, + "epoch": 0.803703481031684, + "flos": 20518818744960.0, + "grad_norm": 1.8038791949413104, + "language_loss": 0.77379185, + "learning_rate": 3.9061723011907245e-07, + "loss": 0.79550028, + "num_input_tokens_seen": 144219020, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.15686035, + "step": 6684, + "time_per_iteration": 2.787869691848755 + }, + { + "auxiliary_loss_clip": 0.01130098, + "auxiliary_loss_mlp": 0.01045102, + "balance_loss_clip": 1.04185891, + "balance_loss_mlp": 1.02859116, + "epoch": 0.803823723922323, + "flos": 22854807838080.0, + "grad_norm": 2.0885314069222747, + "language_loss": 0.79638541, + "learning_rate": 3.901548819273179e-07, + "loss": 0.81813741, + "num_input_tokens_seen": 144239035, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.16503906, + "step": 6685, + "time_per_iteration": 2.8136584758758545 + }, + { + "auxiliary_loss_clip": 0.01126579, + "auxiliary_loss_mlp": 0.01044828, + "balance_loss_clip": 1.04116523, + "balance_loss_mlp": 1.02936661, + "epoch": 0.8039439668129622, + "flos": 21362145235200.0, + "grad_norm": 2.0937281374721355, + "language_loss": 0.69185203, + "learning_rate": 3.896927779477881e-07, + "loss": 0.71356606, + "num_input_tokens_seen": 144258295, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15454102, + "step": 6686, + "time_per_iteration": 2.8013858795166016 + }, + { + "auxiliary_loss_clip": 0.01125981, + "auxiliary_loss_mlp": 0.01039423, + "balance_loss_clip": 1.04149663, + "balance_loss_mlp": 1.02473617, + "epoch": 0.8040642097036013, + "flos": 23802382575360.0, + "grad_norm": 2.1378830494668684, + "language_loss": 0.67198163, + "learning_rate": 3.892309182505833e-07, + "loss": 0.69363564, + "num_input_tokens_seen": 144276110, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.14697266, + "step": 6687, + "time_per_iteration": 2.7993268966674805 + }, + { + "auxiliary_loss_clip": 0.01127619, + "auxiliary_loss_mlp": 0.01037131, + "balance_loss_clip": 1.04063284, + "balance_loss_mlp": 1.02232528, + "epoch": 0.8041844525942403, + "flos": 25922046009600.0, + "grad_norm": 2.3948710063648933, + "language_loss": 0.86337173, + "learning_rate": 3.887693029057675e-07, + "loss": 0.88501918, + "num_input_tokens_seen": 144295620, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.14794922, + "step": 6688, + "time_per_iteration": 3.6126668453216553 + }, + { + "auxiliary_loss_clip": 0.01120033, + "auxiliary_loss_mlp": 0.01039582, + "balance_loss_clip": 1.03903031, + "balance_loss_mlp": 1.0252707, + "epoch": 0.8043046954848795, + "flos": 25191120153600.0, + "grad_norm": 1.6502849679850806, + "language_loss": 0.81539226, + "learning_rate": 3.8830793198336684e-07, + "loss": 0.83698845, + "num_input_tokens_seen": 144315210, + "router_z_loss_clip": 0.81054688, + "router_z_loss_mlp": 0.14312744, + "step": 6689, + "time_per_iteration": 3.6149752140045166 + }, + { + "auxiliary_loss_clip": 0.01124522, + "auxiliary_loss_mlp": 0.01036373, + "balance_loss_clip": 1.03849578, + "balance_loss_mlp": 1.02060103, + "epoch": 0.8044249383755185, + "flos": 41719185123840.0, + "grad_norm": 1.712510314929142, + "language_loss": 0.70460612, + "learning_rate": 3.878468055533721e-07, + "loss": 0.72621506, + "num_input_tokens_seen": 144337750, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.15753174, + "step": 6690, + "time_per_iteration": 2.9832732677459717 + }, + { + "auxiliary_loss_clip": 0.011321, + "auxiliary_loss_mlp": 0.01045415, + "balance_loss_clip": 1.04325199, + "balance_loss_mlp": 1.02907181, + "epoch": 0.8045451812661576, + "flos": 20631434860800.0, + "grad_norm": 2.4533041572522554, + "language_loss": 0.84487569, + "learning_rate": 3.8738592368573464e-07, + "loss": 0.86665082, + "num_input_tokens_seen": 144355305, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.16326904, + "step": 6691, + "time_per_iteration": 2.799795389175415 + }, + { + "auxiliary_loss_clip": 0.01131491, + "auxiliary_loss_mlp": 0.01034938, + "balance_loss_clip": 1.0423274, + "balance_loss_mlp": 1.01953614, + "epoch": 0.8046654241567968, + "flos": 29711806254720.0, + "grad_norm": 1.9518524465730933, + "language_loss": 0.88054621, + "learning_rate": 3.8692528645037137e-07, + "loss": 0.90221053, + "num_input_tokens_seen": 144374485, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.1541748, + "step": 6692, + "time_per_iteration": 2.899529218673706 + }, + { + "auxiliary_loss_clip": 0.01125536, + "auxiliary_loss_mlp": 0.01037471, + "balance_loss_clip": 1.04027283, + "balance_loss_mlp": 1.02277279, + "epoch": 0.8047856670474358, + "flos": 17671389851520.0, + "grad_norm": 2.217224200415831, + "language_loss": 0.77820003, + "learning_rate": 3.8646489391715907e-07, + "loss": 0.79983014, + "num_input_tokens_seen": 144388780, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.14709473, + "step": 6693, + "time_per_iteration": 2.782787322998047 + }, + { + "auxiliary_loss_clip": 0.01127828, + "auxiliary_loss_mlp": 0.0103758, + "balance_loss_clip": 1.042027, + "balance_loss_mlp": 1.02203524, + "epoch": 0.8049059099380749, + "flos": 17120699464320.0, + "grad_norm": 2.0678226078729454, + "language_loss": 0.88222706, + "learning_rate": 3.8600474615593903e-07, + "loss": 0.90388119, + "num_input_tokens_seen": 144403395, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15533447, + "step": 6694, + "time_per_iteration": 3.571254014968872 + }, + { + "auxiliary_loss_clip": 0.01032156, + "auxiliary_loss_mlp": 0.01001752, + "balance_loss_clip": 1.00455165, + "balance_loss_mlp": 1.00039887, + "epoch": 0.805026152828714, + "flos": 62212903240320.0, + "grad_norm": 0.7811897064322127, + "language_loss": 0.59673822, + "learning_rate": 3.8554484323651605e-07, + "loss": 0.61707723, + "num_input_tokens_seen": 144465265, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.0135498, + "step": 6695, + "time_per_iteration": 3.287731170654297 + }, + { + "auxiliary_loss_clip": 0.01123531, + "auxiliary_loss_mlp": 0.01042459, + "balance_loss_clip": 1.04028416, + "balance_loss_mlp": 1.02825558, + "epoch": 0.8051463957193531, + "flos": 21688608971520.0, + "grad_norm": 1.524155272051201, + "language_loss": 0.78885001, + "learning_rate": 3.85085185228657e-07, + "loss": 0.81050992, + "num_input_tokens_seen": 144484235, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.14190674, + "step": 6696, + "time_per_iteration": 2.851752281188965 + }, + { + "auxiliary_loss_clip": 0.01126157, + "auxiliary_loss_mlp": 0.01040767, + "balance_loss_clip": 1.04020822, + "balance_loss_mlp": 1.02606833, + "epoch": 0.8052666386099921, + "flos": 32051458535040.0, + "grad_norm": 2.110663053950855, + "language_loss": 0.72901559, + "learning_rate": 3.8462577220209114e-07, + "loss": 0.7506848, + "num_input_tokens_seen": 144504610, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.14697266, + "step": 6697, + "time_per_iteration": 2.802570104598999 + }, + { + "auxiliary_loss_clip": 0.01031622, + "auxiliary_loss_mlp": 0.01002352, + "balance_loss_clip": 1.00419879, + "balance_loss_mlp": 1.00094497, + "epoch": 0.8053868815006313, + "flos": 67157875768320.0, + "grad_norm": 0.7078369772953957, + "language_loss": 0.58976734, + "learning_rate": 3.8416660422651127e-07, + "loss": 0.61010706, + "num_input_tokens_seen": 144574260, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01403809, + "step": 6698, + "time_per_iteration": 3.3559484481811523 + }, + { + "auxiliary_loss_clip": 0.01127733, + "auxiliary_loss_mlp": 0.01038191, + "balance_loss_clip": 1.04012227, + "balance_loss_mlp": 1.02277744, + "epoch": 0.8055071243912704, + "flos": 23837000307840.0, + "grad_norm": 1.757449917000916, + "language_loss": 0.67780262, + "learning_rate": 3.837076813715723e-07, + "loss": 0.69946188, + "num_input_tokens_seen": 144594145, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.15405273, + "step": 6699, + "time_per_iteration": 2.7751376628875732 + }, + { + "auxiliary_loss_clip": 0.0113264, + "auxiliary_loss_mlp": 0.01038762, + "balance_loss_clip": 1.0436523, + "balance_loss_mlp": 1.02212, + "epoch": 0.8056273672819094, + "flos": 21324510760320.0, + "grad_norm": 1.7387179499000829, + "language_loss": 0.75375277, + "learning_rate": 3.832490037068941e-07, + "loss": 0.7754668, + "num_input_tokens_seen": 144612935, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.16650391, + "step": 6700, + "time_per_iteration": 2.824899435043335 + }, + { + "auxiliary_loss_clip": 0.0112155, + "auxiliary_loss_mlp": 0.01043171, + "balance_loss_clip": 1.03744757, + "balance_loss_mlp": 1.02772141, + "epoch": 0.8057476101725486, + "flos": 25768383626880.0, + "grad_norm": 1.7810670296648623, + "language_loss": 0.7567085, + "learning_rate": 3.827905713020554e-07, + "loss": 0.77835566, + "num_input_tokens_seen": 144630580, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15454102, + "step": 6701, + "time_per_iteration": 2.835275173187256 + }, + { + "auxiliary_loss_clip": 0.01133833, + "auxiliary_loss_mlp": 0.01043385, + "balance_loss_clip": 1.0436933, + "balance_loss_mlp": 1.02620721, + "epoch": 0.8058678530631876, + "flos": 24535283679360.0, + "grad_norm": 2.142522580158281, + "language_loss": 0.68758464, + "learning_rate": 3.823323842266017e-07, + "loss": 0.70935684, + "num_input_tokens_seen": 144649975, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.171875, + "step": 6702, + "time_per_iteration": 2.7787084579467773 + }, + { + "auxiliary_loss_clip": 0.01126906, + "auxiliary_loss_mlp": 0.01034723, + "balance_loss_clip": 1.04026461, + "balance_loss_mlp": 1.01980972, + "epoch": 0.8059880959538267, + "flos": 24753728240640.0, + "grad_norm": 2.840140689783453, + "language_loss": 0.73033798, + "learning_rate": 3.818744425500393e-07, + "loss": 0.7519542, + "num_input_tokens_seen": 144667990, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.14916992, + "step": 6703, + "time_per_iteration": 2.867361307144165 + }, + { + "auxiliary_loss_clip": 0.01125519, + "auxiliary_loss_mlp": 0.01036793, + "balance_loss_clip": 1.04056823, + "balance_loss_mlp": 1.02229691, + "epoch": 0.8061083388444659, + "flos": 22196349671040.0, + "grad_norm": 1.7043205864118407, + "language_loss": 0.80588436, + "learning_rate": 3.8141674634183675e-07, + "loss": 0.8275075, + "num_input_tokens_seen": 144687020, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.14489746, + "step": 6704, + "time_per_iteration": 2.7698326110839844 + }, + { + "auxiliary_loss_clip": 0.01125649, + "auxiliary_loss_mlp": 0.01039404, + "balance_loss_clip": 1.04139042, + "balance_loss_mlp": 1.02436578, + "epoch": 0.8062285817351049, + "flos": 30044195735040.0, + "grad_norm": 3.399757574380384, + "language_loss": 0.66466844, + "learning_rate": 3.809592956714278e-07, + "loss": 0.68631899, + "num_input_tokens_seen": 144710255, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.15045166, + "step": 6705, + "time_per_iteration": 2.856924057006836 + }, + { + "auxiliary_loss_clip": 0.01132203, + "auxiliary_loss_mlp": 0.01041728, + "balance_loss_clip": 1.0450294, + "balance_loss_mlp": 1.02633846, + "epoch": 0.806348824625744, + "flos": 22782591544320.0, + "grad_norm": 2.045174899192654, + "language_loss": 0.74753356, + "learning_rate": 3.805020906082057e-07, + "loss": 0.76927292, + "num_input_tokens_seen": 144728830, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15393066, + "step": 6706, + "time_per_iteration": 2.7721962928771973 + }, + { + "auxiliary_loss_clip": 0.0112685, + "auxiliary_loss_mlp": 0.01032985, + "balance_loss_clip": 1.0388974, + "balance_loss_mlp": 1.01668882, + "epoch": 0.8064690675163831, + "flos": 23404600385280.0, + "grad_norm": 2.3972251809691016, + "language_loss": 0.81491446, + "learning_rate": 3.8004513122152917e-07, + "loss": 0.8365128, + "num_input_tokens_seen": 144747140, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.16296387, + "step": 6707, + "time_per_iteration": 2.818596124649048 + }, + { + "auxiliary_loss_clip": 0.01121693, + "auxiliary_loss_mlp": 0.01036644, + "balance_loss_clip": 1.04071999, + "balance_loss_mlp": 1.02320313, + "epoch": 0.8065893104070222, + "flos": 24060903736320.0, + "grad_norm": 1.873544780328197, + "language_loss": 0.66923928, + "learning_rate": 3.79588417580718e-07, + "loss": 0.6908226, + "num_input_tokens_seen": 144765250, + "router_z_loss_clip": 0.80957031, + "router_z_loss_mlp": 0.13446045, + "step": 6708, + "time_per_iteration": 2.7630627155303955 + }, + { + "auxiliary_loss_clip": 0.01127883, + "auxiliary_loss_mlp": 0.01045281, + "balance_loss_clip": 1.04186523, + "balance_loss_mlp": 1.03006983, + "epoch": 0.8067095532976613, + "flos": 22305410340480.0, + "grad_norm": 2.196981388133858, + "language_loss": 0.76513445, + "learning_rate": 3.791319497550558e-07, + "loss": 0.78686613, + "num_input_tokens_seen": 144783080, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.15209961, + "step": 6709, + "time_per_iteration": 2.773190498352051 + }, + { + "auxiliary_loss_clip": 0.01124968, + "auxiliary_loss_mlp": 0.01044722, + "balance_loss_clip": 1.0406822, + "balance_loss_mlp": 1.02971387, + "epoch": 0.8068297961883004, + "flos": 17129498296320.0, + "grad_norm": 2.5653724947903034, + "language_loss": 0.70847046, + "learning_rate": 3.78675727813788e-07, + "loss": 0.73016739, + "num_input_tokens_seen": 144800645, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.14990234, + "step": 6710, + "time_per_iteration": 2.7993340492248535 + }, + { + "auxiliary_loss_clip": 0.0113014, + "auxiliary_loss_mlp": 0.01045795, + "balance_loss_clip": 1.04054141, + "balance_loss_mlp": 1.02982092, + "epoch": 0.8069500390789395, + "flos": 22018843635840.0, + "grad_norm": 1.6628203258062257, + "language_loss": 0.73668826, + "learning_rate": 3.782197518261225e-07, + "loss": 0.75844765, + "num_input_tokens_seen": 144820085, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.15991211, + "step": 6711, + "time_per_iteration": 2.761220693588257 + }, + { + "auxiliary_loss_clip": 0.01125632, + "auxiliary_loss_mlp": 0.01034886, + "balance_loss_clip": 1.04055166, + "balance_loss_mlp": 1.01863754, + "epoch": 0.8070702819695785, + "flos": 19244241567360.0, + "grad_norm": 2.0121499014237796, + "language_loss": 0.95484829, + "learning_rate": 3.777640218612319e-07, + "loss": 0.97645342, + "num_input_tokens_seen": 144838070, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.16235352, + "step": 6712, + "time_per_iteration": 3.6606318950653076 + }, + { + "auxiliary_loss_clip": 0.01125382, + "auxiliary_loss_mlp": 0.01033453, + "balance_loss_clip": 1.04071057, + "balance_loss_mlp": 1.01892161, + "epoch": 0.8071905248602176, + "flos": 21544320038400.0, + "grad_norm": 2.066685006634478, + "language_loss": 0.72169018, + "learning_rate": 3.773085379882488e-07, + "loss": 0.7432785, + "num_input_tokens_seen": 144857125, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.14532471, + "step": 6713, + "time_per_iteration": 2.7660460472106934 + }, + { + "auxiliary_loss_clip": 0.01125415, + "auxiliary_loss_mlp": 0.01040927, + "balance_loss_clip": 1.03884459, + "balance_loss_mlp": 1.02477407, + "epoch": 0.8073107677508568, + "flos": 37268309105280.0, + "grad_norm": 2.0411799045684114, + "language_loss": 0.75861037, + "learning_rate": 3.768533002762715e-07, + "loss": 0.78027374, + "num_input_tokens_seen": 144880660, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.16149902, + "step": 6714, + "time_per_iteration": 2.9204354286193848 + }, + { + "auxiliary_loss_clip": 0.0112148, + "auxiliary_loss_mlp": 0.01036279, + "balance_loss_clip": 1.03882313, + "balance_loss_mlp": 1.02216411, + "epoch": 0.8074310106414958, + "flos": 28366269759360.0, + "grad_norm": 1.6778145565682345, + "language_loss": 0.76913905, + "learning_rate": 3.763983087943572e-07, + "loss": 0.79071671, + "num_input_tokens_seen": 144900050, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.14123535, + "step": 6715, + "time_per_iteration": 2.8716626167297363 + }, + { + "auxiliary_loss_clip": 0.01123945, + "auxiliary_loss_mlp": 0.01041871, + "balance_loss_clip": 1.03931844, + "balance_loss_mlp": 1.02708936, + "epoch": 0.8075512535321349, + "flos": 24281646768000.0, + "grad_norm": 1.6648194237614349, + "language_loss": 0.80839312, + "learning_rate": 3.759435636115282e-07, + "loss": 0.8300513, + "num_input_tokens_seen": 144920835, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14776611, + "step": 6716, + "time_per_iteration": 2.877067804336548 + }, + { + "auxiliary_loss_clip": 0.01125081, + "auxiliary_loss_mlp": 0.01035958, + "balance_loss_clip": 1.04233861, + "balance_loss_mlp": 1.0213604, + "epoch": 0.807671496422774, + "flos": 26030855283840.0, + "grad_norm": 1.8785369882288228, + "language_loss": 0.73313212, + "learning_rate": 3.7548906479676967e-07, + "loss": 0.7547425, + "num_input_tokens_seen": 144940430, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.14593506, + "step": 6717, + "time_per_iteration": 3.6457021236419678 + }, + { + "auxiliary_loss_clip": 0.01127921, + "auxiliary_loss_mlp": 0.01037671, + "balance_loss_clip": 1.03984356, + "balance_loss_mlp": 1.02182817, + "epoch": 0.8077917393134131, + "flos": 23730740899200.0, + "grad_norm": 1.8998862291995657, + "language_loss": 0.71262854, + "learning_rate": 3.7503481241902855e-07, + "loss": 0.7342844, + "num_input_tokens_seen": 144960405, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.15844727, + "step": 6718, + "time_per_iteration": 3.651113748550415 + }, + { + "auxiliary_loss_clip": 0.0112319, + "auxiliary_loss_mlp": 0.01035712, + "balance_loss_clip": 1.04044461, + "balance_loss_mlp": 1.02231908, + "epoch": 0.8079119822040521, + "flos": 18402028398720.0, + "grad_norm": 1.7238433334883727, + "language_loss": 0.80689633, + "learning_rate": 3.745808065472145e-07, + "loss": 0.82848537, + "num_input_tokens_seen": 144977700, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.1338501, + "step": 6719, + "time_per_iteration": 2.804295539855957 + }, + { + "auxiliary_loss_clip": 0.0112431, + "auxiliary_loss_mlp": 0.01034863, + "balance_loss_clip": 1.04144561, + "balance_loss_mlp": 1.0205102, + "epoch": 0.8080322250946913, + "flos": 23621787970560.0, + "grad_norm": 1.4896729765104342, + "language_loss": 0.7620976, + "learning_rate": 3.741270472501994e-07, + "loss": 0.78368926, + "num_input_tokens_seen": 144998340, + "router_z_loss_clip": 0.82763672, + "router_z_loss_mlp": 0.14349365, + "step": 6720, + "time_per_iteration": 2.8236942291259766 + }, + { + "auxiliary_loss_clip": 0.01125173, + "auxiliary_loss_mlp": 0.01039051, + "balance_loss_clip": 1.04199421, + "balance_loss_mlp": 1.02292156, + "epoch": 0.8081524679853304, + "flos": 22820692896000.0, + "grad_norm": 1.6133373036219607, + "language_loss": 0.72775817, + "learning_rate": 3.736735345968183e-07, + "loss": 0.74940038, + "num_input_tokens_seen": 145017950, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.16125488, + "step": 6721, + "time_per_iteration": 2.8691704273223877 + }, + { + "auxiliary_loss_clip": 0.01122986, + "auxiliary_loss_mlp": 0.01037391, + "balance_loss_clip": 1.04089808, + "balance_loss_mlp": 1.02257288, + "epoch": 0.8082727108759694, + "flos": 17640004343040.0, + "grad_norm": 1.5382490470018644, + "language_loss": 0.78766084, + "learning_rate": 3.7322026865586986e-07, + "loss": 0.80926466, + "num_input_tokens_seen": 145036985, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.14819336, + "step": 6722, + "time_per_iteration": 2.8592917919158936 + }, + { + "auxiliary_loss_clip": 0.01129641, + "auxiliary_loss_mlp": 0.01033657, + "balance_loss_clip": 1.04346633, + "balance_loss_mlp": 1.01920915, + "epoch": 0.8083929537666086, + "flos": 25958172113280.0, + "grad_norm": 1.9087099505516463, + "language_loss": 0.73453009, + "learning_rate": 3.7276724949611206e-07, + "loss": 0.75616312, + "num_input_tokens_seen": 145057095, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.14453125, + "step": 6723, + "time_per_iteration": 3.7459847927093506 + }, + { + "auxiliary_loss_clip": 0.01131036, + "auxiliary_loss_mlp": 0.01043329, + "balance_loss_clip": 1.04235721, + "balance_loss_mlp": 1.0266515, + "epoch": 0.8085131966572476, + "flos": 27089178629760.0, + "grad_norm": 1.9061943372838026, + "language_loss": 0.75221765, + "learning_rate": 3.723144771862694e-07, + "loss": 0.77396131, + "num_input_tokens_seen": 145077735, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.16662598, + "step": 6724, + "time_per_iteration": 2.8633811473846436 + }, + { + "auxiliary_loss_clip": 0.01128075, + "auxiliary_loss_mlp": 0.01047825, + "balance_loss_clip": 1.04053211, + "balance_loss_mlp": 1.03185058, + "epoch": 0.8086334395478867, + "flos": 23988543788160.0, + "grad_norm": 1.6482205974318354, + "language_loss": 0.76733077, + "learning_rate": 3.718619517950263e-07, + "loss": 0.78908968, + "num_input_tokens_seen": 145098330, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.159729, + "step": 6725, + "time_per_iteration": 2.9904520511627197 + }, + { + "auxiliary_loss_clip": 0.01127567, + "auxiliary_loss_mlp": 0.01045707, + "balance_loss_clip": 1.04303384, + "balance_loss_mlp": 1.03099668, + "epoch": 0.8087536824385259, + "flos": 20405879406720.0, + "grad_norm": 2.241986859411789, + "language_loss": 0.7683605, + "learning_rate": 3.714096733910301e-07, + "loss": 0.79009324, + "num_input_tokens_seen": 145115855, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.14703369, + "step": 6726, + "time_per_iteration": 2.92630934715271 + }, + { + "auxiliary_loss_clip": 0.01131496, + "auxiliary_loss_mlp": 0.01037307, + "balance_loss_clip": 1.04235959, + "balance_loss_mlp": 1.02045095, + "epoch": 0.8088739253291649, + "flos": 25919639798400.0, + "grad_norm": 2.062478945725689, + "language_loss": 0.70264179, + "learning_rate": 3.709576420428926e-07, + "loss": 0.72432977, + "num_input_tokens_seen": 145136655, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.16845703, + "step": 6727, + "time_per_iteration": 3.021005392074585 + }, + { + "auxiliary_loss_clip": 0.01125262, + "auxiliary_loss_mlp": 0.01041865, + "balance_loss_clip": 1.03948057, + "balance_loss_mlp": 1.0263083, + "epoch": 0.808994168219804, + "flos": 28402072640640.0, + "grad_norm": 2.085531120203277, + "language_loss": 0.73518109, + "learning_rate": 3.7050585781918463e-07, + "loss": 0.75685239, + "num_input_tokens_seen": 145156955, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.15551758, + "step": 6728, + "time_per_iteration": 3.083320140838623 + }, + { + "auxiliary_loss_clip": 0.011292, + "auxiliary_loss_mlp": 0.01042785, + "balance_loss_clip": 1.04098582, + "balance_loss_mlp": 1.02478433, + "epoch": 0.8091144111104431, + "flos": 17421056991360.0, + "grad_norm": 2.1605918923230725, + "language_loss": 0.68788671, + "learning_rate": 3.700543207884428e-07, + "loss": 0.70960665, + "num_input_tokens_seen": 145173865, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.17993164, + "step": 6729, + "time_per_iteration": 2.900583267211914 + }, + { + "auxiliary_loss_clip": 0.01122859, + "auxiliary_loss_mlp": 0.01039255, + "balance_loss_clip": 1.03860974, + "balance_loss_mlp": 1.02483046, + "epoch": 0.8092346540010822, + "flos": 32153803361280.0, + "grad_norm": 1.6616355318633207, + "language_loss": 0.71042925, + "learning_rate": 3.6960303101916466e-07, + "loss": 0.73205042, + "num_input_tokens_seen": 145193780, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14404297, + "step": 6730, + "time_per_iteration": 2.980302095413208 + }, + { + "auxiliary_loss_clip": 0.01032336, + "auxiliary_loss_mlp": 0.01002405, + "balance_loss_clip": 1.00460291, + "balance_loss_mlp": 1.00096881, + "epoch": 0.8093548968917212, + "flos": 58035093390720.0, + "grad_norm": 0.7435566348390494, + "language_loss": 0.55569178, + "learning_rate": 3.6915198857981047e-07, + "loss": 0.5760392, + "num_input_tokens_seen": 145258980, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01434326, + "step": 6731, + "time_per_iteration": 3.403817653656006 + }, + { + "auxiliary_loss_clip": 0.01129412, + "auxiliary_loss_mlp": 0.01041176, + "balance_loss_clip": 1.04242921, + "balance_loss_mlp": 1.02563143, + "epoch": 0.8094751397823604, + "flos": 27381599251200.0, + "grad_norm": 1.6362190400522618, + "language_loss": 0.67986268, + "learning_rate": 3.687011935388027e-07, + "loss": 0.70156854, + "num_input_tokens_seen": 145281875, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15551758, + "step": 6732, + "time_per_iteration": 2.994067668914795 + }, + { + "auxiliary_loss_clip": 0.01126304, + "auxiliary_loss_mlp": 0.01036792, + "balance_loss_clip": 1.04060411, + "balance_loss_mlp": 1.02149713, + "epoch": 0.8095953826729995, + "flos": 24061083304320.0, + "grad_norm": 2.077776423638587, + "language_loss": 0.72621679, + "learning_rate": 3.6825064596452646e-07, + "loss": 0.74784768, + "num_input_tokens_seen": 145302220, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.1529541, + "step": 6733, + "time_per_iteration": 2.9434897899627686 + }, + { + "auxiliary_loss_clip": 0.01124726, + "auxiliary_loss_mlp": 0.01031585, + "balance_loss_clip": 1.04149318, + "balance_loss_mlp": 1.01680338, + "epoch": 0.8097156255636385, + "flos": 23951412103680.0, + "grad_norm": 1.5710189188339614, + "language_loss": 0.70261002, + "learning_rate": 3.678003459253305e-07, + "loss": 0.72417319, + "num_input_tokens_seen": 145323070, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.14782715, + "step": 6734, + "time_per_iteration": 2.940091371536255 + }, + { + "auxiliary_loss_clip": 0.01125306, + "auxiliary_loss_mlp": 0.01033964, + "balance_loss_clip": 1.04037392, + "balance_loss_mlp": 1.01976061, + "epoch": 0.8098358684542777, + "flos": 21799142098560.0, + "grad_norm": 2.002993294884643, + "language_loss": 0.74101281, + "learning_rate": 3.673502934895236e-07, + "loss": 0.76260555, + "num_input_tokens_seen": 145342575, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.14215088, + "step": 6735, + "time_per_iteration": 2.897965669631958 + }, + { + "auxiliary_loss_clip": 0.01031919, + "auxiliary_loss_mlp": 0.00999888, + "balance_loss_clip": 1.00431395, + "balance_loss_mlp": 0.99846977, + "epoch": 0.8099561113449167, + "flos": 68809515966720.0, + "grad_norm": 0.7025593065022232, + "language_loss": 0.57932246, + "learning_rate": 3.669004887253802e-07, + "loss": 0.59964049, + "num_input_tokens_seen": 145408865, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01416016, + "step": 6736, + "time_per_iteration": 3.4708831310272217 + }, + { + "auxiliary_loss_clip": 0.01130137, + "auxiliary_loss_mlp": 0.0103878, + "balance_loss_clip": 1.04554319, + "balance_loss_mlp": 1.02419424, + "epoch": 0.8100763542355558, + "flos": 23586056916480.0, + "grad_norm": 1.5407690899412307, + "language_loss": 0.7874313, + "learning_rate": 3.664509317011335e-07, + "loss": 0.80912042, + "num_input_tokens_seen": 145429200, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.14587402, + "step": 6737, + "time_per_iteration": 2.871154546737671 + }, + { + "auxiliary_loss_clip": 0.01128599, + "auxiliary_loss_mlp": 0.01042381, + "balance_loss_clip": 1.04200745, + "balance_loss_mlp": 1.02489281, + "epoch": 0.810196597126195, + "flos": 31650408207360.0, + "grad_norm": 2.418297312948435, + "language_loss": 0.73752761, + "learning_rate": 3.6600162248498134e-07, + "loss": 0.75923741, + "num_input_tokens_seen": 145452830, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.17480469, + "step": 6738, + "time_per_iteration": 3.07610821723938 + }, + { + "auxiliary_loss_clip": 0.01119267, + "auxiliary_loss_mlp": 0.01032057, + "balance_loss_clip": 1.03836679, + "balance_loss_mlp": 1.01694679, + "epoch": 0.810316840016834, + "flos": 24900459298560.0, + "grad_norm": 1.75702446128688, + "language_loss": 0.76014149, + "learning_rate": 3.6555256114508426e-07, + "loss": 0.78165478, + "num_input_tokens_seen": 145472625, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.15118408, + "step": 6739, + "time_per_iteration": 2.9779741764068604 + }, + { + "auxiliary_loss_clip": 0.01124107, + "auxiliary_loss_mlp": 0.01035536, + "balance_loss_clip": 1.03851557, + "balance_loss_mlp": 1.01986027, + "epoch": 0.8104370829074731, + "flos": 27965003950080.0, + "grad_norm": 1.9629452430948147, + "language_loss": 0.73296213, + "learning_rate": 3.651037477495642e-07, + "loss": 0.7545585, + "num_input_tokens_seen": 145494075, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15673828, + "step": 6740, + "time_per_iteration": 3.135244131088257 + }, + { + "auxiliary_loss_clip": 0.01127742, + "auxiliary_loss_mlp": 0.01037131, + "balance_loss_clip": 1.04142523, + "balance_loss_mlp": 1.0207994, + "epoch": 0.8105573257981122, + "flos": 24640752988800.0, + "grad_norm": 2.2259510996209824, + "language_loss": 0.6781323, + "learning_rate": 3.6465518236650584e-07, + "loss": 0.69978106, + "num_input_tokens_seen": 145514220, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.16308594, + "step": 6741, + "time_per_iteration": 3.7801990509033203 + }, + { + "auxiliary_loss_clip": 0.01123861, + "auxiliary_loss_mlp": 0.01038463, + "balance_loss_clip": 1.0399555, + "balance_loss_mlp": 1.02369308, + "epoch": 0.8106775686887513, + "flos": 26358935132160.0, + "grad_norm": 1.7984580677863593, + "language_loss": 0.78485894, + "learning_rate": 3.642068650639558e-07, + "loss": 0.8064822, + "num_input_tokens_seen": 145533965, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.14764404, + "step": 6742, + "time_per_iteration": 2.979449510574341 + }, + { + "auxiliary_loss_clip": 0.01122709, + "auxiliary_loss_mlp": 0.01037546, + "balance_loss_clip": 1.03984213, + "balance_loss_mlp": 1.02222729, + "epoch": 0.8107978115793903, + "flos": 27271892136960.0, + "grad_norm": 1.9414697482646548, + "language_loss": 0.64416099, + "learning_rate": 3.6375879590992334e-07, + "loss": 0.6657635, + "num_input_tokens_seen": 145554310, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.15319824, + "step": 6743, + "time_per_iteration": 3.0191211700439453 + }, + { + "auxiliary_loss_clip": 0.01127966, + "auxiliary_loss_mlp": 0.01039046, + "balance_loss_clip": 1.0412271, + "balance_loss_mlp": 1.02415037, + "epoch": 0.8109180544700295, + "flos": 24934322845440.0, + "grad_norm": 2.253358474127961, + "language_loss": 0.81184232, + "learning_rate": 3.6331097497238173e-07, + "loss": 0.83351243, + "num_input_tokens_seen": 145573755, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.14886475, + "step": 6744, + "time_per_iteration": 3.0486888885498047 + }, + { + "auxiliary_loss_clip": 0.01120418, + "auxiliary_loss_mlp": 0.01033091, + "balance_loss_clip": 1.03844547, + "balance_loss_mlp": 1.01837456, + "epoch": 0.8110382973606686, + "flos": 21105383840640.0, + "grad_norm": 1.9420130202353052, + "language_loss": 0.80069339, + "learning_rate": 3.628634023192627e-07, + "loss": 0.82222855, + "num_input_tokens_seen": 145594000, + "router_z_loss_clip": 0.81982422, + "router_z_loss_mlp": 0.14709473, + "step": 6745, + "time_per_iteration": 2.9520139694213867 + }, + { + "auxiliary_loss_clip": 0.01124399, + "auxiliary_loss_mlp": 0.01037885, + "balance_loss_clip": 1.03835392, + "balance_loss_mlp": 1.02168405, + "epoch": 0.8111585402513076, + "flos": 15414081500160.0, + "grad_norm": 2.338477916736515, + "language_loss": 0.75139248, + "learning_rate": 3.624160780184644e-07, + "loss": 0.77301538, + "num_input_tokens_seen": 145611215, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.1619873, + "step": 6746, + "time_per_iteration": 3.8500912189483643 + }, + { + "auxiliary_loss_clip": 0.01128194, + "auxiliary_loss_mlp": 0.01030732, + "balance_loss_clip": 1.04328477, + "balance_loss_mlp": 1.01683223, + "epoch": 0.8112787831419467, + "flos": 24095736950400.0, + "grad_norm": 1.6158969688160985, + "language_loss": 0.74154055, + "learning_rate": 3.6196900213784496e-07, + "loss": 0.76312983, + "num_input_tokens_seen": 145630530, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.13885498, + "step": 6747, + "time_per_iteration": 3.8160674571990967 + }, + { + "auxiliary_loss_clip": 0.01125849, + "auxiliary_loss_mlp": 0.01037954, + "balance_loss_clip": 1.04058063, + "balance_loss_mlp": 1.02330923, + "epoch": 0.8113990260325858, + "flos": 20483374999680.0, + "grad_norm": 1.9918603483794275, + "language_loss": 0.86924368, + "learning_rate": 3.6152217474522527e-07, + "loss": 0.89088166, + "num_input_tokens_seen": 145647345, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.14642334, + "step": 6748, + "time_per_iteration": 2.8850436210632324 + }, + { + "auxiliary_loss_clip": 0.01122055, + "auxiliary_loss_mlp": 0.01039999, + "balance_loss_clip": 1.04057026, + "balance_loss_mlp": 1.02514553, + "epoch": 0.8115192689232249, + "flos": 24901141656960.0, + "grad_norm": 1.6151212821238756, + "language_loss": 0.72987139, + "learning_rate": 3.6107559590838975e-07, + "loss": 0.7514919, + "num_input_tokens_seen": 145666330, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.14868164, + "step": 6749, + "time_per_iteration": 2.9929540157318115 + }, + { + "auxiliary_loss_clip": 0.01126055, + "auxiliary_loss_mlp": 0.01040775, + "balance_loss_clip": 1.03979862, + "balance_loss_mlp": 1.02528429, + "epoch": 0.811639511813864, + "flos": 24057204635520.0, + "grad_norm": 2.6079780781490154, + "language_loss": 0.66158384, + "learning_rate": 3.606292656950822e-07, + "loss": 0.68325216, + "num_input_tokens_seen": 145684740, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.15484619, + "step": 6750, + "time_per_iteration": 2.9988608360290527 + }, + { + "auxiliary_loss_clip": 0.01124979, + "auxiliary_loss_mlp": 0.01035764, + "balance_loss_clip": 1.03913224, + "balance_loss_mlp": 1.0198133, + "epoch": 0.8117597547045031, + "flos": 23185150243200.0, + "grad_norm": 2.032438207741826, + "language_loss": 0.86570489, + "learning_rate": 3.601831841730121e-07, + "loss": 0.88731235, + "num_input_tokens_seen": 145702660, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15960693, + "step": 6751, + "time_per_iteration": 3.020528793334961 + }, + { + "auxiliary_loss_clip": 0.01125201, + "auxiliary_loss_mlp": 0.01038971, + "balance_loss_clip": 1.04105496, + "balance_loss_mlp": 1.02412963, + "epoch": 0.8118799975951422, + "flos": 23040250778880.0, + "grad_norm": 1.5656633471576995, + "language_loss": 0.73103988, + "learning_rate": 3.5973735140984916e-07, + "loss": 0.75268161, + "num_input_tokens_seen": 145722830, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.1484375, + "step": 6752, + "time_per_iteration": 3.866323709487915 + }, + { + "auxiliary_loss_clip": 0.01122951, + "auxiliary_loss_mlp": 0.01039205, + "balance_loss_clip": 1.03985989, + "balance_loss_mlp": 1.02492332, + "epoch": 0.8120002404857812, + "flos": 24639962889600.0, + "grad_norm": 1.9925703645219446, + "language_loss": 0.7968089, + "learning_rate": 3.5929176747322607e-07, + "loss": 0.81843042, + "num_input_tokens_seen": 145741935, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.14276123, + "step": 6753, + "time_per_iteration": 2.883559465408325 + }, + { + "auxiliary_loss_clip": 0.01032437, + "auxiliary_loss_mlp": 0.01002911, + "balance_loss_clip": 1.00474966, + "balance_loss_mlp": 1.00148034, + "epoch": 0.8121204833764204, + "flos": 57415742156160.0, + "grad_norm": 0.8067861426291109, + "language_loss": 0.56156909, + "learning_rate": 3.588464324307372e-07, + "loss": 0.58192253, + "num_input_tokens_seen": 145805560, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01428223, + "step": 6754, + "time_per_iteration": 3.306448221206665 + }, + { + "auxiliary_loss_clip": 0.01125132, + "auxiliary_loss_mlp": 0.01032443, + "balance_loss_clip": 1.03956699, + "balance_loss_mlp": 1.01821554, + "epoch": 0.8122407262670595, + "flos": 19464589549440.0, + "grad_norm": 2.4748541305408462, + "language_loss": 0.75315481, + "learning_rate": 3.584013463499391e-07, + "loss": 0.77473056, + "num_input_tokens_seen": 145824180, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.14239502, + "step": 6755, + "time_per_iteration": 2.7607085704803467 + }, + { + "auxiliary_loss_clip": 0.01032029, + "auxiliary_loss_mlp": 0.01003717, + "balance_loss_clip": 1.0043987, + "balance_loss_mlp": 1.00225663, + "epoch": 0.8123609691576985, + "flos": 56425325472000.0, + "grad_norm": 0.7407417129610058, + "language_loss": 0.64434475, + "learning_rate": 3.579565092983521e-07, + "loss": 0.66470224, + "num_input_tokens_seen": 145885300, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.0145874, + "step": 6756, + "time_per_iteration": 3.142133951187134 + }, + { + "auxiliary_loss_clip": 0.01125843, + "auxiliary_loss_mlp": 0.01035895, + "balance_loss_clip": 1.04018009, + "balance_loss_mlp": 1.0211966, + "epoch": 0.8124812120483377, + "flos": 20631973564800.0, + "grad_norm": 3.3564657734423275, + "language_loss": 0.84134173, + "learning_rate": 3.575119213434565e-07, + "loss": 0.86295915, + "num_input_tokens_seen": 145903815, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.14697266, + "step": 6757, + "time_per_iteration": 2.853194236755371 + }, + { + "auxiliary_loss_clip": 0.01125156, + "auxiliary_loss_mlp": 0.01040452, + "balance_loss_clip": 1.04126263, + "balance_loss_mlp": 1.02541959, + "epoch": 0.8126014549389767, + "flos": 22492397566080.0, + "grad_norm": 1.8742166143838948, + "language_loss": 0.81655794, + "learning_rate": 3.5706758255269765e-07, + "loss": 0.83821404, + "num_input_tokens_seen": 145922270, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.15032959, + "step": 6758, + "time_per_iteration": 2.814138412475586 + }, + { + "auxiliary_loss_clip": 0.01129606, + "auxiliary_loss_mlp": 0.01038923, + "balance_loss_clip": 1.04069567, + "balance_loss_mlp": 1.0221262, + "epoch": 0.8127216978296158, + "flos": 23287961946240.0, + "grad_norm": 1.726766025385837, + "language_loss": 0.69777268, + "learning_rate": 3.566234929934795e-07, + "loss": 0.71945798, + "num_input_tokens_seen": 145941470, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.16796875, + "step": 6759, + "time_per_iteration": 2.821930170059204 + }, + { + "auxiliary_loss_clip": 0.01128024, + "auxiliary_loss_mlp": 0.01038694, + "balance_loss_clip": 1.04157019, + "balance_loss_mlp": 1.0246098, + "epoch": 0.812841940720255, + "flos": 25154994049920.0, + "grad_norm": 1.401792194054226, + "language_loss": 0.71620595, + "learning_rate": 3.561796527331706e-07, + "loss": 0.73787314, + "num_input_tokens_seen": 145963145, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.14080811, + "step": 6760, + "time_per_iteration": 2.814927816390991 + }, + { + "auxiliary_loss_clip": 0.01128131, + "auxiliary_loss_mlp": 0.01039662, + "balance_loss_clip": 1.04084253, + "balance_loss_mlp": 1.02384245, + "epoch": 0.812962183610894, + "flos": 26648446752000.0, + "grad_norm": 1.7956848553711633, + "language_loss": 0.77694821, + "learning_rate": 3.5573606183910163e-07, + "loss": 0.79862618, + "num_input_tokens_seen": 145983150, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.1583252, + "step": 6761, + "time_per_iteration": 2.8899574279785156 + }, + { + "auxiliary_loss_clip": 0.01133681, + "auxiliary_loss_mlp": 0.01038471, + "balance_loss_clip": 1.04185009, + "balance_loss_mlp": 1.02148342, + "epoch": 0.8130824265015331, + "flos": 24966965329920.0, + "grad_norm": 1.7881048048003467, + "language_loss": 0.79034913, + "learning_rate": 3.5529272037856493e-07, + "loss": 0.81207061, + "num_input_tokens_seen": 146001365, + "router_z_loss_clip": 0.91894531, + "router_z_loss_mlp": 0.1697998, + "step": 6762, + "time_per_iteration": 2.9449758529663086 + }, + { + "auxiliary_loss_clip": 0.01032614, + "auxiliary_loss_mlp": 0.01002947, + "balance_loss_clip": 1.00485182, + "balance_loss_mlp": 1.0015223, + "epoch": 0.8132026693921722, + "flos": 67622918175360.0, + "grad_norm": 0.7058976252566274, + "language_loss": 0.53828132, + "learning_rate": 3.548496284188149e-07, + "loss": 0.5586369, + "num_input_tokens_seen": 146061570, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01422119, + "step": 6763, + "time_per_iteration": 3.2994167804718018 + }, + { + "auxiliary_loss_clip": 0.01122957, + "auxiliary_loss_mlp": 0.01038316, + "balance_loss_clip": 1.03937078, + "balance_loss_mlp": 1.0243206, + "epoch": 0.8133229122828113, + "flos": 19495149045120.0, + "grad_norm": 1.6851535828811073, + "language_loss": 0.79066324, + "learning_rate": 3.544067860270681e-07, + "loss": 0.81227601, + "num_input_tokens_seen": 146079145, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.14001465, + "step": 6764, + "time_per_iteration": 2.7874197959899902 + }, + { + "auxiliary_loss_clip": 0.01129713, + "auxiliary_loss_mlp": 0.01040055, + "balance_loss_clip": 1.04054451, + "balance_loss_mlp": 1.02326989, + "epoch": 0.8134431551734503, + "flos": 20668135582080.0, + "grad_norm": 1.7946721506503789, + "language_loss": 0.70724064, + "learning_rate": 3.539641932705029e-07, + "loss": 0.72893834, + "num_input_tokens_seen": 146097625, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.16772461, + "step": 6765, + "time_per_iteration": 2.740617513656616 + }, + { + "auxiliary_loss_clip": 0.01129159, + "auxiliary_loss_mlp": 0.0104343, + "balance_loss_clip": 1.0397377, + "balance_loss_mlp": 1.02765834, + "epoch": 0.8135633980640895, + "flos": 21507332008320.0, + "grad_norm": 2.1800552461358333, + "language_loss": 0.77396548, + "learning_rate": 3.53521850216262e-07, + "loss": 0.79569137, + "num_input_tokens_seen": 146117195, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.15759277, + "step": 6766, + "time_per_iteration": 2.8220622539520264 + }, + { + "auxiliary_loss_clip": 0.01127215, + "auxiliary_loss_mlp": 0.01034613, + "balance_loss_clip": 1.0409416, + "balance_loss_mlp": 1.01871085, + "epoch": 0.8136836409547286, + "flos": 20554442058240.0, + "grad_norm": 1.6874660472722551, + "language_loss": 0.76820147, + "learning_rate": 3.530797569314461e-07, + "loss": 0.78981972, + "num_input_tokens_seen": 146136220, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15887451, + "step": 6767, + "time_per_iteration": 2.7921900749206543 + }, + { + "auxiliary_loss_clip": 0.01127466, + "auxiliary_loss_mlp": 0.01034871, + "balance_loss_clip": 1.04185605, + "balance_loss_mlp": 1.01955223, + "epoch": 0.8138038838453676, + "flos": 20299045380480.0, + "grad_norm": 1.7251959876077942, + "language_loss": 0.78102338, + "learning_rate": 3.5263791348312235e-07, + "loss": 0.80264682, + "num_input_tokens_seen": 146155415, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15332031, + "step": 6768, + "time_per_iteration": 2.7985596656799316 + }, + { + "auxiliary_loss_clip": 0.01124331, + "auxiliary_loss_mlp": 0.01031594, + "balance_loss_clip": 1.0398196, + "balance_loss_mlp": 1.01577473, + "epoch": 0.8139241267360068, + "flos": 29789840551680.0, + "grad_norm": 2.169709163942608, + "language_loss": 0.70508528, + "learning_rate": 3.521963199383171e-07, + "loss": 0.72664458, + "num_input_tokens_seen": 146178370, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.15820312, + "step": 6769, + "time_per_iteration": 2.9906389713287354 + }, + { + "auxiliary_loss_clip": 0.0112841, + "auxiliary_loss_mlp": 0.01037673, + "balance_loss_clip": 1.04030085, + "balance_loss_mlp": 1.02161574, + "epoch": 0.8140443696266458, + "flos": 19713270384000.0, + "grad_norm": 2.412415386717813, + "language_loss": 0.76885432, + "learning_rate": 3.517549763640197e-07, + "loss": 0.79051518, + "num_input_tokens_seen": 146196010, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.16052246, + "step": 6770, + "time_per_iteration": 3.6462440490722656 + }, + { + "auxiliary_loss_clip": 0.01126037, + "auxiliary_loss_mlp": 0.01037223, + "balance_loss_clip": 1.04211342, + "balance_loss_mlp": 1.02286994, + "epoch": 0.8141646125172849, + "flos": 27160568910720.0, + "grad_norm": 1.81172567892008, + "language_loss": 0.7108776, + "learning_rate": 3.513138828271829e-07, + "loss": 0.73251021, + "num_input_tokens_seen": 146215880, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14355469, + "step": 6771, + "time_per_iteration": 2.8350796699523926 + }, + { + "auxiliary_loss_clip": 0.01123702, + "auxiliary_loss_mlp": 0.01036476, + "balance_loss_clip": 1.04011106, + "balance_loss_mlp": 1.02115726, + "epoch": 0.8142848554079241, + "flos": 39673102700160.0, + "grad_norm": 1.807399219040811, + "language_loss": 0.69964689, + "learning_rate": 3.508730393947179e-07, + "loss": 0.72124863, + "num_input_tokens_seen": 146239135, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.15332031, + "step": 6772, + "time_per_iteration": 2.9745192527770996 + }, + { + "auxiliary_loss_clip": 0.01123093, + "auxiliary_loss_mlp": 0.01037371, + "balance_loss_clip": 1.03986752, + "balance_loss_mlp": 1.02229726, + "epoch": 0.8144050982985631, + "flos": 22237288197120.0, + "grad_norm": 2.0909525921558565, + "language_loss": 0.72183102, + "learning_rate": 3.504324461335024e-07, + "loss": 0.74343562, + "num_input_tokens_seen": 146259245, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.15081787, + "step": 6773, + "time_per_iteration": 2.822848081588745 + }, + { + "auxiliary_loss_clip": 0.01129071, + "auxiliary_loss_mlp": 0.01046466, + "balance_loss_clip": 1.04228091, + "balance_loss_mlp": 1.03132665, + "epoch": 0.8145253411892022, + "flos": 23038239617280.0, + "grad_norm": 1.622806085426821, + "language_loss": 0.88351405, + "learning_rate": 3.499921031103732e-07, + "loss": 0.90526944, + "num_input_tokens_seen": 146280015, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15124512, + "step": 6774, + "time_per_iteration": 2.8585190773010254 + }, + { + "auxiliary_loss_clip": 0.0112973, + "auxiliary_loss_mlp": 0.0104308, + "balance_loss_clip": 1.04213452, + "balance_loss_mlp": 1.02810705, + "epoch": 0.8146455840798413, + "flos": 24827668387200.0, + "grad_norm": 1.8768851568428786, + "language_loss": 0.77988005, + "learning_rate": 3.4955201039212987e-07, + "loss": 0.80160815, + "num_input_tokens_seen": 146300935, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.14990234, + "step": 6775, + "time_per_iteration": 3.5334360599517822 + }, + { + "auxiliary_loss_clip": 0.01127098, + "auxiliary_loss_mlp": 0.01036872, + "balance_loss_clip": 1.04021263, + "balance_loss_mlp": 1.02126777, + "epoch": 0.8147658269704804, + "flos": 19974520978560.0, + "grad_norm": 1.7466682532191535, + "language_loss": 0.65228331, + "learning_rate": 3.4911216804553465e-07, + "loss": 0.67392302, + "num_input_tokens_seen": 146319835, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.15576172, + "step": 6776, + "time_per_iteration": 3.6148593425750732 + }, + { + "auxiliary_loss_clip": 0.01127327, + "auxiliary_loss_mlp": 0.01038158, + "balance_loss_clip": 1.03976798, + "balance_loss_mlp": 1.02165949, + "epoch": 0.8148860698611194, + "flos": 21178031097600.0, + "grad_norm": 1.8392818469070427, + "language_loss": 0.70068049, + "learning_rate": 3.4867257613731017e-07, + "loss": 0.72233534, + "num_input_tokens_seen": 146339030, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16491699, + "step": 6777, + "time_per_iteration": 2.861624002456665 + }, + { + "auxiliary_loss_clip": 0.01123933, + "auxiliary_loss_mlp": 0.01045727, + "balance_loss_clip": 1.03948998, + "balance_loss_mlp": 1.03114736, + "epoch": 0.8150063127517585, + "flos": 19606903234560.0, + "grad_norm": 1.6334433670196618, + "language_loss": 0.8566224, + "learning_rate": 3.4823323473414343e-07, + "loss": 0.87831903, + "num_input_tokens_seen": 146358550, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.14575195, + "step": 6778, + "time_per_iteration": 2.836270809173584 + }, + { + "auxiliary_loss_clip": 0.01127082, + "auxiliary_loss_mlp": 0.01041558, + "balance_loss_clip": 1.03907037, + "balance_loss_mlp": 1.0243386, + "epoch": 0.8151265556423977, + "flos": 22638374438400.0, + "grad_norm": 1.9822864003858638, + "language_loss": 0.75749516, + "learning_rate": 3.477941439026812e-07, + "loss": 0.77918154, + "num_input_tokens_seen": 146376770, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.17218018, + "step": 6779, + "time_per_iteration": 2.843679428100586 + }, + { + "auxiliary_loss_clip": 0.01121935, + "auxiliary_loss_mlp": 0.01035844, + "balance_loss_clip": 1.04036999, + "balance_loss_mlp": 1.02186036, + "epoch": 0.8152467985330367, + "flos": 17968048277760.0, + "grad_norm": 1.7627994431962712, + "language_loss": 0.72843337, + "learning_rate": 3.473553037095349e-07, + "loss": 0.75001121, + "num_input_tokens_seen": 146395795, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.13977051, + "step": 6780, + "time_per_iteration": 2.823495388031006 + }, + { + "auxiliary_loss_clip": 0.01123785, + "auxiliary_loss_mlp": 0.01038589, + "balance_loss_clip": 1.04219174, + "balance_loss_mlp": 1.02413452, + "epoch": 0.8153670414236758, + "flos": 24969012405120.0, + "grad_norm": 1.7581449715894197, + "language_loss": 0.83572984, + "learning_rate": 3.469167142212743e-07, + "loss": 0.85735357, + "num_input_tokens_seen": 146417640, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.14453125, + "step": 6781, + "time_per_iteration": 3.7436907291412354 + }, + { + "auxiliary_loss_clip": 0.01125503, + "auxiliary_loss_mlp": 0.01045588, + "balance_loss_clip": 1.03973269, + "balance_loss_mlp": 1.02979279, + "epoch": 0.8154872843143149, + "flos": 31066069754880.0, + "grad_norm": 2.9170126957754463, + "language_loss": 0.63556933, + "learning_rate": 3.4647837550443337e-07, + "loss": 0.65728021, + "num_input_tokens_seen": 146436205, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.15783691, + "step": 6782, + "time_per_iteration": 2.9835870265960693 + }, + { + "auxiliary_loss_clip": 0.01128562, + "auxiliary_loss_mlp": 0.01038315, + "balance_loss_clip": 1.04049015, + "balance_loss_mlp": 1.02271092, + "epoch": 0.815607527204954, + "flos": 19391654983680.0, + "grad_norm": 1.7003247132124062, + "language_loss": 0.74631697, + "learning_rate": 3.460402876255086e-07, + "loss": 0.76798576, + "num_input_tokens_seen": 146453595, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.15612793, + "step": 6783, + "time_per_iteration": 2.855576753616333 + }, + { + "auxiliary_loss_clip": 0.01122396, + "auxiliary_loss_mlp": 0.01039149, + "balance_loss_clip": 1.03829634, + "balance_loss_mlp": 1.0246532, + "epoch": 0.815727770095593, + "flos": 26140418743680.0, + "grad_norm": 1.9632248556864946, + "language_loss": 0.71574855, + "learning_rate": 3.456024506509574e-07, + "loss": 0.73736399, + "num_input_tokens_seen": 146474515, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.14489746, + "step": 6784, + "time_per_iteration": 2.888266086578369 + }, + { + "auxiliary_loss_clip": 0.01122838, + "auxiliary_loss_mlp": 0.01037686, + "balance_loss_clip": 1.0408299, + "balance_loss_mlp": 1.02218866, + "epoch": 0.8158480129862322, + "flos": 25337527989120.0, + "grad_norm": 1.5821876311560072, + "language_loss": 0.73931694, + "learning_rate": 3.4516486464719873e-07, + "loss": 0.76092219, + "num_input_tokens_seen": 146493905, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.15509033, + "step": 6785, + "time_per_iteration": 2.8597218990325928 + }, + { + "auxiliary_loss_clip": 0.0112565, + "auxiliary_loss_mlp": 0.01038202, + "balance_loss_clip": 1.03964484, + "balance_loss_mlp": 1.02238321, + "epoch": 0.8159682558768713, + "flos": 34423645559040.0, + "grad_norm": 1.6864046043295562, + "language_loss": 0.62050509, + "learning_rate": 3.4472752968061445e-07, + "loss": 0.64214373, + "num_input_tokens_seen": 146518335, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.15820312, + "step": 6786, + "time_per_iteration": 3.009446144104004 + }, + { + "auxiliary_loss_clip": 0.01122724, + "auxiliary_loss_mlp": 0.01039413, + "balance_loss_clip": 1.03826761, + "balance_loss_mlp": 1.02424979, + "epoch": 0.8160884987675103, + "flos": 18653223185280.0, + "grad_norm": 1.7752389111871627, + "language_loss": 0.74236083, + "learning_rate": 3.442904458175475e-07, + "loss": 0.76398218, + "num_input_tokens_seen": 146535655, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15161133, + "step": 6787, + "time_per_iteration": 2.8950164318084717 + }, + { + "auxiliary_loss_clip": 0.01124078, + "auxiliary_loss_mlp": 0.01042859, + "balance_loss_clip": 1.03844821, + "balance_loss_mlp": 1.02707553, + "epoch": 0.8162087416581495, + "flos": 31430527102080.0, + "grad_norm": 1.5213269455069087, + "language_loss": 0.76194859, + "learning_rate": 3.438536131243044e-07, + "loss": 0.78361797, + "num_input_tokens_seen": 146556815, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15783691, + "step": 6788, + "time_per_iteration": 2.8588783740997314 + }, + { + "auxiliary_loss_clip": 0.01128414, + "auxiliary_loss_mlp": 0.01041998, + "balance_loss_clip": 1.03958297, + "balance_loss_mlp": 1.02510619, + "epoch": 0.8163289845487885, + "flos": 37593910915200.0, + "grad_norm": 2.926901945606837, + "language_loss": 0.62359291, + "learning_rate": 3.434170316671503e-07, + "loss": 0.64529705, + "num_input_tokens_seen": 146581845, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.16882324, + "step": 6789, + "time_per_iteration": 2.8903086185455322 + }, + { + "auxiliary_loss_clip": 0.01123852, + "auxiliary_loss_mlp": 0.01036433, + "balance_loss_clip": 1.03920436, + "balance_loss_mlp": 1.02200913, + "epoch": 0.8164492274394276, + "flos": 13953989554560.0, + "grad_norm": 2.331422090094393, + "language_loss": 0.89532065, + "learning_rate": 3.4298070151231583e-07, + "loss": 0.91692352, + "num_input_tokens_seen": 146597245, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14422607, + "step": 6790, + "time_per_iteration": 2.7426700592041016 + }, + { + "auxiliary_loss_clip": 0.01126962, + "auxiliary_loss_mlp": 0.0103138, + "balance_loss_clip": 1.04186749, + "balance_loss_mlp": 1.01594281, + "epoch": 0.8165694703300668, + "flos": 28986554747520.0, + "grad_norm": 2.368306734753656, + "language_loss": 0.59898669, + "learning_rate": 3.425446227259916e-07, + "loss": 0.62057006, + "num_input_tokens_seen": 146618210, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.15429688, + "step": 6791, + "time_per_iteration": 2.8352386951446533 + }, + { + "auxiliary_loss_clip": 0.01122356, + "auxiliary_loss_mlp": 0.01037398, + "balance_loss_clip": 1.03921998, + "balance_loss_mlp": 1.02376032, + "epoch": 0.8166897132207058, + "flos": 25118365155840.0, + "grad_norm": 3.57535704424722, + "language_loss": 0.82572794, + "learning_rate": 3.421087953743296e-07, + "loss": 0.84732544, + "num_input_tokens_seen": 146637975, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.13647461, + "step": 6792, + "time_per_iteration": 2.824673652648926 + }, + { + "auxiliary_loss_clip": 0.01127504, + "auxiliary_loss_mlp": 0.01045057, + "balance_loss_clip": 1.03906381, + "balance_loss_mlp": 1.02965474, + "epoch": 0.8168099561113449, + "flos": 23148593176320.0, + "grad_norm": 3.3394741111648774, + "language_loss": 0.79972088, + "learning_rate": 3.416732195234464e-07, + "loss": 0.82144642, + "num_input_tokens_seen": 146658030, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.1541748, + "step": 6793, + "time_per_iteration": 2.755507707595825 + }, + { + "auxiliary_loss_clip": 0.01123601, + "auxiliary_loss_mlp": 0.01038245, + "balance_loss_clip": 1.04030466, + "balance_loss_mlp": 1.02352297, + "epoch": 0.816930199001984, + "flos": 18407666833920.0, + "grad_norm": 2.283783855494798, + "language_loss": 0.79417312, + "learning_rate": 3.4123789523941613e-07, + "loss": 0.81579161, + "num_input_tokens_seen": 146677855, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.1472168, + "step": 6794, + "time_per_iteration": 2.790188789367676 + }, + { + "auxiliary_loss_clip": 0.01123114, + "auxiliary_loss_mlp": 0.01028753, + "balance_loss_clip": 1.03932285, + "balance_loss_mlp": 1.01404893, + "epoch": 0.8170504418926231, + "flos": 21251324799360.0, + "grad_norm": 1.4579601055776026, + "language_loss": 0.63432002, + "learning_rate": 3.4080282258827884e-07, + "loss": 0.65583867, + "num_input_tokens_seen": 146696230, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.14703369, + "step": 6795, + "time_per_iteration": 2.7375731468200684 + }, + { + "auxiliary_loss_clip": 0.01125009, + "auxiliary_loss_mlp": 0.0103457, + "balance_loss_clip": 1.04103851, + "balance_loss_mlp": 1.02051568, + "epoch": 0.8171706847832622, + "flos": 19099234362240.0, + "grad_norm": 2.098339317026063, + "language_loss": 0.72680724, + "learning_rate": 3.403680016360342e-07, + "loss": 0.74840295, + "num_input_tokens_seen": 146714835, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.14056396, + "step": 6796, + "time_per_iteration": 2.7926554679870605 + }, + { + "auxiliary_loss_clip": 0.01126556, + "auxiliary_loss_mlp": 0.0104456, + "balance_loss_clip": 1.04078722, + "balance_loss_mlp": 1.0292058, + "epoch": 0.8172909276739013, + "flos": 21470128496640.0, + "grad_norm": 1.4857121167211378, + "language_loss": 0.67313039, + "learning_rate": 3.3993343244864403e-07, + "loss": 0.6948415, + "num_input_tokens_seen": 146734425, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15380859, + "step": 6797, + "time_per_iteration": 2.750441789627075 + }, + { + "auxiliary_loss_clip": 0.01125966, + "auxiliary_loss_mlp": 0.01036143, + "balance_loss_clip": 1.04033089, + "balance_loss_mlp": 1.02162302, + "epoch": 0.8174111705645404, + "flos": 27599792417280.0, + "grad_norm": 1.5719794652045425, + "language_loss": 0.725061, + "learning_rate": 3.394991150920323e-07, + "loss": 0.74668211, + "num_input_tokens_seen": 146757545, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.14532471, + "step": 6798, + "time_per_iteration": 2.8562350273132324 + }, + { + "auxiliary_loss_clip": 0.01129917, + "auxiliary_loss_mlp": 0.01041973, + "balance_loss_clip": 1.04094505, + "balance_loss_mlp": 1.02547407, + "epoch": 0.8175314134551794, + "flos": 14064594508800.0, + "grad_norm": 2.4161328482106446, + "language_loss": 0.74242437, + "learning_rate": 3.3906504963208396e-07, + "loss": 0.76414335, + "num_input_tokens_seen": 146774240, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.16491699, + "step": 6799, + "time_per_iteration": 3.6839301586151123 + }, + { + "auxiliary_loss_clip": 0.01125519, + "auxiliary_loss_mlp": 0.01040485, + "balance_loss_clip": 1.03852916, + "balance_loss_mlp": 1.0242722, + "epoch": 0.8176516563458186, + "flos": 22708076780160.0, + "grad_norm": 1.8987437319159544, + "language_loss": 0.66752708, + "learning_rate": 3.3863123613464774e-07, + "loss": 0.68918711, + "num_input_tokens_seen": 146793140, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.1619873, + "step": 6800, + "time_per_iteration": 2.8682804107666016 + }, + { + "auxiliary_loss_clip": 0.01125377, + "auxiliary_loss_mlp": 0.01036019, + "balance_loss_clip": 1.04011774, + "balance_loss_mlp": 1.0213387, + "epoch": 0.8177718992364577, + "flos": 21945406279680.0, + "grad_norm": 1.7083742734388465, + "language_loss": 0.74845934, + "learning_rate": 3.381976746655317e-07, + "loss": 0.77007329, + "num_input_tokens_seen": 146812895, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.14660645, + "step": 6801, + "time_per_iteration": 2.9070680141448975 + }, + { + "auxiliary_loss_clip": 0.01124795, + "auxiliary_loss_mlp": 0.01038788, + "balance_loss_clip": 1.04173613, + "balance_loss_mlp": 1.02419662, + "epoch": 0.8178921421270967, + "flos": 22017443005440.0, + "grad_norm": 1.922387575867486, + "language_loss": 0.6745438, + "learning_rate": 3.3776436529050756e-07, + "loss": 0.69617963, + "num_input_tokens_seen": 146832445, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.14593506, + "step": 6802, + "time_per_iteration": 2.836829423904419 + }, + { + "auxiliary_loss_clip": 0.01123208, + "auxiliary_loss_mlp": 0.01034554, + "balance_loss_clip": 1.03978741, + "balance_loss_mlp": 1.01994479, + "epoch": 0.8180123850177359, + "flos": 33183111496320.0, + "grad_norm": 1.7635164567074118, + "language_loss": 0.72414082, + "learning_rate": 3.373313080753073e-07, + "loss": 0.74571848, + "num_input_tokens_seen": 146856505, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.14593506, + "step": 6803, + "time_per_iteration": 2.962400436401367 + }, + { + "auxiliary_loss_clip": 0.01126499, + "auxiliary_loss_mlp": 0.01040929, + "balance_loss_clip": 1.03916693, + "balance_loss_mlp": 1.02506185, + "epoch": 0.8181326279083749, + "flos": 22091167670400.0, + "grad_norm": 1.7900481553411383, + "language_loss": 0.77696514, + "learning_rate": 3.3689850308562527e-07, + "loss": 0.79863948, + "num_input_tokens_seen": 146876950, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15856934, + "step": 6804, + "time_per_iteration": 3.5981290340423584 + }, + { + "auxiliary_loss_clip": 0.01125217, + "auxiliary_loss_mlp": 0.01040834, + "balance_loss_clip": 1.04165852, + "balance_loss_mlp": 1.02655315, + "epoch": 0.818252870799014, + "flos": 15705747936000.0, + "grad_norm": 1.817058063374672, + "language_loss": 0.77871889, + "learning_rate": 3.364659503871183e-07, + "loss": 0.8003794, + "num_input_tokens_seen": 146894885, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14282227, + "step": 6805, + "time_per_iteration": 3.6730008125305176 + }, + { + "auxiliary_loss_clip": 0.01121506, + "auxiliary_loss_mlp": 0.01028347, + "balance_loss_clip": 1.0384624, + "balance_loss_mlp": 1.01389837, + "epoch": 0.8183731136896532, + "flos": 18770687637120.0, + "grad_norm": 4.997441700767705, + "language_loss": 0.84103441, + "learning_rate": 3.3603365004540417e-07, + "loss": 0.86253297, + "num_input_tokens_seen": 146913180, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.14440918, + "step": 6806, + "time_per_iteration": 2.9190359115600586 + }, + { + "auxiliary_loss_clip": 0.01126713, + "auxiliary_loss_mlp": 0.01033308, + "balance_loss_clip": 1.04170084, + "balance_loss_mlp": 1.01797795, + "epoch": 0.8184933565802922, + "flos": 26541792293760.0, + "grad_norm": 2.3724318979420387, + "language_loss": 0.77275062, + "learning_rate": 3.356016021260624e-07, + "loss": 0.79435086, + "num_input_tokens_seen": 146933510, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15319824, + "step": 6807, + "time_per_iteration": 2.855623483657837 + }, + { + "auxiliary_loss_clip": 0.01128028, + "auxiliary_loss_mlp": 0.01047356, + "balance_loss_clip": 1.04065585, + "balance_loss_mlp": 1.03040457, + "epoch": 0.8186135994709313, + "flos": 17530117660800.0, + "grad_norm": 2.5204796656106483, + "language_loss": 0.65858126, + "learning_rate": 3.35169806694634e-07, + "loss": 0.68033504, + "num_input_tokens_seen": 146951760, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.16955566, + "step": 6808, + "time_per_iteration": 2.8178582191467285 + }, + { + "auxiliary_loss_clip": 0.01032367, + "auxiliary_loss_mlp": 0.01001549, + "balance_loss_clip": 1.00462568, + "balance_loss_mlp": 1.00024343, + "epoch": 0.8187338423615703, + "flos": 63480300675840.0, + "grad_norm": 0.7153232056243165, + "language_loss": 0.6064707, + "learning_rate": 3.3473826381662186e-07, + "loss": 0.62680984, + "num_input_tokens_seen": 147022900, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01306152, + "step": 6809, + "time_per_iteration": 4.306704521179199 + }, + { + "auxiliary_loss_clip": 0.01125105, + "auxiliary_loss_mlp": 0.01038105, + "balance_loss_clip": 1.04078746, + "balance_loss_mlp": 1.02318585, + "epoch": 0.8188540852522095, + "flos": 17529974006400.0, + "grad_norm": 1.9082485975165413, + "language_loss": 0.81584531, + "learning_rate": 3.3430697355749216e-07, + "loss": 0.83747745, + "num_input_tokens_seen": 147040590, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.14923096, + "step": 6810, + "time_per_iteration": 2.8396527767181396 + }, + { + "auxiliary_loss_clip": 0.01126734, + "auxiliary_loss_mlp": 0.01035622, + "balance_loss_clip": 1.04058409, + "balance_loss_mlp": 1.02085185, + "epoch": 0.8189743281428485, + "flos": 14392530702720.0, + "grad_norm": 1.8820214125097798, + "language_loss": 0.75489402, + "learning_rate": 3.3387593598266907e-07, + "loss": 0.77651757, + "num_input_tokens_seen": 147057200, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.14776611, + "step": 6811, + "time_per_iteration": 2.8777353763580322 + }, + { + "auxiliary_loss_clip": 0.0112454, + "auxiliary_loss_mlp": 0.01035352, + "balance_loss_clip": 1.03911948, + "balance_loss_mlp": 1.02023602, + "epoch": 0.8190945710334876, + "flos": 25080479285760.0, + "grad_norm": 1.6431121105093784, + "language_loss": 0.78379381, + "learning_rate": 3.3344515115754225e-07, + "loss": 0.80539274, + "num_input_tokens_seen": 147076180, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.15124512, + "step": 6812, + "time_per_iteration": 2.897374391555786 + }, + { + "auxiliary_loss_clip": 0.01125967, + "auxiliary_loss_mlp": 0.01034571, + "balance_loss_clip": 1.0392319, + "balance_loss_mlp": 1.01977146, + "epoch": 0.8192148139241268, + "flos": 21507152440320.0, + "grad_norm": 2.6907060442261637, + "language_loss": 0.80138463, + "learning_rate": 3.33014619147461e-07, + "loss": 0.82298994, + "num_input_tokens_seen": 147094205, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.14801025, + "step": 6813, + "time_per_iteration": 2.897005081176758 + }, + { + "auxiliary_loss_clip": 0.01127969, + "auxiliary_loss_mlp": 0.01048037, + "balance_loss_clip": 1.04130161, + "balance_loss_mlp": 1.03293312, + "epoch": 0.8193350568147658, + "flos": 23952166289280.0, + "grad_norm": 1.819959329419859, + "language_loss": 0.71819854, + "learning_rate": 3.325843400177362e-07, + "loss": 0.73995858, + "num_input_tokens_seen": 147115545, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.15100098, + "step": 6814, + "time_per_iteration": 2.8944365978240967 + }, + { + "auxiliary_loss_clip": 0.01128154, + "auxiliary_loss_mlp": 0.01036998, + "balance_loss_clip": 1.04221296, + "balance_loss_mlp": 1.02084529, + "epoch": 0.8194552997054049, + "flos": 20559469962240.0, + "grad_norm": 1.8269583047839972, + "language_loss": 0.73527241, + "learning_rate": 3.32154313833642e-07, + "loss": 0.75692391, + "num_input_tokens_seen": 147135700, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.16162109, + "step": 6815, + "time_per_iteration": 2.842597484588623 + }, + { + "auxiliary_loss_clip": 0.01129625, + "auxiliary_loss_mlp": 0.01040339, + "balance_loss_clip": 1.04172468, + "balance_loss_mlp": 1.02348316, + "epoch": 0.819575542596044, + "flos": 26031753123840.0, + "grad_norm": 1.9891367912853515, + "language_loss": 0.59353352, + "learning_rate": 3.3172454066041164e-07, + "loss": 0.61523318, + "num_input_tokens_seen": 147155205, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.1685791, + "step": 6816, + "time_per_iteration": 2.8632967472076416 + }, + { + "auxiliary_loss_clip": 0.01124038, + "auxiliary_loss_mlp": 0.01032177, + "balance_loss_clip": 1.04036844, + "balance_loss_mlp": 1.01789582, + "epoch": 0.8196957854866831, + "flos": 29096944220160.0, + "grad_norm": 1.947378265499727, + "language_loss": 0.75968271, + "learning_rate": 3.3129502056324234e-07, + "loss": 0.78124487, + "num_input_tokens_seen": 147176570, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14294434, + "step": 6817, + "time_per_iteration": 2.8643715381622314 + }, + { + "auxiliary_loss_clip": 0.01032292, + "auxiliary_loss_mlp": 0.01001638, + "balance_loss_clip": 1.00440431, + "balance_loss_mlp": 1.00013566, + "epoch": 0.8198160283773221, + "flos": 69033631898880.0, + "grad_norm": 0.8002064521530045, + "language_loss": 0.59754395, + "learning_rate": 3.3086575360729165e-07, + "loss": 0.61788332, + "num_input_tokens_seen": 147234105, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01501465, + "step": 6818, + "time_per_iteration": 3.1673197746276855 + }, + { + "auxiliary_loss_clip": 0.01127971, + "auxiliary_loss_mlp": 0.01046002, + "balance_loss_clip": 1.04034519, + "balance_loss_mlp": 1.03117251, + "epoch": 0.8199362712679613, + "flos": 16618058496000.0, + "grad_norm": 1.8042693476399594, + "language_loss": 0.71572816, + "learning_rate": 3.3043673985767906e-07, + "loss": 0.73746789, + "num_input_tokens_seen": 147253170, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.14831543, + "step": 6819, + "time_per_iteration": 2.800962448120117 + }, + { + "auxiliary_loss_clip": 0.0112476, + "auxiliary_loss_mlp": 0.01044505, + "balance_loss_clip": 1.04029775, + "balance_loss_mlp": 1.02925825, + "epoch": 0.8200565141586004, + "flos": 21757664868480.0, + "grad_norm": 1.7648144085466044, + "language_loss": 0.77868211, + "learning_rate": 3.3000797937948564e-07, + "loss": 0.80037475, + "num_input_tokens_seen": 147271465, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15258789, + "step": 6820, + "time_per_iteration": 2.80935001373291 + }, + { + "auxiliary_loss_clip": 0.01032087, + "auxiliary_loss_mlp": 0.01003469, + "balance_loss_clip": 1.00452113, + "balance_loss_mlp": 1.00196648, + "epoch": 0.8201767570492394, + "flos": 69807112392960.0, + "grad_norm": 0.9394599171919241, + "language_loss": 0.64998966, + "learning_rate": 3.295794722377534e-07, + "loss": 0.67034519, + "num_input_tokens_seen": 147335070, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01501465, + "step": 6821, + "time_per_iteration": 3.3667662143707275 + }, + { + "auxiliary_loss_clip": 0.01121502, + "auxiliary_loss_mlp": 0.01035424, + "balance_loss_clip": 1.03787971, + "balance_loss_mlp": 1.02150059, + "epoch": 0.8202969999398786, + "flos": 23111892455040.0, + "grad_norm": 1.6663670387668879, + "language_loss": 0.80149043, + "learning_rate": 3.291512184974876e-07, + "loss": 0.82305968, + "num_input_tokens_seen": 147355460, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.13909912, + "step": 6822, + "time_per_iteration": 2.928431510925293 + }, + { + "auxiliary_loss_clip": 0.01127633, + "auxiliary_loss_mlp": 0.01043368, + "balance_loss_clip": 1.04135966, + "balance_loss_mlp": 1.02700043, + "epoch": 0.8204172428305176, + "flos": 28220616109440.0, + "grad_norm": 1.5785674650578398, + "language_loss": 0.66480732, + "learning_rate": 3.2872321822365346e-07, + "loss": 0.68651736, + "num_input_tokens_seen": 147375675, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.16357422, + "step": 6823, + "time_per_iteration": 3.118898630142212 + }, + { + "auxiliary_loss_clip": 0.01122721, + "auxiliary_loss_mlp": 0.01034063, + "balance_loss_clip": 1.03955698, + "balance_loss_mlp": 1.01987088, + "epoch": 0.8205374857211567, + "flos": 20887011106560.0, + "grad_norm": 1.95052653588115, + "language_loss": 0.73352933, + "learning_rate": 3.282954714811783e-07, + "loss": 0.75509715, + "num_input_tokens_seen": 147394580, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.14202881, + "step": 6824, + "time_per_iteration": 2.9834303855895996 + }, + { + "auxiliary_loss_clip": 0.01125022, + "auxiliary_loss_mlp": 0.01038616, + "balance_loss_clip": 1.04004335, + "balance_loss_mlp": 1.0227493, + "epoch": 0.8206577286117959, + "flos": 13152140294400.0, + "grad_norm": 2.4601615207937866, + "language_loss": 0.70869064, + "learning_rate": 3.2786797833495093e-07, + "loss": 0.73032701, + "num_input_tokens_seen": 147409935, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15881348, + "step": 6825, + "time_per_iteration": 2.9318454265594482 + }, + { + "auxiliary_loss_clip": 0.01124773, + "auxiliary_loss_mlp": 0.01036507, + "balance_loss_clip": 1.03964996, + "balance_loss_mlp": 1.02176642, + "epoch": 0.8207779715024349, + "flos": 25265634917760.0, + "grad_norm": 1.8637385715004096, + "language_loss": 0.72819382, + "learning_rate": 3.274407388498213e-07, + "loss": 0.74980658, + "num_input_tokens_seen": 147428065, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.14746094, + "step": 6826, + "time_per_iteration": 3.097909450531006 + }, + { + "auxiliary_loss_clip": 0.01129043, + "auxiliary_loss_mlp": 0.01040231, + "balance_loss_clip": 1.04309416, + "balance_loss_mlp": 1.02648652, + "epoch": 0.820898214393074, + "flos": 19610243199360.0, + "grad_norm": 1.8934763422177643, + "language_loss": 0.74461341, + "learning_rate": 3.270137530906021e-07, + "loss": 0.76630616, + "num_input_tokens_seen": 147447300, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.13745117, + "step": 6827, + "time_per_iteration": 2.9984641075134277 + }, + { + "auxiliary_loss_clip": 0.01121491, + "auxiliary_loss_mlp": 0.01030976, + "balance_loss_clip": 1.03869283, + "balance_loss_mlp": 1.01720142, + "epoch": 0.8210184572837131, + "flos": 15596615439360.0, + "grad_norm": 1.884964374695166, + "language_loss": 0.83560067, + "learning_rate": 3.265870211220665e-07, + "loss": 0.85712528, + "num_input_tokens_seen": 147465135, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.13769531, + "step": 6828, + "time_per_iteration": 3.633496046066284 + }, + { + "auxiliary_loss_clip": 0.01131096, + "auxiliary_loss_mlp": 0.01036368, + "balance_loss_clip": 1.04193354, + "balance_loss_mlp": 1.01910615, + "epoch": 0.8211387001743522, + "flos": 20813932886400.0, + "grad_norm": 1.8030581979833036, + "language_loss": 0.81305969, + "learning_rate": 3.2616054300894934e-07, + "loss": 0.83473432, + "num_input_tokens_seen": 147484585, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.17272949, + "step": 6829, + "time_per_iteration": 2.950378656387329 + }, + { + "auxiliary_loss_clip": 0.0112392, + "auxiliary_loss_mlp": 0.01039575, + "balance_loss_clip": 1.03944778, + "balance_loss_mlp": 1.02461386, + "epoch": 0.8212589430649913, + "flos": 27704579368320.0, + "grad_norm": 2.272176439633692, + "language_loss": 0.84676522, + "learning_rate": 3.2573431881594693e-07, + "loss": 0.86840016, + "num_input_tokens_seen": 147504130, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14953613, + "step": 6830, + "time_per_iteration": 2.780734062194824 + }, + { + "auxiliary_loss_clip": 0.01126375, + "auxiliary_loss_mlp": 0.01040703, + "balance_loss_clip": 1.03867936, + "balance_loss_mlp": 1.02534342, + "epoch": 0.8213791859556304, + "flos": 22455625017600.0, + "grad_norm": 2.1011683493383435, + "language_loss": 0.65462959, + "learning_rate": 3.2530834860771663e-07, + "loss": 0.67630041, + "num_input_tokens_seen": 147523510, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.15356445, + "step": 6831, + "time_per_iteration": 2.7833588123321533 + }, + { + "auxiliary_loss_clip": 0.01125348, + "auxiliary_loss_mlp": 0.0104, + "balance_loss_clip": 1.03922021, + "balance_loss_mlp": 1.02412128, + "epoch": 0.8214994288462695, + "flos": 16654471908480.0, + "grad_norm": 1.9283869877173634, + "language_loss": 0.74316663, + "learning_rate": 3.248826324488794e-07, + "loss": 0.76482016, + "num_input_tokens_seen": 147540805, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15881348, + "step": 6832, + "time_per_iteration": 3.573686361312866 + }, + { + "auxiliary_loss_clip": 0.01125699, + "auxiliary_loss_mlp": 0.01037585, + "balance_loss_clip": 1.04257512, + "balance_loss_mlp": 1.02285063, + "epoch": 0.8216196717369085, + "flos": 25221787390080.0, + "grad_norm": 1.7281877169346302, + "language_loss": 0.87781644, + "learning_rate": 3.244571704040138e-07, + "loss": 0.89944935, + "num_input_tokens_seen": 147560965, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.1472168, + "step": 6833, + "time_per_iteration": 2.773926019668579 + }, + { + "auxiliary_loss_clip": 0.01125596, + "auxiliary_loss_mlp": 0.01041378, + "balance_loss_clip": 1.03824151, + "balance_loss_mlp": 1.02554703, + "epoch": 0.8217399146275477, + "flos": 25371930240000.0, + "grad_norm": 1.788453378390535, + "language_loss": 0.73225832, + "learning_rate": 3.2403196253766374e-07, + "loss": 0.75392807, + "num_input_tokens_seen": 147580045, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15820312, + "step": 6834, + "time_per_iteration": 3.581620216369629 + }, + { + "auxiliary_loss_clip": 0.01131777, + "auxiliary_loss_mlp": 0.01041524, + "balance_loss_clip": 1.04153812, + "balance_loss_mlp": 1.02505493, + "epoch": 0.8218601575181868, + "flos": 25629625388160.0, + "grad_norm": 2.529412980592546, + "language_loss": 0.78775346, + "learning_rate": 3.2360700891433254e-07, + "loss": 0.80948645, + "num_input_tokens_seen": 147599070, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.16473389, + "step": 6835, + "time_per_iteration": 2.7992780208587646 + }, + { + "auxiliary_loss_clip": 0.0103209, + "auxiliary_loss_mlp": 0.01000512, + "balance_loss_clip": 1.00432611, + "balance_loss_mlp": 0.99910498, + "epoch": 0.8219804004088258, + "flos": 67660229427840.0, + "grad_norm": 0.8232929738672116, + "language_loss": 0.57270104, + "learning_rate": 3.231823095984847e-07, + "loss": 0.59302706, + "num_input_tokens_seen": 147653710, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01403809, + "step": 6836, + "time_per_iteration": 3.2053892612457275 + }, + { + "auxiliary_loss_clip": 0.01122879, + "auxiliary_loss_mlp": 0.01036852, + "balance_loss_clip": 1.03942096, + "balance_loss_mlp": 1.02263641, + "epoch": 0.822100643299465, + "flos": 19464266327040.0, + "grad_norm": 2.0950026066380474, + "language_loss": 0.76254988, + "learning_rate": 3.2275786465454814e-07, + "loss": 0.7841472, + "num_input_tokens_seen": 147670360, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14215088, + "step": 6837, + "time_per_iteration": 2.8032796382904053 + }, + { + "auxiliary_loss_clip": 0.01124467, + "auxiliary_loss_mlp": 0.01034008, + "balance_loss_clip": 1.03912246, + "balance_loss_mlp": 1.01958346, + "epoch": 0.822220886190104, + "flos": 24681368292480.0, + "grad_norm": 1.9101681970041953, + "language_loss": 0.75608373, + "learning_rate": 3.2233367414690917e-07, + "loss": 0.77766848, + "num_input_tokens_seen": 147692550, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.14416504, + "step": 6838, + "time_per_iteration": 3.762436628341675 + }, + { + "auxiliary_loss_clip": 0.01118772, + "auxiliary_loss_mlp": 0.01033764, + "balance_loss_clip": 1.03914762, + "balance_loss_mlp": 1.01925063, + "epoch": 0.8223411290807431, + "flos": 27819062991360.0, + "grad_norm": 3.5506179817987906, + "language_loss": 0.84759486, + "learning_rate": 3.219097381399183e-07, + "loss": 0.86912024, + "num_input_tokens_seen": 147709725, + "router_z_loss_clip": 0.79638672, + "router_z_loss_mlp": 0.14532471, + "step": 6839, + "time_per_iteration": 2.7770111560821533 + }, + { + "auxiliary_loss_clip": 0.01128833, + "auxiliary_loss_mlp": 0.01037651, + "balance_loss_clip": 1.04186535, + "balance_loss_mlp": 1.02166462, + "epoch": 0.8224613719713821, + "flos": 23218546913280.0, + "grad_norm": 1.7818086741313222, + "language_loss": 0.80961466, + "learning_rate": 3.2148605669788584e-07, + "loss": 0.83127952, + "num_input_tokens_seen": 147729615, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15979004, + "step": 6840, + "time_per_iteration": 2.831873893737793 + }, + { + "auxiliary_loss_clip": 0.0112725, + "auxiliary_loss_mlp": 0.01037545, + "balance_loss_clip": 1.04130507, + "balance_loss_mlp": 1.02284622, + "epoch": 0.8225816148620213, + "flos": 15706250726400.0, + "grad_norm": 2.200728555108632, + "language_loss": 0.77306473, + "learning_rate": 3.2106262988508405e-07, + "loss": 0.79471272, + "num_input_tokens_seen": 147747665, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.14697266, + "step": 6841, + "time_per_iteration": 2.760485887527466 + }, + { + "auxiliary_loss_clip": 0.01125845, + "auxiliary_loss_mlp": 0.01040593, + "balance_loss_clip": 1.03842103, + "balance_loss_mlp": 1.02428508, + "epoch": 0.8227018577526604, + "flos": 18515111391360.0, + "grad_norm": 2.2930413909574687, + "language_loss": 0.73991287, + "learning_rate": 3.206394577657465e-07, + "loss": 0.76157725, + "num_input_tokens_seen": 147765445, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.16320801, + "step": 6842, + "time_per_iteration": 2.823310613632202 + }, + { + "auxiliary_loss_clip": 0.01131972, + "auxiliary_loss_mlp": 0.01043321, + "balance_loss_clip": 1.04043889, + "balance_loss_mlp": 1.02664328, + "epoch": 0.8228221006432994, + "flos": 22236785406720.0, + "grad_norm": 2.961635436519299, + "language_loss": 0.71964955, + "learning_rate": 3.202165404040675e-07, + "loss": 0.74140251, + "num_input_tokens_seen": 147783365, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.16687012, + "step": 6843, + "time_per_iteration": 2.8764708042144775 + }, + { + "auxiliary_loss_clip": 0.01130236, + "auxiliary_loss_mlp": 0.01045069, + "balance_loss_clip": 1.04207015, + "balance_loss_mlp": 1.02972651, + "epoch": 0.8229423435339386, + "flos": 24097532630400.0, + "grad_norm": 2.59716839528566, + "language_loss": 0.74592584, + "learning_rate": 3.1979387786420396e-07, + "loss": 0.76767886, + "num_input_tokens_seen": 147803605, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15356445, + "step": 6844, + "time_per_iteration": 2.859645366668701 + }, + { + "auxiliary_loss_clip": 0.01126042, + "auxiliary_loss_mlp": 0.01035195, + "balance_loss_clip": 1.04098129, + "balance_loss_mlp": 1.02032948, + "epoch": 0.8230625864245776, + "flos": 23878549365120.0, + "grad_norm": 1.7226733480590786, + "language_loss": 0.8188597, + "learning_rate": 3.1937147021027346e-07, + "loss": 0.84047198, + "num_input_tokens_seen": 147822060, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.14868164, + "step": 6845, + "time_per_iteration": 2.781435966491699 + }, + { + "auxiliary_loss_clip": 0.01122762, + "auxiliary_loss_mlp": 0.01036097, + "balance_loss_clip": 1.03976822, + "balance_loss_mlp": 1.02114177, + "epoch": 0.8231828293152167, + "flos": 16581106379520.0, + "grad_norm": 2.5589511264963187, + "language_loss": 0.76367414, + "learning_rate": 3.189493175063547e-07, + "loss": 0.78526264, + "num_input_tokens_seen": 147839295, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.14953613, + "step": 6846, + "time_per_iteration": 2.8465399742126465 + }, + { + "auxiliary_loss_clip": 0.01127817, + "auxiliary_loss_mlp": 0.01043176, + "balance_loss_clip": 1.04194391, + "balance_loss_mlp": 1.02679646, + "epoch": 0.8233030722058559, + "flos": 18880071528960.0, + "grad_norm": 1.7556876909892338, + "language_loss": 0.67548895, + "learning_rate": 3.1852741981648776e-07, + "loss": 0.69719899, + "num_input_tokens_seen": 147857945, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.16381836, + "step": 6847, + "time_per_iteration": 2.9882237911224365 + }, + { + "auxiliary_loss_clip": 0.01124835, + "auxiliary_loss_mlp": 0.01038045, + "balance_loss_clip": 1.04107976, + "balance_loss_mlp": 1.02314949, + "epoch": 0.8234233150964949, + "flos": 28439024757120.0, + "grad_norm": 1.9381739019638715, + "language_loss": 0.69934404, + "learning_rate": 3.1810577720467404e-07, + "loss": 0.7209729, + "num_input_tokens_seen": 147879675, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.14892578, + "step": 6848, + "time_per_iteration": 2.871548652648926 + }, + { + "auxiliary_loss_clip": 0.01128373, + "auxiliary_loss_mlp": 0.01039292, + "balance_loss_clip": 1.04101038, + "balance_loss_mlp": 1.02407503, + "epoch": 0.823543557987134, + "flos": 33765941577600.0, + "grad_norm": 1.4655542627172267, + "language_loss": 0.56666046, + "learning_rate": 3.176843897348769e-07, + "loss": 0.58833712, + "num_input_tokens_seen": 147902870, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15246582, + "step": 6849, + "time_per_iteration": 2.937197208404541 + }, + { + "auxiliary_loss_clip": 0.01125169, + "auxiliary_loss_mlp": 0.0104324, + "balance_loss_clip": 1.04043531, + "balance_loss_mlp": 1.02757001, + "epoch": 0.8236638008777731, + "flos": 17092366611840.0, + "grad_norm": 2.5948662395215254, + "language_loss": 0.75401449, + "learning_rate": 3.1726325747102034e-07, + "loss": 0.7756986, + "num_input_tokens_seen": 147921245, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.15679932, + "step": 6850, + "time_per_iteration": 2.7134311199188232 + }, + { + "auxiliary_loss_clip": 0.01124566, + "auxiliary_loss_mlp": 0.0104216, + "balance_loss_clip": 1.03844571, + "balance_loss_mlp": 1.02709222, + "epoch": 0.8237840437684122, + "flos": 61639982334720.0, + "grad_norm": 1.5763744633935317, + "language_loss": 0.64004421, + "learning_rate": 3.1684238047698974e-07, + "loss": 0.66171145, + "num_input_tokens_seen": 147949515, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15075684, + "step": 6851, + "time_per_iteration": 3.063194990158081 + }, + { + "auxiliary_loss_clip": 0.01126845, + "auxiliary_loss_mlp": 0.01038932, + "balance_loss_clip": 1.03938293, + "balance_loss_mlp": 1.02227867, + "epoch": 0.8239042866590512, + "flos": 27309023821440.0, + "grad_norm": 1.87021372868488, + "language_loss": 0.53001297, + "learning_rate": 3.1642175881663155e-07, + "loss": 0.55167073, + "num_input_tokens_seen": 147969245, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.16650391, + "step": 6852, + "time_per_iteration": 2.7799394130706787 + }, + { + "auxiliary_loss_clip": 0.01123793, + "auxiliary_loss_mlp": 0.0102825, + "balance_loss_clip": 1.03887057, + "balance_loss_mlp": 1.01380205, + "epoch": 0.8240245295496904, + "flos": 21726351187200.0, + "grad_norm": 2.555591061645946, + "language_loss": 0.83862519, + "learning_rate": 3.160013925537537e-07, + "loss": 0.86014557, + "num_input_tokens_seen": 147990080, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.14459229, + "step": 6853, + "time_per_iteration": 2.7164113521575928 + }, + { + "auxiliary_loss_clip": 0.01126462, + "auxiliary_loss_mlp": 0.01035289, + "balance_loss_clip": 1.04039323, + "balance_loss_mlp": 1.01926684, + "epoch": 0.8241447724403295, + "flos": 20009318279040.0, + "grad_norm": 1.9192934369732506, + "language_loss": 0.75583643, + "learning_rate": 3.155812817521266e-07, + "loss": 0.7774539, + "num_input_tokens_seen": 148010455, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.16015625, + "step": 6854, + "time_per_iteration": 2.7909905910491943 + }, + { + "auxiliary_loss_clip": 0.01128017, + "auxiliary_loss_mlp": 0.01038474, + "balance_loss_clip": 1.04022717, + "balance_loss_mlp": 1.022452, + "epoch": 0.8242650153309685, + "flos": 22272983337600.0, + "grad_norm": 1.896942099400985, + "language_loss": 0.77894026, + "learning_rate": 3.151614264754787e-07, + "loss": 0.80060518, + "num_input_tokens_seen": 148028400, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.16015625, + "step": 6855, + "time_per_iteration": 2.7172656059265137 + }, + { + "auxiliary_loss_clip": 0.01125376, + "auxiliary_loss_mlp": 0.0103678, + "balance_loss_clip": 1.03806114, + "balance_loss_mlp": 1.02187896, + "epoch": 0.8243852582216077, + "flos": 22309971367680.0, + "grad_norm": 1.9574080075345444, + "language_loss": 0.79454988, + "learning_rate": 3.147418267875035e-07, + "loss": 0.81617141, + "num_input_tokens_seen": 148046530, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.14904785, + "step": 6856, + "time_per_iteration": 2.8010685443878174 + }, + { + "auxiliary_loss_clip": 0.01127548, + "auxiliary_loss_mlp": 0.0104312, + "balance_loss_clip": 1.03976417, + "balance_loss_mlp": 1.02793241, + "epoch": 0.8245055011122467, + "flos": 24645421756800.0, + "grad_norm": 2.161844206808043, + "language_loss": 0.65040982, + "learning_rate": 3.1432248275185315e-07, + "loss": 0.67211652, + "num_input_tokens_seen": 148067040, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.15185547, + "step": 6857, + "time_per_iteration": 3.5525994300842285 + }, + { + "auxiliary_loss_clip": 0.01127161, + "auxiliary_loss_mlp": 0.01043634, + "balance_loss_clip": 1.03951859, + "balance_loss_mlp": 1.02816117, + "epoch": 0.8246257440028858, + "flos": 17487275713920.0, + "grad_norm": 4.10116354661439, + "language_loss": 0.77671528, + "learning_rate": 3.139033944321412e-07, + "loss": 0.79842323, + "num_input_tokens_seen": 148084400, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.15478516, + "step": 6858, + "time_per_iteration": 2.779250383377075 + }, + { + "auxiliary_loss_clip": 0.0112636, + "auxiliary_loss_mlp": 0.01040095, + "balance_loss_clip": 1.03891873, + "balance_loss_mlp": 1.02492523, + "epoch": 0.824745986893525, + "flos": 25010130499200.0, + "grad_norm": 1.7577992752746026, + "language_loss": 0.78693825, + "learning_rate": 3.1348456189194507e-07, + "loss": 0.80860281, + "num_input_tokens_seen": 148104860, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15155029, + "step": 6859, + "time_per_iteration": 2.868845224380493 + }, + { + "auxiliary_loss_clip": 0.01125055, + "auxiliary_loss_mlp": 0.01038067, + "balance_loss_clip": 1.04010797, + "balance_loss_mlp": 1.02401245, + "epoch": 0.824866229784164, + "flos": 18772698798720.0, + "grad_norm": 1.602888708484449, + "language_loss": 0.82994968, + "learning_rate": 3.1306598519479876e-07, + "loss": 0.85158086, + "num_input_tokens_seen": 148124680, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.14038086, + "step": 6860, + "time_per_iteration": 2.7781007289886475 + }, + { + "auxiliary_loss_clip": 0.01123338, + "auxiliary_loss_mlp": 0.01037169, + "balance_loss_clip": 1.04031169, + "balance_loss_mlp": 1.0231204, + "epoch": 0.8249864726748031, + "flos": 23842171866240.0, + "grad_norm": 1.6610110550358597, + "language_loss": 0.78401065, + "learning_rate": 3.1264766440420177e-07, + "loss": 0.80561572, + "num_input_tokens_seen": 148147150, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.14044189, + "step": 6861, + "time_per_iteration": 3.5812478065490723 + }, + { + "auxiliary_loss_clip": 0.01123664, + "auxiliary_loss_mlp": 0.01035201, + "balance_loss_clip": 1.0406338, + "balance_loss_mlp": 1.02037752, + "epoch": 0.8251067155654422, + "flos": 20303103617280.0, + "grad_norm": 1.9371827511506812, + "language_loss": 0.6918844, + "learning_rate": 3.122295995836124e-07, + "loss": 0.71347308, + "num_input_tokens_seen": 148167020, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.14813232, + "step": 6862, + "time_per_iteration": 2.765512228012085 + }, + { + "auxiliary_loss_clip": 0.01126291, + "auxiliary_loss_mlp": 0.01038657, + "balance_loss_clip": 1.03925288, + "balance_loss_mlp": 1.0228138, + "epoch": 0.8252269584560813, + "flos": 25009699536000.0, + "grad_norm": 2.6073302847851663, + "language_loss": 0.77278906, + "learning_rate": 3.118117907964508e-07, + "loss": 0.79443848, + "num_input_tokens_seen": 148188965, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15844727, + "step": 6863, + "time_per_iteration": 3.590261220932007 + }, + { + "auxiliary_loss_clip": 0.01124797, + "auxiliary_loss_mlp": 0.01047613, + "balance_loss_clip": 1.04189825, + "balance_loss_mlp": 1.03205669, + "epoch": 0.8253472013467203, + "flos": 17128564542720.0, + "grad_norm": 2.012014009661009, + "language_loss": 0.80275267, + "learning_rate": 3.1139423810609856e-07, + "loss": 0.82447678, + "num_input_tokens_seen": 148205660, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.15563965, + "step": 6864, + "time_per_iteration": 2.843358278274536 + }, + { + "auxiliary_loss_clip": 0.0112627, + "auxiliary_loss_mlp": 0.01038765, + "balance_loss_clip": 1.03934097, + "balance_loss_mlp": 1.02287412, + "epoch": 0.8254674442373595, + "flos": 22414794232320.0, + "grad_norm": 1.6972535544002734, + "language_loss": 0.75414467, + "learning_rate": 3.1097694157589714e-07, + "loss": 0.77579504, + "num_input_tokens_seen": 148225545, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.15893555, + "step": 6865, + "time_per_iteration": 2.7674202919006348 + }, + { + "auxiliary_loss_clip": 0.01126591, + "auxiliary_loss_mlp": 0.01042801, + "balance_loss_clip": 1.04202437, + "balance_loss_mlp": 1.02681494, + "epoch": 0.8255876871279986, + "flos": 24786765774720.0, + "grad_norm": 2.901288218083331, + "language_loss": 0.75673443, + "learning_rate": 3.105599012691511e-07, + "loss": 0.77842832, + "num_input_tokens_seen": 148243975, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.15991211, + "step": 6866, + "time_per_iteration": 2.837911605834961 + }, + { + "auxiliary_loss_clip": 0.01123137, + "auxiliary_loss_mlp": 0.01037095, + "balance_loss_clip": 1.03995168, + "balance_loss_mlp": 1.02203321, + "epoch": 0.8257079300186376, + "flos": 27455431656960.0, + "grad_norm": 1.5140898015021058, + "language_loss": 0.82384074, + "learning_rate": 3.101431172491249e-07, + "loss": 0.84544301, + "num_input_tokens_seen": 148265520, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.1505127, + "step": 6867, + "time_per_iteration": 3.7245194911956787 + }, + { + "auxiliary_loss_clip": 0.01127177, + "auxiliary_loss_mlp": 0.01039828, + "balance_loss_clip": 1.03943336, + "balance_loss_mlp": 1.02467656, + "epoch": 0.8258281729092768, + "flos": 16471866142080.0, + "grad_norm": 2.7024815320061792, + "language_loss": 0.72019792, + "learning_rate": 3.097265895790444e-07, + "loss": 0.7418679, + "num_input_tokens_seen": 148283730, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15161133, + "step": 6868, + "time_per_iteration": 2.7410950660705566 + }, + { + "auxiliary_loss_clip": 0.01133275, + "auxiliary_loss_mlp": 0.01043364, + "balance_loss_clip": 1.04503942, + "balance_loss_mlp": 1.02829611, + "epoch": 0.8259484157999158, + "flos": 21433822824960.0, + "grad_norm": 1.875932869715316, + "language_loss": 0.83459806, + "learning_rate": 3.093103183220962e-07, + "loss": 0.85636449, + "num_input_tokens_seen": 148303775, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.15075684, + "step": 6869, + "time_per_iteration": 2.840390205383301 + }, + { + "auxiliary_loss_clip": 0.01033041, + "auxiliary_loss_mlp": 0.01001687, + "balance_loss_clip": 1.00516105, + "balance_loss_mlp": 1.00030446, + "epoch": 0.8260686586905549, + "flos": 58322342453760.0, + "grad_norm": 0.816657727845898, + "language_loss": 0.59381366, + "learning_rate": 3.0889430354142796e-07, + "loss": 0.61416101, + "num_input_tokens_seen": 148365285, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01385498, + "step": 6870, + "time_per_iteration": 3.3085262775421143 + }, + { + "auxiliary_loss_clip": 0.01126084, + "auxiliary_loss_mlp": 0.01030024, + "balance_loss_clip": 1.03885221, + "balance_loss_mlp": 1.0137881, + "epoch": 0.826188901581194, + "flos": 27527288814720.0, + "grad_norm": 1.7683278688299195, + "language_loss": 0.70360076, + "learning_rate": 3.084785453001497e-07, + "loss": 0.72516185, + "num_input_tokens_seen": 148386200, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.16235352, + "step": 6871, + "time_per_iteration": 2.9265177249908447 + }, + { + "auxiliary_loss_clip": 0.01124762, + "auxiliary_loss_mlp": 0.01037751, + "balance_loss_clip": 1.03810096, + "balance_loss_mlp": 1.02276611, + "epoch": 0.8263091444718331, + "flos": 23696051339520.0, + "grad_norm": 2.1809829965859784, + "language_loss": 0.81856424, + "learning_rate": 3.080630436613314e-07, + "loss": 0.8401894, + "num_input_tokens_seen": 148403970, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.14990234, + "step": 6872, + "time_per_iteration": 2.7559845447540283 + }, + { + "auxiliary_loss_clip": 0.01124165, + "auxiliary_loss_mlp": 0.0103447, + "balance_loss_clip": 1.04025817, + "balance_loss_mlp": 1.02100587, + "epoch": 0.8264293873624722, + "flos": 17165157523200.0, + "grad_norm": 1.964715541727119, + "language_loss": 0.86269116, + "learning_rate": 3.076477986880039e-07, + "loss": 0.88427746, + "num_input_tokens_seen": 148421765, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.13464355, + "step": 6873, + "time_per_iteration": 2.7898359298706055 + }, + { + "auxiliary_loss_clip": 0.0112487, + "auxiliary_loss_mlp": 0.01042019, + "balance_loss_clip": 1.04054868, + "balance_loss_mlp": 1.02784491, + "epoch": 0.8265496302531112, + "flos": 24098645952000.0, + "grad_norm": 2.148709007115889, + "language_loss": 0.69186401, + "learning_rate": 3.0723281044315986e-07, + "loss": 0.71353292, + "num_input_tokens_seen": 148443720, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.14178467, + "step": 6874, + "time_per_iteration": 2.8027753829956055 + }, + { + "auxiliary_loss_clip": 0.0111877, + "auxiliary_loss_mlp": 0.01038869, + "balance_loss_clip": 1.03687263, + "balance_loss_mlp": 1.02447486, + "epoch": 0.8266698731437504, + "flos": 14099894599680.0, + "grad_norm": 2.210928366264871, + "language_loss": 0.76038301, + "learning_rate": 3.068180789897521e-07, + "loss": 0.78195941, + "num_input_tokens_seen": 148462130, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.14379883, + "step": 6875, + "time_per_iteration": 2.756885051727295 + }, + { + "auxiliary_loss_clip": 0.0112923, + "auxiliary_loss_mlp": 0.01037017, + "balance_loss_clip": 1.03974247, + "balance_loss_mlp": 1.02054226, + "epoch": 0.8267901160343895, + "flos": 30777563715840.0, + "grad_norm": 1.637922162598292, + "language_loss": 0.81740785, + "learning_rate": 3.064036043906966e-07, + "loss": 0.83907032, + "num_input_tokens_seen": 148485570, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.16479492, + "step": 6876, + "time_per_iteration": 2.833786964416504 + }, + { + "auxiliary_loss_clip": 0.01129371, + "auxiliary_loss_mlp": 0.01042785, + "balance_loss_clip": 1.04044902, + "balance_loss_mlp": 1.0259527, + "epoch": 0.8269103589250285, + "flos": 40624915242240.0, + "grad_norm": 2.500321871007266, + "language_loss": 0.67993629, + "learning_rate": 3.059893867088668e-07, + "loss": 0.70165789, + "num_input_tokens_seen": 148509715, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.16845703, + "step": 6877, + "time_per_iteration": 2.916893482208252 + }, + { + "auxiliary_loss_clip": 0.01121346, + "auxiliary_loss_mlp": 0.01038877, + "balance_loss_clip": 1.03864312, + "balance_loss_mlp": 1.02466726, + "epoch": 0.8270306018156677, + "flos": 30263645877120.0, + "grad_norm": 1.8648924843681194, + "language_loss": 0.67084002, + "learning_rate": 3.055754260071004e-07, + "loss": 0.69244218, + "num_input_tokens_seen": 148532010, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.14208984, + "step": 6878, + "time_per_iteration": 2.861781597137451 + }, + { + "auxiliary_loss_clip": 0.0112318, + "auxiliary_loss_mlp": 0.01036971, + "balance_loss_clip": 1.03896558, + "balance_loss_mlp": 1.02285647, + "epoch": 0.8271508447063067, + "flos": 25226599812480.0, + "grad_norm": 1.8446327804579907, + "language_loss": 0.7359575, + "learning_rate": 3.051617223481948e-07, + "loss": 0.75755906, + "num_input_tokens_seen": 148553330, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14099121, + "step": 6879, + "time_per_iteration": 2.823122262954712 + }, + { + "auxiliary_loss_clip": 0.01130813, + "auxiliary_loss_mlp": 0.01047611, + "balance_loss_clip": 1.0437026, + "balance_loss_mlp": 1.0286926, + "epoch": 0.8272710875969458, + "flos": 17566602900480.0, + "grad_norm": 1.9009923682489427, + "language_loss": 0.75413966, + "learning_rate": 3.047482757949078e-07, + "loss": 0.77592385, + "num_input_tokens_seen": 148570960, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.18908691, + "step": 6880, + "time_per_iteration": 2.8004150390625 + }, + { + "auxiliary_loss_clip": 0.01119948, + "auxiliary_loss_mlp": 0.0103398, + "balance_loss_clip": 1.03779697, + "balance_loss_mlp": 1.01979399, + "epoch": 0.827391330487585, + "flos": 19755465886080.0, + "grad_norm": 1.9135050385962107, + "language_loss": 0.85969651, + "learning_rate": 3.043350864099605e-07, + "loss": 0.88123584, + "num_input_tokens_seen": 148589520, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.14196777, + "step": 6881, + "time_per_iteration": 2.791285753250122 + }, + { + "auxiliary_loss_clip": 0.01122686, + "auxiliary_loss_mlp": 0.01033571, + "balance_loss_clip": 1.03739846, + "balance_loss_mlp": 1.01885486, + "epoch": 0.827511573378224, + "flos": 16835174254080.0, + "grad_norm": 2.066063007638569, + "language_loss": 0.80761039, + "learning_rate": 3.039221542560315e-07, + "loss": 0.82917285, + "num_input_tokens_seen": 148606085, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.14727783, + "step": 6882, + "time_per_iteration": 2.772831678390503 + }, + { + "auxiliary_loss_clip": 0.01121599, + "auxiliary_loss_mlp": 0.01036952, + "balance_loss_clip": 1.03863573, + "balance_loss_mlp": 1.02173519, + "epoch": 0.8276318162688631, + "flos": 18369242259840.0, + "grad_norm": 1.8849305695688974, + "language_loss": 0.73847252, + "learning_rate": 3.0350947939576356e-07, + "loss": 0.76005805, + "num_input_tokens_seen": 148625240, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.15216064, + "step": 6883, + "time_per_iteration": 2.8806862831115723 + }, + { + "auxiliary_loss_clip": 0.01129154, + "auxiliary_loss_mlp": 0.01045387, + "balance_loss_clip": 1.04206204, + "balance_loss_mlp": 1.02961564, + "epoch": 0.8277520591595022, + "flos": 19352691705600.0, + "grad_norm": 1.6470424285697323, + "language_loss": 0.72413492, + "learning_rate": 3.0309706189175876e-07, + "loss": 0.74588031, + "num_input_tokens_seen": 148645075, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15777588, + "step": 6884, + "time_per_iteration": 2.848500967025757 + }, + { + "auxiliary_loss_clip": 0.01032282, + "auxiliary_loss_mlp": 0.01004354, + "balance_loss_clip": 1.00436568, + "balance_loss_mlp": 1.00294137, + "epoch": 0.8278723020501413, + "flos": 67918858329600.0, + "grad_norm": 0.7761336028435485, + "language_loss": 0.57396489, + "learning_rate": 3.0268490180658045e-07, + "loss": 0.59433126, + "num_input_tokens_seen": 148707855, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01409912, + "step": 6885, + "time_per_iteration": 3.355299472808838 + }, + { + "auxiliary_loss_clip": 0.01127692, + "auxiliary_loss_mlp": 0.01037295, + "balance_loss_clip": 1.04167104, + "balance_loss_mlp": 1.02217364, + "epoch": 0.8279925449407803, + "flos": 18185738653440.0, + "grad_norm": 2.33661942747089, + "language_loss": 0.79395497, + "learning_rate": 3.0227299920275305e-07, + "loss": 0.81560481, + "num_input_tokens_seen": 148724170, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15130615, + "step": 6886, + "time_per_iteration": 3.5760607719421387 + }, + { + "auxiliary_loss_clip": 0.01135313, + "auxiliary_loss_mlp": 0.01040564, + "balance_loss_clip": 1.04318357, + "balance_loss_mlp": 1.02331376, + "epoch": 0.8281127878314195, + "flos": 20631434860800.0, + "grad_norm": 2.068481065342149, + "language_loss": 0.85858721, + "learning_rate": 3.018613541427613e-07, + "loss": 0.880346, + "num_input_tokens_seen": 148743690, + "router_z_loss_clip": 0.92089844, + "router_z_loss_mlp": 0.17248535, + "step": 6887, + "time_per_iteration": 2.8211607933044434 + }, + { + "auxiliary_loss_clip": 0.01124607, + "auxiliary_loss_mlp": 0.01036277, + "balance_loss_clip": 1.04054213, + "balance_loss_mlp": 1.02142382, + "epoch": 0.8282330307220586, + "flos": 18004282122240.0, + "grad_norm": 1.620042665074983, + "language_loss": 0.73916411, + "learning_rate": 3.0144996668905243e-07, + "loss": 0.76077294, + "num_input_tokens_seen": 148761070, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.14849854, + "step": 6888, + "time_per_iteration": 2.776933193206787 + }, + { + "auxiliary_loss_clip": 0.01128107, + "auxiliary_loss_mlp": 0.01038628, + "balance_loss_clip": 1.04280114, + "balance_loss_mlp": 1.02296388, + "epoch": 0.8283532736126976, + "flos": 20084120352000.0, + "grad_norm": 2.072753332185525, + "language_loss": 0.82392311, + "learning_rate": 3.010388369040331e-07, + "loss": 0.84559041, + "num_input_tokens_seen": 148779730, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.15673828, + "step": 6889, + "time_per_iteration": 2.700080156326294 + }, + { + "auxiliary_loss_clip": 0.0112609, + "auxiliary_loss_mlp": 0.01040355, + "balance_loss_clip": 1.0411396, + "balance_loss_mlp": 1.02620459, + "epoch": 0.8284735165033368, + "flos": 31868421805440.0, + "grad_norm": 1.3854003417348462, + "language_loss": 0.82444865, + "learning_rate": 3.0062796485007156e-07, + "loss": 0.84611309, + "num_input_tokens_seen": 148800670, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.14160156, + "step": 6890, + "time_per_iteration": 3.5322484970092773 + }, + { + "auxiliary_loss_clip": 0.01126725, + "auxiliary_loss_mlp": 0.01041818, + "balance_loss_clip": 1.0402329, + "balance_loss_mlp": 1.02605867, + "epoch": 0.8285937593939758, + "flos": 26651319840000.0, + "grad_norm": 2.1656130005360303, + "language_loss": 0.65185267, + "learning_rate": 3.002173505894965e-07, + "loss": 0.67353815, + "num_input_tokens_seen": 148819820, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15759277, + "step": 6891, + "time_per_iteration": 2.79752254486084 + }, + { + "auxiliary_loss_clip": 0.01129225, + "auxiliary_loss_mlp": 0.01040125, + "balance_loss_clip": 1.0403378, + "balance_loss_mlp": 1.02368557, + "epoch": 0.8287140022846149, + "flos": 20193683811840.0, + "grad_norm": 2.2337325537207224, + "language_loss": 0.62669629, + "learning_rate": 2.998069941845973e-07, + "loss": 0.64838982, + "num_input_tokens_seen": 148838890, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.16448975, + "step": 6892, + "time_per_iteration": 3.4908089637756348 + }, + { + "auxiliary_loss_clip": 0.01032062, + "auxiliary_loss_mlp": 0.01000984, + "balance_loss_clip": 1.00428891, + "balance_loss_mlp": 0.99972075, + "epoch": 0.8288342451752541, + "flos": 70755980019840.0, + "grad_norm": 0.740934636457203, + "language_loss": 0.57455271, + "learning_rate": 2.993968956976258e-07, + "loss": 0.59488314, + "num_input_tokens_seen": 148906635, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01263428, + "step": 6893, + "time_per_iteration": 3.3961033821105957 + }, + { + "auxiliary_loss_clip": 0.01133365, + "auxiliary_loss_mlp": 0.01038273, + "balance_loss_clip": 1.04197752, + "balance_loss_mlp": 1.02195346, + "epoch": 0.8289544880658931, + "flos": 24572235795840.0, + "grad_norm": 3.105939653070882, + "language_loss": 0.70229387, + "learning_rate": 2.9898705519079313e-07, + "loss": 0.72401023, + "num_input_tokens_seen": 148925740, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.16333008, + "step": 6894, + "time_per_iteration": 2.7777700424194336 + }, + { + "auxiliary_loss_clip": 0.01124576, + "auxiliary_loss_mlp": 0.01033765, + "balance_loss_clip": 1.04118979, + "balance_loss_mlp": 1.01833904, + "epoch": 0.8290747309565322, + "flos": 22273378387200.0, + "grad_norm": 1.7099847472224394, + "language_loss": 0.74745321, + "learning_rate": 2.985774727262715e-07, + "loss": 0.76903659, + "num_input_tokens_seen": 148944585, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.1541748, + "step": 6895, + "time_per_iteration": 2.905827760696411 + }, + { + "auxiliary_loss_clip": 0.01124434, + "auxiliary_loss_mlp": 0.0103251, + "balance_loss_clip": 1.04071462, + "balance_loss_mlp": 1.01824021, + "epoch": 0.8291949738471713, + "flos": 23255570856960.0, + "grad_norm": 5.907188581975579, + "language_loss": 0.81348765, + "learning_rate": 2.981681483661949e-07, + "loss": 0.83505702, + "num_input_tokens_seen": 148964170, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14282227, + "step": 6896, + "time_per_iteration": 3.61623215675354 + }, + { + "auxiliary_loss_clip": 0.01126552, + "auxiliary_loss_mlp": 0.01040883, + "balance_loss_clip": 1.0412699, + "balance_loss_mlp": 1.02639937, + "epoch": 0.8293152167378104, + "flos": 52555768185600.0, + "grad_norm": 1.6683515324816836, + "language_loss": 0.70947587, + "learning_rate": 2.9775908217265633e-07, + "loss": 0.73115027, + "num_input_tokens_seen": 148989405, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.14489746, + "step": 6897, + "time_per_iteration": 3.006141424179077 + }, + { + "auxiliary_loss_clip": 0.01031753, + "auxiliary_loss_mlp": 0.01003508, + "balance_loss_clip": 1.0039475, + "balance_loss_mlp": 1.00207758, + "epoch": 0.8294354596284494, + "flos": 63356156294400.0, + "grad_norm": 0.8308201282306961, + "language_loss": 0.50414705, + "learning_rate": 2.9735027420771253e-07, + "loss": 0.52449965, + "num_input_tokens_seen": 149049740, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01428223, + "step": 6898, + "time_per_iteration": 3.230736494064331 + }, + { + "auxiliary_loss_clip": 0.01120495, + "auxiliary_loss_mlp": 0.01038012, + "balance_loss_clip": 1.03948998, + "balance_loss_mlp": 1.02332568, + "epoch": 0.8295557025190886, + "flos": 24827021942400.0, + "grad_norm": 1.9381924825715366, + "language_loss": 0.71471405, + "learning_rate": 2.969417245333774e-07, + "loss": 0.73629916, + "num_input_tokens_seen": 149069120, + "router_z_loss_clip": 0.80908203, + "router_z_loss_mlp": 0.14709473, + "step": 6899, + "time_per_iteration": 2.903144359588623 + }, + { + "auxiliary_loss_clip": 0.01122735, + "auxiliary_loss_mlp": 0.01039094, + "balance_loss_clip": 1.0394845, + "balance_loss_mlp": 1.02506304, + "epoch": 0.8296759454097277, + "flos": 25118580637440.0, + "grad_norm": 2.0809162240449197, + "language_loss": 0.77841419, + "learning_rate": 2.9653343321162915e-07, + "loss": 0.8000325, + "num_input_tokens_seen": 149088630, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.14050293, + "step": 6900, + "time_per_iteration": 2.822817802429199 + }, + { + "auxiliary_loss_clip": 0.0112936, + "auxiliary_loss_mlp": 0.01034845, + "balance_loss_clip": 1.04138756, + "balance_loss_mlp": 1.01916885, + "epoch": 0.8297961883003667, + "flos": 24132581326080.0, + "grad_norm": 2.228100515448187, + "language_loss": 0.64935881, + "learning_rate": 2.9612540030440446e-07, + "loss": 0.67100084, + "num_input_tokens_seen": 149109175, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.15686035, + "step": 6901, + "time_per_iteration": 2.880368709564209 + }, + { + "auxiliary_loss_clip": 0.01031844, + "auxiliary_loss_mlp": 0.01004113, + "balance_loss_clip": 1.00411057, + "balance_loss_mlp": 1.00269401, + "epoch": 0.8299164311910058, + "flos": 67446561375360.0, + "grad_norm": 0.8502590433246869, + "language_loss": 0.64164752, + "learning_rate": 2.9571762587360206e-07, + "loss": 0.66200709, + "num_input_tokens_seen": 149165560, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01416016, + "step": 6902, + "time_per_iteration": 3.2416505813598633 + }, + { + "auxiliary_loss_clip": 0.01121264, + "auxiliary_loss_mlp": 0.01035383, + "balance_loss_clip": 1.0384047, + "balance_loss_mlp": 1.02054095, + "epoch": 0.8300366740816449, + "flos": 25228682801280.0, + "grad_norm": 1.6147344725676567, + "language_loss": 0.73985595, + "learning_rate": 2.953101099810806e-07, + "loss": 0.76142246, + "num_input_tokens_seen": 149185165, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.14855957, + "step": 6903, + "time_per_iteration": 2.861504316329956 + }, + { + "auxiliary_loss_clip": 0.01123129, + "auxiliary_loss_mlp": 0.01036319, + "balance_loss_clip": 1.04025829, + "balance_loss_mlp": 1.02131081, + "epoch": 0.830156916972284, + "flos": 18041018757120.0, + "grad_norm": 4.113826198673441, + "language_loss": 0.82604063, + "learning_rate": 2.9490285268865965e-07, + "loss": 0.84763503, + "num_input_tokens_seen": 149202655, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.14990234, + "step": 6904, + "time_per_iteration": 2.816044807434082 + }, + { + "auxiliary_loss_clip": 0.0112937, + "auxiliary_loss_mlp": 0.01042064, + "balance_loss_clip": 1.04099905, + "balance_loss_mlp": 1.02589345, + "epoch": 0.830277159862923, + "flos": 26322485806080.0, + "grad_norm": 2.141205274905818, + "language_loss": 0.79449356, + "learning_rate": 2.9449585405812085e-07, + "loss": 0.81620789, + "num_input_tokens_seen": 149220035, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.1618042, + "step": 6905, + "time_per_iteration": 2.820967197418213 + }, + { + "auxiliary_loss_clip": 0.01127807, + "auxiliary_loss_mlp": 0.01041307, + "balance_loss_clip": 1.04129362, + "balance_loss_mlp": 1.0260663, + "epoch": 0.8303974027535622, + "flos": 19938861751680.0, + "grad_norm": 1.9835592767303079, + "language_loss": 0.73851901, + "learning_rate": 2.940891141512043e-07, + "loss": 0.76021016, + "num_input_tokens_seen": 149238055, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15240479, + "step": 6906, + "time_per_iteration": 2.9007527828216553 + }, + { + "auxiliary_loss_clip": 0.01126002, + "auxiliary_loss_mlp": 0.01038786, + "balance_loss_clip": 1.04053664, + "balance_loss_mlp": 1.02301431, + "epoch": 0.8305176456442013, + "flos": 17165552572800.0, + "grad_norm": 2.1677371369157554, + "language_loss": 0.71915942, + "learning_rate": 2.9368263302961385e-07, + "loss": 0.74080729, + "num_input_tokens_seen": 149256755, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15771484, + "step": 6907, + "time_per_iteration": 2.7695891857147217 + }, + { + "auxiliary_loss_clip": 0.01126398, + "auxiliary_loss_mlp": 0.0103507, + "balance_loss_clip": 1.03915966, + "balance_loss_mlp": 1.01929903, + "epoch": 0.8306378885348403, + "flos": 25627614226560.0, + "grad_norm": 1.7746405977394604, + "language_loss": 0.79719526, + "learning_rate": 2.9327641075501075e-07, + "loss": 0.81880999, + "num_input_tokens_seen": 149275745, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15771484, + "step": 6908, + "time_per_iteration": 2.8172104358673096 + }, + { + "auxiliary_loss_clip": 0.01125082, + "auxiliary_loss_mlp": 0.01044616, + "balance_loss_clip": 1.03939319, + "balance_loss_mlp": 1.02805805, + "epoch": 0.8307581314254795, + "flos": 33947864985600.0, + "grad_norm": 2.0847663653034134, + "language_loss": 0.66502178, + "learning_rate": 2.9287044738901866e-07, + "loss": 0.68671876, + "num_input_tokens_seen": 149293730, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.16552734, + "step": 6909, + "time_per_iteration": 2.8112480640411377 + }, + { + "auxiliary_loss_clip": 0.01123921, + "auxiliary_loss_mlp": 0.01042927, + "balance_loss_clip": 1.03981602, + "balance_loss_mlp": 1.0267024, + "epoch": 0.8308783743161186, + "flos": 17562724231680.0, + "grad_norm": 3.276309780326278, + "language_loss": 0.90794462, + "learning_rate": 2.9246474299322274e-07, + "loss": 0.92961311, + "num_input_tokens_seen": 149309290, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.16210938, + "step": 6910, + "time_per_iteration": 2.7344906330108643 + }, + { + "auxiliary_loss_clip": 0.01031908, + "auxiliary_loss_mlp": 0.01003163, + "balance_loss_clip": 1.00416398, + "balance_loss_mlp": 1.00172067, + "epoch": 0.8309986172067576, + "flos": 69412885649280.0, + "grad_norm": 0.8884117851248363, + "language_loss": 0.63133729, + "learning_rate": 2.920592976291678e-07, + "loss": 0.65168798, + "num_input_tokens_seen": 149366620, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.0144043, + "step": 6911, + "time_per_iteration": 3.2383720874786377 + }, + { + "auxiliary_loss_clip": 0.01127604, + "auxiliary_loss_mlp": 0.01042825, + "balance_loss_clip": 1.04144216, + "balance_loss_mlp": 1.0273279, + "epoch": 0.8311188600973968, + "flos": 22309755886080.0, + "grad_norm": 1.9340289917653977, + "language_loss": 0.80489039, + "learning_rate": 2.916541113583595e-07, + "loss": 0.82659477, + "num_input_tokens_seen": 149385120, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15478516, + "step": 6912, + "time_per_iteration": 2.754249095916748 + }, + { + "auxiliary_loss_clip": 0.01124842, + "auxiliary_loss_mlp": 0.01037255, + "balance_loss_clip": 1.04145432, + "balance_loss_mlp": 1.02091718, + "epoch": 0.8312391029880358, + "flos": 18770077105920.0, + "grad_norm": 2.097089814764407, + "language_loss": 0.67072284, + "learning_rate": 2.912491842422642e-07, + "loss": 0.69234371, + "num_input_tokens_seen": 149402825, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.16308594, + "step": 6913, + "time_per_iteration": 2.812666177749634 + }, + { + "auxiliary_loss_clip": 0.0112847, + "auxiliary_loss_mlp": 0.0103389, + "balance_loss_clip": 1.04271722, + "balance_loss_mlp": 1.0193342, + "epoch": 0.8313593458786749, + "flos": 20376648714240.0, + "grad_norm": 1.5699650738172939, + "language_loss": 0.71012402, + "learning_rate": 2.9084451634230857e-07, + "loss": 0.73174763, + "num_input_tokens_seen": 149422125, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.14550781, + "step": 6914, + "time_per_iteration": 3.663729667663574 + }, + { + "auxiliary_loss_clip": 0.01124416, + "auxiliary_loss_mlp": 0.0104113, + "balance_loss_clip": 1.03810513, + "balance_loss_mlp": 1.02651477, + "epoch": 0.831479588769314, + "flos": 32124069878400.0, + "grad_norm": 1.9003286386875486, + "language_loss": 0.7149384, + "learning_rate": 2.9044010771988125e-07, + "loss": 0.7365939, + "num_input_tokens_seen": 149441940, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.14611816, + "step": 6915, + "time_per_iteration": 2.876465320587158 + }, + { + "auxiliary_loss_clip": 0.01123416, + "auxiliary_loss_mlp": 0.01042725, + "balance_loss_clip": 1.03924894, + "balance_loss_mlp": 1.02751422, + "epoch": 0.8315998316599531, + "flos": 45185929338240.0, + "grad_norm": 1.6416447906507652, + "language_loss": 0.71999913, + "learning_rate": 2.900359584363303e-07, + "loss": 0.74166048, + "num_input_tokens_seen": 149465045, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.15209961, + "step": 6916, + "time_per_iteration": 3.072416067123413 + }, + { + "auxiliary_loss_clip": 0.01132883, + "auxiliary_loss_mlp": 0.01048155, + "balance_loss_clip": 1.04334736, + "balance_loss_mlp": 1.03216934, + "epoch": 0.8317200745505922, + "flos": 18363747479040.0, + "grad_norm": 2.321293553831638, + "language_loss": 0.84547341, + "learning_rate": 2.8963206855296494e-07, + "loss": 0.86728376, + "num_input_tokens_seen": 149481285, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.15991211, + "step": 6917, + "time_per_iteration": 2.7469189167022705 + }, + { + "auxiliary_loss_clip": 0.01122924, + "auxiliary_loss_mlp": 0.01038405, + "balance_loss_clip": 1.0391891, + "balance_loss_mlp": 1.02362895, + "epoch": 0.8318403174412313, + "flos": 24206557386240.0, + "grad_norm": 1.7251647304304132, + "language_loss": 0.76746786, + "learning_rate": 2.892284381310548e-07, + "loss": 0.78908116, + "num_input_tokens_seen": 149502700, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.14770508, + "step": 6918, + "time_per_iteration": 2.8499252796173096 + }, + { + "auxiliary_loss_clip": 0.01125055, + "auxiliary_loss_mlp": 0.01036976, + "balance_loss_clip": 1.03810418, + "balance_loss_mlp": 1.02100194, + "epoch": 0.8319605603318704, + "flos": 22418780641920.0, + "grad_norm": 2.547657430123319, + "language_loss": 0.72924185, + "learning_rate": 2.888250672318302e-07, + "loss": 0.75086212, + "num_input_tokens_seen": 149520100, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.15979004, + "step": 6919, + "time_per_iteration": 3.6475212574005127 + }, + { + "auxiliary_loss_clip": 0.01129328, + "auxiliary_loss_mlp": 0.01043829, + "balance_loss_clip": 1.04298115, + "balance_loss_mlp": 1.02919054, + "epoch": 0.8320808032225094, + "flos": 37414501459200.0, + "grad_norm": 1.6349518018028002, + "language_loss": 0.68670893, + "learning_rate": 2.884219559164831e-07, + "loss": 0.70844054, + "num_input_tokens_seen": 149543245, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.14648438, + "step": 6920, + "time_per_iteration": 3.766388416290283 + }, + { + "auxiliary_loss_clip": 0.01124609, + "auxiliary_loss_mlp": 0.01041687, + "balance_loss_clip": 1.04019272, + "balance_loss_mlp": 1.02578461, + "epoch": 0.8322010461131486, + "flos": 12787395638400.0, + "grad_norm": 2.048770408207159, + "language_loss": 0.81635177, + "learning_rate": 2.880191042461635e-07, + "loss": 0.83801466, + "num_input_tokens_seen": 149559185, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.15917969, + "step": 6921, + "time_per_iteration": 2.825610399246216 + }, + { + "auxiliary_loss_clip": 0.01123632, + "auxiliary_loss_mlp": 0.01034417, + "balance_loss_clip": 1.04090738, + "balance_loss_mlp": 1.0198791, + "epoch": 0.8323212890037877, + "flos": 15815455050240.0, + "grad_norm": 1.9837505053628504, + "language_loss": 0.8033725, + "learning_rate": 2.876165122819849e-07, + "loss": 0.82495296, + "num_input_tokens_seen": 149577165, + "router_z_loss_clip": 0.82763672, + "router_z_loss_mlp": 0.14538574, + "step": 6922, + "time_per_iteration": 2.7699337005615234 + }, + { + "auxiliary_loss_clip": 0.01122577, + "auxiliary_loss_mlp": 0.01037986, + "balance_loss_clip": 1.03814077, + "balance_loss_mlp": 1.02263188, + "epoch": 0.8324415318944267, + "flos": 21719276208000.0, + "grad_norm": 1.5432101376270237, + "language_loss": 0.79422772, + "learning_rate": 2.872141800850201e-07, + "loss": 0.81583333, + "num_input_tokens_seen": 149594340, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15356445, + "step": 6923, + "time_per_iteration": 2.7745344638824463 + }, + { + "auxiliary_loss_clip": 0.01121593, + "auxiliary_loss_mlp": 0.01035697, + "balance_loss_clip": 1.03814602, + "balance_loss_mlp": 1.02192283, + "epoch": 0.8325617747850659, + "flos": 34198700636160.0, + "grad_norm": 1.9364947110833584, + "language_loss": 0.73077738, + "learning_rate": 2.868121077163024e-07, + "loss": 0.75235027, + "num_input_tokens_seen": 149613895, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.13757324, + "step": 6924, + "time_per_iteration": 2.8060519695281982 + }, + { + "auxiliary_loss_clip": 0.01127056, + "auxiliary_loss_mlp": 0.01042437, + "balance_loss_clip": 1.04020357, + "balance_loss_mlp": 1.02653408, + "epoch": 0.8326820176757049, + "flos": 18369457741440.0, + "grad_norm": 2.616447673942138, + "language_loss": 0.72504485, + "learning_rate": 2.864102952368257e-07, + "loss": 0.74673986, + "num_input_tokens_seen": 149631820, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15930176, + "step": 6925, + "time_per_iteration": 3.64446759223938 + }, + { + "auxiliary_loss_clip": 0.01123699, + "auxiliary_loss_mlp": 0.01036391, + "balance_loss_clip": 1.03943408, + "balance_loss_mlp": 1.02137089, + "epoch": 0.832802260566344, + "flos": 35991325716480.0, + "grad_norm": 1.2757193285883532, + "language_loss": 0.59156513, + "learning_rate": 2.860087427075444e-07, + "loss": 0.61316603, + "num_input_tokens_seen": 149656070, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.15014648, + "step": 6926, + "time_per_iteration": 2.936553716659546 + }, + { + "auxiliary_loss_clip": 0.01124851, + "auxiliary_loss_mlp": 0.01038996, + "balance_loss_clip": 1.04066193, + "balance_loss_mlp": 1.02487576, + "epoch": 0.8329225034569832, + "flos": 14244434928000.0, + "grad_norm": 3.0723612143376804, + "language_loss": 0.86081421, + "learning_rate": 2.856074501893744e-07, + "loss": 0.88245273, + "num_input_tokens_seen": 149671270, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14117432, + "step": 6927, + "time_per_iteration": 2.727365732192993 + }, + { + "auxiliary_loss_clip": 0.01129191, + "auxiliary_loss_mlp": 0.01049753, + "balance_loss_clip": 1.04158235, + "balance_loss_mlp": 1.03265822, + "epoch": 0.8330427463476222, + "flos": 18077468083200.0, + "grad_norm": 1.699035009489134, + "language_loss": 0.81977713, + "learning_rate": 2.8520641774319054e-07, + "loss": 0.84156656, + "num_input_tokens_seen": 149689360, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.17102051, + "step": 6928, + "time_per_iteration": 2.8076870441436768 + }, + { + "auxiliary_loss_clip": 0.01126868, + "auxiliary_loss_mlp": 0.0103685, + "balance_loss_clip": 1.03904164, + "balance_loss_mlp": 1.02049494, + "epoch": 0.8331629892382613, + "flos": 18040839189120.0, + "grad_norm": 2.4148224648342884, + "language_loss": 0.75736696, + "learning_rate": 2.848056454298309e-07, + "loss": 0.7790041, + "num_input_tokens_seen": 149706685, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.16369629, + "step": 6929, + "time_per_iteration": 2.747506856918335 + }, + { + "auxiliary_loss_clip": 0.01125276, + "auxiliary_loss_mlp": 0.01040289, + "balance_loss_clip": 1.0390532, + "balance_loss_mlp": 1.02417183, + "epoch": 0.8332832321289004, + "flos": 17457398576640.0, + "grad_norm": 1.949434957938022, + "language_loss": 0.64955449, + "learning_rate": 2.844051333100905e-07, + "loss": 0.67121011, + "num_input_tokens_seen": 149724230, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.16137695, + "step": 6930, + "time_per_iteration": 2.804474115371704 + }, + { + "auxiliary_loss_clip": 0.01122478, + "auxiliary_loss_mlp": 0.01029471, + "balance_loss_clip": 1.04175246, + "balance_loss_mlp": 1.0158515, + "epoch": 0.8334034750195395, + "flos": 15084852416640.0, + "grad_norm": 1.6978992316956536, + "language_loss": 0.83784866, + "learning_rate": 2.840048814447269e-07, + "loss": 0.85936815, + "num_input_tokens_seen": 149742395, + "router_z_loss_clip": 0.80761719, + "router_z_loss_mlp": 0.1361084, + "step": 6931, + "time_per_iteration": 2.8396685123443604 + }, + { + "auxiliary_loss_clip": 0.01128735, + "auxiliary_loss_mlp": 0.01043742, + "balance_loss_clip": 1.04133606, + "balance_loss_mlp": 1.02825665, + "epoch": 0.8335237179101785, + "flos": 19427170556160.0, + "grad_norm": 2.389604562485655, + "language_loss": 0.74066389, + "learning_rate": 2.836048898944587e-07, + "loss": 0.76238871, + "num_input_tokens_seen": 149760820, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15472412, + "step": 6932, + "time_per_iteration": 2.8135883808135986 + }, + { + "auxiliary_loss_clip": 0.01121413, + "auxiliary_loss_mlp": 0.01041022, + "balance_loss_clip": 1.04017234, + "balance_loss_mlp": 1.02671087, + "epoch": 0.8336439608008177, + "flos": 21762046327680.0, + "grad_norm": 3.849423910346169, + "language_loss": 0.72811472, + "learning_rate": 2.832051587199642e-07, + "loss": 0.74973911, + "num_input_tokens_seen": 149778075, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.14312744, + "step": 6933, + "time_per_iteration": 2.802363395690918 + }, + { + "auxiliary_loss_clip": 0.01032877, + "auxiliary_loss_mlp": 0.01001316, + "balance_loss_clip": 1.00493717, + "balance_loss_mlp": 0.999951, + "epoch": 0.8337642036914568, + "flos": 59702783990400.0, + "grad_norm": 0.801898241772991, + "language_loss": 0.57637721, + "learning_rate": 2.828056879818821e-07, + "loss": 0.59671915, + "num_input_tokens_seen": 149837150, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01367188, + "step": 6934, + "time_per_iteration": 3.1874337196350098 + }, + { + "auxiliary_loss_clip": 0.01120188, + "auxiliary_loss_mlp": 0.01036934, + "balance_loss_clip": 1.03804588, + "balance_loss_mlp": 1.02203321, + "epoch": 0.8338844465820958, + "flos": 27162185022720.0, + "grad_norm": 1.8362347332184352, + "language_loss": 0.83405638, + "learning_rate": 2.824064777408117e-07, + "loss": 0.85562754, + "num_input_tokens_seen": 149856940, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.14904785, + "step": 6935, + "time_per_iteration": 2.7857396602630615 + }, + { + "auxiliary_loss_clip": 0.0112776, + "auxiliary_loss_mlp": 0.01047765, + "balance_loss_clip": 1.03967237, + "balance_loss_mlp": 1.03181481, + "epoch": 0.8340046894727349, + "flos": 30481264425600.0, + "grad_norm": 1.7828616670340727, + "language_loss": 0.7586863, + "learning_rate": 2.8200752805731263e-07, + "loss": 0.78044152, + "num_input_tokens_seen": 149879930, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.15942383, + "step": 6936, + "time_per_iteration": 2.7640740871429443 + }, + { + "auxiliary_loss_clip": 0.01129201, + "auxiliary_loss_mlp": 0.01035931, + "balance_loss_clip": 1.04194391, + "balance_loss_mlp": 1.0203743, + "epoch": 0.834124932363374, + "flos": 27126166659840.0, + "grad_norm": 1.466649522417788, + "language_loss": 0.80935037, + "learning_rate": 2.8160883899190625e-07, + "loss": 0.8310017, + "num_input_tokens_seen": 149903200, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15563965, + "step": 6937, + "time_per_iteration": 2.926858425140381 + }, + { + "auxiliary_loss_clip": 0.01129492, + "auxiliary_loss_mlp": 0.01039146, + "balance_loss_clip": 1.04176688, + "balance_loss_mlp": 1.02400637, + "epoch": 0.8342451752540131, + "flos": 24569865498240.0, + "grad_norm": 3.0175615861750065, + "language_loss": 0.73233819, + "learning_rate": 2.8121041060507234e-07, + "loss": 0.75402457, + "num_input_tokens_seen": 149922230, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15161133, + "step": 6938, + "time_per_iteration": 2.8750572204589844 + }, + { + "auxiliary_loss_clip": 0.01128201, + "auxiliary_loss_mlp": 0.01037551, + "balance_loss_clip": 1.04091311, + "balance_loss_mlp": 1.02175546, + "epoch": 0.8343654181446521, + "flos": 26615085995520.0, + "grad_norm": 1.7510163319029846, + "language_loss": 0.71556473, + "learning_rate": 2.808122429572528e-07, + "loss": 0.73722219, + "num_input_tokens_seen": 149942435, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15783691, + "step": 6939, + "time_per_iteration": 2.804210662841797 + }, + { + "auxiliary_loss_clip": 0.0112759, + "auxiliary_loss_mlp": 0.01034515, + "balance_loss_clip": 1.03958058, + "balance_loss_mlp": 1.01771808, + "epoch": 0.8344856610352913, + "flos": 20777268078720.0, + "grad_norm": 2.5456698245569163, + "language_loss": 0.75992709, + "learning_rate": 2.804143361088489e-07, + "loss": 0.78154814, + "num_input_tokens_seen": 149961615, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.16809082, + "step": 6940, + "time_per_iteration": 2.782240390777588 + }, + { + "auxiliary_loss_clip": 0.01126465, + "auxiliary_loss_mlp": 0.01036251, + "balance_loss_clip": 1.0401901, + "balance_loss_mlp": 1.02207744, + "epoch": 0.8346059039259304, + "flos": 26095960684800.0, + "grad_norm": 2.3361347314383325, + "language_loss": 0.77779728, + "learning_rate": 2.8001669012022277e-07, + "loss": 0.79942441, + "num_input_tokens_seen": 149979585, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.14172363, + "step": 6941, + "time_per_iteration": 2.7640910148620605 + }, + { + "auxiliary_loss_clip": 0.01128257, + "auxiliary_loss_mlp": 0.01039852, + "balance_loss_clip": 1.04260826, + "balance_loss_mlp": 1.0241878, + "epoch": 0.8347261468165694, + "flos": 29027708755200.0, + "grad_norm": 1.534504898681868, + "language_loss": 0.69363123, + "learning_rate": 2.7961930505169795e-07, + "loss": 0.7153123, + "num_input_tokens_seen": 150003830, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15661621, + "step": 6942, + "time_per_iteration": 2.8762643337249756 + }, + { + "auxiliary_loss_clip": 0.01126461, + "auxiliary_loss_mlp": 0.01043568, + "balance_loss_clip": 1.04002774, + "balance_loss_mlp": 1.02872086, + "epoch": 0.8348463897072086, + "flos": 26396461866240.0, + "grad_norm": 1.7997979245735156, + "language_loss": 0.76331925, + "learning_rate": 2.792221809635558e-07, + "loss": 0.78501952, + "num_input_tokens_seen": 150024460, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.14837646, + "step": 6943, + "time_per_iteration": 3.689427137374878 + }, + { + "auxiliary_loss_clip": 0.01129328, + "auxiliary_loss_mlp": 0.0104291, + "balance_loss_clip": 1.04156232, + "balance_loss_mlp": 1.02636409, + "epoch": 0.8349666325978476, + "flos": 23367720096000.0, + "grad_norm": 3.6574481289940888, + "language_loss": 0.74635243, + "learning_rate": 2.788253179160411e-07, + "loss": 0.76807481, + "num_input_tokens_seen": 150045620, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.1652832, + "step": 6944, + "time_per_iteration": 2.833066701889038 + }, + { + "auxiliary_loss_clip": 0.01122278, + "auxiliary_loss_mlp": 0.01040792, + "balance_loss_clip": 1.04063153, + "balance_loss_mlp": 1.02627265, + "epoch": 0.8350868754884867, + "flos": 12896528135040.0, + "grad_norm": 2.5097772423729263, + "language_loss": 0.65376002, + "learning_rate": 2.7842871596935725e-07, + "loss": 0.67539072, + "num_input_tokens_seen": 150064135, + "router_z_loss_clip": 0.81640625, + "router_z_loss_mlp": 0.1451416, + "step": 6945, + "time_per_iteration": 2.8302061557769775 + }, + { + "auxiliary_loss_clip": 0.01124519, + "auxiliary_loss_mlp": 0.01038031, + "balance_loss_clip": 1.03866529, + "balance_loss_mlp": 1.02303421, + "epoch": 0.8352071183791259, + "flos": 26505522535680.0, + "grad_norm": 1.563642598714707, + "language_loss": 0.69074202, + "learning_rate": 2.780323751836682e-07, + "loss": 0.71236753, + "num_input_tokens_seen": 150085350, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15002441, + "step": 6946, + "time_per_iteration": 2.8203999996185303 + }, + { + "auxiliary_loss_clip": 0.01124288, + "auxiliary_loss_mlp": 0.010399, + "balance_loss_clip": 1.04065955, + "balance_loss_mlp": 1.02411056, + "epoch": 0.8353273612697649, + "flos": 20668063754880.0, + "grad_norm": 1.4458676302988487, + "language_loss": 0.78585541, + "learning_rate": 2.7763629561909876e-07, + "loss": 0.80749726, + "num_input_tokens_seen": 150106180, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.15789795, + "step": 6947, + "time_per_iteration": 2.8702521324157715 + }, + { + "auxiliary_loss_clip": 0.01122159, + "auxiliary_loss_mlp": 0.01037751, + "balance_loss_clip": 1.0380621, + "balance_loss_mlp": 1.02200913, + "epoch": 0.835447604160404, + "flos": 19754137082880.0, + "grad_norm": 4.796714446791629, + "language_loss": 0.76934195, + "learning_rate": 2.772404773357335e-07, + "loss": 0.790941, + "num_input_tokens_seen": 150125585, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.15734863, + "step": 6948, + "time_per_iteration": 3.546511650085449 + }, + { + "auxiliary_loss_clip": 0.0112855, + "auxiliary_loss_mlp": 0.01041613, + "balance_loss_clip": 1.04087162, + "balance_loss_mlp": 1.02612734, + "epoch": 0.8355678470510431, + "flos": 23435842239360.0, + "grad_norm": 1.7861911219908158, + "language_loss": 0.7822268, + "learning_rate": 2.7684492039361853e-07, + "loss": 0.80392843, + "num_input_tokens_seen": 150144810, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.15490723, + "step": 6949, + "time_per_iteration": 3.5483431816101074 + }, + { + "auxiliary_loss_clip": 0.01128331, + "auxiliary_loss_mlp": 0.01042037, + "balance_loss_clip": 1.04263067, + "balance_loss_mlp": 1.02714753, + "epoch": 0.8356880899416822, + "flos": 21214588164480.0, + "grad_norm": 1.7297313656679694, + "language_loss": 0.83717865, + "learning_rate": 2.764496248527586e-07, + "loss": 0.85888231, + "num_input_tokens_seen": 150163785, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.14916992, + "step": 6950, + "time_per_iteration": 2.7475690841674805 + }, + { + "auxiliary_loss_clip": 0.01125252, + "auxiliary_loss_mlp": 0.01039342, + "balance_loss_clip": 1.0392797, + "balance_loss_mlp": 1.02436924, + "epoch": 0.8358083328323213, + "flos": 28037543466240.0, + "grad_norm": 1.9131189416497658, + "language_loss": 0.78571868, + "learning_rate": 2.760545907731211e-07, + "loss": 0.80736458, + "num_input_tokens_seen": 150184360, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.1496582, + "step": 6951, + "time_per_iteration": 2.7584152221679688 + }, + { + "auxiliary_loss_clip": 0.01126818, + "auxiliary_loss_mlp": 0.01044631, + "balance_loss_clip": 1.04008818, + "balance_loss_mlp": 1.02801299, + "epoch": 0.8359285757229604, + "flos": 27783655159680.0, + "grad_norm": 1.761547126725323, + "language_loss": 0.67820019, + "learning_rate": 2.75659818214631e-07, + "loss": 0.69991469, + "num_input_tokens_seen": 150205465, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.16601562, + "step": 6952, + "time_per_iteration": 2.797828435897827 + }, + { + "auxiliary_loss_clip": 0.01124287, + "auxiliary_loss_mlp": 0.01034457, + "balance_loss_clip": 1.04048347, + "balance_loss_mlp": 1.02039611, + "epoch": 0.8360488186135995, + "flos": 21435115714560.0, + "grad_norm": 1.796841452098034, + "language_loss": 0.78188956, + "learning_rate": 2.752653072371749e-07, + "loss": 0.80347699, + "num_input_tokens_seen": 150224900, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.140625, + "step": 6953, + "time_per_iteration": 2.716309070587158 + }, + { + "auxiliary_loss_clip": 0.01123763, + "auxiliary_loss_mlp": 0.01037759, + "balance_loss_clip": 1.04122472, + "balance_loss_mlp": 1.02372253, + "epoch": 0.8361690615042385, + "flos": 27632327160960.0, + "grad_norm": 1.7903390096980638, + "language_loss": 0.74732149, + "learning_rate": 2.7487105790060105e-07, + "loss": 0.76893675, + "num_input_tokens_seen": 150244310, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.14044189, + "step": 6954, + "time_per_iteration": 3.6413159370422363 + }, + { + "auxiliary_loss_clip": 0.01124274, + "auxiliary_loss_mlp": 0.01039948, + "balance_loss_clip": 1.04003072, + "balance_loss_mlp": 1.02562499, + "epoch": 0.8362893043948777, + "flos": 39202529598720.0, + "grad_norm": 1.8106297466063226, + "language_loss": 0.69353461, + "learning_rate": 2.7447707026471587e-07, + "loss": 0.71517682, + "num_input_tokens_seen": 150267285, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14337158, + "step": 6955, + "time_per_iteration": 2.938624858856201 + }, + { + "auxiliary_loss_clip": 0.01122942, + "auxiliary_loss_mlp": 0.01039331, + "balance_loss_clip": 1.03928471, + "balance_loss_mlp": 1.02472794, + "epoch": 0.8364095472855168, + "flos": 24785329230720.0, + "grad_norm": 2.139705870845271, + "language_loss": 0.79687244, + "learning_rate": 2.740833443892874e-07, + "loss": 0.81849515, + "num_input_tokens_seen": 150285455, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14611816, + "step": 6956, + "time_per_iteration": 2.800265312194824 + }, + { + "auxiliary_loss_clip": 0.01122165, + "auxiliary_loss_mlp": 0.01040901, + "balance_loss_clip": 1.03819537, + "balance_loss_mlp": 1.02647066, + "epoch": 0.8365297901761558, + "flos": 22743412784640.0, + "grad_norm": 1.7520155829818798, + "language_loss": 0.79756355, + "learning_rate": 2.7368988033404327e-07, + "loss": 0.8191942, + "num_input_tokens_seen": 150302970, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.14434814, + "step": 6957, + "time_per_iteration": 2.8511908054351807 + }, + { + "auxiliary_loss_clip": 0.01122761, + "auxiliary_loss_mlp": 0.01036411, + "balance_loss_clip": 1.03964591, + "balance_loss_mlp": 1.02133107, + "epoch": 0.836650033066795, + "flos": 28396003242240.0, + "grad_norm": 1.6079311745669145, + "language_loss": 0.84623897, + "learning_rate": 2.732966781586712e-07, + "loss": 0.86783069, + "num_input_tokens_seen": 150322715, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.15075684, + "step": 6958, + "time_per_iteration": 2.869025945663452 + }, + { + "auxiliary_loss_clip": 0.01124021, + "auxiliary_loss_mlp": 0.01039737, + "balance_loss_clip": 1.03942621, + "balance_loss_mlp": 1.02373362, + "epoch": 0.836770275957434, + "flos": 22236857233920.0, + "grad_norm": 1.6039791533482017, + "language_loss": 0.66812456, + "learning_rate": 2.729037379228205e-07, + "loss": 0.68976212, + "num_input_tokens_seen": 150342900, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.16009521, + "step": 6959, + "time_per_iteration": 2.8442676067352295 + }, + { + "auxiliary_loss_clip": 0.01128864, + "auxiliary_loss_mlp": 0.01040236, + "balance_loss_clip": 1.04175639, + "balance_loss_mlp": 1.02540612, + "epoch": 0.8368905188480731, + "flos": 22491930689280.0, + "grad_norm": 3.1486192866416403, + "language_loss": 0.80211353, + "learning_rate": 2.725110596860998e-07, + "loss": 0.8238045, + "num_input_tokens_seen": 150363580, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.14831543, + "step": 6960, + "time_per_iteration": 2.827514886856079 + }, + { + "auxiliary_loss_clip": 0.01124177, + "auxiliary_loss_mlp": 0.01034535, + "balance_loss_clip": 1.04151368, + "balance_loss_mlp": 1.01916933, + "epoch": 0.8370107617387123, + "flos": 13370405287680.0, + "grad_norm": 1.7821287684750873, + "language_loss": 0.70169318, + "learning_rate": 2.7211864350807776e-07, + "loss": 0.72328031, + "num_input_tokens_seen": 150381780, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.15368652, + "step": 6961, + "time_per_iteration": 2.7494378089904785 + }, + { + "auxiliary_loss_clip": 0.01124803, + "auxiliary_loss_mlp": 0.01045299, + "balance_loss_clip": 1.03956139, + "balance_loss_mlp": 1.02973676, + "epoch": 0.8371310046293513, + "flos": 25261289372160.0, + "grad_norm": 1.6189763720486703, + "language_loss": 0.73623073, + "learning_rate": 2.717264894482836e-07, + "loss": 0.75793177, + "num_input_tokens_seen": 150402120, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.15557861, + "step": 6962, + "time_per_iteration": 2.846436023712158 + }, + { + "auxiliary_loss_clip": 0.01130454, + "auxiliary_loss_mlp": 0.01039595, + "balance_loss_clip": 1.04186296, + "balance_loss_mlp": 1.02407408, + "epoch": 0.8372512475199904, + "flos": 19792705311360.0, + "grad_norm": 1.848939090132968, + "language_loss": 0.81018704, + "learning_rate": 2.7133459756620646e-07, + "loss": 0.83188754, + "num_input_tokens_seen": 150419315, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.15509033, + "step": 6963, + "time_per_iteration": 2.7746424674987793 + }, + { + "auxiliary_loss_clip": 0.01128291, + "auxiliary_loss_mlp": 0.01037274, + "balance_loss_clip": 1.04236841, + "balance_loss_mlp": 1.02140701, + "epoch": 0.8373714904106295, + "flos": 19391224020480.0, + "grad_norm": 1.5913111108881979, + "language_loss": 0.73642325, + "learning_rate": 2.7094296792129733e-07, + "loss": 0.75807893, + "num_input_tokens_seen": 150438915, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.15869141, + "step": 6964, + "time_per_iteration": 2.7959089279174805 + }, + { + "auxiliary_loss_clip": 0.01122173, + "auxiliary_loss_mlp": 0.01035674, + "balance_loss_clip": 1.03826666, + "balance_loss_mlp": 1.02111244, + "epoch": 0.8374917333012686, + "flos": 14975935401600.0, + "grad_norm": 1.7855885813547816, + "language_loss": 0.75633478, + "learning_rate": 2.7055160057296424e-07, + "loss": 0.77791321, + "num_input_tokens_seen": 150456155, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14575195, + "step": 6965, + "time_per_iteration": 2.8365097045898438 + }, + { + "auxiliary_loss_clip": 0.0112592, + "auxiliary_loss_mlp": 0.01047455, + "balance_loss_clip": 1.03973365, + "balance_loss_mlp": 1.03096819, + "epoch": 0.8376119761919076, + "flos": 30331839847680.0, + "grad_norm": 1.589398100633404, + "language_loss": 0.72381312, + "learning_rate": 2.7016049558057896e-07, + "loss": 0.74554688, + "num_input_tokens_seen": 150478115, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.16491699, + "step": 6966, + "time_per_iteration": 2.8616793155670166 + }, + { + "auxiliary_loss_clip": 0.01122105, + "auxiliary_loss_mlp": 0.01042217, + "balance_loss_clip": 1.03848052, + "balance_loss_mlp": 1.02666616, + "epoch": 0.8377322190825467, + "flos": 29423336129280.0, + "grad_norm": 1.7699067023869717, + "language_loss": 0.70881593, + "learning_rate": 2.6976965300347074e-07, + "loss": 0.73045921, + "num_input_tokens_seen": 150500725, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.15557861, + "step": 6967, + "time_per_iteration": 2.8805878162384033 + }, + { + "auxiliary_loss_clip": 0.01121566, + "auxiliary_loss_mlp": 0.01042022, + "balance_loss_clip": 1.03831601, + "balance_loss_mlp": 1.02639377, + "epoch": 0.8378524619731859, + "flos": 26687086807680.0, + "grad_norm": 2.7969932101667956, + "language_loss": 0.69363022, + "learning_rate": 2.693790729009309e-07, + "loss": 0.71526611, + "num_input_tokens_seen": 150522335, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.15637207, + "step": 6968, + "time_per_iteration": 2.8829233646392822 + }, + { + "auxiliary_loss_clip": 0.0112061, + "auxiliary_loss_mlp": 0.01037147, + "balance_loss_clip": 1.03774655, + "balance_loss_mlp": 1.02190566, + "epoch": 0.8379727048638249, + "flos": 20703866636160.0, + "grad_norm": 1.9716852276316643, + "language_loss": 0.88821578, + "learning_rate": 2.6898875533220946e-07, + "loss": 0.90979326, + "num_input_tokens_seen": 150541640, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.15240479, + "step": 6969, + "time_per_iteration": 2.797226905822754 + }, + { + "auxiliary_loss_clip": 0.01118963, + "auxiliary_loss_mlp": 0.01030154, + "balance_loss_clip": 1.03890371, + "balance_loss_mlp": 1.01642656, + "epoch": 0.838092947754464, + "flos": 20084084438400.0, + "grad_norm": 1.7397737893761118, + "language_loss": 0.81832689, + "learning_rate": 2.685987003565171e-07, + "loss": 0.83981812, + "num_input_tokens_seen": 150559680, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.137146, + "step": 6970, + "time_per_iteration": 2.749432325363159 + }, + { + "auxiliary_loss_clip": 0.01123767, + "auxiliary_loss_mlp": 0.01035954, + "balance_loss_clip": 1.03898907, + "balance_loss_mlp": 1.02011108, + "epoch": 0.8382131906451031, + "flos": 18113270964480.0, + "grad_norm": 2.3480825225009947, + "language_loss": 0.75381106, + "learning_rate": 2.6820890803302566e-07, + "loss": 0.77540827, + "num_input_tokens_seen": 150575205, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15844727, + "step": 6971, + "time_per_iteration": 2.779254913330078 + }, + { + "auxiliary_loss_clip": 0.01123059, + "auxiliary_loss_mlp": 0.01040588, + "balance_loss_clip": 1.04077911, + "balance_loss_mlp": 1.02541256, + "epoch": 0.8383334335357422, + "flos": 17092653920640.0, + "grad_norm": 2.290689977997778, + "language_loss": 0.8193239, + "learning_rate": 2.6781937842086557e-07, + "loss": 0.84096038, + "num_input_tokens_seen": 150593995, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.1517334, + "step": 6972, + "time_per_iteration": 3.565075397491455 + }, + { + "auxiliary_loss_clip": 0.011253, + "auxiliary_loss_mlp": 0.01041378, + "balance_loss_clip": 1.03975058, + "balance_loss_mlp": 1.02619028, + "epoch": 0.8384536764263812, + "flos": 20704728562560.0, + "grad_norm": 1.69734591804179, + "language_loss": 0.67471635, + "learning_rate": 2.6743011157912933e-07, + "loss": 0.69638312, + "num_input_tokens_seen": 150613715, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15185547, + "step": 6973, + "time_per_iteration": 2.745962381362915 + }, + { + "auxiliary_loss_clip": 0.01125765, + "auxiliary_loss_mlp": 0.01040071, + "balance_loss_clip": 1.04050362, + "balance_loss_mlp": 1.02553916, + "epoch": 0.8385739193170204, + "flos": 28986842056320.0, + "grad_norm": 1.773588786127424, + "language_loss": 0.65302563, + "learning_rate": 2.6704110756686725e-07, + "loss": 0.67468405, + "num_input_tokens_seen": 150634540, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.14526367, + "step": 6974, + "time_per_iteration": 2.8535287380218506 + }, + { + "auxiliary_loss_clip": 0.01126707, + "auxiliary_loss_mlp": 0.01036614, + "balance_loss_clip": 1.03949702, + "balance_loss_mlp": 1.02105737, + "epoch": 0.8386941622076595, + "flos": 23438068882560.0, + "grad_norm": 2.4563693437393614, + "language_loss": 0.83938122, + "learning_rate": 2.6665236644309085e-07, + "loss": 0.86101443, + "num_input_tokens_seen": 150654850, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.15551758, + "step": 6975, + "time_per_iteration": 2.7665014266967773 + }, + { + "auxiliary_loss_clip": 0.0112151, + "auxiliary_loss_mlp": 0.01036057, + "balance_loss_clip": 1.03933644, + "balance_loss_mlp": 1.02177596, + "epoch": 0.8388144050982985, + "flos": 23002724044800.0, + "grad_norm": 1.862577841515683, + "language_loss": 0.79697084, + "learning_rate": 2.662638882667727e-07, + "loss": 0.81854653, + "num_input_tokens_seen": 150673790, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.14282227, + "step": 6976, + "time_per_iteration": 2.8265504837036133 + }, + { + "auxiliary_loss_clip": 0.01129204, + "auxiliary_loss_mlp": 0.0103489, + "balance_loss_clip": 1.04123986, + "balance_loss_mlp": 1.01955962, + "epoch": 0.8389346479889377, + "flos": 24280353878400.0, + "grad_norm": 1.7944969994198687, + "language_loss": 0.73323274, + "learning_rate": 2.658756730968443e-07, + "loss": 0.75487369, + "num_input_tokens_seen": 150692255, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.15332031, + "step": 6977, + "time_per_iteration": 3.498528003692627 + }, + { + "auxiliary_loss_clip": 0.01127319, + "auxiliary_loss_mlp": 0.01034282, + "balance_loss_clip": 1.0416429, + "balance_loss_mlp": 1.01916647, + "epoch": 0.8390548908795767, + "flos": 21215019127680.0, + "grad_norm": 2.8605590063549844, + "language_loss": 0.88488185, + "learning_rate": 2.654877209921975e-07, + "loss": 0.90649796, + "num_input_tokens_seen": 150709790, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15130615, + "step": 6978, + "time_per_iteration": 3.629225015640259 + }, + { + "auxiliary_loss_clip": 0.0113508, + "auxiliary_loss_mlp": 0.01039459, + "balance_loss_clip": 1.04146016, + "balance_loss_mlp": 1.02325833, + "epoch": 0.8391751337702158, + "flos": 35627299332480.0, + "grad_norm": 2.499413806289433, + "language_loss": 0.62887621, + "learning_rate": 2.651000320116843e-07, + "loss": 0.65062153, + "num_input_tokens_seen": 150730675, + "router_z_loss_clip": 0.93554688, + "router_z_loss_mlp": 0.1619873, + "step": 6979, + "time_per_iteration": 2.906787633895874 + }, + { + "auxiliary_loss_clip": 0.01127529, + "auxiliary_loss_mlp": 0.01036363, + "balance_loss_clip": 1.04135573, + "balance_loss_mlp": 1.02041245, + "epoch": 0.839295376660855, + "flos": 21325229032320.0, + "grad_norm": 1.8681291277180507, + "language_loss": 0.76234621, + "learning_rate": 2.647126062141163e-07, + "loss": 0.78398514, + "num_input_tokens_seen": 150749750, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.1595459, + "step": 6980, + "time_per_iteration": 2.748425006866455 + }, + { + "auxiliary_loss_clip": 0.01123493, + "auxiliary_loss_mlp": 0.01041861, + "balance_loss_clip": 1.03873205, + "balance_loss_mlp": 1.02541006, + "epoch": 0.839415619551494, + "flos": 18442535961600.0, + "grad_norm": 1.8040218492122755, + "language_loss": 0.83978957, + "learning_rate": 2.643254436582669e-07, + "loss": 0.8614431, + "num_input_tokens_seen": 150769240, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.16442871, + "step": 6981, + "time_per_iteration": 2.7623562812805176 + }, + { + "auxiliary_loss_clip": 0.01128206, + "auxiliary_loss_mlp": 0.01036083, + "balance_loss_clip": 1.04156518, + "balance_loss_mlp": 1.02192044, + "epoch": 0.8395358624421331, + "flos": 23221958705280.0, + "grad_norm": 1.8170256593195953, + "language_loss": 0.82358646, + "learning_rate": 2.6393854440286743e-07, + "loss": 0.84522939, + "num_input_tokens_seen": 150788410, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.14160156, + "step": 6982, + "time_per_iteration": 2.8279926776885986 + }, + { + "auxiliary_loss_clip": 0.01121083, + "auxiliary_loss_mlp": 0.01042734, + "balance_loss_clip": 1.0388546, + "balance_loss_mlp": 1.02683163, + "epoch": 0.8396561053327722, + "flos": 24381657210240.0, + "grad_norm": 1.9391848918406083, + "language_loss": 0.70888871, + "learning_rate": 2.6355190850661045e-07, + "loss": 0.73052686, + "num_input_tokens_seen": 150805245, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.15905762, + "step": 6983, + "time_per_iteration": 3.6479902267456055 + }, + { + "auxiliary_loss_clip": 0.01123211, + "auxiliary_loss_mlp": 0.01036196, + "balance_loss_clip": 1.04052162, + "balance_loss_mlp": 1.02074647, + "epoch": 0.8397763482234113, + "flos": 22237755073920.0, + "grad_norm": 2.518036260617623, + "language_loss": 0.86699724, + "learning_rate": 2.631655360281486e-07, + "loss": 0.88859129, + "num_input_tokens_seen": 150824920, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.15441895, + "step": 6984, + "time_per_iteration": 2.7982420921325684 + }, + { + "auxiliary_loss_clip": 0.01127065, + "auxiliary_loss_mlp": 0.01039129, + "balance_loss_clip": 1.04000831, + "balance_loss_mlp": 1.02267861, + "epoch": 0.8398965911140504, + "flos": 22163743100160.0, + "grad_norm": 1.800798213450512, + "language_loss": 0.65668094, + "learning_rate": 2.6277942702609323e-07, + "loss": 0.67834294, + "num_input_tokens_seen": 150844400, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.16467285, + "step": 6985, + "time_per_iteration": 2.7628395557403564 + }, + { + "auxiliary_loss_clip": 0.01129128, + "auxiliary_loss_mlp": 0.01040263, + "balance_loss_clip": 1.04258418, + "balance_loss_mlp": 1.02581429, + "epoch": 0.8400168340046895, + "flos": 21542775753600.0, + "grad_norm": 1.8537560279131902, + "language_loss": 0.87552166, + "learning_rate": 2.623935815590186e-07, + "loss": 0.8972156, + "num_input_tokens_seen": 150862780, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.14465332, + "step": 6986, + "time_per_iteration": 2.824176073074341 + }, + { + "auxiliary_loss_clip": 0.01128251, + "auxiliary_loss_mlp": 0.01044391, + "balance_loss_clip": 1.04189301, + "balance_loss_mlp": 1.02833378, + "epoch": 0.8401370768953286, + "flos": 22491966602880.0, + "grad_norm": 2.9515405832446135, + "language_loss": 0.8092851, + "learning_rate": 2.6200799968545516e-07, + "loss": 0.83101153, + "num_input_tokens_seen": 150883075, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.16064453, + "step": 6987, + "time_per_iteration": 2.785176992416382 + }, + { + "auxiliary_loss_clip": 0.01032551, + "auxiliary_loss_mlp": 0.0100217, + "balance_loss_clip": 1.00452662, + "balance_loss_mlp": 1.00082898, + "epoch": 0.8402573197859676, + "flos": 59238890818560.0, + "grad_norm": 0.786253972166531, + "language_loss": 0.56427479, + "learning_rate": 2.616226814638969e-07, + "loss": 0.58462191, + "num_input_tokens_seen": 150948180, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01342773, + "step": 6988, + "time_per_iteration": 3.318763256072998 + }, + { + "auxiliary_loss_clip": 0.01124783, + "auxiliary_loss_mlp": 0.01041292, + "balance_loss_clip": 1.03829122, + "balance_loss_mlp": 1.02654588, + "epoch": 0.8403775626766068, + "flos": 22674608282880.0, + "grad_norm": 2.0746121348559274, + "language_loss": 0.77310014, + "learning_rate": 2.612376269527954e-07, + "loss": 0.79476082, + "num_input_tokens_seen": 150967885, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.14733887, + "step": 6989, + "time_per_iteration": 2.8220508098602295 + }, + { + "auxiliary_loss_clip": 0.01121825, + "auxiliary_loss_mlp": 0.01036139, + "balance_loss_clip": 1.03927302, + "balance_loss_mlp": 1.02100515, + "epoch": 0.8404978055672458, + "flos": 19609704495360.0, + "grad_norm": 2.000330729208879, + "language_loss": 0.68019491, + "learning_rate": 2.608528362105635e-07, + "loss": 0.7017746, + "num_input_tokens_seen": 150987255, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.15118408, + "step": 6990, + "time_per_iteration": 2.7169690132141113 + }, + { + "auxiliary_loss_clip": 0.01123123, + "auxiliary_loss_mlp": 0.01038286, + "balance_loss_clip": 1.03873432, + "balance_loss_mlp": 1.02436256, + "epoch": 0.8406180484578849, + "flos": 27526929678720.0, + "grad_norm": 3.1924794417693647, + "language_loss": 0.73207748, + "learning_rate": 2.6046830929557374e-07, + "loss": 0.75369155, + "num_input_tokens_seen": 151006905, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.13934326, + "step": 6991, + "time_per_iteration": 2.789621114730835 + }, + { + "auxiliary_loss_clip": 0.011285, + "auxiliary_loss_mlp": 0.01038543, + "balance_loss_clip": 1.04215527, + "balance_loss_mlp": 1.02363598, + "epoch": 0.8407382913485241, + "flos": 22127473342080.0, + "grad_norm": 1.8237186582606668, + "language_loss": 0.8501246, + "learning_rate": 2.6008404626615776e-07, + "loss": 0.87179506, + "num_input_tokens_seen": 151025405, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.14904785, + "step": 6992, + "time_per_iteration": 2.807105302810669 + }, + { + "auxiliary_loss_clip": 0.01126224, + "auxiliary_loss_mlp": 0.01043278, + "balance_loss_clip": 1.04039645, + "balance_loss_mlp": 1.02738714, + "epoch": 0.8408585342391631, + "flos": 13918473982080.0, + "grad_norm": 2.642722673128113, + "language_loss": 0.73379117, + "learning_rate": 2.597000471806092e-07, + "loss": 0.75548613, + "num_input_tokens_seen": 151041970, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15893555, + "step": 6993, + "time_per_iteration": 2.833498001098633 + }, + { + "auxiliary_loss_clip": 0.01129295, + "auxiliary_loss_mlp": 0.01044914, + "balance_loss_clip": 1.04274249, + "balance_loss_mlp": 1.02866578, + "epoch": 0.8409787771298022, + "flos": 20187865808640.0, + "grad_norm": 2.0235569333116525, + "language_loss": 0.73183864, + "learning_rate": 2.593163120971793e-07, + "loss": 0.75358075, + "num_input_tokens_seen": 151060835, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.16247559, + "step": 6994, + "time_per_iteration": 2.7220747470855713 + }, + { + "auxiliary_loss_clip": 0.01125356, + "auxiliary_loss_mlp": 0.0103525, + "balance_loss_clip": 1.03991866, + "balance_loss_mlp": 1.02032495, + "epoch": 0.8410990200204413, + "flos": 23142523777920.0, + "grad_norm": 2.0455763745731907, + "language_loss": 0.68747514, + "learning_rate": 2.5893284107408165e-07, + "loss": 0.70908123, + "num_input_tokens_seen": 151078205, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.14935303, + "step": 6995, + "time_per_iteration": 2.746204137802124 + }, + { + "auxiliary_loss_clip": 0.01128293, + "auxiliary_loss_mlp": 0.01044452, + "balance_loss_clip": 1.04150629, + "balance_loss_mlp": 1.02959836, + "epoch": 0.8412192629110804, + "flos": 24027219757440.0, + "grad_norm": 1.7861490499822825, + "language_loss": 0.7777611, + "learning_rate": 2.5854963416948726e-07, + "loss": 0.79948848, + "num_input_tokens_seen": 151100470, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.1484375, + "step": 6996, + "time_per_iteration": 2.7815213203430176 + }, + { + "auxiliary_loss_clip": 0.01122542, + "auxiliary_loss_mlp": 0.01036051, + "balance_loss_clip": 1.03820014, + "balance_loss_mlp": 1.02062488, + "epoch": 0.8413395058017195, + "flos": 25591703604480.0, + "grad_norm": 1.5338249889905997, + "language_loss": 0.69394094, + "learning_rate": 2.5816669144152816e-07, + "loss": 0.71552688, + "num_input_tokens_seen": 151121650, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.1541748, + "step": 6997, + "time_per_iteration": 2.7594847679138184 + }, + { + "auxiliary_loss_clip": 0.01032083, + "auxiliary_loss_mlp": 0.01002113, + "balance_loss_clip": 1.00416911, + "balance_loss_mlp": 1.00086772, + "epoch": 0.8414597486923585, + "flos": 63635396624640.0, + "grad_norm": 0.8537106035498034, + "language_loss": 0.66306901, + "learning_rate": 2.5778401294829777e-07, + "loss": 0.683411, + "num_input_tokens_seen": 151180390, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01245117, + "step": 6998, + "time_per_iteration": 3.388211965560913 + }, + { + "auxiliary_loss_clip": 0.01122486, + "auxiliary_loss_mlp": 0.01038861, + "balance_loss_clip": 1.03881741, + "balance_loss_mlp": 1.02403784, + "epoch": 0.8415799915829977, + "flos": 19098731571840.0, + "grad_norm": 1.8538994322342313, + "language_loss": 0.65141273, + "learning_rate": 2.574015987478473e-07, + "loss": 0.6730262, + "num_input_tokens_seen": 151198520, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.14819336, + "step": 6999, + "time_per_iteration": 2.7497305870056152 + }, + { + "auxiliary_loss_clip": 0.01129895, + "auxiliary_loss_mlp": 0.01035981, + "balance_loss_clip": 1.04216957, + "balance_loss_mlp": 1.01821887, + "epoch": 0.8417002344736367, + "flos": 19821612781440.0, + "grad_norm": 17.595767681973886, + "language_loss": 0.87000984, + "learning_rate": 2.570194488981887e-07, + "loss": 0.89166856, + "num_input_tokens_seen": 151215065, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.1776123, + "step": 7000, + "time_per_iteration": 2.8441426753997803 + }, + { + "auxiliary_loss_clip": 0.01032757, + "auxiliary_loss_mlp": 0.01000458, + "balance_loss_clip": 1.00461626, + "balance_loss_mlp": 0.9991231, + "epoch": 0.8418204773642758, + "flos": 62161516834560.0, + "grad_norm": 0.8381826282900114, + "language_loss": 0.60336453, + "learning_rate": 2.566375634572939e-07, + "loss": 0.62369668, + "num_input_tokens_seen": 151275705, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.0133667, + "step": 7001, + "time_per_iteration": 3.8980908393859863 + }, + { + "auxiliary_loss_clip": 0.01125927, + "auxiliary_loss_mlp": 0.01036727, + "balance_loss_clip": 1.04011559, + "balance_loss_mlp": 1.02170086, + "epoch": 0.841940720254915, + "flos": 17092905315840.0, + "grad_norm": 1.7839698496189387, + "language_loss": 0.76301098, + "learning_rate": 2.562559424830943e-07, + "loss": 0.78463757, + "num_input_tokens_seen": 151293665, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15008545, + "step": 7002, + "time_per_iteration": 2.825901508331299 + }, + { + "auxiliary_loss_clip": 0.01125781, + "auxiliary_loss_mlp": 0.01034289, + "balance_loss_clip": 1.04178071, + "balance_loss_mlp": 1.01873779, + "epoch": 0.842060963145554, + "flos": 16283586026880.0, + "grad_norm": 2.1070867905896837, + "language_loss": 0.70216155, + "learning_rate": 2.5587458603348256e-07, + "loss": 0.72376221, + "num_input_tokens_seen": 151310955, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15545654, + "step": 7003, + "time_per_iteration": 2.855217218399048 + }, + { + "auxiliary_loss_clip": 0.01121209, + "auxiliary_loss_mlp": 0.01037556, + "balance_loss_clip": 1.03793252, + "balance_loss_mlp": 1.02061629, + "epoch": 0.8421812060361931, + "flos": 21908238681600.0, + "grad_norm": 2.5890011454787354, + "language_loss": 0.8425895, + "learning_rate": 2.554934941663085e-07, + "loss": 0.86417711, + "num_input_tokens_seen": 151328490, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.16931152, + "step": 7004, + "time_per_iteration": 2.912278413772583 + }, + { + "auxiliary_loss_clip": 0.01127014, + "auxiliary_loss_mlp": 0.01035408, + "balance_loss_clip": 1.03969479, + "balance_loss_mlp": 1.01912391, + "epoch": 0.8423014489268322, + "flos": 27777693502080.0, + "grad_norm": 1.9304303397279678, + "language_loss": 0.73757136, + "learning_rate": 2.5511266693938484e-07, + "loss": 0.75919563, + "num_input_tokens_seen": 151346950, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.16296387, + "step": 7005, + "time_per_iteration": 2.890125036239624 + }, + { + "auxiliary_loss_clip": 0.01128255, + "auxiliary_loss_mlp": 0.01039897, + "balance_loss_clip": 1.04255974, + "balance_loss_mlp": 1.02411401, + "epoch": 0.8424216918174713, + "flos": 25117610970240.0, + "grad_norm": 1.5179763153631627, + "language_loss": 0.77610397, + "learning_rate": 2.547321044104822e-07, + "loss": 0.79778552, + "num_input_tokens_seen": 151368445, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15795898, + "step": 7006, + "time_per_iteration": 3.651230573654175 + }, + { + "auxiliary_loss_clip": 0.01126817, + "auxiliary_loss_mlp": 0.01039816, + "balance_loss_clip": 1.04072976, + "balance_loss_mlp": 1.02423477, + "epoch": 0.8425419347081103, + "flos": 24748448941440.0, + "grad_norm": 1.6758440774432193, + "language_loss": 0.76822782, + "learning_rate": 2.5435180663733113e-07, + "loss": 0.7898941, + "num_input_tokens_seen": 151388745, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15576172, + "step": 7007, + "time_per_iteration": 3.5888671875 + }, + { + "auxiliary_loss_clip": 0.01125683, + "auxiliary_loss_mlp": 0.01039873, + "balance_loss_clip": 1.04014158, + "balance_loss_mlp": 1.0246613, + "epoch": 0.8426621775987495, + "flos": 24820916630400.0, + "grad_norm": 2.473309215263551, + "language_loss": 0.72060496, + "learning_rate": 2.539717736776241e-07, + "loss": 0.74226058, + "num_input_tokens_seen": 151404970, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15197754, + "step": 7008, + "time_per_iteration": 2.773298978805542 + }, + { + "auxiliary_loss_clip": 0.01125241, + "auxiliary_loss_mlp": 0.01039826, + "balance_loss_clip": 1.0401113, + "balance_loss_mlp": 1.02497244, + "epoch": 0.8427824204893886, + "flos": 23550074467200.0, + "grad_norm": 1.7453767376067173, + "language_loss": 0.76241219, + "learning_rate": 2.535920055890097e-07, + "loss": 0.78406286, + "num_input_tokens_seen": 151426265, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.14855957, + "step": 7009, + "time_per_iteration": 2.7658936977386475 + }, + { + "auxiliary_loss_clip": 0.01131712, + "auxiliary_loss_mlp": 0.01048946, + "balance_loss_clip": 1.04215419, + "balance_loss_mlp": 1.03294837, + "epoch": 0.8429026633800276, + "flos": 16143858120960.0, + "grad_norm": 2.577110956158816, + "language_loss": 0.64650106, + "learning_rate": 2.5321250242910006e-07, + "loss": 0.66830766, + "num_input_tokens_seen": 151444180, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.16003418, + "step": 7010, + "time_per_iteration": 2.7583916187286377 + }, + { + "auxiliary_loss_clip": 0.01122986, + "auxiliary_loss_mlp": 0.01036479, + "balance_loss_clip": 1.03963852, + "balance_loss_mlp": 1.02169132, + "epoch": 0.8430229062706668, + "flos": 22198540400640.0, + "grad_norm": 1.6771464456975684, + "language_loss": 0.86125004, + "learning_rate": 2.5283326425546493e-07, + "loss": 0.88284469, + "num_input_tokens_seen": 151463290, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.14788818, + "step": 7011, + "time_per_iteration": 2.750197410583496 + }, + { + "auxiliary_loss_clip": 0.01120868, + "auxiliary_loss_mlp": 0.01037979, + "balance_loss_clip": 1.04102314, + "balance_loss_mlp": 1.02375746, + "epoch": 0.8431431491613058, + "flos": 35330317683840.0, + "grad_norm": 1.7972919550906483, + "language_loss": 0.69120073, + "learning_rate": 2.5245429112563443e-07, + "loss": 0.71278924, + "num_input_tokens_seen": 151483965, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.14190674, + "step": 7012, + "time_per_iteration": 3.6760222911834717 + }, + { + "auxiliary_loss_clip": 0.01128312, + "auxiliary_loss_mlp": 0.01040043, + "balance_loss_clip": 1.0424149, + "balance_loss_mlp": 1.02554774, + "epoch": 0.8432633920519449, + "flos": 25812374808960.0, + "grad_norm": 1.8041568378362698, + "language_loss": 0.81887394, + "learning_rate": 2.5207558309709865e-07, + "loss": 0.84055752, + "num_input_tokens_seen": 151503700, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.1449585, + "step": 7013, + "time_per_iteration": 2.7574400901794434 + }, + { + "auxiliary_loss_clip": 0.01032191, + "auxiliary_loss_mlp": 0.01003907, + "balance_loss_clip": 1.00419044, + "balance_loss_mlp": 1.00256562, + "epoch": 0.8433836349425841, + "flos": 64959531592320.0, + "grad_norm": 0.657585008375777, + "language_loss": 0.56306577, + "learning_rate": 2.516971402273065e-07, + "loss": 0.58342671, + "num_input_tokens_seen": 151569765, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01342773, + "step": 7014, + "time_per_iteration": 3.2560691833496094 + }, + { + "auxiliary_loss_clip": 0.0112255, + "auxiliary_loss_mlp": 0.01035465, + "balance_loss_clip": 1.03782606, + "balance_loss_mlp": 1.01968193, + "epoch": 0.8435038778332231, + "flos": 20229989483520.0, + "grad_norm": 2.075093163235687, + "language_loss": 0.67731529, + "learning_rate": 2.513189625736687e-07, + "loss": 0.69889539, + "num_input_tokens_seen": 151586660, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15759277, + "step": 7015, + "time_per_iteration": 2.7253360748291016 + }, + { + "auxiliary_loss_clip": 0.01130022, + "auxiliary_loss_mlp": 0.01045795, + "balance_loss_clip": 1.04101133, + "balance_loss_mlp": 1.02881932, + "epoch": 0.8436241207238622, + "flos": 20992229020800.0, + "grad_norm": 2.5418162291266353, + "language_loss": 0.71409845, + "learning_rate": 2.509410501935534e-07, + "loss": 0.73585665, + "num_input_tokens_seen": 151602295, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.16967773, + "step": 7016, + "time_per_iteration": 2.8106322288513184 + }, + { + "auxiliary_loss_clip": 0.01130015, + "auxiliary_loss_mlp": 0.01041238, + "balance_loss_clip": 1.04314399, + "balance_loss_mlp": 1.02448881, + "epoch": 0.8437443636145013, + "flos": 14682257804160.0, + "grad_norm": 3.755042889139711, + "language_loss": 0.7508384, + "learning_rate": 2.5056340314429116e-07, + "loss": 0.77255082, + "num_input_tokens_seen": 151619760, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.1673584, + "step": 7017, + "time_per_iteration": 2.766282081604004 + }, + { + "auxiliary_loss_clip": 0.01131226, + "auxiliary_loss_mlp": 0.01042239, + "balance_loss_clip": 1.0418601, + "balance_loss_mlp": 1.02595472, + "epoch": 0.8438646065051404, + "flos": 21608814908160.0, + "grad_norm": 2.072646631585593, + "language_loss": 0.80017626, + "learning_rate": 2.5018602148316904e-07, + "loss": 0.82191086, + "num_input_tokens_seen": 151635795, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.1628418, + "step": 7018, + "time_per_iteration": 2.8440279960632324 + }, + { + "auxiliary_loss_clip": 0.01122801, + "auxiliary_loss_mlp": 0.0103752, + "balance_loss_clip": 1.04165626, + "balance_loss_mlp": 1.02263021, + "epoch": 0.8439848493957794, + "flos": 23289937194240.0, + "grad_norm": 1.9503148266207602, + "language_loss": 0.80173492, + "learning_rate": 2.498089052674359e-07, + "loss": 0.82333809, + "num_input_tokens_seen": 151653770, + "router_z_loss_clip": 0.81054688, + "router_z_loss_mlp": 0.14898682, + "step": 7019, + "time_per_iteration": 2.7847230434417725 + }, + { + "auxiliary_loss_clip": 0.01126704, + "auxiliary_loss_mlp": 0.0104275, + "balance_loss_clip": 1.04084241, + "balance_loss_mlp": 1.02700281, + "epoch": 0.8441050922864186, + "flos": 19719339782400.0, + "grad_norm": 2.0338415817210325, + "language_loss": 0.75365072, + "learning_rate": 2.494320545543007e-07, + "loss": 0.77534527, + "num_input_tokens_seen": 151673340, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15734863, + "step": 7020, + "time_per_iteration": 2.9322967529296875 + }, + { + "auxiliary_loss_clip": 0.01127836, + "auxiliary_loss_mlp": 0.01044808, + "balance_loss_clip": 1.03999829, + "balance_loss_mlp": 1.0284164, + "epoch": 0.8442253351770577, + "flos": 21835268202240.0, + "grad_norm": 1.67467745442316, + "language_loss": 0.66553485, + "learning_rate": 2.490554694009308e-07, + "loss": 0.68726128, + "num_input_tokens_seen": 151694205, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.16381836, + "step": 7021, + "time_per_iteration": 2.7754547595977783 + }, + { + "auxiliary_loss_clip": 0.01126373, + "auxiliary_loss_mlp": 0.01037972, + "balance_loss_clip": 1.04003596, + "balance_loss_mlp": 1.02259386, + "epoch": 0.8443455780676967, + "flos": 34346365447680.0, + "grad_norm": 1.5553245541743206, + "language_loss": 0.78511488, + "learning_rate": 2.4867914986445426e-07, + "loss": 0.80675828, + "num_input_tokens_seen": 151716595, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15380859, + "step": 7022, + "time_per_iteration": 2.8375933170318604 + }, + { + "auxiliary_loss_clip": 0.01123211, + "auxiliary_loss_mlp": 0.01033208, + "balance_loss_clip": 1.03895092, + "balance_loss_mlp": 1.01880717, + "epoch": 0.8444658209583359, + "flos": 48214599281280.0, + "grad_norm": 1.856994885756041, + "language_loss": 0.71410662, + "learning_rate": 2.483030960019581e-07, + "loss": 0.7356708, + "num_input_tokens_seen": 151740525, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14416504, + "step": 7023, + "time_per_iteration": 3.0168604850769043 + }, + { + "auxiliary_loss_clip": 0.01032172, + "auxiliary_loss_mlp": 0.01002363, + "balance_loss_clip": 1.00422549, + "balance_loss_mlp": 1.00110555, + "epoch": 0.8445860638489749, + "flos": 68484773105280.0, + "grad_norm": 0.7339415297499942, + "language_loss": 0.55400908, + "learning_rate": 2.479273078704891e-07, + "loss": 0.57435441, + "num_input_tokens_seen": 151793890, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01257324, + "step": 7024, + "time_per_iteration": 3.1136395931243896 + }, + { + "auxiliary_loss_clip": 0.01032242, + "auxiliary_loss_mlp": 0.01001115, + "balance_loss_clip": 1.00427115, + "balance_loss_mlp": 0.99985182, + "epoch": 0.844706306739614, + "flos": 62833331882880.0, + "grad_norm": 0.7794869567410929, + "language_loss": 0.64644158, + "learning_rate": 2.475517855270552e-07, + "loss": 0.66677511, + "num_input_tokens_seen": 151853970, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01263428, + "step": 7025, + "time_per_iteration": 3.242543935775757 + }, + { + "auxiliary_loss_clip": 0.01123921, + "auxiliary_loss_mlp": 0.0103483, + "balance_loss_clip": 1.04047632, + "balance_loss_mlp": 1.01995265, + "epoch": 0.8448265496302532, + "flos": 14976114969600.0, + "grad_norm": 1.9590092642012316, + "language_loss": 0.73307198, + "learning_rate": 2.4717652902862143e-07, + "loss": 0.75465953, + "num_input_tokens_seen": 151872945, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.14862061, + "step": 7026, + "time_per_iteration": 2.8203909397125244 + }, + { + "auxiliary_loss_clip": 0.01125168, + "auxiliary_loss_mlp": 0.01040761, + "balance_loss_clip": 1.03886008, + "balance_loss_mlp": 1.02498984, + "epoch": 0.8449467925208922, + "flos": 23441265192960.0, + "grad_norm": 1.7589122716318655, + "language_loss": 0.81288064, + "learning_rate": 2.4680153843211495e-07, + "loss": 0.83453989, + "num_input_tokens_seen": 151892875, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.15795898, + "step": 7027, + "time_per_iteration": 2.818711042404175 + }, + { + "auxiliary_loss_clip": 0.01125894, + "auxiliary_loss_mlp": 0.01041495, + "balance_loss_clip": 1.04211378, + "balance_loss_mlp": 1.02575934, + "epoch": 0.8450670354115313, + "flos": 22748045639040.0, + "grad_norm": 1.9298128070645977, + "language_loss": 0.7256487, + "learning_rate": 2.464268137944212e-07, + "loss": 0.74732256, + "num_input_tokens_seen": 151914170, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.15734863, + "step": 7028, + "time_per_iteration": 2.8178892135620117 + }, + { + "auxiliary_loss_clip": 0.01130384, + "auxiliary_loss_mlp": 0.01044196, + "balance_loss_clip": 1.0414896, + "balance_loss_mlp": 1.02731633, + "epoch": 0.8451872783021703, + "flos": 29825571605760.0, + "grad_norm": 2.0315553566034703, + "language_loss": 0.78254831, + "learning_rate": 2.46052355172385e-07, + "loss": 0.80429411, + "num_input_tokens_seen": 151932210, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.16870117, + "step": 7029, + "time_per_iteration": 2.883399486541748 + }, + { + "auxiliary_loss_clip": 0.01127172, + "auxiliary_loss_mlp": 0.01039981, + "balance_loss_clip": 1.04019547, + "balance_loss_mlp": 1.02349401, + "epoch": 0.8453075211928095, + "flos": 21870029589120.0, + "grad_norm": 1.767182027847302, + "language_loss": 0.74693489, + "learning_rate": 2.456781626228128e-07, + "loss": 0.76860642, + "num_input_tokens_seen": 151951715, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.16491699, + "step": 7030, + "time_per_iteration": 3.649400234222412 + }, + { + "auxiliary_loss_clip": 0.01032122, + "auxiliary_loss_mlp": 0.01000932, + "balance_loss_clip": 1.00413084, + "balance_loss_mlp": 0.99959129, + "epoch": 0.8454277640834486, + "flos": 58751869288320.0, + "grad_norm": 0.921756968494006, + "language_loss": 0.66290098, + "learning_rate": 2.453042362024675e-07, + "loss": 0.68323153, + "num_input_tokens_seen": 152004960, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01342773, + "step": 7031, + "time_per_iteration": 3.261944532394409 + }, + { + "auxiliary_loss_clip": 0.01124686, + "auxiliary_loss_mlp": 0.01039097, + "balance_loss_clip": 1.04057562, + "balance_loss_mlp": 1.0242312, + "epoch": 0.8455480069740876, + "flos": 27090076469760.0, + "grad_norm": 1.680921693735029, + "language_loss": 0.73299813, + "learning_rate": 2.449305759680751e-07, + "loss": 0.75463599, + "num_input_tokens_seen": 152026285, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.14855957, + "step": 7032, + "time_per_iteration": 2.7701547145843506 + }, + { + "auxiliary_loss_clip": 0.01128035, + "auxiliary_loss_mlp": 0.0103593, + "balance_loss_clip": 1.0418942, + "balance_loss_mlp": 1.02239382, + "epoch": 0.8456682498647268, + "flos": 27198670262400.0, + "grad_norm": 1.409142648990293, + "language_loss": 0.75270039, + "learning_rate": 2.445571819763188e-07, + "loss": 0.77434003, + "num_input_tokens_seen": 152048585, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.13537598, + "step": 7033, + "time_per_iteration": 2.820693016052246 + }, + { + "auxiliary_loss_clip": 0.01123736, + "auxiliary_loss_mlp": 0.01040146, + "balance_loss_clip": 1.03957224, + "balance_loss_mlp": 1.02454138, + "epoch": 0.8457884927553658, + "flos": 20631901737600.0, + "grad_norm": 1.7552272232214288, + "language_loss": 0.58441848, + "learning_rate": 2.4418405428384227e-07, + "loss": 0.60605729, + "num_input_tokens_seen": 152068795, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.15606689, + "step": 7034, + "time_per_iteration": 3.5477898120880127 + }, + { + "auxiliary_loss_clip": 0.01123567, + "auxiliary_loss_mlp": 0.01037119, + "balance_loss_clip": 1.03954923, + "balance_loss_mlp": 1.02224123, + "epoch": 0.8459087356460049, + "flos": 15299023259520.0, + "grad_norm": 1.6789066941942123, + "language_loss": 0.71542501, + "learning_rate": 2.4381119294724864e-07, + "loss": 0.73703194, + "num_input_tokens_seen": 152086240, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14880371, + "step": 7035, + "time_per_iteration": 2.7365424633026123 + }, + { + "auxiliary_loss_clip": 0.01125563, + "auxiliary_loss_mlp": 0.0104494, + "balance_loss_clip": 1.04071963, + "balance_loss_mlp": 1.03002727, + "epoch": 0.846028978536644, + "flos": 18843155326080.0, + "grad_norm": 1.9458117477049053, + "language_loss": 0.53969109, + "learning_rate": 2.434385980231004e-07, + "loss": 0.56139618, + "num_input_tokens_seen": 152105080, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14916992, + "step": 7036, + "time_per_iteration": 3.5236504077911377 + }, + { + "auxiliary_loss_clip": 0.01124084, + "auxiliary_loss_mlp": 0.01035511, + "balance_loss_clip": 1.04017055, + "balance_loss_mlp": 1.02089572, + "epoch": 0.8461492214272831, + "flos": 52661740285440.0, + "grad_norm": 1.4570416588209845, + "language_loss": 0.65534627, + "learning_rate": 2.4306626956792043e-07, + "loss": 0.67694223, + "num_input_tokens_seen": 152130025, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.14624023, + "step": 7037, + "time_per_iteration": 2.9843759536743164 + }, + { + "auxiliary_loss_clip": 0.01125649, + "auxiliary_loss_mlp": 0.01032342, + "balance_loss_clip": 1.03800857, + "balance_loss_mlp": 1.01710689, + "epoch": 0.8462694643179222, + "flos": 18588405093120.0, + "grad_norm": 1.716822625546848, + "language_loss": 0.75956929, + "learning_rate": 2.4269420763819017e-07, + "loss": 0.78114915, + "num_input_tokens_seen": 152148070, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.15222168, + "step": 7038, + "time_per_iteration": 2.895984172821045 + }, + { + "auxiliary_loss_clip": 0.01120021, + "auxiliary_loss_mlp": 0.01036409, + "balance_loss_clip": 1.03750849, + "balance_loss_mlp": 1.02130568, + "epoch": 0.8463897072085613, + "flos": 24387080163840.0, + "grad_norm": 2.390715447714546, + "language_loss": 0.83859271, + "learning_rate": 2.4232241229035223e-07, + "loss": 0.86015701, + "num_input_tokens_seen": 152165825, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.15100098, + "step": 7039, + "time_per_iteration": 2.8843905925750732 + }, + { + "auxiliary_loss_clip": 0.01032689, + "auxiliary_loss_mlp": 0.0100332, + "balance_loss_clip": 1.00475717, + "balance_loss_mlp": 1.00210369, + "epoch": 0.8465099500992004, + "flos": 68702140258560.0, + "grad_norm": 0.7556123203618621, + "language_loss": 0.56766641, + "learning_rate": 2.419508835808064e-07, + "loss": 0.58802652, + "num_input_tokens_seen": 152222380, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.012146, + "step": 7040, + "time_per_iteration": 3.295811414718628 + }, + { + "auxiliary_loss_clip": 0.01126768, + "auxiliary_loss_mlp": 0.01035042, + "balance_loss_clip": 1.04048395, + "balance_loss_mlp": 1.0187459, + "epoch": 0.8466301929898394, + "flos": 13735724561280.0, + "grad_norm": 2.185038701704339, + "language_loss": 0.62878138, + "learning_rate": 2.415796215659134e-07, + "loss": 0.65039945, + "num_input_tokens_seen": 152239085, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.1630249, + "step": 7041, + "time_per_iteration": 3.7003121376037598 + }, + { + "auxiliary_loss_clip": 0.01126822, + "auxiliary_loss_mlp": 0.01044952, + "balance_loss_clip": 1.041327, + "balance_loss_mlp": 1.02947879, + "epoch": 0.8467504358804786, + "flos": 19241260738560.0, + "grad_norm": 1.9086010050823201, + "language_loss": 0.77345872, + "learning_rate": 2.412086263019939e-07, + "loss": 0.79517651, + "num_input_tokens_seen": 152257110, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15472412, + "step": 7042, + "time_per_iteration": 2.74124813079834 + }, + { + "auxiliary_loss_clip": 0.0111915, + "auxiliary_loss_mlp": 0.01036874, + "balance_loss_clip": 1.03977394, + "balance_loss_mlp": 1.02244401, + "epoch": 0.8468706787711177, + "flos": 21324115710720.0, + "grad_norm": 1.9630809795449684, + "language_loss": 0.79824173, + "learning_rate": 2.408378978453276e-07, + "loss": 0.81980205, + "num_input_tokens_seen": 152277230, + "router_z_loss_clip": 0.79394531, + "router_z_loss_mlp": 0.14428711, + "step": 7043, + "time_per_iteration": 2.8005900382995605 + }, + { + "auxiliary_loss_clip": 0.01032377, + "auxiliary_loss_mlp": 0.01002275, + "balance_loss_clip": 1.00435615, + "balance_loss_mlp": 1.00102353, + "epoch": 0.8469909216617567, + "flos": 64877439058560.0, + "grad_norm": 0.8242606894866632, + "language_loss": 0.6398437, + "learning_rate": 2.404674362521533e-07, + "loss": 0.66019022, + "num_input_tokens_seen": 152335725, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01251221, + "step": 7044, + "time_per_iteration": 3.2084860801696777 + }, + { + "auxiliary_loss_clip": 0.01126839, + "auxiliary_loss_mlp": 0.01041759, + "balance_loss_clip": 1.04168963, + "balance_loss_mlp": 1.02636957, + "epoch": 0.8471111645523959, + "flos": 19280583152640.0, + "grad_norm": 2.1378278071098045, + "language_loss": 0.74808806, + "learning_rate": 2.4009724157866997e-07, + "loss": 0.76977402, + "num_input_tokens_seen": 152352785, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15393066, + "step": 7045, + "time_per_iteration": 2.7691893577575684 + }, + { + "auxiliary_loss_clip": 0.01124549, + "auxiliary_loss_mlp": 0.01031091, + "balance_loss_clip": 1.04055929, + "balance_loss_mlp": 1.01567113, + "epoch": 0.8472314074430349, + "flos": 22015826893440.0, + "grad_norm": 3.869678782573797, + "language_loss": 0.76941371, + "learning_rate": 2.3972731388103564e-07, + "loss": 0.79097009, + "num_input_tokens_seen": 152371265, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.15435791, + "step": 7046, + "time_per_iteration": 2.842752456665039 + }, + { + "auxiliary_loss_clip": 0.01032184, + "auxiliary_loss_mlp": 0.01000881, + "balance_loss_clip": 1.00430298, + "balance_loss_mlp": 0.99957603, + "epoch": 0.847351650333674, + "flos": 57882580243200.0, + "grad_norm": 0.8042304474567348, + "language_loss": 0.62334478, + "learning_rate": 2.393576532153687e-07, + "loss": 0.64367545, + "num_input_tokens_seen": 152435050, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01306152, + "step": 7047, + "time_per_iteration": 3.383780002593994 + }, + { + "auxiliary_loss_clip": 0.0103233, + "auxiliary_loss_mlp": 0.01001537, + "balance_loss_clip": 1.00434434, + "balance_loss_mlp": 1.00019002, + "epoch": 0.8474718932243132, + "flos": 41284238313600.0, + "grad_norm": 0.937609586214936, + "language_loss": 0.57723576, + "learning_rate": 2.389882596377453e-07, + "loss": 0.59757441, + "num_input_tokens_seen": 152489315, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01348877, + "step": 7048, + "time_per_iteration": 3.207017183303833 + }, + { + "auxiliary_loss_clip": 0.01123617, + "auxiliary_loss_mlp": 0.01035558, + "balance_loss_clip": 1.03877091, + "balance_loss_mlp": 1.01976252, + "epoch": 0.8475921361149522, + "flos": 38180906974080.0, + "grad_norm": 1.6573485546947915, + "language_loss": 0.76539761, + "learning_rate": 2.386191332042031e-07, + "loss": 0.78698933, + "num_input_tokens_seen": 152511210, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15795898, + "step": 7049, + "time_per_iteration": 2.8750522136688232 + }, + { + "auxiliary_loss_clip": 0.01131753, + "auxiliary_loss_mlp": 0.01040063, + "balance_loss_clip": 1.04167795, + "balance_loss_mlp": 1.02447093, + "epoch": 0.8477123790055913, + "flos": 25375054723200.0, + "grad_norm": 1.7054103267957714, + "language_loss": 0.7274701, + "learning_rate": 2.3825027397073794e-07, + "loss": 0.74918824, + "num_input_tokens_seen": 152531685, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.15588379, + "step": 7050, + "time_per_iteration": 2.8073132038116455 + }, + { + "auxiliary_loss_clip": 0.01123911, + "auxiliary_loss_mlp": 0.01034148, + "balance_loss_clip": 1.04048538, + "balance_loss_mlp": 1.01894331, + "epoch": 0.8478326218962304, + "flos": 30225185389440.0, + "grad_norm": 3.0034322102877145, + "language_loss": 0.67089856, + "learning_rate": 2.3788168199330515e-07, + "loss": 0.69247913, + "num_input_tokens_seen": 152553245, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.15203857, + "step": 7051, + "time_per_iteration": 2.8464133739471436 + }, + { + "auxiliary_loss_clip": 0.01122849, + "auxiliary_loss_mlp": 0.01034436, + "balance_loss_clip": 1.03935111, + "balance_loss_mlp": 1.01940346, + "epoch": 0.8479528647868695, + "flos": 38213800853760.0, + "grad_norm": 1.5349356632054434, + "language_loss": 0.72600508, + "learning_rate": 2.3751335732782074e-07, + "loss": 0.74757802, + "num_input_tokens_seen": 152574505, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.1505127, + "step": 7052, + "time_per_iteration": 2.861016273498535 + }, + { + "auxiliary_loss_clip": 0.01125296, + "auxiliary_loss_mlp": 0.01038514, + "balance_loss_clip": 1.04106688, + "balance_loss_mlp": 1.02363682, + "epoch": 0.8480731076775085, + "flos": 20957790856320.0, + "grad_norm": 1.8846961290215554, + "language_loss": 0.79494315, + "learning_rate": 2.371453000301582e-07, + "loss": 0.81658125, + "num_input_tokens_seen": 152593190, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.14874268, + "step": 7053, + "time_per_iteration": 2.76432466506958 + }, + { + "auxiliary_loss_clip": 0.01124249, + "auxiliary_loss_mlp": 0.01035706, + "balance_loss_clip": 1.03851783, + "balance_loss_mlp": 1.02036417, + "epoch": 0.8481933505681477, + "flos": 32596510487040.0, + "grad_norm": 1.752092616521739, + "language_loss": 0.74455678, + "learning_rate": 2.3677751015615222e-07, + "loss": 0.76615632, + "num_input_tokens_seen": 152615265, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15332031, + "step": 7054, + "time_per_iteration": 2.786644458770752 + }, + { + "auxiliary_loss_clip": 0.01126476, + "auxiliary_loss_mlp": 0.01040132, + "balance_loss_clip": 1.04032278, + "balance_loss_mlp": 1.02437234, + "epoch": 0.8483135934587868, + "flos": 20741177888640.0, + "grad_norm": 1.8166599536221093, + "language_loss": 0.85247231, + "learning_rate": 2.3640998776159593e-07, + "loss": 0.87413836, + "num_input_tokens_seen": 152632770, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15759277, + "step": 7055, + "time_per_iteration": 2.7469065189361572 + }, + { + "auxiliary_loss_clip": 0.01122101, + "auxiliary_loss_mlp": 0.01033217, + "balance_loss_clip": 1.04111075, + "balance_loss_mlp": 1.01980639, + "epoch": 0.8484338363494258, + "flos": 21653057485440.0, + "grad_norm": 1.8183695104617514, + "language_loss": 0.8141914, + "learning_rate": 2.3604273290224253e-07, + "loss": 0.83574456, + "num_input_tokens_seen": 152653485, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.1340332, + "step": 7056, + "time_per_iteration": 2.7614293098449707 + }, + { + "auxiliary_loss_clip": 0.01128461, + "auxiliary_loss_mlp": 0.01046281, + "balance_loss_clip": 1.04132938, + "balance_loss_mlp": 1.03024697, + "epoch": 0.848554079240065, + "flos": 15013964926080.0, + "grad_norm": 1.7623168233373199, + "language_loss": 0.74465287, + "learning_rate": 2.356757456338039e-07, + "loss": 0.76640034, + "num_input_tokens_seen": 152670970, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.16040039, + "step": 7057, + "time_per_iteration": 2.7109382152557373 + }, + { + "auxiliary_loss_clip": 0.01031993, + "auxiliary_loss_mlp": 0.00999546, + "balance_loss_clip": 1.00422084, + "balance_loss_mlp": 0.99821073, + "epoch": 0.848674322130704, + "flos": 68060453742720.0, + "grad_norm": 0.7555035010793506, + "language_loss": 0.58969426, + "learning_rate": 2.3530902601195147e-07, + "loss": 0.61000967, + "num_input_tokens_seen": 152739460, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.0133667, + "step": 7058, + "time_per_iteration": 3.357064723968506 + }, + { + "auxiliary_loss_clip": 0.01128506, + "auxiliary_loss_mlp": 0.01038748, + "balance_loss_clip": 1.04013515, + "balance_loss_mlp": 1.0225234, + "epoch": 0.8487945650213431, + "flos": 18475788977280.0, + "grad_norm": 8.187502435286335, + "language_loss": 0.79187977, + "learning_rate": 2.34942574092317e-07, + "loss": 0.81355226, + "num_input_tokens_seen": 152754710, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.16235352, + "step": 7059, + "time_per_iteration": 3.5902915000915527 + }, + { + "auxiliary_loss_clip": 0.01126183, + "auxiliary_loss_mlp": 0.01034978, + "balance_loss_clip": 1.03978157, + "balance_loss_mlp": 1.0194931, + "epoch": 0.8489148079119821, + "flos": 23473189405440.0, + "grad_norm": 1.7580928037237835, + "language_loss": 0.76490569, + "learning_rate": 2.3457638993049045e-07, + "loss": 0.78651726, + "num_input_tokens_seen": 152772700, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.15490723, + "step": 7060, + "time_per_iteration": 2.8655588626861572 + }, + { + "auxiliary_loss_clip": 0.0113219, + "auxiliary_loss_mlp": 0.0103841, + "balance_loss_clip": 1.04282403, + "balance_loss_mlp": 1.02261436, + "epoch": 0.8490350508026213, + "flos": 19937604775680.0, + "grad_norm": 2.168815545711369, + "language_loss": 0.6453439, + "learning_rate": 2.3421047358202252e-07, + "loss": 0.66704988, + "num_input_tokens_seen": 152791550, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.15789795, + "step": 7061, + "time_per_iteration": 2.8143470287323 + }, + { + "auxiliary_loss_clip": 0.01126073, + "auxiliary_loss_mlp": 0.01035797, + "balance_loss_clip": 1.04162467, + "balance_loss_mlp": 1.02055001, + "epoch": 0.8491552936932604, + "flos": 24279958828800.0, + "grad_norm": 2.1942410993656756, + "language_loss": 0.83622622, + "learning_rate": 2.3384482510242144e-07, + "loss": 0.85784495, + "num_input_tokens_seen": 152809410, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.15246582, + "step": 7062, + "time_per_iteration": 2.7787322998046875 + }, + { + "auxiliary_loss_clip": 0.01124475, + "auxiliary_loss_mlp": 0.01040442, + "balance_loss_clip": 1.03928185, + "balance_loss_mlp": 1.02416992, + "epoch": 0.8492755365838994, + "flos": 22522526098560.0, + "grad_norm": 1.8806132472344814, + "language_loss": 0.77966446, + "learning_rate": 2.3347944454715575e-07, + "loss": 0.80131364, + "num_input_tokens_seen": 152825800, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.16271973, + "step": 7063, + "time_per_iteration": 3.6310925483703613 + }, + { + "auxiliary_loss_clip": 0.01126923, + "auxiliary_loss_mlp": 0.01039586, + "balance_loss_clip": 1.03982997, + "balance_loss_mlp": 1.02331424, + "epoch": 0.8493957794745386, + "flos": 26980441182720.0, + "grad_norm": 1.7066545040736651, + "language_loss": 0.67182285, + "learning_rate": 2.331143319716542e-07, + "loss": 0.69348794, + "num_input_tokens_seen": 152845330, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.16271973, + "step": 7064, + "time_per_iteration": 2.8386735916137695 + }, + { + "auxiliary_loss_clip": 0.01126382, + "auxiliary_loss_mlp": 0.01039993, + "balance_loss_clip": 1.04052782, + "balance_loss_mlp": 1.02450776, + "epoch": 0.8495160223651776, + "flos": 29861985018240.0, + "grad_norm": 1.8877594015167687, + "language_loss": 0.66207528, + "learning_rate": 2.3274948743130363e-07, + "loss": 0.68373907, + "num_input_tokens_seen": 152865165, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15478516, + "step": 7065, + "time_per_iteration": 3.7231438159942627 + }, + { + "auxiliary_loss_clip": 0.0112329, + "auxiliary_loss_mlp": 0.01037976, + "balance_loss_clip": 1.03806043, + "balance_loss_mlp": 1.02252638, + "epoch": 0.8496362652558167, + "flos": 23075443128960.0, + "grad_norm": 1.5655565989201863, + "language_loss": 0.79639435, + "learning_rate": 2.3238491098145085e-07, + "loss": 0.81800705, + "num_input_tokens_seen": 152884695, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.15441895, + "step": 7066, + "time_per_iteration": 2.831270694732666 + }, + { + "auxiliary_loss_clip": 0.01126335, + "auxiliary_loss_mlp": 0.01041548, + "balance_loss_clip": 1.03929937, + "balance_loss_mlp": 1.02611065, + "epoch": 0.8497565081464559, + "flos": 14609107756800.0, + "grad_norm": 2.7896562923758026, + "language_loss": 0.73075509, + "learning_rate": 2.3202060267740141e-07, + "loss": 0.7524339, + "num_input_tokens_seen": 152902220, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15441895, + "step": 7067, + "time_per_iteration": 2.92006516456604 + }, + { + "auxiliary_loss_clip": 0.01124097, + "auxiliary_loss_mlp": 0.01035365, + "balance_loss_clip": 1.03855157, + "balance_loss_mlp": 1.01936769, + "epoch": 0.8498767510370949, + "flos": 21136446126720.0, + "grad_norm": 2.216093838327313, + "language_loss": 0.77452892, + "learning_rate": 2.3165656257442044e-07, + "loss": 0.7961235, + "num_input_tokens_seen": 152920740, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15991211, + "step": 7068, + "time_per_iteration": 2.727398633956909 + }, + { + "auxiliary_loss_clip": 0.01122093, + "auxiliary_loss_mlp": 0.01038752, + "balance_loss_clip": 1.03828466, + "balance_loss_mlp": 1.02405977, + "epoch": 0.849996993927734, + "flos": 23654538195840.0, + "grad_norm": 1.8431344805630994, + "language_loss": 0.89994168, + "learning_rate": 2.31292790727734e-07, + "loss": 0.9215501, + "num_input_tokens_seen": 152938305, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.14691162, + "step": 7069, + "time_per_iteration": 2.705665349960327 + }, + { + "auxiliary_loss_clip": 0.01120147, + "auxiliary_loss_mlp": 0.01037847, + "balance_loss_clip": 1.03673434, + "balance_loss_mlp": 1.02274942, + "epoch": 0.8501172368183731, + "flos": 20558069331840.0, + "grad_norm": 2.288355767922611, + "language_loss": 0.80234849, + "learning_rate": 2.3092928719252392e-07, + "loss": 0.82392842, + "num_input_tokens_seen": 152956705, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.15106201, + "step": 7070, + "time_per_iteration": 3.6622962951660156 + }, + { + "auxiliary_loss_clip": 0.01124808, + "auxiliary_loss_mlp": 0.01036836, + "balance_loss_clip": 1.03976214, + "balance_loss_mlp": 1.02118349, + "epoch": 0.8502374797090122, + "flos": 22272624201600.0, + "grad_norm": 1.9964775644917894, + "language_loss": 0.78684962, + "learning_rate": 2.3056605202393475e-07, + "loss": 0.80846608, + "num_input_tokens_seen": 152974265, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.15649414, + "step": 7071, + "time_per_iteration": 2.7809195518493652 + }, + { + "auxiliary_loss_clip": 0.01125237, + "auxiliary_loss_mlp": 0.01037409, + "balance_loss_clip": 1.03866923, + "balance_loss_mlp": 1.0214107, + "epoch": 0.8503577225996513, + "flos": 23659817495040.0, + "grad_norm": 1.859010795744197, + "language_loss": 0.66775978, + "learning_rate": 2.3020308527706888e-07, + "loss": 0.68938625, + "num_input_tokens_seen": 152993680, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.16003418, + "step": 7072, + "time_per_iteration": 2.737086534500122 + }, + { + "auxiliary_loss_clip": 0.01125162, + "auxiliary_loss_mlp": 0.0103415, + "balance_loss_clip": 1.04116488, + "balance_loss_mlp": 1.01923668, + "epoch": 0.8504779654902904, + "flos": 26758513002240.0, + "grad_norm": 2.0099022159120294, + "language_loss": 0.8892864, + "learning_rate": 2.2984038700698715e-07, + "loss": 0.91087955, + "num_input_tokens_seen": 153012990, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.14904785, + "step": 7073, + "time_per_iteration": 2.764512062072754 + }, + { + "auxiliary_loss_clip": 0.01121359, + "auxiliary_loss_mlp": 0.010355, + "balance_loss_clip": 1.04042161, + "balance_loss_mlp": 1.02105212, + "epoch": 0.8505982083809295, + "flos": 26468247196800.0, + "grad_norm": 1.9206922544123848, + "language_loss": 0.79050511, + "learning_rate": 2.2947795726871222e-07, + "loss": 0.81207371, + "num_input_tokens_seen": 153034015, + "router_z_loss_clip": 0.80957031, + "router_z_loss_mlp": 0.14440918, + "step": 7074, + "time_per_iteration": 2.7427079677581787 + }, + { + "auxiliary_loss_clip": 0.01125435, + "auxiliary_loss_mlp": 0.01039199, + "balance_loss_clip": 1.0411973, + "balance_loss_mlp": 1.0230701, + "epoch": 0.8507184512715685, + "flos": 20303390926080.0, + "grad_norm": 1.9223883224884604, + "language_loss": 0.8563602, + "learning_rate": 2.2911579611722253e-07, + "loss": 0.87800646, + "num_input_tokens_seen": 153053160, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.16113281, + "step": 7075, + "time_per_iteration": 2.7619144916534424 + }, + { + "auxiliary_loss_clip": 0.01123479, + "auxiliary_loss_mlp": 0.01035256, + "balance_loss_clip": 1.04117751, + "balance_loss_mlp": 1.02014613, + "epoch": 0.8508386941622077, + "flos": 19025186474880.0, + "grad_norm": 1.9233562805656141, + "language_loss": 0.87413019, + "learning_rate": 2.2875390360745905e-07, + "loss": 0.8957175, + "num_input_tokens_seen": 153072565, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.15106201, + "step": 7076, + "time_per_iteration": 2.786092519760132 + }, + { + "auxiliary_loss_clip": 0.01130536, + "auxiliary_loss_mlp": 0.01042383, + "balance_loss_clip": 1.04100609, + "balance_loss_mlp": 1.02564585, + "epoch": 0.8509589370528468, + "flos": 16433405654400.0, + "grad_norm": 1.726681873799334, + "language_loss": 0.77618599, + "learning_rate": 2.2839227979432008e-07, + "loss": 0.79791522, + "num_input_tokens_seen": 153090215, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.1673584, + "step": 7077, + "time_per_iteration": 2.8020036220550537 + }, + { + "auxiliary_loss_clip": 0.01125494, + "auxiliary_loss_mlp": 0.01039128, + "balance_loss_clip": 1.03906965, + "balance_loss_mlp": 1.02359533, + "epoch": 0.8510791799434858, + "flos": 18259714713600.0, + "grad_norm": 1.9250326290287003, + "language_loss": 0.85267407, + "learning_rate": 2.2803092473266373e-07, + "loss": 0.87432027, + "num_input_tokens_seen": 153107740, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15539551, + "step": 7078, + "time_per_iteration": 2.7822840213775635 + }, + { + "auxiliary_loss_clip": 0.011247, + "auxiliary_loss_mlp": 0.0103842, + "balance_loss_clip": 1.03986168, + "balance_loss_mlp": 1.02364993, + "epoch": 0.851199422834125, + "flos": 23441372933760.0, + "grad_norm": 3.5084671850331617, + "language_loss": 0.865888, + "learning_rate": 2.2766983847730724e-07, + "loss": 0.88751924, + "num_input_tokens_seen": 153127410, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.14770508, + "step": 7079, + "time_per_iteration": 2.741485834121704 + }, + { + "auxiliary_loss_clip": 0.01128193, + "auxiliary_loss_mlp": 0.01047142, + "balance_loss_clip": 1.0406177, + "balance_loss_mlp": 1.03015494, + "epoch": 0.851319665724764, + "flos": 16289404030080.0, + "grad_norm": 2.0278933818696148, + "language_loss": 0.66730523, + "learning_rate": 2.2730902108302663e-07, + "loss": 0.6890586, + "num_input_tokens_seen": 153144325, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.1697998, + "step": 7080, + "time_per_iteration": 2.771453380584717 + }, + { + "auxiliary_loss_clip": 0.0112722, + "auxiliary_loss_mlp": 0.01044236, + "balance_loss_clip": 1.04016662, + "balance_loss_mlp": 1.02788043, + "epoch": 0.8514399086154031, + "flos": 18989347680000.0, + "grad_norm": 1.6588013650460594, + "language_loss": 0.68647027, + "learning_rate": 2.269484726045583e-07, + "loss": 0.70818478, + "num_input_tokens_seen": 153163240, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.16369629, + "step": 7081, + "time_per_iteration": 2.7241334915161133 + }, + { + "auxiliary_loss_clip": 0.01124997, + "auxiliary_loss_mlp": 0.01036584, + "balance_loss_clip": 1.03992915, + "balance_loss_mlp": 1.02198136, + "epoch": 0.8515601515060423, + "flos": 24571194301440.0, + "grad_norm": 1.6072577761023086, + "language_loss": 0.78976297, + "learning_rate": 2.2658819309659672e-07, + "loss": 0.81137884, + "num_input_tokens_seen": 153183440, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.14611816, + "step": 7082, + "time_per_iteration": 2.832796573638916 + }, + { + "auxiliary_loss_clip": 0.01118256, + "auxiliary_loss_mlp": 0.01036522, + "balance_loss_clip": 1.03852201, + "balance_loss_mlp": 1.02270567, + "epoch": 0.8516803943966813, + "flos": 19529443555200.0, + "grad_norm": 1.836055868322218, + "language_loss": 0.84798479, + "learning_rate": 2.2622818261379706e-07, + "loss": 0.86953259, + "num_input_tokens_seen": 153200460, + "router_z_loss_clip": 0.79736328, + "router_z_loss_mlp": 0.13842773, + "step": 7083, + "time_per_iteration": 2.786731719970703 + }, + { + "auxiliary_loss_clip": 0.0113022, + "auxiliary_loss_mlp": 0.01045599, + "balance_loss_clip": 1.04401124, + "balance_loss_mlp": 1.02989304, + "epoch": 0.8518006372873204, + "flos": 20265792364800.0, + "grad_norm": 1.6525089801190498, + "language_loss": 0.75117606, + "learning_rate": 2.2586844121077142e-07, + "loss": 0.7729342, + "num_input_tokens_seen": 153218970, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15692139, + "step": 7084, + "time_per_iteration": 2.8230228424072266 + }, + { + "auxiliary_loss_clip": 0.01127655, + "auxiliary_loss_mlp": 0.01042472, + "balance_loss_clip": 1.04120541, + "balance_loss_mlp": 1.02703476, + "epoch": 0.8519208801779595, + "flos": 24133227770880.0, + "grad_norm": 1.6851475834983767, + "language_loss": 0.72087616, + "learning_rate": 2.2550896894209215e-07, + "loss": 0.74257743, + "num_input_tokens_seen": 153238485, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.15429688, + "step": 7085, + "time_per_iteration": 2.7811548709869385 + }, + { + "auxiliary_loss_clip": 0.01032071, + "auxiliary_loss_mlp": 0.0100089, + "balance_loss_clip": 1.00417209, + "balance_loss_mlp": 0.99953109, + "epoch": 0.8520411230685986, + "flos": 63035223252480.0, + "grad_norm": 0.6806273695889447, + "language_loss": 0.56591654, + "learning_rate": 2.2514976586229184e-07, + "loss": 0.58624613, + "num_input_tokens_seen": 153306430, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01361084, + "step": 7086, + "time_per_iteration": 3.4228670597076416 + }, + { + "auxiliary_loss_clip": 0.01032135, + "auxiliary_loss_mlp": 0.01001487, + "balance_loss_clip": 1.00448966, + "balance_loss_mlp": 1.0001992, + "epoch": 0.8521613659592376, + "flos": 65836865283840.0, + "grad_norm": 0.7548390878781439, + "language_loss": 0.54673791, + "learning_rate": 2.247908320258609e-07, + "loss": 0.56707418, + "num_input_tokens_seen": 153366520, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01287842, + "step": 7087, + "time_per_iteration": 3.2756242752075195 + }, + { + "auxiliary_loss_clip": 0.01125148, + "auxiliary_loss_mlp": 0.01039543, + "balance_loss_clip": 1.03897548, + "balance_loss_mlp": 1.02354491, + "epoch": 0.8522816088498768, + "flos": 23112323418240.0, + "grad_norm": 2.0773032467562578, + "language_loss": 0.79215622, + "learning_rate": 2.2443216748724914e-07, + "loss": 0.81380314, + "num_input_tokens_seen": 153387230, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.16003418, + "step": 7088, + "time_per_iteration": 3.6398720741271973 + }, + { + "auxiliary_loss_clip": 0.0112655, + "auxiliary_loss_mlp": 0.01032672, + "balance_loss_clip": 1.04077113, + "balance_loss_mlp": 1.01795554, + "epoch": 0.8524018517405159, + "flos": 31758140073600.0, + "grad_norm": 1.9305400491890496, + "language_loss": 0.74500883, + "learning_rate": 2.2407377230086588e-07, + "loss": 0.76660103, + "num_input_tokens_seen": 153409585, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.14709473, + "step": 7089, + "time_per_iteration": 2.832672357559204 + }, + { + "auxiliary_loss_clip": 0.01126996, + "auxiliary_loss_mlp": 0.01037058, + "balance_loss_clip": 1.04135227, + "balance_loss_mlp": 1.02233577, + "epoch": 0.8525220946311549, + "flos": 18690318956160.0, + "grad_norm": 1.8021682481383134, + "language_loss": 0.83452505, + "learning_rate": 2.23715646521079e-07, + "loss": 0.85616559, + "num_input_tokens_seen": 153427105, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.14733887, + "step": 7090, + "time_per_iteration": 2.7623965740203857 + }, + { + "auxiliary_loss_clip": 0.01128844, + "auxiliary_loss_mlp": 0.01036597, + "balance_loss_clip": 1.04009235, + "balance_loss_mlp": 1.0205636, + "epoch": 0.852642337521794, + "flos": 21793216354560.0, + "grad_norm": 1.9299465151984125, + "language_loss": 0.84347165, + "learning_rate": 2.2335779020221724e-07, + "loss": 0.86512607, + "num_input_tokens_seen": 153443725, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.16040039, + "step": 7091, + "time_per_iteration": 2.797339677810669 + }, + { + "auxiliary_loss_clip": 0.01032434, + "auxiliary_loss_mlp": 0.00999384, + "balance_loss_clip": 1.00452244, + "balance_loss_mlp": 0.99816841, + "epoch": 0.8527625804124331, + "flos": 69040132260480.0, + "grad_norm": 0.8032567824397691, + "language_loss": 0.56396234, + "learning_rate": 2.2300020339856497e-07, + "loss": 0.58428049, + "num_input_tokens_seen": 153506410, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.012146, + "step": 7092, + "time_per_iteration": 4.141516208648682 + }, + { + "auxiliary_loss_clip": 0.01124253, + "auxiliary_loss_mlp": 0.01041605, + "balance_loss_clip": 1.03881633, + "balance_loss_mlp": 1.02653658, + "epoch": 0.8528828233030722, + "flos": 26979399688320.0, + "grad_norm": 2.851361434019193, + "language_loss": 0.78391552, + "learning_rate": 2.2264288616436966e-07, + "loss": 0.80557418, + "num_input_tokens_seen": 153526665, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15075684, + "step": 7093, + "time_per_iteration": 2.7995026111602783 + }, + { + "auxiliary_loss_clip": 0.01124656, + "auxiliary_loss_mlp": 0.01036485, + "balance_loss_clip": 1.03917718, + "balance_loss_mlp": 1.01922333, + "epoch": 0.8530030661937112, + "flos": 17487598936320.0, + "grad_norm": 2.6448958096114095, + "language_loss": 0.72859156, + "learning_rate": 2.222858385538351e-07, + "loss": 0.75020289, + "num_input_tokens_seen": 153543465, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.17272949, + "step": 7094, + "time_per_iteration": 3.5442049503326416 + }, + { + "auxiliary_loss_clip": 0.0112344, + "auxiliary_loss_mlp": 0.01030178, + "balance_loss_clip": 1.03954482, + "balance_loss_mlp": 1.01599216, + "epoch": 0.8531233090843504, + "flos": 22160798184960.0, + "grad_norm": 1.5829221242256404, + "language_loss": 0.68193334, + "learning_rate": 2.2192906062112527e-07, + "loss": 0.70346951, + "num_input_tokens_seen": 153563340, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.14190674, + "step": 7095, + "time_per_iteration": 2.766143321990967 + }, + { + "auxiliary_loss_clip": 0.01124104, + "auxiliary_loss_mlp": 0.01039639, + "balance_loss_clip": 1.03834808, + "balance_loss_mlp": 1.02392709, + "epoch": 0.8532435519749895, + "flos": 37635388145280.0, + "grad_norm": 1.591017631474793, + "language_loss": 0.70491177, + "learning_rate": 2.2157255242036377e-07, + "loss": 0.72654915, + "num_input_tokens_seen": 153587005, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15710449, + "step": 7096, + "time_per_iteration": 2.8634023666381836 + }, + { + "auxiliary_loss_clip": 0.011227, + "auxiliary_loss_mlp": 0.01043936, + "balance_loss_clip": 1.04024243, + "balance_loss_mlp": 1.02892721, + "epoch": 0.8533637948656285, + "flos": 21398163598080.0, + "grad_norm": 1.5820872878737882, + "language_loss": 0.74108505, + "learning_rate": 2.2121631400563135e-07, + "loss": 0.76275146, + "num_input_tokens_seen": 153606835, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.15002441, + "step": 7097, + "time_per_iteration": 2.766730546951294 + }, + { + "auxiliary_loss_clip": 0.01032701, + "auxiliary_loss_mlp": 0.0100075, + "balance_loss_clip": 1.00471616, + "balance_loss_mlp": 0.99940884, + "epoch": 0.8534840377562677, + "flos": 53345122490880.0, + "grad_norm": 0.77851100652861, + "language_loss": 0.52943933, + "learning_rate": 2.208603454309701e-07, + "loss": 0.54977387, + "num_input_tokens_seen": 153664925, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.01342773, + "step": 7098, + "time_per_iteration": 4.089183568954468 + }, + { + "auxiliary_loss_clip": 0.01126909, + "auxiliary_loss_mlp": 0.01044093, + "balance_loss_clip": 1.03925979, + "balance_loss_mlp": 1.0282023, + "epoch": 0.8536042806469067, + "flos": 20814148368000.0, + "grad_norm": 1.8993580836435036, + "language_loss": 0.71042877, + "learning_rate": 2.2050464675037994e-07, + "loss": 0.73213881, + "num_input_tokens_seen": 153683550, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.15881348, + "step": 7099, + "time_per_iteration": 2.7788636684417725 + }, + { + "auxiliary_loss_clip": 0.01130244, + "auxiliary_loss_mlp": 0.01036262, + "balance_loss_clip": 1.04119039, + "balance_loss_mlp": 1.01981139, + "epoch": 0.8537245235375458, + "flos": 24681368292480.0, + "grad_norm": 1.9746275771462902, + "language_loss": 0.73020399, + "learning_rate": 2.2014921801782016e-07, + "loss": 0.75186902, + "num_input_tokens_seen": 153703040, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.16430664, + "step": 7100, + "time_per_iteration": 2.926621675491333 + }, + { + "auxiliary_loss_clip": 0.01124575, + "auxiliary_loss_mlp": 0.01037672, + "balance_loss_clip": 1.0399878, + "balance_loss_mlp": 1.02229404, + "epoch": 0.853844766428185, + "flos": 24384817607040.0, + "grad_norm": 1.8637173307236952, + "language_loss": 0.73938608, + "learning_rate": 2.1979405928720872e-07, + "loss": 0.76100856, + "num_input_tokens_seen": 153722695, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.15393066, + "step": 7101, + "time_per_iteration": 2.8710274696350098 + }, + { + "auxiliary_loss_clip": 0.01122865, + "auxiliary_loss_mlp": 0.010335, + "balance_loss_clip": 1.03950751, + "balance_loss_mlp": 1.01911187, + "epoch": 0.853965009318824, + "flos": 20955707867520.0, + "grad_norm": 1.4697734117475256, + "language_loss": 0.79581928, + "learning_rate": 2.1943917061242257e-07, + "loss": 0.81738293, + "num_input_tokens_seen": 153742550, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.1439209, + "step": 7102, + "time_per_iteration": 2.8545279502868652 + }, + { + "auxiliary_loss_clip": 0.01134071, + "auxiliary_loss_mlp": 0.01045162, + "balance_loss_clip": 1.0423708, + "balance_loss_mlp": 1.02934265, + "epoch": 0.8540852522094631, + "flos": 24201816791040.0, + "grad_norm": 1.6979480131960465, + "language_loss": 0.66384584, + "learning_rate": 2.1908455204729903e-07, + "loss": 0.68563819, + "num_input_tokens_seen": 153761700, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.15808105, + "step": 7103, + "time_per_iteration": 2.869504690170288 + }, + { + "auxiliary_loss_clip": 0.01129242, + "auxiliary_loss_mlp": 0.01040162, + "balance_loss_clip": 1.04225028, + "balance_loss_mlp": 1.02387834, + "epoch": 0.8542054951001022, + "flos": 25082921410560.0, + "grad_norm": 1.8493444577960234, + "language_loss": 0.78487021, + "learning_rate": 2.1873020364563265e-07, + "loss": 0.80656421, + "num_input_tokens_seen": 153780765, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.16271973, + "step": 7104, + "time_per_iteration": 2.840919017791748 + }, + { + "auxiliary_loss_clip": 0.01127482, + "auxiliary_loss_mlp": 0.01036734, + "balance_loss_clip": 1.04150343, + "balance_loss_mlp": 1.02278614, + "epoch": 0.8543257379907413, + "flos": 24316551809280.0, + "grad_norm": 2.3204458985741865, + "language_loss": 0.76747257, + "learning_rate": 2.183761254611789e-07, + "loss": 0.78911471, + "num_input_tokens_seen": 153801090, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.13946533, + "step": 7105, + "time_per_iteration": 2.834582805633545 + }, + { + "auxiliary_loss_clip": 0.01123236, + "auxiliary_loss_mlp": 0.01032728, + "balance_loss_clip": 1.04010844, + "balance_loss_mlp": 1.0174458, + "epoch": 0.8544459808813804, + "flos": 55286630467200.0, + "grad_norm": 4.734031058736478, + "language_loss": 0.703403, + "learning_rate": 2.1802231754764987e-07, + "loss": 0.72496259, + "num_input_tokens_seen": 153826530, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.152771, + "step": 7106, + "time_per_iteration": 3.0677595138549805 + }, + { + "auxiliary_loss_clip": 0.01123585, + "auxiliary_loss_mlp": 0.01037603, + "balance_loss_clip": 1.03842318, + "balance_loss_mlp": 1.02232623, + "epoch": 0.8545662237720195, + "flos": 25776248705280.0, + "grad_norm": 1.809985810106765, + "language_loss": 0.76533461, + "learning_rate": 2.17668779958718e-07, + "loss": 0.78694654, + "num_input_tokens_seen": 153849110, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.15283203, + "step": 7107, + "time_per_iteration": 2.7837400436401367 + }, + { + "auxiliary_loss_clip": 0.01126444, + "auxiliary_loss_mlp": 0.01034699, + "balance_loss_clip": 1.04172945, + "balance_loss_mlp": 1.01907086, + "epoch": 0.8546864666626586, + "flos": 11108320427520.0, + "grad_norm": 2.5014224962688165, + "language_loss": 0.80960697, + "learning_rate": 2.1731551274801553e-07, + "loss": 0.83121836, + "num_input_tokens_seen": 153865550, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15637207, + "step": 7108, + "time_per_iteration": 2.7437849044799805 + }, + { + "auxiliary_loss_clip": 0.0112562, + "auxiliary_loss_mlp": 0.01039947, + "balance_loss_clip": 1.04091287, + "balance_loss_mlp": 1.02498615, + "epoch": 0.8548067095532976, + "flos": 25520169669120.0, + "grad_norm": 2.1781122887902438, + "language_loss": 0.61340594, + "learning_rate": 2.169625159691324e-07, + "loss": 0.63506162, + "num_input_tokens_seen": 153885425, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.14978027, + "step": 7109, + "time_per_iteration": 2.7671031951904297 + }, + { + "auxiliary_loss_clip": 0.01127083, + "auxiliary_loss_mlp": 0.01038488, + "balance_loss_clip": 1.04003763, + "balance_loss_mlp": 1.02295506, + "epoch": 0.8549269524439368, + "flos": 24717853532160.0, + "grad_norm": 2.408506801973433, + "language_loss": 0.74039662, + "learning_rate": 2.1660978967561784e-07, + "loss": 0.76205236, + "num_input_tokens_seen": 153904760, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15539551, + "step": 7110, + "time_per_iteration": 2.794504404067993 + }, + { + "auxiliary_loss_clip": 0.01121952, + "auxiliary_loss_mlp": 0.01031784, + "balance_loss_clip": 1.03700185, + "balance_loss_mlp": 1.01643014, + "epoch": 0.8550471953345758, + "flos": 19825599191040.0, + "grad_norm": 2.165465149152973, + "language_loss": 0.78828597, + "learning_rate": 2.1625733392098035e-07, + "loss": 0.80982339, + "num_input_tokens_seen": 153920370, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.15332031, + "step": 7111, + "time_per_iteration": 2.734221935272217 + }, + { + "auxiliary_loss_clip": 0.01121815, + "auxiliary_loss_mlp": 0.01035465, + "balance_loss_clip": 1.03867662, + "balance_loss_mlp": 1.02160084, + "epoch": 0.8551674382252149, + "flos": 22820441500800.0, + "grad_norm": 1.61916477672425, + "language_loss": 0.79749489, + "learning_rate": 2.159051487586867e-07, + "loss": 0.8190676, + "num_input_tokens_seen": 153940500, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.13873291, + "step": 7112, + "time_per_iteration": 2.8976848125457764 + }, + { + "auxiliary_loss_clip": 0.01132975, + "auxiliary_loss_mlp": 0.01042847, + "balance_loss_clip": 1.04486012, + "balance_loss_mlp": 1.02651548, + "epoch": 0.8552876811158541, + "flos": 20631255292800.0, + "grad_norm": 2.161704909810232, + "language_loss": 0.72544169, + "learning_rate": 2.155532342421642e-07, + "loss": 0.74719983, + "num_input_tokens_seen": 153958500, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.16333008, + "step": 7113, + "time_per_iteration": 2.835763931274414 + }, + { + "auxiliary_loss_clip": 0.01126836, + "auxiliary_loss_mlp": 0.01048767, + "balance_loss_clip": 1.04186106, + "balance_loss_mlp": 1.03319275, + "epoch": 0.8554079240064931, + "flos": 23112359331840.0, + "grad_norm": 2.040992915276908, + "language_loss": 0.78216791, + "learning_rate": 2.1520159042479636e-07, + "loss": 0.8039239, + "num_input_tokens_seen": 153976790, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15570068, + "step": 7114, + "time_per_iteration": 2.965477466583252 + }, + { + "auxiliary_loss_clip": 0.01124683, + "auxiliary_loss_mlp": 0.01035722, + "balance_loss_clip": 1.04080141, + "balance_loss_mlp": 1.02004623, + "epoch": 0.8555281668971322, + "flos": 22128047959680.0, + "grad_norm": 2.307705412711343, + "language_loss": 0.71086431, + "learning_rate": 2.148502173599287e-07, + "loss": 0.73246837, + "num_input_tokens_seen": 153994930, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.15686035, + "step": 7115, + "time_per_iteration": 2.787130355834961 + }, + { + "auxiliary_loss_clip": 0.01126127, + "auxiliary_loss_mlp": 0.01034685, + "balance_loss_clip": 1.04166579, + "balance_loss_mlp": 1.01970053, + "epoch": 0.8556484097877713, + "flos": 31139040234240.0, + "grad_norm": 1.6743958719480472, + "language_loss": 0.65682948, + "learning_rate": 2.1449911510086372e-07, + "loss": 0.67843759, + "num_input_tokens_seen": 154014400, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14990234, + "step": 7116, + "time_per_iteration": 2.789921283721924 + }, + { + "auxiliary_loss_clip": 0.01124979, + "auxiliary_loss_mlp": 0.01034161, + "balance_loss_clip": 1.0408783, + "balance_loss_mlp": 1.0193373, + "epoch": 0.8557686526784104, + "flos": 24316551809280.0, + "grad_norm": 2.0427877678651933, + "language_loss": 0.76797235, + "learning_rate": 2.141482837008628e-07, + "loss": 0.78956378, + "num_input_tokens_seen": 154034940, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.14813232, + "step": 7117, + "time_per_iteration": 3.6367897987365723 + }, + { + "auxiliary_loss_clip": 0.01124118, + "auxiliary_loss_mlp": 0.01034518, + "balance_loss_clip": 1.03985596, + "balance_loss_mlp": 1.01817441, + "epoch": 0.8558888955690495, + "flos": 17712723427200.0, + "grad_norm": 1.8522678161910926, + "language_loss": 0.71912402, + "learning_rate": 2.1379772321314826e-07, + "loss": 0.74071044, + "num_input_tokens_seen": 154052985, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.16339111, + "step": 7118, + "time_per_iteration": 2.7842254638671875 + }, + { + "auxiliary_loss_clip": 0.01125672, + "auxiliary_loss_mlp": 0.01041839, + "balance_loss_clip": 1.04046965, + "balance_loss_mlp": 1.02693725, + "epoch": 0.8560091384596886, + "flos": 19171702051200.0, + "grad_norm": 2.077830817036451, + "language_loss": 0.81280148, + "learning_rate": 2.1344743369089802e-07, + "loss": 0.83447659, + "num_input_tokens_seen": 154068765, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.14904785, + "step": 7119, + "time_per_iteration": 2.844228506088257 + }, + { + "auxiliary_loss_clip": 0.01124495, + "auxiliary_loss_mlp": 0.01035428, + "balance_loss_clip": 1.04039192, + "balance_loss_mlp": 1.02158761, + "epoch": 0.8561293813503277, + "flos": 23914855036800.0, + "grad_norm": 1.9354947291288138, + "language_loss": 0.82196432, + "learning_rate": 2.130974151872522e-07, + "loss": 0.84356356, + "num_input_tokens_seen": 154089100, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.13848877, + "step": 7120, + "time_per_iteration": 2.8366994857788086 + }, + { + "auxiliary_loss_clip": 0.01125862, + "auxiliary_loss_mlp": 0.01037963, + "balance_loss_clip": 1.03857732, + "balance_loss_mlp": 1.0228951, + "epoch": 0.8562496242409667, + "flos": 22529206028160.0, + "grad_norm": 1.8794062159088816, + "language_loss": 0.78335661, + "learning_rate": 2.1274766775530773e-07, + "loss": 0.80499488, + "num_input_tokens_seen": 154108965, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.15075684, + "step": 7121, + "time_per_iteration": 3.6575751304626465 + }, + { + "auxiliary_loss_clip": 0.01130753, + "auxiliary_loss_mlp": 0.01036333, + "balance_loss_clip": 1.0405314, + "balance_loss_mlp": 1.0206095, + "epoch": 0.8563698671316058, + "flos": 14712745472640.0, + "grad_norm": 1.927583761063592, + "language_loss": 0.79513443, + "learning_rate": 2.1239819144812077e-07, + "loss": 0.81680524, + "num_input_tokens_seen": 154123425, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.15734863, + "step": 7122, + "time_per_iteration": 2.8094215393066406 + }, + { + "auxiliary_loss_clip": 0.01125423, + "auxiliary_loss_mlp": 0.01041145, + "balance_loss_clip": 1.03943276, + "balance_loss_mlp": 1.02556443, + "epoch": 0.856490110022245, + "flos": 39167768211840.0, + "grad_norm": 1.795484736390973, + "language_loss": 0.69919229, + "learning_rate": 2.1204898631870716e-07, + "loss": 0.72085798, + "num_input_tokens_seen": 154148315, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15588379, + "step": 7123, + "time_per_iteration": 3.727463483810425 + }, + { + "auxiliary_loss_clip": 0.01122816, + "auxiliary_loss_mlp": 0.01028958, + "balance_loss_clip": 1.03986299, + "balance_loss_mlp": 1.01375866, + "epoch": 0.856610352912884, + "flos": 29059345658880.0, + "grad_norm": 1.8696688160582968, + "language_loss": 0.75958896, + "learning_rate": 2.1170005242004006e-07, + "loss": 0.78110671, + "num_input_tokens_seen": 154169665, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.15197754, + "step": 7124, + "time_per_iteration": 2.8806233406066895 + }, + { + "auxiliary_loss_clip": 0.01124554, + "auxiliary_loss_mlp": 0.01045273, + "balance_loss_clip": 1.04191208, + "balance_loss_mlp": 1.03109884, + "epoch": 0.8567305958035231, + "flos": 23878333883520.0, + "grad_norm": 1.7099719182546014, + "language_loss": 0.78134018, + "learning_rate": 2.1135138980505384e-07, + "loss": 0.80303848, + "num_input_tokens_seen": 154190335, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.14172363, + "step": 7125, + "time_per_iteration": 2.8479299545288086 + }, + { + "auxiliary_loss_clip": 0.01126785, + "auxiliary_loss_mlp": 0.0104165, + "balance_loss_clip": 1.04184318, + "balance_loss_mlp": 1.0260222, + "epoch": 0.8568508386941622, + "flos": 22200120599040.0, + "grad_norm": 3.275932824847789, + "language_loss": 0.72261024, + "learning_rate": 2.110029985266395e-07, + "loss": 0.74429464, + "num_input_tokens_seen": 154210040, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.15612793, + "step": 7126, + "time_per_iteration": 2.818892478942871 + }, + { + "auxiliary_loss_clip": 0.01127032, + "auxiliary_loss_mlp": 0.01035763, + "balance_loss_clip": 1.0429225, + "balance_loss_mlp": 1.02063549, + "epoch": 0.8569710815848013, + "flos": 17307507121920.0, + "grad_norm": 1.5611137307402212, + "language_loss": 0.73782337, + "learning_rate": 2.1065487863764787e-07, + "loss": 0.75945127, + "num_input_tokens_seen": 154228385, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.15124512, + "step": 7127, + "time_per_iteration": 3.6885926723480225 + }, + { + "auxiliary_loss_clip": 0.01123278, + "auxiliary_loss_mlp": 0.01036998, + "balance_loss_clip": 1.03885674, + "balance_loss_mlp": 1.02220392, + "epoch": 0.8570913244754403, + "flos": 23732285184000.0, + "grad_norm": 1.5193179486041468, + "language_loss": 0.85747015, + "learning_rate": 2.1030703019088846e-07, + "loss": 0.8790729, + "num_input_tokens_seen": 154249015, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.14807129, + "step": 7128, + "time_per_iteration": 2.7566301822662354 + }, + { + "auxiliary_loss_clip": 0.0112113, + "auxiliary_loss_mlp": 0.01036248, + "balance_loss_clip": 1.03832555, + "balance_loss_mlp": 1.02183604, + "epoch": 0.8572115673660795, + "flos": 20048748433920.0, + "grad_norm": 1.8115764813203363, + "language_loss": 0.70972705, + "learning_rate": 2.099594532391291e-07, + "loss": 0.73130083, + "num_input_tokens_seen": 154267700, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.14404297, + "step": 7129, + "time_per_iteration": 2.7325689792633057 + }, + { + "auxiliary_loss_clip": 0.01122375, + "auxiliary_loss_mlp": 0.01035735, + "balance_loss_clip": 1.03858316, + "balance_loss_mlp": 1.02041721, + "epoch": 0.8573318102567186, + "flos": 27160389342720.0, + "grad_norm": 1.5665206939570515, + "language_loss": 0.79271996, + "learning_rate": 2.0961214783509806e-07, + "loss": 0.81430113, + "num_input_tokens_seen": 154290580, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.15319824, + "step": 7130, + "time_per_iteration": 2.8327722549438477 + }, + { + "auxiliary_loss_clip": 0.01126164, + "auxiliary_loss_mlp": 0.01037299, + "balance_loss_clip": 1.04058719, + "balance_loss_mlp": 1.02279162, + "epoch": 0.8574520531473576, + "flos": 24936585402240.0, + "grad_norm": 1.606486423562692, + "language_loss": 0.74522042, + "learning_rate": 2.0926511403148051e-07, + "loss": 0.766855, + "num_input_tokens_seen": 154309545, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.14501953, + "step": 7131, + "time_per_iteration": 2.734909772872925 + }, + { + "auxiliary_loss_clip": 0.01124136, + "auxiliary_loss_mlp": 0.01039094, + "balance_loss_clip": 1.04040647, + "balance_loss_mlp": 1.02383566, + "epoch": 0.8575722960379968, + "flos": 18771154513920.0, + "grad_norm": 2.0466932656809935, + "language_loss": 0.76245326, + "learning_rate": 2.0891835188092143e-07, + "loss": 0.78408557, + "num_input_tokens_seen": 154326545, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.15246582, + "step": 7132, + "time_per_iteration": 2.826855182647705 + }, + { + "auxiliary_loss_clip": 0.01123991, + "auxiliary_loss_mlp": 0.01040521, + "balance_loss_clip": 1.03984463, + "balance_loss_mlp": 1.02524996, + "epoch": 0.8576925389286358, + "flos": 22200300167040.0, + "grad_norm": 1.7560534296151298, + "language_loss": 0.81439894, + "learning_rate": 2.0857186143602434e-07, + "loss": 0.83604407, + "num_input_tokens_seen": 154345190, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.15252686, + "step": 7133, + "time_per_iteration": 2.7795777320861816 + }, + { + "auxiliary_loss_clip": 0.01125435, + "auxiliary_loss_mlp": 0.01032737, + "balance_loss_clip": 1.03967142, + "balance_loss_mlp": 1.01782417, + "epoch": 0.8578127818192749, + "flos": 22894345733760.0, + "grad_norm": 1.7372324276567517, + "language_loss": 0.67394495, + "learning_rate": 2.0822564274935094e-07, + "loss": 0.6955266, + "num_input_tokens_seen": 154364615, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.14910889, + "step": 7134, + "time_per_iteration": 2.817350149154663 + }, + { + "auxiliary_loss_clip": 0.01129654, + "auxiliary_loss_mlp": 0.01039491, + "balance_loss_clip": 1.04122639, + "balance_loss_mlp": 1.02288508, + "epoch": 0.8579330247099141, + "flos": 34824839541120.0, + "grad_norm": 2.2836237742702186, + "language_loss": 0.66651124, + "learning_rate": 2.078796958734239e-07, + "loss": 0.68820268, + "num_input_tokens_seen": 154387335, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.16601562, + "step": 7135, + "time_per_iteration": 2.9246082305908203 + }, + { + "auxiliary_loss_clip": 0.01125852, + "auxiliary_loss_mlp": 0.01034266, + "balance_loss_clip": 1.04006517, + "balance_loss_mlp": 1.01891232, + "epoch": 0.8580532676005531, + "flos": 19755681367680.0, + "grad_norm": 1.8499616721885583, + "language_loss": 0.75141084, + "learning_rate": 2.0753402086072124e-07, + "loss": 0.77301198, + "num_input_tokens_seen": 154405965, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15344238, + "step": 7136, + "time_per_iteration": 2.8545000553131104 + }, + { + "auxiliary_loss_clip": 0.01128191, + "auxiliary_loss_mlp": 0.01039077, + "balance_loss_clip": 1.04280066, + "balance_loss_mlp": 1.02270913, + "epoch": 0.8581735104911922, + "flos": 22739318634240.0, + "grad_norm": 2.372937412677304, + "language_loss": 0.74869943, + "learning_rate": 2.071886177636828e-07, + "loss": 0.77037209, + "num_input_tokens_seen": 154422750, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.16369629, + "step": 7137, + "time_per_iteration": 2.853771448135376 + }, + { + "auxiliary_loss_clip": 0.01125001, + "auxiliary_loss_mlp": 0.01033749, + "balance_loss_clip": 1.03969264, + "balance_loss_mlp": 1.01905036, + "epoch": 0.8582937533818313, + "flos": 23149131880320.0, + "grad_norm": 1.6631863383217034, + "language_loss": 0.83582377, + "learning_rate": 2.0684348663470575e-07, + "loss": 0.85741127, + "num_input_tokens_seen": 154442930, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.14697266, + "step": 7138, + "time_per_iteration": 2.775733709335327 + }, + { + "auxiliary_loss_clip": 0.01126964, + "auxiliary_loss_mlp": 0.01045994, + "balance_loss_clip": 1.03905964, + "balance_loss_mlp": 1.02985334, + "epoch": 0.8584139962724704, + "flos": 19498668577920.0, + "grad_norm": 2.0279809629951453, + "language_loss": 0.61484087, + "learning_rate": 2.0649862752614555e-07, + "loss": 0.63657039, + "num_input_tokens_seen": 154461640, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.16137695, + "step": 7139, + "time_per_iteration": 2.767399549484253 + }, + { + "auxiliary_loss_clip": 0.0103287, + "auxiliary_loss_mlp": 0.01003662, + "balance_loss_clip": 1.00503135, + "balance_loss_mlp": 1.0023154, + "epoch": 0.8585342391631094, + "flos": 71276577788160.0, + "grad_norm": 0.7529725070352467, + "language_loss": 0.57020468, + "learning_rate": 2.0615404049031838e-07, + "loss": 0.59057003, + "num_input_tokens_seen": 154518610, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01348877, + "step": 7140, + "time_per_iteration": 3.3014540672302246 + }, + { + "auxiliary_loss_clip": 0.01125697, + "auxiliary_loss_mlp": 0.01043812, + "balance_loss_clip": 1.03924942, + "balance_loss_mlp": 1.02633572, + "epoch": 0.8586544820537486, + "flos": 10815432929280.0, + "grad_norm": 2.4877751396061116, + "language_loss": 0.78202003, + "learning_rate": 2.0580972557949616e-07, + "loss": 0.80371511, + "num_input_tokens_seen": 154533700, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.17468262, + "step": 7141, + "time_per_iteration": 2.827488660812378 + }, + { + "auxiliary_loss_clip": 0.01032757, + "auxiliary_loss_mlp": 0.01001213, + "balance_loss_clip": 1.00475168, + "balance_loss_mlp": 0.99992573, + "epoch": 0.8587747249443877, + "flos": 64811184422400.0, + "grad_norm": 0.7994217951895941, + "language_loss": 0.54213417, + "learning_rate": 2.054656828459125e-07, + "loss": 0.56247389, + "num_input_tokens_seen": 154597810, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.01287842, + "step": 7142, + "time_per_iteration": 3.329770565032959 + }, + { + "auxiliary_loss_clip": 0.01126305, + "auxiliary_loss_mlp": 0.01045831, + "balance_loss_clip": 1.04044235, + "balance_loss_mlp": 1.02942777, + "epoch": 0.8588949678350267, + "flos": 26834607964800.0, + "grad_norm": 1.7116548412506902, + "language_loss": 0.77284604, + "learning_rate": 2.051219123417578e-07, + "loss": 0.79456735, + "num_input_tokens_seen": 154617870, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.1640625, + "step": 7143, + "time_per_iteration": 2.8048248291015625 + }, + { + "auxiliary_loss_clip": 0.01127677, + "auxiliary_loss_mlp": 0.01037959, + "balance_loss_clip": 1.03998375, + "balance_loss_mlp": 1.02219963, + "epoch": 0.8590152107256659, + "flos": 26104256726400.0, + "grad_norm": 2.0414542881399327, + "language_loss": 0.5977791, + "learning_rate": 2.0477841411918196e-07, + "loss": 0.61943543, + "num_input_tokens_seen": 154637395, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15759277, + "step": 7144, + "time_per_iteration": 2.875537395477295 + }, + { + "auxiliary_loss_clip": 0.01119184, + "auxiliary_loss_mlp": 0.01036423, + "balance_loss_clip": 1.03812659, + "balance_loss_mlp": 1.02162337, + "epoch": 0.859135453616305, + "flos": 26140885620480.0, + "grad_norm": 2.6520165933613624, + "language_loss": 0.74982989, + "learning_rate": 2.0443518823029326e-07, + "loss": 0.77138597, + "num_input_tokens_seen": 154657935, + "router_z_loss_clip": 0.80957031, + "router_z_loss_mlp": 0.14801025, + "step": 7145, + "time_per_iteration": 2.824308395385742 + }, + { + "auxiliary_loss_clip": 0.01125082, + "auxiliary_loss_mlp": 0.01038655, + "balance_loss_clip": 1.04084253, + "balance_loss_mlp": 1.02372444, + "epoch": 0.859255696506944, + "flos": 12969319046400.0, + "grad_norm": 1.9776391077781266, + "language_loss": 0.76241672, + "learning_rate": 2.0409223472715854e-07, + "loss": 0.7840541, + "num_input_tokens_seen": 154675080, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.14941406, + "step": 7146, + "time_per_iteration": 3.6245718002319336 + }, + { + "auxiliary_loss_clip": 0.01123007, + "auxiliary_loss_mlp": 0.01036484, + "balance_loss_clip": 1.03819001, + "balance_loss_mlp": 1.02250051, + "epoch": 0.8593759393975832, + "flos": 18475753063680.0, + "grad_norm": 1.8501281044442022, + "language_loss": 0.74582821, + "learning_rate": 2.0374955366180434e-07, + "loss": 0.76742315, + "num_input_tokens_seen": 154692720, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.14001465, + "step": 7147, + "time_per_iteration": 2.8583078384399414 + }, + { + "auxiliary_loss_clip": 0.01124304, + "auxiliary_loss_mlp": 0.01032475, + "balance_loss_clip": 1.038481, + "balance_loss_mlp": 1.0166198, + "epoch": 0.8594961822882222, + "flos": 22200156512640.0, + "grad_norm": 2.352630021827806, + "language_loss": 0.72900712, + "learning_rate": 2.034071450862147e-07, + "loss": 0.75057489, + "num_input_tokens_seen": 154710190, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15856934, + "step": 7148, + "time_per_iteration": 2.85622239112854 + }, + { + "auxiliary_loss_clip": 0.01130492, + "auxiliary_loss_mlp": 0.01043886, + "balance_loss_clip": 1.04154181, + "balance_loss_mlp": 1.02756596, + "epoch": 0.8596164251788613, + "flos": 23294749616640.0, + "grad_norm": 1.57670988004707, + "language_loss": 0.7673111, + "learning_rate": 2.030650090523327e-07, + "loss": 0.78905487, + "num_input_tokens_seen": 154729380, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.16320801, + "step": 7149, + "time_per_iteration": 2.8382632732391357 + }, + { + "auxiliary_loss_clip": 0.01126616, + "auxiliary_loss_mlp": 0.01042546, + "balance_loss_clip": 1.04091144, + "balance_loss_mlp": 1.02774048, + "epoch": 0.8597366680695004, + "flos": 31649905416960.0, + "grad_norm": 1.7957991092237684, + "language_loss": 0.59539998, + "learning_rate": 2.0272314561205995e-07, + "loss": 0.6170916, + "num_input_tokens_seen": 154749775, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.14825439, + "step": 7150, + "time_per_iteration": 3.6439459323883057 + }, + { + "auxiliary_loss_clip": 0.01124231, + "auxiliary_loss_mlp": 0.01035486, + "balance_loss_clip": 1.03852129, + "balance_loss_mlp": 1.02096581, + "epoch": 0.8598569109601395, + "flos": 21287738211840.0, + "grad_norm": 2.6653547544820486, + "language_loss": 0.72937208, + "learning_rate": 2.023815548172567e-07, + "loss": 0.75096929, + "num_input_tokens_seen": 154769845, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.14526367, + "step": 7151, + "time_per_iteration": 2.771864652633667 + }, + { + "auxiliary_loss_clip": 0.01128552, + "auxiliary_loss_mlp": 0.01040552, + "balance_loss_clip": 1.0408349, + "balance_loss_mlp": 1.02594841, + "epoch": 0.8599771538507786, + "flos": 25447809720960.0, + "grad_norm": 1.6230097573737627, + "language_loss": 0.65637207, + "learning_rate": 2.0204023671974267e-07, + "loss": 0.67806309, + "num_input_tokens_seen": 154789230, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.14599609, + "step": 7152, + "time_per_iteration": 3.5949273109436035 + }, + { + "auxiliary_loss_clip": 0.01120603, + "auxiliary_loss_mlp": 0.01038622, + "balance_loss_clip": 1.03580022, + "balance_loss_mlp": 1.02375674, + "epoch": 0.8600973967414177, + "flos": 16723958768640.0, + "grad_norm": 2.1159148211958607, + "language_loss": 0.8108145, + "learning_rate": 2.0169919137129532e-07, + "loss": 0.83240676, + "num_input_tokens_seen": 154807670, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14855957, + "step": 7153, + "time_per_iteration": 2.7557616233825684 + }, + { + "auxiliary_loss_clip": 0.01129, + "auxiliary_loss_mlp": 0.01038047, + "balance_loss_clip": 1.0410918, + "balance_loss_mlp": 1.02214491, + "epoch": 0.8602176396320568, + "flos": 25227928615680.0, + "grad_norm": 2.312315804484809, + "language_loss": 0.70957363, + "learning_rate": 2.013584188236508e-07, + "loss": 0.73124409, + "num_input_tokens_seen": 154825575, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.15893555, + "step": 7154, + "time_per_iteration": 2.867429494857788 + }, + { + "auxiliary_loss_clip": 0.01125205, + "auxiliary_loss_mlp": 0.01034299, + "balance_loss_clip": 1.03815949, + "balance_loss_mlp": 1.01900482, + "epoch": 0.8603378825226958, + "flos": 20412236113920.0, + "grad_norm": 1.806956132503773, + "language_loss": 0.79716623, + "learning_rate": 2.0101791912850396e-07, + "loss": 0.81876123, + "num_input_tokens_seen": 154845115, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.15283203, + "step": 7155, + "time_per_iteration": 2.7779500484466553 + }, + { + "auxiliary_loss_clip": 0.01129481, + "auxiliary_loss_mlp": 0.01039754, + "balance_loss_clip": 1.04179955, + "balance_loss_mlp": 1.02342284, + "epoch": 0.8604581254133349, + "flos": 34930201109760.0, + "grad_norm": 1.8788176941356478, + "language_loss": 0.64191318, + "learning_rate": 2.006776923375082e-07, + "loss": 0.66360557, + "num_input_tokens_seen": 154866770, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16339111, + "step": 7156, + "time_per_iteration": 3.7554681301116943 + }, + { + "auxiliary_loss_clip": 0.01124558, + "auxiliary_loss_mlp": 0.01032935, + "balance_loss_clip": 1.03977466, + "balance_loss_mlp": 1.01800978, + "epoch": 0.860578368303974, + "flos": 22596538072320.0, + "grad_norm": 1.5900380920582076, + "language_loss": 0.71320188, + "learning_rate": 2.003377385022764e-07, + "loss": 0.73477674, + "num_input_tokens_seen": 154885595, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.14910889, + "step": 7157, + "time_per_iteration": 2.858973503112793 + }, + { + "auxiliary_loss_clip": 0.01126659, + "auxiliary_loss_mlp": 0.01031175, + "balance_loss_clip": 1.04116988, + "balance_loss_mlp": 1.0159936, + "epoch": 0.8606986111946131, + "flos": 21324331192320.0, + "grad_norm": 3.1722549486386167, + "language_loss": 0.77437055, + "learning_rate": 1.9999805767437826e-07, + "loss": 0.79594886, + "num_input_tokens_seen": 154904485, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.1519165, + "step": 7158, + "time_per_iteration": 2.944627285003662 + }, + { + "auxiliary_loss_clip": 0.01122945, + "auxiliary_loss_mlp": 0.01038192, + "balance_loss_clip": 1.0394212, + "balance_loss_mlp": 1.0241133, + "epoch": 0.8608188540852522, + "flos": 28877206769280.0, + "grad_norm": 1.8930768198715404, + "language_loss": 0.71971369, + "learning_rate": 1.9965864990534386e-07, + "loss": 0.74132514, + "num_input_tokens_seen": 154925010, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.14093018, + "step": 7159, + "time_per_iteration": 2.880875587463379 + }, + { + "auxiliary_loss_clip": 0.01124437, + "auxiliary_loss_mlp": 0.01033099, + "balance_loss_clip": 1.03949416, + "balance_loss_mlp": 1.01827574, + "epoch": 0.8609390969758913, + "flos": 29716187713920.0, + "grad_norm": 1.6872635716734827, + "language_loss": 0.77594829, + "learning_rate": 1.9931951524666092e-07, + "loss": 0.79752368, + "num_input_tokens_seen": 154946100, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.14813232, + "step": 7160, + "time_per_iteration": 2.9215757846832275 + }, + { + "auxiliary_loss_clip": 0.01124879, + "auxiliary_loss_mlp": 0.01035501, + "balance_loss_clip": 1.04011357, + "balance_loss_mlp": 1.01963425, + "epoch": 0.8610593398665304, + "flos": 21249349551360.0, + "grad_norm": 1.6889939555533382, + "language_loss": 0.81111377, + "learning_rate": 1.9898065374977534e-07, + "loss": 0.83271766, + "num_input_tokens_seen": 154966305, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15856934, + "step": 7161, + "time_per_iteration": 2.8107452392578125 + }, + { + "auxiliary_loss_clip": 0.0111786, + "auxiliary_loss_mlp": 0.01031614, + "balance_loss_clip": 1.03813732, + "balance_loss_mlp": 1.01829219, + "epoch": 0.8611795827571694, + "flos": 14830102183680.0, + "grad_norm": 1.7502060722034332, + "language_loss": 0.72928262, + "learning_rate": 1.9864206546609342e-07, + "loss": 0.75077736, + "num_input_tokens_seen": 154985145, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.13311768, + "step": 7162, + "time_per_iteration": 2.8541533946990967 + }, + { + "auxiliary_loss_clip": 0.011252, + "auxiliary_loss_mlp": 0.01035799, + "balance_loss_clip": 1.03816831, + "balance_loss_mlp": 1.02186346, + "epoch": 0.8612998256478086, + "flos": 24243258107520.0, + "grad_norm": 1.777822633637712, + "language_loss": 0.84468329, + "learning_rate": 1.983037504469771e-07, + "loss": 0.86629325, + "num_input_tokens_seen": 155003855, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.13928223, + "step": 7163, + "time_per_iteration": 2.8670053482055664 + }, + { + "auxiliary_loss_clip": 0.01126558, + "auxiliary_loss_mlp": 0.01045498, + "balance_loss_clip": 1.04031444, + "balance_loss_mlp": 1.02966714, + "epoch": 0.8614200685384477, + "flos": 21252653602560.0, + "grad_norm": 1.8862628888903417, + "language_loss": 0.66848218, + "learning_rate": 1.9796570874374984e-07, + "loss": 0.69020277, + "num_input_tokens_seen": 155023960, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15856934, + "step": 7164, + "time_per_iteration": 2.9358935356140137 + }, + { + "auxiliary_loss_clip": 0.01123947, + "auxiliary_loss_mlp": 0.01041784, + "balance_loss_clip": 1.03905189, + "balance_loss_mlp": 1.02721632, + "epoch": 0.8615403114290867, + "flos": 20007738080640.0, + "grad_norm": 1.5995747069212847, + "language_loss": 0.77482998, + "learning_rate": 1.976279404076917e-07, + "loss": 0.79648733, + "num_input_tokens_seen": 155043360, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.14562988, + "step": 7165, + "time_per_iteration": 2.7882120609283447 + }, + { + "auxiliary_loss_clip": 0.01122824, + "auxiliary_loss_mlp": 0.01043027, + "balance_loss_clip": 1.03781176, + "balance_loss_mlp": 1.02748203, + "epoch": 0.8616605543197259, + "flos": 29789373674880.0, + "grad_norm": 1.6531221815594963, + "language_loss": 0.75998509, + "learning_rate": 1.9729044549004193e-07, + "loss": 0.78164363, + "num_input_tokens_seen": 155064745, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.15551758, + "step": 7166, + "time_per_iteration": 2.822990655899048 + }, + { + "auxiliary_loss_clip": 0.01123405, + "auxiliary_loss_mlp": 0.01036132, + "balance_loss_clip": 1.03947484, + "balance_loss_mlp": 1.02089703, + "epoch": 0.8617807972103649, + "flos": 28911609020160.0, + "grad_norm": 1.5464648698161167, + "language_loss": 0.70374984, + "learning_rate": 1.9695322404199822e-07, + "loss": 0.72534519, + "num_input_tokens_seen": 155086790, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.15234375, + "step": 7167, + "time_per_iteration": 2.8568379878997803 + }, + { + "auxiliary_loss_clip": 0.01124562, + "auxiliary_loss_mlp": 0.0104207, + "balance_loss_clip": 1.03872299, + "balance_loss_mlp": 1.02549982, + "epoch": 0.861901040101004, + "flos": 27673804391040.0, + "grad_norm": 1.9942188213498033, + "language_loss": 0.82079542, + "learning_rate": 1.9661627611471654e-07, + "loss": 0.84246171, + "num_input_tokens_seen": 155106585, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.16577148, + "step": 7168, + "time_per_iteration": 2.7745790481567383 + }, + { + "auxiliary_loss_clip": 0.01127929, + "auxiliary_loss_mlp": 0.01038446, + "balance_loss_clip": 1.04009938, + "balance_loss_mlp": 1.02223313, + "epoch": 0.8620212829916432, + "flos": 49748056755840.0, + "grad_norm": 2.05548564410577, + "language_loss": 0.70448565, + "learning_rate": 1.9627960175931246e-07, + "loss": 0.72614944, + "num_input_tokens_seen": 155131285, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.1619873, + "step": 7169, + "time_per_iteration": 2.9670674800872803 + }, + { + "auxiliary_loss_clip": 0.01121488, + "auxiliary_loss_mlp": 0.01031335, + "balance_loss_clip": 1.03952205, + "balance_loss_mlp": 1.0163269, + "epoch": 0.8621415258822822, + "flos": 21138672769920.0, + "grad_norm": 1.9566422649940491, + "language_loss": 0.74229991, + "learning_rate": 1.9594320102685847e-07, + "loss": 0.7638281, + "num_input_tokens_seen": 155150555, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.15014648, + "step": 7170, + "time_per_iteration": 2.7133939266204834 + }, + { + "auxiliary_loss_clip": 0.01119942, + "auxiliary_loss_mlp": 0.0103845, + "balance_loss_clip": 1.03742099, + "balance_loss_mlp": 1.02424645, + "epoch": 0.8622617687729213, + "flos": 21689039934720.0, + "grad_norm": 1.8529797811518716, + "language_loss": 0.64151168, + "learning_rate": 1.956070739683864e-07, + "loss": 0.66309559, + "num_input_tokens_seen": 155169890, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.14190674, + "step": 7171, + "time_per_iteration": 2.7689366340637207 + }, + { + "auxiliary_loss_clip": 0.01119824, + "auxiliary_loss_mlp": 0.01034643, + "balance_loss_clip": 1.03864634, + "balance_loss_mlp": 1.01974201, + "epoch": 0.8623820116635604, + "flos": 26250592734720.0, + "grad_norm": 1.6216515682726091, + "language_loss": 0.74247885, + "learning_rate": 1.9527122063488678e-07, + "loss": 0.76402354, + "num_input_tokens_seen": 155191005, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.14892578, + "step": 7172, + "time_per_iteration": 2.8073861598968506 + }, + { + "auxiliary_loss_clip": 0.0112112, + "auxiliary_loss_mlp": 0.01034595, + "balance_loss_clip": 1.03645849, + "balance_loss_mlp": 1.01932478, + "epoch": 0.8625022545541995, + "flos": 19647554451840.0, + "grad_norm": 1.5323769016402615, + "language_loss": 0.8042469, + "learning_rate": 1.9493564107730755e-07, + "loss": 0.825804, + "num_input_tokens_seen": 155211005, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.15270996, + "step": 7173, + "time_per_iteration": 2.929915189743042 + }, + { + "auxiliary_loss_clip": 0.01120077, + "auxiliary_loss_mlp": 0.01041063, + "balance_loss_clip": 1.0382942, + "balance_loss_mlp": 1.02645993, + "epoch": 0.8626224974448385, + "flos": 21908382336000.0, + "grad_norm": 1.8437693776467052, + "language_loss": 0.60520434, + "learning_rate": 1.9460033534655684e-07, + "loss": 0.6268158, + "num_input_tokens_seen": 155230365, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.14611816, + "step": 7174, + "time_per_iteration": 3.628166437149048 + }, + { + "auxiliary_loss_clip": 0.0112071, + "auxiliary_loss_mlp": 0.01039523, + "balance_loss_clip": 1.03727651, + "balance_loss_mlp": 1.02447891, + "epoch": 0.8627427403354777, + "flos": 23331198942720.0, + "grad_norm": 1.543418253335662, + "language_loss": 0.84337413, + "learning_rate": 1.9426530349349978e-07, + "loss": 0.86497647, + "num_input_tokens_seen": 155250815, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.15063477, + "step": 7175, + "time_per_iteration": 2.7904582023620605 + }, + { + "auxiliary_loss_clip": 0.01121029, + "auxiliary_loss_mlp": 0.01031766, + "balance_loss_clip": 1.039464, + "balance_loss_mlp": 1.01681685, + "epoch": 0.8628629832261168, + "flos": 16362877299840.0, + "grad_norm": 1.833036165281915, + "language_loss": 0.64557236, + "learning_rate": 1.9393054556896038e-07, + "loss": 0.66710031, + "num_input_tokens_seen": 155268515, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.14929199, + "step": 7176, + "time_per_iteration": 2.7739500999450684 + }, + { + "auxiliary_loss_clip": 0.01123131, + "auxiliary_loss_mlp": 0.0104041, + "balance_loss_clip": 1.03754926, + "balance_loss_mlp": 1.02491856, + "epoch": 0.8629832261167558, + "flos": 28103941756800.0, + "grad_norm": 2.634932366513048, + "language_loss": 0.69107455, + "learning_rate": 1.9359606162372133e-07, + "loss": 0.71270996, + "num_input_tokens_seen": 155290120, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15484619, + "step": 7177, + "time_per_iteration": 2.792459011077881 + }, + { + "auxiliary_loss_clip": 0.01122735, + "auxiliary_loss_mlp": 0.01035161, + "balance_loss_clip": 1.0399735, + "balance_loss_mlp": 1.01997375, + "epoch": 0.863103469007395, + "flos": 20230061310720.0, + "grad_norm": 1.6309763315174894, + "language_loss": 0.70619428, + "learning_rate": 1.9326185170852293e-07, + "loss": 0.72777319, + "num_input_tokens_seen": 155309085, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.15185547, + "step": 7178, + "time_per_iteration": 2.8084635734558105 + }, + { + "auxiliary_loss_clip": 0.0112713, + "auxiliary_loss_mlp": 0.01033739, + "balance_loss_clip": 1.04318881, + "balance_loss_mlp": 1.0189985, + "epoch": 0.863223711898034, + "flos": 24498547044480.0, + "grad_norm": 1.811477981657862, + "language_loss": 0.72401726, + "learning_rate": 1.9292791587406598e-07, + "loss": 0.74562597, + "num_input_tokens_seen": 155327945, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.1473999, + "step": 7179, + "time_per_iteration": 3.594414234161377 + }, + { + "auxiliary_loss_clip": 0.01125097, + "auxiliary_loss_mlp": 0.01041942, + "balance_loss_clip": 1.04057813, + "balance_loss_mlp": 1.0267781, + "epoch": 0.8633439547886731, + "flos": 17675376261120.0, + "grad_norm": 2.1442715774647305, + "language_loss": 0.86668789, + "learning_rate": 1.9259425417100661e-07, + "loss": 0.88835829, + "num_input_tokens_seen": 155344060, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.1517334, + "step": 7180, + "time_per_iteration": 2.8824257850646973 + }, + { + "auxiliary_loss_clip": 0.01125787, + "auxiliary_loss_mlp": 0.01036922, + "balance_loss_clip": 1.03887558, + "balance_loss_mlp": 1.02167487, + "epoch": 0.8634641976793123, + "flos": 12895055677440.0, + "grad_norm": 1.907441683638739, + "language_loss": 0.74750936, + "learning_rate": 1.9226086664996234e-07, + "loss": 0.76913643, + "num_input_tokens_seen": 155362305, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15246582, + "step": 7181, + "time_per_iteration": 3.678274631500244 + }, + { + "auxiliary_loss_clip": 0.01124533, + "auxiliary_loss_mlp": 0.01041955, + "balance_loss_clip": 1.0399859, + "balance_loss_mlp": 1.02742362, + "epoch": 0.8635844405699513, + "flos": 23878980328320.0, + "grad_norm": 2.245453766811532, + "language_loss": 0.74036467, + "learning_rate": 1.9192775336150712e-07, + "loss": 0.76202953, + "num_input_tokens_seen": 155382605, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.14550781, + "step": 7182, + "time_per_iteration": 2.7990591526031494 + }, + { + "auxiliary_loss_clip": 0.01032667, + "auxiliary_loss_mlp": 0.01001661, + "balance_loss_clip": 1.00482798, + "balance_loss_mlp": 1.00041509, + "epoch": 0.8637046834605904, + "flos": 60453387521280.0, + "grad_norm": 0.7588817159275365, + "language_loss": 0.5621196, + "learning_rate": 1.915949143561739e-07, + "loss": 0.58246291, + "num_input_tokens_seen": 155437280, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01245117, + "step": 7183, + "time_per_iteration": 3.3089938163757324 + }, + { + "auxiliary_loss_clip": 0.01125069, + "auxiliary_loss_mlp": 0.01041541, + "balance_loss_clip": 1.04034996, + "balance_loss_mlp": 1.02613962, + "epoch": 0.8638249263512295, + "flos": 20558751690240.0, + "grad_norm": 2.1200112705816148, + "language_loss": 0.78033584, + "learning_rate": 1.9126234968445498e-07, + "loss": 0.80200195, + "num_input_tokens_seen": 155456970, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15386963, + "step": 7184, + "time_per_iteration": 2.8534305095672607 + }, + { + "auxiliary_loss_clip": 0.01125738, + "auxiliary_loss_mlp": 0.01039611, + "balance_loss_clip": 1.03906488, + "balance_loss_mlp": 1.02441156, + "epoch": 0.8639451692418686, + "flos": 26615768353920.0, + "grad_norm": 1.4324873433575667, + "language_loss": 0.67830646, + "learning_rate": 1.9093005939679884e-07, + "loss": 0.69995987, + "num_input_tokens_seen": 155478925, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.15197754, + "step": 7185, + "time_per_iteration": 3.7368593215942383 + }, + { + "auxiliary_loss_clip": 0.0112554, + "auxiliary_loss_mlp": 0.01038588, + "balance_loss_clip": 1.04080832, + "balance_loss_mlp": 1.02301967, + "epoch": 0.8640654121325076, + "flos": 15122450977920.0, + "grad_norm": 2.2452237465184464, + "language_loss": 0.76739442, + "learning_rate": 1.9059804354361452e-07, + "loss": 0.78903568, + "num_input_tokens_seen": 155496700, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15576172, + "step": 7186, + "time_per_iteration": 2.8303282260894775 + }, + { + "auxiliary_loss_clip": 0.01126481, + "auxiliary_loss_mlp": 0.0103574, + "balance_loss_clip": 1.04040837, + "balance_loss_mlp": 1.02015901, + "epoch": 0.8641856550231467, + "flos": 31869068250240.0, + "grad_norm": 1.6230109135312876, + "language_loss": 0.70434839, + "learning_rate": 1.902663021752684e-07, + "loss": 0.72597063, + "num_input_tokens_seen": 155518130, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15576172, + "step": 7187, + "time_per_iteration": 2.9010963439941406 + }, + { + "auxiliary_loss_clip": 0.01127978, + "auxiliary_loss_mlp": 0.01040258, + "balance_loss_clip": 1.04154038, + "balance_loss_mlp": 1.02553558, + "epoch": 0.8643058979137859, + "flos": 14976545932800.0, + "grad_norm": 2.303380714931531, + "language_loss": 0.82299483, + "learning_rate": 1.8993483534208556e-07, + "loss": 0.84467721, + "num_input_tokens_seen": 155537040, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.14703369, + "step": 7188, + "time_per_iteration": 2.7493226528167725 + }, + { + "auxiliary_loss_clip": 0.01127179, + "auxiliary_loss_mlp": 0.01038902, + "balance_loss_clip": 1.04052615, + "balance_loss_mlp": 1.02366674, + "epoch": 0.8644261408044249, + "flos": 13115726881920.0, + "grad_norm": 2.277748208265564, + "language_loss": 0.74758059, + "learning_rate": 1.8960364309434884e-07, + "loss": 0.76924145, + "num_input_tokens_seen": 155554535, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.15228271, + "step": 7189, + "time_per_iteration": 2.775386333465576 + }, + { + "auxiliary_loss_clip": 0.0112478, + "auxiliary_loss_mlp": 0.0102727, + "balance_loss_clip": 1.04084849, + "balance_loss_mlp": 1.01313806, + "epoch": 0.864546383695064, + "flos": 20850920916480.0, + "grad_norm": 1.7310849809231614, + "language_loss": 0.78340459, + "learning_rate": 1.8927272548229967e-07, + "loss": 0.80492508, + "num_input_tokens_seen": 155574225, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14117432, + "step": 7190, + "time_per_iteration": 2.778594970703125 + }, + { + "auxiliary_loss_clip": 0.01128897, + "auxiliary_loss_mlp": 0.01040501, + "balance_loss_clip": 1.04252505, + "balance_loss_mlp": 1.02498603, + "epoch": 0.8646666265857031, + "flos": 21324582587520.0, + "grad_norm": 1.6113554507944552, + "language_loss": 0.8318854, + "learning_rate": 1.8894208255613876e-07, + "loss": 0.8535794, + "num_input_tokens_seen": 155593540, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.15509033, + "step": 7191, + "time_per_iteration": 2.8040080070495605 + }, + { + "auxiliary_loss_clip": 0.01121307, + "auxiliary_loss_mlp": 0.01033643, + "balance_loss_clip": 1.03804827, + "balance_loss_mlp": 1.01937366, + "epoch": 0.8647868694763422, + "flos": 19750833031680.0, + "grad_norm": 1.8640150213471807, + "language_loss": 0.7768302, + "learning_rate": 1.8861171436602397e-07, + "loss": 0.79837966, + "num_input_tokens_seen": 155610655, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.1427002, + "step": 7192, + "time_per_iteration": 2.723710060119629 + }, + { + "auxiliary_loss_clip": 0.01127668, + "auxiliary_loss_mlp": 0.01042196, + "balance_loss_clip": 1.04192448, + "balance_loss_mlp": 1.02586436, + "epoch": 0.8649071123669813, + "flos": 26176760328960.0, + "grad_norm": 2.0488966601845093, + "language_loss": 0.8016758, + "learning_rate": 1.882816209620719e-07, + "loss": 0.82337445, + "num_input_tokens_seen": 155627365, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.16333008, + "step": 7193, + "time_per_iteration": 2.814734935760498 + }, + { + "auxiliary_loss_clip": 0.01126564, + "auxiliary_loss_mlp": 0.01034174, + "balance_loss_clip": 1.04038405, + "balance_loss_mlp": 1.0178659, + "epoch": 0.8650273552576204, + "flos": 20302888135680.0, + "grad_norm": 2.3388158713199934, + "language_loss": 0.7667641, + "learning_rate": 1.8795180239435738e-07, + "loss": 0.7883715, + "num_input_tokens_seen": 155646220, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.16320801, + "step": 7194, + "time_per_iteration": 2.7494847774505615 + }, + { + "auxiliary_loss_clip": 0.01126969, + "auxiliary_loss_mlp": 0.01039744, + "balance_loss_clip": 1.0417918, + "balance_loss_mlp": 1.02428246, + "epoch": 0.8651475981482595, + "flos": 23951088881280.0, + "grad_norm": 2.4283795366410232, + "language_loss": 0.75673592, + "learning_rate": 1.8762225871291348e-07, + "loss": 0.77840304, + "num_input_tokens_seen": 155662095, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15466309, + "step": 7195, + "time_per_iteration": 2.7541487216949463 + }, + { + "auxiliary_loss_clip": 0.01123098, + "auxiliary_loss_mlp": 0.01032465, + "balance_loss_clip": 1.03766739, + "balance_loss_mlp": 1.01728356, + "epoch": 0.8652678410388985, + "flos": 21684622561920.0, + "grad_norm": 1.6164456409891474, + "language_loss": 0.8103407, + "learning_rate": 1.8729298996773201e-07, + "loss": 0.83189631, + "num_input_tokens_seen": 155680845, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15185547, + "step": 7196, + "time_per_iteration": 2.8744606971740723 + }, + { + "auxiliary_loss_clip": 0.01032166, + "auxiliary_loss_mlp": 0.01000199, + "balance_loss_clip": 1.00446105, + "balance_loss_mlp": 0.99891204, + "epoch": 0.8653880839295377, + "flos": 65224660855680.0, + "grad_norm": 0.8338647253807843, + "language_loss": 0.60918641, + "learning_rate": 1.8696399620876301e-07, + "loss": 0.62951005, + "num_input_tokens_seen": 155737875, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01287842, + "step": 7197, + "time_per_iteration": 3.281545877456665 + }, + { + "auxiliary_loss_clip": 0.01127662, + "auxiliary_loss_mlp": 0.01036843, + "balance_loss_clip": 1.0408895, + "balance_loss_mlp": 1.02229965, + "epoch": 0.8655083268201768, + "flos": 17749172753280.0, + "grad_norm": 2.044545817288897, + "language_loss": 0.7907089, + "learning_rate": 1.866352774859141e-07, + "loss": 0.81235397, + "num_input_tokens_seen": 155753100, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.14526367, + "step": 7198, + "time_per_iteration": 2.8490471839904785 + }, + { + "auxiliary_loss_clip": 0.01122736, + "auxiliary_loss_mlp": 0.01035455, + "balance_loss_clip": 1.0394398, + "balance_loss_mlp": 1.02067351, + "epoch": 0.8656285697108158, + "flos": 20703974376960.0, + "grad_norm": 2.100815451721886, + "language_loss": 0.69056565, + "learning_rate": 1.8630683384905188e-07, + "loss": 0.71214759, + "num_input_tokens_seen": 155772430, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.14770508, + "step": 7199, + "time_per_iteration": 2.7973451614379883 + }, + { + "auxiliary_loss_clip": 0.01123727, + "auxiliary_loss_mlp": 0.01042455, + "balance_loss_clip": 1.03955102, + "balance_loss_mlp": 1.02722609, + "epoch": 0.865748812601455, + "flos": 18653833716480.0, + "grad_norm": 1.8377936186326205, + "language_loss": 0.88669777, + "learning_rate": 1.8597866534800045e-07, + "loss": 0.90835959, + "num_input_tokens_seen": 155787545, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.15209961, + "step": 7200, + "time_per_iteration": 2.7803969383239746 + }, + { + "auxiliary_loss_clip": 0.01126149, + "auxiliary_loss_mlp": 0.01038132, + "balance_loss_clip": 1.04174125, + "balance_loss_mlp": 1.02280211, + "epoch": 0.865869055492094, + "flos": 70652554807680.0, + "grad_norm": 2.0756590508048176, + "language_loss": 0.7462346, + "learning_rate": 1.8565077203254398e-07, + "loss": 0.7678774, + "num_input_tokens_seen": 155813005, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.15319824, + "step": 7201, + "time_per_iteration": 3.208850860595703 + }, + { + "auxiliary_loss_clip": 0.01132325, + "auxiliary_loss_mlp": 0.01038699, + "balance_loss_clip": 1.04099655, + "balance_loss_mlp": 1.02175975, + "epoch": 0.8659892983827331, + "flos": 17383961220480.0, + "grad_norm": 2.273908280501263, + "language_loss": 0.72540772, + "learning_rate": 1.8532315395242203e-07, + "loss": 0.747118, + "num_input_tokens_seen": 155829455, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.16931152, + "step": 7202, + "time_per_iteration": 2.7522549629211426 + }, + { + "auxiliary_loss_clip": 0.0112259, + "auxiliary_loss_mlp": 0.01036846, + "balance_loss_clip": 1.0384661, + "balance_loss_mlp": 1.02281487, + "epoch": 0.8661095412733723, + "flos": 17895221452800.0, + "grad_norm": 2.826162741940104, + "language_loss": 0.71713626, + "learning_rate": 1.849958111573353e-07, + "loss": 0.73873061, + "num_input_tokens_seen": 155848060, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14013672, + "step": 7203, + "time_per_iteration": 3.685633897781372 + }, + { + "auxiliary_loss_clip": 0.01122086, + "auxiliary_loss_mlp": 0.01038353, + "balance_loss_clip": 1.03865433, + "balance_loss_mlp": 1.02323699, + "epoch": 0.8662297841640113, + "flos": 18224163227520.0, + "grad_norm": 1.7678164351223862, + "language_loss": 0.63895261, + "learning_rate": 1.8466874369694074e-07, + "loss": 0.66055703, + "num_input_tokens_seen": 155865755, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.15124512, + "step": 7204, + "time_per_iteration": 2.796898365020752 + }, + { + "auxiliary_loss_clip": 0.01121897, + "auxiliary_loss_mlp": 0.01040889, + "balance_loss_clip": 1.03900766, + "balance_loss_mlp": 1.02735841, + "epoch": 0.8663500270546504, + "flos": 16362159027840.0, + "grad_norm": 2.550320600390742, + "language_loss": 0.70164543, + "learning_rate": 1.843419516208542e-07, + "loss": 0.72327328, + "num_input_tokens_seen": 155882680, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.13543701, + "step": 7205, + "time_per_iteration": 2.807605028152466 + }, + { + "auxiliary_loss_clip": 0.01128721, + "auxiliary_loss_mlp": 0.01043645, + "balance_loss_clip": 1.04064298, + "balance_loss_mlp": 1.02699161, + "epoch": 0.8664702699452895, + "flos": 17894431353600.0, + "grad_norm": 1.9491159119208783, + "language_loss": 0.79613566, + "learning_rate": 1.8401543497865047e-07, + "loss": 0.81785929, + "num_input_tokens_seen": 155900680, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.16650391, + "step": 7206, + "time_per_iteration": 2.894751787185669 + }, + { + "auxiliary_loss_clip": 0.01126024, + "auxiliary_loss_mlp": 0.01039365, + "balance_loss_clip": 1.04090059, + "balance_loss_mlp": 1.02341461, + "epoch": 0.8665905128359286, + "flos": 30736373794560.0, + "grad_norm": 2.2953857107710554, + "language_loss": 0.63722324, + "learning_rate": 1.836891938198608e-07, + "loss": 0.65887713, + "num_input_tokens_seen": 155921105, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.15942383, + "step": 7207, + "time_per_iteration": 2.898409366607666 + }, + { + "auxiliary_loss_clip": 0.01126347, + "auxiliary_loss_mlp": 0.0103725, + "balance_loss_clip": 1.0401113, + "balance_loss_mlp": 1.02081084, + "epoch": 0.8667107557265676, + "flos": 18656419495680.0, + "grad_norm": 2.035469722948994, + "language_loss": 0.71296883, + "learning_rate": 1.8336322819397677e-07, + "loss": 0.73460484, + "num_input_tokens_seen": 155938640, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.16424561, + "step": 7208, + "time_per_iteration": 3.591872453689575 + }, + { + "auxiliary_loss_clip": 0.01126178, + "auxiliary_loss_mlp": 0.01038342, + "balance_loss_clip": 1.03973722, + "balance_loss_mlp": 1.02186704, + "epoch": 0.8668309986172068, + "flos": 20083725302400.0, + "grad_norm": 1.783804917501763, + "language_loss": 0.62971056, + "learning_rate": 1.8303753815044654e-07, + "loss": 0.65135574, + "num_input_tokens_seen": 155957945, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.16491699, + "step": 7209, + "time_per_iteration": 2.809572696685791 + }, + { + "auxiliary_loss_clip": 0.01134687, + "auxiliary_loss_mlp": 0.01044569, + "balance_loss_clip": 1.04230344, + "balance_loss_mlp": 1.02686644, + "epoch": 0.8669512415078459, + "flos": 21615099788160.0, + "grad_norm": 2.166083441613613, + "language_loss": 0.71014524, + "learning_rate": 1.827121237386773e-07, + "loss": 0.73193771, + "num_input_tokens_seen": 155975390, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.17700195, + "step": 7210, + "time_per_iteration": 3.562234401702881 + }, + { + "auxiliary_loss_clip": 0.01126417, + "auxiliary_loss_mlp": 0.01037434, + "balance_loss_clip": 1.03947222, + "balance_loss_mlp": 1.02213907, + "epoch": 0.8670714843984849, + "flos": 17703601372800.0, + "grad_norm": 2.2763062297459475, + "language_loss": 0.75269175, + "learning_rate": 1.8238698500803374e-07, + "loss": 0.77433026, + "num_input_tokens_seen": 155988155, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.1529541, + "step": 7211, + "time_per_iteration": 2.729393243789673 + }, + { + "auxiliary_loss_clip": 0.01032277, + "auxiliary_loss_mlp": 0.01002424, + "balance_loss_clip": 1.00460351, + "balance_loss_mlp": 1.00118434, + "epoch": 0.8671917272891241, + "flos": 60705483125760.0, + "grad_norm": 0.7142688657388394, + "language_loss": 0.56231731, + "learning_rate": 1.820621220078391e-07, + "loss": 0.58266431, + "num_input_tokens_seen": 156052065, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01239014, + "step": 7212, + "time_per_iteration": 3.3205511569976807 + }, + { + "auxiliary_loss_clip": 0.01124236, + "auxiliary_loss_mlp": 0.01041096, + "balance_loss_clip": 1.03884721, + "balance_loss_mlp": 1.02578926, + "epoch": 0.8673119701797631, + "flos": 20451881750400.0, + "grad_norm": 1.848676143109424, + "language_loss": 0.68112266, + "learning_rate": 1.8173753478737553e-07, + "loss": 0.70277596, + "num_input_tokens_seen": 156072500, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.15289307, + "step": 7213, + "time_per_iteration": 2.747934103012085 + }, + { + "auxiliary_loss_clip": 0.01128829, + "auxiliary_loss_mlp": 0.01037867, + "balance_loss_clip": 1.04199767, + "balance_loss_mlp": 1.022668, + "epoch": 0.8674322130704022, + "flos": 19647410797440.0, + "grad_norm": 2.355502539090305, + "language_loss": 0.79881299, + "learning_rate": 1.8141322339588205e-07, + "loss": 0.82047999, + "num_input_tokens_seen": 156089840, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.1519165, + "step": 7214, + "time_per_iteration": 3.6343228816986084 + }, + { + "auxiliary_loss_clip": 0.01125495, + "auxiliary_loss_mlp": 0.01037107, + "balance_loss_clip": 1.04070568, + "balance_loss_mlp": 1.02183628, + "epoch": 0.8675524559610414, + "flos": 26025001367040.0, + "grad_norm": 1.8404908267108004, + "language_loss": 0.70189494, + "learning_rate": 1.810891878825569e-07, + "loss": 0.72352093, + "num_input_tokens_seen": 156109815, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15270996, + "step": 7215, + "time_per_iteration": 2.741553783416748 + }, + { + "auxiliary_loss_clip": 0.01122307, + "auxiliary_loss_mlp": 0.01037658, + "balance_loss_clip": 1.03857183, + "balance_loss_mlp": 1.02264988, + "epoch": 0.8676726988516804, + "flos": 15049444584960.0, + "grad_norm": 1.8845368361126966, + "language_loss": 0.72305262, + "learning_rate": 1.8076542829655561e-07, + "loss": 0.74465233, + "num_input_tokens_seen": 156128620, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.15002441, + "step": 7216, + "time_per_iteration": 2.794247627258301 + }, + { + "auxiliary_loss_clip": 0.01129403, + "auxiliary_loss_mlp": 0.01042811, + "balance_loss_clip": 1.0403564, + "balance_loss_mlp": 1.02682495, + "epoch": 0.8677929417423195, + "flos": 16288111140480.0, + "grad_norm": 1.9096280611670462, + "language_loss": 0.79430622, + "learning_rate": 1.8044194468699203e-07, + "loss": 0.81602836, + "num_input_tokens_seen": 156145930, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.15979004, + "step": 7217, + "time_per_iteration": 2.733959436416626 + }, + { + "auxiliary_loss_clip": 0.01122774, + "auxiliary_loss_mlp": 0.01038388, + "balance_loss_clip": 1.03987503, + "balance_loss_mlp": 1.02211618, + "epoch": 0.8679131846329585, + "flos": 18844160906880.0, + "grad_norm": 3.1418680897943942, + "language_loss": 0.75997305, + "learning_rate": 1.8011873710293912e-07, + "loss": 0.78158468, + "num_input_tokens_seen": 156164435, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.16271973, + "step": 7218, + "time_per_iteration": 2.8022422790527344 + }, + { + "auxiliary_loss_clip": 0.01124278, + "auxiliary_loss_mlp": 0.01037189, + "balance_loss_clip": 1.03942955, + "balance_loss_mlp": 1.02157307, + "epoch": 0.8680334275235977, + "flos": 33620718890880.0, + "grad_norm": 1.7603073567632594, + "language_loss": 0.69588757, + "learning_rate": 1.7979580559342677e-07, + "loss": 0.71750224, + "num_input_tokens_seen": 156185165, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.15625, + "step": 7219, + "time_per_iteration": 2.8885130882263184 + }, + { + "auxiliary_loss_clip": 0.01126168, + "auxiliary_loss_mlp": 0.01040042, + "balance_loss_clip": 1.04108, + "balance_loss_mlp": 1.02417529, + "epoch": 0.8681536704142367, + "flos": 24681152810880.0, + "grad_norm": 1.587257358240235, + "language_loss": 0.6654135, + "learning_rate": 1.7947315020744358e-07, + "loss": 0.68707561, + "num_input_tokens_seen": 156206260, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15869141, + "step": 7220, + "time_per_iteration": 2.8438363075256348 + }, + { + "auxiliary_loss_clip": 0.01122761, + "auxiliary_loss_mlp": 0.01033075, + "balance_loss_clip": 1.03875828, + "balance_loss_mlp": 1.01731563, + "epoch": 0.8682739133048758, + "flos": 20011042131840.0, + "grad_norm": 1.8122703322182527, + "language_loss": 0.80328804, + "learning_rate": 1.7915077099393594e-07, + "loss": 0.82484639, + "num_input_tokens_seen": 156222860, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.15759277, + "step": 7221, + "time_per_iteration": 2.8792812824249268 + }, + { + "auxiliary_loss_clip": 0.01124798, + "auxiliary_loss_mlp": 0.0103612, + "balance_loss_clip": 1.03930402, + "balance_loss_mlp": 1.02012181, + "epoch": 0.868394156195515, + "flos": 16654759217280.0, + "grad_norm": 1.87010575581786, + "language_loss": 0.73336565, + "learning_rate": 1.788286680018083e-07, + "loss": 0.75497484, + "num_input_tokens_seen": 156241570, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15991211, + "step": 7222, + "time_per_iteration": 2.769538402557373 + }, + { + "auxiliary_loss_clip": 0.01124064, + "auxiliary_loss_mlp": 0.01034284, + "balance_loss_clip": 1.04028368, + "balance_loss_mlp": 1.01965714, + "epoch": 0.868514399086154, + "flos": 28001381448960.0, + "grad_norm": 1.5191755973744716, + "language_loss": 0.72107357, + "learning_rate": 1.7850684127992443e-07, + "loss": 0.74265707, + "num_input_tokens_seen": 156261315, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.14624023, + "step": 7223, + "time_per_iteration": 2.856372117996216 + }, + { + "auxiliary_loss_clip": 0.01125979, + "auxiliary_loss_mlp": 0.01039429, + "balance_loss_clip": 1.04175735, + "balance_loss_mlp": 1.02542186, + "epoch": 0.8686346419767931, + "flos": 20084587228800.0, + "grad_norm": 1.6508696581759144, + "language_loss": 0.70390189, + "learning_rate": 1.7818529087710378e-07, + "loss": 0.72555602, + "num_input_tokens_seen": 156281670, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.13995361, + "step": 7224, + "time_per_iteration": 2.854635238647461 + }, + { + "auxiliary_loss_clip": 0.01123339, + "auxiliary_loss_mlp": 0.01039869, + "balance_loss_clip": 1.03974938, + "balance_loss_mlp": 1.02506351, + "epoch": 0.8687548848674322, + "flos": 18223516782720.0, + "grad_norm": 1.6551789309169962, + "language_loss": 0.84096098, + "learning_rate": 1.7786401684212637e-07, + "loss": 0.86259305, + "num_input_tokens_seen": 156300500, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14794922, + "step": 7225, + "time_per_iteration": 2.764204740524292 + }, + { + "auxiliary_loss_clip": 0.01032742, + "auxiliary_loss_mlp": 0.01002229, + "balance_loss_clip": 1.00501609, + "balance_loss_mlp": 1.00098884, + "epoch": 0.8688751277580713, + "flos": 70457885049600.0, + "grad_norm": 0.7794637935847101, + "language_loss": 0.55977565, + "learning_rate": 1.7754301922372883e-07, + "loss": 0.58012533, + "num_input_tokens_seen": 156350145, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01239014, + "step": 7226, + "time_per_iteration": 3.1580514907836914 + }, + { + "auxiliary_loss_clip": 0.01128307, + "auxiliary_loss_mlp": 0.01042888, + "balance_loss_clip": 1.04022133, + "balance_loss_mlp": 1.02744484, + "epoch": 0.8689953706487104, + "flos": 26906788344960.0, + "grad_norm": 2.2692132704303676, + "language_loss": 0.81221616, + "learning_rate": 1.7722229807060617e-07, + "loss": 0.83392811, + "num_input_tokens_seen": 156368725, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.15423584, + "step": 7227, + "time_per_iteration": 2.959364891052246 + }, + { + "auxiliary_loss_clip": 0.01121958, + "auxiliary_loss_mlp": 0.01034166, + "balance_loss_clip": 1.03838491, + "balance_loss_mlp": 1.02013516, + "epoch": 0.8691156135393495, + "flos": 34637385438720.0, + "grad_norm": 2.4127100625970685, + "language_loss": 0.81898689, + "learning_rate": 1.7690185343141172e-07, + "loss": 0.84054816, + "num_input_tokens_seen": 156388640, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.14031982, + "step": 7228, + "time_per_iteration": 2.8405110836029053 + }, + { + "auxiliary_loss_clip": 0.0112196, + "auxiliary_loss_mlp": 0.01034715, + "balance_loss_clip": 1.03882504, + "balance_loss_mlp": 1.02055299, + "epoch": 0.8692358564299886, + "flos": 18989814556800.0, + "grad_norm": 2.7335530688885807, + "language_loss": 0.69811088, + "learning_rate": 1.7658168535475615e-07, + "loss": 0.71967757, + "num_input_tokens_seen": 156406425, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.1416626, + "step": 7229, + "time_per_iteration": 2.689929246902466 + }, + { + "auxiliary_loss_clip": 0.01126081, + "auxiliary_loss_mlp": 0.0104284, + "balance_loss_clip": 1.04031861, + "balance_loss_mlp": 1.0266273, + "epoch": 0.8693560993206276, + "flos": 30370839039360.0, + "grad_norm": 1.6161618103431612, + "language_loss": 0.64500004, + "learning_rate": 1.7626179388920948e-07, + "loss": 0.66668928, + "num_input_tokens_seen": 156427705, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.16210938, + "step": 7230, + "time_per_iteration": 2.8245956897735596 + }, + { + "auxiliary_loss_clip": 0.01125848, + "auxiliary_loss_mlp": 0.01041396, + "balance_loss_clip": 1.04109263, + "balance_loss_mlp": 1.02623248, + "epoch": 0.8694763422112668, + "flos": 27200430028800.0, + "grad_norm": 1.5988605443714687, + "language_loss": 0.80687839, + "learning_rate": 1.7594217908329866e-07, + "loss": 0.82855082, + "num_input_tokens_seen": 156449890, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.1517334, + "step": 7231, + "time_per_iteration": 2.780198812484741 + }, + { + "auxiliary_loss_clip": 0.01123732, + "auxiliary_loss_mlp": 0.01037868, + "balance_loss_clip": 1.04084134, + "balance_loss_mlp": 1.02429008, + "epoch": 0.8695965851019059, + "flos": 26139161767680.0, + "grad_norm": 1.7260518847534403, + "language_loss": 0.74092168, + "learning_rate": 1.7562284098550895e-07, + "loss": 0.7625376, + "num_input_tokens_seen": 156469600, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.13574219, + "step": 7232, + "time_per_iteration": 3.507439613342285 + }, + { + "auxiliary_loss_clip": 0.01032416, + "auxiliary_loss_mlp": 0.01002065, + "balance_loss_clip": 1.00474215, + "balance_loss_mlp": 1.0007422, + "epoch": 0.8697168279925449, + "flos": 67332616456320.0, + "grad_norm": 0.8351546465587956, + "language_loss": 0.62238622, + "learning_rate": 1.753037796442838e-07, + "loss": 0.64273107, + "num_input_tokens_seen": 156529040, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01324463, + "step": 7233, + "time_per_iteration": 3.279496908187866 + }, + { + "auxiliary_loss_clip": 0.01126387, + "auxiliary_loss_mlp": 0.01047158, + "balance_loss_clip": 1.03934658, + "balance_loss_mlp": 1.03118443, + "epoch": 0.8698370708831841, + "flos": 19718693337600.0, + "grad_norm": 2.037099390479271, + "language_loss": 0.75293243, + "learning_rate": 1.74984995108024e-07, + "loss": 0.77466786, + "num_input_tokens_seen": 156546970, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.15979004, + "step": 7234, + "time_per_iteration": 2.7154760360717773 + }, + { + "auxiliary_loss_clip": 0.0112579, + "auxiliary_loss_mlp": 0.01037178, + "balance_loss_clip": 1.04029274, + "balance_loss_mlp": 1.02191913, + "epoch": 0.8699573137738231, + "flos": 12859971068160.0, + "grad_norm": 2.0295676312859654, + "language_loss": 0.83497679, + "learning_rate": 1.7466648742508981e-07, + "loss": 0.85660648, + "num_input_tokens_seen": 156563155, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15252686, + "step": 7235, + "time_per_iteration": 2.75315260887146 + }, + { + "auxiliary_loss_clip": 0.01126366, + "auxiliary_loss_mlp": 0.01032244, + "balance_loss_clip": 1.03971624, + "balance_loss_mlp": 1.01791477, + "epoch": 0.8700775566644622, + "flos": 17420733768960.0, + "grad_norm": 1.7321217318964248, + "language_loss": 0.84454036, + "learning_rate": 1.7434825664379837e-07, + "loss": 0.86612648, + "num_input_tokens_seen": 156581660, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.14318848, + "step": 7236, + "time_per_iteration": 2.74040150642395 + }, + { + "auxiliary_loss_clip": 0.01129499, + "auxiliary_loss_mlp": 0.01040876, + "balance_loss_clip": 1.0412848, + "balance_loss_mlp": 1.02525926, + "epoch": 0.8701977995551013, + "flos": 13735221770880.0, + "grad_norm": 2.3266814168991417, + "language_loss": 0.86510307, + "learning_rate": 1.740303028124246e-07, + "loss": 0.88680685, + "num_input_tokens_seen": 156597720, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15612793, + "step": 7237, + "time_per_iteration": 3.6377711296081543 + }, + { + "auxiliary_loss_clip": 0.01124651, + "auxiliary_loss_mlp": 0.01033356, + "balance_loss_clip": 1.04051697, + "balance_loss_mlp": 1.01849031, + "epoch": 0.8703180424457404, + "flos": 30555707362560.0, + "grad_norm": 1.8098710883080311, + "language_loss": 0.75652885, + "learning_rate": 1.7371262597920212e-07, + "loss": 0.77810895, + "num_input_tokens_seen": 156619780, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.14868164, + "step": 7238, + "time_per_iteration": 2.8776280879974365 + }, + { + "auxiliary_loss_clip": 0.01123841, + "auxiliary_loss_mlp": 0.01039447, + "balance_loss_clip": 1.04216051, + "balance_loss_mlp": 1.02439046, + "epoch": 0.8704382853363795, + "flos": 19608986223360.0, + "grad_norm": 1.5031936391201464, + "language_loss": 0.76405585, + "learning_rate": 1.7339522619232195e-07, + "loss": 0.78568876, + "num_input_tokens_seen": 156638160, + "router_z_loss_clip": 0.81591797, + "router_z_loss_mlp": 0.1505127, + "step": 7239, + "time_per_iteration": 3.5561418533325195 + }, + { + "auxiliary_loss_clip": 0.01125876, + "auxiliary_loss_mlp": 0.01039044, + "balance_loss_clip": 1.03852201, + "balance_loss_mlp": 1.02260518, + "epoch": 0.8705585282270186, + "flos": 26613900846720.0, + "grad_norm": 1.7772285040943199, + "language_loss": 0.75615501, + "learning_rate": 1.730781034999338e-07, + "loss": 0.77780426, + "num_input_tokens_seen": 156659740, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.16455078, + "step": 7240, + "time_per_iteration": 2.825897216796875 + }, + { + "auxiliary_loss_clip": 0.0112156, + "auxiliary_loss_mlp": 0.01035862, + "balance_loss_clip": 1.04047537, + "balance_loss_mlp": 1.02077055, + "epoch": 0.8706787711176577, + "flos": 34090465979520.0, + "grad_norm": 1.6729825225619948, + "language_loss": 0.73411465, + "learning_rate": 1.7276125795014497e-07, + "loss": 0.75568891, + "num_input_tokens_seen": 156678190, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.15100098, + "step": 7241, + "time_per_iteration": 2.863035202026367 + }, + { + "auxiliary_loss_clip": 0.01131238, + "auxiliary_loss_mlp": 0.01032952, + "balance_loss_clip": 1.04249144, + "balance_loss_mlp": 1.01757443, + "epoch": 0.8707990140082967, + "flos": 14611513968000.0, + "grad_norm": 1.8732436910917216, + "language_loss": 0.67413384, + "learning_rate": 1.7244468959102054e-07, + "loss": 0.69577575, + "num_input_tokens_seen": 156695245, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.15380859, + "step": 7242, + "time_per_iteration": 2.75791597366333 + }, + { + "auxiliary_loss_clip": 0.0112583, + "auxiliary_loss_mlp": 0.01034664, + "balance_loss_clip": 1.0399965, + "balance_loss_mlp": 1.0192976, + "epoch": 0.8709192568989359, + "flos": 20084156265600.0, + "grad_norm": 2.0556328936096055, + "language_loss": 0.85376012, + "learning_rate": 1.7212839847058348e-07, + "loss": 0.87536508, + "num_input_tokens_seen": 156710375, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15368652, + "step": 7243, + "time_per_iteration": 3.7335362434387207 + }, + { + "auxiliary_loss_clip": 0.01125656, + "auxiliary_loss_mlp": 0.01040289, + "balance_loss_clip": 1.03998017, + "balance_loss_mlp": 1.02493525, + "epoch": 0.871039499789575, + "flos": 16727083251840.0, + "grad_norm": 2.292507545401771, + "language_loss": 0.73710823, + "learning_rate": 1.718123846368147e-07, + "loss": 0.75876772, + "num_input_tokens_seen": 156729420, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15356445, + "step": 7244, + "time_per_iteration": 2.784287452697754 + }, + { + "auxiliary_loss_clip": 0.01122782, + "auxiliary_loss_mlp": 0.01039903, + "balance_loss_clip": 1.04167891, + "balance_loss_mlp": 1.02510953, + "epoch": 0.871159742680214, + "flos": 21068790860160.0, + "grad_norm": 2.089955014842227, + "language_loss": 0.71284127, + "learning_rate": 1.714966481376543e-07, + "loss": 0.7344681, + "num_input_tokens_seen": 156746100, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.14788818, + "step": 7245, + "time_per_iteration": 2.8404855728149414 + }, + { + "auxiliary_loss_clip": 0.01122789, + "auxiliary_loss_mlp": 0.01035223, + "balance_loss_clip": 1.03738999, + "balance_loss_mlp": 1.019714, + "epoch": 0.8712799855708532, + "flos": 28256526731520.0, + "grad_norm": 2.042986201698563, + "language_loss": 0.82896554, + "learning_rate": 1.7118118902099797e-07, + "loss": 0.85054564, + "num_input_tokens_seen": 156764185, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.15515137, + "step": 7246, + "time_per_iteration": 2.8591225147247314 + }, + { + "auxiliary_loss_clip": 0.01125299, + "auxiliary_loss_mlp": 0.01041179, + "balance_loss_clip": 1.04049134, + "balance_loss_mlp": 1.02577162, + "epoch": 0.8714002284614922, + "flos": 22236677665920.0, + "grad_norm": 1.58356861629446, + "language_loss": 0.8061552, + "learning_rate": 1.7086600733470146e-07, + "loss": 0.82782006, + "num_input_tokens_seen": 156784855, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.1539917, + "step": 7247, + "time_per_iteration": 2.8457531929016113 + }, + { + "auxiliary_loss_clip": 0.01118828, + "auxiliary_loss_mlp": 0.01032698, + "balance_loss_clip": 1.03737044, + "balance_loss_mlp": 1.01879251, + "epoch": 0.8715204713521313, + "flos": 21431919404160.0, + "grad_norm": 1.9238641346884366, + "language_loss": 0.76788425, + "learning_rate": 1.7055110312657738e-07, + "loss": 0.7893995, + "num_input_tokens_seen": 156804350, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.13903809, + "step": 7248, + "time_per_iteration": 2.855391502380371 + }, + { + "auxiliary_loss_clip": 0.0112784, + "auxiliary_loss_mlp": 0.01043229, + "balance_loss_clip": 1.04139102, + "balance_loss_mlp": 1.02792263, + "epoch": 0.8716407142427703, + "flos": 23440439180160.0, + "grad_norm": 2.0499696636378992, + "language_loss": 0.74299842, + "learning_rate": 1.702364764443962e-07, + "loss": 0.76470912, + "num_input_tokens_seen": 156823425, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15307617, + "step": 7249, + "time_per_iteration": 2.7257306575775146 + }, + { + "auxiliary_loss_clip": 0.01126201, + "auxiliary_loss_mlp": 0.01041136, + "balance_loss_clip": 1.0410701, + "balance_loss_mlp": 1.0249356, + "epoch": 0.8717609571334095, + "flos": 27958683156480.0, + "grad_norm": 2.143405749125772, + "language_loss": 0.72553515, + "learning_rate": 1.6992212733588685e-07, + "loss": 0.74720854, + "num_input_tokens_seen": 156843090, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.16204834, + "step": 7250, + "time_per_iteration": 2.809964656829834 + }, + { + "auxiliary_loss_clip": 0.01123314, + "auxiliary_loss_mlp": 0.01038636, + "balance_loss_clip": 1.03940094, + "balance_loss_mlp": 1.02369952, + "epoch": 0.8718812000240486, + "flos": 25479482538240.0, + "grad_norm": 2.5180008984267754, + "language_loss": 0.74828929, + "learning_rate": 1.6960805584873538e-07, + "loss": 0.76990879, + "num_input_tokens_seen": 156861090, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.14941406, + "step": 7251, + "time_per_iteration": 2.8173699378967285 + }, + { + "auxiliary_loss_clip": 0.01123949, + "auxiliary_loss_mlp": 0.01037235, + "balance_loss_clip": 1.03796387, + "balance_loss_mlp": 1.02291238, + "epoch": 0.8720014429146876, + "flos": 23403056100480.0, + "grad_norm": 1.5358669583478142, + "language_loss": 0.77978468, + "learning_rate": 1.6929426203058684e-07, + "loss": 0.80139655, + "num_input_tokens_seen": 156881515, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.14312744, + "step": 7252, + "time_per_iteration": 2.7575814723968506 + }, + { + "auxiliary_loss_clip": 0.01128584, + "auxiliary_loss_mlp": 0.01038743, + "balance_loss_clip": 1.03953552, + "balance_loss_mlp": 1.02124262, + "epoch": 0.8721216858053268, + "flos": 24352821567360.0, + "grad_norm": 1.941720659203584, + "language_loss": 0.8012675, + "learning_rate": 1.689807459290431e-07, + "loss": 0.82294077, + "num_input_tokens_seen": 156900170, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.17504883, + "step": 7253, + "time_per_iteration": 2.7558085918426514 + }, + { + "auxiliary_loss_clip": 0.01125284, + "auxiliary_loss_mlp": 0.01033602, + "balance_loss_clip": 1.04127991, + "balance_loss_mlp": 1.01868856, + "epoch": 0.8722419286959658, + "flos": 33869687034240.0, + "grad_norm": 2.703215416025533, + "language_loss": 0.7064743, + "learning_rate": 1.6866750759166437e-07, + "loss": 0.72806323, + "num_input_tokens_seen": 156920150, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14910889, + "step": 7254, + "time_per_iteration": 2.8253228664398193 + }, + { + "auxiliary_loss_clip": 0.01127717, + "auxiliary_loss_mlp": 0.01036962, + "balance_loss_clip": 1.03898656, + "balance_loss_mlp": 1.02242994, + "epoch": 0.8723621715866049, + "flos": 18369385914240.0, + "grad_norm": 2.7096511920220996, + "language_loss": 0.77164274, + "learning_rate": 1.6835454706596865e-07, + "loss": 0.79328954, + "num_input_tokens_seen": 156937980, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.14538574, + "step": 7255, + "time_per_iteration": 2.7641031742095947 + }, + { + "auxiliary_loss_clip": 0.01127987, + "auxiliary_loss_mlp": 0.01041923, + "balance_loss_clip": 1.04162717, + "balance_loss_mlp": 1.02647305, + "epoch": 0.8724824144772441, + "flos": 22013348855040.0, + "grad_norm": 1.974032049459894, + "language_loss": 0.73753405, + "learning_rate": 1.680418643994317e-07, + "loss": 0.75923312, + "num_input_tokens_seen": 156956550, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.15429688, + "step": 7256, + "time_per_iteration": 2.719364881515503 + }, + { + "auxiliary_loss_clip": 0.01031968, + "auxiliary_loss_mlp": 0.01001696, + "balance_loss_clip": 1.00419736, + "balance_loss_mlp": 1.00050378, + "epoch": 0.8726026573678831, + "flos": 66698720213760.0, + "grad_norm": 0.8827336215189238, + "language_loss": 0.64508885, + "learning_rate": 1.6772945963948738e-07, + "loss": 0.66542554, + "num_input_tokens_seen": 157014715, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01190186, + "step": 7257, + "time_per_iteration": 3.334503412246704 + }, + { + "auxiliary_loss_clip": 0.01127628, + "auxiliary_loss_mlp": 0.01038453, + "balance_loss_clip": 1.04077244, + "balance_loss_mlp": 1.02184749, + "epoch": 0.8727229002585222, + "flos": 13370908078080.0, + "grad_norm": 2.2514743950763227, + "language_loss": 0.77879095, + "learning_rate": 1.6741733283352733e-07, + "loss": 0.80045176, + "num_input_tokens_seen": 157032320, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.16601562, + "step": 7258, + "time_per_iteration": 2.727776050567627 + }, + { + "auxiliary_loss_clip": 0.01125684, + "auxiliary_loss_mlp": 0.01036033, + "balance_loss_clip": 1.03807712, + "balance_loss_mlp": 1.01999927, + "epoch": 0.8728431431491613, + "flos": 21796987282560.0, + "grad_norm": 1.505715379503929, + "language_loss": 0.83828604, + "learning_rate": 1.6710548402890102e-07, + "loss": 0.85990322, + "num_input_tokens_seen": 157052845, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16027832, + "step": 7259, + "time_per_iteration": 2.84948468208313 + }, + { + "auxiliary_loss_clip": 0.01130342, + "auxiliary_loss_mlp": 0.01038758, + "balance_loss_clip": 1.04053378, + "balance_loss_mlp": 1.02262342, + "epoch": 0.8729633860398004, + "flos": 36173823742080.0, + "grad_norm": 2.021281610282088, + "language_loss": 0.66558403, + "learning_rate": 1.6679391327291527e-07, + "loss": 0.68727505, + "num_input_tokens_seen": 157074050, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.16131592, + "step": 7260, + "time_per_iteration": 2.879256010055542 + }, + { + "auxiliary_loss_clip": 0.01123402, + "auxiliary_loss_mlp": 0.01036938, + "balance_loss_clip": 1.0402106, + "balance_loss_mlp": 1.02190566, + "epoch": 0.8730836289304394, + "flos": 16359680989440.0, + "grad_norm": 2.5012929798178813, + "language_loss": 0.67716902, + "learning_rate": 1.6648262061283492e-07, + "loss": 0.69877243, + "num_input_tokens_seen": 157089350, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.15039062, + "step": 7261, + "time_per_iteration": 2.7613658905029297 + }, + { + "auxiliary_loss_clip": 0.01123047, + "auxiliary_loss_mlp": 0.01031589, + "balance_loss_clip": 1.03863537, + "balance_loss_mlp": 1.01776683, + "epoch": 0.8732038718210786, + "flos": 21215126868480.0, + "grad_norm": 2.0773021144027846, + "language_loss": 0.73497057, + "learning_rate": 1.6617160609588353e-07, + "loss": 0.75651693, + "num_input_tokens_seen": 157108525, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.13824463, + "step": 7262, + "time_per_iteration": 3.616903781890869 + }, + { + "auxiliary_loss_clip": 0.01127381, + "auxiliary_loss_mlp": 0.01039051, + "balance_loss_clip": 1.04003012, + "balance_loss_mlp": 1.02479374, + "epoch": 0.8733241147117177, + "flos": 16610696208000.0, + "grad_norm": 2.305365558316713, + "language_loss": 0.71971524, + "learning_rate": 1.6586086976924163e-07, + "loss": 0.74137956, + "num_input_tokens_seen": 157124025, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.14257812, + "step": 7263, + "time_per_iteration": 2.7672483921051025 + }, + { + "auxiliary_loss_clip": 0.01121479, + "auxiliary_loss_mlp": 0.01034565, + "balance_loss_clip": 1.03869081, + "balance_loss_mlp": 1.01901972, + "epoch": 0.8734443576023567, + "flos": 20193935207040.0, + "grad_norm": 1.7148812199372343, + "language_loss": 0.78162134, + "learning_rate": 1.6555041168004747e-07, + "loss": 0.80318177, + "num_input_tokens_seen": 157143345, + "router_z_loss_clip": 0.828125, + "router_z_loss_mlp": 0.15563965, + "step": 7264, + "time_per_iteration": 2.85842227935791 + }, + { + "auxiliary_loss_clip": 0.01122243, + "auxiliary_loss_mlp": 0.01032508, + "balance_loss_clip": 1.040048, + "balance_loss_mlp": 1.01918006, + "epoch": 0.8735646004929959, + "flos": 18041162411520.0, + "grad_norm": 1.6539146356082346, + "language_loss": 0.68980181, + "learning_rate": 1.6524023187539715e-07, + "loss": 0.71134931, + "num_input_tokens_seen": 157161630, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.13323975, + "step": 7265, + "time_per_iteration": 2.815873146057129 + }, + { + "auxiliary_loss_clip": 0.01130359, + "auxiliary_loss_mlp": 0.010408, + "balance_loss_clip": 1.04320824, + "balance_loss_mlp": 1.025195, + "epoch": 0.873684843383635, + "flos": 20262344659200.0, + "grad_norm": 1.7609919136727534, + "language_loss": 0.74543577, + "learning_rate": 1.649303304023446e-07, + "loss": 0.7671473, + "num_input_tokens_seen": 157181385, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15588379, + "step": 7266, + "time_per_iteration": 3.4990615844726562 + }, + { + "auxiliary_loss_clip": 0.01121586, + "auxiliary_loss_mlp": 0.01039227, + "balance_loss_clip": 1.03857076, + "balance_loss_mlp": 1.02485013, + "epoch": 0.873805086274274, + "flos": 16947287579520.0, + "grad_norm": 1.6961363255184514, + "language_loss": 0.7874549, + "learning_rate": 1.6462070730790246e-07, + "loss": 0.80906302, + "num_input_tokens_seen": 157200545, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.14379883, + "step": 7267, + "time_per_iteration": 2.804785966873169 + }, + { + "auxiliary_loss_clip": 0.01122554, + "auxiliary_loss_mlp": 0.01037554, + "balance_loss_clip": 1.03959131, + "balance_loss_mlp": 1.02099562, + "epoch": 0.8739253291649132, + "flos": 18041270152320.0, + "grad_norm": 2.381794238940957, + "language_loss": 0.78713596, + "learning_rate": 1.6431136263903912e-07, + "loss": 0.80873704, + "num_input_tokens_seen": 157219545, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.16552734, + "step": 7268, + "time_per_iteration": 3.6958913803100586 + }, + { + "auxiliary_loss_clip": 0.0112585, + "auxiliary_loss_mlp": 0.01036367, + "balance_loss_clip": 1.03985381, + "balance_loss_mlp": 1.02103627, + "epoch": 0.8740455720555522, + "flos": 21325085377920.0, + "grad_norm": 2.5977201761984583, + "language_loss": 0.73913723, + "learning_rate": 1.6400229644268282e-07, + "loss": 0.76075935, + "num_input_tokens_seen": 157237900, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15319824, + "step": 7269, + "time_per_iteration": 2.8553502559661865 + }, + { + "auxiliary_loss_clip": 0.01125649, + "auxiliary_loss_mlp": 0.01041864, + "balance_loss_clip": 1.03995109, + "balance_loss_mlp": 1.0263195, + "epoch": 0.8741658149461913, + "flos": 15158684822400.0, + "grad_norm": 2.1062145884000123, + "language_loss": 0.81028008, + "learning_rate": 1.6369350876571852e-07, + "loss": 0.83195519, + "num_input_tokens_seen": 157256055, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.15539551, + "step": 7270, + "time_per_iteration": 2.744082450866699 + }, + { + "auxiliary_loss_clip": 0.01125311, + "auxiliary_loss_mlp": 0.01035068, + "balance_loss_clip": 1.0400573, + "balance_loss_mlp": 1.01989269, + "epoch": 0.8742860578368304, + "flos": 23039855729280.0, + "grad_norm": 1.8623335345495935, + "language_loss": 0.81456393, + "learning_rate": 1.6338499965498874e-07, + "loss": 0.83616775, + "num_input_tokens_seen": 157274785, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.15161133, + "step": 7271, + "time_per_iteration": 2.834890842437744 + }, + { + "auxiliary_loss_clip": 0.01127684, + "auxiliary_loss_mlp": 0.01036341, + "balance_loss_clip": 1.04028714, + "balance_loss_mlp": 1.02179706, + "epoch": 0.8744063007274695, + "flos": 28145347159680.0, + "grad_norm": 1.8521280570296224, + "language_loss": 0.77513039, + "learning_rate": 1.630767691572943e-07, + "loss": 0.79677057, + "num_input_tokens_seen": 157294805, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.14538574, + "step": 7272, + "time_per_iteration": 3.7325620651245117 + }, + { + "auxiliary_loss_clip": 0.01032219, + "auxiliary_loss_mlp": 0.00999834, + "balance_loss_clip": 1.00462127, + "balance_loss_mlp": 0.9985823, + "epoch": 0.8745265436181086, + "flos": 64034076654720.0, + "grad_norm": 0.7557340830894698, + "language_loss": 0.53485489, + "learning_rate": 1.6276881731939306e-07, + "loss": 0.55517542, + "num_input_tokens_seen": 157356695, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01251221, + "step": 7273, + "time_per_iteration": 3.3211193084716797 + }, + { + "auxiliary_loss_clip": 0.01123178, + "auxiliary_loss_mlp": 0.01036809, + "balance_loss_clip": 1.03926253, + "balance_loss_mlp": 1.02196109, + "epoch": 0.8746467865087477, + "flos": 28658618553600.0, + "grad_norm": 2.0390296558702032, + "language_loss": 0.75360894, + "learning_rate": 1.6246114418800193e-07, + "loss": 0.77520883, + "num_input_tokens_seen": 157376975, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.14862061, + "step": 7274, + "time_per_iteration": 2.763253688812256 + }, + { + "auxiliary_loss_clip": 0.01127825, + "auxiliary_loss_mlp": 0.01034637, + "balance_loss_clip": 1.04112387, + "balance_loss_mlp": 1.01977181, + "epoch": 0.8747670293993868, + "flos": 23985850268160.0, + "grad_norm": 1.6529505790439034, + "language_loss": 0.76801878, + "learning_rate": 1.6215374980979423e-07, + "loss": 0.78964341, + "num_input_tokens_seen": 157397385, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.14868164, + "step": 7275, + "time_per_iteration": 2.787991523742676 + }, + { + "auxiliary_loss_clip": 0.01123989, + "auxiliary_loss_mlp": 0.01031728, + "balance_loss_clip": 1.03990364, + "balance_loss_mlp": 1.01704764, + "epoch": 0.8748872722900258, + "flos": 45221624478720.0, + "grad_norm": 1.8322094091388381, + "language_loss": 0.68842465, + "learning_rate": 1.6184663423140133e-07, + "loss": 0.7099818, + "num_input_tokens_seen": 157417685, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.14697266, + "step": 7276, + "time_per_iteration": 2.9072940349578857 + }, + { + "auxiliary_loss_clip": 0.01127152, + "auxiliary_loss_mlp": 0.0103861, + "balance_loss_clip": 1.04319155, + "balance_loss_mlp": 1.02277887, + "epoch": 0.875007515180665, + "flos": 19754280737280.0, + "grad_norm": 1.8201816325912303, + "language_loss": 0.643767, + "learning_rate": 1.615397974994126e-07, + "loss": 0.66542459, + "num_input_tokens_seen": 157435490, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.1583252, + "step": 7277, + "time_per_iteration": 2.781968593597412 + }, + { + "auxiliary_loss_clip": 0.01121948, + "auxiliary_loss_mlp": 0.01036449, + "balance_loss_clip": 1.03949273, + "balance_loss_mlp": 1.02209568, + "epoch": 0.875127758071304, + "flos": 22710734386560.0, + "grad_norm": 1.5889884838521435, + "language_loss": 0.80608308, + "learning_rate": 1.6123323966037438e-07, + "loss": 0.82766706, + "num_input_tokens_seen": 157454010, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.14349365, + "step": 7278, + "time_per_iteration": 2.772714853286743 + }, + { + "auxiliary_loss_clip": 0.0112648, + "auxiliary_loss_mlp": 0.01031759, + "balance_loss_clip": 1.0405184, + "balance_loss_mlp": 1.01702523, + "epoch": 0.8752480009619431, + "flos": 23403846199680.0, + "grad_norm": 2.0897156591897397, + "language_loss": 0.78561604, + "learning_rate": 1.6092696076079216e-07, + "loss": 0.80719846, + "num_input_tokens_seen": 157472385, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.14746094, + "step": 7279, + "time_per_iteration": 2.9629158973693848 + }, + { + "auxiliary_loss_clip": 0.0112287, + "auxiliary_loss_mlp": 0.01037887, + "balance_loss_clip": 1.0404712, + "balance_loss_mlp": 1.02346849, + "epoch": 0.8753682438525822, + "flos": 26213101914240.0, + "grad_norm": 2.032903003163213, + "language_loss": 0.73639143, + "learning_rate": 1.6062096084712785e-07, + "loss": 0.75799894, + "num_input_tokens_seen": 157493735, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.14416504, + "step": 7280, + "time_per_iteration": 2.9363274574279785 + }, + { + "auxiliary_loss_clip": 0.01129094, + "auxiliary_loss_mlp": 0.01039274, + "balance_loss_clip": 1.04141891, + "balance_loss_mlp": 1.0233593, + "epoch": 0.8754884867432213, + "flos": 23326745656320.0, + "grad_norm": 1.7038639321606825, + "language_loss": 0.70777881, + "learning_rate": 1.6031523996580098e-07, + "loss": 0.7294625, + "num_input_tokens_seen": 157511295, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.15893555, + "step": 7281, + "time_per_iteration": 2.92087721824646 + }, + { + "auxiliary_loss_clip": 0.01127435, + "auxiliary_loss_mlp": 0.01041871, + "balance_loss_clip": 1.03948212, + "balance_loss_mlp": 1.02639794, + "epoch": 0.8756087296338604, + "flos": 12495226412160.0, + "grad_norm": 2.340334881735797, + "language_loss": 0.66051072, + "learning_rate": 1.6000979816318981e-07, + "loss": 0.68220377, + "num_input_tokens_seen": 157529760, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.15478516, + "step": 7282, + "time_per_iteration": 2.859527111053467 + }, + { + "auxiliary_loss_clip": 0.01125729, + "auxiliary_loss_mlp": 0.01037503, + "balance_loss_clip": 1.0415318, + "balance_loss_mlp": 1.02257752, + "epoch": 0.8757289725244994, + "flos": 18952898353920.0, + "grad_norm": 2.014521422461195, + "language_loss": 0.75229359, + "learning_rate": 1.5970463548562886e-07, + "loss": 0.7739259, + "num_input_tokens_seen": 157548915, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.14904785, + "step": 7283, + "time_per_iteration": 2.8365354537963867 + }, + { + "auxiliary_loss_clip": 0.01123939, + "auxiliary_loss_mlp": 0.01036864, + "balance_loss_clip": 1.03875971, + "balance_loss_mlp": 1.02202225, + "epoch": 0.8758492154151386, + "flos": 25265958140160.0, + "grad_norm": 1.6870519084311983, + "language_loss": 0.70824909, + "learning_rate": 1.5939975197941192e-07, + "loss": 0.72985709, + "num_input_tokens_seen": 157570570, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.14855957, + "step": 7284, + "time_per_iteration": 2.8951356410980225 + }, + { + "auxiliary_loss_clip": 0.01032277, + "auxiliary_loss_mlp": 0.00998907, + "balance_loss_clip": 1.00467253, + "balance_loss_mlp": 0.99761909, + "epoch": 0.8759694583057777, + "flos": 65571664193280.0, + "grad_norm": 0.8085619066900276, + "language_loss": 0.53372836, + "learning_rate": 1.5909514769078892e-07, + "loss": 0.55404019, + "num_input_tokens_seen": 157635675, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01287842, + "step": 7285, + "time_per_iteration": 3.3963637351989746 + }, + { + "auxiliary_loss_clip": 0.01124915, + "auxiliary_loss_mlp": 0.01033231, + "balance_loss_clip": 1.0417093, + "balance_loss_mlp": 1.01849699, + "epoch": 0.8760897011964167, + "flos": 25446193608960.0, + "grad_norm": 1.6609194457755516, + "language_loss": 0.77712125, + "learning_rate": 1.5879082266596867e-07, + "loss": 0.79870272, + "num_input_tokens_seen": 157657015, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.1472168, + "step": 7286, + "time_per_iteration": 2.873481512069702 + }, + { + "auxiliary_loss_clip": 0.01119364, + "auxiliary_loss_mlp": 0.01030855, + "balance_loss_clip": 1.03725743, + "balance_loss_mlp": 1.01706886, + "epoch": 0.8762099440870559, + "flos": 28984830894720.0, + "grad_norm": 1.653643011935102, + "language_loss": 0.72083223, + "learning_rate": 1.5848677695111645e-07, + "loss": 0.74233437, + "num_input_tokens_seen": 157678615, + "router_z_loss_clip": 0.82080078, + "router_z_loss_mlp": 0.13800049, + "step": 7287, + "time_per_iteration": 2.968808174133301 + }, + { + "auxiliary_loss_clip": 0.01130035, + "auxiliary_loss_mlp": 0.0104534, + "balance_loss_clip": 1.04104888, + "balance_loss_mlp": 1.0288775, + "epoch": 0.8763301869776949, + "flos": 21609461352960.0, + "grad_norm": 2.274854024728098, + "language_loss": 0.69805902, + "learning_rate": 1.5818301059235562e-07, + "loss": 0.71981275, + "num_input_tokens_seen": 157693790, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.16467285, + "step": 7288, + "time_per_iteration": 2.9190568923950195 + }, + { + "auxiliary_loss_clip": 0.01126878, + "auxiliary_loss_mlp": 0.01040241, + "balance_loss_clip": 1.03960848, + "balance_loss_mlp": 1.0237422, + "epoch": 0.876450429868334, + "flos": 24644416176000.0, + "grad_norm": 1.5876288357359378, + "language_loss": 0.81529975, + "learning_rate": 1.578795236357684e-07, + "loss": 0.83697093, + "num_input_tokens_seen": 157715255, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.16516113, + "step": 7289, + "time_per_iteration": 2.889338731765747 + }, + { + "auxiliary_loss_clip": 0.01126665, + "auxiliary_loss_mlp": 0.01038848, + "balance_loss_clip": 1.04206717, + "balance_loss_mlp": 1.02389872, + "epoch": 0.8765706727589732, + "flos": 20260046188800.0, + "grad_norm": 2.490907332143591, + "language_loss": 0.85473835, + "learning_rate": 1.5757631612739218e-07, + "loss": 0.87639344, + "num_input_tokens_seen": 157728800, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.14953613, + "step": 7290, + "time_per_iteration": 3.4882147312164307 + }, + { + "auxiliary_loss_clip": 0.01031599, + "auxiliary_loss_mlp": 0.01000548, + "balance_loss_clip": 1.0041095, + "balance_loss_mlp": 0.99929589, + "epoch": 0.8766909156496122, + "flos": 71371165276800.0, + "grad_norm": 0.789925695318356, + "language_loss": 0.61407506, + "learning_rate": 1.572733881132242e-07, + "loss": 0.63439655, + "num_input_tokens_seen": 157789445, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01251221, + "step": 7291, + "time_per_iteration": 3.4078726768493652 + }, + { + "auxiliary_loss_clip": 0.01032343, + "auxiliary_loss_mlp": 0.01000996, + "balance_loss_clip": 1.00452471, + "balance_loss_mlp": 0.99972069, + "epoch": 0.8768111585402513, + "flos": 69523490603520.0, + "grad_norm": 0.7801016000188119, + "language_loss": 0.58533579, + "learning_rate": 1.5697073963921814e-07, + "loss": 0.60566914, + "num_input_tokens_seen": 157848685, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01275635, + "step": 7292, + "time_per_iteration": 3.2952919006347656 + }, + { + "auxiliary_loss_clip": 0.01126897, + "auxiliary_loss_mlp": 0.01036046, + "balance_loss_clip": 1.03917682, + "balance_loss_mlp": 1.01978016, + "epoch": 0.8769314014308904, + "flos": 18838558385280.0, + "grad_norm": 2.366608667522757, + "language_loss": 0.85294712, + "learning_rate": 1.566683707512857e-07, + "loss": 0.87457657, + "num_input_tokens_seen": 157866360, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16259766, + "step": 7293, + "time_per_iteration": 2.870978355407715 + }, + { + "auxiliary_loss_clip": 0.01126393, + "auxiliary_loss_mlp": 0.01033789, + "balance_loss_clip": 1.04084516, + "balance_loss_mlp": 1.01803005, + "epoch": 0.8770516443215295, + "flos": 14976402278400.0, + "grad_norm": 1.964038301596771, + "language_loss": 0.79805362, + "learning_rate": 1.5636628149529553e-07, + "loss": 0.81965542, + "num_input_tokens_seen": 157884150, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15759277, + "step": 7294, + "time_per_iteration": 3.5786008834838867 + }, + { + "auxiliary_loss_clip": 0.01126141, + "auxiliary_loss_mlp": 0.01038724, + "balance_loss_clip": 1.04012752, + "balance_loss_mlp": 1.02261877, + "epoch": 0.8771718872121685, + "flos": 31649654021760.0, + "grad_norm": 2.009811604707951, + "language_loss": 0.79440498, + "learning_rate": 1.560644719170743e-07, + "loss": 0.81605363, + "num_input_tokens_seen": 157905020, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.16082764, + "step": 7295, + "time_per_iteration": 2.887542963027954 + }, + { + "auxiliary_loss_clip": 0.01129171, + "auxiliary_loss_mlp": 0.01042281, + "balance_loss_clip": 1.04025877, + "balance_loss_mlp": 1.02581251, + "epoch": 0.8772921301028077, + "flos": 36095466222720.0, + "grad_norm": 1.8338014666488893, + "language_loss": 0.72149789, + "learning_rate": 1.5576294206240692e-07, + "loss": 0.74321246, + "num_input_tokens_seen": 157924545, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.16461182, + "step": 7296, + "time_per_iteration": 2.903996467590332 + }, + { + "auxiliary_loss_clip": 0.0112824, + "auxiliary_loss_mlp": 0.01041101, + "balance_loss_clip": 1.04174411, + "balance_loss_mlp": 1.02664125, + "epoch": 0.8774123729934468, + "flos": 57116961849600.0, + "grad_norm": 1.733030669426083, + "language_loss": 0.67829269, + "learning_rate": 1.5546169197703507e-07, + "loss": 0.6999861, + "num_input_tokens_seen": 157950820, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.14465332, + "step": 7297, + "time_per_iteration": 3.7907426357269287 + }, + { + "auxiliary_loss_clip": 0.01129795, + "auxiliary_loss_mlp": 0.01041618, + "balance_loss_clip": 1.04209292, + "balance_loss_mlp": 1.02648997, + "epoch": 0.8775326158840858, + "flos": 23914495900800.0, + "grad_norm": 2.37301792921071, + "language_loss": 0.77356178, + "learning_rate": 1.5516072170665774e-07, + "loss": 0.79527587, + "num_input_tokens_seen": 157968790, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15130615, + "step": 7298, + "time_per_iteration": 2.935235023498535 + }, + { + "auxiliary_loss_clip": 0.01121985, + "auxiliary_loss_mlp": 0.01034298, + "balance_loss_clip": 1.04019737, + "balance_loss_mlp": 1.01971233, + "epoch": 0.877652858774725, + "flos": 17123285243520.0, + "grad_norm": 2.578287144156563, + "language_loss": 0.86911857, + "learning_rate": 1.5486003129693214e-07, + "loss": 0.89068139, + "num_input_tokens_seen": 157986155, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.14593506, + "step": 7299, + "time_per_iteration": 2.8394947052001953 + }, + { + "auxiliary_loss_clip": 0.01125101, + "auxiliary_loss_mlp": 0.0103915, + "balance_loss_clip": 1.03810883, + "balance_loss_mlp": 1.02399814, + "epoch": 0.877773101665364, + "flos": 16508961912960.0, + "grad_norm": 1.868254089182872, + "language_loss": 0.77990711, + "learning_rate": 1.545596207934725e-07, + "loss": 0.80154961, + "num_input_tokens_seen": 158004640, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.15148926, + "step": 7300, + "time_per_iteration": 2.8178484439849854 + }, + { + "auxiliary_loss_clip": 0.01120695, + "auxiliary_loss_mlp": 0.01034589, + "balance_loss_clip": 1.03895617, + "balance_loss_mlp": 1.02012336, + "epoch": 0.8778933445560031, + "flos": 22053209973120.0, + "grad_norm": 2.508994651290465, + "language_loss": 0.7774266, + "learning_rate": 1.5425949024185147e-07, + "loss": 0.7989794, + "num_input_tokens_seen": 158024665, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.14447021, + "step": 7301, + "time_per_iteration": 3.7264459133148193 + }, + { + "auxiliary_loss_clip": 0.01123463, + "auxiliary_loss_mlp": 0.01035018, + "balance_loss_clip": 1.03990304, + "balance_loss_mlp": 1.01986635, + "epoch": 0.8780135874466423, + "flos": 22564757514240.0, + "grad_norm": 1.842431272938934, + "language_loss": 0.67732912, + "learning_rate": 1.5395963968759818e-07, + "loss": 0.69891393, + "num_input_tokens_seen": 158044940, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.15148926, + "step": 7302, + "time_per_iteration": 2.812256097793579 + }, + { + "auxiliary_loss_clip": 0.01121119, + "auxiliary_loss_mlp": 0.01034759, + "balance_loss_clip": 1.03828192, + "balance_loss_mlp": 1.01951802, + "epoch": 0.8781338303372813, + "flos": 61531999073280.0, + "grad_norm": 1.6213262871003087, + "language_loss": 0.64679074, + "learning_rate": 1.536600691761998e-07, + "loss": 0.6683495, + "num_input_tokens_seen": 158070770, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.15228271, + "step": 7303, + "time_per_iteration": 3.1653730869293213 + }, + { + "auxiliary_loss_clip": 0.01123461, + "auxiliary_loss_mlp": 0.01036581, + "balance_loss_clip": 1.03975463, + "balance_loss_mlp": 1.02234769, + "epoch": 0.8782540732279204, + "flos": 22674751937280.0, + "grad_norm": 1.8774147941401211, + "language_loss": 0.71563995, + "learning_rate": 1.5336077875310084e-07, + "loss": 0.73724043, + "num_input_tokens_seen": 158089995, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.14227295, + "step": 7304, + "time_per_iteration": 2.949650526046753 + }, + { + "auxiliary_loss_clip": 0.01124146, + "auxiliary_loss_mlp": 0.01038049, + "balance_loss_clip": 1.03885579, + "balance_loss_mlp": 1.02344549, + "epoch": 0.8783743161185595, + "flos": 16070348937600.0, + "grad_norm": 1.8091618908846785, + "language_loss": 0.7418561, + "learning_rate": 1.5306176846370321e-07, + "loss": 0.76347804, + "num_input_tokens_seen": 158108140, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.14624023, + "step": 7305, + "time_per_iteration": 2.8878157138824463 + }, + { + "auxiliary_loss_clip": 0.01124931, + "auxiliary_loss_mlp": 0.01041303, + "balance_loss_clip": 1.03884625, + "balance_loss_mlp": 1.02550805, + "epoch": 0.8784945590091986, + "flos": 26067879227520.0, + "grad_norm": 1.9803973836283624, + "language_loss": 0.74009955, + "learning_rate": 1.5276303835336712e-07, + "loss": 0.76176184, + "num_input_tokens_seen": 158128680, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15795898, + "step": 7306, + "time_per_iteration": 2.882115602493286 + }, + { + "auxiliary_loss_clip": 0.01032193, + "auxiliary_loss_mlp": 0.01002348, + "balance_loss_clip": 1.00463605, + "balance_loss_mlp": 1.00108433, + "epoch": 0.8786148018998376, + "flos": 62720643939840.0, + "grad_norm": 0.8231286157129704, + "language_loss": 0.53482378, + "learning_rate": 1.524645884674094e-07, + "loss": 0.55516922, + "num_input_tokens_seen": 158185610, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01263428, + "step": 7307, + "time_per_iteration": 3.342550039291382 + }, + { + "auxiliary_loss_clip": 0.01126367, + "auxiliary_loss_mlp": 0.01035108, + "balance_loss_clip": 1.04015756, + "balance_loss_mlp": 1.01932442, + "epoch": 0.8787350447904768, + "flos": 21652734263040.0, + "grad_norm": 2.145072831522986, + "language_loss": 0.79310417, + "learning_rate": 1.521664188511047e-07, + "loss": 0.81471896, + "num_input_tokens_seen": 158205635, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.15795898, + "step": 7308, + "time_per_iteration": 2.8878989219665527 + }, + { + "auxiliary_loss_clip": 0.01124506, + "auxiliary_loss_mlp": 0.01039927, + "balance_loss_clip": 1.0398438, + "balance_loss_mlp": 1.02497804, + "epoch": 0.8788552876811159, + "flos": 25478476957440.0, + "grad_norm": 1.9745294413419805, + "language_loss": 0.80632627, + "learning_rate": 1.518685295496851e-07, + "loss": 0.82797056, + "num_input_tokens_seen": 158223495, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.14941406, + "step": 7309, + "time_per_iteration": 2.8400447368621826 + }, + { + "auxiliary_loss_clip": 0.01123851, + "auxiliary_loss_mlp": 0.01041141, + "balance_loss_clip": 1.04133725, + "balance_loss_mlp": 1.02680039, + "epoch": 0.8789755305717549, + "flos": 22310222762880.0, + "grad_norm": 1.618988081776821, + "language_loss": 0.8530432, + "learning_rate": 1.5157092060833975e-07, + "loss": 0.87469316, + "num_input_tokens_seen": 158243145, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.14337158, + "step": 7310, + "time_per_iteration": 2.8189857006073 + }, + { + "auxiliary_loss_clip": 0.01121525, + "auxiliary_loss_mlp": 0.01033083, + "balance_loss_clip": 1.03833818, + "balance_loss_mlp": 1.01708472, + "epoch": 0.879095773462394, + "flos": 29310971408640.0, + "grad_norm": 1.741641894377908, + "language_loss": 0.65780127, + "learning_rate": 1.5127359207221658e-07, + "loss": 0.67934734, + "num_input_tokens_seen": 158262625, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.15997314, + "step": 7311, + "time_per_iteration": 2.8761346340179443 + }, + { + "auxiliary_loss_clip": 0.01126597, + "auxiliary_loss_mlp": 0.01044311, + "balance_loss_clip": 1.03941083, + "balance_loss_mlp": 1.02836704, + "epoch": 0.8792160163530331, + "flos": 16690023394560.0, + "grad_norm": 1.8165920689333712, + "language_loss": 0.73140186, + "learning_rate": 1.5097654398641923e-07, + "loss": 0.75311089, + "num_input_tokens_seen": 158280530, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15942383, + "step": 7312, + "time_per_iteration": 2.7629342079162598 + }, + { + "auxiliary_loss_clip": 0.01125262, + "auxiliary_loss_mlp": 0.01040663, + "balance_loss_clip": 1.04019618, + "balance_loss_mlp": 1.02600002, + "epoch": 0.8793362592436722, + "flos": 24499301230080.0, + "grad_norm": 1.641879392707524, + "language_loss": 0.72910011, + "learning_rate": 1.5067977639601014e-07, + "loss": 0.75075936, + "num_input_tokens_seen": 158303290, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.14672852, + "step": 7313, + "time_per_iteration": 2.8593387603759766 + }, + { + "auxiliary_loss_clip": 0.01124376, + "auxiliary_loss_mlp": 0.01033168, + "balance_loss_clip": 1.04053867, + "balance_loss_mlp": 1.01826644, + "epoch": 0.8794565021343113, + "flos": 14538399834240.0, + "grad_norm": 2.0491974366128023, + "language_loss": 0.70967889, + "learning_rate": 1.5038328934600864e-07, + "loss": 0.73125434, + "num_input_tokens_seen": 158319925, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.14904785, + "step": 7314, + "time_per_iteration": 2.7936413288116455 + }, + { + "auxiliary_loss_clip": 0.01124642, + "auxiliary_loss_mlp": 0.01039827, + "balance_loss_clip": 1.03952098, + "balance_loss_mlp": 1.02524769, + "epoch": 0.8795767450249504, + "flos": 39530286224640.0, + "grad_norm": 2.01253798524256, + "language_loss": 0.70135605, + "learning_rate": 1.5008708288139161e-07, + "loss": 0.72300076, + "num_input_tokens_seen": 158342285, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.14581299, + "step": 7315, + "time_per_iteration": 2.9451632499694824 + }, + { + "auxiliary_loss_clip": 0.01125531, + "auxiliary_loss_mlp": 0.01037131, + "balance_loss_clip": 1.03868604, + "balance_loss_mlp": 1.0219202, + "epoch": 0.8796969879155895, + "flos": 22960672197120.0, + "grad_norm": 1.8741750073354795, + "language_loss": 0.73295474, + "learning_rate": 1.497911570470931e-07, + "loss": 0.75458133, + "num_input_tokens_seen": 158362290, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.15197754, + "step": 7316, + "time_per_iteration": 2.888615131378174 + }, + { + "auxiliary_loss_clip": 0.01122313, + "auxiliary_loss_mlp": 0.01037824, + "balance_loss_clip": 1.03868103, + "balance_loss_mlp": 1.02276182, + "epoch": 0.8798172308062285, + "flos": 28362427004160.0, + "grad_norm": 1.6619638247240223, + "language_loss": 0.85488558, + "learning_rate": 1.494955118880048e-07, + "loss": 0.87648702, + "num_input_tokens_seen": 158383275, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.15057373, + "step": 7317, + "time_per_iteration": 2.943859577178955 + }, + { + "auxiliary_loss_clip": 0.01122008, + "auxiliary_loss_mlp": 0.01033982, + "balance_loss_clip": 1.03866911, + "balance_loss_mlp": 1.01939726, + "epoch": 0.8799374736968677, + "flos": 23988974751360.0, + "grad_norm": 2.407788622763913, + "language_loss": 0.72623098, + "learning_rate": 1.4920014744897634e-07, + "loss": 0.74779093, + "num_input_tokens_seen": 158402690, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.14587402, + "step": 7318, + "time_per_iteration": 2.839872121810913 + }, + { + "auxiliary_loss_clip": 0.01126879, + "auxiliary_loss_mlp": 0.01048024, + "balance_loss_clip": 1.04139113, + "balance_loss_mlp": 1.03305089, + "epoch": 0.8800577165875068, + "flos": 25630271832960.0, + "grad_norm": 1.8188938509488477, + "language_loss": 0.86206174, + "learning_rate": 1.4890506377481392e-07, + "loss": 0.8838107, + "num_input_tokens_seen": 158421780, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.14984131, + "step": 7319, + "time_per_iteration": 3.664609909057617 + }, + { + "auxiliary_loss_clip": 0.01118974, + "auxiliary_loss_mlp": 0.0103741, + "balance_loss_clip": 1.03818035, + "balance_loss_mlp": 1.02389753, + "epoch": 0.8801779594781458, + "flos": 23440331439360.0, + "grad_norm": 1.642393351174405, + "language_loss": 0.64238167, + "learning_rate": 1.486102609102815e-07, + "loss": 0.66394556, + "num_input_tokens_seen": 158442330, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.1350708, + "step": 7320, + "time_per_iteration": 2.954000234603882 + }, + { + "auxiliary_loss_clip": 0.01123788, + "auxiliary_loss_mlp": 0.01034094, + "balance_loss_clip": 1.0391171, + "balance_loss_mlp": 1.01981902, + "epoch": 0.880298202368785, + "flos": 11508580656000.0, + "grad_norm": 2.582693774550504, + "language_loss": 0.85414839, + "learning_rate": 1.483157389001004e-07, + "loss": 0.87572724, + "num_input_tokens_seen": 158459890, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14257812, + "step": 7321, + "time_per_iteration": 2.8156614303588867 + }, + { + "auxiliary_loss_clip": 0.01127625, + "auxiliary_loss_mlp": 0.01034516, + "balance_loss_clip": 1.03855276, + "balance_loss_mlp": 1.01836896, + "epoch": 0.880418445259424, + "flos": 22671447886080.0, + "grad_norm": 2.635984500247354, + "language_loss": 0.79293585, + "learning_rate": 1.4802149778894933e-07, + "loss": 0.81455725, + "num_input_tokens_seen": 158478680, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.16143799, + "step": 7322, + "time_per_iteration": 2.867907762527466 + }, + { + "auxiliary_loss_clip": 0.01121222, + "auxiliary_loss_mlp": 0.01038453, + "balance_loss_clip": 1.03674829, + "balance_loss_mlp": 1.02464294, + "epoch": 0.8805386881500631, + "flos": 20522158709760.0, + "grad_norm": 1.748590630740307, + "language_loss": 0.87455487, + "learning_rate": 1.4772753762146484e-07, + "loss": 0.89615166, + "num_input_tokens_seen": 158497935, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.13812256, + "step": 7323, + "time_per_iteration": 3.6362547874450684 + }, + { + "auxiliary_loss_clip": 0.01126763, + "auxiliary_loss_mlp": 0.01036067, + "balance_loss_clip": 1.03977299, + "balance_loss_mlp": 1.02046216, + "epoch": 0.8806589310407023, + "flos": 36538891620480.0, + "grad_norm": 1.5099213910037468, + "language_loss": 0.70411313, + "learning_rate": 1.474338584422401e-07, + "loss": 0.72574139, + "num_input_tokens_seen": 158523145, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15600586, + "step": 7324, + "time_per_iteration": 2.966092824935913 + }, + { + "auxiliary_loss_clip": 0.01122061, + "auxiliary_loss_mlp": 0.01039701, + "balance_loss_clip": 1.03922403, + "balance_loss_mlp": 1.0249666, + "epoch": 0.8807791739313413, + "flos": 23440187784960.0, + "grad_norm": 1.6227297299519714, + "language_loss": 0.75776768, + "learning_rate": 1.4714046029582595e-07, + "loss": 0.77938533, + "num_input_tokens_seen": 158542210, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.14733887, + "step": 7325, + "time_per_iteration": 2.8497061729431152 + }, + { + "auxiliary_loss_clip": 0.01120535, + "auxiliary_loss_mlp": 0.01041123, + "balance_loss_clip": 1.0372647, + "balance_loss_mlp": 1.02554202, + "epoch": 0.8808994168219804, + "flos": 25956843310080.0, + "grad_norm": 1.6793987540956574, + "language_loss": 0.75600523, + "learning_rate": 1.46847343226731e-07, + "loss": 0.77762187, + "num_input_tokens_seen": 158563250, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.15588379, + "step": 7326, + "time_per_iteration": 3.67781138420105 + }, + { + "auxiliary_loss_clip": 0.01126404, + "auxiliary_loss_mlp": 0.01036914, + "balance_loss_clip": 1.0409658, + "balance_loss_mlp": 1.02171469, + "epoch": 0.8810196597126195, + "flos": 17092079303040.0, + "grad_norm": 1.8502209559750498, + "language_loss": 0.69641453, + "learning_rate": 1.465545072794203e-07, + "loss": 0.71804768, + "num_input_tokens_seen": 158581125, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15185547, + "step": 7327, + "time_per_iteration": 2.8990893363952637 + }, + { + "auxiliary_loss_clip": 0.01122411, + "auxiliary_loss_mlp": 0.01038774, + "balance_loss_clip": 1.03835356, + "balance_loss_mlp": 1.02369428, + "epoch": 0.8811399026032586, + "flos": 23002831785600.0, + "grad_norm": 1.73173979912348, + "language_loss": 0.75684011, + "learning_rate": 1.4626195249831774e-07, + "loss": 0.77845192, + "num_input_tokens_seen": 158602025, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.15087891, + "step": 7328, + "time_per_iteration": 2.9391541481018066 + }, + { + "auxiliary_loss_clip": 0.01121445, + "auxiliary_loss_mlp": 0.01034125, + "balance_loss_clip": 1.03818703, + "balance_loss_mlp": 1.0203203, + "epoch": 0.8812601454938976, + "flos": 14463813242880.0, + "grad_norm": 1.7783934105278272, + "language_loss": 0.71853852, + "learning_rate": 1.4596967892780244e-07, + "loss": 0.74009418, + "num_input_tokens_seen": 158618355, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.13793945, + "step": 7329, + "time_per_iteration": 3.793748140335083 + }, + { + "auxiliary_loss_clip": 0.01123158, + "auxiliary_loss_mlp": 0.01037394, + "balance_loss_clip": 1.03955841, + "balance_loss_mlp": 1.02285087, + "epoch": 0.8813803883845368, + "flos": 22493223578880.0, + "grad_norm": 1.7152229926566767, + "language_loss": 0.74798107, + "learning_rate": 1.4567768661221314e-07, + "loss": 0.76958656, + "num_input_tokens_seen": 158638925, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.14538574, + "step": 7330, + "time_per_iteration": 2.8186750411987305 + }, + { + "auxiliary_loss_clip": 0.01128431, + "auxiliary_loss_mlp": 0.01041493, + "balance_loss_clip": 1.04038858, + "balance_loss_mlp": 1.02572143, + "epoch": 0.8815006312751759, + "flos": 21506901045120.0, + "grad_norm": 2.063441525687441, + "language_loss": 0.74467528, + "learning_rate": 1.4538597559584442e-07, + "loss": 0.76637453, + "num_input_tokens_seen": 158656715, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.15771484, + "step": 7331, + "time_per_iteration": 2.805203914642334 + }, + { + "auxiliary_loss_clip": 0.01125442, + "auxiliary_loss_mlp": 0.01037037, + "balance_loss_clip": 1.03938878, + "balance_loss_mlp": 1.02117038, + "epoch": 0.8816208741658149, + "flos": 22784566792320.0, + "grad_norm": 1.9995912526369508, + "language_loss": 0.79290962, + "learning_rate": 1.4509454592294823e-07, + "loss": 0.81453443, + "num_input_tokens_seen": 158677200, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15856934, + "step": 7332, + "time_per_iteration": 2.7378838062286377 + }, + { + "auxiliary_loss_clip": 0.01124571, + "auxiliary_loss_mlp": 0.01042947, + "balance_loss_clip": 1.0387336, + "balance_loss_mlp": 1.02749729, + "epoch": 0.8817411170564541, + "flos": 17779409026560.0, + "grad_norm": 1.9917811035850115, + "language_loss": 0.78732455, + "learning_rate": 1.448033976377354e-07, + "loss": 0.80899972, + "num_input_tokens_seen": 158692185, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.15454102, + "step": 7333, + "time_per_iteration": 2.797104597091675 + }, + { + "auxiliary_loss_clip": 0.01123123, + "auxiliary_loss_mlp": 0.01035608, + "balance_loss_clip": 1.03874099, + "balance_loss_mlp": 1.02084947, + "epoch": 0.8818613599470931, + "flos": 18551812112640.0, + "grad_norm": 2.0596384257975746, + "language_loss": 0.7413649, + "learning_rate": 1.445125307843713e-07, + "loss": 0.76295221, + "num_input_tokens_seen": 158710410, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14758301, + "step": 7334, + "time_per_iteration": 2.8530185222625732 + }, + { + "auxiliary_loss_clip": 0.01120673, + "auxiliary_loss_mlp": 0.01035718, + "balance_loss_clip": 1.03844595, + "balance_loss_mlp": 1.02171135, + "epoch": 0.8819816028377322, + "flos": 27599792417280.0, + "grad_norm": 1.527123220980034, + "language_loss": 0.75484902, + "learning_rate": 1.442219454069813e-07, + "loss": 0.7764129, + "num_input_tokens_seen": 158731435, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.14019775, + "step": 7335, + "time_per_iteration": 2.8519773483276367 + }, + { + "auxiliary_loss_clip": 0.01121697, + "auxiliary_loss_mlp": 0.01031677, + "balance_loss_clip": 1.03880572, + "balance_loss_mlp": 1.01610231, + "epoch": 0.8821018457283714, + "flos": 23404600385280.0, + "grad_norm": 2.031790476846809, + "language_loss": 0.66276777, + "learning_rate": 1.4393164154964676e-07, + "loss": 0.6843015, + "num_input_tokens_seen": 158750965, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.15570068, + "step": 7336, + "time_per_iteration": 2.805856943130493 + }, + { + "auxiliary_loss_clip": 0.01125704, + "auxiliary_loss_mlp": 0.0103443, + "balance_loss_clip": 1.04037499, + "balance_loss_mlp": 1.02102494, + "epoch": 0.8822220886190104, + "flos": 29132459792640.0, + "grad_norm": 1.7921687605563517, + "language_loss": 0.93757808, + "learning_rate": 1.4364161925640649e-07, + "loss": 0.9591794, + "num_input_tokens_seen": 158772365, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.13427734, + "step": 7337, + "time_per_iteration": 2.866974115371704 + }, + { + "auxiliary_loss_clip": 0.01124782, + "auxiliary_loss_mlp": 0.01037322, + "balance_loss_clip": 1.04046512, + "balance_loss_mlp": 1.0225935, + "epoch": 0.8823423315096495, + "flos": 20485422074880.0, + "grad_norm": 1.7990753095131353, + "language_loss": 0.85239482, + "learning_rate": 1.4335187857125663e-07, + "loss": 0.87401587, + "num_input_tokens_seen": 158791065, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.14709473, + "step": 7338, + "time_per_iteration": 2.8789169788360596 + }, + { + "auxiliary_loss_clip": 0.01124392, + "auxiliary_loss_mlp": 0.01037483, + "balance_loss_clip": 1.03904343, + "balance_loss_mlp": 1.02279687, + "epoch": 0.8824625744002886, + "flos": 24206377818240.0, + "grad_norm": 1.691463358877026, + "language_loss": 0.75439215, + "learning_rate": 1.4306241953815023e-07, + "loss": 0.77601093, + "num_input_tokens_seen": 158812125, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.14685059, + "step": 7339, + "time_per_iteration": 2.8759312629699707 + }, + { + "auxiliary_loss_clip": 0.01122788, + "auxiliary_loss_mlp": 0.01034611, + "balance_loss_clip": 1.03786969, + "balance_loss_mlp": 1.01925635, + "epoch": 0.8825828172909277, + "flos": 24679500785280.0, + "grad_norm": 1.7064666807724886, + "language_loss": 0.71055222, + "learning_rate": 1.4277324220099862e-07, + "loss": 0.73212618, + "num_input_tokens_seen": 158834035, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.15344238, + "step": 7340, + "time_per_iteration": 2.979940414428711 + }, + { + "auxiliary_loss_clip": 0.01121303, + "auxiliary_loss_mlp": 0.01036303, + "balance_loss_clip": 1.039222, + "balance_loss_mlp": 1.02204537, + "epoch": 0.8827030601815667, + "flos": 22456163721600.0, + "grad_norm": 1.82469536756625, + "language_loss": 0.74286097, + "learning_rate": 1.4248434660366938e-07, + "loss": 0.76443708, + "num_input_tokens_seen": 158853510, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.1427002, + "step": 7341, + "time_per_iteration": 2.86211895942688 + }, + { + "auxiliary_loss_clip": 0.0112666, + "auxiliary_loss_mlp": 0.01039954, + "balance_loss_clip": 1.03941298, + "balance_loss_mlp": 1.02446878, + "epoch": 0.8828233030722058, + "flos": 19865639877120.0, + "grad_norm": 1.6688990339434333, + "language_loss": 0.70464587, + "learning_rate": 1.4219573278998808e-07, + "loss": 0.72631204, + "num_input_tokens_seen": 158871970, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.15496826, + "step": 7342, + "time_per_iteration": 2.89778733253479 + }, + { + "auxiliary_loss_clip": 0.0112819, + "auxiliary_loss_mlp": 0.0103891, + "balance_loss_clip": 1.04110837, + "balance_loss_mlp": 1.02329385, + "epoch": 0.882943545962845, + "flos": 39347213581440.0, + "grad_norm": 2.0385438599480614, + "language_loss": 0.64797282, + "learning_rate": 1.4190740080373685e-07, + "loss": 0.66964388, + "num_input_tokens_seen": 158892250, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15600586, + "step": 7343, + "time_per_iteration": 2.9389801025390625 + }, + { + "auxiliary_loss_clip": 0.01127273, + "auxiliary_loss_mlp": 0.01033155, + "balance_loss_clip": 1.04057479, + "balance_loss_mlp": 1.0174787, + "epoch": 0.883063788853484, + "flos": 19054524908160.0, + "grad_norm": 1.8206409730247688, + "language_loss": 0.84448957, + "learning_rate": 1.4161935068865538e-07, + "loss": 0.86609387, + "num_input_tokens_seen": 158907395, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15673828, + "step": 7344, + "time_per_iteration": 2.8265631198883057 + }, + { + "auxiliary_loss_clip": 0.01124955, + "auxiliary_loss_mlp": 0.01043898, + "balance_loss_clip": 1.0391047, + "balance_loss_mlp": 1.02819777, + "epoch": 0.8831840317441231, + "flos": 18733196816640.0, + "grad_norm": 1.836081670241577, + "language_loss": 0.75544477, + "learning_rate": 1.4133158248844113e-07, + "loss": 0.77713323, + "num_input_tokens_seen": 158926300, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.15698242, + "step": 7345, + "time_per_iteration": 2.8492367267608643 + }, + { + "auxiliary_loss_clip": 0.01124736, + "auxiliary_loss_mlp": 0.0103451, + "balance_loss_clip": 1.038131, + "balance_loss_mlp": 1.0191555, + "epoch": 0.8833042746347622, + "flos": 26827712553600.0, + "grad_norm": 1.8054866403930716, + "language_loss": 0.73152155, + "learning_rate": 1.4104409624674785e-07, + "loss": 0.75311399, + "num_input_tokens_seen": 158946085, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15356445, + "step": 7346, + "time_per_iteration": 2.8367509841918945 + }, + { + "auxiliary_loss_clip": 0.01126317, + "auxiliary_loss_mlp": 0.01040545, + "balance_loss_clip": 1.0431453, + "balance_loss_mlp": 1.02508354, + "epoch": 0.8834245175254013, + "flos": 26104077158400.0, + "grad_norm": 1.7187301296182937, + "language_loss": 0.78518474, + "learning_rate": 1.407568920071873e-07, + "loss": 0.80685329, + "num_input_tokens_seen": 158964950, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.15472412, + "step": 7347, + "time_per_iteration": 2.8992323875427246 + }, + { + "auxiliary_loss_clip": 0.01129513, + "auxiliary_loss_mlp": 0.01039137, + "balance_loss_clip": 1.04048276, + "balance_loss_mlp": 1.02245951, + "epoch": 0.8835447604160404, + "flos": 30629036977920.0, + "grad_norm": 2.0107047894447088, + "language_loss": 0.6868993, + "learning_rate": 1.4046996981332782e-07, + "loss": 0.7085858, + "num_input_tokens_seen": 158984835, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.16687012, + "step": 7348, + "time_per_iteration": 3.6059839725494385 + }, + { + "auxiliary_loss_clip": 0.0112985, + "auxiliary_loss_mlp": 0.01034967, + "balance_loss_clip": 1.03843999, + "balance_loss_mlp": 1.01887333, + "epoch": 0.8836650033066795, + "flos": 24718356322560.0, + "grad_norm": 1.8550942594194988, + "language_loss": 0.78030074, + "learning_rate": 1.4018332970869516e-07, + "loss": 0.80194885, + "num_input_tokens_seen": 159002775, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.16101074, + "step": 7349, + "time_per_iteration": 2.8818039894104004 + }, + { + "auxiliary_loss_clip": 0.01122717, + "auxiliary_loss_mlp": 0.01040266, + "balance_loss_clip": 1.03829157, + "balance_loss_mlp": 1.02615714, + "epoch": 0.8837852461973186, + "flos": 25413371556480.0, + "grad_norm": 1.8609363084007489, + "language_loss": 0.852808, + "learning_rate": 1.398969717367733e-07, + "loss": 0.87443781, + "num_input_tokens_seen": 159024100, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.14111328, + "step": 7350, + "time_per_iteration": 2.9685568809509277 + }, + { + "auxiliary_loss_clip": 0.01121273, + "auxiliary_loss_mlp": 0.01034712, + "balance_loss_clip": 1.03802228, + "balance_loss_mlp": 1.02018023, + "epoch": 0.8839054890879576, + "flos": 17822574195840.0, + "grad_norm": 1.9879990346857273, + "language_loss": 0.76475525, + "learning_rate": 1.396108959410014e-07, + "loss": 0.78631502, + "num_input_tokens_seen": 159043315, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.14538574, + "step": 7351, + "time_per_iteration": 2.9121556282043457 + }, + { + "auxiliary_loss_clip": 0.01126575, + "auxiliary_loss_mlp": 0.01038054, + "balance_loss_clip": 1.03881109, + "balance_loss_mlp": 1.02259278, + "epoch": 0.8840257319785968, + "flos": 23769021818880.0, + "grad_norm": 1.5303602124845315, + "language_loss": 0.81387705, + "learning_rate": 1.3932510236477745e-07, + "loss": 0.83552337, + "num_input_tokens_seen": 159063985, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15478516, + "step": 7352, + "time_per_iteration": 3.655067205429077 + }, + { + "auxiliary_loss_clip": 0.01127429, + "auxiliary_loss_mlp": 0.01041602, + "balance_loss_clip": 1.03931308, + "balance_loss_mlp": 1.02572298, + "epoch": 0.8841459748692359, + "flos": 29059776622080.0, + "grad_norm": 2.0022419109858802, + "language_loss": 0.55787218, + "learning_rate": 1.3903959105145636e-07, + "loss": 0.57956249, + "num_input_tokens_seen": 159084475, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15893555, + "step": 7353, + "time_per_iteration": 2.973393440246582 + }, + { + "auxiliary_loss_clip": 0.01121703, + "auxiliary_loss_mlp": 0.01033123, + "balance_loss_clip": 1.03844786, + "balance_loss_mlp": 1.01861572, + "epoch": 0.8842662177598749, + "flos": 24311523905280.0, + "grad_norm": 2.1108578079695004, + "language_loss": 0.82954031, + "learning_rate": 1.387543620443492e-07, + "loss": 0.85108858, + "num_input_tokens_seen": 159101320, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.1451416, + "step": 7354, + "time_per_iteration": 2.802429676055908 + }, + { + "auxiliary_loss_clip": 0.01122661, + "auxiliary_loss_mlp": 0.01042043, + "balance_loss_clip": 1.04028428, + "balance_loss_mlp": 1.02699864, + "epoch": 0.8843864606505141, + "flos": 25007867942400.0, + "grad_norm": 1.8228583752661593, + "language_loss": 0.84024894, + "learning_rate": 1.3846941538672606e-07, + "loss": 0.86189592, + "num_input_tokens_seen": 159120025, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.1505127, + "step": 7355, + "time_per_iteration": 3.6511685848236084 + }, + { + "auxiliary_loss_clip": 0.01124599, + "auxiliary_loss_mlp": 0.01044586, + "balance_loss_clip": 1.03980398, + "balance_loss_mlp": 1.02945209, + "epoch": 0.8845067035411531, + "flos": 28183915388160.0, + "grad_norm": 4.618557391050462, + "language_loss": 0.80726731, + "learning_rate": 1.3818475112181193e-07, + "loss": 0.82895923, + "num_input_tokens_seen": 159138820, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.15142822, + "step": 7356, + "time_per_iteration": 2.8608920574188232 + }, + { + "auxiliary_loss_clip": 0.01119381, + "auxiliary_loss_mlp": 0.01039403, + "balance_loss_clip": 1.03817117, + "balance_loss_mlp": 1.02537179, + "epoch": 0.8846269464317922, + "flos": 12853219311360.0, + "grad_norm": 1.9109488858335453, + "language_loss": 0.79602438, + "learning_rate": 1.3790036929279091e-07, + "loss": 0.81761223, + "num_input_tokens_seen": 159155975, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.14031982, + "step": 7357, + "time_per_iteration": 2.785273551940918 + }, + { + "auxiliary_loss_clip": 0.01127972, + "auxiliary_loss_mlp": 0.01032724, + "balance_loss_clip": 1.04243088, + "balance_loss_mlp": 1.0173341, + "epoch": 0.8847471893224313, + "flos": 18624351628800.0, + "grad_norm": 2.107398861557654, + "language_loss": 0.58538193, + "learning_rate": 1.3761626994280363e-07, + "loss": 0.60698891, + "num_input_tokens_seen": 159173445, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15393066, + "step": 7358, + "time_per_iteration": 3.7419493198394775 + }, + { + "auxiliary_loss_clip": 0.01124841, + "auxiliary_loss_mlp": 0.01031736, + "balance_loss_clip": 1.03866386, + "balance_loss_mlp": 1.01721048, + "epoch": 0.8848674322130704, + "flos": 35769433449600.0, + "grad_norm": 1.6191215273409543, + "language_loss": 0.73657048, + "learning_rate": 1.3733245311494735e-07, + "loss": 0.75813621, + "num_input_tokens_seen": 159196100, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.14526367, + "step": 7359, + "time_per_iteration": 2.944018840789795 + }, + { + "auxiliary_loss_clip": 0.01125235, + "auxiliary_loss_mlp": 0.01040615, + "balance_loss_clip": 1.04089928, + "balance_loss_mlp": 1.02531445, + "epoch": 0.8849876751037095, + "flos": 24243760897920.0, + "grad_norm": 1.9662520817219016, + "language_loss": 0.70879871, + "learning_rate": 1.3704891885227676e-07, + "loss": 0.73045719, + "num_input_tokens_seen": 159216145, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.15307617, + "step": 7360, + "time_per_iteration": 2.8900814056396484 + }, + { + "auxiliary_loss_clip": 0.01128078, + "auxiliary_loss_mlp": 0.01041969, + "balance_loss_clip": 1.04082298, + "balance_loss_mlp": 1.0263288, + "epoch": 0.8851079179943486, + "flos": 21500580251520.0, + "grad_norm": 1.8750338197507421, + "language_loss": 0.77600402, + "learning_rate": 1.367656671978037e-07, + "loss": 0.79770446, + "num_input_tokens_seen": 159233610, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.15637207, + "step": 7361, + "time_per_iteration": 2.8124592304229736 + }, + { + "auxiliary_loss_clip": 0.01124333, + "auxiliary_loss_mlp": 0.01033155, + "balance_loss_clip": 1.04021239, + "balance_loss_mlp": 1.01963103, + "epoch": 0.8852281608849877, + "flos": 15300711198720.0, + "grad_norm": 1.7740310092413125, + "language_loss": 0.73583084, + "learning_rate": 1.36482698194498e-07, + "loss": 0.75740576, + "num_input_tokens_seen": 159250155, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.13525391, + "step": 7362, + "time_per_iteration": 2.8080742359161377 + }, + { + "auxiliary_loss_clip": 0.01127121, + "auxiliary_loss_mlp": 0.01037912, + "balance_loss_clip": 1.03941584, + "balance_loss_mlp": 1.02182496, + "epoch": 0.8853484037756267, + "flos": 23295719283840.0, + "grad_norm": 2.2134021191708944, + "language_loss": 0.71937907, + "learning_rate": 1.3620001188528506e-07, + "loss": 0.74102938, + "num_input_tokens_seen": 159270875, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.16113281, + "step": 7363, + "time_per_iteration": 2.8651516437530518 + }, + { + "auxiliary_loss_clip": 0.01127457, + "auxiliary_loss_mlp": 0.01040174, + "balance_loss_clip": 1.03922462, + "balance_loss_mlp": 1.02379465, + "epoch": 0.8854686466662659, + "flos": 25114773795840.0, + "grad_norm": 2.3721734383585145, + "language_loss": 0.73210573, + "learning_rate": 1.3591760831304865e-07, + "loss": 0.75378203, + "num_input_tokens_seen": 159288565, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.16394043, + "step": 7364, + "time_per_iteration": 2.8919692039489746 + }, + { + "auxiliary_loss_clip": 0.0112451, + "auxiliary_loss_mlp": 0.01040948, + "balance_loss_clip": 1.04111099, + "balance_loss_mlp": 1.02528393, + "epoch": 0.885588889556905, + "flos": 21390873137280.0, + "grad_norm": 1.5820535193428127, + "language_loss": 0.79181921, + "learning_rate": 1.356354875206287e-07, + "loss": 0.81347382, + "num_input_tokens_seen": 159306400, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.15649414, + "step": 7365, + "time_per_iteration": 2.883044719696045 + }, + { + "auxiliary_loss_clip": 0.01124059, + "auxiliary_loss_mlp": 0.0104048, + "balance_loss_clip": 1.040241, + "balance_loss_mlp": 1.02616894, + "epoch": 0.885709132447544, + "flos": 26906752431360.0, + "grad_norm": 1.963804725632878, + "language_loss": 0.69583917, + "learning_rate": 1.3535364955082296e-07, + "loss": 0.71748459, + "num_input_tokens_seen": 159326250, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.14306641, + "step": 7366, + "time_per_iteration": 2.8080246448516846 + }, + { + "auxiliary_loss_clip": 0.01123633, + "auxiliary_loss_mlp": 0.01034261, + "balance_loss_clip": 1.04056025, + "balance_loss_mlp": 1.02047992, + "epoch": 0.8858293753381832, + "flos": 26103394800000.0, + "grad_norm": 1.6401947186222121, + "language_loss": 0.64243227, + "learning_rate": 1.3507209444638613e-07, + "loss": 0.66401118, + "num_input_tokens_seen": 159348250, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.13787842, + "step": 7367, + "time_per_iteration": 2.917996644973755 + }, + { + "auxiliary_loss_clip": 0.01124283, + "auxiliary_loss_mlp": 0.01038859, + "balance_loss_clip": 1.03911102, + "balance_loss_mlp": 1.02388597, + "epoch": 0.8859496182288222, + "flos": 23292810282240.0, + "grad_norm": 1.8404613086408121, + "language_loss": 0.74179643, + "learning_rate": 1.347908222500298e-07, + "loss": 0.76342785, + "num_input_tokens_seen": 159368325, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.1496582, + "step": 7368, + "time_per_iteration": 2.7723331451416016 + }, + { + "auxiliary_loss_clip": 0.01120807, + "auxiliary_loss_mlp": 0.01040018, + "balance_loss_clip": 1.03860891, + "balance_loss_mlp": 1.02590394, + "epoch": 0.8860698611194613, + "flos": 16872916469760.0, + "grad_norm": 1.8971445923263572, + "language_loss": 0.70078182, + "learning_rate": 1.3450983300442276e-07, + "loss": 0.72239006, + "num_input_tokens_seen": 159387555, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.14117432, + "step": 7369, + "time_per_iteration": 2.779723882675171 + }, + { + "auxiliary_loss_clip": 0.01126321, + "auxiliary_loss_mlp": 0.01036849, + "balance_loss_clip": 1.04073966, + "balance_loss_mlp": 1.02181053, + "epoch": 0.8861901040101005, + "flos": 24681404206080.0, + "grad_norm": 1.9106085642236266, + "language_loss": 0.73675334, + "learning_rate": 1.3422912675219068e-07, + "loss": 0.75838506, + "num_input_tokens_seen": 159407310, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15039062, + "step": 7370, + "time_per_iteration": 2.8241512775421143 + }, + { + "auxiliary_loss_clip": 0.01122327, + "auxiliary_loss_mlp": 0.01039045, + "balance_loss_clip": 1.04017329, + "balance_loss_mlp": 1.02460861, + "epoch": 0.8863103469007395, + "flos": 24423026699520.0, + "grad_norm": 1.510318924818396, + "language_loss": 0.79109848, + "learning_rate": 1.339487035359166e-07, + "loss": 0.81271219, + "num_input_tokens_seen": 159427680, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.14453125, + "step": 7371, + "time_per_iteration": 2.7926690578460693 + }, + { + "auxiliary_loss_clip": 0.01121105, + "auxiliary_loss_mlp": 0.01030271, + "balance_loss_clip": 1.0398581, + "balance_loss_mlp": 1.01573944, + "epoch": 0.8864305897913786, + "flos": 22053964158720.0, + "grad_norm": 2.0786128031045705, + "language_loss": 0.84654629, + "learning_rate": 1.336685633981409e-07, + "loss": 0.86805999, + "num_input_tokens_seen": 159448765, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.14538574, + "step": 7372, + "time_per_iteration": 2.8052449226379395 + }, + { + "auxiliary_loss_clip": 0.01123183, + "auxiliary_loss_mlp": 0.01041068, + "balance_loss_clip": 1.03983402, + "balance_loss_mlp": 1.02567816, + "epoch": 0.8865508326820177, + "flos": 19099449843840.0, + "grad_norm": 1.8637595976292314, + "language_loss": 0.75011015, + "learning_rate": 1.333887063813597e-07, + "loss": 0.77175272, + "num_input_tokens_seen": 159466870, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.15393066, + "step": 7373, + "time_per_iteration": 2.7355387210845947 + }, + { + "auxiliary_loss_clip": 0.01122335, + "auxiliary_loss_mlp": 0.01037013, + "balance_loss_clip": 1.03923333, + "balance_loss_mlp": 1.02304196, + "epoch": 0.8866710755726568, + "flos": 15414189240960.0, + "grad_norm": 1.8794627229627319, + "language_loss": 0.66478324, + "learning_rate": 1.331091325280278e-07, + "loss": 0.68637669, + "num_input_tokens_seen": 159485840, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.13970947, + "step": 7374, + "time_per_iteration": 2.768803358078003 + }, + { + "auxiliary_loss_clip": 0.01123612, + "auxiliary_loss_mlp": 0.01033124, + "balance_loss_clip": 1.03900027, + "balance_loss_mlp": 1.01658988, + "epoch": 0.8867913184632958, + "flos": 20083689388800.0, + "grad_norm": 1.5574804975587224, + "language_loss": 0.7845462, + "learning_rate": 1.3282984188055625e-07, + "loss": 0.8061136, + "num_input_tokens_seen": 159505630, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.16540527, + "step": 7375, + "time_per_iteration": 2.750253677368164 + }, + { + "auxiliary_loss_clip": 0.01124698, + "auxiliary_loss_mlp": 0.01038695, + "balance_loss_clip": 1.04003263, + "balance_loss_mlp": 1.02392471, + "epoch": 0.8869115613539349, + "flos": 23365852588800.0, + "grad_norm": 2.1241671027930957, + "language_loss": 0.79647374, + "learning_rate": 1.3255083448131288e-07, + "loss": 0.81810766, + "num_input_tokens_seen": 159524675, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.14770508, + "step": 7376, + "time_per_iteration": 2.8002705574035645 + }, + { + "auxiliary_loss_clip": 0.01123668, + "auxiliary_loss_mlp": 0.01035999, + "balance_loss_clip": 1.03732514, + "balance_loss_mlp": 1.02043033, + "epoch": 0.8870318042445741, + "flos": 21286840371840.0, + "grad_norm": 1.9623839532909233, + "language_loss": 0.78938806, + "learning_rate": 1.3227211037262365e-07, + "loss": 0.81098473, + "num_input_tokens_seen": 159541915, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.15563965, + "step": 7377, + "time_per_iteration": 3.5461182594299316 + }, + { + "auxiliary_loss_clip": 0.01127388, + "auxiliary_loss_mlp": 0.01033099, + "balance_loss_clip": 1.03919494, + "balance_loss_mlp": 1.01735735, + "epoch": 0.8871520471352131, + "flos": 20010862563840.0, + "grad_norm": 1.9809107116883466, + "language_loss": 0.85342783, + "learning_rate": 1.319936695967696e-07, + "loss": 0.87503272, + "num_input_tokens_seen": 159559740, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.1574707, + "step": 7378, + "time_per_iteration": 2.8520641326904297 + }, + { + "auxiliary_loss_clip": 0.01133102, + "auxiliary_loss_mlp": 0.01042201, + "balance_loss_clip": 1.04116273, + "balance_loss_mlp": 1.02399826, + "epoch": 0.8872722900258522, + "flos": 22601422321920.0, + "grad_norm": 2.121580015329716, + "language_loss": 0.82423329, + "learning_rate": 1.3171551219599097e-07, + "loss": 0.84598637, + "num_input_tokens_seen": 159578265, + "router_z_loss_clip": 0.91894531, + "router_z_loss_mlp": 0.18200684, + "step": 7379, + "time_per_iteration": 2.8588759899139404 + }, + { + "auxiliary_loss_clip": 0.01122331, + "auxiliary_loss_mlp": 0.01036608, + "balance_loss_clip": 1.03973258, + "balance_loss_mlp": 1.02122414, + "epoch": 0.8873925329164913, + "flos": 22163276223360.0, + "grad_norm": 1.9595809485426492, + "language_loss": 0.78055245, + "learning_rate": 1.3143763821248377e-07, + "loss": 0.80214185, + "num_input_tokens_seen": 159595350, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.1539917, + "step": 7380, + "time_per_iteration": 2.758493423461914 + }, + { + "auxiliary_loss_clip": 0.01124268, + "auxiliary_loss_mlp": 0.01038652, + "balance_loss_clip": 1.04144609, + "balance_loss_mlp": 1.02468109, + "epoch": 0.8875127758071304, + "flos": 19208223204480.0, + "grad_norm": 1.687064366563365, + "language_loss": 0.72322643, + "learning_rate": 1.3116004768840118e-07, + "loss": 0.74485564, + "num_input_tokens_seen": 159613725, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.13983154, + "step": 7381, + "time_per_iteration": 3.578788995742798 + }, + { + "auxiliary_loss_clip": 0.0112369, + "auxiliary_loss_mlp": 0.01034216, + "balance_loss_clip": 1.03823256, + "balance_loss_mlp": 1.01746678, + "epoch": 0.8876330186977694, + "flos": 18110900666880.0, + "grad_norm": 1.5717182933871026, + "language_loss": 0.73812628, + "learning_rate": 1.3088274066585348e-07, + "loss": 0.75970531, + "num_input_tokens_seen": 159631335, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.16748047, + "step": 7382, + "time_per_iteration": 2.919867753982544 + }, + { + "auxiliary_loss_clip": 0.01123375, + "auxiliary_loss_mlp": 0.01034117, + "balance_loss_clip": 1.03987026, + "balance_loss_mlp": 1.01932335, + "epoch": 0.8877532615884086, + "flos": 22009434272640.0, + "grad_norm": 2.0875152139848585, + "language_loss": 0.90333986, + "learning_rate": 1.3060571718690749e-07, + "loss": 0.92491472, + "num_input_tokens_seen": 159648830, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.14794922, + "step": 7383, + "time_per_iteration": 2.7827835083007812 + }, + { + "auxiliary_loss_clip": 0.01031884, + "auxiliary_loss_mlp": 0.0100139, + "balance_loss_clip": 1.00414968, + "balance_loss_mlp": 1.00016236, + "epoch": 0.8878735044790477, + "flos": 72136924346880.0, + "grad_norm": 0.7472220074680198, + "language_loss": 0.56952125, + "learning_rate": 1.3032897729358805e-07, + "loss": 0.589854, + "num_input_tokens_seen": 159709785, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01226807, + "step": 7384, + "time_per_iteration": 4.124742746353149 + }, + { + "auxiliary_loss_clip": 0.01125365, + "auxiliary_loss_mlp": 0.01037226, + "balance_loss_clip": 1.03904939, + "balance_loss_mlp": 1.0205487, + "epoch": 0.8879937473696867, + "flos": 27526355061120.0, + "grad_norm": 1.9807128306989237, + "language_loss": 0.79739189, + "learning_rate": 1.3005252102787645e-07, + "loss": 0.81901777, + "num_input_tokens_seen": 159728725, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.16687012, + "step": 7385, + "time_per_iteration": 2.8894693851470947 + }, + { + "auxiliary_loss_clip": 0.01127421, + "auxiliary_loss_mlp": 0.01038727, + "balance_loss_clip": 1.04175174, + "balance_loss_mlp": 1.02388525, + "epoch": 0.8881139902603259, + "flos": 22234091886720.0, + "grad_norm": 1.505629767729136, + "language_loss": 0.73524213, + "learning_rate": 1.297763484317105e-07, + "loss": 0.75690365, + "num_input_tokens_seen": 159747020, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.14825439, + "step": 7386, + "time_per_iteration": 2.798750400543213 + }, + { + "auxiliary_loss_clip": 0.01129254, + "auxiliary_loss_mlp": 0.01037614, + "balance_loss_clip": 1.04146647, + "balance_loss_mlp": 1.02243268, + "epoch": 0.888234233150965, + "flos": 20299548170880.0, + "grad_norm": 2.1693504683221803, + "language_loss": 0.70498037, + "learning_rate": 1.2950045954698551e-07, + "loss": 0.72664911, + "num_input_tokens_seen": 159764855, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.15179443, + "step": 7387, + "time_per_iteration": 3.7743003368377686 + }, + { + "auxiliary_loss_clip": 0.01121261, + "auxiliary_loss_mlp": 0.01033477, + "balance_loss_clip": 1.03911495, + "balance_loss_mlp": 1.01834917, + "epoch": 0.888354476041604, + "flos": 18147996437760.0, + "grad_norm": 1.536864104610313, + "language_loss": 0.75390363, + "learning_rate": 1.2922485441555343e-07, + "loss": 0.775451, + "num_input_tokens_seen": 159783935, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.15136719, + "step": 7388, + "time_per_iteration": 2.7806918621063232 + }, + { + "auxiliary_loss_clip": 0.01124015, + "auxiliary_loss_mlp": 0.01036272, + "balance_loss_clip": 1.03934836, + "balance_loss_mlp": 1.02131164, + "epoch": 0.8884747189322432, + "flos": 22014282608640.0, + "grad_norm": 1.9593834764321758, + "language_loss": 0.81557059, + "learning_rate": 1.2894953307922363e-07, + "loss": 0.83717346, + "num_input_tokens_seen": 159802895, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.14978027, + "step": 7389, + "time_per_iteration": 2.794994592666626 + }, + { + "auxiliary_loss_clip": 0.01126299, + "auxiliary_loss_mlp": 0.01044432, + "balance_loss_clip": 1.04080486, + "balance_loss_mlp": 1.02900577, + "epoch": 0.8885949618228822, + "flos": 19786779567360.0, + "grad_norm": 1.8282211501128487, + "language_loss": 0.83868313, + "learning_rate": 1.2867449557976208e-07, + "loss": 0.86039054, + "num_input_tokens_seen": 159820995, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.1541748, + "step": 7390, + "time_per_iteration": 2.830707550048828 + }, + { + "auxiliary_loss_clip": 0.01127551, + "auxiliary_loss_mlp": 0.01038207, + "balance_loss_clip": 1.04052925, + "balance_loss_mlp": 1.02254939, + "epoch": 0.8887152047135213, + "flos": 20047599198720.0, + "grad_norm": 2.013007645724607, + "language_loss": 0.75778818, + "learning_rate": 1.283997419588916e-07, + "loss": 0.77944577, + "num_input_tokens_seen": 159840465, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15643311, + "step": 7391, + "time_per_iteration": 2.8282740116119385 + }, + { + "auxiliary_loss_clip": 0.01125848, + "auxiliary_loss_mlp": 0.01039723, + "balance_loss_clip": 1.03869963, + "balance_loss_mlp": 1.02366519, + "epoch": 0.8888354476041604, + "flos": 18588117784320.0, + "grad_norm": 2.29348188885942, + "language_loss": 0.62216753, + "learning_rate": 1.2812527225829216e-07, + "loss": 0.64382321, + "num_input_tokens_seen": 159858690, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.16052246, + "step": 7392, + "time_per_iteration": 2.8185980319976807 + }, + { + "auxiliary_loss_clip": 0.01129662, + "auxiliary_loss_mlp": 0.01038744, + "balance_loss_clip": 1.04043031, + "balance_loss_mlp": 1.02154255, + "epoch": 0.8889556904947995, + "flos": 21689794120320.0, + "grad_norm": 2.578449843125877, + "language_loss": 0.76664221, + "learning_rate": 1.2785108651960052e-07, + "loss": 0.78832632, + "num_input_tokens_seen": 159880325, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.171875, + "step": 7393, + "time_per_iteration": 2.774432897567749 + }, + { + "auxiliary_loss_clip": 0.01126301, + "auxiliary_loss_mlp": 0.01039778, + "balance_loss_clip": 1.04084647, + "balance_loss_mlp": 1.02487087, + "epoch": 0.8890759333854386, + "flos": 27381204201600.0, + "grad_norm": 2.268250109278146, + "language_loss": 0.80361342, + "learning_rate": 1.2757718478441094e-07, + "loss": 0.82527423, + "num_input_tokens_seen": 159901070, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.14904785, + "step": 7394, + "time_per_iteration": 2.8267931938171387 + }, + { + "auxiliary_loss_clip": 0.01123339, + "auxiliary_loss_mlp": 0.01035334, + "balance_loss_clip": 1.03992534, + "balance_loss_mlp": 1.02037358, + "epoch": 0.8891961762760777, + "flos": 24498834353280.0, + "grad_norm": 1.7665073371118845, + "language_loss": 0.77066445, + "learning_rate": 1.2730356709427302e-07, + "loss": 0.79225123, + "num_input_tokens_seen": 159919750, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.14959717, + "step": 7395, + "time_per_iteration": 2.760791540145874 + }, + { + "auxiliary_loss_clip": 0.01124416, + "auxiliary_loss_mlp": 0.01038363, + "balance_loss_clip": 1.03914011, + "balance_loss_mlp": 1.02230573, + "epoch": 0.8893164191667168, + "flos": 41499770895360.0, + "grad_norm": 2.3608574929726913, + "language_loss": 0.60039937, + "learning_rate": 1.2703023349069542e-07, + "loss": 0.62202716, + "num_input_tokens_seen": 159944600, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.16052246, + "step": 7396, + "time_per_iteration": 2.910740375518799 + }, + { + "auxiliary_loss_clip": 0.01123669, + "auxiliary_loss_mlp": 0.01035311, + "balance_loss_clip": 1.03920662, + "balance_loss_mlp": 1.02056491, + "epoch": 0.8894366620573558, + "flos": 33583623120000.0, + "grad_norm": 1.646665885726525, + "language_loss": 0.61220634, + "learning_rate": 1.2675718401514223e-07, + "loss": 0.63379616, + "num_input_tokens_seen": 159968780, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.14758301, + "step": 7397, + "time_per_iteration": 2.812706708908081 + }, + { + "auxiliary_loss_clip": 0.01123848, + "auxiliary_loss_mlp": 0.0104643, + "balance_loss_clip": 1.03989959, + "balance_loss_mlp": 1.0313921, + "epoch": 0.889556904947995, + "flos": 16909832672640.0, + "grad_norm": 2.1268091106126588, + "language_loss": 0.74676228, + "learning_rate": 1.264844187090346e-07, + "loss": 0.76846504, + "num_input_tokens_seen": 159985905, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.15026855, + "step": 7398, + "time_per_iteration": 2.725343942642212 + }, + { + "auxiliary_loss_clip": 0.01122321, + "auxiliary_loss_mlp": 0.01032991, + "balance_loss_clip": 1.03823745, + "balance_loss_mlp": 1.01888895, + "epoch": 0.889677147838634, + "flos": 26030855283840.0, + "grad_norm": 1.6032737350439492, + "language_loss": 0.75191462, + "learning_rate": 1.262119376137516e-07, + "loss": 0.77346778, + "num_input_tokens_seen": 160006965, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14117432, + "step": 7399, + "time_per_iteration": 2.792862892150879 + }, + { + "auxiliary_loss_clip": 0.01122941, + "auxiliary_loss_mlp": 0.01041587, + "balance_loss_clip": 1.03918815, + "balance_loss_mlp": 1.02712703, + "epoch": 0.8897973907292731, + "flos": 26468283110400.0, + "grad_norm": 1.6623330333810187, + "language_loss": 0.85163009, + "learning_rate": 1.2593974077062707e-07, + "loss": 0.8732754, + "num_input_tokens_seen": 160028585, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.14453125, + "step": 7400, + "time_per_iteration": 2.844440221786499 + }, + { + "auxiliary_loss_clip": 0.0112266, + "auxiliary_loss_mlp": 0.01041781, + "balance_loss_clip": 1.03878784, + "balance_loss_mlp": 1.02603316, + "epoch": 0.8899176336199123, + "flos": 26249694894720.0, + "grad_norm": 1.6562793604812502, + "language_loss": 0.63521063, + "learning_rate": 1.2566782822095423e-07, + "loss": 0.65685499, + "num_input_tokens_seen": 160048840, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.15759277, + "step": 7401, + "time_per_iteration": 2.8389317989349365 + }, + { + "auxiliary_loss_clip": 0.0112631, + "auxiliary_loss_mlp": 0.01037193, + "balance_loss_clip": 1.0406673, + "balance_loss_mlp": 1.02112401, + "epoch": 0.8900378765105513, + "flos": 20811742156800.0, + "grad_norm": 1.8207820730552087, + "language_loss": 0.71241045, + "learning_rate": 1.2539620000598162e-07, + "loss": 0.73404545, + "num_input_tokens_seen": 160068175, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.16064453, + "step": 7402, + "time_per_iteration": 2.8456108570098877 + }, + { + "auxiliary_loss_clip": 0.01123788, + "auxiliary_loss_mlp": 0.01044263, + "balance_loss_clip": 1.03934252, + "balance_loss_mlp": 1.029356, + "epoch": 0.8901581194011904, + "flos": 16472333018880.0, + "grad_norm": 2.0597149249351214, + "language_loss": 0.79816592, + "learning_rate": 1.2512485616691492e-07, + "loss": 0.81984639, + "num_input_tokens_seen": 160085230, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.14910889, + "step": 7403, + "time_per_iteration": 2.8848700523376465 + }, + { + "auxiliary_loss_clip": 0.01132846, + "auxiliary_loss_mlp": 0.01044932, + "balance_loss_clip": 1.04176521, + "balance_loss_mlp": 1.02793312, + "epoch": 0.8902783622918296, + "flos": 35155253773440.0, + "grad_norm": 1.4677196751739476, + "language_loss": 0.8095383, + "learning_rate": 1.2485379674491681e-07, + "loss": 0.83131605, + "num_input_tokens_seen": 160111425, + "router_z_loss_clip": 0.91015625, + "router_z_loss_mlp": 0.17004395, + "step": 7404, + "time_per_iteration": 2.9020233154296875 + }, + { + "auxiliary_loss_clip": 0.01127568, + "auxiliary_loss_mlp": 0.01041554, + "balance_loss_clip": 1.04033256, + "balance_loss_mlp": 1.02625906, + "epoch": 0.8903986051824686, + "flos": 17201068145280.0, + "grad_norm": 2.1557845706176986, + "language_loss": 0.79201984, + "learning_rate": 1.2458302178110657e-07, + "loss": 0.81371105, + "num_input_tokens_seen": 160129790, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.1529541, + "step": 7405, + "time_per_iteration": 2.7444329261779785 + }, + { + "auxiliary_loss_clip": 0.01117795, + "auxiliary_loss_mlp": 0.01031282, + "balance_loss_clip": 1.03768146, + "balance_loss_mlp": 1.01771605, + "epoch": 0.8905188480731077, + "flos": 25483863997440.0, + "grad_norm": 1.8604633302717617, + "language_loss": 0.82583272, + "learning_rate": 1.2431253131656118e-07, + "loss": 0.84732348, + "num_input_tokens_seen": 160149265, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.13568115, + "step": 7406, + "time_per_iteration": 3.580176830291748 + }, + { + "auxiliary_loss_clip": 0.01124751, + "auxiliary_loss_mlp": 0.01035246, + "balance_loss_clip": 1.0381279, + "balance_loss_mlp": 1.01959336, + "epoch": 0.8906390909637467, + "flos": 23365888502400.0, + "grad_norm": 1.6374513676359324, + "language_loss": 0.7682355, + "learning_rate": 1.240423253923133e-07, + "loss": 0.78983545, + "num_input_tokens_seen": 160168870, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15661621, + "step": 7407, + "time_per_iteration": 2.7679669857025146 + }, + { + "auxiliary_loss_clip": 0.01130978, + "auxiliary_loss_mlp": 0.01038214, + "balance_loss_clip": 1.04311156, + "balance_loss_mlp": 1.02147722, + "epoch": 0.8907593338543859, + "flos": 21068790860160.0, + "grad_norm": 4.388661892004906, + "language_loss": 0.69033992, + "learning_rate": 1.237724040493533e-07, + "loss": 0.71203184, + "num_input_tokens_seen": 160187495, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.1673584, + "step": 7408, + "time_per_iteration": 2.861473560333252 + }, + { + "auxiliary_loss_clip": 0.01131289, + "auxiliary_loss_mlp": 0.01041995, + "balance_loss_clip": 1.04189324, + "balance_loss_mlp": 1.02558601, + "epoch": 0.8908795767450249, + "flos": 21869562712320.0, + "grad_norm": 2.1566451175183605, + "language_loss": 0.72843009, + "learning_rate": 1.2350276732862773e-07, + "loss": 0.75016296, + "num_input_tokens_seen": 160208520, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.16424561, + "step": 7409, + "time_per_iteration": 2.783607006072998 + }, + { + "auxiliary_loss_clip": 0.01031881, + "auxiliary_loss_mlp": 0.01002175, + "balance_loss_clip": 1.00407553, + "balance_loss_mlp": 1.00082207, + "epoch": 0.890999819635664, + "flos": 66307869348480.0, + "grad_norm": 0.8650417639529704, + "language_loss": 0.56732428, + "learning_rate": 1.2323341527103993e-07, + "loss": 0.58766484, + "num_input_tokens_seen": 160263720, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.0135498, + "step": 7410, + "time_per_iteration": 4.016891241073608 + }, + { + "auxiliary_loss_clip": 0.01125316, + "auxiliary_loss_mlp": 0.01030149, + "balance_loss_clip": 1.0398519, + "balance_loss_mlp": 1.01527154, + "epoch": 0.8911200625263032, + "flos": 26869908055680.0, + "grad_norm": 2.253027714980059, + "language_loss": 0.84776437, + "learning_rate": 1.2296434791745135e-07, + "loss": 0.86931896, + "num_input_tokens_seen": 160282170, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.14880371, + "step": 7411, + "time_per_iteration": 2.8157236576080322 + }, + { + "auxiliary_loss_clip": 0.01127397, + "auxiliary_loss_mlp": 0.01036289, + "balance_loss_clip": 1.04021406, + "balance_loss_mlp": 1.02077436, + "epoch": 0.8912403054169422, + "flos": 20885825957760.0, + "grad_norm": 1.6683445513889419, + "language_loss": 0.7639606, + "learning_rate": 1.2269556530867875e-07, + "loss": 0.78559744, + "num_input_tokens_seen": 160300725, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15509033, + "step": 7412, + "time_per_iteration": 2.759831666946411 + }, + { + "auxiliary_loss_clip": 0.01132212, + "auxiliary_loss_mlp": 0.01040694, + "balance_loss_clip": 1.04210269, + "balance_loss_mlp": 1.02375495, + "epoch": 0.8913605483075813, + "flos": 27016567286400.0, + "grad_norm": 1.8514880497637953, + "language_loss": 0.81851423, + "learning_rate": 1.2242706748549614e-07, + "loss": 0.84024328, + "num_input_tokens_seen": 160318720, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.16931152, + "step": 7413, + "time_per_iteration": 3.6390771865844727 + }, + { + "auxiliary_loss_clip": 0.01122925, + "auxiliary_loss_mlp": 0.01040405, + "balance_loss_clip": 1.03708959, + "balance_loss_mlp": 1.02528942, + "epoch": 0.8914807911982204, + "flos": 23621500661760.0, + "grad_norm": 1.789673068923594, + "language_loss": 0.82213074, + "learning_rate": 1.2215885448863473e-07, + "loss": 0.84376401, + "num_input_tokens_seen": 160339595, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15100098, + "step": 7414, + "time_per_iteration": 2.876784324645996 + }, + { + "auxiliary_loss_clip": 0.01127588, + "auxiliary_loss_mlp": 0.01037759, + "balance_loss_clip": 1.04070282, + "balance_loss_mlp": 1.02322698, + "epoch": 0.8916010340888595, + "flos": 24462277286400.0, + "grad_norm": 2.385562411169223, + "language_loss": 0.80359763, + "learning_rate": 1.2189092635878152e-07, + "loss": 0.8252511, + "num_input_tokens_seen": 160361045, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.14538574, + "step": 7415, + "time_per_iteration": 2.798997640609741 + }, + { + "auxiliary_loss_clip": 0.01125877, + "auxiliary_loss_mlp": 0.01033862, + "balance_loss_clip": 1.04082358, + "balance_loss_mlp": 1.01944947, + "epoch": 0.8917212769794985, + "flos": 21215773313280.0, + "grad_norm": 1.5838886059702824, + "language_loss": 0.77647567, + "learning_rate": 1.216232831365822e-07, + "loss": 0.79807305, + "num_input_tokens_seen": 160379990, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.14416504, + "step": 7416, + "time_per_iteration": 3.672441005706787 + }, + { + "auxiliary_loss_clip": 0.01125157, + "auxiliary_loss_mlp": 0.01040032, + "balance_loss_clip": 1.04109764, + "balance_loss_mlp": 1.02498794, + "epoch": 0.8918415198701377, + "flos": 25513992529920.0, + "grad_norm": 1.9612621818513876, + "language_loss": 0.80686504, + "learning_rate": 1.2135592486263678e-07, + "loss": 0.82851696, + "num_input_tokens_seen": 160399240, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.15057373, + "step": 7417, + "time_per_iteration": 2.7771146297454834 + }, + { + "auxiliary_loss_clip": 0.01122554, + "auxiliary_loss_mlp": 0.01032256, + "balance_loss_clip": 1.03783023, + "balance_loss_mlp": 1.01718843, + "epoch": 0.8919617627607768, + "flos": 37853006693760.0, + "grad_norm": 1.5828659517382824, + "language_loss": 0.61394501, + "learning_rate": 1.2108885157750415e-07, + "loss": 0.6354931, + "num_input_tokens_seen": 160421600, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15063477, + "step": 7418, + "time_per_iteration": 2.8744583129882812 + }, + { + "auxiliary_loss_clip": 0.01124018, + "auxiliary_loss_mlp": 0.01044581, + "balance_loss_clip": 1.04097867, + "balance_loss_mlp": 1.02974582, + "epoch": 0.8920820056514158, + "flos": 26213676531840.0, + "grad_norm": 1.6707104247963682, + "language_loss": 0.803146, + "learning_rate": 1.2082206332169897e-07, + "loss": 0.82483196, + "num_input_tokens_seen": 160441695, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.1484375, + "step": 7419, + "time_per_iteration": 2.8736205101013184 + }, + { + "auxiliary_loss_clip": 0.01120752, + "auxiliary_loss_mlp": 0.01036386, + "balance_loss_clip": 1.03750515, + "balance_loss_mlp": 1.02172279, + "epoch": 0.892202248542055, + "flos": 17383135207680.0, + "grad_norm": 2.20831619462881, + "language_loss": 0.73623133, + "learning_rate": 1.2055556013569225e-07, + "loss": 0.75780272, + "num_input_tokens_seen": 160457205, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.14660645, + "step": 7420, + "time_per_iteration": 2.7874016761779785 + }, + { + "auxiliary_loss_clip": 0.01121472, + "auxiliary_loss_mlp": 0.01037951, + "balance_loss_clip": 1.03787124, + "balance_loss_mlp": 1.0241766, + "epoch": 0.892322491432694, + "flos": 21324223451520.0, + "grad_norm": 1.582220641879877, + "language_loss": 0.82094586, + "learning_rate": 1.2028934205991315e-07, + "loss": 0.84254014, + "num_input_tokens_seen": 160476525, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.13763428, + "step": 7421, + "time_per_iteration": 2.8322629928588867 + }, + { + "auxiliary_loss_clip": 0.01124409, + "auxiliary_loss_mlp": 0.01039807, + "balance_loss_clip": 1.03998435, + "balance_loss_mlp": 1.02530527, + "epoch": 0.8924427343233331, + "flos": 24029374573440.0, + "grad_norm": 1.4049372541540053, + "language_loss": 0.76764858, + "learning_rate": 1.2002340913474607e-07, + "loss": 0.78929073, + "num_input_tokens_seen": 160500160, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.1451416, + "step": 7422, + "time_per_iteration": 2.821415424346924 + }, + { + "auxiliary_loss_clip": 0.01126665, + "auxiliary_loss_mlp": 0.01039964, + "balance_loss_clip": 1.0396173, + "balance_loss_mlp": 1.02399015, + "epoch": 0.8925629772139723, + "flos": 30008069631360.0, + "grad_norm": 2.239724545539439, + "language_loss": 0.74354708, + "learning_rate": 1.1975776140053317e-07, + "loss": 0.76521337, + "num_input_tokens_seen": 160520130, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.15979004, + "step": 7423, + "time_per_iteration": 3.103224277496338 + }, + { + "auxiliary_loss_clip": 0.01131349, + "auxiliary_loss_mlp": 0.01041779, + "balance_loss_clip": 1.04120481, + "balance_loss_mlp": 1.026425, + "epoch": 0.8926832201046113, + "flos": 22601709630720.0, + "grad_norm": 1.9483160039145846, + "language_loss": 0.73241544, + "learning_rate": 1.194923988975729e-07, + "loss": 0.75414675, + "num_input_tokens_seen": 160539730, + "router_z_loss_clip": 0.90087891, + "router_z_loss_mlp": 0.15344238, + "step": 7424, + "time_per_iteration": 2.854858875274658 + }, + { + "auxiliary_loss_clip": 0.01125637, + "auxiliary_loss_mlp": 0.01030103, + "balance_loss_clip": 1.03845787, + "balance_loss_mlp": 1.01439166, + "epoch": 0.8928034629952504, + "flos": 13297722117120.0, + "grad_norm": 2.32148073134527, + "language_loss": 0.73545945, + "learning_rate": 1.192273216661206e-07, + "loss": 0.75701684, + "num_input_tokens_seen": 160557820, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15716553, + "step": 7425, + "time_per_iteration": 2.7942605018615723 + }, + { + "auxiliary_loss_clip": 0.01032028, + "auxiliary_loss_mlp": 0.01002939, + "balance_loss_clip": 1.0044173, + "balance_loss_mlp": 1.00166941, + "epoch": 0.8929237058858895, + "flos": 54854556744960.0, + "grad_norm": 0.7705516947878678, + "language_loss": 0.57449198, + "learning_rate": 1.189625297463881e-07, + "loss": 0.59484166, + "num_input_tokens_seen": 160619510, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01269531, + "step": 7426, + "time_per_iteration": 3.2694342136383057 + }, + { + "auxiliary_loss_clip": 0.01119903, + "auxiliary_loss_mlp": 0.01034125, + "balance_loss_clip": 1.03855276, + "balance_loss_mlp": 1.01886582, + "epoch": 0.8930439487765286, + "flos": 28883850785280.0, + "grad_norm": 1.6035518749120814, + "language_loss": 0.79646575, + "learning_rate": 1.1869802317854394e-07, + "loss": 0.81800604, + "num_input_tokens_seen": 160643295, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.15258789, + "step": 7427, + "time_per_iteration": 2.855865716934204 + }, + { + "auxiliary_loss_clip": 0.01125763, + "auxiliary_loss_mlp": 0.01036978, + "balance_loss_clip": 1.03914082, + "balance_loss_mlp": 1.02143264, + "epoch": 0.8931641916671677, + "flos": 22419283432320.0, + "grad_norm": 2.1226575133559864, + "language_loss": 0.71860003, + "learning_rate": 1.1843380200271425e-07, + "loss": 0.74022746, + "num_input_tokens_seen": 160662495, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15539551, + "step": 7428, + "time_per_iteration": 2.8631834983825684 + }, + { + "auxiliary_loss_clip": 0.01126881, + "auxiliary_loss_mlp": 0.01036865, + "balance_loss_clip": 1.04052401, + "balance_loss_mlp": 1.02085567, + "epoch": 0.8932844345578068, + "flos": 25843149786240.0, + "grad_norm": 1.6286104854606895, + "language_loss": 0.80745071, + "learning_rate": 1.181698662589805e-07, + "loss": 0.82908821, + "num_input_tokens_seen": 160682080, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.15991211, + "step": 7429, + "time_per_iteration": 2.8904387950897217 + }, + { + "auxiliary_loss_clip": 0.01124735, + "auxiliary_loss_mlp": 0.01034335, + "balance_loss_clip": 1.04010201, + "balance_loss_mlp": 1.0184201, + "epoch": 0.8934046774484459, + "flos": 22925803069440.0, + "grad_norm": 1.747291545801822, + "language_loss": 0.7607708, + "learning_rate": 1.1790621598738249e-07, + "loss": 0.78236151, + "num_input_tokens_seen": 160700395, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15924072, + "step": 7430, + "time_per_iteration": 2.7535619735717773 + }, + { + "auxiliary_loss_clip": 0.01120735, + "auxiliary_loss_mlp": 0.01031403, + "balance_loss_clip": 1.03980839, + "balance_loss_mlp": 1.01819491, + "epoch": 0.8935249203390849, + "flos": 24462097718400.0, + "grad_norm": 1.9631612025041292, + "language_loss": 0.74960995, + "learning_rate": 1.1764285122791461e-07, + "loss": 0.77113134, + "num_input_tokens_seen": 160721115, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.13214111, + "step": 7431, + "time_per_iteration": 2.830164909362793 + }, + { + "auxiliary_loss_clip": 0.01122276, + "auxiliary_loss_mlp": 0.01039178, + "balance_loss_clip": 1.03828192, + "balance_loss_mlp": 1.02358568, + "epoch": 0.8936451632297241, + "flos": 15742735966080.0, + "grad_norm": 1.853732094380243, + "language_loss": 0.77252388, + "learning_rate": 1.173797720205294e-07, + "loss": 0.79413843, + "num_input_tokens_seen": 160739150, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15600586, + "step": 7432, + "time_per_iteration": 2.7081212997436523 + }, + { + "auxiliary_loss_clip": 0.01129555, + "auxiliary_loss_mlp": 0.01044615, + "balance_loss_clip": 1.04095018, + "balance_loss_mlp": 1.02778244, + "epoch": 0.8937654061203631, + "flos": 35115500396160.0, + "grad_norm": 2.48899740367621, + "language_loss": 0.72067022, + "learning_rate": 1.1711697840513602e-07, + "loss": 0.74241191, + "num_input_tokens_seen": 160758585, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.16821289, + "step": 7433, + "time_per_iteration": 2.8655221462249756 + }, + { + "auxiliary_loss_clip": 0.01120865, + "auxiliary_loss_mlp": 0.01030237, + "balance_loss_clip": 1.03850746, + "balance_loss_mlp": 1.01653969, + "epoch": 0.8938856490110022, + "flos": 16107444708480.0, + "grad_norm": 2.1042355093331206, + "language_loss": 0.71162635, + "learning_rate": 1.1685447042160012e-07, + "loss": 0.73313737, + "num_input_tokens_seen": 160776620, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.13708496, + "step": 7434, + "time_per_iteration": 2.7645435333251953 + }, + { + "auxiliary_loss_clip": 0.01127291, + "auxiliary_loss_mlp": 0.01037394, + "balance_loss_clip": 1.03988099, + "balance_loss_mlp": 1.02274954, + "epoch": 0.8940058919016414, + "flos": 20704189858560.0, + "grad_norm": 1.5423863557274105, + "language_loss": 0.71763825, + "learning_rate": 1.1659224810974367e-07, + "loss": 0.73928505, + "num_input_tokens_seen": 160796580, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.14654541, + "step": 7435, + "time_per_iteration": 3.4483864307403564 + }, + { + "auxiliary_loss_clip": 0.01126627, + "auxiliary_loss_mlp": 0.01035581, + "balance_loss_clip": 1.04098201, + "balance_loss_mlp": 1.02109122, + "epoch": 0.8941261347922804, + "flos": 25229041937280.0, + "grad_norm": 1.3471590807191756, + "language_loss": 0.68525857, + "learning_rate": 1.1633031150934591e-07, + "loss": 0.70688069, + "num_input_tokens_seen": 160819610, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.1449585, + "step": 7436, + "time_per_iteration": 2.834566116333008 + }, + { + "auxiliary_loss_clip": 0.01127886, + "auxiliary_loss_mlp": 0.0104246, + "balance_loss_clip": 1.04211009, + "balance_loss_mlp": 1.02703476, + "epoch": 0.8942463776829195, + "flos": 19537236806400.0, + "grad_norm": 1.9450801442738004, + "language_loss": 0.79766512, + "learning_rate": 1.1606866066014176e-07, + "loss": 0.81936854, + "num_input_tokens_seen": 160838660, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15405273, + "step": 7437, + "time_per_iteration": 2.853698968887329 + }, + { + "auxiliary_loss_clip": 0.01124827, + "auxiliary_loss_mlp": 0.01035526, + "balance_loss_clip": 1.03915644, + "balance_loss_mlp": 1.0198853, + "epoch": 0.8943666205735585, + "flos": 22301567585280.0, + "grad_norm": 2.1100251408469326, + "language_loss": 0.75432372, + "learning_rate": 1.1580729560182434e-07, + "loss": 0.77592719, + "num_input_tokens_seen": 160854515, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15649414, + "step": 7438, + "time_per_iteration": 2.855865001678467 + }, + { + "auxiliary_loss_clip": 0.01122188, + "auxiliary_loss_mlp": 0.01041983, + "balance_loss_clip": 1.03795671, + "balance_loss_mlp": 1.02707028, + "epoch": 0.8944868634641977, + "flos": 18912893581440.0, + "grad_norm": 1.6811494474287838, + "language_loss": 0.70802665, + "learning_rate": 1.1554621637404171e-07, + "loss": 0.72966838, + "num_input_tokens_seen": 160872605, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14898682, + "step": 7439, + "time_per_iteration": 3.6431162357330322 + }, + { + "auxiliary_loss_clip": 0.01124887, + "auxiliary_loss_mlp": 0.01039247, + "balance_loss_clip": 1.03941846, + "balance_loss_mlp": 1.02546644, + "epoch": 0.8946071063548368, + "flos": 14460904241280.0, + "grad_norm": 2.0173170012080024, + "language_loss": 0.61300111, + "learning_rate": 1.1528542301639999e-07, + "loss": 0.63464248, + "num_input_tokens_seen": 160889395, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.13806152, + "step": 7440, + "time_per_iteration": 2.7940316200256348 + }, + { + "auxiliary_loss_clip": 0.01126831, + "auxiliary_loss_mlp": 0.01044329, + "balance_loss_clip": 1.03755927, + "balance_loss_mlp": 1.02772367, + "epoch": 0.8947273492454758, + "flos": 20084084438400.0, + "grad_norm": 2.1732485841497002, + "language_loss": 0.823273, + "learning_rate": 1.1502491556846105e-07, + "loss": 0.84498465, + "num_input_tokens_seen": 160907890, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.16601562, + "step": 7441, + "time_per_iteration": 2.8204710483551025 + }, + { + "auxiliary_loss_clip": 0.01122724, + "auxiliary_loss_mlp": 0.01041554, + "balance_loss_clip": 1.038306, + "balance_loss_mlp": 1.02630758, + "epoch": 0.894847592136115, + "flos": 18550555136640.0, + "grad_norm": 2.3785359602740352, + "language_loss": 0.81256413, + "learning_rate": 1.1476469406974331e-07, + "loss": 0.83420694, + "num_input_tokens_seen": 160923490, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.15252686, + "step": 7442, + "time_per_iteration": 3.5348238945007324 + }, + { + "auxiliary_loss_clip": 0.01124847, + "auxiliary_loss_mlp": 0.01032292, + "balance_loss_clip": 1.0410769, + "balance_loss_mlp": 1.01805806, + "epoch": 0.894967835026754, + "flos": 23478468704640.0, + "grad_norm": 2.061663574472331, + "language_loss": 0.77216661, + "learning_rate": 1.1450475855972341e-07, + "loss": 0.79373801, + "num_input_tokens_seen": 160944280, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.14233398, + "step": 7443, + "time_per_iteration": 2.8551084995269775 + }, + { + "auxiliary_loss_clip": 0.01126754, + "auxiliary_loss_mlp": 0.01036588, + "balance_loss_clip": 1.04136753, + "balance_loss_mlp": 1.0205785, + "epoch": 0.8950880779173931, + "flos": 15188310564480.0, + "grad_norm": 2.1048748247169984, + "language_loss": 0.7069028, + "learning_rate": 1.1424510907783158e-07, + "loss": 0.72853619, + "num_input_tokens_seen": 160961560, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.16015625, + "step": 7444, + "time_per_iteration": 2.9033474922180176 + }, + { + "auxiliary_loss_clip": 0.01122373, + "auxiliary_loss_mlp": 0.01038706, + "balance_loss_clip": 1.03793311, + "balance_loss_mlp": 1.02362573, + "epoch": 0.8952083208080323, + "flos": 22091957769600.0, + "grad_norm": 1.5756830501778796, + "language_loss": 0.82673717, + "learning_rate": 1.1398574566345787e-07, + "loss": 0.8483479, + "num_input_tokens_seen": 160982195, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15063477, + "step": 7445, + "time_per_iteration": 3.7346034049987793 + }, + { + "auxiliary_loss_clip": 0.01128436, + "auxiliary_loss_mlp": 0.01037054, + "balance_loss_clip": 1.03939211, + "balance_loss_mlp": 1.02085352, + "epoch": 0.8953285636986713, + "flos": 23254026572160.0, + "grad_norm": 2.113614046231425, + "language_loss": 0.82370055, + "learning_rate": 1.1372666835594702e-07, + "loss": 0.84535545, + "num_input_tokens_seen": 161000520, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.16210938, + "step": 7446, + "time_per_iteration": 2.8119897842407227 + }, + { + "auxiliary_loss_clip": 0.01122486, + "auxiliary_loss_mlp": 0.01030236, + "balance_loss_clip": 1.03850698, + "balance_loss_mlp": 1.0157882, + "epoch": 0.8954488065893104, + "flos": 16362661818240.0, + "grad_norm": 1.8836359582779925, + "language_loss": 0.71825635, + "learning_rate": 1.1346787719460071e-07, + "loss": 0.73978353, + "num_input_tokens_seen": 161019405, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14453125, + "step": 7447, + "time_per_iteration": 2.783510446548462 + }, + { + "auxiliary_loss_clip": 0.01124459, + "auxiliary_loss_mlp": 0.01036574, + "balance_loss_clip": 1.03910422, + "balance_loss_mlp": 1.02102935, + "epoch": 0.8955690494799495, + "flos": 18257883120000.0, + "grad_norm": 1.6582342725628305, + "language_loss": 0.72196317, + "learning_rate": 1.1320937221867732e-07, + "loss": 0.74357355, + "num_input_tokens_seen": 161036985, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.15539551, + "step": 7448, + "time_per_iteration": 2.851041555404663 + }, + { + "auxiliary_loss_clip": 0.01119741, + "auxiliary_loss_mlp": 0.01034496, + "balance_loss_clip": 1.03964698, + "balance_loss_mlp": 1.01998222, + "epoch": 0.8956892923705886, + "flos": 25447486498560.0, + "grad_norm": 1.7679013693628212, + "language_loss": 0.79609716, + "learning_rate": 1.1295115346739192e-07, + "loss": 0.81763953, + "num_input_tokens_seen": 161056985, + "router_z_loss_clip": 0.80175781, + "router_z_loss_mlp": 0.1451416, + "step": 7449, + "time_per_iteration": 2.832401990890503 + }, + { + "auxiliary_loss_clip": 0.01127733, + "auxiliary_loss_mlp": 0.01033028, + "balance_loss_clip": 1.04059887, + "balance_loss_mlp": 1.01689935, + "epoch": 0.8958095352612276, + "flos": 52661883939840.0, + "grad_norm": 3.299184344202994, + "language_loss": 0.72871876, + "learning_rate": 1.1269322097991629e-07, + "loss": 0.7503264, + "num_input_tokens_seen": 161080270, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.16137695, + "step": 7450, + "time_per_iteration": 3.0270590782165527 + }, + { + "auxiliary_loss_clip": 0.0113055, + "auxiliary_loss_mlp": 0.01045974, + "balance_loss_clip": 1.04084635, + "balance_loss_mlp": 1.02993441, + "epoch": 0.8959297781518668, + "flos": 23186335392000.0, + "grad_norm": 1.916401914112907, + "language_loss": 0.67888427, + "learning_rate": 1.1243557479537846e-07, + "loss": 0.7006495, + "num_input_tokens_seen": 161100160, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.16027832, + "step": 7451, + "time_per_iteration": 2.7746760845184326 + }, + { + "auxiliary_loss_clip": 0.01125883, + "auxiliary_loss_mlp": 0.01039088, + "balance_loss_clip": 1.0395968, + "balance_loss_mlp": 1.02307844, + "epoch": 0.8960500210425059, + "flos": 20334309557760.0, + "grad_norm": 2.0839254393664577, + "language_loss": 0.69051939, + "learning_rate": 1.121782149528634e-07, + "loss": 0.71216905, + "num_input_tokens_seen": 161117260, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.16015625, + "step": 7452, + "time_per_iteration": 2.777162551879883 + }, + { + "auxiliary_loss_clip": 0.0113047, + "auxiliary_loss_mlp": 0.01037578, + "balance_loss_clip": 1.04225564, + "balance_loss_mlp": 1.02250981, + "epoch": 0.8961702639331449, + "flos": 19901694153600.0, + "grad_norm": 1.9851898132212216, + "language_loss": 0.79022765, + "learning_rate": 1.1192114149141208e-07, + "loss": 0.81190813, + "num_input_tokens_seen": 161136895, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.1506958, + "step": 7453, + "time_per_iteration": 2.817366600036621 + }, + { + "auxiliary_loss_clip": 0.01130695, + "auxiliary_loss_mlp": 0.01046245, + "balance_loss_clip": 1.04075491, + "balance_loss_mlp": 1.0294838, + "epoch": 0.8962905068237841, + "flos": 12896348567040.0, + "grad_norm": 2.1481320671062445, + "language_loss": 0.65485954, + "learning_rate": 1.1166435445002197e-07, + "loss": 0.67662895, + "num_input_tokens_seen": 161154565, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.1673584, + "step": 7454, + "time_per_iteration": 2.7963390350341797 + }, + { + "auxiliary_loss_clip": 0.01129149, + "auxiliary_loss_mlp": 0.0103368, + "balance_loss_clip": 1.04179692, + "balance_loss_mlp": 1.01831365, + "epoch": 0.8964107497144231, + "flos": 23440331439360.0, + "grad_norm": 2.0655385901158283, + "language_loss": 0.68251514, + "learning_rate": 1.1140785386764818e-07, + "loss": 0.7041434, + "num_input_tokens_seen": 161173265, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.15350342, + "step": 7455, + "time_per_iteration": 2.8503105640411377 + }, + { + "auxiliary_loss_clip": 0.01125129, + "auxiliary_loss_mlp": 0.01046822, + "balance_loss_clip": 1.040766, + "balance_loss_mlp": 1.03164685, + "epoch": 0.8965309926050622, + "flos": 19500176949120.0, + "grad_norm": 1.9604531425243545, + "language_loss": 0.69878519, + "learning_rate": 1.1115163978320153e-07, + "loss": 0.7205047, + "num_input_tokens_seen": 161191995, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.15185547, + "step": 7456, + "time_per_iteration": 2.7525634765625 + }, + { + "auxiliary_loss_clip": 0.0112616, + "auxiliary_loss_mlp": 0.01033, + "balance_loss_clip": 1.04038811, + "balance_loss_mlp": 1.01735938, + "epoch": 0.8966512354957014, + "flos": 28658008022400.0, + "grad_norm": 1.950338974571287, + "language_loss": 0.82230598, + "learning_rate": 1.1089571223554917e-07, + "loss": 0.84389758, + "num_input_tokens_seen": 161212880, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15649414, + "step": 7457, + "time_per_iteration": 2.8139984607696533 + }, + { + "auxiliary_loss_clip": 0.01123563, + "auxiliary_loss_mlp": 0.01036724, + "balance_loss_clip": 1.03856754, + "balance_loss_mlp": 1.02067888, + "epoch": 0.8967714783863404, + "flos": 23370916406400.0, + "grad_norm": 1.9447855429964434, + "language_loss": 0.85812515, + "learning_rate": 1.1064007126351537e-07, + "loss": 0.87972802, + "num_input_tokens_seen": 161233595, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.16027832, + "step": 7458, + "time_per_iteration": 2.753408432006836 + }, + { + "auxiliary_loss_clip": 0.0112231, + "auxiliary_loss_mlp": 0.01036819, + "balance_loss_clip": 1.03821254, + "balance_loss_mlp": 1.02123785, + "epoch": 0.8968917212769795, + "flos": 24535175938560.0, + "grad_norm": 2.011702873712943, + "language_loss": 0.76345515, + "learning_rate": 1.1038471690588003e-07, + "loss": 0.78504646, + "num_input_tokens_seen": 161252740, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15576172, + "step": 7459, + "time_per_iteration": 2.7566487789154053 + }, + { + "auxiliary_loss_clip": 0.01119648, + "auxiliary_loss_mlp": 0.01035184, + "balance_loss_clip": 1.03759849, + "balance_loss_mlp": 1.02083158, + "epoch": 0.8970119641676186, + "flos": 23475416048640.0, + "grad_norm": 1.794052885055755, + "language_loss": 0.80059582, + "learning_rate": 1.1012964920138145e-07, + "loss": 0.82214415, + "num_input_tokens_seen": 161272325, + "router_z_loss_clip": 0.82080078, + "router_z_loss_mlp": 0.14355469, + "step": 7460, + "time_per_iteration": 2.8438682556152344 + }, + { + "auxiliary_loss_clip": 0.0112517, + "auxiliary_loss_mlp": 0.01035855, + "balance_loss_clip": 1.03926528, + "balance_loss_mlp": 1.02142477, + "epoch": 0.8971322070582577, + "flos": 24538192680960.0, + "grad_norm": 1.666658173214514, + "language_loss": 0.7551167, + "learning_rate": 1.0987486818871205e-07, + "loss": 0.77672696, + "num_input_tokens_seen": 161295915, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.14416504, + "step": 7461, + "time_per_iteration": 2.827618360519409 + }, + { + "auxiliary_loss_clip": 0.01127988, + "auxiliary_loss_mlp": 0.01034297, + "balance_loss_clip": 1.04118478, + "balance_loss_mlp": 1.01953864, + "epoch": 0.8972524499488967, + "flos": 21797454159360.0, + "grad_norm": 2.3757136227863116, + "language_loss": 0.73157716, + "learning_rate": 1.0962037390652245e-07, + "loss": 0.75319999, + "num_input_tokens_seen": 161314935, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.14758301, + "step": 7462, + "time_per_iteration": 2.894774913787842 + }, + { + "auxiliary_loss_clip": 0.01126336, + "auxiliary_loss_mlp": 0.01040883, + "balance_loss_clip": 1.04065895, + "balance_loss_mlp": 1.02648222, + "epoch": 0.8973726928395359, + "flos": 21726243446400.0, + "grad_norm": 1.6400351476448112, + "language_loss": 0.71906728, + "learning_rate": 1.0936616639341911e-07, + "loss": 0.74073946, + "num_input_tokens_seen": 161335225, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.14398193, + "step": 7463, + "time_per_iteration": 2.831599473953247 + }, + { + "auxiliary_loss_clip": 0.0103219, + "auxiliary_loss_mlp": 0.01003831, + "balance_loss_clip": 1.00455642, + "balance_loss_mlp": 1.00251353, + "epoch": 0.897492935730175, + "flos": 53837100097920.0, + "grad_norm": 0.7436493851158689, + "language_loss": 0.54857719, + "learning_rate": 1.0911224568796473e-07, + "loss": 0.56893742, + "num_input_tokens_seen": 161393420, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01318359, + "step": 7464, + "time_per_iteration": 4.059817314147949 + }, + { + "auxiliary_loss_clip": 0.01121324, + "auxiliary_loss_mlp": 0.01041337, + "balance_loss_clip": 1.0399133, + "balance_loss_mlp": 1.02601838, + "epoch": 0.897613178620814, + "flos": 18290346036480.0, + "grad_norm": 2.0707855026628006, + "language_loss": 0.70713508, + "learning_rate": 1.0885861182867984e-07, + "loss": 0.72876167, + "num_input_tokens_seen": 161411525, + "router_z_loss_clip": 0.81445312, + "router_z_loss_mlp": 0.15313721, + "step": 7465, + "time_per_iteration": 2.8803598880767822 + }, + { + "auxiliary_loss_clip": 0.01124227, + "auxiliary_loss_mlp": 0.01035034, + "balance_loss_clip": 1.03974676, + "balance_loss_mlp": 1.0196557, + "epoch": 0.8977334215114532, + "flos": 32993718059520.0, + "grad_norm": 1.8004860502066282, + "language_loss": 0.70760345, + "learning_rate": 1.0860526485403942e-07, + "loss": 0.72919601, + "num_input_tokens_seen": 161432800, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.15368652, + "step": 7466, + "time_per_iteration": 2.8329412937164307 + }, + { + "auxiliary_loss_clip": 0.01125042, + "auxiliary_loss_mlp": 0.01040199, + "balance_loss_clip": 1.04121184, + "balance_loss_mlp": 1.02552462, + "epoch": 0.8978536644020922, + "flos": 15195636938880.0, + "grad_norm": 1.4951858912548885, + "language_loss": 0.77194244, + "learning_rate": 1.0835220480247675e-07, + "loss": 0.79359484, + "num_input_tokens_seen": 161451295, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.14672852, + "step": 7467, + "time_per_iteration": 2.805337429046631 + }, + { + "auxiliary_loss_clip": 0.01125813, + "auxiliary_loss_mlp": 0.01037888, + "balance_loss_clip": 1.04073572, + "balance_loss_mlp": 1.02289772, + "epoch": 0.8979739072927313, + "flos": 18004389863040.0, + "grad_norm": 2.3795804272510996, + "language_loss": 0.83951497, + "learning_rate": 1.0809943171238067e-07, + "loss": 0.86115205, + "num_input_tokens_seen": 161469220, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.14984131, + "step": 7468, + "time_per_iteration": 3.659581184387207 + }, + { + "auxiliary_loss_clip": 0.01133414, + "auxiliary_loss_mlp": 0.01045457, + "balance_loss_clip": 1.04142678, + "balance_loss_mlp": 1.0281831, + "epoch": 0.8980941501833704, + "flos": 22271546793600.0, + "grad_norm": 2.0479083471823, + "language_loss": 0.63066882, + "learning_rate": 1.078469456220965e-07, + "loss": 0.65245759, + "num_input_tokens_seen": 161489375, + "router_z_loss_clip": 0.91992188, + "router_z_loss_mlp": 0.17272949, + "step": 7469, + "time_per_iteration": 2.8340003490448 + }, + { + "auxiliary_loss_clip": 0.01122371, + "auxiliary_loss_mlp": 0.01037499, + "balance_loss_clip": 1.03946531, + "balance_loss_mlp": 1.02241933, + "epoch": 0.8982143930740095, + "flos": 37560729726720.0, + "grad_norm": 2.2662905289126756, + "language_loss": 0.69545436, + "learning_rate": 1.0759474656992606e-07, + "loss": 0.71705306, + "num_input_tokens_seen": 161512145, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.15075684, + "step": 7470, + "time_per_iteration": 2.9333529472351074 + }, + { + "auxiliary_loss_clip": 0.0112371, + "auxiliary_loss_mlp": 0.01040277, + "balance_loss_clip": 1.03934121, + "balance_loss_mlp": 1.02438617, + "epoch": 0.8983346359646486, + "flos": 18076893465600.0, + "grad_norm": 2.167408827375982, + "language_loss": 0.7812534, + "learning_rate": 1.0734283459412785e-07, + "loss": 0.80289328, + "num_input_tokens_seen": 161528995, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.15893555, + "step": 7471, + "time_per_iteration": 3.5355384349823 + }, + { + "auxiliary_loss_clip": 0.01128412, + "auxiliary_loss_mlp": 0.01040611, + "balance_loss_clip": 1.04009283, + "balance_loss_mlp": 1.02402902, + "epoch": 0.8984548788552876, + "flos": 20558895344640.0, + "grad_norm": 2.431296219726756, + "language_loss": 0.80427837, + "learning_rate": 1.0709120973291707e-07, + "loss": 0.82596862, + "num_input_tokens_seen": 161548775, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.16589355, + "step": 7472, + "time_per_iteration": 2.7697834968566895 + }, + { + "auxiliary_loss_clip": 0.01129921, + "auxiliary_loss_mlp": 0.01043243, + "balance_loss_clip": 1.04215097, + "balance_loss_mlp": 1.02752542, + "epoch": 0.8985751217459268, + "flos": 17785442511360.0, + "grad_norm": 2.246666831177516, + "language_loss": 0.77498436, + "learning_rate": 1.0683987202446475e-07, + "loss": 0.79671597, + "num_input_tokens_seen": 161566960, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.15704346, + "step": 7473, + "time_per_iteration": 2.8163979053497314 + }, + { + "auxiliary_loss_clip": 0.01126683, + "auxiliary_loss_mlp": 0.0103457, + "balance_loss_clip": 1.04085469, + "balance_loss_mlp": 1.01940036, + "epoch": 0.8986953646365659, + "flos": 21617003208960.0, + "grad_norm": 1.7219507562137149, + "language_loss": 0.69723642, + "learning_rate": 1.0658882150689862e-07, + "loss": 0.71884894, + "num_input_tokens_seen": 161585820, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15179443, + "step": 7474, + "time_per_iteration": 3.670870780944824 + }, + { + "auxiliary_loss_clip": 0.01127107, + "auxiliary_loss_mlp": 0.01045096, + "balance_loss_clip": 1.03951883, + "balance_loss_mlp": 1.0294137, + "epoch": 0.8988156075272049, + "flos": 14027355083520.0, + "grad_norm": 2.7935909931143397, + "language_loss": 0.78938246, + "learning_rate": 1.0633805821830288e-07, + "loss": 0.81110454, + "num_input_tokens_seen": 161602505, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.15679932, + "step": 7475, + "time_per_iteration": 2.8856894969940186 + }, + { + "auxiliary_loss_clip": 0.01127087, + "auxiliary_loss_mlp": 0.01033984, + "balance_loss_clip": 1.04062676, + "balance_loss_mlp": 1.01794434, + "epoch": 0.8989358504178441, + "flos": 29059202004480.0, + "grad_norm": 4.681581907136429, + "language_loss": 0.83016938, + "learning_rate": 1.0608758219671753e-07, + "loss": 0.85178006, + "num_input_tokens_seen": 161621545, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.16052246, + "step": 7476, + "time_per_iteration": 2.778895616531372 + }, + { + "auxiliary_loss_clip": 0.01126125, + "auxiliary_loss_mlp": 0.01036949, + "balance_loss_clip": 1.04124165, + "balance_loss_mlp": 1.02170205, + "epoch": 0.8990560933084831, + "flos": 20230420446720.0, + "grad_norm": 1.6953803549943511, + "language_loss": 0.7047143, + "learning_rate": 1.0583739348014065e-07, + "loss": 0.72634506, + "num_input_tokens_seen": 161642630, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.15240479, + "step": 7477, + "time_per_iteration": 2.8254945278167725 + }, + { + "auxiliary_loss_clip": 0.01126997, + "auxiliary_loss_mlp": 0.01042033, + "balance_loss_clip": 1.04245949, + "balance_loss_mlp": 1.02653575, + "epoch": 0.8991763361991222, + "flos": 25520672459520.0, + "grad_norm": 1.8027258065558274, + "language_loss": 0.84815443, + "learning_rate": 1.0558749210652518e-07, + "loss": 0.86984473, + "num_input_tokens_seen": 161662560, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.15484619, + "step": 7478, + "time_per_iteration": 2.7810702323913574 + }, + { + "auxiliary_loss_clip": 0.01125499, + "auxiliary_loss_mlp": 0.01037833, + "balance_loss_clip": 1.04088354, + "balance_loss_mlp": 1.02201366, + "epoch": 0.8992965790897613, + "flos": 25119191168640.0, + "grad_norm": 1.5634517126383327, + "language_loss": 0.8569243, + "learning_rate": 1.053378781137808e-07, + "loss": 0.87855762, + "num_input_tokens_seen": 161683480, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.15808105, + "step": 7479, + "time_per_iteration": 2.7604846954345703 + }, + { + "auxiliary_loss_clip": 0.01128353, + "auxiliary_loss_mlp": 0.01040488, + "balance_loss_clip": 1.04054213, + "balance_loss_mlp": 1.02418613, + "epoch": 0.8994168219804004, + "flos": 16070815814400.0, + "grad_norm": 1.763443378621589, + "language_loss": 0.7762835, + "learning_rate": 1.0508855153977392e-07, + "loss": 0.79797184, + "num_input_tokens_seen": 161699945, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.16290283, + "step": 7480, + "time_per_iteration": 2.887017011642456 + }, + { + "auxiliary_loss_clip": 0.01123505, + "auxiliary_loss_mlp": 0.01041519, + "balance_loss_clip": 1.03866506, + "balance_loss_mlp": 1.02565181, + "epoch": 0.8995370648710395, + "flos": 24825764966400.0, + "grad_norm": 2.6126229039943514, + "language_loss": 0.67169583, + "learning_rate": 1.0483951242232669e-07, + "loss": 0.69334608, + "num_input_tokens_seen": 161720420, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15881348, + "step": 7481, + "time_per_iteration": 2.799128532409668 + }, + { + "auxiliary_loss_clip": 0.01031343, + "auxiliary_loss_mlp": 0.0100295, + "balance_loss_clip": 1.00378168, + "balance_loss_mlp": 1.00172806, + "epoch": 0.8996573077616786, + "flos": 63116238378240.0, + "grad_norm": 0.9881305718088099, + "language_loss": 0.57747066, + "learning_rate": 1.0459076079921936e-07, + "loss": 0.59781361, + "num_input_tokens_seen": 161773080, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01220703, + "step": 7482, + "time_per_iteration": 3.377002000808716 + }, + { + "auxiliary_loss_clip": 0.01122688, + "auxiliary_loss_mlp": 0.01033791, + "balance_loss_clip": 1.03990161, + "balance_loss_mlp": 1.01906264, + "epoch": 0.8997775506523177, + "flos": 18219674027520.0, + "grad_norm": 2.471569964107993, + "language_loss": 0.85044712, + "learning_rate": 1.0434229670818618e-07, + "loss": 0.8720119, + "num_input_tokens_seen": 161789755, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.14715576, + "step": 7483, + "time_per_iteration": 2.774184465408325 + }, + { + "auxiliary_loss_clip": 0.01122696, + "auxiliary_loss_mlp": 0.01039512, + "balance_loss_clip": 1.03922009, + "balance_loss_mlp": 1.02582085, + "epoch": 0.8998977935429567, + "flos": 24166768095360.0, + "grad_norm": 1.5183617966028236, + "language_loss": 0.79792601, + "learning_rate": 1.0409412018691944e-07, + "loss": 0.81954801, + "num_input_tokens_seen": 161810220, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.13696289, + "step": 7484, + "time_per_iteration": 2.772090196609497 + }, + { + "auxiliary_loss_clip": 0.01127058, + "auxiliary_loss_mlp": 0.01042723, + "balance_loss_clip": 1.04037619, + "balance_loss_mlp": 1.02655804, + "epoch": 0.9000180364335959, + "flos": 20773030273920.0, + "grad_norm": 2.1061748790206316, + "language_loss": 0.7538048, + "learning_rate": 1.0384623127306724e-07, + "loss": 0.77550268, + "num_input_tokens_seen": 161827565, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.16162109, + "step": 7485, + "time_per_iteration": 2.8615541458129883 + }, + { + "auxiliary_loss_clip": 0.01124624, + "auxiliary_loss_mlp": 0.01047598, + "balance_loss_clip": 1.0401746, + "balance_loss_mlp": 1.03315568, + "epoch": 0.900138279324235, + "flos": 19205745166080.0, + "grad_norm": 1.6836567199749648, + "language_loss": 0.79280436, + "learning_rate": 1.0359863000423397e-07, + "loss": 0.81452656, + "num_input_tokens_seen": 161845700, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.14434814, + "step": 7486, + "time_per_iteration": 2.8389668464660645 + }, + { + "auxiliary_loss_clip": 0.01124424, + "auxiliary_loss_mlp": 0.01035142, + "balance_loss_clip": 1.03893209, + "balance_loss_mlp": 1.02075291, + "epoch": 0.900258522214874, + "flos": 28731158069760.0, + "grad_norm": 1.8192913703849747, + "language_loss": 0.72001046, + "learning_rate": 1.0335131641798112e-07, + "loss": 0.74160612, + "num_input_tokens_seen": 161867660, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.1439209, + "step": 7487, + "time_per_iteration": 2.898172378540039 + }, + { + "auxiliary_loss_clip": 0.0103178, + "auxiliary_loss_mlp": 0.01002373, + "balance_loss_clip": 1.00424576, + "balance_loss_mlp": 1.00125873, + "epoch": 0.9003787651055132, + "flos": 58280685655680.0, + "grad_norm": 0.80872886495334, + "language_loss": 0.55623019, + "learning_rate": 1.0310429055182512e-07, + "loss": 0.5765717, + "num_input_tokens_seen": 161921980, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01116943, + "step": 7488, + "time_per_iteration": 3.1883106231689453 + }, + { + "auxiliary_loss_clip": 0.01128866, + "auxiliary_loss_mlp": 0.01038326, + "balance_loss_clip": 1.04115713, + "balance_loss_mlp": 1.02228022, + "epoch": 0.9004990079961522, + "flos": 25556475340800.0, + "grad_norm": 1.9316517280525365, + "language_loss": 0.73307443, + "learning_rate": 1.0285755244324024e-07, + "loss": 0.75474632, + "num_input_tokens_seen": 161942725, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.16040039, + "step": 7489, + "time_per_iteration": 2.898653268814087 + }, + { + "auxiliary_loss_clip": 0.01123547, + "auxiliary_loss_mlp": 0.01035575, + "balance_loss_clip": 1.03921485, + "balance_loss_mlp": 1.02023256, + "epoch": 0.9006192508867913, + "flos": 23335185352320.0, + "grad_norm": 1.417514665332287, + "language_loss": 0.68755519, + "learning_rate": 1.0261110212965629e-07, + "loss": 0.70914638, + "num_input_tokens_seen": 161964520, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.15344238, + "step": 7490, + "time_per_iteration": 2.8598663806915283 + }, + { + "auxiliary_loss_clip": 0.01126033, + "auxiliary_loss_mlp": 0.01036299, + "balance_loss_clip": 1.04159737, + "balance_loss_mlp": 1.02150512, + "epoch": 0.9007394937774305, + "flos": 18040300485120.0, + "grad_norm": 1.941607560033383, + "language_loss": 0.79143524, + "learning_rate": 1.023649396484596e-07, + "loss": 0.81305856, + "num_input_tokens_seen": 161983575, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.14794922, + "step": 7491, + "time_per_iteration": 2.734558582305908 + }, + { + "auxiliary_loss_clip": 0.01125111, + "auxiliary_loss_mlp": 0.01034154, + "balance_loss_clip": 1.03954768, + "balance_loss_mlp": 1.01912165, + "epoch": 0.9008597366680695, + "flos": 43068456633600.0, + "grad_norm": 1.9920355670892933, + "language_loss": 0.68020505, + "learning_rate": 1.0211906503699275e-07, + "loss": 0.70179772, + "num_input_tokens_seen": 162006550, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15032959, + "step": 7492, + "time_per_iteration": 2.9662294387817383 + }, + { + "auxiliary_loss_clip": 0.0113083, + "auxiliary_loss_mlp": 0.01040237, + "balance_loss_clip": 1.04312074, + "balance_loss_mlp": 1.0245018, + "epoch": 0.9009799795587086, + "flos": 14939055112320.0, + "grad_norm": 2.4864957319947765, + "language_loss": 0.82532477, + "learning_rate": 1.0187347833255455e-07, + "loss": 0.84703547, + "num_input_tokens_seen": 162022455, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15734863, + "step": 7493, + "time_per_iteration": 3.4507839679718018 + }, + { + "auxiliary_loss_clip": 0.01123453, + "auxiliary_loss_mlp": 0.01036853, + "balance_loss_clip": 1.04090834, + "balance_loss_mlp": 1.02132559, + "epoch": 0.9011002224493477, + "flos": 21579584215680.0, + "grad_norm": 1.8519758166360651, + "language_loss": 0.79624343, + "learning_rate": 1.0162817957240056e-07, + "loss": 0.81784654, + "num_input_tokens_seen": 162042350, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.1552124, + "step": 7494, + "time_per_iteration": 2.799226999282837 + }, + { + "auxiliary_loss_clip": 0.0103172, + "auxiliary_loss_mlp": 0.0100361, + "balance_loss_clip": 1.00411737, + "balance_loss_mlp": 1.00237012, + "epoch": 0.9012204653399868, + "flos": 71166367883520.0, + "grad_norm": 0.8808685088475435, + "language_loss": 0.62978464, + "learning_rate": 1.0138316879374253e-07, + "loss": 0.65013796, + "num_input_tokens_seen": 162111640, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01239014, + "step": 7495, + "time_per_iteration": 3.451643466949463 + }, + { + "auxiliary_loss_clip": 0.01125947, + "auxiliary_loss_mlp": 0.01039428, + "balance_loss_clip": 1.03946805, + "balance_loss_mlp": 1.02413321, + "epoch": 0.9013407082306258, + "flos": 15594963413760.0, + "grad_norm": 2.4212773952688718, + "language_loss": 0.74136275, + "learning_rate": 1.0113844603374833e-07, + "loss": 0.76301652, + "num_input_tokens_seen": 162128165, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15283203, + "step": 7496, + "time_per_iteration": 2.7640926837921143 + }, + { + "auxiliary_loss_clip": 0.01128696, + "auxiliary_loss_mlp": 0.01038828, + "balance_loss_clip": 1.04030776, + "balance_loss_mlp": 1.0231998, + "epoch": 0.901460951121265, + "flos": 15049157276160.0, + "grad_norm": 2.3571055028103323, + "language_loss": 0.71950734, + "learning_rate": 1.0089401132954178e-07, + "loss": 0.74118257, + "num_input_tokens_seen": 162146145, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.15637207, + "step": 7497, + "time_per_iteration": 3.518516778945923 + }, + { + "auxiliary_loss_clip": 0.01123359, + "auxiliary_loss_mlp": 0.01039811, + "balance_loss_clip": 1.03883493, + "balance_loss_mlp": 1.02431369, + "epoch": 0.9015811940119041, + "flos": 22236857233920.0, + "grad_norm": 1.615459795712192, + "language_loss": 0.72099864, + "learning_rate": 1.006498647182037e-07, + "loss": 0.74263042, + "num_input_tokens_seen": 162164800, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.1550293, + "step": 7498, + "time_per_iteration": 2.863189458847046 + }, + { + "auxiliary_loss_clip": 0.01123763, + "auxiliary_loss_mlp": 0.01035406, + "balance_loss_clip": 1.04046774, + "balance_loss_mlp": 1.02061224, + "epoch": 0.9017014369025431, + "flos": 24973824827520.0, + "grad_norm": 2.0003325580910407, + "language_loss": 0.7139622, + "learning_rate": 1.004060062367713e-07, + "loss": 0.73555392, + "num_input_tokens_seen": 162185895, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.14794922, + "step": 7499, + "time_per_iteration": 2.8023622035980225 + }, + { + "auxiliary_loss_clip": 0.01123851, + "auxiliary_loss_mlp": 0.0103798, + "balance_loss_clip": 1.03937864, + "balance_loss_mlp": 1.02217329, + "epoch": 0.9018216797931822, + "flos": 18114168804480.0, + "grad_norm": 1.7955509149405318, + "language_loss": 0.6934998, + "learning_rate": 1.0016243592223728e-07, + "loss": 0.71511805, + "num_input_tokens_seen": 162206295, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.15795898, + "step": 7500, + "time_per_iteration": 3.6232364177703857 + }, + { + "auxiliary_loss_clip": 0.01127287, + "auxiliary_loss_mlp": 0.01039314, + "balance_loss_clip": 1.04166472, + "balance_loss_mlp": 1.02407336, + "epoch": 0.9019419226838213, + "flos": 37268452759680.0, + "grad_norm": 1.7145125427672732, + "language_loss": 0.65647876, + "learning_rate": 9.991915381155114e-08, + "loss": 0.67814481, + "num_input_tokens_seen": 162229275, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15246582, + "step": 7501, + "time_per_iteration": 2.896590232849121 + }, + { + "auxiliary_loss_clip": 0.01125463, + "auxiliary_loss_mlp": 0.01041065, + "balance_loss_clip": 1.03941619, + "balance_loss_mlp": 1.02692699, + "epoch": 0.9020621655744604, + "flos": 23441121538560.0, + "grad_norm": 2.225446085871749, + "language_loss": 0.74643284, + "learning_rate": 9.967615994161871e-08, + "loss": 0.76809812, + "num_input_tokens_seen": 162248935, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.14135742, + "step": 7502, + "time_per_iteration": 2.839815855026245 + }, + { + "auxiliary_loss_clip": 0.01122129, + "auxiliary_loss_mlp": 0.01045465, + "balance_loss_clip": 1.03948951, + "balance_loss_mlp": 1.03031373, + "epoch": 0.9021824084650995, + "flos": 22857465444480.0, + "grad_norm": 1.7324768106112458, + "language_loss": 0.78231943, + "learning_rate": 9.943345434930161e-08, + "loss": 0.80399543, + "num_input_tokens_seen": 162269185, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.15148926, + "step": 7503, + "time_per_iteration": 3.7006709575653076 + }, + { + "auxiliary_loss_clip": 0.01123959, + "auxiliary_loss_mlp": 0.0103755, + "balance_loss_clip": 1.0392108, + "balance_loss_mlp": 1.02243984, + "epoch": 0.9023026513557386, + "flos": 22127581082880.0, + "grad_norm": 1.9081798958640444, + "language_loss": 0.68649763, + "learning_rate": 9.919103707141885e-08, + "loss": 0.70811266, + "num_input_tokens_seen": 162288065, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.15100098, + "step": 7504, + "time_per_iteration": 2.786865711212158 + }, + { + "auxiliary_loss_clip": 0.01131529, + "auxiliary_loss_mlp": 0.01035407, + "balance_loss_clip": 1.04143214, + "balance_loss_mlp": 1.01909959, + "epoch": 0.9024228942463777, + "flos": 24199087357440.0, + "grad_norm": 2.2471355164895304, + "language_loss": 0.76841962, + "learning_rate": 9.89489081447441e-08, + "loss": 0.79008901, + "num_input_tokens_seen": 162305265, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.16308594, + "step": 7505, + "time_per_iteration": 2.845200777053833 + }, + { + "auxiliary_loss_clip": 0.01127355, + "auxiliary_loss_mlp": 0.01041677, + "balance_loss_clip": 1.0411725, + "balance_loss_mlp": 1.02519059, + "epoch": 0.9025431371370167, + "flos": 25008262992000.0, + "grad_norm": 1.9138751087129096, + "language_loss": 0.83070964, + "learning_rate": 9.870706760600844e-08, + "loss": 0.85239995, + "num_input_tokens_seen": 162325215, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.16479492, + "step": 7506, + "time_per_iteration": 2.850311040878296 + }, + { + "auxiliary_loss_clip": 0.01129931, + "auxiliary_loss_mlp": 0.01043319, + "balance_loss_clip": 1.04330218, + "balance_loss_mlp": 1.02805996, + "epoch": 0.9026633800276559, + "flos": 18952862440320.0, + "grad_norm": 2.516561340135737, + "language_loss": 0.72868043, + "learning_rate": 9.846551549189918e-08, + "loss": 0.75041294, + "num_input_tokens_seen": 162344820, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15270996, + "step": 7507, + "time_per_iteration": 2.8371760845184326 + }, + { + "auxiliary_loss_clip": 0.01126765, + "auxiliary_loss_mlp": 0.01037697, + "balance_loss_clip": 1.03929293, + "balance_loss_mlp": 1.02186608, + "epoch": 0.902783622918295, + "flos": 32416059536640.0, + "grad_norm": 1.8199201856992506, + "language_loss": 0.68756193, + "learning_rate": 9.822425183905902e-08, + "loss": 0.70920652, + "num_input_tokens_seen": 162365345, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.1583252, + "step": 7508, + "time_per_iteration": 2.877562999725342 + }, + { + "auxiliary_loss_clip": 0.01031967, + "auxiliary_loss_mlp": 0.01005171, + "balance_loss_clip": 1.00419903, + "balance_loss_mlp": 1.00386608, + "epoch": 0.902903865808934, + "flos": 63717453244800.0, + "grad_norm": 0.9062219322335967, + "language_loss": 0.75126392, + "learning_rate": 9.798327668408823e-08, + "loss": 0.77163529, + "num_input_tokens_seen": 162426980, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01306152, + "step": 7509, + "time_per_iteration": 3.444812536239624 + }, + { + "auxiliary_loss_clip": 0.01129735, + "auxiliary_loss_mlp": 0.01038567, + "balance_loss_clip": 1.04168916, + "balance_loss_mlp": 1.02150822, + "epoch": 0.9030241086995732, + "flos": 23804034600960.0, + "grad_norm": 1.8525525091443695, + "language_loss": 0.68901062, + "learning_rate": 9.774259006354158e-08, + "loss": 0.7106936, + "num_input_tokens_seen": 162447050, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.1706543, + "step": 7510, + "time_per_iteration": 2.9129960536956787 + }, + { + "auxiliary_loss_clip": 0.01123707, + "auxiliary_loss_mlp": 0.01037989, + "balance_loss_clip": 1.03899455, + "balance_loss_mlp": 1.022277, + "epoch": 0.9031443515902122, + "flos": 26395887248640.0, + "grad_norm": 2.114072067375579, + "language_loss": 0.76129246, + "learning_rate": 9.750219201393184e-08, + "loss": 0.78290945, + "num_input_tokens_seen": 162467015, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.15710449, + "step": 7511, + "time_per_iteration": 2.8338141441345215 + }, + { + "auxiliary_loss_clip": 0.01123417, + "auxiliary_loss_mlp": 0.01032136, + "balance_loss_clip": 1.03864205, + "balance_loss_mlp": 1.01730609, + "epoch": 0.9032645944808513, + "flos": 24939350749440.0, + "grad_norm": 1.8719676462442807, + "language_loss": 0.77878231, + "learning_rate": 9.726208257172697e-08, + "loss": 0.80033785, + "num_input_tokens_seen": 162488710, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.1484375, + "step": 7512, + "time_per_iteration": 2.8625245094299316 + }, + { + "auxiliary_loss_clip": 0.01124363, + "auxiliary_loss_mlp": 0.01041961, + "balance_loss_clip": 1.03984797, + "balance_loss_mlp": 1.02631485, + "epoch": 0.9033848373714904, + "flos": 21178821196800.0, + "grad_norm": 1.9150050606918072, + "language_loss": 0.75007331, + "learning_rate": 9.702226177335115e-08, + "loss": 0.77173656, + "num_input_tokens_seen": 162507205, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.15631104, + "step": 7513, + "time_per_iteration": 2.7203640937805176 + }, + { + "auxiliary_loss_clip": 0.01125007, + "auxiliary_loss_mlp": 0.01037958, + "balance_loss_clip": 1.039271, + "balance_loss_mlp": 1.02272916, + "epoch": 0.9035050802621295, + "flos": 26286359702400.0, + "grad_norm": 1.7077121317201425, + "language_loss": 0.72546065, + "learning_rate": 9.67827296551853e-08, + "loss": 0.74709028, + "num_input_tokens_seen": 162528490, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15216064, + "step": 7514, + "time_per_iteration": 2.7852330207824707 + }, + { + "auxiliary_loss_clip": 0.01126958, + "auxiliary_loss_mlp": 0.01034175, + "balance_loss_clip": 1.04207194, + "balance_loss_mlp": 1.01921463, + "epoch": 0.9036253231527686, + "flos": 24204546224640.0, + "grad_norm": 2.157532591079541, + "language_loss": 0.68586266, + "learning_rate": 9.65434862535659e-08, + "loss": 0.70747399, + "num_input_tokens_seen": 162547860, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.1496582, + "step": 7515, + "time_per_iteration": 2.8534436225891113 + }, + { + "auxiliary_loss_clip": 0.01129164, + "auxiliary_loss_mlp": 0.01043187, + "balance_loss_clip": 1.04064417, + "balance_loss_mlp": 1.02788091, + "epoch": 0.9037455660434077, + "flos": 18072655660800.0, + "grad_norm": 2.6252062163749024, + "language_loss": 0.65593439, + "learning_rate": 9.630453160478635e-08, + "loss": 0.67765796, + "num_input_tokens_seen": 162563215, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.1529541, + "step": 7516, + "time_per_iteration": 2.7391340732574463 + }, + { + "auxiliary_loss_clip": 0.01122674, + "auxiliary_loss_mlp": 0.01039815, + "balance_loss_clip": 1.03861415, + "balance_loss_mlp": 1.02474678, + "epoch": 0.9038658089340468, + "flos": 24060795995520.0, + "grad_norm": 1.5775612342860434, + "language_loss": 0.82412982, + "learning_rate": 9.60658657450959e-08, + "loss": 0.84575462, + "num_input_tokens_seen": 162583515, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15063477, + "step": 7517, + "time_per_iteration": 2.863679885864258 + }, + { + "auxiliary_loss_clip": 0.01123387, + "auxiliary_loss_mlp": 0.01034452, + "balance_loss_clip": 1.03899646, + "balance_loss_mlp": 1.01984882, + "epoch": 0.9039860518246858, + "flos": 21834298535040.0, + "grad_norm": 1.8629814328029595, + "language_loss": 0.79534161, + "learning_rate": 9.582748871069979e-08, + "loss": 0.81691992, + "num_input_tokens_seen": 162602955, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.14587402, + "step": 7518, + "time_per_iteration": 2.735868453979492 + }, + { + "auxiliary_loss_clip": 0.01123228, + "auxiliary_loss_mlp": 0.01036822, + "balance_loss_clip": 1.03881645, + "balance_loss_mlp": 1.02211773, + "epoch": 0.904106294715325, + "flos": 26614870513920.0, + "grad_norm": 1.7773388455234194, + "language_loss": 0.83198023, + "learning_rate": 9.558940053775954e-08, + "loss": 0.85358077, + "num_input_tokens_seen": 162621595, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14703369, + "step": 7519, + "time_per_iteration": 2.791781187057495 + }, + { + "auxiliary_loss_clip": 0.01124561, + "auxiliary_loss_mlp": 0.01038657, + "balance_loss_clip": 1.03931582, + "balance_loss_mlp": 1.02292109, + "epoch": 0.904226537605964, + "flos": 17785693906560.0, + "grad_norm": 1.7691858941036442, + "language_loss": 0.67854357, + "learning_rate": 9.535160126239294e-08, + "loss": 0.70017576, + "num_input_tokens_seen": 162638220, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.1572876, + "step": 7520, + "time_per_iteration": 2.719749689102173 + }, + { + "auxiliary_loss_clip": 0.01121718, + "auxiliary_loss_mlp": 0.01036138, + "balance_loss_clip": 1.03886843, + "balance_loss_mlp": 1.02106977, + "epoch": 0.9043467804966031, + "flos": 24790428961920.0, + "grad_norm": 1.4728753447603213, + "language_loss": 0.70817333, + "learning_rate": 9.511409092067424e-08, + "loss": 0.72975183, + "num_input_tokens_seen": 162658575, + "router_z_loss_clip": 0.828125, + "router_z_loss_mlp": 0.15063477, + "step": 7521, + "time_per_iteration": 2.881744623184204 + }, + { + "auxiliary_loss_clip": 0.01126094, + "auxiliary_loss_mlp": 0.01039603, + "balance_loss_clip": 1.03922379, + "balance_loss_mlp": 1.02309287, + "epoch": 0.9044670233872423, + "flos": 22632125472000.0, + "grad_norm": 1.72636523642697, + "language_loss": 0.67638069, + "learning_rate": 9.487686954863327e-08, + "loss": 0.69803768, + "num_input_tokens_seen": 162678295, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.16516113, + "step": 7522, + "time_per_iteration": 3.5102274417877197 + }, + { + "auxiliary_loss_clip": 0.0112618, + "auxiliary_loss_mlp": 0.01035836, + "balance_loss_clip": 1.04046249, + "balance_loss_mlp": 1.02000463, + "epoch": 0.9045872662778813, + "flos": 23771320289280.0, + "grad_norm": 1.9405333561350129, + "language_loss": 0.7751739, + "learning_rate": 9.46399371822566e-08, + "loss": 0.79679406, + "num_input_tokens_seen": 162698070, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15820312, + "step": 7523, + "time_per_iteration": 2.7815158367156982 + }, + { + "auxiliary_loss_clip": 0.01125985, + "auxiliary_loss_mlp": 0.01036409, + "balance_loss_clip": 1.0407089, + "balance_loss_mlp": 1.02087653, + "epoch": 0.9047075091685204, + "flos": 15191039998080.0, + "grad_norm": 1.9330695538293627, + "language_loss": 0.72624177, + "learning_rate": 9.440329385748657e-08, + "loss": 0.74786568, + "num_input_tokens_seen": 162715140, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.1552124, + "step": 7524, + "time_per_iteration": 2.843533515930176 + }, + { + "auxiliary_loss_clip": 0.01123656, + "auxiliary_loss_mlp": 0.01032998, + "balance_loss_clip": 1.04119456, + "balance_loss_mlp": 1.01902652, + "epoch": 0.9048277520591596, + "flos": 18003707504640.0, + "grad_norm": 1.7446252041341042, + "language_loss": 0.70964658, + "learning_rate": 9.416693961022137e-08, + "loss": 0.73121309, + "num_input_tokens_seen": 162733390, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.13970947, + "step": 7525, + "time_per_iteration": 2.769479990005493 + }, + { + "auxiliary_loss_clip": 0.0112515, + "auxiliary_loss_mlp": 0.01036026, + "balance_loss_clip": 1.04036117, + "balance_loss_mlp": 1.02160776, + "epoch": 0.9049479949497986, + "flos": 21872471713920.0, + "grad_norm": 9.23489490091792, + "language_loss": 0.77373081, + "learning_rate": 9.393087447631654e-08, + "loss": 0.79534256, + "num_input_tokens_seen": 162751670, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.14434814, + "step": 7526, + "time_per_iteration": 3.634798765182495 + }, + { + "auxiliary_loss_clip": 0.011173, + "auxiliary_loss_mlp": 0.01041267, + "balance_loss_clip": 1.03767788, + "balance_loss_mlp": 1.02684271, + "epoch": 0.9050682378404377, + "flos": 20773928113920.0, + "grad_norm": 1.6871702007343836, + "language_loss": 0.72914433, + "learning_rate": 9.36950984915823e-08, + "loss": 0.75072998, + "num_input_tokens_seen": 162770025, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.14416504, + "step": 7527, + "time_per_iteration": 2.9110469818115234 + }, + { + "auxiliary_loss_clip": 0.01126912, + "auxiliary_loss_mlp": 0.0104425, + "balance_loss_clip": 1.04218721, + "balance_loss_mlp": 1.02955198, + "epoch": 0.9051884807310768, + "flos": 21580015178880.0, + "grad_norm": 1.6849977305890347, + "language_loss": 0.69515526, + "learning_rate": 9.345961169178607e-08, + "loss": 0.71686691, + "num_input_tokens_seen": 162789710, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.14697266, + "step": 7528, + "time_per_iteration": 2.7738640308380127 + }, + { + "auxiliary_loss_clip": 0.01127646, + "auxiliary_loss_mlp": 0.01038858, + "balance_loss_clip": 1.04222202, + "balance_loss_mlp": 1.02369416, + "epoch": 0.9053087236217159, + "flos": 21908059113600.0, + "grad_norm": 1.406841396978457, + "language_loss": 0.72863579, + "learning_rate": 9.322441411265081e-08, + "loss": 0.75030082, + "num_input_tokens_seen": 162810695, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15155029, + "step": 7529, + "time_per_iteration": 3.613516092300415 + }, + { + "auxiliary_loss_clip": 0.01127318, + "auxiliary_loss_mlp": 0.01041293, + "balance_loss_clip": 1.04167044, + "balance_loss_mlp": 1.02494907, + "epoch": 0.9054289665123549, + "flos": 17055809544960.0, + "grad_norm": 1.8140142332997673, + "language_loss": 0.73132759, + "learning_rate": 9.298950578985554e-08, + "loss": 0.75301373, + "num_input_tokens_seen": 162827770, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.16357422, + "step": 7530, + "time_per_iteration": 2.7655203342437744 + }, + { + "auxiliary_loss_clip": 0.01126152, + "auxiliary_loss_mlp": 0.01042025, + "balance_loss_clip": 1.03982604, + "balance_loss_mlp": 1.02631354, + "epoch": 0.905549209402994, + "flos": 20777268078720.0, + "grad_norm": 1.6776268152790106, + "language_loss": 0.70969057, + "learning_rate": 9.275488675903665e-08, + "loss": 0.73137236, + "num_input_tokens_seen": 162846715, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15710449, + "step": 7531, + "time_per_iteration": 2.7920665740966797 + }, + { + "auxiliary_loss_clip": 0.01124926, + "auxiliary_loss_mlp": 0.01037758, + "balance_loss_clip": 1.03944433, + "balance_loss_mlp": 1.02257121, + "epoch": 0.9056694522936332, + "flos": 21686813291520.0, + "grad_norm": 1.9522301800545117, + "language_loss": 0.73783535, + "learning_rate": 9.252055705578454e-08, + "loss": 0.75946224, + "num_input_tokens_seen": 162866215, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.1517334, + "step": 7532, + "time_per_iteration": 3.706888437271118 + }, + { + "auxiliary_loss_clip": 0.01121815, + "auxiliary_loss_mlp": 0.01034149, + "balance_loss_clip": 1.03917646, + "balance_loss_mlp": 1.01922417, + "epoch": 0.9057896951842722, + "flos": 29569133433600.0, + "grad_norm": 1.6615634841791005, + "language_loss": 0.72274035, + "learning_rate": 9.228651671564747e-08, + "loss": 0.74430001, + "num_input_tokens_seen": 162888245, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.14929199, + "step": 7533, + "time_per_iteration": 2.8398070335388184 + }, + { + "auxiliary_loss_clip": 0.01123571, + "auxiliary_loss_mlp": 0.01038226, + "balance_loss_clip": 1.04101133, + "balance_loss_mlp": 1.02340817, + "epoch": 0.9059099380749113, + "flos": 27892248952320.0, + "grad_norm": 1.4390258214174123, + "language_loss": 0.77951264, + "learning_rate": 9.205276577412901e-08, + "loss": 0.80113065, + "num_input_tokens_seen": 162911025, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.14807129, + "step": 7534, + "time_per_iteration": 2.8601810932159424 + }, + { + "auxiliary_loss_clip": 0.0112784, + "auxiliary_loss_mlp": 0.01045476, + "balance_loss_clip": 1.04023051, + "balance_loss_mlp": 1.0295018, + "epoch": 0.9060301809655504, + "flos": 17748993185280.0, + "grad_norm": 2.28501252429212, + "language_loss": 0.76666319, + "learning_rate": 9.181930426668905e-08, + "loss": 0.78839636, + "num_input_tokens_seen": 162927820, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.1595459, + "step": 7535, + "time_per_iteration": 2.723055362701416 + }, + { + "auxiliary_loss_clip": 0.01125264, + "auxiliary_loss_mlp": 0.010399, + "balance_loss_clip": 1.04092681, + "balance_loss_mlp": 1.02460527, + "epoch": 0.9061504238561895, + "flos": 31759432963200.0, + "grad_norm": 1.586244592693973, + "language_loss": 0.67891854, + "learning_rate": 9.158613222874346e-08, + "loss": 0.70057023, + "num_input_tokens_seen": 162949445, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.15283203, + "step": 7536, + "time_per_iteration": 2.853757381439209 + }, + { + "auxiliary_loss_clip": 0.01124433, + "auxiliary_loss_mlp": 0.01035629, + "balance_loss_clip": 1.03997135, + "balance_loss_mlp": 1.02060843, + "epoch": 0.9062706667468285, + "flos": 20048066075520.0, + "grad_norm": 1.4803530414681751, + "language_loss": 0.82055146, + "learning_rate": 9.135324969566394e-08, + "loss": 0.84215206, + "num_input_tokens_seen": 162968945, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.15026855, + "step": 7537, + "time_per_iteration": 2.8191723823547363 + }, + { + "auxiliary_loss_clip": 0.01127765, + "auxiliary_loss_mlp": 0.01040771, + "balance_loss_clip": 1.04174149, + "balance_loss_mlp": 1.02672255, + "epoch": 0.9063909096374677, + "flos": 18437292576000.0, + "grad_norm": 3.0527826783508822, + "language_loss": 0.75784254, + "learning_rate": 9.112065670277913e-08, + "loss": 0.7795279, + "num_input_tokens_seen": 162985310, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.14044189, + "step": 7538, + "time_per_iteration": 2.711632013320923 + }, + { + "auxiliary_loss_clip": 0.01117835, + "auxiliary_loss_mlp": 0.01036083, + "balance_loss_clip": 1.03721762, + "balance_loss_mlp": 1.02263033, + "epoch": 0.9065111525281068, + "flos": 33547353361920.0, + "grad_norm": 1.8847956969761006, + "language_loss": 0.72832251, + "learning_rate": 9.088835328537303e-08, + "loss": 0.74986172, + "num_input_tokens_seen": 163006900, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.13452148, + "step": 7539, + "time_per_iteration": 2.8999547958374023 + }, + { + "auxiliary_loss_clip": 0.01125742, + "auxiliary_loss_mlp": 0.01041132, + "balance_loss_clip": 1.04031253, + "balance_loss_mlp": 1.02515769, + "epoch": 0.9066313954187458, + "flos": 23367863750400.0, + "grad_norm": 1.9770916239263991, + "language_loss": 0.71692944, + "learning_rate": 9.065633947868568e-08, + "loss": 0.73859817, + "num_input_tokens_seen": 163026505, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.15966797, + "step": 7540, + "time_per_iteration": 2.7833292484283447 + }, + { + "auxiliary_loss_clip": 0.01125436, + "auxiliary_loss_mlp": 0.01036905, + "balance_loss_clip": 1.04019499, + "balance_loss_mlp": 1.02220631, + "epoch": 0.906751638309385, + "flos": 26249623067520.0, + "grad_norm": 1.9985610304330503, + "language_loss": 0.80084282, + "learning_rate": 9.042461531791379e-08, + "loss": 0.82246625, + "num_input_tokens_seen": 163044925, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.14709473, + "step": 7541, + "time_per_iteration": 2.907078266143799 + }, + { + "auxiliary_loss_clip": 0.01120258, + "auxiliary_loss_mlp": 0.01033102, + "balance_loss_clip": 1.03835297, + "balance_loss_mlp": 1.01816523, + "epoch": 0.906871881200024, + "flos": 16544477485440.0, + "grad_norm": 1.6387296404228306, + "language_loss": 0.77993309, + "learning_rate": 9.019318083820903e-08, + "loss": 0.80146664, + "num_input_tokens_seen": 163063505, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.14941406, + "step": 7542, + "time_per_iteration": 2.8020002841949463 + }, + { + "auxiliary_loss_clip": 0.01127405, + "auxiliary_loss_mlp": 0.01046415, + "balance_loss_clip": 1.04083514, + "balance_loss_mlp": 1.03081048, + "epoch": 0.9069921240906631, + "flos": 24605129675520.0, + "grad_norm": 1.686273611791028, + "language_loss": 0.85161793, + "learning_rate": 8.996203607468045e-08, + "loss": 0.8733561, + "num_input_tokens_seen": 163082505, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.15600586, + "step": 7543, + "time_per_iteration": 2.8214595317840576 + }, + { + "auxiliary_loss_clip": 0.01123945, + "auxiliary_loss_mlp": 0.01037015, + "balance_loss_clip": 1.03786922, + "balance_loss_mlp": 1.02180445, + "epoch": 0.9071123669813023, + "flos": 25374731500800.0, + "grad_norm": 1.7323814460906979, + "language_loss": 0.75511861, + "learning_rate": 8.973118106239241e-08, + "loss": 0.77672815, + "num_input_tokens_seen": 163105110, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15222168, + "step": 7544, + "time_per_iteration": 2.8706393241882324 + }, + { + "auxiliary_loss_clip": 0.01122191, + "auxiliary_loss_mlp": 0.01042591, + "balance_loss_clip": 1.0379746, + "balance_loss_mlp": 1.02669477, + "epoch": 0.9072326098719413, + "flos": 26725798690560.0, + "grad_norm": 1.9863735644006222, + "language_loss": 0.94483614, + "learning_rate": 8.95006158363656e-08, + "loss": 0.96648395, + "num_input_tokens_seen": 163125295, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.15899658, + "step": 7545, + "time_per_iteration": 2.8927628993988037 + }, + { + "auxiliary_loss_clip": 0.0112805, + "auxiliary_loss_mlp": 0.01043519, + "balance_loss_clip": 1.03985238, + "balance_loss_mlp": 1.02668679, + "epoch": 0.9073528527625804, + "flos": 23878800760320.0, + "grad_norm": 1.9237934227629963, + "language_loss": 0.77605617, + "learning_rate": 8.9270340431576e-08, + "loss": 0.79777181, + "num_input_tokens_seen": 163144385, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.16809082, + "step": 7546, + "time_per_iteration": 2.859386682510376 + }, + { + "auxiliary_loss_clip": 0.01123712, + "auxiliary_loss_mlp": 0.01039847, + "balance_loss_clip": 1.03845096, + "balance_loss_mlp": 1.02561331, + "epoch": 0.9074730956532195, + "flos": 37852144767360.0, + "grad_norm": 1.9444360478650529, + "language_loss": 0.73414636, + "learning_rate": 8.904035488295658e-08, + "loss": 0.75578201, + "num_input_tokens_seen": 163163885, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.14233398, + "step": 7547, + "time_per_iteration": 2.88500714302063 + }, + { + "auxiliary_loss_clip": 0.01032265, + "auxiliary_loss_mlp": 0.0100461, + "balance_loss_clip": 1.00453806, + "balance_loss_mlp": 1.00334001, + "epoch": 0.9075933385438586, + "flos": 65173307385600.0, + "grad_norm": 0.6601656111360603, + "language_loss": 0.5325368, + "learning_rate": 8.881065922539632e-08, + "loss": 0.55290556, + "num_input_tokens_seen": 163224325, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01269531, + "step": 7548, + "time_per_iteration": 3.2970523834228516 + }, + { + "auxiliary_loss_clip": 0.01123682, + "auxiliary_loss_mlp": 0.01028965, + "balance_loss_clip": 1.0402143, + "balance_loss_mlp": 1.01470757, + "epoch": 0.9077135814344977, + "flos": 19931571290880.0, + "grad_norm": 1.7505823424520235, + "language_loss": 0.73482776, + "learning_rate": 8.85812534937389e-08, + "loss": 0.75635427, + "num_input_tokens_seen": 163242425, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.14257812, + "step": 7549, + "time_per_iteration": 2.7759592533111572 + }, + { + "auxiliary_loss_clip": 0.01129066, + "auxiliary_loss_mlp": 0.0104607, + "balance_loss_clip": 1.04248452, + "balance_loss_mlp": 1.03075743, + "epoch": 0.9078338243251368, + "flos": 17529650784000.0, + "grad_norm": 5.448271801420523, + "language_loss": 0.67069811, + "learning_rate": 8.835213772278583e-08, + "loss": 0.69244945, + "num_input_tokens_seen": 163259280, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15313721, + "step": 7550, + "time_per_iteration": 2.7794840335845947 + }, + { + "auxiliary_loss_clip": 0.01120532, + "auxiliary_loss_mlp": 0.01038979, + "balance_loss_clip": 1.03718948, + "balance_loss_mlp": 1.02482295, + "epoch": 0.9079540672157759, + "flos": 28803410277120.0, + "grad_norm": 1.6203054184348142, + "language_loss": 0.78783071, + "learning_rate": 8.812331194729373e-08, + "loss": 0.80942583, + "num_input_tokens_seen": 163278925, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.1416626, + "step": 7551, + "time_per_iteration": 3.638753652572632 + }, + { + "auxiliary_loss_clip": 0.01129108, + "auxiliary_loss_mlp": 0.01044627, + "balance_loss_clip": 1.04208541, + "balance_loss_mlp": 1.02746081, + "epoch": 0.9080743101064149, + "flos": 23513840622720.0, + "grad_norm": 1.8652017080214798, + "language_loss": 0.72395664, + "learning_rate": 8.789477620197461e-08, + "loss": 0.74569398, + "num_input_tokens_seen": 163298450, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.171875, + "step": 7552, + "time_per_iteration": 2.791015863418579 + }, + { + "auxiliary_loss_clip": 0.01126404, + "auxiliary_loss_mlp": 0.01038285, + "balance_loss_clip": 1.0408442, + "balance_loss_mlp": 1.02234721, + "epoch": 0.9081945529970541, + "flos": 22778102344320.0, + "grad_norm": 2.4317896877411624, + "language_loss": 0.78845537, + "learning_rate": 8.766653052149831e-08, + "loss": 0.81010228, + "num_input_tokens_seen": 163313635, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15942383, + "step": 7553, + "time_per_iteration": 2.868950843811035 + }, + { + "auxiliary_loss_clip": 0.01123594, + "auxiliary_loss_mlp": 0.01040345, + "balance_loss_clip": 1.03812659, + "balance_loss_mlp": 1.02390587, + "epoch": 0.9083147958876931, + "flos": 18873714821760.0, + "grad_norm": 2.095961954554804, + "language_loss": 0.74375153, + "learning_rate": 8.743857494048823e-08, + "loss": 0.76539087, + "num_input_tokens_seen": 163330450, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.16430664, + "step": 7554, + "time_per_iteration": 2.734179973602295 + }, + { + "auxiliary_loss_clip": 0.01122705, + "auxiliary_loss_mlp": 0.01035957, + "balance_loss_clip": 1.03954983, + "balance_loss_mlp": 1.02122259, + "epoch": 0.9084350387783322, + "flos": 18909374048640.0, + "grad_norm": 2.829186857780171, + "language_loss": 0.62764591, + "learning_rate": 8.721090949352605e-08, + "loss": 0.64923251, + "num_input_tokens_seen": 163346690, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.1472168, + "step": 7555, + "time_per_iteration": 3.540705442428589 + }, + { + "auxiliary_loss_clip": 0.01130302, + "auxiliary_loss_mlp": 0.01041547, + "balance_loss_clip": 1.04170203, + "balance_loss_mlp": 1.02533412, + "epoch": 0.9085552816689714, + "flos": 20595488325120.0, + "grad_norm": 1.9217070195620116, + "language_loss": 0.72876084, + "learning_rate": 8.698353421514793e-08, + "loss": 0.75047934, + "num_input_tokens_seen": 163365065, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.16223145, + "step": 7556, + "time_per_iteration": 2.872565507888794 + }, + { + "auxiliary_loss_clip": 0.01124349, + "auxiliary_loss_mlp": 0.01035858, + "balance_loss_clip": 1.04114568, + "balance_loss_mlp": 1.02095711, + "epoch": 0.9086755245596104, + "flos": 18113163223680.0, + "grad_norm": 2.085350111475917, + "language_loss": 0.80272698, + "learning_rate": 8.67564491398467e-08, + "loss": 0.82432908, + "num_input_tokens_seen": 163382070, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.14904785, + "step": 7557, + "time_per_iteration": 2.7383248805999756 + }, + { + "auxiliary_loss_clip": 0.01124464, + "auxiliary_loss_mlp": 0.01039298, + "balance_loss_clip": 1.03998685, + "balance_loss_mlp": 1.02440858, + "epoch": 0.9087957674502495, + "flos": 19129793857920.0, + "grad_norm": 1.7287752297939574, + "language_loss": 0.73473966, + "learning_rate": 8.652965430207104e-08, + "loss": 0.75637716, + "num_input_tokens_seen": 163399975, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.14892578, + "step": 7558, + "time_per_iteration": 3.595871686935425 + }, + { + "auxiliary_loss_clip": 0.01126126, + "auxiliary_loss_mlp": 0.01038411, + "balance_loss_clip": 1.03907037, + "balance_loss_mlp": 1.02153087, + "epoch": 0.9089160103408886, + "flos": 18109930999680.0, + "grad_norm": 2.507323064546106, + "language_loss": 0.65759277, + "learning_rate": 8.630314973622521e-08, + "loss": 0.67923814, + "num_input_tokens_seen": 163417520, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.16870117, + "step": 7559, + "time_per_iteration": 2.738253593444824 + }, + { + "auxiliary_loss_clip": 0.01123945, + "auxiliary_loss_mlp": 0.01034541, + "balance_loss_clip": 1.04090929, + "balance_loss_mlp": 1.01990187, + "epoch": 0.9090362532315277, + "flos": 33364855336320.0, + "grad_norm": 1.9343074659967656, + "language_loss": 0.71237725, + "learning_rate": 8.607693547666995e-08, + "loss": 0.73396212, + "num_input_tokens_seen": 163440060, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.14648438, + "step": 7560, + "time_per_iteration": 2.814976215362549 + }, + { + "auxiliary_loss_clip": 0.01031744, + "auxiliary_loss_mlp": 0.01003295, + "balance_loss_clip": 1.00413322, + "balance_loss_mlp": 1.00206685, + "epoch": 0.9091564961221668, + "flos": 71480585082240.0, + "grad_norm": 0.8837722359443546, + "language_loss": 0.57966334, + "learning_rate": 8.585101155772201e-08, + "loss": 0.60001367, + "num_input_tokens_seen": 163502180, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01226807, + "step": 7561, + "time_per_iteration": 4.253288507461548 + }, + { + "auxiliary_loss_clip": 0.01123555, + "auxiliary_loss_mlp": 0.01037286, + "balance_loss_clip": 1.03779662, + "balance_loss_mlp": 1.02150273, + "epoch": 0.9092767390128058, + "flos": 24712574232960.0, + "grad_norm": 1.9876902740803324, + "language_loss": 0.68635011, + "learning_rate": 8.562537801365377e-08, + "loss": 0.70795858, + "num_input_tokens_seen": 163521915, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.15783691, + "step": 7562, + "time_per_iteration": 2.8213770389556885 + }, + { + "auxiliary_loss_clip": 0.01124522, + "auxiliary_loss_mlp": 0.01045978, + "balance_loss_clip": 1.03921652, + "balance_loss_mlp": 1.02953863, + "epoch": 0.909396981903445, + "flos": 23586487879680.0, + "grad_norm": 1.626196383003775, + "language_loss": 0.70003134, + "learning_rate": 8.540003487869362e-08, + "loss": 0.72173631, + "num_input_tokens_seen": 163543585, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.16430664, + "step": 7563, + "time_per_iteration": 2.8243346214294434 + }, + { + "auxiliary_loss_clip": 0.01123123, + "auxiliary_loss_mlp": 0.01031306, + "balance_loss_clip": 1.03909624, + "balance_loss_mlp": 1.01619029, + "epoch": 0.909517224794084, + "flos": 23404169422080.0, + "grad_norm": 1.7554252720180008, + "language_loss": 0.79730117, + "learning_rate": 8.517498218702557e-08, + "loss": 0.81884551, + "num_input_tokens_seen": 163561515, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.15124512, + "step": 7564, + "time_per_iteration": 2.8254950046539307 + }, + { + "auxiliary_loss_clip": 0.01124704, + "auxiliary_loss_mlp": 0.01034307, + "balance_loss_clip": 1.03909588, + "balance_loss_mlp": 1.02025807, + "epoch": 0.9096374676847231, + "flos": 19208618254080.0, + "grad_norm": 1.5973827939957286, + "language_loss": 0.69694185, + "learning_rate": 8.49502199727905e-08, + "loss": 0.71853203, + "num_input_tokens_seen": 163579540, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.14056396, + "step": 7565, + "time_per_iteration": 2.835646152496338 + }, + { + "auxiliary_loss_clip": 0.01128361, + "auxiliary_loss_mlp": 0.01038068, + "balance_loss_clip": 1.04021323, + "balance_loss_mlp": 1.0222609, + "epoch": 0.9097577105753623, + "flos": 33292495388160.0, + "grad_norm": 2.3577413140024728, + "language_loss": 0.66473567, + "learning_rate": 8.472574827008428e-08, + "loss": 0.68639994, + "num_input_tokens_seen": 163600425, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.15808105, + "step": 7566, + "time_per_iteration": 2.879286050796509 + }, + { + "auxiliary_loss_clip": 0.01122472, + "auxiliary_loss_mlp": 0.01034733, + "balance_loss_clip": 1.03912711, + "balance_loss_mlp": 1.02008212, + "epoch": 0.9098779534660013, + "flos": 21906443001600.0, + "grad_norm": 1.9453393119572577, + "language_loss": 0.8393079, + "learning_rate": 8.450156711295942e-08, + "loss": 0.86087996, + "num_input_tokens_seen": 163620595, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.14642334, + "step": 7567, + "time_per_iteration": 2.802480697631836 + }, + { + "auxiliary_loss_clip": 0.01123568, + "auxiliary_loss_mlp": 0.01037822, + "balance_loss_clip": 1.03910923, + "balance_loss_mlp": 1.02251518, + "epoch": 0.9099981963566404, + "flos": 25730354102400.0, + "grad_norm": 2.2383547917367284, + "language_loss": 0.87066799, + "learning_rate": 8.427767653542383e-08, + "loss": 0.89228189, + "num_input_tokens_seen": 163635765, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15289307, + "step": 7568, + "time_per_iteration": 2.980881929397583 + }, + { + "auxiliary_loss_clip": 0.01121434, + "auxiliary_loss_mlp": 0.01034994, + "balance_loss_clip": 1.03817964, + "balance_loss_mlp": 1.0200454, + "epoch": 0.9101184392472795, + "flos": 21069437304960.0, + "grad_norm": 2.0057424371129087, + "language_loss": 0.70017219, + "learning_rate": 8.405407657144125e-08, + "loss": 0.72173649, + "num_input_tokens_seen": 163654925, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.14953613, + "step": 7569, + "time_per_iteration": 2.8839738368988037 + }, + { + "auxiliary_loss_clip": 0.01121173, + "auxiliary_loss_mlp": 0.01037141, + "balance_loss_clip": 1.03908217, + "balance_loss_mlp": 1.02236557, + "epoch": 0.9102386821379186, + "flos": 24752614919040.0, + "grad_norm": 1.768175458888786, + "language_loss": 0.72612017, + "learning_rate": 8.383076725493232e-08, + "loss": 0.74770331, + "num_input_tokens_seen": 163672245, + "router_z_loss_clip": 0.81982422, + "router_z_loss_mlp": 0.14776611, + "step": 7570, + "time_per_iteration": 2.906013250350952 + }, + { + "auxiliary_loss_clip": 0.01122581, + "auxiliary_loss_mlp": 0.0103746, + "balance_loss_clip": 1.03884077, + "balance_loss_mlp": 1.02286923, + "epoch": 0.9103589250285576, + "flos": 22562818179840.0, + "grad_norm": 3.054928417203115, + "language_loss": 0.68012619, + "learning_rate": 8.360774861977216e-08, + "loss": 0.70172668, + "num_input_tokens_seen": 163691365, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.14599609, + "step": 7571, + "time_per_iteration": 2.881366491317749 + }, + { + "auxiliary_loss_clip": 0.01124221, + "auxiliary_loss_mlp": 0.01034919, + "balance_loss_clip": 1.04051208, + "balance_loss_mlp": 1.0208528, + "epoch": 0.9104791679191968, + "flos": 25373474524800.0, + "grad_norm": 1.864745101362786, + "language_loss": 0.74836087, + "learning_rate": 8.338502069979281e-08, + "loss": 0.76995224, + "num_input_tokens_seen": 163711675, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.140625, + "step": 7572, + "time_per_iteration": 2.842005491256714 + }, + { + "auxiliary_loss_clip": 0.01124613, + "auxiliary_loss_mlp": 0.01036068, + "balance_loss_clip": 1.03941965, + "balance_loss_mlp": 1.02042782, + "epoch": 0.9105994108098359, + "flos": 14426681558400.0, + "grad_norm": 2.422443370153529, + "language_loss": 0.79368758, + "learning_rate": 8.316258352878214e-08, + "loss": 0.81529438, + "num_input_tokens_seen": 163728095, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.15649414, + "step": 7573, + "time_per_iteration": 2.7600512504577637 + }, + { + "auxiliary_loss_clip": 0.0112633, + "auxiliary_loss_mlp": 0.01039928, + "balance_loss_clip": 1.03934753, + "balance_loss_mlp": 1.02374578, + "epoch": 0.9107196537004749, + "flos": 26718292748160.0, + "grad_norm": 1.9502107049122408, + "language_loss": 0.71555793, + "learning_rate": 8.294043714048338e-08, + "loss": 0.73722053, + "num_input_tokens_seen": 163747175, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.16168213, + "step": 7574, + "time_per_iteration": 2.7959189414978027 + }, + { + "auxiliary_loss_clip": 0.01031972, + "auxiliary_loss_mlp": 0.01001857, + "balance_loss_clip": 1.0042789, + "balance_loss_mlp": 1.00062299, + "epoch": 0.9108398965911141, + "flos": 66532634703360.0, + "grad_norm": 0.7519472464164831, + "language_loss": 0.60434568, + "learning_rate": 8.271858156859624e-08, + "loss": 0.62468398, + "num_input_tokens_seen": 163812545, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.0123291, + "step": 7575, + "time_per_iteration": 3.3514139652252197 + }, + { + "auxiliary_loss_clip": 0.01125212, + "auxiliary_loss_mlp": 0.01035752, + "balance_loss_clip": 1.04064608, + "balance_loss_mlp": 1.01994479, + "epoch": 0.9109601394817531, + "flos": 25411073086080.0, + "grad_norm": 1.6465636268449926, + "language_loss": 0.7397083, + "learning_rate": 8.249701684677557e-08, + "loss": 0.76131797, + "num_input_tokens_seen": 163833870, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15808105, + "step": 7576, + "time_per_iteration": 2.7801625728607178 + }, + { + "auxiliary_loss_clip": 0.01123543, + "auxiliary_loss_mlp": 0.01034974, + "balance_loss_clip": 1.0411737, + "balance_loss_mlp": 1.01916647, + "epoch": 0.9110803823723922, + "flos": 22747794243840.0, + "grad_norm": 1.6678215368364762, + "language_loss": 0.80981565, + "learning_rate": 8.227574300863294e-08, + "loss": 0.83140081, + "num_input_tokens_seen": 163854040, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.15820312, + "step": 7577, + "time_per_iteration": 2.828521490097046 + }, + { + "auxiliary_loss_clip": 0.01129325, + "auxiliary_loss_mlp": 0.01033422, + "balance_loss_clip": 1.04048228, + "balance_loss_mlp": 1.01806164, + "epoch": 0.9112006252630314, + "flos": 48469924131840.0, + "grad_norm": 1.720839155400939, + "language_loss": 0.69903874, + "learning_rate": 8.205476008773548e-08, + "loss": 0.72066617, + "num_input_tokens_seen": 163878040, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.15362549, + "step": 7578, + "time_per_iteration": 2.9585602283477783 + }, + { + "auxiliary_loss_clip": 0.01124599, + "auxiliary_loss_mlp": 0.01036606, + "balance_loss_clip": 1.04179335, + "balance_loss_mlp": 1.02154422, + "epoch": 0.9113208681536704, + "flos": 30009649829760.0, + "grad_norm": 2.245948981406776, + "language_loss": 0.82874894, + "learning_rate": 8.183406811760596e-08, + "loss": 0.85036105, + "num_input_tokens_seen": 163897770, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.15045166, + "step": 7579, + "time_per_iteration": 2.8923122882843018 + }, + { + "auxiliary_loss_clip": 0.01122276, + "auxiliary_loss_mlp": 0.01037081, + "balance_loss_clip": 1.03932071, + "balance_loss_mlp": 1.0226326, + "epoch": 0.9114411110443095, + "flos": 25594971742080.0, + "grad_norm": 3.6507369409769272, + "language_loss": 0.74128187, + "learning_rate": 8.161366713172313e-08, + "loss": 0.76287544, + "num_input_tokens_seen": 163920160, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.14453125, + "step": 7580, + "time_per_iteration": 3.490206718444824 + }, + { + "auxiliary_loss_clip": 0.011319, + "auxiliary_loss_mlp": 0.01035353, + "balance_loss_clip": 1.04172564, + "balance_loss_mlp": 1.01831794, + "epoch": 0.9115613539349486, + "flos": 18399729928320.0, + "grad_norm": 2.6159725388804516, + "language_loss": 0.84179676, + "learning_rate": 8.139355716352137e-08, + "loss": 0.8634693, + "num_input_tokens_seen": 163935000, + "router_z_loss_clip": 0.90185547, + "router_z_loss_mlp": 0.17041016, + "step": 7581, + "time_per_iteration": 2.7820165157318115 + }, + { + "auxiliary_loss_clip": 0.0112284, + "auxiliary_loss_mlp": 0.01039191, + "balance_loss_clip": 1.03785121, + "balance_loss_mlp": 1.02489758, + "epoch": 0.9116815968255877, + "flos": 21726171619200.0, + "grad_norm": 1.5103419780030303, + "language_loss": 0.70026976, + "learning_rate": 8.117373824639196e-08, + "loss": 0.72189009, + "num_input_tokens_seen": 163955265, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.1428833, + "step": 7582, + "time_per_iteration": 2.823946714401245 + }, + { + "auxiliary_loss_clip": 0.01031788, + "auxiliary_loss_mlp": 0.01002462, + "balance_loss_clip": 1.00411034, + "balance_loss_mlp": 1.00119233, + "epoch": 0.9118018397162267, + "flos": 65363526835200.0, + "grad_norm": 0.7204006548061513, + "language_loss": 0.59270054, + "learning_rate": 8.095421041368067e-08, + "loss": 0.61304301, + "num_input_tokens_seen": 164014680, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01269531, + "step": 7583, + "time_per_iteration": 3.312570333480835 + }, + { + "auxiliary_loss_clip": 0.01125139, + "auxiliary_loss_mlp": 0.01039229, + "balance_loss_clip": 1.03992748, + "balance_loss_mlp": 1.02402949, + "epoch": 0.9119220826068659, + "flos": 20922885815040.0, + "grad_norm": 2.4016148020414056, + "language_loss": 0.70908785, + "learning_rate": 8.073497369868999e-08, + "loss": 0.73073149, + "num_input_tokens_seen": 164033140, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.15197754, + "step": 7584, + "time_per_iteration": 3.602860450744629 + }, + { + "auxiliary_loss_clip": 0.01127287, + "auxiliary_loss_mlp": 0.01036039, + "balance_loss_clip": 1.04076076, + "balance_loss_mlp": 1.01963544, + "epoch": 0.912042325497505, + "flos": 28366449327360.0, + "grad_norm": 1.5817631970554487, + "language_loss": 0.75231993, + "learning_rate": 8.051602813467772e-08, + "loss": 0.77395326, + "num_input_tokens_seen": 164054995, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.16387939, + "step": 7585, + "time_per_iteration": 2.856806755065918 + }, + { + "auxiliary_loss_clip": 0.01122234, + "auxiliary_loss_mlp": 0.01039933, + "balance_loss_clip": 1.03985119, + "balance_loss_mlp": 1.02550888, + "epoch": 0.912162568388144, + "flos": 17566782468480.0, + "grad_norm": 2.11396587451854, + "language_loss": 0.71175551, + "learning_rate": 8.029737375485756e-08, + "loss": 0.73337716, + "num_input_tokens_seen": 164074225, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.14422607, + "step": 7586, + "time_per_iteration": 3.6468305587768555 + }, + { + "auxiliary_loss_clip": 0.01124063, + "auxiliary_loss_mlp": 0.01037285, + "balance_loss_clip": 1.03976893, + "balance_loss_mlp": 1.02260387, + "epoch": 0.9122828112787832, + "flos": 19827897661440.0, + "grad_norm": 1.6769696045591609, + "language_loss": 0.72310185, + "learning_rate": 8.007901059239986e-08, + "loss": 0.74471533, + "num_input_tokens_seen": 164093505, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14678955, + "step": 7587, + "time_per_iteration": 2.8152260780334473 + }, + { + "auxiliary_loss_clip": 0.01122897, + "auxiliary_loss_mlp": 0.01034173, + "balance_loss_clip": 1.03834748, + "balance_loss_mlp": 1.01965916, + "epoch": 0.9124030541694222, + "flos": 20813789232000.0, + "grad_norm": 1.5356965550621906, + "language_loss": 0.80141795, + "learning_rate": 7.986093868042964e-08, + "loss": 0.82298863, + "num_input_tokens_seen": 164113750, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.14520264, + "step": 7588, + "time_per_iteration": 2.864063024520874 + }, + { + "auxiliary_loss_clip": 0.01122447, + "auxiliary_loss_mlp": 0.01036976, + "balance_loss_clip": 1.03828669, + "balance_loss_mlp": 1.02215195, + "epoch": 0.9125232970600613, + "flos": 25192305302400.0, + "grad_norm": 1.6111809177345995, + "language_loss": 0.68045479, + "learning_rate": 7.964315805202826e-08, + "loss": 0.70204902, + "num_input_tokens_seen": 164134330, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.14831543, + "step": 7589, + "time_per_iteration": 2.949638843536377 + }, + { + "auxiliary_loss_clip": 0.01123779, + "auxiliary_loss_mlp": 0.0103825, + "balance_loss_clip": 1.03897762, + "balance_loss_mlp": 1.02396309, + "epoch": 0.9126435399507005, + "flos": 19719591177600.0, + "grad_norm": 1.7226879272279159, + "language_loss": 0.73541057, + "learning_rate": 7.942566874023304e-08, + "loss": 0.75703084, + "num_input_tokens_seen": 164153515, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.14294434, + "step": 7590, + "time_per_iteration": 3.656440258026123 + }, + { + "auxiliary_loss_clip": 0.01123838, + "auxiliary_loss_mlp": 0.01035521, + "balance_loss_clip": 1.03705549, + "balance_loss_mlp": 1.01958287, + "epoch": 0.9127637828413395, + "flos": 19573614305280.0, + "grad_norm": 3.9125570205972644, + "language_loss": 0.69369447, + "learning_rate": 7.920847077803649e-08, + "loss": 0.71528804, + "num_input_tokens_seen": 164171305, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15942383, + "step": 7591, + "time_per_iteration": 2.851738214492798 + }, + { + "auxiliary_loss_clip": 0.0112431, + "auxiliary_loss_mlp": 0.01035352, + "balance_loss_clip": 1.0383482, + "balance_loss_mlp": 1.02109432, + "epoch": 0.9128840257319786, + "flos": 20230635928320.0, + "grad_norm": 1.752375961229601, + "language_loss": 0.82190859, + "learning_rate": 7.899156419838826e-08, + "loss": 0.84350514, + "num_input_tokens_seen": 164190275, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.14245605, + "step": 7592, + "time_per_iteration": 2.7289741039276123 + }, + { + "auxiliary_loss_clip": 0.01124667, + "auxiliary_loss_mlp": 0.0103857, + "balance_loss_clip": 1.04006302, + "balance_loss_mlp": 1.02339482, + "epoch": 0.9130042686226177, + "flos": 24858658846080.0, + "grad_norm": 1.873435523106092, + "language_loss": 0.65418452, + "learning_rate": 7.87749490341918e-08, + "loss": 0.67581689, + "num_input_tokens_seen": 164210550, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.15161133, + "step": 7593, + "time_per_iteration": 2.837496519088745 + }, + { + "auxiliary_loss_clip": 0.01127481, + "auxiliary_loss_mlp": 0.0104076, + "balance_loss_clip": 1.03978956, + "balance_loss_mlp": 1.02440476, + "epoch": 0.9131245115132568, + "flos": 23581747284480.0, + "grad_norm": 1.9139603383864268, + "language_loss": 0.83394831, + "learning_rate": 7.855862531830836e-08, + "loss": 0.85563076, + "num_input_tokens_seen": 164226660, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.16351318, + "step": 7594, + "time_per_iteration": 2.822570323944092 + }, + { + "auxiliary_loss_clip": 0.01126267, + "auxiliary_loss_mlp": 0.01039673, + "balance_loss_clip": 1.04075992, + "balance_loss_mlp": 1.02454567, + "epoch": 0.9132447544038959, + "flos": 19931607204480.0, + "grad_norm": 1.8728124420172627, + "language_loss": 0.72451359, + "learning_rate": 7.834259308355373e-08, + "loss": 0.74617296, + "num_input_tokens_seen": 164245425, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15130615, + "step": 7595, + "time_per_iteration": 2.772378921508789 + }, + { + "auxiliary_loss_clip": 0.01125348, + "auxiliary_loss_mlp": 0.01040811, + "balance_loss_clip": 1.03931284, + "balance_loss_mlp": 1.02419353, + "epoch": 0.9133649972945349, + "flos": 21981747864960.0, + "grad_norm": 2.2488968429888887, + "language_loss": 0.74598438, + "learning_rate": 7.812685236269989e-08, + "loss": 0.76764596, + "num_input_tokens_seen": 164264085, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.1661377, + "step": 7596, + "time_per_iteration": 2.8075613975524902 + }, + { + "auxiliary_loss_clip": 0.01031864, + "auxiliary_loss_mlp": 0.00999599, + "balance_loss_clip": 1.00420856, + "balance_loss_mlp": 0.99839479, + "epoch": 0.9134852401851741, + "flos": 71240523511680.0, + "grad_norm": 0.7999229519328153, + "language_loss": 0.58624327, + "learning_rate": 7.791140318847445e-08, + "loss": 0.60655797, + "num_input_tokens_seen": 164322220, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01202393, + "step": 7597, + "time_per_iteration": 3.301452398300171 + }, + { + "auxiliary_loss_clip": 0.01121742, + "auxiliary_loss_mlp": 0.0103452, + "balance_loss_clip": 1.03824449, + "balance_loss_mlp": 1.01995254, + "epoch": 0.9136054830758131, + "flos": 23626923615360.0, + "grad_norm": 1.6371641981874108, + "language_loss": 0.80423725, + "learning_rate": 7.769624559356081e-08, + "loss": 0.82579988, + "num_input_tokens_seen": 164345615, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.14562988, + "step": 7598, + "time_per_iteration": 2.8435230255126953 + }, + { + "auxiliary_loss_clip": 0.01124939, + "auxiliary_loss_mlp": 0.01039378, + "balance_loss_clip": 1.03938031, + "balance_loss_mlp": 1.02407169, + "epoch": 0.9137257259664522, + "flos": 23438858981760.0, + "grad_norm": 2.8648781131300622, + "language_loss": 0.75080633, + "learning_rate": 7.748137961059842e-08, + "loss": 0.77244949, + "num_input_tokens_seen": 164359595, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15319824, + "step": 7599, + "time_per_iteration": 2.8699474334716797 + }, + { + "auxiliary_loss_clip": 0.01119506, + "auxiliary_loss_mlp": 0.01036967, + "balance_loss_clip": 1.03902078, + "balance_loss_mlp": 1.02265048, + "epoch": 0.9138459688570914, + "flos": 19127854523520.0, + "grad_norm": 2.3781212018887095, + "language_loss": 0.65649754, + "learning_rate": 7.726680527218211e-08, + "loss": 0.67806232, + "num_input_tokens_seen": 164376635, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.14312744, + "step": 7600, + "time_per_iteration": 2.8960676193237305 + }, + { + "auxiliary_loss_clip": 0.01122329, + "auxiliary_loss_mlp": 0.01038274, + "balance_loss_clip": 1.03717041, + "balance_loss_mlp": 1.02274108, + "epoch": 0.9139662117477304, + "flos": 46281240714240.0, + "grad_norm": 1.6552189366143581, + "language_loss": 0.75654697, + "learning_rate": 7.70525226108627e-08, + "loss": 0.778153, + "num_input_tokens_seen": 164400305, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15533447, + "step": 7601, + "time_per_iteration": 2.9946470260620117 + }, + { + "auxiliary_loss_clip": 0.01125579, + "auxiliary_loss_mlp": 0.01042246, + "balance_loss_clip": 1.04177356, + "balance_loss_mlp": 1.02624786, + "epoch": 0.9140864546383695, + "flos": 22273198819200.0, + "grad_norm": 1.6952558412048253, + "language_loss": 0.79879415, + "learning_rate": 7.683853165914666e-08, + "loss": 0.82047236, + "num_input_tokens_seen": 164418075, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.15979004, + "step": 7602, + "time_per_iteration": 2.751682758331299 + }, + { + "auxiliary_loss_clip": 0.01126793, + "auxiliary_loss_mlp": 0.01041515, + "balance_loss_clip": 1.04083133, + "balance_loss_mlp": 1.02669716, + "epoch": 0.9142066975290086, + "flos": 17530009920000.0, + "grad_norm": 1.678344072284379, + "language_loss": 0.77145863, + "learning_rate": 7.662483244949602e-08, + "loss": 0.79314172, + "num_input_tokens_seen": 164435335, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.14807129, + "step": 7603, + "time_per_iteration": 2.860771656036377 + }, + { + "auxiliary_loss_clip": 0.01121222, + "auxiliary_loss_mlp": 0.01035688, + "balance_loss_clip": 1.03883171, + "balance_loss_mlp": 1.02107334, + "epoch": 0.9143269404196477, + "flos": 17712148809600.0, + "grad_norm": 2.5836606341687554, + "language_loss": 0.81139022, + "learning_rate": 7.641142501432951e-08, + "loss": 0.83295929, + "num_input_tokens_seen": 164451530, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.14611816, + "step": 7604, + "time_per_iteration": 2.7496023178100586 + }, + { + "auxiliary_loss_clip": 0.01124537, + "auxiliary_loss_mlp": 0.01036481, + "balance_loss_clip": 1.0391854, + "balance_loss_mlp": 1.02207446, + "epoch": 0.9144471833102867, + "flos": 33323414019840.0, + "grad_norm": 1.556641072074067, + "language_loss": 0.73739588, + "learning_rate": 7.619830938602013e-08, + "loss": 0.75900602, + "num_input_tokens_seen": 164472755, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.144104, + "step": 7605, + "time_per_iteration": 2.9227986335754395 + }, + { + "auxiliary_loss_clip": 0.01127794, + "auxiliary_loss_mlp": 0.0104891, + "balance_loss_clip": 1.0413059, + "balance_loss_mlp": 1.03360379, + "epoch": 0.9145674262009259, + "flos": 21068970428160.0, + "grad_norm": 1.7980393130608678, + "language_loss": 0.82748187, + "learning_rate": 7.598548559689777e-08, + "loss": 0.84924889, + "num_input_tokens_seen": 164491155, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15307617, + "step": 7606, + "time_per_iteration": 2.8713366985321045 + }, + { + "auxiliary_loss_clip": 0.01125184, + "auxiliary_loss_mlp": 0.01037589, + "balance_loss_clip": 1.03905189, + "balance_loss_mlp": 1.02231872, + "epoch": 0.914687669091565, + "flos": 16800269212800.0, + "grad_norm": 2.1958569232161094, + "language_loss": 0.81052232, + "learning_rate": 7.577295367924751e-08, + "loss": 0.83215004, + "num_input_tokens_seen": 164507555, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15270996, + "step": 7607, + "time_per_iteration": 2.7919600009918213 + }, + { + "auxiliary_loss_clip": 0.01129176, + "auxiliary_loss_mlp": 0.01041904, + "balance_loss_clip": 1.04143679, + "balance_loss_mlp": 1.02503538, + "epoch": 0.914807911982204, + "flos": 25773627012480.0, + "grad_norm": 1.5700749664849718, + "language_loss": 0.82143188, + "learning_rate": 7.556071366531002e-08, + "loss": 0.84314263, + "num_input_tokens_seen": 164528525, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16870117, + "step": 7608, + "time_per_iteration": 2.811488389968872 + }, + { + "auxiliary_loss_clip": 0.01127367, + "auxiliary_loss_mlp": 0.01048121, + "balance_loss_clip": 1.04074681, + "balance_loss_mlp": 1.03112221, + "epoch": 0.9149281548728432, + "flos": 19208043636480.0, + "grad_norm": 1.952899866363438, + "language_loss": 0.79177141, + "learning_rate": 7.53487655872822e-08, + "loss": 0.81352627, + "num_input_tokens_seen": 164547695, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.17004395, + "step": 7609, + "time_per_iteration": 3.429699420928955 + }, + { + "auxiliary_loss_clip": 0.01121679, + "auxiliary_loss_mlp": 0.01038751, + "balance_loss_clip": 1.037027, + "balance_loss_mlp": 1.0225091, + "epoch": 0.9150483977634822, + "flos": 26870554500480.0, + "grad_norm": 1.8202214241938726, + "language_loss": 0.7393291, + "learning_rate": 7.513710947731656e-08, + "loss": 0.7609334, + "num_input_tokens_seen": 164568905, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.16241455, + "step": 7610, + "time_per_iteration": 2.914057731628418 + }, + { + "auxiliary_loss_clip": 0.01127079, + "auxiliary_loss_mlp": 0.01037183, + "balance_loss_clip": 1.0415597, + "balance_loss_mlp": 1.02230608, + "epoch": 0.9151686406541213, + "flos": 21908956953600.0, + "grad_norm": 1.8278612187965408, + "language_loss": 0.85342014, + "learning_rate": 7.492574536752095e-08, + "loss": 0.87506276, + "num_input_tokens_seen": 164588895, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.14892578, + "step": 7611, + "time_per_iteration": 2.8073201179504395 + }, + { + "auxiliary_loss_clip": 0.01121884, + "auxiliary_loss_mlp": 0.01040368, + "balance_loss_clip": 1.0395453, + "balance_loss_mlp": 1.02561581, + "epoch": 0.9152888835447605, + "flos": 27308556944640.0, + "grad_norm": 1.868697177949604, + "language_loss": 0.77961057, + "learning_rate": 7.471467328995907e-08, + "loss": 0.80123311, + "num_input_tokens_seen": 164607705, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.14746094, + "step": 7612, + "time_per_iteration": 2.8214333057403564 + }, + { + "auxiliary_loss_clip": 0.01126787, + "auxiliary_loss_mlp": 0.01032845, + "balance_loss_clip": 1.04022276, + "balance_loss_mlp": 1.01592922, + "epoch": 0.9154091264353995, + "flos": 13370728510080.0, + "grad_norm": 2.169473522747968, + "language_loss": 0.60663581, + "learning_rate": 7.450389327665018e-08, + "loss": 0.62823218, + "num_input_tokens_seen": 164625540, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.16906738, + "step": 7613, + "time_per_iteration": 3.5673999786376953 + }, + { + "auxiliary_loss_clip": 0.01128766, + "auxiliary_loss_mlp": 0.01041814, + "balance_loss_clip": 1.04108465, + "balance_loss_mlp": 1.02493358, + "epoch": 0.9155293693260386, + "flos": 20193037367040.0, + "grad_norm": 2.7514386768498653, + "language_loss": 0.67138064, + "learning_rate": 7.429340535957029e-08, + "loss": 0.69308639, + "num_input_tokens_seen": 164640735, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.16870117, + "step": 7614, + "time_per_iteration": 2.7828965187072754 + }, + { + "auxiliary_loss_clip": 0.01127559, + "auxiliary_loss_mlp": 0.01032518, + "balance_loss_clip": 1.04277921, + "balance_loss_mlp": 1.01827884, + "epoch": 0.9156496122166777, + "flos": 19354990176000.0, + "grad_norm": 2.1039199606187724, + "language_loss": 0.71073097, + "learning_rate": 7.40832095706494e-08, + "loss": 0.73233175, + "num_input_tokens_seen": 164657430, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.14239502, + "step": 7615, + "time_per_iteration": 3.6318180561065674 + }, + { + "auxiliary_loss_clip": 0.0112648, + "auxiliary_loss_mlp": 0.01046117, + "balance_loss_clip": 1.04193783, + "balance_loss_mlp": 1.03146029, + "epoch": 0.9157698551073168, + "flos": 21107287261440.0, + "grad_norm": 1.725938583176545, + "language_loss": 0.80187666, + "learning_rate": 7.387330594177443e-08, + "loss": 0.82360268, + "num_input_tokens_seen": 164679505, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.14648438, + "step": 7616, + "time_per_iteration": 2.7618777751922607 + }, + { + "auxiliary_loss_clip": 0.01122325, + "auxiliary_loss_mlp": 0.01034789, + "balance_loss_clip": 1.03821445, + "balance_loss_mlp": 1.01993597, + "epoch": 0.9158900979979558, + "flos": 25193167228800.0, + "grad_norm": 1.637228426134167, + "language_loss": 0.79151762, + "learning_rate": 7.366369450478749e-08, + "loss": 0.81308877, + "num_input_tokens_seen": 164700615, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14855957, + "step": 7617, + "time_per_iteration": 2.80718731880188 + }, + { + "auxiliary_loss_clip": 0.01122666, + "auxiliary_loss_mlp": 0.01039602, + "balance_loss_clip": 1.03849483, + "balance_loss_mlp": 1.02466512, + "epoch": 0.916010340888595, + "flos": 30146648302080.0, + "grad_norm": 1.688786999788839, + "language_loss": 0.66159952, + "learning_rate": 7.345437529148646e-08, + "loss": 0.68322217, + "num_input_tokens_seen": 164719625, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.1494751, + "step": 7618, + "time_per_iteration": 2.84771990776062 + }, + { + "auxiliary_loss_clip": 0.01122047, + "auxiliary_loss_mlp": 0.01040523, + "balance_loss_clip": 1.03922749, + "balance_loss_mlp": 1.0251689, + "epoch": 0.9161305837792341, + "flos": 17091827907840.0, + "grad_norm": 2.447491011639103, + "language_loss": 0.72819293, + "learning_rate": 7.324534833362483e-08, + "loss": 0.74981862, + "num_input_tokens_seen": 164737200, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.15350342, + "step": 7619, + "time_per_iteration": 3.5830228328704834 + }, + { + "auxiliary_loss_clip": 0.01122245, + "auxiliary_loss_mlp": 0.01039993, + "balance_loss_clip": 1.03787076, + "balance_loss_mlp": 1.02526498, + "epoch": 0.9162508266698731, + "flos": 22893699288960.0, + "grad_norm": 1.5521905970280299, + "language_loss": 0.68141174, + "learning_rate": 7.303661366291192e-08, + "loss": 0.70303404, + "num_input_tokens_seen": 164757870, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.1472168, + "step": 7620, + "time_per_iteration": 2.8624305725097656 + }, + { + "auxiliary_loss_clip": 0.01125182, + "auxiliary_loss_mlp": 0.0103306, + "balance_loss_clip": 1.04236841, + "balance_loss_mlp": 1.01857591, + "epoch": 0.9163710695605123, + "flos": 19974808287360.0, + "grad_norm": 1.9825890568465796, + "language_loss": 0.81369781, + "learning_rate": 7.28281713110126e-08, + "loss": 0.83528018, + "num_input_tokens_seen": 164775945, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.1449585, + "step": 7621, + "time_per_iteration": 2.7700469493865967 + }, + { + "auxiliary_loss_clip": 0.01122672, + "auxiliary_loss_mlp": 0.01039698, + "balance_loss_clip": 1.04087472, + "balance_loss_mlp": 1.02589357, + "epoch": 0.9164913124511513, + "flos": 22783812606720.0, + "grad_norm": 3.462238503325419, + "language_loss": 0.7726717, + "learning_rate": 7.262002130954759e-08, + "loss": 0.79429543, + "num_input_tokens_seen": 164794400, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.13781738, + "step": 7622, + "time_per_iteration": 2.804955244064331 + }, + { + "auxiliary_loss_clip": 0.01128045, + "auxiliary_loss_mlp": 0.01042556, + "balance_loss_clip": 1.04301214, + "balance_loss_mlp": 1.02685022, + "epoch": 0.9166115553417904, + "flos": 24900854348160.0, + "grad_norm": 1.7909191578053796, + "language_loss": 0.79120433, + "learning_rate": 7.241216369009296e-08, + "loss": 0.81291032, + "num_input_tokens_seen": 164814585, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15704346, + "step": 7623, + "time_per_iteration": 2.847679853439331 + }, + { + "auxiliary_loss_clip": 0.0112266, + "auxiliary_loss_mlp": 0.01039649, + "balance_loss_clip": 1.03823483, + "balance_loss_mlp": 1.02478385, + "epoch": 0.9167317982324296, + "flos": 25702919089920.0, + "grad_norm": 1.8410509691554977, + "language_loss": 0.66348159, + "learning_rate": 7.220459848418037e-08, + "loss": 0.68510467, + "num_input_tokens_seen": 164834660, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.14880371, + "step": 7624, + "time_per_iteration": 2.808086395263672 + }, + { + "auxiliary_loss_clip": 0.01122326, + "auxiliary_loss_mlp": 0.01036797, + "balance_loss_clip": 1.0387907, + "balance_loss_mlp": 1.02187848, + "epoch": 0.9168520411230686, + "flos": 15632813370240.0, + "grad_norm": 2.4802340632137576, + "language_loss": 0.7969451, + "learning_rate": 7.199732572329708e-08, + "loss": 0.81853628, + "num_input_tokens_seen": 164852560, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.14929199, + "step": 7625, + "time_per_iteration": 2.863980531692505 + }, + { + "auxiliary_loss_clip": 0.01127177, + "auxiliary_loss_mlp": 0.01039269, + "balance_loss_clip": 1.04148388, + "balance_loss_mlp": 1.02365303, + "epoch": 0.9169722840137077, + "flos": 30258151096320.0, + "grad_norm": 2.160887780791171, + "language_loss": 0.76049989, + "learning_rate": 7.179034543888684e-08, + "loss": 0.78216434, + "num_input_tokens_seen": 164872065, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15600586, + "step": 7626, + "time_per_iteration": 2.834519386291504 + }, + { + "auxiliary_loss_clip": 0.01124362, + "auxiliary_loss_mlp": 0.01035799, + "balance_loss_clip": 1.03854775, + "balance_loss_mlp": 1.02023005, + "epoch": 0.9170925269043467, + "flos": 22491643380480.0, + "grad_norm": 1.950704838064312, + "language_loss": 0.77162528, + "learning_rate": 7.158365766234808e-08, + "loss": 0.79322684, + "num_input_tokens_seen": 164890915, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15576172, + "step": 7627, + "time_per_iteration": 2.8643293380737305 + }, + { + "auxiliary_loss_clip": 0.01126603, + "auxiliary_loss_mlp": 0.0103686, + "balance_loss_clip": 1.03968048, + "balance_loss_mlp": 1.021065, + "epoch": 0.9172127697949859, + "flos": 22893914770560.0, + "grad_norm": 2.002236162883617, + "language_loss": 0.72349191, + "learning_rate": 7.137726242503527e-08, + "loss": 0.74512661, + "num_input_tokens_seen": 164909835, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15795898, + "step": 7628, + "time_per_iteration": 2.763728141784668 + }, + { + "auxiliary_loss_clip": 0.01124734, + "auxiliary_loss_mlp": 0.01033416, + "balance_loss_clip": 1.03931475, + "balance_loss_mlp": 1.01903355, + "epoch": 0.917333012685625, + "flos": 17451867882240.0, + "grad_norm": 1.9137277188020883, + "language_loss": 0.78416681, + "learning_rate": 7.11711597582585e-08, + "loss": 0.80574834, + "num_input_tokens_seen": 164927195, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.14373779, + "step": 7629, + "time_per_iteration": 2.814744710922241 + }, + { + "auxiliary_loss_clip": 0.01120177, + "auxiliary_loss_mlp": 0.01033746, + "balance_loss_clip": 1.03818107, + "balance_loss_mlp": 1.01931524, + "epoch": 0.917453255576264, + "flos": 14318949692160.0, + "grad_norm": 1.6876642469918448, + "language_loss": 0.79837716, + "learning_rate": 7.096534969328271e-08, + "loss": 0.81991637, + "num_input_tokens_seen": 164944640, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.14422607, + "step": 7630, + "time_per_iteration": 2.907679796218872 + }, + { + "auxiliary_loss_clip": 0.01122559, + "auxiliary_loss_mlp": 0.01041606, + "balance_loss_clip": 1.03784609, + "balance_loss_mlp": 1.02699137, + "epoch": 0.9175734984669032, + "flos": 20741177888640.0, + "grad_norm": 4.015422254676053, + "language_loss": 0.84103358, + "learning_rate": 7.075983226132987e-08, + "loss": 0.86267525, + "num_input_tokens_seen": 164963570, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14624023, + "step": 7631, + "time_per_iteration": 2.872036933898926 + }, + { + "auxiliary_loss_clip": 0.01129826, + "auxiliary_loss_mlp": 0.01041185, + "balance_loss_clip": 1.04217649, + "balance_loss_mlp": 1.02502072, + "epoch": 0.9176937413575422, + "flos": 14830497233280.0, + "grad_norm": 2.846125568381108, + "language_loss": 0.79754013, + "learning_rate": 7.055460749357656e-08, + "loss": 0.81925029, + "num_input_tokens_seen": 164979850, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.16174316, + "step": 7632, + "time_per_iteration": 2.7681844234466553 + }, + { + "auxiliary_loss_clip": 0.01123851, + "auxiliary_loss_mlp": 0.0103925, + "balance_loss_clip": 1.03913021, + "balance_loss_mlp": 1.02355051, + "epoch": 0.9178139842481813, + "flos": 18474603828480.0, + "grad_norm": 1.610574311748408, + "language_loss": 0.70565355, + "learning_rate": 7.034967542115521e-08, + "loss": 0.72728455, + "num_input_tokens_seen": 164998115, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.15710449, + "step": 7633, + "time_per_iteration": 2.7372283935546875 + }, + { + "auxiliary_loss_clip": 0.01121433, + "auxiliary_loss_mlp": 0.01039904, + "balance_loss_clip": 1.03859735, + "balance_loss_mlp": 1.02472854, + "epoch": 0.9179342271388204, + "flos": 20047455544320.0, + "grad_norm": 2.0290695885402696, + "language_loss": 0.75192201, + "learning_rate": 7.014503607515388e-08, + "loss": 0.77353543, + "num_input_tokens_seen": 165017420, + "router_z_loss_clip": 0.828125, + "router_z_loss_mlp": 0.15185547, + "step": 7634, + "time_per_iteration": 2.776280164718628 + }, + { + "auxiliary_loss_clip": 0.01131429, + "auxiliary_loss_mlp": 0.01038373, + "balance_loss_clip": 1.04364395, + "balance_loss_mlp": 1.02360296, + "epoch": 0.9180544700294595, + "flos": 24676232647680.0, + "grad_norm": 2.77798191878845, + "language_loss": 0.68334389, + "learning_rate": 6.994068948661592e-08, + "loss": 0.70504189, + "num_input_tokens_seen": 165035575, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.14752197, + "step": 7635, + "time_per_iteration": 2.8272297382354736 + }, + { + "auxiliary_loss_clip": 0.01127266, + "auxiliary_loss_mlp": 0.01044305, + "balance_loss_clip": 1.03857207, + "balance_loss_mlp": 1.02660263, + "epoch": 0.9181747129200986, + "flos": 16727478301440.0, + "grad_norm": 1.9686365090740978, + "language_loss": 0.76216877, + "learning_rate": 6.973663568654142e-08, + "loss": 0.78388447, + "num_input_tokens_seen": 165053280, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.17700195, + "step": 7636, + "time_per_iteration": 2.742841958999634 + }, + { + "auxiliary_loss_clip": 0.01125739, + "auxiliary_loss_mlp": 0.01041047, + "balance_loss_clip": 1.04127073, + "balance_loss_mlp": 1.02589583, + "epoch": 0.9182949558107377, + "flos": 24271626873600.0, + "grad_norm": 4.057708986653468, + "language_loss": 0.65353167, + "learning_rate": 6.953287470588386e-08, + "loss": 0.67519951, + "num_input_tokens_seen": 165071235, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15161133, + "step": 7637, + "time_per_iteration": 2.7556660175323486 + }, + { + "auxiliary_loss_clip": 0.01125236, + "auxiliary_loss_mlp": 0.01040925, + "balance_loss_clip": 1.03955066, + "balance_loss_mlp": 1.02527308, + "epoch": 0.9184151987013768, + "flos": 22082117443200.0, + "grad_norm": 2.3389287900797653, + "language_loss": 0.85685086, + "learning_rate": 6.932940657555452e-08, + "loss": 0.87851244, + "num_input_tokens_seen": 165087365, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15661621, + "step": 7638, + "time_per_iteration": 3.4491066932678223 + }, + { + "auxiliary_loss_clip": 0.01117917, + "auxiliary_loss_mlp": 0.0103562, + "balance_loss_clip": 1.03875709, + "balance_loss_mlp": 1.02111197, + "epoch": 0.9185354415920158, + "flos": 32166732257280.0, + "grad_norm": 1.3771098639864412, + "language_loss": 0.76525474, + "learning_rate": 6.912623132641938e-08, + "loss": 0.78679013, + "num_input_tokens_seen": 165112455, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.1449585, + "step": 7639, + "time_per_iteration": 2.9025352001190186 + }, + { + "auxiliary_loss_clip": 0.01127484, + "auxiliary_loss_mlp": 0.01035608, + "balance_loss_clip": 1.04089975, + "balance_loss_mlp": 1.01776266, + "epoch": 0.918655684482655, + "flos": 20997831542400.0, + "grad_norm": 1.9721784905835065, + "language_loss": 0.76518238, + "learning_rate": 6.892334898929952e-08, + "loss": 0.78681332, + "num_input_tokens_seen": 165132700, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.17858887, + "step": 7640, + "time_per_iteration": 2.7501351833343506 + }, + { + "auxiliary_loss_clip": 0.01124451, + "auxiliary_loss_mlp": 0.01036714, + "balance_loss_clip": 1.04017019, + "balance_loss_mlp": 1.02156258, + "epoch": 0.918775927373294, + "flos": 15560704817280.0, + "grad_norm": 1.8787003313552235, + "language_loss": 0.84656823, + "learning_rate": 6.872075959497236e-08, + "loss": 0.8681798, + "num_input_tokens_seen": 165151475, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.15148926, + "step": 7641, + "time_per_iteration": 2.789811372756958 + }, + { + "auxiliary_loss_clip": 0.01123114, + "auxiliary_loss_mlp": 0.01036898, + "balance_loss_clip": 1.03863788, + "balance_loss_mlp": 1.02117443, + "epoch": 0.9188961702639331, + "flos": 29934057657600.0, + "grad_norm": 1.8543327327687626, + "language_loss": 0.82825243, + "learning_rate": 6.85184631741702e-08, + "loss": 0.84985244, + "num_input_tokens_seen": 165172040, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.1574707, + "step": 7642, + "time_per_iteration": 3.6909358501434326 + }, + { + "auxiliary_loss_clip": 0.01122906, + "auxiliary_loss_mlp": 0.01032274, + "balance_loss_clip": 1.03760326, + "balance_loss_mlp": 1.01726532, + "epoch": 0.9190164131545723, + "flos": 20701244943360.0, + "grad_norm": 6.274086619007707, + "language_loss": 0.77289116, + "learning_rate": 6.831645975758161e-08, + "loss": 0.79444289, + "num_input_tokens_seen": 165189980, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.15014648, + "step": 7643, + "time_per_iteration": 2.8395040035247803 + }, + { + "auxiliary_loss_clip": 0.01128116, + "auxiliary_loss_mlp": 0.01038696, + "balance_loss_clip": 1.04029107, + "balance_loss_mlp": 1.02205491, + "epoch": 0.9191366560452113, + "flos": 25629912696960.0, + "grad_norm": 1.9464950195593382, + "language_loss": 0.67470872, + "learning_rate": 6.811474937585026e-08, + "loss": 0.69637692, + "num_input_tokens_seen": 165209770, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.16650391, + "step": 7644, + "time_per_iteration": 3.640228748321533 + }, + { + "auxiliary_loss_clip": 0.01122683, + "auxiliary_loss_mlp": 0.01035241, + "balance_loss_clip": 1.0416615, + "balance_loss_mlp": 1.0206027, + "epoch": 0.9192568989358504, + "flos": 21434325615360.0, + "grad_norm": 1.6899554684323594, + "language_loss": 0.79235697, + "learning_rate": 6.79133320595755e-08, + "loss": 0.81393623, + "num_input_tokens_seen": 165229690, + "router_z_loss_clip": 0.81005859, + "router_z_loss_mlp": 0.14624023, + "step": 7645, + "time_per_iteration": 2.8924965858459473 + }, + { + "auxiliary_loss_clip": 0.01125106, + "auxiliary_loss_mlp": 0.01028988, + "balance_loss_clip": 1.04024577, + "balance_loss_mlp": 1.01472425, + "epoch": 0.9193771418264896, + "flos": 23185078416000.0, + "grad_norm": 1.6458821791527414, + "language_loss": 0.7527082, + "learning_rate": 6.771220783931198e-08, + "loss": 0.77424914, + "num_input_tokens_seen": 165249850, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.14257812, + "step": 7646, + "time_per_iteration": 2.776763677597046 + }, + { + "auxiliary_loss_clip": 0.01031725, + "auxiliary_loss_mlp": 0.01000644, + "balance_loss_clip": 1.00415933, + "balance_loss_mlp": 0.99936813, + "epoch": 0.9194973847171286, + "flos": 70582963184640.0, + "grad_norm": 0.8589335151857873, + "language_loss": 0.64556772, + "learning_rate": 6.751137674556994e-08, + "loss": 0.66589135, + "num_input_tokens_seen": 165310235, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01275635, + "step": 7647, + "time_per_iteration": 3.3161940574645996 + }, + { + "auxiliary_loss_clip": 0.01126613, + "auxiliary_loss_mlp": 0.01035083, + "balance_loss_clip": 1.03985119, + "balance_loss_mlp": 1.01997936, + "epoch": 0.9196176276077677, + "flos": 14720682378240.0, + "grad_norm": 2.150650690335145, + "language_loss": 0.77433944, + "learning_rate": 6.731083880881572e-08, + "loss": 0.79595637, + "num_input_tokens_seen": 165326455, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.15106201, + "step": 7648, + "time_per_iteration": 3.6272099018096924 + }, + { + "auxiliary_loss_clip": 0.0112157, + "auxiliary_loss_mlp": 0.0104391, + "balance_loss_clip": 1.03896642, + "balance_loss_mlp": 1.02910471, + "epoch": 0.9197378704984068, + "flos": 23294893271040.0, + "grad_norm": 1.9063999491119272, + "language_loss": 0.80769598, + "learning_rate": 6.711059405947072e-08, + "loss": 0.82935083, + "num_input_tokens_seen": 165344645, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.14807129, + "step": 7649, + "time_per_iteration": 2.855762243270874 + }, + { + "auxiliary_loss_clip": 0.01121176, + "auxiliary_loss_mlp": 0.01040771, + "balance_loss_clip": 1.03748465, + "balance_loss_mlp": 1.025846, + "epoch": 0.9198581133890459, + "flos": 20302564913280.0, + "grad_norm": 1.5912305947776295, + "language_loss": 0.77062058, + "learning_rate": 6.691064252791156e-08, + "loss": 0.79224002, + "num_input_tokens_seen": 165364120, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14916992, + "step": 7650, + "time_per_iteration": 2.776385545730591 + }, + { + "auxiliary_loss_clip": 0.01124134, + "auxiliary_loss_mlp": 0.010279, + "balance_loss_clip": 1.03774619, + "balance_loss_mlp": 1.01205683, + "epoch": 0.9199783562796849, + "flos": 17675663569920.0, + "grad_norm": 1.5211440277945818, + "language_loss": 0.78068161, + "learning_rate": 6.67109842444713e-08, + "loss": 0.80220199, + "num_input_tokens_seen": 165383050, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.1583252, + "step": 7651, + "time_per_iteration": 2.807893753051758 + }, + { + "auxiliary_loss_clip": 0.01125381, + "auxiliary_loss_mlp": 0.01045874, + "balance_loss_clip": 1.03989005, + "balance_loss_mlp": 1.03134286, + "epoch": 0.9200985991703241, + "flos": 17676022705920.0, + "grad_norm": 1.935057635457296, + "language_loss": 0.76797354, + "learning_rate": 6.651161923943704e-08, + "loss": 0.78968614, + "num_input_tokens_seen": 165400955, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.14526367, + "step": 7652, + "time_per_iteration": 2.7823359966278076 + }, + { + "auxiliary_loss_clip": 0.01128584, + "auxiliary_loss_mlp": 0.01036125, + "balance_loss_clip": 1.04065418, + "balance_loss_mlp": 1.02085471, + "epoch": 0.9202188420609632, + "flos": 20996574566400.0, + "grad_norm": 1.6496704483433973, + "language_loss": 0.77066588, + "learning_rate": 6.631254754305326e-08, + "loss": 0.79231304, + "num_input_tokens_seen": 165420415, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.15270996, + "step": 7653, + "time_per_iteration": 2.7263741493225098 + }, + { + "auxiliary_loss_clip": 0.01124484, + "auxiliary_loss_mlp": 0.0103814, + "balance_loss_clip": 1.03856206, + "balance_loss_mlp": 1.02386439, + "epoch": 0.9203390849516022, + "flos": 13918222586880.0, + "grad_norm": 1.8312437321445738, + "language_loss": 0.78469157, + "learning_rate": 6.611376918551848e-08, + "loss": 0.80631781, + "num_input_tokens_seen": 165439200, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.14294434, + "step": 7654, + "time_per_iteration": 2.794542074203491 + }, + { + "auxiliary_loss_clip": 0.01125135, + "auxiliary_loss_mlp": 0.0104062, + "balance_loss_clip": 1.0383904, + "balance_loss_mlp": 1.02509332, + "epoch": 0.9204593278422414, + "flos": 21175912195200.0, + "grad_norm": 2.0370199322623685, + "language_loss": 0.79642618, + "learning_rate": 6.591528419698744e-08, + "loss": 0.8180837, + "num_input_tokens_seen": 165458985, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15533447, + "step": 7655, + "time_per_iteration": 2.7650976181030273 + }, + { + "auxiliary_loss_clip": 0.01124383, + "auxiliary_loss_mlp": 0.01035651, + "balance_loss_clip": 1.04018378, + "balance_loss_mlp": 1.02187645, + "epoch": 0.9205795707328804, + "flos": 14501375890560.0, + "grad_norm": 2.5764488262258025, + "language_loss": 0.83457267, + "learning_rate": 6.571709260756986e-08, + "loss": 0.85617304, + "num_input_tokens_seen": 165475630, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.13763428, + "step": 7656, + "time_per_iteration": 2.7949159145355225 + }, + { + "auxiliary_loss_clip": 0.01130002, + "auxiliary_loss_mlp": 0.01042276, + "balance_loss_clip": 1.04360771, + "balance_loss_mlp": 1.0268563, + "epoch": 0.9206998136235195, + "flos": 22417559579520.0, + "grad_norm": 3.320279360997751, + "language_loss": 0.76420414, + "learning_rate": 6.551919444733122e-08, + "loss": 0.78592694, + "num_input_tokens_seen": 165493445, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.1539917, + "step": 7657, + "time_per_iteration": 2.721355676651001 + }, + { + "auxiliary_loss_clip": 0.01124051, + "auxiliary_loss_mlp": 0.01036129, + "balance_loss_clip": 1.03929913, + "balance_loss_mlp": 1.02072108, + "epoch": 0.9208200565141585, + "flos": 53358407544960.0, + "grad_norm": 2.067376609468654, + "language_loss": 0.66038322, + "learning_rate": 6.53215897462931e-08, + "loss": 0.68198502, + "num_input_tokens_seen": 165517200, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.1539917, + "step": 7658, + "time_per_iteration": 3.0338590145111084 + }, + { + "auxiliary_loss_clip": 0.01124964, + "auxiliary_loss_mlp": 0.01035355, + "balance_loss_clip": 1.04110062, + "balance_loss_mlp": 1.02012038, + "epoch": 0.9209402994047977, + "flos": 30589139946240.0, + "grad_norm": 1.9974049778370178, + "language_loss": 0.74591947, + "learning_rate": 6.512427853443103e-08, + "loss": 0.76752263, + "num_input_tokens_seen": 165539280, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.15228271, + "step": 7659, + "time_per_iteration": 2.860360860824585 + }, + { + "auxiliary_loss_clip": 0.01122126, + "auxiliary_loss_mlp": 0.01035682, + "balance_loss_clip": 1.03956199, + "balance_loss_mlp": 1.02025628, + "epoch": 0.9210605422954368, + "flos": 29132711187840.0, + "grad_norm": 1.5449340080861553, + "language_loss": 0.75816184, + "learning_rate": 6.492726084167799e-08, + "loss": 0.77973992, + "num_input_tokens_seen": 165561395, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.15429688, + "step": 7660, + "time_per_iteration": 2.784553050994873 + }, + { + "auxiliary_loss_clip": 0.01031422, + "auxiliary_loss_mlp": 0.01000922, + "balance_loss_clip": 1.00385499, + "balance_loss_mlp": 0.99957544, + "epoch": 0.9211807851860758, + "flos": 54853838472960.0, + "grad_norm": 0.7805311707227945, + "language_loss": 0.57563663, + "learning_rate": 6.473053669792072e-08, + "loss": 0.59596014, + "num_input_tokens_seen": 165616085, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01348877, + "step": 7661, + "time_per_iteration": 3.198054075241089 + }, + { + "auxiliary_loss_clip": 0.01126811, + "auxiliary_loss_mlp": 0.01037769, + "balance_loss_clip": 1.04021859, + "balance_loss_mlp": 1.0231123, + "epoch": 0.921301028076715, + "flos": 19201974238080.0, + "grad_norm": 2.2739635961723725, + "language_loss": 0.72668159, + "learning_rate": 6.453410613300248e-08, + "loss": 0.74832743, + "num_input_tokens_seen": 165634015, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.14648438, + "step": 7662, + "time_per_iteration": 2.810412645339966 + }, + { + "auxiliary_loss_clip": 0.01126071, + "auxiliary_loss_mlp": 0.01040006, + "balance_loss_clip": 1.04154539, + "balance_loss_mlp": 1.02548635, + "epoch": 0.921421270967354, + "flos": 27526893765120.0, + "grad_norm": 1.5956649180258435, + "language_loss": 0.58431447, + "learning_rate": 6.43379691767214e-08, + "loss": 0.60597521, + "num_input_tokens_seen": 165653220, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14526367, + "step": 7663, + "time_per_iteration": 2.8954265117645264 + }, + { + "auxiliary_loss_clip": 0.01031157, + "auxiliary_loss_mlp": 0.0100077, + "balance_loss_clip": 1.00365078, + "balance_loss_mlp": 0.99953002, + "epoch": 0.9215415138579931, + "flos": 70209311955840.0, + "grad_norm": 0.718197735338011, + "language_loss": 0.55136114, + "learning_rate": 6.414212585883105e-08, + "loss": 0.57168043, + "num_input_tokens_seen": 165715850, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01239014, + "step": 7664, + "time_per_iteration": 3.3230655193328857 + }, + { + "auxiliary_loss_clip": 0.01124852, + "auxiliary_loss_mlp": 0.01034835, + "balance_loss_clip": 1.03846383, + "balance_loss_mlp": 1.01945662, + "epoch": 0.9216617567486323, + "flos": 35553107790720.0, + "grad_norm": 1.5275421208186517, + "language_loss": 0.69777095, + "learning_rate": 6.394657620904143e-08, + "loss": 0.71936786, + "num_input_tokens_seen": 165738960, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.15380859, + "step": 7665, + "time_per_iteration": 2.9387528896331787 + }, + { + "auxiliary_loss_clip": 0.01129447, + "auxiliary_loss_mlp": 0.01034052, + "balance_loss_clip": 1.04154825, + "balance_loss_mlp": 1.01990223, + "epoch": 0.9217819996392713, + "flos": 29533330552320.0, + "grad_norm": 1.658697447038936, + "language_loss": 0.72012001, + "learning_rate": 6.375132025701657e-08, + "loss": 0.74175501, + "num_input_tokens_seen": 165761260, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.14141846, + "step": 7666, + "time_per_iteration": 2.8809404373168945 + }, + { + "auxiliary_loss_clip": 0.01128365, + "auxiliary_loss_mlp": 0.01037459, + "balance_loss_clip": 1.04087567, + "balance_loss_mlp": 1.02047133, + "epoch": 0.9219022425299104, + "flos": 14574669592320.0, + "grad_norm": 2.9742739614144793, + "language_loss": 0.69323325, + "learning_rate": 6.355635803237724e-08, + "loss": 0.71489143, + "num_input_tokens_seen": 165776960, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.1697998, + "step": 7667, + "time_per_iteration": 3.635392665863037 + }, + { + "auxiliary_loss_clip": 0.01127166, + "auxiliary_loss_mlp": 0.0103334, + "balance_loss_clip": 1.04025149, + "balance_loss_mlp": 1.01734197, + "epoch": 0.9220224854205495, + "flos": 18077503996800.0, + "grad_norm": 3.9858423928410676, + "language_loss": 0.79536408, + "learning_rate": 6.336168956469867e-08, + "loss": 0.81696916, + "num_input_tokens_seen": 165795435, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.16003418, + "step": 7668, + "time_per_iteration": 2.7730462551116943 + }, + { + "auxiliary_loss_clip": 0.01124368, + "auxiliary_loss_mlp": 0.01032815, + "balance_loss_clip": 1.04176235, + "balance_loss_mlp": 1.01800346, + "epoch": 0.9221427283111886, + "flos": 24790464875520.0, + "grad_norm": 1.7129523849412345, + "language_loss": 0.72062826, + "learning_rate": 6.316731488351168e-08, + "loss": 0.74220002, + "num_input_tokens_seen": 165816625, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.14807129, + "step": 7669, + "time_per_iteration": 2.8607771396636963 + }, + { + "auxiliary_loss_clip": 0.01123051, + "auxiliary_loss_mlp": 0.01038197, + "balance_loss_clip": 1.03864861, + "balance_loss_mlp": 1.02307546, + "epoch": 0.9222629712018277, + "flos": 13845036625920.0, + "grad_norm": 1.7497525825215894, + "language_loss": 0.63597548, + "learning_rate": 6.297323401830334e-08, + "loss": 0.65758801, + "num_input_tokens_seen": 165835410, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.15130615, + "step": 7670, + "time_per_iteration": 3.5326740741729736 + }, + { + "auxiliary_loss_clip": 0.01125973, + "auxiliary_loss_mlp": 0.01042066, + "balance_loss_clip": 1.0392158, + "balance_loss_mlp": 1.02623463, + "epoch": 0.9223832140924668, + "flos": 21616177196160.0, + "grad_norm": 2.9300224901106655, + "language_loss": 0.68814254, + "learning_rate": 6.277944699851523e-08, + "loss": 0.70982289, + "num_input_tokens_seen": 165854930, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.1583252, + "step": 7671, + "time_per_iteration": 2.851691484451294 + }, + { + "auxiliary_loss_clip": 0.01125586, + "auxiliary_loss_mlp": 0.01034865, + "balance_loss_clip": 1.03851962, + "balance_loss_mlp": 1.0189383, + "epoch": 0.9225034569831059, + "flos": 21142084561920.0, + "grad_norm": 2.0716615972130783, + "language_loss": 0.73123771, + "learning_rate": 6.25859538535447e-08, + "loss": 0.75284219, + "num_input_tokens_seen": 165875725, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.15917969, + "step": 7672, + "time_per_iteration": 2.843308687210083 + }, + { + "auxiliary_loss_clip": 0.01126697, + "auxiliary_loss_mlp": 0.01034601, + "balance_loss_clip": 1.03952074, + "balance_loss_mlp": 1.01922333, + "epoch": 0.9226236998737449, + "flos": 12495046844160.0, + "grad_norm": 3.110011108361518, + "language_loss": 0.77664578, + "learning_rate": 6.239275461274474e-08, + "loss": 0.79825878, + "num_input_tokens_seen": 165892100, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15368652, + "step": 7673, + "time_per_iteration": 3.5612356662750244 + }, + { + "auxiliary_loss_clip": 0.01126023, + "auxiliary_loss_mlp": 0.01044288, + "balance_loss_clip": 1.04097986, + "balance_loss_mlp": 1.02926195, + "epoch": 0.9227439427643841, + "flos": 26214071581440.0, + "grad_norm": 1.7129946897068158, + "language_loss": 0.85900027, + "learning_rate": 6.219984930542299e-08, + "loss": 0.88070339, + "num_input_tokens_seen": 165912840, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.15039062, + "step": 7674, + "time_per_iteration": 2.785712480545044 + }, + { + "auxiliary_loss_clip": 0.0112361, + "auxiliary_loss_mlp": 0.01037389, + "balance_loss_clip": 1.03902364, + "balance_loss_mlp": 1.02209496, + "epoch": 0.9228641856550232, + "flos": 17967581400960.0, + "grad_norm": 1.9410872197713844, + "language_loss": 0.75948942, + "learning_rate": 6.200723796084383e-08, + "loss": 0.78109944, + "num_input_tokens_seen": 165930935, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.1529541, + "step": 7675, + "time_per_iteration": 2.7249526977539062 + }, + { + "auxiliary_loss_clip": 0.01031485, + "auxiliary_loss_mlp": 0.01001689, + "balance_loss_clip": 1.00386405, + "balance_loss_mlp": 1.0004673, + "epoch": 0.9229844285456622, + "flos": 70420609710720.0, + "grad_norm": 0.8007013079574808, + "language_loss": 0.62994736, + "learning_rate": 6.181492060822546e-08, + "loss": 0.65027916, + "num_input_tokens_seen": 165991110, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01220703, + "step": 7676, + "time_per_iteration": 3.192288875579834 + }, + { + "auxiliary_loss_clip": 0.0112436, + "auxiliary_loss_mlp": 0.01041293, + "balance_loss_clip": 1.04089057, + "balance_loss_mlp": 1.02642179, + "epoch": 0.9231046714363014, + "flos": 17967832796160.0, + "grad_norm": 1.9543509579027722, + "language_loss": 0.81669343, + "learning_rate": 6.162289727674274e-08, + "loss": 0.83834994, + "num_input_tokens_seen": 166008790, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.14880371, + "step": 7677, + "time_per_iteration": 3.6414992809295654 + }, + { + "auxiliary_loss_clip": 0.01118586, + "auxiliary_loss_mlp": 0.01031492, + "balance_loss_clip": 1.0364511, + "balance_loss_mlp": 1.01684117, + "epoch": 0.9232249143269404, + "flos": 17858233422720.0, + "grad_norm": 2.6803287147911634, + "language_loss": 0.87720007, + "learning_rate": 6.143116799552527e-08, + "loss": 0.89870083, + "num_input_tokens_seen": 166025035, + "router_z_loss_clip": 0.82080078, + "router_z_loss_mlp": 0.14660645, + "step": 7678, + "time_per_iteration": 2.773580312728882 + }, + { + "auxiliary_loss_clip": 0.01126529, + "auxiliary_loss_mlp": 0.01039129, + "balance_loss_clip": 1.03953838, + "balance_loss_mlp": 1.02388263, + "epoch": 0.9233451572175795, + "flos": 23404384903680.0, + "grad_norm": 2.012069109171905, + "language_loss": 0.55531788, + "learning_rate": 6.123973279365802e-08, + "loss": 0.57697445, + "num_input_tokens_seen": 166044010, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.15234375, + "step": 7679, + "time_per_iteration": 2.799272298812866 + }, + { + "auxiliary_loss_clip": 0.01126591, + "auxiliary_loss_mlp": 0.01038613, + "balance_loss_clip": 1.04200602, + "balance_loss_mlp": 1.02275801, + "epoch": 0.9234654001082186, + "flos": 17999326045440.0, + "grad_norm": 2.0694408873436467, + "language_loss": 0.7811656, + "learning_rate": 6.10485917001824e-08, + "loss": 0.80281758, + "num_input_tokens_seen": 166061865, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.15881348, + "step": 7680, + "time_per_iteration": 2.7136740684509277 + }, + { + "auxiliary_loss_clip": 0.01121978, + "auxiliary_loss_mlp": 0.01036359, + "balance_loss_clip": 1.03874445, + "balance_loss_mlp": 1.02226281, + "epoch": 0.9235856429988577, + "flos": 24750747411840.0, + "grad_norm": 1.4986742344551758, + "language_loss": 0.81117094, + "learning_rate": 6.085774474409322e-08, + "loss": 0.83275437, + "num_input_tokens_seen": 166082425, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.14093018, + "step": 7681, + "time_per_iteration": 2.793621778488159 + }, + { + "auxiliary_loss_clip": 0.01126565, + "auxiliary_loss_mlp": 0.01045621, + "balance_loss_clip": 1.04259133, + "balance_loss_mlp": 1.03098249, + "epoch": 0.9237058858894968, + "flos": 14099894599680.0, + "grad_norm": 2.213306457880079, + "language_loss": 0.69899571, + "learning_rate": 6.066719195434267e-08, + "loss": 0.72071755, + "num_input_tokens_seen": 166100225, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.1463623, + "step": 7682, + "time_per_iteration": 2.7276575565338135 + }, + { + "auxiliary_loss_clip": 0.01129801, + "auxiliary_loss_mlp": 0.01044723, + "balance_loss_clip": 1.04093134, + "balance_loss_mlp": 1.02818847, + "epoch": 0.9238261287801359, + "flos": 28694529175680.0, + "grad_norm": 1.8713445662626693, + "language_loss": 0.66295016, + "learning_rate": 6.047693335983717e-08, + "loss": 0.68469542, + "num_input_tokens_seen": 166122570, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.16540527, + "step": 7683, + "time_per_iteration": 2.8738856315612793 + }, + { + "auxiliary_loss_clip": 0.01124832, + "auxiliary_loss_mlp": 0.01036687, + "balance_loss_clip": 1.03741193, + "balance_loss_mlp": 1.0202477, + "epoch": 0.923946371670775, + "flos": 23111856541440.0, + "grad_norm": 3.1101994643029505, + "language_loss": 0.8275919, + "learning_rate": 6.028696898943853e-08, + "loss": 0.84920716, + "num_input_tokens_seen": 166141630, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.16430664, + "step": 7684, + "time_per_iteration": 2.820051908493042 + }, + { + "auxiliary_loss_clip": 0.01128544, + "auxiliary_loss_mlp": 0.01035863, + "balance_loss_clip": 1.04064453, + "balance_loss_mlp": 1.01930523, + "epoch": 0.924066614561414, + "flos": 21867120587520.0, + "grad_norm": 2.1184414085889363, + "language_loss": 0.70615625, + "learning_rate": 6.00972988719648e-08, + "loss": 0.72780037, + "num_input_tokens_seen": 166159865, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.16552734, + "step": 7685, + "time_per_iteration": 2.778658151626587 + }, + { + "auxiliary_loss_clip": 0.0113044, + "auxiliary_loss_mlp": 0.01038117, + "balance_loss_clip": 1.03984857, + "balance_loss_mlp": 1.02093935, + "epoch": 0.9241868574520532, + "flos": 28511887495680.0, + "grad_norm": 2.6145203926643323, + "language_loss": 0.70465934, + "learning_rate": 5.990792303618807e-08, + "loss": 0.72634494, + "num_input_tokens_seen": 166179445, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.171875, + "step": 7686, + "time_per_iteration": 2.8675003051757812 + }, + { + "auxiliary_loss_clip": 0.01123983, + "auxiliary_loss_mlp": 0.0103577, + "balance_loss_clip": 1.03985035, + "balance_loss_mlp": 1.02018893, + "epoch": 0.9243071003426923, + "flos": 30518324282880.0, + "grad_norm": 1.718785396045228, + "language_loss": 0.69426763, + "learning_rate": 5.971884151083695e-08, + "loss": 0.71586514, + "num_input_tokens_seen": 166201855, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.15576172, + "step": 7687, + "time_per_iteration": 2.8053154945373535 + }, + { + "auxiliary_loss_clip": 0.01123238, + "auxiliary_loss_mlp": 0.01034088, + "balance_loss_clip": 1.04124689, + "balance_loss_mlp": 1.01966405, + "epoch": 0.9244273432333313, + "flos": 28658331244800.0, + "grad_norm": 1.9080077074133568, + "language_loss": 0.74493599, + "learning_rate": 5.9530054324595124e-08, + "loss": 0.76650923, + "num_input_tokens_seen": 166221970, + "router_z_loss_clip": 0.81982422, + "router_z_loss_mlp": 0.14416504, + "step": 7688, + "time_per_iteration": 2.8216519355773926 + }, + { + "auxiliary_loss_clip": 0.01031903, + "auxiliary_loss_mlp": 0.01001504, + "balance_loss_clip": 1.00427794, + "balance_loss_mlp": 1.00028253, + "epoch": 0.9245475861239704, + "flos": 66230589237120.0, + "grad_norm": 0.7210502396962561, + "language_loss": 0.57541287, + "learning_rate": 5.934156150610103e-08, + "loss": 0.59574699, + "num_input_tokens_seen": 166279335, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01220703, + "step": 7689, + "time_per_iteration": 3.3728232383728027 + }, + { + "auxiliary_loss_clip": 0.01129749, + "auxiliary_loss_mlp": 0.01035748, + "balance_loss_clip": 1.04276848, + "balance_loss_mlp": 1.02199745, + "epoch": 0.9246678290146095, + "flos": 24239918142720.0, + "grad_norm": 2.0357278058463657, + "language_loss": 0.7936846, + "learning_rate": 5.915336308394914e-08, + "loss": 0.81533957, + "num_input_tokens_seen": 166298170, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.13757324, + "step": 7690, + "time_per_iteration": 2.885572910308838 + }, + { + "auxiliary_loss_clip": 0.01121299, + "auxiliary_loss_mlp": 0.01037278, + "balance_loss_clip": 1.03811562, + "balance_loss_mlp": 1.02308047, + "epoch": 0.9247880719052486, + "flos": 18988808976000.0, + "grad_norm": 1.5575719901515592, + "language_loss": 0.76770592, + "learning_rate": 5.89654590866886e-08, + "loss": 0.78929168, + "num_input_tokens_seen": 166317670, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.1418457, + "step": 7691, + "time_per_iteration": 2.834083318710327 + }, + { + "auxiliary_loss_clip": 0.01127312, + "auxiliary_loss_mlp": 0.01041992, + "balance_loss_clip": 1.0397917, + "balance_loss_mlp": 1.02594626, + "epoch": 0.9249083147958876, + "flos": 24024095274240.0, + "grad_norm": 3.1252169008170503, + "language_loss": 0.88333166, + "learning_rate": 5.877784954282483e-08, + "loss": 0.90502477, + "num_input_tokens_seen": 166337010, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.16064453, + "step": 7692, + "time_per_iteration": 2.8082540035247803 + }, + { + "auxiliary_loss_clip": 0.0112849, + "auxiliary_loss_mlp": 0.01044523, + "balance_loss_clip": 1.04084051, + "balance_loss_mlp": 1.02740443, + "epoch": 0.9250285576865268, + "flos": 30773972355840.0, + "grad_norm": 1.6986062422309847, + "language_loss": 0.7253089, + "learning_rate": 5.8590534480817963e-08, + "loss": 0.74703908, + "num_input_tokens_seen": 166358735, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.17114258, + "step": 7693, + "time_per_iteration": 2.8807408809661865 + }, + { + "auxiliary_loss_clip": 0.0112518, + "auxiliary_loss_mlp": 0.01039806, + "balance_loss_clip": 1.04102504, + "balance_loss_mlp": 1.02506018, + "epoch": 0.9251488005771659, + "flos": 10633581348480.0, + "grad_norm": 2.0416469666800134, + "language_loss": 0.72428757, + "learning_rate": 5.840351392908349e-08, + "loss": 0.74593747, + "num_input_tokens_seen": 166374455, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.14758301, + "step": 7694, + "time_per_iteration": 2.7102699279785156 + }, + { + "auxiliary_loss_clip": 0.01126568, + "auxiliary_loss_mlp": 0.01035124, + "balance_loss_clip": 1.04119492, + "balance_loss_mlp": 1.0195787, + "epoch": 0.9252690434678049, + "flos": 23586416052480.0, + "grad_norm": 2.371447630662487, + "language_loss": 0.71052277, + "learning_rate": 5.821678791599205e-08, + "loss": 0.73213971, + "num_input_tokens_seen": 166393900, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15551758, + "step": 7695, + "time_per_iteration": 3.500600576400757 + }, + { + "auxiliary_loss_clip": 0.01123071, + "auxiliary_loss_mlp": 0.01037173, + "balance_loss_clip": 1.03896403, + "balance_loss_mlp": 1.0228436, + "epoch": 0.9253892863584441, + "flos": 21469158829440.0, + "grad_norm": 1.6586919735203116, + "language_loss": 0.80885696, + "learning_rate": 5.803035646986965e-08, + "loss": 0.83045942, + "num_input_tokens_seen": 166413235, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.14324951, + "step": 7696, + "time_per_iteration": 2.7679362297058105 + }, + { + "auxiliary_loss_clip": 0.01129385, + "auxiliary_loss_mlp": 0.01042706, + "balance_loss_clip": 1.04051268, + "balance_loss_mlp": 1.02686274, + "epoch": 0.9255095292490831, + "flos": 17456680304640.0, + "grad_norm": 2.253866854555953, + "language_loss": 0.67321765, + "learning_rate": 5.7844219618998766e-08, + "loss": 0.69493854, + "num_input_tokens_seen": 166427560, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.1583252, + "step": 7697, + "time_per_iteration": 2.7080748081207275 + }, + { + "auxiliary_loss_clip": 0.0112328, + "auxiliary_loss_mlp": 0.01042626, + "balance_loss_clip": 1.03850627, + "balance_loss_mlp": 1.02749848, + "epoch": 0.9256297721397222, + "flos": 24750675584640.0, + "grad_norm": 1.7909884569898777, + "language_loss": 0.71676683, + "learning_rate": 5.765837739161505e-08, + "loss": 0.73842591, + "num_input_tokens_seen": 166446680, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.15130615, + "step": 7698, + "time_per_iteration": 2.795398712158203 + }, + { + "auxiliary_loss_clip": 0.01122395, + "auxiliary_loss_mlp": 0.0103717, + "balance_loss_clip": 1.03880215, + "balance_loss_mlp": 1.02202463, + "epoch": 0.9257500150303614, + "flos": 23112215677440.0, + "grad_norm": 1.7128278233593313, + "language_loss": 0.74827886, + "learning_rate": 5.7472829815911504e-08, + "loss": 0.76987451, + "num_input_tokens_seen": 166465505, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.15142822, + "step": 7699, + "time_per_iteration": 3.505744457244873 + }, + { + "auxiliary_loss_clip": 0.01126606, + "auxiliary_loss_mlp": 0.01038981, + "balance_loss_clip": 1.04097617, + "balance_loss_mlp": 1.02311397, + "epoch": 0.9258702579210004, + "flos": 22564685687040.0, + "grad_norm": 1.7895339410133941, + "language_loss": 0.81547713, + "learning_rate": 5.7287576920035164e-08, + "loss": 0.83713299, + "num_input_tokens_seen": 166484520, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15869141, + "step": 7700, + "time_per_iteration": 2.800759792327881 + }, + { + "auxiliary_loss_clip": 0.01122262, + "auxiliary_loss_mlp": 0.01037712, + "balance_loss_clip": 1.03860545, + "balance_loss_mlp": 1.02190518, + "epoch": 0.9259905008116395, + "flos": 30004298703360.0, + "grad_norm": 1.8108622056613575, + "language_loss": 0.76674408, + "learning_rate": 5.7102618732088435e-08, + "loss": 0.78834379, + "num_input_tokens_seen": 166503850, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.15808105, + "step": 7701, + "time_per_iteration": 2.8056352138519287 + }, + { + "auxiliary_loss_clip": 0.01124103, + "auxiliary_loss_mlp": 0.01035589, + "balance_loss_clip": 1.04025292, + "balance_loss_mlp": 1.02118278, + "epoch": 0.9261107437022786, + "flos": 24572128055040.0, + "grad_norm": 1.6837752208809107, + "language_loss": 0.74519473, + "learning_rate": 5.6917955280130216e-08, + "loss": 0.76679164, + "num_input_tokens_seen": 166525330, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.14398193, + "step": 7702, + "time_per_iteration": 2.8268370628356934 + }, + { + "auxiliary_loss_clip": 0.01123483, + "auxiliary_loss_mlp": 0.01040734, + "balance_loss_clip": 1.03790259, + "balance_loss_mlp": 1.02477205, + "epoch": 0.9262309865929177, + "flos": 22018448586240.0, + "grad_norm": 2.384063638818514, + "language_loss": 0.7244097, + "learning_rate": 5.6733586592172755e-08, + "loss": 0.74605179, + "num_input_tokens_seen": 166544825, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.1595459, + "step": 7703, + "time_per_iteration": 3.690938949584961 + }, + { + "auxiliary_loss_clip": 0.01117588, + "auxiliary_loss_mlp": 0.01033948, + "balance_loss_clip": 1.03608418, + "balance_loss_mlp": 1.02002466, + "epoch": 0.9263512294835567, + "flos": 20339481116160.0, + "grad_norm": 2.1388863818142445, + "language_loss": 0.7976681, + "learning_rate": 5.6549512696185244e-08, + "loss": 0.81918347, + "num_input_tokens_seen": 166563325, + "router_z_loss_clip": 0.81445312, + "router_z_loss_mlp": 0.13928223, + "step": 7704, + "time_per_iteration": 2.824946880340576 + }, + { + "auxiliary_loss_clip": 0.01121574, + "auxiliary_loss_mlp": 0.01038316, + "balance_loss_clip": 1.03894806, + "balance_loss_mlp": 1.02359939, + "epoch": 0.9264714723741959, + "flos": 21215378263680.0, + "grad_norm": 1.5125879385888543, + "language_loss": 0.68060827, + "learning_rate": 5.636573362009156e-08, + "loss": 0.70220721, + "num_input_tokens_seen": 166583385, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.14727783, + "step": 7705, + "time_per_iteration": 2.787219524383545 + }, + { + "auxiliary_loss_clip": 0.01127905, + "auxiliary_loss_mlp": 0.01041618, + "balance_loss_clip": 1.03912807, + "balance_loss_mlp": 1.02421367, + "epoch": 0.926591715264835, + "flos": 18004964480640.0, + "grad_norm": 2.0368708918389036, + "language_loss": 0.77192283, + "learning_rate": 5.618224939177074e-08, + "loss": 0.79361808, + "num_input_tokens_seen": 166601290, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.1741333, + "step": 7706, + "time_per_iteration": 3.6657824516296387 + }, + { + "auxiliary_loss_clip": 0.01120288, + "auxiliary_loss_mlp": 0.01037813, + "balance_loss_clip": 1.03805208, + "balance_loss_mlp": 1.02311492, + "epoch": 0.926711958155474, + "flos": 36167969825280.0, + "grad_norm": 1.7634026905546216, + "language_loss": 0.70062047, + "learning_rate": 5.599906003905719e-08, + "loss": 0.72220147, + "num_input_tokens_seen": 166623835, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.14697266, + "step": 7707, + "time_per_iteration": 2.85806941986084 + }, + { + "auxiliary_loss_clip": 0.01122462, + "auxiliary_loss_mlp": 0.01038273, + "balance_loss_clip": 1.04089808, + "balance_loss_mlp": 1.02082086, + "epoch": 0.9268322010461132, + "flos": 21032736583680.0, + "grad_norm": 2.15942023684487, + "language_loss": 0.81914091, + "learning_rate": 5.581616558974023e-08, + "loss": 0.84074819, + "num_input_tokens_seen": 166642400, + "router_z_loss_clip": 0.81591797, + "router_z_loss_mlp": 0.17437744, + "step": 7708, + "time_per_iteration": 2.7876830101013184 + }, + { + "auxiliary_loss_clip": 0.01128794, + "auxiliary_loss_mlp": 0.01042328, + "balance_loss_clip": 1.04037404, + "balance_loss_mlp": 1.02649736, + "epoch": 0.9269524439367522, + "flos": 22964838174720.0, + "grad_norm": 1.7937982657547686, + "language_loss": 0.79377794, + "learning_rate": 5.5633566071565444e-08, + "loss": 0.81548905, + "num_input_tokens_seen": 166661640, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.1583252, + "step": 7709, + "time_per_iteration": 2.8584370613098145 + }, + { + "auxiliary_loss_clip": 0.01119978, + "auxiliary_loss_mlp": 0.01034261, + "balance_loss_clip": 1.0380373, + "balance_loss_mlp": 1.01976538, + "epoch": 0.9270726868273913, + "flos": 41975551468800.0, + "grad_norm": 2.2372501696415212, + "language_loss": 0.70707285, + "learning_rate": 5.5451261512232896e-08, + "loss": 0.72861528, + "num_input_tokens_seen": 166684320, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.14489746, + "step": 7710, + "time_per_iteration": 2.9290623664855957 + }, + { + "auxiliary_loss_clip": 0.01128416, + "auxiliary_loss_mlp": 0.01040045, + "balance_loss_clip": 1.04083622, + "balance_loss_mlp": 1.0243516, + "epoch": 0.9271929297180305, + "flos": 19791771557760.0, + "grad_norm": 1.7976925345301213, + "language_loss": 0.62341595, + "learning_rate": 5.5269251939397576e-08, + "loss": 0.64510053, + "num_input_tokens_seen": 166703835, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.15679932, + "step": 7711, + "time_per_iteration": 2.7831838130950928 + }, + { + "auxiliary_loss_clip": 0.01124559, + "auxiliary_loss_mlp": 0.0104103, + "balance_loss_clip": 1.0372262, + "balance_loss_mlp": 1.02410221, + "epoch": 0.9273131726086695, + "flos": 19968343839360.0, + "grad_norm": 2.07667242379194, + "language_loss": 0.76798594, + "learning_rate": 5.508753738067073e-08, + "loss": 0.78964186, + "num_input_tokens_seen": 166723375, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.16918945, + "step": 7712, + "time_per_iteration": 2.733269214630127 + }, + { + "auxiliary_loss_clip": 0.01124711, + "auxiliary_loss_mlp": 0.01032135, + "balance_loss_clip": 1.03958738, + "balance_loss_mlp": 1.01710224, + "epoch": 0.9274334154993086, + "flos": 23258587599360.0, + "grad_norm": 2.7615079166141707, + "language_loss": 0.79353952, + "learning_rate": 5.4906117863617875e-08, + "loss": 0.815108, + "num_input_tokens_seen": 166742760, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.15039062, + "step": 7713, + "time_per_iteration": 2.8113644123077393 + }, + { + "auxiliary_loss_clip": 0.01120528, + "auxiliary_loss_mlp": 0.01034825, + "balance_loss_clip": 1.03842318, + "balance_loss_mlp": 1.02031732, + "epoch": 0.9275536583899477, + "flos": 31795343585280.0, + "grad_norm": 1.7244476062150214, + "language_loss": 0.77966183, + "learning_rate": 5.4724993415760533e-08, + "loss": 0.80121529, + "num_input_tokens_seen": 166761115, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.14501953, + "step": 7714, + "time_per_iteration": 2.880180597305298 + }, + { + "auxiliary_loss_clip": 0.01130045, + "auxiliary_loss_mlp": 0.01044938, + "balance_loss_clip": 1.04089451, + "balance_loss_mlp": 1.02914286, + "epoch": 0.9276739012805868, + "flos": 18696998885760.0, + "grad_norm": 2.434981156882808, + "language_loss": 0.74957043, + "learning_rate": 5.454416406457496e-08, + "loss": 0.77132022, + "num_input_tokens_seen": 166780210, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.15783691, + "step": 7715, + "time_per_iteration": 2.8934695720672607 + }, + { + "auxiliary_loss_clip": 0.01121148, + "auxiliary_loss_mlp": 0.01034101, + "balance_loss_clip": 1.03952718, + "balance_loss_mlp": 1.02105951, + "epoch": 0.9277941441712259, + "flos": 13879079740800.0, + "grad_norm": 2.453393358215403, + "language_loss": 0.73802584, + "learning_rate": 5.436362983749299e-08, + "loss": 0.75957835, + "num_input_tokens_seen": 166795380, + "router_z_loss_clip": 0.81591797, + "router_z_loss_mlp": 0.13043213, + "step": 7716, + "time_per_iteration": 2.7504050731658936 + }, + { + "auxiliary_loss_clip": 0.01123721, + "auxiliary_loss_mlp": 0.01033285, + "balance_loss_clip": 1.0400846, + "balance_loss_mlp": 1.01894987, + "epoch": 0.927914387061865, + "flos": 23258659426560.0, + "grad_norm": 1.8574555681850915, + "language_loss": 0.64376128, + "learning_rate": 5.418339076190137e-08, + "loss": 0.66533136, + "num_input_tokens_seen": 166814890, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14312744, + "step": 7717, + "time_per_iteration": 2.80723237991333 + }, + { + "auxiliary_loss_clip": 0.0112504, + "auxiliary_loss_mlp": 0.0103732, + "balance_loss_clip": 1.04156995, + "balance_loss_mlp": 1.02092826, + "epoch": 0.9280346299525041, + "flos": 18073733068800.0, + "grad_norm": 2.331052744256837, + "language_loss": 0.88910818, + "learning_rate": 5.400344686514202e-08, + "loss": 0.91073173, + "num_input_tokens_seen": 166832475, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.16381836, + "step": 7718, + "time_per_iteration": 2.7538931369781494 + }, + { + "auxiliary_loss_clip": 0.01124791, + "auxiliary_loss_mlp": 0.01034567, + "balance_loss_clip": 1.04085875, + "balance_loss_mlp": 1.01909375, + "epoch": 0.9281548728431431, + "flos": 22342901160960.0, + "grad_norm": 2.023732468308524, + "language_loss": 0.66645908, + "learning_rate": 5.38237981745131e-08, + "loss": 0.68805265, + "num_input_tokens_seen": 166850590, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.15478516, + "step": 7719, + "time_per_iteration": 2.7740797996520996 + }, + { + "auxiliary_loss_clip": 0.01124712, + "auxiliary_loss_mlp": 0.01032294, + "balance_loss_clip": 1.04013515, + "balance_loss_mlp": 1.01735663, + "epoch": 0.9282751157337822, + "flos": 18843765857280.0, + "grad_norm": 1.5914803744559418, + "language_loss": 0.81258768, + "learning_rate": 5.364444471726592e-08, + "loss": 0.83415771, + "num_input_tokens_seen": 166869795, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.14929199, + "step": 7720, + "time_per_iteration": 2.756898880004883 + }, + { + "auxiliary_loss_clip": 0.01122779, + "auxiliary_loss_mlp": 0.01031975, + "balance_loss_clip": 1.03782439, + "balance_loss_mlp": 1.01651907, + "epoch": 0.9283953586244214, + "flos": 25556834476800.0, + "grad_norm": 2.0054616327481747, + "language_loss": 0.7999773, + "learning_rate": 5.346538652060939e-08, + "loss": 0.82152486, + "num_input_tokens_seen": 166891150, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.15454102, + "step": 7721, + "time_per_iteration": 2.747220993041992 + }, + { + "auxiliary_loss_clip": 0.01119854, + "auxiliary_loss_mlp": 0.01036697, + "balance_loss_clip": 1.03981662, + "balance_loss_mlp": 1.02258229, + "epoch": 0.9285156015150604, + "flos": 18223480869120.0, + "grad_norm": 2.0614536343560395, + "language_loss": 0.70083737, + "learning_rate": 5.3286623611705994e-08, + "loss": 0.72240287, + "num_input_tokens_seen": 166909195, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.14117432, + "step": 7722, + "time_per_iteration": 2.7977449893951416 + }, + { + "auxiliary_loss_clip": 0.01031753, + "auxiliary_loss_mlp": 0.01000545, + "balance_loss_clip": 1.00409651, + "balance_loss_mlp": 0.99931687, + "epoch": 0.9286358444056995, + "flos": 66400017690240.0, + "grad_norm": 0.8135157415684158, + "language_loss": 0.60582548, + "learning_rate": 5.3108156017673824e-08, + "loss": 0.6261484, + "num_input_tokens_seen": 166970955, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01226807, + "step": 7723, + "time_per_iteration": 3.364863872528076 + }, + { + "auxiliary_loss_clip": 0.01126764, + "auxiliary_loss_mlp": 0.01045538, + "balance_loss_clip": 1.03872085, + "balance_loss_mlp": 1.02911055, + "epoch": 0.9287560872963386, + "flos": 22345630594560.0, + "grad_norm": 1.6573931514349056, + "language_loss": 0.71748525, + "learning_rate": 5.2929983765586775e-08, + "loss": 0.73920828, + "num_input_tokens_seen": 166989735, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.16430664, + "step": 7724, + "time_per_iteration": 3.4948229789733887 + }, + { + "auxiliary_loss_clip": 0.01124227, + "auxiliary_loss_mlp": 0.01039393, + "balance_loss_clip": 1.04012835, + "balance_loss_mlp": 1.0236876, + "epoch": 0.9288763301869777, + "flos": 25700225569920.0, + "grad_norm": 1.9246201764490494, + "language_loss": 0.6248976, + "learning_rate": 5.275210688247278e-08, + "loss": 0.64653385, + "num_input_tokens_seen": 167010060, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.15704346, + "step": 7725, + "time_per_iteration": 2.877352237701416 + }, + { + "auxiliary_loss_clip": 0.0112503, + "auxiliary_loss_mlp": 0.01035227, + "balance_loss_clip": 1.04005075, + "balance_loss_mlp": 1.02129102, + "epoch": 0.9289965730776167, + "flos": 12312046028160.0, + "grad_norm": 2.043286349490006, + "language_loss": 0.84585327, + "learning_rate": 5.257452539531604e-08, + "loss": 0.86745584, + "num_input_tokens_seen": 167027130, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.13934326, + "step": 7726, + "time_per_iteration": 2.732926607131958 + }, + { + "auxiliary_loss_clip": 0.01119883, + "auxiliary_loss_mlp": 0.01037527, + "balance_loss_clip": 1.0372839, + "balance_loss_mlp": 1.02295983, + "epoch": 0.9291168159682559, + "flos": 26685973486080.0, + "grad_norm": 1.592294239395634, + "language_loss": 0.68239534, + "learning_rate": 5.2397239331055445e-08, + "loss": 0.70396942, + "num_input_tokens_seen": 167049130, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.14562988, + "step": 7727, + "time_per_iteration": 2.8412208557128906 + }, + { + "auxiliary_loss_clip": 0.01124528, + "auxiliary_loss_mlp": 0.01038433, + "balance_loss_clip": 1.0398829, + "balance_loss_mlp": 1.02323365, + "epoch": 0.929237058858895, + "flos": 14538256179840.0, + "grad_norm": 1.8522772833674517, + "language_loss": 0.8097229, + "learning_rate": 5.2220248716585036e-08, + "loss": 0.83135247, + "num_input_tokens_seen": 167066810, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.15216064, + "step": 7728, + "time_per_iteration": 3.589512825012207 + }, + { + "auxiliary_loss_clip": 0.01125395, + "auxiliary_loss_mlp": 0.01036785, + "balance_loss_clip": 1.03962481, + "balance_loss_mlp": 1.02127576, + "epoch": 0.929357301749534, + "flos": 23835456023040.0, + "grad_norm": 2.0804547602717562, + "language_loss": 0.75982249, + "learning_rate": 5.204355357875445e-08, + "loss": 0.78144419, + "num_input_tokens_seen": 167085155, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.1550293, + "step": 7729, + "time_per_iteration": 2.829439640045166 + }, + { + "auxiliary_loss_clip": 0.01124353, + "auxiliary_loss_mlp": 0.0103461, + "balance_loss_clip": 1.03941381, + "balance_loss_mlp": 1.01823032, + "epoch": 0.9294775446401732, + "flos": 12969319046400.0, + "grad_norm": 4.278555929005181, + "language_loss": 0.70413566, + "learning_rate": 5.1867153944367584e-08, + "loss": 0.72572529, + "num_input_tokens_seen": 167101545, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.16369629, + "step": 7730, + "time_per_iteration": 2.913384437561035 + }, + { + "auxiliary_loss_clip": 0.01126026, + "auxiliary_loss_mlp": 0.01039673, + "balance_loss_clip": 1.04031157, + "balance_loss_mlp": 1.02430665, + "epoch": 0.9295977875308122, + "flos": 26211809024640.0, + "grad_norm": 1.518992962439478, + "language_loss": 0.73316139, + "learning_rate": 5.16910498401848e-08, + "loss": 0.75481844, + "num_input_tokens_seen": 167120995, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15356445, + "step": 7731, + "time_per_iteration": 3.6193902492523193 + }, + { + "auxiliary_loss_clip": 0.01123008, + "auxiliary_loss_mlp": 0.01037533, + "balance_loss_clip": 1.03927946, + "balance_loss_mlp": 1.02259612, + "epoch": 0.9297180304214513, + "flos": 16472297105280.0, + "grad_norm": 1.9711839265350608, + "language_loss": 0.83524334, + "learning_rate": 5.151524129292073e-08, + "loss": 0.85684884, + "num_input_tokens_seen": 167138890, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.14941406, + "step": 7732, + "time_per_iteration": 2.8482775688171387 + }, + { + "auxiliary_loss_clip": 0.0112562, + "auxiliary_loss_mlp": 0.01036618, + "balance_loss_clip": 1.03911042, + "balance_loss_mlp": 1.02139497, + "epoch": 0.9298382733120905, + "flos": 24060436859520.0, + "grad_norm": 2.036775279215881, + "language_loss": 0.66575074, + "learning_rate": 5.1339728329245155e-08, + "loss": 0.68737316, + "num_input_tokens_seen": 167159455, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15228271, + "step": 7733, + "time_per_iteration": 2.7516138553619385 + }, + { + "auxiliary_loss_clip": 0.01132513, + "auxiliary_loss_mlp": 0.01048821, + "balance_loss_clip": 1.04218769, + "balance_loss_mlp": 1.03179765, + "epoch": 0.9299585162027295, + "flos": 22127652910080.0, + "grad_norm": 2.0093329208143342, + "language_loss": 0.79453957, + "learning_rate": 5.116451097578367e-08, + "loss": 0.81635296, + "num_input_tokens_seen": 167178495, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.17028809, + "step": 7734, + "time_per_iteration": 3.7352561950683594 + }, + { + "auxiliary_loss_clip": 0.01123213, + "auxiliary_loss_mlp": 0.01033774, + "balance_loss_clip": 1.03980911, + "balance_loss_mlp": 1.019207, + "epoch": 0.9300787590933686, + "flos": 21471780522240.0, + "grad_norm": 1.6751721924297778, + "language_loss": 0.74058795, + "learning_rate": 5.0989589259115895e-08, + "loss": 0.7621578, + "num_input_tokens_seen": 167199380, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.14575195, + "step": 7735, + "time_per_iteration": 2.828732967376709 + }, + { + "auxiliary_loss_clip": 0.01132688, + "auxiliary_loss_mlp": 0.01043251, + "balance_loss_clip": 1.04210997, + "balance_loss_mlp": 1.02569103, + "epoch": 0.9301990019840077, + "flos": 17779588594560.0, + "grad_norm": 1.866017136475606, + "language_loss": 0.7138285, + "learning_rate": 5.081496320577816e-08, + "loss": 0.73558795, + "num_input_tokens_seen": 167216500, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.17565918, + "step": 7736, + "time_per_iteration": 2.725109577178955 + }, + { + "auxiliary_loss_clip": 0.01031398, + "auxiliary_loss_mlp": 0.0100319, + "balance_loss_clip": 1.00387108, + "balance_loss_mlp": 1.00193787, + "epoch": 0.9303192448746468, + "flos": 58896122307840.0, + "grad_norm": 0.9192431147494127, + "language_loss": 0.61246514, + "learning_rate": 5.0640632842260835e-08, + "loss": 0.63281101, + "num_input_tokens_seen": 167276760, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01251221, + "step": 7737, + "time_per_iteration": 3.3388357162475586 + }, + { + "auxiliary_loss_clip": 0.0112585, + "auxiliary_loss_mlp": 0.01038229, + "balance_loss_clip": 1.03923452, + "balance_loss_mlp": 1.02254057, + "epoch": 0.9304394877652858, + "flos": 57663522172800.0, + "grad_norm": 1.4408212641067595, + "language_loss": 0.72682291, + "learning_rate": 5.0466598195009426e-08, + "loss": 0.74846369, + "num_input_tokens_seen": 167303630, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.15698242, + "step": 7738, + "time_per_iteration": 3.103492498397827 + }, + { + "auxiliary_loss_clip": 0.01121926, + "auxiliary_loss_mlp": 0.01036615, + "balance_loss_clip": 1.03878462, + "balance_loss_mlp": 1.02239311, + "epoch": 0.930559730655925, + "flos": 20996143603200.0, + "grad_norm": 2.2207270392618548, + "language_loss": 0.70062506, + "learning_rate": 5.0292859290425036e-08, + "loss": 0.72221041, + "num_input_tokens_seen": 167321500, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.14221191, + "step": 7739, + "time_per_iteration": 2.774667978286743 + }, + { + "auxiliary_loss_clip": 0.01121462, + "auxiliary_loss_mlp": 0.0103522, + "balance_loss_clip": 1.03916347, + "balance_loss_mlp": 1.02059352, + "epoch": 0.9306799735465641, + "flos": 23258264376960.0, + "grad_norm": 1.8383810925225483, + "language_loss": 0.77788699, + "learning_rate": 5.011941615486348e-08, + "loss": 0.79945385, + "num_input_tokens_seen": 167340615, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.14611816, + "step": 7740, + "time_per_iteration": 2.8103244304656982 + }, + { + "auxiliary_loss_clip": 0.01124029, + "auxiliary_loss_mlp": 0.01034908, + "balance_loss_clip": 1.03984356, + "balance_loss_mlp": 1.02042365, + "epoch": 0.9308002164372031, + "flos": 15231547560960.0, + "grad_norm": 1.9979306737026878, + "language_loss": 0.84388351, + "learning_rate": 4.994626881463659e-08, + "loss": 0.86547279, + "num_input_tokens_seen": 167356870, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.14489746, + "step": 7741, + "time_per_iteration": 2.702604293823242 + }, + { + "auxiliary_loss_clip": 0.01125125, + "auxiliary_loss_mlp": 0.01037426, + "balance_loss_clip": 1.04046702, + "balance_loss_mlp": 1.02309108, + "epoch": 0.9309204593278423, + "flos": 30847481539200.0, + "grad_norm": 1.9485487947177043, + "language_loss": 0.71187687, + "learning_rate": 4.9773417296009814e-08, + "loss": 0.73350239, + "num_input_tokens_seen": 167378390, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14337158, + "step": 7742, + "time_per_iteration": 2.834954023361206 + }, + { + "auxiliary_loss_clip": 0.01126962, + "auxiliary_loss_mlp": 0.01040969, + "balance_loss_clip": 1.04041719, + "balance_loss_mlp": 1.02381492, + "epoch": 0.9310407022184813, + "flos": 23037269950080.0, + "grad_norm": 1.828798661547723, + "language_loss": 0.65266037, + "learning_rate": 4.960086162520527e-08, + "loss": 0.67433965, + "num_input_tokens_seen": 167398480, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.17150879, + "step": 7743, + "time_per_iteration": 2.778225898742676 + }, + { + "auxiliary_loss_clip": 0.01126287, + "auxiliary_loss_mlp": 0.01036637, + "balance_loss_clip": 1.04029036, + "balance_loss_mlp": 1.0213728, + "epoch": 0.9311609451091204, + "flos": 22127976132480.0, + "grad_norm": 1.9449716290323547, + "language_loss": 0.82574201, + "learning_rate": 4.942860182839936e-08, + "loss": 0.84737128, + "num_input_tokens_seen": 167416825, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15258789, + "step": 7744, + "time_per_iteration": 2.8035213947296143 + }, + { + "auxiliary_loss_clip": 0.0112863, + "auxiliary_loss_mlp": 0.01048332, + "balance_loss_clip": 1.04188502, + "balance_loss_mlp": 1.03267956, + "epoch": 0.9312811879997596, + "flos": 21099206701440.0, + "grad_norm": 1.7403111455839648, + "language_loss": 0.79609692, + "learning_rate": 4.925663793172341e-08, + "loss": 0.8178665, + "num_input_tokens_seen": 167434785, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.15649414, + "step": 7745, + "time_per_iteration": 2.843682289123535 + }, + { + "auxiliary_loss_clip": 0.01032265, + "auxiliary_loss_mlp": 0.01001235, + "balance_loss_clip": 1.00459969, + "balance_loss_mlp": 0.99998301, + "epoch": 0.9314014308903986, + "flos": 67148179096320.0, + "grad_norm": 0.7841175560668547, + "language_loss": 0.56522292, + "learning_rate": 4.908496996126477e-08, + "loss": 0.58555788, + "num_input_tokens_seen": 167498245, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01251221, + "step": 7746, + "time_per_iteration": 3.368736982345581 + }, + { + "auxiliary_loss_clip": 0.01126522, + "auxiliary_loss_mlp": 0.01038745, + "balance_loss_clip": 1.04193163, + "balance_loss_mlp": 1.02334344, + "epoch": 0.9315216737810377, + "flos": 22565583527040.0, + "grad_norm": 1.6184216741388424, + "language_loss": 0.7652697, + "learning_rate": 4.89135979430646e-08, + "loss": 0.78692234, + "num_input_tokens_seen": 167518290, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.1541748, + "step": 7747, + "time_per_iteration": 2.8461499214172363 + }, + { + "auxiliary_loss_clip": 0.01124681, + "auxiliary_loss_mlp": 0.01040992, + "balance_loss_clip": 1.03963304, + "balance_loss_mlp": 1.02492821, + "epoch": 0.9316419166716768, + "flos": 23984054588160.0, + "grad_norm": 1.6547204825596944, + "language_loss": 0.85466796, + "learning_rate": 4.874252190312078e-08, + "loss": 0.87632465, + "num_input_tokens_seen": 167538675, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.1605835, + "step": 7748, + "time_per_iteration": 2.7971484661102295 + }, + { + "auxiliary_loss_clip": 0.01126332, + "auxiliary_loss_mlp": 0.01041451, + "balance_loss_clip": 1.03959012, + "balance_loss_mlp": 1.02560782, + "epoch": 0.9317621595623159, + "flos": 30230464688640.0, + "grad_norm": 2.604995398167616, + "language_loss": 0.65118945, + "learning_rate": 4.857174186738477e-08, + "loss": 0.6728673, + "num_input_tokens_seen": 167562025, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.15844727, + "step": 7749, + "time_per_iteration": 2.866725444793701 + }, + { + "auxiliary_loss_clip": 0.01130003, + "auxiliary_loss_mlp": 0.01043461, + "balance_loss_clip": 1.04305696, + "balance_loss_mlp": 1.02791619, + "epoch": 0.931882402452955, + "flos": 15742735966080.0, + "grad_norm": 2.205567918927828, + "language_loss": 0.73142427, + "learning_rate": 4.840125786176408e-08, + "loss": 0.75315887, + "num_input_tokens_seen": 167578230, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.15545654, + "step": 7750, + "time_per_iteration": 2.7693748474121094 + }, + { + "auxiliary_loss_clip": 0.01122247, + "auxiliary_loss_mlp": 0.01038957, + "balance_loss_clip": 1.04063761, + "balance_loss_mlp": 1.02412701, + "epoch": 0.932002645343594, + "flos": 28366521154560.0, + "grad_norm": 1.7845346105116546, + "language_loss": 0.77075279, + "learning_rate": 4.823106991212067e-08, + "loss": 0.79236478, + "num_input_tokens_seen": 167597470, + "router_z_loss_clip": 0.81689453, + "router_z_loss_mlp": 0.14819336, + "step": 7751, + "time_per_iteration": 2.90493106842041 + }, + { + "auxiliary_loss_clip": 0.01122992, + "auxiliary_loss_mlp": 0.01028892, + "balance_loss_clip": 1.0378809, + "balance_loss_mlp": 1.01446724, + "epoch": 0.9321228882342332, + "flos": 15341146934400.0, + "grad_norm": 1.8915465132706595, + "language_loss": 0.83404553, + "learning_rate": 4.806117804427212e-08, + "loss": 0.85556436, + "num_input_tokens_seen": 167615405, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.14398193, + "step": 7752, + "time_per_iteration": 2.888573408126831 + }, + { + "auxiliary_loss_clip": 0.01126246, + "auxiliary_loss_mlp": 0.01042076, + "balance_loss_clip": 1.0401541, + "balance_loss_mlp": 1.02633989, + "epoch": 0.9322431311248722, + "flos": 17895365107200.0, + "grad_norm": 1.7924156963158153, + "language_loss": 0.64264148, + "learning_rate": 4.7891582283990926e-08, + "loss": 0.66432464, + "num_input_tokens_seen": 167634130, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.15722656, + "step": 7753, + "time_per_iteration": 3.440563678741455 + }, + { + "auxiliary_loss_clip": 0.01124287, + "auxiliary_loss_mlp": 0.01037377, + "balance_loss_clip": 1.04009509, + "balance_loss_mlp": 1.02264297, + "epoch": 0.9323633740155113, + "flos": 24169713010560.0, + "grad_norm": 1.7153925291389798, + "language_loss": 0.72782755, + "learning_rate": 4.772228265700473e-08, + "loss": 0.74944425, + "num_input_tokens_seen": 167654990, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.14746094, + "step": 7754, + "time_per_iteration": 2.8765878677368164 + }, + { + "auxiliary_loss_clip": 0.01125166, + "auxiliary_loss_mlp": 0.01036854, + "balance_loss_clip": 1.0405997, + "balance_loss_mlp": 1.02136898, + "epoch": 0.9324836169061504, + "flos": 15043482927360.0, + "grad_norm": 2.0964506065862576, + "language_loss": 0.75905919, + "learning_rate": 4.75532791889961e-08, + "loss": 0.7806794, + "num_input_tokens_seen": 167671690, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.15478516, + "step": 7755, + "time_per_iteration": 2.7088522911071777 + }, + { + "auxiliary_loss_clip": 0.01125106, + "auxiliary_loss_mlp": 0.01043056, + "balance_loss_clip": 1.03958297, + "balance_loss_mlp": 1.0285362, + "epoch": 0.9326038597967895, + "flos": 18624890332800.0, + "grad_norm": 1.887720836103973, + "language_loss": 0.65528589, + "learning_rate": 4.738457190560252e-08, + "loss": 0.6769675, + "num_input_tokens_seen": 167690800, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.1451416, + "step": 7756, + "time_per_iteration": 2.8627700805664062 + }, + { + "auxiliary_loss_clip": 0.0112278, + "auxiliary_loss_mlp": 0.01044288, + "balance_loss_clip": 1.03885961, + "balance_loss_mlp": 1.0289582, + "epoch": 0.9327241026874286, + "flos": 18952646958720.0, + "grad_norm": 2.174728936037287, + "language_loss": 0.78912294, + "learning_rate": 4.721616083241664e-08, + "loss": 0.81079364, + "num_input_tokens_seen": 167709055, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.1529541, + "step": 7757, + "time_per_iteration": 3.600348472595215 + }, + { + "auxiliary_loss_clip": 0.01124977, + "auxiliary_loss_mlp": 0.01040048, + "balance_loss_clip": 1.04127586, + "balance_loss_mlp": 1.0238117, + "epoch": 0.9328443455780677, + "flos": 29570282668800.0, + "grad_norm": 1.6300633423033437, + "language_loss": 0.77816784, + "learning_rate": 4.7048045994986684e-08, + "loss": 0.7998181, + "num_input_tokens_seen": 167729915, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.16235352, + "step": 7758, + "time_per_iteration": 2.830124616622925 + }, + { + "auxiliary_loss_clip": 0.01128113, + "auxiliary_loss_mlp": 0.01041685, + "balance_loss_clip": 1.04219413, + "balance_loss_mlp": 1.02661693, + "epoch": 0.9329645884687068, + "flos": 30081722469120.0, + "grad_norm": 3.2548833788907925, + "language_loss": 0.91388255, + "learning_rate": 4.688022741881559e-08, + "loss": 0.93558049, + "num_input_tokens_seen": 167750440, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.1506958, + "step": 7759, + "time_per_iteration": 2.8330111503601074 + }, + { + "auxiliary_loss_clip": 0.01119439, + "auxiliary_loss_mlp": 0.01032018, + "balance_loss_clip": 1.03786981, + "balance_loss_mlp": 1.01724219, + "epoch": 0.9330848313593458, + "flos": 21867982513920.0, + "grad_norm": 1.5702460914216152, + "language_loss": 0.75713229, + "learning_rate": 4.671270512936076e-08, + "loss": 0.77864683, + "num_input_tokens_seen": 167769600, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.14788818, + "step": 7760, + "time_per_iteration": 3.5819590091705322 + }, + { + "auxiliary_loss_clip": 0.01123175, + "auxiliary_loss_mlp": 0.01034778, + "balance_loss_clip": 1.0385921, + "balance_loss_mlp": 1.02105165, + "epoch": 0.933205074249985, + "flos": 22127221946880.0, + "grad_norm": 1.698054174368507, + "language_loss": 0.82698929, + "learning_rate": 4.6545479152035884e-08, + "loss": 0.84856886, + "num_input_tokens_seen": 167788770, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.137146, + "step": 7761, + "time_per_iteration": 2.7228660583496094 + }, + { + "auxiliary_loss_clip": 0.01123962, + "auxiliary_loss_mlp": 0.01037411, + "balance_loss_clip": 1.03836644, + "balance_loss_mlp": 1.02301073, + "epoch": 0.9333253171406241, + "flos": 15341254675200.0, + "grad_norm": 1.9695174467363092, + "language_loss": 0.76178563, + "learning_rate": 4.637854951220821e-08, + "loss": 0.78339934, + "num_input_tokens_seen": 167805555, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.14404297, + "step": 7762, + "time_per_iteration": 2.808603048324585 + }, + { + "auxiliary_loss_clip": 0.01123196, + "auxiliary_loss_mlp": 0.01037131, + "balance_loss_clip": 1.03741741, + "balance_loss_mlp": 1.02232552, + "epoch": 0.9334455600312631, + "flos": 15706142985600.0, + "grad_norm": 1.7792317308198333, + "language_loss": 0.74700522, + "learning_rate": 4.621191623520171e-08, + "loss": 0.76860851, + "num_input_tokens_seen": 167823985, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.14794922, + "step": 7763, + "time_per_iteration": 3.637963056564331 + }, + { + "auxiliary_loss_clip": 0.01123518, + "auxiliary_loss_mlp": 0.01038438, + "balance_loss_clip": 1.03883696, + "balance_loss_mlp": 1.0237515, + "epoch": 0.9335658029219023, + "flos": 22163563532160.0, + "grad_norm": 2.1755530418994735, + "language_loss": 0.84207457, + "learning_rate": 4.604557934629372e-08, + "loss": 0.86369413, + "num_input_tokens_seen": 167843060, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.14672852, + "step": 7764, + "time_per_iteration": 2.814769983291626 + }, + { + "auxiliary_loss_clip": 0.01119632, + "auxiliary_loss_mlp": 0.01038644, + "balance_loss_clip": 1.0385015, + "balance_loss_mlp": 1.0233736, + "epoch": 0.9336860458125413, + "flos": 20266833859200.0, + "grad_norm": 1.6397517418257754, + "language_loss": 0.80426562, + "learning_rate": 4.587953887071805e-08, + "loss": 0.82584834, + "num_input_tokens_seen": 167862880, + "router_z_loss_clip": 0.81103516, + "router_z_loss_mlp": 0.15270996, + "step": 7765, + "time_per_iteration": 2.794149160385132 + }, + { + "auxiliary_loss_clip": 0.01123225, + "auxiliary_loss_mlp": 0.01035287, + "balance_loss_clip": 1.03875804, + "balance_loss_mlp": 1.02039814, + "epoch": 0.9338062887031804, + "flos": 20919689504640.0, + "grad_norm": 1.6696352386388236, + "language_loss": 0.85531294, + "learning_rate": 4.5713794833662554e-08, + "loss": 0.87689805, + "num_input_tokens_seen": 167882095, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14904785, + "step": 7766, + "time_per_iteration": 2.8113553524017334 + }, + { + "auxiliary_loss_clip": 0.01127786, + "auxiliary_loss_mlp": 0.01037807, + "balance_loss_clip": 1.04076266, + "balance_loss_mlp": 1.02170205, + "epoch": 0.9339265315938196, + "flos": 23221635482880.0, + "grad_norm": 1.7922605223880175, + "language_loss": 0.6338082, + "learning_rate": 4.5548347260270236e-08, + "loss": 0.65546417, + "num_input_tokens_seen": 167901385, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.16113281, + "step": 7767, + "time_per_iteration": 2.8165998458862305 + }, + { + "auxiliary_loss_clip": 0.0112236, + "auxiliary_loss_mlp": 0.01034201, + "balance_loss_clip": 1.03866744, + "balance_loss_mlp": 1.01953888, + "epoch": 0.9340467744844586, + "flos": 22820261932800.0, + "grad_norm": 1.7098304784580962, + "language_loss": 0.69710112, + "learning_rate": 4.538319617564012e-08, + "loss": 0.71866679, + "num_input_tokens_seen": 167920405, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.14648438, + "step": 7768, + "time_per_iteration": 2.7708535194396973 + }, + { + "auxiliary_loss_clip": 0.01126151, + "auxiliary_loss_mlp": 0.01041054, + "balance_loss_clip": 1.03920162, + "balance_loss_mlp": 1.02528834, + "epoch": 0.9341670173750977, + "flos": 23660428026240.0, + "grad_norm": 1.966414812300516, + "language_loss": 0.747419, + "learning_rate": 4.521834160482485e-08, + "loss": 0.76909101, + "num_input_tokens_seen": 167939145, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.15753174, + "step": 7769, + "time_per_iteration": 2.876560688018799 + }, + { + "auxiliary_loss_clip": 0.01123392, + "auxiliary_loss_mlp": 0.01030723, + "balance_loss_clip": 1.04047418, + "balance_loss_mlp": 1.01639962, + "epoch": 0.9342872602657368, + "flos": 24824256595200.0, + "grad_norm": 1.5395646983646254, + "language_loss": 0.82148987, + "learning_rate": 4.5053783572832846e-08, + "loss": 0.84303105, + "num_input_tokens_seen": 167959325, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.14324951, + "step": 7770, + "time_per_iteration": 2.905505418777466 + }, + { + "auxiliary_loss_clip": 0.01124958, + "auxiliary_loss_mlp": 0.01049562, + "balance_loss_clip": 1.03956246, + "balance_loss_mlp": 1.03431511, + "epoch": 0.9344075031563759, + "flos": 25771831332480.0, + "grad_norm": 3.9923043445792814, + "language_loss": 0.76266235, + "learning_rate": 4.488952210462771e-08, + "loss": 0.78440756, + "num_input_tokens_seen": 167979530, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.15258789, + "step": 7771, + "time_per_iteration": 2.851569414138794 + }, + { + "auxiliary_loss_clip": 0.01124434, + "auxiliary_loss_mlp": 0.01037828, + "balance_loss_clip": 1.04006231, + "balance_loss_mlp": 1.02269459, + "epoch": 0.9345277460470149, + "flos": 25551303782400.0, + "grad_norm": 2.065659998705575, + "language_loss": 0.85955203, + "learning_rate": 4.4725557225127495e-08, + "loss": 0.88117468, + "num_input_tokens_seen": 167997870, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.15148926, + "step": 7772, + "time_per_iteration": 2.765866756439209 + }, + { + "auxiliary_loss_clip": 0.01124937, + "auxiliary_loss_mlp": 0.01032689, + "balance_loss_clip": 1.04333436, + "balance_loss_mlp": 1.0188787, + "epoch": 0.9346479889376541, + "flos": 34313112432000.0, + "grad_norm": 1.5512971958891253, + "language_loss": 0.79370379, + "learning_rate": 4.456188895920565e-08, + "loss": 0.81528008, + "num_input_tokens_seen": 168019625, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.13812256, + "step": 7773, + "time_per_iteration": 2.8661928176879883 + }, + { + "auxiliary_loss_clip": 0.01125023, + "auxiliary_loss_mlp": 0.01038807, + "balance_loss_clip": 1.03959405, + "balance_loss_mlp": 1.02155733, + "epoch": 0.9347682318282932, + "flos": 19093739581440.0, + "grad_norm": 2.1155574673350235, + "language_loss": 0.85461688, + "learning_rate": 4.439851733169031e-08, + "loss": 0.87625515, + "num_input_tokens_seen": 168037415, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.17236328, + "step": 7774, + "time_per_iteration": 2.820652961730957 + }, + { + "auxiliary_loss_clip": 0.01122406, + "auxiliary_loss_mlp": 0.01036881, + "balance_loss_clip": 1.03844523, + "balance_loss_mlp": 1.02172971, + "epoch": 0.9348884747189322, + "flos": 26249587153920.0, + "grad_norm": 2.215179318208932, + "language_loss": 0.69231367, + "learning_rate": 4.4235442367365204e-08, + "loss": 0.71390653, + "num_input_tokens_seen": 168057725, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.15161133, + "step": 7775, + "time_per_iteration": 2.79470157623291 + }, + { + "auxiliary_loss_clip": 0.01124834, + "auxiliary_loss_mlp": 0.01043196, + "balance_loss_clip": 1.03838086, + "balance_loss_mlp": 1.02833033, + "epoch": 0.9350087176095714, + "flos": 18333080242560.0, + "grad_norm": 1.9246248520082183, + "language_loss": 0.79688239, + "learning_rate": 4.4072664090968545e-08, + "loss": 0.81856263, + "num_input_tokens_seen": 168076110, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.14880371, + "step": 7776, + "time_per_iteration": 2.8339405059814453 + }, + { + "auxiliary_loss_clip": 0.01126002, + "auxiliary_loss_mlp": 0.01036864, + "balance_loss_clip": 1.04134893, + "balance_loss_mlp": 1.02224326, + "epoch": 0.9351289605002104, + "flos": 19318253541120.0, + "grad_norm": 1.9438871452441249, + "language_loss": 0.84765863, + "learning_rate": 4.391018252719347e-08, + "loss": 0.86928737, + "num_input_tokens_seen": 168095905, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.14605713, + "step": 7777, + "time_per_iteration": 2.8109045028686523 + }, + { + "auxiliary_loss_clip": 0.01125204, + "auxiliary_loss_mlp": 0.010416, + "balance_loss_clip": 1.03931975, + "balance_loss_mlp": 1.02561402, + "epoch": 0.9352492033908495, + "flos": 18799990156800.0, + "grad_norm": 1.7351226624132299, + "language_loss": 0.69132781, + "learning_rate": 4.374799770068849e-08, + "loss": 0.71299589, + "num_input_tokens_seen": 168112580, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15979004, + "step": 7778, + "time_per_iteration": 2.8704142570495605 + }, + { + "auxiliary_loss_clip": 0.01123454, + "auxiliary_loss_mlp": 0.01040204, + "balance_loss_clip": 1.03932738, + "balance_loss_mlp": 1.02553558, + "epoch": 0.9353694462814887, + "flos": 29530134241920.0, + "grad_norm": 2.6142601049304055, + "language_loss": 0.74450272, + "learning_rate": 4.358610963605658e-08, + "loss": 0.76613927, + "num_input_tokens_seen": 168133030, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.14678955, + "step": 7779, + "time_per_iteration": 2.8980281352996826 + }, + { + "auxiliary_loss_clip": 0.01129192, + "auxiliary_loss_mlp": 0.01048421, + "balance_loss_clip": 1.04079902, + "balance_loss_mlp": 1.03258443, + "epoch": 0.9354896891721277, + "flos": 30665450390400.0, + "grad_norm": 2.7746948057449865, + "language_loss": 0.68674517, + "learning_rate": 4.342451835785677e-08, + "loss": 0.70852137, + "num_input_tokens_seen": 168153940, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.1583252, + "step": 7780, + "time_per_iteration": 2.880070447921753 + }, + { + "auxiliary_loss_clip": 0.01120715, + "auxiliary_loss_mlp": 0.01034478, + "balance_loss_clip": 1.03755569, + "balance_loss_mlp": 1.01954663, + "epoch": 0.9356099320627668, + "flos": 19463907191040.0, + "grad_norm": 1.6664307229702153, + "language_loss": 0.75211608, + "learning_rate": 4.3263223890601665e-08, + "loss": 0.77366793, + "num_input_tokens_seen": 168172650, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.14935303, + "step": 7781, + "time_per_iteration": 2.7730329036712646 + }, + { + "auxiliary_loss_clip": 0.01124507, + "auxiliary_loss_mlp": 0.01039474, + "balance_loss_clip": 1.04120171, + "balance_loss_mlp": 1.02507317, + "epoch": 0.9357301749534058, + "flos": 19098156954240.0, + "grad_norm": 1.5867750054286442, + "language_loss": 0.79124022, + "learning_rate": 4.31022262587597e-08, + "loss": 0.81288004, + "num_input_tokens_seen": 168191325, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.144104, + "step": 7782, + "time_per_iteration": 3.5572474002838135 + }, + { + "auxiliary_loss_clip": 0.01131023, + "auxiliary_loss_mlp": 0.01037864, + "balance_loss_clip": 1.0428369, + "balance_loss_mlp": 1.02172875, + "epoch": 0.935850417844045, + "flos": 23550361776000.0, + "grad_norm": 1.5686185983943475, + "language_loss": 0.65792477, + "learning_rate": 4.2941525486754225e-08, + "loss": 0.67961371, + "num_input_tokens_seen": 168211645, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.16131592, + "step": 7783, + "time_per_iteration": 2.8361058235168457 + }, + { + "auxiliary_loss_clip": 0.01117411, + "auxiliary_loss_mlp": 0.01037036, + "balance_loss_clip": 1.0388931, + "balance_loss_mlp": 1.02348197, + "epoch": 0.935970660734684, + "flos": 18588333265920.0, + "grad_norm": 1.8789213805459946, + "language_loss": 0.79594415, + "learning_rate": 4.278112159896286e-08, + "loss": 0.81748861, + "num_input_tokens_seen": 168229485, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.13568115, + "step": 7784, + "time_per_iteration": 2.7971630096435547 + }, + { + "auxiliary_loss_clip": 0.01116836, + "auxiliary_loss_mlp": 0.01035813, + "balance_loss_clip": 1.03697336, + "balance_loss_mlp": 1.02147794, + "epoch": 0.9360909036253231, + "flos": 20631255292800.0, + "grad_norm": 4.492045748971127, + "language_loss": 0.6760717, + "learning_rate": 4.2621014619719896e-08, + "loss": 0.69759816, + "num_input_tokens_seen": 168247250, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.14331055, + "step": 7785, + "time_per_iteration": 2.811060667037964 + }, + { + "auxiliary_loss_clip": 0.01031651, + "auxiliary_loss_mlp": 0.01003718, + "balance_loss_clip": 1.00396001, + "balance_loss_mlp": 1.00246584, + "epoch": 0.9362111465159623, + "flos": 61791421052160.0, + "grad_norm": 0.7174058178457625, + "language_loss": 0.58570802, + "learning_rate": 4.246120457331215e-08, + "loss": 0.6060617, + "num_input_tokens_seen": 168309425, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01251221, + "step": 7786, + "time_per_iteration": 4.0547661781311035 + }, + { + "auxiliary_loss_clip": 0.0112606, + "auxiliary_loss_mlp": 0.01041184, + "balance_loss_clip": 1.04171181, + "balance_loss_mlp": 1.0261519, + "epoch": 0.9363313894066013, + "flos": 24170395368960.0, + "grad_norm": 2.4739787378335723, + "language_loss": 0.71938038, + "learning_rate": 4.2301691483983325e-08, + "loss": 0.74105287, + "num_input_tokens_seen": 168329545, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.15045166, + "step": 7787, + "time_per_iteration": 2.899456024169922 + }, + { + "auxiliary_loss_clip": 0.01126473, + "auxiliary_loss_mlp": 0.0103629, + "balance_loss_clip": 1.03904212, + "balance_loss_mlp": 1.02135348, + "epoch": 0.9364516322972404, + "flos": 20120354196480.0, + "grad_norm": 1.828661161547252, + "language_loss": 0.75722307, + "learning_rate": 4.214247537593163e-08, + "loss": 0.77885067, + "num_input_tokens_seen": 168348795, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.14929199, + "step": 7788, + "time_per_iteration": 2.861377477645874 + }, + { + "auxiliary_loss_clip": 0.01124639, + "auxiliary_loss_mlp": 0.01041927, + "balance_loss_clip": 1.04141307, + "balance_loss_mlp": 1.02724648, + "epoch": 0.9365718751878795, + "flos": 20703758895360.0, + "grad_norm": 1.8802878830744274, + "language_loss": 0.80432922, + "learning_rate": 4.1983556273309293e-08, + "loss": 0.82599485, + "num_input_tokens_seen": 168367545, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.14691162, + "step": 7789, + "time_per_iteration": 3.633624315261841 + }, + { + "auxiliary_loss_clip": 0.01131469, + "auxiliary_loss_mlp": 0.01043291, + "balance_loss_clip": 1.04236627, + "balance_loss_mlp": 1.02737665, + "epoch": 0.9366921180785186, + "flos": 18655270260480.0, + "grad_norm": 2.1581259566599247, + "language_loss": 0.68942124, + "learning_rate": 4.182493420022526e-08, + "loss": 0.71116877, + "num_input_tokens_seen": 168383215, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.15917969, + "step": 7790, + "time_per_iteration": 2.7410683631896973 + }, + { + "auxiliary_loss_clip": 0.01118741, + "auxiliary_loss_mlp": 0.01031201, + "balance_loss_clip": 1.03610516, + "balance_loss_mlp": 1.0172056, + "epoch": 0.9368123609691577, + "flos": 25774955815680.0, + "grad_norm": 1.8712773735947204, + "language_loss": 0.7851817, + "learning_rate": 4.166660918074139e-08, + "loss": 0.80668116, + "num_input_tokens_seen": 168403120, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.13983154, + "step": 7791, + "time_per_iteration": 2.8713936805725098 + }, + { + "auxiliary_loss_clip": 0.01123668, + "auxiliary_loss_mlp": 0.01034496, + "balance_loss_clip": 1.03933942, + "balance_loss_mlp": 1.01966655, + "epoch": 0.9369326038597968, + "flos": 25553386771200.0, + "grad_norm": 1.4543173041100697, + "language_loss": 0.73401809, + "learning_rate": 4.15085812388758e-08, + "loss": 0.75559974, + "num_input_tokens_seen": 168425340, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.14831543, + "step": 7792, + "time_per_iteration": 3.980090856552124 + }, + { + "auxiliary_loss_clip": 0.0112654, + "auxiliary_loss_mlp": 0.01037352, + "balance_loss_clip": 1.04183948, + "balance_loss_mlp": 1.02149701, + "epoch": 0.9370528467504359, + "flos": 23220019370880.0, + "grad_norm": 1.7304661405905926, + "language_loss": 0.78152585, + "learning_rate": 4.135085039860153e-08, + "loss": 0.80316472, + "num_input_tokens_seen": 168444740, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.15856934, + "step": 7793, + "time_per_iteration": 2.9319069385528564 + }, + { + "auxiliary_loss_clip": 0.01125479, + "auxiliary_loss_mlp": 0.01041363, + "balance_loss_clip": 1.04007161, + "balance_loss_mlp": 1.02535343, + "epoch": 0.9371730896410749, + "flos": 24967468120320.0, + "grad_norm": 2.1195591351068295, + "language_loss": 0.79018962, + "learning_rate": 4.1193416683845906e-08, + "loss": 0.81185806, + "num_input_tokens_seen": 168463670, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.16003418, + "step": 7794, + "time_per_iteration": 2.7830214500427246 + }, + { + "auxiliary_loss_clip": 0.01118526, + "auxiliary_loss_mlp": 0.01032908, + "balance_loss_clip": 1.03838265, + "balance_loss_mlp": 1.01915121, + "epoch": 0.9372933325317141, + "flos": 15553091134080.0, + "grad_norm": 2.1344635349868586, + "language_loss": 0.8352952, + "learning_rate": 4.103628011849136e-08, + "loss": 0.85680956, + "num_input_tokens_seen": 168479030, + "router_z_loss_clip": 0.80175781, + "router_z_loss_mlp": 0.13769531, + "step": 7795, + "time_per_iteration": 2.6925570964813232 + }, + { + "auxiliary_loss_clip": 0.01127374, + "auxiliary_loss_mlp": 0.01040259, + "balance_loss_clip": 1.04045856, + "balance_loss_mlp": 1.024261, + "epoch": 0.9374135754223532, + "flos": 21871861182720.0, + "grad_norm": 1.7843847471889245, + "language_loss": 0.75656152, + "learning_rate": 4.0879440726375506e-08, + "loss": 0.77823782, + "num_input_tokens_seen": 168496815, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15991211, + "step": 7796, + "time_per_iteration": 2.790374279022217 + }, + { + "auxiliary_loss_clip": 0.01125674, + "auxiliary_loss_mlp": 0.01034525, + "balance_loss_clip": 1.03950632, + "balance_loss_mlp": 1.01769257, + "epoch": 0.9375338183129922, + "flos": 22631048064000.0, + "grad_norm": 2.391640941713415, + "language_loss": 0.56209159, + "learning_rate": 4.0722898531291074e-08, + "loss": 0.58369356, + "num_input_tokens_seen": 168514055, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.16833496, + "step": 7797, + "time_per_iteration": 2.822662353515625 + }, + { + "auxiliary_loss_clip": 0.0112752, + "auxiliary_loss_mlp": 0.01040829, + "balance_loss_clip": 1.04028332, + "balance_loss_mlp": 1.0258441, + "epoch": 0.9376540612036314, + "flos": 26104292640000.0, + "grad_norm": 1.7337202010809485, + "language_loss": 0.7678054, + "learning_rate": 4.0566653556985295e-08, + "loss": 0.78948891, + "num_input_tokens_seen": 168534600, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.14978027, + "step": 7798, + "time_per_iteration": 2.882326126098633 + }, + { + "auxiliary_loss_clip": 0.0113133, + "auxiliary_loss_mlp": 0.01035366, + "balance_loss_clip": 1.04187822, + "balance_loss_mlp": 1.01835454, + "epoch": 0.9377743040942704, + "flos": 19717580016000.0, + "grad_norm": 76.79136007783985, + "language_loss": 0.81443048, + "learning_rate": 4.0410705827159886e-08, + "loss": 0.83609742, + "num_input_tokens_seen": 168551895, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.16992188, + "step": 7799, + "time_per_iteration": 2.8190677165985107 + }, + { + "auxiliary_loss_clip": 0.01124463, + "auxiliary_loss_mlp": 0.01040204, + "balance_loss_clip": 1.03994608, + "balance_loss_mlp": 1.02524376, + "epoch": 0.9378945469849095, + "flos": 15267530010240.0, + "grad_norm": 2.1043028961262014, + "language_loss": 0.71345496, + "learning_rate": 4.0255055365472356e-08, + "loss": 0.7351017, + "num_input_tokens_seen": 168569990, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14941406, + "step": 7800, + "time_per_iteration": 2.8449018001556396 + }, + { + "auxiliary_loss_clip": 0.01125644, + "auxiliary_loss_mlp": 0.01037549, + "balance_loss_clip": 1.03905284, + "balance_loss_mlp": 1.02307665, + "epoch": 0.9380147898755486, + "flos": 20591394174720.0, + "grad_norm": 2.2955283442581877, + "language_loss": 0.74984545, + "learning_rate": 4.009970219553471e-08, + "loss": 0.77147734, + "num_input_tokens_seen": 168586940, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.14501953, + "step": 7801, + "time_per_iteration": 2.8976097106933594 + }, + { + "auxiliary_loss_clip": 0.01129885, + "auxiliary_loss_mlp": 0.01041019, + "balance_loss_clip": 1.03947806, + "balance_loss_mlp": 1.02512825, + "epoch": 0.9381350327661877, + "flos": 26281116316800.0, + "grad_norm": 2.363027002575479, + "language_loss": 0.77371716, + "learning_rate": 3.99446463409141e-08, + "loss": 0.79542619, + "num_input_tokens_seen": 168604795, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.15893555, + "step": 7802, + "time_per_iteration": 2.8789398670196533 + }, + { + "auxiliary_loss_clip": 0.01127661, + "auxiliary_loss_mlp": 0.01038789, + "balance_loss_clip": 1.03932655, + "balance_loss_mlp": 1.02205205, + "epoch": 0.9382552756568268, + "flos": 23586344225280.0, + "grad_norm": 2.175034063119574, + "language_loss": 0.69078565, + "learning_rate": 3.978988782513215e-08, + "loss": 0.71245015, + "num_input_tokens_seen": 168622290, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.16723633, + "step": 7803, + "time_per_iteration": 2.8804571628570557 + }, + { + "auxiliary_loss_clip": 0.01124626, + "auxiliary_loss_mlp": 0.01037768, + "balance_loss_clip": 1.03828812, + "balance_loss_mlp": 1.02186561, + "epoch": 0.9383755185474659, + "flos": 28438809275520.0, + "grad_norm": 1.6782687268000913, + "language_loss": 0.76314199, + "learning_rate": 3.963542667166586e-08, + "loss": 0.78476596, + "num_input_tokens_seen": 168642395, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.15905762, + "step": 7804, + "time_per_iteration": 2.863882303237915 + }, + { + "auxiliary_loss_clip": 0.01124043, + "auxiliary_loss_mlp": 0.01032168, + "balance_loss_clip": 1.04045868, + "balance_loss_mlp": 1.0175705, + "epoch": 0.938495761438105, + "flos": 20449583280000.0, + "grad_norm": 2.118812833326313, + "language_loss": 0.68404192, + "learning_rate": 3.9481262903946486e-08, + "loss": 0.70560408, + "num_input_tokens_seen": 168661840, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14593506, + "step": 7805, + "time_per_iteration": 2.8625760078430176 + }, + { + "auxiliary_loss_clip": 0.01031461, + "auxiliary_loss_mlp": 0.0100242, + "balance_loss_clip": 1.00387144, + "balance_loss_mlp": 1.00109661, + "epoch": 0.938616004328744, + "flos": 69302711658240.0, + "grad_norm": 0.7769453702820341, + "language_loss": 0.54488873, + "learning_rate": 3.932739654536066e-08, + "loss": 0.56522757, + "num_input_tokens_seen": 168724540, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01324463, + "step": 7806, + "time_per_iteration": 3.331732749938965 + }, + { + "auxiliary_loss_clip": 0.0112137, + "auxiliary_loss_mlp": 0.0103622, + "balance_loss_clip": 1.03918958, + "balance_loss_mlp": 1.022457, + "epoch": 0.9387362472193832, + "flos": 18911636605440.0, + "grad_norm": 2.050204229965863, + "language_loss": 0.73836708, + "learning_rate": 3.917382761925014e-08, + "loss": 0.75994295, + "num_input_tokens_seen": 168740375, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.13751221, + "step": 7807, + "time_per_iteration": 2.8213281631469727 + }, + { + "auxiliary_loss_clip": 0.01121859, + "auxiliary_loss_mlp": 0.0103281, + "balance_loss_clip": 1.03894234, + "balance_loss_mlp": 1.0186832, + "epoch": 0.9388564901100223, + "flos": 26501967089280.0, + "grad_norm": 2.2721644550622138, + "language_loss": 0.79208612, + "learning_rate": 3.9020556148910754e-08, + "loss": 0.81363285, + "num_input_tokens_seen": 168759730, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.14129639, + "step": 7808, + "time_per_iteration": 2.955214500427246 + }, + { + "auxiliary_loss_clip": 0.01031736, + "auxiliary_loss_mlp": 0.01001984, + "balance_loss_clip": 1.00394392, + "balance_loss_mlp": 1.00066042, + "epoch": 0.9389767330006613, + "flos": 58941083157120.0, + "grad_norm": 0.7046514529635207, + "language_loss": 0.56663233, + "learning_rate": 3.8867582157593895e-08, + "loss": 0.58696955, + "num_input_tokens_seen": 168813935, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01324463, + "step": 7809, + "time_per_iteration": 3.196207046508789 + }, + { + "auxiliary_loss_clip": 0.01125521, + "auxiliary_loss_mlp": 0.01040659, + "balance_loss_clip": 1.04151177, + "balance_loss_mlp": 1.02542436, + "epoch": 0.9390969758913005, + "flos": 31102554994560.0, + "grad_norm": 1.8336315994418455, + "language_loss": 0.7648654, + "learning_rate": 3.871490566850544e-08, + "loss": 0.78652722, + "num_input_tokens_seen": 168838145, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15234375, + "step": 7810, + "time_per_iteration": 2.97493314743042 + }, + { + "auxiliary_loss_clip": 0.01125477, + "auxiliary_loss_mlp": 0.01031026, + "balance_loss_clip": 1.0391438, + "balance_loss_mlp": 1.01634526, + "epoch": 0.9392172187819395, + "flos": 22419391173120.0, + "grad_norm": 4.285838622015322, + "language_loss": 0.70609057, + "learning_rate": 3.856252670480642e-08, + "loss": 0.72765553, + "num_input_tokens_seen": 168856805, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.14685059, + "step": 7811, + "time_per_iteration": 3.6355907917022705 + }, + { + "auxiliary_loss_clip": 0.01128023, + "auxiliary_loss_mlp": 0.0103724, + "balance_loss_clip": 1.04213548, + "balance_loss_mlp": 1.02237475, + "epoch": 0.9393374616725786, + "flos": 19719483436800.0, + "grad_norm": 1.7176550211051442, + "language_loss": 0.80958229, + "learning_rate": 3.841044528961279e-08, + "loss": 0.83123493, + "num_input_tokens_seen": 168874600, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.14868164, + "step": 7812, + "time_per_iteration": 2.9108147621154785 + }, + { + "auxiliary_loss_clip": 0.01125434, + "auxiliary_loss_mlp": 0.01036309, + "balance_loss_clip": 1.0394516, + "balance_loss_mlp": 1.01958394, + "epoch": 0.9394577045632178, + "flos": 24170215800960.0, + "grad_norm": 2.094786113709635, + "language_loss": 0.78815609, + "learning_rate": 3.825866144599477e-08, + "loss": 0.8097735, + "num_input_tokens_seen": 168893655, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.16711426, + "step": 7813, + "time_per_iteration": 2.8570356369018555 + }, + { + "auxiliary_loss_clip": 0.01124653, + "auxiliary_loss_mlp": 0.01043501, + "balance_loss_clip": 1.03927028, + "balance_loss_mlp": 1.02796781, + "epoch": 0.9395779474538568, + "flos": 19023929498880.0, + "grad_norm": 1.9057803635029889, + "language_loss": 0.75570542, + "learning_rate": 3.8107175196978145e-08, + "loss": 0.77738702, + "num_input_tokens_seen": 168909960, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.15533447, + "step": 7814, + "time_per_iteration": 2.8276078701019287 + }, + { + "auxiliary_loss_clip": 0.01125099, + "auxiliary_loss_mlp": 0.01038438, + "balance_loss_clip": 1.041816, + "balance_loss_mlp": 1.02347767, + "epoch": 0.9396981903444959, + "flos": 14319129260160.0, + "grad_norm": 2.3382903999365277, + "language_loss": 0.76893687, + "learning_rate": 3.7955986565542996e-08, + "loss": 0.79057223, + "num_input_tokens_seen": 168928040, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.14978027, + "step": 7815, + "time_per_iteration": 3.5353307723999023 + }, + { + "auxiliary_loss_clip": 0.01125671, + "auxiliary_loss_mlp": 0.01040516, + "balance_loss_clip": 1.03897393, + "balance_loss_mlp": 1.02494752, + "epoch": 0.9398184332351349, + "flos": 34787564202240.0, + "grad_norm": 1.8026742981496036, + "language_loss": 0.68498135, + "learning_rate": 3.780509557462497e-08, + "loss": 0.70664328, + "num_input_tokens_seen": 168948240, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.15563965, + "step": 7816, + "time_per_iteration": 3.0441982746124268 + }, + { + "auxiliary_loss_clip": 0.0112715, + "auxiliary_loss_mlp": 0.01035118, + "balance_loss_clip": 1.04024112, + "balance_loss_mlp": 1.01931059, + "epoch": 0.9399386761257741, + "flos": 25372253462400.0, + "grad_norm": 1.4997168221536343, + "language_loss": 0.75451297, + "learning_rate": 3.765450224711375e-08, + "loss": 0.77613568, + "num_input_tokens_seen": 168968745, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15808105, + "step": 7817, + "time_per_iteration": 2.8508214950561523 + }, + { + "auxiliary_loss_clip": 0.01121605, + "auxiliary_loss_mlp": 0.01032197, + "balance_loss_clip": 1.03755581, + "balance_loss_mlp": 1.01709354, + "epoch": 0.9400589190164131, + "flos": 27304965584640.0, + "grad_norm": 1.642303467745893, + "language_loss": 0.7988385, + "learning_rate": 3.750420660585396e-08, + "loss": 0.82037657, + "num_input_tokens_seen": 168990685, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.15112305, + "step": 7818, + "time_per_iteration": 3.6813061237335205 + }, + { + "auxiliary_loss_clip": 0.0112323, + "auxiliary_loss_mlp": 0.01043027, + "balance_loss_clip": 1.03871369, + "balance_loss_mlp": 1.02747011, + "epoch": 0.9401791619070522, + "flos": 23399859790080.0, + "grad_norm": 1.613705823889983, + "language_loss": 0.79444611, + "learning_rate": 3.735420867364603e-08, + "loss": 0.81610864, + "num_input_tokens_seen": 169011665, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15576172, + "step": 7819, + "time_per_iteration": 2.916001319885254 + }, + { + "auxiliary_loss_clip": 0.0111563, + "auxiliary_loss_mlp": 0.01033569, + "balance_loss_clip": 1.03589201, + "balance_loss_mlp": 1.01911533, + "epoch": 0.9402994047976914, + "flos": 35881403120640.0, + "grad_norm": 1.5459379745100799, + "language_loss": 0.6183905, + "learning_rate": 3.7204508473244186e-08, + "loss": 0.63988256, + "num_input_tokens_seen": 169035290, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.14459229, + "step": 7820, + "time_per_iteration": 2.914292573928833 + }, + { + "auxiliary_loss_clip": 0.01121595, + "auxiliary_loss_mlp": 0.01037269, + "balance_loss_clip": 1.0384866, + "balance_loss_mlp": 1.02230859, + "epoch": 0.9404196476883304, + "flos": 22236821320320.0, + "grad_norm": 1.6462106592380814, + "language_loss": 0.69233519, + "learning_rate": 3.7055106027357395e-08, + "loss": 0.71392381, + "num_input_tokens_seen": 169055155, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.14953613, + "step": 7821, + "time_per_iteration": 3.8599865436553955 + }, + { + "auxiliary_loss_clip": 0.01124368, + "auxiliary_loss_mlp": 0.01038298, + "balance_loss_clip": 1.03936887, + "balance_loss_mlp": 1.02261055, + "epoch": 0.9405398905789695, + "flos": 18915802583040.0, + "grad_norm": 2.190774775789854, + "language_loss": 0.71826017, + "learning_rate": 3.690600135865063e-08, + "loss": 0.73988688, + "num_input_tokens_seen": 169072080, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.15698242, + "step": 7822, + "time_per_iteration": 3.022052526473999 + }, + { + "auxiliary_loss_clip": 0.01031919, + "auxiliary_loss_mlp": 0.01001916, + "balance_loss_clip": 1.00427675, + "balance_loss_mlp": 1.00065279, + "epoch": 0.9406601334696086, + "flos": 70274130048000.0, + "grad_norm": 0.789066248416685, + "language_loss": 0.5808655, + "learning_rate": 3.675719448974246e-08, + "loss": 0.60120386, + "num_input_tokens_seen": 169137170, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01263428, + "step": 7823, + "time_per_iteration": 3.4159727096557617 + }, + { + "auxiliary_loss_clip": 0.01127573, + "auxiliary_loss_mlp": 0.01040617, + "balance_loss_clip": 1.04107237, + "balance_loss_mlp": 1.02504873, + "epoch": 0.9407803763602477, + "flos": 22165071903360.0, + "grad_norm": 2.065063627269004, + "language_loss": 0.59930551, + "learning_rate": 3.6608685443207054e-08, + "loss": 0.62098742, + "num_input_tokens_seen": 169156320, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15576172, + "step": 7824, + "time_per_iteration": 2.8523099422454834 + }, + { + "auxiliary_loss_clip": 0.01123199, + "auxiliary_loss_mlp": 0.010376, + "balance_loss_clip": 1.0403266, + "balance_loss_mlp": 1.02213883, + "epoch": 0.9409006192508867, + "flos": 18879496911360.0, + "grad_norm": 2.1172086576819353, + "language_loss": 0.66962606, + "learning_rate": 3.646047424157306e-08, + "loss": 0.69123411, + "num_input_tokens_seen": 169173295, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.15466309, + "step": 7825, + "time_per_iteration": 2.859135627746582 + }, + { + "auxiliary_loss_clip": 0.01128772, + "auxiliary_loss_mlp": 0.01043405, + "balance_loss_clip": 1.04005659, + "balance_loss_mlp": 1.02564287, + "epoch": 0.9410208621415259, + "flos": 23368258800000.0, + "grad_norm": 2.3973421107937023, + "language_loss": 0.68755549, + "learning_rate": 3.631256090732382e-08, + "loss": 0.70927727, + "num_input_tokens_seen": 169193755, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.17773438, + "step": 7826, + "time_per_iteration": 2.825395107269287 + }, + { + "auxiliary_loss_clip": 0.01123199, + "auxiliary_loss_mlp": 0.01027557, + "balance_loss_clip": 1.04090834, + "balance_loss_mlp": 1.01389587, + "epoch": 0.941141105032165, + "flos": 22742227635840.0, + "grad_norm": 1.560419609814472, + "language_loss": 0.8308779, + "learning_rate": 3.6164945462897833e-08, + "loss": 0.8523854, + "num_input_tokens_seen": 169213045, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.13671875, + "step": 7827, + "time_per_iteration": 2.939894437789917 + }, + { + "auxiliary_loss_clip": 0.01122942, + "auxiliary_loss_mlp": 0.01035855, + "balance_loss_clip": 1.04029965, + "balance_loss_mlp": 1.02124023, + "epoch": 0.941261347922804, + "flos": 20704908130560.0, + "grad_norm": 1.7378234048177363, + "language_loss": 0.7543838, + "learning_rate": 3.6017627930687856e-08, + "loss": 0.77597177, + "num_input_tokens_seen": 169232870, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.14624023, + "step": 7828, + "time_per_iteration": 2.8788533210754395 + }, + { + "auxiliary_loss_clip": 0.01122939, + "auxiliary_loss_mlp": 0.0103875, + "balance_loss_clip": 1.03753638, + "balance_loss_mlp": 1.02357507, + "epoch": 0.9413815908134432, + "flos": 19421998997760.0, + "grad_norm": 2.34970654877419, + "language_loss": 0.77326572, + "learning_rate": 3.587060833304267e-08, + "loss": 0.79488254, + "num_input_tokens_seen": 169251060, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.1517334, + "step": 7829, + "time_per_iteration": 2.900874376296997 + }, + { + "auxiliary_loss_clip": 0.01126348, + "auxiliary_loss_mlp": 0.01039002, + "balance_loss_clip": 1.03993201, + "balance_loss_mlp": 1.02321887, + "epoch": 0.9415018337040822, + "flos": 17493452853120.0, + "grad_norm": 2.1070768496821684, + "language_loss": 0.64769047, + "learning_rate": 3.5723886692264225e-08, + "loss": 0.66934395, + "num_input_tokens_seen": 169268600, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15795898, + "step": 7830, + "time_per_iteration": 2.902355194091797 + }, + { + "auxiliary_loss_clip": 0.0111964, + "auxiliary_loss_mlp": 0.01035867, + "balance_loss_clip": 1.03623998, + "balance_loss_mlp": 1.02218151, + "epoch": 0.9416220765947213, + "flos": 31831613343360.0, + "grad_norm": 2.2229697399554307, + "language_loss": 0.61999005, + "learning_rate": 3.557746303061071e-08, + "loss": 0.64154512, + "num_input_tokens_seen": 169290355, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.13696289, + "step": 7831, + "time_per_iteration": 3.0551137924194336 + }, + { + "auxiliary_loss_clip": 0.01124873, + "auxiliary_loss_mlp": 0.01033702, + "balance_loss_clip": 1.04045045, + "balance_loss_mlp": 1.01888442, + "epoch": 0.9417423194853605, + "flos": 23511973115520.0, + "grad_norm": 2.0353298019949038, + "language_loss": 0.72108531, + "learning_rate": 3.543133737029391e-08, + "loss": 0.74267101, + "num_input_tokens_seen": 169310865, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.14819336, + "step": 7832, + "time_per_iteration": 2.899277687072754 + }, + { + "auxiliary_loss_clip": 0.0112608, + "auxiliary_loss_mlp": 0.01043569, + "balance_loss_clip": 1.03931463, + "balance_loss_mlp": 1.02783334, + "epoch": 0.9418625623759995, + "flos": 23915106432000.0, + "grad_norm": 1.784340353852663, + "language_loss": 0.69208586, + "learning_rate": 3.5285509733481214e-08, + "loss": 0.71378231, + "num_input_tokens_seen": 169330590, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.15734863, + "step": 7833, + "time_per_iteration": 2.8171634674072266 + }, + { + "auxiliary_loss_clip": 0.01126038, + "auxiliary_loss_mlp": 0.01046517, + "balance_loss_clip": 1.0395155, + "balance_loss_mlp": 1.03048396, + "epoch": 0.9419828052666386, + "flos": 18076965292800.0, + "grad_norm": 1.7264047512307712, + "language_loss": 0.76231563, + "learning_rate": 3.513998014229469e-08, + "loss": 0.78404123, + "num_input_tokens_seen": 169349540, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.16040039, + "step": 7834, + "time_per_iteration": 2.870079517364502 + }, + { + "auxiliary_loss_clip": 0.01124176, + "auxiliary_loss_mlp": 0.01032956, + "balance_loss_clip": 1.03985238, + "balance_loss_mlp": 1.01947904, + "epoch": 0.9421030481572777, + "flos": 17712328377600.0, + "grad_norm": 2.4839676929629833, + "language_loss": 0.86242437, + "learning_rate": 3.499474861881069e-08, + "loss": 0.88399565, + "num_input_tokens_seen": 169366765, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.13482666, + "step": 7835, + "time_per_iteration": 2.8794496059417725 + }, + { + "auxiliary_loss_clip": 0.0112639, + "auxiliary_loss_mlp": 0.01040294, + "balance_loss_clip": 1.04155588, + "balance_loss_mlp": 1.02479684, + "epoch": 0.9422232910479168, + "flos": 20194114775040.0, + "grad_norm": 2.4424582166704014, + "language_loss": 0.67789078, + "learning_rate": 3.4849815185061136e-08, + "loss": 0.6995576, + "num_input_tokens_seen": 169386655, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.15478516, + "step": 7836, + "time_per_iteration": 2.8505725860595703 + }, + { + "auxiliary_loss_clip": 0.01121324, + "auxiliary_loss_mlp": 0.01035997, + "balance_loss_clip": 1.03825521, + "balance_loss_mlp": 1.02107787, + "epoch": 0.9423435339385559, + "flos": 18442571875200.0, + "grad_norm": 3.0547989049319275, + "language_loss": 0.75789583, + "learning_rate": 3.470517986303223e-08, + "loss": 0.77946901, + "num_input_tokens_seen": 169405640, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.14910889, + "step": 7837, + "time_per_iteration": 2.8109724521636963 + }, + { + "auxiliary_loss_clip": 0.01128292, + "auxiliary_loss_mlp": 0.01042522, + "balance_loss_clip": 1.04191959, + "balance_loss_mlp": 1.02781785, + "epoch": 0.942463776829195, + "flos": 20080636732800.0, + "grad_norm": 1.6161115078812864, + "language_loss": 0.79175866, + "learning_rate": 3.4560842674664856e-08, + "loss": 0.81346673, + "num_input_tokens_seen": 169424155, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.14715576, + "step": 7838, + "time_per_iteration": 2.8556039333343506 + }, + { + "auxiliary_loss_clip": 0.01123722, + "auxiliary_loss_mlp": 0.01031656, + "balance_loss_clip": 1.03871632, + "balance_loss_mlp": 1.01670694, + "epoch": 0.9425840197198341, + "flos": 22636255536000.0, + "grad_norm": 1.7088868954271637, + "language_loss": 0.75399488, + "learning_rate": 3.441680364185506e-08, + "loss": 0.7755487, + "num_input_tokens_seen": 169444025, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.14953613, + "step": 7839, + "time_per_iteration": 2.8973400592803955 + }, + { + "auxiliary_loss_clip": 0.01129657, + "auxiliary_loss_mlp": 0.01036667, + "balance_loss_clip": 1.04092288, + "balance_loss_mlp": 1.02035952, + "epoch": 0.9427042626104731, + "flos": 19937892084480.0, + "grad_norm": 2.4146733269245058, + "language_loss": 0.74651641, + "learning_rate": 3.427306278645314e-08, + "loss": 0.76817966, + "num_input_tokens_seen": 169462480, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.1628418, + "step": 7840, + "time_per_iteration": 3.4748318195343018 + }, + { + "auxiliary_loss_clip": 0.01124449, + "auxiliary_loss_mlp": 0.0103439, + "balance_loss_clip": 1.03938866, + "balance_loss_mlp": 1.01971567, + "epoch": 0.9428245055011123, + "flos": 22856998567680.0, + "grad_norm": 1.6444194314533545, + "language_loss": 0.72663921, + "learning_rate": 3.4129620130264767e-08, + "loss": 0.7482276, + "num_input_tokens_seen": 169480840, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.14660645, + "step": 7841, + "time_per_iteration": 2.8887033462524414 + }, + { + "auxiliary_loss_clip": 0.01125676, + "auxiliary_loss_mlp": 0.01040739, + "balance_loss_clip": 1.04171813, + "balance_loss_mlp": 1.02623081, + "epoch": 0.9429447483917514, + "flos": 20951757371520.0, + "grad_norm": 2.0695476754930855, + "language_loss": 0.77650511, + "learning_rate": 3.398647569505009e-08, + "loss": 0.79816926, + "num_input_tokens_seen": 169498265, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.1451416, + "step": 7842, + "time_per_iteration": 2.9522042274475098 + }, + { + "auxiliary_loss_clip": 0.01127165, + "auxiliary_loss_mlp": 0.01040931, + "balance_loss_clip": 1.03928328, + "balance_loss_mlp": 1.02549314, + "epoch": 0.9430649912823904, + "flos": 18843658116480.0, + "grad_norm": 2.2096045825235424, + "language_loss": 0.74829113, + "learning_rate": 3.384362950252373e-08, + "loss": 0.76997209, + "num_input_tokens_seen": 169515235, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.15441895, + "step": 7843, + "time_per_iteration": 2.892476797103882 + }, + { + "auxiliary_loss_clip": 0.01121449, + "auxiliary_loss_mlp": 0.01033724, + "balance_loss_clip": 1.03875411, + "balance_loss_mlp": 1.01785707, + "epoch": 0.9431852341730296, + "flos": 32556038837760.0, + "grad_norm": 1.8121065391806481, + "language_loss": 0.5697006, + "learning_rate": 3.3701081574355473e-08, + "loss": 0.59125233, + "num_input_tokens_seen": 169537195, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.15869141, + "step": 7844, + "time_per_iteration": 3.6547775268554688 + }, + { + "auxiliary_loss_clip": 0.01031789, + "auxiliary_loss_mlp": 0.01000278, + "balance_loss_clip": 1.00412464, + "balance_loss_mlp": 0.99899668, + "epoch": 0.9433054770636686, + "flos": 66904490252160.0, + "grad_norm": 0.6378938189915633, + "language_loss": 0.51649511, + "learning_rate": 3.3558831932169796e-08, + "loss": 0.53681576, + "num_input_tokens_seen": 169605865, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01281738, + "step": 7845, + "time_per_iteration": 3.4138786792755127 + }, + { + "auxiliary_loss_clip": 0.01123786, + "auxiliary_loss_mlp": 0.01035785, + "balance_loss_clip": 1.03886688, + "balance_loss_mlp": 1.02066898, + "epoch": 0.9434257199543077, + "flos": 26140346916480.0, + "grad_norm": 1.8110345929875862, + "language_loss": 0.8854062, + "learning_rate": 3.341688059754588e-08, + "loss": 0.90700185, + "num_input_tokens_seen": 169621520, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15124512, + "step": 7846, + "time_per_iteration": 2.880279302597046 + }, + { + "auxiliary_loss_clip": 0.0112328, + "auxiliary_loss_mlp": 0.01034559, + "balance_loss_clip": 1.03959966, + "balance_loss_mlp": 1.02027822, + "epoch": 0.9435459628449467, + "flos": 25003486483200.0, + "grad_norm": 2.385761380997505, + "language_loss": 0.77548379, + "learning_rate": 3.327522759201762e-08, + "loss": 0.79706222, + "num_input_tokens_seen": 169641390, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.14276123, + "step": 7847, + "time_per_iteration": 3.7312796115875244 + }, + { + "auxiliary_loss_clip": 0.01124241, + "auxiliary_loss_mlp": 0.01037713, + "balance_loss_clip": 1.03795576, + "balance_loss_mlp": 1.02266884, + "epoch": 0.9436662057355859, + "flos": 22163240309760.0, + "grad_norm": 2.155324689917951, + "language_loss": 0.66833305, + "learning_rate": 3.313387293707359e-08, + "loss": 0.68995261, + "num_input_tokens_seen": 169660095, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15063477, + "step": 7848, + "time_per_iteration": 2.9037797451019287 + }, + { + "auxiliary_loss_clip": 0.01126871, + "auxiliary_loss_mlp": 0.01041791, + "balance_loss_clip": 1.04045105, + "balance_loss_mlp": 1.02702093, + "epoch": 0.943786448626225, + "flos": 20118522602880.0, + "grad_norm": 1.8454786798637464, + "language_loss": 0.68812168, + "learning_rate": 3.29928166541571e-08, + "loss": 0.70980835, + "num_input_tokens_seen": 169679050, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.14782715, + "step": 7849, + "time_per_iteration": 2.8531370162963867 + }, + { + "auxiliary_loss_clip": 0.01123883, + "auxiliary_loss_mlp": 0.01039935, + "balance_loss_clip": 1.03988159, + "balance_loss_mlp": 1.02442622, + "epoch": 0.943906691516864, + "flos": 22090808534400.0, + "grad_norm": 1.8693559436369687, + "language_loss": 0.80271292, + "learning_rate": 3.2852058764666346e-08, + "loss": 0.82435113, + "num_input_tokens_seen": 169698150, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.15490723, + "step": 7850, + "time_per_iteration": 3.7674524784088135 + }, + { + "auxiliary_loss_clip": 0.011218, + "auxiliary_loss_mlp": 0.01037949, + "balance_loss_clip": 1.04132247, + "balance_loss_mlp": 1.02345347, + "epoch": 0.9440269344075032, + "flos": 35298501212160.0, + "grad_norm": 2.1207419268420704, + "language_loss": 0.68884975, + "learning_rate": 3.2711599289954264e-08, + "loss": 0.71044725, + "num_input_tokens_seen": 169722185, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.14489746, + "step": 7851, + "time_per_iteration": 2.9390852451324463 + }, + { + "auxiliary_loss_clip": 0.01125188, + "auxiliary_loss_mlp": 0.01037263, + "balance_loss_clip": 1.04033363, + "balance_loss_mlp": 1.02283907, + "epoch": 0.9441471772981422, + "flos": 19238136255360.0, + "grad_norm": 1.8060792562263568, + "language_loss": 0.77867782, + "learning_rate": 3.257143825132847e-08, + "loss": 0.80030239, + "num_input_tokens_seen": 169740355, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14428711, + "step": 7852, + "time_per_iteration": 2.8382301330566406 + }, + { + "auxiliary_loss_clip": 0.01119408, + "auxiliary_loss_mlp": 0.01032938, + "balance_loss_clip": 1.03877771, + "balance_loss_mlp": 1.01918697, + "epoch": 0.9442674201887813, + "flos": 25739799379200.0, + "grad_norm": 1.816161249300884, + "language_loss": 0.76242292, + "learning_rate": 3.243157567005106e-08, + "loss": 0.78394639, + "num_input_tokens_seen": 169758535, + "router_z_loss_clip": 0.80615234, + "router_z_loss_mlp": 0.1373291, + "step": 7853, + "time_per_iteration": 2.753720998764038 + }, + { + "auxiliary_loss_clip": 0.0113114, + "auxiliary_loss_mlp": 0.01049829, + "balance_loss_clip": 1.04316247, + "balance_loss_mlp": 1.03348577, + "epoch": 0.9443876630794205, + "flos": 15523321737600.0, + "grad_norm": 1.9686593974549473, + "language_loss": 0.64090395, + "learning_rate": 3.2292011567339296e-08, + "loss": 0.66271365, + "num_input_tokens_seen": 169776340, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.16345215, + "step": 7854, + "time_per_iteration": 2.7472429275512695 + }, + { + "auxiliary_loss_clip": 0.01121598, + "auxiliary_loss_mlp": 0.01041062, + "balance_loss_clip": 1.03774965, + "balance_loss_mlp": 1.02682209, + "epoch": 0.9445079059700595, + "flos": 13400821128960.0, + "grad_norm": 2.1178593086558752, + "language_loss": 0.5619328, + "learning_rate": 3.21527459643649e-08, + "loss": 0.58355939, + "num_input_tokens_seen": 169793225, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.14239502, + "step": 7855, + "time_per_iteration": 2.6978230476379395 + }, + { + "auxiliary_loss_clip": 0.01126963, + "auxiliary_loss_mlp": 0.01045581, + "balance_loss_clip": 1.04138231, + "balance_loss_mlp": 1.03078759, + "epoch": 0.9446281488606986, + "flos": 23659242877440.0, + "grad_norm": 2.078985602513331, + "language_loss": 0.73829496, + "learning_rate": 3.2013778882254536e-08, + "loss": 0.76002038, + "num_input_tokens_seen": 169812020, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.14782715, + "step": 7856, + "time_per_iteration": 2.7509829998016357 + }, + { + "auxiliary_loss_clip": 0.01122458, + "auxiliary_loss_mlp": 0.01032253, + "balance_loss_clip": 1.03909922, + "balance_loss_mlp": 1.01782274, + "epoch": 0.9447483917513377, + "flos": 25557337267200.0, + "grad_norm": 1.8662402618348708, + "language_loss": 0.75942487, + "learning_rate": 3.1875110342088676e-08, + "loss": 0.780972, + "num_input_tokens_seen": 169833470, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.14422607, + "step": 7857, + "time_per_iteration": 2.800081491470337 + }, + { + "auxiliary_loss_clip": 0.0112185, + "auxiliary_loss_mlp": 0.01033197, + "balance_loss_clip": 1.03899145, + "balance_loss_mlp": 1.01838529, + "epoch": 0.9448686346419768, + "flos": 24535463247360.0, + "grad_norm": 2.5863258891046756, + "language_loss": 0.65566516, + "learning_rate": 3.1736740364904035e-08, + "loss": 0.67721564, + "num_input_tokens_seen": 169854000, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.14801025, + "step": 7858, + "time_per_iteration": 2.8731861114501953 + }, + { + "auxiliary_loss_clip": 0.01126581, + "auxiliary_loss_mlp": 0.01039128, + "balance_loss_clip": 1.03972983, + "balance_loss_mlp": 1.02396441, + "epoch": 0.9449888775326158, + "flos": 14721256995840.0, + "grad_norm": 1.9865521841201466, + "language_loss": 0.76978934, + "learning_rate": 3.159866897169094e-08, + "loss": 0.79144645, + "num_input_tokens_seen": 169872200, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15148926, + "step": 7859, + "time_per_iteration": 2.8743300437927246 + }, + { + "auxiliary_loss_clip": 0.01123993, + "auxiliary_loss_mlp": 0.01046901, + "balance_loss_clip": 1.03945136, + "balance_loss_mlp": 1.03132081, + "epoch": 0.945109120423255, + "flos": 15447873219840.0, + "grad_norm": 1.644100597729304, + "language_loss": 0.75602269, + "learning_rate": 3.146089618339487e-08, + "loss": 0.77773166, + "num_input_tokens_seen": 169889055, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.15600586, + "step": 7860, + "time_per_iteration": 2.7621116638183594 + }, + { + "auxiliary_loss_clip": 0.01127341, + "auxiliary_loss_mlp": 0.01038175, + "balance_loss_clip": 1.04135501, + "balance_loss_mlp": 1.02319026, + "epoch": 0.9452293633138941, + "flos": 25448097029760.0, + "grad_norm": 1.7675211832427704, + "language_loss": 0.68075502, + "learning_rate": 3.132342202091554e-08, + "loss": 0.70241016, + "num_input_tokens_seen": 169909280, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.14978027, + "step": 7861, + "time_per_iteration": 2.774662971496582 + }, + { + "auxiliary_loss_clip": 0.01127237, + "auxiliary_loss_mlp": 0.01038293, + "balance_loss_clip": 1.0413518, + "balance_loss_mlp": 1.02216387, + "epoch": 0.9453496062045331, + "flos": 21215342350080.0, + "grad_norm": 1.8943086529374376, + "language_loss": 0.68570083, + "learning_rate": 3.1186246505107595e-08, + "loss": 0.7073561, + "num_input_tokens_seen": 169928420, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.16113281, + "step": 7862, + "time_per_iteration": 2.7451870441436768 + }, + { + "auxiliary_loss_clip": 0.01126305, + "auxiliary_loss_mlp": 0.01040948, + "balance_loss_clip": 1.04150581, + "balance_loss_mlp": 1.02592731, + "epoch": 0.9454698490951723, + "flos": 20010898477440.0, + "grad_norm": 1.6876118376762017, + "language_loss": 0.83510512, + "learning_rate": 3.104936965678084e-08, + "loss": 0.85677767, + "num_input_tokens_seen": 169946750, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.15014648, + "step": 7863, + "time_per_iteration": 2.827361583709717 + }, + { + "auxiliary_loss_clip": 0.01122682, + "auxiliary_loss_mlp": 0.01040761, + "balance_loss_clip": 1.03761029, + "balance_loss_mlp": 1.02508485, + "epoch": 0.9455900919858113, + "flos": 21069652786560.0, + "grad_norm": 1.7876102882984204, + "language_loss": 0.81796539, + "learning_rate": 3.091279149669956e-08, + "loss": 0.83959985, + "num_input_tokens_seen": 169965540, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.15673828, + "step": 7864, + "time_per_iteration": 2.8419606685638428 + }, + { + "auxiliary_loss_clip": 0.01121043, + "auxiliary_loss_mlp": 0.01037906, + "balance_loss_clip": 1.03822839, + "balance_loss_mlp": 1.02323675, + "epoch": 0.9457103348764504, + "flos": 20740854666240.0, + "grad_norm": 1.7940204004074851, + "language_loss": 0.73738635, + "learning_rate": 3.0776512045581624e-08, + "loss": 0.75897586, + "num_input_tokens_seen": 169984330, + "router_z_loss_clip": 0.82763672, + "router_z_loss_mlp": 0.14666748, + "step": 7865, + "time_per_iteration": 2.8240480422973633 + }, + { + "auxiliary_loss_clip": 0.01124222, + "auxiliary_loss_mlp": 0.01046744, + "balance_loss_clip": 1.03903675, + "balance_loss_mlp": 1.03117561, + "epoch": 0.9458305777670896, + "flos": 21428363957760.0, + "grad_norm": 1.901135485862934, + "language_loss": 0.77544296, + "learning_rate": 3.0640531324101384e-08, + "loss": 0.79715264, + "num_input_tokens_seen": 170002095, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.15563965, + "step": 7866, + "time_per_iteration": 2.862184762954712 + }, + { + "auxiliary_loss_clip": 0.01128008, + "auxiliary_loss_mlp": 0.01038634, + "balance_loss_clip": 1.04191637, + "balance_loss_mlp": 1.02213502, + "epoch": 0.9459508206577286, + "flos": 20011185786240.0, + "grad_norm": 1.8510245384973019, + "language_loss": 0.76086396, + "learning_rate": 3.0504849352886554e-08, + "loss": 0.78253043, + "num_input_tokens_seen": 170020240, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.16503906, + "step": 7867, + "time_per_iteration": 2.888427257537842 + }, + { + "auxiliary_loss_clip": 0.01125643, + "auxiliary_loss_mlp": 0.01036227, + "balance_loss_clip": 1.04000235, + "balance_loss_mlp": 1.02086055, + "epoch": 0.9460710635483677, + "flos": 12166428291840.0, + "grad_norm": 2.6287004254782387, + "language_loss": 0.71892858, + "learning_rate": 3.036946615252023e-08, + "loss": 0.7405473, + "num_input_tokens_seen": 170035770, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15350342, + "step": 7868, + "time_per_iteration": 2.879995822906494 + }, + { + "auxiliary_loss_clip": 0.01122667, + "auxiliary_loss_mlp": 0.01041904, + "balance_loss_clip": 1.03979254, + "balance_loss_mlp": 1.02642488, + "epoch": 0.9461913064390068, + "flos": 34276196229120.0, + "grad_norm": 2.110997933926462, + "language_loss": 0.66865009, + "learning_rate": 3.0234381743539984e-08, + "loss": 0.69029582, + "num_input_tokens_seen": 170053385, + "router_z_loss_clip": 0.828125, + "router_z_loss_mlp": 0.15460205, + "step": 7869, + "time_per_iteration": 3.6395676136016846 + }, + { + "auxiliary_loss_clip": 0.0112712, + "auxiliary_loss_mlp": 0.01036139, + "balance_loss_clip": 1.04118109, + "balance_loss_mlp": 1.0200696, + "epoch": 0.9463115493296459, + "flos": 19463763536640.0, + "grad_norm": 2.3239951347142256, + "language_loss": 0.80222142, + "learning_rate": 3.0099596146437863e-08, + "loss": 0.82385397, + "num_input_tokens_seen": 170070490, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.16064453, + "step": 7870, + "time_per_iteration": 2.7767982482910156 + }, + { + "auxiliary_loss_clip": 0.01031953, + "auxiliary_loss_mlp": 0.01001198, + "balance_loss_clip": 1.00434875, + "balance_loss_mlp": 0.99995816, + "epoch": 0.946431792220285, + "flos": 70570824387840.0, + "grad_norm": 0.7735617192022956, + "language_loss": 0.60089386, + "learning_rate": 2.996510938166086e-08, + "loss": 0.62122536, + "num_input_tokens_seen": 170133465, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01239014, + "step": 7871, + "time_per_iteration": 3.4471354484558105 + }, + { + "auxiliary_loss_clip": 0.01124755, + "auxiliary_loss_mlp": 0.01040867, + "balance_loss_clip": 1.04042888, + "balance_loss_mlp": 1.02619815, + "epoch": 0.9465520351109241, + "flos": 18947906363520.0, + "grad_norm": 1.9245957247829764, + "language_loss": 0.73448652, + "learning_rate": 2.983092146960997e-08, + "loss": 0.7561428, + "num_input_tokens_seen": 170150810, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.14648438, + "step": 7872, + "time_per_iteration": 2.9214396476745605 + }, + { + "auxiliary_loss_clip": 0.01129971, + "auxiliary_loss_mlp": 0.01040101, + "balance_loss_clip": 1.0409497, + "balance_loss_mlp": 1.02462769, + "epoch": 0.9466722780015632, + "flos": 19135647774720.0, + "grad_norm": 4.43248015373193, + "language_loss": 0.79900551, + "learning_rate": 2.9697032430642256e-08, + "loss": 0.82070619, + "num_input_tokens_seen": 170169025, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.15490723, + "step": 7873, + "time_per_iteration": 3.5795154571533203 + }, + { + "auxiliary_loss_clip": 0.01120338, + "auxiliary_loss_mlp": 0.01034823, + "balance_loss_clip": 1.03825974, + "balance_loss_mlp": 1.02151299, + "epoch": 0.9467925208922022, + "flos": 17237912520960.0, + "grad_norm": 2.152455452802752, + "language_loss": 0.73911548, + "learning_rate": 2.9563442285067906e-08, + "loss": 0.76066709, + "num_input_tokens_seen": 170186070, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.13311768, + "step": 7874, + "time_per_iteration": 2.846834421157837 + }, + { + "auxiliary_loss_clip": 0.01124904, + "auxiliary_loss_mlp": 0.01044133, + "balance_loss_clip": 1.03947377, + "balance_loss_mlp": 1.0283618, + "epoch": 0.9469127637828414, + "flos": 29169016859520.0, + "grad_norm": 2.0887779255596737, + "language_loss": 0.79853535, + "learning_rate": 2.943015105315294e-08, + "loss": 0.82022572, + "num_input_tokens_seen": 170206265, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15771484, + "step": 7875, + "time_per_iteration": 2.8479886054992676 + }, + { + "auxiliary_loss_clip": 0.0113124, + "auxiliary_loss_mlp": 0.0104676, + "balance_loss_clip": 1.04134464, + "balance_loss_mlp": 1.03045213, + "epoch": 0.9470330066734804, + "flos": 26030460234240.0, + "grad_norm": 2.240681526700593, + "language_loss": 0.66105741, + "learning_rate": 2.929715875511718e-08, + "loss": 0.68283737, + "num_input_tokens_seen": 170225300, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.16308594, + "step": 7876, + "time_per_iteration": 3.6116256713867188 + }, + { + "auxiliary_loss_clip": 0.0112322, + "auxiliary_loss_mlp": 0.01047302, + "balance_loss_clip": 1.03818882, + "balance_loss_mlp": 1.03250217, + "epoch": 0.9471532495641195, + "flos": 23440906056960.0, + "grad_norm": 1.8206465728899481, + "language_loss": 0.69682354, + "learning_rate": 2.9164465411135375e-08, + "loss": 0.71852875, + "num_input_tokens_seen": 170245070, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.14788818, + "step": 7877, + "time_per_iteration": 2.8410701751708984 + }, + { + "auxiliary_loss_clip": 0.01124781, + "auxiliary_loss_mlp": 0.01038092, + "balance_loss_clip": 1.04015207, + "balance_loss_mlp": 1.02375722, + "epoch": 0.9472734924547586, + "flos": 15815850099840.0, + "grad_norm": 1.913331536635627, + "language_loss": 0.80483741, + "learning_rate": 2.9032071041337426e-08, + "loss": 0.82646608, + "num_input_tokens_seen": 170263305, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.14337158, + "step": 7878, + "time_per_iteration": 2.7426917552948 + }, + { + "auxiliary_loss_clip": 0.01124015, + "auxiliary_loss_mlp": 0.01038639, + "balance_loss_clip": 1.03972149, + "balance_loss_mlp": 1.02368426, + "epoch": 0.9473937353453977, + "flos": 11181793697280.0, + "grad_norm": 1.6014554658250775, + "language_loss": 0.7294718, + "learning_rate": 2.889997566580704e-08, + "loss": 0.75109833, + "num_input_tokens_seen": 170281460, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.14959717, + "step": 7879, + "time_per_iteration": 3.7657864093780518 + }, + { + "auxiliary_loss_clip": 0.01125824, + "auxiliary_loss_mlp": 0.01035779, + "balance_loss_clip": 1.03984487, + "balance_loss_mlp": 1.01923251, + "epoch": 0.9475139782360368, + "flos": 25775530433280.0, + "grad_norm": 1.6088040235920695, + "language_loss": 0.70030618, + "learning_rate": 2.8768179304583086e-08, + "loss": 0.72192216, + "num_input_tokens_seen": 170303515, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.16540527, + "step": 7880, + "time_per_iteration": 2.8971567153930664 + }, + { + "auxiliary_loss_clip": 0.01124674, + "auxiliary_loss_mlp": 0.01036101, + "balance_loss_clip": 1.04142523, + "balance_loss_mlp": 1.02177238, + "epoch": 0.9476342211266758, + "flos": 22820046451200.0, + "grad_norm": 1.7550535687844528, + "language_loss": 0.73740184, + "learning_rate": 2.8636681977659117e-08, + "loss": 0.7590096, + "num_input_tokens_seen": 170323165, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.14331055, + "step": 7881, + "time_per_iteration": 2.8292720317840576 + }, + { + "auxiliary_loss_clip": 0.01130706, + "auxiliary_loss_mlp": 0.01039955, + "balance_loss_clip": 1.04310489, + "balance_loss_mlp": 1.02368283, + "epoch": 0.947754464017315, + "flos": 20193611984640.0, + "grad_norm": 2.168589228553856, + "language_loss": 0.77880204, + "learning_rate": 2.850548370498318e-08, + "loss": 0.80050874, + "num_input_tokens_seen": 170341005, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.1628418, + "step": 7882, + "time_per_iteration": 2.8337008953094482 + }, + { + "auxiliary_loss_clip": 0.01120717, + "auxiliary_loss_mlp": 0.01037179, + "balance_loss_clip": 1.03830695, + "balance_loss_mlp": 1.02260518, + "epoch": 0.9478747069079541, + "flos": 24717925359360.0, + "grad_norm": 1.7091789261679535, + "language_loss": 0.71234107, + "learning_rate": 2.8374584506457798e-08, + "loss": 0.7339201, + "num_input_tokens_seen": 170362280, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.14569092, + "step": 7883, + "time_per_iteration": 2.974658727645874 + }, + { + "auxiliary_loss_clip": 0.01124237, + "auxiliary_loss_mlp": 0.0103223, + "balance_loss_clip": 1.03873873, + "balance_loss_mlp": 1.0171622, + "epoch": 0.9479949497985931, + "flos": 21361355136000.0, + "grad_norm": 2.0531187092152643, + "language_loss": 0.6784662, + "learning_rate": 2.824398440193998e-08, + "loss": 0.70003092, + "num_input_tokens_seen": 170381080, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15075684, + "step": 7884, + "time_per_iteration": 2.919429063796997 + }, + { + "auxiliary_loss_clip": 0.01129304, + "auxiliary_loss_mlp": 0.01033357, + "balance_loss_clip": 1.0409019, + "balance_loss_mlp": 1.01833618, + "epoch": 0.9481151926892323, + "flos": 18148606968960.0, + "grad_norm": 2.1216673013148206, + "language_loss": 0.71656537, + "learning_rate": 2.811368341124232e-08, + "loss": 0.73819196, + "num_input_tokens_seen": 170400150, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.15014648, + "step": 7885, + "time_per_iteration": 2.970777988433838 + }, + { + "auxiliary_loss_clip": 0.01128327, + "auxiliary_loss_mlp": 0.01040457, + "balance_loss_clip": 1.04296792, + "balance_loss_mlp": 1.02547884, + "epoch": 0.9482354355798713, + "flos": 22128012046080.0, + "grad_norm": 3.191251393367567, + "language_loss": 0.68344855, + "learning_rate": 2.7983681554131222e-08, + "loss": 0.70513642, + "num_input_tokens_seen": 170420410, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.14984131, + "step": 7886, + "time_per_iteration": 2.8881349563598633 + }, + { + "auxiliary_loss_clip": 0.01127843, + "auxiliary_loss_mlp": 0.01037939, + "balance_loss_clip": 1.03998208, + "balance_loss_mlp": 1.02324057, + "epoch": 0.9483556784705104, + "flos": 19063072344960.0, + "grad_norm": 2.4970219950680663, + "language_loss": 0.7008242, + "learning_rate": 2.7853978850327365e-08, + "loss": 0.72248203, + "num_input_tokens_seen": 170439580, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.14709473, + "step": 7887, + "time_per_iteration": 2.876413106918335 + }, + { + "auxiliary_loss_clip": 0.01123736, + "auxiliary_loss_mlp": 0.01035091, + "balance_loss_clip": 1.04238749, + "balance_loss_mlp": 1.02091694, + "epoch": 0.9484759213611496, + "flos": 25777110631680.0, + "grad_norm": 1.7281138096305682, + "language_loss": 0.87078327, + "learning_rate": 2.7724575319507225e-08, + "loss": 0.89237154, + "num_input_tokens_seen": 170459290, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.14178467, + "step": 7888, + "time_per_iteration": 2.8956382274627686 + }, + { + "auxiliary_loss_clip": 0.01119563, + "auxiliary_loss_mlp": 0.0103441, + "balance_loss_clip": 1.03805733, + "balance_loss_mlp": 1.02055752, + "epoch": 0.9485961642517886, + "flos": 20667740532480.0, + "grad_norm": 1.7710535161437697, + "language_loss": 0.7735306, + "learning_rate": 2.759547098130044e-08, + "loss": 0.79507035, + "num_input_tokens_seen": 170478020, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.1385498, + "step": 7889, + "time_per_iteration": 2.9775781631469727 + }, + { + "auxiliary_loss_clip": 0.01121933, + "auxiliary_loss_mlp": 0.01037961, + "balance_loss_clip": 1.0382427, + "balance_loss_mlp": 1.02305412, + "epoch": 0.9487164071424277, + "flos": 22674069578880.0, + "grad_norm": 1.953452276840878, + "language_loss": 0.76881421, + "learning_rate": 2.746666585529267e-08, + "loss": 0.79041308, + "num_input_tokens_seen": 170498295, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.14910889, + "step": 7890, + "time_per_iteration": 2.857950210571289 + }, + { + "auxiliary_loss_clip": 0.01126286, + "auxiliary_loss_mlp": 0.01038332, + "balance_loss_clip": 1.04140687, + "balance_loss_mlp": 1.02325261, + "epoch": 0.9488366500330668, + "flos": 38726461716480.0, + "grad_norm": 2.8232075109550285, + "language_loss": 0.73864233, + "learning_rate": 2.73381599610234e-08, + "loss": 0.76028848, + "num_input_tokens_seen": 170518695, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15075684, + "step": 7891, + "time_per_iteration": 3.033271312713623 + }, + { + "auxiliary_loss_clip": 0.01126251, + "auxiliary_loss_mlp": 0.01037666, + "balance_loss_clip": 1.03915501, + "balance_loss_mlp": 1.02235973, + "epoch": 0.9489568929237059, + "flos": 27890920149120.0, + "grad_norm": 1.6376265252553786, + "language_loss": 0.71477365, + "learning_rate": 2.7209953317987033e-08, + "loss": 0.73641288, + "num_input_tokens_seen": 170539735, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.15319824, + "step": 7892, + "time_per_iteration": 2.877887725830078 + }, + { + "auxiliary_loss_clip": 0.01121723, + "auxiliary_loss_mlp": 0.01037978, + "balance_loss_clip": 1.0388993, + "balance_loss_mlp": 1.0232203, + "epoch": 0.9490771358143449, + "flos": 33580642291200.0, + "grad_norm": 2.153357865019921, + "language_loss": 0.78484428, + "learning_rate": 2.7082045945631793e-08, + "loss": 0.80644131, + "num_input_tokens_seen": 170561950, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.14758301, + "step": 7893, + "time_per_iteration": 2.98174786567688 + }, + { + "auxiliary_loss_clip": 0.01123836, + "auxiliary_loss_mlp": 0.01033976, + "balance_loss_clip": 1.03913724, + "balance_loss_mlp": 1.01898551, + "epoch": 0.9491973787049841, + "flos": 14793796512000.0, + "grad_norm": 1.9498293455482532, + "language_loss": 0.69365633, + "learning_rate": 2.6954437863361712e-08, + "loss": 0.71523452, + "num_input_tokens_seen": 170579865, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.14996338, + "step": 7894, + "time_per_iteration": 2.9244160652160645 + }, + { + "auxiliary_loss_clip": 0.01123202, + "auxiliary_loss_mlp": 0.01029742, + "balance_loss_clip": 1.04035342, + "balance_loss_mlp": 1.0160569, + "epoch": 0.9493176215956232, + "flos": 25332535998720.0, + "grad_norm": 1.9841196799916891, + "language_loss": 0.70753384, + "learning_rate": 2.6827129090534862e-08, + "loss": 0.72906333, + "num_input_tokens_seen": 170600165, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.13684082, + "step": 7895, + "time_per_iteration": 2.8328964710235596 + }, + { + "auxiliary_loss_clip": 0.01124899, + "auxiliary_loss_mlp": 0.01032745, + "balance_loss_clip": 1.04025042, + "balance_loss_mlp": 1.01685417, + "epoch": 0.9494378644862622, + "flos": 21029971236480.0, + "grad_norm": 4.483255572653402, + "language_loss": 0.7799682, + "learning_rate": 2.670011964646335e-08, + "loss": 0.80154467, + "num_input_tokens_seen": 170618845, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.15869141, + "step": 7896, + "time_per_iteration": 2.90067195892334 + }, + { + "auxiliary_loss_clip": 0.01126899, + "auxiliary_loss_mlp": 0.01036061, + "balance_loss_clip": 1.04011524, + "balance_loss_mlp": 1.01997924, + "epoch": 0.9495581073769014, + "flos": 15195134148480.0, + "grad_norm": 2.3502653254771064, + "language_loss": 0.68310177, + "learning_rate": 2.657340955041487e-08, + "loss": 0.7047314, + "num_input_tokens_seen": 170637620, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.1607666, + "step": 7897, + "time_per_iteration": 2.801625967025757 + }, + { + "auxiliary_loss_clip": 0.0112915, + "auxiliary_loss_mlp": 0.01035473, + "balance_loss_clip": 1.0414021, + "balance_loss_mlp": 1.01871216, + "epoch": 0.9496783502675404, + "flos": 28616566705920.0, + "grad_norm": 1.955924785978881, + "language_loss": 0.71855646, + "learning_rate": 2.6446998821611167e-08, + "loss": 0.74020272, + "num_input_tokens_seen": 170657815, + "router_z_loss_clip": 0.87646484, + "router_z_loss_mlp": 0.1675415, + "step": 7898, + "time_per_iteration": 3.5656018257141113 + }, + { + "auxiliary_loss_clip": 0.01124512, + "auxiliary_loss_mlp": 0.01039824, + "balance_loss_clip": 1.03964889, + "balance_loss_mlp": 1.02504158, + "epoch": 0.9497985931581795, + "flos": 14866874732160.0, + "grad_norm": 2.411628466801888, + "language_loss": 0.71640742, + "learning_rate": 2.6320887479228228e-08, + "loss": 0.73805076, + "num_input_tokens_seen": 170674415, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.14794922, + "step": 7899, + "time_per_iteration": 2.844968795776367 + }, + { + "auxiliary_loss_clip": 0.01126008, + "auxiliary_loss_mlp": 0.01037987, + "balance_loss_clip": 1.03892279, + "balance_loss_mlp": 1.02352726, + "epoch": 0.9499188360488187, + "flos": 27193319136000.0, + "grad_norm": 2.3740609298318693, + "language_loss": 0.72458124, + "learning_rate": 2.619507554239786e-08, + "loss": 0.74622113, + "num_input_tokens_seen": 170692975, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.14453125, + "step": 7900, + "time_per_iteration": 2.876821279525757 + }, + { + "auxiliary_loss_clip": 0.01122473, + "auxiliary_loss_mlp": 0.01038479, + "balance_loss_clip": 1.03920972, + "balance_loss_mlp": 1.02264774, + "epoch": 0.9500390789394577, + "flos": 24316479982080.0, + "grad_norm": 1.616466592302764, + "language_loss": 0.69607562, + "learning_rate": 2.606956303020502e-08, + "loss": 0.71768516, + "num_input_tokens_seen": 170713780, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.1583252, + "step": 7901, + "time_per_iteration": 3.6869213581085205 + }, + { + "auxiliary_loss_clip": 0.01124555, + "auxiliary_loss_mlp": 0.01044296, + "balance_loss_clip": 1.0407896, + "balance_loss_mlp": 1.02971077, + "epoch": 0.9501593218300968, + "flos": 14354752573440.0, + "grad_norm": 1.6713616923103392, + "language_loss": 0.84437138, + "learning_rate": 2.5944349961690036e-08, + "loss": 0.8660599, + "num_input_tokens_seen": 170730800, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14593506, + "step": 7902, + "time_per_iteration": 2.817732095718384 + }, + { + "auxiliary_loss_clip": 0.01122537, + "auxiliary_loss_mlp": 0.01040568, + "balance_loss_clip": 1.03888679, + "balance_loss_mlp": 1.0257268, + "epoch": 0.9502795647207359, + "flos": 38728113742080.0, + "grad_norm": 1.6090844225680536, + "language_loss": 0.73226184, + "learning_rate": 2.581943635584749e-08, + "loss": 0.7538929, + "num_input_tokens_seen": 170753630, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.14855957, + "step": 7903, + "time_per_iteration": 2.9817073345184326 + }, + { + "auxiliary_loss_clip": 0.01123435, + "auxiliary_loss_mlp": 0.01036908, + "balance_loss_clip": 1.04077077, + "balance_loss_mlp": 1.02255535, + "epoch": 0.950399807611375, + "flos": 40808023799040.0, + "grad_norm": 1.4933555549709672, + "language_loss": 0.65156096, + "learning_rate": 2.569482223162689e-08, + "loss": 0.67316437, + "num_input_tokens_seen": 170777605, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.14343262, + "step": 7904, + "time_per_iteration": 2.998552083969116 + }, + { + "auxiliary_loss_clip": 0.0112721, + "auxiliary_loss_mlp": 0.01037434, + "balance_loss_clip": 1.03899336, + "balance_loss_mlp": 1.02106643, + "epoch": 0.950520050502014, + "flos": 23440403266560.0, + "grad_norm": 1.5470688484839474, + "language_loss": 0.72474897, + "learning_rate": 2.5570507607932e-08, + "loss": 0.74639535, + "num_input_tokens_seen": 170797520, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.16381836, + "step": 7905, + "time_per_iteration": 3.6434667110443115 + }, + { + "auxiliary_loss_clip": 0.01126721, + "auxiliary_loss_mlp": 0.01038417, + "balance_loss_clip": 1.04063082, + "balance_loss_mlp": 1.02235913, + "epoch": 0.9506402933926532, + "flos": 17783718658560.0, + "grad_norm": 2.606608650942047, + "language_loss": 0.63453859, + "learning_rate": 2.54464925036213e-08, + "loss": 0.65618992, + "num_input_tokens_seen": 170814810, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.16040039, + "step": 7906, + "time_per_iteration": 2.855146646499634 + }, + { + "auxiliary_loss_clip": 0.01125078, + "auxiliary_loss_mlp": 0.01039053, + "balance_loss_clip": 1.03953481, + "balance_loss_mlp": 1.02468276, + "epoch": 0.9507605362832923, + "flos": 32561928668160.0, + "grad_norm": 2.4299914340194415, + "language_loss": 0.61111474, + "learning_rate": 2.532277693750773e-08, + "loss": 0.63275599, + "num_input_tokens_seen": 170835735, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.14367676, + "step": 7907, + "time_per_iteration": 3.008058786392212 + }, + { + "auxiliary_loss_clip": 0.01126456, + "auxiliary_loss_mlp": 0.01036106, + "balance_loss_clip": 1.04239178, + "balance_loss_mlp": 1.02088273, + "epoch": 0.9508807791739313, + "flos": 19602054898560.0, + "grad_norm": 1.9535892211542742, + "language_loss": 0.7593658, + "learning_rate": 2.5199360928358948e-08, + "loss": 0.78099138, + "num_input_tokens_seen": 170852970, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.15234375, + "step": 7908, + "time_per_iteration": 3.7834980487823486 + }, + { + "auxiliary_loss_clip": 0.01116408, + "auxiliary_loss_mlp": 0.0103051, + "balance_loss_clip": 1.0376606, + "balance_loss_mlp": 1.0175283, + "epoch": 0.9510010220645704, + "flos": 21471852349440.0, + "grad_norm": 1.7273679639264339, + "language_loss": 0.87097865, + "learning_rate": 2.507624449489665e-08, + "loss": 0.89244783, + "num_input_tokens_seen": 170871600, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.12982178, + "step": 7909, + "time_per_iteration": 2.9458844661712646 + }, + { + "auxiliary_loss_clip": 0.01126113, + "auxiliary_loss_mlp": 0.01035104, + "balance_loss_clip": 1.03945923, + "balance_loss_mlp": 1.01934433, + "epoch": 0.9511212649552095, + "flos": 18879999701760.0, + "grad_norm": 1.8239157583090135, + "language_loss": 0.64953637, + "learning_rate": 2.495342765579811e-08, + "loss": 0.67114854, + "num_input_tokens_seen": 170890260, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.15759277, + "step": 7910, + "time_per_iteration": 2.9279510974884033 + }, + { + "auxiliary_loss_clip": 0.01122616, + "auxiliary_loss_mlp": 0.01035216, + "balance_loss_clip": 1.0375067, + "balance_loss_mlp": 1.021209, + "epoch": 0.9512415078458486, + "flos": 20810521094400.0, + "grad_norm": 1.6376391541335782, + "language_loss": 0.7082395, + "learning_rate": 2.4830910429693984e-08, + "loss": 0.72981787, + "num_input_tokens_seen": 170910220, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.14019775, + "step": 7911, + "time_per_iteration": 2.837578535079956 + }, + { + "auxiliary_loss_clip": 0.0112509, + "auxiliary_loss_mlp": 0.01042713, + "balance_loss_clip": 1.04004455, + "balance_loss_mlp": 1.02665591, + "epoch": 0.9513617507364877, + "flos": 18369565482240.0, + "grad_norm": 1.779968120854482, + "language_loss": 0.79784745, + "learning_rate": 2.470869283517052e-08, + "loss": 0.81952548, + "num_input_tokens_seen": 170928255, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.16052246, + "step": 7912, + "time_per_iteration": 2.810580015182495 + }, + { + "auxiliary_loss_clip": 0.01122552, + "auxiliary_loss_mlp": 0.01033548, + "balance_loss_clip": 1.03913713, + "balance_loss_mlp": 1.01976693, + "epoch": 0.9514819936271268, + "flos": 25010166412800.0, + "grad_norm": 1.5949619830397466, + "language_loss": 0.76744014, + "learning_rate": 2.458677489076777e-08, + "loss": 0.78900111, + "num_input_tokens_seen": 170949265, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.13787842, + "step": 7913, + "time_per_iteration": 2.8456640243530273 + }, + { + "auxiliary_loss_clip": 0.01124826, + "auxiliary_loss_mlp": 0.01034782, + "balance_loss_clip": 1.04115891, + "balance_loss_mlp": 1.02101374, + "epoch": 0.9516022365177659, + "flos": 18662129758080.0, + "grad_norm": 1.5741887243012105, + "language_loss": 0.83096826, + "learning_rate": 2.446515661498072e-08, + "loss": 0.85256428, + "num_input_tokens_seen": 170968595, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.13781738, + "step": 7914, + "time_per_iteration": 2.813668727874756 + }, + { + "auxiliary_loss_clip": 0.01123217, + "auxiliary_loss_mlp": 0.01034534, + "balance_loss_clip": 1.03962553, + "balance_loss_mlp": 1.01938236, + "epoch": 0.9517224794084049, + "flos": 25372109808000.0, + "grad_norm": 2.832526883961496, + "language_loss": 0.74459517, + "learning_rate": 2.434383802625861e-08, + "loss": 0.76617265, + "num_input_tokens_seen": 170987550, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.15155029, + "step": 7915, + "time_per_iteration": 2.7930805683135986 + }, + { + "auxiliary_loss_clip": 0.0112282, + "auxiliary_loss_mlp": 0.01034601, + "balance_loss_clip": 1.03985548, + "balance_loss_mlp": 1.01919913, + "epoch": 0.9518427222990441, + "flos": 21470918595840.0, + "grad_norm": 1.7573332891818445, + "language_loss": 0.74252558, + "learning_rate": 2.4222819143005168e-08, + "loss": 0.76409984, + "num_input_tokens_seen": 171007145, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.15393066, + "step": 7916, + "time_per_iteration": 2.81391978263855 + }, + { + "auxiliary_loss_clip": 0.01124551, + "auxiliary_loss_mlp": 0.01035873, + "balance_loss_clip": 1.04027939, + "balance_loss_mlp": 1.02162731, + "epoch": 0.9519629651896832, + "flos": 21033634423680.0, + "grad_norm": 2.910564002828512, + "language_loss": 0.81103468, + "learning_rate": 2.4102099983579706e-08, + "loss": 0.83263898, + "num_input_tokens_seen": 171026295, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.14251709, + "step": 7917, + "time_per_iteration": 2.8118906021118164 + }, + { + "auxiliary_loss_clip": 0.01126343, + "auxiliary_loss_mlp": 0.01041499, + "balance_loss_clip": 1.04067254, + "balance_loss_mlp": 1.02511919, + "epoch": 0.9520832080803222, + "flos": 21689219502720.0, + "grad_norm": 7.237625667487571, + "language_loss": 0.77005732, + "learning_rate": 2.3981680566294236e-08, + "loss": 0.79173577, + "num_input_tokens_seen": 171045895, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.16369629, + "step": 7918, + "time_per_iteration": 2.7724926471710205 + }, + { + "auxiliary_loss_clip": 0.01121666, + "auxiliary_loss_mlp": 0.01038718, + "balance_loss_clip": 1.03932989, + "balance_loss_mlp": 1.02440727, + "epoch": 0.9522034509709614, + "flos": 23145289125120.0, + "grad_norm": 1.7341380436606846, + "language_loss": 0.73405582, + "learning_rate": 2.3861560909416822e-08, + "loss": 0.7556597, + "num_input_tokens_seen": 171065445, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.14312744, + "step": 7919, + "time_per_iteration": 2.7988357543945312 + }, + { + "auxiliary_loss_clip": 0.0112582, + "auxiliary_loss_mlp": 0.01037844, + "balance_loss_clip": 1.0411942, + "balance_loss_mlp": 1.02362275, + "epoch": 0.9523236938616004, + "flos": 24679428958080.0, + "grad_norm": 1.7428181766372985, + "language_loss": 0.82763374, + "learning_rate": 2.3741741031169325e-08, + "loss": 0.84927046, + "num_input_tokens_seen": 171085015, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.14202881, + "step": 7920, + "time_per_iteration": 2.825222969055176 + }, + { + "auxiliary_loss_clip": 0.01125295, + "auxiliary_loss_mlp": 0.01053964, + "balance_loss_clip": 1.04007339, + "balance_loss_mlp": 1.0383836, + "epoch": 0.9524439367522395, + "flos": 22672309812480.0, + "grad_norm": 1.9627737024458856, + "language_loss": 0.71392059, + "learning_rate": 2.3622220949728544e-08, + "loss": 0.73571324, + "num_input_tokens_seen": 171103900, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.15588379, + "step": 7921, + "time_per_iteration": 2.8266682624816895 + }, + { + "auxiliary_loss_clip": 0.01123105, + "auxiliary_loss_mlp": 0.01034618, + "balance_loss_clip": 1.03786123, + "balance_loss_mlp": 1.01887083, + "epoch": 0.9525641796428787, + "flos": 34055525024640.0, + "grad_norm": 3.3605389590099977, + "language_loss": 0.61067522, + "learning_rate": 2.3503000683225526e-08, + "loss": 0.6322524, + "num_input_tokens_seen": 171121615, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.15722656, + "step": 7922, + "time_per_iteration": 2.8319034576416016 + }, + { + "auxiliary_loss_clip": 0.01123522, + "auxiliary_loss_mlp": 0.01035597, + "balance_loss_clip": 1.03825307, + "balance_loss_mlp": 1.02001679, + "epoch": 0.9526844225335177, + "flos": 16727083251840.0, + "grad_norm": 2.112322297659448, + "language_loss": 0.84553629, + "learning_rate": 2.3384080249745585e-08, + "loss": 0.86712748, + "num_input_tokens_seen": 171139505, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.15576172, + "step": 7923, + "time_per_iteration": 2.867647409439087 + }, + { + "auxiliary_loss_clip": 0.01122777, + "auxiliary_loss_mlp": 0.01034804, + "balance_loss_clip": 1.03984118, + "balance_loss_mlp": 1.02012932, + "epoch": 0.9528046654241568, + "flos": 36939367330560.0, + "grad_norm": 2.2566505397884318, + "language_loss": 0.83200324, + "learning_rate": 2.3265459667329178e-08, + "loss": 0.8535791, + "num_input_tokens_seen": 171158995, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.14654541, + "step": 7924, + "time_per_iteration": 2.8992114067077637 + }, + { + "auxiliary_loss_clip": 0.01127186, + "auxiliary_loss_mlp": 0.01029163, + "balance_loss_clip": 1.03984666, + "balance_loss_mlp": 1.01448858, + "epoch": 0.9529249083147959, + "flos": 18255010032000.0, + "grad_norm": 2.2134390192345132, + "language_loss": 0.86678183, + "learning_rate": 2.31471389539708e-08, + "loss": 0.8883453, + "num_input_tokens_seen": 171176120, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.14678955, + "step": 7925, + "time_per_iteration": 2.797053337097168 + }, + { + "auxiliary_loss_clip": 0.01124957, + "auxiliary_loss_mlp": 0.01037964, + "balance_loss_clip": 1.04204714, + "balance_loss_mlp": 1.02192998, + "epoch": 0.953045151205435, + "flos": 28658438985600.0, + "grad_norm": 2.2646215822419196, + "language_loss": 0.72569704, + "learning_rate": 2.3029118127619872e-08, + "loss": 0.74732625, + "num_input_tokens_seen": 171195835, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.16040039, + "step": 7926, + "time_per_iteration": 2.8609135150909424 + }, + { + "auxiliary_loss_clip": 0.01124398, + "auxiliary_loss_mlp": 0.01036074, + "balance_loss_clip": 1.04016209, + "balance_loss_mlp": 1.02143502, + "epoch": 0.953165394096074, + "flos": 21835232288640.0, + "grad_norm": 2.020229375444863, + "language_loss": 0.86670482, + "learning_rate": 2.2911397206179628e-08, + "loss": 0.88830954, + "num_input_tokens_seen": 171212585, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.14630127, + "step": 7927, + "time_per_iteration": 3.4738972187042236 + }, + { + "auxiliary_loss_clip": 0.01123702, + "auxiliary_loss_mlp": 0.01038302, + "balance_loss_clip": 1.03998137, + "balance_loss_mlp": 1.024086, + "epoch": 0.9532856369867132, + "flos": 19975059682560.0, + "grad_norm": 1.7100569450468617, + "language_loss": 0.6280852, + "learning_rate": 2.279397620750845e-08, + "loss": 0.64970529, + "num_input_tokens_seen": 171231630, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.14208984, + "step": 7928, + "time_per_iteration": 2.810960531234741 + }, + { + "auxiliary_loss_clip": 0.01121597, + "auxiliary_loss_mlp": 0.01036273, + "balance_loss_clip": 1.03871512, + "balance_loss_mlp": 1.02150321, + "epoch": 0.9534058798773523, + "flos": 15049588239360.0, + "grad_norm": 1.8315956776564906, + "language_loss": 0.78534114, + "learning_rate": 2.2676855149419195e-08, + "loss": 0.80691981, + "num_input_tokens_seen": 171248800, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.14764404, + "step": 7929, + "time_per_iteration": 2.7952451705932617 + }, + { + "auxiliary_loss_clip": 0.01121851, + "auxiliary_loss_mlp": 0.01037192, + "balance_loss_clip": 1.03957748, + "balance_loss_mlp": 1.02214754, + "epoch": 0.9535261227679913, + "flos": 17602800831360.0, + "grad_norm": 2.0471273830655963, + "language_loss": 0.75579107, + "learning_rate": 2.2560034049678988e-08, + "loss": 0.77738154, + "num_input_tokens_seen": 171263150, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.15039062, + "step": 7930, + "time_per_iteration": 3.6737048625946045 + }, + { + "auxiliary_loss_clip": 0.01127575, + "auxiliary_loss_mlp": 0.01039674, + "balance_loss_clip": 1.04028976, + "balance_loss_mlp": 1.02498722, + "epoch": 0.9536463656586305, + "flos": 23142954741120.0, + "grad_norm": 1.662491582607327, + "language_loss": 0.75294054, + "learning_rate": 2.2443512926008988e-08, + "loss": 0.77461302, + "num_input_tokens_seen": 171282480, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.14666748, + "step": 7931, + "time_per_iteration": 2.8757195472717285 + }, + { + "auxiliary_loss_clip": 0.01125626, + "auxiliary_loss_mlp": 0.01038524, + "balance_loss_clip": 1.03961015, + "balance_loss_mlp": 1.02334893, + "epoch": 0.9537666085492695, + "flos": 18625033987200.0, + "grad_norm": 2.1896364008203437, + "language_loss": 0.70031339, + "learning_rate": 2.2327291796085946e-08, + "loss": 0.72195494, + "num_input_tokens_seen": 171300840, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.1517334, + "step": 7932, + "time_per_iteration": 2.840374708175659 + }, + { + "auxiliary_loss_clip": 0.01127697, + "auxiliary_loss_mlp": 0.01041367, + "balance_loss_clip": 1.04155052, + "balance_loss_mlp": 1.02462995, + "epoch": 0.9538868514399086, + "flos": 18989347680000.0, + "grad_norm": 2.566984251081194, + "language_loss": 0.77539092, + "learning_rate": 2.2211370677540197e-08, + "loss": 0.79708159, + "num_input_tokens_seen": 171317365, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.16723633, + "step": 7933, + "time_per_iteration": 2.9327316284179688 + }, + { + "auxiliary_loss_clip": 0.0112643, + "auxiliary_loss_mlp": 0.01036223, + "balance_loss_clip": 1.0407393, + "balance_loss_mlp": 1.02175736, + "epoch": 0.9540070943305478, + "flos": 16800556521600.0, + "grad_norm": 2.301540487152773, + "language_loss": 0.7835499, + "learning_rate": 2.2095749587957012e-08, + "loss": 0.80517638, + "num_input_tokens_seen": 171335270, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.14471436, + "step": 7934, + "time_per_iteration": 3.661393880844116 + }, + { + "auxiliary_loss_clip": 0.01126056, + "auxiliary_loss_mlp": 0.01034486, + "balance_loss_clip": 1.04080153, + "balance_loss_mlp": 1.0198586, + "epoch": 0.9541273372211868, + "flos": 20156911263360.0, + "grad_norm": 2.1074299950192508, + "language_loss": 0.69760507, + "learning_rate": 2.1980428544876138e-08, + "loss": 0.71921051, + "num_input_tokens_seen": 171353910, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.14611816, + "step": 7935, + "time_per_iteration": 2.7561142444610596 + }, + { + "auxiliary_loss_clip": 0.01122136, + "auxiliary_loss_mlp": 0.01032998, + "balance_loss_clip": 1.03893876, + "balance_loss_mlp": 1.01697659, + "epoch": 0.9542475801118259, + "flos": 26725511381760.0, + "grad_norm": 1.52704849893355, + "language_loss": 0.74200511, + "learning_rate": 2.1865407565791584e-08, + "loss": 0.76355648, + "num_input_tokens_seen": 171375480, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.16015625, + "step": 7936, + "time_per_iteration": 2.8649637699127197 + }, + { + "auxiliary_loss_clip": 0.0112245, + "auxiliary_loss_mlp": 0.01040344, + "balance_loss_clip": 1.03777778, + "balance_loss_mlp": 1.02551472, + "epoch": 0.954367823002465, + "flos": 23330911633920.0, + "grad_norm": 2.3148708743644493, + "language_loss": 0.77473259, + "learning_rate": 2.175068666815183e-08, + "loss": 0.79636049, + "num_input_tokens_seen": 171396320, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.14819336, + "step": 7937, + "time_per_iteration": 3.6877920627593994 + }, + { + "auxiliary_loss_clip": 0.01129208, + "auxiliary_loss_mlp": 0.01040269, + "balance_loss_clip": 1.04189134, + "balance_loss_mlp": 1.02484941, + "epoch": 0.9544880658931041, + "flos": 14902713527040.0, + "grad_norm": 1.9442419362428347, + "language_loss": 0.7917335, + "learning_rate": 2.163626586935985e-08, + "loss": 0.81342828, + "num_input_tokens_seen": 171412860, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.15435791, + "step": 7938, + "time_per_iteration": 2.785069704055786 + }, + { + "auxiliary_loss_clip": 0.01127468, + "auxiliary_loss_mlp": 0.01047304, + "balance_loss_clip": 1.0418222, + "balance_loss_mlp": 1.03107929, + "epoch": 0.9546083087837431, + "flos": 29095902725760.0, + "grad_norm": 2.9809489214581224, + "language_loss": 0.62953436, + "learning_rate": 2.1522145186773755e-08, + "loss": 0.65128213, + "num_input_tokens_seen": 171431780, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.16210938, + "step": 7939, + "time_per_iteration": 2.890500545501709 + }, + { + "auxiliary_loss_clip": 0.01123424, + "auxiliary_loss_mlp": 0.01034567, + "balance_loss_clip": 1.03996587, + "balance_loss_mlp": 1.02053046, + "epoch": 0.9547285516743822, + "flos": 21142335957120.0, + "grad_norm": 1.6753386037302471, + "language_loss": 0.85164988, + "learning_rate": 2.140832463770481e-08, + "loss": 0.8732298, + "num_input_tokens_seen": 171450975, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.14025879, + "step": 7940, + "time_per_iteration": 2.788747787475586 + }, + { + "auxiliary_loss_clip": 0.01126112, + "auxiliary_loss_mlp": 0.01036025, + "balance_loss_clip": 1.04030418, + "balance_loss_mlp": 1.02039635, + "epoch": 0.9548487945650214, + "flos": 27490157130240.0, + "grad_norm": 3.097482854976619, + "language_loss": 0.76045513, + "learning_rate": 2.129480423941987e-08, + "loss": 0.78207648, + "num_input_tokens_seen": 171467645, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15637207, + "step": 7941, + "time_per_iteration": 2.9310219287872314 + }, + { + "auxiliary_loss_clip": 0.01119468, + "auxiliary_loss_mlp": 0.01035594, + "balance_loss_clip": 1.03812909, + "balance_loss_mlp": 1.02066922, + "epoch": 0.9549690374556604, + "flos": 22273198819200.0, + "grad_norm": 1.7539814234014193, + "language_loss": 0.80401975, + "learning_rate": 2.1181584009140052e-08, + "loss": 0.82557034, + "num_input_tokens_seen": 171487185, + "router_z_loss_clip": 0.81347656, + "router_z_loss_mlp": 0.14916992, + "step": 7942, + "time_per_iteration": 2.8091447353363037 + }, + { + "auxiliary_loss_clip": 0.01125476, + "auxiliary_loss_mlp": 0.01032387, + "balance_loss_clip": 1.04131007, + "balance_loss_mlp": 1.01843977, + "epoch": 0.9550892803462995, + "flos": 17595294888960.0, + "grad_norm": 2.0832337616060284, + "language_loss": 0.837762, + "learning_rate": 2.10686639640405e-08, + "loss": 0.85934067, + "num_input_tokens_seen": 171501275, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.13934326, + "step": 7943, + "time_per_iteration": 2.79925537109375 + }, + { + "auxiliary_loss_clip": 0.01123693, + "auxiliary_loss_mlp": 0.0103617, + "balance_loss_clip": 1.03980792, + "balance_loss_mlp": 1.02036345, + "epoch": 0.9552095232369386, + "flos": 24353144789760.0, + "grad_norm": 1.6494029509803434, + "language_loss": 0.813169, + "learning_rate": 2.0956044121251294e-08, + "loss": 0.83476764, + "num_input_tokens_seen": 171520060, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.15808105, + "step": 7944, + "time_per_iteration": 2.8796560764312744 + }, + { + "auxiliary_loss_clip": 0.01128019, + "auxiliary_loss_mlp": 0.01037877, + "balance_loss_clip": 1.04057252, + "balance_loss_mlp": 1.0217123, + "epoch": 0.9553297661275777, + "flos": 22746860490240.0, + "grad_norm": 1.7976321715553376, + "language_loss": 0.80895293, + "learning_rate": 2.084372449785654e-08, + "loss": 0.83061188, + "num_input_tokens_seen": 171539895, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.16162109, + "step": 7945, + "time_per_iteration": 2.9135384559631348 + }, + { + "auxiliary_loss_clip": 0.0112433, + "auxiliary_loss_mlp": 0.01038011, + "balance_loss_clip": 1.04171801, + "balance_loss_mlp": 1.02339566, + "epoch": 0.9554500090182168, + "flos": 15413866018560.0, + "grad_norm": 1.7191992757057069, + "language_loss": 0.68454421, + "learning_rate": 2.0731705110895282e-08, + "loss": 0.70616764, + "num_input_tokens_seen": 171557385, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.14611816, + "step": 7946, + "time_per_iteration": 2.8487253189086914 + }, + { + "auxiliary_loss_clip": 0.01127919, + "auxiliary_loss_mlp": 0.01047565, + "balance_loss_clip": 1.04016662, + "balance_loss_mlp": 1.03123379, + "epoch": 0.9555702519088559, + "flos": 23513517400320.0, + "grad_norm": 2.0980743683172856, + "language_loss": 0.86844504, + "learning_rate": 2.0619985977360587e-08, + "loss": 0.8901999, + "num_input_tokens_seen": 171575705, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.16357422, + "step": 7947, + "time_per_iteration": 2.8711745738983154 + }, + { + "auxiliary_loss_clip": 0.01120164, + "auxiliary_loss_mlp": 0.01036775, + "balance_loss_clip": 1.03905296, + "balance_loss_mlp": 1.0227437, + "epoch": 0.955690494799495, + "flos": 22962072827520.0, + "grad_norm": 1.6153523942253372, + "language_loss": 0.76719642, + "learning_rate": 2.0508567114200237e-08, + "loss": 0.78876591, + "num_input_tokens_seen": 171595620, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.14031982, + "step": 7948, + "time_per_iteration": 2.8634421825408936 + }, + { + "auxiliary_loss_clip": 0.01121518, + "auxiliary_loss_mlp": 0.01032621, + "balance_loss_clip": 1.03968954, + "balance_loss_mlp": 1.01822662, + "epoch": 0.955810737690134, + "flos": 26031250333440.0, + "grad_norm": 2.2300482974620417, + "language_loss": 0.78549236, + "learning_rate": 2.0397448538316485e-08, + "loss": 0.80703378, + "num_input_tokens_seen": 171616660, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.14385986, + "step": 7949, + "time_per_iteration": 2.8583996295928955 + }, + { + "auxiliary_loss_clip": 0.01125881, + "auxiliary_loss_mlp": 0.01036218, + "balance_loss_clip": 1.0414114, + "balance_loss_mlp": 1.02176428, + "epoch": 0.9559309805807732, + "flos": 20849951249280.0, + "grad_norm": 1.8852062049609384, + "language_loss": 0.66765797, + "learning_rate": 2.028663026656563e-08, + "loss": 0.68927896, + "num_input_tokens_seen": 171635515, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.14453125, + "step": 7950, + "time_per_iteration": 2.8981568813323975 + }, + { + "auxiliary_loss_clip": 0.01121712, + "auxiliary_loss_mlp": 0.01040673, + "balance_loss_clip": 1.03841519, + "balance_loss_mlp": 1.0257895, + "epoch": 0.9560512234714122, + "flos": 21578219498880.0, + "grad_norm": 1.9001196577498154, + "language_loss": 0.71887171, + "learning_rate": 2.0176112315758885e-08, + "loss": 0.74049556, + "num_input_tokens_seen": 171653305, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.14892578, + "step": 7951, + "time_per_iteration": 2.8416976928710938 + }, + { + "auxiliary_loss_clip": 0.01126744, + "auxiliary_loss_mlp": 0.01037234, + "balance_loss_clip": 1.03996849, + "balance_loss_mlp": 1.02178407, + "epoch": 0.9561714663620513, + "flos": 17450144029440.0, + "grad_norm": 2.3931618336425173, + "language_loss": 0.69399244, + "learning_rate": 2.0065894702661957e-08, + "loss": 0.7156322, + "num_input_tokens_seen": 171669980, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15441895, + "step": 7952, + "time_per_iteration": 2.831582546234131 + }, + { + "auxiliary_loss_clip": 0.01123308, + "auxiliary_loss_mlp": 0.01034159, + "balance_loss_clip": 1.03860998, + "balance_loss_mlp": 1.01835227, + "epoch": 0.9562917092526905, + "flos": 26098510550400.0, + "grad_norm": 2.371255639617801, + "language_loss": 0.77833498, + "learning_rate": 1.9955977443994577e-08, + "loss": 0.79990971, + "num_input_tokens_seen": 171689970, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15808105, + "step": 7953, + "time_per_iteration": 2.8742194175720215 + }, + { + "auxiliary_loss_clip": 0.01130025, + "auxiliary_loss_mlp": 0.01043198, + "balance_loss_clip": 1.04030442, + "balance_loss_mlp": 1.02625799, + "epoch": 0.9564119521433295, + "flos": 24096742531200.0, + "grad_norm": 2.7217718510042355, + "language_loss": 0.61966705, + "learning_rate": 1.9846360556430965e-08, + "loss": 0.64139926, + "num_input_tokens_seen": 171708270, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.16943359, + "step": 7954, + "time_per_iteration": 2.831092119216919 + }, + { + "auxiliary_loss_clip": 0.01123959, + "auxiliary_loss_mlp": 0.01037931, + "balance_loss_clip": 1.03875983, + "balance_loss_mlp": 1.0235064, + "epoch": 0.9565321950339686, + "flos": 32008903896960.0, + "grad_norm": 2.2639504503775436, + "language_loss": 0.61535239, + "learning_rate": 1.973704405660004e-08, + "loss": 0.63697129, + "num_input_tokens_seen": 171729385, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.14440918, + "step": 7955, + "time_per_iteration": 2.901167631149292 + }, + { + "auxiliary_loss_clip": 0.01118868, + "auxiliary_loss_mlp": 0.01035564, + "balance_loss_clip": 1.03549254, + "balance_loss_mlp": 1.02131832, + "epoch": 0.9566524379246077, + "flos": 23588642695680.0, + "grad_norm": 1.4538972974363558, + "language_loss": 0.77916956, + "learning_rate": 1.9628027961085203e-08, + "loss": 0.80071384, + "num_input_tokens_seen": 171752615, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.14263916, + "step": 7956, + "time_per_iteration": 3.627124071121216 + }, + { + "auxiliary_loss_clip": 0.01122645, + "auxiliary_loss_mlp": 0.01034182, + "balance_loss_clip": 1.0392977, + "balance_loss_mlp": 1.01905489, + "epoch": 0.9567726808152468, + "flos": 38067716240640.0, + "grad_norm": 1.859760212507514, + "language_loss": 0.83806872, + "learning_rate": 1.9519312286423894e-08, + "loss": 0.85963702, + "num_input_tokens_seen": 171775810, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.15112305, + "step": 7957, + "time_per_iteration": 2.9140400886535645 + }, + { + "auxiliary_loss_clip": 0.01125076, + "auxiliary_loss_mlp": 0.01040591, + "balance_loss_clip": 1.04068267, + "balance_loss_mlp": 1.02588069, + "epoch": 0.9568929237058859, + "flos": 22744059229440.0, + "grad_norm": 1.566731262735906, + "language_loss": 0.77635282, + "learning_rate": 1.9410897049108255e-08, + "loss": 0.79800946, + "num_input_tokens_seen": 171795090, + "router_z_loss_clip": 0.84375, + "router_z_loss_mlp": 0.14715576, + "step": 7958, + "time_per_iteration": 2.831468343734741 + }, + { + "auxiliary_loss_clip": 0.01132878, + "auxiliary_loss_mlp": 0.01040037, + "balance_loss_clip": 1.04453099, + "balance_loss_mlp": 1.02436113, + "epoch": 0.957013166596525, + "flos": 23841633162240.0, + "grad_norm": 2.062406219355117, + "language_loss": 0.91286373, + "learning_rate": 1.9302782265584905e-08, + "loss": 0.93459284, + "num_input_tokens_seen": 171815755, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.15661621, + "step": 7959, + "time_per_iteration": 3.6137959957122803 + }, + { + "auxiliary_loss_clip": 0.01124026, + "auxiliary_loss_mlp": 0.01031488, + "balance_loss_clip": 1.03889906, + "balance_loss_mlp": 1.01774275, + "epoch": 0.9571334094871641, + "flos": 17639286071040.0, + "grad_norm": 2.4817213291829288, + "language_loss": 0.87093151, + "learning_rate": 1.9194967952254282e-08, + "loss": 0.89248669, + "num_input_tokens_seen": 171834330, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.13739014, + "step": 7960, + "time_per_iteration": 2.822279930114746 + }, + { + "auxiliary_loss_clip": 0.01124592, + "auxiliary_loss_mlp": 0.01043203, + "balance_loss_clip": 1.03978956, + "balance_loss_mlp": 1.02707362, + "epoch": 0.9572536523778031, + "flos": 15369623441280.0, + "grad_norm": 2.297603668061719, + "language_loss": 0.81447989, + "learning_rate": 1.9087454125472635e-08, + "loss": 0.83615786, + "num_input_tokens_seen": 171848805, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.16113281, + "step": 7961, + "time_per_iteration": 2.82373046875 + }, + { + "auxiliary_loss_clip": 0.01126965, + "auxiliary_loss_mlp": 0.01042033, + "balance_loss_clip": 1.03956795, + "balance_loss_mlp": 1.02626121, + "epoch": 0.9573738952684423, + "flos": 24969838417920.0, + "grad_norm": 2.220518905561043, + "language_loss": 0.78545928, + "learning_rate": 1.8980240801548696e-08, + "loss": 0.80714917, + "num_input_tokens_seen": 171867995, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.15783691, + "step": 7962, + "time_per_iteration": 2.8894126415252686 + }, + { + "auxiliary_loss_clip": 0.011224, + "auxiliary_loss_mlp": 0.01034358, + "balance_loss_clip": 1.0392983, + "balance_loss_mlp": 1.01950419, + "epoch": 0.9574941381590814, + "flos": 25769461034880.0, + "grad_norm": 1.8749745656458061, + "language_loss": 0.74027771, + "learning_rate": 1.8873327996747458e-08, + "loss": 0.76184529, + "num_input_tokens_seen": 171886495, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.14862061, + "step": 7963, + "time_per_iteration": 3.6487159729003906 + }, + { + "auxiliary_loss_clip": 0.01125715, + "auxiliary_loss_mlp": 0.01039236, + "balance_loss_clip": 1.04059696, + "balance_loss_mlp": 1.0235604, + "epoch": 0.9576143810497204, + "flos": 32307178435200.0, + "grad_norm": 1.9126793257320611, + "language_loss": 0.65671301, + "learning_rate": 1.8766715727287053e-08, + "loss": 0.67836249, + "num_input_tokens_seen": 171908200, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.15673828, + "step": 7964, + "time_per_iteration": 2.826697587966919 + }, + { + "auxiliary_loss_clip": 0.01126083, + "auxiliary_loss_mlp": 0.0103772, + "balance_loss_clip": 1.03892684, + "balance_loss_mlp": 1.02280664, + "epoch": 0.9577346239403596, + "flos": 27745733376000.0, + "grad_norm": 1.6321445276628133, + "language_loss": 0.79372931, + "learning_rate": 1.8660404009340546e-08, + "loss": 0.81536734, + "num_input_tokens_seen": 171928650, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.14916992, + "step": 7965, + "time_per_iteration": 3.7782294750213623 + }, + { + "auxiliary_loss_clip": 0.01031403, + "auxiliary_loss_mlp": 0.01002471, + "balance_loss_clip": 1.00388288, + "balance_loss_mlp": 1.00120723, + "epoch": 0.9578548668309986, + "flos": 57468313710720.0, + "grad_norm": 0.8713935978965591, + "language_loss": 0.5951919, + "learning_rate": 1.8554392859035485e-08, + "loss": 0.61553067, + "num_input_tokens_seen": 171986400, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01263428, + "step": 7966, + "time_per_iteration": 3.310122489929199 + }, + { + "auxiliary_loss_clip": 0.01126135, + "auxiliary_loss_mlp": 0.01044323, + "balance_loss_clip": 1.03992534, + "balance_loss_mlp": 1.02830172, + "epoch": 0.9579751097216377, + "flos": 19756040503680.0, + "grad_norm": 1.8475738177085959, + "language_loss": 0.78827542, + "learning_rate": 1.8448682292453444e-08, + "loss": 0.80997998, + "num_input_tokens_seen": 172005475, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.16027832, + "step": 7967, + "time_per_iteration": 2.8270201683044434 + }, + { + "auxiliary_loss_clip": 0.01125739, + "auxiliary_loss_mlp": 0.01038418, + "balance_loss_clip": 1.04023838, + "balance_loss_mlp": 1.02365398, + "epoch": 0.9580953526122769, + "flos": 18041270152320.0, + "grad_norm": 1.8932192802856076, + "language_loss": 0.65979457, + "learning_rate": 1.8343272325631154e-08, + "loss": 0.68143606, + "num_input_tokens_seen": 172024420, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.14764404, + "step": 7968, + "time_per_iteration": 2.932575225830078 + }, + { + "auxiliary_loss_clip": 0.0112817, + "auxiliary_loss_mlp": 0.01039399, + "balance_loss_clip": 1.04074228, + "balance_loss_mlp": 1.02416384, + "epoch": 0.9582155955029159, + "flos": 24270154416000.0, + "grad_norm": 2.1214798974050133, + "language_loss": 0.78125054, + "learning_rate": 1.8238162974558492e-08, + "loss": 0.80292618, + "num_input_tokens_seen": 172038350, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.15234375, + "step": 7969, + "time_per_iteration": 2.8844120502471924 + }, + { + "auxiliary_loss_clip": 0.01124127, + "auxiliary_loss_mlp": 0.01033106, + "balance_loss_clip": 1.040048, + "balance_loss_mlp": 1.01859832, + "epoch": 0.958335838393555, + "flos": 22783309816320.0, + "grad_norm": 1.829243537655694, + "language_loss": 0.74794888, + "learning_rate": 1.8133354255181144e-08, + "loss": 0.76952124, + "num_input_tokens_seen": 172058665, + "router_z_loss_clip": 0.84179688, + "router_z_loss_mlp": 0.1451416, + "step": 7970, + "time_per_iteration": 2.847853899002075 + }, + { + "auxiliary_loss_clip": 0.01121382, + "auxiliary_loss_mlp": 0.01035824, + "balance_loss_clip": 1.03874016, + "balance_loss_mlp": 1.02184057, + "epoch": 0.958456081284194, + "flos": 16911484698240.0, + "grad_norm": 1.7119219272998507, + "language_loss": 0.74595869, + "learning_rate": 1.802884618339795e-08, + "loss": 0.76753068, + "num_input_tokens_seen": 172077470, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.13989258, + "step": 7971, + "time_per_iteration": 2.862557888031006 + }, + { + "auxiliary_loss_clip": 0.01128815, + "auxiliary_loss_mlp": 0.01034508, + "balance_loss_clip": 1.04123378, + "balance_loss_mlp": 1.01967287, + "epoch": 0.9585763241748332, + "flos": 19974951941760.0, + "grad_norm": 1.893023842748759, + "language_loss": 0.80845952, + "learning_rate": 1.7924638775062894e-08, + "loss": 0.83009267, + "num_input_tokens_seen": 172096590, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.14825439, + "step": 7972, + "time_per_iteration": 2.8581223487854004 + }, + { + "auxiliary_loss_clip": 0.01121063, + "auxiliary_loss_mlp": 0.01030309, + "balance_loss_clip": 1.03831041, + "balance_loss_mlp": 1.01594436, + "epoch": 0.9586965670654722, + "flos": 21395649646080.0, + "grad_norm": 1.9552790669071995, + "language_loss": 0.81764126, + "learning_rate": 1.7820732045984444e-08, + "loss": 0.83915508, + "num_input_tokens_seen": 172116735, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.14373779, + "step": 7973, + "time_per_iteration": 2.998307466506958 + }, + { + "auxiliary_loss_clip": 0.01127088, + "auxiliary_loss_mlp": 0.01037338, + "balance_loss_clip": 1.03974783, + "balance_loss_mlp": 1.02111411, + "epoch": 0.9588168099561113, + "flos": 21435115714560.0, + "grad_norm": 1.720641907215369, + "language_loss": 0.74169564, + "learning_rate": 1.7717126011924655e-08, + "loss": 0.76333988, + "num_input_tokens_seen": 172138320, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.16235352, + "step": 7974, + "time_per_iteration": 2.918627977371216 + }, + { + "auxiliary_loss_clip": 0.01127556, + "auxiliary_loss_mlp": 0.01045628, + "balance_loss_clip": 1.04135644, + "balance_loss_mlp": 1.02982092, + "epoch": 0.9589370528467505, + "flos": 11763761852160.0, + "grad_norm": 2.7186126596779907, + "language_loss": 0.76531863, + "learning_rate": 1.7613820688600957e-08, + "loss": 0.78705049, + "num_input_tokens_seen": 172154225, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15808105, + "step": 7975, + "time_per_iteration": 2.918839931488037 + }, + { + "auxiliary_loss_clip": 0.01126065, + "auxiliary_loss_mlp": 0.01032311, + "balance_loss_clip": 1.04147673, + "balance_loss_mlp": 1.01804161, + "epoch": 0.9590572957373895, + "flos": 23441516588160.0, + "grad_norm": 1.7554480382618665, + "language_loss": 0.78426087, + "learning_rate": 1.7510816091684588e-08, + "loss": 0.80584455, + "num_input_tokens_seen": 172174150, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.14282227, + "step": 7976, + "time_per_iteration": 2.8092784881591797 + }, + { + "auxiliary_loss_clip": 0.01129273, + "auxiliary_loss_mlp": 0.01040153, + "balance_loss_clip": 1.04040504, + "balance_loss_mlp": 1.02417886, + "epoch": 0.9591775386280286, + "flos": 22528272274560.0, + "grad_norm": 2.651331004939145, + "language_loss": 0.7884872, + "learning_rate": 1.740811223680083e-08, + "loss": 0.8101815, + "num_input_tokens_seen": 172191005, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.15966797, + "step": 7977, + "time_per_iteration": 2.930259943008423 + }, + { + "auxiliary_loss_clip": 0.01126221, + "auxiliary_loss_mlp": 0.01036016, + "balance_loss_clip": 1.04108667, + "balance_loss_mlp": 1.02117491, + "epoch": 0.9592977815186677, + "flos": 18186959715840.0, + "grad_norm": 2.079943044167869, + "language_loss": 0.73998535, + "learning_rate": 1.7305709139530334e-08, + "loss": 0.76160771, + "num_input_tokens_seen": 172209785, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.14831543, + "step": 7978, + "time_per_iteration": 2.793680191040039 + }, + { + "auxiliary_loss_clip": 0.01124606, + "auxiliary_loss_mlp": 0.01037596, + "balance_loss_clip": 1.03867507, + "balance_loss_mlp": 1.02221835, + "epoch": 0.9594180244093068, + "flos": 16537797555840.0, + "grad_norm": 2.2479739815850563, + "language_loss": 0.74142832, + "learning_rate": 1.7203606815407334e-08, + "loss": 0.76305038, + "num_input_tokens_seen": 172224380, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.15393066, + "step": 7979, + "time_per_iteration": 2.6979780197143555 + }, + { + "auxiliary_loss_clip": 0.01130174, + "auxiliary_loss_mlp": 0.0104626, + "balance_loss_clip": 1.04285276, + "balance_loss_mlp": 1.02942729, + "epoch": 0.9595382672999458, + "flos": 20554334317440.0, + "grad_norm": 1.6266931532802278, + "language_loss": 0.78994888, + "learning_rate": 1.7101805279920557e-08, + "loss": 0.81171322, + "num_input_tokens_seen": 172242540, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.16809082, + "step": 7980, + "time_per_iteration": 2.8023767471313477 + }, + { + "auxiliary_loss_clip": 0.01126437, + "auxiliary_loss_mlp": 0.01044393, + "balance_loss_clip": 1.04070306, + "balance_loss_mlp": 1.02828765, + "epoch": 0.959658510190585, + "flos": 22638266697600.0, + "grad_norm": 1.882893067182101, + "language_loss": 0.81146735, + "learning_rate": 1.7000304548513643e-08, + "loss": 0.83317566, + "num_input_tokens_seen": 172262645, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.16113281, + "step": 7981, + "time_per_iteration": 2.7620463371276855 + }, + { + "auxiliary_loss_clip": 0.01125333, + "auxiliary_loss_mlp": 0.01038727, + "balance_loss_clip": 1.03877461, + "balance_loss_mlp": 1.02278829, + "epoch": 0.9597787530812241, + "flos": 19135252725120.0, + "grad_norm": 2.1474540745328192, + "language_loss": 0.82929617, + "learning_rate": 1.6899104636583394e-08, + "loss": 0.85093683, + "num_input_tokens_seen": 172280695, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.15942383, + "step": 7982, + "time_per_iteration": 2.777702569961548 + }, + { + "auxiliary_loss_clip": 0.01031482, + "auxiliary_loss_mlp": 0.00999285, + "balance_loss_clip": 1.00401258, + "balance_loss_mlp": 0.99797362, + "epoch": 0.9598989959718631, + "flos": 60098124055680.0, + "grad_norm": 0.7321766464997506, + "language_loss": 0.61940962, + "learning_rate": 1.6798205559482638e-08, + "loss": 0.63971728, + "num_input_tokens_seen": 172343075, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01312256, + "step": 7983, + "time_per_iteration": 3.450258255004883 + }, + { + "auxiliary_loss_clip": 0.01124451, + "auxiliary_loss_mlp": 0.01037249, + "balance_loss_clip": 1.03858495, + "balance_loss_mlp": 1.02275968, + "epoch": 0.9600192388625023, + "flos": 20886795624960.0, + "grad_norm": 2.4111165180965988, + "language_loss": 0.76870668, + "learning_rate": 1.669760733251713e-08, + "loss": 0.79032367, + "num_input_tokens_seen": 172361950, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.14501953, + "step": 7984, + "time_per_iteration": 2.734351634979248 + }, + { + "auxiliary_loss_clip": 0.01124085, + "auxiliary_loss_mlp": 0.01038793, + "balance_loss_clip": 1.03993475, + "balance_loss_mlp": 1.02426136, + "epoch": 0.9601394817531413, + "flos": 20445740524800.0, + "grad_norm": 2.0734070941136316, + "language_loss": 0.8255055, + "learning_rate": 1.659730997094755e-08, + "loss": 0.84713423, + "num_input_tokens_seen": 172380440, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.14526367, + "step": 7985, + "time_per_iteration": 3.438091516494751 + }, + { + "auxiliary_loss_clip": 0.01118763, + "auxiliary_loss_mlp": 0.01034198, + "balance_loss_clip": 1.0379622, + "balance_loss_mlp": 1.02008379, + "epoch": 0.9602597246437804, + "flos": 21507152440320.0, + "grad_norm": 2.397195833435067, + "language_loss": 0.625974, + "learning_rate": 1.6497313489989283e-08, + "loss": 0.64750361, + "num_input_tokens_seen": 172400265, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.14099121, + "step": 7986, + "time_per_iteration": 2.7675223350524902 + }, + { + "auxiliary_loss_clip": 0.01124333, + "auxiliary_loss_mlp": 0.01028504, + "balance_loss_clip": 1.038311, + "balance_loss_mlp": 1.01446128, + "epoch": 0.9603799675344196, + "flos": 29935099152000.0, + "grad_norm": 2.0444160241912113, + "language_loss": 0.70153904, + "learning_rate": 1.639761790481131e-08, + "loss": 0.72306734, + "num_input_tokens_seen": 172421145, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.14031982, + "step": 7987, + "time_per_iteration": 2.821197032928467 + }, + { + "auxiliary_loss_clip": 0.01123776, + "auxiliary_loss_mlp": 0.0103945, + "balance_loss_clip": 1.04028487, + "balance_loss_mlp": 1.02435839, + "epoch": 0.9605002104250586, + "flos": 28001525103360.0, + "grad_norm": 3.1703257511795337, + "language_loss": 0.79221094, + "learning_rate": 1.6298223230537754e-08, + "loss": 0.81384319, + "num_input_tokens_seen": 172438945, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.15112305, + "step": 7988, + "time_per_iteration": 2.8284003734588623 + }, + { + "auxiliary_loss_clip": 0.01127101, + "auxiliary_loss_mlp": 0.0103712, + "balance_loss_clip": 1.03884745, + "balance_loss_mlp": 1.02131271, + "epoch": 0.9606204533156977, + "flos": 35590490870400.0, + "grad_norm": 2.426489670609893, + "language_loss": 0.69936115, + "learning_rate": 1.619912948224611e-08, + "loss": 0.72100335, + "num_input_tokens_seen": 172460150, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15808105, + "step": 7989, + "time_per_iteration": 3.627483606338501 + }, + { + "auxiliary_loss_clip": 0.01126087, + "auxiliary_loss_mlp": 0.01035381, + "balance_loss_clip": 1.0402894, + "balance_loss_mlp": 1.02055144, + "epoch": 0.9607406962063368, + "flos": 26574614346240.0, + "grad_norm": 2.1189756426082362, + "language_loss": 0.60836881, + "learning_rate": 1.6100336674969682e-08, + "loss": 0.62998348, + "num_input_tokens_seen": 172478990, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.1484375, + "step": 7990, + "time_per_iteration": 2.9503815174102783 + }, + { + "auxiliary_loss_clip": 0.01123288, + "auxiliary_loss_mlp": 0.01043908, + "balance_loss_clip": 1.03855252, + "balance_loss_mlp": 1.02891707, + "epoch": 0.9608609390969759, + "flos": 25331781813120.0, + "grad_norm": 1.8067753289945292, + "language_loss": 0.76506293, + "learning_rate": 1.600184482369449e-08, + "loss": 0.78673488, + "num_input_tokens_seen": 172498905, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14990234, + "step": 7991, + "time_per_iteration": 2.867609977722168 + }, + { + "auxiliary_loss_clip": 0.01128037, + "auxiliary_loss_mlp": 0.01040138, + "balance_loss_clip": 1.04028046, + "balance_loss_mlp": 1.02391338, + "epoch": 0.960981181987615, + "flos": 21069114082560.0, + "grad_norm": 2.5715458384790435, + "language_loss": 0.89323598, + "learning_rate": 1.5903653943362126e-08, + "loss": 0.91491771, + "num_input_tokens_seen": 172517900, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.16229248, + "step": 7992, + "time_per_iteration": 3.7159266471862793 + }, + { + "auxiliary_loss_clip": 0.01123162, + "auxiliary_loss_mlp": 0.01038578, + "balance_loss_clip": 1.03985727, + "balance_loss_mlp": 1.02379584, + "epoch": 0.9611014248782541, + "flos": 17823256554240.0, + "grad_norm": 2.011605930567893, + "language_loss": 0.76903152, + "learning_rate": 1.580576404886802e-08, + "loss": 0.79064894, + "num_input_tokens_seen": 172536430, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.14794922, + "step": 7993, + "time_per_iteration": 2.8870906829833984 + }, + { + "auxiliary_loss_clip": 0.01120303, + "auxiliary_loss_mlp": 0.01037441, + "balance_loss_clip": 1.03948808, + "balance_loss_mlp": 1.02360046, + "epoch": 0.9612216677688932, + "flos": 19354631040000.0, + "grad_norm": 1.923856196591274, + "language_loss": 0.79930985, + "learning_rate": 1.570817515506162e-08, + "loss": 0.82088733, + "num_input_tokens_seen": 172555120, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.1383667, + "step": 7994, + "time_per_iteration": 3.718695878982544 + }, + { + "auxiliary_loss_clip": 0.01123452, + "auxiliary_loss_mlp": 0.0103602, + "balance_loss_clip": 1.0403583, + "balance_loss_mlp": 1.0215714, + "epoch": 0.9613419106595322, + "flos": 15808739207040.0, + "grad_norm": 1.9399556197148105, + "language_loss": 0.81399453, + "learning_rate": 1.561088727674753e-08, + "loss": 0.83558929, + "num_input_tokens_seen": 172569330, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.14440918, + "step": 7995, + "time_per_iteration": 2.8185606002807617 + }, + { + "auxiliary_loss_clip": 0.01130907, + "auxiliary_loss_mlp": 0.01039883, + "balance_loss_clip": 1.04060185, + "balance_loss_mlp": 1.02251387, + "epoch": 0.9614621535501714, + "flos": 25702488126720.0, + "grad_norm": 2.18384503255086, + "language_loss": 0.70947534, + "learning_rate": 1.551390042868417e-08, + "loss": 0.73118317, + "num_input_tokens_seen": 172591100, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.17376709, + "step": 7996, + "time_per_iteration": 2.833496332168579 + }, + { + "auxiliary_loss_clip": 0.01121179, + "auxiliary_loss_mlp": 0.01042114, + "balance_loss_clip": 1.03837204, + "balance_loss_mlp": 1.02637243, + "epoch": 0.9615823964408104, + "flos": 17819054663040.0, + "grad_norm": 1.933164959074061, + "language_loss": 0.70576721, + "learning_rate": 1.5417214625584207e-08, + "loss": 0.72740018, + "num_input_tokens_seen": 172608755, + "router_z_loss_clip": 0.828125, + "router_z_loss_mlp": 0.1572876, + "step": 7997, + "time_per_iteration": 2.723245143890381 + }, + { + "auxiliary_loss_clip": 0.01126153, + "auxiliary_loss_mlp": 0.01039023, + "balance_loss_clip": 1.04054284, + "balance_loss_mlp": 1.02364504, + "epoch": 0.9617026393314495, + "flos": 20190020624640.0, + "grad_norm": 2.958616910666909, + "language_loss": 0.85412908, + "learning_rate": 1.5320829882114806e-08, + "loss": 0.87578082, + "num_input_tokens_seen": 172626830, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.15380859, + "step": 7998, + "time_per_iteration": 2.8565642833709717 + }, + { + "auxiliary_loss_clip": 0.01123061, + "auxiliary_loss_mlp": 0.01035761, + "balance_loss_clip": 1.03747892, + "balance_loss_mlp": 1.020365, + "epoch": 0.9618228822220887, + "flos": 20267013427200.0, + "grad_norm": 1.8688159329889495, + "language_loss": 0.78970146, + "learning_rate": 1.5224746212897378e-08, + "loss": 0.81128967, + "num_input_tokens_seen": 172646125, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15386963, + "step": 7999, + "time_per_iteration": 2.845991373062134 + }, + { + "auxiliary_loss_clip": 0.01123285, + "auxiliary_loss_mlp": 0.01030577, + "balance_loss_clip": 1.04089308, + "balance_loss_mlp": 1.01522279, + "epoch": 0.9619431251127277, + "flos": 21031300039680.0, + "grad_norm": 1.6685422689204974, + "language_loss": 0.77066052, + "learning_rate": 1.512896363250804e-08, + "loss": 0.79219913, + "num_input_tokens_seen": 172666235, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.15356445, + "step": 8000, + "time_per_iteration": 2.965097188949585 + }, + { + "auxiliary_loss_clip": 0.01124347, + "auxiliary_loss_mlp": 0.01033598, + "balance_loss_clip": 1.04056108, + "balance_loss_mlp": 1.02000785, + "epoch": 0.9620633680033668, + "flos": 22382654538240.0, + "grad_norm": 2.425083465907396, + "language_loss": 0.75183833, + "learning_rate": 1.503348215547673e-08, + "loss": 0.77341777, + "num_input_tokens_seen": 172687325, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.1361084, + "step": 8001, + "time_per_iteration": 2.8011770248413086 + }, + { + "auxiliary_loss_clip": 0.0112021, + "auxiliary_loss_mlp": 0.01041496, + "balance_loss_clip": 1.03767908, + "balance_loss_mlp": 1.02728009, + "epoch": 0.962183610894006, + "flos": 18471730740480.0, + "grad_norm": 1.8150561125546485, + "language_loss": 0.80887747, + "learning_rate": 1.4938301796288078e-08, + "loss": 0.83049452, + "num_input_tokens_seen": 172703895, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.14202881, + "step": 8002, + "time_per_iteration": 2.817833423614502 + }, + { + "auxiliary_loss_clip": 0.01125218, + "auxiliary_loss_mlp": 0.01041643, + "balance_loss_clip": 1.040326, + "balance_loss_mlp": 1.02508473, + "epoch": 0.962303853784645, + "flos": 18435245500800.0, + "grad_norm": 2.14606904211085, + "language_loss": 0.81675041, + "learning_rate": 1.4843422569380537e-08, + "loss": 0.83841896, + "num_input_tokens_seen": 172720650, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.16564941, + "step": 8003, + "time_per_iteration": 2.911639451980591 + }, + { + "auxiliary_loss_clip": 0.01122198, + "auxiliary_loss_mlp": 0.01040176, + "balance_loss_clip": 1.0383867, + "balance_loss_mlp": 1.02537012, + "epoch": 0.9624240966752841, + "flos": 26391074826240.0, + "grad_norm": 2.1090756716809844, + "language_loss": 0.82645607, + "learning_rate": 1.4748844489147483e-08, + "loss": 0.84807986, + "num_input_tokens_seen": 172737640, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14807129, + "step": 8004, + "time_per_iteration": 2.864851236343384 + }, + { + "auxiliary_loss_clip": 0.01120676, + "auxiliary_loss_mlp": 0.01041982, + "balance_loss_clip": 1.03639555, + "balance_loss_mlp": 1.02704549, + "epoch": 0.9625443395659231, + "flos": 14647675985280.0, + "grad_norm": 1.8143645728465836, + "language_loss": 0.70805341, + "learning_rate": 1.4654567569936326e-08, + "loss": 0.72967994, + "num_input_tokens_seen": 172755215, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.14953613, + "step": 8005, + "time_per_iteration": 2.8296029567718506 + }, + { + "auxiliary_loss_clip": 0.01128195, + "auxiliary_loss_mlp": 0.01041701, + "balance_loss_clip": 1.04252303, + "balance_loss_mlp": 1.02702641, + "epoch": 0.9626645824565623, + "flos": 18367626147840.0, + "grad_norm": 1.8138584249620886, + "language_loss": 0.82979292, + "learning_rate": 1.456059182604874e-08, + "loss": 0.85149193, + "num_input_tokens_seen": 172774020, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.14672852, + "step": 8006, + "time_per_iteration": 2.8297159671783447 + }, + { + "auxiliary_loss_clip": 0.01126847, + "auxiliary_loss_mlp": 0.01033688, + "balance_loss_clip": 1.04114485, + "balance_loss_mlp": 1.01834023, + "epoch": 0.9627848253472013, + "flos": 16580424021120.0, + "grad_norm": 2.88028524212721, + "language_loss": 0.76626939, + "learning_rate": 1.4466917271740653e-08, + "loss": 0.78787476, + "num_input_tokens_seen": 172792220, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15338135, + "step": 8007, + "time_per_iteration": 2.8507907390594482 + }, + { + "auxiliary_loss_clip": 0.01128796, + "auxiliary_loss_mlp": 0.01036932, + "balance_loss_clip": 1.04001892, + "balance_loss_mlp": 1.02074373, + "epoch": 0.9629050682378404, + "flos": 20886867452160.0, + "grad_norm": 1.905412793831109, + "language_loss": 0.67560297, + "learning_rate": 1.4373543921222697e-08, + "loss": 0.69726026, + "num_input_tokens_seen": 172811805, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.1619873, + "step": 8008, + "time_per_iteration": 2.805950164794922 + }, + { + "auxiliary_loss_clip": 0.01125265, + "auxiliary_loss_mlp": 0.0103855, + "balance_loss_clip": 1.04005468, + "balance_loss_mlp": 1.02253985, + "epoch": 0.9630253111284796, + "flos": 17019252478080.0, + "grad_norm": 2.050881173618672, + "language_loss": 0.78062141, + "learning_rate": 1.428047178865932e-08, + "loss": 0.8022595, + "num_input_tokens_seen": 172828595, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.16015625, + "step": 8009, + "time_per_iteration": 2.8231801986694336 + }, + { + "auxiliary_loss_clip": 0.01122572, + "auxiliary_loss_mlp": 0.0103964, + "balance_loss_clip": 1.03790069, + "balance_loss_mlp": 1.02457213, + "epoch": 0.9631455540191186, + "flos": 20338942412160.0, + "grad_norm": 1.8649873144317128, + "language_loss": 0.74428713, + "learning_rate": 1.4187700888169451e-08, + "loss": 0.76590931, + "num_input_tokens_seen": 172847770, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15057373, + "step": 8010, + "time_per_iteration": 2.8597400188446045 + }, + { + "auxiliary_loss_clip": 0.01031772, + "auxiliary_loss_mlp": 0.01002669, + "balance_loss_clip": 1.00410533, + "balance_loss_mlp": 1.00142336, + "epoch": 0.9632657969097577, + "flos": 65956700033280.0, + "grad_norm": 0.753570546130334, + "language_loss": 0.57000828, + "learning_rate": 1.40952312338265e-08, + "loss": 0.59035265, + "num_input_tokens_seen": 172912415, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01245117, + "step": 8011, + "time_per_iteration": 3.4184956550598145 + }, + { + "auxiliary_loss_clip": 0.01125708, + "auxiliary_loss_mlp": 0.01035375, + "balance_loss_clip": 1.03902411, + "balance_loss_mlp": 1.02039027, + "epoch": 0.9633860398003968, + "flos": 44419523823360.0, + "grad_norm": 1.7448375534827776, + "language_loss": 0.68938875, + "learning_rate": 1.4003062839657909e-08, + "loss": 0.71099949, + "num_input_tokens_seen": 172934895, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.14978027, + "step": 8012, + "time_per_iteration": 3.0212998390197754 + }, + { + "auxiliary_loss_clip": 0.01121031, + "auxiliary_loss_mlp": 0.01040196, + "balance_loss_clip": 1.03741455, + "balance_loss_mlp": 1.02611148, + "epoch": 0.9635062826910359, + "flos": 24827704300800.0, + "grad_norm": 1.5416375103448592, + "language_loss": 0.79993945, + "learning_rate": 1.391119571964583e-08, + "loss": 0.82155174, + "num_input_tokens_seen": 172955835, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14093018, + "step": 8013, + "time_per_iteration": 2.8698244094848633 + }, + { + "auxiliary_loss_clip": 0.01121787, + "auxiliary_loss_mlp": 0.01033827, + "balance_loss_clip": 1.03885818, + "balance_loss_mlp": 1.01963568, + "epoch": 0.9636265255816749, + "flos": 15961360095360.0, + "grad_norm": 1.9972266450110212, + "language_loss": 0.72536314, + "learning_rate": 1.3819629887726225e-08, + "loss": 0.74691927, + "num_input_tokens_seen": 172973925, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.14202881, + "step": 8014, + "time_per_iteration": 3.5230250358581543 + }, + { + "auxiliary_loss_clip": 0.0112612, + "auxiliary_loss_mlp": 0.01038487, + "balance_loss_clip": 1.03934574, + "balance_loss_mlp": 1.02236986, + "epoch": 0.9637467684723141, + "flos": 22601781457920.0, + "grad_norm": 1.849639832283371, + "language_loss": 0.7612263, + "learning_rate": 1.3728365357789317e-08, + "loss": 0.78287238, + "num_input_tokens_seen": 172993290, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.16113281, + "step": 8015, + "time_per_iteration": 2.881591558456421 + }, + { + "auxiliary_loss_clip": 0.01125254, + "auxiliary_loss_mlp": 0.01037439, + "balance_loss_clip": 1.04041696, + "balance_loss_mlp": 1.02203083, + "epoch": 0.9638670113629532, + "flos": 17565812801280.0, + "grad_norm": 2.492679431628529, + "language_loss": 0.76359445, + "learning_rate": 1.3637402143680254e-08, + "loss": 0.78522134, + "num_input_tokens_seen": 173008190, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.15405273, + "step": 8016, + "time_per_iteration": 2.844583749771118 + }, + { + "auxiliary_loss_clip": 0.01031966, + "auxiliary_loss_mlp": 0.01000416, + "balance_loss_clip": 1.00435734, + "balance_loss_mlp": 0.99910492, + "epoch": 0.9639872542535922, + "flos": 55072139379840.0, + "grad_norm": 0.7332201986777804, + "language_loss": 0.55043453, + "learning_rate": 1.3546740259197998e-08, + "loss": 0.57075834, + "num_input_tokens_seen": 173061000, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01312256, + "step": 8017, + "time_per_iteration": 3.285832166671753 + }, + { + "auxiliary_loss_clip": 0.01128823, + "auxiliary_loss_mlp": 0.01032864, + "balance_loss_clip": 1.04094517, + "balance_loss_mlp": 1.01753402, + "epoch": 0.9641074971442314, + "flos": 24134484746880.0, + "grad_norm": 3.6515754118611623, + "language_loss": 0.70101213, + "learning_rate": 1.3456379718095989e-08, + "loss": 0.72262895, + "num_input_tokens_seen": 173081415, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.15332031, + "step": 8018, + "time_per_iteration": 3.636384963989258 + }, + { + "auxiliary_loss_clip": 0.01031957, + "auxiliary_loss_mlp": 0.01002327, + "balance_loss_clip": 1.00427258, + "balance_loss_mlp": 1.00104511, + "epoch": 0.9642277400348704, + "flos": 66747416077440.0, + "grad_norm": 0.9013468029605461, + "language_loss": 0.62010425, + "learning_rate": 1.3366320534081487e-08, + "loss": 0.64044708, + "num_input_tokens_seen": 173144095, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01281738, + "step": 8019, + "time_per_iteration": 3.283385992050171 + }, + { + "auxiliary_loss_clip": 0.01125894, + "auxiliary_loss_mlp": 0.01036869, + "balance_loss_clip": 1.04202902, + "balance_loss_mlp": 1.02155066, + "epoch": 0.9643479829255095, + "flos": 30920272450560.0, + "grad_norm": 4.415686679881405, + "language_loss": 0.7569648, + "learning_rate": 1.3276562720816675e-08, + "loss": 0.77859235, + "num_input_tokens_seen": 173165605, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.15307617, + "step": 8020, + "time_per_iteration": 2.9158780574798584 + }, + { + "auxiliary_loss_clip": 0.0112596, + "auxiliary_loss_mlp": 0.01040925, + "balance_loss_clip": 1.03965533, + "balance_loss_mlp": 1.0259105, + "epoch": 0.9644682258161487, + "flos": 20048245643520.0, + "grad_norm": 2.2063098574696185, + "language_loss": 0.8246907, + "learning_rate": 1.3187106291917549e-08, + "loss": 0.84635955, + "num_input_tokens_seen": 173182595, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.15020752, + "step": 8021, + "time_per_iteration": 3.6285581588745117 + }, + { + "auxiliary_loss_clip": 0.01122011, + "auxiliary_loss_mlp": 0.01035419, + "balance_loss_clip": 1.03856206, + "balance_loss_mlp": 1.0209229, + "epoch": 0.9645884687067877, + "flos": 21178713456000.0, + "grad_norm": 1.784036121419025, + "language_loss": 0.70451576, + "learning_rate": 1.309795126095503e-08, + "loss": 0.72609007, + "num_input_tokens_seen": 173200895, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.14483643, + "step": 8022, + "time_per_iteration": 2.8426856994628906 + }, + { + "auxiliary_loss_clip": 0.01120922, + "auxiliary_loss_mlp": 0.01035535, + "balance_loss_clip": 1.03775716, + "balance_loss_mlp": 1.02075291, + "epoch": 0.9647087115974268, + "flos": 18945967029120.0, + "grad_norm": 5.1047185451030685, + "language_loss": 0.80990332, + "learning_rate": 1.3009097641453192e-08, + "loss": 0.83146787, + "num_input_tokens_seen": 173218745, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.14794922, + "step": 8023, + "time_per_iteration": 3.7227089405059814 + }, + { + "auxiliary_loss_clip": 0.01125311, + "auxiliary_loss_mlp": 0.0103474, + "balance_loss_clip": 1.03898144, + "balance_loss_mlp": 1.02007687, + "epoch": 0.9648289544880659, + "flos": 16545088016640.0, + "grad_norm": 1.5360398224507243, + "language_loss": 0.75756699, + "learning_rate": 1.2920545446891474e-08, + "loss": 0.77916741, + "num_input_tokens_seen": 173235465, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.14685059, + "step": 8024, + "time_per_iteration": 2.756197690963745 + }, + { + "auxiliary_loss_clip": 0.01126303, + "auxiliary_loss_mlp": 0.01040877, + "balance_loss_clip": 1.0403682, + "balance_loss_mlp": 1.02445018, + "epoch": 0.964949197378705, + "flos": 24057527857920.0, + "grad_norm": 1.6370169535807653, + "language_loss": 0.70540351, + "learning_rate": 1.2832294690703127e-08, + "loss": 0.72707528, + "num_input_tokens_seen": 173254440, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.16430664, + "step": 8025, + "time_per_iteration": 2.7985973358154297 + }, + { + "auxiliary_loss_clip": 0.01124051, + "auxiliary_loss_mlp": 0.01034679, + "balance_loss_clip": 1.0391748, + "balance_loss_mlp": 1.01955104, + "epoch": 0.965069440269344, + "flos": 23365565280000.0, + "grad_norm": 1.9604963904217825, + "language_loss": 0.77710867, + "learning_rate": 1.2744345386275668e-08, + "loss": 0.79869598, + "num_input_tokens_seen": 173273980, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15124512, + "step": 8026, + "time_per_iteration": 2.8304126262664795 + }, + { + "auxiliary_loss_clip": 0.01127498, + "auxiliary_loss_mlp": 0.01043552, + "balance_loss_clip": 1.04148543, + "balance_loss_mlp": 1.02880633, + "epoch": 0.9651896831599832, + "flos": 25374875155200.0, + "grad_norm": 1.6100909004519577, + "language_loss": 0.78346103, + "learning_rate": 1.265669754695109e-08, + "loss": 0.80517149, + "num_input_tokens_seen": 173293550, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.14733887, + "step": 8027, + "time_per_iteration": 2.8602583408355713 + }, + { + "auxiliary_loss_clip": 0.01125193, + "auxiliary_loss_mlp": 0.0104036, + "balance_loss_clip": 1.03896403, + "balance_loss_mlp": 1.02436221, + "epoch": 0.9653099260506223, + "flos": 22272875596800.0, + "grad_norm": 2.092910003271333, + "language_loss": 0.82124329, + "learning_rate": 1.2569351186025201e-08, + "loss": 0.84289885, + "num_input_tokens_seen": 173312005, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.16003418, + "step": 8028, + "time_per_iteration": 2.8796000480651855 + }, + { + "auxiliary_loss_clip": 0.01120103, + "auxiliary_loss_mlp": 0.01033037, + "balance_loss_clip": 1.03773808, + "balance_loss_mlp": 1.01885116, + "epoch": 0.9654301689412613, + "flos": 26760847386240.0, + "grad_norm": 1.4905059345913234, + "language_loss": 0.75395417, + "learning_rate": 1.2482306316748737e-08, + "loss": 0.77548558, + "num_input_tokens_seen": 173332450, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.14196777, + "step": 8029, + "time_per_iteration": 2.885387897491455 + }, + { + "auxiliary_loss_clip": 0.01124142, + "auxiliary_loss_mlp": 0.01034008, + "balance_loss_clip": 1.03756452, + "balance_loss_mlp": 1.01918447, + "epoch": 0.9655504118319005, + "flos": 17412689122560.0, + "grad_norm": 1.844443634494172, + "language_loss": 0.78643292, + "learning_rate": 1.2395562952326021e-08, + "loss": 0.80801439, + "num_input_tokens_seen": 173349610, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.14813232, + "step": 8030, + "time_per_iteration": 2.8599092960357666 + }, + { + "auxiliary_loss_clip": 0.01132122, + "auxiliary_loss_mlp": 0.01045126, + "balance_loss_clip": 1.04203022, + "balance_loss_mlp": 1.02831793, + "epoch": 0.9656706547225395, + "flos": 22126970551680.0, + "grad_norm": 2.1175210272597007, + "language_loss": 0.81272119, + "learning_rate": 1.2309121105916309e-08, + "loss": 0.83449364, + "num_input_tokens_seen": 173367900, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.16796875, + "step": 8031, + "time_per_iteration": 2.8140130043029785 + }, + { + "auxiliary_loss_clip": 0.01125123, + "auxiliary_loss_mlp": 0.01036773, + "balance_loss_clip": 1.03920484, + "balance_loss_mlp": 1.02150226, + "epoch": 0.9657908976131786, + "flos": 37049289926400.0, + "grad_norm": 1.7263666179341968, + "language_loss": 0.69062036, + "learning_rate": 1.222298079063222e-08, + "loss": 0.71223927, + "num_input_tokens_seen": 173389040, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15270996, + "step": 8032, + "time_per_iteration": 2.9170944690704346 + }, + { + "auxiliary_loss_clip": 0.01122774, + "auxiliary_loss_mlp": 0.01034768, + "balance_loss_clip": 1.03924441, + "balance_loss_mlp": 1.02011728, + "epoch": 0.9659111405038178, + "flos": 24389809597440.0, + "grad_norm": 2.0647479414738936, + "language_loss": 0.73150063, + "learning_rate": 1.2137142019541524e-08, + "loss": 0.75307602, + "num_input_tokens_seen": 173407595, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.14648438, + "step": 8033, + "time_per_iteration": 2.8474926948547363 + }, + { + "auxiliary_loss_clip": 0.01123468, + "auxiliary_loss_mlp": 0.01040149, + "balance_loss_clip": 1.03947377, + "balance_loss_mlp": 1.02486682, + "epoch": 0.9660313833944568, + "flos": 25009412227200.0, + "grad_norm": 1.9068375887289568, + "language_loss": 0.73096132, + "learning_rate": 1.2051604805666027e-08, + "loss": 0.75259757, + "num_input_tokens_seen": 173424720, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.15283203, + "step": 8034, + "time_per_iteration": 2.7906277179718018 + }, + { + "auxiliary_loss_clip": 0.01123183, + "auxiliary_loss_mlp": 0.01035015, + "balance_loss_clip": 1.03934765, + "balance_loss_mlp": 1.0195775, + "epoch": 0.9661516262850959, + "flos": 11801575895040.0, + "grad_norm": 2.118815349996112, + "language_loss": 0.78083217, + "learning_rate": 1.196636916198135e-08, + "loss": 0.80241418, + "num_input_tokens_seen": 173442260, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.15454102, + "step": 8035, + "time_per_iteration": 2.7900187969207764 + }, + { + "auxiliary_loss_clip": 0.01128183, + "auxiliary_loss_mlp": 0.01033067, + "balance_loss_clip": 1.04115093, + "balance_loss_mlp": 1.01820111, + "epoch": 0.9662718691757349, + "flos": 20047778766720.0, + "grad_norm": 1.8594192488715298, + "language_loss": 0.7676574, + "learning_rate": 1.1881435101418036e-08, + "loss": 0.78926992, + "num_input_tokens_seen": 173461675, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.14855957, + "step": 8036, + "time_per_iteration": 2.7644104957580566 + }, + { + "auxiliary_loss_clip": 0.01031438, + "auxiliary_loss_mlp": 0.01000188, + "balance_loss_clip": 1.00387788, + "balance_loss_mlp": 0.99888915, + "epoch": 0.9663921120663741, + "flos": 68027703517440.0, + "grad_norm": 0.7331376223721073, + "language_loss": 0.65517116, + "learning_rate": 1.1796802636860003e-08, + "loss": 0.67548746, + "num_input_tokens_seen": 173530205, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01300049, + "step": 8037, + "time_per_iteration": 3.304659366607666 + }, + { + "auxiliary_loss_clip": 0.01125385, + "auxiliary_loss_mlp": 0.01043539, + "balance_loss_clip": 1.03966022, + "balance_loss_mlp": 1.02764857, + "epoch": 0.9665123549570132, + "flos": 26322916769280.0, + "grad_norm": 2.0515770757671135, + "language_loss": 0.73459446, + "learning_rate": 1.1712471781146316e-08, + "loss": 0.7562837, + "num_input_tokens_seen": 173549540, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15905762, + "step": 8038, + "time_per_iteration": 2.80246639251709 + }, + { + "auxiliary_loss_clip": 0.01120854, + "auxiliary_loss_mlp": 0.0103381, + "balance_loss_clip": 1.03727353, + "balance_loss_mlp": 1.01950455, + "epoch": 0.9666325978476522, + "flos": 43941121557120.0, + "grad_norm": 2.3381606848823955, + "language_loss": 0.66856933, + "learning_rate": 1.1628442547069628e-08, + "loss": 0.69011593, + "num_input_tokens_seen": 173571740, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.14300537, + "step": 8039, + "time_per_iteration": 2.9069418907165527 + }, + { + "auxiliary_loss_clip": 0.0112406, + "auxiliary_loss_mlp": 0.01041708, + "balance_loss_clip": 1.03816926, + "balance_loss_mlp": 1.02543616, + "epoch": 0.9667528407382914, + "flos": 21543422198400.0, + "grad_norm": 3.9132225058824583, + "language_loss": 0.76875079, + "learning_rate": 1.1544714947377521e-08, + "loss": 0.79040849, + "num_input_tokens_seen": 173589425, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.16259766, + "step": 8040, + "time_per_iteration": 2.802786350250244 + }, + { + "auxiliary_loss_clip": 0.01127621, + "auxiliary_loss_mlp": 0.01041092, + "balance_loss_clip": 1.04125643, + "balance_loss_mlp": 1.02536273, + "epoch": 0.9668730836289304, + "flos": 23878585278720.0, + "grad_norm": 2.994958031278598, + "language_loss": 0.7037698, + "learning_rate": 1.1461288994770945e-08, + "loss": 0.72545695, + "num_input_tokens_seen": 173608500, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.15722656, + "step": 8041, + "time_per_iteration": 2.7287418842315674 + }, + { + "auxiliary_loss_clip": 0.01126748, + "auxiliary_loss_mlp": 0.01041829, + "balance_loss_clip": 1.03897834, + "balance_loss_mlp": 1.02632046, + "epoch": 0.9669933265195695, + "flos": 28293011971200.0, + "grad_norm": 2.016501453180631, + "language_loss": 0.77369189, + "learning_rate": 1.1378164701906002e-08, + "loss": 0.79537773, + "num_input_tokens_seen": 173630265, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.1550293, + "step": 8042, + "time_per_iteration": 2.8497121334075928 + }, + { + "auxiliary_loss_clip": 0.01130527, + "auxiliary_loss_mlp": 0.01038099, + "balance_loss_clip": 1.0432601, + "balance_loss_mlp": 1.02237558, + "epoch": 0.9671135694102087, + "flos": 22454763091200.0, + "grad_norm": 1.6922692015850134, + "language_loss": 0.66462779, + "learning_rate": 1.1295342081392156e-08, + "loss": 0.68631405, + "num_input_tokens_seen": 173649625, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.15716553, + "step": 8043, + "time_per_iteration": 3.5960278511047363 + }, + { + "auxiliary_loss_clip": 0.01119518, + "auxiliary_loss_mlp": 0.0103997, + "balance_loss_clip": 1.03710938, + "balance_loss_mlp": 1.02556956, + "epoch": 0.9672338123008477, + "flos": 20155941596160.0, + "grad_norm": 1.5312068364986153, + "language_loss": 0.69423717, + "learning_rate": 1.1212821145793804e-08, + "loss": 0.71583205, + "num_input_tokens_seen": 173669240, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.14398193, + "step": 8044, + "time_per_iteration": 2.9952056407928467 + }, + { + "auxiliary_loss_clip": 0.01123914, + "auxiliary_loss_mlp": 0.01041167, + "balance_loss_clip": 1.04046559, + "balance_loss_mlp": 1.02622986, + "epoch": 0.9673540551914868, + "flos": 16977487939200.0, + "grad_norm": 2.1244489574522323, + "language_loss": 0.7897312, + "learning_rate": 1.1130601907629156e-08, + "loss": 0.81138194, + "num_input_tokens_seen": 173686970, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.14953613, + "step": 8045, + "time_per_iteration": 2.8490395545959473 + }, + { + "auxiliary_loss_clip": 0.0103169, + "auxiliary_loss_mlp": 0.0100265, + "balance_loss_clip": 1.0041852, + "balance_loss_mlp": 1.00134468, + "epoch": 0.9674742980821259, + "flos": 61892903952000.0, + "grad_norm": 0.8297507157536045, + "language_loss": 0.64762431, + "learning_rate": 1.1048684379370899e-08, + "loss": 0.66796774, + "num_input_tokens_seen": 173747655, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01306152, + "step": 8046, + "time_per_iteration": 4.0537190437316895 + }, + { + "auxiliary_loss_clip": 0.01123106, + "auxiliary_loss_mlp": 0.0103298, + "balance_loss_clip": 1.04122925, + "balance_loss_mlp": 1.01803684, + "epoch": 0.967594540972765, + "flos": 18697824898560.0, + "grad_norm": 1.9043630639590114, + "language_loss": 0.74525416, + "learning_rate": 1.0967068573445759e-08, + "loss": 0.76681501, + "num_input_tokens_seen": 173765140, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.14935303, + "step": 8047, + "time_per_iteration": 2.8447282314300537 + }, + { + "auxiliary_loss_clip": 0.01126417, + "auxiliary_loss_mlp": 0.01034844, + "balance_loss_clip": 1.0391165, + "balance_loss_mlp": 1.01979995, + "epoch": 0.967714783863404, + "flos": 20777411733120.0, + "grad_norm": 2.595520080709252, + "language_loss": 0.64876974, + "learning_rate": 1.0885754502234945e-08, + "loss": 0.67038238, + "num_input_tokens_seen": 173784800, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.1505127, + "step": 8048, + "time_per_iteration": 2.9204254150390625 + }, + { + "auxiliary_loss_clip": 0.01122506, + "auxiliary_loss_mlp": 0.01033381, + "balance_loss_clip": 1.03873348, + "balance_loss_mlp": 1.01899219, + "epoch": 0.9678350267540432, + "flos": 23185473465600.0, + "grad_norm": 1.683620662831669, + "language_loss": 0.78281075, + "learning_rate": 1.08047421780737e-08, + "loss": 0.80436963, + "num_input_tokens_seen": 173803990, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.14404297, + "step": 8049, + "time_per_iteration": 2.804372787475586 + }, + { + "auxiliary_loss_clip": 0.01125249, + "auxiliary_loss_mlp": 0.01037266, + "balance_loss_clip": 1.03895032, + "balance_loss_mlp": 1.02122617, + "epoch": 0.9679552696446823, + "flos": 21726063878400.0, + "grad_norm": 2.002441133451849, + "language_loss": 0.73899806, + "learning_rate": 1.0724031613251305e-08, + "loss": 0.76062322, + "num_input_tokens_seen": 173821890, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.16021729, + "step": 8050, + "time_per_iteration": 3.6029534339904785 + }, + { + "auxiliary_loss_clip": 0.01128931, + "auxiliary_loss_mlp": 0.0103964, + "balance_loss_clip": 1.04123306, + "balance_loss_mlp": 1.02368999, + "epoch": 0.9680755125353213, + "flos": 26869046129280.0, + "grad_norm": 2.4063440868832795, + "language_loss": 0.66941893, + "learning_rate": 1.0643622820011744e-08, + "loss": 0.69110465, + "num_input_tokens_seen": 173842945, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.1595459, + "step": 8051, + "time_per_iteration": 2.879647970199585 + }, + { + "auxiliary_loss_clip": 0.01127037, + "auxiliary_loss_mlp": 0.01039451, + "balance_loss_clip": 1.03976858, + "balance_loss_mlp": 1.02276158, + "epoch": 0.9681957554259605, + "flos": 28325008010880.0, + "grad_norm": 2.223233385891318, + "language_loss": 0.67984879, + "learning_rate": 1.0563515810552814e-08, + "loss": 0.70151371, + "num_input_tokens_seen": 173859915, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.16687012, + "step": 8052, + "time_per_iteration": 3.7896721363067627 + }, + { + "auxiliary_loss_clip": 0.01127275, + "auxiliary_loss_mlp": 0.01039455, + "balance_loss_clip": 1.04259825, + "balance_loss_mlp": 1.0237906, + "epoch": 0.9683159983165995, + "flos": 20557674282240.0, + "grad_norm": 1.532575219041402, + "language_loss": 0.73424375, + "learning_rate": 1.0483710597026795e-08, + "loss": 0.75591105, + "num_input_tokens_seen": 173879775, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15661621, + "step": 8053, + "time_per_iteration": 2.9525439739227295 + }, + { + "auxiliary_loss_clip": 0.01120771, + "auxiliary_loss_mlp": 0.01039341, + "balance_loss_clip": 1.03972888, + "balance_loss_mlp": 1.02433288, + "epoch": 0.9684362412072386, + "flos": 24207958016640.0, + "grad_norm": 2.2663463338891083, + "language_loss": 0.74057144, + "learning_rate": 1.0404207191540227e-08, + "loss": 0.76217252, + "num_input_tokens_seen": 173900230, + "router_z_loss_clip": 0.80957031, + "router_z_loss_mlp": 0.15014648, + "step": 8054, + "time_per_iteration": 2.958104372024536 + }, + { + "auxiliary_loss_clip": 0.01122879, + "auxiliary_loss_mlp": 0.01033816, + "balance_loss_clip": 1.03807986, + "balance_loss_mlp": 1.0194509, + "epoch": 0.9685564840978778, + "flos": 22346241125760.0, + "grad_norm": 1.9757576743345848, + "language_loss": 0.74773329, + "learning_rate": 1.0325005606153236e-08, + "loss": 0.76930028, + "num_input_tokens_seen": 173919690, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.14367676, + "step": 8055, + "time_per_iteration": 2.8738458156585693 + }, + { + "auxiliary_loss_clip": 0.0112394, + "auxiliary_loss_mlp": 0.01037301, + "balance_loss_clip": 1.03864551, + "balance_loss_mlp": 1.02150559, + "epoch": 0.9686767269885168, + "flos": 14386389477120.0, + "grad_norm": 2.6101795395986436, + "language_loss": 0.79287136, + "learning_rate": 1.0246105852881104e-08, + "loss": 0.81448376, + "num_input_tokens_seen": 173934790, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.15808105, + "step": 8056, + "time_per_iteration": 2.9136850833892822 + }, + { + "auxiliary_loss_clip": 0.01128512, + "auxiliary_loss_mlp": 0.01038899, + "balance_loss_clip": 1.04041874, + "balance_loss_mlp": 1.02218628, + "epoch": 0.9687969698791559, + "flos": 21287630471040.0, + "grad_norm": 1.819211998823647, + "language_loss": 0.79046881, + "learning_rate": 1.0167507943692476e-08, + "loss": 0.81214297, + "num_input_tokens_seen": 173953875, + "router_z_loss_clip": 0.88085938, + "router_z_loss_mlp": 0.16711426, + "step": 8057, + "time_per_iteration": 2.832763671875 + }, + { + "auxiliary_loss_clip": 0.01125811, + "auxiliary_loss_mlp": 0.01041, + "balance_loss_clip": 1.04108536, + "balance_loss_mlp": 1.025527, + "epoch": 0.968917212769795, + "flos": 19828328624640.0, + "grad_norm": 1.9937176379422368, + "language_loss": 0.7138899, + "learning_rate": 1.008921189051093e-08, + "loss": 0.73555803, + "num_input_tokens_seen": 173971220, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15490723, + "step": 8058, + "time_per_iteration": 2.855790138244629 + }, + { + "auxiliary_loss_clip": 0.01126664, + "auxiliary_loss_mlp": 0.01040649, + "balance_loss_clip": 1.04119372, + "balance_loss_mlp": 1.02567673, + "epoch": 0.9690374556604341, + "flos": 21681749473920.0, + "grad_norm": 2.0413684874747764, + "language_loss": 0.77439868, + "learning_rate": 1.0011217705213848e-08, + "loss": 0.79607189, + "num_input_tokens_seen": 173989095, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.14978027, + "step": 8059, + "time_per_iteration": 2.799830675125122 + }, + { + "auxiliary_loss_clip": 0.01122233, + "auxiliary_loss_mlp": 0.01037306, + "balance_loss_clip": 1.0387969, + "balance_loss_mlp": 1.02248836, + "epoch": 0.9691576985510731, + "flos": 32635437851520.0, + "grad_norm": 2.2173201771401803, + "language_loss": 0.74572486, + "learning_rate": 9.933525399632658e-09, + "loss": 0.76732028, + "num_input_tokens_seen": 174007330, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.14819336, + "step": 8060, + "time_per_iteration": 2.936023712158203 + }, + { + "auxiliary_loss_clip": 0.01124446, + "auxiliary_loss_mlp": 0.0104005, + "balance_loss_clip": 1.03756285, + "balance_loss_mlp": 1.02365851, + "epoch": 0.9692779414417123, + "flos": 35663174040960.0, + "grad_norm": 2.0071663399042246, + "language_loss": 0.65129912, + "learning_rate": 9.856134985553488e-09, + "loss": 0.67294401, + "num_input_tokens_seen": 174027055, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.1640625, + "step": 8061, + "time_per_iteration": 2.9655447006225586 + }, + { + "auxiliary_loss_clip": 0.01122512, + "auxiliary_loss_mlp": 0.0103366, + "balance_loss_clip": 1.03828979, + "balance_loss_mlp": 1.01964128, + "epoch": 0.9693981843323514, + "flos": 28366952117760.0, + "grad_norm": 2.2295782721850306, + "language_loss": 0.7354455, + "learning_rate": 9.77904647471628e-09, + "loss": 0.75700724, + "num_input_tokens_seen": 174050235, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.14031982, + "step": 8062, + "time_per_iteration": 2.869539976119995 + }, + { + "auxiliary_loss_clip": 0.01123569, + "auxiliary_loss_mlp": 0.01042696, + "balance_loss_clip": 1.03886104, + "balance_loss_mlp": 1.02790761, + "epoch": 0.9695184272229904, + "flos": 23622865378560.0, + "grad_norm": 1.446593457936534, + "language_loss": 0.73784292, + "learning_rate": 9.702259878815454e-09, + "loss": 0.75950557, + "num_input_tokens_seen": 174070560, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.14776611, + "step": 8063, + "time_per_iteration": 2.8918230533599854 + }, + { + "auxiliary_loss_clip": 0.01128724, + "auxiliary_loss_mlp": 0.01043775, + "balance_loss_clip": 1.04083049, + "balance_loss_mlp": 1.02647734, + "epoch": 0.9696386701136296, + "flos": 23294677789440.0, + "grad_norm": 2.2907571493082197, + "language_loss": 0.74846458, + "learning_rate": 9.625775209499254e-09, + "loss": 0.77018958, + "num_input_tokens_seen": 174090565, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.17297363, + "step": 8064, + "time_per_iteration": 2.798147439956665 + }, + { + "auxiliary_loss_clip": 0.01124169, + "auxiliary_loss_mlp": 0.01032343, + "balance_loss_clip": 1.04037714, + "balance_loss_mlp": 1.01726878, + "epoch": 0.9697589130042686, + "flos": 15121876360320.0, + "grad_norm": 1.9967922694015225, + "language_loss": 0.7386865, + "learning_rate": 9.549592478370172e-09, + "loss": 0.76025158, + "num_input_tokens_seen": 174108745, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.15063477, + "step": 8065, + "time_per_iteration": 2.882874011993408 + }, + { + "auxiliary_loss_clip": 0.01125905, + "auxiliary_loss_mlp": 0.01042448, + "balance_loss_clip": 1.04057431, + "balance_loss_mlp": 1.02835774, + "epoch": 0.9698791558949077, + "flos": 18879532824960.0, + "grad_norm": 1.6525836974975285, + "language_loss": 0.79455572, + "learning_rate": 9.473711696985632e-09, + "loss": 0.81623924, + "num_input_tokens_seen": 174128075, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.14086914, + "step": 8066, + "time_per_iteration": 2.7952237129211426 + }, + { + "auxiliary_loss_clip": 0.01124927, + "auxiliary_loss_mlp": 0.01044208, + "balance_loss_clip": 1.03827548, + "balance_loss_mlp": 1.0281862, + "epoch": 0.9699993987855468, + "flos": 17931455297280.0, + "grad_norm": 2.3774585599731375, + "language_loss": 0.75989139, + "learning_rate": 9.398132876856201e-09, + "loss": 0.78158271, + "num_input_tokens_seen": 174147040, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.16015625, + "step": 8067, + "time_per_iteration": 2.8377685546875 + }, + { + "auxiliary_loss_clip": 0.01031753, + "auxiliary_loss_mlp": 0.01000729, + "balance_loss_clip": 1.00419164, + "balance_loss_mlp": 0.99947131, + "epoch": 0.9701196416761859, + "flos": 67182186297600.0, + "grad_norm": 0.7742921224684902, + "language_loss": 0.6083461, + "learning_rate": 9.322856029447379e-09, + "loss": 0.62867093, + "num_input_tokens_seen": 174208225, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01257324, + "step": 8068, + "time_per_iteration": 3.215027093887329 + }, + { + "auxiliary_loss_clip": 0.01120569, + "auxiliary_loss_mlp": 0.01039692, + "balance_loss_clip": 1.03933632, + "balance_loss_mlp": 1.02464819, + "epoch": 0.970239884566825, + "flos": 24277804012800.0, + "grad_norm": 2.127971151250718, + "language_loss": 0.80553287, + "learning_rate": 9.247881166178695e-09, + "loss": 0.8271355, + "num_input_tokens_seen": 174226935, + "router_z_loss_clip": 0.81347656, + "router_z_loss_mlp": 0.15039062, + "step": 8069, + "time_per_iteration": 2.807417631149292 + }, + { + "auxiliary_loss_clip": 0.01125514, + "auxiliary_loss_mlp": 0.01036081, + "balance_loss_clip": 1.04019356, + "balance_loss_mlp": 1.02083468, + "epoch": 0.970360127457464, + "flos": 25301689194240.0, + "grad_norm": 2.306374216381604, + "language_loss": 0.76402891, + "learning_rate": 9.173208298423274e-09, + "loss": 0.78564489, + "num_input_tokens_seen": 174248140, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.15246582, + "step": 8070, + "time_per_iteration": 2.8774068355560303 + }, + { + "auxiliary_loss_clip": 0.01123243, + "auxiliary_loss_mlp": 0.01039049, + "balance_loss_clip": 1.03739572, + "balance_loss_mlp": 1.02376032, + "epoch": 0.9704803703481032, + "flos": 29572473398400.0, + "grad_norm": 1.6664370293245347, + "language_loss": 0.76115072, + "learning_rate": 9.09883743750961e-09, + "loss": 0.78277361, + "num_input_tokens_seen": 174271030, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.15289307, + "step": 8071, + "time_per_iteration": 2.822556972503662 + }, + { + "auxiliary_loss_clip": 0.01125616, + "auxiliary_loss_mlp": 0.01036289, + "balance_loss_clip": 1.04099655, + "balance_loss_mlp": 1.02166188, + "epoch": 0.9706006132387422, + "flos": 17380046638080.0, + "grad_norm": 1.8730077728940695, + "language_loss": 0.83915156, + "learning_rate": 9.024768594719124e-09, + "loss": 0.86077058, + "num_input_tokens_seen": 174289410, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.14624023, + "step": 8072, + "time_per_iteration": 3.54929518699646 + }, + { + "auxiliary_loss_clip": 0.01124731, + "auxiliary_loss_mlp": 0.01035778, + "balance_loss_clip": 1.03941751, + "balance_loss_mlp": 1.01969659, + "epoch": 0.9707208561293813, + "flos": 18186421011840.0, + "grad_norm": 2.019004340644314, + "language_loss": 0.72223479, + "learning_rate": 8.95100178128816e-09, + "loss": 0.74383986, + "num_input_tokens_seen": 174308550, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.16088867, + "step": 8073, + "time_per_iteration": 2.9316563606262207 + }, + { + "auxiliary_loss_clip": 0.01126629, + "auxiliary_loss_mlp": 0.01040796, + "balance_loss_clip": 1.04035234, + "balance_loss_mlp": 1.02500045, + "epoch": 0.9708410990200205, + "flos": 31248388212480.0, + "grad_norm": 2.9250519886473696, + "language_loss": 0.70203757, + "learning_rate": 8.877537008407321e-09, + "loss": 0.72371185, + "num_input_tokens_seen": 174328600, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15795898, + "step": 8074, + "time_per_iteration": 2.864043951034546 + }, + { + "auxiliary_loss_clip": 0.01124055, + "auxiliary_loss_mlp": 0.01043316, + "balance_loss_clip": 1.04002345, + "balance_loss_mlp": 1.02886772, + "epoch": 0.9709613419106595, + "flos": 30554450386560.0, + "grad_norm": 1.6196596685470672, + "language_loss": 0.68648398, + "learning_rate": 8.804374287221028e-09, + "loss": 0.70815766, + "num_input_tokens_seen": 174349835, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.14453125, + "step": 8075, + "time_per_iteration": 3.70354962348938 + }, + { + "auxiliary_loss_clip": 0.01122893, + "auxiliary_loss_mlp": 0.01035502, + "balance_loss_clip": 1.03987479, + "balance_loss_mlp": 1.02048182, + "epoch": 0.9710815848012986, + "flos": 23730166281600.0, + "grad_norm": 1.7483126905137634, + "language_loss": 0.84591508, + "learning_rate": 8.731513628827958e-09, + "loss": 0.86749899, + "num_input_tokens_seen": 174369200, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.15026855, + "step": 8076, + "time_per_iteration": 2.85225510597229 + }, + { + "auxiliary_loss_clip": 0.01128369, + "auxiliary_loss_mlp": 0.01034624, + "balance_loss_clip": 1.04087985, + "balance_loss_mlp": 1.01928794, + "epoch": 0.9712018276919377, + "flos": 23761875012480.0, + "grad_norm": 1.9472093547088127, + "language_loss": 0.82989681, + "learning_rate": 8.658955044280825e-09, + "loss": 0.85152674, + "num_input_tokens_seen": 174388125, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.15332031, + "step": 8077, + "time_per_iteration": 2.898037910461426 + }, + { + "auxiliary_loss_clip": 0.01123866, + "auxiliary_loss_mlp": 0.01038428, + "balance_loss_clip": 1.03933167, + "balance_loss_mlp": 1.024189, + "epoch": 0.9713220705825768, + "flos": 23330983461120.0, + "grad_norm": 1.537701470674167, + "language_loss": 0.77294803, + "learning_rate": 8.586698544587268e-09, + "loss": 0.79457098, + "num_input_tokens_seen": 174409735, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.14239502, + "step": 8078, + "time_per_iteration": 2.888598680496216 + }, + { + "auxiliary_loss_clip": 0.01129003, + "auxiliary_loss_mlp": 0.01036955, + "balance_loss_clip": 1.04317451, + "balance_loss_mlp": 1.02098143, + "epoch": 0.9714423134732159, + "flos": 22200946611840.0, + "grad_norm": 1.9971701350917404, + "language_loss": 0.7435869, + "learning_rate": 8.514744140707853e-09, + "loss": 0.76524651, + "num_input_tokens_seen": 174428875, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.15979004, + "step": 8079, + "time_per_iteration": 3.5900042057037354 + }, + { + "auxiliary_loss_clip": 0.01122653, + "auxiliary_loss_mlp": 0.01035301, + "balance_loss_clip": 1.03943443, + "balance_loss_mlp": 1.02144277, + "epoch": 0.971562556363855, + "flos": 20229917656320.0, + "grad_norm": 1.6669041733591232, + "language_loss": 0.7652207, + "learning_rate": 8.443091843558515e-09, + "loss": 0.78680027, + "num_input_tokens_seen": 174447960, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.1383667, + "step": 8080, + "time_per_iteration": 2.925114870071411 + }, + { + "auxiliary_loss_clip": 0.01125015, + "auxiliary_loss_mlp": 0.0104641, + "balance_loss_clip": 1.03922057, + "balance_loss_mlp": 1.03007841, + "epoch": 0.9716827992544941, + "flos": 24970197553920.0, + "grad_norm": 2.6746577629950785, + "language_loss": 0.64613879, + "learning_rate": 8.37174166400878e-09, + "loss": 0.667853, + "num_input_tokens_seen": 174463535, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.16326904, + "step": 8081, + "time_per_iteration": 3.724249839782715 + }, + { + "auxiliary_loss_clip": 0.01128699, + "auxiliary_loss_mlp": 0.01043505, + "balance_loss_clip": 1.04382551, + "balance_loss_mlp": 1.02878273, + "epoch": 0.9718030421451331, + "flos": 24681476033280.0, + "grad_norm": 1.8144483141553234, + "language_loss": 0.85052097, + "learning_rate": 8.300693612881992e-09, + "loss": 0.87224305, + "num_input_tokens_seen": 174483600, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.14733887, + "step": 8082, + "time_per_iteration": 2.804258346557617 + }, + { + "auxiliary_loss_clip": 0.01127573, + "auxiliary_loss_mlp": 0.01041672, + "balance_loss_clip": 1.04019594, + "balance_loss_mlp": 1.02601969, + "epoch": 0.9719232850357723, + "flos": 22090700793600.0, + "grad_norm": 2.064999594342141, + "language_loss": 0.81452852, + "learning_rate": 8.22994770095664e-09, + "loss": 0.83622098, + "num_input_tokens_seen": 174502175, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.15661621, + "step": 8083, + "time_per_iteration": 2.807511568069458 + }, + { + "auxiliary_loss_clip": 0.01128527, + "auxiliary_loss_mlp": 0.01038484, + "balance_loss_clip": 1.04203546, + "balance_loss_mlp": 1.02261722, + "epoch": 0.9720435279264114, + "flos": 23656908493440.0, + "grad_norm": 2.0779749467010813, + "language_loss": 0.75173718, + "learning_rate": 8.159503938964585e-09, + "loss": 0.77340734, + "num_input_tokens_seen": 174519495, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.15869141, + "step": 8084, + "time_per_iteration": 2.7823498249053955 + }, + { + "auxiliary_loss_clip": 0.01118506, + "auxiliary_loss_mlp": 0.01034531, + "balance_loss_clip": 1.03930748, + "balance_loss_mlp": 1.02054763, + "epoch": 0.9721637708170504, + "flos": 28365910623360.0, + "grad_norm": 2.124119212510937, + "language_loss": 0.70482904, + "learning_rate": 8.089362337592164e-09, + "loss": 0.72635937, + "num_input_tokens_seen": 174543120, + "router_z_loss_clip": 0.79296875, + "router_z_loss_mlp": 0.13970947, + "step": 8085, + "time_per_iteration": 2.8707993030548096 + }, + { + "auxiliary_loss_clip": 0.01123393, + "auxiliary_loss_mlp": 0.01034773, + "balance_loss_clip": 1.03955197, + "balance_loss_mlp": 1.02055168, + "epoch": 0.9722840137076896, + "flos": 29130807767040.0, + "grad_norm": 1.5207603719572447, + "language_loss": 0.72334969, + "learning_rate": 8.019522907479536e-09, + "loss": 0.74493134, + "num_input_tokens_seen": 174563480, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.14215088, + "step": 8086, + "time_per_iteration": 2.8811917304992676 + }, + { + "auxiliary_loss_clip": 0.01124165, + "auxiliary_loss_mlp": 0.010338, + "balance_loss_clip": 1.04074454, + "balance_loss_mlp": 1.01971531, + "epoch": 0.9724042565983286, + "flos": 19243954258560.0, + "grad_norm": 1.878049536951635, + "language_loss": 0.76976973, + "learning_rate": 7.949985659221558e-09, + "loss": 0.79134935, + "num_input_tokens_seen": 174580745, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.14080811, + "step": 8087, + "time_per_iteration": 2.7636547088623047 + }, + { + "auxiliary_loss_clip": 0.0112597, + "auxiliary_loss_mlp": 0.01039661, + "balance_loss_clip": 1.04095054, + "balance_loss_mlp": 1.02468884, + "epoch": 0.9725244994889677, + "flos": 23039676161280.0, + "grad_norm": 2.0165742767511743, + "language_loss": 0.79203832, + "learning_rate": 7.880750603366904e-09, + "loss": 0.8136946, + "num_input_tokens_seen": 174599615, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.1496582, + "step": 8088, + "time_per_iteration": 2.951399564743042 + }, + { + "auxiliary_loss_clip": 0.01130459, + "auxiliary_loss_mlp": 0.01043517, + "balance_loss_clip": 1.04082572, + "balance_loss_mlp": 1.02720904, + "epoch": 0.9726447423796069, + "flos": 23367468700800.0, + "grad_norm": 1.8074554847147888, + "language_loss": 0.79692876, + "learning_rate": 7.811817750418282e-09, + "loss": 0.81866854, + "num_input_tokens_seen": 174618375, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.16296387, + "step": 8089, + "time_per_iteration": 2.8309080600738525 + }, + { + "auxiliary_loss_clip": 0.01127192, + "auxiliary_loss_mlp": 0.0104059, + "balance_loss_clip": 1.04016662, + "balance_loss_mlp": 1.02422261, + "epoch": 0.9727649852702459, + "flos": 26541648639360.0, + "grad_norm": 1.549741463100741, + "language_loss": 0.80198252, + "learning_rate": 7.743187110833105e-09, + "loss": 0.82366031, + "num_input_tokens_seen": 174641135, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.16381836, + "step": 8090, + "time_per_iteration": 2.9737439155578613 + }, + { + "auxiliary_loss_clip": 0.01123443, + "auxiliary_loss_mlp": 0.01036272, + "balance_loss_clip": 1.03861272, + "balance_loss_mlp": 1.02171683, + "epoch": 0.972885228160885, + "flos": 20522338277760.0, + "grad_norm": 1.4644136775373917, + "language_loss": 0.80692554, + "learning_rate": 7.674858695022602e-09, + "loss": 0.82852262, + "num_input_tokens_seen": 174659490, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.14550781, + "step": 8091, + "time_per_iteration": 2.850266695022583 + }, + { + "auxiliary_loss_clip": 0.01128684, + "auxiliary_loss_mlp": 0.01038739, + "balance_loss_clip": 1.04069185, + "balance_loss_mlp": 1.02308667, + "epoch": 0.9730054710515241, + "flos": 17566064196480.0, + "grad_norm": 4.382851451855737, + "language_loss": 0.76245695, + "learning_rate": 7.606832513351591e-09, + "loss": 0.78413123, + "num_input_tokens_seen": 174677440, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.15649414, + "step": 8092, + "time_per_iteration": 2.85139536857605 + }, + { + "auxiliary_loss_clip": 0.01031694, + "auxiliary_loss_mlp": 0.00999868, + "balance_loss_clip": 1.00410199, + "balance_loss_mlp": 0.99854463, + "epoch": 0.9731257139421632, + "flos": 68972010117120.0, + "grad_norm": 0.8317128067407097, + "language_loss": 0.63963771, + "learning_rate": 7.539108576140264e-09, + "loss": 0.65995336, + "num_input_tokens_seen": 174741550, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01324463, + "step": 8093, + "time_per_iteration": 3.45468807220459 + }, + { + "auxiliary_loss_clip": 0.01119582, + "auxiliary_loss_mlp": 0.0103019, + "balance_loss_clip": 1.03824902, + "balance_loss_mlp": 1.01706457, + "epoch": 0.9732459568328022, + "flos": 18478841633280.0, + "grad_norm": 1.9415695353361926, + "language_loss": 0.70700556, + "learning_rate": 7.471686893661732e-09, + "loss": 0.72850323, + "num_input_tokens_seen": 174759845, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.13122559, + "step": 8094, + "time_per_iteration": 2.875845432281494 + }, + { + "auxiliary_loss_clip": 0.01122164, + "auxiliary_loss_mlp": 0.01041305, + "balance_loss_clip": 1.03923666, + "balance_loss_mlp": 1.02665973, + "epoch": 0.9733661997234414, + "flos": 20883886623360.0, + "grad_norm": 1.6815011167419704, + "language_loss": 0.64030147, + "learning_rate": 7.4045674761442636e-09, + "loss": 0.66193616, + "num_input_tokens_seen": 174777175, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.14654541, + "step": 8095, + "time_per_iteration": 2.928272008895874 + }, + { + "auxiliary_loss_clip": 0.01123446, + "auxiliary_loss_mlp": 0.01034555, + "balance_loss_clip": 1.03950417, + "balance_loss_mlp": 1.01944506, + "epoch": 0.9734864426140805, + "flos": 23766795175680.0, + "grad_norm": 1.8254700880795343, + "language_loss": 0.74481797, + "learning_rate": 7.337750333769488e-09, + "loss": 0.76639795, + "num_input_tokens_seen": 174796980, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.15087891, + "step": 8096, + "time_per_iteration": 2.864407539367676 + }, + { + "auxiliary_loss_clip": 0.01126806, + "auxiliary_loss_mlp": 0.01036354, + "balance_loss_clip": 1.03909469, + "balance_loss_mlp": 1.02104163, + "epoch": 0.9736066855047195, + "flos": 35042422176000.0, + "grad_norm": 1.7512955502392353, + "language_loss": 0.72960031, + "learning_rate": 7.2712354766737425e-09, + "loss": 0.75123197, + "num_input_tokens_seen": 174817310, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.15313721, + "step": 8097, + "time_per_iteration": 2.9473862648010254 + }, + { + "auxiliary_loss_clip": 0.01123138, + "auxiliary_loss_mlp": 0.01045729, + "balance_loss_clip": 1.03998351, + "balance_loss_mlp": 1.03049433, + "epoch": 0.9737269283953586, + "flos": 20410620001920.0, + "grad_norm": 1.7179744554457073, + "language_loss": 0.80861568, + "learning_rate": 7.2050229149469565e-09, + "loss": 0.83030432, + "num_input_tokens_seen": 174837320, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.15234375, + "step": 8098, + "time_per_iteration": 2.8757591247558594 + }, + { + "auxiliary_loss_clip": 0.01123206, + "auxiliary_loss_mlp": 0.01041417, + "balance_loss_clip": 1.03827775, + "balance_loss_mlp": 1.02642083, + "epoch": 0.9738471712859977, + "flos": 28911680847360.0, + "grad_norm": 1.6149395361109145, + "language_loss": 0.63059098, + "learning_rate": 7.139112658633984e-09, + "loss": 0.65223718, + "num_input_tokens_seen": 174857470, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.15002441, + "step": 8099, + "time_per_iteration": 2.7942497730255127 + }, + { + "auxiliary_loss_clip": 0.01122504, + "auxiliary_loss_mlp": 0.01034602, + "balance_loss_clip": 1.03803134, + "balance_loss_mlp": 1.01879525, + "epoch": 0.9739674141766368, + "flos": 27782326356480.0, + "grad_norm": 1.9657317233736178, + "language_loss": 0.70481384, + "learning_rate": 7.073504717733048e-09, + "loss": 0.72638488, + "num_input_tokens_seen": 174877035, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15820312, + "step": 8100, + "time_per_iteration": 3.542754650115967 + }, + { + "auxiliary_loss_clip": 0.01031642, + "auxiliary_loss_mlp": 0.01000436, + "balance_loss_clip": 1.00407314, + "balance_loss_mlp": 0.99912435, + "epoch": 0.9740876570672758, + "flos": 68863057188480.0, + "grad_norm": 0.735938803459866, + "language_loss": 0.57204664, + "learning_rate": 7.008199102196855e-09, + "loss": 0.59236741, + "num_input_tokens_seen": 174938460, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01312256, + "step": 8101, + "time_per_iteration": 3.2637720108032227 + }, + { + "auxiliary_loss_clip": 0.01032073, + "auxiliary_loss_mlp": 0.01000187, + "balance_loss_clip": 1.00437093, + "balance_loss_mlp": 0.99889958, + "epoch": 0.974207899957915, + "flos": 58236622646400.0, + "grad_norm": 0.7950376648884873, + "language_loss": 0.58972096, + "learning_rate": 6.9431958219321464e-09, + "loss": 0.61004353, + "num_input_tokens_seen": 174994625, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01287842, + "step": 8102, + "time_per_iteration": 3.2750346660614014 + }, + { + "auxiliary_loss_clip": 0.01126306, + "auxiliary_loss_mlp": 0.01036462, + "balance_loss_clip": 1.04067349, + "balance_loss_mlp": 1.02117336, + "epoch": 0.9743281428485541, + "flos": 22600057605120.0, + "grad_norm": 1.5322717496235891, + "language_loss": 0.77683163, + "learning_rate": 6.878494886800146e-09, + "loss": 0.79845929, + "num_input_tokens_seen": 175015400, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15283203, + "step": 8103, + "time_per_iteration": 2.8817219734191895 + }, + { + "auxiliary_loss_clip": 0.01127933, + "auxiliary_loss_mlp": 0.01032605, + "balance_loss_clip": 1.04161143, + "balance_loss_mlp": 1.01811481, + "epoch": 0.9744483857391931, + "flos": 20008815488640.0, + "grad_norm": 1.7178870566656699, + "language_loss": 0.76156467, + "learning_rate": 6.814096306615669e-09, + "loss": 0.78317004, + "num_input_tokens_seen": 175033540, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.14483643, + "step": 8104, + "time_per_iteration": 3.5976696014404297 + }, + { + "auxiliary_loss_clip": 0.01124065, + "auxiliary_loss_mlp": 0.01047201, + "balance_loss_clip": 1.03823841, + "balance_loss_mlp": 1.03184652, + "epoch": 0.9745686286298323, + "flos": 17675268520320.0, + "grad_norm": 2.1250247326993366, + "language_loss": 0.65482891, + "learning_rate": 6.750000091148011e-09, + "loss": 0.67654157, + "num_input_tokens_seen": 175050835, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.15338135, + "step": 8105, + "time_per_iteration": 2.8649256229400635 + }, + { + "auxiliary_loss_clip": 0.01125157, + "auxiliary_loss_mlp": 0.0103485, + "balance_loss_clip": 1.04053974, + "balance_loss_mlp": 1.0196569, + "epoch": 0.9746888715204713, + "flos": 29460252332160.0, + "grad_norm": 1.983777836951069, + "language_loss": 0.72412288, + "learning_rate": 6.686206250120729e-09, + "loss": 0.74572301, + "num_input_tokens_seen": 175072330, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15197754, + "step": 8106, + "time_per_iteration": 2.8880226612091064 + }, + { + "auxiliary_loss_clip": 0.0112472, + "auxiliary_loss_mlp": 0.01039454, + "balance_loss_clip": 1.03787446, + "balance_loss_mlp": 1.02471352, + "epoch": 0.9748091144111104, + "flos": 18479308510080.0, + "grad_norm": 2.1958831767766367, + "language_loss": 0.74834728, + "learning_rate": 6.622714793210749e-09, + "loss": 0.76998901, + "num_input_tokens_seen": 175091250, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.1472168, + "step": 8107, + "time_per_iteration": 3.645979881286621 + }, + { + "auxiliary_loss_clip": 0.01123712, + "auxiliary_loss_mlp": 0.0103666, + "balance_loss_clip": 1.03988266, + "balance_loss_mlp": 1.02099037, + "epoch": 0.9749293573017496, + "flos": 20665154753280.0, + "grad_norm": 1.7503928155787536, + "language_loss": 0.78534865, + "learning_rate": 6.559525730050364e-09, + "loss": 0.80695242, + "num_input_tokens_seen": 175111350, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.15673828, + "step": 8108, + "time_per_iteration": 2.8609936237335205 + }, + { + "auxiliary_loss_clip": 0.01123916, + "auxiliary_loss_mlp": 0.01039742, + "balance_loss_clip": 1.03992581, + "balance_loss_mlp": 1.02417374, + "epoch": 0.9750496001923886, + "flos": 18478590238080.0, + "grad_norm": 1.837993802313715, + "language_loss": 0.75962883, + "learning_rate": 6.496639070224574e-09, + "loss": 0.78126544, + "num_input_tokens_seen": 175129835, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.15551758, + "step": 8109, + "time_per_iteration": 2.8302175998687744 + }, + { + "auxiliary_loss_clip": 0.01122355, + "auxiliary_loss_mlp": 0.01034738, + "balance_loss_clip": 1.03886139, + "balance_loss_mlp": 1.01906157, + "epoch": 0.9751698430830277, + "flos": 19572967860480.0, + "grad_norm": 2.1403107998555675, + "language_loss": 0.83664715, + "learning_rate": 6.4340548232739714e-09, + "loss": 0.85821807, + "num_input_tokens_seen": 175146035, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.15686035, + "step": 8110, + "time_per_iteration": 3.7334699630737305 + }, + { + "auxiliary_loss_clip": 0.01125034, + "auxiliary_loss_mlp": 0.01037187, + "balance_loss_clip": 1.04025126, + "balance_loss_mlp": 1.02251244, + "epoch": 0.9752900859736668, + "flos": 23550325862400.0, + "grad_norm": 1.670273914574363, + "language_loss": 0.79111814, + "learning_rate": 6.371772998692071e-09, + "loss": 0.81274033, + "num_input_tokens_seen": 175165290, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14672852, + "step": 8111, + "time_per_iteration": 2.854025363922119 + }, + { + "auxiliary_loss_clip": 0.01123169, + "auxiliary_loss_mlp": 0.01031689, + "balance_loss_clip": 1.03758383, + "balance_loss_mlp": 1.01640058, + "epoch": 0.9754103288643059, + "flos": 20303211358080.0, + "grad_norm": 2.6271044791925373, + "language_loss": 0.65623915, + "learning_rate": 6.309793605927094e-09, + "loss": 0.67778772, + "num_input_tokens_seen": 175183610, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15301514, + "step": 8112, + "time_per_iteration": 2.8071792125701904 + }, + { + "auxiliary_loss_clip": 0.01120353, + "auxiliary_loss_mlp": 0.01040483, + "balance_loss_clip": 1.037148, + "balance_loss_mlp": 1.02570748, + "epoch": 0.975530571754945, + "flos": 19350680544000.0, + "grad_norm": 1.765224283565562, + "language_loss": 0.8023268, + "learning_rate": 6.248116654381297e-09, + "loss": 0.82393515, + "num_input_tokens_seen": 175202080, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.14782715, + "step": 8113, + "time_per_iteration": 2.910114049911499 + }, + { + "auxiliary_loss_clip": 0.01120289, + "auxiliary_loss_mlp": 0.01035423, + "balance_loss_clip": 1.03812528, + "balance_loss_mlp": 1.02098703, + "epoch": 0.9756508146455841, + "flos": 23583399310080.0, + "grad_norm": 1.915600815994186, + "language_loss": 0.72949165, + "learning_rate": 6.186742153410751e-09, + "loss": 0.7510488, + "num_input_tokens_seen": 175221575, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.14428711, + "step": 8114, + "time_per_iteration": 2.8472654819488525 + }, + { + "auxiliary_loss_clip": 0.01131331, + "auxiliary_loss_mlp": 0.01043812, + "balance_loss_clip": 1.04230022, + "balance_loss_mlp": 1.02771831, + "epoch": 0.9757710575362232, + "flos": 22966921163520.0, + "grad_norm": 1.898379460667057, + "language_loss": 0.87473869, + "learning_rate": 6.125670112326453e-09, + "loss": 0.8964901, + "num_input_tokens_seen": 175240835, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.1607666, + "step": 8115, + "time_per_iteration": 2.8966457843780518 + }, + { + "auxiliary_loss_clip": 0.01122641, + "auxiliary_loss_mlp": 0.01035367, + "balance_loss_clip": 1.03926444, + "balance_loss_mlp": 1.01970267, + "epoch": 0.9758913004268622, + "flos": 27966009530880.0, + "grad_norm": 1.646471374973321, + "language_loss": 0.70215935, + "learning_rate": 6.064900540392548e-09, + "loss": 0.72373945, + "num_input_tokens_seen": 175262930, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.15673828, + "step": 8116, + "time_per_iteration": 2.8868653774261475 + }, + { + "auxiliary_loss_clip": 0.01117248, + "auxiliary_loss_mlp": 0.01031987, + "balance_loss_clip": 1.0383122, + "balance_loss_mlp": 1.01789069, + "epoch": 0.9760115433175014, + "flos": 22200156512640.0, + "grad_norm": 1.8419372032686914, + "language_loss": 0.78686273, + "learning_rate": 6.0044334468278835e-09, + "loss": 0.80835509, + "num_input_tokens_seen": 175282275, + "router_z_loss_clip": 0.7890625, + "router_z_loss_mlp": 0.14086914, + "step": 8117, + "time_per_iteration": 2.870997428894043 + }, + { + "auxiliary_loss_clip": 0.01124016, + "auxiliary_loss_mlp": 0.01035515, + "balance_loss_clip": 1.03969431, + "balance_loss_mlp": 1.01977897, + "epoch": 0.9761317862081405, + "flos": 26250736389120.0, + "grad_norm": 1.6186537794987814, + "language_loss": 0.71539879, + "learning_rate": 5.944268840805345e-09, + "loss": 0.73699403, + "num_input_tokens_seen": 175303020, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.15740967, + "step": 8118, + "time_per_iteration": 2.958224058151245 + }, + { + "auxiliary_loss_clip": 0.01119157, + "auxiliary_loss_mlp": 0.01033798, + "balance_loss_clip": 1.03861654, + "balance_loss_mlp": 1.02012432, + "epoch": 0.9762520290987795, + "flos": 26575440359040.0, + "grad_norm": 1.9092985868517756, + "language_loss": 0.6425364, + "learning_rate": 5.88440673145163e-09, + "loss": 0.66406596, + "num_input_tokens_seen": 175324070, + "router_z_loss_clip": 0.80566406, + "router_z_loss_mlp": 0.13671875, + "step": 8119, + "time_per_iteration": 2.764827251434326 + }, + { + "auxiliary_loss_clip": 0.01124744, + "auxiliary_loss_mlp": 0.010421, + "balance_loss_clip": 1.04195929, + "balance_loss_mlp": 1.02644765, + "epoch": 0.9763722719894187, + "flos": 18005036307840.0, + "grad_norm": 2.1018031647550117, + "language_loss": 0.82554829, + "learning_rate": 5.824847127848142e-09, + "loss": 0.84721673, + "num_input_tokens_seen": 175342595, + "router_z_loss_clip": 0.82763672, + "router_z_loss_mlp": 0.15649414, + "step": 8120, + "time_per_iteration": 2.830386161804199 + }, + { + "auxiliary_loss_clip": 0.01125426, + "auxiliary_loss_mlp": 0.01037826, + "balance_loss_clip": 1.04065275, + "balance_loss_mlp": 1.02282321, + "epoch": 0.9764925148800577, + "flos": 22455660931200.0, + "grad_norm": 1.9082888528604292, + "language_loss": 0.78574872, + "learning_rate": 5.765590039029433e-09, + "loss": 0.80738121, + "num_input_tokens_seen": 175361915, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15008545, + "step": 8121, + "time_per_iteration": 2.7818095684051514 + }, + { + "auxiliary_loss_clip": 0.01122527, + "auxiliary_loss_mlp": 0.01034657, + "balance_loss_clip": 1.03950214, + "balance_loss_mlp": 1.01979709, + "epoch": 0.9766127577706968, + "flos": 36757084786560.0, + "grad_norm": 1.5186249751656375, + "language_loss": 0.71220285, + "learning_rate": 5.706635473985422e-09, + "loss": 0.73377472, + "num_input_tokens_seen": 175385785, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.14862061, + "step": 8122, + "time_per_iteration": 2.8615314960479736 + }, + { + "auxiliary_loss_clip": 0.01123785, + "auxiliary_loss_mlp": 0.01037664, + "balance_loss_clip": 1.03898203, + "balance_loss_mlp": 1.02240515, + "epoch": 0.976733000661336, + "flos": 22309971367680.0, + "grad_norm": 2.353427542110456, + "language_loss": 0.85388529, + "learning_rate": 5.6479834416591764e-09, + "loss": 0.87549978, + "num_input_tokens_seen": 175405145, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.15258789, + "step": 8123, + "time_per_iteration": 2.8113853931427 + }, + { + "auxiliary_loss_clip": 0.0112796, + "auxiliary_loss_mlp": 0.01036101, + "balance_loss_clip": 1.04013085, + "balance_loss_mlp": 1.02019811, + "epoch": 0.976853243551975, + "flos": 25810938264960.0, + "grad_norm": 2.0801290322447272, + "language_loss": 0.68419045, + "learning_rate": 5.589633950947803e-09, + "loss": 0.70583105, + "num_input_tokens_seen": 175422645, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.15893555, + "step": 8124, + "time_per_iteration": 2.789691209793091 + }, + { + "auxiliary_loss_clip": 0.01129356, + "auxiliary_loss_mlp": 0.01042232, + "balance_loss_clip": 1.04201281, + "balance_loss_mlp": 1.02544701, + "epoch": 0.9769734864426141, + "flos": 21397445326080.0, + "grad_norm": 2.1438302064561885, + "language_loss": 0.7010231, + "learning_rate": 5.5315870107035535e-09, + "loss": 0.72273898, + "num_input_tokens_seen": 175440695, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.16784668, + "step": 8125, + "time_per_iteration": 2.848480224609375 + }, + { + "auxiliary_loss_clip": 0.01125883, + "auxiliary_loss_mlp": 0.01036326, + "balance_loss_clip": 1.0403173, + "balance_loss_mlp": 1.02041173, + "epoch": 0.9770937293332532, + "flos": 13990977584640.0, + "grad_norm": 1.6856986896619481, + "language_loss": 0.78875363, + "learning_rate": 5.473842629731607e-09, + "loss": 0.81037575, + "num_input_tokens_seen": 175459195, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15917969, + "step": 8126, + "time_per_iteration": 2.8714120388031006 + }, + { + "auxiliary_loss_clip": 0.01129646, + "auxiliary_loss_mlp": 0.01033047, + "balance_loss_clip": 1.04130518, + "balance_loss_mlp": 1.01734734, + "epoch": 0.9772139722238923, + "flos": 17931994001280.0, + "grad_norm": 2.1875784299336174, + "language_loss": 0.78059626, + "learning_rate": 5.416400816792066e-09, + "loss": 0.80222321, + "num_input_tokens_seen": 175476710, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.15710449, + "step": 8127, + "time_per_iteration": 2.8705079555511475 + }, + { + "auxiliary_loss_clip": 0.01121185, + "auxiliary_loss_mlp": 0.01035399, + "balance_loss_clip": 1.03826582, + "balance_loss_mlp": 1.0202589, + "epoch": 0.9773342151145313, + "flos": 20446171488000.0, + "grad_norm": 2.240046104863113, + "language_loss": 0.78412271, + "learning_rate": 5.359261580598407e-09, + "loss": 0.80568862, + "num_input_tokens_seen": 175492550, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.15136719, + "step": 8128, + "time_per_iteration": 2.7839672565460205 + }, + { + "auxiliary_loss_clip": 0.01129631, + "auxiliary_loss_mlp": 0.01039858, + "balance_loss_clip": 1.04096413, + "balance_loss_mlp": 1.02343106, + "epoch": 0.9774544580051704, + "flos": 11837306949120.0, + "grad_norm": 2.2567919769392506, + "language_loss": 0.78264642, + "learning_rate": 5.302424929819027e-09, + "loss": 0.80434132, + "num_input_tokens_seen": 175506560, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.16430664, + "step": 8129, + "time_per_iteration": 3.518850088119507 + }, + { + "auxiliary_loss_clip": 0.01126909, + "auxiliary_loss_mlp": 0.01037508, + "balance_loss_clip": 1.03858232, + "balance_loss_mlp": 1.02155781, + "epoch": 0.9775747008958096, + "flos": 13479932833920.0, + "grad_norm": 2.0324068481529514, + "language_loss": 0.73269087, + "learning_rate": 5.24589087307592e-09, + "loss": 0.75433505, + "num_input_tokens_seen": 175524180, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.1595459, + "step": 8130, + "time_per_iteration": 2.861058473587036 + }, + { + "auxiliary_loss_clip": 0.01125943, + "auxiliary_loss_mlp": 0.01038235, + "balance_loss_clip": 1.03954506, + "balance_loss_mlp": 1.02257085, + "epoch": 0.9776949437864486, + "flos": 59532314042880.0, + "grad_norm": 1.4147863331681307, + "language_loss": 0.65125686, + "learning_rate": 5.189659418944891e-09, + "loss": 0.67289865, + "num_input_tokens_seen": 175554355, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.15661621, + "step": 8131, + "time_per_iteration": 3.1612861156463623 + }, + { + "auxiliary_loss_clip": 0.01123434, + "auxiliary_loss_mlp": 0.01031934, + "balance_loss_clip": 1.04009473, + "balance_loss_mlp": 1.01631153, + "epoch": 0.9778151866770877, + "flos": 21178605715200.0, + "grad_norm": 2.137316733983066, + "language_loss": 0.78576374, + "learning_rate": 5.133730575956674e-09, + "loss": 0.80731738, + "num_input_tokens_seen": 175574025, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.15606689, + "step": 8132, + "time_per_iteration": 2.9221103191375732 + }, + { + "auxiliary_loss_clip": 0.01125974, + "auxiliary_loss_mlp": 0.01039353, + "balance_loss_clip": 1.04033637, + "balance_loss_mlp": 1.02416551, + "epoch": 0.9779354295677268, + "flos": 20886795624960.0, + "grad_norm": 1.8911531201889262, + "language_loss": 0.71893352, + "learning_rate": 5.0781043525953696e-09, + "loss": 0.74058676, + "num_input_tokens_seen": 175592090, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15197754, + "step": 8133, + "time_per_iteration": 3.6856725215911865 + }, + { + "auxiliary_loss_clip": 0.01124017, + "auxiliary_loss_mlp": 0.01037244, + "balance_loss_clip": 1.03946555, + "balance_loss_mlp": 1.02287984, + "epoch": 0.9780556724583659, + "flos": 23440618748160.0, + "grad_norm": 1.8890424349883819, + "language_loss": 0.73844445, + "learning_rate": 5.0227807572995605e-09, + "loss": 0.76005703, + "num_input_tokens_seen": 175614065, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.14373779, + "step": 8134, + "time_per_iteration": 2.831922769546509 + }, + { + "auxiliary_loss_clip": 0.01124038, + "auxiliary_loss_mlp": 0.01037763, + "balance_loss_clip": 1.03867936, + "balance_loss_mlp": 1.02351713, + "epoch": 0.9781759153490049, + "flos": 20923244951040.0, + "grad_norm": 2.070249165827375, + "language_loss": 0.67259729, + "learning_rate": 4.967759798461646e-09, + "loss": 0.6942153, + "num_input_tokens_seen": 175632410, + "router_z_loss_clip": 0.85302734, + "router_z_loss_mlp": 0.14257812, + "step": 8135, + "time_per_iteration": 2.818798780441284 + }, + { + "auxiliary_loss_clip": 0.01121954, + "auxiliary_loss_mlp": 0.01040162, + "balance_loss_clip": 1.04084194, + "balance_loss_mlp": 1.02608347, + "epoch": 0.9782961582396441, + "flos": 28293191539200.0, + "grad_norm": 1.8571813072127303, + "language_loss": 0.74971199, + "learning_rate": 4.913041484428282e-09, + "loss": 0.7713331, + "num_input_tokens_seen": 175652885, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.14080811, + "step": 8136, + "time_per_iteration": 3.6156961917877197 + }, + { + "auxiliary_loss_clip": 0.01120843, + "auxiliary_loss_mlp": 0.01035061, + "balance_loss_clip": 1.03887844, + "balance_loss_mlp": 1.02098274, + "epoch": 0.9784164011302832, + "flos": 25552955808000.0, + "grad_norm": 1.8601999054617815, + "language_loss": 0.74549222, + "learning_rate": 4.858625823500384e-09, + "loss": 0.76705122, + "num_input_tokens_seen": 175670585, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.14074707, + "step": 8137, + "time_per_iteration": 2.860118865966797 + }, + { + "auxiliary_loss_clip": 0.01124818, + "auxiliary_loss_mlp": 0.01038118, + "balance_loss_clip": 1.03945684, + "balance_loss_mlp": 1.02281165, + "epoch": 0.9785366440209222, + "flos": 29965945956480.0, + "grad_norm": 1.9069749979604014, + "language_loss": 0.73140043, + "learning_rate": 4.80451282393246e-09, + "loss": 0.75302982, + "num_input_tokens_seen": 175690570, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.15307617, + "step": 8138, + "time_per_iteration": 2.9420292377471924 + }, + { + "auxiliary_loss_clip": 0.01125575, + "auxiliary_loss_mlp": 0.01040319, + "balance_loss_clip": 1.03821874, + "balance_loss_mlp": 1.02391529, + "epoch": 0.9786568869115614, + "flos": 32343591847680.0, + "grad_norm": 2.6907661118203725, + "language_loss": 0.676108, + "learning_rate": 4.750702493933722e-09, + "loss": 0.69776696, + "num_input_tokens_seen": 175710455, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.16418457, + "step": 8139, + "time_per_iteration": 3.665942430496216 + }, + { + "auxiliary_loss_clip": 0.01125472, + "auxiliary_loss_mlp": 0.01040865, + "balance_loss_clip": 1.04145205, + "balance_loss_mlp": 1.02657199, + "epoch": 0.9787771298022004, + "flos": 23331414424320.0, + "grad_norm": 1.8342110197688364, + "language_loss": 0.84966248, + "learning_rate": 4.697194841666974e-09, + "loss": 0.87132585, + "num_input_tokens_seen": 175729380, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.14282227, + "step": 8140, + "time_per_iteration": 2.765307664871216 + }, + { + "auxiliary_loss_clip": 0.01129213, + "auxiliary_loss_mlp": 0.01043317, + "balance_loss_clip": 1.03903079, + "balance_loss_mlp": 1.02700865, + "epoch": 0.9788973726928395, + "flos": 21468548298240.0, + "grad_norm": 1.9237268090124784, + "language_loss": 0.82025027, + "learning_rate": 4.6439898752492764e-09, + "loss": 0.84197557, + "num_input_tokens_seen": 175749520, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.16296387, + "step": 8141, + "time_per_iteration": 2.7471821308135986 + }, + { + "auxiliary_loss_clip": 0.01031791, + "auxiliary_loss_mlp": 0.0100104, + "balance_loss_clip": 1.00406981, + "balance_loss_mlp": 0.9997465, + "epoch": 0.9790176155834787, + "flos": 68897459439360.0, + "grad_norm": 0.7468911921100413, + "language_loss": 0.63692033, + "learning_rate": 4.591087602751731e-09, + "loss": 0.65724868, + "num_input_tokens_seen": 175811380, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01293945, + "step": 8142, + "time_per_iteration": 3.337963342666626 + }, + { + "auxiliary_loss_clip": 0.0112221, + "auxiliary_loss_mlp": 0.01040598, + "balance_loss_clip": 1.03974688, + "balance_loss_mlp": 1.02652526, + "epoch": 0.9791378584741177, + "flos": 21430877909760.0, + "grad_norm": 1.676132944074245, + "language_loss": 0.71954393, + "learning_rate": 4.538488032199916e-09, + "loss": 0.74117202, + "num_input_tokens_seen": 175829480, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.14074707, + "step": 8143, + "time_per_iteration": 2.7858104705810547 + }, + { + "auxiliary_loss_clip": 0.01127838, + "auxiliary_loss_mlp": 0.01038196, + "balance_loss_clip": 1.03915405, + "balance_loss_mlp": 1.02168536, + "epoch": 0.9792581013647568, + "flos": 20153032594560.0, + "grad_norm": 1.8431642361072023, + "language_loss": 0.69150436, + "learning_rate": 4.486191171572784e-09, + "loss": 0.71316469, + "num_input_tokens_seen": 175846750, + "router_z_loss_clip": 0.88769531, + "router_z_loss_mlp": 0.16503906, + "step": 8144, + "time_per_iteration": 2.711397171020508 + }, + { + "auxiliary_loss_clip": 0.01124922, + "auxiliary_loss_mlp": 0.01037501, + "balance_loss_clip": 1.04157698, + "balance_loss_mlp": 1.02344632, + "epoch": 0.9793783442553959, + "flos": 23728191033600.0, + "grad_norm": 1.484083883393897, + "language_loss": 0.77659142, + "learning_rate": 4.434197028803766e-09, + "loss": 0.79821563, + "num_input_tokens_seen": 175865975, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.14050293, + "step": 8145, + "time_per_iteration": 2.7776901721954346 + }, + { + "auxiliary_loss_clip": 0.01125202, + "auxiliary_loss_mlp": 0.01044417, + "balance_loss_clip": 1.04018152, + "balance_loss_mlp": 1.02737021, + "epoch": 0.979498587146035, + "flos": 23038742407680.0, + "grad_norm": 2.0525101823129113, + "language_loss": 0.81984544, + "learning_rate": 4.3825056117805514e-09, + "loss": 0.84154165, + "num_input_tokens_seen": 175881860, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.17022705, + "step": 8146, + "time_per_iteration": 2.779609203338623 + }, + { + "auxiliary_loss_clip": 0.01123738, + "auxiliary_loss_mlp": 0.01046841, + "balance_loss_clip": 1.03916883, + "balance_loss_mlp": 1.03143883, + "epoch": 0.979618830036674, + "flos": 14318841951360.0, + "grad_norm": 2.7986195703110615, + "language_loss": 0.7942183, + "learning_rate": 4.331116928344425e-09, + "loss": 0.81592405, + "num_input_tokens_seen": 175898175, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.15405273, + "step": 8147, + "time_per_iteration": 2.9433557987213135 + }, + { + "auxiliary_loss_clip": 0.01124154, + "auxiliary_loss_mlp": 0.01034098, + "balance_loss_clip": 1.03828156, + "balance_loss_mlp": 1.0177902, + "epoch": 0.9797390729273132, + "flos": 16727514215040.0, + "grad_norm": 1.89337533004436, + "language_loss": 0.62663776, + "learning_rate": 4.28003098629115e-09, + "loss": 0.64822024, + "num_input_tokens_seen": 175914310, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.16296387, + "step": 8148, + "time_per_iteration": 2.9444124698638916 + }, + { + "auxiliary_loss_clip": 0.01124786, + "auxiliary_loss_mlp": 0.01035801, + "balance_loss_clip": 1.03853512, + "balance_loss_mlp": 1.02078056, + "epoch": 0.9798593158179523, + "flos": 24532661986560.0, + "grad_norm": 1.8982268518347574, + "language_loss": 0.78416693, + "learning_rate": 4.229247793370305e-09, + "loss": 0.80577278, + "num_input_tokens_seen": 175933435, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.15002441, + "step": 8149, + "time_per_iteration": 2.864550828933716 + }, + { + "auxiliary_loss_clip": 0.01127297, + "auxiliary_loss_mlp": 0.01037699, + "balance_loss_clip": 1.04017735, + "balance_loss_mlp": 1.0227623, + "epoch": 0.9799795587085913, + "flos": 27308808339840.0, + "grad_norm": 1.6040453721870778, + "language_loss": 0.70375717, + "learning_rate": 4.178767357285951e-09, + "loss": 0.72540712, + "num_input_tokens_seen": 175955065, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.14935303, + "step": 8150, + "time_per_iteration": 3.1104066371917725 + }, + { + "auxiliary_loss_clip": 0.01125333, + "auxiliary_loss_mlp": 0.01033865, + "balance_loss_clip": 1.03994656, + "balance_loss_mlp": 1.01736021, + "epoch": 0.9800998015992305, + "flos": 26286575184000.0, + "grad_norm": 1.9494873106967368, + "language_loss": 0.71535206, + "learning_rate": 4.128589685695516e-09, + "loss": 0.73694408, + "num_input_tokens_seen": 175975490, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.16491699, + "step": 8151, + "time_per_iteration": 2.902681589126587 + }, + { + "auxiliary_loss_clip": 0.01125299, + "auxiliary_loss_mlp": 0.01038063, + "balance_loss_clip": 1.04041374, + "balance_loss_mlp": 1.02162433, + "epoch": 0.9802200444898695, + "flos": 16723635546240.0, + "grad_norm": 1.8569535784088869, + "language_loss": 0.85138559, + "learning_rate": 4.078714786211135e-09, + "loss": 0.87301922, + "num_input_tokens_seen": 175991340, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.16442871, + "step": 8152, + "time_per_iteration": 2.8340649604797363 + }, + { + "auxiliary_loss_clip": 0.01119467, + "auxiliary_loss_mlp": 0.01036558, + "balance_loss_clip": 1.03770256, + "balance_loss_mlp": 1.0222832, + "epoch": 0.9803402873805086, + "flos": 24900459298560.0, + "grad_norm": 1.672535546746373, + "language_loss": 0.77000821, + "learning_rate": 4.029142666398977e-09, + "loss": 0.79156846, + "num_input_tokens_seen": 176011505, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.14276123, + "step": 8153, + "time_per_iteration": 2.846874952316284 + }, + { + "auxiliary_loss_clip": 0.01122665, + "auxiliary_loss_mlp": 0.01047968, + "balance_loss_clip": 1.03920174, + "balance_loss_mlp": 1.03287649, + "epoch": 0.9804605302711478, + "flos": 22564937082240.0, + "grad_norm": 2.3067892600197046, + "language_loss": 0.79966193, + "learning_rate": 3.979873333778805e-09, + "loss": 0.82136822, + "num_input_tokens_seen": 176029680, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.15087891, + "step": 8154, + "time_per_iteration": 2.841327667236328 + }, + { + "auxiliary_loss_clip": 0.01127498, + "auxiliary_loss_mlp": 0.0103527, + "balance_loss_clip": 1.04019344, + "balance_loss_mlp": 1.0204525, + "epoch": 0.9805807731617868, + "flos": 38905368382080.0, + "grad_norm": 1.9462222294116915, + "language_loss": 0.73863918, + "learning_rate": 3.930906795824862e-09, + "loss": 0.76026684, + "num_input_tokens_seen": 176050355, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.1484375, + "step": 8155, + "time_per_iteration": 2.957052230834961 + }, + { + "auxiliary_loss_clip": 0.01122758, + "auxiliary_loss_mlp": 0.01039681, + "balance_loss_clip": 1.03862739, + "balance_loss_mlp": 1.02493501, + "epoch": 0.9807010160524259, + "flos": 17821999578240.0, + "grad_norm": 2.1483987262575597, + "language_loss": 0.76821959, + "learning_rate": 3.882243059965207e-09, + "loss": 0.78984398, + "num_input_tokens_seen": 176068070, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.14733887, + "step": 8156, + "time_per_iteration": 2.8112354278564453 + }, + { + "auxiliary_loss_clip": 0.01125066, + "auxiliary_loss_mlp": 0.01041601, + "balance_loss_clip": 1.03885102, + "balance_loss_mlp": 1.02593732, + "epoch": 0.980821258943065, + "flos": 13552975140480.0, + "grad_norm": 2.521207528954899, + "language_loss": 0.65219098, + "learning_rate": 3.833882133582156e-09, + "loss": 0.67385763, + "num_input_tokens_seen": 176083730, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15661621, + "step": 8157, + "time_per_iteration": 2.8822875022888184 + }, + { + "auxiliary_loss_clip": 0.01125491, + "auxiliary_loss_mlp": 0.01040226, + "balance_loss_clip": 1.04054737, + "balance_loss_mlp": 1.02525949, + "epoch": 0.9809415018337041, + "flos": 21689794120320.0, + "grad_norm": 1.7111050509171957, + "language_loss": 0.78465807, + "learning_rate": 3.785824024012285e-09, + "loss": 0.80631518, + "num_input_tokens_seen": 176102730, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.14971924, + "step": 8158, + "time_per_iteration": 3.5234928131103516 + }, + { + "auxiliary_loss_clip": 0.01121979, + "auxiliary_loss_mlp": 0.01041124, + "balance_loss_clip": 1.04125261, + "balance_loss_mlp": 1.027457, + "epoch": 0.9810617447243432, + "flos": 23294857357440.0, + "grad_norm": 1.4828551636398482, + "language_loss": 0.78518039, + "learning_rate": 3.738068738545541e-09, + "loss": 0.80681145, + "num_input_tokens_seen": 176121815, + "router_z_loss_clip": 0.80761719, + "router_z_loss_mlp": 0.13665771, + "step": 8159, + "time_per_iteration": 2.8563921451568604 + }, + { + "auxiliary_loss_clip": 0.01128498, + "auxiliary_loss_mlp": 0.01038382, + "balance_loss_clip": 1.04280007, + "balance_loss_mlp": 1.02263451, + "epoch": 0.9811819876149822, + "flos": 18332038748160.0, + "grad_norm": 2.626061069186683, + "language_loss": 0.7875579, + "learning_rate": 3.6906162844265733e-09, + "loss": 0.80922669, + "num_input_tokens_seen": 176138900, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.15740967, + "step": 8160, + "time_per_iteration": 2.8109562397003174 + }, + { + "auxiliary_loss_clip": 0.01125117, + "auxiliary_loss_mlp": 0.01042756, + "balance_loss_clip": 1.03847218, + "balance_loss_mlp": 1.02607918, + "epoch": 0.9813022305056214, + "flos": 22601961025920.0, + "grad_norm": 2.0177654307719064, + "language_loss": 0.70867443, + "learning_rate": 3.643466668853845e-09, + "loss": 0.73035312, + "num_input_tokens_seen": 176156925, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.16687012, + "step": 8161, + "time_per_iteration": 2.765758752822876 + }, + { + "auxiliary_loss_clip": 0.01126863, + "auxiliary_loss_mlp": 0.01034187, + "balance_loss_clip": 1.04276896, + "balance_loss_mlp": 1.02026343, + "epoch": 0.9814224733962604, + "flos": 25413335642880.0, + "grad_norm": 1.9036071411062132, + "language_loss": 0.75111914, + "learning_rate": 3.59661989898008e-09, + "loss": 0.77272964, + "num_input_tokens_seen": 176177980, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.13928223, + "step": 8162, + "time_per_iteration": 3.6478121280670166 + }, + { + "auxiliary_loss_clip": 0.01120575, + "auxiliary_loss_mlp": 0.01037155, + "balance_loss_clip": 1.03763616, + "balance_loss_mlp": 1.02236152, + "epoch": 0.9815427162868995, + "flos": 25007185584000.0, + "grad_norm": 5.421284351029682, + "language_loss": 0.76446211, + "learning_rate": 3.5500759819115934e-09, + "loss": 0.78603935, + "num_input_tokens_seen": 176198345, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.14807129, + "step": 8163, + "time_per_iteration": 2.8970816135406494 + }, + { + "auxiliary_loss_clip": 0.01125719, + "auxiliary_loss_mlp": 0.01041865, + "balance_loss_clip": 1.04032707, + "balance_loss_mlp": 1.02631378, + "epoch": 0.9816629591775387, + "flos": 20662604887680.0, + "grad_norm": 1.792546540008224, + "language_loss": 0.81037581, + "learning_rate": 3.5038349247094034e-09, + "loss": 0.83205163, + "num_input_tokens_seen": 176215605, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.15545654, + "step": 8164, + "time_per_iteration": 2.7799153327941895 + }, + { + "auxiliary_loss_clip": 0.01124854, + "auxiliary_loss_mlp": 0.01037735, + "balance_loss_clip": 1.03784311, + "balance_loss_mlp": 1.02258313, + "epoch": 0.9817832020681777, + "flos": 17712220636800.0, + "grad_norm": 2.2157540860703424, + "language_loss": 0.77310437, + "learning_rate": 3.4578967343878994e-09, + "loss": 0.79473025, + "num_input_tokens_seen": 176231810, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.15161133, + "step": 8165, + "time_per_iteration": 3.5226128101348877 + }, + { + "auxiliary_loss_clip": 0.01128791, + "auxiliary_loss_mlp": 0.01035465, + "balance_loss_clip": 1.04311943, + "balance_loss_mlp": 1.02098107, + "epoch": 0.9819034449588168, + "flos": 22530032040960.0, + "grad_norm": 1.802975636903494, + "language_loss": 0.81043565, + "learning_rate": 3.4122614179161733e-09, + "loss": 0.83207822, + "num_input_tokens_seen": 176251770, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.14489746, + "step": 8166, + "time_per_iteration": 2.725372314453125 + }, + { + "auxiliary_loss_clip": 0.01122816, + "auxiliary_loss_mlp": 0.01033976, + "balance_loss_clip": 1.03900146, + "balance_loss_mlp": 1.02005267, + "epoch": 0.9820236878494559, + "flos": 20011221699840.0, + "grad_norm": 2.0774627889178845, + "language_loss": 0.78080297, + "learning_rate": 3.36692898221691e-09, + "loss": 0.80237085, + "num_input_tokens_seen": 176270135, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.1394043, + "step": 8167, + "time_per_iteration": 2.8355050086975098 + }, + { + "auxiliary_loss_clip": 0.01121544, + "auxiliary_loss_mlp": 0.01037192, + "balance_loss_clip": 1.03921282, + "balance_loss_mlp": 1.02346504, + "epoch": 0.982143930740095, + "flos": 18807316531200.0, + "grad_norm": 1.6512929444978524, + "language_loss": 0.73356628, + "learning_rate": 3.3218994341668305e-09, + "loss": 0.75515366, + "num_input_tokens_seen": 176289065, + "router_z_loss_clip": 0.82421875, + "router_z_loss_mlp": 0.137146, + "step": 8168, + "time_per_iteration": 3.7099132537841797 + }, + { + "auxiliary_loss_clip": 0.011231, + "auxiliary_loss_mlp": 0.01035001, + "balance_loss_clip": 1.04077804, + "balance_loss_mlp": 1.01975417, + "epoch": 0.982264173630734, + "flos": 26578026138240.0, + "grad_norm": 1.5898731135360118, + "language_loss": 0.75543451, + "learning_rate": 3.2771727805971373e-09, + "loss": 0.77701551, + "num_input_tokens_seen": 176310450, + "router_z_loss_clip": 0.82421875, + "router_z_loss_mlp": 0.15252686, + "step": 8169, + "time_per_iteration": 2.8639352321624756 + }, + { + "auxiliary_loss_clip": 0.01123795, + "auxiliary_loss_mlp": 0.01036774, + "balance_loss_clip": 1.03837204, + "balance_loss_mlp": 1.02090716, + "epoch": 0.9823844165213732, + "flos": 22014462176640.0, + "grad_norm": 1.8445934610723247, + "language_loss": 0.77455705, + "learning_rate": 3.232749028292847e-09, + "loss": 0.79616272, + "num_input_tokens_seen": 176327415, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15869141, + "step": 8170, + "time_per_iteration": 2.9332222938537598 + }, + { + "auxiliary_loss_clip": 0.01124905, + "auxiliary_loss_mlp": 0.01037088, + "balance_loss_clip": 1.03880095, + "balance_loss_mlp": 1.0226692, + "epoch": 0.9825046594120123, + "flos": 21908166854400.0, + "grad_norm": 1.7127603984453899, + "language_loss": 0.88601768, + "learning_rate": 3.188628183992792e-09, + "loss": 0.9076376, + "num_input_tokens_seen": 176347680, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.14416504, + "step": 8171, + "time_per_iteration": 2.8938100337982178 + }, + { + "auxiliary_loss_clip": 0.01031788, + "auxiliary_loss_mlp": 0.0099835, + "balance_loss_clip": 1.00429678, + "balance_loss_mlp": 0.99710989, + "epoch": 0.9826249023026513, + "flos": 59494610718720.0, + "grad_norm": 0.733238941524587, + "language_loss": 0.62488854, + "learning_rate": 3.1448102543902844e-09, + "loss": 0.64518988, + "num_input_tokens_seen": 176411595, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01239014, + "step": 8172, + "time_per_iteration": 3.3006887435913086 + }, + { + "auxiliary_loss_clip": 0.01123971, + "auxiliary_loss_mlp": 0.01038638, + "balance_loss_clip": 1.04089105, + "balance_loss_mlp": 1.02333188, + "epoch": 0.9827451451932905, + "flos": 16071031296000.0, + "grad_norm": 2.0030119187382285, + "language_loss": 0.67756784, + "learning_rate": 3.1012952461324515e-09, + "loss": 0.69919395, + "num_input_tokens_seen": 176430570, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.15313721, + "step": 8173, + "time_per_iteration": 2.900345802307129 + }, + { + "auxiliary_loss_clip": 0.01124117, + "auxiliary_loss_mlp": 0.01042215, + "balance_loss_clip": 1.0408566, + "balance_loss_mlp": 1.02717686, + "epoch": 0.9828653880839295, + "flos": 20262775622400.0, + "grad_norm": 2.322786903742075, + "language_loss": 0.73766673, + "learning_rate": 3.0580831658204575e-09, + "loss": 0.75933003, + "num_input_tokens_seen": 176448150, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.15045166, + "step": 8174, + "time_per_iteration": 2.947669506072998 + }, + { + "auxiliary_loss_clip": 0.01125535, + "auxiliary_loss_mlp": 0.01034442, + "balance_loss_clip": 1.04271829, + "balance_loss_mlp": 1.01995194, + "epoch": 0.9829856309745686, + "flos": 21616141282560.0, + "grad_norm": 1.6280475742799871, + "language_loss": 0.78079772, + "learning_rate": 3.015174020009281e-09, + "loss": 0.80239749, + "num_input_tokens_seen": 176467475, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.14489746, + "step": 8175, + "time_per_iteration": 2.942885160446167 + }, + { + "auxiliary_loss_clip": 0.0112479, + "auxiliary_loss_mlp": 0.01039715, + "balance_loss_clip": 1.04111636, + "balance_loss_mlp": 1.0262866, + "epoch": 0.9831058738652078, + "flos": 23764209396480.0, + "grad_norm": 1.7229190109714418, + "language_loss": 0.74866289, + "learning_rate": 2.9725678152086043e-09, + "loss": 0.77030796, + "num_input_tokens_seen": 176486045, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.13421631, + "step": 8176, + "time_per_iteration": 2.9099795818328857 + }, + { + "auxiliary_loss_clip": 0.0111994, + "auxiliary_loss_mlp": 0.01035897, + "balance_loss_clip": 1.03756821, + "balance_loss_mlp": 1.02147853, + "epoch": 0.9832261167558468, + "flos": 11320911072000.0, + "grad_norm": 2.6592718551914816, + "language_loss": 0.82503021, + "learning_rate": 2.930264557881257e-09, + "loss": 0.84658861, + "num_input_tokens_seen": 176501230, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.14416504, + "step": 8177, + "time_per_iteration": 2.977620840072632 + }, + { + "auxiliary_loss_clip": 0.01032268, + "auxiliary_loss_mlp": 0.0100182, + "balance_loss_clip": 1.00467587, + "balance_loss_mlp": 1.00057971, + "epoch": 0.9833463596464859, + "flos": 60000304343040.0, + "grad_norm": 0.828699267134454, + "language_loss": 0.58163393, + "learning_rate": 2.8882642544452163e-09, + "loss": 0.60197484, + "num_input_tokens_seen": 176565955, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01239014, + "step": 8178, + "time_per_iteration": 3.3482565879821777 + }, + { + "auxiliary_loss_clip": 0.01123296, + "auxiliary_loss_mlp": 0.01035329, + "balance_loss_clip": 1.03787458, + "balance_loss_mlp": 1.02017784, + "epoch": 0.983466602537125, + "flos": 13626699805440.0, + "grad_norm": 4.912058364096866, + "language_loss": 0.74319744, + "learning_rate": 2.8465669112716083e-09, + "loss": 0.76478374, + "num_input_tokens_seen": 176583480, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.15142822, + "step": 8179, + "time_per_iteration": 2.931541681289673 + }, + { + "auxiliary_loss_clip": 0.01128432, + "auxiliary_loss_mlp": 0.01039762, + "balance_loss_clip": 1.04107976, + "balance_loss_mlp": 1.02277446, + "epoch": 0.9835868454277641, + "flos": 22926844563840.0, + "grad_norm": 1.9285522652133529, + "language_loss": 0.76194626, + "learning_rate": 2.8051725346858177e-09, + "loss": 0.78362823, + "num_input_tokens_seen": 176603740, + "router_z_loss_clip": 0.87353516, + "router_z_loss_mlp": 0.1697998, + "step": 8180, + "time_per_iteration": 2.864955425262451 + }, + { + "auxiliary_loss_clip": 0.01124504, + "auxiliary_loss_mlp": 0.01032272, + "balance_loss_clip": 1.03800631, + "balance_loss_mlp": 1.01702464, + "epoch": 0.9837070883184031, + "flos": 27673409341440.0, + "grad_norm": 2.2178720221728914, + "language_loss": 0.70629978, + "learning_rate": 2.7640811309674883e-09, + "loss": 0.72786754, + "num_input_tokens_seen": 176623240, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15222168, + "step": 8181, + "time_per_iteration": 2.8042213916778564 + }, + { + "auxiliary_loss_clip": 0.01119309, + "auxiliary_loss_mlp": 0.01038088, + "balance_loss_clip": 1.03856754, + "balance_loss_mlp": 1.02334213, + "epoch": 0.9838273312090423, + "flos": 29241951425280.0, + "grad_norm": 1.5756508371461588, + "language_loss": 0.80680364, + "learning_rate": 2.7232927063498557e-09, + "loss": 0.8283776, + "num_input_tokens_seen": 176643615, + "router_z_loss_clip": 0.80761719, + "router_z_loss_mlp": 0.14758301, + "step": 8182, + "time_per_iteration": 2.8829309940338135 + }, + { + "auxiliary_loss_clip": 0.01125451, + "auxiliary_loss_mlp": 0.01036605, + "balance_loss_clip": 1.03927541, + "balance_loss_mlp": 1.02177501, + "epoch": 0.9839475740996814, + "flos": 40110207304320.0, + "grad_norm": 1.7406612892081128, + "language_loss": 0.68831593, + "learning_rate": 2.682807267020859e-09, + "loss": 0.7099365, + "num_input_tokens_seen": 176666375, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.14831543, + "step": 8183, + "time_per_iteration": 2.9319119453430176 + }, + { + "auxiliary_loss_clip": 0.01128028, + "auxiliary_loss_mlp": 0.0103748, + "balance_loss_clip": 1.04229426, + "balance_loss_mlp": 1.02203095, + "epoch": 0.9840678169903204, + "flos": 24169389788160.0, + "grad_norm": 1.6225808105428492, + "language_loss": 0.62492263, + "learning_rate": 2.642624819121808e-09, + "loss": 0.64657772, + "num_input_tokens_seen": 176686525, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15478516, + "step": 8184, + "time_per_iteration": 2.8569839000701904 + }, + { + "auxiliary_loss_clip": 0.01120932, + "auxiliary_loss_mlp": 0.01042681, + "balance_loss_clip": 1.03784728, + "balance_loss_mlp": 1.02801871, + "epoch": 0.9841880598809596, + "flos": 14684484447360.0, + "grad_norm": 1.9206413274047207, + "language_loss": 0.61760795, + "learning_rate": 2.6027453687487154e-09, + "loss": 0.63924414, + "num_input_tokens_seen": 176703615, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.14672852, + "step": 8185, + "time_per_iteration": 2.801551580429077 + }, + { + "auxiliary_loss_clip": 0.01124667, + "auxiliary_loss_mlp": 0.01045156, + "balance_loss_clip": 1.03917491, + "balance_loss_mlp": 1.02933693, + "epoch": 0.9843083027715986, + "flos": 22344768668160.0, + "grad_norm": 2.185438993196434, + "language_loss": 0.53505945, + "learning_rate": 2.5631689219509643e-09, + "loss": 0.55675769, + "num_input_tokens_seen": 176722295, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15820312, + "step": 8186, + "time_per_iteration": 2.837559223175049 + }, + { + "auxiliary_loss_clip": 0.01120226, + "auxiliary_loss_mlp": 0.01034217, + "balance_loss_clip": 1.03954816, + "balance_loss_mlp": 1.02070439, + "epoch": 0.9844285456622377, + "flos": 21800111765760.0, + "grad_norm": 1.6503998963811024, + "language_loss": 0.83268839, + "learning_rate": 2.523895484732197e-09, + "loss": 0.85423285, + "num_input_tokens_seen": 176741750, + "router_z_loss_clip": 0.80615234, + "router_z_loss_mlp": 0.13525391, + "step": 8187, + "time_per_iteration": 3.704861879348755 + }, + { + "auxiliary_loss_clip": 0.01129183, + "auxiliary_loss_mlp": 0.01036559, + "balance_loss_clip": 1.04134083, + "balance_loss_mlp": 1.02038264, + "epoch": 0.9845487885528769, + "flos": 18035380321920.0, + "grad_norm": 1.8864607220751657, + "language_loss": 0.74656391, + "learning_rate": 2.4849250630505357e-09, + "loss": 0.76822138, + "num_input_tokens_seen": 176759995, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.16174316, + "step": 8188, + "time_per_iteration": 2.810354709625244 + }, + { + "auxiliary_loss_clip": 0.0112341, + "auxiliary_loss_mlp": 0.01041106, + "balance_loss_clip": 1.03779948, + "balance_loss_mlp": 1.02597821, + "epoch": 0.9846690314435159, + "flos": 25228610974080.0, + "grad_norm": 1.8879391513236359, + "language_loss": 0.73666251, + "learning_rate": 2.4462576628172528e-09, + "loss": 0.7583077, + "num_input_tokens_seen": 176778625, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15118408, + "step": 8189, + "time_per_iteration": 2.8547563552856445 + }, + { + "auxiliary_loss_clip": 0.01126811, + "auxiliary_loss_mlp": 0.01037886, + "balance_loss_clip": 1.04055643, + "balance_loss_mlp": 1.02329528, + "epoch": 0.984789274334155, + "flos": 18552171248640.0, + "grad_norm": 2.0620491303319146, + "language_loss": 0.73779523, + "learning_rate": 2.407893289898766e-09, + "loss": 0.75944221, + "num_input_tokens_seen": 176797655, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.14599609, + "step": 8190, + "time_per_iteration": 2.8578310012817383 + }, + { + "auxiliary_loss_clip": 0.01121398, + "auxiliary_loss_mlp": 0.0103702, + "balance_loss_clip": 1.03717089, + "balance_loss_mlp": 1.02108216, + "epoch": 0.984909517224794, + "flos": 27345437233920.0, + "grad_norm": 1.9861862513512378, + "language_loss": 0.84080613, + "learning_rate": 2.3698319501144202e-09, + "loss": 0.86239034, + "num_input_tokens_seen": 176818640, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.15930176, + "step": 8191, + "time_per_iteration": 3.6538729667663574 + }, + { + "auxiliary_loss_clip": 0.01128507, + "auxiliary_loss_mlp": 0.01037326, + "balance_loss_clip": 1.03979897, + "balance_loss_mlp": 1.01996911, + "epoch": 0.9850297601154332, + "flos": 18734058743040.0, + "grad_norm": 8.401212639442473, + "language_loss": 0.73447335, + "learning_rate": 2.3320736492382644e-09, + "loss": 0.75613177, + "num_input_tokens_seen": 176837475, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.17370605, + "step": 8192, + "time_per_iteration": 2.9955451488494873 + }, + { + "auxiliary_loss_clip": 0.01122158, + "auxiliary_loss_mlp": 0.01032702, + "balance_loss_clip": 1.04011369, + "balance_loss_mlp": 1.01721644, + "epoch": 0.9851500030060723, + "flos": 22308247514880.0, + "grad_norm": 1.5873875880636763, + "language_loss": 0.67627358, + "learning_rate": 2.29461839299816e-09, + "loss": 0.69782221, + "num_input_tokens_seen": 176857190, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.15490723, + "step": 8193, + "time_per_iteration": 2.89485502243042 + }, + { + "auxiliary_loss_clip": 0.01123208, + "auxiliary_loss_mlp": 0.01036845, + "balance_loss_clip": 1.04016662, + "balance_loss_mlp": 1.02084744, + "epoch": 0.9852702458967113, + "flos": 26353691746560.0, + "grad_norm": 1.5303703977944358, + "language_loss": 0.79889631, + "learning_rate": 2.257466187076229e-09, + "loss": 0.8204968, + "num_input_tokens_seen": 176876395, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.15979004, + "step": 8194, + "time_per_iteration": 3.6900272369384766 + }, + { + "auxiliary_loss_clip": 0.01126364, + "auxiliary_loss_mlp": 0.01040263, + "balance_loss_clip": 1.03962469, + "balance_loss_mlp": 1.0252192, + "epoch": 0.9853904887873505, + "flos": 20883599314560.0, + "grad_norm": 1.75203696902993, + "language_loss": 0.70907879, + "learning_rate": 2.2206170371081854e-09, + "loss": 0.73074508, + "num_input_tokens_seen": 176894980, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.15032959, + "step": 8195, + "time_per_iteration": 2.8778347969055176 + }, + { + "auxiliary_loss_clip": 0.01130365, + "auxiliary_loss_mlp": 0.01044344, + "balance_loss_clip": 1.04263854, + "balance_loss_mlp": 1.02886474, + "epoch": 0.9855107316779895, + "flos": 25263444188160.0, + "grad_norm": 1.697208888932639, + "language_loss": 0.84959936, + "learning_rate": 2.1840709486842247e-09, + "loss": 0.87134647, + "num_input_tokens_seen": 176914600, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.15478516, + "step": 8196, + "time_per_iteration": 2.8793399333953857 + }, + { + "auxiliary_loss_clip": 0.01128561, + "auxiliary_loss_mlp": 0.01036792, + "balance_loss_clip": 1.03999567, + "balance_loss_mlp": 1.02086544, + "epoch": 0.9856309745686286, + "flos": 19062102677760.0, + "grad_norm": 2.195704502835843, + "language_loss": 0.79286772, + "learning_rate": 2.1478279273481335e-09, + "loss": 0.81452131, + "num_input_tokens_seen": 176933085, + "router_z_loss_clip": 0.88427734, + "router_z_loss_mlp": 0.15917969, + "step": 8197, + "time_per_iteration": 3.821503162384033 + }, + { + "auxiliary_loss_clip": 0.01125458, + "auxiliary_loss_mlp": 0.01047284, + "balance_loss_clip": 1.04046249, + "balance_loss_mlp": 1.03228724, + "epoch": 0.9857512174592677, + "flos": 34130758060800.0, + "grad_norm": 2.557121994628148, + "language_loss": 0.80088162, + "learning_rate": 2.1118879785981815e-09, + "loss": 0.82260907, + "num_input_tokens_seen": 176953225, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.15002441, + "step": 8198, + "time_per_iteration": 3.049250841140747 + }, + { + "auxiliary_loss_clip": 0.01121347, + "auxiliary_loss_mlp": 0.01030672, + "balance_loss_clip": 1.03822017, + "balance_loss_mlp": 1.01579499, + "epoch": 0.9858714603499068, + "flos": 25994693266560.0, + "grad_norm": 1.843836935531333, + "language_loss": 0.7950685, + "learning_rate": 2.0762511078862288e-09, + "loss": 0.81658876, + "num_input_tokens_seen": 176973570, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.14880371, + "step": 8199, + "time_per_iteration": 2.9225969314575195 + }, + { + "auxiliary_loss_clip": 0.01127659, + "auxiliary_loss_mlp": 0.0104178, + "balance_loss_clip": 1.0405364, + "balance_loss_mlp": 1.02692628, + "epoch": 0.9859917032405459, + "flos": 23696230907520.0, + "grad_norm": 1.7500788846131918, + "language_loss": 0.64878058, + "learning_rate": 2.0409173206186183e-09, + "loss": 0.67047501, + "num_input_tokens_seen": 176992810, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.14855957, + "step": 8200, + "time_per_iteration": 2.831390619277954 + }, + { + "auxiliary_loss_clip": 0.01124182, + "auxiliary_loss_mlp": 0.01034403, + "balance_loss_clip": 1.03989649, + "balance_loss_mlp": 1.02022934, + "epoch": 0.986111946131185, + "flos": 19938287134080.0, + "grad_norm": 1.999703325937717, + "language_loss": 0.87163937, + "learning_rate": 2.0058866221550617e-09, + "loss": 0.89322519, + "num_input_tokens_seen": 177011050, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.14160156, + "step": 8201, + "time_per_iteration": 2.795659065246582 + }, + { + "auxiliary_loss_clip": 0.01125208, + "auxiliary_loss_mlp": 0.01034708, + "balance_loss_clip": 1.03967106, + "balance_loss_mlp": 1.02035522, + "epoch": 0.9862321890218241, + "flos": 19828831415040.0, + "grad_norm": 1.9629573575013448, + "language_loss": 0.75261009, + "learning_rate": 1.971159017809976e-09, + "loss": 0.7742092, + "num_input_tokens_seen": 177029340, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.14367676, + "step": 8202, + "time_per_iteration": 2.8134727478027344 + }, + { + "auxiliary_loss_clip": 0.01126068, + "auxiliary_loss_mlp": 0.01037813, + "balance_loss_clip": 1.03965306, + "balance_loss_mlp": 1.02181506, + "epoch": 0.9863524319124631, + "flos": 21652051904640.0, + "grad_norm": 1.9305357001848125, + "language_loss": 0.78050125, + "learning_rate": 1.93673451285159e-09, + "loss": 0.80214006, + "num_input_tokens_seen": 177048390, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.16003418, + "step": 8203, + "time_per_iteration": 2.8597309589385986 + }, + { + "auxiliary_loss_clip": 0.01031782, + "auxiliary_loss_mlp": 0.01000682, + "balance_loss_clip": 1.00419354, + "balance_loss_mlp": 0.99940032, + "epoch": 0.9864726748031023, + "flos": 52769977920000.0, + "grad_norm": 0.7392420311257788, + "language_loss": 0.56423086, + "learning_rate": 1.9026131125019495e-09, + "loss": 0.58455551, + "num_input_tokens_seen": 177105760, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01281738, + "step": 8204, + "time_per_iteration": 3.239381790161133 + }, + { + "auxiliary_loss_clip": 0.01121747, + "auxiliary_loss_mlp": 0.01039535, + "balance_loss_clip": 1.03832412, + "balance_loss_mlp": 1.02486026, + "epoch": 0.9865929176937414, + "flos": 23364631526400.0, + "grad_norm": 1.7259936110072365, + "language_loss": 0.87038684, + "learning_rate": 1.8687948219371363e-09, + "loss": 0.89199966, + "num_input_tokens_seen": 177124985, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.14685059, + "step": 8205, + "time_per_iteration": 2.890209436416626 + }, + { + "auxiliary_loss_clip": 0.01128969, + "auxiliary_loss_mlp": 0.01040452, + "balance_loss_clip": 1.03839183, + "balance_loss_mlp": 1.02344084, + "epoch": 0.9867131605843804, + "flos": 21616679986560.0, + "grad_norm": 1.9858951074558766, + "language_loss": 0.88728428, + "learning_rate": 1.835279646287491e-09, + "loss": 0.90897858, + "num_input_tokens_seen": 177142995, + "router_z_loss_clip": 0.90576172, + "router_z_loss_mlp": 0.17016602, + "step": 8206, + "time_per_iteration": 2.9021780490875244 + }, + { + "auxiliary_loss_clip": 0.01131833, + "auxiliary_loss_mlp": 0.01039083, + "balance_loss_clip": 1.04269361, + "balance_loss_mlp": 1.02233458, + "epoch": 0.9868334034750196, + "flos": 22271403139200.0, + "grad_norm": 1.8153572214196954, + "language_loss": 0.76573718, + "learning_rate": 1.8020675906371685e-09, + "loss": 0.78744632, + "num_input_tokens_seen": 177162390, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.16748047, + "step": 8207, + "time_per_iteration": 2.788780927658081 + }, + { + "auxiliary_loss_clip": 0.0112128, + "auxiliary_loss_mlp": 0.0103491, + "balance_loss_clip": 1.03851056, + "balance_loss_mlp": 1.02029455, + "epoch": 0.9869536463656586, + "flos": 25809573548160.0, + "grad_norm": 1.9270639999723522, + "language_loss": 0.74906075, + "learning_rate": 1.7691586600243612e-09, + "loss": 0.77062261, + "num_input_tokens_seen": 177181290, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.14611816, + "step": 8208, + "time_per_iteration": 2.887766122817993 + }, + { + "auxiliary_loss_clip": 0.01124757, + "auxiliary_loss_mlp": 0.0103783, + "balance_loss_clip": 1.03949237, + "balance_loss_mlp": 1.02238071, + "epoch": 0.9870738892562977, + "flos": 16398500613120.0, + "grad_norm": 2.2328307190498493, + "language_loss": 0.86927676, + "learning_rate": 1.7365528594415202e-09, + "loss": 0.89090264, + "num_input_tokens_seen": 177195360, + "router_z_loss_clip": 0.85253906, + "router_z_loss_mlp": 0.15441895, + "step": 8209, + "time_per_iteration": 2.810676336288452 + }, + { + "auxiliary_loss_clip": 0.01126554, + "auxiliary_loss_mlp": 0.0104093, + "balance_loss_clip": 1.03834784, + "balance_loss_mlp": 1.0258683, + "epoch": 0.9871941321469369, + "flos": 35481358373760.0, + "grad_norm": 1.7358501791375167, + "language_loss": 0.67550981, + "learning_rate": 1.7042501938346888e-09, + "loss": 0.69718462, + "num_input_tokens_seen": 177218090, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.15057373, + "step": 8210, + "time_per_iteration": 3.087440252304077 + }, + { + "auxiliary_loss_clip": 0.01120949, + "auxiliary_loss_mlp": 0.01032597, + "balance_loss_clip": 1.0387615, + "balance_loss_mlp": 1.01831603, + "epoch": 0.9873143750375759, + "flos": 21434217874560.0, + "grad_norm": 1.9396079561658224, + "language_loss": 0.76569414, + "learning_rate": 1.6722506681043913e-09, + "loss": 0.7872296, + "num_input_tokens_seen": 177237050, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.14294434, + "step": 8211, + "time_per_iteration": 2.898620128631592 + }, + { + "auxiliary_loss_clip": 0.0112763, + "auxiliary_loss_mlp": 0.01034091, + "balance_loss_clip": 1.04016221, + "balance_loss_mlp": 1.01992869, + "epoch": 0.987434617928215, + "flos": 16326499800960.0, + "grad_norm": 2.2778612987243294, + "language_loss": 0.68919575, + "learning_rate": 1.640554287104745e-09, + "loss": 0.71081293, + "num_input_tokens_seen": 177255325, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.14172363, + "step": 8212, + "time_per_iteration": 2.840787887573242 + }, + { + "auxiliary_loss_clip": 0.01124089, + "auxiliary_loss_mlp": 0.01035783, + "balance_loss_clip": 1.03767633, + "balance_loss_mlp": 1.02023804, + "epoch": 0.9875548608188541, + "flos": 17851984456320.0, + "grad_norm": 2.226854622779086, + "language_loss": 0.80014527, + "learning_rate": 1.609161055644348e-09, + "loss": 0.82174397, + "num_input_tokens_seen": 177271250, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.15539551, + "step": 8213, + "time_per_iteration": 2.8778507709503174 + }, + { + "auxiliary_loss_clip": 0.01129452, + "auxiliary_loss_mlp": 0.01040783, + "balance_loss_clip": 1.0399555, + "balance_loss_mlp": 1.02629864, + "epoch": 0.9876751037094932, + "flos": 26132876887680.0, + "grad_norm": 2.587575372110408, + "language_loss": 0.68346018, + "learning_rate": 1.5780709784849467e-09, + "loss": 0.70516253, + "num_input_tokens_seen": 177288270, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.14465332, + "step": 8214, + "time_per_iteration": 2.8706977367401123 + }, + { + "auxiliary_loss_clip": 0.01129036, + "auxiliary_loss_mlp": 0.01035497, + "balance_loss_clip": 1.04118896, + "balance_loss_mlp": 1.01884329, + "epoch": 0.9877953466001322, + "flos": 15991344973440.0, + "grad_norm": 2.143421070023933, + "language_loss": 0.82165074, + "learning_rate": 1.5472840603436565e-09, + "loss": 0.84329617, + "num_input_tokens_seen": 177305500, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.16650391, + "step": 8215, + "time_per_iteration": 2.8321995735168457 + }, + { + "auxiliary_loss_clip": 0.01123285, + "auxiliary_loss_mlp": 0.01039815, + "balance_loss_clip": 1.03897095, + "balance_loss_mlp": 1.02429366, + "epoch": 0.9879155894907714, + "flos": 18806777827200.0, + "grad_norm": 1.8300198741525846, + "language_loss": 0.78126299, + "learning_rate": 1.5168003058900757e-09, + "loss": 0.80289406, + "num_input_tokens_seen": 177323500, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.15527344, + "step": 8216, + "time_per_iteration": 3.5374081134796143 + }, + { + "auxiliary_loss_clip": 0.0112149, + "auxiliary_loss_mlp": 0.01037932, + "balance_loss_clip": 1.03895807, + "balance_loss_mlp": 1.02288735, + "epoch": 0.9880358323814105, + "flos": 22382044007040.0, + "grad_norm": 1.8503400133538626, + "language_loss": 0.91888559, + "learning_rate": 1.4866197197491715e-09, + "loss": 0.94047976, + "num_input_tokens_seen": 177342860, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.1505127, + "step": 8217, + "time_per_iteration": 2.8518710136413574 + }, + { + "auxiliary_loss_clip": 0.01132425, + "auxiliary_loss_mlp": 0.01043917, + "balance_loss_clip": 1.04150486, + "balance_loss_mlp": 1.0274663, + "epoch": 0.9881560752720495, + "flos": 15668831733120.0, + "grad_norm": 2.66407810477405, + "language_loss": 0.78512704, + "learning_rate": 1.4567423064988371e-09, + "loss": 0.80689049, + "num_input_tokens_seen": 177360210, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.16430664, + "step": 8218, + "time_per_iteration": 2.84012508392334 + }, + { + "auxiliary_loss_clip": 0.01126351, + "auxiliary_loss_mlp": 0.01041492, + "balance_loss_clip": 1.04023468, + "balance_loss_mlp": 1.02571476, + "epoch": 0.9882763181626887, + "flos": 21500113374720.0, + "grad_norm": 1.9050883487219956, + "language_loss": 0.78002918, + "learning_rate": 1.4271680706718913e-09, + "loss": 0.80170763, + "num_input_tokens_seen": 177377885, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.15789795, + "step": 8219, + "time_per_iteration": 2.8498268127441406 + }, + { + "auxiliary_loss_clip": 0.01130528, + "auxiliary_loss_mlp": 0.01042351, + "balance_loss_clip": 1.04364097, + "balance_loss_mlp": 1.0260905, + "epoch": 0.9883965610533277, + "flos": 28034598551040.0, + "grad_norm": 1.6752270650139354, + "language_loss": 0.82405919, + "learning_rate": 1.3978970167543013e-09, + "loss": 0.845788, + "num_input_tokens_seen": 177398065, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.16271973, + "step": 8220, + "time_per_iteration": 3.6225063800811768 + }, + { + "auxiliary_loss_clip": 0.01123986, + "auxiliary_loss_mlp": 0.01041522, + "balance_loss_clip": 1.04013705, + "balance_loss_mlp": 1.02561915, + "epoch": 0.9885168039439668, + "flos": 14098601710080.0, + "grad_norm": 2.300395720867246, + "language_loss": 0.77970934, + "learning_rate": 1.3689291491867372e-09, + "loss": 0.80136442, + "num_input_tokens_seen": 177416380, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.15905762, + "step": 8221, + "time_per_iteration": 2.8839197158813477 + }, + { + "auxiliary_loss_clip": 0.01127059, + "auxiliary_loss_mlp": 0.01032867, + "balance_loss_clip": 1.04038382, + "balance_loss_mlp": 1.01735759, + "epoch": 0.988637046834606, + "flos": 26432013352320.0, + "grad_norm": 2.641644413325276, + "language_loss": 0.7352978, + "learning_rate": 1.3402644723636836e-09, + "loss": 0.75689709, + "num_input_tokens_seen": 177438410, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.15515137, + "step": 8222, + "time_per_iteration": 2.908493995666504 + }, + { + "auxiliary_loss_clip": 0.01127147, + "auxiliary_loss_mlp": 0.01038067, + "balance_loss_clip": 1.04211819, + "balance_loss_mlp": 1.0226649, + "epoch": 0.988757289725245, + "flos": 25229113764480.0, + "grad_norm": 2.0215506046214675, + "language_loss": 0.83516753, + "learning_rate": 1.311902990633218e-09, + "loss": 0.85681963, + "num_input_tokens_seen": 177457375, + "router_z_loss_clip": 0.84960938, + "router_z_loss_mlp": 0.1541748, + "step": 8223, + "time_per_iteration": 3.696124792098999 + }, + { + "auxiliary_loss_clip": 0.01123768, + "auxiliary_loss_mlp": 0.01039133, + "balance_loss_clip": 1.03919542, + "balance_loss_mlp": 1.02401745, + "epoch": 0.9888775326158841, + "flos": 26359042872960.0, + "grad_norm": 1.6367037783526952, + "language_loss": 0.71592677, + "learning_rate": 1.2838447082978987e-09, + "loss": 0.73755586, + "num_input_tokens_seen": 177478530, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.15124512, + "step": 8224, + "time_per_iteration": 2.8397347927093506 + }, + { + "auxiliary_loss_clip": 0.01124698, + "auxiliary_loss_mlp": 0.01035268, + "balance_loss_clip": 1.04066658, + "balance_loss_mlp": 1.01993728, + "epoch": 0.9889977755065231, + "flos": 24316120846080.0, + "grad_norm": 2.6372035252773323, + "language_loss": 0.8278029, + "learning_rate": 1.2560896296143208e-09, + "loss": 0.84940255, + "num_input_tokens_seen": 177496995, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.15319824, + "step": 8225, + "time_per_iteration": 3.6842844486236572 + }, + { + "auxiliary_loss_clip": 0.01125828, + "auxiliary_loss_mlp": 0.01034784, + "balance_loss_clip": 1.04091954, + "balance_loss_mlp": 1.01960826, + "epoch": 0.9891180183971623, + "flos": 18951066760320.0, + "grad_norm": 2.222057145319412, + "language_loss": 0.82272339, + "learning_rate": 1.2286377587926722e-09, + "loss": 0.84432954, + "num_input_tokens_seen": 177513785, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15185547, + "step": 8226, + "time_per_iteration": 2.8175225257873535 + }, + { + "auxiliary_loss_clip": 0.01122533, + "auxiliary_loss_mlp": 0.01043444, + "balance_loss_clip": 1.03706694, + "balance_loss_mlp": 1.02708805, + "epoch": 0.9892382612878013, + "flos": 26176580760960.0, + "grad_norm": 2.362340796619429, + "language_loss": 0.74754161, + "learning_rate": 1.2014890999973992e-09, + "loss": 0.76920134, + "num_input_tokens_seen": 177530705, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.16345215, + "step": 8227, + "time_per_iteration": 2.817105293273926 + }, + { + "auxiliary_loss_clip": 0.01120977, + "auxiliary_loss_mlp": 0.01035873, + "balance_loss_clip": 1.03858781, + "balance_loss_mlp": 1.02032793, + "epoch": 0.9893585041784404, + "flos": 25449605400960.0, + "grad_norm": 1.5475434015587723, + "language_loss": 0.78146499, + "learning_rate": 1.1746436573472073e-09, + "loss": 0.80303347, + "num_input_tokens_seen": 177552440, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.15551758, + "step": 8228, + "time_per_iteration": 2.860445976257324 + }, + { + "auxiliary_loss_clip": 0.01128442, + "auxiliary_loss_mlp": 0.01037327, + "balance_loss_clip": 1.04102302, + "balance_loss_mlp": 1.02233028, + "epoch": 0.9894787470690796, + "flos": 20189302352640.0, + "grad_norm": 2.422979066337175, + "language_loss": 0.69218141, + "learning_rate": 1.1481014349141726e-09, + "loss": 0.71383905, + "num_input_tokens_seen": 177569660, + "router_z_loss_clip": 0.875, + "router_z_loss_mlp": 0.15002441, + "step": 8229, + "time_per_iteration": 2.83134388923645 + }, + { + "auxiliary_loss_clip": 0.01127746, + "auxiliary_loss_mlp": 0.01037522, + "balance_loss_clip": 1.04078317, + "balance_loss_mlp": 1.02139354, + "epoch": 0.9895989899597186, + "flos": 24644308435200.0, + "grad_norm": 1.8742524154848548, + "language_loss": 0.84528828, + "learning_rate": 1.121862436724852e-09, + "loss": 0.86694098, + "num_input_tokens_seen": 177588500, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.16125488, + "step": 8230, + "time_per_iteration": 2.8795976638793945 + }, + { + "auxiliary_loss_clip": 0.01128235, + "auxiliary_loss_mlp": 0.01038109, + "balance_loss_clip": 1.04436326, + "balance_loss_mlp": 1.02326798, + "epoch": 0.9897192328503577, + "flos": 21799034357760.0, + "grad_norm": 1.7349195911940878, + "language_loss": 0.70696127, + "learning_rate": 1.0959266667598388e-09, + "loss": 0.72862476, + "num_input_tokens_seen": 177607315, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.1484375, + "step": 8231, + "time_per_iteration": 2.8779008388519287 + }, + { + "auxiliary_loss_clip": 0.01131891, + "auxiliary_loss_mlp": 0.01037223, + "balance_loss_clip": 1.04105997, + "balance_loss_mlp": 1.02041435, + "epoch": 0.9898394757409968, + "flos": 21325229032320.0, + "grad_norm": 2.0510515482847222, + "language_loss": 0.74662942, + "learning_rate": 1.0702941289533196e-09, + "loss": 0.76832056, + "num_input_tokens_seen": 177625990, + "router_z_loss_clip": 0.90869141, + "router_z_loss_mlp": 0.16809082, + "step": 8232, + "time_per_iteration": 2.8589932918548584 + }, + { + "auxiliary_loss_clip": 0.01119673, + "auxiliary_loss_mlp": 0.01040585, + "balance_loss_clip": 1.03832257, + "balance_loss_mlp": 1.02588081, + "epoch": 0.9899597186316359, + "flos": 18545024442240.0, + "grad_norm": 1.8536544677095743, + "language_loss": 0.88931131, + "learning_rate": 1.0449648271939615e-09, + "loss": 0.91091394, + "num_input_tokens_seen": 177642335, + "router_z_loss_clip": 0.81347656, + "router_z_loss_mlp": 0.14715576, + "step": 8233, + "time_per_iteration": 2.8678486347198486 + }, + { + "auxiliary_loss_clip": 0.01129464, + "auxiliary_loss_mlp": 0.01039603, + "balance_loss_clip": 1.04202604, + "balance_loss_mlp": 1.02387929, + "epoch": 0.990079961522275, + "flos": 23766723348480.0, + "grad_norm": 1.5390871827946964, + "language_loss": 0.72699934, + "learning_rate": 1.0199387653240243e-09, + "loss": 0.74869001, + "num_input_tokens_seen": 177662025, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15734863, + "step": 8234, + "time_per_iteration": 2.9884510040283203 + }, + { + "auxiliary_loss_clip": 0.01124042, + "auxiliary_loss_mlp": 0.01039201, + "balance_loss_clip": 1.04100811, + "balance_loss_mlp": 1.02517033, + "epoch": 0.9902002044129141, + "flos": 16399182971520.0, + "grad_norm": 1.5678855070618667, + "language_loss": 0.70486999, + "learning_rate": 9.952159471400267e-10, + "loss": 0.72650242, + "num_input_tokens_seen": 177679065, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.14031982, + "step": 8235, + "time_per_iteration": 2.9708361625671387 + }, + { + "auxiliary_loss_clip": 0.01121921, + "auxiliary_loss_mlp": 0.01035611, + "balance_loss_clip": 1.03899145, + "balance_loss_mlp": 1.02013779, + "epoch": 0.9903204473035532, + "flos": 22559657783040.0, + "grad_norm": 2.1293747986188634, + "language_loss": 0.84043616, + "learning_rate": 9.707963763923022e-10, + "loss": 0.86201149, + "num_input_tokens_seen": 177698115, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.15466309, + "step": 8236, + "time_per_iteration": 2.938044548034668 + }, + { + "auxiliary_loss_clip": 0.0112174, + "auxiliary_loss_mlp": 0.01036238, + "balance_loss_clip": 1.03919625, + "balance_loss_mlp": 1.02291036, + "epoch": 0.9904406901941922, + "flos": 16144001775360.0, + "grad_norm": 1.717696900983593, + "language_loss": 0.78922647, + "learning_rate": 9.466800567854427e-10, + "loss": 0.81080627, + "num_input_tokens_seen": 177716715, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.13336182, + "step": 8237, + "time_per_iteration": 2.8482697010040283 + }, + { + "auxiliary_loss_clip": 0.01125241, + "auxiliary_loss_mlp": 0.01041694, + "balance_loss_clip": 1.03911436, + "balance_loss_mlp": 1.02617347, + "epoch": 0.9905609330848314, + "flos": 26651499408000.0, + "grad_norm": 2.382003365452772, + "language_loss": 0.6846422, + "learning_rate": 9.228669919778553e-10, + "loss": 0.70631158, + "num_input_tokens_seen": 177735640, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.15515137, + "step": 8238, + "time_per_iteration": 2.853494882583618 + }, + { + "auxiliary_loss_clip": 0.01124854, + "auxiliary_loss_mlp": 0.01036232, + "balance_loss_clip": 1.03998947, + "balance_loss_mlp": 1.02078271, + "epoch": 0.9906811759754705, + "flos": 23111820627840.0, + "grad_norm": 2.700888292474172, + "language_loss": 0.79271501, + "learning_rate": 8.993571855817617e-10, + "loss": 0.81432587, + "num_input_tokens_seen": 177754470, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.15466309, + "step": 8239, + "time_per_iteration": 2.896904468536377 + }, + { + "auxiliary_loss_clip": 0.01123448, + "auxiliary_loss_mlp": 0.01033107, + "balance_loss_clip": 1.03799558, + "balance_loss_mlp": 1.01733541, + "epoch": 0.9908014188661095, + "flos": 22090593052800.0, + "grad_norm": 1.7617540395701177, + "language_loss": 0.74911731, + "learning_rate": 8.761506411638642e-10, + "loss": 0.77068287, + "num_input_tokens_seen": 177773935, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.15783691, + "step": 8240, + "time_per_iteration": 2.8568153381347656 + }, + { + "auxiliary_loss_clip": 0.01124952, + "auxiliary_loss_mlp": 0.01039136, + "balance_loss_clip": 1.03949261, + "balance_loss_mlp": 1.02314985, + "epoch": 0.9909216617567487, + "flos": 19242948677760.0, + "grad_norm": 1.860660161987124, + "language_loss": 0.7378459, + "learning_rate": 8.53247362244236e-10, + "loss": 0.75948679, + "num_input_tokens_seen": 177792745, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.15991211, + "step": 8241, + "time_per_iteration": 2.7786381244659424 + }, + { + "auxiliary_loss_clip": 0.01121616, + "auxiliary_loss_mlp": 0.01037618, + "balance_loss_clip": 1.03896022, + "balance_loss_mlp": 1.02259719, + "epoch": 0.9910419046473877, + "flos": 23621213352960.0, + "grad_norm": 1.5676393243746454, + "language_loss": 0.68278539, + "learning_rate": 8.306473522976532e-10, + "loss": 0.70437777, + "num_input_tokens_seen": 177812150, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.15026855, + "step": 8242, + "time_per_iteration": 2.794935464859009 + }, + { + "auxiliary_loss_clip": 0.01124648, + "auxiliary_loss_mlp": 0.01036045, + "balance_loss_clip": 1.04048514, + "balance_loss_mlp": 1.02060699, + "epoch": 0.9911621475380268, + "flos": 22711380831360.0, + "grad_norm": 1.7445926665900056, + "language_loss": 0.71641469, + "learning_rate": 8.083506147522623e-10, + "loss": 0.73802161, + "num_input_tokens_seen": 177831545, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.15441895, + "step": 8243, + "time_per_iteration": 2.820329427719116 + }, + { + "auxiliary_loss_clip": 0.01122406, + "auxiliary_loss_mlp": 0.01037562, + "balance_loss_clip": 1.0385654, + "balance_loss_mlp": 1.02317953, + "epoch": 0.991282390428666, + "flos": 13516956777600.0, + "grad_norm": 2.7759771054308797, + "language_loss": 0.85518742, + "learning_rate": 7.863571529906909e-10, + "loss": 0.87678707, + "num_input_tokens_seen": 177847130, + "router_z_loss_clip": 0.83935547, + "router_z_loss_mlp": 0.14385986, + "step": 8244, + "time_per_iteration": 2.798107862472534 + }, + { + "auxiliary_loss_clip": 0.01031961, + "auxiliary_loss_mlp": 0.00999588, + "balance_loss_clip": 1.00438178, + "balance_loss_mlp": 0.99835986, + "epoch": 0.991402633319305, + "flos": 61830492071040.0, + "grad_norm": 0.7230654854142139, + "language_loss": 0.59673572, + "learning_rate": 7.646669703489372e-10, + "loss": 0.61705112, + "num_input_tokens_seen": 177911440, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01226807, + "step": 8245, + "time_per_iteration": 4.03278660774231 + }, + { + "auxiliary_loss_clip": 0.01121945, + "auxiliary_loss_mlp": 0.01043984, + "balance_loss_clip": 1.03887832, + "balance_loss_mlp": 1.02880883, + "epoch": 0.9915228762099441, + "flos": 18770148933120.0, + "grad_norm": 1.8726061387217177, + "language_loss": 0.57267857, + "learning_rate": 7.432800701177023e-10, + "loss": 0.59433782, + "num_input_tokens_seen": 177929440, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.15185547, + "step": 8246, + "time_per_iteration": 2.83901309967041 + }, + { + "auxiliary_loss_clip": 0.01031813, + "auxiliary_loss_mlp": 0.01001056, + "balance_loss_clip": 1.00421429, + "balance_loss_mlp": 0.99984592, + "epoch": 0.9916431191005832, + "flos": 65936660244480.0, + "grad_norm": 0.8005603796220656, + "language_loss": 0.57863998, + "learning_rate": 7.221964555415017e-10, + "loss": 0.59896874, + "num_input_tokens_seen": 177989100, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01208496, + "step": 8247, + "time_per_iteration": 3.2764248847961426 + }, + { + "auxiliary_loss_clip": 0.0112068, + "auxiliary_loss_mlp": 0.01033206, + "balance_loss_clip": 1.03737867, + "balance_loss_mlp": 1.01836479, + "epoch": 0.9917633619912223, + "flos": 16581573256320.0, + "grad_norm": 1.9143721346338762, + "language_loss": 0.74976993, + "learning_rate": 7.01416129818222e-10, + "loss": 0.77130884, + "num_input_tokens_seen": 178006720, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.1484375, + "step": 8248, + "time_per_iteration": 2.913646697998047 + }, + { + "auxiliary_loss_clip": 0.01124623, + "auxiliary_loss_mlp": 0.01035757, + "balance_loss_clip": 1.03848195, + "balance_loss_mlp": 1.02112961, + "epoch": 0.9918836048818613, + "flos": 25411108999680.0, + "grad_norm": 2.1331174462223292, + "language_loss": 0.58928609, + "learning_rate": 6.809390961006745e-10, + "loss": 0.61088991, + "num_input_tokens_seen": 178026850, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.14630127, + "step": 8249, + "time_per_iteration": 3.661184072494507 + }, + { + "auxiliary_loss_clip": 0.01123968, + "auxiliary_loss_mlp": 0.01041482, + "balance_loss_clip": 1.03940296, + "balance_loss_mlp": 1.02604485, + "epoch": 0.9920038477725005, + "flos": 25046867134080.0, + "grad_norm": 1.7280027698772824, + "language_loss": 0.68597996, + "learning_rate": 6.607653574948191e-10, + "loss": 0.70763451, + "num_input_tokens_seen": 178047630, + "router_z_loss_clip": 0.84667969, + "router_z_loss_mlp": 0.15435791, + "step": 8250, + "time_per_iteration": 2.884185791015625 + }, + { + "auxiliary_loss_clip": 0.01122412, + "auxiliary_loss_mlp": 0.01031668, + "balance_loss_clip": 1.03898489, + "balance_loss_mlp": 1.01767313, + "epoch": 0.9921240906631396, + "flos": 21829773421440.0, + "grad_norm": 1.9083985651512978, + "language_loss": 0.81852227, + "learning_rate": 6.408949170613187e-10, + "loss": 0.8400631, + "num_input_tokens_seen": 178066895, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.13995361, + "step": 8251, + "time_per_iteration": 2.795323610305786 + }, + { + "auxiliary_loss_clip": 0.01126982, + "auxiliary_loss_mlp": 0.01039706, + "balance_loss_clip": 1.04063678, + "balance_loss_mlp": 1.02187228, + "epoch": 0.9922443335537786, + "flos": 24864225454080.0, + "grad_norm": 1.6128334607739796, + "language_loss": 0.81904703, + "learning_rate": 6.213277778144288e-10, + "loss": 0.84071392, + "num_input_tokens_seen": 178088540, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.1784668, + "step": 8252, + "time_per_iteration": 3.652036190032959 + }, + { + "auxiliary_loss_clip": 0.01125205, + "auxiliary_loss_mlp": 0.01040709, + "balance_loss_clip": 1.03815281, + "balance_loss_mlp": 1.02468753, + "epoch": 0.9923645764444178, + "flos": 21613088626560.0, + "grad_norm": 2.5426032489818744, + "language_loss": 0.67419297, + "learning_rate": 6.020639427224416e-10, + "loss": 0.6958521, + "num_input_tokens_seen": 178106185, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.16027832, + "step": 8253, + "time_per_iteration": 2.8119451999664307 + }, + { + "auxiliary_loss_clip": 0.01122248, + "auxiliary_loss_mlp": 0.01035784, + "balance_loss_clip": 1.03877687, + "balance_loss_mlp": 1.02015615, + "epoch": 0.9924848193350568, + "flos": 25001798544000.0, + "grad_norm": 1.9056754356603076, + "language_loss": 0.72715759, + "learning_rate": 5.831034147076864e-10, + "loss": 0.74873793, + "num_input_tokens_seen": 178123435, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.15637207, + "step": 8254, + "time_per_iteration": 3.6956331729888916 + }, + { + "auxiliary_loss_clip": 0.01031923, + "auxiliary_loss_mlp": 0.01000295, + "balance_loss_clip": 1.00427437, + "balance_loss_mlp": 0.99904901, + "epoch": 0.9926050622256959, + "flos": 68912543151360.0, + "grad_norm": 0.6883548162036446, + "language_loss": 0.55696988, + "learning_rate": 5.644461966463065e-10, + "loss": 0.57729208, + "num_input_tokens_seen": 178191045, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01245117, + "step": 8255, + "time_per_iteration": 3.4331700801849365 + }, + { + "auxiliary_loss_clip": 0.01122254, + "auxiliary_loss_mlp": 0.01041324, + "balance_loss_clip": 1.03956509, + "balance_loss_mlp": 1.02655947, + "epoch": 0.9927253051163349, + "flos": 20923675914240.0, + "grad_norm": 1.6446151299987666, + "language_loss": 0.75743425, + "learning_rate": 5.460922913687049e-10, + "loss": 0.77907002, + "num_input_tokens_seen": 178210135, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.14776611, + "step": 8256, + "time_per_iteration": 2.810981035232544 + }, + { + "auxiliary_loss_clip": 0.01126301, + "auxiliary_loss_mlp": 0.01034246, + "balance_loss_clip": 1.03920758, + "balance_loss_mlp": 1.01709223, + "epoch": 0.9928455480069741, + "flos": 22308211601280.0, + "grad_norm": 1.8851496348856676, + "language_loss": 0.75461972, + "learning_rate": 5.280417016593208e-10, + "loss": 0.77622515, + "num_input_tokens_seen": 178229925, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.17150879, + "step": 8257, + "time_per_iteration": 2.8002512454986572 + }, + { + "auxiliary_loss_clip": 0.01123168, + "auxiliary_loss_mlp": 0.0103811, + "balance_loss_clip": 1.04234505, + "balance_loss_mlp": 1.02354836, + "epoch": 0.9929657908976132, + "flos": 17383889393280.0, + "grad_norm": 1.636686927533411, + "language_loss": 0.74660534, + "learning_rate": 5.102944302559642e-10, + "loss": 0.7682181, + "num_input_tokens_seen": 178247420, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.14544678, + "step": 8258, + "time_per_iteration": 2.9583330154418945 + }, + { + "auxiliary_loss_clip": 0.01132832, + "auxiliary_loss_mlp": 0.01041532, + "balance_loss_clip": 1.04260051, + "balance_loss_mlp": 1.02606487, + "epoch": 0.9930860337882522, + "flos": 22674680110080.0, + "grad_norm": 1.9329977483234808, + "language_loss": 0.79580176, + "learning_rate": 4.9285047985137e-10, + "loss": 0.81754541, + "num_input_tokens_seen": 178266840, + "router_z_loss_clip": 0.90283203, + "router_z_loss_mlp": 0.15460205, + "step": 8259, + "time_per_iteration": 2.852898120880127 + }, + { + "auxiliary_loss_clip": 0.01128632, + "auxiliary_loss_mlp": 0.0104991, + "balance_loss_clip": 1.04164553, + "balance_loss_mlp": 1.03518748, + "epoch": 0.9932062766788914, + "flos": 28147789284480.0, + "grad_norm": 1.9748165914464835, + "language_loss": 0.74489295, + "learning_rate": 4.757098530916436e-10, + "loss": 0.76667833, + "num_input_tokens_seen": 178287285, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.14715576, + "step": 8260, + "time_per_iteration": 2.829692840576172 + }, + { + "auxiliary_loss_clip": 0.01130654, + "auxiliary_loss_mlp": 0.01043214, + "balance_loss_clip": 1.04230165, + "balance_loss_mlp": 1.02758551, + "epoch": 0.9933265195695304, + "flos": 20156659868160.0, + "grad_norm": 2.8914683034548365, + "language_loss": 0.77250946, + "learning_rate": 4.5887255257670563e-10, + "loss": 0.79424816, + "num_input_tokens_seen": 178304325, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.15637207, + "step": 8261, + "time_per_iteration": 2.786109685897827 + }, + { + "auxiliary_loss_clip": 0.01124305, + "auxiliary_loss_mlp": 0.01041801, + "balance_loss_clip": 1.03879035, + "balance_loss_mlp": 1.02513003, + "epoch": 0.9934467624601695, + "flos": 21362037494400.0, + "grad_norm": 2.7097669236335036, + "language_loss": 0.77089298, + "learning_rate": 4.4233858086117906e-10, + "loss": 0.79255402, + "num_input_tokens_seen": 178322850, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.16662598, + "step": 8262, + "time_per_iteration": 2.787424325942993 + }, + { + "auxiliary_loss_clip": 0.0112689, + "auxiliary_loss_mlp": 0.01050409, + "balance_loss_clip": 1.04097986, + "balance_loss_mlp": 1.03510249, + "epoch": 0.9935670053508087, + "flos": 19756040503680.0, + "grad_norm": 4.867282198792473, + "language_loss": 0.68044931, + "learning_rate": 4.261079404528356e-10, + "loss": 0.70222229, + "num_input_tokens_seen": 178342330, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.1529541, + "step": 8263, + "time_per_iteration": 2.8301925659179688 + }, + { + "auxiliary_loss_clip": 0.01129881, + "auxiliary_loss_mlp": 0.01041129, + "balance_loss_clip": 1.04208517, + "balance_loss_mlp": 1.02413547, + "epoch": 0.9936872482414477, + "flos": 21978838863360.0, + "grad_norm": 1.9119059427868121, + "language_loss": 0.69258463, + "learning_rate": 4.1018063381437205e-10, + "loss": 0.71429479, + "num_input_tokens_seen": 178362715, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.17010498, + "step": 8264, + "time_per_iteration": 2.7331955432891846 + }, + { + "auxiliary_loss_clip": 0.0103189, + "auxiliary_loss_mlp": 0.00999278, + "balance_loss_clip": 1.00425863, + "balance_loss_mlp": 0.99796027, + "epoch": 0.9938074911320868, + "flos": 69810667839360.0, + "grad_norm": 0.8648704374330425, + "language_loss": 0.61041892, + "learning_rate": 3.9455666336141167e-10, + "loss": 0.63073051, + "num_input_tokens_seen": 178426495, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01318359, + "step": 8265, + "time_per_iteration": 3.456169366836548 + }, + { + "auxiliary_loss_clip": 0.01124485, + "auxiliary_loss_mlp": 0.01038199, + "balance_loss_clip": 1.04076695, + "balance_loss_mlp": 1.02264822, + "epoch": 0.9939277340227259, + "flos": 15084170058240.0, + "grad_norm": 2.688528835725869, + "language_loss": 0.83275801, + "learning_rate": 3.7923603146450267e-10, + "loss": 0.85438484, + "num_input_tokens_seen": 178442555, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.15557861, + "step": 8266, + "time_per_iteration": 2.7553412914276123 + }, + { + "auxiliary_loss_clip": 0.01125645, + "auxiliary_loss_mlp": 0.01034897, + "balance_loss_clip": 1.03920841, + "balance_loss_mlp": 1.02054429, + "epoch": 0.994047976913365, + "flos": 17712364291200.0, + "grad_norm": 1.9605805511115983, + "language_loss": 0.80920172, + "learning_rate": 3.642187404473418e-10, + "loss": 0.83080721, + "num_input_tokens_seen": 178460715, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.14379883, + "step": 8267, + "time_per_iteration": 2.808150291442871 + }, + { + "auxiliary_loss_clip": 0.01122869, + "auxiliary_loss_mlp": 0.01040118, + "balance_loss_clip": 1.03761125, + "balance_loss_mlp": 1.02503228, + "epoch": 0.994168219804004, + "flos": 19171558396800.0, + "grad_norm": 2.8362422587364753, + "language_loss": 0.86101902, + "learning_rate": 3.495047925885508e-10, + "loss": 0.88264883, + "num_input_tokens_seen": 178479050, + "router_z_loss_clip": 0.85205078, + "router_z_loss_mlp": 0.15106201, + "step": 8268, + "time_per_iteration": 2.7680208683013916 + }, + { + "auxiliary_loss_clip": 0.01124458, + "auxiliary_loss_mlp": 0.01042065, + "balance_loss_clip": 1.03905249, + "balance_loss_mlp": 1.02656746, + "epoch": 0.9942884626946432, + "flos": 17851589406720.0, + "grad_norm": 2.4163991856635514, + "language_loss": 0.83050013, + "learning_rate": 3.350941901199e-10, + "loss": 0.85216534, + "num_input_tokens_seen": 178495970, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.15490723, + "step": 8269, + "time_per_iteration": 2.85084867477417 + }, + { + "auxiliary_loss_clip": 0.0112419, + "auxiliary_loss_mlp": 0.01038721, + "balance_loss_clip": 1.04027224, + "balance_loss_mlp": 1.02268779, + "epoch": 0.9944087055852823, + "flos": 18796578364800.0, + "grad_norm": 2.4296613147837114, + "language_loss": 0.83610439, + "learning_rate": 3.2098693522764066e-10, + "loss": 0.85773349, + "num_input_tokens_seen": 178509170, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.16015625, + "step": 8270, + "time_per_iteration": 2.940730094909668 + }, + { + "auxiliary_loss_clip": 0.01126358, + "auxiliary_loss_mlp": 0.01037333, + "balance_loss_clip": 1.04075241, + "balance_loss_mlp": 1.02164483, + "epoch": 0.9945289484759213, + "flos": 20996969616000.0, + "grad_norm": 1.8644364640353241, + "language_loss": 0.81323028, + "learning_rate": 3.071830300516165e-10, + "loss": 0.83486718, + "num_input_tokens_seen": 178527000, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.15673828, + "step": 8271, + "time_per_iteration": 2.8787906169891357 + }, + { + "auxiliary_loss_clip": 0.0113176, + "auxiliary_loss_mlp": 0.01044652, + "balance_loss_clip": 1.04261065, + "balance_loss_mlp": 1.02785552, + "epoch": 0.9946491913665605, + "flos": 14756952136320.0, + "grad_norm": 1.9620386014933051, + "language_loss": 0.70531851, + "learning_rate": 2.9368247668615234e-10, + "loss": 0.72708261, + "num_input_tokens_seen": 178545590, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.16784668, + "step": 8272, + "time_per_iteration": 2.8410542011260986 + }, + { + "auxiliary_loss_clip": 0.01132266, + "auxiliary_loss_mlp": 0.01041303, + "balance_loss_clip": 1.04259408, + "balance_loss_mlp": 1.02419674, + "epoch": 0.9947694342571995, + "flos": 12669931186560.0, + "grad_norm": 2.633614788625469, + "language_loss": 0.6188972, + "learning_rate": 2.804852771789434e-10, + "loss": 0.64063287, + "num_input_tokens_seen": 178558890, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.17102051, + "step": 8273, + "time_per_iteration": 2.826859951019287 + }, + { + "auxiliary_loss_clip": 0.01121213, + "auxiliary_loss_mlp": 0.01035859, + "balance_loss_clip": 1.03922105, + "balance_loss_mlp": 1.02139258, + "epoch": 0.9948896771478386, + "flos": 18843442634880.0, + "grad_norm": 11.294959383796602, + "language_loss": 0.55635488, + "learning_rate": 2.675914335321661e-10, + "loss": 0.57792568, + "num_input_tokens_seen": 178577645, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.14483643, + "step": 8274, + "time_per_iteration": 3.6431400775909424 + }, + { + "auxiliary_loss_clip": 0.01132348, + "auxiliary_loss_mlp": 0.01041061, + "balance_loss_clip": 1.04111803, + "balance_loss_mlp": 1.02399039, + "epoch": 0.9950099200384778, + "flos": 24900207903360.0, + "grad_norm": 2.1613018705203024, + "language_loss": 0.78784257, + "learning_rate": 2.550009477018111e-10, + "loss": 0.80957663, + "num_input_tokens_seen": 178596415, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.1706543, + "step": 8275, + "time_per_iteration": 2.8750946521759033 + }, + { + "auxiliary_loss_clip": 0.01123969, + "auxiliary_loss_mlp": 0.0104473, + "balance_loss_clip": 1.03906548, + "balance_loss_mlp": 1.03040075, + "epoch": 0.9951301629291168, + "flos": 23733613987200.0, + "grad_norm": 2.114813147802362, + "language_loss": 0.63099521, + "learning_rate": 2.4271382159790634e-10, + "loss": 0.65268219, + "num_input_tokens_seen": 178613845, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14331055, + "step": 8276, + "time_per_iteration": 2.8378589153289795 + }, + { + "auxiliary_loss_clip": 0.01123868, + "auxiliary_loss_mlp": 0.01045498, + "balance_loss_clip": 1.04063058, + "balance_loss_mlp": 1.029953, + "epoch": 0.9952504058197559, + "flos": 22236893147520.0, + "grad_norm": 2.33351478740675, + "language_loss": 0.85987866, + "learning_rate": 2.3073005708429406e-10, + "loss": 0.88157237, + "num_input_tokens_seen": 178633490, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.15563965, + "step": 8277, + "time_per_iteration": 2.9996278285980225 + }, + { + "auxiliary_loss_clip": 0.01121615, + "auxiliary_loss_mlp": 0.01038251, + "balance_loss_clip": 1.0388341, + "balance_loss_mlp": 1.02416086, + "epoch": 0.995370648710395, + "flos": 21211032718080.0, + "grad_norm": 1.7153910763203895, + "language_loss": 0.7199381, + "learning_rate": 2.190496559788535e-10, + "loss": 0.74153674, + "num_input_tokens_seen": 178651775, + "router_z_loss_clip": 0.828125, + "router_z_loss_mlp": 0.14093018, + "step": 8278, + "time_per_iteration": 3.5966899394989014 + }, + { + "auxiliary_loss_clip": 0.01126902, + "auxiliary_loss_mlp": 0.01038431, + "balance_loss_clip": 1.04006243, + "balance_loss_mlp": 1.02296388, + "epoch": 0.9954908916010341, + "flos": 14866731077760.0, + "grad_norm": 2.5455930280090833, + "language_loss": 0.76739854, + "learning_rate": 2.0767262005372265e-10, + "loss": 0.78905189, + "num_input_tokens_seen": 178669290, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.15472412, + "step": 8279, + "time_per_iteration": 2.994351863861084 + }, + { + "auxiliary_loss_clip": 0.01124667, + "auxiliary_loss_mlp": 0.01034286, + "balance_loss_clip": 1.03865552, + "balance_loss_mlp": 1.01925969, + "epoch": 0.9956111344916732, + "flos": 19208259118080.0, + "grad_norm": 1.7622544148587982, + "language_loss": 0.75090468, + "learning_rate": 1.965989510346322e-10, + "loss": 0.7724942, + "num_input_tokens_seen": 178688410, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.15026855, + "step": 8280, + "time_per_iteration": 2.9123222827911377 + }, + { + "auxiliary_loss_clip": 0.01126853, + "auxiliary_loss_mlp": 0.01036441, + "balance_loss_clip": 1.03900361, + "balance_loss_mlp": 1.02024043, + "epoch": 0.9957313773823123, + "flos": 20047060494720.0, + "grad_norm": 1.9168243146613806, + "language_loss": 0.7114836, + "learning_rate": 1.8582865060134955e-10, + "loss": 0.73311651, + "num_input_tokens_seen": 178706600, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.16186523, + "step": 8281, + "time_per_iteration": 3.63614821434021 + }, + { + "auxiliary_loss_clip": 0.01032028, + "auxiliary_loss_mlp": 0.01002896, + "balance_loss_clip": 1.00443792, + "balance_loss_mlp": 1.00159669, + "epoch": 0.9958516202729514, + "flos": 57483253768320.0, + "grad_norm": 0.8234092947260883, + "language_loss": 0.55715352, + "learning_rate": 1.7536172038790098e-10, + "loss": 0.57750273, + "num_input_tokens_seen": 178766910, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.01300049, + "step": 8282, + "time_per_iteration": 3.413656711578369 + }, + { + "auxiliary_loss_clip": 0.01126784, + "auxiliary_loss_mlp": 0.01041195, + "balance_loss_clip": 1.04157054, + "balance_loss_mlp": 1.02597225, + "epoch": 0.9959718631635904, + "flos": 27782900974080.0, + "grad_norm": 2.212751400101354, + "language_loss": 0.69507563, + "learning_rate": 1.651981619819054e-10, + "loss": 0.71675539, + "num_input_tokens_seen": 178784060, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.15228271, + "step": 8283, + "time_per_iteration": 3.776904344558716 + }, + { + "auxiliary_loss_clip": 0.01125439, + "auxiliary_loss_mlp": 0.01028398, + "balance_loss_clip": 1.03934979, + "balance_loss_mlp": 1.01397967, + "epoch": 0.9960921060542296, + "flos": 24024095274240.0, + "grad_norm": 2.184519228080359, + "language_loss": 0.70380598, + "learning_rate": 1.5533797692546257e-10, + "loss": 0.72534436, + "num_input_tokens_seen": 178802795, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.14422607, + "step": 8284, + "time_per_iteration": 2.8031256198883057 + }, + { + "auxiliary_loss_clip": 0.01126854, + "auxiliary_loss_mlp": 0.01035695, + "balance_loss_clip": 1.03883755, + "balance_loss_mlp": 1.02020991, + "epoch": 0.9962123489448687, + "flos": 18697393935360.0, + "grad_norm": 2.303377086791211, + "language_loss": 0.84217227, + "learning_rate": 1.4578116671404296e-10, + "loss": 0.86379772, + "num_input_tokens_seen": 178821075, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.15478516, + "step": 8285, + "time_per_iteration": 2.877659320831299 + }, + { + "auxiliary_loss_clip": 0.0112556, + "auxiliary_loss_mlp": 0.01042633, + "balance_loss_clip": 1.04165411, + "balance_loss_mlp": 1.02778542, + "epoch": 0.9963325918355077, + "flos": 20010754823040.0, + "grad_norm": 2.0347375283219624, + "language_loss": 0.7112506, + "learning_rate": 1.3652773279759777e-10, + "loss": 0.73293251, + "num_input_tokens_seen": 178837725, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.14855957, + "step": 8286, + "time_per_iteration": 2.791185140609741 + }, + { + "auxiliary_loss_clip": 0.01126338, + "auxiliary_loss_mlp": 0.0104048, + "balance_loss_clip": 1.04030204, + "balance_loss_mlp": 1.0246315, + "epoch": 0.9964528347261468, + "flos": 33108488991360.0, + "grad_norm": 1.6196948084169078, + "language_loss": 0.62861246, + "learning_rate": 1.2757767657989305e-10, + "loss": 0.65028059, + "num_input_tokens_seen": 178861515, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.1585083, + "step": 8287, + "time_per_iteration": 2.833238124847412 + }, + { + "auxiliary_loss_clip": 0.011253, + "auxiliary_loss_mlp": 0.0103981, + "balance_loss_clip": 1.03978753, + "balance_loss_mlp": 1.02490354, + "epoch": 0.9965730776167859, + "flos": 23109342589440.0, + "grad_norm": 1.8845729843284984, + "language_loss": 0.87030488, + "learning_rate": 1.1893099941850948e-10, + "loss": 0.89195597, + "num_input_tokens_seen": 178880410, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.14910889, + "step": 8288, + "time_per_iteration": 2.8319783210754395 + }, + { + "auxiliary_loss_clip": 0.01129132, + "auxiliary_loss_mlp": 0.01031026, + "balance_loss_clip": 1.04053378, + "balance_loss_mlp": 1.01555824, + "epoch": 0.996693320507425, + "flos": 22965843755520.0, + "grad_norm": 2.0028651323858524, + "language_loss": 0.77138078, + "learning_rate": 1.105877026252866e-10, + "loss": 0.79298234, + "num_input_tokens_seen": 178898740, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.15466309, + "step": 8289, + "time_per_iteration": 2.8163390159606934 + }, + { + "auxiliary_loss_clip": 0.011282, + "auxiliary_loss_mlp": 0.01039437, + "balance_loss_clip": 1.04104257, + "balance_loss_mlp": 1.02323616, + "epoch": 0.996813563398064, + "flos": 13222740476160.0, + "grad_norm": 2.0387369908651447, + "language_loss": 0.71898836, + "learning_rate": 1.0254778746565663e-10, + "loss": 0.74066472, + "num_input_tokens_seen": 178914015, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.16186523, + "step": 8290, + "time_per_iteration": 2.8932137489318848 + }, + { + "auxiliary_loss_clip": 0.01120049, + "auxiliary_loss_mlp": 0.01038719, + "balance_loss_clip": 1.03899348, + "balance_loss_mlp": 1.0242703, + "epoch": 0.9969338062887032, + "flos": 14647855553280.0, + "grad_norm": 2.0095426872354567, + "language_loss": 0.73032963, + "learning_rate": 9.481125515953259e-11, + "loss": 0.7519173, + "num_input_tokens_seen": 178932075, + "router_z_loss_clip": 0.81054688, + "router_z_loss_mlp": 0.14440918, + "step": 8291, + "time_per_iteration": 2.951951503753662 + }, + { + "auxiliary_loss_clip": 0.01124253, + "auxiliary_loss_mlp": 0.010383, + "balance_loss_clip": 1.03953385, + "balance_loss_mlp": 1.02336872, + "epoch": 0.9970540491793423, + "flos": 25735741142400.0, + "grad_norm": 4.142887021303506, + "language_loss": 0.80028987, + "learning_rate": 8.737810688064228e-11, + "loss": 0.82191539, + "num_input_tokens_seen": 178951910, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.14941406, + "step": 8292, + "time_per_iteration": 2.9149844646453857 + }, + { + "auxiliary_loss_clip": 0.01131894, + "auxiliary_loss_mlp": 0.01042308, + "balance_loss_clip": 1.04150748, + "balance_loss_mlp": 1.02585673, + "epoch": 0.9971742920699813, + "flos": 21470236237440.0, + "grad_norm": 1.988919450994899, + "language_loss": 0.79340065, + "learning_rate": 8.024834375608414e-11, + "loss": 0.81514269, + "num_input_tokens_seen": 178970500, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.16442871, + "step": 8293, + "time_per_iteration": 3.0338337421417236 + }, + { + "auxiliary_loss_clip": 0.01031644, + "auxiliary_loss_mlp": 0.01001964, + "balance_loss_clip": 1.00415194, + "balance_loss_mlp": 1.00069416, + "epoch": 0.9972945349606205, + "flos": 72211223629440.0, + "grad_norm": 0.8223688195628194, + "language_loss": 0.62845898, + "learning_rate": 7.342196686788149e-11, + "loss": 0.64879507, + "num_input_tokens_seen": 179023665, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.01269531, + "step": 8294, + "time_per_iteration": 3.289482831954956 + }, + { + "auxiliary_loss_clip": 0.01129785, + "auxiliary_loss_mlp": 0.01047312, + "balance_loss_clip": 1.04154539, + "balance_loss_mlp": 1.03217232, + "epoch": 0.9974147778512595, + "flos": 19678293515520.0, + "grad_norm": 1.815662970449116, + "language_loss": 0.68856865, + "learning_rate": 6.689897725142834e-11, + "loss": 0.71033961, + "num_input_tokens_seen": 179043140, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15136719, + "step": 8295, + "time_per_iteration": 2.920555353164673 + }, + { + "auxiliary_loss_clip": 0.01123402, + "auxiliary_loss_mlp": 0.01036967, + "balance_loss_clip": 1.03874719, + "balance_loss_mlp": 1.0213505, + "epoch": 0.9975350207418986, + "flos": 15960821391360.0, + "grad_norm": 2.1006873295295776, + "language_loss": 0.8822453, + "learning_rate": 6.067937589615545e-11, + "loss": 0.90384895, + "num_input_tokens_seen": 179061215, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.15612793, + "step": 8296, + "time_per_iteration": 2.8353164196014404 + }, + { + "auxiliary_loss_clip": 0.01031913, + "auxiliary_loss_mlp": 0.01001537, + "balance_loss_clip": 1.00429296, + "balance_loss_mlp": 1.00024414, + "epoch": 0.9976552636325378, + "flos": 59961879768960.0, + "grad_norm": 0.7505798561083857, + "language_loss": 0.57686639, + "learning_rate": 5.476316374575241e-11, + "loss": 0.59720087, + "num_input_tokens_seen": 179124700, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.01293945, + "step": 8297, + "time_per_iteration": 3.314850330352783 + }, + { + "auxiliary_loss_clip": 0.01125615, + "auxiliary_loss_mlp": 0.01043252, + "balance_loss_clip": 1.03866744, + "balance_loss_mlp": 1.02678967, + "epoch": 0.9977755065231768, + "flos": 22487872452480.0, + "grad_norm": 1.8628458063261537, + "language_loss": 0.7256726, + "learning_rate": 4.9150341697723476e-11, + "loss": 0.7473613, + "num_input_tokens_seen": 179144590, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.16467285, + "step": 8298, + "time_per_iteration": 2.9150891304016113 + }, + { + "auxiliary_loss_clip": 0.0112976, + "auxiliary_loss_mlp": 0.01041548, + "balance_loss_clip": 1.04395926, + "balance_loss_mlp": 1.02562201, + "epoch": 0.9978957494138159, + "flos": 26030280666240.0, + "grad_norm": 2.1036911666550315, + "language_loss": 0.66789806, + "learning_rate": 4.384091060338768e-11, + "loss": 0.6896112, + "num_input_tokens_seen": 179165060, + "router_z_loss_clip": 0.85742188, + "router_z_loss_mlp": 0.15924072, + "step": 8299, + "time_per_iteration": 2.9057681560516357 + }, + { + "auxiliary_loss_clip": 0.01122697, + "auxiliary_loss_mlp": 0.0103921, + "balance_loss_clip": 1.03896999, + "balance_loss_mlp": 1.02362919, + "epoch": 0.998015992304455, + "flos": 22637835734400.0, + "grad_norm": 2.3357735178803263, + "language_loss": 0.74111217, + "learning_rate": 3.883487126810081e-11, + "loss": 0.76273125, + "num_input_tokens_seen": 179184320, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.15576172, + "step": 8300, + "time_per_iteration": 2.9192206859588623 + }, + { + "auxiliary_loss_clip": 0.0112209, + "auxiliary_loss_mlp": 0.01036414, + "balance_loss_clip": 1.03908944, + "balance_loss_mlp": 1.02190626, + "epoch": 0.9981362351950941, + "flos": 18223444955520.0, + "grad_norm": 1.8115089296996334, + "language_loss": 0.79162848, + "learning_rate": 3.41322244516995e-11, + "loss": 0.81321353, + "num_input_tokens_seen": 179202265, + "router_z_loss_clip": 0.83105469, + "router_z_loss_mlp": 0.1449585, + "step": 8301, + "time_per_iteration": 2.805922031402588 + }, + { + "auxiliary_loss_clip": 0.01123865, + "auxiliary_loss_mlp": 0.01036426, + "balance_loss_clip": 1.03788519, + "balance_loss_mlp": 1.02119112, + "epoch": 0.9982564780857331, + "flos": 33474095573760.0, + "grad_norm": 1.7797855150521456, + "language_loss": 0.63123947, + "learning_rate": 2.9732970866946925e-11, + "loss": 0.6528424, + "num_input_tokens_seen": 179222145, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15222168, + "step": 8302, + "time_per_iteration": 2.895575761795044 + }, + { + "auxiliary_loss_clip": 0.01128359, + "auxiliary_loss_mlp": 0.01039133, + "balance_loss_clip": 1.03885818, + "balance_loss_mlp": 1.02183628, + "epoch": 0.9983767209763723, + "flos": 15523465392000.0, + "grad_norm": 2.126698255855645, + "language_loss": 0.77882588, + "learning_rate": 2.563711118175327e-11, + "loss": 0.80050081, + "num_input_tokens_seen": 179239030, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.17297363, + "step": 8303, + "time_per_iteration": 3.559765100479126 + }, + { + "auxiliary_loss_clip": 0.01123547, + "auxiliary_loss_mlp": 0.01033845, + "balance_loss_clip": 1.04072952, + "balance_loss_mlp": 1.02004051, + "epoch": 0.9984969638670114, + "flos": 19974377324160.0, + "grad_norm": 2.363065710413715, + "language_loss": 0.83842373, + "learning_rate": 2.184464601717728e-11, + "loss": 0.85999763, + "num_input_tokens_seen": 179257345, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.13806152, + "step": 8304, + "time_per_iteration": 2.8511366844177246 + }, + { + "auxiliary_loss_clip": 0.01127646, + "auxiliary_loss_mlp": 0.01042716, + "balance_loss_clip": 1.0414331, + "balance_loss_mlp": 1.02734959, + "epoch": 0.9986172067576504, + "flos": 20375750874240.0, + "grad_norm": 2.494039599904145, + "language_loss": 0.77605975, + "learning_rate": 1.8355575948758585e-11, + "loss": 0.79776335, + "num_input_tokens_seen": 179275330, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.15368652, + "step": 8305, + "time_per_iteration": 2.9325520992279053 + }, + { + "auxiliary_loss_clip": 0.01127812, + "auxiliary_loss_mlp": 0.01041439, + "balance_loss_clip": 1.04037428, + "balance_loss_mlp": 1.02618003, + "epoch": 0.9987374496482896, + "flos": 23727903724800.0, + "grad_norm": 2.24635084854164, + "language_loss": 0.73765028, + "learning_rate": 1.5169901505407424e-11, + "loss": 0.75934279, + "num_input_tokens_seen": 179292395, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.15258789, + "step": 8306, + "time_per_iteration": 3.663205862045288 + }, + { + "auxiliary_loss_clip": 0.01121709, + "auxiliary_loss_mlp": 0.01034134, + "balance_loss_clip": 1.03775525, + "balance_loss_mlp": 1.01945901, + "epoch": 0.9988576925389286, + "flos": 25044029959680.0, + "grad_norm": 2.008872831761696, + "language_loss": 0.73943704, + "learning_rate": 1.228762317073695e-11, + "loss": 0.76099551, + "num_input_tokens_seen": 179311225, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.14672852, + "step": 8307, + "time_per_iteration": 2.8569624423980713 + }, + { + "auxiliary_loss_clip": 0.01120394, + "auxiliary_loss_mlp": 0.01035459, + "balance_loss_clip": 1.03795695, + "balance_loss_mlp": 1.02008724, + "epoch": 0.9989779354295677, + "flos": 31285627637760.0, + "grad_norm": 4.25242941075593, + "language_loss": 0.79219908, + "learning_rate": 9.70874138195299e-12, + "loss": 0.81375754, + "num_input_tokens_seen": 179333135, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.15380859, + "step": 8308, + "time_per_iteration": 2.926264524459839 + }, + { + "auxiliary_loss_clip": 0.01127886, + "auxiliary_loss_mlp": 0.01038664, + "balance_loss_clip": 1.03962111, + "balance_loss_mlp": 1.02302396, + "epoch": 0.9990981783202069, + "flos": 19573398823680.0, + "grad_norm": 1.5261978138896664, + "language_loss": 0.74664491, + "learning_rate": 7.433256530076093e-12, + "loss": 0.76831043, + "num_input_tokens_seen": 179353090, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.15637207, + "step": 8309, + "time_per_iteration": 2.8140876293182373 + }, + { + "auxiliary_loss_clip": 0.0112199, + "auxiliary_loss_mlp": 0.01027532, + "balance_loss_clip": 1.03917933, + "balance_loss_mlp": 1.01322699, + "epoch": 0.9992184212108459, + "flos": 17199667514880.0, + "grad_norm": 1.9990677446691358, + "language_loss": 0.75444299, + "learning_rate": 5.46116896038562e-12, + "loss": 0.77593821, + "num_input_tokens_seen": 179367500, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.14318848, + "step": 8310, + "time_per_iteration": 3.5957279205322266 + }, + { + "auxiliary_loss_clip": 0.01127064, + "auxiliary_loss_mlp": 0.01038849, + "balance_loss_clip": 1.04067159, + "balance_loss_mlp": 1.02401328, + "epoch": 0.999338664101485, + "flos": 46497853681920.0, + "grad_norm": 1.8791750608770572, + "language_loss": 0.6180793, + "learning_rate": 3.792478972197699e-12, + "loss": 0.63973844, + "num_input_tokens_seen": 179388085, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.14849854, + "step": 8311, + "time_per_iteration": 2.9923226833343506 + }, + { + "auxiliary_loss_clip": 0.01121199, + "auxiliary_loss_mlp": 0.01039248, + "balance_loss_clip": 1.03630841, + "balance_loss_mlp": 1.02438903, + "epoch": 0.9994589069921241, + "flos": 15158253859200.0, + "grad_norm": 2.7793745867678363, + "language_loss": 0.70575523, + "learning_rate": 2.4271868181990895e-12, + "loss": 0.72735977, + "num_input_tokens_seen": 179405250, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.14855957, + "step": 8312, + "time_per_iteration": 3.7800915241241455 + }, + { + "auxiliary_loss_clip": 0.01124165, + "auxiliary_loss_mlp": 0.01037599, + "balance_loss_clip": 1.03879786, + "balance_loss_mlp": 1.02161348, + "epoch": 0.9995791498827632, + "flos": 12531460256640.0, + "grad_norm": 3.0416253754112828, + "language_loss": 0.8134625, + "learning_rate": 1.3652927060014973e-12, + "loss": 0.83508015, + "num_input_tokens_seen": 179420845, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.15979004, + "step": 8313, + "time_per_iteration": 2.7920541763305664 + }, + { + "auxiliary_loss_clip": 0.01128789, + "auxiliary_loss_mlp": 0.01037727, + "balance_loss_clip": 1.04065299, + "balance_loss_mlp": 1.02161038, + "epoch": 0.9996993927734023, + "flos": 19245175320960.0, + "grad_norm": 1.965198535492122, + "language_loss": 0.63768613, + "learning_rate": 6.067967965872612e-13, + "loss": 0.65935135, + "num_input_tokens_seen": 179440455, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.16101074, + "step": 8314, + "time_per_iteration": 2.8270111083984375 + }, + { + "auxiliary_loss_clip": 0.01126321, + "auxiliary_loss_mlp": 0.01035686, + "balance_loss_clip": 1.04034019, + "balance_loss_mlp": 1.02048063, + "epoch": 0.9998196356640414, + "flos": 62952804518400.0, + "grad_norm": 2.6538563550302827, + "language_loss": 0.77015293, + "learning_rate": 1.5169920497548615e-13, + "loss": 0.79177302, + "num_input_tokens_seen": 179465075, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.15216064, + "step": 8315, + "time_per_iteration": 3.167142868041992 + }, + { + "auxiliary_loss_clip": 0.01081853, + "auxiliary_loss_mlp": 0.01016971, + "balance_loss_clip": 1.0226835, + "balance_loss_mlp": 1.00848317, + "epoch": 0.9999398785546805, + "flos": 50922375073920.0, + "grad_norm": 1.1222269924074246, + "language_loss": 0.54995394, + "learning_rate": 0.0, + "loss": 0.57094216, + "num_input_tokens_seen": 179513955, + "router_z_loss_clip": 0.59130859, + "router_z_loss_mlp": 0.08493042, + "step": 8316, + "time_per_iteration": 3.3832051753997803 + } + ], + "logging_steps": 1.0, + "max_steps": 8316, + "num_input_tokens_seen": 179513955, + "num_train_epochs": 1, + "save_steps": 3328, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 6.996749092776837e+17, + "train_batch_size": 5, + "trial_name": null, + "trial_params": null +}