{ "best_metric": null, "best_model_checkpoint": null, "epoch": 0.20009018487900196, "eval_steps": 500, "global_step": 3328, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "auxiliary_loss_clip": 0.04675119, "auxiliary_loss_mlp": 0.02099638, "balance_loss_clip": 1.77205122, "balance_loss_mlp": 2.27198935, "epoch": 6.012325266796934e-05, "flos": 24455432897280.0, "grad_norm": 54.633257353768954, "language_loss": 2.84989119, "learning_rate": 0.0, "loss": 1.94246852, "num_input_tokens_seen": 19155, "router_z_loss_clip": 3.28125, "router_z_loss_mlp": 24.0, "step": 1, "time_per_iteration": 19.307077646255493 }, { "auxiliary_loss_clip": 0.03143228, "auxiliary_loss_mlp": 0.01384828, "balance_loss_clip": 1.18112338, "balance_loss_mlp": 1.51281738, "epoch": 0.00012024650533593868, "flos": 20225010188160.0, "grad_norm": 36.066244838101376, "language_loss": 1.82575774, "learning_rate": 4e-06, "loss": 1.8710382, "num_input_tokens_seen": 36175, "router_z_loss_clip": 2.03125, "router_z_loss_mlp": 16.25, "step": 2, "time_per_iteration": 2.4628143310546875 }, { "auxiliary_loss_clip": 0.03080973, "auxiliary_loss_mlp": 0.0137341, "balance_loss_clip": 1.17828774, "balance_loss_mlp": 1.51664853, "epoch": 0.000180369758003908, "flos": 22308835996800.0, "grad_norm": 32.82768891459672, "language_loss": 1.57214069, "learning_rate": 3.999999964312572e-06, "loss": 1.61668456, "num_input_tokens_seen": 54870, "router_z_loss_clip": 1.953125, "router_z_loss_mlp": 15.625, "step": 3, "time_per_iteration": 2.409597635269165 }, { "auxiliary_loss_clip": 0.03109568, "auxiliary_loss_mlp": 0.01353198, "balance_loss_clip": 1.127177, "balance_loss_mlp": 1.51108968, "epoch": 0.00024049301067187735, "flos": 22413680409600.0, "grad_norm": 24.052977564940104, "language_loss": 1.37424958, "learning_rate": 3.99999985725029e-06, "loss": 1.41887736, "num_input_tokens_seen": 74575, "router_z_loss_clip": 2.265625, "router_z_loss_mlp": 16.0, "step": 4, "time_per_iteration": 2.4043149948120117 }, { "auxiliary_loss_clip": 0.03127305, "auxiliary_loss_mlp": 0.01403953, "balance_loss_clip": 1.18498981, "balance_loss_mlp": 1.50664127, "epoch": 0.0003006162633398467, "flos": 21395927099520.0, "grad_norm": 21.146805273774273, "language_loss": 1.415416, "learning_rate": 3.999999678813158e-06, "loss": 1.46072853, "num_input_tokens_seen": 92580, "router_z_loss_clip": 2.1875, "router_z_loss_mlp": 16.25, "step": 5, "time_per_iteration": 2.3708319664001465 }, { "auxiliary_loss_clip": 0.03063031, "auxiliary_loss_mlp": 0.01405226, "balance_loss_clip": 1.18578613, "balance_loss_mlp": 1.50378633, "epoch": 0.000360739516007816, "flos": 21651316761600.0, "grad_norm": 6.818791628039744, "language_loss": 1.17620254, "learning_rate": 3.999999429001183e-06, "loss": 1.22088504, "num_input_tokens_seen": 109705, "router_z_loss_clip": 2.1875, "router_z_loss_mlp": 15.5625, "step": 6, "time_per_iteration": 2.3820738792419434 }, { "auxiliary_loss_clip": 0.02998424, "auxiliary_loss_mlp": 0.01370393, "balance_loss_clip": 1.17241049, "balance_loss_mlp": 1.50680101, "epoch": 0.0004208627686757854, "flos": 27158586312960.0, "grad_norm": 4.94199637075652, "language_loss": 1.14681077, "learning_rate": 3.9999991078143714e-06, "loss": 1.19049883, "num_input_tokens_seen": 129425, "router_z_loss_clip": 1.984375, "router_z_loss_mlp": 14.9375, "step": 7, "time_per_iteration": 2.679643392562866 }, { "auxiliary_loss_clip": 0.02951132, "auxiliary_loss_mlp": 0.01311899, "balance_loss_clip": 1.13022375, "balance_loss_mlp": 1.49760175, "epoch": 0.0004809860213437547, "flos": 31317824292480.0, "grad_norm": 23.61535064956306, "language_loss": 0.9544208, "learning_rate": 3.999998715252736e-06, "loss": 0.99705112, "num_input_tokens_seen": 149210, "router_z_loss_clip": 1.8125, "router_z_loss_mlp": 14.5625, "step": 8, "time_per_iteration": 2.6806282997131348 }, { "auxiliary_loss_clip": 0.02969223, "auxiliary_loss_mlp": 0.01355113, "balance_loss_clip": 1.16456866, "balance_loss_mlp": 1.50150013, "epoch": 0.000541109274011724, "flos": 32159056435200.0, "grad_norm": 5.007732554313651, "language_loss": 1.11774778, "learning_rate": 3.999998251316293e-06, "loss": 1.16099107, "num_input_tokens_seen": 169055, "router_z_loss_clip": 1.90625, "router_z_loss_mlp": 14.6875, "step": 9, "time_per_iteration": 2.6619770526885986 }, { "auxiliary_loss_clip": 0.02872593, "auxiliary_loss_mlp": 0.01315594, "balance_loss_clip": 1.14421892, "balance_loss_mlp": 1.50026464, "epoch": 0.0006012325266796934, "flos": 18915801914880.0, "grad_norm": 3.139622664196079, "language_loss": 1.06644702, "learning_rate": 3.9999977160050555e-06, "loss": 1.10832882, "num_input_tokens_seen": 188045, "router_z_loss_clip": 1.7109375, "router_z_loss_mlp": 13.75, "step": 10, "time_per_iteration": 2.564631223678589 }, { "auxiliary_loss_clip": 0.02790104, "auxiliary_loss_mlp": 0.01297754, "balance_loss_clip": 1.13801312, "balance_loss_mlp": 1.4966042, "epoch": 0.0006613557793476627, "flos": 20773879672320.0, "grad_norm": 8.748217914557543, "language_loss": 1.10217166, "learning_rate": 3.9999971093190445e-06, "loss": 1.14305019, "num_input_tokens_seen": 207035, "router_z_loss_clip": 1.6015625, "router_z_loss_mlp": 12.9375, "step": 11, "time_per_iteration": 2.6023213863372803 }, { "auxiliary_loss_clip": 0.02668227, "auxiliary_loss_mlp": 0.01248991, "balance_loss_clip": 1.09602213, "balance_loss_mlp": 1.48630834, "epoch": 0.000721479032015632, "flos": 16580740896000.0, "grad_norm": 4.094164265162429, "language_loss": 1.09107757, "learning_rate": 3.999996431258282e-06, "loss": 1.13024974, "num_input_tokens_seen": 223225, "router_z_loss_clip": 1.53125, "router_z_loss_mlp": 11.875, "step": 12, "time_per_iteration": 2.528917074203491 }, { "auxiliary_loss_clip": 0.02649503, "auxiliary_loss_mlp": 0.0122052, "balance_loss_clip": 1.08004415, "balance_loss_mlp": 1.48940086, "epoch": 0.0007816022846836014, "flos": 23804340618240.0, "grad_norm": 3.2974397637514343, "language_loss": 0.99341649, "learning_rate": 3.999995681822791e-06, "loss": 1.03211677, "num_input_tokens_seen": 242570, "router_z_loss_clip": 1.40625, "router_z_loss_mlp": 11.625, "step": 13, "time_per_iteration": 2.6297576427459717 }, { "auxiliary_loss_clip": 0.02601865, "auxiliary_loss_mlp": 0.01248735, "balance_loss_clip": 1.10196495, "balance_loss_mlp": 1.48509979, "epoch": 0.0008417255373515708, "flos": 19171191576960.0, "grad_norm": 4.238777303539045, "language_loss": 1.06023884, "learning_rate": 3.999994861012598e-06, "loss": 1.09874487, "num_input_tokens_seen": 261215, "router_z_loss_clip": 1.46875, "router_z_loss_mlp": 11.125, "step": 14, "time_per_iteration": 2.5761733055114746 }, { "auxiliary_loss_clip": 0.02539715, "auxiliary_loss_mlp": 0.01209442, "balance_loss_clip": 1.07802558, "balance_loss_mlp": 1.48428929, "epoch": 0.00090184879001954, "flos": 26394372362880.0, "grad_norm": 2.5879242228297796, "language_loss": 0.9828856, "learning_rate": 3.999993968827733e-06, "loss": 1.02037716, "num_input_tokens_seen": 280035, "router_z_loss_clip": 1.3125, "router_z_loss_mlp": 10.5625, "step": 15, "time_per_iteration": 2.602414846420288 }, { "auxiliary_loss_clip": 0.02482357, "auxiliary_loss_mlp": 0.01201394, "balance_loss_clip": 1.07560468, "balance_loss_mlp": 1.47833943, "epoch": 0.0009619720426875094, "flos": 24678391305600.0, "grad_norm": 3.0687345466579785, "language_loss": 0.99246669, "learning_rate": 3.999993005268228e-06, "loss": 1.02930415, "num_input_tokens_seen": 300265, "router_z_loss_clip": 1.2578125, "router_z_loss_mlp": 10.0625, "step": 16, "time_per_iteration": 2.56613826751709 }, { "auxiliary_loss_clip": 0.02431807, "auxiliary_loss_mlp": 0.01216623, "balance_loss_clip": 1.10113311, "balance_loss_mlp": 1.47049284, "epoch": 0.0010220952953554788, "flos": 18623543990400.0, "grad_norm": 3.1215760991906873, "language_loss": 1.01441908, "learning_rate": 3.999991970334118e-06, "loss": 1.05090332, "num_input_tokens_seen": 317375, "router_z_loss_clip": 1.15625, "router_z_loss_mlp": 9.625, "step": 17, "time_per_iteration": 5.36155366897583 }, { "auxiliary_loss_clip": 0.02306633, "auxiliary_loss_mlp": 0.01184744, "balance_loss_clip": 1.07888603, "balance_loss_mlp": 1.46231151, "epoch": 0.001082218548023448, "flos": 26141286850560.0, "grad_norm": 2.330774005332857, "language_loss": 0.99674374, "learning_rate": 3.999990864025439e-06, "loss": 1.03165746, "num_input_tokens_seen": 337975, "router_z_loss_clip": 1.0625, "router_z_loss_mlp": 8.4375, "step": 18, "time_per_iteration": 2.5890016555786133 }, { "auxiliary_loss_clip": 0.02248247, "auxiliary_loss_mlp": 0.011974, "balance_loss_clip": 1.07666516, "balance_loss_mlp": 1.44878852, "epoch": 0.0011423418006914173, "flos": 19608758046720.0, "grad_norm": 3.298295705619953, "language_loss": 0.91169536, "learning_rate": 3.99998968634223e-06, "loss": 0.94615185, "num_input_tokens_seen": 356635, "router_z_loss_clip": 1.203125, "router_z_loss_mlp": 8.0, "step": 19, "time_per_iteration": 2.57875919342041 }, { "auxiliary_loss_clip": 0.02162659, "auxiliary_loss_mlp": 0.01173688, "balance_loss_clip": 1.08027506, "balance_loss_mlp": 1.44382668, "epoch": 0.0012024650533593868, "flos": 17894382912000.0, "grad_norm": 2.4149043095773175, "language_loss": 1.03630507, "learning_rate": 3.999988437284535e-06, "loss": 1.06966853, "num_input_tokens_seen": 375625, "router_z_loss_clip": 0.9375, "router_z_loss_mlp": 7.1875, "step": 20, "time_per_iteration": 2.589801788330078 }, { "auxiliary_loss_clip": 0.02108563, "auxiliary_loss_mlp": 0.01184161, "balance_loss_clip": 1.08779252, "balance_loss_mlp": 1.42771745, "epoch": 0.001262588306027356, "flos": 21250967667840.0, "grad_norm": 2.4908713227019383, "language_loss": 0.9450531, "learning_rate": 3.999987116852396e-06, "loss": 0.97798038, "num_input_tokens_seen": 394350, "router_z_loss_clip": 0.96484375, "router_z_loss_mlp": 6.8125, "step": 21, "time_per_iteration": 2.6463260650634766 }, { "auxiliary_loss_clip": 0.0205811, "auxiliary_loss_mlp": 0.01174418, "balance_loss_clip": 1.08300817, "balance_loss_mlp": 1.41683114, "epoch": 0.0013227115586953253, "flos": 26102882488320.0, "grad_norm": 2.5239968437472995, "language_loss": 0.96045399, "learning_rate": 3.999985725045861e-06, "loss": 0.99277925, "num_input_tokens_seen": 413255, "router_z_loss_clip": 0.9140625, "router_z_loss_mlp": 6.40625, "step": 22, "time_per_iteration": 2.61918306350708 }, { "auxiliary_loss_clip": 0.020383, "auxiliary_loss_mlp": 0.01197761, "balance_loss_clip": 1.10830641, "balance_loss_mlp": 1.41866231, "epoch": 0.0013828348113632948, "flos": 23950242656640.0, "grad_norm": 1.8613463886433936, "language_loss": 0.83544517, "learning_rate": 3.999984261864982e-06, "loss": 0.86780572, "num_input_tokens_seen": 433065, "router_z_loss_clip": 0.89453125, "router_z_loss_mlp": 6.1875, "step": 23, "time_per_iteration": 2.5596020221710205 }, { "auxiliary_loss_clip": 0.02009691, "auxiliary_loss_mlp": 0.01183879, "balance_loss_clip": 1.09533048, "balance_loss_mlp": 1.41109812, "epoch": 0.001442958064031264, "flos": 15958972759680.0, "grad_norm": 2.070711216257559, "language_loss": 1.0156163, "learning_rate": 3.999982727309807e-06, "loss": 1.04755211, "num_input_tokens_seen": 451175, "router_z_loss_clip": 0.8828125, "router_z_loss_mlp": 6.0, "step": 24, "time_per_iteration": 2.5257441997528076 }, { "auxiliary_loss_clip": 0.01941478, "auxiliary_loss_mlp": 0.01207248, "balance_loss_clip": 1.11855638, "balance_loss_mlp": 1.39769328, "epoch": 0.0015030813166992333, "flos": 18107527962240.0, "grad_norm": 3.830467856535708, "language_loss": 0.93178821, "learning_rate": 3.999981121380394e-06, "loss": 0.96327549, "num_input_tokens_seen": 468775, "router_z_loss_clip": 0.88671875, "router_z_loss_mlp": 5.4375, "step": 25, "time_per_iteration": 2.5001440048217773 }, { "auxiliary_loss_clip": 0.01920455, "auxiliary_loss_mlp": 0.01198773, "balance_loss_clip": 1.10831678, "balance_loss_mlp": 1.39311361, "epoch": 0.0015632045693672028, "flos": 22233528460800.0, "grad_norm": 2.107854477846936, "language_loss": 1.00563216, "learning_rate": 3.9999794440768e-06, "loss": 1.03682446, "num_input_tokens_seen": 488530, "router_z_loss_clip": 0.90625, "router_z_loss_mlp": 5.28125, "step": 26, "time_per_iteration": 2.573878526687622 }, { "auxiliary_loss_clip": 0.01911834, "auxiliary_loss_mlp": 0.01198717, "balance_loss_clip": 1.11126542, "balance_loss_mlp": 1.38922739, "epoch": 0.001623327822035172, "flos": 23990706789120.0, "grad_norm": 2.122014738041503, "language_loss": 0.89966214, "learning_rate": 3.999977695399084e-06, "loss": 0.93076766, "num_input_tokens_seen": 510495, "router_z_loss_clip": 0.875, "router_z_loss_mlp": 5.21875, "step": 27, "time_per_iteration": 2.562899351119995 }, { "auxiliary_loss_clip": 0.01883569, "auxiliary_loss_mlp": 0.01206261, "balance_loss_clip": 1.12271953, "balance_loss_mlp": 1.37992167, "epoch": 0.0016834510747031415, "flos": 19676769108480.0, "grad_norm": 2.0938647514651083, "language_loss": 0.99451423, "learning_rate": 3.999975875347308e-06, "loss": 1.02541244, "num_input_tokens_seen": 528605, "router_z_loss_clip": 0.8359375, "router_z_loss_mlp": 5.03125, "step": 28, "time_per_iteration": 2.5451371669769287 }, { "auxiliary_loss_clip": 0.01877581, "auxiliary_loss_mlp": 0.01182278, "balance_loss_clip": 1.09611368, "balance_loss_mlp": 1.37079179, "epoch": 0.0017435743273711108, "flos": 20922749176320.0, "grad_norm": 2.1468105105526787, "language_loss": 0.96911222, "learning_rate": 3.999973983921538e-06, "loss": 0.99971074, "num_input_tokens_seen": 548515, "router_z_loss_clip": 0.859375, "router_z_loss_mlp": 5.0625, "step": 29, "time_per_iteration": 2.53570818901062 }, { "auxiliary_loss_clip": 0.01867424, "auxiliary_loss_mlp": 0.01180375, "balance_loss_clip": 1.09459209, "balance_loss_mlp": 1.36146772, "epoch": 0.00180369758003908, "flos": 19528178895360.0, "grad_norm": 3.1842010867040664, "language_loss": 1.11302865, "learning_rate": 3.9999720211218405e-06, "loss": 1.14350665, "num_input_tokens_seen": 564025, "router_z_loss_clip": 0.859375, "router_z_loss_mlp": 5.0625, "step": 30, "time_per_iteration": 2.5348143577575684 }, { "auxiliary_loss_clip": 0.0182161, "auxiliary_loss_mlp": 0.01175429, "balance_loss_clip": 1.08797705, "balance_loss_mlp": 1.35064387, "epoch": 0.0018638208327070496, "flos": 27451961400960.0, "grad_norm": 2.3505293065017008, "language_loss": 0.96483362, "learning_rate": 3.999969986948286e-06, "loss": 0.99480397, "num_input_tokens_seen": 583345, "router_z_loss_clip": 0.875, "router_z_loss_mlp": 4.71875, "step": 31, "time_per_iteration": 2.5954768657684326 }, { "auxiliary_loss_clip": 0.01802228, "auxiliary_loss_mlp": 0.01171508, "balance_loss_clip": 1.08448565, "balance_loss_mlp": 1.33967805, "epoch": 0.0019239440853750188, "flos": 13588614489600.0, "grad_norm": 2.0416038758443933, "language_loss": 0.88528389, "learning_rate": 3.999967881400949e-06, "loss": 0.91502124, "num_input_tokens_seen": 600010, "router_z_loss_clip": 0.8671875, "router_z_loss_mlp": 4.625, "step": 32, "time_per_iteration": 2.538137435913086 }, { "auxiliary_loss_clip": 0.01803808, "auxiliary_loss_mlp": 0.0116373, "balance_loss_clip": 1.07575357, "balance_loss_mlp": 1.33184898, "epoch": 0.001984067338042988, "flos": 11253099623040.0, "grad_norm": 2.7298373076256124, "language_loss": 0.87022352, "learning_rate": 3.999965704479901e-06, "loss": 0.89989889, "num_input_tokens_seen": 616295, "router_z_loss_clip": 0.87890625, "router_z_loss_mlp": 4.71875, "step": 33, "time_per_iteration": 2.531190872192383 }, { "auxiliary_loss_clip": 0.01769014, "auxiliary_loss_mlp": 0.01165698, "balance_loss_clip": 1.07939088, "balance_loss_mlp": 1.32378912, "epoch": 0.0020441905907109576, "flos": 22385051228160.0, "grad_norm": 2.060972244943001, "language_loss": 0.86651742, "learning_rate": 3.999963456185222e-06, "loss": 0.89586449, "num_input_tokens_seen": 637640, "router_z_loss_clip": 0.86328125, "router_z_loss_mlp": 4.4375, "step": 34, "time_per_iteration": 2.5855860710144043 }, { "auxiliary_loss_clip": 0.01742666, "auxiliary_loss_mlp": 0.0113303, "balance_loss_clip": 1.04314601, "balance_loss_mlp": 1.30449271, "epoch": 0.0021043138433789266, "flos": 49776858489600.0, "grad_norm": 2.195255265685214, "language_loss": 0.70702922, "learning_rate": 3.999961136516991e-06, "loss": 0.7357862, "num_input_tokens_seen": 659710, "router_z_loss_clip": 0.8984375, "router_z_loss_mlp": 4.375, "step": 35, "time_per_iteration": 2.8236277103424072 }, { "auxiliary_loss_clip": 0.0174311, "auxiliary_loss_mlp": 0.01142913, "balance_loss_clip": 1.05574691, "balance_loss_mlp": 1.3051616, "epoch": 0.002164437096046896, "flos": 20556929283840.0, "grad_norm": 2.0848739687058857, "language_loss": 0.8459003, "learning_rate": 3.999958745475293e-06, "loss": 0.87476051, "num_input_tokens_seen": 679670, "router_z_loss_clip": 0.87109375, "router_z_loss_mlp": 4.375, "step": 36, "time_per_iteration": 2.5762763023376465 }, { "auxiliary_loss_clip": 0.01730108, "auxiliary_loss_mlp": 0.01154336, "balance_loss_clip": 1.06540632, "balance_loss_mlp": 1.29269588, "epoch": 0.0022245603487148656, "flos": 26541077362560.0, "grad_norm": 3.013775673998062, "language_loss": 0.87594348, "learning_rate": 3.999956283060211e-06, "loss": 0.9047879, "num_input_tokens_seen": 700170, "router_z_loss_clip": 0.88671875, "router_z_loss_mlp": 4.375, "step": 37, "time_per_iteration": 2.625054359436035 }, { "auxiliary_loss_clip": 0.01715788, "auxiliary_loss_mlp": 0.01164623, "balance_loss_clip": 1.07306981, "balance_loss_mlp": 1.29187417, "epoch": 0.0022846836013828346, "flos": 20337185986560.0, "grad_norm": 1.8630027244681364, "language_loss": 0.99655676, "learning_rate": 3.9999537492718345e-06, "loss": 1.02536082, "num_input_tokens_seen": 718545, "router_z_loss_clip": 0.9140625, "router_z_loss_mlp": 4.25, "step": 38, "time_per_iteration": 2.559971332550049 }, { "auxiliary_loss_clip": 0.01694284, "auxiliary_loss_mlp": 0.01137003, "balance_loss_clip": 1.04478264, "balance_loss_mlp": 1.28287303, "epoch": 0.002344806854050804, "flos": 26246445465600.0, "grad_norm": 2.21150037379473, "language_loss": 0.81611729, "learning_rate": 3.999951144110252e-06, "loss": 0.84443015, "num_input_tokens_seen": 739865, "router_z_loss_clip": 0.921875, "router_z_loss_mlp": 4.125, "step": 39, "time_per_iteration": 2.6174371242523193 }, { "auxiliary_loss_clip": 0.01697233, "auxiliary_loss_mlp": 0.01149935, "balance_loss_clip": 1.05900264, "balance_loss_mlp": 1.27529538, "epoch": 0.0024049301067187736, "flos": 11800747209600.0, "grad_norm": 2.8311092766346047, "language_loss": 0.83641642, "learning_rate": 3.999948467575558e-06, "loss": 0.86488813, "num_input_tokens_seen": 755770, "router_z_loss_clip": 0.90625, "router_z_loss_mlp": 4.21875, "step": 40, "time_per_iteration": 2.546539783477783 }, { "auxiliary_loss_clip": 0.01683985, "auxiliary_loss_mlp": 0.01153351, "balance_loss_clip": 1.06179821, "balance_loss_mlp": 1.2719717, "epoch": 0.0024650533593867426, "flos": 20630456340480.0, "grad_norm": 2.7020804722368825, "language_loss": 0.88915122, "learning_rate": 3.999945719667849e-06, "loss": 0.91752458, "num_input_tokens_seen": 773440, "router_z_loss_clip": 0.9140625, "router_z_loss_mlp": 4.125, "step": 41, "time_per_iteration": 2.5499825477600098 }, { "auxiliary_loss_clip": 0.01660993, "auxiliary_loss_mlp": 0.01141511, "balance_loss_clip": 1.05691957, "balance_loss_mlp": 1.26325691, "epoch": 0.002525176612054712, "flos": 18405127324800.0, "grad_norm": 2.0646008580873874, "language_loss": 0.92708147, "learning_rate": 3.999942900387221e-06, "loss": 0.9551065, "num_input_tokens_seen": 790455, "router_z_loss_clip": 0.84765625, "router_z_loss_mlp": 3.96875, "step": 42, "time_per_iteration": 2.537623167037964 }, { "auxiliary_loss_clip": 0.01651451, "auxiliary_loss_mlp": 0.01167617, "balance_loss_clip": 1.07568288, "balance_loss_mlp": 1.2582351, "epoch": 0.0025852998647226816, "flos": 28182763313280.0, "grad_norm": 5.185354719665505, "language_loss": 0.93673301, "learning_rate": 3.999940009733775e-06, "loss": 0.96492368, "num_input_tokens_seen": 810645, "router_z_loss_clip": 0.91796875, "router_z_loss_mlp": 3.9375, "step": 43, "time_per_iteration": 2.595491647720337 }, { "auxiliary_loss_clip": 0.016468, "auxiliary_loss_mlp": 0.01148236, "balance_loss_clip": 1.06078386, "balance_loss_mlp": 1.25160539, "epoch": 0.0026454231173906506, "flos": 14282233937280.0, "grad_norm": 5.263731246682537, "language_loss": 0.88909531, "learning_rate": 3.9999370477076146e-06, "loss": 0.91704565, "num_input_tokens_seen": 827470, "router_z_loss_clip": 0.875, "router_z_loss_mlp": 3.953125, "step": 44, "time_per_iteration": 2.51104736328125 }, { "auxiliary_loss_clip": 0.01636214, "auxiliary_loss_mlp": 0.01135016, "balance_loss_clip": 1.05443072, "balance_loss_mlp": 1.24934185, "epoch": 0.00270554637005862, "flos": 22418114152320.0, "grad_norm": 2.629764911890971, "language_loss": 0.94870114, "learning_rate": 3.9999340143088455e-06, "loss": 0.97641337, "num_input_tokens_seen": 847285, "router_z_loss_clip": 0.8046875, "router_z_loss_mlp": 3.875, "step": 45, "time_per_iteration": 2.719869613647461 }, { "auxiliary_loss_clip": 0.01637104, "auxiliary_loss_mlp": 0.01135423, "balance_loss_clip": 1.05626822, "balance_loss_mlp": 1.24332213, "epoch": 0.0027656696227265896, "flos": 23984702035200.0, "grad_norm": 1.653387096013355, "language_loss": 0.99931061, "learning_rate": 3.999930909537576e-06, "loss": 1.02703583, "num_input_tokens_seen": 867545, "router_z_loss_clip": 0.79296875, "router_z_loss_mlp": 3.9375, "step": 46, "time_per_iteration": 2.7189016342163086 }, { "auxiliary_loss_clip": 0.01620508, "auxiliary_loss_mlp": 0.01147488, "balance_loss_clip": 1.06122851, "balance_loss_mlp": 1.23951733, "epoch": 0.0028257928753945586, "flos": 37668001731840.0, "grad_norm": 2.1541038360868874, "language_loss": 0.84128428, "learning_rate": 3.999927733393916e-06, "loss": 0.86896425, "num_input_tokens_seen": 889915, "router_z_loss_clip": 0.86328125, "router_z_loss_mlp": 3.8125, "step": 47, "time_per_iteration": 2.7456581592559814 }, { "auxiliary_loss_clip": 0.01600178, "auxiliary_loss_mlp": 0.01142669, "balance_loss_clip": 1.05531228, "balance_loss_mlp": 1.23255134, "epoch": 0.002885916128062528, "flos": 22453481226240.0, "grad_norm": 1.773987463822889, "language_loss": 0.85155529, "learning_rate": 3.99992448587798e-06, "loss": 0.87898374, "num_input_tokens_seen": 908975, "router_z_loss_clip": 0.875, "router_z_loss_mlp": 3.671875, "step": 48, "time_per_iteration": 2.7141005992889404 }, { "auxiliary_loss_clip": 0.01593724, "auxiliary_loss_mlp": 0.01130042, "balance_loss_clip": 1.04640508, "balance_loss_mlp": 1.22513652, "epoch": 0.0029460393807304976, "flos": 27011671845120.0, "grad_norm": 3.2920442272937573, "language_loss": 0.8657636, "learning_rate": 3.999921166989884e-06, "loss": 0.89300132, "num_input_tokens_seen": 929810, "router_z_loss_clip": 0.8359375, "router_z_loss_mlp": 3.6875, "step": 49, "time_per_iteration": 2.6943843364715576 }, { "auxiliary_loss_clip": 0.01589589, "auxiliary_loss_mlp": 0.01155181, "balance_loss_clip": 1.07488132, "balance_loss_mlp": 1.2209065, "epoch": 0.0030061626333984666, "flos": 15850916501760.0, "grad_norm": 2.2248531444274304, "language_loss": 0.88049072, "learning_rate": 3.999917776729746e-06, "loss": 0.90793836, "num_input_tokens_seen": 948650, "router_z_loss_clip": 0.8046875, "router_z_loss_mlp": 3.6875, "step": 50, "time_per_iteration": 2.5544683933258057 }, { "auxiliary_loss_clip": 0.01584487, "auxiliary_loss_mlp": 0.01129517, "balance_loss_clip": 1.05141068, "balance_loss_mlp": 1.21761751, "epoch": 0.003066285886066436, "flos": 31825845619200.0, "grad_norm": 4.248983325462892, "language_loss": 0.83911979, "learning_rate": 3.999914315097687e-06, "loss": 0.86625981, "num_input_tokens_seen": 966455, "router_z_loss_clip": 0.78125, "router_z_loss_mlp": 3.671875, "step": 51, "time_per_iteration": 2.647359609603882 }, { "auxiliary_loss_clip": 0.01567797, "auxiliary_loss_mlp": 0.01153382, "balance_loss_clip": 1.07098413, "balance_loss_mlp": 1.21582484, "epoch": 0.0031264091387344056, "flos": 41425878188160.0, "grad_norm": 1.8429176982897937, "language_loss": 0.91580838, "learning_rate": 3.999910782093829e-06, "loss": 0.94302016, "num_input_tokens_seen": 988110, "router_z_loss_clip": 0.82421875, "router_z_loss_mlp": 3.53125, "step": 52, "time_per_iteration": 2.7222397327423096 }, { "auxiliary_loss_clip": 0.01572168, "auxiliary_loss_mlp": 0.01145655, "balance_loss_clip": 1.06015766, "balance_loss_mlp": 1.21593559, "epoch": 0.0031865323914023747, "flos": 23439812446080.0, "grad_norm": 2.1818325670174197, "language_loss": 0.88794315, "learning_rate": 3.999907177718301e-06, "loss": 0.91512132, "num_input_tokens_seen": 1008550, "router_z_loss_clip": 0.85546875, "router_z_loss_mlp": 3.5625, "step": 53, "time_per_iteration": 2.5715830326080322 }, { "auxiliary_loss_clip": 0.01566318, "auxiliary_loss_mlp": 0.01159099, "balance_loss_clip": 1.07393575, "balance_loss_mlp": 1.21152437, "epoch": 0.003246655644070344, "flos": 14428310532480.0, "grad_norm": 2.336043391261303, "language_loss": 0.79594576, "learning_rate": 3.99990350197123e-06, "loss": 0.82319993, "num_input_tokens_seen": 1026840, "router_z_loss_clip": 0.8515625, "router_z_loss_mlp": 3.546875, "step": 54, "time_per_iteration": 2.5533878803253174 }, { "auxiliary_loss_clip": 0.01562585, "auxiliary_loss_mlp": 0.01153723, "balance_loss_clip": 1.06917977, "balance_loss_mlp": 1.20680356, "epoch": 0.0033067788967383136, "flos": 35916793246080.0, "grad_norm": 2.6399092445371815, "language_loss": 0.77894688, "learning_rate": 3.999899754852747e-06, "loss": 0.80610991, "num_input_tokens_seen": 1048875, "router_z_loss_clip": 0.84765625, "router_z_loss_mlp": 3.5625, "step": 55, "time_per_iteration": 2.686530828475952 }, { "auxiliary_loss_clip": 0.01560018, "auxiliary_loss_mlp": 0.01133782, "balance_loss_clip": 1.05348277, "balance_loss_mlp": 1.20421016, "epoch": 0.003366902149406283, "flos": 22957836860160.0, "grad_norm": 3.5583647739648487, "language_loss": 0.83599192, "learning_rate": 3.999895936362987e-06, "loss": 0.86292994, "num_input_tokens_seen": 1066435, "router_z_loss_clip": 0.8046875, "router_z_loss_mlp": 3.5625, "step": 56, "time_per_iteration": 4.309218645095825 }, { "auxiliary_loss_clip": 0.01562721, "auxiliary_loss_mlp": 0.01154903, "balance_loss_clip": 1.07555699, "balance_loss_mlp": 1.19991803, "epoch": 0.003427025402074252, "flos": 26581506583680.0, "grad_norm": 1.804282673250652, "language_loss": 0.90663362, "learning_rate": 3.9998920465020845e-06, "loss": 0.93380976, "num_input_tokens_seen": 1090330, "router_z_loss_clip": 0.79296875, "router_z_loss_mlp": 3.625, "step": 57, "time_per_iteration": 4.142320156097412 }, { "auxiliary_loss_clip": 0.01542008, "auxiliary_loss_mlp": 0.01138251, "balance_loss_clip": 1.05480409, "balance_loss_mlp": 1.20625949, "epoch": 0.0034871486547422216, "flos": 23950068099840.0, "grad_norm": 2.146597421329481, "language_loss": 0.9681412, "learning_rate": 3.999888085270179e-06, "loss": 0.9949438, "num_input_tokens_seen": 1109840, "router_z_loss_clip": 0.8359375, "router_z_loss_mlp": 3.34375, "step": 58, "time_per_iteration": 2.59883451461792 }, { "auxiliary_loss_clip": 0.01538591, "auxiliary_loss_mlp": 0.01136056, "balance_loss_clip": 1.0576638, "balance_loss_mlp": 1.20107901, "epoch": 0.003547271907410191, "flos": 21213924848640.0, "grad_norm": 2.2963116984050234, "language_loss": 0.8566339, "learning_rate": 3.9998840526674135e-06, "loss": 0.88338029, "num_input_tokens_seen": 1128415, "router_z_loss_clip": 0.78125, "router_z_loss_mlp": 3.375, "step": 59, "time_per_iteration": 2.588517189025879 }, { "auxiliary_loss_clip": 0.01543832, "auxiliary_loss_mlp": 0.01121869, "balance_loss_clip": 1.04376316, "balance_loss_mlp": 1.19997227, "epoch": 0.00360739516007816, "flos": 16504071816960.0, "grad_norm": 3.6106206237954153, "language_loss": 0.90589786, "learning_rate": 3.999879948693929e-06, "loss": 0.9325549, "num_input_tokens_seen": 1146515, "router_z_loss_clip": 0.78125, "router_z_loss_mlp": 3.4375, "step": 60, "time_per_iteration": 2.5708141326904297 }, { "auxiliary_loss_clip": 0.01536828, "auxiliary_loss_mlp": 0.01128284, "balance_loss_clip": 1.05480361, "balance_loss_mlp": 1.19439209, "epoch": 0.0036675184127461296, "flos": 19463763703680.0, "grad_norm": 3.0867057656717347, "language_loss": 0.86699647, "learning_rate": 3.999875773349874e-06, "loss": 0.89364761, "num_input_tokens_seen": 1166330, "router_z_loss_clip": 0.734375, "router_z_loss_mlp": 3.421875, "step": 61, "time_per_iteration": 2.733415365219116 }, { "auxiliary_loss_clip": 0.01530636, "auxiliary_loss_mlp": 0.01119644, "balance_loss_clip": 1.0487386, "balance_loss_mlp": 1.19737685, "epoch": 0.003727641665414099, "flos": 20956335770880.0, "grad_norm": 1.9044505707105237, "language_loss": 0.86141676, "learning_rate": 3.999871526635397e-06, "loss": 0.88791955, "num_input_tokens_seen": 1186010, "router_z_loss_clip": 0.7109375, "router_z_loss_mlp": 3.328125, "step": 62, "time_per_iteration": 2.644710063934326 }, { "auxiliary_loss_clip": 0.01525647, "auxiliary_loss_mlp": 0.01135761, "balance_loss_clip": 1.06042099, "balance_loss_mlp": 1.19628096, "epoch": 0.003787764918082068, "flos": 18405057502080.0, "grad_norm": 1.984669322035565, "language_loss": 0.94111091, "learning_rate": 3.999867208550649e-06, "loss": 0.96772498, "num_input_tokens_seen": 1204985, "router_z_loss_clip": 0.75, "router_z_loss_mlp": 3.296875, "step": 63, "time_per_iteration": 2.6939220428466797 }, { "auxiliary_loss_clip": 0.01522687, "auxiliary_loss_mlp": 0.01124747, "balance_loss_clip": 1.05078948, "balance_loss_mlp": 1.19448018, "epoch": 0.0038478881707500376, "flos": 12458406090240.0, "grad_norm": 6.071362458483904, "language_loss": 0.95735359, "learning_rate": 3.999862819095785e-06, "loss": 0.98382795, "num_input_tokens_seen": 1223545, "router_z_loss_clip": 0.7421875, "router_z_loss_mlp": 3.28125, "step": 64, "time_per_iteration": 2.6445887088775635 }, { "auxiliary_loss_clip": 0.01532445, "auxiliary_loss_mlp": 0.01135816, "balance_loss_clip": 1.06204915, "balance_loss_mlp": 1.19461608, "epoch": 0.003908011423418007, "flos": 13552479365760.0, "grad_norm": 1.8832393227624737, "language_loss": 0.82848072, "learning_rate": 3.999858358270962e-06, "loss": 0.85516334, "num_input_tokens_seen": 1241175, "router_z_loss_clip": 0.73828125, "router_z_loss_mlp": 3.375, "step": 65, "time_per_iteration": 2.6585493087768555 }, { "auxiliary_loss_clip": 0.01523418, "auxiliary_loss_mlp": 0.01124821, "balance_loss_clip": 1.05129278, "balance_loss_mlp": 1.19321489, "epoch": 0.003968134676085976, "flos": 18332473052160.0, "grad_norm": 1.8421961963241207, "language_loss": 0.83222592, "learning_rate": 3.999853826076338e-06, "loss": 0.85870826, "num_input_tokens_seen": 1259315, "router_z_loss_clip": 0.734375, "router_z_loss_mlp": 3.296875, "step": 66, "time_per_iteration": 2.7048535346984863 }, { "auxiliary_loss_clip": 0.01520742, "auxiliary_loss_mlp": 0.01120062, "balance_loss_clip": 1.03871369, "balance_loss_mlp": 1.18489003, "epoch": 0.004028257928753946, "flos": 20484205188480.0, "grad_norm": 2.219008712215282, "language_loss": 0.94136697, "learning_rate": 3.999849222512075e-06, "loss": 0.96777511, "num_input_tokens_seen": 1277055, "router_z_loss_clip": 0.8125, "router_z_loss_mlp": 3.359375, "step": 67, "time_per_iteration": 2.617933511734009 }, { "auxiliary_loss_clip": 0.01507726, "auxiliary_loss_mlp": 0.01130344, "balance_loss_clip": 1.04932952, "balance_loss_mlp": 1.18441331, "epoch": 0.004088381181421915, "flos": 18842833440000.0, "grad_norm": 2.354307623528934, "language_loss": 0.92092949, "learning_rate": 3.9998445475783365e-06, "loss": 0.94731021, "num_input_tokens_seen": 1294355, "router_z_loss_clip": 0.80859375, "router_z_loss_mlp": 3.234375, "step": 68, "time_per_iteration": 2.577284097671509 }, { "auxiliary_loss_clip": 0.01497895, "auxiliary_loss_mlp": 0.01131936, "balance_loss_clip": 1.0565958, "balance_loss_mlp": 1.18115139, "epoch": 0.004148504434089885, "flos": 19426790707200.0, "grad_norm": 2.9681617808792784, "language_loss": 0.9445743, "learning_rate": 3.999839801275292e-06, "loss": 0.97087264, "num_input_tokens_seen": 1313525, "router_z_loss_clip": 0.75, "router_z_loss_mlp": 3.171875, "step": 69, "time_per_iteration": 2.5266003608703613 }, { "auxiliary_loss_clip": 0.01495954, "auxiliary_loss_mlp": 0.01127572, "balance_loss_clip": 1.05685711, "balance_loss_mlp": 1.18270397, "epoch": 0.004208627686757853, "flos": 20810049707520.0, "grad_norm": 2.724199016749798, "language_loss": 0.96660733, "learning_rate": 3.999834983603108e-06, "loss": 0.99284261, "num_input_tokens_seen": 1330505, "router_z_loss_clip": 0.70703125, "router_z_loss_mlp": 3.125, "step": 70, "time_per_iteration": 2.590308666229248 }, { "auxiliary_loss_clip": 0.01504126, "auxiliary_loss_mlp": 0.01123484, "balance_loss_clip": 1.04795337, "balance_loss_mlp": 1.17364883, "epoch": 0.004268750939425823, "flos": 19097629608960.0, "grad_norm": 6.624474777525524, "language_loss": 0.91867542, "learning_rate": 3.9998300945619576e-06, "loss": 0.94495147, "num_input_tokens_seen": 1349615, "router_z_loss_clip": 0.7578125, "router_z_loss_mlp": 3.3125, "step": 71, "time_per_iteration": 2.5628535747528076 }, { "auxiliary_loss_clip": 0.01797056, "auxiliary_loss_mlp": 0.01431044, "balance_loss_clip": 1.39118016, "balance_loss_mlp": 1.42536378, "epoch": 0.004328874192093792, "flos": 52436889377280.0, "grad_norm": 2.2573204514841265, "language_loss": 0.65688801, "learning_rate": 3.999825134152016e-06, "loss": 0.68916899, "num_input_tokens_seen": 1410275, "router_z_loss_clip": 0.3984375, "router_z_loss_mlp": 3.71875, "step": 72, "time_per_iteration": 3.1397647857666016 }, { "auxiliary_loss_clip": 0.01734601, "auxiliary_loss_mlp": 0.01323334, "balance_loss_clip": 1.28423309, "balance_loss_mlp": 1.40884876, "epoch": 0.004388997444761762, "flos": 66469459115520.0, "grad_norm": 2.081010780673298, "language_loss": 0.63639885, "learning_rate": 3.999820102373459e-06, "loss": 0.66697824, "num_input_tokens_seen": 1473020, "router_z_loss_clip": 0.390625, "router_z_loss_mlp": 3.25, "step": 73, "time_per_iteration": 3.1842033863067627 }, { "auxiliary_loss_clip": 0.01489288, "auxiliary_loss_mlp": 0.01117901, "balance_loss_clip": 1.04642332, "balance_loss_mlp": 1.17090213, "epoch": 0.004449120697429731, "flos": 18951971950080.0, "grad_norm": 3.399614259340687, "language_loss": 0.83614337, "learning_rate": 3.999814999226467e-06, "loss": 0.86221528, "num_input_tokens_seen": 1490385, "router_z_loss_clip": 0.71484375, "router_z_loss_mlp": 3.1875, "step": 74, "time_per_iteration": 2.5562846660614014 }, { "auxiliary_loss_clip": 0.0150131, "auxiliary_loss_mlp": 0.01129769, "balance_loss_clip": 1.05662227, "balance_loss_mlp": 1.17718089, "epoch": 0.004509243950097701, "flos": 21104437224960.0, "grad_norm": 2.068454009829982, "language_loss": 0.94926447, "learning_rate": 3.999809824711222e-06, "loss": 0.97557527, "num_input_tokens_seen": 1509725, "router_z_loss_clip": 0.73046875, "router_z_loss_mlp": 3.25, "step": 75, "time_per_iteration": 2.530402421951294 }, { "auxiliary_loss_clip": 0.01481729, "auxiliary_loss_mlp": 0.01138677, "balance_loss_clip": 1.06629276, "balance_loss_mlp": 1.17169082, "epoch": 0.004569367202765669, "flos": 20697838997760.0, "grad_norm": 2.4761529519656147, "language_loss": 0.86071384, "learning_rate": 3.9998045788279075e-06, "loss": 0.88691783, "num_input_tokens_seen": 1527245, "router_z_loss_clip": 0.72265625, "router_z_loss_mlp": 3.09375, "step": 76, "time_per_iteration": 2.5472307205200195 }, { "auxiliary_loss_clip": 0.01492312, "auxiliary_loss_mlp": 0.01127005, "balance_loss_clip": 1.05362022, "balance_loss_mlp": 1.17710352, "epoch": 0.004629490455433639, "flos": 28657198045440.0, "grad_norm": 1.769193740793799, "language_loss": 0.90393454, "learning_rate": 3.9997992615767125e-06, "loss": 0.93012774, "num_input_tokens_seen": 1548930, "router_z_loss_clip": 0.734375, "router_z_loss_mlp": 3.15625, "step": 77, "time_per_iteration": 2.5652127265930176 }, { "auxiliary_loss_clip": 0.01483891, "auxiliary_loss_mlp": 0.01125642, "balance_loss_clip": 1.05130339, "balance_loss_mlp": 1.17786443, "epoch": 0.004689613708101608, "flos": 11071621042560.0, "grad_norm": 3.8040509470686623, "language_loss": 0.8998087, "learning_rate": 3.9997938729578266e-06, "loss": 0.92590404, "num_input_tokens_seen": 1565695, "router_z_loss_clip": 0.7421875, "router_z_loss_mlp": 3.0625, "step": 78, "time_per_iteration": 2.5184528827667236 }, { "auxiliary_loss_clip": 0.01486718, "auxiliary_loss_mlp": 0.01138278, "balance_loss_clip": 1.06508303, "balance_loss_mlp": 1.17505836, "epoch": 0.004749736960769578, "flos": 21798021761280.0, "grad_norm": 7.844066050884662, "language_loss": 0.80664903, "learning_rate": 3.99978841297144e-06, "loss": 0.83289897, "num_input_tokens_seen": 1582625, "router_z_loss_clip": 0.734375, "router_z_loss_mlp": 3.125, "step": 79, "time_per_iteration": 2.558370351791382 }, { "auxiliary_loss_clip": 0.01497252, "auxiliary_loss_mlp": 0.01123498, "balance_loss_clip": 1.05368876, "balance_loss_mlp": 1.18235707, "epoch": 0.004809860213437547, "flos": 19791563258880.0, "grad_norm": 3.243026160233554, "language_loss": 0.90004849, "learning_rate": 3.99978288161775e-06, "loss": 0.92625594, "num_input_tokens_seen": 1601725, "router_z_loss_clip": 0.69921875, "router_z_loss_mlp": 3.15625, "step": 80, "time_per_iteration": 2.5560710430145264 }, { "auxiliary_loss_clip": 0.01486012, "auxiliary_loss_mlp": 0.01134552, "balance_loss_clip": 1.06226373, "balance_loss_mlp": 1.18250656, "epoch": 0.004869983466105517, "flos": 26573232591360.0, "grad_norm": 2.165405405963283, "language_loss": 0.93016237, "learning_rate": 3.999777278896952e-06, "loss": 0.95636809, "num_input_tokens_seen": 1622420, "router_z_loss_clip": 0.72265625, "router_z_loss_mlp": 3.03125, "step": 81, "time_per_iteration": 2.637070894241333 }, { "auxiliary_loss_clip": 0.01490725, "auxiliary_loss_mlp": 0.01150368, "balance_loss_clip": 1.07927203, "balance_loss_mlp": 1.18020296, "epoch": 0.004930106718773485, "flos": 12822550237440.0, "grad_norm": 2.6227018187074114, "language_loss": 0.94184101, "learning_rate": 3.999771604809249e-06, "loss": 0.96825194, "num_input_tokens_seen": 1640715, "router_z_loss_clip": 0.7109375, "router_z_loss_mlp": 3.09375, "step": 82, "time_per_iteration": 2.5250260829925537 }, { "auxiliary_loss_clip": 0.01483064, "auxiliary_loss_mlp": 0.01149186, "balance_loss_clip": 1.07451355, "balance_loss_mlp": 1.17058671, "epoch": 0.004990229971441455, "flos": 25773756301440.0, "grad_norm": 2.157109061404475, "language_loss": 0.85192108, "learning_rate": 3.999765859354839e-06, "loss": 0.87824357, "num_input_tokens_seen": 1662210, "router_z_loss_clip": 0.74609375, "router_z_loss_mlp": 3.125, "step": 83, "time_per_iteration": 2.627007484436035 }, { "auxiliary_loss_clip": 0.0147668, "auxiliary_loss_mlp": 0.01154582, "balance_loss_clip": 1.07733428, "balance_loss_mlp": 1.16956425, "epoch": 0.005050353224109424, "flos": 17456292771840.0, "grad_norm": 2.7224666668920112, "language_loss": 0.90668142, "learning_rate": 3.999760042533931e-06, "loss": 0.93299407, "num_input_tokens_seen": 1681070, "router_z_loss_clip": 0.7734375, "router_z_loss_mlp": 3.078125, "step": 84, "time_per_iteration": 2.4960057735443115 }, { "auxiliary_loss_clip": 0.01597049, "auxiliary_loss_mlp": 0.01233618, "balance_loss_clip": 1.19604373, "balance_loss_mlp": 1.38286448, "epoch": 0.005110476476777394, "flos": 69802269235200.0, "grad_norm": 1.0747967940372232, "language_loss": 0.61884308, "learning_rate": 3.999754154346731e-06, "loss": 0.64714968, "num_input_tokens_seen": 1747140, "router_z_loss_clip": 0.375, "router_z_loss_mlp": 2.140625, "step": 85, "time_per_iteration": 3.219843626022339 }, { "auxiliary_loss_clip": 0.01462354, "auxiliary_loss_mlp": 0.011176, "balance_loss_clip": 1.04264069, "balance_loss_mlp": 1.15787935, "epoch": 0.005170599729445363, "flos": 24788961181440.0, "grad_norm": 2.109722333055802, "language_loss": 0.89139509, "learning_rate": 3.999748194793449e-06, "loss": 0.9171946, "num_input_tokens_seen": 1767475, "router_z_loss_clip": 0.75, "router_z_loss_mlp": 3.046875, "step": 86, "time_per_iteration": 2.5646045207977295 }, { "auxiliary_loss_clip": 0.01472103, "auxiliary_loss_mlp": 0.01136714, "balance_loss_clip": 1.06227946, "balance_loss_mlp": 1.16680181, "epoch": 0.005230722982113333, "flos": 23256937411200.0, "grad_norm": 2.3874485555726417, "language_loss": 0.80222178, "learning_rate": 3.999742163874298e-06, "loss": 0.82830989, "num_input_tokens_seen": 1784980, "router_z_loss_clip": 0.7421875, "router_z_loss_mlp": 3.0625, "step": 87, "time_per_iteration": 2.568986177444458 }, { "auxiliary_loss_clip": 0.01458642, "auxiliary_loss_mlp": 0.01126635, "balance_loss_clip": 1.0532496, "balance_loss_mlp": 1.16615224, "epoch": 0.005290846234781301, "flos": 16726957136640.0, "grad_norm": 1.8012755792208095, "language_loss": 0.94010115, "learning_rate": 3.999736061589492e-06, "loss": 0.96595389, "num_input_tokens_seen": 1803030, "router_z_loss_clip": 0.734375, "router_z_loss_mlp": 2.9375, "step": 88, "time_per_iteration": 2.5278608798980713 }, { "auxiliary_loss_clip": 0.01458957, "auxiliary_loss_mlp": 0.01112408, "balance_loss_clip": 1.04107285, "balance_loss_mlp": 1.15566182, "epoch": 0.005350969487449271, "flos": 20885043041280.0, "grad_norm": 2.0496604196188133, "language_loss": 0.84080839, "learning_rate": 3.999729887939251e-06, "loss": 0.86652201, "num_input_tokens_seen": 1822865, "router_z_loss_clip": 0.71484375, "router_z_loss_mlp": 3.03125, "step": 89, "time_per_iteration": 2.563391923904419 }, { "auxiliary_loss_clip": 0.01449543, "auxiliary_loss_mlp": 0.01111738, "balance_loss_clip": 1.04006934, "balance_loss_mlp": 1.15215576, "epoch": 0.00541109274011724, "flos": 26208878976000.0, "grad_norm": 2.0047324414422962, "language_loss": 0.89549929, "learning_rate": 3.9997236429237945e-06, "loss": 0.92111206, "num_input_tokens_seen": 1842435, "router_z_loss_clip": 0.71875, "router_z_loss_mlp": 2.96875, "step": 90, "time_per_iteration": 2.5636699199676514 }, { "auxiliary_loss_clip": 0.01434598, "auxiliary_loss_mlp": 0.01115269, "balance_loss_clip": 1.04503119, "balance_loss_mlp": 1.14833903, "epoch": 0.00547121599278521, "flos": 21177510433920.0, "grad_norm": 3.218059252263368, "language_loss": 0.84463358, "learning_rate": 3.999717326543345e-06, "loss": 0.87013233, "num_input_tokens_seen": 1860065, "router_z_loss_clip": 0.703125, "router_z_loss_mlp": 2.859375, "step": 91, "time_per_iteration": 2.5507023334503174 }, { "auxiliary_loss_clip": 0.01435673, "auxiliary_loss_mlp": 0.01108879, "balance_loss_clip": 1.03992856, "balance_loss_mlp": 1.14235711, "epoch": 0.005531339245453179, "flos": 19717791822720.0, "grad_norm": 2.168802380764744, "language_loss": 0.85291636, "learning_rate": 3.9997109387981275e-06, "loss": 0.87836194, "num_input_tokens_seen": 1878135, "router_z_loss_clip": 0.6875, "router_z_loss_mlp": 2.9375, "step": 92, "time_per_iteration": 2.539388418197632 }, { "auxiliary_loss_clip": 0.01438361, "auxiliary_loss_mlp": 0.01139437, "balance_loss_clip": 1.06667149, "balance_loss_mlp": 1.14034557, "epoch": 0.005591462498121149, "flos": 17635222823040.0, "grad_norm": 2.7323537714010415, "language_loss": 0.89827538, "learning_rate": 3.99970447968837e-06, "loss": 0.92405343, "num_input_tokens_seen": 1894895, "router_z_loss_clip": 0.7265625, "router_z_loss_mlp": 2.984375, "step": 93, "time_per_iteration": 2.520329475402832 }, { "auxiliary_loss_clip": 0.01444438, "auxiliary_loss_mlp": 0.01130383, "balance_loss_clip": 1.05375481, "balance_loss_mlp": 1.14248276, "epoch": 0.005651585750789117, "flos": 20010189392640.0, "grad_norm": 3.029902315947606, "language_loss": 0.85671586, "learning_rate": 3.9996979492143045e-06, "loss": 0.88246405, "num_input_tokens_seen": 1913220, "router_z_loss_clip": 0.765625, "router_z_loss_mlp": 3.015625, "step": 94, "time_per_iteration": 2.518771171569824 }, { "auxiliary_loss_clip": 0.01454722, "auxiliary_loss_mlp": 0.01115659, "balance_loss_clip": 1.07884717, "balance_loss_mlp": 1.27629483, "epoch": 0.005711709003457087, "flos": 59809917185280.0, "grad_norm": 1.1802114848498737, "language_loss": 0.6768719, "learning_rate": 3.999691347376162e-06, "loss": 0.7025758, "num_input_tokens_seen": 1970970, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.78125, "step": 95, "time_per_iteration": 4.534137964248657 }, { "auxiliary_loss_clip": 0.01433724, "auxiliary_loss_mlp": 0.01147286, "balance_loss_clip": 1.07719111, "balance_loss_mlp": 1.137779, "epoch": 0.005771832256125056, "flos": 15558693488640.0, "grad_norm": 3.5253256287489165, "language_loss": 0.88525021, "learning_rate": 3.99968467417418e-06, "loss": 0.91106033, "num_input_tokens_seen": 1988930, "router_z_loss_clip": 0.703125, "router_z_loss_mlp": 2.953125, "step": 96, "time_per_iteration": 4.071526050567627 }, { "auxiliary_loss_clip": 0.01420394, "auxiliary_loss_mlp": 0.01132117, "balance_loss_clip": 1.06559837, "balance_loss_mlp": 1.1362958, "epoch": 0.005831955508793026, "flos": 22527287573760.0, "grad_norm": 2.891057314816126, "language_loss": 0.88203895, "learning_rate": 3.999677929608596e-06, "loss": 0.90756404, "num_input_tokens_seen": 2006285, "router_z_loss_clip": 0.66796875, "router_z_loss_mlp": 2.84375, "step": 97, "time_per_iteration": 4.145650863647461 }, { "auxiliary_loss_clip": 0.01413049, "auxiliary_loss_mlp": 0.01130992, "balance_loss_clip": 1.0612781, "balance_loss_mlp": 1.13330138, "epoch": 0.005892078761460995, "flos": 22048872946560.0, "grad_norm": 2.0178304849889797, "language_loss": 0.75365317, "learning_rate": 3.99967111367965e-06, "loss": 0.77909356, "num_input_tokens_seen": 2024905, "router_z_loss_clip": 0.69921875, "router_z_loss_mlp": 2.796875, "step": 98, "time_per_iteration": 2.5823192596435547 }, { "auxiliary_loss_clip": 0.01407603, "auxiliary_loss_mlp": 0.0104898, "balance_loss_clip": 1.01293111, "balance_loss_mlp": 1.23561692, "epoch": 0.005952202014128965, "flos": 68535689598720.0, "grad_norm": 0.963339581518281, "language_loss": 0.65151054, "learning_rate": 3.999664226387586e-06, "loss": 0.67607635, "num_input_tokens_seen": 2086220, "router_z_loss_clip": 0.36132812, "router_z_loss_mlp": 1.71875, "step": 99, "time_per_iteration": 3.2279751300811768 }, { "auxiliary_loss_clip": 0.01421633, "auxiliary_loss_mlp": 0.01152154, "balance_loss_clip": 1.07628894, "balance_loss_mlp": 1.13284802, "epoch": 0.006012325266796933, "flos": 22959931541760.0, "grad_norm": 2.089015714107569, "language_loss": 0.8961187, "learning_rate": 3.999657267732648e-06, "loss": 0.92185652, "num_input_tokens_seen": 2103365, "router_z_loss_clip": 0.7578125, "router_z_loss_mlp": 2.890625, "step": 100, "time_per_iteration": 2.543591022491455 }, { "auxiliary_loss_clip": 0.01423763, "auxiliary_loss_mlp": 0.01135114, "balance_loss_clip": 1.06478059, "balance_loss_mlp": 1.13237977, "epoch": 0.006072448519464903, "flos": 17346979704960.0, "grad_norm": 2.0200724237283785, "language_loss": 0.89709979, "learning_rate": 3.999650237715088e-06, "loss": 0.92268854, "num_input_tokens_seen": 2121995, "router_z_loss_clip": 0.703125, "router_z_loss_mlp": 2.90625, "step": 101, "time_per_iteration": 2.6019370555877686 }, { "auxiliary_loss_clip": 0.01417283, "auxiliary_loss_mlp": 0.01138533, "balance_loss_clip": 1.06633949, "balance_loss_mlp": 1.13459754, "epoch": 0.006132571772132872, "flos": 24679962316800.0, "grad_norm": 2.382432542633007, "language_loss": 0.89427447, "learning_rate": 3.9996431363351536e-06, "loss": 0.91983271, "num_input_tokens_seen": 2141815, "router_z_loss_clip": 0.72265625, "router_z_loss_mlp": 2.828125, "step": 102, "time_per_iteration": 2.55599045753479 }, { "auxiliary_loss_clip": 0.01412183, "auxiliary_loss_mlp": 0.01121469, "balance_loss_clip": 1.05390084, "balance_loss_mlp": 1.13109529, "epoch": 0.006192695024800842, "flos": 21464741122560.0, "grad_norm": 2.1856308121545287, "language_loss": 0.86522692, "learning_rate": 3.9996359635931e-06, "loss": 0.89056349, "num_input_tokens_seen": 2161125, "router_z_loss_clip": 0.67578125, "router_z_loss_mlp": 2.8125, "step": 103, "time_per_iteration": 2.555086374282837 }, { "auxiliary_loss_clip": 0.01414161, "auxiliary_loss_mlp": 0.01133673, "balance_loss_clip": 1.06314886, "balance_loss_mlp": 1.13492, "epoch": 0.006252818277468811, "flos": 17419459420800.0, "grad_norm": 2.0955448234560823, "language_loss": 0.92823404, "learning_rate": 3.999628719489181e-06, "loss": 0.9537124, "num_input_tokens_seen": 2179510, "router_z_loss_clip": 0.70703125, "router_z_loss_mlp": 2.78125, "step": 104, "time_per_iteration": 2.5785515308380127 }, { "auxiliary_loss_clip": 0.01407541, "auxiliary_loss_mlp": 0.01126941, "balance_loss_clip": 1.05708396, "balance_loss_mlp": 1.12816191, "epoch": 0.006312941530136781, "flos": 19098537304320.0, "grad_norm": 14.34565848265555, "language_loss": 0.94836128, "learning_rate": 3.999621404023658e-06, "loss": 0.97370607, "num_input_tokens_seen": 2197870, "router_z_loss_clip": 0.69921875, "router_z_loss_mlp": 2.796875, "step": 105, "time_per_iteration": 2.543741226196289 }, { "auxiliary_loss_clip": 0.01410998, "auxiliary_loss_mlp": 0.01137532, "balance_loss_clip": 1.06157172, "balance_loss_mlp": 1.12886274, "epoch": 0.006373064782804749, "flos": 24059695368960.0, "grad_norm": 2.226532575855215, "language_loss": 0.85008109, "learning_rate": 3.9996140171967904e-06, "loss": 0.87556636, "num_input_tokens_seen": 2217495, "router_z_loss_clip": 0.7578125, "router_z_loss_mlp": 2.8125, "step": 106, "time_per_iteration": 2.576078414916992 }, { "auxiliary_loss_clip": 0.01402094, "auxiliary_loss_mlp": 0.01119371, "balance_loss_clip": 1.04894197, "balance_loss_mlp": 1.12047982, "epoch": 0.006433188035472719, "flos": 18331460622720.0, "grad_norm": 2.3678014845405873, "language_loss": 0.81457663, "learning_rate": 3.9996065590088426e-06, "loss": 0.8397913, "num_input_tokens_seen": 2236520, "router_z_loss_clip": 0.703125, "router_z_loss_mlp": 2.8125, "step": 107, "time_per_iteration": 2.497641086578369 }, { "auxiliary_loss_clip": 0.01341496, "auxiliary_loss_mlp": 0.0110217, "balance_loss_clip": 1.07508552, "balance_loss_mlp": 1.18681765, "epoch": 0.006493311288140688, "flos": 62558907816960.0, "grad_norm": 0.9461185842272075, "language_loss": 0.64579511, "learning_rate": 3.999599029460081e-06, "loss": 0.67023176, "num_input_tokens_seen": 2300140, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.546875, "step": 108, "time_per_iteration": 3.157834053039551 }, { "auxiliary_loss_clip": 0.01398715, "auxiliary_loss_mlp": 0.01113337, "balance_loss_clip": 1.04224062, "balance_loss_mlp": 1.12512159, "epoch": 0.006553434540808658, "flos": 19499130777600.0, "grad_norm": 2.0634755752913914, "language_loss": 0.96250588, "learning_rate": 3.999591428550772e-06, "loss": 0.98762637, "num_input_tokens_seen": 2317320, "router_z_loss_clip": 0.7109375, "router_z_loss_mlp": 2.734375, "step": 109, "time_per_iteration": 2.527080535888672 }, { "auxiliary_loss_clip": 0.01397787, "auxiliary_loss_mlp": 0.01115048, "balance_loss_clip": 1.04652655, "balance_loss_mlp": 1.12549496, "epoch": 0.006613557793476627, "flos": 21104088111360.0, "grad_norm": 1.7137254663805555, "language_loss": 0.83182019, "learning_rate": 3.999583756281189e-06, "loss": 0.85694849, "num_input_tokens_seen": 2337820, "router_z_loss_clip": 0.6875, "router_z_loss_mlp": 2.71875, "step": 110, "time_per_iteration": 2.596374273300171 }, { "auxiliary_loss_clip": 0.01399336, "auxiliary_loss_mlp": 0.01111445, "balance_loss_clip": 1.04158795, "balance_loss_mlp": 1.12313271, "epoch": 0.006673681046144597, "flos": 26029564899840.0, "grad_norm": 2.0155584080496984, "language_loss": 0.81827509, "learning_rate": 3.999576012651605e-06, "loss": 0.84338289, "num_input_tokens_seen": 2358560, "router_z_loss_clip": 0.6953125, "router_z_loss_mlp": 2.75, "step": 111, "time_per_iteration": 2.590143918991089 }, { "auxiliary_loss_clip": 0.01389427, "auxiliary_loss_mlp": 0.0111467, "balance_loss_clip": 1.04719806, "balance_loss_mlp": 1.11860859, "epoch": 0.006733804298812566, "flos": 23146681737600.0, "grad_norm": 2.370401313774829, "language_loss": 0.92597079, "learning_rate": 3.999568197662297e-06, "loss": 0.95101178, "num_input_tokens_seen": 2379005, "router_z_loss_clip": 0.671875, "router_z_loss_mlp": 2.703125, "step": 112, "time_per_iteration": 2.553640127182007 }, { "auxiliary_loss_clip": 0.01399439, "auxiliary_loss_mlp": 0.01106971, "balance_loss_clip": 1.04140568, "balance_loss_mlp": 1.12583899, "epoch": 0.006793927551480535, "flos": 11763669479040.0, "grad_norm": 2.2048966935764076, "language_loss": 0.77447867, "learning_rate": 3.999560311313543e-06, "loss": 0.79954273, "num_input_tokens_seen": 2395610, "router_z_loss_clip": 0.65625, "router_z_loss_mlp": 2.734375, "step": 113, "time_per_iteration": 2.5330731868743896 }, { "auxiliary_loss_clip": 0.01391044, "auxiliary_loss_mlp": 0.01108191, "balance_loss_clip": 1.03609276, "balance_loss_mlp": 1.12234282, "epoch": 0.006854050804148504, "flos": 19170947197440.0, "grad_norm": 1.788522558774708, "language_loss": 0.91970974, "learning_rate": 3.999552353605626e-06, "loss": 0.94470197, "num_input_tokens_seen": 2415005, "router_z_loss_clip": 0.72265625, "router_z_loss_mlp": 2.6875, "step": 114, "time_per_iteration": 2.5473427772521973 }, { "auxiliary_loss_clip": 0.0139664, "auxiliary_loss_mlp": 0.01111945, "balance_loss_clip": 1.04928839, "balance_loss_mlp": 1.11947298, "epoch": 0.006914174056816474, "flos": 21980792062080.0, "grad_norm": 2.5305389445574975, "language_loss": 0.93227071, "learning_rate": 3.999544324538829e-06, "loss": 0.95735657, "num_input_tokens_seen": 2433965, "router_z_loss_clip": 0.625, "router_z_loss_mlp": 2.765625, "step": 115, "time_per_iteration": 2.5604898929595947 }, { "auxiliary_loss_clip": 0.01389107, "auxiliary_loss_mlp": 0.01104315, "balance_loss_clip": 1.03507793, "balance_loss_mlp": 1.11516285, "epoch": 0.006974297309484443, "flos": 16288238592000.0, "grad_norm": 2.5654792929808563, "language_loss": 0.80363703, "learning_rate": 3.999536224113438e-06, "loss": 0.82857126, "num_input_tokens_seen": 2451605, "router_z_loss_clip": 0.69140625, "router_z_loss_mlp": 2.734375, "step": 116, "time_per_iteration": 2.5479090213775635 }, { "auxiliary_loss_clip": 0.01381443, "auxiliary_loss_mlp": 0.01104956, "balance_loss_clip": 1.0391047, "balance_loss_mlp": 1.11800122, "epoch": 0.007034420562152413, "flos": 26102812665600.0, "grad_norm": 3.7731086756359464, "language_loss": 0.86852342, "learning_rate": 3.9995280523297416e-06, "loss": 0.89338744, "num_input_tokens_seen": 2472035, "router_z_loss_clip": 0.65625, "router_z_loss_mlp": 2.625, "step": 117, "time_per_iteration": 2.556678056716919 }, { "auxiliary_loss_clip": 0.01386747, "auxiliary_loss_mlp": 0.0111943, "balance_loss_clip": 1.04842925, "balance_loss_mlp": 1.12321186, "epoch": 0.007094543814820382, "flos": 14203889112960.0, "grad_norm": 2.1802516405551553, "language_loss": 0.82808697, "learning_rate": 3.9995198091880334e-06, "loss": 0.8531487, "num_input_tokens_seen": 2489285, "router_z_loss_clip": 0.7109375, "router_z_loss_mlp": 2.640625, "step": 118, "time_per_iteration": 2.588897705078125 }, { "auxiliary_loss_clip": 0.01394154, "auxiliary_loss_mlp": 0.01123586, "balance_loss_clip": 1.05520773, "balance_loss_mlp": 1.11883044, "epoch": 0.007154667067488351, "flos": 14975120246400.0, "grad_norm": 2.874811437563815, "language_loss": 0.97947919, "learning_rate": 3.999511494688606e-06, "loss": 1.00465655, "num_input_tokens_seen": 2506460, "router_z_loss_clip": 0.68359375, "router_z_loss_mlp": 2.75, "step": 119, "time_per_iteration": 2.5415806770324707 }, { "auxiliary_loss_clip": 0.01395516, "auxiliary_loss_mlp": 0.01106237, "balance_loss_clip": 1.03919351, "balance_loss_mlp": 1.1161859, "epoch": 0.00721479032015632, "flos": 20192261466240.0, "grad_norm": 2.3987781398524306, "language_loss": 0.87784606, "learning_rate": 3.999503108831758e-06, "loss": 0.90286356, "num_input_tokens_seen": 2525565, "router_z_loss_clip": 0.671875, "router_z_loss_mlp": 2.796875, "step": 120, "time_per_iteration": 2.533524513244629 }, { "auxiliary_loss_clip": 0.01384108, "auxiliary_loss_mlp": 0.01112804, "balance_loss_clip": 1.04556966, "balance_loss_mlp": 1.11985481, "epoch": 0.00727491357282429, "flos": 23146158067200.0, "grad_norm": 1.8486992631891273, "language_loss": 0.92068368, "learning_rate": 3.999494651617787e-06, "loss": 0.94565284, "num_input_tokens_seen": 2546605, "router_z_loss_clip": 0.671875, "router_z_loss_mlp": 2.640625, "step": 121, "time_per_iteration": 2.5171191692352295 }, { "auxiliary_loss_clip": 0.01389371, "auxiliary_loss_mlp": 0.01132887, "balance_loss_clip": 1.06698799, "balance_loss_mlp": 1.12244916, "epoch": 0.007335036825492259, "flos": 15520812796800.0, "grad_norm": 2.3026404931976407, "language_loss": 0.88909745, "learning_rate": 3.999486123046994e-06, "loss": 0.91432005, "num_input_tokens_seen": 2560730, "router_z_loss_clip": 0.65625, "router_z_loss_mlp": 2.671875, "step": 122, "time_per_iteration": 2.5436108112335205 }, { "auxiliary_loss_clip": 0.01383648, "auxiliary_loss_mlp": 0.01112574, "balance_loss_clip": 1.04171658, "balance_loss_mlp": 1.11539435, "epoch": 0.007395160078160229, "flos": 24242221290240.0, "grad_norm": 3.7241053052078352, "language_loss": 0.91549945, "learning_rate": 3.999477523119686e-06, "loss": 0.94046164, "num_input_tokens_seen": 2579550, "router_z_loss_clip": 0.7109375, "router_z_loss_mlp": 2.6875, "step": 123, "time_per_iteration": 2.5250282287597656 }, { "auxiliary_loss_clip": 0.01382203, "auxiliary_loss_mlp": 0.01103122, "balance_loss_clip": 1.03736663, "balance_loss_mlp": 1.11126471, "epoch": 0.007455283330828198, "flos": 31758428050560.0, "grad_norm": 5.948156608363036, "language_loss": 0.69979113, "learning_rate": 3.999468851836168e-06, "loss": 0.72464442, "num_input_tokens_seen": 2600390, "router_z_loss_clip": 0.65625, "router_z_loss_mlp": 2.71875, "step": 124, "time_per_iteration": 2.675476551055908 }, { "auxiliary_loss_clip": 0.01374967, "auxiliary_loss_mlp": 0.01109007, "balance_loss_clip": 1.04382336, "balance_loss_mlp": 1.11420441, "epoch": 0.007515406583496167, "flos": 26613941103360.0, "grad_norm": 2.139508831646684, "language_loss": 0.87107795, "learning_rate": 3.999460109196749e-06, "loss": 0.89591765, "num_input_tokens_seen": 2620770, "router_z_loss_clip": 0.65234375, "router_z_loss_mlp": 2.609375, "step": 125, "time_per_iteration": 2.5488245487213135 }, { "auxiliary_loss_clip": 0.01384091, "auxiliary_loss_mlp": 0.01117652, "balance_loss_clip": 1.04979801, "balance_loss_mlp": 1.1116432, "epoch": 0.007575529836164136, "flos": 18222706137600.0, "grad_norm": 2.6663440653428, "language_loss": 0.80915189, "learning_rate": 3.999451295201743e-06, "loss": 0.83416933, "num_input_tokens_seen": 2639900, "router_z_loss_clip": 0.6796875, "router_z_loss_mlp": 2.71875, "step": 126, "time_per_iteration": 2.552436590194702 }, { "auxiliary_loss_clip": 0.01383761, "auxiliary_loss_mlp": 0.01104154, "balance_loss_clip": 1.03906584, "balance_loss_mlp": 1.11447549, "epoch": 0.007635653088832106, "flos": 21579325804800.0, "grad_norm": 2.829396146717874, "language_loss": 0.66536784, "learning_rate": 3.999442409851463e-06, "loss": 0.690247, "num_input_tokens_seen": 2657450, "router_z_loss_clip": 0.6484375, "router_z_loss_mlp": 2.6875, "step": 127, "time_per_iteration": 2.5131595134735107 }, { "auxiliary_loss_clip": 0.01375587, "auxiliary_loss_mlp": 0.0110898, "balance_loss_clip": 1.04360604, "balance_loss_mlp": 1.1157546, "epoch": 0.007695776341500075, "flos": 25373861055360.0, "grad_norm": 3.1321009293481024, "language_loss": 0.86887217, "learning_rate": 3.999433453146227e-06, "loss": 0.89371789, "num_input_tokens_seen": 2678150, "router_z_loss_clip": 0.65234375, "router_z_loss_mlp": 2.59375, "step": 128, "time_per_iteration": 2.658592700958252 }, { "auxiliary_loss_clip": 0.01373952, "auxiliary_loss_mlp": 0.01117951, "balance_loss_clip": 1.04957247, "balance_loss_mlp": 1.10953844, "epoch": 0.007755899594168045, "flos": 22342876439040.0, "grad_norm": 1.8980833921998537, "language_loss": 0.83853519, "learning_rate": 3.9994244250863535e-06, "loss": 0.86345422, "num_input_tokens_seen": 2698290, "router_z_loss_clip": 0.68359375, "router_z_loss_mlp": 2.640625, "step": 129, "time_per_iteration": 2.5144636631011963 }, { "auxiliary_loss_clip": 0.01367964, "auxiliary_loss_mlp": 0.01104102, "balance_loss_clip": 1.03801274, "balance_loss_mlp": 1.10930061, "epoch": 0.007816022846836013, "flos": 22637124311040.0, "grad_norm": 2.1901717057431904, "language_loss": 0.96096313, "learning_rate": 3.999415325672166e-06, "loss": 0.98568374, "num_input_tokens_seen": 2717630, "router_z_loss_clip": 0.66015625, "router_z_loss_mlp": 2.578125, "step": 130, "time_per_iteration": 2.513084888458252 }, { "auxiliary_loss_clip": 0.01366017, "auxiliary_loss_mlp": 0.0110484, "balance_loss_clip": 1.04018116, "balance_loss_mlp": 1.11058915, "epoch": 0.007876146099503984, "flos": 18182032536960.0, "grad_norm": 2.2948658674142837, "language_loss": 0.80744946, "learning_rate": 3.9994061549039886e-06, "loss": 0.83215797, "num_input_tokens_seen": 2735835, "router_z_loss_clip": 0.6484375, "router_z_loss_mlp": 2.5625, "step": 131, "time_per_iteration": 2.493565797805786 }, { "auxiliary_loss_clip": 0.01374832, "auxiliary_loss_mlp": 0.0110615, "balance_loss_clip": 1.04330242, "balance_loss_mlp": 1.11081851, "epoch": 0.007936269352171952, "flos": 27118436382720.0, "grad_norm": 2.7280402886740327, "language_loss": 0.82224703, "learning_rate": 3.9993969127821485e-06, "loss": 0.84705681, "num_input_tokens_seen": 2756335, "router_z_loss_clip": 0.62890625, "router_z_loss_mlp": 2.640625, "step": 132, "time_per_iteration": 2.5553696155548096 }, { "auxiliary_loss_clip": 0.01363389, "auxiliary_loss_mlp": 0.01108456, "balance_loss_clip": 1.04355907, "balance_loss_mlp": 1.10452795, "epoch": 0.007996392604839923, "flos": 19025324449920.0, "grad_norm": 1.8894809484000292, "language_loss": 0.94121611, "learning_rate": 3.9993875993069755e-06, "loss": 0.96593451, "num_input_tokens_seen": 2775090, "router_z_loss_clip": 0.6484375, "router_z_loss_mlp": 2.59375, "step": 133, "time_per_iteration": 2.4987971782684326 }, { "auxiliary_loss_clip": 0.01359644, "auxiliary_loss_mlp": 0.01107564, "balance_loss_clip": 1.04538476, "balance_loss_mlp": 1.11143434, "epoch": 0.008056515857507891, "flos": 25482964654080.0, "grad_norm": 1.743450272302483, "language_loss": 0.72378039, "learning_rate": 3.9993782144788025e-06, "loss": 0.74845243, "num_input_tokens_seen": 2795320, "router_z_loss_clip": 0.62109375, "router_z_loss_mlp": 2.484375, "step": 134, "time_per_iteration": 3.978874683380127 }, { "auxiliary_loss_clip": 0.0136099, "auxiliary_loss_mlp": 0.01107337, "balance_loss_clip": 1.04010284, "balance_loss_mlp": 1.10283303, "epoch": 0.00811663911017586, "flos": 20556545258880.0, "grad_norm": 4.696246405269054, "language_loss": 0.87248123, "learning_rate": 3.999368758297964e-06, "loss": 0.89716446, "num_input_tokens_seen": 2812815, "router_z_loss_clip": 0.671875, "router_z_loss_mlp": 2.578125, "step": 135, "time_per_iteration": 5.2392988204956055 }, { "auxiliary_loss_clip": 0.01362226, "auxiliary_loss_mlp": 0.01106394, "balance_loss_clip": 1.03935051, "balance_loss_mlp": 1.10586715, "epoch": 0.00817676236284383, "flos": 18798947994240.0, "grad_norm": 1.8825613314785814, "language_loss": 0.87810934, "learning_rate": 3.999359230764798e-06, "loss": 0.90279549, "num_input_tokens_seen": 2830445, "router_z_loss_clip": 0.671875, "router_z_loss_mlp": 2.5625, "step": 136, "time_per_iteration": 3.84043025970459 }, { "auxiliary_loss_clip": 0.01359523, "auxiliary_loss_mlp": 0.01091745, "balance_loss_clip": 1.0304718, "balance_loss_mlp": 1.10392106, "epoch": 0.008236885615511799, "flos": 23872596059520.0, "grad_norm": 2.1865065382853994, "language_loss": 0.82736731, "learning_rate": 3.999349631879643e-06, "loss": 0.85187995, "num_input_tokens_seen": 2846965, "router_z_loss_clip": 0.61328125, "router_z_loss_mlp": 2.5625, "step": 137, "time_per_iteration": 2.4926602840423584 }, { "auxiliary_loss_clip": 0.01355187, "auxiliary_loss_mlp": 0.01092266, "balance_loss_clip": 1.03099298, "balance_loss_mlp": 1.10456729, "epoch": 0.00829700886817977, "flos": 24642500561280.0, "grad_norm": 1.8798885548199515, "language_loss": 0.8933351, "learning_rate": 3.9993399616428425e-06, "loss": 0.91780961, "num_input_tokens_seen": 2867520, "router_z_loss_clip": 0.61328125, "router_z_loss_mlp": 2.5, "step": 138, "time_per_iteration": 2.527296543121338 }, { "auxiliary_loss_clip": 0.01346701, "auxiliary_loss_mlp": 0.01092938, "balance_loss_clip": 1.02894676, "balance_loss_mlp": 1.1021167, "epoch": 0.008357132120847738, "flos": 25260917207040.0, "grad_norm": 2.235375654561318, "language_loss": 0.9069339, "learning_rate": 3.999330220054742e-06, "loss": 0.93133026, "num_input_tokens_seen": 2885675, "router_z_loss_clip": 0.640625, "router_z_loss_mlp": 2.4375, "step": 139, "time_per_iteration": 2.5068202018737793 }, { "auxiliary_loss_clip": 0.01364767, "auxiliary_loss_mlp": 0.01099099, "balance_loss_clip": 1.03525043, "balance_loss_mlp": 1.10740197, "epoch": 0.008417255373515706, "flos": 27343660763520.0, "grad_norm": 2.359048254552134, "language_loss": 0.84722054, "learning_rate": 3.9993204071156894e-06, "loss": 0.87185919, "num_input_tokens_seen": 2905960, "router_z_loss_clip": 0.63671875, "router_z_loss_mlp": 2.5625, "step": 140, "time_per_iteration": 2.596464157104492 }, { "auxiliary_loss_clip": 0.01350993, "auxiliary_loss_mlp": 0.01098609, "balance_loss_clip": 1.03685844, "balance_loss_mlp": 1.10390186, "epoch": 0.008477378626183677, "flos": 17638120465920.0, "grad_norm": 7.5568190931205255, "language_loss": 0.8278836, "learning_rate": 3.999310522826034e-06, "loss": 0.85237962, "num_input_tokens_seen": 2922780, "router_z_loss_clip": 0.6171875, "router_z_loss_mlp": 2.46875, "step": 141, "time_per_iteration": 2.5674993991851807 }, { "auxiliary_loss_clip": 0.01353554, "auxiliary_loss_mlp": 0.01096234, "balance_loss_clip": 1.0332911, "balance_loss_mlp": 1.10089064, "epoch": 0.008537501878851645, "flos": 13880488389120.0, "grad_norm": 2.45829146302685, "language_loss": 0.79976183, "learning_rate": 3.999300567186129e-06, "loss": 0.8242597, "num_input_tokens_seen": 2938765, "router_z_loss_clip": 0.62890625, "router_z_loss_mlp": 2.53125, "step": 142, "time_per_iteration": 2.4487762451171875 }, { "auxiliary_loss_clip": 0.01293382, "auxiliary_loss_mlp": 0.01083486, "balance_loss_clip": 1.05849934, "balance_loss_mlp": 1.14034057, "epoch": 0.008597625131519616, "flos": 71244320832000.0, "grad_norm": 1.0288296118365807, "language_loss": 0.66773266, "learning_rate": 3.999290540196329e-06, "loss": 0.69150138, "num_input_tokens_seen": 3006665, "router_z_loss_clip": 0.24902344, "router_z_loss_mlp": 1.53125, "step": 143, "time_per_iteration": 3.269559144973755 }, { "auxiliary_loss_clip": 0.0135308, "auxiliary_loss_mlp": 0.01103362, "balance_loss_clip": 1.04156375, "balance_loss_mlp": 1.10496008, "epoch": 0.008657748384187584, "flos": 17601880608000.0, "grad_norm": 2.110396981770736, "language_loss": 0.83264089, "learning_rate": 3.999280441856992e-06, "loss": 0.85720533, "num_input_tokens_seen": 3024335, "router_z_loss_clip": 0.6171875, "router_z_loss_mlp": 2.46875, "step": 144, "time_per_iteration": 2.4857771396636963 }, { "auxiliary_loss_clip": 0.01341317, "auxiliary_loss_mlp": 0.01097107, "balance_loss_clip": 1.03240073, "balance_loss_mlp": 1.09779024, "epoch": 0.008717871636855555, "flos": 19714405420800.0, "grad_norm": 2.0613967466713885, "language_loss": 0.87342119, "learning_rate": 3.9992702721684805e-06, "loss": 0.89780545, "num_input_tokens_seen": 3043300, "router_z_loss_clip": 0.6484375, "router_z_loss_mlp": 2.4375, "step": 145, "time_per_iteration": 2.5223348140716553 }, { "auxiliary_loss_clip": 0.01353492, "auxiliary_loss_mlp": 0.01098152, "balance_loss_clip": 1.03301656, "balance_loss_mlp": 1.0989728, "epoch": 0.008777994889523523, "flos": 24716271997440.0, "grad_norm": 1.8762206084414534, "language_loss": 0.85667378, "learning_rate": 3.999260031131154e-06, "loss": 0.88119018, "num_input_tokens_seen": 3064610, "router_z_loss_clip": 0.6484375, "router_z_loss_mlp": 2.546875, "step": 146, "time_per_iteration": 2.530491828918457 }, { "auxiliary_loss_clip": 0.0127963, "auxiliary_loss_mlp": 0.01026299, "balance_loss_clip": 1.00312459, "balance_loss_mlp": 1.14265537, "epoch": 0.008838118142191492, "flos": 70128916715520.0, "grad_norm": 0.8141797365584135, "language_loss": 0.59914416, "learning_rate": 3.999249718745381e-06, "loss": 0.62220341, "num_input_tokens_seen": 3130385, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 1.375, "step": 147, "time_per_iteration": 3.2129898071289062 }, { "auxiliary_loss_clip": 0.01345115, "auxiliary_loss_mlp": 0.01096029, "balance_loss_clip": 1.03628087, "balance_loss_mlp": 1.10191417, "epoch": 0.008898241394859462, "flos": 20043845809920.0, "grad_norm": 1.8636165565969143, "language_loss": 0.83679867, "learning_rate": 3.999239335011527e-06, "loss": 0.86121005, "num_input_tokens_seen": 3149760, "router_z_loss_clip": 0.59765625, "router_z_loss_mlp": 2.4375, "step": 148, "time_per_iteration": 2.566246747970581 }, { "auxiliary_loss_clip": 0.01350047, "auxiliary_loss_mlp": 0.01117175, "balance_loss_clip": 1.05318344, "balance_loss_mlp": 1.09972465, "epoch": 0.008958364647527431, "flos": 10742843969280.0, "grad_norm": 2.214139356241396, "language_loss": 0.87434661, "learning_rate": 3.999228879929965e-06, "loss": 0.89901882, "num_input_tokens_seen": 3164500, "router_z_loss_clip": 0.640625, "router_z_loss_mlp": 2.5, "step": 149, "time_per_iteration": 2.482739210128784 }, { "auxiliary_loss_clip": 0.01352019, "auxiliary_loss_mlp": 0.01106148, "balance_loss_clip": 1.04434991, "balance_loss_mlp": 1.10022926, "epoch": 0.009018487900195401, "flos": 29126326250880.0, "grad_norm": 2.052042896827704, "language_loss": 0.92434806, "learning_rate": 3.999218353501066e-06, "loss": 0.94892967, "num_input_tokens_seen": 3182455, "router_z_loss_clip": 0.6171875, "router_z_loss_mlp": 2.515625, "step": 150, "time_per_iteration": 2.5825510025024414 }, { "auxiliary_loss_clip": 0.01343467, "auxiliary_loss_mlp": 0.01100108, "balance_loss_clip": 1.03931189, "balance_loss_mlp": 1.09753084, "epoch": 0.00907861115286337, "flos": 32962268240640.0, "grad_norm": 2.0323430661128237, "language_loss": 0.73467743, "learning_rate": 3.999207755725208e-06, "loss": 0.75911319, "num_input_tokens_seen": 3203995, "router_z_loss_clip": 0.609375, "router_z_loss_mlp": 2.453125, "step": 151, "time_per_iteration": 2.621089458465576 }, { "auxiliary_loss_clip": 0.01348765, "auxiliary_loss_mlp": 0.01104451, "balance_loss_clip": 1.0417949, "balance_loss_mlp": 1.09938061, "epoch": 0.009138734405531338, "flos": 21761362967040.0, "grad_norm": 2.3198228318377594, "language_loss": 0.87541401, "learning_rate": 3.999197086602766e-06, "loss": 0.89994621, "num_input_tokens_seen": 3222575, "router_z_loss_clip": 0.625, "router_z_loss_mlp": 2.484375, "step": 152, "time_per_iteration": 2.5434858798980713 }, { "auxiliary_loss_clip": 0.01342767, "auxiliary_loss_mlp": 0.01094005, "balance_loss_clip": 1.03306484, "balance_loss_mlp": 1.10060859, "epoch": 0.009198857658199309, "flos": 20841681265920.0, "grad_norm": 3.727614359588284, "language_loss": 0.8170656, "learning_rate": 3.9991863461341234e-06, "loss": 0.84143329, "num_input_tokens_seen": 3240180, "router_z_loss_clip": 0.609375, "router_z_loss_mlp": 2.421875, "step": 153, "time_per_iteration": 2.5529229640960693 }, { "auxiliary_loss_clip": 0.01340483, "auxiliary_loss_mlp": 0.01101178, "balance_loss_clip": 1.04081035, "balance_loss_mlp": 1.09398246, "epoch": 0.009258980910867277, "flos": 24826213468800.0, "grad_norm": 2.0004419209962077, "language_loss": 0.88920546, "learning_rate": 3.999175534319662e-06, "loss": 0.91362202, "num_input_tokens_seen": 3259800, "router_z_loss_clip": 0.6015625, "router_z_loss_mlp": 2.46875, "step": 154, "time_per_iteration": 2.498523712158203 }, { "auxiliary_loss_clip": 0.0134595, "auxiliary_loss_mlp": 0.01134348, "balance_loss_clip": 1.07116711, "balance_loss_mlp": 1.09875727, "epoch": 0.009319104163535248, "flos": 28766511112320.0, "grad_norm": 2.1190994788647393, "language_loss": 0.88754398, "learning_rate": 3.999164651159769e-06, "loss": 0.91234696, "num_input_tokens_seen": 3280400, "router_z_loss_clip": 0.6328125, "router_z_loss_mlp": 2.46875, "step": 155, "time_per_iteration": 2.5487866401672363 }, { "auxiliary_loss_clip": 0.01347773, "auxiliary_loss_mlp": 0.01106749, "balance_loss_clip": 1.04685855, "balance_loss_mlp": 1.09622169, "epoch": 0.009379227416203216, "flos": 16581055098240.0, "grad_norm": 2.63691628375773, "language_loss": 0.85156655, "learning_rate": 3.999153696654832e-06, "loss": 0.87611175, "num_input_tokens_seen": 3297600, "router_z_loss_clip": 0.59765625, "router_z_loss_mlp": 2.515625, "step": 156, "time_per_iteration": 2.453019380569458 }, { "auxiliary_loss_clip": 0.01344155, "auxiliary_loss_mlp": 0.01096222, "balance_loss_clip": 1.03523493, "balance_loss_mlp": 1.0993228, "epoch": 0.009439350668871187, "flos": 18329016827520.0, "grad_norm": 2.1940432790252355, "language_loss": 0.98876888, "learning_rate": 3.9991426708052416e-06, "loss": 1.01317263, "num_input_tokens_seen": 3313635, "router_z_loss_clip": 0.609375, "router_z_loss_mlp": 2.4375, "step": 157, "time_per_iteration": 2.4774229526519775 }, { "auxiliary_loss_clip": 0.01336484, "auxiliary_loss_mlp": 0.01122648, "balance_loss_clip": 1.06270957, "balance_loss_mlp": 1.09593248, "epoch": 0.009499473921539155, "flos": 24348846182400.0, "grad_norm": 4.935499386420861, "language_loss": 0.87422907, "learning_rate": 3.999131573611392e-06, "loss": 0.8988204, "num_input_tokens_seen": 3333735, "router_z_loss_clip": 0.6015625, "router_z_loss_mlp": 2.40625, "step": 158, "time_per_iteration": 2.5238518714904785 }, { "auxiliary_loss_clip": 0.01340118, "auxiliary_loss_mlp": 0.01117905, "balance_loss_clip": 1.05481958, "balance_loss_mlp": 1.09583664, "epoch": 0.009559597174207124, "flos": 16398389531520.0, "grad_norm": 2.8192939651199223, "language_loss": 0.85176593, "learning_rate": 3.999120405073679e-06, "loss": 0.87634623, "num_input_tokens_seen": 3348800, "router_z_loss_clip": 0.6328125, "router_z_loss_mlp": 2.4375, "step": 159, "time_per_iteration": 2.479688882827759 }, { "auxiliary_loss_clip": 0.0133384, "auxiliary_loss_mlp": 0.01106715, "balance_loss_clip": 1.04439223, "balance_loss_mlp": 1.09363759, "epoch": 0.009619720426875094, "flos": 22855785356160.0, "grad_norm": 1.9515699711388057, "language_loss": 0.85391754, "learning_rate": 3.9991091651925014e-06, "loss": 0.87832308, "num_input_tokens_seen": 3368595, "router_z_loss_clip": 0.625, "router_z_loss_mlp": 2.40625, "step": 160, "time_per_iteration": 2.4945571422576904 }, { "auxiliary_loss_clip": 0.01338913, "auxiliary_loss_mlp": 0.01099574, "balance_loss_clip": 1.04101801, "balance_loss_mlp": 1.09515488, "epoch": 0.009679843679543063, "flos": 19134009112320.0, "grad_norm": 2.5362498189638, "language_loss": 0.90829933, "learning_rate": 3.999097853968259e-06, "loss": 0.93268418, "num_input_tokens_seen": 3384975, "router_z_loss_clip": 0.5859375, "router_z_loss_mlp": 2.4375, "step": 161, "time_per_iteration": 2.4981765747070312 }, { "auxiliary_loss_clip": 0.01340961, "auxiliary_loss_mlp": 0.01090779, "balance_loss_clip": 1.03322458, "balance_loss_mlp": 1.09601188, "epoch": 0.009739966932211033, "flos": 20301958558080.0, "grad_norm": 2.448191667557215, "language_loss": 0.90737391, "learning_rate": 3.999086471401357e-06, "loss": 0.93169141, "num_input_tokens_seen": 3404755, "router_z_loss_clip": 0.57421875, "router_z_loss_mlp": 2.453125, "step": 162, "time_per_iteration": 2.5141706466674805 }, { "auxiliary_loss_clip": 0.01249292, "auxiliary_loss_mlp": 0.01097223, "balance_loss_clip": 1.0758605, "balance_loss_mlp": 1.11713386, "epoch": 0.009800090184879002, "flos": 67031073112320.0, "grad_norm": 1.1788821400625664, "language_loss": 0.72151339, "learning_rate": 3.9990750174922005e-06, "loss": 0.74497843, "num_input_tokens_seen": 3467210, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 1.3203125, "step": 163, "time_per_iteration": 3.0873191356658936 }, { "auxiliary_loss_clip": 0.01329569, "auxiliary_loss_mlp": 0.01099409, "balance_loss_clip": 1.0405674, "balance_loss_mlp": 1.09513378, "epoch": 0.00986021343754697, "flos": 17163755556480.0, "grad_norm": 2.5235900048992903, "language_loss": 0.83601165, "learning_rate": 3.9990634922412e-06, "loss": 0.86030143, "num_input_tokens_seen": 3483220, "router_z_loss_clip": 0.58984375, "router_z_loss_mlp": 2.34375, "step": 164, "time_per_iteration": 2.525221824645996 }, { "auxiliary_loss_clip": 0.01322732, "auxiliary_loss_mlp": 0.01090181, "balance_loss_clip": 1.03172088, "balance_loss_mlp": 1.08707738, "epoch": 0.00992033669021494, "flos": 17748445962240.0, "grad_norm": 2.0227093827211897, "language_loss": 0.88425285, "learning_rate": 3.9990518956487655e-06, "loss": 0.908382, "num_input_tokens_seen": 3501465, "router_z_loss_clip": 0.5859375, "router_z_loss_mlp": 2.34375, "step": 165, "time_per_iteration": 2.4771060943603516 }, { "auxiliary_loss_clip": 0.01331029, "auxiliary_loss_mlp": 0.01106093, "balance_loss_clip": 1.0435797, "balance_loss_mlp": 1.09030557, "epoch": 0.00998045994288291, "flos": 25296109724160.0, "grad_norm": 2.4914540557712663, "language_loss": 0.7945292, "learning_rate": 3.9990402277153105e-06, "loss": 0.81890035, "num_input_tokens_seen": 3520480, "router_z_loss_clip": 0.625, "router_z_loss_mlp": 2.40625, "step": 166, "time_per_iteration": 2.5510506629943848 }, { "auxiliary_loss_clip": 0.01329746, "auxiliary_loss_mlp": 0.0110877, "balance_loss_clip": 1.04873562, "balance_loss_mlp": 1.08858144, "epoch": 0.01004058319555088, "flos": 32297801644800.0, "grad_norm": 2.232365353759961, "language_loss": 0.9095034, "learning_rate": 3.999028488441252e-06, "loss": 0.93388855, "num_input_tokens_seen": 3539570, "router_z_loss_clip": 0.6015625, "router_z_loss_mlp": 2.40625, "step": 167, "time_per_iteration": 2.5629281997680664 }, { "auxiliary_loss_clip": 0.01327017, "auxiliary_loss_mlp": 0.01124412, "balance_loss_clip": 1.066715, "balance_loss_mlp": 1.09082603, "epoch": 0.010100706448218848, "flos": 11319365116800.0, "grad_norm": 5.029144748726258, "language_loss": 0.89412969, "learning_rate": 3.999016677827009e-06, "loss": 0.91864395, "num_input_tokens_seen": 3555465, "router_z_loss_clip": 0.578125, "router_z_loss_mlp": 2.359375, "step": 168, "time_per_iteration": 2.4647042751312256 }, { "auxiliary_loss_clip": 0.01318749, "auxiliary_loss_mlp": 0.01095637, "balance_loss_clip": 1.03674769, "balance_loss_mlp": 1.08649659, "epoch": 0.010160829700886819, "flos": 29718103662720.0, "grad_norm": 1.7973544688290481, "language_loss": 0.86207986, "learning_rate": 3.999004795873003e-06, "loss": 0.88622367, "num_input_tokens_seen": 3578970, "router_z_loss_clip": 0.5859375, "router_z_loss_mlp": 2.3125, "step": 169, "time_per_iteration": 2.584573984146118 }, { "auxiliary_loss_clip": 0.01321291, "auxiliary_loss_mlp": 0.01098267, "balance_loss_clip": 1.03785205, "balance_loss_mlp": 1.0876385, "epoch": 0.010220952953554787, "flos": 20411306536320.0, "grad_norm": 1.9927921484930773, "language_loss": 0.83916354, "learning_rate": 3.998992842579657e-06, "loss": 0.86335915, "num_input_tokens_seen": 3597275, "router_z_loss_clip": 0.6015625, "router_z_loss_mlp": 2.34375, "step": 170, "time_per_iteration": 2.49104642868042 }, { "auxiliary_loss_clip": 0.01333332, "auxiliary_loss_mlp": 0.01111543, "balance_loss_clip": 1.05127096, "balance_loss_mlp": 1.08931541, "epoch": 0.010281076206222756, "flos": 31283783850240.0, "grad_norm": 2.252100548603337, "language_loss": 0.89015782, "learning_rate": 3.9989808179474e-06, "loss": 0.91460657, "num_input_tokens_seen": 3618905, "router_z_loss_clip": 0.6015625, "router_z_loss_mlp": 2.4375, "step": 171, "time_per_iteration": 2.5218472480773926 }, { "auxiliary_loss_clip": 0.01321781, "auxiliary_loss_mlp": 0.01102424, "balance_loss_clip": 1.04322505, "balance_loss_mlp": 1.08822954, "epoch": 0.010341199458890726, "flos": 21981176087040.0, "grad_norm": 2.44019530662096, "language_loss": 0.88130593, "learning_rate": 3.998968721976658e-06, "loss": 0.90554798, "num_input_tokens_seen": 3639610, "router_z_loss_clip": 0.59375, "router_z_loss_mlp": 2.328125, "step": 172, "time_per_iteration": 2.5287842750549316 }, { "auxiliary_loss_clip": 0.01312446, "auxiliary_loss_mlp": 0.01096913, "balance_loss_clip": 1.03950167, "balance_loss_mlp": 1.0837512, "epoch": 0.010401322711558695, "flos": 30809209472640.0, "grad_norm": 1.803783117703605, "language_loss": 0.80085742, "learning_rate": 3.998956554667865e-06, "loss": 0.82495105, "num_input_tokens_seen": 3664030, "router_z_loss_clip": 0.5703125, "router_z_loss_mlp": 2.28125, "step": 173, "time_per_iteration": 2.5845518112182617 }, { "auxiliary_loss_clip": 0.01322341, "auxiliary_loss_mlp": 0.01106042, "balance_loss_clip": 1.04863095, "balance_loss_mlp": 1.08603311, "epoch": 0.010461445964226665, "flos": 24714037670400.0, "grad_norm": 1.8406425535950643, "language_loss": 0.82000279, "learning_rate": 3.998944316021455e-06, "loss": 0.84428656, "num_input_tokens_seen": 3683615, "router_z_loss_clip": 0.57421875, "router_z_loss_mlp": 2.375, "step": 174, "time_per_iteration": 5.432522535324097 }, { "auxiliary_loss_clip": 0.01321477, "auxiliary_loss_mlp": 0.01108623, "balance_loss_clip": 1.04925632, "balance_loss_mlp": 1.08508635, "epoch": 0.010521569216894634, "flos": 27709096631040.0, "grad_norm": 3.0811273337718794, "language_loss": 0.72226876, "learning_rate": 3.9989320060378634e-06, "loss": 0.74656975, "num_input_tokens_seen": 3704540, "router_z_loss_clip": 0.59375, "router_z_loss_mlp": 2.375, "step": 175, "time_per_iteration": 3.943849563598633 }, { "auxiliary_loss_clip": 0.0132293, "auxiliary_loss_mlp": 0.01111667, "balance_loss_clip": 1.0503459, "balance_loss_mlp": 1.08794677, "epoch": 0.010581692469562603, "flos": 12457533306240.0, "grad_norm": 2.5844798526471795, "language_loss": 0.96816218, "learning_rate": 3.998919624717531e-06, "loss": 0.99250817, "num_input_tokens_seen": 3721320, "router_z_loss_clip": 0.61328125, "router_z_loss_mlp": 2.359375, "step": 176, "time_per_iteration": 2.4978418350219727 }, { "auxiliary_loss_clip": 0.013113, "auxiliary_loss_mlp": 0.01106073, "balance_loss_clip": 1.05104566, "balance_loss_mlp": 1.0845108, "epoch": 0.010641815722230573, "flos": 19426581239040.0, "grad_norm": 2.4681889646441357, "language_loss": 0.76004493, "learning_rate": 3.998907172060898e-06, "loss": 0.78421861, "num_input_tokens_seen": 3739385, "router_z_loss_clip": 0.55078125, "router_z_loss_mlp": 2.265625, "step": 177, "time_per_iteration": 2.4572675228118896 }, { "auxiliary_loss_clip": 0.01319403, "auxiliary_loss_mlp": 0.0109398, "balance_loss_clip": 1.03518641, "balance_loss_mlp": 1.08404136, "epoch": 0.010701938974898541, "flos": 18331600268160.0, "grad_norm": 2.4012411743357758, "language_loss": 0.75504708, "learning_rate": 3.9988946480684115e-06, "loss": 0.77918088, "num_input_tokens_seen": 3756360, "router_z_loss_clip": 0.5859375, "router_z_loss_mlp": 2.359375, "step": 178, "time_per_iteration": 2.4346606731414795 }, { "auxiliary_loss_clip": 0.01326096, "auxiliary_loss_mlp": 0.01092432, "balance_loss_clip": 1.0330658, "balance_loss_mlp": 1.08672309, "epoch": 0.010762062227566512, "flos": 19203102426240.0, "grad_norm": 2.2127760174419295, "language_loss": 0.8330009, "learning_rate": 3.998882052740516e-06, "loss": 0.85718614, "num_input_tokens_seen": 3773930, "router_z_loss_clip": 0.59375, "router_z_loss_mlp": 2.40625, "step": 179, "time_per_iteration": 2.4346044063568115 }, { "auxiliary_loss_clip": 0.01315178, "auxiliary_loss_mlp": 0.01085268, "balance_loss_clip": 1.0242331, "balance_loss_mlp": 1.07958055, "epoch": 0.01082218548023448, "flos": 31424239716480.0, "grad_norm": 1.9845702506607297, "language_loss": 0.83313191, "learning_rate": 3.9988693860776616e-06, "loss": 0.85713637, "num_input_tokens_seen": 3793630, "router_z_loss_clip": 0.609375, "router_z_loss_mlp": 2.359375, "step": 180, "time_per_iteration": 2.625657558441162 }, { "auxiliary_loss_clip": 0.01318714, "auxiliary_loss_mlp": 0.01090594, "balance_loss_clip": 1.03275371, "balance_loss_mlp": 1.08474123, "epoch": 0.01088230873290245, "flos": 25045258538880.0, "grad_norm": 2.7088351245404394, "language_loss": 0.77022505, "learning_rate": 3.998856648080301e-06, "loss": 0.79431814, "num_input_tokens_seen": 3813610, "router_z_loss_clip": 0.578125, "router_z_loss_mlp": 2.34375, "step": 181, "time_per_iteration": 2.4812734127044678 }, { "auxiliary_loss_clip": 0.01312901, "auxiliary_loss_mlp": 0.01094778, "balance_loss_clip": 1.03638959, "balance_loss_mlp": 1.0799551, "epoch": 0.01094243198557042, "flos": 22892304504960.0, "grad_norm": 2.8413661282454994, "language_loss": 0.75974607, "learning_rate": 3.998843838748888e-06, "loss": 0.78382289, "num_input_tokens_seen": 3831390, "router_z_loss_clip": 0.5859375, "router_z_loss_mlp": 2.328125, "step": 182, "time_per_iteration": 2.48538875579834 }, { "auxiliary_loss_clip": 0.01313813, "auxiliary_loss_mlp": 0.01107224, "balance_loss_clip": 1.04876375, "balance_loss_mlp": 1.08262396, "epoch": 0.011002555238238388, "flos": 17164104670080.0, "grad_norm": 2.0731930225158735, "language_loss": 0.86371708, "learning_rate": 3.9988309580838796e-06, "loss": 0.88792747, "num_input_tokens_seen": 3849705, "router_z_loss_clip": 0.5859375, "router_z_loss_mlp": 2.3125, "step": 183, "time_per_iteration": 2.426077365875244 }, { "auxiliary_loss_clip": 0.01314354, "auxiliary_loss_mlp": 0.01103889, "balance_loss_clip": 1.04771781, "balance_loss_mlp": 1.08373165, "epoch": 0.011062678490906358, "flos": 22309045464960.0, "grad_norm": 2.040898168728987, "language_loss": 0.85582656, "learning_rate": 3.998818006085736e-06, "loss": 0.88000894, "num_input_tokens_seen": 3869230, "router_z_loss_clip": 0.5625, "router_z_loss_mlp": 2.3125, "step": 184, "time_per_iteration": 2.4764184951782227 }, { "auxiliary_loss_clip": 0.01309698, "auxiliary_loss_mlp": 0.0109481, "balance_loss_clip": 1.0387814, "balance_loss_mlp": 1.0818429, "epoch": 0.011122801743574327, "flos": 24387250544640.0, "grad_norm": 1.9350126346676748, "language_loss": 0.8281703, "learning_rate": 3.99880498275492e-06, "loss": 0.85221541, "num_input_tokens_seen": 3889735, "router_z_loss_clip": 0.55859375, "router_z_loss_mlp": 2.28125, "step": 185, "time_per_iteration": 2.5032644271850586 }, { "auxiliary_loss_clip": 0.01317324, "auxiliary_loss_mlp": 0.01090762, "balance_loss_clip": 1.03339851, "balance_loss_mlp": 1.08348095, "epoch": 0.011182924996242297, "flos": 18149283815040.0, "grad_norm": 2.111643803506232, "language_loss": 0.70618719, "learning_rate": 3.9987918880918946e-06, "loss": 0.73026806, "num_input_tokens_seen": 3908855, "router_z_loss_clip": 0.57421875, "router_z_loss_mlp": 2.34375, "step": 186, "time_per_iteration": 2.4431312084198 }, { "auxiliary_loss_clip": 0.01311425, "auxiliary_loss_mlp": 0.01093057, "balance_loss_clip": 1.03919864, "balance_loss_mlp": 1.07877183, "epoch": 0.011243048248910266, "flos": 15485899570560.0, "grad_norm": 2.569855957246881, "language_loss": 1.01050854, "learning_rate": 3.9987787220971295e-06, "loss": 1.03455341, "num_input_tokens_seen": 3923865, "router_z_loss_clip": 0.5390625, "router_z_loss_mlp": 2.328125, "step": 187, "time_per_iteration": 2.4374749660491943 }, { "auxiliary_loss_clip": 0.01303492, "auxiliary_loss_mlp": 0.01099116, "balance_loss_clip": 1.04146647, "balance_loss_mlp": 1.0788306, "epoch": 0.011303171501578235, "flos": 40915273420800.0, "grad_norm": 2.548479597549839, "language_loss": 0.7428776, "learning_rate": 3.9987654847710925e-06, "loss": 0.76690364, "num_input_tokens_seen": 3946870, "router_z_loss_clip": 0.578125, "router_z_loss_mlp": 2.25, "step": 188, "time_per_iteration": 2.6361711025238037 }, { "auxiliary_loss_clip": 0.01240898, "auxiliary_loss_mlp": 0.01070704, "balance_loss_clip": 1.04972339, "balance_loss_mlp": 1.1219821, "epoch": 0.011363294754246205, "flos": 66299607884160.0, "grad_norm": 0.7354062841985437, "language_loss": 0.56136906, "learning_rate": 3.998752176114257e-06, "loss": 0.58448505, "num_input_tokens_seen": 4010005, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 1.1875, "step": 189, "time_per_iteration": 3.160338878631592 }, { "auxiliary_loss_clip": 0.01305183, "auxiliary_loss_mlp": 0.01101284, "balance_loss_clip": 1.04456365, "balance_loss_mlp": 1.0788312, "epoch": 0.011423418006914174, "flos": 24899112120960.0, "grad_norm": 2.1583810659975446, "language_loss": 0.93964595, "learning_rate": 3.998738796127097e-06, "loss": 0.96371061, "num_input_tokens_seen": 4029035, "router_z_loss_clip": 0.56640625, "router_z_loss_mlp": 2.265625, "step": 190, "time_per_iteration": 2.4950332641601562 }, { "auxiliary_loss_clip": 0.0130113, "auxiliary_loss_mlp": 0.0108689, "balance_loss_clip": 1.03446198, "balance_loss_mlp": 1.07827342, "epoch": 0.011483541259582144, "flos": 19790865031680.0, "grad_norm": 2.8798929290255653, "language_loss": 0.84118086, "learning_rate": 3.998725344810092e-06, "loss": 0.8650611, "num_input_tokens_seen": 4046995, "router_z_loss_clip": 0.5234375, "router_z_loss_mlp": 2.234375, "step": 191, "time_per_iteration": 2.474900484085083 }, { "auxiliary_loss_clip": 0.01305111, "auxiliary_loss_mlp": 0.01097284, "balance_loss_clip": 1.04456949, "balance_loss_mlp": 1.0755477, "epoch": 0.011543664512250112, "flos": 26175746229120.0, "grad_norm": 1.8549438389464499, "language_loss": 0.91263413, "learning_rate": 3.99871182216372e-06, "loss": 0.93665814, "num_input_tokens_seen": 4065865, "router_z_loss_clip": 0.52734375, "router_z_loss_mlp": 2.296875, "step": 192, "time_per_iteration": 2.534113645553589 }, { "auxiliary_loss_clip": 0.01302597, "auxiliary_loss_mlp": 0.0110381, "balance_loss_clip": 1.04594564, "balance_loss_mlp": 1.07853484, "epoch": 0.011603787764918083, "flos": 23767856380800.0, "grad_norm": 2.087142401840174, "language_loss": 0.86185181, "learning_rate": 3.998698228188465e-06, "loss": 0.88591588, "num_input_tokens_seen": 4085305, "router_z_loss_clip": 0.578125, "router_z_loss_mlp": 2.25, "step": 193, "time_per_iteration": 2.4675991535186768 }, { "auxiliary_loss_clip": 0.01300011, "auxiliary_loss_mlp": 0.01100296, "balance_loss_clip": 1.04460144, "balance_loss_mlp": 1.07354963, "epoch": 0.011663911017586051, "flos": 25953594048000.0, "grad_norm": 2.5113253385065124, "language_loss": 0.91893256, "learning_rate": 3.9986845628848115e-06, "loss": 0.94293571, "num_input_tokens_seen": 4105185, "router_z_loss_clip": 0.5546875, "router_z_loss_mlp": 2.265625, "step": 194, "time_per_iteration": 2.4987685680389404 }, { "auxiliary_loss_clip": 0.01306807, "auxiliary_loss_mlp": 0.01093733, "balance_loss_clip": 1.0397315, "balance_loss_mlp": 1.07947361, "epoch": 0.01172403427025402, "flos": 17894173443840.0, "grad_norm": 2.357181387982265, "language_loss": 0.88895011, "learning_rate": 3.998670826253246e-06, "loss": 0.91295552, "num_input_tokens_seen": 4123160, "router_z_loss_clip": 0.5390625, "router_z_loss_mlp": 2.28125, "step": 195, "time_per_iteration": 2.435065507888794 }, { "auxiliary_loss_clip": 0.01300711, "auxiliary_loss_mlp": 0.0109839, "balance_loss_clip": 1.04267144, "balance_loss_mlp": 1.07752693, "epoch": 0.01178415752292199, "flos": 17893579950720.0, "grad_norm": 2.2137038281252814, "language_loss": 0.84706283, "learning_rate": 3.998657018294261e-06, "loss": 0.87105381, "num_input_tokens_seen": 4140425, "router_z_loss_clip": 0.55859375, "router_z_loss_mlp": 2.234375, "step": 196, "time_per_iteration": 2.449734926223755 }, { "auxiliary_loss_clip": 0.01303728, "auxiliary_loss_mlp": 0.01095433, "balance_loss_clip": 1.03992963, "balance_loss_mlp": 1.07673109, "epoch": 0.011844280775589959, "flos": 22892444150400.0, "grad_norm": 2.21390164074501, "language_loss": 0.9224143, "learning_rate": 3.998643139008348e-06, "loss": 0.94640595, "num_input_tokens_seen": 4159555, "router_z_loss_clip": 0.5546875, "router_z_loss_mlp": 2.265625, "step": 197, "time_per_iteration": 2.449037551879883 }, { "auxiliary_loss_clip": 0.0129758, "auxiliary_loss_mlp": 0.01095406, "balance_loss_clip": 1.04073656, "balance_loss_mlp": 1.07462072, "epoch": 0.01190440402825793, "flos": 26979097680000.0, "grad_norm": 1.9318876007715875, "language_loss": 0.78542089, "learning_rate": 3.998629188396002e-06, "loss": 0.80935079, "num_input_tokens_seen": 4180480, "router_z_loss_clip": 0.546875, "router_z_loss_mlp": 2.21875, "step": 198, "time_per_iteration": 2.5457894802093506 }, { "auxiliary_loss_clip": 0.01295783, "auxiliary_loss_mlp": 0.01092376, "balance_loss_clip": 1.03916073, "balance_loss_mlp": 1.07411599, "epoch": 0.011964527280925898, "flos": 20520549780480.0, "grad_norm": 1.9546481104925915, "language_loss": 0.87513494, "learning_rate": 3.9986151664577225e-06, "loss": 0.8990165, "num_input_tokens_seen": 4198835, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.21875, "step": 199, "time_per_iteration": 2.43874454498291 }, { "auxiliary_loss_clip": 0.01303553, "auxiliary_loss_mlp": 0.01105744, "balance_loss_clip": 1.04752231, "balance_loss_mlp": 1.07472932, "epoch": 0.012024650533593867, "flos": 27744742995840.0, "grad_norm": 1.9422627423945882, "language_loss": 0.8069098, "learning_rate": 3.998601073194007e-06, "loss": 0.83100271, "num_input_tokens_seen": 4219335, "router_z_loss_clip": 0.58203125, "router_z_loss_mlp": 2.28125, "step": 200, "time_per_iteration": 2.496249198913574 }, { "auxiliary_loss_clip": 0.01295899, "auxiliary_loss_mlp": 0.01089628, "balance_loss_clip": 1.033409, "balance_loss_mlp": 1.07088518, "epoch": 0.012084773786261837, "flos": 10451249360640.0, "grad_norm": 2.2723968814470914, "language_loss": 0.86802953, "learning_rate": 3.998586908605362e-06, "loss": 0.89188486, "num_input_tokens_seen": 4236940, "router_z_loss_clip": 0.5625, "router_z_loss_mlp": 2.25, "step": 201, "time_per_iteration": 2.4100606441497803 }, { "auxiliary_loss_clip": 0.01300387, "auxiliary_loss_mlp": 0.01099245, "balance_loss_clip": 1.04326439, "balance_loss_mlp": 1.07630789, "epoch": 0.012144897038929806, "flos": 23104821150720.0, "grad_norm": 1.7247674530220773, "language_loss": 0.83746284, "learning_rate": 3.99857267269229e-06, "loss": 0.86145914, "num_input_tokens_seen": 4256755, "router_z_loss_clip": 0.55859375, "router_z_loss_mlp": 2.25, "step": 202, "time_per_iteration": 2.495171546936035 }, { "auxiliary_loss_clip": 0.01290128, "auxiliary_loss_mlp": 0.01090998, "balance_loss_clip": 1.03797424, "balance_loss_mlp": 1.06840992, "epoch": 0.012205020291597776, "flos": 21032132065920.0, "grad_norm": 1.8188762982178481, "language_loss": 0.89072442, "learning_rate": 3.9985583654553e-06, "loss": 0.9145357, "num_input_tokens_seen": 4276505, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.21875, "step": 203, "time_per_iteration": 2.4408295154571533 }, { "auxiliary_loss_clip": 0.01245051, "auxiliary_loss_mlp": 0.0107076, "balance_loss_clip": 1.04958832, "balance_loss_mlp": 1.12554657, "epoch": 0.012265143544265745, "flos": 68444846507520.0, "grad_norm": 0.9907239168515048, "language_loss": 0.61084068, "learning_rate": 3.998543986894904e-06, "loss": 0.63399887, "num_input_tokens_seen": 4330965, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 1.1953125, "step": 204, "time_per_iteration": 3.0198233127593994 }, { "auxiliary_loss_clip": 0.01300788, "auxiliary_loss_mlp": 0.01098754, "balance_loss_clip": 1.04201007, "balance_loss_mlp": 1.07181823, "epoch": 0.012325266796933715, "flos": 17018307365760.0, "grad_norm": 2.2994236391340808, "language_loss": 0.90849137, "learning_rate": 3.9985295370116135e-06, "loss": 0.93248677, "num_input_tokens_seen": 4348200, "router_z_loss_clip": 0.56640625, "router_z_loss_mlp": 2.28125, "step": 205, "time_per_iteration": 2.4178733825683594 }, { "auxiliary_loss_clip": 0.0130442, "auxiliary_loss_mlp": 0.01111671, "balance_loss_clip": 1.0553323, "balance_loss_mlp": 1.07182193, "epoch": 0.012385390049601683, "flos": 20189119443840.0, "grad_norm": 2.3596882713470264, "language_loss": 0.88420367, "learning_rate": 3.998515015805945e-06, "loss": 0.90836465, "num_input_tokens_seen": 4365460, "router_z_loss_clip": 0.5625, "router_z_loss_mlp": 2.3125, "step": 206, "time_per_iteration": 2.469921588897705 }, { "auxiliary_loss_clip": 0.01295143, "auxiliary_loss_mlp": 0.01092373, "balance_loss_clip": 1.03915834, "balance_loss_mlp": 1.07029605, "epoch": 0.012445513302269652, "flos": 16252208202240.0, "grad_norm": 2.085862417309902, "language_loss": 0.94610721, "learning_rate": 3.998500423278416e-06, "loss": 0.96998239, "num_input_tokens_seen": 4383650, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.25, "step": 207, "time_per_iteration": 2.4121739864349365 }, { "auxiliary_loss_clip": 0.01296356, "auxiliary_loss_mlp": 0.01095686, "balance_loss_clip": 1.04142165, "balance_loss_mlp": 1.07297158, "epoch": 0.012505636554937622, "flos": 23768240405760.0, "grad_norm": 2.3786309079219756, "language_loss": 0.74965876, "learning_rate": 3.998485759429547e-06, "loss": 0.77357912, "num_input_tokens_seen": 4403765, "router_z_loss_clip": 0.54296875, "router_z_loss_mlp": 2.234375, "step": 208, "time_per_iteration": 2.555402994155884 }, { "auxiliary_loss_clip": 0.01286573, "auxiliary_loss_mlp": 0.01087111, "balance_loss_clip": 1.0330857, "balance_loss_mlp": 1.06945479, "epoch": 0.012565759807605591, "flos": 30590234225280.0, "grad_norm": 9.246201609274017, "language_loss": 0.98260844, "learning_rate": 3.998471024259863e-06, "loss": 1.00634527, "num_input_tokens_seen": 4421935, "router_z_loss_clip": 0.5390625, "router_z_loss_mlp": 2.171875, "step": 209, "time_per_iteration": 2.5096023082733154 }, { "auxiliary_loss_clip": 0.01297111, "auxiliary_loss_mlp": 0.01103891, "balance_loss_clip": 1.04996026, "balance_loss_mlp": 1.07318473, "epoch": 0.012625883060273561, "flos": 40111956881280.0, "grad_norm": 2.8645082772694743, "language_loss": 0.84888291, "learning_rate": 3.998456217769888e-06, "loss": 0.87289298, "num_input_tokens_seen": 4441470, "router_z_loss_clip": 0.5390625, "router_z_loss_mlp": 2.234375, "step": 210, "time_per_iteration": 2.654449939727783 }, { "auxiliary_loss_clip": 0.01288543, "auxiliary_loss_mlp": 0.01107358, "balance_loss_clip": 1.05447721, "balance_loss_mlp": 1.06919765, "epoch": 0.01268600631294153, "flos": 27087956899200.0, "grad_norm": 2.373016707420503, "language_loss": 0.96118057, "learning_rate": 3.998441339960152e-06, "loss": 0.98513967, "num_input_tokens_seen": 4459950, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.1875, "step": 211, "time_per_iteration": 2.4860005378723145 }, { "auxiliary_loss_clip": 0.01300727, "auxiliary_loss_mlp": 0.01111718, "balance_loss_clip": 1.0554986, "balance_loss_mlp": 1.07486534, "epoch": 0.012746129565609499, "flos": 16981823128320.0, "grad_norm": 2.2582161522130466, "language_loss": 0.94642508, "learning_rate": 3.998426390831185e-06, "loss": 0.97054946, "num_input_tokens_seen": 4478390, "router_z_loss_clip": 0.5625, "router_z_loss_mlp": 2.25, "step": 212, "time_per_iteration": 2.4837257862091064 }, { "auxiliary_loss_clip": 0.01289522, "auxiliary_loss_mlp": 0.01092615, "balance_loss_clip": 1.04164124, "balance_loss_mlp": 1.0723896, "epoch": 0.012806252818277469, "flos": 46531786216320.0, "grad_norm": 1.6722825749467651, "language_loss": 0.75558621, "learning_rate": 3.998411370383521e-06, "loss": 0.7794075, "num_input_tokens_seen": 4501665, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 2.171875, "step": 213, "time_per_iteration": 5.504590272903442 }, { "auxiliary_loss_clip": 0.01289584, "auxiliary_loss_mlp": 0.01098085, "balance_loss_clip": 1.0462532, "balance_loss_mlp": 1.06899095, "epoch": 0.012866376070945438, "flos": 14387846400000.0, "grad_norm": 6.023317248147852, "language_loss": 0.85730284, "learning_rate": 3.9983962786176945e-06, "loss": 0.88117963, "num_input_tokens_seen": 4519055, "router_z_loss_clip": 0.51953125, "router_z_loss_mlp": 2.203125, "step": 214, "time_per_iteration": 3.83119797706604 }, { "auxiliary_loss_clip": 0.01287974, "auxiliary_loss_mlp": 0.01113797, "balance_loss_clip": 1.05791199, "balance_loss_mlp": 1.07126224, "epoch": 0.012926499323613408, "flos": 26139611105280.0, "grad_norm": 1.9864199504565263, "language_loss": 0.76788223, "learning_rate": 3.9983811155342465e-06, "loss": 0.79189986, "num_input_tokens_seen": 4540870, "router_z_loss_clip": 0.55859375, "router_z_loss_mlp": 2.15625, "step": 215, "time_per_iteration": 3.860884666442871 }, { "auxiliary_loss_clip": 0.01301141, "auxiliary_loss_mlp": 0.01100734, "balance_loss_clip": 1.04892564, "balance_loss_mlp": 1.07581246, "epoch": 0.012986622576281377, "flos": 30115904227200.0, "grad_norm": 2.076965771857895, "language_loss": 0.89427274, "learning_rate": 3.998365881133717e-06, "loss": 0.91829151, "num_input_tokens_seen": 4560395, "router_z_loss_clip": 0.515625, "router_z_loss_mlp": 2.25, "step": 216, "time_per_iteration": 2.5689918994903564 }, { "auxiliary_loss_clip": 0.01289735, "auxiliary_loss_mlp": 0.01099849, "balance_loss_clip": 1.04475069, "balance_loss_mlp": 1.06854296, "epoch": 0.013046745828949347, "flos": 13953177573120.0, "grad_norm": 2.822835372352441, "language_loss": 0.93123031, "learning_rate": 3.998350575416648e-06, "loss": 0.95512605, "num_input_tokens_seen": 4575785, "router_z_loss_clip": 0.55078125, "router_z_loss_mlp": 2.21875, "step": 217, "time_per_iteration": 2.40677809715271 }, { "auxiliary_loss_clip": 0.0128734, "auxiliary_loss_mlp": 0.01093811, "balance_loss_clip": 1.03728223, "balance_loss_mlp": 1.06867433, "epoch": 0.013106869081617315, "flos": 17346874970880.0, "grad_norm": 1.96990105044481, "language_loss": 0.92702591, "learning_rate": 3.9983351983835885e-06, "loss": 0.95083737, "num_input_tokens_seen": 4594985, "router_z_loss_clip": 0.56640625, "router_z_loss_mlp": 2.1875, "step": 218, "time_per_iteration": 2.459202766418457 }, { "auxiliary_loss_clip": 0.01281061, "auxiliary_loss_mlp": 0.01092205, "balance_loss_clip": 1.0364871, "balance_loss_mlp": 1.06463194, "epoch": 0.013166992334285284, "flos": 25883732684160.0, "grad_norm": 2.0566708002092895, "language_loss": 0.85948598, "learning_rate": 3.998319750035087e-06, "loss": 0.88321859, "num_input_tokens_seen": 4616125, "router_z_loss_clip": 0.55859375, "router_z_loss_mlp": 2.15625, "step": 219, "time_per_iteration": 2.4813380241394043 }, { "auxiliary_loss_clip": 0.01284623, "auxiliary_loss_mlp": 0.01085259, "balance_loss_clip": 1.03380799, "balance_loss_mlp": 1.06691563, "epoch": 0.013227115586953254, "flos": 31174610428800.0, "grad_norm": 1.7846173857072796, "language_loss": 0.87097883, "learning_rate": 3.998304230371692e-06, "loss": 0.8946777, "num_input_tokens_seen": 4637795, "router_z_loss_clip": 0.515625, "router_z_loss_mlp": 2.171875, "step": 220, "time_per_iteration": 2.534933090209961 }, { "auxiliary_loss_clip": 0.01278089, "auxiliary_loss_mlp": 0.01087831, "balance_loss_clip": 1.03773904, "balance_loss_mlp": 1.06271708, "epoch": 0.013287238839621223, "flos": 20408513627520.0, "grad_norm": 1.8386479521990724, "language_loss": 0.86070645, "learning_rate": 3.99828863939396e-06, "loss": 0.88436568, "num_input_tokens_seen": 4656835, "router_z_loss_clip": 0.5, "router_z_loss_mlp": 2.15625, "step": 221, "time_per_iteration": 2.4281139373779297 }, { "auxiliary_loss_clip": 0.01285994, "auxiliary_loss_mlp": 0.01091569, "balance_loss_clip": 1.03704226, "balance_loss_mlp": 1.06290507, "epoch": 0.013347362092289193, "flos": 14136262076160.0, "grad_norm": 2.093337358758933, "language_loss": 0.91403848, "learning_rate": 3.998272977102448e-06, "loss": 0.93781406, "num_input_tokens_seen": 4673015, "router_z_loss_clip": 0.546875, "router_z_loss_mlp": 2.21875, "step": 222, "time_per_iteration": 2.4241268634796143 }, { "auxiliary_loss_clip": 0.01277546, "auxiliary_loss_mlp": 0.01089054, "balance_loss_clip": 1.03266788, "balance_loss_mlp": 1.06376529, "epoch": 0.013407485344957162, "flos": 21796660218240.0, "grad_norm": 2.2189139260236526, "language_loss": 0.94726562, "learning_rate": 3.998257243497712e-06, "loss": 0.97093159, "num_input_tokens_seen": 4692355, "router_z_loss_clip": 0.5625, "router_z_loss_mlp": 2.140625, "step": 223, "time_per_iteration": 2.4427013397216797 }, { "auxiliary_loss_clip": 0.01275896, "auxiliary_loss_mlp": 0.01088529, "balance_loss_clip": 1.03576696, "balance_loss_mlp": 1.06117606, "epoch": 0.013467608597625132, "flos": 18620716170240.0, "grad_norm": 2.8381194576812163, "language_loss": 0.87496227, "learning_rate": 3.998241438580316e-06, "loss": 0.89860654, "num_input_tokens_seen": 4710080, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.15625, "step": 224, "time_per_iteration": 2.4462950229644775 }, { "auxiliary_loss_clip": 0.01276996, "auxiliary_loss_mlp": 0.01084642, "balance_loss_clip": 1.02935266, "balance_loss_mlp": 1.06108713, "epoch": 0.013527731850293101, "flos": 18551308654080.0, "grad_norm": 2.1273599177350144, "language_loss": 0.88692373, "learning_rate": 3.998225562350823e-06, "loss": 0.9105401, "num_input_tokens_seen": 4728980, "router_z_loss_clip": 0.5546875, "router_z_loss_mlp": 2.15625, "step": 225, "time_per_iteration": 2.4228098392486572 }, { "auxiliary_loss_clip": 0.01272484, "auxiliary_loss_mlp": 0.01101737, "balance_loss_clip": 1.04432523, "balance_loss_mlp": 1.06158555, "epoch": 0.01358785510296107, "flos": 19164558418560.0, "grad_norm": 1.815503315808218, "language_loss": 0.98583525, "learning_rate": 3.998209614809799e-06, "loss": 1.00957751, "num_input_tokens_seen": 4747020, "router_z_loss_clip": 0.57421875, "router_z_loss_mlp": 2.109375, "step": 226, "time_per_iteration": 2.455432176589966 }, { "auxiliary_loss_clip": 0.01278348, "auxiliary_loss_mlp": 0.01087459, "balance_loss_clip": 1.03457808, "balance_loss_mlp": 1.06498325, "epoch": 0.01364797835562904, "flos": 23328858545280.0, "grad_norm": 2.7114359242763126, "language_loss": 0.90125763, "learning_rate": 3.9981935959578145e-06, "loss": 0.92491573, "num_input_tokens_seen": 4765000, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.125, "step": 227, "time_per_iteration": 2.450788736343384 }, { "auxiliary_loss_clip": 0.01199205, "auxiliary_loss_mlp": 0.01042575, "balance_loss_clip": 1.0257901, "balance_loss_mlp": 1.0826751, "epoch": 0.013708101608297009, "flos": 70989943599360.0, "grad_norm": 0.9286409859088512, "language_loss": 0.57483828, "learning_rate": 3.99817750579544e-06, "loss": 0.59725606, "num_input_tokens_seen": 4833210, "router_z_loss_clip": 0.16796875, "router_z_loss_mlp": 1.171875, "step": 228, "time_per_iteration": 3.1797547340393066 }, { "auxiliary_loss_clip": 0.01271341, "auxiliary_loss_mlp": 0.01092144, "balance_loss_clip": 1.04128909, "balance_loss_mlp": 1.06173563, "epoch": 0.013768224860964979, "flos": 16324268981760.0, "grad_norm": 2.2875216044741458, "language_loss": 0.86467117, "learning_rate": 3.998161344323251e-06, "loss": 0.88830602, "num_input_tokens_seen": 4850120, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 2.09375, "step": 229, "time_per_iteration": 2.410629987716675 }, { "auxiliary_loss_clip": 0.01274439, "auxiliary_loss_mlp": 0.01091076, "balance_loss_clip": 1.03631115, "balance_loss_mlp": 1.05819178, "epoch": 0.013828348113632948, "flos": 20192017086720.0, "grad_norm": 4.939246484573645, "language_loss": 0.83541977, "learning_rate": 3.998145111541823e-06, "loss": 0.85907495, "num_input_tokens_seen": 4866215, "router_z_loss_clip": 0.546875, "router_z_loss_mlp": 2.15625, "step": 230, "time_per_iteration": 2.4956488609313965 }, { "auxiliary_loss_clip": 0.01271041, "auxiliary_loss_mlp": 0.01091215, "balance_loss_clip": 1.04019332, "balance_loss_mlp": 1.05893278, "epoch": 0.013888471366300916, "flos": 20740013786880.0, "grad_norm": 1.8456005596458809, "language_loss": 0.89727223, "learning_rate": 3.998128807451736e-06, "loss": 0.9208948, "num_input_tokens_seen": 4885630, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 2.125, "step": 231, "time_per_iteration": 2.4690020084381104 }, { "auxiliary_loss_clip": 0.01272094, "auxiliary_loss_mlp": 0.01096445, "balance_loss_clip": 1.0455898, "balance_loss_mlp": 1.05948091, "epoch": 0.013948594618968886, "flos": 22089546547200.0, "grad_norm": 2.7312656935955193, "language_loss": 0.83418334, "learning_rate": 3.9981124320535715e-06, "loss": 0.85786867, "num_input_tokens_seen": 4905570, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 2.125, "step": 232, "time_per_iteration": 2.4867002964019775 }, { "auxiliary_loss_clip": 0.01279185, "auxiliary_loss_mlp": 0.01088507, "balance_loss_clip": 1.03290725, "balance_loss_mlp": 1.05728006, "epoch": 0.014008717871636855, "flos": 19062087978240.0, "grad_norm": 3.2014549539474055, "language_loss": 0.73482341, "learning_rate": 3.998095985347915e-06, "loss": 0.75850034, "num_input_tokens_seen": 4923535, "router_z_loss_clip": 0.5546875, "router_z_loss_mlp": 2.21875, "step": 233, "time_per_iteration": 2.4370038509368896 }, { "auxiliary_loss_clip": 0.01278501, "auxiliary_loss_mlp": 0.0110119, "balance_loss_clip": 1.04537606, "balance_loss_mlp": 1.06214762, "epoch": 0.014068841124304825, "flos": 14530152568320.0, "grad_norm": 2.3160215159406676, "language_loss": 0.84934628, "learning_rate": 3.998079467335351e-06, "loss": 0.87314326, "num_input_tokens_seen": 4939200, "router_z_loss_clip": 0.55859375, "router_z_loss_mlp": 2.15625, "step": 234, "time_per_iteration": 2.4662678241729736 }, { "auxiliary_loss_clip": 0.01272153, "auxiliary_loss_mlp": 0.01089142, "balance_loss_clip": 1.03957438, "balance_loss_mlp": 1.05987799, "epoch": 0.014128964376972794, "flos": 18076420074240.0, "grad_norm": 2.50023724061121, "language_loss": 0.88307524, "learning_rate": 3.998062878016471e-06, "loss": 0.90668821, "num_input_tokens_seen": 4956620, "router_z_loss_clip": 0.49609375, "router_z_loss_mlp": 2.125, "step": 235, "time_per_iteration": 2.4285151958465576 }, { "auxiliary_loss_clip": 0.01270289, "auxiliary_loss_mlp": 0.01100538, "balance_loss_clip": 1.0480144, "balance_loss_mlp": 1.06112003, "epoch": 0.014189087629640764, "flos": 25333257277440.0, "grad_norm": 2.2535235264048796, "language_loss": 0.85064286, "learning_rate": 3.998046217391867e-06, "loss": 0.87435114, "num_input_tokens_seen": 4975650, "router_z_loss_clip": 0.52734375, "router_z_loss_mlp": 2.09375, "step": 236, "time_per_iteration": 2.5147581100463867 }, { "auxiliary_loss_clip": 0.01272199, "auxiliary_loss_mlp": 0.01086088, "balance_loss_clip": 1.03139448, "balance_loss_mlp": 1.05876279, "epoch": 0.014249210882308733, "flos": 36138212288640.0, "grad_norm": 1.9080197335876328, "language_loss": 0.81960863, "learning_rate": 3.9980294854621325e-06, "loss": 0.8431915, "num_input_tokens_seen": 4997415, "router_z_loss_clip": 0.546875, "router_z_loss_mlp": 2.140625, "step": 237, "time_per_iteration": 2.575861692428589 }, { "auxiliary_loss_clip": 0.01267233, "auxiliary_loss_mlp": 0.0110332, "balance_loss_clip": 1.04819798, "balance_loss_mlp": 1.05855203, "epoch": 0.014309334134976702, "flos": 12932142595200.0, "grad_norm": 2.3071106764312312, "language_loss": 0.76282841, "learning_rate": 3.998012682227866e-06, "loss": 0.78653395, "num_input_tokens_seen": 5013905, "router_z_loss_clip": 0.55078125, "router_z_loss_mlp": 2.09375, "step": 238, "time_per_iteration": 2.439086675643921 }, { "auxiliary_loss_clip": 0.0126541, "auxiliary_loss_mlp": 0.01091746, "balance_loss_clip": 1.04174924, "balance_loss_mlp": 1.05934358, "epoch": 0.014369457387644672, "flos": 20776463112960.0, "grad_norm": 2.189294612300346, "language_loss": 0.86273628, "learning_rate": 3.9979958076896655e-06, "loss": 0.88630784, "num_input_tokens_seen": 5033645, "router_z_loss_clip": 0.5, "router_z_loss_mlp": 2.0625, "step": 239, "time_per_iteration": 2.4491758346557617 }, { "auxiliary_loss_clip": 0.01255269, "auxiliary_loss_mlp": 0.0108271, "balance_loss_clip": 1.03245175, "balance_loss_mlp": 1.05461502, "epoch": 0.01442958064031264, "flos": 25847353180800.0, "grad_norm": 2.0680005127153183, "language_loss": 0.92302793, "learning_rate": 3.997978861848135e-06, "loss": 0.94640774, "num_input_tokens_seen": 5052875, "router_z_loss_clip": 0.50390625, "router_z_loss_mlp": 2.015625, "step": 240, "time_per_iteration": 2.511864185333252 }, { "auxiliary_loss_clip": 0.01260894, "auxiliary_loss_mlp": 0.01088302, "balance_loss_clip": 1.03828204, "balance_loss_mlp": 1.05558801, "epoch": 0.014489703892980611, "flos": 28218479500800.0, "grad_norm": 2.0359378222038345, "language_loss": 0.84616089, "learning_rate": 3.997961844703877e-06, "loss": 0.86965281, "num_input_tokens_seen": 5075005, "router_z_loss_clip": 0.5, "router_z_loss_mlp": 2.046875, "step": 241, "time_per_iteration": 2.5065231323242188 }, { "auxiliary_loss_clip": 0.01259675, "auxiliary_loss_mlp": 0.01095578, "balance_loss_clip": 1.04260087, "balance_loss_mlp": 1.06072092, "epoch": 0.01454982714564858, "flos": 22489860729600.0, "grad_norm": 2.1281324971549664, "language_loss": 0.87685394, "learning_rate": 3.997944756257501e-06, "loss": 0.90040648, "num_input_tokens_seen": 5091875, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 1.984375, "step": 242, "time_per_iteration": 2.4625511169433594 }, { "auxiliary_loss_clip": 0.01262409, "auxiliary_loss_mlp": 0.01079613, "balance_loss_clip": 1.02866244, "balance_loss_mlp": 1.05539179, "epoch": 0.014609950398316548, "flos": 21652119722880.0, "grad_norm": 2.0902873867775877, "language_loss": 0.85707223, "learning_rate": 3.997927596509616e-06, "loss": 0.88049245, "num_input_tokens_seen": 5111290, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 2.0625, "step": 243, "time_per_iteration": 2.4473350048065186 }, { "auxiliary_loss_clip": 0.01269157, "auxiliary_loss_mlp": 0.0109046, "balance_loss_clip": 1.03746009, "balance_loss_mlp": 1.05946577, "epoch": 0.014670073650984519, "flos": 21868965377280.0, "grad_norm": 1.566170571801324, "language_loss": 0.83990335, "learning_rate": 3.997910365460834e-06, "loss": 0.86349952, "num_input_tokens_seen": 5132265, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.09375, "step": 244, "time_per_iteration": 2.5066041946411133 }, { "auxiliary_loss_clip": 0.01268433, "auxiliary_loss_mlp": 0.01101666, "balance_loss_clip": 1.04601955, "balance_loss_mlp": 1.05904102, "epoch": 0.014730196903652487, "flos": 23182642304640.0, "grad_norm": 2.681066411928938, "language_loss": 0.78249276, "learning_rate": 3.9978930631117705e-06, "loss": 0.80619383, "num_input_tokens_seen": 5148575, "router_z_loss_clip": 0.5546875, "router_z_loss_mlp": 2.09375, "step": 245, "time_per_iteration": 2.4403247833251953 }, { "auxiliary_loss_clip": 0.01270861, "auxiliary_loss_mlp": 0.01090869, "balance_loss_clip": 1.03546047, "balance_loss_mlp": 1.05506361, "epoch": 0.014790320156320457, "flos": 23221465603200.0, "grad_norm": 1.9837610932174923, "language_loss": 0.83586812, "learning_rate": 3.997875689463043e-06, "loss": 0.85948539, "num_input_tokens_seen": 5170415, "router_z_loss_clip": 0.5546875, "router_z_loss_mlp": 2.15625, "step": 246, "time_per_iteration": 2.513209581375122 }, { "auxiliary_loss_clip": 0.01264415, "auxiliary_loss_mlp": 0.01085882, "balance_loss_clip": 1.03235722, "balance_loss_mlp": 1.05463314, "epoch": 0.014850443408988426, "flos": 15814571909760.0, "grad_norm": 2.458078485465398, "language_loss": 0.89064759, "learning_rate": 3.9978582445152705e-06, "loss": 0.9141506, "num_input_tokens_seen": 5188565, "router_z_loss_clip": 0.53515625, "router_z_loss_mlp": 2.09375, "step": 247, "time_per_iteration": 2.411815881729126 }, { "auxiliary_loss_clip": 0.01265272, "auxiliary_loss_mlp": 0.01078643, "balance_loss_clip": 1.02516592, "balance_loss_mlp": 1.05134773, "epoch": 0.014910566661656396, "flos": 22780617465600.0, "grad_norm": 2.05474741939736, "language_loss": 0.77996743, "learning_rate": 3.997840728269077e-06, "loss": 0.8034066, "num_input_tokens_seen": 5207810, "router_z_loss_clip": 0.53515625, "router_z_loss_mlp": 2.140625, "step": 248, "time_per_iteration": 2.5023632049560547 }, { "auxiliary_loss_clip": 0.01266455, "auxiliary_loss_mlp": 0.01094696, "balance_loss_clip": 1.04407978, "balance_loss_mlp": 1.05812287, "epoch": 0.014970689914324365, "flos": 26863954416000.0, "grad_norm": 1.9115641066417266, "language_loss": 0.83001065, "learning_rate": 3.997823140725088e-06, "loss": 0.8536222, "num_input_tokens_seen": 5226210, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 2.09375, "step": 249, "time_per_iteration": 2.513101100921631 }, { "auxiliary_loss_clip": 0.01264516, "auxiliary_loss_mlp": 0.01087825, "balance_loss_clip": 1.03763819, "balance_loss_mlp": 1.05710781, "epoch": 0.015030813166992334, "flos": 13984948776960.0, "grad_norm": 3.3408272000276846, "language_loss": 0.92655754, "learning_rate": 3.997805481883929e-06, "loss": 0.95008093, "num_input_tokens_seen": 5241660, "router_z_loss_clip": 0.5, "router_z_loss_mlp": 2.078125, "step": 250, "time_per_iteration": 2.465294361114502 }, { "auxiliary_loss_clip": 0.01271166, "auxiliary_loss_mlp": 0.01107342, "balance_loss_clip": 1.05267227, "balance_loss_mlp": 1.05898547, "epoch": 0.015090936419660304, "flos": 24716656022400.0, "grad_norm": 2.6396424242306686, "language_loss": 0.96257102, "learning_rate": 3.997787751746231e-06, "loss": 0.98635614, "num_input_tokens_seen": 5261090, "router_z_loss_clip": 0.546875, "router_z_loss_mlp": 2.125, "step": 251, "time_per_iteration": 2.4686830043792725 }, { "auxiliary_loss_clip": 0.01261888, "auxiliary_loss_mlp": 0.01093252, "balance_loss_clip": 1.04177761, "balance_loss_mlp": 1.05475163, "epoch": 0.015151059672328273, "flos": 25737621177600.0, "grad_norm": 2.3097383613973905, "language_loss": 0.83784211, "learning_rate": 3.997769950312628e-06, "loss": 0.86139357, "num_input_tokens_seen": 5279175, "router_z_loss_clip": 0.515625, "router_z_loss_mlp": 2.0625, "step": 252, "time_per_iteration": 3.93217134475708 }, { "auxiliary_loss_clip": 0.01259553, "auxiliary_loss_mlp": 0.01094354, "balance_loss_clip": 1.04118657, "balance_loss_mlp": 1.05521631, "epoch": 0.015211182924996243, "flos": 21870152363520.0, "grad_norm": 2.096397039732292, "language_loss": 0.97462344, "learning_rate": 3.997752077583753e-06, "loss": 0.99816239, "num_input_tokens_seen": 5296975, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.046875, "step": 253, "time_per_iteration": 5.36123251914978 }, { "auxiliary_loss_clip": 0.01183241, "auxiliary_loss_mlp": 0.01029227, "balance_loss_clip": 1.01310992, "balance_loss_mlp": 1.07796979, "epoch": 0.015271306177664212, "flos": 66891734409600.0, "grad_norm": 0.838537053241808, "language_loss": 0.55493897, "learning_rate": 3.997734133560246e-06, "loss": 0.57706368, "num_input_tokens_seen": 5358375, "router_z_loss_clip": 0.16113281, "router_z_loss_mlp": 1.0546875, "step": 254, "time_per_iteration": 3.1195199489593506 }, { "auxiliary_loss_clip": 0.01263656, "auxiliary_loss_mlp": 0.01102612, "balance_loss_clip": 1.04877758, "balance_loss_mlp": 1.05328155, "epoch": 0.01533142943033218, "flos": 26832846528000.0, "grad_norm": 2.146919372189757, "language_loss": 0.89907759, "learning_rate": 3.997716118242746e-06, "loss": 0.92274028, "num_input_tokens_seen": 5377255, "router_z_loss_clip": 0.5390625, "router_z_loss_mlp": 2.109375, "step": 255, "time_per_iteration": 3.897091865539551 }, { "auxiliary_loss_clip": 0.01261725, "auxiliary_loss_mlp": 0.01104855, "balance_loss_clip": 1.05206895, "balance_loss_mlp": 1.05353701, "epoch": 0.01539155268300015, "flos": 20812702970880.0, "grad_norm": 2.1854471015532435, "language_loss": 0.84855503, "learning_rate": 3.997698031631898e-06, "loss": 0.87222087, "num_input_tokens_seen": 5395320, "router_z_loss_clip": 0.52734375, "router_z_loss_mlp": 2.078125, "step": 256, "time_per_iteration": 2.495079755783081 }, { "auxiliary_loss_clip": 0.01264272, "auxiliary_loss_mlp": 0.01096694, "balance_loss_clip": 1.04481387, "balance_loss_mlp": 1.05318999, "epoch": 0.01545167593566812, "flos": 15960927795840.0, "grad_norm": 3.135350269790941, "language_loss": 0.70954847, "learning_rate": 3.997679873728344e-06, "loss": 0.73315823, "num_input_tokens_seen": 5411970, "router_z_loss_clip": 0.51953125, "router_z_loss_mlp": 2.109375, "step": 257, "time_per_iteration": 2.3912618160247803 }, { "auxiliary_loss_clip": 0.01261377, "auxiliary_loss_mlp": 0.01097662, "balance_loss_clip": 1.04547238, "balance_loss_mlp": 1.05512738, "epoch": 0.01551179918833609, "flos": 22600640073600.0, "grad_norm": 2.5275914771710566, "language_loss": 0.94030905, "learning_rate": 3.9976616445327355e-06, "loss": 0.96389937, "num_input_tokens_seen": 5430245, "router_z_loss_clip": 0.5234375, "router_z_loss_mlp": 2.0625, "step": 258, "time_per_iteration": 2.523621082305908 }, { "auxiliary_loss_clip": 0.0125721, "auxiliary_loss_mlp": 0.01084906, "balance_loss_clip": 1.03304982, "balance_loss_mlp": 1.05085206, "epoch": 0.015571922441004058, "flos": 22815705248640.0, "grad_norm": 2.701701695211177, "language_loss": 0.92466164, "learning_rate": 3.9976433440457205e-06, "loss": 0.94808275, "num_input_tokens_seen": 5448905, "router_z_loss_clip": 0.51953125, "router_z_loss_mlp": 2.0625, "step": 259, "time_per_iteration": 2.45497465133667 }, { "auxiliary_loss_clip": 0.01253468, "auxiliary_loss_mlp": 0.0108324, "balance_loss_clip": 1.03674841, "balance_loss_mlp": 1.05464232, "epoch": 0.015632045693672027, "flos": 18946595600640.0, "grad_norm": 1.7812673312303993, "language_loss": 0.96986514, "learning_rate": 3.997624972267954e-06, "loss": 0.99323225, "num_input_tokens_seen": 5466405, "router_z_loss_clip": 0.46484375, "router_z_loss_mlp": 1.9921875, "step": 260, "time_per_iteration": 2.4638025760650635 }, { "auxiliary_loss_clip": 0.01264476, "auxiliary_loss_mlp": 0.01097613, "balance_loss_clip": 1.04480374, "balance_loss_mlp": 1.05541444, "epoch": 0.015692168946339995, "flos": 29970421125120.0, "grad_norm": 2.0705749401091733, "language_loss": 0.87201715, "learning_rate": 3.99760652920009e-06, "loss": 0.89563799, "num_input_tokens_seen": 5487055, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.09375, "step": 261, "time_per_iteration": 2.5186092853546143 }, { "auxiliary_loss_clip": 0.0126125, "auxiliary_loss_mlp": 0.01086343, "balance_loss_clip": 1.03577399, "balance_loss_mlp": 1.05366397, "epoch": 0.015752292199007967, "flos": 19391039608320.0, "grad_norm": 1.9911084105028154, "language_loss": 0.66606891, "learning_rate": 3.997588014842788e-06, "loss": 0.68954486, "num_input_tokens_seen": 5506600, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 2.078125, "step": 262, "time_per_iteration": 2.4510912895202637 }, { "auxiliary_loss_clip": 0.01257533, "auxiliary_loss_mlp": 0.01103056, "balance_loss_clip": 1.0508666, "balance_loss_mlp": 1.0543381, "epoch": 0.015812415451675936, "flos": 20338756997760.0, "grad_norm": 2.204412624175132, "language_loss": 0.6779955, "learning_rate": 3.997569429196708e-06, "loss": 0.70160139, "num_input_tokens_seen": 5524350, "router_z_loss_clip": 0.5234375, "router_z_loss_mlp": 2.03125, "step": 263, "time_per_iteration": 2.4451043605804443 }, { "auxiliary_loss_clip": 0.01260264, "auxiliary_loss_mlp": 0.01090615, "balance_loss_clip": 1.03990364, "balance_loss_mlp": 1.05124879, "epoch": 0.015872538704343905, "flos": 17524583124480.0, "grad_norm": 2.9410747460535283, "language_loss": 0.84258455, "learning_rate": 3.997550772262513e-06, "loss": 0.86609334, "num_input_tokens_seen": 5542145, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 2.09375, "step": 264, "time_per_iteration": 2.4145796298980713 }, { "auxiliary_loss_clip": 0.01264681, "auxiliary_loss_mlp": 0.0108678, "balance_loss_clip": 1.03683114, "balance_loss_mlp": 1.05565, "epoch": 0.015932661957011873, "flos": 15259802405760.0, "grad_norm": 3.7930459922362205, "language_loss": 1.03443956, "learning_rate": 3.997532044040869e-06, "loss": 1.05795407, "num_input_tokens_seen": 5557920, "router_z_loss_clip": 0.5, "router_z_loss_mlp": 2.078125, "step": 265, "time_per_iteration": 2.4558472633361816 }, { "auxiliary_loss_clip": 0.01265797, "auxiliary_loss_mlp": 0.01091431, "balance_loss_clip": 1.03757191, "balance_loss_mlp": 1.05655909, "epoch": 0.015992785209679845, "flos": 20301504710400.0, "grad_norm": 6.033841447363089, "language_loss": 0.74710017, "learning_rate": 3.997513244532445e-06, "loss": 0.77067244, "num_input_tokens_seen": 5576290, "router_z_loss_clip": 0.5390625, "router_z_loss_mlp": 2.09375, "step": 266, "time_per_iteration": 2.552818775177002 }, { "auxiliary_loss_clip": 0.01253089, "auxiliary_loss_mlp": 0.01081126, "balance_loss_clip": 1.03139186, "balance_loss_mlp": 1.05281532, "epoch": 0.016052908462347814, "flos": 23361397799040.0, "grad_norm": 1.8207170903870495, "language_loss": 0.89983177, "learning_rate": 3.997494373737912e-06, "loss": 0.9231739, "num_input_tokens_seen": 5595205, "router_z_loss_clip": 0.49804688, "router_z_loss_mlp": 2.0, "step": 267, "time_per_iteration": 2.4479634761810303 }, { "auxiliary_loss_clip": 0.01263384, "auxiliary_loss_mlp": 0.01093478, "balance_loss_clip": 1.04264736, "balance_loss_mlp": 1.05432463, "epoch": 0.016113031715015783, "flos": 21285566691840.0, "grad_norm": 2.3169364211275987, "language_loss": 0.8484515, "learning_rate": 3.997475431657943e-06, "loss": 0.87202013, "num_input_tokens_seen": 5612645, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 2.09375, "step": 268, "time_per_iteration": 2.439906120300293 }, { "auxiliary_loss_clip": 0.01254088, "auxiliary_loss_mlp": 0.01081637, "balance_loss_clip": 1.03114021, "balance_loss_mlp": 1.05426097, "epoch": 0.01617315496768375, "flos": 18913742144640.0, "grad_norm": 2.45556176865787, "language_loss": 0.88008893, "learning_rate": 3.9974564182932135e-06, "loss": 0.9034462, "num_input_tokens_seen": 5628345, "router_z_loss_clip": 0.50390625, "router_z_loss_mlp": 2.0, "step": 269, "time_per_iteration": 2.409527063369751 }, { "auxiliary_loss_clip": 0.0126133, "auxiliary_loss_mlp": 0.01089492, "balance_loss_clip": 1.03787434, "balance_loss_mlp": 1.05400348, "epoch": 0.01623327822035172, "flos": 16545513467520.0, "grad_norm": 2.600771597384193, "language_loss": 0.96567738, "learning_rate": 3.997437333644403e-06, "loss": 0.98918557, "num_input_tokens_seen": 5645940, "router_z_loss_clip": 0.515625, "router_z_loss_mlp": 2.078125, "step": 270, "time_per_iteration": 2.4156806468963623 }, { "auxiliary_loss_clip": 0.01258176, "auxiliary_loss_mlp": 0.0109777, "balance_loss_clip": 1.04743981, "balance_loss_mlp": 1.05751896, "epoch": 0.016293401473019692, "flos": 23512361984640.0, "grad_norm": 2.3087201569184472, "language_loss": 0.85398507, "learning_rate": 3.9974181777121915e-06, "loss": 0.87754458, "num_input_tokens_seen": 5665690, "router_z_loss_clip": 0.50390625, "router_z_loss_mlp": 2.0, "step": 271, "time_per_iteration": 2.4505057334899902 }, { "auxiliary_loss_clip": 0.01259898, "auxiliary_loss_mlp": 0.01091065, "balance_loss_clip": 1.03847027, "balance_loss_mlp": 1.05401468, "epoch": 0.01635352472568766, "flos": 29014988325120.0, "grad_norm": 8.633948262091137, "language_loss": 0.80753708, "learning_rate": 3.997398950497263e-06, "loss": 0.8310467, "num_input_tokens_seen": 5683190, "router_z_loss_clip": 0.52734375, "router_z_loss_mlp": 2.0625, "step": 272, "time_per_iteration": 2.4930429458618164 }, { "auxiliary_loss_clip": 0.01254961, "auxiliary_loss_mlp": 0.01091194, "balance_loss_clip": 1.0410068, "balance_loss_mlp": 1.05233335, "epoch": 0.01641364797835563, "flos": 13369674153600.0, "grad_norm": 2.121148494629337, "language_loss": 0.80297101, "learning_rate": 3.9973796520003044e-06, "loss": 0.82643253, "num_input_tokens_seen": 5699780, "router_z_loss_clip": 0.50390625, "router_z_loss_mlp": 2.03125, "step": 273, "time_per_iteration": 2.396026611328125 }, { "auxiliary_loss_clip": 0.01254567, "auxiliary_loss_mlp": 0.01091044, "balance_loss_clip": 1.03914022, "balance_loss_mlp": 1.05181062, "epoch": 0.016473771231023598, "flos": 18877292818560.0, "grad_norm": 2.2415439484369513, "language_loss": 0.90765822, "learning_rate": 3.997360282222004e-06, "loss": 0.93111444, "num_input_tokens_seen": 5716980, "router_z_loss_clip": 0.51953125, "router_z_loss_mlp": 2.03125, "step": 274, "time_per_iteration": 2.4306771755218506 }, { "auxiliary_loss_clip": 0.01256585, "auxiliary_loss_mlp": 0.010984, "balance_loss_clip": 1.04537582, "balance_loss_mlp": 1.05353796, "epoch": 0.016533894483691566, "flos": 22600535339520.0, "grad_norm": 1.8461598322284212, "language_loss": 0.87523705, "learning_rate": 3.997340841163053e-06, "loss": 0.8987869, "num_input_tokens_seen": 5737780, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.03125, "step": 275, "time_per_iteration": 2.4361367225646973 }, { "auxiliary_loss_clip": 0.01259522, "auxiliary_loss_mlp": 0.01098736, "balance_loss_clip": 1.04533052, "balance_loss_mlp": 1.05604446, "epoch": 0.01659401773635954, "flos": 21506112950400.0, "grad_norm": 1.694260442445138, "language_loss": 0.80209416, "learning_rate": 3.9973213288241445e-06, "loss": 0.8256768, "num_input_tokens_seen": 5758330, "router_z_loss_clip": 0.53515625, "router_z_loss_mlp": 2.03125, "step": 276, "time_per_iteration": 2.4542930126190186 }, { "auxiliary_loss_clip": 0.01250956, "auxiliary_loss_mlp": 0.01088707, "balance_loss_clip": 1.04045093, "balance_loss_mlp": 1.05300093, "epoch": 0.016654140989027507, "flos": 32849673505920.0, "grad_norm": 1.7758054657349884, "language_loss": 0.80436337, "learning_rate": 3.997301745205976e-06, "loss": 0.82775998, "num_input_tokens_seen": 5778340, "router_z_loss_clip": 0.48242188, "router_z_loss_mlp": 1.984375, "step": 277, "time_per_iteration": 2.5271694660186768 }, { "auxiliary_loss_clip": 0.01251341, "auxiliary_loss_mlp": 0.01082503, "balance_loss_clip": 1.02964544, "balance_loss_mlp": 1.05135834, "epoch": 0.016714264241695476, "flos": 12305591602560.0, "grad_norm": 2.840228512453406, "language_loss": 0.79760599, "learning_rate": 3.997282090309246e-06, "loss": 0.82094443, "num_input_tokens_seen": 5794295, "router_z_loss_clip": 0.52734375, "router_z_loss_mlp": 2.0, "step": 278, "time_per_iteration": 2.4087443351745605 }, { "auxiliary_loss_clip": 0.0125048, "auxiliary_loss_mlp": 0.01082849, "balance_loss_clip": 1.03437805, "balance_loss_mlp": 1.05186558, "epoch": 0.016774387494363444, "flos": 27122625745920.0, "grad_norm": 1.9624673467401972, "language_loss": 0.90430892, "learning_rate": 3.9972623641346555e-06, "loss": 0.92764223, "num_input_tokens_seen": 5814405, "router_z_loss_clip": 0.484375, "router_z_loss_mlp": 1.984375, "step": 279, "time_per_iteration": 2.4882657527923584 }, { "auxiliary_loss_clip": 0.01252485, "auxiliary_loss_mlp": 0.01087283, "balance_loss_clip": 1.03421128, "balance_loss_mlp": 1.05146337, "epoch": 0.016834510747031413, "flos": 20190515898240.0, "grad_norm": 3.3592851885107806, "language_loss": 0.93480706, "learning_rate": 3.9972425666829085e-06, "loss": 0.95820475, "num_input_tokens_seen": 5832795, "router_z_loss_clip": 0.53125, "router_z_loss_mlp": 2.0, "step": 280, "time_per_iteration": 2.450284481048584 }, { "auxiliary_loss_clip": 0.0125678, "auxiliary_loss_mlp": 0.01087997, "balance_loss_clip": 1.03652251, "balance_loss_mlp": 1.0515883, "epoch": 0.016894633999699385, "flos": 27272961527040.0, "grad_norm": 2.204190539641557, "language_loss": 0.73594493, "learning_rate": 3.997222697954712e-06, "loss": 0.75939268, "num_input_tokens_seen": 5855750, "router_z_loss_clip": 0.515625, "router_z_loss_mlp": 2.0625, "step": 281, "time_per_iteration": 2.5007758140563965 }, { "auxiliary_loss_clip": 0.01255022, "auxiliary_loss_mlp": 0.01095399, "balance_loss_clip": 1.04506898, "balance_loss_mlp": 1.05505633, "epoch": 0.016954757252367354, "flos": 14902081948800.0, "grad_norm": 2.637264410239938, "language_loss": 0.79733199, "learning_rate": 3.997202757950775e-06, "loss": 0.82083619, "num_input_tokens_seen": 5872610, "router_z_loss_clip": 0.50390625, "router_z_loss_mlp": 2.0, "step": 282, "time_per_iteration": 2.4414730072021484 }, { "auxiliary_loss_clip": 0.01256517, "auxiliary_loss_mlp": 0.01099689, "balance_loss_clip": 1.04740334, "balance_loss_mlp": 1.05429316, "epoch": 0.017014880505035322, "flos": 21357802028160.0, "grad_norm": 2.070819037653251, "language_loss": 0.77117169, "learning_rate": 3.997182746671809e-06, "loss": 0.79473376, "num_input_tokens_seen": 5892985, "router_z_loss_clip": 0.5234375, "router_z_loss_mlp": 2.015625, "step": 283, "time_per_iteration": 2.445768117904663 }, { "auxiliary_loss_clip": 0.01257915, "auxiliary_loss_mlp": 0.01084586, "balance_loss_clip": 1.03706956, "balance_loss_mlp": 1.05535746, "epoch": 0.01707500375770329, "flos": 35331753726720.0, "grad_norm": 2.1373580986706613, "language_loss": 0.83959854, "learning_rate": 3.997162664118528e-06, "loss": 0.86302352, "num_input_tokens_seen": 5914060, "router_z_loss_clip": 0.47460938, "router_z_loss_mlp": 2.03125, "step": 284, "time_per_iteration": 2.573779344558716 }, { "auxiliary_loss_clip": 0.01247415, "auxiliary_loss_mlp": 0.01085849, "balance_loss_clip": 1.03487539, "balance_loss_mlp": 1.04878318, "epoch": 0.01713512701037126, "flos": 23581071273600.0, "grad_norm": 2.3576344067130917, "language_loss": 0.96618634, "learning_rate": 3.99714251029165e-06, "loss": 0.989519, "num_input_tokens_seen": 5932860, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 1.984375, "step": 285, "time_per_iteration": 2.444382667541504 }, { "auxiliary_loss_clip": 0.0125138, "auxiliary_loss_mlp": 0.01083147, "balance_loss_clip": 1.0370605, "balance_loss_mlp": 1.05309486, "epoch": 0.01719525026303923, "flos": 27633474892800.0, "grad_norm": 8.623728045445985, "language_loss": 0.93435287, "learning_rate": 3.997122285191892e-06, "loss": 0.95769811, "num_input_tokens_seen": 5952725, "router_z_loss_clip": 0.4609375, "router_z_loss_mlp": 1.984375, "step": 286, "time_per_iteration": 2.5120911598205566 }, { "auxiliary_loss_clip": 0.01248755, "auxiliary_loss_mlp": 0.01089959, "balance_loss_clip": 1.03903317, "balance_loss_mlp": 1.05168724, "epoch": 0.0172553735157072, "flos": 26978504186880.0, "grad_norm": 2.0538479761604704, "language_loss": 0.91652668, "learning_rate": 3.997101988819976e-06, "loss": 0.93991387, "num_input_tokens_seen": 5970560, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 1.96875, "step": 287, "time_per_iteration": 2.463068723678589 }, { "auxiliary_loss_clip": 0.01250456, "auxiliary_loss_mlp": 0.01078526, "balance_loss_clip": 1.03065181, "balance_loss_mlp": 1.05379772, "epoch": 0.01731549676837517, "flos": 14055962215680.0, "grad_norm": 3.2909414233324563, "language_loss": 1.01652026, "learning_rate": 3.997081621176629e-06, "loss": 1.03981018, "num_input_tokens_seen": 5982980, "router_z_loss_clip": 0.47851562, "router_z_loss_mlp": 1.96875, "step": 288, "time_per_iteration": 2.422941207885742 }, { "auxiliary_loss_clip": 0.01164027, "auxiliary_loss_mlp": 0.01052735, "balance_loss_clip": 1.03757143, "balance_loss_mlp": 1.06516135, "epoch": 0.017375620021043137, "flos": 66506885959680.0, "grad_norm": 0.9025926466434199, "language_loss": 0.63966572, "learning_rate": 3.997061182262575e-06, "loss": 0.66183341, "num_input_tokens_seen": 6049445, "router_z_loss_clip": 0.15136719, "router_z_loss_mlp": 0.98828125, "step": 289, "time_per_iteration": 3.1877288818359375 }, { "auxiliary_loss_clip": 0.01247348, "auxiliary_loss_mlp": 0.01082034, "balance_loss_clip": 1.03451705, "balance_loss_mlp": 1.05231452, "epoch": 0.01743574327371111, "flos": 15224435331840.0, "grad_norm": 3.2195614434280504, "language_loss": 0.88081455, "learning_rate": 3.997040672078545e-06, "loss": 0.90410841, "num_input_tokens_seen": 6064150, "router_z_loss_clip": 0.4765625, "router_z_loss_mlp": 1.9453125, "step": 290, "time_per_iteration": 2.4281256198883057 }, { "auxiliary_loss_clip": 0.01248587, "auxiliary_loss_mlp": 0.01079803, "balance_loss_clip": 1.03347826, "balance_loss_mlp": 1.05308676, "epoch": 0.017495866526379078, "flos": 25372708980480.0, "grad_norm": 2.006050207544469, "language_loss": 0.83882666, "learning_rate": 3.997020090625269e-06, "loss": 0.86211061, "num_input_tokens_seen": 6083920, "router_z_loss_clip": 0.46289062, "router_z_loss_mlp": 1.953125, "step": 291, "time_per_iteration": 2.4570248126983643 }, { "auxiliary_loss_clip": 0.0125354, "auxiliary_loss_mlp": 0.01099016, "balance_loss_clip": 1.04727936, "balance_loss_mlp": 1.05850148, "epoch": 0.017555989779047047, "flos": 26358272150400.0, "grad_norm": 1.7788976427156116, "language_loss": 0.72319877, "learning_rate": 3.996999437903485e-06, "loss": 0.74672437, "num_input_tokens_seen": 6105460, "router_z_loss_clip": 0.515625, "router_z_loss_mlp": 1.9453125, "step": 292, "time_per_iteration": 3.9080538749694824 }, { "auxiliary_loss_clip": 0.01247077, "auxiliary_loss_mlp": 0.01092912, "balance_loss_clip": 1.04296315, "balance_loss_mlp": 1.05412185, "epoch": 0.017616113031715015, "flos": 22337919025920.0, "grad_norm": 2.160237997478328, "language_loss": 0.86493468, "learning_rate": 3.996978713913927e-06, "loss": 0.88833451, "num_input_tokens_seen": 6122890, "router_z_loss_clip": 0.5, "router_z_loss_mlp": 1.9296875, "step": 293, "time_per_iteration": 5.241613388061523 }, { "auxiliary_loss_clip": 0.01246615, "auxiliary_loss_mlp": 0.01079838, "balance_loss_clip": 1.03301287, "balance_loss_mlp": 1.05239558, "epoch": 0.017676236284382984, "flos": 20155881962880.0, "grad_norm": 3.2612184299665374, "language_loss": 0.80483878, "learning_rate": 3.996957918657335e-06, "loss": 0.8281033, "num_input_tokens_seen": 6142890, "router_z_loss_clip": 0.46875, "router_z_loss_mlp": 1.9375, "step": 294, "time_per_iteration": 3.8060572147369385 }, { "auxiliary_loss_clip": 0.0124757, "auxiliary_loss_mlp": 0.01089644, "balance_loss_clip": 1.03969526, "balance_loss_mlp": 1.05109572, "epoch": 0.017736359537050956, "flos": 25222303376640.0, "grad_norm": 2.5812548654115948, "language_loss": 0.83908248, "learning_rate": 3.996937052134452e-06, "loss": 0.86245465, "num_input_tokens_seen": 6162030, "router_z_loss_clip": 0.5, "router_z_loss_mlp": 1.96875, "step": 295, "time_per_iteration": 2.4810874462127686 }, { "auxiliary_loss_clip": 0.01248576, "auxiliary_loss_mlp": 0.01091159, "balance_loss_clip": 1.04524004, "balance_loss_mlp": 1.05808198, "epoch": 0.017796482789718925, "flos": 20337779479680.0, "grad_norm": 2.0460008989613, "language_loss": 0.83856666, "learning_rate": 3.996916114346023e-06, "loss": 0.86196399, "num_input_tokens_seen": 6180540, "router_z_loss_clip": 0.4609375, "router_z_loss_mlp": 1.90625, "step": 296, "time_per_iteration": 2.4240550994873047 }, { "auxiliary_loss_clip": 0.01255662, "auxiliary_loss_mlp": 0.0108459, "balance_loss_clip": 1.03647757, "balance_loss_mlp": 1.05740452, "epoch": 0.017856606042386893, "flos": 22378208601600.0, "grad_norm": 2.4621250106449386, "language_loss": 0.87520307, "learning_rate": 3.996895105292794e-06, "loss": 0.89860559, "num_input_tokens_seen": 6199425, "router_z_loss_clip": 0.48046875, "router_z_loss_mlp": 1.984375, "step": 297, "time_per_iteration": 2.4576117992401123 }, { "auxiliary_loss_clip": 0.01250089, "auxiliary_loss_mlp": 0.01076338, "balance_loss_clip": 1.03039491, "balance_loss_mlp": 1.05379272, "epoch": 0.017916729295054862, "flos": 20229024994560.0, "grad_norm": 2.270374893224995, "language_loss": 0.88099438, "learning_rate": 3.996874024975515e-06, "loss": 0.90425873, "num_input_tokens_seen": 6219170, "router_z_loss_clip": 0.4609375, "router_z_loss_mlp": 1.9609375, "step": 298, "time_per_iteration": 2.444200277328491 }, { "auxiliary_loss_clip": 0.0124723, "auxiliary_loss_mlp": 0.01087312, "balance_loss_clip": 1.03714836, "balance_loss_mlp": 1.05417967, "epoch": 0.01797685254772283, "flos": 19389957356160.0, "grad_norm": 2.25462221963985, "language_loss": 0.88106245, "learning_rate": 3.996852873394939e-06, "loss": 0.90440786, "num_input_tokens_seen": 6237930, "router_z_loss_clip": 0.50390625, "router_z_loss_mlp": 1.9296875, "step": 299, "time_per_iteration": 2.4332218170166016 }, { "auxiliary_loss_clip": 0.01257521, "auxiliary_loss_mlp": 0.01077923, "balance_loss_clip": 1.02790248, "balance_loss_mlp": 1.05698752, "epoch": 0.018036975800390802, "flos": 24424851945600.0, "grad_norm": 3.01009565599283, "language_loss": 0.63656032, "learning_rate": 3.996831650551821e-06, "loss": 0.65991479, "num_input_tokens_seen": 6257170, "router_z_loss_clip": 0.5, "router_z_loss_mlp": 2.0, "step": 300, "time_per_iteration": 2.512960433959961 }, { "auxiliary_loss_clip": 0.01250638, "auxiliary_loss_mlp": 0.01089998, "balance_loss_clip": 1.04167092, "balance_loss_mlp": 1.05774963, "epoch": 0.01809709905305877, "flos": 15778017849600.0, "grad_norm": 2.705806939759899, "language_loss": 0.87975717, "learning_rate": 3.996810356446917e-06, "loss": 0.90316349, "num_input_tokens_seen": 6274780, "router_z_loss_clip": 0.484375, "router_z_loss_mlp": 1.9296875, "step": 301, "time_per_iteration": 2.449906826019287 }, { "auxiliary_loss_clip": 0.01170518, "auxiliary_loss_mlp": 0.01035771, "balance_loss_clip": 1.02175176, "balance_loss_mlp": 1.07485867, "epoch": 0.01815722230572674, "flos": 67344592055040.0, "grad_norm": 0.9665553873254724, "language_loss": 0.62200117, "learning_rate": 3.996788991080988e-06, "loss": 0.64406407, "num_input_tokens_seen": 6340435, "router_z_loss_clip": 0.140625, "router_z_loss_mlp": 0.95703125, "step": 302, "time_per_iteration": 3.235015392303467 }, { "auxiliary_loss_clip": 0.01245816, "auxiliary_loss_mlp": 0.01094046, "balance_loss_clip": 1.04636252, "balance_loss_mlp": 1.04997766, "epoch": 0.01821734555839471, "flos": 15484747495680.0, "grad_norm": 2.2391786324512637, "language_loss": 0.89078534, "learning_rate": 3.996767554454796e-06, "loss": 0.91418391, "num_input_tokens_seen": 6358160, "router_z_loss_clip": 0.4765625, "router_z_loss_mlp": 1.9609375, "step": 303, "time_per_iteration": 2.4486289024353027 }, { "auxiliary_loss_clip": 0.01252791, "auxiliary_loss_mlp": 0.01097796, "balance_loss_clip": 1.04799032, "balance_loss_mlp": 1.05749726, "epoch": 0.018277468811062677, "flos": 24096284340480.0, "grad_norm": 1.7268145183362635, "language_loss": 0.79628664, "learning_rate": 3.996746046569107e-06, "loss": 0.81979251, "num_input_tokens_seen": 6378485, "router_z_loss_clip": 0.5, "router_z_loss_mlp": 1.953125, "step": 304, "time_per_iteration": 2.498051643371582 }, { "auxiliary_loss_clip": 0.01242654, "auxiliary_loss_mlp": 0.01073108, "balance_loss_clip": 1.02733171, "balance_loss_mlp": 1.05657101, "epoch": 0.01833759206373065, "flos": 20958290807040.0, "grad_norm": 1.6486581714926711, "language_loss": 0.82408345, "learning_rate": 3.996724467424687e-06, "loss": 0.84724116, "num_input_tokens_seen": 6397845, "router_z_loss_clip": 0.45703125, "router_z_loss_mlp": 1.859375, "step": 305, "time_per_iteration": 2.4394264221191406 }, { "auxiliary_loss_clip": 0.0124937, "auxiliary_loss_mlp": 0.01077638, "balance_loss_clip": 1.03140867, "balance_loss_mlp": 1.05194199, "epoch": 0.018397715316398618, "flos": 19389747888000.0, "grad_norm": 1.952855863430056, "language_loss": 0.90433657, "learning_rate": 3.996702817022308e-06, "loss": 0.9276067, "num_input_tokens_seen": 6416475, "router_z_loss_clip": 0.46289062, "router_z_loss_mlp": 1.9765625, "step": 306, "time_per_iteration": 2.4558498859405518 }, { "auxiliary_loss_clip": 0.01239261, "auxiliary_loss_mlp": 0.0108027, "balance_loss_clip": 1.03404021, "balance_loss_mlp": 1.04859948, "epoch": 0.018457838569066586, "flos": 29131248752640.0, "grad_norm": 2.143075141284067, "language_loss": 0.86084306, "learning_rate": 3.996681095362741e-06, "loss": 0.88403845, "num_input_tokens_seen": 6437520, "router_z_loss_clip": 0.4609375, "router_z_loss_mlp": 1.90625, "step": 307, "time_per_iteration": 2.537835121154785 }, { "auxiliary_loss_clip": 0.01241134, "auxiliary_loss_mlp": 0.01080609, "balance_loss_clip": 1.03178144, "balance_loss_mlp": 1.05186296, "epoch": 0.018517961821734555, "flos": 19207640903040.0, "grad_norm": 2.3437069898355904, "language_loss": 0.71195388, "learning_rate": 3.996659302446762e-06, "loss": 0.73517132, "num_input_tokens_seen": 6455680, "router_z_loss_clip": 0.48828125, "router_z_loss_mlp": 1.890625, "step": 308, "time_per_iteration": 2.4432265758514404 }, { "auxiliary_loss_clip": 0.01246949, "auxiliary_loss_mlp": 0.01087794, "balance_loss_clip": 1.04046774, "balance_loss_mlp": 1.05282617, "epoch": 0.018578085074402523, "flos": 19862053027200.0, "grad_norm": 2.6018867667266163, "language_loss": 0.91403347, "learning_rate": 3.996637438275148e-06, "loss": 0.93738091, "num_input_tokens_seen": 6474880, "router_z_loss_clip": 0.47265625, "router_z_loss_mlp": 1.9375, "step": 309, "time_per_iteration": 2.4650886058807373 }, { "auxiliary_loss_clip": 0.0125455, "auxiliary_loss_mlp": 0.01084848, "balance_loss_clip": 1.03559089, "balance_loss_mlp": 1.05105126, "epoch": 0.018638208327070496, "flos": 29605648573440.0, "grad_norm": 1.9363741747675771, "language_loss": 0.72133344, "learning_rate": 3.99661550284868e-06, "loss": 0.74472737, "num_input_tokens_seen": 6495945, "router_z_loss_clip": 0.4921875, "router_z_loss_mlp": 2.03125, "step": 310, "time_per_iteration": 2.515129566192627 }, { "auxiliary_loss_clip": 0.01245354, "auxiliary_loss_mlp": 0.01099865, "balance_loss_clip": 1.05294418, "balance_loss_mlp": 1.05505824, "epoch": 0.018698331579738464, "flos": 45729866131200.0, "grad_norm": 2.0692631349636943, "language_loss": 0.73453295, "learning_rate": 3.996593496168141e-06, "loss": 0.75798512, "num_input_tokens_seen": 6519930, "router_z_loss_clip": 0.46875, "router_z_loss_mlp": 1.8984375, "step": 311, "time_per_iteration": 2.6794707775115967 }, { "auxiliary_loss_clip": 0.01254028, "auxiliary_loss_mlp": 0.01086933, "balance_loss_clip": 1.04008377, "balance_loss_mlp": 1.05406392, "epoch": 0.018758454832406433, "flos": 20482669088640.0, "grad_norm": 3.6044382373216695, "language_loss": 0.90822446, "learning_rate": 3.996571418234316e-06, "loss": 0.93163407, "num_input_tokens_seen": 6535070, "router_z_loss_clip": 0.46875, "router_z_loss_mlp": 2.0, "step": 312, "time_per_iteration": 2.454665184020996 }, { "auxiliary_loss_clip": 0.01253057, "auxiliary_loss_mlp": 0.01093846, "balance_loss_clip": 1.04518485, "balance_loss_mlp": 1.05319011, "epoch": 0.0188185780850744, "flos": 15776900686080.0, "grad_norm": 2.1685699705162365, "language_loss": 0.89634204, "learning_rate": 3.996549269047992e-06, "loss": 0.91981101, "num_input_tokens_seen": 6554135, "router_z_loss_clip": 0.48828125, "router_z_loss_mlp": 2.0, "step": 313, "time_per_iteration": 2.413755178451538 }, { "auxiliary_loss_clip": 0.01254911, "auxiliary_loss_mlp": 0.01084852, "balance_loss_clip": 1.03831267, "balance_loss_mlp": 1.0554781, "epoch": 0.018878701337742373, "flos": 22454633301120.0, "grad_norm": 2.2131754408423623, "language_loss": 0.72605658, "learning_rate": 3.996527048609961e-06, "loss": 0.7494542, "num_input_tokens_seen": 6572275, "router_z_loss_clip": 0.46484375, "router_z_loss_mlp": 1.9921875, "step": 314, "time_per_iteration": 2.461585283279419 }, { "auxiliary_loss_clip": 0.01247425, "auxiliary_loss_mlp": 0.01093213, "balance_loss_clip": 1.04669785, "balance_loss_mlp": 1.0528543, "epoch": 0.018938824590410342, "flos": 30992189241600.0, "grad_norm": 2.4088693693289045, "language_loss": 0.88752794, "learning_rate": 3.996504756921015e-06, "loss": 0.91093433, "num_input_tokens_seen": 6594520, "router_z_loss_clip": 0.46484375, "router_z_loss_mlp": 1.9453125, "step": 315, "time_per_iteration": 2.527616024017334 }, { "auxiliary_loss_clip": 0.01245421, "auxiliary_loss_mlp": 0.0107802, "balance_loss_clip": 1.03062224, "balance_loss_mlp": 1.05360627, "epoch": 0.01899894784307831, "flos": 23257775283840.0, "grad_norm": 1.883027643759866, "language_loss": 0.80180895, "learning_rate": 3.996482393981951e-06, "loss": 0.82504332, "num_input_tokens_seen": 6614245, "router_z_loss_clip": 0.47460938, "router_z_loss_mlp": 1.921875, "step": 316, "time_per_iteration": 2.5028390884399414 }, { "auxiliary_loss_clip": 0.01244454, "auxiliary_loss_mlp": 0.01081262, "balance_loss_clip": 1.03372157, "balance_loss_mlp": 1.05157375, "epoch": 0.01905907109574628, "flos": 17456921176320.0, "grad_norm": 2.341369116632892, "language_loss": 0.89989537, "learning_rate": 3.996459959793564e-06, "loss": 0.92315257, "num_input_tokens_seen": 6632015, "router_z_loss_clip": 0.4765625, "router_z_loss_mlp": 1.9296875, "step": 317, "time_per_iteration": 2.4208786487579346 }, { "auxiliary_loss_clip": 0.01239179, "auxiliary_loss_mlp": 0.01081755, "balance_loss_clip": 1.03349936, "balance_loss_mlp": 1.04840732, "epoch": 0.019119194348414248, "flos": 14969499517440.0, "grad_norm": 4.3113874763269395, "language_loss": 0.90558648, "learning_rate": 3.996437454356658e-06, "loss": 0.92879575, "num_input_tokens_seen": 6649015, "router_z_loss_clip": 0.484375, "router_z_loss_mlp": 1.90625, "step": 318, "time_per_iteration": 2.414809226989746 }, { "auxiliary_loss_clip": 0.01241514, "auxiliary_loss_mlp": 0.01070685, "balance_loss_clip": 1.02605319, "balance_loss_mlp": 1.0510509, "epoch": 0.01917931760108222, "flos": 25481672933760.0, "grad_norm": 6.179517827759149, "language_loss": 0.93067336, "learning_rate": 3.996414877672034e-06, "loss": 0.95379531, "num_input_tokens_seen": 6669225, "router_z_loss_clip": 0.4453125, "router_z_loss_mlp": 1.90625, "step": 319, "time_per_iteration": 2.4777655601501465 }, { "auxiliary_loss_clip": 0.01141162, "auxiliary_loss_mlp": 0.01034181, "balance_loss_clip": 1.02130675, "balance_loss_mlp": 1.05140138, "epoch": 0.01923944085375019, "flos": 71553722100480.0, "grad_norm": 0.9027108154994729, "language_loss": 0.59722847, "learning_rate": 3.996392229740498e-06, "loss": 0.6189819, "num_input_tokens_seen": 6725775, "router_z_loss_clip": 0.12890625, "router_z_loss_mlp": 0.8984375, "step": 320, "time_per_iteration": 3.004598379135132 }, { "auxiliary_loss_clip": 0.0123998, "auxiliary_loss_mlp": 0.01086659, "balance_loss_clip": 1.03847432, "balance_loss_mlp": 1.0485394, "epoch": 0.019299564106418157, "flos": 19681482142080.0, "grad_norm": 3.3648702202179, "language_loss": 0.89259684, "learning_rate": 3.99636951056286e-06, "loss": 0.91586322, "num_input_tokens_seen": 6744170, "router_z_loss_clip": 0.48242188, "router_z_loss_mlp": 1.9140625, "step": 321, "time_per_iteration": 2.4724040031433105 }, { "auxiliary_loss_clip": 0.01247364, "auxiliary_loss_mlp": 0.0109448, "balance_loss_clip": 1.04455495, "balance_loss_mlp": 1.05249262, "epoch": 0.019359687359086126, "flos": 24386063558400.0, "grad_norm": 2.611546329556763, "language_loss": 0.82608497, "learning_rate": 3.996346720139928e-06, "loss": 0.84950346, "num_input_tokens_seen": 6764565, "router_z_loss_clip": 0.49804688, "router_z_loss_mlp": 1.9453125, "step": 322, "time_per_iteration": 2.472501039505005 }, { "auxiliary_loss_clip": 0.0124952, "auxiliary_loss_mlp": 0.01084466, "balance_loss_clip": 1.03673482, "balance_loss_mlp": 1.05295157, "epoch": 0.019419810611754094, "flos": 23950242656640.0, "grad_norm": 2.3185725465210574, "language_loss": 0.72154129, "learning_rate": 3.996323858472518e-06, "loss": 0.74488103, "num_input_tokens_seen": 6785310, "router_z_loss_clip": 0.4765625, "router_z_loss_mlp": 1.96875, "step": 323, "time_per_iteration": 2.49354887008667 }, { "auxiliary_loss_clip": 0.01238313, "auxiliary_loss_mlp": 0.0107448, "balance_loss_clip": 1.02827406, "balance_loss_mlp": 1.04739833, "epoch": 0.019479933864422067, "flos": 22159233354240.0, "grad_norm": 1.9805260952758794, "language_loss": 0.92195767, "learning_rate": 3.996300925561445e-06, "loss": 0.94508559, "num_input_tokens_seen": 6803290, "router_z_loss_clip": 0.4609375, "router_z_loss_mlp": 1.90625, "step": 324, "time_per_iteration": 2.4647061824798584 }, { "auxiliary_loss_clip": 0.01249478, "auxiliary_loss_mlp": 0.01078997, "balance_loss_clip": 1.03441286, "balance_loss_mlp": 1.05450225, "epoch": 0.019540057117090035, "flos": 22235727876480.0, "grad_norm": 5.209329635238033, "language_loss": 0.64969045, "learning_rate": 3.996277921407525e-06, "loss": 0.67297518, "num_input_tokens_seen": 6822570, "router_z_loss_clip": 0.4453125, "router_z_loss_mlp": 1.9453125, "step": 325, "time_per_iteration": 2.4915707111358643 }, { "auxiliary_loss_clip": 0.01247938, "auxiliary_loss_mlp": 0.01081587, "balance_loss_clip": 1.03504801, "balance_loss_mlp": 1.05839956, "epoch": 0.019600180369758004, "flos": 23075633387520.0, "grad_norm": 3.213050020436582, "language_loss": 0.76240909, "learning_rate": 3.996254846011582e-06, "loss": 0.78570437, "num_input_tokens_seen": 6841910, "router_z_loss_clip": 0.46484375, "router_z_loss_mlp": 1.890625, "step": 326, "time_per_iteration": 2.4705817699432373 }, { "auxiliary_loss_clip": 0.01243276, "auxiliary_loss_mlp": 0.01090596, "balance_loss_clip": 1.04548764, "balance_loss_mlp": 1.05709124, "epoch": 0.019660303622425972, "flos": 25409681976960.0, "grad_norm": 3.3797375509014422, "language_loss": 0.79048991, "learning_rate": 3.99623169937444e-06, "loss": 0.81382859, "num_input_tokens_seen": 6862480, "router_z_loss_clip": 0.45117188, "router_z_loss_mlp": 1.8671875, "step": 327, "time_per_iteration": 2.4997310638427734 }, { "auxiliary_loss_clip": 0.01250367, "auxiliary_loss_mlp": 0.01083052, "balance_loss_clip": 1.03579772, "balance_loss_mlp": 1.05862069, "epoch": 0.01972042687509394, "flos": 23656448632320.0, "grad_norm": 2.246546113724492, "language_loss": 0.80648839, "learning_rate": 3.996208481496923e-06, "loss": 0.82982254, "num_input_tokens_seen": 6882015, "router_z_loss_clip": 0.47265625, "router_z_loss_mlp": 1.921875, "step": 328, "time_per_iteration": 2.4818077087402344 }, { "auxiliary_loss_clip": 0.01246085, "auxiliary_loss_mlp": 0.01091705, "balance_loss_clip": 1.04483187, "balance_loss_mlp": 1.05255795, "epoch": 0.019780550127761913, "flos": 18222496669440.0, "grad_norm": 3.1820907475393647, "language_loss": 0.93123943, "learning_rate": 3.996185192379858e-06, "loss": 0.95461732, "num_input_tokens_seen": 6899785, "router_z_loss_clip": 0.46875, "router_z_loss_mlp": 1.9375, "step": 329, "time_per_iteration": 2.426272392272949 }, { "auxiliary_loss_clip": 0.01251169, "auxiliary_loss_mlp": 0.01090414, "balance_loss_clip": 1.0451622, "balance_loss_mlp": 1.05434823, "epoch": 0.01984067338042988, "flos": 22417695216000.0, "grad_norm": 2.3763564534737656, "language_loss": 0.74106705, "learning_rate": 3.996161832024081e-06, "loss": 0.76448292, "num_input_tokens_seen": 6918575, "router_z_loss_clip": 0.453125, "router_z_loss_mlp": 1.96875, "step": 330, "time_per_iteration": 2.4598453044891357 }, { "auxiliary_loss_clip": 0.01249553, "auxiliary_loss_mlp": 0.01101458, "balance_loss_clip": 1.05377388, "balance_loss_mlp": 1.05429959, "epoch": 0.01990079663309785, "flos": 17054267932800.0, "grad_norm": 2.746237667836226, "language_loss": 0.92803168, "learning_rate": 3.996138400430422e-06, "loss": 0.95154178, "num_input_tokens_seen": 6936965, "router_z_loss_clip": 0.4765625, "router_z_loss_mlp": 1.953125, "step": 331, "time_per_iteration": 3.8557064533233643 }, { "auxiliary_loss_clip": 0.01237676, "auxiliary_loss_mlp": 0.01075151, "balance_loss_clip": 1.03235459, "balance_loss_mlp": 1.05177975, "epoch": 0.01996091988576582, "flos": 15960857973120.0, "grad_norm": 3.6761560054958493, "language_loss": 0.92356098, "learning_rate": 3.996114897599718e-06, "loss": 0.94668925, "num_input_tokens_seen": 6953475, "router_z_loss_clip": 0.42773438, "router_z_loss_mlp": 1.859375, "step": 332, "time_per_iteration": 3.852618455886841 }, { "auxiliary_loss_clip": 0.01240775, "auxiliary_loss_mlp": 0.01078938, "balance_loss_clip": 1.0318985, "balance_loss_mlp": 1.05442214, "epoch": 0.02002104313843379, "flos": 23585330459520.0, "grad_norm": 2.542787881565984, "language_loss": 0.74470538, "learning_rate": 3.996091323532807e-06, "loss": 0.76790249, "num_input_tokens_seen": 6971630, "router_z_loss_clip": 0.47070312, "router_z_loss_mlp": 1.8671875, "step": 333, "time_per_iteration": 3.8072381019592285 }, { "auxiliary_loss_clip": 0.01244784, "auxiliary_loss_mlp": 0.01077039, "balance_loss_clip": 1.03154898, "balance_loss_mlp": 1.05381489, "epoch": 0.02008116639110176, "flos": 34093454158080.0, "grad_norm": 2.2161167871731506, "language_loss": 0.78280437, "learning_rate": 3.996067678230532e-06, "loss": 0.80602264, "num_input_tokens_seen": 6992775, "router_z_loss_clip": 0.45507812, "router_z_loss_mlp": 1.90625, "step": 334, "time_per_iteration": 3.9226768016815186 }, { "auxiliary_loss_clip": 0.01243757, "auxiliary_loss_mlp": 0.0107732, "balance_loss_clip": 1.03049493, "balance_loss_mlp": 1.04973888, "epoch": 0.020141289643769728, "flos": 19682669128320.0, "grad_norm": 1.8925571247576105, "language_loss": 0.82887501, "learning_rate": 3.996043961693736e-06, "loss": 0.85208577, "num_input_tokens_seen": 7011425, "router_z_loss_clip": 0.46875, "router_z_loss_mlp": 1.9375, "step": 335, "time_per_iteration": 2.5068249702453613 }, { "auxiliary_loss_clip": 0.01240044, "auxiliary_loss_mlp": 0.01073121, "balance_loss_clip": 1.02844167, "balance_loss_mlp": 1.05044055, "epoch": 0.020201412896437697, "flos": 20739525027840.0, "grad_norm": 2.570482808050445, "language_loss": 0.9190805, "learning_rate": 3.996020173923266e-06, "loss": 0.9422121, "num_input_tokens_seen": 7029450, "router_z_loss_clip": 0.44726562, "router_z_loss_mlp": 1.8984375, "step": 336, "time_per_iteration": 2.420025587081909 }, { "auxiliary_loss_clip": 0.01243367, "auxiliary_loss_mlp": 0.01077506, "balance_loss_clip": 1.03118145, "balance_loss_mlp": 1.0501318, "epoch": 0.020261536149105665, "flos": 20265474320640.0, "grad_norm": 2.029482100880366, "language_loss": 0.87759602, "learning_rate": 3.99599631491997e-06, "loss": 0.90080476, "num_input_tokens_seen": 7047555, "router_z_loss_clip": 0.46289062, "router_z_loss_mlp": 1.9296875, "step": 337, "time_per_iteration": 2.4157845973968506 }, { "auxiliary_loss_clip": 0.01236485, "auxiliary_loss_mlp": 0.01086757, "balance_loss_clip": 1.04064679, "balance_loss_mlp": 1.04916739, "epoch": 0.020321659401773638, "flos": 25847562648960.0, "grad_norm": 1.5226922127191085, "language_loss": 0.89615571, "learning_rate": 3.995972384684699e-06, "loss": 0.91938806, "num_input_tokens_seen": 7068185, "router_z_loss_clip": 0.4609375, "router_z_loss_mlp": 1.875, "step": 338, "time_per_iteration": 2.4866647720336914 }, { "auxiliary_loss_clip": 0.01238625, "auxiliary_loss_mlp": 0.01073964, "balance_loss_clip": 1.02556515, "balance_loss_mlp": 1.04621446, "epoch": 0.020381782654441606, "flos": 17494033818240.0, "grad_norm": 2.4574020059875217, "language_loss": 0.84838378, "learning_rate": 3.995948383218309e-06, "loss": 0.87150961, "num_input_tokens_seen": 7085955, "router_z_loss_clip": 0.484375, "router_z_loss_mlp": 1.921875, "step": 339, "time_per_iteration": 2.436677932739258 }, { "auxiliary_loss_clip": 0.01243425, "auxiliary_loss_mlp": 0.01085429, "balance_loss_clip": 1.0390569, "balance_loss_mlp": 1.05143118, "epoch": 0.020441905907109575, "flos": 24242779872000.0, "grad_norm": 1.9556810807536034, "language_loss": 0.88591182, "learning_rate": 3.995924310521655e-06, "loss": 0.90920031, "num_input_tokens_seen": 7106345, "router_z_loss_clip": 0.46289062, "router_z_loss_mlp": 1.921875, "step": 340, "time_per_iteration": 2.4788992404937744 }, { "auxiliary_loss_clip": 0.01240924, "auxiliary_loss_mlp": 0.01079023, "balance_loss_clip": 1.03181577, "balance_loss_mlp": 1.05091405, "epoch": 0.020502029159777543, "flos": 22232306563200.0, "grad_norm": 2.1547913145760376, "language_loss": 0.87746286, "learning_rate": 3.995900166595596e-06, "loss": 0.9006623, "num_input_tokens_seen": 7125070, "router_z_loss_clip": 0.47265625, "router_z_loss_mlp": 1.8984375, "step": 341, "time_per_iteration": 2.4392197132110596 }, { "auxiliary_loss_clip": 0.01244465, "auxiliary_loss_mlp": 0.01084104, "balance_loss_clip": 1.0366106, "balance_loss_mlp": 1.05005693, "epoch": 0.020562152412445512, "flos": 23986726894080.0, "grad_norm": 2.1928377266058137, "language_loss": 0.79686862, "learning_rate": 3.995875951440995e-06, "loss": 0.82015431, "num_input_tokens_seen": 7144675, "router_z_loss_clip": 0.47460938, "router_z_loss_mlp": 1.9375, "step": 342, "time_per_iteration": 2.454228162765503 }, { "auxiliary_loss_clip": 0.01231521, "auxiliary_loss_mlp": 0.01077618, "balance_loss_clip": 1.03029144, "balance_loss_mlp": 1.04603922, "epoch": 0.020622275665113484, "flos": 26974210089600.0, "grad_norm": 1.8821152658598543, "language_loss": 0.8900106, "learning_rate": 3.995851665058715e-06, "loss": 0.91310197, "num_input_tokens_seen": 7165505, "router_z_loss_clip": 0.47265625, "router_z_loss_mlp": 1.8515625, "step": 343, "time_per_iteration": 2.4975695610046387 }, { "auxiliary_loss_clip": 0.01247423, "auxiliary_loss_mlp": 0.0108777, "balance_loss_clip": 1.04194629, "balance_loss_mlp": 1.05635118, "epoch": 0.020682398917781453, "flos": 22599627644160.0, "grad_norm": 2.4061841581096366, "language_loss": 0.77623147, "learning_rate": 3.995827307449623e-06, "loss": 0.79958338, "num_input_tokens_seen": 7184605, "router_z_loss_clip": 0.45703125, "router_z_loss_mlp": 1.90625, "step": 344, "time_per_iteration": 2.500241279602051 }, { "auxiliary_loss_clip": 0.01228052, "auxiliary_loss_mlp": 0.01089187, "balance_loss_clip": 1.04031157, "balance_loss_mlp": 1.04616785, "epoch": 0.02074252217044942, "flos": 15012686736000.0, "grad_norm": 2.1031120440135336, "language_loss": 0.74481457, "learning_rate": 3.995802878614588e-06, "loss": 0.76798695, "num_input_tokens_seen": 7203065, "router_z_loss_clip": 0.48828125, "router_z_loss_mlp": 1.8203125, "step": 345, "time_per_iteration": 2.4321231842041016 }, { "auxiliary_loss_clip": 0.01236457, "auxiliary_loss_mlp": 0.01083525, "balance_loss_clip": 1.03290892, "balance_loss_mlp": 1.05203187, "epoch": 0.02080264542311739, "flos": 25336783324800.0, "grad_norm": 2.1070814705930667, "language_loss": 0.89819229, "learning_rate": 3.995778378554483e-06, "loss": 0.92139214, "num_input_tokens_seen": 7222995, "router_z_loss_clip": 0.5078125, "router_z_loss_mlp": 1.84375, "step": 346, "time_per_iteration": 2.500624179840088 }, { "auxiliary_loss_clip": 0.01233687, "auxiliary_loss_mlp": 0.01080794, "balance_loss_clip": 1.03635287, "balance_loss_mlp": 1.04964137, "epoch": 0.02086276867578536, "flos": 24387669480960.0, "grad_norm": 2.0655644820909558, "language_loss": 0.78656721, "learning_rate": 3.99575380727018e-06, "loss": 0.80971205, "num_input_tokens_seen": 7244625, "router_z_loss_clip": 0.4453125, "router_z_loss_mlp": 1.84375, "step": 347, "time_per_iteration": 2.4814603328704834 }, { "auxiliary_loss_clip": 0.01237051, "auxiliary_loss_mlp": 0.01080866, "balance_loss_clip": 1.03516102, "balance_loss_mlp": 1.05103707, "epoch": 0.02092289192845333, "flos": 24461056892160.0, "grad_norm": 1.8886253600904628, "language_loss": 0.70518041, "learning_rate": 3.995729164762559e-06, "loss": 0.72835958, "num_input_tokens_seen": 7263255, "router_z_loss_clip": 0.45703125, "router_z_loss_mlp": 1.859375, "step": 348, "time_per_iteration": 2.498539447784424 }, { "auxiliary_loss_clip": 0.01240629, "auxiliary_loss_mlp": 0.0108724, "balance_loss_clip": 1.04062903, "balance_loss_mlp": 1.04997635, "epoch": 0.0209830151811213, "flos": 17450392752000.0, "grad_norm": 9.238146016146256, "language_loss": 0.76325005, "learning_rate": 3.995704451032496e-06, "loss": 0.78652877, "num_input_tokens_seen": 7279275, "router_z_loss_clip": 0.46484375, "router_z_loss_mlp": 1.90625, "step": 349, "time_per_iteration": 2.4045448303222656 }, { "auxiliary_loss_clip": 0.01223683, "auxiliary_loss_mlp": 0.01074173, "balance_loss_clip": 1.03178263, "balance_loss_mlp": 1.04728103, "epoch": 0.021043138433789268, "flos": 24572778842880.0, "grad_norm": 1.7558642765462482, "language_loss": 0.85043454, "learning_rate": 3.995679666080876e-06, "loss": 0.87341309, "num_input_tokens_seen": 7300180, "router_z_loss_clip": 0.42382812, "router_z_loss_mlp": 1.765625, "step": 350, "time_per_iteration": 2.5335793495178223 }, { "auxiliary_loss_clip": 0.01231843, "auxiliary_loss_mlp": 0.0107489, "balance_loss_clip": 1.03364384, "balance_loss_mlp": 1.05170834, "epoch": 0.021103261686457236, "flos": 24453132013440.0, "grad_norm": 7.130091306898022, "language_loss": 0.79452366, "learning_rate": 3.995654809908581e-06, "loss": 0.81759101, "num_input_tokens_seen": 7317430, "router_z_loss_clip": 0.41210938, "router_z_loss_mlp": 1.8046875, "step": 351, "time_per_iteration": 2.4338746070861816 }, { "auxiliary_loss_clip": 0.01236511, "auxiliary_loss_mlp": 0.01092286, "balance_loss_clip": 1.0449841, "balance_loss_mlp": 1.0526005, "epoch": 0.021163384939125205, "flos": 14682233917440.0, "grad_norm": 3.84086245600335, "language_loss": 0.87032181, "learning_rate": 3.9956298825165005e-06, "loss": 0.89360976, "num_input_tokens_seen": 7334875, "router_z_loss_clip": 0.47265625, "router_z_loss_mlp": 1.84375, "step": 352, "time_per_iteration": 2.449504852294922 }, { "auxiliary_loss_clip": 0.01233454, "auxiliary_loss_mlp": 0.01077861, "balance_loss_clip": 1.03132141, "balance_loss_mlp": 1.05080497, "epoch": 0.021223508191793177, "flos": 24492199691520.0, "grad_norm": 1.8489810839612493, "language_loss": 0.82099515, "learning_rate": 3.995604883905522e-06, "loss": 0.84410834, "num_input_tokens_seen": 7355185, "router_z_loss_clip": 0.46484375, "router_z_loss_mlp": 1.828125, "step": 353, "time_per_iteration": 2.4712696075439453 }, { "auxiliary_loss_clip": 0.01229593, "auxiliary_loss_mlp": 0.01073406, "balance_loss_clip": 1.03025222, "balance_loss_mlp": 1.0502218, "epoch": 0.021283631444461146, "flos": 24126030685440.0, "grad_norm": 1.8250425149469043, "language_loss": 0.80346203, "learning_rate": 3.995579814076539e-06, "loss": 0.82649195, "num_input_tokens_seen": 7374425, "router_z_loss_clip": 0.43164062, "router_z_loss_mlp": 1.796875, "step": 354, "time_per_iteration": 2.4686806201934814 }, { "auxiliary_loss_clip": 0.01236871, "auxiliary_loss_mlp": 0.01079676, "balance_loss_clip": 1.03323174, "balance_loss_mlp": 1.05005431, "epoch": 0.021343754697129114, "flos": 25191055843200.0, "grad_norm": 3.0584252208007134, "language_loss": 0.80488598, "learning_rate": 3.9955546730304455e-06, "loss": 0.82805151, "num_input_tokens_seen": 7394175, "router_z_loss_clip": 0.46484375, "router_z_loss_mlp": 1.875, "step": 355, "time_per_iteration": 2.4575672149658203 }, { "auxiliary_loss_clip": 0.01232122, "auxiliary_loss_mlp": 0.01078482, "balance_loss_clip": 1.03416014, "balance_loss_mlp": 1.04956889, "epoch": 0.021403877949797083, "flos": 17273243180160.0, "grad_norm": 3.327945812803813, "language_loss": 0.88961899, "learning_rate": 3.995529460768139e-06, "loss": 0.91272497, "num_input_tokens_seen": 7412645, "router_z_loss_clip": 0.44140625, "router_z_loss_mlp": 1.828125, "step": 356, "time_per_iteration": 2.4201087951660156 }, { "auxiliary_loss_clip": 0.01229322, "auxiliary_loss_mlp": 0.01077796, "balance_loss_clip": 1.03042173, "balance_loss_mlp": 1.04959869, "epoch": 0.021464001202465055, "flos": 30916183478400.0, "grad_norm": 2.6518396998699507, "language_loss": 0.79755867, "learning_rate": 3.995504177290519e-06, "loss": 0.82062984, "num_input_tokens_seen": 7432275, "router_z_loss_clip": 0.47265625, "router_z_loss_mlp": 1.796875, "step": 357, "time_per_iteration": 2.510831356048584 }, { "auxiliary_loss_clip": 0.0123213, "auxiliary_loss_mlp": 0.01073983, "balance_loss_clip": 1.03171182, "balance_loss_mlp": 1.0482384, "epoch": 0.021524124455133024, "flos": 18185418938880.0, "grad_norm": 2.8196932538713564, "language_loss": 0.76050007, "learning_rate": 3.995478822598488e-06, "loss": 0.78356123, "num_input_tokens_seen": 7450245, "router_z_loss_clip": 0.421875, "router_z_loss_mlp": 1.84375, "step": 358, "time_per_iteration": 2.4422390460968018 }, { "auxiliary_loss_clip": 0.01229352, "auxiliary_loss_mlp": 0.01079093, "balance_loss_clip": 1.03245842, "balance_loss_mlp": 1.04627466, "epoch": 0.021584247707800992, "flos": 13805006296320.0, "grad_norm": 2.3266225841257038, "language_loss": 0.88053858, "learning_rate": 3.995453396692951e-06, "loss": 0.90362304, "num_input_tokens_seen": 7466845, "router_z_loss_clip": 0.46679688, "router_z_loss_mlp": 1.828125, "step": 359, "time_per_iteration": 2.3980555534362793 }, { "auxiliary_loss_clip": 0.01234026, "auxiliary_loss_mlp": 0.01068698, "balance_loss_clip": 1.02614033, "balance_loss_mlp": 1.05061042, "epoch": 0.02164437096046896, "flos": 23293596205440.0, "grad_norm": 4.100534030530065, "language_loss": 0.7596643, "learning_rate": 3.995427899574816e-06, "loss": 0.78269148, "num_input_tokens_seen": 7485450, "router_z_loss_clip": 0.42578125, "router_z_loss_mlp": 1.828125, "step": 360, "time_per_iteration": 2.508373737335205 }, { "auxiliary_loss_clip": 0.01130479, "auxiliary_loss_mlp": 0.01015122, "balance_loss_clip": 1.00186574, "balance_loss_mlp": 1.03843045, "epoch": 0.02170449421313693, "flos": 68896237875840.0, "grad_norm": 0.8318827427639371, "language_loss": 0.64908099, "learning_rate": 3.99540233124499e-06, "loss": 0.67053699, "num_input_tokens_seen": 7553780, "router_z_loss_clip": 0.1328125, "router_z_loss_mlp": 0.921875, "step": 361, "time_per_iteration": 3.1652159690856934 }, { "auxiliary_loss_clip": 0.0122958, "auxiliary_loss_mlp": 0.01074758, "balance_loss_clip": 1.03010178, "balance_loss_mlp": 1.04697037, "epoch": 0.0217646174658049, "flos": 25227365523840.0, "grad_norm": 3.2735007447015194, "language_loss": 0.77731925, "learning_rate": 3.995376691704389e-06, "loss": 0.80036259, "num_input_tokens_seen": 7574155, "router_z_loss_clip": 0.4453125, "router_z_loss_mlp": 1.828125, "step": 362, "time_per_iteration": 2.497349262237549 }, { "auxiliary_loss_clip": 0.01232158, "auxiliary_loss_mlp": 0.01070141, "balance_loss_clip": 1.02698779, "balance_loss_mlp": 1.04921007, "epoch": 0.02182474071847287, "flos": 22892025214080.0, "grad_norm": 2.257508093659879, "language_loss": 0.9193871, "learning_rate": 3.995350980953926e-06, "loss": 0.94241005, "num_input_tokens_seen": 7592320, "router_z_loss_clip": 0.43164062, "router_z_loss_mlp": 1.828125, "step": 363, "time_per_iteration": 2.4355170726776123 }, { "auxiliary_loss_clip": 0.0122669, "auxiliary_loss_mlp": 0.01077318, "balance_loss_clip": 1.03614295, "balance_loss_mlp": 1.04583764, "epoch": 0.02188486397114084, "flos": 23657879998080.0, "grad_norm": 3.088695106876858, "language_loss": 0.89338195, "learning_rate": 3.99532519899452e-06, "loss": 0.91642201, "num_input_tokens_seen": 7611185, "router_z_loss_clip": 0.41210938, "router_z_loss_mlp": 1.8125, "step": 364, "time_per_iteration": 2.4619293212890625 }, { "auxiliary_loss_clip": 0.01231817, "auxiliary_loss_mlp": 0.01075566, "balance_loss_clip": 1.03019536, "balance_loss_mlp": 1.04998064, "epoch": 0.021944987223808807, "flos": 21542562276480.0, "grad_norm": 2.164351222240482, "language_loss": 0.78897971, "learning_rate": 3.99529934582709e-06, "loss": 0.81205356, "num_input_tokens_seen": 7631970, "router_z_loss_clip": 0.453125, "router_z_loss_mlp": 1.8203125, "step": 365, "time_per_iteration": 2.4631075859069824 }, { "auxiliary_loss_clip": 0.0123104, "auxiliary_loss_mlp": 0.01081015, "balance_loss_clip": 1.03552461, "balance_loss_mlp": 1.04984212, "epoch": 0.022005110476476776, "flos": 16069961571840.0, "grad_norm": 2.7063772335893934, "language_loss": 0.83782774, "learning_rate": 3.995273421452558e-06, "loss": 0.86094832, "num_input_tokens_seen": 7649745, "router_z_loss_clip": 0.453125, "router_z_loss_mlp": 1.8125, "step": 366, "time_per_iteration": 2.45685076713562 }, { "auxiliary_loss_clip": 0.01234878, "auxiliary_loss_mlp": 0.01070655, "balance_loss_clip": 1.02695298, "balance_loss_mlp": 1.05081034, "epoch": 0.022065233729144748, "flos": 21432655716480.0, "grad_norm": 2.1036187774214907, "language_loss": 0.86579663, "learning_rate": 3.995247425871851e-06, "loss": 0.888852, "num_input_tokens_seen": 7668830, "router_z_loss_clip": 0.4375, "router_z_loss_mlp": 1.84375, "step": 367, "time_per_iteration": 2.4273951053619385 }, { "auxiliary_loss_clip": 0.01234498, "auxiliary_loss_mlp": 0.01090663, "balance_loss_clip": 1.04426718, "balance_loss_mlp": 1.04885268, "epoch": 0.022125356981812717, "flos": 21542632099200.0, "grad_norm": 2.387828187297824, "language_loss": 0.84244931, "learning_rate": 3.995221359085895e-06, "loss": 0.86570096, "num_input_tokens_seen": 7687240, "router_z_loss_clip": 0.46484375, "router_z_loss_mlp": 1.859375, "step": 368, "time_per_iteration": 2.4499919414520264 }, { "auxiliary_loss_clip": 0.01234171, "auxiliary_loss_mlp": 0.01067317, "balance_loss_clip": 1.02347231, "balance_loss_mlp": 1.04798639, "epoch": 0.022185480234480685, "flos": 20703110613120.0, "grad_norm": 2.3411944361606665, "language_loss": 0.74964315, "learning_rate": 3.995195221095621e-06, "loss": 0.77265799, "num_input_tokens_seen": 7704440, "router_z_loss_clip": 0.4375, "router_z_loss_mlp": 1.859375, "step": 369, "time_per_iteration": 2.4479618072509766 }, { "auxiliary_loss_clip": 0.01229101, "auxiliary_loss_mlp": 0.0107853, "balance_loss_clip": 1.03704512, "balance_loss_mlp": 1.04931593, "epoch": 0.022245603487148654, "flos": 25191998449920.0, "grad_norm": 2.161917099101355, "language_loss": 0.82162476, "learning_rate": 3.995169011901963e-06, "loss": 0.84470105, "num_input_tokens_seen": 7727160, "router_z_loss_clip": 0.4140625, "router_z_loss_mlp": 1.796875, "step": 370, "time_per_iteration": 3.883336305618286 }, { "auxiliary_loss_clip": 0.01231991, "auxiliary_loss_mlp": 0.0108399, "balance_loss_clip": 1.04186118, "balance_loss_mlp": 1.05016744, "epoch": 0.022305726739816623, "flos": 21394914670080.0, "grad_norm": 2.3533588348566785, "language_loss": 0.81352019, "learning_rate": 3.995142731505854e-06, "loss": 0.83667994, "num_input_tokens_seen": 7747730, "router_z_loss_clip": 0.421875, "router_z_loss_mlp": 1.8203125, "step": 371, "time_per_iteration": 2.4541006088256836 }, { "auxiliary_loss_clip": 0.01233079, "auxiliary_loss_mlp": 0.0108394, "balance_loss_clip": 1.03885484, "balance_loss_mlp": 1.05017495, "epoch": 0.022365849992484595, "flos": 22491047715840.0, "grad_norm": 2.63447056724904, "language_loss": 0.83084446, "learning_rate": 3.995116379908234e-06, "loss": 0.85401469, "num_input_tokens_seen": 7766765, "router_z_loss_clip": 0.45117188, "router_z_loss_mlp": 1.828125, "step": 372, "time_per_iteration": 5.315749168395996 }, { "auxiliary_loss_clip": 0.01225089, "auxiliary_loss_mlp": 0.01074048, "balance_loss_clip": 1.02908194, "balance_loss_mlp": 1.04825759, "epoch": 0.022425973245152563, "flos": 17855664347520.0, "grad_norm": 5.51553422257015, "language_loss": 0.78102767, "learning_rate": 3.995089957110041e-06, "loss": 0.80401897, "num_input_tokens_seen": 7784010, "router_z_loss_clip": 0.44921875, "router_z_loss_mlp": 1.765625, "step": 373, "time_per_iteration": 3.8593719005584717 }, { "auxiliary_loss_clip": 0.01229883, "auxiliary_loss_mlp": 0.01077793, "balance_loss_clip": 1.03256512, "balance_loss_mlp": 1.04971766, "epoch": 0.022486096497820532, "flos": 15482233877760.0, "grad_norm": 2.5033796908762147, "language_loss": 0.76996267, "learning_rate": 3.995063463112221e-06, "loss": 0.79303944, "num_input_tokens_seen": 7801305, "router_z_loss_clip": 0.453125, "router_z_loss_mlp": 1.8046875, "step": 374, "time_per_iteration": 2.419429302215576 }, { "auxiliary_loss_clip": 0.01228989, "auxiliary_loss_mlp": 0.01072966, "balance_loss_clip": 1.0256635, "balance_loss_mlp": 1.0454855, "epoch": 0.0225462197504885, "flos": 27782868067200.0, "grad_norm": 1.936964279705477, "language_loss": 0.85860884, "learning_rate": 3.995036897915717e-06, "loss": 0.88162833, "num_input_tokens_seen": 7823965, "router_z_loss_clip": 0.47265625, "router_z_loss_mlp": 1.8359375, "step": 375, "time_per_iteration": 2.4995079040527344 }, { "auxiliary_loss_clip": 0.0123226, "auxiliary_loss_mlp": 0.01084578, "balance_loss_clip": 1.03849185, "balance_loss_mlp": 1.05112147, "epoch": 0.02260634300315647, "flos": 19974438293760.0, "grad_norm": 2.2991912097388605, "language_loss": 0.88661456, "learning_rate": 3.995010261521478e-06, "loss": 0.90978289, "num_input_tokens_seen": 7842115, "router_z_loss_clip": 0.4609375, "router_z_loss_mlp": 1.8125, "step": 376, "time_per_iteration": 2.418308973312378 }, { "auxiliary_loss_clip": 0.0122968, "auxiliary_loss_mlp": 0.01071277, "balance_loss_clip": 1.02774143, "balance_loss_mlp": 1.04656196, "epoch": 0.02266646625582444, "flos": 16027437669120.0, "grad_norm": 2.4116974172152337, "language_loss": 0.74843597, "learning_rate": 3.9949835539304545e-06, "loss": 0.77144551, "num_input_tokens_seen": 7857830, "router_z_loss_clip": 0.43554688, "router_z_loss_mlp": 1.828125, "step": 377, "time_per_iteration": 2.4013772010803223 }, { "auxiliary_loss_clip": 0.01224565, "auxiliary_loss_mlp": 0.01075962, "balance_loss_clip": 1.03197408, "balance_loss_mlp": 1.04923904, "epoch": 0.02272658950849241, "flos": 20403800593920.0, "grad_norm": 2.256167411950087, "language_loss": 0.9871459, "learning_rate": 3.9949567751436e-06, "loss": 1.01015115, "num_input_tokens_seen": 7875840, "router_z_loss_clip": 0.43945312, "router_z_loss_mlp": 1.75, "step": 378, "time_per_iteration": 2.4311933517456055 }, { "auxiliary_loss_clip": 0.01114269, "auxiliary_loss_mlp": 0.010195, "balance_loss_clip": 1.0085324, "balance_loss_mlp": 1.02366507, "epoch": 0.02278671276116038, "flos": 69843990176640.0, "grad_norm": 0.9572279775789706, "language_loss": 0.75515658, "learning_rate": 3.99492992516187e-06, "loss": 0.77649432, "num_input_tokens_seen": 7940190, "router_z_loss_clip": 0.10986328, "router_z_loss_mlp": 0.90625, "step": 379, "time_per_iteration": 3.109609842300415 }, { "auxiliary_loss_clip": 0.01233634, "auxiliary_loss_mlp": 0.01072328, "balance_loss_clip": 1.02812481, "balance_loss_mlp": 1.04682207, "epoch": 0.022846836013828347, "flos": 38507243927040.0, "grad_norm": 2.251694170849771, "language_loss": 0.78342873, "learning_rate": 3.994903003986222e-06, "loss": 0.80648834, "num_input_tokens_seen": 7960840, "router_z_loss_clip": 0.44140625, "router_z_loss_mlp": 1.8671875, "step": 380, "time_per_iteration": 2.5509915351867676 }, { "auxiliary_loss_clip": 0.01224327, "auxiliary_loss_mlp": 0.01077174, "balance_loss_clip": 1.03294706, "balance_loss_mlp": 1.0472827, "epoch": 0.02290695926649632, "flos": 20958430452480.0, "grad_norm": 2.3818750916319096, "language_loss": 0.95732051, "learning_rate": 3.9948760116176174e-06, "loss": 0.98033547, "num_input_tokens_seen": 7975500, "router_z_loss_clip": 0.44335938, "router_z_loss_mlp": 1.7734375, "step": 381, "time_per_iteration": 2.4247195720672607 }, { "auxiliary_loss_clip": 0.01233709, "auxiliary_loss_mlp": 0.01091844, "balance_loss_clip": 1.04683065, "balance_loss_mlp": 1.0478375, "epoch": 0.022967082519164288, "flos": 24021325918080.0, "grad_norm": 5.971767421602091, "language_loss": 0.87281406, "learning_rate": 3.994848948057019e-06, "loss": 0.89606953, "num_input_tokens_seen": 7993880, "router_z_loss_clip": 0.45117188, "router_z_loss_mlp": 1.859375, "step": 382, "time_per_iteration": 2.4535419940948486 }, { "auxiliary_loss_clip": 0.01232721, "auxiliary_loss_mlp": 0.01079831, "balance_loss_clip": 1.03834581, "balance_loss_mlp": 1.04840291, "epoch": 0.023027205771832256, "flos": 20996066764800.0, "grad_norm": 2.0497498489302672, "language_loss": 0.84413087, "learning_rate": 3.994821813305394e-06, "loss": 0.8672564, "num_input_tokens_seen": 8012730, "router_z_loss_clip": 0.4140625, "router_z_loss_mlp": 1.84375, "step": 383, "time_per_iteration": 2.427741765975952 }, { "auxiliary_loss_clip": 0.01224827, "auxiliary_loss_mlp": 0.01096017, "balance_loss_clip": 1.05219579, "balance_loss_mlp": 1.04934549, "epoch": 0.023087329024500225, "flos": 21359757064320.0, "grad_norm": 2.510335613890191, "language_loss": 0.82757276, "learning_rate": 3.99479460736371e-06, "loss": 0.8507812, "num_input_tokens_seen": 8031275, "router_z_loss_clip": 0.4375, "router_z_loss_mlp": 1.7578125, "step": 384, "time_per_iteration": 2.416593074798584 }, { "auxiliary_loss_clip": 0.01226601, "auxiliary_loss_mlp": 0.0107713, "balance_loss_clip": 1.03562117, "balance_loss_mlp": 1.05061913, "epoch": 0.023147452277168194, "flos": 21871339349760.0, "grad_norm": 2.094474581463722, "language_loss": 0.88727117, "learning_rate": 3.994767330232937e-06, "loss": 0.91030848, "num_input_tokens_seen": 8051600, "router_z_loss_clip": 0.4140625, "router_z_loss_mlp": 1.7578125, "step": 385, "time_per_iteration": 2.4557037353515625 }, { "auxiliary_loss_clip": 0.01231744, "auxiliary_loss_mlp": 0.01084369, "balance_loss_clip": 1.04207373, "balance_loss_mlp": 1.05157328, "epoch": 0.023207575529836166, "flos": 18915697180800.0, "grad_norm": 2.4968411771950567, "language_loss": 0.69599569, "learning_rate": 3.994739981914049e-06, "loss": 0.71915674, "num_input_tokens_seen": 8070600, "router_z_loss_clip": 0.421875, "router_z_loss_mlp": 1.796875, "step": 386, "time_per_iteration": 2.4219396114349365 }, { "auxiliary_loss_clip": 0.01227764, "auxiliary_loss_mlp": 0.01078979, "balance_loss_clip": 1.03563499, "balance_loss_mlp": 1.05063343, "epoch": 0.023267698782504134, "flos": 25044839602560.0, "grad_norm": 9.04059514600862, "language_loss": 0.87687516, "learning_rate": 3.994712562408022e-06, "loss": 0.89994264, "num_input_tokens_seen": 8090680, "router_z_loss_clip": 0.43359375, "router_z_loss_mlp": 1.765625, "step": 387, "time_per_iteration": 2.4649181365966797 }, { "auxiliary_loss_clip": 0.01228789, "auxiliary_loss_mlp": 0.01068427, "balance_loss_clip": 1.02572608, "balance_loss_mlp": 1.05026042, "epoch": 0.023327822035172103, "flos": 28877883949440.0, "grad_norm": 1.990605289011327, "language_loss": 0.83328348, "learning_rate": 3.994685071715835e-06, "loss": 0.85625565, "num_input_tokens_seen": 8114610, "router_z_loss_clip": 0.42773438, "router_z_loss_mlp": 1.7890625, "step": 388, "time_per_iteration": 2.5158395767211914 }, { "auxiliary_loss_clip": 0.01225898, "auxiliary_loss_mlp": 0.01072797, "balance_loss_clip": 1.03352904, "balance_loss_mlp": 1.04716516, "epoch": 0.02338794528784007, "flos": 27120426330240.0, "grad_norm": 2.7565455513494936, "language_loss": 0.9320004, "learning_rate": 3.9946575098384686e-06, "loss": 0.95498735, "num_input_tokens_seen": 8133975, "router_z_loss_clip": 0.39257812, "router_z_loss_mlp": 1.7890625, "step": 389, "time_per_iteration": 2.494342803955078 }, { "auxiliary_loss_clip": 0.0122056, "auxiliary_loss_mlp": 0.01076788, "balance_loss_clip": 1.0343492, "balance_loss_mlp": 1.04805517, "epoch": 0.02344806854050804, "flos": 21321352702080.0, "grad_norm": 3.0699235696824085, "language_loss": 0.87314248, "learning_rate": 3.9946298767769065e-06, "loss": 0.89611602, "num_input_tokens_seen": 8153570, "router_z_loss_clip": 0.42382812, "router_z_loss_mlp": 1.7265625, "step": 390, "time_per_iteration": 2.453216552734375 }, { "auxiliary_loss_clip": 0.01223692, "auxiliary_loss_mlp": 0.01069994, "balance_loss_clip": 1.0310601, "balance_loss_mlp": 1.04871178, "epoch": 0.023508191793176012, "flos": 24788856447360.0, "grad_norm": 4.83313340351558, "language_loss": 0.88527739, "learning_rate": 3.994602172532135e-06, "loss": 0.90821421, "num_input_tokens_seen": 8170075, "router_z_loss_clip": 0.38867188, "router_z_loss_mlp": 1.75, "step": 391, "time_per_iteration": 2.455122947692871 }, { "auxiliary_loss_clip": 0.01220029, "auxiliary_loss_mlp": 0.01064478, "balance_loss_clip": 1.02511477, "balance_loss_mlp": 1.04577327, "epoch": 0.02356831504584398, "flos": 25994162914560.0, "grad_norm": 3.6976594815005357, "language_loss": 0.86070114, "learning_rate": 3.994574397105143e-06, "loss": 0.88354623, "num_input_tokens_seen": 8190420, "router_z_loss_clip": 0.39453125, "router_z_loss_mlp": 1.7421875, "step": 392, "time_per_iteration": 2.462695837020874 }, { "auxiliary_loss_clip": 0.01219756, "auxiliary_loss_mlp": 0.01065758, "balance_loss_clip": 1.02577543, "balance_loss_mlp": 1.04501486, "epoch": 0.02362843829851195, "flos": 19061459573760.0, "grad_norm": 1.9887120025248302, "language_loss": 0.88842404, "learning_rate": 3.994546550496921e-06, "loss": 0.91127914, "num_input_tokens_seen": 8208790, "router_z_loss_clip": 0.40039062, "router_z_loss_mlp": 1.75, "step": 393, "time_per_iteration": 2.4153332710266113 }, { "auxiliary_loss_clip": 0.01226983, "auxiliary_loss_mlp": 0.01083042, "balance_loss_clip": 1.03962576, "balance_loss_mlp": 1.05139089, "epoch": 0.023688561551179918, "flos": 16070101217280.0, "grad_norm": 3.2261350159623565, "language_loss": 0.81036854, "learning_rate": 3.994518632708464e-06, "loss": 0.83346879, "num_input_tokens_seen": 8226885, "router_z_loss_clip": 0.43359375, "router_z_loss_mlp": 1.75, "step": 394, "time_per_iteration": 2.4195621013641357 }, { "auxiliary_loss_clip": 0.01222919, "auxiliary_loss_mlp": 0.0107631, "balance_loss_clip": 1.03420556, "balance_loss_mlp": 1.04789519, "epoch": 0.023748684803847887, "flos": 21723342629760.0, "grad_norm": 1.9563544568378761, "language_loss": 0.85760427, "learning_rate": 3.994490643740766e-06, "loss": 0.88059652, "num_input_tokens_seen": 8246825, "router_z_loss_clip": 0.421875, "router_z_loss_mlp": 1.75, "step": 395, "time_per_iteration": 2.447772741317749 }, { "auxiliary_loss_clip": 0.01105912, "auxiliary_loss_mlp": 0.01019347, "balance_loss_clip": 1.00837958, "balance_loss_mlp": 1.0205375, "epoch": 0.02380880805651586, "flos": 61923175136640.0, "grad_norm": 0.91434065681227, "language_loss": 0.63803995, "learning_rate": 3.994462583594828e-06, "loss": 0.65929258, "num_input_tokens_seen": 8302835, "router_z_loss_clip": 0.10986328, "router_z_loss_mlp": 0.8515625, "step": 396, "time_per_iteration": 2.896740198135376 }, { "auxiliary_loss_clip": 0.01220118, "auxiliary_loss_mlp": 0.01063434, "balance_loss_clip": 1.02218795, "balance_loss_mlp": 1.04511356, "epoch": 0.023868931309183827, "flos": 20265299763840.0, "grad_norm": 2.704976210508278, "language_loss": 0.83204758, "learning_rate": 3.994434452271651e-06, "loss": 0.85488307, "num_input_tokens_seen": 8320745, "router_z_loss_clip": 0.4140625, "router_z_loss_mlp": 1.75, "step": 397, "time_per_iteration": 2.4413938522338867 }, { "auxiliary_loss_clip": 0.0122405, "auxiliary_loss_mlp": 0.01069104, "balance_loss_clip": 1.02654648, "balance_loss_mlp": 1.04862189, "epoch": 0.023929054561851796, "flos": 21138128553600.0, "grad_norm": 2.4026621418980687, "language_loss": 0.84061825, "learning_rate": 3.994406249772239e-06, "loss": 0.86354977, "num_input_tokens_seen": 8339540, "router_z_loss_clip": 0.42578125, "router_z_loss_mlp": 1.75, "step": 398, "time_per_iteration": 2.4305572509765625 }, { "auxiliary_loss_clip": 0.01224106, "auxiliary_loss_mlp": 0.01077038, "balance_loss_clip": 1.03304994, "balance_loss_mlp": 1.04774022, "epoch": 0.023989177814519765, "flos": 13697683176960.0, "grad_norm": 3.4160456873041443, "language_loss": 0.8576498, "learning_rate": 3.994377976097598e-06, "loss": 0.88066125, "num_input_tokens_seen": 8354890, "router_z_loss_clip": 0.43945312, "router_z_loss_mlp": 1.765625, "step": 399, "time_per_iteration": 2.400832176208496 }, { "auxiliary_loss_clip": 0.01219711, "auxiliary_loss_mlp": 0.01079695, "balance_loss_clip": 1.03799558, "balance_loss_mlp": 1.04665935, "epoch": 0.024049301067187733, "flos": 26320845306240.0, "grad_norm": 2.980519631223286, "language_loss": 0.85427976, "learning_rate": 3.9943496312487365e-06, "loss": 0.87727386, "num_input_tokens_seen": 8375845, "router_z_loss_clip": 0.41601562, "router_z_loss_mlp": 1.734375, "step": 400, "time_per_iteration": 2.48085618019104 }, { "auxiliary_loss_clip": 0.01224102, "auxiliary_loss_mlp": 0.01069693, "balance_loss_clip": 1.02756393, "balance_loss_mlp": 1.05105746, "epoch": 0.024109424319855705, "flos": 24424293363840.0, "grad_norm": 1.8415507104795583, "language_loss": 0.7897774, "learning_rate": 3.994321215226667e-06, "loss": 0.81271529, "num_input_tokens_seen": 8395240, "router_z_loss_clip": 0.421875, "router_z_loss_mlp": 1.734375, "step": 401, "time_per_iteration": 2.4944117069244385 }, { "auxiliary_loss_clip": 0.01219562, "auxiliary_loss_mlp": 0.01078446, "balance_loss_clip": 1.03874898, "balance_loss_mlp": 1.04854548, "epoch": 0.024169547572523674, "flos": 29603169866880.0, "grad_norm": 2.2638273233506365, "language_loss": 0.78047067, "learning_rate": 3.994292728032404e-06, "loss": 0.80345076, "num_input_tokens_seen": 8416950, "router_z_loss_clip": 0.39648438, "router_z_loss_mlp": 1.703125, "step": 402, "time_per_iteration": 2.5082874298095703 }, { "auxiliary_loss_clip": 0.0122181, "auxiliary_loss_mlp": 0.01069983, "balance_loss_clip": 1.02740133, "balance_loss_mlp": 1.04404521, "epoch": 0.024229670825191642, "flos": 22600360782720.0, "grad_norm": 2.746171059960839, "language_loss": 0.94473672, "learning_rate": 3.994264169666963e-06, "loss": 0.96765459, "num_input_tokens_seen": 8433660, "router_z_loss_clip": 0.42578125, "router_z_loss_mlp": 1.7734375, "step": 403, "time_per_iteration": 2.4508306980133057 }, { "auxiliary_loss_clip": 0.01228746, "auxiliary_loss_mlp": 0.01071185, "balance_loss_clip": 1.0266248, "balance_loss_mlp": 1.04774153, "epoch": 0.02428979407785961, "flos": 18149283815040.0, "grad_norm": 2.778395112563993, "language_loss": 0.9934364, "learning_rate": 3.994235540131364e-06, "loss": 1.01643562, "num_input_tokens_seen": 8450180, "router_z_loss_clip": 0.4453125, "router_z_loss_mlp": 1.8125, "step": 404, "time_per_iteration": 2.392777919769287 }, { "auxiliary_loss_clip": 0.01224341, "auxiliary_loss_mlp": 0.01079139, "balance_loss_clip": 1.03729606, "balance_loss_mlp": 1.04792476, "epoch": 0.024349917330527583, "flos": 15304071876480.0, "grad_norm": 3.1709684895581076, "language_loss": 0.87440234, "learning_rate": 3.994206839426627e-06, "loss": 0.8974371, "num_input_tokens_seen": 8467775, "router_z_loss_clip": 0.41796875, "router_z_loss_mlp": 1.765625, "step": 405, "time_per_iteration": 2.414794445037842 }, { "auxiliary_loss_clip": 0.01232461, "auxiliary_loss_mlp": 0.01072855, "balance_loss_clip": 1.02982068, "balance_loss_mlp": 1.05070579, "epoch": 0.024410040583195552, "flos": 20772937065600.0, "grad_norm": 3.0795526101990034, "language_loss": 0.93019068, "learning_rate": 3.994178067553779e-06, "loss": 0.95324385, "num_input_tokens_seen": 8486765, "router_z_loss_clip": 0.43164062, "router_z_loss_mlp": 1.8203125, "step": 406, "time_per_iteration": 2.4340293407440186 }, { "auxiliary_loss_clip": 0.01225436, "auxiliary_loss_mlp": 0.01079018, "balance_loss_clip": 1.03405201, "balance_loss_mlp": 1.04839635, "epoch": 0.02447016383586352, "flos": 21797777381760.0, "grad_norm": 4.538106572159414, "language_loss": 0.86687589, "learning_rate": 3.994149224513846e-06, "loss": 0.88992041, "num_input_tokens_seen": 8506515, "router_z_loss_clip": 0.44921875, "router_z_loss_mlp": 1.765625, "step": 407, "time_per_iteration": 2.441693067550659 }, { "auxiliary_loss_clip": 0.01223369, "auxiliary_loss_mlp": 0.01074858, "balance_loss_clip": 1.03272939, "balance_loss_mlp": 1.04847205, "epoch": 0.02453028708853149, "flos": 33946714247040.0, "grad_norm": 2.046892011184857, "language_loss": 0.73118854, "learning_rate": 3.994120310307856e-06, "loss": 0.75417078, "num_input_tokens_seen": 8528035, "router_z_loss_clip": 0.41992188, "router_z_loss_mlp": 1.75, "step": 408, "time_per_iteration": 2.5407135486602783 }, { "auxiliary_loss_clip": 0.01228314, "auxiliary_loss_mlp": 0.01074045, "balance_loss_clip": 1.03263187, "balance_loss_mlp": 1.05185044, "epoch": 0.024590410341199458, "flos": 21792086830080.0, "grad_norm": 2.7996715546795694, "language_loss": 0.92269748, "learning_rate": 3.994091324936841e-06, "loss": 0.94572109, "num_input_tokens_seen": 8546455, "router_z_loss_clip": 0.4140625, "router_z_loss_mlp": 1.765625, "step": 409, "time_per_iteration": 3.8719043731689453 }, { "auxiliary_loss_clip": 0.01225605, "auxiliary_loss_mlp": 0.01066815, "balance_loss_clip": 1.02556872, "balance_loss_mlp": 1.0501914, "epoch": 0.02465053359386743, "flos": 35113371972480.0, "grad_norm": 2.2476422238018245, "language_loss": 0.81878775, "learning_rate": 3.994062268401836e-06, "loss": 0.84171194, "num_input_tokens_seen": 8568450, "router_z_loss_clip": 0.41210938, "router_z_loss_mlp": 1.7578125, "step": 410, "time_per_iteration": 2.5387113094329834 }, { "auxiliary_loss_clip": 0.01226802, "auxiliary_loss_mlp": 0.01076319, "balance_loss_clip": 1.03333235, "balance_loss_mlp": 1.04942226, "epoch": 0.0247106568465354, "flos": 27450250744320.0, "grad_norm": 2.654348941693858, "language_loss": 0.77659327, "learning_rate": 3.994033140703878e-06, "loss": 0.7996245, "num_input_tokens_seen": 8589340, "router_z_loss_clip": 0.4296875, "router_z_loss_mlp": 1.7734375, "step": 411, "time_per_iteration": 3.9723191261291504 }, { "auxiliary_loss_clip": 0.0122833, "auxiliary_loss_mlp": 0.0108574, "balance_loss_clip": 1.04299116, "balance_loss_mlp": 1.04967451, "epoch": 0.024770780099203367, "flos": 20702761499520.0, "grad_norm": 2.2231464753261045, "language_loss": 0.86391199, "learning_rate": 3.994003941844007e-06, "loss": 0.88705271, "num_input_tokens_seen": 8607150, "router_z_loss_clip": 0.42773438, "router_z_loss_mlp": 1.7890625, "step": 412, "time_per_iteration": 3.839991331100464 }, { "auxiliary_loss_clip": 0.01101658, "auxiliary_loss_mlp": 0.01014881, "balance_loss_clip": 1.00386584, "balance_loss_mlp": 1.02291083, "epoch": 0.024830903351871336, "flos": 69548625141120.0, "grad_norm": 0.8306108238647311, "language_loss": 0.5848062, "learning_rate": 3.993974671823265e-06, "loss": 0.60597157, "num_input_tokens_seen": 8669865, "router_z_loss_clip": 0.11035156, "router_z_loss_mlp": 0.7890625, "step": 413, "time_per_iteration": 4.496323585510254 }, { "auxiliary_loss_clip": 0.01224728, "auxiliary_loss_mlp": 0.0107864, "balance_loss_clip": 1.03374577, "balance_loss_mlp": 1.04808426, "epoch": 0.024891026604539304, "flos": 32269102640640.0, "grad_norm": 2.8425881831129383, "language_loss": 0.80029666, "learning_rate": 3.9939453306426955e-06, "loss": 0.8233304, "num_input_tokens_seen": 8690235, "router_z_loss_clip": 0.44921875, "router_z_loss_mlp": 1.765625, "step": 414, "time_per_iteration": 2.5428895950317383 }, { "auxiliary_loss_clip": 0.01218806, "auxiliary_loss_mlp": 0.01077296, "balance_loss_clip": 1.03511941, "balance_loss_mlp": 1.04687631, "epoch": 0.024951149857207276, "flos": 18839377215360.0, "grad_norm": 2.7484104693387423, "language_loss": 0.7967658, "learning_rate": 3.9939159183033466e-06, "loss": 0.81972682, "num_input_tokens_seen": 8706295, "router_z_loss_clip": 0.421875, "router_z_loss_mlp": 1.71875, "step": 415, "time_per_iteration": 2.4227468967437744 }, { "auxiliary_loss_clip": 0.01230682, "auxiliary_loss_mlp": 0.01078942, "balance_loss_clip": 1.03686094, "balance_loss_mlp": 1.05102003, "epoch": 0.025011273109875245, "flos": 15376307212800.0, "grad_norm": 3.0735378724622233, "language_loss": 0.95692146, "learning_rate": 3.9938864348062675e-06, "loss": 0.98001772, "num_input_tokens_seen": 8724200, "router_z_loss_clip": 0.41992188, "router_z_loss_mlp": 1.796875, "step": 416, "time_per_iteration": 2.4133734703063965 }, { "auxiliary_loss_clip": 0.01223109, "auxiliary_loss_mlp": 0.01067797, "balance_loss_clip": 1.02349901, "balance_loss_mlp": 1.04716539, "epoch": 0.025071396362543213, "flos": 18914545105920.0, "grad_norm": 2.036718056261979, "language_loss": 0.77308404, "learning_rate": 3.993856880152509e-06, "loss": 0.79599309, "num_input_tokens_seen": 8744170, "router_z_loss_clip": 0.44140625, "router_z_loss_mlp": 1.7578125, "step": 417, "time_per_iteration": 2.454113245010376 }, { "auxiliary_loss_clip": 0.01220784, "auxiliary_loss_mlp": 0.01082056, "balance_loss_clip": 1.03978431, "balance_loss_mlp": 1.05053282, "epoch": 0.025131519615211182, "flos": 25336783324800.0, "grad_norm": 1.654779187260334, "language_loss": 0.76904714, "learning_rate": 3.9938272543431286e-06, "loss": 0.79207551, "num_input_tokens_seen": 8765120, "router_z_loss_clip": 0.421875, "router_z_loss_mlp": 1.703125, "step": 418, "time_per_iteration": 2.50669002532959 }, { "auxiliary_loss_clip": 0.0122798, "auxiliary_loss_mlp": 0.01067283, "balance_loss_clip": 1.02725244, "balance_loss_mlp": 1.05104196, "epoch": 0.02519164286787915, "flos": 18952146506880.0, "grad_norm": 4.729071356260811, "language_loss": 0.81378472, "learning_rate": 3.993797557379182e-06, "loss": 0.83673733, "num_input_tokens_seen": 8783500, "router_z_loss_clip": 0.40039062, "router_z_loss_mlp": 1.765625, "step": 419, "time_per_iteration": 2.421539545059204 }, { "auxiliary_loss_clip": 0.01219551, "auxiliary_loss_mlp": 0.01074075, "balance_loss_clip": 1.0309689, "balance_loss_mlp": 1.04970813, "epoch": 0.025251766120547123, "flos": 17420122736640.0, "grad_norm": 2.2544890987226625, "language_loss": 0.7359246, "learning_rate": 3.9937677892617295e-06, "loss": 0.75886083, "num_input_tokens_seen": 8801175, "router_z_loss_clip": 0.43164062, "router_z_loss_mlp": 1.6953125, "step": 420, "time_per_iteration": 2.417229175567627 }, { "auxiliary_loss_clip": 0.01220992, "auxiliary_loss_mlp": 0.01075999, "balance_loss_clip": 1.03506291, "balance_loss_mlp": 1.04477525, "epoch": 0.02531188937321509, "flos": 25044281020800.0, "grad_norm": 1.9282394845206294, "language_loss": 0.78481078, "learning_rate": 3.993737949991833e-06, "loss": 0.80778074, "num_input_tokens_seen": 8820215, "router_z_loss_clip": 0.41015625, "router_z_loss_mlp": 1.7578125, "step": 421, "time_per_iteration": 2.5036356449127197 }, { "auxiliary_loss_clip": 0.01219951, "auxiliary_loss_mlp": 0.01077345, "balance_loss_clip": 1.0353117, "balance_loss_mlp": 1.05088842, "epoch": 0.02537201262588306, "flos": 30590897541120.0, "grad_norm": 2.08196997858772, "language_loss": 0.81678551, "learning_rate": 3.993708039570557e-06, "loss": 0.8397584, "num_input_tokens_seen": 8839660, "router_z_loss_clip": 0.41992188, "router_z_loss_mlp": 1.6875, "step": 422, "time_per_iteration": 2.503122568130493 }, { "auxiliary_loss_clip": 0.01221051, "auxiliary_loss_mlp": 0.01069051, "balance_loss_clip": 1.03059411, "balance_loss_mlp": 1.04643059, "epoch": 0.02543213587855103, "flos": 26064233746560.0, "grad_norm": 1.8956315955091694, "language_loss": 0.83241171, "learning_rate": 3.99367805799897e-06, "loss": 0.85531271, "num_input_tokens_seen": 8859280, "router_z_loss_clip": 0.38476562, "router_z_loss_mlp": 1.75, "step": 423, "time_per_iteration": 2.467223882675171 }, { "auxiliary_loss_clip": 0.01215465, "auxiliary_loss_mlp": 0.01064725, "balance_loss_clip": 1.02540994, "balance_loss_mlp": 1.04517615, "epoch": 0.025492259131218997, "flos": 36021498013440.0, "grad_norm": 2.03259954828413, "language_loss": 0.74068058, "learning_rate": 3.993648005278142e-06, "loss": 0.76348245, "num_input_tokens_seen": 8880560, "router_z_loss_clip": 0.39257812, "router_z_loss_mlp": 1.703125, "step": 424, "time_per_iteration": 2.521575689315796 }, { "auxiliary_loss_clip": 0.01225442, "auxiliary_loss_mlp": 0.01083442, "balance_loss_clip": 1.04109931, "balance_loss_mlp": 1.04883325, "epoch": 0.02555238238388697, "flos": 18587059752960.0, "grad_norm": 2.815339780265551, "language_loss": 0.8265295, "learning_rate": 3.993617881409143e-06, "loss": 0.84961832, "num_input_tokens_seen": 8899155, "router_z_loss_clip": 0.42382812, "router_z_loss_mlp": 1.765625, "step": 425, "time_per_iteration": 2.4312291145324707 }, { "auxiliary_loss_clip": 0.01222589, "auxiliary_loss_mlp": 0.01077014, "balance_loss_clip": 1.03600645, "balance_loss_mlp": 1.0447588, "epoch": 0.025612505636554938, "flos": 24242046733440.0, "grad_norm": 3.8650408966719287, "language_loss": 0.85035753, "learning_rate": 3.993587686393052e-06, "loss": 0.87335354, "num_input_tokens_seen": 8917890, "router_z_loss_clip": 0.41015625, "router_z_loss_mlp": 1.78125, "step": 426, "time_per_iteration": 2.4371635913848877 }, { "auxiliary_loss_clip": 0.01098027, "auxiliary_loss_mlp": 0.01017479, "balance_loss_clip": 1.00841868, "balance_loss_mlp": 1.0204736, "epoch": 0.025672628889222907, "flos": 60583661936640.0, "grad_norm": 0.8871951760081042, "language_loss": 0.57136494, "learning_rate": 3.993557420230944e-06, "loss": 0.59252, "num_input_tokens_seen": 8978260, "router_z_loss_clip": 0.09082031, "router_z_loss_mlp": 0.7734375, "step": 427, "time_per_iteration": 3.173518657684326 }, { "auxiliary_loss_clip": 0.01222285, "auxiliary_loss_mlp": 0.01077646, "balance_loss_clip": 1.03778279, "balance_loss_mlp": 1.04927766, "epoch": 0.025732752141890875, "flos": 19572238897920.0, "grad_norm": 3.820220230684572, "language_loss": 0.87785196, "learning_rate": 3.9935270829239e-06, "loss": 0.90085125, "num_input_tokens_seen": 8994460, "router_z_loss_clip": 0.3984375, "router_z_loss_mlp": 1.734375, "step": 428, "time_per_iteration": 2.404013156890869 }, { "auxiliary_loss_clip": 0.01218422, "auxiliary_loss_mlp": 0.01069172, "balance_loss_clip": 1.02976179, "balance_loss_mlp": 1.04609978, "epoch": 0.025792875394558847, "flos": 31282945977600.0, "grad_norm": 1.7895895121794416, "language_loss": 0.85488737, "learning_rate": 3.993496674473002e-06, "loss": 0.87776333, "num_input_tokens_seen": 9016670, "router_z_loss_clip": 0.39453125, "router_z_loss_mlp": 1.7265625, "step": 429, "time_per_iteration": 2.5000152587890625 }, { "auxiliary_loss_clip": 0.01218331, "auxiliary_loss_mlp": 0.01076159, "balance_loss_clip": 1.0330286, "balance_loss_mlp": 1.04326129, "epoch": 0.025852998647226816, "flos": 32378241150720.0, "grad_norm": 3.5761364735444694, "language_loss": 0.88163298, "learning_rate": 3.993466194879335e-06, "loss": 0.90457785, "num_input_tokens_seen": 9039720, "router_z_loss_clip": 0.43164062, "router_z_loss_mlp": 1.75, "step": 430, "time_per_iteration": 2.5054399967193604 }, { "auxiliary_loss_clip": 0.01221007, "auxiliary_loss_mlp": 0.01064797, "balance_loss_clip": 1.02712727, "balance_loss_mlp": 1.05101562, "epoch": 0.025913121899894784, "flos": 20192261466240.0, "grad_norm": 2.0882154604930507, "language_loss": 0.83566093, "learning_rate": 3.993435644143989e-06, "loss": 0.85851896, "num_input_tokens_seen": 9059850, "router_z_loss_clip": 0.37695312, "router_z_loss_mlp": 1.703125, "step": 431, "time_per_iteration": 2.413207769393921 }, { "auxiliary_loss_clip": 0.01219247, "auxiliary_loss_mlp": 0.01062272, "balance_loss_clip": 1.02312338, "balance_loss_mlp": 1.04578114, "epoch": 0.025973245152562753, "flos": 14719556027520.0, "grad_norm": 4.11964093237474, "language_loss": 0.86177897, "learning_rate": 3.993405022268051e-06, "loss": 0.8845942, "num_input_tokens_seen": 9077590, "router_z_loss_clip": 0.390625, "router_z_loss_mlp": 1.734375, "step": 432, "time_per_iteration": 2.3806257247924805 }, { "auxiliary_loss_clip": 0.01222087, "auxiliary_loss_mlp": 0.01061348, "balance_loss_clip": 1.02436972, "balance_loss_mlp": 1.04628515, "epoch": 0.02603336840523072, "flos": 30991665571200.0, "grad_norm": 2.6666100728953968, "language_loss": 0.75972843, "learning_rate": 3.993374329252616e-06, "loss": 0.78256285, "num_input_tokens_seen": 9099880, "router_z_loss_clip": 0.37109375, "router_z_loss_mlp": 1.7578125, "step": 433, "time_per_iteration": 2.51416277885437 }, { "auxiliary_loss_clip": 0.01221008, "auxiliary_loss_mlp": 0.01080675, "balance_loss_clip": 1.0364722, "balance_loss_mlp": 1.04617822, "epoch": 0.026093491657898694, "flos": 17673347894400.0, "grad_norm": 1.959187456798102, "language_loss": 0.89468384, "learning_rate": 3.993343565098778e-06, "loss": 0.91770065, "num_input_tokens_seen": 9118620, "router_z_loss_clip": 0.44140625, "router_z_loss_mlp": 1.7421875, "step": 434, "time_per_iteration": 2.4000027179718018 }, { "auxiliary_loss_clip": 0.01221612, "auxiliary_loss_mlp": 0.01081696, "balance_loss_clip": 1.04037833, "balance_loss_mlp": 1.05102932, "epoch": 0.026153614910566662, "flos": 17856921156480.0, "grad_norm": 2.096397488032798, "language_loss": 0.79408079, "learning_rate": 3.993312729807637e-06, "loss": 0.81711388, "num_input_tokens_seen": 9135655, "router_z_loss_clip": 0.4140625, "router_z_loss_mlp": 1.703125, "step": 435, "time_per_iteration": 2.4256155490875244 }, { "auxiliary_loss_clip": 0.01217634, "auxiliary_loss_mlp": 0.01078944, "balance_loss_clip": 1.03753078, "balance_loss_mlp": 1.04609227, "epoch": 0.02621373816323463, "flos": 20010084658560.0, "grad_norm": 2.5845390045116687, "language_loss": 0.86184919, "learning_rate": 3.993281823380292e-06, "loss": 0.88481498, "num_input_tokens_seen": 9153520, "router_z_loss_clip": 0.4140625, "router_z_loss_mlp": 1.71875, "step": 436, "time_per_iteration": 2.4278886318206787 }, { "auxiliary_loss_clip": 0.012181, "auxiliary_loss_mlp": 0.01087453, "balance_loss_clip": 1.04573023, "balance_loss_mlp": 1.0470084, "epoch": 0.0262738614159026, "flos": 19280190441600.0, "grad_norm": 4.714988608425289, "language_loss": 0.74434, "learning_rate": 3.993250845817845e-06, "loss": 0.76739556, "num_input_tokens_seen": 9170750, "router_z_loss_clip": 0.41796875, "router_z_loss_mlp": 1.7109375, "step": 437, "time_per_iteration": 2.4319581985473633 }, { "auxiliary_loss_clip": 0.01215749, "auxiliary_loss_mlp": 0.01087014, "balance_loss_clip": 1.04405141, "balance_loss_mlp": 1.0473398, "epoch": 0.026333984668570568, "flos": 18806209557120.0, "grad_norm": 5.086534458318834, "language_loss": 0.91138661, "learning_rate": 3.9932197971214026e-06, "loss": 0.93441427, "num_input_tokens_seen": 9188430, "router_z_loss_clip": 0.4296875, "router_z_loss_mlp": 1.6875, "step": 438, "time_per_iteration": 2.4063069820404053 }, { "auxiliary_loss_clip": 0.0109988, "auxiliary_loss_mlp": 0.01032375, "balance_loss_clip": 1.02302861, "balance_loss_mlp": 1.02063978, "epoch": 0.02639410792123854, "flos": 64568403607680.0, "grad_norm": 0.8590789451679222, "language_loss": 0.62551713, "learning_rate": 3.9931886772920735e-06, "loss": 0.64683968, "num_input_tokens_seen": 9255835, "router_z_loss_clip": 0.09326172, "router_z_loss_mlp": 0.7890625, "step": 439, "time_per_iteration": 3.1644980907440186 }, { "auxiliary_loss_clip": 0.01225435, "auxiliary_loss_mlp": 0.01080469, "balance_loss_clip": 1.03738666, "balance_loss_mlp": 1.04984474, "epoch": 0.02645423117390651, "flos": 28472263240320.0, "grad_norm": 6.737809188874736, "language_loss": 0.75231874, "learning_rate": 3.993157486330967e-06, "loss": 0.77537781, "num_input_tokens_seen": 9276835, "router_z_loss_clip": 0.4296875, "router_z_loss_mlp": 1.7578125, "step": 440, "time_per_iteration": 2.4854986667633057 }, { "auxiliary_loss_clip": 0.01219877, "auxiliary_loss_mlp": 0.01067345, "balance_loss_clip": 1.02543068, "balance_loss_mlp": 1.04653168, "epoch": 0.026514354426574478, "flos": 18550261313280.0, "grad_norm": 2.6045460134394824, "language_loss": 0.82804596, "learning_rate": 3.993126224239198e-06, "loss": 0.85091817, "num_input_tokens_seen": 9295075, "router_z_loss_clip": 0.41796875, "router_z_loss_mlp": 1.734375, "step": 441, "time_per_iteration": 2.4888463020324707 }, { "auxiliary_loss_clip": 0.01220228, "auxiliary_loss_mlp": 0.01074593, "balance_loss_clip": 1.0329653, "balance_loss_mlp": 1.04525423, "epoch": 0.026574477679242446, "flos": 20666766021120.0, "grad_norm": 2.4595969775424327, "language_loss": 0.78507668, "learning_rate": 3.99309489101788e-06, "loss": 0.80802488, "num_input_tokens_seen": 9314205, "router_z_loss_clip": 0.41601562, "router_z_loss_mlp": 1.75, "step": 442, "time_per_iteration": 2.4291138648986816 }, { "auxiliary_loss_clip": 0.01090726, "auxiliary_loss_mlp": 0.01011347, "balance_loss_clip": 1.00285959, "balance_loss_mlp": 1.01481843, "epoch": 0.026634600931910415, "flos": 57953026414080.0, "grad_norm": 0.9492683728905594, "language_loss": 0.644611, "learning_rate": 3.993063486668132e-06, "loss": 0.66563171, "num_input_tokens_seen": 9367395, "router_z_loss_clip": 0.08496094, "router_z_loss_mlp": 0.7578125, "step": 443, "time_per_iteration": 2.9280450344085693 }, { "auxiliary_loss_clip": 0.01222362, "auxiliary_loss_mlp": 0.01079253, "balance_loss_clip": 1.03905571, "balance_loss_mlp": 1.05135286, "epoch": 0.026694724184578387, "flos": 15814222796160.0, "grad_norm": 2.0330872338587667, "language_loss": 0.82178068, "learning_rate": 3.993032011191076e-06, "loss": 0.8447969, "num_input_tokens_seen": 9385185, "router_z_loss_clip": 0.40234375, "router_z_loss_mlp": 1.7109375, "step": 444, "time_per_iteration": 2.4048736095428467 }, { "auxiliary_loss_clip": 0.01221715, "auxiliary_loss_mlp": 0.01070907, "balance_loss_clip": 1.02789617, "balance_loss_mlp": 1.04752612, "epoch": 0.026754847437246355, "flos": 23439149130240.0, "grad_norm": 2.2354540032417507, "language_loss": 0.95266509, "learning_rate": 3.993000464587833e-06, "loss": 0.9755913, "num_input_tokens_seen": 9403225, "router_z_loss_clip": 0.4296875, "router_z_loss_mlp": 1.7421875, "step": 445, "time_per_iteration": 2.434636354446411 }, { "auxiliary_loss_clip": 0.01221157, "auxiliary_loss_mlp": 0.01077582, "balance_loss_clip": 1.03552508, "balance_loss_mlp": 1.04775894, "epoch": 0.026814970689914324, "flos": 17341009862400.0, "grad_norm": 2.175289182417039, "language_loss": 0.91126347, "learning_rate": 3.9929688468595305e-06, "loss": 0.93425083, "num_input_tokens_seen": 9420540, "router_z_loss_clip": 0.421875, "router_z_loss_mlp": 1.734375, "step": 446, "time_per_iteration": 2.3854146003723145 }, { "auxiliary_loss_clip": 0.01220393, "auxiliary_loss_mlp": 0.01072297, "balance_loss_clip": 1.03069329, "balance_loss_mlp": 1.04886007, "epoch": 0.026875093942582293, "flos": 17893754507520.0, "grad_norm": 2.6742440873310374, "language_loss": 0.79533404, "learning_rate": 3.992937158007296e-06, "loss": 0.81826091, "num_input_tokens_seen": 9438840, "router_z_loss_clip": 0.41601562, "router_z_loss_mlp": 1.71875, "step": 447, "time_per_iteration": 2.436859130859375 }, { "auxiliary_loss_clip": 0.01217204, "auxiliary_loss_mlp": 0.01061085, "balance_loss_clip": 1.02317667, "balance_loss_mlp": 1.0459522, "epoch": 0.026935217195250265, "flos": 21722958604800.0, "grad_norm": 2.7414730375156515, "language_loss": 0.86134863, "learning_rate": 3.992905398032262e-06, "loss": 0.88413143, "num_input_tokens_seen": 9457215, "router_z_loss_clip": 0.37890625, "router_z_loss_mlp": 1.7109375, "step": 448, "time_per_iteration": 2.4112770557403564 }, { "auxiliary_loss_clip": 0.01212256, "auxiliary_loss_mlp": 0.01073785, "balance_loss_clip": 1.03573346, "balance_loss_mlp": 1.04552698, "epoch": 0.026995340447918233, "flos": 23621570317440.0, "grad_norm": 2.07756945998201, "language_loss": 0.88353348, "learning_rate": 3.992873566935559e-06, "loss": 0.90639389, "num_input_tokens_seen": 9475615, "router_z_loss_clip": 0.37890625, "router_z_loss_mlp": 1.671875, "step": 449, "time_per_iteration": 3.893214225769043 }, { "auxiliary_loss_clip": 0.01223196, "auxiliary_loss_mlp": 0.01071437, "balance_loss_clip": 1.02995205, "balance_loss_mlp": 1.04884255, "epoch": 0.027055463700586202, "flos": 17930308567680.0, "grad_norm": 2.1023873559554254, "language_loss": 0.80007172, "learning_rate": 3.992841664718326e-06, "loss": 0.82301807, "num_input_tokens_seen": 9493975, "router_z_loss_clip": 0.4140625, "router_z_loss_mlp": 1.7421875, "step": 450, "time_per_iteration": 2.4293415546417236 }, { "auxiliary_loss_clip": 0.01213427, "auxiliary_loss_mlp": 0.01068471, "balance_loss_clip": 1.02751112, "balance_loss_mlp": 1.04947925, "epoch": 0.02711558695325417, "flos": 25117738254720.0, "grad_norm": 1.6694600928474144, "language_loss": 0.81280768, "learning_rate": 3.9928096913817e-06, "loss": 0.83562666, "num_input_tokens_seen": 9514810, "router_z_loss_clip": 0.41015625, "router_z_loss_mlp": 1.640625, "step": 451, "time_per_iteration": 5.31993293762207 }, { "auxiliary_loss_clip": 0.01217688, "auxiliary_loss_mlp": 0.01082696, "balance_loss_clip": 1.03939891, "balance_loss_mlp": 1.04973805, "epoch": 0.02717571020592214, "flos": 24238520686080.0, "grad_norm": 1.8430085290678004, "language_loss": 0.76597822, "learning_rate": 3.992777646926822e-06, "loss": 0.78898203, "num_input_tokens_seen": 9533635, "router_z_loss_clip": 0.43359375, "router_z_loss_mlp": 1.6796875, "step": 452, "time_per_iteration": 2.4252092838287354 }, { "auxiliary_loss_clip": 0.01216012, "auxiliary_loss_mlp": 0.01073324, "balance_loss_clip": 1.03448617, "balance_loss_mlp": 1.04734445, "epoch": 0.02723583345859011, "flos": 25117773166080.0, "grad_norm": 1.8424704499023885, "language_loss": 0.72687912, "learning_rate": 3.992745531354836e-06, "loss": 0.74977249, "num_input_tokens_seen": 9555420, "router_z_loss_clip": 0.38867188, "router_z_loss_mlp": 1.6875, "step": 453, "time_per_iteration": 3.840744733810425 }, { "auxiliary_loss_clip": 0.01213416, "auxiliary_loss_mlp": 0.01077839, "balance_loss_clip": 1.03923905, "balance_loss_mlp": 1.04579771, "epoch": 0.02729595671125808, "flos": 42739939140480.0, "grad_norm": 1.9164462172076624, "language_loss": 0.81865823, "learning_rate": 3.992713344666888e-06, "loss": 0.84157073, "num_input_tokens_seen": 9578950, "router_z_loss_clip": 0.38671875, "router_z_loss_mlp": 1.671875, "step": 454, "time_per_iteration": 2.601329803466797 }, { "auxiliary_loss_clip": 0.01217129, "auxiliary_loss_mlp": 0.01059361, "balance_loss_clip": 1.02178609, "balance_loss_mlp": 1.04818177, "epoch": 0.02735607996392605, "flos": 21430002453120.0, "grad_norm": 1.8384435634960097, "language_loss": 0.75141943, "learning_rate": 3.992681086864125e-06, "loss": 0.77418435, "num_input_tokens_seen": 9598160, "router_z_loss_clip": 0.375, "router_z_loss_mlp": 1.6875, "step": 455, "time_per_iteration": 2.434659481048584 }, { "auxiliary_loss_clip": 0.01217381, "auxiliary_loss_mlp": 0.01073987, "balance_loss_clip": 1.03364635, "balance_loss_mlp": 1.0450505, "epoch": 0.027416203216594017, "flos": 20850199637760.0, "grad_norm": 3.601715071411152, "language_loss": 0.80229902, "learning_rate": 3.992648757947702e-06, "loss": 0.82521272, "num_input_tokens_seen": 9616010, "router_z_loss_clip": 0.40234375, "router_z_loss_mlp": 1.71875, "step": 456, "time_per_iteration": 2.421617031097412 }, { "auxiliary_loss_clip": 0.01210392, "auxiliary_loss_mlp": 0.0107454, "balance_loss_clip": 1.0342474, "balance_loss_mlp": 1.04569876, "epoch": 0.027476326469261986, "flos": 13223667381120.0, "grad_norm": 2.3636243778557464, "language_loss": 0.81195503, "learning_rate": 3.99261635791877e-06, "loss": 0.83480436, "num_input_tokens_seen": 9634000, "router_z_loss_clip": 0.40234375, "router_z_loss_mlp": 1.6484375, "step": 457, "time_per_iteration": 2.4353220462799072 }, { "auxiliary_loss_clip": 0.01215576, "auxiliary_loss_mlp": 0.01069389, "balance_loss_clip": 1.02940655, "balance_loss_mlp": 1.04417586, "epoch": 0.027536449721929958, "flos": 24023385688320.0, "grad_norm": 2.3286607514782713, "language_loss": 0.9358151, "learning_rate": 3.992583886778485e-06, "loss": 0.95866472, "num_input_tokens_seen": 9653455, "router_z_loss_clip": 0.3984375, "router_z_loss_mlp": 1.71875, "step": 458, "time_per_iteration": 2.451904058456421 }, { "auxiliary_loss_clip": 0.01211667, "auxiliary_loss_mlp": 0.01075366, "balance_loss_clip": 1.03721941, "balance_loss_mlp": 1.04523611, "epoch": 0.027596572974597926, "flos": 13005215804160.0, "grad_norm": 2.2284725754265655, "language_loss": 0.78291839, "learning_rate": 3.9925513445280075e-06, "loss": 0.80578876, "num_input_tokens_seen": 9669650, "router_z_loss_clip": 0.3828125, "router_z_loss_mlp": 1.6640625, "step": 459, "time_per_iteration": 2.4427225589752197 }, { "auxiliary_loss_clip": 0.0121507, "auxiliary_loss_mlp": 0.01068839, "balance_loss_clip": 1.02799821, "balance_loss_mlp": 1.0481329, "epoch": 0.027656696227265895, "flos": 26141810520960.0, "grad_norm": 1.873779291517176, "language_loss": 0.8316347, "learning_rate": 3.9925187311684975e-06, "loss": 0.85447371, "num_input_tokens_seen": 9691415, "router_z_loss_clip": 0.40820312, "router_z_loss_mlp": 1.671875, "step": 460, "time_per_iteration": 2.481241226196289 }, { "auxiliary_loss_clip": 0.01087725, "auxiliary_loss_mlp": 0.01037586, "balance_loss_clip": 1.02890778, "balance_loss_mlp": 1.01300073, "epoch": 0.027716819479933864, "flos": 60693917610240.0, "grad_norm": 1.5945212505311077, "language_loss": 0.73599243, "learning_rate": 3.9924860467011195e-06, "loss": 0.75724554, "num_input_tokens_seen": 9755605, "router_z_loss_clip": 0.08691406, "router_z_loss_mlp": 0.75, "step": 461, "time_per_iteration": 3.049412488937378 }, { "auxiliary_loss_clip": 0.01213652, "auxiliary_loss_mlp": 0.01069088, "balance_loss_clip": 1.03408813, "balance_loss_mlp": 1.05053163, "epoch": 0.027776942732601832, "flos": 31210605907200.0, "grad_norm": 2.472476215251796, "language_loss": 0.8088612, "learning_rate": 3.99245329112704e-06, "loss": 0.83168852, "num_input_tokens_seen": 9776270, "router_z_loss_clip": 0.34960938, "router_z_loss_mlp": 1.6328125, "step": 462, "time_per_iteration": 2.4856672286987305 }, { "auxiliary_loss_clip": 0.01214272, "auxiliary_loss_mlp": 0.01065799, "balance_loss_clip": 1.02932096, "balance_loss_mlp": 1.0486722, "epoch": 0.027837065985269804, "flos": 22673538725760.0, "grad_norm": 2.2505299356194177, "language_loss": 0.89811778, "learning_rate": 3.992420464447427e-06, "loss": 0.92091846, "num_input_tokens_seen": 9794465, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.65625, "step": 463, "time_per_iteration": 2.449388265609741 }, { "auxiliary_loss_clip": 0.01083923, "auxiliary_loss_mlp": 0.01013799, "balance_loss_clip": 1.00612223, "balance_loss_mlp": 1.01178098, "epoch": 0.027897189237937773, "flos": 62179437582720.0, "grad_norm": 0.8732072034279693, "language_loss": 0.5900414, "learning_rate": 3.992387566663454e-06, "loss": 0.6110186, "num_input_tokens_seen": 9849685, "router_z_loss_clip": 0.07666016, "router_z_loss_mlp": 0.72265625, "step": 464, "time_per_iteration": 3.076657295227051 }, { "auxiliary_loss_clip": 0.01224219, "auxiliary_loss_mlp": 0.01071273, "balance_loss_clip": 1.02985942, "balance_loss_mlp": 1.05110717, "epoch": 0.02795731249060574, "flos": 24492164780160.0, "grad_norm": 2.8476587695432993, "language_loss": 0.80872023, "learning_rate": 3.992354597776293e-06, "loss": 0.83167517, "num_input_tokens_seen": 9869505, "router_z_loss_clip": 0.4140625, "router_z_loss_mlp": 1.734375, "step": 465, "time_per_iteration": 2.460120916366577 }, { "auxiliary_loss_clip": 0.01211628, "auxiliary_loss_mlp": 0.01077742, "balance_loss_clip": 1.03806865, "balance_loss_mlp": 1.04559112, "epoch": 0.02801743574327371, "flos": 23731860902400.0, "grad_norm": 2.0807330331238814, "language_loss": 0.78305185, "learning_rate": 3.992321557787121e-06, "loss": 0.80594552, "num_input_tokens_seen": 9890950, "router_z_loss_clip": 0.39648438, "router_z_loss_mlp": 1.65625, "step": 466, "time_per_iteration": 2.440420627593994 }, { "auxiliary_loss_clip": 0.01210946, "auxiliary_loss_mlp": 0.01068143, "balance_loss_clip": 1.02928114, "balance_loss_mlp": 1.04549003, "epoch": 0.02807755899594168, "flos": 20628117279360.0, "grad_norm": 1.807040788688562, "language_loss": 0.87426627, "learning_rate": 3.992288446697118e-06, "loss": 0.89705718, "num_input_tokens_seen": 9911265, "router_z_loss_clip": 0.38867188, "router_z_loss_mlp": 1.65625, "step": 467, "time_per_iteration": 2.429304361343384 }, { "auxiliary_loss_clip": 0.01218809, "auxiliary_loss_mlp": 0.01091612, "balance_loss_clip": 1.05005586, "balance_loss_mlp": 1.04589593, "epoch": 0.02813768224860965, "flos": 19243566558720.0, "grad_norm": 2.1615589645067237, "language_loss": 0.86052179, "learning_rate": 3.9922552645074644e-06, "loss": 0.88362604, "num_input_tokens_seen": 9929025, "router_z_loss_clip": 0.41601562, "router_z_loss_mlp": 1.7265625, "step": 468, "time_per_iteration": 2.4182615280151367 }, { "auxiliary_loss_clip": 0.01216098, "auxiliary_loss_mlp": 0.01075451, "balance_loss_clip": 1.03553975, "balance_loss_mlp": 1.04683185, "epoch": 0.02819780550127762, "flos": 20812912439040.0, "grad_norm": 2.4673660674462172, "language_loss": 0.91542101, "learning_rate": 3.992222011219346e-06, "loss": 0.93833661, "num_input_tokens_seen": 9945190, "router_z_loss_clip": 0.3984375, "router_z_loss_mlp": 1.6953125, "step": 469, "time_per_iteration": 2.420708179473877 }, { "auxiliary_loss_clip": 0.01208904, "auxiliary_loss_mlp": 0.01087393, "balance_loss_clip": 1.0482924, "balance_loss_mlp": 1.04541838, "epoch": 0.028257928753945588, "flos": 19973111662080.0, "grad_norm": 3.359520458405969, "language_loss": 0.80823982, "learning_rate": 3.992188686833948e-06, "loss": 0.83120275, "num_input_tokens_seen": 9962820, "router_z_loss_clip": 0.390625, "router_z_loss_mlp": 1.6328125, "step": 470, "time_per_iteration": 2.426966428756714 }, { "auxiliary_loss_clip": 0.01214693, "auxiliary_loss_mlp": 0.01074015, "balance_loss_clip": 1.0333643, "balance_loss_mlp": 1.0463984, "epoch": 0.028318052006613557, "flos": 20483472049920.0, "grad_norm": 2.0266611816436004, "language_loss": 0.92974067, "learning_rate": 3.992155291352461e-06, "loss": 0.95262778, "num_input_tokens_seen": 9982595, "router_z_loss_clip": 0.40625, "router_z_loss_mlp": 1.6796875, "step": 471, "time_per_iteration": 2.417511224746704 }, { "auxiliary_loss_clip": 0.01211363, "auxiliary_loss_mlp": 0.01070777, "balance_loss_clip": 1.03117526, "balance_loss_mlp": 1.0441376, "epoch": 0.02837817525928153, "flos": 28513495422720.0, "grad_norm": 2.0100640893231168, "language_loss": 0.76147437, "learning_rate": 3.992121824776075e-06, "loss": 0.78429568, "num_input_tokens_seen": 10004645, "router_z_loss_clip": 0.39648438, "router_z_loss_mlp": 1.671875, "step": 472, "time_per_iteration": 2.49124813079834 }, { "auxiliary_loss_clip": 0.01216394, "auxiliary_loss_mlp": 0.01074437, "balance_loss_clip": 1.03524113, "balance_loss_mlp": 1.04516983, "epoch": 0.028438298511949497, "flos": 18550680249600.0, "grad_norm": 2.9233453117850345, "language_loss": 0.9328692, "learning_rate": 3.9920882871059865e-06, "loss": 0.95577747, "num_input_tokens_seen": 10022555, "router_z_loss_clip": 0.39257812, "router_z_loss_mlp": 1.7109375, "step": 473, "time_per_iteration": 2.430454730987549 }, { "auxiliary_loss_clip": 0.01219842, "auxiliary_loss_mlp": 0.01089857, "balance_loss_clip": 1.05104256, "balance_loss_mlp": 1.04805279, "epoch": 0.028498421764617466, "flos": 16909273589760.0, "grad_norm": 3.6126159175986055, "language_loss": 0.88592136, "learning_rate": 3.992054678343391e-06, "loss": 0.90901834, "num_input_tokens_seen": 10041025, "router_z_loss_clip": 0.38867188, "router_z_loss_mlp": 1.71875, "step": 474, "time_per_iteration": 2.3969032764434814 }, { "auxiliary_loss_clip": 0.0121063, "auxiliary_loss_mlp": 0.01080179, "balance_loss_clip": 1.03936172, "balance_loss_mlp": 1.04794836, "epoch": 0.028558545017285435, "flos": 27777561540480.0, "grad_norm": 2.2069611979958164, "language_loss": 0.78739357, "learning_rate": 3.992020998489488e-06, "loss": 0.81030166, "num_input_tokens_seen": 10060775, "router_z_loss_clip": 0.40820312, "router_z_loss_mlp": 1.625, "step": 475, "time_per_iteration": 2.4957873821258545 }, { "auxiliary_loss_clip": 0.01078655, "auxiliary_loss_mlp": 0.0105164, "balance_loss_clip": 1.04286611, "balance_loss_mlp": 1.01238871, "epoch": 0.028618668269953403, "flos": 65651060868480.0, "grad_norm": 0.9168251895118754, "language_loss": 0.66889834, "learning_rate": 3.991987247545479e-06, "loss": 0.69020128, "num_input_tokens_seen": 10120225, "router_z_loss_clip": 0.08789062, "router_z_loss_mlp": 0.6640625, "step": 476, "time_per_iteration": 2.975771188735962 }, { "auxiliary_loss_clip": 0.01212109, "auxiliary_loss_mlp": 0.01079319, "balance_loss_clip": 1.03947902, "balance_loss_mlp": 1.04676247, "epoch": 0.028678791522621375, "flos": 21936208389120.0, "grad_norm": 2.333401231724457, "language_loss": 0.83673292, "learning_rate": 3.99195342551257e-06, "loss": 0.85964721, "num_input_tokens_seen": 10137880, "router_z_loss_clip": 0.3984375, "router_z_loss_mlp": 1.65625, "step": 477, "time_per_iteration": 2.425384521484375 }, { "auxiliary_loss_clip": 0.01219363, "auxiliary_loss_mlp": 0.01076292, "balance_loss_clip": 1.03509319, "balance_loss_mlp": 1.04858422, "epoch": 0.028738914775289344, "flos": 24570963452160.0, "grad_norm": 2.376657919351714, "language_loss": 0.81632209, "learning_rate": 3.991919532391967e-06, "loss": 0.8392787, "num_input_tokens_seen": 10156930, "router_z_loss_clip": 0.41210938, "router_z_loss_mlp": 1.703125, "step": 478, "time_per_iteration": 2.463796854019165 }, { "auxiliary_loss_clip": 0.01212562, "auxiliary_loss_mlp": 0.01074185, "balance_loss_clip": 1.03467917, "balance_loss_mlp": 1.04589367, "epoch": 0.028799038027957313, "flos": 23256867588480.0, "grad_norm": 1.98261751435751, "language_loss": 0.8049897, "learning_rate": 3.991885568184879e-06, "loss": 0.82785714, "num_input_tokens_seen": 10176295, "router_z_loss_clip": 0.39453125, "router_z_loss_mlp": 1.6640625, "step": 479, "time_per_iteration": 2.42974591255188 }, { "auxiliary_loss_clip": 0.01211035, "auxiliary_loss_mlp": 0.01074508, "balance_loss_clip": 1.03156877, "balance_loss_mlp": 1.04642069, "epoch": 0.02885916128062528, "flos": 22163003781120.0, "grad_norm": 2.75380698709829, "language_loss": 0.7387349, "learning_rate": 3.991851532892521e-06, "loss": 0.76159036, "num_input_tokens_seen": 10195790, "router_z_loss_clip": 0.4296875, "router_z_loss_mlp": 1.6484375, "step": 480, "time_per_iteration": 2.4637868404388428 }, { "auxiliary_loss_clip": 0.01211482, "auxiliary_loss_mlp": 0.01066037, "balance_loss_clip": 1.03067946, "balance_loss_mlp": 1.04742777, "epoch": 0.02891928453329325, "flos": 22931651473920.0, "grad_norm": 1.727787042430347, "language_loss": 0.8761667, "learning_rate": 3.991817426516103e-06, "loss": 0.89894187, "num_input_tokens_seen": 10218405, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.640625, "step": 481, "time_per_iteration": 2.4658827781677246 }, { "auxiliary_loss_clip": 0.01075504, "auxiliary_loss_mlp": 0.01015323, "balance_loss_clip": 1.00783658, "balance_loss_mlp": 1.01079535, "epoch": 0.028979407785961222, "flos": 57430202670720.0, "grad_norm": 0.949522922299035, "language_loss": 0.66014594, "learning_rate": 3.991783249056846e-06, "loss": 0.68105423, "num_input_tokens_seen": 10271005, "router_z_loss_clip": 0.07470703, "router_z_loss_mlp": 0.6484375, "step": 482, "time_per_iteration": 2.7990095615386963 }, { "auxiliary_loss_clip": 0.01219808, "auxiliary_loss_mlp": 0.01078013, "balance_loss_clip": 1.03855503, "balance_loss_mlp": 1.04768646, "epoch": 0.02903953103862919, "flos": 16721929900800.0, "grad_norm": 2.632323154501168, "language_loss": 0.78217971, "learning_rate": 3.991749000515968e-06, "loss": 0.80515796, "num_input_tokens_seen": 10288405, "router_z_loss_clip": 0.39453125, "router_z_loss_mlp": 1.71875, "step": 483, "time_per_iteration": 2.390429973602295 }, { "auxiliary_loss_clip": 0.01213693, "auxiliary_loss_mlp": 0.01084099, "balance_loss_clip": 1.04392576, "balance_loss_mlp": 1.04758871, "epoch": 0.02909965429129716, "flos": 16762708235520.0, "grad_norm": 2.6496364320357797, "language_loss": 0.74926507, "learning_rate": 3.991714680894691e-06, "loss": 0.77224296, "num_input_tokens_seen": 10306875, "router_z_loss_clip": 0.40234375, "router_z_loss_mlp": 1.65625, "step": 484, "time_per_iteration": 2.4088382720947266 }, { "auxiliary_loss_clip": 0.0121332, "auxiliary_loss_mlp": 0.0107373, "balance_loss_clip": 1.03410459, "balance_loss_mlp": 1.04610586, "epoch": 0.029159777543965128, "flos": 19784511164160.0, "grad_norm": 2.0936797742723923, "language_loss": 0.83411169, "learning_rate": 3.991680290194241e-06, "loss": 0.85698223, "num_input_tokens_seen": 10323965, "router_z_loss_clip": 0.39648438, "router_z_loss_mlp": 1.671875, "step": 485, "time_per_iteration": 2.4037091732025146 }, { "auxiliary_loss_clip": 0.01216594, "auxiliary_loss_mlp": 0.01070947, "balance_loss_clip": 1.03299022, "balance_loss_mlp": 1.05073392, "epoch": 0.029219900796633096, "flos": 19641751148160.0, "grad_norm": 1.8682562352937333, "language_loss": 0.83862162, "learning_rate": 3.991645828415844e-06, "loss": 0.86149704, "num_input_tokens_seen": 10342620, "router_z_loss_clip": 0.37890625, "router_z_loss_mlp": 1.65625, "step": 486, "time_per_iteration": 2.408877372741699 }, { "auxiliary_loss_clip": 0.01212274, "auxiliary_loss_mlp": 0.01086459, "balance_loss_clip": 1.04642892, "balance_loss_mlp": 1.04686427, "epoch": 0.02928002404930107, "flos": 25884500734080.0, "grad_norm": 2.1830433578473407, "language_loss": 0.88530236, "learning_rate": 3.991611295560732e-06, "loss": 0.90828967, "num_input_tokens_seen": 10364610, "router_z_loss_clip": 0.40039062, "router_z_loss_mlp": 1.65625, "step": 487, "time_per_iteration": 2.484448194503784 }, { "auxiliary_loss_clip": 0.01215214, "auxiliary_loss_mlp": 0.01064765, "balance_loss_clip": 1.02649832, "balance_loss_mlp": 1.04820597, "epoch": 0.029340147301969037, "flos": 20659399724160.0, "grad_norm": 4.5142003549058325, "language_loss": 0.87968355, "learning_rate": 3.991576691630134e-06, "loss": 0.90248334, "num_input_tokens_seen": 10380910, "router_z_loss_clip": 0.3828125, "router_z_loss_mlp": 1.671875, "step": 488, "time_per_iteration": 2.423797607421875 }, { "auxiliary_loss_clip": 0.01209508, "auxiliary_loss_mlp": 0.01070475, "balance_loss_clip": 1.03206539, "balance_loss_mlp": 1.04734719, "epoch": 0.029400270554637006, "flos": 24426806981760.0, "grad_norm": 7.590058532803281, "language_loss": 0.88534021, "learning_rate": 3.991542016625289e-06, "loss": 0.90814012, "num_input_tokens_seen": 10400665, "router_z_loss_clip": 0.38476562, "router_z_loss_mlp": 1.625, "step": 489, "time_per_iteration": 3.889488458633423 }, { "auxiliary_loss_clip": 0.01204857, "auxiliary_loss_mlp": 0.01073573, "balance_loss_clip": 1.03483009, "balance_loss_mlp": 1.04278088, "epoch": 0.029460393807304974, "flos": 20119851573120.0, "grad_norm": 1.9148623633063457, "language_loss": 0.88380492, "learning_rate": 3.99150727054743e-06, "loss": 0.90658921, "num_input_tokens_seen": 10420150, "router_z_loss_clip": 0.38671875, "router_z_loss_mlp": 1.625, "step": 490, "time_per_iteration": 5.217238187789917 }, { "auxiliary_loss_clip": 0.012128, "auxiliary_loss_mlp": 0.01068954, "balance_loss_clip": 1.03185582, "balance_loss_mlp": 1.04768872, "epoch": 0.029520517059972943, "flos": 17674953816960.0, "grad_norm": 3.2853586318233647, "language_loss": 0.91206759, "learning_rate": 3.9914724533978e-06, "loss": 0.93488508, "num_input_tokens_seen": 10438210, "router_z_loss_clip": 0.37109375, "router_z_loss_mlp": 1.6484375, "step": 491, "time_per_iteration": 2.4247934818267822 }, { "auxiliary_loss_clip": 0.0120445, "auxiliary_loss_mlp": 0.01066569, "balance_loss_clip": 1.03044844, "balance_loss_mlp": 1.04490709, "epoch": 0.029580640312640915, "flos": 18952181418240.0, "grad_norm": 2.3581296560745586, "language_loss": 0.85065138, "learning_rate": 3.991437565177642e-06, "loss": 0.87336159, "num_input_tokens_seen": 10455125, "router_z_loss_clip": 0.36132812, "router_z_loss_mlp": 1.59375, "step": 492, "time_per_iteration": 3.793816089630127 }, { "auxiliary_loss_clip": 0.0121162, "auxiliary_loss_mlp": 0.01078789, "balance_loss_clip": 1.03990269, "balance_loss_mlp": 1.047333, "epoch": 0.029640763565308884, "flos": 18725351114880.0, "grad_norm": 4.274363010344045, "language_loss": 0.83796686, "learning_rate": 3.991402605888198e-06, "loss": 0.8608709, "num_input_tokens_seen": 10470990, "router_z_loss_clip": 0.38867188, "router_z_loss_mlp": 1.640625, "step": 493, "time_per_iteration": 2.3795204162597656 }, { "auxiliary_loss_clip": 0.01205968, "auxiliary_loss_mlp": 0.0106367, "balance_loss_clip": 1.02585661, "balance_loss_mlp": 1.04183817, "epoch": 0.029700886817976852, "flos": 20594251393920.0, "grad_norm": 1.7770761529375936, "language_loss": 0.86436814, "learning_rate": 3.991367575530719e-06, "loss": 0.88706452, "num_input_tokens_seen": 10490685, "router_z_loss_clip": 0.37890625, "router_z_loss_mlp": 1.640625, "step": 494, "time_per_iteration": 2.4433693885803223 }, { "auxiliary_loss_clip": 0.01209465, "auxiliary_loss_mlp": 0.01067738, "balance_loss_clip": 1.03328609, "balance_loss_mlp": 1.04554904, "epoch": 0.02976101007064482, "flos": 22235762787840.0, "grad_norm": 2.355404434060518, "language_loss": 0.86683035, "learning_rate": 3.9913324741064535e-06, "loss": 0.88960236, "num_input_tokens_seen": 10509435, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.6328125, "step": 495, "time_per_iteration": 2.4427053928375244 }, { "auxiliary_loss_clip": 0.01204284, "auxiliary_loss_mlp": 0.01065663, "balance_loss_clip": 1.0279218, "balance_loss_mlp": 1.04706717, "epoch": 0.029821133323312793, "flos": 23731511788800.0, "grad_norm": 1.962976155669888, "language_loss": 0.61746514, "learning_rate": 3.991297301616653e-06, "loss": 0.64016461, "num_input_tokens_seen": 10530050, "router_z_loss_clip": 0.37695312, "router_z_loss_mlp": 1.5703125, "step": 496, "time_per_iteration": 2.4484548568725586 }, { "auxiliary_loss_clip": 0.01204454, "auxiliary_loss_mlp": 0.01066235, "balance_loss_clip": 1.03109229, "balance_loss_mlp": 1.04732299, "epoch": 0.02988125657598076, "flos": 22418393443200.0, "grad_norm": 1.729398170481444, "language_loss": 0.8813799, "learning_rate": 3.991262058062575e-06, "loss": 0.90408683, "num_input_tokens_seen": 10551370, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.5703125, "step": 497, "time_per_iteration": 2.5011401176452637 }, { "auxiliary_loss_clip": 0.0120954, "auxiliary_loss_mlp": 0.01069592, "balance_loss_clip": 1.03247035, "balance_loss_mlp": 1.04552197, "epoch": 0.02994137982864873, "flos": 13844248531200.0, "grad_norm": 2.6030676199930602, "language_loss": 0.84617573, "learning_rate": 3.991226743445477e-06, "loss": 0.86896706, "num_input_tokens_seen": 10569225, "router_z_loss_clip": 0.37109375, "router_z_loss_mlp": 1.640625, "step": 498, "time_per_iteration": 2.522249221801758 }, { "auxiliary_loss_clip": 0.01206957, "auxiliary_loss_mlp": 0.01070789, "balance_loss_clip": 1.03540766, "balance_loss_mlp": 1.04600763, "epoch": 0.0300015030813167, "flos": 23907404551680.0, "grad_norm": 6.119065223903574, "language_loss": 0.78805482, "learning_rate": 3.991191357766617e-06, "loss": 0.81083238, "num_input_tokens_seen": 10586170, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.609375, "step": 499, "time_per_iteration": 2.5096781253814697 }, { "auxiliary_loss_clip": 0.01208767, "auxiliary_loss_mlp": 0.01069197, "balance_loss_clip": 1.03391087, "balance_loss_mlp": 1.04836917, "epoch": 0.030061626333984667, "flos": 22015740199680.0, "grad_norm": 1.90305937173952, "language_loss": 0.82357585, "learning_rate": 3.991155901027261e-06, "loss": 0.84635556, "num_input_tokens_seen": 10606205, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.6015625, "step": 500, "time_per_iteration": 2.4628171920776367 }, { "auxiliary_loss_clip": 0.01200767, "auxiliary_loss_mlp": 0.01073662, "balance_loss_clip": 1.0358727, "balance_loss_mlp": 1.04500973, "epoch": 0.03012174958665264, "flos": 23038625479680.0, "grad_norm": 2.535327279683379, "language_loss": 0.8793115, "learning_rate": 3.991120373228672e-06, "loss": 0.90205586, "num_input_tokens_seen": 10625995, "router_z_loss_clip": 0.37890625, "router_z_loss_mlp": 1.5546875, "step": 501, "time_per_iteration": 2.4772112369537354 }, { "auxiliary_loss_clip": 0.01207747, "auxiliary_loss_mlp": 0.01063189, "balance_loss_clip": 1.0281651, "balance_loss_mlp": 1.04432535, "epoch": 0.030181872839320608, "flos": 18952251240960.0, "grad_norm": 2.510197501519828, "language_loss": 0.86130059, "learning_rate": 3.991084774372118e-06, "loss": 0.88400996, "num_input_tokens_seen": 10644105, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.6328125, "step": 502, "time_per_iteration": 2.4320156574249268 }, { "auxiliary_loss_clip": 0.01205823, "auxiliary_loss_mlp": 0.01068997, "balance_loss_clip": 1.0321852, "balance_loss_mlp": 1.04956007, "epoch": 0.030241996091988577, "flos": 16727061870720.0, "grad_norm": 2.3152219203109867, "language_loss": 0.8469739, "learning_rate": 3.991049104458871e-06, "loss": 0.86972207, "num_input_tokens_seen": 10661090, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.5625, "step": 503, "time_per_iteration": 2.4763457775115967 }, { "auxiliary_loss_clip": 0.01199079, "auxiliary_loss_mlp": 0.01069118, "balance_loss_clip": 1.03326011, "balance_loss_mlp": 1.04358077, "epoch": 0.030302119344656545, "flos": 28620015580800.0, "grad_norm": 2.3787165165537334, "language_loss": 0.88057989, "learning_rate": 3.991013363490202e-06, "loss": 0.90326184, "num_input_tokens_seen": 10682380, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.5546875, "step": 504, "time_per_iteration": 2.498375654220581 }, { "auxiliary_loss_clip": 0.01201017, "auxiliary_loss_mlp": 0.01058491, "balance_loss_clip": 1.02403963, "balance_loss_mlp": 1.04350054, "epoch": 0.030362242597324514, "flos": 15668425491840.0, "grad_norm": 2.3046057205140835, "language_loss": 0.77504301, "learning_rate": 3.9909775514673885e-06, "loss": 0.79763812, "num_input_tokens_seen": 10699925, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.578125, "step": 505, "time_per_iteration": 2.5052103996276855 }, { "auxiliary_loss_clip": 0.01077902, "auxiliary_loss_mlp": 0.01015402, "balance_loss_clip": 1.0084877, "balance_loss_mlp": 1.01223004, "epoch": 0.030422365849992486, "flos": 72122107034880.0, "grad_norm": 0.841552000589698, "language_loss": 0.54989272, "learning_rate": 3.990941668391708e-06, "loss": 0.57082576, "num_input_tokens_seen": 10766525, "router_z_loss_clip": 0.06933594, "router_z_loss_mlp": 0.65625, "step": 506, "time_per_iteration": 3.3168389797210693 }, { "auxiliary_loss_clip": 0.01204627, "auxiliary_loss_mlp": 0.01071427, "balance_loss_clip": 1.03626084, "balance_loss_mlp": 1.04464555, "epoch": 0.030482489102660455, "flos": 19426790707200.0, "grad_norm": 2.1395024000496523, "language_loss": 0.83131456, "learning_rate": 3.99090571426444e-06, "loss": 0.85407519, "num_input_tokens_seen": 10786725, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.6015625, "step": 507, "time_per_iteration": 2.4338178634643555 }, { "auxiliary_loss_clip": 0.01205381, "auxiliary_loss_mlp": 0.01067995, "balance_loss_clip": 1.03156519, "balance_loss_mlp": 1.04393148, "epoch": 0.030542612355328423, "flos": 20374787387520.0, "grad_norm": 2.278497143714966, "language_loss": 0.87978184, "learning_rate": 3.990869689086868e-06, "loss": 0.90251565, "num_input_tokens_seen": 10805390, "router_z_loss_clip": 0.36523438, "router_z_loss_mlp": 1.609375, "step": 508, "time_per_iteration": 2.524904489517212 }, { "auxiliary_loss_clip": 0.01206445, "auxiliary_loss_mlp": 0.01057358, "balance_loss_clip": 1.02202475, "balance_loss_mlp": 1.04496431, "epoch": 0.030602735607996392, "flos": 34675945148160.0, "grad_norm": 1.9490258310118795, "language_loss": 0.71126789, "learning_rate": 3.990833592860279e-06, "loss": 0.73390591, "num_input_tokens_seen": 10828030, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.6171875, "step": 509, "time_per_iteration": 2.56103777885437 }, { "auxiliary_loss_clip": 0.01200486, "auxiliary_loss_mlp": 0.01061738, "balance_loss_clip": 1.02790678, "balance_loss_mlp": 1.04403806, "epoch": 0.03066285886066436, "flos": 23657565795840.0, "grad_norm": 2.0781282707932016, "language_loss": 0.81924725, "learning_rate": 3.990797425585959e-06, "loss": 0.84186947, "num_input_tokens_seen": 10845240, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.5625, "step": 510, "time_per_iteration": 2.517728805541992 }, { "auxiliary_loss_clip": 0.01205482, "auxiliary_loss_mlp": 0.01062984, "balance_loss_clip": 1.02786517, "balance_loss_mlp": 1.04849553, "epoch": 0.030722982113332332, "flos": 23001861951360.0, "grad_norm": 2.2066502130873005, "language_loss": 0.83270842, "learning_rate": 3.9907611872652e-06, "loss": 0.85539317, "num_input_tokens_seen": 10864325, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.5703125, "step": 511, "time_per_iteration": 2.4717540740966797 }, { "auxiliary_loss_clip": 0.01203057, "auxiliary_loss_mlp": 0.01067064, "balance_loss_clip": 1.03380466, "balance_loss_mlp": 1.04298186, "epoch": 0.0307831053660003, "flos": 24749788769280.0, "grad_norm": 2.1679052234875598, "language_loss": 0.817909, "learning_rate": 3.990724877899296e-06, "loss": 0.84061021, "num_input_tokens_seen": 10883860, "router_z_loss_clip": 0.33203125, "router_z_loss_mlp": 1.6015625, "step": 512, "time_per_iteration": 2.5127251148223877 }, { "auxiliary_loss_clip": 0.01199515, "auxiliary_loss_mlp": 0.01066431, "balance_loss_clip": 1.02985787, "balance_loss_mlp": 1.04325438, "epoch": 0.03084322861866827, "flos": 26139680928000.0, "grad_norm": 1.8918276302591983, "language_loss": 0.86687189, "learning_rate": 3.990688497489541e-06, "loss": 0.88953137, "num_input_tokens_seen": 10904555, "router_z_loss_clip": 0.36523438, "router_z_loss_mlp": 1.5625, "step": 513, "time_per_iteration": 2.4929020404815674 }, { "auxiliary_loss_clip": 0.01207001, "auxiliary_loss_mlp": 0.0106846, "balance_loss_clip": 1.03360355, "balance_loss_mlp": 1.04672039, "epoch": 0.03090335187133624, "flos": 18770283901440.0, "grad_norm": 1.5700776694139793, "language_loss": 0.78757954, "learning_rate": 3.990652046037234e-06, "loss": 0.81033409, "num_input_tokens_seen": 10923700, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.609375, "step": 514, "time_per_iteration": 2.515777111053467 }, { "auxiliary_loss_clip": 0.01199241, "auxiliary_loss_mlp": 0.0106353, "balance_loss_clip": 1.0313673, "balance_loss_mlp": 1.04487944, "epoch": 0.030963475124004207, "flos": 23220767376000.0, "grad_norm": 3.2378638813729714, "language_loss": 0.76955855, "learning_rate": 3.990615523543677e-06, "loss": 0.79218626, "num_input_tokens_seen": 10942730, "router_z_loss_clip": 0.32226562, "router_z_loss_mlp": 1.546875, "step": 515, "time_per_iteration": 2.475161075592041 }, { "auxiliary_loss_clip": 0.01198662, "auxiliary_loss_mlp": 0.01059837, "balance_loss_clip": 1.02440774, "balance_loss_mlp": 1.0403347, "epoch": 0.03102359837667218, "flos": 42523861536000.0, "grad_norm": 3.4578561136515913, "language_loss": 0.82421023, "learning_rate": 3.990578930010171e-06, "loss": 0.8467952, "num_input_tokens_seen": 10967120, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.5859375, "step": 516, "time_per_iteration": 2.6455483436584473 }, { "auxiliary_loss_clip": 0.01197751, "auxiliary_loss_mlp": 0.01061516, "balance_loss_clip": 1.0260396, "balance_loss_mlp": 1.04530215, "epoch": 0.031083721629340148, "flos": 21175939422720.0, "grad_norm": 1.725880235288346, "language_loss": 0.78557986, "learning_rate": 3.990542265438024e-06, "loss": 0.80817252, "num_input_tokens_seen": 10986775, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.5234375, "step": 517, "time_per_iteration": 2.499014139175415 }, { "auxiliary_loss_clip": 0.01194293, "auxiliary_loss_mlp": 0.01056726, "balance_loss_clip": 1.02375305, "balance_loss_mlp": 1.04368186, "epoch": 0.031143844882008116, "flos": 29714891817600.0, "grad_norm": 1.5230128811737134, "language_loss": 0.9046182, "learning_rate": 3.990505529828544e-06, "loss": 0.92712843, "num_input_tokens_seen": 11011360, "router_z_loss_clip": 0.33007812, "router_z_loss_mlp": 1.5078125, "step": 518, "time_per_iteration": 2.533905029296875 }, { "auxiliary_loss_clip": 0.01208601, "auxiliary_loss_mlp": 0.01072366, "balance_loss_clip": 1.03586388, "balance_loss_mlp": 1.04859257, "epoch": 0.031203968134676085, "flos": 23111349575040.0, "grad_norm": 3.0270756451628125, "language_loss": 0.86141729, "learning_rate": 3.9904687231830424e-06, "loss": 0.88422704, "num_input_tokens_seen": 11030150, "router_z_loss_clip": 0.36523438, "router_z_loss_mlp": 1.6015625, "step": 519, "time_per_iteration": 2.497805118560791 }, { "auxiliary_loss_clip": 0.01199843, "auxiliary_loss_mlp": 0.01066668, "balance_loss_clip": 1.03123891, "balance_loss_mlp": 1.04227221, "epoch": 0.03126409138734405, "flos": 20953473039360.0, "grad_norm": 2.5245551099703847, "language_loss": 0.86705911, "learning_rate": 3.990431845502831e-06, "loss": 0.88972425, "num_input_tokens_seen": 11049145, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.578125, "step": 520, "time_per_iteration": 2.4443423748016357 }, { "auxiliary_loss_clip": 0.01202682, "auxiliary_loss_mlp": 0.01074112, "balance_loss_clip": 1.03861165, "balance_loss_mlp": 1.04245722, "epoch": 0.031324214640012026, "flos": 21649117345920.0, "grad_norm": 1.7543559854752624, "language_loss": 0.89257371, "learning_rate": 3.990394896789228e-06, "loss": 0.91534168, "num_input_tokens_seen": 11068835, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.6015625, "step": 521, "time_per_iteration": 2.474047899246216 }, { "auxiliary_loss_clip": 0.01196938, "auxiliary_loss_mlp": 0.01070582, "balance_loss_clip": 1.03491497, "balance_loss_mlp": 1.04307437, "epoch": 0.03138433789267999, "flos": 23440196471040.0, "grad_norm": 2.037463426747011, "language_loss": 0.70534217, "learning_rate": 3.9903578770435505e-06, "loss": 0.72801739, "num_input_tokens_seen": 11088980, "router_z_loss_clip": 0.35742188, "router_z_loss_mlp": 1.5390625, "step": 522, "time_per_iteration": 2.466909170150757 }, { "auxiliary_loss_clip": 0.01202988, "auxiliary_loss_mlp": 0.0106513, "balance_loss_clip": 1.03029704, "balance_loss_mlp": 1.04272258, "epoch": 0.03144446114534796, "flos": 18981369181440.0, "grad_norm": 4.602702188577186, "language_loss": 0.84968263, "learning_rate": 3.99032078626712e-06, "loss": 0.87236381, "num_input_tokens_seen": 11104300, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.6015625, "step": 523, "time_per_iteration": 2.43253231048584 }, { "auxiliary_loss_clip": 0.01203881, "auxiliary_loss_mlp": 0.01065043, "balance_loss_clip": 1.03044844, "balance_loss_mlp": 1.04492521, "epoch": 0.031504584398015935, "flos": 22636600640640.0, "grad_norm": 2.931477636015078, "language_loss": 0.89870876, "learning_rate": 3.990283624461261e-06, "loss": 0.92139804, "num_input_tokens_seen": 11123335, "router_z_loss_clip": 0.34570312, "router_z_loss_mlp": 1.59375, "step": 524, "time_per_iteration": 2.4421632289886475 }, { "auxiliary_loss_clip": 0.01205348, "auxiliary_loss_mlp": 0.01072081, "balance_loss_clip": 1.03605592, "balance_loss_mlp": 1.04450417, "epoch": 0.0315647076506839, "flos": 25296004990080.0, "grad_norm": 3.1770234004138236, "language_loss": 0.79840553, "learning_rate": 3.9902463916273e-06, "loss": 0.82117987, "num_input_tokens_seen": 11140880, "router_z_loss_clip": 0.36132812, "router_z_loss_mlp": 1.609375, "step": 525, "time_per_iteration": 2.512315034866333 }, { "auxiliary_loss_clip": 0.01197829, "auxiliary_loss_mlp": 0.01059955, "balance_loss_clip": 1.02438259, "balance_loss_mlp": 1.04082167, "epoch": 0.03162483090335187, "flos": 16981892951040.0, "grad_norm": 1.974038699837512, "language_loss": 0.80296195, "learning_rate": 3.990209087766563e-06, "loss": 0.82553983, "num_input_tokens_seen": 11158710, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.5703125, "step": 526, "time_per_iteration": 2.4241766929626465 }, { "auxiliary_loss_clip": 0.01206056, "auxiliary_loss_mlp": 0.01059794, "balance_loss_clip": 1.02348268, "balance_loss_mlp": 1.04648757, "epoch": 0.03168495415601984, "flos": 18733485461760.0, "grad_norm": 2.082035333094545, "language_loss": 0.81417549, "learning_rate": 3.990171712880383e-06, "loss": 0.83683401, "num_input_tokens_seen": 11177550, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.59375, "step": 527, "time_per_iteration": 2.393235921859741 }, { "auxiliary_loss_clip": 0.01193488, "auxiliary_loss_mlp": 0.01064555, "balance_loss_clip": 1.03058052, "balance_loss_mlp": 1.03917575, "epoch": 0.03174507740868781, "flos": 21213820114560.0, "grad_norm": 1.9535105862033473, "language_loss": 0.93562591, "learning_rate": 3.990134266970095e-06, "loss": 0.95820642, "num_input_tokens_seen": 11196230, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.546875, "step": 528, "time_per_iteration": 3.870699644088745 }, { "auxiliary_loss_clip": 0.01199748, "auxiliary_loss_mlp": 0.01055606, "balance_loss_clip": 1.02167869, "balance_loss_mlp": 1.04302394, "epoch": 0.03180520066135578, "flos": 24786587208960.0, "grad_norm": 1.9424774422442086, "language_loss": 0.83959383, "learning_rate": 3.9900967500370335e-06, "loss": 0.86214739, "num_input_tokens_seen": 11214935, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.5625, "step": 529, "time_per_iteration": 2.437253952026367 }, { "auxiliary_loss_clip": 0.01198756, "auxiliary_loss_mlp": 0.01064429, "balance_loss_clip": 1.03158712, "balance_loss_mlp": 1.04470348, "epoch": 0.03186532391402375, "flos": 24863081731200.0, "grad_norm": 2.219900736369779, "language_loss": 0.90270782, "learning_rate": 3.990059162082539e-06, "loss": 0.9253397, "num_input_tokens_seen": 11235310, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.5390625, "step": 530, "time_per_iteration": 5.275681972503662 }, { "auxiliary_loss_clip": 0.01192801, "auxiliary_loss_mlp": 0.01061732, "balance_loss_clip": 1.02620769, "balance_loss_mlp": 1.03718972, "epoch": 0.03192544716669172, "flos": 21213994671360.0, "grad_norm": 2.2176436304325775, "language_loss": 0.76117861, "learning_rate": 3.9900215031079515e-06, "loss": 0.78372395, "num_input_tokens_seen": 11254425, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.5546875, "step": 531, "time_per_iteration": 2.421377420425415 }, { "auxiliary_loss_clip": 0.0119355, "auxiliary_loss_mlp": 0.01057381, "balance_loss_clip": 1.02307296, "balance_loss_mlp": 1.04110742, "epoch": 0.03198557041935969, "flos": 24352058027520.0, "grad_norm": 2.0795541844250462, "language_loss": 0.904203, "learning_rate": 3.989983773114616e-06, "loss": 0.92671233, "num_input_tokens_seen": 11274595, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.5234375, "step": 532, "time_per_iteration": 3.8595516681671143 }, { "auxiliary_loss_clip": 0.01078112, "auxiliary_loss_mlp": 0.0100629, "balance_loss_clip": 0.99875575, "balance_loss_mlp": 1.01723647, "epoch": 0.032045693672027656, "flos": 61824056186880.0, "grad_norm": 0.7257353694709775, "language_loss": 0.5794214, "learning_rate": 3.989945972103877e-06, "loss": 0.60026538, "num_input_tokens_seen": 11336705, "router_z_loss_clip": 0.07519531, "router_z_loss_mlp": 0.609375, "step": 533, "time_per_iteration": 3.104865550994873 }, { "auxiliary_loss_clip": 0.01193289, "auxiliary_loss_mlp": 0.01067067, "balance_loss_clip": 1.03375959, "balance_loss_mlp": 1.04044938, "epoch": 0.03210581692469563, "flos": 28399958081280.0, "grad_norm": 1.6510738722888407, "language_loss": 0.8620894, "learning_rate": 3.989908100077087e-06, "loss": 0.88469297, "num_input_tokens_seen": 11356820, "router_z_loss_clip": 0.33203125, "router_z_loss_mlp": 1.5234375, "step": 534, "time_per_iteration": 2.4982473850250244 }, { "auxiliary_loss_clip": 0.01195723, "auxiliary_loss_mlp": 0.01056269, "balance_loss_clip": 1.02060103, "balance_loss_mlp": 1.04258776, "epoch": 0.03216594017736359, "flos": 24716551288320.0, "grad_norm": 2.2136679875308443, "language_loss": 0.7724539, "learning_rate": 3.989870157035594e-06, "loss": 0.79497379, "num_input_tokens_seen": 11376645, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.53125, "step": 535, "time_per_iteration": 2.4496872425079346 }, { "auxiliary_loss_clip": 0.01195513, "auxiliary_loss_mlp": 0.0106181, "balance_loss_clip": 1.02802587, "balance_loss_mlp": 1.04010534, "epoch": 0.032226063430031565, "flos": 31174121669760.0, "grad_norm": 2.1996955541292476, "language_loss": 0.80698258, "learning_rate": 3.989832142980754e-06, "loss": 0.82955575, "num_input_tokens_seen": 11397310, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.5546875, "step": 536, "time_per_iteration": 2.5376601219177246 }, { "auxiliary_loss_clip": 0.01194075, "auxiliary_loss_mlp": 0.01062913, "balance_loss_clip": 1.02779412, "balance_loss_mlp": 1.04131877, "epoch": 0.03228618668269954, "flos": 32196832392960.0, "grad_norm": 1.9626146690772939, "language_loss": 0.69564807, "learning_rate": 3.989794057913923e-06, "loss": 0.71821791, "num_input_tokens_seen": 11418475, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.53125, "step": 537, "time_per_iteration": 2.4927241802215576 }, { "auxiliary_loss_clip": 0.01197973, "auxiliary_loss_mlp": 0.0106739, "balance_loss_clip": 1.03355789, "balance_loss_mlp": 1.04507327, "epoch": 0.0323463099353675, "flos": 22669174805760.0, "grad_norm": 2.1653692978086414, "language_loss": 0.82236588, "learning_rate": 3.9897559018364615e-06, "loss": 0.84501946, "num_input_tokens_seen": 11436630, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.53125, "step": 538, "time_per_iteration": 2.4447684288024902 }, { "auxiliary_loss_clip": 0.01201374, "auxiliary_loss_mlp": 0.01062495, "balance_loss_clip": 1.02725673, "balance_loss_mlp": 1.04155743, "epoch": 0.032406433188035474, "flos": 26903999612160.0, "grad_norm": 1.7500385242252072, "language_loss": 0.79262614, "learning_rate": 3.98971767474973e-06, "loss": 0.81526482, "num_input_tokens_seen": 11457275, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.6015625, "step": 539, "time_per_iteration": 2.4402785301208496 }, { "auxiliary_loss_clip": 0.0119521, "auxiliary_loss_mlp": 0.01064609, "balance_loss_clip": 1.02927542, "balance_loss_mlp": 1.04309082, "epoch": 0.03246655644070344, "flos": 31502584540800.0, "grad_norm": 3.1364036052751243, "language_loss": 0.77135301, "learning_rate": 3.989679376655092e-06, "loss": 0.79395115, "num_input_tokens_seen": 11476925, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.5234375, "step": 540, "time_per_iteration": 2.5234434604644775 }, { "auxiliary_loss_clip": 0.01202534, "auxiliary_loss_mlp": 0.01065013, "balance_loss_clip": 1.02903533, "balance_loss_mlp": 1.04551792, "epoch": 0.03252667969337141, "flos": 23217311151360.0, "grad_norm": 2.7523998804954344, "language_loss": 0.85123587, "learning_rate": 3.989641007553916e-06, "loss": 0.87391126, "num_input_tokens_seen": 11496830, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.5703125, "step": 541, "time_per_iteration": 2.4256014823913574 }, { "auxiliary_loss_clip": 0.01195641, "auxiliary_loss_mlp": 0.01061551, "balance_loss_clip": 1.02438188, "balance_loss_mlp": 1.04368758, "epoch": 0.032586802946039384, "flos": 14756563935360.0, "grad_norm": 2.2643202941631486, "language_loss": 0.88175774, "learning_rate": 3.989602567447569e-06, "loss": 0.90432966, "num_input_tokens_seen": 11515605, "router_z_loss_clip": 0.37109375, "router_z_loss_mlp": 1.515625, "step": 542, "time_per_iteration": 2.4088408946990967 }, { "auxiliary_loss_clip": 0.01196528, "auxiliary_loss_mlp": 0.01065616, "balance_loss_clip": 1.0302583, "balance_loss_mlp": 1.04263008, "epoch": 0.03264692619870735, "flos": 24279508488960.0, "grad_norm": 1.975439138667125, "language_loss": 0.70890611, "learning_rate": 3.989564056337426e-06, "loss": 0.73152757, "num_input_tokens_seen": 11536230, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.5390625, "step": 543, "time_per_iteration": 2.4294607639312744 }, { "auxiliary_loss_clip": 0.0119393, "auxiliary_loss_mlp": 0.01060699, "balance_loss_clip": 1.0262711, "balance_loss_mlp": 1.03910398, "epoch": 0.03270704945137532, "flos": 22892060125440.0, "grad_norm": 2.6781412261206756, "language_loss": 0.91309845, "learning_rate": 3.989525474224858e-06, "loss": 0.93564469, "num_input_tokens_seen": 11554715, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.546875, "step": 544, "time_per_iteration": 2.438074827194214 }, { "auxiliary_loss_clip": 0.01197985, "auxiliary_loss_mlp": 0.01053669, "balance_loss_clip": 1.02126789, "balance_loss_mlp": 1.04320121, "epoch": 0.032767172704043286, "flos": 18040040570880.0, "grad_norm": 3.2598794138189326, "language_loss": 0.65982533, "learning_rate": 3.989486821111244e-06, "loss": 0.68234193, "num_input_tokens_seen": 11571370, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.546875, "step": 545, "time_per_iteration": 2.3907058238983154 }, { "auxiliary_loss_clip": 0.01197121, "auxiliary_loss_mlp": 0.01058319, "balance_loss_clip": 1.02453566, "balance_loss_mlp": 1.04238844, "epoch": 0.03282729595671126, "flos": 22527636687360.0, "grad_norm": 2.2970789725101652, "language_loss": 0.91792428, "learning_rate": 3.9894480969979635e-06, "loss": 0.94047862, "num_input_tokens_seen": 11588560, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.546875, "step": 546, "time_per_iteration": 2.4523770809173584 }, { "auxiliary_loss_clip": 0.01193368, "auxiliary_loss_mlp": 0.01060539, "balance_loss_clip": 1.02456117, "balance_loss_mlp": 1.03936839, "epoch": 0.03288741920937923, "flos": 20409630791040.0, "grad_norm": 3.55375524184824, "language_loss": 0.81702125, "learning_rate": 3.989409301886398e-06, "loss": 0.83956033, "num_input_tokens_seen": 11605685, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.5390625, "step": 547, "time_per_iteration": 2.4157586097717285 }, { "auxiliary_loss_clip": 0.01195076, "auxiliary_loss_mlp": 0.01056152, "balance_loss_clip": 1.02089, "balance_loss_mlp": 1.0420146, "epoch": 0.032947542462047195, "flos": 20776916960640.0, "grad_norm": 1.8755243921553955, "language_loss": 0.80964327, "learning_rate": 3.989370435777931e-06, "loss": 0.83215559, "num_input_tokens_seen": 11626290, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.53125, "step": 548, "time_per_iteration": 2.4845027923583984 }, { "auxiliary_loss_clip": 0.01196537, "auxiliary_loss_mlp": 0.01059157, "balance_loss_clip": 1.02194011, "balance_loss_mlp": 1.04265714, "epoch": 0.03300766571471517, "flos": 19900247921280.0, "grad_norm": 4.870889560058078, "language_loss": 0.67086864, "learning_rate": 3.989331498673951e-06, "loss": 0.69342566, "num_input_tokens_seen": 11643950, "router_z_loss_clip": 0.37304688, "router_z_loss_mlp": 1.5390625, "step": 549, "time_per_iteration": 2.459944009780884 }, { "auxiliary_loss_clip": 0.01191409, "auxiliary_loss_mlp": 0.01063525, "balance_loss_clip": 1.02819109, "balance_loss_mlp": 1.04134452, "epoch": 0.03306778896738313, "flos": 17966792805120.0, "grad_norm": 2.04638296747371, "language_loss": 0.85720515, "learning_rate": 3.9892924905758475e-06, "loss": 0.87975454, "num_input_tokens_seen": 11662560, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.5, "step": 550, "time_per_iteration": 2.4711291790008545 }, { "auxiliary_loss_clip": 0.01194971, "auxiliary_loss_mlp": 0.01069821, "balance_loss_clip": 1.03541744, "balance_loss_mlp": 1.04545546, "epoch": 0.033127912220051105, "flos": 21652294279680.0, "grad_norm": 1.7071244062597453, "language_loss": 0.81147861, "learning_rate": 3.989253411485011e-06, "loss": 0.83412647, "num_input_tokens_seen": 11682265, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.5, "step": 551, "time_per_iteration": 2.4221384525299072 }, { "auxiliary_loss_clip": 0.01197454, "auxiliary_loss_mlp": 0.01065646, "balance_loss_clip": 1.03043151, "balance_loss_mlp": 1.04241097, "epoch": 0.03318803547271908, "flos": 30187127134080.0, "grad_norm": 2.8232966739904155, "language_loss": 0.86134279, "learning_rate": 3.989214261402838e-06, "loss": 0.88397378, "num_input_tokens_seen": 11699300, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.5546875, "step": 552, "time_per_iteration": 2.518507719039917 }, { "auxiliary_loss_clip": 0.01197266, "auxiliary_loss_mlp": 0.01063903, "balance_loss_clip": 1.02642334, "balance_loss_mlp": 1.04250526, "epoch": 0.03324815872538704, "flos": 20374996855680.0, "grad_norm": 2.3913134793234097, "language_loss": 0.92376202, "learning_rate": 3.989175040330724e-06, "loss": 0.9463737, "num_input_tokens_seen": 11716955, "router_z_loss_clip": 0.375, "router_z_loss_mlp": 1.546875, "step": 553, "time_per_iteration": 2.4046833515167236 }, { "auxiliary_loss_clip": 0.01197331, "auxiliary_loss_mlp": 0.01065382, "balance_loss_clip": 1.02694941, "balance_loss_mlp": 1.04425418, "epoch": 0.033308281978055014, "flos": 24493526323200.0, "grad_norm": 2.3784682061349676, "language_loss": 0.78795719, "learning_rate": 3.98913574827007e-06, "loss": 0.81058431, "num_input_tokens_seen": 11736130, "router_z_loss_clip": 0.38476562, "router_z_loss_mlp": 1.53125, "step": 554, "time_per_iteration": 2.4820733070373535 }, { "auxiliary_loss_clip": 0.0119128, "auxiliary_loss_mlp": 0.01069872, "balance_loss_clip": 1.03301299, "balance_loss_mlp": 1.04178536, "epoch": 0.03336840523072298, "flos": 23399313402240.0, "grad_norm": 2.431197399805768, "language_loss": 0.81781608, "learning_rate": 3.989096385222278e-06, "loss": 0.84042764, "num_input_tokens_seen": 11754425, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.4921875, "step": 555, "time_per_iteration": 2.4461193084716797 }, { "auxiliary_loss_clip": 0.01195652, "auxiliary_loss_mlp": 0.01064587, "balance_loss_clip": 1.02787089, "balance_loss_mlp": 1.04220772, "epoch": 0.03342852848339095, "flos": 30549386067840.0, "grad_norm": 2.9516103816303603, "language_loss": 0.88176799, "learning_rate": 3.989056951188753e-06, "loss": 0.90437037, "num_input_tokens_seen": 11772845, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.53125, "step": 556, "time_per_iteration": 2.528106689453125 }, { "auxiliary_loss_clip": 0.01194919, "auxiliary_loss_mlp": 0.01068536, "balance_loss_clip": 1.03270173, "balance_loss_mlp": 1.04206967, "epoch": 0.03348865173605892, "flos": 22892199770880.0, "grad_norm": 1.9707480879266779, "language_loss": 0.83577824, "learning_rate": 3.989017446170901e-06, "loss": 0.85841274, "num_input_tokens_seen": 11792850, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.53125, "step": 557, "time_per_iteration": 2.4847426414489746 }, { "auxiliary_loss_clip": 0.0119705, "auxiliary_loss_mlp": 0.0105911, "balance_loss_clip": 1.02418184, "balance_loss_mlp": 1.04479933, "epoch": 0.03354877498872689, "flos": 17675058551040.0, "grad_norm": 3.8172882719549515, "language_loss": 0.93698788, "learning_rate": 3.988977870170133e-06, "loss": 0.95954949, "num_input_tokens_seen": 11809670, "router_z_loss_clip": 0.34960938, "router_z_loss_mlp": 1.5234375, "step": 558, "time_per_iteration": 2.411505699157715 }, { "auxiliary_loss_clip": 0.01193043, "auxiliary_loss_mlp": 0.01063056, "balance_loss_clip": 1.02886689, "balance_loss_mlp": 1.04174602, "epoch": 0.03360889824139486, "flos": 21651910254720.0, "grad_norm": 6.270277986351199, "language_loss": 0.76974529, "learning_rate": 3.988938223187861e-06, "loss": 0.7923063, "num_input_tokens_seen": 11829665, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.515625, "step": 559, "time_per_iteration": 2.419795036315918 }, { "auxiliary_loss_clip": 0.01193975, "auxiliary_loss_mlp": 0.01067361, "balance_loss_clip": 1.03369641, "balance_loss_mlp": 1.04131722, "epoch": 0.033669021494062826, "flos": 21794740093440.0, "grad_norm": 2.815050991405286, "language_loss": 0.87094873, "learning_rate": 3.9888985052255005e-06, "loss": 0.89356208, "num_input_tokens_seen": 11848190, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.53125, "step": 560, "time_per_iteration": 2.4539504051208496 }, { "auxiliary_loss_clip": 0.01190911, "auxiliary_loss_mlp": 0.01056281, "balance_loss_clip": 1.02352214, "balance_loss_mlp": 1.04093623, "epoch": 0.0337291447467308, "flos": 21865299684480.0, "grad_norm": 3.150492713638745, "language_loss": 0.80860865, "learning_rate": 3.988858716284468e-06, "loss": 0.83108056, "num_input_tokens_seen": 11864795, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.5, "step": 561, "time_per_iteration": 2.392932891845703 }, { "auxiliary_loss_clip": 0.01192948, "auxiliary_loss_mlp": 0.01064501, "balance_loss_clip": 1.0318135, "balance_loss_mlp": 1.04135442, "epoch": 0.03378926799939877, "flos": 24244734908160.0, "grad_norm": 1.7802178247380682, "language_loss": 0.81872559, "learning_rate": 3.988818856366184e-06, "loss": 0.84130007, "num_input_tokens_seen": 11885275, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.515625, "step": 562, "time_per_iteration": 2.4758028984069824 }, { "auxiliary_loss_clip": 0.01197868, "auxiliary_loss_mlp": 0.01073311, "balance_loss_clip": 1.03945541, "balance_loss_mlp": 1.0440259, "epoch": 0.033849391252066735, "flos": 16506899637120.0, "grad_norm": 1.9287515962020005, "language_loss": 0.83921456, "learning_rate": 3.9887789254720704e-06, "loss": 0.86192638, "num_input_tokens_seen": 11903595, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.5390625, "step": 563, "time_per_iteration": 2.3996310234069824 }, { "auxiliary_loss_clip": 0.01195139, "auxiliary_loss_mlp": 0.01065566, "balance_loss_clip": 1.02963662, "balance_loss_mlp": 1.04248786, "epoch": 0.03390951450473471, "flos": 15668390580480.0, "grad_norm": 2.33407814831408, "language_loss": 0.93336153, "learning_rate": 3.988738923603553e-06, "loss": 0.95596856, "num_input_tokens_seen": 11917815, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.5234375, "step": 564, "time_per_iteration": 2.3867733478546143 }, { "auxiliary_loss_clip": 0.01194909, "auxiliary_loss_mlp": 0.01066091, "balance_loss_clip": 1.03147316, "balance_loss_mlp": 1.03992391, "epoch": 0.03396963775740267, "flos": 22673678371200.0, "grad_norm": 3.090948314160313, "language_loss": 0.94309002, "learning_rate": 3.98869885076206e-06, "loss": 0.96570009, "num_input_tokens_seen": 11936305, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.546875, "step": 565, "time_per_iteration": 2.473637104034424 }, { "auxiliary_loss_clip": 0.01080434, "auxiliary_loss_mlp": 0.01023652, "balance_loss_clip": 1.01535463, "balance_loss_mlp": 1.01858997, "epoch": 0.034029761010070644, "flos": 64388984797440.0, "grad_norm": 1.148770634874019, "language_loss": 0.54870236, "learning_rate": 3.9886587069490195e-06, "loss": 0.56974322, "num_input_tokens_seen": 11998940, "router_z_loss_clip": 0.08300781, "router_z_loss_mlp": 0.6171875, "step": 566, "time_per_iteration": 3.11896014213562 }, { "auxiliary_loss_clip": 0.01195853, "auxiliary_loss_mlp": 0.01062324, "balance_loss_clip": 1.0245589, "balance_loss_mlp": 1.04374218, "epoch": 0.034089884262738616, "flos": 25003188483840.0, "grad_norm": 2.358639810820114, "language_loss": 0.76279438, "learning_rate": 3.988618492165865e-06, "loss": 0.78537619, "num_input_tokens_seen": 12018860, "router_z_loss_clip": 0.37890625, "router_z_loss_mlp": 1.515625, "step": 567, "time_per_iteration": 3.895256280899048 }, { "auxiliary_loss_clip": 0.01189289, "auxiliary_loss_mlp": 0.01067985, "balance_loss_clip": 1.03274679, "balance_loss_mlp": 1.04122317, "epoch": 0.03415000751540658, "flos": 28437838773120.0, "grad_norm": 2.024071973675408, "language_loss": 0.80621415, "learning_rate": 3.988578206414032e-06, "loss": 0.82878685, "num_input_tokens_seen": 12039675, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.484375, "step": 568, "time_per_iteration": 2.4694976806640625 }, { "auxiliary_loss_clip": 0.01193456, "auxiliary_loss_mlp": 0.01061731, "balance_loss_clip": 1.02859068, "balance_loss_mlp": 1.0454756, "epoch": 0.034210130768074554, "flos": 21467708588160.0, "grad_norm": 1.9569454038555405, "language_loss": 0.8628267, "learning_rate": 3.988537849694959e-06, "loss": 0.8853786, "num_input_tokens_seen": 12057680, "router_z_loss_clip": 0.33203125, "router_z_loss_mlp": 1.484375, "step": 569, "time_per_iteration": 3.8459970951080322 }, { "auxiliary_loss_clip": 0.0119573, "auxiliary_loss_mlp": 0.0106184, "balance_loss_clip": 1.02598178, "balance_loss_mlp": 1.04251552, "epoch": 0.03427025402074252, "flos": 18696512465280.0, "grad_norm": 1.8131955516308138, "language_loss": 0.95423174, "learning_rate": 3.988497422010084e-06, "loss": 0.97680748, "num_input_tokens_seen": 12076135, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.53125, "step": 570, "time_per_iteration": 3.7499139308929443 }, { "auxiliary_loss_clip": 0.01191599, "auxiliary_loss_mlp": 0.01061289, "balance_loss_clip": 1.02357149, "balance_loss_mlp": 1.03753138, "epoch": 0.03433037727341049, "flos": 20848942828800.0, "grad_norm": 2.495821873687206, "language_loss": 0.79018605, "learning_rate": 3.988456923360852e-06, "loss": 0.81271493, "num_input_tokens_seen": 12094785, "router_z_loss_clip": 0.37695312, "router_z_loss_mlp": 1.5390625, "step": 571, "time_per_iteration": 3.85359787940979 }, { "auxiliary_loss_clip": 0.01195057, "auxiliary_loss_mlp": 0.01068013, "balance_loss_clip": 1.03172517, "balance_loss_mlp": 1.04233098, "epoch": 0.03439050052607846, "flos": 25409123395200.0, "grad_norm": 2.40142219818747, "language_loss": 0.80008596, "learning_rate": 3.988416353748707e-06, "loss": 0.82271665, "num_input_tokens_seen": 12114590, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.53125, "step": 572, "time_per_iteration": 2.4332334995269775 }, { "auxiliary_loss_clip": 0.01198195, "auxiliary_loss_mlp": 0.01057882, "balance_loss_clip": 1.02295363, "balance_loss_mlp": 1.04672611, "epoch": 0.03445062377874643, "flos": 17639167806720.0, "grad_norm": 2.9297505078403385, "language_loss": 0.84247696, "learning_rate": 3.988375713175097e-06, "loss": 0.86503768, "num_input_tokens_seen": 12132390, "router_z_loss_clip": 0.34960938, "router_z_loss_mlp": 1.515625, "step": 573, "time_per_iteration": 2.392399787902832 }, { "auxiliary_loss_clip": 0.01193016, "auxiliary_loss_mlp": 0.01064463, "balance_loss_clip": 1.0300827, "balance_loss_mlp": 1.04029369, "epoch": 0.0345107470314144, "flos": 16763546108160.0, "grad_norm": 2.3845255873691547, "language_loss": 0.76166523, "learning_rate": 3.988335001641473e-06, "loss": 0.78423995, "num_input_tokens_seen": 12149035, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.53125, "step": 574, "time_per_iteration": 2.3899247646331787 }, { "auxiliary_loss_clip": 0.01194761, "auxiliary_loss_mlp": 0.01054424, "balance_loss_clip": 1.02185607, "balance_loss_mlp": 1.04313254, "epoch": 0.03457087028408237, "flos": 14683560549120.0, "grad_norm": 2.8759034033766717, "language_loss": 0.83530688, "learning_rate": 3.988294219149287e-06, "loss": 0.8577987, "num_input_tokens_seen": 12167530, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.515625, "step": 575, "time_per_iteration": 2.404356002807617 }, { "auxiliary_loss_clip": 0.01191842, "auxiliary_loss_mlp": 0.01066171, "balance_loss_clip": 1.03207731, "balance_loss_mlp": 1.04431152, "epoch": 0.03463099353675034, "flos": 20010259215360.0, "grad_norm": 2.1975776036879133, "language_loss": 0.83930761, "learning_rate": 3.9882533656999945e-06, "loss": 0.86188769, "num_input_tokens_seen": 12186340, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.4765625, "step": 576, "time_per_iteration": 2.404324769973755 }, { "auxiliary_loss_clip": 0.01192762, "auxiliary_loss_mlp": 0.01065608, "balance_loss_clip": 1.03306341, "balance_loss_mlp": 1.04672194, "epoch": 0.03469111678941831, "flos": 25299984885120.0, "grad_norm": 2.136277743417185, "language_loss": 0.86451602, "learning_rate": 3.988212441295054e-06, "loss": 0.88709974, "num_input_tokens_seen": 12204090, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.4609375, "step": 577, "time_per_iteration": 2.4424991607666016 }, { "auxiliary_loss_clip": 0.01193216, "auxiliary_loss_mlp": 0.0106795, "balance_loss_clip": 1.03554869, "balance_loss_mlp": 1.04519367, "epoch": 0.034751240042086275, "flos": 23258264042880.0, "grad_norm": 2.0062231336852197, "language_loss": 0.72245854, "learning_rate": 3.9881714459359255e-06, "loss": 0.74507022, "num_input_tokens_seen": 12224850, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.484375, "step": 578, "time_per_iteration": 2.41383695602417 }, { "auxiliary_loss_clip": 0.01192622, "auxiliary_loss_mlp": 0.010612, "balance_loss_clip": 1.02734423, "balance_loss_mlp": 1.04283249, "epoch": 0.03481136329475425, "flos": 23768100760320.0, "grad_norm": 1.9369968368063495, "language_loss": 0.77471632, "learning_rate": 3.988130379624073e-06, "loss": 0.7972545, "num_input_tokens_seen": 12244935, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.5, "step": 579, "time_per_iteration": 2.420255422592163 }, { "auxiliary_loss_clip": 0.01192129, "auxiliary_loss_mlp": 0.01062744, "balance_loss_clip": 1.02752972, "balance_loss_mlp": 1.04205263, "epoch": 0.03487148654742222, "flos": 20156475456000.0, "grad_norm": 2.55101409008302, "language_loss": 0.86368865, "learning_rate": 3.988089242360961e-06, "loss": 0.88623732, "num_input_tokens_seen": 12262140, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.5, "step": 580, "time_per_iteration": 2.4236578941345215 }, { "auxiliary_loss_clip": 0.01196551, "auxiliary_loss_mlp": 0.01058375, "balance_loss_clip": 1.02592683, "balance_loss_mlp": 1.04238605, "epoch": 0.034931609800090184, "flos": 15668669871360.0, "grad_norm": 2.332495529883519, "language_loss": 0.82363093, "learning_rate": 3.988048034148057e-06, "loss": 0.8461802, "num_input_tokens_seen": 12280930, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.5390625, "step": 581, "time_per_iteration": 2.393260955810547 }, { "auxiliary_loss_clip": 0.0119314, "auxiliary_loss_mlp": 0.01060886, "balance_loss_clip": 1.02884281, "balance_loss_mlp": 1.04386783, "epoch": 0.034991733052758156, "flos": 16361451446400.0, "grad_norm": 2.557332275981054, "language_loss": 0.76911843, "learning_rate": 3.988006754986834e-06, "loss": 0.7916587, "num_input_tokens_seen": 12299125, "router_z_loss_clip": 0.3203125, "router_z_loss_mlp": 1.4921875, "step": 582, "time_per_iteration": 2.378066062927246 }, { "auxiliary_loss_clip": 0.01193353, "auxiliary_loss_mlp": 0.01065852, "balance_loss_clip": 1.03054273, "balance_loss_mlp": 1.04665709, "epoch": 0.03505185630542612, "flos": 19386396397440.0, "grad_norm": 2.182466588591563, "language_loss": 0.87432832, "learning_rate": 3.987965404878763e-06, "loss": 0.89692038, "num_input_tokens_seen": 12316905, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.46875, "step": 583, "time_per_iteration": 2.4018166065216064 }, { "auxiliary_loss_clip": 0.0119538, "auxiliary_loss_mlp": 0.01065533, "balance_loss_clip": 1.03060496, "balance_loss_mlp": 1.0420208, "epoch": 0.03511197955809409, "flos": 21322784067840.0, "grad_norm": 2.4167703716027362, "language_loss": 0.80618572, "learning_rate": 3.987923983825321e-06, "loss": 0.82879484, "num_input_tokens_seen": 12335070, "router_z_loss_clip": 0.34960938, "router_z_loss_mlp": 1.53125, "step": 584, "time_per_iteration": 2.3943240642547607 }, { "auxiliary_loss_clip": 0.01192112, "auxiliary_loss_mlp": 0.01060208, "balance_loss_clip": 1.02647233, "balance_loss_mlp": 1.04246902, "epoch": 0.035172102810762065, "flos": 14135738405760.0, "grad_norm": 5.747430825665412, "language_loss": 0.92533493, "learning_rate": 3.9878824918279845e-06, "loss": 0.9478581, "num_input_tokens_seen": 12350315, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.5, "step": 585, "time_per_iteration": 2.3824782371520996 }, { "auxiliary_loss_clip": 0.01193938, "auxiliary_loss_mlp": 0.01061532, "balance_loss_clip": 1.02808166, "balance_loss_mlp": 1.04445136, "epoch": 0.03523222606343003, "flos": 20296023626880.0, "grad_norm": 2.282708667245842, "language_loss": 0.87457907, "learning_rate": 3.9878409288882364e-06, "loss": 0.89713371, "num_input_tokens_seen": 12366030, "router_z_loss_clip": 0.33398438, "router_z_loss_mlp": 1.5, "step": 586, "time_per_iteration": 2.3860182762145996 }, { "auxiliary_loss_clip": 0.01198949, "auxiliary_loss_mlp": 0.01060182, "balance_loss_clip": 1.02730393, "balance_loss_mlp": 1.04620695, "epoch": 0.035292349316098, "flos": 20374787387520.0, "grad_norm": 1.9012974248253003, "language_loss": 0.76167411, "learning_rate": 3.987799295007558e-06, "loss": 0.7842654, "num_input_tokens_seen": 12384895, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.53125, "step": 587, "time_per_iteration": 2.4081978797912598 }, { "auxiliary_loss_clip": 0.01193189, "auxiliary_loss_mlp": 0.01059531, "balance_loss_clip": 1.02417374, "balance_loss_mlp": 1.04124045, "epoch": 0.03535247256876597, "flos": 21467848233600.0, "grad_norm": 1.754797954220294, "language_loss": 0.78395927, "learning_rate": 3.987757590187436e-06, "loss": 0.80648649, "num_input_tokens_seen": 12404980, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.5234375, "step": 588, "time_per_iteration": 2.410745620727539 }, { "auxiliary_loss_clip": 0.01197837, "auxiliary_loss_mlp": 0.01064675, "balance_loss_clip": 1.02633774, "balance_loss_mlp": 1.04193711, "epoch": 0.03541259582143394, "flos": 23621919431040.0, "grad_norm": 2.7924246346076744, "language_loss": 0.93870485, "learning_rate": 3.987715814429359e-06, "loss": 0.96132994, "num_input_tokens_seen": 12423835, "router_z_loss_clip": 0.3828125, "router_z_loss_mlp": 1.5625, "step": 589, "time_per_iteration": 2.423064947128296 }, { "auxiliary_loss_clip": 0.01199126, "auxiliary_loss_mlp": 0.01063146, "balance_loss_clip": 1.02962434, "balance_loss_mlp": 1.04581523, "epoch": 0.03547271907410191, "flos": 33725050824960.0, "grad_norm": 2.876152200613965, "language_loss": 0.83852893, "learning_rate": 3.987673967734818e-06, "loss": 0.86115164, "num_input_tokens_seen": 12443135, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.5390625, "step": 590, "time_per_iteration": 2.5066776275634766 }, { "auxiliary_loss_clip": 0.01190277, "auxiliary_loss_mlp": 0.01061021, "balance_loss_clip": 1.02862036, "balance_loss_mlp": 1.04210639, "epoch": 0.03553284232676988, "flos": 21141619689600.0, "grad_norm": 2.041992487210075, "language_loss": 0.86693615, "learning_rate": 3.987632050105306e-06, "loss": 0.88944912, "num_input_tokens_seen": 12462895, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.484375, "step": 591, "time_per_iteration": 2.4259727001190186 }, { "auxiliary_loss_clip": 0.01194508, "auxiliary_loss_mlp": 0.01070402, "balance_loss_clip": 1.034091, "balance_loss_mlp": 1.04161382, "epoch": 0.03559296557943785, "flos": 20045591377920.0, "grad_norm": 2.0630956610298865, "language_loss": 0.82878077, "learning_rate": 3.987590061542319e-06, "loss": 0.85142994, "num_input_tokens_seen": 12481515, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.53125, "step": 592, "time_per_iteration": 2.3929755687713623 }, { "auxiliary_loss_clip": 0.01072431, "auxiliary_loss_mlp": 0.01007916, "balance_loss_clip": 1.00109756, "balance_loss_mlp": 1.01578867, "epoch": 0.035653088832105814, "flos": 60331239740160.0, "grad_norm": 0.8880340868301633, "language_loss": 0.59840667, "learning_rate": 3.987548002047354e-06, "loss": 0.61921012, "num_input_tokens_seen": 12548220, "router_z_loss_clip": 0.06835938, "router_z_loss_mlp": 0.56640625, "step": 593, "time_per_iteration": 3.1164820194244385 }, { "auxiliary_loss_clip": 0.01193037, "auxiliary_loss_mlp": 0.01062465, "balance_loss_clip": 1.02639222, "balance_loss_mlp": 1.04370463, "epoch": 0.035713212084773786, "flos": 20112310719360.0, "grad_norm": 2.1609314140189433, "language_loss": 0.8677175, "learning_rate": 3.987505871621915e-06, "loss": 0.89027262, "num_input_tokens_seen": 12566105, "router_z_loss_clip": 0.36132812, "router_z_loss_mlp": 1.4921875, "step": 594, "time_per_iteration": 2.3931171894073486 }, { "auxiliary_loss_clip": 0.01194122, "auxiliary_loss_mlp": 0.01063129, "balance_loss_clip": 1.02932084, "balance_loss_mlp": 1.04368234, "epoch": 0.03577333533744176, "flos": 26284605448320.0, "grad_norm": 1.9805413200314534, "language_loss": 0.84035844, "learning_rate": 3.987463670267502e-06, "loss": 0.86293095, "num_input_tokens_seen": 12586680, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.5078125, "step": 595, "time_per_iteration": 2.455754280090332 }, { "auxiliary_loss_clip": 0.01191518, "auxiliary_loss_mlp": 0.01065451, "balance_loss_clip": 1.03123832, "balance_loss_mlp": 1.04367185, "epoch": 0.035833458590109724, "flos": 10888955475840.0, "grad_norm": 2.8540243898721607, "language_loss": 0.9549948, "learning_rate": 3.987421397985625e-06, "loss": 0.97756451, "num_input_tokens_seen": 12601605, "router_z_loss_clip": 0.34179688, "router_z_loss_mlp": 1.4765625, "step": 596, "time_per_iteration": 2.373399257659912 }, { "auxiliary_loss_clip": 0.0119328, "auxiliary_loss_mlp": 0.01066329, "balance_loss_clip": 1.0362879, "balance_loss_mlp": 1.04383075, "epoch": 0.035893581842777696, "flos": 22089127610880.0, "grad_norm": 7.13882265388366, "language_loss": 0.82787955, "learning_rate": 3.98737905477779e-06, "loss": 0.85047561, "num_input_tokens_seen": 12620365, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.5, "step": 597, "time_per_iteration": 2.412506580352783 }, { "auxiliary_loss_clip": 0.01191588, "auxiliary_loss_mlp": 0.0106389, "balance_loss_clip": 1.02886677, "balance_loss_mlp": 1.04515767, "epoch": 0.03595370509544566, "flos": 23037263936640.0, "grad_norm": 1.9765483286078758, "language_loss": 0.81232685, "learning_rate": 3.987336640645508e-06, "loss": 0.83488166, "num_input_tokens_seen": 12641140, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.46875, "step": 598, "time_per_iteration": 2.4125797748565674 }, { "auxiliary_loss_clip": 0.01189158, "auxiliary_loss_mlp": 0.01063772, "balance_loss_clip": 1.02927327, "balance_loss_mlp": 1.04330945, "epoch": 0.03601382834811363, "flos": 20776672581120.0, "grad_norm": 1.919076803637372, "language_loss": 0.81268477, "learning_rate": 3.987294155590295e-06, "loss": 0.83521414, "num_input_tokens_seen": 12661080, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.453125, "step": 599, "time_per_iteration": 2.4201183319091797 }, { "auxiliary_loss_clip": 0.01195125, "auxiliary_loss_mlp": 0.01064769, "balance_loss_clip": 1.03124774, "balance_loss_mlp": 1.04239631, "epoch": 0.036073951600781605, "flos": 23950487036160.0, "grad_norm": 2.7999164737974818, "language_loss": 0.85811245, "learning_rate": 3.987251599613664e-06, "loss": 0.88071138, "num_input_tokens_seen": 12678270, "router_z_loss_clip": 0.33398438, "router_z_loss_mlp": 1.53125, "step": 600, "time_per_iteration": 2.4171955585479736 }, { "auxiliary_loss_clip": 0.01190231, "auxiliary_loss_mlp": 0.01068489, "balance_loss_clip": 1.03244054, "balance_loss_mlp": 1.04157639, "epoch": 0.03613407485344957, "flos": 18911403083520.0, "grad_norm": 2.2791175764803575, "language_loss": 0.81738359, "learning_rate": 3.987208972717135e-06, "loss": 0.83997083, "num_input_tokens_seen": 12697295, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.484375, "step": 601, "time_per_iteration": 2.4082608222961426 }, { "auxiliary_loss_clip": 0.01187035, "auxiliary_loss_mlp": 0.01053517, "balance_loss_clip": 1.02047205, "balance_loss_mlp": 1.04106665, "epoch": 0.03619419810611754, "flos": 23037438493440.0, "grad_norm": 2.7534643093390185, "language_loss": 0.75187588, "learning_rate": 3.987166274902231e-06, "loss": 0.77428138, "num_input_tokens_seen": 12716165, "router_z_loss_clip": 0.33007812, "router_z_loss_mlp": 1.4609375, "step": 602, "time_per_iteration": 2.407188892364502 }, { "auxiliary_loss_clip": 0.01185516, "auxiliary_loss_mlp": 0.01061491, "balance_loss_clip": 1.02737272, "balance_loss_mlp": 1.04045725, "epoch": 0.03625432135878551, "flos": 29456569601280.0, "grad_norm": 2.065777789073131, "language_loss": 0.79639304, "learning_rate": 3.987123506170473e-06, "loss": 0.81886303, "num_input_tokens_seen": 12735475, "router_z_loss_clip": 0.34179688, "router_z_loss_mlp": 1.453125, "step": 603, "time_per_iteration": 2.4654366970062256 }, { "auxiliary_loss_clip": 0.01189171, "auxiliary_loss_mlp": 0.01056694, "balance_loss_clip": 1.02481782, "balance_loss_mlp": 1.04381037, "epoch": 0.03631444461145348, "flos": 23507544216960.0, "grad_norm": 1.8258687398138511, "language_loss": 0.86671007, "learning_rate": 3.987080666523389e-06, "loss": 0.88916874, "num_input_tokens_seen": 12754540, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.453125, "step": 604, "time_per_iteration": 2.4170491695404053 }, { "auxiliary_loss_clip": 0.01192464, "auxiliary_loss_mlp": 0.01061029, "balance_loss_clip": 1.02710176, "balance_loss_mlp": 1.04573047, "epoch": 0.03637456786412145, "flos": 16617190222080.0, "grad_norm": 2.399624764457191, "language_loss": 0.80515403, "learning_rate": 3.987037755962506e-06, "loss": 0.82768893, "num_input_tokens_seen": 12773050, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.46875, "step": 605, "time_per_iteration": 2.4204325675964355 }, { "auxiliary_loss_clip": 0.01190658, "auxiliary_loss_mlp": 0.01063163, "balance_loss_clip": 1.03068995, "balance_loss_mlp": 1.04383564, "epoch": 0.03643469111678942, "flos": 15850916501760.0, "grad_norm": 2.413621551612539, "language_loss": 0.85129428, "learning_rate": 3.986994774489359e-06, "loss": 0.87383258, "num_input_tokens_seen": 12791240, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.46875, "step": 606, "time_per_iteration": 3.8434371948242188 }, { "auxiliary_loss_clip": 0.01193657, "auxiliary_loss_mlp": 0.01066992, "balance_loss_clip": 1.03149116, "balance_loss_mlp": 1.04402304, "epoch": 0.03649481436945739, "flos": 23619335990400.0, "grad_norm": 5.737261281953924, "language_loss": 0.8204093, "learning_rate": 3.986951722105479e-06, "loss": 0.84301579, "num_input_tokens_seen": 12812245, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.4921875, "step": 607, "time_per_iteration": 2.426976442337036 }, { "auxiliary_loss_clip": 0.01191488, "auxiliary_loss_mlp": 0.01062892, "balance_loss_clip": 1.03022838, "balance_loss_mlp": 1.04630244, "epoch": 0.036554937622125354, "flos": 21754694897280.0, "grad_norm": 3.23599896243447, "language_loss": 0.83184808, "learning_rate": 3.986908598812402e-06, "loss": 0.85439193, "num_input_tokens_seen": 12831085, "router_z_loss_clip": 0.32617188, "router_z_loss_mlp": 1.453125, "step": 608, "time_per_iteration": 2.4064114093780518 }, { "auxiliary_loss_clip": 0.01190389, "auxiliary_loss_mlp": 0.01057344, "balance_loss_clip": 1.02160549, "balance_loss_mlp": 1.04478228, "epoch": 0.036615060874793326, "flos": 17818865907840.0, "grad_norm": 2.8337069348270045, "language_loss": 0.81716424, "learning_rate": 3.986865404611669e-06, "loss": 0.83964157, "num_input_tokens_seen": 12849115, "router_z_loss_clip": 0.35742188, "router_z_loss_mlp": 1.453125, "step": 609, "time_per_iteration": 3.824737548828125 }, { "auxiliary_loss_clip": 0.0119741, "auxiliary_loss_mlp": 0.0107586, "balance_loss_clip": 1.04396009, "balance_loss_mlp": 1.0502708, "epoch": 0.0366751841274613, "flos": 26752791047040.0, "grad_norm": 1.9162090268784777, "language_loss": 0.79127526, "learning_rate": 3.98682213950482e-06, "loss": 0.814008, "num_input_tokens_seen": 12868005, "router_z_loss_clip": 0.3203125, "router_z_loss_mlp": 1.46875, "step": 610, "time_per_iteration": 2.4709088802337646 }, { "auxiliary_loss_clip": 0.0119466, "auxiliary_loss_mlp": 0.01063045, "balance_loss_clip": 1.02954769, "balance_loss_mlp": 1.04573941, "epoch": 0.03673530738012926, "flos": 22195961971200.0, "grad_norm": 2.3790511540834864, "language_loss": 0.87558019, "learning_rate": 3.986778803493401e-06, "loss": 0.89815724, "num_input_tokens_seen": 12886890, "router_z_loss_clip": 0.33398438, "router_z_loss_mlp": 1.484375, "step": 611, "time_per_iteration": 3.8363935947418213 }, { "auxiliary_loss_clip": 0.01190653, "auxiliary_loss_mlp": 0.01072803, "balance_loss_clip": 1.03725433, "balance_loss_mlp": 1.04390585, "epoch": 0.036795430632797235, "flos": 24680485987200.0, "grad_norm": 2.235179936131584, "language_loss": 0.72158015, "learning_rate": 3.986735396578956e-06, "loss": 0.74421477, "num_input_tokens_seen": 12906130, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.46875, "step": 612, "time_per_iteration": 2.440546989440918 }, { "auxiliary_loss_clip": 0.01189836, "auxiliary_loss_mlp": 0.01059669, "balance_loss_clip": 1.02497888, "balance_loss_mlp": 1.04274333, "epoch": 0.0368555538854652, "flos": 17747957203200.0, "grad_norm": 3.122582402275691, "language_loss": 0.79163623, "learning_rate": 3.986691918763034e-06, "loss": 0.81413126, "num_input_tokens_seen": 12925260, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.46875, "step": 613, "time_per_iteration": 2.392113208770752 }, { "auxiliary_loss_clip": 0.01188545, "auxiliary_loss_mlp": 0.01073452, "balance_loss_clip": 1.0383811, "balance_loss_mlp": 1.04238963, "epoch": 0.03691567713813317, "flos": 20593518255360.0, "grad_norm": 1.9597988037603629, "language_loss": 0.93362963, "learning_rate": 3.98664837004719e-06, "loss": 0.95624959, "num_input_tokens_seen": 12944590, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.4609375, "step": 614, "time_per_iteration": 2.461533308029175 }, { "auxiliary_loss_clip": 0.01193101, "auxiliary_loss_mlp": 0.01070644, "balance_loss_clip": 1.03383267, "balance_loss_mlp": 1.04623306, "epoch": 0.036975800390801145, "flos": 33649149795840.0, "grad_norm": 2.718999563947092, "language_loss": 0.73057652, "learning_rate": 3.986604750432974e-06, "loss": 0.753214, "num_input_tokens_seen": 12964785, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.46875, "step": 615, "time_per_iteration": 2.5033388137817383 }, { "auxiliary_loss_clip": 0.01193439, "auxiliary_loss_mlp": 0.01063049, "balance_loss_clip": 1.02983749, "balance_loss_mlp": 1.04255581, "epoch": 0.03703592364346911, "flos": 28292425493760.0, "grad_norm": 2.5644905920844607, "language_loss": 0.81399232, "learning_rate": 3.986561059921947e-06, "loss": 0.83655715, "num_input_tokens_seen": 12986705, "router_z_loss_clip": 0.33203125, "router_z_loss_mlp": 1.5078125, "step": 616, "time_per_iteration": 2.496835231781006 }, { "auxiliary_loss_clip": 0.0118991, "auxiliary_loss_mlp": 0.01069732, "balance_loss_clip": 1.03745008, "balance_loss_mlp": 1.04359233, "epoch": 0.03709604689613708, "flos": 31502863831680.0, "grad_norm": 2.289703141175505, "language_loss": 0.67923647, "learning_rate": 3.986517298515664e-06, "loss": 0.70183289, "num_input_tokens_seen": 13010560, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.4609375, "step": 617, "time_per_iteration": 2.4922900199890137 }, { "auxiliary_loss_clip": 0.01193598, "auxiliary_loss_mlp": 0.01065017, "balance_loss_clip": 1.02799022, "balance_loss_mlp": 1.04717958, "epoch": 0.03715617014880505, "flos": 19608374021760.0, "grad_norm": 2.4143698713390025, "language_loss": 0.79980433, "learning_rate": 3.9864734662156884e-06, "loss": 0.82239044, "num_input_tokens_seen": 13028935, "router_z_loss_clip": 0.37109375, "router_z_loss_mlp": 1.46875, "step": 618, "time_per_iteration": 2.4274299144744873 }, { "auxiliary_loss_clip": 0.01195582, "auxiliary_loss_mlp": 0.01072538, "balance_loss_clip": 1.03458214, "balance_loss_mlp": 1.04337478, "epoch": 0.03721629340147302, "flos": 15923291483520.0, "grad_norm": 2.799342678041712, "language_loss": 0.91307116, "learning_rate": 3.986429563023585e-06, "loss": 0.93575239, "num_input_tokens_seen": 13046000, "router_z_loss_clip": 0.37890625, "router_z_loss_mlp": 1.5234375, "step": 619, "time_per_iteration": 2.3912317752838135 }, { "auxiliary_loss_clip": 0.01192505, "auxiliary_loss_mlp": 0.01069496, "balance_loss_clip": 1.03688025, "balance_loss_mlp": 1.04688096, "epoch": 0.03727641665414099, "flos": 21103075681920.0, "grad_norm": 2.9199639193863978, "language_loss": 0.94099218, "learning_rate": 3.986385588940921e-06, "loss": 0.9636122, "num_input_tokens_seen": 13062995, "router_z_loss_clip": 0.32617188, "router_z_loss_mlp": 1.453125, "step": 620, "time_per_iteration": 2.392190933227539 }, { "auxiliary_loss_clip": 0.0118776, "auxiliary_loss_mlp": 0.0106845, "balance_loss_clip": 1.03087556, "balance_loss_mlp": 1.04218006, "epoch": 0.037336539906808956, "flos": 24130604073600.0, "grad_norm": 1.7018149861947345, "language_loss": 0.76863194, "learning_rate": 3.986341543969264e-06, "loss": 0.79119402, "num_input_tokens_seen": 13084120, "router_z_loss_clip": 0.375, "router_z_loss_mlp": 1.453125, "step": 621, "time_per_iteration": 2.441282033920288 }, { "auxiliary_loss_clip": 0.01191084, "auxiliary_loss_mlp": 0.01061931, "balance_loss_clip": 1.02786076, "balance_loss_mlp": 1.04571021, "epoch": 0.03739666315947693, "flos": 22345285322880.0, "grad_norm": 2.809079720400529, "language_loss": 0.8644613, "learning_rate": 3.986297428110187e-06, "loss": 0.88699144, "num_input_tokens_seen": 13100035, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.453125, "step": 622, "time_per_iteration": 2.4315948486328125 }, { "auxiliary_loss_clip": 0.01194275, "auxiliary_loss_mlp": 0.01059122, "balance_loss_clip": 1.02452755, "balance_loss_mlp": 1.04649282, "epoch": 0.0374567864121449, "flos": 20448454089600.0, "grad_norm": 2.2307814029927964, "language_loss": 0.89798784, "learning_rate": 3.986253241365264e-06, "loss": 0.9205218, "num_input_tokens_seen": 13118070, "router_z_loss_clip": 0.34570312, "router_z_loss_mlp": 1.484375, "step": 623, "time_per_iteration": 2.4165964126586914 }, { "auxiliary_loss_clip": 0.01193625, "auxiliary_loss_mlp": 0.01071854, "balance_loss_clip": 1.03711653, "balance_loss_mlp": 1.04731357, "epoch": 0.037516909664812866, "flos": 19207047409920.0, "grad_norm": 1.8172144217237507, "language_loss": 0.84119725, "learning_rate": 3.986208983736073e-06, "loss": 0.86385202, "num_input_tokens_seen": 13136355, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.4609375, "step": 624, "time_per_iteration": 2.416917562484741 }, { "auxiliary_loss_clip": 0.01191925, "auxiliary_loss_mlp": 0.01053709, "balance_loss_clip": 1.01808918, "balance_loss_mlp": 1.04193032, "epoch": 0.03757703291748084, "flos": 35003814526080.0, "grad_norm": 3.2844482048489367, "language_loss": 0.66283631, "learning_rate": 3.986164655224191e-06, "loss": 0.68529266, "num_input_tokens_seen": 13155435, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.5, "step": 625, "time_per_iteration": 2.5336241722106934 }, { "auxiliary_loss_clip": 0.01185041, "auxiliary_loss_mlp": 0.01067365, "balance_loss_clip": 1.03231764, "balance_loss_mlp": 1.04417443, "epoch": 0.0376371561701488, "flos": 25482720274560.0, "grad_norm": 2.101032365646545, "language_loss": 0.7704007, "learning_rate": 3.986120255831202e-06, "loss": 0.79292476, "num_input_tokens_seen": 13174295, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.40625, "step": 626, "time_per_iteration": 2.439168930053711 }, { "auxiliary_loss_clip": 0.0118929, "auxiliary_loss_mlp": 0.01064691, "balance_loss_clip": 1.02992964, "balance_loss_mlp": 1.04543984, "epoch": 0.037697279422816775, "flos": 18184685800320.0, "grad_norm": 1.7948574546239324, "language_loss": 0.81407958, "learning_rate": 3.986075785558691e-06, "loss": 0.83661938, "num_input_tokens_seen": 13192500, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.4375, "step": 627, "time_per_iteration": 2.3719449043273926 }, { "auxiliary_loss_clip": 0.01192957, "auxiliary_loss_mlp": 0.01069615, "balance_loss_clip": 1.03344655, "balance_loss_mlp": 1.04659152, "epoch": 0.03775740267548475, "flos": 24643128965760.0, "grad_norm": 1.630872127286863, "language_loss": 0.88502806, "learning_rate": 3.986031244408243e-06, "loss": 0.90765381, "num_input_tokens_seen": 13213470, "router_z_loss_clip": 0.36132812, "router_z_loss_mlp": 1.4609375, "step": 628, "time_per_iteration": 2.4616754055023193 }, { "auxiliary_loss_clip": 0.01187756, "auxiliary_loss_mlp": 0.01060824, "balance_loss_clip": 1.02670586, "balance_loss_mlp": 1.04001284, "epoch": 0.03781752592815271, "flos": 21287137703040.0, "grad_norm": 3.1648139815741545, "language_loss": 0.79559755, "learning_rate": 3.985986632381449e-06, "loss": 0.81808335, "num_input_tokens_seen": 13232365, "router_z_loss_clip": 0.34179688, "router_z_loss_mlp": 1.4765625, "step": 629, "time_per_iteration": 2.4048831462860107 }, { "auxiliary_loss_clip": 0.0118929, "auxiliary_loss_mlp": 0.01059391, "balance_loss_clip": 1.02613187, "balance_loss_mlp": 1.04272497, "epoch": 0.037877649180820684, "flos": 22088569029120.0, "grad_norm": 4.483336565305342, "language_loss": 0.76847458, "learning_rate": 3.9859419494799e-06, "loss": 0.79096138, "num_input_tokens_seen": 13251920, "router_z_loss_clip": 0.33203125, "router_z_loss_mlp": 1.46875, "step": 630, "time_per_iteration": 2.4224445819854736 }, { "auxiliary_loss_clip": 0.0119296, "auxiliary_loss_mlp": 0.01068971, "balance_loss_clip": 1.03406703, "balance_loss_mlp": 1.04508269, "epoch": 0.03793777243348865, "flos": 14500476046080.0, "grad_norm": 3.327037065085722, "language_loss": 0.91509634, "learning_rate": 3.985897195705192e-06, "loss": 0.93771565, "num_input_tokens_seen": 13267440, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.484375, "step": 631, "time_per_iteration": 2.3660621643066406 }, { "auxiliary_loss_clip": 0.0118907, "auxiliary_loss_mlp": 0.01076706, "balance_loss_clip": 1.04106295, "balance_loss_mlp": 1.04433274, "epoch": 0.03799789568615662, "flos": 21907334828160.0, "grad_norm": 1.6012248644307439, "language_loss": 0.91935283, "learning_rate": 3.985852371058921e-06, "loss": 0.94201052, "num_input_tokens_seen": 13287850, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.4453125, "step": 632, "time_per_iteration": 2.4410040378570557 }, { "auxiliary_loss_clip": 0.01187046, "auxiliary_loss_mlp": 0.01058762, "balance_loss_clip": 1.02428651, "balance_loss_mlp": 1.04275036, "epoch": 0.03805801893882459, "flos": 24825864355200.0, "grad_norm": 1.8387130062056452, "language_loss": 0.83061844, "learning_rate": 3.985807475542687e-06, "loss": 0.85307658, "num_input_tokens_seen": 13307760, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.4453125, "step": 633, "time_per_iteration": 2.44438099861145 }, { "auxiliary_loss_clip": 0.0118841, "auxiliary_loss_mlp": 0.01060442, "balance_loss_clip": 1.02737331, "balance_loss_mlp": 1.04320788, "epoch": 0.03811814219149256, "flos": 30481619385600.0, "grad_norm": 1.6646039138205775, "language_loss": 0.69604558, "learning_rate": 3.985762509158093e-06, "loss": 0.71853411, "num_input_tokens_seen": 13331230, "router_z_loss_clip": 0.33007812, "router_z_loss_mlp": 1.453125, "step": 634, "time_per_iteration": 2.516108989715576 }, { "auxiliary_loss_clip": 0.0107484, "auxiliary_loss_mlp": 0.01026803, "balance_loss_clip": 1.01931655, "balance_loss_mlp": 1.01821148, "epoch": 0.03817826544416053, "flos": 66989561639040.0, "grad_norm": 0.9040725694158229, "language_loss": 0.61635339, "learning_rate": 3.985717471906742e-06, "loss": 0.63736987, "num_input_tokens_seen": 13394760, "router_z_loss_clip": 0.07470703, "router_z_loss_mlp": 0.56640625, "step": 635, "time_per_iteration": 3.0856924057006836 }, { "auxiliary_loss_clip": 0.01185486, "auxiliary_loss_mlp": 0.01057715, "balance_loss_clip": 1.02440786, "balance_loss_mlp": 1.04052567, "epoch": 0.038238388696828496, "flos": 20484309922560.0, "grad_norm": 2.7305143207672726, "language_loss": 0.83529603, "learning_rate": 3.985672363790243e-06, "loss": 0.857728, "num_input_tokens_seen": 13412775, "router_z_loss_clip": 0.33203125, "router_z_loss_mlp": 1.4453125, "step": 636, "time_per_iteration": 2.430511951446533 }, { "auxiliary_loss_clip": 0.01187786, "auxiliary_loss_mlp": 0.01063098, "balance_loss_clip": 1.02938521, "balance_loss_mlp": 1.04468215, "epoch": 0.03829851194949647, "flos": 17964977414400.0, "grad_norm": 2.82889058687413, "language_loss": 0.79160106, "learning_rate": 3.985627184810206e-06, "loss": 0.81410992, "num_input_tokens_seen": 13427835, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.4296875, "step": 637, "time_per_iteration": 2.394228219985962 }, { "auxiliary_loss_clip": 0.01189064, "auxiliary_loss_mlp": 0.01074372, "balance_loss_clip": 1.03841865, "balance_loss_mlp": 1.04261327, "epoch": 0.03835863520216444, "flos": 22455401351040.0, "grad_norm": 2.291592706612894, "language_loss": 0.83631814, "learning_rate": 3.985581934968241e-06, "loss": 0.85895246, "num_input_tokens_seen": 13447295, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.46875, "step": 638, "time_per_iteration": 2.4217000007629395 }, { "auxiliary_loss_clip": 0.01196848, "auxiliary_loss_mlp": 0.01064665, "balance_loss_clip": 1.02909291, "balance_loss_mlp": 1.04514432, "epoch": 0.038418758454832405, "flos": 22163317983360.0, "grad_norm": 3.242665113678473, "language_loss": 0.70392871, "learning_rate": 3.985536614265964e-06, "loss": 0.72654378, "num_input_tokens_seen": 13468455, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.515625, "step": 639, "time_per_iteration": 2.434626579284668 }, { "auxiliary_loss_clip": 0.01188056, "auxiliary_loss_mlp": 0.01069986, "balance_loss_clip": 1.03379369, "balance_loss_mlp": 1.04202294, "epoch": 0.03847888170750038, "flos": 22746332643840.0, "grad_norm": 5.606862574968034, "language_loss": 0.84624588, "learning_rate": 3.985491222704994e-06, "loss": 0.86882633, "num_input_tokens_seen": 13489085, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.4609375, "step": 640, "time_per_iteration": 2.431072235107422 }, { "auxiliary_loss_clip": 0.01191819, "auxiliary_loss_mlp": 0.01069098, "balance_loss_clip": 1.03369319, "balance_loss_mlp": 1.04466319, "epoch": 0.03853900496016834, "flos": 22710092785920.0, "grad_norm": 2.7125576891372547, "language_loss": 0.82238823, "learning_rate": 3.985445760286949e-06, "loss": 0.84499741, "num_input_tokens_seen": 13509120, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.4765625, "step": 641, "time_per_iteration": 2.419487714767456 }, { "auxiliary_loss_clip": 0.01068748, "auxiliary_loss_mlp": 0.01012281, "balance_loss_clip": 1.00498581, "balance_loss_mlp": 1.01523471, "epoch": 0.038599128212836314, "flos": 70395652569600.0, "grad_norm": 0.8889064780781849, "language_loss": 0.65465635, "learning_rate": 3.985400227013452e-06, "loss": 0.67546666, "num_input_tokens_seen": 13562005, "router_z_loss_clip": 0.07275391, "router_z_loss_mlp": 0.53515625, "step": 642, "time_per_iteration": 3.0247628688812256 }, { "auxiliary_loss_clip": 0.01191587, "auxiliary_loss_mlp": 0.01053832, "balance_loss_clip": 1.02100134, "balance_loss_mlp": 1.04353416, "epoch": 0.03865925146550429, "flos": 23294015141760.0, "grad_norm": 1.994719867029607, "language_loss": 0.79217535, "learning_rate": 3.985354622886128e-06, "loss": 0.81462955, "num_input_tokens_seen": 13582185, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.484375, "step": 643, "time_per_iteration": 2.430391788482666 }, { "auxiliary_loss_clip": 0.01188071, "auxiliary_loss_mlp": 0.01072142, "balance_loss_clip": 1.03945529, "balance_loss_mlp": 1.04187131, "epoch": 0.03871937471817225, "flos": 21429478782720.0, "grad_norm": 1.7826475929274195, "language_loss": 0.82554638, "learning_rate": 3.985308947906604e-06, "loss": 0.84814858, "num_input_tokens_seen": 13599555, "router_z_loss_clip": 0.32617188, "router_z_loss_mlp": 1.46875, "step": 644, "time_per_iteration": 2.419905185699463 }, { "auxiliary_loss_clip": 0.01190707, "auxiliary_loss_mlp": 0.01065234, "balance_loss_clip": 1.02885127, "balance_loss_mlp": 1.04173517, "epoch": 0.038779497970840224, "flos": 34275875345280.0, "grad_norm": 2.5220844419508697, "language_loss": 0.82106018, "learning_rate": 3.985263202076511e-06, "loss": 0.84361959, "num_input_tokens_seen": 13621160, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.484375, "step": 645, "time_per_iteration": 3.919546604156494 }, { "auxiliary_loss_clip": 0.01194109, "auxiliary_loss_mlp": 0.01069637, "balance_loss_clip": 1.03518534, "balance_loss_mlp": 1.04296374, "epoch": 0.03883962122350819, "flos": 22747065782400.0, "grad_norm": 2.5742157379080894, "language_loss": 0.81492043, "learning_rate": 3.985217385397481e-06, "loss": 0.83755791, "num_input_tokens_seen": 13641915, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.515625, "step": 646, "time_per_iteration": 2.4276058673858643 }, { "auxiliary_loss_clip": 0.01192965, "auxiliary_loss_mlp": 0.01076988, "balance_loss_clip": 1.04070067, "balance_loss_mlp": 1.04868424, "epoch": 0.03889974447617616, "flos": 21944726760960.0, "grad_norm": 1.7673884490273624, "language_loss": 0.81530958, "learning_rate": 3.985171497871149e-06, "loss": 0.83800912, "num_input_tokens_seen": 13661410, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.4453125, "step": 647, "time_per_iteration": 2.4281909465789795 }, { "auxiliary_loss_clip": 0.01189059, "auxiliary_loss_mlp": 0.01066854, "balance_loss_clip": 1.03259313, "balance_loss_mlp": 1.04276097, "epoch": 0.03895986772884413, "flos": 31503457324800.0, "grad_norm": 1.9542556086114053, "language_loss": 0.8414427, "learning_rate": 3.985125539499152e-06, "loss": 0.86400187, "num_input_tokens_seen": 13681705, "router_z_loss_clip": 0.34179688, "router_z_loss_mlp": 1.46875, "step": 648, "time_per_iteration": 5.353910207748413 }, { "auxiliary_loss_clip": 0.01189546, "auxiliary_loss_mlp": 0.01059021, "balance_loss_clip": 1.02609563, "balance_loss_mlp": 1.04522252, "epoch": 0.0390199909815121, "flos": 19900003541760.0, "grad_norm": 2.0317345177524047, "language_loss": 0.84429526, "learning_rate": 3.9850795102831315e-06, "loss": 0.86678088, "num_input_tokens_seen": 13700400, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.4375, "step": 649, "time_per_iteration": 2.4122958183288574 }, { "auxiliary_loss_clip": 0.0118993, "auxiliary_loss_mlp": 0.01065205, "balance_loss_clip": 1.02891779, "balance_loss_mlp": 1.04358125, "epoch": 0.03908011423418007, "flos": 21611515944960.0, "grad_norm": 1.8540114561548637, "language_loss": 0.79612905, "learning_rate": 3.9850334102247295e-06, "loss": 0.81868041, "num_input_tokens_seen": 13720145, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.4609375, "step": 650, "time_per_iteration": 2.431824207305908 }, { "auxiliary_loss_clip": 0.01184543, "auxiliary_loss_mlp": 0.01065113, "balance_loss_clip": 1.03082883, "balance_loss_mlp": 1.04022264, "epoch": 0.039140237486848035, "flos": 18660412252800.0, "grad_norm": 2.2190536894804413, "language_loss": 0.78213829, "learning_rate": 3.984987239325592e-06, "loss": 0.80463487, "num_input_tokens_seen": 13737500, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.4453125, "step": 651, "time_per_iteration": 3.855437994003296 }, { "auxiliary_loss_clip": 0.01187028, "auxiliary_loss_mlp": 0.01068847, "balance_loss_clip": 1.0326786, "balance_loss_mlp": 1.04069173, "epoch": 0.03920036073951601, "flos": 18660132961920.0, "grad_norm": 3.7514277523504167, "language_loss": 0.87278444, "learning_rate": 3.984940997587364e-06, "loss": 0.89534318, "num_input_tokens_seen": 13754750, "router_z_loss_clip": 0.36132812, "router_z_loss_mlp": 1.4609375, "step": 652, "time_per_iteration": 2.42366099357605 }, { "auxiliary_loss_clip": 0.01177951, "auxiliary_loss_mlp": 0.01062097, "balance_loss_clip": 1.02807426, "balance_loss_mlp": 1.0395844, "epoch": 0.03926048399218398, "flos": 31353226277760.0, "grad_norm": 2.587133442199089, "language_loss": 0.79192305, "learning_rate": 3.984894685011699e-06, "loss": 0.8143236, "num_input_tokens_seen": 13771990, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.3828125, "step": 653, "time_per_iteration": 2.447777271270752 }, { "auxiliary_loss_clip": 0.01190905, "auxiliary_loss_mlp": 0.01072108, "balance_loss_clip": 1.03312695, "balance_loss_mlp": 1.04188657, "epoch": 0.039320607244851945, "flos": 29602297082880.0, "grad_norm": 2.3614461284920583, "language_loss": 0.8583045, "learning_rate": 3.984848301600248e-06, "loss": 0.8809346, "num_input_tokens_seen": 13792750, "router_z_loss_clip": 0.390625, "router_z_loss_mlp": 1.4921875, "step": 654, "time_per_iteration": 2.454594612121582 }, { "auxiliary_loss_clip": 0.01065384, "auxiliary_loss_mlp": 0.01015938, "balance_loss_clip": 1.00940573, "balance_loss_mlp": 1.01152527, "epoch": 0.03938073049751992, "flos": 66531151221120.0, "grad_norm": 0.7151695118935018, "language_loss": 0.49906549, "learning_rate": 3.984801847354667e-06, "loss": 0.51987869, "num_input_tokens_seen": 13858570, "router_z_loss_clip": 0.06542969, "router_z_loss_mlp": 0.5390625, "step": 655, "time_per_iteration": 3.1038155555725098 }, { "auxiliary_loss_clip": 0.01184583, "auxiliary_loss_mlp": 0.01064343, "balance_loss_clip": 1.03072572, "balance_loss_mlp": 1.0431056, "epoch": 0.03944085375018788, "flos": 23366704325760.0, "grad_norm": 2.3215592261136413, "language_loss": 0.80955482, "learning_rate": 3.984755322276614e-06, "loss": 0.83204401, "num_input_tokens_seen": 13876335, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.4140625, "step": 656, "time_per_iteration": 2.4134862422943115 }, { "auxiliary_loss_clip": 0.01196188, "auxiliary_loss_mlp": 0.01077883, "balance_loss_clip": 1.03923488, "balance_loss_mlp": 1.04710519, "epoch": 0.039500977002855854, "flos": 18547398581760.0, "grad_norm": 2.5540636579470912, "language_loss": 0.76357615, "learning_rate": 3.9847087263677485e-06, "loss": 0.78631687, "num_input_tokens_seen": 13892640, "router_z_loss_clip": 0.38671875, "router_z_loss_mlp": 1.484375, "step": 657, "time_per_iteration": 2.357095718383789 }, { "auxiliary_loss_clip": 0.01192952, "auxiliary_loss_mlp": 0.01061956, "balance_loss_clip": 1.02576399, "balance_loss_mlp": 1.04402542, "epoch": 0.039561100255523826, "flos": 25336992792960.0, "grad_norm": 1.9004998410713654, "language_loss": 0.8134166, "learning_rate": 3.984662059629734e-06, "loss": 0.83596575, "num_input_tokens_seen": 13910085, "router_z_loss_clip": 0.36132812, "router_z_loss_mlp": 1.4921875, "step": 658, "time_per_iteration": 2.44077205657959 }, { "auxiliary_loss_clip": 0.01183464, "auxiliary_loss_mlp": 0.01061993, "balance_loss_clip": 1.0252049, "balance_loss_mlp": 1.04198444, "epoch": 0.03962122350819179, "flos": 18219005533440.0, "grad_norm": 2.0504285700224885, "language_loss": 0.9085809, "learning_rate": 3.984615322064235e-06, "loss": 0.93103546, "num_input_tokens_seen": 13928800, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.4140625, "step": 659, "time_per_iteration": 2.4118030071258545 }, { "auxiliary_loss_clip": 0.01187671, "auxiliary_loss_mlp": 0.01062738, "balance_loss_clip": 1.027619, "balance_loss_mlp": 1.04179525, "epoch": 0.03968134676085976, "flos": 20521178184960.0, "grad_norm": 2.7338367910896078, "language_loss": 0.78944838, "learning_rate": 3.9845685136729215e-06, "loss": 0.81195241, "num_input_tokens_seen": 13948325, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.453125, "step": 660, "time_per_iteration": 2.4234118461608887 }, { "auxiliary_loss_clip": 0.01187967, "auxiliary_loss_mlp": 0.01062019, "balance_loss_clip": 1.02594614, "balance_loss_mlp": 1.04496956, "epoch": 0.03974147001352773, "flos": 22421395820160.0, "grad_norm": 1.6276707879309493, "language_loss": 0.81347334, "learning_rate": 3.984521634457461e-06, "loss": 0.8359732, "num_input_tokens_seen": 13969090, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.4296875, "step": 661, "time_per_iteration": 2.444849729537964 }, { "auxiliary_loss_clip": 0.01060966, "auxiliary_loss_mlp": 0.01010318, "balance_loss_clip": 1.00388038, "balance_loss_mlp": 1.00876069, "epoch": 0.0398015932661957, "flos": 71125267495680.0, "grad_norm": 0.9225560296975938, "language_loss": 0.69447446, "learning_rate": 3.98447468441953e-06, "loss": 0.71518731, "num_input_tokens_seen": 14037555, "router_z_loss_clip": 0.06445312, "router_z_loss_mlp": 0.5234375, "step": 662, "time_per_iteration": 3.17541766166687 }, { "auxiliary_loss_clip": 0.01189675, "auxiliary_loss_mlp": 0.01070304, "balance_loss_clip": 1.03501832, "balance_loss_mlp": 1.0454495, "epoch": 0.03986171651886367, "flos": 16799995434240.0, "grad_norm": 1.8352192519331945, "language_loss": 0.82945752, "learning_rate": 3.984427663560801e-06, "loss": 0.85205734, "num_input_tokens_seen": 14055765, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.4453125, "step": 663, "time_per_iteration": 2.4322595596313477 }, { "auxiliary_loss_clip": 0.01194363, "auxiliary_loss_mlp": 0.01058858, "balance_loss_clip": 1.02304792, "balance_loss_mlp": 1.04646635, "epoch": 0.03992183977153164, "flos": 24533920632960.0, "grad_norm": 2.3611688473755743, "language_loss": 0.87116724, "learning_rate": 3.984380571882954e-06, "loss": 0.89369941, "num_input_tokens_seen": 14074195, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.4765625, "step": 664, "time_per_iteration": 2.4412331581115723 }, { "auxiliary_loss_clip": 0.01183807, "auxiliary_loss_mlp": 0.01064496, "balance_loss_clip": 1.0302825, "balance_loss_mlp": 1.0417701, "epoch": 0.03998196302419961, "flos": 15595003169280.0, "grad_norm": 2.1207161045014273, "language_loss": 0.84756935, "learning_rate": 3.984333409387668e-06, "loss": 0.8700524, "num_input_tokens_seen": 14090215, "router_z_loss_clip": 0.34179688, "router_z_loss_mlp": 1.421875, "step": 665, "time_per_iteration": 2.3960518836975098 }, { "auxiliary_loss_clip": 0.01195958, "auxiliary_loss_mlp": 0.01069138, "balance_loss_clip": 1.03170609, "balance_loss_mlp": 1.04581308, "epoch": 0.04004208627686758, "flos": 25303790223360.0, "grad_norm": 2.1601006873638107, "language_loss": 0.81672788, "learning_rate": 3.984286176076628e-06, "loss": 0.83937883, "num_input_tokens_seen": 14112150, "router_z_loss_clip": 0.375, "router_z_loss_mlp": 1.5, "step": 666, "time_per_iteration": 2.4813365936279297 }, { "auxiliary_loss_clip": 0.01185139, "auxiliary_loss_mlp": 0.01059999, "balance_loss_clip": 1.02240062, "balance_loss_mlp": 1.04265499, "epoch": 0.04010220952953555, "flos": 23474760583680.0, "grad_norm": 1.9467012559641645, "language_loss": 0.86658657, "learning_rate": 3.984238871951518e-06, "loss": 0.88903797, "num_input_tokens_seen": 14131475, "router_z_loss_clip": 0.37695312, "router_z_loss_mlp": 1.421875, "step": 667, "time_per_iteration": 2.4209389686584473 }, { "auxiliary_loss_clip": 0.01183022, "auxiliary_loss_mlp": 0.01060967, "balance_loss_clip": 1.02634835, "balance_loss_mlp": 1.04454565, "epoch": 0.04016233278220352, "flos": 18616247516160.0, "grad_norm": 2.0724305921822808, "language_loss": 0.80607831, "learning_rate": 3.984191497014026e-06, "loss": 0.82851821, "num_input_tokens_seen": 14146165, "router_z_loss_clip": 0.34570312, "router_z_loss_mlp": 1.3828125, "step": 668, "time_per_iteration": 2.380335807800293 }, { "auxiliary_loss_clip": 0.01057149, "auxiliary_loss_mlp": 0.01008952, "balance_loss_clip": 1.00287223, "balance_loss_mlp": 1.00661552, "epoch": 0.040222456034871484, "flos": 70902801112320.0, "grad_norm": 0.7811409650925238, "language_loss": 0.6007818, "learning_rate": 3.984144051265844e-06, "loss": 0.62144279, "num_input_tokens_seen": 14215005, "router_z_loss_clip": 0.06079102, "router_z_loss_mlp": 0.5078125, "step": 669, "time_per_iteration": 3.1673338413238525 }, { "auxiliary_loss_clip": 0.01183658, "auxiliary_loss_mlp": 0.01063918, "balance_loss_clip": 1.02872753, "balance_loss_mlp": 1.04043889, "epoch": 0.040282579287539456, "flos": 23763701928960.0, "grad_norm": 1.7426617425744348, "language_loss": 0.86253875, "learning_rate": 3.984096534708665e-06, "loss": 0.88501447, "num_input_tokens_seen": 14235510, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.4296875, "step": 670, "time_per_iteration": 2.4509003162384033 }, { "auxiliary_loss_clip": 0.01184797, "auxiliary_loss_mlp": 0.01061892, "balance_loss_clip": 1.02713096, "balance_loss_mlp": 1.04237092, "epoch": 0.04034270254020743, "flos": 18477537217920.0, "grad_norm": 6.661894128042471, "language_loss": 0.74786806, "learning_rate": 3.9840489473441835e-06, "loss": 0.77033496, "num_input_tokens_seen": 14254565, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.4296875, "step": 671, "time_per_iteration": 2.4203484058380127 }, { "auxiliary_loss_clip": 0.01189964, "auxiliary_loss_mlp": 0.01067379, "balance_loss_clip": 1.03228378, "balance_loss_mlp": 1.04595947, "epoch": 0.040402825792875394, "flos": 17200903109760.0, "grad_norm": 1.921792455658059, "language_loss": 0.92102182, "learning_rate": 3.984001289174099e-06, "loss": 0.94359517, "num_input_tokens_seen": 14271885, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.4375, "step": 672, "time_per_iteration": 2.3875322341918945 }, { "auxiliary_loss_clip": 0.01188382, "auxiliary_loss_mlp": 0.01067851, "balance_loss_clip": 1.03127718, "balance_loss_mlp": 1.04510617, "epoch": 0.040462949045543366, "flos": 19171156665600.0, "grad_norm": 5.893165256633166, "language_loss": 0.90170169, "learning_rate": 3.983953560200113e-06, "loss": 0.92426401, "num_input_tokens_seen": 14289670, "router_z_loss_clip": 0.36523438, "router_z_loss_mlp": 1.4375, "step": 673, "time_per_iteration": 2.4174492359161377 }, { "auxiliary_loss_clip": 0.01184043, "auxiliary_loss_mlp": 0.01070975, "balance_loss_clip": 1.03382957, "balance_loss_mlp": 1.04199457, "epoch": 0.04052307229821133, "flos": 24018812300160.0, "grad_norm": 1.8851288699257294, "language_loss": 0.74678195, "learning_rate": 3.983905760423926e-06, "loss": 0.76933217, "num_input_tokens_seen": 14309285, "router_z_loss_clip": 0.37109375, "router_z_loss_mlp": 1.421875, "step": 674, "time_per_iteration": 2.4453983306884766 }, { "auxiliary_loss_clip": 0.01192146, "auxiliary_loss_mlp": 0.01056601, "balance_loss_clip": 1.01995611, "balance_loss_mlp": 1.043841, "epoch": 0.0405831955508793, "flos": 16435641818880.0, "grad_norm": 2.672433531864122, "language_loss": 0.77962393, "learning_rate": 3.983857889847247e-06, "loss": 0.80211139, "num_input_tokens_seen": 14328300, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.484375, "step": 675, "time_per_iteration": 2.4712469577789307 }, { "auxiliary_loss_clip": 0.01188736, "auxiliary_loss_mlp": 0.01067069, "balance_loss_clip": 1.03259361, "balance_loss_mlp": 1.04343069, "epoch": 0.040643318803547275, "flos": 24278775350400.0, "grad_norm": 1.777188058958025, "language_loss": 0.76703358, "learning_rate": 3.983809948471783e-06, "loss": 0.78959161, "num_input_tokens_seen": 14346395, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.453125, "step": 676, "time_per_iteration": 2.42793607711792 }, { "auxiliary_loss_clip": 0.01190334, "auxiliary_loss_mlp": 0.01066163, "balance_loss_clip": 1.03082967, "balance_loss_mlp": 1.04389369, "epoch": 0.04070344205621524, "flos": 17711123852160.0, "grad_norm": 2.550943853737293, "language_loss": 0.84916627, "learning_rate": 3.983761936299245e-06, "loss": 0.87173128, "num_input_tokens_seen": 14364605, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.4609375, "step": 677, "time_per_iteration": 2.4127068519592285 }, { "auxiliary_loss_clip": 0.01185319, "auxiliary_loss_mlp": 0.01062526, "balance_loss_clip": 1.02757335, "balance_loss_mlp": 1.04432964, "epoch": 0.04076356530888321, "flos": 26176444456320.0, "grad_norm": 1.9381617410757228, "language_loss": 0.76106936, "learning_rate": 3.983713853331345e-06, "loss": 0.78354776, "num_input_tokens_seen": 14385265, "router_z_loss_clip": 0.34960938, "router_z_loss_mlp": 1.40625, "step": 678, "time_per_iteration": 2.5064332485198975 }, { "auxiliary_loss_clip": 0.01187, "auxiliary_loss_mlp": 0.01064472, "balance_loss_clip": 1.028018, "balance_loss_mlp": 1.04322159, "epoch": 0.04082368856155118, "flos": 35771973459840.0, "grad_norm": 1.9634592798462205, "language_loss": 0.82002586, "learning_rate": 3.9836656995698015e-06, "loss": 0.84254062, "num_input_tokens_seen": 14406090, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.4375, "step": 679, "time_per_iteration": 2.528810739517212 }, { "auxiliary_loss_clip": 0.01190761, "auxiliary_loss_mlp": 0.01058053, "balance_loss_clip": 1.02450764, "balance_loss_mlp": 1.04950869, "epoch": 0.04088381181421915, "flos": 28145406291840.0, "grad_norm": 3.685127405500079, "language_loss": 0.76211154, "learning_rate": 3.983617475016331e-06, "loss": 0.78459966, "num_input_tokens_seen": 14425130, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.4140625, "step": 680, "time_per_iteration": 2.4731523990631104 }, { "auxiliary_loss_clip": 0.01187026, "auxiliary_loss_mlp": 0.01062935, "balance_loss_clip": 1.02447796, "balance_loss_mlp": 1.03947425, "epoch": 0.04094393506688712, "flos": 27596501896320.0, "grad_norm": 1.9793853535666257, "language_loss": 0.83050603, "learning_rate": 3.9835691796726555e-06, "loss": 0.85300565, "num_input_tokens_seen": 14447355, "router_z_loss_clip": 0.38476562, "router_z_loss_mlp": 1.4765625, "step": 681, "time_per_iteration": 2.4579238891601562 }, { "auxiliary_loss_clip": 0.01188714, "auxiliary_loss_mlp": 0.01065756, "balance_loss_clip": 1.02758527, "balance_loss_mlp": 1.04244184, "epoch": 0.04100405831955509, "flos": 23110930638720.0, "grad_norm": 1.850859883141676, "language_loss": 0.71165198, "learning_rate": 3.9835208135404986e-06, "loss": 0.73419666, "num_input_tokens_seen": 14466790, "router_z_loss_clip": 0.3828125, "router_z_loss_mlp": 1.4609375, "step": 682, "time_per_iteration": 2.428924798965454 }, { "auxiliary_loss_clip": 0.01183112, "auxiliary_loss_mlp": 0.01065471, "balance_loss_clip": 1.0303278, "balance_loss_mlp": 1.04071558, "epoch": 0.04106418157222306, "flos": 20155707406080.0, "grad_norm": 1.6317417738527569, "language_loss": 0.72059846, "learning_rate": 3.9834723766215865e-06, "loss": 0.74308419, "num_input_tokens_seen": 14485195, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.421875, "step": 683, "time_per_iteration": 2.3904831409454346 }, { "auxiliary_loss_clip": 0.01184685, "auxiliary_loss_mlp": 0.0106614, "balance_loss_clip": 1.03223693, "balance_loss_mlp": 1.04592633, "epoch": 0.041124304824891024, "flos": 17419738711680.0, "grad_norm": 2.182818138980505, "language_loss": 0.81072485, "learning_rate": 3.983423868917646e-06, "loss": 0.83323312, "num_input_tokens_seen": 14503370, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.3828125, "step": 684, "time_per_iteration": 2.383819818496704 }, { "auxiliary_loss_clip": 0.01188177, "auxiliary_loss_mlp": 0.01062482, "balance_loss_clip": 1.02621865, "balance_loss_mlp": 1.04425693, "epoch": 0.041184428077558996, "flos": 25778853360000.0, "grad_norm": 1.6602237229884422, "language_loss": 0.9059425, "learning_rate": 3.983375290430411e-06, "loss": 0.92844909, "num_input_tokens_seen": 14526415, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.4375, "step": 685, "time_per_iteration": 3.942558526992798 }, { "auxiliary_loss_clip": 0.01185363, "auxiliary_loss_mlp": 0.01061075, "balance_loss_clip": 1.0242871, "balance_loss_mlp": 1.04252374, "epoch": 0.04124455133022697, "flos": 22963701968640.0, "grad_norm": 2.020637774355877, "language_loss": 0.88082665, "learning_rate": 3.983326641161613e-06, "loss": 0.90329105, "num_input_tokens_seen": 14546595, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.4296875, "step": 686, "time_per_iteration": 2.41579270362854 }, { "auxiliary_loss_clip": 0.01187174, "auxiliary_loss_mlp": 0.01067066, "balance_loss_clip": 1.02963471, "balance_loss_mlp": 1.04243541, "epoch": 0.04130467458289493, "flos": 21287975575680.0, "grad_norm": 1.8269890377548201, "language_loss": 0.71391737, "learning_rate": 3.9832779211129894e-06, "loss": 0.73645979, "num_input_tokens_seen": 14566590, "router_z_loss_clip": 0.375, "router_z_loss_mlp": 1.453125, "step": 687, "time_per_iteration": 2.451077461242676 }, { "auxiliary_loss_clip": 0.01183612, "auxiliary_loss_mlp": 0.01060901, "balance_loss_clip": 1.02704561, "balance_loss_mlp": 1.046556, "epoch": 0.041364797835562905, "flos": 19973216396160.0, "grad_norm": 1.5390372221479989, "language_loss": 0.8611179, "learning_rate": 3.983229130286278e-06, "loss": 0.88356304, "num_input_tokens_seen": 14585965, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.3671875, "step": 688, "time_per_iteration": 5.2253546714782715 }, { "auxiliary_loss_clip": 0.01181434, "auxiliary_loss_mlp": 0.01070965, "balance_loss_clip": 1.03534508, "balance_loss_mlp": 1.04390609, "epoch": 0.04142492108823087, "flos": 21905205235200.0, "grad_norm": 1.8610843488901465, "language_loss": 0.83315575, "learning_rate": 3.98318026868322e-06, "loss": 0.85567975, "num_input_tokens_seen": 14606015, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.375, "step": 689, "time_per_iteration": 2.4173686504364014 }, { "auxiliary_loss_clip": 0.01183212, "auxiliary_loss_mlp": 0.01069871, "balance_loss_clip": 1.03622985, "balance_loss_mlp": 1.04125214, "epoch": 0.04148504434089884, "flos": 27638292660480.0, "grad_norm": 2.3858573184890948, "language_loss": 0.68026263, "learning_rate": 3.9831313363055606e-06, "loss": 0.70279348, "num_input_tokens_seen": 14629955, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.421875, "step": 690, "time_per_iteration": 3.858558416366577 }, { "auxiliary_loss_clip": 0.01178753, "auxiliary_loss_mlp": 0.01064627, "balance_loss_clip": 1.02952015, "balance_loss_mlp": 1.03993392, "epoch": 0.041545167593566815, "flos": 20517442669440.0, "grad_norm": 2.23165164324267, "language_loss": 0.74733639, "learning_rate": 3.9830823331550445e-06, "loss": 0.76977026, "num_input_tokens_seen": 14648000, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.390625, "step": 691, "time_per_iteration": 2.3846516609191895 }, { "auxiliary_loss_clip": 0.01179734, "auxiliary_loss_mlp": 0.01071205, "balance_loss_clip": 1.03522789, "balance_loss_mlp": 1.04067516, "epoch": 0.04160529084623478, "flos": 11868269512320.0, "grad_norm": 2.2544470318593404, "language_loss": 0.84076923, "learning_rate": 3.983033259233421e-06, "loss": 0.86327863, "num_input_tokens_seen": 14662235, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.390625, "step": 692, "time_per_iteration": 2.367363214492798 }, { "auxiliary_loss_clip": 0.01186203, "auxiliary_loss_mlp": 0.01064559, "balance_loss_clip": 1.02877247, "balance_loss_mlp": 1.04257679, "epoch": 0.04166541409890275, "flos": 14827472640000.0, "grad_norm": 2.7512087519687785, "language_loss": 0.88303667, "learning_rate": 3.982984114542442e-06, "loss": 0.90554428, "num_input_tokens_seen": 14676065, "router_z_loss_clip": 0.35742188, "router_z_loss_mlp": 1.4375, "step": 693, "time_per_iteration": 2.3654260635375977 }, { "auxiliary_loss_clip": 0.01184472, "auxiliary_loss_mlp": 0.01060751, "balance_loss_clip": 1.02789724, "balance_loss_mlp": 1.04375386, "epoch": 0.04172553735157072, "flos": 25807063605120.0, "grad_norm": 2.1493026141193754, "language_loss": 0.81644607, "learning_rate": 3.98293489908386e-06, "loss": 0.8388983, "num_input_tokens_seen": 14694955, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.40625, "step": 694, "time_per_iteration": 2.4572255611419678 }, { "auxiliary_loss_clip": 0.01185629, "auxiliary_loss_mlp": 0.01062164, "balance_loss_clip": 1.02723527, "balance_loss_mlp": 1.04134369, "epoch": 0.04178566060423869, "flos": 24278670616320.0, "grad_norm": 1.9838667020409235, "language_loss": 0.8338263, "learning_rate": 3.982885612859432e-06, "loss": 0.85630423, "num_input_tokens_seen": 14715510, "router_z_loss_clip": 0.34960938, "router_z_loss_mlp": 1.4375, "step": 695, "time_per_iteration": 2.423081398010254 }, { "auxiliary_loss_clip": 0.01187447, "auxiliary_loss_mlp": 0.0107011, "balance_loss_clip": 1.03351307, "balance_loss_mlp": 1.0442071, "epoch": 0.04184578385690666, "flos": 18221065303680.0, "grad_norm": 2.0993753783977223, "language_loss": 0.84214848, "learning_rate": 3.982836255870918e-06, "loss": 0.86472404, "num_input_tokens_seen": 14731755, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.4296875, "step": 696, "time_per_iteration": 2.3784396648406982 }, { "auxiliary_loss_clip": 0.01182217, "auxiliary_loss_mlp": 0.01071365, "balance_loss_clip": 1.03605509, "balance_loss_mlp": 1.04098535, "epoch": 0.041905907109574626, "flos": 22775450584320.0, "grad_norm": 2.124818166614912, "language_loss": 0.9306224, "learning_rate": 3.982786828120078e-06, "loss": 0.95315826, "num_input_tokens_seen": 14750810, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.4140625, "step": 697, "time_per_iteration": 2.40679669380188 }, { "auxiliary_loss_clip": 0.01178436, "auxiliary_loss_mlp": 0.01060785, "balance_loss_clip": 1.02650046, "balance_loss_mlp": 1.04041481, "epoch": 0.0419660303622426, "flos": 20155916874240.0, "grad_norm": 2.274826421908768, "language_loss": 0.8352983, "learning_rate": 3.982737329608676e-06, "loss": 0.85769051, "num_input_tokens_seen": 14768435, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.3828125, "step": 698, "time_per_iteration": 2.4244003295898438 }, { "auxiliary_loss_clip": 0.01183744, "auxiliary_loss_mlp": 0.01074651, "balance_loss_clip": 1.03903079, "balance_loss_mlp": 1.04241085, "epoch": 0.042026153614910564, "flos": 23075249362560.0, "grad_norm": 2.40989884291235, "language_loss": 0.91279924, "learning_rate": 3.98268776033848e-06, "loss": 0.9353832, "num_input_tokens_seen": 14786690, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.4140625, "step": 699, "time_per_iteration": 2.4093446731567383 }, { "auxiliary_loss_clip": 0.0106356, "auxiliary_loss_mlp": 0.01011575, "balance_loss_clip": 1.00621021, "balance_loss_mlp": 1.01348925, "epoch": 0.042086276867578536, "flos": 64491734528640.0, "grad_norm": 0.8829531456366362, "language_loss": 0.67870784, "learning_rate": 3.9826381203112575e-06, "loss": 0.6994592, "num_input_tokens_seen": 14853840, "router_z_loss_clip": 0.05371094, "router_z_loss_mlp": 0.5, "step": 700, "time_per_iteration": 3.081566333770752 }, { "auxiliary_loss_clip": 0.01188067, "auxiliary_loss_mlp": 0.01067494, "balance_loss_clip": 1.02786875, "balance_loss_mlp": 1.04447913, "epoch": 0.04214640012024651, "flos": 15486109038720.0, "grad_norm": 2.4898711189618576, "language_loss": 0.88516855, "learning_rate": 3.98258840952878e-06, "loss": 0.90772414, "num_input_tokens_seen": 14869580, "router_z_loss_clip": 0.39648438, "router_z_loss_mlp": 1.4375, "step": 701, "time_per_iteration": 2.3736653327941895 }, { "auxiliary_loss_clip": 0.01184988, "auxiliary_loss_mlp": 0.0107199, "balance_loss_clip": 1.03656125, "balance_loss_mlp": 1.04638183, "epoch": 0.04220652337291447, "flos": 23875947550080.0, "grad_norm": 1.7072433387743238, "language_loss": 0.67324317, "learning_rate": 3.982538627992822e-06, "loss": 0.69581294, "num_input_tokens_seen": 14891065, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.390625, "step": 702, "time_per_iteration": 2.4374215602874756 }, { "auxiliary_loss_clip": 0.0105983, "auxiliary_loss_mlp": 0.01005343, "balance_loss_clip": 0.99969298, "balance_loss_mlp": 1.00805283, "epoch": 0.042266646625582445, "flos": 63792145238400.0, "grad_norm": 0.83391300942417, "language_loss": 0.60691524, "learning_rate": 3.98248877570516e-06, "loss": 0.62756693, "num_input_tokens_seen": 14954815, "router_z_loss_clip": 0.05639648, "router_z_loss_mlp": 0.515625, "step": 703, "time_per_iteration": 3.142807722091675 }, { "auxiliary_loss_clip": 0.01056487, "auxiliary_loss_mlp": 0.01007253, "balance_loss_clip": 1.00181758, "balance_loss_mlp": 1.00678504, "epoch": 0.04232676987825041, "flos": 50015521877760.0, "grad_norm": 1.0100126866570873, "language_loss": 0.57689762, "learning_rate": 3.982438852667574e-06, "loss": 0.59753501, "num_input_tokens_seen": 15003050, "router_z_loss_clip": 0.05444336, "router_z_loss_mlp": 0.49609375, "step": 704, "time_per_iteration": 2.9380300045013428 }, { "auxiliary_loss_clip": 0.01187186, "auxiliary_loss_mlp": 0.01060862, "balance_loss_clip": 1.02455127, "balance_loss_mlp": 1.04617953, "epoch": 0.04238689313091838, "flos": 21615041992320.0, "grad_norm": 2.3619916673472745, "language_loss": 0.87409616, "learning_rate": 3.982388858881844e-06, "loss": 0.89657664, "num_input_tokens_seen": 15021990, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.40625, "step": 705, "time_per_iteration": 2.4390175342559814 }, { "auxiliary_loss_clip": 0.01174888, "auxiliary_loss_mlp": 0.01060965, "balance_loss_clip": 1.02885032, "balance_loss_mlp": 1.04063582, "epoch": 0.042447016383586354, "flos": 19134113846400.0, "grad_norm": 1.8293907438004477, "language_loss": 0.71343666, "learning_rate": 3.982338794349755e-06, "loss": 0.7357952, "num_input_tokens_seen": 15040700, "router_z_loss_clip": 0.3203125, "router_z_loss_mlp": 1.34375, "step": 706, "time_per_iteration": 2.386361598968506 }, { "auxiliary_loss_clip": 0.01177439, "auxiliary_loss_mlp": 0.01066396, "balance_loss_clip": 1.02932167, "balance_loss_mlp": 1.04061365, "epoch": 0.04250713963625432, "flos": 24424851945600.0, "grad_norm": 2.0036013581762693, "language_loss": 0.93354023, "learning_rate": 3.982288659073094e-06, "loss": 0.95597857, "num_input_tokens_seen": 15056725, "router_z_loss_clip": 0.37109375, "router_z_loss_mlp": 1.3671875, "step": 707, "time_per_iteration": 2.44316029548645 }, { "auxiliary_loss_clip": 0.01181426, "auxiliary_loss_mlp": 0.01070727, "balance_loss_clip": 1.03467846, "balance_loss_mlp": 1.04033101, "epoch": 0.04256726288892229, "flos": 30366231742080.0, "grad_norm": 2.361387935515631, "language_loss": 0.8126626, "learning_rate": 3.98223845305365e-06, "loss": 0.8351841, "num_input_tokens_seen": 15077550, "router_z_loss_clip": 0.36132812, "router_z_loss_mlp": 1.40625, "step": 708, "time_per_iteration": 2.48256254196167 }, { "auxiliary_loss_clip": 0.01187485, "auxiliary_loss_mlp": 0.01067734, "balance_loss_clip": 1.03063631, "balance_loss_mlp": 1.04343319, "epoch": 0.04262738614159026, "flos": 16361730737280.0, "grad_norm": 2.66574885580616, "language_loss": 0.81993365, "learning_rate": 3.982188176293213e-06, "loss": 0.84248579, "num_input_tokens_seen": 15094955, "router_z_loss_clip": 0.37109375, "router_z_loss_mlp": 1.4375, "step": 709, "time_per_iteration": 2.407764434814453 }, { "auxiliary_loss_clip": 0.0118863, "auxiliary_loss_mlp": 0.0105774, "balance_loss_clip": 1.0218581, "balance_loss_mlp": 1.04400229, "epoch": 0.04268750939425823, "flos": 20411341447680.0, "grad_norm": 2.3436319116749598, "language_loss": 0.84847897, "learning_rate": 3.982137828793581e-06, "loss": 0.87094259, "num_input_tokens_seen": 15113395, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.4453125, "step": 710, "time_per_iteration": 2.4123568534851074 }, { "auxiliary_loss_clip": 0.0118838, "auxiliary_loss_mlp": 0.01067121, "balance_loss_clip": 1.03183496, "balance_loss_mlp": 1.04673469, "epoch": 0.0427476326469262, "flos": 20301923646720.0, "grad_norm": 2.682782371000687, "language_loss": 0.84520423, "learning_rate": 3.982087410556547e-06, "loss": 0.86775929, "num_input_tokens_seen": 15132920, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.421875, "step": 711, "time_per_iteration": 2.4092376232147217 }, { "auxiliary_loss_clip": 0.01179871, "auxiliary_loss_mlp": 0.01063639, "balance_loss_clip": 1.02687526, "balance_loss_mlp": 1.04195905, "epoch": 0.042807755899594166, "flos": 21649780661760.0, "grad_norm": 1.7518729085008558, "language_loss": 0.85324287, "learning_rate": 3.982036921583912e-06, "loss": 0.875678, "num_input_tokens_seen": 15153115, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.375, "step": 712, "time_per_iteration": 2.4427237510681152 }, { "auxiliary_loss_clip": 0.01187882, "auxiliary_loss_mlp": 0.01059618, "balance_loss_clip": 1.02557182, "balance_loss_mlp": 1.04309583, "epoch": 0.04286787915226214, "flos": 21433912525440.0, "grad_norm": 3.1329119544886876, "language_loss": 0.91045451, "learning_rate": 3.981986361877479e-06, "loss": 0.93292952, "num_input_tokens_seen": 15172770, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.453125, "step": 713, "time_per_iteration": 2.4084465503692627 }, { "auxiliary_loss_clip": 0.01058632, "auxiliary_loss_mlp": 0.01018882, "balance_loss_clip": 1.01318336, "balance_loss_mlp": 1.00909543, "epoch": 0.04292800240493011, "flos": 66394256313600.0, "grad_norm": 0.8888646137103391, "language_loss": 0.63704062, "learning_rate": 3.9819357314390494e-06, "loss": 0.65781581, "num_input_tokens_seen": 15240055, "router_z_loss_clip": 0.05688477, "router_z_loss_mlp": 0.49609375, "step": 714, "time_per_iteration": 3.1690354347229004 }, { "auxiliary_loss_clip": 0.01181426, "auxiliary_loss_mlp": 0.01074384, "balance_loss_clip": 1.04050469, "balance_loss_mlp": 1.04453063, "epoch": 0.042988125657598075, "flos": 31648905515520.0, "grad_norm": 2.1541672162311065, "language_loss": 0.74600798, "learning_rate": 3.981885030270432e-06, "loss": 0.76856601, "num_input_tokens_seen": 15261585, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.375, "step": 715, "time_per_iteration": 2.5548620223999023 }, { "auxiliary_loss_clip": 0.01186135, "auxiliary_loss_mlp": 0.01066759, "balance_loss_clip": 1.02880299, "balance_loss_mlp": 1.04575384, "epoch": 0.04304824891026605, "flos": 33247264602240.0, "grad_norm": 1.8331696717597785, "language_loss": 0.72439748, "learning_rate": 3.981834258373437e-06, "loss": 0.74692643, "num_input_tokens_seen": 15281160, "router_z_loss_clip": 0.37890625, "router_z_loss_mlp": 1.40625, "step": 716, "time_per_iteration": 2.5122344493865967 }, { "auxiliary_loss_clip": 0.01051827, "auxiliary_loss_mlp": 0.01005289, "balance_loss_clip": 0.99956697, "balance_loss_mlp": 1.00391102, "epoch": 0.04310837216293401, "flos": 64061080508160.0, "grad_norm": 0.9005979170358152, "language_loss": 0.65497255, "learning_rate": 3.981783415749874e-06, "loss": 0.67554367, "num_input_tokens_seen": 15344505, "router_z_loss_clip": 0.05712891, "router_z_loss_mlp": 0.48046875, "step": 717, "time_per_iteration": 3.0917444229125977 }, { "auxiliary_loss_clip": 0.01050883, "auxiliary_loss_mlp": 0.01005029, "balance_loss_clip": 0.99964118, "balance_loss_mlp": 1.00407958, "epoch": 0.043168495415601985, "flos": 61340719057920.0, "grad_norm": 0.9752943296857631, "language_loss": 0.58790207, "learning_rate": 3.9817325024015596e-06, "loss": 0.6084612, "num_input_tokens_seen": 15404050, "router_z_loss_clip": 0.05395508, "router_z_loss_mlp": 0.46875, "step": 718, "time_per_iteration": 2.9039950370788574 }, { "auxiliary_loss_clip": 0.01183786, "auxiliary_loss_mlp": 0.01068357, "balance_loss_clip": 1.03123498, "balance_loss_mlp": 1.04652083, "epoch": 0.04322861866826996, "flos": 20703215347200.0, "grad_norm": 1.9132710050399087, "language_loss": 0.91328299, "learning_rate": 3.9816815183303086e-06, "loss": 0.93580437, "num_input_tokens_seen": 15424190, "router_z_loss_clip": 0.37109375, "router_z_loss_mlp": 1.375, "step": 719, "time_per_iteration": 2.425180435180664 }, { "auxiliary_loss_clip": 0.01181178, "auxiliary_loss_mlp": 0.01071679, "balance_loss_clip": 1.03806162, "balance_loss_mlp": 1.04153848, "epoch": 0.04328874192093792, "flos": 30372027027840.0, "grad_norm": 1.6259532171414055, "language_loss": 0.66515422, "learning_rate": 3.981630463537942e-06, "loss": 0.68768275, "num_input_tokens_seen": 15446500, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.3984375, "step": 720, "time_per_iteration": 2.474658250808716 }, { "auxiliary_loss_clip": 0.01180216, "auxiliary_loss_mlp": 0.01068949, "balance_loss_clip": 1.03428292, "balance_loss_mlp": 1.04478395, "epoch": 0.043348865173605894, "flos": 21943714331520.0, "grad_norm": 2.3921306796946364, "language_loss": 0.77202111, "learning_rate": 3.981579338026282e-06, "loss": 0.79451281, "num_input_tokens_seen": 15465830, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.359375, "step": 721, "time_per_iteration": 2.426039457321167 }, { "auxiliary_loss_clip": 0.01183751, "auxiliary_loss_mlp": 0.01077762, "balance_loss_clip": 1.04156971, "balance_loss_mlp": 1.04415536, "epoch": 0.04340898842627386, "flos": 15263433187200.0, "grad_norm": 2.822492763484581, "language_loss": 0.88540536, "learning_rate": 3.981528141797153e-06, "loss": 0.9080205, "num_input_tokens_seen": 15479985, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.390625, "step": 722, "time_per_iteration": 2.366525888442993 }, { "auxiliary_loss_clip": 0.01190227, "auxiliary_loss_mlp": 0.01067458, "balance_loss_clip": 1.03372216, "balance_loss_mlp": 1.04493773, "epoch": 0.04346911167894183, "flos": 27964172090880.0, "grad_norm": 1.9351530887289412, "language_loss": 0.84070444, "learning_rate": 3.981476874852382e-06, "loss": 0.86328125, "num_input_tokens_seen": 15501545, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.453125, "step": 723, "time_per_iteration": 2.4779179096221924 }, { "auxiliary_loss_clip": 0.01187966, "auxiliary_loss_mlp": 0.01072437, "balance_loss_clip": 1.03755665, "balance_loss_mlp": 1.04717469, "epoch": 0.0435292349316098, "flos": 29240910933120.0, "grad_norm": 1.9139788895422787, "language_loss": 0.82327592, "learning_rate": 3.981425537193796e-06, "loss": 0.84587997, "num_input_tokens_seen": 15521725, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.40625, "step": 724, "time_per_iteration": 3.8979430198669434 }, { "auxiliary_loss_clip": 0.01182732, "auxiliary_loss_mlp": 0.01068706, "balance_loss_clip": 1.03420663, "balance_loss_mlp": 1.04488754, "epoch": 0.04358935818427777, "flos": 20557313308800.0, "grad_norm": 1.8369900418297336, "language_loss": 0.79121196, "learning_rate": 3.981374128823232e-06, "loss": 0.81372637, "num_input_tokens_seen": 15540910, "router_z_loss_clip": 0.34570312, "router_z_loss_mlp": 1.3828125, "step": 725, "time_per_iteration": 2.4161336421966553 }, { "auxiliary_loss_clip": 0.01194352, "auxiliary_loss_mlp": 0.01072789, "balance_loss_clip": 1.03611982, "balance_loss_mlp": 1.04817867, "epoch": 0.04364948143694574, "flos": 14464061631360.0, "grad_norm": 2.12562964942191, "language_loss": 0.86453843, "learning_rate": 3.981322649742521e-06, "loss": 0.88720989, "num_input_tokens_seen": 15558640, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.4609375, "step": 726, "time_per_iteration": 2.3939080238342285 }, { "auxiliary_loss_clip": 0.01053085, "auxiliary_loss_mlp": 0.01004662, "balance_loss_clip": 0.99972701, "balance_loss_mlp": 1.00666237, "epoch": 0.043709604689613706, "flos": 50064610982400.0, "grad_norm": 0.9096007450487622, "language_loss": 0.55918157, "learning_rate": 3.9812710999535005e-06, "loss": 0.579759, "num_input_tokens_seen": 15612975, "router_z_loss_clip": 0.04931641, "router_z_loss_mlp": 0.46484375, "step": 727, "time_per_iteration": 4.51263689994812 }, { "auxiliary_loss_clip": 0.01187576, "auxiliary_loss_mlp": 0.01065319, "balance_loss_clip": 1.02750552, "balance_loss_mlp": 1.04635787, "epoch": 0.04376972794228168, "flos": 13990709151360.0, "grad_norm": 1.8794382856077294, "language_loss": 0.81984973, "learning_rate": 3.981219479458012e-06, "loss": 0.84237874, "num_input_tokens_seen": 15631070, "router_z_loss_clip": 0.37890625, "router_z_loss_mlp": 1.4140625, "step": 728, "time_per_iteration": 3.7680346965789795 }, { "auxiliary_loss_clip": 0.01179165, "auxiliary_loss_mlp": 0.01065663, "balance_loss_clip": 1.03216577, "balance_loss_mlp": 1.04445708, "epoch": 0.04382985119494965, "flos": 22009037218560.0, "grad_norm": 2.383410342674767, "language_loss": 0.76899624, "learning_rate": 3.981167788257896e-06, "loss": 0.79144454, "num_input_tokens_seen": 15647825, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.34375, "step": 729, "time_per_iteration": 3.801180124282837 }, { "auxiliary_loss_clip": 0.01184388, "auxiliary_loss_mlp": 0.01062557, "balance_loss_clip": 1.02722335, "balance_loss_mlp": 1.04415679, "epoch": 0.043889974447617615, "flos": 24205387939200.0, "grad_norm": 2.001423814994023, "language_loss": 0.9496327, "learning_rate": 3.9811160263549985e-06, "loss": 0.97210211, "num_input_tokens_seen": 15668260, "router_z_loss_clip": 0.35351562, "router_z_loss_mlp": 1.40625, "step": 730, "time_per_iteration": 2.4248015880584717 }, { "auxiliary_loss_clip": 0.01182056, "auxiliary_loss_mlp": 0.0106814, "balance_loss_clip": 1.03128076, "balance_loss_mlp": 1.04220426, "epoch": 0.04395009770028559, "flos": 17273592293760.0, "grad_norm": 2.3730318760653777, "language_loss": 0.8861438, "learning_rate": 3.981064193751166e-06, "loss": 0.90864581, "num_input_tokens_seen": 15685630, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.3984375, "step": 731, "time_per_iteration": 2.3986494541168213 }, { "auxiliary_loss_clip": 0.01182096, "auxiliary_loss_mlp": 0.01063611, "balance_loss_clip": 1.02999365, "balance_loss_mlp": 1.04369128, "epoch": 0.04401022095295355, "flos": 12309536586240.0, "grad_norm": 2.8991978654045716, "language_loss": 0.88705492, "learning_rate": 3.981012290448247e-06, "loss": 0.90951192, "num_input_tokens_seen": 15698645, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.3828125, "step": 732, "time_per_iteration": 2.382936716079712 }, { "auxiliary_loss_clip": 0.0118338, "auxiliary_loss_mlp": 0.01062163, "balance_loss_clip": 1.02690125, "balance_loss_mlp": 1.04321599, "epoch": 0.044070344205621524, "flos": 20958605009280.0, "grad_norm": 2.0845642292686395, "language_loss": 0.86170357, "learning_rate": 3.980960316448097e-06, "loss": 0.88415903, "num_input_tokens_seen": 15716775, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.40625, "step": 733, "time_per_iteration": 2.4018895626068115 }, { "auxiliary_loss_clip": 0.01187338, "auxiliary_loss_mlp": 0.01066768, "balance_loss_clip": 1.03024256, "balance_loss_mlp": 1.0461601, "epoch": 0.044130467458289496, "flos": 13844423088000.0, "grad_norm": 4.290017182560329, "language_loss": 0.90916038, "learning_rate": 3.980908271752567e-06, "loss": 0.93170148, "num_input_tokens_seen": 15733320, "router_z_loss_clip": 0.36523438, "router_z_loss_mlp": 1.4140625, "step": 734, "time_per_iteration": 2.3619699478149414 }, { "auxiliary_loss_clip": 0.01180765, "auxiliary_loss_mlp": 0.01060107, "balance_loss_clip": 1.02715731, "balance_loss_mlp": 1.04557741, "epoch": 0.04419059071095746, "flos": 28653881466240.0, "grad_norm": 1.915025457554586, "language_loss": 0.77842975, "learning_rate": 3.980856156363518e-06, "loss": 0.80083847, "num_input_tokens_seen": 15752705, "router_z_loss_clip": 0.33007812, "router_z_loss_mlp": 1.3515625, "step": 735, "time_per_iteration": 2.490703582763672 }, { "auxiliary_loss_clip": 0.01177451, "auxiliary_loss_mlp": 0.01060491, "balance_loss_clip": 1.0279355, "balance_loss_mlp": 1.04102802, "epoch": 0.04425071396362543, "flos": 28182065086080.0, "grad_norm": 2.359563242556638, "language_loss": 0.88532102, "learning_rate": 3.980803970282806e-06, "loss": 0.90770042, "num_input_tokens_seen": 15772800, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.3671875, "step": 736, "time_per_iteration": 2.4679150581359863 }, { "auxiliary_loss_clip": 0.01180427, "auxiliary_loss_mlp": 0.01066654, "balance_loss_clip": 1.03225017, "balance_loss_mlp": 1.046525, "epoch": 0.0443108372162934, "flos": 23657356327680.0, "grad_norm": 1.934699423655556, "language_loss": 0.84254616, "learning_rate": 3.980751713512298e-06, "loss": 0.86501706, "num_input_tokens_seen": 15793665, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.34375, "step": 737, "time_per_iteration": 2.4656014442443848 }, { "auxiliary_loss_clip": 0.01185789, "auxiliary_loss_mlp": 0.01069531, "balance_loss_clip": 1.03195643, "balance_loss_mlp": 1.04592919, "epoch": 0.04437096046896137, "flos": 33978590184960.0, "grad_norm": 1.8838707245702677, "language_loss": 0.84660316, "learning_rate": 3.980699386053855e-06, "loss": 0.86915642, "num_input_tokens_seen": 15813175, "router_z_loss_clip": 0.375, "router_z_loss_mlp": 1.3984375, "step": 738, "time_per_iteration": 2.5087547302246094 }, { "auxiliary_loss_clip": 0.01054058, "auxiliary_loss_mlp": 0.0100866, "balance_loss_clip": 1.00396276, "balance_loss_mlp": 1.00775146, "epoch": 0.04443108372162934, "flos": 67394379386880.0, "grad_norm": 0.8607474266972598, "language_loss": 0.59154689, "learning_rate": 3.9806469879093465e-06, "loss": 0.61217415, "num_input_tokens_seen": 15872050, "router_z_loss_clip": 0.046875, "router_z_loss_mlp": 0.46289062, "step": 739, "time_per_iteration": 3.008528470993042 }, { "auxiliary_loss_clip": 0.01178647, "auxiliary_loss_mlp": 0.01063742, "balance_loss_clip": 1.03043461, "balance_loss_mlp": 1.0452528, "epoch": 0.04449120697429731, "flos": 29751376055040.0, "grad_norm": 2.074928094832132, "language_loss": 0.90996939, "learning_rate": 3.9805945190806415e-06, "loss": 0.93239331, "num_input_tokens_seen": 15891085, "router_z_loss_clip": 0.33203125, "router_z_loss_mlp": 1.3359375, "step": 740, "time_per_iteration": 2.461949348449707 }, { "auxiliary_loss_clip": 0.01184099, "auxiliary_loss_mlp": 0.01063316, "balance_loss_clip": 1.02836418, "balance_loss_mlp": 1.04532051, "epoch": 0.04455133022696528, "flos": 36500645779200.0, "grad_norm": 1.9530878257015465, "language_loss": 0.71967971, "learning_rate": 3.980541979569614e-06, "loss": 0.74215388, "num_input_tokens_seen": 15914225, "router_z_loss_clip": 0.34960938, "router_z_loss_mlp": 1.390625, "step": 741, "time_per_iteration": 2.5428102016448975 }, { "auxiliary_loss_clip": 0.01177469, "auxiliary_loss_mlp": 0.01066288, "balance_loss_clip": 1.03188419, "balance_loss_mlp": 1.04174197, "epoch": 0.044611453479633245, "flos": 28802401856640.0, "grad_norm": 1.9194179518673538, "language_loss": 0.88805389, "learning_rate": 3.980489369378136e-06, "loss": 0.91049147, "num_input_tokens_seen": 15934540, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.359375, "step": 742, "time_per_iteration": 2.461562395095825 }, { "auxiliary_loss_clip": 0.01177628, "auxiliary_loss_mlp": 0.01059539, "balance_loss_clip": 1.0233475, "balance_loss_mlp": 1.04239082, "epoch": 0.04467157673230122, "flos": 20009945013120.0, "grad_norm": 1.8610295912199888, "language_loss": 0.83681965, "learning_rate": 3.980436688508087e-06, "loss": 0.8591913, "num_input_tokens_seen": 15952560, "router_z_loss_clip": 0.36132812, "router_z_loss_mlp": 1.3515625, "step": 743, "time_per_iteration": 2.410418748855591 }, { "auxiliary_loss_clip": 0.01180885, "auxiliary_loss_mlp": 0.0106705, "balance_loss_clip": 1.031955, "balance_loss_mlp": 1.04375339, "epoch": 0.04473169998496919, "flos": 18003975269760.0, "grad_norm": 2.0392217011253617, "language_loss": 0.79766238, "learning_rate": 3.980383936961348e-06, "loss": 0.82014179, "num_input_tokens_seen": 15970620, "router_z_loss_clip": 0.3515625, "router_z_loss_mlp": 1.375, "step": 744, "time_per_iteration": 2.3963050842285156 }, { "auxiliary_loss_clip": 0.01175583, "auxiliary_loss_mlp": 0.01060617, "balance_loss_clip": 1.02821624, "balance_loss_mlp": 1.04345059, "epoch": 0.044791823237637154, "flos": 20630665808640.0, "grad_norm": 2.062579014162858, "language_loss": 0.85017085, "learning_rate": 3.980331114739799e-06, "loss": 0.87253284, "num_input_tokens_seen": 15987325, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.3203125, "step": 745, "time_per_iteration": 2.41471791267395 }, { "auxiliary_loss_clip": 0.01179399, "auxiliary_loss_mlp": 0.01054924, "balance_loss_clip": 1.0201149, "balance_loss_mlp": 1.04224062, "epoch": 0.04485194649030513, "flos": 31174819896960.0, "grad_norm": 1.8628480270544208, "language_loss": 0.68768948, "learning_rate": 3.980278221845328e-06, "loss": 0.7100327, "num_input_tokens_seen": 16008310, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.375, "step": 746, "time_per_iteration": 2.485377788543701 }, { "auxiliary_loss_clip": 0.01184604, "auxiliary_loss_mlp": 0.01069791, "balance_loss_clip": 1.03326559, "balance_loss_mlp": 1.04763985, "epoch": 0.04491206974297309, "flos": 26142019989120.0, "grad_norm": 3.637756553710533, "language_loss": 0.68110108, "learning_rate": 3.98022525827982e-06, "loss": 0.70364505, "num_input_tokens_seen": 16029620, "router_z_loss_clip": 0.36523438, "router_z_loss_mlp": 1.3671875, "step": 747, "time_per_iteration": 2.474304437637329 }, { "auxiliary_loss_clip": 0.01187419, "auxiliary_loss_mlp": 0.01082806, "balance_loss_clip": 1.04728198, "balance_loss_mlp": 1.04606819, "epoch": 0.044972192995641064, "flos": 20666626375680.0, "grad_norm": 2.2752135269186105, "language_loss": 0.66599447, "learning_rate": 3.980172224045168e-06, "loss": 0.68869674, "num_input_tokens_seen": 16049065, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.4140625, "step": 748, "time_per_iteration": 2.415919303894043 }, { "auxiliary_loss_clip": 0.01183825, "auxiliary_loss_mlp": 0.0107098, "balance_loss_clip": 1.03421593, "balance_loss_mlp": 1.04668474, "epoch": 0.045032316248309036, "flos": 16105922138880.0, "grad_norm": 3.0247764736933203, "language_loss": 0.76647866, "learning_rate": 3.980119119143262e-06, "loss": 0.78902674, "num_input_tokens_seen": 16066765, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.3671875, "step": 749, "time_per_iteration": 2.461630344390869 }, { "auxiliary_loss_clip": 0.01184256, "auxiliary_loss_mlp": 0.01067013, "balance_loss_clip": 1.03275204, "balance_loss_mlp": 1.04713106, "epoch": 0.045092439500977, "flos": 17857863763200.0, "grad_norm": 1.9949543408587387, "language_loss": 0.88806438, "learning_rate": 3.980065943575998e-06, "loss": 0.91057712, "num_input_tokens_seen": 16085980, "router_z_loss_clip": 0.34179688, "router_z_loss_mlp": 1.375, "step": 750, "time_per_iteration": 2.389357089996338 }, { "auxiliary_loss_clip": 0.01185133, "auxiliary_loss_mlp": 0.01076102, "balance_loss_clip": 1.03704882, "balance_loss_mlp": 1.0451926, "epoch": 0.04515256275364497, "flos": 24461650385280.0, "grad_norm": 4.792346322114513, "language_loss": 0.74504662, "learning_rate": 3.9800126973452725e-06, "loss": 0.76765895, "num_input_tokens_seen": 16106260, "router_z_loss_clip": 0.390625, "router_z_loss_mlp": 1.3984375, "step": 751, "time_per_iteration": 2.466661214828491 }, { "auxiliary_loss_clip": 0.01177467, "auxiliary_loss_mlp": 0.01066612, "balance_loss_clip": 1.02989578, "balance_loss_mlp": 1.04091656, "epoch": 0.04521268600631294, "flos": 20915522524800.0, "grad_norm": 1.8841999039596504, "language_loss": 0.68607342, "learning_rate": 3.979959380452989e-06, "loss": 0.70851421, "num_input_tokens_seen": 16123475, "router_z_loss_clip": 0.3671875, "router_z_loss_mlp": 1.359375, "step": 752, "time_per_iteration": 2.3971285820007324 }, { "auxiliary_loss_clip": 0.01178181, "auxiliary_loss_mlp": 0.01054287, "balance_loss_clip": 1.01993108, "balance_loss_mlp": 1.04174387, "epoch": 0.04527280925898091, "flos": 13370512026240.0, "grad_norm": 2.5533987417203603, "language_loss": 0.9229058, "learning_rate": 3.979905992901047e-06, "loss": 0.94523054, "num_input_tokens_seen": 16138335, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.359375, "step": 753, "time_per_iteration": 2.3955495357513428 }, { "auxiliary_loss_clip": 0.01183752, "auxiliary_loss_mlp": 0.01066287, "balance_loss_clip": 1.03231251, "balance_loss_mlp": 1.04663646, "epoch": 0.04533293251164888, "flos": 23253551009280.0, "grad_norm": 1.9030511128020393, "language_loss": 0.91005522, "learning_rate": 3.979852534691353e-06, "loss": 0.93255562, "num_input_tokens_seen": 16157110, "router_z_loss_clip": 0.33984375, "router_z_loss_mlp": 1.375, "step": 754, "time_per_iteration": 2.4177744388580322 }, { "auxiliary_loss_clip": 0.01172809, "auxiliary_loss_mlp": 0.01066015, "balance_loss_clip": 1.03161073, "balance_loss_mlp": 1.04385495, "epoch": 0.04539305576431685, "flos": 12421188714240.0, "grad_norm": 2.3406486534664896, "language_loss": 0.78643274, "learning_rate": 3.979799005825816e-06, "loss": 0.80882096, "num_input_tokens_seen": 16174155, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.2890625, "step": 755, "time_per_iteration": 2.388094902038574 }, { "auxiliary_loss_clip": 0.01182913, "auxiliary_loss_mlp": 0.01077339, "balance_loss_clip": 1.04102838, "balance_loss_mlp": 1.04511786, "epoch": 0.04545317901698482, "flos": 16070066305920.0, "grad_norm": 2.104218086446312, "language_loss": 0.78481936, "learning_rate": 3.979745406306345e-06, "loss": 0.80742186, "num_input_tokens_seen": 16192240, "router_z_loss_clip": 0.36328125, "router_z_loss_mlp": 1.375, "step": 756, "time_per_iteration": 2.375627279281616 }, { "auxiliary_loss_clip": 0.01054118, "auxiliary_loss_mlp": 0.0100707, "balance_loss_clip": 1.00187278, "balance_loss_mlp": 1.00968564, "epoch": 0.045513302269652785, "flos": 66392475834240.0, "grad_norm": 0.8078967859892556, "language_loss": 0.62762362, "learning_rate": 3.979691736134852e-06, "loss": 0.6482355, "num_input_tokens_seen": 16255775, "router_z_loss_clip": 0.05200195, "router_z_loss_mlp": 0.4453125, "step": 757, "time_per_iteration": 3.08660626411438 }, { "auxiliary_loss_clip": 0.01180996, "auxiliary_loss_mlp": 0.01061264, "balance_loss_clip": 1.02705073, "balance_loss_mlp": 1.04657936, "epoch": 0.04557342552232076, "flos": 21470082560640.0, "grad_norm": 2.0456393741536685, "language_loss": 0.84172112, "learning_rate": 3.979637995313254e-06, "loss": 0.86414373, "num_input_tokens_seen": 16277015, "router_z_loss_clip": 0.34179688, "router_z_loss_mlp": 1.34375, "step": 758, "time_per_iteration": 2.445692539215088 }, { "auxiliary_loss_clip": 0.01172712, "auxiliary_loss_mlp": 0.01062261, "balance_loss_clip": 1.02950215, "balance_loss_mlp": 1.03983974, "epoch": 0.04563354877498873, "flos": 23731546700160.0, "grad_norm": 1.9123058886883226, "language_loss": 0.88420147, "learning_rate": 3.979584183843468e-06, "loss": 0.90655118, "num_input_tokens_seen": 16296005, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.328125, "step": 759, "time_per_iteration": 2.4633443355560303 }, { "auxiliary_loss_clip": 0.01183593, "auxiliary_loss_mlp": 0.01063908, "balance_loss_clip": 1.02807355, "balance_loss_mlp": 1.04788387, "epoch": 0.045693672027656694, "flos": 25734653712000.0, "grad_norm": 2.305370252748593, "language_loss": 0.73975301, "learning_rate": 3.979530301727414e-06, "loss": 0.76222801, "num_input_tokens_seen": 16315300, "router_z_loss_clip": 0.35742188, "router_z_loss_mlp": 1.359375, "step": 760, "time_per_iteration": 2.4366261959075928 }, { "auxiliary_loss_clip": 0.01179764, "auxiliary_loss_mlp": 0.01055823, "balance_loss_clip": 1.02196741, "balance_loss_mlp": 1.0474813, "epoch": 0.045753795280324666, "flos": 19718001290880.0, "grad_norm": 1.965163134473522, "language_loss": 0.82210457, "learning_rate": 3.979476348967016e-06, "loss": 0.84446049, "num_input_tokens_seen": 16333820, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.328125, "step": 761, "time_per_iteration": 2.4243874549865723 }, { "auxiliary_loss_clip": 0.01175688, "auxiliary_loss_mlp": 0.01062024, "balance_loss_clip": 1.02773905, "balance_loss_mlp": 1.04460287, "epoch": 0.04581391853299264, "flos": 23254737995520.0, "grad_norm": 1.669881972747545, "language_loss": 0.7976234, "learning_rate": 3.979422325564199e-06, "loss": 0.82000047, "num_input_tokens_seen": 16355290, "router_z_loss_clip": 0.34179688, "router_z_loss_mlp": 1.3125, "step": 762, "time_per_iteration": 2.4524641036987305 }, { "auxiliary_loss_clip": 0.01048775, "auxiliary_loss_mlp": 0.01005676, "balance_loss_clip": 1.0008601, "balance_loss_mlp": 1.00495434, "epoch": 0.0458740417856606, "flos": 64227896317440.0, "grad_norm": 0.9980029867422425, "language_loss": 0.58720791, "learning_rate": 3.979368231520891e-06, "loss": 0.60775238, "num_input_tokens_seen": 16415995, "router_z_loss_clip": 0.0480957, "router_z_loss_mlp": 0.4375, "step": 763, "time_per_iteration": 3.0900869369506836 }, { "auxiliary_loss_clip": 0.01180184, "auxiliary_loss_mlp": 0.01071259, "balance_loss_clip": 1.03927493, "balance_loss_mlp": 1.04385817, "epoch": 0.045934165038328575, "flos": 20769271372800.0, "grad_norm": 2.0523051948717885, "language_loss": 0.87536454, "learning_rate": 3.979314066839022e-06, "loss": 0.89787894, "num_input_tokens_seen": 16433120, "router_z_loss_clip": 0.3203125, "router_z_loss_mlp": 1.359375, "step": 764, "time_per_iteration": 3.8769683837890625 }, { "auxiliary_loss_clip": 0.01179282, "auxiliary_loss_mlp": 0.01075631, "balance_loss_clip": 1.03967762, "balance_loss_mlp": 1.04498005, "epoch": 0.04599428829099654, "flos": 30261596797440.0, "grad_norm": 2.4330211626417233, "language_loss": 0.85370469, "learning_rate": 3.979259831520526e-06, "loss": 0.87625384, "num_input_tokens_seen": 16453360, "router_z_loss_clip": 0.359375, "router_z_loss_mlp": 1.34375, "step": 765, "time_per_iteration": 2.490962266921997 }, { "auxiliary_loss_clip": 0.01181964, "auxiliary_loss_mlp": 0.01070433, "balance_loss_clip": 1.03176188, "balance_loss_mlp": 1.04626715, "epoch": 0.04605441154366451, "flos": 23037822518400.0, "grad_norm": 2.832774509546428, "language_loss": 0.88183564, "learning_rate": 3.979205525567337e-06, "loss": 0.90435958, "num_input_tokens_seen": 16471160, "router_z_loss_clip": 0.38671875, "router_z_loss_mlp": 1.359375, "step": 766, "time_per_iteration": 2.4284579753875732 }, { "auxiliary_loss_clip": 0.01174969, "auxiliary_loss_mlp": 0.01072638, "balance_loss_clip": 1.03797174, "balance_loss_mlp": 1.04154038, "epoch": 0.046114534796332485, "flos": 22016333692800.0, "grad_norm": 13.411650425654186, "language_loss": 0.83985424, "learning_rate": 3.979151148981395e-06, "loss": 0.86233032, "num_input_tokens_seen": 16488940, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.3359375, "step": 767, "time_per_iteration": 3.95444393157959 }, { "auxiliary_loss_clip": 0.01178257, "auxiliary_loss_mlp": 0.01061193, "balance_loss_clip": 1.02736187, "balance_loss_mlp": 1.04422903, "epoch": 0.04617465804900045, "flos": 29861073146880.0, "grad_norm": 5.4049371492260905, "language_loss": 0.8675254, "learning_rate": 3.979096701764638e-06, "loss": 0.88991988, "num_input_tokens_seen": 16509505, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.34375, "step": 768, "time_per_iteration": 2.4905314445495605 }, { "auxiliary_loss_clip": 0.01176369, "auxiliary_loss_mlp": 0.01066782, "balance_loss_clip": 1.0337851, "balance_loss_mlp": 1.04094982, "epoch": 0.04623478130166842, "flos": 25628866692480.0, "grad_norm": 2.37342504986601, "language_loss": 0.75016659, "learning_rate": 3.979042183919012e-06, "loss": 0.77259809, "num_input_tokens_seen": 16528840, "router_z_loss_clip": 0.33007812, "router_z_loss_mlp": 1.3515625, "step": 769, "time_per_iteration": 3.8787219524383545 }, { "auxiliary_loss_clip": 0.01175254, "auxiliary_loss_mlp": 0.0106277, "balance_loss_clip": 1.02986789, "balance_loss_mlp": 1.04421747, "epoch": 0.04629490455433639, "flos": 20448035153280.0, "grad_norm": 40.211569394658184, "language_loss": 0.8622731, "learning_rate": 3.97898759544646e-06, "loss": 0.88465333, "num_input_tokens_seen": 16548335, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.3046875, "step": 770, "time_per_iteration": 2.41145920753479 }, { "auxiliary_loss_clip": 0.01179355, "auxiliary_loss_mlp": 0.01064699, "balance_loss_clip": 1.03155899, "balance_loss_mlp": 1.0424161, "epoch": 0.04635502780700436, "flos": 23147624344320.0, "grad_norm": 2.290435336935505, "language_loss": 0.8721177, "learning_rate": 3.978932936348932e-06, "loss": 0.89455825, "num_input_tokens_seen": 16567725, "router_z_loss_clip": 0.33203125, "router_z_loss_mlp": 1.3671875, "step": 771, "time_per_iteration": 2.4312503337860107 }, { "auxiliary_loss_clip": 0.01181348, "auxiliary_loss_mlp": 0.0107427, "balance_loss_clip": 1.03600359, "balance_loss_mlp": 1.04299688, "epoch": 0.04641515105967233, "flos": 23290977853440.0, "grad_norm": 2.206256766876312, "language_loss": 0.83575541, "learning_rate": 3.978878206628377e-06, "loss": 0.85831153, "num_input_tokens_seen": 16588175, "router_z_loss_clip": 0.3828125, "router_z_loss_mlp": 1.375, "step": 772, "time_per_iteration": 2.4327445030212402 }, { "auxiliary_loss_clip": 0.01177164, "auxiliary_loss_mlp": 0.01059886, "balance_loss_clip": 1.02746129, "balance_loss_mlp": 1.04654682, "epoch": 0.046475274312340296, "flos": 25114142384640.0, "grad_norm": 1.9569844693108625, "language_loss": 0.73629689, "learning_rate": 3.978823406286751e-06, "loss": 0.75866747, "num_input_tokens_seen": 16607735, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.3046875, "step": 773, "time_per_iteration": 2.450957775115967 }, { "auxiliary_loss_clip": 0.0117497, "auxiliary_loss_mlp": 0.01059978, "balance_loss_clip": 1.02726662, "balance_loss_mlp": 1.0445869, "epoch": 0.04653539756500827, "flos": 25263745027200.0, "grad_norm": 2.052677547720233, "language_loss": 0.78662962, "learning_rate": 3.978768535326006e-06, "loss": 0.80897909, "num_input_tokens_seen": 16627225, "router_z_loss_clip": 0.32617188, "router_z_loss_mlp": 1.3046875, "step": 774, "time_per_iteration": 2.43341326713562 }, { "auxiliary_loss_clip": 0.01171919, "auxiliary_loss_mlp": 0.01058004, "balance_loss_clip": 1.02596056, "balance_loss_mlp": 1.04186547, "epoch": 0.046595520817676234, "flos": 35402802076800.0, "grad_norm": 2.122867568169163, "language_loss": 0.73343658, "learning_rate": 3.978713593748103e-06, "loss": 0.75573587, "num_input_tokens_seen": 16647785, "router_z_loss_clip": 0.3203125, "router_z_loss_mlp": 1.296875, "step": 775, "time_per_iteration": 2.53930926322937 }, { "auxiliary_loss_clip": 0.01176197, "auxiliary_loss_mlp": 0.01062381, "balance_loss_clip": 1.02788162, "balance_loss_mlp": 1.04260957, "epoch": 0.046655644070344206, "flos": 18111577680000.0, "grad_norm": 1.6785940587675907, "language_loss": 0.76859474, "learning_rate": 3.9786585815550015e-06, "loss": 0.79098046, "num_input_tokens_seen": 16667555, "router_z_loss_clip": 0.34570312, "router_z_loss_mlp": 1.3359375, "step": 776, "time_per_iteration": 2.4250969886779785 }, { "auxiliary_loss_clip": 0.0116975, "auxiliary_loss_mlp": 0.01059828, "balance_loss_clip": 1.02811885, "balance_loss_mlp": 1.04102755, "epoch": 0.04671576732301218, "flos": 29204007759360.0, "grad_norm": 4.3993855916972695, "language_loss": 0.7100842, "learning_rate": 3.978603498748664e-06, "loss": 0.73238003, "num_input_tokens_seen": 16686875, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.28125, "step": 777, "time_per_iteration": 2.4765748977661133 }, { "auxiliary_loss_clip": 0.01172053, "auxiliary_loss_mlp": 0.01070464, "balance_loss_clip": 1.03491557, "balance_loss_mlp": 1.04171491, "epoch": 0.04677589057568014, "flos": 30477115820160.0, "grad_norm": 1.8863215391272792, "language_loss": 0.7640267, "learning_rate": 3.978548345331058e-06, "loss": 0.78645194, "num_input_tokens_seen": 16706420, "router_z_loss_clip": 0.35546875, "router_z_loss_mlp": 1.296875, "step": 778, "time_per_iteration": 2.472606897354126 }, { "auxiliary_loss_clip": 0.01171305, "auxiliary_loss_mlp": 0.0106156, "balance_loss_clip": 1.02784729, "balance_loss_mlp": 1.04161787, "epoch": 0.046836013828348115, "flos": 20556649992960.0, "grad_norm": 2.3938019710870857, "language_loss": 0.78961205, "learning_rate": 3.978493121304151e-06, "loss": 0.81194067, "num_input_tokens_seen": 16726390, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.296875, "step": 779, "time_per_iteration": 2.429203987121582 }, { "auxiliary_loss_clip": 0.01165012, "auxiliary_loss_mlp": 0.01053025, "balance_loss_clip": 1.0218637, "balance_loss_mlp": 1.03878808, "epoch": 0.04689613708101608, "flos": 25446201125760.0, "grad_norm": 1.7069824197535406, "language_loss": 0.77102339, "learning_rate": 3.978437826669914e-06, "loss": 0.79320371, "num_input_tokens_seen": 16748965, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.265625, "step": 780, "time_per_iteration": 2.483863115310669 }, { "auxiliary_loss_clip": 0.01170145, "auxiliary_loss_mlp": 0.01059947, "balance_loss_clip": 1.02910745, "balance_loss_mlp": 1.04262638, "epoch": 0.04695626033368405, "flos": 23000325851520.0, "grad_norm": 1.9470585388344062, "language_loss": 0.76273519, "learning_rate": 3.9783824614303195e-06, "loss": 0.78503609, "num_input_tokens_seen": 16768620, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.2734375, "step": 781, "time_per_iteration": 2.428377628326416 }, { "auxiliary_loss_clip": 0.01176939, "auxiliary_loss_mlp": 0.01072466, "balance_loss_clip": 1.03796661, "balance_loss_mlp": 1.04257929, "epoch": 0.047016383586352024, "flos": 29132051713920.0, "grad_norm": 2.209148364136706, "language_loss": 0.73881859, "learning_rate": 3.978327025587344e-06, "loss": 0.76131266, "num_input_tokens_seen": 16789755, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.34375, "step": 782, "time_per_iteration": 2.456827402114868 }, { "auxiliary_loss_clip": 0.01171062, "auxiliary_loss_mlp": 0.01053215, "balance_loss_clip": 1.02298379, "balance_loss_mlp": 1.04179227, "epoch": 0.04707650683901999, "flos": 14975434448640.0, "grad_norm": 3.168953013497748, "language_loss": 0.80221462, "learning_rate": 3.978271519142967e-06, "loss": 0.82445741, "num_input_tokens_seen": 16807585, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.296875, "step": 783, "time_per_iteration": 2.392314910888672 }, { "auxiliary_loss_clip": 0.01166711, "auxiliary_loss_mlp": 0.01056846, "balance_loss_clip": 1.02725816, "balance_loss_mlp": 1.04139662, "epoch": 0.04713663009168796, "flos": 21650094864000.0, "grad_norm": 2.5535269215120957, "language_loss": 0.81434727, "learning_rate": 3.978215942099167e-06, "loss": 0.8365829, "num_input_tokens_seen": 16827220, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.25, "step": 784, "time_per_iteration": 2.4085748195648193 }, { "auxiliary_loss_clip": 0.0117326, "auxiliary_loss_mlp": 0.01058399, "balance_loss_clip": 1.02728581, "balance_loss_mlp": 1.0409857, "epoch": 0.04719675334435593, "flos": 21324320167680.0, "grad_norm": 2.743894936329185, "language_loss": 0.80728829, "learning_rate": 3.9781602944579285e-06, "loss": 0.82960492, "num_input_tokens_seen": 16846230, "router_z_loss_clip": 0.31054688, "router_z_loss_mlp": 1.3203125, "step": 785, "time_per_iteration": 2.4267308712005615 }, { "auxiliary_loss_clip": 0.01173528, "auxiliary_loss_mlp": 0.01055203, "balance_loss_clip": 1.02485251, "balance_loss_mlp": 1.04502642, "epoch": 0.0472568765970239, "flos": 17930413301760.0, "grad_norm": 1.9076731639279216, "language_loss": 0.89660287, "learning_rate": 3.978104576221238e-06, "loss": 0.91889018, "num_input_tokens_seen": 16865325, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.28125, "step": 786, "time_per_iteration": 2.4019699096679688 }, { "auxiliary_loss_clip": 0.01171585, "auxiliary_loss_mlp": 0.01058545, "balance_loss_clip": 1.02435589, "balance_loss_mlp": 1.03746784, "epoch": 0.04731699984969187, "flos": 18076350251520.0, "grad_norm": 3.8833038203919887, "language_loss": 0.76674724, "learning_rate": 3.978048787391084e-06, "loss": 0.78904855, "num_input_tokens_seen": 16882930, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.34375, "step": 787, "time_per_iteration": 2.373610258102417 }, { "auxiliary_loss_clip": 0.01176536, "auxiliary_loss_mlp": 0.01058107, "balance_loss_clip": 1.02644455, "balance_loss_mlp": 1.04419088, "epoch": 0.047377123102359836, "flos": 23183968936320.0, "grad_norm": 4.053649185534547, "language_loss": 0.80823344, "learning_rate": 3.9779929279694565e-06, "loss": 0.83057988, "num_input_tokens_seen": 16900710, "router_z_loss_clip": 0.31640625, "router_z_loss_mlp": 1.328125, "step": 788, "time_per_iteration": 2.4188549518585205 }, { "auxiliary_loss_clip": 0.01171514, "auxiliary_loss_mlp": 0.01060823, "balance_loss_clip": 1.02610922, "balance_loss_mlp": 1.04387689, "epoch": 0.04743724635502781, "flos": 22746681757440.0, "grad_norm": 2.0244616281489547, "language_loss": 0.84739041, "learning_rate": 3.977936997958349e-06, "loss": 0.86971378, "num_input_tokens_seen": 16919210, "router_z_loss_clip": 0.34765625, "router_z_loss_mlp": 1.2734375, "step": 789, "time_per_iteration": 2.4152109622955322 }, { "auxiliary_loss_clip": 0.01172058, "auxiliary_loss_mlp": 0.01064209, "balance_loss_clip": 1.03479958, "balance_loss_mlp": 1.04249954, "epoch": 0.04749736960769577, "flos": 17237736460800.0, "grad_norm": 2.4696918852654024, "language_loss": 0.81907129, "learning_rate": 3.977880997359758e-06, "loss": 0.84143388, "num_input_tokens_seen": 16937125, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.296875, "step": 790, "time_per_iteration": 2.41382098197937 }, { "auxiliary_loss_clip": 0.01168927, "auxiliary_loss_mlp": 0.01055267, "balance_loss_clip": 1.02436805, "balance_loss_mlp": 1.04008615, "epoch": 0.047557492860363745, "flos": 40477672039680.0, "grad_norm": 2.1721899593907517, "language_loss": 0.8778193, "learning_rate": 3.977824926175682e-06, "loss": 0.90006131, "num_input_tokens_seen": 16958610, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.28125, "step": 791, "time_per_iteration": 2.5817065238952637 }, { "auxiliary_loss_clip": 0.01172967, "auxiliary_loss_mlp": 0.01058892, "balance_loss_clip": 1.02744484, "balance_loss_mlp": 1.04207683, "epoch": 0.04761761611303172, "flos": 18697001224320.0, "grad_norm": 2.1744271782528704, "language_loss": 0.90019238, "learning_rate": 3.977768784408122e-06, "loss": 0.92251098, "num_input_tokens_seen": 16977300, "router_z_loss_clip": 0.31445312, "router_z_loss_mlp": 1.3046875, "step": 792, "time_per_iteration": 2.4133920669555664 }, { "auxiliary_loss_clip": 0.01170189, "auxiliary_loss_mlp": 0.01068953, "balance_loss_clip": 1.03831649, "balance_loss_mlp": 1.038118, "epoch": 0.04767773936569968, "flos": 20920968696960.0, "grad_norm": 1.9371333153222121, "language_loss": 0.73367131, "learning_rate": 3.977712572059081e-06, "loss": 0.75606275, "num_input_tokens_seen": 16994950, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.3203125, "step": 793, "time_per_iteration": 2.385101795196533 }, { "auxiliary_loss_clip": 0.01173409, "auxiliary_loss_mlp": 0.01053075, "balance_loss_clip": 1.02229476, "balance_loss_mlp": 1.04118943, "epoch": 0.047737862618367655, "flos": 23731546700160.0, "grad_norm": 2.663201040336238, "language_loss": 0.85657656, "learning_rate": 3.977656289130567e-06, "loss": 0.8788414, "num_input_tokens_seen": 17014760, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.328125, "step": 794, "time_per_iteration": 2.423823833465576 }, { "auxiliary_loss_clip": 0.01172468, "auxiliary_loss_mlp": 0.01062428, "balance_loss_clip": 1.03012252, "balance_loss_mlp": 1.0396831, "epoch": 0.04779798587103562, "flos": 23694643526400.0, "grad_norm": 2.6155256508860307, "language_loss": 0.69553244, "learning_rate": 3.977599935624586e-06, "loss": 0.71788138, "num_input_tokens_seen": 17032715, "router_z_loss_clip": 0.32226562, "router_z_loss_mlp": 1.328125, "step": 795, "time_per_iteration": 2.4013278484344482 }, { "auxiliary_loss_clip": 0.01169285, "auxiliary_loss_mlp": 0.01065853, "balance_loss_clip": 1.03307092, "balance_loss_mlp": 1.04058623, "epoch": 0.04785810912370359, "flos": 23182572481920.0, "grad_norm": 2.438329569382553, "language_loss": 0.80910087, "learning_rate": 3.977543511543151e-06, "loss": 0.83145225, "num_input_tokens_seen": 17052215, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.2890625, "step": 796, "time_per_iteration": 2.425485610961914 }, { "auxiliary_loss_clip": 0.01169091, "auxiliary_loss_mlp": 0.01057849, "balance_loss_clip": 1.02528143, "balance_loss_mlp": 1.04067898, "epoch": 0.047918232376371564, "flos": 18039656545920.0, "grad_norm": 2.4290765407806587, "language_loss": 0.81627935, "learning_rate": 3.977487016888274e-06, "loss": 0.83854878, "num_input_tokens_seen": 17069225, "router_z_loss_clip": 0.32617188, "router_z_loss_mlp": 1.28125, "step": 797, "time_per_iteration": 2.385831117630005 }, { "auxiliary_loss_clip": 0.01053266, "auxiliary_loss_mlp": 0.01004369, "balance_loss_clip": 0.99902827, "balance_loss_mlp": 1.01167202, "epoch": 0.04797835562903953, "flos": 62439400632960.0, "grad_norm": 0.9135418856904373, "language_loss": 0.64484239, "learning_rate": 3.977430451661972e-06, "loss": 0.66541874, "num_input_tokens_seen": 17126680, "router_z_loss_clip": 0.0534668, "router_z_loss_mlp": 0.41601562, "step": 798, "time_per_iteration": 2.9660699367523193 }, { "auxiliary_loss_clip": 0.01174345, "auxiliary_loss_mlp": 0.01058284, "balance_loss_clip": 1.02762318, "balance_loss_mlp": 1.03999674, "epoch": 0.0480384788817075, "flos": 21506217684480.0, "grad_norm": 1.9946740548056843, "language_loss": 0.90883076, "learning_rate": 3.9773738158662655e-06, "loss": 0.93115699, "num_input_tokens_seen": 17144835, "router_z_loss_clip": 0.30664062, "router_z_loss_mlp": 1.34375, "step": 799, "time_per_iteration": 2.409356117248535 }, { "auxiliary_loss_clip": 0.01171554, "auxiliary_loss_mlp": 0.01055537, "balance_loss_clip": 1.02487683, "balance_loss_mlp": 1.04478359, "epoch": 0.048098602134375466, "flos": 21725611868160.0, "grad_norm": 2.0226049753764235, "language_loss": 0.86634338, "learning_rate": 3.977317109503172e-06, "loss": 0.8886143, "num_input_tokens_seen": 17165030, "router_z_loss_clip": 0.30664062, "router_z_loss_mlp": 1.265625, "step": 800, "time_per_iteration": 2.443582773208618 }, { "auxiliary_loss_clip": 0.01173783, "auxiliary_loss_mlp": 0.01063135, "balance_loss_clip": 1.03245056, "balance_loss_mlp": 1.04335332, "epoch": 0.04815872538704344, "flos": 22929940817280.0, "grad_norm": 3.5504707427095092, "language_loss": 0.83880752, "learning_rate": 3.977260332574718e-06, "loss": 0.86117673, "num_input_tokens_seen": 17184895, "router_z_loss_clip": 0.30664062, "router_z_loss_mlp": 1.3046875, "step": 801, "time_per_iteration": 2.430955410003662 }, { "auxiliary_loss_clip": 0.01170878, "auxiliary_loss_mlp": 0.01060558, "balance_loss_clip": 1.02977824, "balance_loss_mlp": 1.04109502, "epoch": 0.04821884863971141, "flos": 43173176601600.0, "grad_norm": 2.5459362418907205, "language_loss": 0.79219079, "learning_rate": 3.977203485082928e-06, "loss": 0.81450516, "num_input_tokens_seen": 17208225, "router_z_loss_clip": 0.30664062, "router_z_loss_mlp": 1.296875, "step": 802, "time_per_iteration": 2.6055495738983154 }, { "auxiliary_loss_clip": 0.01170224, "auxiliary_loss_mlp": 0.01054641, "balance_loss_clip": 1.02426648, "balance_loss_mlp": 1.04124594, "epoch": 0.048278971892379376, "flos": 18619145159040.0, "grad_norm": 1.745606544997716, "language_loss": 0.86103964, "learning_rate": 3.977146567029833e-06, "loss": 0.88328832, "num_input_tokens_seen": 17226305, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.2890625, "step": 803, "time_per_iteration": 3.791219711303711 }, { "auxiliary_loss_clip": 0.0116548, "auxiliary_loss_mlp": 0.0105238, "balance_loss_clip": 1.02250648, "balance_loss_mlp": 1.04170287, "epoch": 0.04833909514504735, "flos": 20229024994560.0, "grad_norm": 2.2811621272757576, "language_loss": 0.85222125, "learning_rate": 3.977089578417462e-06, "loss": 0.87439978, "num_input_tokens_seen": 17244545, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.234375, "step": 804, "time_per_iteration": 2.4115700721740723 }, { "auxiliary_loss_clip": 0.0116977, "auxiliary_loss_mlp": 0.01048593, "balance_loss_clip": 1.01893377, "balance_loss_mlp": 1.04267776, "epoch": 0.04839921839771532, "flos": 24644001749760.0, "grad_norm": 2.490447923729626, "language_loss": 0.86260319, "learning_rate": 3.9770325192478504e-06, "loss": 0.88478678, "num_input_tokens_seen": 17265730, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.265625, "step": 805, "time_per_iteration": 2.454223155975342 }, { "auxiliary_loss_clip": 0.01163325, "auxiliary_loss_mlp": 0.01052235, "balance_loss_clip": 1.02326751, "balance_loss_mlp": 1.03973639, "epoch": 0.048459341650383285, "flos": 24826283291520.0, "grad_norm": 2.60907289230247, "language_loss": 0.67868835, "learning_rate": 3.9769753895230324e-06, "loss": 0.70084393, "num_input_tokens_seen": 17284820, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.234375, "step": 806, "time_per_iteration": 5.261777639389038 }, { "auxiliary_loss_clip": 0.01165136, "auxiliary_loss_mlp": 0.0105429, "balance_loss_clip": 1.02570355, "balance_loss_mlp": 1.04075348, "epoch": 0.04851946490305126, "flos": 22162130997120.0, "grad_norm": 5.154972087774901, "language_loss": 0.7642802, "learning_rate": 3.976918189245049e-06, "loss": 0.78647453, "num_input_tokens_seen": 17305085, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.2421875, "step": 807, "time_per_iteration": 2.4211854934692383 }, { "auxiliary_loss_clip": 0.01164869, "auxiliary_loss_mlp": 0.0106269, "balance_loss_clip": 1.0339365, "balance_loss_mlp": 1.03880262, "epoch": 0.04857958815571922, "flos": 19791004677120.0, "grad_norm": 2.46456303042586, "language_loss": 0.86459714, "learning_rate": 3.9768609184159405e-06, "loss": 0.88687277, "num_input_tokens_seen": 17322715, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.265625, "step": 808, "time_per_iteration": 3.7626330852508545 }, { "auxiliary_loss_clip": 0.01170461, "auxiliary_loss_mlp": 0.01053986, "balance_loss_clip": 1.02507806, "balance_loss_mlp": 1.04022026, "epoch": 0.048639711408387194, "flos": 18696966312960.0, "grad_norm": 2.3646723308193276, "language_loss": 0.89717674, "learning_rate": 3.976803577037751e-06, "loss": 0.91942126, "num_input_tokens_seen": 17341455, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.296875, "step": 809, "time_per_iteration": 2.4033820629119873 }, { "auxiliary_loss_clip": 0.01170753, "auxiliary_loss_mlp": 0.01060366, "balance_loss_clip": 1.02941978, "balance_loss_mlp": 1.04319715, "epoch": 0.048699834661055166, "flos": 24862348592640.0, "grad_norm": 1.9856387765986683, "language_loss": 0.84460419, "learning_rate": 3.976746165112527e-06, "loss": 0.8669154, "num_input_tokens_seen": 17360765, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.28125, "step": 810, "time_per_iteration": 2.501471757888794 }, { "auxiliary_loss_clip": 0.01171227, "auxiliary_loss_mlp": 0.01050112, "balance_loss_clip": 1.02076244, "balance_loss_mlp": 1.04236078, "epoch": 0.04875995791372313, "flos": 20702970967680.0, "grad_norm": 5.847958811419739, "language_loss": 0.80468845, "learning_rate": 3.976688682642317e-06, "loss": 0.82690179, "num_input_tokens_seen": 17380625, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.2890625, "step": 811, "time_per_iteration": 2.4055933952331543 }, { "auxiliary_loss_clip": 0.01161484, "auxiliary_loss_mlp": 0.01058142, "balance_loss_clip": 1.02891231, "balance_loss_mlp": 1.03926706, "epoch": 0.048820081166391104, "flos": 18587304132480.0, "grad_norm": 1.782692438004299, "language_loss": 0.74147636, "learning_rate": 3.976631129629173e-06, "loss": 0.76367265, "num_input_tokens_seen": 17399355, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.21875, "step": 812, "time_per_iteration": 2.411888360977173 }, { "auxiliary_loss_clip": 0.01164119, "auxiliary_loss_mlp": 0.01059981, "balance_loss_clip": 1.03265858, "balance_loss_mlp": 1.04155898, "epoch": 0.04888020441905907, "flos": 22706322359040.0, "grad_norm": 1.956145964727686, "language_loss": 0.89826584, "learning_rate": 3.9765735060751475e-06, "loss": 0.92050683, "num_input_tokens_seen": 17418240, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.2265625, "step": 813, "time_per_iteration": 2.4079573154449463 }, { "auxiliary_loss_clip": 0.011635, "auxiliary_loss_mlp": 0.01050011, "balance_loss_clip": 1.02287924, "balance_loss_mlp": 1.04093742, "epoch": 0.04894032767172704, "flos": 22783235817600.0, "grad_norm": 2.4332683364797165, "language_loss": 0.74885005, "learning_rate": 3.976515811982298e-06, "loss": 0.77098519, "num_input_tokens_seen": 17436250, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.2265625, "step": 814, "time_per_iteration": 2.4603400230407715 }, { "auxiliary_loss_clip": 0.0116791, "auxiliary_loss_mlp": 0.01061613, "balance_loss_clip": 1.03182244, "balance_loss_mlp": 1.04210234, "epoch": 0.04900045092439501, "flos": 25515084971520.0, "grad_norm": 2.394562762664596, "language_loss": 0.83616436, "learning_rate": 3.976458047352684e-06, "loss": 0.85845953, "num_input_tokens_seen": 17455750, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.2578125, "step": 815, "time_per_iteration": 2.4485135078430176 }, { "auxiliary_loss_clip": 0.01166777, "auxiliary_loss_mlp": 0.01055653, "balance_loss_clip": 1.02430129, "balance_loss_mlp": 1.03958774, "epoch": 0.04906057417706298, "flos": 25956945538560.0, "grad_norm": 2.157888550916716, "language_loss": 0.90636873, "learning_rate": 3.976400212188366e-06, "loss": 0.92859304, "num_input_tokens_seen": 17474995, "router_z_loss_clip": 0.31445312, "router_z_loss_mlp": 1.2734375, "step": 816, "time_per_iteration": 2.444671630859375 }, { "auxiliary_loss_clip": 0.01168071, "auxiliary_loss_mlp": 0.01055868, "balance_loss_clip": 1.02784157, "balance_loss_mlp": 1.04176772, "epoch": 0.04912069742973095, "flos": 18623648724480.0, "grad_norm": 2.630038287340091, "language_loss": 0.79744601, "learning_rate": 3.976342306491408e-06, "loss": 0.81968546, "num_input_tokens_seen": 17493395, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.265625, "step": 817, "time_per_iteration": 2.4015910625457764 }, { "auxiliary_loss_clip": 0.01165215, "auxiliary_loss_mlp": 0.01062397, "balance_loss_clip": 1.03328562, "balance_loss_mlp": 1.04173994, "epoch": 0.049180820682398915, "flos": 23698553598720.0, "grad_norm": 2.698144736986534, "language_loss": 0.84772664, "learning_rate": 3.976284330263878e-06, "loss": 0.87000275, "num_input_tokens_seen": 17514565, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.234375, "step": 818, "time_per_iteration": 2.4503378868103027 }, { "auxiliary_loss_clip": 0.01169224, "auxiliary_loss_mlp": 0.01056172, "balance_loss_clip": 1.0257256, "balance_loss_mlp": 1.04177284, "epoch": 0.04924094393506689, "flos": 22419266227200.0, "grad_norm": 3.614464798815647, "language_loss": 0.7506969, "learning_rate": 3.976226283507843e-06, "loss": 0.77295083, "num_input_tokens_seen": 17534590, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.2734375, "step": 819, "time_per_iteration": 2.423330783843994 }, { "auxiliary_loss_clip": 0.01167272, "auxiliary_loss_mlp": 0.01055403, "balance_loss_clip": 1.02767491, "balance_loss_mlp": 1.04262042, "epoch": 0.04930106718773486, "flos": 15737448983040.0, "grad_norm": 2.234062200713571, "language_loss": 0.85044587, "learning_rate": 3.976168166225375e-06, "loss": 0.87267256, "num_input_tokens_seen": 17551900, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.25, "step": 820, "time_per_iteration": 2.420631170272827 }, { "auxiliary_loss_clip": 0.01168213, "auxiliary_loss_mlp": 0.01054757, "balance_loss_clip": 1.02502584, "balance_loss_mlp": 1.04121029, "epoch": 0.049361190440402825, "flos": 26249412931200.0, "grad_norm": 2.000199948064709, "language_loss": 0.90914762, "learning_rate": 3.976109978418549e-06, "loss": 0.93137735, "num_input_tokens_seen": 17571485, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.265625, "step": 821, "time_per_iteration": 2.4498400688171387 }, { "auxiliary_loss_clip": 0.01167231, "auxiliary_loss_mlp": 0.01064154, "balance_loss_clip": 1.03509116, "balance_loss_mlp": 1.0418961, "epoch": 0.0494213136930708, "flos": 21251281870080.0, "grad_norm": 1.832087213668366, "language_loss": 0.8943603, "learning_rate": 3.976051720089441e-06, "loss": 0.91667426, "num_input_tokens_seen": 17591410, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.25, "step": 822, "time_per_iteration": 2.4395205974578857 }, { "auxiliary_loss_clip": 0.01166594, "auxiliary_loss_mlp": 0.01057127, "balance_loss_clip": 1.02496481, "balance_loss_mlp": 1.0421176, "epoch": 0.04948143694573876, "flos": 27964241913600.0, "grad_norm": 6.6433608209893436, "language_loss": 0.67021036, "learning_rate": 3.9759933912401304e-06, "loss": 0.69244754, "num_input_tokens_seen": 17612010, "router_z_loss_clip": 0.32226562, "router_z_loss_mlp": 1.25, "step": 823, "time_per_iteration": 2.4612014293670654 }, { "auxiliary_loss_clip": 0.01050969, "auxiliary_loss_mlp": 0.01017236, "balance_loss_clip": 1.01137078, "balance_loss_mlp": 1.00644159, "epoch": 0.049541560198406734, "flos": 66178250398080.0, "grad_norm": 1.3172251732625322, "language_loss": 0.62187296, "learning_rate": 3.975934991872698e-06, "loss": 0.642555, "num_input_tokens_seen": 17673430, "router_z_loss_clip": 0.05859375, "router_z_loss_mlp": 0.4453125, "step": 824, "time_per_iteration": 3.1524150371551514 }, { "auxiliary_loss_clip": 0.0116972, "auxiliary_loss_mlp": 0.01061632, "balance_loss_clip": 1.02976692, "balance_loss_mlp": 1.04172587, "epoch": 0.049601683451074706, "flos": 22891606277760.0, "grad_norm": 1.8254047317461848, "language_loss": 0.90296292, "learning_rate": 3.975876521989229e-06, "loss": 0.9252764, "num_input_tokens_seen": 17689545, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.28125, "step": 825, "time_per_iteration": 2.4061501026153564 }, { "auxiliary_loss_clip": 0.01170339, "auxiliary_loss_mlp": 0.0106166, "balance_loss_clip": 1.02899635, "balance_loss_mlp": 1.04297948, "epoch": 0.04966180670374267, "flos": 21432585893760.0, "grad_norm": 2.234887316682884, "language_loss": 0.66441983, "learning_rate": 3.975817981591809e-06, "loss": 0.6867398, "num_input_tokens_seen": 17705965, "router_z_loss_clip": 0.32617188, "router_z_loss_mlp": 1.2734375, "step": 826, "time_per_iteration": 2.4316608905792236 }, { "auxiliary_loss_clip": 0.01170613, "auxiliary_loss_mlp": 0.01059237, "balance_loss_clip": 1.02803993, "balance_loss_mlp": 1.04257441, "epoch": 0.04972192995641064, "flos": 23106392161920.0, "grad_norm": 2.0648308008719636, "language_loss": 0.78250402, "learning_rate": 3.975759370682528e-06, "loss": 0.80480254, "num_input_tokens_seen": 17724580, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.28125, "step": 827, "time_per_iteration": 2.4437127113342285 }, { "auxiliary_loss_clip": 0.01172813, "auxiliary_loss_mlp": 0.01065139, "balance_loss_clip": 1.03342938, "balance_loss_mlp": 1.04325986, "epoch": 0.04978205320907861, "flos": 40404563919360.0, "grad_norm": 1.6663251374112558, "language_loss": 0.78703785, "learning_rate": 3.975700689263477e-06, "loss": 0.80941737, "num_input_tokens_seen": 17747755, "router_z_loss_clip": 0.31640625, "router_z_loss_mlp": 1.296875, "step": 828, "time_per_iteration": 2.6153178215026855 }, { "auxiliary_loss_clip": 0.01162054, "auxiliary_loss_mlp": 0.01054695, "balance_loss_clip": 1.02622795, "balance_loss_mlp": 1.0404613, "epoch": 0.04984217646174658, "flos": 25227365523840.0, "grad_norm": 2.043692486008699, "language_loss": 0.83223975, "learning_rate": 3.97564193733675e-06, "loss": 0.85440719, "num_input_tokens_seen": 17768550, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.21875, "step": 829, "time_per_iteration": 2.4516046047210693 }, { "auxiliary_loss_clip": 0.01167924, "auxiliary_loss_mlp": 0.01064178, "balance_loss_clip": 1.0296793, "balance_loss_mlp": 1.03958869, "epoch": 0.04990229971441455, "flos": 15958763291520.0, "grad_norm": 1.9552786395143507, "language_loss": 0.75125033, "learning_rate": 3.975583114904446e-06, "loss": 0.77357137, "num_input_tokens_seen": 17786080, "router_z_loss_clip": 0.34375, "router_z_loss_mlp": 1.28125, "step": 830, "time_per_iteration": 2.410106897354126 }, { "auxiliary_loss_clip": 0.01168344, "auxiliary_loss_mlp": 0.01063958, "balance_loss_clip": 1.03321362, "balance_loss_mlp": 1.039922, "epoch": 0.04996242296708252, "flos": 18404149806720.0, "grad_norm": 1.9775454545213287, "language_loss": 0.79518765, "learning_rate": 3.975524221968661e-06, "loss": 0.81751066, "num_input_tokens_seen": 17803635, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.28125, "step": 831, "time_per_iteration": 2.4046993255615234 }, { "auxiliary_loss_clip": 0.01170035, "auxiliary_loss_mlp": 0.01065907, "balance_loss_clip": 1.03642654, "balance_loss_mlp": 1.04187751, "epoch": 0.05002254621975049, "flos": 17857095713280.0, "grad_norm": 2.480317958948075, "language_loss": 0.91368961, "learning_rate": 3.975465258531499e-06, "loss": 0.93604904, "num_input_tokens_seen": 17822190, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.28125, "step": 832, "time_per_iteration": 2.4149022102355957 }, { "auxiliary_loss_clip": 0.01162742, "auxiliary_loss_mlp": 0.01063583, "balance_loss_clip": 1.03423357, "balance_loss_mlp": 1.04163289, "epoch": 0.050082669472418455, "flos": 45658538490240.0, "grad_norm": 2.126591231557392, "language_loss": 0.83265626, "learning_rate": 3.9754062245950625e-06, "loss": 0.85491955, "num_input_tokens_seen": 17846915, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.2109375, "step": 833, "time_per_iteration": 2.5969738960266113 }, { "auxiliary_loss_clip": 0.01164549, "auxiliary_loss_mlp": 0.01053187, "balance_loss_clip": 1.02181077, "balance_loss_mlp": 1.0378592, "epoch": 0.05014279272508643, "flos": 37960538947200.0, "grad_norm": 2.5697450136119495, "language_loss": 0.82563829, "learning_rate": 3.975347120161459e-06, "loss": 0.84781563, "num_input_tokens_seen": 17867270, "router_z_loss_clip": 0.31445312, "router_z_loss_mlp": 1.265625, "step": 834, "time_per_iteration": 2.5611331462860107 }, { "auxiliary_loss_clip": 0.0116794, "auxiliary_loss_mlp": 0.01056384, "balance_loss_clip": 1.02417338, "balance_loss_mlp": 1.04004765, "epoch": 0.0502029159777544, "flos": 20995124158080.0, "grad_norm": 2.2147712592351776, "language_loss": 0.91696298, "learning_rate": 3.975287945232799e-06, "loss": 0.93920618, "num_input_tokens_seen": 17884880, "router_z_loss_clip": 0.32226562, "router_z_loss_mlp": 1.28125, "step": 835, "time_per_iteration": 2.3931682109832764 }, { "auxiliary_loss_clip": 0.01169635, "auxiliary_loss_mlp": 0.0106848, "balance_loss_clip": 1.03572142, "balance_loss_mlp": 1.03864908, "epoch": 0.050263039230422364, "flos": 15887156359680.0, "grad_norm": 8.942841368746471, "language_loss": 0.76724601, "learning_rate": 3.975228699811193e-06, "loss": 0.78962719, "num_input_tokens_seen": 17903695, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.3125, "step": 836, "time_per_iteration": 2.4156110286712646 }, { "auxiliary_loss_clip": 0.01163802, "auxiliary_loss_mlp": 0.01066891, "balance_loss_clip": 1.03853154, "balance_loss_mlp": 1.04223275, "epoch": 0.050323162483090336, "flos": 23731616522880.0, "grad_norm": 2.121859205210282, "language_loss": 0.83580768, "learning_rate": 3.975169383898755e-06, "loss": 0.8581146, "num_input_tokens_seen": 17920745, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.21875, "step": 837, "time_per_iteration": 2.415476083755493 }, { "auxiliary_loss_clip": 0.01164628, "auxiliary_loss_mlp": 0.01063927, "balance_loss_clip": 1.03463697, "balance_loss_mlp": 1.0413357, "epoch": 0.0503832857357583, "flos": 20265195029760.0, "grad_norm": 2.4680140494286156, "language_loss": 0.7328164, "learning_rate": 3.975109997497604e-06, "loss": 0.75510192, "num_input_tokens_seen": 17938220, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.234375, "step": 838, "time_per_iteration": 2.435312271118164 }, { "auxiliary_loss_clip": 0.01160911, "auxiliary_loss_mlp": 0.01063132, "balance_loss_clip": 1.03287721, "balance_loss_mlp": 1.03826404, "epoch": 0.05044340898842627, "flos": 17784057415680.0, "grad_norm": 2.914937148434028, "language_loss": 0.83091825, "learning_rate": 3.975050540609857e-06, "loss": 0.85315871, "num_input_tokens_seen": 17957325, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.2265625, "step": 839, "time_per_iteration": 2.4032273292541504 }, { "auxiliary_loss_clip": 0.01159855, "auxiliary_loss_mlp": 0.01056911, "balance_loss_clip": 1.02784753, "balance_loss_mlp": 1.04024374, "epoch": 0.050503532241094246, "flos": 22965412625280.0, "grad_norm": 1.7597979747182033, "language_loss": 0.8568148, "learning_rate": 3.9749910132376355e-06, "loss": 0.87898248, "num_input_tokens_seen": 17975875, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.203125, "step": 840, "time_per_iteration": 2.453213691711426 }, { "auxiliary_loss_clip": 0.0116538, "auxiliary_loss_mlp": 0.01057513, "balance_loss_clip": 1.02687609, "balance_loss_mlp": 1.04037189, "epoch": 0.05056365549376221, "flos": 22776078988800.0, "grad_norm": 1.9817718431560314, "language_loss": 0.9464941, "learning_rate": 3.974931415383066e-06, "loss": 0.96872306, "num_input_tokens_seen": 17994340, "router_z_loss_clip": 0.30664062, "router_z_loss_mlp": 1.25, "step": 841, "time_per_iteration": 2.447728157043457 }, { "auxiliary_loss_clip": 0.01166064, "auxiliary_loss_mlp": 0.01059588, "balance_loss_clip": 1.03023815, "balance_loss_mlp": 1.03955817, "epoch": 0.05062377874643018, "flos": 30915729630720.0, "grad_norm": 2.1937082555241596, "language_loss": 0.77494878, "learning_rate": 3.974871747048274e-06, "loss": 0.79720527, "num_input_tokens_seen": 18015260, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.265625, "step": 842, "time_per_iteration": 3.926527976989746 }, { "auxiliary_loss_clip": 0.01172329, "auxiliary_loss_mlp": 0.01070474, "balance_loss_clip": 1.03757167, "balance_loss_mlp": 1.04413319, "epoch": 0.05068390199909815, "flos": 19646115068160.0, "grad_norm": 2.3304262601049843, "language_loss": 0.78067744, "learning_rate": 3.97481200823539e-06, "loss": 0.80310547, "num_input_tokens_seen": 18033960, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.28125, "step": 843, "time_per_iteration": 2.4005467891693115 }, { "auxiliary_loss_clip": 0.01167731, "auxiliary_loss_mlp": 0.01048925, "balance_loss_clip": 1.01936114, "balance_loss_mlp": 1.04121172, "epoch": 0.05074402525176612, "flos": 37960573858560.0, "grad_norm": 2.398313254478142, "language_loss": 0.83207279, "learning_rate": 3.974752198946545e-06, "loss": 0.85423934, "num_input_tokens_seen": 18056700, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.265625, "step": 844, "time_per_iteration": 2.590186595916748 }, { "auxiliary_loss_clip": 0.01160503, "auxiliary_loss_mlp": 0.01058272, "balance_loss_clip": 1.02806437, "balance_loss_mlp": 1.03811467, "epoch": 0.05080414850443409, "flos": 22053516157440.0, "grad_norm": 2.354765899656259, "language_loss": 0.76544082, "learning_rate": 3.974692319183873e-06, "loss": 0.78762859, "num_input_tokens_seen": 18075815, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.2265625, "step": 845, "time_per_iteration": 2.4088919162750244 }, { "auxiliary_loss_clip": 0.01161682, "auxiliary_loss_mlp": 0.01059081, "balance_loss_clip": 1.02811074, "balance_loss_mlp": 1.03650367, "epoch": 0.05086427175710206, "flos": 20224870542720.0, "grad_norm": 1.7049887007445408, "language_loss": 0.87393314, "learning_rate": 3.974632368949513e-06, "loss": 0.89614075, "num_input_tokens_seen": 18095095, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.25, "step": 846, "time_per_iteration": 3.855921506881714 }, { "auxiliary_loss_clip": 0.01166303, "auxiliary_loss_mlp": 0.01054591, "balance_loss_clip": 1.02493167, "balance_loss_mlp": 1.04270983, "epoch": 0.05092439500977003, "flos": 15158309483520.0, "grad_norm": 2.118490048729354, "language_loss": 0.87353724, "learning_rate": 3.974572348245602e-06, "loss": 0.89574617, "num_input_tokens_seen": 18112675, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.234375, "step": 847, "time_per_iteration": 2.3899219036102295 }, { "auxiliary_loss_clip": 0.01158489, "auxiliary_loss_mlp": 0.01055071, "balance_loss_clip": 1.0248158, "balance_loss_mlp": 1.03820229, "epoch": 0.050984518262437994, "flos": 22054039827840.0, "grad_norm": 2.2959085046618943, "language_loss": 0.81882077, "learning_rate": 3.974512257074284e-06, "loss": 0.84095639, "num_input_tokens_seen": 18130745, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.203125, "step": 848, "time_per_iteration": 3.773952007293701 }, { "auxiliary_loss_clip": 0.01164005, "auxiliary_loss_mlp": 0.01058961, "balance_loss_clip": 1.02872968, "balance_loss_mlp": 1.04196203, "epoch": 0.05104464151510597, "flos": 30224065219200.0, "grad_norm": 2.2019890987313504, "language_loss": 0.87174815, "learning_rate": 3.974452095437701e-06, "loss": 0.89397776, "num_input_tokens_seen": 18152410, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.21875, "step": 849, "time_per_iteration": 2.490201711654663 }, { "auxiliary_loss_clip": 0.01159582, "auxiliary_loss_mlp": 0.01052927, "balance_loss_clip": 1.02360129, "balance_loss_mlp": 1.03816199, "epoch": 0.05110476476777394, "flos": 18331914470400.0, "grad_norm": 2.0239864852213465, "language_loss": 0.83400553, "learning_rate": 3.974391863338003e-06, "loss": 0.8561306, "num_input_tokens_seen": 18170870, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.21875, "step": 850, "time_per_iteration": 2.468676805496216 }, { "auxiliary_loss_clip": 0.01160646, "auxiliary_loss_mlp": 0.01051729, "balance_loss_clip": 1.02292764, "balance_loss_mlp": 1.03897095, "epoch": 0.051164888020441904, "flos": 37997197741440.0, "grad_norm": 1.9923005803260347, "language_loss": 0.65049136, "learning_rate": 3.974331560777338e-06, "loss": 0.67261505, "num_input_tokens_seen": 18191555, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.21875, "step": 851, "time_per_iteration": 2.573582172393799 }, { "auxiliary_loss_clip": 0.01158165, "auxiliary_loss_mlp": 0.0105297, "balance_loss_clip": 1.02218974, "balance_loss_mlp": 1.03646827, "epoch": 0.051225011273109876, "flos": 23037543227520.0, "grad_norm": 2.750400779171418, "language_loss": 0.83152038, "learning_rate": 3.974271187757857e-06, "loss": 0.85363173, "num_input_tokens_seen": 18208620, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.21875, "step": 852, "time_per_iteration": 2.4141244888305664 }, { "auxiliary_loss_clip": 0.01166429, "auxiliary_loss_mlp": 0.0106459, "balance_loss_clip": 1.03333354, "balance_loss_mlp": 1.04123831, "epoch": 0.05128513452577785, "flos": 18258841261440.0, "grad_norm": 2.0184171365603514, "language_loss": 0.80007803, "learning_rate": 3.974210744281717e-06, "loss": 0.82238829, "num_input_tokens_seen": 18226370, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.25, "step": 853, "time_per_iteration": 2.405571937561035 }, { "auxiliary_loss_clip": 0.01160161, "auxiliary_loss_mlp": 0.01055026, "balance_loss_clip": 1.02531874, "balance_loss_mlp": 1.03994238, "epoch": 0.05134525777844581, "flos": 27197723813760.0, "grad_norm": 1.9780043386979285, "language_loss": 0.75332499, "learning_rate": 3.974150230351074e-06, "loss": 0.77547681, "num_input_tokens_seen": 18247075, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.203125, "step": 854, "time_per_iteration": 2.4725728034973145 }, { "auxiliary_loss_clip": 0.01164939, "auxiliary_loss_mlp": 0.01053258, "balance_loss_clip": 1.02355087, "balance_loss_mlp": 1.04014957, "epoch": 0.051405381031113785, "flos": 28361099871360.0, "grad_norm": 2.159727043901847, "language_loss": 0.81719911, "learning_rate": 3.974089645968087e-06, "loss": 0.83938104, "num_input_tokens_seen": 18265680, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.25, "step": 855, "time_per_iteration": 2.4578967094421387 }, { "auxiliary_loss_clip": 0.01051068, "auxiliary_loss_mlp": 0.01009121, "balance_loss_clip": 1.00416172, "balance_loss_mlp": 1.01090991, "epoch": 0.05146550428378175, "flos": 65614855921920.0, "grad_norm": 0.9758737055950394, "language_loss": 0.65607464, "learning_rate": 3.974028991134917e-06, "loss": 0.67667657, "num_input_tokens_seen": 18327015, "router_z_loss_clip": 0.04956055, "router_z_loss_mlp": 0.40234375, "step": 856, "time_per_iteration": 3.051262855529785 }, { "auxiliary_loss_clip": 0.01158072, "auxiliary_loss_mlp": 0.01047891, "balance_loss_clip": 1.01911426, "balance_loss_mlp": 1.03875589, "epoch": 0.05152562753644972, "flos": 22053760536960.0, "grad_norm": 3.191385805610801, "language_loss": 0.76746464, "learning_rate": 3.973968265853732e-06, "loss": 0.7895242, "num_input_tokens_seen": 18345235, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.1953125, "step": 857, "time_per_iteration": 2.4042773246765137 }, { "auxiliary_loss_clip": 0.01162847, "auxiliary_loss_mlp": 0.01053205, "balance_loss_clip": 1.02392697, "balance_loss_mlp": 1.03982496, "epoch": 0.051585750789117694, "flos": 18508714928640.0, "grad_norm": 2.3876640997155048, "language_loss": 0.88652521, "learning_rate": 3.973907470126697e-06, "loss": 0.9086858, "num_input_tokens_seen": 18362350, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.234375, "step": 858, "time_per_iteration": 2.413517475128174 }, { "auxiliary_loss_clip": 0.01161464, "auxiliary_loss_mlp": 0.01053192, "balance_loss_clip": 1.02322316, "balance_loss_mlp": 1.03962851, "epoch": 0.05164587404178566, "flos": 23729172727680.0, "grad_norm": 2.7678383094014634, "language_loss": 0.75064158, "learning_rate": 3.973846603955982e-06, "loss": 0.77278817, "num_input_tokens_seen": 18383390, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.21875, "step": 859, "time_per_iteration": 2.435861825942993 }, { "auxiliary_loss_clip": 0.01168612, "auxiliary_loss_mlp": 0.0106378, "balance_loss_clip": 1.0315932, "balance_loss_mlp": 1.04018474, "epoch": 0.05170599729445363, "flos": 16251963822720.0, "grad_norm": 2.468489423187343, "language_loss": 0.90703034, "learning_rate": 3.973785667343758e-06, "loss": 0.92935425, "num_input_tokens_seen": 18399220, "router_z_loss_clip": 0.32226562, "router_z_loss_mlp": 1.28125, "step": 860, "time_per_iteration": 2.389540672302246 }, { "auxiliary_loss_clip": 0.01163009, "auxiliary_loss_mlp": 0.01049808, "balance_loss_clip": 1.02217507, "balance_loss_mlp": 1.04159057, "epoch": 0.0517661205471216, "flos": 23984841680640.0, "grad_norm": 1.99597550647492, "language_loss": 0.82325977, "learning_rate": 3.973724660292202e-06, "loss": 0.84538794, "num_input_tokens_seen": 18419005, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.21875, "step": 861, "time_per_iteration": 2.447273015975952 }, { "auxiliary_loss_clip": 0.01163287, "auxiliary_loss_mlp": 0.01049733, "balance_loss_clip": 1.02100301, "balance_loss_mlp": 1.04005361, "epoch": 0.05182624379978957, "flos": 29276452563840.0, "grad_norm": 2.184600867013007, "language_loss": 0.78252262, "learning_rate": 3.973663582803489e-06, "loss": 0.80465281, "num_input_tokens_seen": 18440550, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.234375, "step": 862, "time_per_iteration": 2.4645631313323975 }, { "auxiliary_loss_clip": 0.01160461, "auxiliary_loss_mlp": 0.01060547, "balance_loss_clip": 1.03124499, "balance_loss_mlp": 1.04275537, "epoch": 0.05188636705245754, "flos": 24169671751680.0, "grad_norm": 1.8707139286249292, "language_loss": 0.89435291, "learning_rate": 3.9736024348798e-06, "loss": 0.91656297, "num_input_tokens_seen": 18461950, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.171875, "step": 863, "time_per_iteration": 2.467291831970215 }, { "auxiliary_loss_clip": 0.01164169, "auxiliary_loss_mlp": 0.01061407, "balance_loss_clip": 1.03003168, "balance_loss_mlp": 1.04240823, "epoch": 0.051946490305125506, "flos": 26759494028160.0, "grad_norm": 2.655600598569303, "language_loss": 0.75558275, "learning_rate": 3.973541216523316e-06, "loss": 0.77783847, "num_input_tokens_seen": 18480555, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.21875, "step": 864, "time_per_iteration": 2.4739580154418945 }, { "auxiliary_loss_clip": 0.01165025, "auxiliary_loss_mlp": 0.01055548, "balance_loss_clip": 1.02510238, "balance_loss_mlp": 1.04179323, "epoch": 0.05200661355779348, "flos": 21501574473600.0, "grad_norm": 1.9851203173179528, "language_loss": 0.78729963, "learning_rate": 3.973479927736224e-06, "loss": 0.80950534, "num_input_tokens_seen": 18499645, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.234375, "step": 865, "time_per_iteration": 2.481898546218872 }, { "auxiliary_loss_clip": 0.0116096, "auxiliary_loss_mlp": 0.01056058, "balance_loss_clip": 1.02567112, "balance_loss_mlp": 1.03859043, "epoch": 0.05206673681046144, "flos": 18113497804800.0, "grad_norm": 2.155310896039154, "language_loss": 0.85959566, "learning_rate": 3.973418568520709e-06, "loss": 0.88176584, "num_input_tokens_seen": 18516810, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.21875, "step": 866, "time_per_iteration": 2.3983001708984375 }, { "auxiliary_loss_clip": 0.01165422, "auxiliary_loss_mlp": 0.01058009, "balance_loss_clip": 1.02902925, "balance_loss_mlp": 1.04385662, "epoch": 0.052126860063129415, "flos": 17523396138240.0, "grad_norm": 2.813766549619152, "language_loss": 0.87160748, "learning_rate": 3.973357138878961e-06, "loss": 0.89384174, "num_input_tokens_seen": 18532510, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.21875, "step": 867, "time_per_iteration": 2.3934857845306396 }, { "auxiliary_loss_clip": 0.01154742, "auxiliary_loss_mlp": 0.01062666, "balance_loss_clip": 1.03545022, "balance_loss_mlp": 1.0386498, "epoch": 0.05218698331579739, "flos": 32596692727680.0, "grad_norm": 1.4836901961628903, "language_loss": 0.6341002, "learning_rate": 3.973295638813174e-06, "loss": 0.65627426, "num_input_tokens_seen": 18557380, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.15625, "step": 868, "time_per_iteration": 2.5175845623016357 }, { "auxiliary_loss_clip": 0.01165022, "auxiliary_loss_mlp": 0.01062688, "balance_loss_clip": 1.03052521, "balance_loss_mlp": 1.04034626, "epoch": 0.05224710656846535, "flos": 22126205341440.0, "grad_norm": 4.559647925660619, "language_loss": 0.83260775, "learning_rate": 3.973234068325541e-06, "loss": 0.85488486, "num_input_tokens_seen": 18575720, "router_z_loss_clip": 0.3203125, "router_z_loss_mlp": 1.25, "step": 869, "time_per_iteration": 2.4412894248962402 }, { "auxiliary_loss_clip": 0.01163911, "auxiliary_loss_mlp": 0.01054839, "balance_loss_clip": 1.02630055, "balance_loss_mlp": 1.04024363, "epoch": 0.052307229821133325, "flos": 11144310226560.0, "grad_norm": 2.142193581806339, "language_loss": 0.87373012, "learning_rate": 3.973172427418259e-06, "loss": 0.89591759, "num_input_tokens_seen": 18592185, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.234375, "step": 870, "time_per_iteration": 2.4127585887908936 }, { "auxiliary_loss_clip": 0.01164902, "auxiliary_loss_mlp": 0.01054859, "balance_loss_clip": 1.02622497, "balance_loss_mlp": 1.04133844, "epoch": 0.05236735307380129, "flos": 19127271219840.0, "grad_norm": 2.4752433989170615, "language_loss": 0.80509758, "learning_rate": 3.97311071609353e-06, "loss": 0.82729518, "num_input_tokens_seen": 18609560, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.234375, "step": 871, "time_per_iteration": 2.425719976425171 }, { "auxiliary_loss_clip": 0.01161646, "auxiliary_loss_mlp": 0.01046634, "balance_loss_clip": 1.01799941, "balance_loss_mlp": 1.04076588, "epoch": 0.05242747632646926, "flos": 20959582527360.0, "grad_norm": 2.287777610954603, "language_loss": 0.81213582, "learning_rate": 3.973048934353554e-06, "loss": 0.83421862, "num_input_tokens_seen": 18629405, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.2109375, "step": 872, "time_per_iteration": 2.431184768676758 }, { "auxiliary_loss_clip": 0.01048326, "auxiliary_loss_mlp": 0.01032536, "balance_loss_clip": 1.02783895, "balance_loss_mlp": 1.00866389, "epoch": 0.052487599579137234, "flos": 65017632337920.0, "grad_norm": 0.9004996555808237, "language_loss": 0.61653852, "learning_rate": 3.972987082200538e-06, "loss": 0.63734716, "num_input_tokens_seen": 18681480, "router_z_loss_clip": 0.046875, "router_z_loss_mlp": 0.39648438, "step": 873, "time_per_iteration": 2.950486421585083 }, { "auxiliary_loss_clip": 0.01160999, "auxiliary_loss_mlp": 0.01047396, "balance_loss_clip": 1.01950097, "balance_loss_mlp": 1.03949547, "epoch": 0.0525477228318052, "flos": 23287905653760.0, "grad_norm": 2.1513200790654716, "language_loss": 0.88312685, "learning_rate": 3.972925159636687e-06, "loss": 0.90521085, "num_input_tokens_seen": 18700390, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.21875, "step": 874, "time_per_iteration": 2.431154727935791 }, { "auxiliary_loss_clip": 0.01162635, "auxiliary_loss_mlp": 0.01057579, "balance_loss_clip": 1.02794373, "balance_loss_mlp": 1.03995252, "epoch": 0.05260784608447317, "flos": 32228952710400.0, "grad_norm": 1.788658450643276, "language_loss": 0.74017358, "learning_rate": 3.972863166664212e-06, "loss": 0.76237571, "num_input_tokens_seen": 18721280, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.2265625, "step": 875, "time_per_iteration": 2.5350773334503174 }, { "auxiliary_loss_clip": 0.01161192, "auxiliary_loss_mlp": 0.0105587, "balance_loss_clip": 1.0261867, "balance_loss_mlp": 1.04077697, "epoch": 0.052667969337141136, "flos": 24462034410240.0, "grad_norm": 2.113354624868253, "language_loss": 0.9275443, "learning_rate": 3.972801103285326e-06, "loss": 0.9497149, "num_input_tokens_seen": 18741545, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.203125, "step": 876, "time_per_iteration": 2.459304094314575 }, { "auxiliary_loss_clip": 0.01048974, "auxiliary_loss_mlp": 0.01004426, "balance_loss_clip": 0.99989587, "balance_loss_mlp": 1.01024711, "epoch": 0.05272809258980911, "flos": 57780938989440.0, "grad_norm": 0.8418676385507674, "language_loss": 0.62896293, "learning_rate": 3.9727389695022434e-06, "loss": 0.64949697, "num_input_tokens_seen": 18801400, "router_z_loss_clip": 0.04541016, "router_z_loss_mlp": 0.38671875, "step": 877, "time_per_iteration": 3.1060640811920166 }, { "auxiliary_loss_clip": 0.01162697, "auxiliary_loss_mlp": 0.01058139, "balance_loss_clip": 1.02790785, "balance_loss_mlp": 1.04045391, "epoch": 0.05278821584247708, "flos": 17419843445760.0, "grad_norm": 2.6596222001650593, "language_loss": 0.85823625, "learning_rate": 3.972676765317181e-06, "loss": 0.88044465, "num_input_tokens_seen": 18819670, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.21875, "step": 878, "time_per_iteration": 2.431715726852417 }, { "auxiliary_loss_clip": 0.01163295, "auxiliary_loss_mlp": 0.01053228, "balance_loss_clip": 1.02420092, "balance_loss_mlp": 1.04162955, "epoch": 0.052848339095145046, "flos": 26136154880640.0, "grad_norm": 1.9255771100967056, "language_loss": 0.8295579, "learning_rate": 3.97261449073236e-06, "loss": 0.85172307, "num_input_tokens_seen": 18840580, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.21875, "step": 879, "time_per_iteration": 2.469244956970215 }, { "auxiliary_loss_clip": 0.01158954, "auxiliary_loss_mlp": 0.01061843, "balance_loss_clip": 1.03066969, "balance_loss_mlp": 1.03983855, "epoch": 0.05290846234781302, "flos": 16471148538240.0, "grad_norm": 2.0801283716945176, "language_loss": 0.84291494, "learning_rate": 3.9725521457500005e-06, "loss": 0.86512297, "num_input_tokens_seen": 18859295, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.1875, "step": 880, "time_per_iteration": 2.4176676273345947 }, { "auxiliary_loss_clip": 0.01163223, "auxiliary_loss_mlp": 0.01052989, "balance_loss_clip": 1.02311563, "balance_loss_mlp": 1.03948319, "epoch": 0.05296858560048098, "flos": 19864147708800.0, "grad_norm": 2.2089813172294055, "language_loss": 0.86675858, "learning_rate": 3.97248973037233e-06, "loss": 0.88892066, "num_input_tokens_seen": 18877485, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.234375, "step": 881, "time_per_iteration": 3.8267643451690674 }, { "auxiliary_loss_clip": 0.01160676, "auxiliary_loss_mlp": 0.01053967, "balance_loss_clip": 1.02216184, "balance_loss_mlp": 1.03855371, "epoch": 0.053028708853148955, "flos": 24387460012800.0, "grad_norm": 1.9561697480028104, "language_loss": 0.87807399, "learning_rate": 3.972427244601574e-06, "loss": 0.90022039, "num_input_tokens_seen": 18898275, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.21875, "step": 882, "time_per_iteration": 2.4512674808502197 }, { "auxiliary_loss_clip": 0.0116422, "auxiliary_loss_mlp": 0.01053706, "balance_loss_clip": 1.02218688, "balance_loss_mlp": 1.03948426, "epoch": 0.05308883210581693, "flos": 36391681825920.0, "grad_norm": 2.673534550737762, "language_loss": 0.69085759, "learning_rate": 3.972364688439964e-06, "loss": 0.71303678, "num_input_tokens_seen": 18920665, "router_z_loss_clip": 0.31445312, "router_z_loss_mlp": 1.25, "step": 883, "time_per_iteration": 2.545847177505493 }, { "auxiliary_loss_clip": 0.01158398, "auxiliary_loss_mlp": 0.01055761, "balance_loss_clip": 1.02661395, "balance_loss_mlp": 1.04118681, "epoch": 0.05314895535848489, "flos": 22854039788160.0, "grad_norm": 3.3655480070090205, "language_loss": 0.76206219, "learning_rate": 3.9723020618897325e-06, "loss": 0.78420377, "num_input_tokens_seen": 18939835, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.171875, "step": 884, "time_per_iteration": 2.4771573543548584 }, { "auxiliary_loss_clip": 0.01158304, "auxiliary_loss_mlp": 0.01053657, "balance_loss_clip": 1.02576232, "balance_loss_mlp": 1.04068482, "epoch": 0.053209078611152864, "flos": 12859488322560.0, "grad_norm": 2.1558972408245882, "language_loss": 0.8541072, "learning_rate": 3.972239364953113e-06, "loss": 0.87622678, "num_input_tokens_seen": 18958405, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.1796875, "step": 885, "time_per_iteration": 5.304793834686279 }, { "auxiliary_loss_clip": 0.01160873, "auxiliary_loss_mlp": 0.01058031, "balance_loss_clip": 1.02813315, "balance_loss_mlp": 1.03778291, "epoch": 0.05326920186382083, "flos": 12163844016000.0, "grad_norm": 2.5549299977840665, "language_loss": 0.85519499, "learning_rate": 3.9721765976323435e-06, "loss": 0.87738407, "num_input_tokens_seen": 18975445, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.234375, "step": 886, "time_per_iteration": 2.3972959518432617 }, { "auxiliary_loss_clip": 0.01159147, "auxiliary_loss_mlp": 0.01056786, "balance_loss_clip": 1.02691174, "balance_loss_mlp": 1.0391463, "epoch": 0.0533293251164888, "flos": 22703564361600.0, "grad_norm": 2.058024973151939, "language_loss": 0.88818395, "learning_rate": 3.972113759929665e-06, "loss": 0.91034329, "num_input_tokens_seen": 18991930, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.203125, "step": 887, "time_per_iteration": 3.844373941421509 }, { "auxiliary_loss_clip": 0.01159157, "auxiliary_loss_mlp": 0.01071229, "balance_loss_clip": 1.03975785, "balance_loss_mlp": 1.0372448, "epoch": 0.053389448369156774, "flos": 26939785622400.0, "grad_norm": 1.9006520640832827, "language_loss": 0.74924183, "learning_rate": 3.9720508518473186e-06, "loss": 0.77154565, "num_input_tokens_seen": 19009790, "router_z_loss_clip": 0.31445312, "router_z_loss_mlp": 1.21875, "step": 888, "time_per_iteration": 2.4814414978027344 }, { "auxiliary_loss_clip": 0.01159249, "auxiliary_loss_mlp": 0.01061684, "balance_loss_clip": 1.0303793, "balance_loss_mlp": 1.03939033, "epoch": 0.05344957162182474, "flos": 25555165079040.0, "grad_norm": 2.078289993135767, "language_loss": 0.88061041, "learning_rate": 3.97198787338755e-06, "loss": 0.90281975, "num_input_tokens_seen": 19030170, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.203125, "step": 889, "time_per_iteration": 2.435617685317993 }, { "auxiliary_loss_clip": 0.01158143, "auxiliary_loss_mlp": 0.01051812, "balance_loss_clip": 1.02282035, "balance_loss_mlp": 1.03846538, "epoch": 0.05350969487449271, "flos": 19718559872640.0, "grad_norm": 2.57841427158968, "language_loss": 0.88126409, "learning_rate": 3.971924824552607e-06, "loss": 0.9033637, "num_input_tokens_seen": 19048075, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.1953125, "step": 890, "time_per_iteration": 2.4183461666107178 }, { "auxiliary_loss_clip": 0.01160598, "auxiliary_loss_mlp": 0.0106087, "balance_loss_clip": 1.03183031, "balance_loss_mlp": 1.03818941, "epoch": 0.053569818127160676, "flos": 27015128069760.0, "grad_norm": 2.234304571881244, "language_loss": 0.93175459, "learning_rate": 3.97186170534474e-06, "loss": 0.95396924, "num_input_tokens_seen": 19067465, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.21875, "step": 891, "time_per_iteration": 2.4461190700531006 }, { "auxiliary_loss_clip": 0.01161554, "auxiliary_loss_mlp": 0.01060467, "balance_loss_clip": 1.02894807, "balance_loss_mlp": 1.03905725, "epoch": 0.05362994137982865, "flos": 13187497345920.0, "grad_norm": 2.2302235095241234, "language_loss": 0.71824193, "learning_rate": 3.9717985157662e-06, "loss": 0.74046212, "num_input_tokens_seen": 19085505, "router_z_loss_clip": 0.31445312, "router_z_loss_mlp": 1.2265625, "step": 892, "time_per_iteration": 2.4148337841033936 }, { "auxiliary_loss_clip": 0.01162488, "auxiliary_loss_mlp": 0.01068779, "balance_loss_clip": 1.03904855, "balance_loss_mlp": 1.03886676, "epoch": 0.05369006463249662, "flos": 28656744197760.0, "grad_norm": 1.8684471729019887, "language_loss": 0.82398784, "learning_rate": 3.971735255819244e-06, "loss": 0.8463006, "num_input_tokens_seen": 19104360, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.234375, "step": 893, "time_per_iteration": 2.4615511894226074 }, { "auxiliary_loss_clip": 0.01161726, "auxiliary_loss_mlp": 0.01058905, "balance_loss_clip": 1.02868581, "balance_loss_mlp": 1.03838944, "epoch": 0.053750187885164585, "flos": 28911889480320.0, "grad_norm": 2.5652337661280993, "language_loss": 0.81720483, "learning_rate": 3.971671925506129e-06, "loss": 0.83941114, "num_input_tokens_seen": 19124680, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.234375, "step": 894, "time_per_iteration": 2.492638349533081 }, { "auxiliary_loss_clip": 0.01157092, "auxiliary_loss_mlp": 0.01059332, "balance_loss_clip": 1.02929115, "balance_loss_mlp": 1.03694069, "epoch": 0.05381031113783256, "flos": 15157925458560.0, "grad_norm": 3.843311202633951, "language_loss": 0.75092781, "learning_rate": 3.9716085248291125e-06, "loss": 0.77309203, "num_input_tokens_seen": 19142895, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.203125, "step": 895, "time_per_iteration": 2.380563974380493 }, { "auxiliary_loss_clip": 0.0116928, "auxiliary_loss_mlp": 0.01058747, "balance_loss_clip": 1.02970707, "balance_loss_mlp": 1.0438832, "epoch": 0.05387043439050053, "flos": 21834156885120.0, "grad_norm": 2.4046720652318454, "language_loss": 0.86494035, "learning_rate": 3.97154505379046e-06, "loss": 0.88722062, "num_input_tokens_seen": 19163125, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.25, "step": 896, "time_per_iteration": 2.43613338470459 }, { "auxiliary_loss_clip": 0.01164866, "auxiliary_loss_mlp": 0.01054395, "balance_loss_clip": 1.02225626, "balance_loss_mlp": 1.03944147, "epoch": 0.053930557643168495, "flos": 17309378304000.0, "grad_norm": 6.219713565318159, "language_loss": 0.88008451, "learning_rate": 3.971481512392438e-06, "loss": 0.90227711, "num_input_tokens_seen": 19179385, "router_z_loss_clip": 0.3203125, "router_z_loss_mlp": 1.25, "step": 897, "time_per_iteration": 2.381504535675049 }, { "auxiliary_loss_clip": 0.01159566, "auxiliary_loss_mlp": 0.01061987, "balance_loss_clip": 1.03080177, "balance_loss_mlp": 1.03830087, "epoch": 0.05399068089583647, "flos": 17347503375360.0, "grad_norm": 1.7870846423499203, "language_loss": 0.90078026, "learning_rate": 3.97141790063731e-06, "loss": 0.92299581, "num_input_tokens_seen": 19198725, "router_z_loss_clip": 0.31054688, "router_z_loss_mlp": 1.21875, "step": 898, "time_per_iteration": 2.3894176483154297 }, { "auxiliary_loss_clip": 0.01164325, "auxiliary_loss_mlp": 0.01069351, "balance_loss_clip": 1.03890502, "balance_loss_mlp": 1.03923023, "epoch": 0.05405080414850443, "flos": 17486178762240.0, "grad_norm": 2.6068244252625465, "language_loss": 0.92166436, "learning_rate": 3.971354218527349e-06, "loss": 0.94400114, "num_input_tokens_seen": 19212380, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.25, "step": 899, "time_per_iteration": 2.374274969100952 }, { "auxiliary_loss_clip": 0.01159553, "auxiliary_loss_mlp": 0.01058724, "balance_loss_clip": 1.02880204, "balance_loss_mlp": 1.03960335, "epoch": 0.054110927401172404, "flos": 24495690827520.0, "grad_norm": 2.03080882462333, "language_loss": 0.75723553, "learning_rate": 3.971290466064827e-06, "loss": 0.77941823, "num_input_tokens_seen": 19232235, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.203125, "step": 900, "time_per_iteration": 2.5287563800811768 }, { "auxiliary_loss_clip": 0.01161827, "auxiliary_loss_mlp": 0.01055122, "balance_loss_clip": 1.0256772, "balance_loss_mlp": 1.03749549, "epoch": 0.054171050653840376, "flos": 22928928387840.0, "grad_norm": 3.188934588277214, "language_loss": 0.73738217, "learning_rate": 3.971226643252019e-06, "loss": 0.7595517, "num_input_tokens_seen": 19251460, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.25, "step": 901, "time_per_iteration": 2.408686876296997 }, { "auxiliary_loss_clip": 0.01154546, "auxiliary_loss_mlp": 0.01061796, "balance_loss_clip": 1.03520036, "balance_loss_mlp": 1.03875566, "epoch": 0.05423117390650834, "flos": 12932352063360.0, "grad_norm": 2.102878927305348, "language_loss": 0.8485086, "learning_rate": 3.971162750091202e-06, "loss": 0.87067199, "num_input_tokens_seen": 19269060, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.15625, "step": 902, "time_per_iteration": 2.3947722911834717 }, { "auxiliary_loss_clip": 0.01158079, "auxiliary_loss_mlp": 0.0105659, "balance_loss_clip": 1.02650177, "balance_loss_mlp": 1.03704238, "epoch": 0.05429129715917631, "flos": 19900317744000.0, "grad_norm": 2.06799905004133, "language_loss": 0.86127782, "learning_rate": 3.971098786584657e-06, "loss": 0.88342452, "num_input_tokens_seen": 19288620, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.2109375, "step": 903, "time_per_iteration": 2.4634947776794434 }, { "auxiliary_loss_clip": 0.01156319, "auxiliary_loss_mlp": 0.01050202, "balance_loss_clip": 1.02140117, "balance_loss_mlp": 1.03751755, "epoch": 0.05435142041184428, "flos": 16907702578560.0, "grad_norm": 2.4141723378563067, "language_loss": 0.75000405, "learning_rate": 3.971034752734668e-06, "loss": 0.77206928, "num_input_tokens_seen": 19306615, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.1875, "step": 904, "time_per_iteration": 2.384925365447998 }, { "auxiliary_loss_clip": 0.01162563, "auxiliary_loss_mlp": 0.01054817, "balance_loss_clip": 1.02602839, "balance_loss_mlp": 1.04079747, "epoch": 0.05441154366451225, "flos": 23947275191040.0, "grad_norm": 2.6027161513202386, "language_loss": 0.85758334, "learning_rate": 3.970970648543517e-06, "loss": 0.87975711, "num_input_tokens_seen": 19321680, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.21875, "step": 905, "time_per_iteration": 2.423539161682129 }, { "auxiliary_loss_clip": 0.01157356, "auxiliary_loss_mlp": 0.01053976, "balance_loss_clip": 1.02722573, "balance_loss_mlp": 1.0414567, "epoch": 0.05447166691718022, "flos": 19974333559680.0, "grad_norm": 3.568337735308201, "language_loss": 0.74576402, "learning_rate": 3.970906474013494e-06, "loss": 0.76787734, "num_input_tokens_seen": 19339760, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.15625, "step": 906, "time_per_iteration": 2.4099528789520264 }, { "auxiliary_loss_clip": 0.0116229, "auxiliary_loss_mlp": 0.01055882, "balance_loss_clip": 1.02797496, "balance_loss_mlp": 1.03734863, "epoch": 0.05453179016984819, "flos": 24935351978880.0, "grad_norm": 1.9587767139692178, "language_loss": 0.86923331, "learning_rate": 3.97084222914689e-06, "loss": 0.891415, "num_input_tokens_seen": 19359585, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.25, "step": 907, "time_per_iteration": 2.467411994934082 }, { "auxiliary_loss_clip": 0.0116156, "auxiliary_loss_mlp": 0.01063412, "balance_loss_clip": 1.03406227, "balance_loss_mlp": 1.04133844, "epoch": 0.05459191342251616, "flos": 18114091297920.0, "grad_norm": 3.51243039724905, "language_loss": 0.86991906, "learning_rate": 3.970777913945995e-06, "loss": 0.89216876, "num_input_tokens_seen": 19378590, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.203125, "step": 908, "time_per_iteration": 2.421147584915161 }, { "auxiliary_loss_clip": 0.01160336, "auxiliary_loss_mlp": 0.01057968, "balance_loss_clip": 1.02871394, "balance_loss_mlp": 1.04014111, "epoch": 0.054652036675184125, "flos": 19207291789440.0, "grad_norm": 2.1778556300511402, "language_loss": 0.89483535, "learning_rate": 3.970713528413106e-06, "loss": 0.91701841, "num_input_tokens_seen": 19397910, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.203125, "step": 909, "time_per_iteration": 2.433941602706909 }, { "auxiliary_loss_clip": 0.01161949, "auxiliary_loss_mlp": 0.01063548, "balance_loss_clip": 1.03183877, "balance_loss_mlp": 1.03985989, "epoch": 0.0547121599278521, "flos": 16324827563520.0, "grad_norm": 4.50022788736104, "language_loss": 0.71124053, "learning_rate": 3.9706490725505205e-06, "loss": 0.73349547, "num_input_tokens_seen": 19415950, "router_z_loss_clip": 0.31640625, "router_z_loss_mlp": 1.21875, "step": 910, "time_per_iteration": 2.3976662158966064 }, { "auxiliary_loss_clip": 0.01156027, "auxiliary_loss_mlp": 0.01053817, "balance_loss_clip": 1.02468252, "balance_loss_mlp": 1.0386641, "epoch": 0.05477228318052007, "flos": 20337988947840.0, "grad_norm": 1.814052792527829, "language_loss": 0.83245134, "learning_rate": 3.970584546360539e-06, "loss": 0.85454977, "num_input_tokens_seen": 19435275, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.171875, "step": 911, "time_per_iteration": 2.4349710941314697 }, { "auxiliary_loss_clip": 0.01159004, "auxiliary_loss_mlp": 0.01054041, "balance_loss_clip": 1.02214098, "balance_loss_mlp": 1.03775454, "epoch": 0.054832406433188034, "flos": 21972238778880.0, "grad_norm": 3.1255396403293156, "language_loss": 0.75924587, "learning_rate": 3.970519949845464e-06, "loss": 0.78137636, "num_input_tokens_seen": 19452090, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.2109375, "step": 912, "time_per_iteration": 2.39033842086792 }, { "auxiliary_loss_clip": 0.01155644, "auxiliary_loss_mlp": 0.01056618, "balance_loss_clip": 1.02753103, "balance_loss_mlp": 1.03938627, "epoch": 0.054892529685856006, "flos": 16398005506560.0, "grad_norm": 2.468181012949965, "language_loss": 0.82650316, "learning_rate": 3.9704552830076005e-06, "loss": 0.84862584, "num_input_tokens_seen": 19470865, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.15625, "step": 913, "time_per_iteration": 2.4073081016540527 }, { "auxiliary_loss_clip": 0.01157295, "auxiliary_loss_mlp": 0.01052201, "balance_loss_clip": 1.02399659, "balance_loss_mlp": 1.04099143, "epoch": 0.05495265293852397, "flos": 23911279712640.0, "grad_norm": 2.064636756581716, "language_loss": 0.8323791, "learning_rate": 3.9703905458492564e-06, "loss": 0.85447407, "num_input_tokens_seen": 19492145, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.15625, "step": 914, "time_per_iteration": 2.4616618156433105 }, { "auxiliary_loss_clip": 0.01161332, "auxiliary_loss_mlp": 0.01057481, "balance_loss_clip": 1.02823925, "balance_loss_mlp": 1.04124427, "epoch": 0.055012776191191944, "flos": 23585819218560.0, "grad_norm": 3.304359884839736, "language_loss": 0.8976059, "learning_rate": 3.970325738372742e-06, "loss": 0.91979396, "num_input_tokens_seen": 19511015, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.203125, "step": 915, "time_per_iteration": 2.444795608520508 }, { "auxiliary_loss_clip": 0.01158095, "auxiliary_loss_mlp": 0.01058386, "balance_loss_clip": 1.03014517, "balance_loss_mlp": 1.03939843, "epoch": 0.055072899443859916, "flos": 17527585501440.0, "grad_norm": 1.733987534023442, "language_loss": 0.89628351, "learning_rate": 3.970260860580371e-06, "loss": 0.91844833, "num_input_tokens_seen": 19529040, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.1875, "step": 916, "time_per_iteration": 2.412970542907715 }, { "auxiliary_loss_clip": 0.01160748, "auxiliary_loss_mlp": 0.01062094, "balance_loss_clip": 1.03136241, "balance_loss_mlp": 1.03984213, "epoch": 0.05513302269652788, "flos": 21686160165120.0, "grad_norm": 4.549710784254671, "language_loss": 0.79854846, "learning_rate": 3.970195912474457e-06, "loss": 0.82077694, "num_input_tokens_seen": 19549540, "router_z_loss_clip": 0.30664062, "router_z_loss_mlp": 1.2109375, "step": 917, "time_per_iteration": 2.4622411727905273 }, { "auxiliary_loss_clip": 0.0116097, "auxiliary_loss_mlp": 0.01055916, "balance_loss_clip": 1.02758026, "balance_loss_mlp": 1.03956866, "epoch": 0.05519314594919585, "flos": 21612353817600.0, "grad_norm": 2.0877874916789505, "language_loss": 0.79856837, "learning_rate": 3.9701308940573195e-06, "loss": 0.82073724, "num_input_tokens_seen": 19567570, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.2109375, "step": 918, "time_per_iteration": 2.42089581489563 }, { "auxiliary_loss_clip": 0.01158156, "auxiliary_loss_mlp": 0.01048415, "balance_loss_clip": 1.0192802, "balance_loss_mlp": 1.03866529, "epoch": 0.05525326920186382, "flos": 21797498090880.0, "grad_norm": 1.8702226836959464, "language_loss": 0.88963503, "learning_rate": 3.970065805331279e-06, "loss": 0.91170073, "num_input_tokens_seen": 19585330, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.1953125, "step": 919, "time_per_iteration": 2.4145870208740234 }, { "auxiliary_loss_clip": 0.01155088, "auxiliary_loss_mlp": 0.01054277, "balance_loss_clip": 1.02604783, "balance_loss_mlp": 1.0378617, "epoch": 0.05531339245453179, "flos": 28438362443520.0, "grad_norm": 2.341967609968428, "language_loss": 0.86990917, "learning_rate": 3.970000646298656e-06, "loss": 0.89200282, "num_input_tokens_seen": 19604970, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.171875, "step": 920, "time_per_iteration": 2.4621634483337402 }, { "auxiliary_loss_clip": 0.0115865, "auxiliary_loss_mlp": 0.01054321, "balance_loss_clip": 1.02559161, "balance_loss_mlp": 1.03921771, "epoch": 0.05537351570719976, "flos": 37373718948480.0, "grad_norm": 2.1703236391249847, "language_loss": 0.65769506, "learning_rate": 3.969935416961778e-06, "loss": 0.67982477, "num_input_tokens_seen": 19626235, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.1953125, "step": 921, "time_per_iteration": 3.9598336219787598 }, { "auxiliary_loss_clip": 0.01163449, "auxiliary_loss_mlp": 0.01055815, "balance_loss_clip": 1.0239861, "balance_loss_mlp": 1.04239345, "epoch": 0.05543363895986773, "flos": 20083437158400.0, "grad_norm": 4.7013371739042045, "language_loss": 0.71693504, "learning_rate": 3.969870117322973e-06, "loss": 0.7391277, "num_input_tokens_seen": 19644305, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.2109375, "step": 922, "time_per_iteration": 2.426201581954956 }, { "auxiliary_loss_clip": 0.01161821, "auxiliary_loss_mlp": 0.01062252, "balance_loss_clip": 1.03109097, "balance_loss_mlp": 1.03957999, "epoch": 0.0554937622125357, "flos": 24532105242240.0, "grad_norm": 2.542450463504702, "language_loss": 0.82041645, "learning_rate": 3.96980474738457e-06, "loss": 0.84265721, "num_input_tokens_seen": 19662130, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.21875, "step": 923, "time_per_iteration": 2.434980869293213 }, { "auxiliary_loss_clip": 0.01159792, "auxiliary_loss_mlp": 0.01056812, "balance_loss_clip": 1.02710509, "balance_loss_mlp": 1.0386827, "epoch": 0.055553885465203665, "flos": 14319172022400.0, "grad_norm": 2.0976337306139627, "language_loss": 0.78356576, "learning_rate": 3.969739307148902e-06, "loss": 0.80573177, "num_input_tokens_seen": 19680715, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.2109375, "step": 924, "time_per_iteration": 2.3935599327087402 }, { "auxiliary_loss_clip": 0.01158258, "auxiliary_loss_mlp": 0.01051702, "balance_loss_clip": 1.0231396, "balance_loss_mlp": 1.03932667, "epoch": 0.05561400871787164, "flos": 27379900621440.0, "grad_norm": 1.9906794558315535, "language_loss": 1.0172838, "learning_rate": 3.969673796618306e-06, "loss": 1.03938341, "num_input_tokens_seen": 19700535, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.1875, "step": 925, "time_per_iteration": 5.295424461364746 }, { "auxiliary_loss_clip": 0.01158902, "auxiliary_loss_mlp": 0.01047981, "balance_loss_clip": 1.01815462, "balance_loss_mlp": 1.03811073, "epoch": 0.05567413197053961, "flos": 23219999326080.0, "grad_norm": 1.9889175308308498, "language_loss": 0.80755478, "learning_rate": 3.969608215795117e-06, "loss": 0.82962364, "num_input_tokens_seen": 19718825, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.203125, "step": 926, "time_per_iteration": 2.4128670692443848 }, { "auxiliary_loss_clip": 0.01166394, "auxiliary_loss_mlp": 0.01050217, "balance_loss_clip": 1.01919866, "balance_loss_mlp": 1.04073811, "epoch": 0.055734255223207574, "flos": 25263779938560.0, "grad_norm": 2.2196935082358995, "language_loss": 0.7284615, "learning_rate": 3.969542564681679e-06, "loss": 0.75062764, "num_input_tokens_seen": 19739080, "router_z_loss_clip": 0.31054688, "router_z_loss_mlp": 1.2578125, "step": 927, "time_per_iteration": 3.88985538482666 }, { "auxiliary_loss_clip": 0.01047593, "auxiliary_loss_mlp": 0.01003839, "balance_loss_clip": 0.99959511, "balance_loss_mlp": 1.00975871, "epoch": 0.055794378475875546, "flos": 66499519662720.0, "grad_norm": 0.7940120874990624, "language_loss": 0.59834445, "learning_rate": 3.969476843280333e-06, "loss": 0.61885875, "num_input_tokens_seen": 19802960, "router_z_loss_clip": 0.04248047, "router_z_loss_mlp": 0.37890625, "step": 928, "time_per_iteration": 3.073519706726074 }, { "auxiliary_loss_clip": 0.01161437, "auxiliary_loss_mlp": 0.01059613, "balance_loss_clip": 1.02945352, "balance_loss_mlp": 1.04066682, "epoch": 0.05585450172854351, "flos": 25336469122560.0, "grad_norm": 2.741727701465678, "language_loss": 0.94735438, "learning_rate": 3.969411051593424e-06, "loss": 0.96956486, "num_input_tokens_seen": 19822765, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.203125, "step": 929, "time_per_iteration": 2.4511215686798096 }, { "auxiliary_loss_clip": 0.01159328, "auxiliary_loss_mlp": 0.01056288, "balance_loss_clip": 1.02419722, "balance_loss_mlp": 1.03711009, "epoch": 0.05591462498121148, "flos": 33910334743680.0, "grad_norm": 2.0145312235320367, "language_loss": 0.71520591, "learning_rate": 3.9693451896233e-06, "loss": 0.73736215, "num_input_tokens_seen": 19843590, "router_z_loss_clip": 0.3203125, "router_z_loss_mlp": 1.21875, "step": 930, "time_per_iteration": 2.519730567932129 }, { "auxiliary_loss_clip": 0.01162948, "auxiliary_loss_mlp": 0.01054579, "balance_loss_clip": 1.02428746, "balance_loss_mlp": 1.04021239, "epoch": 0.055974748233879455, "flos": 17929924542720.0, "grad_norm": 7.092155019963012, "language_loss": 0.84803557, "learning_rate": 3.969279257372313e-06, "loss": 0.87021089, "num_input_tokens_seen": 19860230, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.2265625, "step": 931, "time_per_iteration": 2.387993097305298 }, { "auxiliary_loss_clip": 0.01163628, "auxiliary_loss_mlp": 0.01061806, "balance_loss_clip": 1.03116918, "balance_loss_mlp": 1.03927064, "epoch": 0.05603487148654742, "flos": 24020906981760.0, "grad_norm": 1.7785238580422558, "language_loss": 0.83289844, "learning_rate": 3.969213254842814e-06, "loss": 0.85515279, "num_input_tokens_seen": 19880795, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.25, "step": 932, "time_per_iteration": 2.460404634475708 }, { "auxiliary_loss_clip": 0.01163613, "auxiliary_loss_mlp": 0.01056253, "balance_loss_clip": 1.02413821, "balance_loss_mlp": 1.04108763, "epoch": 0.05609499473921539, "flos": 17306899597440.0, "grad_norm": 2.233640975121556, "language_loss": 0.73572028, "learning_rate": 3.9691471820371594e-06, "loss": 0.75791895, "num_input_tokens_seen": 19897960, "router_z_loss_clip": 0.3203125, "router_z_loss_mlp": 1.2265625, "step": 933, "time_per_iteration": 2.430478096008301 }, { "auxiliary_loss_clip": 0.0115802, "auxiliary_loss_mlp": 0.01060328, "balance_loss_clip": 1.02950001, "balance_loss_mlp": 1.03795409, "epoch": 0.05615511799188336, "flos": 20993727703680.0, "grad_norm": 2.738979518074269, "language_loss": 0.86471808, "learning_rate": 3.969081038957708e-06, "loss": 0.88690156, "num_input_tokens_seen": 19913315, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.203125, "step": 934, "time_per_iteration": 2.3857738971710205 }, { "auxiliary_loss_clip": 0.01153999, "auxiliary_loss_mlp": 0.01058784, "balance_loss_clip": 1.03041255, "balance_loss_mlp": 1.03940582, "epoch": 0.05621524124455133, "flos": 17272614775680.0, "grad_norm": 2.1381230663333164, "language_loss": 0.80012619, "learning_rate": 3.969014825606819e-06, "loss": 0.82225406, "num_input_tokens_seen": 19928790, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.1484375, "step": 935, "time_per_iteration": 2.3873655796051025 }, { "auxiliary_loss_clip": 0.01043219, "auxiliary_loss_mlp": 0.01005184, "balance_loss_clip": 1.00051081, "balance_loss_mlp": 1.00608182, "epoch": 0.0562753644972193, "flos": 58716332668800.0, "grad_norm": 0.8227501343542768, "language_loss": 0.69188774, "learning_rate": 3.968948541986855e-06, "loss": 0.71237177, "num_input_tokens_seen": 19988785, "router_z_loss_clip": 0.04663086, "router_z_loss_mlp": 0.37109375, "step": 936, "time_per_iteration": 2.932800531387329 }, { "auxiliary_loss_clip": 0.01158892, "auxiliary_loss_mlp": 0.01053648, "balance_loss_clip": 1.02417982, "balance_loss_mlp": 1.03832948, "epoch": 0.05633548774988727, "flos": 17456083303680.0, "grad_norm": 2.9910462478789834, "language_loss": 0.75406981, "learning_rate": 3.968882188100183e-06, "loss": 0.77619517, "num_input_tokens_seen": 20007685, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.203125, "step": 937, "time_per_iteration": 2.412381410598755 }, { "auxiliary_loss_clip": 0.01042286, "auxiliary_loss_mlp": 0.01003793, "balance_loss_clip": 0.99935871, "balance_loss_mlp": 1.00569856, "epoch": 0.05639561100255524, "flos": 70651426256640.0, "grad_norm": 0.8602694962622135, "language_loss": 0.64379501, "learning_rate": 3.9688157639491704e-06, "loss": 0.66425586, "num_input_tokens_seen": 20072750, "router_z_loss_clip": 0.04443359, "router_z_loss_mlp": 0.3671875, "step": 938, "time_per_iteration": 3.023224353790283 }, { "auxiliary_loss_clip": 0.01166904, "auxiliary_loss_mlp": 0.01058033, "balance_loss_clip": 1.02677608, "balance_loss_mlp": 1.03905725, "epoch": 0.056455734255223204, "flos": 20484938327040.0, "grad_norm": 2.812712479682215, "language_loss": 0.79116201, "learning_rate": 3.968749269536188e-06, "loss": 0.81341136, "num_input_tokens_seen": 20089070, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.28125, "step": 939, "time_per_iteration": 2.4041600227355957 }, { "auxiliary_loss_clip": 0.01158136, "auxiliary_loss_mlp": 0.01054269, "balance_loss_clip": 1.02573061, "balance_loss_mlp": 1.03815985, "epoch": 0.056515857507891176, "flos": 22052503728000.0, "grad_norm": 1.798056398583246, "language_loss": 0.73791158, "learning_rate": 3.9686827048636074e-06, "loss": 0.76003563, "num_input_tokens_seen": 20108790, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.203125, "step": 940, "time_per_iteration": 2.4448323249816895 }, { "auxiliary_loss_clip": 0.01159927, "auxiliary_loss_mlp": 0.01061686, "balance_loss_clip": 1.03264642, "balance_loss_mlp": 1.04005516, "epoch": 0.05657598076055915, "flos": 24024153738240.0, "grad_norm": 1.873028528359329, "language_loss": 0.70337206, "learning_rate": 3.968616069933806e-06, "loss": 0.7255882, "num_input_tokens_seen": 20128455, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.203125, "step": 941, "time_per_iteration": 2.443727731704712 }, { "auxiliary_loss_clip": 0.01156185, "auxiliary_loss_mlp": 0.01053917, "balance_loss_clip": 1.02378058, "balance_loss_mlp": 1.03895378, "epoch": 0.05663610401322711, "flos": 20479701623040.0, "grad_norm": 1.8997356430322379, "language_loss": 0.806705, "learning_rate": 3.96854936474916e-06, "loss": 0.82880604, "num_input_tokens_seen": 20145775, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.171875, "step": 942, "time_per_iteration": 2.40793514251709 }, { "auxiliary_loss_clip": 0.01156782, "auxiliary_loss_mlp": 0.01056196, "balance_loss_clip": 1.02615571, "balance_loss_mlp": 1.03941846, "epoch": 0.056696227265895086, "flos": 21067987898880.0, "grad_norm": 2.2031773806584423, "language_loss": 0.880005, "learning_rate": 3.968482589312052e-06, "loss": 0.90213478, "num_input_tokens_seen": 20164315, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.171875, "step": 943, "time_per_iteration": 2.4251973628997803 }, { "auxiliary_loss_clip": 0.01160592, "auxiliary_loss_mlp": 0.01055373, "balance_loss_clip": 1.02628577, "balance_loss_mlp": 1.04070008, "epoch": 0.05675635051856306, "flos": 17820367096320.0, "grad_norm": 2.2903411620696725, "language_loss": 0.74629074, "learning_rate": 3.968415743624863e-06, "loss": 0.76845038, "num_input_tokens_seen": 20182760, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.1953125, "step": 944, "time_per_iteration": 2.4026761054992676 }, { "auxiliary_loss_clip": 0.01155027, "auxiliary_loss_mlp": 0.01055523, "balance_loss_clip": 1.02712703, "balance_loss_mlp": 1.03648019, "epoch": 0.05681647377123102, "flos": 23113758458880.0, "grad_norm": 1.5914147172454032, "language_loss": 0.79131436, "learning_rate": 3.9683488276899794e-06, "loss": 0.81341994, "num_input_tokens_seen": 20203830, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.1875, "step": 945, "time_per_iteration": 2.5026559829711914 }, { "auxiliary_loss_clip": 0.01158836, "auxiliary_loss_mlp": 0.01053737, "balance_loss_clip": 1.02420902, "balance_loss_mlp": 1.03774977, "epoch": 0.056876597023898995, "flos": 16069612458240.0, "grad_norm": 2.2531418065513664, "language_loss": 0.82614088, "learning_rate": 3.96828184150979e-06, "loss": 0.8482666, "num_input_tokens_seen": 20220365, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.2109375, "step": 946, "time_per_iteration": 2.393521785736084 }, { "auxiliary_loss_clip": 0.01164953, "auxiliary_loss_mlp": 0.01056321, "balance_loss_clip": 1.0244925, "balance_loss_mlp": 1.04142892, "epoch": 0.05693672027656696, "flos": 16834734103680.0, "grad_norm": 1.9265319793675204, "language_loss": 0.79115474, "learning_rate": 3.968214785086684e-06, "loss": 0.81336749, "num_input_tokens_seen": 20238640, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.234375, "step": 947, "time_per_iteration": 2.4404330253601074 }, { "auxiliary_loss_clip": 0.01163781, "auxiliary_loss_mlp": 0.01061873, "balance_loss_clip": 1.0308311, "balance_loss_mlp": 1.04187322, "epoch": 0.05699684352923493, "flos": 21388281511680.0, "grad_norm": 3.7219270590303255, "language_loss": 0.8536315, "learning_rate": 3.968147658423056e-06, "loss": 0.87588805, "num_input_tokens_seen": 20251025, "router_z_loss_clip": 0.31054688, "router_z_loss_mlp": 1.21875, "step": 948, "time_per_iteration": 2.3871397972106934 }, { "auxiliary_loss_clip": 0.01161505, "auxiliary_loss_mlp": 0.01060613, "balance_loss_clip": 1.0270915, "balance_loss_mlp": 1.04082263, "epoch": 0.057056966781902904, "flos": 15559391715840.0, "grad_norm": 1.8594148227814742, "language_loss": 0.87232089, "learning_rate": 3.9680804615213e-06, "loss": 0.8945421, "num_input_tokens_seen": 20269775, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.203125, "step": 949, "time_per_iteration": 2.4071102142333984 }, { "auxiliary_loss_clip": 0.01155606, "auxiliary_loss_mlp": 0.01054039, "balance_loss_clip": 1.02651358, "balance_loss_mlp": 1.03915536, "epoch": 0.05711709003457087, "flos": 19936836892800.0, "grad_norm": 2.011604601070385, "language_loss": 0.78427905, "learning_rate": 3.968013194383815e-06, "loss": 0.8063755, "num_input_tokens_seen": 20287715, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.1640625, "step": 950, "time_per_iteration": 2.3976221084594727 }, { "auxiliary_loss_clip": 0.0116531, "auxiliary_loss_mlp": 0.01060047, "balance_loss_clip": 1.03012562, "balance_loss_mlp": 1.04249465, "epoch": 0.05717721328723884, "flos": 30331493072640.0, "grad_norm": 2.233275216295547, "language_loss": 0.82126546, "learning_rate": 3.967945857013002e-06, "loss": 0.84351903, "num_input_tokens_seen": 20307070, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.2265625, "step": 951, "time_per_iteration": 2.5157175064086914 }, { "auxiliary_loss_clip": 0.01157948, "auxiliary_loss_mlp": 0.0106173, "balance_loss_clip": 1.03192782, "balance_loss_mlp": 1.03772712, "epoch": 0.05723733653990681, "flos": 23653376432640.0, "grad_norm": 2.473968384898655, "language_loss": 0.86654651, "learning_rate": 3.967878449411263e-06, "loss": 0.88874328, "num_input_tokens_seen": 20324945, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.203125, "step": 952, "time_per_iteration": 2.41511607170105 }, { "auxiliary_loss_clip": 0.0116263, "auxiliary_loss_mlp": 0.01060063, "balance_loss_clip": 1.02770972, "balance_loss_mlp": 1.03811193, "epoch": 0.05729745979257478, "flos": 22054633320960.0, "grad_norm": 1.921396913652021, "language_loss": 0.79379117, "learning_rate": 3.967810971581004e-06, "loss": 0.81601816, "num_input_tokens_seen": 20346135, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.2421875, "step": 953, "time_per_iteration": 2.4393908977508545 }, { "auxiliary_loss_clip": 0.01162789, "auxiliary_loss_mlp": 0.01057679, "balance_loss_clip": 1.0272572, "balance_loss_mlp": 1.04243231, "epoch": 0.05735758304524275, "flos": 19603486431360.0, "grad_norm": 2.1109943663128177, "language_loss": 0.86476898, "learning_rate": 3.967743423524633e-06, "loss": 0.88697374, "num_input_tokens_seen": 20364450, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.203125, "step": 954, "time_per_iteration": 2.4184181690216064 }, { "auxiliary_loss_clip": 0.01162818, "auxiliary_loss_mlp": 0.01056128, "balance_loss_clip": 1.02434659, "balance_loss_mlp": 1.040133, "epoch": 0.057417706297910716, "flos": 19098013633920.0, "grad_norm": 2.4544342404404973, "language_loss": 0.87799019, "learning_rate": 3.967675805244562e-06, "loss": 0.90017962, "num_input_tokens_seen": 20383500, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.2265625, "step": 955, "time_per_iteration": 2.424464225769043 }, { "auxiliary_loss_clip": 0.01158728, "auxiliary_loss_mlp": 0.01057114, "balance_loss_clip": 1.0270493, "balance_loss_mlp": 1.03928959, "epoch": 0.05747782955057869, "flos": 16653569725440.0, "grad_norm": 2.2318971564074923, "language_loss": 0.89087892, "learning_rate": 3.967608116743202e-06, "loss": 0.9130373, "num_input_tokens_seen": 20400295, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.1953125, "step": 956, "time_per_iteration": 2.369446039199829 }, { "auxiliary_loss_clip": 0.01159596, "auxiliary_loss_mlp": 0.01056179, "balance_loss_clip": 1.02831984, "balance_loss_mlp": 1.0415616, "epoch": 0.05753795280324665, "flos": 14501174273280.0, "grad_norm": 2.8091922156104077, "language_loss": 0.75586867, "learning_rate": 3.96754035802297e-06, "loss": 0.7780264, "num_input_tokens_seen": 20419085, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.1796875, "step": 957, "time_per_iteration": 2.4029459953308105 }, { "auxiliary_loss_clip": 0.01164806, "auxiliary_loss_mlp": 0.01067315, "balance_loss_clip": 1.03374577, "balance_loss_mlp": 1.04087114, "epoch": 0.057598076055914625, "flos": 18075372733440.0, "grad_norm": 2.0366571588287563, "language_loss": 0.79876363, "learning_rate": 3.967472529086284e-06, "loss": 0.82108486, "num_input_tokens_seen": 20437465, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.234375, "step": 958, "time_per_iteration": 2.418729066848755 }, { "auxiliary_loss_clip": 0.01158242, "auxiliary_loss_mlp": 0.01049047, "balance_loss_clip": 1.02119946, "balance_loss_mlp": 1.03792882, "epoch": 0.0576581993085826, "flos": 22123586989440.0, "grad_norm": 2.6994391636694663, "language_loss": 0.88083041, "learning_rate": 3.967404629935564e-06, "loss": 0.90290332, "num_input_tokens_seen": 20456235, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.203125, "step": 959, "time_per_iteration": 2.4110212326049805 }, { "auxiliary_loss_clip": 0.01153443, "auxiliary_loss_mlp": 0.01049098, "balance_loss_clip": 1.01970053, "balance_loss_mlp": 1.03854108, "epoch": 0.05771832256125056, "flos": 33180370704000.0, "grad_norm": 10.099762558081204, "language_loss": 0.7851907, "learning_rate": 3.9673366605732335e-06, "loss": 0.80721611, "num_input_tokens_seen": 20476825, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.1484375, "step": 960, "time_per_iteration": 2.5175628662109375 }, { "auxiliary_loss_clip": 0.01158291, "auxiliary_loss_mlp": 0.01052655, "balance_loss_clip": 1.0229001, "balance_loss_mlp": 1.03849053, "epoch": 0.057778445813918534, "flos": 24169008435840.0, "grad_norm": 2.056220829596911, "language_loss": 0.93077898, "learning_rate": 3.967268621001718e-06, "loss": 0.95288843, "num_input_tokens_seen": 20496965, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.1953125, "step": 961, "time_per_iteration": 3.873457193374634 }, { "auxiliary_loss_clip": 0.01158597, "auxiliary_loss_mlp": 0.01059004, "balance_loss_clip": 1.02731872, "balance_loss_mlp": 1.03795218, "epoch": 0.0578385690665865, "flos": 29641748785920.0, "grad_norm": 2.839739402560139, "language_loss": 0.68123364, "learning_rate": 3.967200511223446e-06, "loss": 0.70340973, "num_input_tokens_seen": 20518035, "router_z_loss_clip": 0.31640625, "router_z_loss_mlp": 1.2109375, "step": 962, "time_per_iteration": 2.462498664855957 }, { "auxiliary_loss_clip": 0.01158783, "auxiliary_loss_mlp": 0.01056919, "balance_loss_clip": 1.02739072, "balance_loss_mlp": 1.04080355, "epoch": 0.05789869231925447, "flos": 20884414636800.0, "grad_norm": 2.6681272796667352, "language_loss": 0.88147473, "learning_rate": 3.967132331240848e-06, "loss": 0.90363169, "num_input_tokens_seen": 20534740, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.1796875, "step": 963, "time_per_iteration": 2.4187171459198 }, { "auxiliary_loss_clip": 0.01162174, "auxiliary_loss_mlp": 0.01047752, "balance_loss_clip": 1.01761627, "balance_loss_mlp": 1.04180944, "epoch": 0.057958815571922444, "flos": 26029914013440.0, "grad_norm": 2.1959636755272665, "language_loss": 0.8503716, "learning_rate": 3.9670640810563575e-06, "loss": 0.87247086, "num_input_tokens_seen": 20553485, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.203125, "step": 964, "time_per_iteration": 3.964810609817505 }, { "auxiliary_loss_clip": 0.01157798, "auxiliary_loss_mlp": 0.01061291, "balance_loss_clip": 1.03020108, "balance_loss_mlp": 1.04076898, "epoch": 0.05801893882459041, "flos": 18076699365120.0, "grad_norm": 2.6028477809575405, "language_loss": 0.77876091, "learning_rate": 3.96699576067241e-06, "loss": 0.80095172, "num_input_tokens_seen": 20572155, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.171875, "step": 965, "time_per_iteration": 3.8165664672851562 }, { "auxiliary_loss_clip": 0.0115381, "auxiliary_loss_mlp": 0.01051914, "balance_loss_clip": 1.02453136, "balance_loss_mlp": 1.03798401, "epoch": 0.05807906207725838, "flos": 17747922291840.0, "grad_norm": 2.042103237634218, "language_loss": 0.81013924, "learning_rate": 3.966927370091442e-06, "loss": 0.83219647, "num_input_tokens_seen": 20590395, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.15625, "step": 966, "time_per_iteration": 2.4498939514160156 }, { "auxiliary_loss_clip": 0.01156764, "auxiliary_loss_mlp": 0.01054726, "balance_loss_clip": 1.0254128, "balance_loss_mlp": 1.03877592, "epoch": 0.058139185329926346, "flos": 18039412166400.0, "grad_norm": 1.9158014889501103, "language_loss": 0.76398164, "learning_rate": 3.9668589093158975e-06, "loss": 0.78609657, "num_input_tokens_seen": 20608435, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.1796875, "step": 967, "time_per_iteration": 3.8259549140930176 }, { "auxiliary_loss_clip": 0.01040902, "auxiliary_loss_mlp": 0.01004826, "balance_loss_clip": 1.00084472, "balance_loss_mlp": 1.00524974, "epoch": 0.05819930858259432, "flos": 62360287758720.0, "grad_norm": 1.1378855262618766, "language_loss": 0.57294559, "learning_rate": 3.966790378348217e-06, "loss": 0.59340286, "num_input_tokens_seen": 20668575, "router_z_loss_clip": 0.03979492, "router_z_loss_mlp": 0.35546875, "step": 968, "time_per_iteration": 2.9690871238708496 }, { "auxiliary_loss_clip": 0.01163325, "auxiliary_loss_mlp": 0.01056222, "balance_loss_clip": 1.0261575, "balance_loss_mlp": 1.04513001, "epoch": 0.05825943183526229, "flos": 19134358225920.0, "grad_norm": 1.995117496071628, "language_loss": 0.82372129, "learning_rate": 3.966721777190847e-06, "loss": 0.84591675, "num_input_tokens_seen": 20687355, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.1796875, "step": 969, "time_per_iteration": 2.413395643234253 }, { "auxiliary_loss_clip": 0.01158774, "auxiliary_loss_mlp": 0.0105649, "balance_loss_clip": 1.02590108, "balance_loss_mlp": 1.03951955, "epoch": 0.058319555087930255, "flos": 29021202547200.0, "grad_norm": 2.6883084318848938, "language_loss": 0.78030252, "learning_rate": 3.966653105846237e-06, "loss": 0.80245519, "num_input_tokens_seen": 20705710, "router_z_loss_clip": 0.30664062, "router_z_loss_mlp": 1.1953125, "step": 970, "time_per_iteration": 2.459301710128784 }, { "auxiliary_loss_clip": 0.0116011, "auxiliary_loss_mlp": 0.01059005, "balance_loss_clip": 1.02681899, "balance_loss_mlp": 1.03973424, "epoch": 0.05837967834059823, "flos": 18879003475200.0, "grad_norm": 2.4646959244174207, "language_loss": 0.92010236, "learning_rate": 3.966584364316835e-06, "loss": 0.94229347, "num_input_tokens_seen": 20722405, "router_z_loss_clip": 0.32226562, "router_z_loss_mlp": 1.203125, "step": 971, "time_per_iteration": 2.41672945022583 }, { "auxiliary_loss_clip": 0.01153373, "auxiliary_loss_mlp": 0.01046757, "balance_loss_clip": 1.01958954, "balance_loss_mlp": 1.03688502, "epoch": 0.05843980159326619, "flos": 25701870078720.0, "grad_norm": 1.82764836766903, "language_loss": 0.85977405, "learning_rate": 3.966515552605096e-06, "loss": 0.88177538, "num_input_tokens_seen": 20741480, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.1640625, "step": 972, "time_per_iteration": 2.4400033950805664 }, { "auxiliary_loss_clip": 0.01156008, "auxiliary_loss_mlp": 0.01055734, "balance_loss_clip": 1.02808905, "balance_loss_mlp": 1.03966117, "epoch": 0.058499924845934165, "flos": 25551080449920.0, "grad_norm": 2.605468634185153, "language_loss": 0.87558317, "learning_rate": 3.966446670713476e-06, "loss": 0.89770055, "num_input_tokens_seen": 20759685, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.1640625, "step": 973, "time_per_iteration": 2.4687106609344482 }, { "auxiliary_loss_clip": 0.01156814, "auxiliary_loss_mlp": 0.01056868, "balance_loss_clip": 1.02561092, "balance_loss_mlp": 1.03824723, "epoch": 0.05856004809860214, "flos": 16435222882560.0, "grad_norm": 2.345147396479656, "language_loss": 0.74578172, "learning_rate": 3.9663777186444325e-06, "loss": 0.76791859, "num_input_tokens_seen": 20778180, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.1875, "step": 974, "time_per_iteration": 2.4109954833984375 }, { "auxiliary_loss_clip": 0.01153303, "auxiliary_loss_mlp": 0.01056384, "balance_loss_clip": 1.02710593, "balance_loss_mlp": 1.03830385, "epoch": 0.0586201713512701, "flos": 39457230554880.0, "grad_norm": 2.0401418582323214, "language_loss": 0.76616645, "learning_rate": 3.966308696400426e-06, "loss": 0.78826332, "num_input_tokens_seen": 20802705, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.1484375, "step": 975, "time_per_iteration": 2.5860371589660645 }, { "auxiliary_loss_clip": 0.01156982, "auxiliary_loss_mlp": 0.01055984, "balance_loss_clip": 1.02589536, "balance_loss_mlp": 1.03819263, "epoch": 0.058680294603938074, "flos": 23364120885120.0, "grad_norm": 2.3569782231287117, "language_loss": 0.76396739, "learning_rate": 3.96623960398392e-06, "loss": 0.78609711, "num_input_tokens_seen": 20822540, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.1875, "step": 976, "time_per_iteration": 2.426626443862915 }, { "auxiliary_loss_clip": 0.01154381, "auxiliary_loss_mlp": 0.01055215, "balance_loss_clip": 1.02517378, "balance_loss_mlp": 1.0368669, "epoch": 0.05874041785660604, "flos": 32230698278400.0, "grad_norm": 1.9095241002161987, "language_loss": 0.8741101, "learning_rate": 3.9661704413973805e-06, "loss": 0.89620602, "num_input_tokens_seen": 20844175, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.171875, "step": 977, "time_per_iteration": 2.501909017562866 }, { "auxiliary_loss_clip": 0.01155156, "auxiliary_loss_mlp": 0.01054398, "balance_loss_clip": 1.02616882, "balance_loss_mlp": 1.03929973, "epoch": 0.05880054110927401, "flos": 22308940730880.0, "grad_norm": 1.84656154429845, "language_loss": 0.79312801, "learning_rate": 3.966101208643276e-06, "loss": 0.81522357, "num_input_tokens_seen": 20864730, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.15625, "step": 978, "time_per_iteration": 2.425081729888916 }, { "auxiliary_loss_clip": 0.01160624, "auxiliary_loss_mlp": 0.01057484, "balance_loss_clip": 1.02701402, "balance_loss_mlp": 1.03972781, "epoch": 0.05886066436194198, "flos": 27379237305600.0, "grad_norm": 2.6661522830253337, "language_loss": 0.80714297, "learning_rate": 3.966031905724076e-06, "loss": 0.82932401, "num_input_tokens_seen": 20885200, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.203125, "step": 979, "time_per_iteration": 2.495361328125 }, { "auxiliary_loss_clip": 0.01042219, "auxiliary_loss_mlp": 0.01005382, "balance_loss_clip": 1.00101852, "balance_loss_mlp": 1.00593567, "epoch": 0.05892078761460995, "flos": 59581725338880.0, "grad_norm": 0.9253505532266535, "language_loss": 0.59051669, "learning_rate": 3.965962532642255e-06, "loss": 0.61099267, "num_input_tokens_seen": 20940325, "router_z_loss_clip": 0.04370117, "router_z_loss_mlp": 0.36328125, "step": 980, "time_per_iteration": 2.9525704383850098 }, { "auxiliary_loss_clip": 0.01152762, "auxiliary_loss_mlp": 0.01056813, "balance_loss_clip": 1.02776158, "balance_loss_mlp": 1.0376246, "epoch": 0.05898091086727792, "flos": 15413175475200.0, "grad_norm": 2.060047658786371, "language_loss": 0.86316341, "learning_rate": 3.9658930894002885e-06, "loss": 0.88525915, "num_input_tokens_seen": 20958220, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.1484375, "step": 981, "time_per_iteration": 2.410400152206421 }, { "auxiliary_loss_clip": 0.01156705, "auxiliary_loss_mlp": 0.01054771, "balance_loss_clip": 1.02780557, "balance_loss_mlp": 1.04009056, "epoch": 0.059041034119945886, "flos": 23654319039360.0, "grad_norm": 2.1100833836001347, "language_loss": 0.79749936, "learning_rate": 3.965823576000653e-06, "loss": 0.81961417, "num_input_tokens_seen": 20978920, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.171875, "step": 982, "time_per_iteration": 2.4408376216888428 }, { "auxiliary_loss_clip": 0.01163271, "auxiliary_loss_mlp": 0.01055369, "balance_loss_clip": 1.02616262, "balance_loss_mlp": 1.04094839, "epoch": 0.05910115737261386, "flos": 24752930791680.0, "grad_norm": 2.1948144071786544, "language_loss": 0.84188688, "learning_rate": 3.965753992445833e-06, "loss": 0.86407328, "num_input_tokens_seen": 20999490, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.2265625, "step": 983, "time_per_iteration": 2.453981637954712 }, { "auxiliary_loss_clip": 0.01157224, "auxiliary_loss_mlp": 0.01063252, "balance_loss_clip": 1.03204322, "balance_loss_mlp": 1.04023576, "epoch": 0.05916128062528183, "flos": 11727953291520.0, "grad_norm": 1.9029363631781626, "language_loss": 0.84873164, "learning_rate": 3.9656843387383075e-06, "loss": 0.87093639, "num_input_tokens_seen": 21017865, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.171875, "step": 984, "time_per_iteration": 2.387319564819336 }, { "auxiliary_loss_clip": 0.01152499, "auxiliary_loss_mlp": 0.01055111, "balance_loss_clip": 1.02781165, "balance_loss_mlp": 1.04044604, "epoch": 0.059221403877949795, "flos": 21902063212800.0, "grad_norm": 2.5293821744566185, "language_loss": 0.77352715, "learning_rate": 3.965614614880566e-06, "loss": 0.79560328, "num_input_tokens_seen": 21035900, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.125, "step": 985, "time_per_iteration": 2.4488773345947266 }, { "auxiliary_loss_clip": 0.01159218, "auxiliary_loss_mlp": 0.01057503, "balance_loss_clip": 1.02842832, "balance_loss_mlp": 1.04079485, "epoch": 0.05928152713061777, "flos": 20513742065280.0, "grad_norm": 2.9893172371468024, "language_loss": 0.90492582, "learning_rate": 3.965544820875094e-06, "loss": 0.92709303, "num_input_tokens_seen": 21053235, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.1875, "step": 986, "time_per_iteration": 2.443754196166992 }, { "auxiliary_loss_clip": 0.0115854, "auxiliary_loss_mlp": 0.0106082, "balance_loss_clip": 1.02829957, "balance_loss_mlp": 1.03743863, "epoch": 0.05934165038328574, "flos": 24494084904960.0, "grad_norm": 1.9787948727060412, "language_loss": 0.75887883, "learning_rate": 3.965474956724383e-06, "loss": 0.78107238, "num_input_tokens_seen": 21073090, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.2109375, "step": 987, "time_per_iteration": 2.464308977127075 }, { "auxiliary_loss_clip": 0.01158062, "auxiliary_loss_mlp": 0.0105224, "balance_loss_clip": 1.02315331, "balance_loss_mlp": 1.0372479, "epoch": 0.059401773635953704, "flos": 38726498465280.0, "grad_norm": 2.1718172609657764, "language_loss": 0.71649158, "learning_rate": 3.965405022430928e-06, "loss": 0.73859465, "num_input_tokens_seen": 21094895, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.2109375, "step": 988, "time_per_iteration": 2.544602870941162 }, { "auxiliary_loss_clip": 0.01040901, "auxiliary_loss_mlp": 0.01004285, "balance_loss_clip": 0.9999463, "balance_loss_mlp": 1.00449657, "epoch": 0.059461896888621676, "flos": 58020618539520.0, "grad_norm": 0.9235199678427388, "language_loss": 0.71133971, "learning_rate": 3.965335017997222e-06, "loss": 0.73179162, "num_input_tokens_seen": 21147555, "router_z_loss_clip": 0.04345703, "router_z_loss_mlp": 0.36328125, "step": 989, "time_per_iteration": 2.8911666870117188 }, { "auxiliary_loss_clip": 0.01161321, "auxiliary_loss_mlp": 0.01062306, "balance_loss_clip": 1.02914214, "balance_loss_mlp": 1.03849137, "epoch": 0.05952202014128964, "flos": 22126659189120.0, "grad_norm": 1.9788870334050774, "language_loss": 0.77683198, "learning_rate": 3.965264943425766e-06, "loss": 0.79906827, "num_input_tokens_seen": 21167845, "router_z_loss_clip": 0.33203125, "router_z_loss_mlp": 1.2265625, "step": 990, "time_per_iteration": 2.424315929412842 }, { "auxiliary_loss_clip": 0.01153705, "auxiliary_loss_mlp": 0.01049102, "balance_loss_clip": 1.01887035, "balance_loss_mlp": 1.0374701, "epoch": 0.059582143393957614, "flos": 20444823308160.0, "grad_norm": 2.493270788674315, "language_loss": 0.85957623, "learning_rate": 3.965194798719059e-06, "loss": 0.88160431, "num_input_tokens_seen": 21185085, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.1640625, "step": 991, "time_per_iteration": 2.4135589599609375 }, { "auxiliary_loss_clip": 0.01158141, "auxiliary_loss_mlp": 0.01059113, "balance_loss_clip": 1.02890551, "balance_loss_mlp": 1.03746819, "epoch": 0.059642266646625586, "flos": 20593832457600.0, "grad_norm": 2.002948820668704, "language_loss": 0.76866162, "learning_rate": 3.965124583879604e-06, "loss": 0.79083419, "num_input_tokens_seen": 21204230, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.2109375, "step": 992, "time_per_iteration": 2.4235281944274902 }, { "auxiliary_loss_clip": 0.01162354, "auxiliary_loss_mlp": 0.01059023, "balance_loss_clip": 1.03100932, "balance_loss_mlp": 1.04212487, "epoch": 0.05970238989929355, "flos": 19351692639360.0, "grad_norm": 2.425489978258854, "language_loss": 0.74587756, "learning_rate": 3.965054298909908e-06, "loss": 0.76809132, "num_input_tokens_seen": 21222655, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.203125, "step": 993, "time_per_iteration": 2.4192397594451904 }, { "auxiliary_loss_clip": 0.01157142, "auxiliary_loss_mlp": 0.01057147, "balance_loss_clip": 1.02805996, "balance_loss_mlp": 1.04095399, "epoch": 0.05976251315196152, "flos": 30262713960960.0, "grad_norm": 3.039263163999806, "language_loss": 0.79152131, "learning_rate": 3.964983943812479e-06, "loss": 0.8136642, "num_input_tokens_seen": 21242310, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.15625, "step": 994, "time_per_iteration": 2.4724812507629395 }, { "auxiliary_loss_clip": 0.01154745, "auxiliary_loss_mlp": 0.0106191, "balance_loss_clip": 1.032179, "balance_loss_mlp": 1.03938246, "epoch": 0.05982263640462949, "flos": 23184038759040.0, "grad_norm": 2.871359616757743, "language_loss": 0.8020556, "learning_rate": 3.964913518589827e-06, "loss": 0.82422209, "num_input_tokens_seen": 21261410, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.15625, "step": 995, "time_per_iteration": 2.4379522800445557 }, { "auxiliary_loss_clip": 0.01155303, "auxiliary_loss_mlp": 0.01063218, "balance_loss_clip": 1.03384519, "balance_loss_mlp": 1.03777981, "epoch": 0.05988275965729746, "flos": 27849761965440.0, "grad_norm": 2.2098995933085894, "language_loss": 0.8701334, "learning_rate": 3.964843023244466e-06, "loss": 0.89231861, "num_input_tokens_seen": 21280080, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.171875, "step": 996, "time_per_iteration": 2.4437999725341797 }, { "auxiliary_loss_clip": 0.01159118, "auxiliary_loss_mlp": 0.01064769, "balance_loss_clip": 1.03220057, "balance_loss_mlp": 1.04127562, "epoch": 0.05994288290996543, "flos": 24678880064640.0, "grad_norm": 3.8608888713812597, "language_loss": 0.88007629, "learning_rate": 3.964772457778912e-06, "loss": 0.90231526, "num_input_tokens_seen": 21296765, "router_z_loss_clip": 0.32421875, "router_z_loss_mlp": 1.1796875, "step": 997, "time_per_iteration": 2.443021297454834 }, { "auxiliary_loss_clip": 0.01038178, "auxiliary_loss_mlp": 0.01003311, "balance_loss_clip": 0.99923432, "balance_loss_mlp": 1.003317, "epoch": 0.0600030061626334, "flos": 69925965782400.0, "grad_norm": 1.0099586250694919, "language_loss": 0.75391841, "learning_rate": 3.964701822195683e-06, "loss": 0.7743333, "num_input_tokens_seen": 21363345, "router_z_loss_clip": 0.04077148, "router_z_loss_mlp": 0.34765625, "step": 998, "time_per_iteration": 3.1126739978790283 }, { "auxiliary_loss_clip": 0.01157325, "auxiliary_loss_mlp": 0.01061226, "balance_loss_clip": 1.03156662, "balance_loss_mlp": 1.0397613, "epoch": 0.06006312941530137, "flos": 26538982680960.0, "grad_norm": 2.049004464099992, "language_loss": 0.75884998, "learning_rate": 3.9646311164973e-06, "loss": 0.78103548, "num_input_tokens_seen": 21385290, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.171875, "step": 999, "time_per_iteration": 2.4973855018615723 }, { "auxiliary_loss_clip": 0.01157772, "auxiliary_loss_mlp": 0.01056214, "balance_loss_clip": 1.02531469, "balance_loss_mlp": 1.03844833, "epoch": 0.060123252667969335, "flos": 27342787979520.0, "grad_norm": 1.7610372518845114, "language_loss": 0.82862902, "learning_rate": 3.9645603406862846e-06, "loss": 0.85076886, "num_input_tokens_seen": 21407625, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.1953125, "step": 1000, "time_per_iteration": 2.4534127712249756 }, { "auxiliary_loss_clip": 0.01156995, "auxiliary_loss_mlp": 0.01057862, "balance_loss_clip": 1.02845311, "balance_loss_mlp": 1.03943276, "epoch": 0.06018337592063731, "flos": 27015477183360.0, "grad_norm": 4.36013563541894, "language_loss": 0.85889578, "learning_rate": 3.964489494765166e-06, "loss": 0.88104439, "num_input_tokens_seen": 21426835, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.171875, "step": 1001, "time_per_iteration": 3.905280113220215 }, { "auxiliary_loss_clip": 0.01154827, "auxiliary_loss_mlp": 0.01052253, "balance_loss_clip": 1.02483511, "balance_loss_mlp": 1.04005575, "epoch": 0.06024349917330528, "flos": 25591788961920.0, "grad_norm": 2.18419448899442, "language_loss": 0.74045211, "learning_rate": 3.96441857873647e-06, "loss": 0.76252288, "num_input_tokens_seen": 21444920, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.1484375, "step": 1002, "time_per_iteration": 2.4591424465179443 }, { "auxiliary_loss_clip": 0.01155412, "auxiliary_loss_mlp": 0.01053095, "balance_loss_clip": 1.02345991, "balance_loss_mlp": 1.03853083, "epoch": 0.060303622425973244, "flos": 26132279719680.0, "grad_norm": 2.3045699377531497, "language_loss": 0.75484115, "learning_rate": 3.964347592602728e-06, "loss": 0.77692622, "num_input_tokens_seen": 21463555, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.171875, "step": 1003, "time_per_iteration": 2.4360148906707764 }, { "auxiliary_loss_clip": 0.01162091, "auxiliary_loss_mlp": 0.01052213, "balance_loss_clip": 1.0206461, "balance_loss_mlp": 1.04050243, "epoch": 0.060363745678641216, "flos": 20376114019200.0, "grad_norm": 2.4424858968040235, "language_loss": 0.69722307, "learning_rate": 3.964276536366473e-06, "loss": 0.71936619, "num_input_tokens_seen": 21481990, "router_z_loss_clip": 0.31640625, "router_z_loss_mlp": 1.21875, "step": 1004, "time_per_iteration": 5.30731987953186 }, { "auxiliary_loss_clip": 0.01162351, "auxiliary_loss_mlp": 0.01062383, "balance_loss_clip": 1.03300953, "balance_loss_mlp": 1.04186904, "epoch": 0.06042386893130918, "flos": 17748201582720.0, "grad_norm": 2.2180426128467743, "language_loss": 0.83568144, "learning_rate": 3.964205410030241e-06, "loss": 0.85792875, "num_input_tokens_seen": 21500385, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.203125, "step": 1005, "time_per_iteration": 2.4257004261016846 }, { "auxiliary_loss_clip": 0.01037841, "auxiliary_loss_mlp": 0.01008964, "balance_loss_clip": 1.00495863, "balance_loss_mlp": 1.00299001, "epoch": 0.06048399218397715, "flos": 68535061194240.0, "grad_norm": 0.9146307501256149, "language_loss": 0.59042352, "learning_rate": 3.964134213596571e-06, "loss": 0.61089152, "num_input_tokens_seen": 21561040, "router_z_loss_clip": 0.04003906, "router_z_loss_mlp": 0.34765625, "step": 1006, "time_per_iteration": 3.080986261367798 }, { "auxiliary_loss_clip": 0.0115253, "auxiliary_loss_mlp": 0.01054203, "balance_loss_clip": 1.02444792, "balance_loss_mlp": 1.03542399, "epoch": 0.060544115436645125, "flos": 23257391258880.0, "grad_norm": 4.936143666476675, "language_loss": 0.74330884, "learning_rate": 3.964062947068003e-06, "loss": 0.76537621, "num_input_tokens_seen": 21580655, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.171875, "step": 1007, "time_per_iteration": 3.882143497467041 }, { "auxiliary_loss_clip": 0.01155834, "auxiliary_loss_mlp": 0.01052496, "balance_loss_clip": 1.02348018, "balance_loss_mlp": 1.03754771, "epoch": 0.06060423868931309, "flos": 23877309093120.0, "grad_norm": 1.740738530070099, "language_loss": 0.80621183, "learning_rate": 3.9639916104470804e-06, "loss": 0.82829511, "num_input_tokens_seen": 21599650, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.1875, "step": 1008, "time_per_iteration": 2.4477415084838867 }, { "auxiliary_loss_clip": 0.01158471, "auxiliary_loss_mlp": 0.01053834, "balance_loss_clip": 1.0252471, "balance_loss_mlp": 1.04050589, "epoch": 0.06066436194198106, "flos": 18727236328320.0, "grad_norm": 1.7879989603434563, "language_loss": 0.77816951, "learning_rate": 3.9639202037363494e-06, "loss": 0.80029255, "num_input_tokens_seen": 21617550, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.1796875, "step": 1009, "time_per_iteration": 2.418851613998413 }, { "auxiliary_loss_clip": 0.01152606, "auxiliary_loss_mlp": 0.01048527, "balance_loss_clip": 1.02015519, "balance_loss_mlp": 1.03950274, "epoch": 0.06072448519464903, "flos": 24639428361600.0, "grad_norm": 1.7972262110622221, "language_loss": 0.92497772, "learning_rate": 3.9638487269383575e-06, "loss": 0.94698906, "num_input_tokens_seen": 21635865, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.1328125, "step": 1010, "time_per_iteration": 2.451969623565674 }, { "auxiliary_loss_clip": 0.01159785, "auxiliary_loss_mlp": 0.01057949, "balance_loss_clip": 1.0270505, "balance_loss_mlp": 1.03740323, "epoch": 0.060784608447317, "flos": 17378017770240.0, "grad_norm": 3.6712956960038796, "language_loss": 0.71411031, "learning_rate": 3.9637771800556576e-06, "loss": 0.73628759, "num_input_tokens_seen": 21653945, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.2265625, "step": 1011, "time_per_iteration": 2.4284327030181885 }, { "auxiliary_loss_clip": 0.01158641, "auxiliary_loss_mlp": 0.01067488, "balance_loss_clip": 1.03451467, "balance_loss_mlp": 1.03772664, "epoch": 0.06084473169998497, "flos": 23691187301760.0, "grad_norm": 2.2199777315452334, "language_loss": 0.8743695, "learning_rate": 3.963705563090801e-06, "loss": 0.89663088, "num_input_tokens_seen": 21671230, "router_z_loss_clip": 0.33007812, "router_z_loss_mlp": 1.2109375, "step": 1012, "time_per_iteration": 2.440599203109741 }, { "auxiliary_loss_clip": 0.01152822, "auxiliary_loss_mlp": 0.01049775, "balance_loss_clip": 1.02074802, "balance_loss_mlp": 1.03609371, "epoch": 0.06090485495265294, "flos": 23545320174720.0, "grad_norm": 2.3187281216043703, "language_loss": 0.76561666, "learning_rate": 3.963633876046344e-06, "loss": 0.78764266, "num_input_tokens_seen": 21691155, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.171875, "step": 1013, "time_per_iteration": 2.475726366043091 }, { "auxiliary_loss_clip": 0.01157383, "auxiliary_loss_mlp": 0.01060122, "balance_loss_clip": 1.02834129, "balance_loss_mlp": 1.03816497, "epoch": 0.06096497820532091, "flos": 20338268238720.0, "grad_norm": 2.384126716604103, "language_loss": 0.85482019, "learning_rate": 3.963562118924844e-06, "loss": 0.87699521, "num_input_tokens_seen": 21707405, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.1953125, "step": 1014, "time_per_iteration": 2.417125701904297 }, { "auxiliary_loss_clip": 0.01160361, "auxiliary_loss_mlp": 0.01056803, "balance_loss_clip": 1.02313781, "balance_loss_mlp": 1.03910649, "epoch": 0.061025101457988874, "flos": 26937935320320.0, "grad_norm": 2.246085630291627, "language_loss": 0.73297465, "learning_rate": 3.963490291728864e-06, "loss": 0.75514627, "num_input_tokens_seen": 21728090, "router_z_loss_clip": 0.3359375, "router_z_loss_mlp": 1.2109375, "step": 1015, "time_per_iteration": 2.483424186706543 }, { "auxiliary_loss_clip": 0.0115166, "auxiliary_loss_mlp": 0.0105429, "balance_loss_clip": 1.02441573, "balance_loss_mlp": 1.03619528, "epoch": 0.061085224710656846, "flos": 25373861055360.0, "grad_norm": 1.6809549189182948, "language_loss": 0.7901845, "learning_rate": 3.963418394460966e-06, "loss": 0.812244, "num_input_tokens_seen": 21747950, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.15625, "step": 1016, "time_per_iteration": 2.439598798751831 }, { "auxiliary_loss_clip": 0.01157869, "auxiliary_loss_mlp": 0.01047841, "balance_loss_clip": 1.01918364, "balance_loss_mlp": 1.03926146, "epoch": 0.06114534796332482, "flos": 24823664939520.0, "grad_norm": 1.7343598991326854, "language_loss": 0.75973874, "learning_rate": 3.9633464271237166e-06, "loss": 0.78179586, "num_input_tokens_seen": 21767900, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.1875, "step": 1017, "time_per_iteration": 2.4395880699157715 }, { "auxiliary_loss_clip": 0.0116119, "auxiliary_loss_mlp": 0.01061209, "balance_loss_clip": 1.03021479, "balance_loss_mlp": 1.04041672, "epoch": 0.061205471215992784, "flos": 20630386517760.0, "grad_norm": 2.478866834537974, "language_loss": 0.85801131, "learning_rate": 3.963274389719682e-06, "loss": 0.88023531, "num_input_tokens_seen": 21787375, "router_z_loss_clip": 0.31054688, "router_z_loss_mlp": 1.203125, "step": 1018, "time_per_iteration": 2.4153647422790527 }, { "auxiliary_loss_clip": 0.01155674, "auxiliary_loss_mlp": 0.01058413, "balance_loss_clip": 1.02789545, "balance_loss_mlp": 1.03959513, "epoch": 0.061265594468660756, "flos": 16507423307520.0, "grad_norm": 7.811235902239468, "language_loss": 0.76732063, "learning_rate": 3.963202282251436e-06, "loss": 0.78946149, "num_input_tokens_seen": 21806275, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.15625, "step": 1019, "time_per_iteration": 2.393094062805176 }, { "auxiliary_loss_clip": 0.01156278, "auxiliary_loss_mlp": 0.01053328, "balance_loss_clip": 1.02164245, "balance_loss_mlp": 1.03888106, "epoch": 0.06132571772132872, "flos": 26245118833920.0, "grad_norm": 2.2044878885481083, "language_loss": 0.84023499, "learning_rate": 3.96313010472155e-06, "loss": 0.86233103, "num_input_tokens_seen": 21826430, "router_z_loss_clip": 0.31640625, "router_z_loss_mlp": 1.171875, "step": 1020, "time_per_iteration": 2.4407336711883545 }, { "auxiliary_loss_clip": 0.01159264, "auxiliary_loss_mlp": 0.01055516, "balance_loss_clip": 1.02580905, "balance_loss_mlp": 1.04091072, "epoch": 0.06138584097399669, "flos": 37413275385600.0, "grad_norm": 2.1251693510396987, "language_loss": 0.79392493, "learning_rate": 3.963057857132601e-06, "loss": 0.8160727, "num_input_tokens_seen": 21847800, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.1875, "step": 1021, "time_per_iteration": 2.55255126953125 }, { "auxiliary_loss_clip": 0.01154293, "auxiliary_loss_mlp": 0.01057299, "balance_loss_clip": 1.03082263, "balance_loss_mlp": 1.03812289, "epoch": 0.061445964226664665, "flos": 17419703800320.0, "grad_norm": 1.8709309273080235, "language_loss": 0.87560797, "learning_rate": 3.962985539487165e-06, "loss": 0.89772391, "num_input_tokens_seen": 21863385, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.15625, "step": 1022, "time_per_iteration": 2.377963066101074 }, { "auxiliary_loss_clip": 0.01157659, "auxiliary_loss_mlp": 0.01053407, "balance_loss_clip": 1.02510691, "balance_loss_mlp": 1.04006875, "epoch": 0.06150608747933263, "flos": 22598964328320.0, "grad_norm": 3.5561262995454856, "language_loss": 0.82924676, "learning_rate": 3.962913151787826e-06, "loss": 0.85135746, "num_input_tokens_seen": 21881880, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.171875, "step": 1023, "time_per_iteration": 2.4481008052825928 }, { "auxiliary_loss_clip": 0.01039811, "auxiliary_loss_mlp": 0.01006344, "balance_loss_clip": 1.00257671, "balance_loss_mlp": 1.0057925, "epoch": 0.0615662107320006, "flos": 56738712816000.0, "grad_norm": 0.892435407282451, "language_loss": 0.65076607, "learning_rate": 3.962840694037165e-06, "loss": 0.67122757, "num_input_tokens_seen": 21940550, "router_z_loss_clip": 0.03759766, "router_z_loss_mlp": 0.33984375, "step": 1024, "time_per_iteration": 3.0637269020080566 }, { "auxiliary_loss_clip": 0.011566, "auxiliary_loss_mlp": 0.01058846, "balance_loss_clip": 1.027637, "balance_loss_mlp": 1.03874087, "epoch": 0.06162633398466857, "flos": 22563701988480.0, "grad_norm": 2.121991078512927, "language_loss": 0.88018882, "learning_rate": 3.962768166237768e-06, "loss": 0.90234327, "num_input_tokens_seen": 21958390, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.1796875, "step": 1025, "time_per_iteration": 2.4291317462921143 }, { "auxiliary_loss_clip": 0.01158191, "auxiliary_loss_mlp": 0.0104929, "balance_loss_clip": 1.02213371, "balance_loss_mlp": 1.04083061, "epoch": 0.06168645723733654, "flos": 25591928607360.0, "grad_norm": 1.944980768345882, "language_loss": 0.84539229, "learning_rate": 3.9626955683922264e-06, "loss": 0.86746705, "num_input_tokens_seen": 21978625, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.171875, "step": 1026, "time_per_iteration": 2.4521775245666504 }, { "auxiliary_loss_clip": 0.01161698, "auxiliary_loss_mlp": 0.01052541, "balance_loss_clip": 1.02338207, "balance_loss_mlp": 1.04109931, "epoch": 0.06174658049000451, "flos": 15996993096960.0, "grad_norm": 2.258709835405207, "language_loss": 0.82325631, "learning_rate": 3.962622900503127e-06, "loss": 0.84539866, "num_input_tokens_seen": 21996035, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.203125, "step": 1027, "time_per_iteration": 2.4000582695007324 }, { "auxiliary_loss_clip": 0.01152861, "auxiliary_loss_mlp": 0.01051801, "balance_loss_clip": 1.02383435, "balance_loss_mlp": 1.0380646, "epoch": 0.06180670374267248, "flos": 11285324674560.0, "grad_norm": 2.537469324710815, "language_loss": 0.84134269, "learning_rate": 3.962550162573065e-06, "loss": 0.86338931, "num_input_tokens_seen": 22011625, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.1484375, "step": 1028, "time_per_iteration": 2.358281373977661 }, { "auxiliary_loss_clip": 0.01037697, "auxiliary_loss_mlp": 0.01007238, "balance_loss_clip": 1.00339997, "balance_loss_mlp": 1.00318575, "epoch": 0.06186682699534045, "flos": 65127224695680.0, "grad_norm": 0.9629596283558131, "language_loss": 0.60529995, "learning_rate": 3.962477354604636e-06, "loss": 0.62574935, "num_input_tokens_seen": 22066035, "router_z_loss_clip": 0.03833008, "router_z_loss_mlp": 0.34375, "step": 1029, "time_per_iteration": 2.864759683609009 }, { "auxiliary_loss_clip": 0.01150987, "auxiliary_loss_mlp": 0.01053206, "balance_loss_clip": 1.02463198, "balance_loss_mlp": 1.03651297, "epoch": 0.061926950248008414, "flos": 21104681604480.0, "grad_norm": 4.849598134920486, "language_loss": 0.82339936, "learning_rate": 3.962404476600438e-06, "loss": 0.84544134, "num_input_tokens_seen": 22085015, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.140625, "step": 1030, "time_per_iteration": 2.394225597381592 }, { "auxiliary_loss_clip": 0.01161727, "auxiliary_loss_mlp": 0.01062051, "balance_loss_clip": 1.03181958, "balance_loss_mlp": 1.0407145, "epoch": 0.061987073500676386, "flos": 17747503355520.0, "grad_norm": 2.727803687845987, "language_loss": 0.7986154, "learning_rate": 3.962331528563072e-06, "loss": 0.82085317, "num_input_tokens_seen": 22102775, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.2109375, "step": 1031, "time_per_iteration": 2.4070053100585938 }, { "auxiliary_loss_clip": 0.01156936, "auxiliary_loss_mlp": 0.01061886, "balance_loss_clip": 1.03142774, "balance_loss_mlp": 1.03971457, "epoch": 0.06204719675334436, "flos": 21835134403200.0, "grad_norm": 1.6632160897947257, "language_loss": 0.77500129, "learning_rate": 3.962258510495142e-06, "loss": 0.79718953, "num_input_tokens_seen": 22121680, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.171875, "step": 1032, "time_per_iteration": 2.404686450958252 }, { "auxiliary_loss_clip": 0.01158514, "auxiliary_loss_mlp": 0.01062509, "balance_loss_clip": 1.0319314, "balance_loss_mlp": 1.03803051, "epoch": 0.06210732000601232, "flos": 19352705068800.0, "grad_norm": 2.252385851128305, "language_loss": 0.88763595, "learning_rate": 3.962185422399254e-06, "loss": 0.90984619, "num_input_tokens_seen": 22138155, "router_z_loss_clip": 0.30664062, "router_z_loss_mlp": 1.203125, "step": 1033, "time_per_iteration": 2.3899827003479004 }, { "auxiliary_loss_clip": 0.01156533, "auxiliary_loss_mlp": 0.01060124, "balance_loss_clip": 1.03165722, "balance_loss_mlp": 1.03904891, "epoch": 0.062167443258680295, "flos": 24748357403520.0, "grad_norm": 2.1112015481619135, "language_loss": 0.85067034, "learning_rate": 3.962112264278014e-06, "loss": 0.87283695, "num_input_tokens_seen": 22157420, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.171875, "step": 1034, "time_per_iteration": 2.417952299118042 }, { "auxiliary_loss_clip": 0.01152025, "auxiliary_loss_mlp": 0.0105143, "balance_loss_clip": 1.02109075, "balance_loss_mlp": 1.04026592, "epoch": 0.06222756651134827, "flos": 34457074634880.0, "grad_norm": 2.0175474623906156, "language_loss": 0.80539238, "learning_rate": 3.962039036134035e-06, "loss": 0.82742691, "num_input_tokens_seen": 22178620, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.1171875, "step": 1035, "time_per_iteration": 2.5118720531463623 }, { "auxiliary_loss_clip": 0.01158328, "auxiliary_loss_mlp": 0.01052676, "balance_loss_clip": 1.02084732, "balance_loss_mlp": 1.0402739, "epoch": 0.06228768976401623, "flos": 25665281107200.0, "grad_norm": 2.7436331301329893, "language_loss": 0.78723359, "learning_rate": 3.961965737969931e-06, "loss": 0.8093437, "num_input_tokens_seen": 22197125, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.1796875, "step": 1036, "time_per_iteration": 2.429211139678955 }, { "auxiliary_loss_clip": 0.01154011, "auxiliary_loss_mlp": 0.01055222, "balance_loss_clip": 1.02690959, "balance_loss_mlp": 1.03979087, "epoch": 0.062347813016684205, "flos": 25294608535680.0, "grad_norm": 1.8662455074359048, "language_loss": 0.86611468, "learning_rate": 3.961892369788315e-06, "loss": 0.88820702, "num_input_tokens_seen": 22217575, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.140625, "step": 1037, "time_per_iteration": 2.4833195209503174 }, { "auxiliary_loss_clip": 0.01152175, "auxiliary_loss_mlp": 0.01053944, "balance_loss_clip": 1.02109003, "balance_loss_mlp": 1.03708446, "epoch": 0.06240793626935217, "flos": 26905815002880.0, "grad_norm": 2.290920851884523, "language_loss": 0.80359685, "learning_rate": 3.961818931591808e-06, "loss": 0.82565802, "num_input_tokens_seen": 22236840, "router_z_loss_clip": 0.328125, "router_z_loss_mlp": 1.15625, "step": 1038, "time_per_iteration": 2.4463109970092773 }, { "auxiliary_loss_clip": 0.01153569, "auxiliary_loss_mlp": 0.01057785, "balance_loss_clip": 1.02872229, "balance_loss_mlp": 1.0402391, "epoch": 0.06246805952202014, "flos": 21614727790080.0, "grad_norm": 3.0631812454019824, "language_loss": 0.85687834, "learning_rate": 3.961745423383028e-06, "loss": 0.87899184, "num_input_tokens_seen": 22256465, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.1328125, "step": 1039, "time_per_iteration": 2.4449520111083984 }, { "auxiliary_loss_clip": 0.01155556, "auxiliary_loss_mlp": 0.01059921, "balance_loss_clip": 1.0290575, "balance_loss_mlp": 1.03914809, "epoch": 0.0625281827746881, "flos": 19311053950080.0, "grad_norm": 1.8935037623048254, "language_loss": 0.80690914, "learning_rate": 3.961671845164602e-06, "loss": 0.82906389, "num_input_tokens_seen": 22274025, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.1640625, "step": 1040, "time_per_iteration": 3.8361432552337646 }, { "auxiliary_loss_clip": 0.01157663, "auxiliary_loss_mlp": 0.01059855, "balance_loss_clip": 1.03005266, "balance_loss_mlp": 1.04205906, "epoch": 0.06258830602735609, "flos": 27744533527680.0, "grad_norm": 8.969843761282052, "language_loss": 0.69530857, "learning_rate": 3.961598196939153e-06, "loss": 0.71748376, "num_input_tokens_seen": 22292245, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.15625, "step": 1041, "time_per_iteration": 2.4421133995056152 }, { "auxiliary_loss_clip": 0.01153717, "auxiliary_loss_mlp": 0.01054046, "balance_loss_clip": 1.02290881, "balance_loss_mlp": 1.03592014, "epoch": 0.06264842928002405, "flos": 23221465603200.0, "grad_norm": 2.1600309028167017, "language_loss": 0.81277382, "learning_rate": 3.961524478709311e-06, "loss": 0.8348515, "num_input_tokens_seen": 22311455, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.1796875, "step": 1042, "time_per_iteration": 2.4429678916931152 }, { "auxiliary_loss_clip": 0.01155927, "auxiliary_loss_mlp": 0.01047444, "balance_loss_clip": 1.01816618, "balance_loss_mlp": 1.03883386, "epoch": 0.06270855253269202, "flos": 38397965771520.0, "grad_norm": 1.6556398191388253, "language_loss": 0.76052594, "learning_rate": 3.961450690477705e-06, "loss": 0.78255963, "num_input_tokens_seen": 22333750, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.171875, "step": 1043, "time_per_iteration": 3.930548906326294 }, { "auxiliary_loss_clip": 0.0115033, "auxiliary_loss_mlp": 0.01048839, "balance_loss_clip": 1.02066958, "balance_loss_mlp": 1.03881478, "epoch": 0.06276867578535998, "flos": 22452503708160.0, "grad_norm": 2.1727494522463116, "language_loss": 0.92467427, "learning_rate": 3.961376832246969e-06, "loss": 0.946666, "num_input_tokens_seen": 22351940, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.1171875, "step": 1044, "time_per_iteration": 3.851454973220825 }, { "auxiliary_loss_clip": 0.01153806, "auxiliary_loss_mlp": 0.01051929, "balance_loss_clip": 1.02408147, "balance_loss_mlp": 1.03988838, "epoch": 0.06282879903802796, "flos": 22929312412800.0, "grad_norm": 2.6175374391353987, "language_loss": 0.86091137, "learning_rate": 3.96130290401974e-06, "loss": 0.88296872, "num_input_tokens_seen": 22372085, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.140625, "step": 1045, "time_per_iteration": 2.4562478065490723 }, { "auxiliary_loss_clip": 0.01147997, "auxiliary_loss_mlp": 0.01058226, "balance_loss_clip": 1.03123653, "balance_loss_mlp": 1.0371809, "epoch": 0.06288892229069593, "flos": 34817937114240.0, "grad_norm": 2.050724739218883, "language_loss": 0.78363693, "learning_rate": 3.961228905798655e-06, "loss": 0.80569911, "num_input_tokens_seen": 22392020, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.109375, "step": 1046, "time_per_iteration": 3.8668949604034424 }, { "auxiliary_loss_clip": 0.01154558, "auxiliary_loss_mlp": 0.01061772, "balance_loss_clip": 1.03345942, "balance_loss_mlp": 1.03891706, "epoch": 0.06294904554336389, "flos": 19426127391360.0, "grad_norm": 2.827178720419603, "language_loss": 0.77426672, "learning_rate": 3.961154837586356e-06, "loss": 0.79642999, "num_input_tokens_seen": 22411180, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.15625, "step": 1047, "time_per_iteration": 2.403306007385254 }, { "auxiliary_loss_clip": 0.01158847, "auxiliary_loss_mlp": 0.0105585, "balance_loss_clip": 1.02571368, "balance_loss_mlp": 1.03932309, "epoch": 0.06300916879603187, "flos": 40660267783680.0, "grad_norm": 2.2582086957955054, "language_loss": 0.7676698, "learning_rate": 3.961080699385484e-06, "loss": 0.78981674, "num_input_tokens_seen": 22435105, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.1953125, "step": 1048, "time_per_iteration": 2.585777521133423 }, { "auxiliary_loss_clip": 0.01159709, "auxiliary_loss_mlp": 0.01053575, "balance_loss_clip": 1.02439284, "balance_loss_mlp": 1.040627, "epoch": 0.06306929204869983, "flos": 23803048897920.0, "grad_norm": 2.868288878903169, "language_loss": 0.77440327, "learning_rate": 3.961006491198688e-06, "loss": 0.79653615, "num_input_tokens_seen": 22452710, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.1953125, "step": 1049, "time_per_iteration": 2.448014259338379 }, { "auxiliary_loss_clip": 0.01154987, "auxiliary_loss_mlp": 0.0105744, "balance_loss_clip": 1.02704167, "balance_loss_mlp": 1.03871131, "epoch": 0.0631294153013678, "flos": 18914824396800.0, "grad_norm": 2.1328334411159666, "language_loss": 0.83224154, "learning_rate": 3.960932213028614e-06, "loss": 0.85436583, "num_input_tokens_seen": 22470175, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.1640625, "step": 1050, "time_per_iteration": 2.428130626678467 }, { "auxiliary_loss_clip": 0.01153997, "auxiliary_loss_mlp": 0.0105438, "balance_loss_clip": 1.02810645, "balance_loss_mlp": 1.04045296, "epoch": 0.06318953855403578, "flos": 24279019729920.0, "grad_norm": 2.0519250920100536, "language_loss": 0.76973629, "learning_rate": 3.960857864877913e-06, "loss": 0.79182005, "num_input_tokens_seen": 22490020, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.1328125, "step": 1051, "time_per_iteration": 2.432659864425659 }, { "auxiliary_loss_clip": 0.01155369, "auxiliary_loss_mlp": 0.01063899, "balance_loss_clip": 1.03567064, "balance_loss_mlp": 1.03871274, "epoch": 0.06324966180670374, "flos": 22527811244160.0, "grad_norm": 2.0842928281458883, "language_loss": 0.80101454, "learning_rate": 3.960783446749239e-06, "loss": 0.82320726, "num_input_tokens_seen": 22509685, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.171875, "step": 1052, "time_per_iteration": 2.4301910400390625 }, { "auxiliary_loss_clip": 0.01156457, "auxiliary_loss_mlp": 0.01054979, "balance_loss_clip": 1.0256772, "balance_loss_mlp": 1.03852856, "epoch": 0.06330978505937171, "flos": 15777214888320.0, "grad_norm": 2.4383696598633495, "language_loss": 0.78276086, "learning_rate": 3.960708958645247e-06, "loss": 0.8048752, "num_input_tokens_seen": 22527905, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.1796875, "step": 1053, "time_per_iteration": 2.3998453617095947 }, { "auxiliary_loss_clip": 0.01154537, "auxiliary_loss_mlp": 0.01047982, "balance_loss_clip": 1.02014709, "balance_loss_mlp": 1.03847611, "epoch": 0.06336990831203967, "flos": 21470012737920.0, "grad_norm": 1.9432053338143196, "language_loss": 0.84447843, "learning_rate": 3.960634400568597e-06, "loss": 0.86650366, "num_input_tokens_seen": 22546335, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.15625, "step": 1054, "time_per_iteration": 2.4102094173431396 }, { "auxiliary_loss_clip": 0.01153083, "auxiliary_loss_mlp": 0.01058387, "balance_loss_clip": 1.03110003, "balance_loss_mlp": 1.03914165, "epoch": 0.06343003156470765, "flos": 18477886331520.0, "grad_norm": 2.41561749478276, "language_loss": 0.85629678, "learning_rate": 3.9605597725219485e-06, "loss": 0.87841147, "num_input_tokens_seen": 22563885, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.140625, "step": 1055, "time_per_iteration": 2.3968770503997803 }, { "auxiliary_loss_clip": 0.01155734, "auxiliary_loss_mlp": 0.01060315, "balance_loss_clip": 1.02896309, "balance_loss_mlp": 1.03833985, "epoch": 0.06349015481737562, "flos": 25153733733120.0, "grad_norm": 2.6139634882193867, "language_loss": 0.8117063, "learning_rate": 3.960485074507964e-06, "loss": 0.83386678, "num_input_tokens_seen": 22583035, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.171875, "step": 1056, "time_per_iteration": 2.4474260807037354 }, { "auxiliary_loss_clip": 0.01159162, "auxiliary_loss_mlp": 0.01056713, "balance_loss_clip": 1.02288127, "balance_loss_mlp": 1.03690875, "epoch": 0.06355027807004358, "flos": 26870517751680.0, "grad_norm": 2.441038065041055, "language_loss": 0.80776274, "learning_rate": 3.960410306529311e-06, "loss": 0.82992148, "num_input_tokens_seen": 22605055, "router_z_loss_clip": 0.33789062, "router_z_loss_mlp": 1.21875, "step": 1057, "time_per_iteration": 2.497753858566284 }, { "auxiliary_loss_clip": 0.01145479, "auxiliary_loss_mlp": 0.01053419, "balance_loss_clip": 1.02721691, "balance_loss_mlp": 1.03655159, "epoch": 0.06361040132271156, "flos": 21395647808640.0, "grad_norm": 1.8377619202367705, "language_loss": 0.83484435, "learning_rate": 3.960335468588656e-06, "loss": 0.85683334, "num_input_tokens_seen": 22623760, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.09375, "step": 1058, "time_per_iteration": 2.4473915100097656 }, { "auxiliary_loss_clip": 0.01150486, "auxiliary_loss_mlp": 0.01053281, "balance_loss_clip": 1.02176166, "balance_loss_mlp": 1.03529727, "epoch": 0.06367052457537953, "flos": 25732733587200.0, "grad_norm": 2.1521473809757206, "language_loss": 0.87502033, "learning_rate": 3.960260560688672e-06, "loss": 0.89705795, "num_input_tokens_seen": 22643000, "router_z_loss_clip": 0.31640625, "router_z_loss_mlp": 1.1484375, "step": 1059, "time_per_iteration": 2.487631320953369 }, { "auxiliary_loss_clip": 0.01157359, "auxiliary_loss_mlp": 0.0105957, "balance_loss_clip": 1.03045952, "balance_loss_mlp": 1.04114223, "epoch": 0.0637306478280475, "flos": 17630684346240.0, "grad_norm": 2.4567529624321938, "language_loss": 0.91952676, "learning_rate": 3.96018558283203e-06, "loss": 0.94169605, "num_input_tokens_seen": 22660460, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.15625, "step": 1060, "time_per_iteration": 2.4338083267211914 }, { "auxiliary_loss_clip": 0.01153993, "auxiliary_loss_mlp": 0.01054981, "balance_loss_clip": 1.02560759, "balance_loss_mlp": 1.03712416, "epoch": 0.06379077108071547, "flos": 13661757521280.0, "grad_norm": 2.079161993849353, "language_loss": 0.8758902, "learning_rate": 3.960110535021406e-06, "loss": 0.89797997, "num_input_tokens_seen": 22679270, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.171875, "step": 1061, "time_per_iteration": 2.447174549102783 }, { "auxiliary_loss_clip": 0.01159231, "auxiliary_loss_mlp": 0.01055013, "balance_loss_clip": 1.02487659, "balance_loss_mlp": 1.03590798, "epoch": 0.06385089433338344, "flos": 28477499944320.0, "grad_norm": 2.4121893917422734, "language_loss": 0.7742179, "learning_rate": 3.96003541725948e-06, "loss": 0.79636031, "num_input_tokens_seen": 22699330, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.234375, "step": 1062, "time_per_iteration": 2.530794620513916 }, { "auxiliary_loss_clip": 0.01152508, "auxiliary_loss_mlp": 0.01056336, "balance_loss_clip": 1.02829826, "balance_loss_mlp": 1.03505027, "epoch": 0.0639110175860514, "flos": 24310057795200.0, "grad_norm": 3.397358567992743, "language_loss": 0.8646583, "learning_rate": 3.959960229548932e-06, "loss": 0.88674676, "num_input_tokens_seen": 22717945, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.171875, "step": 1063, "time_per_iteration": 2.493764638900757 }, { "auxiliary_loss_clip": 0.01152675, "auxiliary_loss_mlp": 0.01061486, "balance_loss_clip": 1.03192258, "balance_loss_mlp": 1.03749716, "epoch": 0.06397114083871938, "flos": 22089686192640.0, "grad_norm": 1.8895788334406478, "language_loss": 0.79841852, "learning_rate": 3.9598849718924456e-06, "loss": 0.82056022, "num_input_tokens_seen": 22736790, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.1484375, "step": 1064, "time_per_iteration": 2.552570343017578 }, { "auxiliary_loss_clip": 0.01156097, "auxiliary_loss_mlp": 0.01062561, "balance_loss_clip": 1.03197181, "balance_loss_mlp": 1.03786206, "epoch": 0.06403126409138735, "flos": 19571819961600.0, "grad_norm": 2.915547150127337, "language_loss": 0.84240711, "learning_rate": 3.9598096442927045e-06, "loss": 0.86459368, "num_input_tokens_seen": 22754745, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.1796875, "step": 1065, "time_per_iteration": 2.483699321746826 }, { "auxiliary_loss_clip": 0.01153172, "auxiliary_loss_mlp": 0.01055382, "balance_loss_clip": 1.02670002, "balance_loss_mlp": 1.03946292, "epoch": 0.06409138734405531, "flos": 40805820708480.0, "grad_norm": 2.1064109549280228, "language_loss": 0.68253148, "learning_rate": 3.959734246752399e-06, "loss": 0.70461702, "num_input_tokens_seen": 22776780, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.140625, "step": 1066, "time_per_iteration": 2.613372564315796 }, { "auxiliary_loss_clip": 0.01153307, "auxiliary_loss_mlp": 0.01068058, "balance_loss_clip": 1.03811312, "balance_loss_mlp": 1.03984094, "epoch": 0.06415151059672328, "flos": 20440773590400.0, "grad_norm": 2.221808413280424, "language_loss": 0.9024362, "learning_rate": 3.959658779274219e-06, "loss": 0.92464983, "num_input_tokens_seen": 22793915, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.1328125, "step": 1067, "time_per_iteration": 2.4078049659729004 }, { "auxiliary_loss_clip": 0.01153334, "auxiliary_loss_mlp": 0.01056416, "balance_loss_clip": 1.02794874, "balance_loss_mlp": 1.03792787, "epoch": 0.06421163384939126, "flos": 18071218281600.0, "grad_norm": 2.0953299155703515, "language_loss": 0.83557618, "learning_rate": 3.959583241860859e-06, "loss": 0.85767376, "num_input_tokens_seen": 22812670, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.15625, "step": 1068, "time_per_iteration": 2.43798565864563 }, { "auxiliary_loss_clip": 0.01149899, "auxiliary_loss_mlp": 0.01055215, "balance_loss_clip": 1.02734399, "balance_loss_mlp": 1.03755999, "epoch": 0.06427175710205922, "flos": 25118261925120.0, "grad_norm": 2.7684988106959607, "language_loss": 0.89493138, "learning_rate": 3.959507634515013e-06, "loss": 0.91698253, "num_input_tokens_seen": 22832440, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.125, "step": 1069, "time_per_iteration": 2.4779961109161377 }, { "auxiliary_loss_clip": 0.01156154, "auxiliary_loss_mlp": 0.01066524, "balance_loss_clip": 1.03642344, "balance_loss_mlp": 1.03853703, "epoch": 0.06433188035472719, "flos": 17379693515520.0, "grad_norm": 2.6462331204119565, "language_loss": 0.95468295, "learning_rate": 3.95943195723938e-06, "loss": 0.97690964, "num_input_tokens_seen": 22845495, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.171875, "step": 1070, "time_per_iteration": 2.40043306350708 }, { "auxiliary_loss_clip": 0.01154162, "auxiliary_loss_mlp": 0.01050874, "balance_loss_clip": 1.02147698, "balance_loss_mlp": 1.03742683, "epoch": 0.06439200360739517, "flos": 23545250352000.0, "grad_norm": 1.9726107770921453, "language_loss": 0.88081366, "learning_rate": 3.959356210036661e-06, "loss": 0.90286404, "num_input_tokens_seen": 22865390, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.171875, "step": 1071, "time_per_iteration": 2.4496536254882812 }, { "auxiliary_loss_clip": 0.01149584, "auxiliary_loss_mlp": 0.01051725, "balance_loss_clip": 1.02471256, "balance_loss_mlp": 1.03599989, "epoch": 0.06445212686006313, "flos": 21978732291840.0, "grad_norm": 1.9189707447936222, "language_loss": 0.76146531, "learning_rate": 3.959280392909559e-06, "loss": 0.78347838, "num_input_tokens_seen": 22885495, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.140625, "step": 1072, "time_per_iteration": 2.4486169815063477 }, { "auxiliary_loss_clip": 0.01156692, "auxiliary_loss_mlp": 0.01058945, "balance_loss_clip": 1.02768803, "balance_loss_mlp": 1.03829575, "epoch": 0.0645122501127311, "flos": 25920112187520.0, "grad_norm": 2.0618225953997027, "language_loss": 0.80716658, "learning_rate": 3.9592045058607785e-06, "loss": 0.82932299, "num_input_tokens_seen": 22904845, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.1875, "step": 1073, "time_per_iteration": 2.4605281352996826 }, { "auxiliary_loss_clip": 0.01144171, "auxiliary_loss_mlp": 0.01053222, "balance_loss_clip": 1.02463543, "balance_loss_mlp": 1.03527343, "epoch": 0.06457237336539907, "flos": 25624956620160.0, "grad_norm": 1.6866784670733426, "language_loss": 0.80415916, "learning_rate": 3.95912854889303e-06, "loss": 0.82613313, "num_input_tokens_seen": 22925940, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.09375, "step": 1074, "time_per_iteration": 2.4511263370513916 }, { "auxiliary_loss_clip": 0.01153531, "auxiliary_loss_mlp": 0.01052702, "balance_loss_clip": 1.0233289, "balance_loss_mlp": 1.03641522, "epoch": 0.06463249661806704, "flos": 19462960742400.0, "grad_norm": 2.532306893728656, "language_loss": 0.78886366, "learning_rate": 3.959052522009023e-06, "loss": 0.81092602, "num_input_tokens_seen": 22944375, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.171875, "step": 1075, "time_per_iteration": 2.4653217792510986 }, { "auxiliary_loss_clip": 0.01157863, "auxiliary_loss_mlp": 0.01054332, "balance_loss_clip": 1.02711701, "balance_loss_mlp": 1.03994346, "epoch": 0.064692619870735, "flos": 24496912725120.0, "grad_norm": 5.248740749478744, "language_loss": 0.87301528, "learning_rate": 3.95897642521147e-06, "loss": 0.89513719, "num_input_tokens_seen": 22959145, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.1796875, "step": 1076, "time_per_iteration": 2.558877944946289 }, { "auxiliary_loss_clip": 0.01149619, "auxiliary_loss_mlp": 0.01046069, "balance_loss_clip": 1.01819813, "balance_loss_mlp": 1.03587496, "epoch": 0.06475274312340297, "flos": 17017748784000.0, "grad_norm": 2.1191892004808404, "language_loss": 0.80661476, "learning_rate": 3.958900258503089e-06, "loss": 0.82857162, "num_input_tokens_seen": 22978100, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.140625, "step": 1077, "time_per_iteration": 2.5130233764648438 }, { "auxiliary_loss_clip": 0.01156957, "auxiliary_loss_mlp": 0.01062066, "balance_loss_clip": 1.0318104, "balance_loss_mlp": 1.03646731, "epoch": 0.06481286637607095, "flos": 24571207831680.0, "grad_norm": 2.564596832680389, "language_loss": 0.91844654, "learning_rate": 3.958824021886595e-06, "loss": 0.94063681, "num_input_tokens_seen": 22997285, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.203125, "step": 1078, "time_per_iteration": 2.5395843982696533 }, { "auxiliary_loss_clip": 0.01160052, "auxiliary_loss_mlp": 0.01057451, "balance_loss_clip": 1.02744603, "balance_loss_mlp": 1.03943264, "epoch": 0.06487298962873891, "flos": 21104576870400.0, "grad_norm": 2.0316399257948365, "language_loss": 0.78641224, "learning_rate": 3.9587477153647115e-06, "loss": 0.80858719, "num_input_tokens_seen": 23016285, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.203125, "step": 1079, "time_per_iteration": 2.4766886234283447 }, { "auxiliary_loss_clip": 0.01151624, "auxiliary_loss_mlp": 0.01057974, "balance_loss_clip": 1.02979302, "balance_loss_mlp": 1.0372957, "epoch": 0.06493311288140688, "flos": 24607028753280.0, "grad_norm": 2.63909668044762, "language_loss": 0.68948388, "learning_rate": 3.95867133894016e-06, "loss": 0.71157992, "num_input_tokens_seen": 23036420, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.140625, "step": 1080, "time_per_iteration": 3.8689091205596924 }, { "auxiliary_loss_clip": 0.01151384, "auxiliary_loss_mlp": 0.01055946, "balance_loss_clip": 1.02595294, "balance_loss_mlp": 1.03560901, "epoch": 0.06499323613407486, "flos": 25336818236160.0, "grad_norm": 1.7999405252280114, "language_loss": 0.72002423, "learning_rate": 3.958594892615667e-06, "loss": 0.7420975, "num_input_tokens_seen": 23056945, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.15625, "step": 1081, "time_per_iteration": 2.4564967155456543 }, { "auxiliary_loss_clip": 0.0114835, "auxiliary_loss_mlp": 0.01051298, "balance_loss_clip": 1.02166224, "balance_loss_mlp": 1.03567648, "epoch": 0.06505335938674282, "flos": 20374682653440.0, "grad_norm": 2.8791195051438643, "language_loss": 0.84015405, "learning_rate": 3.95851837639396e-06, "loss": 0.86215043, "num_input_tokens_seen": 23074940, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.125, "step": 1082, "time_per_iteration": 2.4006152153015137 }, { "auxiliary_loss_clip": 0.01159429, "auxiliary_loss_mlp": 0.01064826, "balance_loss_clip": 1.03422475, "balance_loss_mlp": 1.03717995, "epoch": 0.06511348263941079, "flos": 25336748413440.0, "grad_norm": 5.4199843251700655, "language_loss": 0.82377207, "learning_rate": 3.9584417902777695e-06, "loss": 0.84601462, "num_input_tokens_seen": 23093420, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.21875, "step": 1083, "time_per_iteration": 3.9749627113342285 }, { "auxiliary_loss_clip": 0.01156056, "auxiliary_loss_mlp": 0.01057279, "balance_loss_clip": 1.02689254, "balance_loss_mlp": 1.0394218, "epoch": 0.06517360589207877, "flos": 20331949282560.0, "grad_norm": 2.566877750876929, "language_loss": 0.79550064, "learning_rate": 3.95836513426983e-06, "loss": 0.81763399, "num_input_tokens_seen": 23111550, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.1640625, "step": 1084, "time_per_iteration": 2.466878890991211 }, { "auxiliary_loss_clip": 0.01152333, "auxiliary_loss_mlp": 0.01053444, "balance_loss_clip": 1.02485764, "balance_loss_mlp": 1.03761244, "epoch": 0.06523372914474673, "flos": 31680432339840.0, "grad_norm": 5.835992372464286, "language_loss": 0.66288763, "learning_rate": 3.958288408372877e-06, "loss": 0.68494546, "num_input_tokens_seen": 23130335, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.1484375, "step": 1085, "time_per_iteration": 3.9148972034454346 }, { "auxiliary_loss_clip": 0.01147609, "auxiliary_loss_mlp": 0.0105211, "balance_loss_clip": 1.02476358, "balance_loss_mlp": 1.03516364, "epoch": 0.0652938523974147, "flos": 20777091517440.0, "grad_norm": 2.121651067054262, "language_loss": 0.76523113, "learning_rate": 3.9582116125896474e-06, "loss": 0.78722835, "num_input_tokens_seen": 23152380, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.125, "step": 1086, "time_per_iteration": 2.4858388900756836 }, { "auxiliary_loss_clip": 0.01148721, "auxiliary_loss_mlp": 0.01047731, "balance_loss_clip": 1.0208497, "balance_loss_mlp": 1.03547835, "epoch": 0.06535397565008266, "flos": 16690053962880.0, "grad_norm": 3.2443909718870723, "language_loss": 0.85044527, "learning_rate": 3.958134746922882e-06, "loss": 0.87240976, "num_input_tokens_seen": 23171630, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.1328125, "step": 1087, "time_per_iteration": 2.390460252761841 }, { "auxiliary_loss_clip": 0.01147463, "auxiliary_loss_mlp": 0.01053357, "balance_loss_clip": 1.02568889, "balance_loss_mlp": 1.0346911, "epoch": 0.06541409890275064, "flos": 26867061527040.0, "grad_norm": 2.6455725952910427, "language_loss": 0.77596116, "learning_rate": 3.958057811375325e-06, "loss": 0.7979694, "num_input_tokens_seen": 23192520, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.125, "step": 1088, "time_per_iteration": 2.4677419662475586 }, { "auxiliary_loss_clip": 0.01149935, "auxiliary_loss_mlp": 0.01057884, "balance_loss_clip": 1.02992916, "balance_loss_mlp": 1.03685796, "epoch": 0.06547422215541861, "flos": 20520584691840.0, "grad_norm": 1.7317476980719246, "language_loss": 0.71197081, "learning_rate": 3.957980805949722e-06, "loss": 0.73404896, "num_input_tokens_seen": 23210710, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.125, "step": 1089, "time_per_iteration": 2.386992931365967 }, { "auxiliary_loss_clip": 0.01147423, "auxiliary_loss_mlp": 0.01050398, "balance_loss_clip": 1.02330232, "balance_loss_mlp": 1.0368315, "epoch": 0.06553434540808657, "flos": 22015565642880.0, "grad_norm": 1.8712122613700142, "language_loss": 0.85494733, "learning_rate": 3.957903730648819e-06, "loss": 0.87692559, "num_input_tokens_seen": 23230305, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.109375, "step": 1090, "time_per_iteration": 2.4183759689331055 }, { "auxiliary_loss_clip": 0.01153315, "auxiliary_loss_mlp": 0.01055158, "balance_loss_clip": 1.02670288, "balance_loss_mlp": 1.03855705, "epoch": 0.06559446866075455, "flos": 24607482600960.0, "grad_norm": 2.0463246774747117, "language_loss": 0.71929127, "learning_rate": 3.957826585475369e-06, "loss": 0.74137598, "num_input_tokens_seen": 23249015, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.1484375, "step": 1091, "time_per_iteration": 2.446702241897583 }, { "auxiliary_loss_clip": 0.01148337, "auxiliary_loss_mlp": 0.01053979, "balance_loss_clip": 1.02652502, "balance_loss_mlp": 1.03667808, "epoch": 0.06565459191342252, "flos": 24273678291840.0, "grad_norm": 2.577704198220226, "language_loss": 0.82610309, "learning_rate": 3.957749370432124e-06, "loss": 0.84812617, "num_input_tokens_seen": 23265105, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.1171875, "step": 1092, "time_per_iteration": 2.3923897743225098 }, { "auxiliary_loss_clip": 0.01152296, "auxiliary_loss_mlp": 0.01055245, "balance_loss_clip": 1.02546632, "balance_loss_mlp": 1.03611541, "epoch": 0.06571471516609048, "flos": 24786063538560.0, "grad_norm": 1.8958847964951662, "language_loss": 0.7130363, "learning_rate": 3.957672085521841e-06, "loss": 0.73511177, "num_input_tokens_seen": 23283950, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.15625, "step": 1093, "time_per_iteration": 2.4138355255126953 }, { "auxiliary_loss_clip": 0.01149541, "auxiliary_loss_mlp": 0.01052042, "balance_loss_clip": 1.02276444, "balance_loss_mlp": 1.03724301, "epoch": 0.06577483841875846, "flos": 26212858871040.0, "grad_norm": 1.6711946765405614, "language_loss": 0.87978733, "learning_rate": 3.957594730747276e-06, "loss": 0.90180314, "num_input_tokens_seen": 23305005, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.125, "step": 1094, "time_per_iteration": 2.4436089992523193 }, { "auxiliary_loss_clip": 0.01151625, "auxiliary_loss_mlp": 0.0105448, "balance_loss_clip": 1.02482069, "balance_loss_mlp": 1.037292, "epoch": 0.06583496167142643, "flos": 25079683006080.0, "grad_norm": 2.2043409811576806, "language_loss": 0.81170315, "learning_rate": 3.957517306111191e-06, "loss": 0.8337642, "num_input_tokens_seen": 23323220, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.140625, "step": 1095, "time_per_iteration": 2.4198005199432373 }, { "auxiliary_loss_clip": 0.0114609, "auxiliary_loss_mlp": 0.01048581, "balance_loss_clip": 1.02168703, "balance_loss_mlp": 1.03538322, "epoch": 0.06589508492409439, "flos": 25628622312960.0, "grad_norm": 2.126074922761706, "language_loss": 0.6998198, "learning_rate": 3.957439811616349e-06, "loss": 0.72176647, "num_input_tokens_seen": 23342235, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.109375, "step": 1096, "time_per_iteration": 2.4462523460388184 }, { "auxiliary_loss_clip": 0.01152339, "auxiliary_loss_mlp": 0.01050908, "balance_loss_clip": 1.02412224, "balance_loss_mlp": 1.039469, "epoch": 0.06595520817676236, "flos": 23620173863040.0, "grad_norm": 1.8544065519083277, "language_loss": 0.77033997, "learning_rate": 3.957362247265515e-06, "loss": 0.79237241, "num_input_tokens_seen": 23363680, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.125, "step": 1097, "time_per_iteration": 2.40742564201355 }, { "auxiliary_loss_clip": 0.0115117, "auxiliary_loss_mlp": 0.0106126, "balance_loss_clip": 1.03356814, "balance_loss_mlp": 1.03739858, "epoch": 0.06601533142943034, "flos": 33800323449600.0, "grad_norm": 1.966380454277295, "language_loss": 0.78213745, "learning_rate": 3.957284613061456e-06, "loss": 0.80426174, "num_input_tokens_seen": 23385590, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.140625, "step": 1098, "time_per_iteration": 2.5256941318511963 }, { "auxiliary_loss_clip": 0.01150204, "auxiliary_loss_mlp": 0.01060173, "balance_loss_clip": 1.03039408, "balance_loss_mlp": 1.03755939, "epoch": 0.0660754546820983, "flos": 20258352403200.0, "grad_norm": 4.2821213113645795, "language_loss": 0.81474102, "learning_rate": 3.957206909006945e-06, "loss": 0.8368448, "num_input_tokens_seen": 23402945, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.125, "step": 1099, "time_per_iteration": 2.382452964782715 }, { "auxiliary_loss_clip": 0.01142698, "auxiliary_loss_mlp": 0.01050563, "balance_loss_clip": 1.02282298, "balance_loss_mlp": 1.03261256, "epoch": 0.06613557793476627, "flos": 19353158916480.0, "grad_norm": 3.173236474032098, "language_loss": 0.82873213, "learning_rate": 3.957129135104754e-06, "loss": 0.85066473, "num_input_tokens_seen": 23421410, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.1015625, "step": 1100, "time_per_iteration": 2.41461443901062 }, { "auxiliary_loss_clip": 0.01149334, "auxiliary_loss_mlp": 0.01056582, "balance_loss_clip": 1.02933121, "balance_loss_mlp": 1.03661847, "epoch": 0.06619570118743424, "flos": 13771698992640.0, "grad_norm": 2.412003980507769, "language_loss": 0.73175687, "learning_rate": 3.957051291357658e-06, "loss": 0.75381601, "num_input_tokens_seen": 23438870, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.125, "step": 1101, "time_per_iteration": 2.445695638656616 }, { "auxiliary_loss_clip": 0.01146847, "auxiliary_loss_mlp": 0.01058545, "balance_loss_clip": 1.02983987, "balance_loss_mlp": 1.0356338, "epoch": 0.06625582444010221, "flos": 17856921156480.0, "grad_norm": 2.432250688453231, "language_loss": 0.85938394, "learning_rate": 3.956973377768437e-06, "loss": 0.8814379, "num_input_tokens_seen": 23456975, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.109375, "step": 1102, "time_per_iteration": 2.3959429264068604 }, { "auxiliary_loss_clip": 0.01148477, "auxiliary_loss_mlp": 0.01051257, "balance_loss_clip": 1.02208626, "balance_loss_mlp": 1.03743041, "epoch": 0.06631594769277017, "flos": 11837894762880.0, "grad_norm": 4.580157305379214, "language_loss": 0.81804848, "learning_rate": 3.956895394339869e-06, "loss": 0.84004581, "num_input_tokens_seen": 23473440, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.109375, "step": 1103, "time_per_iteration": 2.3681488037109375 }, { "auxiliary_loss_clip": 0.01150511, "auxiliary_loss_mlp": 0.01060027, "balance_loss_clip": 1.03302574, "balance_loss_mlp": 1.03946304, "epoch": 0.06637607094543815, "flos": 19792296397440.0, "grad_norm": 1.8496289460402604, "language_loss": 0.81953788, "learning_rate": 3.956817341074738e-06, "loss": 0.84164321, "num_input_tokens_seen": 23493880, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.109375, "step": 1104, "time_per_iteration": 2.4122419357299805 }, { "auxiliary_loss_clip": 0.01143497, "auxiliary_loss_mlp": 0.01047648, "balance_loss_clip": 1.0187993, "balance_loss_mlp": 1.03398204, "epoch": 0.06643619419810612, "flos": 25484430931200.0, "grad_norm": 1.8390153581956532, "language_loss": 0.80658793, "learning_rate": 3.95673921797583e-06, "loss": 0.82849944, "num_input_tokens_seen": 23514920, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.09375, "step": 1105, "time_per_iteration": 2.446779727935791 }, { "auxiliary_loss_clip": 0.01144253, "auxiliary_loss_mlp": 0.01052386, "balance_loss_clip": 1.02591014, "balance_loss_mlp": 1.0359776, "epoch": 0.06649631745077408, "flos": 16945583270400.0, "grad_norm": 2.004951132118422, "language_loss": 0.96369636, "learning_rate": 3.956661025045933e-06, "loss": 0.98566276, "num_input_tokens_seen": 23531635, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0859375, "step": 1106, "time_per_iteration": 2.384737014770508 }, { "auxiliary_loss_clip": 0.01149602, "auxiliary_loss_mlp": 0.0104897, "balance_loss_clip": 1.02033615, "balance_loss_mlp": 1.03467488, "epoch": 0.06655644070344206, "flos": 17857619383680.0, "grad_norm": 3.070621473217749, "language_loss": 0.8192116, "learning_rate": 3.9565827622878365e-06, "loss": 0.84119731, "num_input_tokens_seen": 23551020, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.15625, "step": 1107, "time_per_iteration": 2.3848822116851807 }, { "auxiliary_loss_clip": 0.01043134, "auxiliary_loss_mlp": 0.01023949, "balance_loss_clip": 1.01925182, "balance_loss_mlp": 1.00925303, "epoch": 0.06661656395611003, "flos": 61416236062080.0, "grad_norm": 0.7916552539491276, "language_loss": 0.56714582, "learning_rate": 3.956504429704334e-06, "loss": 0.5878166, "num_input_tokens_seen": 23610675, "router_z_loss_clip": 0.046875, "router_z_loss_mlp": 0.33984375, "step": 1108, "time_per_iteration": 2.987449884414673 }, { "auxiliary_loss_clip": 0.01147292, "auxiliary_loss_mlp": 0.01054553, "balance_loss_clip": 1.0233202, "balance_loss_mlp": 1.03442478, "epoch": 0.066676687208778, "flos": 20661948253440.0, "grad_norm": 3.414117940188091, "language_loss": 0.72846961, "learning_rate": 3.956426027298221e-06, "loss": 0.7504881, "num_input_tokens_seen": 23628710, "router_z_loss_clip": 0.3125, "router_z_loss_mlp": 1.125, "step": 1109, "time_per_iteration": 2.4279117584228516 }, { "auxiliary_loss_clip": 0.01147421, "auxiliary_loss_mlp": 0.01054663, "balance_loss_clip": 1.02586257, "balance_loss_mlp": 1.03525794, "epoch": 0.06673681046144596, "flos": 20922225505920.0, "grad_norm": 2.0648281320744832, "language_loss": 0.7821449, "learning_rate": 3.956347555072296e-06, "loss": 0.80416572, "num_input_tokens_seen": 23649160, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.125, "step": 1110, "time_per_iteration": 2.421630382537842 }, { "auxiliary_loss_clip": 0.01148098, "auxiliary_loss_mlp": 0.01051408, "balance_loss_clip": 1.0239898, "balance_loss_mlp": 1.03642201, "epoch": 0.06679693371411394, "flos": 31064494400640.0, "grad_norm": 3.521541261242907, "language_loss": 0.71108806, "learning_rate": 3.95626901302936e-06, "loss": 0.73308313, "num_input_tokens_seen": 23671995, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.1171875, "step": 1111, "time_per_iteration": 2.4702768325805664 }, { "auxiliary_loss_clip": 0.01152294, "auxiliary_loss_mlp": 0.01052218, "balance_loss_clip": 1.02451348, "balance_loss_mlp": 1.03722906, "epoch": 0.0668570569667819, "flos": 21725053286400.0, "grad_norm": 2.0572766136120872, "language_loss": 0.78350592, "learning_rate": 3.956190401172214e-06, "loss": 0.80555105, "num_input_tokens_seen": 23690705, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.1484375, "step": 1112, "time_per_iteration": 2.4014744758605957 }, { "auxiliary_loss_clip": 0.01148717, "auxiliary_loss_mlp": 0.01057763, "balance_loss_clip": 1.0297966, "balance_loss_mlp": 1.03698647, "epoch": 0.06691718021944987, "flos": 22746158087040.0, "grad_norm": 2.2200154515730315, "language_loss": 0.79009718, "learning_rate": 3.956111719503664e-06, "loss": 0.81216192, "num_input_tokens_seen": 23709990, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.1171875, "step": 1113, "time_per_iteration": 2.4003682136535645 }, { "auxiliary_loss_clip": 0.01143582, "auxiliary_loss_mlp": 0.01045722, "balance_loss_clip": 1.01901865, "balance_loss_mlp": 1.03396714, "epoch": 0.06697730347211785, "flos": 16544675594880.0, "grad_norm": 1.8213310860122236, "language_loss": 0.82533109, "learning_rate": 3.956032968026519e-06, "loss": 0.84722418, "num_input_tokens_seen": 23728485, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.09375, "step": 1114, "time_per_iteration": 2.3902881145477295 }, { "auxiliary_loss_clip": 0.01045519, "auxiliary_loss_mlp": 0.01003938, "balance_loss_clip": 0.99950367, "balance_loss_mlp": 1.0111022, "epoch": 0.06703742672478581, "flos": 59779123499520.0, "grad_norm": 0.8243039787841735, "language_loss": 0.58152986, "learning_rate": 3.955954146743589e-06, "loss": 0.60202444, "num_input_tokens_seen": 23786650, "router_z_loss_clip": 0.04443359, "router_z_loss_mlp": 0.34375, "step": 1115, "time_per_iteration": 2.975740432739258 }, { "auxiliary_loss_clip": 0.01148229, "auxiliary_loss_mlp": 0.01056833, "balance_loss_clip": 1.02709055, "balance_loss_mlp": 1.03565681, "epoch": 0.06709754997745378, "flos": 16799262295680.0, "grad_norm": 3.130615164771175, "language_loss": 0.9187237, "learning_rate": 3.9558752556576874e-06, "loss": 0.94077432, "num_input_tokens_seen": 23802555, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.125, "step": 1116, "time_per_iteration": 2.3603169918060303 }, { "auxiliary_loss_clip": 0.01154845, "auxiliary_loss_mlp": 0.01060873, "balance_loss_clip": 1.03114212, "balance_loss_mlp": 1.03859985, "epoch": 0.06715767323012176, "flos": 22122923673600.0, "grad_norm": 2.095241715439275, "language_loss": 0.87228984, "learning_rate": 3.955796294771628e-06, "loss": 0.89444697, "num_input_tokens_seen": 23822945, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.1640625, "step": 1117, "time_per_iteration": 2.437349319458008 }, { "auxiliary_loss_clip": 0.01041756, "auxiliary_loss_mlp": 0.01006534, "balance_loss_clip": 1.00224292, "balance_loss_mlp": 1.00733614, "epoch": 0.06721779648278972, "flos": 66615363020160.0, "grad_norm": 0.8524745008189767, "language_loss": 0.59762853, "learning_rate": 3.95571726408823e-06, "loss": 0.61811143, "num_input_tokens_seen": 23874075, "router_z_loss_clip": 0.04296875, "router_z_loss_mlp": 0.34375, "step": 1118, "time_per_iteration": 3.0528414249420166 }, { "auxiliary_loss_clip": 0.01146149, "auxiliary_loss_mlp": 0.01046751, "balance_loss_clip": 1.01955914, "balance_loss_mlp": 1.03465438, "epoch": 0.06727791973545769, "flos": 22381385535360.0, "grad_norm": 6.487344051059983, "language_loss": 0.82986391, "learning_rate": 3.955638163610314e-06, "loss": 0.85179293, "num_input_tokens_seen": 23889720, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.1171875, "step": 1119, "time_per_iteration": 2.419816732406616 }, { "auxiliary_loss_clip": 0.01144101, "auxiliary_loss_mlp": 0.01050539, "balance_loss_clip": 1.02450359, "balance_loss_mlp": 1.03527403, "epoch": 0.06733804298812565, "flos": 24279054641280.0, "grad_norm": 1.8616435906553814, "language_loss": 0.8482362, "learning_rate": 3.955558993340703e-06, "loss": 0.87018257, "num_input_tokens_seen": 23909385, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.09375, "step": 1120, "time_per_iteration": 3.8828959465026855 }, { "auxiliary_loss_clip": 0.01156273, "auxiliary_loss_mlp": 0.01058698, "balance_loss_clip": 1.03104162, "balance_loss_mlp": 1.0411582, "epoch": 0.06739816624079363, "flos": 15917496197760.0, "grad_norm": 2.2660236821839623, "language_loss": 0.78819853, "learning_rate": 3.955479753282221e-06, "loss": 0.81034827, "num_input_tokens_seen": 23926830, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.1484375, "step": 1121, "time_per_iteration": 2.4081673622131348 }, { "auxiliary_loss_clip": 0.01039947, "auxiliary_loss_mlp": 0.01005861, "balance_loss_clip": 1.00168824, "balance_loss_mlp": 1.00650489, "epoch": 0.0674582894934616, "flos": 71394656613120.0, "grad_norm": 0.7533590153096971, "language_loss": 0.58349454, "learning_rate": 3.955400443437696e-06, "loss": 0.60395265, "num_input_tokens_seen": 23992640, "router_z_loss_clip": 0.04174805, "router_z_loss_mlp": 0.3359375, "step": 1122, "time_per_iteration": 4.462320566177368 }, { "auxiliary_loss_clip": 0.0115262, "auxiliary_loss_mlp": 0.01052663, "balance_loss_clip": 1.02430344, "balance_loss_mlp": 1.03925991, "epoch": 0.06751841274612956, "flos": 25263779938560.0, "grad_norm": 2.039678532660783, "language_loss": 0.71565163, "learning_rate": 3.95532106380996e-06, "loss": 0.7377044, "num_input_tokens_seen": 24011135, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.1328125, "step": 1123, "time_per_iteration": 3.810786724090576 }, { "auxiliary_loss_clip": 0.01150952, "auxiliary_loss_mlp": 0.0105428, "balance_loss_clip": 1.02476323, "balance_loss_mlp": 1.03756046, "epoch": 0.06757853599879754, "flos": 23801687354880.0, "grad_norm": 1.86152894512166, "language_loss": 0.79015303, "learning_rate": 3.9552416144018445e-06, "loss": 0.81220531, "num_input_tokens_seen": 24030695, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.1328125, "step": 1124, "time_per_iteration": 2.4354350566864014 }, { "auxiliary_loss_clip": 0.01145399, "auxiliary_loss_mlp": 0.01044236, "balance_loss_clip": 1.01797402, "balance_loss_mlp": 1.0362289, "epoch": 0.0676386592514655, "flos": 21032655736320.0, "grad_norm": 2.7282165640532234, "language_loss": 0.71316373, "learning_rate": 3.955162095216186e-06, "loss": 0.7350601, "num_input_tokens_seen": 24050680, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 1.09375, "step": 1125, "time_per_iteration": 3.7452244758605957 }, { "auxiliary_loss_clip": 0.01145681, "auxiliary_loss_mlp": 0.01052374, "balance_loss_clip": 1.02295339, "balance_loss_mlp": 1.03685808, "epoch": 0.06769878250413347, "flos": 25555165079040.0, "grad_norm": 2.6751849311736544, "language_loss": 0.81167436, "learning_rate": 3.95508250625582e-06, "loss": 0.83365488, "num_input_tokens_seen": 24067205, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.0859375, "step": 1126, "time_per_iteration": 2.447014331817627 }, { "auxiliary_loss_clip": 0.01040105, "auxiliary_loss_mlp": 0.01005089, "balance_loss_clip": 1.00074983, "balance_loss_mlp": 1.00653863, "epoch": 0.06775890575680145, "flos": 70651740458880.0, "grad_norm": 0.7816526170598117, "language_loss": 0.59801042, "learning_rate": 3.95500284752359e-06, "loss": 0.61846232, "num_input_tokens_seen": 24131320, "router_z_loss_clip": 0.04345703, "router_z_loss_mlp": 0.3359375, "step": 1127, "time_per_iteration": 3.066779851913452 }, { "auxiliary_loss_clip": 0.01147014, "auxiliary_loss_mlp": 0.01048138, "balance_loss_clip": 1.02029109, "balance_loss_mlp": 1.03688431, "epoch": 0.06781902900946941, "flos": 24234575702400.0, "grad_norm": 2.224979259672447, "language_loss": 0.81246132, "learning_rate": 3.954923119022337e-06, "loss": 0.83441287, "num_input_tokens_seen": 24149930, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.1015625, "step": 1128, "time_per_iteration": 2.4281606674194336 }, { "auxiliary_loss_clip": 0.01154245, "auxiliary_loss_mlp": 0.01046281, "balance_loss_clip": 1.01767111, "balance_loss_mlp": 1.03841734, "epoch": 0.06787915226213738, "flos": 22416473318400.0, "grad_norm": 2.7069058363169156, "language_loss": 0.75399923, "learning_rate": 3.9548433207549065e-06, "loss": 0.77600449, "num_input_tokens_seen": 24169590, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.15625, "step": 1129, "time_per_iteration": 2.4316508769989014 }, { "auxiliary_loss_clip": 0.01145738, "auxiliary_loss_mlp": 0.01046612, "balance_loss_clip": 1.01823974, "balance_loss_mlp": 1.03559637, "epoch": 0.06793927551480534, "flos": 37705393664640.0, "grad_norm": 1.7740383949338567, "language_loss": 0.71722078, "learning_rate": 3.954763452724146e-06, "loss": 0.73914433, "num_input_tokens_seen": 24189965, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.09375, "step": 1130, "time_per_iteration": 2.5574941635131836 }, { "auxiliary_loss_clip": 0.01145337, "auxiliary_loss_mlp": 0.01049854, "balance_loss_clip": 1.02341366, "balance_loss_mlp": 1.03672767, "epoch": 0.06799939876747332, "flos": 20630351606400.0, "grad_norm": 2.5622001697638903, "language_loss": 0.80953151, "learning_rate": 3.954683514932906e-06, "loss": 0.83148336, "num_input_tokens_seen": 24208045, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0859375, "step": 1131, "time_per_iteration": 2.46608567237854 }, { "auxiliary_loss_clip": 0.0114364, "auxiliary_loss_mlp": 0.0105958, "balance_loss_clip": 1.0310415, "balance_loss_mlp": 1.0365063, "epoch": 0.06805952202014129, "flos": 14863921966080.0, "grad_norm": 10.230628435090006, "language_loss": 0.80578613, "learning_rate": 3.95460350738404e-06, "loss": 0.82781839, "num_input_tokens_seen": 24223805, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.0703125, "step": 1132, "time_per_iteration": 2.406174421310425 }, { "auxiliary_loss_clip": 0.01144118, "auxiliary_loss_mlp": 0.01054145, "balance_loss_clip": 1.02660751, "balance_loss_mlp": 1.03547812, "epoch": 0.06811964527280925, "flos": 48907555747200.0, "grad_norm": 1.5189189589491072, "language_loss": 0.63690358, "learning_rate": 3.954523430080402e-06, "loss": 0.65888619, "num_input_tokens_seen": 24249475, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.0859375, "step": 1133, "time_per_iteration": 2.6687326431274414 }, { "auxiliary_loss_clip": 0.01150547, "auxiliary_loss_mlp": 0.01053353, "balance_loss_clip": 1.02378917, "balance_loss_mlp": 1.03586221, "epoch": 0.06817976852547723, "flos": 15376377035520.0, "grad_norm": 2.2196303995449114, "language_loss": 0.74988973, "learning_rate": 3.9544432830248504e-06, "loss": 0.77192879, "num_input_tokens_seen": 24267980, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.1484375, "step": 1134, "time_per_iteration": 2.389263391494751 }, { "auxiliary_loss_clip": 0.01144706, "auxiliary_loss_mlp": 0.01052305, "balance_loss_clip": 1.02647233, "balance_loss_mlp": 1.03708625, "epoch": 0.0682398917781452, "flos": 20154694976640.0, "grad_norm": 3.295505603148525, "language_loss": 0.8708508, "learning_rate": 3.954363066220246e-06, "loss": 0.89282089, "num_input_tokens_seen": 24286805, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 1.078125, "step": 1135, "time_per_iteration": 2.395012378692627 }, { "auxiliary_loss_clip": 0.01146483, "auxiliary_loss_mlp": 0.01045769, "balance_loss_clip": 1.01824403, "balance_loss_mlp": 1.03473854, "epoch": 0.06830001503081316, "flos": 23439498243840.0, "grad_norm": 2.9357431467981527, "language_loss": 0.77959895, "learning_rate": 3.954282779669451e-06, "loss": 0.80152142, "num_input_tokens_seen": 24305855, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.1171875, "step": 1136, "time_per_iteration": 2.437164783477783 }, { "auxiliary_loss_clip": 0.01149479, "auxiliary_loss_mlp": 0.01056481, "balance_loss_clip": 1.02828848, "balance_loss_mlp": 1.0376966, "epoch": 0.06836013828348114, "flos": 34348389972480.0, "grad_norm": 10.53788335757046, "language_loss": 0.83737171, "learning_rate": 3.95420242337533e-06, "loss": 0.85943127, "num_input_tokens_seen": 24326535, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.1171875, "step": 1137, "time_per_iteration": 2.509925127029419 }, { "auxiliary_loss_clip": 0.01143737, "auxiliary_loss_mlp": 0.01049197, "balance_loss_clip": 1.02167106, "balance_loss_mlp": 1.0355711, "epoch": 0.06842026153614911, "flos": 23147729078400.0, "grad_norm": 2.548998429243754, "language_loss": 0.78280199, "learning_rate": 3.954121997340752e-06, "loss": 0.80473137, "num_input_tokens_seen": 24345810, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.078125, "step": 1138, "time_per_iteration": 2.451308012008667 }, { "auxiliary_loss_clip": 0.01147673, "auxiliary_loss_mlp": 0.01060586, "balance_loss_clip": 1.03071165, "balance_loss_mlp": 1.03628469, "epoch": 0.06848038478881707, "flos": 24607796803200.0, "grad_norm": 2.4230119432478325, "language_loss": 0.85318613, "learning_rate": 3.9540415015685855e-06, "loss": 0.8752687, "num_input_tokens_seen": 24366095, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.109375, "step": 1139, "time_per_iteration": 2.418158769607544 }, { "auxiliary_loss_clip": 0.01145708, "auxiliary_loss_mlp": 0.01045786, "balance_loss_clip": 1.01917887, "balance_loss_mlp": 1.0365721, "epoch": 0.06854050804148504, "flos": 40879382676480.0, "grad_norm": 1.802646008694186, "language_loss": 0.74583817, "learning_rate": 3.953960936061706e-06, "loss": 0.76775312, "num_input_tokens_seen": 24388665, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.09375, "step": 1140, "time_per_iteration": 2.5871405601501465 }, { "auxiliary_loss_clip": 0.01144081, "auxiliary_loss_mlp": 0.01060288, "balance_loss_clip": 1.02993786, "balance_loss_mlp": 1.03546023, "epoch": 0.06860063129415302, "flos": 31685005728000.0, "grad_norm": 2.353283521657733, "language_loss": 0.6831162, "learning_rate": 3.9538803008229845e-06, "loss": 0.70515984, "num_input_tokens_seen": 24407705, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.0859375, "step": 1141, "time_per_iteration": 2.4802167415618896 }, { "auxiliary_loss_clip": 0.01147212, "auxiliary_loss_mlp": 0.01055887, "balance_loss_clip": 1.02670479, "balance_loss_mlp": 1.03654337, "epoch": 0.06866075454682098, "flos": 26540798071680.0, "grad_norm": 2.4042942679845396, "language_loss": 0.78867722, "learning_rate": 3.953799595855303e-06, "loss": 0.81070817, "num_input_tokens_seen": 24428390, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.109375, "step": 1142, "time_per_iteration": 2.4773917198181152 }, { "auxiliary_loss_clip": 0.01144319, "auxiliary_loss_mlp": 0.01050934, "balance_loss_clip": 1.02454114, "balance_loss_mlp": 1.03604782, "epoch": 0.06872087779948895, "flos": 29788453785600.0, "grad_norm": 1.8654347049738194, "language_loss": 0.6836428, "learning_rate": 3.953718821161539e-06, "loss": 0.70559537, "num_input_tokens_seen": 24450810, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0859375, "step": 1143, "time_per_iteration": 2.5048325061798096 }, { "auxiliary_loss_clip": 0.01138427, "auxiliary_loss_mlp": 0.01050695, "balance_loss_clip": 1.02438569, "balance_loss_mlp": 1.03550994, "epoch": 0.06878100105215693, "flos": 26939960179200.0, "grad_norm": 1.722226436663597, "language_loss": 0.74243826, "learning_rate": 3.953637976744576e-06, "loss": 0.76432949, "num_input_tokens_seen": 24469965, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 1.03125, "step": 1144, "time_per_iteration": 2.498687267303467 }, { "auxiliary_loss_clip": 0.01146537, "auxiliary_loss_mlp": 0.01055603, "balance_loss_clip": 1.02723074, "balance_loss_mlp": 1.03431416, "epoch": 0.06884112430482489, "flos": 10669980228480.0, "grad_norm": 4.7915259833189205, "language_loss": 0.9168638, "learning_rate": 3.953557062607299e-06, "loss": 0.93888521, "num_input_tokens_seen": 24486370, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.125, "step": 1145, "time_per_iteration": 2.358703136444092 }, { "auxiliary_loss_clip": 0.01150496, "auxiliary_loss_mlp": 0.01056676, "balance_loss_clip": 1.02692151, "balance_loss_mlp": 1.03589928, "epoch": 0.06890124755749286, "flos": 20192610579840.0, "grad_norm": 2.3528228070886286, "language_loss": 0.81935954, "learning_rate": 3.953476078752595e-06, "loss": 0.84143126, "num_input_tokens_seen": 24503780, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.140625, "step": 1146, "time_per_iteration": 2.4289767742156982 }, { "auxiliary_loss_clip": 0.01142062, "auxiliary_loss_mlp": 0.01057507, "balance_loss_clip": 1.0310905, "balance_loss_mlp": 1.03638935, "epoch": 0.06896137081016084, "flos": 20448174798720.0, "grad_norm": 2.3180790737616364, "language_loss": 0.84927756, "learning_rate": 3.953395025183355e-06, "loss": 0.87127328, "num_input_tokens_seen": 24522320, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 1.0625, "step": 1147, "time_per_iteration": 2.3913071155548096 }, { "auxiliary_loss_clip": 0.01145703, "auxiliary_loss_mlp": 0.01050701, "balance_loss_clip": 1.02339029, "balance_loss_mlp": 1.03590751, "epoch": 0.0690214940628288, "flos": 18367735392000.0, "grad_norm": 1.9433216530482342, "language_loss": 0.85627848, "learning_rate": 3.9533139019024715e-06, "loss": 0.87824255, "num_input_tokens_seen": 24540445, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.09375, "step": 1148, "time_per_iteration": 2.3985259532928467 }, { "auxiliary_loss_clip": 0.01142808, "auxiliary_loss_mlp": 0.01051462, "balance_loss_clip": 1.02472377, "balance_loss_mlp": 1.0347476, "epoch": 0.06908161731549677, "flos": 20556999106560.0, "grad_norm": 2.5802358550337074, "language_loss": 0.69454765, "learning_rate": 3.953232708912839e-06, "loss": 0.71649039, "num_input_tokens_seen": 24557105, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.078125, "step": 1149, "time_per_iteration": 2.38120436668396 }, { "auxiliary_loss_clip": 0.01148519, "auxiliary_loss_mlp": 0.01048599, "balance_loss_clip": 1.02033401, "balance_loss_mlp": 1.03614509, "epoch": 0.06914174056816474, "flos": 27562426542720.0, "grad_norm": 2.0275636731663966, "language_loss": 0.83030009, "learning_rate": 3.953151446217356e-06, "loss": 0.85227126, "num_input_tokens_seen": 24578240, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.125, "step": 1150, "time_per_iteration": 2.4575576782226562 }, { "auxiliary_loss_clip": 0.01150229, "auxiliary_loss_mlp": 0.01055713, "balance_loss_clip": 1.02748466, "balance_loss_mlp": 1.03936958, "epoch": 0.06920186382083271, "flos": 15303129269760.0, "grad_norm": 3.4158487911616255, "language_loss": 0.8146646, "learning_rate": 3.953070113818921e-06, "loss": 0.83672404, "num_input_tokens_seen": 24593585, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.109375, "step": 1151, "time_per_iteration": 2.356996774673462 }, { "auxiliary_loss_clip": 0.01142556, "auxiliary_loss_mlp": 0.01047837, "balance_loss_clip": 1.0213964, "balance_loss_mlp": 1.03563929, "epoch": 0.06926198707350067, "flos": 25190078325120.0, "grad_norm": 2.1900678702636296, "language_loss": 0.85472023, "learning_rate": 3.952988711720439e-06, "loss": 0.87662417, "num_input_tokens_seen": 24613110, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0625, "step": 1152, "time_per_iteration": 2.472827196121216 }, { "auxiliary_loss_clip": 0.01142016, "auxiliary_loss_mlp": 0.01049989, "balance_loss_clip": 1.02365518, "balance_loss_mlp": 1.03515291, "epoch": 0.06932211032616864, "flos": 13255438584960.0, "grad_norm": 1.9839593451496706, "language_loss": 0.90736151, "learning_rate": 3.952907239924813e-06, "loss": 0.92928159, "num_input_tokens_seen": 24628795, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 1.0703125, "step": 1153, "time_per_iteration": 2.3746461868286133 }, { "auxiliary_loss_clip": 0.01144548, "auxiliary_loss_mlp": 0.01048424, "balance_loss_clip": 1.01949143, "balance_loss_mlp": 1.035882, "epoch": 0.06938223357883662, "flos": 24826213468800.0, "grad_norm": 2.2986110293559463, "language_loss": 0.81671846, "learning_rate": 3.95282569843495e-06, "loss": 0.83864814, "num_input_tokens_seen": 24645480, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.09375, "step": 1154, "time_per_iteration": 2.451007843017578 }, { "auxiliary_loss_clip": 0.01141782, "auxiliary_loss_mlp": 0.01054789, "balance_loss_clip": 1.02863431, "balance_loss_mlp": 1.03702188, "epoch": 0.06944235683150458, "flos": 27266852039040.0, "grad_norm": 1.8435908644227317, "language_loss": 0.75050694, "learning_rate": 3.952744087253762e-06, "loss": 0.77247268, "num_input_tokens_seen": 24664630, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.046875, "step": 1155, "time_per_iteration": 2.449211597442627 }, { "auxiliary_loss_clip": 0.01141457, "auxiliary_loss_mlp": 0.01045484, "balance_loss_clip": 1.01813757, "balance_loss_mlp": 1.0339638, "epoch": 0.06950248008417255, "flos": 25806993782400.0, "grad_norm": 1.809875608216508, "language_loss": 0.70478129, "learning_rate": 3.952662406384161e-06, "loss": 0.72665071, "num_input_tokens_seen": 24684210, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.078125, "step": 1156, "time_per_iteration": 2.47178316116333 }, { "auxiliary_loss_clip": 0.01144698, "auxiliary_loss_mlp": 0.01056941, "balance_loss_clip": 1.02679288, "balance_loss_mlp": 1.03540492, "epoch": 0.06956260333684053, "flos": 22270501457280.0, "grad_norm": 2.042998869215872, "language_loss": 0.75011253, "learning_rate": 3.952580655829061e-06, "loss": 0.77212894, "num_input_tokens_seen": 24702490, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.09375, "step": 1157, "time_per_iteration": 2.4180266857147217 }, { "auxiliary_loss_clip": 0.01144855, "auxiliary_loss_mlp": 0.01055636, "balance_loss_clip": 1.02833712, "balance_loss_mlp": 1.03508615, "epoch": 0.0696227265895085, "flos": 29680048414080.0, "grad_norm": 1.9123471701501298, "language_loss": 0.71525955, "learning_rate": 3.952498835591381e-06, "loss": 0.73726451, "num_input_tokens_seen": 24724340, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.09375, "step": 1158, "time_per_iteration": 2.50199031829834 }, { "auxiliary_loss_clip": 0.01144933, "auxiliary_loss_mlp": 0.0105089, "balance_loss_clip": 1.0223515, "balance_loss_mlp": 1.03502059, "epoch": 0.06968284984217646, "flos": 25522276711680.0, "grad_norm": 1.8495643318440533, "language_loss": 0.79798836, "learning_rate": 3.952416945674039e-06, "loss": 0.81994659, "num_input_tokens_seen": 24745550, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.09375, "step": 1159, "time_per_iteration": 3.928196668624878 }, { "auxiliary_loss_clip": 0.01149889, "auxiliary_loss_mlp": 0.01054699, "balance_loss_clip": 1.02370465, "balance_loss_mlp": 1.0370692, "epoch": 0.06974297309484444, "flos": 20697315327360.0, "grad_norm": 2.9024267789836666, "language_loss": 0.80438364, "learning_rate": 3.952334986079957e-06, "loss": 0.82642949, "num_input_tokens_seen": 24762575, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.125, "step": 1160, "time_per_iteration": 2.3827168941497803 }, { "auxiliary_loss_clip": 0.01143649, "auxiliary_loss_mlp": 0.01054615, "balance_loss_clip": 1.02514696, "balance_loss_mlp": 1.03302932, "epoch": 0.0698030963475124, "flos": 26503999632000.0, "grad_norm": 1.6930892433628664, "language_loss": 0.756661, "learning_rate": 3.9522529568120635e-06, "loss": 0.77864367, "num_input_tokens_seen": 24782605, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.109375, "step": 1161, "time_per_iteration": 2.4427478313446045 }, { "auxiliary_loss_clip": 0.01140831, "auxiliary_loss_mlp": 0.01054918, "balance_loss_clip": 1.02531838, "balance_loss_mlp": 1.03243947, "epoch": 0.06986321960018037, "flos": 23039288795520.0, "grad_norm": 1.789627523228887, "language_loss": 0.82873094, "learning_rate": 3.952170857873283e-06, "loss": 0.85068834, "num_input_tokens_seen": 24802910, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.0859375, "step": 1162, "time_per_iteration": 5.221744537353516 }, { "auxiliary_loss_clip": 0.01137981, "auxiliary_loss_mlp": 0.01047741, "balance_loss_clip": 1.01998901, "balance_loss_mlp": 1.03151393, "epoch": 0.06992334285284833, "flos": 28583566254720.0, "grad_norm": 2.1630598390116518, "language_loss": 0.78933895, "learning_rate": 3.952088689266547e-06, "loss": 0.81119615, "num_input_tokens_seen": 24823305, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0625, "step": 1163, "time_per_iteration": 2.46140193939209 }, { "auxiliary_loss_clip": 0.01143545, "auxiliary_loss_mlp": 0.01054695, "balance_loss_clip": 1.02511919, "balance_loss_mlp": 1.03399253, "epoch": 0.06998346610551631, "flos": 20594286305280.0, "grad_norm": 2.0283146772607057, "language_loss": 0.79181325, "learning_rate": 3.952006450994786e-06, "loss": 0.81379569, "num_input_tokens_seen": 24842155, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.09375, "step": 1164, "time_per_iteration": 3.7580206394195557 }, { "auxiliary_loss_clip": 0.01143306, "auxiliary_loss_mlp": 0.01055278, "balance_loss_clip": 1.02756214, "balance_loss_mlp": 1.0347116, "epoch": 0.07004358935818428, "flos": 22527706510080.0, "grad_norm": 1.5771235008123332, "language_loss": 0.72730517, "learning_rate": 3.951924143060937e-06, "loss": 0.74929094, "num_input_tokens_seen": 24862080, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0859375, "step": 1165, "time_per_iteration": 2.418278694152832 }, { "auxiliary_loss_clip": 0.01142289, "auxiliary_loss_mlp": 0.01051947, "balance_loss_clip": 1.02405202, "balance_loss_mlp": 1.03415227, "epoch": 0.07010371261085224, "flos": 28948722831360.0, "grad_norm": 2.5634817613944993, "language_loss": 0.80783445, "learning_rate": 3.951841765467935e-06, "loss": 0.82977676, "num_input_tokens_seen": 24886165, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.078125, "step": 1166, "time_per_iteration": 2.481705904006958 }, { "auxiliary_loss_clip": 0.0113909, "auxiliary_loss_mlp": 0.01044312, "balance_loss_clip": 1.01615429, "balance_loss_mlp": 1.03347373, "epoch": 0.07016383586352022, "flos": 23658054554880.0, "grad_norm": 2.060449597299873, "language_loss": 0.84201783, "learning_rate": 3.951759318218722e-06, "loss": 0.86385179, "num_input_tokens_seen": 24905775, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.0546875, "step": 1167, "time_per_iteration": 2.4210996627807617 }, { "auxiliary_loss_clip": 0.01144835, "auxiliary_loss_mlp": 0.01052096, "balance_loss_clip": 1.02310443, "balance_loss_mlp": 1.03402662, "epoch": 0.07022395911618819, "flos": 19791109411200.0, "grad_norm": 2.2660493238016324, "language_loss": 0.89404839, "learning_rate": 3.951676801316239e-06, "loss": 0.91601771, "num_input_tokens_seen": 24924295, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.109375, "step": 1168, "time_per_iteration": 2.4255263805389404 }, { "auxiliary_loss_clip": 0.01145153, "auxiliary_loss_mlp": 0.01062315, "balance_loss_clip": 1.03074884, "balance_loss_mlp": 1.03228617, "epoch": 0.07028408236885615, "flos": 21688080289920.0, "grad_norm": 6.326228126873958, "language_loss": 0.88479823, "learning_rate": 3.951594214763431e-06, "loss": 0.90687293, "num_input_tokens_seen": 24943210, "router_z_loss_clip": 0.31640625, "router_z_loss_mlp": 1.125, "step": 1169, "time_per_iteration": 2.4024498462677 }, { "auxiliary_loss_clip": 0.01145541, "auxiliary_loss_mlp": 0.01061234, "balance_loss_clip": 1.03035831, "balance_loss_mlp": 1.03636777, "epoch": 0.07034420562152413, "flos": 25629076160640.0, "grad_norm": 3.762141607568952, "language_loss": 0.83485639, "learning_rate": 3.951511558563246e-06, "loss": 0.85692418, "num_input_tokens_seen": 24960360, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.09375, "step": 1170, "time_per_iteration": 2.454987049102783 }, { "auxiliary_loss_clip": 0.01142319, "auxiliary_loss_mlp": 0.01058716, "balance_loss_clip": 1.02900887, "balance_loss_mlp": 1.03398812, "epoch": 0.0704043288741921, "flos": 20809491125760.0, "grad_norm": 2.0649427383890804, "language_loss": 0.75835848, "learning_rate": 3.951428832718633e-06, "loss": 0.7803688, "num_input_tokens_seen": 24978290, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.078125, "step": 1171, "time_per_iteration": 2.4012880325317383 }, { "auxiliary_loss_clip": 0.01142165, "auxiliary_loss_mlp": 0.0104635, "balance_loss_clip": 1.01909852, "balance_loss_mlp": 1.03408408, "epoch": 0.07046445212686006, "flos": 25591998430080.0, "grad_norm": 1.8672122014430101, "language_loss": 0.88891184, "learning_rate": 3.951346037232546e-06, "loss": 0.91079688, "num_input_tokens_seen": 24997055, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.078125, "step": 1172, "time_per_iteration": 2.453601121902466 }, { "auxiliary_loss_clip": 0.01143036, "auxiliary_loss_mlp": 0.01048481, "balance_loss_clip": 1.01798701, "balance_loss_mlp": 1.03233933, "epoch": 0.07052457537952803, "flos": 25555793483520.0, "grad_norm": 2.2028699019088385, "language_loss": 0.82122999, "learning_rate": 3.951263172107937e-06, "loss": 0.84314519, "num_input_tokens_seen": 25017490, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.109375, "step": 1173, "time_per_iteration": 2.4472427368164062 }, { "auxiliary_loss_clip": 0.0114247, "auxiliary_loss_mlp": 0.01051073, "balance_loss_clip": 1.02131832, "balance_loss_mlp": 1.03417015, "epoch": 0.070584698632196, "flos": 17967525943680.0, "grad_norm": 56.39445814219136, "language_loss": 0.8231191, "learning_rate": 3.951180237347765e-06, "loss": 0.84505451, "num_input_tokens_seen": 25035660, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.078125, "step": 1174, "time_per_iteration": 2.4031693935394287 }, { "auxiliary_loss_clip": 0.01142534, "auxiliary_loss_mlp": 0.01051656, "balance_loss_clip": 1.02391613, "balance_loss_mlp": 1.03413618, "epoch": 0.07064482188486397, "flos": 25369811337600.0, "grad_norm": 2.066140012195251, "language_loss": 0.85233241, "learning_rate": 3.951097232954989e-06, "loss": 0.87427437, "num_input_tokens_seen": 25054785, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.078125, "step": 1175, "time_per_iteration": 2.438138484954834 }, { "auxiliary_loss_clip": 0.01145033, "auxiliary_loss_mlp": 0.01059015, "balance_loss_clip": 1.02916503, "balance_loss_mlp": 1.03509188, "epoch": 0.07070494513753194, "flos": 24898693184640.0, "grad_norm": 1.9238158226412332, "language_loss": 0.83100969, "learning_rate": 3.951014158932572e-06, "loss": 0.85305011, "num_input_tokens_seen": 25075180, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.1015625, "step": 1176, "time_per_iteration": 2.4777917861938477 }, { "auxiliary_loss_clip": 0.01142268, "auxiliary_loss_mlp": 0.01058178, "balance_loss_clip": 1.02932918, "balance_loss_mlp": 1.03444862, "epoch": 0.07076506839019991, "flos": 22337569912320.0, "grad_norm": 4.217455463468714, "language_loss": 0.74490559, "learning_rate": 3.950931015283479e-06, "loss": 0.76691002, "num_input_tokens_seen": 25093035, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.078125, "step": 1177, "time_per_iteration": 2.4201037883758545 }, { "auxiliary_loss_clip": 0.01147636, "auxiliary_loss_mlp": 0.01051709, "balance_loss_clip": 1.0224309, "balance_loss_mlp": 1.03535318, "epoch": 0.07082519164286788, "flos": 18659818759680.0, "grad_norm": 2.1383008413969153, "language_loss": 0.86319709, "learning_rate": 3.950847802010675e-06, "loss": 0.88519061, "num_input_tokens_seen": 25112520, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.125, "step": 1178, "time_per_iteration": 2.4174270629882812 }, { "auxiliary_loss_clip": 0.01068847, "auxiliary_loss_mlp": 0.01008342, "balance_loss_clip": 1.00047445, "balance_loss_mlp": 1.01997209, "epoch": 0.07088531489553584, "flos": 63650676942720.0, "grad_norm": 0.8365517648153916, "language_loss": 0.63280094, "learning_rate": 3.950764519117132e-06, "loss": 0.65357292, "num_input_tokens_seen": 25177760, "router_z_loss_clip": 0.07861328, "router_z_loss_mlp": 0.48828125, "step": 1179, "time_per_iteration": 3.1840317249298096 }, { "auxiliary_loss_clip": 0.01145276, "auxiliary_loss_mlp": 0.0105284, "balance_loss_clip": 1.02395582, "balance_loss_mlp": 1.0354408, "epoch": 0.07094543814820382, "flos": 21571819862400.0, "grad_norm": 2.5128410300265416, "language_loss": 0.83514106, "learning_rate": 3.9506811666058215e-06, "loss": 0.85712224, "num_input_tokens_seen": 25195260, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.1015625, "step": 1180, "time_per_iteration": 2.446951389312744 }, { "auxiliary_loss_clip": 0.011423, "auxiliary_loss_mlp": 0.01053974, "balance_loss_clip": 1.02662778, "balance_loss_mlp": 1.03505707, "epoch": 0.07100556140087179, "flos": 22088883231360.0, "grad_norm": 2.205994704076047, "language_loss": 0.87598801, "learning_rate": 3.950597744479717e-06, "loss": 0.89795077, "num_input_tokens_seen": 25212740, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.0703125, "step": 1181, "time_per_iteration": 2.4032480716705322 }, { "auxiliary_loss_clip": 0.0114354, "auxiliary_loss_mlp": 0.01055523, "balance_loss_clip": 1.02886796, "balance_loss_mlp": 1.03676593, "epoch": 0.07106568465353975, "flos": 47920491388800.0, "grad_norm": 2.04683450933953, "language_loss": 0.83846635, "learning_rate": 3.950514252741797e-06, "loss": 0.86045694, "num_input_tokens_seen": 25236420, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0703125, "step": 1182, "time_per_iteration": 2.7398793697357178 }, { "auxiliary_loss_clip": 0.01141069, "auxiliary_loss_mlp": 0.01048225, "balance_loss_clip": 1.01976967, "balance_loss_mlp": 1.03627372, "epoch": 0.07112580790620772, "flos": 23439672800640.0, "grad_norm": 3.851505484815403, "language_loss": 0.7913717, "learning_rate": 3.950430691395042e-06, "loss": 0.81326461, "num_input_tokens_seen": 25255120, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.046875, "step": 1183, "time_per_iteration": 2.4907233715057373 }, { "auxiliary_loss_clip": 0.01145502, "auxiliary_loss_mlp": 0.01055042, "balance_loss_clip": 1.02502549, "balance_loss_mlp": 1.03338456, "epoch": 0.0711859311588757, "flos": 31867531649280.0, "grad_norm": 2.0698299749698843, "language_loss": 0.78832853, "learning_rate": 3.95034706044243e-06, "loss": 0.81033391, "num_input_tokens_seen": 25275150, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.125, "step": 1184, "time_per_iteration": 2.532350540161133 }, { "auxiliary_loss_clip": 0.01141494, "auxiliary_loss_mlp": 0.01055609, "balance_loss_clip": 1.0272969, "balance_loss_mlp": 1.03506601, "epoch": 0.07124605441154366, "flos": 19609281717120.0, "grad_norm": 1.9766682763801302, "language_loss": 0.76702213, "learning_rate": 3.95026335988695e-06, "loss": 0.78899324, "num_input_tokens_seen": 25293680, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.0625, "step": 1185, "time_per_iteration": 2.4000062942504883 }, { "auxiliary_loss_clip": 0.0114118, "auxiliary_loss_mlp": 0.01052528, "balance_loss_clip": 1.02598047, "balance_loss_mlp": 1.03560448, "epoch": 0.07130617766421163, "flos": 14683560549120.0, "grad_norm": 2.3640395760624795, "language_loss": 0.65478528, "learning_rate": 3.950179589731587e-06, "loss": 0.67672229, "num_input_tokens_seen": 25310050, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0546875, "step": 1186, "time_per_iteration": 2.405527353286743 }, { "auxiliary_loss_clip": 0.01141241, "auxiliary_loss_mlp": 0.01048348, "balance_loss_clip": 1.02125204, "balance_loss_mlp": 1.0361383, "epoch": 0.07136630091687961, "flos": 26066712453120.0, "grad_norm": 1.8983866206856574, "language_loss": 0.69567817, "learning_rate": 3.950095749979331e-06, "loss": 0.717574, "num_input_tokens_seen": 25331020, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.046875, "step": 1187, "time_per_iteration": 2.451582670211792 }, { "auxiliary_loss_clip": 0.01142274, "auxiliary_loss_mlp": 0.01049525, "balance_loss_clip": 1.02415752, "balance_loss_mlp": 1.03641868, "epoch": 0.07142642416954757, "flos": 15668285846400.0, "grad_norm": 2.595109808513564, "language_loss": 0.79029095, "learning_rate": 3.950011840633174e-06, "loss": 0.81220895, "num_input_tokens_seen": 25347875, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 1.0625, "step": 1188, "time_per_iteration": 2.4079363346099854 }, { "auxiliary_loss_clip": 0.01143543, "auxiliary_loss_mlp": 0.01049544, "balance_loss_clip": 1.02231669, "balance_loss_mlp": 1.03669262, "epoch": 0.07148654742221554, "flos": 19754310971520.0, "grad_norm": 1.9246330997835532, "language_loss": 0.84834594, "learning_rate": 3.9499278616961106e-06, "loss": 0.87027681, "num_input_tokens_seen": 25366715, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.0703125, "step": 1189, "time_per_iteration": 2.4651424884796143 }, { "auxiliary_loss_clip": 0.01140024, "auxiliary_loss_mlp": 0.01049078, "balance_loss_clip": 1.02243471, "balance_loss_mlp": 1.03461695, "epoch": 0.07154667067488352, "flos": 23470850511360.0, "grad_norm": 1.8648074508764025, "language_loss": 0.76680577, "learning_rate": 3.949843813171137e-06, "loss": 0.78869677, "num_input_tokens_seen": 25385450, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.0546875, "step": 1190, "time_per_iteration": 2.448935031890869 }, { "auxiliary_loss_clip": 0.01145334, "auxiliary_loss_mlp": 0.01063064, "balance_loss_clip": 1.03400064, "balance_loss_mlp": 1.03551149, "epoch": 0.07160679392755148, "flos": 18331949381760.0, "grad_norm": 2.0180827544920383, "language_loss": 0.75543731, "learning_rate": 3.949759695061254e-06, "loss": 0.77752125, "num_input_tokens_seen": 25403940, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.09375, "step": 1191, "time_per_iteration": 2.420780897140503 }, { "auxiliary_loss_clip": 0.01143815, "auxiliary_loss_mlp": 0.01053508, "balance_loss_clip": 1.02427769, "balance_loss_mlp": 1.03578496, "epoch": 0.07166691718021945, "flos": 17746106901120.0, "grad_norm": 3.7572314550321306, "language_loss": 0.74226058, "learning_rate": 3.949675507369463e-06, "loss": 0.76423383, "num_input_tokens_seen": 25420410, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.078125, "step": 1192, "time_per_iteration": 2.41892409324646 }, { "auxiliary_loss_clip": 0.01139894, "auxiliary_loss_mlp": 0.0104925, "balance_loss_clip": 1.02218974, "balance_loss_mlp": 1.03323972, "epoch": 0.07172704043288743, "flos": 22450932696960.0, "grad_norm": 2.174877482137923, "language_loss": 0.78133452, "learning_rate": 3.949591250098768e-06, "loss": 0.80322599, "num_input_tokens_seen": 25439415, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.0703125, "step": 1193, "time_per_iteration": 2.4201836585998535 }, { "auxiliary_loss_clip": 0.01145021, "auxiliary_loss_mlp": 0.01050061, "balance_loss_clip": 1.02161729, "balance_loss_mlp": 1.03653884, "epoch": 0.07178716368555539, "flos": 23221081578240.0, "grad_norm": 2.173482196864183, "language_loss": 0.85534096, "learning_rate": 3.949506923252175e-06, "loss": 0.87729174, "num_input_tokens_seen": 25458715, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.0859375, "step": 1194, "time_per_iteration": 2.473048448562622 }, { "auxiliary_loss_clip": 0.01141008, "auxiliary_loss_mlp": 0.01053987, "balance_loss_clip": 1.02797532, "balance_loss_mlp": 1.03503466, "epoch": 0.07184728693822336, "flos": 25149788749440.0, "grad_norm": 2.39331448397616, "language_loss": 0.81294763, "learning_rate": 3.9494225268326965e-06, "loss": 0.83489752, "num_input_tokens_seen": 25477985, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 1.0625, "step": 1195, "time_per_iteration": 2.423985719680786 }, { "auxiliary_loss_clip": 0.01142658, "auxiliary_loss_mlp": 0.01044567, "balance_loss_clip": 1.01861489, "balance_loss_mlp": 1.03682089, "epoch": 0.07190741019089132, "flos": 22710127697280.0, "grad_norm": 1.9294649589775585, "language_loss": 0.7980628, "learning_rate": 3.949338060843342e-06, "loss": 0.81993502, "num_input_tokens_seen": 25497110, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 1.0625, "step": 1196, "time_per_iteration": 2.4577596187591553 }, { "auxiliary_loss_clip": 0.01137897, "auxiliary_loss_mlp": 0.01062488, "balance_loss_clip": 1.03280461, "balance_loss_mlp": 1.03426504, "epoch": 0.0719675334435593, "flos": 29348548254720.0, "grad_norm": 3.0521239106417553, "language_loss": 0.70851308, "learning_rate": 3.949253525287126e-06, "loss": 0.73051691, "num_input_tokens_seen": 25516555, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.0390625, "step": 1197, "time_per_iteration": 2.4613070487976074 }, { "auxiliary_loss_clip": 0.01138148, "auxiliary_loss_mlp": 0.01048628, "balance_loss_clip": 1.02221167, "balance_loss_mlp": 1.03309786, "epoch": 0.07202765669622727, "flos": 17638818693120.0, "grad_norm": 4.029424760912505, "language_loss": 0.85489368, "learning_rate": 3.9491689201670655e-06, "loss": 0.8767615, "num_input_tokens_seen": 25533895, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 1.046875, "step": 1198, "time_per_iteration": 2.4109957218170166 }, { "auxiliary_loss_clip": 0.01142444, "auxiliary_loss_mlp": 0.01053808, "balance_loss_clip": 1.02474451, "balance_loss_mlp": 1.03578997, "epoch": 0.07208777994889523, "flos": 21432969918720.0, "grad_norm": 2.2387513912187056, "language_loss": 0.83341557, "learning_rate": 3.94908424548618e-06, "loss": 0.85537809, "num_input_tokens_seen": 25554195, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.0703125, "step": 1199, "time_per_iteration": 3.895958185195923 }, { "auxiliary_loss_clip": 0.01146576, "auxiliary_loss_mlp": 0.01055465, "balance_loss_clip": 1.02858305, "balance_loss_mlp": 1.03858709, "epoch": 0.07214790320156321, "flos": 26939715799680.0, "grad_norm": 2.230486717462451, "language_loss": 0.75693011, "learning_rate": 3.9489995012474924e-06, "loss": 0.77895045, "num_input_tokens_seen": 25574155, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.078125, "step": 1200, "time_per_iteration": 2.5096075534820557 }, { "auxiliary_loss_clip": 0.0114182, "auxiliary_loss_mlp": 0.01056701, "balance_loss_clip": 1.02809119, "balance_loss_mlp": 1.03691626, "epoch": 0.07220802645423118, "flos": 23878775370240.0, "grad_norm": 2.228077108250191, "language_loss": 0.8275224, "learning_rate": 3.948914687454027e-06, "loss": 0.84950757, "num_input_tokens_seen": 25592735, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.046875, "step": 1201, "time_per_iteration": 2.4473869800567627 }, { "auxiliary_loss_clip": 0.01143448, "auxiliary_loss_mlp": 0.01054304, "balance_loss_clip": 1.02470398, "balance_loss_mlp": 1.03431582, "epoch": 0.07226814970689914, "flos": 19242658863360.0, "grad_norm": 2.3020796644932813, "language_loss": 0.68767619, "learning_rate": 3.948829804108807e-06, "loss": 0.70965374, "num_input_tokens_seen": 25611510, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.09375, "step": 1202, "time_per_iteration": 5.258546590805054 }, { "auxiliary_loss_clip": 0.01142037, "auxiliary_loss_mlp": 0.01052944, "balance_loss_clip": 1.02426255, "balance_loss_mlp": 1.03484988, "epoch": 0.07232827295956712, "flos": 19171017020160.0, "grad_norm": 2.6378065975950515, "language_loss": 0.87662745, "learning_rate": 3.948744851214865e-06, "loss": 0.89857721, "num_input_tokens_seen": 25629560, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.078125, "step": 1203, "time_per_iteration": 3.90230393409729 }, { "auxiliary_loss_clip": 0.01145265, "auxiliary_loss_mlp": 0.01051896, "balance_loss_clip": 1.02352452, "balance_loss_mlp": 1.03499961, "epoch": 0.07238839621223508, "flos": 17638783781760.0, "grad_norm": 1.9076009635896547, "language_loss": 0.78297997, "learning_rate": 3.948659828775233e-06, "loss": 0.80495155, "num_input_tokens_seen": 25648330, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.109375, "step": 1204, "time_per_iteration": 2.4286630153656006 }, { "auxiliary_loss_clip": 0.01141302, "auxiliary_loss_mlp": 0.01051983, "balance_loss_clip": 1.02534008, "balance_loss_mlp": 1.0341996, "epoch": 0.07244851946490305, "flos": 28291168684800.0, "grad_norm": 1.6565434766687437, "language_loss": 0.82000256, "learning_rate": 3.9485747367929436e-06, "loss": 0.8419354, "num_input_tokens_seen": 25669470, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0703125, "step": 1205, "time_per_iteration": 2.4822306632995605 }, { "auxiliary_loss_clip": 0.01144075, "auxiliary_loss_mlp": 0.01054803, "balance_loss_clip": 1.02466643, "balance_loss_mlp": 1.03717446, "epoch": 0.07250864271757101, "flos": 22563736899840.0, "grad_norm": 2.0059740658311545, "language_loss": 0.7660293, "learning_rate": 3.948489575271035e-06, "loss": 0.78801811, "num_input_tokens_seen": 25690470, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.0625, "step": 1206, "time_per_iteration": 2.473125457763672 }, { "auxiliary_loss_clip": 0.01143495, "auxiliary_loss_mlp": 0.01050002, "balance_loss_clip": 1.02232158, "balance_loss_mlp": 1.03645301, "epoch": 0.072568765970239, "flos": 21761328055680.0, "grad_norm": 2.467119437823108, "language_loss": 0.77418441, "learning_rate": 3.948404344212544e-06, "loss": 0.79611939, "num_input_tokens_seen": 25709205, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0703125, "step": 1207, "time_per_iteration": 2.404226779937744 }, { "auxiliary_loss_clip": 0.01140906, "auxiliary_loss_mlp": 0.01048589, "balance_loss_clip": 1.02285171, "balance_loss_mlp": 1.03607512, "epoch": 0.07262888922290696, "flos": 25518541196160.0, "grad_norm": 2.4181522111205536, "language_loss": 0.79696399, "learning_rate": 3.948319043620516e-06, "loss": 0.81885892, "num_input_tokens_seen": 25728485, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 1.046875, "step": 1208, "time_per_iteration": 2.4818239212036133 }, { "auxiliary_loss_clip": 0.01141341, "auxiliary_loss_mlp": 0.01045132, "balance_loss_clip": 1.01904941, "balance_loss_mlp": 1.03661633, "epoch": 0.07268901247557492, "flos": 21245626229760.0, "grad_norm": 2.396122437366724, "language_loss": 0.78514445, "learning_rate": 3.948233673497991e-06, "loss": 0.80700922, "num_input_tokens_seen": 25747730, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.046875, "step": 1209, "time_per_iteration": 2.4050676822662354 }, { "auxiliary_loss_clip": 0.0114268, "auxiliary_loss_mlp": 0.01055349, "balance_loss_clip": 1.0285393, "balance_loss_mlp": 1.03766227, "epoch": 0.0727491357282429, "flos": 25478251620480.0, "grad_norm": 2.593857403459947, "language_loss": 0.8194046, "learning_rate": 3.948148233848018e-06, "loss": 0.84138495, "num_input_tokens_seen": 25768050, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.0546875, "step": 1210, "time_per_iteration": 2.4727799892425537 }, { "auxiliary_loss_clip": 0.01140298, "auxiliary_loss_mlp": 0.01054591, "balance_loss_clip": 1.02724493, "balance_loss_mlp": 1.03657985, "epoch": 0.07280925898091087, "flos": 24461021980800.0, "grad_norm": 1.7600465661830798, "language_loss": 0.84463573, "learning_rate": 3.948062724673646e-06, "loss": 0.8665846, "num_input_tokens_seen": 25787985, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.0390625, "step": 1211, "time_per_iteration": 2.4353771209716797 }, { "auxiliary_loss_clip": 0.01137977, "auxiliary_loss_mlp": 0.01042181, "balance_loss_clip": 1.01647985, "balance_loss_mlp": 1.03354049, "epoch": 0.07286938223357883, "flos": 18287435531520.0, "grad_norm": 2.3416196011709323, "language_loss": 0.90241849, "learning_rate": 3.947977145977927e-06, "loss": 0.92422009, "num_input_tokens_seen": 25803620, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 1.046875, "step": 1212, "time_per_iteration": 2.371009111404419 }, { "auxiliary_loss_clip": 0.0114141, "auxiliary_loss_mlp": 0.01046974, "balance_loss_clip": 1.01823235, "balance_loss_mlp": 1.03664112, "epoch": 0.07292950548624681, "flos": 21213750291840.0, "grad_norm": 2.0401905048381983, "language_loss": 0.72653985, "learning_rate": 3.947891497763914e-06, "loss": 0.7484237, "num_input_tokens_seen": 25823315, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.046875, "step": 1213, "time_per_iteration": 2.4208662509918213 }, { "auxiliary_loss_clip": 0.01143135, "auxiliary_loss_mlp": 0.01044519, "balance_loss_clip": 1.01767325, "balance_loss_mlp": 1.03473377, "epoch": 0.07298962873891478, "flos": 24640929550080.0, "grad_norm": 1.7914246045079538, "language_loss": 0.84198576, "learning_rate": 3.947805780034664e-06, "loss": 0.86386228, "num_input_tokens_seen": 25842605, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.0859375, "step": 1214, "time_per_iteration": 2.448817253112793 }, { "auxiliary_loss_clip": 0.01146305, "auxiliary_loss_mlp": 0.01049021, "balance_loss_clip": 1.021662, "balance_loss_mlp": 1.03676653, "epoch": 0.07304975199158274, "flos": 27051542484480.0, "grad_norm": 2.7801557012053837, "language_loss": 0.84115255, "learning_rate": 3.947719992793236e-06, "loss": 0.86310577, "num_input_tokens_seen": 25863030, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.09375, "step": 1215, "time_per_iteration": 2.4775185585021973 }, { "auxiliary_loss_clip": 0.01141298, "auxiliary_loss_mlp": 0.01048689, "balance_loss_clip": 1.0214498, "balance_loss_mlp": 1.03517699, "epoch": 0.07310987524425071, "flos": 33548075809920.0, "grad_norm": 2.0020276426992583, "language_loss": 0.80888009, "learning_rate": 3.9476341360426924e-06, "loss": 0.83077991, "num_input_tokens_seen": 25888015, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.0625, "step": 1216, "time_per_iteration": 2.6004252433776855 }, { "auxiliary_loss_clip": 0.01145917, "auxiliary_loss_mlp": 0.01044281, "balance_loss_clip": 1.01780462, "balance_loss_mlp": 1.0389626, "epoch": 0.07316999849691869, "flos": 28109690104320.0, "grad_norm": 2.2619077943459716, "language_loss": 0.76420432, "learning_rate": 3.9475482097860955e-06, "loss": 0.78610629, "num_input_tokens_seen": 25908660, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0703125, "step": 1217, "time_per_iteration": 2.4730021953582764 }, { "auxiliary_loss_clip": 0.0113805, "auxiliary_loss_mlp": 0.01051466, "balance_loss_clip": 1.02534699, "balance_loss_mlp": 1.03639913, "epoch": 0.07323012174958665, "flos": 14391721560960.0, "grad_norm": 2.01205891769929, "language_loss": 0.86498004, "learning_rate": 3.947462214026512e-06, "loss": 0.88687515, "num_input_tokens_seen": 25927215, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.015625, "step": 1218, "time_per_iteration": 2.4093971252441406 }, { "auxiliary_loss_clip": 0.01141823, "auxiliary_loss_mlp": 0.01057249, "balance_loss_clip": 1.03045118, "balance_loss_mlp": 1.03431439, "epoch": 0.07329024500225462, "flos": 21615356194560.0, "grad_norm": 1.733751255105255, "language_loss": 0.86781025, "learning_rate": 3.947376148767013e-06, "loss": 0.88980097, "num_input_tokens_seen": 25945500, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.078125, "step": 1219, "time_per_iteration": 2.4079973697662354 }, { "auxiliary_loss_clip": 0.01139537, "auxiliary_loss_mlp": 0.0105074, "balance_loss_clip": 1.02428699, "balance_loss_mlp": 1.03407681, "epoch": 0.0733503682549226, "flos": 13223318267520.0, "grad_norm": 2.5594197446101523, "language_loss": 0.84308958, "learning_rate": 3.947290014010668e-06, "loss": 0.86499238, "num_input_tokens_seen": 25963105, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 1.0546875, "step": 1220, "time_per_iteration": 2.3945207595825195 }, { "auxiliary_loss_clip": 0.01140314, "auxiliary_loss_mlp": 0.01054931, "balance_loss_clip": 1.02772701, "balance_loss_mlp": 1.034724, "epoch": 0.07341049150759056, "flos": 20885915825280.0, "grad_norm": 4.876721254970153, "language_loss": 0.76878929, "learning_rate": 3.9472038097605516e-06, "loss": 0.7907418, "num_input_tokens_seen": 25981690, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.0546875, "step": 1221, "time_per_iteration": 2.4245474338531494 }, { "auxiliary_loss_clip": 0.01143765, "auxiliary_loss_mlp": 0.01053598, "balance_loss_clip": 1.02473783, "balance_loss_mlp": 1.03804398, "epoch": 0.07347061476025853, "flos": 15412721627520.0, "grad_norm": 2.9206585572953103, "language_loss": 0.91950142, "learning_rate": 3.94711753601974e-06, "loss": 0.94147503, "num_input_tokens_seen": 25999890, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0546875, "step": 1222, "time_per_iteration": 2.3849406242370605 }, { "auxiliary_loss_clip": 0.01148281, "auxiliary_loss_mlp": 0.01049477, "balance_loss_clip": 1.0222857, "balance_loss_mlp": 1.03966355, "epoch": 0.0735307380129265, "flos": 11108070368640.0, "grad_norm": 2.3887842126224474, "language_loss": 0.90904081, "learning_rate": 3.947031192791312e-06, "loss": 0.93101841, "num_input_tokens_seen": 26016445, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.0859375, "step": 1223, "time_per_iteration": 2.439429759979248 }, { "auxiliary_loss_clip": 0.01142685, "auxiliary_loss_mlp": 0.01054166, "balance_loss_clip": 1.02712965, "balance_loss_mlp": 1.03838503, "epoch": 0.07359086126559447, "flos": 23731267409280.0, "grad_norm": 2.124319472946957, "language_loss": 0.81972909, "learning_rate": 3.9469447800783485e-06, "loss": 0.84169757, "num_input_tokens_seen": 26036080, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.046875, "step": 1224, "time_per_iteration": 2.4450085163116455 }, { "auxiliary_loss_clip": 0.01140637, "auxiliary_loss_mlp": 0.01059567, "balance_loss_clip": 1.03000343, "balance_loss_mlp": 1.03389513, "epoch": 0.07365098451826244, "flos": 20992296337920.0, "grad_norm": 2.4175849250634416, "language_loss": 0.83184677, "learning_rate": 3.946858297883935e-06, "loss": 0.85384881, "num_input_tokens_seen": 26055805, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.0625, "step": 1225, "time_per_iteration": 2.4287068843841553 }, { "auxiliary_loss_clip": 0.01141, "auxiliary_loss_mlp": 0.01049981, "balance_loss_clip": 1.02364779, "balance_loss_mlp": 1.03529191, "epoch": 0.0737111077709304, "flos": 19932682440960.0, "grad_norm": 2.011532145546205, "language_loss": 0.90203059, "learning_rate": 3.946771746211156e-06, "loss": 0.9239403, "num_input_tokens_seen": 26073905, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 1.0546875, "step": 1226, "time_per_iteration": 2.3896918296813965 }, { "auxiliary_loss_clip": 0.01149355, "auxiliary_loss_mlp": 0.01046833, "balance_loss_clip": 1.01710188, "balance_loss_mlp": 1.03556871, "epoch": 0.07377123102359838, "flos": 16580601250560.0, "grad_norm": 2.5947243267518085, "language_loss": 0.76146984, "learning_rate": 3.946685125063101e-06, "loss": 0.78343177, "num_input_tokens_seen": 26091700, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.140625, "step": 1227, "time_per_iteration": 2.448315143585205 }, { "auxiliary_loss_clip": 0.01141519, "auxiliary_loss_mlp": 0.01054329, "balance_loss_clip": 1.02749527, "balance_loss_mlp": 1.03901958, "epoch": 0.07383135427626634, "flos": 28327338720000.0, "grad_norm": 1.6501807133980002, "language_loss": 0.85381699, "learning_rate": 3.9465984344428615e-06, "loss": 0.87577546, "num_input_tokens_seen": 26114105, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.0234375, "step": 1228, "time_per_iteration": 2.4647083282470703 }, { "auxiliary_loss_clip": 0.01144267, "auxiliary_loss_mlp": 0.0104705, "balance_loss_clip": 1.01946509, "balance_loss_mlp": 1.03646779, "epoch": 0.07389147752893431, "flos": 20046149959680.0, "grad_norm": 2.6306386628314633, "language_loss": 0.79686767, "learning_rate": 3.946511674353531e-06, "loss": 0.81878078, "num_input_tokens_seen": 26131165, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.078125, "step": 1229, "time_per_iteration": 2.4147233963012695 }, { "auxiliary_loss_clip": 0.01143158, "auxiliary_loss_mlp": 0.01053877, "balance_loss_clip": 1.0235498, "balance_loss_mlp": 1.03548288, "epoch": 0.07395160078160229, "flos": 18113148691200.0, "grad_norm": 2.6013210374683204, "language_loss": 0.78361106, "learning_rate": 3.9464248447982065e-06, "loss": 0.80558145, "num_input_tokens_seen": 26150040, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.078125, "step": 1230, "time_per_iteration": 2.377610445022583 }, { "auxiliary_loss_clip": 0.01139386, "auxiliary_loss_mlp": 0.01046705, "balance_loss_clip": 1.01846445, "balance_loss_mlp": 1.03478765, "epoch": 0.07401172403427025, "flos": 23585784307200.0, "grad_norm": 2.1179927963574534, "language_loss": 0.81063914, "learning_rate": 3.946337945779986e-06, "loss": 0.83249998, "num_input_tokens_seen": 26169380, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.046875, "step": 1231, "time_per_iteration": 2.456991672515869 }, { "auxiliary_loss_clip": 0.01143164, "auxiliary_loss_mlp": 0.01050602, "balance_loss_clip": 1.0215987, "balance_loss_mlp": 1.03441632, "epoch": 0.07407184728693822, "flos": 26358691086720.0, "grad_norm": 2.1789172637567487, "language_loss": 0.94616294, "learning_rate": 3.94625097730197e-06, "loss": 0.96810061, "num_input_tokens_seen": 26189420, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0859375, "step": 1232, "time_per_iteration": 2.4660682678222656 }, { "auxiliary_loss_clip": 0.01139915, "auxiliary_loss_mlp": 0.01049601, "balance_loss_clip": 1.02361298, "balance_loss_mlp": 1.03494775, "epoch": 0.0741319705396062, "flos": 22199348373120.0, "grad_norm": 1.8981820065754686, "language_loss": 0.81123012, "learning_rate": 3.946163939367264e-06, "loss": 0.83312529, "num_input_tokens_seen": 26209300, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 1.046875, "step": 1233, "time_per_iteration": 2.4436287879943848 }, { "auxiliary_loss_clip": 0.01146831, "auxiliary_loss_mlp": 0.01059435, "balance_loss_clip": 1.02751112, "balance_loss_mlp": 1.03701162, "epoch": 0.07419209379227416, "flos": 39198978161280.0, "grad_norm": 2.275438771641701, "language_loss": 0.70302069, "learning_rate": 3.9460768319789724e-06, "loss": 0.72508335, "num_input_tokens_seen": 26228110, "router_z_loss_clip": 0.3203125, "router_z_loss_mlp": 1.1015625, "step": 1234, "time_per_iteration": 2.5459272861480713 }, { "auxiliary_loss_clip": 0.01144109, "auxiliary_loss_mlp": 0.01052466, "balance_loss_clip": 1.024261, "balance_loss_mlp": 1.03747475, "epoch": 0.07425221704494213, "flos": 22780617465600.0, "grad_norm": 1.9535843715857266, "language_loss": 0.77411473, "learning_rate": 3.945989655140205e-06, "loss": 0.79608047, "num_input_tokens_seen": 26247020, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.0625, "step": 1235, "time_per_iteration": 2.465358018875122 }, { "auxiliary_loss_clip": 0.01142052, "auxiliary_loss_mlp": 0.01047285, "balance_loss_clip": 1.0195688, "balance_loss_mlp": 1.03596783, "epoch": 0.0743123402976101, "flos": 22271897911680.0, "grad_norm": 2.1740463529522676, "language_loss": 0.8237235, "learning_rate": 3.945902408854073e-06, "loss": 0.84561688, "num_input_tokens_seen": 26265750, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0625, "step": 1236, "time_per_iteration": 2.433403730392456 }, { "auxiliary_loss_clip": 0.01142526, "auxiliary_loss_mlp": 0.01054264, "balance_loss_clip": 1.02632093, "balance_loss_mlp": 1.03503644, "epoch": 0.07437246355027807, "flos": 29313739762560.0, "grad_norm": 2.189070570878333, "language_loss": 0.7565378, "learning_rate": 3.945815093123688e-06, "loss": 0.77850574, "num_input_tokens_seen": 26287905, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.078125, "step": 1237, "time_per_iteration": 2.515289545059204 }, { "auxiliary_loss_clip": 0.01141188, "auxiliary_loss_mlp": 0.01051467, "balance_loss_clip": 1.02457392, "balance_loss_mlp": 1.03395224, "epoch": 0.07443258680294604, "flos": 31943293032960.0, "grad_norm": 1.6945120127682058, "language_loss": 0.77806079, "learning_rate": 3.945727707952168e-06, "loss": 0.79998732, "num_input_tokens_seen": 26311795, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.0703125, "step": 1238, "time_per_iteration": 2.539829730987549 }, { "auxiliary_loss_clip": 0.01146616, "auxiliary_loss_mlp": 0.01052864, "balance_loss_clip": 1.02389622, "balance_loss_mlp": 1.03531945, "epoch": 0.074492710055614, "flos": 22674167130240.0, "grad_norm": 1.9737472469547397, "language_loss": 0.86791956, "learning_rate": 3.945640253342632e-06, "loss": 0.88991439, "num_input_tokens_seen": 26330330, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.1171875, "step": 1239, "time_per_iteration": 3.8863275051116943 }, { "auxiliary_loss_clip": 0.01143857, "auxiliary_loss_mlp": 0.01046158, "balance_loss_clip": 1.01618838, "balance_loss_mlp": 1.03571773, "epoch": 0.07455283330828198, "flos": 21283925857920.0, "grad_norm": 1.87822766520297, "language_loss": 0.88759482, "learning_rate": 3.9455527292981996e-06, "loss": 0.909495, "num_input_tokens_seen": 26348865, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.078125, "step": 1240, "time_per_iteration": 2.4443845748901367 }, { "auxiliary_loss_clip": 0.01144719, "auxiliary_loss_mlp": 0.01053837, "balance_loss_clip": 1.02510726, "balance_loss_mlp": 1.0371182, "epoch": 0.07461295656094995, "flos": 24387285456000.0, "grad_norm": 2.065054906840633, "language_loss": 0.89345175, "learning_rate": 3.945465135821995e-06, "loss": 0.91543734, "num_input_tokens_seen": 26368210, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.078125, "step": 1241, "time_per_iteration": 3.873291254043579 }, { "auxiliary_loss_clip": 0.01042863, "auxiliary_loss_mlp": 0.01006544, "balance_loss_clip": 1.00244331, "balance_loss_mlp": 1.0090704, "epoch": 0.07467307981361791, "flos": 62106608753280.0, "grad_norm": 0.889362179100009, "language_loss": 0.63114607, "learning_rate": 3.9453774729171435e-06, "loss": 0.65164018, "num_input_tokens_seen": 26424890, "router_z_loss_clip": 0.04101562, "router_z_loss_mlp": 0.33789062, "step": 1242, "time_per_iteration": 6.085329055786133 }, { "auxiliary_loss_clip": 0.01153597, "auxiliary_loss_mlp": 0.01054763, "balance_loss_clip": 1.0232439, "balance_loss_mlp": 1.03737187, "epoch": 0.07473320306628589, "flos": 24861999479040.0, "grad_norm": 2.764946515618584, "language_loss": 0.62563753, "learning_rate": 3.945289740586775e-06, "loss": 0.64772117, "num_input_tokens_seen": 26446405, "router_z_loss_clip": 0.31445312, "router_z_loss_mlp": 1.1640625, "step": 1243, "time_per_iteration": 2.450406074523926 }, { "auxiliary_loss_clip": 0.0114227, "auxiliary_loss_mlp": 0.01044545, "balance_loss_clip": 1.01800931, "balance_loss_mlp": 1.03594351, "epoch": 0.07479332631895386, "flos": 24896354123520.0, "grad_norm": 1.845577187094996, "language_loss": 0.76297748, "learning_rate": 3.945201938834018e-06, "loss": 0.78484559, "num_input_tokens_seen": 26466070, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0625, "step": 1244, "time_per_iteration": 2.448031187057495 }, { "auxiliary_loss_clip": 0.01147805, "auxiliary_loss_mlp": 0.01054192, "balance_loss_clip": 1.02586758, "balance_loss_mlp": 1.03727245, "epoch": 0.07485344957162182, "flos": 17814467076480.0, "grad_norm": 3.423545890676594, "language_loss": 0.69239521, "learning_rate": 3.945114067662009e-06, "loss": 0.71441513, "num_input_tokens_seen": 26479350, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.109375, "step": 1245, "time_per_iteration": 2.3707425594329834 }, { "auxiliary_loss_clip": 0.01144149, "auxiliary_loss_mlp": 0.01052193, "balance_loss_clip": 1.02332044, "balance_loss_mlp": 1.03658879, "epoch": 0.0749135728242898, "flos": 25009018680960.0, "grad_norm": 1.7770744627600272, "language_loss": 0.88667941, "learning_rate": 3.9450261270738815e-06, "loss": 0.90864277, "num_input_tokens_seen": 26498255, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.078125, "step": 1246, "time_per_iteration": 2.4962313175201416 }, { "auxiliary_loss_clip": 0.01152627, "auxiliary_loss_mlp": 0.01055953, "balance_loss_clip": 1.0230515, "balance_loss_mlp": 1.03740907, "epoch": 0.07497369607695777, "flos": 17821100234880.0, "grad_norm": 2.4445910492133485, "language_loss": 0.88317931, "learning_rate": 3.944938117072776e-06, "loss": 0.90526509, "num_input_tokens_seen": 26515375, "router_z_loss_clip": 0.33007812, "router_z_loss_mlp": 1.15625, "step": 1247, "time_per_iteration": 2.4028384685516357 }, { "auxiliary_loss_clip": 0.01141357, "auxiliary_loss_mlp": 0.01053357, "balance_loss_clip": 1.02529478, "balance_loss_mlp": 1.03495264, "epoch": 0.07503381932962573, "flos": 15120219323520.0, "grad_norm": 2.405659657463368, "language_loss": 0.64709055, "learning_rate": 3.944850037661831e-06, "loss": 0.66903764, "num_input_tokens_seen": 26533595, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.0625, "step": 1248, "time_per_iteration": 2.442790985107422 }, { "auxiliary_loss_clip": 0.01141608, "auxiliary_loss_mlp": 0.01052677, "balance_loss_clip": 1.02704656, "balance_loss_mlp": 1.03831017, "epoch": 0.0750939425822937, "flos": 12816091635840.0, "grad_norm": 2.2582700393608524, "language_loss": 0.74438941, "learning_rate": 3.944761888844191e-06, "loss": 0.76633227, "num_input_tokens_seen": 26549405, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 1.03125, "step": 1249, "time_per_iteration": 2.3979647159576416 }, { "auxiliary_loss_clip": 0.01147036, "auxiliary_loss_mlp": 0.01063925, "balance_loss_clip": 1.03414643, "balance_loss_mlp": 1.0371995, "epoch": 0.07515406583496168, "flos": 24205702141440.0, "grad_norm": 3.3231514265364357, "language_loss": 0.8245669, "learning_rate": 3.944673670623001e-06, "loss": 0.84667647, "num_input_tokens_seen": 26567200, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.09375, "step": 1250, "time_per_iteration": 2.469184398651123 }, { "auxiliary_loss_clip": 0.01144078, "auxiliary_loss_mlp": 0.01053375, "balance_loss_clip": 1.02534926, "balance_loss_mlp": 1.03903031, "epoch": 0.07521418908762964, "flos": 26686944489600.0, "grad_norm": 6.605391922579396, "language_loss": 0.669029, "learning_rate": 3.944585383001411e-06, "loss": 0.69100344, "num_input_tokens_seen": 26586190, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.0546875, "step": 1251, "time_per_iteration": 2.5036938190460205 }, { "auxiliary_loss_clip": 0.01039191, "auxiliary_loss_mlp": 0.0100688, "balance_loss_clip": 1.00211179, "balance_loss_mlp": 1.00628459, "epoch": 0.0752743123402976, "flos": 59091788096640.0, "grad_norm": 0.8883458310414257, "language_loss": 0.70415509, "learning_rate": 3.944497025982571e-06, "loss": 0.72461569, "num_input_tokens_seen": 26650710, "router_z_loss_clip": 0.04760742, "router_z_loss_mlp": 0.33007812, "step": 1252, "time_per_iteration": 3.095370054244995 }, { "auxiliary_loss_clip": 0.01142421, "auxiliary_loss_mlp": 0.01061465, "balance_loss_clip": 1.03253305, "balance_loss_mlp": 1.03514695, "epoch": 0.07533443559296558, "flos": 23475912658560.0, "grad_norm": 2.1062586387472946, "language_loss": 0.79992402, "learning_rate": 3.944408599569633e-06, "loss": 0.82196289, "num_input_tokens_seen": 26669000, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0703125, "step": 1253, "time_per_iteration": 2.4121005535125732 }, { "auxiliary_loss_clip": 0.01146671, "auxiliary_loss_mlp": 0.01057295, "balance_loss_clip": 1.0272783, "balance_loss_mlp": 1.03715527, "epoch": 0.07539455884563355, "flos": 20878270237440.0, "grad_norm": 2.9367962325269223, "language_loss": 0.9338783, "learning_rate": 3.9443201037657545e-06, "loss": 0.95591795, "num_input_tokens_seen": 26683075, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.09375, "step": 1254, "time_per_iteration": 2.419344663619995 }, { "auxiliary_loss_clip": 0.01139612, "auxiliary_loss_mlp": 0.01047563, "balance_loss_clip": 1.01966774, "balance_loss_mlp": 1.03530228, "epoch": 0.07545468209830151, "flos": 27671669786880.0, "grad_norm": 4.104633237380502, "language_loss": 0.87933367, "learning_rate": 3.944231538574092e-06, "loss": 0.90120542, "num_input_tokens_seen": 26701875, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.046875, "step": 1255, "time_per_iteration": 2.4580881595611572 }, { "auxiliary_loss_clip": 0.01140922, "auxiliary_loss_mlp": 0.01051102, "balance_loss_clip": 1.02168155, "balance_loss_mlp": 1.03676486, "epoch": 0.0755148053509695, "flos": 14136122430720.0, "grad_norm": 1.7522842390054543, "language_loss": 0.79969382, "learning_rate": 3.9441429039978086e-06, "loss": 0.82161403, "num_input_tokens_seen": 26719050, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.046875, "step": 1256, "time_per_iteration": 2.4150822162628174 }, { "auxiliary_loss_clip": 0.01040318, "auxiliary_loss_mlp": 0.01004824, "balance_loss_clip": 1.00017476, "balance_loss_mlp": 1.00681496, "epoch": 0.07557492860363746, "flos": 58232506780800.0, "grad_norm": 0.7715466436615287, "language_loss": 0.58031034, "learning_rate": 3.944054200040065e-06, "loss": 0.60076171, "num_input_tokens_seen": 26780650, "router_z_loss_clip": 0.04638672, "router_z_loss_mlp": 0.3359375, "step": 1257, "time_per_iteration": 3.169145107269287 }, { "auxiliary_loss_clip": 0.01143633, "auxiliary_loss_mlp": 0.01056884, "balance_loss_clip": 1.02752304, "balance_loss_mlp": 1.03853524, "epoch": 0.07563505185630542, "flos": 24643233699840.0, "grad_norm": 2.6403445214651766, "language_loss": 0.89664084, "learning_rate": 3.943965426704027e-06, "loss": 0.91864598, "num_input_tokens_seen": 26798725, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.0546875, "step": 1258, "time_per_iteration": 2.4530155658721924 }, { "auxiliary_loss_clip": 0.01142097, "auxiliary_loss_mlp": 0.01055937, "balance_loss_clip": 1.02931738, "balance_loss_mlp": 1.03846896, "epoch": 0.07569517510897339, "flos": 15522104517120.0, "grad_norm": 2.0561855104483153, "language_loss": 0.80861282, "learning_rate": 3.943876583992864e-06, "loss": 0.83059323, "num_input_tokens_seen": 26817005, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.03125, "step": 1259, "time_per_iteration": 2.41066837310791 }, { "auxiliary_loss_clip": 0.01141372, "auxiliary_loss_mlp": 0.01054204, "balance_loss_clip": 1.02615404, "balance_loss_mlp": 1.03635323, "epoch": 0.07575529836164137, "flos": 22927462110720.0, "grad_norm": 1.7786873652159558, "language_loss": 0.75696343, "learning_rate": 3.943787671909746e-06, "loss": 0.7789191, "num_input_tokens_seen": 26836655, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.046875, "step": 1260, "time_per_iteration": 2.4653754234313965 }, { "auxiliary_loss_clip": 0.01142378, "auxiliary_loss_mlp": 0.0105818, "balance_loss_clip": 1.0282104, "balance_loss_mlp": 1.03585958, "epoch": 0.07581542161430933, "flos": 19499410068480.0, "grad_norm": 2.2183576565645375, "language_loss": 0.84589267, "learning_rate": 3.943698690457846e-06, "loss": 0.86789823, "num_input_tokens_seen": 26854925, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.0625, "step": 1261, "time_per_iteration": 2.437059164047241 }, { "auxiliary_loss_clip": 0.0114615, "auxiliary_loss_mlp": 0.01060723, "balance_loss_clip": 1.03254223, "balance_loss_mlp": 1.03722024, "epoch": 0.0758755448669773, "flos": 24972290064000.0, "grad_norm": 1.8577573636488671, "language_loss": 0.83029902, "learning_rate": 3.943609639640339e-06, "loss": 0.85236776, "num_input_tokens_seen": 26876170, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.0859375, "step": 1262, "time_per_iteration": 2.51898455619812 }, { "auxiliary_loss_clip": 0.01139744, "auxiliary_loss_mlp": 0.01054691, "balance_loss_clip": 1.02629542, "balance_loss_mlp": 1.03495586, "epoch": 0.07593566811964528, "flos": 22746856314240.0, "grad_norm": 3.162190663494559, "language_loss": 0.82544661, "learning_rate": 3.943520519460405e-06, "loss": 0.84739101, "num_input_tokens_seen": 26895005, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.046875, "step": 1263, "time_per_iteration": 2.421804189682007 }, { "auxiliary_loss_clip": 0.01144962, "auxiliary_loss_mlp": 0.01046337, "balance_loss_clip": 1.01865637, "balance_loss_mlp": 1.03497577, "epoch": 0.07599579137231324, "flos": 23111279752320.0, "grad_norm": 2.6172494042949146, "language_loss": 0.76007628, "learning_rate": 3.943431329921221e-06, "loss": 0.78198922, "num_input_tokens_seen": 26913930, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.1015625, "step": 1264, "time_per_iteration": 2.436509847640991 }, { "auxiliary_loss_clip": 0.01143242, "auxiliary_loss_mlp": 0.0105781, "balance_loss_clip": 1.02933121, "balance_loss_mlp": 1.03629994, "epoch": 0.07605591462498121, "flos": 14501174273280.0, "grad_norm": 2.5946179933620526, "language_loss": 0.8096326, "learning_rate": 3.943342071025974e-06, "loss": 0.8316431, "num_input_tokens_seen": 26931485, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.0703125, "step": 1265, "time_per_iteration": 2.3973848819732666 }, { "auxiliary_loss_clip": 0.01144294, "auxiliary_loss_mlp": 0.01049988, "balance_loss_clip": 1.02011418, "balance_loss_mlp": 1.03644371, "epoch": 0.07611603787764919, "flos": 23513060211840.0, "grad_norm": 3.287075522816002, "language_loss": 0.65693021, "learning_rate": 3.9432527427778455e-06, "loss": 0.678873, "num_input_tokens_seen": 26951670, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.078125, "step": 1266, "time_per_iteration": 2.455113172531128 }, { "auxiliary_loss_clip": 0.0114309, "auxiliary_loss_mlp": 0.01054833, "balance_loss_clip": 1.02599609, "balance_loss_mlp": 1.03630745, "epoch": 0.07617616113031715, "flos": 21506112950400.0, "grad_norm": 2.377862188966611, "language_loss": 0.79258627, "learning_rate": 3.943163345180026e-06, "loss": 0.81456548, "num_input_tokens_seen": 26970335, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0625, "step": 1267, "time_per_iteration": 2.5339574813842773 }, { "auxiliary_loss_clip": 0.0114187, "auxiliary_loss_mlp": 0.01043139, "balance_loss_clip": 1.01756895, "balance_loss_mlp": 1.03486967, "epoch": 0.07623628438298512, "flos": 14572327357440.0, "grad_norm": 2.5288995682024065, "language_loss": 0.72980249, "learning_rate": 3.9430738782357054e-06, "loss": 0.75165266, "num_input_tokens_seen": 26986025, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 1.0703125, "step": 1268, "time_per_iteration": 2.394282579421997 }, { "auxiliary_loss_clip": 0.01143821, "auxiliary_loss_mlp": 0.01048508, "balance_loss_clip": 1.02023184, "balance_loss_mlp": 1.03516436, "epoch": 0.07629640763565308, "flos": 14719521116160.0, "grad_norm": 2.747619032322646, "language_loss": 0.82369566, "learning_rate": 3.9429843419480755e-06, "loss": 0.84561896, "num_input_tokens_seen": 27004045, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.0859375, "step": 1269, "time_per_iteration": 2.429852247238159 }, { "auxiliary_loss_clip": 0.01141087, "auxiliary_loss_mlp": 0.01050495, "balance_loss_clip": 1.02332711, "balance_loss_mlp": 1.03607571, "epoch": 0.07635653088832106, "flos": 14902047037440.0, "grad_norm": 2.3775327902069257, "language_loss": 0.88504201, "learning_rate": 3.942894736320334e-06, "loss": 0.90695786, "num_input_tokens_seen": 27022070, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.046875, "step": 1270, "time_per_iteration": 2.413921356201172 }, { "auxiliary_loss_clip": 0.01145719, "auxiliary_loss_mlp": 0.01053373, "balance_loss_clip": 1.02552581, "balance_loss_mlp": 1.03685999, "epoch": 0.07641665414098903, "flos": 26650355518080.0, "grad_norm": 2.3193044847198054, "language_loss": 0.71426392, "learning_rate": 3.942805061355676e-06, "loss": 0.73625481, "num_input_tokens_seen": 27041755, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0859375, "step": 1271, "time_per_iteration": 2.488715410232544 }, { "auxiliary_loss_clip": 0.01140245, "auxiliary_loss_mlp": 0.01051859, "balance_loss_clip": 1.02447629, "balance_loss_mlp": 1.03839946, "epoch": 0.07647677739365699, "flos": 25191614424960.0, "grad_norm": 1.6024138693327201, "language_loss": 0.82551324, "learning_rate": 3.9427153170573026e-06, "loss": 0.8474344, "num_input_tokens_seen": 27061540, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.015625, "step": 1272, "time_per_iteration": 2.4781711101531982 }, { "auxiliary_loss_clip": 0.01141554, "auxiliary_loss_mlp": 0.01053515, "balance_loss_clip": 1.02572775, "balance_loss_mlp": 1.0339067, "epoch": 0.07653690064632497, "flos": 20557103840640.0, "grad_norm": 4.691640888023605, "language_loss": 0.7996034, "learning_rate": 3.9426255034284174e-06, "loss": 0.82155412, "num_input_tokens_seen": 27081395, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.078125, "step": 1273, "time_per_iteration": 2.424798011779785 }, { "auxiliary_loss_clip": 0.01141954, "auxiliary_loss_mlp": 0.01055519, "balance_loss_clip": 1.0280174, "balance_loss_mlp": 1.03463483, "epoch": 0.07659702389899294, "flos": 22268336952960.0, "grad_norm": 2.39038723505383, "language_loss": 0.81201237, "learning_rate": 3.942535620472224e-06, "loss": 0.83398712, "num_input_tokens_seen": 27101175, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.078125, "step": 1274, "time_per_iteration": 2.4857890605926514 }, { "auxiliary_loss_clip": 0.01144799, "auxiliary_loss_mlp": 0.01058775, "balance_loss_clip": 1.03149986, "balance_loss_mlp": 1.03649175, "epoch": 0.0766571471516609, "flos": 32634713064960.0, "grad_norm": 2.102679445046312, "language_loss": 0.73003268, "learning_rate": 3.942445668191932e-06, "loss": 0.7520684, "num_input_tokens_seen": 27124505, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.078125, "step": 1275, "time_per_iteration": 2.533172845840454 }, { "auxiliary_loss_clip": 0.01145421, "auxiliary_loss_mlp": 0.01054721, "balance_loss_clip": 1.0252409, "balance_loss_mlp": 1.03737283, "epoch": 0.07671727040432888, "flos": 15266505386880.0, "grad_norm": 2.224435838407383, "language_loss": 0.79420996, "learning_rate": 3.94235564659075e-06, "loss": 0.81621134, "num_input_tokens_seen": 27140960, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.078125, "step": 1276, "time_per_iteration": 2.540464162826538 }, { "auxiliary_loss_clip": 0.0114624, "auxiliary_loss_mlp": 0.01051885, "balance_loss_clip": 1.0239898, "balance_loss_mlp": 1.03762674, "epoch": 0.07677739365699685, "flos": 28182833136000.0, "grad_norm": 2.2290197674949424, "language_loss": 0.59222054, "learning_rate": 3.942265555671892e-06, "loss": 0.61420172, "num_input_tokens_seen": 27160985, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.0859375, "step": 1277, "time_per_iteration": 2.5006768703460693 }, { "auxiliary_loss_clip": 0.01145807, "auxiliary_loss_mlp": 0.01057558, "balance_loss_clip": 1.02946019, "balance_loss_mlp": 1.03459835, "epoch": 0.07683751690966481, "flos": 18295150942080.0, "grad_norm": 3.6354157239331477, "language_loss": 0.75029022, "learning_rate": 3.942175395438572e-06, "loss": 0.77232379, "num_input_tokens_seen": 27178390, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.109375, "step": 1278, "time_per_iteration": 3.87268328666687 }, { "auxiliary_loss_clip": 0.01136778, "auxiliary_loss_mlp": 0.01051673, "balance_loss_clip": 1.02626991, "balance_loss_mlp": 1.03386188, "epoch": 0.07689764016233278, "flos": 21980024012160.0, "grad_norm": 2.747090015732431, "language_loss": 0.88341421, "learning_rate": 3.942085165894009e-06, "loss": 0.90529871, "num_input_tokens_seen": 27197505, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 1.03125, "step": 1279, "time_per_iteration": 2.445218086242676 }, { "auxiliary_loss_clip": 0.01139971, "auxiliary_loss_mlp": 0.01048246, "balance_loss_clip": 1.020661, "balance_loss_mlp": 1.03736174, "epoch": 0.07695776341500075, "flos": 22234924915200.0, "grad_norm": 2.417511747291879, "language_loss": 0.82531738, "learning_rate": 3.9419948670414206e-06, "loss": 0.84719956, "num_input_tokens_seen": 27214260, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0234375, "step": 1280, "time_per_iteration": 2.472209930419922 }, { "auxiliary_loss_clip": 0.01140022, "auxiliary_loss_mlp": 0.01056315, "balance_loss_clip": 1.02799058, "balance_loss_mlp": 1.0359323, "epoch": 0.07701788666766872, "flos": 16142825312640.0, "grad_norm": 3.092359493923539, "language_loss": 0.75768244, "learning_rate": 3.941904498884032e-06, "loss": 0.7796458, "num_input_tokens_seen": 27232525, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.0390625, "step": 1281, "time_per_iteration": 4.002227067947388 }, { "auxiliary_loss_clip": 0.01144663, "auxiliary_loss_mlp": 0.0105056, "balance_loss_clip": 1.02142501, "balance_loss_mlp": 1.03378582, "epoch": 0.07707800992033668, "flos": 19462053047040.0, "grad_norm": 3.3006494765725862, "language_loss": 0.74827677, "learning_rate": 3.941814061425067e-06, "loss": 0.77022898, "num_input_tokens_seen": 27249800, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.109375, "step": 1282, "time_per_iteration": 3.923555374145508 }, { "auxiliary_loss_clip": 0.01143046, "auxiliary_loss_mlp": 0.01054118, "balance_loss_clip": 1.02739167, "balance_loss_mlp": 1.03525567, "epoch": 0.07713813317300466, "flos": 18989259148800.0, "grad_norm": 2.6287145402917154, "language_loss": 0.83850062, "learning_rate": 3.941723554667752e-06, "loss": 0.86047232, "num_input_tokens_seen": 27268895, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.078125, "step": 1283, "time_per_iteration": 2.483353853225708 }, { "auxiliary_loss_clip": 0.01142422, "auxiliary_loss_mlp": 0.01056764, "balance_loss_clip": 1.02675915, "balance_loss_mlp": 1.03554845, "epoch": 0.07719825642567263, "flos": 18112974134400.0, "grad_norm": 4.00045806440098, "language_loss": 0.74790585, "learning_rate": 3.941632978615318e-06, "loss": 0.7698977, "num_input_tokens_seen": 27288180, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.0703125, "step": 1284, "time_per_iteration": 2.4336962699890137 }, { "auxiliary_loss_clip": 0.01139409, "auxiliary_loss_mlp": 0.01058026, "balance_loss_clip": 1.03098965, "balance_loss_mlp": 1.03460002, "epoch": 0.0772583796783406, "flos": 42192780312960.0, "grad_norm": 1.9606313210845743, "language_loss": 0.76300985, "learning_rate": 3.941542333270999e-06, "loss": 0.78498423, "num_input_tokens_seen": 27311815, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.046875, "step": 1285, "time_per_iteration": 2.6642353534698486 }, { "auxiliary_loss_clip": 0.01146061, "auxiliary_loss_mlp": 0.01060958, "balance_loss_clip": 1.03332567, "balance_loss_mlp": 1.0383873, "epoch": 0.07731850293100857, "flos": 24752546766720.0, "grad_norm": 2.016961446663883, "language_loss": 0.83767694, "learning_rate": 3.9414516186380275e-06, "loss": 0.85974705, "num_input_tokens_seen": 27331890, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.078125, "step": 1286, "time_per_iteration": 2.4510388374328613 }, { "auxiliary_loss_clip": 0.01143516, "auxiliary_loss_mlp": 0.01049856, "balance_loss_clip": 1.02274823, "balance_loss_mlp": 1.03486085, "epoch": 0.07737862618367654, "flos": 17564942522880.0, "grad_norm": 2.2536591918310656, "language_loss": 0.770509, "learning_rate": 3.941360834719641e-06, "loss": 0.79244268, "num_input_tokens_seen": 27348320, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.0859375, "step": 1287, "time_per_iteration": 2.44064998626709 }, { "auxiliary_loss_clip": 0.01138396, "auxiliary_loss_mlp": 0.01049698, "balance_loss_clip": 1.0231142, "balance_loss_mlp": 1.03492832, "epoch": 0.0774387494363445, "flos": 25626038872320.0, "grad_norm": 2.0322938983618326, "language_loss": 0.84395438, "learning_rate": 3.941269981519081e-06, "loss": 0.86583531, "num_input_tokens_seen": 27367670, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.03125, "step": 1288, "time_per_iteration": 2.4835731983184814 }, { "auxiliary_loss_clip": 0.01138414, "auxiliary_loss_mlp": 0.01050989, "balance_loss_clip": 1.02489436, "balance_loss_mlp": 1.03338683, "epoch": 0.07749887268901248, "flos": 12239046817920.0, "grad_norm": 2.157235414053665, "language_loss": 0.85084462, "learning_rate": 3.941179059039589e-06, "loss": 0.87273872, "num_input_tokens_seen": 27385485, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.046875, "step": 1289, "time_per_iteration": 2.520219564437866 }, { "auxiliary_loss_clip": 0.01140713, "auxiliary_loss_mlp": 0.01047597, "balance_loss_clip": 1.01990509, "balance_loss_mlp": 1.03482342, "epoch": 0.07755899594168045, "flos": 25080590701440.0, "grad_norm": 1.9354442156770693, "language_loss": 0.85018635, "learning_rate": 3.941088067284409e-06, "loss": 0.87206948, "num_input_tokens_seen": 27405110, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0625, "step": 1290, "time_per_iteration": 2.470550060272217 }, { "auxiliary_loss_clip": 0.01136217, "auxiliary_loss_mlp": 0.01061076, "balance_loss_clip": 1.03229904, "balance_loss_mlp": 1.03412962, "epoch": 0.07761911919434841, "flos": 14245540231680.0, "grad_norm": 2.307083349569191, "language_loss": 0.90523207, "learning_rate": 3.9409970062567895e-06, "loss": 0.92720503, "num_input_tokens_seen": 27422855, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0234375, "step": 1291, "time_per_iteration": 2.440746784210205 }, { "auxiliary_loss_clip": 0.01041045, "auxiliary_loss_mlp": 0.01016423, "balance_loss_clip": 1.01241791, "balance_loss_mlp": 1.00740957, "epoch": 0.07767924244701638, "flos": 67233463597440.0, "grad_norm": 0.8793135900668437, "language_loss": 0.65063083, "learning_rate": 3.94090587595998e-06, "loss": 0.67120552, "num_input_tokens_seen": 27487190, "router_z_loss_clip": 0.04003906, "router_z_loss_mlp": 0.3359375, "step": 1292, "time_per_iteration": 3.168759822845459 }, { "auxiliary_loss_clip": 0.0114126, "auxiliary_loss_mlp": 0.01046543, "balance_loss_clip": 1.01861262, "balance_loss_mlp": 1.03376663, "epoch": 0.07773936569968436, "flos": 28549316344320.0, "grad_norm": 2.045054047714249, "language_loss": 0.87551838, "learning_rate": 3.940814676397232e-06, "loss": 0.89739639, "num_input_tokens_seen": 27510465, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.0703125, "step": 1293, "time_per_iteration": 2.5071475505828857 }, { "auxiliary_loss_clip": 0.01144298, "auxiliary_loss_mlp": 0.01053189, "balance_loss_clip": 1.02447128, "balance_loss_mlp": 1.03891051, "epoch": 0.07779948895235232, "flos": 27489039131520.0, "grad_norm": 2.1716084836194733, "language_loss": 0.84672004, "learning_rate": 3.940723407571801e-06, "loss": 0.8686949, "num_input_tokens_seen": 27528645, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.0546875, "step": 1294, "time_per_iteration": 2.5061745643615723 }, { "auxiliary_loss_clip": 0.01141525, "auxiliary_loss_mlp": 0.01048977, "balance_loss_clip": 1.02082014, "balance_loss_mlp": 1.03697085, "epoch": 0.07785961220502029, "flos": 18222322112640.0, "grad_norm": 2.381994821509741, "language_loss": 0.79361206, "learning_rate": 3.9406320694869425e-06, "loss": 0.81551707, "num_input_tokens_seen": 27546165, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.046875, "step": 1295, "time_per_iteration": 2.407604694366455 }, { "auxiliary_loss_clip": 0.01140814, "auxiliary_loss_mlp": 0.01047431, "balance_loss_clip": 1.02001262, "balance_loss_mlp": 1.03543675, "epoch": 0.07791973545768827, "flos": 24607063664640.0, "grad_norm": 2.319744815127164, "language_loss": 0.87795794, "learning_rate": 3.940540662145918e-06, "loss": 0.89984035, "num_input_tokens_seen": 27566520, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.0546875, "step": 1296, "time_per_iteration": 2.4614198207855225 }, { "auxiliary_loss_clip": 0.01141737, "auxiliary_loss_mlp": 0.01049657, "balance_loss_clip": 1.02112985, "balance_loss_mlp": 1.03465438, "epoch": 0.07797985871035623, "flos": 14281221507840.0, "grad_norm": 3.0816995918719856, "language_loss": 0.96446133, "learning_rate": 3.940449185551989e-06, "loss": 0.98637521, "num_input_tokens_seen": 27581960, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.0703125, "step": 1297, "time_per_iteration": 2.4358835220336914 }, { "auxiliary_loss_clip": 0.01140842, "auxiliary_loss_mlp": 0.01050633, "balance_loss_clip": 1.0230006, "balance_loss_mlp": 1.03406048, "epoch": 0.0780399819630242, "flos": 26609367715200.0, "grad_norm": 2.114068414625776, "language_loss": 0.7612232, "learning_rate": 3.94035763970842e-06, "loss": 0.78313792, "num_input_tokens_seen": 27601415, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0703125, "step": 1298, "time_per_iteration": 2.452118396759033 }, { "auxiliary_loss_clip": 0.0114226, "auxiliary_loss_mlp": 0.01051076, "balance_loss_clip": 1.02457619, "balance_loss_mlp": 1.03616405, "epoch": 0.07810010521569218, "flos": 21833458657920.0, "grad_norm": 1.7562543677043454, "language_loss": 0.80491579, "learning_rate": 3.940266024618478e-06, "loss": 0.82684916, "num_input_tokens_seen": 27621490, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0625, "step": 1299, "time_per_iteration": 2.448579788208008 }, { "auxiliary_loss_clip": 0.01138035, "auxiliary_loss_mlp": 0.01050805, "balance_loss_clip": 1.0229218, "balance_loss_mlp": 1.03488827, "epoch": 0.07816022846836014, "flos": 25080101942400.0, "grad_norm": 2.063325759243455, "language_loss": 0.85981327, "learning_rate": 3.940174340285432e-06, "loss": 0.88170165, "num_input_tokens_seen": 27640600, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.03125, "step": 1300, "time_per_iteration": 2.4218318462371826 }, { "auxiliary_loss_clip": 0.0114285, "auxiliary_loss_mlp": 0.01046773, "balance_loss_clip": 1.01909256, "balance_loss_mlp": 1.03690553, "epoch": 0.0782203517210281, "flos": 40915901825280.0, "grad_norm": 2.2657372489361336, "language_loss": 0.71694589, "learning_rate": 3.940082586712555e-06, "loss": 0.73884213, "num_input_tokens_seen": 27663070, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0625, "step": 1301, "time_per_iteration": 2.6418845653533936 }, { "auxiliary_loss_clip": 0.01147725, "auxiliary_loss_mlp": 0.01058281, "balance_loss_clip": 1.03042173, "balance_loss_mlp": 1.03863072, "epoch": 0.07828047497369607, "flos": 41170418703360.0, "grad_norm": 1.5101327011812837, "language_loss": 0.7031014, "learning_rate": 3.939990763903122e-06, "loss": 0.72516143, "num_input_tokens_seen": 27686425, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.0859375, "step": 1302, "time_per_iteration": 2.741868734359741 }, { "auxiliary_loss_clip": 0.01142976, "auxiliary_loss_mlp": 0.01052096, "balance_loss_clip": 1.02437949, "balance_loss_mlp": 1.03674889, "epoch": 0.07834059822636405, "flos": 23507160192000.0, "grad_norm": 1.988705315306982, "language_loss": 0.82179976, "learning_rate": 3.939898871860407e-06, "loss": 0.84375048, "num_input_tokens_seen": 27704900, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0625, "step": 1303, "time_per_iteration": 2.4571609497070312 }, { "auxiliary_loss_clip": 0.01142584, "auxiliary_loss_mlp": 0.01059815, "balance_loss_clip": 1.03139567, "balance_loss_mlp": 1.03558779, "epoch": 0.07840072147903202, "flos": 20192854959360.0, "grad_norm": 2.581953731053822, "language_loss": 0.74705011, "learning_rate": 3.939806910587693e-06, "loss": 0.76907408, "num_input_tokens_seen": 27724890, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.0703125, "step": 1304, "time_per_iteration": 2.433095932006836 }, { "auxiliary_loss_clip": 0.01146743, "auxiliary_loss_mlp": 0.01056951, "balance_loss_clip": 1.02690983, "balance_loss_mlp": 1.04023051, "epoch": 0.07846084473169998, "flos": 21359757064320.0, "grad_norm": 1.8012610446750759, "language_loss": 0.76330793, "learning_rate": 3.9397148800882595e-06, "loss": 0.78534484, "num_input_tokens_seen": 27743115, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.0625, "step": 1305, "time_per_iteration": 2.4251163005828857 }, { "auxiliary_loss_clip": 0.01145606, "auxiliary_loss_mlp": 0.01058712, "balance_loss_clip": 1.02882612, "balance_loss_mlp": 1.03599501, "epoch": 0.07852096798436796, "flos": 25409786711040.0, "grad_norm": 1.709212760868719, "language_loss": 0.84957409, "learning_rate": 3.939622780365391e-06, "loss": 0.87161732, "num_input_tokens_seen": 27763570, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.09375, "step": 1306, "time_per_iteration": 2.4500534534454346 }, { "auxiliary_loss_clip": 0.01141706, "auxiliary_loss_mlp": 0.01042407, "balance_loss_clip": 1.01584721, "balance_loss_mlp": 1.03744817, "epoch": 0.07858109123703592, "flos": 24570335047680.0, "grad_norm": 2.6426056403295197, "language_loss": 0.9069171, "learning_rate": 3.939530611422375e-06, "loss": 0.92875826, "num_input_tokens_seen": 27780030, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0390625, "step": 1307, "time_per_iteration": 2.4576053619384766 }, { "auxiliary_loss_clip": 0.01142213, "auxiliary_loss_mlp": 0.01051842, "balance_loss_clip": 1.02225399, "balance_loss_mlp": 1.03556371, "epoch": 0.07864121448970389, "flos": 20697978643200.0, "grad_norm": 1.8719671173611063, "language_loss": 0.8353464, "learning_rate": 3.939438373262501e-06, "loss": 0.85728693, "num_input_tokens_seen": 27796225, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.0625, "step": 1308, "time_per_iteration": 2.3935189247131348 }, { "auxiliary_loss_clip": 0.01139905, "auxiliary_loss_mlp": 0.0104803, "balance_loss_clip": 1.02133918, "balance_loss_mlp": 1.03627121, "epoch": 0.07870133774237187, "flos": 22965412625280.0, "grad_norm": 1.4422409899536226, "language_loss": 0.77097666, "learning_rate": 3.93934606588906e-06, "loss": 0.79285604, "num_input_tokens_seen": 27815975, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.0390625, "step": 1309, "time_per_iteration": 2.4520504474639893 }, { "auxiliary_loss_clip": 0.01148205, "auxiliary_loss_mlp": 0.01060707, "balance_loss_clip": 1.03188252, "balance_loss_mlp": 1.03713357, "epoch": 0.07876146099503983, "flos": 18841855921920.0, "grad_norm": 2.109083590132941, "language_loss": 0.80204201, "learning_rate": 3.939253689305346e-06, "loss": 0.82413113, "num_input_tokens_seen": 27832255, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.109375, "step": 1310, "time_per_iteration": 2.374415397644043 }, { "auxiliary_loss_clip": 0.01138736, "auxiliary_loss_mlp": 0.01049991, "balance_loss_clip": 1.02394342, "balance_loss_mlp": 1.03848791, "epoch": 0.0788215842477078, "flos": 23804654820480.0, "grad_norm": 1.6889419547689029, "language_loss": 0.72608209, "learning_rate": 3.939161243514657e-06, "loss": 0.74796939, "num_input_tokens_seen": 27852180, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.0078125, "step": 1311, "time_per_iteration": 2.4493556022644043 }, { "auxiliary_loss_clip": 0.01143532, "auxiliary_loss_mlp": 0.01063003, "balance_loss_clip": 1.03531039, "balance_loss_mlp": 1.03952634, "epoch": 0.07888170750037576, "flos": 21578837045760.0, "grad_norm": 3.1876067717240892, "language_loss": 0.85806346, "learning_rate": 3.939068728520291e-06, "loss": 0.88012886, "num_input_tokens_seen": 27871435, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0390625, "step": 1312, "time_per_iteration": 2.4188222885131836 }, { "auxiliary_loss_clip": 0.01141501, "auxiliary_loss_mlp": 0.01058017, "balance_loss_clip": 1.03069377, "balance_loss_mlp": 1.03968775, "epoch": 0.07894183075304374, "flos": 19863833506560.0, "grad_norm": 2.420747931174189, "language_loss": 0.81749922, "learning_rate": 3.938976144325549e-06, "loss": 0.83949447, "num_input_tokens_seen": 27890625, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.015625, "step": 1313, "time_per_iteration": 2.436534881591797 }, { "auxiliary_loss_clip": 0.01149076, "auxiliary_loss_mlp": 0.01059925, "balance_loss_clip": 1.02850103, "balance_loss_mlp": 1.03694129, "epoch": 0.07900195400571171, "flos": 16142546021760.0, "grad_norm": 2.417584199048771, "language_loss": 0.72915339, "learning_rate": 3.9388834909337375e-06, "loss": 0.75124347, "num_input_tokens_seen": 27906530, "router_z_loss_clip": 0.31445312, "router_z_loss_mlp": 1.1171875, "step": 1314, "time_per_iteration": 2.3856356143951416 }, { "auxiliary_loss_clip": 0.01141154, "auxiliary_loss_mlp": 0.01053615, "balance_loss_clip": 1.02693546, "balance_loss_mlp": 1.03433251, "epoch": 0.07906207725837967, "flos": 23729347284480.0, "grad_norm": 1.614323519360908, "language_loss": 0.79576534, "learning_rate": 3.938790768348161e-06, "loss": 0.81771302, "num_input_tokens_seen": 27926725, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.0703125, "step": 1315, "time_per_iteration": 2.4572107791900635 }, { "auxiliary_loss_clip": 0.01140939, "auxiliary_loss_mlp": 0.01057704, "balance_loss_clip": 1.02779472, "balance_loss_mlp": 1.03500342, "epoch": 0.07912220051104765, "flos": 24314770828800.0, "grad_norm": 1.9746038132442256, "language_loss": 0.73879462, "learning_rate": 3.938697976572129e-06, "loss": 0.76078105, "num_input_tokens_seen": 27947875, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.0625, "step": 1316, "time_per_iteration": 2.4434773921966553 }, { "auxiliary_loss_clip": 0.01147625, "auxiliary_loss_mlp": 0.01056223, "balance_loss_clip": 1.02733898, "balance_loss_mlp": 1.03697991, "epoch": 0.07918232376371562, "flos": 18879038386560.0, "grad_norm": 3.399225985719132, "language_loss": 0.65351379, "learning_rate": 3.938605115608954e-06, "loss": 0.67555225, "num_input_tokens_seen": 27965040, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.1015625, "step": 1317, "time_per_iteration": 2.584772825241089 }, { "auxiliary_loss_clip": 0.01148, "auxiliary_loss_mlp": 0.01062528, "balance_loss_clip": 1.03260612, "balance_loss_mlp": 1.03737366, "epoch": 0.07924244701638358, "flos": 27375187587840.0, "grad_norm": 2.590912805255077, "language_loss": 0.7312218, "learning_rate": 3.938512185461948e-06, "loss": 0.75332707, "num_input_tokens_seen": 27985330, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.109375, "step": 1318, "time_per_iteration": 3.980745315551758 }, { "auxiliary_loss_clip": 0.01143958, "auxiliary_loss_mlp": 0.0105347, "balance_loss_clip": 1.02559924, "balance_loss_mlp": 1.03717411, "epoch": 0.07930257026905156, "flos": 25119134709120.0, "grad_norm": 1.67985434599967, "language_loss": 0.90111381, "learning_rate": 3.938419186134429e-06, "loss": 0.92308807, "num_input_tokens_seen": 28007615, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0625, "step": 1319, "time_per_iteration": 2.4653007984161377 }, { "auxiliary_loss_clip": 0.01142388, "auxiliary_loss_mlp": 0.01056527, "balance_loss_clip": 1.02764225, "balance_loss_mlp": 1.03432322, "epoch": 0.07936269352171953, "flos": 21833423746560.0, "grad_norm": 1.8351091050344135, "language_loss": 0.79586965, "learning_rate": 3.9383261176297155e-06, "loss": 0.81785882, "num_input_tokens_seen": 28027765, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.078125, "step": 1320, "time_per_iteration": 3.920444965362549 }, { "auxiliary_loss_clip": 0.01143597, "auxiliary_loss_mlp": 0.01056051, "balance_loss_clip": 1.02728558, "balance_loss_mlp": 1.03809261, "epoch": 0.07942281677438749, "flos": 16939124668800.0, "grad_norm": 2.9078392525385057, "language_loss": 0.69522524, "learning_rate": 3.938232979951129e-06, "loss": 0.71722168, "num_input_tokens_seen": 28044225, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.0546875, "step": 1321, "time_per_iteration": 5.2817018032073975 }, { "auxiliary_loss_clip": 0.01140461, "auxiliary_loss_mlp": 0.01058528, "balance_loss_clip": 1.03015637, "balance_loss_mlp": 1.03602624, "epoch": 0.07948294002705546, "flos": 18986012392320.0, "grad_norm": 2.1799705885269205, "language_loss": 0.84114683, "learning_rate": 3.938139773101993e-06, "loss": 0.86313665, "num_input_tokens_seen": 28062915, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.046875, "step": 1322, "time_per_iteration": 2.416301965713501 }, { "auxiliary_loss_clip": 0.01139827, "auxiliary_loss_mlp": 0.01054368, "balance_loss_clip": 1.0266279, "balance_loss_mlp": 1.03306556, "epoch": 0.07954306327972344, "flos": 21652364102400.0, "grad_norm": 2.3530515704260577, "language_loss": 0.90426469, "learning_rate": 3.938046497085634e-06, "loss": 0.92620659, "num_input_tokens_seen": 28082175, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0703125, "step": 1323, "time_per_iteration": 2.456897735595703 }, { "auxiliary_loss_clip": 0.01137344, "auxiliary_loss_mlp": 0.01053257, "balance_loss_clip": 1.02537346, "balance_loss_mlp": 1.0350616, "epoch": 0.0796031865323914, "flos": 23219196364800.0, "grad_norm": 1.726783845318455, "language_loss": 0.82554126, "learning_rate": 3.937953151905381e-06, "loss": 0.84744722, "num_input_tokens_seen": 28102645, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0234375, "step": 1324, "time_per_iteration": 2.426100730895996 }, { "auxiliary_loss_clip": 0.01141442, "auxiliary_loss_mlp": 0.01052664, "balance_loss_clip": 1.02341056, "balance_loss_mlp": 1.03469324, "epoch": 0.07966330978505937, "flos": 23293421648640.0, "grad_norm": 4.397938593012299, "language_loss": 0.79089087, "learning_rate": 3.937859737564564e-06, "loss": 0.81283194, "num_input_tokens_seen": 28122805, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.0703125, "step": 1325, "time_per_iteration": 2.5013134479522705 }, { "auxiliary_loss_clip": 0.01143631, "auxiliary_loss_mlp": 0.01060608, "balance_loss_clip": 1.03113937, "balance_loss_mlp": 1.03763437, "epoch": 0.07972343303772735, "flos": 18362952535680.0, "grad_norm": 2.3257959431325914, "language_loss": 0.88397908, "learning_rate": 3.937766254066519e-06, "loss": 0.9060216, "num_input_tokens_seen": 28140530, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.0625, "step": 1326, "time_per_iteration": 2.4037437438964844 }, { "auxiliary_loss_clip": 0.01137361, "auxiliary_loss_mlp": 0.01047272, "balance_loss_clip": 1.01801825, "balance_loss_mlp": 1.03456819, "epoch": 0.07978355629039531, "flos": 21761432789760.0, "grad_norm": 2.0451963149136407, "language_loss": 0.83130109, "learning_rate": 3.937672701414581e-06, "loss": 0.85314745, "num_input_tokens_seen": 28159640, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.03125, "step": 1327, "time_per_iteration": 2.4422802925109863 }, { "auxiliary_loss_clip": 0.01141986, "auxiliary_loss_mlp": 0.01053909, "balance_loss_clip": 1.02343893, "balance_loss_mlp": 1.03526497, "epoch": 0.07984367954306328, "flos": 18550331136000.0, "grad_norm": 2.109895088853339, "language_loss": 0.78819835, "learning_rate": 3.937579079612087e-06, "loss": 0.81015736, "num_input_tokens_seen": 28177050, "router_z_loss_clip": 0.3046875, "router_z_loss_mlp": 1.0625, "step": 1328, "time_per_iteration": 2.3991034030914307 }, { "auxiliary_loss_clip": 0.01143765, "auxiliary_loss_mlp": 0.01053721, "balance_loss_clip": 1.02444315, "balance_loss_mlp": 1.03603315, "epoch": 0.07990380279573125, "flos": 16903268835840.0, "grad_norm": 2.470183114375481, "language_loss": 0.7324903, "learning_rate": 3.9374853886623805e-06, "loss": 0.75446516, "num_input_tokens_seen": 28193245, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.078125, "step": 1329, "time_per_iteration": 2.444092035293579 }, { "auxiliary_loss_clip": 0.01138578, "auxiliary_loss_mlp": 0.0104502, "balance_loss_clip": 1.01756668, "balance_loss_mlp": 1.03349066, "epoch": 0.07996392604839922, "flos": 24097192035840.0, "grad_norm": 1.7260779122665668, "language_loss": 0.8116973, "learning_rate": 3.937391628568805e-06, "loss": 0.83353329, "num_input_tokens_seen": 28213570, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.046875, "step": 1330, "time_per_iteration": 2.4590070247650146 }, { "auxiliary_loss_clip": 0.01139686, "auxiliary_loss_mlp": 0.01050342, "balance_loss_clip": 1.02336502, "balance_loss_mlp": 1.03483081, "epoch": 0.08002404930106718, "flos": 14277974751360.0, "grad_norm": 5.953708099505188, "language_loss": 0.88954514, "learning_rate": 3.937297799334706e-06, "loss": 0.9114455, "num_input_tokens_seen": 28229980, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.046875, "step": 1331, "time_per_iteration": 2.418520212173462 }, { "auxiliary_loss_clip": 0.01143133, "auxiliary_loss_mlp": 0.01050024, "balance_loss_clip": 1.02038908, "balance_loss_mlp": 1.03411341, "epoch": 0.08008417255373516, "flos": 40404633742080.0, "grad_norm": 1.9768105749615845, "language_loss": 0.73450077, "learning_rate": 3.937203900963431e-06, "loss": 0.75643235, "num_input_tokens_seen": 28253840, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.09375, "step": 1332, "time_per_iteration": 2.569986343383789 }, { "auxiliary_loss_clip": 0.01139288, "auxiliary_loss_mlp": 0.01047808, "balance_loss_clip": 1.01986504, "balance_loss_mlp": 1.03435731, "epoch": 0.08014429580640313, "flos": 18477921242880.0, "grad_norm": 1.9158417669999368, "language_loss": 0.82308197, "learning_rate": 3.9371099334583315e-06, "loss": 0.84495294, "num_input_tokens_seen": 28271675, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.046875, "step": 1333, "time_per_iteration": 2.416245460510254 }, { "auxiliary_loss_clip": 0.0114136, "auxiliary_loss_mlp": 0.01049533, "balance_loss_clip": 1.0222342, "balance_loss_mlp": 1.03392458, "epoch": 0.0802044190590711, "flos": 22052398993920.0, "grad_norm": 2.22853006873138, "language_loss": 0.74815822, "learning_rate": 3.937015896822762e-06, "loss": 0.7700671, "num_input_tokens_seen": 28291850, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.078125, "step": 1334, "time_per_iteration": 2.415252685546875 }, { "auxiliary_loss_clip": 0.01139333, "auxiliary_loss_mlp": 0.01049732, "balance_loss_clip": 1.02207494, "balance_loss_mlp": 1.03632402, "epoch": 0.08026454231173906, "flos": 24570963452160.0, "grad_norm": 1.8146781566640344, "language_loss": 0.80229247, "learning_rate": 3.936921791060078e-06, "loss": 0.82418305, "num_input_tokens_seen": 28310780, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.03125, "step": 1335, "time_per_iteration": 2.476264476776123 }, { "auxiliary_loss_clip": 0.01042149, "auxiliary_loss_mlp": 0.01018921, "balance_loss_clip": 1.01470125, "balance_loss_mlp": 1.00973916, "epoch": 0.08032466556440704, "flos": 52579195545600.0, "grad_norm": 0.7418359836964853, "language_loss": 0.5600881, "learning_rate": 3.936827616173636e-06, "loss": 0.58069885, "num_input_tokens_seen": 28369985, "router_z_loss_clip": 0.04223633, "router_z_loss_mlp": 0.32421875, "step": 1336, "time_per_iteration": 3.1065118312835693 }, { "auxiliary_loss_clip": 0.01139751, "auxiliary_loss_mlp": 0.01056251, "balance_loss_clip": 1.02970338, "balance_loss_mlp": 1.0362134, "epoch": 0.080384788817075, "flos": 23841453260160.0, "grad_norm": 2.0979713849682615, "language_loss": 0.67448568, "learning_rate": 3.9367333721668006e-06, "loss": 0.6964457, "num_input_tokens_seen": 28388670, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.03125, "step": 1337, "time_per_iteration": 2.450666904449463 }, { "auxiliary_loss_clip": 0.01140381, "auxiliary_loss_mlp": 0.01054173, "balance_loss_clip": 1.02661204, "balance_loss_mlp": 1.03676331, "epoch": 0.08044491206974297, "flos": 25299565948800.0, "grad_norm": 2.2552804727475815, "language_loss": 0.86439645, "learning_rate": 3.936639059042932e-06, "loss": 0.88634193, "num_input_tokens_seen": 28411845, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0390625, "step": 1338, "time_per_iteration": 2.505598783493042 }, { "auxiliary_loss_clip": 0.01138751, "auxiliary_loss_mlp": 0.01067749, "balance_loss_clip": 1.03838754, "balance_loss_mlp": 1.03342259, "epoch": 0.08050503532241095, "flos": 22375625160960.0, "grad_norm": 4.359292809857966, "language_loss": 0.87319863, "learning_rate": 3.936544676805397e-06, "loss": 0.89526367, "num_input_tokens_seen": 28427875, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.0546875, "step": 1339, "time_per_iteration": 2.425258159637451 }, { "auxiliary_loss_clip": 0.01133709, "auxiliary_loss_mlp": 0.01047137, "balance_loss_clip": 1.02039886, "balance_loss_mlp": 1.03435993, "epoch": 0.08056515857507891, "flos": 18368433619200.0, "grad_norm": 2.1293160007814826, "language_loss": 0.89519572, "learning_rate": 3.936450225457564e-06, "loss": 0.91700423, "num_input_tokens_seen": 28446615, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.9921875, "step": 1340, "time_per_iteration": 2.40386700630188 }, { "auxiliary_loss_clip": 0.01036387, "auxiliary_loss_mlp": 0.01003174, "balance_loss_clip": 0.99928826, "balance_loss_mlp": 1.00538206, "epoch": 0.08062528182774688, "flos": 51345329719680.0, "grad_norm": 0.8708482508804375, "language_loss": 0.64813459, "learning_rate": 3.936355705002804e-06, "loss": 0.66853023, "num_input_tokens_seen": 28505290, "router_z_loss_clip": 0.03881836, "router_z_loss_mlp": 0.31054688, "step": 1341, "time_per_iteration": 3.0302042961120605 }, { "auxiliary_loss_clip": 0.01145959, "auxiliary_loss_mlp": 0.01054501, "balance_loss_clip": 1.02598643, "balance_loss_mlp": 1.03534555, "epoch": 0.08068540508041486, "flos": 17598843319680.0, "grad_norm": 2.07789975291421, "language_loss": 0.89729524, "learning_rate": 3.936261115444489e-06, "loss": 0.91929984, "num_input_tokens_seen": 28522735, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.109375, "step": 1342, "time_per_iteration": 2.4231953620910645 }, { "auxiliary_loss_clip": 0.01147974, "auxiliary_loss_mlp": 0.01057106, "balance_loss_clip": 1.02868652, "balance_loss_mlp": 1.03958821, "epoch": 0.08074552833308282, "flos": 10560422782080.0, "grad_norm": 2.5990374723313217, "language_loss": 0.76440805, "learning_rate": 3.936166456785997e-06, "loss": 0.78645885, "num_input_tokens_seen": 28539460, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.078125, "step": 1343, "time_per_iteration": 2.42657470703125 }, { "auxiliary_loss_clip": 0.0103502, "auxiliary_loss_mlp": 0.01009421, "balance_loss_clip": 1.00532043, "balance_loss_mlp": 1.00449371, "epoch": 0.08080565158575079, "flos": 60837026739840.0, "grad_norm": 0.8016447465790755, "language_loss": 0.57401437, "learning_rate": 3.936071729030702e-06, "loss": 0.59445882, "num_input_tokens_seen": 28599855, "router_z_loss_clip": 0.04101562, "router_z_loss_mlp": 0.3046875, "step": 1344, "time_per_iteration": 3.0285229682922363 }, { "auxiliary_loss_clip": 0.01142824, "auxiliary_loss_mlp": 0.01056169, "balance_loss_clip": 1.02847719, "balance_loss_mlp": 1.03656745, "epoch": 0.08086577483841875, "flos": 18331390800000.0, "grad_norm": 3.3861312365131355, "language_loss": 0.86296439, "learning_rate": 3.935976932181989e-06, "loss": 0.88495433, "num_input_tokens_seen": 28617585, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.0625, "step": 1345, "time_per_iteration": 2.429169178009033 }, { "auxiliary_loss_clip": 0.0113923, "auxiliary_loss_mlp": 0.01056697, "balance_loss_clip": 1.03047085, "balance_loss_mlp": 1.03582883, "epoch": 0.08092589809108673, "flos": 21542527365120.0, "grad_norm": 1.8247206662094533, "language_loss": 0.87417907, "learning_rate": 3.935882066243239e-06, "loss": 0.89613831, "num_input_tokens_seen": 28636355, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.03125, "step": 1346, "time_per_iteration": 2.438117027282715 }, { "auxiliary_loss_clip": 0.01140205, "auxiliary_loss_mlp": 0.01051821, "balance_loss_clip": 1.02616668, "balance_loss_mlp": 1.03634501, "epoch": 0.0809860213437547, "flos": 22126903568640.0, "grad_norm": 1.9378568526291882, "language_loss": 0.92655408, "learning_rate": 3.935787131217838e-06, "loss": 0.94847435, "num_input_tokens_seen": 28656260, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 1.0390625, "step": 1347, "time_per_iteration": 2.4197306632995605 }, { "auxiliary_loss_clip": 0.01136978, "auxiliary_loss_mlp": 0.01047197, "balance_loss_clip": 1.01831245, "balance_loss_mlp": 1.03443682, "epoch": 0.08104614459642266, "flos": 21724424881920.0, "grad_norm": 2.006159913199672, "language_loss": 0.89071, "learning_rate": 3.9356921271091734e-06, "loss": 0.91255176, "num_input_tokens_seen": 28675865, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0234375, "step": 1348, "time_per_iteration": 2.4586288928985596 }, { "auxiliary_loss_clip": 0.01137366, "auxiliary_loss_mlp": 0.01047615, "balance_loss_clip": 1.02150774, "balance_loss_mlp": 1.0380547, "epoch": 0.08110626784909064, "flos": 23950731415680.0, "grad_norm": 1.9698497285486793, "language_loss": 0.76631665, "learning_rate": 3.935597053920635e-06, "loss": 0.7881664, "num_input_tokens_seen": 28696255, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9921875, "step": 1349, "time_per_iteration": 2.4439492225646973 }, { "auxiliary_loss_clip": 0.01140006, "auxiliary_loss_mlp": 0.0105312, "balance_loss_clip": 1.02532005, "balance_loss_mlp": 1.03573346, "epoch": 0.0811663911017586, "flos": 19024696045440.0, "grad_norm": 2.4233555394337256, "language_loss": 0.88450396, "learning_rate": 3.935501911655618e-06, "loss": 0.90643525, "num_input_tokens_seen": 28713905, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0390625, "step": 1350, "time_per_iteration": 2.412860631942749 }, { "auxiliary_loss_clip": 0.01135942, "auxiliary_loss_mlp": 0.01058571, "balance_loss_clip": 1.03073609, "balance_loss_mlp": 1.03386545, "epoch": 0.08122651435442657, "flos": 15340381557120.0, "grad_norm": 2.1531664901380094, "language_loss": 0.8194319, "learning_rate": 3.935406700317516e-06, "loss": 0.84137702, "num_input_tokens_seen": 28732075, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0234375, "step": 1351, "time_per_iteration": 2.4073026180267334 }, { "auxiliary_loss_clip": 0.011395, "auxiliary_loss_mlp": 0.01051922, "balance_loss_clip": 1.02209556, "balance_loss_mlp": 1.03398967, "epoch": 0.08128663760709455, "flos": 23220453173760.0, "grad_norm": 2.489233034819534, "language_loss": 0.75422478, "learning_rate": 3.935311419909728e-06, "loss": 0.77613902, "num_input_tokens_seen": 28751150, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.0546875, "step": 1352, "time_per_iteration": 2.5176920890808105 }, { "auxiliary_loss_clip": 0.01141523, "auxiliary_loss_mlp": 0.01056237, "balance_loss_clip": 1.02710199, "balance_loss_mlp": 1.03585172, "epoch": 0.08134676085976252, "flos": 22964539841280.0, "grad_norm": 1.82520977696158, "language_loss": 0.83126086, "learning_rate": 3.935216070435652e-06, "loss": 0.85323852, "num_input_tokens_seen": 28773360, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.0546875, "step": 1353, "time_per_iteration": 2.468277931213379 }, { "auxiliary_loss_clip": 0.01033342, "auxiliary_loss_mlp": 0.01007523, "balance_loss_clip": 1.00339806, "balance_loss_mlp": 1.00308514, "epoch": 0.08140688411243048, "flos": 64319369679360.0, "grad_norm": 0.8483378491422867, "language_loss": 0.59735012, "learning_rate": 3.935120651898694e-06, "loss": 0.61775875, "num_input_tokens_seen": 28833390, "router_z_loss_clip": 0.04125977, "router_z_loss_mlp": 0.30273438, "step": 1354, "time_per_iteration": 3.0853233337402344 }, { "auxiliary_loss_clip": 0.01136525, "auxiliary_loss_mlp": 0.01052364, "balance_loss_clip": 1.02514863, "balance_loss_mlp": 1.03431463, "epoch": 0.08146700736509845, "flos": 22490768424960.0, "grad_norm": 1.8495339968426354, "language_loss": 0.82956147, "learning_rate": 3.935025164302257e-06, "loss": 0.85145044, "num_input_tokens_seen": 28852430, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.0234375, "step": 1355, "time_per_iteration": 2.464796781539917 }, { "auxiliary_loss_clip": 0.01139687, "auxiliary_loss_mlp": 0.01059226, "balance_loss_clip": 1.0298171, "balance_loss_mlp": 1.03361726, "epoch": 0.08152713061776642, "flos": 20446813255680.0, "grad_norm": 1.9276991028738168, "language_loss": 0.7095387, "learning_rate": 3.934929607649749e-06, "loss": 0.73152781, "num_input_tokens_seen": 28870685, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.0625, "step": 1356, "time_per_iteration": 2.4853670597076416 }, { "auxiliary_loss_clip": 0.01139779, "auxiliary_loss_mlp": 0.01051055, "balance_loss_clip": 1.02342236, "balance_loss_mlp": 1.03437948, "epoch": 0.08158725387043439, "flos": 23549090601600.0, "grad_norm": 1.8537448997099917, "language_loss": 0.70516974, "learning_rate": 3.934833981944582e-06, "loss": 0.72707808, "num_input_tokens_seen": 28889860, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.0546875, "step": 1357, "time_per_iteration": 2.4754977226257324 }, { "auxiliary_loss_clip": 0.0114063, "auxiliary_loss_mlp": 0.01053673, "balance_loss_clip": 1.02531266, "balance_loss_mlp": 1.03652501, "epoch": 0.08164737712310235, "flos": 22016263870080.0, "grad_norm": 2.1873732431405237, "language_loss": 0.84406656, "learning_rate": 3.934738287190166e-06, "loss": 0.86600959, "num_input_tokens_seen": 28905865, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.0390625, "step": 1358, "time_per_iteration": 3.949376106262207 }, { "auxiliary_loss_clip": 0.01141411, "auxiliary_loss_mlp": 0.01053594, "balance_loss_clip": 1.02345777, "balance_loss_mlp": 1.03640282, "epoch": 0.08170750037577033, "flos": 23366704325760.0, "grad_norm": 2.0468282200358843, "language_loss": 1.0262934, "learning_rate": 3.934642523389917e-06, "loss": 1.04824352, "num_input_tokens_seen": 28925250, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.0546875, "step": 1359, "time_per_iteration": 3.8985743522644043 }, { "auxiliary_loss_clip": 0.01136888, "auxiliary_loss_mlp": 0.01049909, "balance_loss_clip": 1.02070248, "balance_loss_mlp": 1.03373373, "epoch": 0.0817676236284383, "flos": 28396850970240.0, "grad_norm": 2.0307171832386377, "language_loss": 0.83083647, "learning_rate": 3.934546690547253e-06, "loss": 0.85270447, "num_input_tokens_seen": 28943445, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.03125, "step": 1360, "time_per_iteration": 2.471463680267334 }, { "auxiliary_loss_clip": 0.01140642, "auxiliary_loss_mlp": 0.01051129, "balance_loss_clip": 1.02199435, "balance_loss_mlp": 1.03499973, "epoch": 0.08182774688110626, "flos": 19207885282560.0, "grad_norm": 2.2149181348842157, "language_loss": 0.72330105, "learning_rate": 3.934450788665594e-06, "loss": 0.74521875, "num_input_tokens_seen": 28962695, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.0546875, "step": 1361, "time_per_iteration": 5.282749176025391 }, { "auxiliary_loss_clip": 0.01132788, "auxiliary_loss_mlp": 0.01056152, "balance_loss_clip": 1.02720809, "balance_loss_mlp": 1.03063273, "epoch": 0.08188787013377424, "flos": 22782991438080.0, "grad_norm": 2.91924500043117, "language_loss": 0.76753962, "learning_rate": 3.934354817748363e-06, "loss": 0.78942901, "num_input_tokens_seen": 28982120, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0234375, "step": 1362, "time_per_iteration": 2.4537808895111084 }, { "auxiliary_loss_clip": 0.01139277, "auxiliary_loss_mlp": 0.01047336, "balance_loss_clip": 1.01981115, "balance_loss_mlp": 1.03720856, "epoch": 0.08194799338644221, "flos": 16467273377280.0, "grad_norm": 2.5230938950134862, "language_loss": 0.7296077, "learning_rate": 3.934258777798984e-06, "loss": 0.75147378, "num_input_tokens_seen": 28998100, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.0234375, "step": 1363, "time_per_iteration": 2.3879504203796387 }, { "auxiliary_loss_clip": 0.01139257, "auxiliary_loss_mlp": 0.01046154, "balance_loss_clip": 1.01670909, "balance_loss_mlp": 1.03688741, "epoch": 0.08200811663911017, "flos": 23912536521600.0, "grad_norm": 2.0120773302425747, "language_loss": 0.77598512, "learning_rate": 3.934162668820884e-06, "loss": 0.79783922, "num_input_tokens_seen": 29017095, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.0234375, "step": 1364, "time_per_iteration": 2.460463762283325 }, { "auxiliary_loss_clip": 0.01139736, "auxiliary_loss_mlp": 0.01051396, "balance_loss_clip": 1.02369142, "balance_loss_mlp": 1.03548634, "epoch": 0.08206823989177814, "flos": 17895534986880.0, "grad_norm": 11.193197151022844, "language_loss": 0.81889302, "learning_rate": 3.934066490817495e-06, "loss": 0.84080428, "num_input_tokens_seen": 29037240, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.046875, "step": 1365, "time_per_iteration": 2.552370548248291 }, { "auxiliary_loss_clip": 0.01138806, "auxiliary_loss_mlp": 0.01046396, "balance_loss_clip": 1.01888299, "balance_loss_mlp": 1.03748155, "epoch": 0.08212836314444612, "flos": 22087172574720.0, "grad_norm": 2.109783001922283, "language_loss": 0.82045788, "learning_rate": 3.9339702437922465e-06, "loss": 0.84230983, "num_input_tokens_seen": 29056250, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.015625, "step": 1366, "time_per_iteration": 2.615504264831543 }, { "auxiliary_loss_clip": 0.01138705, "auxiliary_loss_mlp": 0.010455, "balance_loss_clip": 1.02046609, "balance_loss_mlp": 1.03385198, "epoch": 0.08218848639711408, "flos": 17596678815360.0, "grad_norm": 1.770323620790888, "language_loss": 0.81591201, "learning_rate": 3.933873927748575e-06, "loss": 0.83775401, "num_input_tokens_seen": 29073380, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 1.046875, "step": 1367, "time_per_iteration": 2.467405080795288 }, { "auxiliary_loss_clip": 0.01145546, "auxiliary_loss_mlp": 0.01059798, "balance_loss_clip": 1.03116357, "balance_loss_mlp": 1.03625274, "epoch": 0.08224860964978205, "flos": 17856886245120.0, "grad_norm": 2.036747963087789, "language_loss": 0.82997632, "learning_rate": 3.933777542689918e-06, "loss": 0.85202968, "num_input_tokens_seen": 29091330, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.09375, "step": 1368, "time_per_iteration": 2.5678632259368896 }, { "auxiliary_loss_clip": 0.01133686, "auxiliary_loss_mlp": 0.0104746, "balance_loss_clip": 1.02059031, "balance_loss_mlp": 1.03601241, "epoch": 0.08230873290245003, "flos": 25226388005760.0, "grad_norm": 1.7779988400973337, "language_loss": 0.81281292, "learning_rate": 3.933681088619715e-06, "loss": 0.83462441, "num_input_tokens_seen": 29110375, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.9765625, "step": 1369, "time_per_iteration": 2.535598039627075 }, { "auxiliary_loss_clip": 0.01136147, "auxiliary_loss_mlp": 0.01048409, "balance_loss_clip": 1.02279139, "balance_loss_mlp": 1.03621101, "epoch": 0.08236885615511799, "flos": 31758567696000.0, "grad_norm": 2.113249725053309, "language_loss": 0.74624491, "learning_rate": 3.933584565541407e-06, "loss": 0.76809049, "num_input_tokens_seen": 29129395, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 1.0, "step": 1370, "time_per_iteration": 2.606149196624756 }, { "auxiliary_loss_clip": 0.01141181, "auxiliary_loss_mlp": 0.01055651, "balance_loss_clip": 1.02685022, "balance_loss_mlp": 1.03535104, "epoch": 0.08242897940778596, "flos": 23184702074880.0, "grad_norm": 1.5566014351480937, "language_loss": 0.74512672, "learning_rate": 3.9334879734584405e-06, "loss": 0.76709503, "num_input_tokens_seen": 29148650, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0625, "step": 1371, "time_per_iteration": 2.5078554153442383 }, { "auxiliary_loss_clip": 0.01137903, "auxiliary_loss_mlp": 0.01052755, "balance_loss_clip": 1.02305984, "balance_loss_mlp": 1.03381038, "epoch": 0.08248910266045394, "flos": 34490172470400.0, "grad_norm": 2.0681939965691374, "language_loss": 0.71125972, "learning_rate": 3.933391312374262e-06, "loss": 0.73316634, "num_input_tokens_seen": 29170785, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.046875, "step": 1372, "time_per_iteration": 2.66995906829834 }, { "auxiliary_loss_clip": 0.01141069, "auxiliary_loss_mlp": 0.01055284, "balance_loss_clip": 1.02529144, "balance_loss_mlp": 1.03453314, "epoch": 0.0825492259131219, "flos": 13435590533760.0, "grad_norm": 3.663715416242882, "language_loss": 0.87991744, "learning_rate": 3.93329458229232e-06, "loss": 0.90188098, "num_input_tokens_seen": 29185210, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.0625, "step": 1373, "time_per_iteration": 2.432610034942627 }, { "auxiliary_loss_clip": 0.01136937, "auxiliary_loss_mlp": 0.01053465, "balance_loss_clip": 1.02511716, "balance_loss_mlp": 1.03353488, "epoch": 0.08260934916578987, "flos": 25811252968320.0, "grad_norm": 1.8320037522503072, "language_loss": 0.82148111, "learning_rate": 3.933197783216068e-06, "loss": 0.8433851, "num_input_tokens_seen": 29205210, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.03125, "step": 1374, "time_per_iteration": 2.4978749752044678 }, { "auxiliary_loss_clip": 0.01037488, "auxiliary_loss_mlp": 0.01014386, "balance_loss_clip": 1.00916481, "balance_loss_mlp": 1.00536513, "epoch": 0.08266947241845783, "flos": 63456909563520.0, "grad_norm": 0.8173504943743701, "language_loss": 0.60599476, "learning_rate": 3.93310091514896e-06, "loss": 0.62651354, "num_input_tokens_seen": 29265350, "router_z_loss_clip": 0.05224609, "router_z_loss_mlp": 0.3203125, "step": 1375, "time_per_iteration": 3.042445182800293 }, { "auxiliary_loss_clip": 0.01035876, "auxiliary_loss_mlp": 0.01008424, "balance_loss_clip": 1.00348854, "balance_loss_mlp": 1.00414395, "epoch": 0.08272959567112581, "flos": 69990346062720.0, "grad_norm": 0.9075145685559574, "language_loss": 0.62212205, "learning_rate": 3.933003978094452e-06, "loss": 0.64256501, "num_input_tokens_seen": 29321475, "router_z_loss_clip": 0.04931641, "router_z_loss_mlp": 0.31640625, "step": 1376, "time_per_iteration": 3.0153872966766357 }, { "auxiliary_loss_clip": 0.01142638, "auxiliary_loss_mlp": 0.01054963, "balance_loss_clip": 1.02731788, "balance_loss_mlp": 1.03678632, "epoch": 0.08278971892379378, "flos": 20412144408960.0, "grad_norm": 1.6758916856217034, "language_loss": 0.82464159, "learning_rate": 3.9329069720560045e-06, "loss": 0.84661758, "num_input_tokens_seen": 29341405, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0625, "step": 1377, "time_per_iteration": 2.5013442039489746 }, { "auxiliary_loss_clip": 0.01137221, "auxiliary_loss_mlp": 0.01052512, "balance_loss_clip": 1.02425957, "balance_loss_mlp": 1.03591537, "epoch": 0.08284984217646174, "flos": 26249028906240.0, "grad_norm": 1.8833560528177287, "language_loss": 0.84713018, "learning_rate": 3.932809897037079e-06, "loss": 0.8690275, "num_input_tokens_seen": 29361955, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.015625, "step": 1378, "time_per_iteration": 2.4867005348205566 }, { "auxiliary_loss_clip": 0.01138837, "auxiliary_loss_mlp": 0.01053253, "balance_loss_clip": 1.02495241, "balance_loss_mlp": 1.03472888, "epoch": 0.08290996542912972, "flos": 27193569361920.0, "grad_norm": 2.1981360833435644, "language_loss": 0.87588495, "learning_rate": 3.932712753041141e-06, "loss": 0.89780581, "num_input_tokens_seen": 29382395, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.0390625, "step": 1379, "time_per_iteration": 2.477638006210327 }, { "auxiliary_loss_clip": 0.01137617, "auxiliary_loss_mlp": 0.01054187, "balance_loss_clip": 1.02743649, "balance_loss_mlp": 1.03673005, "epoch": 0.08297008868179769, "flos": 38616661728000.0, "grad_norm": 2.1679296386762332, "language_loss": 0.7849893, "learning_rate": 3.932615540071656e-06, "loss": 0.80690736, "num_input_tokens_seen": 29404460, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.015625, "step": 1380, "time_per_iteration": 2.553020477294922 }, { "auxiliary_loss_clip": 0.01137195, "auxiliary_loss_mlp": 0.01059167, "balance_loss_clip": 1.03142667, "balance_loss_mlp": 1.03819525, "epoch": 0.08303021193446565, "flos": 19973705155200.0, "grad_norm": 2.502140765456767, "language_loss": 0.85779071, "learning_rate": 3.932518258132094e-06, "loss": 0.8797543, "num_input_tokens_seen": 29422675, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 0.98828125, "step": 1381, "time_per_iteration": 2.4220502376556396 }, { "auxiliary_loss_clip": 0.01146824, "auxiliary_loss_mlp": 0.01056244, "balance_loss_clip": 1.02714479, "balance_loss_mlp": 1.03916287, "epoch": 0.08309033518713363, "flos": 13661792432640.0, "grad_norm": 2.8855093131695493, "language_loss": 0.88018179, "learning_rate": 3.932420907225926e-06, "loss": 0.9022125, "num_input_tokens_seen": 29439840, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.078125, "step": 1382, "time_per_iteration": 2.3828964233398438 }, { "auxiliary_loss_clip": 0.01138958, "auxiliary_loss_mlp": 0.01054701, "balance_loss_clip": 1.02839184, "balance_loss_mlp": 1.03570044, "epoch": 0.0831504584398016, "flos": 17967560855040.0, "grad_norm": 2.632102141344648, "language_loss": 0.77463621, "learning_rate": 3.932323487356626e-06, "loss": 0.7965728, "num_input_tokens_seen": 29457360, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 1.03125, "step": 1383, "time_per_iteration": 2.403245210647583 }, { "auxiliary_loss_clip": 0.01142298, "auxiliary_loss_mlp": 0.01054448, "balance_loss_clip": 1.02700639, "balance_loss_mlp": 1.03616834, "epoch": 0.08321058169246956, "flos": 22600290960000.0, "grad_norm": 6.694357317480596, "language_loss": 0.82948864, "learning_rate": 3.932225998527672e-06, "loss": 0.85145605, "num_input_tokens_seen": 29477040, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.0625, "step": 1384, "time_per_iteration": 2.4460690021514893 }, { "auxiliary_loss_clip": 0.01147629, "auxiliary_loss_mlp": 0.01055719, "balance_loss_clip": 1.02738309, "balance_loss_mlp": 1.03878617, "epoch": 0.08327070494513754, "flos": 22849501311360.0, "grad_norm": 2.7198915303661058, "language_loss": 0.85049307, "learning_rate": 3.932128440742542e-06, "loss": 0.87252659, "num_input_tokens_seen": 29492010, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.0859375, "step": 1385, "time_per_iteration": 2.4074478149414062 }, { "auxiliary_loss_clip": 0.01144683, "auxiliary_loss_mlp": 0.01051475, "balance_loss_clip": 1.02263844, "balance_loss_mlp": 1.03906059, "epoch": 0.0833308281978055, "flos": 22781909185920.0, "grad_norm": 1.7272321262773471, "language_loss": 0.68542445, "learning_rate": 3.932030814004719e-06, "loss": 0.70738602, "num_input_tokens_seen": 29511850, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0546875, "step": 1386, "time_per_iteration": 2.489827871322632 }, { "auxiliary_loss_clip": 0.01138937, "auxiliary_loss_mlp": 0.0105088, "balance_loss_clip": 1.02410579, "balance_loss_mlp": 1.03381312, "epoch": 0.08339095145047347, "flos": 20811585807360.0, "grad_norm": 1.6662190524934888, "language_loss": 0.81894517, "learning_rate": 3.9319331183176844e-06, "loss": 0.84084338, "num_input_tokens_seen": 29531415, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.046875, "step": 1387, "time_per_iteration": 2.442065715789795 }, { "auxiliary_loss_clip": 0.0114151, "auxiliary_loss_mlp": 0.01063807, "balance_loss_clip": 1.03293192, "balance_loss_mlp": 1.03462207, "epoch": 0.08345107470314143, "flos": 18514335657600.0, "grad_norm": 1.9618631684366505, "language_loss": 0.77150124, "learning_rate": 3.931835353684927e-06, "loss": 0.79355443, "num_input_tokens_seen": 29549525, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.0703125, "step": 1388, "time_per_iteration": 2.442744731903076 }, { "auxiliary_loss_clip": 0.01136028, "auxiliary_loss_mlp": 0.01059897, "balance_loss_clip": 1.03164482, "balance_loss_mlp": 1.03585863, "epoch": 0.08351119795580941, "flos": 18806558670720.0, "grad_norm": 1.9977385433797352, "language_loss": 0.78928244, "learning_rate": 3.931737520109935e-06, "loss": 0.81124169, "num_input_tokens_seen": 29568705, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.0, "step": 1389, "time_per_iteration": 2.403454542160034 }, { "auxiliary_loss_clip": 0.01140771, "auxiliary_loss_mlp": 0.01045638, "balance_loss_clip": 1.01771963, "balance_loss_mlp": 1.03638148, "epoch": 0.08357132120847738, "flos": 18440843512320.0, "grad_norm": 2.5662322532793325, "language_loss": 0.87396991, "learning_rate": 3.931639617596201e-06, "loss": 0.89583397, "num_input_tokens_seen": 29585855, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.046875, "step": 1390, "time_per_iteration": 2.444549798965454 }, { "auxiliary_loss_clip": 0.01136063, "auxiliary_loss_mlp": 0.01063131, "balance_loss_clip": 1.03559399, "balance_loss_mlp": 1.03334212, "epoch": 0.08363144446114534, "flos": 25921124616960.0, "grad_norm": 2.3827728135287236, "language_loss": 0.86620837, "learning_rate": 3.931541646147217e-06, "loss": 0.88820034, "num_input_tokens_seen": 29607280, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.03125, "step": 1391, "time_per_iteration": 2.4772896766662598 }, { "auxiliary_loss_clip": 0.01147064, "auxiliary_loss_mlp": 0.01066716, "balance_loss_clip": 1.03882098, "balance_loss_mlp": 1.03849733, "epoch": 0.08369156771381332, "flos": 18040319861760.0, "grad_norm": 2.5776007911349925, "language_loss": 0.87413985, "learning_rate": 3.93144360576648e-06, "loss": 0.89627767, "num_input_tokens_seen": 29624130, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.0859375, "step": 1392, "time_per_iteration": 2.43986439704895 }, { "auxiliary_loss_clip": 0.0113886, "auxiliary_loss_mlp": 0.01055026, "balance_loss_clip": 1.02673769, "balance_loss_mlp": 1.03480148, "epoch": 0.08375169096648129, "flos": 22673992573440.0, "grad_norm": 2.5201146235582197, "language_loss": 0.79845703, "learning_rate": 3.931345496457489e-06, "loss": 0.82039583, "num_input_tokens_seen": 29643210, "router_z_loss_clip": 0.28320312, "router_z_loss_mlp": 1.0390625, "step": 1393, "time_per_iteration": 2.4255776405334473 }, { "auxiliary_loss_clip": 0.01137911, "auxiliary_loss_mlp": 0.01047337, "balance_loss_clip": 1.02095628, "balance_loss_mlp": 1.03682518, "epoch": 0.08381181421914925, "flos": 26102044615680.0, "grad_norm": 3.8426116391483442, "language_loss": 0.84546328, "learning_rate": 3.931247318223746e-06, "loss": 0.86731571, "num_input_tokens_seen": 29663920, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 1.0078125, "step": 1394, "time_per_iteration": 2.4559414386749268 }, { "auxiliary_loss_clip": 0.01141432, "auxiliary_loss_mlp": 0.0104905, "balance_loss_clip": 1.0208931, "balance_loss_mlp": 1.0367496, "epoch": 0.08387193747181723, "flos": 20628780595200.0, "grad_norm": 2.1271812036602222, "language_loss": 0.82844597, "learning_rate": 3.931149071068753e-06, "loss": 0.85035086, "num_input_tokens_seen": 29683825, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.046875, "step": 1395, "time_per_iteration": 2.403179407119751 }, { "auxiliary_loss_clip": 0.01141043, "auxiliary_loss_mlp": 0.01050477, "balance_loss_clip": 1.02043593, "balance_loss_mlp": 1.03645396, "epoch": 0.0839320607244852, "flos": 13442363337600.0, "grad_norm": 2.805372604291138, "language_loss": 0.82337093, "learning_rate": 3.931050754996018e-06, "loss": 0.84528613, "num_input_tokens_seen": 29698775, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.046875, "step": 1396, "time_per_iteration": 2.3986716270446777 }, { "auxiliary_loss_clip": 0.01139023, "auxiliary_loss_mlp": 0.01051594, "balance_loss_clip": 1.0220654, "balance_loss_mlp": 1.03866041, "epoch": 0.08399218397715316, "flos": 23476122126720.0, "grad_norm": 1.9699496155025322, "language_loss": 0.76609969, "learning_rate": 3.930952370009048e-06, "loss": 0.78800583, "num_input_tokens_seen": 29719430, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.0, "step": 1397, "time_per_iteration": 3.893310308456421 }, { "auxiliary_loss_clip": 0.01136244, "auxiliary_loss_mlp": 0.01046049, "balance_loss_clip": 1.01685452, "balance_loss_mlp": 1.03420091, "epoch": 0.08405230722982113, "flos": 25919553605760.0, "grad_norm": 2.245557912887348, "language_loss": 0.7817446, "learning_rate": 3.930853916111355e-06, "loss": 0.80356753, "num_input_tokens_seen": 29739685, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.0234375, "step": 1398, "time_per_iteration": 3.903414726257324 }, { "auxiliary_loss_clip": 0.01131837, "auxiliary_loss_mlp": 0.01047987, "balance_loss_clip": 1.0217495, "balance_loss_mlp": 1.03283024, "epoch": 0.0841124304824891, "flos": 17966478602880.0, "grad_norm": 2.612612922286341, "language_loss": 0.95172715, "learning_rate": 3.930755393306453e-06, "loss": 0.9735254, "num_input_tokens_seen": 29756165, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.98828125, "step": 1399, "time_per_iteration": 2.4221866130828857 }, { "auxiliary_loss_clip": 0.01139385, "auxiliary_loss_mlp": 0.01057325, "balance_loss_clip": 1.02733231, "balance_loss_mlp": 1.03419042, "epoch": 0.08417255373515707, "flos": 25628482667520.0, "grad_norm": 1.9415604543083347, "language_loss": 0.81517625, "learning_rate": 3.930656801597857e-06, "loss": 0.83714336, "num_input_tokens_seen": 29776425, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.0546875, "step": 1400, "time_per_iteration": 5.18965220451355 }, { "auxiliary_loss_clip": 0.01136377, "auxiliary_loss_mlp": 0.01053098, "balance_loss_clip": 1.0243212, "balance_loss_mlp": 1.0334307, "epoch": 0.08423267698782504, "flos": 26248540147200.0, "grad_norm": 3.0763122953915043, "language_loss": 0.86442995, "learning_rate": 3.930558140989087e-06, "loss": 0.88632476, "num_input_tokens_seen": 29796440, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.03125, "step": 1401, "time_per_iteration": 2.4589450359344482 }, { "auxiliary_loss_clip": 0.01140493, "auxiliary_loss_mlp": 0.01053125, "balance_loss_clip": 1.02307224, "balance_loss_mlp": 1.03402793, "epoch": 0.08429280024049302, "flos": 20118699498240.0, "grad_norm": 2.3133765135270075, "language_loss": 0.87033337, "learning_rate": 3.930459411483662e-06, "loss": 0.89226949, "num_input_tokens_seen": 29814755, "router_z_loss_clip": 0.30078125, "router_z_loss_mlp": 1.0625, "step": 1402, "time_per_iteration": 2.4530887603759766 }, { "auxiliary_loss_clip": 0.01134145, "auxiliary_loss_mlp": 0.01047832, "balance_loss_clip": 1.02120042, "balance_loss_mlp": 1.03132677, "epoch": 0.08435292349316098, "flos": 42922849086720.0, "grad_norm": 2.048879929905967, "language_loss": 0.8895582, "learning_rate": 3.930360613085106e-06, "loss": 0.91137803, "num_input_tokens_seen": 29834785, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.03125, "step": 1403, "time_per_iteration": 2.6017262935638428 }, { "auxiliary_loss_clip": 0.01139954, "auxiliary_loss_mlp": 0.01052537, "balance_loss_clip": 1.02278185, "balance_loss_mlp": 1.0341984, "epoch": 0.08441304674582895, "flos": 22856169381120.0, "grad_norm": 2.3078835344609447, "language_loss": 0.80272245, "learning_rate": 3.930261745796945e-06, "loss": 0.82464731, "num_input_tokens_seen": 29854695, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.0625, "step": 1404, "time_per_iteration": 2.4601595401763916 }, { "auxiliary_loss_clip": 0.01142983, "auxiliary_loss_mlp": 0.01063266, "balance_loss_clip": 1.03136539, "balance_loss_mlp": 1.03684366, "epoch": 0.08447316999849692, "flos": 18696512465280.0, "grad_norm": 1.9814480155180556, "language_loss": 0.83600795, "learning_rate": 3.930162809622709e-06, "loss": 0.85807049, "num_input_tokens_seen": 29872180, "router_z_loss_clip": 0.31835938, "router_z_loss_mlp": 1.0625, "step": 1405, "time_per_iteration": 2.4039623737335205 }, { "auxiliary_loss_clip": 0.01137342, "auxiliary_loss_mlp": 0.0105122, "balance_loss_clip": 1.022228, "balance_loss_mlp": 1.0332588, "epoch": 0.08453329325116489, "flos": 25482790097280.0, "grad_norm": 1.6255358588896107, "language_loss": 0.80443799, "learning_rate": 3.930063804565927e-06, "loss": 0.82632363, "num_input_tokens_seen": 29893205, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0390625, "step": 1406, "time_per_iteration": 2.510904550552368 }, { "auxiliary_loss_clip": 0.01140187, "auxiliary_loss_mlp": 0.01056026, "balance_loss_clip": 1.0283339, "balance_loss_mlp": 1.03640819, "epoch": 0.08459341650383286, "flos": 20919083483520.0, "grad_norm": 1.957000793352056, "language_loss": 0.79425609, "learning_rate": 3.929964730630132e-06, "loss": 0.81621814, "num_input_tokens_seen": 29911970, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0390625, "step": 1407, "time_per_iteration": 2.423840045928955 }, { "auxiliary_loss_clip": 0.01135099, "auxiliary_loss_mlp": 0.01050036, "balance_loss_clip": 1.02234411, "balance_loss_mlp": 1.03416073, "epoch": 0.08465353975650082, "flos": 13042223712000.0, "grad_norm": 2.3275697224793697, "language_loss": 0.91585648, "learning_rate": 3.92986558781886e-06, "loss": 0.9377079, "num_input_tokens_seen": 29929925, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0078125, "step": 1408, "time_per_iteration": 2.4315226078033447 }, { "auxiliary_loss_clip": 0.01043673, "auxiliary_loss_mlp": 0.01023297, "balance_loss_clip": 1.01826644, "balance_loss_mlp": 1.01066768, "epoch": 0.0847136630091688, "flos": 60874174293120.0, "grad_norm": 0.8792341838331387, "language_loss": 0.61765254, "learning_rate": 3.92976637613565e-06, "loss": 0.63832223, "num_input_tokens_seen": 29985950, "router_z_loss_clip": 0.05029297, "router_z_loss_mlp": 0.33007812, "step": 1409, "time_per_iteration": 3.1208980083465576 }, { "auxiliary_loss_clip": 0.01131074, "auxiliary_loss_mlp": 0.01055388, "balance_loss_clip": 1.02793384, "balance_loss_mlp": 1.03469133, "epoch": 0.08477378626183676, "flos": 22045661101440.0, "grad_norm": 1.6652926113525195, "language_loss": 0.86648887, "learning_rate": 3.9296670955840415e-06, "loss": 0.88835347, "num_input_tokens_seen": 30004330, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.96484375, "step": 1410, "time_per_iteration": 2.4426138401031494 }, { "auxiliary_loss_clip": 0.01136838, "auxiliary_loss_mlp": 0.01047834, "balance_loss_clip": 1.01812696, "balance_loss_mlp": 1.03348505, "epoch": 0.08483390951450473, "flos": 16689146267520.0, "grad_norm": 2.071857028368419, "language_loss": 0.74074405, "learning_rate": 3.929567746167578e-06, "loss": 0.76259077, "num_input_tokens_seen": 30022555, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 1.03125, "step": 1411, "time_per_iteration": 2.4424655437469482 }, { "auxiliary_loss_clip": 0.01036848, "auxiliary_loss_mlp": 0.01007644, "balance_loss_clip": 1.00278056, "balance_loss_mlp": 1.00472724, "epoch": 0.08489403276717271, "flos": 51581341710720.0, "grad_norm": 0.9068689782583981, "language_loss": 0.56724936, "learning_rate": 3.929468327889805e-06, "loss": 0.58769429, "num_input_tokens_seen": 30077220, "router_z_loss_clip": 0.04858398, "router_z_loss_mlp": 0.3203125, "step": 1412, "time_per_iteration": 3.0075435638427734 }, { "auxiliary_loss_clip": 0.01133906, "auxiliary_loss_mlp": 0.01054027, "balance_loss_clip": 1.02596474, "balance_loss_mlp": 1.03316736, "epoch": 0.08495415601984067, "flos": 17091380574720.0, "grad_norm": 2.4888670092824627, "language_loss": 0.88898432, "learning_rate": 3.9293688407542715e-06, "loss": 0.91086364, "num_input_tokens_seen": 30094600, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.0078125, "step": 1413, "time_per_iteration": 2.515660524368286 }, { "auxiliary_loss_clip": 0.01138069, "auxiliary_loss_mlp": 0.01049608, "balance_loss_clip": 1.02165365, "balance_loss_mlp": 1.03670883, "epoch": 0.08501427927250864, "flos": 23147310142080.0, "grad_norm": 1.928976151337458, "language_loss": 0.88079464, "learning_rate": 3.929269284764526e-06, "loss": 0.9026714, "num_input_tokens_seen": 30114475, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.015625, "step": 1414, "time_per_iteration": 2.4387221336364746 }, { "auxiliary_loss_clip": 0.01138837, "auxiliary_loss_mlp": 0.01055666, "balance_loss_clip": 1.02861762, "balance_loss_mlp": 1.03549552, "epoch": 0.08507440252517662, "flos": 19062437091840.0, "grad_norm": 1.8104022752795743, "language_loss": 0.77125359, "learning_rate": 3.929169659924123e-06, "loss": 0.79319859, "num_input_tokens_seen": 30133350, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.03125, "step": 1415, "time_per_iteration": 2.4585282802581787 }, { "auxiliary_loss_clip": 0.01136626, "auxiliary_loss_mlp": 0.01053604, "balance_loss_clip": 1.02770007, "balance_loss_mlp": 1.03540778, "epoch": 0.08513452577784458, "flos": 60180137775360.0, "grad_norm": 1.7518766502615744, "language_loss": 0.70400184, "learning_rate": 3.929069966236617e-06, "loss": 0.72590417, "num_input_tokens_seen": 30159005, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 1.015625, "step": 1416, "time_per_iteration": 2.7807602882385254 }, { "auxiliary_loss_clip": 0.01142408, "auxiliary_loss_mlp": 0.01058259, "balance_loss_clip": 1.02874279, "balance_loss_mlp": 1.03745627, "epoch": 0.08519464903051255, "flos": 27307246348800.0, "grad_norm": 2.068740206450198, "language_loss": 0.74673724, "learning_rate": 3.928970203705565e-06, "loss": 0.76874387, "num_input_tokens_seen": 30179450, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.046875, "step": 1417, "time_per_iteration": 2.503241539001465 }, { "auxiliary_loss_clip": 0.01135854, "auxiliary_loss_mlp": 0.01048919, "balance_loss_clip": 1.02104759, "balance_loss_mlp": 1.03411698, "epoch": 0.08525477228318051, "flos": 20265404497920.0, "grad_norm": 2.8020629614021364, "language_loss": 0.82518953, "learning_rate": 3.92887037233453e-06, "loss": 0.84703726, "num_input_tokens_seen": 30197235, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.015625, "step": 1418, "time_per_iteration": 2.4216010570526123 }, { "auxiliary_loss_clip": 0.01036904, "auxiliary_loss_mlp": 0.01013136, "balance_loss_clip": 1.00758064, "balance_loss_mlp": 1.00450683, "epoch": 0.08531489553584849, "flos": 67611923268480.0, "grad_norm": 0.892372631416078, "language_loss": 0.56662297, "learning_rate": 3.928770472127073e-06, "loss": 0.58712339, "num_input_tokens_seen": 30257410, "router_z_loss_clip": 0.05566406, "router_z_loss_mlp": 0.32421875, "step": 1419, "time_per_iteration": 3.0361785888671875 }, { "auxiliary_loss_clip": 0.01135059, "auxiliary_loss_mlp": 0.010583, "balance_loss_clip": 1.03101301, "balance_loss_mlp": 1.03346896, "epoch": 0.08537501878851646, "flos": 27525732837120.0, "grad_norm": 2.225891915285972, "language_loss": 0.69978249, "learning_rate": 3.928670503086758e-06, "loss": 0.72171611, "num_input_tokens_seen": 30277865, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.015625, "step": 1420, "time_per_iteration": 2.5303280353546143 }, { "auxiliary_loss_clip": 0.011341, "auxiliary_loss_mlp": 0.01041267, "balance_loss_clip": 1.01250148, "balance_loss_mlp": 1.0326556, "epoch": 0.08543514204118442, "flos": 22783131083520.0, "grad_norm": 1.5346284285593206, "language_loss": 0.88313144, "learning_rate": 3.9285704652171545e-06, "loss": 0.90488505, "num_input_tokens_seen": 30298545, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.015625, "step": 1421, "time_per_iteration": 2.4855990409851074 }, { "auxiliary_loss_clip": 0.01035216, "auxiliary_loss_mlp": 0.01009874, "balance_loss_clip": 1.00470078, "balance_loss_mlp": 1.00329804, "epoch": 0.0854952652938524, "flos": 60987362520960.0, "grad_norm": 0.8049145610308904, "language_loss": 0.63468266, "learning_rate": 3.9284703585218324e-06, "loss": 0.65513355, "num_input_tokens_seen": 30361725, "router_z_loss_clip": 0.05175781, "router_z_loss_mlp": 0.3203125, "step": 1422, "time_per_iteration": 3.057037115097046 }, { "auxiliary_loss_clip": 0.01132589, "auxiliary_loss_mlp": 0.01055404, "balance_loss_clip": 1.0286057, "balance_loss_mlp": 1.03614104, "epoch": 0.08555538854652037, "flos": 28036791452160.0, "grad_norm": 3.313427635387682, "language_loss": 0.83097607, "learning_rate": 3.928370183004363e-06, "loss": 0.85285604, "num_input_tokens_seen": 30382180, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.96484375, "step": 1423, "time_per_iteration": 2.525609016418457 }, { "auxiliary_loss_clip": 0.01138477, "auxiliary_loss_mlp": 0.01063673, "balance_loss_clip": 1.03676784, "balance_loss_mlp": 1.03663898, "epoch": 0.08561551179918833, "flos": 23508277355520.0, "grad_norm": 1.6560125375036239, "language_loss": 0.75101602, "learning_rate": 3.9282699386683236e-06, "loss": 0.77303749, "num_input_tokens_seen": 30402980, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.015625, "step": 1424, "time_per_iteration": 2.5153520107269287 }, { "auxiliary_loss_clip": 0.01137275, "auxiliary_loss_mlp": 0.01058406, "balance_loss_clip": 1.03109503, "balance_loss_mlp": 1.0375545, "epoch": 0.08567563505185631, "flos": 17926084293120.0, "grad_norm": 1.8755866914873893, "language_loss": 0.76020384, "learning_rate": 3.928169625517289e-06, "loss": 0.78216064, "num_input_tokens_seen": 30420800, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.99609375, "step": 1425, "time_per_iteration": 2.46828293800354 }, { "auxiliary_loss_clip": 0.01134093, "auxiliary_loss_mlp": 0.01046632, "balance_loss_clip": 1.02071571, "balance_loss_mlp": 1.03487504, "epoch": 0.08573575830452428, "flos": 19718490049920.0, "grad_norm": 2.9193753758221637, "language_loss": 0.93008298, "learning_rate": 3.9280692435548405e-06, "loss": 0.95189023, "num_input_tokens_seen": 30439620, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.9921875, "step": 1426, "time_per_iteration": 2.4237146377563477 }, { "auxiliary_loss_clip": 0.01141535, "auxiliary_loss_mlp": 0.01061609, "balance_loss_clip": 1.03209257, "balance_loss_mlp": 1.03872645, "epoch": 0.08579588155719224, "flos": 17930587858560.0, "grad_norm": 2.0509856314306787, "language_loss": 0.75465858, "learning_rate": 3.927968792784561e-06, "loss": 0.77669007, "num_input_tokens_seen": 30457300, "router_z_loss_clip": 0.29492188, "router_z_loss_mlp": 1.03125, "step": 1427, "time_per_iteration": 2.5031518936157227 }, { "auxiliary_loss_clip": 0.0113477, "auxiliary_loss_mlp": 0.01049683, "balance_loss_clip": 1.02349269, "balance_loss_mlp": 1.03487051, "epoch": 0.08585600480986022, "flos": 16032429993600.0, "grad_norm": 2.3071386430294982, "language_loss": 0.82328194, "learning_rate": 3.927868273210033e-06, "loss": 0.84512639, "num_input_tokens_seen": 30471580, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.0, "step": 1428, "time_per_iteration": 2.5303401947021484 }, { "auxiliary_loss_clip": 0.01142845, "auxiliary_loss_mlp": 0.0106289, "balance_loss_clip": 1.03399324, "balance_loss_mlp": 1.03674901, "epoch": 0.08591612806252819, "flos": 28656185616000.0, "grad_norm": 2.24419618106378, "language_loss": 0.79911095, "learning_rate": 3.927767684834847e-06, "loss": 0.8211683, "num_input_tokens_seen": 30492720, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0625, "step": 1429, "time_per_iteration": 2.5428974628448486 }, { "auxiliary_loss_clip": 0.01141666, "auxiliary_loss_mlp": 0.01056747, "balance_loss_clip": 1.02868533, "balance_loss_mlp": 1.03754735, "epoch": 0.08597625131519615, "flos": 20958081338880.0, "grad_norm": 2.76991814960215, "language_loss": 0.88487703, "learning_rate": 3.9276670276625894e-06, "loss": 0.90686119, "num_input_tokens_seen": 30509535, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.0390625, "step": 1430, "time_per_iteration": 2.4304347038269043 }, { "auxiliary_loss_clip": 0.01137127, "auxiliary_loss_mlp": 0.01052006, "balance_loss_clip": 1.02481472, "balance_loss_mlp": 1.03735805, "epoch": 0.08603637456786412, "flos": 23255296577280.0, "grad_norm": 1.6513236082284355, "language_loss": 0.81535912, "learning_rate": 3.927566301696856e-06, "loss": 0.83725047, "num_input_tokens_seen": 30529490, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.0, "step": 1431, "time_per_iteration": 2.4605906009674072 }, { "auxiliary_loss_clip": 0.01136667, "auxiliary_loss_mlp": 0.01054383, "balance_loss_clip": 1.02739429, "balance_loss_mlp": 1.03375912, "epoch": 0.0860964978205321, "flos": 28692914232960.0, "grad_norm": 1.9114593628809293, "language_loss": 0.77429157, "learning_rate": 3.927465506941238e-06, "loss": 0.79620206, "num_input_tokens_seen": 30550205, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.03125, "step": 1432, "time_per_iteration": 2.466153860092163 }, { "auxiliary_loss_clip": 0.01136558, "auxiliary_loss_mlp": 0.01058598, "balance_loss_clip": 1.02967751, "balance_loss_mlp": 1.03431463, "epoch": 0.08615662107320006, "flos": 19317372906240.0, "grad_norm": 2.704543968863709, "language_loss": 0.72969025, "learning_rate": 3.927364643399335e-06, "loss": 0.75164181, "num_input_tokens_seen": 30568830, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0234375, "step": 1433, "time_per_iteration": 2.426859140396118 }, { "auxiliary_loss_clip": 0.0114178, "auxiliary_loss_mlp": 0.01059604, "balance_loss_clip": 1.02967048, "balance_loss_mlp": 1.03753674, "epoch": 0.08621674432586802, "flos": 15850776856320.0, "grad_norm": 2.5010258393633356, "language_loss": 0.85818481, "learning_rate": 3.927263711074745e-06, "loss": 0.8801986, "num_input_tokens_seen": 30585730, "router_z_loss_clip": 0.29882812, "router_z_loss_mlp": 1.0390625, "step": 1434, "time_per_iteration": 2.3832969665527344 }, { "auxiliary_loss_clip": 0.0113731, "auxiliary_loss_mlp": 0.01055979, "balance_loss_clip": 1.02808404, "balance_loss_mlp": 1.03510058, "epoch": 0.086276867578536, "flos": 14099777838720.0, "grad_norm": 2.47929626062069, "language_loss": 0.78560674, "learning_rate": 3.927162709971072e-06, "loss": 0.8075397, "num_input_tokens_seen": 30603180, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.0234375, "step": 1435, "time_per_iteration": 2.4245572090148926 }, { "auxiliary_loss_clip": 0.01036822, "auxiliary_loss_mlp": 0.01042827, "balance_loss_clip": 1.03898871, "balance_loss_mlp": 1.00600958, "epoch": 0.08633699083120397, "flos": 70181250710400.0, "grad_norm": 0.923907410816164, "language_loss": 0.57990175, "learning_rate": 3.927061640091918e-06, "loss": 0.60069823, "num_input_tokens_seen": 30668895, "router_z_loss_clip": 0.03833008, "router_z_loss_mlp": 0.30859375, "step": 1436, "time_per_iteration": 4.62123966217041 }, { "auxiliary_loss_clip": 0.01136609, "auxiliary_loss_mlp": 0.01054357, "balance_loss_clip": 1.02512634, "balance_loss_mlp": 1.0350647, "epoch": 0.08639711408387193, "flos": 30297592275840.0, "grad_norm": 2.785802479640344, "language_loss": 0.68792832, "learning_rate": 3.926960501440891e-06, "loss": 0.70983791, "num_input_tokens_seen": 30688955, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.015625, "step": 1437, "time_per_iteration": 2.506444215774536 }, { "auxiliary_loss_clip": 0.01136806, "auxiliary_loss_mlp": 0.0104706, "balance_loss_clip": 1.01931965, "balance_loss_mlp": 1.0348177, "epoch": 0.08645723733653991, "flos": 20296791676800.0, "grad_norm": 2.169010760070846, "language_loss": 0.72614551, "learning_rate": 3.9268592940216014e-06, "loss": 0.74798417, "num_input_tokens_seen": 30706095, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.015625, "step": 1438, "time_per_iteration": 3.8132054805755615 }, { "auxiliary_loss_clip": 0.01132794, "auxiliary_loss_mlp": 0.0104806, "balance_loss_clip": 1.01983142, "balance_loss_mlp": 1.03471184, "epoch": 0.08651736058920788, "flos": 32889195031680.0, "grad_norm": 1.600592663775302, "language_loss": 0.64091539, "learning_rate": 3.9267580178376596e-06, "loss": 0.6627239, "num_input_tokens_seen": 30729025, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 0.98046875, "step": 1439, "time_per_iteration": 2.560499906539917 }, { "auxiliary_loss_clip": 0.01137702, "auxiliary_loss_mlp": 0.01049448, "balance_loss_clip": 1.02102852, "balance_loss_mlp": 1.03593493, "epoch": 0.08657748384187584, "flos": 22636286438400.0, "grad_norm": 2.5636767582097706, "language_loss": 0.87194371, "learning_rate": 3.92665667289268e-06, "loss": 0.89381528, "num_input_tokens_seen": 30746155, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.015625, "step": 1440, "time_per_iteration": 5.246348857879639 }, { "auxiliary_loss_clip": 0.01142193, "auxiliary_loss_mlp": 0.01058538, "balance_loss_clip": 1.02762711, "balance_loss_mlp": 1.03654242, "epoch": 0.08663760709454381, "flos": 23657286504960.0, "grad_norm": 3.2939192965722217, "language_loss": 0.8352201, "learning_rate": 3.92655525919028e-06, "loss": 0.85722744, "num_input_tokens_seen": 30761410, "router_z_loss_clip": 0.30859375, "router_z_loss_mlp": 1.0546875, "step": 1441, "time_per_iteration": 2.458915948867798 }, { "auxiliary_loss_clip": 0.01033196, "auxiliary_loss_mlp": 0.01004597, "balance_loss_clip": 1.00054348, "balance_loss_mlp": 1.00218558, "epoch": 0.08669773034721179, "flos": 62683688482560.0, "grad_norm": 0.8420296727267951, "language_loss": 0.60429287, "learning_rate": 3.926453776734078e-06, "loss": 0.62467074, "num_input_tokens_seen": 30823010, "router_z_loss_clip": 0.04052734, "router_z_loss_mlp": 0.31054688, "step": 1442, "time_per_iteration": 3.161848783493042 }, { "auxiliary_loss_clip": 0.01139796, "auxiliary_loss_mlp": 0.01052902, "balance_loss_clip": 1.02550721, "balance_loss_mlp": 1.03469789, "epoch": 0.08675785359987975, "flos": 20666451818880.0, "grad_norm": 2.652059476450735, "language_loss": 0.78552687, "learning_rate": 3.9263522255276965e-06, "loss": 0.80745387, "num_input_tokens_seen": 30841980, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.0546875, "step": 1443, "time_per_iteration": 2.423205852508545 }, { "auxiliary_loss_clip": 0.01135757, "auxiliary_loss_mlp": 0.01049801, "balance_loss_clip": 1.02301478, "balance_loss_mlp": 1.03341126, "epoch": 0.08681797685254772, "flos": 26939960179200.0, "grad_norm": 1.604042055689223, "language_loss": 0.82368612, "learning_rate": 3.9262506055747596e-06, "loss": 0.84554166, "num_input_tokens_seen": 30863280, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.0234375, "step": 1444, "time_per_iteration": 2.4947876930236816 }, { "auxiliary_loss_clip": 0.01139058, "auxiliary_loss_mlp": 0.01054381, "balance_loss_clip": 1.02591348, "balance_loss_mlp": 1.03583121, "epoch": 0.0868781001052157, "flos": 17711856990720.0, "grad_norm": 2.8698712932591914, "language_loss": 0.87018931, "learning_rate": 3.926148916878893e-06, "loss": 0.8921237, "num_input_tokens_seen": 30881710, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 1.03125, "step": 1445, "time_per_iteration": 2.438837766647339 }, { "auxiliary_loss_clip": 0.01140454, "auxiliary_loss_mlp": 0.01055732, "balance_loss_clip": 1.02794433, "balance_loss_mlp": 1.0385077, "epoch": 0.08693822335788366, "flos": 19895639621760.0, "grad_norm": 1.8482243248422658, "language_loss": 0.81103694, "learning_rate": 3.926047159443727e-06, "loss": 0.83299881, "num_input_tokens_seen": 30900225, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0234375, "step": 1446, "time_per_iteration": 2.46681547164917 }, { "auxiliary_loss_clip": 0.01033918, "auxiliary_loss_mlp": 0.01006089, "balance_loss_clip": 1.00213134, "balance_loss_mlp": 1.00254512, "epoch": 0.08699834661055163, "flos": 67020878995200.0, "grad_norm": 0.7253279505632818, "language_loss": 0.54759985, "learning_rate": 3.925945333272891e-06, "loss": 0.56799996, "num_input_tokens_seen": 30959580, "router_z_loss_clip": 0.03955078, "router_z_loss_mlp": 0.31445312, "step": 1447, "time_per_iteration": 3.1271657943725586 }, { "auxiliary_loss_clip": 0.01134434, "auxiliary_loss_mlp": 0.0105079, "balance_loss_clip": 1.02214408, "balance_loss_mlp": 1.03559732, "epoch": 0.0870584698632196, "flos": 13479650536320.0, "grad_norm": 2.368430375578426, "language_loss": 0.84644473, "learning_rate": 3.925843438370021e-06, "loss": 0.86829698, "num_input_tokens_seen": 30976775, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 0.984375, "step": 1448, "time_per_iteration": 2.5329737663269043 }, { "auxiliary_loss_clip": 0.01140375, "auxiliary_loss_mlp": 0.01052026, "balance_loss_clip": 1.02382088, "balance_loss_mlp": 1.03608632, "epoch": 0.08711859311588757, "flos": 16106096695680.0, "grad_norm": 2.6427047070415206, "language_loss": 0.80531889, "learning_rate": 3.925741474738752e-06, "loss": 0.82724291, "num_input_tokens_seen": 30990495, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.046875, "step": 1449, "time_per_iteration": 2.3928134441375732 }, { "auxiliary_loss_clip": 0.01135613, "auxiliary_loss_mlp": 0.01043545, "balance_loss_clip": 1.01903582, "balance_loss_mlp": 1.03557396, "epoch": 0.08717871636855554, "flos": 38470829512320.0, "grad_norm": 1.5675969670229246, "language_loss": 0.71181607, "learning_rate": 3.925639442382724e-06, "loss": 0.73360765, "num_input_tokens_seen": 31014080, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 1.0, "step": 1450, "time_per_iteration": 2.5841941833496094 }, { "auxiliary_loss_clip": 0.01136972, "auxiliary_loss_mlp": 0.01054337, "balance_loss_clip": 1.0267638, "balance_loss_mlp": 1.03637421, "epoch": 0.0872388396212235, "flos": 17599681192320.0, "grad_norm": 1.771054780384107, "language_loss": 0.83204961, "learning_rate": 3.925537341305578e-06, "loss": 0.85396278, "num_input_tokens_seen": 31031210, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.0078125, "step": 1451, "time_per_iteration": 2.3957407474517822 }, { "auxiliary_loss_clip": 0.01133863, "auxiliary_loss_mlp": 0.0105886, "balance_loss_clip": 1.03336096, "balance_loss_mlp": 1.03573465, "epoch": 0.08729896287389148, "flos": 25258368677760.0, "grad_norm": 2.1923603807858347, "language_loss": 0.74339652, "learning_rate": 3.925435171510957e-06, "loss": 0.76532376, "num_input_tokens_seen": 31049710, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.984375, "step": 1452, "time_per_iteration": 2.4761288166046143 }, { "auxiliary_loss_clip": 0.0113897, "auxiliary_loss_mlp": 0.0105538, "balance_loss_clip": 1.02777123, "balance_loss_mlp": 1.03575897, "epoch": 0.08735908612655945, "flos": 15631557229440.0, "grad_norm": 3.009200128085401, "language_loss": 0.79649633, "learning_rate": 3.925332933002507e-06, "loss": 0.81843984, "num_input_tokens_seen": 31066160, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 1.03125, "step": 1453, "time_per_iteration": 2.4006617069244385 }, { "auxiliary_loss_clip": 0.01135753, "auxiliary_loss_mlp": 0.01051944, "balance_loss_clip": 1.02537227, "balance_loss_mlp": 1.03667951, "epoch": 0.08741920937922741, "flos": 20338617352320.0, "grad_norm": 1.875711402079848, "language_loss": 0.70716834, "learning_rate": 3.925230625783877e-06, "loss": 0.72904533, "num_input_tokens_seen": 31085270, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.9921875, "step": 1454, "time_per_iteration": 2.452676773071289 }, { "auxiliary_loss_clip": 0.01034214, "auxiliary_loss_mlp": 0.0101123, "balance_loss_clip": 1.00710583, "balance_loss_mlp": 1.00364447, "epoch": 0.08747933263189539, "flos": 62816252515200.0, "grad_norm": 0.7824902349415341, "language_loss": 0.58511788, "learning_rate": 3.925128249858719e-06, "loss": 0.60557228, "num_input_tokens_seen": 31148445, "router_z_loss_clip": 0.04125977, "router_z_loss_mlp": 0.3046875, "step": 1455, "time_per_iteration": 3.0468578338623047 }, { "auxiliary_loss_clip": 0.01134979, "auxiliary_loss_mlp": 0.01048629, "balance_loss_clip": 1.02142525, "balance_loss_mlp": 1.03400826, "epoch": 0.08753945588456336, "flos": 33034503576960.0, "grad_norm": 1.5610199804777385, "language_loss": 0.77557188, "learning_rate": 3.925025805230685e-06, "loss": 0.79740798, "num_input_tokens_seen": 31168770, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.0078125, "step": 1456, "time_per_iteration": 2.5630991458892822 }, { "auxiliary_loss_clip": 0.01132645, "auxiliary_loss_mlp": 0.01054823, "balance_loss_clip": 1.02566481, "balance_loss_mlp": 1.03343987, "epoch": 0.08759957913723132, "flos": 35545911206400.0, "grad_norm": 2.3625478373839406, "language_loss": 0.71963835, "learning_rate": 3.924923291903433e-06, "loss": 0.74151307, "num_input_tokens_seen": 31189270, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 0.9921875, "step": 1457, "time_per_iteration": 2.551595449447632 }, { "auxiliary_loss_clip": 0.01130536, "auxiliary_loss_mlp": 0.01044576, "balance_loss_clip": 1.01885056, "balance_loss_mlp": 1.03258061, "epoch": 0.0876597023898993, "flos": 23910092726400.0, "grad_norm": 1.5815599312414572, "language_loss": 0.86436832, "learning_rate": 3.924820709880619e-06, "loss": 0.88611948, "num_input_tokens_seen": 31210385, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.98046875, "step": 1458, "time_per_iteration": 2.4821531772613525 }, { "auxiliary_loss_clip": 0.01140857, "auxiliary_loss_mlp": 0.0104819, "balance_loss_clip": 1.02134418, "balance_loss_mlp": 1.03808141, "epoch": 0.08771982564256726, "flos": 18113043957120.0, "grad_norm": 1.6349072283959376, "language_loss": 0.8053205, "learning_rate": 3.924718059165906e-06, "loss": 0.82721102, "num_input_tokens_seen": 31229745, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.03125, "step": 1459, "time_per_iteration": 2.4148335456848145 }, { "auxiliary_loss_clip": 0.01137273, "auxiliary_loss_mlp": 0.01055503, "balance_loss_clip": 1.02746463, "balance_loss_mlp": 1.03450203, "epoch": 0.08777994889523523, "flos": 17711054029440.0, "grad_norm": 2.099146642925664, "language_loss": 0.84267873, "learning_rate": 3.924615339762956e-06, "loss": 0.8646065, "num_input_tokens_seen": 31248280, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.0234375, "step": 1460, "time_per_iteration": 2.470891237258911 }, { "auxiliary_loss_clip": 0.01130985, "auxiliary_loss_mlp": 0.01051176, "balance_loss_clip": 1.02502179, "balance_loss_mlp": 1.03299022, "epoch": 0.0878400721479032, "flos": 12819198746880.0, "grad_norm": 2.6593803727230347, "language_loss": 0.81124723, "learning_rate": 3.924512551675435e-06, "loss": 0.83306885, "num_input_tokens_seen": 31262190, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.98046875, "step": 1461, "time_per_iteration": 2.421342372894287 }, { "auxiliary_loss_clip": 0.01138365, "auxiliary_loss_mlp": 0.01052623, "balance_loss_clip": 1.02715981, "balance_loss_mlp": 1.03707671, "epoch": 0.08790019540057117, "flos": 26391579454080.0, "grad_norm": 1.7269677394716834, "language_loss": 0.76201111, "learning_rate": 3.924409694907011e-06, "loss": 0.783921, "num_input_tokens_seen": 31283690, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 1.015625, "step": 1462, "time_per_iteration": 2.6441681385040283 }, { "auxiliary_loss_clip": 0.01139179, "auxiliary_loss_mlp": 0.01055463, "balance_loss_clip": 1.02630436, "balance_loss_mlp": 1.03623247, "epoch": 0.08796031865323914, "flos": 19133066505600.0, "grad_norm": 1.7974264920681688, "language_loss": 0.74233687, "learning_rate": 3.924306769461356e-06, "loss": 0.7642833, "num_input_tokens_seen": 31302505, "router_z_loss_clip": 0.29101562, "router_z_loss_mlp": 1.03125, "step": 1463, "time_per_iteration": 2.507075309753418 }, { "auxiliary_loss_clip": 0.01137699, "auxiliary_loss_mlp": 0.01051707, "balance_loss_clip": 1.02271533, "balance_loss_mlp": 1.03376389, "epoch": 0.0880204419059071, "flos": 26063186405760.0, "grad_norm": 1.8813185484463697, "language_loss": 0.83247638, "learning_rate": 3.924203775342142e-06, "loss": 0.85437036, "num_input_tokens_seen": 31323070, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0390625, "step": 1464, "time_per_iteration": 2.582576274871826 }, { "auxiliary_loss_clip": 0.01135477, "auxiliary_loss_mlp": 0.01055543, "balance_loss_clip": 1.02949548, "balance_loss_mlp": 1.03474152, "epoch": 0.08808056515857508, "flos": 22376881969920.0, "grad_norm": 1.893011339821771, "language_loss": 0.78369987, "learning_rate": 3.924100712553046e-06, "loss": 0.80561006, "num_input_tokens_seen": 31341880, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 1.0078125, "step": 1465, "time_per_iteration": 2.4433510303497314 }, { "auxiliary_loss_clip": 0.01138777, "auxiliary_loss_mlp": 0.01050684, "balance_loss_clip": 1.02324176, "balance_loss_mlp": 1.03619003, "epoch": 0.08814068841124305, "flos": 23184178404480.0, "grad_norm": 2.4569350502347165, "language_loss": 0.84995323, "learning_rate": 3.923997581097744e-06, "loss": 0.87184787, "num_input_tokens_seen": 31361995, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.0234375, "step": 1466, "time_per_iteration": 2.514909267425537 }, { "auxiliary_loss_clip": 0.01137144, "auxiliary_loss_mlp": 0.01049688, "balance_loss_clip": 1.02200794, "balance_loss_mlp": 1.03487492, "epoch": 0.08820081166391101, "flos": 25154117758080.0, "grad_norm": 2.1428941119918825, "language_loss": 0.84030366, "learning_rate": 3.923894380979917e-06, "loss": 0.86217201, "num_input_tokens_seen": 31381515, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.015625, "step": 1467, "time_per_iteration": 2.4532175064086914 }, { "auxiliary_loss_clip": 0.01136589, "auxiliary_loss_mlp": 0.01050094, "balance_loss_clip": 1.02235413, "balance_loss_mlp": 1.03442502, "epoch": 0.08826093491657899, "flos": 22230735552000.0, "grad_norm": 1.8526771256998313, "language_loss": 0.75296938, "learning_rate": 3.9237911122032485e-06, "loss": 0.7748363, "num_input_tokens_seen": 31400345, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0234375, "step": 1468, "time_per_iteration": 2.4733340740203857 }, { "auxiliary_loss_clip": 0.01134615, "auxiliary_loss_mlp": 0.01045144, "balance_loss_clip": 1.0202769, "balance_loss_mlp": 1.03503835, "epoch": 0.08832105816924696, "flos": 22125751493760.0, "grad_norm": 5.546335350124982, "language_loss": 0.8053264, "learning_rate": 3.923687774771424e-06, "loss": 0.827124, "num_input_tokens_seen": 31419620, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.99609375, "step": 1469, "time_per_iteration": 2.4505934715270996 }, { "auxiliary_loss_clip": 0.01139914, "auxiliary_loss_mlp": 0.01053373, "balance_loss_clip": 1.02658677, "balance_loss_mlp": 1.03651309, "epoch": 0.08838118142191492, "flos": 17565536016000.0, "grad_norm": 1.989153980440257, "language_loss": 0.77890998, "learning_rate": 3.923584368688132e-06, "loss": 0.80084276, "num_input_tokens_seen": 31437970, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 1.03125, "step": 1470, "time_per_iteration": 2.4670047760009766 }, { "auxiliary_loss_clip": 0.01132672, "auxiliary_loss_mlp": 0.01050743, "balance_loss_clip": 1.0239327, "balance_loss_mlp": 1.03389835, "epoch": 0.0884413046745829, "flos": 20776148910720.0, "grad_norm": 1.8836256182417797, "language_loss": 0.83851361, "learning_rate": 3.923480893957061e-06, "loss": 0.86034775, "num_input_tokens_seen": 31457040, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.98828125, "step": 1471, "time_per_iteration": 2.4457244873046875 }, { "auxiliary_loss_clip": 0.01130809, "auxiliary_loss_mlp": 0.01046425, "balance_loss_clip": 1.02320361, "balance_loss_mlp": 1.0354538, "epoch": 0.08850142792725087, "flos": 22124424862080.0, "grad_norm": 5.684584918680441, "language_loss": 0.83179504, "learning_rate": 3.923377350581905e-06, "loss": 0.85356736, "num_input_tokens_seen": 31477520, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.953125, "step": 1472, "time_per_iteration": 2.4930155277252197 }, { "auxiliary_loss_clip": 0.01135368, "auxiliary_loss_mlp": 0.0104367, "balance_loss_clip": 1.01808798, "balance_loss_mlp": 1.03654361, "epoch": 0.08856155117991883, "flos": 22417660304640.0, "grad_norm": 2.271622402276832, "language_loss": 0.82474113, "learning_rate": 3.923273738566359e-06, "loss": 0.84653151, "num_input_tokens_seen": 31495575, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.98828125, "step": 1473, "time_per_iteration": 2.4435055255889893 }, { "auxiliary_loss_clip": 0.01136878, "auxiliary_loss_mlp": 0.01047921, "balance_loss_clip": 1.02318478, "balance_loss_mlp": 1.03624725, "epoch": 0.0886216744325868, "flos": 29935647544320.0, "grad_norm": 1.5623027892790873, "language_loss": 0.78689879, "learning_rate": 3.92317005791412e-06, "loss": 0.80874676, "num_input_tokens_seen": 31520020, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 1.0078125, "step": 1474, "time_per_iteration": 2.5359132289886475 }, { "auxiliary_loss_clip": 0.01132754, "auxiliary_loss_mlp": 0.01048824, "balance_loss_clip": 1.02194262, "balance_loss_mlp": 1.03589225, "epoch": 0.08868179768525478, "flos": 23981839303680.0, "grad_norm": 1.6698744687384766, "language_loss": 0.79016858, "learning_rate": 3.923066308628889e-06, "loss": 0.81198436, "num_input_tokens_seen": 31539265, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.96875, "step": 1475, "time_per_iteration": 2.441384792327881 }, { "auxiliary_loss_clip": 0.01133227, "auxiliary_loss_mlp": 0.01045926, "balance_loss_clip": 1.02074862, "balance_loss_mlp": 1.03375602, "epoch": 0.08874192093792274, "flos": 43175934599040.0, "grad_norm": 1.5978715824027918, "language_loss": 0.73998678, "learning_rate": 3.922962490714368e-06, "loss": 0.76177835, "num_input_tokens_seen": 31563425, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.99609375, "step": 1476, "time_per_iteration": 4.0761120319366455 }, { "auxiliary_loss_clip": 0.01136907, "auxiliary_loss_mlp": 0.01049839, "balance_loss_clip": 1.02361321, "balance_loss_mlp": 1.03561664, "epoch": 0.0888020441905907, "flos": 32851104871680.0, "grad_norm": 1.8367264592435533, "language_loss": 0.74373507, "learning_rate": 3.922858604174262e-06, "loss": 0.76560253, "num_input_tokens_seen": 31584525, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.015625, "step": 1477, "time_per_iteration": 2.597036123275757 }, { "auxiliary_loss_clip": 0.0113435, "auxiliary_loss_mlp": 0.01055158, "balance_loss_clip": 1.02894378, "balance_loss_mlp": 1.03508937, "epoch": 0.08886216744325869, "flos": 23148217837440.0, "grad_norm": 1.8871903181689216, "language_loss": 0.86721641, "learning_rate": 3.922754649012279e-06, "loss": 0.88911152, "num_input_tokens_seen": 31603325, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9921875, "step": 1478, "time_per_iteration": 3.840590476989746 }, { "auxiliary_loss_clip": 0.0113549, "auxiliary_loss_mlp": 0.01053253, "balance_loss_clip": 1.02746797, "balance_loss_mlp": 1.03530025, "epoch": 0.08892229069592665, "flos": 23330464467840.0, "grad_norm": 3.261643036609131, "language_loss": 0.77389818, "learning_rate": 3.922650625232128e-06, "loss": 0.79578561, "num_input_tokens_seen": 31624820, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 1.0, "step": 1479, "time_per_iteration": 4.07110071182251 }, { "auxiliary_loss_clip": 0.01130919, "auxiliary_loss_mlp": 0.01043634, "balance_loss_clip": 1.01843274, "balance_loss_mlp": 1.03348565, "epoch": 0.08898241394859462, "flos": 26212579580160.0, "grad_norm": 2.419935582481106, "language_loss": 0.78363329, "learning_rate": 3.922546532837522e-06, "loss": 0.80537885, "num_input_tokens_seen": 31646080, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.9765625, "step": 1480, "time_per_iteration": 4.018751382827759 }, { "auxiliary_loss_clip": 0.01133847, "auxiliary_loss_mlp": 0.01052865, "balance_loss_clip": 1.02467179, "balance_loss_mlp": 1.03337443, "epoch": 0.0890425372012626, "flos": 23549474626560.0, "grad_norm": 2.081024641177727, "language_loss": 0.66308194, "learning_rate": 3.9224423718321756e-06, "loss": 0.68494904, "num_input_tokens_seen": 31665770, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.0078125, "step": 1481, "time_per_iteration": 2.4800703525543213 }, { "auxiliary_loss_clip": 0.01134249, "auxiliary_loss_mlp": 0.01047694, "balance_loss_clip": 1.02288699, "balance_loss_mlp": 1.03551579, "epoch": 0.08910266045393056, "flos": 23001687394560.0, "grad_norm": 1.864432448296234, "language_loss": 0.9653616, "learning_rate": 3.922338142219806e-06, "loss": 0.98718101, "num_input_tokens_seen": 31683805, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.98828125, "step": 1482, "time_per_iteration": -0.18274378776550293 }, { "auxiliary_loss_clip": 0.01137095, "auxiliary_loss_mlp": 0.01052594, "balance_loss_clip": 1.02611768, "balance_loss_mlp": 1.03577983, "epoch": 0.08916278370659853, "flos": 31935298331520.0, "grad_norm": 1.9234200730524673, "language_loss": 0.7877143, "learning_rate": 3.922233844004133e-06, "loss": 0.8096112, "num_input_tokens_seen": 31704630, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.015625, "step": 1483, "time_per_iteration": 2.5001602172851562 }, { "auxiliary_loss_clip": 0.01132379, "auxiliary_loss_mlp": 0.01056567, "balance_loss_clip": 1.03047252, "balance_loss_mlp": 1.03430367, "epoch": 0.08922290695926649, "flos": 17529435803520.0, "grad_norm": 2.3630293380418683, "language_loss": 0.85483754, "learning_rate": 3.922129477188879e-06, "loss": 0.87672698, "num_input_tokens_seen": 31723255, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.98046875, "step": 1484, "time_per_iteration": 2.4240922927856445 }, { "auxiliary_loss_clip": 0.01142654, "auxiliary_loss_mlp": 0.01049875, "balance_loss_clip": 1.02124047, "balance_loss_mlp": 1.03831923, "epoch": 0.08928303021193447, "flos": 32123689361280.0, "grad_norm": 1.6040066629630427, "language_loss": 0.80224192, "learning_rate": 3.922025041777768e-06, "loss": 0.82416725, "num_input_tokens_seen": 31747045, "router_z_loss_clip": 0.28710938, "router_z_loss_mlp": 1.0390625, "step": 1485, "time_per_iteration": 2.524641275405884 }, { "auxiliary_loss_clip": 0.01133156, "auxiliary_loss_mlp": 0.01051077, "balance_loss_clip": 1.02557778, "balance_loss_mlp": 1.03244472, "epoch": 0.08934315346460243, "flos": 22124180482560.0, "grad_norm": 2.0853469502904693, "language_loss": 0.82999718, "learning_rate": 3.921920537774528e-06, "loss": 0.85183954, "num_input_tokens_seen": 31766615, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 1.0078125, "step": 1486, "time_per_iteration": 2.4714694023132324 }, { "auxiliary_loss_clip": 0.01132548, "auxiliary_loss_mlp": 0.01056082, "balance_loss_clip": 1.02856851, "balance_loss_mlp": 1.03458941, "epoch": 0.0894032767172704, "flos": 22564470038400.0, "grad_norm": 1.6977251970071152, "language_loss": 0.76376575, "learning_rate": 3.921815965182887e-06, "loss": 0.78565204, "num_input_tokens_seen": 31785855, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.9765625, "step": 1487, "time_per_iteration": 2.472322940826416 }, { "auxiliary_loss_clip": 0.01135312, "auxiliary_loss_mlp": 0.01050471, "balance_loss_clip": 1.02255261, "balance_loss_mlp": 1.03422713, "epoch": 0.08946339996993838, "flos": 20192366200320.0, "grad_norm": 2.0317647278322477, "language_loss": 0.82573104, "learning_rate": 3.921711324006578e-06, "loss": 0.8475889, "num_input_tokens_seen": 31804210, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 1.0078125, "step": 1488, "time_per_iteration": 2.4572408199310303 }, { "auxiliary_loss_clip": 0.01132356, "auxiliary_loss_mlp": 0.01048558, "balance_loss_clip": 1.02412033, "balance_loss_mlp": 1.0337038, "epoch": 0.08952352322260634, "flos": 48358372060800.0, "grad_norm": 2.8944739791810865, "language_loss": 0.72003675, "learning_rate": 3.921606614249335e-06, "loss": 0.74184585, "num_input_tokens_seen": 31826150, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.98828125, "step": 1489, "time_per_iteration": 2.759321451187134 }, { "auxiliary_loss_clip": 0.01130591, "auxiliary_loss_mlp": 0.01049125, "balance_loss_clip": 1.02280331, "balance_loss_mlp": 1.03231263, "epoch": 0.08958364647527431, "flos": 31791805176960.0, "grad_norm": 1.7753833348466836, "language_loss": 0.89858687, "learning_rate": 3.921501835914894e-06, "loss": 0.92038399, "num_input_tokens_seen": 31848060, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.984375, "step": 1490, "time_per_iteration": 2.541839122772217 }, { "auxiliary_loss_clip": 0.01140402, "auxiliary_loss_mlp": 0.01056209, "balance_loss_clip": 1.02856421, "balance_loss_mlp": 1.03494847, "epoch": 0.08964376972794229, "flos": 23367053439360.0, "grad_norm": 2.3138583614972386, "language_loss": 0.73459613, "learning_rate": 3.921396989006997e-06, "loss": 0.75656223, "num_input_tokens_seen": 31870040, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0546875, "step": 1491, "time_per_iteration": 2.482553482055664 }, { "auxiliary_loss_clip": 0.01132926, "auxiliary_loss_mlp": 0.01044359, "balance_loss_clip": 1.01953959, "balance_loss_mlp": 1.03456628, "epoch": 0.08970389298061025, "flos": 23293666028160.0, "grad_norm": 1.9212727717432074, "language_loss": 0.76900983, "learning_rate": 3.9212920735293824e-06, "loss": 0.79078269, "num_input_tokens_seen": 31890400, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.984375, "step": 1492, "time_per_iteration": 2.4586904048919678 }, { "auxiliary_loss_clip": 0.01132489, "auxiliary_loss_mlp": 0.01048568, "balance_loss_clip": 1.02209187, "balance_loss_mlp": 1.03553343, "epoch": 0.08976401623327822, "flos": 33760417898880.0, "grad_norm": 2.0921066681155245, "language_loss": 0.70533705, "learning_rate": 3.921187089485796e-06, "loss": 0.72714764, "num_input_tokens_seen": 31913435, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.96875, "step": 1493, "time_per_iteration": 2.540811777114868 }, { "auxiliary_loss_clip": 0.01132349, "auxiliary_loss_mlp": 0.01048068, "balance_loss_clip": 1.02154422, "balance_loss_mlp": 1.03320432, "epoch": 0.08982413948594618, "flos": 23910302194560.0, "grad_norm": 1.8346011499961192, "language_loss": 0.86851084, "learning_rate": 3.921082036879985e-06, "loss": 0.89031506, "num_input_tokens_seen": 31932435, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.9921875, "step": 1494, "time_per_iteration": 2.462740421295166 }, { "auxiliary_loss_clip": 0.01133166, "auxiliary_loss_mlp": 0.01054281, "balance_loss_clip": 1.02751827, "balance_loss_mlp": 1.03466368, "epoch": 0.08988426273861416, "flos": 16836584405760.0, "grad_norm": 1.7800069294718281, "language_loss": 0.83029783, "learning_rate": 3.9209769157156976e-06, "loss": 0.85217232, "num_input_tokens_seen": 31950125, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.984375, "step": 1495, "time_per_iteration": 2.403963804244995 }, { "auxiliary_loss_clip": 0.0113773, "auxiliary_loss_mlp": 0.01058836, "balance_loss_clip": 1.03222859, "balance_loss_mlp": 1.03652573, "epoch": 0.08994438599128213, "flos": 14792489591040.0, "grad_norm": 1.8861377359210703, "language_loss": 0.69612455, "learning_rate": 3.920871725996685e-06, "loss": 0.7180903, "num_input_tokens_seen": 31968050, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 1.015625, "step": 1496, "time_per_iteration": 2.3890879154205322 }, { "auxiliary_loss_clip": 0.01132691, "auxiliary_loss_mlp": 0.01049185, "balance_loss_clip": 1.024616, "balance_loss_mlp": 1.0346992, "epoch": 0.09000450924395009, "flos": 17383359208320.0, "grad_norm": 1.665119356571217, "language_loss": 0.79898089, "learning_rate": 3.920766467726702e-06, "loss": 0.82079965, "num_input_tokens_seen": 31985675, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.9765625, "step": 1497, "time_per_iteration": 2.4362621307373047 }, { "auxiliary_loss_clip": 0.01135802, "auxiliary_loss_mlp": 0.01049072, "balance_loss_clip": 1.02345371, "balance_loss_mlp": 1.03321958, "epoch": 0.09006463249661807, "flos": 24279159375360.0, "grad_norm": 2.704740676644494, "language_loss": 0.8292343, "learning_rate": 3.920661140909505e-06, "loss": 0.85108304, "num_input_tokens_seen": 32005180, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 1.0234375, "step": 1498, "time_per_iteration": 2.4597747325897217 }, { "auxiliary_loss_clip": 0.01136483, "auxiliary_loss_mlp": 0.01056237, "balance_loss_clip": 1.0302614, "balance_loss_mlp": 1.03528881, "epoch": 0.09012475574928604, "flos": 13661094205440.0, "grad_norm": 4.45428008519945, "language_loss": 0.78773302, "learning_rate": 3.920555745548851e-06, "loss": 0.8096602, "num_input_tokens_seen": 32022970, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 1.015625, "step": 1499, "time_per_iteration": 2.419222593307495 }, { "auxiliary_loss_clip": 0.01130284, "auxiliary_loss_mlp": 0.01055989, "balance_loss_clip": 1.03046608, "balance_loss_mlp": 1.03425586, "epoch": 0.090184879001954, "flos": 23326728952320.0, "grad_norm": 1.7092410090117585, "language_loss": 0.93098229, "learning_rate": 3.920450281648503e-06, "loss": 0.95284498, "num_input_tokens_seen": 32043055, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9609375, "step": 1500, "time_per_iteration": 2.435533285140991 }, { "auxiliary_loss_clip": 0.01133888, "auxiliary_loss_mlp": 0.01047049, "balance_loss_clip": 1.02153802, "balance_loss_mlp": 1.03357148, "epoch": 0.09024500225462198, "flos": 23001582660480.0, "grad_norm": 2.196175652597993, "language_loss": 0.74589396, "learning_rate": 3.920344749212226e-06, "loss": 0.76770335, "num_input_tokens_seen": 32061900, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 1.0, "step": 1501, "time_per_iteration": 2.46028733253479 }, { "auxiliary_loss_clip": 0.01044632, "auxiliary_loss_mlp": 0.01025841, "balance_loss_clip": 1.02193058, "balance_loss_mlp": 1.00976682, "epoch": 0.09030512550728995, "flos": 62185966007040.0, "grad_norm": 0.7316882177862591, "language_loss": 0.58222729, "learning_rate": 3.920239148243783e-06, "loss": 0.60293198, "num_input_tokens_seen": 32122745, "router_z_loss_clip": 0.0390625, "router_z_loss_mlp": 0.34765625, "step": 1502, "time_per_iteration": 3.0684406757354736 }, { "auxiliary_loss_clip": 0.01127273, "auxiliary_loss_mlp": 0.01045086, "balance_loss_clip": 1.02217412, "balance_loss_mlp": 1.03078341, "epoch": 0.09036524875995791, "flos": 38799152737920.0, "grad_norm": 2.273966761436493, "language_loss": 0.69753504, "learning_rate": 3.920133478746944e-06, "loss": 0.71925861, "num_input_tokens_seen": 32145125, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.96484375, "step": 1503, "time_per_iteration": 2.6171352863311768 }, { "auxiliary_loss_clip": 0.01133858, "auxiliary_loss_mlp": 0.01047987, "balance_loss_clip": 1.02242839, "balance_loss_mlp": 1.03417444, "epoch": 0.09042537201262588, "flos": 21688987985280.0, "grad_norm": 2.231932536970297, "language_loss": 0.85978246, "learning_rate": 3.920027740725481e-06, "loss": 0.88160092, "num_input_tokens_seen": 32166255, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 1.0, "step": 1504, "time_per_iteration": 2.4569787979125977 }, { "auxiliary_loss_clip": 0.01140245, "auxiliary_loss_mlp": 0.01054718, "balance_loss_clip": 1.0257622, "balance_loss_mlp": 1.03655267, "epoch": 0.09048549526529386, "flos": 22266102625920.0, "grad_norm": 2.087997650270069, "language_loss": 0.72479331, "learning_rate": 3.919921934183167e-06, "loss": 0.7467429, "num_input_tokens_seen": 32184010, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0390625, "step": 1505, "time_per_iteration": 2.4890501499176025 }, { "auxiliary_loss_clip": 0.01132126, "auxiliary_loss_mlp": 0.01046802, "balance_loss_clip": 1.02039695, "balance_loss_mlp": 1.03433979, "epoch": 0.09054561851796182, "flos": 14610068403840.0, "grad_norm": 2.0171736463787093, "language_loss": 0.80757898, "learning_rate": 3.919816059123778e-06, "loss": 0.82936823, "num_input_tokens_seen": 32201635, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.9765625, "step": 1506, "time_per_iteration": 2.4429192543029785 }, { "auxiliary_loss_clip": 0.01132694, "auxiliary_loss_mlp": 0.01044174, "balance_loss_clip": 1.0197835, "balance_loss_mlp": 1.03535104, "epoch": 0.09060574177062979, "flos": 27634941169920.0, "grad_norm": 1.9727846762699803, "language_loss": 0.75965023, "learning_rate": 3.919710115551092e-06, "loss": 0.78141892, "num_input_tokens_seen": 32221940, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.97265625, "step": 1507, "time_per_iteration": 2.5400660037994385 }, { "auxiliary_loss_clip": 0.01042598, "auxiliary_loss_mlp": 0.01005991, "balance_loss_clip": 1.00198567, "balance_loss_mlp": 1.00916338, "epoch": 0.09066586502329776, "flos": 66082657495680.0, "grad_norm": 0.7293385187982612, "language_loss": 0.57651293, "learning_rate": 3.91960410346889e-06, "loss": 0.59699887, "num_input_tokens_seen": 32276495, "router_z_loss_clip": 0.04003906, "router_z_loss_mlp": 0.33398438, "step": 1508, "time_per_iteration": 2.967487335205078 }, { "auxiliary_loss_clip": 0.0113586, "auxiliary_loss_mlp": 0.0105616, "balance_loss_clip": 1.02919483, "balance_loss_mlp": 1.03609443, "epoch": 0.09072598827596573, "flos": 18915452801280.0, "grad_norm": 2.2190963044476137, "language_loss": 0.85160971, "learning_rate": 3.919498022880955e-06, "loss": 0.87352985, "num_input_tokens_seen": 32294130, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.99609375, "step": 1509, "time_per_iteration": 2.5288822650909424 }, { "auxiliary_loss_clip": 0.01141463, "auxiliary_loss_mlp": 0.01054211, "balance_loss_clip": 1.02641118, "balance_loss_mlp": 1.03611588, "epoch": 0.0907861115286337, "flos": 24820732385280.0, "grad_norm": 2.3260571756947472, "language_loss": 0.84302211, "learning_rate": 3.9193918737910735e-06, "loss": 0.86497879, "num_input_tokens_seen": 32313555, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.046875, "step": 1510, "time_per_iteration": 2.466231346130371 }, { "auxiliary_loss_clip": 0.01134825, "auxiliary_loss_mlp": 0.01050364, "balance_loss_clip": 1.02267158, "balance_loss_mlp": 1.03401542, "epoch": 0.09084623478130167, "flos": 21651770609280.0, "grad_norm": 1.912535516932508, "language_loss": 0.85478687, "learning_rate": 3.919285656203033e-06, "loss": 0.87663877, "num_input_tokens_seen": 32331430, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0078125, "step": 1511, "time_per_iteration": 2.4451420307159424 }, { "auxiliary_loss_clip": 0.01132187, "auxiliary_loss_mlp": 0.01048022, "balance_loss_clip": 1.02150965, "balance_loss_mlp": 1.03599679, "epoch": 0.09090635803396964, "flos": 27637943546880.0, "grad_norm": 1.7413602600900544, "language_loss": 0.85064685, "learning_rate": 3.919179370120624e-06, "loss": 0.87244892, "num_input_tokens_seen": 32353705, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.96484375, "step": 1512, "time_per_iteration": 2.4990103244781494 }, { "auxiliary_loss_clip": 0.0112873, "auxiliary_loss_mlp": 0.0104476, "balance_loss_clip": 1.02056026, "balance_loss_mlp": 1.03212404, "epoch": 0.0909664812866376, "flos": 17668355569920.0, "grad_norm": 2.479651232075728, "language_loss": 0.86426342, "learning_rate": 3.919073015547641e-06, "loss": 0.88599831, "num_input_tokens_seen": 32370520, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.96484375, "step": 1513, "time_per_iteration": 2.4611003398895264 }, { "auxiliary_loss_clip": 0.01134348, "auxiliary_loss_mlp": 0.01049628, "balance_loss_clip": 1.02377188, "balance_loss_mlp": 1.03542376, "epoch": 0.09102660453930557, "flos": 23950312479360.0, "grad_norm": 1.8659367863772227, "language_loss": 0.86158764, "learning_rate": 3.918966592487878e-06, "loss": 0.88342738, "num_input_tokens_seen": 32389105, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.98828125, "step": 1514, "time_per_iteration": 2.4910011291503906 }, { "auxiliary_loss_clip": 0.01134255, "auxiliary_loss_mlp": 0.01058063, "balance_loss_clip": 1.03387547, "balance_loss_mlp": 1.03549898, "epoch": 0.09108672779197355, "flos": 25811741727360.0, "grad_norm": 1.844292454397013, "language_loss": 0.90314281, "learning_rate": 3.918860100945134e-06, "loss": 0.92506593, "num_input_tokens_seen": 32408065, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.984375, "step": 1515, "time_per_iteration": 3.9674715995788574 }, { "auxiliary_loss_clip": 0.01134262, "auxiliary_loss_mlp": 0.01046491, "balance_loss_clip": 1.02043211, "balance_loss_mlp": 1.03435397, "epoch": 0.09114685104464151, "flos": 29638292561280.0, "grad_norm": 2.112813599939862, "language_loss": 0.85246992, "learning_rate": 3.9187535409232076e-06, "loss": 0.87427747, "num_input_tokens_seen": 32427225, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 1.0, "step": 1516, "time_per_iteration": 2.516740322113037 }, { "auxiliary_loss_clip": 0.01138227, "auxiliary_loss_mlp": 0.01050867, "balance_loss_clip": 1.02510583, "balance_loss_mlp": 1.03607202, "epoch": 0.09120697429730948, "flos": 33728227758720.0, "grad_norm": 1.4460689545829237, "language_loss": 0.80797648, "learning_rate": 3.918646912425904e-06, "loss": 0.82986748, "num_input_tokens_seen": 32450510, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 1.0234375, "step": 1517, "time_per_iteration": 3.999890089035034 }, { "auxiliary_loss_clip": 0.01140348, "auxiliary_loss_mlp": 0.01059854, "balance_loss_clip": 1.03282976, "balance_loss_mlp": 1.03763103, "epoch": 0.09126709754997746, "flos": 18400519025280.0, "grad_norm": 1.570144501988006, "language_loss": 0.77740484, "learning_rate": 3.918540215457027e-06, "loss": 0.79940683, "num_input_tokens_seen": 32468425, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.03125, "step": 1518, "time_per_iteration": 2.447437047958374 }, { "auxiliary_loss_clip": 0.01133414, "auxiliary_loss_mlp": 0.01051801, "balance_loss_clip": 1.02459717, "balance_loss_mlp": 1.03456867, "epoch": 0.09132722080264542, "flos": 22090838267520.0, "grad_norm": 1.6758766032308245, "language_loss": 0.86130202, "learning_rate": 3.918433450020386e-06, "loss": 0.88315415, "num_input_tokens_seen": 32487510, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 0.98828125, "step": 1519, "time_per_iteration": 3.8800432682037354 }, { "auxiliary_loss_clip": 0.01136126, "auxiliary_loss_mlp": 0.01051936, "balance_loss_clip": 1.02413607, "balance_loss_mlp": 1.03529108, "epoch": 0.09138734405531339, "flos": 21032062243200.0, "grad_norm": 2.3850002057706474, "language_loss": 0.72785783, "learning_rate": 3.9183266161197885e-06, "loss": 0.74973845, "num_input_tokens_seen": 32507250, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 1.0078125, "step": 1520, "time_per_iteration": 3.880931854248047 }, { "auxiliary_loss_clip": 0.01135703, "auxiliary_loss_mlp": 0.01054564, "balance_loss_clip": 1.02647829, "balance_loss_mlp": 1.03522754, "epoch": 0.09144746730798137, "flos": 20082913488000.0, "grad_norm": 2.5358907338691727, "language_loss": 0.85057628, "learning_rate": 3.91821971375905e-06, "loss": 0.87247896, "num_input_tokens_seen": 32526045, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 1.0, "step": 1521, "time_per_iteration": 2.469526767730713 }, { "auxiliary_loss_clip": 0.01136681, "auxiliary_loss_mlp": 0.0105437, "balance_loss_clip": 1.02826309, "balance_loss_mlp": 1.03459895, "epoch": 0.09150759056064933, "flos": 22777265975040.0, "grad_norm": 2.7776145419894873, "language_loss": 0.83937508, "learning_rate": 3.918112742941983e-06, "loss": 0.86128557, "num_input_tokens_seen": 32546575, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 1.015625, "step": 1522, "time_per_iteration": 2.5015242099761963 }, { "auxiliary_loss_clip": 0.01129337, "auxiliary_loss_mlp": 0.01053417, "balance_loss_clip": 1.0273335, "balance_loss_mlp": 1.03404737, "epoch": 0.0915677138133173, "flos": 27562950213120.0, "grad_norm": 1.9851311064106862, "language_loss": 0.81124741, "learning_rate": 3.9180057036724066e-06, "loss": 0.83307493, "num_input_tokens_seen": 32568795, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.953125, "step": 1523, "time_per_iteration": 2.5210797786712646 }, { "auxiliary_loss_clip": 0.01135162, "auxiliary_loss_mlp": 0.01050573, "balance_loss_clip": 1.02509856, "balance_loss_mlp": 1.03610897, "epoch": 0.09162783706598528, "flos": 17673836653440.0, "grad_norm": 2.434152104453912, "language_loss": 0.74915415, "learning_rate": 3.9178985959541406e-06, "loss": 0.77101147, "num_input_tokens_seen": 32587010, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9921875, "step": 1524, "time_per_iteration": 2.440109968185425 }, { "auxiliary_loss_clip": 0.01134508, "auxiliary_loss_mlp": 0.01055211, "balance_loss_clip": 1.02829385, "balance_loss_mlp": 1.03354788, "epoch": 0.09168796031865324, "flos": 18477223015680.0, "grad_norm": 2.5147764071717886, "language_loss": 0.86025923, "learning_rate": 3.917791419791006e-06, "loss": 0.88215643, "num_input_tokens_seen": 32602375, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 1.0078125, "step": 1525, "time_per_iteration": 2.432311773300171 }, { "auxiliary_loss_clip": 0.01133316, "auxiliary_loss_mlp": 0.0104995, "balance_loss_clip": 1.02396262, "balance_loss_mlp": 1.03515005, "epoch": 0.0917480835713212, "flos": 29386324212480.0, "grad_norm": 2.096285881342677, "language_loss": 0.7531842, "learning_rate": 3.91768417518683e-06, "loss": 0.77501684, "num_input_tokens_seen": 32621460, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.98046875, "step": 1526, "time_per_iteration": 2.508889675140381 }, { "auxiliary_loss_clip": 0.01133753, "auxiliary_loss_mlp": 0.01048868, "balance_loss_clip": 1.02340519, "balance_loss_mlp": 1.0355401, "epoch": 0.09180820682398917, "flos": 19828222053120.0, "grad_norm": 2.155732744211786, "language_loss": 0.77275509, "learning_rate": 3.917576862145438e-06, "loss": 0.79458129, "num_input_tokens_seen": 32640440, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.984375, "step": 1527, "time_per_iteration": 2.504575490951538 }, { "auxiliary_loss_clip": 0.01134489, "auxiliary_loss_mlp": 0.01052293, "balance_loss_clip": 1.02495801, "balance_loss_mlp": 1.03549433, "epoch": 0.09186833007665715, "flos": 23840720121600.0, "grad_norm": 2.472114783236302, "language_loss": 0.78673851, "learning_rate": 3.91746948067066e-06, "loss": 0.80860639, "num_input_tokens_seen": 32660020, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.9921875, "step": 1528, "time_per_iteration": 2.497410297393799 }, { "auxiliary_loss_clip": 0.01133446, "auxiliary_loss_mlp": 0.01044216, "balance_loss_clip": 1.01814508, "balance_loss_mlp": 1.03460348, "epoch": 0.09192845332932512, "flos": 12931898215680.0, "grad_norm": 2.7292163851837303, "language_loss": 0.77312195, "learning_rate": 3.91736203076633e-06, "loss": 0.79489857, "num_input_tokens_seen": 32678170, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.98828125, "step": 1529, "time_per_iteration": 2.433762550354004 }, { "auxiliary_loss_clip": 0.01133114, "auxiliary_loss_mlp": 0.01049329, "balance_loss_clip": 1.02217329, "balance_loss_mlp": 1.03224063, "epoch": 0.09198857658199308, "flos": 24567123202560.0, "grad_norm": 1.8967014901884687, "language_loss": 0.8285197, "learning_rate": 3.9172545124362795e-06, "loss": 0.85034418, "num_input_tokens_seen": 32697540, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.0078125, "step": 1530, "time_per_iteration": 2.4446945190429688 }, { "auxiliary_loss_clip": 0.0113179, "auxiliary_loss_mlp": 0.01055151, "balance_loss_clip": 1.02943802, "balance_loss_mlp": 1.03457022, "epoch": 0.09204869983466106, "flos": 20265893256960.0, "grad_norm": 2.6123554775526823, "language_loss": 0.83155543, "learning_rate": 3.9171469256843484e-06, "loss": 0.85342479, "num_input_tokens_seen": 32716805, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.96875, "step": 1531, "time_per_iteration": 2.4508118629455566 }, { "auxiliary_loss_clip": 0.01131607, "auxiliary_loss_mlp": 0.01050721, "balance_loss_clip": 1.02392328, "balance_loss_mlp": 1.03305411, "epoch": 0.09210882308732903, "flos": 20884624104960.0, "grad_norm": 3.281697007764632, "language_loss": 0.81480652, "learning_rate": 3.917039270514375e-06, "loss": 0.83662981, "num_input_tokens_seen": 32736385, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.984375, "step": 1532, "time_per_iteration": 2.4293484687805176 }, { "auxiliary_loss_clip": 0.01135024, "auxiliary_loss_mlp": 0.01053959, "balance_loss_clip": 1.02642202, "balance_loss_mlp": 1.0354228, "epoch": 0.09216894633999699, "flos": 30955006776960.0, "grad_norm": 2.533561225734148, "language_loss": 0.83641398, "learning_rate": 3.9169315469302e-06, "loss": 0.85830384, "num_input_tokens_seen": 32757140, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 0.99609375, "step": 1533, "time_per_iteration": 2.507021188735962 }, { "auxiliary_loss_clip": 0.01133847, "auxiliary_loss_mlp": 0.01046285, "balance_loss_clip": 1.02035689, "balance_loss_mlp": 1.03520799, "epoch": 0.09222906959266497, "flos": 13150733817600.0, "grad_norm": 2.012719717570496, "language_loss": 0.90133536, "learning_rate": 3.91682375493567e-06, "loss": 0.92313659, "num_input_tokens_seen": 32774860, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.984375, "step": 1534, "time_per_iteration": 2.387143611907959 }, { "auxiliary_loss_clip": 0.01133891, "auxiliary_loss_mlp": 0.01064751, "balance_loss_clip": 1.03456783, "balance_loss_mlp": 1.03374577, "epoch": 0.09228919284533293, "flos": 25993290130560.0, "grad_norm": 1.9768533857068011, "language_loss": 0.75789332, "learning_rate": 3.916715894534631e-06, "loss": 0.77987975, "num_input_tokens_seen": 32795250, "router_z_loss_clip": 0.30273438, "router_z_loss_mlp": 1.0, "step": 1535, "time_per_iteration": 2.490509271621704 }, { "auxiliary_loss_clip": 0.01127005, "auxiliary_loss_mlp": 0.01048225, "balance_loss_clip": 1.02288067, "balance_loss_mlp": 1.03255856, "epoch": 0.0923493160980009, "flos": 18659818759680.0, "grad_norm": 1.6612240374319367, "language_loss": 0.81129748, "learning_rate": 3.916607965730932e-06, "loss": 0.83304977, "num_input_tokens_seen": 32813805, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9453125, "step": 1536, "time_per_iteration": 2.3843331336975098 }, { "auxiliary_loss_clip": 0.0112877, "auxiliary_loss_mlp": 0.01049071, "balance_loss_clip": 1.02415609, "balance_loss_mlp": 1.03268063, "epoch": 0.09240943935066886, "flos": 21139559919360.0, "grad_norm": 3.3094259332098237, "language_loss": 0.89304686, "learning_rate": 3.9164999685284245e-06, "loss": 0.91482526, "num_input_tokens_seen": 32830960, "router_z_loss_clip": 0.24902344, "router_z_loss_mlp": 0.9609375, "step": 1537, "time_per_iteration": 2.4380135536193848 }, { "auxiliary_loss_clip": 0.01130553, "auxiliary_loss_mlp": 0.01052005, "balance_loss_clip": 1.02496839, "balance_loss_mlp": 1.03329217, "epoch": 0.09246956260333684, "flos": 20591458485120.0, "grad_norm": 2.218947463308206, "language_loss": 0.81051397, "learning_rate": 3.916391902930963e-06, "loss": 0.83233953, "num_input_tokens_seen": 32848275, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.97265625, "step": 1538, "time_per_iteration": 2.4207215309143066 }, { "auxiliary_loss_clip": 0.01047669, "auxiliary_loss_mlp": 0.01006016, "balance_loss_clip": 1.00153339, "balance_loss_mlp": 1.01349711, "epoch": 0.09252968585600481, "flos": 67555153664640.0, "grad_norm": 0.7330743763034323, "language_loss": 0.57387245, "learning_rate": 3.916283768942404e-06, "loss": 0.59440935, "num_input_tokens_seen": 32917730, "router_z_loss_clip": 0.04492188, "router_z_loss_mlp": 0.34179688, "step": 1539, "time_per_iteration": 3.207620620727539 }, { "auxiliary_loss_clip": 0.01134897, "auxiliary_loss_mlp": 0.01046346, "balance_loss_clip": 1.02051282, "balance_loss_mlp": 1.03614211, "epoch": 0.09258980910867277, "flos": 17382905360640.0, "grad_norm": 3.029941687293074, "language_loss": 0.67660999, "learning_rate": 3.916175566566607e-06, "loss": 0.69842243, "num_input_tokens_seen": 32934910, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.98828125, "step": 1540, "time_per_iteration": 2.4448249340057373 }, { "auxiliary_loss_clip": 0.01133389, "auxiliary_loss_mlp": 0.01046343, "balance_loss_clip": 1.01944923, "balance_loss_mlp": 1.03585327, "epoch": 0.09264993236134075, "flos": 19864880847360.0, "grad_norm": 1.941268230130487, "language_loss": 0.83593309, "learning_rate": 3.916067295807433e-06, "loss": 0.85773039, "num_input_tokens_seen": 32953840, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.9765625, "step": 1541, "time_per_iteration": 2.517001152038574 }, { "auxiliary_loss_clip": 0.01044401, "auxiliary_loss_mlp": 0.01004169, "balance_loss_clip": 1.00006866, "balance_loss_mlp": 1.0115366, "epoch": 0.09271005561400872, "flos": 62281558909440.0, "grad_norm": 0.8820698294636038, "language_loss": 0.61850953, "learning_rate": 3.915958956668745e-06, "loss": 0.63899529, "num_input_tokens_seen": 33011410, "router_z_loss_clip": 0.04101562, "router_z_loss_mlp": 0.328125, "step": 1542, "time_per_iteration": 3.1013567447662354 }, { "auxiliary_loss_clip": 0.01131936, "auxiliary_loss_mlp": 0.01053385, "balance_loss_clip": 1.02901816, "balance_loss_mlp": 1.0335089, "epoch": 0.09277017886667668, "flos": 23328788722560.0, "grad_norm": 1.826825102411416, "language_loss": 0.82780075, "learning_rate": 3.915850549154412e-06, "loss": 0.84965402, "num_input_tokens_seen": 33031675, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.984375, "step": 1543, "time_per_iteration": 2.4943008422851562 }, { "auxiliary_loss_clip": 0.0113051, "auxiliary_loss_mlp": 0.01050692, "balance_loss_clip": 1.02413249, "balance_loss_mlp": 1.03472304, "epoch": 0.09283030211934466, "flos": 54743183435520.0, "grad_norm": 1.8056748607287116, "language_loss": 0.72402155, "learning_rate": 3.9157420732682995e-06, "loss": 0.74583352, "num_input_tokens_seen": 33056355, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.95703125, "step": 1544, "time_per_iteration": 2.719080924987793 }, { "auxiliary_loss_clip": 0.0113308, "auxiliary_loss_mlp": 0.01047957, "balance_loss_clip": 1.02156389, "balance_loss_mlp": 1.03499806, "epoch": 0.09289042537201263, "flos": 30333517931520.0, "grad_norm": 2.5526843283161744, "language_loss": 0.77444106, "learning_rate": 3.91563352901428e-06, "loss": 0.79625142, "num_input_tokens_seen": 33079520, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.98046875, "step": 1545, "time_per_iteration": 2.5514965057373047 }, { "auxiliary_loss_clip": 0.01129955, "auxiliary_loss_mlp": 0.01049175, "balance_loss_clip": 1.02300811, "balance_loss_mlp": 1.03411222, "epoch": 0.09295054862468059, "flos": 17745932344320.0, "grad_norm": 2.695182505976073, "language_loss": 0.74121594, "learning_rate": 3.915524916396229e-06, "loss": 0.76300728, "num_input_tokens_seen": 33096135, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.95703125, "step": 1546, "time_per_iteration": 2.397153854370117 }, { "auxiliary_loss_clip": 0.01133425, "auxiliary_loss_mlp": 0.01045961, "balance_loss_clip": 1.02051032, "balance_loss_mlp": 1.03395939, "epoch": 0.09301067187734856, "flos": 23656937391360.0, "grad_norm": 1.798481983913879, "language_loss": 0.8445034, "learning_rate": 3.91541623541802e-06, "loss": 0.86629736, "num_input_tokens_seen": 33115245, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9921875, "step": 1547, "time_per_iteration": 2.485987901687622 }, { "auxiliary_loss_clip": 0.01131087, "auxiliary_loss_mlp": 0.01051158, "balance_loss_clip": 1.02568305, "balance_loss_mlp": 1.03294826, "epoch": 0.09307079513001654, "flos": 27526465975680.0, "grad_norm": 2.1358068082235433, "language_loss": 0.67515683, "learning_rate": 3.9153074860835326e-06, "loss": 0.69697928, "num_input_tokens_seen": 33136640, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.98046875, "step": 1548, "time_per_iteration": 2.512816905975342 }, { "auxiliary_loss_clip": 0.01134184, "auxiliary_loss_mlp": 0.01056214, "balance_loss_clip": 1.02990484, "balance_loss_mlp": 1.0346415, "epoch": 0.0931309183826845, "flos": 20626406622720.0, "grad_norm": 1.908903974905939, "language_loss": 0.83415234, "learning_rate": 3.915198668396649e-06, "loss": 0.85605627, "num_input_tokens_seen": 33155060, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.9921875, "step": 1549, "time_per_iteration": 2.4752960205078125 }, { "auxiliary_loss_clip": 0.011351, "auxiliary_loss_mlp": 0.01042165, "balance_loss_clip": 1.01690459, "balance_loss_mlp": 1.03577685, "epoch": 0.09319104163535247, "flos": 29019701358720.0, "grad_norm": 1.6348414684861816, "language_loss": 0.75787747, "learning_rate": 3.91508978236125e-06, "loss": 0.77965015, "num_input_tokens_seen": 33175420, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.9921875, "step": 1550, "time_per_iteration": 2.4757087230682373 }, { "auxiliary_loss_clip": 0.0113549, "auxiliary_loss_mlp": 0.01054175, "balance_loss_clip": 1.02480149, "balance_loss_mlp": 1.03360713, "epoch": 0.09325116488802045, "flos": 25300368910080.0, "grad_norm": 2.818707974293908, "language_loss": 0.82972282, "learning_rate": 3.914980827981223e-06, "loss": 0.85161948, "num_input_tokens_seen": 33194120, "router_z_loss_clip": 0.29296875, "router_z_loss_mlp": 1.015625, "step": 1551, "time_per_iteration": 2.5096378326416016 }, { "auxiliary_loss_clip": 0.01036712, "auxiliary_loss_mlp": 0.01005773, "balance_loss_clip": 1.00164795, "balance_loss_mlp": 1.00625217, "epoch": 0.09331128814068841, "flos": 61532880514560.0, "grad_norm": 0.7464859419302465, "language_loss": 0.61793554, "learning_rate": 3.914871805260456e-06, "loss": 0.63836038, "num_input_tokens_seen": 33261080, "router_z_loss_clip": 0.04125977, "router_z_loss_mlp": 0.3046875, "step": 1552, "time_per_iteration": 3.1823689937591553 }, { "auxiliary_loss_clip": 0.01035259, "auxiliary_loss_mlp": 0.01005669, "balance_loss_clip": 1.00163996, "balance_loss_mlp": 1.0053786, "epoch": 0.09337141139335638, "flos": 53290515052800.0, "grad_norm": 0.8366902008839252, "language_loss": 0.59049493, "learning_rate": 3.91476271420284e-06, "loss": 0.61090428, "num_input_tokens_seen": 33330235, "router_z_loss_clip": 0.0402832, "router_z_loss_mlp": 0.29882812, "step": 1553, "time_per_iteration": 3.234530210494995 }, { "auxiliary_loss_clip": 0.01133075, "auxiliary_loss_mlp": 0.01050295, "balance_loss_clip": 1.02307987, "balance_loss_mlp": 1.03343916, "epoch": 0.09343153464602436, "flos": 23475738101760.0, "grad_norm": 1.8482203056914184, "language_loss": 0.87292784, "learning_rate": 3.914653554812269e-06, "loss": 0.89476156, "num_input_tokens_seen": 33349035, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 0.99609375, "step": 1554, "time_per_iteration": 2.4784629344940186 }, { "auxiliary_loss_clip": 0.01130778, "auxiliary_loss_mlp": 0.01048842, "balance_loss_clip": 1.02203178, "balance_loss_mlp": 1.03553355, "epoch": 0.09349165789869232, "flos": 19352495600640.0, "grad_norm": 1.8549894743094775, "language_loss": 0.81752455, "learning_rate": 3.914544327092637e-06, "loss": 0.83932072, "num_input_tokens_seen": 33368060, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.953125, "step": 1555, "time_per_iteration": 3.996166944503784 }, { "auxiliary_loss_clip": 0.01034732, "auxiliary_loss_mlp": 0.01003965, "balance_loss_clip": 1.00012672, "balance_loss_mlp": 1.00511003, "epoch": 0.09355178115136029, "flos": 67499572913280.0, "grad_norm": 0.8684699997120257, "language_loss": 0.5964554, "learning_rate": 3.914435031047844e-06, "loss": 0.61684233, "num_input_tokens_seen": 33430825, "router_z_loss_clip": 0.03833008, "router_z_loss_mlp": 0.296875, "step": 1556, "time_per_iteration": 3.087555408477783 }, { "auxiliary_loss_clip": 0.01132066, "auxiliary_loss_mlp": 0.01051652, "balance_loss_clip": 1.02517581, "balance_loss_mlp": 1.03376365, "epoch": 0.09361190440402825, "flos": 37340132353920.0, "grad_norm": 1.8014126832529527, "language_loss": 0.8437897, "learning_rate": 3.9143256666817875e-06, "loss": 0.86562681, "num_input_tokens_seen": 33454855, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.98046875, "step": 1557, "time_per_iteration": 3.954483985900879 }, { "auxiliary_loss_clip": 0.01131742, "auxiliary_loss_mlp": 0.01051749, "balance_loss_clip": 1.02405715, "balance_loss_mlp": 1.03336477, "epoch": 0.09367202765669623, "flos": 24898553539200.0, "grad_norm": 1.7844932698726639, "language_loss": 0.77857816, "learning_rate": 3.914216233998373e-06, "loss": 0.80041307, "num_input_tokens_seen": 33476000, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 0.984375, "step": 1558, "time_per_iteration": 3.984492540359497 }, { "auxiliary_loss_clip": 0.01135984, "auxiliary_loss_mlp": 0.01048342, "balance_loss_clip": 1.02124536, "balance_loss_mlp": 1.03523898, "epoch": 0.0937321509093642, "flos": 15704665349760.0, "grad_norm": 1.858447994209009, "language_loss": 0.79866064, "learning_rate": 3.914106733001505e-06, "loss": 0.82050389, "num_input_tokens_seen": 33493845, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 1.0078125, "step": 1559, "time_per_iteration": 3.8980712890625 }, { "auxiliary_loss_clip": 0.01127696, "auxiliary_loss_mlp": 0.01048208, "balance_loss_clip": 1.02249503, "balance_loss_mlp": 1.03341794, "epoch": 0.09379227416203216, "flos": 20482704000000.0, "grad_norm": 3.145335706741672, "language_loss": 0.76307881, "learning_rate": 3.9139971636950914e-06, "loss": 0.78483784, "num_input_tokens_seen": 33510850, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.94140625, "step": 1560, "time_per_iteration": 2.4804866313934326 }, { "auxiliary_loss_clip": 0.01137378, "auxiliary_loss_mlp": 0.01049168, "balance_loss_clip": 1.02276349, "balance_loss_mlp": 1.03392553, "epoch": 0.09385239741470014, "flos": 24351359800320.0, "grad_norm": 1.684229005364842, "language_loss": 0.80698353, "learning_rate": 3.913887526083042e-06, "loss": 0.82884896, "num_input_tokens_seen": 33530430, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 1.0390625, "step": 1561, "time_per_iteration": 2.5235869884490967 }, { "auxiliary_loss_clip": 0.01130155, "auxiliary_loss_mlp": 0.01042075, "balance_loss_clip": 1.01642132, "balance_loss_mlp": 1.03278327, "epoch": 0.0939125206673681, "flos": 33290102707200.0, "grad_norm": 6.11597418955832, "language_loss": 0.61490536, "learning_rate": 3.91377782016927e-06, "loss": 0.63662767, "num_input_tokens_seen": 33551975, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.97265625, "step": 1562, "time_per_iteration": 2.5785610675811768 }, { "auxiliary_loss_clip": 0.01132565, "auxiliary_loss_mlp": 0.01051863, "balance_loss_clip": 1.02629256, "balance_loss_mlp": 1.03629041, "epoch": 0.09397264392003607, "flos": 19243915672320.0, "grad_norm": 9.651361957153787, "language_loss": 0.84796524, "learning_rate": 3.9136680459576905e-06, "loss": 0.86980951, "num_input_tokens_seen": 33569850, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.96484375, "step": 1563, "time_per_iteration": 2.445735216140747 }, { "auxiliary_loss_clip": 0.01126767, "auxiliary_loss_mlp": 0.01042546, "balance_loss_clip": 1.01910937, "balance_loss_mlp": 1.03224373, "epoch": 0.09403276717270405, "flos": 19316919058560.0, "grad_norm": 1.689177225733662, "language_loss": 0.75749022, "learning_rate": 3.913558203452221e-06, "loss": 0.77918339, "num_input_tokens_seen": 33590510, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.9453125, "step": 1564, "time_per_iteration": 2.600019931793213 }, { "auxiliary_loss_clip": 0.01131358, "auxiliary_loss_mlp": 0.01046324, "balance_loss_clip": 1.02163541, "balance_loss_mlp": 1.03432405, "epoch": 0.09409289042537201, "flos": 23582432816640.0, "grad_norm": 2.075383688901369, "language_loss": 0.80019706, "learning_rate": 3.913448292656782e-06, "loss": 0.82197386, "num_input_tokens_seen": 33608810, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.96875, "step": 1565, "time_per_iteration": 2.5562853813171387 }, { "auxiliary_loss_clip": 0.01128414, "auxiliary_loss_mlp": 0.01053288, "balance_loss_clip": 1.0280869, "balance_loss_mlp": 1.03111577, "epoch": 0.09415301367803998, "flos": 20077572049920.0, "grad_norm": 1.9569232200602484, "language_loss": 0.75231785, "learning_rate": 3.913338313575295e-06, "loss": 0.77413487, "num_input_tokens_seen": 33627265, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.97265625, "step": 1566, "time_per_iteration": 2.5157737731933594 }, { "auxiliary_loss_clip": 0.01128828, "auxiliary_loss_mlp": 0.01056516, "balance_loss_clip": 1.03018296, "balance_loss_mlp": 1.03255665, "epoch": 0.09421313693070796, "flos": 21061215095040.0, "grad_norm": 1.8935387915162705, "language_loss": 0.77399063, "learning_rate": 3.913228266211685e-06, "loss": 0.79584408, "num_input_tokens_seen": 33644810, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.9609375, "step": 1567, "time_per_iteration": 2.4394407272338867 }, { "auxiliary_loss_clip": 0.01132407, "auxiliary_loss_mlp": 0.01048987, "balance_loss_clip": 1.02391696, "balance_loss_mlp": 1.03506601, "epoch": 0.09427326018337592, "flos": 24315015208320.0, "grad_norm": 1.8373050423611277, "language_loss": 0.82380879, "learning_rate": 3.91311815056988e-06, "loss": 0.84562278, "num_input_tokens_seen": 33665665, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.97265625, "step": 1568, "time_per_iteration": 2.4735612869262695 }, { "auxiliary_loss_clip": 0.01133743, "auxiliary_loss_mlp": 0.01047137, "balance_loss_clip": 1.02000451, "balance_loss_mlp": 1.035182, "epoch": 0.09433338343604389, "flos": 20262925791360.0, "grad_norm": 3.0445244276773686, "language_loss": 0.76563734, "learning_rate": 3.9130079666538094e-06, "loss": 0.78744614, "num_input_tokens_seen": 33684760, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 0.984375, "step": 1569, "time_per_iteration": 2.4458184242248535 }, { "auxiliary_loss_clip": 0.01128653, "auxiliary_loss_mlp": 0.01054956, "balance_loss_clip": 1.02970767, "balance_loss_mlp": 1.03367496, "epoch": 0.09439350668871185, "flos": 12742355111040.0, "grad_norm": 2.085834786434566, "language_loss": 0.85499036, "learning_rate": 3.912897714467405e-06, "loss": 0.87682647, "num_input_tokens_seen": 33700750, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.94921875, "step": 1570, "time_per_iteration": 2.400621175765991 }, { "auxiliary_loss_clip": 0.01132324, "auxiliary_loss_mlp": 0.01047512, "balance_loss_clip": 1.02268112, "balance_loss_mlp": 1.03605151, "epoch": 0.09445362994137983, "flos": 25960960344960.0, "grad_norm": 1.7147238482541927, "language_loss": 0.76369232, "learning_rate": 3.912787394014602e-06, "loss": 0.78549063, "num_input_tokens_seen": 33724430, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.9609375, "step": 1571, "time_per_iteration": 2.503005266189575 }, { "auxiliary_loss_clip": 0.01125115, "auxiliary_loss_mlp": 0.01049007, "balance_loss_clip": 1.02462876, "balance_loss_mlp": 1.03338134, "epoch": 0.0945137531940478, "flos": 19714440332160.0, "grad_norm": 1.5826613962874685, "language_loss": 0.79275006, "learning_rate": 3.912677005299337e-06, "loss": 0.81449127, "num_input_tokens_seen": 33743455, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.91796875, "step": 1572, "time_per_iteration": 2.4463391304016113 }, { "auxiliary_loss_clip": 0.01128702, "auxiliary_loss_mlp": 0.0105154, "balance_loss_clip": 1.02736449, "balance_loss_mlp": 1.03395796, "epoch": 0.09457387644671576, "flos": 23616089233920.0, "grad_norm": 1.9490236600921087, "language_loss": 0.87449968, "learning_rate": 3.912566548325549e-06, "loss": 0.8963021, "num_input_tokens_seen": 33763435, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.94921875, "step": 1573, "time_per_iteration": 2.4512505531311035 }, { "auxiliary_loss_clip": 0.01130399, "auxiliary_loss_mlp": 0.0105662, "balance_loss_clip": 1.0299294, "balance_loss_mlp": 1.0338819, "epoch": 0.09463399969938374, "flos": 26906059382400.0, "grad_norm": 3.9008503413191, "language_loss": 0.81712437, "learning_rate": 3.912456023097182e-06, "loss": 0.83899456, "num_input_tokens_seen": 33784325, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.96484375, "step": 1574, "time_per_iteration": 2.4951813220977783 }, { "auxiliary_loss_clip": 0.01129031, "auxiliary_loss_mlp": 0.01044704, "balance_loss_clip": 1.01958644, "balance_loss_mlp": 1.03409958, "epoch": 0.0946941229520517, "flos": 23658438579840.0, "grad_norm": 1.8567349415175596, "language_loss": 0.81094515, "learning_rate": 3.912345429618178e-06, "loss": 0.83268249, "num_input_tokens_seen": 33802510, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.94921875, "step": 1575, "time_per_iteration": 2.443317174911499 }, { "auxiliary_loss_clip": 0.01126348, "auxiliary_loss_mlp": 0.01052319, "balance_loss_clip": 1.02739215, "balance_loss_mlp": 1.03227639, "epoch": 0.09475424620471967, "flos": 24132908223360.0, "grad_norm": 2.4286094261598494, "language_loss": 0.86847895, "learning_rate": 3.912234767892486e-06, "loss": 0.89026564, "num_input_tokens_seen": 33819980, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.94140625, "step": 1576, "time_per_iteration": 2.446354866027832 }, { "auxiliary_loss_clip": 0.01039027, "auxiliary_loss_mlp": 0.01006219, "balance_loss_clip": 1.00252378, "balance_loss_mlp": 1.00889707, "epoch": 0.09481436945738765, "flos": 68422815573120.0, "grad_norm": 0.9876111566020145, "language_loss": 0.65870196, "learning_rate": 3.912124037924053e-06, "loss": 0.67915446, "num_input_tokens_seen": 33878925, "router_z_loss_clip": 0.03686523, "router_z_loss_mlp": 0.30078125, "step": 1577, "time_per_iteration": 3.0272223949432373 }, { "auxiliary_loss_clip": 0.01129314, "auxiliary_loss_mlp": 0.01042959, "balance_loss_clip": 1.01878285, "balance_loss_mlp": 1.03307056, "epoch": 0.09487449271005562, "flos": 16653150789120.0, "grad_norm": 2.0004661920780817, "language_loss": 0.79035044, "learning_rate": 3.912013239716831e-06, "loss": 0.81207317, "num_input_tokens_seen": 33897600, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.9609375, "step": 1578, "time_per_iteration": 2.4237899780273438 }, { "auxiliary_loss_clip": 0.01127281, "auxiliary_loss_mlp": 0.01051388, "balance_loss_clip": 1.02628279, "balance_loss_mlp": 1.03219068, "epoch": 0.09493461596272358, "flos": 24274655809920.0, "grad_norm": 1.757623102725029, "language_loss": 0.78247732, "learning_rate": 3.911902373274776e-06, "loss": 0.80426395, "num_input_tokens_seen": 33917365, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.953125, "step": 1579, "time_per_iteration": 2.4414737224578857 }, { "auxiliary_loss_clip": 0.011302, "auxiliary_loss_mlp": 0.01053094, "balance_loss_clip": 1.02664101, "balance_loss_mlp": 1.03369892, "epoch": 0.09499473921539155, "flos": 21869139934080.0, "grad_norm": 2.1555160267596505, "language_loss": 0.72842288, "learning_rate": 3.911791438601842e-06, "loss": 0.75025582, "num_input_tokens_seen": 33936680, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.96484375, "step": 1580, "time_per_iteration": 2.4454803466796875 }, { "auxiliary_loss_clip": 0.01125442, "auxiliary_loss_mlp": 0.01051209, "balance_loss_clip": 1.02759409, "balance_loss_mlp": 1.03187084, "epoch": 0.09505486246805953, "flos": 33545736748800.0, "grad_norm": 1.9535485397853518, "language_loss": 0.77732539, "learning_rate": 3.91168043570199e-06, "loss": 0.79909194, "num_input_tokens_seen": 33960685, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.9375, "step": 1581, "time_per_iteration": 2.5470573902130127 }, { "auxiliary_loss_clip": 0.01127665, "auxiliary_loss_mlp": 0.01049812, "balance_loss_clip": 1.02551746, "balance_loss_mlp": 1.0330832, "epoch": 0.09511498572072749, "flos": 21214273962240.0, "grad_norm": 1.921177984257379, "language_loss": 0.87140906, "learning_rate": 3.911569364579181e-06, "loss": 0.89318383, "num_input_tokens_seen": 33980015, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.9453125, "step": 1582, "time_per_iteration": 2.4393773078918457 }, { "auxiliary_loss_clip": 0.0112801, "auxiliary_loss_mlp": 0.01042294, "balance_loss_clip": 1.01530528, "balance_loss_mlp": 1.03308201, "epoch": 0.09517510897339546, "flos": 14610382606080.0, "grad_norm": 1.9526238988739044, "language_loss": 0.66777384, "learning_rate": 3.9114582252373786e-06, "loss": 0.68947685, "num_input_tokens_seen": 33997705, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.94921875, "step": 1583, "time_per_iteration": 2.408489465713501 }, { "auxiliary_loss_clip": 0.01130276, "auxiliary_loss_mlp": 0.01044433, "balance_loss_clip": 1.01768279, "balance_loss_mlp": 1.03359652, "epoch": 0.09523523222606343, "flos": 27816140459520.0, "grad_norm": 2.123446702097312, "language_loss": 0.70384932, "learning_rate": 3.911347017680548e-06, "loss": 0.72559643, "num_input_tokens_seen": 34017465, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.96484375, "step": 1584, "time_per_iteration": 2.4857635498046875 }, { "auxiliary_loss_clip": 0.01128904, "auxiliary_loss_mlp": 0.01043143, "balance_loss_clip": 1.01943183, "balance_loss_mlp": 1.03342712, "epoch": 0.0952953554787314, "flos": 20705170383360.0, "grad_norm": 1.4986264749081961, "language_loss": 0.81038153, "learning_rate": 3.911235741912659e-06, "loss": 0.832102, "num_input_tokens_seen": 34038550, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.95703125, "step": 1585, "time_per_iteration": 2.4599831104278564 }, { "auxiliary_loss_clip": 0.01131759, "auxiliary_loss_mlp": 0.01052134, "balance_loss_clip": 1.02458525, "balance_loss_mlp": 1.0344888, "epoch": 0.09535547873139937, "flos": 24786552297600.0, "grad_norm": 1.7153409836079414, "language_loss": 0.71711075, "learning_rate": 3.911124397937683e-06, "loss": 0.73894966, "num_input_tokens_seen": 34058665, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 0.97265625, "step": 1586, "time_per_iteration": 2.5065290927886963 }, { "auxiliary_loss_clip": 0.0103869, "auxiliary_loss_mlp": 0.01004867, "balance_loss_clip": 1.00143409, "balance_loss_mlp": 1.00877166, "epoch": 0.09541560198406734, "flos": 71909208230400.0, "grad_norm": 0.8043839504118597, "language_loss": 0.5548532, "learning_rate": 3.911012985759594e-06, "loss": 0.57528877, "num_input_tokens_seen": 34109655, "router_z_loss_clip": 0.03442383, "router_z_loss_mlp": 0.29882812, "step": 1587, "time_per_iteration": 2.8876850605010986 }, { "auxiliary_loss_clip": 0.01131229, "auxiliary_loss_mlp": 0.01053932, "balance_loss_clip": 1.02865958, "balance_loss_mlp": 1.03474832, "epoch": 0.09547572523673531, "flos": 28981436641920.0, "grad_norm": 1.6962794613973096, "language_loss": 0.80978215, "learning_rate": 3.910901505382367e-06, "loss": 0.83163375, "num_input_tokens_seen": 34131115, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.96484375, "step": 1588, "time_per_iteration": 2.5049076080322266 }, { "auxiliary_loss_clip": 0.01128484, "auxiliary_loss_mlp": 0.01048779, "balance_loss_clip": 1.02406645, "balance_loss_mlp": 1.03440237, "epoch": 0.09553584848940327, "flos": 24132768577920.0, "grad_norm": 1.5583499472319169, "language_loss": 0.81693327, "learning_rate": 3.910789956809981e-06, "loss": 0.8387059, "num_input_tokens_seen": 34151925, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.94140625, "step": 1589, "time_per_iteration": 2.444581985473633 }, { "auxiliary_loss_clip": 0.01129849, "auxiliary_loss_mlp": 0.01049648, "balance_loss_clip": 1.02476883, "balance_loss_mlp": 1.0345633, "epoch": 0.09559597174207124, "flos": 42849706055040.0, "grad_norm": 1.4846773363066526, "language_loss": 0.64840114, "learning_rate": 3.910678340046415e-06, "loss": 0.67019612, "num_input_tokens_seen": 34175395, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.953125, "step": 1590, "time_per_iteration": 2.6288366317749023 }, { "auxiliary_loss_clip": 0.01127174, "auxiliary_loss_mlp": 0.01048047, "balance_loss_clip": 1.02297759, "balance_loss_mlp": 1.03421664, "epoch": 0.09565609499473922, "flos": 32669486645760.0, "grad_norm": 1.9044171830956573, "language_loss": 0.83177459, "learning_rate": 3.910566655095655e-06, "loss": 0.85352671, "num_input_tokens_seen": 34197760, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.9296875, "step": 1591, "time_per_iteration": 2.530949592590332 }, { "auxiliary_loss_clip": 0.01131455, "auxiliary_loss_mlp": 0.01058947, "balance_loss_clip": 1.03254199, "balance_loss_mlp": 1.03324676, "epoch": 0.09571621824740718, "flos": 18477432483840.0, "grad_norm": 2.665017419106436, "language_loss": 0.74207127, "learning_rate": 3.9104549019616855e-06, "loss": 0.76397526, "num_input_tokens_seen": 34215330, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.984375, "step": 1592, "time_per_iteration": 2.3857264518737793 }, { "auxiliary_loss_clip": 0.01126811, "auxiliary_loss_mlp": 0.0104743, "balance_loss_clip": 1.02271771, "balance_loss_mlp": 1.03202522, "epoch": 0.09577634150007515, "flos": 29386219478400.0, "grad_norm": 1.841806432839428, "language_loss": 0.74010116, "learning_rate": 3.910343080648495e-06, "loss": 0.76184356, "num_input_tokens_seen": 34237745, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.94921875, "step": 1593, "time_per_iteration": 2.4896321296691895 }, { "auxiliary_loss_clip": 0.01128345, "auxiliary_loss_mlp": 0.01050887, "balance_loss_clip": 1.02528095, "balance_loss_mlp": 1.03316426, "epoch": 0.09583646475274313, "flos": 22746716668800.0, "grad_norm": 1.7383322301339936, "language_loss": 0.69956505, "learning_rate": 3.910231191160074e-06, "loss": 0.72135735, "num_input_tokens_seen": 34256565, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.953125, "step": 1594, "time_per_iteration": 3.940382480621338 }, { "auxiliary_loss_clip": 0.01129306, "auxiliary_loss_mlp": 0.01044947, "balance_loss_clip": 1.01942444, "balance_loss_mlp": 1.03309989, "epoch": 0.0958965880054111, "flos": 23217346062720.0, "grad_norm": 2.301854053189111, "language_loss": 0.8258779, "learning_rate": 3.910119233500415e-06, "loss": 0.84762043, "num_input_tokens_seen": 34275970, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.96484375, "step": 1595, "time_per_iteration": 2.4429819583892822 }, { "auxiliary_loss_clip": 0.01131211, "auxiliary_loss_mlp": 0.01047621, "balance_loss_clip": 1.02132368, "balance_loss_mlp": 1.03454077, "epoch": 0.09595671125807906, "flos": 21323377560960.0, "grad_norm": 2.15645836466016, "language_loss": 0.84589171, "learning_rate": 3.910007207673514e-06, "loss": 0.86767995, "num_input_tokens_seen": 34295490, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.96484375, "step": 1596, "time_per_iteration": 3.870670795440674 }, { "auxiliary_loss_clip": 0.01132189, "auxiliary_loss_mlp": 0.0104588, "balance_loss_clip": 1.01948714, "balance_loss_mlp": 1.03406048, "epoch": 0.09601683451074704, "flos": 39601910695680.0, "grad_norm": 1.8493887818937222, "language_loss": 0.69076598, "learning_rate": 3.909895113683369e-06, "loss": 0.71254671, "num_input_tokens_seen": 34319990, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.98046875, "step": 1597, "time_per_iteration": 2.581110715866089 }, { "auxiliary_loss_clip": 0.01130091, "auxiliary_loss_mlp": 0.01040875, "balance_loss_clip": 1.01574564, "balance_loss_mlp": 1.03267622, "epoch": 0.096076957763415, "flos": 23731581611520.0, "grad_norm": 2.043402237761072, "language_loss": 0.74736744, "learning_rate": 3.9097829515339805e-06, "loss": 0.76907706, "num_input_tokens_seen": 34339225, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.97265625, "step": 1598, "time_per_iteration": 3.935260057449341 }, { "auxiliary_loss_clip": 0.01134141, "auxiliary_loss_mlp": 0.01051248, "balance_loss_clip": 1.02383006, "balance_loss_mlp": 1.0345974, "epoch": 0.09613708101608297, "flos": 34676678286720.0, "grad_norm": 1.5883270883652745, "language_loss": 0.69103479, "learning_rate": 3.909670721229351e-06, "loss": 0.71288872, "num_input_tokens_seen": 34361020, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 0.9921875, "step": 1599, "time_per_iteration": 2.53983998298645 }, { "auxiliary_loss_clip": 0.01129399, "auxiliary_loss_mlp": 0.01047688, "balance_loss_clip": 1.02234411, "balance_loss_mlp": 1.03334928, "epoch": 0.09619720426875093, "flos": 20739001357440.0, "grad_norm": 2.1501053197670674, "language_loss": 0.84326446, "learning_rate": 3.909558422773485e-06, "loss": 0.8650353, "num_input_tokens_seen": 34378630, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9609375, "step": 1600, "time_per_iteration": 2.480672597885132 }, { "auxiliary_loss_clip": 0.01130237, "auxiliary_loss_mlp": 0.01051023, "balance_loss_clip": 1.0260725, "balance_loss_mlp": 1.03436995, "epoch": 0.09625732752141891, "flos": 13041874598400.0, "grad_norm": 3.115910981380097, "language_loss": 0.803262, "learning_rate": 3.909446056170392e-06, "loss": 0.82507461, "num_input_tokens_seen": 34397110, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.95703125, "step": 1601, "time_per_iteration": 2.407433032989502 }, { "auxiliary_loss_clip": 0.01133421, "auxiliary_loss_mlp": 0.01050606, "balance_loss_clip": 1.02371204, "balance_loss_mlp": 1.03521669, "epoch": 0.09631745077408688, "flos": 22272526316160.0, "grad_norm": 2.789109957056121, "language_loss": 0.82325655, "learning_rate": 3.9093336214240805e-06, "loss": 0.84509683, "num_input_tokens_seen": 34414165, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.984375, "step": 1602, "time_per_iteration": 2.4383020401000977 }, { "auxiliary_loss_clip": 0.01128537, "auxiliary_loss_mlp": 0.01051186, "balance_loss_clip": 1.02412581, "balance_loss_mlp": 1.0341289, "epoch": 0.09637757402675484, "flos": 24753105348480.0, "grad_norm": 2.012458938720864, "language_loss": 0.62555087, "learning_rate": 3.9092211185385625e-06, "loss": 0.64734805, "num_input_tokens_seen": 34434445, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.9453125, "step": 1603, "time_per_iteration": 2.452920913696289 }, { "auxiliary_loss_clip": 0.01132477, "auxiliary_loss_mlp": 0.01048033, "balance_loss_clip": 1.01939869, "balance_loss_mlp": 1.03522003, "epoch": 0.09643769727942282, "flos": 22524739044480.0, "grad_norm": 5.2125661320797105, "language_loss": 0.71050173, "learning_rate": 3.909108547517855e-06, "loss": 0.73230684, "num_input_tokens_seen": 34453095, "router_z_loss_clip": 0.28515625, "router_z_loss_mlp": 0.97265625, "step": 1604, "time_per_iteration": 2.5001332759857178 }, { "auxiliary_loss_clip": 0.01129197, "auxiliary_loss_mlp": 0.01049554, "balance_loss_clip": 1.02432883, "balance_loss_mlp": 1.0339098, "epoch": 0.09649782053209079, "flos": 30919674614400.0, "grad_norm": 2.1476828999693787, "language_loss": 0.79755807, "learning_rate": 3.908995908365974e-06, "loss": 0.81934559, "num_input_tokens_seen": 34473680, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.953125, "step": 1605, "time_per_iteration": 2.516160249710083 }, { "auxiliary_loss_clip": 0.01129773, "auxiliary_loss_mlp": 0.01046575, "balance_loss_clip": 1.02044487, "balance_loss_mlp": 1.03133607, "epoch": 0.09655794378475875, "flos": 25336469122560.0, "grad_norm": 2.1327881144518552, "language_loss": 0.74646139, "learning_rate": 3.908883201086939e-06, "loss": 0.76822495, "num_input_tokens_seen": 34492610, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.984375, "step": 1606, "time_per_iteration": 2.4860920906066895 }, { "auxiliary_loss_clip": 0.01131643, "auxiliary_loss_mlp": 0.01043237, "balance_loss_clip": 1.01773846, "balance_loss_mlp": 1.03393614, "epoch": 0.09661806703742673, "flos": 22344971120640.0, "grad_norm": 1.7854779361931754, "language_loss": 0.75499034, "learning_rate": 3.908770425684774e-06, "loss": 0.77673924, "num_input_tokens_seen": 34511855, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9765625, "step": 1607, "time_per_iteration": 2.439432382583618 }, { "auxiliary_loss_clip": 0.01129381, "auxiliary_loss_mlp": 0.0104156, "balance_loss_clip": 1.01736069, "balance_loss_mlp": 1.03235519, "epoch": 0.0966781902900947, "flos": 17456606974080.0, "grad_norm": 1.966699091936902, "language_loss": 0.86513656, "learning_rate": 3.908657582163501e-06, "loss": 0.88684595, "num_input_tokens_seen": 34528905, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.97265625, "step": 1608, "time_per_iteration": 2.420522451400757 }, { "auxiliary_loss_clip": 0.01134142, "auxiliary_loss_mlp": 0.01055626, "balance_loss_clip": 1.02823162, "balance_loss_mlp": 1.0341233, "epoch": 0.09673831354276266, "flos": 36902496061440.0, "grad_norm": 2.6149778361642504, "language_loss": 0.71525943, "learning_rate": 3.90854467052715e-06, "loss": 0.73715711, "num_input_tokens_seen": 34548480, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 1.0, "step": 1609, "time_per_iteration": 2.5289125442504883 }, { "auxiliary_loss_clip": 0.01130136, "auxiliary_loss_mlp": 0.01046521, "balance_loss_clip": 1.02195179, "balance_loss_mlp": 1.03249955, "epoch": 0.09679843679543064, "flos": 20700422438400.0, "grad_norm": 2.195539894108231, "language_loss": 0.84416944, "learning_rate": 3.908431690779748e-06, "loss": 0.86593604, "num_input_tokens_seen": 34565410, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.9765625, "step": 1610, "time_per_iteration": 2.4299871921539307 }, { "auxiliary_loss_clip": 0.01129956, "auxiliary_loss_mlp": 0.0104887, "balance_loss_clip": 1.02248859, "balance_loss_mlp": 1.03369439, "epoch": 0.0968585600480986, "flos": 23513269680000.0, "grad_norm": 2.117691948263346, "language_loss": 0.6709789, "learning_rate": 3.9083186429253284e-06, "loss": 0.69276714, "num_input_tokens_seen": 34584840, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.9609375, "step": 1611, "time_per_iteration": 2.4413843154907227 }, { "auxiliary_loss_clip": 0.01129039, "auxiliary_loss_mlp": 0.01048062, "balance_loss_clip": 1.02288496, "balance_loss_mlp": 1.03424144, "epoch": 0.09691868330076657, "flos": 20120026129920.0, "grad_norm": 1.79228238953497, "language_loss": 0.8106401, "learning_rate": 3.908205526967925e-06, "loss": 0.83241117, "num_input_tokens_seen": 34603360, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.9453125, "step": 1612, "time_per_iteration": 2.458664894104004 }, { "auxiliary_loss_clip": 0.01130977, "auxiliary_loss_mlp": 0.01048114, "balance_loss_clip": 1.02069569, "balance_loss_mlp": 1.03401744, "epoch": 0.09697880655343454, "flos": 16543767899520.0, "grad_norm": 2.123354943874359, "language_loss": 0.8074218, "learning_rate": 3.9080923429115755e-06, "loss": 0.82921273, "num_input_tokens_seen": 34620760, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.96875, "step": 1613, "time_per_iteration": 2.4841091632843018 }, { "auxiliary_loss_clip": 0.01131463, "auxiliary_loss_mlp": 0.01046134, "balance_loss_clip": 1.01866841, "balance_loss_mlp": 1.03485274, "epoch": 0.09703892980610251, "flos": 26102987222400.0, "grad_norm": 2.1194127715365556, "language_loss": 0.84466386, "learning_rate": 3.907979090760318e-06, "loss": 0.86643982, "num_input_tokens_seen": 34640695, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.96484375, "step": 1614, "time_per_iteration": 2.508568286895752 }, { "auxiliary_loss_clip": 0.0104157, "auxiliary_loss_mlp": 0.01004943, "balance_loss_clip": 1.00150931, "balance_loss_mlp": 1.01060295, "epoch": 0.09709905305877048, "flos": 60440273516160.0, "grad_norm": 0.7053662651999016, "language_loss": 0.54595566, "learning_rate": 3.907865770518194e-06, "loss": 0.56642085, "num_input_tokens_seen": 34702395, "router_z_loss_clip": 0.03442383, "router_z_loss_mlp": 0.30859375, "step": 1615, "time_per_iteration": 3.030308723449707 }, { "auxiliary_loss_clip": 0.01039482, "auxiliary_loss_mlp": 0.01002298, "balance_loss_clip": 0.99905533, "balance_loss_mlp": 1.00889277, "epoch": 0.09715917631143844, "flos": 57636503228160.0, "grad_norm": 0.8212475208661101, "language_loss": 0.58202291, "learning_rate": 3.9077523821892495e-06, "loss": 0.60244071, "num_input_tokens_seen": 34768910, "router_z_loss_clip": 0.0324707, "router_z_loss_mlp": 0.3046875, "step": 1616, "time_per_iteration": 3.2261805534362793 }, { "auxiliary_loss_clip": 0.01131525, "auxiliary_loss_mlp": 0.01057625, "balance_loss_clip": 1.03033817, "balance_loss_mlp": 1.03429449, "epoch": 0.09721929956410642, "flos": 20557173663360.0, "grad_norm": 1.883023529993233, "language_loss": 0.68717158, "learning_rate": 3.907638925777529e-06, "loss": 0.70906311, "num_input_tokens_seen": 34787680, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.97265625, "step": 1617, "time_per_iteration": 2.468594789505005 }, { "auxiliary_loss_clip": 0.01130611, "auxiliary_loss_mlp": 0.01056749, "balance_loss_clip": 1.02909219, "balance_loss_mlp": 1.0327003, "epoch": 0.09727942281677439, "flos": 27343137093120.0, "grad_norm": 1.8482913576706792, "language_loss": 0.80511546, "learning_rate": 3.907525401287082e-06, "loss": 0.82698905, "num_input_tokens_seen": 34808330, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 0.98046875, "step": 1618, "time_per_iteration": 2.519648313522339 }, { "auxiliary_loss_clip": 0.01125056, "auxiliary_loss_mlp": 0.01046007, "balance_loss_clip": 1.02254701, "balance_loss_mlp": 1.03280425, "epoch": 0.09733954606944235, "flos": 24898867741440.0, "grad_norm": 1.656348963972433, "language_loss": 0.93125695, "learning_rate": 3.907411808721961e-06, "loss": 0.95296764, "num_input_tokens_seen": 34830020, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.921875, "step": 1619, "time_per_iteration": 2.4830868244171143 }, { "auxiliary_loss_clip": 0.01126306, "auxiliary_loss_mlp": 0.01051411, "balance_loss_clip": 1.02572119, "balance_loss_mlp": 1.03497815, "epoch": 0.09739966932211033, "flos": 31502584540800.0, "grad_norm": 2.14999931966844, "language_loss": 0.88552165, "learning_rate": 3.907298148086219e-06, "loss": 0.9072988, "num_input_tokens_seen": 34850330, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.91015625, "step": 1620, "time_per_iteration": 2.555567741394043 }, { "auxiliary_loss_clip": 0.01129719, "auxiliary_loss_mlp": 0.01056381, "balance_loss_clip": 1.02976167, "balance_loss_mlp": 1.033777, "epoch": 0.0974597925747783, "flos": 23877623295360.0, "grad_norm": 1.937261380343017, "language_loss": 0.77111161, "learning_rate": 3.907184419383912e-06, "loss": 0.79297262, "num_input_tokens_seen": 34871640, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.9609375, "step": 1621, "time_per_iteration": 2.5151984691619873 }, { "auxiliary_loss_clip": 0.01128858, "auxiliary_loss_mlp": 0.01066381, "balance_loss_clip": 1.04022598, "balance_loss_mlp": 1.0327791, "epoch": 0.09751991582744626, "flos": 17018621568000.0, "grad_norm": 1.9740811110808778, "language_loss": 0.77910846, "learning_rate": 3.907070622619099e-06, "loss": 0.80106086, "num_input_tokens_seen": 34888100, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9609375, "step": 1622, "time_per_iteration": 2.4174482822418213 }, { "auxiliary_loss_clip": 0.01130166, "auxiliary_loss_mlp": 0.01057601, "balance_loss_clip": 1.02941966, "balance_loss_mlp": 1.03194284, "epoch": 0.09758003908011423, "flos": 28401564003840.0, "grad_norm": 4.7039111582580535, "language_loss": 0.85681069, "learning_rate": 3.906956757795841e-06, "loss": 0.87868834, "num_input_tokens_seen": 34910485, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 0.984375, "step": 1623, "time_per_iteration": 2.5555121898651123 }, { "auxiliary_loss_clip": 0.01127703, "auxiliary_loss_mlp": 0.01053942, "balance_loss_clip": 1.02783489, "balance_loss_mlp": 1.03434443, "epoch": 0.09764016233278221, "flos": 18143488529280.0, "grad_norm": 2.193997572133753, "language_loss": 0.79843217, "learning_rate": 3.906842824918201e-06, "loss": 0.8202486, "num_input_tokens_seen": 34928615, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.93359375, "step": 1624, "time_per_iteration": 2.418097496032715 }, { "auxiliary_loss_clip": 0.01127799, "auxiliary_loss_mlp": 0.01048748, "balance_loss_clip": 1.02371407, "balance_loss_mlp": 1.03221011, "epoch": 0.09770028558545017, "flos": 15265004198400.0, "grad_norm": 2.217296861129646, "language_loss": 0.8578465, "learning_rate": 3.906728823990246e-06, "loss": 0.87961197, "num_input_tokens_seen": 34946045, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.95703125, "step": 1625, "time_per_iteration": 2.4446890354156494 }, { "auxiliary_loss_clip": 0.01130691, "auxiliary_loss_mlp": 0.01056346, "balance_loss_clip": 1.03190827, "balance_loss_mlp": 1.03437328, "epoch": 0.09776040883811814, "flos": 23471444004480.0, "grad_norm": 2.178532157941631, "language_loss": 0.85360849, "learning_rate": 3.906614755016044e-06, "loss": 0.87547886, "num_input_tokens_seen": 34962865, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.96484375, "step": 1626, "time_per_iteration": 2.485790967941284 }, { "auxiliary_loss_clip": 0.01134409, "auxiliary_loss_mlp": 0.01051877, "balance_loss_clip": 1.02445865, "balance_loss_mlp": 1.03664041, "epoch": 0.09782053209078612, "flos": 24498309179520.0, "grad_norm": 2.56867210436732, "language_loss": 0.83493525, "learning_rate": 3.9065006179996655e-06, "loss": 0.85679817, "num_input_tokens_seen": 34983505, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 0.9765625, "step": 1627, "time_per_iteration": 2.505154609680176 }, { "auxiliary_loss_clip": 0.01125757, "auxiliary_loss_mlp": 0.01049378, "balance_loss_clip": 1.02329516, "balance_loss_mlp": 1.03237844, "epoch": 0.09788065534345408, "flos": 21579081425280.0, "grad_norm": 2.4486454381811202, "language_loss": 0.8416543, "learning_rate": 3.906386412945184e-06, "loss": 0.8634057, "num_input_tokens_seen": 35001825, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9296875, "step": 1628, "time_per_iteration": 2.4483213424682617 }, { "auxiliary_loss_clip": 0.01126192, "auxiliary_loss_mlp": 0.01044646, "balance_loss_clip": 1.01932597, "balance_loss_mlp": 1.03139532, "epoch": 0.09794077859612205, "flos": 23841313614720.0, "grad_norm": 1.6683207470752828, "language_loss": 0.75619781, "learning_rate": 3.906272139856676e-06, "loss": 0.77790618, "num_input_tokens_seen": 35023075, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9453125, "step": 1629, "time_per_iteration": 2.4707629680633545 }, { "auxiliary_loss_clip": 0.011295, "auxiliary_loss_mlp": 0.01048848, "balance_loss_clip": 1.02209711, "balance_loss_mlp": 1.03475523, "epoch": 0.09800090184879003, "flos": 23658752782080.0, "grad_norm": 1.8837128082629686, "language_loss": 0.78327549, "learning_rate": 3.906157798738218e-06, "loss": 0.80505896, "num_input_tokens_seen": 35043480, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.9453125, "step": 1630, "time_per_iteration": 2.4542155265808105 }, { "auxiliary_loss_clip": 0.01128359, "auxiliary_loss_mlp": 0.01050031, "balance_loss_clip": 1.02196896, "balance_loss_mlp": 1.03394079, "epoch": 0.09806102510145799, "flos": 17054826514560.0, "grad_norm": 2.2129601684385456, "language_loss": 0.86369681, "learning_rate": 3.906043389593892e-06, "loss": 0.8854807, "num_input_tokens_seen": 35061490, "router_z_loss_clip": 0.28125, "router_z_loss_mlp": 0.9453125, "step": 1631, "time_per_iteration": 2.3962056636810303 }, { "auxiliary_loss_clip": 0.01126007, "auxiliary_loss_mlp": 0.01048049, "balance_loss_clip": 1.0227648, "balance_loss_mlp": 1.03338003, "epoch": 0.09812114835412596, "flos": 23877344004480.0, "grad_norm": 2.0547621584516267, "language_loss": 0.83182806, "learning_rate": 3.9059289124277804e-06, "loss": 0.85356867, "num_input_tokens_seen": 35079670, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.92578125, "step": 1632, "time_per_iteration": 2.4445884227752686 }, { "auxiliary_loss_clip": 0.01129925, "auxiliary_loss_mlp": 0.01054484, "balance_loss_clip": 1.02919996, "balance_loss_mlp": 1.03536582, "epoch": 0.09818127160679392, "flos": 20594425950720.0, "grad_norm": 1.9895039626173088, "language_loss": 0.78635532, "learning_rate": 3.9058143672439684e-06, "loss": 0.80819941, "num_input_tokens_seen": 35099205, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9453125, "step": 1633, "time_per_iteration": 2.442203998565674 }, { "auxiliary_loss_clip": 0.01124605, "auxiliary_loss_mlp": 0.01049381, "balance_loss_clip": 1.02381027, "balance_loss_mlp": 1.03329194, "epoch": 0.0982413948594619, "flos": 15486423240960.0, "grad_norm": 2.3402569957392636, "language_loss": 0.73614502, "learning_rate": 3.905699754046544e-06, "loss": 0.75788486, "num_input_tokens_seen": 35115270, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.9140625, "step": 1634, "time_per_iteration": 3.9503259658813477 }, { "auxiliary_loss_clip": 0.01130281, "auxiliary_loss_mlp": 0.01056121, "balance_loss_clip": 1.02904904, "balance_loss_mlp": 1.03145909, "epoch": 0.09830151811212987, "flos": 24206784393600.0, "grad_norm": 2.5834969716202623, "language_loss": 0.72645545, "learning_rate": 3.905585072839597e-06, "loss": 0.74831951, "num_input_tokens_seen": 35134065, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 0.98828125, "step": 1635, "time_per_iteration": 2.446293592453003 }, { "auxiliary_loss_clip": 0.01132306, "auxiliary_loss_mlp": 0.01046668, "balance_loss_clip": 1.0192616, "balance_loss_mlp": 1.03499234, "epoch": 0.09836164136479783, "flos": 20593553166720.0, "grad_norm": 2.563211383861435, "language_loss": 0.78225213, "learning_rate": 3.905470323627221e-06, "loss": 0.80404186, "num_input_tokens_seen": 35154870, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.97265625, "step": 1636, "time_per_iteration": 3.8364906311035156 }, { "auxiliary_loss_clip": 0.01128111, "auxiliary_loss_mlp": 0.01056342, "balance_loss_clip": 1.03126025, "balance_loss_mlp": 1.03416944, "epoch": 0.09842176461746581, "flos": 19933241022720.0, "grad_norm": 1.9148254897281238, "language_loss": 0.69535017, "learning_rate": 3.9053555064135106e-06, "loss": 0.71719474, "num_input_tokens_seen": 35171850, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.9375, "step": 1637, "time_per_iteration": 2.4151923656463623 }, { "auxiliary_loss_clip": 0.01127765, "auxiliary_loss_mlp": 0.01053649, "balance_loss_clip": 1.02756572, "balance_loss_mlp": 1.03222847, "epoch": 0.09848188787013377, "flos": 21213610646400.0, "grad_norm": 2.1697528708543414, "language_loss": 0.7735889, "learning_rate": 3.905240621202563e-06, "loss": 0.79540306, "num_input_tokens_seen": 35188795, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.953125, "step": 1638, "time_per_iteration": 3.8829591274261475 }, { "auxiliary_loss_clip": 0.01125303, "auxiliary_loss_mlp": 0.01041223, "balance_loss_clip": 1.01531875, "balance_loss_mlp": 1.03256583, "epoch": 0.09854201112280174, "flos": 30152912135040.0, "grad_norm": 1.5071899996243445, "language_loss": 0.72347581, "learning_rate": 3.905125667998478e-06, "loss": 0.74514115, "num_input_tokens_seen": 35212100, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.92578125, "step": 1639, "time_per_iteration": 2.500878095626831 }, { "auxiliary_loss_clip": 0.01127181, "auxiliary_loss_mlp": 0.01039899, "balance_loss_clip": 1.01400685, "balance_loss_mlp": 1.03245223, "epoch": 0.09860213437546972, "flos": 21794740093440.0, "grad_norm": 1.704259235748373, "language_loss": 0.88319802, "learning_rate": 3.90501064680536e-06, "loss": 0.90486884, "num_input_tokens_seen": 35230390, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.9453125, "step": 1640, "time_per_iteration": 2.4600088596343994 }, { "auxiliary_loss_clip": 0.0113121, "auxiliary_loss_mlp": 0.01042793, "balance_loss_clip": 1.01701999, "balance_loss_mlp": 1.03482461, "epoch": 0.09866225762813768, "flos": 21834471087360.0, "grad_norm": 2.5186735761485917, "language_loss": 0.80387259, "learning_rate": 3.904895557627311e-06, "loss": 0.82561255, "num_input_tokens_seen": 35250405, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.96484375, "step": 1641, "time_per_iteration": 2.4450650215148926 }, { "auxiliary_loss_clip": 0.01128525, "auxiliary_loss_mlp": 0.01048598, "balance_loss_clip": 1.02246714, "balance_loss_mlp": 1.03371489, "epoch": 0.09872238088080565, "flos": 17598982965120.0, "grad_norm": 2.5171415456479145, "language_loss": 0.86056006, "learning_rate": 3.90478040046844e-06, "loss": 0.88233137, "num_input_tokens_seen": 35262820, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9453125, "step": 1642, "time_per_iteration": 2.3926727771759033 }, { "auxiliary_loss_clip": 0.01130981, "auxiliary_loss_mlp": 0.01045888, "balance_loss_clip": 1.02081847, "balance_loss_mlp": 1.03485167, "epoch": 0.09878250413347361, "flos": 27634906258560.0, "grad_norm": 1.5784985955077508, "language_loss": 0.80769372, "learning_rate": 3.9046651753328565e-06, "loss": 0.82946241, "num_input_tokens_seen": 35284490, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.9609375, "step": 1643, "time_per_iteration": 2.501354694366455 }, { "auxiliary_loss_clip": 0.01126589, "auxiliary_loss_mlp": 0.01054205, "balance_loss_clip": 1.0288372, "balance_loss_mlp": 1.03275657, "epoch": 0.0988426273861416, "flos": 16543802810880.0, "grad_norm": 1.9392785792078961, "language_loss": 0.82399493, "learning_rate": 3.904549882224672e-06, "loss": 0.8458029, "num_input_tokens_seen": 35302815, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9375, "step": 1644, "time_per_iteration": 2.427469491958618 }, { "auxiliary_loss_clip": 0.01124519, "auxiliary_loss_mlp": 0.01045704, "balance_loss_clip": 1.02088428, "balance_loss_mlp": 1.03299594, "epoch": 0.09890275063880956, "flos": 21214204139520.0, "grad_norm": 1.8836345415938323, "language_loss": 0.68441319, "learning_rate": 3.904434521148001e-06, "loss": 0.70611537, "num_input_tokens_seen": 35321175, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.91796875, "step": 1645, "time_per_iteration": 2.4562041759490967 }, { "auxiliary_loss_clip": 0.01037503, "auxiliary_loss_mlp": 0.01022829, "balance_loss_clip": 1.01913321, "balance_loss_mlp": 1.01097751, "epoch": 0.09896287389147752, "flos": 59376225876480.0, "grad_norm": 0.857572721094008, "language_loss": 0.60793686, "learning_rate": 3.904319092106961e-06, "loss": 0.62854016, "num_input_tokens_seen": 35381740, "router_z_loss_clip": 0.03686523, "router_z_loss_mlp": 0.265625, "step": 1646, "time_per_iteration": 3.085836172103882 }, { "auxiliary_loss_clip": 0.01124927, "auxiliary_loss_mlp": 0.01050674, "balance_loss_clip": 1.02424479, "balance_loss_mlp": 1.03245282, "epoch": 0.0990229971441455, "flos": 29641399672320.0, "grad_norm": 2.1236808177994075, "language_loss": 0.73563886, "learning_rate": 3.904203595105671e-06, "loss": 0.75739485, "num_input_tokens_seen": 35403760, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.92578125, "step": 1647, "time_per_iteration": 2.485992670059204 }, { "auxiliary_loss_clip": 0.01126761, "auxiliary_loss_mlp": 0.01049389, "balance_loss_clip": 1.02456927, "balance_loss_mlp": 1.03380871, "epoch": 0.09908312039681347, "flos": 21833807771520.0, "grad_norm": 2.0031847019009117, "language_loss": 0.84025264, "learning_rate": 3.904088030148253e-06, "loss": 0.86201417, "num_input_tokens_seen": 35424050, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.9296875, "step": 1648, "time_per_iteration": 2.454493522644043 }, { "auxiliary_loss_clip": 0.01034882, "auxiliary_loss_mlp": 0.01007785, "balance_loss_clip": 1.00447071, "balance_loss_mlp": 1.00925016, "epoch": 0.09914324364948143, "flos": 57560951312640.0, "grad_norm": 0.7264387503758062, "language_loss": 0.5566957, "learning_rate": 3.90397239723883e-06, "loss": 0.57712233, "num_input_tokens_seen": 35481690, "router_z_loss_clip": 0.03320312, "router_z_loss_mlp": 0.2578125, "step": 1649, "time_per_iteration": 3.019339084625244 }, { "auxiliary_loss_clip": 0.01120967, "auxiliary_loss_mlp": 0.01050197, "balance_loss_clip": 1.0244596, "balance_loss_mlp": 1.03003716, "epoch": 0.09920336690214941, "flos": 34122711744000.0, "grad_norm": 2.1078890210501404, "language_loss": 0.89719647, "learning_rate": 3.903856696381531e-06, "loss": 0.91890812, "num_input_tokens_seen": 35498635, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.91015625, "step": 1650, "time_per_iteration": 2.5014076232910156 }, { "auxiliary_loss_clip": 0.01032729, "auxiliary_loss_mlp": 0.01006617, "balance_loss_clip": 1.0029211, "balance_loss_mlp": 1.00755525, "epoch": 0.09926349015481738, "flos": 71212514716800.0, "grad_norm": 0.7942200012221744, "language_loss": 0.63744354, "learning_rate": 3.903740927580484e-06, "loss": 0.65783697, "num_input_tokens_seen": 35565720, "router_z_loss_clip": 0.03686523, "router_z_loss_mlp": 0.25195312, "step": 1651, "time_per_iteration": 3.259958505630493 }, { "auxiliary_loss_clip": 0.01123818, "auxiliary_loss_mlp": 0.01055441, "balance_loss_clip": 1.03074133, "balance_loss_mlp": 1.03246665, "epoch": 0.09932361340748534, "flos": 23147589432960.0, "grad_norm": 2.3108050741700272, "language_loss": 0.8803277, "learning_rate": 3.90362509083982e-06, "loss": 0.90212035, "num_input_tokens_seen": 35586000, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.9140625, "step": 1652, "time_per_iteration": 2.527146816253662 }, { "auxiliary_loss_clip": 0.01128132, "auxiliary_loss_mlp": 0.01048056, "balance_loss_clip": 1.02420235, "balance_loss_mlp": 1.03502429, "epoch": 0.09938373666015332, "flos": 19827628560000.0, "grad_norm": 2.0249951043498418, "language_loss": 0.82159197, "learning_rate": 3.903509186163673e-06, "loss": 0.84335381, "num_input_tokens_seen": 35604355, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.9296875, "step": 1653, "time_per_iteration": 2.4585115909576416 }, { "auxiliary_loss_clip": 0.01127948, "auxiliary_loss_mlp": 0.01055208, "balance_loss_clip": 1.02905321, "balance_loss_mlp": 1.0347662, "epoch": 0.09944385991282129, "flos": 22089581458560.0, "grad_norm": 2.025344607574988, "language_loss": 0.79414368, "learning_rate": 3.903393213556179e-06, "loss": 0.81597531, "num_input_tokens_seen": 35625495, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9296875, "step": 1654, "time_per_iteration": 2.5175716876983643 }, { "auxiliary_loss_clip": 0.01127728, "auxiliary_loss_mlp": 0.01055725, "balance_loss_clip": 1.03126287, "balance_loss_mlp": 1.03671968, "epoch": 0.09950398316548925, "flos": 19827838028160.0, "grad_norm": 1.631934643293413, "language_loss": 0.81203735, "learning_rate": 3.903277173021479e-06, "loss": 0.83387184, "num_input_tokens_seen": 35645030, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.91015625, "step": 1655, "time_per_iteration": 2.5070693492889404 }, { "auxiliary_loss_clip": 0.01122712, "auxiliary_loss_mlp": 0.01046783, "balance_loss_clip": 1.02300107, "balance_loss_mlp": 1.03238094, "epoch": 0.09956410641815722, "flos": 25002699724800.0, "grad_norm": 1.8733174755885336, "language_loss": 0.80317938, "learning_rate": 3.903161064563712e-06, "loss": 0.8248744, "num_input_tokens_seen": 35664305, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.90234375, "step": 1656, "time_per_iteration": 2.480952739715576 }, { "auxiliary_loss_clip": 0.01129665, "auxiliary_loss_mlp": 0.01052113, "balance_loss_clip": 1.02768731, "balance_loss_mlp": 1.03797841, "epoch": 0.0996242296708252, "flos": 19316709590400.0, "grad_norm": 1.677361575413214, "language_loss": 0.88713956, "learning_rate": 3.9030448881870206e-06, "loss": 0.90895736, "num_input_tokens_seen": 35684060, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.91796875, "step": 1657, "time_per_iteration": 2.445462465286255 }, { "auxiliary_loss_clip": 0.01131792, "auxiliary_loss_mlp": 0.01045285, "balance_loss_clip": 1.01911855, "balance_loss_mlp": 1.03498387, "epoch": 0.09968435292349316, "flos": 21870536388480.0, "grad_norm": 2.4750170046506597, "language_loss": 0.84711289, "learning_rate": 3.902928643895554e-06, "loss": 0.86888373, "num_input_tokens_seen": 35703250, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.96875, "step": 1658, "time_per_iteration": 2.4258053302764893 }, { "auxiliary_loss_clip": 0.01031565, "auxiliary_loss_mlp": 0.01005096, "balance_loss_clip": 1.00178158, "balance_loss_mlp": 1.00624537, "epoch": 0.09974447617616113, "flos": 65381636839680.0, "grad_norm": 0.9018418713282724, "language_loss": 0.60856706, "learning_rate": 3.9028123316934575e-06, "loss": 0.62893367, "num_input_tokens_seen": 35762165, "router_z_loss_clip": 0.03320312, "router_z_loss_mlp": 0.25390625, "step": 1659, "time_per_iteration": 3.0654025077819824 }, { "auxiliary_loss_clip": 0.01127454, "auxiliary_loss_mlp": 0.01047572, "balance_loss_clip": 1.02435017, "balance_loss_mlp": 1.03469169, "epoch": 0.0998045994288291, "flos": 23658682959360.0, "grad_norm": 4.848363324112766, "language_loss": 0.85086519, "learning_rate": 3.902695951584885e-06, "loss": 0.87261546, "num_input_tokens_seen": 35781520, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.92578125, "step": 1660, "time_per_iteration": 2.453413963317871 }, { "auxiliary_loss_clip": 0.0112804, "auxiliary_loss_mlp": 0.01046614, "balance_loss_clip": 1.02045989, "balance_loss_mlp": 1.03721237, "epoch": 0.09986472268149707, "flos": 19608688224000.0, "grad_norm": 1.9418971738798911, "language_loss": 0.80042166, "learning_rate": 3.902579503573987e-06, "loss": 0.82216817, "num_input_tokens_seen": 35799565, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.90625, "step": 1661, "time_per_iteration": 2.4215495586395264 }, { "auxiliary_loss_clip": 0.01128891, "auxiliary_loss_mlp": 0.01046257, "balance_loss_clip": 1.01982832, "balance_loss_mlp": 1.03200734, "epoch": 0.09992484593416504, "flos": 26212125732480.0, "grad_norm": 1.8201023566804326, "language_loss": 0.83474773, "learning_rate": 3.902462987664922e-06, "loss": 0.85649925, "num_input_tokens_seen": 35821085, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.96875, "step": 1662, "time_per_iteration": 2.4538283348083496 }, { "auxiliary_loss_clip": 0.01127811, "auxiliary_loss_mlp": 0.01049377, "balance_loss_clip": 1.02310371, "balance_loss_mlp": 1.03647661, "epoch": 0.09998496918683301, "flos": 17492672275200.0, "grad_norm": 2.0821206111460366, "language_loss": 0.88856053, "learning_rate": 3.902346403861846e-06, "loss": 0.91033244, "num_input_tokens_seen": 35839840, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9140625, "step": 1663, "time_per_iteration": 2.412142753601074 }, { "auxiliary_loss_clip": 0.01128487, "auxiliary_loss_mlp": 0.01050009, "balance_loss_clip": 1.02400947, "balance_loss_mlp": 1.03475738, "epoch": 0.10004509243950098, "flos": 22783794399360.0, "grad_norm": 1.6838586642707083, "language_loss": 0.70417583, "learning_rate": 3.9022297521689196e-06, "loss": 0.72596073, "num_input_tokens_seen": 35861545, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9375, "step": 1664, "time_per_iteration": 2.4572854042053223 }, { "auxiliary_loss_clip": 0.0112979, "auxiliary_loss_mlp": 0.01051836, "balance_loss_clip": 1.02657557, "balance_loss_mlp": 1.03681624, "epoch": 0.10010521569216894, "flos": 16252452581760.0, "grad_norm": 2.3885815761458833, "language_loss": 0.78945351, "learning_rate": 3.902113032590307e-06, "loss": 0.8112697, "num_input_tokens_seen": 35878295, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.9296875, "step": 1665, "time_per_iteration": 2.4018146991729736 }, { "auxiliary_loss_clip": 0.01133934, "auxiliary_loss_mlp": 0.0106273, "balance_loss_clip": 1.03639627, "balance_loss_mlp": 1.03929853, "epoch": 0.10016533894483691, "flos": 23401512817920.0, "grad_norm": 1.7453432919123004, "language_loss": 0.70129985, "learning_rate": 3.901996245130174e-06, "loss": 0.72326648, "num_input_tokens_seen": 35898990, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.9453125, "step": 1666, "time_per_iteration": 2.441276788711548 }, { "auxiliary_loss_clip": 0.01128306, "auxiliary_loss_mlp": 0.0106182, "balance_loss_clip": 1.03455698, "balance_loss_mlp": 1.03533638, "epoch": 0.10022546219750489, "flos": 19353158916480.0, "grad_norm": 2.1816995475096856, "language_loss": 0.78218007, "learning_rate": 3.901879389792686e-06, "loss": 0.80408126, "num_input_tokens_seen": 35916225, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.9296875, "step": 1667, "time_per_iteration": 2.3924455642700195 }, { "auxiliary_loss_clip": 0.01128903, "auxiliary_loss_mlp": 0.01055359, "balance_loss_clip": 1.02883434, "balance_loss_mlp": 1.03445554, "epoch": 0.10028558545017285, "flos": 27084640320000.0, "grad_norm": 2.5698885473644046, "language_loss": 0.77251256, "learning_rate": 3.9017624665820155e-06, "loss": 0.79435515, "num_input_tokens_seen": 35934630, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.9453125, "step": 1668, "time_per_iteration": 2.4461889266967773 }, { "auxiliary_loss_clip": 0.01126996, "auxiliary_loss_mlp": 0.01049487, "balance_loss_clip": 1.02247405, "balance_loss_mlp": 1.03288484, "epoch": 0.10034570870284082, "flos": 25845991637760.0, "grad_norm": 2.261072014975546, "language_loss": 0.78254324, "learning_rate": 3.901645475502334e-06, "loss": 0.80430806, "num_input_tokens_seen": 35953855, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.94140625, "step": 1669, "time_per_iteration": 2.4449715614318848 }, { "auxiliary_loss_clip": 0.01131728, "auxiliary_loss_mlp": 0.01064008, "balance_loss_clip": 1.03719783, "balance_loss_mlp": 1.0359478, "epoch": 0.1004058319555088, "flos": 26248400501760.0, "grad_norm": 3.2733356873499346, "language_loss": 0.85289216, "learning_rate": 3.901528416557817e-06, "loss": 0.87484956, "num_input_tokens_seen": 35974555, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.9609375, "step": 1670, "time_per_iteration": 2.451265335083008 }, { "auxiliary_loss_clip": 0.01121068, "auxiliary_loss_mlp": 0.01044712, "balance_loss_clip": 1.02132273, "balance_loss_mlp": 1.03208125, "epoch": 0.10046595520817676, "flos": 25373302473600.0, "grad_norm": 1.5981438708977425, "language_loss": 0.77034068, "learning_rate": 3.901411289752643e-06, "loss": 0.79199851, "num_input_tokens_seen": 35996830, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.890625, "step": 1671, "time_per_iteration": 2.4486119747161865 }, { "auxiliary_loss_clip": 0.01029094, "auxiliary_loss_mlp": 0.01005826, "balance_loss_clip": 1.00208318, "balance_loss_mlp": 1.00429285, "epoch": 0.10052607846084473, "flos": 67458934224000.0, "grad_norm": 0.7714351547673884, "language_loss": 0.60759377, "learning_rate": 3.901294095090991e-06, "loss": 0.62794292, "num_input_tokens_seen": 36054465, "router_z_loss_clip": 0.03735352, "router_z_loss_mlp": 0.24804688, "step": 1672, "time_per_iteration": 3.0407564640045166 }, { "auxiliary_loss_clip": 0.01131427, "auxiliary_loss_mlp": 0.01055055, "balance_loss_clip": 1.02826881, "balance_loss_mlp": 1.03504014, "epoch": 0.10058620171351271, "flos": 21359442862080.0, "grad_norm": 2.0496448377842214, "language_loss": 0.76836884, "learning_rate": 3.901176832577043e-06, "loss": 0.79023367, "num_input_tokens_seen": 36073480, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.96484375, "step": 1673, "time_per_iteration": 3.8078036308288574 }, { "auxiliary_loss_clip": 0.01125647, "auxiliary_loss_mlp": 0.01041383, "balance_loss_clip": 1.01638508, "balance_loss_mlp": 1.03168154, "epoch": 0.10064632496618067, "flos": 16799192472960.0, "grad_norm": 2.1322106564210506, "language_loss": 0.73229301, "learning_rate": 3.901059502214984e-06, "loss": 0.75396329, "num_input_tokens_seen": 36091830, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.94140625, "step": 1674, "time_per_iteration": 2.386910915374756 }, { "auxiliary_loss_clip": 0.0112982, "auxiliary_loss_mlp": 0.01044697, "balance_loss_clip": 1.0194962, "balance_loss_mlp": 1.03423774, "epoch": 0.10070644821884864, "flos": 23623280974080.0, "grad_norm": 2.1319658939344626, "language_loss": 0.79347897, "learning_rate": 3.900942104009003e-06, "loss": 0.81522405, "num_input_tokens_seen": 36111400, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.953125, "step": 1675, "time_per_iteration": 3.822953939437866 }, { "auxiliary_loss_clip": 0.01125259, "auxiliary_loss_mlp": 0.01056213, "balance_loss_clip": 1.03017747, "balance_loss_mlp": 1.03400826, "epoch": 0.1007665714715166, "flos": 24461406005760.0, "grad_norm": 2.608486034898942, "language_loss": 0.81366646, "learning_rate": 3.900824637963287e-06, "loss": 0.83548117, "num_input_tokens_seen": 36129345, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.9140625, "step": 1676, "time_per_iteration": 2.427243232727051 }, { "auxiliary_loss_clip": 0.01130224, "auxiliary_loss_mlp": 0.01053137, "balance_loss_clip": 1.02775776, "balance_loss_mlp": 1.03403139, "epoch": 0.10082669472418458, "flos": 16798214954880.0, "grad_norm": 2.002097366993846, "language_loss": 0.8618263, "learning_rate": 3.9007071040820285e-06, "loss": 0.88365984, "num_input_tokens_seen": 36146255, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.96484375, "step": 1677, "time_per_iteration": 3.9178919792175293 }, { "auxiliary_loss_clip": 0.01127388, "auxiliary_loss_mlp": 0.01050704, "balance_loss_clip": 1.02513361, "balance_loss_mlp": 1.03335094, "epoch": 0.10088681797685255, "flos": 13552653922560.0, "grad_norm": 2.024853065057127, "language_loss": 0.85943526, "learning_rate": 3.900589502369423e-06, "loss": 0.88121617, "num_input_tokens_seen": 36164050, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.94140625, "step": 1678, "time_per_iteration": 2.389723062515259 }, { "auxiliary_loss_clip": 0.0113194, "auxiliary_loss_mlp": 0.0104625, "balance_loss_clip": 1.01992822, "balance_loss_mlp": 1.03805208, "epoch": 0.10094694122952051, "flos": 25264513077120.0, "grad_norm": 2.1347749920294357, "language_loss": 0.89958286, "learning_rate": 3.9004718328296676e-06, "loss": 0.92136478, "num_input_tokens_seen": 36183530, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.94140625, "step": 1679, "time_per_iteration": 2.444307565689087 }, { "auxiliary_loss_clip": 0.01029359, "auxiliary_loss_mlp": 0.01012468, "balance_loss_clip": 1.00867677, "balance_loss_mlp": 1.0046916, "epoch": 0.10100706448218849, "flos": 69850762980480.0, "grad_norm": 0.7774210750054823, "language_loss": 0.52998149, "learning_rate": 3.900354095466962e-06, "loss": 0.55039978, "num_input_tokens_seen": 36248550, "router_z_loss_clip": 0.0378418, "router_z_loss_mlp": 0.24707031, "step": 1680, "time_per_iteration": 3.087531328201294 }, { "auxiliary_loss_clip": 0.01125771, "auxiliary_loss_mlp": 0.0104457, "balance_loss_clip": 1.01921439, "balance_loss_mlp": 1.03489673, "epoch": 0.10106718773485646, "flos": 20006244408960.0, "grad_norm": 1.7655943226966269, "language_loss": 0.76840341, "learning_rate": 3.900236290285506e-06, "loss": 0.79010677, "num_input_tokens_seen": 36266065, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.91015625, "step": 1681, "time_per_iteration": 2.3962461948394775 }, { "auxiliary_loss_clip": 0.01132826, "auxiliary_loss_mlp": 0.01058513, "balance_loss_clip": 1.02968824, "balance_loss_mlp": 1.03300309, "epoch": 0.10112731098752442, "flos": 13478987220480.0, "grad_norm": 2.3429988295473376, "language_loss": 0.93676221, "learning_rate": 3.900118417289504e-06, "loss": 0.95867562, "num_input_tokens_seen": 36280960, "router_z_loss_clip": 0.2890625, "router_z_loss_mlp": 1.0, "step": 1682, "time_per_iteration": 2.3901689052581787 }, { "auxiliary_loss_clip": 0.01129013, "auxiliary_loss_mlp": 0.01044731, "balance_loss_clip": 1.01974475, "balance_loss_mlp": 1.03455329, "epoch": 0.1011874342401924, "flos": 18514894239360.0, "grad_norm": 2.5183537344737164, "language_loss": 0.87868714, "learning_rate": 3.900000476483164e-06, "loss": 0.9004246, "num_input_tokens_seen": 36299010, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.9453125, "step": 1683, "time_per_iteration": 2.381648302078247 }, { "auxiliary_loss_clip": 0.01128587, "auxiliary_loss_mlp": 0.01050638, "balance_loss_clip": 1.02581882, "balance_loss_mlp": 1.03547812, "epoch": 0.10124755749286037, "flos": 20701853804160.0, "grad_norm": 1.7995433586265865, "language_loss": 0.7452631, "learning_rate": 3.8998824678706946e-06, "loss": 0.76705539, "num_input_tokens_seen": 36318400, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.93359375, "step": 1684, "time_per_iteration": 2.424262523651123 }, { "auxiliary_loss_clip": 0.01026459, "auxiliary_loss_mlp": 0.01004289, "balance_loss_clip": 1.00049782, "balance_loss_mlp": 1.00221205, "epoch": 0.10130768074552833, "flos": 56106015557760.0, "grad_norm": 0.7860492151257247, "language_loss": 0.61080587, "learning_rate": 3.899764391456306e-06, "loss": 0.63111335, "num_input_tokens_seen": 36381815, "router_z_loss_clip": 0.0378418, "router_z_loss_mlp": 0.2421875, "step": 1685, "time_per_iteration": 3.1197712421417236 }, { "auxiliary_loss_clip": 0.01127716, "auxiliary_loss_mlp": 0.01050595, "balance_loss_clip": 1.02488136, "balance_loss_mlp": 1.03425086, "epoch": 0.1013678039981963, "flos": 33400916962560.0, "grad_norm": 2.7186454758616514, "language_loss": 0.61819071, "learning_rate": 3.8996462472442145e-06, "loss": 0.63997382, "num_input_tokens_seen": 36404320, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.9375, "step": 1686, "time_per_iteration": 2.557370185852051 }, { "auxiliary_loss_clip": 0.01129063, "auxiliary_loss_mlp": 0.0104935, "balance_loss_clip": 1.02246845, "balance_loss_mlp": 1.03644657, "epoch": 0.10142792725086427, "flos": 31903980975360.0, "grad_norm": 2.880887399024693, "language_loss": 0.81339729, "learning_rate": 3.8995280352386344e-06, "loss": 0.83518136, "num_input_tokens_seen": 36427510, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.92578125, "step": 1687, "time_per_iteration": 2.5703492164611816 }, { "auxiliary_loss_clip": 0.01131885, "auxiliary_loss_mlp": 0.01052296, "balance_loss_clip": 1.02561688, "balance_loss_mlp": 1.03477442, "epoch": 0.10148805050353224, "flos": 28474637212800.0, "grad_norm": 1.9758616894600414, "language_loss": 0.71980017, "learning_rate": 3.899409755443785e-06, "loss": 0.741642, "num_input_tokens_seen": 36448230, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.97265625, "step": 1688, "time_per_iteration": 2.483548164367676 }, { "auxiliary_loss_clip": 0.01126728, "auxiliary_loss_mlp": 0.0105274, "balance_loss_clip": 1.02780175, "balance_loss_mlp": 1.03525734, "epoch": 0.1015481737562002, "flos": 25147903536000.0, "grad_norm": 2.4480015311261627, "language_loss": 0.86638576, "learning_rate": 3.899291407863887e-06, "loss": 0.88818043, "num_input_tokens_seen": 36464395, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.9140625, "step": 1689, "time_per_iteration": 2.476289749145508 }, { "auxiliary_loss_clip": 0.01123046, "auxiliary_loss_mlp": 0.01043488, "balance_loss_clip": 1.01890743, "balance_loss_mlp": 1.03043246, "epoch": 0.10160829700886818, "flos": 30881479720320.0, "grad_norm": 1.7647076627727838, "language_loss": 0.88198733, "learning_rate": 3.899172992503165e-06, "loss": 0.90365267, "num_input_tokens_seen": 36486475, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.92578125, "step": 1690, "time_per_iteration": 2.602295160293579 }, { "auxiliary_loss_clip": 0.01126595, "auxiliary_loss_mlp": 0.01044899, "balance_loss_clip": 1.02053297, "balance_loss_mlp": 1.03325152, "epoch": 0.10166842026153615, "flos": 20410992334080.0, "grad_norm": 2.4958265577871694, "language_loss": 0.83553779, "learning_rate": 3.899054509365843e-06, "loss": 0.85725272, "num_input_tokens_seen": 36505310, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.93359375, "step": 1691, "time_per_iteration": 2.4795753955841064 }, { "auxiliary_loss_clip": 0.01127851, "auxiliary_loss_mlp": 0.01048939, "balance_loss_clip": 1.02351224, "balance_loss_mlp": 1.03416157, "epoch": 0.10172854351420411, "flos": 33475491360000.0, "grad_norm": 1.5062507315521056, "language_loss": 0.6655491, "learning_rate": 3.89893595845615e-06, "loss": 0.68731701, "num_input_tokens_seen": 36529820, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9375, "step": 1692, "time_per_iteration": 2.5656216144561768 }, { "auxiliary_loss_clip": 0.01126495, "auxiliary_loss_mlp": 0.01053501, "balance_loss_clip": 1.02723956, "balance_loss_mlp": 1.03327119, "epoch": 0.1017886667668721, "flos": 23549195335680.0, "grad_norm": 1.6614169439877764, "language_loss": 0.75763559, "learning_rate": 3.898817339778319e-06, "loss": 0.77943558, "num_input_tokens_seen": 36549000, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9296875, "step": 1693, "time_per_iteration": 2.5104470252990723 }, { "auxiliary_loss_clip": 0.01126969, "auxiliary_loss_mlp": 0.01048846, "balance_loss_clip": 1.02250028, "balance_loss_mlp": 1.03276646, "epoch": 0.10184879001954006, "flos": 23294922837120.0, "grad_norm": 1.6836904940668604, "language_loss": 0.8728255, "learning_rate": 3.898698653336581e-06, "loss": 0.89458358, "num_input_tokens_seen": 36567515, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.9453125, "step": 1694, "time_per_iteration": 2.4375650882720947 }, { "auxiliary_loss_clip": 0.01130871, "auxiliary_loss_mlp": 0.01048342, "balance_loss_clip": 1.02084053, "balance_loss_mlp": 1.0317018, "epoch": 0.10190891327220802, "flos": 18332123938560.0, "grad_norm": 2.3117364563831915, "language_loss": 0.7957328, "learning_rate": 3.8985798991351715e-06, "loss": 0.81752491, "num_input_tokens_seen": 36586190, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 0.9921875, "step": 1695, "time_per_iteration": 2.3535845279693604 }, { "auxiliary_loss_clip": 0.0112953, "auxiliary_loss_mlp": 0.01053714, "balance_loss_clip": 1.02674866, "balance_loss_mlp": 1.03284919, "epoch": 0.10196903652487599, "flos": 26464268638080.0, "grad_norm": 1.771862970932022, "language_loss": 0.86338663, "learning_rate": 3.898461077178329e-06, "loss": 0.88521904, "num_input_tokens_seen": 36607495, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.96875, "step": 1696, "time_per_iteration": 2.464308261871338 }, { "auxiliary_loss_clip": 0.01124274, "auxiliary_loss_mlp": 0.0105109, "balance_loss_clip": 1.02643776, "balance_loss_mlp": 1.0333662, "epoch": 0.10202915977754397, "flos": 21868511529600.0, "grad_norm": 1.8515751852928584, "language_loss": 0.82061327, "learning_rate": 3.898342187470296e-06, "loss": 0.84236693, "num_input_tokens_seen": 36628555, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.91015625, "step": 1697, "time_per_iteration": 2.4317948818206787 }, { "auxiliary_loss_clip": 0.01128753, "auxiliary_loss_mlp": 0.01047324, "balance_loss_clip": 1.02047801, "balance_loss_mlp": 1.03356361, "epoch": 0.10208928303021193, "flos": 22308661440000.0, "grad_norm": 2.003597479785428, "language_loss": 0.80216718, "learning_rate": 3.898223230015311e-06, "loss": 0.82392788, "num_input_tokens_seen": 36646250, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.94921875, "step": 1698, "time_per_iteration": 2.4211795330047607 }, { "auxiliary_loss_clip": 0.01126562, "auxiliary_loss_mlp": 0.01038661, "balance_loss_clip": 1.01368654, "balance_loss_mlp": 1.03277946, "epoch": 0.1021494062828799, "flos": 22124529596160.0, "grad_norm": 3.2384389278339683, "language_loss": 0.75553715, "learning_rate": 3.8981042048176235e-06, "loss": 0.77718937, "num_input_tokens_seen": 36666675, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.9375, "step": 1699, "time_per_iteration": 2.40696120262146 }, { "auxiliary_loss_clip": 0.01126844, "auxiliary_loss_mlp": 0.01043679, "balance_loss_clip": 1.01847851, "balance_loss_mlp": 1.03395414, "epoch": 0.10220952953554788, "flos": 19645696131840.0, "grad_norm": 1.660742949698858, "language_loss": 0.79711759, "learning_rate": 3.897985111881478e-06, "loss": 0.81882286, "num_input_tokens_seen": 36685225, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.9296875, "step": 1700, "time_per_iteration": 2.3866331577301025 }, { "auxiliary_loss_clip": 0.01125573, "auxiliary_loss_mlp": 0.01046093, "balance_loss_clip": 1.02139282, "balance_loss_mlp": 1.03189898, "epoch": 0.10226965278821584, "flos": 29786044901760.0, "grad_norm": 1.7820552114416843, "language_loss": 0.77036595, "learning_rate": 3.897865951211127e-06, "loss": 0.79208261, "num_input_tokens_seen": 36705985, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.9375, "step": 1701, "time_per_iteration": 2.4395692348480225 }, { "auxiliary_loss_clip": 0.01129793, "auxiliary_loss_mlp": 0.01043111, "balance_loss_clip": 1.01714671, "balance_loss_mlp": 1.03423548, "epoch": 0.10232977604088381, "flos": 27015581917440.0, "grad_norm": 2.8523336534744077, "language_loss": 0.78233707, "learning_rate": 3.897746722810822e-06, "loss": 0.80406612, "num_input_tokens_seen": 36725815, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.953125, "step": 1702, "time_per_iteration": 2.436391592025757 }, { "auxiliary_loss_clip": 0.01124749, "auxiliary_loss_mlp": 0.01042127, "balance_loss_clip": 1.01767766, "balance_loss_mlp": 1.03302264, "epoch": 0.10238989929355179, "flos": 20776463112960.0, "grad_norm": 2.0057943972936663, "language_loss": 0.94855535, "learning_rate": 3.897627426684818e-06, "loss": 0.97022408, "num_input_tokens_seen": 36742345, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.91796875, "step": 1703, "time_per_iteration": 2.3969039916992188 }, { "auxiliary_loss_clip": 0.01033287, "auxiliary_loss_mlp": 0.01006063, "balance_loss_clip": 1.00258231, "balance_loss_mlp": 1.00845075, "epoch": 0.10245002254621975, "flos": 57695297690880.0, "grad_norm": 0.8679415430569597, "language_loss": 0.55032927, "learning_rate": 3.897508062837372e-06, "loss": 0.57072276, "num_input_tokens_seen": 36798775, "router_z_loss_clip": 0.03491211, "router_z_loss_mlp": 0.24804688, "step": 1704, "time_per_iteration": 2.9495198726654053 }, { "auxiliary_loss_clip": 0.01126333, "auxiliary_loss_mlp": 0.01041418, "balance_loss_clip": 1.01575208, "balance_loss_mlp": 1.03269148, "epoch": 0.10251014579888772, "flos": 16799192472960.0, "grad_norm": 2.1628112329074147, "language_loss": 0.83624583, "learning_rate": 3.897388631272745e-06, "loss": 0.85792339, "num_input_tokens_seen": 36816295, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.9375, "step": 1705, "time_per_iteration": 2.3643715381622314 }, { "auxiliary_loss_clip": 0.01028383, "auxiliary_loss_mlp": 0.01004697, "balance_loss_clip": 1.0013113, "balance_loss_mlp": 1.00360084, "epoch": 0.1025702690515557, "flos": 68562328832640.0, "grad_norm": 0.7604867521608939, "language_loss": 0.60402644, "learning_rate": 3.8972691319951975e-06, "loss": 0.62435722, "num_input_tokens_seen": 36882030, "router_z_loss_clip": 0.03393555, "router_z_loss_mlp": 0.24804688, "step": 1706, "time_per_iteration": 3.0974771976470947 }, { "auxiliary_loss_clip": 0.01129113, "auxiliary_loss_mlp": 0.01045736, "balance_loss_clip": 1.02063107, "balance_loss_mlp": 1.03540444, "epoch": 0.10263039230422366, "flos": 14865737356800.0, "grad_norm": 2.324386550135008, "language_loss": 0.86008024, "learning_rate": 3.897149565008996e-06, "loss": 0.88182867, "num_input_tokens_seen": 36899245, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.9375, "step": 1707, "time_per_iteration": 2.3840553760528564 }, { "auxiliary_loss_clip": 0.01126693, "auxiliary_loss_mlp": 0.0104306, "balance_loss_clip": 1.01841986, "balance_loss_mlp": 1.03309667, "epoch": 0.10269051555689163, "flos": 25336434211200.0, "grad_norm": 1.4926963578695893, "language_loss": 0.7271347, "learning_rate": 3.897029930318406e-06, "loss": 0.74883235, "num_input_tokens_seen": 36920950, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.9375, "step": 1708, "time_per_iteration": 2.418433904647827 }, { "auxiliary_loss_clip": 0.01124308, "auxiliary_loss_mlp": 0.01049843, "balance_loss_clip": 1.02458286, "balance_loss_mlp": 1.03183699, "epoch": 0.10275063880955959, "flos": 21067778430720.0, "grad_norm": 1.7060560352128098, "language_loss": 0.91177273, "learning_rate": 3.8969102279276974e-06, "loss": 0.93351424, "num_input_tokens_seen": 36938900, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.92578125, "step": 1709, "time_per_iteration": 2.3767495155334473 }, { "auxiliary_loss_clip": 0.01125528, "auxiliary_loss_mlp": 0.01043814, "balance_loss_clip": 1.01924491, "balance_loss_mlp": 1.03394461, "epoch": 0.10281076206222757, "flos": 30365638248960.0, "grad_norm": 2.6524481322250177, "language_loss": 0.88083231, "learning_rate": 3.896790457841142e-06, "loss": 0.90252578, "num_input_tokens_seen": 36957010, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.9140625, "step": 1710, "time_per_iteration": 2.436161994934082 }, { "auxiliary_loss_clip": 0.01120898, "auxiliary_loss_mlp": 0.01043799, "balance_loss_clip": 1.01883698, "balance_loss_mlp": 1.03276277, "epoch": 0.10287088531489554, "flos": 22417241368320.0, "grad_norm": 2.085302421561381, "language_loss": 0.79199672, "learning_rate": 3.896670620063015e-06, "loss": 0.81364369, "num_input_tokens_seen": 36977690, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.8828125, "step": 1711, "time_per_iteration": 2.4026315212249756 }, { "auxiliary_loss_clip": 0.01127818, "auxiliary_loss_mlp": 0.01048941, "balance_loss_clip": 1.0223453, "balance_loss_mlp": 1.03480744, "epoch": 0.1029310085675635, "flos": 25114910434560.0, "grad_norm": 2.9395482296819866, "language_loss": 0.73571283, "learning_rate": 3.896550714597592e-06, "loss": 0.75748044, "num_input_tokens_seen": 36997300, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.9296875, "step": 1712, "time_per_iteration": 2.428074598312378 }, { "auxiliary_loss_clip": 0.01124127, "auxiliary_loss_mlp": 0.01050446, "balance_loss_clip": 1.02728355, "balance_loss_mlp": 1.03561556, "epoch": 0.10299113182023148, "flos": 19864601556480.0, "grad_norm": 1.8027700892132426, "language_loss": 0.86771899, "learning_rate": 3.896430741449153e-06, "loss": 0.88946474, "num_input_tokens_seen": 37016110, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.88671875, "step": 1713, "time_per_iteration": 3.7723255157470703 }, { "auxiliary_loss_clip": 0.01124946, "auxiliary_loss_mlp": 0.01043759, "balance_loss_clip": 1.02115691, "balance_loss_mlp": 1.03272271, "epoch": 0.10305125507289944, "flos": 20446603787520.0, "grad_norm": 1.5695577095444464, "language_loss": 0.72571588, "learning_rate": 3.8963107006219785e-06, "loss": 0.74740291, "num_input_tokens_seen": 37036405, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.921875, "step": 1714, "time_per_iteration": 2.414334535598755 }, { "auxiliary_loss_clip": 0.01130338, "auxiliary_loss_mlp": 0.01050869, "balance_loss_clip": 1.02520323, "balance_loss_mlp": 1.03358746, "epoch": 0.10311137832556741, "flos": 26249552576640.0, "grad_norm": 2.3658421272882806, "language_loss": 0.90832257, "learning_rate": 3.896190592120353e-06, "loss": 0.93013465, "num_input_tokens_seen": 37057580, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.96875, "step": 1715, "time_per_iteration": 3.8215456008911133 }, { "auxiliary_loss_clip": 0.01126425, "auxiliary_loss_mlp": 0.01044727, "balance_loss_clip": 1.01991987, "balance_loss_mlp": 1.03373456, "epoch": 0.10317150157823539, "flos": 35297468904960.0, "grad_norm": 2.4404738181742807, "language_loss": 0.75811809, "learning_rate": 3.896070415948563e-06, "loss": 0.77982962, "num_input_tokens_seen": 37079120, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.92578125, "step": 1716, "time_per_iteration": 2.520768165588379 }, { "auxiliary_loss_clip": 0.01131289, "auxiliary_loss_mlp": 0.01053478, "balance_loss_clip": 1.02689457, "balance_loss_mlp": 1.03468513, "epoch": 0.10323162483090335, "flos": 25738738341120.0, "grad_norm": 1.8637532906378036, "language_loss": 0.8557725, "learning_rate": 3.895950172110897e-06, "loss": 0.87762022, "num_input_tokens_seen": 37099710, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.96484375, "step": 1717, "time_per_iteration": 3.9256229400634766 }, { "auxiliary_loss_clip": 0.01126121, "auxiliary_loss_mlp": 0.01052604, "balance_loss_clip": 1.02903628, "balance_loss_mlp": 1.03377187, "epoch": 0.10329174808357132, "flos": 16288936819200.0, "grad_norm": 1.8295567488097717, "language_loss": 0.8306402, "learning_rate": 3.895829860611646e-06, "loss": 0.85242748, "num_input_tokens_seen": 37117775, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.921875, "step": 1718, "time_per_iteration": 2.3728268146514893 }, { "auxiliary_loss_clip": 0.01126617, "auxiliary_loss_mlp": 0.01044083, "balance_loss_clip": 1.02005076, "balance_loss_mlp": 1.0349555, "epoch": 0.10335187133623928, "flos": 36685615495680.0, "grad_norm": 1.9096384483571365, "language_loss": 0.72850704, "learning_rate": 3.895709481455105e-06, "loss": 0.75021404, "num_input_tokens_seen": 37140280, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.91796875, "step": 1719, "time_per_iteration": 2.532106399536133 }, { "auxiliary_loss_clip": 0.0112546, "auxiliary_loss_mlp": 0.01044799, "balance_loss_clip": 1.02068257, "balance_loss_mlp": 1.03439403, "epoch": 0.10341199458890726, "flos": 14974771132800.0, "grad_norm": 2.126221701693877, "language_loss": 0.92706668, "learning_rate": 3.895589034645568e-06, "loss": 0.94876933, "num_input_tokens_seen": 37158350, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.91015625, "step": 1720, "time_per_iteration": 2.364673137664795 }, { "auxiliary_loss_clip": 0.01125416, "auxiliary_loss_mlp": 0.01043045, "balance_loss_clip": 1.01660395, "balance_loss_mlp": 1.03304362, "epoch": 0.10347211784157523, "flos": 21030561054720.0, "grad_norm": 2.0938448238925855, "language_loss": 0.79727536, "learning_rate": 3.8954685201873344e-06, "loss": 0.81895995, "num_input_tokens_seen": 37177120, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.921875, "step": 1721, "time_per_iteration": 2.389920949935913 }, { "auxiliary_loss_clip": 0.01129995, "auxiliary_loss_mlp": 0.01047324, "balance_loss_clip": 1.02155101, "balance_loss_mlp": 1.03391886, "epoch": 0.1035322410942432, "flos": 19791074499840.0, "grad_norm": 3.5107157029297373, "language_loss": 0.80865979, "learning_rate": 3.895347938084706e-06, "loss": 0.83043295, "num_input_tokens_seen": 37195895, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.9609375, "step": 1722, "time_per_iteration": 2.397848129272461 }, { "auxiliary_loss_clip": 0.01026926, "auxiliary_loss_mlp": 0.01003916, "balance_loss_clip": 0.99988711, "balance_loss_mlp": 1.00182128, "epoch": 0.10359236434691117, "flos": 52696014554880.0, "grad_norm": 0.9208959392453442, "language_loss": 0.6713531, "learning_rate": 3.895227288341984e-06, "loss": 0.6916616, "num_input_tokens_seen": 37247270, "router_z_loss_clip": 0.0402832, "router_z_loss_mlp": 0.25195312, "step": 1723, "time_per_iteration": 2.7822790145874023 }, { "auxiliary_loss_clip": 0.01127009, "auxiliary_loss_mlp": 0.01054408, "balance_loss_clip": 1.02935004, "balance_loss_mlp": 1.03282261, "epoch": 0.10365248759957914, "flos": 18404429097600.0, "grad_norm": 3.3654607027691053, "language_loss": 0.77949142, "learning_rate": 3.8951065709634755e-06, "loss": 0.80130565, "num_input_tokens_seen": 37265595, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.94140625, "step": 1724, "time_per_iteration": 2.4069485664367676 }, { "auxiliary_loss_clip": 0.0113112, "auxiliary_loss_mlp": 0.0105769, "balance_loss_clip": 1.03269172, "balance_loss_mlp": 1.03416538, "epoch": 0.1037126108522471, "flos": 47551878587520.0, "grad_norm": 1.7015623873393253, "language_loss": 0.74837613, "learning_rate": 3.8949857859534884e-06, "loss": 0.77026427, "num_input_tokens_seen": 37286660, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.97265625, "step": 1725, "time_per_iteration": 2.601824998855591 }, { "auxiliary_loss_clip": 0.01125892, "auxiliary_loss_mlp": 0.01055733, "balance_loss_clip": 1.03295231, "balance_loss_mlp": 1.034518, "epoch": 0.10377273410491508, "flos": 22815670337280.0, "grad_norm": 1.9342342234349115, "language_loss": 0.74688578, "learning_rate": 3.894864933316333e-06, "loss": 0.76870197, "num_input_tokens_seen": 37304915, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.9140625, "step": 1726, "time_per_iteration": 2.403597116470337 }, { "auxiliary_loss_clip": 0.0112702, "auxiliary_loss_mlp": 0.01050749, "balance_loss_clip": 1.02503598, "balance_loss_mlp": 1.03321671, "epoch": 0.10383285735758305, "flos": 26137551335040.0, "grad_norm": 1.9402703290147678, "language_loss": 0.72989267, "learning_rate": 3.894744013056322e-06, "loss": 0.75167036, "num_input_tokens_seen": 37325265, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.9375, "step": 1727, "time_per_iteration": 2.416836738586426 }, { "auxiliary_loss_clip": 0.01125883, "auxiliary_loss_mlp": 0.01053341, "balance_loss_clip": 1.02731776, "balance_loss_mlp": 1.03287256, "epoch": 0.10389298061025101, "flos": 17090856904320.0, "grad_norm": 2.1202233462362714, "language_loss": 0.8460077, "learning_rate": 3.894623025177772e-06, "loss": 0.8678, "num_input_tokens_seen": 37341650, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.9296875, "step": 1728, "time_per_iteration": 2.358876943588257 }, { "auxiliary_loss_clip": 0.01125422, "auxiliary_loss_mlp": 0.01048718, "balance_loss_clip": 1.02230167, "balance_loss_mlp": 1.03374028, "epoch": 0.10395310386291898, "flos": 20775485594880.0, "grad_norm": 2.289753052034552, "language_loss": 0.70360857, "learning_rate": 3.894501969684999e-06, "loss": 0.7253499, "num_input_tokens_seen": 37360270, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.91796875, "step": 1729, "time_per_iteration": 2.4003336429595947 }, { "auxiliary_loss_clip": 0.01123786, "auxiliary_loss_mlp": 0.01048196, "balance_loss_clip": 1.02238703, "balance_loss_mlp": 1.032902, "epoch": 0.10401322711558696, "flos": 12819792240000.0, "grad_norm": 2.4446926699856104, "language_loss": 0.81571615, "learning_rate": 3.894380846582324e-06, "loss": 0.83743596, "num_input_tokens_seen": 37375225, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.91015625, "step": 1730, "time_per_iteration": 2.360623598098755 }, { "auxiliary_loss_clip": 0.01121029, "auxiliary_loss_mlp": 0.0103754, "balance_loss_clip": 1.01527131, "balance_loss_mlp": 1.03012872, "epoch": 0.10407335036825492, "flos": 23183584911360.0, "grad_norm": 1.7366924144958773, "language_loss": 0.75999582, "learning_rate": 3.89425965587407e-06, "loss": 0.78158152, "num_input_tokens_seen": 37395165, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.91015625, "step": 1731, "time_per_iteration": 2.3825974464416504 }, { "auxiliary_loss_clip": 0.01120133, "auxiliary_loss_mlp": 0.01043516, "balance_loss_clip": 1.02133155, "balance_loss_mlp": 1.03189421, "epoch": 0.10413347362092289, "flos": 26102987222400.0, "grad_norm": 2.4071442074963687, "language_loss": 0.82738227, "learning_rate": 3.894138397564562e-06, "loss": 0.84901875, "num_input_tokens_seen": 37414845, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.8828125, "step": 1732, "time_per_iteration": 2.423374891281128 }, { "auxiliary_loss_clip": 0.01122988, "auxiliary_loss_mlp": 0.01041163, "balance_loss_clip": 1.01872814, "balance_loss_mlp": 1.03322685, "epoch": 0.10419359687359087, "flos": 12640233784320.0, "grad_norm": 2.216770909856606, "language_loss": 0.83156657, "learning_rate": 3.894017071658125e-06, "loss": 0.85320818, "num_input_tokens_seen": 37432490, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.8984375, "step": 1733, "time_per_iteration": 2.3525197505950928 }, { "auxiliary_loss_clip": 0.01125276, "auxiliary_loss_mlp": 0.01040831, "balance_loss_clip": 1.01758552, "balance_loss_mlp": 1.03219247, "epoch": 0.10425372012625883, "flos": 12124427224320.0, "grad_norm": 2.1742975654993333, "language_loss": 0.76333314, "learning_rate": 3.893895678159092e-06, "loss": 0.78499418, "num_input_tokens_seen": 37449435, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.9296875, "step": 1734, "time_per_iteration": 2.3762547969818115 }, { "auxiliary_loss_clip": 0.01120024, "auxiliary_loss_mlp": 0.01038903, "balance_loss_clip": 1.01643229, "balance_loss_mlp": 1.03039026, "epoch": 0.1043138433789268, "flos": 25336399299840.0, "grad_norm": 1.7597136442224786, "language_loss": 0.75126266, "learning_rate": 3.8937742170717935e-06, "loss": 0.77285194, "num_input_tokens_seen": 37469105, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.8984375, "step": 1735, "time_per_iteration": 2.4201161861419678 }, { "auxiliary_loss_clip": 0.01123744, "auxiliary_loss_mlp": 0.01043558, "balance_loss_clip": 1.01808333, "balance_loss_mlp": 1.0322299, "epoch": 0.10437396663159478, "flos": 29165917599360.0, "grad_norm": 1.638933424974742, "language_loss": 0.7859149, "learning_rate": 3.893652688400565e-06, "loss": 0.80758798, "num_input_tokens_seen": 37490540, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9140625, "step": 1736, "time_per_iteration": 2.5261738300323486 }, { "auxiliary_loss_clip": 0.01122254, "auxiliary_loss_mlp": 0.01055381, "balance_loss_clip": 1.03001356, "balance_loss_mlp": 1.03328729, "epoch": 0.10443408988426274, "flos": 25079822651520.0, "grad_norm": 1.8328233590421816, "language_loss": 0.70703518, "learning_rate": 3.893531092149743e-06, "loss": 0.72881156, "num_input_tokens_seen": 37511905, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.890625, "step": 1737, "time_per_iteration": 2.435969352722168 }, { "auxiliary_loss_clip": 0.01126007, "auxiliary_loss_mlp": 0.01052773, "balance_loss_clip": 1.02487803, "balance_loss_mlp": 1.03007066, "epoch": 0.1044942131369307, "flos": 26758481598720.0, "grad_norm": 1.781210849877685, "language_loss": 0.81362653, "learning_rate": 3.893409428323666e-06, "loss": 0.83541435, "num_input_tokens_seen": 37533635, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 0.9609375, "step": 1738, "time_per_iteration": 2.429069757461548 }, { "auxiliary_loss_clip": 0.01124425, "auxiliary_loss_mlp": 0.01053833, "balance_loss_clip": 1.02822733, "balance_loss_mlp": 1.03121376, "epoch": 0.10455433638959867, "flos": 18441576650880.0, "grad_norm": 1.8175202610349077, "language_loss": 0.74855512, "learning_rate": 3.8932876969266785e-06, "loss": 0.7703377, "num_input_tokens_seen": 37552035, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.93359375, "step": 1739, "time_per_iteration": 2.4002676010131836 }, { "auxiliary_loss_clip": 0.01121714, "auxiliary_loss_mlp": 0.01050747, "balance_loss_clip": 1.02702391, "balance_loss_mlp": 1.03128552, "epoch": 0.10461445964226665, "flos": 23217939555840.0, "grad_norm": 3.3126482199985987, "language_loss": 0.77350897, "learning_rate": 3.893165897963123e-06, "loss": 0.79523361, "num_input_tokens_seen": 37571540, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.90625, "step": 1740, "time_per_iteration": 2.390667676925659 }, { "auxiliary_loss_clip": 0.01124322, "auxiliary_loss_mlp": 0.01046332, "balance_loss_clip": 1.02300251, "balance_loss_mlp": 1.03353405, "epoch": 0.10467458289493461, "flos": 24344307705600.0, "grad_norm": 2.0689127159114054, "language_loss": 0.8588016, "learning_rate": 3.893044031437346e-06, "loss": 0.88050812, "num_input_tokens_seen": 37588265, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.90625, "step": 1741, "time_per_iteration": 2.3992786407470703 }, { "auxiliary_loss_clip": 0.01124591, "auxiliary_loss_mlp": 0.0104551, "balance_loss_clip": 1.01955867, "balance_loss_mlp": 1.03237677, "epoch": 0.10473470614760258, "flos": 21286893323520.0, "grad_norm": 2.532080922859773, "language_loss": 0.75275385, "learning_rate": 3.892922097353697e-06, "loss": 0.77445483, "num_input_tokens_seen": 37606860, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.921875, "step": 1742, "time_per_iteration": 2.376932382583618 }, { "auxiliary_loss_clip": 0.01124536, "auxiliary_loss_mlp": 0.01048128, "balance_loss_clip": 1.02493, "balance_loss_mlp": 1.03433001, "epoch": 0.10479482940027056, "flos": 21686195076480.0, "grad_norm": 1.9615184924185378, "language_loss": 0.86979914, "learning_rate": 3.8928000957165275e-06, "loss": 0.89152575, "num_input_tokens_seen": 37625210, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.90234375, "step": 1743, "time_per_iteration": 2.388972520828247 }, { "auxiliary_loss_clip": 0.01121899, "auxiliary_loss_mlp": 0.01047988, "balance_loss_clip": 1.02152383, "balance_loss_mlp": 1.0318445, "epoch": 0.10485495265293852, "flos": 21572797380480.0, "grad_norm": 1.980346536726369, "language_loss": 0.75399542, "learning_rate": 3.8926780265301915e-06, "loss": 0.77569425, "num_input_tokens_seen": 37644110, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.90234375, "step": 1744, "time_per_iteration": 2.3914849758148193 }, { "auxiliary_loss_clip": 0.01124598, "auxiliary_loss_mlp": 0.01050524, "balance_loss_clip": 1.02765965, "balance_loss_mlp": 1.03332758, "epoch": 0.10491507590560649, "flos": 37960399301760.0, "grad_norm": 1.8428502923857146, "language_loss": 0.78735441, "learning_rate": 3.8925558897990445e-06, "loss": 0.80910563, "num_input_tokens_seen": 37665800, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.9140625, "step": 1745, "time_per_iteration": 2.5292773246765137 }, { "auxiliary_loss_clip": 0.01122432, "auxiliary_loss_mlp": 0.01057055, "balance_loss_clip": 1.03259289, "balance_loss_mlp": 1.0313642, "epoch": 0.10497519915827447, "flos": 26395070590080.0, "grad_norm": 2.7675134126447194, "language_loss": 0.82449567, "learning_rate": 3.892433685527447e-06, "loss": 0.84629059, "num_input_tokens_seen": 37685095, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.91015625, "step": 1746, "time_per_iteration": 2.4199109077453613 }, { "auxiliary_loss_clip": 0.0112421, "auxiliary_loss_mlp": 0.0105103, "balance_loss_clip": 1.02669919, "balance_loss_mlp": 1.03329909, "epoch": 0.10503532241094243, "flos": 40660581985920.0, "grad_norm": 1.6092919705029667, "language_loss": 0.69958377, "learning_rate": 3.892311413719759e-06, "loss": 0.72133613, "num_input_tokens_seen": 37707445, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.90625, "step": 1747, "time_per_iteration": 2.571589708328247 }, { "auxiliary_loss_clip": 0.01128556, "auxiliary_loss_mlp": 0.01052369, "balance_loss_clip": 1.02750218, "balance_loss_mlp": 1.03342628, "epoch": 0.1050954456636104, "flos": 29788104672000.0, "grad_norm": 2.2964050379853744, "language_loss": 0.84260982, "learning_rate": 3.892189074380345e-06, "loss": 0.86441904, "num_input_tokens_seen": 37728325, "router_z_loss_clip": 0.24902344, "router_z_loss_mlp": 0.953125, "step": 1748, "time_per_iteration": 2.4443163871765137 }, { "auxiliary_loss_clip": 0.01117316, "auxiliary_loss_mlp": 0.01042685, "balance_loss_clip": 1.01779461, "balance_loss_mlp": 1.02924657, "epoch": 0.10515556891627838, "flos": 23947694127360.0, "grad_norm": 2.0563322181054393, "language_loss": 0.71392345, "learning_rate": 3.892066667513569e-06, "loss": 0.73552346, "num_input_tokens_seen": 37748910, "router_z_loss_clip": 0.24902344, "router_z_loss_mlp": 0.8828125, "step": 1749, "time_per_iteration": 2.41198468208313 }, { "auxiliary_loss_clip": 0.01118943, "auxiliary_loss_mlp": 0.01046165, "balance_loss_clip": 1.02150106, "balance_loss_mlp": 1.03067636, "epoch": 0.10521569216894634, "flos": 18258631793280.0, "grad_norm": 2.09474354965328, "language_loss": 0.81900769, "learning_rate": 3.891944193123801e-06, "loss": 0.84065878, "num_input_tokens_seen": 37765745, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.8828125, "step": 1750, "time_per_iteration": 2.3500618934631348 }, { "auxiliary_loss_clip": 0.01127944, "auxiliary_loss_mlp": 0.01055871, "balance_loss_clip": 1.03163528, "balance_loss_mlp": 1.03505528, "epoch": 0.10527581542161431, "flos": 15630056040960.0, "grad_norm": 2.155919689446535, "language_loss": 0.92280161, "learning_rate": 3.891821651215411e-06, "loss": 0.9446398, "num_input_tokens_seen": 37780520, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.9296875, "step": 1751, "time_per_iteration": 2.38798189163208 }, { "auxiliary_loss_clip": 0.01121446, "auxiliary_loss_mlp": 0.01042405, "balance_loss_clip": 1.01945722, "balance_loss_mlp": 1.03214347, "epoch": 0.10533593867428227, "flos": 18295569878400.0, "grad_norm": 3.1524830188228834, "language_loss": 0.78899848, "learning_rate": 3.8916990417927735e-06, "loss": 0.810637, "num_input_tokens_seen": 37799515, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.89453125, "step": 1752, "time_per_iteration": 2.3802599906921387 }, { "auxiliary_loss_clip": 0.01121154, "auxiliary_loss_mlp": 0.01045128, "balance_loss_clip": 1.0206542, "balance_loss_mlp": 1.03210664, "epoch": 0.10539606192695025, "flos": 29021935685760.0, "grad_norm": 1.8613766788519057, "language_loss": 0.75671118, "learning_rate": 3.891576364860262e-06, "loss": 0.77837402, "num_input_tokens_seen": 37818695, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.890625, "step": 1753, "time_per_iteration": 3.81923508644104 }, { "auxiliary_loss_clip": 0.01124279, "auxiliary_loss_mlp": 0.01053431, "balance_loss_clip": 1.02843332, "balance_loss_mlp": 1.03089023, "epoch": 0.10545618517961822, "flos": 19968433539840.0, "grad_norm": 1.8995117140353865, "language_loss": 0.83522022, "learning_rate": 3.891453620422258e-06, "loss": 0.85699737, "num_input_tokens_seen": 37837860, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.93359375, "step": 1754, "time_per_iteration": 3.7905337810516357 }, { "auxiliary_loss_clip": 0.01128729, "auxiliary_loss_mlp": 0.01050098, "balance_loss_clip": 1.0233947, "balance_loss_mlp": 1.03546464, "epoch": 0.10551630843228618, "flos": 16142511110400.0, "grad_norm": 2.7992506175283154, "language_loss": 0.6898886, "learning_rate": 3.891330808483137e-06, "loss": 0.71167684, "num_input_tokens_seen": 37856260, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.93359375, "step": 1755, "time_per_iteration": 2.345918655395508 }, { "auxiliary_loss_clip": 0.01127338, "auxiliary_loss_mlp": 0.01054693, "balance_loss_clip": 1.0284667, "balance_loss_mlp": 1.03414655, "epoch": 0.10557643168495416, "flos": 23439009484800.0, "grad_norm": 2.095319590645789, "language_loss": 0.76325703, "learning_rate": 3.891207929047286e-06, "loss": 0.78507727, "num_input_tokens_seen": 37876960, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9296875, "step": 1756, "time_per_iteration": 2.4101569652557373 }, { "auxiliary_loss_clip": 0.01124114, "auxiliary_loss_mlp": 0.0104714, "balance_loss_clip": 1.02310705, "balance_loss_mlp": 1.03179097, "epoch": 0.10563655493762213, "flos": 21797951938560.0, "grad_norm": 1.7351198219289925, "language_loss": 0.79872441, "learning_rate": 3.8910849821190884e-06, "loss": 0.82043695, "num_input_tokens_seen": 37897070, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.921875, "step": 1757, "time_per_iteration": 3.8162782192230225 }, { "auxiliary_loss_clip": 0.0112361, "auxiliary_loss_mlp": 0.01044144, "balance_loss_clip": 1.01888347, "balance_loss_mlp": 1.03206515, "epoch": 0.10569667819029009, "flos": 53798782625280.0, "grad_norm": 1.5596724518209486, "language_loss": 0.78979349, "learning_rate": 3.890961967702933e-06, "loss": 0.81147099, "num_input_tokens_seen": 37923635, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.9140625, "step": 1758, "time_per_iteration": 2.692870616912842 }, { "auxiliary_loss_clip": 0.01127386, "auxiliary_loss_mlp": 0.01040808, "balance_loss_clip": 1.01633477, "balance_loss_mlp": 1.03447723, "epoch": 0.10575680144295807, "flos": 22924529556480.0, "grad_norm": 1.7044128071529396, "language_loss": 0.91619074, "learning_rate": 3.890838885803208e-06, "loss": 0.93787271, "num_input_tokens_seen": 37942650, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.9296875, "step": 1759, "time_per_iteration": 2.395379066467285 }, { "auxiliary_loss_clip": 0.01126309, "auxiliary_loss_mlp": 0.01055059, "balance_loss_clip": 1.02841604, "balance_loss_mlp": 1.0312767, "epoch": 0.10581692469562604, "flos": 14135808228480.0, "grad_norm": 1.9812907161966353, "language_loss": 0.77218324, "learning_rate": 3.890715736424307e-06, "loss": 0.79399687, "num_input_tokens_seen": 37960660, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.953125, "step": 1760, "time_per_iteration": 2.3628458976745605 }, { "auxiliary_loss_clip": 0.01124844, "auxiliary_loss_mlp": 0.01051971, "balance_loss_clip": 1.0255419, "balance_loss_mlp": 1.03168297, "epoch": 0.105877047948294, "flos": 25957469208960.0, "grad_norm": 3.0718633757371125, "language_loss": 0.8935079, "learning_rate": 3.890592519570626e-06, "loss": 0.91527599, "num_input_tokens_seen": 37978625, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.9296875, "step": 1761, "time_per_iteration": 2.4184231758117676 }, { "auxiliary_loss_clip": 0.01125491, "auxiliary_loss_mlp": 0.01053337, "balance_loss_clip": 1.02795768, "balance_loss_mlp": 1.03289485, "epoch": 0.10593717120096197, "flos": 30663447079680.0, "grad_norm": 2.260293059938474, "language_loss": 0.7777102, "learning_rate": 3.89046923524656e-06, "loss": 0.79949844, "num_input_tokens_seen": 38000005, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.92578125, "step": 1762, "time_per_iteration": 2.473712205886841 }, { "auxiliary_loss_clip": 0.01029718, "auxiliary_loss_mlp": 0.01002481, "balance_loss_clip": 0.99895269, "balance_loss_mlp": 1.00415659, "epoch": 0.10599729445362994, "flos": 66432905055360.0, "grad_norm": 0.7567652517232661, "language_loss": 0.60488772, "learning_rate": 3.8903458834565105e-06, "loss": 0.62520969, "num_input_tokens_seen": 38066165, "router_z_loss_clip": 0.03540039, "router_z_loss_mlp": 0.25585938, "step": 1763, "time_per_iteration": 3.18863844871521 }, { "auxiliary_loss_clip": 0.0112237, "auxiliary_loss_mlp": 0.01042503, "balance_loss_clip": 1.01810062, "balance_loss_mlp": 1.03152835, "epoch": 0.10605741770629791, "flos": 23947135545600.0, "grad_norm": 1.7391662287955905, "language_loss": 0.79645944, "learning_rate": 3.890222464204879e-06, "loss": 0.8181082, "num_input_tokens_seen": 38086150, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.90625, "step": 1764, "time_per_iteration": 2.401545524597168 }, { "auxiliary_loss_clip": 0.01123566, "auxiliary_loss_mlp": 0.01048595, "balance_loss_clip": 1.02396607, "balance_loss_mlp": 1.03268123, "epoch": 0.10611754095896588, "flos": 19386605865600.0, "grad_norm": 1.916558263318626, "language_loss": 0.80186951, "learning_rate": 3.89009897749607e-06, "loss": 0.82359111, "num_input_tokens_seen": 38104205, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.91015625, "step": 1765, "time_per_iteration": 2.383697748184204 }, { "auxiliary_loss_clip": 0.01120346, "auxiliary_loss_mlp": 0.01050923, "balance_loss_clip": 1.02638936, "balance_loss_mlp": 1.03064609, "epoch": 0.10617766421163385, "flos": 22236635571840.0, "grad_norm": 1.840882934516297, "language_loss": 0.76780617, "learning_rate": 3.88997542333449e-06, "loss": 0.78951889, "num_input_tokens_seen": 38122005, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.8984375, "step": 1766, "time_per_iteration": 2.376946210861206 }, { "auxiliary_loss_clip": 0.01125036, "auxiliary_loss_mlp": 0.01054262, "balance_loss_clip": 1.02844131, "balance_loss_mlp": 1.03260565, "epoch": 0.10623778746430182, "flos": 28403100103680.0, "grad_norm": 1.5994413753068162, "language_loss": 0.77417314, "learning_rate": 3.889851801724549e-06, "loss": 0.79596615, "num_input_tokens_seen": 38143365, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.92578125, "step": 1767, "time_per_iteration": 2.4704442024230957 }, { "auxiliary_loss_clip": 0.01028991, "auxiliary_loss_mlp": 0.0100806, "balance_loss_clip": 1.00436473, "balance_loss_mlp": 1.00339031, "epoch": 0.10629791071696978, "flos": 64231282719360.0, "grad_norm": 0.6754453633366562, "language_loss": 0.57893264, "learning_rate": 3.889728112670658e-06, "loss": 0.59930313, "num_input_tokens_seen": 38210035, "router_z_loss_clip": 0.03686523, "router_z_loss_mlp": 0.25585938, "step": 1768, "time_per_iteration": 3.1264593601226807 }, { "auxiliary_loss_clip": 0.01125592, "auxiliary_loss_mlp": 0.01039103, "balance_loss_clip": 1.01608372, "balance_loss_mlp": 1.03276598, "epoch": 0.10635803396963776, "flos": 22746472289280.0, "grad_norm": 1.3944339810139335, "language_loss": 0.86446828, "learning_rate": 3.8896043561772325e-06, "loss": 0.88611525, "num_input_tokens_seen": 38231230, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.9296875, "step": 1769, "time_per_iteration": 2.437068462371826 }, { "auxiliary_loss_clip": 0.0112854, "auxiliary_loss_mlp": 0.01050713, "balance_loss_clip": 1.02455854, "balance_loss_mlp": 1.0353334, "epoch": 0.10641815722230573, "flos": 31394214080640.0, "grad_norm": 2.81559546028732, "language_loss": 0.61949551, "learning_rate": 3.889480532248688e-06, "loss": 0.64128804, "num_input_tokens_seen": 38253890, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.9296875, "step": 1770, "time_per_iteration": 2.472952365875244 }, { "auxiliary_loss_clip": 0.01029669, "auxiliary_loss_mlp": 0.01003379, "balance_loss_clip": 0.99985087, "balance_loss_mlp": 1.00370085, "epoch": 0.1064782804749737, "flos": 58550077307520.0, "grad_norm": 1.1358638098678222, "language_loss": 0.57037234, "learning_rate": 3.889356640889444e-06, "loss": 0.59070289, "num_input_tokens_seen": 38304290, "router_z_loss_clip": 0.03540039, "router_z_loss_mlp": 0.25976562, "step": 1771, "time_per_iteration": 2.942660093307495 }, { "auxiliary_loss_clip": 0.01125322, "auxiliary_loss_mlp": 0.01058676, "balance_loss_clip": 1.0330584, "balance_loss_mlp": 1.03383589, "epoch": 0.10653840372764166, "flos": 23986691982720.0, "grad_norm": 1.6196900827448717, "language_loss": 0.88175607, "learning_rate": 3.8892326821039205e-06, "loss": 0.9035961, "num_input_tokens_seen": 38324725, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.9140625, "step": 1772, "time_per_iteration": 2.4082319736480713 }, { "auxiliary_loss_clip": 0.0112954, "auxiliary_loss_mlp": 0.01046109, "balance_loss_clip": 1.01948977, "balance_loss_mlp": 1.03332782, "epoch": 0.10659852698030964, "flos": 18293719576320.0, "grad_norm": 3.5867802309173427, "language_loss": 0.7572273, "learning_rate": 3.889108655896542e-06, "loss": 0.77898383, "num_input_tokens_seen": 38340735, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.96484375, "step": 1773, "time_per_iteration": 2.360600709915161 }, { "auxiliary_loss_clip": 0.01127654, "auxiliary_loss_mlp": 0.01051195, "balance_loss_clip": 1.02645874, "balance_loss_mlp": 1.03547025, "epoch": 0.1066586502329776, "flos": 32159230992000.0, "grad_norm": 1.8622521371175176, "language_loss": 0.82763404, "learning_rate": 3.888984562271736e-06, "loss": 0.84942257, "num_input_tokens_seen": 38361315, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.921875, "step": 1774, "time_per_iteration": 2.489536762237549 }, { "auxiliary_loss_clip": 0.01129712, "auxiliary_loss_mlp": 0.01051944, "balance_loss_clip": 1.02571797, "balance_loss_mlp": 1.03411245, "epoch": 0.10671877348564557, "flos": 17784197061120.0, "grad_norm": 2.2850490467324875, "language_loss": 0.76627076, "learning_rate": 3.888860401233929e-06, "loss": 0.78808731, "num_input_tokens_seen": 38377425, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.95703125, "step": 1775, "time_per_iteration": 2.3686110973358154 }, { "auxiliary_loss_clip": 0.01029928, "auxiliary_loss_mlp": 0.01003173, "balance_loss_clip": 0.99997795, "balance_loss_mlp": 1.00443852, "epoch": 0.10677889673831355, "flos": 63506695029120.0, "grad_norm": 0.8170942420256979, "language_loss": 0.57425374, "learning_rate": 3.8887361727875535e-06, "loss": 0.5945847, "num_input_tokens_seen": 38440275, "router_z_loss_clip": 0.03198242, "router_z_loss_mlp": 0.25390625, "step": 1776, "time_per_iteration": 3.05979585647583 }, { "auxiliary_loss_clip": 0.01126727, "auxiliary_loss_mlp": 0.010466, "balance_loss_clip": 1.02071953, "balance_loss_mlp": 1.03543973, "epoch": 0.10683901999098151, "flos": 22016612983680.0, "grad_norm": 1.5249944085465545, "language_loss": 0.8304826, "learning_rate": 3.888611876937043e-06, "loss": 0.85221589, "num_input_tokens_seen": 38461820, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.9140625, "step": 1777, "time_per_iteration": 2.4524457454681396 }, { "auxiliary_loss_clip": 0.01125564, "auxiliary_loss_mlp": 0.01047993, "balance_loss_clip": 1.02347136, "balance_loss_mlp": 1.03583741, "epoch": 0.10689914324364948, "flos": 25041872136960.0, "grad_norm": 3.6853021162900017, "language_loss": 0.87512541, "learning_rate": 3.888487513686832e-06, "loss": 0.89686102, "num_input_tokens_seen": 38482235, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.8984375, "step": 1778, "time_per_iteration": 2.4289801120758057 }, { "auxiliary_loss_clip": 0.01129984, "auxiliary_loss_mlp": 0.01050385, "balance_loss_clip": 1.024683, "balance_loss_mlp": 1.0366596, "epoch": 0.10695926649631746, "flos": 16434210453120.0, "grad_norm": 1.9132762909614143, "language_loss": 0.84370452, "learning_rate": 3.88836308304136e-06, "loss": 0.8655082, "num_input_tokens_seen": 38500690, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.93359375, "step": 1779, "time_per_iteration": 2.385535955429077 }, { "auxiliary_loss_clip": 0.01121678, "auxiliary_loss_mlp": 0.01045359, "balance_loss_clip": 1.02161276, "balance_loss_mlp": 1.03248048, "epoch": 0.10701938974898542, "flos": 16978366903680.0, "grad_norm": 1.940612983001018, "language_loss": 0.67382878, "learning_rate": 3.888238585005066e-06, "loss": 0.69549918, "num_input_tokens_seen": 38518405, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.890625, "step": 1780, "time_per_iteration": 2.3779759407043457 }, { "auxiliary_loss_clip": 0.01124395, "auxiliary_loss_mlp": 0.01048141, "balance_loss_clip": 1.02261877, "balance_loss_mlp": 1.03375697, "epoch": 0.10707951300165339, "flos": 15887191271040.0, "grad_norm": 2.120473397927048, "language_loss": 0.91888499, "learning_rate": 3.888114019582395e-06, "loss": 0.94061041, "num_input_tokens_seen": 38535060, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.90625, "step": 1781, "time_per_iteration": 2.366016387939453 }, { "auxiliary_loss_clip": 0.01125598, "auxiliary_loss_mlp": 0.01046815, "balance_loss_clip": 1.02105403, "balance_loss_mlp": 1.03395641, "epoch": 0.10713963625432135, "flos": 14246273370240.0, "grad_norm": 1.961229098536527, "language_loss": 0.79416013, "learning_rate": 3.887989386777791e-06, "loss": 0.81588423, "num_input_tokens_seen": 38552855, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.91796875, "step": 1782, "time_per_iteration": 2.359746217727661 }, { "auxiliary_loss_clip": 0.01126018, "auxiliary_loss_mlp": 0.01047457, "balance_loss_clip": 1.02292371, "balance_loss_mlp": 1.03313398, "epoch": 0.10719975950698933, "flos": 16756040165760.0, "grad_norm": 2.134353445152127, "language_loss": 0.78729677, "learning_rate": 3.887864686595703e-06, "loss": 0.80903149, "num_input_tokens_seen": 38570075, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.9296875, "step": 1783, "time_per_iteration": 2.3823351860046387 }, { "auxiliary_loss_clip": 0.01127351, "auxiliary_loss_mlp": 0.01051587, "balance_loss_clip": 1.02654111, "balance_loss_mlp": 1.03401184, "epoch": 0.1072598827596573, "flos": 22709534204160.0, "grad_norm": 1.9319803999437355, "language_loss": 0.86656928, "learning_rate": 3.887739919040579e-06, "loss": 0.88835871, "num_input_tokens_seen": 38587970, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.93359375, "step": 1784, "time_per_iteration": 2.3890769481658936 }, { "auxiliary_loss_clip": 0.01128957, "auxiliary_loss_mlp": 0.01048849, "balance_loss_clip": 1.02149022, "balance_loss_mlp": 1.03402746, "epoch": 0.10732000601232526, "flos": 23257146879360.0, "grad_norm": 2.588319712177381, "language_loss": 1.0069952, "learning_rate": 3.887615084116874e-06, "loss": 1.02877331, "num_input_tokens_seen": 38605840, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.94921875, "step": 1785, "time_per_iteration": 2.4029221534729004 }, { "auxiliary_loss_clip": 0.01123074, "auxiliary_loss_mlp": 0.01048438, "balance_loss_clip": 1.02413082, "balance_loss_mlp": 1.0350312, "epoch": 0.10738012926499324, "flos": 24205911609600.0, "grad_norm": 1.3298221024401562, "language_loss": 0.84858882, "learning_rate": 3.887490181829042e-06, "loss": 0.87030399, "num_input_tokens_seen": 38627070, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.8828125, "step": 1786, "time_per_iteration": 2.4200692176818848 }, { "auxiliary_loss_clip": 0.0112323, "auxiliary_loss_mlp": 0.01046963, "balance_loss_clip": 1.01944935, "balance_loss_mlp": 1.03104043, "epoch": 0.1074402525176612, "flos": 20922016037760.0, "grad_norm": 1.716800751345172, "language_loss": 0.78385222, "learning_rate": 3.887365212181542e-06, "loss": 0.80555415, "num_input_tokens_seen": 38645840, "router_z_loss_clip": 0.27539062, "router_z_loss_mlp": 0.921875, "step": 1787, "time_per_iteration": 2.38824462890625 }, { "auxiliary_loss_clip": 0.01128656, "auxiliary_loss_mlp": 0.01050491, "balance_loss_clip": 1.02328789, "balance_loss_mlp": 1.03389168, "epoch": 0.10750037577032917, "flos": 16945967295360.0, "grad_norm": 1.7573360887890106, "language_loss": 0.82472336, "learning_rate": 3.88724017517883e-06, "loss": 0.84651482, "num_input_tokens_seen": 38664770, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 0.9453125, "step": 1788, "time_per_iteration": 2.3621251583099365 }, { "auxiliary_loss_clip": 0.01123994, "auxiliary_loss_mlp": 0.01050739, "balance_loss_clip": 1.02641988, "balance_loss_mlp": 1.03294051, "epoch": 0.10756049902299715, "flos": 20265509232000.0, "grad_norm": 1.838983977324372, "language_loss": 0.78195995, "learning_rate": 3.887115070825373e-06, "loss": 0.8037073, "num_input_tokens_seen": 38683865, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.91015625, "step": 1789, "time_per_iteration": 2.3786821365356445 }, { "auxiliary_loss_clip": 0.01126292, "auxiliary_loss_mlp": 0.01060179, "balance_loss_clip": 1.03047156, "balance_loss_mlp": 1.0334549, "epoch": 0.10762062227566511, "flos": 23585400282240.0, "grad_norm": 2.6683776947526225, "language_loss": 0.745278, "learning_rate": 3.886989899125632e-06, "loss": 0.76714271, "num_input_tokens_seen": 38702485, "router_z_loss_clip": 0.296875, "router_z_loss_mlp": 0.9296875, "step": 1790, "time_per_iteration": 2.4150373935699463 }, { "auxiliary_loss_clip": 0.01127052, "auxiliary_loss_mlp": 0.01061173, "balance_loss_clip": 1.03416061, "balance_loss_mlp": 1.03454113, "epoch": 0.10768074552833308, "flos": 24309638858880.0, "grad_norm": 2.0808194179309796, "language_loss": 0.78436476, "learning_rate": 3.886864660084075e-06, "loss": 0.806247, "num_input_tokens_seen": 38722475, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.92578125, "step": 1791, "time_per_iteration": 2.408830404281616 }, { "auxiliary_loss_clip": 0.01119221, "auxiliary_loss_mlp": 0.0104741, "balance_loss_clip": 1.02353215, "balance_loss_mlp": 1.03260255, "epoch": 0.10774086878100106, "flos": 25298832810240.0, "grad_norm": 1.8902431432210107, "language_loss": 0.70625722, "learning_rate": 3.886739353705173e-06, "loss": 0.72792351, "num_input_tokens_seen": 38743285, "router_z_loss_clip": 0.23925781, "router_z_loss_mlp": 0.8671875, "step": 1792, "time_per_iteration": 3.9579362869262695 }, { "auxiliary_loss_clip": 0.01125953, "auxiliary_loss_mlp": 0.01043045, "balance_loss_clip": 1.0159843, "balance_loss_mlp": 1.03193712, "epoch": 0.10780099203366902, "flos": 22052957575680.0, "grad_norm": 1.8560698854611348, "language_loss": 0.75875032, "learning_rate": 3.886613979993396e-06, "loss": 0.78044033, "num_input_tokens_seen": 38763035, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.9375, "step": 1793, "time_per_iteration": 2.3957271575927734 }, { "auxiliary_loss_clip": 0.01124997, "auxiliary_loss_mlp": 0.01057005, "balance_loss_clip": 1.0321852, "balance_loss_mlp": 1.03507137, "epoch": 0.10786111528633699, "flos": 22746367555200.0, "grad_norm": 1.5432225075588661, "language_loss": 0.85082167, "learning_rate": 3.886488538953219e-06, "loss": 0.87264168, "num_input_tokens_seen": 38784900, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.8984375, "step": 1794, "time_per_iteration": 3.90265154838562 }, { "auxiliary_loss_clip": 0.01129109, "auxiliary_loss_mlp": 0.01044994, "balance_loss_clip": 1.01956701, "balance_loss_mlp": 1.0332222, "epoch": 0.10792123853900495, "flos": 20849990169600.0, "grad_norm": 1.9717999863669853, "language_loss": 0.7450695, "learning_rate": 3.8863630305891196e-06, "loss": 0.76681054, "num_input_tokens_seen": 38804695, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9609375, "step": 1795, "time_per_iteration": 2.3823306560516357 }, { "auxiliary_loss_clip": 0.01127177, "auxiliary_loss_mlp": 0.01049021, "balance_loss_clip": 1.02110219, "balance_loss_mlp": 1.03276324, "epoch": 0.10798136179167293, "flos": 17747747735040.0, "grad_norm": 2.4639211054087333, "language_loss": 0.81476229, "learning_rate": 3.8862374549055755e-06, "loss": 0.83652425, "num_input_tokens_seen": 38822395, "router_z_loss_clip": 0.27929688, "router_z_loss_mlp": 0.9453125, "step": 1796, "time_per_iteration": 3.7188520431518555 }, { "auxiliary_loss_clip": 0.01128867, "auxiliary_loss_mlp": 0.01058443, "balance_loss_clip": 1.03150129, "balance_loss_mlp": 1.03387094, "epoch": 0.1080414850443409, "flos": 13588789046400.0, "grad_norm": 2.2364337055104055, "language_loss": 0.73790944, "learning_rate": 3.886111811907069e-06, "loss": 0.75978255, "num_input_tokens_seen": 38839865, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.94921875, "step": 1797, "time_per_iteration": 3.7533183097839355 }, { "auxiliary_loss_clip": 0.01125669, "auxiliary_loss_mlp": 0.01046034, "balance_loss_clip": 1.02181053, "balance_loss_mlp": 1.03294826, "epoch": 0.10810160829700886, "flos": 16252487493120.0, "grad_norm": 2.129713269732791, "language_loss": 0.81432426, "learning_rate": 3.885986101598082e-06, "loss": 0.83604133, "num_input_tokens_seen": 38857300, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.9296875, "step": 1798, "time_per_iteration": 2.378171443939209 }, { "auxiliary_loss_clip": 0.01125419, "auxiliary_loss_mlp": 0.01047582, "balance_loss_clip": 1.02235758, "balance_loss_mlp": 1.0329802, "epoch": 0.10816173154967684, "flos": 15157122497280.0, "grad_norm": 2.29817641696348, "language_loss": 0.85187292, "learning_rate": 3.885860323983104e-06, "loss": 0.87360299, "num_input_tokens_seen": 38874960, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.92578125, "step": 1799, "time_per_iteration": 2.3468503952026367 }, { "auxiliary_loss_clip": 0.01122342, "auxiliary_loss_mlp": 0.01054019, "balance_loss_clip": 1.02958083, "balance_loss_mlp": 1.03431249, "epoch": 0.10822185480234481, "flos": 17784371617920.0, "grad_norm": 1.871905884018768, "language_loss": 0.76835096, "learning_rate": 3.885734479066622e-06, "loss": 0.79011458, "num_input_tokens_seen": 38893610, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.8828125, "step": 1800, "time_per_iteration": 2.383742094039917 }, { "auxiliary_loss_clip": 0.01119787, "auxiliary_loss_mlp": 0.01041734, "balance_loss_clip": 1.0178442, "balance_loss_mlp": 1.03130066, "epoch": 0.10828197805501277, "flos": 25555479281280.0, "grad_norm": 1.5186469739563766, "language_loss": 0.7293545, "learning_rate": 3.885608566853126e-06, "loss": 0.75096971, "num_input_tokens_seen": 38913485, "router_z_loss_clip": 0.23925781, "router_z_loss_mlp": 0.88671875, "step": 1801, "time_per_iteration": 2.4169530868530273 }, { "auxiliary_loss_clip": 0.01129902, "auxiliary_loss_mlp": 0.01046469, "balance_loss_clip": 1.02205503, "balance_loss_mlp": 1.03348994, "epoch": 0.10834210130768075, "flos": 28983217121280.0, "grad_norm": 1.8508048354535698, "language_loss": 0.65805316, "learning_rate": 3.8854825873471115e-06, "loss": 0.67981684, "num_input_tokens_seen": 38935650, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.9609375, "step": 1802, "time_per_iteration": 2.4483463764190674 }, { "auxiliary_loss_clip": 0.01123685, "auxiliary_loss_mlp": 0.01052333, "balance_loss_clip": 1.02596378, "balance_loss_mlp": 1.03211331, "epoch": 0.10840222456034872, "flos": 20263239993600.0, "grad_norm": 3.1106185134885775, "language_loss": 0.81412292, "learning_rate": 3.885356540553073e-06, "loss": 0.83588308, "num_input_tokens_seen": 38954130, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.9140625, "step": 1803, "time_per_iteration": 2.37387752532959 }, { "auxiliary_loss_clip": 0.01120541, "auxiliary_loss_mlp": 0.01048911, "balance_loss_clip": 1.02432966, "balance_loss_mlp": 1.03114367, "epoch": 0.10846234781301668, "flos": 19862087938560.0, "grad_norm": 1.572617341333409, "language_loss": 0.91127855, "learning_rate": 3.88523042647551e-06, "loss": 0.93297303, "num_input_tokens_seen": 38972905, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.89453125, "step": 1804, "time_per_iteration": 2.3934566974639893 }, { "auxiliary_loss_clip": 0.01123549, "auxiliary_loss_mlp": 0.0104879, "balance_loss_clip": 1.02301693, "balance_loss_mlp": 1.03132033, "epoch": 0.10852247106568465, "flos": 26467829596800.0, "grad_norm": 2.1018894025612136, "language_loss": 0.76497591, "learning_rate": 3.885104245118921e-06, "loss": 0.78669924, "num_input_tokens_seen": 38993255, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.921875, "step": 1805, "time_per_iteration": 2.423750638961792 }, { "auxiliary_loss_clip": 0.01120164, "auxiliary_loss_mlp": 0.01040732, "balance_loss_clip": 1.01696146, "balance_loss_mlp": 1.03138614, "epoch": 0.10858259431835263, "flos": 30080188039680.0, "grad_norm": 1.9556052539861366, "language_loss": 0.86247486, "learning_rate": 3.8849779964878125e-06, "loss": 0.88408375, "num_input_tokens_seen": 39012610, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.890625, "step": 1806, "time_per_iteration": 2.451107978820801 }, { "auxiliary_loss_clip": 0.01122921, "auxiliary_loss_mlp": 0.01049866, "balance_loss_clip": 1.02549911, "balance_loss_mlp": 1.03105104, "epoch": 0.10864271757102059, "flos": 19062157800960.0, "grad_norm": 3.0931004384736887, "language_loss": 0.81229842, "learning_rate": 3.884851680586687e-06, "loss": 0.83402628, "num_input_tokens_seen": 39030120, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.91796875, "step": 1807, "time_per_iteration": 2.3664300441741943 }, { "auxiliary_loss_clip": 0.01120947, "auxiliary_loss_mlp": 0.01043449, "balance_loss_clip": 1.020895, "balance_loss_mlp": 1.03195763, "epoch": 0.10870284082368856, "flos": 24713967847680.0, "grad_norm": 1.8282252096160712, "language_loss": 0.7888785, "learning_rate": 3.884725297420053e-06, "loss": 0.81052244, "num_input_tokens_seen": 39049875, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.890625, "step": 1808, "time_per_iteration": 2.4256677627563477 }, { "auxiliary_loss_clip": 0.01125638, "auxiliary_loss_mlp": 0.01046616, "balance_loss_clip": 1.02194023, "balance_loss_mlp": 1.03399539, "epoch": 0.10876296407635654, "flos": 20626720824960.0, "grad_norm": 1.7326907662304973, "language_loss": 0.79196876, "learning_rate": 3.884598846992422e-06, "loss": 0.81369132, "num_input_tokens_seen": 39068935, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.9140625, "step": 1809, "time_per_iteration": 2.39125394821167 }, { "auxiliary_loss_clip": 0.01121593, "auxiliary_loss_mlp": 0.01045871, "balance_loss_clip": 1.0217669, "balance_loss_mlp": 1.03222513, "epoch": 0.1088230873290245, "flos": 21578767223040.0, "grad_norm": 1.9900982360126007, "language_loss": 0.84929574, "learning_rate": 3.884472329308306e-06, "loss": 0.87097037, "num_input_tokens_seen": 39087370, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.89453125, "step": 1810, "time_per_iteration": 2.3882687091827393 }, { "auxiliary_loss_clip": 0.01127982, "auxiliary_loss_mlp": 0.01052776, "balance_loss_clip": 1.02653766, "balance_loss_mlp": 1.03371501, "epoch": 0.10888321058169247, "flos": 26467829596800.0, "grad_norm": 2.0813929656714656, "language_loss": 0.63652569, "learning_rate": 3.8843457443722195e-06, "loss": 0.6583333, "num_input_tokens_seen": 39106635, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.94140625, "step": 1811, "time_per_iteration": 2.4178736209869385 }, { "auxiliary_loss_clip": 0.01122244, "auxiliary_loss_mlp": 0.01045659, "balance_loss_clip": 1.02147138, "balance_loss_mlp": 1.03190422, "epoch": 0.10894333383436045, "flos": 25847423003520.0, "grad_norm": 2.200120322991929, "language_loss": 0.74163443, "learning_rate": 3.884219092188681e-06, "loss": 0.76331341, "num_input_tokens_seen": 39126335, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.90625, "step": 1812, "time_per_iteration": 2.4389612674713135 }, { "auxiliary_loss_clip": 0.01122401, "auxiliary_loss_mlp": 0.0104553, "balance_loss_clip": 1.02191472, "balance_loss_mlp": 1.03145349, "epoch": 0.10900345708702841, "flos": 19536068862720.0, "grad_norm": 1.67541571038205, "language_loss": 0.72409236, "learning_rate": 3.884092372762209e-06, "loss": 0.74577165, "num_input_tokens_seen": 39144820, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.91015625, "step": 1813, "time_per_iteration": 2.4204447269439697 }, { "auxiliary_loss_clip": 0.01120348, "auxiliary_loss_mlp": 0.01047142, "balance_loss_clip": 1.02408659, "balance_loss_mlp": 1.03367043, "epoch": 0.10906358033969638, "flos": 23622163810560.0, "grad_norm": 1.8415215996899577, "language_loss": 0.82487369, "learning_rate": 3.883965586097327e-06, "loss": 0.84654868, "num_input_tokens_seen": 39165945, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8671875, "step": 1814, "time_per_iteration": 2.419451951980591 }, { "auxiliary_loss_clip": 0.01123526, "auxiliary_loss_mlp": 0.0104773, "balance_loss_clip": 1.02416265, "balance_loss_mlp": 1.03223526, "epoch": 0.10912370359236434, "flos": 21213680469120.0, "grad_norm": 4.420405023215869, "language_loss": 0.84061807, "learning_rate": 3.88383873219856e-06, "loss": 0.86233068, "num_input_tokens_seen": 39183520, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.9140625, "step": 1815, "time_per_iteration": 2.373497247695923 }, { "auxiliary_loss_clip": 0.0112164, "auxiliary_loss_mlp": 0.01044655, "balance_loss_clip": 1.01982379, "balance_loss_mlp": 1.03311396, "epoch": 0.10918382684503232, "flos": 13552339720320.0, "grad_norm": 4.895839077195346, "language_loss": 0.71816218, "learning_rate": 3.8837118110704345e-06, "loss": 0.73982519, "num_input_tokens_seen": 39201190, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.88671875, "step": 1816, "time_per_iteration": 2.3697516918182373 }, { "auxiliary_loss_clip": 0.01126296, "auxiliary_loss_mlp": 0.01053606, "balance_loss_clip": 1.02730894, "balance_loss_mlp": 1.03459978, "epoch": 0.10924395009770028, "flos": 27963089838720.0, "grad_norm": 2.297323944015786, "language_loss": 0.72977591, "learning_rate": 3.88358482271748e-06, "loss": 0.75157493, "num_input_tokens_seen": 39221210, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.91796875, "step": 1817, "time_per_iteration": 2.427844524383545 }, { "auxiliary_loss_clip": 0.01123589, "auxiliary_loss_mlp": 0.01046912, "balance_loss_clip": 1.02059031, "balance_loss_mlp": 1.0316056, "epoch": 0.10930407335036825, "flos": 25592557011840.0, "grad_norm": 1.665588613083356, "language_loss": 0.67563391, "learning_rate": 3.883457767144228e-06, "loss": 0.69733888, "num_input_tokens_seen": 39242025, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.921875, "step": 1818, "time_per_iteration": 2.426520586013794 }, { "auxiliary_loss_clip": 0.01123798, "auxiliary_loss_mlp": 0.01051526, "balance_loss_clip": 1.02620637, "balance_loss_mlp": 1.03213441, "epoch": 0.10936419660303623, "flos": 18405197147520.0, "grad_norm": 2.4183236947991564, "language_loss": 0.73805034, "learning_rate": 3.883330644355212e-06, "loss": 0.75980365, "num_input_tokens_seen": 39259870, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.9140625, "step": 1819, "time_per_iteration": 2.3530497550964355 }, { "auxiliary_loss_clip": 0.01124492, "auxiliary_loss_mlp": 0.01050427, "balance_loss_clip": 1.02703798, "balance_loss_mlp": 1.03328323, "epoch": 0.1094243198557042, "flos": 23838974553600.0, "grad_norm": 3.5626405489282345, "language_loss": 0.7400474, "learning_rate": 3.8832034543549716e-06, "loss": 0.76179659, "num_input_tokens_seen": 39278500, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.91015625, "step": 1820, "time_per_iteration": 2.417438507080078 }, { "auxiliary_loss_clip": 0.01122324, "auxiliary_loss_mlp": 0.01050176, "balance_loss_clip": 1.02416396, "balance_loss_mlp": 1.03208113, "epoch": 0.10948444310837216, "flos": 14643166239360.0, "grad_norm": 2.4897420477094308, "language_loss": 0.82555467, "learning_rate": 3.883076197148043e-06, "loss": 0.84727973, "num_input_tokens_seen": 39294800, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.90234375, "step": 1821, "time_per_iteration": 2.348799467086792 }, { "auxiliary_loss_clip": 0.01119461, "auxiliary_loss_mlp": 0.01048744, "balance_loss_clip": 1.02570057, "balance_loss_mlp": 1.03041005, "epoch": 0.10954456636104014, "flos": 27817571825280.0, "grad_norm": 2.5166504603033286, "language_loss": 0.76038003, "learning_rate": 3.8829488727389684e-06, "loss": 0.78206205, "num_input_tokens_seen": 39314625, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.890625, "step": 1822, "time_per_iteration": 2.4672350883483887 }, { "auxiliary_loss_clip": 0.01120139, "auxiliary_loss_mlp": 0.01039119, "balance_loss_clip": 1.01685095, "balance_loss_mlp": 1.03165674, "epoch": 0.1096046896137081, "flos": 33619508184960.0, "grad_norm": 1.7645762476545976, "language_loss": 0.79805642, "learning_rate": 3.882821481132294e-06, "loss": 0.81964904, "num_input_tokens_seen": 39336465, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.8828125, "step": 1823, "time_per_iteration": 2.4774742126464844 }, { "auxiliary_loss_clip": 0.01121685, "auxiliary_loss_mlp": 0.01041736, "balance_loss_clip": 1.01870477, "balance_loss_mlp": 1.03323531, "epoch": 0.10966481286637607, "flos": 26978783477760.0, "grad_norm": 1.5088373435187543, "language_loss": 0.79352868, "learning_rate": 3.882694022332562e-06, "loss": 0.81516284, "num_input_tokens_seen": 39357930, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8828125, "step": 1824, "time_per_iteration": 2.421736717224121 }, { "auxiliary_loss_clip": 0.01121125, "auxiliary_loss_mlp": 0.01048417, "balance_loss_clip": 1.02428925, "balance_loss_mlp": 1.03207612, "epoch": 0.10972493611904403, "flos": 23035518368640.0, "grad_norm": 1.8666654077198064, "language_loss": 0.8807869, "learning_rate": 3.882566496344324e-06, "loss": 0.90248227, "num_input_tokens_seen": 39376380, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.88671875, "step": 1825, "time_per_iteration": 2.3920836448669434 }, { "auxiliary_loss_clip": 0.01116614, "auxiliary_loss_mlp": 0.01045545, "balance_loss_clip": 1.02299047, "balance_loss_mlp": 1.03023314, "epoch": 0.10978505937171201, "flos": 38103194229120.0, "grad_norm": 2.47998846056717, "language_loss": 0.76288664, "learning_rate": 3.88243890317213e-06, "loss": 0.78450823, "num_input_tokens_seen": 39399935, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.8671875, "step": 1826, "time_per_iteration": 2.5153608322143555 }, { "auxiliary_loss_clip": 0.01041664, "auxiliary_loss_mlp": 0.01008415, "balance_loss_clip": 1.00543463, "balance_loss_mlp": 1.01317477, "epoch": 0.10984518262437998, "flos": 59846645802240.0, "grad_norm": 0.8523878617208052, "language_loss": 0.54961729, "learning_rate": 3.882311242820534e-06, "loss": 0.57011807, "num_input_tokens_seen": 39460685, "router_z_loss_clip": 0.02978516, "router_z_loss_mlp": 0.28515625, "step": 1827, "time_per_iteration": 3.0348868370056152 }, { "auxiliary_loss_clip": 0.01038473, "auxiliary_loss_mlp": 0.01004865, "balance_loss_clip": 1.00183761, "balance_loss_mlp": 1.00998783, "epoch": 0.10990530587704794, "flos": 66716295494400.0, "grad_norm": 0.7314426430528725, "language_loss": 0.553303, "learning_rate": 3.882183515294092e-06, "loss": 0.57373631, "num_input_tokens_seen": 39524765, "router_z_loss_clip": 0.03027344, "router_z_loss_mlp": 0.28515625, "step": 1828, "time_per_iteration": 3.0552117824554443 }, { "auxiliary_loss_clip": 0.01125022, "auxiliary_loss_mlp": 0.01047309, "balance_loss_clip": 1.02119052, "balance_loss_mlp": 1.03243375, "epoch": 0.10996542912971592, "flos": 25446026568960.0, "grad_norm": 3.020088416281762, "language_loss": 0.84540451, "learning_rate": 3.882055720597362e-06, "loss": 0.86712778, "num_input_tokens_seen": 39543640, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.92578125, "step": 1829, "time_per_iteration": 2.4032886028289795 }, { "auxiliary_loss_clip": 0.01123522, "auxiliary_loss_mlp": 0.01051236, "balance_loss_clip": 1.02602315, "balance_loss_mlp": 1.03332651, "epoch": 0.11002555238238389, "flos": 44016503425920.0, "grad_norm": 2.1794525035170795, "language_loss": 0.88641047, "learning_rate": 3.8819278587349045e-06, "loss": 0.90815806, "num_input_tokens_seen": 39567525, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.90234375, "step": 1830, "time_per_iteration": 2.5679574012756348 }, { "auxiliary_loss_clip": 0.01034133, "auxiliary_loss_mlp": 0.01003377, "balance_loss_clip": 0.99980056, "balance_loss_mlp": 1.00614595, "epoch": 0.11008567563505185, "flos": 54061781097600.0, "grad_norm": 0.6857460363847238, "language_loss": 0.55485028, "learning_rate": 3.881799929711282e-06, "loss": 0.57522535, "num_input_tokens_seen": 39628470, "router_z_loss_clip": 0.03564453, "router_z_loss_mlp": 0.27929688, "step": 1831, "time_per_iteration": 3.04540753364563 }, { "auxiliary_loss_clip": 0.01129426, "auxiliary_loss_mlp": 0.01050821, "balance_loss_clip": 1.02491689, "balance_loss_mlp": 1.0353601, "epoch": 0.11014579888771983, "flos": 24242011822080.0, "grad_norm": 2.222061058726195, "language_loss": 0.91241372, "learning_rate": 3.881671933531061e-06, "loss": 0.9342162, "num_input_tokens_seen": 39646670, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.94140625, "step": 1832, "time_per_iteration": 3.7834312915802 }, { "auxiliary_loss_clip": 0.01035121, "auxiliary_loss_mlp": 0.01002943, "balance_loss_clip": 0.99962914, "balance_loss_mlp": 1.00814128, "epoch": 0.1102059221403878, "flos": 57740684325120.0, "grad_norm": 0.7083011127659482, "language_loss": 0.59934974, "learning_rate": 3.881543870198809e-06, "loss": 0.61973035, "num_input_tokens_seen": 39712915, "router_z_loss_clip": 0.03320312, "router_z_loss_mlp": 0.26953125, "step": 1833, "time_per_iteration": 4.4285361766815186 }, { "auxiliary_loss_clip": 0.01122688, "auxiliary_loss_mlp": 0.01042234, "balance_loss_clip": 1.0180105, "balance_loss_mlp": 1.03290153, "epoch": 0.11026604539305576, "flos": 16795107843840.0, "grad_norm": 6.5647103964142275, "language_loss": 0.80468589, "learning_rate": 3.881415739719096e-06, "loss": 0.82633519, "num_input_tokens_seen": 39730650, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.8984375, "step": 1834, "time_per_iteration": 2.365412712097168 }, { "auxiliary_loss_clip": 0.01127512, "auxiliary_loss_mlp": 0.01049115, "balance_loss_clip": 1.0236876, "balance_loss_mlp": 1.0368191, "epoch": 0.11032616864572373, "flos": 23986936362240.0, "grad_norm": 3.0585102867786986, "language_loss": 0.90389204, "learning_rate": 3.881287542096494e-06, "loss": 0.92565829, "num_input_tokens_seen": 39751065, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.90625, "step": 1835, "time_per_iteration": 2.432892084121704 }, { "auxiliary_loss_clip": 0.01126549, "auxiliary_loss_mlp": 0.01044755, "balance_loss_clip": 1.02037621, "balance_loss_mlp": 1.03564322, "epoch": 0.1103862918983917, "flos": 19682110546560.0, "grad_norm": 2.1670359810007205, "language_loss": 0.63784945, "learning_rate": 3.881159277335581e-06, "loss": 0.65956241, "num_input_tokens_seen": 39769245, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.90625, "step": 1836, "time_per_iteration": 5.125218868255615 }, { "auxiliary_loss_clip": 0.01123524, "auxiliary_loss_mlp": 0.0104534, "balance_loss_clip": 1.02220166, "balance_loss_mlp": 1.03421807, "epoch": 0.11044641515105967, "flos": 32159510282880.0, "grad_norm": 1.921355031365176, "language_loss": 0.72566742, "learning_rate": 3.88103094544093e-06, "loss": 0.74735606, "num_input_tokens_seen": 39790830, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.890625, "step": 1837, "time_per_iteration": 2.4840970039367676 }, { "auxiliary_loss_clip": 0.01127698, "auxiliary_loss_mlp": 0.01051693, "balance_loss_clip": 1.02830374, "balance_loss_mlp": 1.03508973, "epoch": 0.11050653840372764, "flos": 16688343306240.0, "grad_norm": 2.6567733589711198, "language_loss": 0.7852577, "learning_rate": 3.880902546417125e-06, "loss": 0.80705154, "num_input_tokens_seen": 39809475, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.92578125, "step": 1838, "time_per_iteration": 2.3714590072631836 }, { "auxiliary_loss_clip": 0.01126018, "auxiliary_loss_mlp": 0.01053658, "balance_loss_clip": 1.02975667, "balance_loss_mlp": 1.03530455, "epoch": 0.11056666165639562, "flos": 21207989917440.0, "grad_norm": 1.8341679071863268, "language_loss": 0.71916747, "learning_rate": 3.880774080268745e-06, "loss": 0.74096417, "num_input_tokens_seen": 39826355, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.90625, "step": 1839, "time_per_iteration": 2.373873472213745 }, { "auxiliary_loss_clip": 0.01129867, "auxiliary_loss_mlp": 0.0104687, "balance_loss_clip": 1.02095413, "balance_loss_mlp": 1.0373522, "epoch": 0.11062678490906358, "flos": 19164663152640.0, "grad_norm": 2.2181662106134747, "language_loss": 0.7848084, "learning_rate": 3.880645547000377e-06, "loss": 0.80657578, "num_input_tokens_seen": 39845335, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.92578125, "step": 1840, "time_per_iteration": 2.397270679473877 }, { "auxiliary_loss_clip": 0.0111785, "auxiliary_loss_mlp": 0.0104291, "balance_loss_clip": 1.02027237, "balance_loss_mlp": 1.03210425, "epoch": 0.11068690816173155, "flos": 24894259441920.0, "grad_norm": 1.6079820018701225, "language_loss": 0.87717295, "learning_rate": 3.880516946616606e-06, "loss": 0.89878058, "num_input_tokens_seen": 39865065, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.859375, "step": 1841, "time_per_iteration": 2.405466079711914 }, { "auxiliary_loss_clip": 0.01119553, "auxiliary_loss_mlp": 0.01044184, "balance_loss_clip": 1.02077103, "balance_loss_mlp": 1.03343081, "epoch": 0.11074703141439952, "flos": 16471427829120.0, "grad_norm": 1.962118133830366, "language_loss": 0.90375423, "learning_rate": 3.880388279122023e-06, "loss": 0.92539161, "num_input_tokens_seen": 39882780, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.859375, "step": 1842, "time_per_iteration": 2.362666606903076 }, { "auxiliary_loss_clip": 0.0112011, "auxiliary_loss_mlp": 0.01046667, "balance_loss_clip": 1.02385056, "balance_loss_mlp": 1.03192472, "epoch": 0.11080715466706749, "flos": 19171401045120.0, "grad_norm": 2.339226252612978, "language_loss": 0.85794604, "learning_rate": 3.880259544521219e-06, "loss": 0.87961382, "num_input_tokens_seen": 39900295, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8828125, "step": 1843, "time_per_iteration": 2.3702476024627686 }, { "auxiliary_loss_clip": 0.01122537, "auxiliary_loss_mlp": 0.01047296, "balance_loss_clip": 1.02326322, "balance_loss_mlp": 1.03436565, "epoch": 0.11086727791973545, "flos": 27703580636160.0, "grad_norm": 1.9907211936404086, "language_loss": 0.74612248, "learning_rate": 3.880130742818789e-06, "loss": 0.76782072, "num_input_tokens_seen": 39922075, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.8828125, "step": 1844, "time_per_iteration": 2.4470207691192627 }, { "auxiliary_loss_clip": 0.01125056, "auxiliary_loss_mlp": 0.01043043, "balance_loss_clip": 1.01850975, "balance_loss_mlp": 1.03429544, "epoch": 0.11092740117240343, "flos": 18513986544000.0, "grad_norm": 2.2664377638723683, "language_loss": 0.75605702, "learning_rate": 3.880001874019328e-06, "loss": 0.77773809, "num_input_tokens_seen": 39940115, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.90625, "step": 1845, "time_per_iteration": 2.352602243423462 }, { "auxiliary_loss_clip": 0.01121297, "auxiliary_loss_mlp": 0.01043874, "balance_loss_clip": 1.0212239, "balance_loss_mlp": 1.03393865, "epoch": 0.1109875244250714, "flos": 20521387653120.0, "grad_norm": 1.6060166262770896, "language_loss": 0.76185834, "learning_rate": 3.879872938127438e-06, "loss": 0.78351009, "num_input_tokens_seen": 39959920, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.875, "step": 1846, "time_per_iteration": 2.3857526779174805 }, { "auxiliary_loss_clip": 0.0112376, "auxiliary_loss_mlp": 0.01043863, "balance_loss_clip": 1.02034295, "balance_loss_mlp": 1.03274524, "epoch": 0.11104764767773936, "flos": 14097787891200.0, "grad_norm": 2.8021533328888744, "language_loss": 0.85970891, "learning_rate": 3.879743935147717e-06, "loss": 0.88138521, "num_input_tokens_seen": 39974755, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.91015625, "step": 1847, "time_per_iteration": 2.3417770862579346 }, { "auxiliary_loss_clip": 0.01122761, "auxiliary_loss_mlp": 0.01049376, "balance_loss_clip": 1.02574825, "balance_loss_mlp": 1.03167677, "epoch": 0.11110777093040733, "flos": 20593483344000.0, "grad_norm": 2.008085509007359, "language_loss": 0.77417588, "learning_rate": 3.87961486508477e-06, "loss": 0.79589725, "num_input_tokens_seen": 39993355, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.91015625, "step": 1848, "time_per_iteration": 2.385127067565918 }, { "auxiliary_loss_clip": 0.01121406, "auxiliary_loss_mlp": 0.01042318, "balance_loss_clip": 1.02090836, "balance_loss_mlp": 1.03614879, "epoch": 0.11116789418307531, "flos": 21869035200000.0, "grad_norm": 2.3025183471435877, "language_loss": 0.77871823, "learning_rate": 3.879485727943204e-06, "loss": 0.80035543, "num_input_tokens_seen": 40012410, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.8515625, "step": 1849, "time_per_iteration": 2.3708243370056152 }, { "auxiliary_loss_clip": 0.01122495, "auxiliary_loss_mlp": 0.01046036, "balance_loss_clip": 1.02386284, "balance_loss_mlp": 1.03078926, "epoch": 0.11122801743574327, "flos": 15522209251200.0, "grad_norm": 3.133443799544841, "language_loss": 0.712363, "learning_rate": 3.879356523727627e-06, "loss": 0.73404837, "num_input_tokens_seen": 40029315, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.91796875, "step": 1850, "time_per_iteration": 2.3596272468566895 }, { "auxiliary_loss_clip": 0.01124561, "auxiliary_loss_mlp": 0.01046046, "balance_loss_clip": 1.02158463, "balance_loss_mlp": 1.03467679, "epoch": 0.11128814068841124, "flos": 14391407358720.0, "grad_norm": 2.085739423799335, "language_loss": 0.81019771, "learning_rate": 3.87922725244265e-06, "loss": 0.8319037, "num_input_tokens_seen": 40045765, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.8984375, "step": 1851, "time_per_iteration": 2.337360382080078 }, { "auxiliary_loss_clip": 0.01121203, "auxiliary_loss_mlp": 0.01045591, "balance_loss_clip": 1.0227859, "balance_loss_mlp": 1.03309608, "epoch": 0.11134826394107922, "flos": 16653011143680.0, "grad_norm": 2.4333394722702217, "language_loss": 0.88124275, "learning_rate": 3.879097914092886e-06, "loss": 0.90291065, "num_input_tokens_seen": 40061660, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8828125, "step": 1852, "time_per_iteration": 2.357152223587036 }, { "auxiliary_loss_clip": 0.01122967, "auxiliary_loss_mlp": 0.01046162, "balance_loss_clip": 1.02071118, "balance_loss_mlp": 1.03313994, "epoch": 0.11140838719374718, "flos": 16690053962880.0, "grad_norm": 2.310298758811435, "language_loss": 0.72288007, "learning_rate": 3.878968508682952e-06, "loss": 0.74457133, "num_input_tokens_seen": 40080180, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.8984375, "step": 1853, "time_per_iteration": 2.3621346950531006 }, { "auxiliary_loss_clip": 0.0103504, "auxiliary_loss_mlp": 0.01007875, "balance_loss_clip": 1.00453675, "balance_loss_mlp": 1.00779462, "epoch": 0.11146851044641515, "flos": 60973397976960.0, "grad_norm": 0.7850775594182413, "language_loss": 0.53635836, "learning_rate": 3.878839036217464e-06, "loss": 0.55678749, "num_input_tokens_seen": 40138910, "router_z_loss_clip": 0.03344727, "router_z_loss_mlp": 0.2734375, "step": 1854, "time_per_iteration": 2.982922315597534 }, { "auxiliary_loss_clip": 0.01130098, "auxiliary_loss_mlp": 0.01046836, "balance_loss_clip": 1.02103877, "balance_loss_mlp": 1.03421712, "epoch": 0.11152863369908313, "flos": 22192924682880.0, "grad_norm": 2.521424876635544, "language_loss": 0.84896588, "learning_rate": 3.878709496701045e-06, "loss": 0.87073517, "num_input_tokens_seen": 40157745, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.95703125, "step": 1855, "time_per_iteration": 2.3863637447357178 }, { "auxiliary_loss_clip": 0.01120315, "auxiliary_loss_mlp": 0.0104773, "balance_loss_clip": 1.02462673, "balance_loss_mlp": 1.03160822, "epoch": 0.11158875695175109, "flos": 19536487799040.0, "grad_norm": 2.214380250338141, "language_loss": 0.81937933, "learning_rate": 3.8785798901383155e-06, "loss": 0.84105986, "num_input_tokens_seen": 40175375, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.88671875, "step": 1856, "time_per_iteration": 2.386831521987915 }, { "auxiliary_loss_clip": 0.0112035, "auxiliary_loss_mlp": 0.01044205, "balance_loss_clip": 1.02087581, "balance_loss_mlp": 1.03325868, "epoch": 0.11164888020441906, "flos": 25441662648960.0, "grad_norm": 2.387040860028858, "language_loss": 0.83130205, "learning_rate": 3.878450216533902e-06, "loss": 0.85294759, "num_input_tokens_seen": 40195715, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.87109375, "step": 1857, "time_per_iteration": 2.4100728034973145 }, { "auxiliary_loss_clip": 0.01122763, "auxiliary_loss_mlp": 0.01041734, "balance_loss_clip": 1.01819038, "balance_loss_mlp": 1.0308063, "epoch": 0.11170900345708702, "flos": 15631836520320.0, "grad_norm": 2.108834383109374, "language_loss": 0.82937002, "learning_rate": 3.878320475892433e-06, "loss": 0.85101503, "num_input_tokens_seen": 40213975, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.921875, "step": 1858, "time_per_iteration": 2.378499984741211 }, { "auxiliary_loss_clip": 0.0112419, "auxiliary_loss_mlp": 0.01053308, "balance_loss_clip": 1.02928758, "balance_loss_mlp": 1.03413117, "epoch": 0.111769126709755, "flos": 23038311277440.0, "grad_norm": 2.439230848213365, "language_loss": 0.91331965, "learning_rate": 3.878190668218537e-06, "loss": 0.93509459, "num_input_tokens_seen": 40233905, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.8984375, "step": 1859, "time_per_iteration": 2.3891844749450684 }, { "auxiliary_loss_clip": 0.01121581, "auxiliary_loss_mlp": 0.01046617, "balance_loss_clip": 1.02209592, "balance_loss_mlp": 1.03094959, "epoch": 0.11182924996242297, "flos": 20849641056000.0, "grad_norm": 2.1701555389504694, "language_loss": 0.81527424, "learning_rate": 3.878060793516847e-06, "loss": 0.83695626, "num_input_tokens_seen": 40252810, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.90625, "step": 1860, "time_per_iteration": 2.385209798812866 }, { "auxiliary_loss_clip": 0.01117795, "auxiliary_loss_mlp": 0.01048985, "balance_loss_clip": 1.02597761, "balance_loss_mlp": 1.03135908, "epoch": 0.11188937321509093, "flos": 17454407558400.0, "grad_norm": 4.339887688424837, "language_loss": 0.74721307, "learning_rate": 3.8779308517919995e-06, "loss": 0.76888084, "num_input_tokens_seen": 40272000, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8671875, "step": 1861, "time_per_iteration": 2.3749191761016846 }, { "auxiliary_loss_clip": 0.01121004, "auxiliary_loss_mlp": 0.01039688, "balance_loss_clip": 1.01765776, "balance_loss_mlp": 1.03207588, "epoch": 0.11194949646775891, "flos": 24094818063360.0, "grad_norm": 1.8191287058658938, "language_loss": 0.88780928, "learning_rate": 3.87780084304863e-06, "loss": 0.9094162, "num_input_tokens_seen": 40290660, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.890625, "step": 1862, "time_per_iteration": 2.390875816345215 }, { "auxiliary_loss_clip": 0.01122035, "auxiliary_loss_mlp": 0.01049775, "balance_loss_clip": 1.02638614, "balance_loss_mlp": 1.03333008, "epoch": 0.11200961972042688, "flos": 25152756215040.0, "grad_norm": 2.4417652901407396, "language_loss": 0.86999977, "learning_rate": 3.877670767291379e-06, "loss": 0.89171791, "num_input_tokens_seen": 40307820, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.88671875, "step": 1863, "time_per_iteration": 2.3952221870422363 }, { "auxiliary_loss_clip": 0.01123151, "auxiliary_loss_mlp": 0.01043867, "balance_loss_clip": 1.01883328, "balance_loss_mlp": 1.03344309, "epoch": 0.11206974297309484, "flos": 21287242437120.0, "grad_norm": 1.7716005915019037, "language_loss": 0.63989413, "learning_rate": 3.877540624524888e-06, "loss": 0.66156435, "num_input_tokens_seen": 40327430, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.8984375, "step": 1864, "time_per_iteration": 2.387942314147949 }, { "auxiliary_loss_clip": 0.01121606, "auxiliary_loss_mlp": 0.01043734, "balance_loss_clip": 1.02109623, "balance_loss_mlp": 1.0336833, "epoch": 0.11212986622576282, "flos": 18914998953600.0, "grad_norm": 2.8383906767286464, "language_loss": 0.74338508, "learning_rate": 3.877410414753802e-06, "loss": 0.76503849, "num_input_tokens_seen": 40344545, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.87890625, "step": 1865, "time_per_iteration": 2.3607590198516846 }, { "auxiliary_loss_clip": 0.0112189, "auxiliary_loss_mlp": 0.01046953, "balance_loss_clip": 1.02070308, "balance_loss_mlp": 1.03154969, "epoch": 0.11218998947843078, "flos": 22053655802880.0, "grad_norm": 9.87993134600205, "language_loss": 0.84361953, "learning_rate": 3.877280137982767e-06, "loss": 0.86530793, "num_input_tokens_seen": 40362300, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.90234375, "step": 1866, "time_per_iteration": 2.3785462379455566 }, { "auxiliary_loss_clip": 0.01120683, "auxiliary_loss_mlp": 0.01045652, "balance_loss_clip": 1.02070165, "balance_loss_mlp": 1.03133845, "epoch": 0.11225011273109875, "flos": 24570544515840.0, "grad_norm": 1.7207529171668403, "language_loss": 0.81263578, "learning_rate": 3.877149794216433e-06, "loss": 0.83429909, "num_input_tokens_seen": 40384720, "router_z_loss_clip": 0.24902344, "router_z_loss_mlp": 0.89453125, "step": 1867, "time_per_iteration": 2.4535868167877197 }, { "auxiliary_loss_clip": 0.01124865, "auxiliary_loss_mlp": 0.01051785, "balance_loss_clip": 1.02877796, "balance_loss_mlp": 1.03491139, "epoch": 0.11231023598376672, "flos": 28437419836800.0, "grad_norm": 2.0254021408977803, "language_loss": 0.86644781, "learning_rate": 3.877019383459451e-06, "loss": 0.88821429, "num_input_tokens_seen": 40404000, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.8984375, "step": 1868, "time_per_iteration": 2.432866334915161 }, { "auxiliary_loss_clip": 0.01122161, "auxiliary_loss_mlp": 0.01043699, "balance_loss_clip": 1.02059674, "balance_loss_mlp": 1.03400004, "epoch": 0.1123703592364347, "flos": 14425657269120.0, "grad_norm": 2.5856270805718995, "language_loss": 0.68023825, "learning_rate": 3.876888905716476e-06, "loss": 0.70189679, "num_input_tokens_seen": 40418665, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.87890625, "step": 1869, "time_per_iteration": 2.3502705097198486 }, { "auxiliary_loss_clip": 0.01125969, "auxiliary_loss_mlp": 0.01052933, "balance_loss_clip": 1.02717185, "balance_loss_mlp": 1.03294349, "epoch": 0.11243048248910266, "flos": 22235204206080.0, "grad_norm": 1.536104041632161, "language_loss": 0.77442759, "learning_rate": 3.876758360992165e-06, "loss": 0.79621661, "num_input_tokens_seen": 40437870, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.9296875, "step": 1870, "time_per_iteration": 2.3861958980560303 }, { "auxiliary_loss_clip": 0.01123982, "auxiliary_loss_mlp": 0.01045221, "balance_loss_clip": 1.02044964, "balance_loss_mlp": 1.03114092, "epoch": 0.11249060574177062, "flos": 18583289326080.0, "grad_norm": 2.2165975537900806, "language_loss": 0.7623505, "learning_rate": 3.8766277492911736e-06, "loss": 0.7840426, "num_input_tokens_seen": 40455570, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.92578125, "step": 1871, "time_per_iteration": 3.832282781600952 }, { "auxiliary_loss_clip": 0.0112357, "auxiliary_loss_mlp": 0.01040434, "balance_loss_clip": 1.01706851, "balance_loss_mlp": 1.03323805, "epoch": 0.1125507289944386, "flos": 22855471153920.0, "grad_norm": 1.9322766154803015, "language_loss": 0.81456953, "learning_rate": 3.876497070618166e-06, "loss": 0.83620954, "num_input_tokens_seen": 40473600, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.90234375, "step": 1872, "time_per_iteration": 2.3943119049072266 }, { "auxiliary_loss_clip": 0.01125733, "auxiliary_loss_mlp": 0.01052084, "balance_loss_clip": 1.02839732, "balance_loss_mlp": 1.03431916, "epoch": 0.11261085224710657, "flos": 19675547210880.0, "grad_norm": 2.3744857363701612, "language_loss": 0.82998043, "learning_rate": 3.876366324977806e-06, "loss": 0.8517586, "num_input_tokens_seen": 40490025, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.9140625, "step": 1873, "time_per_iteration": 3.711221933364868 }, { "auxiliary_loss_clip": 0.01125359, "auxiliary_loss_mlp": 0.01049416, "balance_loss_clip": 1.02316654, "balance_loss_mlp": 1.03108621, "epoch": 0.11267097549977453, "flos": 26062173976320.0, "grad_norm": 1.8842552987423473, "language_loss": 0.92325759, "learning_rate": 3.876235512374757e-06, "loss": 0.94500536, "num_input_tokens_seen": 40511580, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.94140625, "step": 1874, "time_per_iteration": 2.4347848892211914 }, { "auxiliary_loss_clip": 0.01119011, "auxiliary_loss_mlp": 0.01047171, "balance_loss_clip": 1.02326918, "balance_loss_mlp": 1.03145373, "epoch": 0.11273109875244251, "flos": 21067010380800.0, "grad_norm": 1.4604694796120459, "language_loss": 0.7536639, "learning_rate": 3.876104632813689e-06, "loss": 0.77532566, "num_input_tokens_seen": 40530155, "router_z_loss_clip": 0.23925781, "router_z_loss_mlp": 0.875, "step": 1875, "time_per_iteration": 5.135155439376831 }, { "auxiliary_loss_clip": 0.01120029, "auxiliary_loss_mlp": 0.01045988, "balance_loss_clip": 1.02455413, "balance_loss_mlp": 1.03409672, "epoch": 0.11279122200511048, "flos": 27087782342400.0, "grad_norm": 2.0494182757181982, "language_loss": 0.71384954, "learning_rate": 3.875973686299272e-06, "loss": 0.73550969, "num_input_tokens_seen": 40549500, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.859375, "step": 1876, "time_per_iteration": 2.4182865619659424 }, { "auxiliary_loss_clip": 0.01121093, "auxiliary_loss_mlp": 0.01044036, "balance_loss_clip": 1.02175617, "balance_loss_mlp": 1.03341925, "epoch": 0.11285134525777844, "flos": 20187024762240.0, "grad_norm": 1.8334081916707283, "language_loss": 0.7652418, "learning_rate": 3.875842672836182e-06, "loss": 0.78689313, "num_input_tokens_seen": 40567475, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.875, "step": 1877, "time_per_iteration": 2.3859004974365234 }, { "auxiliary_loss_clip": 0.0112174, "auxiliary_loss_mlp": 0.01055278, "balance_loss_clip": 1.02965963, "balance_loss_mlp": 1.03242016, "epoch": 0.11291146851044641, "flos": 12457638040320.0, "grad_norm": 2.5233166777136145, "language_loss": 0.87412786, "learning_rate": 3.87571159242909e-06, "loss": 0.89589804, "num_input_tokens_seen": 40583280, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.890625, "step": 1878, "time_per_iteration": 2.3492658138275146 }, { "auxiliary_loss_clip": 0.01121535, "auxiliary_loss_mlp": 0.01042434, "balance_loss_clip": 1.01773417, "balance_loss_mlp": 1.03220189, "epoch": 0.11297159176311439, "flos": 23841173969280.0, "grad_norm": 2.1187437654021233, "language_loss": 0.80941617, "learning_rate": 3.875580445082677e-06, "loss": 0.83105588, "num_input_tokens_seen": 40603080, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.89453125, "step": 1879, "time_per_iteration": 2.4078209400177 }, { "auxiliary_loss_clip": 0.0112127, "auxiliary_loss_mlp": 0.0105136, "balance_loss_clip": 1.02637398, "balance_loss_mlp": 1.03222597, "epoch": 0.11303171501578235, "flos": 29929363499520.0, "grad_norm": 2.0738310531410757, "language_loss": 0.69966519, "learning_rate": 3.875449230801622e-06, "loss": 0.72139156, "num_input_tokens_seen": 40623255, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.890625, "step": 1880, "time_per_iteration": 2.4354565143585205 }, { "auxiliary_loss_clip": 0.01121064, "auxiliary_loss_mlp": 0.01045815, "balance_loss_clip": 1.0205431, "balance_loss_mlp": 1.03177834, "epoch": 0.11309183826845032, "flos": 16179623752320.0, "grad_norm": 1.6998560748807998, "language_loss": 0.71996421, "learning_rate": 3.875317949590609e-06, "loss": 0.74163306, "num_input_tokens_seen": 40641570, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.89453125, "step": 1881, "time_per_iteration": 2.3698432445526123 }, { "auxiliary_loss_clip": 0.01122401, "auxiliary_loss_mlp": 0.01048771, "balance_loss_clip": 1.02403498, "balance_loss_mlp": 1.03180218, "epoch": 0.1131519615211183, "flos": 12019897013760.0, "grad_norm": 2.16034106561837, "language_loss": 0.74119371, "learning_rate": 3.875186601454322e-06, "loss": 0.76290548, "num_input_tokens_seen": 40658775, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.90625, "step": 1882, "time_per_iteration": 2.3546245098114014 }, { "auxiliary_loss_clip": 0.01119348, "auxiliary_loss_mlp": 0.01047118, "balance_loss_clip": 1.0215472, "balance_loss_mlp": 1.03185344, "epoch": 0.11321208477378626, "flos": 26248924172160.0, "grad_norm": 2.046400534186846, "language_loss": 0.79340416, "learning_rate": 3.8750551863974484e-06, "loss": 0.81506884, "num_input_tokens_seen": 40679555, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.875, "step": 1883, "time_per_iteration": 2.4280452728271484 }, { "auxiliary_loss_clip": 0.01121617, "auxiliary_loss_mlp": 0.01044822, "balance_loss_clip": 1.01896548, "balance_loss_mlp": 1.03018355, "epoch": 0.11327220802645423, "flos": 13625517663360.0, "grad_norm": 2.5416306163806515, "language_loss": 0.77227646, "learning_rate": 3.874923704424679e-06, "loss": 0.79394084, "num_input_tokens_seen": 40697295, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.9140625, "step": 1884, "time_per_iteration": 2.3503458499908447 }, { "auxiliary_loss_clip": 0.01040015, "auxiliary_loss_mlp": 0.01006052, "balance_loss_clip": 1.00242758, "balance_loss_mlp": 1.01261711, "epoch": 0.1133323312791222, "flos": 57188672818560.0, "grad_norm": 0.7855083275624904, "language_loss": 0.55201423, "learning_rate": 3.8747921555407045e-06, "loss": 0.57247484, "num_input_tokens_seen": 40758095, "router_z_loss_clip": 0.03613281, "router_z_loss_mlp": 0.2734375, "step": 1885, "time_per_iteration": 2.9273834228515625 }, { "auxiliary_loss_clip": 0.01113865, "auxiliary_loss_mlp": 0.01042374, "balance_loss_clip": 1.01996267, "balance_loss_mlp": 1.03008294, "epoch": 0.11339245453179017, "flos": 24350591750400.0, "grad_norm": 1.9656073476571887, "language_loss": 0.90563154, "learning_rate": 3.874660539750222e-06, "loss": 0.92719388, "num_input_tokens_seen": 40777140, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.83984375, "step": 1886, "time_per_iteration": 2.406250476837158 }, { "auxiliary_loss_clip": 0.01123372, "auxiliary_loss_mlp": 0.01042115, "balance_loss_clip": 1.01962066, "balance_loss_mlp": 1.03481531, "epoch": 0.11345257778445814, "flos": 22669698476160.0, "grad_norm": 1.9569472103942396, "language_loss": 0.85377294, "learning_rate": 3.874528857057926e-06, "loss": 0.87542778, "num_input_tokens_seen": 40797505, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.88671875, "step": 1887, "time_per_iteration": 2.4290239810943604 }, { "auxiliary_loss_clip": 0.01123608, "auxiliary_loss_mlp": 0.01048515, "balance_loss_clip": 1.02505445, "balance_loss_mlp": 1.03379381, "epoch": 0.11351270103712612, "flos": 20987408747520.0, "grad_norm": 3.57150940087995, "language_loss": 0.75795519, "learning_rate": 3.874397107468516e-06, "loss": 0.77967644, "num_input_tokens_seen": 40812970, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.8984375, "step": 1888, "time_per_iteration": 2.40659236907959 }, { "auxiliary_loss_clip": 0.01125696, "auxiliary_loss_mlp": 0.01049427, "balance_loss_clip": 1.02421427, "balance_loss_mlp": 1.03497028, "epoch": 0.11357282428979408, "flos": 37346241841920.0, "grad_norm": 2.520445784871137, "language_loss": 0.68051779, "learning_rate": 3.874265290986696e-06, "loss": 0.70226902, "num_input_tokens_seen": 40837745, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.90625, "step": 1889, "time_per_iteration": 2.5671088695526123 }, { "auxiliary_loss_clip": 0.01123114, "auxiliary_loss_mlp": 0.01041424, "balance_loss_clip": 1.01773691, "balance_loss_mlp": 1.03432798, "epoch": 0.11363294754246205, "flos": 21756091351680.0, "grad_norm": 2.475836393648607, "language_loss": 0.8416034, "learning_rate": 3.874133407617169e-06, "loss": 0.86324883, "num_input_tokens_seen": 40856490, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.88671875, "step": 1890, "time_per_iteration": 2.384964942932129 }, { "auxiliary_loss_clip": 0.01118093, "auxiliary_loss_mlp": 0.01043316, "balance_loss_clip": 1.02026057, "balance_loss_mlp": 1.03270805, "epoch": 0.11369307079513001, "flos": 22600535339520.0, "grad_norm": 2.160870064589821, "language_loss": 0.64799368, "learning_rate": 3.874001457364642e-06, "loss": 0.66960776, "num_input_tokens_seen": 40874070, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8515625, "step": 1891, "time_per_iteration": 2.3964099884033203 }, { "auxiliary_loss_clip": 0.01121113, "auxiliary_loss_mlp": 0.01038546, "balance_loss_clip": 1.01606369, "balance_loss_mlp": 1.03340304, "epoch": 0.11375319404779799, "flos": 21943190661120.0, "grad_norm": 2.5961754883451422, "language_loss": 0.8853538, "learning_rate": 3.873869440233822e-06, "loss": 0.90695035, "num_input_tokens_seen": 40892425, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.875, "step": 1892, "time_per_iteration": 2.4157590866088867 }, { "auxiliary_loss_clip": 0.01123236, "auxiliary_loss_mlp": 0.01056673, "balance_loss_clip": 1.03193736, "balance_loss_mlp": 1.03481007, "epoch": 0.11381331730046595, "flos": 26394267628800.0, "grad_norm": 2.38988995888122, "language_loss": 0.73289359, "learning_rate": 3.8737373562294225e-06, "loss": 0.75469267, "num_input_tokens_seen": 40912190, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.8828125, "step": 1893, "time_per_iteration": 2.435520887374878 }, { "auxiliary_loss_clip": 0.01119479, "auxiliary_loss_mlp": 0.01054012, "balance_loss_clip": 1.02993214, "balance_loss_mlp": 1.03295314, "epoch": 0.11387344055313392, "flos": 23803607479680.0, "grad_norm": 2.002772720280015, "language_loss": 0.7954644, "learning_rate": 3.873605205356157e-06, "loss": 0.81719935, "num_input_tokens_seen": 40928395, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.8671875, "step": 1894, "time_per_iteration": 2.38785982131958 }, { "auxiliary_loss_clip": 0.01122086, "auxiliary_loss_mlp": 0.01047838, "balance_loss_clip": 1.02388895, "balance_loss_mlp": 1.03158116, "epoch": 0.1139335638058019, "flos": 34521699294720.0, "grad_norm": 5.640230676070867, "language_loss": 0.80075616, "learning_rate": 3.873472987618742e-06, "loss": 0.82245541, "num_input_tokens_seen": 40946555, "router_z_loss_clip": 0.23925781, "router_z_loss_mlp": 0.90625, "step": 1895, "time_per_iteration": 2.4901442527770996 }, { "auxiliary_loss_clip": 0.01037659, "auxiliary_loss_mlp": 0.01010939, "balance_loss_clip": 1.00774467, "balance_loss_mlp": 1.01096821, "epoch": 0.11399368705846986, "flos": 70584148333440.0, "grad_norm": 0.799269048333181, "language_loss": 0.63373232, "learning_rate": 3.873340703021894e-06, "loss": 0.65421826, "num_input_tokens_seen": 41004910, "router_z_loss_clip": 0.03198242, "router_z_loss_mlp": 0.26757812, "step": 1896, "time_per_iteration": 3.1031527519226074 }, { "auxiliary_loss_clip": 0.01120183, "auxiliary_loss_mlp": 0.0105012, "balance_loss_clip": 1.02465725, "balance_loss_mlp": 1.03358936, "epoch": 0.11405381031113783, "flos": 21323203004160.0, "grad_norm": 1.8792429588436772, "language_loss": 0.84862256, "learning_rate": 3.873208351570335e-06, "loss": 0.87032557, "num_input_tokens_seen": 41026385, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.8671875, "step": 1897, "time_per_iteration": 2.421992540359497 }, { "auxiliary_loss_clip": 0.01120028, "auxiliary_loss_mlp": 0.01045303, "balance_loss_clip": 1.02270079, "balance_loss_mlp": 1.0324626, "epoch": 0.11411393356380581, "flos": 19718594784000.0, "grad_norm": 2.787889135189672, "language_loss": 0.79151994, "learning_rate": 3.873075933268788e-06, "loss": 0.81317323, "num_input_tokens_seen": 41045315, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.875, "step": 1898, "time_per_iteration": 2.4150617122650146 }, { "auxiliary_loss_clip": 0.0112114, "auxiliary_loss_mlp": 0.01050052, "balance_loss_clip": 1.02486324, "balance_loss_mlp": 1.0317378, "epoch": 0.11417405681647377, "flos": 17529470714880.0, "grad_norm": 2.0055423221469075, "language_loss": 0.73206705, "learning_rate": 3.87294344812198e-06, "loss": 0.75377893, "num_input_tokens_seen": 41063390, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.89453125, "step": 1899, "time_per_iteration": 2.3959221839904785 }, { "auxiliary_loss_clip": 0.01034788, "auxiliary_loss_mlp": 0.01003222, "balance_loss_clip": 0.99971706, "balance_loss_mlp": 1.00755334, "epoch": 0.11423418006914174, "flos": 59671416355200.0, "grad_norm": 0.9087447150687288, "language_loss": 0.63396221, "learning_rate": 3.8728108961346386e-06, "loss": 0.65434235, "num_input_tokens_seen": 41124180, "router_z_loss_clip": 0.03515625, "router_z_loss_mlp": 0.2734375, "step": 1900, "time_per_iteration": 2.9975342750549316 }, { "auxiliary_loss_clip": 0.01122627, "auxiliary_loss_mlp": 0.01046789, "balance_loss_clip": 1.02257764, "balance_loss_mlp": 1.03412795, "epoch": 0.1142943033218097, "flos": 22962096046080.0, "grad_norm": 1.662102926602138, "language_loss": 0.78009129, "learning_rate": 3.872678277311493e-06, "loss": 0.80178547, "num_input_tokens_seen": 41143485, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.88671875, "step": 1901, "time_per_iteration": 2.405158758163452 }, { "auxiliary_loss_clip": 0.01121845, "auxiliary_loss_mlp": 0.01041315, "balance_loss_clip": 1.0184269, "balance_loss_mlp": 1.0350672, "epoch": 0.11435442657447768, "flos": 18255385036800.0, "grad_norm": 2.0287733645949926, "language_loss": 0.83728218, "learning_rate": 3.872545591657276e-06, "loss": 0.85891378, "num_input_tokens_seen": 41161695, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8671875, "step": 1902, "time_per_iteration": 2.3576552867889404 }, { "auxiliary_loss_clip": 0.01117738, "auxiliary_loss_mlp": 0.0104449, "balance_loss_clip": 1.01995707, "balance_loss_mlp": 1.0303036, "epoch": 0.11441454982714565, "flos": 24060044482560.0, "grad_norm": 1.6977257217677675, "language_loss": 0.77722776, "learning_rate": 3.872412839176725e-06, "loss": 0.79885, "num_input_tokens_seen": 41181715, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.875, "step": 1903, "time_per_iteration": 2.4129691123962402 }, { "auxiliary_loss_clip": 0.0112092, "auxiliary_loss_mlp": 0.01038833, "balance_loss_clip": 1.01737499, "balance_loss_mlp": 1.03335369, "epoch": 0.11447467307981361, "flos": 25336538945280.0, "grad_norm": 2.289445239864963, "language_loss": 0.75533634, "learning_rate": 3.872280019874576e-06, "loss": 0.77693391, "num_input_tokens_seen": 41201770, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.875, "step": 1904, "time_per_iteration": 2.4143829345703125 }, { "auxiliary_loss_clip": 0.01118086, "auxiliary_loss_mlp": 0.01044327, "balance_loss_clip": 1.01951957, "balance_loss_mlp": 1.03191447, "epoch": 0.11453479633248159, "flos": 21724983463680.0, "grad_norm": 2.360565416980462, "language_loss": 0.91935968, "learning_rate": 3.872147133755568e-06, "loss": 0.94098371, "num_input_tokens_seen": 41220590, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.86328125, "step": 1905, "time_per_iteration": 2.3952243328094482 }, { "auxiliary_loss_clip": 0.01121045, "auxiliary_loss_mlp": 0.01047477, "balance_loss_clip": 1.02266955, "balance_loss_mlp": 1.03007984, "epoch": 0.11459491958514956, "flos": 12968871212160.0, "grad_norm": 2.7913693138781923, "language_loss": 0.77344108, "learning_rate": 3.872014180824446e-06, "loss": 0.79512632, "num_input_tokens_seen": 41237250, "router_z_loss_clip": 0.24902344, "router_z_loss_mlp": 0.91015625, "step": 1906, "time_per_iteration": 2.3560922145843506 }, { "auxiliary_loss_clip": 0.01119823, "auxiliary_loss_mlp": 0.01049127, "balance_loss_clip": 1.02524948, "balance_loss_mlp": 1.03317046, "epoch": 0.11465504283781752, "flos": 22710162608640.0, "grad_norm": 11.269839278915923, "language_loss": 0.81792992, "learning_rate": 3.8718811610859526e-06, "loss": 0.83961946, "num_input_tokens_seen": 41256680, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.8671875, "step": 1907, "time_per_iteration": 2.3930797576904297 }, { "auxiliary_loss_clip": 0.01120222, "auxiliary_loss_mlp": 0.01054948, "balance_loss_clip": 1.03223836, "balance_loss_mlp": 1.03404033, "epoch": 0.1147151660904855, "flos": 23397428188800.0, "grad_norm": 2.608949679238145, "language_loss": 0.84991479, "learning_rate": 3.8717480745448356e-06, "loss": 0.87166649, "num_input_tokens_seen": 41270955, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.86328125, "step": 1908, "time_per_iteration": 2.362637996673584 }, { "auxiliary_loss_clip": 0.01034183, "auxiliary_loss_mlp": 0.01003795, "balance_loss_clip": 1.00045669, "balance_loss_mlp": 1.00746334, "epoch": 0.11477528934315347, "flos": 63009044242560.0, "grad_norm": 0.9166563959521401, "language_loss": 0.60988611, "learning_rate": 3.871614921205845e-06, "loss": 0.63026589, "num_input_tokens_seen": 41319180, "router_z_loss_clip": 0.03344727, "router_z_loss_mlp": 0.26757812, "step": 1909, "time_per_iteration": 2.77168607711792 }, { "auxiliary_loss_clip": 0.01121819, "auxiliary_loss_mlp": 0.01041726, "balance_loss_clip": 1.01943362, "balance_loss_mlp": 1.0347178, "epoch": 0.11483541259582143, "flos": 16324687918080.0, "grad_norm": 1.8870721212084607, "language_loss": 0.78994447, "learning_rate": 3.871481701073731e-06, "loss": 0.81157988, "num_input_tokens_seen": 41337480, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.87109375, "step": 1910, "time_per_iteration": 2.3642501831054688 }, { "auxiliary_loss_clip": 0.01123226, "auxiliary_loss_mlp": 0.01042623, "balance_loss_clip": 1.02014017, "balance_loss_mlp": 1.03540301, "epoch": 0.1148955358484894, "flos": 21579325804800.0, "grad_norm": 2.1832236962668934, "language_loss": 0.77382857, "learning_rate": 3.8713484141532505e-06, "loss": 0.79548711, "num_input_tokens_seen": 41354650, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.87890625, "step": 1911, "time_per_iteration": 3.7580933570861816 }, { "auxiliary_loss_clip": 0.01116111, "auxiliary_loss_mlp": 0.01042138, "balance_loss_clip": 1.01986945, "balance_loss_mlp": 1.03272152, "epoch": 0.11495565910115738, "flos": 27672437836800.0, "grad_norm": 1.8401151809725036, "language_loss": 0.79115731, "learning_rate": 3.871215060449158e-06, "loss": 0.81273973, "num_input_tokens_seen": 41376935, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.8359375, "step": 1912, "time_per_iteration": 2.4457361698150635 }, { "auxiliary_loss_clip": 0.01116913, "auxiliary_loss_mlp": 0.01054898, "balance_loss_clip": 1.03103209, "balance_loss_mlp": 1.03193891, "epoch": 0.11501578235382534, "flos": 20631294213120.0, "grad_norm": 1.8881752293686607, "language_loss": 0.77768546, "learning_rate": 3.871081639966213e-06, "loss": 0.79940355, "num_input_tokens_seen": 41396105, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.8515625, "step": 1913, "time_per_iteration": 3.730783224105835 }, { "auxiliary_loss_clip": 0.01120003, "auxiliary_loss_mlp": 0.01040292, "balance_loss_clip": 1.01674795, "balance_loss_mlp": 1.03115487, "epoch": 0.1150759056064933, "flos": 19828012584960.0, "grad_norm": 2.030156090053584, "language_loss": 0.7035594, "learning_rate": 3.870948152709178e-06, "loss": 0.72516233, "num_input_tokens_seen": 41415600, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.890625, "step": 1914, "time_per_iteration": 2.431093215942383 }, { "auxiliary_loss_clip": 0.0103309, "auxiliary_loss_mlp": 0.01008316, "balance_loss_clip": 1.00535917, "balance_loss_mlp": 1.0072974, "epoch": 0.11513602885916129, "flos": 70041981830400.0, "grad_norm": 0.7608967370047242, "language_loss": 0.61050045, "learning_rate": 3.870814598682816e-06, "loss": 0.63091445, "num_input_tokens_seen": 41478760, "router_z_loss_clip": 0.02954102, "router_z_loss_mlp": 0.2578125, "step": 1915, "time_per_iteration": 5.959998846054077 }, { "auxiliary_loss_clip": 0.01121487, "auxiliary_loss_mlp": 0.01043383, "balance_loss_clip": 1.01954126, "balance_loss_mlp": 1.03480065, "epoch": 0.11519615211182925, "flos": 15740835384960.0, "grad_norm": 6.53600990937075, "language_loss": 0.92811406, "learning_rate": 3.8706809778918935e-06, "loss": 0.94976276, "num_input_tokens_seen": 41495720, "router_z_loss_clip": 0.23925781, "router_z_loss_mlp": 0.8671875, "step": 1916, "time_per_iteration": 2.364123821258545 }, { "auxiliary_loss_clip": 0.01118829, "auxiliary_loss_mlp": 0.01047289, "balance_loss_clip": 1.02338719, "balance_loss_mlp": 1.0321542, "epoch": 0.11525627536449722, "flos": 20666591464320.0, "grad_norm": 1.9052463671782878, "language_loss": 0.72640043, "learning_rate": 3.870547290341179e-06, "loss": 0.74806166, "num_input_tokens_seen": 41513585, "router_z_loss_clip": 0.23925781, "router_z_loss_mlp": 0.8671875, "step": 1917, "time_per_iteration": 2.4193167686462402 }, { "auxiliary_loss_clip": 0.01118869, "auxiliary_loss_mlp": 0.0103757, "balance_loss_clip": 1.01358509, "balance_loss_mlp": 1.03429604, "epoch": 0.1153163986171652, "flos": 20302237848960.0, "grad_norm": 2.388180552467478, "language_loss": 0.74289095, "learning_rate": 3.870413536035442e-06, "loss": 0.76445532, "num_input_tokens_seen": 41533390, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.84375, "step": 1918, "time_per_iteration": 2.3907649517059326 }, { "auxiliary_loss_clip": 0.01121428, "auxiliary_loss_mlp": 0.01037743, "balance_loss_clip": 1.01305485, "balance_loss_mlp": 1.03278852, "epoch": 0.11537652186983316, "flos": 17638364845440.0, "grad_norm": 2.183824722391978, "language_loss": 0.86369371, "learning_rate": 3.870279714979458e-06, "loss": 0.88528538, "num_input_tokens_seen": 41551015, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.88671875, "step": 1919, "time_per_iteration": 2.3742287158966064 }, { "auxiliary_loss_clip": 0.01117779, "auxiliary_loss_mlp": 0.01045815, "balance_loss_clip": 1.02177036, "balance_loss_mlp": 1.03159249, "epoch": 0.11543664512250112, "flos": 21068337012480.0, "grad_norm": 3.6738136039291676, "language_loss": 0.86615455, "learning_rate": 3.870145827178002e-06, "loss": 0.8877905, "num_input_tokens_seen": 41568055, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.859375, "step": 1920, "time_per_iteration": 2.3817317485809326 }, { "auxiliary_loss_clip": 0.01117626, "auxiliary_loss_mlp": 0.01040966, "balance_loss_clip": 1.01758945, "balance_loss_mlp": 1.03209615, "epoch": 0.11549676837516909, "flos": 22746437377920.0, "grad_norm": 2.0350494362644977, "language_loss": 0.79077518, "learning_rate": 3.8700118726358525e-06, "loss": 0.81236112, "num_input_tokens_seen": 41587435, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.85546875, "step": 1921, "time_per_iteration": 2.4029366970062256 }, { "auxiliary_loss_clip": 0.01123317, "auxiliary_loss_mlp": 0.01051315, "balance_loss_clip": 1.02592325, "balance_loss_mlp": 1.03322721, "epoch": 0.11555689162783707, "flos": 19168049554560.0, "grad_norm": 1.9432903044582477, "language_loss": 0.78655696, "learning_rate": 3.869877851357789e-06, "loss": 0.80830324, "num_input_tokens_seen": 41604975, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.90234375, "step": 1922, "time_per_iteration": 2.364842414855957 }, { "auxiliary_loss_clip": 0.01125324, "auxiliary_loss_mlp": 0.01050104, "balance_loss_clip": 1.02689362, "balance_loss_mlp": 1.03513312, "epoch": 0.11561701488050503, "flos": 24570893629440.0, "grad_norm": 2.1217780923341003, "language_loss": 0.8439163, "learning_rate": 3.869743763348595e-06, "loss": 0.86567056, "num_input_tokens_seen": 41626155, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.90234375, "step": 1923, "time_per_iteration": 2.4334733486175537 }, { "auxiliary_loss_clip": 0.01122675, "auxiliary_loss_mlp": 0.01044974, "balance_loss_clip": 1.02002394, "balance_loss_mlp": 1.03377521, "epoch": 0.115677138133173, "flos": 17091590042880.0, "grad_norm": 2.242921719131463, "language_loss": 0.80798101, "learning_rate": 3.869609608613055e-06, "loss": 0.82965755, "num_input_tokens_seen": 41644805, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.890625, "step": 1924, "time_per_iteration": 2.3767452239990234 }, { "auxiliary_loss_clip": 0.01033338, "auxiliary_loss_mlp": 0.01004751, "balance_loss_clip": 1.00162804, "balance_loss_mlp": 1.0078938, "epoch": 0.11573726138584098, "flos": 62700515758080.0, "grad_norm": 0.8289051333358758, "language_loss": 0.61181498, "learning_rate": 3.869475387155958e-06, "loss": 0.63219583, "num_input_tokens_seen": 41709345, "router_z_loss_clip": 0.03125, "router_z_loss_mlp": 0.25390625, "step": 1925, "time_per_iteration": 3.051574230194092 }, { "auxiliary_loss_clip": 0.01118964, "auxiliary_loss_mlp": 0.01048068, "balance_loss_clip": 1.02341545, "balance_loss_mlp": 1.03163743, "epoch": 0.11579738463850894, "flos": 22600046580480.0, "grad_norm": 1.8870413846579273, "language_loss": 0.75285721, "learning_rate": 3.8693410989820925e-06, "loss": 0.77452743, "num_input_tokens_seen": 41730210, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.875, "step": 1926, "time_per_iteration": 2.398479461669922 }, { "auxiliary_loss_clip": 0.01120091, "auxiliary_loss_mlp": 0.0104481, "balance_loss_clip": 1.01912022, "balance_loss_mlp": 1.03248537, "epoch": 0.11585750789117691, "flos": 21725053286400.0, "grad_norm": 4.134645563731568, "language_loss": 0.72157353, "learning_rate": 3.869206744096252e-06, "loss": 0.74322253, "num_input_tokens_seen": 41750270, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.875, "step": 1927, "time_per_iteration": 2.406965494155884 }, { "auxiliary_loss_clip": 0.01117803, "auxiliary_loss_mlp": 0.0104096, "balance_loss_clip": 1.01684427, "balance_loss_mlp": 1.03162479, "epoch": 0.11591763114384489, "flos": 26286316104960.0, "grad_norm": 1.534695631001158, "language_loss": 0.86650527, "learning_rate": 3.869072322503232e-06, "loss": 0.88809288, "num_input_tokens_seen": 41772975, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.859375, "step": 1928, "time_per_iteration": 2.432307243347168 }, { "auxiliary_loss_clip": 0.01120868, "auxiliary_loss_mlp": 0.01047544, "balance_loss_clip": 1.02378583, "balance_loss_mlp": 1.03301144, "epoch": 0.11597775439651285, "flos": 22999418156160.0, "grad_norm": 1.7562016611887232, "language_loss": 0.77448833, "learning_rate": 3.868937834207828e-06, "loss": 0.79617244, "num_input_tokens_seen": 41791765, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.87890625, "step": 1929, "time_per_iteration": 2.4027504920959473 }, { "auxiliary_loss_clip": 0.01117126, "auxiliary_loss_mlp": 0.01050525, "balance_loss_clip": 1.02811384, "balance_loss_mlp": 1.0315845, "epoch": 0.11603787764918082, "flos": 31940360478720.0, "grad_norm": 2.9383215708820356, "language_loss": 0.76913202, "learning_rate": 3.86880327921484e-06, "loss": 0.79080844, "num_input_tokens_seen": 41815615, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.85546875, "step": 1930, "time_per_iteration": 2.471935510635376 }, { "auxiliary_loss_clip": 0.01119107, "auxiliary_loss_mlp": 0.01045321, "balance_loss_clip": 1.02261209, "balance_loss_mlp": 1.03244591, "epoch": 0.1160980009018488, "flos": 22270606191360.0, "grad_norm": 1.950474949962868, "language_loss": 0.72070694, "learning_rate": 3.8686686575290695e-06, "loss": 0.74235123, "num_input_tokens_seen": 41834810, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.8671875, "step": 1931, "time_per_iteration": 2.3999249935150146 }, { "auxiliary_loss_clip": 0.01123334, "auxiliary_loss_mlp": 0.01045584, "balance_loss_clip": 1.0216229, "balance_loss_mlp": 1.03538382, "epoch": 0.11615812415451676, "flos": 22782537590400.0, "grad_norm": 1.6694959414934365, "language_loss": 0.82114506, "learning_rate": 3.868533969155322e-06, "loss": 0.84283423, "num_input_tokens_seen": 41854975, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.8828125, "step": 1932, "time_per_iteration": 2.4132401943206787 }, { "auxiliary_loss_clip": 0.01029327, "auxiliary_loss_mlp": 0.01009468, "balance_loss_clip": 1.00627303, "balance_loss_mlp": 1.00398624, "epoch": 0.11621824740718473, "flos": 67142864885760.0, "grad_norm": 0.77813532920461, "language_loss": 0.61104012, "learning_rate": 3.868399214098404e-06, "loss": 0.631428, "num_input_tokens_seen": 41911105, "router_z_loss_clip": 0.03198242, "router_z_loss_mlp": 0.25390625, "step": 1933, "time_per_iteration": 2.89139986038208 }, { "auxiliary_loss_clip": 0.01121111, "auxiliary_loss_mlp": 0.01044965, "balance_loss_clip": 1.02174306, "balance_loss_mlp": 1.03182209, "epoch": 0.11627837065985269, "flos": 20374892121600.0, "grad_norm": 5.832653992836649, "language_loss": 0.85950387, "learning_rate": 3.868264392363124e-06, "loss": 0.88116461, "num_input_tokens_seen": 41931750, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.89453125, "step": 1934, "time_per_iteration": 2.4212839603424072 }, { "auxiliary_loss_clip": 0.01126041, "auxiliary_loss_mlp": 0.01047781, "balance_loss_clip": 1.02274728, "balance_loss_mlp": 1.03563976, "epoch": 0.11633849391252067, "flos": 21724739084160.0, "grad_norm": 2.285134865303062, "language_loss": 0.65957439, "learning_rate": 3.868129503954293e-06, "loss": 0.68131256, "num_input_tokens_seen": 41949400, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.90625, "step": 1935, "time_per_iteration": 2.410168409347534 }, { "auxiliary_loss_clip": 0.01124111, "auxiliary_loss_mlp": 0.01048578, "balance_loss_clip": 1.02572513, "balance_loss_mlp": 1.03325069, "epoch": 0.11639861716518864, "flos": 18804394166400.0, "grad_norm": 2.468005328842679, "language_loss": 0.75913846, "learning_rate": 3.867994548876726e-06, "loss": 0.78086537, "num_input_tokens_seen": 41968100, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.90625, "step": 1936, "time_per_iteration": 2.3786065578460693 }, { "auxiliary_loss_clip": 0.01123324, "auxiliary_loss_mlp": 0.01044361, "balance_loss_clip": 1.01954186, "balance_loss_mlp": 1.03270447, "epoch": 0.1164587404178566, "flos": 21213924848640.0, "grad_norm": 2.03835241920668, "language_loss": 0.8434478, "learning_rate": 3.867859527135238e-06, "loss": 0.86512464, "num_input_tokens_seen": 41986375, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.90625, "step": 1937, "time_per_iteration": 2.4055237770080566 }, { "auxiliary_loss_clip": 0.011168, "auxiliary_loss_mlp": 0.01036059, "balance_loss_clip": 1.01487517, "balance_loss_mlp": 1.0320996, "epoch": 0.11651886367052458, "flos": 27817397268480.0, "grad_norm": 1.9669668728451497, "language_loss": 0.76133978, "learning_rate": 3.867724438734649e-06, "loss": 0.78286839, "num_input_tokens_seen": 42006055, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.84765625, "step": 1938, "time_per_iteration": 2.444209575653076 }, { "auxiliary_loss_clip": 0.0112379, "auxiliary_loss_mlp": 0.01045809, "balance_loss_clip": 1.02169299, "balance_loss_mlp": 1.03271842, "epoch": 0.11657898692319255, "flos": 22888568989440.0, "grad_norm": 2.4333915561606583, "language_loss": 0.79423189, "learning_rate": 3.867589283679779e-06, "loss": 0.81592792, "num_input_tokens_seen": 42024995, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.91015625, "step": 1939, "time_per_iteration": 2.3905863761901855 }, { "auxiliary_loss_clip": 0.01120133, "auxiliary_loss_mlp": 0.01053959, "balance_loss_clip": 1.02989054, "balance_loss_mlp": 1.03123116, "epoch": 0.11663911017586051, "flos": 24314770828800.0, "grad_norm": 2.2356411317656377, "language_loss": 0.8636415, "learning_rate": 3.867454061975451e-06, "loss": 0.88538247, "num_input_tokens_seen": 42042640, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.88671875, "step": 1940, "time_per_iteration": 2.4451136589050293 }, { "auxiliary_loss_clip": 0.01118435, "auxiliary_loss_mlp": 0.01055251, "balance_loss_clip": 1.03233874, "balance_loss_mlp": 1.03401971, "epoch": 0.11669923342852849, "flos": 42338507794560.0, "grad_norm": 1.3780814995012212, "language_loss": 0.75742328, "learning_rate": 3.8673187736264914e-06, "loss": 0.77916014, "num_input_tokens_seen": 42067005, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.84375, "step": 1941, "time_per_iteration": 2.576957941055298 }, { "auxiliary_loss_clip": 0.01117836, "auxiliary_loss_mlp": 0.01047996, "balance_loss_clip": 1.02486932, "balance_loss_mlp": 1.0315516, "epoch": 0.11675935668119646, "flos": 14641560316800.0, "grad_norm": 2.077467512953499, "language_loss": 0.88486266, "learning_rate": 3.8671834186377275e-06, "loss": 0.90652096, "num_input_tokens_seen": 42082295, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.86328125, "step": 1942, "time_per_iteration": 2.3597772121429443 }, { "auxiliary_loss_clip": 0.01115714, "auxiliary_loss_mlp": 0.01043805, "balance_loss_clip": 1.02228785, "balance_loss_mlp": 1.03200805, "epoch": 0.11681947993386442, "flos": 35115012806400.0, "grad_norm": 1.6107241451107719, "language_loss": 0.68025339, "learning_rate": 3.867047997013991e-06, "loss": 0.70184863, "num_input_tokens_seen": 42105295, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.8359375, "step": 1943, "time_per_iteration": 2.5141799449920654 }, { "auxiliary_loss_clip": 0.01115098, "auxiliary_loss_mlp": 0.01037105, "balance_loss_clip": 1.01480138, "balance_loss_mlp": 1.03127599, "epoch": 0.11687960318653239, "flos": 38981713570560.0, "grad_norm": 3.075908187618805, "language_loss": 0.69172108, "learning_rate": 3.866912508760114e-06, "loss": 0.71324313, "num_input_tokens_seen": 42125520, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.8359375, "step": 1944, "time_per_iteration": 2.5394175052642822 }, { "auxiliary_loss_clip": 0.01116969, "auxiliary_loss_mlp": 0.01040783, "balance_loss_clip": 1.01936138, "balance_loss_mlp": 1.03080392, "epoch": 0.11693972643920036, "flos": 25993778889600.0, "grad_norm": 1.4261852213290416, "language_loss": 0.82534927, "learning_rate": 3.866776953880932e-06, "loss": 0.84692681, "num_input_tokens_seen": 42146335, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.86328125, "step": 1945, "time_per_iteration": 2.4165971279144287 }, { "auxiliary_loss_clip": 0.01115177, "auxiliary_loss_mlp": 0.01047136, "balance_loss_clip": 1.02493882, "balance_loss_mlp": 1.0304879, "epoch": 0.11699984969186833, "flos": 27270866845440.0, "grad_norm": 2.2188461994747657, "language_loss": 0.764691, "learning_rate": 3.8666413323812825e-06, "loss": 0.78631407, "num_input_tokens_seen": 42165320, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.84375, "step": 1946, "time_per_iteration": 2.4419751167297363 }, { "auxiliary_loss_clip": 0.01116491, "auxiliary_loss_mlp": 0.01048205, "balance_loss_clip": 1.02660465, "balance_loss_mlp": 1.03320909, "epoch": 0.1170599729445363, "flos": 15266959234560.0, "grad_norm": 1.852763228158811, "language_loss": 0.68523192, "learning_rate": 3.8665056442660055e-06, "loss": 0.7068789, "num_input_tokens_seen": 42182955, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.8359375, "step": 1947, "time_per_iteration": 2.3548645973205566 }, { "auxiliary_loss_clip": 0.01123669, "auxiliary_loss_mlp": 0.01047806, "balance_loss_clip": 1.02370167, "balance_loss_mlp": 1.03648901, "epoch": 0.11712009619720427, "flos": 17163511176960.0, "grad_norm": 2.2191004921610955, "language_loss": 0.84888136, "learning_rate": 3.866369889539942e-06, "loss": 0.87059611, "num_input_tokens_seen": 42200760, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.87109375, "step": 1948, "time_per_iteration": 2.3758552074432373 }, { "auxiliary_loss_clip": 0.01033309, "auxiliary_loss_mlp": 0.0101984, "balance_loss_clip": 1.01684785, "balance_loss_mlp": 1.0073961, "epoch": 0.11718021944987224, "flos": 70937644515840.0, "grad_norm": 0.8216537331260262, "language_loss": 0.65126908, "learning_rate": 3.86623406820794e-06, "loss": 0.67180055, "num_input_tokens_seen": 42265745, "router_z_loss_clip": 0.02990723, "router_z_loss_mlp": 0.25976562, "step": 1949, "time_per_iteration": 3.048769235610962 }, { "auxiliary_loss_clip": 0.01115862, "auxiliary_loss_mlp": 0.01048048, "balance_loss_clip": 1.02580369, "balance_loss_mlp": 1.03134167, "epoch": 0.1172403427025402, "flos": 27452240691840.0, "grad_norm": 1.6808494109986374, "language_loss": 0.71865463, "learning_rate": 3.8660981802748434e-06, "loss": 0.74029374, "num_input_tokens_seen": 42286245, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.84375, "step": 1950, "time_per_iteration": 2.432399034500122 }, { "auxiliary_loss_clip": 0.01123141, "auxiliary_loss_mlp": 0.01046762, "balance_loss_clip": 1.02400458, "balance_loss_mlp": 1.03402305, "epoch": 0.11730046595520818, "flos": 15667831998720.0, "grad_norm": 2.7210292700723393, "language_loss": 0.76711386, "learning_rate": 3.865962225745504e-06, "loss": 0.78881288, "num_input_tokens_seen": 42302710, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.890625, "step": 1951, "time_per_iteration": 3.768670082092285 }, { "auxiliary_loss_clip": 0.01120962, "auxiliary_loss_mlp": 0.0104838, "balance_loss_clip": 1.02534842, "balance_loss_mlp": 1.03474116, "epoch": 0.11736058920787615, "flos": 25628971426560.0, "grad_norm": 1.7319526892347994, "language_loss": 0.7685138, "learning_rate": 3.865826204624771e-06, "loss": 0.79020721, "num_input_tokens_seen": 42324115, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.86328125, "step": 1952, "time_per_iteration": 2.421802520751953 }, { "auxiliary_loss_clip": 0.01118202, "auxiliary_loss_mlp": 0.01047002, "balance_loss_clip": 1.02407813, "balance_loss_mlp": 1.03074563, "epoch": 0.11742071246054411, "flos": 21433214298240.0, "grad_norm": 1.7038080722742601, "language_loss": 0.71910661, "learning_rate": 3.865690116917501e-06, "loss": 0.74075866, "num_input_tokens_seen": 42342505, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.875, "step": 1953, "time_per_iteration": 3.782447338104248 }, { "auxiliary_loss_clip": 0.01124642, "auxiliary_loss_mlp": 0.0104304, "balance_loss_clip": 1.01981831, "balance_loss_mlp": 1.03440595, "epoch": 0.11748083571321208, "flos": 15996923274240.0, "grad_norm": 2.5555025588281386, "language_loss": 0.79637015, "learning_rate": 3.8655539626285505e-06, "loss": 0.81804705, "num_input_tokens_seen": 42360525, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.90234375, "step": 1954, "time_per_iteration": 3.778489351272583 }, { "auxiliary_loss_clip": 0.01119053, "auxiliary_loss_mlp": 0.01050862, "balance_loss_clip": 1.02672267, "balance_loss_mlp": 1.03155947, "epoch": 0.11754095896588006, "flos": 16179134993280.0, "grad_norm": 1.9247334416091033, "language_loss": 0.85399234, "learning_rate": 3.865417741762777e-06, "loss": 0.87569153, "num_input_tokens_seen": 42377045, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.875, "step": 1955, "time_per_iteration": 3.747711420059204 }, { "auxiliary_loss_clip": 0.01121077, "auxiliary_loss_mlp": 0.01046602, "balance_loss_clip": 1.02414286, "balance_loss_mlp": 1.03478241, "epoch": 0.11760108221854802, "flos": 13260745111680.0, "grad_norm": 2.2783008108075076, "language_loss": 0.77872068, "learning_rate": 3.865281454325043e-06, "loss": 0.80039746, "num_input_tokens_seen": 42393960, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.86328125, "step": 1956, "time_per_iteration": 2.368340492248535 }, { "auxiliary_loss_clip": 0.01116919, "auxiliary_loss_mlp": 0.01044495, "balance_loss_clip": 1.02103484, "balance_loss_mlp": 1.03262877, "epoch": 0.11766120547121599, "flos": 24497296750080.0, "grad_norm": 1.8959509243281567, "language_loss": 0.80642533, "learning_rate": 3.865145100320212e-06, "loss": 0.82803947, "num_input_tokens_seen": 42413160, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.84375, "step": 1957, "time_per_iteration": 2.4359662532806396 }, { "auxiliary_loss_clip": 0.01122664, "auxiliary_loss_mlp": 0.0103978, "balance_loss_clip": 1.01593804, "balance_loss_mlp": 1.03471184, "epoch": 0.11772132872388397, "flos": 17783079897600.0, "grad_norm": 3.407091579068367, "language_loss": 0.77597332, "learning_rate": 3.86500867975315e-06, "loss": 0.79759777, "num_input_tokens_seen": 42432590, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.87890625, "step": 1958, "time_per_iteration": 2.3869376182556152 }, { "auxiliary_loss_clip": 0.01118996, "auxiliary_loss_mlp": 0.01040341, "balance_loss_clip": 1.01607013, "balance_loss_mlp": 1.03330898, "epoch": 0.11778145197655193, "flos": 13216405818240.0, "grad_norm": 2.2072269949487886, "language_loss": 0.7668767, "learning_rate": 3.864872192628725e-06, "loss": 0.78847003, "num_input_tokens_seen": 42450135, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.85546875, "step": 1959, "time_per_iteration": 2.3841238021850586 }, { "auxiliary_loss_clip": 0.01122492, "auxiliary_loss_mlp": 0.01039689, "balance_loss_clip": 1.01795745, "balance_loss_mlp": 1.03416228, "epoch": 0.1178415752292199, "flos": 20229164640000.0, "grad_norm": 1.892203115995961, "language_loss": 0.69768929, "learning_rate": 3.864735638951809e-06, "loss": 0.71931112, "num_input_tokens_seen": 42470050, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.8828125, "step": 1960, "time_per_iteration": 2.4021859169006348 }, { "auxiliary_loss_clip": 0.01123859, "auxiliary_loss_mlp": 0.01043981, "balance_loss_clip": 1.02006721, "balance_loss_mlp": 1.03432, "epoch": 0.11790169848188788, "flos": 13039360980480.0, "grad_norm": 2.654020946733496, "language_loss": 0.81240052, "learning_rate": 3.864599018727275e-06, "loss": 0.83407891, "num_input_tokens_seen": 42484335, "router_z_loss_clip": 0.23925781, "router_z_loss_mlp": 0.8984375, "step": 1961, "time_per_iteration": 2.367949962615967 }, { "auxiliary_loss_clip": 0.0111437, "auxiliary_loss_mlp": 0.01047871, "balance_loss_clip": 1.02488744, "balance_loss_mlp": 1.03202939, "epoch": 0.11796182173455584, "flos": 22264845816960.0, "grad_norm": 2.19753021704276, "language_loss": 0.92440534, "learning_rate": 3.864462331959998e-06, "loss": 0.94602782, "num_input_tokens_seen": 42502720, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.82421875, "step": 1962, "time_per_iteration": 2.4064676761627197 }, { "auxiliary_loss_clip": 0.01122337, "auxiliary_loss_mlp": 0.01049053, "balance_loss_clip": 1.02627194, "balance_loss_mlp": 1.03362584, "epoch": 0.1180219449872238, "flos": 10634229129600.0, "grad_norm": 2.1702189567270125, "language_loss": 0.871997, "learning_rate": 3.864325578654856e-06, "loss": 0.89371091, "num_input_tokens_seen": 42519460, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.88671875, "step": 1963, "time_per_iteration": 2.4086060523986816 }, { "auxiliary_loss_clip": 0.01118009, "auxiliary_loss_mlp": 0.01043094, "balance_loss_clip": 1.02043271, "balance_loss_mlp": 1.03014529, "epoch": 0.11808206823989177, "flos": 20922469885440.0, "grad_norm": 2.0896722423300678, "language_loss": 0.83948267, "learning_rate": 3.864188758816731e-06, "loss": 0.8610937, "num_input_tokens_seen": 42539420, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.87890625, "step": 1964, "time_per_iteration": 2.3962087631225586 }, { "auxiliary_loss_clip": 0.01124048, "auxiliary_loss_mlp": 0.01046546, "balance_loss_clip": 1.02208459, "balance_loss_mlp": 1.03625286, "epoch": 0.11814219149255975, "flos": 20776707492480.0, "grad_norm": 2.1223552877057097, "language_loss": 0.82847214, "learning_rate": 3.864051872450504e-06, "loss": 0.85017812, "num_input_tokens_seen": 42558225, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.875, "step": 1965, "time_per_iteration": 2.375262975692749 }, { "auxiliary_loss_clip": 0.01120428, "auxiliary_loss_mlp": 0.01043547, "balance_loss_clip": 1.01982474, "balance_loss_mlp": 1.03300381, "epoch": 0.11820231474522772, "flos": 48758162572800.0, "grad_norm": 1.6467709144383305, "language_loss": 0.74588215, "learning_rate": 3.863914919561059e-06, "loss": 0.76752186, "num_input_tokens_seen": 42580790, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.875, "step": 1966, "time_per_iteration": 2.6221964359283447 }, { "auxiliary_loss_clip": 0.01129223, "auxiliary_loss_mlp": 0.01048573, "balance_loss_clip": 1.02477837, "balance_loss_mlp": 1.03779209, "epoch": 0.11826243799789568, "flos": 16689669937920.0, "grad_norm": 2.8886502735577246, "language_loss": 0.72988284, "learning_rate": 3.863777900153287e-06, "loss": 0.75166082, "num_input_tokens_seen": 42597355, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.9140625, "step": 1967, "time_per_iteration": 2.3466272354125977 }, { "auxiliary_loss_clip": 0.01121168, "auxiliary_loss_mlp": 0.01043189, "balance_loss_clip": 1.01883435, "balance_loss_mlp": 1.03336382, "epoch": 0.11832256125056366, "flos": 16908924476160.0, "grad_norm": 2.087425676331709, "language_loss": 0.88269222, "learning_rate": 3.863640814232076e-06, "loss": 0.90433586, "num_input_tokens_seen": 42616060, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.875, "step": 1968, "time_per_iteration": 2.3878138065338135 }, { "auxiliary_loss_clip": 0.01120177, "auxiliary_loss_mlp": 0.01045114, "balance_loss_clip": 1.02167749, "balance_loss_mlp": 1.03433907, "epoch": 0.11838268450323162, "flos": 22819301118720.0, "grad_norm": 2.373016368325097, "language_loss": 0.67450416, "learning_rate": 3.863503661802317e-06, "loss": 0.6961571, "num_input_tokens_seen": 42636285, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.859375, "step": 1969, "time_per_iteration": 2.3945834636688232 }, { "auxiliary_loss_clip": 0.01122843, "auxiliary_loss_mlp": 0.01044308, "balance_loss_clip": 1.02043056, "balance_loss_mlp": 1.0355742, "epoch": 0.11844280775589959, "flos": 33544479939840.0, "grad_norm": 2.5095524727065324, "language_loss": 0.80832243, "learning_rate": 3.863366442868906e-06, "loss": 0.82999396, "num_input_tokens_seen": 42658320, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.875, "step": 1970, "time_per_iteration": 2.4908838272094727 }, { "auxiliary_loss_clip": 0.01033128, "auxiliary_loss_mlp": 0.01007311, "balance_loss_clip": 1.00461662, "balance_loss_mlp": 1.0066843, "epoch": 0.11850293100856757, "flos": 66347577959040.0, "grad_norm": 0.8002912839407599, "language_loss": 0.66149813, "learning_rate": 3.863229157436741e-06, "loss": 0.68190253, "num_input_tokens_seen": 42721500, "router_z_loss_clip": 0.02697754, "router_z_loss_mlp": 0.26367188, "step": 1971, "time_per_iteration": 3.002826452255249 }, { "auxiliary_loss_clip": 0.01120792, "auxiliary_loss_mlp": 0.01038308, "balance_loss_clip": 1.01590836, "balance_loss_mlp": 1.03313684, "epoch": 0.11856305426123553, "flos": 24679892494080.0, "grad_norm": 2.2289507829910598, "language_loss": 0.7991339, "learning_rate": 3.863091805510718e-06, "loss": 0.82072496, "num_input_tokens_seen": 42739825, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.875, "step": 1972, "time_per_iteration": 2.423044204711914 }, { "auxiliary_loss_clip": 0.01118167, "auxiliary_loss_mlp": 0.01045811, "balance_loss_clip": 1.02244556, "balance_loss_mlp": 1.03223205, "epoch": 0.1186231775139035, "flos": 24278949907200.0, "grad_norm": 2.1791190440773556, "language_loss": 0.72848439, "learning_rate": 3.862954387095743e-06, "loss": 0.75012422, "num_input_tokens_seen": 42758695, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.859375, "step": 1973, "time_per_iteration": 2.4122936725616455 }, { "auxiliary_loss_clip": 0.01117423, "auxiliary_loss_mlp": 0.01040538, "balance_loss_clip": 1.01789987, "balance_loss_mlp": 1.03277802, "epoch": 0.11868330076657148, "flos": 21756475376640.0, "grad_norm": 1.721549687915635, "language_loss": 0.71981263, "learning_rate": 3.862816902196717e-06, "loss": 0.74139225, "num_input_tokens_seen": 42778510, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.84375, "step": 1974, "time_per_iteration": 2.415842056274414 }, { "auxiliary_loss_clip": 0.01120913, "auxiliary_loss_mlp": 0.01044245, "balance_loss_clip": 1.02083254, "balance_loss_mlp": 1.03460026, "epoch": 0.11874342401923944, "flos": 17192559294720.0, "grad_norm": 2.1320444490964077, "language_loss": 0.78171802, "learning_rate": 3.862679350818547e-06, "loss": 0.80336952, "num_input_tokens_seen": 42793995, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.86328125, "step": 1975, "time_per_iteration": 2.3707268238067627 }, { "auxiliary_loss_clip": 0.01121493, "auxiliary_loss_mlp": 0.01041877, "balance_loss_clip": 1.02053881, "balance_loss_mlp": 1.03448355, "epoch": 0.11880354727190741, "flos": 15228729429120.0, "grad_norm": 5.821912162635384, "language_loss": 0.75312293, "learning_rate": 3.862541732966144e-06, "loss": 0.77475655, "num_input_tokens_seen": 42809000, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.8671875, "step": 1976, "time_per_iteration": 2.361881971359253 }, { "auxiliary_loss_clip": 0.01117401, "auxiliary_loss_mlp": 0.01043903, "balance_loss_clip": 1.02064562, "balance_loss_mlp": 1.03225017, "epoch": 0.11886367052457537, "flos": 27308433335040.0, "grad_norm": 3.8382577631191896, "language_loss": 0.75069487, "learning_rate": 3.862404048644416e-06, "loss": 0.77230787, "num_input_tokens_seen": 42831585, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.8515625, "step": 1977, "time_per_iteration": 2.4624083042144775 }, { "auxiliary_loss_clip": 0.01122901, "auxiliary_loss_mlp": 0.010488, "balance_loss_clip": 1.02482712, "balance_loss_mlp": 1.03556919, "epoch": 0.11892379377724335, "flos": 21797218800000.0, "grad_norm": 2.156976507215717, "language_loss": 0.7394048, "learning_rate": 3.862266297858279e-06, "loss": 0.76112187, "num_input_tokens_seen": 42848420, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.87109375, "step": 1978, "time_per_iteration": 2.385887622833252 }, { "auxiliary_loss_clip": 0.011179, "auxiliary_loss_mlp": 0.01046777, "balance_loss_clip": 1.02485406, "balance_loss_mlp": 1.03311896, "epoch": 0.11898391702991132, "flos": 13990150569600.0, "grad_norm": 1.908963691018837, "language_loss": 0.73343402, "learning_rate": 3.862128480612648e-06, "loss": 0.75508082, "num_input_tokens_seen": 42866645, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.84765625, "step": 1979, "time_per_iteration": 2.3576278686523438 }, { "auxiliary_loss_clip": 0.01120054, "auxiliary_loss_mlp": 0.01044867, "balance_loss_clip": 1.02189517, "balance_loss_mlp": 1.03369176, "epoch": 0.11904404028257928, "flos": 32233176984960.0, "grad_norm": 1.6278011430777886, "language_loss": 0.9859215, "learning_rate": 3.8619905969124415e-06, "loss": 1.00757062, "num_input_tokens_seen": 42888515, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.86328125, "step": 1980, "time_per_iteration": 2.4863715171813965 }, { "auxiliary_loss_clip": 0.01123417, "auxiliary_loss_mlp": 0.01049478, "balance_loss_clip": 1.02595758, "balance_loss_mlp": 1.03414297, "epoch": 0.11910416353524726, "flos": 23585155902720.0, "grad_norm": 1.7044605200764433, "language_loss": 0.8611837, "learning_rate": 3.86185264676258e-06, "loss": 0.88291258, "num_input_tokens_seen": 42909035, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.890625, "step": 1981, "time_per_iteration": 2.4147064685821533 }, { "auxiliary_loss_clip": 0.01123441, "auxiliary_loss_mlp": 0.01047793, "balance_loss_clip": 1.02433228, "balance_loss_mlp": 1.03463411, "epoch": 0.11916428678791523, "flos": 25332000468480.0, "grad_norm": 1.8984009700727715, "language_loss": 0.85393345, "learning_rate": 3.861714630167987e-06, "loss": 0.87564576, "num_input_tokens_seen": 42927555, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.88671875, "step": 1982, "time_per_iteration": 2.4262924194335938 }, { "auxiliary_loss_clip": 0.01118164, "auxiliary_loss_mlp": 0.01041158, "balance_loss_clip": 1.01779306, "balance_loss_mlp": 1.03233421, "epoch": 0.11922441004058319, "flos": 19787513541120.0, "grad_norm": 2.4797697769246785, "language_loss": 0.85202748, "learning_rate": 3.8615765471335874e-06, "loss": 0.87362069, "num_input_tokens_seen": 42945300, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.859375, "step": 1983, "time_per_iteration": 2.377220392227173 }, { "auxiliary_loss_clip": 0.01124226, "auxiliary_loss_mlp": 0.01050751, "balance_loss_clip": 1.02544284, "balance_loss_mlp": 1.03365731, "epoch": 0.11928453329325117, "flos": 21535475270400.0, "grad_norm": 3.3113596146379733, "language_loss": 0.77033579, "learning_rate": 3.8614383976643096e-06, "loss": 0.79208553, "num_input_tokens_seen": 42961295, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.90625, "step": 1984, "time_per_iteration": 2.3807835578918457 }, { "auxiliary_loss_clip": 0.0111945, "auxiliary_loss_mlp": 0.01055403, "balance_loss_clip": 1.03197885, "balance_loss_mlp": 1.03279757, "epoch": 0.11934465654591914, "flos": 20813924868480.0, "grad_norm": 1.8278780443494753, "language_loss": 0.83421803, "learning_rate": 3.861300181765084e-06, "loss": 0.85596657, "num_input_tokens_seen": 42980330, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.8671875, "step": 1985, "time_per_iteration": 2.380126714706421 }, { "auxiliary_loss_clip": 0.01116393, "auxiliary_loss_mlp": 0.01042754, "balance_loss_clip": 1.02002048, "balance_loss_mlp": 1.03147042, "epoch": 0.1194047797985871, "flos": 19059539448960.0, "grad_norm": 2.061105441215783, "language_loss": 0.73861659, "learning_rate": 3.861161899440843e-06, "loss": 0.76020807, "num_input_tokens_seen": 42996125, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.8515625, "step": 1986, "time_per_iteration": 2.3536202907562256 }, { "auxiliary_loss_clip": 0.01121507, "auxiliary_loss_mlp": 0.01049547, "balance_loss_clip": 1.02557421, "balance_loss_mlp": 1.03355265, "epoch": 0.11946490305125507, "flos": 27189798935040.0, "grad_norm": 1.9365010102679958, "language_loss": 0.720505, "learning_rate": 3.86102355069652e-06, "loss": 0.74221563, "num_input_tokens_seen": 43014180, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.87890625, "step": 1987, "time_per_iteration": 2.439728021621704 }, { "auxiliary_loss_clip": 0.0112178, "auxiliary_loss_mlp": 0.0104697, "balance_loss_clip": 1.02298498, "balance_loss_mlp": 1.03436017, "epoch": 0.11952502630392305, "flos": 21139769387520.0, "grad_norm": 2.47030658028113, "language_loss": 0.71941423, "learning_rate": 3.860885135537054e-06, "loss": 0.74110174, "num_input_tokens_seen": 43032120, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.875, "step": 1988, "time_per_iteration": 2.406344413757324 }, { "auxiliary_loss_clip": 0.01119401, "auxiliary_loss_mlp": 0.01054096, "balance_loss_clip": 1.02750063, "balance_loss_mlp": 1.03266072, "epoch": 0.11958514955659101, "flos": 22123237875840.0, "grad_norm": 1.9140912637348366, "language_loss": 0.80716503, "learning_rate": 3.860746653967384e-06, "loss": 0.82889998, "num_input_tokens_seen": 43052215, "router_z_loss_clip": 0.265625, "router_z_loss_mlp": 0.8671875, "step": 1989, "time_per_iteration": 2.383209705352783 }, { "auxiliary_loss_clip": 0.01124657, "auxiliary_loss_mlp": 0.01047651, "balance_loss_clip": 1.02223611, "balance_loss_mlp": 1.03449106, "epoch": 0.11964527280925898, "flos": 17420471850240.0, "grad_norm": 2.7618199204925213, "language_loss": 0.75409639, "learning_rate": 3.860608105992454e-06, "loss": 0.77581948, "num_input_tokens_seen": 43069720, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.90234375, "step": 1990, "time_per_iteration": 3.7754063606262207 }, { "auxiliary_loss_clip": 0.01033002, "auxiliary_loss_mlp": 0.01003007, "balance_loss_clip": 0.99988371, "balance_loss_mlp": 1.00634062, "epoch": 0.11970539606192696, "flos": 70676564302080.0, "grad_norm": 0.846775636511847, "language_loss": 0.55253577, "learning_rate": 3.860469491617206e-06, "loss": 0.57289588, "num_input_tokens_seen": 43123130, "router_z_loss_clip": 0.03125, "router_z_loss_mlp": 0.265625, "step": 1991, "time_per_iteration": 3.015073537826538 }, { "auxiliary_loss_clip": 0.01118721, "auxiliary_loss_mlp": 0.01043284, "balance_loss_clip": 1.02001476, "balance_loss_mlp": 1.03399217, "epoch": 0.11976551931459492, "flos": 21213959760000.0, "grad_norm": 6.539764829323169, "language_loss": 0.78014505, "learning_rate": 3.8603308108465864e-06, "loss": 0.80176508, "num_input_tokens_seen": 43140015, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.84765625, "step": 1992, "time_per_iteration": 3.727733612060547 }, { "auxiliary_loss_clip": 0.01123085, "auxiliary_loss_mlp": 0.01045995, "balance_loss_clip": 1.02146149, "balance_loss_mlp": 1.03341937, "epoch": 0.11982564256726289, "flos": 25988262894720.0, "grad_norm": 1.732685606188902, "language_loss": 0.79110837, "learning_rate": 3.8601920636855466e-06, "loss": 0.8127991, "num_input_tokens_seen": 43160105, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.8984375, "step": 1993, "time_per_iteration": 3.8382067680358887 }, { "auxiliary_loss_clip": 0.01117257, "auxiliary_loss_mlp": 0.01047155, "balance_loss_clip": 1.02362347, "balance_loss_mlp": 1.03113675, "epoch": 0.11988576581993086, "flos": 21649850484480.0, "grad_norm": 1.8914864370530282, "language_loss": 0.82625687, "learning_rate": 3.860053250139036e-06, "loss": 0.84790105, "num_input_tokens_seen": 43179835, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.859375, "step": 1994, "time_per_iteration": 2.388251781463623 }, { "auxiliary_loss_clip": 0.01119523, "auxiliary_loss_mlp": 0.01044532, "balance_loss_clip": 1.02286029, "balance_loss_mlp": 1.03431726, "epoch": 0.11994588907259883, "flos": 17856467308800.0, "grad_norm": 2.0853703367348304, "language_loss": 0.88376117, "learning_rate": 3.859914370212011e-06, "loss": 0.90540171, "num_input_tokens_seen": 43197210, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.8515625, "step": 1995, "time_per_iteration": 3.7453725337982178 }, { "auxiliary_loss_clip": 0.01120729, "auxiliary_loss_mlp": 0.01053838, "balance_loss_clip": 1.02916145, "balance_loss_mlp": 1.03449523, "epoch": 0.1200060123252668, "flos": 24461580562560.0, "grad_norm": 1.9456351794724802, "language_loss": 0.7399205, "learning_rate": 3.859775423909426e-06, "loss": 0.76166618, "num_input_tokens_seen": 43215050, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.86328125, "step": 1996, "time_per_iteration": 2.4598515033721924 }, { "auxiliary_loss_clip": 0.01118924, "auxiliary_loss_mlp": 0.01044023, "balance_loss_clip": 1.01909614, "balance_loss_mlp": 1.03307962, "epoch": 0.12006613557793476, "flos": 18731251134720.0, "grad_norm": 2.0778181248459413, "language_loss": 0.87980461, "learning_rate": 3.8596364112362395e-06, "loss": 0.90143406, "num_input_tokens_seen": 43233900, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.859375, "step": 1997, "time_per_iteration": 2.352095127105713 }, { "auxiliary_loss_clip": 0.01118619, "auxiliary_loss_mlp": 0.01053211, "balance_loss_clip": 1.02774751, "balance_loss_mlp": 1.03132403, "epoch": 0.12012625883060274, "flos": 22266800853120.0, "grad_norm": 2.0035983354663993, "language_loss": 0.78402185, "learning_rate": 3.859497332197413e-06, "loss": 0.80574012, "num_input_tokens_seen": 43252105, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.875, "step": 1998, "time_per_iteration": 2.411452054977417 }, { "auxiliary_loss_clip": 0.01122502, "auxiliary_loss_mlp": 0.01045654, "balance_loss_clip": 1.02150226, "balance_loss_mlp": 1.03462815, "epoch": 0.1201863820832707, "flos": 21757906742400.0, "grad_norm": 1.6482861458018172, "language_loss": 0.73282808, "learning_rate": 3.8593581867979105e-06, "loss": 0.75450969, "num_input_tokens_seen": 43270315, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.87890625, "step": 1999, "time_per_iteration": 2.4018945693969727 }, { "auxiliary_loss_clip": 0.01121568, "auxiliary_loss_mlp": 0.01050488, "balance_loss_clip": 1.02662206, "balance_loss_mlp": 1.03275871, "epoch": 0.12024650533593867, "flos": 21906915891840.0, "grad_norm": 2.312608842513132, "language_loss": 0.74748641, "learning_rate": 3.8592189750426965e-06, "loss": 0.769207, "num_input_tokens_seen": 43289935, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.88671875, "step": 2000, "time_per_iteration": 2.414414167404175 }, { "auxiliary_loss_clip": 0.01120195, "auxiliary_loss_mlp": 0.01045413, "balance_loss_clip": 1.02091551, "balance_loss_mlp": 1.0318017, "epoch": 0.12030662858860665, "flos": 21688150112640.0, "grad_norm": 1.5722378616385124, "language_loss": 0.84657854, "learning_rate": 3.85907969693674e-06, "loss": 0.86823463, "num_input_tokens_seen": 43309325, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.8828125, "step": 2001, "time_per_iteration": 2.3905575275421143 }, { "auxiliary_loss_clip": 0.01118935, "auxiliary_loss_mlp": 0.01042232, "balance_loss_clip": 1.01928473, "balance_loss_mlp": 1.03210068, "epoch": 0.12036675184127461, "flos": 12932386974720.0, "grad_norm": 2.150666824424472, "language_loss": 0.74219608, "learning_rate": 3.858940352485011e-06, "loss": 0.76380777, "num_input_tokens_seen": 43327010, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.8671875, "step": 2002, "time_per_iteration": 2.3468594551086426 }, { "auxiliary_loss_clip": 0.01125384, "auxiliary_loss_mlp": 0.01048133, "balance_loss_clip": 1.02219284, "balance_loss_mlp": 1.03465629, "epoch": 0.12042687509394258, "flos": 20849955258240.0, "grad_norm": 2.2416977745271627, "language_loss": 0.77901542, "learning_rate": 3.8588009416924835e-06, "loss": 0.80075049, "num_input_tokens_seen": 43345650, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.90625, "step": 2003, "time_per_iteration": 2.3935604095458984 }, { "auxiliary_loss_clip": 0.01118354, "auxiliary_loss_mlp": 0.01045372, "balance_loss_clip": 1.02026618, "balance_loss_mlp": 1.03217614, "epoch": 0.12048699834661056, "flos": 23877378915840.0, "grad_norm": 2.3116049897435924, "language_loss": 0.72234046, "learning_rate": 3.858661464564131e-06, "loss": 0.74397773, "num_input_tokens_seen": 43365555, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.86328125, "step": 2004, "time_per_iteration": 2.4032437801361084 }, { "auxiliary_loss_clip": 0.01127669, "auxiliary_loss_mlp": 0.01051112, "balance_loss_clip": 1.02428949, "balance_loss_mlp": 1.03524661, "epoch": 0.12054712159927852, "flos": 19755323400960.0, "grad_norm": 1.6450597877325683, "language_loss": 0.78438574, "learning_rate": 3.858521921104932e-06, "loss": 0.80617362, "num_input_tokens_seen": 43384990, "router_z_loss_clip": 0.26757812, "router_z_loss_mlp": 0.921875, "step": 2005, "time_per_iteration": 2.4171624183654785 }, { "auxiliary_loss_clip": 0.01030416, "auxiliary_loss_mlp": 0.01006522, "balance_loss_clip": 1.00354171, "balance_loss_mlp": 1.00357127, "epoch": 0.12060724485194649, "flos": 51670057075200.0, "grad_norm": 0.9250736729463825, "language_loss": 0.58070427, "learning_rate": 3.858382311319866e-06, "loss": 0.60107362, "num_input_tokens_seen": 43436335, "router_z_loss_clip": 0.02978516, "router_z_loss_mlp": 0.26953125, "step": 2006, "time_per_iteration": 2.8138062953948975 }, { "auxiliary_loss_clip": 0.01120069, "auxiliary_loss_mlp": 0.01043962, "balance_loss_clip": 1.02029848, "balance_loss_mlp": 1.03448987, "epoch": 0.12066736810461445, "flos": 18989398794240.0, "grad_norm": 1.7877193335870534, "language_loss": 0.76783776, "learning_rate": 3.858242635213917e-06, "loss": 0.78947806, "num_input_tokens_seen": 43456495, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.85546875, "step": 2007, "time_per_iteration": 2.4160773754119873 }, { "auxiliary_loss_clip": 0.01121991, "auxiliary_loss_mlp": 0.0105572, "balance_loss_clip": 1.03088856, "balance_loss_mlp": 1.03364897, "epoch": 0.12072749135728243, "flos": 16471043804160.0, "grad_norm": 3.1240115634081933, "language_loss": 0.8271625, "learning_rate": 3.858102892792067e-06, "loss": 0.84893966, "num_input_tokens_seen": 43473085, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.8828125, "step": 2008, "time_per_iteration": 2.364374876022339 }, { "auxiliary_loss_clip": 0.01119849, "auxiliary_loss_mlp": 0.01046688, "balance_loss_clip": 1.02267885, "balance_loss_mlp": 1.03178072, "epoch": 0.1207876146099504, "flos": 18076140783360.0, "grad_norm": 2.175275464065516, "language_loss": 0.83321232, "learning_rate": 3.857963084059304e-06, "loss": 0.85487771, "num_input_tokens_seen": 43491135, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.8828125, "step": 2009, "time_per_iteration": 2.3697962760925293 }, { "auxiliary_loss_clip": 0.01123511, "auxiliary_loss_mlp": 0.01055106, "balance_loss_clip": 1.02812898, "balance_loss_mlp": 1.03362918, "epoch": 0.12084773786261836, "flos": 21870501477120.0, "grad_norm": 1.731425056093623, "language_loss": 0.84200156, "learning_rate": 3.857823209020619e-06, "loss": 0.86378777, "num_input_tokens_seen": 43510440, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.8984375, "step": 2010, "time_per_iteration": 2.382655382156372 }, { "auxiliary_loss_clip": 0.01125417, "auxiliary_loss_mlp": 0.01058759, "balance_loss_clip": 1.03335524, "balance_loss_mlp": 1.03640819, "epoch": 0.12090786111528634, "flos": 18332054115840.0, "grad_norm": 1.7003581849257905, "language_loss": 0.84254408, "learning_rate": 3.857683267681002e-06, "loss": 0.86438584, "num_input_tokens_seen": 43530145, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.890625, "step": 2011, "time_per_iteration": 2.4131879806518555 }, { "auxiliary_loss_clip": 0.01124281, "auxiliary_loss_mlp": 0.01052195, "balance_loss_clip": 1.02688694, "balance_loss_mlp": 1.03430021, "epoch": 0.1209679843679543, "flos": 21104786338560.0, "grad_norm": 1.8372059440576718, "language_loss": 0.95579314, "learning_rate": 3.857543260045448e-06, "loss": 0.9775579, "num_input_tokens_seen": 43549315, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.8984375, "step": 2012, "time_per_iteration": 2.3958005905151367 }, { "auxiliary_loss_clip": 0.01119601, "auxiliary_loss_mlp": 0.0104187, "balance_loss_clip": 1.01671648, "balance_loss_mlp": 1.03354287, "epoch": 0.12102810762062227, "flos": 29239793769600.0, "grad_norm": 2.677766300536327, "language_loss": 0.80141032, "learning_rate": 3.857403186118952e-06, "loss": 0.82302499, "num_input_tokens_seen": 43569240, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.859375, "step": 2013, "time_per_iteration": 2.4480528831481934 }, { "auxiliary_loss_clip": 0.01123554, "auxiliary_loss_mlp": 0.01051356, "balance_loss_clip": 1.02354503, "balance_loss_mlp": 1.03343034, "epoch": 0.12108823087329025, "flos": 17929749985920.0, "grad_norm": 2.5308944401753597, "language_loss": 0.77227497, "learning_rate": 3.857263045906516e-06, "loss": 0.79402405, "num_input_tokens_seen": 43587710, "router_z_loss_clip": 0.27734375, "router_z_loss_mlp": 0.8984375, "step": 2014, "time_per_iteration": 2.358426809310913 }, { "auxiliary_loss_clip": 0.01121764, "auxiliary_loss_mlp": 0.010419, "balance_loss_clip": 1.01609063, "balance_loss_mlp": 1.03438997, "epoch": 0.12114835412595822, "flos": 22090733533440.0, "grad_norm": 3.5941496769831156, "language_loss": 0.86573106, "learning_rate": 3.857122839413138e-06, "loss": 0.88736767, "num_input_tokens_seen": 43606000, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.875, "step": 2015, "time_per_iteration": 2.4904682636260986 }, { "auxiliary_loss_clip": 0.01116226, "auxiliary_loss_mlp": 0.0105065, "balance_loss_clip": 1.02637863, "balance_loss_mlp": 1.03096962, "epoch": 0.12120847737862618, "flos": 20411306536320.0, "grad_norm": 2.4791579545711127, "language_loss": 0.68878114, "learning_rate": 3.856982566643824e-06, "loss": 0.71044993, "num_input_tokens_seen": 43624815, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.8515625, "step": 2016, "time_per_iteration": 2.3910837173461914 }, { "auxiliary_loss_clip": 0.01125592, "auxiliary_loss_mlp": 0.0104802, "balance_loss_clip": 1.02233076, "balance_loss_mlp": 1.03680539, "epoch": 0.12126860063129415, "flos": 22307963212800.0, "grad_norm": 5.124671887293178, "language_loss": 0.80184972, "learning_rate": 3.856842227603578e-06, "loss": 0.82358587, "num_input_tokens_seen": 43643960, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.88671875, "step": 2017, "time_per_iteration": 2.395059585571289 }, { "auxiliary_loss_clip": 0.01122697, "auxiliary_loss_mlp": 0.01047158, "balance_loss_clip": 1.01989436, "balance_loss_mlp": 1.03370953, "epoch": 0.12132872388396213, "flos": 13698416315520.0, "grad_norm": 2.248275907000715, "language_loss": 0.68856907, "learning_rate": 3.856701822297409e-06, "loss": 0.71026766, "num_input_tokens_seen": 43662650, "router_z_loss_clip": 0.2734375, "router_z_loss_mlp": 0.890625, "step": 2018, "time_per_iteration": 2.3690297603607178 }, { "auxiliary_loss_clip": 0.01126354, "auxiliary_loss_mlp": 0.0104908, "balance_loss_clip": 1.02452278, "balance_loss_mlp": 1.03770387, "epoch": 0.12138884713663009, "flos": 26465804737920.0, "grad_norm": 1.794465734915278, "language_loss": 0.72320479, "learning_rate": 3.856561350730329e-06, "loss": 0.74495912, "num_input_tokens_seen": 43684205, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.88671875, "step": 2019, "time_per_iteration": 2.4442644119262695 }, { "auxiliary_loss_clip": 0.01121581, "auxiliary_loss_mlp": 0.01058645, "balance_loss_clip": 1.03263319, "balance_loss_mlp": 1.03215969, "epoch": 0.12144897038929806, "flos": 26140379155200.0, "grad_norm": 2.9071528755660077, "language_loss": 0.92150027, "learning_rate": 3.856420812907349e-06, "loss": 0.94330251, "num_input_tokens_seen": 43706320, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.89453125, "step": 2020, "time_per_iteration": 2.4346835613250732 }, { "auxiliary_loss_clip": 0.01122444, "auxiliary_loss_mlp": 0.01047896, "balance_loss_clip": 1.02292204, "balance_loss_mlp": 1.03476238, "epoch": 0.12150909364196603, "flos": 24716376731520.0, "grad_norm": 2.000040683910714, "language_loss": 0.7741518, "learning_rate": 3.856280208833486e-06, "loss": 0.79585522, "num_input_tokens_seen": 43724805, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.875, "step": 2021, "time_per_iteration": 2.417357921600342 }, { "auxiliary_loss_clip": 0.01120585, "auxiliary_loss_mlp": 0.01046029, "balance_loss_clip": 1.02217555, "balance_loss_mlp": 1.03421116, "epoch": 0.121569216894634, "flos": 25185958784640.0, "grad_norm": 2.02603328388594, "language_loss": 0.80683607, "learning_rate": 3.856139538513758e-06, "loss": 0.82850218, "num_input_tokens_seen": 43742320, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.86328125, "step": 2022, "time_per_iteration": 2.413191080093384 }, { "auxiliary_loss_clip": 0.01124483, "auxiliary_loss_mlp": 0.01053167, "balance_loss_clip": 1.02814507, "balance_loss_mlp": 1.03584802, "epoch": 0.12162934014730196, "flos": 13443236121600.0, "grad_norm": 1.7818911963633228, "language_loss": 0.85147119, "learning_rate": 3.855998801953183e-06, "loss": 0.87324774, "num_input_tokens_seen": 43760665, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.88671875, "step": 2023, "time_per_iteration": 2.382678985595703 }, { "auxiliary_loss_clip": 0.01120644, "auxiliary_loss_mlp": 0.01050331, "balance_loss_clip": 1.02470064, "balance_loss_mlp": 1.03305507, "epoch": 0.12168946339996994, "flos": 16945199245440.0, "grad_norm": 2.298953223732629, "language_loss": 0.85245049, "learning_rate": 3.855857999156786e-06, "loss": 0.87416029, "num_input_tokens_seen": 43779020, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.875, "step": 2024, "time_per_iteration": 2.3559911251068115 }, { "auxiliary_loss_clip": 0.01119247, "auxiliary_loss_mlp": 0.01047792, "balance_loss_clip": 1.02230477, "balance_loss_mlp": 1.03056741, "epoch": 0.12174958665263791, "flos": 29820399546240.0, "grad_norm": 2.6608086756553595, "language_loss": 0.71909428, "learning_rate": 3.85571713012959e-06, "loss": 0.74076468, "num_input_tokens_seen": 43798850, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.890625, "step": 2025, "time_per_iteration": 2.4525861740112305 }, { "auxiliary_loss_clip": 0.01122017, "auxiliary_loss_mlp": 0.01044699, "balance_loss_clip": 1.02079761, "balance_loss_mlp": 1.03384447, "epoch": 0.12180970990530587, "flos": 24640824816000.0, "grad_norm": 1.9378737122767655, "language_loss": 0.76372939, "learning_rate": 3.855576194876624e-06, "loss": 0.78539658, "num_input_tokens_seen": 43820130, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.8828125, "step": 2026, "time_per_iteration": 2.443085193634033 }, { "auxiliary_loss_clip": 0.01121749, "auxiliary_loss_mlp": 0.01048304, "balance_loss_clip": 1.02411687, "balance_loss_mlp": 1.03315139, "epoch": 0.12186983315797385, "flos": 20520654514560.0, "grad_norm": 2.396102644901252, "language_loss": 0.88871133, "learning_rate": 3.855435193402916e-06, "loss": 0.91041183, "num_input_tokens_seen": 43838485, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.88671875, "step": 2027, "time_per_iteration": 2.375792980194092 }, { "auxiliary_loss_clip": 0.01119273, "auxiliary_loss_mlp": 0.0104507, "balance_loss_clip": 1.02220595, "balance_loss_mlp": 1.03226328, "epoch": 0.12192995641064182, "flos": 27817117977600.0, "grad_norm": 1.5858442001493853, "language_loss": 0.7563501, "learning_rate": 3.8552941257135e-06, "loss": 0.77799356, "num_input_tokens_seen": 43859080, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8671875, "step": 2028, "time_per_iteration": 2.433760404586792 }, { "auxiliary_loss_clip": 0.0111943, "auxiliary_loss_mlp": 0.01051931, "balance_loss_clip": 1.02595568, "balance_loss_mlp": 1.03203964, "epoch": 0.12199007966330978, "flos": 22016054401920.0, "grad_norm": 2.2405902070927253, "language_loss": 0.7657541, "learning_rate": 3.855152991813408e-06, "loss": 0.78746778, "num_input_tokens_seen": 43879030, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.875, "step": 2029, "time_per_iteration": 3.77600359916687 }, { "auxiliary_loss_clip": 0.01118043, "auxiliary_loss_mlp": 0.0104532, "balance_loss_clip": 1.02150249, "balance_loss_mlp": 1.03121793, "epoch": 0.12205020291597775, "flos": 23294084964480.0, "grad_norm": 3.673993069551172, "language_loss": 0.7888177, "learning_rate": 3.855011791707678e-06, "loss": 0.81045127, "num_input_tokens_seen": 43898505, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.8671875, "step": 2030, "time_per_iteration": 2.3976783752441406 }, { "auxiliary_loss_clip": 0.01117256, "auxiliary_loss_mlp": 0.01049557, "balance_loss_clip": 1.0247736, "balance_loss_mlp": 1.03186178, "epoch": 0.12211032616864573, "flos": 26030402772480.0, "grad_norm": 2.072864758974824, "language_loss": 0.73834264, "learning_rate": 3.854870525401349e-06, "loss": 0.76001072, "num_input_tokens_seen": 43917945, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.8515625, "step": 2031, "time_per_iteration": 2.418802261352539 }, { "auxiliary_loss_clip": 0.01118716, "auxiliary_loss_mlp": 0.01046682, "balance_loss_clip": 1.02255392, "balance_loss_mlp": 1.03258789, "epoch": 0.12217044942131369, "flos": 20409944993280.0, "grad_norm": 4.745944251459415, "language_loss": 0.74995601, "learning_rate": 3.8547291928994615e-06, "loss": 0.77161002, "num_input_tokens_seen": 43937385, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.859375, "step": 2032, "time_per_iteration": 5.171973705291748 }, { "auxiliary_loss_clip": 0.01111557, "auxiliary_loss_mlp": 0.01039076, "balance_loss_clip": 1.01628351, "balance_loss_mlp": 1.02953041, "epoch": 0.12223057267398166, "flos": 22856029735680.0, "grad_norm": 1.6312332353564754, "language_loss": 0.89163828, "learning_rate": 3.8545877942070605e-06, "loss": 0.91314459, "num_input_tokens_seen": 43958130, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8203125, "step": 2033, "time_per_iteration": 2.4246609210968018 }, { "auxiliary_loss_clip": 0.01124556, "auxiliary_loss_mlp": 0.0104608, "balance_loss_clip": 1.02266693, "balance_loss_mlp": 1.03622246, "epoch": 0.12229069592664964, "flos": 20046533984640.0, "grad_norm": 1.9397284321498525, "language_loss": 0.65490395, "learning_rate": 3.8544463293291914e-06, "loss": 0.67661023, "num_input_tokens_seen": 43976800, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.8828125, "step": 2034, "time_per_iteration": 3.7740495204925537 }, { "auxiliary_loss_clip": 0.011211, "auxiliary_loss_mlp": 0.01050813, "balance_loss_clip": 1.02624369, "balance_loss_mlp": 1.03414273, "epoch": 0.1223508191793176, "flos": 22273119809280.0, "grad_norm": 2.121930359580294, "language_loss": 0.76366186, "learning_rate": 3.8543047982709035e-06, "loss": 0.78538096, "num_input_tokens_seen": 43996620, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.8671875, "step": 2035, "time_per_iteration": 2.409691333770752 }, { "auxiliary_loss_clip": 0.01122696, "auxiliary_loss_mlp": 0.01048104, "balance_loss_clip": 1.02359462, "balance_loss_mlp": 1.03383136, "epoch": 0.12241094243198557, "flos": 21284973198720.0, "grad_norm": 1.832738408115759, "language_loss": 0.71510398, "learning_rate": 3.854163201037247e-06, "loss": 0.73681188, "num_input_tokens_seen": 44016175, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.890625, "step": 2036, "time_per_iteration": 2.399597406387329 }, { "auxiliary_loss_clip": 0.01120529, "auxiliary_loss_mlp": 0.01052217, "balance_loss_clip": 1.02780282, "balance_loss_mlp": 1.03389144, "epoch": 0.12247106568465355, "flos": 17381473994880.0, "grad_norm": 1.7031477508166286, "language_loss": 0.83004296, "learning_rate": 3.854021537633275e-06, "loss": 0.8517704, "num_input_tokens_seen": 44035060, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.8671875, "step": 2037, "time_per_iteration": 2.3796517848968506 }, { "auxiliary_loss_clip": 0.01126316, "auxiliary_loss_mlp": 0.01045922, "balance_loss_clip": 1.0206852, "balance_loss_mlp": 1.03652191, "epoch": 0.12253118893732151, "flos": 27044420567040.0, "grad_norm": 3.2035945862169948, "language_loss": 0.79517901, "learning_rate": 3.853879808064044e-06, "loss": 0.81690133, "num_input_tokens_seen": 44053330, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.8984375, "step": 2038, "time_per_iteration": 2.4297780990600586 }, { "auxiliary_loss_clip": 0.01033306, "auxiliary_loss_mlp": 0.01017511, "balance_loss_clip": 1.01360095, "balance_loss_mlp": 1.00665402, "epoch": 0.12259131218998948, "flos": 53858762208000.0, "grad_norm": 0.8245276463797878, "language_loss": 0.58636552, "learning_rate": 3.8537380123346105e-06, "loss": 0.60687369, "num_input_tokens_seen": 44107575, "router_z_loss_clip": 0.0390625, "router_z_loss_mlp": 0.265625, "step": 2039, "time_per_iteration": 2.911100149154663 }, { "auxiliary_loss_clip": 0.0112325, "auxiliary_loss_mlp": 0.01049507, "balance_loss_clip": 1.02356696, "balance_loss_mlp": 1.03539979, "epoch": 0.12265143544265744, "flos": 17891031421440.0, "grad_norm": 3.018760004112527, "language_loss": 0.80326319, "learning_rate": 3.853596150450037e-06, "loss": 0.82499075, "num_input_tokens_seen": 44126075, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.87890625, "step": 2040, "time_per_iteration": 2.3550355434417725 }, { "auxiliary_loss_clip": 0.01116468, "auxiliary_loss_mlp": 0.01044236, "balance_loss_clip": 1.02149129, "balance_loss_mlp": 1.03211033, "epoch": 0.12271155869532542, "flos": 21798824722560.0, "grad_norm": 1.7984327608047004, "language_loss": 0.82874405, "learning_rate": 3.853454222415384e-06, "loss": 0.8503511, "num_input_tokens_seen": 44145605, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.84375, "step": 2041, "time_per_iteration": 2.387716054916382 }, { "auxiliary_loss_clip": 0.01122322, "auxiliary_loss_mlp": 0.01042929, "balance_loss_clip": 1.01605964, "balance_loss_mlp": 1.03212059, "epoch": 0.12277168194799339, "flos": 19827733294080.0, "grad_norm": 1.7672051121677157, "language_loss": 0.67215192, "learning_rate": 3.853312228235717e-06, "loss": 0.69380438, "num_input_tokens_seen": 44164770, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.90234375, "step": 2042, "time_per_iteration": 2.378965139389038 }, { "auxiliary_loss_clip": 0.01120555, "auxiliary_loss_mlp": 0.01055578, "balance_loss_clip": 1.03185534, "balance_loss_mlp": 1.0333581, "epoch": 0.12283180520066135, "flos": 23219929503360.0, "grad_norm": 1.8628373231360564, "language_loss": 0.81608152, "learning_rate": 3.853170167916106e-06, "loss": 0.83784282, "num_input_tokens_seen": 44184025, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.87109375, "step": 2043, "time_per_iteration": 2.3813624382019043 }, { "auxiliary_loss_clip": 0.0111882, "auxiliary_loss_mlp": 0.01047401, "balance_loss_clip": 1.02171087, "balance_loss_mlp": 1.03120828, "epoch": 0.12289192845332933, "flos": 18587478689280.0, "grad_norm": 1.907887097537687, "language_loss": 0.80282354, "learning_rate": 3.853028041461617e-06, "loss": 0.82448578, "num_input_tokens_seen": 44202950, "router_z_loss_clip": 0.2578125, "router_z_loss_mlp": 0.875, "step": 2044, "time_per_iteration": 2.379453420639038 }, { "auxiliary_loss_clip": 0.01118916, "auxiliary_loss_mlp": 0.01046854, "balance_loss_clip": 1.02279735, "balance_loss_mlp": 1.03514624, "epoch": 0.1229520517059973, "flos": 25768519597440.0, "grad_norm": 1.6706588656358827, "language_loss": 0.78307921, "learning_rate": 3.852885848877323e-06, "loss": 0.80473691, "num_input_tokens_seen": 44221115, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.8359375, "step": 2045, "time_per_iteration": 2.4202218055725098 }, { "auxiliary_loss_clip": 0.01124099, "auxiliary_loss_mlp": 0.0105316, "balance_loss_clip": 1.02633786, "balance_loss_mlp": 1.03491473, "epoch": 0.12301217495866526, "flos": 20886090382080.0, "grad_norm": 2.1187738228664235, "language_loss": 0.67233276, "learning_rate": 3.852743590168301e-06, "loss": 0.69410533, "num_input_tokens_seen": 44240575, "router_z_loss_clip": 0.26953125, "router_z_loss_mlp": 0.890625, "step": 2046, "time_per_iteration": 2.3931007385253906 }, { "auxiliary_loss_clip": 0.01118281, "auxiliary_loss_mlp": 0.01052155, "balance_loss_clip": 1.02762175, "balance_loss_mlp": 1.03496456, "epoch": 0.12307229821133324, "flos": 22377824576640.0, "grad_norm": 2.1682406600206012, "language_loss": 0.72872901, "learning_rate": 3.852601265339625e-06, "loss": 0.75043344, "num_input_tokens_seen": 44257145, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.83203125, "step": 2047, "time_per_iteration": 2.3804879188537598 }, { "auxiliary_loss_clip": 0.01118315, "auxiliary_loss_mlp": 0.01044566, "balance_loss_clip": 1.01955628, "balance_loss_mlp": 1.03371596, "epoch": 0.1231324214640012, "flos": 23366285389440.0, "grad_norm": 1.6043465109701915, "language_loss": 0.76935506, "learning_rate": 3.8524588743963755e-06, "loss": 0.7909838, "num_input_tokens_seen": 44278035, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.84375, "step": 2048, "time_per_iteration": 2.4538443088531494 }, { "auxiliary_loss_clip": 0.01120431, "auxiliary_loss_mlp": 0.01044889, "balance_loss_clip": 1.02201271, "balance_loss_mlp": 1.03345227, "epoch": 0.12319254471666917, "flos": 23766075901440.0, "grad_norm": 1.8330515449971934, "language_loss": 0.84730721, "learning_rate": 3.852316417343634e-06, "loss": 0.86896044, "num_input_tokens_seen": 44296980, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8671875, "step": 2049, "time_per_iteration": 2.402829885482788 }, { "auxiliary_loss_clip": 0.01118217, "auxiliary_loss_mlp": 0.01050867, "balance_loss_clip": 1.02539206, "balance_loss_mlp": 1.03130078, "epoch": 0.12325266796933713, "flos": 23549020778880.0, "grad_norm": 2.288036285795224, "language_loss": 0.75656784, "learning_rate": 3.852173894186484e-06, "loss": 0.77825868, "num_input_tokens_seen": 44318005, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.8671875, "step": 2050, "time_per_iteration": 2.4450416564941406 }, { "auxiliary_loss_clip": 0.01119152, "auxiliary_loss_mlp": 0.01044909, "balance_loss_clip": 1.02019715, "balance_loss_mlp": 1.03360677, "epoch": 0.12331279122200511, "flos": 24422896909440.0, "grad_norm": 2.177058190261101, "language_loss": 0.80784744, "learning_rate": 3.852031304930012e-06, "loss": 0.82948804, "num_input_tokens_seen": 44335260, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.8515625, "step": 2051, "time_per_iteration": 2.494203805923462 }, { "auxiliary_loss_clip": 0.01118979, "auxiliary_loss_mlp": 0.01048398, "balance_loss_clip": 1.02320886, "balance_loss_mlp": 1.0356648, "epoch": 0.12337291447467308, "flos": 25483104299520.0, "grad_norm": 1.7588340293504667, "language_loss": 0.80011177, "learning_rate": 3.851888649579307e-06, "loss": 0.82178557, "num_input_tokens_seen": 44355315, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.83203125, "step": 2052, "time_per_iteration": 2.4480302333831787 }, { "auxiliary_loss_clip": 0.01119589, "auxiliary_loss_mlp": 0.01049361, "balance_loss_clip": 1.02334929, "balance_loss_mlp": 1.03246593, "epoch": 0.12343303772734104, "flos": 23548881133440.0, "grad_norm": 2.062989969890706, "language_loss": 0.7362048, "learning_rate": 3.85174592813946e-06, "loss": 0.75789428, "num_input_tokens_seen": 44373020, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.87109375, "step": 2053, "time_per_iteration": 2.395052909851074 }, { "auxiliary_loss_clip": 0.01117386, "auxiliary_loss_mlp": 0.01044747, "balance_loss_clip": 1.02097631, "balance_loss_mlp": 1.02949238, "epoch": 0.12349316098000902, "flos": 47555299900800.0, "grad_norm": 1.7078781870035997, "language_loss": 0.74674809, "learning_rate": 3.851603140615564e-06, "loss": 0.76836938, "num_input_tokens_seen": 44397525, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.87890625, "step": 2054, "time_per_iteration": 2.6298487186431885 }, { "auxiliary_loss_clip": 0.0111469, "auxiliary_loss_mlp": 0.01037518, "balance_loss_clip": 1.01553583, "balance_loss_mlp": 1.02974188, "epoch": 0.12355328423267699, "flos": 25044804691200.0, "grad_norm": 2.2983846207671887, "language_loss": 0.84969324, "learning_rate": 3.851460287012714e-06, "loss": 0.87121534, "num_input_tokens_seen": 44415890, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.84765625, "step": 2055, "time_per_iteration": 2.4238319396972656 }, { "auxiliary_loss_clip": 0.01117869, "auxiliary_loss_mlp": 0.01047669, "balance_loss_clip": 1.02550793, "balance_loss_mlp": 1.03191376, "epoch": 0.12361340748534495, "flos": 27707909644800.0, "grad_norm": 2.378386507328866, "language_loss": 0.77205324, "learning_rate": 3.85131736733601e-06, "loss": 0.79370862, "num_input_tokens_seen": 44436625, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.859375, "step": 2056, "time_per_iteration": 2.454680919647217 }, { "auxiliary_loss_clip": 0.0111708, "auxiliary_loss_mlp": 0.01043758, "balance_loss_clip": 1.0191288, "balance_loss_mlp": 1.03185117, "epoch": 0.12367353073801293, "flos": 26139401637120.0, "grad_norm": 2.3364790556535238, "language_loss": 0.83136255, "learning_rate": 3.851174381590551e-06, "loss": 0.85297096, "num_input_tokens_seen": 44455265, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.8515625, "step": 2057, "time_per_iteration": 2.4133453369140625 }, { "auxiliary_loss_clip": 0.01122137, "auxiliary_loss_mlp": 0.0104733, "balance_loss_clip": 1.02342844, "balance_loss_mlp": 1.03349555, "epoch": 0.1237336539906809, "flos": 25154850896640.0, "grad_norm": 1.7560536613187636, "language_loss": 0.78054428, "learning_rate": 3.85103132978144e-06, "loss": 0.802239, "num_input_tokens_seen": 44475815, "router_z_loss_clip": 0.23925781, "router_z_loss_mlp": 0.88671875, "step": 2058, "time_per_iteration": 2.4437592029571533 }, { "auxiliary_loss_clip": 0.01117888, "auxiliary_loss_mlp": 0.0104837, "balance_loss_clip": 1.02399194, "balance_loss_mlp": 1.03020191, "epoch": 0.12379377724334886, "flos": 15303687851520.0, "grad_norm": 2.113776375453416, "language_loss": 0.83108556, "learning_rate": 3.850888211913782e-06, "loss": 0.85274816, "num_input_tokens_seen": 44494045, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.87890625, "step": 2059, "time_per_iteration": 2.3637967109680176 }, { "auxiliary_loss_clip": 0.01122157, "auxiliary_loss_mlp": 0.0105333, "balance_loss_clip": 1.02692533, "balance_loss_mlp": 1.03408313, "epoch": 0.12385390049601683, "flos": 21315871618560.0, "grad_norm": 2.294771563660056, "language_loss": 0.8141284, "learning_rate": 3.8507450279926856e-06, "loss": 0.83588326, "num_input_tokens_seen": 44509120, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.8828125, "step": 2060, "time_per_iteration": 2.4067494869232178 }, { "auxiliary_loss_clip": 0.01115806, "auxiliary_loss_mlp": 0.01048251, "balance_loss_clip": 1.02343154, "balance_loss_mlp": 1.03006387, "epoch": 0.1239140237486848, "flos": 15115576112640.0, "grad_norm": 2.2533304529817393, "language_loss": 0.85985982, "learning_rate": 3.850601778023259e-06, "loss": 0.88150042, "num_input_tokens_seen": 44525780, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.859375, "step": 2061, "time_per_iteration": 2.3487155437469482 }, { "auxiliary_loss_clip": 0.01117927, "auxiliary_loss_mlp": 0.01044717, "balance_loss_clip": 1.02045834, "balance_loss_mlp": 1.03339148, "epoch": 0.12397414700135277, "flos": 21975834648960.0, "grad_norm": 1.807106900031189, "language_loss": 0.84427786, "learning_rate": 3.850458462010615e-06, "loss": 0.86590421, "num_input_tokens_seen": 44543125, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.84375, "step": 2062, "time_per_iteration": 2.4029412269592285 }, { "auxiliary_loss_clip": 0.0111791, "auxiliary_loss_mlp": 0.01050326, "balance_loss_clip": 1.02610242, "balance_loss_mlp": 1.03355742, "epoch": 0.12403427025402074, "flos": 13400223459840.0, "grad_norm": 1.7333092179216898, "language_loss": 0.78806698, "learning_rate": 3.850315079959869e-06, "loss": 0.80974936, "num_input_tokens_seen": 44560275, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.84375, "step": 2063, "time_per_iteration": 2.377467393875122 }, { "auxiliary_loss_clip": 0.01117095, "auxiliary_loss_mlp": 0.01046062, "balance_loss_clip": 1.02002692, "balance_loss_mlp": 1.03245807, "epoch": 0.12409439350668872, "flos": 15303478383360.0, "grad_norm": 2.240657909027672, "language_loss": 0.79231298, "learning_rate": 3.850171631876137e-06, "loss": 0.81394458, "num_input_tokens_seen": 44577640, "router_z_loss_clip": 0.25976562, "router_z_loss_mlp": 0.84375, "step": 2064, "time_per_iteration": 2.3639748096466064 }, { "auxiliary_loss_clip": 0.01116078, "auxiliary_loss_mlp": 0.01052179, "balance_loss_clip": 1.02820563, "balance_loss_mlp": 1.03158522, "epoch": 0.12415451675935668, "flos": 25008215719680.0, "grad_norm": 3.4474827998857696, "language_loss": 0.92303932, "learning_rate": 3.850028117764539e-06, "loss": 0.94472188, "num_input_tokens_seen": 44594860, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.84375, "step": 2065, "time_per_iteration": 2.410625696182251 }, { "auxiliary_loss_clip": 0.01117973, "auxiliary_loss_mlp": 0.01043903, "balance_loss_clip": 1.01925051, "balance_loss_mlp": 1.03079259, "epoch": 0.12421464001202465, "flos": 23658543313920.0, "grad_norm": 1.8623053813568275, "language_loss": 0.80406475, "learning_rate": 3.849884537630196e-06, "loss": 0.82568353, "num_input_tokens_seen": 44614780, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.875, "step": 2066, "time_per_iteration": 2.4238030910491943 }, { "auxiliary_loss_clip": 0.01035428, "auxiliary_loss_mlp": 0.01008362, "balance_loss_clip": 1.00404668, "balance_loss_mlp": 1.00907445, "epoch": 0.12427476326469263, "flos": 65729440604160.0, "grad_norm": 0.8776658503758344, "language_loss": 0.63336056, "learning_rate": 3.849740891478233e-06, "loss": 0.65379852, "num_input_tokens_seen": 44671240, "router_z_loss_clip": 0.04321289, "router_z_loss_mlp": 0.26367188, "step": 2067, "time_per_iteration": 2.973073720932007 }, { "auxiliary_loss_clip": 0.01116555, "auxiliary_loss_mlp": 0.01042225, "balance_loss_clip": 1.01896691, "balance_loss_mlp": 1.03140807, "epoch": 0.12433488651736059, "flos": 24534269746560.0, "grad_norm": 2.794207783635333, "language_loss": 0.9301703, "learning_rate": 3.849597179313775e-06, "loss": 0.95175815, "num_input_tokens_seen": 44691050, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.8515625, "step": 2068, "time_per_iteration": 2.4296932220458984 }, { "auxiliary_loss_clip": 0.01119675, "auxiliary_loss_mlp": 0.01043389, "balance_loss_clip": 1.02181268, "balance_loss_mlp": 1.03442335, "epoch": 0.12439500977002856, "flos": 21030630877440.0, "grad_norm": 1.8576322137631927, "language_loss": 0.81259358, "learning_rate": 3.849453401141952e-06, "loss": 0.83422422, "num_input_tokens_seen": 44709850, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.8515625, "step": 2069, "time_per_iteration": 3.7766854763031006 }, { "auxiliary_loss_clip": 0.0111926, "auxiliary_loss_mlp": 0.0105096, "balance_loss_clip": 1.02696347, "balance_loss_mlp": 1.03305292, "epoch": 0.12445513302269653, "flos": 26829495037440.0, "grad_norm": 1.798662539204355, "language_loss": 0.77407026, "learning_rate": 3.8493095569678945e-06, "loss": 0.79577243, "num_input_tokens_seen": 44731475, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.86328125, "step": 2070, "time_per_iteration": 2.4453954696655273 }, { "auxiliary_loss_clip": 0.01119335, "auxiliary_loss_mlp": 0.01041812, "balance_loss_clip": 1.01684988, "balance_loss_mlp": 1.03424931, "epoch": 0.1245152562753645, "flos": 18367944860160.0, "grad_norm": 2.31678494920857, "language_loss": 0.8035953, "learning_rate": 3.849165646796735e-06, "loss": 0.82520676, "num_input_tokens_seen": 44749685, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.8515625, "step": 2071, "time_per_iteration": 2.3660426139831543 }, { "auxiliary_loss_clip": 0.01118492, "auxiliary_loss_mlp": 0.01049068, "balance_loss_clip": 1.02392673, "balance_loss_mlp": 1.03493595, "epoch": 0.12457537952803246, "flos": 33106634179200.0, "grad_norm": 1.6516694868819906, "language_loss": 0.7830193, "learning_rate": 3.849021670633611e-06, "loss": 0.80469489, "num_input_tokens_seen": 44772165, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.8359375, "step": 2072, "time_per_iteration": 5.2420947551727295 }, { "auxiliary_loss_clip": 0.01117737, "auxiliary_loss_mlp": 0.01051804, "balance_loss_clip": 1.02922606, "balance_loss_mlp": 1.03530848, "epoch": 0.12463550278070043, "flos": 22269209736960.0, "grad_norm": 2.4421081395701836, "language_loss": 0.74980325, "learning_rate": 3.8488776284836595e-06, "loss": 0.77149862, "num_input_tokens_seen": 44790580, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.8203125, "step": 2073, "time_per_iteration": 3.8428778648376465 }, { "auxiliary_loss_clip": 0.01116143, "auxiliary_loss_mlp": 0.01049777, "balance_loss_clip": 1.02605462, "balance_loss_mlp": 1.03189266, "epoch": 0.12469562603336841, "flos": 14678288933760.0, "grad_norm": 2.1570640646911725, "language_loss": 0.90657204, "learning_rate": 3.8487335203520215e-06, "loss": 0.92823124, "num_input_tokens_seen": 44806730, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.84375, "step": 2074, "time_per_iteration": 2.376634359359741 }, { "auxiliary_loss_clip": 0.01118504, "auxiliary_loss_mlp": 0.01048479, "balance_loss_clip": 1.0228374, "balance_loss_mlp": 1.03194141, "epoch": 0.12475574928603637, "flos": 24643617724800.0, "grad_norm": 2.37343051951324, "language_loss": 0.83716631, "learning_rate": 3.84858934624384e-06, "loss": 0.85883617, "num_input_tokens_seen": 44825550, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.86328125, "step": 2075, "time_per_iteration": 2.3994948863983154 }, { "auxiliary_loss_clip": 0.01117635, "auxiliary_loss_mlp": 0.01050142, "balance_loss_clip": 1.02473879, "balance_loss_mlp": 1.03250575, "epoch": 0.12481587253870434, "flos": 21761886637440.0, "grad_norm": 2.710930549668586, "language_loss": 0.73307014, "learning_rate": 3.8484451061642585e-06, "loss": 0.75474799, "num_input_tokens_seen": 44844155, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.8515625, "step": 2076, "time_per_iteration": 2.40032696723938 }, { "auxiliary_loss_clip": 0.01115359, "auxiliary_loss_mlp": 0.0104041, "balance_loss_clip": 1.01834464, "balance_loss_mlp": 1.03534937, "epoch": 0.12487599579137232, "flos": 21431503641600.0, "grad_norm": 1.7213159578200155, "language_loss": 0.75646508, "learning_rate": 3.8483008001184275e-06, "loss": 0.77802277, "num_input_tokens_seen": 44863780, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.80078125, "step": 2077, "time_per_iteration": 2.4050636291503906 }, { "auxiliary_loss_clip": 0.01114992, "auxiliary_loss_mlp": 0.0104251, "balance_loss_clip": 1.01846504, "balance_loss_mlp": 1.03148556, "epoch": 0.12493611904404028, "flos": 16106690188800.0, "grad_norm": 2.7697005372158348, "language_loss": 0.81957054, "learning_rate": 3.848156428111495e-06, "loss": 0.84114563, "num_input_tokens_seen": 44881480, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.8359375, "step": 2078, "time_per_iteration": 2.392759323120117 }, { "auxiliary_loss_clip": 0.01119904, "auxiliary_loss_mlp": 0.01045197, "balance_loss_clip": 1.02247524, "balance_loss_mlp": 1.0352838, "epoch": 0.12499624229670825, "flos": 21579186159360.0, "grad_norm": 1.691066522987815, "language_loss": 0.75001132, "learning_rate": 3.8480119901486135e-06, "loss": 0.77166235, "num_input_tokens_seen": 44900390, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.84765625, "step": 2079, "time_per_iteration": 2.3888022899627686 }, { "auxiliary_loss_clip": 0.01123983, "auxiliary_loss_mlp": 0.01051593, "balance_loss_clip": 1.02683294, "balance_loss_mlp": 1.03642082, "epoch": 0.1250563655493762, "flos": 25697960006400.0, "grad_norm": 2.096393689326478, "language_loss": 0.8320049, "learning_rate": 3.847867486234937e-06, "loss": 0.8537606, "num_input_tokens_seen": 44920375, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.875, "step": 2080, "time_per_iteration": 2.4352903366088867 }, { "auxiliary_loss_clip": 0.01116862, "auxiliary_loss_mlp": 0.01050311, "balance_loss_clip": 1.02584922, "balance_loss_mlp": 1.03381598, "epoch": 0.12511648880204418, "flos": 16908575362560.0, "grad_norm": 2.0247473194895234, "language_loss": 0.84366202, "learning_rate": 3.847722916375624e-06, "loss": 0.8653338, "num_input_tokens_seen": 44938415, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.828125, "step": 2081, "time_per_iteration": 2.3475122451782227 }, { "auxiliary_loss_clip": 0.01116416, "auxiliary_loss_mlp": 0.01041025, "balance_loss_clip": 1.01849461, "balance_loss_mlp": 1.03229547, "epoch": 0.12517661205471217, "flos": 17566513534080.0, "grad_norm": 1.7118357765139873, "language_loss": 0.76701432, "learning_rate": 3.847578280575832e-06, "loss": 0.78858876, "num_input_tokens_seen": 44957135, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.83984375, "step": 2082, "time_per_iteration": 2.3789234161376953 }, { "auxiliary_loss_clip": 0.01127754, "auxiliary_loss_mlp": 0.01042905, "balance_loss_clip": 1.01657128, "balance_loss_mlp": 1.03636873, "epoch": 0.12523673530738014, "flos": 16032883841280.0, "grad_norm": 2.459898948127579, "language_loss": 0.79046977, "learning_rate": 3.847433578840725e-06, "loss": 0.81217635, "num_input_tokens_seen": 44974480, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.9140625, "step": 2083, "time_per_iteration": 2.355137586593628 }, { "auxiliary_loss_clip": 0.01117899, "auxiliary_loss_mlp": 0.01042609, "balance_loss_clip": 1.0177182, "balance_loss_mlp": 1.0319339, "epoch": 0.1252968585600481, "flos": 18806733227520.0, "grad_norm": 3.531654342694931, "language_loss": 0.90191615, "learning_rate": 3.847288811175465e-06, "loss": 0.92352128, "num_input_tokens_seen": 44990310, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.859375, "step": 2084, "time_per_iteration": 2.3714373111724854 }, { "auxiliary_loss_clip": 0.01118012, "auxiliary_loss_mlp": 0.01041976, "balance_loss_clip": 1.01832533, "balance_loss_mlp": 1.03389311, "epoch": 0.12535698181271607, "flos": 27270343175040.0, "grad_norm": 1.9371363425061896, "language_loss": 0.7973994, "learning_rate": 3.84714397758522e-06, "loss": 0.81899923, "num_input_tokens_seen": 45010720, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.84375, "step": 2085, "time_per_iteration": 2.435650587081909 }, { "auxiliary_loss_clip": 0.01115806, "auxiliary_loss_mlp": 0.0104591, "balance_loss_clip": 1.0222826, "balance_loss_mlp": 1.0319972, "epoch": 0.12541710506538403, "flos": 22053027398400.0, "grad_norm": 1.941421465247455, "language_loss": 0.88167977, "learning_rate": 3.846999078075156e-06, "loss": 0.90329695, "num_input_tokens_seen": 45030360, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.8359375, "step": 2086, "time_per_iteration": 2.3734970092773438 }, { "auxiliary_loss_clip": 0.01112312, "auxiliary_loss_mlp": 0.01044926, "balance_loss_clip": 1.02293229, "balance_loss_mlp": 1.03155839, "epoch": 0.125477228318052, "flos": 12602388003840.0, "grad_norm": 2.0588020364240602, "language_loss": 0.87000966, "learning_rate": 3.8468541126504476e-06, "loss": 0.89158201, "num_input_tokens_seen": 45045085, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.80859375, "step": 2087, "time_per_iteration": 2.3449716567993164 }, { "auxiliary_loss_clip": 0.01116882, "auxiliary_loss_mlp": 0.01043699, "balance_loss_clip": 1.0192132, "balance_loss_mlp": 1.03264987, "epoch": 0.12553735157071996, "flos": 23877413827200.0, "grad_norm": 1.8887299433909066, "language_loss": 0.73232102, "learning_rate": 3.846709081316266e-06, "loss": 0.75392687, "num_input_tokens_seen": 45065145, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.84375, "step": 2088, "time_per_iteration": 2.399785280227661 }, { "auxiliary_loss_clip": 0.01038205, "auxiliary_loss_mlp": 0.01002933, "balance_loss_clip": 0.9987132, "balance_loss_mlp": 1.01034844, "epoch": 0.12559747482338796, "flos": 69917482321920.0, "grad_norm": 0.7564133700942366, "language_loss": 0.61721826, "learning_rate": 3.846563984077788e-06, "loss": 0.63762963, "num_input_tokens_seen": 45126230, "router_z_loss_clip": 0.04223633, "router_z_loss_mlp": 0.27734375, "step": 2089, "time_per_iteration": 3.0131125450134277 }, { "auxiliary_loss_clip": 0.01112258, "auxiliary_loss_mlp": 0.01043145, "balance_loss_clip": 1.02019763, "balance_loss_mlp": 1.03125, "epoch": 0.12565759807605592, "flos": 24278426236800.0, "grad_norm": 3.37125939911503, "language_loss": 0.77657014, "learning_rate": 3.846418820940191e-06, "loss": 0.79812419, "num_input_tokens_seen": 45145545, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.80859375, "step": 2090, "time_per_iteration": 2.400271415710449 }, { "auxiliary_loss_clip": 0.01036273, "auxiliary_loss_mlp": 0.01004513, "balance_loss_clip": 1.00050783, "balance_loss_mlp": 1.00848293, "epoch": 0.12571772132872389, "flos": 56448375016320.0, "grad_norm": 0.7524356481684067, "language_loss": 0.59461302, "learning_rate": 3.846273591908656e-06, "loss": 0.61502087, "num_input_tokens_seen": 45206845, "router_z_loss_clip": 0.04003906, "router_z_loss_mlp": 0.27734375, "step": 2091, "time_per_iteration": 2.971236228942871 }, { "auxiliary_loss_clip": 0.0111571, "auxiliary_loss_mlp": 0.01041371, "balance_loss_clip": 1.01947236, "balance_loss_mlp": 1.03398085, "epoch": 0.12577784458139185, "flos": 41244225050880.0, "grad_norm": 2.0347088973012024, "language_loss": 0.6315937, "learning_rate": 3.846128296988365e-06, "loss": 0.65316451, "num_input_tokens_seen": 45228495, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.81640625, "step": 2092, "time_per_iteration": 2.5666489601135254 }, { "auxiliary_loss_clip": 0.01119389, "auxiliary_loss_mlp": 0.0105116, "balance_loss_clip": 1.02620912, "balance_loss_mlp": 1.03349912, "epoch": 0.12583796783405982, "flos": 19754485528320.0, "grad_norm": 4.127986859516993, "language_loss": 0.80670291, "learning_rate": 3.845982936184505e-06, "loss": 0.82840842, "num_input_tokens_seen": 45245720, "router_z_loss_clip": 0.24902344, "router_z_loss_mlp": 0.859375, "step": 2093, "time_per_iteration": 2.372065305709839 }, { "auxiliary_loss_clip": 0.01117134, "auxiliary_loss_mlp": 0.01052396, "balance_loss_clip": 1.02943611, "balance_loss_mlp": 1.03341269, "epoch": 0.12589809108672778, "flos": 22600989187200.0, "grad_norm": 1.8013595234153057, "language_loss": 0.75998724, "learning_rate": 3.845837509502262e-06, "loss": 0.78168249, "num_input_tokens_seen": 45265650, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.8359375, "step": 2094, "time_per_iteration": 2.4176156520843506 }, { "auxiliary_loss_clip": 0.01112457, "auxiliary_loss_mlp": 0.01048578, "balance_loss_clip": 1.02617884, "balance_loss_mlp": 1.03079164, "epoch": 0.12595821433939577, "flos": 45221111665920.0, "grad_norm": 1.9134547374868065, "language_loss": 0.76899022, "learning_rate": 3.845692016946826e-06, "loss": 0.79060054, "num_input_tokens_seen": 45287790, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.81640625, "step": 2095, "time_per_iteration": 2.5881059169769287 }, { "auxiliary_loss_clip": 0.01116577, "auxiliary_loss_mlp": 0.01042894, "balance_loss_clip": 1.01988709, "balance_loss_mlp": 1.03235626, "epoch": 0.12601833759206374, "flos": 14318927642880.0, "grad_norm": 2.3136884654316052, "language_loss": 0.82832527, "learning_rate": 3.845546458523391e-06, "loss": 0.84991997, "num_input_tokens_seen": 45305720, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.84375, "step": 2096, "time_per_iteration": 2.38016676902771 }, { "auxiliary_loss_clip": 0.01114748, "auxiliary_loss_mlp": 0.01045874, "balance_loss_clip": 1.02244925, "balance_loss_mlp": 1.03167081, "epoch": 0.1260784608447317, "flos": 21287172614400.0, "grad_norm": 2.0683992218288885, "language_loss": 0.7564081, "learning_rate": 3.845400834237148e-06, "loss": 0.7780143, "num_input_tokens_seen": 45325290, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.83203125, "step": 2097, "time_per_iteration": 2.400761127471924 }, { "auxiliary_loss_clip": 0.01115307, "auxiliary_loss_mlp": 0.01049884, "balance_loss_clip": 1.0281167, "balance_loss_mlp": 1.0332402, "epoch": 0.12613858409739967, "flos": 26250076247040.0, "grad_norm": 3.3089693939595457, "language_loss": 0.8743059, "learning_rate": 3.8452551440932975e-06, "loss": 0.89595783, "num_input_tokens_seen": 45344465, "router_z_loss_clip": 0.21777344, "router_z_loss_mlp": 0.8203125, "step": 2098, "time_per_iteration": 2.4255878925323486 }, { "auxiliary_loss_clip": 0.01119626, "auxiliary_loss_mlp": 0.01057123, "balance_loss_clip": 1.03171968, "balance_loss_mlp": 1.03246355, "epoch": 0.12619870735006763, "flos": 21578906868480.0, "grad_norm": 1.9861322343369732, "language_loss": 0.69507301, "learning_rate": 3.8451093880970365e-06, "loss": 0.71684051, "num_input_tokens_seen": 45362465, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.87109375, "step": 2099, "time_per_iteration": 2.3840246200561523 }, { "auxiliary_loss_clip": 0.01116001, "auxiliary_loss_mlp": 0.0105122, "balance_loss_clip": 1.02680588, "balance_loss_mlp": 1.03224981, "epoch": 0.1262588306027356, "flos": 23365936275840.0, "grad_norm": 2.4534387303493603, "language_loss": 0.81588322, "learning_rate": 3.844963566253569e-06, "loss": 0.83755541, "num_input_tokens_seen": 45382700, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.83984375, "step": 2100, "time_per_iteration": 2.4223971366882324 }, { "auxiliary_loss_clip": 0.01119178, "auxiliary_loss_mlp": 0.01048629, "balance_loss_clip": 1.0248704, "balance_loss_mlp": 1.03249133, "epoch": 0.12631895385540357, "flos": 23948113063680.0, "grad_norm": 1.9546686395482318, "language_loss": 0.80489665, "learning_rate": 3.844817678568097e-06, "loss": 0.82657468, "num_input_tokens_seen": 45401005, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.8671875, "step": 2101, "time_per_iteration": 2.4068095684051514 }, { "auxiliary_loss_clip": 0.01033474, "auxiliary_loss_mlp": 0.01005857, "balance_loss_clip": 1.0021373, "balance_loss_mlp": 1.00711823, "epoch": 0.12637907710807156, "flos": 70278868471680.0, "grad_norm": 0.7011092592635109, "language_loss": 0.57050014, "learning_rate": 3.8446717250458275e-06, "loss": 0.59089339, "num_input_tokens_seen": 45466555, "router_z_loss_clip": 0.03710938, "router_z_loss_mlp": 0.26367188, "step": 2102, "time_per_iteration": 3.098021984100342 }, { "auxiliary_loss_clip": 0.01117671, "auxiliary_loss_mlp": 0.01044396, "balance_loss_clip": 1.02066135, "balance_loss_mlp": 1.03210449, "epoch": 0.12643920036073952, "flos": 18914126169600.0, "grad_norm": 2.1890385512611754, "language_loss": 0.93189907, "learning_rate": 3.844525705691969e-06, "loss": 0.9535197, "num_input_tokens_seen": 45485165, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.85546875, "step": 2103, "time_per_iteration": 2.3683717250823975 }, { "auxiliary_loss_clip": 0.01109523, "auxiliary_loss_mlp": 0.01034552, "balance_loss_clip": 1.01373827, "balance_loss_mlp": 1.03146935, "epoch": 0.1264993236134075, "flos": 27781227233280.0, "grad_norm": 2.0237792358659945, "language_loss": 0.77780366, "learning_rate": 3.844379620511733e-06, "loss": 0.7992444, "num_input_tokens_seen": 45504630, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.78125, "step": 2104, "time_per_iteration": 2.473353385925293 }, { "auxiliary_loss_clip": 0.011191, "auxiliary_loss_mlp": 0.01046612, "balance_loss_clip": 1.02412975, "balance_loss_mlp": 1.03558755, "epoch": 0.12655944686607545, "flos": 24753524284800.0, "grad_norm": 3.309421676063958, "language_loss": 0.81139278, "learning_rate": 3.844233469510333e-06, "loss": 0.83304989, "num_input_tokens_seen": 45524885, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.8359375, "step": 2105, "time_per_iteration": 2.4122915267944336 }, { "auxiliary_loss_clip": 0.01121015, "auxiliary_loss_mlp": 0.01043026, "balance_loss_clip": 1.0182066, "balance_loss_mlp": 1.03295863, "epoch": 0.12661957011874342, "flos": 24131930705280.0, "grad_norm": 2.622898736258822, "language_loss": 0.83117187, "learning_rate": 3.844087252692984e-06, "loss": 0.85281229, "num_input_tokens_seen": 45545000, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.8828125, "step": 2106, "time_per_iteration": 2.431649684906006 }, { "auxiliary_loss_clip": 0.01115561, "auxiliary_loss_mlp": 0.01042881, "balance_loss_clip": 1.01909912, "balance_loss_mlp": 1.03455234, "epoch": 0.12667969337141138, "flos": 24568519656960.0, "grad_norm": 1.8570276402480308, "language_loss": 0.7331838, "learning_rate": 3.843940970064904e-06, "loss": 0.75476825, "num_input_tokens_seen": 45564210, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.80859375, "step": 2107, "time_per_iteration": 2.404420852661133 }, { "auxiliary_loss_clip": 0.0111377, "auxiliary_loss_mlp": 0.01038722, "balance_loss_clip": 1.01615608, "balance_loss_mlp": 1.03279042, "epoch": 0.12673981662407935, "flos": 22960699591680.0, "grad_norm": 1.886671777421692, "language_loss": 0.78911781, "learning_rate": 3.843794621631314e-06, "loss": 0.81064278, "num_input_tokens_seen": 45583030, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.8125, "step": 2108, "time_per_iteration": 2.4051084518432617 }, { "auxiliary_loss_clip": 0.01114943, "auxiliary_loss_mlp": 0.01042829, "balance_loss_clip": 1.02022719, "balance_loss_mlp": 1.03060329, "epoch": 0.12679993987674734, "flos": 17273906496000.0, "grad_norm": 1.9965757902599248, "language_loss": 0.75592458, "learning_rate": 3.843648207397438e-06, "loss": 0.7775023, "num_input_tokens_seen": 45602265, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.84375, "step": 2109, "time_per_iteration": 3.766605854034424 }, { "auxiliary_loss_clip": 0.0111483, "auxiliary_loss_mlp": 0.01044969, "balance_loss_clip": 1.02223563, "balance_loss_mlp": 1.0322988, "epoch": 0.1268600631294153, "flos": 17274115964160.0, "grad_norm": 1.7273210348148718, "language_loss": 0.8307693, "learning_rate": 3.843501727368498e-06, "loss": 0.85236728, "num_input_tokens_seen": 45620595, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.82421875, "step": 2110, "time_per_iteration": 2.4035935401916504 }, { "auxiliary_loss_clip": 0.01112926, "auxiliary_loss_mlp": 0.01041728, "balance_loss_clip": 1.01898241, "balance_loss_mlp": 1.03165388, "epoch": 0.12692018638208327, "flos": 24059904837120.0, "grad_norm": 1.7158466888913007, "language_loss": 0.78610981, "learning_rate": 3.8433551815497255e-06, "loss": 0.80765629, "num_input_tokens_seen": 45641140, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.81640625, "step": 2111, "time_per_iteration": 5.266538381576538 }, { "auxiliary_loss_clip": 0.01123097, "auxiliary_loss_mlp": 0.0105393, "balance_loss_clip": 1.027632, "balance_loss_mlp": 1.03377807, "epoch": 0.12698030963475124, "flos": 31830558652800.0, "grad_norm": 2.513129431042387, "language_loss": 0.76426053, "learning_rate": 3.843208569946347e-06, "loss": 0.78603077, "num_input_tokens_seen": 45662315, "router_z_loss_clip": 0.26367188, "router_z_loss_mlp": 0.89453125, "step": 2112, "time_per_iteration": 2.4973158836364746 }, { "auxiliary_loss_clip": 0.01114453, "auxiliary_loss_mlp": 0.0105147, "balance_loss_clip": 1.0288676, "balance_loss_mlp": 1.03119397, "epoch": 0.1270404328874192, "flos": 25186691923200.0, "grad_norm": 1.7756504212212387, "language_loss": 0.85513252, "learning_rate": 3.843061892563596e-06, "loss": 0.87679178, "num_input_tokens_seen": 45680335, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.83203125, "step": 2113, "time_per_iteration": 3.8007261753082275 }, { "auxiliary_loss_clip": 0.0111456, "auxiliary_loss_mlp": 0.01046979, "balance_loss_clip": 1.02329206, "balance_loss_mlp": 1.03161609, "epoch": 0.12710055614008717, "flos": 15996434515200.0, "grad_norm": 2.1560867935703585, "language_loss": 0.73853689, "learning_rate": 3.842915149406707e-06, "loss": 0.76015228, "num_input_tokens_seen": 45696240, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.828125, "step": 2114, "time_per_iteration": 2.369175672531128 }, { "auxiliary_loss_clip": 0.01118727, "auxiliary_loss_mlp": 0.0104384, "balance_loss_clip": 1.02104712, "balance_loss_mlp": 1.03434312, "epoch": 0.12716067939275516, "flos": 15084747515520.0, "grad_norm": 1.9327626940509444, "language_loss": 0.83024955, "learning_rate": 3.842768340480917e-06, "loss": 0.85187531, "num_input_tokens_seen": 45713695, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.84375, "step": 2115, "time_per_iteration": 2.3644826412200928 }, { "auxiliary_loss_clip": 0.0111886, "auxiliary_loss_mlp": 0.01048808, "balance_loss_clip": 1.02586019, "balance_loss_mlp": 1.03423762, "epoch": 0.12722080264542313, "flos": 28365463791360.0, "grad_norm": 1.6253823017253595, "language_loss": 0.86538076, "learning_rate": 3.8426214657914656e-06, "loss": 0.88705742, "num_input_tokens_seen": 45736655, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.84375, "step": 2116, "time_per_iteration": 2.4714713096618652 }, { "auxiliary_loss_clip": 0.01114616, "auxiliary_loss_mlp": 0.01041541, "balance_loss_clip": 1.01851034, "balance_loss_mlp": 1.03221858, "epoch": 0.1272809258980911, "flos": 32378520441600.0, "grad_norm": 1.7531392172531437, "language_loss": 0.70339799, "learning_rate": 3.842474525343594e-06, "loss": 0.72495955, "num_input_tokens_seen": 45758195, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.82421875, "step": 2117, "time_per_iteration": 2.485377788543701 }, { "auxiliary_loss_clip": 0.01114204, "auxiliary_loss_mlp": 0.01045505, "balance_loss_clip": 1.02188969, "balance_loss_mlp": 1.03143668, "epoch": 0.12734104915075906, "flos": 16033477334400.0, "grad_norm": 1.9788775424089131, "language_loss": 0.86027038, "learning_rate": 3.842327519142545e-06, "loss": 0.88186753, "num_input_tokens_seen": 45774280, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.828125, "step": 2118, "time_per_iteration": 2.361762523651123 }, { "auxiliary_loss_clip": 0.01112817, "auxiliary_loss_mlp": 0.01045058, "balance_loss_clip": 1.02202654, "balance_loss_mlp": 1.03070664, "epoch": 0.12740117240342702, "flos": 18259330020480.0, "grad_norm": 2.068224738756466, "language_loss": 0.87385684, "learning_rate": 3.842180447193566e-06, "loss": 0.89543557, "num_input_tokens_seen": 45792760, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8203125, "step": 2119, "time_per_iteration": 2.36799693107605 }, { "auxiliary_loss_clip": 0.01116089, "auxiliary_loss_mlp": 0.01042747, "balance_loss_clip": 1.01886976, "balance_loss_mlp": 1.031811, "epoch": 0.127461295656095, "flos": 12121215379200.0, "grad_norm": 4.792706132267744, "language_loss": 0.87717366, "learning_rate": 3.842033309501905e-06, "loss": 0.89876205, "num_input_tokens_seen": 45804300, "router_z_loss_clip": 0.23925781, "router_z_loss_mlp": 0.84375, "step": 2120, "time_per_iteration": 2.314236640930176 }, { "auxiliary_loss_clip": 0.01111922, "auxiliary_loss_mlp": 0.01036893, "balance_loss_clip": 1.01433849, "balance_loss_mlp": 1.03037643, "epoch": 0.12752141890876295, "flos": 23147973457920.0, "grad_norm": 1.9133798996998994, "language_loss": 0.75380892, "learning_rate": 3.841886106072815e-06, "loss": 0.77529705, "num_input_tokens_seen": 45823780, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.8125, "step": 2121, "time_per_iteration": 2.3969669342041016 }, { "auxiliary_loss_clip": 0.01111711, "auxiliary_loss_mlp": 0.01044018, "balance_loss_clip": 1.02122521, "balance_loss_mlp": 1.03040743, "epoch": 0.12758154216143094, "flos": 21614937258240.0, "grad_norm": 2.479920683180096, "language_loss": 0.83177739, "learning_rate": 3.841738836911547e-06, "loss": 0.85333467, "num_input_tokens_seen": 45840495, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.8125, "step": 2122, "time_per_iteration": 2.3829691410064697 }, { "auxiliary_loss_clip": 0.01115723, "auxiliary_loss_mlp": 0.01043598, "balance_loss_clip": 1.02047181, "balance_loss_mlp": 1.03254235, "epoch": 0.1276416654140989, "flos": 15923954799360.0, "grad_norm": 4.716823850097833, "language_loss": 0.79111636, "learning_rate": 3.8415915020233574e-06, "loss": 0.81270957, "num_input_tokens_seen": 45857735, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.83203125, "step": 2123, "time_per_iteration": 2.3637309074401855 }, { "auxiliary_loss_clip": 0.01115774, "auxiliary_loss_mlp": 0.01051057, "balance_loss_clip": 1.02757287, "balance_loss_mlp": 1.03227031, "epoch": 0.12770178866676687, "flos": 22381595003520.0, "grad_norm": 1.6150056217743856, "language_loss": 0.78939128, "learning_rate": 3.8414441014135045e-06, "loss": 0.81105959, "num_input_tokens_seen": 45876485, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.8359375, "step": 2124, "time_per_iteration": 2.377854108810425 }, { "auxiliary_loss_clip": 0.01115417, "auxiliary_loss_mlp": 0.01044485, "balance_loss_clip": 1.02244282, "balance_loss_mlp": 1.02979326, "epoch": 0.12776191191943484, "flos": 21651421495680.0, "grad_norm": 2.004458347345125, "language_loss": 0.75415641, "learning_rate": 3.8412966350872475e-06, "loss": 0.77575541, "num_input_tokens_seen": 45894645, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.85546875, "step": 2125, "time_per_iteration": 2.3942389488220215 }, { "auxiliary_loss_clip": 0.01112755, "auxiliary_loss_mlp": 0.01042692, "balance_loss_clip": 1.02105582, "balance_loss_mlp": 1.03033376, "epoch": 0.1278220351721028, "flos": 25734479155200.0, "grad_norm": 2.3313958258670318, "language_loss": 0.77859557, "learning_rate": 3.841149103049851e-06, "loss": 0.80015004, "num_input_tokens_seen": 45913755, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.82421875, "step": 2126, "time_per_iteration": 2.4090161323547363 }, { "auxiliary_loss_clip": 0.01112666, "auxiliary_loss_mlp": 0.01045773, "balance_loss_clip": 1.0235647, "balance_loss_mlp": 1.03223729, "epoch": 0.12788215842477077, "flos": 41241676521600.0, "grad_norm": 1.5875149616149478, "language_loss": 0.69364333, "learning_rate": 3.8410015053065785e-06, "loss": 0.71522772, "num_input_tokens_seen": 45936095, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.8046875, "step": 2127, "time_per_iteration": 2.5647130012512207 }, { "auxiliary_loss_clip": 0.01032074, "auxiliary_loss_mlp": 0.01005287, "balance_loss_clip": 1.00142455, "balance_loss_mlp": 1.00704312, "epoch": 0.12794228167743876, "flos": 70873822817280.0, "grad_norm": 0.8476597553826426, "language_loss": 0.62836862, "learning_rate": 3.8408538418626985e-06, "loss": 0.6487422, "num_input_tokens_seen": 46004655, "router_z_loss_clip": 0.03857422, "router_z_loss_mlp": 0.25, "step": 2128, "time_per_iteration": 3.063953399658203 }, { "auxiliary_loss_clip": 0.01113648, "auxiliary_loss_mlp": 0.01036754, "balance_loss_clip": 1.01316237, "balance_loss_mlp": 1.03077292, "epoch": 0.12800240493010673, "flos": 16288797173760.0, "grad_norm": 2.753633175846378, "language_loss": 0.77115464, "learning_rate": 3.840706112723479e-06, "loss": 0.79265857, "num_input_tokens_seen": 46023610, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.828125, "step": 2129, "time_per_iteration": 2.3833889961242676 }, { "auxiliary_loss_clip": 0.0112229, "auxiliary_loss_mlp": 0.01046874, "balance_loss_clip": 1.02174425, "balance_loss_mlp": 1.03358889, "epoch": 0.1280625281827747, "flos": 20630491251840.0, "grad_norm": 2.0155953904423067, "language_loss": 0.79008496, "learning_rate": 3.840558317894194e-06, "loss": 0.81177664, "num_input_tokens_seen": 46041725, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.88671875, "step": 2130, "time_per_iteration": 2.378575325012207 }, { "auxiliary_loss_clip": 0.01115906, "auxiliary_loss_mlp": 0.01041164, "balance_loss_clip": 1.01815629, "balance_loss_mlp": 1.03099012, "epoch": 0.12812265143544266, "flos": 22637124311040.0, "grad_norm": 2.2626543637482257, "language_loss": 0.70868599, "learning_rate": 3.840410457380117e-06, "loss": 0.73025668, "num_input_tokens_seen": 46061095, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.8515625, "step": 2131, "time_per_iteration": 2.4017488956451416 }, { "auxiliary_loss_clip": 0.01116169, "auxiliary_loss_mlp": 0.01041838, "balance_loss_clip": 1.01891422, "balance_loss_mlp": 1.03228021, "epoch": 0.12818277468811062, "flos": 34713267258240.0, "grad_norm": 2.4095980582332497, "language_loss": 0.72449213, "learning_rate": 3.840262531186525e-06, "loss": 0.74607217, "num_input_tokens_seen": 46082670, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.83984375, "step": 2132, "time_per_iteration": 2.497260332107544 }, { "auxiliary_loss_clip": 0.01112032, "auxiliary_loss_mlp": 0.01044994, "balance_loss_clip": 1.02172446, "balance_loss_mlp": 1.03138447, "epoch": 0.1282428979407786, "flos": 23111000461440.0, "grad_norm": 2.3302926949069236, "language_loss": 0.82523346, "learning_rate": 3.840114539318697e-06, "loss": 0.84680378, "num_input_tokens_seen": 46102410, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.80859375, "step": 2133, "time_per_iteration": 2.3946878910064697 }, { "auxiliary_loss_clip": 0.01120332, "auxiliary_loss_mlp": 0.01052204, "balance_loss_clip": 1.02770627, "balance_loss_mlp": 1.03165603, "epoch": 0.12830302119344655, "flos": 20885461977600.0, "grad_norm": 2.325376780580096, "language_loss": 0.79481399, "learning_rate": 3.839966481781914e-06, "loss": 0.81653935, "num_input_tokens_seen": 46121145, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.890625, "step": 2134, "time_per_iteration": 2.3721272945404053 }, { "auxiliary_loss_clip": 0.01116204, "auxiliary_loss_mlp": 0.01045534, "balance_loss_clip": 1.02231169, "balance_loss_mlp": 1.03271592, "epoch": 0.12836314444611455, "flos": 21396695149440.0, "grad_norm": 1.9570237272098825, "language_loss": 0.82733893, "learning_rate": 3.83981835858146e-06, "loss": 0.84895641, "num_input_tokens_seen": 46140740, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.8359375, "step": 2135, "time_per_iteration": 2.391282081604004 }, { "auxiliary_loss_clip": 0.01112881, "auxiliary_loss_mlp": 0.01050256, "balance_loss_clip": 1.02685499, "balance_loss_mlp": 1.03148592, "epoch": 0.1284232676987825, "flos": 13661617875840.0, "grad_norm": 2.3054997403547475, "language_loss": 0.77193314, "learning_rate": 3.839670169722622e-06, "loss": 0.79356444, "num_input_tokens_seen": 46156805, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.8125, "step": 2136, "time_per_iteration": 2.342874050140381 }, { "auxiliary_loss_clip": 0.01033939, "auxiliary_loss_mlp": 0.01001595, "balance_loss_clip": 0.99828076, "balance_loss_mlp": 1.00933623, "epoch": 0.12848339095145048, "flos": 59991709968000.0, "grad_norm": 0.891959578830437, "language_loss": 0.59144431, "learning_rate": 3.839521915210688e-06, "loss": 0.6117996, "num_input_tokens_seen": 46222085, "router_z_loss_clip": 0.03320312, "router_z_loss_mlp": 0.24609375, "step": 2137, "time_per_iteration": 3.1718711853027344 }, { "auxiliary_loss_clip": 0.01112519, "auxiliary_loss_mlp": 0.01044892, "balance_loss_clip": 1.02251649, "balance_loss_mlp": 1.02958333, "epoch": 0.12854351420411844, "flos": 13880523300480.0, "grad_norm": 3.2122022524145843, "language_loss": 0.82409132, "learning_rate": 3.839373595050948e-06, "loss": 0.84566545, "num_input_tokens_seen": 46239970, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.828125, "step": 2138, "time_per_iteration": 2.3435263633728027 }, { "auxiliary_loss_clip": 0.01116734, "auxiliary_loss_mlp": 0.01046684, "balance_loss_clip": 1.02316427, "balance_loss_mlp": 1.03054476, "epoch": 0.1286036374567864, "flos": 22636845020160.0, "grad_norm": 2.599591190734799, "language_loss": 0.78714335, "learning_rate": 3.839225209248696e-06, "loss": 0.80877751, "num_input_tokens_seen": 46257740, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.859375, "step": 2139, "time_per_iteration": 2.3830668926239014 }, { "auxiliary_loss_clip": 0.0111658, "auxiliary_loss_mlp": 0.01042747, "balance_loss_clip": 1.02007365, "balance_loss_mlp": 1.03078604, "epoch": 0.12866376070945437, "flos": 16323884956800.0, "grad_norm": 2.1733964329112556, "language_loss": 0.85316467, "learning_rate": 3.839076757809228e-06, "loss": 0.87475795, "num_input_tokens_seen": 46275445, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.859375, "step": 2140, "time_per_iteration": 2.3494696617126465 }, { "auxiliary_loss_clip": 0.01112102, "auxiliary_loss_mlp": 0.01040162, "balance_loss_clip": 1.01962209, "balance_loss_mlp": 1.02963066, "epoch": 0.12872388396212234, "flos": 11873750595840.0, "grad_norm": 2.4315846790103257, "language_loss": 0.85440862, "learning_rate": 3.83892824073784e-06, "loss": 0.87593126, "num_input_tokens_seen": 46291710, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.82421875, "step": 2141, "time_per_iteration": 2.3729195594787598 }, { "auxiliary_loss_clip": 0.01118117, "auxiliary_loss_mlp": 0.01043187, "balance_loss_clip": 1.01964295, "balance_loss_mlp": 1.03118086, "epoch": 0.12878400721479033, "flos": 28365428880000.0, "grad_norm": 2.0216933033230786, "language_loss": 0.6776073, "learning_rate": 3.838779658039834e-06, "loss": 0.69922036, "num_input_tokens_seen": 46311335, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.87109375, "step": 2142, "time_per_iteration": 2.4438652992248535 }, { "auxiliary_loss_clip": 0.01119154, "auxiliary_loss_mlp": 0.01039734, "balance_loss_clip": 1.01579714, "balance_loss_mlp": 1.03259099, "epoch": 0.1288441304674583, "flos": 25884430911360.0, "grad_norm": 1.9742993299275668, "language_loss": 0.83022559, "learning_rate": 3.838631009720513e-06, "loss": 0.85181445, "num_input_tokens_seen": 46330985, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.8671875, "step": 2143, "time_per_iteration": 2.43601131439209 }, { "auxiliary_loss_clip": 0.01118479, "auxiliary_loss_mlp": 0.01048113, "balance_loss_clip": 1.0251298, "balance_loss_mlp": 1.03445256, "epoch": 0.12890425372012626, "flos": 20812737882240.0, "grad_norm": 1.8165525936827422, "language_loss": 0.81771183, "learning_rate": 3.83848229578518e-06, "loss": 0.83937776, "num_input_tokens_seen": 46351295, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.83984375, "step": 2144, "time_per_iteration": 2.4065892696380615 }, { "auxiliary_loss_clip": 0.01114425, "auxiliary_loss_mlp": 0.01048728, "balance_loss_clip": 1.02498186, "balance_loss_mlp": 1.02971935, "epoch": 0.12896437697279423, "flos": 22564749329280.0, "grad_norm": 2.18530696944292, "language_loss": 0.78207135, "learning_rate": 3.838333516239142e-06, "loss": 0.80370283, "num_input_tokens_seen": 46368600, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.84765625, "step": 2145, "time_per_iteration": 2.3868374824523926 }, { "auxiliary_loss_clip": 0.01117629, "auxiliary_loss_mlp": 0.01048487, "balance_loss_clip": 1.02367997, "balance_loss_mlp": 1.03120184, "epoch": 0.1290245002254622, "flos": 17492811920640.0, "grad_norm": 2.506205522025062, "language_loss": 0.82470876, "learning_rate": 3.83818467108771e-06, "loss": 0.84636986, "num_input_tokens_seen": 46387370, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.8671875, "step": 2146, "time_per_iteration": 2.360330820083618 }, { "auxiliary_loss_clip": 0.01117362, "auxiliary_loss_mlp": 0.01044412, "balance_loss_clip": 1.02117813, "balance_loss_mlp": 1.03334451, "epoch": 0.12908462347813016, "flos": 36314593810560.0, "grad_norm": 3.324351171390452, "language_loss": 0.71070415, "learning_rate": 3.838035760336196e-06, "loss": 0.73232186, "num_input_tokens_seen": 46409570, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.83984375, "step": 2147, "time_per_iteration": 2.5244247913360596 }, { "auxiliary_loss_clip": 0.01115186, "auxiliary_loss_mlp": 0.01044679, "balance_loss_clip": 1.02238691, "balance_loss_mlp": 1.03071284, "epoch": 0.12914474673079815, "flos": 22527601776000.0, "grad_norm": 2.3566456445030193, "language_loss": 0.71779263, "learning_rate": 3.837886783989914e-06, "loss": 0.73939127, "num_input_tokens_seen": 46429320, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.84375, "step": 2148, "time_per_iteration": 3.7509665489196777 }, { "auxiliary_loss_clip": 0.01112953, "auxiliary_loss_mlp": 0.01037325, "balance_loss_clip": 1.01605821, "balance_loss_mlp": 1.03363156, "epoch": 0.12920486998346611, "flos": 21470780787840.0, "grad_norm": 1.4622631739764205, "language_loss": 0.78898561, "learning_rate": 3.837737742054179e-06, "loss": 0.81048834, "num_input_tokens_seen": 46450155, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.79296875, "step": 2149, "time_per_iteration": 2.397650718688965 }, { "auxiliary_loss_clip": 0.0111474, "auxiliary_loss_mlp": 0.01038862, "balance_loss_clip": 1.01599813, "balance_loss_mlp": 1.0322175, "epoch": 0.12926499323613408, "flos": 27307316171520.0, "grad_norm": 1.9198554036163238, "language_loss": 0.76388699, "learning_rate": 3.837588634534312e-06, "loss": 0.78542304, "num_input_tokens_seen": 46470280, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.82421875, "step": 2150, "time_per_iteration": 3.8543319702148438 }, { "auxiliary_loss_clip": 0.01115536, "auxiliary_loss_mlp": 0.01046366, "balance_loss_clip": 1.0236088, "balance_loss_mlp": 1.03230286, "epoch": 0.12932511648880204, "flos": 22090035306240.0, "grad_norm": 2.7470104760426186, "language_loss": 0.70360446, "learning_rate": 3.837439461435634e-06, "loss": 0.72522342, "num_input_tokens_seen": 46487605, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.83203125, "step": 2151, "time_per_iteration": 3.801199197769165 }, { "auxiliary_loss_clip": 0.01115444, "auxiliary_loss_mlp": 0.01038906, "balance_loss_clip": 1.0167923, "balance_loss_mlp": 1.03347862, "epoch": 0.12938523974147, "flos": 20301749089920.0, "grad_norm": 1.893864815881546, "language_loss": 0.84205532, "learning_rate": 3.837290222763467e-06, "loss": 0.86359888, "num_input_tokens_seen": 46505100, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.8203125, "step": 2152, "time_per_iteration": 3.7916481494903564 }, { "auxiliary_loss_clip": 0.01115379, "auxiliary_loss_mlp": 0.01048931, "balance_loss_clip": 1.02667511, "balance_loss_mlp": 1.03076506, "epoch": 0.12944536299413797, "flos": 19498956220800.0, "grad_norm": 1.7597086564245399, "language_loss": 0.78322285, "learning_rate": 3.837140918523139e-06, "loss": 0.80486596, "num_input_tokens_seen": 46524020, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.84765625, "step": 2153, "time_per_iteration": 2.3810129165649414 }, { "auxiliary_loss_clip": 0.01114693, "auxiliary_loss_mlp": 0.01046041, "balance_loss_clip": 1.02345061, "balance_loss_mlp": 1.03024554, "epoch": 0.12950548624680594, "flos": 27706722658560.0, "grad_norm": 1.5810401249289232, "language_loss": 0.80105108, "learning_rate": 3.836991548719977e-06, "loss": 0.82265842, "num_input_tokens_seen": 46544640, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.84375, "step": 2154, "time_per_iteration": 2.4317073822021484 }, { "auxiliary_loss_clip": 0.01117873, "auxiliary_loss_mlp": 0.01045428, "balance_loss_clip": 1.02213466, "balance_loss_mlp": 1.03387988, "epoch": 0.12956560949947393, "flos": 17564802877440.0, "grad_norm": 1.883418700162283, "language_loss": 0.83143741, "learning_rate": 3.836842113359312e-06, "loss": 0.85307044, "num_input_tokens_seen": 46561395, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.83984375, "step": 2155, "time_per_iteration": 2.3684208393096924 }, { "auxiliary_loss_clip": 0.01116601, "auxiliary_loss_mlp": 0.0104286, "balance_loss_clip": 1.01961398, "balance_loss_mlp": 1.03140187, "epoch": 0.1296257327521419, "flos": 20739664673280.0, "grad_norm": 2.60337306720873, "language_loss": 0.75139713, "learning_rate": 3.836692612446477e-06, "loss": 0.77299178, "num_input_tokens_seen": 46579395, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.8515625, "step": 2156, "time_per_iteration": 2.3783843517303467 }, { "auxiliary_loss_clip": 0.01110463, "auxiliary_loss_mlp": 0.01041525, "balance_loss_clip": 1.01924503, "balance_loss_mlp": 1.03018153, "epoch": 0.12968585600480986, "flos": 16394898395520.0, "grad_norm": 1.8736014366258005, "language_loss": 0.86187625, "learning_rate": 3.836543045986806e-06, "loss": 0.88339609, "num_input_tokens_seen": 46597090, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.80078125, "step": 2157, "time_per_iteration": 2.371201515197754 }, { "auxiliary_loss_clip": 0.01114903, "auxiliary_loss_mlp": 0.01045962, "balance_loss_clip": 1.02079701, "balance_loss_mlp": 1.02920556, "epoch": 0.12974597925747783, "flos": 28328281326720.0, "grad_norm": 2.5809651804015252, "language_loss": 0.80159575, "learning_rate": 3.836393413985639e-06, "loss": 0.8232044, "num_input_tokens_seen": 46617355, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.85546875, "step": 2158, "time_per_iteration": 2.4271130561828613 }, { "auxiliary_loss_clip": 0.01119188, "auxiliary_loss_mlp": 0.01044262, "balance_loss_clip": 1.02031255, "balance_loss_mlp": 1.03189647, "epoch": 0.1298061025101458, "flos": 9682357288320.0, "grad_norm": 2.3233885540699752, "language_loss": 0.74530011, "learning_rate": 3.836243716448315e-06, "loss": 0.76693463, "num_input_tokens_seen": 46633130, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.875, "step": 2159, "time_per_iteration": 2.3568336963653564 }, { "auxiliary_loss_clip": 0.01110925, "auxiliary_loss_mlp": 0.01043432, "balance_loss_clip": 1.0205555, "balance_loss_mlp": 1.02931619, "epoch": 0.12986622576281376, "flos": 27708293669760.0, "grad_norm": 1.9939973787459886, "language_loss": 0.82547617, "learning_rate": 3.8360939533801755e-06, "loss": 0.84701967, "num_input_tokens_seen": 46650575, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.81640625, "step": 2160, "time_per_iteration": 2.436048984527588 }, { "auxiliary_loss_clip": 0.01111275, "auxiliary_loss_mlp": 0.01039301, "balance_loss_clip": 1.01673508, "balance_loss_mlp": 1.03352499, "epoch": 0.12992634901548175, "flos": 18801845637120.0, "grad_norm": 1.6115067908069767, "language_loss": 0.8194257, "learning_rate": 3.835944124786566e-06, "loss": 0.84093148, "num_input_tokens_seen": 46668780, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.77734375, "step": 2161, "time_per_iteration": 2.3671443462371826 }, { "auxiliary_loss_clip": 0.0111232, "auxiliary_loss_mlp": 0.01045213, "balance_loss_clip": 1.02225292, "balance_loss_mlp": 1.03120661, "epoch": 0.12998647226814972, "flos": 29126430984960.0, "grad_norm": 2.48374558455811, "language_loss": 0.82309949, "learning_rate": 3.835794230672833e-06, "loss": 0.84467483, "num_input_tokens_seen": 46687550, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.8125, "step": 2162, "time_per_iteration": 2.451164722442627 }, { "auxiliary_loss_clip": 0.01114268, "auxiliary_loss_mlp": 0.01039767, "balance_loss_clip": 1.01642549, "balance_loss_mlp": 1.02928841, "epoch": 0.13004659552081768, "flos": 19572657834240.0, "grad_norm": 2.722292118622819, "language_loss": 0.73022962, "learning_rate": 3.8356442710443264e-06, "loss": 0.75177002, "num_input_tokens_seen": 46706730, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.8515625, "step": 2163, "time_per_iteration": 2.3654446601867676 }, { "auxiliary_loss_clip": 0.01118636, "auxiliary_loss_mlp": 0.01045333, "balance_loss_clip": 1.0205729, "balance_loss_mlp": 1.03252673, "epoch": 0.13010671877348565, "flos": 22489651261440.0, "grad_norm": 2.095764827754389, "language_loss": 0.80722785, "learning_rate": 3.835494245906398e-06, "loss": 0.82886755, "num_input_tokens_seen": 46724250, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.859375, "step": 2164, "time_per_iteration": 2.3870291709899902 }, { "auxiliary_loss_clip": 0.01114403, "auxiliary_loss_mlp": 0.01043411, "balance_loss_clip": 1.02221584, "balance_loss_mlp": 1.03097296, "epoch": 0.1301668420261536, "flos": 23877099624960.0, "grad_norm": 3.8116589712356306, "language_loss": 0.72372723, "learning_rate": 3.835344155264401e-06, "loss": 0.74530542, "num_input_tokens_seen": 46744105, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.83203125, "step": 2165, "time_per_iteration": 2.4053943157196045 }, { "auxiliary_loss_clip": 0.01116581, "auxiliary_loss_mlp": 0.0104437, "balance_loss_clip": 1.02080274, "balance_loss_mlp": 1.03170514, "epoch": 0.13022696527882158, "flos": 23148916064640.0, "grad_norm": 1.9997241389468778, "language_loss": 0.74730682, "learning_rate": 3.835193999123692e-06, "loss": 0.76891643, "num_input_tokens_seen": 46764250, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.84765625, "step": 2166, "time_per_iteration": 2.3938140869140625 }, { "auxiliary_loss_clip": 0.01114391, "auxiliary_loss_mlp": 0.01047195, "balance_loss_clip": 1.02433074, "balance_loss_mlp": 1.03019023, "epoch": 0.13028708853148954, "flos": 26907281280000.0, "grad_norm": 2.0620299613784137, "language_loss": 0.83216614, "learning_rate": 3.83504377748963e-06, "loss": 0.853782, "num_input_tokens_seen": 46786865, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.84375, "step": 2167, "time_per_iteration": 2.4385018348693848 }, { "auxiliary_loss_clip": 0.01115013, "auxiliary_loss_mlp": 0.01049261, "balance_loss_clip": 1.02614653, "balance_loss_mlp": 1.03209972, "epoch": 0.13034721178415754, "flos": 21250409086080.0, "grad_norm": 1.513359311460835, "language_loss": 0.8302232, "learning_rate": 3.834893490367576e-06, "loss": 0.85186589, "num_input_tokens_seen": 46807030, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.828125, "step": 2168, "time_per_iteration": 2.40877628326416 }, { "auxiliary_loss_clip": 0.0111569, "auxiliary_loss_mlp": 0.01046431, "balance_loss_clip": 1.02275646, "balance_loss_mlp": 1.0308814, "epoch": 0.1304073350368255, "flos": 18766338917760.0, "grad_norm": 1.984090395510942, "language_loss": 0.80360681, "learning_rate": 3.834743137762894e-06, "loss": 0.82522798, "num_input_tokens_seen": 46826280, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.84765625, "step": 2169, "time_per_iteration": 2.3826045989990234 }, { "auxiliary_loss_clip": 0.0111662, "auxiliary_loss_mlp": 0.01039846, "balance_loss_clip": 1.01742303, "balance_loss_mlp": 1.03232527, "epoch": 0.13046745828949347, "flos": 28363438932480.0, "grad_norm": 2.2305857081038893, "language_loss": 0.6652239, "learning_rate": 3.834592719680948e-06, "loss": 0.68678856, "num_input_tokens_seen": 46846505, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.84375, "step": 2170, "time_per_iteration": 2.461083173751831 }, { "auxiliary_loss_clip": 0.01114168, "auxiliary_loss_mlp": 0.01043086, "balance_loss_clip": 1.01999474, "balance_loss_mlp": 1.03133297, "epoch": 0.13052758154216143, "flos": 29603798271360.0, "grad_norm": 1.7907177832851473, "language_loss": 0.66911954, "learning_rate": 3.834442236127107e-06, "loss": 0.69069207, "num_input_tokens_seen": 46867380, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.828125, "step": 2171, "time_per_iteration": 2.4362990856170654 }, { "auxiliary_loss_clip": 0.01113873, "auxiliary_loss_mlp": 0.0104092, "balance_loss_clip": 1.01773369, "balance_loss_mlp": 1.03114748, "epoch": 0.1305877047948294, "flos": 19389852622080.0, "grad_norm": 3.7675128109353753, "language_loss": 0.71806937, "learning_rate": 3.834291687106741e-06, "loss": 0.73961735, "num_input_tokens_seen": 46886810, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.828125, "step": 2172, "time_per_iteration": 2.380793333053589 }, { "auxiliary_loss_clip": 0.01112477, "auxiliary_loss_mlp": 0.01036177, "balance_loss_clip": 1.0148859, "balance_loss_mlp": 1.03321958, "epoch": 0.13064782804749736, "flos": 16872579884160.0, "grad_norm": 2.3902403008806186, "language_loss": 0.75815773, "learning_rate": 3.834141072625224e-06, "loss": 0.77964425, "num_input_tokens_seen": 46905620, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.79296875, "step": 2173, "time_per_iteration": 2.3549892902374268 }, { "auxiliary_loss_clip": 0.01115196, "auxiliary_loss_mlp": 0.0105006, "balance_loss_clip": 1.0247401, "balance_loss_mlp": 1.03040016, "epoch": 0.13070795130016533, "flos": 24497925154560.0, "grad_norm": 2.9225325642837494, "language_loss": 0.70756316, "learning_rate": 3.833990392687929e-06, "loss": 0.72921574, "num_input_tokens_seen": 46925120, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.84765625, "step": 2174, "time_per_iteration": 2.4198758602142334 }, { "auxiliary_loss_clip": 0.01028906, "auxiliary_loss_mlp": 0.01013085, "balance_loss_clip": 1.00998521, "balance_loss_mlp": 1.00515437, "epoch": 0.13076807455283332, "flos": 71051042211840.0, "grad_norm": 0.7942536445789119, "language_loss": 0.59030503, "learning_rate": 3.833839647300235e-06, "loss": 0.61072493, "num_input_tokens_seen": 46988195, "router_z_loss_clip": 0.03088379, "router_z_loss_mlp": 0.23828125, "step": 2175, "time_per_iteration": 3.144157648086548 }, { "auxiliary_loss_clip": 0.01114359, "auxiliary_loss_mlp": 0.01043894, "balance_loss_clip": 1.02068412, "balance_loss_mlp": 1.03204215, "epoch": 0.13082819780550128, "flos": 20263519284480.0, "grad_norm": 2.0487518159718525, "language_loss": 0.79935825, "learning_rate": 3.8336888364675215e-06, "loss": 0.82094073, "num_input_tokens_seen": 47004720, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.8203125, "step": 2176, "time_per_iteration": 2.3896164894104004 }, { "auxiliary_loss_clip": 0.01114783, "auxiliary_loss_mlp": 0.0104714, "balance_loss_clip": 1.02339339, "balance_loss_mlp": 1.03182983, "epoch": 0.13088832105816925, "flos": 34202034086400.0, "grad_norm": 1.8814432995606216, "language_loss": 0.74356544, "learning_rate": 3.83353796019517e-06, "loss": 0.76518464, "num_input_tokens_seen": 47024255, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.828125, "step": 2177, "time_per_iteration": 2.495523691177368 }, { "auxiliary_loss_clip": 0.01109811, "auxiliary_loss_mlp": 0.01038365, "balance_loss_clip": 1.0149163, "balance_loss_mlp": 1.02974164, "epoch": 0.13094844431083721, "flos": 17893998887040.0, "grad_norm": 3.209198226754352, "language_loss": 0.81699485, "learning_rate": 3.833387018488565e-06, "loss": 0.8384766, "num_input_tokens_seen": 47042465, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.80078125, "step": 2178, "time_per_iteration": 2.3795485496520996 }, { "auxiliary_loss_clip": 0.01114659, "auxiliary_loss_mlp": 0.01045616, "balance_loss_clip": 1.02328825, "balance_loss_mlp": 1.03250599, "epoch": 0.13100856756350518, "flos": 17310355822080.0, "grad_norm": 2.5560615218726506, "language_loss": 0.74238646, "learning_rate": 3.833236011353094e-06, "loss": 0.76398921, "num_input_tokens_seen": 47060370, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.8203125, "step": 2179, "time_per_iteration": 2.342864751815796 }, { "auxiliary_loss_clip": 0.01110019, "auxiliary_loss_mlp": 0.01040483, "balance_loss_clip": 1.0178926, "balance_loss_mlp": 1.03046405, "epoch": 0.13106869081617314, "flos": 22199453107200.0, "grad_norm": 2.048661509886946, "language_loss": 0.84644121, "learning_rate": 3.833084938794144e-06, "loss": 0.86794627, "num_input_tokens_seen": 47081415, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.79296875, "step": 2180, "time_per_iteration": 2.407223701477051 }, { "auxiliary_loss_clip": 0.01112943, "auxiliary_loss_mlp": 0.01047561, "balance_loss_clip": 1.02467299, "balance_loss_mlp": 1.0332936, "epoch": 0.13112881406884114, "flos": 21762026282880.0, "grad_norm": 1.9753502228991404, "language_loss": 0.89866793, "learning_rate": 3.832933800817109e-06, "loss": 0.92027295, "num_input_tokens_seen": 47099860, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.796875, "step": 2181, "time_per_iteration": 2.3756368160247803 }, { "auxiliary_loss_clip": 0.01114895, "auxiliary_loss_mlp": 0.01039846, "balance_loss_clip": 1.01674366, "balance_loss_mlp": 1.03142238, "epoch": 0.1311889373215091, "flos": 23329975708800.0, "grad_norm": 1.9743377326784572, "language_loss": 0.68522978, "learning_rate": 3.832782597427381e-06, "loss": 0.70677722, "num_input_tokens_seen": 47118540, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8359375, "step": 2182, "time_per_iteration": 2.4033896923065186 }, { "auxiliary_loss_clip": 0.01112122, "auxiliary_loss_mlp": 0.01042636, "balance_loss_clip": 1.01881838, "balance_loss_mlp": 1.03059769, "epoch": 0.13124906057417707, "flos": 21466381956480.0, "grad_norm": 2.201104097311233, "language_loss": 0.78646314, "learning_rate": 3.832631328630357e-06, "loss": 0.8080107, "num_input_tokens_seen": 47136710, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.81640625, "step": 2183, "time_per_iteration": 2.3812484741210938 }, { "auxiliary_loss_clip": 0.01112053, "auxiliary_loss_mlp": 0.01041131, "balance_loss_clip": 1.01770616, "balance_loss_mlp": 1.03083646, "epoch": 0.13130918382684503, "flos": 23254284147840.0, "grad_norm": 1.775200535092224, "language_loss": 0.85511321, "learning_rate": 3.832479994431435e-06, "loss": 0.87664509, "num_input_tokens_seen": 47157155, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.8125, "step": 2184, "time_per_iteration": 2.4094576835632324 }, { "auxiliary_loss_clip": 0.01112983, "auxiliary_loss_mlp": 0.01048421, "balance_loss_clip": 1.02326775, "balance_loss_mlp": 1.02956557, "epoch": 0.131369307079513, "flos": 20849222119680.0, "grad_norm": 1.8957297581875063, "language_loss": 0.81803644, "learning_rate": 3.8323285948360155e-06, "loss": 0.83965051, "num_input_tokens_seen": 47176820, "router_z_loss_clip": 0.25195312, "router_z_loss_mlp": 0.83203125, "step": 2185, "time_per_iteration": 2.3804616928100586 }, { "auxiliary_loss_clip": 0.01114125, "auxiliary_loss_mlp": 0.01049463, "balance_loss_clip": 1.02526331, "balance_loss_mlp": 1.02938843, "epoch": 0.13142943033218096, "flos": 17857375004160.0, "grad_norm": 2.2495711618574887, "language_loss": 0.73018312, "learning_rate": 3.832177129849501e-06, "loss": 0.75181901, "num_input_tokens_seen": 47195855, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.84765625, "step": 2186, "time_per_iteration": 2.3832266330718994 }, { "auxiliary_loss_clip": 0.01111766, "auxiliary_loss_mlp": 0.01042853, "balance_loss_clip": 1.01833165, "balance_loss_mlp": 1.03020239, "epoch": 0.13148955358484893, "flos": 20994984512640.0, "grad_norm": 1.9049824607495724, "language_loss": 0.79982936, "learning_rate": 3.832025599477299e-06, "loss": 0.82137549, "num_input_tokens_seen": 47214535, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.81640625, "step": 2187, "time_per_iteration": 3.7775983810424805 }, { "auxiliary_loss_clip": 0.01029256, "auxiliary_loss_mlp": 0.01001701, "balance_loss_clip": 0.99886429, "balance_loss_mlp": 1.00545847, "epoch": 0.13154967683751692, "flos": 70169206291200.0, "grad_norm": 0.8418638121387123, "language_loss": 0.59020334, "learning_rate": 3.831874003724815e-06, "loss": 0.61051291, "num_input_tokens_seen": 47270300, "router_z_loss_clip": 0.02832031, "router_z_loss_mlp": 0.23828125, "step": 2188, "time_per_iteration": 3.027845859527588 }, { "auxiliary_loss_clip": 0.01113306, "auxiliary_loss_mlp": 0.0104065, "balance_loss_clip": 1.0176661, "balance_loss_mlp": 1.03258336, "epoch": 0.1316098000901849, "flos": 20375101589760.0, "grad_norm": 2.7635906312789493, "language_loss": 0.74211311, "learning_rate": 3.83172234259746e-06, "loss": 0.76365268, "num_input_tokens_seen": 47290720, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.80859375, "step": 2189, "time_per_iteration": 2.404848575592041 }, { "auxiliary_loss_clip": 0.01111108, "auxiliary_loss_mlp": 0.01043428, "balance_loss_clip": 1.02089787, "balance_loss_mlp": 1.03086686, "epoch": 0.13166992334285285, "flos": 23220034237440.0, "grad_norm": 2.841583407847499, "language_loss": 0.72652352, "learning_rate": 3.831570616100646e-06, "loss": 0.74806881, "num_input_tokens_seen": 47311820, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.80078125, "step": 2190, "time_per_iteration": 3.9414379596710205 }, { "auxiliary_loss_clip": 0.01114807, "auxiliary_loss_mlp": 0.01043525, "balance_loss_clip": 1.02108979, "balance_loss_mlp": 1.03255475, "epoch": 0.13173004659552082, "flos": 23329836063360.0, "grad_norm": 2.0700281427836646, "language_loss": 0.74798489, "learning_rate": 3.831418824239789e-06, "loss": 0.7695682, "num_input_tokens_seen": 47331605, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.82421875, "step": 2191, "time_per_iteration": 2.4002647399902344 }, { "auxiliary_loss_clip": 0.01113795, "auxiliary_loss_mlp": 0.01042197, "balance_loss_clip": 1.0187366, "balance_loss_mlp": 1.03179884, "epoch": 0.13179016984818878, "flos": 21250443997440.0, "grad_norm": 1.8918926065518962, "language_loss": 0.79094386, "learning_rate": 3.831266967020304e-06, "loss": 0.81250381, "num_input_tokens_seen": 47350455, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.8203125, "step": 2192, "time_per_iteration": 3.7655372619628906 }, { "auxiliary_loss_clip": 0.01113748, "auxiliary_loss_mlp": 0.01049621, "balance_loss_clip": 1.02704287, "balance_loss_mlp": 1.03156579, "epoch": 0.13185029310085675, "flos": 17777913016320.0, "grad_norm": 1.8315977299639683, "language_loss": 0.85026896, "learning_rate": 3.831115044447613e-06, "loss": 0.87190259, "num_input_tokens_seen": 47368225, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.8203125, "step": 2193, "time_per_iteration": 2.3712708950042725 }, { "auxiliary_loss_clip": 0.01027596, "auxiliary_loss_mlp": 0.01006696, "balance_loss_clip": 1.00401342, "balance_loss_mlp": 1.00445843, "epoch": 0.1319104163535247, "flos": 69848319185280.0, "grad_norm": 0.7516439486959761, "language_loss": 0.5407998, "learning_rate": 3.830963056527136e-06, "loss": 0.56114268, "num_input_tokens_seen": 47427125, "router_z_loss_clip": 0.02685547, "router_z_loss_mlp": 0.23242188, "step": 2194, "time_per_iteration": 2.982822895050049 }, { "auxiliary_loss_clip": 0.01111447, "auxiliary_loss_mlp": 0.0104063, "balance_loss_clip": 1.0185287, "balance_loss_mlp": 1.02902436, "epoch": 0.1319705396061927, "flos": 25191893715840.0, "grad_norm": 2.706274089240783, "language_loss": 0.72682089, "learning_rate": 3.830811003264296e-06, "loss": 0.74834168, "num_input_tokens_seen": 47450275, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.82421875, "step": 2195, "time_per_iteration": 2.4383349418640137 }, { "auxiliary_loss_clip": 0.01116518, "auxiliary_loss_mlp": 0.01036477, "balance_loss_clip": 1.01201487, "balance_loss_mlp": 1.03085577, "epoch": 0.13203066285886067, "flos": 20739420293760.0, "grad_norm": 2.188057333145781, "language_loss": 0.77745765, "learning_rate": 3.830658884664522e-06, "loss": 0.79898763, "num_input_tokens_seen": 47469155, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.859375, "step": 2196, "time_per_iteration": 2.4020564556121826 }, { "auxiliary_loss_clip": 0.01112904, "auxiliary_loss_mlp": 0.01046155, "balance_loss_clip": 1.02268314, "balance_loss_mlp": 1.03074789, "epoch": 0.13209078611152864, "flos": 22053306689280.0, "grad_norm": 2.3314154498043687, "language_loss": 0.74964809, "learning_rate": 3.830506700733241e-06, "loss": 0.77123868, "num_input_tokens_seen": 47488405, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.8203125, "step": 2197, "time_per_iteration": 2.3787591457366943 }, { "auxiliary_loss_clip": 0.01112553, "auxiliary_loss_mlp": 0.01036321, "balance_loss_clip": 1.01301527, "balance_loss_mlp": 1.03028679, "epoch": 0.1321509093641966, "flos": 16284153962880.0, "grad_norm": 1.9713334442069481, "language_loss": 0.79398841, "learning_rate": 3.830354451475884e-06, "loss": 0.81547713, "num_input_tokens_seen": 47505650, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.8203125, "step": 2198, "time_per_iteration": 2.3466055393218994 }, { "auxiliary_loss_clip": 0.01110413, "auxiliary_loss_mlp": 0.01045317, "balance_loss_clip": 1.02369249, "balance_loss_mlp": 1.0311594, "epoch": 0.13221103261686457, "flos": 16982067507840.0, "grad_norm": 2.16452769131162, "language_loss": 0.82794964, "learning_rate": 3.830202136897886e-06, "loss": 0.84950697, "num_input_tokens_seen": 47521540, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.79296875, "step": 2199, "time_per_iteration": 2.3588204383850098 }, { "auxiliary_loss_clip": 0.01114966, "auxiliary_loss_mlp": 0.01045151, "balance_loss_clip": 1.02260888, "balance_loss_mlp": 1.03232229, "epoch": 0.13227115586953253, "flos": 34232373924480.0, "grad_norm": 2.0674267205343058, "language_loss": 0.69267744, "learning_rate": 3.8300497570046804e-06, "loss": 0.71427858, "num_input_tokens_seen": 47543625, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.82421875, "step": 2200, "time_per_iteration": 2.498288631439209 }, { "auxiliary_loss_clip": 0.01110224, "auxiliary_loss_mlp": 0.01044977, "balance_loss_clip": 1.02214885, "balance_loss_mlp": 1.02931237, "epoch": 0.13233127912220052, "flos": 20703599372160.0, "grad_norm": 1.72394146433636, "language_loss": 0.84412003, "learning_rate": 3.829897311801707e-06, "loss": 0.86567205, "num_input_tokens_seen": 47563740, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.80859375, "step": 2201, "time_per_iteration": 2.41595721244812 }, { "auxiliary_loss_clip": 0.0111353, "auxiliary_loss_mlp": 0.0103892, "balance_loss_clip": 1.01606727, "balance_loss_mlp": 1.03146267, "epoch": 0.1323914023748685, "flos": 25804061228160.0, "grad_norm": 1.8939885495026298, "language_loss": 0.8684684, "learning_rate": 3.829744801294406e-06, "loss": 0.88999289, "num_input_tokens_seen": 47582655, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8203125, "step": 2202, "time_per_iteration": 2.4088144302368164 }, { "auxiliary_loss_clip": 0.01108057, "auxiliary_loss_mlp": 0.01042929, "balance_loss_clip": 1.02104235, "balance_loss_mlp": 1.02929723, "epoch": 0.13245152562753645, "flos": 21250478908800.0, "grad_norm": 1.9619794150131111, "language_loss": 0.72687638, "learning_rate": 3.8295922254882186e-06, "loss": 0.74838626, "num_input_tokens_seen": 47600875, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.7890625, "step": 2203, "time_per_iteration": 2.4206366539001465 }, { "auxiliary_loss_clip": 0.011129, "auxiliary_loss_mlp": 0.01045435, "balance_loss_clip": 1.02400172, "balance_loss_mlp": 1.03066564, "epoch": 0.13251164888020442, "flos": 26609856474240.0, "grad_norm": 2.3983479091674726, "language_loss": 0.73204589, "learning_rate": 3.829439584388591e-06, "loss": 0.75362927, "num_input_tokens_seen": 47619250, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.82421875, "step": 2204, "time_per_iteration": 2.413806676864624 }, { "auxiliary_loss_clip": 0.01114894, "auxiliary_loss_mlp": 0.01049636, "balance_loss_clip": 1.02656865, "balance_loss_mlp": 1.03270948, "epoch": 0.13257177213287238, "flos": 29825217313920.0, "grad_norm": 1.627487404452735, "language_loss": 0.78527379, "learning_rate": 3.8292868780009715e-06, "loss": 0.8069191, "num_input_tokens_seen": 47639445, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8203125, "step": 2205, "time_per_iteration": 2.448495626449585 }, { "auxiliary_loss_clip": 0.01112746, "auxiliary_loss_mlp": 0.01042662, "balance_loss_clip": 1.01917768, "balance_loss_mlp": 1.03346515, "epoch": 0.13263189538554035, "flos": 21287382082560.0, "grad_norm": 3.000510399921291, "language_loss": 0.78886485, "learning_rate": 3.829134106330809e-06, "loss": 0.81041896, "num_input_tokens_seen": 47658740, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.79296875, "step": 2206, "time_per_iteration": 2.371365785598755 }, { "auxiliary_loss_clip": 0.01112996, "auxiliary_loss_mlp": 0.0104297, "balance_loss_clip": 1.02140486, "balance_loss_mlp": 1.03161049, "epoch": 0.13269201863820831, "flos": 16873138465920.0, "grad_norm": 1.9451013166631212, "language_loss": 0.74432611, "learning_rate": 3.828981269383554e-06, "loss": 0.76588583, "num_input_tokens_seen": 47676880, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.8125, "step": 2207, "time_per_iteration": 2.374790668487549 }, { "auxiliary_loss_clip": 0.01109145, "auxiliary_loss_mlp": 0.01041137, "balance_loss_clip": 1.01826119, "balance_loss_mlp": 1.02947807, "epoch": 0.1327521418908763, "flos": 23767786558080.0, "grad_norm": 1.6835914547074657, "language_loss": 0.8392238, "learning_rate": 3.828828367164663e-06, "loss": 0.86072659, "num_input_tokens_seen": 47696635, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.796875, "step": 2208, "time_per_iteration": 2.4047040939331055 }, { "auxiliary_loss_clip": 0.01109737, "auxiliary_loss_mlp": 0.010438, "balance_loss_clip": 1.02266455, "balance_loss_mlp": 1.03286314, "epoch": 0.13281226514354427, "flos": 26504383656960.0, "grad_norm": 1.640844661454858, "language_loss": 0.84896123, "learning_rate": 3.828675399679592e-06, "loss": 0.87049663, "num_input_tokens_seen": 47717760, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.76953125, "step": 2209, "time_per_iteration": 2.446331024169922 }, { "auxiliary_loss_clip": 0.01111576, "auxiliary_loss_mlp": 0.01041452, "balance_loss_clip": 1.02059054, "balance_loss_mlp": 1.02965975, "epoch": 0.13287238839621224, "flos": 24497610952320.0, "grad_norm": 3.8266139058343094, "language_loss": 0.82185507, "learning_rate": 3.8285223669337995e-06, "loss": 0.84338534, "num_input_tokens_seen": 47737685, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.8203125, "step": 2210, "time_per_iteration": 2.407039165496826 }, { "auxiliary_loss_clip": 0.01028667, "auxiliary_loss_mlp": 0.01008429, "balance_loss_clip": 1.00596142, "balance_loss_mlp": 1.0048151, "epoch": 0.1329325116488802, "flos": 67691071054080.0, "grad_norm": 0.7631754503202972, "language_loss": 0.57968318, "learning_rate": 3.828369268932747e-06, "loss": 0.60005414, "num_input_tokens_seen": 47802415, "router_z_loss_clip": 0.0246582, "router_z_loss_mlp": 0.23828125, "step": 2211, "time_per_iteration": 3.1298675537109375 }, { "auxiliary_loss_clip": 0.01027464, "auxiliary_loss_mlp": 0.01003969, "balance_loss_clip": 1.00142968, "balance_loss_mlp": 1.00369525, "epoch": 0.13299263490154817, "flos": 72258303715200.0, "grad_norm": 0.7972035907439725, "language_loss": 0.55318034, "learning_rate": 3.828216105681899e-06, "loss": 0.57349467, "num_input_tokens_seen": 47871485, "router_z_loss_clip": 0.02539062, "router_z_loss_mlp": 0.23828125, "step": 2212, "time_per_iteration": 3.120358943939209 }, { "auxiliary_loss_clip": 0.01115866, "auxiliary_loss_mlp": 0.01043653, "balance_loss_clip": 1.02024066, "balance_loss_mlp": 1.03086209, "epoch": 0.13305275815421613, "flos": 17930308567680.0, "grad_norm": 3.686084098762373, "language_loss": 0.74999905, "learning_rate": 3.8280628771867205e-06, "loss": 0.77159429, "num_input_tokens_seen": 47888315, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.8515625, "step": 2213, "time_per_iteration": 2.3642704486846924 }, { "auxiliary_loss_clip": 0.01106028, "auxiliary_loss_mlp": 0.0103986, "balance_loss_clip": 1.01936781, "balance_loss_mlp": 1.02920556, "epoch": 0.13311288140688413, "flos": 22339943884800.0, "grad_norm": 1.9918070964647272, "language_loss": 0.79267049, "learning_rate": 3.8279095834526815e-06, "loss": 0.81412941, "num_input_tokens_seen": 47906600, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.765625, "step": 2214, "time_per_iteration": 2.381934881210327 }, { "auxiliary_loss_clip": 0.01111644, "auxiliary_loss_mlp": 0.01048696, "balance_loss_clip": 1.02578413, "balance_loss_mlp": 1.03067029, "epoch": 0.1331730046595521, "flos": 31867531649280.0, "grad_norm": 1.8632679071922624, "language_loss": 0.69134682, "learning_rate": 3.8277562244852495e-06, "loss": 0.71295023, "num_input_tokens_seen": 47927630, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.8125, "step": 2215, "time_per_iteration": 2.469895601272583 }, { "auxiliary_loss_clip": 0.0110936, "auxiliary_loss_mlp": 0.01038844, "balance_loss_clip": 1.01729095, "balance_loss_mlp": 1.0285362, "epoch": 0.13323312791222006, "flos": 22565447556480.0, "grad_norm": 1.7429587827665565, "language_loss": 0.8103472, "learning_rate": 3.827602800289901e-06, "loss": 0.83182919, "num_input_tokens_seen": 47947935, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.8046875, "step": 2216, "time_per_iteration": 2.4177145957946777 }, { "auxiliary_loss_clip": 0.01110098, "auxiliary_loss_mlp": 0.01050275, "balance_loss_clip": 1.02679074, "balance_loss_mlp": 1.02934813, "epoch": 0.13329325116488802, "flos": 15084433313280.0, "grad_norm": 1.9884474111186918, "language_loss": 0.87180638, "learning_rate": 3.827449310872109e-06, "loss": 0.89341009, "num_input_tokens_seen": 47965515, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.8046875, "step": 2217, "time_per_iteration": 2.3541815280914307 }, { "auxiliary_loss_clip": 0.01110352, "auxiliary_loss_mlp": 0.01042972, "balance_loss_clip": 1.02001226, "balance_loss_mlp": 1.03001714, "epoch": 0.133353374417556, "flos": 27452450160000.0, "grad_norm": 2.0869599672048142, "language_loss": 0.73178005, "learning_rate": 3.827295756237351e-06, "loss": 0.75331324, "num_input_tokens_seen": 47985675, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8046875, "step": 2218, "time_per_iteration": 2.4269580841064453 }, { "auxiliary_loss_clip": 0.01110731, "auxiliary_loss_mlp": 0.01040425, "balance_loss_clip": 1.0173105, "balance_loss_mlp": 1.03010273, "epoch": 0.13341349767022395, "flos": 24093631077120.0, "grad_norm": 1.9308602690623262, "language_loss": 0.87036943, "learning_rate": 3.8271421363911095e-06, "loss": 0.89188099, "num_input_tokens_seen": 48004985, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.8046875, "step": 2219, "time_per_iteration": 2.412137985229492 }, { "auxiliary_loss_clip": 0.01112182, "auxiliary_loss_mlp": 0.01044723, "balance_loss_clip": 1.02320623, "balance_loss_mlp": 1.03274846, "epoch": 0.13347362092289192, "flos": 24132209996160.0, "grad_norm": 1.8041370165092492, "language_loss": 0.77078104, "learning_rate": 3.826988451338864e-06, "loss": 0.79235017, "num_input_tokens_seen": 48024965, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.79296875, "step": 2220, "time_per_iteration": 2.3967785835266113 }, { "auxiliary_loss_clip": 0.01107377, "auxiliary_loss_mlp": 0.01038252, "balance_loss_clip": 1.01708031, "balance_loss_mlp": 1.02819431, "epoch": 0.1335337441755599, "flos": 18435711542400.0, "grad_norm": 7.832869738206291, "language_loss": 0.78862309, "learning_rate": 3.826834701086101e-06, "loss": 0.8100794, "num_input_tokens_seen": 48040890, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.7890625, "step": 2221, "time_per_iteration": 2.364459753036499 }, { "auxiliary_loss_clip": 0.01027571, "auxiliary_loss_mlp": 0.01008219, "balance_loss_clip": 1.00550091, "balance_loss_mlp": 1.00492895, "epoch": 0.13359386742822787, "flos": 50609395837440.0, "grad_norm": 1.0235370816867682, "language_loss": 0.69041914, "learning_rate": 3.826680885638306e-06, "loss": 0.71077704, "num_input_tokens_seen": 48091855, "router_z_loss_clip": 0.02722168, "router_z_loss_mlp": 0.2265625, "step": 2222, "time_per_iteration": 2.852614641189575 }, { "auxiliary_loss_clip": 0.01111476, "auxiliary_loss_mlp": 0.01043529, "balance_loss_clip": 1.02135563, "balance_loss_mlp": 1.03144515, "epoch": 0.13365399068089584, "flos": 22777615088640.0, "grad_norm": 2.5645377689459323, "language_loss": 0.67273825, "learning_rate": 3.826527005000969e-06, "loss": 0.69428831, "num_input_tokens_seen": 48111350, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.80078125, "step": 2223, "time_per_iteration": 2.4104299545288086 }, { "auxiliary_loss_clip": 0.01111142, "auxiliary_loss_mlp": 0.01041346, "balance_loss_clip": 1.01845801, "balance_loss_mlp": 1.02990484, "epoch": 0.1337141139335638, "flos": 12530781072000.0, "grad_norm": 2.3977038475576817, "language_loss": 0.82913315, "learning_rate": 3.826373059179582e-06, "loss": 0.85065806, "num_input_tokens_seen": 48129840, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8125, "step": 2224, "time_per_iteration": 2.357208013534546 }, { "auxiliary_loss_clip": 0.01115536, "auxiliary_loss_mlp": 0.01043834, "balance_loss_clip": 1.01978946, "balance_loss_mlp": 1.03101516, "epoch": 0.13377423718623177, "flos": 23037857429760.0, "grad_norm": 2.4352710360103487, "language_loss": 0.6528067, "learning_rate": 3.826219048179639e-06, "loss": 0.67440045, "num_input_tokens_seen": 48149240, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.84375, "step": 2225, "time_per_iteration": 2.411724090576172 }, { "auxiliary_loss_clip": 0.01111973, "auxiliary_loss_mlp": 0.01051142, "balance_loss_clip": 1.02871895, "balance_loss_mlp": 1.03108084, "epoch": 0.13383436043889974, "flos": 16215479585280.0, "grad_norm": 2.2820423654325768, "language_loss": 0.89218378, "learning_rate": 3.826064972006635e-06, "loss": 0.9138149, "num_input_tokens_seen": 48166330, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.80859375, "step": 2226, "time_per_iteration": 2.3514645099639893 }, { "auxiliary_loss_clip": 0.01111681, "auxiliary_loss_mlp": 0.01046075, "balance_loss_clip": 1.02270985, "balance_loss_mlp": 1.03023171, "epoch": 0.1338944836915677, "flos": 24278530970880.0, "grad_norm": 2.3901396881263177, "language_loss": 0.74010229, "learning_rate": 3.825910830666069e-06, "loss": 0.76167989, "num_input_tokens_seen": 48187600, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.8125, "step": 2227, "time_per_iteration": 3.7839083671569824 }, { "auxiliary_loss_clip": 0.01109886, "auxiliary_loss_mlp": 0.01042187, "balance_loss_clip": 1.02022851, "balance_loss_mlp": 1.02917624, "epoch": 0.1339546069442357, "flos": 17597900712960.0, "grad_norm": 1.9744666336955208, "language_loss": 0.85193986, "learning_rate": 3.825756624163443e-06, "loss": 0.87346053, "num_input_tokens_seen": 48204400, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.80859375, "step": 2228, "time_per_iteration": 2.4063632488250732 }, { "auxiliary_loss_clip": 0.0111183, "auxiliary_loss_mlp": 0.01043096, "balance_loss_clip": 1.02082753, "balance_loss_mlp": 1.03083122, "epoch": 0.13401473019690366, "flos": 18989049680640.0, "grad_norm": 2.210839270971356, "language_loss": 0.80781674, "learning_rate": 3.825602352504259e-06, "loss": 0.82936597, "num_input_tokens_seen": 48222180, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.80859375, "step": 2229, "time_per_iteration": 3.8466081619262695 }, { "auxiliary_loss_clip": 0.01112713, "auxiliary_loss_mlp": 0.01054344, "balance_loss_clip": 1.03212357, "balance_loss_mlp": 1.03109515, "epoch": 0.13407485344957162, "flos": 26942578531200.0, "grad_norm": 1.7333625897589784, "language_loss": 0.73866439, "learning_rate": 3.825448015694023e-06, "loss": 0.76033497, "num_input_tokens_seen": 48243245, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.81640625, "step": 2230, "time_per_iteration": 3.813413619995117 }, { "auxiliary_loss_clip": 0.01114408, "auxiliary_loss_mlp": 0.01052288, "balance_loss_clip": 1.02911401, "balance_loss_mlp": 1.03100502, "epoch": 0.1341349767022396, "flos": 20338338061440.0, "grad_norm": 1.665509430892685, "language_loss": 0.80048466, "learning_rate": 3.8252936137382435e-06, "loss": 0.8221516, "num_input_tokens_seen": 48262600, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.8359375, "step": 2231, "time_per_iteration": 3.74367356300354 }, { "auxiliary_loss_clip": 0.01113262, "auxiliary_loss_mlp": 0.01053751, "balance_loss_clip": 1.03018284, "balance_loss_mlp": 1.03137553, "epoch": 0.13419509995490755, "flos": 29860724033280.0, "grad_norm": 1.6743044726072736, "language_loss": 0.72241318, "learning_rate": 3.82513914664243e-06, "loss": 0.74408329, "num_input_tokens_seen": 48285075, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.81640625, "step": 2232, "time_per_iteration": 2.4403839111328125 }, { "auxiliary_loss_clip": 0.01116062, "auxiliary_loss_mlp": 0.01045066, "balance_loss_clip": 1.02165365, "balance_loss_mlp": 1.0313139, "epoch": 0.13425522320757552, "flos": 26941775569920.0, "grad_norm": 2.475825651129534, "language_loss": 0.65877473, "learning_rate": 3.824984614412095e-06, "loss": 0.68038601, "num_input_tokens_seen": 48301285, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.84765625, "step": 2233, "time_per_iteration": 2.420086145401001 }, { "auxiliary_loss_clip": 0.01110733, "auxiliary_loss_mlp": 0.01040684, "balance_loss_clip": 1.01764095, "balance_loss_mlp": 1.0282203, "epoch": 0.1343153464602435, "flos": 15776411927040.0, "grad_norm": 2.6796130628290333, "language_loss": 0.81137764, "learning_rate": 3.824830017052753e-06, "loss": 0.83289182, "num_input_tokens_seen": 48317835, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.82421875, "step": 2234, "time_per_iteration": 2.3355088233947754 }, { "auxiliary_loss_clip": 0.01114447, "auxiliary_loss_mlp": 0.01045909, "balance_loss_clip": 1.02279413, "balance_loss_mlp": 1.03209209, "epoch": 0.13437546971291148, "flos": 24313653665280.0, "grad_norm": 2.0611952609550626, "language_loss": 0.82459158, "learning_rate": 3.824675354569923e-06, "loss": 0.8461951, "num_input_tokens_seen": 48335670, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.82421875, "step": 2235, "time_per_iteration": 2.398684501647949 }, { "auxiliary_loss_clip": 0.01113562, "auxiliary_loss_mlp": 0.01046007, "balance_loss_clip": 1.02316618, "balance_loss_mlp": 1.03003097, "epoch": 0.13443559296557944, "flos": 26649482734080.0, "grad_norm": 1.8639521609867127, "language_loss": 0.86475575, "learning_rate": 3.824520626969122e-06, "loss": 0.88635147, "num_input_tokens_seen": 48357805, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8359375, "step": 2236, "time_per_iteration": 2.4377503395080566 }, { "auxiliary_loss_clip": 0.01115367, "auxiliary_loss_mlp": 0.01041982, "balance_loss_clip": 1.01909375, "balance_loss_mlp": 1.03230786, "epoch": 0.1344957162182474, "flos": 21795193941120.0, "grad_norm": 1.6210711622806835, "language_loss": 0.77399528, "learning_rate": 3.824365834255874e-06, "loss": 0.79556882, "num_input_tokens_seen": 48377845, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.83203125, "step": 2237, "time_per_iteration": 2.3981072902679443 }, { "auxiliary_loss_clip": 0.01115959, "auxiliary_loss_mlp": 0.01050458, "balance_loss_clip": 1.02498269, "balance_loss_mlp": 1.0319252, "epoch": 0.13455583947091537, "flos": 19864531733760.0, "grad_norm": 3.126998467904437, "language_loss": 0.78480875, "learning_rate": 3.824210976435702e-06, "loss": 0.8064729, "num_input_tokens_seen": 48394735, "router_z_loss_clip": 0.25390625, "router_z_loss_mlp": 0.83984375, "step": 2238, "time_per_iteration": 2.369377851486206 }, { "auxiliary_loss_clip": 0.01108949, "auxiliary_loss_mlp": 0.01037434, "balance_loss_clip": 1.01383018, "balance_loss_mlp": 1.02933514, "epoch": 0.13461596272358334, "flos": 30845519153280.0, "grad_norm": 2.5142370982128113, "language_loss": 0.68518054, "learning_rate": 3.824056053514132e-06, "loss": 0.70664436, "num_input_tokens_seen": 48414200, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.796875, "step": 2239, "time_per_iteration": 2.4480724334716797 }, { "auxiliary_loss_clip": 0.01115627, "auxiliary_loss_mlp": 0.01046975, "balance_loss_clip": 1.02345467, "balance_loss_mlp": 1.0310353, "epoch": 0.1346760859762513, "flos": 12493633518720.0, "grad_norm": 2.4649535556244233, "language_loss": 0.8140105, "learning_rate": 3.823901065496693e-06, "loss": 0.8356365, "num_input_tokens_seen": 48431065, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.84765625, "step": 2240, "time_per_iteration": 2.3443541526794434 }, { "auxiliary_loss_clip": 0.0111462, "auxiliary_loss_mlp": 0.01047804, "balance_loss_clip": 1.02402151, "balance_loss_mlp": 1.03058279, "epoch": 0.1347362092289193, "flos": 21834924935040.0, "grad_norm": 1.6357289943000772, "language_loss": 0.77624297, "learning_rate": 3.823746012388918e-06, "loss": 0.79786718, "num_input_tokens_seen": 48450335, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.83984375, "step": 2241, "time_per_iteration": 2.386347532272339 }, { "auxiliary_loss_clip": 0.01109833, "auxiliary_loss_mlp": 0.0104163, "balance_loss_clip": 1.0201124, "balance_loss_mlp": 1.03068209, "epoch": 0.13479633248158726, "flos": 23508451912320.0, "grad_norm": 1.659205909924718, "language_loss": 0.82998061, "learning_rate": 3.823590894196339e-06, "loss": 0.85149527, "num_input_tokens_seen": 48468555, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.7890625, "step": 2242, "time_per_iteration": 2.4124395847320557 }, { "auxiliary_loss_clip": 0.01114746, "auxiliary_loss_mlp": 0.0104936, "balance_loss_clip": 1.02486253, "balance_loss_mlp": 1.03196406, "epoch": 0.13485645573425523, "flos": 29343241728000.0, "grad_norm": 3.4559024996267147, "language_loss": 0.64423156, "learning_rate": 3.823435710924491e-06, "loss": 0.66587257, "num_input_tokens_seen": 48488515, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.828125, "step": 2243, "time_per_iteration": 2.446357250213623 }, { "auxiliary_loss_clip": 0.01107438, "auxiliary_loss_mlp": 0.0103543, "balance_loss_clip": 1.01361525, "balance_loss_mlp": 1.02859759, "epoch": 0.1349165789869232, "flos": 28035883756800.0, "grad_norm": 1.9068844032861496, "language_loss": 0.72579181, "learning_rate": 3.823280462578913e-06, "loss": 0.7472204, "num_input_tokens_seen": 48510515, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.7890625, "step": 2244, "time_per_iteration": 2.431729316711426 }, { "auxiliary_loss_clip": 0.01111567, "auxiliary_loss_mlp": 0.01045916, "balance_loss_clip": 1.02393365, "balance_loss_mlp": 1.03124416, "epoch": 0.13497670223959116, "flos": 22852713156480.0, "grad_norm": 1.5985867586427198, "language_loss": 0.85773522, "learning_rate": 3.8231251491651455e-06, "loss": 0.87931001, "num_input_tokens_seen": 48529940, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.8046875, "step": 2245, "time_per_iteration": 2.402890920639038 }, { "auxiliary_loss_clip": 0.0110876, "auxiliary_loss_mlp": 0.01037539, "balance_loss_clip": 1.01528263, "balance_loss_mlp": 1.03117847, "epoch": 0.13503682549225912, "flos": 16503757614720.0, "grad_norm": 3.0225897100770207, "language_loss": 0.7903704, "learning_rate": 3.822969770688732e-06, "loss": 0.81183338, "num_input_tokens_seen": 48548190, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.7734375, "step": 2246, "time_per_iteration": 2.375706911087036 }, { "auxiliary_loss_clip": 0.01026554, "auxiliary_loss_mlp": 0.01003031, "balance_loss_clip": 1.00014651, "balance_loss_mlp": 1.00370598, "epoch": 0.1350969487449271, "flos": 70753023912960.0, "grad_norm": 0.7460429799412394, "language_loss": 0.6049459, "learning_rate": 3.8228143271552154e-06, "loss": 0.62524176, "num_input_tokens_seen": 48613165, "router_z_loss_clip": 0.02880859, "router_z_loss_mlp": 0.22851562, "step": 2247, "time_per_iteration": 3.1216773986816406 }, { "auxiliary_loss_clip": 0.01116568, "auxiliary_loss_mlp": 0.01046591, "balance_loss_clip": 1.02258193, "balance_loss_mlp": 1.03152978, "epoch": 0.13515707199759508, "flos": 23074865337600.0, "grad_norm": 1.9198332175671928, "language_loss": 0.81013012, "learning_rate": 3.822658818570145e-06, "loss": 0.83176172, "num_input_tokens_seen": 48631705, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.8515625, "step": 2248, "time_per_iteration": 2.382345199584961 }, { "auxiliary_loss_clip": 0.01108966, "auxiliary_loss_mlp": 0.01039089, "balance_loss_clip": 1.0176791, "balance_loss_mlp": 1.03090119, "epoch": 0.13521719525026304, "flos": 23185225745280.0, "grad_norm": 1.7824571080205176, "language_loss": 0.76665759, "learning_rate": 3.822503244939069e-06, "loss": 0.78813815, "num_input_tokens_seen": 48649740, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.78125, "step": 2249, "time_per_iteration": 2.39690899848938 }, { "auxiliary_loss_clip": 0.01112266, "auxiliary_loss_mlp": 0.01047254, "balance_loss_clip": 1.02599943, "balance_loss_mlp": 1.03164351, "epoch": 0.135277318502931, "flos": 24789764142720.0, "grad_norm": 1.4487850858130753, "language_loss": 0.84145266, "learning_rate": 3.822347606267541e-06, "loss": 0.86304784, "num_input_tokens_seen": 48671565, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.8046875, "step": 2250, "time_per_iteration": 2.437000274658203 }, { "auxiliary_loss_clip": 0.01112671, "auxiliary_loss_mlp": 0.01045562, "balance_loss_clip": 1.02204239, "balance_loss_mlp": 1.03052807, "epoch": 0.13533744175559898, "flos": 21907439562240.0, "grad_norm": 2.9768659822997896, "language_loss": 0.82101446, "learning_rate": 3.8221919025611145e-06, "loss": 0.84259683, "num_input_tokens_seen": 48690425, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.8203125, "step": 2251, "time_per_iteration": 2.400810480117798 }, { "auxiliary_loss_clip": 0.01110933, "auxiliary_loss_mlp": 0.01045632, "balance_loss_clip": 1.02289891, "balance_loss_mlp": 1.02972174, "epoch": 0.13539756500826694, "flos": 21210678092160.0, "grad_norm": 1.6350101086519406, "language_loss": 0.85983527, "learning_rate": 3.822036133825346e-06, "loss": 0.88140088, "num_input_tokens_seen": 48707505, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.8125, "step": 2252, "time_per_iteration": 2.3785557746887207 }, { "auxiliary_loss_clip": 0.01026414, "auxiliary_loss_mlp": 0.01003823, "balance_loss_clip": 1.00139153, "balance_loss_mlp": 1.00329792, "epoch": 0.1354576882609349, "flos": 63238981656960.0, "grad_norm": 0.7738025226790045, "language_loss": 0.61805081, "learning_rate": 3.821880300065794e-06, "loss": 0.63835323, "num_input_tokens_seen": 48775895, "router_z_loss_clip": 0.02429199, "router_z_loss_mlp": 0.23144531, "step": 2253, "time_per_iteration": 3.105794906616211 }, { "auxiliary_loss_clip": 0.01112569, "auxiliary_loss_mlp": 0.01045554, "balance_loss_clip": 1.02329731, "balance_loss_mlp": 1.03244042, "epoch": 0.1355178115136029, "flos": 25481882401920.0, "grad_norm": 1.8286536160423945, "language_loss": 0.89134341, "learning_rate": 3.821724401288022e-06, "loss": 0.91292465, "num_input_tokens_seen": 48798370, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.8046875, "step": 2254, "time_per_iteration": 2.444282054901123 }, { "auxiliary_loss_clip": 0.01115733, "auxiliary_loss_mlp": 0.01052232, "balance_loss_clip": 1.02819943, "balance_loss_mlp": 1.02996039, "epoch": 0.13557793476627086, "flos": 21615879864960.0, "grad_norm": 1.874734162542784, "language_loss": 0.84478366, "learning_rate": 3.821568437497592e-06, "loss": 0.86646336, "num_input_tokens_seen": 48817955, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.859375, "step": 2255, "time_per_iteration": 2.3827905654907227 }, { "auxiliary_loss_clip": 0.01110767, "auxiliary_loss_mlp": 0.0104296, "balance_loss_clip": 1.02015567, "balance_loss_mlp": 1.02874386, "epoch": 0.13563805801893883, "flos": 24927322366080.0, "grad_norm": 2.5800977448637177, "language_loss": 0.74805433, "learning_rate": 3.821412408700069e-06, "loss": 0.76959157, "num_input_tokens_seen": 48836330, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8203125, "step": 2256, "time_per_iteration": 2.392350912094116 }, { "auxiliary_loss_clip": 0.01113995, "auxiliary_loss_mlp": 0.01049828, "balance_loss_clip": 1.02698743, "balance_loss_mlp": 1.03051257, "epoch": 0.1356981812716068, "flos": 14749581663360.0, "grad_norm": 2.636399814040291, "language_loss": 0.83367229, "learning_rate": 3.821256314901023e-06, "loss": 0.8553105, "num_input_tokens_seen": 48851890, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8359375, "step": 2257, "time_per_iteration": 2.3453638553619385 }, { "auxiliary_loss_clip": 0.01117427, "auxiliary_loss_mlp": 0.01045476, "balance_loss_clip": 1.02176523, "balance_loss_mlp": 1.03027248, "epoch": 0.13575830452427476, "flos": 11107791077760.0, "grad_norm": 2.4127163814424946, "language_loss": 0.81851101, "learning_rate": 3.821100156106024e-06, "loss": 0.84013999, "num_input_tokens_seen": 48865510, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.875, "step": 2258, "time_per_iteration": 2.3260347843170166 }, { "auxiliary_loss_clip": 0.01112168, "auxiliary_loss_mlp": 0.01047014, "balance_loss_clip": 1.02205133, "balance_loss_mlp": 1.03000212, "epoch": 0.13581842777694272, "flos": 17959531242240.0, "grad_norm": 2.4992820719003985, "language_loss": 0.82302582, "learning_rate": 3.820943932320644e-06, "loss": 0.84461761, "num_input_tokens_seen": 48882360, "router_z_loss_clip": 0.25, "router_z_loss_mlp": 0.8203125, "step": 2259, "time_per_iteration": 2.3578600883483887 }, { "auxiliary_loss_clip": 0.01115546, "auxiliary_loss_mlp": 0.01044489, "balance_loss_clip": 1.02334094, "balance_loss_mlp": 1.03380466, "epoch": 0.1358785510296107, "flos": 22856029735680.0, "grad_norm": 1.8211233775954654, "language_loss": 0.73700893, "learning_rate": 3.82078764355046e-06, "loss": 0.75860929, "num_input_tokens_seen": 48902700, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.81640625, "step": 2260, "time_per_iteration": 2.391852378845215 }, { "auxiliary_loss_clip": 0.01109938, "auxiliary_loss_mlp": 0.01051966, "balance_loss_clip": 1.03000736, "balance_loss_mlp": 1.02989888, "epoch": 0.13593867428227868, "flos": 25738214670720.0, "grad_norm": 2.3528590007555854, "language_loss": 0.75280863, "learning_rate": 3.820631289801048e-06, "loss": 0.77442765, "num_input_tokens_seen": 48922525, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.80078125, "step": 2261, "time_per_iteration": 2.457080602645874 }, { "auxiliary_loss_clip": 0.0111298, "auxiliary_loss_mlp": 0.01039492, "balance_loss_clip": 1.01767623, "balance_loss_mlp": 1.03044224, "epoch": 0.13599879753494665, "flos": 31247858194560.0, "grad_norm": 3.7731977894452378, "language_loss": 0.63001621, "learning_rate": 3.82047487107799e-06, "loss": 0.65154088, "num_input_tokens_seen": 48942510, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.82421875, "step": 2262, "time_per_iteration": 2.46036696434021 }, { "auxiliary_loss_clip": 0.01111991, "auxiliary_loss_mlp": 0.01041925, "balance_loss_clip": 1.01963282, "balance_loss_mlp": 1.03004408, "epoch": 0.1360589207876146, "flos": 23913898064640.0, "grad_norm": 2.647698818670158, "language_loss": 0.82917178, "learning_rate": 3.820318387386865e-06, "loss": 0.85071099, "num_input_tokens_seen": 48962625, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.8203125, "step": 2263, "time_per_iteration": 2.4298808574676514 }, { "auxiliary_loss_clip": 0.01116094, "auxiliary_loss_mlp": 0.01052066, "balance_loss_clip": 1.0279026, "balance_loss_mlp": 1.03200746, "epoch": 0.13611904404028258, "flos": 19973181484800.0, "grad_norm": 2.037974726999726, "language_loss": 0.87724793, "learning_rate": 3.8201618387332605e-06, "loss": 0.8989296, "num_input_tokens_seen": 48982525, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.84375, "step": 2264, "time_per_iteration": 2.3881897926330566 }, { "auxiliary_loss_clip": 0.01116603, "auxiliary_loss_mlp": 0.01043247, "balance_loss_clip": 1.01901174, "balance_loss_mlp": 1.03215957, "epoch": 0.13617916729295054, "flos": 15339753152640.0, "grad_norm": 3.0086405021950764, "language_loss": 0.71634519, "learning_rate": 3.82000522512276e-06, "loss": 0.73794365, "num_input_tokens_seen": 48997605, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.84375, "step": 2265, "time_per_iteration": 2.362116813659668 }, { "auxiliary_loss_clip": 0.01110729, "auxiliary_loss_mlp": 0.01036144, "balance_loss_clip": 1.01580715, "balance_loss_mlp": 1.03179741, "epoch": 0.1362392905456185, "flos": 27450285655680.0, "grad_norm": 2.2099575569314935, "language_loss": 0.66132319, "learning_rate": 3.819848546560957e-06, "loss": 0.68279195, "num_input_tokens_seen": 49018535, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7890625, "step": 2266, "time_per_iteration": 2.4455068111419678 }, { "auxiliary_loss_clip": 0.01111668, "auxiliary_loss_mlp": 0.01050643, "balance_loss_clip": 1.02792215, "balance_loss_mlp": 1.03137374, "epoch": 0.1362994137982865, "flos": 25007866606080.0, "grad_norm": 1.585383208781827, "language_loss": 0.76206291, "learning_rate": 3.819691803053439e-06, "loss": 0.78368604, "num_input_tokens_seen": 49038865, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.8046875, "step": 2267, "time_per_iteration": 3.8275153636932373 }, { "auxiliary_loss_clip": 0.01110651, "auxiliary_loss_mlp": 0.01041567, "balance_loss_clip": 1.01919103, "balance_loss_mlp": 1.02962959, "epoch": 0.13635953705095447, "flos": 20301993469440.0, "grad_norm": 2.200944711480025, "language_loss": 0.81724751, "learning_rate": 3.819534994605802e-06, "loss": 0.83876967, "num_input_tokens_seen": 49058010, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.80859375, "step": 2268, "time_per_iteration": 2.3766119480133057 }, { "auxiliary_loss_clip": 0.01109573, "auxiliary_loss_mlp": 0.01038756, "balance_loss_clip": 1.01694036, "balance_loss_mlp": 1.03026772, "epoch": 0.13641966030362243, "flos": 31357066527360.0, "grad_norm": 1.7898561510552156, "language_loss": 0.75749362, "learning_rate": 3.819378121223641e-06, "loss": 0.77897686, "num_input_tokens_seen": 49080330, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.79296875, "step": 2269, "time_per_iteration": 3.867666244506836 }, { "auxiliary_loss_clip": 0.01115323, "auxiliary_loss_mlp": 0.01037113, "balance_loss_clip": 1.01519012, "balance_loss_mlp": 1.03264654, "epoch": 0.1364797835562904, "flos": 20477257827840.0, "grad_norm": 2.098228893476109, "language_loss": 0.80965889, "learning_rate": 3.819221182912555e-06, "loss": 0.8311832, "num_input_tokens_seen": 49097035, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.828125, "step": 2270, "time_per_iteration": 5.221672773361206 }, { "auxiliary_loss_clip": 0.01115496, "auxiliary_loss_mlp": 0.01048832, "balance_loss_clip": 1.02586031, "balance_loss_mlp": 1.03060257, "epoch": 0.13653990680895836, "flos": 13077520963200.0, "grad_norm": 2.722045196078793, "language_loss": 0.75869644, "learning_rate": 3.819064179678145e-06, "loss": 0.78033966, "num_input_tokens_seen": 49113945, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.84765625, "step": 2271, "time_per_iteration": 2.361807107925415 }, { "auxiliary_loss_clip": 0.01114825, "auxiliary_loss_mlp": 0.01044332, "balance_loss_clip": 1.02134812, "balance_loss_mlp": 1.03116345, "epoch": 0.13660003006162633, "flos": 16945757827200.0, "grad_norm": 1.8230049855742485, "language_loss": 0.80149591, "learning_rate": 3.8189071115260134e-06, "loss": 0.82308745, "num_input_tokens_seen": 49132855, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8359375, "step": 2272, "time_per_iteration": 2.3897011280059814 }, { "auxiliary_loss_clip": 0.01027994, "auxiliary_loss_mlp": 0.01020793, "balance_loss_clip": 1.01800334, "balance_loss_mlp": 1.00485504, "epoch": 0.1366601533142943, "flos": 68679357310080.0, "grad_norm": 0.6956821962683516, "language_loss": 0.60680348, "learning_rate": 3.818749978461765e-06, "loss": 0.62729138, "num_input_tokens_seen": 49198310, "router_z_loss_clip": 0.0279541, "router_z_loss_mlp": 0.23144531, "step": 2273, "time_per_iteration": 3.1359124183654785 }, { "auxiliary_loss_clip": 0.01109442, "auxiliary_loss_mlp": 0.01042235, "balance_loss_clip": 1.02008581, "balance_loss_mlp": 1.02961373, "epoch": 0.13672027656696228, "flos": 19243252356480.0, "grad_norm": 1.6551537076379452, "language_loss": 0.77221978, "learning_rate": 3.8185927804910096e-06, "loss": 0.79373658, "num_input_tokens_seen": 49217250, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.796875, "step": 2274, "time_per_iteration": 2.3578431606292725 }, { "auxiliary_loss_clip": 0.01112591, "auxiliary_loss_mlp": 0.01045059, "balance_loss_clip": 1.02202773, "balance_loss_mlp": 1.03039908, "epoch": 0.13678039981963025, "flos": 24533780987520.0, "grad_norm": 2.473963726005356, "language_loss": 0.7832284, "learning_rate": 3.818435517619355e-06, "loss": 0.80480492, "num_input_tokens_seen": 49236615, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8203125, "step": 2275, "time_per_iteration": 2.43884539604187 }, { "auxiliary_loss_clip": 0.01110935, "auxiliary_loss_mlp": 0.01041719, "balance_loss_clip": 1.02028525, "balance_loss_mlp": 1.0302285, "epoch": 0.13684052307229821, "flos": 15668425491840.0, "grad_norm": 2.7714124972923755, "language_loss": 0.81413603, "learning_rate": 3.818278189852415e-06, "loss": 0.83566254, "num_input_tokens_seen": 49253935, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.80859375, "step": 2276, "time_per_iteration": 2.362184524536133 }, { "auxiliary_loss_clip": 0.01119973, "auxiliary_loss_mlp": 0.01049788, "balance_loss_clip": 1.02260852, "balance_loss_mlp": 1.03208447, "epoch": 0.13690064632496618, "flos": 28363473843840.0, "grad_norm": 2.480011971937364, "language_loss": 0.69309795, "learning_rate": 3.8181207971958025e-06, "loss": 0.71479559, "num_input_tokens_seen": 49273605, "router_z_loss_clip": 0.27148438, "router_z_loss_mlp": 0.87890625, "step": 2277, "time_per_iteration": 2.4605045318603516 }, { "auxiliary_loss_clip": 0.01112767, "auxiliary_loss_mlp": 0.01054563, "balance_loss_clip": 1.03097129, "balance_loss_mlp": 1.03087139, "epoch": 0.13696076957763414, "flos": 23403642410880.0, "grad_norm": 2.1244063974828564, "language_loss": 0.80648291, "learning_rate": 3.817963339655137e-06, "loss": 0.82815623, "num_input_tokens_seen": 49291785, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.8203125, "step": 2278, "time_per_iteration": 2.3916478157043457 }, { "auxiliary_loss_clip": 0.01112518, "auxiliary_loss_mlp": 0.01041664, "balance_loss_clip": 1.01930034, "balance_loss_mlp": 1.03335357, "epoch": 0.1370208928303021, "flos": 37195068188160.0, "grad_norm": 2.334388181097841, "language_loss": 0.7501992, "learning_rate": 3.8178058172360346e-06, "loss": 0.77174109, "num_input_tokens_seen": 49311405, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.7890625, "step": 2279, "time_per_iteration": 2.517383098602295 }, { "auxiliary_loss_clip": 0.0111511, "auxiliary_loss_mlp": 0.01046556, "balance_loss_clip": 1.02408552, "balance_loss_mlp": 1.03076339, "epoch": 0.13708101608297008, "flos": 26975187607680.0, "grad_norm": 1.8940635031675936, "language_loss": 0.76659471, "learning_rate": 3.817648229944119e-06, "loss": 0.78821135, "num_input_tokens_seen": 49331835, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.84375, "step": 2280, "time_per_iteration": 2.4143829345703125 }, { "auxiliary_loss_clip": 0.01107492, "auxiliary_loss_mlp": 0.01041696, "balance_loss_clip": 1.01880753, "balance_loss_mlp": 1.02764416, "epoch": 0.13714113933563807, "flos": 32555635102080.0, "grad_norm": 1.73368062530258, "language_loss": 0.79739249, "learning_rate": 3.817490577785014e-06, "loss": 0.81888437, "num_input_tokens_seen": 49352290, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.796875, "step": 2281, "time_per_iteration": 2.465832233428955 }, { "auxiliary_loss_clip": 0.01117367, "auxiliary_loss_mlp": 0.01047146, "balance_loss_clip": 1.0238409, "balance_loss_mlp": 1.03159499, "epoch": 0.13720126258830603, "flos": 16100510878080.0, "grad_norm": 1.7304683509529122, "language_loss": 0.83738309, "learning_rate": 3.817332860764346e-06, "loss": 0.85902822, "num_input_tokens_seen": 49370285, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.859375, "step": 2282, "time_per_iteration": 2.3765079975128174 }, { "auxiliary_loss_clip": 0.01109256, "auxiliary_loss_mlp": 0.01042945, "balance_loss_clip": 1.02089167, "balance_loss_mlp": 1.02886319, "epoch": 0.137261385840974, "flos": 18952530531840.0, "grad_norm": 1.6378516218141752, "language_loss": 0.73454171, "learning_rate": 3.817175078887742e-06, "loss": 0.75606376, "num_input_tokens_seen": 49389610, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.8046875, "step": 2283, "time_per_iteration": 2.3719022274017334 }, { "auxiliary_loss_clip": 0.01113893, "auxiliary_loss_mlp": 0.01046806, "balance_loss_clip": 1.02537262, "balance_loss_mlp": 1.03361201, "epoch": 0.13732150909364196, "flos": 23294224609920.0, "grad_norm": 2.343842559962333, "language_loss": 0.83827215, "learning_rate": 3.8170172321608345e-06, "loss": 0.85987914, "num_input_tokens_seen": 49408390, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.8046875, "step": 2284, "time_per_iteration": 2.398379325866699 }, { "auxiliary_loss_clip": 0.01116226, "auxiliary_loss_mlp": 0.01046771, "balance_loss_clip": 1.02210665, "balance_loss_mlp": 1.03020883, "epoch": 0.13738163234630993, "flos": 29349979620480.0, "grad_norm": 1.769826224992319, "language_loss": 0.74995393, "learning_rate": 3.816859320589255e-06, "loss": 0.77158391, "num_input_tokens_seen": 49427725, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.859375, "step": 2285, "time_per_iteration": 2.5012168884277344 }, { "auxiliary_loss_clip": 0.0111146, "auxiliary_loss_mlp": 0.01044058, "balance_loss_clip": 1.02106261, "balance_loss_mlp": 1.03127074, "epoch": 0.1374417555989779, "flos": 26650111138560.0, "grad_norm": 1.7714291008538752, "language_loss": 0.74398136, "learning_rate": 3.81670134417864e-06, "loss": 0.76553655, "num_input_tokens_seen": 49449000, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.80078125, "step": 2286, "time_per_iteration": 2.4326276779174805 }, { "auxiliary_loss_clip": 0.01117541, "auxiliary_loss_mlp": 0.01050704, "balance_loss_clip": 1.02468061, "balance_loss_mlp": 1.03174639, "epoch": 0.1375018788516459, "flos": 28402122585600.0, "grad_norm": 2.0022977447187134, "language_loss": 0.86365223, "learning_rate": 3.8165433029346276e-06, "loss": 0.88533461, "num_input_tokens_seen": 49468360, "router_z_loss_clip": 0.26171875, "router_z_loss_mlp": 0.859375, "step": 2287, "time_per_iteration": 2.4316582679748535 }, { "auxiliary_loss_clip": 0.0111424, "auxiliary_loss_mlp": 0.01047324, "balance_loss_clip": 1.02416182, "balance_loss_mlp": 1.03153014, "epoch": 0.13756200210431385, "flos": 37412297867520.0, "grad_norm": 1.8374540548779694, "language_loss": 0.68856287, "learning_rate": 3.816385196862858e-06, "loss": 0.71017849, "num_input_tokens_seen": 49493450, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.828125, "step": 2288, "time_per_iteration": 2.5205066204071045 }, { "auxiliary_loss_clip": 0.01114996, "auxiliary_loss_mlp": 0.01044784, "balance_loss_clip": 1.02221727, "balance_loss_mlp": 1.03295159, "epoch": 0.13762212535698182, "flos": 22709918229120.0, "grad_norm": 2.3650727449351887, "language_loss": 0.86925477, "learning_rate": 3.816227025968972e-06, "loss": 0.89085257, "num_input_tokens_seen": 49511220, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.8203125, "step": 2289, "time_per_iteration": 2.3779609203338623 }, { "auxiliary_loss_clip": 0.01108751, "auxiliary_loss_mlp": 0.01043833, "balance_loss_clip": 1.02186263, "balance_loss_mlp": 1.02881837, "epoch": 0.13768224860964978, "flos": 23950975795200.0, "grad_norm": 1.8917479365362528, "language_loss": 0.74836767, "learning_rate": 3.8160687902586155e-06, "loss": 0.76989353, "num_input_tokens_seen": 49529820, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.796875, "step": 2290, "time_per_iteration": 2.411607027053833 }, { "auxiliary_loss_clip": 0.01026061, "auxiliary_loss_mlp": 0.01013156, "balance_loss_clip": 1.01010406, "balance_loss_mlp": 1.00359797, "epoch": 0.13774237186231775, "flos": 63586750510080.0, "grad_norm": 0.7022014569495892, "language_loss": 0.51588422, "learning_rate": 3.815910489737436e-06, "loss": 0.5362764, "num_input_tokens_seen": 49595325, "router_z_loss_clip": 0.03051758, "router_z_loss_mlp": 0.22460938, "step": 2291, "time_per_iteration": 3.0735068321228027 }, { "auxiliary_loss_clip": 0.01111884, "auxiliary_loss_mlp": 0.01043238, "balance_loss_clip": 1.01936054, "balance_loss_mlp": 1.03058958, "epoch": 0.1378024951149857, "flos": 24278321502720.0, "grad_norm": 1.7932632724464097, "language_loss": 0.70804548, "learning_rate": 3.815752124411081e-06, "loss": 0.72959673, "num_input_tokens_seen": 49615850, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.81640625, "step": 2292, "time_per_iteration": 2.408328056335449 }, { "auxiliary_loss_clip": 0.01112536, "auxiliary_loss_mlp": 0.01049368, "balance_loss_clip": 1.02622998, "balance_loss_mlp": 1.03175652, "epoch": 0.13786261836765368, "flos": 14020839521280.0, "grad_norm": 2.592992239259999, "language_loss": 0.80301976, "learning_rate": 3.815593694285204e-06, "loss": 0.82463878, "num_input_tokens_seen": 49631860, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.80859375, "step": 2293, "time_per_iteration": 2.3571887016296387 }, { "auxiliary_loss_clip": 0.01113389, "auxiliary_loss_mlp": 0.01049373, "balance_loss_clip": 1.0256741, "balance_loss_mlp": 1.03126013, "epoch": 0.13792274162032167, "flos": 28877360279040.0, "grad_norm": 2.1590697829436465, "language_loss": 0.78471428, "learning_rate": 3.815435199365459e-06, "loss": 0.80634189, "num_input_tokens_seen": 49652145, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.8203125, "step": 2294, "time_per_iteration": 2.459261894226074 }, { "auxiliary_loss_clip": 0.01113719, "auxiliary_loss_mlp": 0.01044648, "balance_loss_clip": 1.02364314, "balance_loss_mlp": 1.03316617, "epoch": 0.13798286487298964, "flos": 21140118501120.0, "grad_norm": 2.209995404447119, "language_loss": 0.80169517, "learning_rate": 3.815276639657501e-06, "loss": 0.82327884, "num_input_tokens_seen": 49669880, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.8046875, "step": 2295, "time_per_iteration": 2.372267484664917 }, { "auxiliary_loss_clip": 0.01111241, "auxiliary_loss_mlp": 0.0104571, "balance_loss_clip": 1.02134347, "balance_loss_mlp": 1.02965164, "epoch": 0.1380429881256576, "flos": 22486509239040.0, "grad_norm": 1.8298599221433658, "language_loss": 0.78164601, "learning_rate": 3.815118015166989e-06, "loss": 0.80321556, "num_input_tokens_seen": 49687255, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.81640625, "step": 2296, "time_per_iteration": 2.387343406677246 }, { "auxiliary_loss_clip": 0.01116511, "auxiliary_loss_mlp": 0.01047356, "balance_loss_clip": 1.0243485, "balance_loss_mlp": 1.0339613, "epoch": 0.13810311137832557, "flos": 21392715254400.0, "grad_norm": 1.833720412786261, "language_loss": 0.78415352, "learning_rate": 3.814959325899584e-06, "loss": 0.80579221, "num_input_tokens_seen": 49706650, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.828125, "step": 2297, "time_per_iteration": 2.3892436027526855 }, { "auxiliary_loss_clip": 0.01111782, "auxiliary_loss_mlp": 0.01047091, "balance_loss_clip": 1.02507257, "balance_loss_mlp": 1.03123212, "epoch": 0.13816323463099353, "flos": 25988786565120.0, "grad_norm": 2.3329139411238775, "language_loss": 0.68648392, "learning_rate": 3.81480057186095e-06, "loss": 0.70807266, "num_input_tokens_seen": 49725715, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.8046875, "step": 2298, "time_per_iteration": 2.412243604660034 }, { "auxiliary_loss_clip": 0.01116407, "auxiliary_loss_mlp": 0.01051851, "balance_loss_clip": 1.02841473, "balance_loss_mlp": 1.0311178, "epoch": 0.1382233578836615, "flos": 19243322179200.0, "grad_norm": 2.047238548430911, "language_loss": 0.86757356, "learning_rate": 3.814641753056751e-06, "loss": 0.88925612, "num_input_tokens_seen": 49744710, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.8515625, "step": 2299, "time_per_iteration": 2.362157106399536 }, { "auxiliary_loss_clip": 0.01111331, "auxiliary_loss_mlp": 0.01052014, "balance_loss_clip": 1.02799284, "balance_loss_mlp": 1.02935147, "epoch": 0.1382834811363295, "flos": 25665106550400.0, "grad_norm": 1.782685339291963, "language_loss": 0.75776196, "learning_rate": 3.8144828694926565e-06, "loss": 0.77939546, "num_input_tokens_seen": 49764300, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.8203125, "step": 2300, "time_per_iteration": 2.427091121673584 }, { "auxiliary_loss_clip": 0.0111164, "auxiliary_loss_mlp": 0.01046812, "balance_loss_clip": 1.02509212, "balance_loss_mlp": 1.03253913, "epoch": 0.13834360438899745, "flos": 19783394000640.0, "grad_norm": 2.815439832138442, "language_loss": 0.8307541, "learning_rate": 3.814323921174335e-06, "loss": 0.85233855, "num_input_tokens_seen": 49778380, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.7890625, "step": 2301, "time_per_iteration": 2.347304582595825 }, { "auxiliary_loss_clip": 0.01109228, "auxiliary_loss_mlp": 0.0104721, "balance_loss_clip": 1.0243578, "balance_loss_mlp": 1.03007984, "epoch": 0.13840372764166542, "flos": 26650634808960.0, "grad_norm": 1.8641862102081654, "language_loss": 0.85776269, "learning_rate": 3.81416490810746e-06, "loss": 0.87932712, "num_input_tokens_seen": 49797460, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.7890625, "step": 2302, "time_per_iteration": 2.4252429008483887 }, { "auxiliary_loss_clip": 0.01025471, "auxiliary_loss_mlp": 0.01006036, "balance_loss_clip": 1.00288916, "balance_loss_mlp": 1.00358677, "epoch": 0.13846385089433338, "flos": 70507444343040.0, "grad_norm": 0.7557448061426103, "language_loss": 0.65586698, "learning_rate": 3.814005830297706e-06, "loss": 0.67618203, "num_input_tokens_seen": 49868005, "router_z_loss_clip": 0.03149414, "router_z_loss_mlp": 0.21875, "step": 2303, "time_per_iteration": 3.1661269664764404 }, { "auxiliary_loss_clip": 0.01109203, "auxiliary_loss_mlp": 0.01046523, "balance_loss_clip": 1.02403975, "balance_loss_mlp": 1.03107214, "epoch": 0.13852397414700135, "flos": 17347747754880.0, "grad_norm": 1.7502725857789592, "language_loss": 0.78403562, "learning_rate": 3.81384668775075e-06, "loss": 0.80559289, "num_input_tokens_seen": 49885825, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.78125, "step": 2304, "time_per_iteration": 2.371805191040039 }, { "auxiliary_loss_clip": 0.0111554, "auxiliary_loss_mlp": 0.01040256, "balance_loss_clip": 1.01833344, "balance_loss_mlp": 1.03259957, "epoch": 0.13858409739966931, "flos": 21542701921920.0, "grad_norm": 2.023868147563291, "language_loss": 0.77400017, "learning_rate": 3.8136874804722724e-06, "loss": 0.79555821, "num_input_tokens_seen": 49905975, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.83203125, "step": 2305, "time_per_iteration": 2.439689874649048 }, { "auxiliary_loss_clip": 0.01109113, "auxiliary_loss_mlp": 0.01044396, "balance_loss_clip": 1.02222347, "balance_loss_mlp": 1.02947581, "epoch": 0.13864422065233728, "flos": 21578837045760.0, "grad_norm": 1.7596167687772786, "language_loss": 0.87383056, "learning_rate": 3.813528208467953e-06, "loss": 0.89536566, "num_input_tokens_seen": 49925800, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.796875, "step": 2306, "time_per_iteration": 2.4078853130340576 }, { "auxiliary_loss_clip": 0.01024399, "auxiliary_loss_mlp": 0.01003031, "balance_loss_clip": 1.00005126, "balance_loss_mlp": 1.00275683, "epoch": 0.13870434390500527, "flos": 53368861743360.0, "grad_norm": 0.8668211906086138, "language_loss": 0.58999717, "learning_rate": 3.813368871743477e-06, "loss": 0.61027151, "num_input_tokens_seen": 49977620, "router_z_loss_clip": 0.02978516, "router_z_loss_mlp": 0.21679688, "step": 2307, "time_per_iteration": 4.485899925231934 }, { "auxiliary_loss_clip": 0.0111669, "auxiliary_loss_mlp": 0.01045172, "balance_loss_clip": 1.02103209, "balance_loss_mlp": 1.03269196, "epoch": 0.13876446715767324, "flos": 22564784240640.0, "grad_norm": 2.4277061628479344, "language_loss": 0.79327637, "learning_rate": 3.813209470304531e-06, "loss": 0.81489497, "num_input_tokens_seen": 49996650, "router_z_loss_clip": 0.2421875, "router_z_loss_mlp": 0.83984375, "step": 2308, "time_per_iteration": 3.7915420532226562 }, { "auxiliary_loss_clip": 0.01112279, "auxiliary_loss_mlp": 0.01039478, "balance_loss_clip": 1.01644635, "balance_loss_mlp": 1.03122795, "epoch": 0.1388245904103412, "flos": 20704157953920.0, "grad_norm": 2.7725072846561845, "language_loss": 0.77483279, "learning_rate": 3.813050004156802e-06, "loss": 0.79635036, "num_input_tokens_seen": 50015640, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.80859375, "step": 2309, "time_per_iteration": 3.744729995727539 }, { "auxiliary_loss_clip": 0.01115686, "auxiliary_loss_mlp": 0.01039425, "balance_loss_clip": 1.01646543, "balance_loss_mlp": 1.03120661, "epoch": 0.13888471366300917, "flos": 20553787261440.0, "grad_norm": 1.9923646727733035, "language_loss": 0.67644227, "learning_rate": 3.812890473305983e-06, "loss": 0.6979934, "num_input_tokens_seen": 50033500, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.84375, "step": 2310, "time_per_iteration": 3.7314958572387695 }, { "auxiliary_loss_clip": 0.01114428, "auxiliary_loss_mlp": 0.01042951, "balance_loss_clip": 1.01878774, "balance_loss_mlp": 1.03177369, "epoch": 0.13894483691567713, "flos": 13837370993280.0, "grad_norm": 1.927902290859768, "language_loss": 0.83659101, "learning_rate": 3.812730877757766e-06, "loss": 0.85816479, "num_input_tokens_seen": 50050075, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.828125, "step": 2311, "time_per_iteration": 2.350458860397339 }, { "auxiliary_loss_clip": 0.01116978, "auxiliary_loss_mlp": 0.01043033, "balance_loss_clip": 1.01932216, "balance_loss_mlp": 1.03216636, "epoch": 0.1390049601683451, "flos": 28030123382400.0, "grad_norm": 1.9602687914704884, "language_loss": 0.81861597, "learning_rate": 3.812571217517847e-06, "loss": 0.84021604, "num_input_tokens_seen": 50070080, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.84765625, "step": 2312, "time_per_iteration": 2.4183192253112793 }, { "auxiliary_loss_clip": 0.01115909, "auxiliary_loss_mlp": 0.01043386, "balance_loss_clip": 1.02151084, "balance_loss_mlp": 1.03175282, "epoch": 0.13906508342101306, "flos": 26755758512640.0, "grad_norm": 1.6884825216715873, "language_loss": 0.86466634, "learning_rate": 3.8124114925919234e-06, "loss": 0.88625932, "num_input_tokens_seen": 50090040, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.83984375, "step": 2313, "time_per_iteration": 2.4165217876434326 }, { "auxiliary_loss_clip": 0.01113991, "auxiliary_loss_mlp": 0.01050481, "balance_loss_clip": 1.02727151, "balance_loss_mlp": 1.03228283, "epoch": 0.13912520667368106, "flos": 24533955544320.0, "grad_norm": 1.9420178761846076, "language_loss": 0.79697347, "learning_rate": 3.812251702985696e-06, "loss": 0.81861818, "num_input_tokens_seen": 50110595, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.8203125, "step": 2314, "time_per_iteration": 2.4216740131378174 }, { "auxiliary_loss_clip": 0.01115102, "auxiliary_loss_mlp": 0.01042126, "balance_loss_clip": 1.01780772, "balance_loss_mlp": 1.03328633, "epoch": 0.13918532992634902, "flos": 19382416502400.0, "grad_norm": 6.510030730031837, "language_loss": 0.85251737, "learning_rate": 3.8120918487048673e-06, "loss": 0.87408972, "num_input_tokens_seen": 50125430, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.81640625, "step": 2315, "time_per_iteration": 2.3296632766723633 }, { "auxiliary_loss_clip": 0.01113169, "auxiliary_loss_mlp": 0.01047482, "balance_loss_clip": 1.02389026, "balance_loss_mlp": 1.03022432, "epoch": 0.139245453179017, "flos": 21322714245120.0, "grad_norm": 2.1342939804528664, "language_loss": 0.77397943, "learning_rate": 3.8119319297551417e-06, "loss": 0.79558593, "num_input_tokens_seen": 50144120, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.828125, "step": 2316, "time_per_iteration": 2.3798413276672363 }, { "auxiliary_loss_clip": 0.01112326, "auxiliary_loss_mlp": 0.01044963, "balance_loss_clip": 1.02010787, "balance_loss_mlp": 1.03112698, "epoch": 0.13930557643168495, "flos": 19499584625280.0, "grad_norm": 1.6088289410619419, "language_loss": 0.76960433, "learning_rate": 3.811771946142226e-06, "loss": 0.79117715, "num_input_tokens_seen": 50162500, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.8125, "step": 2317, "time_per_iteration": 2.3655471801757812 }, { "auxiliary_loss_clip": 0.01115677, "auxiliary_loss_mlp": 0.01044519, "balance_loss_clip": 1.02172649, "balance_loss_mlp": 1.03252137, "epoch": 0.13936569968435292, "flos": 25409647065600.0, "grad_norm": 1.8353313151848425, "language_loss": 0.80771768, "learning_rate": 3.8116118978718298e-06, "loss": 0.82931966, "num_input_tokens_seen": 50182415, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.83203125, "step": 2318, "time_per_iteration": 2.4340999126434326 }, { "auxiliary_loss_clip": 0.01022548, "auxiliary_loss_mlp": 0.01004856, "balance_loss_clip": 1.00195885, "balance_loss_mlp": 1.00148022, "epoch": 0.13942582293702088, "flos": 70767372481920.0, "grad_norm": 0.8539343675442279, "language_loss": 0.59066468, "learning_rate": 3.811451784949665e-06, "loss": 0.61093873, "num_input_tokens_seen": 50245160, "router_z_loss_clip": 0.02893066, "router_z_loss_mlp": 0.2109375, "step": 2319, "time_per_iteration": 3.0526671409606934 }, { "auxiliary_loss_clip": 0.01116876, "auxiliary_loss_mlp": 0.01048714, "balance_loss_clip": 1.02567124, "balance_loss_mlp": 1.03278899, "epoch": 0.13948594618968888, "flos": 35589412627200.0, "grad_norm": 2.446227278608528, "language_loss": 0.65113854, "learning_rate": 3.811291607381446e-06, "loss": 0.67279446, "num_input_tokens_seen": 50268215, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.84375, "step": 2320, "time_per_iteration": 2.50424861907959 }, { "auxiliary_loss_clip": 0.01112892, "auxiliary_loss_mlp": 0.01039067, "balance_loss_clip": 1.01576197, "balance_loss_mlp": 1.03142297, "epoch": 0.13954606944235684, "flos": 21104157934080.0, "grad_norm": 1.5250669828234587, "language_loss": 0.70898479, "learning_rate": 3.8111313651728887e-06, "loss": 0.73050439, "num_input_tokens_seen": 50288575, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.81640625, "step": 2321, "time_per_iteration": 2.387446165084839 }, { "auxiliary_loss_clip": 0.0111377, "auxiliary_loss_mlp": 0.01048863, "balance_loss_clip": 1.02609396, "balance_loss_mlp": 1.03025997, "epoch": 0.1396061926950248, "flos": 25043303502720.0, "grad_norm": 1.8167873724049057, "language_loss": 0.85633826, "learning_rate": 3.810971058329712e-06, "loss": 0.8779645, "num_input_tokens_seen": 50308735, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.8359375, "step": 2322, "time_per_iteration": 2.4280426502227783 }, { "auxiliary_loss_clip": 0.01107724, "auxiliary_loss_mlp": 0.010402, "balance_loss_clip": 1.01770544, "balance_loss_mlp": 1.02950573, "epoch": 0.13966631594769277, "flos": 37632495012480.0, "grad_norm": 1.7811434630357614, "language_loss": 0.67362523, "learning_rate": 3.810810686857636e-06, "loss": 0.69510448, "num_input_tokens_seen": 50331025, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.78125, "step": 2323, "time_per_iteration": 2.543933629989624 }, { "auxiliary_loss_clip": 0.01119111, "auxiliary_loss_mlp": 0.01041933, "balance_loss_clip": 1.01809096, "balance_loss_mlp": 1.03224778, "epoch": 0.13972643920036074, "flos": 16690053962880.0, "grad_norm": 1.88168631675888, "language_loss": 0.88742232, "learning_rate": 3.8106502507623847e-06, "loss": 0.90903276, "num_input_tokens_seen": 50349725, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.8671875, "step": 2324, "time_per_iteration": 2.3862104415893555 }, { "auxiliary_loss_clip": 0.01114269, "auxiliary_loss_mlp": 0.01046201, "balance_loss_clip": 1.02154899, "balance_loss_mlp": 1.02962196, "epoch": 0.1397865624530287, "flos": 23329940797440.0, "grad_norm": 2.4616348831774024, "language_loss": 0.70485055, "learning_rate": 3.810489750049684e-06, "loss": 0.72645521, "num_input_tokens_seen": 50367965, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.84375, "step": 2325, "time_per_iteration": 2.3760411739349365 }, { "auxiliary_loss_clip": 0.01114765, "auxiliary_loss_mlp": 0.0104497, "balance_loss_clip": 1.02260661, "balance_loss_mlp": 1.03328538, "epoch": 0.13984668570569667, "flos": 22777370709120.0, "grad_norm": 2.194120627576144, "language_loss": 0.81632841, "learning_rate": 3.810329184725261e-06, "loss": 0.83792573, "num_input_tokens_seen": 50385605, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.81640625, "step": 2326, "time_per_iteration": 2.393697500228882 }, { "auxiliary_loss_clip": 0.01113238, "auxiliary_loss_mlp": 0.01043437, "balance_loss_clip": 1.02263474, "balance_loss_mlp": 1.0323596, "epoch": 0.13990680895836466, "flos": 19463519324160.0, "grad_norm": 1.679143758752739, "language_loss": 0.88916981, "learning_rate": 3.8101685547948456e-06, "loss": 0.91073656, "num_input_tokens_seen": 50403985, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.80859375, "step": 2327, "time_per_iteration": 2.3681604862213135 }, { "auxiliary_loss_clip": 0.01111782, "auxiliary_loss_mlp": 0.01051522, "balance_loss_clip": 1.02977872, "balance_loss_mlp": 1.03227198, "epoch": 0.13996693221103262, "flos": 20302237848960.0, "grad_norm": 2.3029894270625078, "language_loss": 0.84684706, "learning_rate": 3.8100078602641714e-06, "loss": 0.86848009, "num_input_tokens_seen": 50421590, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.796875, "step": 2328, "time_per_iteration": 2.3757174015045166 }, { "auxiliary_loss_clip": 0.01113621, "auxiliary_loss_mlp": 0.01045352, "balance_loss_clip": 1.02246439, "balance_loss_mlp": 1.03034067, "epoch": 0.1400270554637006, "flos": 26616419809920.0, "grad_norm": 1.5164445313294197, "language_loss": 0.74061275, "learning_rate": 3.8098471011389723e-06, "loss": 0.7622025, "num_input_tokens_seen": 50443945, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.83203125, "step": 2329, "time_per_iteration": 2.428548574447632 }, { "auxiliary_loss_clip": 0.01112129, "auxiliary_loss_mlp": 0.01044525, "balance_loss_clip": 1.02216148, "balance_loss_mlp": 1.02910054, "epoch": 0.14008717871636855, "flos": 19390446115200.0, "grad_norm": 2.297219445526394, "language_loss": 0.7825973, "learning_rate": 3.809686277424986e-06, "loss": 0.80416381, "num_input_tokens_seen": 50462065, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.828125, "step": 2330, "time_per_iteration": 2.3700814247131348 }, { "auxiliary_loss_clip": 0.01110938, "auxiliary_loss_mlp": 0.01038475, "balance_loss_clip": 1.01670766, "balance_loss_mlp": 1.03057289, "epoch": 0.14014730196903652, "flos": 15303373649280.0, "grad_norm": 2.6356683996312147, "language_loss": 0.71626061, "learning_rate": 3.809525389127951e-06, "loss": 0.7377547, "num_input_tokens_seen": 50479565, "router_z_loss_clip": 0.21777344, "router_z_loss_mlp": 0.8046875, "step": 2331, "time_per_iteration": 2.3808858394622803 }, { "auxiliary_loss_clip": 0.01108511, "auxiliary_loss_mlp": 0.01043533, "balance_loss_clip": 1.02311277, "balance_loss_mlp": 1.03137755, "epoch": 0.14020742522170448, "flos": 14938810565760.0, "grad_norm": 1.8915342608415047, "language_loss": 0.7251972, "learning_rate": 3.8093644362536094e-06, "loss": 0.74671763, "num_input_tokens_seen": 50497305, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.7734375, "step": 2332, "time_per_iteration": 2.3650176525115967 }, { "auxiliary_loss_clip": 0.01022938, "auxiliary_loss_mlp": 0.01007537, "balance_loss_clip": 1.00495028, "balance_loss_mlp": 1.00183344, "epoch": 0.14026754847437245, "flos": 48822017316480.0, "grad_norm": 0.8126341124100568, "language_loss": 0.56089938, "learning_rate": 3.809203418807706e-06, "loss": 0.58120418, "num_input_tokens_seen": 50549735, "router_z_loss_clip": 0.02587891, "router_z_loss_mlp": 0.2109375, "step": 2333, "time_per_iteration": 2.8726682662963867 }, { "auxiliary_loss_clip": 0.01113125, "auxiliary_loss_mlp": 0.01046742, "balance_loss_clip": 1.02356756, "balance_loss_mlp": 1.03146529, "epoch": 0.14032767172704044, "flos": 25772150378880.0, "grad_norm": 1.6596230994853203, "language_loss": 0.82564056, "learning_rate": 3.8090423367959862e-06, "loss": 0.84723926, "num_input_tokens_seen": 50570100, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.81640625, "step": 2334, "time_per_iteration": 2.431256055831909 }, { "auxiliary_loss_clip": 0.01109192, "auxiliary_loss_mlp": 0.01039683, "balance_loss_clip": 1.01901162, "balance_loss_mlp": 1.02880728, "epoch": 0.1403877949797084, "flos": 21215216568960.0, "grad_norm": 1.8070704326569684, "language_loss": 0.81511354, "learning_rate": 3.8088811902241984e-06, "loss": 0.83660233, "num_input_tokens_seen": 50589185, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.8046875, "step": 2335, "time_per_iteration": 2.418689012527466 }, { "auxiliary_loss_clip": 0.01118209, "auxiliary_loss_mlp": 0.01050244, "balance_loss_clip": 1.02465022, "balance_loss_mlp": 1.03229046, "epoch": 0.14044791823237637, "flos": 22746856314240.0, "grad_norm": 1.5857080281952594, "language_loss": 0.8213681, "learning_rate": 3.8087199790980943e-06, "loss": 0.84305263, "num_input_tokens_seen": 50609645, "router_z_loss_clip": 0.25585938, "router_z_loss_mlp": 0.859375, "step": 2336, "time_per_iteration": 2.3965723514556885 }, { "auxiliary_loss_clip": 0.01112836, "auxiliary_loss_mlp": 0.01041006, "balance_loss_clip": 1.01849961, "balance_loss_mlp": 1.03063273, "epoch": 0.14050804148504434, "flos": 22963387766400.0, "grad_norm": 1.6075359232070303, "language_loss": 0.80349731, "learning_rate": 3.8085587034234268e-06, "loss": 0.82503575, "num_input_tokens_seen": 50628385, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.8203125, "step": 2337, "time_per_iteration": 2.393561363220215 }, { "auxiliary_loss_clip": 0.01115161, "auxiliary_loss_mlp": 0.01051235, "balance_loss_clip": 1.02847803, "balance_loss_mlp": 1.03135502, "epoch": 0.1405681647377123, "flos": 22199243639040.0, "grad_norm": 3.0750946669038184, "language_loss": 0.79212838, "learning_rate": 3.8083973632059507e-06, "loss": 0.81379235, "num_input_tokens_seen": 50647260, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.83984375, "step": 2338, "time_per_iteration": 2.384571075439453 }, { "auxiliary_loss_clip": 0.01119153, "auxiliary_loss_mlp": 0.01042763, "balance_loss_clip": 1.01802742, "balance_loss_mlp": 1.03551793, "epoch": 0.14062828799038027, "flos": 23731651434240.0, "grad_norm": 2.5063985567556615, "language_loss": 0.79717278, "learning_rate": 3.8082359584514254e-06, "loss": 0.81879199, "num_input_tokens_seen": 50666130, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.8359375, "step": 2339, "time_per_iteration": 2.4049720764160156 }, { "auxiliary_loss_clip": 0.01113783, "auxiliary_loss_mlp": 0.01044787, "balance_loss_clip": 1.02204204, "balance_loss_mlp": 1.03190649, "epoch": 0.14068841124304826, "flos": 39200933197440.0, "grad_norm": 2.0345611069085847, "language_loss": 0.65627486, "learning_rate": 3.8080744891656095e-06, "loss": 0.67786056, "num_input_tokens_seen": 50687440, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.8203125, "step": 2340, "time_per_iteration": 2.5427424907684326 }, { "auxiliary_loss_clip": 0.01112406, "auxiliary_loss_mlp": 0.01043054, "balance_loss_clip": 1.01995158, "balance_loss_mlp": 1.03261447, "epoch": 0.14074853449571623, "flos": 20191283948160.0, "grad_norm": 2.969123965413322, "language_loss": 0.77967715, "learning_rate": 3.807912955354266e-06, "loss": 0.80123174, "num_input_tokens_seen": 50704030, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.796875, "step": 2341, "time_per_iteration": 2.3721840381622314 }, { "auxiliary_loss_clip": 0.0110888, "auxiliary_loss_mlp": 0.01044925, "balance_loss_clip": 1.02176309, "balance_loss_mlp": 1.03030443, "epoch": 0.1408086577483842, "flos": 18404882945280.0, "grad_norm": 1.8885129295917387, "language_loss": 0.80213922, "learning_rate": 3.80775135702316e-06, "loss": 0.8236773, "num_input_tokens_seen": 50723305, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.78515625, "step": 2342, "time_per_iteration": 2.3720855712890625 }, { "auxiliary_loss_clip": 0.01110901, "auxiliary_loss_mlp": 0.01046541, "balance_loss_clip": 1.0253576, "balance_loss_mlp": 1.03243005, "epoch": 0.14086878100105216, "flos": 25263430824960.0, "grad_norm": 1.9349407206328697, "language_loss": 0.78248572, "learning_rate": 3.8075896941780576e-06, "loss": 0.80406016, "num_input_tokens_seen": 50743270, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.78515625, "step": 2343, "time_per_iteration": 2.4285085201263428 }, { "auxiliary_loss_clip": 0.01024611, "auxiliary_loss_mlp": 0.01005424, "balance_loss_clip": 1.00275409, "balance_loss_mlp": 1.00350428, "epoch": 0.14092890425372012, "flos": 65975194730880.0, "grad_norm": 0.9113845422778512, "language_loss": 0.61496663, "learning_rate": 3.807427966824729e-06, "loss": 0.63526696, "num_input_tokens_seen": 50802710, "router_z_loss_clip": 0.0267334, "router_z_loss_mlp": 0.2109375, "step": 2344, "time_per_iteration": 2.959873914718628 }, { "auxiliary_loss_clip": 0.01110289, "auxiliary_loss_mlp": 0.0104094, "balance_loss_clip": 1.01946998, "balance_loss_mlp": 1.03021097, "epoch": 0.1409890275063881, "flos": 23693875476480.0, "grad_norm": 1.5345214861639942, "language_loss": 0.644485, "learning_rate": 3.807266174968946e-06, "loss": 0.66599727, "num_input_tokens_seen": 50822625, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.80078125, "step": 2345, "time_per_iteration": 2.4022178649902344 }, { "auxiliary_loss_clip": 0.0111492, "auxiliary_loss_mlp": 0.0103954, "balance_loss_clip": 1.01692581, "balance_loss_mlp": 1.02984154, "epoch": 0.14104915075905605, "flos": 23622024165120.0, "grad_norm": 3.732364891862398, "language_loss": 0.72913074, "learning_rate": 3.8071043186164813e-06, "loss": 0.75067532, "num_input_tokens_seen": 50842330, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.8515625, "step": 2346, "time_per_iteration": 3.8668501377105713 }, { "auxiliary_loss_clip": 0.01115043, "auxiliary_loss_mlp": 0.01048066, "balance_loss_clip": 1.02484357, "balance_loss_mlp": 1.03207099, "epoch": 0.14110927401172405, "flos": 20594111748480.0, "grad_norm": 3.1926601726259403, "language_loss": 0.77061605, "learning_rate": 3.8069423977731123e-06, "loss": 0.79224718, "num_input_tokens_seen": 50861035, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.828125, "step": 2347, "time_per_iteration": 3.761927366256714 }, { "auxiliary_loss_clip": 0.01112526, "auxiliary_loss_mlp": 0.01043529, "balance_loss_clip": 1.02202344, "balance_loss_mlp": 1.02968144, "epoch": 0.141169397264392, "flos": 28546802726400.0, "grad_norm": 2.338335469689385, "language_loss": 0.76286185, "learning_rate": 3.8067804124446167e-06, "loss": 0.78442234, "num_input_tokens_seen": 50880105, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.828125, "step": 2348, "time_per_iteration": 2.4172322750091553 }, { "auxiliary_loss_clip": 0.01113536, "auxiliary_loss_mlp": 0.01045398, "balance_loss_clip": 1.02209258, "balance_loss_mlp": 1.03189969, "epoch": 0.14122952051705998, "flos": 17091310752000.0, "grad_norm": 1.714294255083456, "language_loss": 0.86320311, "learning_rate": 3.806618362636776e-06, "loss": 0.88479245, "num_input_tokens_seen": 50897720, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.8203125, "step": 2349, "time_per_iteration": 3.782675266265869 }, { "auxiliary_loss_clip": 0.01112457, "auxiliary_loss_mlp": 0.01042913, "balance_loss_clip": 1.02048922, "balance_loss_mlp": 1.03230882, "epoch": 0.14128964376972794, "flos": 28945615720320.0, "grad_norm": 1.6260149264212769, "language_loss": 0.89123261, "learning_rate": 3.806456248355373e-06, "loss": 0.91278625, "num_input_tokens_seen": 50918385, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.80078125, "step": 2350, "time_per_iteration": 3.825753927230835 }, { "auxiliary_loss_clip": 0.01117982, "auxiliary_loss_mlp": 0.01044442, "balance_loss_clip": 1.02080297, "balance_loss_mlp": 1.03451514, "epoch": 0.1413497670223959, "flos": 18988770389760.0, "grad_norm": 1.6992135889614395, "language_loss": 0.81226486, "learning_rate": 3.806294069606194e-06, "loss": 0.83388907, "num_input_tokens_seen": 50938270, "router_z_loss_clip": 0.23632812, "router_z_loss_mlp": 0.8359375, "step": 2351, "time_per_iteration": 2.3727540969848633 }, { "auxiliary_loss_clip": 0.01115487, "auxiliary_loss_mlp": 0.01041299, "balance_loss_clip": 1.01912642, "balance_loss_mlp": 1.03287506, "epoch": 0.14140989027506387, "flos": 29860933501440.0, "grad_norm": 2.4282329595428696, "language_loss": 0.83351785, "learning_rate": 3.806131826395025e-06, "loss": 0.85508567, "num_input_tokens_seen": 50958155, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.828125, "step": 2352, "time_per_iteration": 2.443225860595703 }, { "auxiliary_loss_clip": 0.01023041, "auxiliary_loss_mlp": 0.01003801, "balance_loss_clip": 1.001441, "balance_loss_mlp": 1.00168443, "epoch": 0.14147001352773186, "flos": 62076303826560.0, "grad_norm": 0.9060145135068637, "language_loss": 0.61919022, "learning_rate": 3.805969518727658e-06, "loss": 0.63945866, "num_input_tokens_seen": 51020705, "router_z_loss_clip": 0.02355957, "router_z_loss_mlp": 0.21289062, "step": 2353, "time_per_iteration": 2.951414108276367 }, { "auxiliary_loss_clip": 0.01110853, "auxiliary_loss_mlp": 0.01044185, "balance_loss_clip": 1.02258432, "balance_loss_mlp": 1.0318141, "epoch": 0.14153013678039983, "flos": 22016438426880.0, "grad_norm": 1.6978982977159032, "language_loss": 0.87054855, "learning_rate": 3.805807146609884e-06, "loss": 0.8920989, "num_input_tokens_seen": 51039995, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.7890625, "step": 2354, "time_per_iteration": 2.403468132019043 }, { "auxiliary_loss_clip": 0.01113959, "auxiliary_loss_mlp": 0.0104906, "balance_loss_clip": 1.02529013, "balance_loss_mlp": 1.03166056, "epoch": 0.1415902600330678, "flos": 19719048631680.0, "grad_norm": 2.191536556128632, "language_loss": 0.74257559, "learning_rate": 3.8056447100474976e-06, "loss": 0.76420581, "num_input_tokens_seen": 51059075, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.82421875, "step": 2355, "time_per_iteration": 2.3797683715820312 }, { "auxiliary_loss_clip": 0.01022638, "auxiliary_loss_mlp": 0.01004672, "balance_loss_clip": 1.00213301, "balance_loss_mlp": 1.00106001, "epoch": 0.14165038328573576, "flos": 65897862336000.0, "grad_norm": 0.6813695692162474, "language_loss": 0.51837111, "learning_rate": 3.8054822090462963e-06, "loss": 0.53864413, "num_input_tokens_seen": 51120380, "router_z_loss_clip": 0.02539062, "router_z_loss_mlp": 0.21484375, "step": 2356, "time_per_iteration": 2.9988350868225098 }, { "auxiliary_loss_clip": 0.01111905, "auxiliary_loss_mlp": 0.01049433, "balance_loss_clip": 1.0279392, "balance_loss_mlp": 1.03195858, "epoch": 0.14171050653840372, "flos": 12129349726080.0, "grad_norm": 2.2499940840965778, "language_loss": 0.71124399, "learning_rate": 3.80531964361208e-06, "loss": 0.73285735, "num_input_tokens_seen": 51136950, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.796875, "step": 2357, "time_per_iteration": 2.3525867462158203 }, { "auxiliary_loss_clip": 0.01115533, "auxiliary_loss_mlp": 0.01045419, "balance_loss_clip": 1.02374637, "balance_loss_mlp": 1.0324074, "epoch": 0.1417706297910717, "flos": 20411446181760.0, "grad_norm": 3.382316394087526, "language_loss": 0.81723762, "learning_rate": 3.8051570137506485e-06, "loss": 0.8388471, "num_input_tokens_seen": 51155175, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.83203125, "step": 2358, "time_per_iteration": 2.3947958946228027 }, { "auxiliary_loss_clip": 0.01116677, "auxiliary_loss_mlp": 0.0104764, "balance_loss_clip": 1.02472782, "balance_loss_mlp": 1.03283024, "epoch": 0.14183075304373965, "flos": 22379570144640.0, "grad_norm": 2.021207632014741, "language_loss": 0.71728957, "learning_rate": 3.804994319467807e-06, "loss": 0.73893273, "num_input_tokens_seen": 51174500, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.8359375, "step": 2359, "time_per_iteration": 2.4015228748321533 }, { "auxiliary_loss_clip": 0.01110608, "auxiliary_loss_mlp": 0.01036418, "balance_loss_clip": 1.01448345, "balance_loss_mlp": 1.03044295, "epoch": 0.14189087629640765, "flos": 21579814563840.0, "grad_norm": 2.0088275417241963, "language_loss": 0.75609106, "learning_rate": 3.804831560769361e-06, "loss": 0.77756137, "num_input_tokens_seen": 51194270, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.80078125, "step": 2360, "time_per_iteration": 2.3866117000579834 }, { "auxiliary_loss_clip": 0.01111825, "auxiliary_loss_mlp": 0.01045388, "balance_loss_clip": 1.02290535, "balance_loss_mlp": 1.0320853, "epoch": 0.1419509995490756, "flos": 20007605952000.0, "grad_norm": 1.9198538889155847, "language_loss": 0.81491876, "learning_rate": 3.8046687376611196e-06, "loss": 0.83649093, "num_input_tokens_seen": 51211850, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.796875, "step": 2361, "time_per_iteration": 2.364715337753296 }, { "auxiliary_loss_clip": 0.0111196, "auxiliary_loss_mlp": 0.01043956, "balance_loss_clip": 1.02096045, "balance_loss_mlp": 1.0317812, "epoch": 0.14201112280174358, "flos": 31940116099200.0, "grad_norm": 1.9110472055203933, "language_loss": 0.74089873, "learning_rate": 3.8045058501488927e-06, "loss": 0.76245791, "num_input_tokens_seen": 51233545, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.8046875, "step": 2362, "time_per_iteration": 2.4871034622192383 }, { "auxiliary_loss_clip": 0.01113443, "auxiliary_loss_mlp": 0.01040609, "balance_loss_clip": 1.01844823, "balance_loss_mlp": 1.03260601, "epoch": 0.14207124605441154, "flos": 41462536982400.0, "grad_norm": 1.6877583599104975, "language_loss": 0.73817307, "learning_rate": 3.804342898238494e-06, "loss": 0.75971359, "num_input_tokens_seen": 51257615, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.80859375, "step": 2363, "time_per_iteration": 2.5701701641082764 }, { "auxiliary_loss_clip": 0.01111354, "auxiliary_loss_mlp": 0.01042218, "balance_loss_clip": 1.02141619, "balance_loss_mlp": 1.03112841, "epoch": 0.1421313693070795, "flos": 31903736595840.0, "grad_norm": 1.77503551669461, "language_loss": 0.72893143, "learning_rate": 3.8041798819357386e-06, "loss": 0.75046718, "num_input_tokens_seen": 51279645, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.8046875, "step": 2364, "time_per_iteration": 2.4738662242889404 }, { "auxiliary_loss_clip": 0.01108281, "auxiliary_loss_mlp": 0.01039316, "balance_loss_clip": 1.01900244, "balance_loss_mlp": 1.03060389, "epoch": 0.14219149255974747, "flos": 26869924258560.0, "grad_norm": 2.37641305853089, "language_loss": 0.9059546, "learning_rate": 3.804016801246444e-06, "loss": 0.92743063, "num_input_tokens_seen": 51299775, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.77734375, "step": 2365, "time_per_iteration": 2.4367516040802 }, { "auxiliary_loss_clip": 0.01110698, "auxiliary_loss_mlp": 0.01040614, "balance_loss_clip": 1.01842928, "balance_loss_mlp": 1.02955818, "epoch": 0.14225161581241544, "flos": 27453183298560.0, "grad_norm": 1.680012895617236, "language_loss": 0.65590346, "learning_rate": 3.80385365617643e-06, "loss": 0.67741668, "num_input_tokens_seen": 51319430, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.8125, "step": 2366, "time_per_iteration": 2.4501473903656006 }, { "auxiliary_loss_clip": 0.01107549, "auxiliary_loss_mlp": 0.01038612, "balance_loss_clip": 1.01609302, "balance_loss_mlp": 1.02917778, "epoch": 0.14231173906508343, "flos": 10560667161600.0, "grad_norm": 2.2568984621358297, "language_loss": 0.80072278, "learning_rate": 3.8036904467315196e-06, "loss": 0.82218438, "num_input_tokens_seen": 51336045, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.78125, "step": 2367, "time_per_iteration": 2.36179256439209 }, { "auxiliary_loss_clip": 0.01115014, "auxiliary_loss_mlp": 0.0105425, "balance_loss_clip": 1.03032434, "balance_loss_mlp": 1.03221178, "epoch": 0.1423718623177514, "flos": 28359773239680.0, "grad_norm": 2.2982741802170543, "language_loss": 0.82969856, "learning_rate": 3.8035271729175366e-06, "loss": 0.8513912, "num_input_tokens_seen": 51357030, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.828125, "step": 2368, "time_per_iteration": 2.4400994777679443 }, { "auxiliary_loss_clip": 0.01111815, "auxiliary_loss_mlp": 0.01049996, "balance_loss_clip": 1.02657127, "balance_loss_mlp": 1.03188169, "epoch": 0.14243198557041936, "flos": 19353228739200.0, "grad_norm": 2.225072538149742, "language_loss": 0.86660296, "learning_rate": 3.803363834740308e-06, "loss": 0.88822114, "num_input_tokens_seen": 51374890, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.796875, "step": 2369, "time_per_iteration": 2.3709802627563477 }, { "auxiliary_loss_clip": 0.01111436, "auxiliary_loss_mlp": 0.0104525, "balance_loss_clip": 1.021909, "balance_loss_mlp": 1.0288223, "epoch": 0.14249210882308733, "flos": 28805508967680.0, "grad_norm": 1.5397197418609387, "language_loss": 0.7586726, "learning_rate": 3.8032004322056627e-06, "loss": 0.7802394, "num_input_tokens_seen": 51398100, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.828125, "step": 2370, "time_per_iteration": 2.4590070247650146 }, { "auxiliary_loss_clip": 0.01111055, "auxiliary_loss_mlp": 0.01046763, "balance_loss_clip": 1.02507842, "balance_loss_mlp": 1.03099585, "epoch": 0.1425522320757553, "flos": 21833947416960.0, "grad_norm": 1.7583329430782595, "language_loss": 0.83001393, "learning_rate": 3.8030369653194326e-06, "loss": 0.85159212, "num_input_tokens_seen": 51418745, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.80078125, "step": 2371, "time_per_iteration": 2.3887088298797607 }, { "auxiliary_loss_clip": 0.01113674, "auxiliary_loss_mlp": 0.0104464, "balance_loss_clip": 1.02160835, "balance_loss_mlp": 1.03213429, "epoch": 0.14261235532842326, "flos": 17310495467520.0, "grad_norm": 1.9612660560625004, "language_loss": 0.82770348, "learning_rate": 3.802873434087451e-06, "loss": 0.84928668, "num_input_tokens_seen": 51437455, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.81640625, "step": 2372, "time_per_iteration": 2.3595197200775146 }, { "auxiliary_loss_clip": 0.01111961, "auxiliary_loss_mlp": 0.0104377, "balance_loss_clip": 1.02153802, "balance_loss_mlp": 1.03225899, "epoch": 0.14267247858109125, "flos": 18805755709440.0, "grad_norm": 3.171101260252954, "language_loss": 0.84952039, "learning_rate": 3.8027098385155546e-06, "loss": 0.87107772, "num_input_tokens_seen": 51455710, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.796875, "step": 2373, "time_per_iteration": 2.3554599285125732 }, { "auxiliary_loss_clip": 0.01108555, "auxiliary_loss_mlp": 0.01047167, "balance_loss_clip": 1.0272826, "balance_loss_mlp": 1.02993953, "epoch": 0.14273260183375922, "flos": 11358258238080.0, "grad_norm": 1.9497593446538968, "language_loss": 0.85978901, "learning_rate": 3.802546178609581e-06, "loss": 0.88134623, "num_input_tokens_seen": 51471270, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.78515625, "step": 2374, "time_per_iteration": 2.3471994400024414 }, { "auxiliary_loss_clip": 0.01116023, "auxiliary_loss_mlp": 0.01048151, "balance_loss_clip": 1.02371335, "balance_loss_mlp": 1.03108776, "epoch": 0.14279272508642718, "flos": 27566336615040.0, "grad_norm": 1.6567888506182693, "language_loss": 0.79175425, "learning_rate": 3.8023824543753706e-06, "loss": 0.81339598, "num_input_tokens_seen": 51492705, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.8515625, "step": 2375, "time_per_iteration": 2.4538514614105225 }, { "auxiliary_loss_clip": 0.01114544, "auxiliary_loss_mlp": 0.01052575, "balance_loss_clip": 1.0297699, "balance_loss_mlp": 1.03305233, "epoch": 0.14285284833909515, "flos": 16251649620480.0, "grad_norm": 2.5280824056824294, "language_loss": 0.76490855, "learning_rate": 3.802218665818767e-06, "loss": 0.78657973, "num_input_tokens_seen": 51510780, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.81640625, "step": 2376, "time_per_iteration": 2.356001138687134 }, { "auxiliary_loss_clip": 0.01111805, "auxiliary_loss_mlp": 0.01041119, "balance_loss_clip": 1.0195061, "balance_loss_mlp": 1.03157842, "epoch": 0.1429129715917631, "flos": 19754590262400.0, "grad_norm": 1.822378925866651, "language_loss": 0.93101025, "learning_rate": 3.802054812945615e-06, "loss": 0.95253944, "num_input_tokens_seen": 51531400, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.8046875, "step": 2377, "time_per_iteration": 2.403977870941162 }, { "auxiliary_loss_clip": 0.01109632, "auxiliary_loss_mlp": 0.0104081, "balance_loss_clip": 1.01652718, "balance_loss_mlp": 1.02863622, "epoch": 0.14297309484443108, "flos": 21136173517440.0, "grad_norm": 2.0131222186116404, "language_loss": 0.91564405, "learning_rate": 3.801890895761762e-06, "loss": 0.93714845, "num_input_tokens_seen": 51548215, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.8125, "step": 2378, "time_per_iteration": 2.373382091522217 }, { "auxiliary_loss_clip": 0.01113144, "auxiliary_loss_mlp": 0.01038751, "balance_loss_clip": 1.0161612, "balance_loss_mlp": 1.02985239, "epoch": 0.14303321809709904, "flos": 23585539927680.0, "grad_norm": 1.6098071087419281, "language_loss": 0.73419136, "learning_rate": 3.8017269142730584e-06, "loss": 0.75571024, "num_input_tokens_seen": 51566820, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.83203125, "step": 2379, "time_per_iteration": 2.3976776599884033 }, { "auxiliary_loss_clip": 0.01108895, "auxiliary_loss_mlp": 0.01051303, "balance_loss_clip": 1.02926159, "balance_loss_mlp": 1.0289185, "epoch": 0.14309334134976703, "flos": 15887365827840.0, "grad_norm": 1.961125634825258, "language_loss": 0.78647882, "learning_rate": 3.801562868485355e-06, "loss": 0.80808079, "num_input_tokens_seen": 51585075, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.80078125, "step": 2380, "time_per_iteration": 2.3603386878967285 }, { "auxiliary_loss_clip": 0.01115999, "auxiliary_loss_mlp": 0.01042334, "balance_loss_clip": 1.0200417, "balance_loss_mlp": 1.03406525, "epoch": 0.143153464602435, "flos": 16324687918080.0, "grad_norm": 2.1120644519583975, "language_loss": 0.88181722, "learning_rate": 3.801398758404508e-06, "loss": 0.90340054, "num_input_tokens_seen": 51603185, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.8203125, "step": 2381, "time_per_iteration": 2.377686023712158 }, { "auxiliary_loss_clip": 0.01109959, "auxiliary_loss_mlp": 0.01041204, "balance_loss_clip": 1.0184474, "balance_loss_mlp": 1.0317328, "epoch": 0.14321358785510296, "flos": 17091136195200.0, "grad_norm": 2.218149750101264, "language_loss": 0.76782626, "learning_rate": 3.801234584036372e-06, "loss": 0.78933787, "num_input_tokens_seen": 51620880, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.78125, "step": 2382, "time_per_iteration": 2.4079976081848145 }, { "auxiliary_loss_clip": 0.01110099, "auxiliary_loss_mlp": 0.01046025, "balance_loss_clip": 1.02367353, "balance_loss_mlp": 1.02952552, "epoch": 0.14327371110777093, "flos": 26321718090240.0, "grad_norm": 2.2360123700006933, "language_loss": 0.76984197, "learning_rate": 3.801070345386808e-06, "loss": 0.79140317, "num_input_tokens_seen": 51640170, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.8046875, "step": 2383, "time_per_iteration": 2.426485776901245 }, { "auxiliary_loss_clip": 0.01112413, "auxiliary_loss_mlp": 0.01048505, "balance_loss_clip": 1.02380538, "balance_loss_mlp": 1.02988279, "epoch": 0.1433338343604389, "flos": 18075512378880.0, "grad_norm": 2.3204216584220494, "language_loss": 0.87647116, "learning_rate": 3.8009060424616757e-06, "loss": 0.89808035, "num_input_tokens_seen": 51656580, "router_z_loss_clip": 0.24707031, "router_z_loss_mlp": 0.82421875, "step": 2384, "time_per_iteration": 2.3518261909484863 }, { "auxiliary_loss_clip": 0.01115697, "auxiliary_loss_mlp": 0.01045557, "balance_loss_clip": 1.02165604, "balance_loss_mlp": 1.03101289, "epoch": 0.14339395761310686, "flos": 15521895048960.0, "grad_norm": 2.254275462304245, "language_loss": 0.79344857, "learning_rate": 3.800741675266839e-06, "loss": 0.81506115, "num_input_tokens_seen": 51674645, "router_z_loss_clip": 0.23925781, "router_z_loss_mlp": 0.84375, "step": 2385, "time_per_iteration": 3.7864675521850586 }, { "auxiliary_loss_clip": 0.01109576, "auxiliary_loss_mlp": 0.01043029, "balance_loss_clip": 1.0214045, "balance_loss_mlp": 1.03019714, "epoch": 0.14345408086577485, "flos": 28547500953600.0, "grad_norm": 1.680801224843066, "language_loss": 0.75024277, "learning_rate": 3.8005772438081645e-06, "loss": 0.77176881, "num_input_tokens_seen": 51695770, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.79296875, "step": 2386, "time_per_iteration": 3.8261818885803223 }, { "auxiliary_loss_clip": 0.01111145, "auxiliary_loss_mlp": 0.01040607, "balance_loss_clip": 1.01887536, "balance_loss_mlp": 1.03130329, "epoch": 0.14351420411844282, "flos": 20229024994560.0, "grad_norm": 2.069877764197071, "language_loss": 0.78709936, "learning_rate": 3.80041274809152e-06, "loss": 0.80861688, "num_input_tokens_seen": 51714165, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.796875, "step": 2387, "time_per_iteration": 2.3673291206359863 }, { "auxiliary_loss_clip": 0.01109196, "auxiliary_loss_mlp": 0.01045684, "balance_loss_clip": 1.02303433, "balance_loss_mlp": 1.02931619, "epoch": 0.14357432737111078, "flos": 19864008063360.0, "grad_norm": 2.107675298355465, "language_loss": 0.82349843, "learning_rate": 3.8002481881227753e-06, "loss": 0.84504724, "num_input_tokens_seen": 51734440, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.796875, "step": 2388, "time_per_iteration": 2.386894702911377 }, { "auxiliary_loss_clip": 0.01111769, "auxiliary_loss_mlp": 0.01042883, "balance_loss_clip": 1.02164018, "balance_loss_mlp": 1.03092527, "epoch": 0.14363445062377875, "flos": 28255557231360.0, "grad_norm": 2.801989469982729, "language_loss": 0.82503819, "learning_rate": 3.8000835639078038e-06, "loss": 0.8465848, "num_input_tokens_seen": 51753730, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.80859375, "step": 2389, "time_per_iteration": 5.149309873580933 }, { "auxiliary_loss_clip": 0.01113311, "auxiliary_loss_mlp": 0.01045806, "balance_loss_clip": 1.02178502, "balance_loss_mlp": 1.03083026, "epoch": 0.1436945738764467, "flos": 18185698229760.0, "grad_norm": 1.9683773180119561, "language_loss": 0.83158261, "learning_rate": 3.79991887545248e-06, "loss": 0.85317379, "num_input_tokens_seen": 51771195, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.82421875, "step": 2390, "time_per_iteration": 2.348984479904175 }, { "auxiliary_loss_clip": 0.01108689, "auxiliary_loss_mlp": 0.01044275, "balance_loss_clip": 1.02272224, "balance_loss_mlp": 1.02935779, "epoch": 0.14375469712911468, "flos": 27306687767040.0, "grad_norm": 1.530518587021261, "language_loss": 0.74943447, "learning_rate": 3.799754122762682e-06, "loss": 0.77096415, "num_input_tokens_seen": 51792290, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.79296875, "step": 2391, "time_per_iteration": 2.4368858337402344 }, { "auxiliary_loss_clip": 0.0102576, "auxiliary_loss_mlp": 0.01002298, "balance_loss_clip": 0.99986637, "balance_loss_mlp": 1.00419617, "epoch": 0.14381482038178264, "flos": 56888559838080.0, "grad_norm": 1.0154385386077425, "language_loss": 0.61786532, "learning_rate": 3.7995893058442886e-06, "loss": 0.63814592, "num_input_tokens_seen": 51843675, "router_z_loss_clip": 0.02429199, "router_z_loss_mlp": 0.21582031, "step": 2392, "time_per_iteration": 2.861618995666504 }, { "auxiliary_loss_clip": 0.01111459, "auxiliary_loss_mlp": 0.01049624, "balance_loss_clip": 1.02554417, "balance_loss_mlp": 1.02828789, "epoch": 0.14387494363445064, "flos": 14281326241920.0, "grad_norm": 2.069828487465517, "language_loss": 0.76887888, "learning_rate": 3.7994244247031814e-06, "loss": 0.79048973, "num_input_tokens_seen": 51860285, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.83203125, "step": 2393, "time_per_iteration": 2.358253002166748 }, { "auxiliary_loss_clip": 0.01112962, "auxiliary_loss_mlp": 0.01042185, "balance_loss_clip": 1.02052498, "balance_loss_mlp": 1.03079224, "epoch": 0.1439350668871186, "flos": 26760262078080.0, "grad_norm": 1.8514424364669433, "language_loss": 0.76460016, "learning_rate": 3.799259479345246e-06, "loss": 0.78615165, "num_input_tokens_seen": 51880105, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.8203125, "step": 2394, "time_per_iteration": 2.4324076175689697 }, { "auxiliary_loss_clip": 0.01108887, "auxiliary_loss_mlp": 0.01045604, "balance_loss_clip": 1.02238178, "balance_loss_mlp": 1.02888703, "epoch": 0.14399519013978657, "flos": 40698392855040.0, "grad_norm": 1.6178149714271428, "language_loss": 0.86226803, "learning_rate": 3.799094469776367e-06, "loss": 0.8838129, "num_input_tokens_seen": 51905175, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.80078125, "step": 2395, "time_per_iteration": 2.5615274906158447 }, { "auxiliary_loss_clip": 0.01108712, "auxiliary_loss_mlp": 0.010458, "balance_loss_clip": 1.02436662, "balance_loss_mlp": 1.03135109, "epoch": 0.14405531339245453, "flos": 20556510347520.0, "grad_norm": 1.5520876876903358, "language_loss": 0.82945615, "learning_rate": 3.7989293960024353e-06, "loss": 0.85100126, "num_input_tokens_seen": 51924490, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.7734375, "step": 2396, "time_per_iteration": 2.386688232421875 }, { "auxiliary_loss_clip": 0.01106862, "auxiliary_loss_mlp": 0.01039103, "balance_loss_clip": 1.01921928, "balance_loss_mlp": 1.02991748, "epoch": 0.1441154366451225, "flos": 19571924695680.0, "grad_norm": 2.649614566155054, "language_loss": 0.82483745, "learning_rate": 3.79876425802934e-06, "loss": 0.84629709, "num_input_tokens_seen": 51940490, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.76953125, "step": 2397, "time_per_iteration": 2.375566005706787 }, { "auxiliary_loss_clip": 0.0111361, "auxiliary_loss_mlp": 0.01051843, "balance_loss_clip": 1.02984858, "balance_loss_mlp": 1.03093338, "epoch": 0.14417555989779046, "flos": 18514719682560.0, "grad_norm": 1.6649212933750084, "language_loss": 0.79688466, "learning_rate": 3.798599055862976e-06, "loss": 0.8185392, "num_input_tokens_seen": 51957910, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.828125, "step": 2398, "time_per_iteration": 2.3601315021514893 }, { "auxiliary_loss_clip": 0.01106993, "auxiliary_loss_mlp": 0.01051333, "balance_loss_clip": 1.02938616, "balance_loss_mlp": 1.03000951, "epoch": 0.14423568315045843, "flos": 26030472595200.0, "grad_norm": 10.976560248022666, "language_loss": 0.64585006, "learning_rate": 3.798433789509238e-06, "loss": 0.66743332, "num_input_tokens_seen": 51978010, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.76953125, "step": 2399, "time_per_iteration": 2.4182305335998535 }, { "auxiliary_loss_clip": 0.01110507, "auxiliary_loss_mlp": 0.01045423, "balance_loss_clip": 1.02336943, "balance_loss_mlp": 1.03213882, "epoch": 0.14429580640312642, "flos": 21287661373440.0, "grad_norm": 2.09594890333685, "language_loss": 0.82169539, "learning_rate": 3.798268458974024e-06, "loss": 0.84325469, "num_input_tokens_seen": 51998515, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.78515625, "step": 2400, "time_per_iteration": 2.3954548835754395 }, { "auxiliary_loss_clip": 0.01112839, "auxiliary_loss_mlp": 0.01048561, "balance_loss_clip": 1.02368224, "balance_loss_mlp": 1.03122401, "epoch": 0.14435592965579438, "flos": 25626737099520.0, "grad_norm": 2.025461941102729, "language_loss": 0.74472535, "learning_rate": 3.7981030642632348e-06, "loss": 0.76633936, "num_input_tokens_seen": 52019270, "router_z_loss_clip": 0.24902344, "router_z_loss_mlp": 0.81640625, "step": 2401, "time_per_iteration": 2.4465906620025635 }, { "auxiliary_loss_clip": 0.01109409, "auxiliary_loss_mlp": 0.01038422, "balance_loss_clip": 1.01783419, "balance_loss_mlp": 1.02983093, "epoch": 0.14441605290846235, "flos": 22963981259520.0, "grad_norm": 1.8874126218600147, "language_loss": 0.8074652, "learning_rate": 3.797937605382772e-06, "loss": 0.82894349, "num_input_tokens_seen": 52039315, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.796875, "step": 2402, "time_per_iteration": 2.3857922554016113 }, { "auxiliary_loss_clip": 0.01109774, "auxiliary_loss_mlp": 0.01045489, "balance_loss_clip": 1.02326858, "balance_loss_mlp": 1.03010798, "epoch": 0.14447617616113032, "flos": 17346700414080.0, "grad_norm": 2.4552598884483037, "language_loss": 0.84383583, "learning_rate": 3.7977720823385413e-06, "loss": 0.86538851, "num_input_tokens_seen": 52056555, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.796875, "step": 2403, "time_per_iteration": 2.3651437759399414 }, { "auxiliary_loss_clip": 0.01108275, "auxiliary_loss_mlp": 0.01047624, "balance_loss_clip": 1.02614224, "balance_loss_mlp": 1.02868319, "epoch": 0.14453629941379828, "flos": 24059066964480.0, "grad_norm": 1.9221461120669485, "language_loss": 0.69886154, "learning_rate": 3.797606495136449e-06, "loss": 0.72042048, "num_input_tokens_seen": 52075800, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.796875, "step": 2404, "time_per_iteration": 2.394455909729004 }, { "auxiliary_loss_clip": 0.01105719, "auxiliary_loss_mlp": 0.01043794, "balance_loss_clip": 1.02257442, "balance_loss_mlp": 1.02929091, "epoch": 0.14459642266646625, "flos": 14428659646080.0, "grad_norm": 1.97815403774669, "language_loss": 0.73047113, "learning_rate": 3.7974408437824055e-06, "loss": 0.75196624, "num_input_tokens_seen": 52092585, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.765625, "step": 2405, "time_per_iteration": 2.354947566986084 }, { "auxiliary_loss_clip": 0.01106846, "auxiliary_loss_mlp": 0.01041665, "balance_loss_clip": 1.0204097, "balance_loss_mlp": 1.03113937, "epoch": 0.14465654591913424, "flos": 9866314575360.0, "grad_norm": 4.372737942354544, "language_loss": 0.73072457, "learning_rate": 3.7972751282823216e-06, "loss": 0.7522096, "num_input_tokens_seen": 52108990, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.7578125, "step": 2406, "time_per_iteration": 2.3535666465759277 }, { "auxiliary_loss_clip": 0.01110209, "auxiliary_loss_mlp": 0.01046813, "balance_loss_clip": 1.02440178, "balance_loss_mlp": 1.03055358, "epoch": 0.1447166691718022, "flos": 24971766393600.0, "grad_norm": 2.2657112076268757, "language_loss": 0.75740147, "learning_rate": 3.797109348642111e-06, "loss": 0.77897167, "num_input_tokens_seen": 52125385, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.796875, "step": 2407, "time_per_iteration": 2.403923273086548 }, { "auxiliary_loss_clip": 0.01107285, "auxiliary_loss_mlp": 0.01037676, "balance_loss_clip": 1.01682675, "balance_loss_mlp": 1.02851439, "epoch": 0.14477679242447017, "flos": 21906950803200.0, "grad_norm": 1.4982171274409408, "language_loss": 0.79570168, "learning_rate": 3.796943504867691e-06, "loss": 0.81715131, "num_input_tokens_seen": 52144985, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.78515625, "step": 2408, "time_per_iteration": 2.3764305114746094 }, { "auxiliary_loss_clip": 0.01110048, "auxiliary_loss_mlp": 0.01046355, "balance_loss_clip": 1.0226922, "balance_loss_mlp": 1.0321908, "epoch": 0.14483691567713813, "flos": 20739699584640.0, "grad_norm": 25.981433853447278, "language_loss": 0.82397455, "learning_rate": 3.7967775969649796e-06, "loss": 0.84553862, "num_input_tokens_seen": 52163885, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.78125, "step": 2409, "time_per_iteration": 2.367091178894043 }, { "auxiliary_loss_clip": 0.01109693, "auxiliary_loss_mlp": 0.01047852, "balance_loss_clip": 1.02637005, "balance_loss_mlp": 1.03146958, "epoch": 0.1448970389298061, "flos": 35406258301440.0, "grad_norm": 1.7803194277442216, "language_loss": 0.74579203, "learning_rate": 3.7966116249398974e-06, "loss": 0.76736754, "num_input_tokens_seen": 52184325, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.78125, "step": 2410, "time_per_iteration": 2.5116896629333496 }, { "auxiliary_loss_clip": 0.01107985, "auxiliary_loss_mlp": 0.01041762, "balance_loss_clip": 1.02059031, "balance_loss_mlp": 1.02840614, "epoch": 0.14495716218247406, "flos": 15413454766080.0, "grad_norm": 1.8276983278577725, "language_loss": 0.8123709, "learning_rate": 3.7964455887983675e-06, "loss": 0.83386838, "num_input_tokens_seen": 52202740, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.796875, "step": 2411, "time_per_iteration": 2.356724739074707 }, { "auxiliary_loss_clip": 0.01108443, "auxiliary_loss_mlp": 0.01043607, "balance_loss_clip": 1.0213623, "balance_loss_mlp": 1.03147197, "epoch": 0.14501728543514203, "flos": 33691813344000.0, "grad_norm": 2.289403854800444, "language_loss": 0.70229125, "learning_rate": 3.7962794885463165e-06, "loss": 0.72381175, "num_input_tokens_seen": 52223100, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.76953125, "step": 2412, "time_per_iteration": 2.4828953742980957 }, { "auxiliary_loss_clip": 0.01109675, "auxiliary_loss_mlp": 0.01037952, "balance_loss_clip": 1.0168761, "balance_loss_mlp": 1.03181338, "epoch": 0.14507740868781002, "flos": 15595212637440.0, "grad_norm": 2.979334482442582, "language_loss": 0.76595402, "learning_rate": 3.7961133241896706e-06, "loss": 0.78743023, "num_input_tokens_seen": 52239690, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.78125, "step": 2413, "time_per_iteration": 2.352808713912964 }, { "auxiliary_loss_clip": 0.01028559, "auxiliary_loss_mlp": 0.01003957, "balance_loss_clip": 1.00168025, "balance_loss_mlp": 1.00727987, "epoch": 0.145137531940478, "flos": 66672026023680.0, "grad_norm": 0.8823553427400396, "language_loss": 0.58824027, "learning_rate": 3.79594709573436e-06, "loss": 0.60856533, "num_input_tokens_seen": 52296705, "router_z_loss_clip": 0.02282715, "router_z_loss_mlp": 0.21289062, "step": 2414, "time_per_iteration": 2.949816942214966 }, { "auxiliary_loss_clip": 0.01026435, "auxiliary_loss_mlp": 0.01006749, "balance_loss_clip": 1.00451934, "balance_loss_mlp": 1.00521815, "epoch": 0.14519765519314595, "flos": 67518041022720.0, "grad_norm": 0.8364423695325559, "language_loss": 0.62246674, "learning_rate": 3.7957808031863173e-06, "loss": 0.64279854, "num_input_tokens_seen": 52361830, "router_z_loss_clip": 0.02233887, "router_z_loss_mlp": 0.21191406, "step": 2415, "time_per_iteration": 3.0504634380340576 }, { "auxiliary_loss_clip": 0.01106679, "auxiliary_loss_mlp": 0.01037006, "balance_loss_clip": 1.01550126, "balance_loss_mlp": 1.02929401, "epoch": 0.14525777844581392, "flos": 17198040378240.0, "grad_norm": 1.999221124672743, "language_loss": 0.71876603, "learning_rate": 3.7956144465514775e-06, "loss": 0.7402029, "num_input_tokens_seen": 52379420, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.7734375, "step": 2416, "time_per_iteration": 2.359178304672241 }, { "auxiliary_loss_clip": 0.01025716, "auxiliary_loss_mlp": 0.01003898, "balance_loss_clip": 1.00165713, "balance_loss_mlp": 1.00491476, "epoch": 0.14531790169848188, "flos": 65401152289920.0, "grad_norm": 0.7093771423964166, "language_loss": 0.60392392, "learning_rate": 3.7954480258357765e-06, "loss": 0.62422007, "num_input_tokens_seen": 52446290, "router_z_loss_clip": 0.02246094, "router_z_loss_mlp": 0.20800781, "step": 2417, "time_per_iteration": 3.0886611938476562 }, { "auxiliary_loss_clip": 0.01111104, "auxiliary_loss_mlp": 0.01048751, "balance_loss_clip": 1.02648282, "balance_loss_mlp": 1.02903223, "epoch": 0.14537802495114985, "flos": 32561081274240.0, "grad_norm": 2.5029232264615304, "language_loss": 0.78740788, "learning_rate": 3.7952815410451542e-06, "loss": 0.80900639, "num_input_tokens_seen": 52467295, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.8203125, "step": 2418, "time_per_iteration": 2.4675533771514893 }, { "auxiliary_loss_clip": 0.01106791, "auxiliary_loss_mlp": 0.0103804, "balance_loss_clip": 1.01740527, "balance_loss_mlp": 1.02998948, "epoch": 0.1454381482038178, "flos": 20225743326720.0, "grad_norm": 2.3543541080721186, "language_loss": 0.71558362, "learning_rate": 3.7951149921855515e-06, "loss": 0.73703194, "num_input_tokens_seen": 52487295, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.765625, "step": 2419, "time_per_iteration": 2.4007863998413086 }, { "auxiliary_loss_clip": 0.01107661, "auxiliary_loss_mlp": 0.01041494, "balance_loss_clip": 1.01862967, "balance_loss_mlp": 1.03081453, "epoch": 0.1454982714564858, "flos": 22892025214080.0, "grad_norm": 2.509133605470454, "language_loss": 0.89308, "learning_rate": 3.794948379262913e-06, "loss": 0.91457152, "num_input_tokens_seen": 52504220, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.765625, "step": 2420, "time_per_iteration": 2.3839590549468994 }, { "auxiliary_loss_clip": 0.01108402, "auxiliary_loss_mlp": 0.01040117, "balance_loss_clip": 1.01805139, "balance_loss_mlp": 1.03007388, "epoch": 0.14555839470915377, "flos": 20228815526400.0, "grad_norm": 1.9597872167404442, "language_loss": 0.82786453, "learning_rate": 3.794781702283183e-06, "loss": 0.84934974, "num_input_tokens_seen": 52521900, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.78125, "step": 2421, "time_per_iteration": 2.3867626190185547 }, { "auxiliary_loss_clip": 0.01110605, "auxiliary_loss_mlp": 0.01042945, "balance_loss_clip": 1.0207957, "balance_loss_mlp": 1.03116322, "epoch": 0.14561851796182174, "flos": 22235204206080.0, "grad_norm": 1.6068226050987182, "language_loss": 0.81600267, "learning_rate": 3.7946149612523116e-06, "loss": 0.83753818, "num_input_tokens_seen": 52540495, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.79296875, "step": 2422, "time_per_iteration": 2.388953447341919 }, { "auxiliary_loss_clip": 0.01027698, "auxiliary_loss_mlp": 0.01003126, "balance_loss_clip": 1.00078976, "balance_loss_mlp": 1.00744081, "epoch": 0.1456786412144897, "flos": 52633624222080.0, "grad_norm": 0.9167793141958676, "language_loss": 0.63315821, "learning_rate": 3.794448156176248e-06, "loss": 0.65346646, "num_input_tokens_seen": 52603305, "router_z_loss_clip": 0.02331543, "router_z_loss_mlp": 0.203125, "step": 2423, "time_per_iteration": 3.024770975112915 }, { "auxiliary_loss_clip": 0.01109042, "auxiliary_loss_mlp": 0.01039125, "balance_loss_clip": 1.01955104, "balance_loss_mlp": 1.0321672, "epoch": 0.14573876446715767, "flos": 23220557907840.0, "grad_norm": 3.561868481747033, "language_loss": 0.82309031, "learning_rate": 3.794281287060946e-06, "loss": 0.84457195, "num_input_tokens_seen": 52623435, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.76953125, "step": 2424, "time_per_iteration": 3.805325984954834 }, { "auxiliary_loss_clip": 0.01108945, "auxiliary_loss_mlp": 0.01040561, "balance_loss_clip": 1.01880586, "balance_loss_mlp": 1.03246284, "epoch": 0.14579888771982563, "flos": 18113393070720.0, "grad_norm": 2.3867775837286835, "language_loss": 0.78662455, "learning_rate": 3.7941143539123596e-06, "loss": 0.80811965, "num_input_tokens_seen": 52642255, "router_z_loss_clip": 0.21777344, "router_z_loss_mlp": 0.765625, "step": 2425, "time_per_iteration": 2.3818471431732178 }, { "auxiliary_loss_clip": 0.011079, "auxiliary_loss_mlp": 0.01040786, "balance_loss_clip": 1.01992452, "balance_loss_mlp": 1.03184569, "epoch": 0.14585901097249362, "flos": 23000046560640.0, "grad_norm": 2.004185816551749, "language_loss": 0.8373462, "learning_rate": 3.7939473567364473e-06, "loss": 0.85883307, "num_input_tokens_seen": 52658700, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.76171875, "step": 2426, "time_per_iteration": 3.758450746536255 }, { "auxiliary_loss_clip": 0.01106852, "auxiliary_loss_mlp": 0.01039837, "balance_loss_clip": 1.01924932, "balance_loss_mlp": 1.03191352, "epoch": 0.1459191342251616, "flos": 21907579207680.0, "grad_norm": 1.9805063390766124, "language_loss": 0.87306172, "learning_rate": 3.793780295539169e-06, "loss": 0.89452857, "num_input_tokens_seen": 52678140, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.74609375, "step": 2427, "time_per_iteration": 3.782193899154663 }, { "auxiliary_loss_clip": 0.01112837, "auxiliary_loss_mlp": 0.01038514, "balance_loss_clip": 1.01620984, "balance_loss_mlp": 1.03108454, "epoch": 0.14597925747782955, "flos": 14974631487360.0, "grad_norm": 2.3468969024196586, "language_loss": 0.66859877, "learning_rate": 3.793613170326485e-06, "loss": 0.69011229, "num_input_tokens_seen": 52696825, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.8203125, "step": 2428, "time_per_iteration": 3.758338689804077 }, { "auxiliary_loss_clip": 0.01106052, "auxiliary_loss_mlp": 0.01040569, "balance_loss_clip": 1.01875365, "balance_loss_mlp": 1.02963817, "epoch": 0.14603938073049752, "flos": 21067848253440.0, "grad_norm": 2.593160435012226, "language_loss": 0.83403075, "learning_rate": 3.793445981104362e-06, "loss": 0.855497, "num_input_tokens_seen": 52715125, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.765625, "step": 2429, "time_per_iteration": 2.4017221927642822 }, { "auxiliary_loss_clip": 0.0110459, "auxiliary_loss_mlp": 0.01034364, "balance_loss_clip": 1.01366925, "balance_loss_mlp": 1.02881503, "epoch": 0.14609950398316549, "flos": 19863763683840.0, "grad_norm": 1.7183100895627228, "language_loss": 0.79013276, "learning_rate": 3.7932787278787643e-06, "loss": 0.81152231, "num_input_tokens_seen": 52734015, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.7578125, "step": 2430, "time_per_iteration": 2.3640503883361816 }, { "auxiliary_loss_clip": 0.01108524, "auxiliary_loss_mlp": 0.01045156, "balance_loss_clip": 1.02367413, "balance_loss_mlp": 1.0301621, "epoch": 0.14615962723583345, "flos": 22417765038720.0, "grad_norm": 2.056080596525592, "language_loss": 0.82878721, "learning_rate": 3.7931114106556618e-06, "loss": 0.85032403, "num_input_tokens_seen": 52753025, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.78515625, "step": 2431, "time_per_iteration": 2.3871424198150635 }, { "auxiliary_loss_clip": 0.01110717, "auxiliary_loss_mlp": 0.0104409, "balance_loss_clip": 1.02068925, "balance_loss_mlp": 1.03180575, "epoch": 0.14621975048850142, "flos": 22345145677440.0, "grad_norm": 1.8340758882246042, "language_loss": 0.78707421, "learning_rate": 3.7929440294410256e-06, "loss": 0.80862224, "num_input_tokens_seen": 52773420, "router_z_loss_clip": 0.234375, "router_z_loss_mlp": 0.7890625, "step": 2432, "time_per_iteration": 2.3856136798858643 }, { "auxiliary_loss_clip": 0.01104668, "auxiliary_loss_mlp": 0.01045863, "balance_loss_clip": 1.02277184, "balance_loss_mlp": 1.02831507, "epoch": 0.1462798737411694, "flos": 24388018594560.0, "grad_norm": 2.069365885011866, "language_loss": 0.79872191, "learning_rate": 3.792776584240829e-06, "loss": 0.82022727, "num_input_tokens_seen": 52792870, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.765625, "step": 2433, "time_per_iteration": 2.4183566570281982 }, { "auxiliary_loss_clip": 0.01104393, "auxiliary_loss_mlp": 0.01039623, "balance_loss_clip": 1.01834464, "balance_loss_mlp": 1.03108001, "epoch": 0.14633999699383737, "flos": 19243671292800.0, "grad_norm": 1.8863344199181091, "language_loss": 0.78056562, "learning_rate": 3.7926090750610477e-06, "loss": 0.80200571, "num_input_tokens_seen": 52811615, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.734375, "step": 2434, "time_per_iteration": 2.362306594848633 }, { "auxiliary_loss_clip": 0.01023121, "auxiliary_loss_mlp": 0.01002245, "balance_loss_clip": 0.9998247, "balance_loss_mlp": 1.00283957, "epoch": 0.14640012024650534, "flos": 62657468184960.0, "grad_norm": 0.856942128852628, "language_loss": 0.58509767, "learning_rate": 3.7924415019076593e-06, "loss": 0.60535133, "num_input_tokens_seen": 52873230, "router_z_loss_clip": 0.02416992, "router_z_loss_mlp": 0.203125, "step": 2435, "time_per_iteration": 2.991661787033081 }, { "auxiliary_loss_clip": 0.01104708, "auxiliary_loss_mlp": 0.01040238, "balance_loss_clip": 1.01990104, "balance_loss_mlp": 1.02923751, "epoch": 0.1464602434991733, "flos": 12275426321280.0, "grad_norm": 2.1762567484660567, "language_loss": 0.88035542, "learning_rate": 3.7922738647866447e-06, "loss": 0.90180486, "num_input_tokens_seen": 52889325, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.75390625, "step": 2436, "time_per_iteration": 2.359917640686035 }, { "auxiliary_loss_clip": 0.01109498, "auxiliary_loss_mlp": 0.01038005, "balance_loss_clip": 1.01778746, "balance_loss_mlp": 1.03151584, "epoch": 0.14652036675184127, "flos": 20921282899200.0, "grad_norm": 2.0799352703185243, "language_loss": 0.74614632, "learning_rate": 3.792106163703986e-06, "loss": 0.7676214, "num_input_tokens_seen": 52909705, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.78125, "step": 2437, "time_per_iteration": 2.401500701904297 }, { "auxiliary_loss_clip": 0.01107899, "auxiliary_loss_mlp": 0.01047454, "balance_loss_clip": 1.02295673, "balance_loss_mlp": 1.02991128, "epoch": 0.14658049000450923, "flos": 27702603118080.0, "grad_norm": 2.836756276761295, "language_loss": 0.73944908, "learning_rate": 3.791938398665668e-06, "loss": 0.76100266, "num_input_tokens_seen": 52930300, "router_z_loss_clip": 0.24511719, "router_z_loss_mlp": 0.78125, "step": 2438, "time_per_iteration": 2.4427995681762695 }, { "auxiliary_loss_clip": 0.01107115, "auxiliary_loss_mlp": 0.01039244, "balance_loss_clip": 1.01975298, "balance_loss_mlp": 1.03227019, "epoch": 0.14664061325717723, "flos": 24935351978880.0, "grad_norm": 2.120152516942651, "language_loss": 0.74749863, "learning_rate": 3.7917705696776786e-06, "loss": 0.76896226, "num_input_tokens_seen": 52949955, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.74609375, "step": 2439, "time_per_iteration": 2.42877459526062 }, { "auxiliary_loss_clip": 0.01105312, "auxiliary_loss_mlp": 0.01045753, "balance_loss_clip": 1.02427197, "balance_loss_mlp": 1.0307641, "epoch": 0.1467007365098452, "flos": 40296053813760.0, "grad_norm": 1.8759258491755313, "language_loss": 0.74690181, "learning_rate": 3.7916026767460067e-06, "loss": 0.76841247, "num_input_tokens_seen": 52972905, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.74609375, "step": 2440, "time_per_iteration": 2.543079376220703 }, { "auxiliary_loss_clip": 0.01104207, "auxiliary_loss_mlp": 0.01042874, "balance_loss_clip": 1.02344286, "balance_loss_mlp": 1.03002143, "epoch": 0.14676085976251316, "flos": 26539890376320.0, "grad_norm": 1.5325917258297075, "language_loss": 0.83222544, "learning_rate": 3.791434719876643e-06, "loss": 0.85369635, "num_input_tokens_seen": 52994850, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.7421875, "step": 2441, "time_per_iteration": 2.4438955783843994 }, { "auxiliary_loss_clip": 0.01112085, "auxiliary_loss_mlp": 0.01042917, "balance_loss_clip": 1.01980221, "balance_loss_mlp": 1.03033793, "epoch": 0.14682098301518112, "flos": 23548985867520.0, "grad_norm": 2.030479993575067, "language_loss": 0.72118711, "learning_rate": 3.7912666990755825e-06, "loss": 0.74273717, "num_input_tokens_seen": 53014740, "router_z_loss_clip": 0.23144531, "router_z_loss_mlp": 0.81640625, "step": 2442, "time_per_iteration": 2.3936851024627686 }, { "auxiliary_loss_clip": 0.0111204, "auxiliary_loss_mlp": 0.01041995, "balance_loss_clip": 1.02048922, "balance_loss_mlp": 1.03172112, "epoch": 0.1468811062678491, "flos": 11650411428480.0, "grad_norm": 2.8190333055980825, "language_loss": 0.81732076, "learning_rate": 3.791098614348821e-06, "loss": 0.83886111, "num_input_tokens_seen": 53029780, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.8046875, "step": 2443, "time_per_iteration": 2.3456430435180664 }, { "auxiliary_loss_clip": 0.01107763, "auxiliary_loss_mlp": 0.01042506, "balance_loss_clip": 1.02182293, "balance_loss_mlp": 1.03075087, "epoch": 0.14694122952051705, "flos": 23001512837760.0, "grad_norm": 1.7544449726777838, "language_loss": 0.82959914, "learning_rate": 3.790930465702358e-06, "loss": 0.85110176, "num_input_tokens_seen": 53048620, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.7734375, "step": 2444, "time_per_iteration": 2.3839950561523438 }, { "auxiliary_loss_clip": 0.01108247, "auxiliary_loss_mlp": 0.01039826, "balance_loss_clip": 1.01883376, "balance_loss_mlp": 1.03084493, "epoch": 0.14700135277318502, "flos": 26501835127680.0, "grad_norm": 1.6459977299007913, "language_loss": 0.70786947, "learning_rate": 3.790762253142193e-06, "loss": 0.72935021, "num_input_tokens_seen": 53070055, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.7734375, "step": 2445, "time_per_iteration": 2.4207019805908203 }, { "auxiliary_loss_clip": 0.01025382, "auxiliary_loss_mlp": 0.01003005, "balance_loss_clip": 1.00078809, "balance_loss_mlp": 1.00529063, "epoch": 0.147061476025853, "flos": 59446366531200.0, "grad_norm": 0.8105754912794436, "language_loss": 0.6305809, "learning_rate": 3.7905939766743296e-06, "loss": 0.65086478, "num_input_tokens_seen": 53126945, "router_z_loss_clip": 0.0222168, "router_z_loss_mlp": 0.20117188, "step": 2446, "time_per_iteration": 2.908247232437134 }, { "auxiliary_loss_clip": 0.01108616, "auxiliary_loss_mlp": 0.01047134, "balance_loss_clip": 1.02487803, "balance_loss_mlp": 1.02990222, "epoch": 0.14712159927852098, "flos": 28329607958400.0, "grad_norm": 1.57403669728487, "language_loss": 0.74675715, "learning_rate": 3.790425636304773e-06, "loss": 0.76831472, "num_input_tokens_seen": 53149130, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.7890625, "step": 2447, "time_per_iteration": 2.44604754447937 }, { "auxiliary_loss_clip": 0.01104429, "auxiliary_loss_mlp": 0.01038763, "balance_loss_clip": 1.01849759, "balance_loss_mlp": 1.02910519, "epoch": 0.14718172253118894, "flos": 27088585303680.0, "grad_norm": 2.1219490618768657, "language_loss": 0.85836643, "learning_rate": 3.7902572320395313e-06, "loss": 0.87979835, "num_input_tokens_seen": 53167120, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.75390625, "step": 2448, "time_per_iteration": 2.4199423789978027 }, { "auxiliary_loss_clip": 0.01023192, "auxiliary_loss_mlp": 0.01010136, "balance_loss_clip": 1.00791848, "balance_loss_mlp": 1.00305104, "epoch": 0.1472418457838569, "flos": 66703587759360.0, "grad_norm": 0.7662270412089266, "language_loss": 0.56839919, "learning_rate": 3.790088763884614e-06, "loss": 0.58873248, "num_input_tokens_seen": 53227945, "router_z_loss_clip": 0.0222168, "router_z_loss_mlp": 0.20117188, "step": 2449, "time_per_iteration": 2.975759983062744 }, { "auxiliary_loss_clip": 0.01106258, "auxiliary_loss_mlp": 0.01040438, "balance_loss_clip": 1.01940989, "balance_loss_mlp": 1.03197777, "epoch": 0.14730196903652487, "flos": 19572553100160.0, "grad_norm": 1.8227387694827935, "language_loss": 0.85032845, "learning_rate": 3.789920231846033e-06, "loss": 0.87179542, "num_input_tokens_seen": 53244615, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.7421875, "step": 2450, "time_per_iteration": 2.370150566101074 }, { "auxiliary_loss_clip": 0.01109353, "auxiliary_loss_mlp": 0.01039365, "balance_loss_clip": 1.0171082, "balance_loss_mlp": 1.03172064, "epoch": 0.14736209228919284, "flos": 16070101217280.0, "grad_norm": 2.111219050806394, "language_loss": 0.74913561, "learning_rate": 3.7897516359298034e-06, "loss": 0.77062279, "num_input_tokens_seen": 53262205, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.7734375, "step": 2451, "time_per_iteration": 2.3699350357055664 }, { "auxiliary_loss_clip": 0.01102903, "auxiliary_loss_mlp": 0.01041274, "balance_loss_clip": 1.02171206, "balance_loss_mlp": 1.02980554, "epoch": 0.1474222155418608, "flos": 23038346188800.0, "grad_norm": 1.5959023829120984, "language_loss": 0.82293332, "learning_rate": 3.7895829761419417e-06, "loss": 0.84437507, "num_input_tokens_seen": 53282445, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.73046875, "step": 2452, "time_per_iteration": 2.4012458324432373 }, { "auxiliary_loss_clip": 0.01104116, "auxiliary_loss_mlp": 0.01037933, "balance_loss_clip": 1.01787043, "balance_loss_mlp": 1.0311923, "epoch": 0.1474823387945288, "flos": 17017713872640.0, "grad_norm": 1.9016400293559543, "language_loss": 0.74453282, "learning_rate": 3.789414252488467e-06, "loss": 0.76595336, "num_input_tokens_seen": 53299060, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.73046875, "step": 2453, "time_per_iteration": 2.3514039516448975 }, { "auxiliary_loss_clip": 0.011081, "auxiliary_loss_mlp": 0.01037958, "balance_loss_clip": 1.01665509, "balance_loss_mlp": 1.03024554, "epoch": 0.14754246204719676, "flos": 17894068709760.0, "grad_norm": 2.005732480324612, "language_loss": 0.75826025, "learning_rate": 3.7892454649754006e-06, "loss": 0.77972078, "num_input_tokens_seen": 53315970, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.77734375, "step": 2454, "time_per_iteration": 2.351875066757202 }, { "auxiliary_loss_clip": 0.011096, "auxiliary_loss_mlp": 0.01038054, "balance_loss_clip": 1.01739502, "balance_loss_mlp": 1.02999902, "epoch": 0.14760258529986472, "flos": 13078254101760.0, "grad_norm": 1.874940662609717, "language_loss": 0.83087826, "learning_rate": 3.789076613608766e-06, "loss": 0.85235482, "num_input_tokens_seen": 53332940, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.796875, "step": 2455, "time_per_iteration": 2.3562862873077393 }, { "auxiliary_loss_clip": 0.01110501, "auxiliary_loss_mlp": 0.01041664, "balance_loss_clip": 1.02000332, "balance_loss_mlp": 1.02969289, "epoch": 0.1476627085525327, "flos": 30805194666240.0, "grad_norm": 2.15398892222712, "language_loss": 0.83898067, "learning_rate": 3.788907698394589e-06, "loss": 0.8605023, "num_input_tokens_seen": 53353295, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.80859375, "step": 2456, "time_per_iteration": 2.4573004245758057 }, { "auxiliary_loss_clip": 0.0110531, "auxiliary_loss_mlp": 0.01033511, "balance_loss_clip": 1.012959, "balance_loss_mlp": 1.02993011, "epoch": 0.14772283180520066, "flos": 21688359580800.0, "grad_norm": 1.8900714922399362, "language_loss": 0.84265876, "learning_rate": 3.788738719338898e-06, "loss": 0.86404693, "num_input_tokens_seen": 53373410, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.75390625, "step": 2457, "time_per_iteration": 2.378192901611328 }, { "auxiliary_loss_clip": 0.01104148, "auxiliary_loss_mlp": 0.01035294, "balance_loss_clip": 1.01481414, "balance_loss_mlp": 1.03054821, "epoch": 0.14778295505786862, "flos": 18769411117440.0, "grad_norm": 1.9567926522310815, "language_loss": 0.74974108, "learning_rate": 3.788569676447723e-06, "loss": 0.77113551, "num_input_tokens_seen": 53391430, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.734375, "step": 2458, "time_per_iteration": 2.3949851989746094 }, { "auxiliary_loss_clip": 0.01111074, "auxiliary_loss_mlp": 0.01041301, "balance_loss_clip": 1.01924753, "balance_loss_mlp": 1.03039289, "epoch": 0.1478430783105366, "flos": 22892444150400.0, "grad_norm": 1.8526540964536895, "language_loss": 0.83802259, "learning_rate": 3.7884005697270976e-06, "loss": 0.85954636, "num_input_tokens_seen": 53409960, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.8046875, "step": 2459, "time_per_iteration": 2.386124849319458 }, { "auxiliary_loss_clip": 0.01102934, "auxiliary_loss_mlp": 0.01039875, "balance_loss_clip": 1.02070618, "balance_loss_mlp": 1.03005552, "epoch": 0.14790320156320458, "flos": 15084433313280.0, "grad_norm": 2.3751272671493258, "language_loss": 0.75208676, "learning_rate": 3.7882313991830553e-06, "loss": 0.77351487, "num_input_tokens_seen": 53426160, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.7265625, "step": 2460, "time_per_iteration": 2.3616836071014404 }, { "auxiliary_loss_clip": 0.01109286, "auxiliary_loss_mlp": 0.01040809, "balance_loss_clip": 1.01885045, "balance_loss_mlp": 1.03090692, "epoch": 0.14796332481587254, "flos": 26503580695680.0, "grad_norm": 1.695323430815126, "language_loss": 0.81721282, "learning_rate": 3.788062164821635e-06, "loss": 0.83871377, "num_input_tokens_seen": 53448530, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.78515625, "step": 2461, "time_per_iteration": 2.438246250152588 }, { "auxiliary_loss_clip": 0.01108934, "auxiliary_loss_mlp": 0.01041782, "balance_loss_clip": 1.01964498, "balance_loss_mlp": 1.03071451, "epoch": 0.1480234480685405, "flos": 17562324170880.0, "grad_norm": 3.2664391766448513, "language_loss": 0.65738714, "learning_rate": 3.7878928666488755e-06, "loss": 0.67889428, "num_input_tokens_seen": 53465915, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.78125, "step": 2462, "time_per_iteration": 2.344008207321167 }, { "auxiliary_loss_clip": 0.01106101, "auxiliary_loss_mlp": 0.01047585, "balance_loss_clip": 1.02500653, "balance_loss_mlp": 1.02892792, "epoch": 0.14808357132120847, "flos": 53580121580160.0, "grad_norm": 2.134080966410557, "language_loss": 0.67159075, "learning_rate": 3.787723504670818e-06, "loss": 0.69312757, "num_input_tokens_seen": 53496055, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.7734375, "step": 2463, "time_per_iteration": 2.6848130226135254 }, { "auxiliary_loss_clip": 0.01104801, "auxiliary_loss_mlp": 0.01044845, "balance_loss_clip": 1.02301788, "balance_loss_mlp": 1.02841341, "epoch": 0.14814369457387644, "flos": 19828152230400.0, "grad_norm": 1.6024968555364416, "language_loss": 0.76646018, "learning_rate": 3.7875540788935076e-06, "loss": 0.78795666, "num_input_tokens_seen": 53513790, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.765625, "step": 2464, "time_per_iteration": 3.7671191692352295 }, { "auxiliary_loss_clip": 0.01104148, "auxiliary_loss_mlp": 0.01039635, "balance_loss_clip": 1.01981044, "balance_loss_mlp": 1.03094208, "epoch": 0.1482038178265444, "flos": 23913828241920.0, "grad_norm": 1.6632214555847842, "language_loss": 0.79578698, "learning_rate": 3.7873845893229896e-06, "loss": 0.81722486, "num_input_tokens_seen": 53533410, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.73046875, "step": 2465, "time_per_iteration": 2.411268711090088 }, { "auxiliary_loss_clip": 0.01109104, "auxiliary_loss_mlp": 0.01042305, "balance_loss_clip": 1.02058506, "balance_loss_mlp": 1.0299412, "epoch": 0.1482639410792124, "flos": 24169357549440.0, "grad_norm": 2.5067273738761964, "language_loss": 0.76638633, "learning_rate": 3.7872150359653143e-06, "loss": 0.78790045, "num_input_tokens_seen": 53554775, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.79296875, "step": 2466, "time_per_iteration": 3.780672788619995 }, { "auxiliary_loss_clip": 0.01025631, "auxiliary_loss_mlp": 0.01002746, "balance_loss_clip": 1.00039744, "balance_loss_mlp": 1.00557256, "epoch": 0.14832406433188036, "flos": 66188025578880.0, "grad_norm": 0.7811321793737293, "language_loss": 0.6012544, "learning_rate": 3.787045418826531e-06, "loss": 0.62153816, "num_input_tokens_seen": 53609675, "router_z_loss_clip": 0.0234375, "router_z_loss_mlp": 0.20117188, "step": 2467, "time_per_iteration": 4.413452863693237 }, { "auxiliary_loss_clip": 0.01102364, "auxiliary_loss_mlp": 0.01035207, "balance_loss_clip": 1.01523924, "balance_loss_mlp": 1.02893591, "epoch": 0.14838418758454833, "flos": 25410066001920.0, "grad_norm": 2.1207902730080836, "language_loss": 0.87758505, "learning_rate": 3.7868757379126938e-06, "loss": 0.89896071, "num_input_tokens_seen": 53626950, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.734375, "step": 2468, "time_per_iteration": 3.744333505630493 }, { "auxiliary_loss_clip": 0.01106776, "auxiliary_loss_mlp": 0.01040982, "balance_loss_clip": 1.01863039, "balance_loss_mlp": 1.02884674, "epoch": 0.1484443108372163, "flos": 23288918083200.0, "grad_norm": 2.031910697410879, "language_loss": 0.76093054, "learning_rate": 3.7867059932298578e-06, "loss": 0.78240806, "num_input_tokens_seen": 53644200, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.78125, "step": 2469, "time_per_iteration": 2.4004931449890137 }, { "auxiliary_loss_clip": 0.01105523, "auxiliary_loss_mlp": 0.01045265, "balance_loss_clip": 1.02372372, "balance_loss_mlp": 1.0301652, "epoch": 0.14850443408988426, "flos": 14646797020800.0, "grad_norm": 2.357346122684486, "language_loss": 0.75776291, "learning_rate": 3.786536184784081e-06, "loss": 0.77927077, "num_input_tokens_seen": 53659650, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.75390625, "step": 2470, "time_per_iteration": 2.352248430252075 }, { "auxiliary_loss_clip": 0.01102071, "auxiliary_loss_mlp": 0.01042208, "balance_loss_clip": 1.02152479, "balance_loss_mlp": 1.02874684, "epoch": 0.14856455734255222, "flos": 23547240299520.0, "grad_norm": 1.958258548759413, "language_loss": 0.72282279, "learning_rate": 3.786366312581423e-06, "loss": 0.74426562, "num_input_tokens_seen": 53680275, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.734375, "step": 2471, "time_per_iteration": 2.4054150581359863 }, { "auxiliary_loss_clip": 0.01108779, "auxiliary_loss_mlp": 0.01039437, "balance_loss_clip": 1.01688242, "balance_loss_mlp": 1.02943885, "epoch": 0.1486246805952202, "flos": 18076315340160.0, "grad_norm": 2.567091590849504, "language_loss": 0.89306957, "learning_rate": 3.786196376627947e-06, "loss": 0.91455173, "num_input_tokens_seen": 53698270, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.796875, "step": 2472, "time_per_iteration": 2.360830783843994 }, { "auxiliary_loss_clip": 0.01105657, "auxiliary_loss_mlp": 0.01043823, "balance_loss_clip": 1.02275848, "balance_loss_mlp": 1.02937889, "epoch": 0.14868480384788818, "flos": 19352635246080.0, "grad_norm": 3.2100663064713113, "language_loss": 0.80462313, "learning_rate": 3.7860263769297163e-06, "loss": 0.82611787, "num_input_tokens_seen": 53716845, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.76171875, "step": 2473, "time_per_iteration": 2.360668420791626 }, { "auxiliary_loss_clip": 0.01109533, "auxiliary_loss_mlp": 0.01041752, "balance_loss_clip": 1.02066338, "balance_loss_mlp": 1.02998114, "epoch": 0.14874492710055615, "flos": 22199103993600.0, "grad_norm": 2.565127652122542, "language_loss": 0.77484328, "learning_rate": 3.7858563134927985e-06, "loss": 0.79635614, "num_input_tokens_seen": 53734970, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.796875, "step": 2474, "time_per_iteration": 2.384188413619995 }, { "auxiliary_loss_clip": 0.01106567, "auxiliary_loss_mlp": 0.01043922, "balance_loss_clip": 1.02127266, "balance_loss_mlp": 1.02885747, "epoch": 0.1488050503532241, "flos": 21102447277440.0, "grad_norm": 3.3138585561777703, "language_loss": 0.82387251, "learning_rate": 3.785686186323263e-06, "loss": 0.84537739, "num_input_tokens_seen": 53753415, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.77734375, "step": 2475, "time_per_iteration": 2.36897611618042 }, { "auxiliary_loss_clip": 0.01108567, "auxiliary_loss_mlp": 0.01047392, "balance_loss_clip": 1.0271976, "balance_loss_mlp": 1.03358066, "epoch": 0.14886517360589208, "flos": 12785751797760.0, "grad_norm": 1.8469100074926545, "language_loss": 0.80300593, "learning_rate": 3.785515995427181e-06, "loss": 0.82456547, "num_input_tokens_seen": 53770305, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.75, "step": 2476, "time_per_iteration": 2.3844544887542725 }, { "auxiliary_loss_clip": 0.01100131, "auxiliary_loss_mlp": 0.01044251, "balance_loss_clip": 1.02424812, "balance_loss_mlp": 1.02915752, "epoch": 0.14892529685856004, "flos": 29021586572160.0, "grad_norm": 1.650390794059293, "language_loss": 0.77724421, "learning_rate": 3.7853457408106257e-06, "loss": 0.79868805, "num_input_tokens_seen": 53788895, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.7109375, "step": 2477, "time_per_iteration": 2.4232230186462402 }, { "auxiliary_loss_clip": 0.01023444, "auxiliary_loss_mlp": 0.01002709, "balance_loss_clip": 1.0001694, "balance_loss_mlp": 1.00380075, "epoch": 0.148985420111228, "flos": 61923105313920.0, "grad_norm": 0.8200409354329088, "language_loss": 0.60067445, "learning_rate": 3.785175422479673e-06, "loss": 0.62093598, "num_input_tokens_seen": 53850260, "router_z_loss_clip": 0.02539062, "router_z_loss_mlp": 0.19628906, "step": 2478, "time_per_iteration": 3.0559191703796387 }, { "auxiliary_loss_clip": 0.01106866, "auxiliary_loss_mlp": 0.01040803, "balance_loss_clip": 1.02027547, "balance_loss_mlp": 1.03009081, "epoch": 0.149045543363896, "flos": 23913967887360.0, "grad_norm": 3.065683946321891, "language_loss": 0.70812583, "learning_rate": 3.785005040440402e-06, "loss": 0.72960258, "num_input_tokens_seen": 53867520, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.765625, "step": 2479, "time_per_iteration": 2.4103593826293945 }, { "auxiliary_loss_clip": 0.01105955, "auxiliary_loss_mlp": 0.01036613, "balance_loss_clip": 1.01666939, "balance_loss_mlp": 1.03049088, "epoch": 0.14910566661656396, "flos": 23653411344000.0, "grad_norm": 1.8489014792152119, "language_loss": 0.81115156, "learning_rate": 3.784834594698892e-06, "loss": 0.83257723, "num_input_tokens_seen": 53886620, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.7578125, "step": 2480, "time_per_iteration": 2.4197895526885986 }, { "auxiliary_loss_clip": 0.01108557, "auxiliary_loss_mlp": 0.01041216, "balance_loss_clip": 1.02054477, "balance_loss_mlp": 1.03099465, "epoch": 0.14916578986923193, "flos": 20514440292480.0, "grad_norm": 2.551710077764799, "language_loss": 0.84128141, "learning_rate": 3.7846640852612275e-06, "loss": 0.86277914, "num_input_tokens_seen": 53902230, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.77734375, "step": 2481, "time_per_iteration": 2.365886926651001 }, { "auxiliary_loss_clip": 0.01106418, "auxiliary_loss_mlp": 0.01048773, "balance_loss_clip": 1.02586138, "balance_loss_mlp": 1.02866387, "epoch": 0.1492259131218999, "flos": 22490733513600.0, "grad_norm": 2.197025405151839, "language_loss": 0.7761029, "learning_rate": 3.7844935121334917e-06, "loss": 0.79765475, "num_input_tokens_seen": 53919475, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.77734375, "step": 2482, "time_per_iteration": 2.3777050971984863 }, { "auxiliary_loss_clip": 0.01113147, "auxiliary_loss_mlp": 0.01039511, "balance_loss_clip": 1.01711154, "balance_loss_mlp": 1.03127205, "epoch": 0.14928603637456786, "flos": 23184736986240.0, "grad_norm": 2.387801286820446, "language_loss": 0.78827822, "learning_rate": 3.7843228753217726e-06, "loss": 0.8098048, "num_input_tokens_seen": 53939150, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.8203125, "step": 2483, "time_per_iteration": 2.389840841293335 }, { "auxiliary_loss_clip": 0.01102808, "auxiliary_loss_mlp": 0.01034901, "balance_loss_clip": 1.01655495, "balance_loss_mlp": 1.02996683, "epoch": 0.14934615962723582, "flos": 21652154634240.0, "grad_norm": 1.7036932947225292, "language_loss": 0.70134556, "learning_rate": 3.784152174832161e-06, "loss": 0.72272271, "num_input_tokens_seen": 53958735, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.7265625, "step": 2484, "time_per_iteration": 2.3964152336120605 }, { "auxiliary_loss_clip": 0.0110853, "auxiliary_loss_mlp": 0.01039956, "balance_loss_clip": 1.01690102, "balance_loss_mlp": 1.02992082, "epoch": 0.1494062828799038, "flos": 27009018581760.0, "grad_norm": 1.9461525369762271, "language_loss": 0.84372914, "learning_rate": 3.783981410670747e-06, "loss": 0.86521399, "num_input_tokens_seen": 53975065, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.78515625, "step": 2485, "time_per_iteration": 2.395015239715576 }, { "auxiliary_loss_clip": 0.01109191, "auxiliary_loss_mlp": 0.010414, "balance_loss_clip": 1.01866639, "balance_loss_mlp": 1.03184628, "epoch": 0.14946640613257178, "flos": 21213889937280.0, "grad_norm": 2.3685290325684805, "language_loss": 0.85049736, "learning_rate": 3.7838105828436246e-06, "loss": 0.87200332, "num_input_tokens_seen": 53993330, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.7734375, "step": 2486, "time_per_iteration": 2.380772590637207 }, { "auxiliary_loss_clip": 0.01102168, "auxiliary_loss_mlp": 0.01035574, "balance_loss_clip": 1.01717949, "balance_loss_mlp": 1.0275681, "epoch": 0.14952652938523975, "flos": 13370023267200.0, "grad_norm": 2.4486603015435375, "language_loss": 0.74685532, "learning_rate": 3.7836396913568924e-06, "loss": 0.7682327, "num_input_tokens_seen": 54010515, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.74609375, "step": 2487, "time_per_iteration": 2.348187208175659 }, { "auxiliary_loss_clip": 0.01105893, "auxiliary_loss_mlp": 0.01045336, "balance_loss_clip": 1.0232228, "balance_loss_mlp": 1.0319078, "epoch": 0.1495866526379077, "flos": 35516234684160.0, "grad_norm": 1.970596711140622, "language_loss": 0.71655691, "learning_rate": 3.783468736216647e-06, "loss": 0.73806924, "num_input_tokens_seen": 54031315, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.7421875, "step": 2488, "time_per_iteration": 2.507781982421875 }, { "auxiliary_loss_clip": 0.01108562, "auxiliary_loss_mlp": 0.01045898, "balance_loss_clip": 1.02378464, "balance_loss_mlp": 1.02948689, "epoch": 0.14964677589057568, "flos": 17631976066560.0, "grad_norm": 3.4016547959130485, "language_loss": 0.70374882, "learning_rate": 3.78329771742899e-06, "loss": 0.7252934, "num_input_tokens_seen": 54045965, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.79296875, "step": 2489, "time_per_iteration": 2.3227648735046387 }, { "auxiliary_loss_clip": 0.01107973, "auxiliary_loss_mlp": 0.01046761, "balance_loss_clip": 1.02527964, "balance_loss_mlp": 1.02966833, "epoch": 0.14970689914324364, "flos": 20184476232960.0, "grad_norm": 3.089408897003641, "language_loss": 0.82155168, "learning_rate": 3.7831266350000246e-06, "loss": 0.843099, "num_input_tokens_seen": 54059960, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.78515625, "step": 2490, "time_per_iteration": 2.3514554500579834 }, { "auxiliary_loss_clip": 0.01109634, "auxiliary_loss_mlp": 0.01042063, "balance_loss_clip": 1.02196455, "balance_loss_mlp": 1.03133702, "epoch": 0.1497670223959116, "flos": 37227293239680.0, "grad_norm": 1.8708118573962265, "language_loss": 0.79381263, "learning_rate": 3.7829554889358566e-06, "loss": 0.81532961, "num_input_tokens_seen": 54079330, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.78515625, "step": 2491, "time_per_iteration": 2.5021839141845703 }, { "auxiliary_loss_clip": 0.01106918, "auxiliary_loss_mlp": 0.01040376, "balance_loss_clip": 1.01766706, "balance_loss_mlp": 1.02717793, "epoch": 0.1498271456485796, "flos": 24454877581440.0, "grad_norm": 1.7980417862337945, "language_loss": 0.9063071, "learning_rate": 3.782784279242593e-06, "loss": 0.92778003, "num_input_tokens_seen": 54097555, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.796875, "step": 2492, "time_per_iteration": 2.4079651832580566 }, { "auxiliary_loss_clip": 0.01105784, "auxiliary_loss_mlp": 0.01048743, "balance_loss_clip": 1.02727377, "balance_loss_mlp": 1.02955317, "epoch": 0.14988726890124757, "flos": 16252662049920.0, "grad_norm": 4.15489378567907, "language_loss": 0.78564751, "learning_rate": 3.782613005926345e-06, "loss": 0.80719274, "num_input_tokens_seen": 54115600, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.76171875, "step": 2493, "time_per_iteration": 2.36659836769104 }, { "auxiliary_loss_clip": 0.01106706, "auxiliary_loss_mlp": 0.01043433, "balance_loss_clip": 1.02074707, "balance_loss_mlp": 1.02827418, "epoch": 0.14994739215391553, "flos": 20665544123520.0, "grad_norm": 2.2290031291976846, "language_loss": 0.80210996, "learning_rate": 3.7824416689932236e-06, "loss": 0.82361138, "num_input_tokens_seen": 54135220, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.78515625, "step": 2494, "time_per_iteration": 2.386484146118164 }, { "auxiliary_loss_clip": 0.01104888, "auxiliary_loss_mlp": 0.01045692, "balance_loss_clip": 1.02410316, "balance_loss_mlp": 1.02876639, "epoch": 0.1500075154065835, "flos": 70649961845760.0, "grad_norm": 16.878931362562213, "language_loss": 0.6622541, "learning_rate": 3.782270268449345e-06, "loss": 0.68375999, "num_input_tokens_seen": 54161065, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.76171875, "step": 2495, "time_per_iteration": 2.770857572555542 }, { "auxiliary_loss_clip": 0.01022901, "auxiliary_loss_mlp": 0.01007046, "balance_loss_clip": 1.00465024, "balance_loss_mlp": 1.00302672, "epoch": 0.15006763865925146, "flos": 68008955783040.0, "grad_norm": 0.8867353178954155, "language_loss": 0.59524918, "learning_rate": 3.7820988043008242e-06, "loss": 0.61554861, "num_input_tokens_seen": 54225095, "router_z_loss_clip": 0.02392578, "router_z_loss_mlp": 0.19921875, "step": 2496, "time_per_iteration": 3.040221691131592 }, { "auxiliary_loss_clip": 0.01107105, "auxiliary_loss_mlp": 0.0104946, "balance_loss_clip": 1.02659595, "balance_loss_mlp": 1.02788699, "epoch": 0.15012776191191943, "flos": 18915278244480.0, "grad_norm": 1.9285751832320503, "language_loss": 0.65387583, "learning_rate": 3.7819272765537817e-06, "loss": 0.6754415, "num_input_tokens_seen": 54243750, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.7890625, "step": 2497, "time_per_iteration": 2.3551132678985596 }, { "auxiliary_loss_clip": 0.01109416, "auxiliary_loss_mlp": 0.01040362, "balance_loss_clip": 1.01925039, "balance_loss_mlp": 1.03077793, "epoch": 0.1501878851645874, "flos": 23700054787200.0, "grad_norm": 1.4946450898699988, "language_loss": 0.75399512, "learning_rate": 3.781755685214338e-06, "loss": 0.77549291, "num_input_tokens_seen": 54266185, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.78515625, "step": 2498, "time_per_iteration": 2.451472759246826 }, { "auxiliary_loss_clip": 0.01110941, "auxiliary_loss_mlp": 0.01048753, "balance_loss_clip": 1.02396941, "balance_loss_mlp": 1.03037, "epoch": 0.15024800841725539, "flos": 20411481093120.0, "grad_norm": 3.5897244410704174, "language_loss": 0.72171801, "learning_rate": 3.7815840302886174e-06, "loss": 0.74331498, "num_input_tokens_seen": 54283940, "router_z_loss_clip": 0.24804688, "router_z_loss_mlp": 0.8046875, "step": 2499, "time_per_iteration": 2.3646938800811768 }, { "auxiliary_loss_clip": 0.01106481, "auxiliary_loss_mlp": 0.01044837, "balance_loss_clip": 1.02206814, "balance_loss_mlp": 1.02982831, "epoch": 0.15030813166992335, "flos": 31829685868800.0, "grad_norm": 2.078369693629021, "language_loss": 0.71735597, "learning_rate": 3.7814123117827446e-06, "loss": 0.73886919, "num_input_tokens_seen": 54304830, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.76953125, "step": 2500, "time_per_iteration": 2.465881586074829 }, { "auxiliary_loss_clip": 0.0110495, "auxiliary_loss_mlp": 0.01051187, "balance_loss_clip": 1.0273335, "balance_loss_mlp": 1.02872014, "epoch": 0.15036825492259132, "flos": 35656515993600.0, "grad_norm": 1.7714213938379884, "language_loss": 0.64922321, "learning_rate": 3.7812405297028496e-06, "loss": 0.67078459, "num_input_tokens_seen": 54325595, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.76171875, "step": 2501, "time_per_iteration": 2.4886040687561035 }, { "auxiliary_loss_clip": 0.01102843, "auxiliary_loss_mlp": 0.01047496, "balance_loss_clip": 1.02550197, "balance_loss_mlp": 1.02860188, "epoch": 0.15042837817525928, "flos": 18837317445120.0, "grad_norm": 3.2510901341753407, "language_loss": 0.83437526, "learning_rate": 3.7810686840550627e-06, "loss": 0.85587859, "num_input_tokens_seen": 54342180, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.7421875, "step": 2502, "time_per_iteration": 2.3517165184020996 }, { "auxiliary_loss_clip": 0.01101775, "auxiliary_loss_mlp": 0.01043332, "balance_loss_clip": 1.02318525, "balance_loss_mlp": 1.02661479, "epoch": 0.15048850142792725, "flos": 19534567674240.0, "grad_norm": 1.8809738964560827, "language_loss": 0.77416849, "learning_rate": 3.780896774845515e-06, "loss": 0.79561961, "num_input_tokens_seen": 54360255, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.75, "step": 2503, "time_per_iteration": 3.7314107418060303 }, { "auxiliary_loss_clip": 0.01104913, "auxiliary_loss_mlp": 0.01038449, "balance_loss_clip": 1.01751566, "balance_loss_mlp": 1.02834249, "epoch": 0.1505486246805952, "flos": 22016473338240.0, "grad_norm": 1.886192327546137, "language_loss": 0.8547039, "learning_rate": 3.780724802080342e-06, "loss": 0.8761375, "num_input_tokens_seen": 54378260, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.765625, "step": 2504, "time_per_iteration": 2.382795810699463 }, { "auxiliary_loss_clip": 0.01104583, "auxiliary_loss_mlp": 0.01036428, "balance_loss_clip": 1.01629364, "balance_loss_mlp": 1.02975178, "epoch": 0.15060874793326318, "flos": 20742038645760.0, "grad_norm": 1.6319880378542269, "language_loss": 0.83300298, "learning_rate": 3.780552765765682e-06, "loss": 0.85441315, "num_input_tokens_seen": 54399745, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.75, "step": 2505, "time_per_iteration": 2.389429807662964 }, { "auxiliary_loss_clip": 0.01102799, "auxiliary_loss_mlp": 0.01041655, "balance_loss_clip": 1.02066219, "balance_loss_mlp": 1.02746129, "epoch": 0.15066887118593117, "flos": 16470973981440.0, "grad_norm": 2.5789060434706066, "language_loss": 0.75912398, "learning_rate": 3.7803806659076736e-06, "loss": 0.78056848, "num_input_tokens_seen": 54417105, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.75390625, "step": 2506, "time_per_iteration": 3.8077516555786133 }, { "auxiliary_loss_clip": 0.0110723, "auxiliary_loss_mlp": 0.01043255, "balance_loss_clip": 1.02203536, "balance_loss_mlp": 1.02894378, "epoch": 0.15072899443859913, "flos": 19858457157120.0, "grad_norm": 3.620360501858079, "language_loss": 0.76408052, "learning_rate": 3.7802085025124596e-06, "loss": 0.78558534, "num_input_tokens_seen": 54433920, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.78125, "step": 2507, "time_per_iteration": 3.6984262466430664 }, { "auxiliary_loss_clip": 0.01101382, "auxiliary_loss_mlp": 0.01037609, "balance_loss_clip": 1.01672363, "balance_loss_mlp": 1.02788413, "epoch": 0.1507891176912671, "flos": 20775206304000.0, "grad_norm": 1.8528607674820419, "language_loss": 0.68635213, "learning_rate": 3.780036275586183e-06, "loss": 0.70774198, "num_input_tokens_seen": 54451540, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.734375, "step": 2508, "time_per_iteration": 2.3735949993133545 }, { "auxiliary_loss_clip": 0.0110717, "auxiliary_loss_mlp": 0.01039804, "balance_loss_clip": 1.01870346, "balance_loss_mlp": 1.02987397, "epoch": 0.15084924094393506, "flos": 23585505016320.0, "grad_norm": 1.703027506053113, "language_loss": 0.77397841, "learning_rate": 3.77986398513499e-06, "loss": 0.79544818, "num_input_tokens_seen": 54470800, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.7734375, "step": 2509, "time_per_iteration": 2.409031867980957 }, { "auxiliary_loss_clip": 0.01111394, "auxiliary_loss_mlp": 0.01046605, "balance_loss_clip": 1.02228665, "balance_loss_mlp": 1.02996719, "epoch": 0.15090936419660303, "flos": 18910460476800.0, "grad_norm": 2.139432909344303, "language_loss": 0.79934525, "learning_rate": 3.7796916311650306e-06, "loss": 0.82092535, "num_input_tokens_seen": 54486525, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.8125, "step": 2510, "time_per_iteration": 2.332805871963501 }, { "auxiliary_loss_clip": 0.0110901, "auxiliary_loss_mlp": 0.01047356, "balance_loss_clip": 1.0243609, "balance_loss_mlp": 1.03031814, "epoch": 0.150969487449271, "flos": 17927341102080.0, "grad_norm": 2.0902943664394393, "language_loss": 0.73790503, "learning_rate": 3.779519213682454e-06, "loss": 0.75946862, "num_input_tokens_seen": 54503795, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.7890625, "step": 2511, "time_per_iteration": 2.363016128540039 }, { "auxiliary_loss_clip": 0.01105998, "auxiliary_loss_mlp": 0.01040421, "balance_loss_clip": 1.02032197, "balance_loss_mlp": 1.03014183, "epoch": 0.151029610701939, "flos": 24241941999360.0, "grad_norm": 1.9770655402867399, "language_loss": 0.69080341, "learning_rate": 3.7793467326934147e-06, "loss": 0.71226764, "num_input_tokens_seen": 54523025, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.7578125, "step": 2512, "time_per_iteration": 2.397308349609375 }, { "auxiliary_loss_clip": 0.0110867, "auxiliary_loss_mlp": 0.01042614, "balance_loss_clip": 1.02091765, "balance_loss_mlp": 1.0323503, "epoch": 0.15108973395460695, "flos": 30261212772480.0, "grad_norm": 2.6598247798626575, "language_loss": 0.73773617, "learning_rate": 3.7791741882040677e-06, "loss": 0.75924897, "num_input_tokens_seen": 54545025, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.765625, "step": 2513, "time_per_iteration": 2.45658802986145 }, { "auxiliary_loss_clip": 0.01020814, "auxiliary_loss_mlp": 0.010182, "balance_loss_clip": 1.01548231, "balance_loss_mlp": 1.00186658, "epoch": 0.15114985720727492, "flos": 60434443319040.0, "grad_norm": 0.8834424408720046, "language_loss": 0.64798552, "learning_rate": 3.7790015802205703e-06, "loss": 0.66837567, "num_input_tokens_seen": 54604545, "router_z_loss_clip": 0.02722168, "router_z_loss_mlp": 0.18945312, "step": 2514, "time_per_iteration": 2.968867778778076 }, { "auxiliary_loss_clip": 0.01103398, "auxiliary_loss_mlp": 0.01038986, "balance_loss_clip": 1.01708698, "balance_loss_mlp": 1.02875233, "epoch": 0.15120998045994288, "flos": 20520654514560.0, "grad_norm": 5.903596646191268, "language_loss": 0.73183751, "learning_rate": 3.778828908749082e-06, "loss": 0.75326133, "num_input_tokens_seen": 54620590, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.74609375, "step": 2515, "time_per_iteration": 2.3934273719787598 }, { "auxiliary_loss_clip": 0.0110383, "auxiliary_loss_mlp": 0.01039183, "balance_loss_clip": 1.0183332, "balance_loss_mlp": 1.02974045, "epoch": 0.15127010371261085, "flos": 21177824636160.0, "grad_norm": 1.8364367976316958, "language_loss": 0.7763139, "learning_rate": 3.7786561737957664e-06, "loss": 0.79774404, "num_input_tokens_seen": 54640410, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7421875, "step": 2516, "time_per_iteration": 2.3890445232391357 }, { "auxiliary_loss_clip": 0.01022081, "auxiliary_loss_mlp": 0.01006885, "balance_loss_clip": 1.00452423, "balance_loss_mlp": 1.00267529, "epoch": 0.1513302269652788, "flos": 65317500938880.0, "grad_norm": 0.7261458198856392, "language_loss": 0.54662186, "learning_rate": 3.7784833753667867e-06, "loss": 0.56691152, "num_input_tokens_seen": 54701430, "router_z_loss_clip": 0.02355957, "router_z_loss_mlp": 0.19335938, "step": 2517, "time_per_iteration": 3.0439090728759766 }, { "auxiliary_loss_clip": 0.01105508, "auxiliary_loss_mlp": 0.0104157, "balance_loss_clip": 1.01917112, "balance_loss_mlp": 1.02838504, "epoch": 0.15139035021794678, "flos": 19134812073600.0, "grad_norm": 2.059296618904718, "language_loss": 0.78328919, "learning_rate": 3.7783105134683108e-06, "loss": 0.80475998, "num_input_tokens_seen": 54720845, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.7734375, "step": 2518, "time_per_iteration": 2.396969795227051 }, { "auxiliary_loss_clip": 0.01109758, "auxiliary_loss_mlp": 0.01046193, "balance_loss_clip": 1.02384138, "balance_loss_mlp": 1.03013384, "epoch": 0.15145047347061477, "flos": 26577352131840.0, "grad_norm": 2.0281620108176632, "language_loss": 0.69986463, "learning_rate": 3.7781375881065066e-06, "loss": 0.7214241, "num_input_tokens_seen": 54740495, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.796875, "step": 2519, "time_per_iteration": 2.4198665618896484 }, { "auxiliary_loss_clip": 0.01106441, "auxiliary_loss_mlp": 0.01041322, "balance_loss_clip": 1.02094889, "balance_loss_mlp": 1.02969408, "epoch": 0.15151059672328274, "flos": 20301923646720.0, "grad_norm": 2.499711858783702, "language_loss": 0.78746629, "learning_rate": 3.7779645992875453e-06, "loss": 0.80894399, "num_input_tokens_seen": 54758415, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.76953125, "step": 2520, "time_per_iteration": 2.375433921813965 }, { "auxiliary_loss_clip": 0.0111098, "auxiliary_loss_mlp": 0.01047152, "balance_loss_clip": 1.02419233, "balance_loss_mlp": 1.03058589, "epoch": 0.1515707199759507, "flos": 27227330513280.0, "grad_norm": 1.7823679850582101, "language_loss": 0.74635911, "learning_rate": 3.7777915470176013e-06, "loss": 0.7679404, "num_input_tokens_seen": 54779355, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.8046875, "step": 2521, "time_per_iteration": 2.4408998489379883 }, { "auxiliary_loss_clip": 0.01110329, "auxiliary_loss_mlp": 0.01043944, "balance_loss_clip": 1.0210675, "balance_loss_mlp": 1.03026247, "epoch": 0.15163084322861867, "flos": 23586203243520.0, "grad_norm": 1.9815758785267137, "language_loss": 0.81626248, "learning_rate": 3.7776184313028504e-06, "loss": 0.83780521, "num_input_tokens_seen": 54799465, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.80078125, "step": 2522, "time_per_iteration": 2.3857479095458984 }, { "auxiliary_loss_clip": 0.01027446, "auxiliary_loss_mlp": 0.01011436, "balance_loss_clip": 1.00920677, "balance_loss_mlp": 1.00768375, "epoch": 0.15169096648128663, "flos": 66886427882880.0, "grad_norm": 0.8246270249677684, "language_loss": 0.57857478, "learning_rate": 3.7774452521494703e-06, "loss": 0.59896362, "num_input_tokens_seen": 54857665, "router_z_loss_clip": 0.02233887, "router_z_loss_mlp": 0.19726562, "step": 2523, "time_per_iteration": 2.964157819747925 }, { "auxiliary_loss_clip": 0.01105539, "auxiliary_loss_mlp": 0.01043856, "balance_loss_clip": 1.02152777, "balance_loss_mlp": 1.02922678, "epoch": 0.1517510897339546, "flos": 29094171022080.0, "grad_norm": 1.7355717654770246, "language_loss": 0.74829638, "learning_rate": 3.777272009563641e-06, "loss": 0.76979029, "num_input_tokens_seen": 54879895, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.765625, "step": 2524, "time_per_iteration": 2.434436559677124 }, { "auxiliary_loss_clip": 0.01103708, "auxiliary_loss_mlp": 0.01045557, "balance_loss_clip": 1.02325308, "balance_loss_mlp": 1.0275718, "epoch": 0.1518112129866226, "flos": 18405546261120.0, "grad_norm": 2.2503896736230757, "language_loss": 0.74488342, "learning_rate": 3.7770987035515454e-06, "loss": 0.76637608, "num_input_tokens_seen": 54898245, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.76171875, "step": 2525, "time_per_iteration": 2.3705458641052246 }, { "auxiliary_loss_clip": 0.01110369, "auxiliary_loss_mlp": 0.01043974, "balance_loss_clip": 1.02100289, "balance_loss_mlp": 1.03120804, "epoch": 0.15187133623929056, "flos": 19424451646080.0, "grad_norm": 1.7240981580721493, "language_loss": 0.79515433, "learning_rate": 3.7769253341193677e-06, "loss": 0.81669778, "num_input_tokens_seen": 54917060, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.79296875, "step": 2526, "time_per_iteration": 2.3717384338378906 }, { "auxiliary_loss_clip": 0.01101619, "auxiliary_loss_mlp": 0.01044334, "balance_loss_clip": 1.0238061, "balance_loss_mlp": 1.0290668, "epoch": 0.15193145949195852, "flos": 17565256725120.0, "grad_norm": 1.7227662650603675, "language_loss": 0.84842026, "learning_rate": 3.7767519012732968e-06, "loss": 0.86987978, "num_input_tokens_seen": 54936365, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.7265625, "step": 2527, "time_per_iteration": 2.3688910007476807 }, { "auxiliary_loss_clip": 0.01107457, "auxiliary_loss_mlp": 0.01039621, "balance_loss_clip": 1.01920068, "balance_loss_mlp": 1.0299511, "epoch": 0.15199158274462649, "flos": 36174731437440.0, "grad_norm": 2.117843194012049, "language_loss": 0.69113839, "learning_rate": 3.77657840501952e-06, "loss": 0.71260917, "num_input_tokens_seen": 54961365, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.77734375, "step": 2528, "time_per_iteration": 2.517273187637329 }, { "auxiliary_loss_clip": 0.01107373, "auxiliary_loss_mlp": 0.01045553, "balance_loss_clip": 1.02377367, "balance_loss_mlp": 1.03085923, "epoch": 0.15205170599729445, "flos": 23072980124160.0, "grad_norm": 1.8420005400037076, "language_loss": 0.86785525, "learning_rate": 3.77640484536423e-06, "loss": 0.88938451, "num_input_tokens_seen": 54980750, "router_z_loss_clip": 0.21777344, "router_z_loss_mlp": 0.765625, "step": 2529, "time_per_iteration": 2.3985602855682373 }, { "auxiliary_loss_clip": 0.01104006, "auxiliary_loss_mlp": 0.01037226, "balance_loss_clip": 1.0161376, "balance_loss_mlp": 1.02957439, "epoch": 0.15211182924996242, "flos": 21907299916800.0, "grad_norm": 1.9081949275850647, "language_loss": 0.83878064, "learning_rate": 3.7762312223136206e-06, "loss": 0.86019295, "num_input_tokens_seen": 54999675, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.74609375, "step": 2530, "time_per_iteration": 2.369981050491333 }, { "auxiliary_loss_clip": 0.01107426, "auxiliary_loss_mlp": 0.01041581, "balance_loss_clip": 1.01987326, "balance_loss_mlp": 1.03080463, "epoch": 0.15217195250263038, "flos": 13880662945920.0, "grad_norm": 2.195533786066148, "language_loss": 0.80004138, "learning_rate": 3.7760575358738885e-06, "loss": 0.82153153, "num_input_tokens_seen": 55018295, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.765625, "step": 2531, "time_per_iteration": 2.3685219287872314 }, { "auxiliary_loss_clip": 0.011084, "auxiliary_loss_mlp": 0.01044006, "balance_loss_clip": 1.02345395, "balance_loss_mlp": 1.03170586, "epoch": 0.15223207575529837, "flos": 24534165012480.0, "grad_norm": 1.812882691072394, "language_loss": 0.78945243, "learning_rate": 3.7758837860512306e-06, "loss": 0.81097651, "num_input_tokens_seen": 55037975, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.765625, "step": 2532, "time_per_iteration": 2.4467689990997314 }, { "auxiliary_loss_clip": 0.01106842, "auxiliary_loss_mlp": 0.01041883, "balance_loss_clip": 1.02085471, "balance_loss_mlp": 1.03198981, "epoch": 0.15229219900796634, "flos": 25555618926720.0, "grad_norm": 4.311532411440558, "language_loss": 0.87922168, "learning_rate": 3.775709972851849e-06, "loss": 0.90070897, "num_input_tokens_seen": 55057135, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.75, "step": 2533, "time_per_iteration": 2.43026065826416 }, { "auxiliary_loss_clip": 0.01106862, "auxiliary_loss_mlp": 0.01046504, "balance_loss_clip": 1.02447379, "balance_loss_mlp": 1.03032994, "epoch": 0.1523523222606343, "flos": 18216980674560.0, "grad_norm": 2.273913263189555, "language_loss": 0.78457522, "learning_rate": 3.775536096281946e-06, "loss": 0.80610883, "num_input_tokens_seen": 55075525, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.765625, "step": 2534, "time_per_iteration": 2.351072072982788 }, { "auxiliary_loss_clip": 0.0110932, "auxiliary_loss_mlp": 0.01040589, "balance_loss_clip": 1.01833248, "balance_loss_mlp": 1.02856922, "epoch": 0.15241244551330227, "flos": 13259278834560.0, "grad_norm": 3.0967219246986604, "language_loss": 0.76634681, "learning_rate": 3.7753621563477268e-06, "loss": 0.78784585, "num_input_tokens_seen": 55090845, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.80859375, "step": 2535, "time_per_iteration": 2.3607687950134277 }, { "auxiliary_loss_clip": 0.01112375, "auxiliary_loss_mlp": 0.01041494, "balance_loss_clip": 1.01929736, "balance_loss_mlp": 1.03086329, "epoch": 0.15247256876597023, "flos": 19714649800320.0, "grad_norm": 1.9995141359856081, "language_loss": 0.78141522, "learning_rate": 3.7751881530553993e-06, "loss": 0.80295384, "num_input_tokens_seen": 55108750, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.81640625, "step": 2536, "time_per_iteration": 2.3622381687164307 }, { "auxiliary_loss_clip": 0.01105197, "auxiliary_loss_mlp": 0.01042185, "balance_loss_clip": 1.02139473, "balance_loss_mlp": 1.03205538, "epoch": 0.1525326920186382, "flos": 20374822298880.0, "grad_norm": 3.0883739704224134, "language_loss": 0.76058221, "learning_rate": 3.775014086411173e-06, "loss": 0.78205609, "num_input_tokens_seen": 55126750, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.73046875, "step": 2537, "time_per_iteration": 2.370582342147827 }, { "auxiliary_loss_clip": 0.0110693, "auxiliary_loss_mlp": 0.01042626, "balance_loss_clip": 1.02166939, "balance_loss_mlp": 1.03100705, "epoch": 0.15259281527130616, "flos": 13589103248640.0, "grad_norm": 2.696911866493913, "language_loss": 0.77871943, "learning_rate": 3.7748399564212595e-06, "loss": 0.80021489, "num_input_tokens_seen": 55144690, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7578125, "step": 2538, "time_per_iteration": 2.358154058456421 }, { "auxiliary_loss_clip": 0.01102209, "auxiliary_loss_mlp": 0.0103443, "balance_loss_clip": 1.01554728, "balance_loss_mlp": 1.02940416, "epoch": 0.15265293852397416, "flos": 22859171758080.0, "grad_norm": 2.018316998131727, "language_loss": 0.89714652, "learning_rate": 3.7746657630918735e-06, "loss": 0.91851294, "num_input_tokens_seen": 55166055, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.7265625, "step": 2539, "time_per_iteration": 2.408341646194458 }, { "auxiliary_loss_clip": 0.01106273, "auxiliary_loss_mlp": 0.0104616, "balance_loss_clip": 1.02464235, "balance_loss_mlp": 1.02910089, "epoch": 0.15271306177664212, "flos": 29236931038080.0, "grad_norm": 2.0070992879303664, "language_loss": 0.93298948, "learning_rate": 3.7744915064292313e-06, "loss": 0.95451379, "num_input_tokens_seen": 55186285, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.7734375, "step": 2540, "time_per_iteration": 2.4411098957061768 }, { "auxiliary_loss_clip": 0.01100278, "auxiliary_loss_mlp": 0.01037735, "balance_loss_clip": 1.01820827, "balance_loss_mlp": 1.02696204, "epoch": 0.1527731850293101, "flos": 31244995463040.0, "grad_norm": 1.6043658348108991, "language_loss": 0.75354832, "learning_rate": 3.7743171864395524e-06, "loss": 0.77492845, "num_input_tokens_seen": 55207915, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.734375, "step": 2541, "time_per_iteration": 2.463897466659546 }, { "auxiliary_loss_clip": 0.01101776, "auxiliary_loss_mlp": 0.01041522, "balance_loss_clip": 1.02097011, "balance_loss_mlp": 1.02875042, "epoch": 0.15283330828197805, "flos": 22381001510400.0, "grad_norm": 1.6350841919306038, "language_loss": 0.81249166, "learning_rate": 3.774142803129057e-06, "loss": 0.83392459, "num_input_tokens_seen": 55227860, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.73046875, "step": 2542, "time_per_iteration": 2.385455369949341 }, { "auxiliary_loss_clip": 0.01106315, "auxiliary_loss_mlp": 0.01043719, "balance_loss_clip": 1.02259505, "balance_loss_mlp": 1.02995694, "epoch": 0.15289343153464602, "flos": 25518960132480.0, "grad_norm": 1.7618678381994837, "language_loss": 0.77379119, "learning_rate": 3.7739683565039674e-06, "loss": 0.79529154, "num_input_tokens_seen": 55247330, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.765625, "step": 2543, "time_per_iteration": 3.8703291416168213 }, { "auxiliary_loss_clip": 0.01103589, "auxiliary_loss_mlp": 0.01037816, "balance_loss_clip": 1.01696634, "balance_loss_mlp": 1.02916515, "epoch": 0.15295355478731398, "flos": 22708940711040.0, "grad_norm": 1.898209789518137, "language_loss": 0.86182797, "learning_rate": 3.7737938465705115e-06, "loss": 0.88324201, "num_input_tokens_seen": 55266195, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.74609375, "step": 2544, "time_per_iteration": 2.3836936950683594 }, { "auxiliary_loss_clip": 0.01105948, "auxiliary_loss_mlp": 0.01043145, "balance_loss_clip": 1.02056634, "balance_loss_mlp": 1.02821922, "epoch": 0.15301367803998198, "flos": 23250967568640.0, "grad_norm": 2.012905755527917, "language_loss": 0.8258521, "learning_rate": 3.773619273334916e-06, "loss": 0.84734297, "num_input_tokens_seen": 55283305, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.77734375, "step": 2545, "time_per_iteration": 3.8111298084259033 }, { "auxiliary_loss_clip": 0.01103459, "auxiliary_loss_mlp": 0.01039394, "balance_loss_clip": 1.01785302, "balance_loss_mlp": 1.02929902, "epoch": 0.15307380129264994, "flos": 25885059315840.0, "grad_norm": 2.5491668962390683, "language_loss": 0.71097058, "learning_rate": 3.77344463680341e-06, "loss": 0.73239911, "num_input_tokens_seen": 55303035, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.7421875, "step": 2546, "time_per_iteration": 2.455937147140503 }, { "auxiliary_loss_clip": 0.01103855, "auxiliary_loss_mlp": 0.01043118, "balance_loss_clip": 1.02050447, "balance_loss_mlp": 1.02750611, "epoch": 0.1531339245453179, "flos": 46971482279040.0, "grad_norm": 1.8089644794002437, "language_loss": 0.77684152, "learning_rate": 3.7732699369822276e-06, "loss": 0.79831123, "num_input_tokens_seen": 55327570, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.76171875, "step": 2547, "time_per_iteration": 3.963167190551758 }, { "auxiliary_loss_clip": 0.01105039, "auxiliary_loss_mlp": 0.0104707, "balance_loss_clip": 1.02424145, "balance_loss_mlp": 1.02820408, "epoch": 0.15319404779798587, "flos": 35880588299520.0, "grad_norm": 2.501128504882286, "language_loss": 0.74221045, "learning_rate": 3.7730951738776025e-06, "loss": 0.76373148, "num_input_tokens_seen": 55351090, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.76953125, "step": 2548, "time_per_iteration": 2.5057740211486816 }, { "auxiliary_loss_clip": 0.01107012, "auxiliary_loss_mlp": 0.01040167, "balance_loss_clip": 1.01805377, "balance_loss_mlp": 1.02945113, "epoch": 0.15325417105065384, "flos": 25663500627840.0, "grad_norm": 1.3539366184629202, "language_loss": 0.80474466, "learning_rate": 3.7729203474957715e-06, "loss": 0.82621646, "num_input_tokens_seen": 55371050, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.7734375, "step": 2549, "time_per_iteration": 2.4154052734375 }, { "auxiliary_loss_clip": 0.01105433, "auxiliary_loss_mlp": 0.01038403, "balance_loss_clip": 1.0184834, "balance_loss_mlp": 1.02960038, "epoch": 0.1533142943033218, "flos": 18769830053760.0, "grad_norm": 1.7346121941856547, "language_loss": 0.74965739, "learning_rate": 3.7727454578429735e-06, "loss": 0.77109581, "num_input_tokens_seen": 55390375, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.7578125, "step": 2550, "time_per_iteration": 2.369154691696167 }, { "auxiliary_loss_clip": 0.01107496, "auxiliary_loss_mlp": 0.01047447, "balance_loss_clip": 1.0250597, "balance_loss_mlp": 1.02971649, "epoch": 0.15337441755598977, "flos": 23106392161920.0, "grad_norm": 2.2196697060420028, "language_loss": 0.77113855, "learning_rate": 3.7725705049254507e-06, "loss": 0.79268789, "num_input_tokens_seen": 55408890, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.77734375, "step": 2551, "time_per_iteration": 2.379380941390991 }, { "auxiliary_loss_clip": 0.01020893, "auxiliary_loss_mlp": 0.01003217, "balance_loss_clip": 1.00111914, "balance_loss_mlp": 1.00219727, "epoch": 0.15343454080865776, "flos": 59857712703360.0, "grad_norm": 0.9447315928812187, "language_loss": 0.56754923, "learning_rate": 3.7723954887494457e-06, "loss": 0.58779031, "num_input_tokens_seen": 55463815, "router_z_loss_clip": 0.02099609, "router_z_loss_mlp": 0.1875, "step": 2552, "time_per_iteration": 2.9446704387664795 }, { "auxiliary_loss_clip": 0.01107918, "auxiliary_loss_mlp": 0.01040425, "balance_loss_clip": 1.01816869, "balance_loss_mlp": 1.02918124, "epoch": 0.15349466406132573, "flos": 11910095187840.0, "grad_norm": 2.3160070802126898, "language_loss": 0.88518476, "learning_rate": 3.772220409321205e-06, "loss": 0.90666825, "num_input_tokens_seen": 55481050, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.7890625, "step": 2553, "time_per_iteration": 2.35170841217041 }, { "auxiliary_loss_clip": 0.01108115, "auxiliary_loss_mlp": 0.0104021, "balance_loss_clip": 1.01832306, "balance_loss_mlp": 1.02948189, "epoch": 0.1535547873139937, "flos": 24095795581440.0, "grad_norm": 3.7418927186046984, "language_loss": 0.78330117, "learning_rate": 3.7720452666469766e-06, "loss": 0.80478442, "num_input_tokens_seen": 55500050, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.78515625, "step": 2554, "time_per_iteration": 2.4206840991973877 }, { "auxiliary_loss_clip": 0.01111027, "auxiliary_loss_mlp": 0.01044822, "balance_loss_clip": 1.02211225, "balance_loss_mlp": 1.03178763, "epoch": 0.15361491056666166, "flos": 17565501104640.0, "grad_norm": 2.5973429021719974, "language_loss": 0.77826989, "learning_rate": 3.7718700607330114e-06, "loss": 0.79982841, "num_input_tokens_seen": 55518125, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.79296875, "step": 2555, "time_per_iteration": 2.356719970703125 }, { "auxiliary_loss_clip": 0.01102546, "auxiliary_loss_mlp": 0.01039915, "balance_loss_clip": 1.02055502, "balance_loss_mlp": 1.02782702, "epoch": 0.15367503381932962, "flos": 25044874513920.0, "grad_norm": 1.6601553386463048, "language_loss": 0.77098221, "learning_rate": 3.7716947915855607e-06, "loss": 0.7924068, "num_input_tokens_seen": 55540960, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.74609375, "step": 2556, "time_per_iteration": 2.433222532272339 }, { "auxiliary_loss_clip": 0.01102502, "auxiliary_loss_mlp": 0.01033546, "balance_loss_clip": 1.01278019, "balance_loss_mlp": 1.0288049, "epoch": 0.15373515707199759, "flos": 21506252595840.0, "grad_norm": 1.9197042729823908, "language_loss": 0.89976764, "learning_rate": 3.7715194592108805e-06, "loss": 0.92112815, "num_input_tokens_seen": 55559210, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.734375, "step": 2557, "time_per_iteration": 2.376099109649658 }, { "auxiliary_loss_clip": 0.01105636, "auxiliary_loss_mlp": 0.01042414, "balance_loss_clip": 1.01971674, "balance_loss_mlp": 1.02806902, "epoch": 0.15379528032466555, "flos": 25993534510080.0, "grad_norm": 1.9933591767860965, "language_loss": 0.71279323, "learning_rate": 3.7713440636152276e-06, "loss": 0.73427367, "num_input_tokens_seen": 55578925, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.7734375, "step": 2558, "time_per_iteration": 2.4223055839538574 }, { "auxiliary_loss_clip": 0.0110847, "auxiliary_loss_mlp": 0.01044002, "balance_loss_clip": 1.02246046, "balance_loss_mlp": 1.03012931, "epoch": 0.15385540357733354, "flos": 19276420014720.0, "grad_norm": 2.4142626335273087, "language_loss": 0.91885328, "learning_rate": 3.7711686048048613e-06, "loss": 0.94037807, "num_input_tokens_seen": 55597255, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.78125, "step": 2559, "time_per_iteration": 2.371500253677368 }, { "auxiliary_loss_clip": 0.01106853, "auxiliary_loss_mlp": 0.01046784, "balance_loss_clip": 1.02265644, "balance_loss_mlp": 1.02879667, "epoch": 0.1539155268300015, "flos": 28547850067200.0, "grad_norm": 2.5031426019798815, "language_loss": 0.63263065, "learning_rate": 3.7709930827860445e-06, "loss": 0.65416706, "num_input_tokens_seen": 55619515, "router_z_loss_clip": 0.24023438, "router_z_loss_mlp": 0.78125, "step": 2560, "time_per_iteration": 2.436124324798584 }, { "auxiliary_loss_clip": 0.01105785, "auxiliary_loss_mlp": 0.01048935, "balance_loss_clip": 1.02601123, "balance_loss_mlp": 1.02788079, "epoch": 0.15397565008266947, "flos": 23546821363200.0, "grad_norm": 1.8593641817635, "language_loss": 0.88214654, "learning_rate": 3.770817497565039e-06, "loss": 0.90369374, "num_input_tokens_seen": 55640050, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.78125, "step": 2561, "time_per_iteration": 2.4040687084198 }, { "auxiliary_loss_clip": 0.01103428, "auxiliary_loss_mlp": 0.01036151, "balance_loss_clip": 1.01612425, "balance_loss_mlp": 1.02934492, "epoch": 0.15403577333533744, "flos": 17128842330240.0, "grad_norm": 1.8613872011541217, "language_loss": 0.8302772, "learning_rate": 3.770641849148113e-06, "loss": 0.85167301, "num_input_tokens_seen": 55658695, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.7421875, "step": 2562, "time_per_iteration": 2.362842559814453 }, { "auxiliary_loss_clip": 0.01111663, "auxiliary_loss_mlp": 0.01048927, "balance_loss_clip": 1.024418, "balance_loss_mlp": 1.03037024, "epoch": 0.1540958965880054, "flos": 17893545039360.0, "grad_norm": 2.6615220319396173, "language_loss": 0.74593759, "learning_rate": 3.7704661375415336e-06, "loss": 0.76754344, "num_input_tokens_seen": 55676340, "router_z_loss_clip": 0.24414062, "router_z_loss_mlp": 0.8125, "step": 2563, "time_per_iteration": 2.3713762760162354 }, { "auxiliary_loss_clip": 0.01106939, "auxiliary_loss_mlp": 0.01037819, "balance_loss_clip": 1.01505017, "balance_loss_mlp": 1.02760148, "epoch": 0.15415601984067337, "flos": 32159684839680.0, "grad_norm": 2.127160502792017, "language_loss": 0.7599268, "learning_rate": 3.770290362751572e-06, "loss": 0.78137439, "num_input_tokens_seen": 55698890, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.79296875, "step": 2564, "time_per_iteration": 2.4816644191741943 }, { "auxiliary_loss_clip": 0.01103838, "auxiliary_loss_mlp": 0.01044494, "balance_loss_clip": 1.02382302, "balance_loss_mlp": 1.02820563, "epoch": 0.15421614309334136, "flos": 24023280954240.0, "grad_norm": 2.3929769772717817, "language_loss": 0.70904052, "learning_rate": 3.7701145247845006e-06, "loss": 0.73052388, "num_input_tokens_seen": 55718535, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.7578125, "step": 2565, "time_per_iteration": 2.4122776985168457 }, { "auxiliary_loss_clip": 0.01102412, "auxiliary_loss_mlp": 0.01045623, "balance_loss_clip": 1.02420139, "balance_loss_mlp": 1.02667618, "epoch": 0.15427626634600933, "flos": 24385225685760.0, "grad_norm": 128.62132603744894, "language_loss": 0.72072661, "learning_rate": 3.7699386236465954e-06, "loss": 0.74220693, "num_input_tokens_seen": 55738970, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.7578125, "step": 2566, "time_per_iteration": 2.411787271499634 }, { "auxiliary_loss_clip": 0.01101311, "auxiliary_loss_mlp": 0.01036585, "balance_loss_clip": 1.01612926, "balance_loss_mlp": 1.02717113, "epoch": 0.1543363895986773, "flos": 23330394645120.0, "grad_norm": 1.7865310614845986, "language_loss": 0.85019439, "learning_rate": 3.769762659344134e-06, "loss": 0.87157333, "num_input_tokens_seen": 55759585, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.7421875, "step": 2567, "time_per_iteration": 2.410080671310425 }, { "auxiliary_loss_clip": 0.01107123, "auxiliary_loss_mlp": 0.01041647, "balance_loss_clip": 1.02020097, "balance_loss_mlp": 1.0298934, "epoch": 0.15439651285134526, "flos": 24273294266880.0, "grad_norm": 1.8003980210100428, "language_loss": 0.78207928, "learning_rate": 3.7695866318833946e-06, "loss": 0.80356699, "num_input_tokens_seen": 55779250, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.7734375, "step": 2568, "time_per_iteration": 2.424752950668335 }, { "auxiliary_loss_clip": 0.01105023, "auxiliary_loss_mlp": 0.01037149, "balance_loss_clip": 1.0147258, "balance_loss_mlp": 1.02836204, "epoch": 0.15445663610401322, "flos": 22455052237440.0, "grad_norm": 1.8000672053146936, "language_loss": 0.70164311, "learning_rate": 3.769410541270661e-06, "loss": 0.72306484, "num_input_tokens_seen": 55800470, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.765625, "step": 2569, "time_per_iteration": 2.3858604431152344 }, { "auxiliary_loss_clip": 0.01100376, "auxiliary_loss_mlp": 0.0104229, "balance_loss_clip": 1.02138114, "balance_loss_mlp": 1.02779078, "epoch": 0.1545167593566812, "flos": 22048558744320.0, "grad_norm": 1.6523212296947989, "language_loss": 0.76557863, "learning_rate": 3.7692343875122167e-06, "loss": 0.78700531, "num_input_tokens_seen": 55817795, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7265625, "step": 2570, "time_per_iteration": 2.38033390045166 }, { "auxiliary_loss_clip": 0.01103751, "auxiliary_loss_mlp": 0.01040179, "balance_loss_clip": 1.0186975, "balance_loss_mlp": 1.02916908, "epoch": 0.15457688260934915, "flos": 19317233260800.0, "grad_norm": 2.519584689727318, "language_loss": 0.77578133, "learning_rate": 3.769058170614348e-06, "loss": 0.79722065, "num_input_tokens_seen": 55836125, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.74609375, "step": 2571, "time_per_iteration": 2.377727508544922 }, { "auxiliary_loss_clip": 0.01104421, "auxiliary_loss_mlp": 0.01042916, "balance_loss_clip": 1.02193546, "balance_loss_mlp": 1.02845848, "epoch": 0.15463700586201715, "flos": 24132838400640.0, "grad_norm": 2.585517759659818, "language_loss": 0.82445037, "learning_rate": 3.768881890583344e-06, "loss": 0.84592372, "num_input_tokens_seen": 55855280, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.7578125, "step": 2572, "time_per_iteration": 2.402364730834961 }, { "auxiliary_loss_clip": 0.01107173, "auxiliary_loss_mlp": 0.01036979, "balance_loss_clip": 1.01581907, "balance_loss_mlp": 1.02824247, "epoch": 0.1546971291146851, "flos": 22419789897600.0, "grad_norm": 1.5434582100933483, "language_loss": 0.90369272, "learning_rate": 3.7687055474254946e-06, "loss": 0.92513418, "num_input_tokens_seen": 55875695, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.7890625, "step": 2573, "time_per_iteration": 2.402705669403076 }, { "auxiliary_loss_clip": 0.01106665, "auxiliary_loss_mlp": 0.01041862, "balance_loss_clip": 1.02113163, "balance_loss_mlp": 1.02890539, "epoch": 0.15475725236735308, "flos": 17529261246720.0, "grad_norm": 1.746927606225564, "language_loss": 0.70083201, "learning_rate": 3.7685291411470946e-06, "loss": 0.72231728, "num_input_tokens_seen": 55894575, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.77734375, "step": 2574, "time_per_iteration": 2.3665170669555664 }, { "auxiliary_loss_clip": 0.01105633, "auxiliary_loss_mlp": 0.01043801, "balance_loss_clip": 1.02161622, "balance_loss_mlp": 1.02813995, "epoch": 0.15481737562002104, "flos": 22560734522880.0, "grad_norm": 1.749430047223734, "language_loss": 0.82673597, "learning_rate": 3.768352671754439e-06, "loss": 0.84823036, "num_input_tokens_seen": 55912855, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.7734375, "step": 2575, "time_per_iteration": 2.394659996032715 }, { "auxiliary_loss_clip": 0.01105269, "auxiliary_loss_mlp": 0.01038444, "balance_loss_clip": 1.0179522, "balance_loss_mlp": 1.02768183, "epoch": 0.154877498872689, "flos": 24899391411840.0, "grad_norm": 2.0641206323001935, "language_loss": 0.85018152, "learning_rate": 3.7681761392538246e-06, "loss": 0.87161869, "num_input_tokens_seen": 55932375, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.7734375, "step": 2576, "time_per_iteration": 2.4128005504608154 }, { "auxiliary_loss_clip": 0.01102041, "auxiliary_loss_mlp": 0.01040264, "balance_loss_clip": 1.019629, "balance_loss_mlp": 1.026214, "epoch": 0.15493762212535697, "flos": 28146244164480.0, "grad_norm": 1.7035648936845607, "language_loss": 0.82154602, "learning_rate": 3.7679995436515525e-06, "loss": 0.84296906, "num_input_tokens_seen": 55953970, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.7578125, "step": 2577, "time_per_iteration": 2.43742036819458 }, { "auxiliary_loss_clip": 0.0110781, "auxiliary_loss_mlp": 0.01047088, "balance_loss_clip": 1.02558279, "balance_loss_mlp": 1.03000402, "epoch": 0.15499774537802496, "flos": 25409891445120.0, "grad_norm": 2.681903882832599, "language_loss": 0.76673013, "learning_rate": 3.7678228849539244e-06, "loss": 0.78827906, "num_input_tokens_seen": 55973120, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.78125, "step": 2578, "time_per_iteration": 2.4083006381988525 }, { "auxiliary_loss_clip": 0.01106969, "auxiliary_loss_mlp": 0.01040705, "balance_loss_clip": 1.01773357, "balance_loss_mlp": 1.02976203, "epoch": 0.15505786863069293, "flos": 22090454242560.0, "grad_norm": 2.1068175774035662, "language_loss": 0.82854289, "learning_rate": 3.767646163167245e-06, "loss": 0.85001969, "num_input_tokens_seen": 55993260, "router_z_loss_clip": 0.22949219, "router_z_loss_mlp": 0.76953125, "step": 2579, "time_per_iteration": 2.4026007652282715 }, { "auxiliary_loss_clip": 0.01104262, "auxiliary_loss_mlp": 0.01040047, "balance_loss_clip": 1.02056837, "balance_loss_mlp": 1.03090668, "epoch": 0.1551179918833609, "flos": 18916116117120.0, "grad_norm": 2.178197886575931, "language_loss": 0.80735964, "learning_rate": 3.7674693782978206e-06, "loss": 0.82880276, "num_input_tokens_seen": 56012130, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.734375, "step": 2580, "time_per_iteration": 2.367403268814087 }, { "auxiliary_loss_clip": 0.01024101, "auxiliary_loss_mlp": 0.01002831, "balance_loss_clip": 1.00063789, "balance_loss_mlp": 1.00525832, "epoch": 0.15517811513602886, "flos": 66235821096960.0, "grad_norm": 0.8381763744793062, "language_loss": 0.58836788, "learning_rate": 3.7672925303519605e-06, "loss": 0.60863721, "num_input_tokens_seen": 56079045, "router_z_loss_clip": 0.02197266, "router_z_loss_mlp": 0.18847656, "step": 2581, "time_per_iteration": 3.164335250854492 }, { "auxiliary_loss_clip": 0.01108483, "auxiliary_loss_mlp": 0.01039474, "balance_loss_clip": 1.01849318, "balance_loss_mlp": 1.02883208, "epoch": 0.15523823838869683, "flos": 24020034197760.0, "grad_norm": 2.419429780114484, "language_loss": 0.8530618, "learning_rate": 3.7671156193359764e-06, "loss": 0.87454134, "num_input_tokens_seen": 56098745, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.796875, "step": 2582, "time_per_iteration": 3.7996675968170166 }, { "auxiliary_loss_clip": 0.01103487, "auxiliary_loss_mlp": 0.01044752, "balance_loss_clip": 1.02330661, "balance_loss_mlp": 1.02763653, "epoch": 0.1552983616413648, "flos": 20484030631680.0, "grad_norm": 2.3637131348154115, "language_loss": 0.78676498, "learning_rate": 3.766938645256182e-06, "loss": 0.80824739, "num_input_tokens_seen": 56117655, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.7578125, "step": 2583, "time_per_iteration": 2.4146738052368164 }, { "auxiliary_loss_clip": 0.01103412, "auxiliary_loss_mlp": 0.01042965, "balance_loss_clip": 1.02308095, "balance_loss_mlp": 1.02806306, "epoch": 0.15535848489403276, "flos": 32122362729600.0, "grad_norm": 1.787448485889266, "language_loss": 0.76157773, "learning_rate": 3.766761608118892e-06, "loss": 0.78304148, "num_input_tokens_seen": 56141960, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.75390625, "step": 2584, "time_per_iteration": 2.497995376586914 }, { "auxiliary_loss_clip": 0.01102306, "auxiliary_loss_mlp": 0.01039194, "balance_loss_clip": 1.01690185, "balance_loss_mlp": 1.02709401, "epoch": 0.15541860814670075, "flos": 19097455052160.0, "grad_norm": 2.1192931896292055, "language_loss": 0.75837165, "learning_rate": 3.766584507930424e-06, "loss": 0.77978659, "num_input_tokens_seen": 56161430, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.75, "step": 2585, "time_per_iteration": 5.134189605712891 }, { "auxiliary_loss_clip": 0.01100754, "auxiliary_loss_mlp": 0.01039866, "balance_loss_clip": 1.01944494, "balance_loss_mlp": 1.02741408, "epoch": 0.1554787313993687, "flos": 19171086842880.0, "grad_norm": 2.61309060370953, "language_loss": 0.61490977, "learning_rate": 3.7664073446971e-06, "loss": 0.63631594, "num_input_tokens_seen": 56179390, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.734375, "step": 2586, "time_per_iteration": 3.731452465057373 }, { "auxiliary_loss_clip": 0.011037, "auxiliary_loss_mlp": 0.0103897, "balance_loss_clip": 1.01838279, "balance_loss_mlp": 1.02729452, "epoch": 0.15553885465203668, "flos": 16142895135360.0, "grad_norm": 1.5537971677562452, "language_loss": 0.80979955, "learning_rate": 3.7662301184252413e-06, "loss": 0.83122623, "num_input_tokens_seen": 56198020, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.76171875, "step": 2587, "time_per_iteration": 2.4011423587799072 }, { "auxiliary_loss_clip": 0.01106464, "auxiliary_loss_mlp": 0.01045561, "balance_loss_clip": 1.02371013, "balance_loss_mlp": 1.02809954, "epoch": 0.15559897790470464, "flos": 25336608768000.0, "grad_norm": 1.796175972885769, "language_loss": 0.88489425, "learning_rate": 3.766052829121173e-06, "loss": 0.90641451, "num_input_tokens_seen": 56218165, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.78125, "step": 2588, "time_per_iteration": 2.447763442993164 }, { "auxiliary_loss_clip": 0.01104995, "auxiliary_loss_mlp": 0.01047148, "balance_loss_clip": 1.02462912, "balance_loss_mlp": 1.0295099, "epoch": 0.1556591011573726, "flos": 23147659255680.0, "grad_norm": 2.333558389343231, "language_loss": 0.64972603, "learning_rate": 3.765875476791222e-06, "loss": 0.67124742, "num_input_tokens_seen": 56237160, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.75390625, "step": 2589, "time_per_iteration": 2.421673536300659 }, { "auxiliary_loss_clip": 0.01104252, "auxiliary_loss_mlp": 0.01036745, "balance_loss_clip": 1.01425052, "balance_loss_mlp": 1.02719223, "epoch": 0.15571922441004057, "flos": 25369811337600.0, "grad_norm": 1.7241537957140867, "language_loss": 0.82660699, "learning_rate": 3.765698061441718e-06, "loss": 0.84801698, "num_input_tokens_seen": 56257610, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.76953125, "step": 2590, "time_per_iteration": 2.4393231868743896 }, { "auxiliary_loss_clip": 0.01103367, "auxiliary_loss_mlp": 0.01036363, "balance_loss_clip": 1.01496434, "balance_loss_mlp": 1.02695584, "epoch": 0.15577934766270854, "flos": 14500510957440.0, "grad_norm": 1.987275243360299, "language_loss": 0.79317725, "learning_rate": 3.7655205830789918e-06, "loss": 0.81457454, "num_input_tokens_seen": 56275215, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.765625, "step": 2591, "time_per_iteration": 2.373990774154663 }, { "auxiliary_loss_clip": 0.01102398, "auxiliary_loss_mlp": 0.01047158, "balance_loss_clip": 1.02709532, "balance_loss_mlp": 1.02724028, "epoch": 0.15583947091537653, "flos": 37413031006080.0, "grad_norm": 3.0056621303756965, "language_loss": 0.64931399, "learning_rate": 3.7653430417093777e-06, "loss": 0.67080957, "num_input_tokens_seen": 56297130, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.75390625, "step": 2592, "time_per_iteration": 2.553753614425659 }, { "auxiliary_loss_clip": 0.01107732, "auxiliary_loss_mlp": 0.01042511, "balance_loss_clip": 1.01964641, "balance_loss_mlp": 1.02997577, "epoch": 0.1558995941680445, "flos": 21833668126080.0, "grad_norm": 1.990849898249795, "language_loss": 0.81727475, "learning_rate": 3.765165437339211e-06, "loss": 0.83877718, "num_input_tokens_seen": 56314995, "router_z_loss_clip": 0.22851562, "router_z_loss_mlp": 0.77734375, "step": 2593, "time_per_iteration": 2.397907018661499 }, { "auxiliary_loss_clip": 0.01100068, "auxiliary_loss_mlp": 0.01039175, "balance_loss_clip": 1.01808691, "balance_loss_mlp": 1.02801895, "epoch": 0.15595971742071246, "flos": 19791598170240.0, "grad_norm": 2.0859177285149797, "language_loss": 0.73165357, "learning_rate": 3.764987769974831e-06, "loss": 0.75304604, "num_input_tokens_seen": 56334005, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.71875, "step": 2594, "time_per_iteration": 2.378899335861206 }, { "auxiliary_loss_clip": 0.01098355, "auxiliary_loss_mlp": 0.01035254, "balance_loss_clip": 1.01486969, "balance_loss_mlp": 1.02698159, "epoch": 0.15601984067338043, "flos": 26720984931840.0, "grad_norm": 3.6423690182773587, "language_loss": 0.81098974, "learning_rate": 3.764810039622577e-06, "loss": 0.83232594, "num_input_tokens_seen": 56353795, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.71484375, "step": 2595, "time_per_iteration": 2.4521374702453613 }, { "auxiliary_loss_clip": 0.01100865, "auxiliary_loss_mlp": 0.01038839, "balance_loss_clip": 1.01794171, "balance_loss_mlp": 1.02678692, "epoch": 0.1560799639260484, "flos": 18368293973760.0, "grad_norm": 1.9787245185617681, "language_loss": 0.86365926, "learning_rate": 3.7646322462887927e-06, "loss": 0.88505626, "num_input_tokens_seen": 56373195, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7421875, "step": 2596, "time_per_iteration": 2.3852343559265137 }, { "auxiliary_loss_clip": 0.01099539, "auxiliary_loss_mlp": 0.01037294, "balance_loss_clip": 1.01727879, "balance_loss_mlp": 1.02835238, "epoch": 0.15614008717871636, "flos": 22597951898880.0, "grad_norm": 1.657866688818024, "language_loss": 0.68346548, "learning_rate": 3.764454389979822e-06, "loss": 0.70483381, "num_input_tokens_seen": 56391525, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.7109375, "step": 2597, "time_per_iteration": 2.417179822921753 }, { "auxiliary_loss_clip": 0.01097967, "auxiliary_loss_mlp": 0.01040386, "balance_loss_clip": 1.02059722, "balance_loss_mlp": 1.02725589, "epoch": 0.15620021043138435, "flos": 22745774062080.0, "grad_norm": 1.783937869785717, "language_loss": 0.79627144, "learning_rate": 3.7642764707020134e-06, "loss": 0.81765497, "num_input_tokens_seen": 56410715, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.70703125, "step": 2598, "time_per_iteration": 2.415555238723755 }, { "auxiliary_loss_clip": 0.01096528, "auxiliary_loss_mlp": 0.01032579, "balance_loss_clip": 1.01294494, "balance_loss_mlp": 1.02566028, "epoch": 0.15626033368405232, "flos": 13114109934720.0, "grad_norm": 2.19423517852279, "language_loss": 0.82752991, "learning_rate": 3.764098488461716e-06, "loss": 0.84882104, "num_input_tokens_seen": 56429170, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.7109375, "step": 2599, "time_per_iteration": 2.381692409515381 }, { "auxiliary_loss_clip": 0.01107132, "auxiliary_loss_mlp": 0.01037977, "balance_loss_clip": 1.01541018, "balance_loss_mlp": 1.02894783, "epoch": 0.15632045693672028, "flos": 16471358006400.0, "grad_norm": 2.8044434070259467, "language_loss": 0.81662029, "learning_rate": 3.7639204432652808e-06, "loss": 0.83807135, "num_input_tokens_seen": 56445685, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.78125, "step": 2600, "time_per_iteration": 2.3765108585357666 }, { "auxiliary_loss_clip": 0.01106237, "auxiliary_loss_mlp": 0.0103855, "balance_loss_clip": 1.01822448, "balance_loss_mlp": 1.02960777, "epoch": 0.15638058018938825, "flos": 20849291942400.0, "grad_norm": 1.8183566185622821, "language_loss": 0.884462, "learning_rate": 3.7637423351190628e-06, "loss": 0.9059099, "num_input_tokens_seen": 56465900, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.765625, "step": 2601, "time_per_iteration": 2.4123878479003906 }, { "auxiliary_loss_clip": 0.01107621, "auxiliary_loss_mlp": 0.01056924, "balance_loss_clip": 1.03384519, "balance_loss_mlp": 1.03044391, "epoch": 0.1564407034420562, "flos": 21871129881600.0, "grad_norm": 1.6946203892584524, "language_loss": 0.78171384, "learning_rate": 3.7635641640294177e-06, "loss": 0.80335927, "num_input_tokens_seen": 56485020, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.7734375, "step": 2602, "time_per_iteration": 2.4028422832489014 }, { "auxiliary_loss_clip": 0.01101798, "auxiliary_loss_mlp": 0.01039578, "balance_loss_clip": 1.01946712, "balance_loss_mlp": 1.02789998, "epoch": 0.15650082669472418, "flos": 21833493569280.0, "grad_norm": 3.598417753453261, "language_loss": 0.73629385, "learning_rate": 3.7633859300027036e-06, "loss": 0.7577076, "num_input_tokens_seen": 56505205, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.73828125, "step": 2603, "time_per_iteration": 2.4165844917297363 }, { "auxiliary_loss_clip": 0.01103958, "auxiliary_loss_mlp": 0.01045743, "balance_loss_clip": 1.02513182, "balance_loss_mlp": 1.02844143, "epoch": 0.15656094994739214, "flos": 13799909237760.0, "grad_norm": 2.672069342832569, "language_loss": 0.87356353, "learning_rate": 3.7632076330452823e-06, "loss": 0.89506054, "num_input_tokens_seen": 56521495, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.75390625, "step": 2604, "time_per_iteration": 2.3788928985595703 }, { "auxiliary_loss_clip": 0.01102621, "auxiliary_loss_mlp": 0.01041581, "balance_loss_clip": 1.02092218, "balance_loss_mlp": 1.0275898, "epoch": 0.15662107320006013, "flos": 27306967057920.0, "grad_norm": 1.9506636409082554, "language_loss": 0.85097289, "learning_rate": 3.7630292731635155e-06, "loss": 0.87241483, "num_input_tokens_seen": 56540665, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.75, "step": 2605, "time_per_iteration": 2.4615745544433594 }, { "auxiliary_loss_clip": 0.01107971, "auxiliary_loss_mlp": 0.01042204, "balance_loss_clip": 1.02082968, "balance_loss_mlp": 1.02778459, "epoch": 0.1566811964527281, "flos": 26683942112640.0, "grad_norm": 2.1938220667631048, "language_loss": 0.73083031, "learning_rate": 3.762850850363769e-06, "loss": 0.75233209, "num_input_tokens_seen": 56560805, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.80078125, "step": 2606, "time_per_iteration": 2.446239471435547 }, { "auxiliary_loss_clip": 0.01104533, "auxiliary_loss_mlp": 0.0103651, "balance_loss_clip": 1.01601839, "balance_loss_mlp": 1.0298152, "epoch": 0.15674131970539606, "flos": 16102605559680.0, "grad_norm": 2.216983009685828, "language_loss": 0.76660913, "learning_rate": 3.7626723646524107e-06, "loss": 0.78801954, "num_input_tokens_seen": 56576335, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.74609375, "step": 2607, "time_per_iteration": 2.374535083770752 }, { "auxiliary_loss_clip": 0.01101664, "auxiliary_loss_mlp": 0.01040982, "balance_loss_clip": 1.02083576, "balance_loss_mlp": 1.02859378, "epoch": 0.15680144295806403, "flos": 19168747781760.0, "grad_norm": 2.121054199236041, "language_loss": 0.81724632, "learning_rate": 3.7624938160358096e-06, "loss": 0.83867276, "num_input_tokens_seen": 56595880, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.73046875, "step": 2608, "time_per_iteration": 2.399024486541748 }, { "auxiliary_loss_clip": 0.01107222, "auxiliary_loss_mlp": 0.01045988, "balance_loss_clip": 1.02292061, "balance_loss_mlp": 1.02944684, "epoch": 0.156861566210732, "flos": 20812388768640.0, "grad_norm": 2.286626616381914, "language_loss": 0.72848833, "learning_rate": 3.762315204520338e-06, "loss": 0.75002038, "num_input_tokens_seen": 56615130, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.77734375, "step": 2609, "time_per_iteration": 2.435127019882202 }, { "auxiliary_loss_clip": 0.01103131, "auxiliary_loss_mlp": 0.01038042, "balance_loss_clip": 1.01733494, "balance_loss_mlp": 1.02717757, "epoch": 0.15692168946339996, "flos": 20046883098240.0, "grad_norm": 2.2163270139322533, "language_loss": 0.71791583, "learning_rate": 3.7621365301123696e-06, "loss": 0.73932755, "num_input_tokens_seen": 56634005, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.7578125, "step": 2610, "time_per_iteration": 2.417283296585083 }, { "auxiliary_loss_clip": 0.0110366, "auxiliary_loss_mlp": 0.01042168, "balance_loss_clip": 1.02022147, "balance_loss_mlp": 1.02626252, "epoch": 0.15698181271606793, "flos": 21396939528960.0, "grad_norm": 1.6604171307479039, "language_loss": 0.72618192, "learning_rate": 3.7619577928182816e-06, "loss": 0.74764025, "num_input_tokens_seen": 56653480, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.7734375, "step": 2611, "time_per_iteration": 2.42850399017334 }, { "auxiliary_loss_clip": 0.01102773, "auxiliary_loss_mlp": 0.0104017, "balance_loss_clip": 1.02003598, "balance_loss_mlp": 1.02798891, "epoch": 0.15704193596873592, "flos": 20844858199680.0, "grad_norm": 2.1770139587214623, "language_loss": 0.70722824, "learning_rate": 3.7617789926444525e-06, "loss": 0.72865766, "num_input_tokens_seen": 56672270, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.74609375, "step": 2612, "time_per_iteration": 2.4021122455596924 }, { "auxiliary_loss_clip": 0.01105137, "auxiliary_loss_mlp": 0.01046943, "balance_loss_clip": 1.02634406, "balance_loss_mlp": 1.02847147, "epoch": 0.15710205922140388, "flos": 21761816814720.0, "grad_norm": 1.971352266797598, "language_loss": 0.75976723, "learning_rate": 3.761600129597262e-06, "loss": 0.78128803, "num_input_tokens_seen": 56691510, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.765625, "step": 2613, "time_per_iteration": 2.4084362983703613 }, { "auxiliary_loss_clip": 0.01103495, "auxiliary_loss_mlp": 0.01048327, "balance_loss_clip": 1.02672625, "balance_loss_mlp": 1.02705002, "epoch": 0.15716218247407185, "flos": 25006644708480.0, "grad_norm": 1.6309071429618132, "language_loss": 0.65967524, "learning_rate": 3.761421203683095e-06, "loss": 0.68119335, "num_input_tokens_seen": 56712230, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.765625, "step": 2614, "time_per_iteration": 2.4340224266052246 }, { "auxiliary_loss_clip": 0.01106212, "auxiliary_loss_mlp": 0.01040308, "balance_loss_clip": 1.01833797, "balance_loss_mlp": 1.02878881, "epoch": 0.1572223057267398, "flos": 20190795189120.0, "grad_norm": 2.356254018131287, "language_loss": 0.74882823, "learning_rate": 3.7612422149083362e-06, "loss": 0.77029347, "num_input_tokens_seen": 56727490, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.7734375, "step": 2615, "time_per_iteration": 2.355894088745117 }, { "auxiliary_loss_clip": 0.01100076, "auxiliary_loss_mlp": 0.0104314, "balance_loss_clip": 1.02350664, "balance_loss_mlp": 1.02799809, "epoch": 0.15728242897940778, "flos": 20958465363840.0, "grad_norm": 2.0172598137008455, "language_loss": 0.73029327, "learning_rate": 3.761063163279373e-06, "loss": 0.75172544, "num_input_tokens_seen": 56747385, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.72265625, "step": 2616, "time_per_iteration": 2.3996548652648926 }, { "auxiliary_loss_clip": 0.01103443, "auxiliary_loss_mlp": 0.01042533, "balance_loss_clip": 1.02121794, "balance_loss_mlp": 1.02759457, "epoch": 0.15734255223207574, "flos": 23037194113920.0, "grad_norm": 1.952295697980959, "language_loss": 0.72702718, "learning_rate": 3.7608840488025955e-06, "loss": 0.748487, "num_input_tokens_seen": 56768055, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.7578125, "step": 2617, "time_per_iteration": 2.3851678371429443 }, { "auxiliary_loss_clip": 0.01102768, "auxiliary_loss_mlp": 0.01037228, "balance_loss_clip": 1.01672435, "balance_loss_mlp": 1.02899408, "epoch": 0.15740267548474374, "flos": 20550435770880.0, "grad_norm": 2.7305945018535875, "language_loss": 0.74240804, "learning_rate": 3.760704871484396e-06, "loss": 0.76380801, "num_input_tokens_seen": 56785110, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.734375, "step": 2618, "time_per_iteration": 2.3909361362457275 }, { "auxiliary_loss_clip": 0.01106578, "auxiliary_loss_mlp": 0.01043279, "balance_loss_clip": 1.01898444, "balance_loss_mlp": 1.02754009, "epoch": 0.1574627987374117, "flos": 22666032783360.0, "grad_norm": 1.918600653750494, "language_loss": 0.78889054, "learning_rate": 3.7605256313311684e-06, "loss": 0.8103891, "num_input_tokens_seen": 56804975, "router_z_loss_clip": 0.24316406, "router_z_loss_mlp": 0.7890625, "step": 2619, "time_per_iteration": 2.376849889755249 }, { "auxiliary_loss_clip": 0.01100153, "auxiliary_loss_mlp": 0.01035403, "balance_loss_clip": 1.01598394, "balance_loss_mlp": 1.02774501, "epoch": 0.15752292199007967, "flos": 16799716143360.0, "grad_norm": 1.9631199295647428, "language_loss": 0.76334906, "learning_rate": 3.7603463283493093e-06, "loss": 0.78470463, "num_input_tokens_seen": 56822470, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.72265625, "step": 2620, "time_per_iteration": 2.378913640975952 }, { "auxiliary_loss_clip": 0.01105, "auxiliary_loss_mlp": 0.01036747, "balance_loss_clip": 1.01451433, "balance_loss_mlp": 1.02806258, "epoch": 0.15758304524274763, "flos": 29824693643520.0, "grad_norm": 1.6950638831849694, "language_loss": 0.71077681, "learning_rate": 3.760166962545219e-06, "loss": 0.7321943, "num_input_tokens_seen": 56842100, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.76953125, "step": 2621, "time_per_iteration": 2.445683002471924 }, { "auxiliary_loss_clip": 0.01105344, "auxiliary_loss_mlp": 0.01040413, "balance_loss_clip": 1.01920557, "balance_loss_mlp": 1.02917194, "epoch": 0.1576431684954156, "flos": 53575478369280.0, "grad_norm": 2.043786453576383, "language_loss": 0.72216332, "learning_rate": 3.7599875339252962e-06, "loss": 0.74362087, "num_input_tokens_seen": 56865920, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.76171875, "step": 2622, "time_per_iteration": 4.0393757820129395 }, { "auxiliary_loss_clip": 0.01102739, "auxiliary_loss_mlp": 0.01036596, "balance_loss_clip": 1.01675987, "balance_loss_mlp": 1.0277462, "epoch": 0.15770329174808356, "flos": 20812563325440.0, "grad_norm": 1.728780941941876, "language_loss": 0.87419021, "learning_rate": 3.759808042495947e-06, "loss": 0.89558357, "num_input_tokens_seen": 56885265, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.75, "step": 2623, "time_per_iteration": 2.4093070030212402 }, { "auxiliary_loss_clip": 0.01103387, "auxiliary_loss_mlp": 0.01036534, "balance_loss_clip": 1.01710296, "balance_loss_mlp": 1.02887702, "epoch": 0.15776341500075153, "flos": 24972813734400.0, "grad_norm": 1.685512866388488, "language_loss": 0.81717169, "learning_rate": 3.7596284882635746e-06, "loss": 0.83857095, "num_input_tokens_seen": 56906710, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.74609375, "step": 2624, "time_per_iteration": 5.201894044876099 }, { "auxiliary_loss_clip": 0.01103976, "auxiliary_loss_mlp": 0.01039816, "balance_loss_clip": 1.01782215, "balance_loss_mlp": 1.02759087, "epoch": 0.15782353825341952, "flos": 21906846069120.0, "grad_norm": 2.6155308847246554, "language_loss": 0.7979489, "learning_rate": 3.7594488712345878e-06, "loss": 0.81938678, "num_input_tokens_seen": 56924275, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.765625, "step": 2625, "time_per_iteration": 2.397899866104126 }, { "auxiliary_loss_clip": 0.0110183, "auxiliary_loss_mlp": 0.01039829, "balance_loss_clip": 1.01994491, "balance_loss_mlp": 1.0284586, "epoch": 0.15788366150608749, "flos": 26175990608640.0, "grad_norm": 3.0363684040067476, "language_loss": 0.80167592, "learning_rate": 3.7592691914153967e-06, "loss": 0.82309252, "num_input_tokens_seen": 56941525, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.734375, "step": 2626, "time_per_iteration": 3.8083722591400146 }, { "auxiliary_loss_clip": 0.01104302, "auxiliary_loss_mlp": 0.0103839, "balance_loss_clip": 1.01789784, "balance_loss_mlp": 1.03113103, "epoch": 0.15794378475875545, "flos": 27708572960640.0, "grad_norm": 1.8168098782618698, "language_loss": 0.73536825, "learning_rate": 3.7590894488124134e-06, "loss": 0.75679517, "num_input_tokens_seen": 56962145, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.73046875, "step": 2627, "time_per_iteration": 2.457184076309204 }, { "auxiliary_loss_clip": 0.01102751, "auxiliary_loss_mlp": 0.01041898, "balance_loss_clip": 1.02078581, "balance_loss_mlp": 1.02858937, "epoch": 0.15800390801142342, "flos": 12129349726080.0, "grad_norm": 2.1391588192881947, "language_loss": 0.85239929, "learning_rate": 3.7589096434320534e-06, "loss": 0.87384582, "num_input_tokens_seen": 56977505, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.7421875, "step": 2628, "time_per_iteration": 2.3664088249206543 }, { "auxiliary_loss_clip": 0.01099321, "auxiliary_loss_mlp": 0.01036293, "balance_loss_clip": 1.01702857, "balance_loss_mlp": 1.0267477, "epoch": 0.15806403126409138, "flos": 20703669194880.0, "grad_norm": 1.8327200788202531, "language_loss": 0.76718879, "learning_rate": 3.7587297752807315e-06, "loss": 0.78854489, "num_input_tokens_seen": 56996770, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.7265625, "step": 2629, "time_per_iteration": 2.407407283782959 }, { "auxiliary_loss_clip": 0.01103757, "auxiliary_loss_mlp": 0.01045324, "balance_loss_clip": 1.02343762, "balance_loss_mlp": 1.02720749, "epoch": 0.15812415451675935, "flos": 17820751121280.0, "grad_norm": 2.4884600869974265, "language_loss": 0.73892325, "learning_rate": 3.758549844364869e-06, "loss": 0.76041412, "num_input_tokens_seen": 57014970, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.765625, "step": 2630, "time_per_iteration": 2.4013473987579346 }, { "auxiliary_loss_clip": 0.0110509, "auxiliary_loss_mlp": 0.01041841, "balance_loss_clip": 1.0194056, "balance_loss_mlp": 1.02786446, "epoch": 0.15818427776942734, "flos": 20083018222080.0, "grad_norm": 5.646154236075837, "language_loss": 0.83460271, "learning_rate": 3.7583698506908854e-06, "loss": 0.85607207, "num_input_tokens_seen": 57034045, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.76953125, "step": 2631, "time_per_iteration": 2.435514450073242 }, { "auxiliary_loss_clip": 0.01101833, "auxiliary_loss_mlp": 0.01036206, "balance_loss_clip": 1.01524854, "balance_loss_mlp": 1.02746201, "epoch": 0.1582444010220953, "flos": 21213855025920.0, "grad_norm": 1.702038878764565, "language_loss": 0.78231049, "learning_rate": 3.7581897942652046e-06, "loss": 0.80369091, "num_input_tokens_seen": 57053695, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7421875, "step": 2632, "time_per_iteration": 2.405518054962158 }, { "auxiliary_loss_clip": 0.01104133, "auxiliary_loss_mlp": 0.01050743, "balance_loss_clip": 1.0299412, "balance_loss_mlp": 1.0280633, "epoch": 0.15830452427476327, "flos": 17857375004160.0, "grad_norm": 2.156080894809283, "language_loss": 0.83225524, "learning_rate": 3.7580096750942535e-06, "loss": 0.85380399, "num_input_tokens_seen": 57071290, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.76171875, "step": 2633, "time_per_iteration": 2.364070177078247 }, { "auxiliary_loss_clip": 0.01104832, "auxiliary_loss_mlp": 0.01040668, "balance_loss_clip": 1.02018774, "balance_loss_mlp": 1.02880466, "epoch": 0.15836464752743123, "flos": 24533815898880.0, "grad_norm": 1.6509486986117194, "language_loss": 0.77444232, "learning_rate": 3.7578294931844584e-06, "loss": 0.79589731, "num_input_tokens_seen": 57091465, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.7578125, "step": 2634, "time_per_iteration": 2.4315414428710938 }, { "auxiliary_loss_clip": 0.01104861, "auxiliary_loss_mlp": 0.01038563, "balance_loss_clip": 1.01715338, "balance_loss_mlp": 1.02793598, "epoch": 0.1584247707800992, "flos": 20119781750400.0, "grad_norm": 3.32648958753033, "language_loss": 0.88971549, "learning_rate": 3.757649248542251e-06, "loss": 0.91114974, "num_input_tokens_seen": 57110075, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.76953125, "step": 2635, "time_per_iteration": 2.402858018875122 }, { "auxiliary_loss_clip": 0.01104143, "auxiliary_loss_mlp": 0.01043664, "balance_loss_clip": 1.02177715, "balance_loss_mlp": 1.02616, "epoch": 0.15848489403276717, "flos": 20374927032960.0, "grad_norm": 2.140323991383923, "language_loss": 0.75747037, "learning_rate": 3.757468941174063e-06, "loss": 0.77894843, "num_input_tokens_seen": 57128945, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.78125, "step": 2636, "time_per_iteration": 2.39902400970459 }, { "auxiliary_loss_clip": 0.01107464, "auxiliary_loss_mlp": 0.01042391, "balance_loss_clip": 1.02092123, "balance_loss_mlp": 1.02948594, "epoch": 0.15854501728543513, "flos": 39345368958720.0, "grad_norm": 2.2281739836646084, "language_loss": 0.71450502, "learning_rate": 3.7572885710863293e-06, "loss": 0.73600358, "num_input_tokens_seen": 57152385, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.78125, "step": 2637, "time_per_iteration": 2.563585042953491 }, { "auxiliary_loss_clip": 0.01102063, "auxiliary_loss_mlp": 0.01035121, "balance_loss_clip": 1.01518965, "balance_loss_mlp": 1.02702391, "epoch": 0.15860514053810312, "flos": 24863046819840.0, "grad_norm": 1.9987676543931971, "language_loss": 0.77517295, "learning_rate": 3.7571081382854866e-06, "loss": 0.79654485, "num_input_tokens_seen": 57172620, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.75, "step": 2638, "time_per_iteration": 2.4379286766052246 }, { "auxiliary_loss_clip": 0.01105014, "auxiliary_loss_mlp": 0.01041608, "balance_loss_clip": 1.0185771, "balance_loss_mlp": 1.02786207, "epoch": 0.1586652637907711, "flos": 26176479367680.0, "grad_norm": 1.7938691790713672, "language_loss": 0.75311208, "learning_rate": 3.756927642777974e-06, "loss": 0.77457821, "num_input_tokens_seen": 57194680, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.7734375, "step": 2639, "time_per_iteration": 2.448011636734009 }, { "auxiliary_loss_clip": 0.01106245, "auxiliary_loss_mlp": 0.01049362, "balance_loss_clip": 1.02793968, "balance_loss_mlp": 1.0296948, "epoch": 0.15872538704343905, "flos": 19791039588480.0, "grad_norm": 1.8529297947319283, "language_loss": 0.81090569, "learning_rate": 3.7567470845702337e-06, "loss": 0.83246183, "num_input_tokens_seen": 57214675, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.765625, "step": 2640, "time_per_iteration": 2.4345805644989014 }, { "auxiliary_loss_clip": 0.0110238, "auxiliary_loss_mlp": 0.0104355, "balance_loss_clip": 1.02324867, "balance_loss_mlp": 1.02789259, "epoch": 0.15878551029610702, "flos": 28474113542400.0, "grad_norm": 2.238978024847191, "language_loss": 0.66688108, "learning_rate": 3.756566463668709e-06, "loss": 0.68834043, "num_input_tokens_seen": 57235830, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.74609375, "step": 2641, "time_per_iteration": 2.429933547973633 }, { "auxiliary_loss_clip": 0.01110032, "auxiliary_loss_mlp": 0.01047067, "balance_loss_clip": 1.02529955, "balance_loss_mlp": 1.02991748, "epoch": 0.15884563354877498, "flos": 24205562496000.0, "grad_norm": 2.0593057564250232, "language_loss": 0.75106114, "learning_rate": 3.756385780079845e-06, "loss": 0.77263212, "num_input_tokens_seen": 57255970, "router_z_loss_clip": 0.21777344, "router_z_loss_mlp": 0.80078125, "step": 2642, "time_per_iteration": 2.417956590652466 }, { "auxiliary_loss_clip": 0.01099275, "auxiliary_loss_mlp": 0.01043671, "balance_loss_clip": 1.02241588, "balance_loss_mlp": 1.02721715, "epoch": 0.15890575680144295, "flos": 23948706556800.0, "grad_norm": 1.763166913728333, "language_loss": 0.70588106, "learning_rate": 3.756205033810091e-06, "loss": 0.72731048, "num_input_tokens_seen": 57274435, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.72265625, "step": 2643, "time_per_iteration": 2.4096288681030273 }, { "auxiliary_loss_clip": 0.01099905, "auxiliary_loss_mlp": 0.01037241, "balance_loss_clip": 1.01777434, "balance_loss_mlp": 1.02727807, "epoch": 0.15896588005411091, "flos": 21213959760000.0, "grad_norm": 2.136780877812778, "language_loss": 0.77865797, "learning_rate": 3.7560242248658963e-06, "loss": 0.8000294, "num_input_tokens_seen": 57293115, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.7265625, "step": 2644, "time_per_iteration": 2.4319334030151367 }, { "auxiliary_loss_clip": 0.01100841, "auxiliary_loss_mlp": 0.0104081, "balance_loss_clip": 1.02156985, "balance_loss_mlp": 1.02712774, "epoch": 0.1590260033067789, "flos": 24351255066240.0, "grad_norm": 1.8506680923764118, "language_loss": 0.8223685, "learning_rate": 3.7558433532537145e-06, "loss": 0.84378505, "num_input_tokens_seen": 57312565, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.734375, "step": 2645, "time_per_iteration": 2.4091849327087402 }, { "auxiliary_loss_clip": 0.01103085, "auxiliary_loss_mlp": 0.01038907, "balance_loss_clip": 1.01693726, "balance_loss_mlp": 1.02752018, "epoch": 0.15908612655944687, "flos": 32047648686720.0, "grad_norm": 2.162556548938065, "language_loss": 0.70025808, "learning_rate": 3.75566241898e-06, "loss": 0.72167802, "num_input_tokens_seen": 57333360, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.7578125, "step": 2646, "time_per_iteration": 2.494476079940796 }, { "auxiliary_loss_clip": 0.01098996, "auxiliary_loss_mlp": 0.01037495, "balance_loss_clip": 1.01734889, "balance_loss_mlp": 1.0268023, "epoch": 0.15914624981211484, "flos": 17784406529280.0, "grad_norm": 2.392693770113908, "language_loss": 0.62278962, "learning_rate": 3.7554814220512095e-06, "loss": 0.64415455, "num_input_tokens_seen": 57350575, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.72265625, "step": 2647, "time_per_iteration": 2.339232921600342 }, { "auxiliary_loss_clip": 0.01101412, "auxiliary_loss_mlp": 0.01039351, "balance_loss_clip": 1.01848936, "balance_loss_mlp": 1.02895546, "epoch": 0.1592063730647828, "flos": 17711542788480.0, "grad_norm": 2.094533690090337, "language_loss": 0.89786607, "learning_rate": 3.755300362473803e-06, "loss": 0.91927373, "num_input_tokens_seen": 57367570, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7265625, "step": 2648, "time_per_iteration": 2.3754818439483643 }, { "auxiliary_loss_clip": 0.01099667, "auxiliary_loss_mlp": 0.0103661, "balance_loss_clip": 1.01740479, "balance_loss_mlp": 1.02806485, "epoch": 0.15926649631745077, "flos": 18802648598400.0, "grad_norm": 1.784209771623308, "language_loss": 0.91517699, "learning_rate": 3.7551192402542418e-06, "loss": 0.93653977, "num_input_tokens_seen": 57383980, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.71484375, "step": 2649, "time_per_iteration": 2.3740234375 }, { "auxiliary_loss_clip": 0.01108733, "auxiliary_loss_mlp": 0.01039118, "balance_loss_clip": 1.01682544, "balance_loss_mlp": 1.02754092, "epoch": 0.15932661957011873, "flos": 17565291636480.0, "grad_norm": 2.4707960811613074, "language_loss": 0.71221823, "learning_rate": 3.7549380553989893e-06, "loss": 0.73369676, "num_input_tokens_seen": 57400840, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.8125, "step": 2650, "time_per_iteration": 2.391444206237793 }, { "auxiliary_loss_clip": 0.01097245, "auxiliary_loss_mlp": 0.01033142, "balance_loss_clip": 1.01448572, "balance_loss_mlp": 1.02721882, "epoch": 0.15938674282278673, "flos": 13333504118400.0, "grad_norm": 1.84575017835478, "language_loss": 0.71013993, "learning_rate": 3.7547568079145116e-06, "loss": 0.73144376, "num_input_tokens_seen": 57419230, "router_z_loss_clip": 0.18652344, "router_z_loss_mlp": 0.69921875, "step": 2651, "time_per_iteration": 2.355515956878662 }, { "auxiliary_loss_clip": 0.01102709, "auxiliary_loss_mlp": 0.01037868, "balance_loss_clip": 1.01563537, "balance_loss_mlp": 1.02707005, "epoch": 0.1594468660754547, "flos": 22487835870720.0, "grad_norm": 1.9484677562824262, "language_loss": 0.79622519, "learning_rate": 3.754575497807278e-06, "loss": 0.81763101, "num_input_tokens_seen": 57439315, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.7578125, "step": 2652, "time_per_iteration": 2.4259936809539795 }, { "auxiliary_loss_clip": 0.01100165, "auxiliary_loss_mlp": 0.01039507, "balance_loss_clip": 1.01925349, "balance_loss_mlp": 1.02832174, "epoch": 0.15950698932812266, "flos": 15006577248000.0, "grad_norm": 2.95076321606993, "language_loss": 0.69801968, "learning_rate": 3.7543941250837578e-06, "loss": 0.71941638, "num_input_tokens_seen": 57454635, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.71875, "step": 2653, "time_per_iteration": 2.325303554534912 }, { "auxiliary_loss_clip": 0.011016, "auxiliary_loss_mlp": 0.0103608, "balance_loss_clip": 1.01498032, "balance_loss_mlp": 1.02698135, "epoch": 0.15956711258079062, "flos": 30153715096320.0, "grad_norm": 2.0871788872937076, "language_loss": 0.77066928, "learning_rate": 3.7542126897504235e-06, "loss": 0.79204607, "num_input_tokens_seen": 57476805, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.74609375, "step": 2654, "time_per_iteration": 2.4599967002868652 }, { "auxiliary_loss_clip": 0.01098148, "auxiliary_loss_mlp": 0.01037046, "balance_loss_clip": 1.01642323, "balance_loss_mlp": 1.02560902, "epoch": 0.1596272358334586, "flos": 21031643306880.0, "grad_norm": 1.8948480458854995, "language_loss": 0.81581485, "learning_rate": 3.754031191813752e-06, "loss": 0.83716679, "num_input_tokens_seen": 57496400, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.7265625, "step": 2655, "time_per_iteration": 2.3683502674102783 }, { "auxiliary_loss_clip": 0.01099878, "auxiliary_loss_mlp": 0.01033338, "balance_loss_clip": 1.01475358, "balance_loss_mlp": 1.02669549, "epoch": 0.15968735908612655, "flos": 15267133791360.0, "grad_norm": 1.9719336390073554, "language_loss": 0.73297918, "learning_rate": 3.753849631280218e-06, "loss": 0.75431132, "num_input_tokens_seen": 57513700, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.73046875, "step": 2656, "time_per_iteration": 2.367332696914673 }, { "auxiliary_loss_clip": 0.01095125, "auxiliary_loss_mlp": 0.01037746, "balance_loss_clip": 1.01929212, "balance_loss_mlp": 1.02536428, "epoch": 0.15974748233879452, "flos": 52663791369600.0, "grad_norm": 2.1035566022409644, "language_loss": 0.77869081, "learning_rate": 3.7536680081563023e-06, "loss": 0.80001956, "num_input_tokens_seen": 57536180, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.69921875, "step": 2657, "time_per_iteration": 2.6401469707489014 }, { "auxiliary_loss_clip": 0.01099954, "auxiliary_loss_mlp": 0.01039954, "balance_loss_clip": 1.0214889, "balance_loss_mlp": 1.02863574, "epoch": 0.1598076055914625, "flos": 18732263564160.0, "grad_norm": 1.7498684972558385, "language_loss": 0.74488926, "learning_rate": 3.753486322448487e-06, "loss": 0.76628828, "num_input_tokens_seen": 57555025, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.71484375, "step": 2658, "time_per_iteration": 2.3692216873168945 }, { "auxiliary_loss_clip": 0.01099768, "auxiliary_loss_mlp": 0.01035301, "balance_loss_clip": 1.01396298, "balance_loss_mlp": 1.02635539, "epoch": 0.15986772884413047, "flos": 34347831390720.0, "grad_norm": 1.7291313209764942, "language_loss": 0.75411272, "learning_rate": 3.753304574163255e-06, "loss": 0.77546334, "num_input_tokens_seen": 57577660, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.734375, "step": 2659, "time_per_iteration": 2.4754250049591064 }, { "auxiliary_loss_clip": 0.01099666, "auxiliary_loss_mlp": 0.01040488, "balance_loss_clip": 1.01914978, "balance_loss_mlp": 1.0264492, "epoch": 0.15992785209679844, "flos": 22053865271040.0, "grad_norm": 1.9408709358154512, "language_loss": 0.90600204, "learning_rate": 3.7531227633070924e-06, "loss": 0.92740357, "num_input_tokens_seen": 57596335, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.734375, "step": 2660, "time_per_iteration": 2.3919284343719482 }, { "auxiliary_loss_clip": 0.01102115, "auxiliary_loss_mlp": 0.01036555, "balance_loss_clip": 1.01618242, "balance_loss_mlp": 1.02813041, "epoch": 0.1599879753494664, "flos": 33065436908160.0, "grad_norm": 1.6439194375650927, "language_loss": 0.77577305, "learning_rate": 3.7529408898864887e-06, "loss": 0.79715973, "num_input_tokens_seen": 57616830, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.7421875, "step": 2661, "time_per_iteration": 3.8710927963256836 }, { "auxiliary_loss_clip": 0.01099562, "auxiliary_loss_mlp": 0.01032414, "balance_loss_clip": 1.01285183, "balance_loss_mlp": 1.02597821, "epoch": 0.16004809860213437, "flos": 28036756540800.0, "grad_norm": 2.155112005459171, "language_loss": 0.74525195, "learning_rate": 3.752758953907933e-06, "loss": 0.7665717, "num_input_tokens_seen": 57635515, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.734375, "step": 2662, "time_per_iteration": 2.450744390487671 }, { "auxiliary_loss_clip": 0.0110121, "auxiliary_loss_mlp": 0.01042846, "balance_loss_clip": 1.02243745, "balance_loss_mlp": 1.02744985, "epoch": 0.16010822185480234, "flos": 22779116277120.0, "grad_norm": 1.948664428006108, "language_loss": 0.82199454, "learning_rate": 3.7525769553779192e-06, "loss": 0.84343511, "num_input_tokens_seen": 57654250, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.73828125, "step": 2663, "time_per_iteration": 3.7367546558380127 }, { "auxiliary_loss_clip": 0.01104684, "auxiliary_loss_mlp": 0.0103922, "balance_loss_clip": 1.01894271, "balance_loss_mlp": 1.02900457, "epoch": 0.16016834510747033, "flos": 20082983310720.0, "grad_norm": 1.9397599102893541, "language_loss": 0.80063188, "learning_rate": 3.7523948943029424e-06, "loss": 0.82207096, "num_input_tokens_seen": 57672645, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7578125, "step": 2664, "time_per_iteration": 3.814304828643799 }, { "auxiliary_loss_clip": 0.01099688, "auxiliary_loss_mlp": 0.01039963, "balance_loss_clip": 1.01994777, "balance_loss_mlp": 1.02588677, "epoch": 0.1602284683601383, "flos": 21172902134400.0, "grad_norm": 1.604862513318294, "language_loss": 0.93802118, "learning_rate": 3.752212770689499e-06, "loss": 0.9594177, "num_input_tokens_seen": 57691055, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.73828125, "step": 2665, "time_per_iteration": 2.383065938949585 }, { "auxiliary_loss_clip": 0.01101918, "auxiliary_loss_mlp": 0.01034422, "balance_loss_clip": 1.01403761, "balance_loss_mlp": 1.02683282, "epoch": 0.16028859161280626, "flos": 14646692286720.0, "grad_norm": 2.328704262901842, "language_loss": 0.84797919, "learning_rate": 3.752030584544089e-06, "loss": 0.86934257, "num_input_tokens_seen": 57707235, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.75, "step": 2666, "time_per_iteration": 3.7724173069000244 }, { "auxiliary_loss_clip": 0.01099267, "auxiliary_loss_mlp": 0.01039921, "balance_loss_clip": 1.02032328, "balance_loss_mlp": 1.02699661, "epoch": 0.16034871486547422, "flos": 20989433606400.0, "grad_norm": 2.2293109702379645, "language_loss": 0.81689608, "learning_rate": 3.7518483358732142e-06, "loss": 0.83828795, "num_input_tokens_seen": 57724190, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.72265625, "step": 2667, "time_per_iteration": 2.3819265365600586 }, { "auxiliary_loss_clip": 0.01100691, "auxiliary_loss_mlp": 0.01045485, "balance_loss_clip": 1.02395546, "balance_loss_mlp": 1.02797079, "epoch": 0.1604088381181422, "flos": 21396660238080.0, "grad_norm": 2.223020140601549, "language_loss": 0.74172294, "learning_rate": 3.751666024683379e-06, "loss": 0.76318473, "num_input_tokens_seen": 57743620, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.7265625, "step": 2668, "time_per_iteration": 2.4038949012756348 }, { "auxiliary_loss_clip": 0.01101106, "auxiliary_loss_mlp": 0.01041508, "balance_loss_clip": 1.02031231, "balance_loss_mlp": 1.02621579, "epoch": 0.16046896137081015, "flos": 23875947550080.0, "grad_norm": 1.5919316620720776, "language_loss": 0.77043045, "learning_rate": 3.751483650981089e-06, "loss": 0.79185653, "num_input_tokens_seen": 57764810, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.75, "step": 2669, "time_per_iteration": 2.435511350631714 }, { "auxiliary_loss_clip": 0.01026355, "auxiliary_loss_mlp": 0.01005916, "balance_loss_clip": 1.00378191, "balance_loss_mlp": 1.00718212, "epoch": 0.16052908462347812, "flos": 59803842608640.0, "grad_norm": 0.8001425358573404, "language_loss": 0.55502141, "learning_rate": 3.7513012147728527e-06, "loss": 0.57534409, "num_input_tokens_seen": 57824390, "router_z_loss_clip": 0.0213623, "router_z_loss_mlp": 0.19140625, "step": 2670, "time_per_iteration": 2.9251914024353027 }, { "auxiliary_loss_clip": 0.011007, "auxiliary_loss_mlp": 0.01038048, "balance_loss_clip": 1.01755571, "balance_loss_mlp": 1.02633071, "epoch": 0.1605892078761461, "flos": 18295569878400.0, "grad_norm": 1.9109288235358965, "language_loss": 0.77216643, "learning_rate": 3.751118716065181e-06, "loss": 0.79355395, "num_input_tokens_seen": 57843665, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.7421875, "step": 2671, "time_per_iteration": 2.3948540687561035 }, { "auxiliary_loss_clip": 0.01101305, "auxiliary_loss_mlp": 0.01034247, "balance_loss_clip": 1.01454139, "balance_loss_mlp": 1.02747202, "epoch": 0.16064933112881408, "flos": 32159370637440.0, "grad_norm": 2.0845246797628487, "language_loss": 0.65131581, "learning_rate": 3.750936154864587e-06, "loss": 0.67267138, "num_input_tokens_seen": 57863305, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.73828125, "step": 2672, "time_per_iteration": 2.483060598373413 }, { "auxiliary_loss_clip": 0.01101782, "auxiliary_loss_mlp": 0.01037121, "balance_loss_clip": 1.0153178, "balance_loss_mlp": 1.02615297, "epoch": 0.16070945438148204, "flos": 19827768205440.0, "grad_norm": 2.084943627089922, "language_loss": 0.85613823, "learning_rate": 3.750753531177586e-06, "loss": 0.8775273, "num_input_tokens_seen": 57883025, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.7578125, "step": 2673, "time_per_iteration": 2.3847758769989014 }, { "auxiliary_loss_clip": 0.01102021, "auxiliary_loss_mlp": 0.01043887, "balance_loss_clip": 1.02426529, "balance_loss_mlp": 1.02886093, "epoch": 0.16076957763415, "flos": 18912240956160.0, "grad_norm": 2.5506654545037857, "language_loss": 0.73004067, "learning_rate": 3.750570845010694e-06, "loss": 0.75149977, "num_input_tokens_seen": 57901430, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.73046875, "step": 2674, "time_per_iteration": 2.352660894393921 }, { "auxiliary_loss_clip": 0.01099736, "auxiliary_loss_mlp": 0.01037093, "balance_loss_clip": 1.01462245, "balance_loss_mlp": 1.0261476, "epoch": 0.16082970088681797, "flos": 16763406462720.0, "grad_norm": 1.5455839925373789, "language_loss": 0.8386209, "learning_rate": 3.7503880963704314e-06, "loss": 0.85998923, "num_input_tokens_seen": 57919550, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.734375, "step": 2675, "time_per_iteration": 2.3616671562194824 }, { "auxiliary_loss_clip": 0.01103789, "auxiliary_loss_mlp": 0.01039682, "balance_loss_clip": 1.018749, "balance_loss_mlp": 1.02890682, "epoch": 0.16088982413948594, "flos": 35148878691840.0, "grad_norm": 1.8855339884645612, "language_loss": 0.82327354, "learning_rate": 3.7502052852633206e-06, "loss": 0.84470832, "num_input_tokens_seen": 57939890, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.75, "step": 2676, "time_per_iteration": 2.47837233543396 }, { "auxiliary_loss_clip": 0.01099421, "auxiliary_loss_mlp": 0.01035022, "balance_loss_clip": 1.01689005, "balance_loss_mlp": 1.02847695, "epoch": 0.1609499473921539, "flos": 18624102572160.0, "grad_norm": 2.4144862954961335, "language_loss": 0.73110569, "learning_rate": 3.7500224116958856e-06, "loss": 0.75245011, "num_input_tokens_seen": 57957410, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.7109375, "step": 2677, "time_per_iteration": 2.363727569580078 }, { "auxiliary_loss_clip": 0.01096931, "auxiliary_loss_mlp": 0.01035353, "balance_loss_clip": 1.01620793, "balance_loss_mlp": 1.02643561, "epoch": 0.1610100706448219, "flos": 33144340314240.0, "grad_norm": 1.700185238420181, "language_loss": 0.7650227, "learning_rate": 3.7498394756746522e-06, "loss": 0.78634554, "num_input_tokens_seen": 57977900, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.703125, "step": 2678, "time_per_iteration": 2.464813232421875 }, { "auxiliary_loss_clip": 0.01101983, "auxiliary_loss_mlp": 0.01036238, "balance_loss_clip": 1.01424408, "balance_loss_mlp": 1.0274899, "epoch": 0.16107019389748986, "flos": 34675316743680.0, "grad_norm": 1.8492951103689623, "language_loss": 0.70696336, "learning_rate": 3.749656477206149e-06, "loss": 0.72834557, "num_input_tokens_seen": 57998210, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.7421875, "step": 2679, "time_per_iteration": 2.497814893722534 }, { "auxiliary_loss_clip": 0.01025193, "auxiliary_loss_mlp": 0.01011856, "balance_loss_clip": 1.00947165, "balance_loss_mlp": 1.00566459, "epoch": 0.16113031715015783, "flos": 65710483735680.0, "grad_norm": 0.7914817310226664, "language_loss": 0.51820886, "learning_rate": 3.749473416296906e-06, "loss": 0.53857934, "num_input_tokens_seen": 58059420, "router_z_loss_clip": 0.02380371, "router_z_loss_mlp": 0.1953125, "step": 2680, "time_per_iteration": 3.0602798461914062 }, { "auxiliary_loss_clip": 0.01100346, "auxiliary_loss_mlp": 0.01039493, "balance_loss_clip": 1.01643753, "balance_loss_mlp": 1.0259409, "epoch": 0.1611904404028258, "flos": 20809456214400.0, "grad_norm": 1.8493612624825924, "language_loss": 0.80466175, "learning_rate": 3.749290292953458e-06, "loss": 0.82606018, "num_input_tokens_seen": 58078370, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.7421875, "step": 2681, "time_per_iteration": 2.3731982707977295 }, { "auxiliary_loss_clip": 0.01098794, "auxiliary_loss_mlp": 0.01043045, "balance_loss_clip": 1.02272022, "balance_loss_mlp": 1.02759242, "epoch": 0.16125056365549376, "flos": 27012195515520.0, "grad_norm": 1.9213457875697393, "language_loss": 0.68854344, "learning_rate": 3.749107107182339e-06, "loss": 0.70996189, "num_input_tokens_seen": 58097395, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7109375, "step": 2682, "time_per_iteration": 2.4167189598083496 }, { "auxiliary_loss_clip": 0.01102691, "auxiliary_loss_mlp": 0.010446, "balance_loss_clip": 1.02196264, "balance_loss_mlp": 1.02975488, "epoch": 0.16131068690816172, "flos": 19275651964800.0, "grad_norm": 2.0126592915452126, "language_loss": 0.87158656, "learning_rate": 3.7489238589900855e-06, "loss": 0.89305949, "num_input_tokens_seen": 58115630, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.73046875, "step": 2683, "time_per_iteration": 2.3900914192199707 }, { "auxiliary_loss_clip": 0.01103146, "auxiliary_loss_mlp": 0.0104813, "balance_loss_clip": 1.02664828, "balance_loss_mlp": 1.02847111, "epoch": 0.16137081016082971, "flos": 35336396937600.0, "grad_norm": 1.9732873598850735, "language_loss": 0.74236965, "learning_rate": 3.7487405483832395e-06, "loss": 0.7638824, "num_input_tokens_seen": 58138655, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.74609375, "step": 2684, "time_per_iteration": 2.5172033309936523 }, { "auxiliary_loss_clip": 0.01107227, "auxiliary_loss_mlp": 0.01043533, "balance_loss_clip": 1.02189636, "balance_loss_mlp": 1.03007555, "epoch": 0.16143093341349768, "flos": 34233979847040.0, "grad_norm": 3.002548692009738, "language_loss": 0.70575935, "learning_rate": 3.748557175368341e-06, "loss": 0.72726703, "num_input_tokens_seen": 58157440, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.7734375, "step": 2685, "time_per_iteration": 2.5088298320770264 }, { "auxiliary_loss_clip": 0.01097151, "auxiliary_loss_mlp": 0.01038804, "balance_loss_clip": 1.01793098, "balance_loss_mlp": 1.02681971, "epoch": 0.16149105666616564, "flos": 27998072887680.0, "grad_norm": 1.8848704834199657, "language_loss": 0.716102, "learning_rate": 3.748373739951935e-06, "loss": 0.73746157, "num_input_tokens_seen": 58176660, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.703125, "step": 2686, "time_per_iteration": 2.4251043796539307 }, { "auxiliary_loss_clip": 0.01103306, "auxiliary_loss_mlp": 0.01045832, "balance_loss_clip": 1.0247798, "balance_loss_mlp": 1.03054428, "epoch": 0.1615511799188336, "flos": 19421344535040.0, "grad_norm": 2.039853388909567, "language_loss": 0.81668341, "learning_rate": 3.7481902421405676e-06, "loss": 0.83817482, "num_input_tokens_seen": 58195085, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.7265625, "step": 2687, "time_per_iteration": 2.3885385990142822 }, { "auxiliary_loss_clip": 0.01106753, "auxiliary_loss_mlp": 0.01043671, "balance_loss_clip": 1.01904213, "balance_loss_mlp": 1.02707458, "epoch": 0.16161130317150157, "flos": 22853865231360.0, "grad_norm": 1.776605083139513, "language_loss": 0.71621692, "learning_rate": 3.7480066819407876e-06, "loss": 0.7377212, "num_input_tokens_seen": 58213540, "router_z_loss_clip": 0.24609375, "router_z_loss_mlp": 0.796875, "step": 2688, "time_per_iteration": 2.383474349975586 }, { "auxiliary_loss_clip": 0.01101855, "auxiliary_loss_mlp": 0.01042002, "balance_loss_clip": 1.02196348, "balance_loss_mlp": 1.02826595, "epoch": 0.16167142642416954, "flos": 26109201444480.0, "grad_norm": 3.3158671752621918, "language_loss": 0.75798613, "learning_rate": 3.7478230593591448e-06, "loss": 0.77942467, "num_input_tokens_seen": 58236995, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.734375, "step": 2689, "time_per_iteration": 2.491985321044922 }, { "auxiliary_loss_clip": 0.01101707, "auxiliary_loss_mlp": 0.01040813, "balance_loss_clip": 1.01946294, "balance_loss_mlp": 1.02907157, "epoch": 0.1617315496768375, "flos": 22778662429440.0, "grad_norm": 1.83149192820103, "language_loss": 0.87536496, "learning_rate": 3.747639374402193e-06, "loss": 0.89679015, "num_input_tokens_seen": 58257230, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.7265625, "step": 2690, "time_per_iteration": 2.417999029159546 }, { "auxiliary_loss_clip": 0.01099128, "auxiliary_loss_mlp": 0.01040434, "balance_loss_clip": 1.02136016, "balance_loss_mlp": 1.02734601, "epoch": 0.1617916729295055, "flos": 22016228958720.0, "grad_norm": 1.8029494924009606, "language_loss": 0.88038915, "learning_rate": 3.7474556270764877e-06, "loss": 0.90178472, "num_input_tokens_seen": 58277080, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.71875, "step": 2691, "time_per_iteration": 2.3933136463165283 }, { "auxiliary_loss_clip": 0.01106955, "auxiliary_loss_mlp": 0.0105175, "balance_loss_clip": 1.02844501, "balance_loss_mlp": 1.02773464, "epoch": 0.16185179618217346, "flos": 23437194094080.0, "grad_norm": 2.117898701803228, "language_loss": 0.82161796, "learning_rate": 3.7472718173885864e-06, "loss": 0.84320498, "num_input_tokens_seen": 58294815, "router_z_loss_clip": 0.23339844, "router_z_loss_mlp": 0.79296875, "step": 2692, "time_per_iteration": 2.3905084133148193 }, { "auxiliary_loss_clip": 0.01105115, "auxiliary_loss_mlp": 0.01043661, "balance_loss_clip": 1.02046311, "balance_loss_mlp": 1.02833152, "epoch": 0.16191191943484143, "flos": 25664931993600.0, "grad_norm": 2.1941735514482166, "language_loss": 0.81331909, "learning_rate": 3.747087945345048e-06, "loss": 0.83480686, "num_input_tokens_seen": 58313215, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.765625, "step": 2693, "time_per_iteration": 2.413591146469116 }, { "auxiliary_loss_clip": 0.01100085, "auxiliary_loss_mlp": 0.01043148, "balance_loss_clip": 1.02325225, "balance_loss_mlp": 1.02796292, "epoch": 0.1619720426875094, "flos": 23476226860800.0, "grad_norm": 1.4848001684834402, "language_loss": 0.83649707, "learning_rate": 3.746904010952435e-06, "loss": 0.85792935, "num_input_tokens_seen": 58333215, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.71875, "step": 2694, "time_per_iteration": 2.411134958267212 }, { "auxiliary_loss_clip": 0.01107016, "auxiliary_loss_mlp": 0.01046708, "balance_loss_clip": 1.02434468, "balance_loss_mlp": 1.02975321, "epoch": 0.16203216594017736, "flos": 24132524198400.0, "grad_norm": 1.9641912622310724, "language_loss": 0.69131589, "learning_rate": 3.7467200142173114e-06, "loss": 0.71285313, "num_input_tokens_seen": 58351160, "router_z_loss_clip": 0.22363281, "router_z_loss_mlp": 0.7734375, "step": 2695, "time_per_iteration": 2.4020485877990723 }, { "auxiliary_loss_clip": 0.01106137, "auxiliary_loss_mlp": 0.0104104, "balance_loss_clip": 1.01900971, "balance_loss_mlp": 1.02939367, "epoch": 0.16209228919284532, "flos": 22339943884800.0, "grad_norm": 2.057423905456492, "language_loss": 0.82545096, "learning_rate": 3.7465359551462438e-06, "loss": 0.84692276, "num_input_tokens_seen": 58368505, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.765625, "step": 2696, "time_per_iteration": 2.3817596435546875 }, { "auxiliary_loss_clip": 0.01108746, "auxiliary_loss_mlp": 0.01043349, "balance_loss_clip": 1.02006698, "balance_loss_mlp": 1.0291909, "epoch": 0.1621524124455133, "flos": 15814222796160.0, "grad_norm": 2.223793290993585, "language_loss": 0.88445479, "learning_rate": 3.7463518337458006e-06, "loss": 0.90597576, "num_input_tokens_seen": 58385085, "router_z_loss_clip": 0.23242188, "router_z_loss_mlp": 0.796875, "step": 2697, "time_per_iteration": 2.373197317123413 }, { "auxiliary_loss_clip": 0.01097182, "auxiliary_loss_mlp": 0.01035197, "balance_loss_clip": 1.01657605, "balance_loss_mlp": 1.02738893, "epoch": 0.16221253569818128, "flos": 30185486300160.0, "grad_norm": 1.4813412941126236, "language_loss": 0.80739617, "learning_rate": 3.7461676500225522e-06, "loss": 0.82871991, "num_input_tokens_seen": 58406985, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.69921875, "step": 2698, "time_per_iteration": 2.4726316928863525 }, { "auxiliary_loss_clip": 0.01098231, "auxiliary_loss_mlp": 0.010466, "balance_loss_clip": 1.02551174, "balance_loss_mlp": 1.02775669, "epoch": 0.16227265895084925, "flos": 24604899160320.0, "grad_norm": 1.7044726119749638, "language_loss": 0.77323377, "learning_rate": 3.7459834039830726e-06, "loss": 0.79468215, "num_input_tokens_seen": 58426205, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.703125, "step": 2699, "time_per_iteration": 2.424938678741455 }, { "auxiliary_loss_clip": 0.01099858, "auxiliary_loss_mlp": 0.01036946, "balance_loss_clip": 1.01789641, "balance_loss_mlp": 1.02728641, "epoch": 0.1623327822035172, "flos": 19572308720640.0, "grad_norm": 2.732572726447219, "language_loss": 0.85681903, "learning_rate": 3.745799095633936e-06, "loss": 0.87818706, "num_input_tokens_seen": 58443830, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.7265625, "step": 2700, "time_per_iteration": 3.7618701457977295 }, { "auxiliary_loss_clip": 0.01099364, "auxiliary_loss_mlp": 0.01043829, "balance_loss_clip": 1.02216864, "balance_loss_mlp": 1.0269953, "epoch": 0.16239290545618518, "flos": 26467271015040.0, "grad_norm": 3.9493981243958216, "language_loss": 0.8032552, "learning_rate": 3.7456147249817203e-06, "loss": 0.82468712, "num_input_tokens_seen": 58464405, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.72265625, "step": 2701, "time_per_iteration": 2.429082155227661 }, { "auxiliary_loss_clip": 0.01101654, "auxiliary_loss_mlp": 0.01040281, "balance_loss_clip": 1.01992059, "balance_loss_mlp": 1.02973723, "epoch": 0.16245302870885314, "flos": 15851021235840.0, "grad_norm": 1.9193976785875857, "language_loss": 0.73022813, "learning_rate": 3.745430292033006e-06, "loss": 0.75164747, "num_input_tokens_seen": 58483295, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.71875, "step": 2702, "time_per_iteration": 3.7418477535247803 }, { "auxiliary_loss_clip": 0.01101791, "auxiliary_loss_mlp": 0.01042486, "balance_loss_clip": 1.02016985, "balance_loss_mlp": 1.02802634, "epoch": 0.1625131519615211, "flos": 14755656240000.0, "grad_norm": 2.2755867486125743, "language_loss": 0.72900951, "learning_rate": 3.745245796794374e-06, "loss": 0.75045222, "num_input_tokens_seen": 58501205, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.73828125, "step": 2703, "time_per_iteration": 3.726701021194458 }, { "auxiliary_loss_clip": 0.01102086, "auxiliary_loss_mlp": 0.01038425, "balance_loss_clip": 1.01637101, "balance_loss_mlp": 1.02656317, "epoch": 0.1625732752141891, "flos": 28219247550720.0, "grad_norm": 2.2852940608047865, "language_loss": 0.70878398, "learning_rate": 3.7450612392724084e-06, "loss": 0.73018903, "num_input_tokens_seen": 58522315, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.75390625, "step": 2704, "time_per_iteration": 2.432807683944702 }, { "auxiliary_loss_clip": 0.01026434, "auxiliary_loss_mlp": 0.01003702, "balance_loss_clip": 1.00136578, "balance_loss_mlp": 1.00683141, "epoch": 0.16263339846685707, "flos": 67324727491200.0, "grad_norm": 0.7802057449767931, "language_loss": 0.53309071, "learning_rate": 3.7448766194736967e-06, "loss": 0.55339205, "num_input_tokens_seen": 58586695, "router_z_loss_clip": 0.02331543, "router_z_loss_mlp": 0.19628906, "step": 2705, "time_per_iteration": 3.03957462310791 }, { "auxiliary_loss_clip": 0.01103926, "auxiliary_loss_mlp": 0.01042252, "balance_loss_clip": 1.02042484, "balance_loss_mlp": 1.02761436, "epoch": 0.16269352171952503, "flos": 14318299238400.0, "grad_norm": 2.8690243218573026, "language_loss": 0.75459617, "learning_rate": 3.7446919374048265e-06, "loss": 0.77605796, "num_input_tokens_seen": 58602435, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.765625, "step": 2706, "time_per_iteration": 3.706674814224243 }, { "auxiliary_loss_clip": 0.01101461, "auxiliary_loss_mlp": 0.01033724, "balance_loss_clip": 1.01388717, "balance_loss_mlp": 1.02723992, "epoch": 0.162753644972193, "flos": 28360087441920.0, "grad_norm": 1.8757145565592657, "language_loss": 0.72222096, "learning_rate": 3.7445071930723888e-06, "loss": 0.74357283, "num_input_tokens_seen": 58621275, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.7421875, "step": 2707, "time_per_iteration": 2.4413931369781494 }, { "auxiliary_loss_clip": 0.01103872, "auxiliary_loss_mlp": 0.01043134, "balance_loss_clip": 1.0217123, "balance_loss_mlp": 1.02834046, "epoch": 0.16281376822486096, "flos": 19936836892800.0, "grad_norm": 2.571560648137463, "language_loss": 0.83571339, "learning_rate": 3.7443223864829773e-06, "loss": 0.85718346, "num_input_tokens_seen": 58637550, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.75390625, "step": 2708, "time_per_iteration": 2.38787579536438 }, { "auxiliary_loss_clip": 0.01107758, "auxiliary_loss_mlp": 0.01043605, "balance_loss_clip": 1.01953697, "balance_loss_mlp": 1.02829206, "epoch": 0.16287389147752893, "flos": 21250653465600.0, "grad_norm": 2.0981569629576327, "language_loss": 0.86046529, "learning_rate": 3.7441375176431863e-06, "loss": 0.88197893, "num_input_tokens_seen": 58654135, "router_z_loss_clip": 0.24121094, "router_z_loss_mlp": 0.796875, "step": 2709, "time_per_iteration": 2.376641273498535 }, { "auxiliary_loss_clip": 0.01101736, "auxiliary_loss_mlp": 0.01040952, "balance_loss_clip": 1.02080548, "balance_loss_mlp": 1.02731037, "epoch": 0.1629340147301969, "flos": 19243671292800.0, "grad_norm": 1.6395461203937707, "language_loss": 0.91247582, "learning_rate": 3.7439525865596137e-06, "loss": 0.93390268, "num_input_tokens_seen": 58674320, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.7421875, "step": 2710, "time_per_iteration": 2.399232864379883 }, { "auxiliary_loss_clip": 0.01102421, "auxiliary_loss_mlp": 0.01043115, "balance_loss_clip": 1.02130008, "balance_loss_mlp": 1.02973938, "epoch": 0.16299413798286488, "flos": 21248803163520.0, "grad_norm": 2.4521110439754237, "language_loss": 0.81027466, "learning_rate": 3.7437675932388596e-06, "loss": 0.83173001, "num_input_tokens_seen": 58691000, "router_z_loss_clip": 0.21777344, "router_z_loss_mlp": 0.7265625, "step": 2711, "time_per_iteration": 2.367405891418457 }, { "auxiliary_loss_clip": 0.01104288, "auxiliary_loss_mlp": 0.01039016, "balance_loss_clip": 1.01658082, "balance_loss_mlp": 1.02587044, "epoch": 0.16305426123553285, "flos": 18769585674240.0, "grad_norm": 2.1562163814010247, "language_loss": 0.8089633, "learning_rate": 3.7435825376875253e-06, "loss": 0.83039629, "num_input_tokens_seen": 58710230, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.78125, "step": 2712, "time_per_iteration": 2.3526713848114014 }, { "auxiliary_loss_clip": 0.0110389, "auxiliary_loss_mlp": 0.01043683, "balance_loss_clip": 1.02184391, "balance_loss_mlp": 1.02715445, "epoch": 0.16311438448820081, "flos": 22086648904320.0, "grad_norm": 1.8909279713995406, "language_loss": 0.77136874, "learning_rate": 3.743397419912215e-06, "loss": 0.79284441, "num_input_tokens_seen": 58728610, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.765625, "step": 2713, "time_per_iteration": 2.3635666370391846 }, { "auxiliary_loss_clip": 0.01103318, "auxiliary_loss_mlp": 0.01045318, "balance_loss_clip": 1.02394414, "balance_loss_mlp": 1.03042424, "epoch": 0.16317450774086878, "flos": 16466889352320.0, "grad_norm": 2.6818998512786365, "language_loss": 0.7886489, "learning_rate": 3.7432122399195365e-06, "loss": 0.81013525, "num_input_tokens_seen": 58744385, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.7265625, "step": 2714, "time_per_iteration": 2.347163200378418 }, { "auxiliary_loss_clip": 0.0110423, "auxiliary_loss_mlp": 0.01040664, "balance_loss_clip": 1.02042222, "balance_loss_mlp": 1.02938485, "epoch": 0.16323463099353674, "flos": 24351778736640.0, "grad_norm": 1.650219947186336, "language_loss": 0.77981454, "learning_rate": 3.7430269977160956e-06, "loss": 0.80126345, "num_input_tokens_seen": 58763905, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.75, "step": 2715, "time_per_iteration": 2.402946949005127 }, { "auxiliary_loss_clip": 0.01099133, "auxiliary_loss_mlp": 0.0103536, "balance_loss_clip": 1.01517773, "balance_loss_mlp": 1.0264461, "epoch": 0.1632947542462047, "flos": 24899600880000.0, "grad_norm": 2.4567311825744897, "language_loss": 0.82195216, "learning_rate": 3.742841693308506e-06, "loss": 0.84329712, "num_input_tokens_seen": 58785580, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.7265625, "step": 2716, "time_per_iteration": 2.417941093444824 }, { "auxiliary_loss_clip": 0.01105238, "auxiliary_loss_mlp": 0.01042246, "balance_loss_clip": 1.02075291, "balance_loss_mlp": 1.0306592, "epoch": 0.1633548774988727, "flos": 24899112120960.0, "grad_norm": 1.936804784333361, "language_loss": 0.86132491, "learning_rate": 3.742656326703379e-06, "loss": 0.88279974, "num_input_tokens_seen": 58806075, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.74609375, "step": 2717, "time_per_iteration": 2.3981850147247314 }, { "auxiliary_loss_clip": 0.01100908, "auxiliary_loss_mlp": 0.01038622, "balance_loss_clip": 1.01877379, "balance_loss_mlp": 1.0287087, "epoch": 0.16341500075154067, "flos": 30440596671360.0, "grad_norm": 1.706598655723777, "language_loss": 0.76384556, "learning_rate": 3.7424708979073306e-06, "loss": 0.78524089, "num_input_tokens_seen": 58827405, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.72265625, "step": 2718, "time_per_iteration": 2.4446098804473877 }, { "auxiliary_loss_clip": 0.01102697, "auxiliary_loss_mlp": 0.01037988, "balance_loss_clip": 1.0174011, "balance_loss_mlp": 1.02754319, "epoch": 0.16347512400420863, "flos": 22783410374400.0, "grad_norm": 1.9844112790707533, "language_loss": 0.73798156, "learning_rate": 3.742285406926978e-06, "loss": 0.75938845, "num_input_tokens_seen": 58847205, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.75, "step": 2719, "time_per_iteration": 2.3830654621124268 }, { "auxiliary_loss_clip": 0.0110243, "auxiliary_loss_mlp": 0.0103931, "balance_loss_clip": 1.01919889, "balance_loss_mlp": 1.02733767, "epoch": 0.1635352472568766, "flos": 22632306543360.0, "grad_norm": 1.6711539850838706, "language_loss": 0.72027409, "learning_rate": 3.7420998537689402e-06, "loss": 0.74169153, "num_input_tokens_seen": 58866865, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.75, "step": 2720, "time_per_iteration": 2.3898520469665527 }, { "auxiliary_loss_clip": 0.01099884, "auxiliary_loss_mlp": 0.01038156, "balance_loss_clip": 1.01648426, "balance_loss_mlp": 1.02837658, "epoch": 0.16359537050954456, "flos": 15522104517120.0, "grad_norm": 2.0201840408562517, "language_loss": 0.75201935, "learning_rate": 3.7419142384398404e-06, "loss": 0.77339977, "num_input_tokens_seen": 58885200, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.71484375, "step": 2721, "time_per_iteration": 2.3708364963531494 }, { "auxiliary_loss_clip": 0.01102009, "auxiliary_loss_mlp": 0.01038147, "balance_loss_clip": 1.01720154, "balance_loss_mlp": 1.02607942, "epoch": 0.16365549376221253, "flos": 22089092699520.0, "grad_norm": 2.0125671119970114, "language_loss": 0.79488349, "learning_rate": 3.7417285609463026e-06, "loss": 0.81628501, "num_input_tokens_seen": 58906385, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7578125, "step": 2722, "time_per_iteration": 2.408007860183716 }, { "auxiliary_loss_clip": 0.01104363, "auxiliary_loss_mlp": 0.01044011, "balance_loss_clip": 1.02027655, "balance_loss_mlp": 1.02764964, "epoch": 0.1637156170148805, "flos": 24059276432640.0, "grad_norm": 3.464672780683019, "language_loss": 0.84411418, "learning_rate": 3.7415428212949524e-06, "loss": 0.86559796, "num_input_tokens_seen": 58925040, "router_z_loss_clip": 0.23730469, "router_z_loss_mlp": 0.765625, "step": 2723, "time_per_iteration": 2.450230598449707 }, { "auxiliary_loss_clip": 0.01097816, "auxiliary_loss_mlp": 0.01037045, "balance_loss_clip": 1.0164578, "balance_loss_mlp": 1.02698934, "epoch": 0.1637757402675485, "flos": 26684221403520.0, "grad_norm": 6.8876305134667035, "language_loss": 0.71284223, "learning_rate": 3.7413570194924183e-06, "loss": 0.73419076, "num_input_tokens_seen": 58944790, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.7109375, "step": 2724, "time_per_iteration": 2.427006721496582 }, { "auxiliary_loss_clip": 0.01097508, "auxiliary_loss_mlp": 0.01036383, "balance_loss_clip": 1.01665354, "balance_loss_mlp": 1.02673888, "epoch": 0.16383586352021645, "flos": 16106026872960.0, "grad_norm": 2.22160942867275, "language_loss": 0.70896482, "learning_rate": 3.741171155545332e-06, "loss": 0.73030376, "num_input_tokens_seen": 58962500, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.70703125, "step": 2725, "time_per_iteration": 2.3558108806610107 }, { "auxiliary_loss_clip": 0.01097869, "auxiliary_loss_mlp": 0.01036382, "balance_loss_clip": 1.01664102, "balance_loss_mlp": 1.02822781, "epoch": 0.16389598677288442, "flos": 19165151911680.0, "grad_norm": 2.929731921097319, "language_loss": 0.88497961, "learning_rate": 3.7409852294603255e-06, "loss": 0.90632212, "num_input_tokens_seen": 58980355, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.6953125, "step": 2726, "time_per_iteration": 2.37483549118042 }, { "auxiliary_loss_clip": 0.01105815, "auxiliary_loss_mlp": 0.01041284, "balance_loss_clip": 1.02011204, "balance_loss_mlp": 1.03108537, "epoch": 0.16395611002555238, "flos": 21505938393600.0, "grad_norm": 1.9957275556100098, "language_loss": 0.74080288, "learning_rate": 3.740799241244035e-06, "loss": 0.76227391, "num_input_tokens_seen": 58999505, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.75, "step": 2727, "time_per_iteration": 2.384979009628296 }, { "auxiliary_loss_clip": 0.01097125, "auxiliary_loss_mlp": 0.01038954, "balance_loss_clip": 1.01969004, "balance_loss_mlp": 1.02770567, "epoch": 0.16401623327822035, "flos": 21469838181120.0, "grad_norm": 1.7100790204559277, "language_loss": 0.82165432, "learning_rate": 3.7406131909030972e-06, "loss": 0.84301507, "num_input_tokens_seen": 59017930, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.6953125, "step": 2728, "time_per_iteration": 2.406728982925415 }, { "auxiliary_loss_clip": 0.01102966, "auxiliary_loss_mlp": 0.01040676, "balance_loss_clip": 1.02002883, "balance_loss_mlp": 1.02820706, "epoch": 0.1640763565308883, "flos": 13625378017920.0, "grad_norm": 7.722774197557603, "language_loss": 0.85067058, "learning_rate": 3.740427078444152e-06, "loss": 0.87210703, "num_input_tokens_seen": 59035130, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.74609375, "step": 2729, "time_per_iteration": 2.3586716651916504 }, { "auxiliary_loss_clip": 0.01100578, "auxiliary_loss_mlp": 0.01044189, "balance_loss_clip": 1.02415013, "balance_loss_mlp": 1.02738476, "epoch": 0.16413647978355628, "flos": 15450532496640.0, "grad_norm": 2.248498291358936, "language_loss": 0.72755969, "learning_rate": 3.7402409038738416e-06, "loss": 0.74900734, "num_input_tokens_seen": 59053080, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.73046875, "step": 2730, "time_per_iteration": 2.357088804244995 }, { "auxiliary_loss_clip": 0.01102125, "auxiliary_loss_mlp": 0.01043043, "balance_loss_clip": 1.02051187, "balance_loss_mlp": 1.0260247, "epoch": 0.16419660303622427, "flos": 45876955155840.0, "grad_norm": 1.7213639834879002, "language_loss": 0.74439585, "learning_rate": 3.7400546671988096e-06, "loss": 0.76584756, "num_input_tokens_seen": 59075610, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.76171875, "step": 2731, "time_per_iteration": 2.5883896350860596 }, { "auxiliary_loss_clip": 0.01105179, "auxiliary_loss_mlp": 0.0103957, "balance_loss_clip": 1.01845753, "balance_loss_mlp": 1.02862, "epoch": 0.16425672628889224, "flos": 18951832304640.0, "grad_norm": 2.8613939021894943, "language_loss": 0.79236877, "learning_rate": 3.739868368425702e-06, "loss": 0.81381625, "num_input_tokens_seen": 59094555, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.765625, "step": 2732, "time_per_iteration": 2.36692476272583 }, { "auxiliary_loss_clip": 0.0110444, "auxiliary_loss_mlp": 0.01039292, "balance_loss_clip": 1.01827526, "balance_loss_mlp": 1.02979863, "epoch": 0.1643168495415602, "flos": 24311943008640.0, "grad_norm": 2.5706923919132962, "language_loss": 0.69387078, "learning_rate": 3.7396820075611682e-06, "loss": 0.71530807, "num_input_tokens_seen": 59113515, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.74609375, "step": 2733, "time_per_iteration": 2.3920483589172363 }, { "auxiliary_loss_clip": 0.01103058, "auxiliary_loss_mlp": 0.01040968, "balance_loss_clip": 1.01924813, "balance_loss_mlp": 1.02886534, "epoch": 0.16437697279422817, "flos": 26427330552960.0, "grad_norm": 2.0778114265675827, "language_loss": 0.81116164, "learning_rate": 3.7394955846118585e-06, "loss": 0.83260185, "num_input_tokens_seen": 59133275, "router_z_loss_clip": 0.21777344, "router_z_loss_mlp": 0.7421875, "step": 2734, "time_per_iteration": 2.4219093322753906 }, { "auxiliary_loss_clip": 0.01100096, "auxiliary_loss_mlp": 0.01037873, "balance_loss_clip": 1.01756001, "balance_loss_mlp": 1.02732301, "epoch": 0.16443709604689613, "flos": 34530811159680.0, "grad_norm": 2.1915788350221095, "language_loss": 0.82217395, "learning_rate": 3.739309099584426e-06, "loss": 0.84355366, "num_input_tokens_seen": 59154095, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7265625, "step": 2735, "time_per_iteration": 2.478828191757202 }, { "auxiliary_loss_clip": 0.01099139, "auxiliary_loss_mlp": 0.01036727, "balance_loss_clip": 1.01740348, "balance_loss_mlp": 1.0274241, "epoch": 0.1644972192995641, "flos": 23256937411200.0, "grad_norm": 3.0648714549534146, "language_loss": 0.78555602, "learning_rate": 3.7391225524855256e-06, "loss": 0.80691475, "num_input_tokens_seen": 59173795, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.71875, "step": 2736, "time_per_iteration": 2.388718605041504 }, { "auxiliary_loss_clip": 0.01103637, "auxiliary_loss_mlp": 0.01040628, "balance_loss_clip": 1.0214355, "balance_loss_mlp": 1.03016293, "epoch": 0.1645573425522321, "flos": 26978329630080.0, "grad_norm": 1.7862917661888835, "language_loss": 0.81538427, "learning_rate": 3.738935943321815e-06, "loss": 0.83682692, "num_input_tokens_seen": 59191610, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.734375, "step": 2737, "time_per_iteration": 2.411057472229004 }, { "auxiliary_loss_clip": 0.0110028, "auxiliary_loss_mlp": 0.01038799, "balance_loss_clip": 1.01886678, "balance_loss_mlp": 1.02628779, "epoch": 0.16461746580490005, "flos": 28730480722560.0, "grad_norm": 1.9910116654991181, "language_loss": 0.87328762, "learning_rate": 3.7387492720999536e-06, "loss": 0.89467835, "num_input_tokens_seen": 59213000, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.7421875, "step": 2738, "time_per_iteration": 2.4444847106933594 }, { "auxiliary_loss_clip": 0.01099441, "auxiliary_loss_mlp": 0.01044621, "balance_loss_clip": 1.02408099, "balance_loss_mlp": 1.02695906, "epoch": 0.16467758905756802, "flos": 24929172668160.0, "grad_norm": 1.6850865175004341, "language_loss": 0.71940517, "learning_rate": 3.7385625388266037e-06, "loss": 0.7408458, "num_input_tokens_seen": 59232340, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.7265625, "step": 2739, "time_per_iteration": 2.3921971321105957 }, { "auxiliary_loss_clip": 0.01098789, "auxiliary_loss_mlp": 0.01035218, "balance_loss_clip": 1.01532221, "balance_loss_mlp": 1.02643645, "epoch": 0.16473771231023598, "flos": 24825375596160.0, "grad_norm": 3.8569228628265426, "language_loss": 0.81790274, "learning_rate": 3.7383757435084284e-06, "loss": 0.83924282, "num_input_tokens_seen": 59253950, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.72265625, "step": 2740, "time_per_iteration": 3.817305564880371 }, { "auxiliary_loss_clip": 0.01106972, "auxiliary_loss_mlp": 0.01048364, "balance_loss_clip": 1.02619159, "balance_loss_mlp": 1.0297575, "epoch": 0.16479783556290395, "flos": 39894482822400.0, "grad_norm": 2.491280067494279, "language_loss": 0.68863475, "learning_rate": 3.7381888861520943e-06, "loss": 0.71018815, "num_input_tokens_seen": 59275545, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.7734375, "step": 2741, "time_per_iteration": 2.5311717987060547 }, { "auxiliary_loss_clip": 0.01100268, "auxiliary_loss_mlp": 0.01035299, "balance_loss_clip": 1.01497364, "balance_loss_mlp": 1.02653241, "epoch": 0.16485795881557191, "flos": 19896163292160.0, "grad_norm": 1.7078130198013188, "language_loss": 0.79608095, "learning_rate": 3.73800196676427e-06, "loss": 0.8174367, "num_input_tokens_seen": 59293480, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.73828125, "step": 2742, "time_per_iteration": 3.780141592025757 }, { "auxiliary_loss_clip": 0.01099181, "auxiliary_loss_mlp": 0.0104227, "balance_loss_clip": 1.02131319, "balance_loss_mlp": 1.02702117, "epoch": 0.16491808206823988, "flos": 20555148804480.0, "grad_norm": 2.675536099106907, "language_loss": 0.8468293, "learning_rate": 3.737814985351627e-06, "loss": 0.86824381, "num_input_tokens_seen": 59313435, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.72265625, "step": 2743, "time_per_iteration": 3.7769858837127686 }, { "auxiliary_loss_clip": 0.0109797, "auxiliary_loss_mlp": 0.01038289, "balance_loss_clip": 1.01821387, "balance_loss_mlp": 1.02596939, "epoch": 0.16497820532090787, "flos": 23799802141440.0, "grad_norm": 1.614851134462598, "language_loss": 0.85501188, "learning_rate": 3.7376279419208367e-06, "loss": 0.87637448, "num_input_tokens_seen": 59331535, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.71875, "step": 2744, "time_per_iteration": 2.3961873054504395 }, { "auxiliary_loss_clip": 0.0109583, "auxiliary_loss_mlp": 0.0104075, "balance_loss_clip": 1.02158082, "balance_loss_mlp": 1.02610826, "epoch": 0.16503832857357584, "flos": 25481498376960.0, "grad_norm": 2.0251563213959, "language_loss": 0.82605666, "learning_rate": 3.7374408364785744e-06, "loss": 0.84742248, "num_input_tokens_seen": 59350680, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.69921875, "step": 2745, "time_per_iteration": 3.801182508468628 }, { "auxiliary_loss_clip": 0.01105267, "auxiliary_loss_mlp": 0.01046314, "balance_loss_clip": 1.02546382, "balance_loss_mlp": 1.02833867, "epoch": 0.1650984518262438, "flos": 17675093462400.0, "grad_norm": 2.203588025169381, "language_loss": 0.76188481, "learning_rate": 3.7372536690315187e-06, "loss": 0.7834006, "num_input_tokens_seen": 59367020, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.76953125, "step": 2746, "time_per_iteration": 2.367724895477295 }, { "auxiliary_loss_clip": 0.0109833, "auxiliary_loss_mlp": 0.01038929, "balance_loss_clip": 1.01905644, "balance_loss_mlp": 1.02701068, "epoch": 0.16515857507891177, "flos": 18697315426560.0, "grad_norm": 1.5013443128957897, "language_loss": 0.80648381, "learning_rate": 3.737066439586348e-06, "loss": 0.82785642, "num_input_tokens_seen": 59386075, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.7109375, "step": 2747, "time_per_iteration": 2.379795789718628 }, { "auxiliary_loss_clip": 0.01103344, "auxiliary_loss_mlp": 0.01039918, "balance_loss_clip": 1.01911581, "balance_loss_mlp": 1.02899504, "epoch": 0.16521869833157973, "flos": 15009649447680.0, "grad_norm": 2.0025581701827586, "language_loss": 0.69230592, "learning_rate": 3.7368791481497448e-06, "loss": 0.71373856, "num_input_tokens_seen": 59402690, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.7421875, "step": 2748, "time_per_iteration": 2.341280937194824 }, { "auxiliary_loss_clip": 0.01101063, "auxiliary_loss_mlp": 0.01046647, "balance_loss_clip": 1.02620232, "balance_loss_mlp": 1.02744234, "epoch": 0.1652788215842477, "flos": 22120235498880.0, "grad_norm": 2.13358667618576, "language_loss": 0.87971032, "learning_rate": 3.736691794728392e-06, "loss": 0.90118742, "num_input_tokens_seen": 59421130, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.734375, "step": 2749, "time_per_iteration": 2.374248504638672 }, { "auxiliary_loss_clip": 0.01100285, "auxiliary_loss_mlp": 0.01034894, "balance_loss_clip": 1.01435399, "balance_loss_mlp": 1.0265491, "epoch": 0.16533894483691566, "flos": 18332089027200.0, "grad_norm": 1.9698727226091124, "language_loss": 0.79004288, "learning_rate": 3.736504379328976e-06, "loss": 0.81139457, "num_input_tokens_seen": 59438970, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.73828125, "step": 2750, "time_per_iteration": 2.3773529529571533 }, { "auxiliary_loss_clip": 0.01100886, "auxiliary_loss_mlp": 0.01040034, "balance_loss_clip": 1.019876, "balance_loss_mlp": 1.02821505, "epoch": 0.16539906808958366, "flos": 22381036421760.0, "grad_norm": 1.679330557616043, "language_loss": 0.95238423, "learning_rate": 3.7363169019581865e-06, "loss": 0.97379339, "num_input_tokens_seen": 59458510, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.7265625, "step": 2751, "time_per_iteration": 2.3926827907562256 }, { "auxiliary_loss_clip": 0.01098458, "auxiliary_loss_mlp": 0.010343, "balance_loss_clip": 1.01447558, "balance_loss_mlp": 1.02918196, "epoch": 0.16545919134225162, "flos": 22709988051840.0, "grad_norm": 3.6291009142097597, "language_loss": 0.70971817, "learning_rate": 3.7361293626227125e-06, "loss": 0.73104578, "num_input_tokens_seen": 59477110, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.69140625, "step": 2752, "time_per_iteration": 2.4085581302642822 }, { "auxiliary_loss_clip": 0.01029316, "auxiliary_loss_mlp": 0.01008623, "balance_loss_clip": 1.00564301, "balance_loss_mlp": 1.00707996, "epoch": 0.1655193145949196, "flos": 67799720805120.0, "grad_norm": 0.8051340800737071, "language_loss": 0.54032564, "learning_rate": 3.735941761329248e-06, "loss": 0.56070507, "num_input_tokens_seen": 59541155, "router_z_loss_clip": 0.02978516, "router_z_loss_mlp": 0.22265625, "step": 2753, "time_per_iteration": 3.1193695068359375 }, { "auxiliary_loss_clip": 0.01099621, "auxiliary_loss_mlp": 0.01033631, "balance_loss_clip": 1.01318693, "balance_loss_mlp": 1.02745223, "epoch": 0.16557943784758755, "flos": 24279229198080.0, "grad_norm": 1.8369958358468492, "language_loss": 0.75099742, "learning_rate": 3.735754098084487e-06, "loss": 0.77232993, "num_input_tokens_seen": 59561155, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.71875, "step": 2754, "time_per_iteration": 2.4263463020324707 }, { "auxiliary_loss_clip": 0.01108749, "auxiliary_loss_mlp": 0.01046593, "balance_loss_clip": 1.02282274, "balance_loss_mlp": 1.03078508, "epoch": 0.16563956110025552, "flos": 20082599285760.0, "grad_norm": 2.722434912623219, "language_loss": 0.86311758, "learning_rate": 3.7355663728951265e-06, "loss": 0.88467097, "num_input_tokens_seen": 59580460, "router_z_loss_clip": 0.23828125, "router_z_loss_mlp": 0.77734375, "step": 2755, "time_per_iteration": 2.3921918869018555 }, { "auxiliary_loss_clip": 0.01098502, "auxiliary_loss_mlp": 0.01042226, "balance_loss_clip": 1.02297413, "balance_loss_mlp": 1.02710485, "epoch": 0.16569968435292348, "flos": 28033300316160.0, "grad_norm": 1.9668023918212456, "language_loss": 0.73244894, "learning_rate": 3.7353785857678675e-06, "loss": 0.75385618, "num_input_tokens_seen": 59600025, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.71484375, "step": 2756, "time_per_iteration": 2.442095994949341 }, { "auxiliary_loss_clip": 0.01097588, "auxiliary_loss_mlp": 0.01038453, "balance_loss_clip": 1.01904607, "balance_loss_mlp": 1.02879632, "epoch": 0.16575980760559147, "flos": 26249028906240.0, "grad_norm": 1.7749553241589369, "language_loss": 0.74760187, "learning_rate": 3.7351907367094105e-06, "loss": 0.76896232, "num_input_tokens_seen": 59620600, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.6875, "step": 2757, "time_per_iteration": 2.4306015968322754 }, { "auxiliary_loss_clip": 0.01101508, "auxiliary_loss_mlp": 0.01037136, "balance_loss_clip": 1.01654887, "balance_loss_mlp": 1.02925587, "epoch": 0.16581993085825944, "flos": 26942718176640.0, "grad_norm": 2.1596303075322982, "language_loss": 0.84663153, "learning_rate": 3.7350028257264593e-06, "loss": 0.86801791, "num_input_tokens_seen": 59641385, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.72265625, "step": 2758, "time_per_iteration": 2.438163995742798 }, { "auxiliary_loss_clip": 0.01104018, "auxiliary_loss_mlp": 0.01039387, "balance_loss_clip": 1.0203855, "balance_loss_mlp": 1.03096461, "epoch": 0.1658800541109274, "flos": 21652538659200.0, "grad_norm": 1.886810125326837, "language_loss": 0.79101157, "learning_rate": 3.7348148528257202e-06, "loss": 0.81244564, "num_input_tokens_seen": 59659865, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.73046875, "step": 2759, "time_per_iteration": 2.3999645709991455 }, { "auxiliary_loss_clip": 0.0109939, "auxiliary_loss_mlp": 0.01037362, "balance_loss_clip": 1.01654792, "balance_loss_mlp": 1.02732992, "epoch": 0.16594017736359537, "flos": 16434559566720.0, "grad_norm": 2.2047309594012026, "language_loss": 0.75204885, "learning_rate": 3.734626818013902e-06, "loss": 0.77341634, "num_input_tokens_seen": 59678780, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.72265625, "step": 2760, "time_per_iteration": 2.3776280879974365 }, { "auxiliary_loss_clip": 0.01104055, "auxiliary_loss_mlp": 0.0103902, "balance_loss_clip": 1.01839638, "balance_loss_mlp": 1.0285697, "epoch": 0.16600030061626334, "flos": 22636216615680.0, "grad_norm": 2.606869656949303, "language_loss": 0.73423386, "learning_rate": 3.734438721297714e-06, "loss": 0.75566459, "num_input_tokens_seen": 59698795, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.75390625, "step": 2761, "time_per_iteration": 2.4225025177001953 }, { "auxiliary_loss_clip": 0.01099037, "auxiliary_loss_mlp": 0.01040729, "balance_loss_clip": 1.02139318, "balance_loss_mlp": 1.02757204, "epoch": 0.1660604238689313, "flos": 26395349880960.0, "grad_norm": 3.788373493784275, "language_loss": 0.8883667, "learning_rate": 3.73425056268387e-06, "loss": 0.90976429, "num_input_tokens_seen": 59718795, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.71484375, "step": 2762, "time_per_iteration": 2.479235887527466 }, { "auxiliary_loss_clip": 0.01101874, "auxiliary_loss_mlp": 0.01041349, "balance_loss_clip": 1.02078533, "balance_loss_mlp": 1.02852178, "epoch": 0.16612054712159927, "flos": 23038869859200.0, "grad_norm": 2.36889733150707, "language_loss": 0.8771072, "learning_rate": 3.7340623421790843e-06, "loss": 0.89853942, "num_input_tokens_seen": 59737555, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.734375, "step": 2763, "time_per_iteration": 2.483316421508789 }, { "auxiliary_loss_clip": 0.0102727, "auxiliary_loss_mlp": 0.01003247, "balance_loss_clip": 1.00039816, "balance_loss_mlp": 1.0044899, "epoch": 0.16618067037426726, "flos": 59237864691840.0, "grad_norm": 0.7714551426644473, "language_loss": 0.59774059, "learning_rate": 3.733874059790074e-06, "loss": 0.61804575, "num_input_tokens_seen": 59800915, "router_z_loss_clip": 0.02844238, "router_z_loss_mlp": 0.22753906, "step": 2764, "time_per_iteration": 3.048445224761963 }, { "auxiliary_loss_clip": 0.01105783, "auxiliary_loss_mlp": 0.01040468, "balance_loss_clip": 1.01886737, "balance_loss_mlp": 1.0312283, "epoch": 0.16624079362693522, "flos": 27197584168320.0, "grad_norm": 1.7848183524532986, "language_loss": 0.82096636, "learning_rate": 3.733685715523559e-06, "loss": 0.84242886, "num_input_tokens_seen": 59822910, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.74609375, "step": 2765, "time_per_iteration": 2.4442572593688965 }, { "auxiliary_loss_clip": 0.01107709, "auxiliary_loss_mlp": 0.010432, "balance_loss_clip": 1.01971591, "balance_loss_mlp": 1.02863848, "epoch": 0.1663009168796032, "flos": 10924322549760.0, "grad_norm": 2.682801917681024, "language_loss": 0.69734764, "learning_rate": 3.7334973093862595e-06, "loss": 0.71885675, "num_input_tokens_seen": 59838805, "router_z_loss_clip": 0.23535156, "router_z_loss_mlp": 0.7890625, "step": 2766, "time_per_iteration": 2.3364741802215576 }, { "auxiliary_loss_clip": 0.01100791, "auxiliary_loss_mlp": 0.01040526, "balance_loss_clip": 1.02058244, "balance_loss_mlp": 1.03000283, "epoch": 0.16636104013227115, "flos": 17893475216640.0, "grad_norm": 2.7257714940608744, "language_loss": 0.88355601, "learning_rate": 3.7333088413849008e-06, "loss": 0.9049691, "num_input_tokens_seen": 59855345, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.70703125, "step": 2767, "time_per_iteration": 2.365118980407715 }, { "auxiliary_loss_clip": 0.01026081, "auxiliary_loss_mlp": 0.01000866, "balance_loss_clip": 0.99827963, "balance_loss_mlp": 1.00439978, "epoch": 0.16642116338493912, "flos": 66722335159680.0, "grad_norm": 0.6389215842058035, "language_loss": 0.52877498, "learning_rate": 3.7331203115262078e-06, "loss": 0.54904449, "num_input_tokens_seen": 59917710, "router_z_loss_clip": 0.02587891, "router_z_loss_mlp": 0.21679688, "step": 2768, "time_per_iteration": 3.1787824630737305 }, { "auxiliary_loss_clip": 0.01104612, "auxiliary_loss_mlp": 0.01035194, "balance_loss_clip": 1.01367664, "balance_loss_mlp": 1.02860165, "epoch": 0.16648128663760708, "flos": 19025045159040.0, "grad_norm": 2.534206350664639, "language_loss": 0.85295093, "learning_rate": 3.7329317198169098e-06, "loss": 0.87434894, "num_input_tokens_seen": 59935105, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.7578125, "step": 2769, "time_per_iteration": 2.354337692260742 }, { "auxiliary_loss_clip": 0.01025435, "auxiliary_loss_mlp": 0.01002373, "balance_loss_clip": 0.99979842, "balance_loss_mlp": 1.00393605, "epoch": 0.16654140989027508, "flos": 70131744535680.0, "grad_norm": 0.806276761124177, "language_loss": 0.57446808, "learning_rate": 3.732743066263736e-06, "loss": 0.59474611, "num_input_tokens_seen": 59984085, "router_z_loss_clip": 0.02575684, "router_z_loss_mlp": 0.21484375, "step": 2770, "time_per_iteration": 2.9205851554870605 }, { "auxiliary_loss_clip": 0.01024973, "auxiliary_loss_mlp": 0.01004411, "balance_loss_clip": 1.0017761, "balance_loss_mlp": 1.00355017, "epoch": 0.16660153314294304, "flos": 70269407493120.0, "grad_norm": 0.8895709965521077, "language_loss": 0.56245881, "learning_rate": 3.7325543508734187e-06, "loss": 0.58275265, "num_input_tokens_seen": 60043470, "router_z_loss_clip": 0.02636719, "router_z_loss_mlp": 0.21484375, "step": 2771, "time_per_iteration": 2.8889243602752686 }, { "auxiliary_loss_clip": 0.01101051, "auxiliary_loss_mlp": 0.0104363, "balance_loss_clip": 1.02268481, "balance_loss_mlp": 1.02889836, "epoch": 0.166661656395611, "flos": 23073957642240.0, "grad_norm": 3.431203791623101, "language_loss": 0.70461863, "learning_rate": 3.732365573652694e-06, "loss": 0.7260654, "num_input_tokens_seen": 60063045, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.72265625, "step": 2772, "time_per_iteration": 2.4362192153930664 }, { "auxiliary_loss_clip": 0.01099417, "auxiliary_loss_mlp": 0.01039775, "balance_loss_clip": 1.01942599, "balance_loss_mlp": 1.02762997, "epoch": 0.16672177964827897, "flos": 28365079766400.0, "grad_norm": 3.772922793979306, "language_loss": 0.86091107, "learning_rate": 3.7321767346082977e-06, "loss": 0.882303, "num_input_tokens_seen": 60081945, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.71875, "step": 2773, "time_per_iteration": 2.4295010566711426 }, { "auxiliary_loss_clip": 0.01099592, "auxiliary_loss_mlp": 0.01030765, "balance_loss_clip": 1.01282382, "balance_loss_mlp": 1.02890277, "epoch": 0.16678190290094694, "flos": 19090228400640.0, "grad_norm": 2.2753054762817917, "language_loss": 0.82221007, "learning_rate": 3.7319878337469694e-06, "loss": 0.84351367, "num_input_tokens_seen": 60096820, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.70703125, "step": 2774, "time_per_iteration": 2.39690899848938 }, { "auxiliary_loss_clip": 0.01102119, "auxiliary_loss_mlp": 0.01040889, "balance_loss_clip": 1.02076638, "balance_loss_mlp": 1.02918148, "epoch": 0.1668420261536149, "flos": 21798021761280.0, "grad_norm": 2.396825643784463, "language_loss": 0.8285411, "learning_rate": 3.73179887107545e-06, "loss": 0.84997118, "num_input_tokens_seen": 60116140, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.7265625, "step": 2775, "time_per_iteration": 2.388744354248047 }, { "auxiliary_loss_clip": 0.01099687, "auxiliary_loss_mlp": 0.01039013, "balance_loss_clip": 1.02028513, "balance_loss_mlp": 1.02941787, "epoch": 0.16690214940628287, "flos": 19061529396480.0, "grad_norm": 3.126882021668333, "language_loss": 0.806099, "learning_rate": 3.731609846600485e-06, "loss": 0.82748598, "num_input_tokens_seen": 60134235, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.703125, "step": 2776, "time_per_iteration": 2.386744976043701 }, { "auxiliary_loss_clip": 0.01095309, "auxiliary_loss_mlp": 0.01035522, "balance_loss_clip": 1.0168184, "balance_loss_mlp": 1.02750969, "epoch": 0.16696227265895086, "flos": 18587548512000.0, "grad_norm": 2.1056546406434435, "language_loss": 0.80016923, "learning_rate": 3.731420760328818e-06, "loss": 0.82147753, "num_input_tokens_seen": 60153275, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.67578125, "step": 2777, "time_per_iteration": 2.3581535816192627 }, { "auxiliary_loss_clip": 0.01100116, "auxiliary_loss_mlp": 0.01037908, "balance_loss_clip": 1.01821482, "balance_loss_mlp": 1.02836227, "epoch": 0.16702239591161883, "flos": 23293037623680.0, "grad_norm": 1.775130683428226, "language_loss": 0.85230374, "learning_rate": 3.7312316122671977e-06, "loss": 0.87368405, "num_input_tokens_seen": 60173215, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.71875, "step": 2778, "time_per_iteration": 2.4324114322662354 }, { "auxiliary_loss_clip": 0.01103116, "auxiliary_loss_mlp": 0.01036977, "balance_loss_clip": 1.01637769, "balance_loss_mlp": 1.02825439, "epoch": 0.1670825191642868, "flos": 24424502832000.0, "grad_norm": 2.077255176239374, "language_loss": 0.74045932, "learning_rate": 3.731042402422375e-06, "loss": 0.76186025, "num_input_tokens_seen": 60190515, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.75, "step": 2779, "time_per_iteration": 3.8116507530212402 }, { "auxiliary_loss_clip": 0.01099015, "auxiliary_loss_mlp": 0.01040698, "balance_loss_clip": 1.0210402, "balance_loss_mlp": 1.02855587, "epoch": 0.16714264241695476, "flos": 26796292467840.0, "grad_norm": 3.883328641370763, "language_loss": 0.66294205, "learning_rate": 3.730853130801101e-06, "loss": 0.68433917, "num_input_tokens_seen": 60211655, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.703125, "step": 2780, "time_per_iteration": 2.469529628753662 }, { "auxiliary_loss_clip": 0.01098358, "auxiliary_loss_mlp": 0.01036649, "balance_loss_clip": 1.01637185, "balance_loss_mlp": 1.02710223, "epoch": 0.16720276566962272, "flos": 21834226707840.0, "grad_norm": 2.3808420385732205, "language_loss": 0.78112018, "learning_rate": 3.7306637974101312e-06, "loss": 0.80247027, "num_input_tokens_seen": 60230860, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7109375, "step": 2781, "time_per_iteration": 3.8163065910339355 }, { "auxiliary_loss_clip": 0.01102115, "auxiliary_loss_mlp": 0.01033987, "balance_loss_clip": 1.01564074, "balance_loss_mlp": 1.02904999, "epoch": 0.1672628889222907, "flos": 21469349422080.0, "grad_norm": 1.7121057808521025, "language_loss": 0.74994546, "learning_rate": 3.730474402256223e-06, "loss": 0.77130646, "num_input_tokens_seen": 60250535, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.73046875, "step": 2782, "time_per_iteration": 3.775129556655884 }, { "auxiliary_loss_clip": 0.01103694, "auxiliary_loss_mlp": 0.01036492, "balance_loss_clip": 1.01619673, "balance_loss_mlp": 1.02874088, "epoch": 0.16732301217495865, "flos": 30772690323840.0, "grad_norm": 4.691143884560107, "language_loss": 0.67676735, "learning_rate": 3.7302849453461337e-06, "loss": 0.69816923, "num_input_tokens_seen": 60269530, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.75, "step": 2783, "time_per_iteration": 2.4699010848999023 }, { "auxiliary_loss_clip": 0.01101268, "auxiliary_loss_mlp": 0.01038573, "balance_loss_clip": 1.01932085, "balance_loss_mlp": 1.02963662, "epoch": 0.16738313542762664, "flos": 23473573597440.0, "grad_norm": 1.7539523891610789, "language_loss": 0.70496118, "learning_rate": 3.730095426686626e-06, "loss": 0.72635961, "num_input_tokens_seen": 60289900, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.71875, "step": 2784, "time_per_iteration": 3.8684651851654053 }, { "auxiliary_loss_clip": 0.01102211, "auxiliary_loss_mlp": 0.01042852, "balance_loss_clip": 1.02014303, "balance_loss_mlp": 1.02739811, "epoch": 0.1674432586802946, "flos": 29787790469760.0, "grad_norm": 2.05039647654335, "language_loss": 0.60617006, "learning_rate": 3.729905846284463e-06, "loss": 0.62762076, "num_input_tokens_seen": 60310025, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.75, "step": 2785, "time_per_iteration": 2.4373538494110107 }, { "auxiliary_loss_clip": 0.01025677, "auxiliary_loss_mlp": 0.01008213, "balance_loss_clip": 1.00566173, "balance_loss_mlp": 1.00451803, "epoch": 0.16750338193296258, "flos": 66132547695360.0, "grad_norm": 0.8260912271145021, "language_loss": 0.58771896, "learning_rate": 3.72971620414641e-06, "loss": 0.60805786, "num_input_tokens_seen": 60377800, "router_z_loss_clip": 0.0255127, "router_z_loss_mlp": 0.2109375, "step": 2786, "time_per_iteration": 3.076218605041504 }, { "auxiliary_loss_clip": 0.01101877, "auxiliary_loss_mlp": 0.01039603, "balance_loss_clip": 1.01880097, "balance_loss_mlp": 1.02818251, "epoch": 0.16756350518563054, "flos": 25695760590720.0, "grad_norm": 1.9732594496456566, "language_loss": 0.76632226, "learning_rate": 3.729526500279235e-06, "loss": 0.78773701, "num_input_tokens_seen": 60398215, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.73828125, "step": 2787, "time_per_iteration": 2.409895896911621 }, { "auxiliary_loss_clip": 0.01101295, "auxiliary_loss_mlp": 0.01038415, "balance_loss_clip": 1.0182445, "balance_loss_mlp": 1.02843809, "epoch": 0.1676236284382985, "flos": 23835134304000.0, "grad_norm": 2.4665047237238906, "language_loss": 0.76906705, "learning_rate": 3.729336734689708e-06, "loss": 0.7904641, "num_input_tokens_seen": 60416910, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.73046875, "step": 2788, "time_per_iteration": 2.4283406734466553 }, { "auxiliary_loss_clip": 0.01023516, "auxiliary_loss_mlp": 0.01007237, "balance_loss_clip": 1.00474536, "balance_loss_mlp": 1.00275159, "epoch": 0.16768375169096647, "flos": 59872167872640.0, "grad_norm": 0.8522369178846958, "language_loss": 0.59424734, "learning_rate": 3.7291469073846017e-06, "loss": 0.61455488, "num_input_tokens_seen": 60468660, "router_z_loss_clip": 0.02490234, "router_z_loss_mlp": 0.20800781, "step": 2789, "time_per_iteration": 2.8966400623321533 }, { "auxiliary_loss_clip": 0.01102404, "auxiliary_loss_mlp": 0.0104348, "balance_loss_clip": 1.02137899, "balance_loss_mlp": 1.02846503, "epoch": 0.16774387494363446, "flos": 38434135806720.0, "grad_norm": 1.596248254233782, "language_loss": 0.69839656, "learning_rate": 3.72895701837069e-06, "loss": 0.71985543, "num_input_tokens_seen": 60492370, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.73828125, "step": 2790, "time_per_iteration": 2.550748586654663 }, { "auxiliary_loss_clip": 0.01102681, "auxiliary_loss_mlp": 0.01043039, "balance_loss_clip": 1.02257061, "balance_loss_mlp": 1.0283494, "epoch": 0.16780399819630243, "flos": 22636530817920.0, "grad_norm": 1.8564311737949704, "language_loss": 0.79571879, "learning_rate": 3.7287670676547495e-06, "loss": 0.81717592, "num_input_tokens_seen": 60512655, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.74609375, "step": 2791, "time_per_iteration": 2.3864128589630127 }, { "auxiliary_loss_clip": 0.0110316, "auxiliary_loss_mlp": 0.01046247, "balance_loss_clip": 1.02599347, "balance_loss_mlp": 1.02913141, "epoch": 0.1678641214489704, "flos": 32890102727040.0, "grad_norm": 2.08072886880986, "language_loss": 0.71467054, "learning_rate": 3.7285770552435593e-06, "loss": 0.73616463, "num_input_tokens_seen": 60533090, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7421875, "step": 2792, "time_per_iteration": 2.4916250705718994 }, { "auxiliary_loss_clip": 0.01102263, "auxiliary_loss_mlp": 0.01038455, "balance_loss_clip": 1.0188446, "balance_loss_mlp": 1.02897751, "epoch": 0.16792424470163836, "flos": 19973879712000.0, "grad_norm": 1.916949862508531, "language_loss": 0.71492851, "learning_rate": 3.7283869811439006e-06, "loss": 0.73633564, "num_input_tokens_seen": 60553190, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.734375, "step": 2793, "time_per_iteration": 2.388706922531128 }, { "auxiliary_loss_clip": 0.01103298, "auxiliary_loss_mlp": 0.01038733, "balance_loss_clip": 1.01902735, "balance_loss_mlp": 1.02974629, "epoch": 0.16798436795430632, "flos": 19718839163520.0, "grad_norm": 2.0882395204511353, "language_loss": 0.7694692, "learning_rate": 3.728196845362557e-06, "loss": 0.7908895, "num_input_tokens_seen": 60571995, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.734375, "step": 2794, "time_per_iteration": 2.3973076343536377 }, { "auxiliary_loss_clip": 0.01104216, "auxiliary_loss_mlp": 0.01042443, "balance_loss_clip": 1.02249885, "balance_loss_mlp": 1.03091669, "epoch": 0.1680444912069743, "flos": 28103755173120.0, "grad_norm": 3.660215334389666, "language_loss": 0.7173906, "learning_rate": 3.7280066479063128e-06, "loss": 0.73885721, "num_input_tokens_seen": 60591275, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.73046875, "step": 2795, "time_per_iteration": 2.4465954303741455 }, { "auxiliary_loss_clip": 0.01100168, "auxiliary_loss_mlp": 0.01029214, "balance_loss_clip": 1.00934196, "balance_loss_mlp": 1.02792013, "epoch": 0.16810461445964225, "flos": 18074290481280.0, "grad_norm": 2.0025951283747716, "language_loss": 0.83917654, "learning_rate": 3.7278163887819565e-06, "loss": 0.86047041, "num_input_tokens_seen": 60609235, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.72265625, "step": 2796, "time_per_iteration": 2.364109992980957 }, { "auxiliary_loss_clip": 0.01101342, "auxiliary_loss_mlp": 0.01040593, "balance_loss_clip": 1.02031505, "balance_loss_mlp": 1.02778924, "epoch": 0.16816473771231025, "flos": 23877518561280.0, "grad_norm": 2.6026955903410967, "language_loss": 0.81673908, "learning_rate": 3.727626067996277e-06, "loss": 0.83815849, "num_input_tokens_seen": 60629880, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.734375, "step": 2797, "time_per_iteration": 2.405458450317383 }, { "auxiliary_loss_clip": 0.01095684, "auxiliary_loss_mlp": 0.01032874, "balance_loss_clip": 1.01563668, "balance_loss_mlp": 1.02797866, "epoch": 0.1682248609649782, "flos": 22782502679040.0, "grad_norm": 1.5458723910185148, "language_loss": 0.75072479, "learning_rate": 3.727435685556068e-06, "loss": 0.77201039, "num_input_tokens_seen": 60651175, "router_z_loss_clip": 0.17285156, "router_z_loss_mlp": 0.67578125, "step": 2798, "time_per_iteration": 2.412954330444336 }, { "auxiliary_loss_clip": 0.01102921, "auxiliary_loss_mlp": 0.01039167, "balance_loss_clip": 1.0206418, "balance_loss_mlp": 1.03008187, "epoch": 0.16828498421764618, "flos": 20704053219840.0, "grad_norm": 2.087515366307674, "language_loss": 0.79870963, "learning_rate": 3.7272452414681227e-06, "loss": 0.82013059, "num_input_tokens_seen": 60670210, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.7265625, "step": 2799, "time_per_iteration": 2.3892929553985596 }, { "auxiliary_loss_clip": 0.01103858, "auxiliary_loss_mlp": 0.0103511, "balance_loss_clip": 1.01366425, "balance_loss_mlp": 1.02899194, "epoch": 0.16834510747031414, "flos": 29419422048000.0, "grad_norm": 2.1088789988159067, "language_loss": 0.70523083, "learning_rate": 3.7270547357392375e-06, "loss": 0.72662044, "num_input_tokens_seen": 60690895, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.75, "step": 2800, "time_per_iteration": 2.461484909057617 }, { "auxiliary_loss_clip": 0.01099733, "auxiliary_loss_mlp": 0.01035997, "balance_loss_clip": 1.01558852, "balance_loss_mlp": 1.02740645, "epoch": 0.1684052307229821, "flos": 18144535870080.0, "grad_norm": 1.724561753876366, "language_loss": 0.83576268, "learning_rate": 3.7268641683762113e-06, "loss": 0.85712004, "num_input_tokens_seen": 60708280, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.72265625, "step": 2801, "time_per_iteration": 2.348421335220337 }, { "auxiliary_loss_clip": 0.01101752, "auxiliary_loss_mlp": 0.01041094, "balance_loss_clip": 1.02017236, "balance_loss_mlp": 1.02821028, "epoch": 0.16846535397565007, "flos": 16574177560320.0, "grad_norm": 2.7224930861772654, "language_loss": 0.82470471, "learning_rate": 3.7266735393858456e-06, "loss": 0.84613317, "num_input_tokens_seen": 60724150, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.734375, "step": 2802, "time_per_iteration": 2.3624260425567627 }, { "auxiliary_loss_clip": 0.01102921, "auxiliary_loss_mlp": 0.01043189, "balance_loss_clip": 1.02175534, "balance_loss_mlp": 1.02774191, "epoch": 0.16852547722831807, "flos": 30407568658560.0, "grad_norm": 1.5945284759246205, "language_loss": 0.80595237, "learning_rate": 3.7264828487749422e-06, "loss": 0.82741344, "num_input_tokens_seen": 60746485, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.75390625, "step": 2803, "time_per_iteration": 2.461865186691284 }, { "auxiliary_loss_clip": 0.01099747, "auxiliary_loss_mlp": 0.01042445, "balance_loss_clip": 1.0223465, "balance_loss_mlp": 1.02886486, "epoch": 0.16858560048098603, "flos": 33506110488960.0, "grad_norm": 2.253990294914759, "language_loss": 0.76085579, "learning_rate": 3.726292096550307e-06, "loss": 0.78227776, "num_input_tokens_seen": 60762875, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.70703125, "step": 2804, "time_per_iteration": 2.4685802459716797 }, { "auxiliary_loss_clip": 0.01021925, "auxiliary_loss_mlp": 0.0100264, "balance_loss_clip": 1.00030327, "balance_loss_mlp": 1.00205374, "epoch": 0.168645723733654, "flos": 67367111748480.0, "grad_norm": 0.8313708537078875, "language_loss": 0.55405569, "learning_rate": 3.7261012827187477e-06, "loss": 0.57430136, "num_input_tokens_seen": 60825510, "router_z_loss_clip": 0.02331543, "router_z_loss_mlp": 0.19921875, "step": 2805, "time_per_iteration": 2.996175765991211 }, { "auxiliary_loss_clip": 0.01095749, "auxiliary_loss_mlp": 0.01034293, "balance_loss_clip": 1.01666236, "balance_loss_mlp": 1.02667689, "epoch": 0.16870584698632196, "flos": 21323552117760.0, "grad_norm": 2.471730494349587, "language_loss": 0.72939378, "learning_rate": 3.725910407287074e-06, "loss": 0.75069416, "num_input_tokens_seen": 60844440, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.69140625, "step": 2806, "time_per_iteration": 2.394117593765259 }, { "auxiliary_loss_clip": 0.01098756, "auxiliary_loss_mlp": 0.01038943, "balance_loss_clip": 1.02041769, "balance_loss_mlp": 1.02883208, "epoch": 0.16876597023898993, "flos": 20739699584640.0, "grad_norm": 2.0119891708960393, "language_loss": 0.70036387, "learning_rate": 3.7257194702620964e-06, "loss": 0.72174084, "num_input_tokens_seen": 60863210, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.69921875, "step": 2807, "time_per_iteration": 2.383373498916626 }, { "auxiliary_loss_clip": 0.01101881, "auxiliary_loss_mlp": 0.01041627, "balance_loss_clip": 1.02217221, "balance_loss_mlp": 1.03004301, "epoch": 0.1688260934916579, "flos": 20302447317120.0, "grad_norm": 2.4472358434644166, "language_loss": 0.70172656, "learning_rate": 3.725528471650631e-06, "loss": 0.72316158, "num_input_tokens_seen": 60882510, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.71875, "step": 2808, "time_per_iteration": 2.3989601135253906 }, { "auxiliary_loss_clip": 0.01100398, "auxiliary_loss_mlp": 0.0104131, "balance_loss_clip": 1.02078247, "balance_loss_mlp": 1.02708447, "epoch": 0.16888621674432586, "flos": 20339629781760.0, "grad_norm": 2.292779589312388, "language_loss": 0.80167681, "learning_rate": 3.7253374114594925e-06, "loss": 0.82309389, "num_input_tokens_seen": 60901105, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.734375, "step": 2809, "time_per_iteration": 2.369114398956299 }, { "auxiliary_loss_clip": 0.01109348, "auxiliary_loss_mlp": 0.0104127, "balance_loss_clip": 1.02125466, "balance_loss_mlp": 1.03182256, "epoch": 0.16894633999699385, "flos": 16244108766720.0, "grad_norm": 2.8977700555594548, "language_loss": 0.8793937, "learning_rate": 3.7251462896955e-06, "loss": 0.90089989, "num_input_tokens_seen": 60915340, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.7734375, "step": 2810, "time_per_iteration": 2.3441340923309326 }, { "auxiliary_loss_clip": 0.01103802, "auxiliary_loss_mlp": 0.01047031, "balance_loss_clip": 1.02674174, "balance_loss_mlp": 1.02955842, "epoch": 0.16900646324966181, "flos": 19609142071680.0, "grad_norm": 2.502849501120339, "language_loss": 0.92641753, "learning_rate": 3.724955106365474e-06, "loss": 0.94792581, "num_input_tokens_seen": 60933735, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7421875, "step": 2811, "time_per_iteration": 2.377276659011841 }, { "auxiliary_loss_clip": 0.01100842, "auxiliary_loss_mlp": 0.01038255, "balance_loss_clip": 1.01937222, "balance_loss_mlp": 1.02856755, "epoch": 0.16906658650232978, "flos": 22016997008640.0, "grad_norm": 2.3749282258990276, "language_loss": 0.78287768, "learning_rate": 3.724763861476237e-06, "loss": 0.80426866, "num_input_tokens_seen": 60953105, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.72265625, "step": 2812, "time_per_iteration": 2.377631902694702 }, { "auxiliary_loss_clip": 0.01100864, "auxiliary_loss_mlp": 0.01040717, "balance_loss_clip": 1.02241826, "balance_loss_mlp": 1.02951097, "epoch": 0.16912670975499774, "flos": 11762936340480.0, "grad_norm": 2.81241792316052, "language_loss": 0.7505877, "learning_rate": 3.724572555034615e-06, "loss": 0.77200353, "num_input_tokens_seen": 60969150, "router_z_loss_clip": 0.18261719, "router_z_loss_mlp": 0.71484375, "step": 2813, "time_per_iteration": 2.3591530323028564 }, { "auxiliary_loss_clip": 0.01101558, "auxiliary_loss_mlp": 0.01041879, "balance_loss_clip": 1.02145851, "balance_loss_mlp": 1.02710545, "epoch": 0.1691868330076657, "flos": 17160543711360.0, "grad_norm": 8.40670518179425, "language_loss": 0.68826377, "learning_rate": 3.7243811870474346e-06, "loss": 0.70969814, "num_input_tokens_seen": 60982825, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.7421875, "step": 2814, "time_per_iteration": 2.324979066848755 }, { "auxiliary_loss_clip": 0.01100765, "auxiliary_loss_mlp": 0.01037974, "balance_loss_clip": 1.01856661, "balance_loss_mlp": 1.02842844, "epoch": 0.16924695626033368, "flos": 22415530711680.0, "grad_norm": 2.2746765096551487, "language_loss": 0.61625373, "learning_rate": 3.724189757521525e-06, "loss": 0.63764107, "num_input_tokens_seen": 61000875, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.72265625, "step": 2815, "time_per_iteration": 2.3952457904815674 }, { "auxiliary_loss_clip": 0.01097071, "auxiliary_loss_mlp": 0.01035948, "balance_loss_clip": 1.01794755, "balance_loss_mlp": 1.02694273, "epoch": 0.16930707951300164, "flos": 25738459050240.0, "grad_norm": 3.3403623940253144, "language_loss": 0.82395369, "learning_rate": 3.7239982664637185e-06, "loss": 0.84528393, "num_input_tokens_seen": 61021940, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.703125, "step": 2816, "time_per_iteration": 2.4072141647338867 }, { "auxiliary_loss_clip": 0.01102119, "auxiliary_loss_mlp": 0.01041241, "balance_loss_clip": 1.02181005, "balance_loss_mlp": 1.02807236, "epoch": 0.16936720276566963, "flos": 22745948618880.0, "grad_norm": 3.3116096120669414, "language_loss": 0.86611402, "learning_rate": 3.7238067138808477e-06, "loss": 0.88754761, "num_input_tokens_seen": 61040285, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.7421875, "step": 2817, "time_per_iteration": 2.388659715652466 }, { "auxiliary_loss_clip": 0.01100093, "auxiliary_loss_mlp": 0.01039138, "balance_loss_clip": 1.01989794, "balance_loss_mlp": 1.03019714, "epoch": 0.1694273260183376, "flos": 19572937125120.0, "grad_norm": 1.7000006864629023, "language_loss": 0.8144446, "learning_rate": 3.72361509977975e-06, "loss": 0.83583695, "num_input_tokens_seen": 61059020, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.69921875, "step": 2818, "time_per_iteration": 2.3735878467559814 }, { "auxiliary_loss_clip": 0.01097576, "auxiliary_loss_mlp": 0.01040857, "balance_loss_clip": 1.02055573, "balance_loss_mlp": 1.02641368, "epoch": 0.16948744927100556, "flos": 12457044547200.0, "grad_norm": 2.4156942861080433, "language_loss": 0.82009411, "learning_rate": 3.7234234241672632e-06, "loss": 0.84147841, "num_input_tokens_seen": 61074245, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7109375, "step": 2819, "time_per_iteration": 3.769258737564087 }, { "auxiliary_loss_clip": 0.01023284, "auxiliary_loss_mlp": 0.01007441, "balance_loss_clip": 1.00496185, "balance_loss_mlp": 1.00381994, "epoch": 0.16954757252367353, "flos": 71288731814400.0, "grad_norm": 0.9319188129893082, "language_loss": 0.61062413, "learning_rate": 3.7232316870502274e-06, "loss": 0.63093144, "num_input_tokens_seen": 61127080, "router_z_loss_clip": 0.02478027, "router_z_loss_mlp": 0.1953125, "step": 2820, "time_per_iteration": 2.91005539894104 }, { "auxiliary_loss_clip": 0.01099981, "auxiliary_loss_mlp": 0.01043868, "balance_loss_clip": 1.02462757, "balance_loss_mlp": 1.02774501, "epoch": 0.1696076957763415, "flos": 29605229637120.0, "grad_norm": 3.091531330886817, "language_loss": 0.78350353, "learning_rate": 3.723039888435485e-06, "loss": 0.80494201, "num_input_tokens_seen": 61146955, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.72265625, "step": 2821, "time_per_iteration": 3.8358261585235596 }, { "auxiliary_loss_clip": 0.01102246, "auxiliary_loss_mlp": 0.01045946, "balance_loss_clip": 1.02509594, "balance_loss_mlp": 1.03072417, "epoch": 0.16966781902900946, "flos": 24387460012800.0, "grad_norm": 1.9821330134851807, "language_loss": 0.78271604, "learning_rate": 3.722848028329882e-06, "loss": 0.80419791, "num_input_tokens_seen": 61166605, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.71484375, "step": 2822, "time_per_iteration": 3.8631176948547363 }, { "auxiliary_loss_clip": 0.01099396, "auxiliary_loss_mlp": 0.01037836, "balance_loss_clip": 1.01911986, "balance_loss_mlp": 1.02830386, "epoch": 0.16972794228167745, "flos": 23037717784320.0, "grad_norm": 4.088592431205343, "language_loss": 0.75136393, "learning_rate": 3.7226561067402638e-06, "loss": 0.77273631, "num_input_tokens_seen": 61186535, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.7109375, "step": 2823, "time_per_iteration": 3.7679522037506104 }, { "auxiliary_loss_clip": 0.01100015, "auxiliary_loss_mlp": 0.01039195, "balance_loss_clip": 1.01944184, "balance_loss_mlp": 1.0288341, "epoch": 0.16978806553434542, "flos": 35227153693440.0, "grad_norm": 2.161818085815661, "language_loss": 0.60268676, "learning_rate": 3.7224641236734805e-06, "loss": 0.62407881, "num_input_tokens_seen": 61208965, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.7109375, "step": 2824, "time_per_iteration": 2.519198179244995 }, { "auxiliary_loss_clip": 0.01099241, "auxiliary_loss_mlp": 0.01039886, "balance_loss_clip": 1.0196799, "balance_loss_mlp": 1.02819765, "epoch": 0.16984818878701338, "flos": 32012944928640.0, "grad_norm": 1.6091365944583946, "language_loss": 0.73137844, "learning_rate": 3.7222720791363837e-06, "loss": 0.75276971, "num_input_tokens_seen": 61230670, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.7109375, "step": 2825, "time_per_iteration": 2.457822561264038 }, { "auxiliary_loss_clip": 0.01103609, "auxiliary_loss_mlp": 0.0104685, "balance_loss_clip": 1.02480817, "balance_loss_mlp": 1.0285238, "epoch": 0.16990831203968135, "flos": 22817555550720.0, "grad_norm": 2.0299478360072247, "language_loss": 0.85285699, "learning_rate": 3.7220799731358264e-06, "loss": 0.87436152, "num_input_tokens_seen": 61249510, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.75, "step": 2826, "time_per_iteration": 2.3784821033477783 }, { "auxiliary_loss_clip": 0.01105385, "auxiliary_loss_mlp": 0.01045185, "balance_loss_clip": 1.02478862, "balance_loss_mlp": 1.0288589, "epoch": 0.1699684352923493, "flos": 23038485834240.0, "grad_norm": 1.7377436091686924, "language_loss": 0.82375735, "learning_rate": 3.721887805678665e-06, "loss": 0.84526312, "num_input_tokens_seen": 61269440, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.765625, "step": 2827, "time_per_iteration": 2.3947606086730957 }, { "auxiliary_loss_clip": 0.01102699, "auxiliary_loss_mlp": 0.01037606, "balance_loss_clip": 1.01596928, "balance_loss_mlp": 1.02850986, "epoch": 0.17002855854501728, "flos": 21433039741440.0, "grad_norm": 1.7688978957597494, "language_loss": 0.73898339, "learning_rate": 3.7216955767717558e-06, "loss": 0.76038647, "num_input_tokens_seen": 61288195, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.7421875, "step": 2828, "time_per_iteration": 2.3846895694732666 }, { "auxiliary_loss_clip": 0.0102301, "auxiliary_loss_mlp": 0.01005238, "balance_loss_clip": 1.00283051, "balance_loss_mlp": 1.00294185, "epoch": 0.17008868179768524, "flos": 71450099585280.0, "grad_norm": 0.7636207272108545, "language_loss": 0.56493086, "learning_rate": 3.721503286421961e-06, "loss": 0.58521336, "num_input_tokens_seen": 61350850, "router_z_loss_clip": 0.02404785, "router_z_loss_mlp": 0.20117188, "step": 2829, "time_per_iteration": 3.060309648513794 }, { "auxiliary_loss_clip": 0.01100291, "auxiliary_loss_mlp": 0.01034828, "balance_loss_clip": 1.01595736, "balance_loss_mlp": 1.02777958, "epoch": 0.17014880505035324, "flos": 24899147032320.0, "grad_norm": 1.9485488632433958, "language_loss": 0.83049953, "learning_rate": 3.7213109346361424e-06, "loss": 0.85185075, "num_input_tokens_seen": 61370765, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.7265625, "step": 2830, "time_per_iteration": 2.4133338928222656 }, { "auxiliary_loss_clip": 0.01100308, "auxiliary_loss_mlp": 0.01036126, "balance_loss_clip": 1.01582432, "balance_loss_mlp": 1.02769089, "epoch": 0.1702089283030212, "flos": 29861108058240.0, "grad_norm": 1.8773353166213922, "language_loss": 0.78348982, "learning_rate": 3.721118521421164e-06, "loss": 0.80485415, "num_input_tokens_seen": 61388935, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7265625, "step": 2831, "time_per_iteration": 2.445157527923584 }, { "auxiliary_loss_clip": 0.0110162, "auxiliary_loss_mlp": 0.01043851, "balance_loss_clip": 1.02229834, "balance_loss_mlp": 1.02805817, "epoch": 0.17026905155568917, "flos": 17743348903680.0, "grad_norm": 2.739917273717214, "language_loss": 0.79639959, "learning_rate": 3.7209260467838926e-06, "loss": 0.81785429, "num_input_tokens_seen": 61407350, "router_z_loss_clip": 0.21582031, "router_z_loss_mlp": 0.734375, "step": 2832, "time_per_iteration": 2.355471611022949 }, { "auxiliary_loss_clip": 0.01100581, "auxiliary_loss_mlp": 0.01042989, "balance_loss_clip": 1.02341509, "balance_loss_mlp": 1.0275898, "epoch": 0.17032917480835713, "flos": 23147554521600.0, "grad_norm": 1.6157854847826956, "language_loss": 0.8847543, "learning_rate": 3.720733510731198e-06, "loss": 0.90619004, "num_input_tokens_seen": 61429010, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.73046875, "step": 2833, "time_per_iteration": 2.4187119007110596 }, { "auxiliary_loss_clip": 0.01099931, "auxiliary_loss_mlp": 0.01042799, "balance_loss_clip": 1.02253354, "balance_loss_mlp": 1.02734184, "epoch": 0.1703892980610251, "flos": 39201003020160.0, "grad_norm": 2.2330127490136915, "language_loss": 0.71865654, "learning_rate": 3.72054091326995e-06, "loss": 0.74008387, "num_input_tokens_seen": 61450040, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7265625, "step": 2834, "time_per_iteration": 2.5245070457458496 }, { "auxiliary_loss_clip": 0.01104221, "auxiliary_loss_mlp": 0.01044854, "balance_loss_clip": 1.02518415, "balance_loss_mlp": 1.03048635, "epoch": 0.17044942131369306, "flos": 23037997075200.0, "grad_norm": 2.04637989842674, "language_loss": 0.86782855, "learning_rate": 3.7203482544070227e-06, "loss": 0.8893193, "num_input_tokens_seen": 61468585, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.73828125, "step": 2835, "time_per_iteration": 2.3873298168182373 }, { "auxiliary_loss_clip": 0.0110199, "auxiliary_loss_mlp": 0.0104297, "balance_loss_clip": 1.02076173, "balance_loss_mlp": 1.0270679, "epoch": 0.17050954456636103, "flos": 17054058464640.0, "grad_norm": 2.074709036086154, "language_loss": 0.73609614, "learning_rate": 3.720155534149292e-06, "loss": 0.75754571, "num_input_tokens_seen": 61486330, "router_z_loss_clip": 0.22265625, "router_z_loss_mlp": 0.75, "step": 2836, "time_per_iteration": 2.3513550758361816 }, { "auxiliary_loss_clip": 0.01107355, "auxiliary_loss_mlp": 0.01040434, "balance_loss_clip": 1.01742601, "balance_loss_mlp": 1.02962399, "epoch": 0.17056966781902902, "flos": 16836025824000.0, "grad_norm": 2.084422244258617, "language_loss": 0.80140126, "learning_rate": 3.7199627525036343e-06, "loss": 0.8228792, "num_input_tokens_seen": 61503950, "router_z_loss_clip": 0.23046875, "router_z_loss_mlp": 0.77734375, "step": 2837, "time_per_iteration": 2.3739218711853027 }, { "auxiliary_loss_clip": 0.01098566, "auxiliary_loss_mlp": 0.01039546, "balance_loss_clip": 1.01877975, "balance_loss_mlp": 1.02908397, "epoch": 0.17062979107169698, "flos": 17711577699840.0, "grad_norm": 9.587808322010051, "language_loss": 0.83528239, "learning_rate": 3.7197699094769303e-06, "loss": 0.85666353, "num_input_tokens_seen": 61523550, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.6953125, "step": 2838, "time_per_iteration": 2.3722033500671387 }, { "auxiliary_loss_clip": 0.01098691, "auxiliary_loss_mlp": 0.01034909, "balance_loss_clip": 1.01614571, "balance_loss_mlp": 1.02855921, "epoch": 0.17068991432436495, "flos": 22524040817280.0, "grad_norm": 1.7594720877797905, "language_loss": 0.93555927, "learning_rate": 3.719577005076062e-06, "loss": 0.95689523, "num_input_tokens_seen": 61542720, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.703125, "step": 2839, "time_per_iteration": 2.414508581161499 }, { "auxiliary_loss_clip": 0.01102511, "auxiliary_loss_mlp": 0.01039726, "balance_loss_clip": 1.01892352, "balance_loss_mlp": 1.02878416, "epoch": 0.17075003757703291, "flos": 25881812559360.0, "grad_norm": 2.486884554266925, "language_loss": 0.83609664, "learning_rate": 3.719384039307914e-06, "loss": 0.85751897, "num_input_tokens_seen": 61563040, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.734375, "step": 2840, "time_per_iteration": 2.414292573928833 }, { "auxiliary_loss_clip": 0.01102081, "auxiliary_loss_mlp": 0.01041209, "balance_loss_clip": 1.01995409, "balance_loss_mlp": 1.02834702, "epoch": 0.17081016082970088, "flos": 20119677016320.0, "grad_norm": 1.9077523811333352, "language_loss": 0.75979531, "learning_rate": 3.7191910121793723e-06, "loss": 0.78122818, "num_input_tokens_seen": 61581890, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.734375, "step": 2841, "time_per_iteration": 2.3861541748046875 }, { "auxiliary_loss_clip": 0.0109873, "auxiliary_loss_mlp": 0.0104296, "balance_loss_clip": 1.02227688, "balance_loss_mlp": 1.02622223, "epoch": 0.17087028408236885, "flos": 24935317067520.0, "grad_norm": 1.8118609133322574, "language_loss": 0.76893795, "learning_rate": 3.718997923697326e-06, "loss": 0.79035485, "num_input_tokens_seen": 61602095, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.7265625, "step": 2842, "time_per_iteration": 2.4093101024627686 }, { "auxiliary_loss_clip": 0.01098277, "auxiliary_loss_mlp": 0.01038635, "balance_loss_clip": 1.01902461, "balance_loss_mlp": 1.02864337, "epoch": 0.17093040733503684, "flos": 19056990919680.0, "grad_norm": 1.96747438457974, "language_loss": 0.85524523, "learning_rate": 3.7188047738686655e-06, "loss": 0.87661433, "num_input_tokens_seen": 61620400, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.6953125, "step": 2843, "time_per_iteration": 2.4019691944122314 }, { "auxiliary_loss_clip": 0.01098306, "auxiliary_loss_mlp": 0.01040205, "balance_loss_clip": 1.02007055, "balance_loss_mlp": 1.02914369, "epoch": 0.1709905305877048, "flos": 13078114456320.0, "grad_norm": 1.7886432537207648, "language_loss": 0.68202627, "learning_rate": 3.7186115627002837e-06, "loss": 0.70341146, "num_input_tokens_seen": 61637680, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.69140625, "step": 2844, "time_per_iteration": 2.374447822570801 }, { "auxiliary_loss_clip": 0.01101739, "auxiliary_loss_mlp": 0.01043683, "balance_loss_clip": 1.02196276, "balance_loss_mlp": 1.02908492, "epoch": 0.17105065384037277, "flos": 19208304218880.0, "grad_norm": 2.0010483451506085, "language_loss": 0.78770077, "learning_rate": 3.718418290199076e-06, "loss": 0.80915499, "num_input_tokens_seen": 61655630, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.7265625, "step": 2845, "time_per_iteration": 2.3720314502716064 }, { "auxiliary_loss_clip": 0.01097968, "auxiliary_loss_mlp": 0.01039524, "balance_loss_clip": 1.02000928, "balance_loss_mlp": 1.02587223, "epoch": 0.17111077709304073, "flos": 18514196012160.0, "grad_norm": 4.3704053337883755, "language_loss": 0.77804375, "learning_rate": 3.71822495637194e-06, "loss": 0.79941869, "num_input_tokens_seen": 61673475, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.72265625, "step": 2846, "time_per_iteration": 2.3560523986816406 }, { "auxiliary_loss_clip": 0.01099426, "auxiliary_loss_mlp": 0.01040035, "balance_loss_clip": 1.02113986, "balance_loss_mlp": 1.02861381, "epoch": 0.1711709003457087, "flos": 25081498396800.0, "grad_norm": 1.711799864431428, "language_loss": 0.79977489, "learning_rate": 3.7180315612257748e-06, "loss": 0.8211695, "num_input_tokens_seen": 61693370, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.70703125, "step": 2847, "time_per_iteration": 2.428332805633545 }, { "auxiliary_loss_clip": 0.01097876, "auxiliary_loss_mlp": 0.01039338, "balance_loss_clip": 1.02033615, "balance_loss_mlp": 1.02556586, "epoch": 0.17123102359837666, "flos": 17565431281920.0, "grad_norm": 3.0462412738497533, "language_loss": 0.86679769, "learning_rate": 3.7178381047674825e-06, "loss": 0.88816977, "num_input_tokens_seen": 61710820, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.72265625, "step": 2848, "time_per_iteration": 2.3382651805877686 }, { "auxiliary_loss_clip": 0.01100188, "auxiliary_loss_mlp": 0.01039521, "balance_loss_clip": 1.01960135, "balance_loss_mlp": 1.02768493, "epoch": 0.17129114685104463, "flos": 26172534384000.0, "grad_norm": 2.1077203639678475, "language_loss": 0.75360501, "learning_rate": 3.717644587003967e-06, "loss": 0.77500212, "num_input_tokens_seen": 61729855, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.72265625, "step": 2849, "time_per_iteration": 2.4131977558135986 }, { "auxiliary_loss_clip": 0.01026538, "auxiliary_loss_mlp": 0.01004335, "balance_loss_clip": 1.00199831, "balance_loss_mlp": 1.00581324, "epoch": 0.17135127010371262, "flos": 69266212220160.0, "grad_norm": 0.7842330333871769, "language_loss": 0.57423878, "learning_rate": 3.7174510079421347e-06, "loss": 0.59454751, "num_input_tokens_seen": 61790290, "router_z_loss_clip": 0.02331543, "router_z_loss_mlp": 0.20703125, "step": 2850, "time_per_iteration": 3.004063367843628 }, { "auxiliary_loss_clip": 0.01097386, "auxiliary_loss_mlp": 0.01042264, "balance_loss_clip": 1.02221298, "balance_loss_mlp": 1.02785206, "epoch": 0.1714113933563806, "flos": 23548985867520.0, "grad_norm": 3.1580788317294433, "language_loss": 0.80728292, "learning_rate": 3.7172573675888937e-06, "loss": 0.82867938, "num_input_tokens_seen": 61809265, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.6953125, "step": 2851, "time_per_iteration": 2.409008502960205 }, { "auxiliary_loss_clip": 0.01096345, "auxiliary_loss_mlp": 0.01035792, "balance_loss_clip": 1.01668262, "balance_loss_mlp": 1.02716529, "epoch": 0.17147151660904855, "flos": 21141375310080.0, "grad_norm": 6.502278396988303, "language_loss": 0.93110287, "learning_rate": 3.717063665951155e-06, "loss": 0.95242417, "num_input_tokens_seen": 61828980, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.69140625, "step": 2852, "time_per_iteration": 2.3677895069122314 }, { "auxiliary_loss_clip": 0.01099955, "auxiliary_loss_mlp": 0.01041107, "balance_loss_clip": 1.02143705, "balance_loss_mlp": 1.02682829, "epoch": 0.17153163986171652, "flos": 18623893104000.0, "grad_norm": 1.9697959632093773, "language_loss": 0.68919253, "learning_rate": 3.7168699030358305e-06, "loss": 0.71060312, "num_input_tokens_seen": 61847915, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.734375, "step": 2853, "time_per_iteration": 2.3650238513946533 }, { "auxiliary_loss_clip": 0.01103775, "auxiliary_loss_mlp": 0.01041317, "balance_loss_clip": 1.02082443, "balance_loss_mlp": 1.03048611, "epoch": 0.17159176311438448, "flos": 18222287201280.0, "grad_norm": 2.3573506234697623, "language_loss": 0.66342807, "learning_rate": 3.7166760788498355e-06, "loss": 0.68487895, "num_input_tokens_seen": 61865570, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.734375, "step": 2854, "time_per_iteration": 2.346888303756714 }, { "auxiliary_loss_clip": 0.01094329, "auxiliary_loss_mlp": 0.0103849, "balance_loss_clip": 1.0200839, "balance_loss_mlp": 1.02519643, "epoch": 0.17165188636705245, "flos": 20737988928000.0, "grad_norm": 1.7339636954993085, "language_loss": 0.89137179, "learning_rate": 3.716482193400087e-06, "loss": 0.9127, "num_input_tokens_seen": 61883340, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.69140625, "step": 2855, "time_per_iteration": 2.3888208866119385 }, { "auxiliary_loss_clip": 0.01101268, "auxiliary_loss_mlp": 0.0103767, "balance_loss_clip": 1.01755893, "balance_loss_mlp": 1.02768111, "epoch": 0.17171200961972044, "flos": 24898728096000.0, "grad_norm": 1.9672459927595096, "language_loss": 0.82613242, "learning_rate": 3.7162882466935042e-06, "loss": 0.84752178, "num_input_tokens_seen": 61900610, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.734375, "step": 2856, "time_per_iteration": 2.4004223346710205 }, { "auxiliary_loss_clip": 0.01098951, "auxiliary_loss_mlp": 0.0104157, "balance_loss_clip": 1.02188826, "balance_loss_mlp": 1.02765584, "epoch": 0.1717721328723884, "flos": 20156196165120.0, "grad_norm": 2.041226216781192, "language_loss": 0.86407518, "learning_rate": 3.716094238737009e-06, "loss": 0.8854804, "num_input_tokens_seen": 61916795, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.7109375, "step": 2857, "time_per_iteration": 2.368487596511841 }, { "auxiliary_loss_clip": 0.01100271, "auxiliary_loss_mlp": 0.01043538, "balance_loss_clip": 1.02352262, "balance_loss_mlp": 1.02817392, "epoch": 0.17183225612505637, "flos": 23360699571840.0, "grad_norm": 2.6917011832688993, "language_loss": 0.78101349, "learning_rate": 3.715900169537524e-06, "loss": 0.80245161, "num_input_tokens_seen": 61936665, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.72265625, "step": 2858, "time_per_iteration": 2.407489776611328 }, { "auxiliary_loss_clip": 0.01106981, "auxiliary_loss_mlp": 0.01050912, "balance_loss_clip": 1.02846503, "balance_loss_mlp": 1.0280242, "epoch": 0.17189237937772434, "flos": 18113253425280.0, "grad_norm": 2.327169641792259, "language_loss": 0.76883638, "learning_rate": 3.7157060391019767e-06, "loss": 0.79041535, "num_input_tokens_seen": 61954415, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.7890625, "step": 2859, "time_per_iteration": 3.7524237632751465 }, { "auxiliary_loss_clip": 0.01097007, "auxiliary_loss_mlp": 0.01037272, "balance_loss_clip": 1.01747096, "balance_loss_mlp": 1.02755082, "epoch": 0.1719525026303923, "flos": 23257286524800.0, "grad_norm": 1.9563460985952137, "language_loss": 0.76953274, "learning_rate": 3.7155118474372936e-06, "loss": 0.79087549, "num_input_tokens_seen": 61973940, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.6953125, "step": 2860, "time_per_iteration": 2.4103503227233887 }, { "auxiliary_loss_clip": 0.01099755, "auxiliary_loss_mlp": 0.01034111, "balance_loss_clip": 1.01408386, "balance_loss_mlp": 1.02734613, "epoch": 0.17201262588306027, "flos": 20809456214400.0, "grad_norm": 2.818945073790931, "language_loss": 0.81869853, "learning_rate": 3.7153175945504057e-06, "loss": 0.84003723, "num_input_tokens_seen": 61991845, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.7265625, "step": 2861, "time_per_iteration": 3.7697300910949707 }, { "auxiliary_loss_clip": 0.01098907, "auxiliary_loss_mlp": 0.01039077, "balance_loss_clip": 1.0195142, "balance_loss_mlp": 1.02756715, "epoch": 0.17207274913572823, "flos": 20374822298880.0, "grad_norm": 4.648906746694479, "language_loss": 0.85571301, "learning_rate": 3.7151232804482456e-06, "loss": 0.87709284, "num_input_tokens_seen": 62009395, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.71484375, "step": 2862, "time_per_iteration": 3.7209343910217285 }, { "auxiliary_loss_clip": 0.0109487, "auxiliary_loss_mlp": 0.01037687, "balance_loss_clip": 1.01934016, "balance_loss_mlp": 1.02720714, "epoch": 0.17213287238839622, "flos": 26796501936000.0, "grad_norm": 3.680640271375408, "language_loss": 0.78025091, "learning_rate": 3.7149289051377474e-06, "loss": 0.8015765, "num_input_tokens_seen": 62029005, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.67578125, "step": 2863, "time_per_iteration": 3.7889277935028076 }, { "auxiliary_loss_clip": 0.01097385, "auxiliary_loss_mlp": 0.01042709, "balance_loss_clip": 1.02197862, "balance_loss_mlp": 1.02595067, "epoch": 0.1721929956410642, "flos": 26029634722560.0, "grad_norm": 1.6838877162819452, "language_loss": 0.72329086, "learning_rate": 3.714734468625847e-06, "loss": 0.74469173, "num_input_tokens_seen": 62048730, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.71484375, "step": 2864, "time_per_iteration": 2.438499927520752 }, { "auxiliary_loss_clip": 0.01101815, "auxiliary_loss_mlp": 0.01038164, "balance_loss_clip": 1.01929295, "balance_loss_mlp": 1.02860403, "epoch": 0.17225311889373215, "flos": 22272002645760.0, "grad_norm": 2.0219813649855074, "language_loss": 0.72502583, "learning_rate": 3.714539970919485e-06, "loss": 0.74642563, "num_input_tokens_seen": 62069000, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.734375, "step": 2865, "time_per_iteration": 2.3733339309692383 }, { "auxiliary_loss_clip": 0.01100059, "auxiliary_loss_mlp": 0.01040146, "balance_loss_clip": 1.02076221, "balance_loss_mlp": 1.02951944, "epoch": 0.17231324214640012, "flos": 21286718766720.0, "grad_norm": 2.9961208754105124, "language_loss": 0.78705955, "learning_rate": 3.7143454120256017e-06, "loss": 0.80846155, "num_input_tokens_seen": 62086750, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.70703125, "step": 2866, "time_per_iteration": 2.4031829833984375 }, { "auxiliary_loss_clip": 0.01097716, "auxiliary_loss_mlp": 0.01040117, "balance_loss_clip": 1.02022111, "balance_loss_mlp": 1.02698362, "epoch": 0.17237336539906808, "flos": 19679771485440.0, "grad_norm": 1.706005589775413, "language_loss": 0.79743361, "learning_rate": 3.71415079195114e-06, "loss": 0.81881189, "num_input_tokens_seen": 62106240, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.70703125, "step": 2867, "time_per_iteration": 2.3662564754486084 }, { "auxiliary_loss_clip": 0.01097452, "auxiliary_loss_mlp": 0.01035584, "balance_loss_clip": 1.01467443, "balance_loss_mlp": 1.02625537, "epoch": 0.17243348865173605, "flos": 17528702664960.0, "grad_norm": 1.9585180560379911, "language_loss": 0.79336035, "learning_rate": 3.713956110703046e-06, "loss": 0.81469071, "num_input_tokens_seen": 62124895, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7109375, "step": 2868, "time_per_iteration": 2.3915653228759766 }, { "auxiliary_loss_clip": 0.0110449, "auxiliary_loss_mlp": 0.01038557, "balance_loss_clip": 1.01855421, "balance_loss_mlp": 1.02958059, "epoch": 0.17249361190440402, "flos": 18258876172800.0, "grad_norm": 2.445199233157961, "language_loss": 0.83990276, "learning_rate": 3.713761368288268e-06, "loss": 0.86133319, "num_input_tokens_seen": 62143510, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.75, "step": 2869, "time_per_iteration": 2.3506219387054443 }, { "auxiliary_loss_clip": 0.01100771, "auxiliary_loss_mlp": 0.01042502, "balance_loss_clip": 1.02104473, "balance_loss_mlp": 1.02764046, "epoch": 0.172553735157072, "flos": 21173425804800.0, "grad_norm": 1.7938640814322828, "language_loss": 0.76781571, "learning_rate": 3.713566564713754e-06, "loss": 0.78924841, "num_input_tokens_seen": 62162285, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.73046875, "step": 2870, "time_per_iteration": 2.4121081829071045 }, { "auxiliary_loss_clip": 0.01094729, "auxiliary_loss_mlp": 0.01034617, "balance_loss_clip": 1.0175581, "balance_loss_mlp": 1.02771533, "epoch": 0.17261385840973997, "flos": 22272177202560.0, "grad_norm": 1.7992539394948617, "language_loss": 0.76963258, "learning_rate": 3.7133716999864574e-06, "loss": 0.7909261, "num_input_tokens_seen": 62180970, "router_z_loss_clip": 0.17089844, "router_z_loss_mlp": 0.671875, "step": 2871, "time_per_iteration": 2.3836746215820312 }, { "auxiliary_loss_clip": 0.01097193, "auxiliary_loss_mlp": 0.010355, "balance_loss_clip": 1.01587784, "balance_loss_mlp": 1.02676845, "epoch": 0.17267398166240794, "flos": 27921159429120.0, "grad_norm": 2.6200849364685523, "language_loss": 0.7476573, "learning_rate": 3.7131767741133327e-06, "loss": 0.76898426, "num_input_tokens_seen": 62198965, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.703125, "step": 2872, "time_per_iteration": 2.450266122817993 }, { "auxiliary_loss_clip": 0.01095105, "auxiliary_loss_mlp": 0.01039845, "balance_loss_clip": 1.0204612, "balance_loss_mlp": 1.02686155, "epoch": 0.1727341049150759, "flos": 21944028533760.0, "grad_norm": 6.242095849254738, "language_loss": 0.82088262, "learning_rate": 3.712981787101335e-06, "loss": 0.84223211, "num_input_tokens_seen": 62219890, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.6796875, "step": 2873, "time_per_iteration": 2.385953903198242 }, { "auxiliary_loss_clip": 0.01097729, "auxiliary_loss_mlp": 0.01035709, "balance_loss_clip": 1.01587296, "balance_loss_mlp": 1.0280807, "epoch": 0.17279422816774387, "flos": 18107074114560.0, "grad_norm": 2.0824944934024656, "language_loss": 0.74705172, "learning_rate": 3.7127867389574244e-06, "loss": 0.76838607, "num_input_tokens_seen": 62237140, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.6953125, "step": 2874, "time_per_iteration": 2.3947160243988037 }, { "auxiliary_loss_clip": 0.01097361, "auxiliary_loss_mlp": 0.01040762, "balance_loss_clip": 1.01957846, "balance_loss_mlp": 1.02649164, "epoch": 0.17285435142041183, "flos": 21834366353280.0, "grad_norm": 2.0074995783007985, "language_loss": 0.80613792, "learning_rate": 3.7125916296885606e-06, "loss": 0.82751918, "num_input_tokens_seen": 62255405, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.7109375, "step": 2875, "time_per_iteration": 2.3952105045318604 }, { "auxiliary_loss_clip": 0.01100966, "auxiliary_loss_mlp": 0.01041017, "balance_loss_clip": 1.01912999, "balance_loss_mlp": 1.02759469, "epoch": 0.17291447467307983, "flos": 18367491012480.0, "grad_norm": 2.8247080586996716, "language_loss": 0.87094033, "learning_rate": 3.7123964593017066e-06, "loss": 0.89236015, "num_input_tokens_seen": 62271280, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.734375, "step": 2876, "time_per_iteration": 2.377943515777588 }, { "auxiliary_loss_clip": 0.01097734, "auxiliary_loss_mlp": 0.01039503, "balance_loss_clip": 1.02060854, "balance_loss_mlp": 1.0285635, "epoch": 0.1729745979257478, "flos": 18623648724480.0, "grad_norm": 1.839598799780756, "language_loss": 0.84719235, "learning_rate": 3.7122012278038285e-06, "loss": 0.86856472, "num_input_tokens_seen": 62289140, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.69140625, "step": 2877, "time_per_iteration": 2.348487138748169 }, { "auxiliary_loss_clip": 0.01100964, "auxiliary_loss_mlp": 0.01036013, "balance_loss_clip": 1.01605749, "balance_loss_mlp": 1.02966297, "epoch": 0.17303472117841576, "flos": 22997253651840.0, "grad_norm": 2.2625307315465935, "language_loss": 0.79290515, "learning_rate": 3.7120059352018922e-06, "loss": 0.81427491, "num_input_tokens_seen": 62307490, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.7109375, "step": 2878, "time_per_iteration": 2.4186136722564697 }, { "auxiliary_loss_clip": 0.01095456, "auxiliary_loss_mlp": 0.01037214, "balance_loss_clip": 1.01696002, "balance_loss_mlp": 1.02593291, "epoch": 0.17309484443108372, "flos": 25663256248320.0, "grad_norm": 1.7061990498952924, "language_loss": 0.70197231, "learning_rate": 3.7118105815028677e-06, "loss": 0.72329903, "num_input_tokens_seen": 62328570, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.6953125, "step": 2879, "time_per_iteration": 2.4180474281311035 }, { "auxiliary_loss_clip": 0.01097694, "auxiliary_loss_mlp": 0.01041854, "balance_loss_clip": 1.02081323, "balance_loss_mlp": 1.02637446, "epoch": 0.1731549676837517, "flos": 13552060429440.0, "grad_norm": 2.0531514935022184, "language_loss": 0.8334012, "learning_rate": 3.7116151667137272e-06, "loss": 0.85479665, "num_input_tokens_seen": 62345735, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.71484375, "step": 2880, "time_per_iteration": 2.375086545944214 }, { "auxiliary_loss_clip": 0.01102314, "auxiliary_loss_mlp": 0.01036382, "balance_loss_clip": 1.01543653, "balance_loss_mlp": 1.02918601, "epoch": 0.17321509093641965, "flos": 22855959912960.0, "grad_norm": 2.0841091537860934, "language_loss": 0.80595112, "learning_rate": 3.7114196908414444e-06, "loss": 0.8273381, "num_input_tokens_seen": 62365525, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.734375, "step": 2881, "time_per_iteration": 2.3861749172210693 }, { "auxiliary_loss_clip": 0.01099988, "auxiliary_loss_mlp": 0.01034552, "balance_loss_clip": 1.01553822, "balance_loss_mlp": 1.02886462, "epoch": 0.17327521418908762, "flos": 24351639091200.0, "grad_norm": 2.6595247932454313, "language_loss": 0.77453423, "learning_rate": 3.7112241538929946e-06, "loss": 0.7958796, "num_input_tokens_seen": 62385160, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.7109375, "step": 2882, "time_per_iteration": 2.4411187171936035 }, { "auxiliary_loss_clip": 0.01096081, "auxiliary_loss_mlp": 0.01035144, "balance_loss_clip": 1.01562929, "balance_loss_mlp": 1.02655911, "epoch": 0.1733353374417556, "flos": 33104364940800.0, "grad_norm": 1.8538684903891403, "language_loss": 0.76314259, "learning_rate": 3.711028555875357e-06, "loss": 0.78445482, "num_input_tokens_seen": 62405280, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.6953125, "step": 2883, "time_per_iteration": 2.4880666732788086 }, { "auxiliary_loss_clip": 0.01097012, "auxiliary_loss_mlp": 0.01035835, "balance_loss_clip": 1.01678514, "balance_loss_mlp": 1.02765751, "epoch": 0.17339546069442358, "flos": 24387809126400.0, "grad_norm": 1.8911862531878234, "language_loss": 0.85407919, "learning_rate": 3.7108328967955113e-06, "loss": 0.87540758, "num_input_tokens_seen": 62423665, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.6953125, "step": 2884, "time_per_iteration": 2.4213695526123047 }, { "auxiliary_loss_clip": 0.01098005, "auxiliary_loss_mlp": 0.0104383, "balance_loss_clip": 1.02525759, "balance_loss_mlp": 1.02772915, "epoch": 0.17345558394709154, "flos": 27452938919040.0, "grad_norm": 2.875467068049921, "language_loss": 0.74540174, "learning_rate": 3.7106371766604408e-06, "loss": 0.76682007, "num_input_tokens_seen": 62445170, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.703125, "step": 2885, "time_per_iteration": 2.422321319580078 }, { "auxiliary_loss_clip": 0.01093593, "auxiliary_loss_mlp": 0.01034796, "balance_loss_clip": 1.01702213, "balance_loss_mlp": 1.02822804, "epoch": 0.1735157071997595, "flos": 24680974746240.0, "grad_norm": 1.5230780813505223, "language_loss": 0.70776856, "learning_rate": 3.7104413954771294e-06, "loss": 0.72905242, "num_input_tokens_seen": 62466135, "router_z_loss_clip": 0.17773438, "router_z_loss_mlp": 0.65234375, "step": 2886, "time_per_iteration": 2.4529497623443604 }, { "auxiliary_loss_clip": 0.0109737, "auxiliary_loss_mlp": 0.01034553, "balance_loss_clip": 1.01472795, "balance_loss_mlp": 1.02688885, "epoch": 0.17357583045242747, "flos": 21687870821760.0, "grad_norm": 2.549200029262177, "language_loss": 0.69418108, "learning_rate": 3.710245553252564e-06, "loss": 0.7155003, "num_input_tokens_seen": 62483910, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.703125, "step": 2887, "time_per_iteration": 2.4095358848571777 }, { "auxiliary_loss_clip": 0.01098123, "auxiliary_loss_mlp": 0.01041994, "balance_loss_clip": 1.02282476, "balance_loss_mlp": 1.02799964, "epoch": 0.17363595370509544, "flos": 15374875847040.0, "grad_norm": 1.8093871820287766, "language_loss": 0.853176, "learning_rate": 3.7100496499937345e-06, "loss": 0.87457716, "num_input_tokens_seen": 62501530, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.703125, "step": 2888, "time_per_iteration": 2.3673512935638428 }, { "auxiliary_loss_clip": 0.01097958, "auxiliary_loss_mlp": 0.01039942, "balance_loss_clip": 1.01965261, "balance_loss_mlp": 1.02668357, "epoch": 0.1736960769577634, "flos": 23439812446080.0, "grad_norm": 2.7465663528092237, "language_loss": 0.78378886, "learning_rate": 3.7098536857076315e-06, "loss": 0.80516785, "num_input_tokens_seen": 62521295, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7109375, "step": 2889, "time_per_iteration": 2.4165608882904053 }, { "auxiliary_loss_clip": 0.01094021, "auxiliary_loss_mlp": 0.01035741, "balance_loss_clip": 1.01685858, "balance_loss_mlp": 1.0275017, "epoch": 0.1737562002104314, "flos": 18586850284800.0, "grad_norm": 2.261143993961399, "language_loss": 0.83725846, "learning_rate": 3.7096576604012492e-06, "loss": 0.85855603, "num_input_tokens_seen": 62539615, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.6640625, "step": 2890, "time_per_iteration": 2.373586893081665 }, { "auxiliary_loss_clip": 0.01099012, "auxiliary_loss_mlp": 0.01044074, "balance_loss_clip": 1.02492881, "balance_loss_mlp": 1.02829003, "epoch": 0.17381632346309936, "flos": 15997132742400.0, "grad_norm": 2.1077282277956457, "language_loss": 0.82070744, "learning_rate": 3.7094615740815824e-06, "loss": 0.84213829, "num_input_tokens_seen": 62556820, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.70703125, "step": 2891, "time_per_iteration": 2.340186595916748 }, { "auxiliary_loss_clip": 0.01098339, "auxiliary_loss_mlp": 0.01034982, "balance_loss_clip": 1.01451373, "balance_loss_mlp": 1.02644587, "epoch": 0.17387644671576732, "flos": 13369010837760.0, "grad_norm": 1.9985918358305603, "language_loss": 0.80757391, "learning_rate": 3.709265426755629e-06, "loss": 0.82890713, "num_input_tokens_seen": 62572450, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.71875, "step": 2892, "time_per_iteration": 2.343317985534668 }, { "auxiliary_loss_clip": 0.01100705, "auxiliary_loss_mlp": 0.01040469, "balance_loss_clip": 1.01998901, "balance_loss_mlp": 1.0291729, "epoch": 0.1739365699684353, "flos": 26614290216960.0, "grad_norm": 2.924729453432933, "language_loss": 0.74330664, "learning_rate": 3.7090692184303894e-06, "loss": 0.76471835, "num_input_tokens_seen": 62592580, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.71484375, "step": 2893, "time_per_iteration": 2.4177193641662598 }, { "auxiliary_loss_clip": 0.01099165, "auxiliary_loss_mlp": 0.01040929, "balance_loss_clip": 1.02050877, "balance_loss_mlp": 1.02689755, "epoch": 0.17399669322110325, "flos": 23366843971200.0, "grad_norm": 1.9308889919567662, "language_loss": 0.82883406, "learning_rate": 3.7088729491128665e-06, "loss": 0.85023499, "num_input_tokens_seen": 62611220, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.72265625, "step": 2894, "time_per_iteration": 2.4008944034576416 }, { "auxiliary_loss_clip": 0.010986, "auxiliary_loss_mlp": 0.0103375, "balance_loss_clip": 1.01236343, "balance_loss_mlp": 1.02747357, "epoch": 0.17405681647377122, "flos": 22053027398400.0, "grad_norm": 4.187679183561046, "language_loss": 0.74383038, "learning_rate": 3.708676618810063e-06, "loss": 0.76515388, "num_input_tokens_seen": 62629185, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.7109375, "step": 2895, "time_per_iteration": 2.3879427909851074 }, { "auxiliary_loss_clip": 0.01029121, "auxiliary_loss_mlp": 0.01007236, "balance_loss_clip": 1.00432754, "balance_loss_mlp": 1.00671434, "epoch": 0.1741169397264392, "flos": 61454396044800.0, "grad_norm": 0.8721027012547796, "language_loss": 0.62732995, "learning_rate": 3.7084802275289866e-06, "loss": 0.64769351, "num_input_tokens_seen": 62691895, "router_z_loss_clip": 0.02905273, "router_z_loss_mlp": 0.22460938, "step": 2896, "time_per_iteration": 3.1353063583374023 }, { "auxiliary_loss_clip": 0.01096297, "auxiliary_loss_mlp": 0.01033438, "balance_loss_clip": 1.01472223, "balance_loss_mlp": 1.02518332, "epoch": 0.17417706297910718, "flos": 27016419790080.0, "grad_norm": 2.159513337646916, "language_loss": 0.75981808, "learning_rate": 3.708283775276645e-06, "loss": 0.78111547, "num_input_tokens_seen": 62713790, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.7109375, "step": 2897, "time_per_iteration": 2.430640935897827 }, { "auxiliary_loss_clip": 0.01096242, "auxiliary_loss_mlp": 0.0103623, "balance_loss_clip": 1.01592231, "balance_loss_mlp": 1.02759027, "epoch": 0.17423718623177514, "flos": 33507506943360.0, "grad_norm": 2.226953318823505, "language_loss": 0.69583464, "learning_rate": 3.70808726206005e-06, "loss": 0.71715933, "num_input_tokens_seen": 62736285, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.6875, "step": 2898, "time_per_iteration": 3.8754465579986572 }, { "auxiliary_loss_clip": 0.01102547, "auxiliary_loss_mlp": 0.01040489, "balance_loss_clip": 1.01961541, "balance_loss_mlp": 1.02899253, "epoch": 0.1742973094844431, "flos": 27197409611520.0, "grad_norm": 2.4331251828178315, "language_loss": 0.76197898, "learning_rate": 3.7078906878862145e-06, "loss": 0.78340936, "num_input_tokens_seen": 62756240, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.734375, "step": 2899, "time_per_iteration": 2.4297640323638916 }, { "auxiliary_loss_clip": 0.01094865, "auxiliary_loss_mlp": 0.01038535, "balance_loss_clip": 1.01890135, "balance_loss_mlp": 1.02577353, "epoch": 0.17435743273711107, "flos": 22709638938240.0, "grad_norm": 1.829918370174299, "language_loss": 0.7232179, "learning_rate": 3.7076940527621536e-06, "loss": 0.74455196, "num_input_tokens_seen": 62775910, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.69140625, "step": 2900, "time_per_iteration": 2.4094789028167725 }, { "auxiliary_loss_clip": 0.01101356, "auxiliary_loss_mlp": 0.01043212, "balance_loss_clip": 1.02312541, "balance_loss_mlp": 1.0292275, "epoch": 0.17441755598977904, "flos": 41644853435520.0, "grad_norm": 1.6640230087428245, "language_loss": 0.69881225, "learning_rate": 3.707497356694884e-06, "loss": 0.72025788, "num_input_tokens_seen": 62799385, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.72265625, "step": 2901, "time_per_iteration": 5.310842990875244 }, { "auxiliary_loss_clip": 0.01100747, "auxiliary_loss_mlp": 0.01041757, "balance_loss_clip": 1.0216223, "balance_loss_mlp": 1.02748847, "epoch": 0.174477679242447, "flos": 26285862257280.0, "grad_norm": 2.413978229389291, "language_loss": 0.76461095, "learning_rate": 3.707300599691427e-06, "loss": 0.78603601, "num_input_tokens_seen": 62819380, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.734375, "step": 2902, "time_per_iteration": 3.814647674560547 }, { "auxiliary_loss_clip": 0.01099999, "auxiliary_loss_mlp": 0.01043622, "balance_loss_clip": 1.02456009, "balance_loss_mlp": 1.02765155, "epoch": 0.174537802495115, "flos": 17857444826880.0, "grad_norm": 2.1695329462755764, "language_loss": 0.81537986, "learning_rate": 3.7071037817588023e-06, "loss": 0.83681607, "num_input_tokens_seen": 62836205, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.72265625, "step": 2903, "time_per_iteration": 2.345242500305176 }, { "auxiliary_loss_clip": 0.01098288, "auxiliary_loss_mlp": 0.01034931, "balance_loss_clip": 1.01455855, "balance_loss_mlp": 1.02797866, "epoch": 0.17459792574778296, "flos": 16939927630080.0, "grad_norm": 3.2726253911474683, "language_loss": 0.73323435, "learning_rate": 3.706906902904036e-06, "loss": 0.75456655, "num_input_tokens_seen": 62854045, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.703125, "step": 2904, "time_per_iteration": 2.3807260990142822 }, { "auxiliary_loss_clip": 0.01098457, "auxiliary_loss_mlp": 0.01037204, "balance_loss_clip": 1.0170573, "balance_loss_mlp": 1.02728105, "epoch": 0.17465804900045093, "flos": 25518855398400.0, "grad_norm": 1.9188512777035645, "language_loss": 0.64299375, "learning_rate": 3.7067099631341517e-06, "loss": 0.66435039, "num_input_tokens_seen": 62873075, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.7109375, "step": 2905, "time_per_iteration": 2.408696413040161 }, { "auxiliary_loss_clip": 0.01106027, "auxiliary_loss_mlp": 0.01047717, "balance_loss_clip": 1.02572274, "balance_loss_mlp": 1.03002274, "epoch": 0.1747181722531189, "flos": 24128683948800.0, "grad_norm": 1.7097878330468699, "language_loss": 0.7937634, "learning_rate": 3.70651296245618e-06, "loss": 0.81530094, "num_input_tokens_seen": 62892675, "router_z_loss_clip": 0.21972656, "router_z_loss_mlp": 0.7578125, "step": 2906, "time_per_iteration": 2.4063754081726074 }, { "auxiliary_loss_clip": 0.01099716, "auxiliary_loss_mlp": 0.01047005, "balance_loss_clip": 1.0270853, "balance_loss_mlp": 1.02893543, "epoch": 0.17477829550578686, "flos": 17747852469120.0, "grad_norm": 1.6778656885318153, "language_loss": 0.80657685, "learning_rate": 3.70631590087715e-06, "loss": 0.82804406, "num_input_tokens_seen": 62910675, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.70703125, "step": 2907, "time_per_iteration": 2.3640167713165283 }, { "auxiliary_loss_clip": 0.01098921, "auxiliary_loss_mlp": 0.01042513, "balance_loss_clip": 1.02270007, "balance_loss_mlp": 1.02678657, "epoch": 0.17483841875845482, "flos": 15376446858240.0, "grad_norm": 2.761525710535151, "language_loss": 0.80839372, "learning_rate": 3.706118778404095e-06, "loss": 0.82980806, "num_input_tokens_seen": 62928130, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.72265625, "step": 2908, "time_per_iteration": 2.372986078262329 }, { "auxiliary_loss_clip": 0.0109838, "auxiliary_loss_mlp": 0.01037755, "balance_loss_clip": 1.017609, "balance_loss_mlp": 1.02853179, "epoch": 0.17489854201112282, "flos": 17162359102080.0, "grad_norm": 2.5263439339312135, "language_loss": 0.80055851, "learning_rate": 3.7059215950440487e-06, "loss": 0.8219198, "num_input_tokens_seen": 62944290, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.69921875, "step": 2909, "time_per_iteration": 2.3411989212036133 }, { "auxiliary_loss_clip": 0.01097792, "auxiliary_loss_mlp": 0.01038519, "balance_loss_clip": 1.01776505, "balance_loss_mlp": 1.02661288, "epoch": 0.17495866526379078, "flos": 19754276060160.0, "grad_norm": 2.032621032577187, "language_loss": 0.76930559, "learning_rate": 3.7057243508040494e-06, "loss": 0.79066873, "num_input_tokens_seen": 62963505, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.71484375, "step": 2910, "time_per_iteration": 2.3876938819885254 }, { "auxiliary_loss_clip": 0.01100046, "auxiliary_loss_mlp": 0.01041668, "balance_loss_clip": 1.02018619, "balance_loss_mlp": 1.02748728, "epoch": 0.17501878851645875, "flos": 28509899552640.0, "grad_norm": 2.720788766012272, "language_loss": 0.87412465, "learning_rate": 3.7055270456911354e-06, "loss": 0.89554185, "num_input_tokens_seen": 62985020, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.7265625, "step": 2911, "time_per_iteration": 2.4427218437194824 }, { "auxiliary_loss_clip": 0.01097482, "auxiliary_loss_mlp": 0.01043231, "balance_loss_clip": 1.02215457, "balance_loss_mlp": 1.02574348, "epoch": 0.1750789117691267, "flos": 17930238744960.0, "grad_norm": 2.365782983215061, "language_loss": 0.89540219, "learning_rate": 3.7053296797123485e-06, "loss": 0.91680932, "num_input_tokens_seen": 63001745, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.71875, "step": 2912, "time_per_iteration": 2.364178419113159 }, { "auxiliary_loss_clip": 0.01098745, "auxiliary_loss_mlp": 0.01041352, "balance_loss_clip": 1.01925063, "balance_loss_mlp": 1.02638698, "epoch": 0.17513903502179468, "flos": 18258457236480.0, "grad_norm": 1.9260236057718623, "language_loss": 0.7252481, "learning_rate": 3.7051322528747313e-06, "loss": 0.74664903, "num_input_tokens_seen": 63019750, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.72265625, "step": 2913, "time_per_iteration": 2.3827435970306396 }, { "auxiliary_loss_clip": 0.01096951, "auxiliary_loss_mlp": 0.01034689, "balance_loss_clip": 1.0148052, "balance_loss_mlp": 1.02793205, "epoch": 0.17519915827446264, "flos": 20703669194880.0, "grad_norm": 1.6688855674381564, "language_loss": 0.68798614, "learning_rate": 3.704934765185331e-06, "loss": 0.70930254, "num_input_tokens_seen": 63039500, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.6875, "step": 2914, "time_per_iteration": 2.417940616607666 }, { "auxiliary_loss_clip": 0.01095125, "auxiliary_loss_mlp": 0.0103836, "balance_loss_clip": 1.01779675, "balance_loss_mlp": 1.02606404, "epoch": 0.1752592815271306, "flos": 20522330259840.0, "grad_norm": 1.706189824721809, "language_loss": 0.93502462, "learning_rate": 3.7047372166511945e-06, "loss": 0.95635939, "num_input_tokens_seen": 63059785, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.69140625, "step": 2915, "time_per_iteration": 2.3981058597564697 }, { "auxiliary_loss_clip": 0.01093396, "auxiliary_loss_mlp": 0.01034579, "balance_loss_clip": 1.01422966, "balance_loss_mlp": 1.02555728, "epoch": 0.1753194047797986, "flos": 21798091584000.0, "grad_norm": 1.65925447995177, "language_loss": 0.80993646, "learning_rate": 3.704539607279371e-06, "loss": 0.83121622, "num_input_tokens_seen": 63079385, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.6796875, "step": 2916, "time_per_iteration": 2.411088705062866 }, { "auxiliary_loss_clip": 0.01099869, "auxiliary_loss_mlp": 0.0104288, "balance_loss_clip": 1.02164841, "balance_loss_mlp": 1.02748048, "epoch": 0.17537952803246656, "flos": 20667289691520.0, "grad_norm": 1.5880739721979988, "language_loss": 0.73977023, "learning_rate": 3.704341937076914e-06, "loss": 0.76119775, "num_input_tokens_seen": 63098970, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.72265625, "step": 2917, "time_per_iteration": 2.3958816528320312 }, { "auxiliary_loss_clip": 0.01094786, "auxiliary_loss_mlp": 0.01036699, "balance_loss_clip": 1.01557553, "balance_loss_mlp": 1.02746558, "epoch": 0.17543965128513453, "flos": 23293945319040.0, "grad_norm": 1.9491242793277963, "language_loss": 0.7629177, "learning_rate": 3.7041442060508778e-06, "loss": 0.78423256, "num_input_tokens_seen": 63118750, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.671875, "step": 2918, "time_per_iteration": 2.401737689971924 }, { "auxiliary_loss_clip": 0.01100381, "auxiliary_loss_mlp": 0.01037278, "balance_loss_clip": 1.01521242, "balance_loss_mlp": 1.02643442, "epoch": 0.1754997745378025, "flos": 29094345578880.0, "grad_norm": 3.161543355332977, "language_loss": 0.7428453, "learning_rate": 3.7039464142083183e-06, "loss": 0.76422191, "num_input_tokens_seen": 63136865, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.73828125, "step": 2919, "time_per_iteration": 2.4384982585906982 }, { "auxiliary_loss_clip": 0.01100849, "auxiliary_loss_mlp": 0.01042433, "balance_loss_clip": 1.02054608, "balance_loss_mlp": 1.02537584, "epoch": 0.17555989779047046, "flos": 30370560750720.0, "grad_norm": 2.1375587565939926, "language_loss": 0.74586523, "learning_rate": 3.7037485615562936e-06, "loss": 0.76729798, "num_input_tokens_seen": 63158325, "router_z_loss_clip": 0.21875, "router_z_loss_mlp": 0.75390625, "step": 2920, "time_per_iteration": 2.4775569438934326 }, { "auxiliary_loss_clip": 0.01094259, "auxiliary_loss_mlp": 0.01037155, "balance_loss_clip": 1.01831973, "balance_loss_mlp": 1.02546024, "epoch": 0.17562002104313842, "flos": 23286823401600.0, "grad_norm": 2.1269303189241673, "language_loss": 0.79498994, "learning_rate": 3.703550648101866e-06, "loss": 0.81630409, "num_input_tokens_seen": 63173115, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.6875, "step": 2921, "time_per_iteration": 2.411048650741577 }, { "auxiliary_loss_clip": 0.01102464, "auxiliary_loss_mlp": 0.01043531, "balance_loss_clip": 1.02088106, "balance_loss_mlp": 1.02722764, "epoch": 0.1756801442958064, "flos": 24789345206400.0, "grad_norm": 1.720611101927665, "language_loss": 0.87780988, "learning_rate": 3.7033526738520983e-06, "loss": 0.89926982, "num_input_tokens_seen": 63192880, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.75, "step": 2922, "time_per_iteration": 2.4284417629241943 }, { "auxiliary_loss_clip": 0.01097197, "auxiliary_loss_mlp": 0.01043666, "balance_loss_clip": 1.02282798, "balance_loss_mlp": 1.02513099, "epoch": 0.17574026754847438, "flos": 25770579367680.0, "grad_norm": 2.2000436260039042, "language_loss": 0.62409222, "learning_rate": 3.7031546388140545e-06, "loss": 0.64550078, "num_input_tokens_seen": 63214395, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.71875, "step": 2923, "time_per_iteration": 2.4263389110565186 }, { "auxiliary_loss_clip": 0.01102621, "auxiliary_loss_mlp": 0.01041787, "balance_loss_clip": 1.01973307, "balance_loss_mlp": 1.02732992, "epoch": 0.17580039080114235, "flos": 17455664367360.0, "grad_norm": 2.059515887283728, "language_loss": 0.80213439, "learning_rate": 3.702956542994802e-06, "loss": 0.82357854, "num_input_tokens_seen": 63231020, "router_z_loss_clip": 0.22070312, "router_z_loss_mlp": 0.75390625, "step": 2924, "time_per_iteration": 2.3719711303710938 }, { "auxiliary_loss_clip": 0.0110046, "auxiliary_loss_mlp": 0.01041157, "balance_loss_clip": 1.01857901, "balance_loss_mlp": 1.02674377, "epoch": 0.1758605140538103, "flos": 14863817232000.0, "grad_norm": 3.4252338567491845, "language_loss": 0.7123369, "learning_rate": 3.7027583864014123e-06, "loss": 0.73375309, "num_input_tokens_seen": 63246245, "router_z_loss_clip": 0.22558594, "router_z_loss_mlp": 0.734375, "step": 2925, "time_per_iteration": 2.3547937870025635 }, { "auxiliary_loss_clip": 0.010994, "auxiliary_loss_mlp": 0.01037619, "balance_loss_clip": 1.01755607, "balance_loss_mlp": 1.02856195, "epoch": 0.17592063730647828, "flos": 23003118760320.0, "grad_norm": 1.7061490438206, "language_loss": 0.71652341, "learning_rate": 3.7025601690409555e-06, "loss": 0.73789358, "num_input_tokens_seen": 63267790, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.70703125, "step": 2926, "time_per_iteration": 2.4420547485351562 }, { "auxiliary_loss_clip": 0.01100969, "auxiliary_loss_mlp": 0.01036308, "balance_loss_clip": 1.01409912, "balance_loss_mlp": 1.02715552, "epoch": 0.17598076055914624, "flos": 20740432723200.0, "grad_norm": 1.8321860126161547, "language_loss": 0.8483274, "learning_rate": 3.702361890920505e-06, "loss": 0.86970007, "num_input_tokens_seen": 63286830, "router_z_loss_clip": 0.22167969, "router_z_loss_mlp": 0.73828125, "step": 2927, "time_per_iteration": 2.3879168033599854 }, { "auxiliary_loss_clip": 0.0109817, "auxiliary_loss_mlp": 0.01040642, "balance_loss_clip": 1.02124703, "balance_loss_mlp": 1.02745426, "epoch": 0.1760408838118142, "flos": 34091080185600.0, "grad_norm": 1.9231681298547754, "language_loss": 0.7214148, "learning_rate": 3.702163552047138e-06, "loss": 0.74280298, "num_input_tokens_seen": 63308870, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.70703125, "step": 2928, "time_per_iteration": 2.513338804244995 }, { "auxiliary_loss_clip": 0.01095716, "auxiliary_loss_mlp": 0.01038513, "balance_loss_clip": 1.01797342, "balance_loss_mlp": 1.02707195, "epoch": 0.1761010070644822, "flos": 24167297779200.0, "grad_norm": 1.8471882104266197, "language_loss": 0.83402288, "learning_rate": 3.7019651524279326e-06, "loss": 0.85536516, "num_input_tokens_seen": 63329005, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.6875, "step": 2929, "time_per_iteration": 2.391378164291382 }, { "auxiliary_loss_clip": 0.01098179, "auxiliary_loss_mlp": 0.01043888, "balance_loss_clip": 1.02446914, "balance_loss_mlp": 1.02659583, "epoch": 0.17616113031715017, "flos": 26575536741120.0, "grad_norm": 1.5544288993568705, "language_loss": 0.79389054, "learning_rate": 3.7017666920699693e-06, "loss": 0.81531119, "num_input_tokens_seen": 63349390, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.71875, "step": 2930, "time_per_iteration": 2.433692216873169 }, { "auxiliary_loss_clip": 0.01100173, "auxiliary_loss_mlp": 0.01036324, "balance_loss_clip": 1.01531935, "balance_loss_mlp": 1.02801824, "epoch": 0.17622125356981813, "flos": 25665490575360.0, "grad_norm": 2.2015391258894117, "language_loss": 0.77019572, "learning_rate": 3.701568170980329e-06, "loss": 0.79156071, "num_input_tokens_seen": 63368835, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.72265625, "step": 2931, "time_per_iteration": 2.4104647636413574 }, { "auxiliary_loss_clip": 0.01098125, "auxiliary_loss_mlp": 0.01036678, "balance_loss_clip": 1.0169245, "balance_loss_mlp": 1.02678323, "epoch": 0.1762813768224861, "flos": 16507597864320.0, "grad_norm": 2.7257495806613976, "language_loss": 0.74576712, "learning_rate": 3.7013695891660985e-06, "loss": 0.76711518, "num_input_tokens_seen": 63385220, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.71484375, "step": 2932, "time_per_iteration": 2.381547212600708 }, { "auxiliary_loss_clip": 0.01104459, "auxiliary_loss_mlp": 0.01046361, "balance_loss_clip": 1.02371168, "balance_loss_mlp": 1.02852345, "epoch": 0.17634150007515406, "flos": 11211239036160.0, "grad_norm": 2.7797515402489887, "language_loss": 0.89398766, "learning_rate": 3.701170946634364e-06, "loss": 0.91549587, "num_input_tokens_seen": 63400865, "router_z_loss_clip": 0.2265625, "router_z_loss_mlp": 0.7578125, "step": 2933, "time_per_iteration": 2.340221643447876 }, { "auxiliary_loss_clip": 0.01096472, "auxiliary_loss_mlp": 0.01042185, "balance_loss_clip": 1.02317154, "balance_loss_mlp": 1.02740037, "epoch": 0.17640162332782203, "flos": 23658787693440.0, "grad_norm": 1.7039241733091834, "language_loss": 0.88141811, "learning_rate": 3.700972243392214e-06, "loss": 0.90280473, "num_input_tokens_seen": 63421390, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.69140625, "step": 2934, "time_per_iteration": 2.44293212890625 }, { "auxiliary_loss_clip": 0.01092239, "auxiliary_loss_mlp": 0.01039757, "balance_loss_clip": 1.02050495, "balance_loss_mlp": 1.0247401, "epoch": 0.17646174658049, "flos": 53795012198400.0, "grad_norm": 1.544438353879266, "language_loss": 0.70650262, "learning_rate": 3.70077347944674e-06, "loss": 0.72782254, "num_input_tokens_seen": 63444715, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.67578125, "step": 2935, "time_per_iteration": 2.655700206756592 }, { "auxiliary_loss_clip": 0.01101648, "auxiliary_loss_mlp": 0.01037841, "balance_loss_clip": 1.01676512, "balance_loss_mlp": 1.02728581, "epoch": 0.17652186983315798, "flos": 24242710049280.0, "grad_norm": 2.695729821185055, "language_loss": 0.70003366, "learning_rate": 3.7005746548050353e-06, "loss": 0.72142857, "num_input_tokens_seen": 63465525, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.7421875, "step": 2936, "time_per_iteration": 2.4301464557647705 }, { "auxiliary_loss_clip": 0.01101024, "auxiliary_loss_mlp": 0.01038122, "balance_loss_clip": 1.01888156, "balance_loss_mlp": 1.03025663, "epoch": 0.17658199308582595, "flos": 27453043653120.0, "grad_norm": 1.802780086834324, "language_loss": 0.71520585, "learning_rate": 3.7003757694741956e-06, "loss": 0.7365973, "num_input_tokens_seen": 63485815, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.70703125, "step": 2937, "time_per_iteration": 3.927194118499756 }, { "auxiliary_loss_clip": 0.01102053, "auxiliary_loss_mlp": 0.01042294, "balance_loss_clip": 1.02105069, "balance_loss_mlp": 1.0282346, "epoch": 0.17664211633849392, "flos": 22417590481920.0, "grad_norm": 4.800873366126143, "language_loss": 0.75749171, "learning_rate": 3.7001768234613188e-06, "loss": 0.77893519, "num_input_tokens_seen": 63503905, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.73828125, "step": 2938, "time_per_iteration": 2.4505906105041504 }, { "auxiliary_loss_clip": 0.01098209, "auxiliary_loss_mlp": 0.0103539, "balance_loss_clip": 1.01573217, "balance_loss_mlp": 1.02652121, "epoch": 0.17670223959116188, "flos": 24714037670400.0, "grad_norm": 2.297893445036306, "language_loss": 0.71310973, "learning_rate": 3.6999778167735043e-06, "loss": 0.73444581, "num_input_tokens_seen": 63521985, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.71875, "step": 2939, "time_per_iteration": 2.422348737716675 }, { "auxiliary_loss_clip": 0.01099561, "auxiliary_loss_mlp": 0.01036472, "balance_loss_clip": 1.01658833, "balance_loss_mlp": 1.02866793, "epoch": 0.17676236284382985, "flos": 22525995853440.0, "grad_norm": 2.369397878630063, "language_loss": 0.73411208, "learning_rate": 3.699778749417855e-06, "loss": 0.75547242, "num_input_tokens_seen": 63539830, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.70703125, "step": 2940, "time_per_iteration": 3.8665661811828613 }, { "auxiliary_loss_clip": 0.01098768, "auxiliary_loss_mlp": 0.01037577, "balance_loss_clip": 1.01682305, "balance_loss_mlp": 1.0269599, "epoch": 0.1768224860964978, "flos": 12384355363200.0, "grad_norm": 2.239458217562189, "language_loss": 0.85782027, "learning_rate": 3.699579621401474e-06, "loss": 0.87918377, "num_input_tokens_seen": 63555495, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.71875, "step": 2941, "time_per_iteration": 3.711132287979126 }, { "auxiliary_loss_clip": 0.01097197, "auxiliary_loss_mlp": 0.01032487, "balance_loss_clip": 1.01241255, "balance_loss_mlp": 1.02669907, "epoch": 0.1768826093491658, "flos": 24352197672960.0, "grad_norm": 2.178196462890618, "language_loss": 0.76568735, "learning_rate": 3.699380432731468e-06, "loss": 0.78698421, "num_input_tokens_seen": 63575290, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.703125, "step": 2942, "time_per_iteration": 3.78407621383667 }, { "auxiliary_loss_clip": 0.01098873, "auxiliary_loss_mlp": 0.01038952, "balance_loss_clip": 1.01731586, "balance_loss_mlp": 1.02701664, "epoch": 0.17694273260183377, "flos": 23585923952640.0, "grad_norm": 3.5547954155900796, "language_loss": 0.79881883, "learning_rate": 3.699181183414946e-06, "loss": 0.82019711, "num_input_tokens_seen": 63594670, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.71875, "step": 2943, "time_per_iteration": 2.390895366668701 }, { "auxiliary_loss_clip": 0.01097381, "auxiliary_loss_mlp": 0.01040082, "balance_loss_clip": 1.01857674, "balance_loss_mlp": 1.02504814, "epoch": 0.17700285585450173, "flos": 26759773319040.0, "grad_norm": 2.4635067207059373, "language_loss": 0.80503607, "learning_rate": 3.698981873459018e-06, "loss": 0.82641065, "num_input_tokens_seen": 63614780, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.72265625, "step": 2944, "time_per_iteration": 2.428388833999634 }, { "auxiliary_loss_clip": 0.01096845, "auxiliary_loss_mlp": 0.01047621, "balance_loss_clip": 1.02808249, "balance_loss_mlp": 1.02602494, "epoch": 0.1770629791071697, "flos": 42774712721280.0, "grad_norm": 2.0907109865020654, "language_loss": 0.73149455, "learning_rate": 3.6987825028707976e-06, "loss": 0.75293922, "num_input_tokens_seen": 63637190, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.70703125, "step": 2945, "time_per_iteration": 2.551279067993164 }, { "auxiliary_loss_clip": 0.0109669, "auxiliary_loss_mlp": 0.01039958, "balance_loss_clip": 1.01960874, "balance_loss_mlp": 1.02730441, "epoch": 0.17712310235983766, "flos": 17344675555200.0, "grad_norm": 2.518853551238311, "language_loss": 0.78102767, "learning_rate": 3.698583071657399e-06, "loss": 0.80239409, "num_input_tokens_seen": 63652140, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.6953125, "step": 2946, "time_per_iteration": 2.349885940551758 }, { "auxiliary_loss_clip": 0.01096569, "auxiliary_loss_mlp": 0.01032906, "balance_loss_clip": 1.01380908, "balance_loss_mlp": 1.0272876, "epoch": 0.17718322561250563, "flos": 23877344004480.0, "grad_norm": 2.926468314024668, "language_loss": 0.76134998, "learning_rate": 3.6983835798259404e-06, "loss": 0.78264475, "num_input_tokens_seen": 63671700, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.69140625, "step": 2947, "time_per_iteration": 2.394010543823242 }, { "auxiliary_loss_clip": 0.01094627, "auxiliary_loss_mlp": 0.01039666, "balance_loss_clip": 1.02019954, "balance_loss_mlp": 1.02523696, "epoch": 0.1772433488651736, "flos": 36464859768960.0, "grad_norm": 3.613830340342464, "language_loss": 0.72849512, "learning_rate": 3.6981840273835405e-06, "loss": 0.74983805, "num_input_tokens_seen": 63691685, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.6953125, "step": 2948, "time_per_iteration": 2.5140676498413086 }, { "auxiliary_loss_clip": 0.01096881, "auxiliary_loss_mlp": 0.01034496, "balance_loss_clip": 1.01445663, "balance_loss_mlp": 1.02745891, "epoch": 0.1773034721178416, "flos": 26683592999040.0, "grad_norm": 1.952034909545662, "language_loss": 0.81700194, "learning_rate": 3.6979844143373207e-06, "loss": 0.83831561, "num_input_tokens_seen": 63711720, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.6953125, "step": 2949, "time_per_iteration": 2.415473461151123 }, { "auxiliary_loss_clip": 0.01034546, "auxiliary_loss_mlp": 0.01004541, "balance_loss_clip": 1.0013696, "balance_loss_mlp": 1.01126313, "epoch": 0.17736359537050955, "flos": 57114377712000.0, "grad_norm": 0.8157807708418564, "language_loss": 0.64964092, "learning_rate": 3.6977847406944053e-06, "loss": 0.67003179, "num_input_tokens_seen": 63776280, "router_z_loss_clip": 0.03173828, "router_z_loss_mlp": 0.23242188, "step": 2950, "time_per_iteration": 3.150360345840454 }, { "auxiliary_loss_clip": 0.01094283, "auxiliary_loss_mlp": 0.01034287, "balance_loss_clip": 1.01507056, "balance_loss_mlp": 1.0265522, "epoch": 0.17742371862317752, "flos": 27196990675200.0, "grad_norm": 1.956804133860281, "language_loss": 0.83536267, "learning_rate": 3.6975850064619193e-06, "loss": 0.85664833, "num_input_tokens_seen": 63797535, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.67578125, "step": 2951, "time_per_iteration": 2.4206326007843018 }, { "auxiliary_loss_clip": 0.01097285, "auxiliary_loss_mlp": 0.01038312, "balance_loss_clip": 1.01787972, "balance_loss_mlp": 1.02587032, "epoch": 0.17748384187584548, "flos": 20958639920640.0, "grad_norm": 3.7777157195705753, "language_loss": 0.80479968, "learning_rate": 3.697385211646991e-06, "loss": 0.82615566, "num_input_tokens_seen": 63817045, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.7109375, "step": 2952, "time_per_iteration": 2.4163801670074463 }, { "auxiliary_loss_clip": 0.01095084, "auxiliary_loss_mlp": 0.01032055, "balance_loss_clip": 1.01189661, "balance_loss_mlp": 1.02589631, "epoch": 0.17754396512851345, "flos": 25008809212800.0, "grad_norm": 8.429013698081665, "language_loss": 0.79238909, "learning_rate": 3.697185356256751e-06, "loss": 0.8136605, "num_input_tokens_seen": 63837665, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.69140625, "step": 2953, "time_per_iteration": 2.409651517868042 }, { "auxiliary_loss_clip": 0.01098027, "auxiliary_loss_mlp": 0.01036523, "balance_loss_clip": 1.01796246, "balance_loss_mlp": 1.02777719, "epoch": 0.1776040883811814, "flos": 32050197216000.0, "grad_norm": 1.8421486948225045, "language_loss": 0.88229394, "learning_rate": 3.6969854402983314e-06, "loss": 0.90363944, "num_input_tokens_seen": 63858455, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.703125, "step": 2954, "time_per_iteration": 2.483719825744629 }, { "auxiliary_loss_clip": 0.01097547, "auxiliary_loss_mlp": 0.01041523, "balance_loss_clip": 1.0208286, "balance_loss_mlp": 1.02728677, "epoch": 0.17766421163384938, "flos": 21573216316800.0, "grad_norm": 2.006047180188921, "language_loss": 0.84834766, "learning_rate": 3.6967854637788665e-06, "loss": 0.86973828, "num_input_tokens_seen": 63876935, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.703125, "step": 2955, "time_per_iteration": 2.379448652267456 }, { "auxiliary_loss_clip": 0.01093515, "auxiliary_loss_mlp": 0.0103567, "balance_loss_clip": 1.01706147, "balance_loss_mlp": 1.02667737, "epoch": 0.17772433488651737, "flos": 22418218886400.0, "grad_norm": 2.3926075165858425, "language_loss": 0.70818555, "learning_rate": 3.696585426705493e-06, "loss": 0.72947741, "num_input_tokens_seen": 63896815, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.66796875, "step": 2956, "time_per_iteration": 2.41111421585083 }, { "auxiliary_loss_clip": 0.0109402, "auxiliary_loss_mlp": 0.01038841, "balance_loss_clip": 1.01956499, "balance_loss_mlp": 1.02558947, "epoch": 0.17778445813918534, "flos": 25628273199360.0, "grad_norm": 1.952670960754375, "language_loss": 0.82171714, "learning_rate": 3.6963853290853503e-06, "loss": 0.84304583, "num_input_tokens_seen": 63916140, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.6875, "step": 2957, "time_per_iteration": 2.4054601192474365 }, { "auxiliary_loss_clip": 0.01093597, "auxiliary_loss_mlp": 0.01034734, "balance_loss_clip": 1.01699567, "balance_loss_mlp": 1.02670491, "epoch": 0.1778445813918533, "flos": 25627714617600.0, "grad_norm": 1.7895656136965656, "language_loss": 0.75002372, "learning_rate": 3.6961851709255784e-06, "loss": 0.77130711, "num_input_tokens_seen": 63935220, "router_z_loss_clip": 0.17773438, "router_z_loss_mlp": 0.66796875, "step": 2958, "time_per_iteration": 2.432297706604004 }, { "auxiliary_loss_clip": 0.010995, "auxiliary_loss_mlp": 0.01033267, "balance_loss_clip": 1.01464653, "balance_loss_mlp": 1.02971029, "epoch": 0.17790470464452127, "flos": 22344447450240.0, "grad_norm": 2.2348646910534926, "language_loss": 0.80148596, "learning_rate": 3.6959849522333206e-06, "loss": 0.82281363, "num_input_tokens_seen": 63954550, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.69921875, "step": 2959, "time_per_iteration": 2.386784076690674 }, { "auxiliary_loss_clip": 0.01096945, "auxiliary_loss_mlp": 0.01039385, "balance_loss_clip": 1.01919079, "balance_loss_mlp": 1.02686608, "epoch": 0.17796482789718923, "flos": 18765012286080.0, "grad_norm": 1.75423547245717, "language_loss": 0.51365209, "learning_rate": 3.6957846730157222e-06, "loss": 0.53501546, "num_input_tokens_seen": 63972425, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.703125, "step": 2960, "time_per_iteration": 2.3954639434814453 }, { "auxiliary_loss_clip": 0.01100382, "auxiliary_loss_mlp": 0.01044366, "balance_loss_clip": 1.02450538, "balance_loss_mlp": 1.02857542, "epoch": 0.1780249511498572, "flos": 23439812446080.0, "grad_norm": 1.9753834197272402, "language_loss": 0.88879579, "learning_rate": 3.6955843332799317e-06, "loss": 0.91024327, "num_input_tokens_seen": 63992165, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.71875, "step": 2961, "time_per_iteration": 2.3947105407714844 }, { "auxiliary_loss_clip": 0.01097315, "auxiliary_loss_mlp": 0.01043823, "balance_loss_clip": 1.02297306, "balance_loss_mlp": 1.02615452, "epoch": 0.1780850744025252, "flos": 23366355212160.0, "grad_norm": 1.6960386384330346, "language_loss": 0.79236126, "learning_rate": 3.6953839330330972e-06, "loss": 0.81377268, "num_input_tokens_seen": 64013470, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7109375, "step": 2962, "time_per_iteration": 2.4387855529785156 }, { "auxiliary_loss_clip": 0.01099002, "auxiliary_loss_mlp": 0.01040503, "balance_loss_clip": 1.02032113, "balance_loss_mlp": 1.02963376, "epoch": 0.17814519765519315, "flos": 13771140410880.0, "grad_norm": 2.200560901801987, "language_loss": 0.74530143, "learning_rate": 3.6951834722823715e-06, "loss": 0.76669645, "num_input_tokens_seen": 64030975, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.6953125, "step": 2963, "time_per_iteration": 2.357468366622925 }, { "auxiliary_loss_clip": 0.01098758, "auxiliary_loss_mlp": 0.010387, "balance_loss_clip": 1.01862502, "balance_loss_mlp": 1.0284189, "epoch": 0.17820532090786112, "flos": 21975450624000.0, "grad_norm": 1.658869639699114, "language_loss": 0.78876424, "learning_rate": 3.6949829510349082e-06, "loss": 0.81013888, "num_input_tokens_seen": 64050075, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.703125, "step": 2964, "time_per_iteration": 2.448072671890259 }, { "auxiliary_loss_clip": 0.01097214, "auxiliary_loss_mlp": 0.01040353, "balance_loss_clip": 1.02223301, "balance_loss_mlp": 1.02902925, "epoch": 0.17826544416052909, "flos": 24789589585920.0, "grad_norm": 2.460196336053275, "language_loss": 0.80767095, "learning_rate": 3.6947823692978634e-06, "loss": 0.82904661, "num_input_tokens_seen": 64071920, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.6796875, "step": 2965, "time_per_iteration": 2.414928436279297 }, { "auxiliary_loss_clip": 0.01095953, "auxiliary_loss_mlp": 0.0104086, "balance_loss_clip": 1.02256083, "balance_loss_mlp": 1.02708244, "epoch": 0.17832556741319705, "flos": 13878777732480.0, "grad_norm": 2.4380853617515474, "language_loss": 0.94539225, "learning_rate": 3.6945817270783955e-06, "loss": 0.9667604, "num_input_tokens_seen": 64086835, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.6875, "step": 2966, "time_per_iteration": 2.3657639026641846 }, { "auxiliary_loss_clip": 0.01095675, "auxiliary_loss_mlp": 0.01038397, "balance_loss_clip": 1.01912105, "balance_loss_mlp": 1.02737188, "epoch": 0.17838569066586502, "flos": 36640403418240.0, "grad_norm": 2.6088065706095853, "language_loss": 0.72639889, "learning_rate": 3.6943810243836648e-06, "loss": 0.74773961, "num_input_tokens_seen": 64107360, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.68359375, "step": 2967, "time_per_iteration": 2.495950222015381 }, { "auxiliary_loss_clip": 0.01092959, "auxiliary_loss_mlp": 0.01038678, "balance_loss_clip": 1.01980734, "balance_loss_mlp": 1.02761495, "epoch": 0.17844581391853298, "flos": 18726468278400.0, "grad_norm": 2.019554405112788, "language_loss": 0.77192456, "learning_rate": 3.6941802612208334e-06, "loss": 0.7932409, "num_input_tokens_seen": 64124690, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.65625, "step": 2968, "time_per_iteration": 2.385192632675171 }, { "auxiliary_loss_clip": 0.01096339, "auxiliary_loss_mlp": 0.01039807, "balance_loss_clip": 1.02190185, "balance_loss_mlp": 1.02813447, "epoch": 0.17850593717120097, "flos": 27377107712640.0, "grad_norm": 2.374990649800464, "language_loss": 0.75913197, "learning_rate": 3.6939794375970667e-06, "loss": 0.78049338, "num_input_tokens_seen": 64146315, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.68359375, "step": 2969, "time_per_iteration": 2.4243085384368896 }, { "auxiliary_loss_clip": 0.01040425, "auxiliary_loss_mlp": 0.01015848, "balance_loss_clip": 1.01160467, "balance_loss_mlp": 1.01369655, "epoch": 0.17856606042386894, "flos": 66992913129600.0, "grad_norm": 0.839377654031634, "language_loss": 0.69052625, "learning_rate": 3.693778553519531e-06, "loss": 0.71108902, "num_input_tokens_seen": 64210875, "router_z_loss_clip": 0.04248047, "router_z_loss_mlp": 0.26757812, "step": 2970, "time_per_iteration": 3.1388394832611084 }, { "auxiliary_loss_clip": 0.01099696, "auxiliary_loss_mlp": 0.01035852, "balance_loss_clip": 1.01663566, "balance_loss_mlp": 1.02782619, "epoch": 0.1786261836765369, "flos": 36975499447680.0, "grad_norm": 1.832802303528834, "language_loss": 0.67340553, "learning_rate": 3.6935776089953956e-06, "loss": 0.69476104, "num_input_tokens_seen": 64230740, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.71875, "step": 2971, "time_per_iteration": 2.5004563331604004 }, { "auxiliary_loss_clip": 0.01095868, "auxiliary_loss_mlp": 0.01039161, "balance_loss_clip": 1.01888382, "balance_loss_mlp": 1.02606821, "epoch": 0.17868630692920487, "flos": 24824328255360.0, "grad_norm": 1.695710173332243, "language_loss": 0.89951611, "learning_rate": 3.6933766040318323e-06, "loss": 0.92086643, "num_input_tokens_seen": 64252300, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.6953125, "step": 2972, "time_per_iteration": 2.4530422687530518 }, { "auxiliary_loss_clip": 0.01098219, "auxiliary_loss_mlp": 0.010454, "balance_loss_clip": 1.02532518, "balance_loss_mlp": 1.02798152, "epoch": 0.17874643018187283, "flos": 16981055078400.0, "grad_norm": 2.9073345714967735, "language_loss": 0.87565172, "learning_rate": 3.693175538636014e-06, "loss": 0.89708793, "num_input_tokens_seen": 64270105, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.703125, "step": 2973, "time_per_iteration": 2.346275806427002 }, { "auxiliary_loss_clip": 0.01095117, "auxiliary_loss_mlp": 0.01043492, "balance_loss_clip": 1.02205861, "balance_loss_mlp": 1.02672338, "epoch": 0.1788065534345408, "flos": 21031189459200.0, "grad_norm": 2.6152529641254287, "language_loss": 0.76249814, "learning_rate": 3.692974412815116e-06, "loss": 0.78388429, "num_input_tokens_seen": 64287250, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.68359375, "step": 2974, "time_per_iteration": 2.4068634510040283 }, { "auxiliary_loss_clip": 0.01097236, "auxiliary_loss_mlp": 0.01038095, "balance_loss_clip": 1.01747215, "balance_loss_mlp": 1.02757478, "epoch": 0.17886667668720876, "flos": 23986587248640.0, "grad_norm": 2.789210916353887, "language_loss": 0.74411094, "learning_rate": 3.692773226576315e-06, "loss": 0.76546419, "num_input_tokens_seen": 64307140, "router_z_loss_clip": 0.20605469, "router_z_loss_mlp": 0.6953125, "step": 2975, "time_per_iteration": 2.387455940246582 }, { "auxiliary_loss_clip": 0.01095074, "auxiliary_loss_mlp": 0.01033124, "balance_loss_clip": 1.01447964, "balance_loss_mlp": 1.02747369, "epoch": 0.17892679993987676, "flos": 25738284493440.0, "grad_norm": 1.5661051401773733, "language_loss": 0.72881365, "learning_rate": 3.692571979926793e-06, "loss": 0.75009561, "num_input_tokens_seen": 64328760, "router_z_loss_clip": 0.18652344, "router_z_loss_mlp": 0.67578125, "step": 2976, "time_per_iteration": 2.446561098098755 }, { "auxiliary_loss_clip": 0.01091909, "auxiliary_loss_mlp": 0.01031705, "balance_loss_clip": 1.0149678, "balance_loss_mlp": 1.02750278, "epoch": 0.17898692319254472, "flos": 25698588410880.0, "grad_norm": 1.5147232149525842, "language_loss": 0.77300251, "learning_rate": 3.69237067287373e-06, "loss": 0.79423863, "num_input_tokens_seen": 64348800, "router_z_loss_clip": 0.16699219, "router_z_loss_mlp": 0.64453125, "step": 2977, "time_per_iteration": 3.884591579437256 }, { "auxiliary_loss_clip": 0.01098059, "auxiliary_loss_mlp": 0.01046172, "balance_loss_clip": 1.0268724, "balance_loss_mlp": 1.03055906, "epoch": 0.1790470464452127, "flos": 19316779413120.0, "grad_norm": 2.1145517949177695, "language_loss": 0.79672265, "learning_rate": 3.6921693054243118e-06, "loss": 0.81816506, "num_input_tokens_seen": 64367955, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.67578125, "step": 2978, "time_per_iteration": 2.462244987487793 }, { "auxiliary_loss_clip": 0.01098729, "auxiliary_loss_mlp": 0.01035956, "balance_loss_clip": 1.01608396, "balance_loss_mlp": 1.02767551, "epoch": 0.17910716969788065, "flos": 30042970663680.0, "grad_norm": 1.655474119658755, "language_loss": 0.76386064, "learning_rate": 3.6919678775857235e-06, "loss": 0.78520751, "num_input_tokens_seen": 64389805, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.7109375, "step": 2979, "time_per_iteration": 2.4736757278442383 }, { "auxiliary_loss_clip": 0.01095602, "auxiliary_loss_mlp": 0.01034779, "balance_loss_clip": 1.01668274, "balance_loss_mlp": 1.02867532, "epoch": 0.17916729295054862, "flos": 19426685973120.0, "grad_norm": 1.987467858622668, "language_loss": 0.68844259, "learning_rate": 3.691766389365154e-06, "loss": 0.70974636, "num_input_tokens_seen": 64408220, "router_z_loss_clip": 0.18066406, "router_z_loss_mlp": 0.671875, "step": 2980, "time_per_iteration": 5.1556713581085205 }, { "auxiliary_loss_clip": 0.01100295, "auxiliary_loss_mlp": 0.01039066, "balance_loss_clip": 1.01813245, "balance_loss_mlp": 1.02987719, "epoch": 0.17922741620321658, "flos": 14610661896960.0, "grad_norm": 1.717844595830466, "language_loss": 0.70527929, "learning_rate": 3.6915648407697936e-06, "loss": 0.72667289, "num_input_tokens_seen": 64426380, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.703125, "step": 2981, "time_per_iteration": 2.354583501815796 }, { "auxiliary_loss_clip": 0.01099471, "auxiliary_loss_mlp": 0.01046799, "balance_loss_clip": 1.02537692, "balance_loss_mlp": 1.02893233, "epoch": 0.17928753945588458, "flos": 17164349049600.0, "grad_norm": 2.3933114345657147, "language_loss": 0.81727308, "learning_rate": 3.691363231806836e-06, "loss": 0.83873576, "num_input_tokens_seen": 64444355, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.703125, "step": 2982, "time_per_iteration": 3.780759811401367 }, { "auxiliary_loss_clip": 0.01095086, "auxiliary_loss_mlp": 0.01034561, "balance_loss_clip": 1.01578546, "balance_loss_mlp": 1.02700841, "epoch": 0.17934766270855254, "flos": 31394248992000.0, "grad_norm": 1.502508796775173, "language_loss": 0.8268553, "learning_rate": 3.691161562483474e-06, "loss": 0.84815174, "num_input_tokens_seen": 64467800, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.6796875, "step": 2983, "time_per_iteration": 2.4457900524139404 }, { "auxiliary_loss_clip": 0.01097371, "auxiliary_loss_mlp": 0.01040516, "balance_loss_clip": 1.01946414, "balance_loss_mlp": 1.02622414, "epoch": 0.1794077859612205, "flos": 20813121907200.0, "grad_norm": 1.9901995182426722, "language_loss": 0.8515988, "learning_rate": 3.690959832806907e-06, "loss": 0.87297773, "num_input_tokens_seen": 64487230, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.7109375, "step": 2984, "time_per_iteration": 2.4008259773254395 }, { "auxiliary_loss_clip": 0.01096673, "auxiliary_loss_mlp": 0.01039458, "balance_loss_clip": 1.01822722, "balance_loss_mlp": 1.02621925, "epoch": 0.17946790921388847, "flos": 28985172157440.0, "grad_norm": 1.3482576795547023, "language_loss": 0.89483905, "learning_rate": 3.690758042784333e-06, "loss": 0.9162004, "num_input_tokens_seen": 64509165, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.70703125, "step": 2985, "time_per_iteration": 2.4310142993927 }, { "auxiliary_loss_clip": 0.0109346, "auxiliary_loss_mlp": 0.01037514, "balance_loss_clip": 1.01982343, "balance_loss_mlp": 1.02734017, "epoch": 0.17952803246655644, "flos": 20736452828160.0, "grad_norm": 1.9732589629646102, "language_loss": 0.69493186, "learning_rate": 3.690556192422954e-06, "loss": 0.7162416, "num_input_tokens_seen": 64527940, "router_z_loss_clip": 0.17675781, "router_z_loss_mlp": 0.66015625, "step": 2986, "time_per_iteration": 2.4057769775390625 }, { "auxiliary_loss_clip": 0.01093622, "auxiliary_loss_mlp": 0.0103883, "balance_loss_clip": 1.01949382, "balance_loss_mlp": 1.02628994, "epoch": 0.1795881557192244, "flos": 28254754270080.0, "grad_norm": 2.1704732837980933, "language_loss": 0.77198172, "learning_rate": 3.6903542817299725e-06, "loss": 0.79330623, "num_input_tokens_seen": 64545230, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.671875, "step": 2987, "time_per_iteration": 2.412198066711426 }, { "auxiliary_loss_clip": 0.01098981, "auxiliary_loss_mlp": 0.01040336, "balance_loss_clip": 1.01951039, "balance_loss_mlp": 1.0278964, "epoch": 0.17964827897189237, "flos": 18551029363200.0, "grad_norm": 1.8687050131293323, "language_loss": 0.77970552, "learning_rate": 3.690152310712595e-06, "loss": 0.80109864, "num_input_tokens_seen": 64563820, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7109375, "step": 2988, "time_per_iteration": 2.426103353500366 }, { "auxiliary_loss_clip": 0.01027175, "auxiliary_loss_mlp": 0.01007762, "balance_loss_clip": 1.00511575, "balance_loss_mlp": 1.006791, "epoch": 0.17970840222456036, "flos": 58162261392000.0, "grad_norm": 0.7673995150315861, "language_loss": 0.62703419, "learning_rate": 3.6899502793780295e-06, "loss": 0.64738357, "num_input_tokens_seen": 64621315, "router_z_loss_clip": 0.02648926, "router_z_loss_mlp": 0.203125, "step": 2989, "time_per_iteration": 2.9614171981811523 }, { "auxiliary_loss_clip": 0.01096796, "auxiliary_loss_mlp": 0.01032886, "balance_loss_clip": 1.01431358, "balance_loss_mlp": 1.02748251, "epoch": 0.17976852547722832, "flos": 20299828965120.0, "grad_norm": 2.5947417124638914, "language_loss": 0.70792025, "learning_rate": 3.689748187733485e-06, "loss": 0.72921705, "num_input_tokens_seen": 64639885, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.69140625, "step": 2990, "time_per_iteration": 2.399005889892578 }, { "auxiliary_loss_clip": 0.01096707, "auxiliary_loss_mlp": 0.01041279, "balance_loss_clip": 1.02242041, "balance_loss_mlp": 1.02798045, "epoch": 0.1798286487298963, "flos": 39668001632640.0, "grad_norm": 1.794322181293968, "language_loss": 0.68833303, "learning_rate": 3.6895460357861743e-06, "loss": 0.70971286, "num_input_tokens_seen": 64661220, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.6875, "step": 2991, "time_per_iteration": 2.546729803085327 }, { "auxiliary_loss_clip": 0.01095533, "auxiliary_loss_mlp": 0.01036174, "balance_loss_clip": 1.01694548, "balance_loss_mlp": 1.02731848, "epoch": 0.17988877198256426, "flos": 25519134689280.0, "grad_norm": 1.9749019486282824, "language_loss": 0.83044302, "learning_rate": 3.6893438235433117e-06, "loss": 0.85176003, "num_input_tokens_seen": 64682530, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.6796875, "step": 2992, "time_per_iteration": 2.4183197021484375 }, { "auxiliary_loss_clip": 0.01093847, "auxiliary_loss_mlp": 0.01034821, "balance_loss_clip": 1.01723754, "balance_loss_mlp": 1.02816367, "epoch": 0.17994889523523222, "flos": 18806488848000.0, "grad_norm": 2.2885754724046654, "language_loss": 0.81842172, "learning_rate": 3.689141551012114e-06, "loss": 0.83970839, "num_input_tokens_seen": 64701025, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.65625, "step": 2993, "time_per_iteration": 2.3537867069244385 }, { "auxiliary_loss_clip": 0.01094229, "auxiliary_loss_mlp": 0.01031966, "balance_loss_clip": 1.01268983, "balance_loss_mlp": 1.02611601, "epoch": 0.18000901848790019, "flos": 21103424795520.0, "grad_norm": 1.8544353092325712, "language_loss": 0.78179508, "learning_rate": 3.688939218199799e-06, "loss": 0.80305707, "num_input_tokens_seen": 64719570, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.68359375, "step": 2994, "time_per_iteration": 2.401881456375122 }, { "auxiliary_loss_clip": 0.01097756, "auxiliary_loss_mlp": 0.0103543, "balance_loss_clip": 1.01721501, "balance_loss_mlp": 1.02783489, "epoch": 0.18006914174056818, "flos": 19645416840960.0, "grad_norm": 2.2977123948197695, "language_loss": 0.80790877, "learning_rate": 3.6887368251135875e-06, "loss": 0.82924068, "num_input_tokens_seen": 64738110, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.69921875, "step": 2995, "time_per_iteration": 2.355602741241455 }, { "auxiliary_loss_clip": 0.01096986, "auxiliary_loss_mlp": 0.01036714, "balance_loss_clip": 1.01842737, "balance_loss_mlp": 1.02779651, "epoch": 0.18012926499323614, "flos": 19498886398080.0, "grad_norm": 2.004213944979025, "language_loss": 0.84364128, "learning_rate": 3.688534371760703e-06, "loss": 0.86497831, "num_input_tokens_seen": 64756345, "router_z_loss_clip": 0.18261719, "router_z_loss_mlp": 0.69140625, "step": 2996, "time_per_iteration": 2.3671207427978516 }, { "auxiliary_loss_clip": 0.01091169, "auxiliary_loss_mlp": 0.01036037, "balance_loss_clip": 1.01834655, "balance_loss_mlp": 1.02571321, "epoch": 0.1801893882459041, "flos": 19463519324160.0, "grad_norm": 1.8246987670289778, "language_loss": 0.88096237, "learning_rate": 3.68833185814837e-06, "loss": 0.90223432, "num_input_tokens_seen": 64776375, "router_z_loss_clip": 0.17675781, "router_z_loss_mlp": 0.65625, "step": 2997, "time_per_iteration": 2.428534507751465 }, { "auxiliary_loss_clip": 0.01098237, "auxiliary_loss_mlp": 0.01043539, "balance_loss_clip": 1.02237988, "balance_loss_mlp": 1.0258584, "epoch": 0.18024951149857207, "flos": 26869365676800.0, "grad_norm": 1.749073381435927, "language_loss": 0.85452026, "learning_rate": 3.688129284283816e-06, "loss": 0.875938, "num_input_tokens_seen": 64796210, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.72265625, "step": 2998, "time_per_iteration": 2.436452627182007 }, { "auxiliary_loss_clip": 0.01097172, "auxiliary_loss_mlp": 0.01042816, "balance_loss_clip": 1.02375484, "balance_loss_mlp": 1.02929878, "epoch": 0.18030963475124004, "flos": 30225322028160.0, "grad_norm": 1.898293642111629, "language_loss": 0.84303552, "learning_rate": 3.6879266501742705e-06, "loss": 0.86443543, "num_input_tokens_seen": 64818590, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.6796875, "step": 2999, "time_per_iteration": 2.440263032913208 }, { "auxiliary_loss_clip": 0.01094612, "auxiliary_loss_mlp": 0.01039983, "balance_loss_clip": 1.0202781, "balance_loss_mlp": 1.0264858, "epoch": 0.180369758003908, "flos": 22306462024320.0, "grad_norm": 1.7817234409355363, "language_loss": 0.74977803, "learning_rate": 3.6877239558269642e-06, "loss": 0.77112401, "num_input_tokens_seen": 64838350, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.6796875, "step": 3000, "time_per_iteration": 2.400106191635132 }, { "auxiliary_loss_clip": 0.01094685, "auxiliary_loss_mlp": 0.01045176, "balance_loss_clip": 1.02609098, "balance_loss_mlp": 1.02801108, "epoch": 0.18042988125657597, "flos": 23730918295680.0, "grad_norm": 1.6914341462004998, "language_loss": 0.7138685, "learning_rate": 3.687521201249132e-06, "loss": 0.73526716, "num_input_tokens_seen": 64858065, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.66796875, "step": 3001, "time_per_iteration": 2.4026381969451904 }, { "auxiliary_loss_clip": 0.01097361, "auxiliary_loss_mlp": 0.01037554, "balance_loss_clip": 1.01790857, "balance_loss_mlp": 1.02772808, "epoch": 0.18049000450924396, "flos": 24092548824960.0, "grad_norm": 1.9922244182544901, "language_loss": 0.88416296, "learning_rate": 3.687318386448008e-06, "loss": 0.90551209, "num_input_tokens_seen": 64877305, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.6953125, "step": 3002, "time_per_iteration": 2.4055991172790527 }, { "auxiliary_loss_clip": 0.01094645, "auxiliary_loss_mlp": 0.01039409, "balance_loss_clip": 1.0210861, "balance_loss_mlp": 1.02737689, "epoch": 0.18055012776191193, "flos": 22162096085760.0, "grad_norm": 1.8975920308571603, "language_loss": 0.80576307, "learning_rate": 3.687115511430832e-06, "loss": 0.82710361, "num_input_tokens_seen": 64896955, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.671875, "step": 3003, "time_per_iteration": 2.3824965953826904 }, { "auxiliary_loss_clip": 0.01095178, "auxiliary_loss_mlp": 0.01038835, "balance_loss_clip": 1.01905823, "balance_loss_mlp": 1.02607787, "epoch": 0.1806102510145799, "flos": 28912238593920.0, "grad_norm": 2.2792963212946056, "language_loss": 0.66961324, "learning_rate": 3.6869125762048423e-06, "loss": 0.69095337, "num_input_tokens_seen": 64917080, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.69140625, "step": 3004, "time_per_iteration": 2.422297239303589 }, { "auxiliary_loss_clip": 0.01097773, "auxiliary_loss_mlp": 0.01040511, "balance_loss_clip": 1.02108049, "balance_loss_mlp": 1.02752137, "epoch": 0.18067037426724786, "flos": 19024696045440.0, "grad_norm": 1.6577558114338158, "language_loss": 0.85528255, "learning_rate": 3.6867095807772826e-06, "loss": 0.87666535, "num_input_tokens_seen": 64935215, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.703125, "step": 3005, "time_per_iteration": 2.3702433109283447 }, { "auxiliary_loss_clip": 0.01092354, "auxiliary_loss_mlp": 0.01038303, "balance_loss_clip": 1.01965904, "balance_loss_mlp": 1.02653646, "epoch": 0.18073049751991582, "flos": 27452415248640.0, "grad_norm": 1.545663060047751, "language_loss": 0.8309502, "learning_rate": 3.6865065251553967e-06, "loss": 0.85225677, "num_input_tokens_seen": 64956275, "router_z_loss_clip": 0.18652344, "router_z_loss_mlp": 0.65625, "step": 3006, "time_per_iteration": 2.427565574645996 }, { "auxiliary_loss_clip": 0.01092904, "auxiliary_loss_mlp": 0.01038936, "balance_loss_clip": 1.0199697, "balance_loss_mlp": 1.02515292, "epoch": 0.1807906207725838, "flos": 28727827459200.0, "grad_norm": 1.6693494832060916, "language_loss": 0.77110308, "learning_rate": 3.6863034093464307e-06, "loss": 0.79242146, "num_input_tokens_seen": 64979390, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.6796875, "step": 3007, "time_per_iteration": 2.4451706409454346 }, { "auxiliary_loss_clip": 0.0102929, "auxiliary_loss_mlp": 0.01018946, "balance_loss_clip": 1.01641917, "balance_loss_mlp": 1.00968313, "epoch": 0.18085074402525175, "flos": 64462791144960.0, "grad_norm": 0.7976309932500728, "language_loss": 0.56929284, "learning_rate": 3.686100233357634e-06, "loss": 0.5897752, "num_input_tokens_seen": 65043135, "router_z_loss_clip": 0.02526855, "router_z_loss_mlp": 0.19628906, "step": 3008, "time_per_iteration": 3.161513328552246 }, { "auxiliary_loss_clip": 0.01097169, "auxiliary_loss_mlp": 0.01044263, "balance_loss_clip": 1.02424729, "balance_loss_mlp": 1.02932644, "epoch": 0.18091086727791975, "flos": 23475842835840.0, "grad_norm": 1.9043645543070598, "language_loss": 0.67481375, "learning_rate": 3.6858969971962573e-06, "loss": 0.69622803, "num_input_tokens_seen": 65062845, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.6796875, "step": 3009, "time_per_iteration": 2.3931095600128174 }, { "auxiliary_loss_clip": 0.01096304, "auxiliary_loss_mlp": 0.01034655, "balance_loss_clip": 1.01597512, "balance_loss_mlp": 1.02767563, "epoch": 0.1809709905305877, "flos": 24169322638080.0, "grad_norm": 2.562430365773123, "language_loss": 0.75639015, "learning_rate": 3.685693700869553e-06, "loss": 0.77769971, "num_input_tokens_seen": 65082110, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.6875, "step": 3010, "time_per_iteration": 2.4174158573150635 }, { "auxiliary_loss_clip": 0.01089004, "auxiliary_loss_mlp": 0.01034499, "balance_loss_clip": 1.0177027, "balance_loss_mlp": 1.02533138, "epoch": 0.18103111378325568, "flos": 21649885395840.0, "grad_norm": 1.5937217071721066, "language_loss": 0.67342001, "learning_rate": 3.6854903443847772e-06, "loss": 0.69465506, "num_input_tokens_seen": 65101985, "router_z_loss_clip": 0.16796875, "router_z_loss_mlp": 0.63671875, "step": 3011, "time_per_iteration": 2.4023966789245605 }, { "auxiliary_loss_clip": 0.0109164, "auxiliary_loss_mlp": 0.01035591, "balance_loss_clip": 1.01705384, "balance_loss_mlp": 1.02585912, "epoch": 0.18109123703592364, "flos": 53684965992960.0, "grad_norm": 1.7738741352484462, "language_loss": 0.71349472, "learning_rate": 3.6852869277491865e-06, "loss": 0.73476702, "num_input_tokens_seen": 65129295, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.66015625, "step": 3012, "time_per_iteration": 2.692625045776367 }, { "auxiliary_loss_clip": 0.01093415, "auxiliary_loss_mlp": 0.01038487, "balance_loss_clip": 1.02000999, "balance_loss_mlp": 1.02846265, "epoch": 0.1811513602885916, "flos": 35844104062080.0, "grad_norm": 2.037454486184128, "language_loss": 0.63142848, "learning_rate": 3.68508345097004e-06, "loss": 0.65274751, "num_input_tokens_seen": 65150625, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.6484375, "step": 3013, "time_per_iteration": 2.505274772644043 }, { "auxiliary_loss_clip": 0.01097457, "auxiliary_loss_mlp": 0.01043572, "balance_loss_clip": 1.02592874, "balance_loss_mlp": 1.02976298, "epoch": 0.18121148354125957, "flos": 23731441966080.0, "grad_norm": 1.6507094366992145, "language_loss": 0.76124537, "learning_rate": 3.6848799140546e-06, "loss": 0.78265566, "num_input_tokens_seen": 65170880, "router_z_loss_clip": 0.17675781, "router_z_loss_mlp": 0.6796875, "step": 3014, "time_per_iteration": 2.4147589206695557 }, { "auxiliary_loss_clip": 0.01098058, "auxiliary_loss_mlp": 0.01042203, "balance_loss_clip": 1.0211271, "balance_loss_mlp": 1.02832627, "epoch": 0.18127160679392756, "flos": 28727129232000.0, "grad_norm": 2.1835324373493568, "language_loss": 0.66143107, "learning_rate": 3.6846763170101297e-06, "loss": 0.68283367, "num_input_tokens_seen": 65192530, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.69921875, "step": 3015, "time_per_iteration": 2.45542573928833 }, { "auxiliary_loss_clip": 0.01092742, "auxiliary_loss_mlp": 0.01035633, "balance_loss_clip": 1.01674986, "balance_loss_mlp": 1.02746844, "epoch": 0.18133173004659553, "flos": 20484030631680.0, "grad_norm": 1.6975769583465765, "language_loss": 0.7801252, "learning_rate": 3.684472659843895e-06, "loss": 0.80140895, "num_input_tokens_seen": 65211675, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.6484375, "step": 3016, "time_per_iteration": 2.3770716190338135 }, { "auxiliary_loss_clip": 0.01098174, "auxiliary_loss_mlp": 0.01038127, "balance_loss_clip": 1.0191493, "balance_loss_mlp": 1.02951336, "epoch": 0.1813918532992635, "flos": 22851107233920.0, "grad_norm": 1.6992091223276173, "language_loss": 0.83647573, "learning_rate": 3.6842689425631645e-06, "loss": 0.85783875, "num_input_tokens_seen": 65231185, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.6875, "step": 3017, "time_per_iteration": 3.90226411819458 }, { "auxiliary_loss_clip": 0.01091733, "auxiliary_loss_mlp": 0.01035836, "balance_loss_clip": 1.01820517, "balance_loss_mlp": 1.0266006, "epoch": 0.18145197655193146, "flos": 36063637891200.0, "grad_norm": 4.773021443468642, "language_loss": 0.67354894, "learning_rate": 3.684065165175208e-06, "loss": 0.69482458, "num_input_tokens_seen": 65251645, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.6484375, "step": 3018, "time_per_iteration": 2.5068161487579346 }, { "auxiliary_loss_clip": 0.01095457, "auxiliary_loss_mlp": 0.01034266, "balance_loss_clip": 1.01630139, "balance_loss_mlp": 1.0276463, "epoch": 0.18151209980459942, "flos": 24022827106560.0, "grad_norm": 1.9264538479523363, "language_loss": 0.75988364, "learning_rate": 3.683861327687297e-06, "loss": 0.78118086, "num_input_tokens_seen": 65271125, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.67578125, "step": 3019, "time_per_iteration": 3.7964046001434326 }, { "auxiliary_loss_clip": 0.01097, "auxiliary_loss_mlp": 0.01035523, "balance_loss_clip": 1.01623487, "balance_loss_mlp": 1.02833366, "epoch": 0.1815722230572674, "flos": 23950487036160.0, "grad_norm": 2.1930430614750605, "language_loss": 0.81399328, "learning_rate": 3.683657430106707e-06, "loss": 0.83531857, "num_input_tokens_seen": 65290600, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.6875, "step": 3020, "time_per_iteration": 3.7735393047332764 }, { "auxiliary_loss_clip": 0.0109454, "auxiliary_loss_mlp": 0.01037535, "balance_loss_clip": 1.01861608, "balance_loss_mlp": 1.02751517, "epoch": 0.18163234630993536, "flos": 24385400242560.0, "grad_norm": 1.7764036231911737, "language_loss": 0.77370763, "learning_rate": 3.683453472440714e-06, "loss": 0.79502845, "num_input_tokens_seen": 65311040, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.671875, "step": 3021, "time_per_iteration": 2.4085967540740967 }, { "auxiliary_loss_clip": 0.01090358, "auxiliary_loss_mlp": 0.010367, "balance_loss_clip": 1.01873493, "balance_loss_mlp": 1.02425432, "epoch": 0.18169246956260335, "flos": 24680171784960.0, "grad_norm": 1.6819576893612795, "language_loss": 0.84895861, "learning_rate": 3.6832494546965975e-06, "loss": 0.87022913, "num_input_tokens_seen": 65332115, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.66015625, "step": 3022, "time_per_iteration": 3.787031888961792 }, { "auxiliary_loss_clip": 0.01093793, "auxiliary_loss_mlp": 0.01037292, "balance_loss_clip": 1.01830173, "balance_loss_mlp": 1.02538633, "epoch": 0.1817525928152713, "flos": 24242151467520.0, "grad_norm": 1.8081751746641312, "language_loss": 0.69382024, "learning_rate": 3.6830453768816376e-06, "loss": 0.71513104, "num_input_tokens_seen": 65352210, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.68359375, "step": 3023, "time_per_iteration": 2.3976800441741943 }, { "auxiliary_loss_clip": 0.0109341, "auxiliary_loss_mlp": 0.01036629, "balance_loss_clip": 1.01902175, "balance_loss_mlp": 1.02653098, "epoch": 0.18181271606793928, "flos": 16471148538240.0, "grad_norm": 1.8344109458778812, "language_loss": 0.73903406, "learning_rate": 3.6828412390031174e-06, "loss": 0.76033449, "num_input_tokens_seen": 65370600, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.66796875, "step": 3024, "time_per_iteration": 2.3516170978546143 }, { "auxiliary_loss_clip": 0.01094789, "auxiliary_loss_mlp": 0.0103464, "balance_loss_clip": 1.01550698, "balance_loss_mlp": 1.02728665, "epoch": 0.18187283932060724, "flos": 18580252037760.0, "grad_norm": 1.9590317436915126, "language_loss": 0.8824296, "learning_rate": 3.682637041068322e-06, "loss": 0.90372384, "num_input_tokens_seen": 65387270, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.67578125, "step": 3025, "time_per_iteration": 2.3552322387695312 }, { "auxiliary_loss_clip": 0.01093535, "auxiliary_loss_mlp": 0.01035689, "balance_loss_clip": 1.01683044, "balance_loss_mlp": 1.02792573, "epoch": 0.1819329625732752, "flos": 20265788522880.0, "grad_norm": 1.6937359134732468, "language_loss": 0.78706336, "learning_rate": 3.6824327830845387e-06, "loss": 0.80835557, "num_input_tokens_seen": 65406550, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.65625, "step": 3026, "time_per_iteration": 2.3805599212646484 }, { "auxiliary_loss_clip": 0.01095746, "auxiliary_loss_mlp": 0.01039595, "balance_loss_clip": 1.02073646, "balance_loss_mlp": 1.02767563, "epoch": 0.18199308582594317, "flos": 25914177256320.0, "grad_norm": 1.6651142140429498, "language_loss": 0.75892401, "learning_rate": 3.6822284650590576e-06, "loss": 0.78027743, "num_input_tokens_seen": 65425955, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.6796875, "step": 3027, "time_per_iteration": 2.4328739643096924 }, { "auxiliary_loss_clip": 0.01097257, "auxiliary_loss_mlp": 0.01039279, "balance_loss_clip": 1.01903725, "balance_loss_mlp": 1.02633667, "epoch": 0.18205320907861114, "flos": 15376621415040.0, "grad_norm": 1.9379751917848766, "language_loss": 0.85705507, "learning_rate": 3.68202408699917e-06, "loss": 0.87842047, "num_input_tokens_seen": 65442820, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.7109375, "step": 3028, "time_per_iteration": 2.361747980117798 }, { "auxiliary_loss_clip": 0.01092009, "auxiliary_loss_mlp": 0.01036923, "balance_loss_clip": 1.01812363, "balance_loss_mlp": 1.02600992, "epoch": 0.18211333233127913, "flos": 25623280874880.0, "grad_norm": 1.9534451950594895, "language_loss": 0.82559109, "learning_rate": 3.6818196489121683e-06, "loss": 0.84688038, "num_input_tokens_seen": 65461825, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.66015625, "step": 3029, "time_per_iteration": 2.4177732467651367 }, { "auxiliary_loss_clip": 0.010948, "auxiliary_loss_mlp": 0.01039609, "balance_loss_clip": 1.02011859, "balance_loss_mlp": 1.02715087, "epoch": 0.1821734555839471, "flos": 14975120246400.0, "grad_norm": 1.9630182510346148, "language_loss": 0.77569473, "learning_rate": 3.68161515080535e-06, "loss": 0.79703879, "num_input_tokens_seen": 65479480, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.6796875, "step": 3030, "time_per_iteration": 2.346942663192749 }, { "auxiliary_loss_clip": 0.01093451, "auxiliary_loss_mlp": 0.01034323, "balance_loss_clip": 1.01573825, "balance_loss_mlp": 1.02540433, "epoch": 0.18223357883661506, "flos": 20192959693440.0, "grad_norm": 1.9172145309317545, "language_loss": 0.84994686, "learning_rate": 3.681410592686013e-06, "loss": 0.87122458, "num_input_tokens_seen": 65497775, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.6796875, "step": 3031, "time_per_iteration": 2.3773000240325928 }, { "auxiliary_loss_clip": 0.01094263, "auxiliary_loss_mlp": 0.01035067, "balance_loss_clip": 1.01641083, "balance_loss_mlp": 1.02630711, "epoch": 0.18229370208928303, "flos": 15231068490240.0, "grad_norm": 2.3139330996384486, "language_loss": 0.80105782, "learning_rate": 3.681205974561457e-06, "loss": 0.82235116, "num_input_tokens_seen": 65516505, "router_z_loss_clip": 0.18652344, "router_z_loss_mlp": 0.6796875, "step": 3032, "time_per_iteration": 2.3608875274658203 }, { "auxiliary_loss_clip": 0.01099095, "auxiliary_loss_mlp": 0.01039615, "balance_loss_clip": 1.02026689, "balance_loss_mlp": 1.0278728, "epoch": 0.182353825341951, "flos": 23839393489920.0, "grad_norm": 2.2208230265790116, "language_loss": 0.81210154, "learning_rate": 3.6810012964389846e-06, "loss": 0.83348858, "num_input_tokens_seen": 65536160, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.7109375, "step": 3033, "time_per_iteration": 2.4141690731048584 }, { "auxiliary_loss_clip": 0.0102659, "auxiliary_loss_mlp": 0.01005783, "balance_loss_clip": 1.00319636, "balance_loss_mlp": 1.00633883, "epoch": 0.18241394859461896, "flos": 61188114038400.0, "grad_norm": 0.8954202485514626, "language_loss": 0.63418603, "learning_rate": 3.680796558325899e-06, "loss": 0.65450966, "num_input_tokens_seen": 65589375, "router_z_loss_clip": 0.02587891, "router_z_loss_mlp": 0.20214844, "step": 3034, "time_per_iteration": 2.9214541912078857 }, { "auxiliary_loss_clip": 0.01093738, "auxiliary_loss_mlp": 0.01035767, "balance_loss_clip": 1.01731384, "balance_loss_mlp": 1.02661943, "epoch": 0.18247407184728695, "flos": 18470904059520.0, "grad_norm": 1.9366467761323554, "language_loss": 0.79605818, "learning_rate": 3.6805917602295084e-06, "loss": 0.81735319, "num_input_tokens_seen": 65606720, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.671875, "step": 3035, "time_per_iteration": 2.369335651397705 }, { "auxiliary_loss_clip": 0.01090274, "auxiliary_loss_mlp": 0.01031759, "balance_loss_clip": 1.01418781, "balance_loss_mlp": 1.02530837, "epoch": 0.18253419509995492, "flos": 21794216423040.0, "grad_norm": 1.7195695900240333, "language_loss": 0.84461898, "learning_rate": 3.680386902157121e-06, "loss": 0.8658393, "num_input_tokens_seen": 65625495, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.6484375, "step": 3036, "time_per_iteration": 2.401374101638794 }, { "auxiliary_loss_clip": 0.01092218, "auxiliary_loss_mlp": 0.010331, "balance_loss_clip": 1.01519525, "balance_loss_mlp": 1.02690077, "epoch": 0.18259431835262288, "flos": 20148934602240.0, "grad_norm": 2.0379480875904177, "language_loss": 0.79803252, "learning_rate": 3.680181984116047e-06, "loss": 0.81928569, "num_input_tokens_seen": 65643515, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.65234375, "step": 3037, "time_per_iteration": 2.3707478046417236 }, { "auxiliary_loss_clip": 0.01098544, "auxiliary_loss_mlp": 0.01038427, "balance_loss_clip": 1.01751804, "balance_loss_mlp": 1.02916551, "epoch": 0.18265444160529085, "flos": 16980740876160.0, "grad_norm": 4.894582151194986, "language_loss": 0.79521585, "learning_rate": 3.6799770061136e-06, "loss": 0.81658554, "num_input_tokens_seen": 65658155, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.6953125, "step": 3038, "time_per_iteration": 2.3390767574310303 }, { "auxiliary_loss_clip": 0.01093295, "auxiliary_loss_mlp": 0.01034576, "balance_loss_clip": 1.01652741, "balance_loss_mlp": 1.02583265, "epoch": 0.1827145648579588, "flos": 34421812295040.0, "grad_norm": 2.248150297807195, "language_loss": 0.67581129, "learning_rate": 3.6797719681570953e-06, "loss": 0.69708991, "num_input_tokens_seen": 65679310, "router_z_loss_clip": 0.18066406, "router_z_loss_mlp": 0.671875, "step": 3039, "time_per_iteration": 2.5091495513916016 }, { "auxiliary_loss_clip": 0.01094548, "auxiliary_loss_mlp": 0.01036112, "balance_loss_clip": 1.01676488, "balance_loss_mlp": 1.02687049, "epoch": 0.18277468811062678, "flos": 53285035835520.0, "grad_norm": 2.444654234344379, "language_loss": 0.73460305, "learning_rate": 3.6795668702538505e-06, "loss": 0.75590956, "num_input_tokens_seen": 65705235, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.6796875, "step": 3040, "time_per_iteration": 2.6638429164886475 }, { "auxiliary_loss_clip": 0.01093964, "auxiliary_loss_mlp": 0.01032173, "balance_loss_clip": 1.01376712, "balance_loss_mlp": 1.02748919, "epoch": 0.18283481136329474, "flos": 31649289540480.0, "grad_norm": 2.0313259677411803, "language_loss": 0.60360682, "learning_rate": 3.6793617124111836e-06, "loss": 0.62486821, "num_input_tokens_seen": 65727575, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.66796875, "step": 3041, "time_per_iteration": 2.4727203845977783 }, { "auxiliary_loss_clip": 0.01095961, "auxiliary_loss_mlp": 0.01041796, "balance_loss_clip": 1.02248418, "balance_loss_mlp": 1.02796614, "epoch": 0.18289493461596273, "flos": 53135782306560.0, "grad_norm": 1.6755375784789484, "language_loss": 0.60253775, "learning_rate": 3.6791564946364176e-06, "loss": 0.62391531, "num_input_tokens_seen": 65751370, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.6796875, "step": 3042, "time_per_iteration": 2.6615445613861084 }, { "auxiliary_loss_clip": 0.01093501, "auxiliary_loss_mlp": 0.01032486, "balance_loss_clip": 1.01429451, "balance_loss_mlp": 1.02835739, "epoch": 0.1829550578686307, "flos": 25588297825920.0, "grad_norm": 1.6013928454406494, "language_loss": 0.87596387, "learning_rate": 3.678951216936875e-06, "loss": 0.89722371, "num_input_tokens_seen": 65771040, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.65234375, "step": 3043, "time_per_iteration": 2.425600051879883 }, { "auxiliary_loss_clip": 0.01096452, "auxiliary_loss_mlp": 0.01038411, "balance_loss_clip": 1.01756108, "balance_loss_mlp": 1.02765286, "epoch": 0.18301518112129866, "flos": 22600325871360.0, "grad_norm": 2.1711627912841824, "language_loss": 0.70740992, "learning_rate": 3.6787458793198825e-06, "loss": 0.72875857, "num_input_tokens_seen": 65789345, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.6875, "step": 3044, "time_per_iteration": 2.37648606300354 }, { "auxiliary_loss_clip": 0.01098498, "auxiliary_loss_mlp": 0.01040058, "balance_loss_clip": 1.01912498, "balance_loss_mlp": 1.02705944, "epoch": 0.18307530437396663, "flos": 34019403431040.0, "grad_norm": 2.1079227102188396, "language_loss": 0.64306909, "learning_rate": 3.678540481792768e-06, "loss": 0.66445458, "num_input_tokens_seen": 65810990, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.71484375, "step": 3045, "time_per_iteration": 2.4975714683532715 }, { "auxiliary_loss_clip": 0.01092421, "auxiliary_loss_mlp": 0.01037968, "balance_loss_clip": 1.01928782, "balance_loss_mlp": 1.02710378, "epoch": 0.1831354276266346, "flos": 21278933533440.0, "grad_norm": 2.2035152419714414, "language_loss": 0.79463446, "learning_rate": 3.6783350243628613e-06, "loss": 0.81593835, "num_input_tokens_seen": 65827230, "router_z_loss_clip": 0.18652344, "router_z_loss_mlp": 0.65625, "step": 3046, "time_per_iteration": 2.3456878662109375 }, { "auxiliary_loss_clip": 0.01093784, "auxiliary_loss_mlp": 0.01036512, "balance_loss_clip": 1.01663971, "balance_loss_mlp": 1.02519798, "epoch": 0.18319555087930256, "flos": 21031887686400.0, "grad_norm": 3.27295768454744, "language_loss": 0.78758115, "learning_rate": 3.678129507037495e-06, "loss": 0.80888414, "num_input_tokens_seen": 65845900, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.6875, "step": 3047, "time_per_iteration": 2.380859136581421 }, { "auxiliary_loss_clip": 0.01095002, "auxiliary_loss_mlp": 0.01033843, "balance_loss_clip": 1.01493669, "balance_loss_mlp": 1.02816248, "epoch": 0.18325567413197055, "flos": 34381627453440.0, "grad_norm": 1.5145269717417007, "language_loss": 0.80488312, "learning_rate": 3.6779239298240032e-06, "loss": 0.82617152, "num_input_tokens_seen": 65868730, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.66796875, "step": 3048, "time_per_iteration": 2.49118971824646 }, { "auxiliary_loss_clip": 0.01096412, "auxiliary_loss_mlp": 0.01041874, "balance_loss_clip": 1.02191842, "balance_loss_mlp": 1.02703547, "epoch": 0.18331579738463852, "flos": 20557418042880.0, "grad_norm": 2.404031913488755, "language_loss": 0.8656354, "learning_rate": 3.6777182927297225e-06, "loss": 0.88701832, "num_input_tokens_seen": 65888420, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.69140625, "step": 3049, "time_per_iteration": 2.37909197807312 }, { "auxiliary_loss_clip": 0.0110299, "auxiliary_loss_mlp": 0.01040136, "balance_loss_clip": 1.01970398, "balance_loss_mlp": 1.02914858, "epoch": 0.18337592063730648, "flos": 19606907744640.0, "grad_norm": 2.3591333444806923, "language_loss": 0.76766431, "learning_rate": 3.6775125957619913e-06, "loss": 0.78909552, "num_input_tokens_seen": 65905840, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.73828125, "step": 3050, "time_per_iteration": 2.3622822761535645 }, { "auxiliary_loss_clip": 0.01091689, "auxiliary_loss_mlp": 0.01032873, "balance_loss_clip": 1.01391912, "balance_loss_mlp": 1.02520013, "epoch": 0.18343604388997445, "flos": 20849815612800.0, "grad_norm": 2.0644922428537096, "language_loss": 0.99320161, "learning_rate": 3.6773068389281507e-06, "loss": 1.01444721, "num_input_tokens_seen": 65922845, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.6640625, "step": 3051, "time_per_iteration": 2.3741469383239746 }, { "auxiliary_loss_clip": 0.01093216, "auxiliary_loss_mlp": 0.01036161, "balance_loss_clip": 1.01641965, "balance_loss_mlp": 1.02763176, "epoch": 0.1834961671426424, "flos": 24393080741760.0, "grad_norm": 2.272240720087646, "language_loss": 0.86265355, "learning_rate": 3.6771010222355434e-06, "loss": 0.88394737, "num_input_tokens_seen": 65945555, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.65625, "step": 3052, "time_per_iteration": 2.44637131690979 }, { "auxiliary_loss_clip": 0.01093433, "auxiliary_loss_mlp": 0.010358, "balance_loss_clip": 1.01694083, "balance_loss_mlp": 1.02554989, "epoch": 0.18355629039531038, "flos": 21250548731520.0, "grad_norm": 2.0104145983990582, "language_loss": 0.73043442, "learning_rate": 3.6768951456915147e-06, "loss": 0.75172675, "num_input_tokens_seen": 65963965, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.6796875, "step": 3053, "time_per_iteration": 2.391200542449951 }, { "auxiliary_loss_clip": 0.01098646, "auxiliary_loss_mlp": 0.01038256, "balance_loss_clip": 1.01797879, "balance_loss_mlp": 1.02836823, "epoch": 0.18361641364797834, "flos": 28655277920640.0, "grad_norm": 1.8701847575316863, "language_loss": 0.61304927, "learning_rate": 3.6766892093034123e-06, "loss": 0.63441837, "num_input_tokens_seen": 65985965, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.703125, "step": 3054, "time_per_iteration": 2.433223247528076 }, { "auxiliary_loss_clip": 0.01095956, "auxiliary_loss_mlp": 0.01035546, "balance_loss_clip": 1.01764071, "balance_loss_mlp": 1.02743411, "epoch": 0.18367653690064634, "flos": 20917896497280.0, "grad_norm": 1.9018163121145335, "language_loss": 0.78297484, "learning_rate": 3.6764832130785846e-06, "loss": 0.80428982, "num_input_tokens_seen": 66005645, "router_z_loss_clip": 0.17871094, "router_z_loss_mlp": 0.6875, "step": 3055, "time_per_iteration": 2.4101922512054443 }, { "auxiliary_loss_clip": 0.01093526, "auxiliary_loss_mlp": 0.0103933, "balance_loss_clip": 1.0211978, "balance_loss_mlp": 1.02704954, "epoch": 0.1837366601533143, "flos": 28764381519360.0, "grad_norm": 14.423993688140268, "language_loss": 0.70290178, "learning_rate": 3.6762771570243834e-06, "loss": 0.72423035, "num_input_tokens_seen": 66025675, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.6640625, "step": 3056, "time_per_iteration": 3.8228213787078857 }, { "auxiliary_loss_clip": 0.01095429, "auxiliary_loss_mlp": 0.01036901, "balance_loss_clip": 1.01788759, "balance_loss_mlp": 1.02727616, "epoch": 0.18379678340598227, "flos": 21250374174720.0, "grad_norm": 1.7355973902289035, "language_loss": 0.80511397, "learning_rate": 3.6760710411481623e-06, "loss": 0.82643723, "num_input_tokens_seen": 66046125, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.6796875, "step": 3057, "time_per_iteration": 2.4140725135803223 }, { "auxiliary_loss_clip": 0.01100634, "auxiliary_loss_mlp": 0.01040652, "balance_loss_clip": 1.0181576, "balance_loss_mlp": 1.0272398, "epoch": 0.18385690665865023, "flos": 20448558823680.0, "grad_norm": 2.0946795049792666, "language_loss": 0.82550985, "learning_rate": 3.675864865457277e-06, "loss": 0.84692276, "num_input_tokens_seen": 66064375, "router_z_loss_clip": 0.22460938, "router_z_loss_mlp": 0.734375, "step": 3058, "time_per_iteration": 2.379794120788574 }, { "auxiliary_loss_clip": 0.01096393, "auxiliary_loss_mlp": 0.01040085, "balance_loss_clip": 1.0208447, "balance_loss_mlp": 1.02711689, "epoch": 0.1839170299113182, "flos": 26139366725760.0, "grad_norm": 2.0980446596863476, "language_loss": 0.85711503, "learning_rate": 3.675658629959086e-06, "loss": 0.87847984, "num_input_tokens_seen": 66084590, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.69140625, "step": 3059, "time_per_iteration": 3.891240119934082 }, { "auxiliary_loss_clip": 0.01093734, "auxiliary_loss_mlp": 0.01036044, "balance_loss_clip": 1.01762605, "balance_loss_mlp": 1.02620101, "epoch": 0.18397715316398616, "flos": 31756717393920.0, "grad_norm": 1.6536118193911227, "language_loss": 0.72956884, "learning_rate": 3.6754523346609486e-06, "loss": 0.75086659, "num_input_tokens_seen": 66107105, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.671875, "step": 3060, "time_per_iteration": 2.4973304271698 }, { "auxiliary_loss_clip": 0.01097536, "auxiliary_loss_mlp": 0.01041634, "balance_loss_clip": 1.02152312, "balance_loss_mlp": 1.02810681, "epoch": 0.18403727641665413, "flos": 24610729357440.0, "grad_norm": 1.7066628802824622, "language_loss": 0.72872066, "learning_rate": 3.675245979570227e-06, "loss": 0.75011235, "num_input_tokens_seen": 66129295, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.6953125, "step": 3061, "time_per_iteration": 3.771973133087158 }, { "auxiliary_loss_clip": 0.01096596, "auxiliary_loss_mlp": 0.01043026, "balance_loss_clip": 1.02271307, "balance_loss_mlp": 1.02863383, "epoch": 0.18409739966932212, "flos": 23438800016640.0, "grad_norm": 1.8269389315323057, "language_loss": 0.81693745, "learning_rate": 3.6750395646942857e-06, "loss": 0.83833361, "num_input_tokens_seen": 66146910, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.6796875, "step": 3062, "time_per_iteration": 2.428438425064087 }, { "auxiliary_loss_clip": 0.01100455, "auxiliary_loss_mlp": 0.01040849, "balance_loss_clip": 1.02059579, "balance_loss_mlp": 1.02837312, "epoch": 0.18415752292199009, "flos": 21871025147520.0, "grad_norm": 2.02936284940346, "language_loss": 0.73024154, "learning_rate": 3.674833090040491e-06, "loss": 0.75165462, "num_input_tokens_seen": 66165370, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.71875, "step": 3063, "time_per_iteration": 2.3711354732513428 }, { "auxiliary_loss_clip": 0.01092559, "auxiliary_loss_mlp": 0.01036058, "balance_loss_clip": 1.0182246, "balance_loss_mlp": 1.0251112, "epoch": 0.18421764617465805, "flos": 25409507420160.0, "grad_norm": 1.735898876993406, "language_loss": 0.65679663, "learning_rate": 3.6746265556162116e-06, "loss": 0.67808282, "num_input_tokens_seen": 66186210, "router_z_loss_clip": 0.17773438, "router_z_loss_mlp": 0.67578125, "step": 3064, "time_per_iteration": 2.438000440597534 }, { "auxiliary_loss_clip": 0.01095055, "auxiliary_loss_mlp": 0.01036554, "balance_loss_clip": 1.01711094, "balance_loss_mlp": 1.02754521, "epoch": 0.18427776942732602, "flos": 27196920852480.0, "grad_norm": 2.484580111970709, "language_loss": 0.69019604, "learning_rate": 3.6744199614288174e-06, "loss": 0.71151215, "num_input_tokens_seen": 66204800, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.671875, "step": 3065, "time_per_iteration": 2.418018102645874 }, { "auxiliary_loss_clip": 0.01098874, "auxiliary_loss_mlp": 0.01041382, "balance_loss_clip": 1.01999593, "balance_loss_mlp": 1.02820754, "epoch": 0.18433789267999398, "flos": 27851193331200.0, "grad_norm": 2.250389640607876, "language_loss": 0.72799128, "learning_rate": 3.6742133074856828e-06, "loss": 0.74939388, "num_input_tokens_seen": 66222195, "router_z_loss_clip": 0.21386719, "router_z_loss_mlp": 0.70703125, "step": 3066, "time_per_iteration": 2.4412014484405518 }, { "auxiliary_loss_clip": 0.01094326, "auxiliary_loss_mlp": 0.01037958, "balance_loss_clip": 1.01918304, "balance_loss_mlp": 1.02577353, "epoch": 0.18439801593266195, "flos": 17856013461120.0, "grad_norm": 2.4021419349828457, "language_loss": 0.81847805, "learning_rate": 3.6740065937941815e-06, "loss": 0.83980089, "num_input_tokens_seen": 66239505, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.6875, "step": 3067, "time_per_iteration": 2.353297233581543 }, { "auxiliary_loss_clip": 0.01027585, "auxiliary_loss_mlp": 0.01006485, "balance_loss_clip": 1.00370753, "balance_loss_mlp": 1.00713754, "epoch": 0.18445813918532994, "flos": 56386403619840.0, "grad_norm": 0.9837619196764028, "language_loss": 0.5968374, "learning_rate": 3.673799820361691e-06, "loss": 0.61717808, "num_input_tokens_seen": 66295695, "router_z_loss_clip": 0.02783203, "router_z_loss_mlp": 0.20507812, "step": 3068, "time_per_iteration": 2.925403594970703 }, { "auxiliary_loss_clip": 0.01094959, "auxiliary_loss_mlp": 0.01035708, "balance_loss_clip": 1.01705205, "balance_loss_mlp": 1.0279814, "epoch": 0.1845182624379979, "flos": 20956196125440.0, "grad_norm": 1.7097395201758374, "language_loss": 0.76456642, "learning_rate": 3.67359298719559e-06, "loss": 0.78587306, "num_input_tokens_seen": 66315315, "router_z_loss_clip": 0.18652344, "router_z_loss_mlp": 0.671875, "step": 3069, "time_per_iteration": 2.3808977603912354 }, { "auxiliary_loss_clip": 0.01095065, "auxiliary_loss_mlp": 0.01035625, "balance_loss_clip": 1.01588392, "balance_loss_mlp": 1.02618957, "epoch": 0.18457838569066587, "flos": 20484135365760.0, "grad_norm": 1.818544207956705, "language_loss": 0.84722435, "learning_rate": 3.6733860943032607e-06, "loss": 0.86853123, "num_input_tokens_seen": 66333675, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.6875, "step": 3070, "time_per_iteration": 2.394941568374634 }, { "auxiliary_loss_clip": 0.01095165, "auxiliary_loss_mlp": 0.01036996, "balance_loss_clip": 1.0171473, "balance_loss_mlp": 1.02638769, "epoch": 0.18463850894333383, "flos": 25008844124160.0, "grad_norm": 1.9230594787216562, "language_loss": 0.77398825, "learning_rate": 3.6731791416920863e-06, "loss": 0.79530984, "num_input_tokens_seen": 66354075, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.6875, "step": 3071, "time_per_iteration": 2.402108907699585 }, { "auxiliary_loss_clip": 0.01099213, "auxiliary_loss_mlp": 0.01046013, "balance_loss_clip": 1.02627218, "balance_loss_mlp": 1.02819109, "epoch": 0.1846986321960018, "flos": 16799681232000.0, "grad_norm": 2.5374344837361407, "language_loss": 0.77136636, "learning_rate": 3.6729721293694523e-06, "loss": 0.79281867, "num_input_tokens_seen": 66372520, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.7109375, "step": 3072, "time_per_iteration": 2.424731969833374 }, { "auxiliary_loss_clip": 0.01097982, "auxiliary_loss_mlp": 0.0103529, "balance_loss_clip": 1.01547766, "balance_loss_mlp": 1.02694249, "epoch": 0.18475875544866976, "flos": 20813261552640.0, "grad_norm": 1.872650812020611, "language_loss": 0.86287987, "learning_rate": 3.6727650573427464e-06, "loss": 0.88421261, "num_input_tokens_seen": 66390745, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.7109375, "step": 3073, "time_per_iteration": 2.382587432861328 }, { "auxiliary_loss_clip": 0.01097807, "auxiliary_loss_mlp": 0.01038702, "balance_loss_clip": 1.02026057, "balance_loss_mlp": 1.0290978, "epoch": 0.18481887870133773, "flos": 22600325871360.0, "grad_norm": 2.581206154861645, "language_loss": 0.91659003, "learning_rate": 3.672557925619358e-06, "loss": 0.93795508, "num_input_tokens_seen": 66410525, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.6875, "step": 3074, "time_per_iteration": 2.416276216506958 }, { "auxiliary_loss_clip": 0.01095132, "auxiliary_loss_mlp": 0.0104095, "balance_loss_clip": 1.0198257, "balance_loss_mlp": 1.02679682, "epoch": 0.18487900195400572, "flos": 29457582030720.0, "grad_norm": 1.9061680747010519, "language_loss": 0.64877582, "learning_rate": 3.67235073420668e-06, "loss": 0.67013657, "num_input_tokens_seen": 66432535, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.68359375, "step": 3075, "time_per_iteration": 2.4442203044891357 }, { "auxiliary_loss_clip": 0.01096984, "auxiliary_loss_mlp": 0.01037603, "balance_loss_clip": 1.01770711, "balance_loss_mlp": 1.02914739, "epoch": 0.1849391252066737, "flos": 20627803077120.0, "grad_norm": 1.8140366071750742, "language_loss": 0.72486526, "learning_rate": 3.672143483112106e-06, "loss": 0.74621117, "num_input_tokens_seen": 66450620, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.6796875, "step": 3076, "time_per_iteration": 2.3927721977233887 }, { "auxiliary_loss_clip": 0.0109696, "auxiliary_loss_mlp": 0.01038754, "balance_loss_clip": 1.01946545, "balance_loss_mlp": 1.02690089, "epoch": 0.18499924845934165, "flos": 14427682128000.0, "grad_norm": 2.272661651313579, "language_loss": 0.81143332, "learning_rate": 3.6719361723430325e-06, "loss": 0.83279043, "num_input_tokens_seen": 66467865, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.703125, "step": 3077, "time_per_iteration": 2.3459293842315674 }, { "auxiliary_loss_clip": 0.01093051, "auxiliary_loss_mlp": 0.01037479, "balance_loss_clip": 1.01968122, "balance_loss_mlp": 1.02585387, "epoch": 0.18505937171200962, "flos": 23726659109760.0, "grad_norm": 1.8577127837084841, "language_loss": 0.78537548, "learning_rate": 3.671728801906857e-06, "loss": 0.8066808, "num_input_tokens_seen": 66486245, "router_z_loss_clip": 0.17773438, "router_z_loss_mlp": 0.671875, "step": 3078, "time_per_iteration": 2.426886558532715 }, { "auxiliary_loss_clip": 0.01095366, "auxiliary_loss_mlp": 0.01039953, "balance_loss_clip": 1.02030742, "balance_loss_mlp": 1.02709413, "epoch": 0.18511949496467758, "flos": 25956317134080.0, "grad_norm": 1.8974727382618128, "language_loss": 0.77608848, "learning_rate": 3.6715213718109816e-06, "loss": 0.79744172, "num_input_tokens_seen": 66506510, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.68359375, "step": 3079, "time_per_iteration": 2.4180920124053955 }, { "auxiliary_loss_clip": 0.01090707, "auxiliary_loss_mlp": 0.01038363, "balance_loss_clip": 1.02035069, "balance_loss_mlp": 1.02345252, "epoch": 0.18517961821734555, "flos": 42411895205760.0, "grad_norm": 1.751473176273842, "language_loss": 0.81666404, "learning_rate": 3.671313882062808e-06, "loss": 0.8379547, "num_input_tokens_seen": 66530960, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.671875, "step": 3080, "time_per_iteration": 2.6055829524993896 }, { "auxiliary_loss_clip": 0.01096193, "auxiliary_loss_mlp": 0.01039956, "balance_loss_clip": 1.01905835, "balance_loss_mlp": 1.02513731, "epoch": 0.18523974147001354, "flos": 24096423985920.0, "grad_norm": 1.8256720833864581, "language_loss": 0.73711753, "learning_rate": 3.6711063326697405e-06, "loss": 0.758479, "num_input_tokens_seen": 66550275, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.7109375, "step": 3081, "time_per_iteration": 2.4011452198028564 }, { "auxiliary_loss_clip": 0.01097376, "auxiliary_loss_mlp": 0.01039001, "balance_loss_clip": 1.01956987, "balance_loss_mlp": 1.0292356, "epoch": 0.1852998647226815, "flos": 27374210069760.0, "grad_norm": 2.017635067332807, "language_loss": 0.71629858, "learning_rate": 3.6708987236391867e-06, "loss": 0.73766237, "num_input_tokens_seen": 66569040, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.6796875, "step": 3082, "time_per_iteration": 2.4442226886749268 }, { "auxiliary_loss_clip": 0.01095924, "auxiliary_loss_mlp": 0.01036522, "balance_loss_clip": 1.01576734, "balance_loss_mlp": 1.0284586, "epoch": 0.18535998797534947, "flos": 18331774824960.0, "grad_norm": 2.59461557446974, "language_loss": 0.69121969, "learning_rate": 3.6706910549785562e-06, "loss": 0.71254414, "num_input_tokens_seen": 66587775, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.67578125, "step": 3083, "time_per_iteration": 2.348402261734009 }, { "auxiliary_loss_clip": 0.01095571, "auxiliary_loss_mlp": 0.01035693, "balance_loss_clip": 1.01739383, "balance_loss_mlp": 1.02865887, "epoch": 0.18542011122801744, "flos": 37844522899200.0, "grad_norm": 2.0029760019679537, "language_loss": 0.68881965, "learning_rate": 3.670483326695259e-06, "loss": 0.71013224, "num_input_tokens_seen": 66610800, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.671875, "step": 3084, "time_per_iteration": 2.5439836978912354 }, { "auxiliary_loss_clip": 0.01093441, "auxiliary_loss_mlp": 0.01035632, "balance_loss_clip": 1.01701152, "balance_loss_mlp": 1.02686095, "epoch": 0.1854802344806854, "flos": 25185120912000.0, "grad_norm": 1.7511023900322003, "language_loss": 0.77998507, "learning_rate": 3.6702755387967097e-06, "loss": 0.80127585, "num_input_tokens_seen": 66630960, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.6640625, "step": 3085, "time_per_iteration": 2.4212591648101807 }, { "auxiliary_loss_clip": 0.01093055, "auxiliary_loss_mlp": 0.01037337, "balance_loss_clip": 1.01842999, "balance_loss_mlp": 1.02593207, "epoch": 0.18554035773335337, "flos": 26683662821760.0, "grad_norm": 2.121942337652293, "language_loss": 0.73581004, "learning_rate": 3.6700676912903214e-06, "loss": 0.75711393, "num_input_tokens_seen": 66650585, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.671875, "step": 3086, "time_per_iteration": 2.4404194355010986 }, { "auxiliary_loss_clip": 0.010925, "auxiliary_loss_mlp": 0.01038782, "balance_loss_clip": 1.01865935, "balance_loss_mlp": 1.02658355, "epoch": 0.18560048098602133, "flos": 22345774081920.0, "grad_norm": 2.215072680838077, "language_loss": 0.69519728, "learning_rate": 3.6698597841835144e-06, "loss": 0.71651012, "num_input_tokens_seen": 66670045, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.65625, "step": 3087, "time_per_iteration": 2.3740296363830566 }, { "auxiliary_loss_clip": 0.01092804, "auxiliary_loss_mlp": 0.0103977, "balance_loss_clip": 1.01925397, "balance_loss_mlp": 1.02611017, "epoch": 0.18566060423868933, "flos": 17747573178240.0, "grad_norm": 2.3717575823996118, "language_loss": 0.73237813, "learning_rate": 3.6696518174837064e-06, "loss": 0.75370395, "num_input_tokens_seen": 66688790, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.66796875, "step": 3088, "time_per_iteration": 2.3804128170013428 }, { "auxiliary_loss_clip": 0.01092503, "auxiliary_loss_mlp": 0.01036027, "balance_loss_clip": 1.01862299, "balance_loss_mlp": 1.0264163, "epoch": 0.1857207274913573, "flos": 24676226801280.0, "grad_norm": 1.8183974767075333, "language_loss": 0.91748768, "learning_rate": 3.6694437911983197e-06, "loss": 0.93877304, "num_input_tokens_seen": 66708090, "router_z_loss_clip": 0.17382812, "router_z_loss_mlp": 0.66015625, "step": 3089, "time_per_iteration": 2.4014928340911865 }, { "auxiliary_loss_clip": 0.01090311, "auxiliary_loss_mlp": 0.01034988, "balance_loss_clip": 1.01645148, "balance_loss_mlp": 1.02585053, "epoch": 0.18578085074402526, "flos": 28146558366720.0, "grad_norm": 4.054582797978431, "language_loss": 0.57891083, "learning_rate": 3.669235705334779e-06, "loss": 0.60016382, "num_input_tokens_seen": 66727320, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.64453125, "step": 3090, "time_per_iteration": 2.441551446914673 }, { "auxiliary_loss_clip": 0.01089237, "auxiliary_loss_mlp": 0.01036836, "balance_loss_clip": 1.01866841, "balance_loss_mlp": 1.02500856, "epoch": 0.18584097399669322, "flos": 23950731415680.0, "grad_norm": 1.991318415116826, "language_loss": 0.81947285, "learning_rate": 3.669027559900509e-06, "loss": 0.84073359, "num_input_tokens_seen": 66747505, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.640625, "step": 3091, "time_per_iteration": 2.428163528442383 }, { "auxiliary_loss_clip": 0.01094214, "auxiliary_loss_mlp": 0.01042256, "balance_loss_clip": 1.02330172, "balance_loss_mlp": 1.02634609, "epoch": 0.18590109724936119, "flos": 17200728552960.0, "grad_norm": 5.040958102622603, "language_loss": 0.84161019, "learning_rate": 3.6688193549029397e-06, "loss": 0.86297488, "num_input_tokens_seen": 66766425, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.6796875, "step": 3092, "time_per_iteration": 2.395855188369751 }, { "auxiliary_loss_clip": 0.01096124, "auxiliary_loss_mlp": 0.01040186, "balance_loss_clip": 1.02018285, "balance_loss_mlp": 1.0266664, "epoch": 0.18596122050202915, "flos": 17233791477120.0, "grad_norm": 2.3730589433562215, "language_loss": 0.93141162, "learning_rate": 3.6686110903494995e-06, "loss": 0.95277476, "num_input_tokens_seen": 66781130, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.6953125, "step": 3093, "time_per_iteration": 2.3353850841522217 }, { "auxiliary_loss_clip": 0.01094786, "auxiliary_loss_mlp": 0.01040432, "balance_loss_clip": 1.02232397, "balance_loss_mlp": 1.02821648, "epoch": 0.18602134375469712, "flos": 19019878277760.0, "grad_norm": 1.8480169426734527, "language_loss": 0.77004647, "learning_rate": 3.668402766247622e-06, "loss": 0.7913987, "num_input_tokens_seen": 66797535, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.6640625, "step": 3094, "time_per_iteration": 2.387707233428955 }, { "auxiliary_loss_clip": 0.01094195, "auxiliary_loss_mlp": 0.01037264, "balance_loss_clip": 1.01876283, "balance_loss_mlp": 1.02692091, "epoch": 0.1860814670073651, "flos": 50948229248640.0, "grad_norm": 1.6034813027980024, "language_loss": 0.69743431, "learning_rate": 3.6681943826047413e-06, "loss": 0.71874893, "num_input_tokens_seen": 66821720, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.671875, "step": 3095, "time_per_iteration": 2.652736186981201 }, { "auxiliary_loss_clip": 0.01094172, "auxiliary_loss_mlp": 0.01038766, "balance_loss_clip": 1.01911986, "balance_loss_mlp": 1.02603316, "epoch": 0.18614159026003307, "flos": 19389957356160.0, "grad_norm": 2.022373330053034, "language_loss": 0.80696297, "learning_rate": 3.6679859394282944e-06, "loss": 0.82829237, "num_input_tokens_seen": 66839060, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.6796875, "step": 3096, "time_per_iteration": 3.8538777828216553 }, { "auxiliary_loss_clip": 0.01092448, "auxiliary_loss_mlp": 0.010374, "balance_loss_clip": 1.01905358, "balance_loss_mlp": 1.02626419, "epoch": 0.18620171351270104, "flos": 21797707559040.0, "grad_norm": 2.027790195257226, "language_loss": 0.74775016, "learning_rate": 3.6677774367257194e-06, "loss": 0.76904869, "num_input_tokens_seen": 66857760, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.6640625, "step": 3097, "time_per_iteration": 2.3681282997131348 }, { "auxiliary_loss_clip": 0.01092863, "auxiliary_loss_mlp": 0.01035928, "balance_loss_clip": 1.0169971, "balance_loss_mlp": 1.02733111, "epoch": 0.186261836765369, "flos": 16361940205440.0, "grad_norm": 2.02847078941279, "language_loss": 0.65580666, "learning_rate": 3.6675688745044583e-06, "loss": 0.67709458, "num_input_tokens_seen": 66876460, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.65625, "step": 3098, "time_per_iteration": 2.395278215408325 }, { "auxiliary_loss_clip": 0.01092862, "auxiliary_loss_mlp": 0.01038471, "balance_loss_clip": 1.01805055, "balance_loss_mlp": 1.02456856, "epoch": 0.18632196001803697, "flos": 23368868830080.0, "grad_norm": 1.7684825156704067, "language_loss": 0.6959098, "learning_rate": 3.6673602527719533e-06, "loss": 0.71722305, "num_input_tokens_seen": 66897960, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.68359375, "step": 3099, "time_per_iteration": 5.235820531845093 }, { "auxiliary_loss_clip": 0.01095068, "auxiliary_loss_mlp": 0.0104164, "balance_loss_clip": 1.02186322, "balance_loss_mlp": 1.02683389, "epoch": 0.18638208327070493, "flos": 22490908070400.0, "grad_norm": 1.54436392584293, "language_loss": 0.71356487, "learning_rate": 3.66715157153565e-06, "loss": 0.73493195, "num_input_tokens_seen": 66917675, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.6796875, "step": 3100, "time_per_iteration": 2.3982720375061035 }, { "auxiliary_loss_clip": 0.01094381, "auxiliary_loss_mlp": 0.01045709, "balance_loss_clip": 1.02631426, "balance_loss_mlp": 1.02628279, "epoch": 0.18644220652337293, "flos": 29164067297280.0, "grad_norm": 1.9548442918114333, "language_loss": 0.80125928, "learning_rate": 3.666942830802996e-06, "loss": 0.82266021, "num_input_tokens_seen": 66936000, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.6796875, "step": 3101, "time_per_iteration": 3.767545223236084 }, { "auxiliary_loss_clip": 0.01090047, "auxiliary_loss_mlp": 0.01035671, "balance_loss_clip": 1.01759875, "balance_loss_mlp": 1.02579045, "epoch": 0.1865023297760409, "flos": 24242640226560.0, "grad_norm": 1.8817863431958264, "language_loss": 0.76819777, "learning_rate": 3.6667340305814394e-06, "loss": 0.78945494, "num_input_tokens_seen": 66955700, "router_z_loss_clip": 0.18066406, "router_z_loss_mlp": 0.640625, "step": 3102, "time_per_iteration": 2.4271433353424072 }, { "auxiliary_loss_clip": 0.01095077, "auxiliary_loss_mlp": 0.01033553, "balance_loss_clip": 1.01529002, "balance_loss_mlp": 1.02549887, "epoch": 0.18656245302870886, "flos": 19127899624320.0, "grad_norm": 2.3303646098086364, "language_loss": 0.76854289, "learning_rate": 3.6665251708784325e-06, "loss": 0.78982925, "num_input_tokens_seen": 66972815, "router_z_loss_clip": 0.18261719, "router_z_loss_mlp": 0.6953125, "step": 3103, "time_per_iteration": 2.3667640686035156 }, { "auxiliary_loss_clip": 0.01094081, "auxiliary_loss_mlp": 0.01040071, "balance_loss_clip": 1.0222609, "balance_loss_mlp": 1.0266794, "epoch": 0.18662257628137682, "flos": 17785104756480.0, "grad_norm": 1.6592553548614029, "language_loss": 0.79195917, "learning_rate": 3.6663162517014294e-06, "loss": 0.81330061, "num_input_tokens_seen": 66992280, "router_z_loss_clip": 0.17773438, "router_z_loss_mlp": 0.67578125, "step": 3104, "time_per_iteration": 2.387533187866211 }, { "auxiliary_loss_clip": 0.01093203, "auxiliary_loss_mlp": 0.01032621, "balance_loss_clip": 1.01515698, "balance_loss_mlp": 1.0283078, "epoch": 0.1866826995340448, "flos": 24023246042880.0, "grad_norm": 2.181170976821608, "language_loss": 0.85263824, "learning_rate": 3.6661072730578858e-06, "loss": 0.87389648, "num_input_tokens_seen": 67012220, "router_z_loss_clip": 0.17382812, "router_z_loss_mlp": 0.6484375, "step": 3105, "time_per_iteration": 2.407792568206787 }, { "auxiliary_loss_clip": 0.01095813, "auxiliary_loss_mlp": 0.01037537, "balance_loss_clip": 1.01649654, "balance_loss_mlp": 1.02444923, "epoch": 0.18674282278671275, "flos": 26140030041600.0, "grad_norm": 2.084470877677171, "language_loss": 0.86739075, "learning_rate": 3.665898234955259e-06, "loss": 0.88872427, "num_input_tokens_seen": 67032030, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.71484375, "step": 3106, "time_per_iteration": 2.4420242309570312 }, { "auxiliary_loss_clip": 0.01094503, "auxiliary_loss_mlp": 0.01038425, "balance_loss_clip": 1.01956582, "balance_loss_mlp": 1.02600205, "epoch": 0.18680294603938072, "flos": 19201112478720.0, "grad_norm": 1.9136123432418746, "language_loss": 0.78331274, "learning_rate": 3.6656891374010097e-06, "loss": 0.80464196, "num_input_tokens_seen": 67048920, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.6875, "step": 3107, "time_per_iteration": 2.356750726699829 }, { "auxiliary_loss_clip": 0.01094176, "auxiliary_loss_mlp": 0.0104153, "balance_loss_clip": 1.02033496, "balance_loss_mlp": 1.02441061, "epoch": 0.1868630692920487, "flos": 28543730526720.0, "grad_norm": 2.008652235562938, "language_loss": 0.73930967, "learning_rate": 3.665479980402599e-06, "loss": 0.76066679, "num_input_tokens_seen": 67068645, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.6953125, "step": 3108, "time_per_iteration": 2.44694185256958 }, { "auxiliary_loss_clip": 0.01094503, "auxiliary_loss_mlp": 0.01035724, "balance_loss_clip": 1.01715183, "balance_loss_mlp": 1.02770066, "epoch": 0.18692319254471668, "flos": 17237073144960.0, "grad_norm": 1.7324864186310476, "language_loss": 0.74367827, "learning_rate": 3.665270763967493e-06, "loss": 0.76498055, "num_input_tokens_seen": 67087075, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.66796875, "step": 3109, "time_per_iteration": 2.361837387084961 }, { "auxiliary_loss_clip": 0.01091546, "auxiliary_loss_mlp": 0.01034762, "balance_loss_clip": 1.01597452, "balance_loss_mlp": 1.02526867, "epoch": 0.18698331579738464, "flos": 23184073670400.0, "grad_norm": 1.6227875736907937, "language_loss": 0.84263664, "learning_rate": 3.6650614881031567e-06, "loss": 0.86389971, "num_input_tokens_seen": 67108040, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.6640625, "step": 3110, "time_per_iteration": 2.40545916557312 }, { "auxiliary_loss_clip": 0.01095176, "auxiliary_loss_mlp": 0.01040211, "balance_loss_clip": 1.02092266, "balance_loss_mlp": 1.02745783, "epoch": 0.1870434390500526, "flos": 25515643553280.0, "grad_norm": 2.1454489632159643, "language_loss": 0.84406185, "learning_rate": 3.664852152817059e-06, "loss": 0.86541569, "num_input_tokens_seen": 67127605, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.67578125, "step": 3111, "time_per_iteration": 2.4096744060516357 }, { "auxiliary_loss_clip": 0.01091842, "auxiliary_loss_mlp": 0.01036712, "balance_loss_clip": 1.01865137, "balance_loss_mlp": 1.0261302, "epoch": 0.18710356230272057, "flos": 19499794093440.0, "grad_norm": 1.9619282300149858, "language_loss": 0.76870215, "learning_rate": 3.6646427581166702e-06, "loss": 0.78998768, "num_input_tokens_seen": 67145785, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.65625, "step": 3112, "time_per_iteration": 2.378523111343384 }, { "auxiliary_loss_clip": 0.0109347, "auxiliary_loss_mlp": 0.01037504, "balance_loss_clip": 1.01946771, "balance_loss_mlp": 1.02579355, "epoch": 0.18716368555538854, "flos": 26759633673600.0, "grad_norm": 2.0331328031005156, "language_loss": 0.64472282, "learning_rate": 3.6644333040094636e-06, "loss": 0.66603267, "num_input_tokens_seen": 67165930, "router_z_loss_clip": 0.18066406, "router_z_loss_mlp": 0.67578125, "step": 3113, "time_per_iteration": 2.4664793014526367 }, { "auxiliary_loss_clip": 0.0109792, "auxiliary_loss_mlp": 0.01032934, "balance_loss_clip": 1.01378942, "balance_loss_mlp": 1.02778101, "epoch": 0.1872238088080565, "flos": 25188716782080.0, "grad_norm": 3.8465490341367548, "language_loss": 0.81099665, "learning_rate": 3.6642237905029132e-06, "loss": 0.83230519, "num_input_tokens_seen": 67185830, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.703125, "step": 3114, "time_per_iteration": 2.439657688140869 }, { "auxiliary_loss_clip": 0.01094659, "auxiliary_loss_mlp": 0.01041677, "balance_loss_clip": 1.02135181, "balance_loss_mlp": 1.02711296, "epoch": 0.1872839320607245, "flos": 24133152602880.0, "grad_norm": 1.890025172784249, "language_loss": 0.57458973, "learning_rate": 3.664014217604497e-06, "loss": 0.59595311, "num_input_tokens_seen": 67206930, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.67578125, "step": 3115, "time_per_iteration": 2.4208052158355713 }, { "auxiliary_loss_clip": 0.01091956, "auxiliary_loss_mlp": 0.01034217, "balance_loss_clip": 1.01615715, "balance_loss_mlp": 1.02803063, "epoch": 0.18734405531339246, "flos": 21172867223040.0, "grad_norm": 2.0238768270140497, "language_loss": 0.71198618, "learning_rate": 3.6638045853216938e-06, "loss": 0.733248, "num_input_tokens_seen": 67226290, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.640625, "step": 3116, "time_per_iteration": 2.40444016456604 }, { "auxiliary_loss_clip": 0.01089161, "auxiliary_loss_mlp": 0.01030626, "balance_loss_clip": 1.01394904, "balance_loss_mlp": 1.02528214, "epoch": 0.18740417856606043, "flos": 17236758942720.0, "grad_norm": 1.9591007837706198, "language_loss": 0.78899264, "learning_rate": 3.663594893661985e-06, "loss": 0.81019044, "num_input_tokens_seen": 67244410, "router_z_loss_clip": 0.16601562, "router_z_loss_mlp": 0.640625, "step": 3117, "time_per_iteration": 2.3701653480529785 }, { "auxiliary_loss_clip": 0.01092906, "auxiliary_loss_mlp": 0.01033878, "balance_loss_clip": 1.01637793, "balance_loss_mlp": 1.02760911, "epoch": 0.1874643018187284, "flos": 32556787176960.0, "grad_norm": 1.7784577775703345, "language_loss": 0.84191912, "learning_rate": 3.663385142632853e-06, "loss": 0.86318696, "num_input_tokens_seen": 67264470, "router_z_loss_clip": 0.17480469, "router_z_loss_mlp": 0.65234375, "step": 3118, "time_per_iteration": 2.495004177093506 }, { "auxiliary_loss_clip": 0.01092904, "auxiliary_loss_mlp": 0.01032003, "balance_loss_clip": 1.01405048, "balance_loss_mlp": 1.0258255, "epoch": 0.18752442507139636, "flos": 23257042145280.0, "grad_norm": 10.896413965969675, "language_loss": 0.76097798, "learning_rate": 3.663175332241785e-06, "loss": 0.78222704, "num_input_tokens_seen": 67284315, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.66796875, "step": 3119, "time_per_iteration": 2.3944449424743652 }, { "auxiliary_loss_clip": 0.01094838, "auxiliary_loss_mlp": 0.01039663, "balance_loss_clip": 1.02112603, "balance_loss_mlp": 1.02713871, "epoch": 0.18758454832406432, "flos": 21759896689920.0, "grad_norm": 1.9147952443760252, "language_loss": 0.82168788, "learning_rate": 3.6629654624962666e-06, "loss": 0.84303284, "num_input_tokens_seen": 67302780, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.6796875, "step": 3120, "time_per_iteration": 2.40030574798584 }, { "auxiliary_loss_clip": 0.01089715, "auxiliary_loss_mlp": 0.01034538, "balance_loss_clip": 1.01741982, "balance_loss_mlp": 1.02592897, "epoch": 0.1876446715767323, "flos": 29568919956480.0, "grad_norm": 2.0956589745081087, "language_loss": 0.85304511, "learning_rate": 3.6627555334037893e-06, "loss": 0.87428761, "num_input_tokens_seen": 67323405, "router_z_loss_clip": 0.17089844, "router_z_loss_mlp": 0.63671875, "step": 3121, "time_per_iteration": 2.442066192626953 }, { "auxiliary_loss_clip": 0.01092619, "auxiliary_loss_mlp": 0.01037071, "balance_loss_clip": 1.0192734, "balance_loss_mlp": 1.0265708, "epoch": 0.18770479482940028, "flos": 30338580078720.0, "grad_norm": 1.771444275332751, "language_loss": 0.70667934, "learning_rate": 3.662545544971844e-06, "loss": 0.72797626, "num_input_tokens_seen": 67345800, "router_z_loss_clip": 0.17773438, "router_z_loss_mlp": 0.66015625, "step": 3122, "time_per_iteration": 2.4791295528411865 }, { "auxiliary_loss_clip": 0.01088481, "auxiliary_loss_mlp": 0.01035789, "balance_loss_clip": 1.01690674, "balance_loss_mlp": 1.02441263, "epoch": 0.18776491808206824, "flos": 14464480567680.0, "grad_norm": 2.354608206212646, "language_loss": 0.70926332, "learning_rate": 3.662335497207924e-06, "loss": 0.73050606, "num_input_tokens_seen": 67363575, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.640625, "step": 3123, "time_per_iteration": 2.3511626720428467 }, { "auxiliary_loss_clip": 0.01090677, "auxiliary_loss_mlp": 0.01035649, "balance_loss_clip": 1.01949656, "balance_loss_mlp": 1.02599788, "epoch": 0.1878250413347362, "flos": 24497401484160.0, "grad_norm": 1.9326721085903336, "language_loss": 0.74157596, "learning_rate": 3.662125390119527e-06, "loss": 0.76283926, "num_input_tokens_seen": 67381765, "router_z_loss_clip": 0.16113281, "router_z_loss_mlp": 0.6484375, "step": 3124, "time_per_iteration": 2.408418655395508 }, { "auxiliary_loss_clip": 0.01092443, "auxiliary_loss_mlp": 0.01036397, "balance_loss_clip": 1.01840878, "balance_loss_mlp": 1.02593827, "epoch": 0.18788516458740417, "flos": 39784611173760.0, "grad_norm": 1.6700523984093973, "language_loss": 0.8071084, "learning_rate": 3.66191522371415e-06, "loss": 0.8283968, "num_input_tokens_seen": 67405000, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.6640625, "step": 3125, "time_per_iteration": 2.548044443130493 }, { "auxiliary_loss_clip": 0.01028515, "auxiliary_loss_mlp": 0.01011871, "balance_loss_clip": 1.00909352, "balance_loss_mlp": 1.00887442, "epoch": 0.18794528784007214, "flos": 64696151738880.0, "grad_norm": 0.9630832536964263, "language_loss": 0.63649619, "learning_rate": 3.6617049979992937e-06, "loss": 0.65690005, "num_input_tokens_seen": 67467140, "router_z_loss_clip": 0.02783203, "router_z_loss_mlp": 0.19628906, "step": 3126, "time_per_iteration": 3.0950701236724854 }, { "auxiliary_loss_clip": 0.01090555, "auxiliary_loss_mlp": 0.01034912, "balance_loss_clip": 1.01673269, "balance_loss_mlp": 1.02696335, "epoch": 0.1880054110927401, "flos": 28620783630720.0, "grad_norm": 1.6531561448667726, "language_loss": 0.81115246, "learning_rate": 3.6614947129824603e-06, "loss": 0.83240718, "num_input_tokens_seen": 67487980, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.63671875, "step": 3127, "time_per_iteration": 2.4451937675476074 }, { "auxiliary_loss_clip": 0.01026604, "auxiliary_loss_mlp": 0.01004844, "balance_loss_clip": 1.00189924, "balance_loss_mlp": 1.00730681, "epoch": 0.1880655343454081, "flos": 64485625040640.0, "grad_norm": 0.7640370653681977, "language_loss": 0.61857343, "learning_rate": 3.6612843686711542e-06, "loss": 0.63888794, "num_input_tokens_seen": 67552500, "router_z_loss_clip": 0.02941895, "router_z_loss_mlp": 0.19335938, "step": 3128, "time_per_iteration": 3.1438512802124023 }, { "auxiliary_loss_clip": 0.01095201, "auxiliary_loss_mlp": 0.01029671, "balance_loss_clip": 1.0115037, "balance_loss_mlp": 1.02658224, "epoch": 0.18812565759807606, "flos": 32123095868160.0, "grad_norm": 2.1786778699681593, "language_loss": 0.70593059, "learning_rate": 3.661073965072883e-06, "loss": 0.72717929, "num_input_tokens_seen": 67573295, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.6875, "step": 3129, "time_per_iteration": 2.520672559738159 }, { "auxiliary_loss_clip": 0.01094378, "auxiliary_loss_mlp": 0.01040439, "balance_loss_clip": 1.02124667, "balance_loss_mlp": 1.02671683, "epoch": 0.18818578085074403, "flos": 20623683536640.0, "grad_norm": 2.4633057488059817, "language_loss": 0.85206509, "learning_rate": 3.6608635021951546e-06, "loss": 0.87341321, "num_input_tokens_seen": 67590010, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.67578125, "step": 3130, "time_per_iteration": 2.4170432090759277 }, { "auxiliary_loss_clip": 0.01093009, "auxiliary_loss_mlp": 0.01035344, "balance_loss_clip": 1.01569831, "balance_loss_mlp": 1.025419, "epoch": 0.188245904103412, "flos": 28839235207680.0, "grad_norm": 2.0859429617400593, "language_loss": 0.77010924, "learning_rate": 3.6606529800454794e-06, "loss": 0.7913928, "num_input_tokens_seen": 67611110, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.67578125, "step": 3131, "time_per_iteration": 2.4445645809173584 }, { "auxiliary_loss_clip": 0.01091879, "auxiliary_loss_mlp": 0.01037633, "balance_loss_clip": 1.0196805, "balance_loss_mlp": 1.02741194, "epoch": 0.18830602735607996, "flos": 29419142757120.0, "grad_norm": 2.053265037194725, "language_loss": 0.81552517, "learning_rate": 3.660442398631372e-06, "loss": 0.83682024, "num_input_tokens_seen": 67631990, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.64453125, "step": 3132, "time_per_iteration": 2.4377148151397705 }, { "auxiliary_loss_clip": 0.01094441, "auxiliary_loss_mlp": 0.01040242, "balance_loss_clip": 1.02169347, "balance_loss_mlp": 1.02709687, "epoch": 0.18836615060874792, "flos": 28871774461440.0, "grad_norm": 2.180847825789763, "language_loss": 0.79780543, "learning_rate": 3.660231757960346e-06, "loss": 0.81915224, "num_input_tokens_seen": 67650490, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.671875, "step": 3133, "time_per_iteration": 2.4449267387390137 }, { "auxiliary_loss_clip": 0.01093615, "auxiliary_loss_mlp": 0.01043389, "balance_loss_clip": 1.02441084, "balance_loss_mlp": 1.02740383, "epoch": 0.18842627386141592, "flos": 22600570250880.0, "grad_norm": 11.22870173067583, "language_loss": 0.82609212, "learning_rate": 3.660021058039919e-06, "loss": 0.84746218, "num_input_tokens_seen": 67668860, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.66015625, "step": 3134, "time_per_iteration": 2.400921583175659 }, { "auxiliary_loss_clip": 0.01092727, "auxiliary_loss_mlp": 0.01038447, "balance_loss_clip": 1.01982653, "balance_loss_mlp": 1.02743077, "epoch": 0.18848639711408388, "flos": 24572394817920.0, "grad_norm": 1.5148271158508548, "language_loss": 0.8306362, "learning_rate": 3.659810298877611e-06, "loss": 0.8519479, "num_input_tokens_seen": 67690220, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.65234375, "step": 3135, "time_per_iteration": 3.787660598754883 }, { "auxiliary_loss_clip": 0.01098167, "auxiliary_loss_mlp": 0.010379, "balance_loss_clip": 1.01861191, "balance_loss_mlp": 1.02853096, "epoch": 0.18854652036675185, "flos": 34165514937600.0, "grad_norm": 2.004547343841207, "language_loss": 0.78512704, "learning_rate": 3.659599480480943e-06, "loss": 0.80648768, "num_input_tokens_seen": 67709820, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.6953125, "step": 3136, "time_per_iteration": 2.479468822479248 }, { "auxiliary_loss_clip": 0.01093813, "auxiliary_loss_mlp": 0.0104207, "balance_loss_clip": 1.02251983, "balance_loss_mlp": 1.02774096, "epoch": 0.1886066436194198, "flos": 24199278451200.0, "grad_norm": 2.0835368106190146, "language_loss": 0.81216836, "learning_rate": 3.659388602857438e-06, "loss": 0.83352721, "num_input_tokens_seen": 67729490, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.66015625, "step": 3137, "time_per_iteration": 2.4111809730529785 }, { "auxiliary_loss_clip": 0.01095639, "auxiliary_loss_mlp": 0.01036815, "balance_loss_clip": 1.01908875, "balance_loss_mlp": 1.02895725, "epoch": 0.18866676687208778, "flos": 21250059972480.0, "grad_norm": 1.5100388369519946, "language_loss": 0.80736995, "learning_rate": 3.6591776660146225e-06, "loss": 0.82869452, "num_input_tokens_seen": 67749665, "router_z_loss_clip": 0.17773438, "router_z_loss_mlp": 0.66796875, "step": 3138, "time_per_iteration": 3.7891883850097656 }, { "auxiliary_loss_clip": 0.01095721, "auxiliary_loss_mlp": 0.01039131, "balance_loss_clip": 1.02066553, "balance_loss_mlp": 1.02673495, "epoch": 0.18872689012475574, "flos": 37307069429760.0, "grad_norm": 2.0378947896863555, "language_loss": 0.63375771, "learning_rate": 3.6589666699600247e-06, "loss": 0.65510619, "num_input_tokens_seen": 67776230, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.69140625, "step": 3139, "time_per_iteration": 3.908792495727539 }, { "auxiliary_loss_clip": 0.01092315, "auxiliary_loss_mlp": 0.01037282, "balance_loss_clip": 1.017326, "balance_loss_mlp": 1.02600133, "epoch": 0.1887870133774237, "flos": 21651246938880.0, "grad_norm": 2.4065564903787893, "language_loss": 0.71284431, "learning_rate": 3.6587556147011728e-06, "loss": 0.73414028, "num_input_tokens_seen": 67795080, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.6640625, "step": 3140, "time_per_iteration": 3.7559385299682617 }, { "auxiliary_loss_clip": 0.01094735, "auxiliary_loss_mlp": 0.01037656, "balance_loss_clip": 1.01778328, "balance_loss_mlp": 1.02658761, "epoch": 0.1888471366300917, "flos": 15923745331200.0, "grad_norm": 2.4677283334453546, "language_loss": 0.87063736, "learning_rate": 3.6585445002456004e-06, "loss": 0.89196122, "num_input_tokens_seen": 67813110, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.6796875, "step": 3141, "time_per_iteration": 2.3590939044952393 }, { "auxiliary_loss_clip": 0.01096879, "auxiliary_loss_mlp": 0.01036378, "balance_loss_clip": 1.01556396, "balance_loss_mlp": 1.0269196, "epoch": 0.18890725988275966, "flos": 18550959540480.0, "grad_norm": 1.8405154147025118, "language_loss": 0.7696079, "learning_rate": 3.6583333266008404e-06, "loss": 0.79094052, "num_input_tokens_seen": 67831070, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.69921875, "step": 3142, "time_per_iteration": 2.342622756958008 }, { "auxiliary_loss_clip": 0.01091897, "auxiliary_loss_mlp": 0.01035272, "balance_loss_clip": 1.01705718, "balance_loss_mlp": 1.02641535, "epoch": 0.18896738313542763, "flos": 28839584321280.0, "grad_norm": 1.7804266465807372, "language_loss": 0.78882277, "learning_rate": 3.6581220937744305e-06, "loss": 0.81009448, "num_input_tokens_seen": 67852170, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.65234375, "step": 3143, "time_per_iteration": 2.4450275897979736 }, { "auxiliary_loss_clip": 0.01094285, "auxiliary_loss_mlp": 0.01040665, "balance_loss_clip": 1.02175856, "balance_loss_mlp": 1.02750754, "epoch": 0.1890275063880956, "flos": 22411830107520.0, "grad_norm": 2.287618186149079, "language_loss": 0.71571839, "learning_rate": 3.6579108017739076e-06, "loss": 0.73706782, "num_input_tokens_seen": 67869945, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.66796875, "step": 3144, "time_per_iteration": 2.3720335960388184 }, { "auxiliary_loss_clip": 0.01094998, "auxiliary_loss_mlp": 0.01037076, "balance_loss_clip": 1.01803827, "balance_loss_mlp": 1.02687371, "epoch": 0.18908762964076356, "flos": 24242744960640.0, "grad_norm": 2.6075958756325393, "language_loss": 0.73052001, "learning_rate": 3.6576994506068136e-06, "loss": 0.75184077, "num_input_tokens_seen": 67890240, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.6796875, "step": 3145, "time_per_iteration": 2.4356400966644287 }, { "auxiliary_loss_clip": 0.0109119, "auxiliary_loss_mlp": 0.01034824, "balance_loss_clip": 1.01741934, "balance_loss_mlp": 1.02506208, "epoch": 0.18914775289343153, "flos": 16981962773760.0, "grad_norm": 2.683121301152449, "language_loss": 0.76936823, "learning_rate": 3.6574880402806897e-06, "loss": 0.79062837, "num_input_tokens_seen": 67907825, "router_z_loss_clip": 0.17382812, "router_z_loss_mlp": 0.6640625, "step": 3146, "time_per_iteration": 2.351532220840454 }, { "auxiliary_loss_clip": 0.01093525, "auxiliary_loss_mlp": 0.01039422, "balance_loss_clip": 1.02119517, "balance_loss_mlp": 1.02663589, "epoch": 0.1892078761460995, "flos": 21542701921920.0, "grad_norm": 2.1745879156237082, "language_loss": 0.78983533, "learning_rate": 3.6572765708030813e-06, "loss": 0.81116486, "num_input_tokens_seen": 67926670, "router_z_loss_clip": 0.18261719, "router_z_loss_mlp": 0.66796875, "step": 3147, "time_per_iteration": 2.396304130554199 }, { "auxiliary_loss_clip": 0.01090724, "auxiliary_loss_mlp": 0.01032712, "balance_loss_clip": 1.01547456, "balance_loss_mlp": 1.02646947, "epoch": 0.18926799939876748, "flos": 23000465496960.0, "grad_norm": 2.8520065875250187, "language_loss": 0.66726327, "learning_rate": 3.657065042181536e-06, "loss": 0.68849766, "num_input_tokens_seen": 67943645, "router_z_loss_clip": 0.17285156, "router_z_loss_mlp": 0.640625, "step": 3148, "time_per_iteration": 2.395343065261841 }, { "auxiliary_loss_clip": 0.01091736, "auxiliary_loss_mlp": 0.01030399, "balance_loss_clip": 1.01361406, "balance_loss_mlp": 1.02645969, "epoch": 0.18932812265143545, "flos": 22271932823040.0, "grad_norm": 2.445524490717879, "language_loss": 0.76157504, "learning_rate": 3.6568534544236008e-06, "loss": 0.78279638, "num_input_tokens_seen": 67962345, "router_z_loss_clip": 0.16796875, "router_z_loss_mlp": 0.65234375, "step": 3149, "time_per_iteration": 2.3933093547821045 }, { "auxiliary_loss_clip": 0.01090073, "auxiliary_loss_mlp": 0.01036841, "balance_loss_clip": 1.02015185, "balance_loss_mlp": 1.02690399, "epoch": 0.1893882459041034, "flos": 18623439256320.0, "grad_norm": 3.363665441741508, "language_loss": 0.81101823, "learning_rate": 3.656641807536828e-06, "loss": 0.83228737, "num_input_tokens_seen": 67979760, "router_z_loss_clip": 0.16699219, "router_z_loss_mlp": 0.6328125, "step": 3150, "time_per_iteration": 2.3639278411865234 }, { "auxiliary_loss_clip": 0.01095356, "auxiliary_loss_mlp": 0.01041358, "balance_loss_clip": 1.022928, "balance_loss_mlp": 1.02777815, "epoch": 0.18944836915677138, "flos": 22891885568640.0, "grad_norm": 2.0949354009812304, "language_loss": 0.84872854, "learning_rate": 3.6564301015287706e-06, "loss": 0.87009573, "num_input_tokens_seen": 67996895, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.67578125, "step": 3151, "time_per_iteration": 2.3968522548675537 }, { "auxiliary_loss_clip": 0.01095155, "auxiliary_loss_mlp": 0.01040186, "balance_loss_clip": 1.0226388, "balance_loss_mlp": 1.02819836, "epoch": 0.18950849240943934, "flos": 26795349861120.0, "grad_norm": 1.9176161239989238, "language_loss": 0.74011457, "learning_rate": 3.6562183364069835e-06, "loss": 0.76146793, "num_input_tokens_seen": 68018365, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.66796875, "step": 3152, "time_per_iteration": 2.4383933544158936 }, { "auxiliary_loss_clip": 0.01091615, "auxiliary_loss_mlp": 0.01039334, "balance_loss_clip": 1.02085638, "balance_loss_mlp": 1.02577949, "epoch": 0.1895686156621073, "flos": 24970125559680.0, "grad_norm": 1.8800007116162436, "language_loss": 0.75120592, "learning_rate": 3.6560065121790244e-06, "loss": 0.77251536, "num_input_tokens_seen": 68037985, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.65625, "step": 3153, "time_per_iteration": 2.4134364128112793 }, { "auxiliary_loss_clip": 0.01093964, "auxiliary_loss_mlp": 0.01037649, "balance_loss_clip": 1.01952863, "balance_loss_mlp": 1.0262568, "epoch": 0.1896287389147753, "flos": 21943469952000.0, "grad_norm": 5.572109106942339, "language_loss": 0.79413539, "learning_rate": 3.655794628852453e-06, "loss": 0.8154515, "num_input_tokens_seen": 68057975, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.6796875, "step": 3154, "time_per_iteration": 2.3982646465301514 }, { "auxiliary_loss_clip": 0.01094087, "auxiliary_loss_mlp": 0.01037787, "balance_loss_clip": 1.01841521, "balance_loss_mlp": 1.02587223, "epoch": 0.18968886216744327, "flos": 18178297021440.0, "grad_norm": 2.80094461298542, "language_loss": 0.72725987, "learning_rate": 3.6555826864348297e-06, "loss": 0.74857867, "num_input_tokens_seen": 68074175, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.6796875, "step": 3155, "time_per_iteration": 2.3434884548187256 }, { "auxiliary_loss_clip": 0.01089547, "auxiliary_loss_mlp": 0.01035331, "balance_loss_clip": 1.01737821, "balance_loss_mlp": 1.02359009, "epoch": 0.18974898542011123, "flos": 20411446181760.0, "grad_norm": 2.2744931621323725, "language_loss": 0.7401787, "learning_rate": 3.6553706849337197e-06, "loss": 0.76142752, "num_input_tokens_seen": 68095230, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.65625, "step": 3156, "time_per_iteration": 2.408158540725708 }, { "auxiliary_loss_clip": 0.01093336, "auxiliary_loss_mlp": 0.01032518, "balance_loss_clip": 1.01494694, "balance_loss_mlp": 1.02672505, "epoch": 0.1898091086727792, "flos": 23983968896640.0, "grad_norm": 1.781444490073804, "language_loss": 0.67989981, "learning_rate": 3.6551586243566877e-06, "loss": 0.7011584, "num_input_tokens_seen": 68113805, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.6640625, "step": 3157, "time_per_iteration": 2.396543502807617 }, { "auxiliary_loss_clip": 0.01091825, "auxiliary_loss_mlp": 0.01031665, "balance_loss_clip": 1.01412976, "balance_loss_mlp": 1.02482057, "epoch": 0.18986923192544716, "flos": 27635813953920.0, "grad_norm": 1.7151154497333212, "language_loss": 0.79707837, "learning_rate": 3.654946504711302e-06, "loss": 0.81831336, "num_input_tokens_seen": 68133190, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.671875, "step": 3158, "time_per_iteration": 2.4283323287963867 }, { "auxiliary_loss_clip": 0.01096707, "auxiliary_loss_mlp": 0.01039898, "balance_loss_clip": 1.01951361, "balance_loss_mlp": 1.02673757, "epoch": 0.18992935517811513, "flos": 25482964654080.0, "grad_norm": 2.665136737825096, "language_loss": 0.72027659, "learning_rate": 3.6547343260051323e-06, "loss": 0.74164271, "num_input_tokens_seen": 68152330, "router_z_loss_clip": 0.20410156, "router_z_loss_mlp": 0.69921875, "step": 3159, "time_per_iteration": 2.4149372577667236 }, { "auxiliary_loss_clip": 0.01093615, "auxiliary_loss_mlp": 0.0104252, "balance_loss_clip": 1.02317274, "balance_loss_mlp": 1.02667046, "epoch": 0.1899894784307831, "flos": 17419843445760.0, "grad_norm": 2.4817875191132286, "language_loss": 0.85185206, "learning_rate": 3.6545220882457518e-06, "loss": 0.87321341, "num_input_tokens_seen": 68170185, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.671875, "step": 3160, "time_per_iteration": 2.370260238647461 }, { "auxiliary_loss_clip": 0.01088359, "auxiliary_loss_mlp": 0.01044831, "balance_loss_clip": 1.02783227, "balance_loss_mlp": 1.02554953, "epoch": 0.19004960168345109, "flos": 27490959256320.0, "grad_norm": 1.8624981820899977, "language_loss": 0.73385042, "learning_rate": 3.6543097914407336e-06, "loss": 0.75518227, "num_input_tokens_seen": 68191665, "router_z_loss_clip": 0.16992188, "router_z_loss_mlp": 0.625, "step": 3161, "time_per_iteration": 2.4460105895996094 }, { "auxiliary_loss_clip": 0.01090908, "auxiliary_loss_mlp": 0.01041006, "balance_loss_clip": 1.0234704, "balance_loss_mlp": 1.02615905, "epoch": 0.19010972493611905, "flos": 38653145965440.0, "grad_norm": 1.8150751487237726, "language_loss": 0.80446106, "learning_rate": 3.6540974355976537e-06, "loss": 0.82578015, "num_input_tokens_seen": 68214635, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.6484375, "step": 3162, "time_per_iteration": 2.5258138179779053 }, { "auxiliary_loss_clip": 0.01092994, "auxiliary_loss_mlp": 0.01031712, "balance_loss_clip": 1.01316249, "balance_loss_mlp": 1.02582717, "epoch": 0.19016984818878702, "flos": 19243741115520.0, "grad_norm": 3.1610870978860692, "language_loss": 0.75388765, "learning_rate": 3.653885020724092e-06, "loss": 0.77513468, "num_input_tokens_seen": 68232150, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.671875, "step": 3163, "time_per_iteration": 2.3539695739746094 }, { "auxiliary_loss_clip": 0.01091513, "auxiliary_loss_mlp": 0.01038314, "balance_loss_clip": 1.02019417, "balance_loss_mlp": 1.02672601, "epoch": 0.19022997144145498, "flos": 37595382370560.0, "grad_norm": 2.5051610458785984, "language_loss": 0.74053907, "learning_rate": 3.653672546827628e-06, "loss": 0.76183736, "num_input_tokens_seen": 68253370, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.6484375, "step": 3164, "time_per_iteration": 2.5292487144470215 }, { "auxiliary_loss_clip": 0.01093023, "auxiliary_loss_mlp": 0.01028941, "balance_loss_clip": 1.01123857, "balance_loss_mlp": 1.02741444, "epoch": 0.19029009469412295, "flos": 61528762840320.0, "grad_norm": 1.4450338276097412, "language_loss": 0.66605741, "learning_rate": 3.653460013915844e-06, "loss": 0.68727708, "num_input_tokens_seen": 68278895, "router_z_loss_clip": 0.17675781, "router_z_loss_mlp": 0.65625, "step": 3165, "time_per_iteration": 2.7367405891418457 }, { "auxiliary_loss_clip": 0.01095137, "auxiliary_loss_mlp": 0.01038974, "balance_loss_clip": 1.02067578, "balance_loss_mlp": 1.02840436, "epoch": 0.1903502179467909, "flos": 13953980534400.0, "grad_norm": 2.4984910791317807, "language_loss": 0.73748457, "learning_rate": 3.653247421996326e-06, "loss": 0.75882566, "num_input_tokens_seen": 68294880, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.6640625, "step": 3166, "time_per_iteration": 2.3404548168182373 }, { "auxiliary_loss_clip": 0.01027155, "auxiliary_loss_mlp": 0.01025743, "balance_loss_clip": 1.02271545, "balance_loss_mlp": 1.0086807, "epoch": 0.1904103411994589, "flos": 66896168152320.0, "grad_norm": 0.7936556718366062, "language_loss": 0.50340271, "learning_rate": 3.65303477107666e-06, "loss": 0.52393174, "num_input_tokens_seen": 68359665, "router_z_loss_clip": 0.03027344, "router_z_loss_mlp": 0.18457031, "step": 3167, "time_per_iteration": 3.0473318099975586 }, { "auxiliary_loss_clip": 0.01092101, "auxiliary_loss_mlp": 0.01034347, "balance_loss_clip": 1.01715732, "balance_loss_mlp": 1.02729797, "epoch": 0.19047046445212687, "flos": 21907649030400.0, "grad_norm": 1.9924075328792246, "language_loss": 0.7409988, "learning_rate": 3.6528220611644356e-06, "loss": 0.7622633, "num_input_tokens_seen": 68378950, "router_z_loss_clip": 0.171875, "router_z_loss_mlp": 0.6484375, "step": 3168, "time_per_iteration": 2.402815103530884 }, { "auxiliary_loss_clip": 0.01024807, "auxiliary_loss_mlp": 0.0101354, "balance_loss_clip": 1.01065552, "balance_loss_mlp": 1.00624621, "epoch": 0.19053058770479483, "flos": 59252424595200.0, "grad_norm": 0.8669423342558235, "language_loss": 0.6008268, "learning_rate": 3.652609292267242e-06, "loss": 0.62121028, "num_input_tokens_seen": 68434235, "router_z_loss_clip": 0.02880859, "router_z_loss_mlp": 0.18554688, "step": 3169, "time_per_iteration": 2.9471940994262695 }, { "auxiliary_loss_clip": 0.01095435, "auxiliary_loss_mlp": 0.01041426, "balance_loss_clip": 1.02373552, "balance_loss_mlp": 1.0265286, "epoch": 0.1905907109574628, "flos": 23950172833920.0, "grad_norm": 1.6654320331704824, "language_loss": 0.78398848, "learning_rate": 3.6523964643926754e-06, "loss": 0.8053571, "num_input_tokens_seen": 68453830, "router_z_loss_clip": 0.17675781, "router_z_loss_mlp": 0.6875, "step": 3170, "time_per_iteration": 2.4139318466186523 }, { "auxiliary_loss_clip": 0.01089523, "auxiliary_loss_mlp": 0.01036114, "balance_loss_clip": 1.01806617, "balance_loss_mlp": 1.02465212, "epoch": 0.19065083421013077, "flos": 20811306516480.0, "grad_norm": 1.71655659388284, "language_loss": 0.78177553, "learning_rate": 3.6521835775483285e-06, "loss": 0.80303192, "num_input_tokens_seen": 68473005, "router_z_loss_clip": 0.18066406, "router_z_loss_mlp": 0.6484375, "step": 3171, "time_per_iteration": 2.3902087211608887 }, { "auxiliary_loss_clip": 0.01093806, "auxiliary_loss_mlp": 0.0103914, "balance_loss_clip": 1.01995897, "balance_loss_mlp": 1.02565539, "epoch": 0.19071095746279873, "flos": 31283644204800.0, "grad_norm": 2.0296539216697793, "language_loss": 0.77943277, "learning_rate": 3.6519706317417995e-06, "loss": 0.8007623, "num_input_tokens_seen": 68493470, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.6796875, "step": 3172, "time_per_iteration": 2.4620447158813477 }, { "auxiliary_loss_clip": 0.01093869, "auxiliary_loss_mlp": 0.0104006, "balance_loss_clip": 1.02184463, "balance_loss_mlp": 1.02689338, "epoch": 0.1907710807154667, "flos": 14355237323520.0, "grad_norm": 7.969753133433176, "language_loss": 0.80303502, "learning_rate": 3.6517576269806885e-06, "loss": 0.82437432, "num_input_tokens_seen": 68511290, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.66796875, "step": 3173, "time_per_iteration": 2.364328384399414 }, { "auxiliary_loss_clip": 0.01093834, "auxiliary_loss_mlp": 0.01049215, "balance_loss_clip": 1.03058267, "balance_loss_mlp": 1.02608895, "epoch": 0.1908312039681347, "flos": 26905815002880.0, "grad_norm": 1.5726743213791063, "language_loss": 0.78732854, "learning_rate": 3.651544563272597e-06, "loss": 0.80875897, "num_input_tokens_seen": 68532575, "router_z_loss_clip": 0.18652344, "router_z_loss_mlp": 0.6796875, "step": 3174, "time_per_iteration": 3.873528242111206 }, { "auxiliary_loss_clip": 0.0109644, "auxiliary_loss_mlp": 0.01044016, "balance_loss_clip": 1.02575302, "balance_loss_mlp": 1.02894258, "epoch": 0.19089132722080265, "flos": 14494017444480.0, "grad_norm": 2.560932419383946, "language_loss": 0.81298071, "learning_rate": 3.651331440625127e-06, "loss": 0.83438522, "num_input_tokens_seen": 68548760, "router_z_loss_clip": 0.18261719, "router_z_loss_mlp": 0.671875, "step": 3175, "time_per_iteration": 2.3709824085235596 }, { "auxiliary_loss_clip": 0.01095595, "auxiliary_loss_mlp": 0.01047697, "balance_loss_clip": 1.02912462, "balance_loss_mlp": 1.02780724, "epoch": 0.19095145047347062, "flos": 13952060409600.0, "grad_norm": 2.1343172854609658, "language_loss": 0.85423797, "learning_rate": 3.651118259045887e-06, "loss": 0.87567091, "num_input_tokens_seen": 68563100, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.67578125, "step": 3176, "time_per_iteration": 2.352125883102417 }, { "auxiliary_loss_clip": 0.01097972, "auxiliary_loss_mlp": 0.01050253, "balance_loss_clip": 1.0299871, "balance_loss_mlp": 1.02877474, "epoch": 0.19101157372613858, "flos": 25300648200960.0, "grad_norm": 1.9692737698191998, "language_loss": 0.81437957, "learning_rate": 3.650905018542483e-06, "loss": 0.8358618, "num_input_tokens_seen": 68581650, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.69140625, "step": 3177, "time_per_iteration": 3.792844533920288 }, { "auxiliary_loss_clip": 0.01092264, "auxiliary_loss_mlp": 0.01036427, "balance_loss_clip": 1.01786613, "balance_loss_mlp": 1.02616823, "epoch": 0.19107169697880655, "flos": 20557173663360.0, "grad_norm": 2.7706199197200676, "language_loss": 0.74712181, "learning_rate": 3.650691719122525e-06, "loss": 0.76840878, "num_input_tokens_seen": 68600360, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.6640625, "step": 3178, "time_per_iteration": 3.7311582565307617 }, { "auxiliary_loss_clip": 0.01094936, "auxiliary_loss_mlp": 0.01036207, "balance_loss_clip": 1.01819444, "balance_loss_mlp": 1.02852631, "epoch": 0.19113182023147451, "flos": 22162130997120.0, "grad_norm": 1.6771813715315846, "language_loss": 0.81381947, "learning_rate": 3.6504783607936266e-06, "loss": 0.83513093, "num_input_tokens_seen": 68617885, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.6640625, "step": 3179, "time_per_iteration": 2.387749671936035 }, { "auxiliary_loss_clip": 0.01095876, "auxiliary_loss_mlp": 0.01035741, "balance_loss_clip": 1.01747799, "balance_loss_mlp": 1.02783322, "epoch": 0.19119194348414248, "flos": 18580985176320.0, "grad_norm": 3.571695730743239, "language_loss": 0.80022579, "learning_rate": 3.6502649435634006e-06, "loss": 0.82154197, "num_input_tokens_seen": 68634550, "router_z_loss_clip": 0.18261719, "router_z_loss_mlp": 0.6796875, "step": 3180, "time_per_iteration": 3.7456233501434326 }, { "auxiliary_loss_clip": 0.01092087, "auxiliary_loss_mlp": 0.01042117, "balance_loss_clip": 1.02328157, "balance_loss_mlp": 1.02579308, "epoch": 0.19125206673681047, "flos": 19025603740800.0, "grad_norm": 2.4491028994390365, "language_loss": 0.79080421, "learning_rate": 3.6500514674394634e-06, "loss": 0.81214631, "num_input_tokens_seen": 68651895, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.6640625, "step": 3181, "time_per_iteration": 2.3571255207061768 }, { "auxiliary_loss_clip": 0.01094004, "auxiliary_loss_mlp": 0.01037111, "balance_loss_clip": 1.01886034, "balance_loss_mlp": 1.02600908, "epoch": 0.19131218998947844, "flos": 21689057808000.0, "grad_norm": 1.8849302621669406, "language_loss": 0.73793995, "learning_rate": 3.649837932429434e-06, "loss": 0.75925112, "num_input_tokens_seen": 68671500, "router_z_loss_clip": 0.18261719, "router_z_loss_mlp": 0.6796875, "step": 3182, "time_per_iteration": 2.392728090286255 }, { "auxiliary_loss_clip": 0.01094247, "auxiliary_loss_mlp": 0.01040668, "balance_loss_clip": 1.02185655, "balance_loss_mlp": 1.02753794, "epoch": 0.1913723132421464, "flos": 18441506828160.0, "grad_norm": 1.7585453217167473, "language_loss": 0.64951855, "learning_rate": 3.649624338540933e-06, "loss": 0.67086768, "num_input_tokens_seen": 68690570, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.6640625, "step": 3183, "time_per_iteration": 2.3653316497802734 }, { "auxiliary_loss_clip": 0.01092735, "auxiliary_loss_mlp": 0.01043071, "balance_loss_clip": 1.02306771, "balance_loss_mlp": 1.0258553, "epoch": 0.19143243649481437, "flos": 27158935426560.0, "grad_norm": 1.5167240814876268, "language_loss": 0.73595703, "learning_rate": 3.649410685781582e-06, "loss": 0.75731504, "num_input_tokens_seen": 68709735, "router_z_loss_clip": 0.20019531, "router_z_loss_mlp": 0.66796875, "step": 3184, "time_per_iteration": 2.432704210281372 }, { "auxiliary_loss_clip": 0.01092212, "auxiliary_loss_mlp": 0.01035316, "balance_loss_clip": 1.01520491, "balance_loss_mlp": 1.02478993, "epoch": 0.19149255974748233, "flos": 21718071014400.0, "grad_norm": 1.9895216461375365, "language_loss": 0.88315654, "learning_rate": 3.6491969741590075e-06, "loss": 0.90443182, "num_input_tokens_seen": 68727565, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.671875, "step": 3185, "time_per_iteration": 2.3759915828704834 }, { "auxiliary_loss_clip": 0.01092097, "auxiliary_loss_mlp": 0.01033265, "balance_loss_clip": 1.0140481, "balance_loss_mlp": 1.02576983, "epoch": 0.1915526830001503, "flos": 22962270602880.0, "grad_norm": 2.0860743889738442, "language_loss": 0.72633183, "learning_rate": 3.648983203680834e-06, "loss": 0.74758548, "num_input_tokens_seen": 68748110, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.6640625, "step": 3186, "time_per_iteration": 2.4003496170043945 }, { "auxiliary_loss_clip": 0.01096054, "auxiliary_loss_mlp": 0.0103823, "balance_loss_clip": 1.01659393, "balance_loss_mlp": 1.02652895, "epoch": 0.1916128062528183, "flos": 26139541282560.0, "grad_norm": 1.784433829999298, "language_loss": 0.83411252, "learning_rate": 3.6487693743546927e-06, "loss": 0.8554554, "num_input_tokens_seen": 68769765, "router_z_loss_clip": 0.21679688, "router_z_loss_mlp": 0.6953125, "step": 3187, "time_per_iteration": 2.4265408515930176 }, { "auxiliary_loss_clip": 0.01027159, "auxiliary_loss_mlp": 0.01014839, "balance_loss_clip": 1.01260972, "balance_loss_mlp": 1.00835872, "epoch": 0.19167292950548626, "flos": 54922809847680.0, "grad_norm": 0.853340881450663, "language_loss": 0.55857521, "learning_rate": 3.648555486188213e-06, "loss": 0.57899523, "num_input_tokens_seen": 68826815, "router_z_loss_clip": 0.02233887, "router_z_loss_mlp": 0.1875, "step": 3188, "time_per_iteration": 3.071138858795166 }, { "auxiliary_loss_clip": 0.01093198, "auxiliary_loss_mlp": 0.01037988, "balance_loss_clip": 1.01933169, "balance_loss_mlp": 1.02743053, "epoch": 0.19173305275815422, "flos": 29934286001280.0, "grad_norm": 1.6054637380264414, "language_loss": 0.70125937, "learning_rate": 3.648341539189029e-06, "loss": 0.72257119, "num_input_tokens_seen": 68847585, "router_z_loss_clip": 0.18652344, "router_z_loss_mlp": 0.65625, "step": 3189, "time_per_iteration": 2.4534246921539307 }, { "auxiliary_loss_clip": 0.01088219, "auxiliary_loss_mlp": 0.01031638, "balance_loss_clip": 1.01462686, "balance_loss_mlp": 1.02538657, "epoch": 0.1917931760108222, "flos": 24751359780480.0, "grad_norm": 3.424436763277206, "language_loss": 0.74134934, "learning_rate": 3.648127533364775e-06, "loss": 0.76254797, "num_input_tokens_seen": 68866620, "router_z_loss_clip": 0.16992188, "router_z_loss_mlp": 0.62890625, "step": 3190, "time_per_iteration": 2.4190168380737305 }, { "auxiliary_loss_clip": 0.01092916, "auxiliary_loss_mlp": 0.01048443, "balance_loss_clip": 1.02938151, "balance_loss_mlp": 1.02776313, "epoch": 0.19185329926349015, "flos": 18842554149120.0, "grad_norm": 3.6163422025086005, "language_loss": 0.8435185, "learning_rate": 3.6479134687230887e-06, "loss": 0.86493206, "num_input_tokens_seen": 68885515, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.6484375, "step": 3191, "time_per_iteration": 2.377803087234497 }, { "auxiliary_loss_clip": 0.01090419, "auxiliary_loss_mlp": 0.01032266, "balance_loss_clip": 1.01468277, "balance_loss_mlp": 1.0269171, "epoch": 0.19191342251615812, "flos": 22085880854400.0, "grad_norm": 1.8367942314446566, "language_loss": 0.89690745, "learning_rate": 3.64769934527161e-06, "loss": 0.91813433, "num_input_tokens_seen": 68903225, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.6328125, "step": 3192, "time_per_iteration": 2.38694167137146 }, { "auxiliary_loss_clip": 0.01095632, "auxiliary_loss_mlp": 0.01041773, "balance_loss_clip": 1.02150786, "balance_loss_mlp": 1.02851701, "epoch": 0.19197354576882608, "flos": 22198056652800.0, "grad_norm": 1.7812500685174586, "language_loss": 0.74489391, "learning_rate": 3.64748516301798e-06, "loss": 0.7662679, "num_input_tokens_seen": 68922860, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.671875, "step": 3193, "time_per_iteration": 2.389866352081299 }, { "auxiliary_loss_clip": 0.01096063, "auxiliary_loss_mlp": 0.0103819, "balance_loss_clip": 1.01852024, "balance_loss_mlp": 1.02679372, "epoch": 0.19203366902149407, "flos": 24895132225920.0, "grad_norm": 1.7237964102702663, "language_loss": 0.7463479, "learning_rate": 3.6472709219698422e-06, "loss": 0.76769042, "num_input_tokens_seen": 68943000, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.69140625, "step": 3194, "time_per_iteration": 2.419905185699463 }, { "auxiliary_loss_clip": 0.01022737, "auxiliary_loss_mlp": 0.01002894, "balance_loss_clip": 1.00068891, "balance_loss_mlp": 1.00399566, "epoch": 0.19209379227416204, "flos": 68413633885440.0, "grad_norm": 0.7852784648745245, "language_loss": 0.68454325, "learning_rate": 3.647056622134843e-06, "loss": 0.70479953, "num_input_tokens_seen": 69000255, "router_z_loss_clip": 0.02209473, "router_z_loss_mlp": 0.1875, "step": 3195, "time_per_iteration": 2.9521589279174805 }, { "auxiliary_loss_clip": 0.0109407, "auxiliary_loss_mlp": 0.01042562, "balance_loss_clip": 1.02328563, "balance_loss_mlp": 1.02698195, "epoch": 0.19215391552683, "flos": 22054074739200.0, "grad_norm": 2.5447857934723115, "language_loss": 0.72515213, "learning_rate": 3.6468422635206297e-06, "loss": 0.74651849, "num_input_tokens_seen": 69019665, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.671875, "step": 3196, "time_per_iteration": 2.3983092308044434 }, { "auxiliary_loss_clip": 0.01096925, "auxiliary_loss_mlp": 0.01042756, "balance_loss_clip": 1.02365923, "balance_loss_mlp": 1.03062785, "epoch": 0.19221403877949797, "flos": 20301923646720.0, "grad_norm": 2.1716943994036444, "language_loss": 0.83250105, "learning_rate": 3.6466278461348514e-06, "loss": 0.85389781, "num_input_tokens_seen": 69039055, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.6640625, "step": 3197, "time_per_iteration": 2.3770110607147217 }, { "auxiliary_loss_clip": 0.01092871, "auxiliary_loss_mlp": 0.01035991, "balance_loss_clip": 1.0170486, "balance_loss_mlp": 1.02595544, "epoch": 0.19227416203216594, "flos": 23184213315840.0, "grad_norm": 2.105888060625776, "language_loss": 0.80370164, "learning_rate": 3.646413369985161e-06, "loss": 0.82499027, "num_input_tokens_seen": 69056370, "router_z_loss_clip": 0.18945312, "router_z_loss_mlp": 0.66796875, "step": 3198, "time_per_iteration": 2.3984158039093018 }, { "auxiliary_loss_clip": 0.01094667, "auxiliary_loss_mlp": 0.01039524, "balance_loss_clip": 1.01867473, "balance_loss_mlp": 1.02617788, "epoch": 0.1923342852848339, "flos": 25775397135360.0, "grad_norm": 2.109748742392438, "language_loss": 0.78280067, "learning_rate": 3.6461988350792137e-06, "loss": 0.80414265, "num_input_tokens_seen": 69075915, "router_z_loss_clip": 0.20800781, "router_z_loss_mlp": 0.6875, "step": 3199, "time_per_iteration": 2.41072940826416 }, { "auxiliary_loss_clip": 0.01094885, "auxiliary_loss_mlp": 0.01036543, "balance_loss_clip": 1.0186615, "balance_loss_mlp": 1.02991319, "epoch": 0.19239440853750187, "flos": 17127410964480.0, "grad_norm": 2.4234828034627993, "language_loss": 0.83533007, "learning_rate": 3.6459842414246636e-06, "loss": 0.85664433, "num_input_tokens_seen": 69094145, "router_z_loss_clip": 0.17871094, "router_z_loss_mlp": 0.6484375, "step": 3200, "time_per_iteration": 2.351614475250244 }, { "auxiliary_loss_clip": 0.01095715, "auxiliary_loss_mlp": 0.01044843, "balance_loss_clip": 1.02640176, "balance_loss_mlp": 1.02879262, "epoch": 0.19245453179016986, "flos": 16434175541760.0, "grad_norm": 2.052054101014935, "language_loss": 0.79116702, "learning_rate": 3.6457695890291697e-06, "loss": 0.8125726, "num_input_tokens_seen": 69111110, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.66796875, "step": 3201, "time_per_iteration": 2.348501682281494 }, { "auxiliary_loss_clip": 0.01095271, "auxiliary_loss_mlp": 0.01037754, "balance_loss_clip": 1.01831102, "balance_loss_mlp": 1.02722049, "epoch": 0.19251465504283782, "flos": 20229234462720.0, "grad_norm": 2.270429026092637, "language_loss": 0.69541204, "learning_rate": 3.645554877900393e-06, "loss": 0.71674228, "num_input_tokens_seen": 69130280, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.6796875, "step": 3202, "time_per_iteration": 2.384735584259033 }, { "auxiliary_loss_clip": 0.01092863, "auxiliary_loss_mlp": 0.01032726, "balance_loss_clip": 1.01412892, "balance_loss_mlp": 1.02728343, "epoch": 0.1925747782955058, "flos": 19463344767360.0, "grad_norm": 2.5595947348008443, "language_loss": 0.91117144, "learning_rate": 3.645340108045995e-06, "loss": 0.93242729, "num_input_tokens_seen": 69149570, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.65625, "step": 3203, "time_per_iteration": 2.3753819465637207 }, { "auxiliary_loss_clip": 0.01094895, "auxiliary_loss_mlp": 0.01042478, "balance_loss_clip": 1.02277279, "balance_loss_mlp": 1.02658033, "epoch": 0.19263490154817375, "flos": 17784615997440.0, "grad_norm": 1.9669752936168026, "language_loss": 0.81680238, "learning_rate": 3.6451252794736417e-06, "loss": 0.83817607, "num_input_tokens_seen": 69168190, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.6796875, "step": 3204, "time_per_iteration": 2.368269920349121 }, { "auxiliary_loss_clip": 0.01092571, "auxiliary_loss_mlp": 0.01039142, "balance_loss_clip": 1.02041459, "balance_loss_mlp": 1.02673435, "epoch": 0.19269502480084172, "flos": 17456118215040.0, "grad_norm": 1.988349642468062, "language_loss": 0.75792122, "learning_rate": 3.6449103921909983e-06, "loss": 0.77923828, "num_input_tokens_seen": 69186950, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.66015625, "step": 3205, "time_per_iteration": 2.3562424182891846 }, { "auxiliary_loss_clip": 0.01096497, "auxiliary_loss_mlp": 0.01039308, "balance_loss_clip": 1.02030611, "balance_loss_mlp": 1.02881098, "epoch": 0.19275514805350968, "flos": 21505833659520.0, "grad_norm": 2.9249157336018707, "language_loss": 0.82801032, "learning_rate": 3.644695446205735e-06, "loss": 0.84936833, "num_input_tokens_seen": 69204850, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.67578125, "step": 3206, "time_per_iteration": 2.388993740081787 }, { "auxiliary_loss_clip": 0.01024912, "auxiliary_loss_mlp": 0.01007991, "balance_loss_clip": 1.00558305, "balance_loss_mlp": 1.00593722, "epoch": 0.19281527130617768, "flos": 47693379928320.0, "grad_norm": 0.8365671296608214, "language_loss": 0.60553396, "learning_rate": 3.644480441525521e-06, "loss": 0.62586296, "num_input_tokens_seen": 69259200, "router_z_loss_clip": 0.02404785, "router_z_loss_mlp": 0.18945312, "step": 3207, "time_per_iteration": 2.855283260345459 }, { "auxiliary_loss_clip": 0.01092823, "auxiliary_loss_mlp": 0.0103696, "balance_loss_clip": 1.01725471, "balance_loss_mlp": 1.02535605, "epoch": 0.19287539455884564, "flos": 11800467918720.0, "grad_norm": 5.028053307957577, "language_loss": 0.74671447, "learning_rate": 3.6442653781580305e-06, "loss": 0.76801234, "num_input_tokens_seen": 69275835, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.671875, "step": 3208, "time_per_iteration": 2.3509304523468018 }, { "auxiliary_loss_clip": 0.01094353, "auxiliary_loss_mlp": 0.01039326, "balance_loss_clip": 1.01995444, "balance_loss_mlp": 1.02553701, "epoch": 0.1929355178115136, "flos": 20630386517760.0, "grad_norm": 2.0997662987306325, "language_loss": 0.60876942, "learning_rate": 3.6440502561109384e-06, "loss": 0.63010621, "num_input_tokens_seen": 69294810, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.6875, "step": 3209, "time_per_iteration": 2.3633580207824707 }, { "auxiliary_loss_clip": 0.01096845, "auxiliary_loss_mlp": 0.01043453, "balance_loss_clip": 1.02256715, "balance_loss_mlp": 1.02674937, "epoch": 0.19299564106418157, "flos": 40806309467520.0, "grad_norm": 1.9447056824487978, "language_loss": 0.7999202, "learning_rate": 3.6438350753919213e-06, "loss": 0.82132316, "num_input_tokens_seen": 69316065, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.69921875, "step": 3210, "time_per_iteration": 2.5475172996520996 }, { "auxiliary_loss_clip": 0.01088278, "auxiliary_loss_mlp": 0.01037149, "balance_loss_clip": 1.01960135, "balance_loss_mlp": 1.02380741, "epoch": 0.19305576431684954, "flos": 11360702033280.0, "grad_norm": 2.2666304209789923, "language_loss": 0.82905734, "learning_rate": 3.643619836008659e-06, "loss": 0.85031164, "num_input_tokens_seen": 69332900, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.64453125, "step": 3211, "time_per_iteration": 2.3530471324920654 }, { "auxiliary_loss_clip": 0.01021493, "auxiliary_loss_mlp": 0.01002706, "balance_loss_clip": 1.00033367, "balance_loss_mlp": 1.00280476, "epoch": 0.1931158875695175, "flos": 54509299171200.0, "grad_norm": 0.9651194982063522, "language_loss": 0.63612223, "learning_rate": 3.6434045379688324e-06, "loss": 0.6563642, "num_input_tokens_seen": 69382535, "router_z_loss_clip": 0.02368164, "router_z_loss_mlp": 0.18652344, "step": 3212, "time_per_iteration": 2.9029247760772705 }, { "auxiliary_loss_clip": 0.01093502, "auxiliary_loss_mlp": 0.01041722, "balance_loss_clip": 1.02342355, "balance_loss_mlp": 1.02697301, "epoch": 0.19317601082218547, "flos": 19827419091840.0, "grad_norm": 1.7286706440858817, "language_loss": 0.76006323, "learning_rate": 3.6431891812801254e-06, "loss": 0.78141546, "num_input_tokens_seen": 69400600, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.6640625, "step": 3213, "time_per_iteration": 2.38496732711792 }, { "auxiliary_loss_clip": 0.01096553, "auxiliary_loss_mlp": 0.01041368, "balance_loss_clip": 1.02217579, "balance_loss_mlp": 1.02772832, "epoch": 0.19323613407485346, "flos": 13151222576640.0, "grad_norm": 2.0194626720413957, "language_loss": 0.71029568, "learning_rate": 3.6429737659502237e-06, "loss": 0.73167491, "num_input_tokens_seen": 69417350, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.6875, "step": 3214, "time_per_iteration": 3.7430498600006104 }, { "auxiliary_loss_clip": 0.01092736, "auxiliary_loss_mlp": 0.0103565, "balance_loss_clip": 1.01599288, "balance_loss_mlp": 1.02590609, "epoch": 0.19329625732752143, "flos": 14026390427520.0, "grad_norm": 2.04791721415141, "language_loss": 0.74819297, "learning_rate": 3.642758291986814e-06, "loss": 0.76947683, "num_input_tokens_seen": 69431845, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.66796875, "step": 3215, "time_per_iteration": 2.345534086227417 }, { "auxiliary_loss_clip": 0.01089553, "auxiliary_loss_mlp": 0.01040103, "balance_loss_clip": 1.02125573, "balance_loss_mlp": 1.02407169, "epoch": 0.1933563805801894, "flos": 23440580496000.0, "grad_norm": 3.8363575180347804, "language_loss": 0.88652748, "learning_rate": 3.642542759397587e-06, "loss": 0.90782398, "num_input_tokens_seen": 69453275, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.65625, "step": 3216, "time_per_iteration": 2.4154319763183594 }, { "auxiliary_loss_clip": 0.01092795, "auxiliary_loss_mlp": 0.01040408, "balance_loss_clip": 1.02185881, "balance_loss_mlp": 1.0269376, "epoch": 0.19341650383285736, "flos": 20484275011200.0, "grad_norm": 1.7398186088472865, "language_loss": 0.80092424, "learning_rate": 3.6423271681902336e-06, "loss": 0.82225633, "num_input_tokens_seen": 69471830, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.65625, "step": 3217, "time_per_iteration": 3.7727136611938477 }, { "auxiliary_loss_clip": 0.01096014, "auxiliary_loss_mlp": 0.01038216, "balance_loss_clip": 1.01750958, "balance_loss_mlp": 1.02631855, "epoch": 0.19347662708552532, "flos": 17857514649600.0, "grad_norm": 2.3762710878227398, "language_loss": 0.61644971, "learning_rate": 3.642111518372448e-06, "loss": 0.63779199, "num_input_tokens_seen": 69489320, "router_z_loss_clip": 0.20703125, "router_z_loss_mlp": 0.6953125, "step": 3218, "time_per_iteration": 3.738217830657959 }, { "auxiliary_loss_clip": 0.01093309, "auxiliary_loss_mlp": 0.01041184, "balance_loss_clip": 1.0221107, "balance_loss_mlp": 1.02671599, "epoch": 0.1935367503381933, "flos": 18186256811520.0, "grad_norm": 1.9905440332339441, "language_loss": 0.80267423, "learning_rate": 3.6418958099519267e-06, "loss": 0.82401913, "num_input_tokens_seen": 69506665, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.6640625, "step": 3219, "time_per_iteration": 3.819187641143799 }, { "auxiliary_loss_clip": 0.01094091, "auxiliary_loss_mlp": 0.01043146, "balance_loss_clip": 1.02382195, "balance_loss_mlp": 1.02746654, "epoch": 0.19359687359086128, "flos": 15956319496320.0, "grad_norm": 2.4496600917349647, "language_loss": 0.85869569, "learning_rate": 3.6416800429363674e-06, "loss": 0.88006806, "num_input_tokens_seen": 69523835, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.66796875, "step": 3220, "time_per_iteration": 2.361116886138916 }, { "auxiliary_loss_clip": 0.01088917, "auxiliary_loss_mlp": 0.01035224, "balance_loss_clip": 1.01898813, "balance_loss_mlp": 1.02643895, "epoch": 0.19365699684352924, "flos": 21214134316800.0, "grad_norm": 3.4607468513524915, "language_loss": 0.84419346, "learning_rate": 3.6414642173334704e-06, "loss": 0.86543494, "num_input_tokens_seen": 69542620, "router_z_loss_clip": 0.16210938, "router_z_loss_mlp": 0.625, "step": 3221, "time_per_iteration": 2.382223129272461 }, { "auxiliary_loss_clip": 0.01092355, "auxiliary_loss_mlp": 0.01036781, "balance_loss_clip": 1.01993644, "balance_loss_mlp": 1.0285064, "epoch": 0.1937171200961972, "flos": 17310146353920.0, "grad_norm": 2.191763726116518, "language_loss": 0.86122036, "learning_rate": 3.6412483331509373e-06, "loss": 0.88251173, "num_input_tokens_seen": 69561130, "router_z_loss_clip": 0.16894531, "router_z_loss_mlp": 0.640625, "step": 3222, "time_per_iteration": 2.412787437438965 }, { "auxiliary_loss_clip": 0.0109185, "auxiliary_loss_mlp": 0.01032477, "balance_loss_clip": 1.01404715, "balance_loss_mlp": 1.02545762, "epoch": 0.19377724334886517, "flos": 22634924895360.0, "grad_norm": 2.525734028275212, "language_loss": 0.78428602, "learning_rate": 3.641032390396473e-06, "loss": 0.80552936, "num_input_tokens_seen": 69580425, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.6640625, "step": 3223, "time_per_iteration": 2.4569711685180664 }, { "auxiliary_loss_clip": 0.01090868, "auxiliary_loss_mlp": 0.01034828, "balance_loss_clip": 1.01762629, "balance_loss_mlp": 1.02666807, "epoch": 0.19383736660153314, "flos": 15077136839040.0, "grad_norm": 2.1340963987027926, "language_loss": 0.75416589, "learning_rate": 3.6408163890777843e-06, "loss": 0.77542287, "num_input_tokens_seen": 69597085, "router_z_loss_clip": 0.171875, "router_z_loss_mlp": 0.640625, "step": 3224, "time_per_iteration": 2.358529567718506 }, { "auxiliary_loss_clip": 0.01090426, "auxiliary_loss_mlp": 0.01031402, "balance_loss_clip": 1.01259124, "balance_loss_mlp": 1.02646494, "epoch": 0.1938974898542011, "flos": 47118152367360.0, "grad_norm": 2.2078840050057473, "language_loss": 0.70660877, "learning_rate": 3.640600329202579e-06, "loss": 0.72782701, "num_input_tokens_seen": 69618885, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.640625, "step": 3225, "time_per_iteration": 2.617093563079834 }, { "auxiliary_loss_clip": 0.01090086, "auxiliary_loss_mlp": 0.01033062, "balance_loss_clip": 1.01528764, "balance_loss_mlp": 1.02510476, "epoch": 0.19395761310686907, "flos": 25811357702400.0, "grad_norm": 2.4472171369842837, "language_loss": 0.69760823, "learning_rate": 3.6403842107785686e-06, "loss": 0.71883965, "num_input_tokens_seen": 69638200, "router_z_loss_clip": 0.17773438, "router_z_loss_mlp": 0.6484375, "step": 3226, "time_per_iteration": 2.4120709896087646 }, { "auxiliary_loss_clip": 0.01091227, "auxiliary_loss_mlp": 0.01039145, "balance_loss_clip": 1.02029812, "balance_loss_mlp": 1.02752805, "epoch": 0.19401773635953706, "flos": 23038485834240.0, "grad_norm": 1.6591788837545542, "language_loss": 0.76039732, "learning_rate": 3.6401680338134653e-06, "loss": 0.78170109, "num_input_tokens_seen": 69657550, "router_z_loss_clip": 0.18847656, "router_z_loss_mlp": 0.63671875, "step": 3227, "time_per_iteration": 2.3980982303619385 }, { "auxiliary_loss_clip": 0.01090895, "auxiliary_loss_mlp": 0.01040711, "balance_loss_clip": 1.02276993, "balance_loss_mlp": 1.02430868, "epoch": 0.19407785961220503, "flos": 15919974904320.0, "grad_norm": 1.8649498984843145, "language_loss": 0.69280744, "learning_rate": 3.6399517983149838e-06, "loss": 0.71412349, "num_input_tokens_seen": 69675005, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.66796875, "step": 3228, "time_per_iteration": 2.3586676120758057 }, { "auxiliary_loss_clip": 0.01091473, "auxiliary_loss_mlp": 0.01043592, "balance_loss_clip": 1.02537704, "balance_loss_mlp": 1.02686977, "epoch": 0.194137982864873, "flos": 25920531123840.0, "grad_norm": 2.220200115653601, "language_loss": 0.74391913, "learning_rate": 3.6397355042908407e-06, "loss": 0.76526976, "num_input_tokens_seen": 69696455, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.64453125, "step": 3229, "time_per_iteration": 2.4335381984710693 }, { "auxiliary_loss_clip": 0.01091645, "auxiliary_loss_mlp": 0.01033087, "balance_loss_clip": 1.01598048, "balance_loss_mlp": 1.02630305, "epoch": 0.19419810611754096, "flos": 13260500732160.0, "grad_norm": 2.4094008360195143, "language_loss": 0.65313721, "learning_rate": 3.6395191517487557e-06, "loss": 0.67438447, "num_input_tokens_seen": 69714245, "router_z_loss_clip": 0.17089844, "router_z_loss_mlp": 0.65234375, "step": 3230, "time_per_iteration": 2.3724405765533447 }, { "auxiliary_loss_clip": 0.01089027, "auxiliary_loss_mlp": 0.01036275, "balance_loss_clip": 1.01831055, "balance_loss_mlp": 1.02513218, "epoch": 0.19425822937020892, "flos": 15704665349760.0, "grad_norm": 1.9126845229967357, "language_loss": 0.82243401, "learning_rate": 3.6393027406964494e-06, "loss": 0.84368706, "num_input_tokens_seen": 69731515, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.640625, "step": 3231, "time_per_iteration": 2.372706174850464 }, { "auxiliary_loss_clip": 0.01093246, "auxiliary_loss_mlp": 0.01034572, "balance_loss_clip": 1.01459241, "balance_loss_mlp": 1.02733052, "epoch": 0.1943183526228769, "flos": 23104472037120.0, "grad_norm": 1.882908253566389, "language_loss": 0.87074304, "learning_rate": 3.639086271141645e-06, "loss": 0.89202118, "num_input_tokens_seen": 69748885, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.66015625, "step": 3232, "time_per_iteration": 2.3734681606292725 }, { "auxiliary_loss_clip": 0.01092436, "auxiliary_loss_mlp": 0.01037586, "balance_loss_clip": 1.01937103, "balance_loss_mlp": 1.02718019, "epoch": 0.19437847587554485, "flos": 24711593875200.0, "grad_norm": 1.7653542276973573, "language_loss": 0.85239351, "learning_rate": 3.6388697430920674e-06, "loss": 0.8736937, "num_input_tokens_seen": 69767540, "router_z_loss_clip": 0.18261719, "router_z_loss_mlp": 0.65234375, "step": 3233, "time_per_iteration": 2.4153127670288086 }, { "auxiliary_loss_clip": 0.01093505, "auxiliary_loss_mlp": 0.01042232, "balance_loss_clip": 1.02377868, "balance_loss_mlp": 1.02482581, "epoch": 0.19443859912821285, "flos": 23114910533760.0, "grad_norm": 1.7319351285692142, "language_loss": 0.88985711, "learning_rate": 3.638653156555445e-06, "loss": 0.91121447, "num_input_tokens_seen": 69789340, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.6875, "step": 3234, "time_per_iteration": 2.4098269939422607 }, { "auxiliary_loss_clip": 0.01091998, "auxiliary_loss_mlp": 0.01033936, "balance_loss_clip": 1.01485085, "balance_loss_mlp": 1.02412546, "epoch": 0.1944987223808808, "flos": 15083525617920.0, "grad_norm": 5.7409567116605515, "language_loss": 0.78201854, "learning_rate": 3.638436511539507e-06, "loss": 0.80327791, "num_input_tokens_seen": 69806470, "router_z_loss_clip": 0.19042969, "router_z_loss_mlp": 0.6796875, "step": 3235, "time_per_iteration": 2.35680890083313 }, { "auxiliary_loss_clip": 0.01092563, "auxiliary_loss_mlp": 0.01034089, "balance_loss_clip": 1.01667261, "balance_loss_mlp": 1.02676952, "epoch": 0.19455884563354878, "flos": 17125979598720.0, "grad_norm": 1.953177095274907, "language_loss": 0.79242563, "learning_rate": 3.6382198080519833e-06, "loss": 0.81369209, "num_input_tokens_seen": 69822655, "router_z_loss_clip": 0.17382812, "router_z_loss_mlp": 0.66015625, "step": 3236, "time_per_iteration": 2.345677614212036 }, { "auxiliary_loss_clip": 0.01090901, "auxiliary_loss_mlp": 0.01037931, "balance_loss_clip": 1.01946533, "balance_loss_mlp": 1.02475131, "epoch": 0.19461896888621674, "flos": 20192366200320.0, "grad_norm": 1.503568861209779, "language_loss": 0.7555871, "learning_rate": 3.6380030461006093e-06, "loss": 0.77687538, "num_input_tokens_seen": 69841895, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.6640625, "step": 3237, "time_per_iteration": 2.3915979862213135 }, { "auxiliary_loss_clip": 0.01092935, "auxiliary_loss_mlp": 0.01038811, "balance_loss_clip": 1.02026165, "balance_loss_mlp": 1.02521873, "epoch": 0.1946790921388847, "flos": 25300194353280.0, "grad_norm": 1.5232704083913822, "language_loss": 0.75017565, "learning_rate": 3.6377862256931203e-06, "loss": 0.77149314, "num_input_tokens_seen": 69862220, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.67578125, "step": 3238, "time_per_iteration": 2.416321039199829 }, { "auxiliary_loss_clip": 0.01094228, "auxiliary_loss_mlp": 0.01040916, "balance_loss_clip": 1.02069783, "balance_loss_mlp": 1.02699256, "epoch": 0.19473921539155267, "flos": 20192366200320.0, "grad_norm": 1.9690252501932926, "language_loss": 0.73038596, "learning_rate": 3.637569346837253e-06, "loss": 0.75173736, "num_input_tokens_seen": 69881830, "router_z_loss_clip": 0.20214844, "router_z_loss_mlp": 0.671875, "step": 3239, "time_per_iteration": 2.3719887733459473 }, { "auxiliary_loss_clip": 0.01091792, "auxiliary_loss_mlp": 0.01038523, "balance_loss_clip": 1.02036738, "balance_loss_mlp": 1.02522206, "epoch": 0.19479933864422067, "flos": 20886474407040.0, "grad_norm": 1.7556989267650014, "language_loss": 0.7344541, "learning_rate": 3.6373524095407485e-06, "loss": 0.75575721, "num_input_tokens_seen": 69900515, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.66796875, "step": 3240, "time_per_iteration": 2.3777613639831543 }, { "auxiliary_loss_clip": 0.01091297, "auxiliary_loss_mlp": 0.01034383, "balance_loss_clip": 1.01701427, "balance_loss_mlp": 1.02552462, "epoch": 0.19485946189688863, "flos": 23293945319040.0, "grad_norm": 1.9485447126523352, "language_loss": 0.66402727, "learning_rate": 3.637135413811348e-06, "loss": 0.68528414, "num_input_tokens_seen": 69920060, "router_z_loss_clip": 0.17382812, "router_z_loss_mlp": 0.65625, "step": 3241, "time_per_iteration": 2.399484395980835 }, { "auxiliary_loss_clip": 0.0109156, "auxiliary_loss_mlp": 0.01037781, "balance_loss_clip": 1.01962543, "balance_loss_mlp": 1.02629566, "epoch": 0.1949195851495566, "flos": 23293910407680.0, "grad_norm": 1.9349221812557778, "language_loss": 0.8284806, "learning_rate": 3.636918359656796e-06, "loss": 0.84977406, "num_input_tokens_seen": 69939820, "router_z_loss_clip": 0.18164062, "router_z_loss_mlp": 0.65234375, "step": 3242, "time_per_iteration": 2.4018092155456543 }, { "auxiliary_loss_clip": 0.01023619, "auxiliary_loss_mlp": 0.01003701, "balance_loss_clip": 1.00105453, "balance_loss_mlp": 1.00448895, "epoch": 0.19497970840222456, "flos": 64959536102400.0, "grad_norm": 0.8196101855354372, "language_loss": 0.57456034, "learning_rate": 3.636701247084839e-06, "loss": 0.59483355, "num_input_tokens_seen": 70002145, "router_z_loss_clip": 0.02648926, "router_z_loss_mlp": 0.19140625, "step": 3243, "time_per_iteration": 3.0555503368377686 }, { "auxiliary_loss_clip": 0.01095224, "auxiliary_loss_mlp": 0.01039058, "balance_loss_clip": 1.0204618, "balance_loss_mlp": 1.0275383, "epoch": 0.19503983165489253, "flos": 19643741095680.0, "grad_norm": 2.016429930690371, "language_loss": 0.83302236, "learning_rate": 3.6364840761032238e-06, "loss": 0.85436511, "num_input_tokens_seen": 70020510, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.67578125, "step": 3244, "time_per_iteration": 2.3813095092773438 }, { "auxiliary_loss_clip": 0.01094253, "auxiliary_loss_mlp": 0.01035704, "balance_loss_clip": 1.01697576, "balance_loss_mlp": 1.02831161, "epoch": 0.1950999549075605, "flos": 21140921462400.0, "grad_norm": 1.6515982101360513, "language_loss": 0.7687943, "learning_rate": 3.6362668467197015e-06, "loss": 0.7900939, "num_input_tokens_seen": 70040760, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.65625, "step": 3245, "time_per_iteration": 2.3906078338623047 }, { "auxiliary_loss_clip": 0.01093757, "auxiliary_loss_mlp": 0.01038342, "balance_loss_clip": 1.01860094, "balance_loss_mlp": 1.02652812, "epoch": 0.19516007816022846, "flos": 20883821143680.0, "grad_norm": 1.9215117662172279, "language_loss": 0.84480739, "learning_rate": 3.6360495589420247e-06, "loss": 0.86612833, "num_input_tokens_seen": 70058720, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.671875, "step": 3246, "time_per_iteration": 2.376635789871216 }, { "auxiliary_loss_clip": 0.0109496, "auxiliary_loss_mlp": 0.01038014, "balance_loss_clip": 1.01931024, "balance_loss_mlp": 1.02649212, "epoch": 0.19522020141289645, "flos": 16909552880640.0, "grad_norm": 2.0241418147502372, "language_loss": 0.75633973, "learning_rate": 3.6358322127779476e-06, "loss": 0.77766943, "num_input_tokens_seen": 70076470, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.68359375, "step": 3247, "time_per_iteration": 2.3652443885803223 }, { "auxiliary_loss_clip": 0.01097248, "auxiliary_loss_mlp": 0.01035461, "balance_loss_clip": 1.01579106, "balance_loss_mlp": 1.02903318, "epoch": 0.19528032466556441, "flos": 26723603283840.0, "grad_norm": 1.8681365272471933, "language_loss": 0.75390351, "learning_rate": 3.6356148082352265e-06, "loss": 0.77523059, "num_input_tokens_seen": 70096220, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.6796875, "step": 3248, "time_per_iteration": 2.444396495819092 }, { "auxiliary_loss_clip": 0.01093291, "auxiliary_loss_mlp": 0.01037908, "balance_loss_clip": 1.018417, "balance_loss_mlp": 1.0265578, "epoch": 0.19534044791823238, "flos": 21031748040960.0, "grad_norm": 2.045327479393052, "language_loss": 0.78500307, "learning_rate": 3.63539734532162e-06, "loss": 0.80631506, "num_input_tokens_seen": 70114800, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.6640625, "step": 3249, "time_per_iteration": 2.3759329319000244 }, { "auxiliary_loss_clip": 0.0109475, "auxiliary_loss_mlp": 0.01036823, "balance_loss_clip": 1.01817822, "balance_loss_mlp": 1.02709186, "epoch": 0.19540057117090034, "flos": 22343016084480.0, "grad_norm": 1.5481371088008538, "language_loss": 0.72917652, "learning_rate": 3.6351798240448894e-06, "loss": 0.75049222, "num_input_tokens_seen": 70134930, "router_z_loss_clip": 0.18652344, "router_z_loss_mlp": 0.67578125, "step": 3250, "time_per_iteration": 2.3821792602539062 }, { "auxiliary_loss_clip": 0.01091139, "auxiliary_loss_mlp": 0.01036574, "balance_loss_clip": 1.01860952, "balance_loss_mlp": 1.02606761, "epoch": 0.1954606944235683, "flos": 20300631926400.0, "grad_norm": 2.077856386507019, "language_loss": 0.79381561, "learning_rate": 3.634962244412797e-06, "loss": 0.8150928, "num_input_tokens_seen": 70152045, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.65234375, "step": 3251, "time_per_iteration": 2.3681230545043945 }, { "auxiliary_loss_clip": 0.01094269, "auxiliary_loss_mlp": 0.01043514, "balance_loss_clip": 1.02563214, "balance_loss_mlp": 1.02753401, "epoch": 0.19552081767623627, "flos": 17345932364160.0, "grad_norm": 4.307334667439784, "language_loss": 0.83700019, "learning_rate": 3.6347446064331074e-06, "loss": 0.85837805, "num_input_tokens_seen": 70169240, "router_z_loss_clip": 0.17871094, "router_z_loss_mlp": 0.66796875, "step": 3252, "time_per_iteration": 2.3527212142944336 }, { "auxiliary_loss_clip": 0.01096735, "auxiliary_loss_mlp": 0.01046693, "balance_loss_clip": 1.02541399, "balance_loss_mlp": 1.02695727, "epoch": 0.19558094092890424, "flos": 31976286134400.0, "grad_norm": 1.8757880103444917, "language_loss": 0.73458648, "learning_rate": 3.6345269101135885e-06, "loss": 0.75602067, "num_input_tokens_seen": 70192690, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.69921875, "step": 3253, "time_per_iteration": 2.4764764308929443 }, { "auxiliary_loss_clip": 0.01094574, "auxiliary_loss_mlp": 0.01037477, "balance_loss_clip": 1.01700842, "balance_loss_mlp": 1.02562308, "epoch": 0.19564106418157223, "flos": 22267918016640.0, "grad_norm": 1.850358038220547, "language_loss": 0.76417327, "learning_rate": 3.634309155462008e-06, "loss": 0.78549385, "num_input_tokens_seen": 70209685, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.6875, "step": 3254, "time_per_iteration": 3.743779182434082 }, { "auxiliary_loss_clip": 0.01020729, "auxiliary_loss_mlp": 0.01004913, "balance_loss_clip": 1.00242162, "balance_loss_mlp": 1.00204873, "epoch": 0.1957011874342402, "flos": 54362000678400.0, "grad_norm": 0.7573270102413824, "language_loss": 0.55256647, "learning_rate": 3.6340913424861383e-06, "loss": 0.57282287, "num_input_tokens_seen": 70265050, "router_z_loss_clip": 0.02490234, "router_z_loss_mlp": 0.18652344, "step": 3255, "time_per_iteration": 2.994016408920288 }, { "auxiliary_loss_clip": 0.01096999, "auxiliary_loss_mlp": 0.01036695, "balance_loss_clip": 1.01614356, "balance_loss_mlp": 1.02813172, "epoch": 0.19576131068690816, "flos": 16505817384960.0, "grad_norm": 2.7410325008516776, "language_loss": 0.70526785, "learning_rate": 3.6338734711937512e-06, "loss": 0.72660476, "num_input_tokens_seen": 70281830, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.6875, "step": 3256, "time_per_iteration": 3.76116943359375 }, { "auxiliary_loss_clip": 0.01091378, "auxiliary_loss_mlp": 0.01036129, "balance_loss_clip": 1.01678145, "balance_loss_mlp": 1.02586532, "epoch": 0.19582143393957613, "flos": 14718822888960.0, "grad_norm": 3.4934437635727464, "language_loss": 0.80128163, "learning_rate": 3.6336555415926232e-06, "loss": 0.82255673, "num_input_tokens_seen": 70297420, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.65625, "step": 3257, "time_per_iteration": 3.6858999729156494 }, { "auxiliary_loss_clip": 0.01093835, "auxiliary_loss_mlp": 0.01036658, "balance_loss_clip": 1.01727438, "balance_loss_mlp": 1.02637625, "epoch": 0.1958815571922441, "flos": 24424363186560.0, "grad_norm": 1.9926092582446306, "language_loss": 0.74545258, "learning_rate": 3.6334375536905313e-06, "loss": 0.76675749, "num_input_tokens_seen": 70319210, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.67578125, "step": 3258, "time_per_iteration": 2.413703203201294 }, { "auxiliary_loss_clip": 0.01095093, "auxiliary_loss_mlp": 0.01035571, "balance_loss_clip": 1.01610398, "balance_loss_mlp": 1.02671051, "epoch": 0.19594168044491206, "flos": 24899112120960.0, "grad_norm": 1.9569839831859468, "language_loss": 0.74006474, "learning_rate": 3.633219507495255e-06, "loss": 0.76137137, "num_input_tokens_seen": 70339045, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.68359375, "step": 3259, "time_per_iteration": 3.779968500137329 }, { "auxiliary_loss_clip": 0.01097543, "auxiliary_loss_mlp": 0.01043504, "balance_loss_clip": 1.0222733, "balance_loss_mlp": 1.02805924, "epoch": 0.19600180369758005, "flos": 12056206694400.0, "grad_norm": 2.537177333310713, "language_loss": 0.76501352, "learning_rate": 3.633001403014575e-06, "loss": 0.78642392, "num_input_tokens_seen": 70356505, "router_z_loss_clip": 0.21191406, "router_z_loss_mlp": 0.6953125, "step": 3260, "time_per_iteration": 2.352241039276123 }, { "auxiliary_loss_clip": 0.01094873, "auxiliary_loss_mlp": 0.01039222, "balance_loss_clip": 1.01876605, "balance_loss_mlp": 1.02674258, "epoch": 0.19606192695024802, "flos": 20849152296960.0, "grad_norm": 2.0974309113083542, "language_loss": 0.82169342, "learning_rate": 3.632783240256276e-06, "loss": 0.84303439, "num_input_tokens_seen": 70375410, "router_z_loss_clip": 0.20507812, "router_z_loss_mlp": 0.6796875, "step": 3261, "time_per_iteration": 2.386594295501709 }, { "auxiliary_loss_clip": 0.0109362, "auxiliary_loss_mlp": 0.01039481, "balance_loss_clip": 1.01932275, "balance_loss_mlp": 1.02704477, "epoch": 0.19612205020291598, "flos": 28474253187840.0, "grad_norm": 2.2523880615555, "language_loss": 0.76350236, "learning_rate": 3.632565019228143e-06, "loss": 0.78483337, "num_input_tokens_seen": 70396315, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.6640625, "step": 3262, "time_per_iteration": 2.4290218353271484 }, { "auxiliary_loss_clip": 0.01097967, "auxiliary_loss_mlp": 0.01042585, "balance_loss_clip": 1.02332032, "balance_loss_mlp": 1.02915668, "epoch": 0.19618217345558395, "flos": 25555444369920.0, "grad_norm": 1.6467314770588994, "language_loss": 0.86481088, "learning_rate": 3.6323467399379634e-06, "loss": 0.8862164, "num_input_tokens_seen": 70417945, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.6875, "step": 3263, "time_per_iteration": 2.4395463466644287 }, { "auxiliary_loss_clip": 0.01092486, "auxiliary_loss_mlp": 0.01035799, "balance_loss_clip": 1.01756024, "balance_loss_mlp": 1.02606213, "epoch": 0.1962422967082519, "flos": 25263256268160.0, "grad_norm": 1.672456435112228, "language_loss": 0.73797274, "learning_rate": 3.6321284023935284e-06, "loss": 0.75925559, "num_input_tokens_seen": 70438690, "router_z_loss_clip": 0.18261719, "router_z_loss_mlp": 0.6640625, "step": 3264, "time_per_iteration": 2.4126996994018555 }, { "auxiliary_loss_clip": 0.01093631, "auxiliary_loss_mlp": 0.01035924, "balance_loss_clip": 1.01721966, "balance_loss_mlp": 1.02846122, "epoch": 0.19630241996091988, "flos": 18806349202560.0, "grad_norm": 1.8398535179686513, "language_loss": 0.78879499, "learning_rate": 3.6319100066026284e-06, "loss": 0.81009054, "num_input_tokens_seen": 70455385, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.65234375, "step": 3265, "time_per_iteration": 2.3726553916931152 }, { "auxiliary_loss_clip": 0.01020868, "auxiliary_loss_mlp": 0.01002098, "balance_loss_clip": 0.99966645, "balance_loss_mlp": 1.00293803, "epoch": 0.19636254321358784, "flos": 62318287526400.0, "grad_norm": 0.7818136536489693, "language_loss": 0.53380704, "learning_rate": 3.6316915525730586e-06, "loss": 0.55403674, "num_input_tokens_seen": 70514280, "router_z_loss_clip": 0.02429199, "router_z_loss_mlp": 0.1796875, "step": 3266, "time_per_iteration": 3.045663833618164 }, { "auxiliary_loss_clip": 0.01097612, "auxiliary_loss_mlp": 0.010397, "balance_loss_clip": 1.01870728, "balance_loss_mlp": 1.02730501, "epoch": 0.19642266646625584, "flos": 21068267189760.0, "grad_norm": 1.995501135005288, "language_loss": 0.80237895, "learning_rate": 3.631473040312614e-06, "loss": 0.82375205, "num_input_tokens_seen": 70531800, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.703125, "step": 3267, "time_per_iteration": 2.3809406757354736 }, { "auxiliary_loss_clip": 0.010922, "auxiliary_loss_mlp": 0.01036523, "balance_loss_clip": 1.01773524, "balance_loss_mlp": 1.02631319, "epoch": 0.1964827897189238, "flos": 14537763244800.0, "grad_norm": 9.820342443007977, "language_loss": 0.86726725, "learning_rate": 3.631254469829094e-06, "loss": 0.88855445, "num_input_tokens_seen": 70550615, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.66015625, "step": 3268, "time_per_iteration": 2.3497135639190674 }, { "auxiliary_loss_clip": 0.01094062, "auxiliary_loss_mlp": 0.01034729, "balance_loss_clip": 1.01679969, "balance_loss_mlp": 1.02803731, "epoch": 0.19654291297159177, "flos": 19243636381440.0, "grad_norm": 2.55200700436689, "language_loss": 0.69314349, "learning_rate": 3.631035841130297e-06, "loss": 0.71443152, "num_input_tokens_seen": 70568690, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.66015625, "step": 3269, "time_per_iteration": 2.382796049118042 }, { "auxiliary_loss_clip": 0.01098343, "auxiliary_loss_mlp": 0.01038692, "balance_loss_clip": 1.019189, "balance_loss_mlp": 1.02958822, "epoch": 0.19660303622425973, "flos": 25774524351360.0, "grad_norm": 2.125134277645011, "language_loss": 0.80778444, "learning_rate": 3.6308171542240273e-06, "loss": 0.82915473, "num_input_tokens_seen": 70588665, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.6875, "step": 3270, "time_per_iteration": 2.418290376663208 }, { "auxiliary_loss_clip": 0.01090844, "auxiliary_loss_mlp": 0.01036312, "balance_loss_clip": 1.01775169, "balance_loss_mlp": 1.02518535, "epoch": 0.1966631594769277, "flos": 20594041925760.0, "grad_norm": 2.3224232715776933, "language_loss": 0.83714098, "learning_rate": 3.6305984091180875e-06, "loss": 0.8584125, "num_input_tokens_seen": 70606900, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.65625, "step": 3271, "time_per_iteration": 2.3715765476226807 }, { "auxiliary_loss_clip": 0.01090027, "auxiliary_loss_mlp": 0.01035944, "balance_loss_clip": 1.01740742, "balance_loss_mlp": 1.026191, "epoch": 0.19672328272959566, "flos": 23622059076480.0, "grad_norm": 1.9779680712166663, "language_loss": 0.8020243, "learning_rate": 3.630379605820286e-06, "loss": 0.82328403, "num_input_tokens_seen": 70625955, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.640625, "step": 3272, "time_per_iteration": 2.3996176719665527 }, { "auxiliary_loss_clip": 0.01094305, "auxiliary_loss_mlp": 0.0104103, "balance_loss_clip": 1.02173042, "balance_loss_mlp": 1.0272423, "epoch": 0.19678340598226365, "flos": 23109848386560.0, "grad_norm": 1.9417672352006365, "language_loss": 0.80638385, "learning_rate": 3.630160744338429e-06, "loss": 0.82773721, "num_input_tokens_seen": 70646090, "router_z_loss_clip": 0.19335938, "router_z_loss_mlp": 0.671875, "step": 3273, "time_per_iteration": 2.4034245014190674 }, { "auxiliary_loss_clip": 0.01093358, "auxiliary_loss_mlp": 0.01042803, "balance_loss_clip": 1.02361071, "balance_loss_mlp": 1.02661026, "epoch": 0.19684352923493162, "flos": 24533711164800.0, "grad_norm": 1.685327645217058, "language_loss": 0.77463973, "learning_rate": 3.6299418246803287e-06, "loss": 0.79600132, "num_input_tokens_seen": 70666065, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.66796875, "step": 3274, "time_per_iteration": 2.4098117351531982 }, { "auxiliary_loss_clip": 0.01093006, "auxiliary_loss_mlp": 0.01038849, "balance_loss_clip": 1.01897693, "balance_loss_mlp": 1.02611113, "epoch": 0.19690365248759958, "flos": 21795438320640.0, "grad_norm": 3.613819678474458, "language_loss": 0.81330287, "learning_rate": 3.6297228468537976e-06, "loss": 0.83462143, "num_input_tokens_seen": 70681580, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.66796875, "step": 3275, "time_per_iteration": 2.3586437702178955 }, { "auxiliary_loss_clip": 0.01093968, "auxiliary_loss_mlp": 0.01039103, "balance_loss_clip": 1.01931393, "balance_loss_mlp": 1.02676809, "epoch": 0.19696377574026755, "flos": 19055803933440.0, "grad_norm": 1.8388194076600255, "language_loss": 0.81185746, "learning_rate": 3.6295038108666504e-06, "loss": 0.83318818, "num_input_tokens_seen": 70697745, "router_z_loss_clip": 0.19824219, "router_z_loss_mlp": 0.671875, "step": 3276, "time_per_iteration": 2.3627398014068604 }, { "auxiliary_loss_clip": 0.01093263, "auxiliary_loss_mlp": 0.0103276, "balance_loss_clip": 1.01329339, "balance_loss_mlp": 1.02681422, "epoch": 0.19702389899293551, "flos": 22819545498240.0, "grad_norm": 3.913293022738735, "language_loss": 0.89339715, "learning_rate": 3.629284716726703e-06, "loss": 0.91465741, "num_input_tokens_seen": 70715110, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.6640625, "step": 3277, "time_per_iteration": 2.3836357593536377 }, { "auxiliary_loss_clip": 0.01096947, "auxiliary_loss_mlp": 0.0104411, "balance_loss_clip": 1.02131701, "balance_loss_mlp": 1.02706742, "epoch": 0.19708402224560348, "flos": 22893107466240.0, "grad_norm": 2.121847641070351, "language_loss": 0.62500441, "learning_rate": 3.6290655644417757e-06, "loss": 0.646415, "num_input_tokens_seen": 70734715, "router_z_loss_clip": 0.22753906, "router_z_loss_mlp": 0.69921875, "step": 3278, "time_per_iteration": 2.398437738418579 }, { "auxiliary_loss_clip": 0.01094952, "auxiliary_loss_mlp": 0.01041973, "balance_loss_clip": 1.02227986, "balance_loss_mlp": 1.02898657, "epoch": 0.19714414549827144, "flos": 25661440857600.0, "grad_norm": 2.9479425926662484, "language_loss": 0.73167086, "learning_rate": 3.6288463540196894e-06, "loss": 0.75304008, "num_input_tokens_seen": 70752650, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.66015625, "step": 3279, "time_per_iteration": 2.409766912460327 }, { "auxiliary_loss_clip": 0.01094849, "auxiliary_loss_mlp": 0.01036963, "balance_loss_clip": 1.01818752, "balance_loss_mlp": 1.02610898, "epoch": 0.19720426875093944, "flos": 23914666114560.0, "grad_norm": 1.6378735440973151, "language_loss": 0.8245886, "learning_rate": 3.6286270854682654e-06, "loss": 0.84590667, "num_input_tokens_seen": 70772365, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.6875, "step": 3280, "time_per_iteration": 2.402646780014038 }, { "auxiliary_loss_clip": 0.01096637, "auxiliary_loss_mlp": 0.01039202, "balance_loss_clip": 1.02074862, "balance_loss_mlp": 1.02819932, "epoch": 0.1972643920036074, "flos": 13881081882240.0, "grad_norm": 1.9159719125239376, "language_loss": 0.77710402, "learning_rate": 3.6284077587953307e-06, "loss": 0.79846239, "num_input_tokens_seen": 70790340, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.68359375, "step": 3281, "time_per_iteration": 2.359015703201294 }, { "auxiliary_loss_clip": 0.01091432, "auxiliary_loss_mlp": 0.0104029, "balance_loss_clip": 1.02223015, "balance_loss_mlp": 1.02704263, "epoch": 0.19732451525627537, "flos": 19862611608960.0, "grad_norm": 1.7886278430035771, "language_loss": 0.79787135, "learning_rate": 3.628188374008712e-06, "loss": 0.81918859, "num_input_tokens_seen": 70809295, "router_z_loss_clip": 0.18066406, "router_z_loss_mlp": 0.64453125, "step": 3282, "time_per_iteration": 2.4334022998809814 }, { "auxiliary_loss_clip": 0.01097752, "auxiliary_loss_mlp": 0.01036859, "balance_loss_clip": 1.01765406, "balance_loss_mlp": 1.02917266, "epoch": 0.19738463850894333, "flos": 24972255152640.0, "grad_norm": 2.004615061425067, "language_loss": 0.71514744, "learning_rate": 3.6279689311162382e-06, "loss": 0.73649353, "num_input_tokens_seen": 70828765, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.68359375, "step": 3283, "time_per_iteration": 2.411259412765503 }, { "auxiliary_loss_clip": 0.01093562, "auxiliary_loss_mlp": 0.010486, "balance_loss_clip": 1.02915692, "balance_loss_mlp": 1.02674568, "epoch": 0.1974447617616113, "flos": 18367909948800.0, "grad_norm": 2.0901111004947532, "language_loss": 0.78843147, "learning_rate": 3.6277494301257407e-06, "loss": 0.80985308, "num_input_tokens_seen": 70846805, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.66796875, "step": 3284, "time_per_iteration": 2.363900661468506 }, { "auxiliary_loss_clip": 0.01095873, "auxiliary_loss_mlp": 0.01040929, "balance_loss_clip": 1.01948333, "balance_loss_mlp": 1.02663827, "epoch": 0.19750488501427926, "flos": 22891850657280.0, "grad_norm": 2.1448640694948136, "language_loss": 0.86014587, "learning_rate": 3.6275298710450533e-06, "loss": 0.88151383, "num_input_tokens_seen": 70863805, "router_z_loss_clip": 0.21484375, "router_z_loss_mlp": 0.69140625, "step": 3285, "time_per_iteration": 2.3824238777160645 }, { "auxiliary_loss_clip": 0.01092925, "auxiliary_loss_mlp": 0.01034811, "balance_loss_clip": 1.01686954, "balance_loss_mlp": 1.0281477, "epoch": 0.19756500826694723, "flos": 21870431654400.0, "grad_norm": 2.307979665706718, "language_loss": 0.88518846, "learning_rate": 3.627310253882012e-06, "loss": 0.90646577, "num_input_tokens_seen": 70882660, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.6484375, "step": 3286, "time_per_iteration": 2.3906936645507812 }, { "auxiliary_loss_clip": 0.01095962, "auxiliary_loss_mlp": 0.01041885, "balance_loss_clip": 1.02093983, "balance_loss_mlp": 1.02850592, "epoch": 0.19762513151961522, "flos": 15158065104000.0, "grad_norm": 2.3708893620062006, "language_loss": 0.78196716, "learning_rate": 3.627090578644452e-06, "loss": 0.80334568, "num_input_tokens_seen": 70898765, "router_z_loss_clip": 0.20898438, "router_z_loss_mlp": 0.671875, "step": 3287, "time_per_iteration": 2.351134777069092 }, { "auxiliary_loss_clip": 0.01095035, "auxiliary_loss_mlp": 0.0103711, "balance_loss_clip": 1.01710689, "balance_loss_mlp": 1.02704477, "epoch": 0.1976852547722832, "flos": 16978331992320.0, "grad_norm": 6.362389513168747, "language_loss": 0.81321955, "learning_rate": 3.6268708453402163e-06, "loss": 0.83454096, "num_input_tokens_seen": 70916370, "router_z_loss_clip": 0.19921875, "router_z_loss_mlp": 0.6796875, "step": 3288, "time_per_iteration": 2.357706308364868 }, { "auxiliary_loss_clip": 0.01091467, "auxiliary_loss_mlp": 0.01039245, "balance_loss_clip": 1.02087522, "balance_loss_mlp": 1.02781177, "epoch": 0.19774537802495115, "flos": 20301888735360.0, "grad_norm": 1.8304637570278746, "language_loss": 0.72829801, "learning_rate": 3.626651053977144e-06, "loss": 0.74960506, "num_input_tokens_seen": 70934870, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.63671875, "step": 3289, "time_per_iteration": 2.3803505897521973 }, { "auxiliary_loss_clip": 0.01093329, "auxiliary_loss_mlp": 0.01041591, "balance_loss_clip": 1.02309, "balance_loss_mlp": 1.02678442, "epoch": 0.19780550127761912, "flos": 27234242962560.0, "grad_norm": 2.524989848506488, "language_loss": 0.7940982, "learning_rate": 3.6264312045630802e-06, "loss": 0.81544745, "num_input_tokens_seen": 70955140, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.6640625, "step": 3290, "time_per_iteration": 2.4267165660858154 }, { "auxiliary_loss_clip": 0.01023181, "auxiliary_loss_mlp": 0.01002361, "balance_loss_clip": 0.9999764, "balance_loss_mlp": 1.00474858, "epoch": 0.19786562453028708, "flos": 63547368629760.0, "grad_norm": 0.8859536292225495, "language_loss": 0.60287488, "learning_rate": 3.62621129710587e-06, "loss": 0.6231302, "num_input_tokens_seen": 71012005, "router_z_loss_clip": 0.02380371, "router_z_loss_mlp": 0.18457031, "step": 3291, "time_per_iteration": 3.0441205501556396 }, { "auxiliary_loss_clip": 0.01095104, "auxiliary_loss_mlp": 0.01036338, "balance_loss_clip": 1.01521397, "balance_loss_mlp": 1.02619529, "epoch": 0.19792574778295505, "flos": 26285443320960.0, "grad_norm": 1.710016469752574, "language_loss": 0.81068504, "learning_rate": 3.6259913316133625e-06, "loss": 0.83199942, "num_input_tokens_seen": 71031140, "router_z_loss_clip": 0.2109375, "router_z_loss_mlp": 0.6875, "step": 3292, "time_per_iteration": 2.4184789657592773 }, { "auxiliary_loss_clip": 0.01088314, "auxiliary_loss_mlp": 0.01036573, "balance_loss_clip": 1.01852489, "balance_loss_mlp": 1.02581763, "epoch": 0.19798587103562304, "flos": 19937081272320.0, "grad_norm": 2.164269928396593, "language_loss": 0.81474257, "learning_rate": 3.625771308093406e-06, "loss": 0.8359915, "num_input_tokens_seen": 71050250, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.625, "step": 3293, "time_per_iteration": 3.7663729190826416 }, { "auxiliary_loss_clip": 0.011012, "auxiliary_loss_mlp": 0.01041589, "balance_loss_clip": 1.02060866, "balance_loss_mlp": 1.03096867, "epoch": 0.198045994288291, "flos": 20119258080000.0, "grad_norm": 1.8060343049106946, "language_loss": 0.61091065, "learning_rate": 3.625551226553854e-06, "loss": 0.63233852, "num_input_tokens_seen": 71068665, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.703125, "step": 3294, "time_per_iteration": 2.3722171783447266 }, { "auxiliary_loss_clip": 0.01092659, "auxiliary_loss_mlp": 0.01038661, "balance_loss_clip": 1.01956379, "balance_loss_mlp": 1.02670622, "epoch": 0.19810611754095897, "flos": 17966688071040.0, "grad_norm": 1.9379544072355641, "language_loss": 0.87146139, "learning_rate": 3.6253310870025598e-06, "loss": 0.89277458, "num_input_tokens_seen": 71085320, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.66015625, "step": 3295, "time_per_iteration": 2.3429510593414307 }, { "auxiliary_loss_clip": 0.01092325, "auxiliary_loss_mlp": 0.01037285, "balance_loss_clip": 1.01948702, "balance_loss_mlp": 1.02713811, "epoch": 0.19816624079362694, "flos": 15084119111040.0, "grad_norm": 2.476934992282589, "language_loss": 0.80659974, "learning_rate": 3.6251108894473806e-06, "loss": 0.82789588, "num_input_tokens_seen": 71102020, "router_z_loss_clip": 0.17773438, "router_z_loss_mlp": 0.65234375, "step": 3296, "time_per_iteration": 5.0992112159729 }, { "auxiliary_loss_clip": 0.01095761, "auxiliary_loss_mlp": 0.01034934, "balance_loss_clip": 1.01356006, "balance_loss_mlp": 1.0261941, "epoch": 0.1982263640462949, "flos": 24899147032320.0, "grad_norm": 1.9064652416671366, "language_loss": 0.68187982, "learning_rate": 3.624890633896173e-06, "loss": 0.70318681, "num_input_tokens_seen": 71123390, "router_z_loss_clip": 0.21289062, "router_z_loss_mlp": 0.6953125, "step": 3297, "time_per_iteration": 2.414424180984497 }, { "auxiliary_loss_clip": 0.01090671, "auxiliary_loss_mlp": 0.01034454, "balance_loss_clip": 1.01708508, "balance_loss_mlp": 1.02733803, "epoch": 0.19828648729896287, "flos": 20375136501120.0, "grad_norm": 1.7080929907446294, "language_loss": 0.81423819, "learning_rate": 3.6246703203567996e-06, "loss": 0.83548945, "num_input_tokens_seen": 71141800, "router_z_loss_clip": 0.17382812, "router_z_loss_mlp": 0.6328125, "step": 3298, "time_per_iteration": 2.3621015548706055 }, { "auxiliary_loss_clip": 0.01096755, "auxiliary_loss_mlp": 0.01040343, "balance_loss_clip": 1.01938558, "balance_loss_mlp": 1.02652717, "epoch": 0.19834661055163083, "flos": 18879038386560.0, "grad_norm": 1.8118896067170283, "language_loss": 0.8499251, "learning_rate": 3.624449948837121e-06, "loss": 0.87129605, "num_input_tokens_seen": 71159505, "router_z_loss_clip": 0.20996094, "router_z_loss_mlp": 0.703125, "step": 3299, "time_per_iteration": 3.7855286598205566 }, { "auxiliary_loss_clip": 0.0102453, "auxiliary_loss_mlp": 0.01005913, "balance_loss_clip": 1.00327826, "balance_loss_mlp": 1.00592458, "epoch": 0.19840673380429882, "flos": 60255897292800.0, "grad_norm": 0.7660662056871961, "language_loss": 0.53273189, "learning_rate": 3.6242295193450024e-06, "loss": 0.55303633, "num_input_tokens_seen": 71223265, "router_z_loss_clip": 0.02636719, "router_z_loss_mlp": 0.18554688, "step": 3300, "time_per_iteration": 2.9814531803131104 }, { "auxiliary_loss_clip": 0.01093189, "auxiliary_loss_mlp": 0.01043868, "balance_loss_clip": 1.02426958, "balance_loss_mlp": 1.02672458, "epoch": 0.1984668570569668, "flos": 19900981059840.0, "grad_norm": 2.006506243334892, "language_loss": 0.72906816, "learning_rate": 3.6240090318883103e-06, "loss": 0.75043869, "num_input_tokens_seen": 71242385, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.6640625, "step": 3301, "time_per_iteration": 2.390085458755493 }, { "auxiliary_loss_clip": 0.01093821, "auxiliary_loss_mlp": 0.01034821, "balance_loss_clip": 1.01660538, "balance_loss_mlp": 1.0276103, "epoch": 0.19852698030963475, "flos": 15629916395520.0, "grad_norm": 2.3942277196780233, "language_loss": 0.88054079, "learning_rate": 3.623788486474913e-06, "loss": 0.90182722, "num_input_tokens_seen": 71258990, "router_z_loss_clip": 0.18261719, "router_z_loss_mlp": 0.66015625, "step": 3302, "time_per_iteration": 2.3620595932006836 }, { "auxiliary_loss_clip": 0.01093732, "auxiliary_loss_mlp": 0.01034571, "balance_loss_clip": 1.01491392, "balance_loss_mlp": 1.02640676, "epoch": 0.19858710356230272, "flos": 43141335575040.0, "grad_norm": 1.8277191047193035, "language_loss": 0.73320621, "learning_rate": 3.623567883112682e-06, "loss": 0.7544893, "num_input_tokens_seen": 71282770, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.671875, "step": 3303, "time_per_iteration": 2.573152780532837 }, { "auxiliary_loss_clip": 0.01092105, "auxiliary_loss_mlp": 0.01041146, "balance_loss_clip": 1.02151227, "balance_loss_mlp": 1.02686501, "epoch": 0.19864722681497068, "flos": 35142873937920.0, "grad_norm": 1.8424346043034194, "language_loss": 0.74658036, "learning_rate": 3.6233472218094897e-06, "loss": 0.76791286, "num_input_tokens_seen": 71301410, "router_z_loss_clip": 0.19726562, "router_z_loss_mlp": 0.65234375, "step": 3304, "time_per_iteration": 2.49125599861145 }, { "auxiliary_loss_clip": 0.01091011, "auxiliary_loss_mlp": 0.01033155, "balance_loss_clip": 1.01579833, "balance_loss_mlp": 1.02630901, "epoch": 0.19870735006763865, "flos": 19425219696000.0, "grad_norm": 2.7354527523926517, "language_loss": 0.85860914, "learning_rate": 3.62312650257321e-06, "loss": 0.8798508, "num_input_tokens_seen": 71319670, "router_z_loss_clip": 0.17382812, "router_z_loss_mlp": 0.6484375, "step": 3305, "time_per_iteration": 2.3646278381347656 }, { "auxiliary_loss_clip": 0.01093773, "auxiliary_loss_mlp": 0.01033682, "balance_loss_clip": 1.01421475, "balance_loss_mlp": 1.02594137, "epoch": 0.19876747332030664, "flos": 23546332604160.0, "grad_norm": 1.5847495036578148, "language_loss": 0.68523008, "learning_rate": 3.622905725411721e-06, "loss": 0.70650458, "num_input_tokens_seen": 71339850, "router_z_loss_clip": 0.1953125, "router_z_loss_mlp": 0.6796875, "step": 3306, "time_per_iteration": 2.4286417961120605 }, { "auxiliary_loss_clip": 0.01089723, "auxiliary_loss_mlp": 0.01030964, "balance_loss_clip": 1.01242733, "balance_loss_mlp": 1.02498138, "epoch": 0.1988275965729746, "flos": 19828361698560.0, "grad_norm": 1.9311524076025604, "language_loss": 0.76461613, "learning_rate": 3.622684890332901e-06, "loss": 0.78582305, "num_input_tokens_seen": 71359795, "router_z_loss_clip": 0.18554688, "router_z_loss_mlp": 0.6484375, "step": 3307, "time_per_iteration": 2.4046173095703125 }, { "auxiliary_loss_clip": 0.01092585, "auxiliary_loss_mlp": 0.0103464, "balance_loss_clip": 1.01755738, "balance_loss_mlp": 1.02870154, "epoch": 0.19888771982564257, "flos": 23512501630080.0, "grad_norm": 2.169181339009595, "language_loss": 0.7575652, "learning_rate": 3.622463997344632e-06, "loss": 0.77883744, "num_input_tokens_seen": 71378885, "router_z_loss_clip": 0.17089844, "router_z_loss_mlp": 0.63671875, "step": 3308, "time_per_iteration": 2.406249761581421 }, { "auxiliary_loss_clip": 0.01092841, "auxiliary_loss_mlp": 0.01034602, "balance_loss_clip": 1.01551628, "balance_loss_mlp": 1.02665699, "epoch": 0.19894784307831054, "flos": 18149528194560.0, "grad_norm": 3.2640222401270114, "language_loss": 0.75881577, "learning_rate": 3.622243046454796e-06, "loss": 0.78009021, "num_input_tokens_seen": 71397285, "router_z_loss_clip": 0.19140625, "router_z_loss_mlp": 0.66015625, "step": 3309, "time_per_iteration": 2.353468894958496 }, { "auxiliary_loss_clip": 0.01090891, "auxiliary_loss_mlp": 0.01042274, "balance_loss_clip": 1.02351093, "balance_loss_mlp": 1.02639914, "epoch": 0.1990079663309785, "flos": 24275004923520.0, "grad_norm": 2.1937839839446713, "language_loss": 0.87566149, "learning_rate": 3.6220220376712787e-06, "loss": 0.89699316, "num_input_tokens_seen": 71415775, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.64453125, "step": 3310, "time_per_iteration": 2.4153454303741455 }, { "auxiliary_loss_clip": 0.01090099, "auxiliary_loss_mlp": 0.01032432, "balance_loss_clip": 1.01319122, "balance_loss_mlp": 1.02569306, "epoch": 0.19906808958364647, "flos": 34896212115840.0, "grad_norm": 2.7435653817024566, "language_loss": 0.64026791, "learning_rate": 3.621800971001967e-06, "loss": 0.66149318, "num_input_tokens_seen": 71437315, "router_z_loss_clip": 0.19238281, "router_z_loss_mlp": 0.64453125, "step": 3311, "time_per_iteration": 2.482752561569214 }, { "auxiliary_loss_clip": 0.01094648, "auxiliary_loss_mlp": 0.01034431, "balance_loss_clip": 1.01598966, "balance_loss_mlp": 1.02681947, "epoch": 0.19912821283631443, "flos": 24023734801920.0, "grad_norm": 2.2046276982202686, "language_loss": 0.73585874, "learning_rate": 3.6215798464547505e-06, "loss": 0.75714952, "num_input_tokens_seen": 71456320, "router_z_loss_clip": 0.18457031, "router_z_loss_mlp": 0.67578125, "step": 3312, "time_per_iteration": 2.39709734916687 }, { "auxiliary_loss_clip": 0.01089779, "auxiliary_loss_mlp": 0.01037231, "balance_loss_clip": 1.01886082, "balance_loss_mlp": 1.0253818, "epoch": 0.19918833608898243, "flos": 19858177866240.0, "grad_norm": 2.214729870251275, "language_loss": 0.83765405, "learning_rate": 3.6213586640375207e-06, "loss": 0.85892415, "num_input_tokens_seen": 71475360, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.64453125, "step": 3313, "time_per_iteration": 2.3777294158935547 }, { "auxiliary_loss_clip": 0.01094265, "auxiliary_loss_mlp": 0.01039953, "balance_loss_clip": 1.02130866, "balance_loss_mlp": 1.02869534, "epoch": 0.1992484593416504, "flos": 29094520135680.0, "grad_norm": 4.863015940835911, "language_loss": 0.80661523, "learning_rate": 3.6211374237581706e-06, "loss": 0.82795733, "num_input_tokens_seen": 71496155, "router_z_loss_clip": 0.18652344, "router_z_loss_mlp": 0.65625, "step": 3314, "time_per_iteration": 2.4357516765594482 }, { "auxiliary_loss_clip": 0.01088398, "auxiliary_loss_mlp": 0.01033582, "balance_loss_clip": 1.01609409, "balance_loss_mlp": 1.02526283, "epoch": 0.19930858259431836, "flos": 23293875496320.0, "grad_norm": 1.444588805498063, "language_loss": 0.8718859, "learning_rate": 3.620916125624596e-06, "loss": 0.89310575, "num_input_tokens_seen": 71517295, "router_z_loss_clip": 0.17480469, "router_z_loss_mlp": 0.6328125, "step": 3315, "time_per_iteration": 2.4119343757629395 }, { "auxiliary_loss_clip": 0.01093967, "auxiliary_loss_mlp": 0.01034429, "balance_loss_clip": 1.0160712, "balance_loss_mlp": 1.02733731, "epoch": 0.19936870584698632, "flos": 25377526748160.0, "grad_norm": 1.5396054078786452, "language_loss": 0.7089622, "learning_rate": 3.620694769644694e-06, "loss": 0.73024619, "num_input_tokens_seen": 71540000, "router_z_loss_clip": 0.18359375, "router_z_loss_mlp": 0.66796875, "step": 3316, "time_per_iteration": 2.441154956817627 }, { "auxiliary_loss_clip": 0.01023534, "auxiliary_loss_mlp": 0.01004258, "balance_loss_clip": 1.00170732, "balance_loss_mlp": 1.00516033, "epoch": 0.1994288290996543, "flos": 62164388920320.0, "grad_norm": 0.8381400898957825, "language_loss": 0.66274536, "learning_rate": 3.6204733558263653e-06, "loss": 0.68302333, "num_input_tokens_seen": 71607880, "router_z_loss_clip": 0.0255127, "router_z_loss_mlp": 0.18359375, "step": 3317, "time_per_iteration": 3.0949625968933105 }, { "auxiliary_loss_clip": 0.01095287, "auxiliary_loss_mlp": 0.01039173, "balance_loss_clip": 1.02044487, "balance_loss_mlp": 1.02687836, "epoch": 0.19948895235232225, "flos": 19024835690880.0, "grad_norm": 3.3600161071849532, "language_loss": 0.74098063, "learning_rate": 3.6202518841775104e-06, "loss": 0.76232529, "num_input_tokens_seen": 71625695, "router_z_loss_clip": 0.1875, "router_z_loss_mlp": 0.68359375, "step": 3318, "time_per_iteration": 2.3706955909729004 }, { "auxiliary_loss_clip": 0.01088429, "auxiliary_loss_mlp": 0.01038061, "balance_loss_clip": 1.02025127, "balance_loss_mlp": 1.02606761, "epoch": 0.19954907560499022, "flos": 37814287795200.0, "grad_norm": 1.9049686259150524, "language_loss": 0.78945422, "learning_rate": 3.6200303547060336e-06, "loss": 0.81071913, "num_input_tokens_seen": 71648520, "router_z_loss_clip": 0.17773438, "router_z_loss_mlp": 0.625, "step": 3319, "time_per_iteration": 2.5089056491851807 }, { "auxiliary_loss_clip": 0.01091702, "auxiliary_loss_mlp": 0.01034212, "balance_loss_clip": 1.01383972, "balance_loss_mlp": 1.02566719, "epoch": 0.1996091988576582, "flos": 49563329414400.0, "grad_norm": 2.4920940963466145, "language_loss": 0.76352167, "learning_rate": 3.61980876741984e-06, "loss": 0.78478074, "num_input_tokens_seen": 71672185, "router_z_loss_clip": 0.203125, "router_z_loss_mlp": 0.66015625, "step": 3320, "time_per_iteration": 2.6354074478149414 }, { "auxiliary_loss_clip": 0.01090883, "auxiliary_loss_mlp": 0.01039099, "balance_loss_clip": 1.02159953, "balance_loss_mlp": 1.02575707, "epoch": 0.19966932211032618, "flos": 22634750338560.0, "grad_norm": 2.207245808725498, "language_loss": 0.80172241, "learning_rate": 3.6195871223268392e-06, "loss": 0.82302225, "num_input_tokens_seen": 71692890, "router_z_loss_clip": 0.17578125, "router_z_loss_mlp": 0.65234375, "step": 3321, "time_per_iteration": 2.3941807746887207 }, { "auxiliary_loss_clip": 0.0102174, "auxiliary_loss_mlp": 0.01001946, "balance_loss_clip": 0.99951375, "balance_loss_mlp": 1.00350666, "epoch": 0.19972944536299414, "flos": 54079308466560.0, "grad_norm": 0.8696088293687311, "language_loss": 0.65128511, "learning_rate": 3.61936541943494e-06, "loss": 0.67152202, "num_input_tokens_seen": 71745815, "router_z_loss_clip": 0.02429199, "router_z_loss_mlp": 0.18261719, "step": 3322, "time_per_iteration": 2.8282928466796875 }, { "auxiliary_loss_clip": 0.01021052, "auxiliary_loss_mlp": 0.01002368, "balance_loss_clip": 0.99996036, "balance_loss_mlp": 1.00306439, "epoch": 0.1997895686156621, "flos": 69352204498560.0, "grad_norm": 0.7836553028040963, "language_loss": 0.56965047, "learning_rate": 3.619143658752054e-06, "loss": 0.5898847, "num_input_tokens_seen": 71806915, "router_z_loss_clip": 0.02404785, "router_z_loss_mlp": 0.1796875, "step": 3323, "time_per_iteration": 3.154853105545044 }, { "auxiliary_loss_clip": 0.01092334, "auxiliary_loss_mlp": 0.01036843, "balance_loss_clip": 1.01741147, "balance_loss_mlp": 1.02751148, "epoch": 0.19984969186833007, "flos": 18551064274560.0, "grad_norm": 2.64619446476262, "language_loss": 0.80301172, "learning_rate": 3.6189218402860958e-06, "loss": 0.82430351, "num_input_tokens_seen": 71824645, "router_z_loss_clip": 0.19433594, "router_z_loss_mlp": 0.6484375, "step": 3324, "time_per_iteration": 2.364062786102295 }, { "auxiliary_loss_clip": 0.01089631, "auxiliary_loss_mlp": 0.0104084, "balance_loss_clip": 1.02077699, "balance_loss_mlp": 1.02509081, "epoch": 0.19990981512099804, "flos": 26428552450560.0, "grad_norm": 1.755109624568066, "language_loss": 0.54017216, "learning_rate": 3.6186999640449817e-06, "loss": 0.56147689, "num_input_tokens_seen": 71845125, "router_z_loss_clip": 0.20117188, "router_z_loss_mlp": 0.64453125, "step": 3325, "time_per_iteration": 2.447310447692871 }, { "auxiliary_loss_clip": 0.01091465, "auxiliary_loss_mlp": 0.01034303, "balance_loss_clip": 1.01624262, "balance_loss_mlp": 1.02723396, "epoch": 0.19996993837366603, "flos": 16325071943040.0, "grad_norm": 2.354431826748387, "language_loss": 0.85965687, "learning_rate": 3.6184780300366294e-06, "loss": 0.88091457, "num_input_tokens_seen": 71863500, "router_z_loss_clip": 0.18066406, "router_z_loss_mlp": 0.64453125, "step": 3326, "time_per_iteration": 2.361738443374634 }, { "auxiliary_loss_clip": 0.01090149, "auxiliary_loss_mlp": 0.01036799, "balance_loss_clip": 1.01886964, "balance_loss_mlp": 1.02702761, "epoch": 0.200030061626334, "flos": 20843287188480.0, "grad_norm": 1.8797726577910963, "language_loss": 0.71729505, "learning_rate": 3.6182560382689598e-06, "loss": 0.73856449, "num_input_tokens_seen": 71881845, "router_z_loss_clip": 0.1796875, "router_z_loss_mlp": 0.6328125, "step": 3327, "time_per_iteration": 2.370218515396118 }, { "auxiliary_loss_clip": 0.01093333, "auxiliary_loss_mlp": 0.01038162, "balance_loss_clip": 1.0185287, "balance_loss_mlp": 1.02680278, "epoch": 0.20009018487900196, "flos": 23761677070080.0, "grad_norm": 1.981411592710895, "language_loss": 0.76705289, "learning_rate": 3.6180339887498948e-06, "loss": 0.78836781, "num_input_tokens_seen": 71900940, "router_z_loss_clip": 0.19628906, "router_z_loss_mlp": 0.6640625, "step": 3328, "time_per_iteration": 2.3940131664276123 } ], "logging_steps": 1.0, "max_steps": 16632, "num_input_tokens_seen": 71900940, "num_train_epochs": 1, "save_steps": 3328, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 2.8065853682692915e+17, "train_batch_size": 5, "trial_name": null, "trial_params": null }