diff --git "a/sft/1M3/Full_smoe_share/checkpoint-6893/trainer_state.json" "b/sft/1M3/Full_smoe_share/checkpoint-6893/trainer_state.json" new file mode 100644--- /dev/null +++ "b/sft/1M3/Full_smoe_share/checkpoint-6893/trainer_state.json" @@ -0,0 +1,117214 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.20001741048110963, + "eval_steps": 500, + "global_step": 6893, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "auxiliary_loss_clip": 0.05603412, + "auxiliary_loss_mlp": 0.03114321, + "balance_loss_clip": 2.14666033, + "balance_loss_mlp": 2.84080744, + "epoch": 2.901746851604666e-05, + "flos": 74761303668480.0, + "grad_norm": 2.8245304708204624, + "language_loss": 0.91534865, + "learning_rate": 0.0, + "loss": 0.67611367, + "num_input_tokens_seen": 62630, + "router_z_loss_clip": 34.5, + "router_z_loss_mlp": 2.73046875, + "step": 1, + "time_per_iteration": 21.210593938827515 + }, + { + "auxiliary_loss_clip": 0.03274859, + "auxiliary_loss_mlp": 0.01459778, + "balance_loss_clip": 1.5328176, + "balance_loss_mlp": 1.18588245, + "epoch": 5.803493703209332e-05, + "flos": 25222024085760.0, + "grad_norm": 35.4531218952757, + "language_loss": 1.66993427, + "learning_rate": 3.994399663808758e-07, + "loss": 1.71728075, + "num_input_tokens_seen": 76875, + "router_z_loss_clip": 17.4375, + "router_z_loss_mlp": 2.74023438, + "step": 2, + "time_per_iteration": 2.4828267097473145 + }, + { + "auxiliary_loss_clip": 0.0326227, + "auxiliary_loss_mlp": 0.01469547, + "balance_loss_clip": 1.5236814, + "balance_loss_mlp": 1.19260013, + "epoch": 8.705240554813998e-05, + "flos": 19855035843840.0, + "grad_norm": 57.155353112915165, + "language_loss": 2.32581735, + "learning_rate": 6.330973680030075e-07, + "loss": 2.37313557, + "num_input_tokens_seen": 88305, + "router_z_loss_clip": 17.40625, + "router_z_loss_mlp": 2.76953125, + "step": 3, + "time_per_iteration": 2.4472815990448 + }, + { + "auxiliary_loss_clip": 0.03551114, + "auxiliary_loss_mlp": 0.0191291, + "balance_loss_clip": 1.44429958, + "balance_loss_mlp": 1.75955927, + "epoch": 0.00011606987406418665, + "flos": 69992056719360.0, + "grad_norm": 2.5548336860068344, + "language_loss": 0.66045374, + "learning_rate": 7.988799327617516e-07, + "loss": 0.71509397, + "num_input_tokens_seen": 152570, + "router_z_loss_clip": 21.125, + "router_z_loss_mlp": 1.53125, + "step": 4, + "time_per_iteration": 3.059968948364258 + }, + { + "auxiliary_loss_clip": 0.03297501, + "auxiliary_loss_mlp": 0.01539173, + "balance_loss_clip": 1.54107356, + "balance_loss_mlp": 1.24992347, + "epoch": 0.0001450873425802333, + "flos": 24745669228800.0, + "grad_norm": 47.142083084680294, + "language_loss": 2.25973439, + "learning_rate": 9.274708801606189e-07, + "loss": 2.30810142, + "num_input_tokens_seen": 166005, + "router_z_loss_clip": 17.5625, + "router_z_loss_mlp": 2.89355469, + "step": 5, + "time_per_iteration": 2.4401609897613525 + }, + { + "auxiliary_loss_clip": 0.03251427, + "auxiliary_loss_mlp": 0.01512932, + "balance_loss_clip": 1.52715015, + "balance_loss_mlp": 1.22644877, + "epoch": 0.00017410481109627997, + "flos": 64263439814400.0, + "grad_norm": 23.45378722379591, + "language_loss": 1.52832699, + "learning_rate": 1.0325373343838831e-06, + "loss": 1.57597065, + "num_input_tokens_seen": 189205, + "router_z_loss_clip": 17.234375, + "router_z_loss_mlp": 2.86328125, + "step": 6, + "time_per_iteration": 2.687267303466797 + }, + { + "auxiliary_loss_clip": 0.03258441, + "auxiliary_loss_mlp": 0.0145736, + "balance_loss_clip": 1.53551304, + "balance_loss_mlp": 1.19796014, + "epoch": 0.00020312227961232662, + "flos": 18728981896320.0, + "grad_norm": 35.69190422308644, + "language_loss": 1.76463866, + "learning_rate": 1.1213697556858757e-06, + "loss": 1.81179667, + "num_input_tokens_seen": 202350, + "router_z_loss_clip": 17.234375, + "router_z_loss_mlp": 2.59375, + "step": 7, + "time_per_iteration": 8.356410264968872 + }, + { + "auxiliary_loss_clip": 0.03228472, + "auxiliary_loss_mlp": 0.01449919, + "balance_loss_clip": 1.51401973, + "balance_loss_mlp": 1.19624186, + "epoch": 0.0002321397481283733, + "flos": 12852331493760.0, + "grad_norm": 36.15735595702056, + "language_loss": 2.08638453, + "learning_rate": 1.1983198991426273e-06, + "loss": 2.13316822, + "num_input_tokens_seen": 214925, + "router_z_loss_clip": 17.1640625, + "router_z_loss_mlp": 2.53515625, + "step": 8, + "time_per_iteration": 2.5538549423217773 + }, + { + "auxiliary_loss_clip": 0.03264161, + "auxiliary_loss_mlp": 0.01447531, + "balance_loss_clip": 1.52788246, + "balance_loss_mlp": 1.18279147, + "epoch": 0.00026115721664441994, + "flos": 29417467011840.0, + "grad_norm": 46.54000303124526, + "language_loss": 1.9089942, + "learning_rate": 1.266194736006015e-06, + "loss": 1.95611107, + "num_input_tokens_seen": 234220, + "router_z_loss_clip": 17.3125, + "router_z_loss_mlp": 2.64648438, + "step": 9, + "time_per_iteration": 4.961307525634766 + }, + { + "auxiliary_loss_clip": 0.03472012, + "auxiliary_loss_mlp": 0.01857982, + "balance_loss_clip": 1.44670403, + "balance_loss_mlp": 1.71683872, + "epoch": 0.0002901746851604666, + "flos": 63782928639360.0, + "grad_norm": 2.7130452429604226, + "language_loss": 0.66902542, + "learning_rate": 1.326910846541495e-06, + "loss": 0.72232544, + "num_input_tokens_seen": 290580, + "router_z_loss_clip": 20.25, + "router_z_loss_mlp": 1.4140625, + "step": 10, + "time_per_iteration": 2.923848867416382 + }, + { + "auxiliary_loss_clip": 0.03165447, + "auxiliary_loss_mlp": 0.01533381, + "balance_loss_clip": 1.52500153, + "balance_loss_mlp": 1.25700641, + "epoch": 0.00031919215367651324, + "flos": 22702551932160.0, + "grad_norm": 10.62066737072138, + "language_loss": 1.5693748, + "learning_rate": 1.3818352494454209e-06, + "loss": 1.61636305, + "num_input_tokens_seen": 305540, + "router_z_loss_clip": 16.3984375, + "router_z_loss_mlp": 2.76269531, + "step": 11, + "time_per_iteration": 2.611985921859741 + }, + { + "auxiliary_loss_clip": 0.03179511, + "auxiliary_loss_mlp": 0.01466505, + "balance_loss_clip": 1.53380966, + "balance_loss_mlp": 1.19737828, + "epoch": 0.00034820962219255994, + "flos": 15990569406720.0, + "grad_norm": 14.816110975821323, + "language_loss": 1.53501344, + "learning_rate": 1.431977300764759e-06, + "loss": 1.58147359, + "num_input_tokens_seen": 318590, + "router_z_loss_clip": 16.46875, + "router_z_loss_mlp": 2.69140625, + "step": 12, + "time_per_iteration": 2.7740864753723145 + }, + { + "auxiliary_loss_clip": 0.03145821, + "auxiliary_loss_mlp": 0.01554617, + "balance_loss_clip": 1.46849823, + "balance_loss_mlp": 1.45314562, + "epoch": 0.0003772270907086066, + "flos": 71008727777280.0, + "grad_norm": 2.569390798914373, + "language_loss": 0.63734543, + "learning_rate": 1.4781035166087354e-06, + "loss": 0.68434978, + "num_input_tokens_seen": 382525, + "router_z_loss_clip": 16.75, + "router_z_loss_mlp": 1.015625, + "step": 13, + "time_per_iteration": 3.206904411315918 + }, + { + "auxiliary_loss_clip": 0.03120919, + "auxiliary_loss_mlp": 0.01519744, + "balance_loss_clip": 1.47066641, + "balance_loss_mlp": 1.42132533, + "epoch": 0.00040624455922465323, + "flos": 74774284732800.0, + "grad_norm": 2.4304842164160965, + "language_loss": 0.6334818, + "learning_rate": 1.5208097220667513e-06, + "loss": 0.67988843, + "num_input_tokens_seen": 447495, + "router_z_loss_clip": 16.5, + "router_z_loss_mlp": 0.98046875, + "step": 14, + "time_per_iteration": 3.170572519302368 + }, + { + "auxiliary_loss_clip": 0.03088063, + "auxiliary_loss_mlp": 0.01496311, + "balance_loss_clip": 1.4717474, + "balance_loss_mlp": 1.40170693, + "epoch": 0.0004352620277406999, + "flos": 74764369906560.0, + "grad_norm": 2.6749190307833053, + "language_loss": 0.64686126, + "learning_rate": 1.5605682481636264e-06, + "loss": 0.69270498, + "num_input_tokens_seen": 503225, + "router_z_loss_clip": 16.125, + "router_z_loss_mlp": 0.9453125, + "step": 15, + "time_per_iteration": 3.067427158355713 + }, + { + "auxiliary_loss_clip": 0.03106893, + "auxiliary_loss_mlp": 0.01518949, + "balance_loss_clip": 1.5343883, + "balance_loss_mlp": 1.2446723, + "epoch": 0.0004642794962567466, + "flos": 12495309264000.0, + "grad_norm": 15.486926933594598, + "language_loss": 1.69735599, + "learning_rate": 1.5977598655235032e-06, + "loss": 1.74361444, + "num_input_tokens_seen": 515655, + "router_z_loss_clip": 15.7265625, + "router_z_loss_mlp": 2.7421875, + "step": 16, + "time_per_iteration": 2.58730411529541 + }, + { + "auxiliary_loss_clip": 0.03096529, + "auxiliary_loss_mlp": 0.01465619, + "balance_loss_clip": 1.52979922, + "balance_loss_mlp": 1.21136951, + "epoch": 0.0004932969647727932, + "flos": 22301574433920.0, + "grad_norm": 7.846744896113187, + "language_loss": 1.59019029, + "learning_rate": 1.6326960198921147e-06, + "loss": 1.63581169, + "num_input_tokens_seen": 531020, + "router_z_loss_clip": 15.65625, + "router_z_loss_mlp": 2.54199219, + "step": 17, + "time_per_iteration": 2.633697986602783 + }, + { + "auxiliary_loss_clip": 0.03039613, + "auxiliary_loss_mlp": 0.01453096, + "balance_loss_clip": 1.52606666, + "balance_loss_mlp": 1.20933676, + "epoch": 0.0005223144332888399, + "flos": 14861303614080.0, + "grad_norm": 9.384512127883402, + "language_loss": 1.7185626, + "learning_rate": 1.6656347023868906e-06, + "loss": 1.76348972, + "num_input_tokens_seen": 545155, + "router_z_loss_clip": 15.1328125, + "router_z_loss_mlp": 2.43847656, + "step": 18, + "time_per_iteration": 2.58577561378479 + }, + { + "auxiliary_loss_clip": 0.03000507, + "auxiliary_loss_mlp": 0.01505201, + "balance_loss_clip": 1.53409696, + "balance_loss_mlp": 1.25276327, + "epoch": 0.0005513319018048866, + "flos": 19091240830080.0, + "grad_norm": 8.316845407175842, + "language_loss": 1.20145273, + "learning_rate": 1.696792023158303e-06, + "loss": 1.24650979, + "num_input_tokens_seen": 559275, + "router_z_loss_clip": 14.671875, + "router_z_loss_mlp": 2.52539062, + "step": 19, + "time_per_iteration": 2.6427271366119385 + }, + { + "auxiliary_loss_clip": 0.02886, + "auxiliary_loss_mlp": 0.01459671, + "balance_loss_clip": 1.53287196, + "balance_loss_mlp": 1.22993135, + "epoch": 0.0005803493703209332, + "flos": 32298290403840.0, + "grad_norm": 7.310042711596369, + "language_loss": 1.45324039, + "learning_rate": 1.7263508129223706e-06, + "loss": 1.49669719, + "num_input_tokens_seen": 574905, + "router_z_loss_clip": 13.5234375, + "router_z_loss_mlp": 2.29785156, + "step": 20, + "time_per_iteration": 2.73553729057312 + }, + { + "auxiliary_loss_clip": 0.02401284, + "auxiliary_loss_mlp": 0.01251714, + "balance_loss_clip": 1.54193139, + "balance_loss_mlp": 1.20994329, + "epoch": 0.0006093668388369799, + "flos": 60690356651520.0, + "grad_norm": 1.6085041453087054, + "language_loss": 0.64893621, + "learning_rate": 1.754467123688883e-06, + "loss": 0.68546623, + "num_input_tokens_seen": 636915, + "router_z_loss_clip": 8.5625, + "router_z_loss_mlp": 0.41796875, + "step": 21, + "time_per_iteration": 3.0670342445373535 + }, + { + "auxiliary_loss_clip": 0.02866018, + "auxiliary_loss_mlp": 0.01420139, + "balance_loss_clip": 1.53874898, + "balance_loss_mlp": 1.21853185, + "epoch": 0.0006383843073530265, + "flos": 16319730504960.0, + "grad_norm": 9.202088068964391, + "language_loss": 1.45236158, + "learning_rate": 1.7812752158262967e-06, + "loss": 1.49522316, + "num_input_tokens_seen": 649135, + "router_z_loss_clip": 13.2734375, + "router_z_loss_mlp": 2.01660156, + "step": 22, + "time_per_iteration": 2.6586127281188965 + }, + { + "auxiliary_loss_clip": 0.02711173, + "auxiliary_loss_mlp": 0.0165409, + "balance_loss_clip": 1.54441071, + "balance_loss_mlp": 1.41729224, + "epoch": 0.0006674017758690732, + "flos": 13779065289600.0, + "grad_norm": 7.47430671852817, + "language_loss": 1.41301489, + "learning_rate": 1.806891435649222e-06, + "loss": 1.45666754, + "num_input_tokens_seen": 661465, + "router_z_loss_clip": 11.6875, + "router_z_loss_mlp": 2.36816406, + "step": 23, + "time_per_iteration": 2.5883629322052 + }, + { + "auxiliary_loss_clip": 0.02820873, + "auxiliary_loss_mlp": 0.01527152, + "balance_loss_clip": 1.53766227, + "balance_loss_mlp": 1.29245257, + "epoch": 0.0006964192443851199, + "flos": 40690886912640.0, + "grad_norm": 4.486362640955505, + "language_loss": 1.10189199, + "learning_rate": 1.8314172671456348e-06, + "loss": 1.14537239, + "num_input_tokens_seen": 680760, + "router_z_loss_clip": 12.8359375, + "router_z_loss_mlp": 2.34570312, + "step": 24, + "time_per_iteration": 2.789806604385376 + }, + { + "auxiliary_loss_clip": 0.02756078, + "auxiliary_loss_mlp": 0.01466465, + "balance_loss_clip": 1.52972293, + "balance_loss_mlp": 1.27258253, + "epoch": 0.0007254367129011665, + "flos": 16609719191040.0, + "grad_norm": 7.331045567171508, + "language_loss": 1.79279852, + "learning_rate": 1.8549417603212378e-06, + "loss": 1.83502388, + "num_input_tokens_seen": 693140, + "router_z_loss_clip": 12.265625, + "router_z_loss_mlp": 1.94042969, + "step": 25, + "time_per_iteration": 2.6067922115325928 + }, + { + "auxiliary_loss_clip": 0.02739747, + "auxiliary_loss_mlp": 0.01512038, + "balance_loss_clip": 1.52733231, + "balance_loss_mlp": 1.30566263, + "epoch": 0.0007544541814172132, + "flos": 11611378661760.0, + "grad_norm": 6.396111433687333, + "language_loss": 1.21683621, + "learning_rate": 1.8775434829896112e-06, + "loss": 1.259354, + "num_input_tokens_seen": 704545, + "router_z_loss_clip": 12.125, + "router_z_loss_mlp": 2.06445312, + "step": 26, + "time_per_iteration": 2.7037360668182373 + }, + { + "auxiliary_loss_clip": 0.0280432, + "auxiliary_loss_mlp": 0.01457033, + "balance_loss_clip": 1.53715479, + "balance_loss_mlp": 1.24216974, + "epoch": 0.0007834716499332599, + "flos": 24455261606400.0, + "grad_norm": 3.261184876850625, + "language_loss": 1.03125691, + "learning_rate": 1.8992921040090223e-06, + "loss": 1.07387042, + "num_input_tokens_seen": 723595, + "router_z_loss_clip": 12.671875, + "router_z_loss_mlp": 2.14941406, + "step": 27, + "time_per_iteration": 2.8553102016448975 + }, + { + "auxiliary_loss_clip": 0.02691009, + "auxiliary_loss_mlp": 0.01403677, + "balance_loss_clip": 1.52407157, + "balance_loss_mlp": 1.21885443, + "epoch": 0.0008124891184493065, + "flos": 31678058367360.0, + "grad_norm": 4.903962995526711, + "language_loss": 1.33802319, + "learning_rate": 1.920249688447627e-06, + "loss": 1.37897003, + "num_input_tokens_seen": 737365, + "router_z_loss_clip": 11.671875, + "router_z_loss_mlp": 1.84765625, + "step": 28, + "time_per_iteration": 2.7147858142852783 + }, + { + "auxiliary_loss_clip": 0.02161496, + "auxiliary_loss_mlp": 0.01233571, + "balance_loss_clip": 1.55222535, + "balance_loss_mlp": 1.19752264, + "epoch": 0.0008415065869653532, + "flos": 66611416170240.0, + "grad_norm": 1.257331380583588, + "language_loss": 0.59844661, + "learning_rate": 1.940471765372691e-06, + "loss": 0.63239729, + "num_input_tokens_seen": 804120, + "router_z_loss_clip": 6.09375, + "router_z_loss_mlp": 0.36132812, + "step": 29, + "time_per_iteration": 3.287592887878418 + }, + { + "auxiliary_loss_clip": 0.02660626, + "auxiliary_loss_mlp": 0.01434879, + "balance_loss_clip": 1.52278399, + "balance_loss_mlp": 1.23927999, + "epoch": 0.0008705240554813998, + "flos": 74732286366720.0, + "grad_norm": 2.713091338656989, + "language_loss": 0.91488504, + "learning_rate": 1.9600082145445022e-06, + "loss": 0.95584011, + "num_input_tokens_seen": 831390, + "router_z_loss_clip": 11.375, + "router_z_loss_mlp": 1.95703125, + "step": 30, + "time_per_iteration": 3.100003719329834 + }, + { + "auxiliary_loss_clip": 0.02573309, + "auxiliary_loss_mlp": 0.01439788, + "balance_loss_clip": 1.52417159, + "balance_loss_mlp": 1.25830376, + "epoch": 0.0008995415239974465, + "flos": 74732775125760.0, + "grad_norm": 3.707873799596842, + "language_loss": 1.13495898, + "learning_rate": 1.9789040076651924e-06, + "loss": 1.17508984, + "num_input_tokens_seen": 856215, + "router_z_loss_clip": 10.4921875, + "router_z_loss_mlp": 1.81542969, + "step": 31, + "time_per_iteration": 3.0358688831329346 + }, + { + "auxiliary_loss_clip": 0.02534668, + "auxiliary_loss_mlp": 0.01467588, + "balance_loss_clip": 1.51073241, + "balance_loss_mlp": 1.27761602, + "epoch": 0.0009285589925134932, + "flos": 26679578192640.0, + "grad_norm": 3.057902883053167, + "language_loss": 1.19209409, + "learning_rate": 1.997199831904379e-06, + "loss": 1.2321167, + "num_input_tokens_seen": 870260, + "router_z_loss_clip": 10.2578125, + "router_z_loss_mlp": 1.90039062, + "step": 32, + "time_per_iteration": 2.6428632736206055 + }, + { + "auxiliary_loss_clip": 0.02009267, + "auxiliary_loss_mlp": 0.0115393, + "balance_loss_clip": 1.54129529, + "balance_loss_mlp": 1.10891628, + "epoch": 0.0009575764610295398, + "flos": 71598619975680.0, + "grad_norm": 1.4810889338536815, + "language_loss": 0.63696635, + "learning_rate": 2.014932617448428e-06, + "loss": 0.66859829, + "num_input_tokens_seen": 935505, + "router_z_loss_clip": 4.6875, + "router_z_loss_mlp": 0.44921875, + "step": 33, + "time_per_iteration": 3.098554849624634 + }, + { + "auxiliary_loss_clip": 0.02487295, + "auxiliary_loss_mlp": 0.01437693, + "balance_loss_clip": 1.51201236, + "balance_loss_mlp": 1.26135921, + "epoch": 0.0009865939295455864, + "flos": 29199608928000.0, + "grad_norm": 2.771412565297549, + "language_loss": 1.16969705, + "learning_rate": 2.0321359862729905e-06, + "loss": 1.20894682, + "num_input_tokens_seen": 956585, + "router_z_loss_clip": 9.7421875, + "router_z_loss_mlp": 1.76464844, + "step": 34, + "time_per_iteration": 2.7917463779449463 + }, + { + "auxiliary_loss_clip": 0.02401547, + "auxiliary_loss_mlp": 0.0134993, + "balance_loss_clip": 1.5025171, + "balance_loss_mlp": 1.20306432, + "epoch": 0.0010156113980616332, + "flos": 32444785935360.0, + "grad_norm": 5.216669599842201, + "language_loss": 1.22353089, + "learning_rate": 2.0488406358464945e-06, + "loss": 1.26104569, + "num_input_tokens_seen": 971900, + "router_z_loss_clip": 9.0, + "router_z_loss_mlp": 1.47070312, + "step": 35, + "time_per_iteration": 2.772977352142334 + }, + { + "auxiliary_loss_clip": 0.01935979, + "auxiliary_loss_mlp": 0.01129919, + "balance_loss_clip": 1.52769661, + "balance_loss_mlp": 1.07346189, + "epoch": 0.0010446288665776798, + "flos": 68930522697600.0, + "grad_norm": 1.3936657487118012, + "language_loss": 0.63292074, + "learning_rate": 2.0650746687677663e-06, + "loss": 0.6635797, + "num_input_tokens_seen": 1031825, + "router_z_loss_clip": 4.09375, + "router_z_loss_mlp": 0.56640625, + "step": 36, + "time_per_iteration": 3.0520150661468506 + }, + { + "auxiliary_loss_clip": 0.01920827, + "auxiliary_loss_mlp": 0.01139899, + "balance_loss_clip": 1.52312708, + "balance_loss_mlp": 1.07848191, + "epoch": 0.0010736463350937264, + "flos": 58461641233920.0, + "grad_norm": 1.3095910640256607, + "language_loss": 0.61487472, + "learning_rate": 2.080863877229568e-06, + "loss": 0.64548194, + "num_input_tokens_seen": 1090910, + "router_z_loss_clip": 3.984375, + "router_z_loss_mlp": 0.61328125, + "step": 37, + "time_per_iteration": 3.041842222213745 + }, + { + "auxiliary_loss_clip": 0.02329548, + "auxiliary_loss_mlp": 0.01322078, + "balance_loss_clip": 1.4941361, + "balance_loss_mlp": 1.17921734, + "epoch": 0.0011026638036097732, + "flos": 34962652166400.0, + "grad_norm": 3.1581936288510524, + "language_loss": 1.08860183, + "learning_rate": 2.096231989539179e-06, + "loss": 1.12511814, + "num_input_tokens_seen": 1107990, + "router_z_loss_clip": 8.3515625, + "router_z_loss_mlp": 1.42773438, + "step": 38, + "time_per_iteration": 2.7131896018981934 + }, + { + "auxiliary_loss_clip": 0.0188756, + "auxiliary_loss_mlp": 0.01129178, + "balance_loss_clip": 1.50685978, + "balance_loss_mlp": 1.06089449, + "epoch": 0.0011316812721258198, + "flos": 61786419874560.0, + "grad_norm": 1.1099545654332628, + "language_loss": 0.63496351, + "learning_rate": 2.1112008846117425e-06, + "loss": 0.66513085, + "num_input_tokens_seen": 1174060, + "router_z_loss_clip": 3.8125, + "router_z_loss_mlp": 0.68359375, + "step": 39, + "time_per_iteration": 3.0811476707458496 + }, + { + "auxiliary_loss_clip": 0.02280499, + "auxiliary_loss_mlp": 0.01232919, + "balance_loss_clip": 1.4708581, + "balance_loss_mlp": 1.11800122, + "epoch": 0.0011606987406418664, + "flos": 19274185687680.0, + "grad_norm": 3.0643814287825797, + "language_loss": 1.3068161, + "learning_rate": 2.1257907793032464e-06, + "loss": 1.34195042, + "num_input_tokens_seen": 1187835, + "router_z_loss_clip": 8.0859375, + "router_z_loss_mlp": 1.14794922, + "step": 40, + "time_per_iteration": 2.7388036251068115 + }, + { + "auxiliary_loss_clip": 0.02271193, + "auxiliary_loss_mlp": 0.01275555, + "balance_loss_clip": 1.48185253, + "balance_loss_mlp": 1.1422317, + "epoch": 0.0011897162091579132, + "flos": 13726696383360.0, + "grad_norm": 4.873487766085185, + "language_loss": 1.43272114, + "learning_rate": 2.140020392608441e-06, + "loss": 1.46818864, + "num_input_tokens_seen": 1198420, + "router_z_loss_clip": 7.890625, + "router_z_loss_mlp": 1.33300781, + "step": 41, + "time_per_iteration": 2.5985729694366455 + }, + { + "auxiliary_loss_clip": 0.01833073, + "auxiliary_loss_mlp": 0.01117774, + "balance_loss_clip": 1.46793199, + "balance_loss_mlp": 1.04224277, + "epoch": 0.0012187336776739598, + "flos": 61379332888320.0, + "grad_norm": 0.918598005694619, + "language_loss": 0.59410304, + "learning_rate": 2.153907090069759e-06, + "loss": 0.62361151, + "num_input_tokens_seen": 1260125, + "router_z_loss_clip": 3.65625, + "router_z_loss_mlp": 0.75390625, + "step": 42, + "time_per_iteration": 3.1933703422546387 + }, + { + "auxiliary_loss_clip": 0.01811938, + "auxiliary_loss_mlp": 0.0110144, + "balance_loss_clip": 1.4554013, + "balance_loss_mlp": 1.02552772, + "epoch": 0.0012477511461900063, + "flos": 70177864308480.0, + "grad_norm": 1.00331080662038, + "language_loss": 0.6206072, + "learning_rate": 2.167467011191937e-06, + "loss": 0.64974099, + "num_input_tokens_seen": 1323940, + "router_z_loss_clip": 3.5625, + "router_z_loss_mlp": 0.7578125, + "step": 43, + "time_per_iteration": 3.1391761302948 + }, + { + "auxiliary_loss_clip": 0.0178467, + "auxiliary_loss_mlp": 0.010831, + "balance_loss_clip": 1.43967652, + "balance_loss_mlp": 1.00795043, + "epoch": 0.001276768614706053, + "flos": 74769641521920.0, + "grad_norm": 1.1881478368968181, + "language_loss": 0.60726774, + "learning_rate": 2.180715182207172e-06, + "loss": 0.63594544, + "num_input_tokens_seen": 1387700, + "router_z_loss_clip": 3.453125, + "router_z_loss_mlp": 0.75, + "step": 44, + "time_per_iteration": 3.126199245452881 + }, + { + "auxiliary_loss_clip": 0.0219437, + "auxiliary_loss_mlp": 0.01298254, + "balance_loss_clip": 1.45635843, + "balance_loss_mlp": 1.16092479, + "epoch": 0.0013057860832220998, + "flos": 20769201550080.0, + "grad_norm": 2.718015514132568, + "language_loss": 1.07219601, + "learning_rate": 2.193665616166634e-06, + "loss": 1.1071223, + "num_input_tokens_seen": 1405335, + "router_z_loss_clip": 7.3828125, + "router_z_loss_mlp": 1.37304688, + "step": 45, + "time_per_iteration": 2.6652708053588867 + }, + { + "auxiliary_loss_clip": 0.01728205, + "auxiliary_loss_mlp": 0.01069699, + "balance_loss_clip": 1.40860438, + "balance_loss_mlp": 0.99950892, + "epoch": 0.0013348035517381463, + "flos": 69948241096320.0, + "grad_norm": 1.0380559749508205, + "language_loss": 0.65831256, + "learning_rate": 2.206331402030098e-06, + "loss": 0.68629164, + "num_input_tokens_seen": 1458275, + "router_z_loss_clip": 3.1875, + "router_z_loss_mlp": 0.703125, + "step": 46, + "time_per_iteration": 2.954298257827759 + }, + { + "auxiliary_loss_clip": 0.02095168, + "auxiliary_loss_mlp": 0.01340874, + "balance_loss_clip": 1.44470096, + "balance_loss_mlp": 1.20802712, + "epoch": 0.001363821020254193, + "flos": 18288901808640.0, + "grad_norm": 2.7846409036885498, + "language_loss": 1.02356088, + "learning_rate": 2.2187247841737033e-06, + "loss": 1.05792129, + "num_input_tokens_seen": 1472945, + "router_z_loss_clip": 6.50390625, + "router_z_loss_mlp": 1.32910156, + "step": 47, + "time_per_iteration": 2.601274251937866 + }, + { + "auxiliary_loss_clip": 0.01678638, + "auxiliary_loss_mlp": 0.01071745, + "balance_loss_clip": 1.38308764, + "balance_loss_mlp": 1.00765789, + "epoch": 0.0013928384887702398, + "flos": 68026306728960.0, + "grad_norm": 0.9029402795187478, + "language_loss": 0.56028962, + "learning_rate": 2.230857233526511e-06, + "loss": 0.58779347, + "num_input_tokens_seen": 1537970, + "router_z_loss_clip": 2.953125, + "router_z_loss_mlp": 0.640625, + "step": 48, + "time_per_iteration": 3.1453464031219482 + }, + { + "auxiliary_loss_clip": 0.02114736, + "auxiliary_loss_mlp": 0.01297129, + "balance_loss_clip": 1.42917192, + "balance_loss_mlp": 1.16933715, + "epoch": 0.0014218559572862863, + "flos": 20842798429440.0, + "grad_norm": 3.185106826766501, + "language_loss": 1.00069332, + "learning_rate": 2.2427395113717513e-06, + "loss": 1.03481209, + "num_input_tokens_seen": 1554410, + "router_z_loss_clip": 6.86328125, + "router_z_loss_mlp": 1.27734375, + "step": 49, + "time_per_iteration": 2.628575325012207 + }, + { + "auxiliary_loss_clip": 0.01635232, + "auxiliary_loss_mlp": 0.01071202, + "balance_loss_clip": 1.36307096, + "balance_loss_mlp": 1.01360023, + "epoch": 0.001450873425802333, + "flos": 71726540797440.0, + "grad_norm": 0.9529609040648295, + "language_loss": 0.60865295, + "learning_rate": 2.254381726702114e-06, + "loss": 0.63571727, + "num_input_tokens_seen": 1615650, + "router_z_loss_clip": 2.71875, + "router_z_loss_mlp": 0.57421875, + "step": 50, + "time_per_iteration": 3.266808032989502 + }, + { + "auxiliary_loss_clip": 0.01616269, + "auxiliary_loss_mlp": 0.01068675, + "balance_loss_clip": 1.35608554, + "balance_loss_mlp": 1.01488817, + "epoch": 0.0014798908943183797, + "flos": 74658338507520.0, + "grad_norm": 0.9890726420163949, + "language_loss": 0.62493533, + "learning_rate": 2.265793387895122e-06, + "loss": 0.65178478, + "num_input_tokens_seen": 1679695, + "router_z_loss_clip": 2.59375, + "router_z_loss_mlp": 0.5390625, + "step": 51, + "time_per_iteration": 3.150045156478882 + }, + { + "auxiliary_loss_clip": 0.01599068, + "auxiliary_loss_mlp": 0.01063426, + "balance_loss_clip": 1.35016608, + "balance_loss_mlp": 1.01383495, + "epoch": 0.0015089083628344263, + "flos": 63347980521600.0, + "grad_norm": 0.9877313416677833, + "language_loss": 0.66811001, + "learning_rate": 2.276983449370487e-06, + "loss": 0.69473493, + "num_input_tokens_seen": 1738825, + "router_z_loss_clip": 2.484375, + "router_z_loss_mlp": 0.49609375, + "step": 52, + "time_per_iteration": 2.962188959121704 + }, + { + "auxiliary_loss_clip": 0.01582143, + "auxiliary_loss_mlp": 0.01059098, + "balance_loss_clip": 1.34469986, + "balance_loss_mlp": 1.01274896, + "epoch": 0.001537925831350473, + "flos": 74770374660480.0, + "grad_norm": 0.848657650297548, + "language_loss": 0.58383512, + "learning_rate": 2.287960353803055e-06, + "loss": 0.61024761, + "num_input_tokens_seen": 1797415, + "router_z_loss_clip": 2.375, + "router_z_loss_mlp": 0.46289062, + "step": 53, + "time_per_iteration": 3.053617477416992 + }, + { + "auxiliary_loss_clip": 0.01996323, + "auxiliary_loss_mlp": 0.01178997, + "balance_loss_clip": 1.3854512, + "balance_loss_mlp": 1.08763552, + "epoch": 0.0015669432998665197, + "flos": 28760261978880.0, + "grad_norm": 2.3931307615450357, + "language_loss": 1.0319581, + "learning_rate": 2.2987320703898984e-06, + "loss": 1.06371117, + "num_input_tokens_seen": 1820515, + "router_z_loss_clip": 6.109375, + "router_z_loss_mlp": 0.91357422, + "step": 54, + "time_per_iteration": 2.8062310218811035 + }, + { + "auxiliary_loss_clip": 0.01554533, + "auxiliary_loss_mlp": 0.01044361, + "balance_loss_clip": 1.33505344, + "balance_loss_mlp": 1.00354409, + "epoch": 0.0015959607683825663, + "flos": 61774934037120.0, + "grad_norm": 0.8767393946017232, + "language_loss": 0.54807585, + "learning_rate": 2.30930612960604e-06, + "loss": 0.57406479, + "num_input_tokens_seen": 1875210, + "router_z_loss_clip": 2.1875, + "router_z_loss_mlp": 0.40820312, + "step": 55, + "time_per_iteration": 2.941366195678711 + }, + { + "auxiliary_loss_clip": 0.02002657, + "auxiliary_loss_mlp": 0.01283899, + "balance_loss_clip": 1.39547229, + "balance_loss_mlp": 1.16554809, + "epoch": 0.001624978236898613, + "flos": 13727499344640.0, + "grad_norm": 2.92295858594285, + "language_loss": 1.264503, + "learning_rate": 2.319689654828503e-06, + "loss": 1.29736853, + "num_input_tokens_seen": 1886820, + "router_z_loss_clip": 6.078125, + "router_z_loss_mlp": 1.18408203, + "step": 56, + "time_per_iteration": 2.568812608718872 + }, + { + "auxiliary_loss_clip": 0.01532602, + "auxiliary_loss_mlp": 0.01036263, + "balance_loss_clip": 1.3253541, + "balance_loss_mlp": 0.99945098, + "epoch": 0.0016539957054146595, + "flos": 61243834435200.0, + "grad_norm": 0.7813970386029492, + "language_loss": 0.60254526, + "learning_rate": 2.3298893911613107e-06, + "loss": 0.62823391, + "num_input_tokens_seen": 1952960, + "router_z_loss_clip": 2.0625, + "router_z_loss_mlp": 0.3671875, + "step": 57, + "time_per_iteration": 3.1637184619903564 + }, + { + "auxiliary_loss_clip": 0.02011072, + "auxiliary_loss_mlp": 0.01249702, + "balance_loss_clip": 1.38608909, + "balance_loss_mlp": 1.12210035, + "epoch": 0.0016830131739307063, + "flos": 20042309710080.0, + "grad_norm": 2.4379556457185876, + "language_loss": 1.091712, + "learning_rate": 2.339911731753567e-06, + "loss": 1.12431979, + "num_input_tokens_seen": 1970440, + "router_z_loss_clip": 6.24609375, + "router_z_loss_mlp": 1.27636719, + "step": 58, + "time_per_iteration": 2.7443063259124756 + }, + { + "auxiliary_loss_clip": 0.02000212, + "auxiliary_loss_mlp": 0.0122152, + "balance_loss_clip": 1.37860107, + "balance_loss_mlp": 1.10254931, + "epoch": 0.001712030642446753, + "flos": 22338163405440.0, + "grad_norm": 2.703923456848877, + "language_loss": 0.81196541, + "learning_rate": 2.3497627418677867e-06, + "loss": 0.84418273, + "num_input_tokens_seen": 1985535, + "router_z_loss_clip": 6.21875, + "router_z_loss_mlp": 1.18945312, + "step": 59, + "time_per_iteration": 2.6451656818389893 + }, + { + "auxiliary_loss_clip": 0.01502817, + "auxiliary_loss_mlp": 0.01034954, + "balance_loss_clip": 1.31146049, + "balance_loss_mlp": 1.00119376, + "epoch": 0.0017410481109627995, + "flos": 69924743884800.0, + "grad_norm": 1.1385609243371808, + "language_loss": 0.56981313, + "learning_rate": 2.359448180925378e-06, + "loss": 0.59519082, + "num_input_tokens_seen": 2051030, + "router_z_loss_clip": 1.9140625, + "router_z_loss_mlp": 0.33789062, + "step": 60, + "time_per_iteration": 3.11531138420105 + }, + { + "auxiliary_loss_clip": 0.01935625, + "auxiliary_loss_mlp": 0.01208567, + "balance_loss_clip": 1.3523953, + "balance_loss_mlp": 1.10123134, + "epoch": 0.0017700655794788463, + "flos": 12232867507200.0, + "grad_norm": 3.1130658965361424, + "language_loss": 1.1171881, + "learning_rate": 2.3689735227299243e-06, + "loss": 1.14863002, + "num_input_tokens_seen": 2063875, + "router_z_loss_clip": 5.8359375, + "router_z_loss_mlp": 1.07226562, + "step": 61, + "time_per_iteration": 2.5642151832580566 + }, + { + "auxiliary_loss_clip": 0.01482015, + "auxiliary_loss_mlp": 0.01041465, + "balance_loss_clip": 1.29814148, + "balance_loss_mlp": 1.0080868, + "epoch": 0.001799083047994893, + "flos": 68539913873280.0, + "grad_norm": 0.9035995399815414, + "language_loss": 0.6252172, + "learning_rate": 2.3783439740460682e-06, + "loss": 0.65045202, + "num_input_tokens_seen": 2124740, + "router_z_loss_clip": 1.84375, + "router_z_loss_mlp": 0.33398438, + "step": 62, + "time_per_iteration": 3.1399364471435547 + }, + { + "auxiliary_loss_clip": 0.01890779, + "auxiliary_loss_mlp": 0.01213708, + "balance_loss_clip": 1.35352194, + "balance_loss_mlp": 1.11061597, + "epoch": 0.0018281005165109395, + "flos": 20732996603520.0, + "grad_norm": 2.828640090466237, + "language_loss": 1.10353088, + "learning_rate": 2.3875644916918902e-06, + "loss": 1.13457584, + "num_input_tokens_seen": 2138655, + "router_z_loss_clip": 5.375, + "router_z_loss_mlp": 1.03076172, + "step": 63, + "time_per_iteration": 2.575239896774292 + }, + { + "auxiliary_loss_clip": 0.01918757, + "auxiliary_loss_mlp": 0.01207532, + "balance_loss_clip": 1.3594048, + "balance_loss_mlp": 1.09390163, + "epoch": 0.0018571179850269863, + "flos": 35768237944320.0, + "grad_norm": 2.6640540456408006, + "language_loss": 1.22029424, + "learning_rate": 2.3966397982852547e-06, + "loss": 1.25155711, + "num_input_tokens_seen": 2156155, + "router_z_loss_clip": 5.5859375, + "router_z_loss_mlp": 1.13720703, + "step": 64, + "time_per_iteration": 2.689847946166992 + }, + { + "auxiliary_loss_clip": 0.01908557, + "auxiliary_loss_mlp": 0.0123512, + "balance_loss_clip": 1.35130894, + "balance_loss_mlp": 1.12797451, + "epoch": 0.001886135453543033, + "flos": 29051263094400.0, + "grad_norm": 3.006221705847965, + "language_loss": 1.25664818, + "learning_rate": 2.4055743967693543e-06, + "loss": 1.28808498, + "num_input_tokens_seen": 2172080, + "router_z_loss_clip": 5.5703125, + "router_z_loss_mlp": 1.07177734, + "step": 65, + "time_per_iteration": 2.6699626445770264 + }, + { + "auxiliary_loss_clip": 0.01850071, + "auxiliary_loss_mlp": 0.0120899, + "balance_loss_clip": 1.34611332, + "balance_loss_mlp": 1.10232115, + "epoch": 0.0019151529220590795, + "flos": 15882303680640.0, + "grad_norm": 3.3925344764120164, + "language_loss": 1.05812478, + "learning_rate": 2.4143725838293036e-06, + "loss": 1.08871531, + "num_input_tokens_seen": 2184295, + "router_z_loss_clip": 5.0390625, + "router_z_loss_mlp": 1.06738281, + "step": 66, + "time_per_iteration": 2.5734033584594727 + }, + { + "auxiliary_loss_clip": 0.01435078, + "auxiliary_loss_mlp": 0.01038088, + "balance_loss_clip": 1.26706386, + "balance_loss_mlp": 1.00547194, + "epoch": 0.0019441703905751263, + "flos": 50106087544320.0, + "grad_norm": 0.8987414874430923, + "language_loss": 0.58196008, + "learning_rate": 2.4230384622998466e-06, + "loss": 0.60669172, + "num_input_tokens_seen": 2236095, + "router_z_loss_clip": 1.6796875, + "router_z_loss_mlp": 0.32617188, + "step": 67, + "time_per_iteration": 2.944135904312134 + }, + { + "auxiliary_loss_clip": 0.01425942, + "auxiliary_loss_mlp": 0.01034535, + "balance_loss_clip": 1.25956511, + "balance_loss_mlp": 1.00325441, + "epoch": 0.0019731878590911727, + "flos": 59547056492160.0, + "grad_norm": 0.8281894021276948, + "language_loss": 0.57064307, + "learning_rate": 2.4315759526538664e-06, + "loss": 0.59524786, + "num_input_tokens_seen": 2291030, + "router_z_loss_clip": 1.65625, + "router_z_loss_mlp": 0.3125, + "step": 68, + "time_per_iteration": 3.042628526687622 + }, + { + "auxiliary_loss_clip": 0.01804928, + "auxiliary_loss_mlp": 0.01192643, + "balance_loss_clip": 1.33677793, + "balance_loss_mlp": 1.1003269, + "epoch": 0.0020022053276072195, + "flos": 26459276313600.0, + "grad_norm": 4.663995980292618, + "language_loss": 1.13674164, + "learning_rate": 2.4399888036522294e-06, + "loss": 1.16671729, + "num_input_tokens_seen": 2307790, + "router_z_loss_clip": 4.67578125, + "router_z_loss_mlp": 0.92333984, + "step": 69, + "time_per_iteration": 2.691441535949707 + }, + { + "auxiliary_loss_clip": 0.01793254, + "auxiliary_loss_mlp": 0.01186077, + "balance_loss_clip": 1.32545638, + "balance_loss_mlp": 1.09352279, + "epoch": 0.0020312227961232663, + "flos": 29744009758080.0, + "grad_norm": 6.148443152818621, + "language_loss": 1.14527321, + "learning_rate": 2.4482806022273704e-06, + "loss": 1.17506647, + "num_input_tokens_seen": 2324120, + "router_z_loss_clip": 4.68359375, + "router_z_loss_mlp": 0.92626953, + "step": 70, + "time_per_iteration": 2.6240437030792236 + }, + { + "auxiliary_loss_clip": 0.01756445, + "auxiliary_loss_mlp": 0.01210142, + "balance_loss_clip": 1.31783843, + "balance_loss_mlp": 1.12001991, + "epoch": 0.0020602402646393127, + "flos": 21062332258560.0, + "grad_norm": 3.4437867788890197, + "language_loss": 1.167032, + "learning_rate": 2.456454782665838e-06, + "loss": 1.19669795, + "num_input_tokens_seen": 2339380, + "router_z_loss_clip": 4.38671875, + "router_z_loss_mlp": 0.90087891, + "step": 71, + "time_per_iteration": 2.6848058700561523 + }, + { + "auxiliary_loss_clip": 0.0176945, + "auxiliary_loss_mlp": 0.01200195, + "balance_loss_clip": 1.31795001, + "balance_loss_mlp": 1.10516191, + "epoch": 0.0020892577331553595, + "flos": 11903601674880.0, + "grad_norm": 2.8462632006312014, + "language_loss": 1.01242495, + "learning_rate": 2.464514635148642e-06, + "loss": 1.04212141, + "num_input_tokens_seen": 2350465, + "router_z_loss_clip": 4.51953125, + "router_z_loss_mlp": 0.94873047, + "step": 72, + "time_per_iteration": 2.521902561187744 + }, + { + "auxiliary_loss_clip": 0.01400503, + "auxiliary_loss_mlp": 0.01031983, + "balance_loss_clip": 1.23944473, + "balance_loss_mlp": 1.0022285, + "epoch": 0.0021182752016714063, + "flos": 57401084730240.0, + "grad_norm": 0.7749940844909313, + "language_loss": 0.55299997, + "learning_rate": 2.4724633137025535e-06, + "loss": 0.57732481, + "num_input_tokens_seen": 2411125, + "router_z_loss_clip": 1.609375, + "router_z_loss_mlp": 0.296875, + "step": 73, + "time_per_iteration": 3.1556458473205566 + }, + { + "auxiliary_loss_clip": 0.01787346, + "auxiliary_loss_mlp": 0.01206673, + "balance_loss_clip": 1.33101833, + "balance_loss_mlp": 1.1122117, + "epoch": 0.0021472926701874527, + "flos": 23104367303040.0, + "grad_norm": 2.5517307610360835, + "language_loss": 1.22311556, + "learning_rate": 2.4803038436104442e-06, + "loss": 1.25305581, + "num_input_tokens_seen": 2426450, + "router_z_loss_clip": 4.5703125, + "router_z_loss_mlp": 0.94580078, + "step": 74, + "time_per_iteration": 2.6265010833740234 + }, + { + "auxiliary_loss_clip": 0.01742222, + "auxiliary_loss_mlp": 0.01176881, + "balance_loss_clip": 1.31721318, + "balance_loss_mlp": 1.09419775, + "epoch": 0.0021763101387034995, + "flos": 42086295066240.0, + "grad_norm": 2.374493656861531, + "language_loss": 1.15206575, + "learning_rate": 2.4880391283242453e-06, + "loss": 1.18125677, + "num_input_tokens_seen": 2451490, + "router_z_loss_clip": 4.24804688, + "router_z_loss_mlp": 0.82714844, + "step": 75, + "time_per_iteration": 2.778013229370117 + }, + { + "auxiliary_loss_clip": 0.01389928, + "auxiliary_loss_mlp": 0.01031538, + "balance_loss_clip": 1.23165488, + "balance_loss_mlp": 1.00273705, + "epoch": 0.0022053276072195463, + "flos": 63309785627520.0, + "grad_norm": 0.7879971303960958, + "language_loss": 0.58480287, + "learning_rate": 2.495671955920055e-06, + "loss": 0.60901749, + "num_input_tokens_seen": 2508445, + "router_z_loss_clip": 1.578125, + "router_z_loss_mlp": 0.28710938, + "step": 76, + "time_per_iteration": 2.9487199783325195 + }, + { + "auxiliary_loss_clip": 0.01747903, + "auxiliary_loss_mlp": 0.01179898, + "balance_loss_clip": 1.30610216, + "balance_loss_mlp": 1.09111071, + "epoch": 0.0022343450757355927, + "flos": 38793741477120.0, + "grad_norm": 2.7254843658122745, + "language_loss": 1.10130215, + "learning_rate": 2.5032050051312963e-06, + "loss": 1.13058019, + "num_input_tokens_seen": 2528630, + "router_z_loss_clip": 4.41796875, + "router_z_loss_mlp": 0.88720703, + "step": 77, + "time_per_iteration": 2.7070469856262207 + }, + { + "auxiliary_loss_clip": 0.01707739, + "auxiliary_loss_mlp": 0.01186393, + "balance_loss_clip": 1.299263, + "balance_loss_mlp": 1.10337567, + "epoch": 0.0022633625442516395, + "flos": 30985032412800.0, + "grad_norm": 5.42458442104478, + "language_loss": 1.32932758, + "learning_rate": 2.5106408509926183e-06, + "loss": 1.35826898, + "num_input_tokens_seen": 2542975, + "router_z_loss_clip": 4.08789062, + "router_z_loss_mlp": 0.83056641, + "step": 78, + "time_per_iteration": 2.674792528152466 + }, + { + "auxiliary_loss_clip": 0.01741211, + "auxiliary_loss_mlp": 0.01187923, + "balance_loss_clip": 1.31202543, + "balance_loss_mlp": 1.10438073, + "epoch": 0.0022923800127676863, + "flos": 11247094869120.0, + "grad_norm": 3.023748525943825, + "language_loss": 1.097422, + "learning_rate": 2.517981970124274e-06, + "loss": 1.12671328, + "num_input_tokens_seen": 2554405, + "router_z_loss_clip": 4.29101562, + "router_z_loss_mlp": 0.83544922, + "step": 79, + "time_per_iteration": 2.5389063358306885 + }, + { + "auxiliary_loss_clip": 0.01740442, + "auxiliary_loss_mlp": 0.01180781, + "balance_loss_clip": 1.30552578, + "balance_loss_mlp": 1.09723949, + "epoch": 0.0023213974812837327, + "flos": 65900552376960.0, + "grad_norm": 2.8384146269502137, + "language_loss": 1.08824766, + "learning_rate": 2.525230745684122e-06, + "loss": 1.11745989, + "num_input_tokens_seen": 2574415, + "router_z_loss_clip": 4.34960938, + "router_z_loss_mlp": 0.83544922, + "step": 80, + "time_per_iteration": 2.9225046634674072 + }, + { + "auxiliary_loss_clip": 0.01369651, + "auxiliary_loss_mlp": 0.01028379, + "balance_loss_clip": 1.21840358, + "balance_loss_mlp": 0.99938738, + "epoch": 0.0023504149497997795, + "flos": 59551245855360.0, + "grad_norm": 0.8278200550272514, + "language_loss": 0.59865916, + "learning_rate": 2.53238947201203e-06, + "loss": 0.62263942, + "num_input_tokens_seen": 2634065, + "router_z_loss_clip": 1.515625, + "router_z_loss_mlp": 0.2890625, + "step": 81, + "time_per_iteration": 2.9561257362365723 + }, + { + "auxiliary_loss_clip": 0.01722611, + "auxiliary_loss_mlp": 0.01183744, + "balance_loss_clip": 1.29663312, + "balance_loss_mlp": 1.1032536, + "epoch": 0.0023794324183158263, + "flos": 24235029550080.0, + "grad_norm": 4.084013584800417, + "language_loss": 1.14591873, + "learning_rate": 2.5394603589893167e-06, + "loss": 1.17498231, + "num_input_tokens_seen": 2649080, + "router_z_loss_clip": 4.25976562, + "router_z_loss_mlp": 0.8046875, + "step": 82, + "time_per_iteration": 2.5752158164978027 + }, + { + "auxiliary_loss_clip": 0.01677572, + "auxiliary_loss_mlp": 0.01157417, + "balance_loss_clip": 1.28072691, + "balance_loss_mlp": 1.0787394, + "epoch": 0.0024084498868318727, + "flos": 16610487240960.0, + "grad_norm": 2.6397974315588404, + "language_loss": 1.00337148, + "learning_rate": 2.5464455361339734e-06, + "loss": 1.03172135, + "num_input_tokens_seen": 2663825, + "router_z_loss_clip": 3.96679688, + "router_z_loss_mlp": 0.78710938, + "step": 83, + "time_per_iteration": 9.617888927459717 + }, + { + "auxiliary_loss_clip": 0.01675984, + "auxiliary_loss_mlp": 0.01160869, + "balance_loss_clip": 1.28660619, + "balance_loss_mlp": 1.08562446, + "epoch": 0.0024374673553479195, + "flos": 21571051812480.0, + "grad_norm": 2.7623054047617583, + "language_loss": 1.05187631, + "learning_rate": 2.553347056450635e-06, + "loss": 1.08024478, + "num_input_tokens_seen": 2678215, + "router_z_loss_clip": 3.88867188, + "router_z_loss_mlp": 0.75292969, + "step": 84, + "time_per_iteration": 4.941425561904907 + }, + { + "auxiliary_loss_clip": 0.01358145, + "auxiliary_loss_mlp": 0.01029884, + "balance_loss_clip": 1.21039557, + "balance_loss_mlp": 1.00051033, + "epoch": 0.0024664848238639663, + "flos": 74773062835200.0, + "grad_norm": 0.782072667101282, + "language_loss": 0.5660845, + "learning_rate": 2.5601669000527336e-06, + "loss": 0.58996475, + "num_input_tokens_seen": 2744060, + "router_z_loss_clip": 1.484375, + "router_z_loss_mlp": 0.29296875, + "step": 85, + "time_per_iteration": 3.0932705402374268 + }, + { + "auxiliary_loss_clip": 0.0135273, + "auxiliary_loss_mlp": 0.01031191, + "balance_loss_clip": 1.20768309, + "balance_loss_mlp": 1.00238979, + "epoch": 0.0024955022923800127, + "flos": 60211837290240.0, + "grad_norm": 0.7705792263236412, + "language_loss": 0.58125007, + "learning_rate": 2.5669069775728125e-06, + "loss": 0.60508925, + "num_input_tokens_seen": 2807220, + "router_z_loss_clip": 1.453125, + "router_z_loss_mlp": 0.28710938, + "step": 86, + "time_per_iteration": 3.0483219623565674 + }, + { + "auxiliary_loss_clip": 0.01686733, + "auxiliary_loss_mlp": 0.01181734, + "balance_loss_clip": 1.28293061, + "balance_loss_mlp": 1.10324705, + "epoch": 0.0025245197608960595, + "flos": 13472947555200.0, + "grad_norm": 3.948355482584935, + "language_loss": 1.22705054, + "learning_rate": 2.5735691333756985e-06, + "loss": 1.25573528, + "num_input_tokens_seen": 2818480, + "router_z_loss_clip": 4.04492188, + "router_z_loss_mlp": 0.78515625, + "step": 87, + "time_per_iteration": 2.5939009189605713 + }, + { + "auxiliary_loss_clip": 0.01698763, + "auxiliary_loss_mlp": 0.01163476, + "balance_loss_clip": 1.28322959, + "balance_loss_mlp": 1.08527493, + "epoch": 0.002553537229412106, + "flos": 16281011940480.0, + "grad_norm": 4.343267865267095, + "language_loss": 1.12276697, + "learning_rate": 2.580155148588048e-06, + "loss": 1.15138936, + "num_input_tokens_seen": 2830820, + "router_z_loss_clip": 4.15625, + "router_z_loss_mlp": 0.78173828, + "step": 88, + "time_per_iteration": 2.5774264335632324 + }, + { + "auxiliary_loss_clip": 0.0170423, + "auxiliary_loss_mlp": 0.01178615, + "balance_loss_clip": 1.28863525, + "balance_loss_mlp": 1.09774315, + "epoch": 0.0025825546979281527, + "flos": 16610312684160.0, + "grad_norm": 2.7327668389379927, + "language_loss": 1.13667035, + "learning_rate": 2.5866667439567312e-06, + "loss": 1.16549873, + "num_input_tokens_seen": 2843205, + "router_z_loss_clip": 4.1640625, + "router_z_loss_mlp": 0.80957031, + "step": 89, + "time_per_iteration": 2.571519374847412 + }, + { + "auxiliary_loss_clip": 0.01639275, + "auxiliary_loss_mlp": 0.01151776, + "balance_loss_clip": 1.27095556, + "balance_loss_mlp": 1.0799644, + "epoch": 0.0026115721664441995, + "flos": 12378036407040.0, + "grad_norm": 3.2583608610028025, + "language_loss": 0.90376282, + "learning_rate": 2.5931055825475097e-06, + "loss": 0.93167329, + "num_input_tokens_seen": 2857335, + "router_z_loss_clip": 3.68359375, + "router_z_loss_mlp": 0.71777344, + "step": 90, + "time_per_iteration": 2.755380868911743 + }, + { + "auxiliary_loss_clip": 0.01333536, + "auxiliary_loss_mlp": 0.01028079, + "balance_loss_clip": 1.19990277, + "balance_loss_mlp": 1.0025202, + "epoch": 0.002640589634960246, + "flos": 63900585521280.0, + "grad_norm": 0.8225370599624247, + "language_loss": 0.64415109, + "learning_rate": 2.599473272294611e-06, + "loss": 0.66776729, + "num_input_tokens_seen": 2921595, + "router_z_loss_clip": 1.3359375, + "router_z_loss_mlp": 0.25585938, + "step": 91, + "time_per_iteration": 3.0907649993896484 + }, + { + "auxiliary_loss_clip": 0.01663148, + "auxiliary_loss_mlp": 0.01158348, + "balance_loss_clip": 1.27293909, + "balance_loss_mlp": 1.08157766, + "epoch": 0.0026696071034762927, + "flos": 31899302853120.0, + "grad_norm": 3.4887266036146016, + "language_loss": 1.15547979, + "learning_rate": 2.605771368410974e-06, + "loss": 1.18369472, + "num_input_tokens_seen": 2937400, + "router_z_loss_clip": 3.90234375, + "router_z_loss_mlp": 0.76806641, + "step": 92, + "time_per_iteration": 2.826685905456543 + }, + { + "auxiliary_loss_clip": 0.01323928, + "auxiliary_loss_mlp": 0.010264, + "balance_loss_clip": 1.19229805, + "balance_loss_mlp": 1.00026906, + "epoch": 0.0026986245719923395, + "flos": 74770933242240.0, + "grad_norm": 0.849889654618098, + "language_loss": 0.5802995, + "learning_rate": 2.6120013756682003e-06, + "loss": 0.6038028, + "num_input_tokens_seen": 3002790, + "router_z_loss_clip": 1.3125, + "router_z_loss_mlp": 0.26171875, + "step": 93, + "time_per_iteration": 3.144468307495117 + }, + { + "auxiliary_loss_clip": 0.01614788, + "auxiliary_loss_mlp": 0.01177805, + "balance_loss_clip": 1.25959516, + "balance_loss_mlp": 1.10275126, + "epoch": 0.002727642040508386, + "flos": 32303736576000.0, + "grad_norm": 7.75481051969427, + "language_loss": 1.18147552, + "learning_rate": 2.618164750554579e-06, + "loss": 1.20940149, + "num_input_tokens_seen": 3020540, + "router_z_loss_clip": 3.5546875, + "router_z_loss_mlp": 0.75097656, + "step": 94, + "time_per_iteration": 2.7028446197509766 + }, + { + "auxiliary_loss_clip": 0.01631409, + "auxiliary_loss_mlp": 0.01148771, + "balance_loss_clip": 1.25618196, + "balance_loss_mlp": 1.0775795, + "epoch": 0.0027566595090244327, + "flos": 26568868671360.0, + "grad_norm": 4.072890866742212, + "language_loss": 1.25180888, + "learning_rate": 2.624262903318922e-06, + "loss": 1.27961063, + "num_input_tokens_seen": 3033525, + "router_z_loss_clip": 3.75390625, + "router_z_loss_mlp": 0.71240234, + "step": 95, + "time_per_iteration": 2.6202268600463867 + }, + { + "auxiliary_loss_clip": 0.01313055, + "auxiliary_loss_mlp": 0.01026175, + "balance_loss_clip": 1.1863296, + "balance_loss_mlp": 1.00061679, + "epoch": 0.0027856769775404795, + "flos": 74780882979840.0, + "grad_norm": 0.7210637871698568, + "language_loss": 0.58431375, + "learning_rate": 2.6302971999073867e-06, + "loss": 0.60770601, + "num_input_tokens_seen": 3107020, + "router_z_loss_clip": 1.265625, + "router_z_loss_mlp": 0.25585938, + "step": 96, + "time_per_iteration": 3.2620537281036377 + }, + { + "auxiliary_loss_clip": 0.01660424, + "auxiliary_loss_mlp": 0.0117631, + "balance_loss_clip": 1.27056217, + "balance_loss_mlp": 1.10640574, + "epoch": 0.002814694446056526, + "flos": 46235757864960.0, + "grad_norm": 2.7161985900401953, + "language_loss": 1.12611246, + "learning_rate": 2.636268963799937e-06, + "loss": 1.15447998, + "num_input_tokens_seen": 3125590, + "router_z_loss_clip": 3.90234375, + "router_z_loss_mlp": 0.69921875, + "step": 97, + "time_per_iteration": 2.7948203086853027 + }, + { + "auxiliary_loss_clip": 0.01621453, + "auxiliary_loss_mlp": 0.01132644, + "balance_loss_clip": 1.25435114, + "balance_loss_mlp": 1.06073689, + "epoch": 0.0028437119145725727, + "flos": 11282322297600.0, + "grad_norm": 3.5387290925837083, + "language_loss": 1.14176488, + "learning_rate": 2.642179477752627e-06, + "loss": 1.1693058, + "num_input_tokens_seen": 3135965, + "router_z_loss_clip": 3.671875, + "router_z_loss_mlp": 0.71875, + "step": 98, + "time_per_iteration": 2.5969624519348145 + }, + { + "auxiliary_loss_clip": 0.01616883, + "auxiliary_loss_mlp": 0.01164535, + "balance_loss_clip": 1.25497043, + "balance_loss_mlp": 1.09506011, + "epoch": 0.0028727293830886195, + "flos": 12524322470400.0, + "grad_norm": 3.2356306451421415, + "language_loss": 1.19152105, + "learning_rate": 2.6480299854514357e-06, + "loss": 1.2193352, + "num_input_tokens_seen": 3147755, + "router_z_loss_clip": 3.62304688, + "router_z_loss_mlp": 0.69433594, + "step": 99, + "time_per_iteration": 2.5407729148864746 + }, + { + "auxiliary_loss_clip": 0.01590817, + "auxiliary_loss_mlp": 0.01141677, + "balance_loss_clip": 1.24869227, + "balance_loss_mlp": 1.07453847, + "epoch": 0.002901746851604666, + "flos": 33940255645440.0, + "grad_norm": 3.5226074046102007, + "language_loss": 1.02671933, + "learning_rate": 2.65382169308299e-06, + "loss": 1.05404425, + "num_input_tokens_seen": 3166200, + "router_z_loss_clip": 3.421875, + "router_z_loss_mlp": 0.67138672, + "step": 100, + "time_per_iteration": 2.777618408203125 + }, + { + "auxiliary_loss_clip": 0.01631133, + "auxiliary_loss_mlp": 0.01139693, + "balance_loss_clip": 1.2577666, + "balance_loss_mlp": 1.07131481, + "epoch": 0.0029307643201207127, + "flos": 27263291080320.0, + "grad_norm": 4.180782258410829, + "language_loss": 1.12216783, + "learning_rate": 2.659555770827138e-06, + "loss": 1.14987612, + "num_input_tokens_seen": 3177955, + "router_z_loss_clip": 3.73828125, + "router_z_loss_mlp": 0.68408203, + "step": 101, + "time_per_iteration": 2.6760268211364746 + }, + { + "auxiliary_loss_clip": 0.01303684, + "auxiliary_loss_mlp": 0.01030062, + "balance_loss_clip": 1.1807797, + "balance_loss_mlp": 1.00583887, + "epoch": 0.0029597817886367595, + "flos": 63942236640000.0, + "grad_norm": 0.7907335489596249, + "language_loss": 0.57728267, + "learning_rate": 2.6652333542759976e-06, + "loss": 0.60062015, + "num_input_tokens_seen": 3238025, + "router_z_loss_clip": 1.234375, + "router_z_loss_mlp": 0.2421875, + "step": 102, + "time_per_iteration": 3.10502552986145 + }, + { + "auxiliary_loss_clip": 0.01624499, + "auxiliary_loss_mlp": 0.01151741, + "balance_loss_clip": 1.26301467, + "balance_loss_mlp": 1.07964325, + "epoch": 0.002988799257152806, + "flos": 25000046461440.0, + "grad_norm": 2.6493497247786606, + "language_loss": 1.02380311, + "learning_rate": 2.6708555457837733e-06, + "loss": 1.05156553, + "num_input_tokens_seen": 3255000, + "router_z_loss_clip": 3.609375, + "router_z_loss_mlp": 0.72070312, + "step": 103, + "time_per_iteration": 2.696643114089966 + }, + { + "auxiliary_loss_clip": 0.01297821, + "auxiliary_loss_mlp": 0.01027877, + "balance_loss_clip": 1.17637229, + "balance_loss_mlp": 1.00327206, + "epoch": 0.0030178167256688527, + "flos": 69266107486080.0, + "grad_norm": 0.7059371929460276, + "language_loss": 0.58483589, + "learning_rate": 2.676423415751363e-06, + "loss": 0.60809278, + "num_input_tokens_seen": 3319875, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.24609375, + "step": 104, + "time_per_iteration": 3.1604251861572266 + }, + { + "auxiliary_loss_clip": 0.01624808, + "auxiliary_loss_mlp": 0.0122504, + "balance_loss_clip": 1.2590481, + "balance_loss_mlp": 1.15146375, + "epoch": 0.0030468341941848995, + "flos": 24746332544640.0, + "grad_norm": 3.2441937315148373, + "language_loss": 1.10279441, + "learning_rate": 2.681938003849502e-06, + "loss": 1.13129282, + "num_input_tokens_seen": 3336710, + "router_z_loss_clip": 3.65234375, + "router_z_loss_mlp": 0.73583984, + "step": 105, + "time_per_iteration": 2.6566872596740723 + }, + { + "auxiliary_loss_clip": 0.01598774, + "auxiliary_loss_mlp": 0.01154041, + "balance_loss_clip": 1.24602222, + "balance_loss_mlp": 1.08244395, + "epoch": 0.003075851662700946, + "flos": 24782362934400.0, + "grad_norm": 3.038030153130874, + "language_loss": 1.13466096, + "learning_rate": 2.6874003201839304e-06, + "loss": 1.16218925, + "num_input_tokens_seen": 3350890, + "router_z_loss_clip": 3.52734375, + "router_z_loss_mlp": 0.71606445, + "step": 106, + "time_per_iteration": 2.5922799110412598 + }, + { + "auxiliary_loss_clip": 0.01592673, + "auxiliary_loss_mlp": 0.01148901, + "balance_loss_clip": 1.24588442, + "balance_loss_mlp": 1.08462334, + "epoch": 0.0031048691312169927, + "flos": 17304036865920.0, + "grad_norm": 4.128309674533783, + "language_loss": 1.14610958, + "learning_rate": 2.692811346405858e-06, + "loss": 1.17352533, + "num_input_tokens_seen": 3361720, + "router_z_loss_clip": 3.46679688, + "router_z_loss_mlp": 0.64208984, + "step": 107, + "time_per_iteration": 2.537651538848877 + }, + { + "auxiliary_loss_clip": 0.01598519, + "auxiliary_loss_mlp": 0.01153126, + "balance_loss_clip": 1.24422002, + "balance_loss_mlp": 1.0867976, + "epoch": 0.0031338865997330395, + "flos": 67250608807680.0, + "grad_norm": 4.4140115369686574, + "language_loss": 1.21053362, + "learning_rate": 2.698172036770774e-06, + "loss": 1.23804998, + "num_input_tokens_seen": 3383520, + "router_z_loss_clip": 3.546875, + "router_z_loss_mlp": 0.66357422, + "step": 108, + "time_per_iteration": 2.905881643295288 + }, + { + "auxiliary_loss_clip": 0.01593855, + "auxiliary_loss_mlp": 0.01166224, + "balance_loss_clip": 1.23564315, + "balance_loss_mlp": 1.09593809, + "epoch": 0.003162904068249086, + "flos": 22375241136000.0, + "grad_norm": 3.2414619530826285, + "language_loss": 1.17729616, + "learning_rate": 2.703483319148466e-06, + "loss": 1.20489693, + "num_input_tokens_seen": 3397615, + "router_z_loss_clip": 3.58398438, + "router_z_loss_mlp": 0.703125, + "step": 109, + "time_per_iteration": 2.5988926887512207 + }, + { + "auxiliary_loss_clip": 0.01296484, + "auxiliary_loss_mlp": 0.01032796, + "balance_loss_clip": 1.17483759, + "balance_loss_mlp": 1.0083822, + "epoch": 0.0031919215367651327, + "flos": 72834405926400.0, + "grad_norm": 0.7246544271780021, + "language_loss": 0.56329417, + "learning_rate": 2.708746095986916e-06, + "loss": 0.58658695, + "num_input_tokens_seen": 3461295, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.24414062, + "step": 110, + "time_per_iteration": 3.1555874347686768 + }, + { + "auxiliary_loss_clip": 0.01578414, + "auxiliary_loss_mlp": 0.01155151, + "balance_loss_clip": 1.23722005, + "balance_loss_mlp": 1.08901334, + "epoch": 0.003220939005281179, + "flos": 29051437651200.0, + "grad_norm": 2.8383803607837756, + "language_loss": 1.1608938, + "learning_rate": 2.7139612452325754e-06, + "loss": 1.18822944, + "num_input_tokens_seen": 3477540, + "router_z_loss_clip": 3.41210938, + "router_z_loss_mlp": 0.66162109, + "step": 111, + "time_per_iteration": 2.636653423309326 + }, + { + "auxiliary_loss_clip": 0.01291471, + "auxiliary_loss_mlp": 0.0102882, + "balance_loss_clip": 1.17181969, + "balance_loss_mlp": 1.00545514, + "epoch": 0.003249956473797226, + "flos": 63502296197760.0, + "grad_norm": 0.7498701706654403, + "language_loss": 0.54599881, + "learning_rate": 2.7191296212093786e-06, + "loss": 0.56920171, + "num_input_tokens_seen": 3542620, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.23339844, + "step": 112, + "time_per_iteration": 3.116804599761963 + }, + { + "auxiliary_loss_clip": 0.01590166, + "auxiliary_loss_mlp": 0.01163303, + "balance_loss_clip": 1.23860884, + "balance_loss_mlp": 1.09225464, + "epoch": 0.0032789739423132727, + "flos": 23506706344320.0, + "grad_norm": 2.4628020377236135, + "language_loss": 0.76663458, + "learning_rate": 2.724252055458679e-06, + "loss": 0.79416931, + "num_input_tokens_seen": 3559185, + "router_z_loss_clip": 3.515625, + "router_z_loss_mlp": 0.71044922, + "step": 113, + "time_per_iteration": 2.622398853302002 + }, + { + "auxiliary_loss_clip": 0.01572638, + "auxiliary_loss_mlp": 0.01134853, + "balance_loss_clip": 1.23553109, + "balance_loss_mlp": 1.0716244, + "epoch": 0.003307991410829319, + "flos": 25513269580800.0, + "grad_norm": 2.7919178209609177, + "language_loss": 0.9609586, + "learning_rate": 2.7293293575421866e-06, + "loss": 0.98803347, + "num_input_tokens_seen": 3576220, + "router_z_loss_clip": 3.37304688, + "router_z_loss_mlp": 0.63232422, + "step": 114, + "time_per_iteration": 2.72886323928833 + }, + { + "auxiliary_loss_clip": 0.01569669, + "auxiliary_loss_mlp": 0.01157755, + "balance_loss_clip": 1.22411942, + "balance_loss_mlp": 1.08837509, + "epoch": 0.003337008879345366, + "flos": 16136296888320.0, + "grad_norm": 3.2569295589807434, + "language_loss": 1.13197494, + "learning_rate": 2.7343623158098412e-06, + "loss": 1.15924919, + "num_input_tokens_seen": 3590170, + "router_z_loss_clip": 3.45703125, + "router_z_loss_mlp": 0.69433594, + "step": 115, + "time_per_iteration": 2.5353851318359375 + }, + { + "auxiliary_loss_clip": 0.01558899, + "auxiliary_loss_mlp": 0.01165954, + "balance_loss_clip": 1.23684525, + "balance_loss_mlp": 1.09862423, + "epoch": 0.0033660263478614127, + "flos": 74731623050880.0, + "grad_norm": 2.631416645000653, + "language_loss": 0.80053437, + "learning_rate": 2.7393516981344427e-06, + "loss": 0.82778287, + "num_input_tokens_seen": 3619655, + "router_z_loss_clip": 3.22070312, + "router_z_loss_mlp": 0.67333984, + "step": 116, + "time_per_iteration": 3.012768030166626 + }, + { + "auxiliary_loss_clip": 0.01571465, + "auxiliary_loss_mlp": 0.01149946, + "balance_loss_clip": 1.22903442, + "balance_loss_mlp": 1.08213997, + "epoch": 0.003395043816377459, + "flos": 27701974713600.0, + "grad_norm": 3.5928768734154586, + "language_loss": 1.15721488, + "learning_rate": 2.7442982526147504e-06, + "loss": 1.18442905, + "num_input_tokens_seen": 3636430, + "router_z_loss_clip": 3.42382812, + "router_z_loss_mlp": 0.67773438, + "step": 117, + "time_per_iteration": 2.7391345500946045 + }, + { + "auxiliary_loss_clip": 0.01535184, + "auxiliary_loss_mlp": 0.01138411, + "balance_loss_clip": 1.22173524, + "balance_loss_mlp": 1.07713687, + "epoch": 0.003424061284893506, + "flos": 33826927772160.0, + "grad_norm": 3.420402234440519, + "language_loss": 1.41055489, + "learning_rate": 2.7492027082486626e-06, + "loss": 1.43729079, + "num_input_tokens_seen": 3649100, + "router_z_loss_clip": 3.13671875, + "router_z_loss_mlp": 0.61279297, + "step": 118, + "time_per_iteration": 2.6823954582214355 + }, + { + "auxiliary_loss_clip": 0.01269209, + "auxiliary_loss_mlp": 0.01053991, + "balance_loss_clip": 1.15220892, + "balance_loss_mlp": 1.03081715, + "epoch": 0.0034530787534095527, + "flos": 74778334450560.0, + "grad_norm": 0.7814465714796313, + "language_loss": 0.57115936, + "learning_rate": 2.7540657755779904e-06, + "loss": 0.59439135, + "num_input_tokens_seen": 3718090, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.23144531, + "step": 119, + "time_per_iteration": 3.266496181488037 + }, + { + "auxiliary_loss_clip": 0.0156395, + "auxiliary_loss_mlp": 0.01136067, + "balance_loss_clip": 1.22684193, + "balance_loss_mlp": 1.07126498, + "epoch": 0.003482096221925599, + "flos": 27341985018240.0, + "grad_norm": 2.3134531923282964, + "language_loss": 1.02546346, + "learning_rate": 2.758888147306254e-06, + "loss": 1.05246377, + "num_input_tokens_seen": 3743795, + "router_z_loss_clip": 3.375, + "router_z_loss_mlp": 0.64746094, + "step": 120, + "time_per_iteration": 2.8294639587402344 + }, + { + "auxiliary_loss_clip": 0.01564386, + "auxiliary_loss_mlp": 0.01119279, + "balance_loss_clip": 1.22267127, + "balance_loss_mlp": 1.0571475, + "epoch": 0.003511113690441646, + "flos": 19529400792960.0, + "grad_norm": 2.511066099399071, + "language_loss": 1.07938957, + "learning_rate": 2.7636704988908417e-06, + "loss": 1.10622621, + "num_input_tokens_seen": 3758265, + "router_z_loss_clip": 3.4140625, + "router_z_loss_mlp": 0.62109375, + "step": 121, + "time_per_iteration": 2.5807509422302246 + }, + { + "auxiliary_loss_clip": 0.01547347, + "auxiliary_loss_mlp": 0.01156188, + "balance_loss_clip": 1.22184956, + "balance_loss_mlp": 1.08981228, + "epoch": 0.0035401311589576927, + "flos": 11904090433920.0, + "grad_norm": 2.7896436961774502, + "language_loss": 0.98968017, + "learning_rate": 2.7684134891108e-06, + "loss": 1.01671541, + "num_input_tokens_seen": 3770935, + "router_z_loss_clip": 3.2578125, + "router_z_loss_mlp": 0.66308594, + "step": 122, + "time_per_iteration": 2.5515313148498535 + }, + { + "auxiliary_loss_clip": 0.01538378, + "auxiliary_loss_mlp": 0.01135112, + "balance_loss_clip": 1.2122972, + "balance_loss_mlp": 1.07340884, + "epoch": 0.003569148627473739, + "flos": 23689511556480.0, + "grad_norm": 3.001650433366563, + "language_loss": 0.95656574, + "learning_rate": 2.7731177606114483e-06, + "loss": 0.98330069, + "num_input_tokens_seen": 3786425, + "router_z_loss_clip": 3.25585938, + "router_z_loss_mlp": 0.6171875, + "step": 123, + "time_per_iteration": 2.682016134262085 + }, + { + "auxiliary_loss_clip": 0.0151775, + "auxiliary_loss_mlp": 0.0114034, + "balance_loss_clip": 1.2098856, + "balance_loss_mlp": 1.07858944, + "epoch": 0.003598166095989786, + "flos": 25595000807040.0, + "grad_norm": 2.4383084112801017, + "language_loss": 0.91250312, + "learning_rate": 2.777783940426944e-06, + "loss": 0.93908405, + "num_input_tokens_seen": 3805695, + "router_z_loss_clip": 3.08007812, + "router_z_loss_mlp": 0.61767578, + "step": 124, + "time_per_iteration": 2.6220271587371826 + }, + { + "auxiliary_loss_clip": 0.0127741, + "auxiliary_loss_mlp": 0.0102704, + "balance_loss_clip": 1.15736723, + "balance_loss_mlp": 1.00415218, + "epoch": 0.0036271835645058327, + "flos": 71671204425600.0, + "grad_norm": 0.8108185291115738, + "language_loss": 0.53386706, + "learning_rate": 2.782412640481857e-06, + "loss": 0.55691159, + "num_input_tokens_seen": 3870130, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.22851562, + "step": 125, + "time_per_iteration": 3.1531450748443604 + }, + { + "auxiliary_loss_clip": 0.01520812, + "auxiliary_loss_mlp": 0.01146815, + "balance_loss_clip": 1.21195769, + "balance_loss_mlp": 1.0858748, + "epoch": 0.003656201033021879, + "flos": 15479650437120.0, + "grad_norm": 3.3638089927984005, + "language_loss": 1.00002205, + "learning_rate": 2.787004458072766e-06, + "loss": 1.02669835, + "num_input_tokens_seen": 3883485, + "router_z_loss_clip": 3.08984375, + "router_z_loss_mlp": 0.60986328, + "step": 126, + "time_per_iteration": 2.601975917816162 + }, + { + "auxiliary_loss_clip": 0.01272525, + "auxiliary_loss_mlp": 0.01026561, + "balance_loss_clip": 1.15425217, + "balance_loss_mlp": 1.00338697, + "epoch": 0.003685218501537926, + "flos": 61888261910400.0, + "grad_norm": 0.837374759035491, + "language_loss": 0.57683188, + "learning_rate": 2.7915599763308157e-06, + "loss": 0.59982276, + "num_input_tokens_seen": 3943475, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.23144531, + "step": 127, + "time_per_iteration": 3.098071575164795 + }, + { + "auxiliary_loss_clip": 0.01528622, + "auxiliary_loss_mlp": 0.01128539, + "balance_loss_clip": 1.20966053, + "balance_loss_mlp": 1.06697893, + "epoch": 0.0037142359700539727, + "flos": 10661392033920.0, + "grad_norm": 4.131937679777086, + "language_loss": 1.23390079, + "learning_rate": 2.7960797646661305e-06, + "loss": 1.26047242, + "num_input_tokens_seen": 3953925, + "router_z_loss_clip": 3.1875, + "router_z_loss_mlp": 0.61572266, + "step": 128, + "time_per_iteration": 2.5777745246887207 + }, + { + "auxiliary_loss_clip": 0.01266218, + "auxiliary_loss_mlp": 0.01022186, + "balance_loss_clip": 1.15267956, + "balance_loss_mlp": 1.00034666, + "epoch": 0.003743253438570019, + "flos": 68939285448960.0, + "grad_norm": 0.7334219679139843, + "language_loss": 0.58947337, + "learning_rate": 2.8005643791949446e-06, + "loss": 0.61235738, + "num_input_tokens_seen": 4021235, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.21875, + "step": 129, + "time_per_iteration": 3.1588480472564697 + }, + { + "auxiliary_loss_clip": 0.01512834, + "auxiliary_loss_mlp": 0.01128577, + "balance_loss_clip": 1.20232868, + "balance_loss_mlp": 1.06935418, + "epoch": 0.003772270907086066, + "flos": 22155846952320.0, + "grad_norm": 3.2080605213989966, + "language_loss": 1.08831882, + "learning_rate": 2.80501436315023e-06, + "loss": 1.11473286, + "num_input_tokens_seen": 4034220, + "router_z_loss_clip": 3.10742188, + "router_z_loss_mlp": 0.59228516, + "step": 130, + "time_per_iteration": 2.61716890335083 + }, + { + "auxiliary_loss_clip": 0.01500069, + "auxiliary_loss_mlp": 0.0110008, + "balance_loss_clip": 1.20318604, + "balance_loss_mlp": 1.04042792, + "epoch": 0.0038012883756021127, + "flos": 11940644494080.0, + "grad_norm": 3.1899594329123993, + "language_loss": 0.91764718, + "learning_rate": 2.8094302472765976e-06, + "loss": 0.9436487, + "num_input_tokens_seen": 4046425, + "router_z_loss_clip": 2.97265625, + "router_z_loss_mlp": 0.59619141, + "step": 131, + "time_per_iteration": 2.5269131660461426 + }, + { + "auxiliary_loss_clip": 0.01259347, + "auxiliary_loss_mlp": 0.01024402, + "balance_loss_clip": 1.15061951, + "balance_loss_mlp": 1.00313556, + "epoch": 0.003830305844118159, + "flos": 74762379959040.0, + "grad_norm": 0.7453260548091007, + "language_loss": 0.54960978, + "learning_rate": 2.8138125502101794e-06, + "loss": 0.57244724, + "num_input_tokens_seen": 4107605, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.21289062, + "step": 132, + "time_per_iteration": 3.072739362716675 + }, + { + "auxiliary_loss_clip": 0.01509325, + "auxiliary_loss_mlp": 0.01139573, + "balance_loss_clip": 1.20688045, + "balance_loss_mlp": 1.07376969, + "epoch": 0.003859323312634206, + "flos": 30583217041920.0, + "grad_norm": 3.0894458093397414, + "language_loss": 1.19636655, + "learning_rate": 2.818161778844179e-06, + "loss": 1.22285545, + "num_input_tokens_seen": 4121685, + "router_z_loss_clip": 3.02539062, + "router_z_loss_mlp": 0.65820312, + "step": 133, + "time_per_iteration": 2.6269619464874268 + }, + { + "auxiliary_loss_clip": 0.01503497, + "auxiliary_loss_mlp": 0.01120544, + "balance_loss_clip": 1.19700611, + "balance_loss_mlp": 1.05807877, + "epoch": 0.0038883407811502527, + "flos": 16390394830080.0, + "grad_norm": 2.905250749821365, + "language_loss": 0.99226427, + "learning_rate": 2.8224784286807224e-06, + "loss": 1.01850474, + "num_input_tokens_seen": 4134235, + "router_z_loss_clip": 3.06835938, + "router_z_loss_mlp": 0.62451172, + "step": 134, + "time_per_iteration": 2.6216049194335938 + }, + { + "auxiliary_loss_clip": 0.01516019, + "auxiliary_loss_mlp": 0.01123852, + "balance_loss_clip": 1.20273948, + "balance_loss_mlp": 1.05981255, + "epoch": 0.003917358249666299, + "flos": 30765289115520.0, + "grad_norm": 4.308675205845716, + "language_loss": 1.13121057, + "learning_rate": 2.826762984169642e-06, + "loss": 1.15760922, + "num_input_tokens_seen": 4151530, + "router_z_loss_clip": 3.1328125, + "router_z_loss_mlp": 0.640625, + "step": 135, + "time_per_iteration": 2.5755298137664795 + }, + { + "auxiliary_loss_clip": 0.01490551, + "auxiliary_loss_mlp": 0.01151205, + "balance_loss_clip": 1.1956197, + "balance_loss_mlp": 1.09174347, + "epoch": 0.003946375718182345, + "flos": 11647269406080.0, + "grad_norm": 4.837054839213775, + "language_loss": 1.07395983, + "learning_rate": 2.8310159190347422e-06, + "loss": 1.10037732, + "num_input_tokens_seen": 4161255, + "router_z_loss_clip": 2.94921875, + "router_z_loss_mlp": 0.59472656, + "step": 136, + "time_per_iteration": 2.573303699493408 + }, + { + "auxiliary_loss_clip": 0.01498966, + "auxiliary_loss_mlp": 0.01111842, + "balance_loss_clip": 1.19599223, + "balance_loss_mlp": 1.05333424, + "epoch": 0.003975393186698393, + "flos": 29014185363840.0, + "grad_norm": 3.8770296390613814, + "language_loss": 0.95647979, + "learning_rate": 2.835237696588131e-06, + "loss": 0.98258781, + "num_input_tokens_seen": 4176600, + "router_z_loss_clip": 3.03125, + "router_z_loss_mlp": 0.5847168, + "step": 137, + "time_per_iteration": 2.6956679821014404 + }, + { + "auxiliary_loss_clip": 0.01503453, + "auxiliary_loss_mlp": 0.01138333, + "balance_loss_clip": 1.2006979, + "balance_loss_mlp": 1.07615304, + "epoch": 0.004004410655214439, + "flos": 20370598024320.0, + "grad_norm": 3.4570258292904574, + "language_loss": 1.06589985, + "learning_rate": 2.8394287700331053e-06, + "loss": 1.0923177, + "num_input_tokens_seen": 4189295, + "router_z_loss_clip": 3.02148438, + "router_z_loss_mlp": 0.62109375, + "step": 138, + "time_per_iteration": 2.5339200496673584 + }, + { + "auxiliary_loss_clip": 0.01498898, + "auxiliary_loss_mlp": 0.01133182, + "balance_loss_clip": 1.19553137, + "balance_loss_mlp": 1.07138419, + "epoch": 0.004033428123730485, + "flos": 33028743202560.0, + "grad_norm": 3.42279033780189, + "language_loss": 0.96190655, + "learning_rate": 2.8435895827561136e-06, + "loss": 0.98822725, + "num_input_tokens_seen": 4203510, + "router_z_loss_clip": 3.03320312, + "router_z_loss_mlp": 0.61865234, + "step": 139, + "time_per_iteration": 2.6542246341705322 + }, + { + "auxiliary_loss_clip": 0.01245739, + "auxiliary_loss_mlp": 0.01047259, + "balance_loss_clip": 1.14088392, + "balance_loss_mlp": 1.02627826, + "epoch": 0.004062445592246533, + "flos": 59582318832000.0, + "grad_norm": 0.795116399395695, + "language_loss": 0.57589757, + "learning_rate": 2.847720568608246e-06, + "loss": 0.59882748, + "num_input_tokens_seen": 4251885, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.20996094, + "step": 140, + "time_per_iteration": 2.896568775177002 + }, + { + "auxiliary_loss_clip": 0.01494405, + "auxiliary_loss_mlp": 0.0112615, + "balance_loss_clip": 1.19199526, + "balance_loss_mlp": 1.06559134, + "epoch": 0.004091463060762579, + "flos": 16537483854720.0, + "grad_norm": 3.130038659758698, + "language_loss": 1.08114624, + "learning_rate": 2.8518221521767104e-06, + "loss": 1.10735178, + "num_input_tokens_seen": 4264735, + "router_z_loss_clip": 3.02734375, + "router_z_loss_mlp": 0.60595703, + "step": 141, + "time_per_iteration": 2.5687828063964844 + }, + { + "auxiliary_loss_clip": 0.01489614, + "auxiliary_loss_mlp": 0.01119279, + "balance_loss_clip": 1.1943773, + "balance_loss_mlp": 1.05833936, + "epoch": 0.004120480529278625, + "flos": 21025952755200.0, + "grad_norm": 3.465104210283906, + "language_loss": 0.97214925, + "learning_rate": 2.855894749046714e-06, + "loss": 0.99823821, + "num_input_tokens_seen": 4279280, + "router_z_loss_clip": 2.95117188, + "router_z_loss_mlp": 0.609375, + "step": 142, + "time_per_iteration": 2.6007912158966064 + }, + { + "auxiliary_loss_clip": 0.01489647, + "auxiliary_loss_mlp": 0.01119751, + "balance_loss_clip": 1.18888783, + "balance_loss_mlp": 1.06195831, + "epoch": 0.004149497997794673, + "flos": 23652259269120.0, + "grad_norm": 2.346619817058183, + "language_loss": 1.19564295, + "learning_rate": 2.859938766054156e-06, + "loss": 1.22173691, + "num_input_tokens_seen": 4296225, + "router_z_loss_clip": 3.00976562, + "router_z_loss_mlp": 0.578125, + "step": 143, + "time_per_iteration": 2.63154673576355 + }, + { + "auxiliary_loss_clip": 0.0148878, + "auxiliary_loss_mlp": 0.01145526, + "balance_loss_clip": 1.19021821, + "balance_loss_mlp": 1.08029509, + "epoch": 0.004178515466310719, + "flos": 14021433014400.0, + "grad_norm": 2.8911028634930496, + "language_loss": 1.11926126, + "learning_rate": 2.863954601529518e-06, + "loss": 1.14560437, + "num_input_tokens_seen": 4309835, + "router_z_loss_clip": 2.984375, + "router_z_loss_mlp": 0.65234375, + "step": 144, + "time_per_iteration": 2.5161077976226807 + }, + { + "auxiliary_loss_clip": 0.01510481, + "auxiliary_loss_mlp": 0.01133406, + "balance_loss_clip": 1.19356966, + "balance_loss_mlp": 1.06793582, + "epoch": 0.004207532934826765, + "flos": 35515815747840.0, + "grad_norm": 3.041605672920611, + "language_loss": 1.02768826, + "learning_rate": 2.86794264553331e-06, + "loss": 1.0541271, + "num_input_tokens_seen": 4328955, + "router_z_loss_clip": 3.16796875, + "router_z_loss_mlp": 0.65429688, + "step": 145, + "time_per_iteration": 2.684934377670288 + }, + { + "auxiliary_loss_clip": 0.01479459, + "auxiliary_loss_mlp": 0.01140454, + "balance_loss_clip": 1.18437839, + "balance_loss_mlp": 1.07980061, + "epoch": 0.004236550403342813, + "flos": 74744121317760.0, + "grad_norm": 3.268875475277276, + "language_loss": 0.87929404, + "learning_rate": 2.8719032800834294e-06, + "loss": 0.90549314, + "num_input_tokens_seen": 4354660, + "router_z_loss_clip": 2.94921875, + "router_z_loss_mlp": 0.60693359, + "step": 146, + "time_per_iteration": 2.9971272945404053 + }, + { + "auxiliary_loss_clip": 0.01497723, + "auxiliary_loss_mlp": 0.01131804, + "balance_loss_clip": 1.18319607, + "balance_loss_mlp": 1.06910014, + "epoch": 0.004265567871858859, + "flos": 27409926257280.0, + "grad_norm": 2.7933275758279144, + "language_loss": 1.09615552, + "learning_rate": 2.875836879374759e-06, + "loss": 1.12245083, + "num_input_tokens_seen": 4371170, + "router_z_loss_clip": 3.14257812, + "router_z_loss_mlp": 0.62695312, + "step": 147, + "time_per_iteration": 2.6137611865997314 + }, + { + "auxiliary_loss_clip": 0.01467225, + "auxiliary_loss_mlp": 0.01138828, + "balance_loss_clip": 1.18129659, + "balance_loss_mlp": 1.07683921, + "epoch": 0.004294585340374905, + "flos": 22119083424000.0, + "grad_norm": 5.383607708793792, + "language_loss": 1.01417446, + "learning_rate": 2.8797438099913196e-06, + "loss": 1.04023492, + "num_input_tokens_seen": 4386390, + "router_z_loss_clip": 2.85546875, + "router_z_loss_mlp": 0.61962891, + "step": 148, + "time_per_iteration": 2.547001600265503 + }, + { + "auxiliary_loss_clip": 0.0145974, + "auxiliary_loss_mlp": 0.01114046, + "balance_loss_clip": 1.17346716, + "balance_loss_mlp": 1.05520439, + "epoch": 0.004323602808890953, + "flos": 12450620856960.0, + "grad_norm": 3.8054792678792015, + "language_loss": 1.09283793, + "learning_rate": 2.8836244311112828e-06, + "loss": 1.11857581, + "num_input_tokens_seen": 4398295, + "router_z_loss_clip": 2.86328125, + "router_z_loss_mlp": 0.58886719, + "step": 149, + "time_per_iteration": 2.5567517280578613 + }, + { + "auxiliary_loss_clip": 0.01227929, + "auxiliary_loss_mlp": 0.01024593, + "balance_loss_clip": 1.12428927, + "balance_loss_mlp": 1.00599647, + "epoch": 0.004352620277406999, + "flos": 74784653406720.0, + "grad_norm": 0.692197286756316, + "language_loss": 0.56536484, + "learning_rate": 2.887479094705121e-06, + "loss": 0.58789003, + "num_input_tokens_seen": 4468280, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.18554688, + "step": 150, + "time_per_iteration": 3.281113624572754 + }, + { + "auxiliary_loss_clip": 0.01226799, + "auxiliary_loss_mlp": 0.01022042, + "balance_loss_clip": 1.12399638, + "balance_loss_mlp": 1.00325453, + "epoch": 0.004381637745923045, + "flos": 66419080156800.0, + "grad_norm": 0.6882680257236788, + "language_loss": 0.49958476, + "learning_rate": 2.8913081457271816e-06, + "loss": 0.52207315, + "num_input_tokens_seen": 4530430, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.1875, + "step": 151, + "time_per_iteration": 3.0473270416259766 + }, + { + "auxiliary_loss_clip": 0.01491586, + "auxiliary_loss_mlp": 0.01130111, + "balance_loss_clip": 1.18189907, + "balance_loss_mlp": 1.07231808, + "epoch": 0.004410655214439093, + "flos": 32847962849280.0, + "grad_norm": 2.7244615404956667, + "language_loss": 1.04900336, + "learning_rate": 2.8951119223009308e-06, + "loss": 1.07522035, + "num_input_tokens_seen": 4549530, + "router_z_loss_clip": 3.1015625, + "router_z_loss_mlp": 0.57788086, + "step": 152, + "time_per_iteration": 2.7164175510406494 + }, + { + "auxiliary_loss_clip": 0.01221932, + "auxiliary_loss_mlp": 0.01018911, + "balance_loss_clip": 1.12069976, + "balance_loss_mlp": 0.99993253, + "epoch": 0.004439672682955139, + "flos": 54741191621760.0, + "grad_norm": 0.7259984340438033, + "language_loss": 0.56615031, + "learning_rate": 2.8988907558981293e-06, + "loss": 0.58855879, + "num_input_tokens_seen": 4605815, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.18945312, + "step": 153, + "time_per_iteration": 2.942127227783203 + }, + { + "auxiliary_loss_clip": 0.01470069, + "auxiliary_loss_mlp": 0.01114734, + "balance_loss_clip": 1.18287063, + "balance_loss_mlp": 1.0564642, + "epoch": 0.004468690151471185, + "flos": 19494173364480.0, + "grad_norm": 2.5679299376580893, + "language_loss": 0.95873404, + "learning_rate": 2.902644971512172e-06, + "loss": 0.98458213, + "num_input_tokens_seen": 4623535, + "router_z_loss_clip": 2.87109375, + "router_z_loss_mlp": 0.58251953, + "step": 154, + "time_per_iteration": 2.617633581161499 + }, + { + "auxiliary_loss_clip": 0.01445527, + "auxiliary_loss_mlp": 0.01102534, + "balance_loss_clip": 1.17462516, + "balance_loss_mlp": 1.04567146, + "epoch": 0.004497707619987233, + "flos": 66704427498240.0, + "grad_norm": 4.6220915808108956, + "language_loss": 1.05372477, + "learning_rate": 2.9063748878258113e-06, + "loss": 1.07920539, + "num_input_tokens_seen": 4644870, + "router_z_loss_clip": 2.70507812, + "router_z_loss_mlp": 0.56811523, + "step": 155, + "time_per_iteration": 2.875516891479492 + }, + { + "auxiliary_loss_clip": 0.01461914, + "auxiliary_loss_mlp": 0.01116063, + "balance_loss_clip": 1.17515492, + "balance_loss_mlp": 1.05424142, + "epoch": 0.004526725088503279, + "flos": 30619701279360.0, + "grad_norm": 3.856686560744075, + "language_loss": 1.07924414, + "learning_rate": 2.910080817373494e-06, + "loss": 1.10502398, + "num_input_tokens_seen": 4659890, + "router_z_loss_clip": 2.86523438, + "router_z_loss_mlp": 0.6184082, + "step": 156, + "time_per_iteration": 2.756685972213745 + }, + { + "auxiliary_loss_clip": 0.01469343, + "auxiliary_loss_mlp": 0.01114871, + "balance_loss_clip": 1.17743552, + "balance_loss_mlp": 1.05226278, + "epoch": 0.004555742557019325, + "flos": 36202383100800.0, + "grad_norm": 4.6587680724069305, + "language_loss": 1.2279079, + "learning_rate": 2.9137630666985104e-06, + "loss": 1.25374985, + "num_input_tokens_seen": 4673420, + "router_z_loss_clip": 2.91601562, + "router_z_loss_mlp": 0.62695312, + "step": 157, + "time_per_iteration": 2.657703161239624 + }, + { + "auxiliary_loss_clip": 0.0146187, + "auxiliary_loss_mlp": 0.01122529, + "balance_loss_clip": 1.18234634, + "balance_loss_mlp": 1.06278086, + "epoch": 0.004584760025535373, + "flos": 44848623703680.0, + "grad_norm": 3.134462284080985, + "language_loss": 1.06137705, + "learning_rate": 2.91742193650515e-06, + "loss": 1.08722103, + "num_input_tokens_seen": 4689870, + "router_z_loss_clip": 2.79492188, + "router_z_loss_mlp": 0.59814453, + "step": 158, + "time_per_iteration": 5.975171804428101 + }, + { + "auxiliary_loss_clip": 0.01471233, + "auxiliary_loss_mlp": 0.01123594, + "balance_loss_clip": 1.17753768, + "balance_loss_mlp": 1.0639174, + "epoch": 0.004613777494051419, + "flos": 32588732937600.0, + "grad_norm": 3.5539231980138384, + "language_loss": 1.00364053, + "learning_rate": 2.9210577218060625e-06, + "loss": 1.02958882, + "num_input_tokens_seen": 4706790, + "router_z_loss_clip": 2.93554688, + "router_z_loss_mlp": 0.59741211, + "step": 159, + "time_per_iteration": 8.466437101364136 + }, + { + "auxiliary_loss_clip": 0.01472096, + "auxiliary_loss_mlp": 0.01123154, + "balance_loss_clip": 1.1754508, + "balance_loss_mlp": 1.06672001, + "epoch": 0.004642794962567465, + "flos": 22562165888640.0, + "grad_norm": 2.578414122409144, + "language_loss": 1.09961486, + "learning_rate": 2.9246707120649977e-06, + "loss": 1.12556744, + "num_input_tokens_seen": 4723555, + "router_z_loss_clip": 2.96679688, + "router_z_loss_mlp": 0.56469727, + "step": 160, + "time_per_iteration": 6.252458333969116 + }, + { + "auxiliary_loss_clip": 0.01463708, + "auxiliary_loss_mlp": 0.01125125, + "balance_loss_clip": 1.18126583, + "balance_loss_mlp": 1.06928754, + "epoch": 0.004671812431083513, + "flos": 11429830258560.0, + "grad_norm": 3.417345522063279, + "language_loss": 0.9483608, + "learning_rate": 2.928261191335098e-06, + "loss": 0.97424912, + "num_input_tokens_seen": 4737145, + "router_z_loss_clip": 2.82226562, + "router_z_loss_mlp": 0.55859375, + "step": 161, + "time_per_iteration": 2.558534860610962 + }, + { + "auxiliary_loss_clip": 0.01219236, + "auxiliary_loss_mlp": 0.01060221, + "balance_loss_clip": 1.11973286, + "balance_loss_mlp": 1.04238713, + "epoch": 0.004700829899599559, + "flos": 52356764073600.0, + "grad_norm": 0.9318814071254542, + "language_loss": 0.62203479, + "learning_rate": 2.9318294383929054e-06, + "loss": 0.64482939, + "num_input_tokens_seen": 4785045, + "router_z_loss_clip": 0.9921875, + "router_z_loss_mlp": 0.17871094, + "step": 162, + "time_per_iteration": 2.7978665828704834 + }, + { + "auxiliary_loss_clip": 0.01447203, + "auxiliary_loss_mlp": 0.01134612, + "balance_loss_clip": 1.17048562, + "balance_loss_mlp": 1.07677138, + "epoch": 0.004729847368115605, + "flos": 20586012312960.0, + "grad_norm": 3.050734613649778, + "language_loss": 1.17822933, + "learning_rate": 2.935375726868257e-06, + "loss": 1.20404756, + "num_input_tokens_seen": 4800570, + "router_z_loss_clip": 2.76757812, + "router_z_loss_mlp": 0.578125, + "step": 163, + "time_per_iteration": 2.614384412765503 + }, + { + "auxiliary_loss_clip": 0.01439506, + "auxiliary_loss_mlp": 0.01111334, + "balance_loss_clip": 1.17300236, + "balance_loss_mlp": 1.05380344, + "epoch": 0.004758864836631653, + "flos": 22008304080000.0, + "grad_norm": 2.8275230792870563, + "language_loss": 1.0163672, + "learning_rate": 2.9389003253701925e-06, + "loss": 1.0418756, + "num_input_tokens_seen": 4813210, + "router_z_loss_clip": 2.6640625, + "router_z_loss_mlp": 0.57519531, + "step": 164, + "time_per_iteration": 2.56876277923584 + }, + { + "auxiliary_loss_clip": 0.01435287, + "auxiliary_loss_mlp": 0.01112117, + "balance_loss_clip": 1.16216624, + "balance_loss_mlp": 1.05720925, + "epoch": 0.004787882305147699, + "flos": 13733504098560.0, + "grad_norm": 2.6250637244904085, + "language_loss": 0.96558666, + "learning_rate": 2.9424034976090475e-06, + "loss": 0.99106067, + "num_input_tokens_seen": 4828395, + "router_z_loss_clip": 2.72851562, + "router_z_loss_mlp": 0.54882812, + "step": 165, + "time_per_iteration": 2.554903268814087 + }, + { + "auxiliary_loss_clip": 0.01432231, + "auxiliary_loss_mlp": 0.01113671, + "balance_loss_clip": 1.16667271, + "balance_loss_mlp": 1.06090868, + "epoch": 0.004816899773663745, + "flos": 23945075775360.0, + "grad_norm": 4.052085451502398, + "language_loss": 1.14292622, + "learning_rate": 2.9458855025148492e-06, + "loss": 1.16838515, + "num_input_tokens_seen": 4841540, + "router_z_loss_clip": 2.65820312, + "router_z_loss_mlp": 0.52709961, + "step": 166, + "time_per_iteration": 2.584813356399536 + }, + { + "auxiliary_loss_clip": 0.01450031, + "auxiliary_loss_mlp": 0.01121179, + "balance_loss_clip": 1.17205048, + "balance_loss_mlp": 1.05914211, + "epoch": 0.004845917242179793, + "flos": 16760927756160.0, + "grad_norm": 3.375671533389298, + "language_loss": 1.08060479, + "learning_rate": 2.9493465943521642e-06, + "loss": 1.1063168, + "num_input_tokens_seen": 4854770, + "router_z_loss_clip": 2.77734375, + "router_z_loss_mlp": 0.62011719, + "step": 167, + "time_per_iteration": 2.54451847076416 + }, + { + "auxiliary_loss_clip": 0.01462671, + "auxiliary_loss_mlp": 0.01132252, + "balance_loss_clip": 1.16546535, + "balance_loss_mlp": 1.07610464, + "epoch": 0.004874934710695839, + "flos": 32482701538560.0, + "grad_norm": 2.1635288892014217, + "language_loss": 0.83426666, + "learning_rate": 2.9527870228315107e-06, + "loss": 0.8602159, + "num_input_tokens_seen": 4876160, + "router_z_loss_clip": 2.97265625, + "router_z_loss_mlp": 0.56176758, + "step": 168, + "time_per_iteration": 2.684818744659424 + }, + { + "auxiliary_loss_clip": 0.01413262, + "auxiliary_loss_mlp": 0.01089838, + "balance_loss_clip": 1.16052771, + "balance_loss_mlp": 1.03976977, + "epoch": 0.004903952179211885, + "flos": 34342315395840.0, + "grad_norm": 2.3477663992720736, + "language_loss": 0.98906767, + "learning_rate": 2.956207033217471e-06, + "loss": 1.01409864, + "num_input_tokens_seen": 4900820, + "router_z_loss_clip": 2.52734375, + "router_z_loss_mlp": 0.50073242, + "step": 169, + "time_per_iteration": 2.6769628524780273 + }, + { + "auxiliary_loss_clip": 0.01203671, + "auxiliary_loss_mlp": 0.010172, + "balance_loss_clip": 1.10947657, + "balance_loss_mlp": 1.00041556, + "epoch": 0.004932969647727933, + "flos": 67953861924480.0, + "grad_norm": 0.7452925624390325, + "language_loss": 0.61192322, + "learning_rate": 2.9596068664336094e-06, + "loss": 0.63413203, + "num_input_tokens_seen": 4967690, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.16796875, + "step": 170, + "time_per_iteration": 3.241928815841675 + }, + { + "auxiliary_loss_clip": 0.0143268, + "auxiliary_loss_mlp": 0.0111803, + "balance_loss_clip": 1.15953875, + "balance_loss_mlp": 1.06533957, + "epoch": 0.004961987116243979, + "flos": 16355167401600.0, + "grad_norm": 3.289339543348577, + "language_loss": 0.8487255, + "learning_rate": 2.9629867591643182e-06, + "loss": 0.87423265, + "num_input_tokens_seen": 4981430, + "router_z_loss_clip": 2.73242188, + "router_z_loss_mlp": 0.52636719, + "step": 171, + "time_per_iteration": 2.5210304260253906 + }, + { + "auxiliary_loss_clip": 0.01449344, + "auxiliary_loss_mlp": 0.01113248, + "balance_loss_clip": 1.17141199, + "balance_loss_mlp": 1.05345225, + "epoch": 0.004991004584760025, + "flos": 40543134572160.0, + "grad_norm": 3.7475330075339772, + "language_loss": 1.02590477, + "learning_rate": 2.9663469439536884e-06, + "loss": 1.05153072, + "num_input_tokens_seen": 4997690, + "router_z_loss_clip": 2.77929688, + "router_z_loss_mlp": 0.59765625, + "step": 172, + "time_per_iteration": 2.6378421783447266 + }, + { + "auxiliary_loss_clip": 0.01418653, + "auxiliary_loss_mlp": 0.01105103, + "balance_loss_clip": 1.1582588, + "balance_loss_mlp": 1.05024266, + "epoch": 0.005020022053276072, + "flos": 74729982216960.0, + "grad_norm": 3.236065603881287, + "language_loss": 0.90528691, + "learning_rate": 2.969687649301524e-06, + "loss": 0.93052447, + "num_input_tokens_seen": 5019750, + "router_z_loss_clip": 2.6015625, + "router_z_loss_mlp": 0.5480957, + "step": 173, + "time_per_iteration": 3.0664303302764893 + }, + { + "auxiliary_loss_clip": 0.01198377, + "auxiliary_loss_mlp": 0.01023289, + "balance_loss_clip": 1.10565031, + "balance_loss_mlp": 1.00707626, + "epoch": 0.005049039521792119, + "flos": 73019166174720.0, + "grad_norm": 0.7049198873567876, + "language_loss": 0.56636572, + "learning_rate": 2.9730090997565743e-06, + "loss": 0.58858234, + "num_input_tokens_seen": 5082885, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.16210938, + "step": 174, + "time_per_iteration": 3.1671528816223145 + }, + { + "auxiliary_loss_clip": 0.01435858, + "auxiliary_loss_mlp": 0.01112954, + "balance_loss_clip": 1.15832329, + "balance_loss_mlp": 1.05754578, + "epoch": 0.005078056990308165, + "flos": 28072402905600.0, + "grad_norm": 2.0280002752141932, + "language_loss": 0.97398025, + "learning_rate": 2.976311516007114e-06, + "loss": 0.99946833, + "num_input_tokens_seen": 5100670, + "router_z_loss_clip": 2.77734375, + "router_z_loss_mlp": 0.5546875, + "step": 175, + "time_per_iteration": 2.6669626235961914 + }, + { + "auxiliary_loss_clip": 0.01195928, + "auxiliary_loss_mlp": 0.01020009, + "balance_loss_clip": 1.10344219, + "balance_loss_mlp": 1.00427294, + "epoch": 0.005107074458824212, + "flos": 74755432598400.0, + "grad_norm": 0.7563306513505534, + "language_loss": 0.53540528, + "learning_rate": 2.9795951149689236e-06, + "loss": 0.55756468, + "num_input_tokens_seen": 5153055, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.15722656, + "step": 176, + "time_per_iteration": 3.0163261890411377 + }, + { + "auxiliary_loss_clip": 0.01430126, + "auxiliary_loss_mlp": 0.01096744, + "balance_loss_clip": 1.15222502, + "balance_loss_mlp": 1.04298019, + "epoch": 0.005136091927340259, + "flos": 12823876869120.0, + "grad_norm": 2.889248783274339, + "language_loss": 0.82972586, + "learning_rate": 2.982860109870794e-06, + "loss": 0.85499454, + "num_input_tokens_seen": 5167845, + "router_z_loss_clip": 2.78125, + "router_z_loss_mlp": 0.53759766, + "step": 177, + "time_per_iteration": 2.5624003410339355 + }, + { + "auxiliary_loss_clip": 0.01193714, + "auxiliary_loss_mlp": 0.010165, + "balance_loss_clip": 1.1018641, + "balance_loss_mlp": 1.00057364, + "epoch": 0.005165109395856305, + "flos": 63466056339840.0, + "grad_norm": 0.7397174118148311, + "language_loss": 0.58473343, + "learning_rate": 2.986106710337607e-06, + "loss": 0.60683554, + "num_input_tokens_seen": 5226630, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.15917969, + "step": 178, + "time_per_iteration": 3.0169179439544678 + }, + { + "auxiliary_loss_clip": 0.01190937, + "auxiliary_loss_mlp": 0.01014961, + "balance_loss_clip": 1.10070908, + "balance_loss_mlp": 0.9997018, + "epoch": 0.005194126864372352, + "flos": 72657919670400.0, + "grad_norm": 0.9085904505204333, + "language_loss": 0.65165162, + "learning_rate": 2.9893351224711024e-06, + "loss": 0.67371058, + "num_input_tokens_seen": 5290860, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.15234375, + "step": 179, + "time_per_iteration": 3.134096145629883 + }, + { + "auxiliary_loss_clip": 0.01420162, + "auxiliary_loss_mlp": 0.01114708, + "balance_loss_clip": 1.15279114, + "balance_loss_mlp": 1.05736852, + "epoch": 0.005223144332888399, + "flos": 19165605759360.0, + "grad_norm": 2.5420496513833934, + "language_loss": 0.98292792, + "learning_rate": 2.9925455489283856e-06, + "loss": 1.0082767, + "num_input_tokens_seen": 5305520, + "router_z_loss_clip": 2.67382812, + "router_z_loss_mlp": 0.57397461, + "step": 180, + "time_per_iteration": 2.633831739425659 + }, + { + "auxiliary_loss_clip": 0.01431561, + "auxiliary_loss_mlp": 0.01109803, + "balance_loss_clip": 1.15608454, + "balance_loss_mlp": 1.05086589, + "epoch": 0.005252161801404445, + "flos": 22304053140480.0, + "grad_norm": 2.3548484571753105, + "language_loss": 0.89163876, + "learning_rate": 2.9957381889982656e-06, + "loss": 0.91705239, + "num_input_tokens_seen": 5323950, + "router_z_loss_clip": 2.75390625, + "router_z_loss_mlp": 0.58935547, + "step": 181, + "time_per_iteration": 2.649352550506592 + }, + { + "auxiliary_loss_clip": 0.0118707, + "auxiliary_loss_mlp": 0.01021651, + "balance_loss_clip": 1.09793806, + "balance_loss_mlp": 1.00620198, + "epoch": 0.005281179269920492, + "flos": 55140563197440.0, + "grad_norm": 0.7330571925996588, + "language_loss": 0.54696292, + "learning_rate": 2.998913238675487e-06, + "loss": 0.56905013, + "num_input_tokens_seen": 5385430, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.15429688, + "step": 182, + "time_per_iteration": 3.0756375789642334 + }, + { + "auxiliary_loss_clip": 0.01421232, + "auxiliary_loss_mlp": 0.01115372, + "balance_loss_clip": 1.15495861, + "balance_loss_mlp": 1.05448031, + "epoch": 0.005310196738436539, + "flos": 10334290705920.0, + "grad_norm": 3.193043843885067, + "language_loss": 0.89392376, + "learning_rate": 3.0020708907329318e-06, + "loss": 0.91928983, + "num_input_tokens_seen": 5396130, + "router_z_loss_clip": 2.66601562, + "router_z_loss_mlp": 0.609375, + "step": 183, + "time_per_iteration": 2.551381826400757 + }, + { + "auxiliary_loss_clip": 0.01420026, + "auxiliary_loss_mlp": 0.01112643, + "balance_loss_clip": 1.15258884, + "balance_loss_mlp": 1.05675769, + "epoch": 0.005339214206952585, + "flos": 34852605960960.0, + "grad_norm": 3.0138395919664527, + "language_loss": 1.10904431, + "learning_rate": 3.00521133479185e-06, + "loss": 1.13437104, + "num_input_tokens_seen": 5410825, + "router_z_loss_clip": 2.67773438, + "router_z_loss_mlp": 0.55932617, + "step": 184, + "time_per_iteration": 2.8130505084991455 + }, + { + "auxiliary_loss_clip": 0.01433121, + "auxiliary_loss_mlp": 0.01126937, + "balance_loss_clip": 1.15233302, + "balance_loss_mlp": 1.06623578, + "epoch": 0.005368231675468632, + "flos": 21608583390720.0, + "grad_norm": 2.803752398711878, + "language_loss": 1.01100194, + "learning_rate": 3.008334757390187e-06, + "loss": 1.0366025, + "num_input_tokens_seen": 5424460, + "router_z_loss_clip": 2.80859375, + "router_z_loss_mlp": 0.60693359, + "step": 185, + "time_per_iteration": 2.5940232276916504 + }, + { + "auxiliary_loss_clip": 0.01426944, + "auxiliary_loss_mlp": 0.01111311, + "balance_loss_clip": 1.15987134, + "balance_loss_mlp": 1.05275559, + "epoch": 0.005397249143984679, + "flos": 21316814225280.0, + "grad_norm": 2.696434040658944, + "language_loss": 0.9731583, + "learning_rate": 3.011441342049076e-06, + "loss": 0.99854088, + "num_input_tokens_seen": 5438690, + "router_z_loss_clip": 2.671875, + "router_z_loss_mlp": 0.5859375, + "step": 186, + "time_per_iteration": 2.58048677444458 + }, + { + "auxiliary_loss_clip": 0.01411664, + "auxiliary_loss_mlp": 0.01127772, + "balance_loss_clip": 1.14700496, + "balance_loss_mlp": 1.06926453, + "epoch": 0.005426266612500725, + "flos": 74729318901120.0, + "grad_norm": 1.994661925862772, + "language_loss": 0.89182585, + "learning_rate": 3.0145312693375354e-06, + "loss": 0.91722018, + "num_input_tokens_seen": 5469720, + "router_z_loss_clip": 2.6484375, + "router_z_loss_mlp": 0.58496094, + "step": 187, + "time_per_iteration": 2.971195936203003 + }, + { + "auxiliary_loss_clip": 0.01416307, + "auxiliary_loss_mlp": 0.01119027, + "balance_loss_clip": 1.15049553, + "balance_loss_mlp": 1.06457233, + "epoch": 0.005455284081016772, + "flos": 30400167450240.0, + "grad_norm": 3.2565332680743557, + "language_loss": 1.04794168, + "learning_rate": 3.017604716935455e-06, + "loss": 1.073295, + "num_input_tokens_seen": 5485260, + "router_z_loss_clip": 2.65527344, + "router_z_loss_mlp": 0.54394531, + "step": 188, + "time_per_iteration": 2.654827833175659 + }, + { + "auxiliary_loss_clip": 0.01417853, + "auxiliary_loss_mlp": 0.01102467, + "balance_loss_clip": 1.14718294, + "balance_loss_mlp": 1.04746437, + "epoch": 0.005484301549532819, + "flos": 24928823554560.0, + "grad_norm": 3.8014949489313254, + "language_loss": 1.17889631, + "learning_rate": 3.020661859694898e-06, + "loss": 1.20409942, + "num_input_tokens_seen": 5498850, + "router_z_loss_clip": 2.70898438, + "router_z_loss_mlp": 0.55053711, + "step": 189, + "time_per_iteration": 2.5986385345458984 + }, + { + "auxiliary_loss_clip": 0.01173723, + "auxiliary_loss_mlp": 0.0101575, + "balance_loss_clip": 1.08817387, + "balance_loss_mlp": 1.00096786, + "epoch": 0.005513319018048865, + "flos": 71528898257280.0, + "grad_norm": 0.6951757786543544, + "language_loss": 0.52299571, + "learning_rate": 3.023702869699798e-06, + "loss": 0.5448904, + "num_input_tokens_seen": 5564350, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.14746094, + "step": 190, + "time_per_iteration": 3.167358636856079 + }, + { + "auxiliary_loss_clip": 0.01172577, + "auxiliary_loss_mlp": 0.01015016, + "balance_loss_clip": 1.08731282, + "balance_loss_mlp": 1.00032973, + "epoch": 0.005542336486564912, + "flos": 72591095594880.0, + "grad_norm": 0.7973821207601126, + "language_loss": 0.5467993, + "learning_rate": 3.0267279163240784e-06, + "loss": 0.56867522, + "num_input_tokens_seen": 5633550, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.14648438, + "step": 191, + "time_per_iteration": 3.1951072216033936 + }, + { + "auxiliary_loss_clip": 0.01409573, + "auxiliary_loss_mlp": 0.01089353, + "balance_loss_clip": 1.14386845, + "balance_loss_mlp": 1.03480291, + "epoch": 0.005571353955080959, + "flos": 11429481144960.0, + "grad_norm": 3.099583495028832, + "language_loss": 1.02846837, + "learning_rate": 3.0297371662882626e-06, + "loss": 1.0534575, + "num_input_tokens_seen": 5645160, + "router_z_loss_clip": 2.66015625, + "router_z_loss_mlp": 0.54492188, + "step": 192, + "time_per_iteration": 2.5116803646087646 + }, + { + "auxiliary_loss_clip": 0.01421031, + "auxiliary_loss_mlp": 0.01105942, + "balance_loss_clip": 1.14693713, + "balance_loss_mlp": 1.04848361, + "epoch": 0.005600371423597005, + "flos": 17157157309440.0, + "grad_norm": 9.463142891893499, + "language_loss": 0.87134099, + "learning_rate": 3.032730783714606e-06, + "loss": 0.89661074, + "num_input_tokens_seen": 5659580, + "router_z_loss_clip": 2.74023438, + "router_z_loss_mlp": 0.57470703, + "step": 193, + "time_per_iteration": 2.6173486709594727 + }, + { + "auxiliary_loss_clip": 0.01400204, + "auxiliary_loss_mlp": 0.01101606, + "balance_loss_clip": 1.14765358, + "balance_loss_mlp": 1.04524422, + "epoch": 0.005629388892113052, + "flos": 26967437285760.0, + "grad_norm": 2.7423143630824622, + "language_loss": 1.09127569, + "learning_rate": 3.0357089301808127e-06, + "loss": 1.11629367, + "num_input_tokens_seen": 5673350, + "router_z_loss_clip": 2.52734375, + "router_z_loss_mlp": 0.56347656, + "step": 194, + "time_per_iteration": 2.614104747772217 + }, + { + "auxiliary_loss_clip": 0.01387497, + "auxiliary_loss_mlp": 0.01104537, + "balance_loss_clip": 1.13538933, + "balance_loss_mlp": 1.05101168, + "epoch": 0.005658406360629099, + "flos": 12195056638080.0, + "grad_norm": 3.367726997558351, + "language_loss": 1.2294966, + "learning_rate": 3.038671764772362e-06, + "loss": 1.25441694, + "num_input_tokens_seen": 5684200, + "router_z_loss_clip": 2.5234375, + "router_z_loss_mlp": 0.53540039, + "step": 195, + "time_per_iteration": 2.5705814361572266 + }, + { + "auxiliary_loss_clip": 0.01163142, + "auxiliary_loss_mlp": 0.01016998, + "balance_loss_clip": 1.07945371, + "balance_loss_mlp": 1.00240695, + "epoch": 0.005687423829145145, + "flos": 56458115285760.0, + "grad_norm": 0.7727593708676791, + "language_loss": 0.50743914, + "learning_rate": 3.0416194441335026e-06, + "loss": 0.52924055, + "num_input_tokens_seen": 5738855, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.14550781, + "step": 196, + "time_per_iteration": 3.0094547271728516 + }, + { + "auxiliary_loss_clip": 0.01384072, + "auxiliary_loss_mlp": 0.01095317, + "balance_loss_clip": 1.13896489, + "balance_loss_mlp": 1.04384232, + "epoch": 0.005716441297661192, + "flos": 34596343514880.0, + "grad_norm": 2.902294483723141, + "language_loss": 1.21145463, + "learning_rate": 3.0445521225169482e-06, + "loss": 1.23624849, + "num_input_tokens_seen": 5753825, + "router_z_loss_clip": 2.45117188, + "router_z_loss_mlp": 0.51464844, + "step": 197, + "time_per_iteration": 2.713438034057617 + }, + { + "auxiliary_loss_clip": 0.014112, + "auxiliary_loss_mlp": 0.01129142, + "balance_loss_clip": 1.14528513, + "balance_loss_mlp": 1.06791639, + "epoch": 0.005745458766177239, + "flos": 15115401555840.0, + "grad_norm": 2.7110640326456292, + "language_loss": 1.17014372, + "learning_rate": 3.0474699518323115e-06, + "loss": 1.19554722, + "num_input_tokens_seen": 5767625, + "router_z_loss_clip": 2.66015625, + "router_z_loss_mlp": 0.61254883, + "step": 198, + "time_per_iteration": 2.5646474361419678 + }, + { + "auxiliary_loss_clip": 0.0116027, + "auxiliary_loss_mlp": 0.0101513, + "balance_loss_clip": 1.07811975, + "balance_loss_mlp": 1.00120664, + "epoch": 0.005774476234693285, + "flos": 62031301217280.0, + "grad_norm": 0.772484347389699, + "language_loss": 0.60567671, + "learning_rate": 3.0503730816933237e-06, + "loss": 0.62743074, + "num_input_tokens_seen": 5822440, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.13964844, + "step": 199, + "time_per_iteration": 2.9575488567352295 + }, + { + "auxiliary_loss_clip": 0.01392796, + "auxiliary_loss_mlp": 0.0110838, + "balance_loss_clip": 1.13960862, + "balance_loss_mlp": 1.05487883, + "epoch": 0.005803493703209332, + "flos": 24200709816960.0, + "grad_norm": 2.3438922797878883, + "language_loss": 0.9272362, + "learning_rate": 3.0532616594638653e-06, + "loss": 0.95224798, + "num_input_tokens_seen": 5836670, + "router_z_loss_clip": 2.53320312, + "router_z_loss_mlp": 0.53491211, + "step": 200, + "time_per_iteration": 2.547064781188965 + }, + { + "auxiliary_loss_clip": 0.01387991, + "auxiliary_loss_mlp": 0.01118385, + "balance_loss_clip": 1.14060271, + "balance_loss_mlp": 1.06309581, + "epoch": 0.005832511171725379, + "flos": 39231587237760.0, + "grad_norm": 2.967015356547181, + "language_loss": 0.97732615, + "learning_rate": 3.056135830302854e-06, + "loss": 1.00238991, + "num_input_tokens_seen": 5856935, + "router_z_loss_clip": 2.47460938, + "router_z_loss_mlp": 0.55224609, + "step": 201, + "time_per_iteration": 2.7091312408447266 + }, + { + "auxiliary_loss_clip": 0.01382555, + "auxiliary_loss_mlp": 0.01108952, + "balance_loss_clip": 1.13599038, + "balance_loss_mlp": 1.05299544, + "epoch": 0.005861528640241425, + "flos": 32773423363200.0, + "grad_norm": 2.6959454724166236, + "language_loss": 1.14222395, + "learning_rate": 3.058995737208014e-06, + "loss": 1.16713905, + "num_input_tokens_seen": 5872555, + "router_z_loss_clip": 2.46484375, + "router_z_loss_mlp": 0.55981445, + "step": 202, + "time_per_iteration": 2.690154790878296 + }, + { + "auxiliary_loss_clip": 0.01400391, + "auxiliary_loss_mlp": 0.01123227, + "balance_loss_clip": 1.13834298, + "balance_loss_mlp": 1.07034612, + "epoch": 0.005890546108757472, + "flos": 34096281978240.0, + "grad_norm": 2.8260405945152507, + "language_loss": 1.08136451, + "learning_rate": 3.0618415210585666e-06, + "loss": 1.10660064, + "num_input_tokens_seen": 5891840, + "router_z_loss_clip": 2.62109375, + "router_z_loss_mlp": 0.52856445, + "step": 203, + "time_per_iteration": 2.6950621604919434 + }, + { + "auxiliary_loss_clip": 0.01387246, + "auxiliary_loss_mlp": 0.01109355, + "balance_loss_clip": 1.13899469, + "balance_loss_mlp": 1.05213451, + "epoch": 0.005919563577273519, + "flos": 20149981943040.0, + "grad_norm": 3.9242764187338453, + "language_loss": 1.14278364, + "learning_rate": 3.064673320656874e-06, + "loss": 1.16774964, + "num_input_tokens_seen": 5906690, + "router_z_loss_clip": 2.48046875, + "router_z_loss_mlp": 0.57226562, + "step": 204, + "time_per_iteration": 2.5648953914642334 + }, + { + "auxiliary_loss_clip": 0.01388648, + "auxiliary_loss_mlp": 0.0111229, + "balance_loss_clip": 1.13862252, + "balance_loss_mlp": 1.06043458, + "epoch": 0.005948581045789565, + "flos": 28724196677760.0, + "grad_norm": 3.353368248165843, + "language_loss": 0.99967825, + "learning_rate": 3.0674912727690606e-06, + "loss": 1.02468765, + "num_input_tokens_seen": 5923535, + "router_z_loss_clip": 2.49804688, + "router_z_loss_mlp": 0.51855469, + "step": 205, + "time_per_iteration": 2.6001455783843994 + }, + { + "auxiliary_loss_clip": 0.01150984, + "auxiliary_loss_mlp": 0.01016232, + "balance_loss_clip": 1.07143855, + "balance_loss_mlp": 1.00297618, + "epoch": 0.005977598514305612, + "flos": 67438439389440.0, + "grad_norm": 0.648472657898008, + "language_loss": 0.52646041, + "learning_rate": 3.070295512164649e-06, + "loss": 0.54813254, + "num_input_tokens_seen": 5987620, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.1328125, + "step": 206, + "time_per_iteration": 3.148399591445923 + }, + { + "auxiliary_loss_clip": 0.01394469, + "auxiliary_loss_mlp": 0.01117374, + "balance_loss_clip": 1.13414526, + "balance_loss_mlp": 1.06122625, + "epoch": 0.006006615982821659, + "flos": 34713337080960.0, + "grad_norm": 2.4169328677426662, + "language_loss": 0.98581159, + "learning_rate": 3.073086171655237e-06, + "loss": 1.01092994, + "num_input_tokens_seen": 6003980, + "router_z_loss_clip": 2.6015625, + "router_z_loss_mlp": 0.56201172, + "step": 207, + "time_per_iteration": 2.658815383911133 + }, + { + "auxiliary_loss_clip": 0.01149652, + "auxiliary_loss_mlp": 0.0101401, + "balance_loss_clip": 1.07064497, + "balance_loss_mlp": 1.00065887, + "epoch": 0.006035633451337705, + "flos": 57444621062400.0, + "grad_norm": 0.7461357400774679, + "language_loss": 0.49320263, + "learning_rate": 3.0758633821322388e-06, + "loss": 0.51483923, + "num_input_tokens_seen": 6058280, + "router_z_loss_clip": 0.7890625, + "router_z_loss_mlp": 0.13378906, + "step": 208, + "time_per_iteration": 2.977206230163574 + }, + { + "auxiliary_loss_clip": 0.01390828, + "auxiliary_loss_mlp": 0.01097402, + "balance_loss_clip": 1.13063169, + "balance_loss_mlp": 1.04237533, + "epoch": 0.006064650919853752, + "flos": 23733327179520.0, + "grad_norm": 4.6664756522655715, + "language_loss": 1.16435468, + "learning_rate": 3.078627272603724e-06, + "loss": 1.189237, + "num_input_tokens_seen": 6076740, + "router_z_loss_clip": 2.6015625, + "router_z_loss_mlp": 0.55029297, + "step": 209, + "time_per_iteration": 2.556694984436035 + }, + { + "auxiliary_loss_clip": 0.0114766, + "auxiliary_loss_mlp": 0.01013288, + "balance_loss_clip": 1.06906104, + "balance_loss_mlp": 0.99984163, + "epoch": 0.006093668388369799, + "flos": 62187990865920.0, + "grad_norm": 0.8035737185403703, + "language_loss": 0.58177167, + "learning_rate": 3.081377970230378e-06, + "loss": 0.60338116, + "num_input_tokens_seen": 6136335, + "router_z_loss_clip": 0.7890625, + "router_z_loss_mlp": 0.13476562, + "step": 210, + "time_per_iteration": 3.158872127532959 + }, + { + "auxiliary_loss_clip": 0.01379111, + "auxiliary_loss_mlp": 0.0110607, + "balance_loss_clip": 1.13045502, + "balance_loss_mlp": 1.04484415, + "epoch": 0.006122685856885845, + "flos": 40106580531840.0, + "grad_norm": 3.697108292620776, + "language_loss": 1.01555932, + "learning_rate": 3.0841156003606057e-06, + "loss": 1.04041111, + "num_input_tokens_seen": 6153170, + "router_z_loss_clip": 2.48828125, + "router_z_loss_mlp": 0.61230469, + "step": 211, + "time_per_iteration": 2.7398524284362793 + }, + { + "auxiliary_loss_clip": 0.01390797, + "auxiliary_loss_mlp": 0.01111317, + "balance_loss_clip": 1.13235617, + "balance_loss_mlp": 1.05652857, + "epoch": 0.006151703325401892, + "flos": 74730575710080.0, + "grad_norm": 2.725358833079068, + "language_loss": 0.91916066, + "learning_rate": 3.0868402865648067e-06, + "loss": 0.9441818, + "num_input_tokens_seen": 6175540, + "router_z_loss_clip": 2.58789062, + "router_z_loss_mlp": 0.54711914, + "step": 212, + "time_per_iteration": 2.9422056674957275 + }, + { + "auxiliary_loss_clip": 0.01376486, + "auxiliary_loss_mlp": 0.01101836, + "balance_loss_clip": 1.13387823, + "balance_loss_mlp": 1.05117166, + "epoch": 0.006180720793917939, + "flos": 74733927200640.0, + "grad_norm": 2.551079399388518, + "language_loss": 1.00613534, + "learning_rate": 3.0895521506688455e-06, + "loss": 1.03091848, + "num_input_tokens_seen": 6209680, + "router_z_loss_clip": 2.42578125, + "router_z_loss_mlp": 0.50732422, + "step": 213, + "time_per_iteration": 3.012164354324341 + }, + { + "auxiliary_loss_clip": 0.01144217, + "auxiliary_loss_mlp": 0.01018532, + "balance_loss_clip": 1.06612301, + "balance_loss_mlp": 1.0048945, + "epoch": 0.006209738262433985, + "flos": 69660382003200.0, + "grad_norm": 0.6940384592818486, + "language_loss": 0.56867921, + "learning_rate": 3.092251312786734e-06, + "loss": 0.59030676, + "num_input_tokens_seen": 6273105, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.13671875, + "step": 214, + "time_per_iteration": 3.0688114166259766 + }, + { + "auxiliary_loss_clip": 0.01369576, + "auxiliary_loss_mlp": 0.01088843, + "balance_loss_clip": 1.12775147, + "balance_loss_mlp": 1.03794098, + "epoch": 0.006238755730950032, + "flos": 14239186364160.0, + "grad_norm": 5.095972591232032, + "language_loss": 1.03363681, + "learning_rate": 3.094937891352556e-06, + "loss": 1.0582211, + "num_input_tokens_seen": 6287415, + "router_z_loss_clip": 2.41601562, + "router_z_loss_mlp": 0.5090332, + "step": 215, + "time_per_iteration": 2.60644793510437 + }, + { + "auxiliary_loss_clip": 0.01141814, + "auxiliary_loss_mlp": 0.01018782, + "balance_loss_clip": 1.06406534, + "balance_loss_mlp": 1.00533521, + "epoch": 0.006267773199466079, + "flos": 74774459289600.0, + "grad_norm": 0.7631804918401681, + "language_loss": 0.57154804, + "learning_rate": 3.09761200315165e-06, + "loss": 0.59315401, + "num_input_tokens_seen": 6350530, + "router_z_loss_clip": 0.77734375, + "router_z_loss_mlp": 0.13476562, + "step": 216, + "time_per_iteration": 3.1157338619232178 + }, + { + "auxiliary_loss_clip": 0.01369011, + "auxiliary_loss_mlp": 0.01105851, + "balance_loss_clip": 1.13026595, + "balance_loss_mlp": 1.04929829, + "epoch": 0.006296790667982125, + "flos": 23798894446080.0, + "grad_norm": 2.2585693892749945, + "language_loss": 0.86686289, + "learning_rate": 3.100273763351068e-06, + "loss": 0.89161146, + "num_input_tokens_seen": 6366080, + "router_z_loss_clip": 2.38867188, + "router_z_loss_mlp": 0.56591797, + "step": 217, + "time_per_iteration": 2.5788350105285645 + }, + { + "auxiliary_loss_clip": 0.01138318, + "auxiliary_loss_mlp": 0.01013931, + "balance_loss_clip": 1.06188488, + "balance_loss_mlp": 1.00105619, + "epoch": 0.006325808136498172, + "flos": 74769536787840.0, + "grad_norm": 0.7178325141580802, + "language_loss": 0.59362924, + "learning_rate": 3.102923285529342e-06, + "loss": 0.6151517, + "num_input_tokens_seen": 6426405, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.12890625, + "step": 218, + "time_per_iteration": 3.089313507080078 + }, + { + "auxiliary_loss_clip": 0.01137, + "auxiliary_loss_mlp": 0.01016574, + "balance_loss_clip": 1.0610311, + "balance_loss_mlp": 1.00322306, + "epoch": 0.006354825605014219, + "flos": 74779207234560.0, + "grad_norm": 0.6892237982214503, + "language_loss": 0.53868914, + "learning_rate": 3.105560681705561e-06, + "loss": 0.56022489, + "num_input_tokens_seen": 6492415, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.13378906, + "step": 219, + "time_per_iteration": 3.183962821960449 + }, + { + "auxiliary_loss_clip": 0.01375751, + "auxiliary_loss_mlp": 0.01103308, + "balance_loss_clip": 1.12184799, + "balance_loss_mlp": 1.04792333, + "epoch": 0.006383843073530265, + "flos": 23907544197120.0, + "grad_norm": 3.6924341284978706, + "language_loss": 0.94772995, + "learning_rate": 3.1081860623677917e-06, + "loss": 0.97252053, + "num_input_tokens_seen": 6506585, + "router_z_loss_clip": 2.54296875, + "router_z_loss_mlp": 0.55419922, + "step": 220, + "time_per_iteration": 2.565293073654175 + }, + { + "auxiliary_loss_clip": 0.01133112, + "auxiliary_loss_mlp": 0.01013718, + "balance_loss_clip": 1.05785549, + "balance_loss_mlp": 1.00065243, + "epoch": 0.006412860542046312, + "flos": 70559640558720.0, + "grad_norm": 0.7905340372209563, + "language_loss": 0.54005957, + "learning_rate": 3.11079953650085e-06, + "loss": 0.56152785, + "num_input_tokens_seen": 6560050, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.13085938, + "step": 221, + "time_per_iteration": 2.9704031944274902 + }, + { + "auxiliary_loss_clip": 0.01371209, + "auxiliary_loss_mlp": 0.01103255, + "balance_loss_clip": 1.12523782, + "balance_loss_mlp": 1.05263913, + "epoch": 0.006441878010562358, + "flos": 10405618346880.0, + "grad_norm": 4.091926752934666, + "language_loss": 0.99117529, + "learning_rate": 3.1134012116134513e-06, + "loss": 1.01591992, + "num_input_tokens_seen": 6569480, + "router_z_loss_clip": 2.45703125, + "router_z_loss_mlp": 0.50585938, + "step": 222, + "time_per_iteration": 2.4767134189605713 + }, + { + "auxiliary_loss_clip": 0.01373432, + "auxiliary_loss_mlp": 0.01130775, + "balance_loss_clip": 1.12651443, + "balance_loss_mlp": 1.07360196, + "epoch": 0.006470895479078405, + "flos": 15077311395840.0, + "grad_norm": 2.829767502214582, + "language_loss": 0.93191755, + "learning_rate": 3.1159911937647437e-06, + "loss": 0.95695961, + "num_input_tokens_seen": 6584705, + "router_z_loss_clip": 2.47265625, + "router_z_loss_mlp": 0.57275391, + "step": 223, + "time_per_iteration": 2.5232415199279785 + }, + { + "auxiliary_loss_clip": 0.0136758, + "auxiliary_loss_mlp": 0.01095375, + "balance_loss_clip": 1.12451673, + "balance_loss_mlp": 1.04478228, + "epoch": 0.006499912947594452, + "flos": 24527496942720.0, + "grad_norm": 2.416735701506376, + "language_loss": 1.01618314, + "learning_rate": 3.1185695875902545e-06, + "loss": 1.04081273, + "num_input_tokens_seen": 6601500, + "router_z_loss_clip": 2.4296875, + "router_z_loss_mlp": 0.50610352, + "step": 224, + "time_per_iteration": 2.676933526992798 + }, + { + "auxiliary_loss_clip": 0.01384071, + "auxiliary_loss_mlp": 0.01117072, + "balance_loss_clip": 1.12541974, + "balance_loss_mlp": 1.06209278, + "epoch": 0.006528930416110498, + "flos": 16283036799360.0, + "grad_norm": 2.7133444640229336, + "language_loss": 0.93386519, + "learning_rate": 3.1211364963272528e-06, + "loss": 0.95887661, + "num_input_tokens_seen": 6615935, + "router_z_loss_clip": 2.58398438, + "router_z_loss_mlp": 0.54956055, + "step": 225, + "time_per_iteration": 2.5669455528259277 + }, + { + "auxiliary_loss_clip": 0.01372467, + "auxiliary_loss_mlp": 0.01114169, + "balance_loss_clip": 1.1274085, + "balance_loss_mlp": 1.05995321, + "epoch": 0.006557947884626545, + "flos": 74729668014720.0, + "grad_norm": 3.1585605415366844, + "language_loss": 1.03615224, + "learning_rate": 3.123692021839555e-06, + "loss": 1.06101871, + "num_input_tokens_seen": 6639770, + "router_z_loss_clip": 2.453125, + "router_z_loss_mlp": 0.54199219, + "step": 226, + "time_per_iteration": 2.993429660797119 + }, + { + "auxiliary_loss_clip": 0.01125741, + "auxiliary_loss_mlp": 0.01018612, + "balance_loss_clip": 1.05062842, + "balance_loss_mlp": 1.00478363, + "epoch": 0.006586965353142592, + "flos": 63937802897280.0, + "grad_norm": 0.7404674770617227, + "language_loss": 0.56023437, + "learning_rate": 3.126236264641778e-06, + "loss": 0.58167791, + "num_input_tokens_seen": 6699950, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.13867188, + "step": 227, + "time_per_iteration": 3.067471742630005 + }, + { + "auxiliary_loss_clip": 0.01358762, + "auxiliary_loss_mlp": 0.01095894, + "balance_loss_clip": 1.11458874, + "balance_loss_mlp": 1.04244041, + "epoch": 0.006615982821658638, + "flos": 31315589965440.0, + "grad_norm": 2.639065303400254, + "language_loss": 1.05490482, + "learning_rate": 3.1287693239230624e-06, + "loss": 1.07945132, + "num_input_tokens_seen": 6716875, + "router_z_loss_clip": 2.44140625, + "router_z_loss_mlp": 0.53393555, + "step": 228, + "time_per_iteration": 2.6244328022003174 + }, + { + "auxiliary_loss_clip": 0.01368133, + "auxiliary_loss_mlp": 0.01108796, + "balance_loss_clip": 1.12845945, + "balance_loss_mlp": 1.05753589, + "epoch": 0.006645000290174685, + "flos": 14822864340480.0, + "grad_norm": 3.314080706686952, + "language_loss": 0.88409781, + "learning_rate": 3.1312912975702777e-06, + "loss": 0.90886706, + "num_input_tokens_seen": 6729765, + "router_z_loss_clip": 2.39648438, + "router_z_loss_mlp": 0.51269531, + "step": 229, + "time_per_iteration": 2.5468990802764893 + }, + { + "auxiliary_loss_clip": 0.01120025, + "auxiliary_loss_mlp": 0.01013853, + "balance_loss_clip": 1.04761505, + "balance_loss_mlp": 1.00040674, + "epoch": 0.006674017758690732, + "flos": 53022417655680.0, + "grad_norm": 0.7498031610604586, + "language_loss": 0.5190571, + "learning_rate": 3.133802282190717e-06, + "loss": 0.54039586, + "num_input_tokens_seen": 6789420, + "router_z_loss_clip": 0.72265625, + "router_z_loss_mlp": 0.13476562, + "step": 230, + "time_per_iteration": 3.0127382278442383 + }, + { + "auxiliary_loss_clip": 0.01372008, + "auxiliary_loss_mlp": 0.01111848, + "balance_loss_clip": 1.11908495, + "balance_loss_mlp": 1.05858493, + "epoch": 0.006703035227206778, + "flos": 14493807976320.0, + "grad_norm": 3.025321232536006, + "language_loss": 1.13281024, + "learning_rate": 3.1363023731343034e-06, + "loss": 1.1576488, + "num_input_tokens_seen": 6801630, + "router_z_loss_clip": 2.52929688, + "router_z_loss_mlp": 0.5324707, + "step": 231, + "time_per_iteration": 2.560091733932495 + }, + { + "auxiliary_loss_clip": 0.01357268, + "auxiliary_loss_mlp": 0.01099854, + "balance_loss_clip": 1.11758542, + "balance_loss_mlp": 1.04623365, + "epoch": 0.006732052695722825, + "flos": 11358502617600.0, + "grad_norm": 2.8204943618715452, + "language_loss": 0.88071305, + "learning_rate": 3.1387916645153185e-06, + "loss": 0.90528429, + "num_input_tokens_seen": 6812580, + "router_z_loss_clip": 2.3984375, + "router_z_loss_mlp": 0.53564453, + "step": 232, + "time_per_iteration": 2.5230066776275635 + }, + { + "auxiliary_loss_clip": 0.01363107, + "auxiliary_loss_mlp": 0.01112517, + "balance_loss_clip": 1.11531258, + "balance_loss_mlp": 1.05622649, + "epoch": 0.006761070164238872, + "flos": 18219005533440.0, + "grad_norm": 3.2941824699528937, + "language_loss": 1.08350766, + "learning_rate": 3.1412702492336547e-06, + "loss": 1.10826385, + "num_input_tokens_seen": 6827190, + "router_z_loss_clip": 2.47460938, + "router_z_loss_mlp": 0.56298828, + "step": 233, + "time_per_iteration": 2.5768680572509766 + }, + { + "auxiliary_loss_clip": 0.01346841, + "auxiliary_loss_mlp": 0.01120599, + "balance_loss_clip": 1.11464858, + "balance_loss_mlp": 1.07100797, + "epoch": 0.006790087632754918, + "flos": 23359547496960.0, + "grad_norm": 2.648524131909128, + "language_loss": 1.05120361, + "learning_rate": 3.1437382189956262e-06, + "loss": 1.0758779, + "num_input_tokens_seen": 6840370, + "router_z_loss_clip": 2.32421875, + "router_z_loss_mlp": 0.49584961, + "step": 234, + "time_per_iteration": 5.698854207992554 + }, + { + "auxiliary_loss_clip": 0.01353062, + "auxiliary_loss_mlp": 0.01099658, + "balance_loss_clip": 1.11384392, + "balance_loss_mlp": 1.04329586, + "epoch": 0.006819105101270965, + "flos": 29710178784000.0, + "grad_norm": 2.1767874039828503, + "language_loss": 0.94699556, + "learning_rate": 3.146195664334322e-06, + "loss": 0.97152281, + "num_input_tokens_seen": 6857495, + "router_z_loss_clip": 2.3984375, + "router_z_loss_mlp": 0.5637207, + "step": 235, + "time_per_iteration": 5.377424955368042 + }, + { + "auxiliary_loss_clip": 0.01344247, + "auxiliary_loss_mlp": 0.01109741, + "balance_loss_clip": 1.10846734, + "balance_loss_mlp": 1.05015993, + "epoch": 0.006848122569787012, + "flos": 31786428827520.0, + "grad_norm": 2.6189032071908285, + "language_loss": 1.03044331, + "learning_rate": 3.1486426746295384e-06, + "loss": 1.05498314, + "num_input_tokens_seen": 6874455, + "router_z_loss_clip": 2.35839844, + "router_z_loss_mlp": 0.59594727, + "step": 236, + "time_per_iteration": 7.567396402359009 + }, + { + "auxiliary_loss_clip": 0.01350757, + "auxiliary_loss_mlp": 0.01101245, + "balance_loss_clip": 1.1117698, + "balance_loss_mlp": 1.04412007, + "epoch": 0.006877140038303058, + "flos": 23030630778240.0, + "grad_norm": 6.027839305218062, + "language_loss": 1.11477387, + "learning_rate": 3.151079338127282e-06, + "loss": 1.13929391, + "num_input_tokens_seen": 6887410, + "router_z_loss_clip": 2.38671875, + "router_z_loss_mlp": 0.5703125, + "step": 237, + "time_per_iteration": 2.565986394882202 + }, + { + "auxiliary_loss_clip": 0.01342745, + "auxiliary_loss_mlp": 0.01105369, + "balance_loss_clip": 1.12106299, + "balance_loss_mlp": 1.05549192, + "epoch": 0.006906157506819105, + "flos": 37952998093440.0, + "grad_norm": 2.8216737877892855, + "language_loss": 1.10460436, + "learning_rate": 3.1535057419588662e-06, + "loss": 1.12908554, + "num_input_tokens_seen": 6907445, + "router_z_loss_clip": 2.21679688, + "router_z_loss_mlp": 0.49853516, + "step": 238, + "time_per_iteration": 2.694779872894287 + }, + { + "auxiliary_loss_clip": 0.01342244, + "auxiliary_loss_mlp": 0.01103437, + "balance_loss_clip": 1.11110783, + "balance_loss_mlp": 1.0496738, + "epoch": 0.006935174975335152, + "flos": 40106336152320.0, + "grad_norm": 3.685145314920763, + "language_loss": 0.88919425, + "learning_rate": 3.155921972159608e-06, + "loss": 0.91365111, + "num_input_tokens_seen": 6922595, + "router_z_loss_clip": 2.31347656, + "router_z_loss_mlp": 0.53759766, + "step": 239, + "time_per_iteration": 2.578927755355835 + }, + { + "auxiliary_loss_clip": 0.01327944, + "auxiliary_loss_mlp": 0.01092569, + "balance_loss_clip": 1.10801625, + "balance_loss_mlp": 1.04731727, + "epoch": 0.006964192443851198, + "flos": 43427099986560.0, + "grad_norm": 2.13251581427715, + "language_loss": 0.86107397, + "learning_rate": 3.1583281136871298e-06, + "loss": 0.88527906, + "num_input_tokens_seen": 6943745, + "router_z_loss_clip": 2.19921875, + "router_z_loss_mlp": 0.45239258, + "step": 240, + "time_per_iteration": 2.8548710346221924 + }, + { + "auxiliary_loss_clip": 0.01316749, + "auxiliary_loss_mlp": 0.01091423, + "balance_loss_clip": 1.10546732, + "balance_loss_mlp": 1.04567027, + "epoch": 0.006993209912367245, + "flos": 18946316309760.0, + "grad_norm": 2.543096652165729, + "language_loss": 0.94322699, + "learning_rate": 3.1607242504392867e-06, + "loss": 0.96730876, + "num_input_tokens_seen": 6957530, + "router_z_loss_clip": 2.11132812, + "router_z_loss_mlp": 0.45776367, + "step": 241, + "time_per_iteration": 2.498368978500366 + }, + { + "auxiliary_loss_clip": 0.0110957, + "auxiliary_loss_mlp": 0.01014651, + "balance_loss_clip": 1.03823161, + "balance_loss_mlp": 1.00158536, + "epoch": 0.007022227380883292, + "flos": 74781965232000.0, + "grad_norm": 0.768424320000282, + "language_loss": 0.62322944, + "learning_rate": 3.1631104652717176e-06, + "loss": 0.64447165, + "num_input_tokens_seen": 7022510, + "router_z_loss_clip": 0.7109375, + "router_z_loss_mlp": 0.13085938, + "step": 242, + "time_per_iteration": 3.1712892055511475 + }, + { + "auxiliary_loss_clip": 0.01345053, + "auxiliary_loss_mlp": 0.01090708, + "balance_loss_clip": 1.10559607, + "balance_loss_mlp": 1.03572822, + "epoch": 0.007051244849399338, + "flos": 28504453380480.0, + "grad_norm": 3.2606882156383534, + "language_loss": 1.16821671, + "learning_rate": 3.1654868400150375e-06, + "loss": 1.19257426, + "num_input_tokens_seen": 7036600, + "router_z_loss_clip": 2.39453125, + "router_z_loss_mlp": 0.55004883, + "step": 243, + "time_per_iteration": 2.5345380306243896 + }, + { + "auxiliary_loss_clip": 0.01107501, + "auxiliary_loss_mlp": 0.01013536, + "balance_loss_clip": 1.03683782, + "balance_loss_mlp": 1.00047064, + "epoch": 0.007080262317915385, + "flos": 72254603111040.0, + "grad_norm": 0.9232264686316656, + "language_loss": 0.58564758, + "learning_rate": 3.167853455491676e-06, + "loss": 0.6068579, + "num_input_tokens_seen": 7099645, + "router_z_loss_clip": 0.70703125, + "router_z_loss_mlp": 0.13085938, + "step": 244, + "time_per_iteration": 3.0237810611724854 + }, + { + "auxiliary_loss_clip": 0.01348233, + "auxiliary_loss_mlp": 0.01099489, + "balance_loss_clip": 1.10871875, + "balance_loss_mlp": 1.04920626, + "epoch": 0.007109279786431432, + "flos": 25658543214720.0, + "grad_norm": 2.438814227852582, + "language_loss": 0.87012649, + "learning_rate": 3.1702103915323702e-06, + "loss": 0.89460361, + "num_input_tokens_seen": 7119230, + "router_z_loss_clip": 2.39648438, + "router_z_loss_mlp": 0.50292969, + "step": 245, + "time_per_iteration": 2.57757830619812 + }, + { + "auxiliary_loss_clip": 0.01105954, + "auxiliary_loss_mlp": 0.01012113, + "balance_loss_clip": 1.03517342, + "balance_loss_mlp": 0.99961978, + "epoch": 0.007138297254947478, + "flos": 51781534646400.0, + "grad_norm": 0.7960504118534252, + "language_loss": 0.5428108, + "learning_rate": 3.172557726992324e-06, + "loss": 0.56399143, + "num_input_tokens_seen": 7171815, + "router_z_loss_clip": 0.70703125, + "router_z_loss_mlp": 0.125, + "step": 246, + "time_per_iteration": 2.8637499809265137 + }, + { + "auxiliary_loss_clip": 0.01337002, + "auxiliary_loss_mlp": 0.01104959, + "balance_loss_clip": 1.10274184, + "balance_loss_mlp": 1.05219698, + "epoch": 0.007167314723463525, + "flos": 11867641107840.0, + "grad_norm": 3.8477668567959182, + "language_loss": 1.1349045, + "learning_rate": 3.1748955397670386e-06, + "loss": 1.15932417, + "num_input_tokens_seen": 7183960, + "router_z_loss_clip": 2.34179688, + "router_z_loss_mlp": 0.52807617, + "step": 247, + "time_per_iteration": 2.496349334716797 + }, + { + "auxiliary_loss_clip": 0.0133535, + "auxiliary_loss_mlp": 0.01091246, + "balance_loss_clip": 1.0996139, + "balance_loss_mlp": 1.04148769, + "epoch": 0.007196332191979572, + "flos": 12778315678080.0, + "grad_norm": 3.029808976255468, + "language_loss": 1.07360554, + "learning_rate": 3.17722390680782e-06, + "loss": 1.09787154, + "num_input_tokens_seen": 7195335, + "router_z_loss_clip": 2.35546875, + "router_z_loss_mlp": 0.49780273, + "step": 248, + "time_per_iteration": 2.5451924800872803 + }, + { + "auxiliary_loss_clip": 0.01345534, + "auxiliary_loss_mlp": 0.01106204, + "balance_loss_clip": 1.10802817, + "balance_loss_mlp": 1.05554032, + "epoch": 0.007225349660495618, + "flos": 31103073319680.0, + "grad_norm": 2.44567926176179, + "language_loss": 0.99701953, + "learning_rate": 3.1795429041369805e-06, + "loss": 1.02153683, + "num_input_tokens_seen": 7216575, + "router_z_loss_clip": 2.375, + "router_z_loss_mlp": 0.50708008, + "step": 249, + "time_per_iteration": 2.6406772136688232 + }, + { + "auxiliary_loss_clip": 0.01336016, + "auxiliary_loss_mlp": 0.01105312, + "balance_loss_clip": 1.10581732, + "balance_loss_mlp": 1.0581286, + "epoch": 0.007254367129011665, + "flos": 14204168403840.0, + "grad_norm": 3.098373208369785, + "language_loss": 1.09946609, + "learning_rate": 3.1818526068627325e-06, + "loss": 1.12387943, + "num_input_tokens_seen": 7235890, + "router_z_loss_clip": 2.30078125, + "router_z_loss_mlp": 0.47167969, + "step": 250, + "time_per_iteration": 2.6626663208007812 + }, + { + "auxiliary_loss_clip": 0.0133581, + "auxiliary_loss_mlp": 0.01104225, + "balance_loss_clip": 1.10327077, + "balance_loss_mlp": 1.05461049, + "epoch": 0.007283384597527712, + "flos": 27122555923200.0, + "grad_norm": 2.2377253617138946, + "language_loss": 0.83166546, + "learning_rate": 3.1841530891937837e-06, + "loss": 0.85606575, + "num_input_tokens_seen": 7257230, + "router_z_loss_clip": 2.32421875, + "router_z_loss_mlp": 0.49707031, + "step": 251, + "time_per_iteration": 2.6049063205718994 + }, + { + "auxiliary_loss_clip": 0.01344644, + "auxiliary_loss_mlp": 0.01135372, + "balance_loss_clip": 1.10518527, + "balance_loss_mlp": 1.07924819, + "epoch": 0.007312402066043758, + "flos": 28767733009920.0, + "grad_norm": 2.311675362332758, + "language_loss": 0.94115865, + "learning_rate": 3.186444424453642e-06, + "loss": 0.96595871, + "num_input_tokens_seen": 7276215, + "router_z_loss_clip": 2.39648438, + "router_z_loss_mlp": 0.56152344, + "step": 252, + "time_per_iteration": 2.6627261638641357 + }, + { + "auxiliary_loss_clip": 0.01341296, + "auxiliary_loss_mlp": 0.01107448, + "balance_loss_clip": 1.10690904, + "balance_loss_mlp": 1.05838168, + "epoch": 0.007341419534559805, + "flos": 17302291297920.0, + "grad_norm": 2.5794081776393933, + "language_loss": 0.9313463, + "learning_rate": 3.188726685094643e-06, + "loss": 0.95583379, + "num_input_tokens_seen": 7289725, + "router_z_loss_clip": 2.34570312, + "router_z_loss_mlp": 0.4909668, + "step": 253, + "time_per_iteration": 2.519981622695923 + }, + { + "auxiliary_loss_clip": 0.01346972, + "auxiliary_loss_mlp": 0.01107645, + "balance_loss_clip": 1.10778689, + "balance_loss_mlp": 1.05378628, + "epoch": 0.007370437003075852, + "flos": 36749507016960.0, + "grad_norm": 3.2784542883457033, + "language_loss": 1.12787497, + "learning_rate": 3.1909999427116915e-06, + "loss": 1.15242124, + "num_input_tokens_seen": 7311345, + "router_z_loss_clip": 2.39257812, + "router_z_loss_mlp": 0.53857422, + "step": 254, + "time_per_iteration": 2.669159412384033 + }, + { + "auxiliary_loss_clip": 0.01320621, + "auxiliary_loss_mlp": 0.01093223, + "balance_loss_clip": 1.10142159, + "balance_loss_mlp": 1.04396534, + "epoch": 0.007399454471591898, + "flos": 17448367893120.0, + "grad_norm": 2.946794514396719, + "language_loss": 1.09439051, + "learning_rate": 3.193264268055741e-06, + "loss": 1.11852884, + "num_input_tokens_seen": 7324090, + "router_z_loss_clip": 2.1875, + "router_z_loss_mlp": 0.49243164, + "step": 255, + "time_per_iteration": 2.5787036418914795 + }, + { + "auxiliary_loss_clip": 0.01098079, + "auxiliary_loss_mlp": 0.01025178, + "balance_loss_clip": 1.02754021, + "balance_loss_mlp": 1.01287603, + "epoch": 0.007428471940107945, + "flos": 67920764088960.0, + "grad_norm": 0.788382811410653, + "language_loss": 0.59061122, + "learning_rate": 3.1955197310470064e-06, + "loss": 0.61184376, + "num_input_tokens_seen": 7388060, + "router_z_loss_clip": 0.703125, + "router_z_loss_mlp": 0.12255859, + "step": 256, + "time_per_iteration": 3.1276979446411133 + }, + { + "auxiliary_loss_clip": 0.01337566, + "auxiliary_loss_mlp": 0.01109018, + "balance_loss_clip": 1.10564506, + "balance_loss_mlp": 1.05747199, + "epoch": 0.007457489408623992, + "flos": 26387669381760.0, + "grad_norm": 3.1800290627416974, + "language_loss": 1.02220809, + "learning_rate": 3.197766400787917e-06, + "loss": 1.04667401, + "num_input_tokens_seen": 7402750, + "router_z_loss_clip": 2.31835938, + "router_z_loss_mlp": 0.51611328, + "step": 257, + "time_per_iteration": 2.5296173095703125 + }, + { + "auxiliary_loss_clip": 0.01321218, + "auxiliary_loss_mlp": 0.01089725, + "balance_loss_clip": 1.10314131, + "balance_loss_mlp": 1.04447317, + "epoch": 0.007486506877140038, + "flos": 28905500701440.0, + "grad_norm": 5.724861361264618, + "language_loss": 1.09158731, + "learning_rate": 3.2000043455758205e-06, + "loss": 1.11569667, + "num_input_tokens_seen": 7417125, + "router_z_loss_clip": 2.17773438, + "router_z_loss_mlp": 0.45263672, + "step": 258, + "time_per_iteration": 2.5262691974639893 + }, + { + "auxiliary_loss_clip": 0.01327257, + "auxiliary_loss_mlp": 0.01097417, + "balance_loss_clip": 1.10163164, + "balance_loss_mlp": 1.04537034, + "epoch": 0.007515524345656085, + "flos": 24855889991040.0, + "grad_norm": 3.530691514335101, + "language_loss": 1.02949631, + "learning_rate": 3.2022336329154436e-06, + "loss": 1.053743, + "num_input_tokens_seen": 7430290, + "router_z_loss_clip": 2.25488281, + "router_z_loss_mlp": 0.52050781, + "step": 259, + "time_per_iteration": 2.532068967819214 + }, + { + "auxiliary_loss_clip": 0.01321129, + "auxiliary_loss_mlp": 0.01092761, + "balance_loss_clip": 1.10116005, + "balance_loss_mlp": 1.04052329, + "epoch": 0.007544541814172132, + "flos": 22883367196800.0, + "grad_norm": 3.462910317925729, + "language_loss": 0.9906286, + "learning_rate": 3.204454329531106e-06, + "loss": 1.01476753, + "num_input_tokens_seen": 7441050, + "router_z_loss_clip": 2.20117188, + "router_z_loss_mlp": 0.52246094, + "step": 260, + "time_per_iteration": 2.530834913253784 + }, + { + "auxiliary_loss_clip": 0.01325512, + "auxiliary_loss_mlp": 0.01086934, + "balance_loss_clip": 1.1008718, + "balance_loss_mlp": 1.03903544, + "epoch": 0.007573559282688178, + "flos": 53771933923200.0, + "grad_norm": 2.365797636424978, + "language_loss": 0.76420903, + "learning_rate": 3.2066665013787064e-06, + "loss": 0.78833354, + "num_input_tokens_seen": 7464410, + "router_z_loss_clip": 2.24609375, + "router_z_loss_mlp": 0.47875977, + "step": 261, + "time_per_iteration": 2.7774126529693604 + }, + { + "auxiliary_loss_clip": 0.01327086, + "auxiliary_loss_mlp": 0.0108694, + "balance_loss_clip": 1.10097778, + "balance_loss_mlp": 1.03689551, + "epoch": 0.007602576751204225, + "flos": 23835553240320.0, + "grad_norm": 4.117504319367123, + "language_loss": 1.00473797, + "learning_rate": 3.2088702136574735e-06, + "loss": 1.02887821, + "num_input_tokens_seen": 7478955, + "router_z_loss_clip": 2.265625, + "router_z_loss_mlp": 0.5, + "step": 262, + "time_per_iteration": 2.563304901123047 + }, + { + "auxiliary_loss_clip": 0.01096381, + "auxiliary_loss_mlp": 0.01012501, + "balance_loss_clip": 1.02648497, + "balance_loss_mlp": 0.99991232, + "epoch": 0.007631594219720272, + "flos": 63512280846720.0, + "grad_norm": 0.7365646650546431, + "language_loss": 0.58088815, + "learning_rate": 3.2110655308215014e-06, + "loss": 0.60197699, + "num_input_tokens_seen": 7542680, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.12597656, + "step": 263, + "time_per_iteration": 3.140061140060425 + }, + { + "auxiliary_loss_clip": 0.01311162, + "auxiliary_loss_mlp": 0.01083695, + "balance_loss_clip": 1.08919215, + "balance_loss_mlp": 1.03546333, + "epoch": 0.007660611688236318, + "flos": 27446654874240.0, + "grad_norm": 2.7053314478849875, + "language_loss": 1.08959401, + "learning_rate": 3.2132525165910553e-06, + "loss": 1.11354256, + "num_input_tokens_seen": 7558260, + "router_z_loss_clip": 2.22265625, + "router_z_loss_mlp": 0.48242188, + "step": 264, + "time_per_iteration": 2.634575128555298 + }, + { + "auxiliary_loss_clip": 0.01094834, + "auxiliary_loss_mlp": 0.01013331, + "balance_loss_clip": 1.02569151, + "balance_loss_mlp": 1.00045669, + "epoch": 0.007689629156752365, + "flos": 56999129713920.0, + "grad_norm": 0.7891033676549705, + "language_loss": 0.55800265, + "learning_rate": 3.2154312339636743e-06, + "loss": 0.57908434, + "num_input_tokens_seen": 7609175, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.12890625, + "step": 265, + "time_per_iteration": 2.819581985473633 + }, + { + "auxiliary_loss_clip": 0.01316418, + "auxiliary_loss_mlp": 0.0109432, + "balance_loss_clip": 1.09573114, + "balance_loss_mlp": 1.04778111, + "epoch": 0.007718646625268412, + "flos": 21575136441600.0, + "grad_norm": 2.960718442353231, + "language_loss": 0.96319401, + "learning_rate": 3.2176017452250547e-06, + "loss": 0.98730135, + "num_input_tokens_seen": 7625380, + "router_z_loss_clip": 2.20703125, + "router_z_loss_mlp": 0.46557617, + "step": 266, + "time_per_iteration": 2.537832498550415 + }, + { + "auxiliary_loss_clip": 0.013282, + "auxiliary_loss_mlp": 0.01100976, + "balance_loss_clip": 1.09768534, + "balance_loss_mlp": 1.04737949, + "epoch": 0.007747664093784458, + "flos": 21826650942720.0, + "grad_norm": 3.221291626523253, + "language_loss": 0.98631799, + "learning_rate": 3.219764111959739e-06, + "loss": 1.01060975, + "num_input_tokens_seen": 7639375, + "router_z_loss_clip": 2.30273438, + "router_z_loss_mlp": 0.53588867, + "step": 267, + "time_per_iteration": 2.491034984588623 + }, + { + "auxiliary_loss_clip": 0.01091588, + "auxiliary_loss_mlp": 0.01012313, + "balance_loss_clip": 1.02406645, + "balance_loss_mlp": 1.00043941, + "epoch": 0.007776681562300505, + "flos": 60248915020800.0, + "grad_norm": 0.7227439545196289, + "language_loss": 0.60362428, + "learning_rate": 3.2219183950615983e-06, + "loss": 0.62466329, + "num_input_tokens_seen": 7700035, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.11865234, + "step": 268, + "time_per_iteration": 2.9810123443603516 + }, + { + "auxiliary_loss_clip": 0.01330978, + "auxiliary_loss_mlp": 0.01107161, + "balance_loss_clip": 1.09396482, + "balance_loss_mlp": 1.05227757, + "epoch": 0.007805699030816552, + "flos": 13873994876160.0, + "grad_norm": 3.0101103274218066, + "language_loss": 1.10101986, + "learning_rate": 3.2240646547441223e-06, + "loss": 1.12540126, + "num_input_tokens_seen": 7712075, + "router_z_loss_clip": 2.37304688, + "router_z_loss_mlp": 0.54931641, + "step": 269, + "time_per_iteration": 2.4383251667022705 + }, + { + "auxiliary_loss_clip": 0.01325426, + "auxiliary_loss_mlp": 0.01090348, + "balance_loss_clip": 1.10426652, + "balance_loss_mlp": 1.04244995, + "epoch": 0.007834716499332598, + "flos": 36528471999360.0, + "grad_norm": 2.985906744967804, + "language_loss": 0.8667717, + "learning_rate": 3.2262029505505177e-06, + "loss": 0.8909294, + "num_input_tokens_seen": 7725860, + "router_z_loss_clip": 2.2109375, + "router_z_loss_mlp": 0.47900391, + "step": 270, + "time_per_iteration": 2.677576780319214 + }, + { + "auxiliary_loss_clip": 0.01090048, + "auxiliary_loss_mlp": 0.0101413, + "balance_loss_clip": 1.0227406, + "balance_loss_mlp": 1.0016371, + "epoch": 0.007863733967848644, + "flos": 68385004704000.0, + "grad_norm": 0.7344881670645651, + "language_loss": 0.55378938, + "learning_rate": 3.2283333413636183e-06, + "loss": 0.57483113, + "num_input_tokens_seen": 7790930, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.12451172, + "step": 271, + "time_per_iteration": 3.265427350997925 + }, + { + "auxiliary_loss_clip": 0.01306057, + "auxiliary_loss_mlp": 0.01087128, + "balance_loss_clip": 1.08866453, + "balance_loss_mlp": 1.03746569, + "epoch": 0.00789275143636469, + "flos": 11869770700800.0, + "grad_norm": 2.6836366537279943, + "language_loss": 0.95019901, + "learning_rate": 3.230455885415618e-06, + "loss": 0.97413081, + "num_input_tokens_seen": 7803035, + "router_z_loss_clip": 2.17773438, + "router_z_loss_mlp": 0.49707031, + "step": 272, + "time_per_iteration": 2.429546594619751 + }, + { + "auxiliary_loss_clip": 0.01093864, + "auxiliary_loss_mlp": 0.01015736, + "balance_loss_clip": 1.02469277, + "balance_loss_mlp": 1.00371945, + "epoch": 0.007921768904880739, + "flos": 74779556348160.0, + "grad_norm": 0.6683687513528255, + "language_loss": 0.5675236, + "learning_rate": 3.232570640297618e-06, + "loss": 0.58861959, + "num_input_tokens_seen": 7872035, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.12011719, + "step": 273, + "time_per_iteration": 3.1488003730773926 + }, + { + "auxiliary_loss_clip": 0.01339861, + "auxiliary_loss_mlp": 0.01101219, + "balance_loss_clip": 1.09894907, + "balance_loss_mlp": 1.04848039, + "epoch": 0.007950786373396785, + "flos": 74730575710080.0, + "grad_norm": 2.420275658389808, + "language_loss": 0.94302869, + "learning_rate": 3.2346776629690067e-06, + "loss": 0.96743947, + "num_input_tokens_seen": 7898235, + "router_z_loss_clip": 2.40820312, + "router_z_loss_mlp": 0.52709961, + "step": 274, + "time_per_iteration": 2.876708507537842 + }, + { + "auxiliary_loss_clip": 0.01325209, + "auxiliary_loss_mlp": 0.01097162, + "balance_loss_clip": 1.09567308, + "balance_loss_mlp": 1.04845309, + "epoch": 0.007979803841912832, + "flos": 19970214019200.0, + "grad_norm": 2.9152007024323665, + "language_loss": 0.92621332, + "learning_rate": 3.236777009766659e-06, + "loss": 0.95043713, + "num_input_tokens_seen": 7913715, + "router_z_loss_clip": 2.29296875, + "router_z_loss_mlp": 0.48706055, + "step": 275, + "time_per_iteration": 2.516251802444458 + }, + { + "auxiliary_loss_clip": 0.01328536, + "auxiliary_loss_mlp": 0.01088918, + "balance_loss_clip": 1.099576, + "balance_loss_mlp": 1.03915977, + "epoch": 0.008008821310428878, + "flos": 35880413742720.0, + "grad_norm": 22.811510103205546, + "language_loss": 0.99745989, + "learning_rate": 3.2388687364139807e-06, + "loss": 1.02163434, + "num_input_tokens_seen": 7933165, + "router_z_loss_clip": 2.28710938, + "router_z_loss_mlp": 0.49755859, + "step": 276, + "time_per_iteration": 2.6472787857055664 + }, + { + "auxiliary_loss_clip": 0.01094029, + "auxiliary_loss_mlp": 0.01013071, + "balance_loss_clip": 1.02311325, + "balance_loss_mlp": 1.00048292, + "epoch": 0.008037838778944924, + "flos": 58201329070080.0, + "grad_norm": 0.6913321242770896, + "language_loss": 0.54687995, + "learning_rate": 3.2409528980297825e-06, + "loss": 0.56795096, + "num_input_tokens_seen": 7991740, + "router_z_loss_clip": 0.7109375, + "router_z_loss_mlp": 0.12597656, + "step": 277, + "time_per_iteration": 2.945726156234741 + }, + { + "auxiliary_loss_clip": 0.01095474, + "auxiliary_loss_mlp": 0.0101247, + "balance_loss_clip": 1.02415037, + "balance_loss_mlp": 0.99959511, + "epoch": 0.00806685624746097, + "flos": 70136527392000.0, + "grad_norm": 0.6651285197907466, + "language_loss": 0.47882327, + "learning_rate": 3.2430295491369894e-06, + "loss": 0.49990267, + "num_input_tokens_seen": 8051905, + "router_z_loss_clip": 0.7109375, + "router_z_loss_mlp": 0.12890625, + "step": 278, + "time_per_iteration": 3.005002737045288 + }, + { + "auxiliary_loss_clip": 0.01316892, + "auxiliary_loss_mlp": 0.010938, + "balance_loss_clip": 1.09365201, + "balance_loss_mlp": 1.04528189, + "epoch": 0.008095873715977019, + "flos": 37922204407680.0, + "grad_norm": 2.2005653655424733, + "language_loss": 0.94585115, + "learning_rate": 3.245098743671207e-06, + "loss": 0.96995801, + "num_input_tokens_seen": 8068575, + "router_z_loss_clip": 2.234375, + "router_z_loss_mlp": 0.48535156, + "step": 279, + "time_per_iteration": 2.607048511505127 + }, + { + "auxiliary_loss_clip": 0.01319713, + "auxiliary_loss_mlp": 0.01089463, + "balance_loss_clip": 1.09147596, + "balance_loss_mlp": 1.03901362, + "epoch": 0.008124891184493065, + "flos": 25922416337280.0, + "grad_norm": 4.553133543872547, + "language_loss": 1.03179979, + "learning_rate": 3.2471605349891217e-06, + "loss": 1.05589151, + "num_input_tokens_seen": 8082865, + "router_z_loss_clip": 2.27929688, + "router_z_loss_mlp": 0.50439453, + "step": 280, + "time_per_iteration": 2.42358660697937 + }, + { + "auxiliary_loss_clip": 0.01311727, + "auxiliary_loss_mlp": 0.01096772, + "balance_loss_clip": 1.09327936, + "balance_loss_mlp": 1.04834938, + "epoch": 0.008153908653009112, + "flos": 24893386657920.0, + "grad_norm": 2.527356671340075, + "language_loss": 1.04080892, + "learning_rate": 3.249214975876758e-06, + "loss": 1.06489396, + "num_input_tokens_seen": 8099145, + "router_z_loss_clip": 2.18164062, + "router_z_loss_mlp": 0.48339844, + "step": 281, + "time_per_iteration": 2.520216941833496 + }, + { + "auxiliary_loss_clip": 0.0130832, + "auxiliary_loss_mlp": 0.01086129, + "balance_loss_clip": 1.09165263, + "balance_loss_mlp": 1.03804004, + "epoch": 0.008182926121525158, + "flos": 21572308621440.0, + "grad_norm": 2.673538484475667, + "language_loss": 1.21206427, + "learning_rate": 3.2512621185575862e-06, + "loss": 1.23600876, + "num_input_tokens_seen": 8113860, + "router_z_loss_clip": 2.16796875, + "router_z_loss_mlp": 0.48071289, + "step": 282, + "time_per_iteration": 2.475067377090454 + }, + { + "auxiliary_loss_clip": 0.01320568, + "auxiliary_loss_mlp": 0.01083907, + "balance_loss_clip": 1.09538913, + "balance_loss_mlp": 1.0337435, + "epoch": 0.008211943590041204, + "flos": 11902135397760.0, + "grad_norm": 4.982760815336104, + "language_loss": 1.02269995, + "learning_rate": 3.25330201470049e-06, + "loss": 1.04674459, + "num_input_tokens_seen": 8125035, + "router_z_loss_clip": 2.25390625, + "router_z_loss_mlp": 0.50146484, + "step": 283, + "time_per_iteration": 2.499622106552124 + }, + { + "auxiliary_loss_clip": 0.01097452, + "auxiliary_loss_mlp": 0.01013948, + "balance_loss_clip": 1.02375531, + "balance_loss_mlp": 1.00107348, + "epoch": 0.00824096105855725, + "flos": 72651984739200.0, + "grad_norm": 0.7865571737196375, + "language_loss": 0.60157746, + "learning_rate": 3.2553347154275897e-06, + "loss": 0.62269145, + "num_input_tokens_seen": 8186125, + "router_z_loss_clip": 0.73828125, + "router_z_loss_mlp": 0.12890625, + "step": 284, + "time_per_iteration": 3.042581081390381 + }, + { + "auxiliary_loss_clip": 0.01322793, + "auxiliary_loss_mlp": 0.01095429, + "balance_loss_clip": 1.09059477, + "balance_loss_mlp": 1.04557514, + "epoch": 0.008269978527073299, + "flos": 23894101457280.0, + "grad_norm": 2.1820296952094833, + "language_loss": 0.93939304, + "learning_rate": 3.25736027132193e-06, + "loss": 0.96357524, + "num_input_tokens_seen": 8207535, + "router_z_loss_clip": 2.32421875, + "router_z_loss_mlp": 0.49829102, + "step": 285, + "time_per_iteration": 2.57147216796875 + }, + { + "auxiliary_loss_clip": 0.01305946, + "auxiliary_loss_mlp": 0.01091557, + "balance_loss_clip": 1.09535217, + "balance_loss_mlp": 1.04468441, + "epoch": 0.008298995995589345, + "flos": 40774293884160.0, + "grad_norm": 2.51363138769027, + "language_loss": 0.73077369, + "learning_rate": 3.259378732435032e-06, + "loss": 0.7547487, + "num_input_tokens_seen": 8229160, + "router_z_loss_clip": 2.10253906, + "router_z_loss_mlp": 0.46875, + "step": 286, + "time_per_iteration": 2.672858953475952 + }, + { + "auxiliary_loss_clip": 0.01317193, + "auxiliary_loss_mlp": 0.01102854, + "balance_loss_clip": 1.08877802, + "balance_loss_mlp": 1.05500364, + "epoch": 0.008328013464105392, + "flos": 41201736059520.0, + "grad_norm": 7.35269867152173, + "language_loss": 1.12460196, + "learning_rate": 3.2613901482943165e-06, + "loss": 1.14880252, + "num_input_tokens_seen": 8250280, + "router_z_loss_clip": 2.28515625, + "router_z_loss_mlp": 0.47851562, + "step": 287, + "time_per_iteration": 2.627807855606079 + }, + { + "auxiliary_loss_clip": 0.0129867, + "auxiliary_loss_mlp": 0.0108998, + "balance_loss_clip": 1.09320736, + "balance_loss_mlp": 1.04298711, + "epoch": 0.008357030932621438, + "flos": 35326307554560.0, + "grad_norm": 2.6019418230113773, + "language_loss": 0.95924628, + "learning_rate": 3.263394567910394e-06, + "loss": 0.98313284, + "num_input_tokens_seen": 8266780, + "router_z_loss_clip": 2.05566406, + "router_z_loss_mlp": 0.4699707, + "step": 288, + "time_per_iteration": 2.6925551891326904 + }, + { + "auxiliary_loss_clip": 0.01309107, + "auxiliary_loss_mlp": 0.01092618, + "balance_loss_clip": 1.09223962, + "balance_loss_mlp": 1.04438591, + "epoch": 0.008386048401137484, + "flos": 24532070330880.0, + "grad_norm": 2.5192013702880147, + "language_loss": 0.9203409, + "learning_rate": 3.2653920397842294e-06, + "loss": 0.94435817, + "num_input_tokens_seen": 8288370, + "router_z_loss_clip": 2.16992188, + "router_z_loss_mlp": 0.48217773, + "step": 289, + "time_per_iteration": 2.651501178741455 + }, + { + "auxiliary_loss_clip": 0.01082725, + "auxiliary_loss_mlp": 0.01014116, + "balance_loss_clip": 1.01667929, + "balance_loss_mlp": 1.0011462, + "epoch": 0.00841506586965353, + "flos": 62626325385600.0, + "grad_norm": 0.7408699748782072, + "language_loss": 0.5620985, + "learning_rate": 3.2673826119141857e-06, + "loss": 0.58306682, + "num_input_tokens_seen": 8352020, + "router_z_loss_clip": 0.6640625, + "router_z_loss_mlp": 0.12988281, + "step": 290, + "time_per_iteration": 3.0705480575561523 + }, + { + "auxiliary_loss_clip": 0.01313481, + "auxiliary_loss_mlp": 0.01085001, + "balance_loss_clip": 1.09390092, + "balance_loss_mlp": 1.03779423, + "epoch": 0.008444083338169579, + "flos": 17085305998080.0, + "grad_norm": 5.384016280628247, + "language_loss": 1.05338287, + "learning_rate": 3.2693663318029444e-06, + "loss": 1.07736778, + "num_input_tokens_seen": 8365070, + "router_z_loss_clip": 2.19628906, + "router_z_loss_mlp": 0.47167969, + "step": 291, + "time_per_iteration": 2.4455513954162598 + }, + { + "auxiliary_loss_clip": 0.01306162, + "auxiliary_loss_mlp": 0.0111442, + "balance_loss_clip": 1.09008312, + "balance_loss_mlp": 1.05872536, + "epoch": 0.008473100806685625, + "flos": 7703585360640.0, + "grad_norm": 6.446510307638264, + "language_loss": 1.13860488, + "learning_rate": 3.2713432464643052e-06, + "loss": 1.16281056, + "num_input_tokens_seen": 8372095, + "router_z_loss_clip": 2.15820312, + "router_z_loss_mlp": 0.55688477, + "step": 292, + "time_per_iteration": 2.4577622413635254 + }, + { + "auxiliary_loss_clip": 0.01079689, + "auxiliary_loss_mlp": 0.0101492, + "balance_loss_clip": 1.01418161, + "balance_loss_mlp": 1.00214124, + "epoch": 0.008502118275201672, + "flos": 74776519059840.0, + "grad_norm": 0.8264386128760827, + "language_loss": 0.46094981, + "learning_rate": 3.2733134024298745e-06, + "loss": 0.48189592, + "num_input_tokens_seen": 8430365, + "router_z_loss_clip": 0.65625, + "router_z_loss_mlp": 0.12792969, + "step": 293, + "time_per_iteration": 3.089611053466797 + }, + { + "auxiliary_loss_clip": 0.01292749, + "auxiliary_loss_mlp": 0.01086529, + "balance_loss_clip": 1.08292484, + "balance_loss_mlp": 1.04270756, + "epoch": 0.008531135743717718, + "flos": 17126223978240.0, + "grad_norm": 3.359073647712591, + "language_loss": 0.98850441, + "learning_rate": 3.2752768457556347e-06, + "loss": 1.01229715, + "num_input_tokens_seen": 8443410, + "router_z_loss_clip": 2.09667969, + "router_z_loss_mlp": 0.4387207, + "step": 294, + "time_per_iteration": 2.5561306476593018 + }, + { + "auxiliary_loss_clip": 0.01322517, + "auxiliary_loss_mlp": 0.01095494, + "balance_loss_clip": 1.09053326, + "balance_loss_mlp": 1.04230309, + "epoch": 0.008560153212233764, + "flos": 45542941378560.0, + "grad_norm": 3.9983136278834963, + "language_loss": 0.85806024, + "learning_rate": 3.2772336220284056e-06, + "loss": 0.88224041, + "num_input_tokens_seen": 8462000, + "router_z_loss_clip": 2.31640625, + "router_z_loss_mlp": 0.5324707, + "step": 295, + "time_per_iteration": 2.6855416297912598 + }, + { + "auxiliary_loss_clip": 0.01299898, + "auxiliary_loss_mlp": 0.01094889, + "balance_loss_clip": 1.0914073, + "balance_loss_mlp": 1.04808736, + "epoch": 0.00858917068074981, + "flos": 74729772748800.0, + "grad_norm": 3.3352647930820254, + "language_loss": 0.80347157, + "learning_rate": 3.2791837763721955e-06, + "loss": 0.8274194, + "num_input_tokens_seen": 8484775, + "router_z_loss_clip": 2.08691406, + "router_z_loss_mlp": 0.46826172, + "step": 296, + "time_per_iteration": 2.880319356918335 + }, + { + "auxiliary_loss_clip": 0.0130011, + "auxiliary_loss_mlp": 0.0109977, + "balance_loss_clip": 1.08787751, + "balance_loss_mlp": 1.05332589, + "epoch": 0.008618188149265857, + "flos": 11284137688320.0, + "grad_norm": 3.7233064174027786, + "language_loss": 0.98644406, + "learning_rate": 3.2811273534544436e-06, + "loss": 1.01044285, + "num_input_tokens_seen": 8496330, + "router_z_loss_clip": 2.11914062, + "router_z_loss_mlp": 0.46484375, + "step": 297, + "time_per_iteration": 2.4746463298797607 + }, + { + "auxiliary_loss_clip": 0.0129803, + "auxiliary_loss_mlp": 0.01092755, + "balance_loss_clip": 1.08447731, + "balance_loss_mlp": 1.04366493, + "epoch": 0.008647205617781905, + "flos": 21899793974400.0, + "grad_norm": 4.655286447896814, + "language_loss": 0.97460604, + "learning_rate": 3.2830643974921586e-06, + "loss": 0.99851388, + "num_input_tokens_seen": 8509655, + "router_z_loss_clip": 2.13476562, + "router_z_loss_mlp": 0.4909668, + "step": 298, + "time_per_iteration": 2.494732618331909 + }, + { + "auxiliary_loss_clip": 0.01315128, + "auxiliary_loss_mlp": 0.01091316, + "balance_loss_clip": 1.09122396, + "balance_loss_mlp": 1.0425837, + "epoch": 0.008676223086297952, + "flos": 21536557522560.0, + "grad_norm": 2.798375460738479, + "language_loss": 0.822918, + "learning_rate": 3.2849949522579577e-06, + "loss": 0.84698248, + "num_input_tokens_seen": 8525945, + "router_z_loss_clip": 2.23632812, + "router_z_loss_mlp": 0.48730469, + "step": 299, + "time_per_iteration": 2.439344644546509 + }, + { + "auxiliary_loss_clip": 0.01285892, + "auxiliary_loss_mlp": 0.01075025, + "balance_loss_clip": 1.08356738, + "balance_loss_mlp": 1.03089356, + "epoch": 0.008705240554813998, + "flos": 63128658026880.0, + "grad_norm": 2.579497421342463, + "language_loss": 1.00101137, + "learning_rate": 3.286919061085997e-06, + "loss": 1.02462065, + "num_input_tokens_seen": 8546645, + "router_z_loss_clip": 2.02246094, + "router_z_loss_mlp": 0.44140625, + "step": 300, + "time_per_iteration": 2.82444167137146 + }, + { + "auxiliary_loss_clip": 0.01297624, + "auxiliary_loss_mlp": 0.01077941, + "balance_loss_clip": 1.08202648, + "balance_loss_mlp": 1.03273666, + "epoch": 0.008734258023330044, + "flos": 48480497596800.0, + "grad_norm": 1.9355444855870774, + "language_loss": 0.92215866, + "learning_rate": 3.2888367668778124e-06, + "loss": 0.94591427, + "num_input_tokens_seen": 8582450, + "router_z_loss_clip": 2.15234375, + "router_z_loss_mlp": 0.45214844, + "step": 301, + "time_per_iteration": 3.0110459327697754 + }, + { + "auxiliary_loss_clip": 0.01304893, + "auxiliary_loss_mlp": 0.01085344, + "balance_loss_clip": 1.08296967, + "balance_loss_mlp": 1.03651547, + "epoch": 0.00876327549184609, + "flos": 28942718077440.0, + "grad_norm": 2.509605382456749, + "language_loss": 0.92640197, + "learning_rate": 3.2907481121080574e-06, + "loss": 0.95030433, + "num_input_tokens_seen": 8599775, + "router_z_loss_clip": 2.22265625, + "router_z_loss_mlp": 0.48852539, + "step": 302, + "time_per_iteration": 2.5535922050476074 + }, + { + "auxiliary_loss_clip": 0.01290243, + "auxiliary_loss_mlp": 0.01090898, + "balance_loss_clip": 1.08294046, + "balance_loss_mlp": 1.03830338, + "epoch": 0.008792292960362137, + "flos": 16066610081280.0, + "grad_norm": 3.327319054620999, + "language_loss": 0.96803749, + "learning_rate": 3.2926531388301455e-06, + "loss": 0.99184895, + "num_input_tokens_seen": 8612710, + "router_z_loss_clip": 2.07421875, + "router_z_loss_mlp": 0.52587891, + "step": 303, + "time_per_iteration": 2.461421489715576 + }, + { + "auxiliary_loss_clip": 0.01285766, + "auxiliary_loss_mlp": 0.01094847, + "balance_loss_clip": 1.0785526, + "balance_loss_mlp": 1.04973865, + "epoch": 0.008821310428878185, + "flos": 74729004698880.0, + "grad_norm": 2.0723990007473025, + "language_loss": 0.82880771, + "learning_rate": 3.2945518886818066e-06, + "loss": 0.85261387, + "num_input_tokens_seen": 8634650, + "router_z_loss_clip": 2.07226562, + "router_z_loss_mlp": 0.45092773, + "step": 304, + "time_per_iteration": 2.8480582237243652 + }, + { + "auxiliary_loss_clip": 0.01281054, + "auxiliary_loss_mlp": 0.0108736, + "balance_loss_clip": 1.0793767, + "balance_loss_mlp": 1.03979564, + "epoch": 0.008850327897394232, + "flos": 33211373857920.0, + "grad_norm": 2.7724194223396217, + "language_loss": 1.20742333, + "learning_rate": 3.296444402890543e-06, + "loss": 1.23110747, + "num_input_tokens_seen": 8651660, + "router_z_loss_clip": 2.01464844, + "router_z_loss_mlp": 0.47583008, + "step": 305, + "time_per_iteration": 2.6457247734069824 + }, + { + "auxiliary_loss_clip": 0.01283459, + "auxiliary_loss_mlp": 0.01074087, + "balance_loss_clip": 1.0826273, + "balance_loss_mlp": 1.02933621, + "epoch": 0.008879345365910278, + "flos": 14385612072960.0, + "grad_norm": 3.992132940180427, + "language_loss": 0.79320538, + "learning_rate": 3.298330722279005e-06, + "loss": 0.81678092, + "num_input_tokens_seen": 8665185, + "router_z_loss_clip": 2.00878906, + "router_z_loss_mlp": 0.44799805, + "step": 306, + "time_per_iteration": 2.4178829193115234 + }, + { + "auxiliary_loss_clip": 0.01288969, + "auxiliary_loss_mlp": 0.01082693, + "balance_loss_clip": 1.08130944, + "balance_loss_mlp": 1.03748906, + "epoch": 0.008908362834426324, + "flos": 15954050257920.0, + "grad_norm": 3.738656145352933, + "language_loss": 0.89234066, + "learning_rate": 3.3002108872702717e-06, + "loss": 0.91605723, + "num_input_tokens_seen": 8678150, + "router_z_loss_clip": 2.07714844, + "router_z_loss_mlp": 0.4519043, + "step": 307, + "time_per_iteration": 2.447415590286255 + }, + { + "auxiliary_loss_clip": 0.01074641, + "auxiliary_loss_mlp": 0.01019672, + "balance_loss_clip": 1.00920522, + "balance_loss_mlp": 1.00813258, + "epoch": 0.00893738030294237, + "flos": 63857049189120.0, + "grad_norm": 0.6764153224997438, + "language_loss": 0.46024936, + "learning_rate": 3.3020849378930476e-06, + "loss": 0.4811925, + "num_input_tokens_seen": 8741240, + "router_z_loss_clip": 0.65625, + "router_z_loss_mlp": 0.11523438, + "step": 308, + "time_per_iteration": 3.06259822845459 + }, + { + "auxiliary_loss_clip": 0.01074193, + "auxiliary_loss_mlp": 0.0101572, + "balance_loss_clip": 1.00877881, + "balance_loss_mlp": 1.00418067, + "epoch": 0.008966397771458417, + "flos": 69550789645440.0, + "grad_norm": 0.6935276685541363, + "language_loss": 0.55391514, + "learning_rate": 3.303952913786781e-06, + "loss": 0.57481432, + "num_input_tokens_seen": 8799210, + "router_z_loss_clip": 0.65625, + "router_z_loss_mlp": 0.11523438, + "step": 309, + "time_per_iteration": 2.980060577392578 + }, + { + "auxiliary_loss_clip": 0.01291679, + "auxiliary_loss_mlp": 0.01087764, + "balance_loss_clip": 1.08417261, + "balance_loss_mlp": 1.0411768, + "epoch": 0.008995415239974465, + "flos": 21386326475520.0, + "grad_norm": 3.8218954296452785, + "language_loss": 1.00738311, + "learning_rate": 3.305814854206687e-06, + "loss": 1.03117752, + "num_input_tokens_seen": 8811205, + "router_z_loss_clip": 2.07519531, + "router_z_loss_mlp": 0.46582031, + "step": 310, + "time_per_iteration": 2.5260398387908936 + }, + { + "auxiliary_loss_clip": 0.01289933, + "auxiliary_loss_mlp": 0.01080825, + "balance_loss_clip": 1.07619894, + "balance_loss_mlp": 1.03349936, + "epoch": 0.009024432708490512, + "flos": 19894033699200.0, + "grad_norm": 3.1962972271296066, + "language_loss": 0.94486171, + "learning_rate": 3.307670798028707e-06, + "loss": 0.96856934, + "num_input_tokens_seen": 8824250, + "router_z_loss_clip": 2.13574219, + "router_z_loss_mlp": 0.47314453, + "step": 311, + "time_per_iteration": 5.045663595199585 + }, + { + "auxiliary_loss_clip": 0.01290593, + "auxiliary_loss_mlp": 0.01099719, + "balance_loss_clip": 1.08055937, + "balance_loss_mlp": 1.05401373, + "epoch": 0.009053450177006558, + "flos": 35925032327040.0, + "grad_norm": 2.7893837527846297, + "language_loss": 1.01497269, + "learning_rate": 3.30952078375437e-06, + "loss": 1.03887582, + "num_input_tokens_seen": 8842500, + "router_z_loss_clip": 2.10058594, + "router_z_loss_mlp": 0.45703125, + "step": 312, + "time_per_iteration": 7.580075979232788 + }, + { + "auxiliary_loss_clip": 0.01290532, + "auxiliary_loss_mlp": 0.01094789, + "balance_loss_clip": 1.08879447, + "balance_loss_mlp": 1.04767752, + "epoch": 0.009082467645522604, + "flos": 17596120233600.0, + "grad_norm": 2.2145694461966703, + "language_loss": 0.94722533, + "learning_rate": 3.3113648495155915e-06, + "loss": 0.97107852, + "num_input_tokens_seen": 8858640, + "router_z_loss_clip": 2.01757812, + "router_z_loss_mlp": 0.47094727, + "step": 313, + "time_per_iteration": 4.842338562011719 + }, + { + "auxiliary_loss_clip": 0.01077013, + "auxiliary_loss_mlp": 0.01023375, + "balance_loss_clip": 1.01340103, + "balance_loss_mlp": 1.01212156, + "epoch": 0.00911148511403865, + "flos": 74770968153600.0, + "grad_norm": 0.737231347500376, + "language_loss": 0.54690826, + "learning_rate": 3.3132030330793862e-06, + "loss": 0.56791216, + "num_input_tokens_seen": 8920080, + "router_z_loss_clip": 0.63671875, + "router_z_loss_mlp": 0.11230469, + "step": 314, + "time_per_iteration": 3.0382049083709717 + }, + { + "auxiliary_loss_clip": 0.0107418, + "auxiliary_loss_mlp": 0.01014804, + "balance_loss_clip": 1.01174092, + "balance_loss_mlp": 1.00345564, + "epoch": 0.009140502582554697, + "flos": 65288697200640.0, + "grad_norm": 1.0188463291985512, + "language_loss": 0.53804517, + "learning_rate": 3.3150353718525096e-06, + "loss": 0.55893493, + "num_input_tokens_seen": 8979035, + "router_z_loss_clip": 0.625, + "router_z_loss_mlp": 0.11328125, + "step": 315, + "time_per_iteration": 2.986386775970459 + }, + { + "auxiliary_loss_clip": 0.0129232, + "auxiliary_loss_mlp": 0.01082057, + "balance_loss_clip": 1.07855463, + "balance_loss_mlp": 1.03268051, + "epoch": 0.009169520051070745, + "flos": 16389696602880.0, + "grad_norm": 2.821486130147911, + "language_loss": 0.87147593, + "learning_rate": 3.3168619028860257e-06, + "loss": 0.89521968, + "num_input_tokens_seen": 8992800, + "router_z_loss_clip": 2.14160156, + "router_z_loss_mlp": 0.49389648, + "step": 316, + "time_per_iteration": 2.489290237426758 + }, + { + "auxiliary_loss_clip": 0.01271871, + "auxiliary_loss_mlp": 0.01075779, + "balance_loss_clip": 1.07845235, + "balance_loss_mlp": 1.03365088, + "epoch": 0.009198537519586792, + "flos": 15624190932480.0, + "grad_norm": 5.6445653965441815, + "language_loss": 1.17044473, + "learning_rate": 3.3186826628798026e-06, + "loss": 1.19392121, + "num_input_tokens_seen": 9004985, + "router_z_loss_clip": 1.93457031, + "router_z_loss_mlp": 0.42138672, + "step": 317, + "time_per_iteration": 2.448166847229004 + }, + { + "auxiliary_loss_clip": 0.01281993, + "auxiliary_loss_mlp": 0.0107668, + "balance_loss_clip": 1.07545888, + "balance_loss_mlp": 1.03109455, + "epoch": 0.009227554988102838, + "flos": 29744568339840.0, + "grad_norm": 3.687828714591867, + "language_loss": 1.19036937, + "learning_rate": 3.3204976881869384e-06, + "loss": 1.21395612, + "num_input_tokens_seen": 9020355, + "router_z_loss_clip": 2.06640625, + "router_z_loss_mlp": 0.45629883, + "step": 318, + "time_per_iteration": 2.53352952003479 + }, + { + "auxiliary_loss_clip": 0.01297281, + "auxiliary_loss_mlp": 0.01092886, + "balance_loss_clip": 1.07756352, + "balance_loss_mlp": 1.04248452, + "epoch": 0.009256572456618884, + "flos": 28577945525760.0, + "grad_norm": 5.505538261814145, + "language_loss": 0.92432624, + "learning_rate": 3.3223070148181116e-06, + "loss": 0.94822794, + "num_input_tokens_seen": 9037890, + "router_z_loss_clip": 2.19726562, + "router_z_loss_mlp": 0.50415039, + "step": 319, + "time_per_iteration": 2.528045654296875 + }, + { + "auxiliary_loss_clip": 0.01262192, + "auxiliary_loss_mlp": 0.01084561, + "balance_loss_clip": 1.06978309, + "balance_loss_mlp": 1.04269516, + "epoch": 0.00928558992513493, + "flos": 21026127312000.0, + "grad_norm": 2.6672804607858533, + "language_loss": 0.94803613, + "learning_rate": 3.3241106784458735e-06, + "loss": 0.97150367, + "num_input_tokens_seen": 9052035, + "router_z_loss_clip": 1.92480469, + "router_z_loss_mlp": 0.41870117, + "step": 320, + "time_per_iteration": 2.516500473022461 + }, + { + "auxiliary_loss_clip": 0.01265767, + "auxiliary_loss_mlp": 0.01072723, + "balance_loss_clip": 1.07142305, + "balance_loss_mlp": 1.03142941, + "epoch": 0.009314607393650977, + "flos": 17083665164160.0, + "grad_norm": 3.379028882651893, + "language_loss": 0.88319266, + "learning_rate": 3.3259087144088656e-06, + "loss": 0.90657759, + "num_input_tokens_seen": 9064630, + "router_z_loss_clip": 1.94335938, + "router_z_loss_mlp": 0.41308594, + "step": 321, + "time_per_iteration": 2.385010004043579 + }, + { + "auxiliary_loss_clip": 0.01074599, + "auxiliary_loss_mlp": 0.01037492, + "balance_loss_clip": 1.01446939, + "balance_loss_mlp": 1.02633393, + "epoch": 0.009343624862167025, + "flos": 62034862176000.0, + "grad_norm": 0.7092089020461779, + "language_loss": 0.57601666, + "learning_rate": 3.327701157715974e-06, + "loss": 0.59713757, + "num_input_tokens_seen": 9122435, + "router_z_loss_clip": 0.6015625, + "router_z_loss_mlp": 0.11181641, + "step": 322, + "time_per_iteration": 2.902329206466675 + }, + { + "auxiliary_loss_clip": 0.01280215, + "auxiliary_loss_mlp": 0.01099373, + "balance_loss_clip": 1.0833745, + "balance_loss_mlp": 1.05409729, + "epoch": 0.009372642330683072, + "flos": 32881130507520.0, + "grad_norm": 3.600494976978975, + "language_loss": 0.82931983, + "learning_rate": 3.329488043050418e-06, + "loss": 0.85311568, + "num_input_tokens_seen": 9141455, + "router_z_loss_clip": 1.96972656, + "router_z_loss_mlp": 0.453125, + "step": 323, + "time_per_iteration": 2.5727617740631104 + }, + { + "auxiliary_loss_clip": 0.01288398, + "auxiliary_loss_mlp": 0.01096308, + "balance_loss_clip": 1.07999659, + "balance_loss_mlp": 1.04814756, + "epoch": 0.009401659799199118, + "flos": 31352458227840.0, + "grad_norm": 3.9480827936027785, + "language_loss": 1.17932868, + "learning_rate": 3.3312694047737813e-06, + "loss": 1.20317578, + "num_input_tokens_seen": 9157615, + "router_z_loss_clip": 2.08496094, + "router_z_loss_mlp": 0.48144531, + "step": 324, + "time_per_iteration": 2.569260358810425 + }, + { + "auxiliary_loss_clip": 0.01283557, + "auxiliary_loss_mlp": 0.01096452, + "balance_loss_clip": 1.07990646, + "balance_loss_mlp": 1.05019891, + "epoch": 0.009430677267715164, + "flos": 13726836028800.0, + "grad_norm": 2.479361691672529, + "language_loss": 0.95137292, + "learning_rate": 3.333045276929973e-06, + "loss": 0.975173, + "num_input_tokens_seen": 9169345, + "router_z_loss_clip": 2.03808594, + "router_z_loss_mlp": 0.46264648, + "step": 325, + "time_per_iteration": 2.4673335552215576 + }, + { + "auxiliary_loss_clip": 0.01288604, + "auxiliary_loss_mlp": 0.01101202, + "balance_loss_clip": 1.07899737, + "balance_loss_mlp": 1.0558306, + "epoch": 0.00945969473623121, + "flos": 34857179349120.0, + "grad_norm": 2.1522444532817953, + "language_loss": 1.09006667, + "learning_rate": 3.334815693249133e-06, + "loss": 1.1139648, + "num_input_tokens_seen": 9194125, + "router_z_loss_clip": 2.09570312, + "router_z_loss_mlp": 0.45336914, + "step": 326, + "time_per_iteration": 2.657621145248413 + }, + { + "auxiliary_loss_clip": 0.01062406, + "auxiliary_loss_mlp": 0.01015001, + "balance_loss_clip": 1.00544381, + "balance_loss_mlp": 1.00408137, + "epoch": 0.009488712204747257, + "flos": 62941137914880.0, + "grad_norm": 0.6982523251042201, + "language_loss": 0.52276117, + "learning_rate": 3.3365806871514735e-06, + "loss": 0.54353523, + "num_input_tokens_seen": 9249005, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.109375, + "step": 327, + "time_per_iteration": 2.8451037406921387 + }, + { + "auxiliary_loss_clip": 0.01277833, + "auxiliary_loss_mlp": 0.01084782, + "balance_loss_clip": 1.07610118, + "balance_loss_mlp": 1.0367887, + "epoch": 0.009517729673263305, + "flos": 32479175491200.0, + "grad_norm": 3.194503218012911, + "language_loss": 0.96320796, + "learning_rate": 3.3383402917510684e-06, + "loss": 0.98683405, + "num_input_tokens_seen": 9263710, + "router_z_loss_clip": 2.015625, + "router_z_loss_mlp": 0.48022461, + "step": 328, + "time_per_iteration": 2.5169742107391357 + }, + { + "auxiliary_loss_clip": 0.01060709, + "auxiliary_loss_mlp": 0.01016224, + "balance_loss_clip": 1.00387728, + "balance_loss_mlp": 1.00549531, + "epoch": 0.009546747141779352, + "flos": 70613685210240.0, + "grad_norm": 0.7045042671311939, + "language_loss": 0.56910038, + "learning_rate": 3.340094539859579e-06, + "loss": 0.58986974, + "num_input_tokens_seen": 9328025, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.10742188, + "step": 329, + "time_per_iteration": 3.0697202682495117 + }, + { + "auxiliary_loss_clip": 0.01059944, + "auxiliary_loss_mlp": 0.01012749, + "balance_loss_clip": 1.00340509, + "balance_loss_mlp": 1.00240147, + "epoch": 0.009575764610295398, + "flos": 69595582786560.0, + "grad_norm": 0.7413353327678388, + "language_loss": 0.58828723, + "learning_rate": 3.3418434639899233e-06, + "loss": 0.60901415, + "num_input_tokens_seen": 9393160, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.10351562, + "step": 330, + "time_per_iteration": 3.0630013942718506 + }, + { + "auxiliary_loss_clip": 0.01059919, + "auxiliary_loss_mlp": 0.01012072, + "balance_loss_clip": 1.0033586, + "balance_loss_mlp": 1.0019151, + "epoch": 0.009604782078811444, + "flos": 74774843314560.0, + "grad_norm": 0.8702579728299477, + "language_loss": 0.57496893, + "learning_rate": 3.3435870963598952e-06, + "loss": 0.59568894, + "num_input_tokens_seen": 9459550, + "router_z_loss_clip": 0.56640625, + "router_z_loss_mlp": 0.1015625, + "step": 331, + "time_per_iteration": 3.064107656478882 + }, + { + "auxiliary_loss_clip": 0.01294546, + "auxiliary_loss_mlp": 0.01090274, + "balance_loss_clip": 1.07455575, + "balance_loss_mlp": 1.04082632, + "epoch": 0.00963379954732749, + "flos": 26535875569920.0, + "grad_norm": 2.91063038671211, + "language_loss": 0.86692566, + "learning_rate": 3.3453254688957247e-06, + "loss": 0.89077383, + "num_input_tokens_seen": 9475915, + "router_z_loss_clip": 2.19824219, + "router_z_loss_mlp": 0.49438477, + "step": 332, + "time_per_iteration": 2.5041372776031494 + }, + { + "auxiliary_loss_clip": 0.01276926, + "auxiliary_loss_mlp": 0.01088692, + "balance_loss_clip": 1.07768452, + "balance_loss_mlp": 1.04930532, + "epoch": 0.009662817015843537, + "flos": 15229392744960.0, + "grad_norm": 2.5141564701385986, + "language_loss": 0.82568014, + "learning_rate": 3.347058613235583e-06, + "loss": 0.84933627, + "num_input_tokens_seen": 9491815, + "router_z_loss_clip": 1.9921875, + "router_z_loss_mlp": 0.39404297, + "step": 333, + "time_per_iteration": 2.455409526824951 + }, + { + "auxiliary_loss_clip": 0.01260965, + "auxiliary_loss_mlp": 0.01075078, + "balance_loss_clip": 1.06909966, + "balance_loss_mlp": 1.03540552, + "epoch": 0.009691834484359585, + "flos": 38610656974080.0, + "grad_norm": 2.7095384049651132, + "language_loss": 1.0190419, + "learning_rate": 3.34878656073304e-06, + "loss": 1.04240227, + "num_input_tokens_seen": 9509620, + "router_z_loss_clip": 1.91699219, + "router_z_loss_mlp": 0.39672852, + "step": 334, + "time_per_iteration": 2.614246129989624 + }, + { + "auxiliary_loss_clip": 0.01060344, + "auxiliary_loss_mlp": 0.01017331, + "balance_loss_clip": 1.00270236, + "balance_loss_mlp": 1.00774622, + "epoch": 0.009720851952875632, + "flos": 74780184752640.0, + "grad_norm": 0.752479346561659, + "language_loss": 0.54144502, + "learning_rate": 3.350509342460466e-06, + "loss": 0.56222177, + "num_input_tokens_seen": 9573665, + "router_z_loss_clip": 0.578125, + "router_z_loss_mlp": 0.09570312, + "step": 335, + "time_per_iteration": 3.1358025074005127 + }, + { + "auxiliary_loss_clip": 0.01059422, + "auxiliary_loss_mlp": 0.01021597, + "balance_loss_clip": 1.00308371, + "balance_loss_mlp": 1.01220345, + "epoch": 0.009749869421391678, + "flos": 60832418440320.0, + "grad_norm": 0.7020544901833018, + "language_loss": 0.53209329, + "learning_rate": 3.3522269892123866e-06, + "loss": 0.55290353, + "num_input_tokens_seen": 9635490, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.09375, + "step": 336, + "time_per_iteration": 2.9490883350372314 + }, + { + "auxiliary_loss_clip": 0.01270672, + "auxiliary_loss_mlp": 0.01076565, + "balance_loss_clip": 1.07712412, + "balance_loss_mlp": 1.03412628, + "epoch": 0.009778886889907724, + "flos": 15807868928640.0, + "grad_norm": 2.8962905969733406, + "language_loss": 0.96347594, + "learning_rate": 3.3539395315087827e-06, + "loss": 0.98694831, + "num_input_tokens_seen": 9649630, + "router_z_loss_clip": 1.93457031, + "router_z_loss_mlp": 0.42431641, + "step": 337, + "time_per_iteration": 2.5009167194366455 + }, + { + "auxiliary_loss_clip": 0.01276387, + "auxiliary_loss_mlp": 0.0109916, + "balance_loss_clip": 1.07361174, + "balance_loss_mlp": 1.05271578, + "epoch": 0.00980790435842377, + "flos": 25073992454400.0, + "grad_norm": 3.3522316544604585, + "language_loss": 0.93857312, + "learning_rate": 3.3556469995983466e-06, + "loss": 0.96232855, + "num_input_tokens_seen": 9662900, + "router_z_loss_clip": 2.02929688, + "router_z_loss_mlp": 0.46459961, + "step": 338, + "time_per_iteration": 2.545910120010376 + }, + { + "auxiliary_loss_clip": 0.01296651, + "auxiliary_loss_mlp": 0.0110384, + "balance_loss_clip": 1.07771754, + "balance_loss_mlp": 1.04895639, + "epoch": 0.009836921826939817, + "flos": 13034717769600.0, + "grad_norm": 3.22172551149963, + "language_loss": 1.05499089, + "learning_rate": 3.357349423461686e-06, + "loss": 1.0789957, + "num_input_tokens_seen": 9674755, + "router_z_loss_clip": 2.19335938, + "router_z_loss_mlp": 0.54882812, + "step": 339, + "time_per_iteration": 2.488532304763794 + }, + { + "auxiliary_loss_clip": 0.01285609, + "auxiliary_loss_mlp": 0.01093757, + "balance_loss_clip": 1.07562399, + "balance_loss_mlp": 1.04581118, + "epoch": 0.009865939295455865, + "flos": 31059990835200.0, + "grad_norm": 4.550114539569016, + "language_loss": 1.07451963, + "learning_rate": 3.3590468328144853e-06, + "loss": 1.09831333, + "num_input_tokens_seen": 9691705, + "router_z_loss_clip": 2.09960938, + "router_z_loss_mlp": 0.47924805, + "step": 340, + "time_per_iteration": 2.5524799823760986 + }, + { + "auxiliary_loss_clip": 0.01277659, + "auxiliary_loss_mlp": 0.01075665, + "balance_loss_clip": 1.07980573, + "balance_loss_mlp": 1.02903032, + "epoch": 0.009894956763971912, + "flos": 16498381265280.0, + "grad_norm": 4.406491376740628, + "language_loss": 0.83119214, + "learning_rate": 3.360739257110613e-06, + "loss": 0.85472536, + "num_input_tokens_seen": 9702375, + "router_z_loss_clip": 1.97949219, + "router_z_loss_mlp": 0.46704102, + "step": 341, + "time_per_iteration": 2.4862427711486816 + }, + { + "auxiliary_loss_clip": 0.01281037, + "auxiliary_loss_mlp": 0.01092794, + "balance_loss_clip": 1.07565546, + "balance_loss_mlp": 1.04346538, + "epoch": 0.009923974232487958, + "flos": 28869714691200.0, + "grad_norm": 2.8165674876457913, + "language_loss": 0.91342485, + "learning_rate": 3.3624267255451937e-06, + "loss": 0.93716317, + "num_input_tokens_seen": 9718690, + "router_z_loss_clip": 2.05371094, + "router_z_loss_mlp": 0.49365234, + "step": 342, + "time_per_iteration": 2.513627767562866 + }, + { + "auxiliary_loss_clip": 0.01273792, + "auxiliary_loss_mlp": 0.0109619, + "balance_loss_clip": 1.07587254, + "balance_loss_mlp": 1.05122459, + "epoch": 0.009952991701004004, + "flos": 74052282349440.0, + "grad_norm": 6.891869118713088, + "language_loss": 1.11290562, + "learning_rate": 3.3641092670576266e-06, + "loss": 1.1366055, + "num_input_tokens_seen": 9751970, + "router_z_loss_clip": 1.98046875, + "router_z_loss_mlp": 0.44970703, + "step": 343, + "time_per_iteration": 2.9759068489074707 + }, + { + "auxiliary_loss_clip": 0.01283888, + "auxiliary_loss_mlp": 0.01096849, + "balance_loss_clip": 1.07790637, + "balance_loss_mlp": 1.04701924, + "epoch": 0.00998200916952005, + "flos": 29874374375040.0, + "grad_norm": 1.9697568559704963, + "language_loss": 0.84507048, + "learning_rate": 3.3657869103345642e-06, + "loss": 0.86887783, + "num_input_tokens_seen": 9775060, + "router_z_loss_clip": 2.05957031, + "router_z_loss_mlp": 0.49829102, + "step": 344, + "time_per_iteration": 2.597223997116089 + }, + { + "auxiliary_loss_clip": 0.01070083, + "auxiliary_loss_mlp": 0.01010805, + "balance_loss_clip": 1.00627065, + "balance_loss_mlp": 1.00093412, + "epoch": 0.010011026638036097, + "flos": 64843066206720.0, + "grad_norm": 0.7707332271968923, + "language_loss": 0.5522325, + "learning_rate": 3.3674596838128487e-06, + "loss": 0.57304132, + "num_input_tokens_seen": 9829910, + "router_z_loss_clip": 0.640625, + "router_z_loss_mlp": 0.09863281, + "step": 345, + "time_per_iteration": 2.9073615074157715 + }, + { + "auxiliary_loss_clip": 0.01274014, + "auxiliary_loss_mlp": 0.01102623, + "balance_loss_clip": 1.0745852, + "balance_loss_mlp": 1.05656016, + "epoch": 0.010040044106552144, + "flos": 12670224508800.0, + "grad_norm": 3.189195092986946, + "language_loss": 0.93157572, + "learning_rate": 3.3691276156823998e-06, + "loss": 0.95534211, + "num_input_tokens_seen": 9841350, + "router_z_loss_clip": 1.99316406, + "router_z_loss_mlp": 0.4609375, + "step": 346, + "time_per_iteration": 2.5042996406555176 + }, + { + "auxiliary_loss_clip": 0.01270112, + "auxiliary_loss_mlp": 0.01092974, + "balance_loss_clip": 1.0724268, + "balance_loss_mlp": 1.05065441, + "epoch": 0.010069061575068192, + "flos": 11793520558080.0, + "grad_norm": 3.382270492967928, + "language_loss": 0.8911351, + "learning_rate": 3.3707907338890692e-06, + "loss": 0.91476601, + "num_input_tokens_seen": 9853295, + "router_z_loss_clip": 1.9765625, + "router_z_loss_mlp": 0.4230957, + "step": 347, + "time_per_iteration": 2.4431304931640625 + }, + { + "auxiliary_loss_clip": 0.01273258, + "auxiliary_loss_mlp": 0.010746, + "balance_loss_clip": 1.07602763, + "balance_loss_mlp": 1.02739346, + "epoch": 0.010098079043584238, + "flos": 25988193072000.0, + "grad_norm": 2.9104279404302162, + "language_loss": 0.84435934, + "learning_rate": 3.37244906613745e-06, + "loss": 0.86783791, + "num_input_tokens_seen": 9867620, + "router_z_loss_clip": 1.97167969, + "router_z_loss_mlp": 0.47192383, + "step": 348, + "time_per_iteration": 2.520141363143921 + }, + { + "auxiliary_loss_clip": 0.01069812, + "auxiliary_loss_mlp": 0.01011066, + "balance_loss_clip": 1.00649786, + "balance_loss_mlp": 1.00148118, + "epoch": 0.010127096512100284, + "flos": 64151541440640.0, + "grad_norm": 0.7675448387913206, + "language_loss": 0.56529921, + "learning_rate": 3.3741026398936434e-06, + "loss": 0.58610803, + "num_input_tokens_seen": 9925330, + "router_z_loss_clip": 0.6328125, + "router_z_loss_mlp": 0.09570312, + "step": 349, + "time_per_iteration": 2.904085874557495 + }, + { + "auxiliary_loss_clip": 0.01272874, + "auxiliary_loss_mlp": 0.01088748, + "balance_loss_clip": 1.0746398, + "balance_loss_mlp": 1.04511738, + "epoch": 0.01015611398061633, + "flos": 74733822466560.0, + "grad_norm": 3.7650238588439535, + "language_loss": 0.88009524, + "learning_rate": 3.3757514823879893e-06, + "loss": 0.9037115, + "num_input_tokens_seen": 9955195, + "router_z_loss_clip": 1.98046875, + "router_z_loss_mlp": 0.4362793, + "step": 350, + "time_per_iteration": 2.868281602859497 + }, + { + "auxiliary_loss_clip": 0.01267867, + "auxiliary_loss_mlp": 0.01075415, + "balance_loss_clip": 1.07048535, + "balance_loss_mlp": 1.03266668, + "epoch": 0.010185131449132377, + "flos": 49958616360960.0, + "grad_norm": 3.9982706465949702, + "language_loss": 0.81365639, + "learning_rate": 3.3773956206177575e-06, + "loss": 0.83708924, + "num_input_tokens_seen": 9974750, + "router_z_loss_clip": 1.97460938, + "router_z_loss_mlp": 0.42773438, + "step": 351, + "time_per_iteration": 2.6899232864379883 + }, + { + "auxiliary_loss_clip": 0.01268022, + "auxiliary_loss_mlp": 0.01091524, + "balance_loss_clip": 1.06789374, + "balance_loss_mlp": 1.04620028, + "epoch": 0.010214148917648424, + "flos": 43900522289280.0, + "grad_norm": 3.8338603260718265, + "language_loss": 1.24021864, + "learning_rate": 3.3790350813497995e-06, + "loss": 1.26381397, + "num_input_tokens_seen": 9995975, + "router_z_loss_clip": 2.00195312, + "router_z_loss_mlp": 0.453125, + "step": 352, + "time_per_iteration": 2.719179153442383 + }, + { + "auxiliary_loss_clip": 0.01262414, + "auxiliary_loss_mlp": 0.01074942, + "balance_loss_clip": 1.07272959, + "balance_loss_mlp": 1.03262329, + "epoch": 0.010243166386164472, + "flos": 28686874567680.0, + "grad_norm": 2.237639134995841, + "language_loss": 0.89064288, + "learning_rate": 3.380669891123163e-06, + "loss": 0.91401649, + "num_input_tokens_seen": 10011615, + "router_z_loss_clip": 1.89550781, + "router_z_loss_mlp": 0.4230957, + "step": 353, + "time_per_iteration": 2.52133846282959 + }, + { + "auxiliary_loss_clip": 0.01256209, + "auxiliary_loss_mlp": 0.01072346, + "balance_loss_clip": 1.06898785, + "balance_loss_mlp": 1.0323875, + "epoch": 0.010272183854680518, + "flos": 25988018515200.0, + "grad_norm": 2.8078957159580376, + "language_loss": 0.9834159, + "learning_rate": 3.3823000762516696e-06, + "loss": 1.00670147, + "num_input_tokens_seen": 10027920, + "router_z_loss_clip": 1.87207031, + "router_z_loss_mlp": 0.3996582, + "step": 354, + "time_per_iteration": 2.5372164249420166 + }, + { + "auxiliary_loss_clip": 0.01066449, + "auxiliary_loss_mlp": 0.01008933, + "balance_loss_clip": 1.00565362, + "balance_loss_mlp": 1.0001111, + "epoch": 0.010301201323196564, + "flos": 62403056040960.0, + "grad_norm": 0.781875012171104, + "language_loss": 0.52115363, + "learning_rate": 3.3839256628264573e-06, + "loss": 0.54190749, + "num_input_tokens_seen": 10090375, + "router_z_loss_clip": 0.609375, + "router_z_loss_mlp": 0.08837891, + "step": 355, + "time_per_iteration": 3.0340373516082764 + }, + { + "auxiliary_loss_clip": 0.01065276, + "auxiliary_loss_mlp": 0.01010393, + "balance_loss_clip": 1.00554907, + "balance_loss_mlp": 1.00166655, + "epoch": 0.01033021879171261, + "flos": 68573675024640.0, + "grad_norm": 0.682718223258853, + "language_loss": 0.55449319, + "learning_rate": 3.385546676718483e-06, + "loss": 0.57524991, + "num_input_tokens_seen": 10156230, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.08740234, + "step": 356, + "time_per_iteration": 3.2159173488616943 + }, + { + "auxiliary_loss_clip": 0.01278583, + "auxiliary_loss_mlp": 0.01100416, + "balance_loss_clip": 1.06978333, + "balance_loss_mlp": 1.05061078, + "epoch": 0.010359236260228657, + "flos": 32079664270080.0, + "grad_norm": 2.851990820823694, + "language_loss": 1.08252859, + "learning_rate": 3.387163143580998e-06, + "loss": 1.10631859, + "num_input_tokens_seen": 10172395, + "router_z_loss_clip": 2.08886719, + "router_z_loss_mlp": 0.49804688, + "step": 357, + "time_per_iteration": 2.560187339782715 + }, + { + "auxiliary_loss_clip": 0.01064179, + "auxiliary_loss_mlp": 0.01008756, + "balance_loss_clip": 1.00513053, + "balance_loss_mlp": 0.99993449, + "epoch": 0.010388253728744704, + "flos": 74773726151040.0, + "grad_norm": 0.7813325855987995, + "language_loss": 0.51972723, + "learning_rate": 3.3887750888519783e-06, + "loss": 0.54045659, + "num_input_tokens_seen": 10234795, + "router_z_loss_clip": 0.58984375, + "router_z_loss_mlp": 0.08837891, + "step": 358, + "time_per_iteration": 3.175640344619751 + }, + { + "auxiliary_loss_clip": 0.01272798, + "auxiliary_loss_mlp": 0.01085366, + "balance_loss_clip": 1.06711626, + "balance_loss_mlp": 1.03882647, + "epoch": 0.010417271197260752, + "flos": 14349092924160.0, + "grad_norm": 3.337522174649555, + "language_loss": 0.91218138, + "learning_rate": 3.3903825377565315e-06, + "loss": 0.93576294, + "num_input_tokens_seen": 10247845, + "router_z_loss_clip": 2.05664062, + "router_z_loss_mlp": 0.46557617, + "step": 359, + "time_per_iteration": 2.5152370929718018 + }, + { + "auxiliary_loss_clip": 0.01257227, + "auxiliary_loss_mlp": 0.01094668, + "balance_loss_clip": 1.07260656, + "balance_loss_mlp": 1.05163348, + "epoch": 0.010446288665776798, + "flos": 74731623050880.0, + "grad_norm": 2.2277649895268334, + "language_loss": 0.87653577, + "learning_rate": 3.3919855153092614e-06, + "loss": 0.90005469, + "num_input_tokens_seen": 10271540, + "router_z_loss_clip": 1.84570312, + "router_z_loss_mlp": 0.43041992, + "step": 360, + "time_per_iteration": 2.8551340103149414 + }, + { + "auxiliary_loss_clip": 0.01275713, + "auxiliary_loss_mlp": 0.01091023, + "balance_loss_clip": 1.07259274, + "balance_loss_mlp": 1.04262376, + "epoch": 0.010475306134292844, + "flos": 21645346919040.0, + "grad_norm": 2.111273397481533, + "language_loss": 0.95974255, + "learning_rate": 3.393584046316606e-06, + "loss": 0.98340976, + "num_input_tokens_seen": 10284925, + "router_z_loss_clip": 2.03417969, + "router_z_loss_mlp": 0.48364258, + "step": 361, + "time_per_iteration": 2.491626501083374 + }, + { + "auxiliary_loss_clip": 0.01255702, + "auxiliary_loss_mlp": 0.0107794, + "balance_loss_clip": 1.06961, + "balance_loss_mlp": 1.03695619, + "epoch": 0.01050432360280889, + "flos": 16209439920000.0, + "grad_norm": 3.3973855639980672, + "language_loss": 0.92144465, + "learning_rate": 3.3951781553791414e-06, + "loss": 0.94478106, + "num_input_tokens_seen": 10298005, + "router_z_loss_clip": 1.86132812, + "router_z_loss_mlp": 0.40942383, + "step": 362, + "time_per_iteration": 2.4323530197143555 + }, + { + "auxiliary_loss_clip": 0.01062043, + "auxiliary_loss_mlp": 0.01008252, + "balance_loss_clip": 1.00485039, + "balance_loss_mlp": 0.99995482, + "epoch": 0.010533341071324937, + "flos": 58854973144320.0, + "grad_norm": 0.7678616885073528, + "language_loss": 0.55871898, + "learning_rate": 3.396767866893849e-06, + "loss": 0.57942194, + "num_input_tokens_seen": 10348085, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.08300781, + "step": 363, + "time_per_iteration": 2.771610736846924 + }, + { + "auxiliary_loss_clip": 0.01268921, + "auxiliary_loss_mlp": 0.01079159, + "balance_loss_clip": 1.07163393, + "balance_loss_mlp": 1.03118944, + "epoch": 0.010562358539840984, + "flos": 14092725744000.0, + "grad_norm": 2.779908954025353, + "language_loss": 0.79097176, + "learning_rate": 3.3983532050563628e-06, + "loss": 0.81445253, + "num_input_tokens_seen": 10360400, + "router_z_loss_clip": 1.97265625, + "router_z_loss_mlp": 0.47973633, + "step": 364, + "time_per_iteration": 2.4046504497528076 + }, + { + "auxiliary_loss_clip": 0.01266486, + "auxiliary_loss_mlp": 0.01087543, + "balance_loss_clip": 1.06826723, + "balance_loss_mlp": 1.04214811, + "epoch": 0.010591376008357032, + "flos": 36461682835200.0, + "grad_norm": 3.6393000461724987, + "language_loss": 1.00221348, + "learning_rate": 3.3999341938631724e-06, + "loss": 1.02575374, + "num_input_tokens_seen": 10379230, + "router_z_loss_clip": 1.984375, + "router_z_loss_mlp": 0.45361328, + "step": 365, + "time_per_iteration": 2.5836966037750244 + }, + { + "auxiliary_loss_clip": 0.01276027, + "auxiliary_loss_mlp": 0.01099501, + "balance_loss_clip": 1.07361937, + "balance_loss_mlp": 1.05265212, + "epoch": 0.010620393476873078, + "flos": 37223767192320.0, + "grad_norm": 3.3720040756774083, + "language_loss": 1.01410747, + "learning_rate": 3.401510857113807e-06, + "loss": 1.03786278, + "num_input_tokens_seen": 10395170, + "router_z_loss_clip": 2.02539062, + "router_z_loss_mlp": 0.46826172, + "step": 366, + "time_per_iteration": 2.5862247943878174 + }, + { + "auxiliary_loss_clip": 0.01261141, + "auxiliary_loss_mlp": 0.01083004, + "balance_loss_clip": 1.07521915, + "balance_loss_mlp": 1.04342651, + "epoch": 0.010649410945389124, + "flos": 37114733416320.0, + "grad_norm": 2.9648209689624845, + "language_loss": 0.83151901, + "learning_rate": 3.4030832184129836e-06, + "loss": 0.8549605, + "num_input_tokens_seen": 10410605, + "router_z_loss_clip": 1.859375, + "router_z_loss_mlp": 0.39599609, + "step": 367, + "time_per_iteration": 2.5234215259552 + }, + { + "auxiliary_loss_clip": 0.01264078, + "auxiliary_loss_mlp": 0.01086246, + "balance_loss_clip": 1.07400346, + "balance_loss_mlp": 1.03951573, + "epoch": 0.01067842841390517, + "flos": 38246268447360.0, + "grad_norm": 2.35015267677646, + "language_loss": 0.83703631, + "learning_rate": 3.4046513011727257e-06, + "loss": 0.8605395, + "num_input_tokens_seen": 10429875, + "router_z_loss_clip": 1.89941406, + "router_z_loss_mlp": 0.46728516, + "step": 368, + "time_per_iteration": 2.589895009994507 + }, + { + "auxiliary_loss_clip": 0.01270944, + "auxiliary_loss_mlp": 0.01090345, + "balance_loss_clip": 1.07020962, + "balance_loss_mlp": 1.04514074, + "epoch": 0.010707445882421217, + "flos": 12414311176320.0, + "grad_norm": 3.1821352816710324, + "language_loss": 0.75983083, + "learning_rate": 3.406215128614456e-06, + "loss": 0.78344369, + "num_input_tokens_seen": 10442110, + "router_z_loss_clip": 2.00683594, + "router_z_loss_mlp": 0.45166016, + "step": 369, + "time_per_iteration": 2.4204142093658447 + }, + { + "auxiliary_loss_clip": 0.01269737, + "auxiliary_loss_mlp": 0.01084779, + "balance_loss_clip": 1.07272029, + "balance_loss_mlp": 1.04281723, + "epoch": 0.010736463350937264, + "flos": 28468807015680.0, + "grad_norm": 2.7537121355811647, + "language_loss": 1.02632737, + "learning_rate": 3.4077747237710627e-06, + "loss": 1.0498724, + "num_input_tokens_seen": 10458400, + "router_z_loss_clip": 1.96679688, + "router_z_loss_mlp": 0.41967773, + "step": 370, + "time_per_iteration": 2.4898722171783447 + }, + { + "auxiliary_loss_clip": 0.01058182, + "auxiliary_loss_mlp": 0.01012192, + "balance_loss_clip": 1.00384939, + "balance_loss_mlp": 1.00365663, + "epoch": 0.010765480819453312, + "flos": 72939320161920.0, + "grad_norm": 0.7404757821080358, + "language_loss": 0.50954401, + "learning_rate": 3.4093301094889307e-06, + "loss": 0.53024781, + "num_input_tokens_seen": 10517930, + "router_z_loss_clip": 0.54296875, + "router_z_loss_mlp": 0.08544922, + "step": 371, + "time_per_iteration": 3.126986503601074 + }, + { + "auxiliary_loss_clip": 0.0126104, + "auxiliary_loss_mlp": 0.01095626, + "balance_loss_clip": 1.0695076, + "balance_loss_mlp": 1.04984999, + "epoch": 0.010794498287969358, + "flos": 23503878524160.0, + "grad_norm": 3.0210632121115975, + "language_loss": 1.05000687, + "learning_rate": 3.410881308429951e-06, + "loss": 1.07357359, + "num_input_tokens_seen": 10529360, + "router_z_loss_clip": 1.91503906, + "router_z_loss_mlp": 0.45800781, + "step": 372, + "time_per_iteration": 2.419278144836426 + }, + { + "auxiliary_loss_clip": 0.0105788, + "auxiliary_loss_mlp": 0.01008606, + "balance_loss_clip": 1.00383472, + "balance_loss_mlp": 1.00045216, + "epoch": 0.010823515756485404, + "flos": 66672619516800.0, + "grad_norm": 0.7626833978716668, + "language_loss": 0.54566085, + "learning_rate": 3.412428343073505e-06, + "loss": 0.56632578, + "num_input_tokens_seen": 10589220, + "router_z_loss_clip": 0.5390625, + "router_z_loss_mlp": 0.08154297, + "step": 373, + "time_per_iteration": 2.9457812309265137 + }, + { + "auxiliary_loss_clip": 0.01257549, + "auxiliary_loss_mlp": 0.01068676, + "balance_loss_clip": 1.06971347, + "balance_loss_mlp": 1.02626121, + "epoch": 0.01085253322500145, + "flos": 43539066316800.0, + "grad_norm": 2.454209671317325, + "language_loss": 0.85913873, + "learning_rate": 3.413971235718411e-06, + "loss": 0.88240099, + "num_input_tokens_seen": 10607085, + "router_z_loss_clip": 1.87890625, + "router_z_loss_mlp": 0.42456055, + "step": 374, + "time_per_iteration": 2.6678826808929443 + }, + { + "auxiliary_loss_clip": 0.01268805, + "auxiliary_loss_mlp": 0.01090059, + "balance_loss_clip": 1.06863809, + "balance_loss_mlp": 1.04585671, + "epoch": 0.010881550693517497, + "flos": 23799557761920.0, + "grad_norm": 2.43692541074156, + "language_loss": 1.16477227, + "learning_rate": 3.4155100084848646e-06, + "loss": 1.18836081, + "num_input_tokens_seen": 10625140, + "router_z_loss_clip": 2.00195312, + "router_z_loss_mlp": 0.44189453, + "step": 375, + "time_per_iteration": 2.4608943462371826 + }, + { + "auxiliary_loss_clip": 0.01256516, + "auxiliary_loss_mlp": 0.0108528, + "balance_loss_clip": 1.06961298, + "balance_loss_mlp": 1.04007626, + "epoch": 0.010910568162033544, + "flos": 22120060942080.0, + "grad_norm": 3.207922513114282, + "language_loss": 0.91053963, + "learning_rate": 3.417044683316331e-06, + "loss": 0.93395764, + "num_input_tokens_seen": 10640310, + "router_z_loss_clip": 1.86816406, + "router_z_loss_mlp": 0.4519043, + "step": 376, + "time_per_iteration": 2.4659605026245117 + }, + { + "auxiliary_loss_clip": 0.01255819, + "auxiliary_loss_mlp": 0.01079766, + "balance_loss_clip": 1.06761372, + "balance_loss_mlp": 1.03847182, + "epoch": 0.010939585630549592, + "flos": 32736764568960.0, + "grad_norm": 2.4078042692491337, + "language_loss": 0.8499949, + "learning_rate": 3.4185752819814268e-06, + "loss": 0.87335062, + "num_input_tokens_seen": 10659645, + "router_z_loss_clip": 1.87890625, + "router_z_loss_mlp": 0.4128418, + "step": 377, + "time_per_iteration": 2.5542237758636475 + }, + { + "auxiliary_loss_clip": 0.01257533, + "auxiliary_loss_mlp": 0.01092375, + "balance_loss_clip": 1.07247782, + "balance_loss_mlp": 1.05224919, + "epoch": 0.010968603099065638, + "flos": 30511226085120.0, + "grad_norm": 3.2966034191211695, + "language_loss": 0.97124738, + "learning_rate": 3.420101826075774e-06, + "loss": 0.99474645, + "num_input_tokens_seen": 10675925, + "router_z_loss_clip": 1.84863281, + "router_z_loss_mlp": 0.40112305, + "step": 378, + "time_per_iteration": 2.55072021484375 + }, + { + "auxiliary_loss_clip": 0.0126156, + "auxiliary_loss_mlp": 0.01074657, + "balance_loss_clip": 1.06967974, + "balance_loss_mlp": 1.03453112, + "epoch": 0.010997620567581684, + "flos": 36895513789440.0, + "grad_norm": 3.3229004812527116, + "language_loss": 0.9684996, + "learning_rate": 3.4216243370238263e-06, + "loss": 0.9918617, + "num_input_tokens_seen": 10691055, + "router_z_loss_clip": 1.92089844, + "router_z_loss_mlp": 0.40136719, + "step": 379, + "time_per_iteration": 2.5486268997192383 + }, + { + "auxiliary_loss_clip": 0.01261135, + "auxiliary_loss_mlp": 0.01087692, + "balance_loss_clip": 1.06722772, + "balance_loss_mlp": 1.0429647, + "epoch": 0.01102663803609773, + "flos": 32519569800960.0, + "grad_norm": 2.8578922351448064, + "language_loss": 0.90099525, + "learning_rate": 3.423142836080674e-06, + "loss": 0.92448348, + "num_input_tokens_seen": 10710200, + "router_z_loss_clip": 1.93847656, + "router_z_loss_mlp": 0.44702148, + "step": 380, + "time_per_iteration": 2.595701217651367 + }, + { + "auxiliary_loss_clip": 0.0124852, + "auxiliary_loss_mlp": 0.01093566, + "balance_loss_clip": 1.05883718, + "balance_loss_mlp": 1.05057883, + "epoch": 0.011055655504613777, + "flos": 42514191089280.0, + "grad_norm": 3.8092553070019424, + "language_loss": 1.18656528, + "learning_rate": 3.4246573443338227e-06, + "loss": 1.20998621, + "num_input_tokens_seen": 10725840, + "router_z_loss_clip": 1.89550781, + "router_z_loss_mlp": 0.42944336, + "step": 381, + "time_per_iteration": 2.5257151126861572 + }, + { + "auxiliary_loss_clip": 0.01253068, + "auxiliary_loss_mlp": 0.01075576, + "balance_loss_clip": 1.06589711, + "balance_loss_mlp": 1.03409147, + "epoch": 0.011084672973129824, + "flos": 15624854248320.0, + "grad_norm": 4.438460294981292, + "language_loss": 1.02178311, + "learning_rate": 3.4261678827049543e-06, + "loss": 1.04506946, + "num_input_tokens_seen": 10738165, + "router_z_loss_clip": 1.86914062, + "router_z_loss_mlp": 0.41455078, + "step": 382, + "time_per_iteration": 2.437466859817505 + }, + { + "auxiliary_loss_clip": 0.0126216, + "auxiliary_loss_mlp": 0.01088394, + "balance_loss_clip": 1.06908143, + "balance_loss_mlp": 1.04273677, + "epoch": 0.011113690441645872, + "flos": 15734621162880.0, + "grad_norm": 3.575160231972366, + "language_loss": 1.01136231, + "learning_rate": 3.4276744719516564e-06, + "loss": 1.03486776, + "num_input_tokens_seen": 10753495, + "router_z_loss_clip": 1.93164062, + "router_z_loss_mlp": 0.45605469, + "step": 383, + "time_per_iteration": 2.6205620765686035 + }, + { + "auxiliary_loss_clip": 0.01261226, + "auxiliary_loss_mlp": 0.01091472, + "balance_loss_clip": 1.07217765, + "balance_loss_mlp": 1.04717362, + "epoch": 0.011142707910161918, + "flos": 39485999381760.0, + "grad_norm": 3.9450411721984446, + "language_loss": 1.08046138, + "learning_rate": 3.4291771326691384e-06, + "loss": 1.10398829, + "num_input_tokens_seen": 10772245, + "router_z_loss_clip": 1.89257812, + "router_z_loss_mlp": 0.44311523, + "step": 384, + "time_per_iteration": 2.627092123031616 + }, + { + "auxiliary_loss_clip": 0.01250463, + "auxiliary_loss_mlp": 0.01085231, + "balance_loss_clip": 1.06377864, + "balance_loss_mlp": 1.04365134, + "epoch": 0.011171725378677964, + "flos": 32627521324800.0, + "grad_norm": 2.563394210790961, + "language_loss": 0.76175833, + "learning_rate": 3.4306758852919156e-06, + "loss": 0.78511524, + "num_input_tokens_seen": 10788255, + "router_z_loss_clip": 1.8671875, + "router_z_loss_mlp": 0.41625977, + "step": 385, + "time_per_iteration": 2.4554524421691895 + }, + { + "auxiliary_loss_clip": 0.01252924, + "auxiliary_loss_mlp": 0.01082395, + "balance_loss_clip": 1.06741846, + "balance_loss_mlp": 1.03988504, + "epoch": 0.01120074284719401, + "flos": 29744009758080.0, + "grad_norm": 2.7299414885490836, + "language_loss": 0.89279109, + "learning_rate": 3.4321707500954817e-06, + "loss": 0.91614431, + "num_input_tokens_seen": 10806540, + "router_z_loss_clip": 1.85351562, + "router_z_loss_mlp": 0.42529297, + "step": 386, + "time_per_iteration": 5.106091737747192 + }, + { + "auxiliary_loss_clip": 0.01264773, + "auxiliary_loss_mlp": 0.01081646, + "balance_loss_clip": 1.06720436, + "balance_loss_mlp": 1.03751469, + "epoch": 0.011229760315710057, + "flos": 30552702647040.0, + "grad_norm": 2.3360094005380603, + "language_loss": 0.86433756, + "learning_rate": 3.433661747197952e-06, + "loss": 0.88780177, + "num_input_tokens_seen": 10828090, + "router_z_loss_clip": 1.97753906, + "router_z_loss_mlp": 0.44140625, + "step": 387, + "time_per_iteration": 4.96156120300293 + }, + { + "auxiliary_loss_clip": 0.01068697, + "auxiliary_loss_mlp": 0.01010492, + "balance_loss_clip": 1.01122355, + "balance_loss_mlp": 1.00229084, + "epoch": 0.011258777784226104, + "flos": 62692939992960.0, + "grad_norm": 0.7052611843011187, + "language_loss": 0.53479356, + "learning_rate": 3.4351488965616886e-06, + "loss": 0.55558544, + "num_input_tokens_seen": 10887500, + "router_z_loss_clip": 0.57421875, + "router_z_loss_mlp": 0.08203125, + "step": 388, + "time_per_iteration": 5.474780082702637 + }, + { + "auxiliary_loss_clip": 0.0125136, + "auxiliary_loss_mlp": 0.01091915, + "balance_loss_clip": 1.06402385, + "balance_loss_mlp": 1.05255187, + "epoch": 0.011287795252742152, + "flos": 15407031075840.0, + "grad_norm": 2.4748945669773827, + "language_loss": 0.83345711, + "learning_rate": 3.4366322179949013e-06, + "loss": 0.85688984, + "num_input_tokens_seen": 10903135, + "router_z_loss_clip": 1.87304688, + "router_z_loss_mlp": 0.39404297, + "step": 389, + "time_per_iteration": 4.713484764099121 + }, + { + "auxiliary_loss_clip": 0.01263792, + "auxiliary_loss_mlp": 0.01103531, + "balance_loss_clip": 1.068174, + "balance_loss_mlp": 1.06245136, + "epoch": 0.011316812721258198, + "flos": 30985940108160.0, + "grad_norm": 2.489358414364969, + "language_loss": 1.05505085, + "learning_rate": 3.438111731153238e-06, + "loss": 1.07872415, + "num_input_tokens_seen": 10922040, + "router_z_loss_clip": 1.95507812, + "router_z_loss_mlp": 0.41064453, + "step": 390, + "time_per_iteration": 2.5505220890045166 + }, + { + "auxiliary_loss_clip": 0.01246268, + "auxiliary_loss_mlp": 0.0108215, + "balance_loss_clip": 1.06481743, + "balance_loss_mlp": 1.04028332, + "epoch": 0.011345830189774244, + "flos": 23364295441920.0, + "grad_norm": 2.366107359353078, + "language_loss": 0.90476227, + "learning_rate": 3.439587455541337e-06, + "loss": 0.92804641, + "num_input_tokens_seen": 10937670, + "router_z_loss_clip": 1.81542969, + "router_z_loss_mlp": 0.41821289, + "step": 391, + "time_per_iteration": 2.449899435043335 + }, + { + "auxiliary_loss_clip": 0.0125932, + "auxiliary_loss_mlp": 0.01080741, + "balance_loss_clip": 1.06502032, + "balance_loss_mlp": 1.03594184, + "epoch": 0.01137484765829029, + "flos": 34709322274560.0, + "grad_norm": 3.172922059905694, + "language_loss": 1.01695919, + "learning_rate": 3.4410594105143784e-06, + "loss": 1.04035974, + "num_input_tokens_seen": 10958640, + "router_z_loss_clip": 1.94726562, + "router_z_loss_mlp": 0.44799805, + "step": 392, + "time_per_iteration": 2.6561686992645264 + }, + { + "auxiliary_loss_clip": 0.01251806, + "auxiliary_loss_mlp": 0.01087637, + "balance_loss_clip": 1.06229353, + "balance_loss_mlp": 1.04138434, + "epoch": 0.011403865126806337, + "flos": 25622792115840.0, + "grad_norm": 2.040637701053185, + "language_loss": 1.00438166, + "learning_rate": 3.442527615279605e-06, + "loss": 1.027776, + "num_input_tokens_seen": 10977625, + "router_z_loss_clip": 1.89648438, + "router_z_loss_mlp": 0.46289062, + "step": 393, + "time_per_iteration": 2.5932552814483643 + }, + { + "auxiliary_loss_clip": 0.01254288, + "auxiliary_loss_mlp": 0.01071582, + "balance_loss_clip": 1.06625128, + "balance_loss_mlp": 1.02961993, + "epoch": 0.011432882595322384, + "flos": 31461142890240.0, + "grad_norm": 1.9383829033233142, + "language_loss": 0.84433222, + "learning_rate": 3.443992088897824e-06, + "loss": 0.8675909, + "num_input_tokens_seen": 11004130, + "router_z_loss_clip": 1.87695312, + "router_z_loss_mlp": 0.41992188, + "step": 394, + "time_per_iteration": 2.7879011631011963 + }, + { + "auxiliary_loss_clip": 0.0106326, + "auxiliary_loss_mlp": 0.01007895, + "balance_loss_clip": 1.00776088, + "balance_loss_mlp": 1.00031281, + "epoch": 0.01146190006383843, + "flos": 74767442106240.0, + "grad_norm": 0.6821586703158155, + "language_loss": 0.51562476, + "learning_rate": 3.4454528502848933e-06, + "loss": 0.5363363, + "num_input_tokens_seen": 11068260, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.07568359, + "step": 395, + "time_per_iteration": 3.0658676624298096 + }, + { + "auxiliary_loss_clip": 0.01060435, + "auxiliary_loss_mlp": 0.01007579, + "balance_loss_clip": 1.00612426, + "balance_loss_mlp": 0.99980646, + "epoch": 0.011490917532354478, + "flos": 63245929017600.0, + "grad_norm": 0.6852107495153835, + "language_loss": 0.51910645, + "learning_rate": 3.4469099182131874e-06, + "loss": 0.53978664, + "num_input_tokens_seen": 11135770, + "router_z_loss_clip": 0.54296875, + "router_z_loss_mlp": 0.07763672, + "step": 396, + "time_per_iteration": 3.1812257766723633 + }, + { + "auxiliary_loss_clip": 0.01248825, + "auxiliary_loss_mlp": 0.01083262, + "balance_loss_clip": 1.06485772, + "balance_loss_mlp": 1.04199219, + "epoch": 0.011519935000870524, + "flos": 36021079077120.0, + "grad_norm": 3.2121404650114282, + "language_loss": 0.9852742, + "learning_rate": 3.4483633113130455e-06, + "loss": 1.00859511, + "num_input_tokens_seen": 11152745, + "router_z_loss_clip": 1.84082031, + "router_z_loss_mlp": 0.41259766, + "step": 397, + "time_per_iteration": 2.5919599533081055 + }, + { + "auxiliary_loss_clip": 0.01056236, + "auxiliary_loss_mlp": 0.01009758, + "balance_loss_clip": 1.00439835, + "balance_loss_mlp": 1.00193739, + "epoch": 0.01154895246938657, + "flos": 65348015333760.0, + "grad_norm": 0.7045602794036027, + "language_loss": 0.50584096, + "learning_rate": 3.4498130480741995e-06, + "loss": 0.52650088, + "num_input_tokens_seen": 11204975, + "router_z_loss_clip": 0.515625, + "router_z_loss_mlp": 0.078125, + "step": 398, + "time_per_iteration": 2.858719825744629 + }, + { + "auxiliary_loss_clip": 0.01252947, + "auxiliary_loss_mlp": 0.01089752, + "balance_loss_clip": 1.06541252, + "balance_loss_mlp": 1.04748011, + "epoch": 0.011577969937902617, + "flos": 17707423248000.0, + "grad_norm": 5.378063795657694, + "language_loss": 0.84030002, + "learning_rate": 3.4512591468471864e-06, + "loss": 0.86372703, + "num_input_tokens_seen": 11218985, + "router_z_loss_clip": 1.87402344, + "router_z_loss_mlp": 0.4230957, + "step": 399, + "time_per_iteration": 2.448517084121704 + }, + { + "auxiliary_loss_clip": 0.01253178, + "auxiliary_loss_mlp": 0.01083674, + "balance_loss_clip": 1.06380117, + "balance_loss_mlp": 1.04197502, + "epoch": 0.011606987406418664, + "flos": 30916043832960.0, + "grad_norm": 2.8407419149913262, + "language_loss": 1.11386371, + "learning_rate": 3.452701625844741e-06, + "loss": 1.1372323, + "num_input_tokens_seen": 11239685, + "router_z_loss_clip": 1.89550781, + "router_z_loss_mlp": 0.41723633, + "step": 400, + "time_per_iteration": 2.498701572418213 + }, + { + "auxiliary_loss_clip": 0.01255625, + "auxiliary_loss_mlp": 0.01087649, + "balance_loss_clip": 1.06385028, + "balance_loss_mlp": 1.04640234, + "epoch": 0.01163600487493471, + "flos": 29890854403200.0, + "grad_norm": 2.9280244482705244, + "language_loss": 1.14369464, + "learning_rate": 3.4541405031431746e-06, + "loss": 1.16712737, + "num_input_tokens_seen": 11256765, + "router_z_loss_clip": 1.91796875, + "router_z_loss_mlp": 0.4128418, + "step": 401, + "time_per_iteration": 2.6204473972320557 + }, + { + "auxiliary_loss_clip": 0.01231073, + "auxiliary_loss_mlp": 0.01065773, + "balance_loss_clip": 1.0584358, + "balance_loss_mlp": 1.02826953, + "epoch": 0.011665022343450758, + "flos": 32702095722240.0, + "grad_norm": 2.0749854642221024, + "language_loss": 0.81595057, + "learning_rate": 3.45557579668373e-06, + "loss": 0.83891904, + "num_input_tokens_seen": 11274260, + "router_z_loss_clip": 1.7265625, + "router_z_loss_mlp": 0.375, + "step": 402, + "time_per_iteration": 2.6002204418182373 + }, + { + "auxiliary_loss_clip": 0.01057472, + "auxiliary_loss_mlp": 0.01010389, + "balance_loss_clip": 1.00737906, + "balance_loss_mlp": 1.00242567, + "epoch": 0.011694039811966804, + "flos": 57463300506240.0, + "grad_norm": 0.8029675182308176, + "language_loss": 0.55277675, + "learning_rate": 3.4570075242739278e-06, + "loss": 0.57345533, + "num_input_tokens_seen": 11319090, + "router_z_loss_clip": 0.5, + "router_z_loss_mlp": 0.07958984, + "step": 403, + "time_per_iteration": 2.7400150299072266 + }, + { + "auxiliary_loss_clip": 0.01254204, + "auxiliary_loss_mlp": 0.01077184, + "balance_loss_clip": 1.06875968, + "balance_loss_mlp": 1.03705859, + "epoch": 0.01172305728048285, + "flos": 32482038222720.0, + "grad_norm": 2.7710374748160205, + "language_loss": 1.09113979, + "learning_rate": 3.4584357035888897e-06, + "loss": 1.11445379, + "num_input_tokens_seen": 11345600, + "router_z_loss_clip": 1.85546875, + "router_z_loss_mlp": 0.40112305, + "step": 404, + "time_per_iteration": 2.838379383087158 + }, + { + "auxiliary_loss_clip": 0.01260518, + "auxiliary_loss_mlp": 0.01093929, + "balance_loss_clip": 1.07166982, + "balance_loss_mlp": 1.05199146, + "epoch": 0.011752074748998897, + "flos": 35183023868160.0, + "grad_norm": 6.767648602361475, + "language_loss": 0.80493742, + "learning_rate": 3.4598603521726485e-06, + "loss": 0.82848191, + "num_input_tokens_seen": 11366800, + "router_z_loss_clip": 1.88964844, + "router_z_loss_mlp": 0.41967773, + "step": 405, + "time_per_iteration": 2.5190646648406982 + }, + { + "auxiliary_loss_clip": 0.01243372, + "auxiliary_loss_mlp": 0.01091242, + "balance_loss_clip": 1.06100845, + "balance_loss_mlp": 1.05056798, + "epoch": 0.011781092217514944, + "flos": 31058210355840.0, + "grad_norm": 6.8494421142526, + "language_loss": 0.98062897, + "learning_rate": 3.4612814874394425e-06, + "loss": 1.00397515, + "num_input_tokens_seen": 11382550, + "router_z_loss_clip": 1.82519531, + "router_z_loss_mlp": 0.40698242, + "step": 406, + "time_per_iteration": 2.505539655685425 + }, + { + "auxiliary_loss_clip": 0.01254461, + "auxiliary_loss_mlp": 0.01077607, + "balance_loss_clip": 1.07148695, + "balance_loss_mlp": 1.03733802, + "epoch": 0.01181010968603099, + "flos": 24308137670400.0, + "grad_norm": 3.5094062234560703, + "language_loss": 0.9217726, + "learning_rate": 3.4626991266749886e-06, + "loss": 0.94509327, + "num_input_tokens_seen": 11397580, + "router_z_loss_clip": 1.83007812, + "router_z_loss_mlp": 0.40283203, + "step": 407, + "time_per_iteration": 2.487431526184082 + }, + { + "auxiliary_loss_clip": 0.01246935, + "auxiliary_loss_mlp": 0.0108796, + "balance_loss_clip": 1.06650031, + "balance_loss_mlp": 1.04771459, + "epoch": 0.011839127154547038, + "flos": 7194167579520.0, + "grad_norm": 19.38822473545377, + "language_loss": 0.96014786, + "learning_rate": 3.4641132870377497e-06, + "loss": 0.98349679, + "num_input_tokens_seen": 11405950, + "router_z_loss_clip": 1.80371094, + "router_z_loss_mlp": 0.40258789, + "step": 408, + "time_per_iteration": 2.428882122039795 + }, + { + "auxiliary_loss_clip": 0.01062809, + "auxiliary_loss_mlp": 0.01007694, + "balance_loss_clip": 1.01168609, + "balance_loss_mlp": 0.99996889, + "epoch": 0.011868144623063084, + "flos": 67373814729600.0, + "grad_norm": 0.6940708977817863, + "language_loss": 0.51349473, + "learning_rate": 3.4655239855601753e-06, + "loss": 0.53419977, + "num_input_tokens_seen": 11471845, + "router_z_loss_clip": 0.5078125, + "router_z_loss_mlp": 0.07714844, + "step": 409, + "time_per_iteration": 3.097460985183716 + }, + { + "auxiliary_loss_clip": 0.01240674, + "auxiliary_loss_mlp": 0.01070128, + "balance_loss_clip": 1.06318617, + "balance_loss_mlp": 1.03171873, + "epoch": 0.01189716209157913, + "flos": 22083925818240.0, + "grad_norm": 2.546837657705033, + "language_loss": 0.67532372, + "learning_rate": 3.4669312391499364e-06, + "loss": 0.69843173, + "num_input_tokens_seen": 11485650, + "router_z_loss_clip": 1.77441406, + "router_z_loss_mlp": 0.3840332, + "step": 410, + "time_per_iteration": 2.4010138511657715 + }, + { + "auxiliary_loss_clip": 0.01251135, + "auxiliary_loss_mlp": 0.01082739, + "balance_loss_clip": 1.06966233, + "balance_loss_mlp": 1.04387724, + "epoch": 0.011926179560095177, + "flos": 74731692873600.0, + "grad_norm": 2.7539956041041345, + "language_loss": 0.75460714, + "learning_rate": 3.468335064591138e-06, + "loss": 0.77794588, + "num_input_tokens_seen": 11512540, + "router_z_loss_clip": 1.81542969, + "router_z_loss_mlp": 0.38818359, + "step": 411, + "time_per_iteration": 2.9031307697296143 + }, + { + "auxiliary_loss_clip": 0.01246151, + "auxiliary_loss_mlp": 0.01092408, + "balance_loss_clip": 1.06424737, + "balance_loss_mlp": 1.05094683, + "epoch": 0.011955197028611224, + "flos": 18770353724160.0, + "grad_norm": 4.521411077662914, + "language_loss": 1.00742912, + "learning_rate": 3.469735478545525e-06, + "loss": 1.03081465, + "num_input_tokens_seen": 11526705, + "router_z_loss_clip": 1.81933594, + "router_z_loss_mlp": 0.41455078, + "step": 412, + "time_per_iteration": 2.4475202560424805 + }, + { + "auxiliary_loss_clip": 0.01231595, + "auxiliary_loss_mlp": 0.01075647, + "balance_loss_clip": 1.0618552, + "balance_loss_mlp": 1.03735662, + "epoch": 0.01198421449712727, + "flos": 24235169195520.0, + "grad_norm": 2.4799319982892554, + "language_loss": 0.97080821, + "learning_rate": 3.4711324975536624e-06, + "loss": 0.99388057, + "num_input_tokens_seen": 11542650, + "router_z_loss_clip": 1.69824219, + "router_z_loss_mlp": 0.38330078, + "step": 413, + "time_per_iteration": 2.459949016571045 + }, + { + "auxiliary_loss_clip": 0.0124703, + "auxiliary_loss_mlp": 0.010856, + "balance_loss_clip": 1.06690502, + "balance_loss_mlp": 1.04733396, + "epoch": 0.012013231965643318, + "flos": 47586337966080.0, + "grad_norm": 3.213886169179528, + "language_loss": 0.68975878, + "learning_rate": 3.4725261380361128e-06, + "loss": 0.71308506, + "num_input_tokens_seen": 11561345, + "router_z_loss_clip": 1.80078125, + "router_z_loss_mlp": 0.3828125, + "step": 414, + "time_per_iteration": 2.6877520084381104 + }, + { + "auxiliary_loss_clip": 0.01238675, + "auxiliary_loss_mlp": 0.01065779, + "balance_loss_clip": 1.06482887, + "balance_loss_mlp": 1.02956343, + "epoch": 0.012042249434159364, + "flos": 15151641413760.0, + "grad_norm": 3.4464411577537564, + "language_loss": 0.83200073, + "learning_rate": 3.473916416294592e-06, + "loss": 0.85504526, + "num_input_tokens_seen": 11574225, + "router_z_loss_clip": 1.74023438, + "router_z_loss_mlp": 0.36181641, + "step": 415, + "time_per_iteration": 2.3908469676971436 + }, + { + "auxiliary_loss_clip": 0.01245947, + "auxiliary_loss_mlp": 0.0108831, + "balance_loss_clip": 1.06680214, + "balance_loss_mlp": 1.04403567, + "epoch": 0.01207126690267541, + "flos": 19201042656000.0, + "grad_norm": 3.2483367404520376, + "language_loss": 0.89352697, + "learning_rate": 3.4753033485131146e-06, + "loss": 0.91686952, + "num_input_tokens_seen": 11595270, + "router_z_loss_clip": 1.79199219, + "router_z_loss_mlp": 0.44311523, + "step": 416, + "time_per_iteration": 2.802920341491699 + }, + { + "auxiliary_loss_clip": 0.01239728, + "auxiliary_loss_mlp": 0.01073895, + "balance_loss_clip": 1.06536293, + "balance_loss_mlp": 1.0348177, + "epoch": 0.012100284371191457, + "flos": 16535598641280.0, + "grad_norm": 3.004844281529641, + "language_loss": 0.96442199, + "learning_rate": 3.4766869507591215e-06, + "loss": 0.98755819, + "num_input_tokens_seen": 11608715, + "router_z_loss_clip": 1.7421875, + "router_z_loss_mlp": 0.39086914, + "step": 417, + "time_per_iteration": 2.4243195056915283 + }, + { + "auxiliary_loss_clip": 0.0124145, + "auxiliary_loss_mlp": 0.01085333, + "balance_loss_clip": 1.06376052, + "balance_loss_mlp": 1.04468226, + "epoch": 0.012129301839707504, + "flos": 57801749892480.0, + "grad_norm": 2.458821921360228, + "language_loss": 0.76212275, + "learning_rate": 3.4780672389845997e-06, + "loss": 0.78539056, + "num_input_tokens_seen": 11632190, + "router_z_loss_clip": 1.77734375, + "router_z_loss_mlp": 0.40625, + "step": 418, + "time_per_iteration": 2.7555861473083496 + }, + { + "auxiliary_loss_clip": 0.01246154, + "auxiliary_loss_mlp": 0.0108266, + "balance_loss_clip": 1.06604731, + "balance_loss_mlp": 1.04186618, + "epoch": 0.01215831930822355, + "flos": 28650111039360.0, + "grad_norm": 2.3340993429276367, + "language_loss": 0.72394729, + "learning_rate": 3.4794442290271854e-06, + "loss": 0.74723542, + "num_input_tokens_seen": 11649890, + "router_z_loss_clip": 1.80078125, + "router_z_loss_mlp": 0.40795898, + "step": 419, + "time_per_iteration": 2.5011563301086426 + }, + { + "auxiliary_loss_clip": 0.0124555, + "auxiliary_loss_mlp": 0.01082546, + "balance_loss_clip": 1.06522143, + "balance_loss_mlp": 1.04401779, + "epoch": 0.012187336776739598, + "flos": 11427630842880.0, + "grad_norm": 5.448312426886068, + "language_loss": 0.962322, + "learning_rate": 3.4808179366112537e-06, + "loss": 0.98560292, + "num_input_tokens_seen": 11662145, + "router_z_loss_clip": 1.80371094, + "router_z_loss_mlp": 0.38549805, + "step": 420, + "time_per_iteration": 2.4860715866088867 + }, + { + "auxiliary_loss_clip": 0.01240808, + "auxiliary_loss_mlp": 0.01075009, + "balance_loss_clip": 1.06097889, + "balance_loss_mlp": 1.03474021, + "epoch": 0.012216354245255644, + "flos": 31780108005120.0, + "grad_norm": 2.091508527124366, + "language_loss": 0.94276452, + "learning_rate": 3.482188377348995e-06, + "loss": 0.96592271, + "num_input_tokens_seen": 11685255, + "router_z_loss_clip": 1.79980469, + "router_z_loss_mlp": 0.40234375, + "step": 421, + "time_per_iteration": 2.6266062259674072 + }, + { + "auxiliary_loss_clip": 0.0123103, + "auxiliary_loss_mlp": 0.01078297, + "balance_loss_clip": 1.05936766, + "balance_loss_mlp": 1.040079, + "epoch": 0.01224537171377169, + "flos": 27556631256960.0, + "grad_norm": 2.553689965281223, + "language_loss": 0.90924764, + "learning_rate": 3.4835555667414816e-06, + "loss": 0.93234098, + "num_input_tokens_seen": 11701360, + "router_z_loss_clip": 1.71484375, + "router_z_loss_mlp": 0.38208008, + "step": 422, + "time_per_iteration": 2.4927361011505127 + }, + { + "auxiliary_loss_clip": 0.01240042, + "auxiliary_loss_mlp": 0.0108303, + "balance_loss_clip": 1.06132197, + "balance_loss_mlp": 1.04354823, + "epoch": 0.012274389182287737, + "flos": 25474900129920.0, + "grad_norm": 8.887666729130462, + "language_loss": 1.00451267, + "learning_rate": 3.484919520179718e-06, + "loss": 1.02774346, + "num_input_tokens_seen": 11716840, + "router_z_loss_clip": 1.78710938, + "router_z_loss_mlp": 0.39501953, + "step": 423, + "time_per_iteration": 2.4649465084075928 + }, + { + "auxiliary_loss_clip": 0.01060127, + "auxiliary_loss_mlp": 0.01017058, + "balance_loss_clip": 1.00796616, + "balance_loss_mlp": 1.00957191, + "epoch": 0.012303406650803783, + "flos": 57661920564480.0, + "grad_norm": 0.7332516402017335, + "language_loss": 0.54981637, + "learning_rate": 3.4862802529456826e-06, + "loss": 0.57058823, + "num_input_tokens_seen": 11775725, + "router_z_loss_clip": 0.5234375, + "router_z_loss_mlp": 0.07470703, + "step": 424, + "time_per_iteration": 2.994548797607422 + }, + { + "auxiliary_loss_clip": 0.01245653, + "auxiliary_loss_mlp": 0.01081517, + "balance_loss_clip": 1.06209922, + "balance_loss_mlp": 1.04096222, + "epoch": 0.01233242411931983, + "flos": 27556282143360.0, + "grad_norm": 2.5577173478071087, + "language_loss": 0.93536699, + "learning_rate": 3.487637780213353e-06, + "loss": 0.95863873, + "num_input_tokens_seen": 11792910, + "router_z_loss_clip": 1.8359375, + "router_z_loss_mlp": 0.4050293, + "step": 425, + "time_per_iteration": 2.5004937648773193 + }, + { + "auxiliary_loss_clip": 0.01249639, + "auxiliary_loss_mlp": 0.01078815, + "balance_loss_clip": 1.0680896, + "balance_loss_mlp": 1.03761649, + "epoch": 0.012361441587835878, + "flos": 15810522192000.0, + "grad_norm": 2.8254074097079998, + "language_loss": 0.85641563, + "learning_rate": 3.4889921170497213e-06, + "loss": 0.87970018, + "num_input_tokens_seen": 11805285, + "router_z_loss_clip": 1.81640625, + "router_z_loss_mlp": 0.41210938, + "step": 426, + "time_per_iteration": 2.5499517917633057 + }, + { + "auxiliary_loss_clip": 0.01058415, + "auxiliary_loss_mlp": 0.01009289, + "balance_loss_clip": 1.00694561, + "balance_loss_mlp": 1.00123096, + "epoch": 0.012390459056351924, + "flos": 60622310678400.0, + "grad_norm": 0.6902541365140114, + "language_loss": 0.59296769, + "learning_rate": 3.4903432784158e-06, + "loss": 0.61364472, + "num_input_tokens_seen": 11868365, + "router_z_loss_clip": 0.515625, + "router_z_loss_mlp": 0.08056641, + "step": 427, + "time_per_iteration": 3.0230069160461426 + }, + { + "auxiliary_loss_clip": 0.01250119, + "auxiliary_loss_mlp": 0.01078656, + "balance_loss_clip": 1.0592072, + "balance_loss_mlp": 1.0385536, + "epoch": 0.01241947652486797, + "flos": 25078845133440.0, + "grad_norm": 2.693898647815045, + "language_loss": 1.04567122, + "learning_rate": 3.49169127916761e-06, + "loss": 1.068959, + "num_input_tokens_seen": 11889040, + "router_z_loss_clip": 1.90625, + "router_z_loss_mlp": 0.40136719, + "step": 428, + "time_per_iteration": 2.6321616172790527 + }, + { + "auxiliary_loss_clip": 0.01228317, + "auxiliary_loss_mlp": 0.01066773, + "balance_loss_clip": 1.05721009, + "balance_loss_mlp": 1.03043818, + "epoch": 0.012448493993384017, + "flos": 22517652038400.0, + "grad_norm": 5.658201111885965, + "language_loss": 0.92866778, + "learning_rate": 3.4930361340571636e-06, + "loss": 0.95161867, + "num_input_tokens_seen": 11901770, + "router_z_loss_clip": 1.70996094, + "router_z_loss_mlp": 0.36376953, + "step": 429, + "time_per_iteration": 2.4685301780700684 + }, + { + "auxiliary_loss_clip": 0.01228813, + "auxiliary_loss_mlp": 0.01076964, + "balance_loss_clip": 1.05483377, + "balance_loss_mlp": 1.03907943, + "epoch": 0.012477511461900063, + "flos": 26207901457920.0, + "grad_norm": 2.4669965228061512, + "language_loss": 0.98047715, + "learning_rate": 3.494377857733432e-06, + "loss": 1.00353491, + "num_input_tokens_seen": 11918465, + "router_z_loss_clip": 1.74023438, + "router_z_loss_mlp": 0.37890625, + "step": 430, + "time_per_iteration": 2.599656343460083 + }, + { + "auxiliary_loss_clip": 0.01242663, + "auxiliary_loss_mlp": 0.010864, + "balance_loss_clip": 1.06140876, + "balance_loss_mlp": 1.04441428, + "epoch": 0.01250652893041611, + "flos": 13911770833920.0, + "grad_norm": 3.027814638531197, + "language_loss": 0.97372067, + "learning_rate": 3.4957164647433026e-06, + "loss": 0.9970113, + "num_input_tokens_seen": 11931705, + "router_z_loss_clip": 1.8125, + "router_z_loss_mlp": 0.41967773, + "step": 431, + "time_per_iteration": 2.4365193843841553 + }, + { + "auxiliary_loss_clip": 0.01237794, + "auxiliary_loss_mlp": 0.01083601, + "balance_loss_clip": 1.05988717, + "balance_loss_mlp": 1.04268837, + "epoch": 0.012535546398932158, + "flos": 23944063345920.0, + "grad_norm": 2.3154453549199143, + "language_loss": 0.93002039, + "learning_rate": 3.497051969532526e-06, + "loss": 0.95323431, + "num_input_tokens_seen": 11947765, + "router_z_loss_clip": 1.77832031, + "router_z_loss_mlp": 0.40893555, + "step": 432, + "time_per_iteration": 2.4863619804382324 + }, + { + "auxiliary_loss_clip": 0.01059233, + "auxiliary_loss_mlp": 0.01022169, + "balance_loss_clip": 1.00925088, + "balance_loss_mlp": 1.01468217, + "epoch": 0.012564563867448204, + "flos": 62875465914240.0, + "grad_norm": 0.6860876128192182, + "language_loss": 0.5446946, + "learning_rate": 3.498384386446649e-06, + "loss": 0.5655086, + "num_input_tokens_seen": 12009485, + "router_z_loss_clip": 0.5, + "router_z_loss_mlp": 0.07470703, + "step": 433, + "time_per_iteration": 3.0187249183654785 + }, + { + "auxiliary_loss_clip": 0.01235029, + "auxiliary_loss_mlp": 0.01084307, + "balance_loss_clip": 1.06418228, + "balance_loss_mlp": 1.04465806, + "epoch": 0.01259358133596425, + "flos": 17043829436160.0, + "grad_norm": 3.48986576656221, + "language_loss": 0.98978823, + "learning_rate": 3.499713729731944e-06, + "loss": 1.01298153, + "num_input_tokens_seen": 12020175, + "router_z_loss_clip": 1.70703125, + "router_z_loss_mlp": 0.39672852, + "step": 434, + "time_per_iteration": 2.540928363800049 + }, + { + "auxiliary_loss_clip": 0.01229668, + "auxiliary_loss_mlp": 0.01072927, + "balance_loss_clip": 1.05733466, + "balance_loss_mlp": 1.0373069, + "epoch": 0.012622598804480297, + "flos": 74729493457920.0, + "grad_norm": 2.8512582014066283, + "language_loss": 0.92510194, + "learning_rate": 3.5010400135363173e-06, + "loss": 0.94812787, + "num_input_tokens_seen": 12041690, + "router_z_loss_clip": 1.72265625, + "router_z_loss_mlp": 0.35595703, + "step": 435, + "time_per_iteration": 2.8971903324127197 + }, + { + "auxiliary_loss_clip": 0.01054291, + "auxiliary_loss_mlp": 0.01007786, + "balance_loss_clip": 1.00576198, + "balance_loss_mlp": 1.00039542, + "epoch": 0.012651616272996343, + "flos": 74773865796480.0, + "grad_norm": 0.7191193407290016, + "language_loss": 0.56255794, + "learning_rate": 3.5023632519102177e-06, + "loss": 0.58317864, + "num_input_tokens_seen": 12108800, + "router_z_loss_clip": 0.484375, + "router_z_loss_mlp": 0.07373047, + "step": 436, + "time_per_iteration": 3.1485283374786377 + }, + { + "auxiliary_loss_clip": 0.01233086, + "auxiliary_loss_mlp": 0.01065096, + "balance_loss_clip": 1.06226015, + "balance_loss_mlp": 1.02609086, + "epoch": 0.01268063374151239, + "flos": 15697962368640.0, + "grad_norm": 3.7505339176372567, + "language_loss": 1.01759839, + "learning_rate": 3.503683458807525e-06, + "loss": 1.04058027, + "num_input_tokens_seen": 12120415, + "router_z_loss_clip": 1.70703125, + "router_z_loss_mlp": 0.38964844, + "step": 437, + "time_per_iteration": 2.518829584121704 + }, + { + "auxiliary_loss_clip": 0.01240026, + "auxiliary_loss_mlp": 0.01078809, + "balance_loss_clip": 1.06523693, + "balance_loss_mlp": 1.0426414, + "epoch": 0.012709651210028438, + "flos": 34597705057920.0, + "grad_norm": 2.5716376142939663, + "language_loss": 0.94044209, + "learning_rate": 3.505000648086437e-06, + "loss": 0.96363044, + "num_input_tokens_seen": 12139055, + "router_z_loss_clip": 1.74902344, + "router_z_loss_mlp": 0.36157227, + "step": 438, + "time_per_iteration": 2.589500904083252 + }, + { + "auxiliary_loss_clip": 0.0123325, + "auxiliary_loss_mlp": 0.01072093, + "balance_loss_clip": 1.06755638, + "balance_loss_mlp": 1.03489959, + "epoch": 0.012738668678544484, + "flos": 28286001803520.0, + "grad_norm": 2.379503741420389, + "language_loss": 0.99040782, + "learning_rate": 3.5063148335103383e-06, + "loss": 1.01346123, + "num_input_tokens_seen": 12156030, + "router_z_loss_clip": 1.65625, + "router_z_loss_mlp": 0.37182617, + "step": 439, + "time_per_iteration": 2.529754400253296 + }, + { + "auxiliary_loss_clip": 0.01249001, + "auxiliary_loss_mlp": 0.01082934, + "balance_loss_clip": 1.06445622, + "balance_loss_mlp": 1.04295075, + "epoch": 0.01276768614706053, + "flos": 24950503509120.0, + "grad_norm": 2.836527991277888, + "language_loss": 1.08623195, + "learning_rate": 3.507626028748667e-06, + "loss": 1.10955119, + "num_input_tokens_seen": 12175310, + "router_z_loss_clip": 1.84375, + "router_z_loss_mlp": 0.39990234, + "step": 440, + "time_per_iteration": 2.5360641479492188 + }, + { + "auxiliary_loss_clip": 0.01053266, + "auxiliary_loss_mlp": 0.0100798, + "balance_loss_clip": 1.00582123, + "balance_loss_mlp": 1.00130451, + "epoch": 0.012796703615576577, + "flos": 68350754793600.0, + "grad_norm": 0.7255717591364609, + "language_loss": 0.50223488, + "learning_rate": 3.508934247377766e-06, + "loss": 0.52284729, + "num_input_tokens_seen": 12236095, + "router_z_loss_clip": 0.47460938, + "router_z_loss_mlp": 0.06689453, + "step": 441, + "time_per_iteration": 3.068272829055786 + }, + { + "auxiliary_loss_clip": 0.01052705, + "auxiliary_loss_mlp": 0.01006641, + "balance_loss_clip": 1.00556755, + "balance_loss_mlp": 0.99996489, + "epoch": 0.012825721084092623, + "flos": 66709592513280.0, + "grad_norm": 0.7603181535863429, + "language_loss": 0.56330931, + "learning_rate": 3.510239502881726e-06, + "loss": 0.58390272, + "num_input_tokens_seen": 12292810, + "router_z_loss_clip": 0.47265625, + "router_z_loss_mlp": 0.06689453, + "step": 442, + "time_per_iteration": 3.080322265625 + }, + { + "auxiliary_loss_clip": 0.01237309, + "auxiliary_loss_mlp": 0.01069794, + "balance_loss_clip": 1.06171584, + "balance_loss_mlp": 1.03229105, + "epoch": 0.01285473855260867, + "flos": 27667201132800.0, + "grad_norm": 2.154495431555292, + "language_loss": 0.89384329, + "learning_rate": 3.5115418086532197e-06, + "loss": 0.91691434, + "num_input_tokens_seen": 12312710, + "router_z_loss_clip": 1.75390625, + "router_z_loss_mlp": 0.375, + "step": 443, + "time_per_iteration": 2.533439874649048 + }, + { + "auxiliary_loss_clip": 0.01244017, + "auxiliary_loss_mlp": 0.01095955, + "balance_loss_clip": 1.06095719, + "balance_loss_mlp": 1.05151415, + "epoch": 0.012883756021124716, + "flos": 26210694366720.0, + "grad_norm": 5.822292661254167, + "language_loss": 0.78859264, + "learning_rate": 3.512841177994327e-06, + "loss": 0.81199241, + "num_input_tokens_seen": 12333435, + "router_z_loss_clip": 1.82910156, + "router_z_loss_mlp": 0.44433594, + "step": 444, + "time_per_iteration": 2.8330812454223633 + }, + { + "auxiliary_loss_clip": 0.01249145, + "auxiliary_loss_mlp": 0.01081954, + "balance_loss_clip": 1.05868304, + "balance_loss_mlp": 1.03870451, + "epoch": 0.012912773489640764, + "flos": 16939194491520.0, + "grad_norm": 3.7163281192000985, + "language_loss": 0.98419511, + "learning_rate": 3.5141376241173505e-06, + "loss": 1.00750613, + "num_input_tokens_seen": 12346170, + "router_z_loss_clip": 1.90429688, + "router_z_loss_mlp": 0.43261719, + "step": 445, + "time_per_iteration": 2.431938409805298 + }, + { + "auxiliary_loss_clip": 0.01238396, + "auxiliary_loss_mlp": 0.01067771, + "balance_loss_clip": 1.06536198, + "balance_loss_mlp": 1.0270257, + "epoch": 0.01294179095815681, + "flos": 22849990070400.0, + "grad_norm": 4.220712698574699, + "language_loss": 1.01864815, + "learning_rate": 3.5154311601456196e-06, + "loss": 1.04170978, + "num_input_tokens_seen": 12359885, + "router_z_loss_clip": 1.73144531, + "router_z_loss_mlp": 0.4074707, + "step": 446, + "time_per_iteration": 2.444805145263672 + }, + { + "auxiliary_loss_clip": 0.0121848, + "auxiliary_loss_mlp": 0.01063871, + "balance_loss_clip": 1.05852866, + "balance_loss_mlp": 1.02773881, + "epoch": 0.012970808426672857, + "flos": 31641434484480.0, + "grad_norm": 3.638827848688381, + "language_loss": 1.13065839, + "learning_rate": 3.5167217991142907e-06, + "loss": 1.15348196, + "num_input_tokens_seen": 12373035, + "router_z_loss_clip": 1.59863281, + "router_z_loss_mlp": 0.36096191, + "step": 447, + "time_per_iteration": 2.509554862976074 + }, + { + "auxiliary_loss_clip": 0.01050177, + "auxiliary_loss_mlp": 0.01021083, + "balance_loss_clip": 1.00413477, + "balance_loss_mlp": 1.01469386, + "epoch": 0.012999825895188903, + "flos": 66636030545280.0, + "grad_norm": 0.7897524290914542, + "language_loss": 0.5918144, + "learning_rate": 3.5180095539711303e-06, + "loss": 0.61252695, + "num_input_tokens_seen": 12431200, + "router_z_loss_clip": 0.4609375, + "router_z_loss_mlp": 0.06396484, + "step": 448, + "time_per_iteration": 3.0264666080474854 + }, + { + "auxiliary_loss_clip": 0.01232075, + "auxiliary_loss_mlp": 0.01065767, + "balance_loss_clip": 1.06074166, + "balance_loss_mlp": 1.02981353, + "epoch": 0.01302884336370495, + "flos": 14494401469440.0, + "grad_norm": 4.046450892762601, + "language_loss": 0.98532873, + "learning_rate": 3.5192944375773016e-06, + "loss": 1.0083071, + "num_input_tokens_seen": 12443425, + "router_z_loss_clip": 1.71289062, + "router_z_loss_mlp": 0.359375, + "step": 449, + "time_per_iteration": 2.4435336589813232 + }, + { + "auxiliary_loss_clip": 0.01238189, + "auxiliary_loss_mlp": 0.01090537, + "balance_loss_clip": 1.05791879, + "balance_loss_mlp": 1.05017328, + "epoch": 0.013057860832220996, + "flos": 30847753480320.0, + "grad_norm": 2.480103050542491, + "language_loss": 0.99186397, + "learning_rate": 3.5205764627081286e-06, + "loss": 1.01515126, + "num_input_tokens_seen": 12463685, + "router_z_loss_clip": 1.80175781, + "router_z_loss_mlp": 0.40356445, + "step": 450, + "time_per_iteration": 2.543992280960083 + }, + { + "auxiliary_loss_clip": 0.01231517, + "auxiliary_loss_mlp": 0.01063425, + "balance_loss_clip": 1.05646992, + "balance_loss_mlp": 1.02601767, + "epoch": 0.013086878300737044, + "flos": 23398231150080.0, + "grad_norm": 2.4389608050359284, + "language_loss": 1.06838131, + "learning_rate": 3.521855642053862e-06, + "loss": 1.09133077, + "num_input_tokens_seen": 12481025, + "router_z_loss_clip": 1.74902344, + "router_z_loss_mlp": 0.3737793, + "step": 451, + "time_per_iteration": 2.503248691558838 + }, + { + "auxiliary_loss_clip": 0.01229353, + "auxiliary_loss_mlp": 0.01067444, + "balance_loss_clip": 1.05772936, + "balance_loss_mlp": 1.03077567, + "epoch": 0.01311589576925309, + "flos": 17302396032000.0, + "grad_norm": 3.2775028976184095, + "language_loss": 0.92597169, + "learning_rate": 3.5231319882204308e-06, + "loss": 0.94893968, + "num_input_tokens_seen": 12492870, + "router_z_loss_clip": 1.71777344, + "router_z_loss_mlp": 0.36669922, + "step": 452, + "time_per_iteration": 2.456177234649658 + }, + { + "auxiliary_loss_clip": 0.01231441, + "auxiliary_loss_mlp": 0.01091163, + "balance_loss_clip": 1.05872488, + "balance_loss_mlp": 1.0516572, + "epoch": 0.013144913237769137, + "flos": 34522362610560.0, + "grad_norm": 2.71727122073947, + "language_loss": 0.88935733, + "learning_rate": 3.524405513730189e-06, + "loss": 0.91258335, + "num_input_tokens_seen": 12506590, + "router_z_loss_clip": 1.72753906, + "router_z_loss_mlp": 0.39489746, + "step": 453, + "time_per_iteration": 2.543659210205078 + }, + { + "auxiliary_loss_clip": 0.01238745, + "auxiliary_loss_mlp": 0.01070868, + "balance_loss_clip": 1.0647316, + "balance_loss_mlp": 1.03093243, + "epoch": 0.013173930706285183, + "flos": 16756354368000.0, + "grad_norm": 13.517920969657322, + "language_loss": 0.81427932, + "learning_rate": 3.5256762310226537e-06, + "loss": 0.83737546, + "num_input_tokens_seen": 12519175, + "router_z_loss_clip": 1.73828125, + "router_z_loss_mlp": 0.39916992, + "step": 454, + "time_per_iteration": 2.5563690662384033 + }, + { + "auxiliary_loss_clip": 0.0122992, + "auxiliary_loss_mlp": 0.01073072, + "balance_loss_clip": 1.05936015, + "balance_loss_mlp": 1.03478205, + "epoch": 0.01320294817480123, + "flos": 11319085825920.0, + "grad_norm": 7.732469453409506, + "language_loss": 0.95478237, + "learning_rate": 3.52694415245523e-06, + "loss": 0.97781229, + "num_input_tokens_seen": 12531215, + "router_z_loss_clip": 1.70507812, + "router_z_loss_mlp": 0.38330078, + "step": 455, + "time_per_iteration": 2.4107720851898193 + }, + { + "auxiliary_loss_clip": 0.01230492, + "auxiliary_loss_mlp": 0.01064312, + "balance_loss_clip": 1.06256604, + "balance_loss_mlp": 1.0275476, + "epoch": 0.013231965643317276, + "flos": 41061943509120.0, + "grad_norm": 4.41123818127912, + "language_loss": 0.96144974, + "learning_rate": 3.5282092903039383e-06, + "loss": 0.98439783, + "num_input_tokens_seen": 12552195, + "router_z_loss_clip": 1.6796875, + "router_z_loss_mlp": 0.36767578, + "step": 456, + "time_per_iteration": 2.6315183639526367 + }, + { + "auxiliary_loss_clip": 0.01244332, + "auxiliary_loss_mlp": 0.01086583, + "balance_loss_clip": 1.06191218, + "balance_loss_mlp": 1.04714847, + "epoch": 0.013260983111833324, + "flos": 14786030989440.0, + "grad_norm": 4.408979524214026, + "language_loss": 0.90261197, + "learning_rate": 3.529471656764121e-06, + "loss": 0.92592114, + "num_input_tokens_seen": 12567845, + "router_z_loss_clip": 1.82519531, + "router_z_loss_mlp": 0.39453125, + "step": 457, + "time_per_iteration": 2.4234519004821777 + }, + { + "auxiliary_loss_clip": 0.01252217, + "auxiliary_loss_mlp": 0.01084958, + "balance_loss_clip": 1.0637002, + "balance_loss_mlp": 1.04399753, + "epoch": 0.01329000058034937, + "flos": 74732914771200.0, + "grad_norm": 4.036131581968372, + "language_loss": 0.96857983, + "learning_rate": 3.5307312639511536e-06, + "loss": 0.99195158, + "num_input_tokens_seen": 12594020, + "router_z_loss_clip": 1.88476562, + "router_z_loss_mlp": 0.40966797, + "step": 458, + "time_per_iteration": 2.893519401550293 + }, + { + "auxiliary_loss_clip": 0.01050168, + "auxiliary_loss_mlp": 0.01009404, + "balance_loss_clip": 1.00409162, + "balance_loss_mlp": 1.00315726, + "epoch": 0.013319018048865417, + "flos": 59847902611200.0, + "grad_norm": 0.816942837373143, + "language_loss": 0.58834779, + "learning_rate": 3.531988123901137e-06, + "loss": 0.60894346, + "num_input_tokens_seen": 12657785, + "router_z_loss_clip": 0.4609375, + "router_z_loss_mlp": 0.06225586, + "step": 459, + "time_per_iteration": 3.124546766281128 + }, + { + "auxiliary_loss_clip": 0.01221788, + "auxiliary_loss_mlp": 0.01063002, + "balance_loss_clip": 1.05534649, + "balance_loss_mlp": 1.02654743, + "epoch": 0.013348035517381463, + "flos": 39454577291520.0, + "grad_norm": 2.294210893858189, + "language_loss": 0.93281162, + "learning_rate": 3.533242248571593e-06, + "loss": 0.95565957, + "num_input_tokens_seen": 12673185, + "router_z_loss_clip": 1.66503906, + "router_z_loss_mlp": 0.36425781, + "step": 460, + "time_per_iteration": 2.647644519805908 + }, + { + "auxiliary_loss_clip": 0.01050614, + "auxiliary_loss_mlp": 0.01008515, + "balance_loss_clip": 1.00425518, + "balance_loss_mlp": 1.00222063, + "epoch": 0.01337705298589751, + "flos": 68033812671360.0, + "grad_norm": 0.7329859744391293, + "language_loss": 0.57657492, + "learning_rate": 3.5344936498421413e-06, + "loss": 0.59716624, + "num_input_tokens_seen": 12736475, + "router_z_loss_clip": 0.46289062, + "router_z_loss_mlp": 0.06298828, + "step": 461, + "time_per_iteration": 3.118741273880005 + }, + { + "auxiliary_loss_clip": 0.01239585, + "auxiliary_loss_mlp": 0.01072871, + "balance_loss_clip": 1.06308222, + "balance_loss_mlp": 1.03462863, + "epoch": 0.013406070454413556, + "flos": 28872542511360.0, + "grad_norm": 2.0923778876659416, + "language_loss": 0.82330704, + "learning_rate": 3.5357423395151797e-06, + "loss": 0.84643161, + "num_input_tokens_seen": 12757220, + "router_z_loss_clip": 1.765625, + "router_z_loss_mlp": 0.38256836, + "step": 462, + "time_per_iteration": 5.241103410720825 + }, + { + "auxiliary_loss_clip": 0.01237505, + "auxiliary_loss_mlp": 0.0107689, + "balance_loss_clip": 1.05975461, + "balance_loss_mlp": 1.03776526, + "epoch": 0.013435087922929604, + "flos": 24818044210560.0, + "grad_norm": 2.9107707157630616, + "language_loss": 0.82470113, + "learning_rate": 3.536988329316549e-06, + "loss": 0.84784508, + "num_input_tokens_seen": 12772730, + "router_z_loss_clip": 1.77832031, + "router_z_loss_mlp": 0.39111328, + "step": 463, + "time_per_iteration": 4.8746466636657715 + }, + { + "auxiliary_loss_clip": 0.01239278, + "auxiliary_loss_mlp": 0.01072357, + "balance_loss_clip": 1.06105936, + "balance_loss_mlp": 1.02734375, + "epoch": 0.01346410539144565, + "flos": 26643408157440.0, + "grad_norm": 2.327675794154874, + "language_loss": 0.96837258, + "learning_rate": 3.5382316308961943e-06, + "loss": 0.99148899, + "num_input_tokens_seen": 12785895, + "router_z_loss_clip": 1.78417969, + "router_z_loss_mlp": 0.45019531, + "step": 464, + "time_per_iteration": 4.875885248184204 + }, + { + "auxiliary_loss_clip": 0.01226209, + "auxiliary_loss_mlp": 0.01072806, + "balance_loss_clip": 1.05771685, + "balance_loss_mlp": 1.03468251, + "epoch": 0.013493122859961697, + "flos": 30224204864640.0, + "grad_norm": 1.9848455189214094, + "language_loss": 0.90239179, + "learning_rate": 3.5394722558288188e-06, + "loss": 0.92538196, + "num_input_tokens_seen": 12809695, + "router_z_loss_clip": 1.68457031, + "router_z_loss_mlp": 0.38110352, + "step": 465, + "time_per_iteration": 5.02630090713501 + }, + { + "auxiliary_loss_clip": 0.01229973, + "auxiliary_loss_mlp": 0.01070705, + "balance_loss_clip": 1.05866265, + "balance_loss_mlp": 1.03179514, + "epoch": 0.013522140328477743, + "flos": 31240980656640.0, + "grad_norm": 3.085967662678346, + "language_loss": 0.8794533, + "learning_rate": 3.5407102156145306e-06, + "loss": 0.90245998, + "num_input_tokens_seen": 12827885, + "router_z_loss_clip": 1.71289062, + "router_z_loss_mlp": 0.38916016, + "step": 466, + "time_per_iteration": 2.51175856590271 + }, + { + "auxiliary_loss_clip": 0.01050953, + "auxiliary_loss_mlp": 0.01010247, + "balance_loss_clip": 1.00528133, + "balance_loss_mlp": 1.00380969, + "epoch": 0.01355115779699379, + "flos": 74779870550400.0, + "grad_norm": 0.7336173728209818, + "language_loss": 0.59512281, + "learning_rate": 3.5419455216794824e-06, + "loss": 0.61573482, + "num_input_tokens_seen": 12887480, + "router_z_loss_clip": 0.45703125, + "router_z_loss_mlp": 0.06445312, + "step": 467, + "time_per_iteration": 3.127854108810425 + }, + { + "auxiliary_loss_clip": 0.01050269, + "auxiliary_loss_mlp": 0.0101017, + "balance_loss_clip": 1.00477338, + "balance_loss_mlp": 1.00373292, + "epoch": 0.013580175265509836, + "flos": 63322144248960.0, + "grad_norm": 0.6454870713227916, + "language_loss": 0.59510732, + "learning_rate": 3.543178185376502e-06, + "loss": 0.61571169, + "num_input_tokens_seen": 12950825, + "router_z_loss_clip": 0.45507812, + "router_z_loss_mlp": 0.06445312, + "step": 468, + "time_per_iteration": 3.0699310302734375 + }, + { + "auxiliary_loss_clip": 0.01236542, + "auxiliary_loss_mlp": 0.01083873, + "balance_loss_clip": 1.06519175, + "balance_loss_mlp": 1.04443824, + "epoch": 0.013609192734025884, + "flos": 11393066730240.0, + "grad_norm": 3.896473544007973, + "language_loss": 0.80563164, + "learning_rate": 3.5444082179857223e-06, + "loss": 0.82883573, + "num_input_tokens_seen": 12964055, + "router_z_loss_clip": 1.71484375, + "router_z_loss_mlp": 0.39428711, + "step": 469, + "time_per_iteration": 2.5123417377471924 + }, + { + "auxiliary_loss_clip": 0.01236958, + "auxiliary_loss_mlp": 0.01067765, + "balance_loss_clip": 1.05934834, + "balance_loss_mlp": 1.02587461, + "epoch": 0.01363821020254193, + "flos": 22375555338240.0, + "grad_norm": 3.841785199113265, + "language_loss": 1.14334714, + "learning_rate": 3.545635630715198e-06, + "loss": 1.16639423, + "num_input_tokens_seen": 12977770, + "router_z_loss_clip": 1.77636719, + "router_z_loss_mlp": 0.41918945, + "step": 470, + "time_per_iteration": 2.4253506660461426 + }, + { + "auxiliary_loss_clip": 0.01231637, + "auxiliary_loss_mlp": 0.0107383, + "balance_loss_clip": 1.05997443, + "balance_loss_mlp": 1.03494382, + "epoch": 0.013667227671057977, + "flos": 42878998552320.0, + "grad_norm": 3.444550174534543, + "language_loss": 0.85939407, + "learning_rate": 3.546860434701518e-06, + "loss": 0.88244879, + "num_input_tokens_seen": 12995525, + "router_z_loss_clip": 1.71582031, + "router_z_loss_mlp": 0.38891602, + "step": 471, + "time_per_iteration": 2.699951410293579 + }, + { + "auxiliary_loss_clip": 0.01231499, + "auxiliary_loss_mlp": 0.01079908, + "balance_loss_clip": 1.05617607, + "balance_loss_mlp": 1.03813672, + "epoch": 0.013696245139574023, + "flos": 21135440378880.0, + "grad_norm": 3.6046474236150825, + "language_loss": 0.88066316, + "learning_rate": 3.548082641010414e-06, + "loss": 0.90377724, + "num_input_tokens_seen": 13010905, + "router_z_loss_clip": 1.75390625, + "router_z_loss_mlp": 0.41748047, + "step": 472, + "time_per_iteration": 2.405672788619995 + }, + { + "auxiliary_loss_clip": 0.01048285, + "auxiliary_loss_mlp": 0.01013256, + "balance_loss_clip": 1.00300837, + "balance_loss_mlp": 1.00700986, + "epoch": 0.01372526260809007, + "flos": 74786608442880.0, + "grad_norm": 0.6839798472914224, + "language_loss": 0.54120296, + "learning_rate": 3.5493022606373578e-06, + "loss": 0.56181836, + "num_input_tokens_seen": 13080835, + "router_z_loss_clip": 0.453125, + "router_z_loss_mlp": 0.0625, + "step": 473, + "time_per_iteration": 3.218013286590576 + }, + { + "auxiliary_loss_clip": 0.0122375, + "auxiliary_loss_mlp": 0.01069224, + "balance_loss_clip": 1.05600047, + "balance_loss_mlp": 1.03358078, + "epoch": 0.013754280076606116, + "flos": 16508994318720.0, + "grad_norm": 3.324542048470533, + "language_loss": 0.88000929, + "learning_rate": 3.550519304508158e-06, + "loss": 0.90293896, + "num_input_tokens_seen": 13095550, + "router_z_loss_clip": 1.67480469, + "router_z_loss_mlp": 0.35644531, + "step": 474, + "time_per_iteration": 2.3647897243499756 + }, + { + "auxiliary_loss_clip": 0.0123349, + "auxiliary_loss_mlp": 0.01089487, + "balance_loss_clip": 1.06203854, + "balance_loss_mlp": 1.04921818, + "epoch": 0.013783297545122164, + "flos": 12488082612480.0, + "grad_norm": 7.133368913537497, + "language_loss": 0.90503585, + "learning_rate": 3.551733783479541e-06, + "loss": 0.92826563, + "num_input_tokens_seen": 13108305, + "router_z_loss_clip": 1.71484375, + "router_z_loss_mlp": 0.40307617, + "step": 475, + "time_per_iteration": 2.5177807807922363 + }, + { + "auxiliary_loss_clip": 0.01225385, + "auxiliary_loss_mlp": 0.01074145, + "balance_loss_clip": 1.05722535, + "balance_loss_mlp": 1.037238, + "epoch": 0.01381231501363821, + "flos": 56962612431360.0, + "grad_norm": 4.712509317873693, + "language_loss": 0.94165945, + "learning_rate": 3.552945708339742e-06, + "loss": 0.9646548, + "num_input_tokens_seen": 13128735, + "router_z_loss_clip": 1.68164062, + "router_z_loss_mlp": 0.36914062, + "step": 476, + "time_per_iteration": 2.7438230514526367 + }, + { + "auxiliary_loss_clip": 0.01229959, + "auxiliary_loss_mlp": 0.01071767, + "balance_loss_clip": 1.05742168, + "balance_loss_mlp": 1.03111696, + "epoch": 0.013841332482154257, + "flos": 16206821568000.0, + "grad_norm": 8.720350065332672, + "language_loss": 0.85732913, + "learning_rate": 3.5541550898090704e-06, + "loss": 0.88034636, + "num_input_tokens_seen": 13139930, + "router_z_loss_clip": 1.72363281, + "router_z_loss_mlp": 0.40673828, + "step": 477, + "time_per_iteration": 2.3960981369018555 + }, + { + "auxiliary_loss_clip": 0.01235164, + "auxiliary_loss_mlp": 0.01080081, + "balance_loss_clip": 1.06082797, + "balance_loss_mlp": 1.04143369, + "epoch": 0.013870349950670303, + "flos": 55732935968640.0, + "grad_norm": 3.090735972313077, + "language_loss": 0.79178286, + "learning_rate": 3.5553619385404838e-06, + "loss": 0.81493533, + "num_input_tokens_seen": 13160645, + "router_z_loss_clip": 1.74316406, + "router_z_loss_mlp": 0.38671875, + "step": 478, + "time_per_iteration": 2.655416250228882 + }, + { + "auxiliary_loss_clip": 0.01218627, + "auxiliary_loss_mlp": 0.01066351, + "balance_loss_clip": 1.05384254, + "balance_loss_mlp": 1.03292489, + "epoch": 0.01389936741918635, + "flos": 11647897810560.0, + "grad_norm": 3.388396033818726, + "language_loss": 0.74855614, + "learning_rate": 3.5565662651201502e-06, + "loss": 0.77140594, + "num_input_tokens_seen": 13172570, + "router_z_loss_clip": 1.64746094, + "router_z_loss_mlp": 0.33447266, + "step": 479, + "time_per_iteration": 2.465557813644409 + }, + { + "auxiliary_loss_clip": 0.0123013, + "auxiliary_loss_mlp": 0.01084888, + "balance_loss_clip": 1.06242204, + "balance_loss_mlp": 1.04604983, + "epoch": 0.013928384887702396, + "flos": 37370227812480.0, + "grad_norm": 2.2639468154705087, + "language_loss": 0.91074038, + "learning_rate": 3.5577680800680056e-06, + "loss": 0.93389052, + "num_input_tokens_seen": 13190760, + "router_z_loss_clip": 1.67675781, + "router_z_loss_mlp": 0.38818359, + "step": 480, + "time_per_iteration": 2.5436387062072754 + }, + { + "auxiliary_loss_clip": 0.01234198, + "auxiliary_loss_mlp": 0.01071042, + "balance_loss_clip": 1.06072044, + "balance_loss_mlp": 1.0346837, + "epoch": 0.013957402356218444, + "flos": 37734790896000.0, + "grad_norm": 2.6287682683479354, + "language_loss": 0.84138262, + "learning_rate": 3.5589673938383033e-06, + "loss": 0.86443502, + "num_input_tokens_seen": 13212150, + "router_z_loss_clip": 1.73632812, + "router_z_loss_mlp": 0.36352539, + "step": 481, + "time_per_iteration": 2.7530035972595215 + }, + { + "auxiliary_loss_clip": 0.01228087, + "auxiliary_loss_mlp": 0.01062844, + "balance_loss_clip": 1.05652237, + "balance_loss_mlp": 1.02591324, + "epoch": 0.01398641982473449, + "flos": 32121105920640.0, + "grad_norm": 2.097192814655506, + "language_loss": 1.09813297, + "learning_rate": 3.5601642168201625e-06, + "loss": 1.12104225, + "num_input_tokens_seen": 13235835, + "router_z_loss_clip": 1.71582031, + "router_z_loss_mlp": 0.36914062, + "step": 482, + "time_per_iteration": 2.617661237716675 + }, + { + "auxiliary_loss_clip": 0.01223423, + "auxiliary_loss_mlp": 0.01072119, + "balance_loss_clip": 1.05226898, + "balance_loss_mlp": 1.03518772, + "epoch": 0.014015437293250537, + "flos": 33907925859840.0, + "grad_norm": 2.127060187704156, + "language_loss": 1.05158043, + "learning_rate": 3.5613585593381047e-06, + "loss": 1.07453585, + "num_input_tokens_seen": 13255755, + "router_z_loss_clip": 1.71191406, + "router_z_loss_mlp": 0.36914062, + "step": 483, + "time_per_iteration": 2.5261263847351074 + }, + { + "auxiliary_loss_clip": 0.01231125, + "auxiliary_loss_mlp": 0.01074124, + "balance_loss_clip": 1.05846858, + "balance_loss_mlp": 1.03714514, + "epoch": 0.014044454761766583, + "flos": 16032814018560.0, + "grad_norm": 3.2008867763878723, + "language_loss": 0.87128532, + "learning_rate": 3.5625504316525934e-06, + "loss": 0.89433777, + "num_input_tokens_seen": 13269210, + "router_z_loss_clip": 1.72558594, + "router_z_loss_mlp": 0.36987305, + "step": 484, + "time_per_iteration": 2.446773052215576 + }, + { + "auxiliary_loss_clip": 0.01230104, + "auxiliary_loss_mlp": 0.01075281, + "balance_loss_clip": 1.05689466, + "balance_loss_mlp": 1.03658605, + "epoch": 0.01407347223028263, + "flos": 22633179327360.0, + "grad_norm": 2.7448388514245186, + "language_loss": 0.76148278, + "learning_rate": 3.5637398439605558e-06, + "loss": 0.7845366, + "num_input_tokens_seen": 13283685, + "router_z_loss_clip": 1.73144531, + "router_z_loss_mlp": 0.38720703, + "step": 485, + "time_per_iteration": 2.4394593238830566 + }, + { + "auxiliary_loss_clip": 0.01236477, + "auxiliary_loss_mlp": 0.01078712, + "balance_loss_clip": 1.06012833, + "balance_loss_mlp": 1.04194772, + "epoch": 0.014102489698798676, + "flos": 19565082069120.0, + "grad_norm": 4.334488029858283, + "language_loss": 0.82574821, + "learning_rate": 3.5649268063959134e-06, + "loss": 0.84890014, + "num_input_tokens_seen": 13297210, + "router_z_loss_clip": 1.76464844, + "router_z_loss_mlp": 0.36743164, + "step": 486, + "time_per_iteration": 2.452425003051758 + }, + { + "auxiliary_loss_clip": 0.01213391, + "auxiliary_loss_mlp": 0.01080502, + "balance_loss_clip": 1.05864429, + "balance_loss_mlp": 1.0469327, + "epoch": 0.014131507167314724, + "flos": 15078114357120.0, + "grad_norm": 3.692905724734751, + "language_loss": 0.84963071, + "learning_rate": 3.566111329030094e-06, + "loss": 0.87256962, + "num_input_tokens_seen": 13311410, + "router_z_loss_clip": 1.546875, + "router_z_loss_mlp": 0.3359375, + "step": 487, + "time_per_iteration": 2.4880599975585938 + }, + { + "auxiliary_loss_clip": 0.0105097, + "auxiliary_loss_mlp": 0.01024772, + "balance_loss_clip": 1.00453544, + "balance_loss_mlp": 1.01804841, + "epoch": 0.01416052463583077, + "flos": 64223986245120.0, + "grad_norm": 0.7400686316958199, + "language_loss": 0.54051948, + "learning_rate": 3.567293421872552e-06, + "loss": 0.56127685, + "num_input_tokens_seen": 13372965, + "router_z_loss_clip": 0.46484375, + "router_z_loss_mlp": 0.06738281, + "step": 488, + "time_per_iteration": 3.0368471145629883 + }, + { + "auxiliary_loss_clip": 0.01219063, + "auxiliary_loss_mlp": 0.01070546, + "balance_loss_clip": 1.0566752, + "balance_loss_mlp": 1.03673851, + "epoch": 0.014189542104346817, + "flos": 28687153858560.0, + "grad_norm": 2.631880047957373, + "language_loss": 0.91351658, + "learning_rate": 3.568473094871265e-06, + "loss": 0.93641269, + "num_input_tokens_seen": 13388965, + "router_z_loss_clip": 1.62402344, + "router_z_loss_mlp": 0.33789062, + "step": 489, + "time_per_iteration": 2.465468645095825 + }, + { + "auxiliary_loss_clip": 0.01224177, + "auxiliary_loss_mlp": 0.01059921, + "balance_loss_clip": 1.05752134, + "balance_loss_mlp": 1.02425337, + "epoch": 0.014218559572862863, + "flos": 38536117488000.0, + "grad_norm": 2.7320119082976655, + "language_loss": 1.06925797, + "learning_rate": 3.5696503579132456e-06, + "loss": 1.09209907, + "num_input_tokens_seen": 13405945, + "router_z_loss_clip": 1.66601562, + "router_z_loss_mlp": 0.3560791, + "step": 490, + "time_per_iteration": 2.618497133255005 + }, + { + "auxiliary_loss_clip": 0.01054107, + "auxiliary_loss_mlp": 0.01008326, + "balance_loss_clip": 1.00657499, + "balance_loss_mlp": 1.00145984, + "epoch": 0.01424757704137891, + "flos": 67190346201600.0, + "grad_norm": 0.7122164959803263, + "language_loss": 0.49091375, + "learning_rate": 3.570825220825037e-06, + "loss": 0.51153803, + "num_input_tokens_seen": 13462405, + "router_z_loss_clip": 0.4765625, + "router_z_loss_mlp": 0.06884766, + "step": 491, + "time_per_iteration": 3.042903184890747 + }, + { + "auxiliary_loss_clip": 0.01227977, + "auxiliary_loss_mlp": 0.01069372, + "balance_loss_clip": 1.05956829, + "balance_loss_mlp": 1.03217888, + "epoch": 0.014276594509894956, + "flos": 21828221953920.0, + "grad_norm": 2.40168329964105, + "language_loss": 0.75713289, + "learning_rate": 3.5719976933732e-06, + "loss": 0.78010643, + "num_input_tokens_seen": 13477160, + "router_z_loss_clip": 1.68457031, + "router_z_loss_mlp": 0.37207031, + "step": 492, + "time_per_iteration": 2.433035373687744 + }, + { + "auxiliary_loss_clip": 0.01236435, + "auxiliary_loss_mlp": 0.01083236, + "balance_loss_clip": 1.06317365, + "balance_loss_mlp": 1.04749656, + "epoch": 0.014305611978411003, + "flos": 17996608972800.0, + "grad_norm": 2.864623011241668, + "language_loss": 0.9406625, + "learning_rate": 3.5731677852648057e-06, + "loss": 0.9638592, + "num_input_tokens_seen": 13491465, + "router_z_loss_clip": 1.734375, + "router_z_loss_mlp": 0.35717773, + "step": 493, + "time_per_iteration": 2.4602136611938477 + }, + { + "auxiliary_loss_clip": 0.01240647, + "auxiliary_loss_mlp": 0.01082262, + "balance_loss_clip": 1.06154311, + "balance_loss_mlp": 1.04292309, + "epoch": 0.01433462944692705, + "flos": 27665141362560.0, + "grad_norm": 3.369538229206316, + "language_loss": 0.79701722, + "learning_rate": 3.5743355061479145e-06, + "loss": 0.82024634, + "num_input_tokens_seen": 13506200, + "router_z_loss_clip": 1.79101562, + "router_z_loss_mlp": 0.39331055, + "step": 494, + "time_per_iteration": 2.5184977054595947 + }, + { + "auxiliary_loss_clip": 0.01069806, + "auxiliary_loss_mlp": 0.01015674, + "balance_loss_clip": 1.02020764, + "balance_loss_mlp": 1.00914121, + "epoch": 0.014363646915443097, + "flos": 74774005441920.0, + "grad_norm": 0.9772998129823895, + "language_loss": 0.54735613, + "learning_rate": 3.5755008656120545e-06, + "loss": 0.56821096, + "num_input_tokens_seen": 13572210, + "router_z_loss_clip": 0.49609375, + "router_z_loss_mlp": 0.06542969, + "step": 495, + "time_per_iteration": 3.1317973136901855 + }, + { + "auxiliary_loss_clip": 0.01228069, + "auxiliary_loss_mlp": 0.01071306, + "balance_loss_clip": 1.05970573, + "balance_loss_mlp": 1.03338587, + "epoch": 0.014392664383959143, + "flos": 35290451721600.0, + "grad_norm": 2.686237236763118, + "language_loss": 1.15993202, + "learning_rate": 3.5766638731886958e-06, + "loss": 1.1829257, + "num_input_tokens_seen": 13592030, + "router_z_loss_clip": 1.68457031, + "router_z_loss_mlp": 0.37915039, + "step": 496, + "time_per_iteration": 2.6954095363616943 + }, + { + "auxiliary_loss_clip": 0.01231551, + "auxiliary_loss_mlp": 0.01074291, + "balance_loss_clip": 1.06237125, + "balance_loss_mlp": 1.03633499, + "epoch": 0.01442168185247519, + "flos": 31970421025920.0, + "grad_norm": 2.49690497815799, + "language_loss": 0.94868052, + "learning_rate": 3.5778245383517136e-06, + "loss": 0.97173905, + "num_input_tokens_seen": 13615635, + "router_z_loss_clip": 1.69335938, + "router_z_loss_mlp": 0.37939453, + "step": 497, + "time_per_iteration": 2.7072815895080566 + }, + { + "auxiliary_loss_clip": 0.01222628, + "auxiliary_loss_mlp": 0.01081712, + "balance_loss_clip": 1.05854797, + "balance_loss_mlp": 1.04556763, + "epoch": 0.014450699320991236, + "flos": 26679263990400.0, + "grad_norm": 3.05993207472896, + "language_loss": 0.96546626, + "learning_rate": 3.5789828705178567e-06, + "loss": 0.9885096, + "num_input_tokens_seen": 13630230, + "router_z_loss_clip": 1.64257812, + "router_z_loss_mlp": 0.36132812, + "step": 498, + "time_per_iteration": 2.481520652770996 + }, + { + "auxiliary_loss_clip": 0.01059427, + "auxiliary_loss_mlp": 0.0100547, + "balance_loss_clip": 1.00939393, + "balance_loss_mlp": 0.99927127, + "epoch": 0.014479716789507283, + "flos": 74770304837760.0, + "grad_norm": 0.6773216570954174, + "language_loss": 0.52877951, + "learning_rate": 3.5801388790472013e-06, + "loss": 0.54942852, + "num_input_tokens_seen": 13693550, + "router_z_loss_clip": 0.5, + "router_z_loss_mlp": 0.06176758, + "step": 499, + "time_per_iteration": 3.1644248962402344 + }, + { + "auxiliary_loss_clip": 0.01228055, + "auxiliary_loss_mlp": 0.01077586, + "balance_loss_clip": 1.06088245, + "balance_loss_mlp": 1.03669739, + "epoch": 0.01450873425802333, + "flos": 27446829431040.0, + "grad_norm": 2.761763589021657, + "language_loss": 0.88287711, + "learning_rate": 3.5812925732436083e-06, + "loss": 0.90593356, + "num_input_tokens_seen": 13711270, + "router_z_loss_clip": 1.67089844, + "router_z_loss_mlp": 0.40942383, + "step": 500, + "time_per_iteration": 2.5466952323913574 + }, + { + "auxiliary_loss_clip": 0.01229927, + "auxiliary_loss_mlp": 0.01086181, + "balance_loss_clip": 1.06138706, + "balance_loss_mlp": 1.04901147, + "epoch": 0.014537751726539377, + "flos": 29527443394560.0, + "grad_norm": 2.8233968009583066, + "language_loss": 0.76306695, + "learning_rate": 3.582443962355171e-06, + "loss": 0.78622806, + "num_input_tokens_seen": 13727425, + "router_z_loss_clip": 1.68457031, + "router_z_loss_mlp": 0.37207031, + "step": 501, + "time_per_iteration": 2.569211721420288 + }, + { + "auxiliary_loss_clip": 0.01229513, + "auxiliary_loss_mlp": 0.01087995, + "balance_loss_clip": 1.06046546, + "balance_loss_mlp": 1.04887033, + "epoch": 0.014566769195055423, + "flos": 15989871179520.0, + "grad_norm": 4.098810552065202, + "language_loss": 0.96928847, + "learning_rate": 3.5835930555746595e-06, + "loss": 0.99246359, + "num_input_tokens_seen": 13739685, + "router_z_loss_clip": 1.69140625, + "router_z_loss_mlp": 0.39135742, + "step": 502, + "time_per_iteration": 2.4990339279174805 + }, + { + "auxiliary_loss_clip": 0.0105941, + "auxiliary_loss_mlp": 0.01013659, + "balance_loss_clip": 1.0093565, + "balance_loss_mlp": 1.00762677, + "epoch": 0.01459578666357147, + "flos": 60913241971200.0, + "grad_norm": 0.7603702870749693, + "language_loss": 0.59640956, + "learning_rate": 3.584739862039961e-06, + "loss": 0.61714029, + "num_input_tokens_seen": 13804690, + "router_z_loss_clip": 0.5, + "router_z_loss_mlp": 0.06030273, + "step": 503, + "time_per_iteration": 3.0849997997283936 + }, + { + "auxiliary_loss_clip": 0.01228092, + "auxiliary_loss_mlp": 0.01086167, + "balance_loss_clip": 1.05741119, + "balance_loss_mlp": 1.04191637, + "epoch": 0.014624804132087516, + "flos": 27487293563520.0, + "grad_norm": 2.6167407494148716, + "language_loss": 1.10838103, + "learning_rate": 3.5858843908345178e-06, + "loss": 1.13152361, + "num_input_tokens_seen": 13823020, + "router_z_loss_clip": 1.70800781, + "router_z_loss_mlp": 0.44238281, + "step": 504, + "time_per_iteration": 2.463632822036743 + }, + { + "auxiliary_loss_clip": 0.01228189, + "auxiliary_loss_mlp": 0.01086618, + "balance_loss_clip": 1.05609202, + "balance_loss_mlp": 1.04296339, + "epoch": 0.014653821600603563, + "flos": 12487209828480.0, + "grad_norm": 5.076347306843628, + "language_loss": 0.99482703, + "learning_rate": 3.5870266509877573e-06, + "loss": 1.01797509, + "num_input_tokens_seen": 13834385, + "router_z_loss_clip": 1.72167969, + "router_z_loss_mlp": 0.43676758, + "step": 505, + "time_per_iteration": 2.4023609161376953 + }, + { + "auxiliary_loss_clip": 0.01219129, + "auxiliary_loss_mlp": 0.01079713, + "balance_loss_clip": 1.06135583, + "balance_loss_mlp": 1.0467155, + "epoch": 0.01468283906911961, + "flos": 18581578669440.0, + "grad_norm": 2.824665729687399, + "language_loss": 1.02690482, + "learning_rate": 3.588166651475519e-06, + "loss": 1.04989314, + "num_input_tokens_seen": 13846355, + "router_z_loss_clip": 1.57714844, + "router_z_loss_mlp": 0.33007812, + "step": 506, + "time_per_iteration": 2.3971946239471436 + }, + { + "auxiliary_loss_clip": 0.01214347, + "auxiliary_loss_mlp": 0.01076884, + "balance_loss_clip": 1.05519736, + "balance_loss_mlp": 1.04233754, + "epoch": 0.014711856537635657, + "flos": 24783689566080.0, + "grad_norm": 2.551099266949821, + "language_loss": 1.0001471, + "learning_rate": 3.5893044012204783e-06, + "loss": 1.02305949, + "num_input_tokens_seen": 13860710, + "router_z_loss_clip": 1.59082031, + "router_z_loss_mlp": 0.3449707, + "step": 507, + "time_per_iteration": 2.4626779556274414 + }, + { + "auxiliary_loss_clip": 0.01222814, + "auxiliary_loss_mlp": 0.01064716, + "balance_loss_clip": 1.05789232, + "balance_loss_mlp": 1.0282861, + "epoch": 0.014740874006151703, + "flos": 18143488529280.0, + "grad_norm": 2.9267912127824864, + "language_loss": 0.88121748, + "learning_rate": 3.5904399090925674e-06, + "loss": 0.90409279, + "num_input_tokens_seen": 13875745, + "router_z_loss_clip": 1.64941406, + "router_z_loss_mlp": 0.36425781, + "step": 508, + "time_per_iteration": 2.4084532260894775 + }, + { + "auxiliary_loss_clip": 0.01223432, + "auxiliary_loss_mlp": 0.0107815, + "balance_loss_clip": 1.05808771, + "balance_loss_mlp": 1.04100394, + "epoch": 0.01476989147466775, + "flos": 29927303729280.0, + "grad_norm": 3.27413717342017, + "language_loss": 0.88149965, + "learning_rate": 3.5915731839093863e-06, + "loss": 0.9045155, + "num_input_tokens_seen": 13891275, + "router_z_loss_clip": 1.65332031, + "router_z_loss_mlp": 0.37133789, + "step": 509, + "time_per_iteration": 2.5653185844421387 + }, + { + "auxiliary_loss_clip": 0.01235368, + "auxiliary_loss_mlp": 0.01078032, + "balance_loss_clip": 1.05739164, + "balance_loss_mlp": 1.03368568, + "epoch": 0.014798908943183796, + "flos": 36934162531200.0, + "grad_norm": 2.5323464172836676, + "language_loss": 0.97351551, + "learning_rate": 3.592704234436617e-06, + "loss": 0.99664962, + "num_input_tokens_seen": 13908485, + "router_z_loss_clip": 1.78027344, + "router_z_loss_mlp": 0.44311523, + "step": 510, + "time_per_iteration": 2.5573387145996094 + }, + { + "auxiliary_loss_clip": 0.01219648, + "auxiliary_loss_mlp": 0.01061393, + "balance_loss_clip": 1.05795908, + "balance_loss_mlp": 1.028157, + "epoch": 0.014827926411699843, + "flos": 26687782362240.0, + "grad_norm": 2.054699082578626, + "language_loss": 0.94808888, + "learning_rate": 3.593833069388429e-06, + "loss": 0.97089934, + "num_input_tokens_seen": 13928705, + "router_z_loss_clip": 1.61816406, + "router_z_loss_mlp": 0.33203125, + "step": 511, + "time_per_iteration": 2.5237882137298584 + }, + { + "auxiliary_loss_clip": 0.01224623, + "auxiliary_loss_mlp": 0.01074505, + "balance_loss_clip": 1.06087828, + "balance_loss_mlp": 1.03678691, + "epoch": 0.01485694388021589, + "flos": 33030209479680.0, + "grad_norm": 2.4358311273841657, + "language_loss": 0.94565141, + "learning_rate": 3.594959697427882e-06, + "loss": 0.96864265, + "num_input_tokens_seen": 13947700, + "router_z_loss_clip": 1.63671875, + "router_z_loss_mlp": 0.37744141, + "step": 512, + "time_per_iteration": 2.6488170623779297 + }, + { + "auxiliary_loss_clip": 0.01219209, + "auxiliary_loss_mlp": 0.01073563, + "balance_loss_clip": 1.05547953, + "balance_loss_mlp": 1.04008913, + "epoch": 0.014885961348731937, + "flos": 13765310213760.0, + "grad_norm": 2.3474443696489393, + "language_loss": 0.81667745, + "learning_rate": 3.5960841271673257e-06, + "loss": 0.83960521, + "num_input_tokens_seen": 13960880, + "router_z_loss_clip": 1.63867188, + "router_z_loss_mlp": 0.33496094, + "step": 513, + "time_per_iteration": 2.416160821914673 + }, + { + "auxiliary_loss_clip": 0.01220202, + "auxiliary_loss_mlp": 0.01065075, + "balance_loss_clip": 1.0525775, + "balance_loss_mlp": 1.02940714, + "epoch": 0.014914978817247983, + "flos": 74734241402880.0, + "grad_norm": 2.9373905475005557, + "language_loss": 0.97651821, + "learning_rate": 3.597206367168793e-06, + "loss": 0.99937099, + "num_input_tokens_seen": 13987420, + "router_z_loss_clip": 1.67675781, + "router_z_loss_mlp": 0.35681152, + "step": 514, + "time_per_iteration": 2.8354814052581787 + }, + { + "auxiliary_loss_clip": 0.01048988, + "auxiliary_loss_mlp": 0.01012951, + "balance_loss_clip": 1.00484002, + "balance_loss_mlp": 1.00727642, + "epoch": 0.01494399628576403, + "flos": 56782493527680.0, + "grad_norm": 0.7535834865970222, + "language_loss": 0.57445008, + "learning_rate": 3.598326425944392e-06, + "loss": 0.59506947, + "num_input_tokens_seen": 14040220, + "router_z_loss_clip": 0.44140625, + "router_z_loss_mlp": 0.05664062, + "step": 515, + "time_per_iteration": 2.9235174655914307 + }, + { + "auxiliary_loss_clip": 0.01220525, + "auxiliary_loss_mlp": 0.01091754, + "balance_loss_clip": 1.05954814, + "balance_loss_mlp": 1.05646789, + "epoch": 0.014973013754280076, + "flos": 74731273937280.0, + "grad_norm": 2.313939199544695, + "language_loss": 0.69476414, + "learning_rate": 3.5994443119566963e-06, + "loss": 0.71788692, + "num_input_tokens_seen": 14064065, + "router_z_loss_clip": 1.61035156, + "router_z_loss_mlp": 0.35266113, + "step": 516, + "time_per_iteration": 2.8806166648864746 + }, + { + "auxiliary_loss_clip": 0.0104851, + "auxiliary_loss_mlp": 0.01007859, + "balance_loss_clip": 1.00462759, + "balance_loss_mlp": 1.00182724, + "epoch": 0.015002031222796123, + "flos": 56818419183360.0, + "grad_norm": 0.7389217595794071, + "language_loss": 0.55120373, + "learning_rate": 3.600560033619124e-06, + "loss": 0.57176745, + "num_input_tokens_seen": 14119065, + "router_z_loss_clip": 0.4375, + "router_z_loss_mlp": 0.06030273, + "step": 517, + "time_per_iteration": 2.895589590072632 + }, + { + "auxiliary_loss_clip": 0.01222996, + "auxiliary_loss_mlp": 0.01079712, + "balance_loss_clip": 1.06016421, + "balance_loss_mlp": 1.04023004, + "epoch": 0.01503104869131217, + "flos": 22119083424000.0, + "grad_norm": 3.2452212346952045, + "language_loss": 0.85187858, + "learning_rate": 3.6016735992963195e-06, + "loss": 0.87490565, + "num_input_tokens_seen": 14137580, + "router_z_loss_clip": 1.62597656, + "router_z_loss_mlp": 0.39477539, + "step": 518, + "time_per_iteration": 2.522373676300049 + }, + { + "auxiliary_loss_clip": 0.01232859, + "auxiliary_loss_mlp": 0.010829, + "balance_loss_clip": 1.06098676, + "balance_loss_mlp": 1.04673195, + "epoch": 0.015060066159828217, + "flos": 24491641109760.0, + "grad_norm": 2.3716580433820598, + "language_loss": 1.01743233, + "learning_rate": 3.602785017304531e-06, + "loss": 1.04058981, + "num_input_tokens_seen": 14154380, + "router_z_loss_clip": 1.71875, + "router_z_loss_mlp": 0.36181641, + "step": 519, + "time_per_iteration": 2.4938673973083496 + }, + { + "auxiliary_loss_clip": 0.01047891, + "auxiliary_loss_mlp": 0.01006923, + "balance_loss_clip": 1.00522673, + "balance_loss_mlp": 1.00112975, + "epoch": 0.015089083628344263, + "flos": 64075151652480.0, + "grad_norm": 0.7320348505053219, + "language_loss": 0.56616843, + "learning_rate": 3.603894295911982e-06, + "loss": 0.58671665, + "num_input_tokens_seen": 14211185, + "router_z_loss_clip": 0.42578125, + "router_z_loss_mlp": 0.05786133, + "step": 520, + "time_per_iteration": 2.9006876945495605 + }, + { + "auxiliary_loss_clip": 0.01046455, + "auxiliary_loss_mlp": 0.01005696, + "balance_loss_clip": 1.00400329, + "balance_loss_mlp": 0.99983102, + "epoch": 0.01511810109686031, + "flos": 56569104097920.0, + "grad_norm": 0.7399251252671195, + "language_loss": 0.55432427, + "learning_rate": 3.6050014433392397e-06, + "loss": 0.57484579, + "num_input_tokens_seen": 14271735, + "router_z_loss_clip": 0.42578125, + "router_z_loss_mlp": 0.05859375, + "step": 521, + "time_per_iteration": 2.9587299823760986 + }, + { + "auxiliary_loss_clip": 0.01045944, + "auxiliary_loss_mlp": 0.01005475, + "balance_loss_clip": 1.00349283, + "balance_loss_mlp": 0.99956232, + "epoch": 0.015147118565376356, + "flos": 61772455330560.0, + "grad_norm": 0.7718187653081685, + "language_loss": 0.53797728, + "learning_rate": 3.6061064677595822e-06, + "loss": 0.55849147, + "num_input_tokens_seen": 14327630, + "router_z_loss_clip": 0.42578125, + "router_z_loss_mlp": 0.05908203, + "step": 522, + "time_per_iteration": 2.9420087337493896 + }, + { + "auxiliary_loss_clip": 0.01221649, + "auxiliary_loss_mlp": 0.0106934, + "balance_loss_clip": 1.0580709, + "balance_loss_mlp": 1.03145552, + "epoch": 0.015176136033892403, + "flos": 37406642227200.0, + "grad_norm": 2.60340464265057, + "language_loss": 0.88517845, + "learning_rate": 3.6072093772993584e-06, + "loss": 0.90808839, + "num_input_tokens_seen": 14343715, + "router_z_loss_clip": 1.63671875, + "router_z_loss_mlp": 0.37890625, + "step": 523, + "time_per_iteration": 2.5784754753112793 + }, + { + "auxiliary_loss_clip": 0.01216621, + "auxiliary_loss_mlp": 0.0107649, + "balance_loss_clip": 1.05549598, + "balance_loss_mlp": 1.0394634, + "epoch": 0.01520515350240845, + "flos": 31970700316800.0, + "grad_norm": 3.578155035137143, + "language_loss": 0.83235753, + "learning_rate": 3.6083101800383493e-06, + "loss": 0.85528862, + "num_input_tokens_seen": 14359915, + "router_z_loss_clip": 1.61035156, + "router_z_loss_mlp": 0.37023926, + "step": 524, + "time_per_iteration": 2.629268169403076 + }, + { + "auxiliary_loss_clip": 0.01216662, + "auxiliary_loss_mlp": 0.01073461, + "balance_loss_clip": 1.05408621, + "balance_loss_mlp": 1.03667259, + "epoch": 0.015234170970924497, + "flos": 32371014499200.0, + "grad_norm": 2.152391405535316, + "language_loss": 0.99400961, + "learning_rate": 3.609408884010121e-06, + "loss": 1.01691079, + "num_input_tokens_seen": 14380945, + "router_z_loss_clip": 1.62597656, + "router_z_loss_mlp": 0.36816406, + "step": 525, + "time_per_iteration": 2.534785032272339 + }, + { + "auxiliary_loss_clip": 0.01223186, + "auxiliary_loss_mlp": 0.01081126, + "balance_loss_clip": 1.05795467, + "balance_loss_mlp": 1.04269314, + "epoch": 0.015263188439440543, + "flos": 11612076888960.0, + "grad_norm": 3.386615397280992, + "language_loss": 0.86064434, + "learning_rate": 3.6105054972023773e-06, + "loss": 0.8836875, + "num_input_tokens_seen": 14392745, + "router_z_loss_clip": 1.65234375, + "router_z_loss_mlp": 0.3840332, + "step": 526, + "time_per_iteration": 2.427122116088867 + }, + { + "auxiliary_loss_clip": 0.01048873, + "auxiliary_loss_mlp": 0.01041655, + "balance_loss_clip": 1.00696266, + "balance_loss_mlp": 1.03619492, + "epoch": 0.01529220590795659, + "flos": 74777147464320.0, + "grad_norm": 0.6885672464155814, + "language_loss": 0.50275987, + "learning_rate": 3.611600027557307e-06, + "loss": 0.52366519, + "num_input_tokens_seen": 14459565, + "router_z_loss_clip": 0.41796875, + "router_z_loss_mlp": 0.0546875, + "step": 527, + "time_per_iteration": 3.0811967849731445 + }, + { + "auxiliary_loss_clip": 0.01234796, + "auxiliary_loss_mlp": 0.0108577, + "balance_loss_clip": 1.06296968, + "balance_loss_mlp": 1.04550123, + "epoch": 0.015321223376472636, + "flos": 23213854926720.0, + "grad_norm": 3.09323175092769, + "language_loss": 0.91817778, + "learning_rate": 3.6126924829719315e-06, + "loss": 0.94138348, + "num_input_tokens_seen": 14477405, + "router_z_loss_clip": 1.71777344, + "router_z_loss_mlp": 0.40283203, + "step": 528, + "time_per_iteration": 2.4831936359405518 + }, + { + "auxiliary_loss_clip": 0.01227405, + "auxiliary_loss_mlp": 0.01067983, + "balance_loss_clip": 1.05889368, + "balance_loss_mlp": 1.03129005, + "epoch": 0.015350240844988683, + "flos": 20883541852800.0, + "grad_norm": 3.644082345236275, + "language_loss": 1.02057266, + "learning_rate": 3.613782871298444e-06, + "loss": 1.04352653, + "num_input_tokens_seen": 14490740, + "router_z_loss_clip": 1.68554688, + "router_z_loss_mlp": 0.36694336, + "step": 529, + "time_per_iteration": 2.399630546569824 + }, + { + "auxiliary_loss_clip": 0.01232358, + "auxiliary_loss_mlp": 0.01080638, + "balance_loss_clip": 1.06246305, + "balance_loss_mlp": 1.0419662, + "epoch": 0.01537925831350473, + "flos": 16322977261440.0, + "grad_norm": 2.6715406606299146, + "language_loss": 0.65894222, + "learning_rate": 3.61487120034455e-06, + "loss": 0.68207216, + "num_input_tokens_seen": 14505210, + "router_z_loss_clip": 1.69921875, + "router_z_loss_mlp": 0.38647461, + "step": 530, + "time_per_iteration": 2.412639856338501 + }, + { + "auxiliary_loss_clip": 0.01212855, + "auxiliary_loss_mlp": 0.01070219, + "balance_loss_clip": 1.06043684, + "balance_loss_mlp": 1.03600621, + "epoch": 0.015408275782020777, + "flos": 13728197571840.0, + "grad_norm": 2.6810254300043526, + "language_loss": 0.92130709, + "learning_rate": 3.6159574778738017e-06, + "loss": 0.94413787, + "num_input_tokens_seen": 14516860, + "router_z_loss_clip": 1.52441406, + "router_z_loss_mlp": 0.34228516, + "step": 531, + "time_per_iteration": 2.4052655696868896 + }, + { + "auxiliary_loss_clip": 0.01224742, + "auxiliary_loss_mlp": 0.0106594, + "balance_loss_clip": 1.06102753, + "balance_loss_mlp": 1.02991533, + "epoch": 0.015437293250536823, + "flos": 32264110316160.0, + "grad_norm": 2.7328840025293593, + "language_loss": 0.87357372, + "learning_rate": 3.6170417116059306e-06, + "loss": 0.89648056, + "num_input_tokens_seen": 14535365, + "router_z_loss_clip": 1.63769531, + "router_z_loss_mlp": 0.35998535, + "step": 532, + "time_per_iteration": 2.4928715229034424 + }, + { + "auxiliary_loss_clip": 0.0122018, + "auxiliary_loss_mlp": 0.01072557, + "balance_loss_clip": 1.05930734, + "balance_loss_mlp": 1.03498209, + "epoch": 0.01546631071905287, + "flos": 16498625644800.0, + "grad_norm": 4.1883057780965975, + "language_loss": 0.97800034, + "learning_rate": 3.6181239092171762e-06, + "loss": 1.00092769, + "num_input_tokens_seen": 14547670, + "router_z_loss_clip": 1.60839844, + "router_z_loss_mlp": 0.37609863, + "step": 533, + "time_per_iteration": 2.3913259506225586 + }, + { + "auxiliary_loss_clip": 0.01232395, + "auxiliary_loss_mlp": 0.01093435, + "balance_loss_clip": 1.06258845, + "balance_loss_mlp": 1.04837406, + "epoch": 0.015495328187568916, + "flos": 46129132972800.0, + "grad_norm": 3.098214212832882, + "language_loss": 1.03198695, + "learning_rate": 3.619204078340615e-06, + "loss": 1.0552454, + "num_input_tokens_seen": 14567995, + "router_z_loss_clip": 1.69921875, + "router_z_loss_mlp": 0.45068359, + "step": 534, + "time_per_iteration": 2.6377060413360596 + }, + { + "auxiliary_loss_clip": 0.0104601, + "auxiliary_loss_mlp": 0.01012122, + "balance_loss_clip": 1.00471711, + "balance_loss_mlp": 1.00713933, + "epoch": 0.015524345656084963, + "flos": 74771631469440.0, + "grad_norm": 0.7291629592929073, + "language_loss": 0.52168763, + "learning_rate": 3.620282226566477e-06, + "loss": 0.54226887, + "num_input_tokens_seen": 14630350, + "router_z_loss_clip": 0.4140625, + "router_z_loss_mlp": 0.04980469, + "step": 535, + "time_per_iteration": 3.072789192199707 + }, + { + "auxiliary_loss_clip": 0.01220987, + "auxiliary_loss_mlp": 0.0108452, + "balance_loss_clip": 1.06283748, + "balance_loss_mlp": 1.0467546, + "epoch": 0.01555336312460101, + "flos": 10918003593600.0, + "grad_norm": 2.650672205807192, + "language_loss": 0.89288533, + "learning_rate": 3.621358361442474e-06, + "loss": 0.9159404, + "num_input_tokens_seen": 14642155, + "router_z_loss_clip": 1.58007812, + "router_z_loss_mlp": 0.37792969, + "step": 536, + "time_per_iteration": 2.420781373977661 + }, + { + "auxiliary_loss_clip": 0.01218136, + "auxiliary_loss_mlp": 0.01074234, + "balance_loss_clip": 1.05661678, + "balance_loss_mlp": 1.03789878, + "epoch": 0.015582380593117057, + "flos": 25622792115840.0, + "grad_norm": 2.9388458399031823, + "language_loss": 0.82356083, + "learning_rate": 3.62243249047411e-06, + "loss": 0.84648454, + "num_input_tokens_seen": 14655125, + "router_z_loss_clip": 1.61425781, + "router_z_loss_mlp": 0.36364746, + "step": 537, + "time_per_iteration": 2.6476781368255615 + }, + { + "auxiliary_loss_clip": 0.01221917, + "auxiliary_loss_mlp": 0.01068748, + "balance_loss_clip": 1.06048131, + "balance_loss_mlp": 1.0341537, + "epoch": 0.015611398061633103, + "flos": 22629967482240.0, + "grad_norm": 5.880216723819863, + "language_loss": 0.83424234, + "learning_rate": 3.623504621124998e-06, + "loss": 0.85714895, + "num_input_tokens_seen": 14669050, + "router_z_loss_clip": 1.61425781, + "router_z_loss_mlp": 0.34570312, + "step": 538, + "time_per_iteration": 4.875777006149292 + }, + { + "auxiliary_loss_clip": 0.01214968, + "auxiliary_loss_mlp": 0.01078555, + "balance_loss_clip": 1.0559082, + "balance_loss_mlp": 1.04095614, + "epoch": 0.015640415530149148, + "flos": 39929256403200.0, + "grad_norm": 2.3447603902947853, + "language_loss": 0.75946641, + "learning_rate": 3.624574760817172e-06, + "loss": 0.78240156, + "num_input_tokens_seen": 14691675, + "router_z_loss_clip": 1.59082031, + "router_z_loss_mlp": 0.37597656, + "step": 539, + "time_per_iteration": 7.403366804122925 + }, + { + "auxiliary_loss_clip": 0.01223682, + "auxiliary_loss_mlp": 0.01070875, + "balance_loss_clip": 1.06250596, + "balance_loss_mlp": 1.03422987, + "epoch": 0.015669432998665196, + "flos": 18291171047040.0, + "grad_norm": 5.660004190487082, + "language_loss": 0.88586283, + "learning_rate": 3.6256429169313935e-06, + "loss": 0.90880841, + "num_input_tokens_seen": 14706105, + "router_z_loss_clip": 1.61230469, + "router_z_loss_mlp": 0.36621094, + "step": 540, + "time_per_iteration": 4.908369064331055 + }, + { + "auxiliary_loss_clip": 0.01219645, + "auxiliary_loss_mlp": 0.01078285, + "balance_loss_clip": 1.05714238, + "balance_loss_mlp": 1.04142594, + "epoch": 0.015698450467181244, + "flos": 15953317119360.0, + "grad_norm": 3.2137656089946343, + "language_loss": 0.92031431, + "learning_rate": 3.626709096807456e-06, + "loss": 0.94329369, + "num_input_tokens_seen": 14720355, + "router_z_loss_clip": 1.625, + "router_z_loss_mlp": 0.36889648, + "step": 541, + "time_per_iteration": 2.374861001968384 + }, + { + "auxiliary_loss_clip": 0.0120565, + "auxiliary_loss_mlp": 0.01078856, + "balance_loss_clip": 1.0566119, + "balance_loss_mlp": 1.04421425, + "epoch": 0.01572746793569729, + "flos": 20331006675840.0, + "grad_norm": 3.1110140555437718, + "language_loss": 0.89787936, + "learning_rate": 3.627773307744494e-06, + "loss": 0.92072439, + "num_input_tokens_seen": 14733005, + "router_z_loss_clip": 1.48925781, + "router_z_loss_mlp": 0.34619141, + "step": 542, + "time_per_iteration": 2.4497640132904053 + }, + { + "auxiliary_loss_clip": 0.01212693, + "auxiliary_loss_mlp": 0.01074347, + "balance_loss_clip": 1.05344367, + "balance_loss_mlp": 1.03846526, + "epoch": 0.015756485404213337, + "flos": 30839758778880.0, + "grad_norm": 3.2143729996598776, + "language_loss": 0.97295094, + "learning_rate": 3.6288355570012727e-06, + "loss": 0.9958213, + "num_input_tokens_seen": 14749155, + "router_z_loss_clip": 1.59179688, + "router_z_loss_mlp": 0.35864258, + "step": 543, + "time_per_iteration": 2.7813782691955566 + }, + { + "auxiliary_loss_clip": 0.01210514, + "auxiliary_loss_mlp": 0.01085609, + "balance_loss_clip": 1.05496478, + "balance_loss_mlp": 1.05058503, + "epoch": 0.01578550287272938, + "flos": 11356477758720.0, + "grad_norm": 2.824227979198889, + "language_loss": 0.92680532, + "learning_rate": 3.6298958517964935e-06, + "loss": 0.94976652, + "num_input_tokens_seen": 14760760, + "router_z_loss_clip": 1.55371094, + "router_z_loss_mlp": 0.34985352, + "step": 544, + "time_per_iteration": 2.4084534645080566 + }, + { + "auxiliary_loss_clip": 0.0120233, + "auxiliary_loss_mlp": 0.01067209, + "balance_loss_clip": 1.05353308, + "balance_loss_mlp": 1.03418779, + "epoch": 0.01581452034124543, + "flos": 16317705646080.0, + "grad_norm": 3.1160492019129724, + "language_loss": 0.92845511, + "learning_rate": 3.630954199309085e-06, + "loss": 0.95115054, + "num_input_tokens_seen": 14773135, + "router_z_loss_clip": 1.48730469, + "router_z_loss_mlp": 0.33032227, + "step": 545, + "time_per_iteration": 2.3702540397644043 + }, + { + "auxiliary_loss_clip": 0.01045644, + "auxiliary_loss_mlp": 0.01032254, + "balance_loss_clip": 1.0053345, + "balance_loss_mlp": 1.02731848, + "epoch": 0.015843537809761478, + "flos": 64740665589120.0, + "grad_norm": 0.8403117245847231, + "language_loss": 0.53599334, + "learning_rate": 3.632010606678494e-06, + "loss": 0.55677229, + "num_input_tokens_seen": 14835015, + "router_z_loss_clip": 0.40234375, + "router_z_loss_mlp": 0.04931641, + "step": 546, + "time_per_iteration": 3.11527943611145 + }, + { + "auxiliary_loss_clip": 0.012191, + "auxiliary_loss_mlp": 0.01075714, + "balance_loss_clip": 1.05333698, + "balance_loss_mlp": 1.0360893, + "epoch": 0.015872555278277523, + "flos": 27046585071360.0, + "grad_norm": 3.158285772158721, + "language_loss": 0.97808903, + "learning_rate": 3.6330650810049766e-06, + "loss": 1.00103724, + "num_input_tokens_seen": 14851830, + "router_z_loss_clip": 1.65722656, + "router_z_loss_mlp": 0.39648438, + "step": 547, + "time_per_iteration": 2.513568639755249 + }, + { + "auxiliary_loss_clip": 0.01211716, + "auxiliary_loss_mlp": 0.01077105, + "balance_loss_clip": 1.05770218, + "balance_loss_mlp": 1.04196215, + "epoch": 0.01590157274679357, + "flos": 31461142890240.0, + "grad_norm": 2.4276443872272275, + "language_loss": 0.77711427, + "learning_rate": 3.6341176293498826e-06, + "loss": 0.80000246, + "num_input_tokens_seen": 14869145, + "router_z_loss_clip": 1.54101562, + "router_z_loss_mlp": 0.35131836, + "step": 548, + "time_per_iteration": 2.4927682876586914 + }, + { + "auxiliary_loss_clip": 0.01208569, + "auxiliary_loss_mlp": 0.01069075, + "balance_loss_clip": 1.05324209, + "balance_loss_mlp": 1.03164375, + "epoch": 0.015930590215309615, + "flos": 12524915963520.0, + "grad_norm": 3.049638058676407, + "language_loss": 0.84677637, + "learning_rate": 3.635168258735939e-06, + "loss": 0.86955291, + "num_input_tokens_seen": 14881820, + "router_z_loss_clip": 1.55371094, + "router_z_loss_mlp": 0.37426758, + "step": 549, + "time_per_iteration": 2.3956193923950195 + }, + { + "auxiliary_loss_clip": 0.01231846, + "auxiliary_loss_mlp": 0.01087289, + "balance_loss_clip": 1.05707884, + "balance_loss_mlp": 1.04308653, + "epoch": 0.015959607683825663, + "flos": 24599837013120.0, + "grad_norm": 2.5597742301668682, + "language_loss": 0.96683198, + "learning_rate": 3.6362169761475343e-06, + "loss": 0.99002337, + "num_input_tokens_seen": 14896690, + "router_z_loss_clip": 1.74902344, + "router_z_loss_mlp": 0.44213867, + "step": 550, + "time_per_iteration": 2.4525108337402344 + }, + { + "auxiliary_loss_clip": 0.0122708, + "auxiliary_loss_mlp": 0.01075486, + "balance_loss_clip": 1.06042504, + "balance_loss_mlp": 1.03767288, + "epoch": 0.015988625152341708, + "flos": 10443778329600.0, + "grad_norm": 3.179079098583193, + "language_loss": 0.90559369, + "learning_rate": 3.6372637885309946e-06, + "loss": 0.92861938, + "num_input_tokens_seen": 14907260, + "router_z_loss_clip": 1.6640625, + "router_z_loss_mlp": 0.37817383, + "step": 551, + "time_per_iteration": 2.4137887954711914 + }, + { + "auxiliary_loss_clip": 0.01044368, + "auxiliary_loss_mlp": 0.01007467, + "balance_loss_clip": 1.0029819, + "balance_loss_mlp": 1.00255585, + "epoch": 0.016017642620857756, + "flos": 70506850849920.0, + "grad_norm": 0.802357899009429, + "language_loss": 0.56215024, + "learning_rate": 3.6383087027948565e-06, + "loss": 0.58266854, + "num_input_tokens_seen": 14972845, + "router_z_loss_clip": 0.4140625, + "router_z_loss_mlp": 0.04907227, + "step": 552, + "time_per_iteration": 3.113877296447754 + }, + { + "auxiliary_loss_clip": 0.0121632, + "auxiliary_loss_mlp": 0.01079926, + "balance_loss_clip": 1.05509734, + "balance_loss_mlp": 1.03717709, + "epoch": 0.016046660089373804, + "flos": 31895706983040.0, + "grad_norm": 2.465654922651311, + "language_loss": 0.87671697, + "learning_rate": 3.6393517258101497e-06, + "loss": 0.89967942, + "num_input_tokens_seen": 14989565, + "router_z_loss_clip": 1.61230469, + "router_z_loss_mlp": 0.42773438, + "step": 553, + "time_per_iteration": 2.613224983215332 + }, + { + "auxiliary_loss_clip": 0.01216082, + "auxiliary_loss_mlp": 0.01077257, + "balance_loss_clip": 1.05555105, + "balance_loss_mlp": 1.03965843, + "epoch": 0.01607567755788985, + "flos": 10374196256640.0, + "grad_norm": 3.0854184614035867, + "language_loss": 0.86330509, + "learning_rate": 3.6403928644106584e-06, + "loss": 0.88623846, + "num_input_tokens_seen": 14999900, + "router_z_loss_clip": 1.60644531, + "router_z_loss_mlp": 0.3762207, + "step": 554, + "time_per_iteration": 2.3950746059417725 + }, + { + "auxiliary_loss_clip": 0.01230863, + "auxiliary_loss_mlp": 0.010761, + "balance_loss_clip": 1.0595119, + "balance_loss_mlp": 1.03697586, + "epoch": 0.016104695026405897, + "flos": 23872281857280.0, + "grad_norm": 2.8376941184932787, + "language_loss": 0.9839896, + "learning_rate": 3.6414321253931943e-06, + "loss": 1.00705922, + "num_input_tokens_seen": 15017125, + "router_z_loss_clip": 1.71484375, + "router_z_loss_mlp": 0.39111328, + "step": 555, + "time_per_iteration": 2.498486042022705 + }, + { + "auxiliary_loss_clip": 0.01225843, + "auxiliary_loss_mlp": 0.0106483, + "balance_loss_clip": 1.05861115, + "balance_loss_mlp": 1.02496588, + "epoch": 0.01613371249492194, + "flos": 13074832788480.0, + "grad_norm": 3.934385437109385, + "language_loss": 1.16748095, + "learning_rate": 3.6424695155178653e-06, + "loss": 1.19038773, + "num_input_tokens_seen": 15030265, + "router_z_loss_clip": 1.67285156, + "router_z_loss_mlp": 0.39868164, + "step": 556, + "time_per_iteration": 2.398928165435791 + }, + { + "auxiliary_loss_clip": 0.01205101, + "auxiliary_loss_mlp": 0.01076303, + "balance_loss_clip": 1.05179989, + "balance_loss_mlp": 1.03891897, + "epoch": 0.01616272996343799, + "flos": 14020001648640.0, + "grad_norm": 3.013164153248762, + "language_loss": 0.7343148, + "learning_rate": 3.643505041508334e-06, + "loss": 0.75712883, + "num_input_tokens_seen": 15043175, + "router_z_loss_clip": 1.53320312, + "router_z_loss_mlp": 0.37402344, + "step": 557, + "time_per_iteration": 2.467557430267334 + }, + { + "auxiliary_loss_clip": 0.01043057, + "auxiliary_loss_mlp": 0.01006079, + "balance_loss_clip": 1.00250864, + "balance_loss_mlp": 1.00123906, + "epoch": 0.016191747431954038, + "flos": 63863088854400.0, + "grad_norm": 0.6970707064137883, + "language_loss": 0.54512972, + "learning_rate": 3.644538710052083e-06, + "loss": 0.56562114, + "num_input_tokens_seen": 15102595, + "router_z_loss_clip": 0.40625, + "router_z_loss_mlp": 0.04833984, + "step": 558, + "time_per_iteration": 2.9722752571105957 + }, + { + "auxiliary_loss_clip": 0.01043024, + "auxiliary_loss_mlp": 0.01005546, + "balance_loss_clip": 1.00237918, + "balance_loss_mlp": 1.00080168, + "epoch": 0.016220764900470083, + "flos": 67030549441920.0, + "grad_norm": 0.7466751880093014, + "language_loss": 0.52472997, + "learning_rate": 3.6455705278006725e-06, + "loss": 0.54521561, + "num_input_tokens_seen": 15151495, + "router_z_loss_clip": 0.40625, + "router_z_loss_mlp": 0.04736328, + "step": 559, + "time_per_iteration": 2.9248573780059814 + }, + { + "auxiliary_loss_clip": 0.01209558, + "auxiliary_loss_mlp": 0.01056071, + "balance_loss_clip": 1.05482507, + "balance_loss_mlp": 1.02171493, + "epoch": 0.01624978236898613, + "flos": 14968382353920.0, + "grad_norm": 3.9364810007133677, + "language_loss": 1.07262444, + "learning_rate": 3.6466005013699975e-06, + "loss": 1.09528065, + "num_input_tokens_seen": 15163330, + "router_z_loss_clip": 1.546875, + "router_z_loss_mlp": 0.34350586, + "step": 560, + "time_per_iteration": 2.4511332511901855 + }, + { + "auxiliary_loss_clip": 0.01043035, + "auxiliary_loss_mlp": 0.01007202, + "balance_loss_clip": 1.00336361, + "balance_loss_mlp": 1.00252879, + "epoch": 0.016278799837502175, + "flos": 74769990635520.0, + "grad_norm": 0.7147082563041414, + "language_loss": 0.52617943, + "learning_rate": 3.6476286373405424e-06, + "loss": 0.54668176, + "num_input_tokens_seen": 15225765, + "router_z_loss_clip": 0.39648438, + "router_z_loss_mlp": 0.04663086, + "step": 561, + "time_per_iteration": 3.060439109802246 + }, + { + "auxiliary_loss_clip": 0.01204541, + "auxiliary_loss_mlp": 0.01067419, + "balance_loss_clip": 1.05432653, + "balance_loss_mlp": 1.02974927, + "epoch": 0.016307817306018223, + "flos": 10406421308160.0, + "grad_norm": 7.259039741683454, + "language_loss": 1.12514508, + "learning_rate": 3.6486549422576337e-06, + "loss": 1.14786482, + "num_input_tokens_seen": 15234910, + "router_z_loss_clip": 1.50097656, + "router_z_loss_mlp": 0.37670898, + "step": 562, + "time_per_iteration": 2.4867024421691895 + }, + { + "auxiliary_loss_clip": 0.010446, + "auxiliary_loss_mlp": 0.01014451, + "balance_loss_clip": 1.00401974, + "balance_loss_mlp": 1.00946856, + "epoch": 0.016336834774534268, + "flos": 57877265030400.0, + "grad_norm": 0.7358456558519563, + "language_loss": 0.54504037, + "learning_rate": 3.649679422631688e-06, + "loss": 0.56563091, + "num_input_tokens_seen": 15291310, + "router_z_loss_clip": 0.40625, + "router_z_loss_mlp": 0.04980469, + "step": 563, + "time_per_iteration": 2.833803176879883 + }, + { + "auxiliary_loss_clip": 0.01217404, + "auxiliary_loss_mlp": 0.01076305, + "balance_loss_clip": 1.05627918, + "balance_loss_mlp": 1.03832507, + "epoch": 0.016365852243050316, + "flos": 74731937253120.0, + "grad_norm": 2.941376324652733, + "language_loss": 0.90111446, + "learning_rate": 3.650702084938462e-06, + "loss": 0.92405152, + "num_input_tokens_seen": 15313550, + "router_z_loss_clip": 1.61230469, + "router_z_loss_mlp": 0.37988281, + "step": 564, + "time_per_iteration": 2.847116231918335 + }, + { + "auxiliary_loss_clip": 0.01214708, + "auxiliary_loss_mlp": 0.01078559, + "balance_loss_clip": 1.06092715, + "balance_loss_mlp": 1.04291534, + "epoch": 0.016394869711566364, + "flos": 10479773808000.0, + "grad_norm": 2.767812294847946, + "language_loss": 0.69848698, + "learning_rate": 3.6517229356192984e-06, + "loss": 0.72141963, + "num_input_tokens_seen": 15325600, + "router_z_loss_clip": 1.53613281, + "router_z_loss_mlp": 0.35644531, + "step": 565, + "time_per_iteration": 2.4109983444213867 + }, + { + "auxiliary_loss_clip": 0.01231437, + "auxiliary_loss_mlp": 0.01085349, + "balance_loss_clip": 1.05991292, + "balance_loss_mlp": 1.04376924, + "epoch": 0.01642388718008241, + "flos": 16318438784640.0, + "grad_norm": 3.1666473697699677, + "language_loss": 0.96631753, + "learning_rate": 3.652741981081366e-06, + "loss": 0.98948544, + "num_input_tokens_seen": 15337850, + "router_z_loss_clip": 1.71582031, + "router_z_loss_mlp": 0.41625977, + "step": 566, + "time_per_iteration": 2.3997833728790283 + }, + { + "auxiliary_loss_clip": 0.01045793, + "auxiliary_loss_mlp": 0.01010268, + "balance_loss_clip": 1.00303841, + "balance_loss_mlp": 1.0051893, + "epoch": 0.016452904648598457, + "flos": 55690095997440.0, + "grad_norm": 0.7326266157971788, + "language_loss": 0.53830802, + "learning_rate": 3.6537592276979053e-06, + "loss": 0.55886865, + "num_input_tokens_seen": 15393285, + "router_z_loss_clip": 0.42773438, + "router_z_loss_mlp": 0.05078125, + "step": 567, + "time_per_iteration": 2.949429750442505 + }, + { + "auxiliary_loss_clip": 0.01225706, + "auxiliary_loss_mlp": 0.01084124, + "balance_loss_clip": 1.05587888, + "balance_loss_mlp": 1.04144728, + "epoch": 0.0164819221171145, + "flos": 16901732736000.0, + "grad_norm": 3.4789580181395827, + "language_loss": 0.99389023, + "learning_rate": 3.6547746818084655e-06, + "loss": 1.01698852, + "num_input_tokens_seen": 15406580, + "router_z_loss_clip": 1.70117188, + "router_z_loss_mlp": 0.42700195, + "step": 568, + "time_per_iteration": 2.4276678562164307 + }, + { + "auxiliary_loss_clip": 0.01045582, + "auxiliary_loss_mlp": 0.01004538, + "balance_loss_clip": 1.00229347, + "balance_loss_mlp": 0.99929261, + "epoch": 0.01651093958563055, + "flos": 66568403508480.0, + "grad_norm": 0.6483008104314448, + "language_loss": 0.59681338, + "learning_rate": 3.6557883497191405e-06, + "loss": 0.61731458, + "num_input_tokens_seen": 15473025, + "router_z_loss_clip": 0.43359375, + "router_z_loss_mlp": 0.05249023, + "step": 569, + "time_per_iteration": 3.0942862033843994 + }, + { + "auxiliary_loss_clip": 0.01044679, + "auxiliary_loss_mlp": 0.01005113, + "balance_loss_clip": 1.00197697, + "balance_loss_mlp": 0.99970067, + "epoch": 0.016539957054146598, + "flos": 70716225473280.0, + "grad_norm": 0.6885276676776046, + "language_loss": 0.55608726, + "learning_rate": 3.656800237702806e-06, + "loss": 0.57658517, + "num_input_tokens_seen": 15528640, + "router_z_loss_clip": 0.42578125, + "router_z_loss_mlp": 0.05419922, + "step": 570, + "time_per_iteration": 2.985337972640991 + }, + { + "auxiliary_loss_clip": 0.01214715, + "auxiliary_loss_mlp": 0.01074825, + "balance_loss_clip": 1.05387402, + "balance_loss_mlp": 1.03705931, + "epoch": 0.016568974522662643, + "flos": 24382293131520.0, + "grad_norm": 2.720782551494755, + "language_loss": 1.0069021, + "learning_rate": 3.65781035199935e-06, + "loss": 1.02979755, + "num_input_tokens_seen": 15544405, + "router_z_loss_clip": 1.60644531, + "router_z_loss_mlp": 0.37731934, + "step": 571, + "time_per_iteration": 2.4859554767608643 + }, + { + "auxiliary_loss_clip": 0.01200481, + "auxiliary_loss_mlp": 0.01068928, + "balance_loss_clip": 1.04975462, + "balance_loss_mlp": 1.0318538, + "epoch": 0.01659799199117869, + "flos": 28099984746240.0, + "grad_norm": 3.4789691030879712, + "language_loss": 1.08215046, + "learning_rate": 3.6588186988159077e-06, + "loss": 1.10484457, + "num_input_tokens_seen": 15558795, + "router_z_loss_clip": 1.50683594, + "router_z_loss_mlp": 0.37084961, + "step": 572, + "time_per_iteration": 2.507601737976074 + }, + { + "auxiliary_loss_clip": 0.01212178, + "auxiliary_loss_mlp": 0.01067844, + "balance_loss_clip": 1.05717945, + "balance_loss_mlp": 1.03229606, + "epoch": 0.016627009459694735, + "flos": 16318299139200.0, + "grad_norm": 2.9523803232523225, + "language_loss": 0.8172009, + "learning_rate": 3.6598252843270863e-06, + "loss": 0.84000117, + "num_input_tokens_seen": 15571720, + "router_z_loss_clip": 1.55273438, + "router_z_loss_mlp": 0.35546875, + "step": 573, + "time_per_iteration": 2.3916232585906982 + }, + { + "auxiliary_loss_clip": 0.0121176, + "auxiliary_loss_mlp": 0.01076138, + "balance_loss_clip": 1.05566311, + "balance_loss_mlp": 1.0386703, + "epoch": 0.016656026928210783, + "flos": 26984404206720.0, + "grad_norm": 2.3581782561545706, + "language_loss": 0.78437066, + "learning_rate": 3.6608301146751923e-06, + "loss": 0.80724967, + "num_input_tokens_seen": 15590020, + "router_z_loss_clip": 1.56054688, + "router_z_loss_mlp": 0.37475586, + "step": 574, + "time_per_iteration": 2.5477941036224365 + }, + { + "auxiliary_loss_clip": 0.01221068, + "auxiliary_loss_mlp": 0.01090485, + "balance_loss_clip": 1.06000328, + "balance_loss_mlp": 1.05193233, + "epoch": 0.016685044396726828, + "flos": 16358344335360.0, + "grad_norm": 3.0676702619474057, + "language_loss": 0.90934592, + "learning_rate": 3.66183319597046e-06, + "loss": 0.93246144, + "num_input_tokens_seen": 15604860, + "router_z_loss_clip": 1.61035156, + "router_z_loss_mlp": 0.38549805, + "step": 575, + "time_per_iteration": 2.4659106731414795 + }, + { + "auxiliary_loss_clip": 0.01222048, + "auxiliary_loss_mlp": 0.01082145, + "balance_loss_clip": 1.05633366, + "balance_loss_mlp": 1.04123271, + "epoch": 0.016714061865242876, + "flos": 27847667283840.0, + "grad_norm": 2.623579547076584, + "language_loss": 0.80461299, + "learning_rate": 3.6628345342912697e-06, + "loss": 0.82765496, + "num_input_tokens_seen": 15621045, + "router_z_loss_clip": 1.65917969, + "router_z_loss_mlp": 0.40930176, + "step": 576, + "time_per_iteration": 2.5188562870025635 + }, + { + "auxiliary_loss_clip": 0.01041866, + "auxiliary_loss_mlp": 0.01006687, + "balance_loss_clip": 1.00190258, + "balance_loss_mlp": 1.00158501, + "epoch": 0.016743079333758924, + "flos": 74773411948800.0, + "grad_norm": 0.7195168593232127, + "language_loss": 0.60090923, + "learning_rate": 3.663834135684372e-06, + "loss": 0.62139475, + "num_input_tokens_seen": 15691015, + "router_z_loss_clip": 0.3984375, + "router_z_loss_mlp": 0.05102539, + "step": 577, + "time_per_iteration": 3.161961317062378 + }, + { + "auxiliary_loss_clip": 0.01235447, + "auxiliary_loss_mlp": 0.01082673, + "balance_loss_clip": 1.06071579, + "balance_loss_mlp": 1.04099715, + "epoch": 0.01677209680227497, + "flos": 21497769135360.0, + "grad_norm": 2.2579690517992463, + "language_loss": 0.93494588, + "learning_rate": 3.6648320061651052e-06, + "loss": 0.95812702, + "num_input_tokens_seen": 15709155, + "router_z_loss_clip": 1.74609375, + "router_z_loss_mlp": 0.41674805, + "step": 578, + "time_per_iteration": 2.461158514022827 + }, + { + "auxiliary_loss_clip": 0.01040952, + "auxiliary_loss_mlp": 0.01004897, + "balance_loss_clip": 1.00141001, + "balance_loss_mlp": 1.00010455, + "epoch": 0.016801114270791017, + "flos": 74776972907520.0, + "grad_norm": 0.7239557548371486, + "language_loss": 0.56618202, + "learning_rate": 3.665828151717614e-06, + "loss": 0.58664048, + "num_input_tokens_seen": 15771180, + "router_z_loss_clip": 0.39453125, + "router_z_loss_mlp": 0.04785156, + "step": 579, + "time_per_iteration": 3.123081922531128 + }, + { + "auxiliary_loss_clip": 0.0121184, + "auxiliary_loss_mlp": 0.01074774, + "balance_loss_clip": 1.05630183, + "balance_loss_mlp": 1.03760457, + "epoch": 0.01683013173930706, + "flos": 13654670515200.0, + "grad_norm": 3.770622310522129, + "language_loss": 0.82319868, + "learning_rate": 3.6668225782950615e-06, + "loss": 0.84606487, + "num_input_tokens_seen": 15783845, + "router_z_loss_clip": 1.55664062, + "router_z_loss_mlp": 0.37158203, + "step": 580, + "time_per_iteration": 2.439493179321289 + }, + { + "auxiliary_loss_clip": 0.01041277, + "auxiliary_loss_mlp": 0.01005802, + "balance_loss_clip": 1.00122476, + "balance_loss_mlp": 1.0009619, + "epoch": 0.01685914920782311, + "flos": 60691892751360.0, + "grad_norm": 0.679175637407216, + "language_loss": 0.5460695, + "learning_rate": 3.6678152918198486e-06, + "loss": 0.56654024, + "num_input_tokens_seen": 15846885, + "router_z_loss_clip": 0.40039062, + "router_z_loss_mlp": 0.04833984, + "step": 581, + "time_per_iteration": 3.0071640014648438 + }, + { + "auxiliary_loss_clip": 0.01041857, + "auxiliary_loss_mlp": 0.01006763, + "balance_loss_clip": 1.00134504, + "balance_loss_mlp": 1.00173283, + "epoch": 0.016888166676339158, + "flos": 74768489447040.0, + "grad_norm": 0.6541211419997871, + "language_loss": 0.52256465, + "learning_rate": 3.6688062981838202e-06, + "loss": 0.54305089, + "num_input_tokens_seen": 15914395, + "router_z_loss_clip": 0.40625, + "router_z_loss_mlp": 0.05029297, + "step": 582, + "time_per_iteration": 3.15873384475708 + }, + { + "auxiliary_loss_clip": 0.01204649, + "auxiliary_loss_mlp": 0.01068758, + "balance_loss_clip": 1.05819774, + "balance_loss_mlp": 1.03275681, + "epoch": 0.016917184144855203, + "flos": 12451738020480.0, + "grad_norm": 4.386528450238565, + "language_loss": 0.9331249, + "learning_rate": 3.6697956032484757e-06, + "loss": 0.95585895, + "num_input_tokens_seen": 15925365, + "router_z_loss_clip": 1.46289062, + "router_z_loss_mlp": 0.36010742, + "step": 583, + "time_per_iteration": 2.5298688411712646 + }, + { + "auxiliary_loss_clip": 0.01217299, + "auxiliary_loss_mlp": 0.01073639, + "balance_loss_clip": 1.05534768, + "balance_loss_mlp": 1.033144, + "epoch": 0.01694620161337125, + "flos": 29177403436800.0, + "grad_norm": 3.2424788725971148, + "language_loss": 0.98875427, + "learning_rate": 3.670783212845181e-06, + "loss": 1.01166368, + "num_input_tokens_seen": 15947345, + "router_z_loss_clip": 1.62011719, + "router_z_loss_mlp": 0.40466309, + "step": 584, + "time_per_iteration": 2.577542304992676 + }, + { + "auxiliary_loss_clip": 0.01206437, + "auxiliary_loss_mlp": 0.01078542, + "balance_loss_clip": 1.05139828, + "balance_loss_mlp": 1.03996563, + "epoch": 0.016975219081887295, + "flos": 29390513575680.0, + "grad_norm": 2.122591386474465, + "language_loss": 0.73503315, + "learning_rate": 3.6717691327753693e-06, + "loss": 0.75788295, + "num_input_tokens_seen": 15973560, + "router_z_loss_clip": 1.54980469, + "router_z_loss_mlp": 0.38525391, + "step": 585, + "time_per_iteration": 2.7448880672454834 + }, + { + "auxiliary_loss_clip": 0.01043116, + "auxiliary_loss_mlp": 0.01012791, + "balance_loss_clip": 1.00284028, + "balance_loss_mlp": 1.00749803, + "epoch": 0.017004236550403343, + "flos": 74626706949120.0, + "grad_norm": 0.6679360778167126, + "language_loss": 0.5437268, + "learning_rate": 3.67275336881075e-06, + "loss": 0.56428587, + "num_input_tokens_seen": 16036605, + "router_z_loss_clip": 0.40234375, + "router_z_loss_mlp": 0.05297852, + "step": 586, + "time_per_iteration": 3.080960273742676 + }, + { + "auxiliary_loss_clip": 0.01042387, + "auxiliary_loss_mlp": 0.01008655, + "balance_loss_clip": 1.00209761, + "balance_loss_mlp": 1.00343323, + "epoch": 0.017033254018919388, + "flos": 67442838220800.0, + "grad_norm": 1.843921572420356, + "language_loss": 0.55617535, + "learning_rate": 3.6737359266935092e-06, + "loss": 0.57668579, + "num_input_tokens_seen": 16102765, + "router_z_loss_clip": 0.40234375, + "router_z_loss_mlp": 0.05224609, + "step": 587, + "time_per_iteration": 3.1354315280914307 + }, + { + "auxiliary_loss_clip": 0.01209088, + "auxiliary_loss_mlp": 0.01082085, + "balance_loss_clip": 1.05653834, + "balance_loss_mlp": 1.04467678, + "epoch": 0.017062271487435436, + "flos": 28833160631040.0, + "grad_norm": 3.52248430238267, + "language_loss": 0.74948382, + "learning_rate": 3.6747168121365105e-06, + "loss": 0.77239549, + "num_input_tokens_seen": 16120360, + "router_z_loss_clip": 1.52636719, + "router_z_loss_mlp": 0.37451172, + "step": 588, + "time_per_iteration": 2.5353245735168457 + }, + { + "auxiliary_loss_clip": 0.0120951, + "auxiliary_loss_mlp": 0.01066013, + "balance_loss_clip": 1.05751836, + "balance_loss_mlp": 1.03032207, + "epoch": 0.017091288955951484, + "flos": 12603924103680.0, + "grad_norm": 2.9592142563552573, + "language_loss": 0.99705493, + "learning_rate": 3.6756960308234956e-06, + "loss": 1.0198102, + "num_input_tokens_seen": 16133410, + "router_z_loss_clip": 1.51953125, + "router_z_loss_mlp": 0.35693359, + "step": 589, + "time_per_iteration": 2.4017419815063477 + }, + { + "auxiliary_loss_clip": 0.01218278, + "auxiliary_loss_mlp": 0.01093469, + "balance_loss_clip": 1.06280553, + "balance_loss_mlp": 1.05634761, + "epoch": 0.01712030642446753, + "flos": 12159200805120.0, + "grad_norm": 2.885191692624182, + "language_loss": 0.85533464, + "learning_rate": 3.676673588409281e-06, + "loss": 0.87845206, + "num_input_tokens_seen": 16144475, + "router_z_loss_clip": 1.55566406, + "router_z_loss_mlp": 0.37133789, + "step": 590, + "time_per_iteration": 2.463385820388794 + }, + { + "auxiliary_loss_clip": 0.01220291, + "auxiliary_loss_mlp": 0.01080902, + "balance_loss_clip": 1.05562949, + "balance_loss_mlp": 1.03772473, + "epoch": 0.017149323892983577, + "flos": 35546399965440.0, + "grad_norm": 1.930963832882248, + "language_loss": 0.94577932, + "learning_rate": 3.6776494905199557e-06, + "loss": 0.96879125, + "num_input_tokens_seen": 16163055, + "router_z_loss_clip": 1.64746094, + "router_z_loss_mlp": 0.43200684, + "step": 591, + "time_per_iteration": 2.5602543354034424 + }, + { + "auxiliary_loss_clip": 0.01226076, + "auxiliary_loss_mlp": 0.0109096, + "balance_loss_clip": 1.06049085, + "balance_loss_mlp": 1.04961801, + "epoch": 0.01717834136149962, + "flos": 52583386775040.0, + "grad_norm": 4.468146365034736, + "language_loss": 0.89243031, + "learning_rate": 3.6786237427530713e-06, + "loss": 0.91560066, + "num_input_tokens_seen": 16183490, + "router_z_loss_clip": 1.65625, + "router_z_loss_mlp": 0.41357422, + "step": 592, + "time_per_iteration": 2.898890256881714 + }, + { + "auxiliary_loss_clip": 0.01203495, + "auxiliary_loss_mlp": 0.010704, + "balance_loss_clip": 1.05315733, + "balance_loss_mlp": 1.03606796, + "epoch": 0.01720735883001567, + "flos": 19562743008000.0, + "grad_norm": 3.6140155674423924, + "language_loss": 0.95218801, + "learning_rate": 3.679596350677839e-06, + "loss": 0.97492695, + "num_input_tokens_seen": 16194475, + "router_z_loss_clip": 1.50195312, + "router_z_loss_mlp": 0.34350586, + "step": 593, + "time_per_iteration": 2.4162065982818604 + }, + { + "auxiliary_loss_clip": 0.01217428, + "auxiliary_loss_mlp": 0.0108347, + "balance_loss_clip": 1.0537343, + "balance_loss_mlp": 1.04343987, + "epoch": 0.017236376298531714, + "flos": 30738021477120.0, + "grad_norm": 3.4765155553466682, + "language_loss": 0.91927582, + "learning_rate": 3.6805673198353194e-06, + "loss": 0.94228482, + "num_input_tokens_seen": 16211245, + "router_z_loss_clip": 1.63574219, + "router_z_loss_mlp": 0.40063477, + "step": 594, + "time_per_iteration": 2.5506889820098877 + }, + { + "auxiliary_loss_clip": 0.01043678, + "auxiliary_loss_mlp": 0.01013749, + "balance_loss_clip": 1.00460315, + "balance_loss_mlp": 1.00855172, + "epoch": 0.017265393767047763, + "flos": 57137246519040.0, + "grad_norm": 0.8356850949048538, + "language_loss": 0.56119823, + "learning_rate": 3.6815366557386092e-06, + "loss": 0.58177251, + "num_input_tokens_seen": 16260285, + "router_z_loss_clip": 0.390625, + "router_z_loss_mlp": 0.05200195, + "step": 595, + "time_per_iteration": 2.797882080078125 + }, + { + "auxiliary_loss_clip": 0.0121035, + "auxiliary_loss_mlp": 0.01085139, + "balance_loss_clip": 1.05460584, + "balance_loss_mlp": 1.0489229, + "epoch": 0.01729441123556381, + "flos": 24673887740160.0, + "grad_norm": 3.030697342191339, + "language_loss": 0.91607249, + "learning_rate": 3.6825043638730345e-06, + "loss": 0.93902743, + "num_input_tokens_seen": 16276780, + "router_z_loss_clip": 1.55761719, + "router_z_loss_mlp": 0.36206055, + "step": 596, + "time_per_iteration": 2.5305585861206055 + }, + { + "auxiliary_loss_clip": 0.01206123, + "auxiliary_loss_mlp": 0.01071038, + "balance_loss_clip": 1.05284667, + "balance_loss_mlp": 1.03513181, + "epoch": 0.017323428704079855, + "flos": 40834868826240.0, + "grad_norm": 5.6790630044692065, + "language_loss": 1.03824115, + "learning_rate": 3.6834704496963308e-06, + "loss": 1.06101274, + "num_input_tokens_seen": 16294420, + "router_z_loss_clip": 1.53222656, + "router_z_loss_mlp": 0.35888672, + "step": 597, + "time_per_iteration": 2.5986716747283936 + }, + { + "auxiliary_loss_clip": 0.01040495, + "auxiliary_loss_mlp": 0.01006831, + "balance_loss_clip": 1.00182831, + "balance_loss_mlp": 1.00194323, + "epoch": 0.017352446172595903, + "flos": 71594919371520.0, + "grad_norm": 0.7147222865500354, + "language_loss": 0.5237087, + "learning_rate": 3.6844349186388327e-06, + "loss": 0.54418194, + "num_input_tokens_seen": 16357610, + "router_z_loss_clip": 0.38671875, + "router_z_loss_mlp": 0.04882812, + "step": 598, + "time_per_iteration": 3.1441187858581543 + }, + { + "auxiliary_loss_clip": 0.01214791, + "auxiliary_loss_mlp": 0.01068358, + "balance_loss_clip": 1.05405164, + "balance_loss_mlp": 1.03185606, + "epoch": 0.017381463641111948, + "flos": 22192924682880.0, + "grad_norm": 2.505397267201671, + "language_loss": 0.76338977, + "learning_rate": 3.685397776103655e-06, + "loss": 0.78622121, + "num_input_tokens_seen": 16375270, + "router_z_loss_clip": 1.60449219, + "router_z_loss_mlp": 0.36499023, + "step": 599, + "time_per_iteration": 2.4275758266448975 + }, + { + "auxiliary_loss_clip": 0.0121407, + "auxiliary_loss_mlp": 0.01076636, + "balance_loss_clip": 1.05553341, + "balance_loss_mlp": 1.04063439, + "epoch": 0.017410481109627996, + "flos": 20915522524800.0, + "grad_norm": 2.9397698142463913, + "language_loss": 0.89975667, + "learning_rate": 3.686359027466873e-06, + "loss": 0.92266375, + "num_input_tokens_seen": 16389455, + "router_z_loss_clip": 1.5859375, + "router_z_loss_mlp": 0.35961914, + "step": 600, + "time_per_iteration": 2.478641986846924 + }, + { + "auxiliary_loss_clip": 0.01204295, + "auxiliary_loss_mlp": 0.01061961, + "balance_loss_clip": 1.05436826, + "balance_loss_mlp": 1.02893972, + "epoch": 0.017439498578144044, + "flos": 18290193528960.0, + "grad_norm": 2.89378358163654, + "language_loss": 0.92475665, + "learning_rate": 3.6873186780777043e-06, + "loss": 0.94741917, + "num_input_tokens_seen": 16403635, + "router_z_loss_clip": 1.50097656, + "router_z_loss_mlp": 0.33007812, + "step": 601, + "time_per_iteration": 2.5225956439971924 + }, + { + "auxiliary_loss_clip": 0.01214772, + "auxiliary_loss_mlp": 0.01076153, + "balance_loss_clip": 1.05780375, + "balance_loss_mlp": 1.03955626, + "epoch": 0.01746851604666009, + "flos": 18616666452480.0, + "grad_norm": 3.0396936519571276, + "language_loss": 0.81943661, + "learning_rate": 3.688276733258688e-06, + "loss": 0.84234589, + "num_input_tokens_seen": 16417665, + "router_z_loss_clip": 1.56933594, + "router_z_loss_mlp": 0.36621094, + "step": 602, + "time_per_iteration": 2.4350485801696777 + }, + { + "auxiliary_loss_clip": 0.01041349, + "auxiliary_loss_mlp": 0.01008574, + "balance_loss_clip": 1.00289249, + "balance_loss_mlp": 1.00373363, + "epoch": 0.017497533515176137, + "flos": 74079722678400.0, + "grad_norm": 0.7449924950033974, + "language_loss": 0.53939807, + "learning_rate": 3.689233198305862e-06, + "loss": 0.5598973, + "num_input_tokens_seen": 16483255, + "router_z_loss_clip": 0.38476562, + "router_z_loss_mlp": 0.04833984, + "step": 603, + "time_per_iteration": 3.095670461654663 + }, + { + "auxiliary_loss_clip": 0.0104025, + "auxiliary_loss_mlp": 0.01005894, + "balance_loss_clip": 1.00203323, + "balance_loss_mlp": 1.00098228, + "epoch": 0.01752655098369218, + "flos": 53905196183040.0, + "grad_norm": 0.7537604658552993, + "language_loss": 0.51252711, + "learning_rate": 3.6901880784889333e-06, + "loss": 0.53298855, + "num_input_tokens_seen": 16536955, + "router_z_loss_clip": 0.3828125, + "router_z_loss_mlp": 0.04907227, + "step": 604, + "time_per_iteration": 2.914414882659912 + }, + { + "auxiliary_loss_clip": 0.01217271, + "auxiliary_loss_mlp": 0.01069668, + "balance_loss_clip": 1.05591857, + "balance_loss_mlp": 1.03240347, + "epoch": 0.01755556845220823, + "flos": 26752860869760.0, + "grad_norm": 3.096179474326431, + "language_loss": 1.1088748, + "learning_rate": 3.6911413790514606e-06, + "loss": 1.13174427, + "num_input_tokens_seen": 16550330, + "router_z_loss_clip": 1.61523438, + "router_z_loss_mlp": 0.37255859, + "step": 605, + "time_per_iteration": 2.5081214904785156 + }, + { + "auxiliary_loss_clip": 0.01220324, + "auxiliary_loss_mlp": 0.01077659, + "balance_loss_clip": 1.05889773, + "balance_loss_mlp": 1.04275465, + "epoch": 0.017584585920724274, + "flos": 12486686158080.0, + "grad_norm": 3.6624030119983213, + "language_loss": 0.9022674, + "learning_rate": 3.6920931052110214e-06, + "loss": 0.92524719, + "num_input_tokens_seen": 16560930, + "router_z_loss_clip": 1.61230469, + "router_z_loss_mlp": 0.34912109, + "step": 606, + "time_per_iteration": 2.410839319229126 + }, + { + "auxiliary_loss_clip": 0.01038709, + "auxiliary_loss_mlp": 0.01009838, + "balance_loss_clip": 1.0012213, + "balance_loss_mlp": 1.00516462, + "epoch": 0.017613603389240323, + "flos": 74776239768960.0, + "grad_norm": 0.9383147153341552, + "language_loss": 0.52318412, + "learning_rate": 3.693043262159385e-06, + "loss": 0.54366958, + "num_input_tokens_seen": 16629165, + "router_z_loss_clip": 0.375, + "router_z_loss_mlp": 0.04663086, + "step": 607, + "time_per_iteration": 3.1564393043518066 + }, + { + "auxiliary_loss_clip": 0.01038549, + "auxiliary_loss_mlp": 0.01009934, + "balance_loss_clip": 1.00116682, + "balance_loss_mlp": 1.00538051, + "epoch": 0.01764262085775637, + "flos": 60793560230400.0, + "grad_norm": 0.7366257546590168, + "language_loss": 0.56967884, + "learning_rate": 3.6939918550626825e-06, + "loss": 0.59016365, + "num_input_tokens_seen": 16691410, + "router_z_loss_clip": 0.375, + "router_z_loss_mlp": 0.0456543, + "step": 608, + "time_per_iteration": 3.045623302459717 + }, + { + "auxiliary_loss_clip": 0.0119695, + "auxiliary_loss_mlp": 0.01062539, + "balance_loss_clip": 1.05178034, + "balance_loss_mlp": 1.02716994, + "epoch": 0.017671638326272415, + "flos": 19637631607680.0, + "grad_norm": 4.913003440211704, + "language_loss": 0.80533946, + "learning_rate": 3.694938889061574e-06, + "loss": 0.82793438, + "num_input_tokens_seen": 16707460, + "router_z_loss_clip": 1.45214844, + "router_z_loss_mlp": 0.35351562, + "step": 609, + "time_per_iteration": 2.442518949508667 + }, + { + "auxiliary_loss_clip": 0.01213645, + "auxiliary_loss_mlp": 0.01067832, + "balance_loss_clip": 1.05706143, + "balance_loss_mlp": 1.02880311, + "epoch": 0.017700655794788463, + "flos": 25912780801920.0, + "grad_norm": 12.078127771376025, + "language_loss": 1.00541568, + "learning_rate": 3.695884369271419e-06, + "loss": 1.02823043, + "num_input_tokens_seen": 16721175, + "router_z_loss_clip": 1.56445312, + "router_z_loss_mlp": 0.38989258, + "step": 610, + "time_per_iteration": 2.457926034927368 + }, + { + "auxiliary_loss_clip": 0.01209075, + "auxiliary_loss_mlp": 0.01066809, + "balance_loss_clip": 1.0597856, + "balance_loss_mlp": 1.03146386, + "epoch": 0.017729673263304508, + "flos": 12229865130240.0, + "grad_norm": 3.000760770958378, + "language_loss": 0.8832165, + "learning_rate": 3.6968283007824383e-06, + "loss": 0.90597534, + "num_input_tokens_seen": 16733305, + "router_z_loss_clip": 1.4921875, + "router_z_loss_mlp": 0.35388184, + "step": 611, + "time_per_iteration": 2.4361684322357178 + }, + { + "auxiliary_loss_clip": 0.01207627, + "auxiliary_loss_mlp": 0.01069867, + "balance_loss_clip": 1.0522716, + "balance_loss_mlp": 1.03446221, + "epoch": 0.017758690731820556, + "flos": 13216510552320.0, + "grad_norm": 3.0398406954682504, + "language_loss": 1.11716509, + "learning_rate": 3.697770688659881e-06, + "loss": 1.13994002, + "num_input_tokens_seen": 16744370, + "router_z_loss_clip": 1.55371094, + "router_z_loss_mlp": 0.35424805, + "step": 612, + "time_per_iteration": 2.399001359939575 + }, + { + "auxiliary_loss_clip": 0.01208675, + "auxiliary_loss_mlp": 0.01064075, + "balance_loss_clip": 1.05405402, + "balance_loss_mlp": 1.02981412, + "epoch": 0.017787708200336604, + "flos": 37011355280640.0, + "grad_norm": 2.9320552482337474, + "language_loss": 0.93716055, + "learning_rate": 3.6987115379441873e-06, + "loss": 0.95988804, + "num_input_tokens_seen": 16762245, + "router_z_loss_clip": 1.54589844, + "router_z_loss_mlp": 0.34240723, + "step": 613, + "time_per_iteration": 2.595562696456909 + }, + { + "auxiliary_loss_clip": 0.0103834, + "auxiliary_loss_mlp": 0.01007958, + "balance_loss_clip": 1.00202477, + "balance_loss_mlp": 1.00349987, + "epoch": 0.01781672566885265, + "flos": 60688261969920.0, + "grad_norm": 0.7196316475920849, + "language_loss": 0.58956623, + "learning_rate": 3.6996508536511475e-06, + "loss": 0.61002922, + "num_input_tokens_seen": 16823525, + "router_z_loss_clip": 0.36328125, + "router_z_loss_mlp": 0.04467773, + "step": 614, + "time_per_iteration": 2.981311321258545 + }, + { + "auxiliary_loss_clip": 0.0121275, + "auxiliary_loss_mlp": 0.0106722, + "balance_loss_clip": 1.05510163, + "balance_loss_mlp": 1.03019321, + "epoch": 0.017845743137368697, + "flos": 36824570173440.0, + "grad_norm": 3.3274341867204327, + "language_loss": 0.8991757, + "learning_rate": 3.7005886407720676e-06, + "loss": 0.92197537, + "num_input_tokens_seen": 16839510, + "router_z_loss_clip": 1.57714844, + "router_z_loss_mlp": 0.37060547, + "step": 615, + "time_per_iteration": 9.683468341827393 + }, + { + "auxiliary_loss_clip": 0.01037756, + "auxiliary_loss_mlp": 0.01006179, + "balance_loss_clip": 1.00120854, + "balance_loss_mlp": 1.00152969, + "epoch": 0.01787476060588474, + "flos": 67067382792960.0, + "grad_norm": 0.6806720922177373, + "language_loss": 0.54606247, + "learning_rate": 3.7015249042739234e-06, + "loss": 0.56650186, + "num_input_tokens_seen": 16901575, + "router_z_loss_clip": 0.36523438, + "router_z_loss_mlp": 0.04638672, + "step": 616, + "time_per_iteration": 3.020332098007202 + }, + { + "auxiliary_loss_clip": 0.01204448, + "auxiliary_loss_mlp": 0.01069594, + "balance_loss_clip": 1.05046546, + "balance_loss_mlp": 1.03464198, + "epoch": 0.01790377807440079, + "flos": 18984999962880.0, + "grad_norm": 3.2276764842365284, + "language_loss": 0.94098926, + "learning_rate": 3.7024596490995227e-06, + "loss": 0.96372968, + "num_input_tokens_seen": 16915745, + "router_z_loss_clip": 1.54101562, + "router_z_loss_mlp": 0.34960938, + "step": 617, + "time_per_iteration": 4.945127487182617 + }, + { + "auxiliary_loss_clip": 0.01035923, + "auxiliary_loss_mlp": 0.01003563, + "balance_loss_clip": 1.00065708, + "balance_loss_mlp": 0.9988904, + "epoch": 0.017932795542916834, + "flos": 64951925425920.0, + "grad_norm": 0.7447069182233065, + "language_loss": 0.53618622, + "learning_rate": 3.7033928801676558e-06, + "loss": 0.55658108, + "num_input_tokens_seen": 16975495, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.04663086, + "step": 618, + "time_per_iteration": 2.9310927391052246 + }, + { + "auxiliary_loss_clip": 0.0121106, + "auxiliary_loss_mlp": 0.01083425, + "balance_loss_clip": 1.05453634, + "balance_loss_mlp": 1.04501605, + "epoch": 0.017961813011432883, + "flos": 16938635909760.0, + "grad_norm": 2.8284367472807728, + "language_loss": 0.85331005, + "learning_rate": 3.70432460237326e-06, + "loss": 0.87625486, + "num_input_tokens_seen": 16990735, + "router_z_loss_clip": 1.56347656, + "router_z_loss_mlp": 0.3840332, + "step": 619, + "time_per_iteration": 2.438553810119629 + }, + { + "auxiliary_loss_clip": 0.0103584, + "auxiliary_loss_mlp": 0.01004234, + "balance_loss_clip": 1.00051928, + "balance_loss_mlp": 0.99963236, + "epoch": 0.01799083047994893, + "flos": 62111670900480.0, + "grad_norm": 0.7117361759935381, + "language_loss": 0.51081938, + "learning_rate": 3.705254820587563e-06, + "loss": 0.53122008, + "num_input_tokens_seen": 17051900, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.04589844, + "step": 620, + "time_per_iteration": 2.9380455017089844 + }, + { + "auxiliary_loss_clip": 0.01035844, + "auxiliary_loss_mlp": 0.01007846, + "balance_loss_clip": 1.00064826, + "balance_loss_mlp": 1.00331593, + "epoch": 0.018019847948464975, + "flos": 55720645303680.0, + "grad_norm": 0.6770010933950152, + "language_loss": 0.53755116, + "learning_rate": 3.7061835396582444e-06, + "loss": 0.55798805, + "num_input_tokens_seen": 17105620, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.04541016, + "step": 621, + "time_per_iteration": 2.8429529666900635 + }, + { + "auxiliary_loss_clip": 0.01205309, + "auxiliary_loss_mlp": 0.0106409, + "balance_loss_clip": 1.05917299, + "balance_loss_mlp": 1.02956688, + "epoch": 0.018048865416981023, + "flos": 74728725408000.0, + "grad_norm": 4.0486473755284464, + "language_loss": 1.10458159, + "learning_rate": 3.707110764409583e-06, + "loss": 1.12727559, + "num_input_tokens_seen": 17128415, + "router_z_loss_clip": 1.45996094, + "router_z_loss_mlp": 0.34521484, + "step": 622, + "time_per_iteration": 2.8875319957733154 + }, + { + "auxiliary_loss_clip": 0.01212725, + "auxiliary_loss_mlp": 0.01061023, + "balance_loss_clip": 1.05944836, + "balance_loss_mlp": 1.02790666, + "epoch": 0.018077882885497068, + "flos": 11720412437760.0, + "grad_norm": 2.4782137507892092, + "language_loss": 0.8329283, + "learning_rate": 3.708036499642607e-06, + "loss": 0.8556658, + "num_input_tokens_seen": 17142540, + "router_z_loss_clip": 1.53222656, + "router_z_loss_mlp": 0.33105469, + "step": 623, + "time_per_iteration": 2.461448907852173 + }, + { + "auxiliary_loss_clip": 0.01199453, + "auxiliary_loss_mlp": 0.01070242, + "balance_loss_clip": 1.05109024, + "balance_loss_mlp": 1.03551626, + "epoch": 0.018106900354013116, + "flos": 33721175664000.0, + "grad_norm": 2.8403523373236603, + "language_loss": 1.07398129, + "learning_rate": 3.708960750135246e-06, + "loss": 1.09667826, + "num_input_tokens_seen": 17158405, + "router_z_loss_clip": 1.484375, + "router_z_loss_mlp": 0.34753418, + "step": 624, + "time_per_iteration": 2.675741672515869 + }, + { + "auxiliary_loss_clip": 0.01212285, + "auxiliary_loss_mlp": 0.01066213, + "balance_loss_clip": 1.0572859, + "balance_loss_mlp": 1.0308553, + "epoch": 0.018135917822529164, + "flos": 74729912394240.0, + "grad_norm": 2.495497873427527, + "language_loss": 0.82738525, + "learning_rate": 3.7098835206424755e-06, + "loss": 0.85017025, + "num_input_tokens_seen": 17182745, + "router_z_loss_clip": 1.54882812, + "router_z_loss_mlp": 0.35351562, + "step": 625, + "time_per_iteration": 2.851353645324707 + }, + { + "auxiliary_loss_clip": 0.0121487, + "auxiliary_loss_mlp": 0.01070527, + "balance_loss_clip": 1.05585837, + "balance_loss_mlp": 1.0330596, + "epoch": 0.01816493529104521, + "flos": 20841960556800.0, + "grad_norm": 2.4243082849413295, + "language_loss": 0.97746277, + "learning_rate": 3.7108048158964674e-06, + "loss": 1.00031674, + "num_input_tokens_seen": 17197765, + "router_z_loss_clip": 1.59082031, + "router_z_loss_mlp": 0.37475586, + "step": 626, + "time_per_iteration": 2.4894142150878906 + }, + { + "auxiliary_loss_clip": 0.01214185, + "auxiliary_loss_mlp": 0.01078815, + "balance_loss_clip": 1.06126845, + "balance_loss_mlp": 1.04073966, + "epoch": 0.018193952759561257, + "flos": 11320447368960.0, + "grad_norm": 3.687059545372061, + "language_loss": 0.78013247, + "learning_rate": 3.711724640606732e-06, + "loss": 0.80306244, + "num_input_tokens_seen": 17210110, + "router_z_loss_clip": 1.52734375, + "router_z_loss_mlp": 0.38061523, + "step": 627, + "time_per_iteration": 2.404533863067627 + }, + { + "auxiliary_loss_clip": 0.01215739, + "auxiliary_loss_mlp": 0.01075381, + "balance_loss_clip": 1.05856979, + "balance_loss_mlp": 1.03787756, + "epoch": 0.0182229702280773, + "flos": 38755721139840.0, + "grad_norm": 2.6364117904071898, + "language_loss": 0.89708769, + "learning_rate": 3.712642999460262e-06, + "loss": 0.91999894, + "num_input_tokens_seen": 17231000, + "router_z_loss_clip": 1.5703125, + "router_z_loss_mlp": 0.375, + "step": 628, + "time_per_iteration": 2.6853344440460205 + }, + { + "auxiliary_loss_clip": 0.01204058, + "auxiliary_loss_mlp": 0.010733, + "balance_loss_clip": 1.05356944, + "balance_loss_mlp": 1.04260373, + "epoch": 0.01825198769659335, + "flos": 39270689827200.0, + "grad_norm": 2.258182536378473, + "language_loss": 0.86520886, + "learning_rate": 3.713559897121683e-06, + "loss": 0.88798249, + "num_input_tokens_seen": 17248095, + "router_z_loss_clip": 1.50390625, + "router_z_loss_mlp": 0.30688477, + "step": 629, + "time_per_iteration": 2.5750608444213867 + }, + { + "auxiliary_loss_clip": 0.01213906, + "auxiliary_loss_mlp": 0.01078421, + "balance_loss_clip": 1.05885887, + "balance_loss_mlp": 1.04156101, + "epoch": 0.018281005165109394, + "flos": 22157767077120.0, + "grad_norm": 2.509726164658903, + "language_loss": 0.94818223, + "learning_rate": 3.7144753382333854e-06, + "loss": 0.97110546, + "num_input_tokens_seen": 17263175, + "router_z_loss_clip": 1.55078125, + "router_z_loss_mlp": 0.36889648, + "step": 630, + "time_per_iteration": 2.4872052669525146 + }, + { + "auxiliary_loss_clip": 0.01038445, + "auxiliary_loss_mlp": 0.01004749, + "balance_loss_clip": 1.0034349, + "balance_loss_mlp": 1.00048137, + "epoch": 0.018310022633625442, + "flos": 69156968976000.0, + "grad_norm": 0.7295599919016915, + "language_loss": 0.5087676, + "learning_rate": 3.7153893274156738e-06, + "loss": 0.52919954, + "num_input_tokens_seen": 17327605, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.04272461, + "step": 631, + "time_per_iteration": 3.027191162109375 + }, + { + "auxiliary_loss_clip": 0.01198422, + "auxiliary_loss_mlp": 0.01073542, + "balance_loss_clip": 1.05222154, + "balance_loss_mlp": 1.03742206, + "epoch": 0.01833904010214149, + "flos": 17138058929280.0, + "grad_norm": 4.185062565016966, + "language_loss": 0.9007436, + "learning_rate": 3.7163018692669016e-06, + "loss": 0.92346323, + "num_input_tokens_seen": 17342545, + "router_z_loss_clip": 1.4609375, + "router_z_loss_mlp": 0.36120605, + "step": 632, + "time_per_iteration": 2.546887159347534 + }, + { + "auxiliary_loss_clip": 0.01038699, + "auxiliary_loss_mlp": 0.01008452, + "balance_loss_clip": 1.00289202, + "balance_loss_mlp": 1.00399339, + "epoch": 0.018368057570657535, + "flos": 57008765249280.0, + "grad_norm": 0.6504686298909169, + "language_loss": 0.50104165, + "learning_rate": 3.717212968363613e-06, + "loss": 0.52151316, + "num_input_tokens_seen": 17405975, + "router_z_loss_clip": 0.35742188, + "router_z_loss_mlp": 0.04467773, + "step": 633, + "time_per_iteration": 3.1367757320404053 + }, + { + "auxiliary_loss_clip": 0.01211059, + "auxiliary_loss_mlp": 0.0107857, + "balance_loss_clip": 1.05261803, + "balance_loss_mlp": 1.03956497, + "epoch": 0.018397075039173583, + "flos": 10150612709760.0, + "grad_norm": 3.1661372958270544, + "language_loss": 0.92645383, + "learning_rate": 3.7181226292606785e-06, + "loss": 0.94935012, + "num_input_tokens_seen": 17415530, + "router_z_loss_clip": 1.58496094, + "router_z_loss_mlp": 0.38989258, + "step": 634, + "time_per_iteration": 2.4141783714294434 + }, + { + "auxiliary_loss_clip": 0.01200216, + "auxiliary_loss_mlp": 0.01063218, + "balance_loss_clip": 1.05151749, + "balance_loss_mlp": 1.02776527, + "epoch": 0.018426092507689628, + "flos": 28249936502400.0, + "grad_norm": 2.702542181680624, + "language_loss": 0.80299652, + "learning_rate": 3.7190308564914345e-06, + "loss": 0.82563084, + "num_input_tokens_seen": 17436895, + "router_z_loss_clip": 1.48535156, + "router_z_loss_mlp": 0.3548584, + "step": 635, + "time_per_iteration": 2.7683820724487305 + }, + { + "auxiliary_loss_clip": 0.01220115, + "auxiliary_loss_mlp": 0.01080387, + "balance_loss_clip": 1.05913734, + "balance_loss_mlp": 1.03682828, + "epoch": 0.018455109976205676, + "flos": 24168275297280.0, + "grad_norm": 4.100731218223105, + "language_loss": 0.92885429, + "learning_rate": 3.719937654567814e-06, + "loss": 0.9518593, + "num_input_tokens_seen": 17454505, + "router_z_loss_clip": 1.60839844, + "router_z_loss_mlp": 0.43554688, + "step": 636, + "time_per_iteration": 2.476973533630371 + }, + { + "auxiliary_loss_clip": 0.0103649, + "auxiliary_loss_mlp": 0.01007659, + "balance_loss_clip": 1.00134492, + "balance_loss_mlp": 1.00346327, + "epoch": 0.01848412744472172, + "flos": 63133648485120.0, + "grad_norm": 0.6571775493063435, + "language_loss": 0.5523262, + "learning_rate": 3.7208430279804867e-06, + "loss": 0.57276767, + "num_input_tokens_seen": 17519505, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.04199219, + "step": 637, + "time_per_iteration": 3.04866623878479 + }, + { + "auxiliary_loss_clip": 0.01037222, + "auxiliary_loss_mlp": 0.01006083, + "balance_loss_clip": 1.00195622, + "balance_loss_mlp": 1.00179124, + "epoch": 0.01851314491323777, + "flos": 64667348000640.0, + "grad_norm": 0.6959188327822372, + "language_loss": 0.55901885, + "learning_rate": 3.7217469811989875e-06, + "loss": 0.57945192, + "num_input_tokens_seen": 17580635, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.04296875, + "step": 638, + "time_per_iteration": 3.0158917903900146 + }, + { + "auxiliary_loss_clip": 0.01208027, + "auxiliary_loss_mlp": 0.01066239, + "balance_loss_clip": 1.05316448, + "balance_loss_mlp": 1.02809215, + "epoch": 0.018542162381753817, + "flos": 16792803694080.0, + "grad_norm": 2.9200052851131297, + "language_loss": 0.89019001, + "learning_rate": 3.722649518671853e-06, + "loss": 0.91293263, + "num_input_tokens_seen": 17593785, + "router_z_loss_clip": 1.54882812, + "router_z_loss_mlp": 0.38134766, + "step": 639, + "time_per_iteration": 2.4041764736175537 + }, + { + "auxiliary_loss_clip": 0.01036634, + "auxiliary_loss_mlp": 0.01002619, + "balance_loss_clip": 1.0014565, + "balance_loss_mlp": 0.99842268, + "epoch": 0.01857117985026986, + "flos": 74770828508160.0, + "grad_norm": 0.7040882007066361, + "language_loss": 0.55160099, + "learning_rate": 3.7235506448267494e-06, + "loss": 0.57199353, + "num_input_tokens_seen": 17659045, + "router_z_loss_clip": 0.3515625, + "router_z_loss_mlp": 0.04199219, + "step": 640, + "time_per_iteration": 3.132100820541382 + }, + { + "auxiliary_loss_clip": 0.01036843, + "auxiliary_loss_mlp": 0.01003359, + "balance_loss_clip": 1.00145578, + "balance_loss_mlp": 0.99925804, + "epoch": 0.01860019731878591, + "flos": 68639486670720.0, + "grad_norm": 0.747546690346156, + "language_loss": 0.5291847, + "learning_rate": 3.724450364070606e-06, + "loss": 0.54958677, + "num_input_tokens_seen": 17713855, + "router_z_loss_clip": 0.35351562, + "router_z_loss_mlp": 0.04101562, + "step": 641, + "time_per_iteration": 2.934887409210205 + }, + { + "auxiliary_loss_clip": 0.01036757, + "auxiliary_loss_mlp": 0.01004288, + "balance_loss_clip": 1.00128794, + "balance_loss_mlp": 1.00011563, + "epoch": 0.018629214787301954, + "flos": 65471642058240.0, + "grad_norm": 0.7006305732375798, + "language_loss": 0.52002728, + "learning_rate": 3.7253486807897415e-06, + "loss": 0.5404377, + "num_input_tokens_seen": 17775570, + "router_z_loss_clip": 0.35546875, + "router_z_loss_mlp": 0.04174805, + "step": 642, + "time_per_iteration": 3.055210828781128 + }, + { + "auxiliary_loss_clip": 0.01212461, + "auxiliary_loss_mlp": 0.0106746, + "balance_loss_clip": 1.05461764, + "balance_loss_mlp": 1.03012371, + "epoch": 0.018658232255818002, + "flos": 37625058892800.0, + "grad_norm": 2.6010168276128534, + "language_loss": 0.97835588, + "learning_rate": 3.726245599349994e-06, + "loss": 1.00115514, + "num_input_tokens_seen": 17791295, + "router_z_loss_clip": 1.57910156, + "router_z_loss_mlp": 0.37329102, + "step": 643, + "time_per_iteration": 2.5750985145568848 + }, + { + "auxiliary_loss_clip": 0.01035816, + "auxiliary_loss_mlp": 0.01008712, + "balance_loss_clip": 1.00163126, + "balance_loss_mlp": 1.00442004, + "epoch": 0.01868724972433405, + "flos": 69616985316480.0, + "grad_norm": 0.7722073499110899, + "language_loss": 0.52961802, + "learning_rate": 3.7271411240968497e-06, + "loss": 0.55006331, + "num_input_tokens_seen": 17846505, + "router_z_loss_clip": 0.34179688, + "router_z_loss_mlp": 0.04296875, + "step": 644, + "time_per_iteration": 2.9952733516693115 + }, + { + "auxiliary_loss_clip": 0.01210258, + "auxiliary_loss_mlp": 0.01065221, + "balance_loss_clip": 1.05335999, + "balance_loss_mlp": 1.0309124, + "epoch": 0.018716267192850095, + "flos": 27702812586240.0, + "grad_norm": 2.8164560561815395, + "language_loss": 1.07266808, + "learning_rate": 3.728035259355564e-06, + "loss": 1.09542298, + "num_input_tokens_seen": 17864505, + "router_z_loss_clip": 1.56640625, + "router_z_loss_mlp": 0.34301758, + "step": 645, + "time_per_iteration": 2.551999807357788 + }, + { + "auxiliary_loss_clip": 0.01191956, + "auxiliary_loss_mlp": 0.01060435, + "balance_loss_clip": 1.04880309, + "balance_loss_mlp": 1.02793908, + "epoch": 0.018745284661366143, + "flos": 28431345260160.0, + "grad_norm": 2.5845338158075597, + "language_loss": 0.6957733, + "learning_rate": 3.7289280094312938e-06, + "loss": 0.71829724, + "num_input_tokens_seen": 17884095, + "router_z_loss_clip": 1.43164062, + "router_z_loss_mlp": 0.32495117, + "step": 646, + "time_per_iteration": 2.6199851036071777 + }, + { + "auxiliary_loss_clip": 0.01188373, + "auxiliary_loss_mlp": 0.01064265, + "balance_loss_clip": 1.04878068, + "balance_loss_mlp": 1.03412843, + "epoch": 0.018774302129882188, + "flos": 23542178152320.0, + "grad_norm": 2.60429525498411, + "language_loss": 0.83000016, + "learning_rate": 3.729819378609217e-06, + "loss": 0.85252649, + "num_input_tokens_seen": 17899715, + "router_z_loss_clip": 1.39648438, + "router_z_loss_mlp": 0.30151367, + "step": 647, + "time_per_iteration": 2.4510204792022705 + }, + { + "auxiliary_loss_clip": 0.01209225, + "auxiliary_loss_mlp": 0.01075164, + "balance_loss_clip": 1.05811536, + "balance_loss_mlp": 1.04297709, + "epoch": 0.018803319598398236, + "flos": 11974719847680.0, + "grad_norm": 3.4154645118677474, + "language_loss": 0.98540497, + "learning_rate": 3.730709371154657e-06, + "loss": 1.00824893, + "num_input_tokens_seen": 17909490, + "router_z_loss_clip": 1.51074219, + "router_z_loss_mlp": 0.32177734, + "step": 648, + "time_per_iteration": 2.4300811290740967 + }, + { + "auxiliary_loss_clip": 0.01202307, + "auxiliary_loss_mlp": 0.0106891, + "balance_loss_clip": 1.05309927, + "balance_loss_mlp": 1.03430319, + "epoch": 0.01883233706691428, + "flos": 15990569406720.0, + "grad_norm": 6.0421732324071815, + "language_loss": 0.96395999, + "learning_rate": 3.731597991313208e-06, + "loss": 0.98667216, + "num_input_tokens_seen": 17924135, + "router_z_loss_clip": 1.4921875, + "router_z_loss_mlp": 0.34606934, + "step": 649, + "time_per_iteration": 2.3764450550079346 + }, + { + "auxiliary_loss_clip": 0.01034983, + "auxiliary_loss_mlp": 0.01003885, + "balance_loss_clip": 1.00142908, + "balance_loss_mlp": 0.99964154, + "epoch": 0.01886135453543033, + "flos": 63606477294720.0, + "grad_norm": 0.7373541574833498, + "language_loss": 0.5344305, + "learning_rate": 3.732485243310849e-06, + "loss": 0.55481917, + "num_input_tokens_seen": 17988495, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.04248047, + "step": 650, + "time_per_iteration": 3.1011767387390137 + }, + { + "auxiliary_loss_clip": 0.01200976, + "auxiliary_loss_mlp": 0.0106287, + "balance_loss_clip": 1.05359006, + "balance_loss_mlp": 1.02949142, + "epoch": 0.018890372003946377, + "flos": 30985416437760.0, + "grad_norm": 2.8648695753727185, + "language_loss": 1.01143479, + "learning_rate": 3.733371131354075e-06, + "loss": 1.03407335, + "num_input_tokens_seen": 18003780, + "router_z_loss_clip": 1.47363281, + "router_z_loss_mlp": 0.33337402, + "step": 651, + "time_per_iteration": 2.4998068809509277 + }, + { + "auxiliary_loss_clip": 0.01196772, + "auxiliary_loss_mlp": 0.01073696, + "balance_loss_clip": 1.05333364, + "balance_loss_mlp": 1.03796864, + "epoch": 0.01891938947246242, + "flos": 36783547459200.0, + "grad_norm": 2.7567220917017496, + "language_loss": 0.95508671, + "learning_rate": 3.734255659630009e-06, + "loss": 0.97779131, + "num_input_tokens_seen": 18019785, + "router_z_loss_clip": 1.43359375, + "router_z_loss_mlp": 0.35693359, + "step": 652, + "time_per_iteration": 2.6138010025024414 + }, + { + "auxiliary_loss_clip": 0.01209769, + "auxiliary_loss_mlp": 0.01076121, + "balance_loss_clip": 1.05759668, + "balance_loss_mlp": 1.03926098, + "epoch": 0.01894840694097847, + "flos": 33213887475840.0, + "grad_norm": 2.486919223350404, + "language_loss": 0.96077442, + "learning_rate": 3.7351388323065203e-06, + "loss": 0.98363328, + "num_input_tokens_seen": 18035145, + "router_z_loss_clip": 1.52148438, + "router_z_loss_mlp": 0.36816406, + "step": 653, + "time_per_iteration": 2.5019094944000244 + }, + { + "auxiliary_loss_clip": 0.01211195, + "auxiliary_loss_mlp": 0.01079901, + "balance_loss_clip": 1.05639923, + "balance_loss_mlp": 1.04485321, + "epoch": 0.018977424409494514, + "flos": 18761311681920.0, + "grad_norm": 3.2353403736817423, + "language_loss": 0.99763286, + "learning_rate": 3.7360206535323494e-06, + "loss": 1.02054381, + "num_input_tokens_seen": 18048260, + "router_z_loss_clip": 1.54980469, + "router_z_loss_mlp": 0.35083008, + "step": 654, + "time_per_iteration": 2.4039928913116455 + }, + { + "auxiliary_loss_clip": 0.01196395, + "auxiliary_loss_mlp": 0.01070949, + "balance_loss_clip": 1.05282354, + "balance_loss_mlp": 1.03479254, + "epoch": 0.019006441878010562, + "flos": 12158712046080.0, + "grad_norm": 3.435186083060895, + "language_loss": 1.01670992, + "learning_rate": 3.7369011274372165e-06, + "loss": 1.03938341, + "num_input_tokens_seen": 18060555, + "router_z_loss_clip": 1.43359375, + "router_z_loss_mlp": 0.36157227, + "step": 655, + "time_per_iteration": 2.3564419746398926 + }, + { + "auxiliary_loss_clip": 0.01199462, + "auxiliary_loss_mlp": 0.01073284, + "balance_loss_clip": 1.05159175, + "balance_loss_mlp": 1.03709257, + "epoch": 0.01903545934652661, + "flos": 15921022245120.0, + "grad_norm": 3.6894834664413927, + "language_loss": 0.82605278, + "learning_rate": 3.737780258131944e-06, + "loss": 0.84878021, + "num_input_tokens_seen": 18073535, + "router_z_loss_clip": 1.47949219, + "router_z_loss_mlp": 0.36206055, + "step": 656, + "time_per_iteration": 2.3947219848632812 + }, + { + "auxiliary_loss_clip": 0.01193344, + "auxiliary_loss_mlp": 0.01059635, + "balance_loss_clip": 1.05317187, + "balance_loss_mlp": 1.02822375, + "epoch": 0.019064476815042655, + "flos": 27444664926720.0, + "grad_norm": 2.4987250008156, + "language_loss": 0.98849881, + "learning_rate": 3.738658049708568e-06, + "loss": 1.01102865, + "num_input_tokens_seen": 18089265, + "router_z_loss_clip": 1.40234375, + "router_z_loss_mlp": 0.31420898, + "step": 657, + "time_per_iteration": 2.4487404823303223 + }, + { + "auxiliary_loss_clip": 0.01205291, + "auxiliary_loss_mlp": 0.01066888, + "balance_loss_clip": 1.05398405, + "balance_loss_mlp": 1.03293753, + "epoch": 0.019093494283558703, + "flos": 16500685415040.0, + "grad_norm": 2.98493351715419, + "language_loss": 0.82342106, + "learning_rate": 3.739534506240455e-06, + "loss": 0.84614289, + "num_input_tokens_seen": 18102775, + "router_z_loss_clip": 1.515625, + "router_z_loss_mlp": 0.33959961, + "step": 658, + "time_per_iteration": 2.469813346862793 + }, + { + "auxiliary_loss_clip": 0.01037148, + "auxiliary_loss_mlp": 0.01008546, + "balance_loss_clip": 1.00217748, + "balance_loss_mlp": 1.00439763, + "epoch": 0.019122511752074748, + "flos": 74591828634240.0, + "grad_norm": 0.7002569261473527, + "language_loss": 0.53688276, + "learning_rate": 3.7404096317824104e-06, + "loss": 0.55733967, + "num_input_tokens_seen": 18168635, + "router_z_loss_clip": 0.34960938, + "router_z_loss_mlp": 0.04150391, + "step": 659, + "time_per_iteration": 3.0868256092071533 + }, + { + "auxiliary_loss_clip": 0.01211741, + "auxiliary_loss_mlp": 0.01068765, + "balance_loss_clip": 1.05483174, + "balance_loss_mlp": 1.03293049, + "epoch": 0.019151529220590796, + "flos": 45833767937280.0, + "grad_norm": 2.9335271582118314, + "language_loss": 0.90107656, + "learning_rate": 3.741283430370799e-06, + "loss": 0.92388159, + "num_input_tokens_seen": 18185620, + "router_z_loss_clip": 1.5703125, + "router_z_loss_mlp": 0.3581543, + "step": 660, + "time_per_iteration": 2.6605520248413086 + }, + { + "auxiliary_loss_clip": 0.01184773, + "auxiliary_loss_mlp": 0.01048033, + "balance_loss_clip": 1.04582381, + "balance_loss_mlp": 1.01920772, + "epoch": 0.01918054668910684, + "flos": 47949958442880.0, + "grad_norm": 2.9173407302763055, + "language_loss": 0.9749105, + "learning_rate": 3.74215590602365e-06, + "loss": 0.99723852, + "num_input_tokens_seen": 18203070, + "router_z_loss_clip": 1.38867188, + "router_z_loss_mlp": 0.28833008, + "step": 661, + "time_per_iteration": 2.6588892936706543 + }, + { + "auxiliary_loss_clip": 0.01204912, + "auxiliary_loss_mlp": 0.010718, + "balance_loss_clip": 1.05053115, + "balance_loss_mlp": 1.03114915, + "epoch": 0.01920956415762289, + "flos": 74730366241920.0, + "grad_norm": 2.0654745780096535, + "language_loss": 0.90167373, + "learning_rate": 3.743027062740771e-06, + "loss": 0.92444086, + "num_input_tokens_seen": 18236040, + "router_z_loss_clip": 1.54394531, + "router_z_loss_mlp": 0.40625, + "step": 662, + "time_per_iteration": 2.9456450939178467 + }, + { + "auxiliary_loss_clip": 0.01197224, + "auxiliary_loss_mlp": 0.01070874, + "balance_loss_clip": 1.05113316, + "balance_loss_mlp": 1.03434873, + "epoch": 0.019238581626138937, + "flos": 27672333102720.0, + "grad_norm": 2.9245531317035858, + "language_loss": 1.00648487, + "learning_rate": 3.743896904503857e-06, + "loss": 1.02916586, + "num_input_tokens_seen": 18251270, + "router_z_loss_clip": 1.45898438, + "router_z_loss_mlp": 0.36547852, + "step": 663, + "time_per_iteration": 2.5020687580108643 + }, + { + "auxiliary_loss_clip": 0.01037448, + "auxiliary_loss_mlp": 0.01004414, + "balance_loss_clip": 1.00291359, + "balance_loss_mlp": 1.0002898, + "epoch": 0.01926759909465498, + "flos": 61309646081280.0, + "grad_norm": 0.7564397365036813, + "language_loss": 0.56493741, + "learning_rate": 3.7447654352766005e-06, + "loss": 0.58535606, + "num_input_tokens_seen": 18310320, + "router_z_loss_clip": 0.34570312, + "router_z_loss_mlp": 0.04125977, + "step": 664, + "time_per_iteration": 2.9521727561950684 + }, + { + "auxiliary_loss_clip": 0.01202467, + "auxiliary_loss_mlp": 0.01073425, + "balance_loss_clip": 1.05370259, + "balance_loss_mlp": 1.03946233, + "epoch": 0.01929661656317103, + "flos": 46818213943680.0, + "grad_norm": 2.581962385141825, + "language_loss": 0.75140011, + "learning_rate": 3.7456326590047978e-06, + "loss": 0.77415901, + "num_input_tokens_seen": 18331365, + "router_z_loss_clip": 1.48730469, + "router_z_loss_mlp": 0.33972168, + "step": 665, + "time_per_iteration": 2.679278612136841 + }, + { + "auxiliary_loss_clip": 0.0120005, + "auxiliary_loss_mlp": 0.01066928, + "balance_loss_clip": 1.05249548, + "balance_loss_mlp": 1.03116488, + "epoch": 0.019325634031687074, + "flos": 22169671850880.0, + "grad_norm": 2.171288740988848, + "language_loss": 0.88766408, + "learning_rate": 3.746498579616459e-06, + "loss": 0.91033387, + "num_input_tokens_seen": 18352115, + "router_z_loss_clip": 1.47558594, + "router_z_loss_mlp": 0.35766602, + "step": 666, + "time_per_iteration": 2.495293140411377 + }, + { + "auxiliary_loss_clip": 0.0120622, + "auxiliary_loss_mlp": 0.01080083, + "balance_loss_clip": 1.0529778, + "balance_loss_mlp": 1.0440104, + "epoch": 0.019354651500203122, + "flos": 29305291213440.0, + "grad_norm": 1.9859342133433948, + "language_loss": 0.86860633, + "learning_rate": 3.747363201021913e-06, + "loss": 0.89146936, + "num_input_tokens_seen": 18370185, + "router_z_loss_clip": 1.53222656, + "router_z_loss_mlp": 0.3605957, + "step": 667, + "time_per_iteration": 2.4874274730682373 + }, + { + "auxiliary_loss_clip": 0.01196188, + "auxiliary_loss_mlp": 0.01070705, + "balance_loss_clip": 1.05225182, + "balance_loss_mlp": 1.03680217, + "epoch": 0.01938366896871917, + "flos": 28833684301440.0, + "grad_norm": 2.8922810065430586, + "language_loss": 0.885037, + "learning_rate": 3.7482265271139155e-06, + "loss": 0.90770584, + "num_input_tokens_seen": 18386270, + "router_z_loss_clip": 1.43945312, + "router_z_loss_mlp": 0.33911133, + "step": 668, + "time_per_iteration": 2.520082712173462 + }, + { + "auxiliary_loss_clip": 0.01036859, + "auxiliary_loss_mlp": 0.01004197, + "balance_loss_clip": 1.00223339, + "balance_loss_mlp": 1.00016809, + "epoch": 0.019412686437235215, + "flos": 74779486525440.0, + "grad_norm": 0.6929356987618069, + "language_loss": 0.54124045, + "learning_rate": 3.7490885617677517e-06, + "loss": 0.56165099, + "num_input_tokens_seen": 18454165, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.0402832, + "step": 669, + "time_per_iteration": 3.1645383834838867 + }, + { + "auxiliary_loss_clip": 0.01223454, + "auxiliary_loss_mlp": 0.01074952, + "balance_loss_clip": 1.0588789, + "balance_loss_mlp": 1.03468347, + "epoch": 0.019441703905751263, + "flos": 30365184401280.0, + "grad_norm": 2.47186678470338, + "language_loss": 1.15419054, + "learning_rate": 3.7499493088413417e-06, + "loss": 1.17717457, + "num_input_tokens_seen": 18471430, + "router_z_loss_clip": 1.64453125, + "router_z_loss_mlp": 0.40258789, + "step": 670, + "time_per_iteration": 2.638146162033081 + }, + { + "auxiliary_loss_clip": 0.01193419, + "auxiliary_loss_mlp": 0.01071314, + "balance_loss_clip": 1.05286908, + "balance_loss_mlp": 1.03897285, + "epoch": 0.019470721374267308, + "flos": 27630681984000.0, + "grad_norm": 2.728834792516675, + "language_loss": 0.99257582, + "learning_rate": 3.750808772175345e-06, + "loss": 1.01522315, + "num_input_tokens_seen": 18489680, + "router_z_loss_clip": 1.40722656, + "router_z_loss_mlp": 0.32336426, + "step": 671, + "time_per_iteration": 2.477191209793091 + }, + { + "auxiliary_loss_clip": 0.01192329, + "auxiliary_loss_mlp": 0.01067415, + "balance_loss_clip": 1.05038083, + "balance_loss_mlp": 1.03298736, + "epoch": 0.019499738842783356, + "flos": 32953959336960.0, + "grad_norm": 2.8614731655878813, + "language_loss": 0.80874008, + "learning_rate": 3.7516669555932624e-06, + "loss": 0.83133751, + "num_input_tokens_seen": 18504105, + "router_z_loss_clip": 1.41894531, + "router_z_loss_mlp": 0.34472656, + "step": 672, + "time_per_iteration": 2.4295542240142822 + }, + { + "auxiliary_loss_clip": 0.01036414, + "auxiliary_loss_mlp": 0.01004545, + "balance_loss_clip": 1.00184488, + "balance_loss_mlp": 1.00058758, + "epoch": 0.0195287563112994, + "flos": 65977813082880.0, + "grad_norm": 0.709785661543485, + "language_loss": 0.5806483, + "learning_rate": 3.7525238629015374e-06, + "loss": 0.60105789, + "num_input_tokens_seen": 18562865, + "router_z_loss_clip": 0.34570312, + "router_z_loss_mlp": 0.03955078, + "step": 673, + "time_per_iteration": 2.962172746658325 + }, + { + "auxiliary_loss_clip": 0.01036187, + "auxiliary_loss_mlp": 0.01003321, + "balance_loss_clip": 1.00184059, + "balance_loss_mlp": 0.99929166, + "epoch": 0.01955777377981545, + "flos": 56647344188160.0, + "grad_norm": 0.7023416150159748, + "language_loss": 0.54057795, + "learning_rate": 3.7533794978896586e-06, + "loss": 0.56097305, + "num_input_tokens_seen": 18625005, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.0402832, + "step": 674, + "time_per_iteration": 2.9950685501098633 + }, + { + "auxiliary_loss_clip": 0.01200583, + "auxiliary_loss_mlp": 0.0106709, + "balance_loss_clip": 1.05636168, + "balance_loss_mlp": 1.0317446, + "epoch": 0.019586791248331497, + "flos": 39595591739520.0, + "grad_norm": 2.762961198447772, + "language_loss": 1.0579983, + "learning_rate": 3.7542338643302607e-06, + "loss": 1.08067501, + "num_input_tokens_seen": 18642340, + "router_z_loss_clip": 1.44042969, + "router_z_loss_mlp": 0.35375977, + "step": 675, + "time_per_iteration": 2.5815629959106445 + }, + { + "auxiliary_loss_clip": 0.0103582, + "auxiliary_loss_mlp": 0.01004822, + "balance_loss_clip": 1.00223458, + "balance_loss_mlp": 1.00093555, + "epoch": 0.01961580871684754, + "flos": 53722914641280.0, + "grad_norm": 0.763824340124174, + "language_loss": 0.55513179, + "learning_rate": 3.7550869659792225e-06, + "loss": 0.57553816, + "num_input_tokens_seen": 18699755, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.03881836, + "step": 676, + "time_per_iteration": 2.947244167327881 + }, + { + "auxiliary_loss_clip": 0.01196946, + "auxiliary_loss_mlp": 0.01072522, + "balance_loss_clip": 1.04917359, + "balance_loss_mlp": 1.04002583, + "epoch": 0.01964482618536359, + "flos": 15516204497280.0, + "grad_norm": 3.2841766303907156, + "language_loss": 0.82157135, + "learning_rate": 3.755938806575768e-06, + "loss": 0.844266, + "num_input_tokens_seen": 18711300, + "router_z_loss_clip": 1.47753906, + "router_z_loss_mlp": 0.32495117, + "step": 677, + "time_per_iteration": 2.421269178390503 + }, + { + "auxiliary_loss_clip": 0.01035454, + "auxiliary_loss_mlp": 0.01006119, + "balance_loss_clip": 1.00149012, + "balance_loss_mlp": 1.00204206, + "epoch": 0.019673843653879634, + "flos": 74767581751680.0, + "grad_norm": 0.7809523481518975, + "language_loss": 0.54023147, + "learning_rate": 3.756789389842562e-06, + "loss": 0.56064713, + "num_input_tokens_seen": 18767475, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.04077148, + "step": 678, + "time_per_iteration": 3.023847818374634 + }, + { + "auxiliary_loss_clip": 0.0120352, + "auxiliary_loss_mlp": 0.01068282, + "balance_loss_clip": 1.05737638, + "balance_loss_mlp": 1.03785992, + "epoch": 0.019702861122395682, + "flos": 28248156023040.0, + "grad_norm": 6.536647447260352, + "language_loss": 0.9282335, + "learning_rate": 3.7576387194858126e-06, + "loss": 0.95095158, + "num_input_tokens_seen": 18783635, + "router_z_loss_clip": 1.46191406, + "router_z_loss_mlp": 0.30444336, + "step": 679, + "time_per_iteration": 2.435426712036133 + }, + { + "auxiliary_loss_clip": 0.0118931, + "auxiliary_loss_mlp": 0.01056339, + "balance_loss_clip": 1.05121541, + "balance_loss_mlp": 1.02504659, + "epoch": 0.01973187859091173, + "flos": 50186773296000.0, + "grad_norm": 2.854820932816463, + "language_loss": 1.08126485, + "learning_rate": 3.7584867991953607e-06, + "loss": 1.10372138, + "num_input_tokens_seen": 18807275, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.31298828, + "step": 680, + "time_per_iteration": 2.7501306533813477 + }, + { + "auxiliary_loss_clip": 0.0119608, + "auxiliary_loss_mlp": 0.01061628, + "balance_loss_clip": 1.05513382, + "balance_loss_mlp": 1.0271883, + "epoch": 0.019760896059427775, + "flos": 41276240634240.0, + "grad_norm": 2.732916928454866, + "language_loss": 0.97541249, + "learning_rate": 3.7593336326447845e-06, + "loss": 0.99798954, + "num_input_tokens_seen": 18824035, + "router_z_loss_clip": 1.40820312, + "router_z_loss_mlp": 0.34436035, + "step": 681, + "time_per_iteration": 2.624089479446411 + }, + { + "auxiliary_loss_clip": 0.01208096, + "auxiliary_loss_mlp": 0.01072296, + "balance_loss_clip": 1.05509353, + "balance_loss_mlp": 1.03765392, + "epoch": 0.019789913527943823, + "flos": 25841522983680.0, + "grad_norm": 2.966359851044342, + "language_loss": 0.98177814, + "learning_rate": 3.760179223491489e-06, + "loss": 1.00458205, + "num_input_tokens_seen": 18845295, + "router_z_loss_clip": 1.53222656, + "router_z_loss_mlp": 0.34667969, + "step": 682, + "time_per_iteration": 2.8029632568359375 + }, + { + "auxiliary_loss_clip": 0.01194803, + "auxiliary_loss_mlp": 0.01064007, + "balance_loss_clip": 1.05173206, + "balance_loss_mlp": 1.0313437, + "epoch": 0.019818930996459868, + "flos": 16468146161280.0, + "grad_norm": 3.305109515068567, + "language_loss": 0.81396598, + "learning_rate": 3.761023575376802e-06, + "loss": 0.83655405, + "num_input_tokens_seen": 18859190, + "router_z_loss_clip": 1.4296875, + "router_z_loss_mlp": 0.32641602, + "step": 683, + "time_per_iteration": 2.4048240184783936 + }, + { + "auxiliary_loss_clip": 0.01208101, + "auxiliary_loss_mlp": 0.01072581, + "balance_loss_clip": 1.05425847, + "balance_loss_mlp": 1.03410017, + "epoch": 0.019847948464975916, + "flos": 25183794280320.0, + "grad_norm": 2.7653865253476515, + "language_loss": 1.05674338, + "learning_rate": 3.7618666919260695e-06, + "loss": 1.07955027, + "num_input_tokens_seen": 18873435, + "router_z_loss_clip": 1.53808594, + "router_z_loss_mlp": 0.38476562, + "step": 684, + "time_per_iteration": 2.479614496231079 + }, + { + "auxiliary_loss_clip": 0.01192493, + "auxiliary_loss_mlp": 0.01069994, + "balance_loss_clip": 1.05269229, + "balance_loss_mlp": 1.0401206, + "epoch": 0.01987696593349196, + "flos": 12778734614400.0, + "grad_norm": 3.3931922334107307, + "language_loss": 0.76937097, + "learning_rate": 3.7627085767487498e-06, + "loss": 0.79199588, + "num_input_tokens_seen": 18886415, + "router_z_loss_clip": 1.3984375, + "router_z_loss_mlp": 0.29882812, + "step": 685, + "time_per_iteration": 2.3680825233459473 + }, + { + "auxiliary_loss_clip": 0.01200751, + "auxiliary_loss_mlp": 0.01068373, + "balance_loss_clip": 1.0526334, + "balance_loss_mlp": 1.03547096, + "epoch": 0.01990598340200801, + "flos": 25111140007680.0, + "grad_norm": 3.4975530676207716, + "language_loss": 0.79248303, + "learning_rate": 3.7635492334385024e-06, + "loss": 0.81517422, + "num_input_tokens_seen": 18900680, + "router_z_loss_clip": 1.47949219, + "router_z_loss_mlp": 0.32910156, + "step": 686, + "time_per_iteration": 2.4157774448394775 + }, + { + "auxiliary_loss_clip": 0.01035682, + "auxiliary_loss_mlp": 0.01003428, + "balance_loss_clip": 1.00182474, + "balance_loss_mlp": 0.99980372, + "epoch": 0.019935000870524057, + "flos": 56353619986560.0, + "grad_norm": 0.6771502528846192, + "language_loss": 0.5288012, + "learning_rate": 3.7643886655732852e-06, + "loss": 0.54919231, + "num_input_tokens_seen": 18959550, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.03613281, + "step": 687, + "time_per_iteration": 3.0851378440856934 + }, + { + "auxiliary_loss_clip": 0.01190009, + "auxiliary_loss_mlp": 0.01051975, + "balance_loss_clip": 1.05069041, + "balance_loss_mlp": 1.0223639, + "epoch": 0.0199640183390401, + "flos": 41128069357440.0, + "grad_norm": 6.079353108831381, + "language_loss": 0.76385915, + "learning_rate": 3.76522687671544e-06, + "loss": 0.78627896, + "num_input_tokens_seen": 18975105, + "router_z_loss_clip": 1.39355469, + "router_z_loss_mlp": 0.29614258, + "step": 688, + "time_per_iteration": 2.5849123001098633 + }, + { + "auxiliary_loss_clip": 0.01201379, + "auxiliary_loss_mlp": 0.01076219, + "balance_loss_clip": 1.0535326, + "balance_loss_mlp": 1.04191089, + "epoch": 0.01999303580755615, + "flos": 12742599490560.0, + "grad_norm": 6.8930908926940155, + "language_loss": 1.04935086, + "learning_rate": 3.7660638704117904e-06, + "loss": 1.07212687, + "num_input_tokens_seen": 18985985, + "router_z_loss_clip": 1.47949219, + "router_z_loss_mlp": 0.34313965, + "step": 689, + "time_per_iteration": 4.874150991439819 + }, + { + "auxiliary_loss_clip": 0.01210576, + "auxiliary_loss_mlp": 0.01067648, + "balance_loss_clip": 1.0577774, + "balance_loss_mlp": 1.02932215, + "epoch": 0.020022053276072194, + "flos": 29963229384960.0, + "grad_norm": 3.214491811181341, + "language_loss": 1.07883501, + "learning_rate": 3.766899650193724e-06, + "loss": 1.10161734, + "num_input_tokens_seen": 18999805, + "router_z_loss_clip": 1.52734375, + "router_z_loss_mlp": 0.38354492, + "step": 690, + "time_per_iteration": 2.528341770172119 + }, + { + "auxiliary_loss_clip": 0.01200975, + "auxiliary_loss_mlp": 0.01071774, + "balance_loss_clip": 1.05355811, + "balance_loss_mlp": 1.03663123, + "epoch": 0.020051070744588242, + "flos": 16208462401920.0, + "grad_norm": 3.149152166802156, + "language_loss": 0.99337512, + "learning_rate": 3.7677342195772886e-06, + "loss": 1.01610255, + "num_input_tokens_seen": 19013040, + "router_z_loss_clip": 1.47363281, + "router_z_loss_mlp": 0.35131836, + "step": 691, + "time_per_iteration": 7.189133882522583 + }, + { + "auxiliary_loss_clip": 0.01034555, + "auxiliary_loss_mlp": 0.01011239, + "balance_loss_clip": 1.00160122, + "balance_loss_mlp": 1.00773394, + "epoch": 0.020080088213104287, + "flos": 68276075662080.0, + "grad_norm": 0.7847910202296332, + "language_loss": 0.51206386, + "learning_rate": 3.7685675820632748e-06, + "loss": 0.53252178, + "num_input_tokens_seen": 19076865, + "router_z_loss_clip": 0.328125, + "router_z_loss_mlp": 0.03515625, + "step": 692, + "time_per_iteration": 3.055546760559082 + }, + { + "auxiliary_loss_clip": 0.01201676, + "auxiliary_loss_mlp": 0.01067074, + "balance_loss_clip": 1.05129707, + "balance_loss_mlp": 1.03145397, + "epoch": 0.020109105681620335, + "flos": 30218898337920.0, + "grad_norm": 2.805149479063932, + "language_loss": 0.89432287, + "learning_rate": 3.7693997411373113e-06, + "loss": 0.91701031, + "num_input_tokens_seen": 19091515, + "router_z_loss_clip": 1.50292969, + "router_z_loss_mlp": 0.35668945, + "step": 693, + "time_per_iteration": 2.4059526920318604 + }, + { + "auxiliary_loss_clip": 0.01034587, + "auxiliary_loss_mlp": 0.0100879, + "balance_loss_clip": 1.00118148, + "balance_loss_mlp": 1.00514245, + "epoch": 0.020138123150136383, + "flos": 67430689067520.0, + "grad_norm": 0.7595185686348097, + "language_loss": 0.53814912, + "learning_rate": 3.770230700269945e-06, + "loss": 0.5585829, + "num_input_tokens_seen": 19148115, + "router_z_loss_clip": 0.33398438, + "router_z_loss_mlp": 0.03637695, + "step": 694, + "time_per_iteration": 5.35961389541626 + }, + { + "auxiliary_loss_clip": 0.01179322, + "auxiliary_loss_mlp": 0.01062479, + "balance_loss_clip": 1.05018151, + "balance_loss_mlp": 1.0326407, + "epoch": 0.020167140618652428, + "flos": 32301502248960.0, + "grad_norm": 2.6448476516173423, + "language_loss": 0.82576954, + "learning_rate": 3.7710604629167325e-06, + "loss": 0.84818757, + "num_input_tokens_seen": 19166560, + "router_z_loss_clip": 1.29003906, + "router_z_loss_mlp": 0.2980957, + "step": 695, + "time_per_iteration": 2.516496181488037 + }, + { + "auxiliary_loss_clip": 0.01197561, + "auxiliary_loss_mlp": 0.01060459, + "balance_loss_clip": 1.05409503, + "balance_loss_mlp": 1.02834451, + "epoch": 0.020196158087168476, + "flos": 35179462909440.0, + "grad_norm": 2.9522050073780215, + "language_loss": 1.00194776, + "learning_rate": 3.771889032518326e-06, + "loss": 1.02452803, + "num_input_tokens_seen": 19181395, + "router_z_loss_clip": 1.43457031, + "router_z_loss_mlp": 0.32092285, + "step": 696, + "time_per_iteration": 2.526660203933716 + }, + { + "auxiliary_loss_clip": 0.01187483, + "auxiliary_loss_mlp": 0.01056743, + "balance_loss_clip": 1.04833746, + "balance_loss_mlp": 1.02599871, + "epoch": 0.02022517555568452, + "flos": 28505011962240.0, + "grad_norm": 3.073922398411438, + "language_loss": 0.759987, + "learning_rate": 3.7727164125005555e-06, + "loss": 0.78242934, + "num_input_tokens_seen": 19197925, + "router_z_loss_clip": 1.39160156, + "router_z_loss_mlp": 0.30737305, + "step": 697, + "time_per_iteration": 2.4138567447662354 + }, + { + "auxiliary_loss_clip": 0.01190176, + "auxiliary_loss_mlp": 0.01065673, + "balance_loss_clip": 1.0518558, + "balance_loss_mlp": 1.03401089, + "epoch": 0.02025419302420057, + "flos": 13542389982720.0, + "grad_norm": 3.311849471018633, + "language_loss": 1.0983696, + "learning_rate": 3.7735426062745193e-06, + "loss": 1.12092817, + "num_input_tokens_seen": 19207700, + "router_z_loss_clip": 1.38378906, + "router_z_loss_mlp": 0.31640625, + "step": 698, + "time_per_iteration": 2.3930299282073975 + }, + { + "auxiliary_loss_clip": 0.01204486, + "auxiliary_loss_mlp": 0.01077595, + "balance_loss_clip": 1.06035137, + "balance_loss_mlp": 1.04159379, + "epoch": 0.020283210492716617, + "flos": 27045398085120.0, + "grad_norm": 3.160570929628857, + "language_loss": 1.02444804, + "learning_rate": 3.7743676172366622e-06, + "loss": 1.04726875, + "num_input_tokens_seen": 19222820, + "router_z_loss_clip": 1.44042969, + "router_z_loss_mlp": 0.36010742, + "step": 699, + "time_per_iteration": 2.4631052017211914 + }, + { + "auxiliary_loss_clip": 0.01205962, + "auxiliary_loss_mlp": 0.01067961, + "balance_loss_clip": 1.05776691, + "balance_loss_mlp": 1.03401005, + "epoch": 0.02031222796123266, + "flos": 33619438362240.0, + "grad_norm": 2.230177549990912, + "language_loss": 0.76747388, + "learning_rate": 3.775191448768865e-06, + "loss": 0.79021311, + "num_input_tokens_seen": 19239690, + "router_z_loss_clip": 1.48046875, + "router_z_loss_mlp": 0.33959961, + "step": 700, + "time_per_iteration": 2.4630494117736816 + }, + { + "auxiliary_loss_clip": 0.01191092, + "auxiliary_loss_mlp": 0.01070708, + "balance_loss_clip": 1.05153537, + "balance_loss_mlp": 1.04016662, + "epoch": 0.02034124542974871, + "flos": 11429690613120.0, + "grad_norm": 4.849069928331142, + "language_loss": 1.06554437, + "learning_rate": 3.776014104238524e-06, + "loss": 1.08816242, + "num_input_tokens_seen": 19251150, + "router_z_loss_clip": 1.39453125, + "router_z_loss_mlp": 0.30517578, + "step": 701, + "time_per_iteration": 2.4173805713653564 + }, + { + "auxiliary_loss_clip": 0.01204417, + "auxiliary_loss_mlp": 0.01070222, + "balance_loss_clip": 1.05274296, + "balance_loss_mlp": 1.03417325, + "epoch": 0.020370262898264754, + "flos": 10517410120320.0, + "grad_norm": 2.869631416940182, + "language_loss": 0.84946287, + "learning_rate": 3.7768355869986333e-06, + "loss": 0.87220931, + "num_input_tokens_seen": 19261715, + "router_z_loss_clip": 1.515625, + "router_z_loss_mlp": 0.36035156, + "step": 702, + "time_per_iteration": 2.393442153930664 + }, + { + "auxiliary_loss_clip": 0.01041143, + "auxiliary_loss_mlp": 0.010124, + "balance_loss_clip": 1.00920451, + "balance_loss_mlp": 1.00899112, + "epoch": 0.020399280366780802, + "flos": 57695297690880.0, + "grad_norm": 0.6987982290455604, + "language_loss": 0.51473361, + "learning_rate": 3.7776559003878716e-06, + "loss": 0.53526902, + "num_input_tokens_seen": 19322490, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.03417969, + "step": 703, + "time_per_iteration": 3.018455743789673 + }, + { + "auxiliary_loss_clip": 0.01037284, + "auxiliary_loss_mlp": 0.01007003, + "balance_loss_clip": 1.00509715, + "balance_loss_mlp": 1.00361753, + "epoch": 0.020428297835296847, + "flos": 69301649116800.0, + "grad_norm": 0.7311833012248009, + "language_loss": 0.52377349, + "learning_rate": 3.7784750477306753e-06, + "loss": 0.54421633, + "num_input_tokens_seen": 19388305, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.03393555, + "step": 704, + "time_per_iteration": 3.080904722213745 + }, + { + "auxiliary_loss_clip": 0.01196798, + "auxiliary_loss_mlp": 0.01071214, + "balance_loss_clip": 1.05241537, + "balance_loss_mlp": 1.03742981, + "epoch": 0.020457315303812895, + "flos": 52181745960960.0, + "grad_norm": 3.285560588451551, + "language_loss": 0.93675351, + "learning_rate": 3.7792930323373297e-06, + "loss": 0.95943367, + "num_input_tokens_seen": 19410815, + "router_z_loss_clip": 1.44433594, + "router_z_loss_mlp": 0.33764648, + "step": 705, + "time_per_iteration": 2.7402560710906982 + }, + { + "auxiliary_loss_clip": 0.01210265, + "auxiliary_loss_mlp": 0.01076531, + "balance_loss_clip": 1.05270445, + "balance_loss_mlp": 1.03595209, + "epoch": 0.020486332772328943, + "flos": 31936939165440.0, + "grad_norm": 2.9435036815972957, + "language_loss": 1.06620765, + "learning_rate": 3.780109857504039e-06, + "loss": 1.08907557, + "num_input_tokens_seen": 19426225, + "router_z_loss_clip": 1.57519531, + "router_z_loss_mlp": 0.40551758, + "step": 706, + "time_per_iteration": 2.511474370956421 + }, + { + "auxiliary_loss_clip": 0.0120016, + "auxiliary_loss_mlp": 0.01073175, + "balance_loss_clip": 1.05509567, + "balance_loss_mlp": 1.03939128, + "epoch": 0.020515350240844988, + "flos": 11172590294400.0, + "grad_norm": 3.0274828121560944, + "language_loss": 0.8356033, + "learning_rate": 3.7809255265130137e-06, + "loss": 0.85833669, + "num_input_tokens_seen": 19436275, + "router_z_loss_clip": 1.45019531, + "router_z_loss_mlp": 0.33789062, + "step": 707, + "time_per_iteration": 2.401668071746826 + }, + { + "auxiliary_loss_clip": 0.01205353, + "auxiliary_loss_mlp": 0.01082903, + "balance_loss_clip": 1.05648553, + "balance_loss_mlp": 1.04523325, + "epoch": 0.020544367709361036, + "flos": 16718997346560.0, + "grad_norm": 3.9829276161211933, + "language_loss": 0.6953916, + "learning_rate": 3.7817400426325455e-06, + "loss": 0.71827418, + "num_input_tokens_seen": 19448860, + "router_z_loss_clip": 1.48925781, + "router_z_loss_mlp": 0.37670898, + "step": 708, + "time_per_iteration": 2.3714346885681152 + }, + { + "auxiliary_loss_clip": 0.01051257, + "auxiliary_loss_mlp": 0.01030733, + "balance_loss_clip": 1.01672578, + "balance_loss_mlp": 1.02684629, + "epoch": 0.02057338517787708, + "flos": 64733508760320.0, + "grad_norm": 0.7194746996345418, + "language_loss": 0.52824783, + "learning_rate": 3.782553409117088e-06, + "loss": 0.54906774, + "num_input_tokens_seen": 19509640, + "router_z_loss_clip": 0.34375, + "router_z_loss_mlp": 0.03881836, + "step": 709, + "time_per_iteration": 2.95375394821167 + }, + { + "auxiliary_loss_clip": 0.01205019, + "auxiliary_loss_mlp": 0.01074032, + "balance_loss_clip": 1.05513048, + "balance_loss_mlp": 1.03869879, + "epoch": 0.02060240264639313, + "flos": 20300666837760.0, + "grad_norm": 2.5634340506973228, + "language_loss": 0.95386517, + "learning_rate": 3.783365629207333e-06, + "loss": 0.9766556, + "num_input_tokens_seen": 19527220, + "router_z_loss_clip": 1.49902344, + "router_z_loss_mlp": 0.35351562, + "step": 710, + "time_per_iteration": 2.3995308876037598 + }, + { + "auxiliary_loss_clip": 0.01201836, + "auxiliary_loss_mlp": 0.01089223, + "balance_loss_clip": 1.05645776, + "balance_loss_mlp": 1.05415142, + "epoch": 0.020631420114909177, + "flos": 29892530148480.0, + "grad_norm": 2.846174284286976, + "language_loss": 0.8865571, + "learning_rate": 3.7841767061302886e-06, + "loss": 0.9094677, + "num_input_tokens_seen": 19544115, + "router_z_loss_clip": 1.45605469, + "router_z_loss_mlp": 0.35058594, + "step": 711, + "time_per_iteration": 2.552060604095459 + }, + { + "auxiliary_loss_clip": 0.01201288, + "auxiliary_loss_mlp": 0.01066571, + "balance_loss_clip": 1.05464482, + "balance_loss_mlp": 1.03190458, + "epoch": 0.02066043758342522, + "flos": 31941896578560.0, + "grad_norm": 2.0728492193609656, + "language_loss": 0.71803546, + "learning_rate": 3.7849866430993588e-06, + "loss": 0.74071407, + "num_input_tokens_seen": 19562355, + "router_z_loss_clip": 1.46582031, + "router_z_loss_mlp": 0.34680176, + "step": 712, + "time_per_iteration": 2.5179786682128906 + }, + { + "auxiliary_loss_clip": 0.01206432, + "auxiliary_loss_mlp": 0.01088067, + "balance_loss_clip": 1.05440342, + "balance_loss_mlp": 1.05003929, + "epoch": 0.02068945505194127, + "flos": 37699284176640.0, + "grad_norm": 3.87586314685798, + "language_loss": 0.96345937, + "learning_rate": 3.7857954433144147e-06, + "loss": 0.98640436, + "num_input_tokens_seen": 19581335, + "router_z_loss_clip": 1.51855469, + "router_z_loss_mlp": 0.38037109, + "step": 713, + "time_per_iteration": 2.6033051013946533 + }, + { + "auxiliary_loss_clip": 0.01044523, + "auxiliary_loss_mlp": 0.01008075, + "balance_loss_clip": 1.0096097, + "balance_loss_mlp": 1.00411749, + "epoch": 0.020718472520457314, + "flos": 74771980583040.0, + "grad_norm": 0.7124795941629705, + "language_loss": 0.56589437, + "learning_rate": 3.7866031099618737e-06, + "loss": 0.58642036, + "num_input_tokens_seen": 19642865, + "router_z_loss_clip": 0.34765625, + "router_z_loss_mlp": 0.03955078, + "step": 714, + "time_per_iteration": 3.0512890815734863 + }, + { + "auxiliary_loss_clip": 0.01214214, + "auxiliary_loss_mlp": 0.01085787, + "balance_loss_clip": 1.05855989, + "balance_loss_mlp": 1.04758084, + "epoch": 0.020747489988973362, + "flos": 20340013806720.0, + "grad_norm": 2.3151064845951024, + "language_loss": 0.88912892, + "learning_rate": 3.787409646214775e-06, + "loss": 0.91212893, + "num_input_tokens_seen": 19664640, + "router_z_loss_clip": 1.55761719, + "router_z_loss_mlp": 0.38208008, + "step": 715, + "time_per_iteration": 2.5849757194519043 + }, + { + "auxiliary_loss_clip": 0.01035575, + "auxiliary_loss_mlp": 0.01007289, + "balance_loss_clip": 1.00211382, + "balance_loss_mlp": 1.00347471, + "epoch": 0.020776507457489407, + "flos": 65998515519360.0, + "grad_norm": 0.6305870100265896, + "language_loss": 0.52799815, + "learning_rate": 3.788215055232854e-06, + "loss": 0.54842681, + "num_input_tokens_seen": 19735205, + "router_z_loss_clip": 0.33398438, + "router_z_loss_mlp": 0.03808594, + "step": 716, + "time_per_iteration": 3.2027761936187744 + }, + { + "auxiliary_loss_clip": 0.01035062, + "auxiliary_loss_mlp": 0.01005895, + "balance_loss_clip": 1.0020268, + "balance_loss_mlp": 1.00227058, + "epoch": 0.020805524926005455, + "flos": 56274088176000.0, + "grad_norm": 0.7423425316321922, + "language_loss": 0.56199086, + "learning_rate": 3.789019340162615e-06, + "loss": 0.58240044, + "num_input_tokens_seen": 19792645, + "router_z_loss_clip": 0.33007812, + "router_z_loss_mlp": 0.03613281, + "step": 717, + "time_per_iteration": 2.885756254196167 + }, + { + "auxiliary_loss_clip": 0.01196831, + "auxiliary_loss_mlp": 0.01079518, + "balance_loss_clip": 1.05322826, + "balance_loss_mlp": 1.04213417, + "epoch": 0.020834542394521503, + "flos": 12050620876800.0, + "grad_norm": 3.369823532905971, + "language_loss": 0.8399784, + "learning_rate": 3.7898225041374074e-06, + "loss": 0.86274195, + "num_input_tokens_seen": 19803710, + "router_z_loss_clip": 1.43652344, + "router_z_loss_mlp": 0.3737793, + "step": 718, + "time_per_iteration": 2.3932180404663086 + }, + { + "auxiliary_loss_clip": 0.01035079, + "auxiliary_loss_mlp": 0.01008314, + "balance_loss_clip": 1.00290775, + "balance_loss_mlp": 1.00485718, + "epoch": 0.020863559863037548, + "flos": 68886881631360.0, + "grad_norm": 0.7485392960773527, + "language_loss": 0.53492177, + "learning_rate": 3.790624550277496e-06, + "loss": 0.55535567, + "num_input_tokens_seen": 19857780, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.03466797, + "step": 719, + "time_per_iteration": 2.9055490493774414 + }, + { + "auxiliary_loss_clip": 0.01198473, + "auxiliary_loss_mlp": 0.01076802, + "balance_loss_clip": 1.05289316, + "balance_loss_mlp": 1.04158759, + "epoch": 0.020892577331553596, + "flos": 18288936720000.0, + "grad_norm": 2.5016062379296393, + "language_loss": 0.77041483, + "learning_rate": 3.7914254816901373e-06, + "loss": 0.79316759, + "num_input_tokens_seen": 19870425, + "router_z_loss_clip": 1.45605469, + "router_z_loss_mlp": 0.35229492, + "step": 720, + "time_per_iteration": 2.4748215675354004 + }, + { + "auxiliary_loss_clip": 0.01036262, + "auxiliary_loss_mlp": 0.01005811, + "balance_loss_clip": 1.00435567, + "balance_loss_mlp": 1.00216293, + "epoch": 0.02092159480006964, + "flos": 57369802285440.0, + "grad_norm": 0.7442182102580571, + "language_loss": 0.54783964, + "learning_rate": 3.792225301469649e-06, + "loss": 0.56826043, + "num_input_tokens_seen": 19931845, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.03637695, + "step": 721, + "time_per_iteration": 3.0024187564849854 + }, + { + "auxiliary_loss_clip": 0.01195403, + "auxiliary_loss_mlp": 0.01058279, + "balance_loss_clip": 1.0501318, + "balance_loss_mlp": 1.02234912, + "epoch": 0.02095061226858569, + "flos": 28651018734720.0, + "grad_norm": 2.5666554082535438, + "language_loss": 1.12505651, + "learning_rate": 3.793024012697482e-06, + "loss": 1.14759338, + "num_input_tokens_seen": 19953645, + "router_z_loss_clip": 1.453125, + "router_z_loss_mlp": 0.35888672, + "step": 722, + "time_per_iteration": 2.4891552925109863 + }, + { + "auxiliary_loss_clip": 0.01194817, + "auxiliary_loss_mlp": 0.01072515, + "balance_loss_clip": 1.05077779, + "balance_loss_mlp": 1.03778982, + "epoch": 0.020979629737101737, + "flos": 16152460848000.0, + "grad_norm": 12.380717854442375, + "language_loss": 0.90927237, + "learning_rate": 3.7938216184422938e-06, + "loss": 0.93194574, + "num_input_tokens_seen": 19969390, + "router_z_loss_clip": 1.44238281, + "router_z_loss_mlp": 0.34729004, + "step": 723, + "time_per_iteration": 2.427917718887329 + }, + { + "auxiliary_loss_clip": 0.01033507, + "auxiliary_loss_mlp": 0.01008672, + "balance_loss_clip": 1.00132239, + "balance_loss_mlp": 1.00497699, + "epoch": 0.02100864720561778, + "flos": 70186417591680.0, + "grad_norm": 0.8185469918020232, + "language_loss": 0.53861129, + "learning_rate": 3.7946181217600164e-06, + "loss": 0.55903304, + "num_input_tokens_seen": 20034860, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.03686523, + "step": 724, + "time_per_iteration": 3.142052412033081 + }, + { + "auxiliary_loss_clip": 0.0121326, + "auxiliary_loss_mlp": 0.01068903, + "balance_loss_clip": 1.05538511, + "balance_loss_mlp": 1.03261566, + "epoch": 0.02103766467413383, + "flos": 17523116847360.0, + "grad_norm": 4.214410745034199, + "language_loss": 0.80931771, + "learning_rate": 3.795413525693929e-06, + "loss": 0.83213931, + "num_input_tokens_seen": 20050690, + "router_z_loss_clip": 1.57910156, + "router_z_loss_mlp": 0.36279297, + "step": 725, + "time_per_iteration": 2.4000649452209473 + }, + { + "auxiliary_loss_clip": 0.01205031, + "auxiliary_loss_mlp": 0.01066955, + "balance_loss_clip": 1.05225432, + "balance_loss_mlp": 1.02728236, + "epoch": 0.021066682142649874, + "flos": 34817203975680.0, + "grad_norm": 2.473083737269025, + "language_loss": 0.99815285, + "learning_rate": 3.7962078332747247e-06, + "loss": 1.02087283, + "num_input_tokens_seen": 20067375, + "router_z_loss_clip": 1.52734375, + "router_z_loss_mlp": 0.39697266, + "step": 726, + "time_per_iteration": 2.5727546215057373 + }, + { + "auxiliary_loss_clip": 0.01198979, + "auxiliary_loss_mlp": 0.01076896, + "balance_loss_clip": 1.05183804, + "balance_loss_mlp": 1.04103744, + "epoch": 0.021095699611165922, + "flos": 74730680444160.0, + "grad_norm": 2.865082537796974, + "language_loss": 1.03934634, + "learning_rate": 3.7970010475205834e-06, + "loss": 1.06210518, + "num_input_tokens_seen": 20089985, + "router_z_loss_clip": 1.47265625, + "router_z_loss_mlp": 0.35888672, + "step": 727, + "time_per_iteration": 2.819384813308716 + }, + { + "auxiliary_loss_clip": 0.0119819, + "auxiliary_loss_mlp": 0.01078935, + "balance_loss_clip": 1.04754972, + "balance_loss_mlp": 1.04238534, + "epoch": 0.021124717079681967, + "flos": 18544291470720.0, + "grad_norm": 4.662857497509613, + "language_loss": 0.82153773, + "learning_rate": 3.7977931714372386e-06, + "loss": 0.84430891, + "num_input_tokens_seen": 20107145, + "router_z_loss_clip": 1.50683594, + "router_z_loss_mlp": 0.36547852, + "step": 728, + "time_per_iteration": 2.459000825881958 + }, + { + "auxiliary_loss_clip": 0.01202269, + "auxiliary_loss_mlp": 0.0106773, + "balance_loss_clip": 1.05607677, + "balance_loss_mlp": 1.03483987, + "epoch": 0.021153734548198015, + "flos": 74729598192000.0, + "grad_norm": 4.594161507406858, + "language_loss": 0.66932672, + "learning_rate": 3.7985842080180446e-06, + "loss": 0.69202673, + "num_input_tokens_seen": 20128560, + "router_z_loss_clip": 1.4609375, + "router_z_loss_mlp": 0.32873535, + "step": 729, + "time_per_iteration": 2.8028042316436768 + }, + { + "auxiliary_loss_clip": 0.01196546, + "auxiliary_loss_mlp": 0.01073626, + "balance_loss_clip": 1.04773629, + "balance_loss_mlp": 1.03624165, + "epoch": 0.021182752016714063, + "flos": 16830300360960.0, + "grad_norm": 3.2736403084767196, + "language_loss": 0.77569413, + "learning_rate": 3.7993741602440483e-06, + "loss": 0.79839587, + "num_input_tokens_seen": 20142450, + "router_z_loss_clip": 1.48632812, + "router_z_loss_mlp": 0.3737793, + "step": 730, + "time_per_iteration": 2.3961870670318604 + }, + { + "auxiliary_loss_clip": 0.01208954, + "auxiliary_loss_mlp": 0.01072005, + "balance_loss_clip": 1.05353975, + "balance_loss_mlp": 1.0323565, + "epoch": 0.021211769485230108, + "flos": 22886299751040.0, + "grad_norm": 2.9876675611550723, + "language_loss": 0.87351978, + "learning_rate": 3.8001630310840514e-06, + "loss": 0.89632934, + "num_input_tokens_seen": 20157885, + "router_z_loss_clip": 1.55371094, + "router_z_loss_mlp": 0.39648438, + "step": 731, + "time_per_iteration": 2.4098689556121826 + }, + { + "auxiliary_loss_clip": 0.0119577, + "auxiliary_loss_mlp": 0.01068512, + "balance_loss_clip": 1.05284727, + "balance_loss_mlp": 1.03451383, + "epoch": 0.021240786953746156, + "flos": 13363739222400.0, + "grad_norm": 2.4941156272988536, + "language_loss": 0.79908723, + "learning_rate": 3.800950823494683e-06, + "loss": 0.82173002, + "num_input_tokens_seen": 20172155, + "router_z_loss_clip": 1.42675781, + "router_z_loss_mlp": 0.33996582, + "step": 732, + "time_per_iteration": 2.429682731628418 + }, + { + "auxiliary_loss_clip": 0.01200809, + "auxiliary_loss_mlp": 0.01063752, + "balance_loss_clip": 1.05322671, + "balance_loss_mlp": 1.02622545, + "epoch": 0.0212698044222622, + "flos": 30345981287040.0, + "grad_norm": 2.1897436781967348, + "language_loss": 0.94345951, + "learning_rate": 3.8017375404204606e-06, + "loss": 0.9661051, + "num_input_tokens_seen": 20194245, + "router_z_loss_clip": 1.47558594, + "router_z_loss_mlp": 0.375, + "step": 733, + "time_per_iteration": 2.57663893699646 + }, + { + "auxiliary_loss_clip": 0.01188985, + "auxiliary_loss_mlp": 0.01076348, + "balance_loss_clip": 1.04656208, + "balance_loss_mlp": 1.04084742, + "epoch": 0.02129882189077825, + "flos": 27668143739520.0, + "grad_norm": 2.9550763782322864, + "language_loss": 1.11977482, + "learning_rate": 3.802523184793859e-06, + "loss": 1.14242804, + "num_input_tokens_seen": 20213170, + "router_z_loss_clip": 1.42285156, + "router_z_loss_mlp": 0.35498047, + "step": 734, + "time_per_iteration": 2.4670331478118896 + }, + { + "auxiliary_loss_clip": 0.01047699, + "auxiliary_loss_mlp": 0.01012282, + "balance_loss_clip": 1.01315975, + "balance_loss_mlp": 1.00789499, + "epoch": 0.021327839359294293, + "flos": 68640743479680.0, + "grad_norm": 0.7723701749763511, + "language_loss": 0.52900934, + "learning_rate": 3.8033077595353777e-06, + "loss": 0.54960918, + "num_input_tokens_seen": 20273980, + "router_z_loss_clip": 0.34570312, + "router_z_loss_mlp": 0.04394531, + "step": 735, + "time_per_iteration": 3.0035483837127686 + }, + { + "auxiliary_loss_clip": 0.01200007, + "auxiliary_loss_mlp": 0.01077147, + "balance_loss_clip": 1.05294478, + "balance_loss_mlp": 1.0416224, + "epoch": 0.02135685682781034, + "flos": 30766231722240.0, + "grad_norm": 4.901413629863208, + "language_loss": 0.79729223, + "learning_rate": 3.8040912675536016e-06, + "loss": 0.82006371, + "num_input_tokens_seen": 20288030, + "router_z_loss_clip": 1.46972656, + "router_z_loss_mlp": 0.35522461, + "step": 736, + "time_per_iteration": 2.4205737113952637 + }, + { + "auxiliary_loss_clip": 0.0103964, + "auxiliary_loss_mlp": 0.010103, + "balance_loss_clip": 1.00585747, + "balance_loss_mlp": 1.00624657, + "epoch": 0.02138587429632639, + "flos": 69804468650880.0, + "grad_norm": 0.761883737402973, + "language_loss": 0.60163498, + "learning_rate": 3.8048737117452677e-06, + "loss": 0.62213439, + "num_input_tokens_seen": 20349820, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.04052734, + "step": 737, + "time_per_iteration": 2.979874610900879 + }, + { + "auxiliary_loss_clip": 0.01196391, + "auxiliary_loss_mlp": 0.01058715, + "balance_loss_clip": 1.05183339, + "balance_loss_mlp": 1.02694559, + "epoch": 0.021414891764842434, + "flos": 15806891410560.0, + "grad_norm": 2.886668077560544, + "language_loss": 0.83419204, + "learning_rate": 3.8056550949953317e-06, + "loss": 0.8567431, + "num_input_tokens_seen": 20361700, + "router_z_loss_clip": 1.44628906, + "router_z_loss_mlp": 0.31762695, + "step": 738, + "time_per_iteration": 2.3971238136291504 + }, + { + "auxiliary_loss_clip": 0.0103667, + "auxiliary_loss_mlp": 0.01004262, + "balance_loss_clip": 1.00306511, + "balance_loss_mlp": 1.00016105, + "epoch": 0.021443909233358482, + "flos": 68505908342400.0, + "grad_norm": 0.7319104598712963, + "language_loss": 0.52907419, + "learning_rate": 3.806435420177029e-06, + "loss": 0.54948354, + "num_input_tokens_seen": 20422835, + "router_z_loss_clip": 0.3359375, + "router_z_loss_mlp": 0.04101562, + "step": 739, + "time_per_iteration": 3.0963358879089355 + }, + { + "auxiliary_loss_clip": 0.01036908, + "auxiliary_loss_mlp": 0.01004094, + "balance_loss_clip": 1.00400281, + "balance_loss_mlp": 1.00001681, + "epoch": 0.021472926701874527, + "flos": 74774075264640.0, + "grad_norm": 0.6598902600609827, + "language_loss": 0.57416761, + "learning_rate": 3.8072146901519385e-06, + "loss": 0.59457755, + "num_input_tokens_seen": 20488955, + "router_z_loss_clip": 0.328125, + "router_z_loss_mlp": 0.04077148, + "step": 740, + "time_per_iteration": 3.0725276470184326 + }, + { + "auxiliary_loss_clip": 0.01204274, + "auxiliary_loss_mlp": 0.01078637, + "balance_loss_clip": 1.05265737, + "balance_loss_mlp": 1.03840399, + "epoch": 0.021501944170390575, + "flos": 36933569038080.0, + "grad_norm": 4.0001575519482095, + "language_loss": 1.08313882, + "learning_rate": 3.8079929077700457e-06, + "loss": 1.105968, + "num_input_tokens_seen": 20508400, + "router_z_loss_clip": 1.51464844, + "router_z_loss_mlp": 0.40234375, + "step": 741, + "time_per_iteration": 2.6307785511016846 + }, + { + "auxiliary_loss_clip": 0.0119529, + "auxiliary_loss_mlp": 0.01070711, + "balance_loss_clip": 1.05562353, + "balance_loss_mlp": 1.03501987, + "epoch": 0.021530961638906623, + "flos": 42594351304320.0, + "grad_norm": 2.6919196472213995, + "language_loss": 1.13980794, + "learning_rate": 3.8087700758698065e-06, + "loss": 1.16246784, + "num_input_tokens_seen": 20531245, + "router_z_loss_clip": 1.39746094, + "router_z_loss_mlp": 0.35693359, + "step": 742, + "time_per_iteration": 2.610584020614624 + }, + { + "auxiliary_loss_clip": 0.01039943, + "auxiliary_loss_mlp": 0.01008144, + "balance_loss_clip": 1.00803113, + "balance_loss_mlp": 1.00442457, + "epoch": 0.021559979107422668, + "flos": 65502470655360.0, + "grad_norm": 0.7938371899334024, + "language_loss": 0.46643421, + "learning_rate": 3.809546197278207e-06, + "loss": 0.48691508, + "num_input_tokens_seen": 20586370, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.03710938, + "step": 743, + "time_per_iteration": 2.9522950649261475 + }, + { + "auxiliary_loss_clip": 0.01200643, + "auxiliary_loss_mlp": 0.01067271, + "balance_loss_clip": 1.05172515, + "balance_loss_mlp": 1.02805114, + "epoch": 0.021588996575938716, + "flos": 31749909678720.0, + "grad_norm": 2.2273768220668675, + "language_loss": 0.96860468, + "learning_rate": 3.810321274810827e-06, + "loss": 0.99128377, + "num_input_tokens_seen": 20604280, + "router_z_loss_clip": 1.48828125, + "router_z_loss_mlp": 0.39196777, + "step": 744, + "time_per_iteration": 2.5096099376678467 + }, + { + "auxiliary_loss_clip": 0.01200132, + "auxiliary_loss_mlp": 0.01062937, + "balance_loss_clip": 1.05124974, + "balance_loss_mlp": 1.02464724, + "epoch": 0.02161801404445476, + "flos": 28504418469120.0, + "grad_norm": 3.598469686546852, + "language_loss": 0.99560124, + "learning_rate": 3.8110953112719017e-06, + "loss": 1.01823187, + "num_input_tokens_seen": 20618850, + "router_z_loss_clip": 1.48828125, + "router_z_loss_mlp": 0.3828125, + "step": 745, + "time_per_iteration": 2.5306150913238525 + }, + { + "auxiliary_loss_clip": 0.01184673, + "auxiliary_loss_mlp": 0.01061941, + "balance_loss_clip": 1.04803467, + "balance_loss_mlp": 1.02958739, + "epoch": 0.02164703151297081, + "flos": 32442830899200.0, + "grad_norm": 3.8516761649622806, + "language_loss": 1.10940003, + "learning_rate": 3.81186830945438e-06, + "loss": 1.13186622, + "num_input_tokens_seen": 20632240, + "router_z_loss_clip": 1.3671875, + "router_z_loss_mlp": 0.3236084, + "step": 746, + "time_per_iteration": 2.553525686264038 + }, + { + "auxiliary_loss_clip": 0.01186684, + "auxiliary_loss_mlp": 0.01068898, + "balance_loss_clip": 1.04884613, + "balance_loss_mlp": 1.03296876, + "epoch": 0.021676048981486853, + "flos": 38027537579520.0, + "grad_norm": 2.598660003624722, + "language_loss": 1.08203816, + "learning_rate": 3.812640272139988e-06, + "loss": 1.10459399, + "num_input_tokens_seen": 20648830, + "router_z_loss_clip": 1.37792969, + "router_z_loss_mlp": 0.35913086, + "step": 747, + "time_per_iteration": 2.565298080444336 + }, + { + "auxiliary_loss_clip": 0.01201392, + "auxiliary_loss_mlp": 0.01070658, + "balance_loss_clip": 1.05290842, + "balance_loss_mlp": 1.03179598, + "epoch": 0.0217050664500029, + "flos": 35256690570240.0, + "grad_norm": 2.435237765184403, + "language_loss": 0.73144972, + "learning_rate": 3.813411202099287e-06, + "loss": 0.75417018, + "num_input_tokens_seen": 20668395, + "router_z_loss_clip": 1.48339844, + "router_z_loss_mlp": 0.38891602, + "step": 748, + "time_per_iteration": 2.5794057846069336 + }, + { + "auxiliary_loss_clip": 0.01195954, + "auxiliary_loss_mlp": 0.01060943, + "balance_loss_clip": 1.04869616, + "balance_loss_mlp": 1.02681303, + "epoch": 0.02173408391851895, + "flos": 29422598981760.0, + "grad_norm": 2.5349880770058157, + "language_loss": 1.10549474, + "learning_rate": 3.8141811020917338e-06, + "loss": 1.1280638, + "num_input_tokens_seen": 20688865, + "router_z_loss_clip": 1.47070312, + "router_z_loss_mlp": 0.34143066, + "step": 749, + "time_per_iteration": 2.492826461791992 + }, + { + "auxiliary_loss_clip": 0.01038886, + "auxiliary_loss_mlp": 0.01008665, + "balance_loss_clip": 1.00830424, + "balance_loss_mlp": 1.00499368, + "epoch": 0.021763101387034994, + "flos": 62626709410560.0, + "grad_norm": 0.774465380850936, + "language_loss": 0.56752527, + "learning_rate": 3.81494997486574e-06, + "loss": 0.58800077, + "num_input_tokens_seen": 20753735, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.03662109, + "step": 750, + "time_per_iteration": 3.0114922523498535 + }, + { + "auxiliary_loss_clip": 0.01196462, + "auxiliary_loss_mlp": 0.010585, + "balance_loss_clip": 1.05197811, + "balance_loss_mlp": 1.01742005, + "epoch": 0.021792118855551042, + "flos": 22046429151360.0, + "grad_norm": 2.9854627764304147, + "language_loss": 0.82846856, + "learning_rate": 3.815717823158732e-06, + "loss": 0.85101819, + "num_input_tokens_seen": 20770205, + "router_z_loss_clip": 1.4453125, + "router_z_loss_mlp": 0.41113281, + "step": 751, + "time_per_iteration": 2.5019354820251465 + }, + { + "auxiliary_loss_clip": 0.01189404, + "auxiliary_loss_mlp": 0.01074141, + "balance_loss_clip": 1.0540688, + "balance_loss_mlp": 1.04294443, + "epoch": 0.021821136324067087, + "flos": 11065441731840.0, + "grad_norm": 3.09842860421235, + "language_loss": 0.86538434, + "learning_rate": 3.816484649697207e-06, + "loss": 0.8880198, + "num_input_tokens_seen": 20780935, + "router_z_loss_clip": 1.35449219, + "router_z_loss_mlp": 0.31201172, + "step": 752, + "time_per_iteration": 2.3975436687469482 + }, + { + "auxiliary_loss_clip": 0.01032426, + "auxiliary_loss_mlp": 0.01004054, + "balance_loss_clip": 1.00255704, + "balance_loss_mlp": 1.00023985, + "epoch": 0.021850153792583135, + "flos": 63388409742720.0, + "grad_norm": 0.7410718212620574, + "language_loss": 0.53811717, + "learning_rate": 3.817250457196791e-06, + "loss": 0.55848193, + "num_input_tokens_seen": 20840120, + "router_z_loss_clip": 0.29882812, + "router_z_loss_mlp": 0.03808594, + "step": 753, + "time_per_iteration": 2.902249574661255 + }, + { + "auxiliary_loss_clip": 0.01197704, + "auxiliary_loss_mlp": 0.01065781, + "balance_loss_clip": 1.04874134, + "balance_loss_mlp": 1.03196132, + "epoch": 0.021879171261099183, + "flos": 42733759829760.0, + "grad_norm": 4.349920090855965, + "language_loss": 0.9498775, + "learning_rate": 3.818015248362302e-06, + "loss": 0.97251236, + "num_input_tokens_seen": 20862485, + "router_z_loss_clip": 1.49023438, + "router_z_loss_mlp": 0.33837891, + "step": 754, + "time_per_iteration": 2.619843006134033 + }, + { + "auxiliary_loss_clip": 0.01202198, + "auxiliary_loss_mlp": 0.01068987, + "balance_loss_clip": 1.05335402, + "balance_loss_mlp": 1.03312898, + "epoch": 0.021908188729615228, + "flos": 56055603553920.0, + "grad_norm": 2.2291819032531053, + "language_loss": 0.9799062, + "learning_rate": 3.818779025887801e-06, + "loss": 1.00261807, + "num_input_tokens_seen": 20891275, + "router_z_loss_clip": 1.48925781, + "router_z_loss_mlp": 0.3581543, + "step": 755, + "time_per_iteration": 2.7035868167877197 + }, + { + "auxiliary_loss_clip": 0.01202749, + "auxiliary_loss_mlp": 0.01068413, + "balance_loss_clip": 1.05256772, + "balance_loss_mlp": 1.02699971, + "epoch": 0.021937206198131276, + "flos": 25550905893120.0, + "grad_norm": 2.7024462334536303, + "language_loss": 1.05092013, + "learning_rate": 3.81954179245665e-06, + "loss": 1.07363176, + "num_input_tokens_seen": 20906840, + "router_z_loss_clip": 1.5, + "router_z_loss_mlp": 0.4140625, + "step": 756, + "time_per_iteration": 2.5760440826416016 + }, + { + "auxiliary_loss_clip": 0.01189961, + "auxiliary_loss_mlp": 0.01070971, + "balance_loss_clip": 1.04979539, + "balance_loss_mlp": 1.03614962, + "epoch": 0.02196622366664732, + "flos": 29634277754880.0, + "grad_norm": 3.2784286702709085, + "language_loss": 0.77210832, + "learning_rate": 3.820303550741571e-06, + "loss": 0.79471761, + "num_input_tokens_seen": 20926260, + "router_z_loss_clip": 1.40039062, + "router_z_loss_mlp": 0.34814453, + "step": 757, + "time_per_iteration": 2.5456457138061523 + }, + { + "auxiliary_loss_clip": 0.01195752, + "auxiliary_loss_mlp": 0.01071172, + "balance_loss_clip": 1.05276465, + "balance_loss_mlp": 1.03753114, + "epoch": 0.02199524113516337, + "flos": 17379239667840.0, + "grad_norm": 2.699184792207963, + "language_loss": 1.0071125, + "learning_rate": 3.8210643034047025e-06, + "loss": 1.0297817, + "num_input_tokens_seen": 20940335, + "router_z_loss_clip": 1.43066406, + "router_z_loss_mlp": 0.33605957, + "step": 758, + "time_per_iteration": 2.547914743423462 + }, + { + "auxiliary_loss_clip": 0.01038122, + "auxiliary_loss_mlp": 0.01010105, + "balance_loss_clip": 1.00682116, + "balance_loss_mlp": 1.00664759, + "epoch": 0.022024258603679413, + "flos": 74767372283520.0, + "grad_norm": 0.6767292106741137, + "language_loss": 0.52322739, + "learning_rate": 3.8218240530976505e-06, + "loss": 0.54370964, + "num_input_tokens_seen": 21005590, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.03466797, + "step": 759, + "time_per_iteration": 3.1837759017944336 + }, + { + "auxiliary_loss_clip": 0.01188881, + "auxiliary_loss_mlp": 0.01059774, + "balance_loss_clip": 1.05084717, + "balance_loss_mlp": 1.03005505, + "epoch": 0.02205327607219546, + "flos": 29707979368320.0, + "grad_norm": 2.9718709298689814, + "language_loss": 0.72492063, + "learning_rate": 3.82258280246155e-06, + "loss": 0.74740714, + "num_input_tokens_seen": 21021005, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.29724121, + "step": 760, + "time_per_iteration": 2.4069430828094482 + }, + { + "auxiliary_loss_clip": 0.01036423, + "auxiliary_loss_mlp": 0.01003866, + "balance_loss_clip": 1.0055809, + "balance_loss_mlp": 1.00059986, + "epoch": 0.02208229354071151, + "flos": 66419883118080.0, + "grad_norm": 0.6687546612621593, + "language_loss": 0.51112098, + "learning_rate": 3.823340554127116e-06, + "loss": 0.53152388, + "num_input_tokens_seen": 21082605, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.03271484, + "step": 761, + "time_per_iteration": 2.963020086288452 + }, + { + "auxiliary_loss_clip": 0.0119099, + "auxiliary_loss_mlp": 0.01075285, + "balance_loss_clip": 1.05012989, + "balance_loss_mlp": 1.0401659, + "epoch": 0.022111311009227554, + "flos": 26133222326400.0, + "grad_norm": 3.0895371991700795, + "language_loss": 0.81760585, + "learning_rate": 3.824097310714699e-06, + "loss": 0.84026855, + "num_input_tokens_seen": 21099205, + "router_z_loss_clip": 1.40820312, + "router_z_loss_mlp": 0.35107422, + "step": 762, + "time_per_iteration": 2.4392800331115723 + }, + { + "auxiliary_loss_clip": 0.01176422, + "auxiliary_loss_mlp": 0.01061204, + "balance_loss_clip": 1.04604912, + "balance_loss_mlp": 1.031986, + "epoch": 0.022140328477743602, + "flos": 14530780972800.0, + "grad_norm": 2.7100240274344642, + "language_loss": 0.90219307, + "learning_rate": 3.824853074834342e-06, + "loss": 0.92456931, + "num_input_tokens_seen": 21112380, + "router_z_loss_clip": 1.3046875, + "router_z_loss_mlp": 0.29211426, + "step": 763, + "time_per_iteration": 2.3555538654327393 + }, + { + "auxiliary_loss_clip": 0.01179298, + "auxiliary_loss_mlp": 0.01068366, + "balance_loss_clip": 1.04674029, + "balance_loss_mlp": 1.03629887, + "epoch": 0.022169345946259647, + "flos": 36165584661120.0, + "grad_norm": 3.3792710018817314, + "language_loss": 0.87663376, + "learning_rate": 3.82560784908583e-06, + "loss": 0.89911044, + "num_input_tokens_seen": 21130490, + "router_z_loss_clip": 1.32519531, + "router_z_loss_mlp": 0.32080078, + "step": 764, + "time_per_iteration": 4.921048402786255 + }, + { + "auxiliary_loss_clip": 0.01182536, + "auxiliary_loss_mlp": 0.01064307, + "balance_loss_clip": 1.0493083, + "balance_loss_mlp": 1.03145313, + "epoch": 0.022198363414775695, + "flos": 27700124411520.0, + "grad_norm": 3.2491116434312017, + "language_loss": 0.88955116, + "learning_rate": 3.826361636058748e-06, + "loss": 0.91201961, + "num_input_tokens_seen": 21148605, + "router_z_loss_clip": 1.33203125, + "router_z_loss_mlp": 0.32861328, + "step": 765, + "time_per_iteration": 2.5272254943847656 + }, + { + "auxiliary_loss_clip": 0.01032057, + "auxiliary_loss_mlp": 0.01011121, + "balance_loss_clip": 1.00181055, + "balance_loss_mlp": 1.00804591, + "epoch": 0.022227380883291743, + "flos": 67331116270080.0, + "grad_norm": 0.6883847962879841, + "language_loss": 0.51874995, + "learning_rate": 3.827114438332532e-06, + "loss": 0.53918171, + "num_input_tokens_seen": 21207490, + "router_z_loss_clip": 0.30273438, + "router_z_loss_mlp": 0.03063965, + "step": 766, + "time_per_iteration": 5.151331186294556 + }, + { + "auxiliary_loss_clip": 0.01193837, + "auxiliary_loss_mlp": 0.01066, + "balance_loss_clip": 1.0522244, + "balance_loss_mlp": 1.03123879, + "epoch": 0.022256398351807788, + "flos": 28687747351680.0, + "grad_norm": 2.914012687747211, + "language_loss": 0.97543293, + "learning_rate": 3.827866258476522e-06, + "loss": 0.99803126, + "num_input_tokens_seen": 21222515, + "router_z_loss_clip": 1.41503906, + "router_z_loss_mlp": 0.34765625, + "step": 767, + "time_per_iteration": 2.47771954536438 + }, + { + "auxiliary_loss_clip": 0.01032664, + "auxiliary_loss_mlp": 0.01005967, + "balance_loss_clip": 1.00233006, + "balance_loss_mlp": 1.00277209, + "epoch": 0.022285415820323836, + "flos": 61309192233600.0, + "grad_norm": 0.6980269832011718, + "language_loss": 0.5022018, + "learning_rate": 3.828617099050014e-06, + "loss": 0.52258801, + "num_input_tokens_seen": 21272320, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.03198242, + "step": 768, + "time_per_iteration": 5.319704294204712 + }, + { + "auxiliary_loss_clip": 0.01186364, + "auxiliary_loss_mlp": 0.01066302, + "balance_loss_clip": 1.05146599, + "balance_loss_mlp": 1.03287578, + "epoch": 0.02231443328883988, + "flos": 24016333593600.0, + "grad_norm": 3.256553585479543, + "language_loss": 0.8451761, + "learning_rate": 3.8293669626023145e-06, + "loss": 0.86770272, + "num_input_tokens_seen": 21286465, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.33447266, + "step": 769, + "time_per_iteration": 2.435331344604492 + }, + { + "auxiliary_loss_clip": 0.01189494, + "auxiliary_loss_mlp": 0.01056344, + "balance_loss_clip": 1.04850316, + "balance_loss_mlp": 1.02253664, + "epoch": 0.02234345075735593, + "flos": 20077118202240.0, + "grad_norm": 2.514809354643886, + "language_loss": 1.03360462, + "learning_rate": 3.830115851672791e-06, + "loss": 1.05606306, + "num_input_tokens_seen": 21302395, + "router_z_loss_clip": 1.40917969, + "router_z_loss_mlp": 0.33837891, + "step": 770, + "time_per_iteration": 2.430353879928589 + }, + { + "auxiliary_loss_clip": 0.01185389, + "auxiliary_loss_mlp": 0.01058982, + "balance_loss_clip": 1.04740047, + "balance_loss_mlp": 1.02747536, + "epoch": 0.022372468225871973, + "flos": 39489595251840.0, + "grad_norm": 3.8892721993954886, + "language_loss": 1.02896881, + "learning_rate": 3.830863768790924e-06, + "loss": 1.05141246, + "num_input_tokens_seen": 21319210, + "router_z_loss_clip": 1.38085938, + "router_z_loss_mlp": 0.31481934, + "step": 771, + "time_per_iteration": 4.985408782958984 + }, + { + "auxiliary_loss_clip": 0.0103559, + "auxiliary_loss_mlp": 0.01004316, + "balance_loss_clip": 1.00409651, + "balance_loss_mlp": 1.00109756, + "epoch": 0.02240148569438802, + "flos": 73238944383360.0, + "grad_norm": 0.6909772450857236, + "language_loss": 0.51204169, + "learning_rate": 3.831610716476358e-06, + "loss": 0.53244072, + "num_input_tokens_seen": 21379720, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.03222656, + "step": 772, + "time_per_iteration": 3.0034706592559814 + }, + { + "auxiliary_loss_clip": 0.01177654, + "auxiliary_loss_mlp": 0.01055995, + "balance_loss_clip": 1.04667473, + "balance_loss_mlp": 1.02490544, + "epoch": 0.02243050316290407, + "flos": 15257044408320.0, + "grad_norm": 3.8176104201016057, + "language_loss": 1.0385046, + "learning_rate": 3.83235669723895e-06, + "loss": 1.06084108, + "num_input_tokens_seen": 21390245, + "router_z_loss_clip": 1.31152344, + "router_z_loss_mlp": 0.31079102, + "step": 773, + "time_per_iteration": 2.402055025100708 + }, + { + "auxiliary_loss_clip": 0.0103465, + "auxiliary_loss_mlp": 0.01003025, + "balance_loss_clip": 1.00321245, + "balance_loss_mlp": 1.00004506, + "epoch": 0.022459520631420114, + "flos": 65260207664640.0, + "grad_norm": 0.7339910136600372, + "language_loss": 0.52952051, + "learning_rate": 3.833101713578828e-06, + "loss": 0.54989731, + "num_input_tokens_seen": 21456720, + "router_z_loss_clip": 0.31445312, + "router_z_loss_mlp": 0.02978516, + "step": 774, + "time_per_iteration": 3.0698208808898926 + }, + { + "auxiliary_loss_clip": 0.01204728, + "auxiliary_loss_mlp": 0.01071238, + "balance_loss_clip": 1.05422914, + "balance_loss_mlp": 1.03559446, + "epoch": 0.022488538099936162, + "flos": 27007517393280.0, + "grad_norm": 2.3229502957948918, + "language_loss": 0.94207942, + "learning_rate": 3.83384576798643e-06, + "loss": 0.9648391, + "num_input_tokens_seen": 21471700, + "router_z_loss_clip": 1.50390625, + "router_z_loss_mlp": 0.35620117, + "step": 775, + "time_per_iteration": 2.7339303493499756 + }, + { + "auxiliary_loss_clip": 0.01188144, + "auxiliary_loss_mlp": 0.01061957, + "balance_loss_clip": 1.04726052, + "balance_loss_mlp": 1.03046179, + "epoch": 0.022517555568452207, + "flos": 26899216755840.0, + "grad_norm": 2.8163413766020424, + "language_loss": 1.0322485, + "learning_rate": 3.834588862942565e-06, + "loss": 1.05474949, + "num_input_tokens_seen": 21491595, + "router_z_loss_clip": 1.40917969, + "router_z_loss_mlp": 0.31481934, + "step": 776, + "time_per_iteration": 2.4962668418884277 + }, + { + "auxiliary_loss_clip": 0.01199099, + "auxiliary_loss_mlp": 0.01083693, + "balance_loss_clip": 1.05639756, + "balance_loss_mlp": 1.04769242, + "epoch": 0.022546573036968255, + "flos": 31392154310400.0, + "grad_norm": 2.3702086111313583, + "language_loss": 1.04297364, + "learning_rate": 3.835331000918451e-06, + "loss": 1.06580162, + "num_input_tokens_seen": 21510460, + "router_z_loss_clip": 1.42675781, + "router_z_loss_mlp": 0.36010742, + "step": 777, + "time_per_iteration": 2.5542140007019043 + }, + { + "auxiliary_loss_clip": 0.01032517, + "auxiliary_loss_mlp": 0.0100311, + "balance_loss_clip": 1.00184608, + "balance_loss_mlp": 0.99998695, + "epoch": 0.022575590505484303, + "flos": 74762100668160.0, + "grad_norm": 0.7768441228451879, + "language_loss": 0.55491805, + "learning_rate": 3.836072184375777e-06, + "loss": 0.57527435, + "num_input_tokens_seen": 21569265, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.03125, + "step": 778, + "time_per_iteration": 3.101407766342163 + }, + { + "auxiliary_loss_clip": 0.01181959, + "auxiliary_loss_mlp": 0.01068077, + "balance_loss_clip": 1.0475986, + "balance_loss_mlp": 1.03883481, + "epoch": 0.022604607974000348, + "flos": 15702465934080.0, + "grad_norm": 5.323591744561662, + "language_loss": 1.04124427, + "learning_rate": 3.8368124157667445e-06, + "loss": 1.06374466, + "num_input_tokens_seen": 21581555, + "router_z_loss_clip": 1.34472656, + "router_z_loss_mlp": 0.29223633, + "step": 779, + "time_per_iteration": 2.4908816814422607 + }, + { + "auxiliary_loss_clip": 0.01186797, + "auxiliary_loss_mlp": 0.01060139, + "balance_loss_clip": 1.04988861, + "balance_loss_mlp": 1.0299077, + "epoch": 0.022633625442516396, + "flos": 22886090282880.0, + "grad_norm": 3.2312958237620983, + "language_loss": 1.05457163, + "learning_rate": 3.8375516975341135e-06, + "loss": 1.07704091, + "num_input_tokens_seen": 21598715, + "router_z_loss_clip": 1.36914062, + "router_z_loss_mlp": 0.30224609, + "step": 780, + "time_per_iteration": 2.9099268913269043 + }, + { + "auxiliary_loss_clip": 0.01032708, + "auxiliary_loss_mlp": 0.01003331, + "balance_loss_clip": 1.00177765, + "balance_loss_mlp": 1.00008833, + "epoch": 0.02266264291103244, + "flos": 60762033406080.0, + "grad_norm": 0.6769298142824483, + "language_loss": 0.57448745, + "learning_rate": 3.838290032111259e-06, + "loss": 0.59484792, + "num_input_tokens_seen": 21662190, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.0324707, + "step": 781, + "time_per_iteration": 3.300028085708618 + }, + { + "auxiliary_loss_clip": 0.01192124, + "auxiliary_loss_mlp": 0.01066708, + "balance_loss_clip": 1.05067682, + "balance_loss_mlp": 1.03068256, + "epoch": 0.02269166037954849, + "flos": 39960713404800.0, + "grad_norm": 2.9842259012698658, + "language_loss": 1.04080045, + "learning_rate": 3.8390274219222125e-06, + "loss": 1.06338882, + "num_input_tokens_seen": 21679115, + "router_z_loss_clip": 1.4140625, + "router_z_loss_mlp": 0.36010742, + "step": 782, + "time_per_iteration": 2.7420496940612793 + }, + { + "auxiliary_loss_clip": 0.01188126, + "auxiliary_loss_mlp": 0.01061547, + "balance_loss_clip": 1.04679465, + "balance_loss_mlp": 1.02956343, + "epoch": 0.022720677848064533, + "flos": 39085266263040.0, + "grad_norm": 3.1401905228878397, + "language_loss": 0.82664639, + "learning_rate": 3.839763869381713e-06, + "loss": 0.84914303, + "num_input_tokens_seen": 21697935, + "router_z_loss_clip": 1.41210938, + "router_z_loss_mlp": 0.31994629, + "step": 783, + "time_per_iteration": 2.6164252758026123 + }, + { + "auxiliary_loss_clip": 0.01182979, + "auxiliary_loss_mlp": 0.01065988, + "balance_loss_clip": 1.04437613, + "balance_loss_mlp": 1.03178644, + "epoch": 0.02274969531658058, + "flos": 74743597647360.0, + "grad_norm": 3.1569650158310334, + "language_loss": 0.86894596, + "learning_rate": 3.840499376895254e-06, + "loss": 0.89143562, + "num_input_tokens_seen": 21721375, + "router_z_loss_clip": 1.38769531, + "router_z_loss_mlp": 0.34204102, + "step": 784, + "time_per_iteration": 2.9531869888305664 + }, + { + "auxiliary_loss_clip": 0.011844, + "auxiliary_loss_mlp": 0.01059021, + "balance_loss_clip": 1.04964101, + "balance_loss_mlp": 1.02924275, + "epoch": 0.02277871278509663, + "flos": 14640512976000.0, + "grad_norm": 2.716066893338227, + "language_loss": 0.87695396, + "learning_rate": 3.841233946859129e-06, + "loss": 0.89938819, + "num_input_tokens_seen": 21733045, + "router_z_loss_clip": 1.34570312, + "router_z_loss_mlp": 0.29772949, + "step": 785, + "time_per_iteration": 2.4362189769744873 + }, + { + "auxiliary_loss_clip": 0.01193883, + "auxiliary_loss_mlp": 0.0106938, + "balance_loss_clip": 1.04923463, + "balance_loss_mlp": 1.03442764, + "epoch": 0.022807730253612674, + "flos": 30517056282240.0, + "grad_norm": 12.69232213506155, + "language_loss": 0.89434922, + "learning_rate": 3.8419675816604806e-06, + "loss": 0.91698182, + "num_input_tokens_seen": 21748950, + "router_z_loss_clip": 1.44726562, + "router_z_loss_mlp": 0.34960938, + "step": 786, + "time_per_iteration": 2.5113868713378906 + }, + { + "auxiliary_loss_clip": 0.01032489, + "auxiliary_loss_mlp": 0.01003708, + "balance_loss_clip": 1.00184131, + "balance_loss_mlp": 1.00060844, + "epoch": 0.022836747722128722, + "flos": 56299365866880.0, + "grad_norm": 0.751040438083192, + "language_loss": 0.4964751, + "learning_rate": 3.842700283677345e-06, + "loss": 0.51683712, + "num_input_tokens_seen": 21803175, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.03100586, + "step": 787, + "time_per_iteration": 2.828655242919922 + }, + { + "auxiliary_loss_clip": 0.01186653, + "auxiliary_loss_mlp": 0.01069705, + "balance_loss_clip": 1.04732549, + "balance_loss_mlp": 1.03499115, + "epoch": 0.022865765190644767, + "flos": 26461301172480.0, + "grad_norm": 2.469548746610676, + "language_loss": 0.90641785, + "learning_rate": 3.8434320552787e-06, + "loss": 0.92898142, + "num_input_tokens_seen": 21818255, + "router_z_loss_clip": 1.39257812, + "router_z_loss_mlp": 0.34692383, + "step": 788, + "time_per_iteration": 2.3784825801849365 + }, + { + "auxiliary_loss_clip": 0.01180333, + "auxiliary_loss_mlp": 0.01071331, + "balance_loss_clip": 1.04833937, + "balance_loss_mlp": 1.03953815, + "epoch": 0.022894782659160815, + "flos": 47366804136960.0, + "grad_norm": 5.655871365338351, + "language_loss": 0.90155143, + "learning_rate": 3.844162898824509e-06, + "loss": 0.92406809, + "num_input_tokens_seen": 21835000, + "router_z_loss_clip": 1.32128906, + "router_z_loss_mlp": 0.31762695, + "step": 789, + "time_per_iteration": 2.622175455093384 + }, + { + "auxiliary_loss_clip": 0.01183636, + "auxiliary_loss_mlp": 0.01076272, + "balance_loss_clip": 1.0482378, + "balance_loss_mlp": 1.04422855, + "epoch": 0.02292380012767686, + "flos": 43353642752640.0, + "grad_norm": 3.4207687421347766, + "language_loss": 0.82700801, + "learning_rate": 3.844892816665769e-06, + "loss": 0.84960705, + "num_input_tokens_seen": 21851945, + "router_z_loss_clip": 1.35449219, + "router_z_loss_mlp": 0.32067871, + "step": 790, + "time_per_iteration": 2.501284599304199 + }, + { + "auxiliary_loss_clip": 0.01189559, + "auxiliary_loss_mlp": 0.01066191, + "balance_loss_clip": 1.04958439, + "balance_loss_mlp": 1.03085709, + "epoch": 0.022952817596192908, + "flos": 22812249024000.0, + "grad_norm": 2.9305134056554825, + "language_loss": 0.91609114, + "learning_rate": 3.845621811144555e-06, + "loss": 0.93864858, + "num_input_tokens_seen": 21865095, + "router_z_loss_clip": 1.3984375, + "router_z_loss_mlp": 0.35351562, + "step": 791, + "time_per_iteration": 2.363826036453247 + }, + { + "auxiliary_loss_clip": 0.01172854, + "auxiliary_loss_mlp": 0.01059772, + "balance_loss_clip": 1.04414845, + "balance_loss_mlp": 1.03039932, + "epoch": 0.022981835064708956, + "flos": 31130934451200.0, + "grad_norm": 2.635018962355554, + "language_loss": 0.76606143, + "learning_rate": 3.846349884594063e-06, + "loss": 0.78838772, + "num_input_tokens_seen": 21880565, + "router_z_loss_clip": 1.28808594, + "router_z_loss_mlp": 0.29370117, + "step": 792, + "time_per_iteration": 2.5105605125427246 + }, + { + "auxiliary_loss_clip": 0.01191795, + "auxiliary_loss_mlp": 0.01069973, + "balance_loss_clip": 1.04823053, + "balance_loss_mlp": 1.03803658, + "epoch": 0.023010852533225, + "flos": 17743383815040.0, + "grad_norm": 2.9811388728975605, + "language_loss": 0.88579643, + "learning_rate": 3.847077039338659e-06, + "loss": 0.90841413, + "num_input_tokens_seen": 21896470, + "router_z_loss_clip": 1.43457031, + "router_z_loss_mlp": 0.31945801, + "step": 793, + "time_per_iteration": 2.3598790168762207 + }, + { + "auxiliary_loss_clip": 0.0118083, + "auxiliary_loss_mlp": 0.01066544, + "balance_loss_clip": 1.04609537, + "balance_loss_mlp": 1.03717053, + "epoch": 0.02303987000174105, + "flos": 12268164758400.0, + "grad_norm": 5.399855834813133, + "language_loss": 1.29043615, + "learning_rate": 3.847803277693921e-06, + "loss": 1.31290984, + "num_input_tokens_seen": 21907555, + "router_z_loss_clip": 1.34863281, + "router_z_loss_mlp": 0.29370117, + "step": 794, + "time_per_iteration": 2.430034875869751 + }, + { + "auxiliary_loss_clip": 0.01192503, + "auxiliary_loss_mlp": 0.01069465, + "balance_loss_clip": 1.05158401, + "balance_loss_mlp": 1.0369451, + "epoch": 0.023068887470257093, + "flos": 25626632365440.0, + "grad_norm": 2.1337238982229647, + "language_loss": 0.75455886, + "learning_rate": 3.848528601966682e-06, + "loss": 0.77717853, + "num_input_tokens_seen": 21930305, + "router_z_loss_clip": 1.41015625, + "router_z_loss_mlp": 0.32519531, + "step": 795, + "time_per_iteration": 2.5693838596343994 + }, + { + "auxiliary_loss_clip": 0.01193269, + "auxiliary_loss_mlp": 0.0105866, + "balance_loss_clip": 1.05217278, + "balance_loss_mlp": 1.02821422, + "epoch": 0.02309790493877314, + "flos": 25074725592960.0, + "grad_norm": 2.2586943838164295, + "language_loss": 0.93019938, + "learning_rate": 3.849253014455075e-06, + "loss": 0.95271868, + "num_input_tokens_seen": 21946535, + "router_z_loss_clip": 1.41113281, + "router_z_loss_mlp": 0.3046875, + "step": 796, + "time_per_iteration": 2.5320591926574707 + }, + { + "auxiliary_loss_clip": 0.0119008, + "auxiliary_loss_mlp": 0.01064547, + "balance_loss_clip": 1.04712868, + "balance_loss_mlp": 1.02903426, + "epoch": 0.02312692240728919, + "flos": 18903897141120.0, + "grad_norm": 3.965243936731278, + "language_loss": 1.17547941, + "learning_rate": 3.84997651744858e-06, + "loss": 1.1980257, + "num_input_tokens_seen": 21956120, + "router_z_loss_clip": 1.42871094, + "router_z_loss_mlp": 0.35498047, + "step": 797, + "time_per_iteration": 2.3667781352996826 + }, + { + "auxiliary_loss_clip": 0.01181162, + "auxiliary_loss_mlp": 0.01064743, + "balance_loss_clip": 1.05025876, + "balance_loss_mlp": 1.03086376, + "epoch": 0.023155939875805234, + "flos": 32883679036800.0, + "grad_norm": 2.673336169125238, + "language_loss": 0.83302808, + "learning_rate": 3.850699113228063e-06, + "loss": 0.85548717, + "num_input_tokens_seen": 21980765, + "router_z_loss_clip": 1.30859375, + "router_z_loss_mlp": 0.33825684, + "step": 798, + "time_per_iteration": 2.715712547302246 + }, + { + "auxiliary_loss_clip": 0.01204493, + "auxiliary_loss_mlp": 0.01064464, + "balance_loss_clip": 1.05260444, + "balance_loss_mlp": 1.02808106, + "epoch": 0.023184957344321282, + "flos": 36056585796480.0, + "grad_norm": 3.4572633194961684, + "language_loss": 0.95642364, + "learning_rate": 3.851420804065818e-06, + "loss": 0.97911322, + "num_input_tokens_seen": 21997965, + "router_z_loss_clip": 1.51855469, + "router_z_loss_mlp": 0.36352539, + "step": 799, + "time_per_iteration": 2.4236791133880615 + }, + { + "auxiliary_loss_clip": 0.01193895, + "auxiliary_loss_mlp": 0.010653, + "balance_loss_clip": 1.04757273, + "balance_loss_mlp": 1.03268456, + "epoch": 0.023213974812837327, + "flos": 26791300143360.0, + "grad_norm": 2.0915426878573413, + "language_loss": 0.98436093, + "learning_rate": 3.8521415922256166e-06, + "loss": 1.00695288, + "num_input_tokens_seen": 22017260, + "router_z_loss_clip": 1.46191406, + "router_z_loss_mlp": 0.32641602, + "step": 800, + "time_per_iteration": 2.5161261558532715 + }, + { + "auxiliary_loss_clip": 0.01193456, + "auxiliary_loss_mlp": 0.01078548, + "balance_loss_clip": 1.05162501, + "balance_loss_mlp": 1.04657662, + "epoch": 0.023242992281353375, + "flos": 34453478764800.0, + "grad_norm": 2.4258190065688843, + "language_loss": 0.9811489, + "learning_rate": 3.852861479962747e-06, + "loss": 1.00386894, + "num_input_tokens_seen": 22034900, + "router_z_loss_clip": 1.41796875, + "router_z_loss_mlp": 0.31970215, + "step": 801, + "time_per_iteration": 2.570054531097412 + }, + { + "auxiliary_loss_clip": 0.01178002, + "auxiliary_loss_mlp": 0.01053338, + "balance_loss_clip": 1.04526603, + "balance_loss_mlp": 1.02270174, + "epoch": 0.02327200974986942, + "flos": 12014346107520.0, + "grad_norm": 3.5125531859163717, + "language_loss": 0.91168714, + "learning_rate": 3.853580469524051e-06, + "loss": 0.93400049, + "num_input_tokens_seen": 22047165, + "router_z_loss_clip": 1.32714844, + "router_z_loss_mlp": 0.30615234, + "step": 802, + "time_per_iteration": 2.484914779663086 + }, + { + "auxiliary_loss_clip": 0.01179942, + "auxiliary_loss_mlp": 0.01061034, + "balance_loss_clip": 1.04544377, + "balance_loss_mlp": 1.0283587, + "epoch": 0.023301027218385468, + "flos": 16208252933760.0, + "grad_norm": 3.3719768524380633, + "language_loss": 0.83868951, + "learning_rate": 3.854298563147975e-06, + "loss": 0.86109924, + "num_input_tokens_seen": 22060420, + "router_z_loss_clip": 1.34570312, + "router_z_loss_mlp": 0.32653809, + "step": 803, + "time_per_iteration": 2.401972770690918 + }, + { + "auxiliary_loss_clip": 0.01039591, + "auxiliary_loss_mlp": 0.01002709, + "balance_loss_clip": 1.00549698, + "balance_loss_mlp": 0.9994669, + "epoch": 0.023330044686901516, + "flos": 74769990635520.0, + "grad_norm": 0.719912879895457, + "language_loss": 0.55013061, + "learning_rate": 3.855015763064606e-06, + "loss": 0.57055354, + "num_input_tokens_seen": 22125790, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.0324707, + "step": 804, + "time_per_iteration": 3.0779104232788086 + }, + { + "auxiliary_loss_clip": 0.01038508, + "auxiliary_loss_mlp": 0.01003447, + "balance_loss_clip": 1.00467432, + "balance_loss_mlp": 0.99998993, + "epoch": 0.02335906215541756, + "flos": 56779002391680.0, + "grad_norm": 0.8414891225245246, + "language_loss": 0.56202888, + "learning_rate": 3.855732071495717e-06, + "loss": 0.58244836, + "num_input_tokens_seen": 22182250, + "router_z_loss_clip": 0.33984375, + "router_z_loss_mlp": 0.03466797, + "step": 805, + "time_per_iteration": 2.886181354522705 + }, + { + "auxiliary_loss_clip": 0.01037082, + "auxiliary_loss_mlp": 0.0100339, + "balance_loss_clip": 1.00360453, + "balance_loss_mlp": 0.99981374, + "epoch": 0.02338807962393361, + "flos": 60794991596160.0, + "grad_norm": 0.6836527571967536, + "language_loss": 0.53712082, + "learning_rate": 3.856447490654803e-06, + "loss": 0.55752552, + "num_input_tokens_seen": 22243370, + "router_z_loss_clip": 0.33398438, + "router_z_loss_mlp": 0.03564453, + "step": 806, + "time_per_iteration": 2.9325265884399414 + }, + { + "auxiliary_loss_clip": 0.01180695, + "auxiliary_loss_mlp": 0.01070183, + "balance_loss_clip": 1.04654288, + "balance_loss_mlp": 1.03805614, + "epoch": 0.023417097092449653, + "flos": 36384385351680.0, + "grad_norm": 2.311919003171526, + "language_loss": 0.88154709, + "learning_rate": 3.85716202274713e-06, + "loss": 0.90405589, + "num_input_tokens_seen": 22263715, + "router_z_loss_clip": 1.34082031, + "router_z_loss_mlp": 0.3215332, + "step": 807, + "time_per_iteration": 2.6205289363861084 + }, + { + "auxiliary_loss_clip": 0.01188749, + "auxiliary_loss_mlp": 0.0106378, + "balance_loss_clip": 1.04892778, + "balance_loss_mlp": 1.03291726, + "epoch": 0.0234461145609657, + "flos": 30620015481600.0, + "grad_norm": 3.172941504075171, + "language_loss": 0.9232049, + "learning_rate": 3.857875669969765e-06, + "loss": 0.94573021, + "num_input_tokens_seen": 22278910, + "router_z_loss_clip": 1.3984375, + "router_z_loss_mlp": 0.30883789, + "step": 808, + "time_per_iteration": 2.464385509490967 + }, + { + "auxiliary_loss_clip": 0.01167688, + "auxiliary_loss_mlp": 0.01064814, + "balance_loss_clip": 1.04514933, + "balance_loss_mlp": 1.03381968, + "epoch": 0.02347513202948175, + "flos": 13435381065600.0, + "grad_norm": 3.0046030670518586, + "language_loss": 0.88061297, + "learning_rate": 3.858588434511628e-06, + "loss": 0.90293801, + "num_input_tokens_seen": 22290310, + "router_z_loss_clip": 1.22363281, + "router_z_loss_mlp": 0.30981445, + "step": 809, + "time_per_iteration": 2.4141578674316406 + }, + { + "auxiliary_loss_clip": 0.01196054, + "auxiliary_loss_mlp": 0.01084027, + "balance_loss_clip": 1.0495708, + "balance_loss_mlp": 1.04421067, + "epoch": 0.023504149497997794, + "flos": 52437903672960.0, + "grad_norm": 3.670651580351032, + "language_loss": 1.2844336, + "learning_rate": 3.859300318553524e-06, + "loss": 1.30723453, + "num_input_tokens_seen": 22311180, + "router_z_loss_clip": 1.46289062, + "router_z_loss_mlp": 0.39819336, + "step": 810, + "time_per_iteration": 2.6644091606140137 + }, + { + "auxiliary_loss_clip": 0.01191677, + "auxiliary_loss_mlp": 0.01064519, + "balance_loss_clip": 1.04972887, + "balance_loss_mlp": 1.03015089, + "epoch": 0.023533166966513842, + "flos": 11866558855680.0, + "grad_norm": 3.6097827338489794, + "language_loss": 0.89745629, + "learning_rate": 3.860011324268188e-06, + "loss": 0.92001826, + "num_input_tokens_seen": 22323300, + "router_z_loss_clip": 1.41894531, + "router_z_loss_mlp": 0.34399414, + "step": 811, + "time_per_iteration": 2.3740651607513428 + }, + { + "auxiliary_loss_clip": 0.01187714, + "auxiliary_loss_mlp": 0.01060639, + "balance_loss_clip": 1.04847383, + "balance_loss_mlp": 1.02865481, + "epoch": 0.023562184435029887, + "flos": 22850513740800.0, + "grad_norm": 2.2143192756780965, + "language_loss": 0.84205389, + "learning_rate": 3.860721453820318e-06, + "loss": 0.86453748, + "num_input_tokens_seen": 22340530, + "router_z_loss_clip": 1.39257812, + "router_z_loss_mlp": 0.31982422, + "step": 812, + "time_per_iteration": 2.418813943862915 + }, + { + "auxiliary_loss_clip": 0.01199699, + "auxiliary_loss_mlp": 0.01074726, + "balance_loss_clip": 1.04967546, + "balance_loss_mlp": 1.03777075, + "epoch": 0.023591201903545935, + "flos": 32817972124800.0, + "grad_norm": 2.106402331048749, + "language_loss": 0.87561244, + "learning_rate": 3.861430709366625e-06, + "loss": 0.89835668, + "num_input_tokens_seen": 22365065, + "router_z_loss_clip": 1.50097656, + "router_z_loss_mlp": 0.36938477, + "step": 813, + "time_per_iteration": 2.6510472297668457 + }, + { + "auxiliary_loss_clip": 0.01190395, + "auxiliary_loss_mlp": 0.01075535, + "balance_loss_clip": 1.05289555, + "balance_loss_mlp": 1.04387283, + "epoch": 0.02362021937206198, + "flos": 32736974037120.0, + "grad_norm": 1.994053772592937, + "language_loss": 0.86649323, + "learning_rate": 3.8621390930558644e-06, + "loss": 0.88915253, + "num_input_tokens_seen": 22381910, + "router_z_loss_clip": 1.37597656, + "router_z_loss_mlp": 0.31628418, + "step": 814, + "time_per_iteration": 2.4878129959106445 + }, + { + "auxiliary_loss_clip": 0.01036596, + "auxiliary_loss_mlp": 0.0101313, + "balance_loss_clip": 1.00488138, + "balance_loss_mlp": 1.00938642, + "epoch": 0.023649236840578028, + "flos": 59084910558720.0, + "grad_norm": 0.7232273302560878, + "language_loss": 0.56422663, + "learning_rate": 3.862846607028876e-06, + "loss": 0.58472395, + "num_input_tokens_seen": 22442230, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.03735352, + "step": 815, + "time_per_iteration": 2.9697465896606445 + }, + { + "auxiliary_loss_clip": 0.01185785, + "auxiliary_loss_mlp": 0.01064067, + "balance_loss_clip": 1.05365849, + "balance_loss_mlp": 1.03190434, + "epoch": 0.023678254309094076, + "flos": 21098223002880.0, + "grad_norm": 2.7471161035890415, + "language_loss": 1.00883174, + "learning_rate": 3.863553253418625e-06, + "loss": 1.03133023, + "num_input_tokens_seen": 22455850, + "router_z_loss_clip": 1.32226562, + "router_z_loss_mlp": 0.32177734, + "step": 816, + "time_per_iteration": 2.384800910949707 + }, + { + "auxiliary_loss_clip": 0.0119191, + "auxiliary_loss_mlp": 0.01066717, + "balance_loss_clip": 1.05107045, + "balance_loss_mlp": 1.03228891, + "epoch": 0.02370727177761012, + "flos": 34854037326720.0, + "grad_norm": 2.4505607039224304, + "language_loss": 0.83102584, + "learning_rate": 3.86425903435024e-06, + "loss": 0.85361207, + "num_input_tokens_seen": 22473240, + "router_z_loss_clip": 1.40917969, + "router_z_loss_mlp": 0.34448242, + "step": 817, + "time_per_iteration": 2.5493600368499756 + }, + { + "auxiliary_loss_clip": 0.01190155, + "auxiliary_loss_mlp": 0.01062643, + "balance_loss_clip": 1.04684675, + "balance_loss_mlp": 1.02862048, + "epoch": 0.02373628924612617, + "flos": 25658159189760.0, + "grad_norm": 2.455952793986326, + "language_loss": 0.75850815, + "learning_rate": 3.864963951941051e-06, + "loss": 0.78103608, + "num_input_tokens_seen": 22490925, + "router_z_loss_clip": 1.43457031, + "router_z_loss_mlp": 0.34033203, + "step": 818, + "time_per_iteration": 2.38460111618042 + }, + { + "auxiliary_loss_clip": 0.01183048, + "auxiliary_loss_mlp": 0.01056556, + "balance_loss_clip": 1.04779732, + "balance_loss_mlp": 1.02518034, + "epoch": 0.023765306714642213, + "flos": 11904579192960.0, + "grad_norm": 3.580555840791303, + "language_loss": 0.77675277, + "learning_rate": 3.8656680083006265e-06, + "loss": 0.79914886, + "num_input_tokens_seen": 22502800, + "router_z_loss_clip": 1.35058594, + "router_z_loss_mlp": 0.3137207, + "step": 819, + "time_per_iteration": 2.4003031253814697 + }, + { + "auxiliary_loss_clip": 0.01035218, + "auxiliary_loss_mlp": 0.01003066, + "balance_loss_clip": 1.00293827, + "balance_loss_mlp": 0.99975151, + "epoch": 0.02379432418315826, + "flos": 74772923189760.0, + "grad_norm": 0.6549510363138962, + "language_loss": 0.53170782, + "learning_rate": 3.866371205530811e-06, + "loss": 0.55209059, + "num_input_tokens_seen": 22573000, + "router_z_loss_clip": 0.32421875, + "router_z_loss_mlp": 0.03320312, + "step": 820, + "time_per_iteration": 3.2435152530670166 + }, + { + "auxiliary_loss_clip": 0.01175753, + "auxiliary_loss_mlp": 0.01061115, + "balance_loss_clip": 1.0478766, + "balance_loss_mlp": 1.03069282, + "epoch": 0.02382334165167431, + "flos": 15076368789120.0, + "grad_norm": 3.0052730972135504, + "language_loss": 0.99337047, + "learning_rate": 3.86707354572577e-06, + "loss": 1.0157392, + "num_input_tokens_seen": 22584610, + "router_z_loss_clip": 1.27929688, + "router_z_loss_mlp": 0.30407715, + "step": 821, + "time_per_iteration": 2.4520227909088135 + }, + { + "auxiliary_loss_clip": 0.01181911, + "auxiliary_loss_mlp": 0.0107088, + "balance_loss_clip": 1.04858053, + "balance_loss_mlp": 1.03649998, + "epoch": 0.023852359120190354, + "flos": 10184932442880.0, + "grad_norm": 4.85575448958902, + "language_loss": 0.95953697, + "learning_rate": 3.867775030972013e-06, + "loss": 0.98206484, + "num_input_tokens_seen": 22593075, + "router_z_loss_clip": 1.33300781, + "router_z_loss_mlp": 0.34326172, + "step": 822, + "time_per_iteration": 2.3640010356903076 + }, + { + "auxiliary_loss_clip": 0.01185347, + "auxiliary_loss_mlp": 0.01068204, + "balance_loss_clip": 1.04723668, + "balance_loss_mlp": 1.03103507, + "epoch": 0.023881376588706402, + "flos": 12194498056320.0, + "grad_norm": 4.329136308295026, + "language_loss": 0.94639659, + "learning_rate": 3.868475663348448e-06, + "loss": 0.96893215, + "num_input_tokens_seen": 22604130, + "router_z_loss_clip": 1.37988281, + "router_z_loss_mlp": 0.37207031, + "step": 823, + "time_per_iteration": 2.388925075531006 + }, + { + "auxiliary_loss_clip": 0.01034971, + "auxiliary_loss_mlp": 0.01008781, + "balance_loss_clip": 1.00299382, + "balance_loss_mlp": 1.00565803, + "epoch": 0.023910394057222447, + "flos": 60428089451520.0, + "grad_norm": 0.6930633563778732, + "language_loss": 0.55348766, + "learning_rate": 3.8691754449264e-06, + "loss": 0.5739252, + "num_input_tokens_seen": 22662890, + "router_z_loss_clip": 0.3203125, + "router_z_loss_mlp": 0.03112793, + "step": 824, + "time_per_iteration": 2.9084575176239014 + }, + { + "auxiliary_loss_clip": 0.01185508, + "auxiliary_loss_mlp": 0.01063793, + "balance_loss_clip": 1.04672098, + "balance_loss_mlp": 1.02919805, + "epoch": 0.023939411525738495, + "flos": 24495935207040.0, + "grad_norm": 2.6755557738848132, + "language_loss": 0.82234335, + "learning_rate": 3.869874377769666e-06, + "loss": 0.84483635, + "num_input_tokens_seen": 22678465, + "router_z_loss_clip": 1.38867188, + "router_z_loss_mlp": 0.34570312, + "step": 825, + "time_per_iteration": 2.5203824043273926 + }, + { + "auxiliary_loss_clip": 0.01180993, + "auxiliary_loss_mlp": 0.01063003, + "balance_loss_clip": 1.04662323, + "balance_loss_mlp": 1.03070927, + "epoch": 0.02396842899425454, + "flos": 16864445537280.0, + "grad_norm": 2.523662708663929, + "language_loss": 0.94074905, + "learning_rate": 3.870572463934538e-06, + "loss": 0.96318907, + "num_input_tokens_seen": 22691670, + "router_z_loss_clip": 1.34472656, + "router_z_loss_mlp": 0.32299805, + "step": 826, + "time_per_iteration": 2.3927950859069824 + }, + { + "auxiliary_loss_clip": 0.01187974, + "auxiliary_loss_mlp": 0.0106079, + "balance_loss_clip": 1.04922879, + "balance_loss_mlp": 1.02687478, + "epoch": 0.023997446462770588, + "flos": 11465511534720.0, + "grad_norm": 3.7528090182523224, + "language_loss": 1.02764225, + "learning_rate": 3.871269705469845e-06, + "loss": 1.05012989, + "num_input_tokens_seen": 22702735, + "router_z_loss_clip": 1.38671875, + "router_z_loss_mlp": 0.33935547, + "step": 827, + "time_per_iteration": 2.4109041690826416 + }, + { + "auxiliary_loss_clip": 0.01169542, + "auxiliary_loss_mlp": 0.01057449, + "balance_loss_clip": 1.04465449, + "balance_loss_mlp": 1.03054345, + "epoch": 0.024026463931286636, + "flos": 33758009015040.0, + "grad_norm": 2.977194828316303, + "language_loss": 0.7104938, + "learning_rate": 3.871966104416989e-06, + "loss": 0.73276377, + "num_input_tokens_seen": 22717825, + "router_z_loss_clip": 1.24804688, + "router_z_loss_mlp": 0.26916504, + "step": 828, + "time_per_iteration": 2.4912943840026855 + }, + { + "auxiliary_loss_clip": 0.01184875, + "auxiliary_loss_mlp": 0.0107218, + "balance_loss_clip": 1.04847848, + "balance_loss_mlp": 1.03694153, + "epoch": 0.02405548139980268, + "flos": 35325853706880.0, + "grad_norm": 5.503363498354128, + "language_loss": 0.92427188, + "learning_rate": 3.872661662809979e-06, + "loss": 0.94684243, + "num_input_tokens_seen": 22735085, + "router_z_loss_clip": 1.36328125, + "router_z_loss_mlp": 0.35229492, + "step": 829, + "time_per_iteration": 2.5689401626586914 + }, + { + "auxiliary_loss_clip": 0.01175366, + "auxiliary_loss_mlp": 0.01058554, + "balance_loss_clip": 1.04815698, + "balance_loss_mlp": 1.02665401, + "epoch": 0.02408449886831873, + "flos": 64702053624960.0, + "grad_norm": 1.8030971540051184, + "language_loss": 0.54974514, + "learning_rate": 3.873356382675468e-06, + "loss": 0.57208431, + "num_input_tokens_seen": 22760180, + "router_z_loss_clip": 1.27246094, + "router_z_loss_mlp": 0.31884766, + "step": 830, + "time_per_iteration": 2.7513070106506348 + }, + { + "auxiliary_loss_clip": 0.0118398, + "auxiliary_loss_mlp": 0.01061694, + "balance_loss_clip": 1.04854262, + "balance_loss_mlp": 1.03036582, + "epoch": 0.024113516336834773, + "flos": 25330778570880.0, + "grad_norm": 2.4938292029155598, + "language_loss": 0.85892618, + "learning_rate": 3.87405026603279e-06, + "loss": 0.88138288, + "num_input_tokens_seen": 22776350, + "router_z_loss_clip": 1.35644531, + "router_z_loss_mlp": 0.31323242, + "step": 831, + "time_per_iteration": 2.4444854259490967 + }, + { + "auxiliary_loss_clip": 0.0118347, + "auxiliary_loss_mlp": 0.01059202, + "balance_loss_clip": 1.05167174, + "balance_loss_mlp": 1.02768302, + "epoch": 0.02414253380535082, + "flos": 26243093975040.0, + "grad_norm": 1.666034348864892, + "language_loss": 0.66616428, + "learning_rate": 3.8747433148939905e-06, + "loss": 0.68859094, + "num_input_tokens_seen": 22794025, + "router_z_loss_clip": 1.31835938, + "router_z_loss_mlp": 0.31494141, + "step": 832, + "time_per_iteration": 2.455075979232788 + }, + { + "auxiliary_loss_clip": 0.01183281, + "auxiliary_loss_mlp": 0.01068908, + "balance_loss_clip": 1.05043912, + "balance_loss_mlp": 1.03959441, + "epoch": 0.024171551273866866, + "flos": 32735891784960.0, + "grad_norm": 3.4080885713344786, + "language_loss": 0.83783156, + "learning_rate": 3.875435531263866e-06, + "loss": 0.86035347, + "num_input_tokens_seen": 22809030, + "router_z_loss_clip": 1.33007812, + "router_z_loss_mlp": 0.29321289, + "step": 833, + "time_per_iteration": 2.519310235977173 + }, + { + "auxiliary_loss_clip": 0.01032934, + "auxiliary_loss_mlp": 0.0100316, + "balance_loss_clip": 1.00225759, + "balance_loss_mlp": 1.00006068, + "epoch": 0.024200568742382914, + "flos": 74768943294720.0, + "grad_norm": 0.7304433041823255, + "language_loss": 0.53278351, + "learning_rate": 3.876126917139997e-06, + "loss": 0.55314445, + "num_input_tokens_seen": 22866415, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.03100586, + "step": 834, + "time_per_iteration": 2.9953787326812744 + }, + { + "auxiliary_loss_clip": 0.01182383, + "auxiliary_loss_mlp": 0.01051974, + "balance_loss_clip": 1.04632688, + "balance_loss_mlp": 1.02443671, + "epoch": 0.024229586210898962, + "flos": 17887226083200.0, + "grad_norm": 2.37590186844371, + "language_loss": 0.66947204, + "learning_rate": 3.876817474512782e-06, + "loss": 0.69181561, + "num_input_tokens_seen": 22880520, + "router_z_loss_clip": 1.36132812, + "router_z_loss_mlp": 0.27526855, + "step": 835, + "time_per_iteration": 2.4538586139678955 + }, + { + "auxiliary_loss_clip": 0.01179894, + "auxiliary_loss_mlp": 0.01061175, + "balance_loss_clip": 1.05008864, + "balance_loss_mlp": 1.0292511, + "epoch": 0.024258603679415007, + "flos": 27667445512320.0, + "grad_norm": 2.3869289778048604, + "language_loss": 0.90634394, + "learning_rate": 3.8775072053654756e-06, + "loss": 0.92875469, + "num_input_tokens_seen": 22898465, + "router_z_loss_clip": 1.29882812, + "router_z_loss_mlp": 0.31921387, + "step": 836, + "time_per_iteration": 2.455637216567993 + }, + { + "auxiliary_loss_clip": 0.01201037, + "auxiliary_loss_mlp": 0.01069214, + "balance_loss_clip": 1.05150688, + "balance_loss_mlp": 1.03520393, + "epoch": 0.024287621147931055, + "flos": 22777405620480.0, + "grad_norm": 2.629405165957938, + "language_loss": 1.05821049, + "learning_rate": 3.878196111674215e-06, + "loss": 1.08091295, + "num_input_tokens_seen": 22914685, + "router_z_loss_clip": 1.49414062, + "router_z_loss_mlp": 0.34020996, + "step": 837, + "time_per_iteration": 2.5599751472473145 + }, + { + "auxiliary_loss_clip": 0.01191318, + "auxiliary_loss_mlp": 0.01072461, + "balance_loss_clip": 1.05159664, + "balance_loss_mlp": 1.03889191, + "epoch": 0.0243166386164471, + "flos": 15404482546560.0, + "grad_norm": 3.9534187815519206, + "language_loss": 0.96142632, + "learning_rate": 3.878884195408061e-06, + "loss": 0.9840641, + "num_input_tokens_seen": 22927160, + "router_z_loss_clip": 1.39746094, + "router_z_loss_mlp": 0.33544922, + "step": 838, + "time_per_iteration": 2.5126657485961914 + }, + { + "auxiliary_loss_clip": 0.01196107, + "auxiliary_loss_mlp": 0.01070397, + "balance_loss_clip": 1.05023313, + "balance_loss_mlp": 1.03267932, + "epoch": 0.024345656084963148, + "flos": 11357699656320.0, + "grad_norm": 4.903534679197105, + "language_loss": 0.84442234, + "learning_rate": 3.879571458529031e-06, + "loss": 0.86708742, + "num_input_tokens_seen": 22941175, + "router_z_loss_clip": 1.45800781, + "router_z_loss_mlp": 0.37670898, + "step": 839, + "time_per_iteration": 4.94146466255188 + }, + { + "auxiliary_loss_clip": 0.01032512, + "auxiliary_loss_mlp": 0.01003162, + "balance_loss_clip": 1.00205493, + "balance_loss_mlp": 0.9997049, + "epoch": 0.024374673553479196, + "flos": 58312143325440.0, + "grad_norm": 0.6992049346227985, + "language_loss": 0.52394015, + "learning_rate": 3.88025790299213e-06, + "loss": 0.54429692, + "num_input_tokens_seen": 22992970, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.03466797, + "step": 840, + "time_per_iteration": 2.834063768386841 + }, + { + "auxiliary_loss_clip": 0.01179192, + "auxiliary_loss_mlp": 0.01057422, + "balance_loss_clip": 1.04790676, + "balance_loss_mlp": 1.0229944, + "epoch": 0.02440369102199524, + "flos": 10443184836480.0, + "grad_norm": 3.4192663607852833, + "language_loss": 0.99701273, + "learning_rate": 3.880943530745382e-06, + "loss": 1.0193789, + "num_input_tokens_seen": 23004650, + "router_z_loss_clip": 1.31445312, + "router_z_loss_mlp": 0.34436035, + "step": 841, + "time_per_iteration": 2.3808350563049316 + }, + { + "auxiliary_loss_clip": 0.01180425, + "auxiliary_loss_mlp": 0.01065119, + "balance_loss_clip": 1.04745424, + "balance_loss_mlp": 1.02947509, + "epoch": 0.02443270849051129, + "flos": 19492672176000.0, + "grad_norm": 2.9189629651885713, + "language_loss": 1.02455509, + "learning_rate": 3.881628343729871e-06, + "loss": 1.04701042, + "num_input_tokens_seen": 23017855, + "router_z_loss_clip": 1.32910156, + "router_z_loss_mlp": 0.35668945, + "step": 842, + "time_per_iteration": 4.86422324180603 + }, + { + "auxiliary_loss_clip": 0.01190027, + "auxiliary_loss_mlp": 0.01080599, + "balance_loss_clip": 1.05060208, + "balance_loss_mlp": 1.04679132, + "epoch": 0.024461725959027333, + "flos": 18725665317120.0, + "grad_norm": 2.723607621626494, + "language_loss": 0.89026952, + "learning_rate": 3.882312343879765e-06, + "loss": 0.91297579, + "num_input_tokens_seen": 23030935, + "router_z_loss_clip": 1.39550781, + "router_z_loss_mlp": 0.33813477, + "step": 843, + "time_per_iteration": 2.409306764602661 + }, + { + "auxiliary_loss_clip": 0.01183944, + "auxiliary_loss_mlp": 0.0106298, + "balance_loss_clip": 1.04773211, + "balance_loss_mlp": 1.03198588, + "epoch": 0.02449074342754338, + "flos": 32851139783040.0, + "grad_norm": 2.239058308190672, + "language_loss": 0.98122007, + "learning_rate": 3.882995533122357e-06, + "loss": 1.00368929, + "num_input_tokens_seen": 23051470, + "router_z_loss_clip": 1.36425781, + "router_z_loss_mlp": 0.30969238, + "step": 844, + "time_per_iteration": 5.169459342956543 + }, + { + "auxiliary_loss_clip": 0.01191451, + "auxiliary_loss_mlp": 0.01058327, + "balance_loss_clip": 1.04913568, + "balance_loss_mlp": 1.02745175, + "epoch": 0.024519760896059426, + "flos": 16973968072320.0, + "grad_norm": 10.259589179163461, + "language_loss": 0.99976021, + "learning_rate": 3.88367791337809e-06, + "loss": 1.02225804, + "num_input_tokens_seen": 23062730, + "router_z_loss_clip": 1.42285156, + "router_z_loss_mlp": 0.30871582, + "step": 845, + "time_per_iteration": 2.36834716796875 + }, + { + "auxiliary_loss_clip": 0.01033204, + "auxiliary_loss_mlp": 0.01002847, + "balance_loss_clip": 1.00226974, + "balance_loss_mlp": 0.99905646, + "epoch": 0.024548778364575474, + "flos": 53319528259200.0, + "grad_norm": 0.7238099533152089, + "language_loss": 0.52510852, + "learning_rate": 3.884359486560594e-06, + "loss": 0.54546905, + "num_input_tokens_seen": 23118095, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.0378418, + "step": 846, + "time_per_iteration": 3.280519962310791 + }, + { + "auxiliary_loss_clip": 0.01182724, + "auxiliary_loss_mlp": 0.01067737, + "balance_loss_clip": 1.05601525, + "balance_loss_mlp": 1.03612244, + "epoch": 0.024577795833091522, + "flos": 13619093973120.0, + "grad_norm": 2.697100426738351, + "language_loss": 0.834409, + "learning_rate": 3.885040254576717e-06, + "loss": 0.85691357, + "num_input_tokens_seen": 23131945, + "router_z_loss_clip": 1.26757812, + "router_z_loss_mlp": 0.31604004, + "step": 847, + "time_per_iteration": 2.4697890281677246 + }, + { + "auxiliary_loss_clip": 0.01185066, + "auxiliary_loss_mlp": 0.01064684, + "balance_loss_clip": 1.04841399, + "balance_loss_mlp": 1.03047085, + "epoch": 0.024606813301607567, + "flos": 32298814074240.0, + "grad_norm": 5.181806927989984, + "language_loss": 0.78925264, + "learning_rate": 3.885720219326559e-06, + "loss": 0.81175011, + "num_input_tokens_seen": 23146705, + "router_z_loss_clip": 1.36621094, + "router_z_loss_mlp": 0.34204102, + "step": 848, + "time_per_iteration": 4.925029754638672 + }, + { + "auxiliary_loss_clip": 0.01033325, + "auxiliary_loss_mlp": 0.01003457, + "balance_loss_clip": 1.00242138, + "balance_loss_mlp": 0.99933261, + "epoch": 0.024635830770123615, + "flos": 59630323818240.0, + "grad_norm": 0.8063958593354277, + "language_loss": 0.54966736, + "learning_rate": 3.886399382703498e-06, + "loss": 0.57003522, + "num_input_tokens_seen": 23204840, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.04125977, + "step": 849, + "time_per_iteration": 2.9722986221313477 + }, + { + "auxiliary_loss_clip": 0.01033123, + "auxiliary_loss_mlp": 0.01003423, + "balance_loss_clip": 1.00245309, + "balance_loss_mlp": 0.99958485, + "epoch": 0.02466484823863966, + "flos": 68308335624960.0, + "grad_norm": 0.7851332938462892, + "language_loss": 0.54392731, + "learning_rate": 3.887077746594228e-06, + "loss": 0.56429273, + "num_input_tokens_seen": 23267380, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.03833008, + "step": 850, + "time_per_iteration": 3.014650821685791 + }, + { + "auxiliary_loss_clip": 0.01179228, + "auxiliary_loss_mlp": 0.0107559, + "balance_loss_clip": 1.04807627, + "balance_loss_mlp": 1.04271197, + "epoch": 0.024693865707155708, + "flos": 12267815644800.0, + "grad_norm": 3.283530094699197, + "language_loss": 0.97695029, + "learning_rate": 3.88775531287879e-06, + "loss": 0.99949849, + "num_input_tokens_seen": 23277990, + "router_z_loss_clip": 1.31152344, + "router_z_loss_mlp": 0.32861328, + "step": 851, + "time_per_iteration": 2.395385503768921 + }, + { + "auxiliary_loss_clip": 0.01033516, + "auxiliary_loss_mlp": 0.01007866, + "balance_loss_clip": 1.00262499, + "balance_loss_mlp": 1.0041703, + "epoch": 0.024722883175671756, + "flos": 58498055648640.0, + "grad_norm": 0.8453605079121098, + "language_loss": 0.57410288, + "learning_rate": 3.888432083430597e-06, + "loss": 0.59451675, + "num_input_tokens_seen": 23335060, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.03686523, + "step": 852, + "time_per_iteration": 2.966702461242676 + }, + { + "auxiliary_loss_clip": 0.01178664, + "auxiliary_loss_mlp": 0.0106585, + "balance_loss_clip": 1.0523169, + "balance_loss_mlp": 1.03695381, + "epoch": 0.0247519006441878, + "flos": 43316460288000.0, + "grad_norm": 2.7452830574010125, + "language_loss": 0.92582303, + "learning_rate": 3.889108060116473e-06, + "loss": 0.94826818, + "num_input_tokens_seen": 23351345, + "router_z_loss_clip": 1.26123047, + "router_z_loss_mlp": 0.28869629, + "step": 853, + "time_per_iteration": 2.6020641326904297 + }, + { + "auxiliary_loss_clip": 0.01033469, + "auxiliary_loss_mlp": 0.01011834, + "balance_loss_clip": 1.00254655, + "balance_loss_mlp": 1.00849617, + "epoch": 0.02478091811270385, + "flos": 68925251082240.0, + "grad_norm": 0.7043246032695123, + "language_loss": 0.52265358, + "learning_rate": 3.889783244796675e-06, + "loss": 0.54310662, + "num_input_tokens_seen": 23408405, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.03344727, + "step": 854, + "time_per_iteration": 2.9418437480926514 + }, + { + "auxiliary_loss_clip": 0.01191926, + "auxiliary_loss_mlp": 0.01076694, + "balance_loss_clip": 1.0547812, + "balance_loss_mlp": 1.0442214, + "epoch": 0.024809935581219893, + "flos": 21546856373760.0, + "grad_norm": 2.508609578064674, + "language_loss": 0.83177829, + "learning_rate": 3.890457639324937e-06, + "loss": 0.85446447, + "num_input_tokens_seen": 23427245, + "router_z_loss_clip": 1.37109375, + "router_z_loss_mlp": 0.32470703, + "step": 855, + "time_per_iteration": 2.4761438369750977 + }, + { + "auxiliary_loss_clip": 0.01190374, + "auxiliary_loss_mlp": 0.01058287, + "balance_loss_clip": 1.05352259, + "balance_loss_mlp": 1.02297735, + "epoch": 0.02483895304973594, + "flos": 16791442151040.0, + "grad_norm": 3.261034631891349, + "language_loss": 1.00515032, + "learning_rate": 3.891131245548486e-06, + "loss": 1.02763689, + "num_input_tokens_seen": 23440585, + "router_z_loss_clip": 1.3671875, + "router_z_loss_mlp": 0.35327148, + "step": 856, + "time_per_iteration": 2.4172215461730957 + }, + { + "auxiliary_loss_clip": 0.01178512, + "auxiliary_loss_mlp": 0.01057356, + "balance_loss_clip": 1.05003405, + "balance_loss_mlp": 1.02675486, + "epoch": 0.024867970518251986, + "flos": 15881081783040.0, + "grad_norm": 3.392355199933124, + "language_loss": 0.86601043, + "learning_rate": 3.89180406530808e-06, + "loss": 0.88836902, + "num_input_tokens_seen": 23453110, + "router_z_loss_clip": 1.28320312, + "router_z_loss_mlp": 0.30603027, + "step": 857, + "time_per_iteration": 2.399155855178833 + }, + { + "auxiliary_loss_clip": 0.01187598, + "auxiliary_loss_mlp": 0.0106451, + "balance_loss_clip": 1.04760659, + "balance_loss_mlp": 1.03103614, + "epoch": 0.024896987986768034, + "flos": 11062648823040.0, + "grad_norm": 3.602459929375625, + "language_loss": 0.84125185, + "learning_rate": 3.892476100438039e-06, + "loss": 0.86377293, + "num_input_tokens_seen": 23466395, + "router_z_loss_clip": 1.3984375, + "router_z_loss_mlp": 0.3347168, + "step": 858, + "time_per_iteration": 2.7103538513183594 + }, + { + "auxiliary_loss_clip": 0.01034694, + "auxiliary_loss_mlp": 0.01003643, + "balance_loss_clip": 1.00305152, + "balance_loss_mlp": 1.00037658, + "epoch": 0.024926005455284082, + "flos": 58356552441600.0, + "grad_norm": 0.7268043508869724, + "language_loss": 0.54221612, + "learning_rate": 3.8931473527662725e-06, + "loss": 0.56259948, + "num_input_tokens_seen": 23529420, + "router_z_loss_clip": 0.31640625, + "router_z_loss_mlp": 0.03271484, + "step": 859, + "time_per_iteration": 3.0012259483337402 + }, + { + "auxiliary_loss_clip": 0.01190367, + "auxiliary_loss_mlp": 0.0106195, + "balance_loss_clip": 1.05170095, + "balance_loss_mlp": 1.02995467, + "epoch": 0.024955022923800127, + "flos": 25257007134720.0, + "grad_norm": 2.692738357410124, + "language_loss": 0.69320142, + "learning_rate": 3.893817824114308e-06, + "loss": 0.71572459, + "num_input_tokens_seen": 23544890, + "router_z_loss_clip": 1.38574219, + "router_z_loss_mlp": 0.32006836, + "step": 860, + "time_per_iteration": 2.462371587753296 + }, + { + "auxiliary_loss_clip": 0.01035354, + "auxiliary_loss_mlp": 0.01004454, + "balance_loss_clip": 1.00445533, + "balance_loss_mlp": 1.00125909, + "epoch": 0.024984040392316175, + "flos": 73449087056640.0, + "grad_norm": 0.7387651526943686, + "language_loss": 0.48937726, + "learning_rate": 3.894487516297324e-06, + "loss": 0.50977534, + "num_input_tokens_seen": 23595790, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.03198242, + "step": 861, + "time_per_iteration": 2.9381215572357178 + }, + { + "auxiliary_loss_clip": 0.01170328, + "auxiliary_loss_mlp": 0.01063466, + "balance_loss_clip": 1.04605567, + "balance_loss_mlp": 1.02964604, + "epoch": 0.02501305786083222, + "flos": 12377966584320.0, + "grad_norm": 3.1658955679339083, + "language_loss": 0.91876364, + "learning_rate": 3.895156431124179e-06, + "loss": 0.94110161, + "num_input_tokens_seen": 23606470, + "router_z_loss_clip": 1.24121094, + "router_z_loss_mlp": 0.3380127, + "step": 862, + "time_per_iteration": 2.3670814037323 + }, + { + "auxiliary_loss_clip": 0.01173861, + "auxiliary_loss_mlp": 0.01062737, + "balance_loss_clip": 1.04765964, + "balance_loss_mlp": 1.03050268, + "epoch": 0.025042075329348268, + "flos": 15221991536640.0, + "grad_norm": 2.7392203388114202, + "language_loss": 0.76961195, + "learning_rate": 3.895824570397436e-06, + "loss": 0.791978, + "num_input_tokens_seen": 23618450, + "router_z_loss_clip": 1.26171875, + "router_z_loss_mlp": 0.32214355, + "step": 863, + "time_per_iteration": 2.3888278007507324 + }, + { + "auxiliary_loss_clip": 0.01180094, + "auxiliary_loss_mlp": 0.01075654, + "balance_loss_clip": 1.05179942, + "balance_loss_mlp": 1.04246616, + "epoch": 0.025071092797864316, + "flos": 10880856040320.0, + "grad_norm": 3.0772425773767815, + "language_loss": 0.8513006, + "learning_rate": 3.896491935913401e-06, + "loss": 0.87385809, + "num_input_tokens_seen": 23629475, + "router_z_loss_clip": 1.28417969, + "router_z_loss_mlp": 0.33178711, + "step": 864, + "time_per_iteration": 2.420639753341675 + }, + { + "auxiliary_loss_clip": 0.01177572, + "auxiliary_loss_mlp": 0.01065477, + "balance_loss_clip": 1.04633081, + "balance_loss_mlp": 1.03373194, + "epoch": 0.02510011026638036, + "flos": 24417904584960.0, + "grad_norm": 2.8733684039972984, + "language_loss": 1.0035615, + "learning_rate": 3.897158529462142e-06, + "loss": 1.02599192, + "num_input_tokens_seen": 23646720, + "router_z_loss_clip": 1.31152344, + "router_z_loss_mlp": 0.31750488, + "step": 865, + "time_per_iteration": 2.462392568588257 + }, + { + "auxiliary_loss_clip": 0.01174346, + "auxiliary_loss_mlp": 0.01061961, + "balance_loss_clip": 1.0459969, + "balance_loss_mlp": 1.03244519, + "epoch": 0.02512912773489641, + "flos": 29965079687040.0, + "grad_norm": 3.428990227308138, + "language_loss": 0.78848106, + "learning_rate": 3.8978243528275245e-06, + "loss": 0.81084412, + "num_input_tokens_seen": 23665625, + "router_z_loss_clip": 1.28417969, + "router_z_loss_mlp": 0.29528809, + "step": 866, + "time_per_iteration": 2.503988742828369 + }, + { + "auxiliary_loss_clip": 0.01034688, + "auxiliary_loss_mlp": 0.01011997, + "balance_loss_clip": 1.00501037, + "balance_loss_mlp": 1.00865936, + "epoch": 0.025158145203412453, + "flos": 74774110176000.0, + "grad_norm": 0.6502571459129123, + "language_loss": 0.52877462, + "learning_rate": 3.898489407787237e-06, + "loss": 0.54924142, + "num_input_tokens_seen": 23736630, + "router_z_loss_clip": 0.296875, + "router_z_loss_mlp": 0.03344727, + "step": 867, + "time_per_iteration": 3.2771317958831787 + }, + { + "auxiliary_loss_clip": 0.01199074, + "auxiliary_loss_mlp": 0.01068304, + "balance_loss_clip": 1.04890704, + "balance_loss_mlp": 1.03051448, + "epoch": 0.0251871626719285, + "flos": 32445588896640.0, + "grad_norm": 1.9270896256058576, + "language_loss": 0.83071113, + "learning_rate": 3.89915369611282e-06, + "loss": 0.85338491, + "num_input_tokens_seen": 23764165, + "router_z_loss_clip": 1.50097656, + "router_z_loss_mlp": 0.37817383, + "step": 868, + "time_per_iteration": 2.776620626449585 + }, + { + "auxiliary_loss_clip": 0.0118527, + "auxiliary_loss_mlp": 0.01070167, + "balance_loss_clip": 1.04590964, + "balance_loss_mlp": 1.03680038, + "epoch": 0.025216180140444546, + "flos": 15991686570240.0, + "grad_norm": 3.547038615829003, + "language_loss": 0.85426021, + "learning_rate": 3.899817219569695e-06, + "loss": 0.87681454, + "num_input_tokens_seen": 23776725, + "router_z_loss_clip": 1.39453125, + "router_z_loss_mlp": 0.33374023, + "step": 869, + "time_per_iteration": 2.4944472312927246 + }, + { + "auxiliary_loss_clip": 0.01181364, + "auxiliary_loss_mlp": 0.01063846, + "balance_loss_clip": 1.04675567, + "balance_loss_mlp": 1.03005064, + "epoch": 0.025245197608960594, + "flos": 12560213214720.0, + "grad_norm": 3.0572808149235526, + "language_loss": 0.97203702, + "learning_rate": 3.900479979917193e-06, + "loss": 0.99448907, + "num_input_tokens_seen": 23789535, + "router_z_loss_clip": 1.34472656, + "router_z_loss_mlp": 0.33813477, + "step": 870, + "time_per_iteration": 2.370790958404541 + }, + { + "auxiliary_loss_clip": 0.01182062, + "auxiliary_loss_mlp": 0.01076116, + "balance_loss_clip": 1.04436791, + "balance_loss_mlp": 1.0410682, + "epoch": 0.025274215077476642, + "flos": 47737232328960.0, + "grad_norm": 2.4513391905935644, + "language_loss": 0.90403533, + "learning_rate": 3.901141978908582e-06, + "loss": 0.92661715, + "num_input_tokens_seen": 23812190, + "router_z_loss_clip": 1.37646484, + "router_z_loss_mlp": 0.3503418, + "step": 871, + "time_per_iteration": 2.9306857585906982 + }, + { + "auxiliary_loss_clip": 0.01178839, + "auxiliary_loss_mlp": 0.01062827, + "balance_loss_clip": 1.04687142, + "balance_loss_mlp": 1.03078341, + "epoch": 0.025303232545992687, + "flos": 16404464108160.0, + "grad_norm": 2.9440851020067726, + "language_loss": 0.90896636, + "learning_rate": 3.901803218291094e-06, + "loss": 0.93138295, + "num_input_tokens_seen": 23826655, + "router_z_loss_clip": 1.3203125, + "router_z_loss_mlp": 0.32019043, + "step": 872, + "time_per_iteration": 2.418184518814087 + }, + { + "auxiliary_loss_clip": 0.01177498, + "auxiliary_loss_mlp": 0.01066511, + "balance_loss_clip": 1.04870963, + "balance_loss_mlp": 1.03481364, + "epoch": 0.025332250014508735, + "flos": 47816205557760.0, + "grad_norm": 2.429761213921722, + "language_loss": 0.94157171, + "learning_rate": 3.902463699805952e-06, + "loss": 0.96401179, + "num_input_tokens_seen": 23849390, + "router_z_loss_clip": 1.28808594, + "router_z_loss_mlp": 0.31689453, + "step": 873, + "time_per_iteration": 2.7084100246429443 + }, + { + "auxiliary_loss_clip": 0.01178028, + "auxiliary_loss_mlp": 0.01066588, + "balance_loss_clip": 1.0490123, + "balance_loss_mlp": 1.03521252, + "epoch": 0.02536126748302478, + "flos": 20443147562880.0, + "grad_norm": 2.453372355755464, + "language_loss": 0.85554528, + "learning_rate": 3.903123425188401e-06, + "loss": 0.87799144, + "num_input_tokens_seen": 23865620, + "router_z_loss_clip": 1.29003906, + "router_z_loss_mlp": 0.31347656, + "step": 874, + "time_per_iteration": 2.4212334156036377 + }, + { + "auxiliary_loss_clip": 0.01176163, + "auxiliary_loss_mlp": 0.01065042, + "balance_loss_clip": 1.04429567, + "balance_loss_mlp": 1.03101933, + "epoch": 0.025390284951540828, + "flos": 33940534936320.0, + "grad_norm": 2.5887273800195274, + "language_loss": 1.16717291, + "learning_rate": 3.903782396167732e-06, + "loss": 1.18958497, + "num_input_tokens_seen": 23882290, + "router_z_loss_clip": 1.31835938, + "router_z_loss_mlp": 0.34033203, + "step": 875, + "time_per_iteration": 2.5387072563171387 + }, + { + "auxiliary_loss_clip": 0.01033456, + "auxiliary_loss_mlp": 0.01007694, + "balance_loss_clip": 1.00284791, + "balance_loss_mlp": 1.00457072, + "epoch": 0.025419302420056876, + "flos": 59191779830400.0, + "grad_norm": 0.6649599861039153, + "language_loss": 0.54486215, + "learning_rate": 3.904440614467313e-06, + "loss": 0.56527364, + "num_input_tokens_seen": 23942490, + "router_z_loss_clip": 0.30664062, + "router_z_loss_mlp": 0.03125, + "step": 876, + "time_per_iteration": 3.021780014038086 + }, + { + "auxiliary_loss_clip": 0.01159741, + "auxiliary_loss_mlp": 0.01051908, + "balance_loss_clip": 1.04129982, + "balance_loss_mlp": 1.0243113, + "epoch": 0.02544831988857292, + "flos": 10771368416640.0, + "grad_norm": 3.160309841249461, + "language_loss": 0.93841982, + "learning_rate": 3.905098081804608e-06, + "loss": 0.9605363, + "num_input_tokens_seen": 23954680, + "router_z_loss_clip": 1.18554688, + "router_z_loss_mlp": 0.27587891, + "step": 877, + "time_per_iteration": 2.449155330657959 + }, + { + "auxiliary_loss_clip": 0.01184573, + "auxiliary_loss_mlp": 0.01061415, + "balance_loss_clip": 1.05247617, + "balance_loss_mlp": 1.03127885, + "epoch": 0.02547733735708897, + "flos": 10443324481920.0, + "grad_norm": 2.8448003253157323, + "language_loss": 0.98795891, + "learning_rate": 3.905754799891214e-06, + "loss": 1.01041889, + "num_input_tokens_seen": 23966875, + "router_z_loss_clip": 1.3203125, + "router_z_loss_mlp": 0.30175781, + "step": 878, + "time_per_iteration": 2.3626463413238525 + }, + { + "auxiliary_loss_clip": 0.0118138, + "auxiliary_loss_mlp": 0.01053086, + "balance_loss_clip": 1.04965138, + "balance_loss_mlp": 1.02147174, + "epoch": 0.025506354825605013, + "flos": 39229946403840.0, + "grad_norm": 2.4857301764470683, + "language_loss": 0.99810624, + "learning_rate": 3.9064107704328816e-06, + "loss": 1.02045083, + "num_input_tokens_seen": 23986835, + "router_z_loss_clip": 1.31738281, + "router_z_loss_mlp": 0.3157959, + "step": 879, + "time_per_iteration": 2.5484721660614014 + }, + { + "auxiliary_loss_clip": 0.01179537, + "auxiliary_loss_mlp": 0.01085015, + "balance_loss_clip": 1.04698968, + "balance_loss_mlp": 1.04541397, + "epoch": 0.02553537229412106, + "flos": 12560806707840.0, + "grad_norm": 2.993149487790626, + "language_loss": 1.07440364, + "learning_rate": 3.9070659951295425e-06, + "loss": 1.09704912, + "num_input_tokens_seen": 23998250, + "router_z_loss_clip": 1.32617188, + "router_z_loss_mlp": 0.39599609, + "step": 880, + "time_per_iteration": 2.3790271282196045 + }, + { + "auxiliary_loss_clip": 0.01182192, + "auxiliary_loss_mlp": 0.01071086, + "balance_loss_clip": 1.0474937, + "balance_loss_mlp": 1.03613329, + "epoch": 0.025564389762637106, + "flos": 16979553889920.0, + "grad_norm": 2.864112131566733, + "language_loss": 0.96332049, + "learning_rate": 3.907720475675338e-06, + "loss": 0.98585331, + "num_input_tokens_seen": 24014605, + "router_z_loss_clip": 1.34765625, + "router_z_loss_mlp": 0.34912109, + "step": 881, + "time_per_iteration": 2.422333002090454 + }, + { + "auxiliary_loss_clip": 0.01166487, + "auxiliary_loss_mlp": 0.0106322, + "balance_loss_clip": 1.04143071, + "balance_loss_mlp": 1.03279793, + "epoch": 0.025593407231153154, + "flos": 15841944282240.0, + "grad_norm": 3.062729457217865, + "language_loss": 0.74810827, + "learning_rate": 3.908374213758642e-06, + "loss": 0.77040535, + "num_input_tokens_seen": 24027300, + "router_z_loss_clip": 1.25195312, + "router_z_loss_mlp": 0.30432129, + "step": 882, + "time_per_iteration": 2.378354072570801 + }, + { + "auxiliary_loss_clip": 0.01033575, + "auxiliary_loss_mlp": 0.01009955, + "balance_loss_clip": 1.0034852, + "balance_loss_mlp": 1.00683141, + "epoch": 0.025622424699669202, + "flos": 59818435557120.0, + "grad_norm": 0.6969387314315689, + "language_loss": 0.55060291, + "learning_rate": 3.909027211062089e-06, + "loss": 0.57103813, + "num_input_tokens_seen": 24092200, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.03112793, + "step": 883, + "time_per_iteration": 3.034106492996216 + }, + { + "auxiliary_loss_clip": 0.01187754, + "auxiliary_loss_mlp": 0.01067389, + "balance_loss_clip": 1.05007398, + "balance_loss_mlp": 1.03465438, + "epoch": 0.025651442168185247, + "flos": 27264582800640.0, + "grad_norm": 2.4697357579140724, + "language_loss": 0.919765, + "learning_rate": 3.909679469262601e-06, + "loss": 0.94231641, + "num_input_tokens_seen": 24107905, + "router_z_loss_clip": 1.37695312, + "router_z_loss_mlp": 0.32739258, + "step": 884, + "time_per_iteration": 2.399184226989746 + }, + { + "auxiliary_loss_clip": 0.01176019, + "auxiliary_loss_mlp": 0.01061898, + "balance_loss_clip": 1.04639983, + "balance_loss_mlp": 1.03192902, + "epoch": 0.025680459636701295, + "flos": 50072537727360.0, + "grad_norm": 4.473719850463993, + "language_loss": 0.91684073, + "learning_rate": 3.910330990031413e-06, + "loss": 0.93921995, + "num_input_tokens_seen": 24131570, + "router_z_loss_clip": 1.29589844, + "router_z_loss_mlp": 0.29980469, + "step": 885, + "time_per_iteration": 2.79410719871521 + }, + { + "auxiliary_loss_clip": 0.01189806, + "auxiliary_loss_mlp": 0.01068913, + "balance_loss_clip": 1.04572582, + "balance_loss_mlp": 1.03139758, + "epoch": 0.02570947710521734, + "flos": 17813454647040.0, + "grad_norm": 3.0959066241361852, + "language_loss": 1.13191533, + "learning_rate": 3.910981775034096e-06, + "loss": 1.15450263, + "num_input_tokens_seen": 24143230, + "router_z_loss_clip": 1.44238281, + "router_z_loss_mlp": 0.37536621, + "step": 886, + "time_per_iteration": 2.3839855194091797 + }, + { + "auxiliary_loss_clip": 0.01169417, + "auxiliary_loss_mlp": 0.01060023, + "balance_loss_clip": 1.04530919, + "balance_loss_mlp": 1.02534485, + "epoch": 0.025738494573733388, + "flos": 16687121408640.0, + "grad_norm": 2.74716847581232, + "language_loss": 0.7843805, + "learning_rate": 3.911631825930584e-06, + "loss": 0.8066749, + "num_input_tokens_seen": 24158850, + "router_z_loss_clip": 1.2421875, + "router_z_loss_mlp": 0.34643555, + "step": 887, + "time_per_iteration": 2.4415431022644043 + }, + { + "auxiliary_loss_clip": 0.01171058, + "auxiliary_loss_mlp": 0.01073332, + "balance_loss_clip": 1.04789305, + "balance_loss_mlp": 1.03821337, + "epoch": 0.025767512042249432, + "flos": 27555374448000.0, + "grad_norm": 4.28131890840945, + "language_loss": 0.95905799, + "learning_rate": 3.9122811443752026e-06, + "loss": 0.98150194, + "num_input_tokens_seen": 24173490, + "router_z_loss_clip": 1.23046875, + "router_z_loss_mlp": 0.35095215, + "step": 888, + "time_per_iteration": 2.444268226623535 + }, + { + "auxiliary_loss_clip": 0.01185284, + "auxiliary_loss_mlp": 0.01068024, + "balance_loss_clip": 1.05007482, + "balance_loss_mlp": 1.03257155, + "epoch": 0.02579652951076548, + "flos": 41092702283520.0, + "grad_norm": 8.828696948611118, + "language_loss": 0.91601545, + "learning_rate": 3.912929732016691e-06, + "loss": 0.93854862, + "num_input_tokens_seen": 24189735, + "router_z_loss_clip": 1.35253906, + "router_z_loss_mlp": 0.35449219, + "step": 889, + "time_per_iteration": 2.496457099914551 + }, + { + "auxiliary_loss_clip": 0.01034315, + "auxiliary_loss_mlp": 0.01003983, + "balance_loss_clip": 1.00358295, + "balance_loss_mlp": 1.00088382, + "epoch": 0.02582554697928153, + "flos": 63964197751680.0, + "grad_norm": 0.6832202166580094, + "language_loss": 0.50913435, + "learning_rate": 3.913577590498226e-06, + "loss": 0.52951735, + "num_input_tokens_seen": 24249930, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.03088379, + "step": 890, + "time_per_iteration": 2.9834954738616943 + }, + { + "auxiliary_loss_clip": 0.01172416, + "auxiliary_loss_mlp": 0.0106107, + "balance_loss_clip": 1.04328418, + "balance_loss_mlp": 1.02812064, + "epoch": 0.025854564447797573, + "flos": 25990078285440.0, + "grad_norm": 2.2409720446980916, + "language_loss": 1.03825879, + "learning_rate": 3.91422472145745e-06, + "loss": 1.06059372, + "num_input_tokens_seen": 24266130, + "router_z_loss_clip": 1.29101562, + "router_z_loss_mlp": 0.32946777, + "step": 891, + "time_per_iteration": 2.4645814895629883 + }, + { + "auxiliary_loss_clip": 0.01163749, + "auxiliary_loss_mlp": 0.01059105, + "balance_loss_clip": 1.04392767, + "balance_loss_mlp": 1.03129315, + "epoch": 0.02588358191631362, + "flos": 16391162880000.0, + "grad_norm": 2.4145751697812226, + "language_loss": 0.7972759, + "learning_rate": 3.914871126526495e-06, + "loss": 0.8195045, + "num_input_tokens_seen": 24278130, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.27807617, + "step": 892, + "time_per_iteration": 2.468276262283325 + }, + { + "auxiliary_loss_clip": 0.01171352, + "auxiliary_loss_mlp": 0.01057778, + "balance_loss_clip": 1.04288483, + "balance_loss_mlp": 1.02509034, + "epoch": 0.025912599384829666, + "flos": 12924147893760.0, + "grad_norm": 3.918778747690828, + "language_loss": 1.01282787, + "learning_rate": 3.915516807332006e-06, + "loss": 1.03511918, + "num_input_tokens_seen": 24288865, + "router_z_loss_clip": 1.28613281, + "router_z_loss_mlp": 0.32666016, + "step": 893, + "time_per_iteration": 2.3601579666137695 + }, + { + "auxiliary_loss_clip": 0.01187024, + "auxiliary_loss_mlp": 0.01063043, + "balance_loss_clip": 1.05080736, + "balance_loss_mlp": 1.02872288, + "epoch": 0.025941616853345714, + "flos": 22446498954240.0, + "grad_norm": 3.7494541805657136, + "language_loss": 0.98892832, + "learning_rate": 3.916161765495166e-06, + "loss": 1.01142907, + "num_input_tokens_seen": 24300935, + "router_z_loss_clip": 1.36132812, + "router_z_loss_mlp": 0.34301758, + "step": 894, + "time_per_iteration": 2.4247498512268066 + }, + { + "auxiliary_loss_clip": 0.01034215, + "auxiliary_loss_mlp": 0.01006327, + "balance_loss_clip": 1.00277138, + "balance_loss_mlp": 1.00298941, + "epoch": 0.025970634321861762, + "flos": 72726384579840.0, + "grad_norm": 0.7535542722137364, + "language_loss": 0.5567596, + "learning_rate": 3.916806002631721e-06, + "loss": 0.57716501, + "num_input_tokens_seen": 24366515, + "router_z_loss_clip": 0.31445312, + "router_z_loss_mlp": 0.03344727, + "step": 895, + "time_per_iteration": 3.0532186031341553 + }, + { + "auxiliary_loss_clip": 0.01034222, + "auxiliary_loss_mlp": 0.01004893, + "balance_loss_clip": 1.00346828, + "balance_loss_mlp": 1.00145996, + "epoch": 0.025999651790377807, + "flos": 74776309591680.0, + "grad_norm": 0.6577403761724155, + "language_loss": 0.53104019, + "learning_rate": 3.917449520352006e-06, + "loss": 0.55143136, + "num_input_tokens_seen": 24437660, + "router_z_loss_clip": 0.30859375, + "router_z_loss_mlp": 0.03442383, + "step": 896, + "time_per_iteration": 3.1736555099487305 + }, + { + "auxiliary_loss_clip": 0.0103346, + "auxiliary_loss_mlp": 0.01002896, + "balance_loss_clip": 1.00236869, + "balance_loss_mlp": 0.99946296, + "epoch": 0.026028669258893855, + "flos": 56665744341120.0, + "grad_norm": 0.7352091424589863, + "language_loss": 0.51659101, + "learning_rate": 3.918092320260965e-06, + "loss": 0.53695458, + "num_input_tokens_seen": 24491695, + "router_z_loss_clip": 0.3125, + "router_z_loss_mlp": 0.03442383, + "step": 897, + "time_per_iteration": 2.7878212928771973 + }, + { + "auxiliary_loss_clip": 0.01033096, + "auxiliary_loss_mlp": 0.01002596, + "balance_loss_clip": 1.00271511, + "balance_loss_mlp": 0.99925822, + "epoch": 0.0260576867274099, + "flos": 56352572645760.0, + "grad_norm": 0.7023093553883396, + "language_loss": 0.549438, + "learning_rate": 3.918734403958178e-06, + "loss": 0.56979495, + "num_input_tokens_seen": 24551640, + "router_z_loss_clip": 0.3046875, + "router_z_loss_mlp": 0.03344727, + "step": 898, + "time_per_iteration": 3.304915428161621 + }, + { + "auxiliary_loss_clip": 0.01176662, + "auxiliary_loss_mlp": 0.01049125, + "balance_loss_clip": 1.0446701, + "balance_loss_mlp": 1.01678336, + "epoch": 0.026086704195925948, + "flos": 29234591976960.0, + "grad_norm": 3.0869057340130754, + "language_loss": 0.8508687, + "learning_rate": 3.919375773037884e-06, + "loss": 0.87312657, + "num_input_tokens_seen": 24573575, + "router_z_loss_clip": 1.31884766, + "router_z_loss_mlp": 0.32336426, + "step": 899, + "time_per_iteration": 2.6262972354888916 + }, + { + "auxiliary_loss_clip": 0.01167092, + "auxiliary_loss_mlp": 0.01066467, + "balance_loss_clip": 1.04520869, + "balance_loss_mlp": 1.03761864, + "epoch": 0.026115721664441992, + "flos": 14383971239040.0, + "grad_norm": 4.897496620990621, + "language_loss": 0.90250105, + "learning_rate": 3.9200164290890045e-06, + "loss": 0.9248367, + "num_input_tokens_seen": 24584115, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.28869629, + "step": 900, + "time_per_iteration": 2.5154929161071777 + }, + { + "auxiliary_loss_clip": 0.01034431, + "auxiliary_loss_mlp": 0.01006976, + "balance_loss_clip": 1.00440097, + "balance_loss_mlp": 1.00368571, + "epoch": 0.02614473913295804, + "flos": 61967584252800.0, + "grad_norm": 0.8943155746239743, + "language_loss": 0.54883242, + "learning_rate": 3.92065637369517e-06, + "loss": 0.56924647, + "num_input_tokens_seen": 24644865, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.03295898, + "step": 901, + "time_per_iteration": 2.9327900409698486 + }, + { + "auxiliary_loss_clip": 0.01168345, + "auxiliary_loss_mlp": 0.01057068, + "balance_loss_clip": 1.0462625, + "balance_loss_mlp": 1.02755141, + "epoch": 0.02617375660147409, + "flos": 30511121351040.0, + "grad_norm": 2.5372972503833915, + "language_loss": 0.82907367, + "learning_rate": 3.921295608434737e-06, + "loss": 0.85132778, + "num_input_tokens_seen": 24661255, + "router_z_loss_clip": 1.22167969, + "router_z_loss_mlp": 0.29492188, + "step": 902, + "time_per_iteration": 2.527657985687256 + }, + { + "auxiliary_loss_clip": 0.01034044, + "auxiliary_loss_mlp": 0.01016518, + "balance_loss_clip": 1.00403821, + "balance_loss_mlp": 1.01332343, + "epoch": 0.026202774069990133, + "flos": 58317449852160.0, + "grad_norm": 0.6754188777195904, + "language_loss": 0.51393682, + "learning_rate": 3.92193413488082e-06, + "loss": 0.53444242, + "num_input_tokens_seen": 24722775, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.03198242, + "step": 903, + "time_per_iteration": 3.0624237060546875 + }, + { + "auxiliary_loss_clip": 0.0118481, + "auxiliary_loss_mlp": 0.01071208, + "balance_loss_clip": 1.05066085, + "balance_loss_mlp": 1.03655398, + "epoch": 0.02623179153850618, + "flos": 17338007485440.0, + "grad_norm": 3.048669997863638, + "language_loss": 0.85331196, + "learning_rate": 3.922571954601306e-06, + "loss": 0.87587214, + "num_input_tokens_seen": 24733740, + "router_z_loss_clip": 1.34179688, + "router_z_loss_mlp": 0.34655762, + "step": 904, + "time_per_iteration": 2.3576338291168213 + }, + { + "auxiliary_loss_clip": 0.01186555, + "auxiliary_loss_mlp": 0.01064804, + "balance_loss_clip": 1.05110681, + "balance_loss_mlp": 1.02973258, + "epoch": 0.026260809007022226, + "flos": 19820192440320.0, + "grad_norm": 3.172118394526262, + "language_loss": 0.95671487, + "learning_rate": 3.9232090691588845e-06, + "loss": 0.97922844, + "num_input_tokens_seen": 24745660, + "router_z_loss_clip": 1.35253906, + "router_z_loss_mlp": 0.35058594, + "step": 905, + "time_per_iteration": 2.448408842086792 + }, + { + "auxiliary_loss_clip": 0.01163021, + "auxiliary_loss_mlp": 0.01054906, + "balance_loss_clip": 1.04401183, + "balance_loss_mlp": 1.02796483, + "epoch": 0.026289826475538274, + "flos": 18253011064320.0, + "grad_norm": 2.6373112827416625, + "language_loss": 0.74165112, + "learning_rate": 3.923845480111065e-06, + "loss": 0.76383042, + "num_input_tokens_seen": 24758895, + "router_z_loss_clip": 1.19042969, + "router_z_loss_mlp": 0.26965332, + "step": 906, + "time_per_iteration": 2.3499207496643066 + }, + { + "auxiliary_loss_clip": 0.01175281, + "auxiliary_loss_mlp": 0.01070435, + "balance_loss_clip": 1.04436922, + "balance_loss_mlp": 1.03756857, + "epoch": 0.026318843944054322, + "flos": 16279301283840.0, + "grad_norm": 3.735032758174692, + "language_loss": 0.97886586, + "learning_rate": 3.924481189010205e-06, + "loss": 1.00132298, + "num_input_tokens_seen": 24769185, + "router_z_loss_clip": 1.30810547, + "router_z_loss_mlp": 0.32885742, + "step": 907, + "time_per_iteration": 2.361823797225952 + }, + { + "auxiliary_loss_clip": 0.01171231, + "auxiliary_loss_mlp": 0.01063029, + "balance_loss_clip": 1.04649544, + "balance_loss_mlp": 1.03532445, + "epoch": 0.026347861412570367, + "flos": 12159968855040.0, + "grad_norm": 2.2978124006183607, + "language_loss": 0.87898207, + "learning_rate": 3.925116197403529e-06, + "loss": 0.90132463, + "num_input_tokens_seen": 24784365, + "router_z_loss_clip": 1.24804688, + "router_z_loss_mlp": 0.27697754, + "step": 908, + "time_per_iteration": 2.4097907543182373 + }, + { + "auxiliary_loss_clip": 0.01171874, + "auxiliary_loss_mlp": 0.01057797, + "balance_loss_clip": 1.04760218, + "balance_loss_mlp": 1.02649271, + "epoch": 0.026376878881086415, + "flos": 36021113988480.0, + "grad_norm": 4.336293549833986, + "language_loss": 0.8425653, + "learning_rate": 3.925750506833153e-06, + "loss": 0.86486202, + "num_input_tokens_seen": 24801230, + "router_z_loss_clip": 1.24316406, + "router_z_loss_mlp": 0.31347656, + "step": 909, + "time_per_iteration": 2.590991258621216 + }, + { + "auxiliary_loss_clip": 0.01168577, + "auxiliary_loss_mlp": 0.01050257, + "balance_loss_clip": 1.04911649, + "balance_loss_mlp": 1.02142036, + "epoch": 0.02640589634960246, + "flos": 32261457052800.0, + "grad_norm": 3.080521506239891, + "language_loss": 1.0187993, + "learning_rate": 3.926384118836106e-06, + "loss": 1.04098773, + "num_input_tokens_seen": 24816915, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.28869629, + "step": 910, + "time_per_iteration": 2.512951135635376 + }, + { + "auxiliary_loss_clip": 0.01042151, + "auxiliary_loss_mlp": 0.01002071, + "balance_loss_clip": 1.01222539, + "balance_loss_mlp": 0.99909091, + "epoch": 0.026434913818118508, + "flos": 71434666897920.0, + "grad_norm": 0.7196465738874105, + "language_loss": 0.53258491, + "learning_rate": 3.9270170349443515e-06, + "loss": 0.55302715, + "num_input_tokens_seen": 24883480, + "router_z_loss_clip": 0.30078125, + "router_z_loss_mlp": 0.02978516, + "step": 911, + "time_per_iteration": 3.353287696838379 + }, + { + "auxiliary_loss_clip": 0.0103848, + "auxiliary_loss_mlp": 0.01003201, + "balance_loss_clip": 1.00907993, + "balance_loss_mlp": 1.0003047, + "epoch": 0.026463931286634552, + "flos": 65793611416320.0, + "grad_norm": 0.7448658452170448, + "language_loss": 0.55670971, + "learning_rate": 3.927649256684814e-06, + "loss": 0.57712656, + "num_input_tokens_seen": 24941805, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.02893066, + "step": 912, + "time_per_iteration": 2.92807936668396 + }, + { + "auxiliary_loss_clip": 0.01034243, + "auxiliary_loss_mlp": 0.01004122, + "balance_loss_clip": 1.00495088, + "balance_loss_mlp": 1.00120127, + "epoch": 0.0264929487551506, + "flos": 60322791191040.0, + "grad_norm": 0.6783724944508474, + "language_loss": 0.53095973, + "learning_rate": 3.928280785579394e-06, + "loss": 0.55134338, + "num_input_tokens_seen": 25006025, + "router_z_loss_clip": 0.29296875, + "router_z_loss_mlp": 0.0291748, + "step": 913, + "time_per_iteration": 3.053368330001831 + }, + { + "auxiliary_loss_clip": 0.01172366, + "auxiliary_loss_mlp": 0.01065257, + "balance_loss_clip": 1.04748452, + "balance_loss_mlp": 1.03202176, + "epoch": 0.02652196622366665, + "flos": 21062855928960.0, + "grad_norm": 2.7874178575090207, + "language_loss": 0.89565229, + "learning_rate": 3.928911623144997e-06, + "loss": 0.91802847, + "num_input_tokens_seen": 25019310, + "router_z_loss_clip": 1.24707031, + "router_z_loss_mlp": 0.33215332, + "step": 914, + "time_per_iteration": 2.4568264484405518 + }, + { + "auxiliary_loss_clip": 0.01168499, + "auxiliary_loss_mlp": 0.01064603, + "balance_loss_clip": 1.04658556, + "balance_loss_mlp": 1.03527737, + "epoch": 0.026550983692182693, + "flos": 12890456565120.0, + "grad_norm": 3.9381953595835544, + "language_loss": 0.84962779, + "learning_rate": 3.92954177089355e-06, + "loss": 0.87195885, + "num_input_tokens_seen": 25031890, + "router_z_loss_clip": 1.22070312, + "router_z_loss_mlp": 0.29309082, + "step": 915, + "time_per_iteration": 4.834162473678589 + }, + { + "auxiliary_loss_clip": 0.01189217, + "auxiliary_loss_mlp": 0.01071377, + "balance_loss_clip": 1.04897285, + "balance_loss_mlp": 1.03783202, + "epoch": 0.02658000116069874, + "flos": 13217557893120.0, + "grad_norm": 2.7551102716649662, + "language_loss": 0.75776476, + "learning_rate": 3.9301712303320286e-06, + "loss": 0.78037071, + "num_input_tokens_seen": 25045930, + "router_z_loss_clip": 1.40234375, + "router_z_loss_mlp": 0.33544922, + "step": 916, + "time_per_iteration": 2.409233570098877 + }, + { + "auxiliary_loss_clip": 0.01165986, + "auxiliary_loss_mlp": 0.0106331, + "balance_loss_clip": 1.0457418, + "balance_loss_mlp": 1.03131425, + "epoch": 0.026609018629214786, + "flos": 48022892006400.0, + "grad_norm": 2.38139556877572, + "language_loss": 0.8010264, + "learning_rate": 3.930800002962473e-06, + "loss": 0.82331932, + "num_input_tokens_seen": 25062715, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.31970215, + "step": 917, + "time_per_iteration": 2.6266701221466064 + }, + { + "auxiliary_loss_clip": 0.01044075, + "auxiliary_loss_mlp": 0.01012067, + "balance_loss_clip": 1.01569653, + "balance_loss_mlp": 1.00927722, + "epoch": 0.026638036097730834, + "flos": 65325321083520.0, + "grad_norm": 0.7294554111252383, + "language_loss": 0.50641584, + "learning_rate": 3.931428090282013e-06, + "loss": 0.52697724, + "num_input_tokens_seen": 25125725, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.0279541, + "step": 918, + "time_per_iteration": 5.255882740020752 + }, + { + "auxiliary_loss_clip": 0.01042369, + "auxiliary_loss_mlp": 0.01004105, + "balance_loss_clip": 1.01377821, + "balance_loss_mlp": 1.00137556, + "epoch": 0.026667053566246882, + "flos": 74783571154560.0, + "grad_norm": 0.7036843858168963, + "language_loss": 0.55744064, + "learning_rate": 3.932055493782887e-06, + "loss": 0.57790542, + "num_input_tokens_seen": 25196390, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.02734375, + "step": 919, + "time_per_iteration": 3.190603256225586 + }, + { + "auxiliary_loss_clip": 0.01168166, + "auxiliary_loss_mlp": 0.0107671, + "balance_loss_clip": 1.04575217, + "balance_loss_mlp": 1.0484457, + "epoch": 0.026696071034762927, + "flos": 30185870325120.0, + "grad_norm": 2.693464365712973, + "language_loss": 0.95658827, + "learning_rate": 3.932682214952469e-06, + "loss": 0.97903699, + "num_input_tokens_seen": 25213175, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.28271484, + "step": 920, + "time_per_iteration": 4.909872531890869 + }, + { + "auxiliary_loss_clip": 0.01175992, + "auxiliary_loss_mlp": 0.01067615, + "balance_loss_clip": 1.04597926, + "balance_loss_mlp": 1.03594136, + "epoch": 0.026725088503278975, + "flos": 29049936462720.0, + "grad_norm": 2.970235374451667, + "language_loss": 0.93154216, + "learning_rate": 3.933308255273279e-06, + "loss": 0.9539783, + "num_input_tokens_seen": 25225940, + "router_z_loss_clip": 1.29980469, + "router_z_loss_mlp": 0.3170166, + "step": 921, + "time_per_iteration": 2.8139233589172363 + }, + { + "auxiliary_loss_clip": 0.01032367, + "auxiliary_loss_mlp": 0.01012221, + "balance_loss_clip": 1.00396788, + "balance_loss_mlp": 1.00935972, + "epoch": 0.02675410597179502, + "flos": 60906713546880.0, + "grad_norm": 0.6708319044586742, + "language_loss": 0.50229764, + "learning_rate": 3.933933616223017e-06, + "loss": 0.52274352, + "num_input_tokens_seen": 25287290, + "router_z_loss_clip": 0.28515625, + "router_z_loss_mlp": 0.02856445, + "step": 922, + "time_per_iteration": 3.0158791542053223 + }, + { + "auxiliary_loss_clip": 0.01170602, + "auxiliary_loss_mlp": 0.01073496, + "balance_loss_clip": 1.04499269, + "balance_loss_mlp": 1.04303849, + "epoch": 0.026783123440311068, + "flos": 24125925951360.0, + "grad_norm": 3.6479416404606604, + "language_loss": 0.94016552, + "learning_rate": 3.934558299274573e-06, + "loss": 0.96260655, + "num_input_tokens_seen": 25300365, + "router_z_loss_clip": 1.25585938, + "router_z_loss_mlp": 0.3046875, + "step": 923, + "time_per_iteration": 2.4699504375457764 + }, + { + "auxiliary_loss_clip": 0.0116922, + "auxiliary_loss_mlp": 0.0106257, + "balance_loss_clip": 1.04454374, + "balance_loss_mlp": 1.03404331, + "epoch": 0.026812140908827112, + "flos": 36566562159360.0, + "grad_norm": 2.1824375928492, + "language_loss": 0.98119545, + "learning_rate": 3.9351823058960555e-06, + "loss": 1.00351334, + "num_input_tokens_seen": 25321065, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.28527832, + "step": 924, + "time_per_iteration": 5.049455404281616 + }, + { + "auxiliary_loss_clip": 0.01168553, + "auxiliary_loss_mlp": 0.01055141, + "balance_loss_clip": 1.04305065, + "balance_loss_mlp": 1.0265311, + "epoch": 0.02684115837734316, + "flos": 14565868755840.0, + "grad_norm": 2.9501887362039665, + "language_loss": 0.76448023, + "learning_rate": 3.935805637550806e-06, + "loss": 0.78671718, + "num_input_tokens_seen": 25333200, + "router_z_loss_clip": 1.25585938, + "router_z_loss_mlp": 0.28637695, + "step": 925, + "time_per_iteration": 2.3536925315856934 + }, + { + "auxiliary_loss_clip": 0.01172484, + "auxiliary_loss_mlp": 0.01053933, + "balance_loss_clip": 1.0467031, + "balance_loss_mlp": 1.02303362, + "epoch": 0.02687017584585921, + "flos": 29781750804480.0, + "grad_norm": 5.352135191424085, + "language_loss": 0.89531946, + "learning_rate": 3.936428295697425e-06, + "loss": 0.9175837, + "num_input_tokens_seen": 25350220, + "router_z_loss_clip": 1.2578125, + "router_z_loss_mlp": 0.3092041, + "step": 926, + "time_per_iteration": 2.5082387924194336 + }, + { + "auxiliary_loss_clip": 0.01177068, + "auxiliary_loss_mlp": 0.01061861, + "balance_loss_clip": 1.04890728, + "balance_loss_mlp": 1.02934051, + "epoch": 0.026899193314375253, + "flos": 46928225237760.0, + "grad_norm": 3.1037326082869847, + "language_loss": 0.99163347, + "learning_rate": 3.937050281789788e-06, + "loss": 1.01402283, + "num_input_tokens_seen": 25366365, + "router_z_loss_clip": 1.28076172, + "router_z_loss_mlp": 0.32543945, + "step": 927, + "time_per_iteration": 2.5801522731781006 + }, + { + "auxiliary_loss_clip": 0.01177279, + "auxiliary_loss_mlp": 0.01067483, + "balance_loss_clip": 1.05048931, + "balance_loss_mlp": 1.0349865, + "epoch": 0.0269282107828913, + "flos": 27336224643840.0, + "grad_norm": 2.8738328034454805, + "language_loss": 0.98024684, + "learning_rate": 3.93767159727707e-06, + "loss": 1.00269437, + "num_input_tokens_seen": 25384070, + "router_z_loss_clip": 1.26660156, + "router_z_loss_mlp": 0.32495117, + "step": 928, + "time_per_iteration": 2.5751242637634277 + }, + { + "auxiliary_loss_clip": 0.01036414, + "auxiliary_loss_mlp": 0.01003211, + "balance_loss_clip": 1.00830674, + "balance_loss_mlp": 1.00037372, + "epoch": 0.026957228251407346, + "flos": 68712664613760.0, + "grad_norm": 0.6830942155868118, + "language_loss": 0.50654852, + "learning_rate": 3.938292243603762e-06, + "loss": 0.52694482, + "num_input_tokens_seen": 25446115, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.02832031, + "step": 929, + "time_per_iteration": 3.0315773487091064 + }, + { + "auxiliary_loss_clip": 0.01157634, + "auxiliary_loss_mlp": 0.01047837, + "balance_loss_clip": 1.03857529, + "balance_loss_mlp": 1.0199542, + "epoch": 0.026986245719923394, + "flos": 27551115262080.0, + "grad_norm": 3.4786169983028112, + "language_loss": 1.0284065, + "learning_rate": 3.938912222209695e-06, + "loss": 1.05046117, + "num_input_tokens_seen": 25457680, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.2791748, + "step": 930, + "time_per_iteration": 2.457000255584717 + }, + { + "auxiliary_loss_clip": 0.01033947, + "auxiliary_loss_mlp": 0.01004801, + "balance_loss_clip": 1.00601876, + "balance_loss_mlp": 1.00179684, + "epoch": 0.02701526318843944, + "flos": 59402760376320.0, + "grad_norm": 0.6793476679463405, + "language_loss": 0.50539029, + "learning_rate": 3.939531534530054e-06, + "loss": 0.52577776, + "num_input_tokens_seen": 25518550, + "router_z_loss_clip": 0.27929688, + "router_z_loss_mlp": 0.0300293, + "step": 931, + "time_per_iteration": 2.975346326828003 + }, + { + "auxiliary_loss_clip": 0.01165264, + "auxiliary_loss_mlp": 0.01054886, + "balance_loss_clip": 1.04548526, + "balance_loss_mlp": 1.02566767, + "epoch": 0.027044280656955487, + "flos": 27556735991040.0, + "grad_norm": 2.968523634741282, + "language_loss": 0.75271064, + "learning_rate": 3.9401501819954064e-06, + "loss": 0.77491212, + "num_input_tokens_seen": 25540010, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.2923584, + "step": 932, + "time_per_iteration": 2.5614216327667236 + }, + { + "auxiliary_loss_clip": 0.01178468, + "auxiliary_loss_mlp": 0.0107857, + "balance_loss_clip": 1.04626083, + "balance_loss_mlp": 1.04899454, + "epoch": 0.027073298125471535, + "flos": 31179532930560.0, + "grad_norm": 2.2948654719004615, + "language_loss": 0.89646095, + "learning_rate": 3.940768166031714e-06, + "loss": 0.91903138, + "num_input_tokens_seen": 25563520, + "router_z_loss_clip": 1.3203125, + "router_z_loss_mlp": 0.29577637, + "step": 933, + "time_per_iteration": 2.9592740535736084 + }, + { + "auxiliary_loss_clip": 0.01160102, + "auxiliary_loss_mlp": 0.01061411, + "balance_loss_clip": 1.04155779, + "balance_loss_mlp": 1.03454125, + "epoch": 0.02710231559398758, + "flos": 38318852897280.0, + "grad_norm": 2.495592947655333, + "language_loss": 0.93804073, + "learning_rate": 3.941385488060358e-06, + "loss": 0.96025586, + "num_input_tokens_seen": 25583720, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.26879883, + "step": 934, + "time_per_iteration": 2.6003189086914062 + }, + { + "auxiliary_loss_clip": 0.01164528, + "auxiliary_loss_mlp": 0.01057845, + "balance_loss_clip": 1.04401183, + "balance_loss_mlp": 1.02841234, + "epoch": 0.027131333062503628, + "flos": 36651121205760.0, + "grad_norm": 2.2494192643561433, + "language_loss": 0.94461906, + "learning_rate": 3.942002149498154e-06, + "loss": 0.96684277, + "num_input_tokens_seen": 25606675, + "router_z_loss_clip": 1.20507812, + "router_z_loss_mlp": 0.29455566, + "step": 935, + "time_per_iteration": 2.6517412662506104 + }, + { + "auxiliary_loss_clip": 0.01033595, + "auxiliary_loss_mlp": 0.01006384, + "balance_loss_clip": 1.0054245, + "balance_loss_mlp": 1.00351155, + "epoch": 0.027160350531019672, + "flos": 68554403953920.0, + "grad_norm": 0.7532876886689482, + "language_loss": 0.52390242, + "learning_rate": 3.9426181517573775e-06, + "loss": 0.54430223, + "num_input_tokens_seen": 25658530, + "router_z_loss_clip": 0.28125, + "router_z_loss_mlp": 0.02868652, + "step": 936, + "time_per_iteration": 2.8410050868988037 + }, + { + "auxiliary_loss_clip": 0.01033244, + "auxiliary_loss_mlp": 0.01002911, + "balance_loss_clip": 1.00541592, + "balance_loss_mlp": 1.00021696, + "epoch": 0.02718936799953572, + "flos": 74779311968640.0, + "grad_norm": 0.6328149636268318, + "language_loss": 0.47309268, + "learning_rate": 3.943233496245778e-06, + "loss": 0.49345422, + "num_input_tokens_seen": 25724825, + "router_z_loss_clip": 0.27734375, + "router_z_loss_mlp": 0.02697754, + "step": 937, + "time_per_iteration": 3.153043508529663 + }, + { + "auxiliary_loss_clip": 0.01166627, + "auxiliary_loss_mlp": 0.01055019, + "balance_loss_clip": 1.04483843, + "balance_loss_mlp": 1.02605104, + "epoch": 0.02721838546805177, + "flos": 29274288059520.0, + "grad_norm": 2.6782272573802137, + "language_loss": 0.95452821, + "learning_rate": 3.943848184366598e-06, + "loss": 0.97674465, + "num_input_tokens_seen": 25741665, + "router_z_loss_clip": 1.21728516, + "router_z_loss_mlp": 0.28991699, + "step": 938, + "time_per_iteration": 2.475269079208374 + }, + { + "auxiliary_loss_clip": 0.01182413, + "auxiliary_loss_mlp": 0.01086492, + "balance_loss_clip": 1.05090439, + "balance_loss_mlp": 1.05139732, + "epoch": 0.027247402936567813, + "flos": 36969878718720.0, + "grad_norm": 2.299081484096085, + "language_loss": 0.95266002, + "learning_rate": 3.9444622175186e-06, + "loss": 0.97534907, + "num_input_tokens_seen": 25764955, + "router_z_loss_clip": 1.31445312, + "router_z_loss_mlp": 0.35083008, + "step": 939, + "time_per_iteration": 2.6384856700897217 + }, + { + "auxiliary_loss_clip": 0.01164256, + "auxiliary_loss_mlp": 0.01061743, + "balance_loss_clip": 1.04355299, + "balance_loss_mlp": 1.03016496, + "epoch": 0.02727642040508386, + "flos": 15992000772480.0, + "grad_norm": 2.408476205779271, + "language_loss": 0.79919261, + "learning_rate": 3.945075597096074e-06, + "loss": 0.82145262, + "num_input_tokens_seen": 25778015, + "router_z_loss_clip": 1.20507812, + "router_z_loss_mlp": 0.31616211, + "step": 940, + "time_per_iteration": 2.47756290435791 + }, + { + "auxiliary_loss_clip": 0.01166811, + "auxiliary_loss_mlp": 0.01059145, + "balance_loss_clip": 1.04813933, + "balance_loss_mlp": 1.03071392, + "epoch": 0.027305437873599906, + "flos": 29895776904960.0, + "grad_norm": 2.1435462622137225, + "language_loss": 0.9374302, + "learning_rate": 3.945688324488866e-06, + "loss": 0.95968974, + "num_input_tokens_seen": 25793635, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.2845459, + "step": 941, + "time_per_iteration": 2.485368251800537 + }, + { + "auxiliary_loss_clip": 0.01031652, + "auxiliary_loss_mlp": 0.01003528, + "balance_loss_clip": 1.00417221, + "balance_loss_mlp": 1.00088155, + "epoch": 0.027334455342115954, + "flos": 66053923580160.0, + "grad_norm": 0.7294015365187208, + "language_loss": 0.51287961, + "learning_rate": 3.946300401082393e-06, + "loss": 0.53323138, + "num_input_tokens_seen": 25852085, + "router_z_loss_clip": 0.27539062, + "router_z_loss_mlp": 0.02648926, + "step": 942, + "time_per_iteration": 3.1010611057281494 + }, + { + "auxiliary_loss_clip": 0.01181392, + "auxiliary_loss_mlp": 0.01071222, + "balance_loss_clip": 1.05006647, + "balance_loss_mlp": 1.03512514, + "epoch": 0.027363472810632, + "flos": 30919988816640.0, + "grad_norm": 2.264440978362155, + "language_loss": 1.11449909, + "learning_rate": 3.946911828257664e-06, + "loss": 1.13702524, + "num_input_tokens_seen": 25873450, + "router_z_loss_clip": 1.31347656, + "router_z_loss_mlp": 0.36083984, + "step": 943, + "time_per_iteration": 2.5827231407165527 + }, + { + "auxiliary_loss_clip": 0.01031022, + "auxiliary_loss_mlp": 0.01003094, + "balance_loss_clip": 1.00393879, + "balance_loss_mlp": 1.00042343, + "epoch": 0.027392490279148047, + "flos": 60275798634240.0, + "grad_norm": 0.7385354545727802, + "language_loss": 0.50412744, + "learning_rate": 3.94752260739129e-06, + "loss": 0.5244686, + "num_input_tokens_seen": 25929255, + "router_z_loss_clip": 0.27148438, + "router_z_loss_mlp": 0.0267334, + "step": 944, + "time_per_iteration": 2.94856333732605 + }, + { + "auxiliary_loss_clip": 0.01165527, + "auxiliary_loss_mlp": 0.01062287, + "balance_loss_clip": 1.0436306, + "balance_loss_mlp": 1.03436804, + "epoch": 0.027421507747664095, + "flos": 16790709012480.0, + "grad_norm": 3.4488520082081826, + "language_loss": 0.81100857, + "learning_rate": 3.9481327398555175e-06, + "loss": 0.8332867, + "num_input_tokens_seen": 25940965, + "router_z_loss_clip": 1.21777344, + "router_z_loss_mlp": 0.27954102, + "step": 945, + "time_per_iteration": 2.3961470127105713 + }, + { + "auxiliary_loss_clip": 0.01030632, + "auxiliary_loss_mlp": 0.01003295, + "balance_loss_clip": 1.00332713, + "balance_loss_mlp": 1.00077963, + "epoch": 0.02745052521618014, + "flos": 61859946931200.0, + "grad_norm": 0.661635894973486, + "language_loss": 0.48122483, + "learning_rate": 3.948742227018233e-06, + "loss": 0.50156409, + "num_input_tokens_seen": 26004465, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.02514648, + "step": 946, + "time_per_iteration": 3.310753345489502 + }, + { + "auxiliary_loss_clip": 0.01180483, + "auxiliary_loss_mlp": 0.01070272, + "balance_loss_clip": 1.04832196, + "balance_loss_mlp": 1.03722751, + "epoch": 0.027479542684696188, + "flos": 27923428667520.0, + "grad_norm": 3.7136101491062807, + "language_loss": 0.98913318, + "learning_rate": 3.949351070242994e-06, + "loss": 1.01164079, + "num_input_tokens_seen": 26025085, + "router_z_loss_clip": 1.32226562, + "router_z_loss_mlp": 0.3302002, + "step": 947, + "time_per_iteration": 2.4809274673461914 + }, + { + "auxiliary_loss_clip": 0.01181654, + "auxiliary_loss_mlp": 0.01076903, + "balance_loss_clip": 1.0501349, + "balance_loss_mlp": 1.0427016, + "epoch": 0.027508560153212232, + "flos": 21939490056960.0, + "grad_norm": 2.6111553730598778, + "language_loss": 1.00055289, + "learning_rate": 3.949959270889033e-06, + "loss": 1.02313864, + "num_input_tokens_seen": 26041380, + "router_z_loss_clip": 1.31494141, + "router_z_loss_mlp": 0.34204102, + "step": 948, + "time_per_iteration": 2.4599039554595947 + }, + { + "auxiliary_loss_clip": 0.01177254, + "auxiliary_loss_mlp": 0.01063241, + "balance_loss_clip": 1.04783463, + "balance_loss_mlp": 1.03249657, + "epoch": 0.02753757762172828, + "flos": 57912808527360.0, + "grad_norm": 2.135281466668814, + "language_loss": 0.80670738, + "learning_rate": 3.950566830311289e-06, + "loss": 0.82911229, + "num_input_tokens_seen": 26063680, + "router_z_loss_clip": 1.29492188, + "router_z_loss_mlp": 0.30712891, + "step": 949, + "time_per_iteration": 2.7377853393554688 + }, + { + "auxiliary_loss_clip": 0.01167866, + "auxiliary_loss_mlp": 0.0105665, + "balance_loss_clip": 1.04382491, + "balance_loss_mlp": 1.02651405, + "epoch": 0.02756659509024433, + "flos": 26607971260800.0, + "grad_norm": 2.49314665196564, + "language_loss": 0.93057799, + "learning_rate": 3.951173749860417e-06, + "loss": 0.95282322, + "num_input_tokens_seen": 26080575, + "router_z_loss_clip": 1.24121094, + "router_z_loss_mlp": 0.30126953, + "step": 950, + "time_per_iteration": 2.5342791080474854 + }, + { + "auxiliary_loss_clip": 0.01031704, + "auxiliary_loss_mlp": 0.01008812, + "balance_loss_clip": 1.004493, + "balance_loss_mlp": 1.00636816, + "epoch": 0.027595612558760373, + "flos": 54234636572160.0, + "grad_norm": 0.6893653204713445, + "language_loss": 0.50807911, + "learning_rate": 3.9517800308828105e-06, + "loss": 0.52848428, + "num_input_tokens_seen": 26141950, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.02441406, + "step": 951, + "time_per_iteration": 3.0194544792175293 + }, + { + "auxiliary_loss_clip": 0.01030406, + "auxiliary_loss_mlp": 0.01004567, + "balance_loss_clip": 1.00342214, + "balance_loss_mlp": 1.00192106, + "epoch": 0.02762463002727642, + "flos": 60061710977280.0, + "grad_norm": 0.7249642326576767, + "language_loss": 0.49023309, + "learning_rate": 3.9523856747206175e-06, + "loss": 0.5105828, + "num_input_tokens_seen": 26193555, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.02648926, + "step": 952, + "time_per_iteration": 2.8124594688415527 + }, + { + "auxiliary_loss_clip": 0.01164939, + "auxiliary_loss_mlp": 0.01068995, + "balance_loss_clip": 1.04288769, + "balance_loss_mlp": 1.03891897, + "epoch": 0.027653647495792466, + "flos": 25629948944640.0, + "grad_norm": 2.7716627516671912, + "language_loss": 1.00509143, + "learning_rate": 3.952990682711758e-06, + "loss": 1.02743077, + "num_input_tokens_seen": 26210310, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.30065918, + "step": 953, + "time_per_iteration": 2.433802843093872 + }, + { + "auxiliary_loss_clip": 0.01028622, + "auxiliary_loss_mlp": 0.0100256, + "balance_loss_clip": 1.0019834, + "balance_loss_mlp": 0.99994892, + "epoch": 0.027682664964308514, + "flos": 60803440145280.0, + "grad_norm": 0.7364777252711938, + "language_loss": 0.51536751, + "learning_rate": 3.953595056189946e-06, + "loss": 0.53567934, + "num_input_tokens_seen": 26275435, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.02612305, + "step": 954, + "time_per_iteration": 3.097292423248291 + }, + { + "auxiliary_loss_clip": 0.01172354, + "auxiliary_loss_mlp": 0.01065561, + "balance_loss_clip": 1.04648769, + "balance_loss_mlp": 1.03347015, + "epoch": 0.02771168243282456, + "flos": 28066258506240.0, + "grad_norm": 4.904361965206708, + "language_loss": 1.02472651, + "learning_rate": 3.954198796484698e-06, + "loss": 1.04710579, + "num_input_tokens_seen": 26290105, + "router_z_loss_clip": 1.2578125, + "router_z_loss_mlp": 0.32092285, + "step": 955, + "time_per_iteration": 2.4466965198516846 + }, + { + "auxiliary_loss_clip": 0.01029342, + "auxiliary_loss_mlp": 0.0100338, + "balance_loss_clip": 1.00287199, + "balance_loss_mlp": 1.00076962, + "epoch": 0.027740699901340607, + "flos": 58573153716480.0, + "grad_norm": 0.7071622036111925, + "language_loss": 0.54787421, + "learning_rate": 3.954801904921359e-06, + "loss": 0.56820142, + "num_input_tokens_seen": 26348590, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.02612305, + "step": 956, + "time_per_iteration": 2.9319655895233154 + }, + { + "auxiliary_loss_clip": 0.01028629, + "auxiliary_loss_mlp": 0.01004701, + "balance_loss_clip": 1.00229847, + "balance_loss_mlp": 1.00201893, + "epoch": 0.027769717369856655, + "flos": 60124553291520.0, + "grad_norm": 0.6298524585185243, + "language_loss": 0.47238225, + "learning_rate": 3.955404382821119e-06, + "loss": 0.49271554, + "num_input_tokens_seen": 26411455, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.02685547, + "step": 957, + "time_per_iteration": 3.12951922416687 + }, + { + "auxiliary_loss_clip": 0.01027142, + "auxiliary_loss_mlp": 0.01002203, + "balance_loss_clip": 1.00103199, + "balance_loss_mlp": 0.99964035, + "epoch": 0.0277987348383727, + "flos": 70025154554880.0, + "grad_norm": 0.6849794251514925, + "language_loss": 0.5034281, + "learning_rate": 3.956006231501026e-06, + "loss": 0.52372158, + "num_input_tokens_seen": 26472470, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.02563477, + "step": 958, + "time_per_iteration": 3.1038241386413574 + }, + { + "auxiliary_loss_clip": 0.01172694, + "auxiliary_loss_mlp": 0.01070002, + "balance_loss_clip": 1.04508209, + "balance_loss_mlp": 1.03752911, + "epoch": 0.027827752306888748, + "flos": 44520579768960.0, + "grad_norm": 4.425769755037942, + "language_loss": 1.05256581, + "learning_rate": 3.9566074522740066e-06, + "loss": 1.07499278, + "num_input_tokens_seen": 26493700, + "router_z_loss_clip": 1.27636719, + "router_z_loss_mlp": 0.32470703, + "step": 959, + "time_per_iteration": 2.693526268005371 + }, + { + "auxiliary_loss_clip": 0.01027759, + "auxiliary_loss_mlp": 0.01006414, + "balance_loss_clip": 1.00219703, + "balance_loss_mlp": 1.00388682, + "epoch": 0.027856769775404792, + "flos": 74768524358400.0, + "grad_norm": 0.7963941463704813, + "language_loss": 0.55443323, + "learning_rate": 3.9572080464488815e-06, + "loss": 0.57477498, + "num_input_tokens_seen": 26551680, + "router_z_loss_clip": 0.25585938, + "router_z_loss_mlp": 0.02526855, + "step": 960, + "time_per_iteration": 3.041386604309082 + }, + { + "auxiliary_loss_clip": 0.0117738, + "auxiliary_loss_mlp": 0.01065105, + "balance_loss_clip": 1.04896867, + "balance_loss_mlp": 1.03062963, + "epoch": 0.02788578724392084, + "flos": 12961225624320.0, + "grad_norm": 5.683603011236205, + "language_loss": 0.81540442, + "learning_rate": 3.957808015330385e-06, + "loss": 0.83782923, + "num_input_tokens_seen": 26564550, + "router_z_loss_clip": 1.28417969, + "router_z_loss_mlp": 0.34448242, + "step": 961, + "time_per_iteration": 2.4250028133392334 + }, + { + "auxiliary_loss_clip": 0.01172641, + "auxiliary_loss_mlp": 0.01056277, + "balance_loss_clip": 1.0473969, + "balance_loss_mlp": 1.02447152, + "epoch": 0.02791480471243689, + "flos": 43353398373120.0, + "grad_norm": 2.7798483828409006, + "language_loss": 0.82067478, + "learning_rate": 3.958407360219179e-06, + "loss": 0.84296393, + "num_input_tokens_seen": 26580135, + "router_z_loss_clip": 1.25195312, + "router_z_loss_mlp": 0.31799316, + "step": 962, + "time_per_iteration": 2.5521903038024902 + }, + { + "auxiliary_loss_clip": 0.01183901, + "auxiliary_loss_mlp": 0.01063269, + "balance_loss_clip": 1.0501579, + "balance_loss_mlp": 1.03115368, + "epoch": 0.027943822180952933, + "flos": 21134183569920.0, + "grad_norm": 9.807408028481339, + "language_loss": 1.05200231, + "learning_rate": 3.9590060824118735e-06, + "loss": 1.07447398, + "num_input_tokens_seen": 26592885, + "router_z_loss_clip": 1.33691406, + "router_z_loss_mlp": 0.32128906, + "step": 963, + "time_per_iteration": 2.5406062602996826 + }, + { + "auxiliary_loss_clip": 0.01029088, + "auxiliary_loss_mlp": 0.01016126, + "balance_loss_clip": 1.00304556, + "balance_loss_mlp": 1.01351571, + "epoch": 0.02797283964946898, + "flos": 66930452974080.0, + "grad_norm": 0.6632323282311821, + "language_loss": 0.54601729, + "learning_rate": 3.959604183201038e-06, + "loss": 0.56646943, + "num_input_tokens_seen": 26656860, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.02612305, + "step": 964, + "time_per_iteration": 3.0065438747406006 + }, + { + "auxiliary_loss_clip": 0.01168576, + "auxiliary_loss_mlp": 0.01061676, + "balance_loss_clip": 1.04498112, + "balance_loss_mlp": 1.02841687, + "epoch": 0.028001857117985026, + "flos": 29488201159680.0, + "grad_norm": 2.788100083376861, + "language_loss": 0.85562062, + "learning_rate": 3.960201663875225e-06, + "loss": 0.87792313, + "num_input_tokens_seen": 26671265, + "router_z_loss_clip": 1.23632812, + "router_z_loss_mlp": 0.33300781, + "step": 965, + "time_per_iteration": 2.4763615131378174 + }, + { + "auxiliary_loss_clip": 0.01171796, + "auxiliary_loss_mlp": 0.0106082, + "balance_loss_clip": 1.04899263, + "balance_loss_mlp": 1.02952778, + "epoch": 0.028030874586501074, + "flos": 10661182565760.0, + "grad_norm": 3.149883213176545, + "language_loss": 0.84005201, + "learning_rate": 3.960798525718981e-06, + "loss": 0.86237812, + "num_input_tokens_seen": 26681225, + "router_z_loss_clip": 1.22753906, + "router_z_loss_mlp": 0.31237793, + "step": 966, + "time_per_iteration": 2.3816356658935547 + }, + { + "auxiliary_loss_clip": 0.01166381, + "auxiliary_loss_mlp": 0.01056696, + "balance_loss_clip": 1.04733086, + "balance_loss_mlp": 1.02692938, + "epoch": 0.02805989205501712, + "flos": 12341517258240.0, + "grad_norm": 2.771194595231159, + "language_loss": 0.91253632, + "learning_rate": 3.961394770012866e-06, + "loss": 0.93476707, + "num_input_tokens_seen": 26693995, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.29785156, + "step": 967, + "time_per_iteration": 2.3631699085235596 + }, + { + "auxiliary_loss_clip": 0.01169786, + "auxiliary_loss_mlp": 0.01056615, + "balance_loss_clip": 1.04651809, + "balance_loss_mlp": 1.02583528, + "epoch": 0.028088909523533167, + "flos": 25731511689600.0, + "grad_norm": 2.4495344425821024, + "language_loss": 0.95734352, + "learning_rate": 3.9619903980334684e-06, + "loss": 0.97960758, + "num_input_tokens_seen": 26707370, + "router_z_loss_clip": 1.23388672, + "router_z_loss_mlp": 0.30773926, + "step": 968, + "time_per_iteration": 2.5484917163848877 + }, + { + "auxiliary_loss_clip": 0.01166464, + "auxiliary_loss_mlp": 0.01068509, + "balance_loss_clip": 1.04610074, + "balance_loss_mlp": 1.03787255, + "epoch": 0.028117926992049215, + "flos": 25840754933760.0, + "grad_norm": 2.2886236681293584, + "language_loss": 0.85193396, + "learning_rate": 3.9625854110534254e-06, + "loss": 0.87428367, + "num_input_tokens_seen": 26722675, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.30627441, + "step": 969, + "time_per_iteration": 2.5567264556884766 + }, + { + "auxiliary_loss_clip": 0.0102983, + "auxiliary_loss_mlp": 0.0100255, + "balance_loss_clip": 1.00389814, + "balance_loss_mlp": 1.00004709, + "epoch": 0.02814694446056526, + "flos": 68555346560640.0, + "grad_norm": 0.7126656887253113, + "language_loss": 0.56993365, + "learning_rate": 3.963179810341432e-06, + "loss": 0.59025753, + "num_input_tokens_seen": 26781100, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.02502441, + "step": 970, + "time_per_iteration": 2.952439308166504 + }, + { + "auxiliary_loss_clip": 0.01184507, + "auxiliary_loss_mlp": 0.01067103, + "balance_loss_clip": 1.0529294, + "balance_loss_mlp": 1.03482151, + "epoch": 0.028175961929081308, + "flos": 35435865000960.0, + "grad_norm": 4.014068448519076, + "language_loss": 0.88846326, + "learning_rate": 3.9637735971622635e-06, + "loss": 0.91097939, + "num_input_tokens_seen": 26796080, + "router_z_loss_clip": 1.31640625, + "router_z_loss_mlp": 0.32299805, + "step": 971, + "time_per_iteration": 2.5423710346221924 + }, + { + "auxiliary_loss_clip": 0.01162317, + "auxiliary_loss_mlp": 0.01050445, + "balance_loss_clip": 1.04528272, + "balance_loss_mlp": 1.0235877, + "epoch": 0.028204979397597352, + "flos": 30655836403200.0, + "grad_norm": 2.928543515222659, + "language_loss": 0.95089996, + "learning_rate": 3.964366772776789e-06, + "loss": 0.97302759, + "num_input_tokens_seen": 26812295, + "router_z_loss_clip": 1.17041016, + "router_z_loss_mlp": 0.26867676, + "step": 972, + "time_per_iteration": 2.733560085296631 + }, + { + "auxiliary_loss_clip": 0.01172497, + "auxiliary_loss_mlp": 0.01056907, + "balance_loss_clip": 1.04395545, + "balance_loss_mlp": 1.02659202, + "epoch": 0.0282339968661134, + "flos": 23290105069440.0, + "grad_norm": 2.9362894274071745, + "language_loss": 0.85963964, + "learning_rate": 3.964959338441989e-06, + "loss": 0.88193363, + "num_input_tokens_seen": 26828600, + "router_z_loss_clip": 1.28515625, + "router_z_loss_mlp": 0.3034668, + "step": 973, + "time_per_iteration": 2.4122250080108643 + }, + { + "auxiliary_loss_clip": 0.01027253, + "auxiliary_loss_mlp": 0.01003582, + "balance_loss_clip": 1.00128531, + "balance_loss_mlp": 1.00112653, + "epoch": 0.02826301433462945, + "flos": 68283721249920.0, + "grad_norm": 0.6348914110161605, + "language_loss": 0.52070558, + "learning_rate": 3.96555129541097e-06, + "loss": 0.5410139, + "num_input_tokens_seen": 26896545, + "router_z_loss_clip": 0.25976562, + "router_z_loss_mlp": 0.02453613, + "step": 974, + "time_per_iteration": 3.117342948913574 + }, + { + "auxiliary_loss_clip": 0.01178567, + "auxiliary_loss_mlp": 0.01060532, + "balance_loss_clip": 1.04746974, + "balance_loss_mlp": 1.03001499, + "epoch": 0.028292031803145493, + "flos": 12486895626240.0, + "grad_norm": 2.8103061234245605, + "language_loss": 0.83185399, + "learning_rate": 3.9661426449329815e-06, + "loss": 0.85424501, + "num_input_tokens_seen": 26908390, + "router_z_loss_clip": 1.31054688, + "router_z_loss_mlp": 0.30517578, + "step": 975, + "time_per_iteration": 2.3685081005096436 + }, + { + "auxiliary_loss_clip": 0.01173254, + "auxiliary_loss_mlp": 0.01055617, + "balance_loss_clip": 1.04495955, + "balance_loss_mlp": 1.02524257, + "epoch": 0.02832104927166154, + "flos": 40361795637120.0, + "grad_norm": 2.8521442168391724, + "language_loss": 1.10078394, + "learning_rate": 3.966733388253427e-06, + "loss": 1.12307262, + "num_input_tokens_seen": 26926780, + "router_z_loss_clip": 1.28369141, + "router_z_loss_mlp": 0.30395508, + "step": 976, + "time_per_iteration": 2.6454272270202637 + }, + { + "auxiliary_loss_clip": 0.01171683, + "auxiliary_loss_mlp": 0.01052804, + "balance_loss_clip": 1.05251646, + "balance_loss_mlp": 1.02165508, + "epoch": 0.028350066740177586, + "flos": 16498765290240.0, + "grad_norm": 10.338437309911123, + "language_loss": 0.93625677, + "learning_rate": 3.967323526613891e-06, + "loss": 0.95850164, + "num_input_tokens_seen": 26937800, + "router_z_loss_clip": 1.19042969, + "router_z_loss_mlp": 0.31115723, + "step": 977, + "time_per_iteration": 2.389937162399292 + }, + { + "auxiliary_loss_clip": 0.0102943, + "auxiliary_loss_mlp": 0.0100932, + "balance_loss_clip": 1.00268197, + "balance_loss_mlp": 1.00675738, + "epoch": 0.028379084208693634, + "flos": 63289709729280.0, + "grad_norm": 0.6590942574701915, + "language_loss": 0.49603868, + "learning_rate": 3.967913061252141e-06, + "loss": 0.51642621, + "num_input_tokens_seen": 27003420, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.02563477, + "step": 978, + "time_per_iteration": 3.104931354522705 + }, + { + "auxiliary_loss_clip": 0.0116555, + "auxiliary_loss_mlp": 0.01051975, + "balance_loss_clip": 1.04775214, + "balance_loss_mlp": 1.02312624, + "epoch": 0.02840810167720968, + "flos": 20113392971520.0, + "grad_norm": 3.0065858478441996, + "language_loss": 0.77947652, + "learning_rate": 3.968501993402152e-06, + "loss": 0.80165178, + "num_input_tokens_seen": 27017875, + "router_z_loss_clip": 1.17773438, + "router_z_loss_mlp": 0.28845215, + "step": 979, + "time_per_iteration": 2.3655550479888916 + }, + { + "auxiliary_loss_clip": 0.01028397, + "auxiliary_loss_mlp": 0.01006551, + "balance_loss_clip": 1.00188017, + "balance_loss_mlp": 1.00398767, + "epoch": 0.028437119145725727, + "flos": 74776903084800.0, + "grad_norm": 1.2471809672900733, + "language_loss": 0.59960157, + "learning_rate": 3.969090324294122e-06, + "loss": 0.61995101, + "num_input_tokens_seen": 27079345, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.02563477, + "step": 980, + "time_per_iteration": 3.0933330059051514 + }, + { + "auxiliary_loss_clip": 0.01181683, + "auxiliary_loss_mlp": 0.01087327, + "balance_loss_clip": 1.05266285, + "balance_loss_mlp": 1.05623746, + "epoch": 0.028466136614241775, + "flos": 26065420732800.0, + "grad_norm": 2.9495327538381626, + "language_loss": 0.69747335, + "learning_rate": 3.969678055154481e-06, + "loss": 0.72016346, + "num_input_tokens_seen": 27093595, + "router_z_loss_clip": 1.29052734, + "router_z_loss_mlp": 0.31066895, + "step": 981, + "time_per_iteration": 2.481497049331665 + }, + { + "auxiliary_loss_clip": 0.0102826, + "auxiliary_loss_mlp": 0.01002889, + "balance_loss_clip": 1.00172222, + "balance_loss_mlp": 1.00037396, + "epoch": 0.02849515408275782, + "flos": 72034545611520.0, + "grad_norm": 0.6910634964792011, + "language_loss": 0.54725945, + "learning_rate": 3.970265187205913e-06, + "loss": 0.56757092, + "num_input_tokens_seen": 27154075, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.02514648, + "step": 982, + "time_per_iteration": 3.026261329650879 + }, + { + "auxiliary_loss_clip": 0.01027881, + "auxiliary_loss_mlp": 0.01002974, + "balance_loss_clip": 1.00184369, + "balance_loss_mlp": 1.00058937, + "epoch": 0.028524171551273868, + "flos": 56896135603200.0, + "grad_norm": 0.8290524299774974, + "language_loss": 0.52165747, + "learning_rate": 3.970851721667367e-06, + "loss": 0.54196596, + "num_input_tokens_seen": 27206905, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.02380371, + "step": 983, + "time_per_iteration": 2.9477484226226807 + }, + { + "auxiliary_loss_clip": 0.01028076, + "auxiliary_loss_mlp": 0.01004332, + "balance_loss_clip": 1.00193429, + "balance_loss_mlp": 1.0019362, + "epoch": 0.028553189019789912, + "flos": 68607645644160.0, + "grad_norm": 0.7606923092027195, + "language_loss": 0.56102645, + "learning_rate": 3.971437659754076e-06, + "loss": 0.58135056, + "num_input_tokens_seen": 27270340, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.02392578, + "step": 984, + "time_per_iteration": 3.431955099105835 + }, + { + "auxiliary_loss_clip": 0.0115998, + "auxiliary_loss_mlp": 0.01058816, + "balance_loss_clip": 1.04519176, + "balance_loss_mlp": 1.03287578, + "epoch": 0.02858220648830596, + "flos": 45505200332160.0, + "grad_norm": 2.509091504812241, + "language_loss": 0.82068598, + "learning_rate": 3.9720230026775675e-06, + "loss": 0.84287393, + "num_input_tokens_seen": 27287480, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.25939941, + "step": 985, + "time_per_iteration": 2.6454062461853027 + }, + { + "auxiliary_loss_clip": 0.01176556, + "auxiliary_loss_mlp": 0.0106716, + "balance_loss_clip": 1.04560149, + "balance_loss_mlp": 1.03638065, + "epoch": 0.028611223956822005, + "flos": 25804759455360.0, + "grad_norm": 3.202875328198519, + "language_loss": 1.05768073, + "learning_rate": 3.972607751645682e-06, + "loss": 1.08011794, + "num_input_tokens_seen": 27302950, + "router_z_loss_clip": 1.30957031, + "router_z_loss_mlp": 0.30773926, + "step": 986, + "time_per_iteration": 2.44553804397583 + }, + { + "auxiliary_loss_clip": 0.01158845, + "auxiliary_loss_mlp": 0.0104763, + "balance_loss_clip": 1.04449081, + "balance_loss_mlp": 1.02222669, + "epoch": 0.028640241425338053, + "flos": 11319365116800.0, + "grad_norm": 3.7013288138733422, + "language_loss": 1.04670954, + "learning_rate": 3.973191907862586e-06, + "loss": 1.06877434, + "num_input_tokens_seen": 27314795, + "router_z_loss_clip": 1.14453125, + "router_z_loss_mlp": 0.25415039, + "step": 987, + "time_per_iteration": 2.4417266845703125 + }, + { + "auxiliary_loss_clip": 0.01174614, + "auxiliary_loss_mlp": 0.01060203, + "balance_loss_clip": 1.0488534, + "balance_loss_mlp": 1.02945936, + "epoch": 0.0286692588938541, + "flos": 27665350830720.0, + "grad_norm": 2.031338496457521, + "language_loss": 0.8614921, + "learning_rate": 3.973775472528791e-06, + "loss": 0.88384026, + "num_input_tokens_seen": 27331450, + "router_z_loss_clip": 1.2578125, + "router_z_loss_mlp": 0.30761719, + "step": 988, + "time_per_iteration": 2.4774434566497803 + }, + { + "auxiliary_loss_clip": 0.0102785, + "auxiliary_loss_mlp": 0.01001437, + "balance_loss_clip": 1.00192988, + "balance_loss_mlp": 0.99895757, + "epoch": 0.028698276362370146, + "flos": 61234338545280.0, + "grad_norm": 0.7210565317639822, + "language_loss": 0.52431738, + "learning_rate": 3.9743584468411595e-06, + "loss": 0.54461026, + "num_input_tokens_seen": 27393880, + "router_z_loss_clip": 0.25976562, + "router_z_loss_mlp": 0.02478027, + "step": 989, + "time_per_iteration": 3.0169014930725098 + }, + { + "auxiliary_loss_clip": 0.01171676, + "auxiliary_loss_mlp": 0.01063444, + "balance_loss_clip": 1.04536915, + "balance_loss_mlp": 1.03233063, + "epoch": 0.028727293830886194, + "flos": 32701327672320.0, + "grad_norm": 2.3061690797874923, + "language_loss": 0.82020211, + "learning_rate": 3.97494083199293e-06, + "loss": 0.84255332, + "num_input_tokens_seen": 27413805, + "router_z_loss_clip": 1.26269531, + "router_z_loss_mlp": 0.31140137, + "step": 990, + "time_per_iteration": 4.9777326583862305 + }, + { + "auxiliary_loss_clip": 0.01028078, + "auxiliary_loss_mlp": 0.01002054, + "balance_loss_clip": 1.00212026, + "balance_loss_mlp": 0.99956226, + "epoch": 0.02875631129940224, + "flos": 62404138293120.0, + "grad_norm": 0.6705322075328143, + "language_loss": 0.51955926, + "learning_rate": 3.975522629173727e-06, + "loss": 0.53986061, + "num_input_tokens_seen": 27478830, + "router_z_loss_clip": 0.25976562, + "router_z_loss_mlp": 0.02490234, + "step": 991, + "time_per_iteration": 3.0212056636810303 + }, + { + "auxiliary_loss_clip": 0.01168444, + "auxiliary_loss_mlp": 0.01049493, + "balance_loss_clip": 1.04614782, + "balance_loss_mlp": 1.02309978, + "epoch": 0.028785328767918287, + "flos": 61630988901120.0, + "grad_norm": 2.810515333584412, + "language_loss": 0.7612952, + "learning_rate": 3.976103839569571e-06, + "loss": 0.78347456, + "num_input_tokens_seen": 27499010, + "router_z_loss_clip": 1.22167969, + "router_z_loss_mlp": 0.26367188, + "step": 992, + "time_per_iteration": 2.6815061569213867 + }, + { + "auxiliary_loss_clip": 0.0116117, + "auxiliary_loss_mlp": 0.01055199, + "balance_loss_clip": 1.04387856, + "balance_loss_mlp": 1.02675533, + "epoch": 0.028814346236434335, + "flos": 18841157694720.0, + "grad_norm": 3.0321275710630475, + "language_loss": 0.76717961, + "learning_rate": 3.976684464362904e-06, + "loss": 0.78934324, + "num_input_tokens_seen": 27516385, + "router_z_loss_clip": 1.17333984, + "router_z_loss_mlp": 0.28442383, + "step": 993, + "time_per_iteration": 2.4962494373321533 + }, + { + "auxiliary_loss_clip": 0.01169844, + "auxiliary_loss_mlp": 0.01087148, + "balance_loss_clip": 1.04819393, + "balance_loss_mlp": 1.05514038, + "epoch": 0.02884336370495038, + "flos": 41969685525120.0, + "grad_norm": 2.5018292677264857, + "language_loss": 0.9180845, + "learning_rate": 3.9772645047325895e-06, + "loss": 0.9406544, + "num_input_tokens_seen": 27534935, + "router_z_loss_clip": 1.21777344, + "router_z_loss_mlp": 0.31994629, + "step": 994, + "time_per_iteration": 4.799519300460815 + }, + { + "auxiliary_loss_clip": 0.0115663, + "auxiliary_loss_mlp": 0.01054676, + "balance_loss_clip": 1.0409317, + "balance_loss_mlp": 1.02619648, + "epoch": 0.028872381173466428, + "flos": 11318073396480.0, + "grad_norm": 3.6054166897058786, + "language_loss": 0.88449508, + "learning_rate": 3.977843961853942e-06, + "loss": 0.9066081, + "num_input_tokens_seen": 27546160, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.28442383, + "step": 995, + "time_per_iteration": 2.349382162094116 + }, + { + "auxiliary_loss_clip": 0.01173884, + "auxiliary_loss_mlp": 0.01062643, + "balance_loss_clip": 1.04597449, + "balance_loss_mlp": 1.03018212, + "epoch": 0.028901398641982472, + "flos": 16829462488320.0, + "grad_norm": 3.925620774406624, + "language_loss": 0.95697546, + "learning_rate": 3.978422836898733e-06, + "loss": 0.97934079, + "num_input_tokens_seen": 27560210, + "router_z_loss_clip": 1.28027344, + "router_z_loss_mlp": 0.32446289, + "step": 996, + "time_per_iteration": 2.844914674758911 + }, + { + "auxiliary_loss_clip": 0.01027477, + "auxiliary_loss_mlp": 0.01002853, + "balance_loss_clip": 1.00165808, + "balance_loss_mlp": 1.00040877, + "epoch": 0.02893041611049852, + "flos": 74768524358400.0, + "grad_norm": 0.6987046267825834, + "language_loss": 0.50613153, + "learning_rate": 3.979001131035201e-06, + "loss": 0.52643484, + "num_input_tokens_seen": 27622735, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.02441406, + "step": 997, + "time_per_iteration": 5.4945127964019775 + }, + { + "auxiliary_loss_clip": 0.01162498, + "auxiliary_loss_mlp": 0.01067376, + "balance_loss_clip": 1.0434649, + "balance_loss_mlp": 1.03702545, + "epoch": 0.028959433579014565, + "flos": 12197081496960.0, + "grad_norm": 4.519793439073517, + "language_loss": 0.7784189, + "learning_rate": 3.979578845428077e-06, + "loss": 0.80071765, + "num_input_tokens_seen": 27635605, + "router_z_loss_clip": 1.19140625, + "router_z_loss_mlp": 0.30322266, + "step": 998, + "time_per_iteration": 2.3735923767089844 + }, + { + "auxiliary_loss_clip": 0.01169049, + "auxiliary_loss_mlp": 0.01067027, + "balance_loss_clip": 1.04616165, + "balance_loss_mlp": 1.03566349, + "epoch": 0.028988451047530613, + "flos": 14858475793920.0, + "grad_norm": 3.247261910956228, + "language_loss": 0.99648035, + "learning_rate": 3.9801559812385905e-06, + "loss": 1.01884115, + "num_input_tokens_seen": 27648280, + "router_z_loss_clip": 1.23046875, + "router_z_loss_mlp": 0.31359863, + "step": 999, + "time_per_iteration": 2.4405317306518555 + }, + { + "auxiliary_loss_clip": 0.01028055, + "auxiliary_loss_mlp": 0.01001278, + "balance_loss_clip": 1.00222778, + "balance_loss_mlp": 0.99885839, + "epoch": 0.02901746851604666, + "flos": 63059562846720.0, + "grad_norm": 0.7035480337866395, + "language_loss": 0.53178334, + "learning_rate": 3.980732539624484e-06, + "loss": 0.5520767, + "num_input_tokens_seen": 27710000, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.02416992, + "step": 1000, + "time_per_iteration": 5.6588664054870605 + }, + { + "auxiliary_loss_clip": 0.01168251, + "auxiliary_loss_mlp": 0.01054368, + "balance_loss_clip": 1.04438698, + "balance_loss_mlp": 1.02270639, + "epoch": 0.029046485984562706, + "flos": 29927443374720.0, + "grad_norm": 2.637210414450538, + "language_loss": 1.06428742, + "learning_rate": 3.981308521740032e-06, + "loss": 1.08651364, + "num_input_tokens_seen": 27727140, + "router_z_loss_clip": 1.23730469, + "router_z_loss_mlp": 0.31665039, + "step": 1001, + "time_per_iteration": 2.4776127338409424 + }, + { + "auxiliary_loss_clip": 0.01173449, + "auxiliary_loss_mlp": 0.0105722, + "balance_loss_clip": 1.0464071, + "balance_loss_mlp": 1.02737021, + "epoch": 0.029075503453078754, + "flos": 10809528399360.0, + "grad_norm": 5.377970806690868, + "language_loss": 1.07800722, + "learning_rate": 3.981883928736047e-06, + "loss": 1.1003139, + "num_input_tokens_seen": 27739160, + "router_z_loss_clip": 1.27050781, + "router_z_loss_mlp": 0.29858398, + "step": 1002, + "time_per_iteration": 2.420708179473877 + }, + { + "auxiliary_loss_clip": 0.01030496, + "auxiliary_loss_mlp": 0.01002175, + "balance_loss_clip": 1.00416923, + "balance_loss_mlp": 0.99994618, + "epoch": 0.0291045209215948, + "flos": 74767127904000.0, + "grad_norm": 0.7025594777599908, + "language_loss": 0.54518342, + "learning_rate": 3.982458761759901e-06, + "loss": 0.56551009, + "num_input_tokens_seen": 27796095, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.02233887, + "step": 1003, + "time_per_iteration": 2.9802541732788086 + }, + { + "auxiliary_loss_clip": 0.01164942, + "auxiliary_loss_mlp": 0.01058053, + "balance_loss_clip": 1.04427934, + "balance_loss_mlp": 1.02677298, + "epoch": 0.029133538390110847, + "flos": 26168589400320.0, + "grad_norm": 2.3837965137928316, + "language_loss": 0.84322858, + "learning_rate": 3.983033021955535e-06, + "loss": 0.86545855, + "num_input_tokens_seen": 27809520, + "router_z_loss_clip": 1.20654297, + "router_z_loss_mlp": 0.31274414, + "step": 1004, + "time_per_iteration": 2.4610772132873535 + }, + { + "auxiliary_loss_clip": 0.01170198, + "auxiliary_loss_mlp": 0.0105819, + "balance_loss_clip": 1.0477159, + "balance_loss_mlp": 1.02738595, + "epoch": 0.029162555858626895, + "flos": 21611690501760.0, + "grad_norm": 5.582519155137816, + "language_loss": 0.94038653, + "learning_rate": 3.983606710463473e-06, + "loss": 0.96267039, + "num_input_tokens_seen": 27823675, + "router_z_loss_clip": 1.22460938, + "router_z_loss_mlp": 0.30810547, + "step": 1005, + "time_per_iteration": 2.4103736877441406 + }, + { + "auxiliary_loss_clip": 0.01178036, + "auxiliary_loss_mlp": 0.01061392, + "balance_loss_clip": 1.04517734, + "balance_loss_mlp": 1.02788258, + "epoch": 0.02919157332714294, + "flos": 33756717294720.0, + "grad_norm": 2.4800401419765743, + "language_loss": 0.97621965, + "learning_rate": 3.9841798284208365e-06, + "loss": 0.99861395, + "num_input_tokens_seen": 27839020, + "router_z_loss_clip": 1.328125, + "router_z_loss_mlp": 0.33508301, + "step": 1006, + "time_per_iteration": 2.5104475021362305 + }, + { + "auxiliary_loss_clip": 0.01151555, + "auxiliary_loss_mlp": 0.01055111, + "balance_loss_clip": 1.0397352, + "balance_loss_mlp": 1.02734709, + "epoch": 0.029220590795658988, + "flos": 18581159733120.0, + "grad_norm": 2.5658209794560025, + "language_loss": 0.84917653, + "learning_rate": 3.984752376961359e-06, + "loss": 0.87124312, + "num_input_tokens_seen": 27852880, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.27746582, + "step": 1007, + "time_per_iteration": 2.6526780128479004 + }, + { + "auxiliary_loss_clip": 0.01159495, + "auxiliary_loss_mlp": 0.01056147, + "balance_loss_clip": 1.04280996, + "balance_loss_mlp": 1.0292654, + "epoch": 0.029249608264175032, + "flos": 13328232503040.0, + "grad_norm": 3.3727899094264773, + "language_loss": 0.8715384, + "learning_rate": 3.985324357215394e-06, + "loss": 0.89369482, + "num_input_tokens_seen": 27864250, + "router_z_loss_clip": 1.16748047, + "router_z_loss_mlp": 0.26867676, + "step": 1008, + "time_per_iteration": 2.3814828395843506 + }, + { + "auxiliary_loss_clip": 0.01165422, + "auxiliary_loss_mlp": 0.01061415, + "balance_loss_clip": 1.04397035, + "balance_loss_mlp": 1.03393698, + "epoch": 0.02927862573269108, + "flos": 27664862071680.0, + "grad_norm": 2.9560229872857815, + "language_loss": 0.80067778, + "learning_rate": 3.985895770309937e-06, + "loss": 0.82294613, + "num_input_tokens_seen": 27878130, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.27478027, + "step": 1009, + "time_per_iteration": 2.450294256210327 + }, + { + "auxiliary_loss_clip": 0.01158768, + "auxiliary_loss_mlp": 0.01054641, + "balance_loss_clip": 1.04306436, + "balance_loss_mlp": 1.02470732, + "epoch": 0.029307643201207125, + "flos": 25370963412480.0, + "grad_norm": 2.154115652239771, + "language_loss": 0.84948701, + "learning_rate": 3.986466617368632e-06, + "loss": 0.87162113, + "num_input_tokens_seen": 27897760, + "router_z_loss_clip": 1.15820312, + "router_z_loss_mlp": 0.29919434, + "step": 1010, + "time_per_iteration": 2.5495219230651855 + }, + { + "auxiliary_loss_clip": 0.01030548, + "auxiliary_loss_mlp": 0.01028603, + "balance_loss_clip": 1.00378418, + "balance_loss_mlp": 1.02595687, + "epoch": 0.029336660669723173, + "flos": 60797260834560.0, + "grad_norm": 0.9088994167455353, + "language_loss": 0.49385077, + "learning_rate": 3.98703689951179e-06, + "loss": 0.51444227, + "num_input_tokens_seen": 27954750, + "router_z_loss_clip": 0.26757812, + "router_z_loss_mlp": 0.02648926, + "step": 1011, + "time_per_iteration": 2.978306531906128 + }, + { + "auxiliary_loss_clip": 0.01169974, + "auxiliary_loss_mlp": 0.01054731, + "balance_loss_clip": 1.04564333, + "balance_loss_mlp": 1.02316463, + "epoch": 0.02936567813823922, + "flos": 30291412965120.0, + "grad_norm": 2.5127757967581674, + "language_loss": 0.96296895, + "learning_rate": 3.987606617856395e-06, + "loss": 0.98521602, + "num_input_tokens_seen": 27969840, + "router_z_loss_clip": 1.24414062, + "router_z_loss_mlp": 0.31567383, + "step": 1012, + "time_per_iteration": 2.498608112335205 + }, + { + "auxiliary_loss_clip": 0.01166304, + "auxiliary_loss_mlp": 0.0105974, + "balance_loss_clip": 1.04464889, + "balance_loss_mlp": 1.0306766, + "epoch": 0.029394695606755266, + "flos": 37558164994560.0, + "grad_norm": 2.8108381940427285, + "language_loss": 0.84771419, + "learning_rate": 3.988175773516123e-06, + "loss": 0.86997461, + "num_input_tokens_seen": 27987035, + "router_z_loss_clip": 1.21679688, + "router_z_loss_mlp": 0.29040527, + "step": 1013, + "time_per_iteration": 2.6798219680786133 + }, + { + "auxiliary_loss_clip": 0.01164617, + "auxiliary_loss_mlp": 0.01059083, + "balance_loss_clip": 1.04746604, + "balance_loss_mlp": 1.0330236, + "epoch": 0.029423713075271314, + "flos": 13581457660800.0, + "grad_norm": 2.935119184484765, + "language_loss": 1.10054898, + "learning_rate": 3.988744367601354e-06, + "loss": 1.12278593, + "num_input_tokens_seen": 27999625, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.26062012, + "step": 1014, + "time_per_iteration": 2.4200873374938965 + }, + { + "auxiliary_loss_clip": 0.01168905, + "auxiliary_loss_mlp": 0.01062756, + "balance_loss_clip": 1.04680252, + "balance_loss_mlp": 1.02966356, + "epoch": 0.02945273054378736, + "flos": 34743781653120.0, + "grad_norm": 2.848615564916145, + "language_loss": 1.05713415, + "learning_rate": 3.9893124012191855e-06, + "loss": 1.07945085, + "num_input_tokens_seen": 28013200, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.3314209, + "step": 1015, + "time_per_iteration": 2.548980951309204 + }, + { + "auxiliary_loss_clip": 0.01164307, + "auxiliary_loss_mlp": 0.01058689, + "balance_loss_clip": 1.04618883, + "balance_loss_mlp": 1.03156841, + "epoch": 0.029481748012303407, + "flos": 33357345719040.0, + "grad_norm": 4.244487103815412, + "language_loss": 0.88685751, + "learning_rate": 3.989879875473443e-06, + "loss": 0.90908754, + "num_input_tokens_seen": 28032205, + "router_z_loss_clip": 1.18212891, + "router_z_loss_mlp": 0.27124023, + "step": 1016, + "time_per_iteration": 2.580291509628296 + }, + { + "auxiliary_loss_clip": 0.01165824, + "auxiliary_loss_mlp": 0.01059947, + "balance_loss_clip": 1.04383469, + "balance_loss_mlp": 1.02724791, + "epoch": 0.029510765480819455, + "flos": 44666935655040.0, + "grad_norm": 2.051016363573508, + "language_loss": 0.93430173, + "learning_rate": 3.990446791464694e-06, + "loss": 0.95655942, + "num_input_tokens_seen": 28051985, + "router_z_loss_clip": 1.21875, + "router_z_loss_mlp": 0.32702637, + "step": 1017, + "time_per_iteration": 2.6670446395874023 + }, + { + "auxiliary_loss_clip": 0.01171086, + "auxiliary_loss_mlp": 0.01073778, + "balance_loss_clip": 1.0478518, + "balance_loss_mlp": 1.04273558, + "epoch": 0.0295397829493355, + "flos": 36276573473280.0, + "grad_norm": 9.365781167250457, + "language_loss": 0.88818777, + "learning_rate": 3.991013150290262e-06, + "loss": 0.91063643, + "num_input_tokens_seen": 28066965, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.3104248, + "step": 1018, + "time_per_iteration": 2.5791361331939697 + }, + { + "auxiliary_loss_clip": 0.01040712, + "auxiliary_loss_mlp": 0.01005242, + "balance_loss_clip": 1.01369762, + "balance_loss_mlp": 1.00270259, + "epoch": 0.029568800417851548, + "flos": 74774738580480.0, + "grad_norm": 0.6586607492649322, + "language_loss": 0.50267684, + "learning_rate": 3.991578953044237e-06, + "loss": 0.52313638, + "num_input_tokens_seen": 28135700, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.02539062, + "step": 1019, + "time_per_iteration": 3.222712278366089 + }, + { + "auxiliary_loss_clip": 0.01158521, + "auxiliary_loss_mlp": 0.0105138, + "balance_loss_clip": 1.04346752, + "balance_loss_mlp": 1.02701426, + "epoch": 0.029597817886367592, + "flos": 9826513758720.0, + "grad_norm": 2.703647362949842, + "language_loss": 0.84341085, + "learning_rate": 3.992144200817493e-06, + "loss": 0.86550987, + "num_input_tokens_seen": 28147150, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.24377441, + "step": 1020, + "time_per_iteration": 2.795621395111084 + }, + { + "auxiliary_loss_clip": 0.01161067, + "auxiliary_loss_mlp": 0.01056584, + "balance_loss_clip": 1.04489255, + "balance_loss_mlp": 1.02966642, + "epoch": 0.02962683535488364, + "flos": 66119876737920.0, + "grad_norm": 2.23057235338985, + "language_loss": 0.77061367, + "learning_rate": 3.992708894697692e-06, + "loss": 0.79279017, + "num_input_tokens_seen": 28170010, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.26928711, + "step": 1021, + "time_per_iteration": 2.8491761684417725 + }, + { + "auxiliary_loss_clip": 0.01160629, + "auxiliary_loss_mlp": 0.01060523, + "balance_loss_clip": 1.04832542, + "balance_loss_mlp": 1.03417766, + "epoch": 0.029655852823399685, + "flos": 23541445013760.0, + "grad_norm": 3.2030173432769673, + "language_loss": 0.9181087, + "learning_rate": 3.993273035769305e-06, + "loss": 0.94032019, + "num_input_tokens_seen": 28187690, + "router_z_loss_clip": 1.12255859, + "router_z_loss_mlp": 0.26330566, + "step": 1022, + "time_per_iteration": 2.509235143661499 + }, + { + "auxiliary_loss_clip": 0.01156834, + "auxiliary_loss_mlp": 0.01057316, + "balance_loss_clip": 1.03945446, + "balance_loss_mlp": 1.0285629, + "epoch": 0.029684870291915733, + "flos": 13764611986560.0, + "grad_norm": 3.0375058992979778, + "language_loss": 0.86874723, + "learning_rate": 3.99383662511362e-06, + "loss": 0.89088869, + "num_input_tokens_seen": 28200130, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.28747559, + "step": 1023, + "time_per_iteration": 2.4403717517852783 + }, + { + "auxiliary_loss_clip": 0.01031193, + "auxiliary_loss_mlp": 0.01011973, + "balance_loss_clip": 1.00394464, + "balance_loss_mlp": 1.00932646, + "epoch": 0.02971388776043178, + "flos": 74776658705280.0, + "grad_norm": 0.6708788605282224, + "language_loss": 0.55348003, + "learning_rate": 3.994399663808758e-06, + "loss": 0.57391173, + "num_input_tokens_seen": 28266870, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.02648926, + "step": 1024, + "time_per_iteration": 3.2121284008026123 + }, + { + "auxiliary_loss_clip": 0.01030909, + "auxiliary_loss_mlp": 0.01007271, + "balance_loss_clip": 1.00369906, + "balance_loss_mlp": 1.0045054, + "epoch": 0.029742905228947826, + "flos": 58654675474560.0, + "grad_norm": 0.7111026627678787, + "language_loss": 0.49414665, + "learning_rate": 3.9949621529296794e-06, + "loss": 0.51452845, + "num_input_tokens_seen": 28329815, + "router_z_loss_clip": 0.27148438, + "router_z_loss_mlp": 0.02770996, + "step": 1025, + "time_per_iteration": 3.067807674407959 + }, + { + "auxiliary_loss_clip": 0.01030104, + "auxiliary_loss_mlp": 0.01007711, + "balance_loss_clip": 1.00293207, + "balance_loss_mlp": 1.00493336, + "epoch": 0.029771922697463874, + "flos": 71274695581440.0, + "grad_norm": 0.7813077931323856, + "language_loss": 0.54761368, + "learning_rate": 3.995524093548202e-06, + "loss": 0.56799191, + "num_input_tokens_seen": 28395090, + "router_z_loss_clip": 0.27148438, + "router_z_loss_mlp": 0.02783203, + "step": 1026, + "time_per_iteration": 3.1438136100769043 + }, + { + "auxiliary_loss_clip": 0.01155634, + "auxiliary_loss_mlp": 0.01053921, + "balance_loss_clip": 1.04347467, + "balance_loss_mlp": 1.02643144, + "epoch": 0.02980094016597992, + "flos": 12487244739840.0, + "grad_norm": 3.146280004691934, + "language_loss": 0.87814844, + "learning_rate": 3.996085486733009e-06, + "loss": 0.90024406, + "num_input_tokens_seen": 28406805, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.27502441, + "step": 1027, + "time_per_iteration": 2.349883794784546 + }, + { + "auxiliary_loss_clip": 0.01168605, + "auxiliary_loss_mlp": 0.01063046, + "balance_loss_clip": 1.0446161, + "balance_loss_mlp": 1.03493667, + "epoch": 0.029829957634495967, + "flos": 33439111856640.0, + "grad_norm": 2.180113549118334, + "language_loss": 0.98524332, + "learning_rate": 3.996646333549668e-06, + "loss": 1.0075599, + "num_input_tokens_seen": 28428370, + "router_z_loss_clip": 1.23925781, + "router_z_loss_mlp": 0.28100586, + "step": 1028, + "time_per_iteration": 2.631155490875244 + }, + { + "auxiliary_loss_clip": 0.0115441, + "auxiliary_loss_mlp": 0.01045303, + "balance_loss_clip": 1.04230285, + "balance_loss_mlp": 1.02178359, + "epoch": 0.02985897510301201, + "flos": 34197216318720.0, + "grad_norm": 2.3702911504736095, + "language_loss": 0.81953502, + "learning_rate": 3.997206635060634e-06, + "loss": 0.84153217, + "num_input_tokens_seen": 28444815, + "router_z_loss_clip": 1.12207031, + "router_z_loss_mlp": 0.23510742, + "step": 1029, + "time_per_iteration": 2.533262252807617 + }, + { + "auxiliary_loss_clip": 0.01169957, + "auxiliary_loss_mlp": 0.01061259, + "balance_loss_clip": 1.045789, + "balance_loss_mlp": 1.02969205, + "epoch": 0.02988799257152806, + "flos": 30181331848320.0, + "grad_norm": 2.5049634597010333, + "language_loss": 0.91044664, + "learning_rate": 3.997766392325268e-06, + "loss": 0.93275887, + "num_input_tokens_seen": 28460255, + "router_z_loss_clip": 1.24121094, + "router_z_loss_mlp": 0.31542969, + "step": 1030, + "time_per_iteration": 2.480332612991333 + }, + { + "auxiliary_loss_clip": 0.01034866, + "auxiliary_loss_mlp": 0.01014123, + "balance_loss_clip": 1.00746, + "balance_loss_mlp": 1.01140499, + "epoch": 0.029917010040044108, + "flos": 58202201854080.0, + "grad_norm": 0.7261213823993506, + "language_loss": 0.49574155, + "learning_rate": 3.998325606399846e-06, + "loss": 0.51623142, + "num_input_tokens_seen": 28519810, + "router_z_loss_clip": 0.2734375, + "router_z_loss_mlp": 0.02722168, + "step": 1031, + "time_per_iteration": 2.869279384613037 + }, + { + "auxiliary_loss_clip": 0.01168502, + "auxiliary_loss_mlp": 0.01059691, + "balance_loss_clip": 1.04667342, + "balance_loss_mlp": 1.03103352, + "epoch": 0.029946027508560152, + "flos": 18397272268800.0, + "grad_norm": 4.011649802987375, + "language_loss": 0.7953155, + "learning_rate": 3.998884278337572e-06, + "loss": 0.81759733, + "num_input_tokens_seen": 28533395, + "router_z_loss_clip": 1.21777344, + "router_z_loss_mlp": 0.28662109, + "step": 1032, + "time_per_iteration": 2.3805654048919678 + }, + { + "auxiliary_loss_clip": 0.01161866, + "auxiliary_loss_mlp": 0.01058325, + "balance_loss_clip": 1.04411578, + "balance_loss_mlp": 1.02914214, + "epoch": 0.0299750449770762, + "flos": 20843426833920.0, + "grad_norm": 2.709578324058611, + "language_loss": 1.09328842, + "learning_rate": 3.999442409188591e-06, + "loss": 1.1154902, + "num_input_tokens_seen": 28549595, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.29211426, + "step": 1033, + "time_per_iteration": 2.7743589878082275 + }, + { + "auxiliary_loss_clip": 0.01161063, + "auxiliary_loss_mlp": 0.01059113, + "balance_loss_clip": 1.0438652, + "balance_loss_mlp": 1.03086019, + "epoch": 0.030004062445592245, + "flos": 14823003985920.0, + "grad_norm": 3.593176168401648, + "language_loss": 0.88555855, + "learning_rate": 4e-06, + "loss": 0.90776026, + "num_input_tokens_seen": 28562100, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.2824707, + "step": 1034, + "time_per_iteration": 2.3668735027313232 + }, + { + "auxiliary_loss_clip": 0.01029407, + "auxiliary_loss_mlp": 0.01004773, + "balance_loss_clip": 1.00244021, + "balance_loss_mlp": 1.00206733, + "epoch": 0.030033079914108293, + "flos": 59921185288320.0, + "grad_norm": 0.7381354335526723, + "language_loss": 0.50818598, + "learning_rate": 3.999999991167595e-06, + "loss": 0.5285278, + "num_input_tokens_seen": 28622455, + "router_z_loss_clip": 0.26953125, + "router_z_loss_mlp": 0.02709961, + "step": 1035, + "time_per_iteration": 3.072673797607422 + }, + { + "auxiliary_loss_clip": 0.01167763, + "auxiliary_loss_mlp": 0.01057336, + "balance_loss_clip": 1.04296517, + "balance_loss_mlp": 1.02683032, + "epoch": 0.03006209738262434, + "flos": 62605273835520.0, + "grad_norm": 1.836276173816049, + "language_loss": 1.069345, + "learning_rate": 3.999999964670382e-06, + "loss": 1.09159589, + "num_input_tokens_seen": 28652645, + "router_z_loss_clip": 1.24804688, + "router_z_loss_mlp": 0.30493164, + "step": 1036, + "time_per_iteration": 2.7682595252990723 + }, + { + "auxiliary_loss_clip": 0.01167004, + "auxiliary_loss_mlp": 0.01068554, + "balance_loss_clip": 1.04360271, + "balance_loss_mlp": 1.03591478, + "epoch": 0.030091114851140386, + "flos": 36207899095680.0, + "grad_norm": 2.9655656192800377, + "language_loss": 0.84476316, + "learning_rate": 3.999999920508358e-06, + "loss": 0.86711872, + "num_input_tokens_seen": 28677435, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.32641602, + "step": 1037, + "time_per_iteration": 2.5520782470703125 + }, + { + "auxiliary_loss_clip": 0.0114783, + "auxiliary_loss_mlp": 0.01046685, + "balance_loss_clip": 1.04041958, + "balance_loss_mlp": 1.02434564, + "epoch": 0.030120132319656434, + "flos": 18798703614720.0, + "grad_norm": 2.8068196995940315, + "language_loss": 0.83005166, + "learning_rate": 3.999999858681527e-06, + "loss": 0.85199678, + "num_input_tokens_seen": 28689935, + "router_z_loss_clip": 1.07373047, + "router_z_loss_mlp": 0.22338867, + "step": 1038, + "time_per_iteration": 2.384187698364258 + }, + { + "auxiliary_loss_clip": 0.01156791, + "auxiliary_loss_mlp": 0.01049237, + "balance_loss_clip": 1.04090953, + "balance_loss_mlp": 1.02308226, + "epoch": 0.03014914978817248, + "flos": 25987180642560.0, + "grad_norm": 2.4832453626090545, + "language_loss": 0.93018091, + "learning_rate": 3.999999779189888e-06, + "loss": 0.95224124, + "num_input_tokens_seen": 28708305, + "router_z_loss_clip": 1.15820312, + "router_z_loss_mlp": 0.26159668, + "step": 1039, + "time_per_iteration": 2.5775938034057617 + }, + { + "auxiliary_loss_clip": 0.01034456, + "auxiliary_loss_mlp": 0.01008713, + "balance_loss_clip": 1.00777817, + "balance_loss_mlp": 1.00606608, + "epoch": 0.030178167256688527, + "flos": 61201973848320.0, + "grad_norm": 0.665276418376171, + "language_loss": 0.51264179, + "learning_rate": 3.99999968203344e-06, + "loss": 0.53307354, + "num_input_tokens_seen": 28770205, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.02648926, + "step": 1040, + "time_per_iteration": 3.0227932929992676 + }, + { + "auxiliary_loss_clip": 0.01032653, + "auxiliary_loss_mlp": 0.01013075, + "balance_loss_clip": 1.00603056, + "balance_loss_mlp": 1.01036906, + "epoch": 0.03020718472520457, + "flos": 69185702891520.0, + "grad_norm": 0.8042319077204981, + "language_loss": 0.60776442, + "learning_rate": 3.999999567212187e-06, + "loss": 0.62822169, + "num_input_tokens_seen": 28830940, + "router_z_loss_clip": 0.265625, + "router_z_loss_mlp": 0.02709961, + "step": 1041, + "time_per_iteration": 3.056264638900757 + }, + { + "auxiliary_loss_clip": 0.01163485, + "auxiliary_loss_mlp": 0.01062336, + "balance_loss_clip": 1.04301465, + "balance_loss_mlp": 1.03117454, + "epoch": 0.03023620219372062, + "flos": 12487070183040.0, + "grad_norm": 3.400129296547645, + "language_loss": 0.70932853, + "learning_rate": 3.9999994347261276e-06, + "loss": 0.73158675, + "num_input_tokens_seen": 28843830, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.31152344, + "step": 1042, + "time_per_iteration": 2.370908498764038 + }, + { + "auxiliary_loss_clip": 0.01027735, + "auxiliary_loss_mlp": 0.01002321, + "balance_loss_clip": 1.00164652, + "balance_loss_mlp": 0.99971086, + "epoch": 0.030265219662236668, + "flos": 70973221057920.0, + "grad_norm": 0.7294859796273229, + "language_loss": 0.56679225, + "learning_rate": 3.999999284575265e-06, + "loss": 0.58709282, + "num_input_tokens_seen": 28900160, + "router_z_loss_clip": 0.26171875, + "router_z_loss_mlp": 0.02612305, + "step": 1043, + "time_per_iteration": 2.9768922328948975 + }, + { + "auxiliary_loss_clip": 0.01028523, + "auxiliary_loss_mlp": 0.01003761, + "balance_loss_clip": 1.0029099, + "balance_loss_mlp": 1.00117457, + "epoch": 0.030294237130752712, + "flos": 63308179704960.0, + "grad_norm": 0.6646235675566312, + "language_loss": 0.52801305, + "learning_rate": 3.999999116759598e-06, + "loss": 0.54833591, + "num_input_tokens_seen": 28962035, + "router_z_loss_clip": 0.25585938, + "router_z_loss_mlp": 0.02587891, + "step": 1044, + "time_per_iteration": 2.9661808013916016 + }, + { + "auxiliary_loss_clip": 0.01162047, + "auxiliary_loss_mlp": 0.01057957, + "balance_loss_clip": 1.04500341, + "balance_loss_mlp": 1.02982342, + "epoch": 0.03032325459926876, + "flos": 20516150949120.0, + "grad_norm": 2.609877413609671, + "language_loss": 1.07997799, + "learning_rate": 3.999998931279131e-06, + "loss": 1.1021781, + "num_input_tokens_seen": 28975725, + "router_z_loss_clip": 1.17089844, + "router_z_loss_mlp": 0.28125, + "step": 1045, + "time_per_iteration": 2.4314491748809814 + }, + { + "auxiliary_loss_clip": 0.01169617, + "auxiliary_loss_mlp": 0.01058857, + "balance_loss_clip": 1.04732275, + "balance_loss_mlp": 1.0283041, + "epoch": 0.030352272067784805, + "flos": 32087030567040.0, + "grad_norm": 2.2704258025654864, + "language_loss": 1.04625332, + "learning_rate": 3.999998728133863e-06, + "loss": 1.06853807, + "num_input_tokens_seen": 28997230, + "router_z_loss_clip": 1.22363281, + "router_z_loss_mlp": 0.30529785, + "step": 1046, + "time_per_iteration": 2.787484645843506 + }, + { + "auxiliary_loss_clip": 0.01164692, + "auxiliary_loss_mlp": 0.01068541, + "balance_loss_clip": 1.04656887, + "balance_loss_mlp": 1.03888202, + "epoch": 0.030381289536300853, + "flos": 32041643932800.0, + "grad_norm": 2.4843279831594214, + "language_loss": 0.93969876, + "learning_rate": 3.999998507323797e-06, + "loss": 0.96203107, + "num_input_tokens_seen": 29009920, + "router_z_loss_clip": 1.18164062, + "router_z_loss_mlp": 0.29663086, + "step": 1047, + "time_per_iteration": 2.514040946960449 + }, + { + "auxiliary_loss_clip": 0.01168737, + "auxiliary_loss_mlp": 0.01063343, + "balance_loss_clip": 1.04229486, + "balance_loss_mlp": 1.03075147, + "epoch": 0.0304103070048169, + "flos": 29450913960960.0, + "grad_norm": 3.8904860780935824, + "language_loss": 0.99460548, + "learning_rate": 3.999998268848935e-06, + "loss": 1.01692617, + "num_input_tokens_seen": 29023615, + "router_z_loss_clip": 1.26464844, + "router_z_loss_mlp": 0.3260498, + "step": 1048, + "time_per_iteration": 2.3734710216522217 + }, + { + "auxiliary_loss_clip": 0.01176878, + "auxiliary_loss_mlp": 0.01072589, + "balance_loss_clip": 1.04563785, + "balance_loss_mlp": 1.03861392, + "epoch": 0.030439324473332946, + "flos": 18524145749760.0, + "grad_norm": 2.8162238919478093, + "language_loss": 0.74501872, + "learning_rate": 3.99999801270928e-06, + "loss": 0.76751339, + "num_input_tokens_seen": 29042650, + "router_z_loss_clip": 1.3125, + "router_z_loss_mlp": 0.33959961, + "step": 1049, + "time_per_iteration": 2.450484275817871 + }, + { + "auxiliary_loss_clip": 0.01153135, + "auxiliary_loss_mlp": 0.01044217, + "balance_loss_clip": 1.04142809, + "balance_loss_mlp": 1.01908755, + "epoch": 0.030468341941848994, + "flos": 20047371857280.0, + "grad_norm": 2.0661801700539133, + "language_loss": 0.77671492, + "learning_rate": 3.999997738904832e-06, + "loss": 0.79868841, + "num_input_tokens_seen": 29059945, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.25134277, + "step": 1050, + "time_per_iteration": 2.394695997238159 + }, + { + "auxiliary_loss_clip": 0.01030901, + "auxiliary_loss_mlp": 0.01002941, + "balance_loss_clip": 1.00585842, + "balance_loss_mlp": 1.000175, + "epoch": 0.03049735941036504, + "flos": 67431142915200.0, + "grad_norm": 0.752223843439534, + "language_loss": 0.54085153, + "learning_rate": 3.999997447435595e-06, + "loss": 0.56119001, + "num_input_tokens_seen": 29121730, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.02770996, + "step": 1051, + "time_per_iteration": 3.004603147506714 + }, + { + "auxiliary_loss_clip": 0.01166865, + "auxiliary_loss_mlp": 0.01062852, + "balance_loss_clip": 1.04331124, + "balance_loss_mlp": 1.03343129, + "epoch": 0.030526376878881087, + "flos": 37699074708480.0, + "grad_norm": 2.6148597360078485, + "language_loss": 0.96390295, + "learning_rate": 3.999997138301571e-06, + "loss": 0.98620015, + "num_input_tokens_seen": 29138175, + "router_z_loss_clip": 1.23535156, + "router_z_loss_mlp": 0.29394531, + "step": 1052, + "time_per_iteration": 2.459637403488159 + }, + { + "auxiliary_loss_clip": 0.01150133, + "auxiliary_loss_mlp": 0.01049765, + "balance_loss_clip": 1.04137015, + "balance_loss_mlp": 1.02281177, + "epoch": 0.03055539434739713, + "flos": 12120377506560.0, + "grad_norm": 2.79843005338145, + "language_loss": 0.80195683, + "learning_rate": 3.999996811502763e-06, + "loss": 0.82395577, + "num_input_tokens_seen": 29149015, + "router_z_loss_clip": 1.08691406, + "router_z_loss_mlp": 0.26953125, + "step": 1053, + "time_per_iteration": 2.3700571060180664 + }, + { + "auxiliary_loss_clip": 0.01160323, + "auxiliary_loss_mlp": 0.01057227, + "balance_loss_clip": 1.04188621, + "balance_loss_mlp": 1.03001142, + "epoch": 0.03058441181591318, + "flos": 14974910778240.0, + "grad_norm": 2.0654167481578507, + "language_loss": 0.64017457, + "learning_rate": 3.999996467039174e-06, + "loss": 0.66235006, + "num_input_tokens_seen": 29164455, + "router_z_loss_clip": 1.18457031, + "router_z_loss_mlp": 0.2722168, + "step": 1054, + "time_per_iteration": 2.380074977874756 + }, + { + "auxiliary_loss_clip": 0.01157651, + "auxiliary_loss_mlp": 0.01057018, + "balance_loss_clip": 1.04131603, + "balance_loss_mlp": 1.02893281, + "epoch": 0.030613429284429228, + "flos": 30547849968000.0, + "grad_norm": 2.511997944453479, + "language_loss": 0.96253395, + "learning_rate": 3.999996104910807e-06, + "loss": 0.98468059, + "num_input_tokens_seen": 29180150, + "router_z_loss_clip": 1.16259766, + "router_z_loss_mlp": 0.28088379, + "step": 1055, + "time_per_iteration": 2.498290777206421 + }, + { + "auxiliary_loss_clip": 0.0117013, + "auxiliary_loss_mlp": 0.01060211, + "balance_loss_clip": 1.04395711, + "balance_loss_mlp": 1.02866828, + "epoch": 0.030642446752945272, + "flos": 33721315309440.0, + "grad_norm": 3.0314995626532473, + "language_loss": 0.92709637, + "learning_rate": 3.999995725117666e-06, + "loss": 0.94939977, + "num_input_tokens_seen": 29199245, + "router_z_loss_clip": 1.26074219, + "router_z_loss_mlp": 0.31518555, + "step": 1056, + "time_per_iteration": 2.47772479057312 + }, + { + "auxiliary_loss_clip": 0.01162285, + "auxiliary_loss_mlp": 0.01058784, + "balance_loss_clip": 1.04323554, + "balance_loss_mlp": 1.02983987, + "epoch": 0.03067146422146132, + "flos": 30731004293760.0, + "grad_norm": 2.951941517907532, + "language_loss": 1.0339427, + "learning_rate": 3.999995327659752e-06, + "loss": 1.05615342, + "num_input_tokens_seen": 29215115, + "router_z_loss_clip": 1.18945312, + "router_z_loss_mlp": 0.28955078, + "step": 1057, + "time_per_iteration": 2.509718179702759 + }, + { + "auxiliary_loss_clip": 0.01027037, + "auxiliary_loss_mlp": 0.01010306, + "balance_loss_clip": 1.00183892, + "balance_loss_mlp": 1.00768375, + "epoch": 0.030700481689977365, + "flos": 61935882871680.0, + "grad_norm": 0.6685820368052231, + "language_loss": 0.58962619, + "learning_rate": 3.9999949125370706e-06, + "loss": 0.60999954, + "num_input_tokens_seen": 29285835, + "router_z_loss_clip": 0.25195312, + "router_z_loss_mlp": 0.02624512, + "step": 1058, + "time_per_iteration": 3.2007341384887695 + }, + { + "auxiliary_loss_clip": 0.0116339, + "auxiliary_loss_mlp": 0.01059693, + "balance_loss_clip": 1.04522991, + "balance_loss_mlp": 1.03223896, + "epoch": 0.030729499158493413, + "flos": 29853532293120.0, + "grad_norm": 2.549996113222792, + "language_loss": 0.73269558, + "learning_rate": 3.999994479749624e-06, + "loss": 0.75492644, + "num_input_tokens_seen": 29306070, + "router_z_loss_clip": 1.18066406, + "router_z_loss_mlp": 0.27441406, + "step": 1059, + "time_per_iteration": 2.669445037841797 + }, + { + "auxiliary_loss_clip": 0.01160253, + "auxiliary_loss_mlp": 0.01060996, + "balance_loss_clip": 1.04410279, + "balance_loss_mlp": 1.03108644, + "epoch": 0.03075851662700946, + "flos": 20184511144320.0, + "grad_norm": 2.8444658672082657, + "language_loss": 1.03826904, + "learning_rate": 3.999994029297418e-06, + "loss": 1.06048155, + "num_input_tokens_seen": 29319195, + "router_z_loss_clip": 1.16015625, + "router_z_loss_mlp": 0.2989502, + "step": 1060, + "time_per_iteration": 2.4894027709960938 + }, + { + "auxiliary_loss_clip": 0.01171057, + "auxiliary_loss_mlp": 0.01055113, + "balance_loss_clip": 1.04766667, + "balance_loss_mlp": 1.02455974, + "epoch": 0.030787534095525506, + "flos": 19859888522880.0, + "grad_norm": 3.261467435039726, + "language_loss": 0.88255316, + "learning_rate": 3.999993561180455e-06, + "loss": 0.9048149, + "num_input_tokens_seen": 29334270, + "router_z_loss_clip": 1.23535156, + "router_z_loss_mlp": 0.30554199, + "step": 1061, + "time_per_iteration": 2.3952698707580566 + }, + { + "auxiliary_loss_clip": 0.01027224, + "auxiliary_loss_mlp": 0.010031, + "balance_loss_clip": 1.00163364, + "balance_loss_mlp": 1.00063193, + "epoch": 0.030816551564041554, + "flos": 74047707095040.0, + "grad_norm": 0.7407337832635814, + "language_loss": 0.55448329, + "learning_rate": 3.999993075398739e-06, + "loss": 0.57478654, + "num_input_tokens_seen": 29403110, + "router_z_loss_clip": 0.25585938, + "router_z_loss_mlp": 0.0246582, + "step": 1062, + "time_per_iteration": 3.112001895904541 + }, + { + "auxiliary_loss_clip": 0.01027059, + "auxiliary_loss_mlp": 0.01005307, + "balance_loss_clip": 1.00187922, + "balance_loss_mlp": 1.00291145, + "epoch": 0.0308455690325576, + "flos": 71236814889600.0, + "grad_norm": 0.7324239112244942, + "language_loss": 0.56367129, + "learning_rate": 3.999992571952275e-06, + "loss": 0.58399498, + "num_input_tokens_seen": 29464635, + "router_z_loss_clip": 0.25195312, + "router_z_loss_mlp": 0.02392578, + "step": 1063, + "time_per_iteration": 3.0659265518188477 + }, + { + "auxiliary_loss_clip": 0.01153252, + "auxiliary_loss_mlp": 0.01050484, + "balance_loss_clip": 1.0437696, + "balance_loss_mlp": 1.02691627, + "epoch": 0.030874586501073647, + "flos": 14677520883840.0, + "grad_norm": 3.1947729732495462, + "language_loss": 0.86295736, + "learning_rate": 3.999992050841068e-06, + "loss": 0.88499475, + "num_input_tokens_seen": 29477410, + "router_z_loss_clip": 1.09570312, + "router_z_loss_mlp": 0.2355957, + "step": 1064, + "time_per_iteration": 2.42087984085083 + }, + { + "auxiliary_loss_clip": 0.01027891, + "auxiliary_loss_mlp": 0.01002752, + "balance_loss_clip": 1.0021255, + "balance_loss_mlp": 1.000332, + "epoch": 0.03090360396958969, + "flos": 74760808947840.0, + "grad_norm": 0.7393043498615948, + "language_loss": 0.5301013, + "learning_rate": 3.999991512065121e-06, + "loss": 0.55040777, + "num_input_tokens_seen": 29538140, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.02416992, + "step": 1065, + "time_per_iteration": 2.954691171646118 + }, + { + "auxiliary_loss_clip": 0.01153275, + "auxiliary_loss_mlp": 0.01045387, + "balance_loss_clip": 1.04159617, + "balance_loss_mlp": 1.01793361, + "epoch": 0.03093262143810574, + "flos": 20040668876160.0, + "grad_norm": 3.2675669946814025, + "language_loss": 0.79468036, + "learning_rate": 3.9999909556244405e-06, + "loss": 0.81666696, + "num_input_tokens_seen": 29552520, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.2746582, + "step": 1066, + "time_per_iteration": 2.3987810611724854 + }, + { + "auxiliary_loss_clip": 0.0115944, + "auxiliary_loss_mlp": 0.01063442, + "balance_loss_clip": 1.04020822, + "balance_loss_mlp": 1.03517795, + "epoch": 0.030961638906621788, + "flos": 16758553783680.0, + "grad_norm": 3.8294313780492644, + "language_loss": 0.92168909, + "learning_rate": 3.999990381519031e-06, + "loss": 0.94391787, + "num_input_tokens_seen": 29570035, + "router_z_loss_clip": 1.19335938, + "router_z_loss_mlp": 0.2824707, + "step": 1067, + "time_per_iteration": 4.834253549575806 + }, + { + "auxiliary_loss_clip": 0.0116828, + "auxiliary_loss_mlp": 0.01065182, + "balance_loss_clip": 1.04488957, + "balance_loss_mlp": 1.0334363, + "epoch": 0.030990656375137832, + "flos": 20332647509760.0, + "grad_norm": 3.5008504235637177, + "language_loss": 0.90253955, + "learning_rate": 3.999989789748896e-06, + "loss": 0.92487419, + "num_input_tokens_seen": 29583255, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.31713867, + "step": 1068, + "time_per_iteration": 2.3847358226776123 + }, + { + "auxiliary_loss_clip": 0.01171851, + "auxiliary_loss_mlp": 0.01064156, + "balance_loss_clip": 1.04815125, + "balance_loss_mlp": 1.03249359, + "epoch": 0.03101967384365388, + "flos": 20441192526720.0, + "grad_norm": 3.2439781088869335, + "language_loss": 0.91273928, + "learning_rate": 3.999989180314042e-06, + "loss": 0.93509936, + "num_input_tokens_seen": 29596400, + "router_z_loss_clip": 1.23681641, + "router_z_loss_mlp": 0.31665039, + "step": 1069, + "time_per_iteration": 4.73886775970459 + }, + { + "auxiliary_loss_clip": 0.01026932, + "auxiliary_loss_mlp": 0.01004515, + "balance_loss_clip": 1.00172758, + "balance_loss_mlp": 1.00204778, + "epoch": 0.031048691312169925, + "flos": 74774564023680.0, + "grad_norm": 0.6179663759778453, + "language_loss": 0.50008547, + "learning_rate": 3.999988553214475e-06, + "loss": 0.52039999, + "num_input_tokens_seen": 29666475, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.0246582, + "step": 1070, + "time_per_iteration": 3.1286134719848633 + }, + { + "auxiliary_loss_clip": 0.01026835, + "auxiliary_loss_mlp": 0.01002242, + "balance_loss_clip": 1.00162017, + "balance_loss_mlp": 0.99988192, + "epoch": 0.031077708780685973, + "flos": 57081314787840.0, + "grad_norm": 0.6933723119476635, + "language_loss": 0.54953533, + "learning_rate": 3.9999879084501984e-06, + "loss": 0.56982607, + "num_input_tokens_seen": 29728695, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.02355957, + "step": 1071, + "time_per_iteration": 3.2259933948516846 + }, + { + "auxiliary_loss_clip": 0.0117031, + "auxiliary_loss_mlp": 0.01061491, + "balance_loss_clip": 1.04517293, + "balance_loss_mlp": 1.03075862, + "epoch": 0.03110672624920202, + "flos": 20219738572800.0, + "grad_norm": 3.1502361649689044, + "language_loss": 1.15018463, + "learning_rate": 3.99998724602122e-06, + "loss": 1.17250252, + "num_input_tokens_seen": 29740860, + "router_z_loss_clip": 1.25195312, + "router_z_loss_mlp": 0.30737305, + "step": 1072, + "time_per_iteration": 2.4232232570648193 + }, + { + "auxiliary_loss_clip": 0.01166499, + "auxiliary_loss_mlp": 0.01063305, + "balance_loss_clip": 1.04326177, + "balance_loss_mlp": 1.02984285, + "epoch": 0.031135743717718066, + "flos": 27628552391040.0, + "grad_norm": 2.713205064108277, + "language_loss": 0.99739933, + "learning_rate": 3.999986565927545e-06, + "loss": 1.01969743, + "num_input_tokens_seen": 29757160, + "router_z_loss_clip": 1.23242188, + "router_z_loss_mlp": 0.33459473, + "step": 1073, + "time_per_iteration": 2.54258131980896 + }, + { + "auxiliary_loss_clip": 0.01167543, + "auxiliary_loss_mlp": 0.01056451, + "balance_loss_clip": 1.04611635, + "balance_loss_mlp": 1.02664912, + "epoch": 0.031164761186234114, + "flos": 30949595516160.0, + "grad_norm": 4.846846228424639, + "language_loss": 1.05070949, + "learning_rate": 3.99998586816918e-06, + "loss": 1.07294953, + "num_input_tokens_seen": 29772100, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.29797363, + "step": 1074, + "time_per_iteration": 4.98825478553772 + }, + { + "auxiliary_loss_clip": 0.01165292, + "auxiliary_loss_mlp": 0.01059048, + "balance_loss_clip": 1.04563904, + "balance_loss_mlp": 1.03073525, + "epoch": 0.03119377865475016, + "flos": 33868753447680.0, + "grad_norm": 2.2493310895314127, + "language_loss": 0.92292577, + "learning_rate": 3.99998515274613e-06, + "loss": 0.94516909, + "num_input_tokens_seen": 29791055, + "router_z_loss_clip": 1.19824219, + "router_z_loss_mlp": 0.28283691, + "step": 1075, + "time_per_iteration": 2.5042974948883057 + }, + { + "auxiliary_loss_clip": 0.01027529, + "auxiliary_loss_mlp": 0.01010259, + "balance_loss_clip": 1.00263906, + "balance_loss_mlp": 1.00794649, + "epoch": 0.031222796123266207, + "flos": 67508126196480.0, + "grad_norm": 0.6837603186960344, + "language_loss": 0.50288999, + "learning_rate": 3.999984419658401e-06, + "loss": 0.52326787, + "num_input_tokens_seen": 29847120, + "router_z_loss_clip": 0.24902344, + "router_z_loss_mlp": 0.02307129, + "step": 1076, + "time_per_iteration": 5.371958255767822 + }, + { + "auxiliary_loss_clip": 0.01026752, + "auxiliary_loss_mlp": 0.01005534, + "balance_loss_clip": 1.00208604, + "balance_loss_mlp": 1.00320959, + "epoch": 0.03125181359178225, + "flos": 74772259873920.0, + "grad_norm": 0.7322715670593031, + "language_loss": 0.56369323, + "learning_rate": 3.999983668906002e-06, + "loss": 0.58401608, + "num_input_tokens_seen": 29913725, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.02319336, + "step": 1077, + "time_per_iteration": 3.1331284046173096 + }, + { + "auxiliary_loss_clip": 0.01026667, + "auxiliary_loss_mlp": 0.01006044, + "balance_loss_clip": 1.00204992, + "balance_loss_mlp": 1.00377929, + "epoch": 0.031280831060298296, + "flos": 65247185727360.0, + "grad_norm": 0.724557908755833, + "language_loss": 0.4618555, + "learning_rate": 3.999982900488937e-06, + "loss": 0.48218259, + "num_input_tokens_seen": 29972275, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.02270508, + "step": 1078, + "time_per_iteration": 3.104541301727295 + }, + { + "auxiliary_loss_clip": 0.01165194, + "auxiliary_loss_mlp": 0.01063774, + "balance_loss_clip": 1.04354358, + "balance_loss_mlp": 1.03283906, + "epoch": 0.03130984852881435, + "flos": 31971712746240.0, + "grad_norm": 2.5573019206785683, + "language_loss": 0.79354393, + "learning_rate": 3.999982114407214e-06, + "loss": 0.81583369, + "num_input_tokens_seen": 29987945, + "router_z_loss_clip": 1.21484375, + "router_z_loss_mlp": 0.30932617, + "step": 1079, + "time_per_iteration": 2.4740216732025146 + }, + { + "auxiliary_loss_clip": 0.0102529, + "auxiliary_loss_mlp": 0.01001656, + "balance_loss_clip": 1.00119376, + "balance_loss_mlp": 0.99948651, + "epoch": 0.03133886599733039, + "flos": 65410054686720.0, + "grad_norm": 0.6907859645966288, + "language_loss": 0.53218359, + "learning_rate": 3.999981310660839e-06, + "loss": 0.55245304, + "num_input_tokens_seen": 30051910, + "router_z_loss_clip": 0.24121094, + "router_z_loss_mlp": 0.02172852, + "step": 1080, + "time_per_iteration": 3.0884625911712646 + }, + { + "auxiliary_loss_clip": 0.01025618, + "auxiliary_loss_mlp": 0.01002262, + "balance_loss_clip": 1.00151718, + "balance_loss_mlp": 0.9999854, + "epoch": 0.03136788346584644, + "flos": 63306853073280.0, + "grad_norm": 0.7136656430783328, + "language_loss": 0.53078771, + "learning_rate": 3.99998048924982e-06, + "loss": 0.55106652, + "num_input_tokens_seen": 30110715, + "router_z_loss_clip": 0.24121094, + "router_z_loss_mlp": 0.02282715, + "step": 1081, + "time_per_iteration": 3.1636970043182373 + }, + { + "auxiliary_loss_clip": 0.01167845, + "auxiliary_loss_mlp": 0.01061283, + "balance_loss_clip": 1.04852545, + "balance_loss_mlp": 1.0346036, + "epoch": 0.03139690093436249, + "flos": 74728969787520.0, + "grad_norm": 2.68950936063222, + "language_loss": 0.88391113, + "learning_rate": 3.999979650174164e-06, + "loss": 0.90620244, + "num_input_tokens_seen": 30136620, + "router_z_loss_clip": 1.19335938, + "router_z_loss_mlp": 0.26708984, + "step": 1082, + "time_per_iteration": 2.8451523780822754 + }, + { + "auxiliary_loss_clip": 0.01177812, + "auxiliary_loss_mlp": 0.01058337, + "balance_loss_clip": 1.04678655, + "balance_loss_mlp": 1.02653146, + "epoch": 0.03142591840287853, + "flos": 36100226862720.0, + "grad_norm": 2.5731188786390713, + "language_loss": 1.04715192, + "learning_rate": 3.9999787934338785e-06, + "loss": 1.06951344, + "num_input_tokens_seen": 30157695, + "router_z_loss_clip": 1.31054688, + "router_z_loss_mlp": 0.31860352, + "step": 1083, + "time_per_iteration": 2.517889976501465 + }, + { + "auxiliary_loss_clip": 0.01170605, + "auxiliary_loss_mlp": 0.01065703, + "balance_loss_clip": 1.04553747, + "balance_loss_mlp": 1.03653216, + "epoch": 0.03145493587139458, + "flos": 15659208892800.0, + "grad_norm": 3.2793287211310638, + "language_loss": 0.97288197, + "learning_rate": 3.999977919028971e-06, + "loss": 0.99524498, + "num_input_tokens_seen": 30168260, + "router_z_loss_clip": 1.25048828, + "router_z_loss_mlp": 0.29174805, + "step": 1084, + "time_per_iteration": 2.492215633392334 + }, + { + "auxiliary_loss_clip": 0.01164016, + "auxiliary_loss_mlp": 0.01057053, + "balance_loss_clip": 1.04575431, + "balance_loss_mlp": 1.02847862, + "epoch": 0.03148395333991063, + "flos": 39486034293120.0, + "grad_norm": 2.218961330014959, + "language_loss": 0.99585021, + "learning_rate": 3.999977026959449e-06, + "loss": 1.01806092, + "num_input_tokens_seen": 30185235, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.28552246, + "step": 1085, + "time_per_iteration": 2.6024293899536133 + }, + { + "auxiliary_loss_clip": 0.01163434, + "auxiliary_loss_mlp": 0.01047054, + "balance_loss_clip": 1.04373312, + "balance_loss_mlp": 1.02078009, + "epoch": 0.031512970808426674, + "flos": 33685145274240.0, + "grad_norm": 3.339980859968336, + "language_loss": 0.84746546, + "learning_rate": 3.999976117225321e-06, + "loss": 0.86957037, + "num_input_tokens_seen": 30201055, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.26269531, + "step": 1086, + "time_per_iteration": 2.509643793106079 + }, + { + "auxiliary_loss_clip": 0.01164739, + "auxiliary_loss_mlp": 0.01057861, + "balance_loss_clip": 1.04434049, + "balance_loss_mlp": 1.02705753, + "epoch": 0.03154198827694272, + "flos": 21462436972800.0, + "grad_norm": 3.145635976562394, + "language_loss": 0.80750316, + "learning_rate": 3.999975189826594e-06, + "loss": 0.8297292, + "num_input_tokens_seen": 30214355, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.30810547, + "step": 1087, + "time_per_iteration": 2.4125783443450928 + }, + { + "auxiliary_loss_clip": 0.01170224, + "auxiliary_loss_mlp": 0.01052184, + "balance_loss_clip": 1.04563332, + "balance_loss_mlp": 1.02088022, + "epoch": 0.03157100574545876, + "flos": 29641015647360.0, + "grad_norm": 3.196297488137408, + "language_loss": 0.94886744, + "learning_rate": 3.9999742447632775e-06, + "loss": 0.97109157, + "num_input_tokens_seen": 30230100, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.31286621, + "step": 1088, + "time_per_iteration": 2.5409631729125977 + }, + { + "auxiliary_loss_clip": 0.0116357, + "auxiliary_loss_mlp": 0.01052196, + "balance_loss_clip": 1.04325807, + "balance_loss_mlp": 1.02295399, + "epoch": 0.031600023213974815, + "flos": 32810291625600.0, + "grad_norm": 3.2548950486037334, + "language_loss": 0.98545468, + "learning_rate": 3.99997328203538e-06, + "loss": 1.00761223, + "num_input_tokens_seen": 30249465, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.29248047, + "step": 1089, + "time_per_iteration": 2.5281901359558105 + }, + { + "auxiliary_loss_clip": 0.01167377, + "auxiliary_loss_mlp": 0.01051026, + "balance_loss_clip": 1.04365146, + "balance_loss_mlp": 1.02026987, + "epoch": 0.03162904068249086, + "flos": 23547379944960.0, + "grad_norm": 3.5882507115682256, + "language_loss": 0.95412797, + "learning_rate": 3.999972301642907e-06, + "loss": 0.97631192, + "num_input_tokens_seen": 30265955, + "router_z_loss_clip": 1.23730469, + "router_z_loss_mlp": 0.30725098, + "step": 1090, + "time_per_iteration": 2.505211591720581 + }, + { + "auxiliary_loss_clip": 0.01165967, + "auxiliary_loss_mlp": 0.01063652, + "balance_loss_clip": 1.04541445, + "balance_loss_mlp": 1.03327787, + "epoch": 0.031658058151006904, + "flos": 30696754383360.0, + "grad_norm": 3.0422287685084766, + "language_loss": 0.63076735, + "learning_rate": 3.999971303585871e-06, + "loss": 0.6530636, + "num_input_tokens_seen": 30282960, + "router_z_loss_clip": 1.20605469, + "router_z_loss_mlp": 0.30371094, + "step": 1091, + "time_per_iteration": 2.5057098865509033 + }, + { + "auxiliary_loss_clip": 0.01026346, + "auxiliary_loss_mlp": 0.01009099, + "balance_loss_clip": 1.00231457, + "balance_loss_mlp": 1.00683403, + "epoch": 0.031687075619522956, + "flos": 74590292534400.0, + "grad_norm": 0.7722108263420154, + "language_loss": 0.52622229, + "learning_rate": 3.999970287864279e-06, + "loss": 0.54657674, + "num_input_tokens_seen": 30344585, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.02270508, + "step": 1092, + "time_per_iteration": 3.090754747390747 + }, + { + "auxiliary_loss_clip": 0.01159727, + "auxiliary_loss_mlp": 0.01069225, + "balance_loss_clip": 1.04217911, + "balance_loss_mlp": 1.03737223, + "epoch": 0.031716093088039, + "flos": 33429441409920.0, + "grad_norm": 3.5635041644024943, + "language_loss": 0.87665737, + "learning_rate": 3.9999692544781385e-06, + "loss": 0.89894688, + "num_input_tokens_seen": 30360275, + "router_z_loss_clip": 1.17626953, + "router_z_loss_mlp": 0.3182373, + "step": 1093, + "time_per_iteration": 2.5096585750579834 + }, + { + "auxiliary_loss_clip": 0.01154217, + "auxiliary_loss_mlp": 0.01043845, + "balance_loss_clip": 1.04254055, + "balance_loss_mlp": 1.01982391, + "epoch": 0.031745110556555045, + "flos": 26206295535360.0, + "grad_norm": 2.579434129226873, + "language_loss": 0.93746436, + "learning_rate": 3.999968203427463e-06, + "loss": 0.959445, + "num_input_tokens_seen": 30377165, + "router_z_loss_clip": 1.11767578, + "router_z_loss_mlp": 0.24035645, + "step": 1094, + "time_per_iteration": 2.461799144744873 + }, + { + "auxiliary_loss_clip": 0.011563, + "auxiliary_loss_mlp": 0.01058125, + "balance_loss_clip": 1.04133105, + "balance_loss_mlp": 1.03031373, + "epoch": 0.03177412802507109, + "flos": 16688029104000.0, + "grad_norm": 3.2059387073406946, + "language_loss": 0.84345472, + "learning_rate": 3.999967134712257e-06, + "loss": 0.86559898, + "num_input_tokens_seen": 30391495, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.27819824, + "step": 1095, + "time_per_iteration": 2.690551280975342 + }, + { + "auxiliary_loss_clip": 0.01165257, + "auxiliary_loss_mlp": 0.01059199, + "balance_loss_clip": 1.04361689, + "balance_loss_mlp": 1.02698827, + "epoch": 0.03180314549358714, + "flos": 35547412394880.0, + "grad_norm": 2.2863147502546584, + "language_loss": 0.97679377, + "learning_rate": 3.999966048332532e-06, + "loss": 0.99903834, + "num_input_tokens_seen": 30408975, + "router_z_loss_clip": 1.21484375, + "router_z_loss_mlp": 0.32214355, + "step": 1096, + "time_per_iteration": 2.5115418434143066 + }, + { + "auxiliary_loss_clip": 0.01157684, + "auxiliary_loss_mlp": 0.01056852, + "balance_loss_clip": 1.04006422, + "balance_loss_mlp": 1.02771699, + "epoch": 0.031832162962103186, + "flos": 20074150736640.0, + "grad_norm": 3.988914861227707, + "language_loss": 0.90977842, + "learning_rate": 3.999964944288298e-06, + "loss": 0.93192375, + "num_input_tokens_seen": 30421450, + "router_z_loss_clip": 1.17480469, + "router_z_loss_mlp": 0.29138184, + "step": 1097, + "time_per_iteration": 2.358356237411499 + }, + { + "auxiliary_loss_clip": 0.01149796, + "auxiliary_loss_mlp": 0.01051787, + "balance_loss_clip": 1.04097378, + "balance_loss_mlp": 1.02713418, + "epoch": 0.03186118043061923, + "flos": 29417536834560.0, + "grad_norm": 1.8622738036512132, + "language_loss": 0.75634778, + "learning_rate": 3.999963822579565e-06, + "loss": 0.77836359, + "num_input_tokens_seen": 30437610, + "router_z_loss_clip": 1.08837891, + "router_z_loss_mlp": 0.24633789, + "step": 1098, + "time_per_iteration": 2.452526569366455 + }, + { + "auxiliary_loss_clip": 0.01028305, + "auxiliary_loss_mlp": 0.01006095, + "balance_loss_clip": 1.00342071, + "balance_loss_mlp": 1.00347197, + "epoch": 0.03189019789913528, + "flos": 58099591768320.0, + "grad_norm": 0.8797673517409375, + "language_loss": 0.53403288, + "learning_rate": 3.999962683206341e-06, + "loss": 0.5543769, + "num_input_tokens_seen": 30495365, + "router_z_loss_clip": 0.24902344, + "router_z_loss_mlp": 0.02624512, + "step": 1099, + "time_per_iteration": 2.9173200130462646 + }, + { + "auxiliary_loss_clip": 0.01154627, + "auxiliary_loss_mlp": 0.01042115, + "balance_loss_clip": 1.04156137, + "balance_loss_mlp": 1.01578176, + "epoch": 0.03191921536765133, + "flos": 10626164605440.0, + "grad_norm": 4.340202737343502, + "language_loss": 0.87146962, + "learning_rate": 3.999961526168638e-06, + "loss": 0.89343703, + "num_input_tokens_seen": 30505960, + "router_z_loss_clip": 1.12988281, + "router_z_loss_mlp": 0.26318359, + "step": 1100, + "time_per_iteration": 2.351982593536377 + }, + { + "auxiliary_loss_clip": 0.01028148, + "auxiliary_loss_mlp": 0.01005514, + "balance_loss_clip": 1.00321174, + "balance_loss_mlp": 1.00290287, + "epoch": 0.03194823283616737, + "flos": 74784409027200.0, + "grad_norm": 0.6799058347248863, + "language_loss": 0.49441639, + "learning_rate": 3.999960351466465e-06, + "loss": 0.51475298, + "num_input_tokens_seen": 30574655, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.02612305, + "step": 1101, + "time_per_iteration": 3.2080163955688477 + }, + { + "auxiliary_loss_clip": 0.01161325, + "auxiliary_loss_mlp": 0.01061474, + "balance_loss_clip": 1.04698968, + "balance_loss_mlp": 1.03458071, + "epoch": 0.031977250304683416, + "flos": 33330601751040.0, + "grad_norm": 2.651841759121738, + "language_loss": 1.02156341, + "learning_rate": 3.9999591590998334e-06, + "loss": 1.04379141, + "num_input_tokens_seen": 30592020, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.2689209, + "step": 1102, + "time_per_iteration": 2.5572333335876465 + }, + { + "auxiliary_loss_clip": 0.01151625, + "auxiliary_loss_mlp": 0.01056754, + "balance_loss_clip": 1.04044735, + "balance_loss_mlp": 1.02970541, + "epoch": 0.03200626777319947, + "flos": 16573723712640.0, + "grad_norm": 2.5757830637856016, + "language_loss": 0.80103534, + "learning_rate": 3.9999579490687525e-06, + "loss": 0.82311916, + "num_input_tokens_seen": 30604605, + "router_z_loss_clip": 1.11035156, + "router_z_loss_mlp": 0.27062988, + "step": 1103, + "time_per_iteration": 2.4117143154144287 + }, + { + "auxiliary_loss_clip": 0.01177761, + "auxiliary_loss_mlp": 0.01057657, + "balance_loss_clip": 1.04797196, + "balance_loss_mlp": 1.0262574, + "epoch": 0.03203528524171551, + "flos": 74735568034560.0, + "grad_norm": 2.2127400052282122, + "language_loss": 1.04275465, + "learning_rate": 3.999956721373235e-06, + "loss": 1.06510878, + "num_input_tokens_seen": 30634080, + "router_z_loss_clip": 1.29980469, + "router_z_loss_mlp": 0.3137207, + "step": 1104, + "time_per_iteration": 2.8782734870910645 + }, + { + "auxiliary_loss_clip": 0.01168813, + "auxiliary_loss_mlp": 0.01065057, + "balance_loss_clip": 1.04368615, + "balance_loss_mlp": 1.03673244, + "epoch": 0.03206430271023156, + "flos": 74738744968320.0, + "grad_norm": 2.144277758767941, + "language_loss": 0.76958996, + "learning_rate": 3.999955476013289e-06, + "loss": 0.79192865, + "num_input_tokens_seen": 30658905, + "router_z_loss_clip": 1.25, + "router_z_loss_mlp": 0.2833252, + "step": 1105, + "time_per_iteration": 2.9196910858154297 + }, + { + "auxiliary_loss_clip": 0.01027963, + "auxiliary_loss_mlp": 0.01007255, + "balance_loss_clip": 1.00333214, + "balance_loss_mlp": 1.00479913, + "epoch": 0.03209332017874761, + "flos": 70429937391360.0, + "grad_norm": 0.7070341154598054, + "language_loss": 0.53106999, + "learning_rate": 3.999954212988927e-06, + "loss": 0.55142224, + "num_input_tokens_seen": 30719175, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.02453613, + "step": 1106, + "time_per_iteration": 3.0002641677856445 + }, + { + "auxiliary_loss_clip": 0.01162718, + "auxiliary_loss_mlp": 0.01060173, + "balance_loss_clip": 1.04407024, + "balance_loss_mlp": 1.03002477, + "epoch": 0.03212233764726365, + "flos": 42333829672320.0, + "grad_norm": 2.2635712215046655, + "language_loss": 1.12020397, + "learning_rate": 3.999952932300161e-06, + "loss": 1.14243293, + "num_input_tokens_seen": 30746765, + "router_z_loss_clip": 1.1875, + "router_z_loss_mlp": 0.30151367, + "step": 1107, + "time_per_iteration": 2.7567319869995117 + }, + { + "auxiliary_loss_clip": 0.01161305, + "auxiliary_loss_mlp": 0.0106254, + "balance_loss_clip": 1.0421145, + "balance_loss_mlp": 1.0313549, + "epoch": 0.0321513551157797, + "flos": 12779258284800.0, + "grad_norm": 2.7739065855959346, + "language_loss": 0.90466744, + "learning_rate": 3.9999516339470015e-06, + "loss": 0.92690587, + "num_input_tokens_seen": 30760580, + "router_z_loss_clip": 1.19189453, + "router_z_loss_mlp": 0.31164551, + "step": 1108, + "time_per_iteration": 2.6573691368103027 + }, + { + "auxiliary_loss_clip": 0.01170547, + "auxiliary_loss_mlp": 0.01072063, + "balance_loss_clip": 1.0451088, + "balance_loss_mlp": 1.03942394, + "epoch": 0.03218037258429575, + "flos": 18906969340800.0, + "grad_norm": 4.109495730587151, + "language_loss": 0.90856528, + "learning_rate": 3.999950317929459e-06, + "loss": 0.93099129, + "num_input_tokens_seen": 30773240, + "router_z_loss_clip": 1.25488281, + "router_z_loss_mlp": 0.32629395, + "step": 1109, + "time_per_iteration": 2.396069288253784 + }, + { + "auxiliary_loss_clip": 0.01165771, + "auxiliary_loss_mlp": 0.01052731, + "balance_loss_clip": 1.04137993, + "balance_loss_mlp": 1.02568293, + "epoch": 0.032209390052811794, + "flos": 17924583104640.0, + "grad_norm": 2.499603654920861, + "language_loss": 0.86396044, + "learning_rate": 3.999948984247547e-06, + "loss": 0.88614541, + "num_input_tokens_seen": 30790030, + "router_z_loss_clip": 1.24414062, + "router_z_loss_mlp": 0.27062988, + "step": 1110, + "time_per_iteration": 2.3921778202056885 + }, + { + "auxiliary_loss_clip": 0.01183337, + "auxiliary_loss_mlp": 0.01080026, + "balance_loss_clip": 1.0532136, + "balance_loss_mlp": 1.04830432, + "epoch": 0.03223840752132784, + "flos": 44337425443200.0, + "grad_norm": 2.5634162836369367, + "language_loss": 0.95916402, + "learning_rate": 3.999947632901276e-06, + "loss": 0.9817977, + "num_input_tokens_seen": 30807085, + "router_z_loss_clip": 1.29931641, + "router_z_loss_mlp": 0.31713867, + "step": 1111, + "time_per_iteration": 2.608321189880371 + }, + { + "auxiliary_loss_clip": 0.01027083, + "auxiliary_loss_mlp": 0.01002386, + "balance_loss_clip": 1.00303841, + "balance_loss_mlp": 0.99995458, + "epoch": 0.03226742498984388, + "flos": 70625240870400.0, + "grad_norm": 0.676702664547308, + "language_loss": 0.55717921, + "learning_rate": 3.999946263890658e-06, + "loss": 0.57747394, + "num_input_tokens_seen": 30873065, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.02429199, + "step": 1112, + "time_per_iteration": 3.168478012084961 + }, + { + "auxiliary_loss_clip": 0.01164355, + "auxiliary_loss_mlp": 0.01057491, + "balance_loss_clip": 1.04427385, + "balance_loss_mlp": 1.02805781, + "epoch": 0.032296442458359935, + "flos": 20811306516480.0, + "grad_norm": 2.179711567606258, + "language_loss": 0.91055655, + "learning_rate": 3.999944877215704e-06, + "loss": 0.93277502, + "num_input_tokens_seen": 30892150, + "router_z_loss_clip": 1.20214844, + "router_z_loss_mlp": 0.29443359, + "step": 1113, + "time_per_iteration": 2.406583547592163 + }, + { + "auxiliary_loss_clip": 0.01168681, + "auxiliary_loss_mlp": 0.01066913, + "balance_loss_clip": 1.04572666, + "balance_loss_mlp": 1.03601348, + "epoch": 0.03232545992687598, + "flos": 25259276373120.0, + "grad_norm": 4.662208841319127, + "language_loss": 0.70439422, + "learning_rate": 3.99994347287643e-06, + "loss": 0.72675014, + "num_input_tokens_seen": 30907795, + "router_z_loss_clip": 1.22900391, + "router_z_loss_mlp": 0.30883789, + "step": 1114, + "time_per_iteration": 2.4684298038482666 + }, + { + "auxiliary_loss_clip": 0.01026577, + "auxiliary_loss_mlp": 0.01001086, + "balance_loss_clip": 1.0028336, + "balance_loss_mlp": 0.99877352, + "epoch": 0.032354477395392024, + "flos": 60422782924800.0, + "grad_norm": 0.7196203323371569, + "language_loss": 0.47946858, + "learning_rate": 3.999942050872844e-06, + "loss": 0.49974519, + "num_input_tokens_seen": 30961055, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.02307129, + "step": 1115, + "time_per_iteration": 2.9003961086273193 + }, + { + "auxiliary_loss_clip": 0.01177354, + "auxiliary_loss_mlp": 0.01067815, + "balance_loss_clip": 1.05059612, + "balance_loss_mlp": 1.03316116, + "epoch": 0.032383494863908076, + "flos": 26133152503680.0, + "grad_norm": 3.6469060684345758, + "language_loss": 0.97338074, + "learning_rate": 3.999940611204961e-06, + "loss": 0.9958325, + "num_input_tokens_seen": 30976655, + "router_z_loss_clip": 1.26660156, + "router_z_loss_mlp": 0.34667969, + "step": 1116, + "time_per_iteration": 2.545091390609741 + }, + { + "auxiliary_loss_clip": 0.01161633, + "auxiliary_loss_mlp": 0.0105848, + "balance_loss_clip": 1.04121685, + "balance_loss_mlp": 1.02996469, + "epoch": 0.03241251233242412, + "flos": 16425412790400.0, + "grad_norm": 3.7940028177999117, + "language_loss": 0.91075575, + "learning_rate": 3.999939153872793e-06, + "loss": 0.93295681, + "num_input_tokens_seen": 30986590, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.28503418, + "step": 1117, + "time_per_iteration": 2.3533644676208496 + }, + { + "auxiliary_loss_clip": 0.01171011, + "auxiliary_loss_mlp": 0.01069007, + "balance_loss_clip": 1.04455996, + "balance_loss_mlp": 1.04049182, + "epoch": 0.032441529800940165, + "flos": 33395086765440.0, + "grad_norm": 3.1921038876077503, + "language_loss": 1.29113197, + "learning_rate": 3.999937678876355e-06, + "loss": 1.31353235, + "num_input_tokens_seen": 31007615, + "router_z_loss_clip": 1.26660156, + "router_z_loss_mlp": 0.28527832, + "step": 1118, + "time_per_iteration": 2.596472978591919 + }, + { + "auxiliary_loss_clip": 0.01172224, + "auxiliary_loss_mlp": 0.01062724, + "balance_loss_clip": 1.04557908, + "balance_loss_mlp": 1.03062105, + "epoch": 0.03247054726945621, + "flos": 33137881712640.0, + "grad_norm": 3.2314363187907342, + "language_loss": 0.88350081, + "learning_rate": 3.9999361862156565e-06, + "loss": 0.90585029, + "num_input_tokens_seen": 31024700, + "router_z_loss_clip": 1.26757812, + "router_z_loss_mlp": 0.32080078, + "step": 1119, + "time_per_iteration": 2.526601552963257 + }, + { + "auxiliary_loss_clip": 0.01164353, + "auxiliary_loss_mlp": 0.0105571, + "balance_loss_clip": 1.04174042, + "balance_loss_mlp": 1.02371478, + "epoch": 0.03249956473797226, + "flos": 11940051000960.0, + "grad_norm": 6.107712040274142, + "language_loss": 0.87003982, + "learning_rate": 3.999934675890713e-06, + "loss": 0.89224041, + "num_input_tokens_seen": 31037760, + "router_z_loss_clip": 1.22558594, + "router_z_loss_mlp": 0.32006836, + "step": 1120, + "time_per_iteration": 2.3773651123046875 + }, + { + "auxiliary_loss_clip": 0.01158592, + "auxiliary_loss_mlp": 0.01061257, + "balance_loss_clip": 1.04483557, + "balance_loss_mlp": 1.03217018, + "epoch": 0.032528582206488306, + "flos": 26098518568320.0, + "grad_norm": 2.3451173451488665, + "language_loss": 0.87453318, + "learning_rate": 3.999933147901536e-06, + "loss": 0.89673167, + "num_input_tokens_seen": 31052295, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.2911377, + "step": 1121, + "time_per_iteration": 2.7491862773895264 + }, + { + "auxiliary_loss_clip": 0.01030356, + "auxiliary_loss_mlp": 0.01009316, + "balance_loss_clip": 1.0057404, + "balance_loss_mlp": 1.00703883, + "epoch": 0.03255759967500435, + "flos": 65714184339840.0, + "grad_norm": 0.7698806494712126, + "language_loss": 0.55231863, + "learning_rate": 3.999931602248141e-06, + "loss": 0.57271534, + "num_input_tokens_seen": 31107880, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.02282715, + "step": 1122, + "time_per_iteration": 2.928762197494507 + }, + { + "auxiliary_loss_clip": 0.01153495, + "auxiliary_loss_mlp": 0.01057736, + "balance_loss_clip": 1.03951347, + "balance_loss_mlp": 1.02994788, + "epoch": 0.0325866171435204, + "flos": 33647648607360.0, + "grad_norm": 3.2559532869618777, + "language_loss": 1.0036937, + "learning_rate": 3.999930038930541e-06, + "loss": 1.02580595, + "num_input_tokens_seen": 31122910, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.27783203, + "step": 1123, + "time_per_iteration": 2.509187698364258 + }, + { + "auxiliary_loss_clip": 0.01028204, + "auxiliary_loss_mlp": 0.01002099, + "balance_loss_clip": 1.0040977, + "balance_loss_mlp": 0.99966675, + "epoch": 0.03261563461203645, + "flos": 69699414769920.0, + "grad_norm": 0.6868804104406864, + "language_loss": 0.52343917, + "learning_rate": 3.999928457948749e-06, + "loss": 0.54374218, + "num_input_tokens_seen": 31187025, + "router_z_loss_clip": 0.24121094, + "router_z_loss_mlp": 0.02429199, + "step": 1124, + "time_per_iteration": 3.0339109897613525 + }, + { + "auxiliary_loss_clip": 0.01027948, + "auxiliary_loss_mlp": 0.01003303, + "balance_loss_clip": 1.00406241, + "balance_loss_mlp": 1.00066853, + "epoch": 0.03264465208055249, + "flos": 67912664653440.0, + "grad_norm": 0.7135780437392357, + "language_loss": 0.5136646, + "learning_rate": 3.99992685930278e-06, + "loss": 0.53397709, + "num_input_tokens_seen": 31252435, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.02636719, + "step": 1125, + "time_per_iteration": 3.101377248764038 + }, + { + "auxiliary_loss_clip": 0.01165319, + "auxiliary_loss_mlp": 0.01061627, + "balance_loss_clip": 1.04714394, + "balance_loss_mlp": 1.03318334, + "epoch": 0.032673669549068536, + "flos": 25841592806400.0, + "grad_norm": 2.9810207838200697, + "language_loss": 0.94303083, + "learning_rate": 3.9999252429926475e-06, + "loss": 0.96530032, + "num_input_tokens_seen": 31267990, + "router_z_loss_clip": 1.18066406, + "router_z_loss_mlp": 0.28393555, + "step": 1126, + "time_per_iteration": 2.6719319820404053 + }, + { + "auxiliary_loss_clip": 0.01027283, + "auxiliary_loss_mlp": 0.01003489, + "balance_loss_clip": 1.00346112, + "balance_loss_mlp": 1.00085485, + "epoch": 0.03270268701758459, + "flos": 62047046240640.0, + "grad_norm": 0.7065341141845808, + "language_loss": 0.54930556, + "learning_rate": 3.999923609018365e-06, + "loss": 0.56961334, + "num_input_tokens_seen": 31332135, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.02636719, + "step": 1127, + "time_per_iteration": 3.1063995361328125 + }, + { + "auxiliary_loss_clip": 0.01158355, + "auxiliary_loss_mlp": 0.01055497, + "balance_loss_clip": 1.04509592, + "balance_loss_mlp": 1.02695799, + "epoch": 0.03273170448610063, + "flos": 25330988039040.0, + "grad_norm": 2.2882300038091885, + "language_loss": 0.78373754, + "learning_rate": 3.99992195737995e-06, + "loss": 0.80587608, + "num_input_tokens_seen": 31346340, + "router_z_loss_clip": 1.13183594, + "router_z_loss_mlp": 0.28527832, + "step": 1128, + "time_per_iteration": 2.4700186252593994 + }, + { + "auxiliary_loss_clip": 0.01169184, + "auxiliary_loss_mlp": 0.01056213, + "balance_loss_clip": 1.04624486, + "balance_loss_mlp": 1.02452707, + "epoch": 0.03276072195461668, + "flos": 45251905351680.0, + "grad_norm": 2.8184446080837704, + "language_loss": 0.97445065, + "learning_rate": 3.999920288077414e-06, + "loss": 0.99670458, + "num_input_tokens_seen": 31366845, + "router_z_loss_clip": 1.22949219, + "router_z_loss_mlp": 0.31665039, + "step": 1129, + "time_per_iteration": 2.6740646362304688 + }, + { + "auxiliary_loss_clip": 0.01159582, + "auxiliary_loss_mlp": 0.01075747, + "balance_loss_clip": 1.04759169, + "balance_loss_mlp": 1.04907954, + "epoch": 0.03278973942313273, + "flos": 35619228794880.0, + "grad_norm": 2.402838783568027, + "language_loss": 0.83062357, + "learning_rate": 3.999918601110772e-06, + "loss": 0.85297686, + "num_input_tokens_seen": 31385380, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.2668457, + "step": 1130, + "time_per_iteration": 2.543963670730591 + }, + { + "auxiliary_loss_clip": 0.01176207, + "auxiliary_loss_mlp": 0.01071818, + "balance_loss_clip": 1.05202401, + "balance_loss_mlp": 1.04237342, + "epoch": 0.03281875689164877, + "flos": 14349546771840.0, + "grad_norm": 2.513405891903851, + "language_loss": 0.71124876, + "learning_rate": 3.99991689648004e-06, + "loss": 0.73372906, + "num_input_tokens_seen": 31400620, + "router_z_loss_clip": 1.24316406, + "router_z_loss_mlp": 0.29443359, + "step": 1131, + "time_per_iteration": 2.4552931785583496 + }, + { + "auxiliary_loss_clip": 0.01158755, + "auxiliary_loss_mlp": 0.010596, + "balance_loss_clip": 1.04532862, + "balance_loss_mlp": 1.03242064, + "epoch": 0.03284777436016482, + "flos": 29604810700800.0, + "grad_norm": 2.3661415692685694, + "language_loss": 1.04930854, + "learning_rate": 3.999915174185233e-06, + "loss": 1.07149196, + "num_input_tokens_seen": 31419070, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.27148438, + "step": 1132, + "time_per_iteration": 2.505023241043091 + }, + { + "auxiliary_loss_clip": 0.01169832, + "auxiliary_loss_mlp": 0.01064216, + "balance_loss_clip": 1.04896307, + "balance_loss_mlp": 1.03503323, + "epoch": 0.03287679182868086, + "flos": 20769445929600.0, + "grad_norm": 2.927013596938861, + "language_loss": 0.63478398, + "learning_rate": 3.999913434226366e-06, + "loss": 0.6571244, + "num_input_tokens_seen": 31433095, + "router_z_loss_clip": 1.20800781, + "router_z_loss_mlp": 0.29199219, + "step": 1133, + "time_per_iteration": 2.482492208480835 + }, + { + "auxiliary_loss_clip": 0.01177086, + "auxiliary_loss_mlp": 0.01064945, + "balance_loss_clip": 1.05211437, + "balance_loss_mlp": 1.03020763, + "epoch": 0.032905809297196914, + "flos": 18763127072640.0, + "grad_norm": 2.2902530911804986, + "language_loss": 1.03896308, + "learning_rate": 3.999911676603454e-06, + "loss": 1.06138337, + "num_input_tokens_seen": 31450930, + "router_z_loss_clip": 1.24902344, + "router_z_loss_mlp": 0.34753418, + "step": 1134, + "time_per_iteration": 2.8048102855682373 + }, + { + "auxiliary_loss_clip": 0.01166471, + "auxiliary_loss_mlp": 0.01059515, + "balance_loss_clip": 1.04654121, + "balance_loss_mlp": 1.02968895, + "epoch": 0.03293482676571296, + "flos": 17413454666880.0, + "grad_norm": 2.7759140624301297, + "language_loss": 0.90351582, + "learning_rate": 3.999909901316513e-06, + "loss": 0.92577565, + "num_input_tokens_seen": 31464330, + "router_z_loss_clip": 1.20019531, + "router_z_loss_mlp": 0.2980957, + "step": 1135, + "time_per_iteration": 2.3788483142852783 + }, + { + "auxiliary_loss_clip": 0.01178266, + "auxiliary_loss_mlp": 0.01068896, + "balance_loss_clip": 1.05059338, + "balance_loss_mlp": 1.035851, + "epoch": 0.032963844234229, + "flos": 22627384041600.0, + "grad_norm": 2.3927733545080168, + "language_loss": 0.90589154, + "learning_rate": 3.999908108365559e-06, + "loss": 0.9283632, + "num_input_tokens_seen": 31478310, + "router_z_loss_clip": 1.27636719, + "router_z_loss_mlp": 0.32995605, + "step": 1136, + "time_per_iteration": 2.427126169204712 + }, + { + "auxiliary_loss_clip": 0.0103411, + "auxiliary_loss_mlp": 0.01004433, + "balance_loss_clip": 1.0086242, + "balance_loss_mlp": 1.0018698, + "epoch": 0.032992861702745055, + "flos": 74775471719040.0, + "grad_norm": 0.764053607580345, + "language_loss": 0.5637961, + "learning_rate": 3.999906297750607e-06, + "loss": 0.58418155, + "num_input_tokens_seen": 31545475, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.02563477, + "step": 1137, + "time_per_iteration": 3.1066999435424805 + }, + { + "auxiliary_loss_clip": 0.01032077, + "auxiliary_loss_mlp": 0.01002738, + "balance_loss_clip": 1.00658417, + "balance_loss_mlp": 1.00024652, + "epoch": 0.0330218791712611, + "flos": 67619254654080.0, + "grad_norm": 0.6820303888472745, + "language_loss": 0.52282041, + "learning_rate": 3.999904469471672e-06, + "loss": 0.5431686, + "num_input_tokens_seen": 31606580, + "router_z_loss_clip": 0.25390625, + "router_z_loss_mlp": 0.02490234, + "step": 1138, + "time_per_iteration": 2.973233938217163 + }, + { + "auxiliary_loss_clip": 0.01156893, + "auxiliary_loss_mlp": 0.01053558, + "balance_loss_clip": 1.04579651, + "balance_loss_mlp": 1.02739215, + "epoch": 0.033050896639777144, + "flos": 34887938123520.0, + "grad_norm": 3.823735100201764, + "language_loss": 0.79156816, + "learning_rate": 3.9999026235287725e-06, + "loss": 0.81367266, + "num_input_tokens_seen": 31621655, + "router_z_loss_clip": 1.11083984, + "router_z_loss_mlp": 0.26171875, + "step": 1139, + "time_per_iteration": 2.5890042781829834 + }, + { + "auxiliary_loss_clip": 0.01166053, + "auxiliary_loss_mlp": 0.01054492, + "balance_loss_clip": 1.04637885, + "balance_loss_mlp": 1.02702594, + "epoch": 0.033079914108293196, + "flos": 27409053473280.0, + "grad_norm": 1.9631484025151023, + "language_loss": 0.868007, + "learning_rate": 3.999900759921924e-06, + "loss": 0.89021254, + "num_input_tokens_seen": 31635030, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.2746582, + "step": 1140, + "time_per_iteration": 2.444671630859375 + }, + { + "auxiliary_loss_clip": 0.01175869, + "auxiliary_loss_mlp": 0.01076024, + "balance_loss_clip": 1.05467379, + "balance_loss_mlp": 1.04477894, + "epoch": 0.03310893157680924, + "flos": 10991076802560.0, + "grad_norm": 4.58379649023145, + "language_loss": 0.89138258, + "learning_rate": 3.999898878651142e-06, + "loss": 0.91390145, + "num_input_tokens_seen": 31647265, + "router_z_loss_clip": 1.2109375, + "router_z_loss_mlp": 0.31225586, + "step": 1141, + "time_per_iteration": 2.423398494720459 + }, + { + "auxiliary_loss_clip": 0.01168066, + "auxiliary_loss_mlp": 0.01071555, + "balance_loss_clip": 1.04829323, + "balance_loss_mlp": 1.04134727, + "epoch": 0.033137949045325285, + "flos": 15880837403520.0, + "grad_norm": 2.703996765164102, + "language_loss": 0.60345674, + "learning_rate": 3.999896979716444e-06, + "loss": 0.625853, + "num_input_tokens_seen": 31661660, + "router_z_loss_clip": 1.19824219, + "router_z_loss_mlp": 0.30187988, + "step": 1142, + "time_per_iteration": 2.370190143585205 + }, + { + "auxiliary_loss_clip": 0.01032176, + "auxiliary_loss_mlp": 0.01009239, + "balance_loss_clip": 1.00644433, + "balance_loss_mlp": 1.00653338, + "epoch": 0.03316696651384133, + "flos": 74772190051200.0, + "grad_norm": 0.6539364691752418, + "language_loss": 0.50123739, + "learning_rate": 3.999895063117847e-06, + "loss": 0.52165151, + "num_input_tokens_seen": 31727900, + "router_z_loss_clip": 0.2578125, + "router_z_loss_mlp": 0.02709961, + "step": 1143, + "time_per_iteration": 3.0860800743103027 + }, + { + "auxiliary_loss_clip": 0.01162998, + "auxiliary_loss_mlp": 0.01060124, + "balance_loss_clip": 1.04452682, + "balance_loss_mlp": 1.03088164, + "epoch": 0.03319598398235738, + "flos": 16246866764160.0, + "grad_norm": 5.907477943686975, + "language_loss": 0.81849247, + "learning_rate": 3.999893128855368e-06, + "loss": 0.84072363, + "num_input_tokens_seen": 31742720, + "router_z_loss_clip": 1.18359375, + "router_z_loss_mlp": 0.29248047, + "step": 1144, + "time_per_iteration": 4.928346633911133 + }, + { + "auxiliary_loss_clip": 0.01167403, + "auxiliary_loss_mlp": 0.01064757, + "balance_loss_clip": 1.04787409, + "balance_loss_mlp": 1.03499031, + "epoch": 0.033225001450873426, + "flos": 33829406478720.0, + "grad_norm": 2.465979597551392, + "language_loss": 0.89791429, + "learning_rate": 3.999891176929023e-06, + "loss": 0.92023587, + "num_input_tokens_seen": 31759635, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.29748535, + "step": 1145, + "time_per_iteration": 2.5303475856781006 + }, + { + "auxiliary_loss_clip": 0.01164436, + "auxiliary_loss_mlp": 0.01061836, + "balance_loss_clip": 1.04411554, + "balance_loss_mlp": 1.03162861, + "epoch": 0.03325401891938947, + "flos": 45506561875200.0, + "grad_norm": 2.399356772626893, + "language_loss": 0.78199822, + "learning_rate": 3.999889207338829e-06, + "loss": 0.80426085, + "num_input_tokens_seen": 31777360, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.30187988, + "step": 1146, + "time_per_iteration": 4.991508960723877 + }, + { + "auxiliary_loss_clip": 0.01181467, + "auxiliary_loss_mlp": 0.01066187, + "balance_loss_clip": 1.0520184, + "balance_loss_mlp": 1.03371429, + "epoch": 0.03328303638790552, + "flos": 31021062802560.0, + "grad_norm": 2.477452239367711, + "language_loss": 0.93412656, + "learning_rate": 3.999887220084805e-06, + "loss": 0.95660311, + "num_input_tokens_seen": 31792970, + "router_z_loss_clip": 1.29589844, + "router_z_loss_mlp": 0.32495117, + "step": 1147, + "time_per_iteration": 2.4780101776123047 + }, + { + "auxiliary_loss_clip": 0.01175661, + "auxiliary_loss_mlp": 0.01058532, + "balance_loss_clip": 1.04704273, + "balance_loss_mlp": 1.02386594, + "epoch": 0.03331205385642157, + "flos": 29671250751360.0, + "grad_norm": 2.7368109576845017, + "language_loss": 0.84713292, + "learning_rate": 3.999885215166969e-06, + "loss": 0.86947489, + "num_input_tokens_seen": 31808890, + "router_z_loss_clip": 1.28710938, + "router_z_loss_mlp": 0.34692383, + "step": 1148, + "time_per_iteration": 2.4572153091430664 + }, + { + "auxiliary_loss_clip": 0.01170436, + "auxiliary_loss_mlp": 0.01055487, + "balance_loss_clip": 1.04735994, + "balance_loss_mlp": 1.02647185, + "epoch": 0.03334107132493761, + "flos": 33586584906240.0, + "grad_norm": 1.7614206878082277, + "language_loss": 0.89848959, + "learning_rate": 3.999883192585336e-06, + "loss": 0.92074889, + "num_input_tokens_seen": 31835620, + "router_z_loss_clip": 1.22949219, + "router_z_loss_mlp": 0.2902832, + "step": 1149, + "time_per_iteration": 2.6463117599487305 + }, + { + "auxiliary_loss_clip": 0.01171184, + "auxiliary_loss_mlp": 0.01054906, + "balance_loss_clip": 1.048033, + "balance_loss_mlp": 1.02461505, + "epoch": 0.033370088793453656, + "flos": 30654754151040.0, + "grad_norm": 3.0350181488384864, + "language_loss": 0.88023651, + "learning_rate": 3.999881152339926e-06, + "loss": 0.90249741, + "num_input_tokens_seen": 31849535, + "router_z_loss_clip": 1.23144531, + "router_z_loss_mlp": 0.30310059, + "step": 1150, + "time_per_iteration": 4.920022964477539 + }, + { + "auxiliary_loss_clip": 0.01031774, + "auxiliary_loss_mlp": 0.01003444, + "balance_loss_clip": 1.00620675, + "balance_loss_mlp": 1.00012958, + "epoch": 0.03339910626196971, + "flos": 64476722643840.0, + "grad_norm": 0.6915746339374454, + "language_loss": 0.4794625, + "learning_rate": 3.999879094430756e-06, + "loss": 0.49981469, + "num_input_tokens_seen": 31906495, + "router_z_loss_clip": 0.25585938, + "router_z_loss_mlp": 0.03320312, + "step": 1151, + "time_per_iteration": 5.43794059753418 + }, + { + "auxiliary_loss_clip": 0.01158645, + "auxiliary_loss_mlp": 0.01059144, + "balance_loss_clip": 1.0447619, + "balance_loss_mlp": 1.03074825, + "epoch": 0.03342812373048575, + "flos": 29526780078720.0, + "grad_norm": 3.0201527072588097, + "language_loss": 0.9177109, + "learning_rate": 3.999877018857844e-06, + "loss": 0.93988872, + "num_input_tokens_seen": 31921320, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.28369141, + "step": 1152, + "time_per_iteration": 2.484264612197876 + }, + { + "auxiliary_loss_clip": 0.01165379, + "auxiliary_loss_mlp": 0.01060667, + "balance_loss_clip": 1.04885459, + "balance_loss_mlp": 1.03116238, + "epoch": 0.0334571411990018, + "flos": 32626578718080.0, + "grad_norm": 3.668572787794497, + "language_loss": 0.93242639, + "learning_rate": 3.99987492562121e-06, + "loss": 0.95468688, + "num_input_tokens_seen": 31936155, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.29528809, + "step": 1153, + "time_per_iteration": 2.586482524871826 + }, + { + "auxiliary_loss_clip": 0.01029136, + "auxiliary_loss_mlp": 0.01008925, + "balance_loss_clip": 1.00406504, + "balance_loss_mlp": 1.00619507, + "epoch": 0.03348615866751785, + "flos": 64414437045120.0, + "grad_norm": 0.7029751795881977, + "language_loss": 0.47986883, + "learning_rate": 3.999872814720871e-06, + "loss": 0.50024945, + "num_input_tokens_seen": 31996195, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.02734375, + "step": 1154, + "time_per_iteration": 2.9886081218719482 + }, + { + "auxiliary_loss_clip": 0.01161948, + "auxiliary_loss_mlp": 0.01067679, + "balance_loss_clip": 1.04644775, + "balance_loss_mlp": 1.03700662, + "epoch": 0.03351517613603389, + "flos": 25987320288000.0, + "grad_norm": 2.48594167103639, + "language_loss": 0.76099628, + "learning_rate": 3.999870686156846e-06, + "loss": 0.78329253, + "num_input_tokens_seen": 32012965, + "router_z_loss_clip": 1.15332031, + "router_z_loss_mlp": 0.3067627, + "step": 1155, + "time_per_iteration": 2.4969370365142822 + }, + { + "auxiliary_loss_clip": 0.01168102, + "auxiliary_loss_mlp": 0.01050057, + "balance_loss_clip": 1.04914916, + "balance_loss_mlp": 1.02333081, + "epoch": 0.03354419360454994, + "flos": 19893126003840.0, + "grad_norm": 2.5135381500984098, + "language_loss": 0.88217819, + "learning_rate": 3.999868539929154e-06, + "loss": 0.90435982, + "num_input_tokens_seen": 32033140, + "router_z_loss_clip": 1.18945312, + "router_z_loss_mlp": 0.26708984, + "step": 1156, + "time_per_iteration": 2.5067617893218994 + }, + { + "auxiliary_loss_clip": 0.01028412, + "auxiliary_loss_mlp": 0.01005047, + "balance_loss_clip": 1.0038662, + "balance_loss_mlp": 1.00215065, + "epoch": 0.03357321107306598, + "flos": 56010284876160.0, + "grad_norm": 0.6623151966704665, + "language_loss": 0.55262911, + "learning_rate": 3.999866376037814e-06, + "loss": 0.57296371, + "num_input_tokens_seen": 32088540, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.02893066, + "step": 1157, + "time_per_iteration": 3.036309003829956 + }, + { + "auxiliary_loss_clip": 0.01173246, + "auxiliary_loss_mlp": 0.01063976, + "balance_loss_clip": 1.04956388, + "balance_loss_mlp": 1.03157449, + "epoch": 0.033602228541582034, + "flos": 18689355636480.0, + "grad_norm": 2.72491453585972, + "language_loss": 0.94010693, + "learning_rate": 3.999864194482844e-06, + "loss": 0.96247917, + "num_input_tokens_seen": 32101410, + "router_z_loss_clip": 1.23828125, + "router_z_loss_mlp": 0.32397461, + "step": 1158, + "time_per_iteration": 2.5034637451171875 + }, + { + "auxiliary_loss_clip": 0.01157032, + "auxiliary_loss_mlp": 0.01055807, + "balance_loss_clip": 1.04481912, + "balance_loss_mlp": 1.02770972, + "epoch": 0.03363124601009808, + "flos": 15407589657600.0, + "grad_norm": 2.620870862489153, + "language_loss": 0.81635952, + "learning_rate": 3.999861995264266e-06, + "loss": 0.83848792, + "num_input_tokens_seen": 32116305, + "router_z_loss_clip": 1.12109375, + "router_z_loss_mlp": 0.28088379, + "step": 1159, + "time_per_iteration": 2.442650556564331 + }, + { + "auxiliary_loss_clip": 0.01172309, + "auxiliary_loss_mlp": 0.01070296, + "balance_loss_clip": 1.05274951, + "balance_loss_mlp": 1.04179287, + "epoch": 0.03366026347861412, + "flos": 19749877228800.0, + "grad_norm": 2.4160473895634973, + "language_loss": 0.83795434, + "learning_rate": 3.999859778382096e-06, + "loss": 0.86038041, + "num_input_tokens_seen": 32133150, + "router_z_loss_clip": 1.19482422, + "router_z_loss_mlp": 0.28503418, + "step": 1160, + "time_per_iteration": 2.4890248775482178 + }, + { + "auxiliary_loss_clip": 0.01155278, + "auxiliary_loss_mlp": 0.01056143, + "balance_loss_clip": 1.0433588, + "balance_loss_mlp": 1.03056026, + "epoch": 0.033689280947130175, + "flos": 35472628529280.0, + "grad_norm": 2.711481562521657, + "language_loss": 1.01829731, + "learning_rate": 3.9998575438363555e-06, + "loss": 1.04041159, + "num_input_tokens_seen": 32149280, + "router_z_loss_clip": 1.11767578, + "router_z_loss_mlp": 0.25610352, + "step": 1161, + "time_per_iteration": 2.6275415420532227 + }, + { + "auxiliary_loss_clip": 0.0103385, + "auxiliary_loss_mlp": 0.01005548, + "balance_loss_clip": 1.00925457, + "balance_loss_mlp": 1.0027945, + "epoch": 0.03371829841564622, + "flos": 54082415577600.0, + "grad_norm": 0.7437033138692717, + "language_loss": 0.51287901, + "learning_rate": 3.999855291627064e-06, + "loss": 0.53327298, + "num_input_tokens_seen": 32203435, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.02758789, + "step": 1162, + "time_per_iteration": 2.9323503971099854 + }, + { + "auxiliary_loss_clip": 0.0118361, + "auxiliary_loss_mlp": 0.01075987, + "balance_loss_clip": 1.05158353, + "balance_loss_mlp": 1.04347825, + "epoch": 0.033747315884162264, + "flos": 12816999331200.0, + "grad_norm": 2.287882715927554, + "language_loss": 0.77588558, + "learning_rate": 3.999853021754241e-06, + "loss": 0.79848152, + "num_input_tokens_seen": 32218330, + "router_z_loss_clip": 1.3203125, + "router_z_loss_mlp": 0.32495117, + "step": 1163, + "time_per_iteration": 2.3819334506988525 + }, + { + "auxiliary_loss_clip": 0.01027827, + "auxiliary_loss_mlp": 0.01005163, + "balance_loss_clip": 1.00377369, + "balance_loss_mlp": 1.00275469, + "epoch": 0.033776333352678316, + "flos": 55280704861440.0, + "grad_norm": 0.7421025522691524, + "language_loss": 0.57659626, + "learning_rate": 3.999850734217907e-06, + "loss": 0.59692615, + "num_input_tokens_seen": 32274570, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.02404785, + "step": 1164, + "time_per_iteration": 2.8718080520629883 + }, + { + "auxiliary_loss_clip": 0.01026773, + "auxiliary_loss_mlp": 0.01002727, + "balance_loss_clip": 1.00271249, + "balance_loss_mlp": 1.00037849, + "epoch": 0.03380535082119436, + "flos": 62002986238080.0, + "grad_norm": 0.6733403491570668, + "language_loss": 0.50308514, + "learning_rate": 3.999848429018082e-06, + "loss": 0.52338016, + "num_input_tokens_seen": 32335785, + "router_z_loss_clip": 0.24023438, + "router_z_loss_mlp": 0.0234375, + "step": 1165, + "time_per_iteration": 2.9297173023223877 + }, + { + "auxiliary_loss_clip": 0.01167816, + "auxiliary_loss_mlp": 0.01056302, + "balance_loss_clip": 1.04851031, + "balance_loss_mlp": 1.02807331, + "epoch": 0.033834368289710405, + "flos": 67432471413120.0, + "grad_norm": 2.6876738322066873, + "language_loss": 0.81878591, + "learning_rate": 3.999846106154787e-06, + "loss": 0.84102714, + "num_input_tokens_seen": 32357945, + "router_z_loss_clip": 1.19287109, + "router_z_loss_mlp": 0.28210449, + "step": 1166, + "time_per_iteration": 2.740899085998535 + }, + { + "auxiliary_loss_clip": 0.01161549, + "auxiliary_loss_mlp": 0.01057221, + "balance_loss_clip": 1.04523659, + "balance_loss_mlp": 1.02888489, + "epoch": 0.03386338575822645, + "flos": 29125209087360.0, + "grad_norm": 3.057645956089445, + "language_loss": 0.8583411, + "learning_rate": 3.9998437656280415e-06, + "loss": 0.88052881, + "num_input_tokens_seen": 32374245, + "router_z_loss_clip": 1.16357422, + "router_z_loss_mlp": 0.28356934, + "step": 1167, + "time_per_iteration": 2.475385904312134 + }, + { + "auxiliary_loss_clip": 0.01165699, + "auxiliary_loss_mlp": 0.01061501, + "balance_loss_clip": 1.04794502, + "balance_loss_mlp": 1.0334394, + "epoch": 0.0338924032267425, + "flos": 15588719124480.0, + "grad_norm": 2.5596044706474452, + "language_loss": 0.74587476, + "learning_rate": 3.999841407437867e-06, + "loss": 0.76814675, + "num_input_tokens_seen": 32387125, + "router_z_loss_clip": 1.17675781, + "router_z_loss_mlp": 0.28063965, + "step": 1168, + "time_per_iteration": 2.3998148441314697 + }, + { + "auxiliary_loss_clip": 0.01166749, + "auxiliary_loss_mlp": 0.01061199, + "balance_loss_clip": 1.04969501, + "balance_loss_mlp": 1.0327673, + "epoch": 0.033921420695258546, + "flos": 10841509071360.0, + "grad_norm": 4.887969587534948, + "language_loss": 1.04470432, + "learning_rate": 3.999839031584283e-06, + "loss": 1.0669837, + "num_input_tokens_seen": 32396360, + "router_z_loss_clip": 1.16992188, + "router_z_loss_mlp": 0.28442383, + "step": 1169, + "time_per_iteration": 2.3629825115203857 + }, + { + "auxiliary_loss_clip": 0.0103928, + "auxiliary_loss_mlp": 0.01038127, + "balance_loss_clip": 1.01430821, + "balance_loss_mlp": 1.03561199, + "epoch": 0.03395043816377459, + "flos": 74770304837760.0, + "grad_norm": 0.6859991324186532, + "language_loss": 0.52127916, + "learning_rate": 3.999836638067312e-06, + "loss": 0.54205322, + "num_input_tokens_seen": 32458720, + "router_z_loss_clip": 0.25, + "router_z_loss_mlp": 0.02514648, + "step": 1170, + "time_per_iteration": 3.281968355178833 + }, + { + "auxiliary_loss_clip": 0.01163918, + "auxiliary_loss_mlp": 0.01058542, + "balance_loss_clip": 1.04727674, + "balance_loss_mlp": 1.02891886, + "epoch": 0.03397945563229064, + "flos": 24489651162240.0, + "grad_norm": 2.7307387080496306, + "language_loss": 1.06786776, + "learning_rate": 3.999834226886975e-06, + "loss": 1.09009218, + "num_input_tokens_seen": 32474510, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.29626465, + "step": 1171, + "time_per_iteration": 2.4699742794036865 + }, + { + "auxiliary_loss_clip": 0.01028147, + "auxiliary_loss_mlp": 0.01007129, + "balance_loss_clip": 1.00424314, + "balance_loss_mlp": 1.00470901, + "epoch": 0.03400847310080669, + "flos": 65585772892800.0, + "grad_norm": 0.7572690317750191, + "language_loss": 0.55951238, + "learning_rate": 3.9998317980432924e-06, + "loss": 0.5798651, + "num_input_tokens_seen": 32535245, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.02416992, + "step": 1172, + "time_per_iteration": 3.035816192626953 + }, + { + "auxiliary_loss_clip": 0.01025602, + "auxiliary_loss_mlp": 0.01002263, + "balance_loss_clip": 1.00187528, + "balance_loss_mlp": 0.99985451, + "epoch": 0.03403749056932273, + "flos": 69672531156480.0, + "grad_norm": 0.6286969500874411, + "language_loss": 0.48552084, + "learning_rate": 3.999829351536286e-06, + "loss": 0.50579947, + "num_input_tokens_seen": 32603280, + "router_z_loss_clip": 0.23730469, + "router_z_loss_mlp": 0.02404785, + "step": 1173, + "time_per_iteration": 3.1828677654266357 + }, + { + "auxiliary_loss_clip": 0.01027119, + "auxiliary_loss_mlp": 0.01005147, + "balance_loss_clip": 1.00349772, + "balance_loss_mlp": 1.0027386, + "epoch": 0.034066508037838776, + "flos": 70983799200000.0, + "grad_norm": 0.6772232610354634, + "language_loss": 0.50052047, + "learning_rate": 3.999826887365978e-06, + "loss": 0.52084309, + "num_input_tokens_seen": 32671835, + "router_z_loss_clip": 0.23632812, + "router_z_loss_mlp": 0.02404785, + "step": 1174, + "time_per_iteration": 3.071491003036499 + }, + { + "auxiliary_loss_clip": 0.01166088, + "auxiliary_loss_mlp": 0.01067388, + "balance_loss_clip": 1.05094314, + "balance_loss_mlp": 1.04105449, + "epoch": 0.03409552550635483, + "flos": 35581767039360.0, + "grad_norm": 2.9023179407114608, + "language_loss": 0.99314427, + "learning_rate": 3.9998244055323896e-06, + "loss": 1.01547909, + "num_input_tokens_seen": 32688635, + "router_z_loss_clip": 1.15039062, + "router_z_loss_mlp": 0.26342773, + "step": 1175, + "time_per_iteration": 2.570058584213257 + }, + { + "auxiliary_loss_clip": 0.01169983, + "auxiliary_loss_mlp": 0.01072346, + "balance_loss_clip": 1.05260527, + "balance_loss_mlp": 1.04364026, + "epoch": 0.03412454297487087, + "flos": 24417206357760.0, + "grad_norm": 2.278410015852935, + "language_loss": 0.7594403, + "learning_rate": 3.999821906035542e-06, + "loss": 0.78186357, + "num_input_tokens_seen": 32702555, + "router_z_loss_clip": 1.17431641, + "router_z_loss_mlp": 0.28710938, + "step": 1176, + "time_per_iteration": 2.449171781539917 + }, + { + "auxiliary_loss_clip": 0.01031276, + "auxiliary_loss_mlp": 0.01011851, + "balance_loss_clip": 1.00796723, + "balance_loss_mlp": 1.00940681, + "epoch": 0.03415356044338692, + "flos": 66165540796800.0, + "grad_norm": 0.6576390749810238, + "language_loss": 0.50262529, + "learning_rate": 3.999819388875459e-06, + "loss": 0.52305651, + "num_input_tokens_seen": 32768800, + "router_z_loss_clip": 0.23339844, + "router_z_loss_mlp": 0.02441406, + "step": 1177, + "time_per_iteration": 3.1215062141418457 + }, + { + "auxiliary_loss_clip": 0.01028039, + "auxiliary_loss_mlp": 0.01008519, + "balance_loss_clip": 1.00476003, + "balance_loss_mlp": 1.0059557, + "epoch": 0.03418257791190297, + "flos": 65896884817920.0, + "grad_norm": 0.6612438966366654, + "language_loss": 0.50375426, + "learning_rate": 3.999816854052162e-06, + "loss": 0.52411985, + "num_input_tokens_seen": 32825500, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.02563477, + "step": 1178, + "time_per_iteration": 2.9865682125091553 + }, + { + "auxiliary_loss_clip": 0.0115811, + "auxiliary_loss_mlp": 0.01069517, + "balance_loss_clip": 1.04807508, + "balance_loss_mlp": 1.03967857, + "epoch": 0.03421159538041901, + "flos": 25444281000960.0, + "grad_norm": 2.2422331029473925, + "language_loss": 0.81343812, + "learning_rate": 3.999814301565673e-06, + "loss": 0.83571434, + "num_input_tokens_seen": 32844245, + "router_z_loss_clip": 1.10009766, + "router_z_loss_mlp": 0.29833984, + "step": 1179, + "time_per_iteration": 2.4606053829193115 + }, + { + "auxiliary_loss_clip": 0.01168541, + "auxiliary_loss_mlp": 0.01073792, + "balance_loss_clip": 1.04670191, + "balance_loss_mlp": 1.04133129, + "epoch": 0.03424061284893506, + "flos": 28108084181760.0, + "grad_norm": 3.704570337981739, + "language_loss": 1.14135313, + "learning_rate": 3.999811731416015e-06, + "loss": 1.1637764, + "num_input_tokens_seen": 32866005, + "router_z_loss_clip": 1.21972656, + "router_z_loss_mlp": 0.32446289, + "step": 1180, + "time_per_iteration": 2.6296215057373047 + }, + { + "auxiliary_loss_clip": 0.01173899, + "auxiliary_loss_mlp": 0.01059825, + "balance_loss_clip": 1.04614258, + "balance_loss_mlp": 1.02954555, + "epoch": 0.0342696303174511, + "flos": 70532828634240.0, + "grad_norm": 2.5483027530826683, + "language_loss": 0.73704726, + "learning_rate": 3.99980914360321e-06, + "loss": 0.75938451, + "num_input_tokens_seen": 32886890, + "router_z_loss_clip": 1.27734375, + "router_z_loss_mlp": 0.30310059, + "step": 1181, + "time_per_iteration": 2.7848422527313232 + }, + { + "auxiliary_loss_clip": 0.01157449, + "auxiliary_loss_mlp": 0.01060142, + "balance_loss_clip": 1.04608679, + "balance_loss_mlp": 1.03186536, + "epoch": 0.034298647785967154, + "flos": 24928893377280.0, + "grad_norm": 2.363826611749408, + "language_loss": 0.93276906, + "learning_rate": 3.999806538127282e-06, + "loss": 0.95494497, + "num_input_tokens_seen": 32900750, + "router_z_loss_clip": 1.11523438, + "router_z_loss_mlp": 0.28283691, + "step": 1182, + "time_per_iteration": 2.8196732997894287 + }, + { + "auxiliary_loss_clip": 0.01035154, + "auxiliary_loss_mlp": 0.01031113, + "balance_loss_clip": 1.01137745, + "balance_loss_mlp": 1.02863324, + "epoch": 0.0343276652544832, + "flos": 60902209981440.0, + "grad_norm": 0.7148542575904446, + "language_loss": 0.50040656, + "learning_rate": 3.999803914988253e-06, + "loss": 0.52106923, + "num_input_tokens_seen": 32956575, + "router_z_loss_clip": 0.23828125, + "router_z_loss_mlp": 0.02478027, + "step": 1183, + "time_per_iteration": 2.921283721923828 + }, + { + "auxiliary_loss_clip": 0.01161026, + "auxiliary_loss_mlp": 0.01068098, + "balance_loss_clip": 1.04532063, + "balance_loss_mlp": 1.04035842, + "epoch": 0.03435668272299924, + "flos": 23111454309120.0, + "grad_norm": 2.7791435994434908, + "language_loss": 0.91841453, + "learning_rate": 3.999801274186146e-06, + "loss": 0.94070578, + "num_input_tokens_seen": 32975570, + "router_z_loss_clip": 1.15625, + "router_z_loss_mlp": 0.27746582, + "step": 1184, + "time_per_iteration": 2.4222428798675537 + }, + { + "auxiliary_loss_clip": 0.01174979, + "auxiliary_loss_mlp": 0.01068117, + "balance_loss_clip": 1.04962707, + "balance_loss_mlp": 1.03835082, + "epoch": 0.034385700191515295, + "flos": 74734974541440.0, + "grad_norm": 2.3615565761459, + "language_loss": 0.84050953, + "learning_rate": 3.999798615720986e-06, + "loss": 0.86294049, + "num_input_tokens_seen": 32999770, + "router_z_loss_clip": 1.25390625, + "router_z_loss_mlp": 0.29772949, + "step": 1185, + "time_per_iteration": 2.804069757461548 + }, + { + "auxiliary_loss_clip": 0.01164799, + "auxiliary_loss_mlp": 0.01064998, + "balance_loss_clip": 1.04555202, + "balance_loss_mlp": 1.03549337, + "epoch": 0.03441471766003134, + "flos": 32665716218880.0, + "grad_norm": 3.3654706194910786, + "language_loss": 0.87554699, + "learning_rate": 3.999795939592795e-06, + "loss": 0.89784503, + "num_input_tokens_seen": 33014670, + "router_z_loss_clip": 1.19287109, + "router_z_loss_mlp": 0.29528809, + "step": 1186, + "time_per_iteration": 2.520204782485962 + }, + { + "auxiliary_loss_clip": 0.01160111, + "auxiliary_loss_mlp": 0.01055964, + "balance_loss_clip": 1.05123997, + "balance_loss_mlp": 1.03021526, + "epoch": 0.034443735128547384, + "flos": 30585137166720.0, + "grad_norm": 2.81653917877084, + "language_loss": 0.84486246, + "learning_rate": 3.9997932458015974e-06, + "loss": 0.86702323, + "num_input_tokens_seen": 33028470, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.25756836, + "step": 1187, + "time_per_iteration": 2.4944117069244385 + }, + { + "auxiliary_loss_clip": 0.01024272, + "auxiliary_loss_mlp": 0.01003656, + "balance_loss_clip": 1.00139451, + "balance_loss_mlp": 1.00098562, + "epoch": 0.03447275259706343, + "flos": 54657784650240.0, + "grad_norm": 0.7121959362598825, + "language_loss": 0.48297632, + "learning_rate": 3.999790534347416e-06, + "loss": 0.50325561, + "num_input_tokens_seen": 33083380, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.0267334, + "step": 1188, + "time_per_iteration": 2.8452553749084473 + }, + { + "auxiliary_loss_clip": 0.0116072, + "auxiliary_loss_mlp": 0.0106112, + "balance_loss_clip": 1.04772973, + "balance_loss_mlp": 1.0322957, + "epoch": 0.03450177006557948, + "flos": 31424833209600.0, + "grad_norm": 1.9980220866592597, + "language_loss": 0.93225104, + "learning_rate": 3.999787805230276e-06, + "loss": 0.9544695, + "num_input_tokens_seen": 33100870, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.28820801, + "step": 1189, + "time_per_iteration": 2.5430614948272705 + }, + { + "auxiliary_loss_clip": 0.01162937, + "auxiliary_loss_mlp": 0.01053987, + "balance_loss_clip": 1.04715216, + "balance_loss_mlp": 1.02584207, + "epoch": 0.034530787534095525, + "flos": 30072647185920.0, + "grad_norm": 2.5994809538520007, + "language_loss": 0.81606013, + "learning_rate": 3.9997850584502006e-06, + "loss": 0.8382293, + "num_input_tokens_seen": 33115540, + "router_z_loss_clip": 1.15722656, + "router_z_loss_mlp": 0.28161621, + "step": 1190, + "time_per_iteration": 2.4044642448425293 + }, + { + "auxiliary_loss_clip": 0.01024962, + "auxiliary_loss_mlp": 0.0100452, + "balance_loss_clip": 1.00229287, + "balance_loss_mlp": 1.00195742, + "epoch": 0.03455980500261157, + "flos": 63904914529920.0, + "grad_norm": 0.6777207689445431, + "language_loss": 0.52175903, + "learning_rate": 3.999782294007214e-06, + "loss": 0.54205388, + "num_input_tokens_seen": 33180150, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.02563477, + "step": 1191, + "time_per_iteration": 3.0006814002990723 + }, + { + "auxiliary_loss_clip": 0.01161592, + "auxiliary_loss_mlp": 0.01060634, + "balance_loss_clip": 1.04518974, + "balance_loss_mlp": 1.03134441, + "epoch": 0.03458882247112762, + "flos": 11867187260160.0, + "grad_norm": 3.194620422624371, + "language_loss": 0.83520252, + "learning_rate": 3.999779511901341e-06, + "loss": 0.8574248, + "num_input_tokens_seen": 33190350, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.29296875, + "step": 1192, + "time_per_iteration": 2.3587186336517334 + }, + { + "auxiliary_loss_clip": 0.01159302, + "auxiliary_loss_mlp": 0.01059452, + "balance_loss_clip": 1.04387581, + "balance_loss_mlp": 1.03061533, + "epoch": 0.034617839939643666, + "flos": 26788018475520.0, + "grad_norm": 2.2604184016416564, + "language_loss": 0.8575632, + "learning_rate": 3.999776712132606e-06, + "loss": 0.87975073, + "num_input_tokens_seen": 33207540, + "router_z_loss_clip": 1.15478516, + "router_z_loss_mlp": 0.28820801, + "step": 1193, + "time_per_iteration": 2.5031251907348633 + }, + { + "auxiliary_loss_clip": 0.01171509, + "auxiliary_loss_mlp": 0.01059913, + "balance_loss_clip": 1.04915786, + "balance_loss_mlp": 1.02866864, + "epoch": 0.03464685740815971, + "flos": 30985206969600.0, + "grad_norm": 2.5456340523359597, + "language_loss": 0.72351795, + "learning_rate": 3.999773894701034e-06, + "loss": 0.7458322, + "num_input_tokens_seen": 33222395, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.31225586, + "step": 1194, + "time_per_iteration": 2.4794111251831055 + }, + { + "auxiliary_loss_clip": 0.01024348, + "auxiliary_loss_mlp": 0.01001817, + "balance_loss_clip": 1.00181496, + "balance_loss_mlp": 0.9990986, + "epoch": 0.03467587487667576, + "flos": 57354441287040.0, + "grad_norm": 0.7454875995359979, + "language_loss": 0.53361607, + "learning_rate": 3.9997710596066505e-06, + "loss": 0.55387771, + "num_input_tokens_seen": 33272795, + "router_z_loss_clip": 0.22558594, + "router_z_loss_mlp": 0.02722168, + "step": 1195, + "time_per_iteration": 2.9472098350524902 + }, + { + "auxiliary_loss_clip": 0.01156194, + "auxiliary_loss_mlp": 0.01049983, + "balance_loss_clip": 1.0451051, + "balance_loss_mlp": 1.02333963, + "epoch": 0.03470489234519181, + "flos": 44482733988480.0, + "grad_norm": 6.549944014721286, + "language_loss": 0.9531613, + "learning_rate": 3.9997682068494795e-06, + "loss": 0.97522312, + "num_input_tokens_seen": 33289355, + "router_z_loss_clip": 1.11132812, + "router_z_loss_mlp": 0.26635742, + "step": 1196, + "time_per_iteration": 2.6423211097717285 + }, + { + "auxiliary_loss_clip": 0.01150619, + "auxiliary_loss_mlp": 0.01055552, + "balance_loss_clip": 1.04272544, + "balance_loss_mlp": 1.02793133, + "epoch": 0.03473390981370785, + "flos": 15116274339840.0, + "grad_norm": 2.3928988462840564, + "language_loss": 0.83025098, + "learning_rate": 3.9997653364295455e-06, + "loss": 0.85231268, + "num_input_tokens_seen": 33301815, + "router_z_loss_clip": 1.08007812, + "router_z_loss_mlp": 0.27624512, + "step": 1197, + "time_per_iteration": 2.493323564529419 + }, + { + "auxiliary_loss_clip": 0.01023795, + "auxiliary_loss_mlp": 0.01005906, + "balance_loss_clip": 1.00129342, + "balance_loss_mlp": 1.00318766, + "epoch": 0.034762927282223896, + "flos": 74775751009920.0, + "grad_norm": 0.6462285101600962, + "language_loss": 0.54062712, + "learning_rate": 3.999762448346876e-06, + "loss": 0.56092411, + "num_input_tokens_seen": 33371520, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.02722168, + "step": 1198, + "time_per_iteration": 3.1685776710510254 + }, + { + "auxiliary_loss_clip": 0.01173123, + "auxiliary_loss_mlp": 0.01049304, + "balance_loss_clip": 1.04781473, + "balance_loss_mlp": 1.01958489, + "epoch": 0.03479194475073995, + "flos": 22236565749120.0, + "grad_norm": 2.862353258148059, + "language_loss": 1.06612563, + "learning_rate": 3.999759542601494e-06, + "loss": 1.08834982, + "num_input_tokens_seen": 33386930, + "router_z_loss_clip": 1.25488281, + "router_z_loss_mlp": 0.29736328, + "step": 1199, + "time_per_iteration": 2.465440273284912 + }, + { + "auxiliary_loss_clip": 0.01023935, + "auxiliary_loss_mlp": 0.01005429, + "balance_loss_clip": 1.00115192, + "balance_loss_mlp": 1.00287819, + "epoch": 0.03482096221925599, + "flos": 61601066133120.0, + "grad_norm": 0.7582320383580157, + "language_loss": 0.49873692, + "learning_rate": 3.999756619193427e-06, + "loss": 0.51903057, + "num_input_tokens_seen": 33446410, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.0255127, + "step": 1200, + "time_per_iteration": 2.9144954681396484 + }, + { + "auxiliary_loss_clip": 0.01158308, + "auxiliary_loss_mlp": 0.01048349, + "balance_loss_clip": 1.04203773, + "balance_loss_mlp": 1.02071655, + "epoch": 0.03484997968777204, + "flos": 24964330273920.0, + "grad_norm": 2.59412349900066, + "language_loss": 1.02978253, + "learning_rate": 3.999753678122701e-06, + "loss": 1.05184913, + "num_input_tokens_seen": 33460205, + "router_z_loss_clip": 1.1640625, + "router_z_loss_mlp": 0.27636719, + "step": 1201, + "time_per_iteration": 2.4639697074890137 + }, + { + "auxiliary_loss_clip": 0.01166916, + "auxiliary_loss_mlp": 0.01063321, + "balance_loss_clip": 1.04688668, + "balance_loss_mlp": 1.03328061, + "epoch": 0.03487899715628809, + "flos": 22082529363840.0, + "grad_norm": 2.781506932167829, + "language_loss": 0.92532194, + "learning_rate": 3.999750719389341e-06, + "loss": 0.94762433, + "num_input_tokens_seen": 33474020, + "router_z_loss_clip": 1.19970703, + "router_z_loss_mlp": 0.30029297, + "step": 1202, + "time_per_iteration": 2.4276652336120605 + }, + { + "auxiliary_loss_clip": 0.01158902, + "auxiliary_loss_mlp": 0.01057241, + "balance_loss_clip": 1.04676354, + "balance_loss_mlp": 1.03046679, + "epoch": 0.03490801462480413, + "flos": 16502221514880.0, + "grad_norm": 27.260783124606665, + "language_loss": 0.82148421, + "learning_rate": 3.999747742993374e-06, + "loss": 0.84364569, + "num_input_tokens_seen": 33487345, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.2677002, + "step": 1203, + "time_per_iteration": 2.4171969890594482 + }, + { + "auxiliary_loss_clip": 0.01163183, + "auxiliary_loss_mlp": 0.01061675, + "balance_loss_clip": 1.0497098, + "balance_loss_mlp": 1.03352976, + "epoch": 0.03493703209332018, + "flos": 32046566434560.0, + "grad_norm": 2.787236426148979, + "language_loss": 0.9295404, + "learning_rate": 3.999744748934825e-06, + "loss": 0.9517889, + "num_input_tokens_seen": 33502735, + "router_z_loss_clip": 1.13476562, + "router_z_loss_mlp": 0.28161621, + "step": 1204, + "time_per_iteration": 2.524148941040039 + }, + { + "auxiliary_loss_clip": 0.01023641, + "auxiliary_loss_mlp": 0.01002759, + "balance_loss_clip": 1.00102949, + "balance_loss_mlp": 1.00018418, + "epoch": 0.03496604956183622, + "flos": 74769955724160.0, + "grad_norm": 0.6936051891257912, + "language_loss": 0.54166353, + "learning_rate": 3.999741737213721e-06, + "loss": 0.5619275, + "num_input_tokens_seen": 33567165, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.02575684, + "step": 1205, + "time_per_iteration": 3.0398566722869873 + }, + { + "auxiliary_loss_clip": 0.01160777, + "auxiliary_loss_mlp": 0.01057656, + "balance_loss_clip": 1.0465219, + "balance_loss_mlp": 1.02982068, + "epoch": 0.034995067030352274, + "flos": 44340148529280.0, + "grad_norm": 2.405061868192621, + "language_loss": 0.94436884, + "learning_rate": 3.99973870783009e-06, + "loss": 0.96655327, + "num_input_tokens_seen": 33588305, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.27819824, + "step": 1206, + "time_per_iteration": 2.6581387519836426 + }, + { + "auxiliary_loss_clip": 0.01023578, + "auxiliary_loss_mlp": 0.01002789, + "balance_loss_clip": 1.00080299, + "balance_loss_mlp": 1.00030911, + "epoch": 0.03502408449886832, + "flos": 74758958645760.0, + "grad_norm": 0.7359706738271004, + "language_loss": 0.55336046, + "learning_rate": 3.9997356607839554e-06, + "loss": 0.57362413, + "num_input_tokens_seen": 33644095, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.02478027, + "step": 1207, + "time_per_iteration": 2.9279916286468506 + }, + { + "auxiliary_loss_clip": 0.01162114, + "auxiliary_loss_mlp": 0.0105122, + "balance_loss_clip": 1.04745591, + "balance_loss_mlp": 1.02314615, + "epoch": 0.03505310196738436, + "flos": 13070643425280.0, + "grad_norm": 2.362258114393666, + "language_loss": 0.87997985, + "learning_rate": 3.999732596075348e-06, + "loss": 0.9021132, + "num_input_tokens_seen": 33655200, + "router_z_loss_clip": 1.14794922, + "router_z_loss_mlp": 0.28076172, + "step": 1208, + "time_per_iteration": 2.486952543258667 + }, + { + "auxiliary_loss_clip": 0.01165813, + "auxiliary_loss_mlp": 0.010775, + "balance_loss_clip": 1.04714036, + "balance_loss_mlp": 1.04602849, + "epoch": 0.035082119435900415, + "flos": 35361185869440.0, + "grad_norm": 3.1927693478300276, + "language_loss": 1.00462031, + "learning_rate": 3.9997295137042925e-06, + "loss": 1.02705348, + "num_input_tokens_seen": 33669745, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.31469727, + "step": 1209, + "time_per_iteration": 2.5921671390533447 + }, + { + "auxiliary_loss_clip": 0.01163861, + "auxiliary_loss_mlp": 0.01057584, + "balance_loss_clip": 1.04932809, + "balance_loss_mlp": 1.02967691, + "epoch": 0.03511113690441646, + "flos": 35546574522240.0, + "grad_norm": 2.718209757856871, + "language_loss": 1.00866103, + "learning_rate": 3.999726413670816e-06, + "loss": 1.03087544, + "num_input_tokens_seen": 33688565, + "router_z_loss_clip": 1.14453125, + "router_z_loss_mlp": 0.27905273, + "step": 1210, + "time_per_iteration": 2.697110176086426 + }, + { + "auxiliary_loss_clip": 0.01161584, + "auxiliary_loss_mlp": 0.01055066, + "balance_loss_clip": 1.04709411, + "balance_loss_mlp": 1.02847075, + "epoch": 0.035140154372932504, + "flos": 25293526283520.0, + "grad_norm": 3.7453070039266025, + "language_loss": 0.93975425, + "learning_rate": 3.999723295974948e-06, + "loss": 0.96192074, + "num_input_tokens_seen": 33707555, + "router_z_loss_clip": 1.14404297, + "router_z_loss_mlp": 0.26599121, + "step": 1211, + "time_per_iteration": 2.5222878456115723 + }, + { + "auxiliary_loss_clip": 0.01156738, + "auxiliary_loss_mlp": 0.01055748, + "balance_loss_clip": 1.04448414, + "balance_loss_mlp": 1.02927136, + "epoch": 0.03516917184144855, + "flos": 16902640431360.0, + "grad_norm": 2.4746834946854475, + "language_loss": 0.91743809, + "learning_rate": 3.999720160616714e-06, + "loss": 0.93956298, + "num_input_tokens_seen": 33721680, + "router_z_loss_clip": 1.12158203, + "router_z_loss_mlp": 0.26464844, + "step": 1212, + "time_per_iteration": 2.3784782886505127 + }, + { + "auxiliary_loss_clip": 0.010233, + "auxiliary_loss_mlp": 0.0100528, + "balance_loss_clip": 1.0011065, + "balance_loss_mlp": 1.00233543, + "epoch": 0.0351981893099646, + "flos": 62435246181120.0, + "grad_norm": 0.6781118282658897, + "language_loss": 0.51464188, + "learning_rate": 3.999717007596143e-06, + "loss": 0.53492767, + "num_input_tokens_seen": 33785430, + "router_z_loss_clip": 0.22167969, + "router_z_loss_mlp": 0.02941895, + "step": 1213, + "time_per_iteration": 3.042065382003784 + }, + { + "auxiliary_loss_clip": 0.01165439, + "auxiliary_loss_mlp": 0.01071026, + "balance_loss_clip": 1.05025125, + "balance_loss_mlp": 1.04199886, + "epoch": 0.035227206778480645, + "flos": 70858985489280.0, + "grad_norm": 1.9421911358638457, + "language_loss": 0.83172476, + "learning_rate": 3.999713836913261e-06, + "loss": 0.85408938, + "num_input_tokens_seen": 33822670, + "router_z_loss_clip": 1.15039062, + "router_z_loss_mlp": 0.29016113, + "step": 1214, + "time_per_iteration": 3.0698819160461426 + }, + { + "auxiliary_loss_clip": 0.0116149, + "auxiliary_loss_mlp": 0.01071097, + "balance_loss_clip": 1.04560387, + "balance_loss_mlp": 1.03944683, + "epoch": 0.03525622424699669, + "flos": 21281482062720.0, + "grad_norm": 11.531067013934228, + "language_loss": 1.00978458, + "learning_rate": 3.999710648568098e-06, + "loss": 1.03211045, + "num_input_tokens_seen": 33836880, + "router_z_loss_clip": 1.15917969, + "router_z_loss_mlp": 0.31640625, + "step": 1215, + "time_per_iteration": 2.3925676345825195 + }, + { + "auxiliary_loss_clip": 0.01160335, + "auxiliary_loss_mlp": 0.01056927, + "balance_loss_clip": 1.04523861, + "balance_loss_mlp": 1.02859092, + "epoch": 0.03528524171551274, + "flos": 32592328807680.0, + "grad_norm": 2.0294372983228386, + "language_loss": 0.83065951, + "learning_rate": 3.9997074425606804e-06, + "loss": 0.85283214, + "num_input_tokens_seen": 33855420, + "router_z_loss_clip": 1.15087891, + "router_z_loss_mlp": 0.28308105, + "step": 1216, + "time_per_iteration": 2.5457427501678467 + }, + { + "auxiliary_loss_clip": 0.01164191, + "auxiliary_loss_mlp": 0.01067347, + "balance_loss_clip": 1.05047464, + "balance_loss_mlp": 1.03649533, + "epoch": 0.035314259184028786, + "flos": 26716621011840.0, + "grad_norm": 3.7146149031425906, + "language_loss": 0.86854708, + "learning_rate": 3.999704218891039e-06, + "loss": 0.89086246, + "num_input_tokens_seen": 33868655, + "router_z_loss_clip": 1.13476562, + "router_z_loss_mlp": 0.30859375, + "step": 1217, + "time_per_iteration": 2.474518060684204 + }, + { + "auxiliary_loss_clip": 0.01157589, + "auxiliary_loss_mlp": 0.0105468, + "balance_loss_clip": 1.04463422, + "balance_loss_mlp": 1.02492487, + "epoch": 0.03534327665254483, + "flos": 46570085844480.0, + "grad_norm": 2.506953764639295, + "language_loss": 0.69909155, + "learning_rate": 3.9997009775592e-06, + "loss": 0.72121429, + "num_input_tokens_seen": 33889470, + "router_z_loss_clip": 1.12890625, + "router_z_loss_mlp": 0.29736328, + "step": 1218, + "time_per_iteration": 5.13092565536499 + }, + { + "auxiliary_loss_clip": 0.01023562, + "auxiliary_loss_mlp": 0.01002759, + "balance_loss_clip": 1.00148821, + "balance_loss_mlp": 0.99996948, + "epoch": 0.03537229412106088, + "flos": 61783173118080.0, + "grad_norm": 0.6582896959748433, + "language_loss": 0.48776895, + "learning_rate": 3.9996977185651925e-06, + "loss": 0.50803214, + "num_input_tokens_seen": 33947250, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.0279541, + "step": 1219, + "time_per_iteration": 2.9475533962249756 + }, + { + "auxiliary_loss_clip": 0.01152288, + "auxiliary_loss_mlp": 0.0104834, + "balance_loss_clip": 1.04538107, + "balance_loss_mlp": 1.02108908, + "epoch": 0.03540131158957693, + "flos": 12412495785600.0, + "grad_norm": 2.7349932413387257, + "language_loss": 1.00846159, + "learning_rate": 3.999694441909045e-06, + "loss": 1.03046775, + "num_input_tokens_seen": 33958700, + "router_z_loss_clip": 1.06835938, + "router_z_loss_mlp": 0.27246094, + "step": 1220, + "time_per_iteration": 2.381939649581909 + }, + { + "auxiliary_loss_clip": 0.01022987, + "auxiliary_loss_mlp": 0.01002836, + "balance_loss_clip": 1.00110471, + "balance_loss_mlp": 0.99997497, + "epoch": 0.03543032905809297, + "flos": 69306013036800.0, + "grad_norm": 0.6659944511696503, + "language_loss": 0.48246971, + "learning_rate": 3.999691147590788e-06, + "loss": 0.50272799, + "num_input_tokens_seen": 34024825, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.02856445, + "step": 1221, + "time_per_iteration": 3.226607084274292 + }, + { + "auxiliary_loss_clip": 0.01157426, + "auxiliary_loss_mlp": 0.01052379, + "balance_loss_clip": 1.04773879, + "balance_loss_mlp": 1.0242455, + "epoch": 0.035459346526609016, + "flos": 24966006019200.0, + "grad_norm": 3.375750782533878, + "language_loss": 0.96354753, + "learning_rate": 3.999687835610449e-06, + "loss": 0.98564553, + "num_input_tokens_seen": 34039735, + "router_z_loss_clip": 1.09716797, + "router_z_loss_mlp": 0.28088379, + "step": 1222, + "time_per_iteration": 4.73967719078064 + }, + { + "auxiliary_loss_clip": 0.01168292, + "auxiliary_loss_mlp": 0.01057339, + "balance_loss_clip": 1.04961228, + "balance_loss_mlp": 1.02732217, + "epoch": 0.03548836399512507, + "flos": 25367088251520.0, + "grad_norm": 3.2692538668043998, + "language_loss": 0.9423002, + "learning_rate": 3.999684505968059e-06, + "loss": 0.96455652, + "num_input_tokens_seen": 34057955, + "router_z_loss_clip": 1.18798828, + "router_z_loss_mlp": 0.3001709, + "step": 1223, + "time_per_iteration": 2.503652811050415 + }, + { + "auxiliary_loss_clip": 0.01170304, + "auxiliary_loss_mlp": 0.0107931, + "balance_loss_clip": 1.04634619, + "balance_loss_mlp": 1.04638469, + "epoch": 0.03551738146364111, + "flos": 29161553679360.0, + "grad_norm": 6.729966542291099, + "language_loss": 0.98524451, + "learning_rate": 3.999681158663645e-06, + "loss": 1.00774074, + "num_input_tokens_seen": 34071810, + "router_z_loss_clip": 1.24023438, + "router_z_loss_mlp": 0.3293457, + "step": 1224, + "time_per_iteration": 2.5016305446624756 + }, + { + "auxiliary_loss_clip": 0.0102303, + "auxiliary_loss_mlp": 0.01003124, + "balance_loss_clip": 1.00115228, + "balance_loss_mlp": 1.00007248, + "epoch": 0.03554639893215716, + "flos": 69800942603520.0, + "grad_norm": 0.708619630271325, + "language_loss": 0.50838238, + "learning_rate": 3.999677793697238e-06, + "loss": 0.52864397, + "num_input_tokens_seen": 34126210, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.03051758, + "step": 1225, + "time_per_iteration": 5.323030710220337 + }, + { + "auxiliary_loss_clip": 0.01023171, + "auxiliary_loss_mlp": 0.0100438, + "balance_loss_clip": 1.00120819, + "balance_loss_mlp": 1.00166237, + "epoch": 0.03557541640067321, + "flos": 66564004677120.0, + "grad_norm": 0.6507085124485995, + "language_loss": 0.51761919, + "learning_rate": 3.9996744110688685e-06, + "loss": 0.53789473, + "num_input_tokens_seen": 34188660, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.02722168, + "step": 1226, + "time_per_iteration": 3.0545694828033447 + }, + { + "auxiliary_loss_clip": 0.01148828, + "auxiliary_loss_mlp": 0.01063927, + "balance_loss_clip": 1.04056704, + "balance_loss_mlp": 1.03823757, + "epoch": 0.03560443386918925, + "flos": 19091624855040.0, + "grad_norm": 2.7492990932156935, + "language_loss": 0.8821947, + "learning_rate": 3.999671010778564e-06, + "loss": 0.90432227, + "num_input_tokens_seen": 34202710, + "router_z_loss_clip": 1.08398438, + "router_z_loss_mlp": 0.25695801, + "step": 1227, + "time_per_iteration": 2.359589099884033 + }, + { + "auxiliary_loss_clip": 0.01165527, + "auxiliary_loss_mlp": 0.01080092, + "balance_loss_clip": 1.04796875, + "balance_loss_mlp": 1.04984903, + "epoch": 0.0356334513377053, + "flos": 16135284458880.0, + "grad_norm": 2.3834505974267235, + "language_loss": 0.8094008, + "learning_rate": 3.999667592826357e-06, + "loss": 0.83185697, + "num_input_tokens_seen": 34216430, + "router_z_loss_clip": 1.17626953, + "router_z_loss_mlp": 0.30249023, + "step": 1228, + "time_per_iteration": 4.797964096069336 + }, + { + "auxiliary_loss_clip": 0.01022649, + "auxiliary_loss_mlp": 0.0100363, + "balance_loss_clip": 1.00073862, + "balance_loss_mlp": 1.00091219, + "epoch": 0.03566246880622134, + "flos": 67654796284800.0, + "grad_norm": 0.7142727545746621, + "language_loss": 0.49971312, + "learning_rate": 3.999664157212276e-06, + "loss": 0.5199759, + "num_input_tokens_seen": 34276440, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.02722168, + "step": 1229, + "time_per_iteration": 3.0532937049865723 + }, + { + "auxiliary_loss_clip": 0.01022595, + "auxiliary_loss_mlp": 0.01002465, + "balance_loss_clip": 1.00040364, + "balance_loss_mlp": 0.99978256, + "epoch": 0.035691486274737394, + "flos": 68190923122560.0, + "grad_norm": 0.7521331393372939, + "language_loss": 0.52655655, + "learning_rate": 3.999660703936352e-06, + "loss": 0.54680717, + "num_input_tokens_seen": 34324765, + "router_z_loss_clip": 0.22167969, + "router_z_loss_mlp": 0.02685547, + "step": 1230, + "time_per_iteration": 2.8306050300598145 + }, + { + "auxiliary_loss_clip": 0.01179752, + "auxiliary_loss_mlp": 0.01067559, + "balance_loss_clip": 1.04697871, + "balance_loss_mlp": 1.03276229, + "epoch": 0.03572050374325344, + "flos": 21756684844800.0, + "grad_norm": 4.964794436802193, + "language_loss": 1.17541003, + "learning_rate": 3.999657232998616e-06, + "loss": 1.19788313, + "num_input_tokens_seen": 34337470, + "router_z_loss_clip": 1.32714844, + "router_z_loss_mlp": 0.34765625, + "step": 1231, + "time_per_iteration": 2.838819742202759 + }, + { + "auxiliary_loss_clip": 0.01159508, + "auxiliary_loss_mlp": 0.0105329, + "balance_loss_clip": 1.04512405, + "balance_loss_mlp": 1.02341628, + "epoch": 0.03574952121176948, + "flos": 60906820147200.0, + "grad_norm": 2.578098752906875, + "language_loss": 0.80966055, + "learning_rate": 3.999653744399098e-06, + "loss": 0.83178854, + "num_input_tokens_seen": 34366815, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.29882812, + "step": 1232, + "time_per_iteration": 2.9835774898529053 + }, + { + "auxiliary_loss_clip": 0.0115245, + "auxiliary_loss_mlp": 0.01048271, + "balance_loss_clip": 1.04338503, + "balance_loss_mlp": 1.02146053, + "epoch": 0.035778538680285535, + "flos": 24363543864960.0, + "grad_norm": 2.3258505449572646, + "language_loss": 0.86499614, + "learning_rate": 3.999650238137829e-06, + "loss": 0.88700342, + "num_input_tokens_seen": 34390530, + "router_z_loss_clip": 1.09033203, + "router_z_loss_mlp": 0.26794434, + "step": 1233, + "time_per_iteration": 2.913813352584839 + }, + { + "auxiliary_loss_clip": 0.01164089, + "auxiliary_loss_mlp": 0.01067598, + "balance_loss_clip": 1.04719925, + "balance_loss_mlp": 1.03427887, + "epoch": 0.03580755614880158, + "flos": 47853248376960.0, + "grad_norm": 38.805187583253606, + "language_loss": 1.06221747, + "learning_rate": 3.999646714214839e-06, + "loss": 1.08453441, + "num_input_tokens_seen": 34409270, + "router_z_loss_clip": 1.16796875, + "router_z_loss_mlp": 0.33312988, + "step": 1234, + "time_per_iteration": 2.7547426223754883 + }, + { + "auxiliary_loss_clip": 0.01023878, + "auxiliary_loss_mlp": 0.01002114, + "balance_loss_clip": 1.00225031, + "balance_loss_mlp": 0.99951482, + "epoch": 0.035836573617317624, + "flos": 71642470510080.0, + "grad_norm": 0.7253752491229054, + "language_loss": 0.52225679, + "learning_rate": 3.999643172630161e-06, + "loss": 0.54251671, + "num_input_tokens_seen": 34479295, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.02600098, + "step": 1235, + "time_per_iteration": 3.143007516860962 + }, + { + "auxiliary_loss_clip": 0.01169689, + "auxiliary_loss_mlp": 0.01068061, + "balance_loss_clip": 1.04597235, + "balance_loss_mlp": 1.03618431, + "epoch": 0.03586559108583367, + "flos": 30511365730560.0, + "grad_norm": 2.6548316800539493, + "language_loss": 1.01833439, + "learning_rate": 3.999639613383826e-06, + "loss": 1.04071188, + "num_input_tokens_seen": 34499115, + "router_z_loss_clip": 1.23925781, + "router_z_loss_mlp": 0.31872559, + "step": 1236, + "time_per_iteration": 2.5472559928894043 + }, + { + "auxiliary_loss_clip": 0.01156547, + "auxiliary_loss_mlp": 0.01058164, + "balance_loss_clip": 1.04286385, + "balance_loss_mlp": 1.02896976, + "epoch": 0.03589460855434972, + "flos": 20003276943360.0, + "grad_norm": 3.7639459374851834, + "language_loss": 0.85712844, + "learning_rate": 3.999636036475864e-06, + "loss": 0.87927562, + "num_input_tokens_seen": 34514575, + "router_z_loss_clip": 1.13623047, + "router_z_loss_mlp": 0.29187012, + "step": 1237, + "time_per_iteration": 2.4374349117279053 + }, + { + "auxiliary_loss_clip": 0.01165891, + "auxiliary_loss_mlp": 0.0106187, + "balance_loss_clip": 1.04880023, + "balance_loss_mlp": 1.03335476, + "epoch": 0.035923626022865765, + "flos": 39192554471040.0, + "grad_norm": 2.200186232708484, + "language_loss": 0.93141621, + "learning_rate": 3.999632441906307e-06, + "loss": 0.95369381, + "num_input_tokens_seen": 34531345, + "router_z_loss_clip": 1.17138672, + "router_z_loss_mlp": 0.28515625, + "step": 1238, + "time_per_iteration": 2.580287456512451 + }, + { + "auxiliary_loss_clip": 0.011434, + "auxiliary_loss_mlp": 0.01048017, + "balance_loss_clip": 1.03905773, + "balance_loss_mlp": 1.02270877, + "epoch": 0.03595264349138181, + "flos": 23031189360000.0, + "grad_norm": 3.3776639155188715, + "language_loss": 0.89557266, + "learning_rate": 3.999628829675188e-06, + "loss": 0.91748691, + "num_input_tokens_seen": 34545745, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.25305176, + "step": 1239, + "time_per_iteration": 2.5587522983551025 + }, + { + "auxiliary_loss_clip": 0.01169707, + "auxiliary_loss_mlp": 0.01064326, + "balance_loss_clip": 1.04972363, + "balance_loss_mlp": 1.03429675, + "epoch": 0.03598166095989786, + "flos": 42039442154880.0, + "grad_norm": 2.6632408602718276, + "language_loss": 1.00388432, + "learning_rate": 3.999625199782537e-06, + "loss": 1.02622461, + "num_input_tokens_seen": 34561930, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.30053711, + "step": 1240, + "time_per_iteration": 2.5883612632751465 + }, + { + "auxiliary_loss_clip": 0.01152484, + "auxiliary_loss_mlp": 0.01055163, + "balance_loss_clip": 1.04478955, + "balance_loss_mlp": 1.02409697, + "epoch": 0.036010678428413906, + "flos": 29089842013440.0, + "grad_norm": 4.575876471937341, + "language_loss": 0.87077284, + "learning_rate": 3.999621552228387e-06, + "loss": 0.89284939, + "num_input_tokens_seen": 34579055, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.31054688, + "step": 1241, + "time_per_iteration": 2.457756519317627 + }, + { + "auxiliary_loss_clip": 0.01167615, + "auxiliary_loss_mlp": 0.01066213, + "balance_loss_clip": 1.05190992, + "balance_loss_mlp": 1.03520656, + "epoch": 0.03603969589692995, + "flos": 34013887436160.0, + "grad_norm": 3.860306260061098, + "language_loss": 0.92364705, + "learning_rate": 3.9996178870127715e-06, + "loss": 0.94598538, + "num_input_tokens_seen": 34595190, + "router_z_loss_clip": 1.15820312, + "router_z_loss_mlp": 0.31018066, + "step": 1242, + "time_per_iteration": 2.5480170249938965 + }, + { + "auxiliary_loss_clip": 0.01141898, + "auxiliary_loss_mlp": 0.01042814, + "balance_loss_clip": 1.0391798, + "balance_loss_mlp": 1.01758909, + "epoch": 0.036068713365445995, + "flos": 35439007023360.0, + "grad_norm": 2.2412314661292654, + "language_loss": 0.77982366, + "learning_rate": 3.99961420413572e-06, + "loss": 0.80167079, + "num_input_tokens_seen": 34617685, + "router_z_loss_clip": 1.02685547, + "router_z_loss_mlp": 0.25231934, + "step": 1243, + "time_per_iteration": 2.5237298011779785 + }, + { + "auxiliary_loss_clip": 0.01158954, + "auxiliary_loss_mlp": 0.01059086, + "balance_loss_clip": 1.04596376, + "balance_loss_mlp": 1.03150105, + "epoch": 0.03609773083396205, + "flos": 22775415672960.0, + "grad_norm": 2.7855655372268444, + "language_loss": 0.90899563, + "learning_rate": 3.999610503597269e-06, + "loss": 0.93117601, + "num_input_tokens_seen": 34632625, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.27600098, + "step": 1244, + "time_per_iteration": 2.8324077129364014 + }, + { + "auxiliary_loss_clip": 0.0115072, + "auxiliary_loss_mlp": 0.01052109, + "balance_loss_clip": 1.04158545, + "balance_loss_mlp": 1.02397573, + "epoch": 0.03612674830247809, + "flos": 20915487613440.0, + "grad_norm": 2.6992545244488326, + "language_loss": 0.84865189, + "learning_rate": 3.999606785397447e-06, + "loss": 0.87068021, + "num_input_tokens_seen": 34647585, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.28125, + "step": 1245, + "time_per_iteration": 2.417962074279785 + }, + { + "auxiliary_loss_clip": 0.01159129, + "auxiliary_loss_mlp": 0.01067291, + "balance_loss_clip": 1.04557073, + "balance_loss_mlp": 1.03747714, + "epoch": 0.036155765770994136, + "flos": 14601584943360.0, + "grad_norm": 6.00245240568213, + "language_loss": 0.88738215, + "learning_rate": 3.999603049536289e-06, + "loss": 0.90964627, + "num_input_tokens_seen": 34660910, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.2980957, + "step": 1246, + "time_per_iteration": 2.5022411346435547 + }, + { + "auxiliary_loss_clip": 0.01156978, + "auxiliary_loss_mlp": 0.01049854, + "balance_loss_clip": 1.04329264, + "balance_loss_mlp": 1.02218509, + "epoch": 0.03618478323951019, + "flos": 28152702766080.0, + "grad_norm": 5.851288920090744, + "language_loss": 0.73987037, + "learning_rate": 3.999599296013828e-06, + "loss": 0.76193869, + "num_input_tokens_seen": 34682705, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.27648926, + "step": 1247, + "time_per_iteration": 2.569542169570923 + }, + { + "auxiliary_loss_clip": 0.01158441, + "auxiliary_loss_mlp": 0.01070943, + "balance_loss_clip": 1.04322803, + "balance_loss_mlp": 1.03767145, + "epoch": 0.03621380070802623, + "flos": 33173562988800.0, + "grad_norm": 2.8303195860723243, + "language_loss": 1.05366302, + "learning_rate": 3.999595524830097e-06, + "loss": 1.07595694, + "num_input_tokens_seen": 34696720, + "router_z_loss_clip": 1.15234375, + "router_z_loss_mlp": 0.3326416, + "step": 1248, + "time_per_iteration": 2.521052360534668 + }, + { + "auxiliary_loss_clip": 0.01165543, + "auxiliary_loss_mlp": 0.01069538, + "balance_loss_clip": 1.04428601, + "balance_loss_mlp": 1.03987837, + "epoch": 0.03624281817654228, + "flos": 34378590165120.0, + "grad_norm": 3.021657833963417, + "language_loss": 1.10645533, + "learning_rate": 3.999591735985128e-06, + "loss": 1.12880611, + "num_input_tokens_seen": 34715475, + "router_z_loss_clip": 1.21191406, + "router_z_loss_mlp": 0.29675293, + "step": 1249, + "time_per_iteration": 2.507774829864502 + }, + { + "auxiliary_loss_clip": 0.01168279, + "auxiliary_loss_mlp": 0.01062666, + "balance_loss_clip": 1.04856086, + "balance_loss_mlp": 1.03423405, + "epoch": 0.03627183564505833, + "flos": 18068634840960.0, + "grad_norm": 2.452565642400732, + "language_loss": 0.90052617, + "learning_rate": 3.999587929478957e-06, + "loss": 0.92283559, + "num_input_tokens_seen": 34729995, + "router_z_loss_clip": 1.19726562, + "router_z_loss_mlp": 0.28442383, + "step": 1250, + "time_per_iteration": 2.3924992084503174 + }, + { + "auxiliary_loss_clip": 0.01027231, + "auxiliary_loss_mlp": 0.01005322, + "balance_loss_clip": 1.004457, + "balance_loss_mlp": 1.00259209, + "epoch": 0.03630085311357437, + "flos": 69550126329600.0, + "grad_norm": 0.749764126510509, + "language_loss": 0.53098416, + "learning_rate": 3.999584105311616e-06, + "loss": 0.5513097, + "num_input_tokens_seen": 34794175, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.02734375, + "step": 1251, + "time_per_iteration": 3.0747148990631104 + }, + { + "auxiliary_loss_clip": 0.01164605, + "auxiliary_loss_mlp": 0.01059355, + "balance_loss_clip": 1.04773653, + "balance_loss_mlp": 1.03217494, + "epoch": 0.03632987058209042, + "flos": 27990287654400.0, + "grad_norm": 3.389283951981717, + "language_loss": 1.06706023, + "learning_rate": 3.999580263483139e-06, + "loss": 1.0892998, + "num_input_tokens_seen": 34805505, + "router_z_loss_clip": 1.17041016, + "router_z_loss_mlp": 0.27185059, + "step": 1252, + "time_per_iteration": 2.4788546562194824 + }, + { + "auxiliary_loss_clip": 0.01156671, + "auxiliary_loss_mlp": 0.01064085, + "balance_loss_clip": 1.0464108, + "balance_loss_mlp": 1.03741765, + "epoch": 0.03635888805060646, + "flos": 36530636503680.0, + "grad_norm": 3.031483102536523, + "language_loss": 1.02473712, + "learning_rate": 3.999576403993559e-06, + "loss": 1.04694474, + "num_input_tokens_seen": 34821955, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.26660156, + "step": 1253, + "time_per_iteration": 2.59753680229187 + }, + { + "auxiliary_loss_clip": 0.01160586, + "auxiliary_loss_mlp": 0.01077053, + "balance_loss_clip": 1.04643667, + "balance_loss_mlp": 1.04947948, + "epoch": 0.036387905519122514, + "flos": 12122960947200.0, + "grad_norm": 3.0688245971506283, + "language_loss": 0.9510591, + "learning_rate": 3.999572526842912e-06, + "loss": 0.9734354, + "num_input_tokens_seen": 34833335, + "router_z_loss_clip": 1.14257812, + "router_z_loss_mlp": 0.27587891, + "step": 1254, + "time_per_iteration": 2.3925364017486572 + }, + { + "auxiliary_loss_clip": 0.01166443, + "auxiliary_loss_mlp": 0.01069894, + "balance_loss_clip": 1.05153346, + "balance_loss_mlp": 1.04080665, + "epoch": 0.03641692298763856, + "flos": 20003381677440.0, + "grad_norm": 3.569906992062683, + "language_loss": 1.00487614, + "learning_rate": 3.999568632031231e-06, + "loss": 1.02723956, + "num_input_tokens_seen": 34846095, + "router_z_loss_clip": 1.1484375, + "router_z_loss_mlp": 0.29101562, + "step": 1255, + "time_per_iteration": 2.3924336433410645 + }, + { + "auxiliary_loss_clip": 0.01159358, + "auxiliary_loss_mlp": 0.01077953, + "balance_loss_clip": 1.04723895, + "balance_loss_mlp": 1.04778123, + "epoch": 0.0364459404561546, + "flos": 36386235653760.0, + "grad_norm": 2.392907472356167, + "language_loss": 0.83757401, + "learning_rate": 3.9995647195585516e-06, + "loss": 0.85994709, + "num_input_tokens_seen": 34863535, + "router_z_loss_clip": 1.12011719, + "router_z_loss_mlp": 0.30102539, + "step": 1256, + "time_per_iteration": 2.490316390991211 + }, + { + "auxiliary_loss_clip": 0.01155072, + "auxiliary_loss_mlp": 0.01063236, + "balance_loss_clip": 1.04560423, + "balance_loss_mlp": 1.03602052, + "epoch": 0.036474957924670655, + "flos": 30730236243840.0, + "grad_norm": 3.5741111659254803, + "language_loss": 0.83005077, + "learning_rate": 3.999560789424907e-06, + "loss": 0.85223383, + "num_input_tokens_seen": 34879955, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.27233887, + "step": 1257, + "time_per_iteration": 2.694365978240967 + }, + { + "auxiliary_loss_clip": 0.01029442, + "auxiliary_loss_mlp": 0.01007791, + "balance_loss_clip": 1.00650203, + "balance_loss_mlp": 1.00523996, + "epoch": 0.0365039753931867, + "flos": 65673964586880.0, + "grad_norm": 0.7324642968389117, + "language_loss": 0.52110243, + "learning_rate": 3.999556841630332e-06, + "loss": 0.54147476, + "num_input_tokens_seen": 34934395, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.0255127, + "step": 1258, + "time_per_iteration": 3.0226659774780273 + }, + { + "auxiliary_loss_clip": 0.01028803, + "auxiliary_loss_mlp": 0.01007088, + "balance_loss_clip": 1.00570011, + "balance_loss_mlp": 1.00444138, + "epoch": 0.036532992861702744, + "flos": 74764684108800.0, + "grad_norm": 0.6909029706872533, + "language_loss": 0.45154747, + "learning_rate": 3.999552876174862e-06, + "loss": 0.47190636, + "num_input_tokens_seen": 34990630, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.02648926, + "step": 1259, + "time_per_iteration": 2.994269609451294 + }, + { + "auxiliary_loss_clip": 0.0102678, + "auxiliary_loss_mlp": 0.01001698, + "balance_loss_clip": 1.00378418, + "balance_loss_mlp": 0.99909961, + "epoch": 0.03656201033021879, + "flos": 52923403440000.0, + "grad_norm": 0.6376155138372407, + "language_loss": 0.50290895, + "learning_rate": 3.9995488930585315e-06, + "loss": 0.52319372, + "num_input_tokens_seen": 35054365, + "router_z_loss_clip": 0.23046875, + "router_z_loss_mlp": 0.02600098, + "step": 1260, + "time_per_iteration": 3.1136038303375244 + }, + { + "auxiliary_loss_clip": 0.01157553, + "auxiliary_loss_mlp": 0.01059725, + "balance_loss_clip": 1.04713559, + "balance_loss_mlp": 1.0311389, + "epoch": 0.03659102779873484, + "flos": 12778664791680.0, + "grad_norm": 2.769365196065814, + "language_loss": 0.97023487, + "learning_rate": 3.999544892281377e-06, + "loss": 0.99240768, + "num_input_tokens_seen": 35067450, + "router_z_loss_clip": 1.10351562, + "router_z_loss_mlp": 0.2857666, + "step": 1261, + "time_per_iteration": 2.443155288696289 + }, + { + "auxiliary_loss_clip": 0.01160493, + "auxiliary_loss_mlp": 0.0106081, + "balance_loss_clip": 1.04412568, + "balance_loss_mlp": 1.03335571, + "epoch": 0.036620045267250885, + "flos": 34996936988160.0, + "grad_norm": 3.14572807263639, + "language_loss": 1.00341058, + "learning_rate": 3.999540873843432e-06, + "loss": 1.02562356, + "num_input_tokens_seen": 35088020, + "router_z_loss_clip": 1.16455078, + "router_z_loss_mlp": 0.27490234, + "step": 1262, + "time_per_iteration": 2.5585882663726807 + }, + { + "auxiliary_loss_clip": 0.0115657, + "auxiliary_loss_mlp": 0.01066772, + "balance_loss_clip": 1.04481006, + "balance_loss_mlp": 1.03681421, + "epoch": 0.03664906273576693, + "flos": 15483176484480.0, + "grad_norm": 3.19220103141015, + "language_loss": 0.87042034, + "learning_rate": 3.9995368377447335e-06, + "loss": 0.89265382, + "num_input_tokens_seen": 35100805, + "router_z_loss_clip": 1.11669922, + "router_z_loss_mlp": 0.29992676, + "step": 1263, + "time_per_iteration": 2.439981698989868 + }, + { + "auxiliary_loss_clip": 0.01029667, + "auxiliary_loss_mlp": 0.01014715, + "balance_loss_clip": 1.00708449, + "balance_loss_mlp": 1.01177013, + "epoch": 0.03667808020428298, + "flos": 57509455190400.0, + "grad_norm": 0.6555512400027768, + "language_loss": 0.44385231, + "learning_rate": 3.999532783985316e-06, + "loss": 0.46429613, + "num_input_tokens_seen": 35158335, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.02941895, + "step": 1264, + "time_per_iteration": 2.9431204795837402 + }, + { + "auxiliary_loss_clip": 0.01028955, + "auxiliary_loss_mlp": 0.01011944, + "balance_loss_clip": 1.00624382, + "balance_loss_mlp": 1.00934577, + "epoch": 0.036707097672799026, + "flos": 74771701292160.0, + "grad_norm": 0.6901476628973815, + "language_loss": 0.54769635, + "learning_rate": 3.999528712565216e-06, + "loss": 0.56810534, + "num_input_tokens_seen": 35224360, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.02600098, + "step": 1265, + "time_per_iteration": 3.0629327297210693 + }, + { + "auxiliary_loss_clip": 0.01159436, + "auxiliary_loss_mlp": 0.01060088, + "balance_loss_clip": 1.04763281, + "balance_loss_mlp": 1.03184724, + "epoch": 0.03673611514131507, + "flos": 29344079600640.0, + "grad_norm": 2.2002421880048058, + "language_loss": 1.0002017, + "learning_rate": 3.9995246234844694e-06, + "loss": 1.02239692, + "num_input_tokens_seen": 35242600, + "router_z_loss_clip": 1.11669922, + "router_z_loss_mlp": 0.28222656, + "step": 1266, + "time_per_iteration": 2.56701397895813 + }, + { + "auxiliary_loss_clip": 0.01155921, + "auxiliary_loss_mlp": 0.01065661, + "balance_loss_clip": 1.04752207, + "balance_loss_mlp": 1.03711057, + "epoch": 0.036765132609831115, + "flos": 74729004698880.0, + "grad_norm": 2.3272883782623857, + "language_loss": 1.09791923, + "learning_rate": 3.999520516743112e-06, + "loss": 1.12013519, + "num_input_tokens_seen": 35266790, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.28540039, + "step": 1267, + "time_per_iteration": 2.815260648727417 + }, + { + "auxiliary_loss_clip": 0.01160989, + "auxiliary_loss_mlp": 0.01051133, + "balance_loss_clip": 1.04631996, + "balance_loss_mlp": 1.0237031, + "epoch": 0.03679415007834717, + "flos": 14277939840000.0, + "grad_norm": 2.70820500501361, + "language_loss": 0.73885566, + "learning_rate": 3.99951639234118e-06, + "loss": 0.76097685, + "num_input_tokens_seen": 35279500, + "router_z_loss_clip": 1.14648438, + "router_z_loss_mlp": 0.27441406, + "step": 1268, + "time_per_iteration": 2.5104904174804688 + }, + { + "auxiliary_loss_clip": 0.01025203, + "auxiliary_loss_mlp": 0.01013264, + "balance_loss_clip": 1.00225222, + "balance_loss_mlp": 1.01083219, + "epoch": 0.03682316754686321, + "flos": 61522162727040.0, + "grad_norm": 0.7343145689982654, + "language_loss": 0.51686096, + "learning_rate": 3.999512250278711e-06, + "loss": 0.53724563, + "num_input_tokens_seen": 35338470, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.02429199, + "step": 1269, + "time_per_iteration": 2.921720504760742 + }, + { + "auxiliary_loss_clip": 0.01162262, + "auxiliary_loss_mlp": 0.01053393, + "balance_loss_clip": 1.05103993, + "balance_loss_mlp": 1.02595067, + "epoch": 0.036852185015379256, + "flos": 12743507185920.0, + "grad_norm": 27.07018267899468, + "language_loss": 0.85416478, + "learning_rate": 3.99950809055574e-06, + "loss": 0.87632132, + "num_input_tokens_seen": 35357605, + "router_z_loss_clip": 1.11083984, + "router_z_loss_mlp": 0.27453613, + "step": 1270, + "time_per_iteration": 2.896876096725464 + }, + { + "auxiliary_loss_clip": 0.01173218, + "auxiliary_loss_mlp": 0.01073102, + "balance_loss_clip": 1.04867291, + "balance_loss_mlp": 1.03885341, + "epoch": 0.03688120248389531, + "flos": 14165345105280.0, + "grad_norm": 3.319030039945695, + "language_loss": 0.86407971, + "learning_rate": 3.999503913172305e-06, + "loss": 0.88654292, + "num_input_tokens_seen": 35372910, + "router_z_loss_clip": 1.24609375, + "router_z_loss_mlp": 0.34265137, + "step": 1271, + "time_per_iteration": 2.3762521743774414 + }, + { + "auxiliary_loss_clip": 0.01026392, + "auxiliary_loss_mlp": 0.01008208, + "balance_loss_clip": 1.00325835, + "balance_loss_mlp": 1.00582433, + "epoch": 0.03691021995241135, + "flos": 58675344865920.0, + "grad_norm": 0.722693279369546, + "language_loss": 0.48548797, + "learning_rate": 3.999499718128441e-06, + "loss": 0.50583392, + "num_input_tokens_seen": 35426990, + "router_z_loss_clip": 0.23144531, + "router_z_loss_mlp": 0.02380371, + "step": 1272, + "time_per_iteration": 2.9947261810302734 + }, + { + "auxiliary_loss_clip": 0.01157535, + "auxiliary_loss_mlp": 0.01064493, + "balance_loss_clip": 1.04265821, + "balance_loss_mlp": 1.035537, + "epoch": 0.0369392374209274, + "flos": 56816954772480.0, + "grad_norm": 2.4347076364498768, + "language_loss": 0.64065462, + "learning_rate": 3.999495505424188e-06, + "loss": 0.66287494, + "num_input_tokens_seen": 35449765, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.29003906, + "step": 1273, + "time_per_iteration": 2.752802610397339 + }, + { + "auxiliary_loss_clip": 0.01159263, + "auxiliary_loss_mlp": 0.01056209, + "balance_loss_clip": 1.05035257, + "balance_loss_mlp": 1.02637148, + "epoch": 0.03696825488944344, + "flos": 12310653749760.0, + "grad_norm": 2.631812784836928, + "language_loss": 0.84126306, + "learning_rate": 3.9994912750595805e-06, + "loss": 0.8634178, + "num_input_tokens_seen": 35461620, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.2980957, + "step": 1274, + "time_per_iteration": 2.5157220363616943 + }, + { + "auxiliary_loss_clip": 0.01168797, + "auxiliary_loss_mlp": 0.01076583, + "balance_loss_clip": 1.04845119, + "balance_loss_mlp": 1.0459938, + "epoch": 0.03699727235795949, + "flos": 21070850630400.0, + "grad_norm": 2.555325304639503, + "language_loss": 0.80487621, + "learning_rate": 3.999487027034657e-06, + "loss": 0.82733005, + "num_input_tokens_seen": 35476800, + "router_z_loss_clip": 1.20410156, + "router_z_loss_mlp": 0.30566406, + "step": 1275, + "time_per_iteration": 2.45000958442688 + }, + { + "auxiliary_loss_clip": 0.01151221, + "auxiliary_loss_mlp": 0.01054818, + "balance_loss_clip": 1.04343331, + "balance_loss_mlp": 1.02835369, + "epoch": 0.03702628982647554, + "flos": 17666400533760.0, + "grad_norm": 3.4757012012269515, + "language_loss": 0.83693278, + "learning_rate": 3.999482761349455e-06, + "loss": 0.85899317, + "num_input_tokens_seen": 35489345, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.26489258, + "step": 1276, + "time_per_iteration": 2.4471426010131836 + }, + { + "auxiliary_loss_clip": 0.0116654, + "auxiliary_loss_mlp": 0.01059624, + "balance_loss_clip": 1.0478487, + "balance_loss_mlp": 1.02759218, + "epoch": 0.03705530729499158, + "flos": 25990846335360.0, + "grad_norm": 2.178851549631137, + "language_loss": 0.81452501, + "learning_rate": 3.999478478004013e-06, + "loss": 0.83678663, + "num_input_tokens_seen": 35508105, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.32080078, + "step": 1277, + "time_per_iteration": 2.51887583732605 + }, + { + "auxiliary_loss_clip": 0.01156952, + "auxiliary_loss_mlp": 0.01059161, + "balance_loss_clip": 1.04472029, + "balance_loss_mlp": 1.02938259, + "epoch": 0.037084324763507634, + "flos": 34232792860800.0, + "grad_norm": 2.138581237162335, + "language_loss": 0.75540936, + "learning_rate": 3.999474176998368e-06, + "loss": 0.77757049, + "num_input_tokens_seen": 35524875, + "router_z_loss_clip": 1.12304688, + "router_z_loss_mlp": 0.29785156, + "step": 1278, + "time_per_iteration": 2.539072275161743 + }, + { + "auxiliary_loss_clip": 0.01166831, + "auxiliary_loss_mlp": 0.01055495, + "balance_loss_clip": 1.04805553, + "balance_loss_mlp": 1.02603817, + "epoch": 0.03711334223202368, + "flos": 14529873277440.0, + "grad_norm": 8.559483058245032, + "language_loss": 0.82198131, + "learning_rate": 3.999469858332557e-06, + "loss": 0.84420466, + "num_input_tokens_seen": 35536425, + "router_z_loss_clip": 1.18847656, + "router_z_loss_mlp": 0.29467773, + "step": 1279, + "time_per_iteration": 2.434657335281372 + }, + { + "auxiliary_loss_clip": 0.01026, + "auxiliary_loss_mlp": 0.01012186, + "balance_loss_clip": 1.00325429, + "balance_loss_mlp": 1.00962341, + "epoch": 0.03714235970053972, + "flos": 65687580017280.0, + "grad_norm": 0.6305140020272215, + "language_loss": 0.46744949, + "learning_rate": 3.99946552200662e-06, + "loss": 0.48783135, + "num_input_tokens_seen": 35598590, + "router_z_loss_clip": 0.22753906, + "router_z_loss_mlp": 0.02563477, + "step": 1280, + "time_per_iteration": 3.0438039302825928 + }, + { + "auxiliary_loss_clip": 0.01177513, + "auxiliary_loss_mlp": 0.01083052, + "balance_loss_clip": 1.05003119, + "balance_loss_mlp": 1.05136609, + "epoch": 0.037171377169055775, + "flos": 62002429522560.0, + "grad_norm": 22.502760925942262, + "language_loss": 0.96679205, + "learning_rate": 3.999461168020593e-06, + "loss": 0.98939764, + "num_input_tokens_seen": 35626065, + "router_z_loss_clip": 1.27539062, + "router_z_loss_mlp": 0.31689453, + "step": 1281, + "time_per_iteration": 2.8132758140563965 + }, + { + "auxiliary_loss_clip": 0.01149634, + "auxiliary_loss_mlp": 0.0105488, + "balance_loss_clip": 1.04534972, + "balance_loss_mlp": 1.02777159, + "epoch": 0.03720039463757182, + "flos": 51160990273920.0, + "grad_norm": 3.2185462966045746, + "language_loss": 0.89864361, + "learning_rate": 3.999456796374517e-06, + "loss": 0.92068875, + "num_input_tokens_seen": 35646075, + "router_z_loss_clip": 1.04150391, + "router_z_loss_mlp": 0.2713623, + "step": 1282, + "time_per_iteration": 2.6518402099609375 + }, + { + "auxiliary_loss_clip": 0.01164797, + "auxiliary_loss_mlp": 0.01063367, + "balance_loss_clip": 1.04834449, + "balance_loss_mlp": 1.03513813, + "epoch": 0.037229412106087864, + "flos": 30911575178880.0, + "grad_norm": 2.646785524690227, + "language_loss": 0.83403945, + "learning_rate": 3.9994524070684295e-06, + "loss": 0.8563211, + "num_input_tokens_seen": 35660425, + "router_z_loss_clip": 1.1640625, + "router_z_loss_mlp": 0.28234863, + "step": 1283, + "time_per_iteration": 2.7953853607177734 + }, + { + "auxiliary_loss_clip": 0.01024514, + "auxiliary_loss_mlp": 0.01002353, + "balance_loss_clip": 1.00237298, + "balance_loss_mlp": 0.99998057, + "epoch": 0.03725842957460391, + "flos": 74771840937600.0, + "grad_norm": 0.7311840565371566, + "language_loss": 0.53405505, + "learning_rate": 3.999448000102369e-06, + "loss": 0.55432373, + "num_input_tokens_seen": 35722125, + "router_z_loss_clip": 0.22167969, + "router_z_loss_mlp": 0.02368164, + "step": 1284, + "time_per_iteration": 3.0810675621032715 + }, + { + "auxiliary_loss_clip": 0.01153561, + "auxiliary_loss_mlp": 0.01051417, + "balance_loss_clip": 1.04591703, + "balance_loss_mlp": 1.02534556, + "epoch": 0.03728744704311996, + "flos": 27118296737280.0, + "grad_norm": 2.1577827412948922, + "language_loss": 0.74582839, + "learning_rate": 3.999443575476374e-06, + "loss": 0.76787817, + "num_input_tokens_seen": 35738030, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.26062012, + "step": 1285, + "time_per_iteration": 2.5567119121551514 + }, + { + "auxiliary_loss_clip": 0.01025304, + "auxiliary_loss_mlp": 0.01001489, + "balance_loss_clip": 1.00322366, + "balance_loss_mlp": 0.99909288, + "epoch": 0.037316464511636005, + "flos": 62303308686720.0, + "grad_norm": 0.669918644334085, + "language_loss": 0.51966178, + "learning_rate": 3.999439133190486e-06, + "loss": 0.53992969, + "num_input_tokens_seen": 35802555, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.02392578, + "step": 1286, + "time_per_iteration": 3.1178152561187744 + }, + { + "auxiliary_loss_clip": 0.01157965, + "auxiliary_loss_mlp": 0.01060325, + "balance_loss_clip": 1.04593003, + "balance_loss_mlp": 1.03229916, + "epoch": 0.03734548198015205, + "flos": 22303459647360.0, + "grad_norm": 2.6202228456300594, + "language_loss": 0.72077119, + "learning_rate": 3.999434673244741e-06, + "loss": 0.74295408, + "num_input_tokens_seen": 35817845, + "router_z_loss_clip": 1.11962891, + "router_z_loss_mlp": 0.28039551, + "step": 1287, + "time_per_iteration": 2.5132896900177 + }, + { + "auxiliary_loss_clip": 0.01167599, + "auxiliary_loss_mlp": 0.01057116, + "balance_loss_clip": 1.04770041, + "balance_loss_mlp": 1.02497745, + "epoch": 0.0373744994486681, + "flos": 31243773565440.0, + "grad_norm": 1.9185483301431863, + "language_loss": 0.88902593, + "learning_rate": 3.99943019563918e-06, + "loss": 0.91127312, + "num_input_tokens_seen": 35838180, + "router_z_loss_clip": 1.19921875, + "router_z_loss_mlp": 0.32104492, + "step": 1288, + "time_per_iteration": 2.577796459197998 + }, + { + "auxiliary_loss_clip": 0.01164428, + "auxiliary_loss_mlp": 0.01070152, + "balance_loss_clip": 1.04901934, + "balance_loss_mlp": 1.03949118, + "epoch": 0.037403516917184146, + "flos": 74730086951040.0, + "grad_norm": 2.0926064212987914, + "language_loss": 0.8681252, + "learning_rate": 3.999425700373843e-06, + "loss": 0.89047098, + "num_input_tokens_seen": 35865075, + "router_z_loss_clip": 1.15673828, + "router_z_loss_mlp": 0.30688477, + "step": 1289, + "time_per_iteration": 2.863882303237915 + }, + { + "auxiliary_loss_clip": 0.01161286, + "auxiliary_loss_mlp": 0.01061238, + "balance_loss_clip": 1.04754949, + "balance_loss_mlp": 1.03151894, + "epoch": 0.03743253438570019, + "flos": 27921159429120.0, + "grad_norm": 3.044601286508377, + "language_loss": 0.80903435, + "learning_rate": 3.999421187448769e-06, + "loss": 0.83125961, + "num_input_tokens_seen": 35879290, + "router_z_loss_clip": 1.13720703, + "router_z_loss_mlp": 0.29699707, + "step": 1290, + "time_per_iteration": 2.518390417098999 + }, + { + "auxiliary_loss_clip": 0.01177091, + "auxiliary_loss_mlp": 0.0106999, + "balance_loss_clip": 1.04779816, + "balance_loss_mlp": 1.03626537, + "epoch": 0.037461551854216235, + "flos": 25767786458880.0, + "grad_norm": 2.4068123447472054, + "language_loss": 1.08765697, + "learning_rate": 3.999416656863998e-06, + "loss": 1.11012781, + "num_input_tokens_seen": 35895065, + "router_z_loss_clip": 1.29199219, + "router_z_loss_mlp": 0.33728027, + "step": 1291, + "time_per_iteration": 2.50107741355896 + }, + { + "auxiliary_loss_clip": 0.01158163, + "auxiliary_loss_mlp": 0.0106185, + "balance_loss_clip": 1.04381514, + "balance_loss_mlp": 1.03612471, + "epoch": 0.03749056932273229, + "flos": 19123326236160.0, + "grad_norm": 4.643222806891402, + "language_loss": 1.00551796, + "learning_rate": 3.9994121086195695e-06, + "loss": 1.02771831, + "num_input_tokens_seen": 35905865, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.25720215, + "step": 1292, + "time_per_iteration": 2.452967405319214 + }, + { + "auxiliary_loss_clip": 0.01022557, + "auxiliary_loss_mlp": 0.0102641, + "balance_loss_clip": 1.00102878, + "balance_loss_mlp": 1.02416933, + "epoch": 0.03751958679124833, + "flos": 69848528653440.0, + "grad_norm": 0.6809636534466839, + "language_loss": 0.47636005, + "learning_rate": 3.999407542715524e-06, + "loss": 0.49684972, + "num_input_tokens_seen": 35968590, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.02246094, + "step": 1293, + "time_per_iteration": 3.1334915161132812 + }, + { + "auxiliary_loss_clip": 0.01155363, + "auxiliary_loss_mlp": 0.0105585, + "balance_loss_clip": 1.04262209, + "balance_loss_mlp": 1.0287416, + "epoch": 0.037548604259764376, + "flos": 20550784884480.0, + "grad_norm": 2.5536815663805594, + "language_loss": 0.82590824, + "learning_rate": 3.999402959151903e-06, + "loss": 0.84802037, + "num_input_tokens_seen": 35982540, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.27111816, + "step": 1294, + "time_per_iteration": 4.800007104873657 + }, + { + "auxiliary_loss_clip": 0.01157423, + "auxiliary_loss_mlp": 0.01065649, + "balance_loss_clip": 1.04568172, + "balance_loss_mlp": 1.03798103, + "epoch": 0.03757762172828043, + "flos": 23981874215040.0, + "grad_norm": 3.115399646906947, + "language_loss": 0.80107975, + "learning_rate": 3.9993983579287454e-06, + "loss": 0.82331049, + "num_input_tokens_seen": 35999585, + "router_z_loss_clip": 1.11669922, + "router_z_loss_mlp": 0.2767334, + "step": 1295, + "time_per_iteration": 2.7797799110412598 + }, + { + "auxiliary_loss_clip": 0.01159137, + "auxiliary_loss_mlp": 0.01063418, + "balance_loss_clip": 1.04531562, + "balance_loss_mlp": 1.03428292, + "epoch": 0.03760663919679647, + "flos": 32956193664000.0, + "grad_norm": 2.350672401486514, + "language_loss": 0.88970524, + "learning_rate": 3.999393739046093e-06, + "loss": 0.9119308, + "num_input_tokens_seen": 36015895, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.29138184, + "step": 1296, + "time_per_iteration": 2.492565631866455 + }, + { + "auxiliary_loss_clip": 0.01157362, + "auxiliary_loss_mlp": 0.01055876, + "balance_loss_clip": 1.04757452, + "balance_loss_mlp": 1.02768326, + "epoch": 0.03763565666531252, + "flos": 26648295747840.0, + "grad_norm": 2.871554144004983, + "language_loss": 0.89734173, + "learning_rate": 3.999389102503985e-06, + "loss": 0.91947412, + "num_input_tokens_seen": 36029950, + "router_z_loss_clip": 1.09765625, + "router_z_loss_mlp": 0.28222656, + "step": 1297, + "time_per_iteration": 2.459803342819214 + }, + { + "auxiliary_loss_clip": 0.01161462, + "auxiliary_loss_mlp": 0.01053679, + "balance_loss_clip": 1.05000544, + "balance_loss_mlp": 1.02767992, + "epoch": 0.03766467413382856, + "flos": 14605355370240.0, + "grad_norm": 2.5007533032506286, + "language_loss": 0.85299098, + "learning_rate": 3.999384448302464e-06, + "loss": 0.87514246, + "num_input_tokens_seen": 36041645, + "router_z_loss_clip": 1.11523438, + "router_z_loss_mlp": 0.26025391, + "step": 1298, + "time_per_iteration": 4.667646408081055 + }, + { + "auxiliary_loss_clip": 0.01158551, + "auxiliary_loss_mlp": 0.01060895, + "balance_loss_clip": 1.04503274, + "balance_loss_mlp": 1.03272629, + "epoch": 0.03769369160234461, + "flos": 29496614797440.0, + "grad_norm": 1.884202621400397, + "language_loss": 0.85410994, + "learning_rate": 3.999379776441571e-06, + "loss": 0.87630439, + "num_input_tokens_seen": 36062860, + "router_z_loss_clip": 1.13574219, + "router_z_loss_mlp": 0.28161621, + "step": 1299, + "time_per_iteration": 2.5270323753356934 + }, + { + "auxiliary_loss_clip": 0.01149735, + "auxiliary_loss_mlp": 0.01056903, + "balance_loss_clip": 1.04775, + "balance_loss_mlp": 1.03271556, + "epoch": 0.03772270907086066, + "flos": 39997057996800.0, + "grad_norm": 3.6130768687988173, + "language_loss": 0.66486466, + "learning_rate": 3.999375086921346e-06, + "loss": 0.68693101, + "num_input_tokens_seen": 36079885, + "router_z_loss_clip": 1.01953125, + "router_z_loss_mlp": 0.24194336, + "step": 1300, + "time_per_iteration": 2.530292272567749 + }, + { + "auxiliary_loss_clip": 0.01157641, + "auxiliary_loss_mlp": 0.01057818, + "balance_loss_clip": 1.04585314, + "balance_loss_mlp": 1.03252184, + "epoch": 0.0377517265393767, + "flos": 32372655333120.0, + "grad_norm": 2.14528919454135, + "language_loss": 0.83663332, + "learning_rate": 3.999370379741831e-06, + "loss": 0.85878789, + "num_input_tokens_seen": 36097680, + "router_z_loss_clip": 1.11767578, + "router_z_loss_mlp": 0.25305176, + "step": 1301, + "time_per_iteration": 5.044700384140015 + }, + { + "auxiliary_loss_clip": 0.01041198, + "auxiliary_loss_mlp": 0.01010445, + "balance_loss_clip": 1.01848459, + "balance_loss_mlp": 1.00796545, + "epoch": 0.037780744007892754, + "flos": 62691613361280.0, + "grad_norm": 0.7102102985691682, + "language_loss": 0.50409764, + "learning_rate": 3.999365654903069e-06, + "loss": 0.5246141, + "num_input_tokens_seen": 36158395, + "router_z_loss_clip": 0.22753906, + "router_z_loss_mlp": 0.02478027, + "step": 1302, + "time_per_iteration": 3.0560290813446045 + }, + { + "auxiliary_loss_clip": 0.01159942, + "auxiliary_loss_mlp": 0.01054677, + "balance_loss_clip": 1.04511666, + "balance_loss_mlp": 1.02681768, + "epoch": 0.0378097614764088, + "flos": 33392084388480.0, + "grad_norm": 6.350036381870851, + "language_loss": 0.73874122, + "learning_rate": 3.999360912405099e-06, + "loss": 0.7608875, + "num_input_tokens_seen": 36172330, + "router_z_loss_clip": 1.14794922, + "router_z_loss_mlp": 0.27856445, + "step": 1303, + "time_per_iteration": 2.5640811920166016 + }, + { + "auxiliary_loss_clip": 0.01162278, + "auxiliary_loss_mlp": 0.01064218, + "balance_loss_clip": 1.04566598, + "balance_loss_mlp": 1.03538179, + "epoch": 0.03783877894492484, + "flos": 26094119736960.0, + "grad_norm": 5.6232775955784415, + "language_loss": 1.02722132, + "learning_rate": 3.999356152247965e-06, + "loss": 1.04948616, + "num_input_tokens_seen": 36185345, + "router_z_loss_clip": 1.16552734, + "router_z_loss_mlp": 0.28845215, + "step": 1304, + "time_per_iteration": 5.955381870269775 + }, + { + "auxiliary_loss_clip": 0.01170002, + "auxiliary_loss_mlp": 0.01071538, + "balance_loss_clip": 1.04602945, + "balance_loss_mlp": 1.03920817, + "epoch": 0.037867796413440895, + "flos": 40032983652480.0, + "grad_norm": 2.8210041953601253, + "language_loss": 0.88317102, + "learning_rate": 3.999351374431708e-06, + "loss": 0.90558636, + "num_input_tokens_seen": 36200395, + "router_z_loss_clip": 1.24023438, + "router_z_loss_mlp": 0.32348633, + "step": 1305, + "time_per_iteration": 2.8076815605163574 + }, + { + "auxiliary_loss_clip": 0.01148955, + "auxiliary_loss_mlp": 0.0104848, + "balance_loss_clip": 1.04229712, + "balance_loss_mlp": 1.02076399, + "epoch": 0.03789681388195694, + "flos": 24601303290240.0, + "grad_norm": 2.7082372248863917, + "language_loss": 0.89356321, + "learning_rate": 3.9993465789563715e-06, + "loss": 0.91553766, + "num_input_tokens_seen": 36215545, + "router_z_loss_clip": 1.06689453, + "router_z_loss_mlp": 0.27722168, + "step": 1306, + "time_per_iteration": 2.4883909225463867 + }, + { + "auxiliary_loss_clip": 0.01161429, + "auxiliary_loss_mlp": 0.01062938, + "balance_loss_clip": 1.04708791, + "balance_loss_mlp": 1.0342207, + "epoch": 0.037925831350472984, + "flos": 17009230412160.0, + "grad_norm": 4.776124484070097, + "language_loss": 0.97606075, + "learning_rate": 3.999341765821997e-06, + "loss": 0.99830437, + "num_input_tokens_seen": 36227725, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.28735352, + "step": 1307, + "time_per_iteration": 2.4019978046417236 + }, + { + "auxiliary_loss_clip": 0.01158508, + "auxiliary_loss_mlp": 0.01064491, + "balance_loss_clip": 1.04718065, + "balance_loss_mlp": 1.03713298, + "epoch": 0.03795484881898903, + "flos": 24016508150400.0, + "grad_norm": 3.447038233387574, + "language_loss": 0.81001484, + "learning_rate": 3.999336935028626e-06, + "loss": 0.83224487, + "num_input_tokens_seen": 36243030, + "router_z_loss_clip": 1.11328125, + "router_z_loss_mlp": 0.27355957, + "step": 1308, + "time_per_iteration": 2.4971370697021484 + }, + { + "auxiliary_loss_clip": 0.01152741, + "auxiliary_loss_mlp": 0.01052686, + "balance_loss_clip": 1.04634094, + "balance_loss_mlp": 1.02666295, + "epoch": 0.03798386628750508, + "flos": 18981543738240.0, + "grad_norm": 2.795100028510942, + "language_loss": 0.71558571, + "learning_rate": 3.999332086576302e-06, + "loss": 0.73763996, + "num_input_tokens_seen": 36255530, + "router_z_loss_clip": 1.06347656, + "router_z_loss_mlp": 0.26037598, + "step": 1309, + "time_per_iteration": 2.3939809799194336 + }, + { + "auxiliary_loss_clip": 0.01161795, + "auxiliary_loss_mlp": 0.01060099, + "balance_loss_clip": 1.04637969, + "balance_loss_mlp": 1.03240705, + "epoch": 0.038012883756021125, + "flos": 34670289507840.0, + "grad_norm": 2.599886315933657, + "language_loss": 0.84706581, + "learning_rate": 3.999327220465069e-06, + "loss": 0.86928475, + "num_input_tokens_seen": 36273665, + "router_z_loss_clip": 1.15527344, + "router_z_loss_mlp": 0.27697754, + "step": 1310, + "time_per_iteration": 2.5913758277893066 + }, + { + "auxiliary_loss_clip": 0.0115729, + "auxiliary_loss_mlp": 0.01061527, + "balance_loss_clip": 1.04342341, + "balance_loss_mlp": 1.03359628, + "epoch": 0.03804190122453717, + "flos": 32371642903680.0, + "grad_norm": 2.47087578417044, + "language_loss": 0.7688511, + "learning_rate": 3.999322336694969e-06, + "loss": 0.79103923, + "num_input_tokens_seen": 36288415, + "router_z_loss_clip": 1.13867188, + "router_z_loss_mlp": 0.27929688, + "step": 1311, + "time_per_iteration": 2.5480868816375732 + }, + { + "auxiliary_loss_clip": 0.01168561, + "auxiliary_loss_mlp": 0.0106336, + "balance_loss_clip": 1.04892623, + "balance_loss_mlp": 1.0341537, + "epoch": 0.03807091869305322, + "flos": 30217117858560.0, + "grad_norm": 2.665740078296969, + "language_loss": 1.1229775, + "learning_rate": 3.9993174352660435e-06, + "loss": 1.14529657, + "num_input_tokens_seen": 36303815, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.29211426, + "step": 1312, + "time_per_iteration": 2.513087034225464 + }, + { + "auxiliary_loss_clip": 0.01161761, + "auxiliary_loss_mlp": 0.0105634, + "balance_loss_clip": 1.04711449, + "balance_loss_mlp": 1.02927995, + "epoch": 0.038099936161569266, + "flos": 33940046177280.0, + "grad_norm": 3.0256501387888757, + "language_loss": 0.98742139, + "learning_rate": 3.9993125161783395e-06, + "loss": 1.00960243, + "num_input_tokens_seen": 36323515, + "router_z_loss_clip": 1.14648438, + "router_z_loss_mlp": 0.27038574, + "step": 1313, + "time_per_iteration": 2.514920711517334 + }, + { + "auxiliary_loss_clip": 0.01171116, + "auxiliary_loss_mlp": 0.01061181, + "balance_loss_clip": 1.04876494, + "balance_loss_mlp": 1.03031766, + "epoch": 0.03812895363008531, + "flos": 13809858975360.0, + "grad_norm": 2.655977267794288, + "language_loss": 0.97173464, + "learning_rate": 3.999307579431897e-06, + "loss": 0.99405766, + "num_input_tokens_seen": 36337335, + "router_z_loss_clip": 1.22265625, + "router_z_loss_mlp": 0.30847168, + "step": 1314, + "time_per_iteration": 2.4593451023101807 + }, + { + "auxiliary_loss_clip": 0.01154387, + "auxiliary_loss_mlp": 0.01045847, + "balance_loss_clip": 1.04890215, + "balance_loss_mlp": 1.02185035, + "epoch": 0.038157971098601355, + "flos": 11429446233600.0, + "grad_norm": 4.336115432486163, + "language_loss": 0.70464897, + "learning_rate": 3.999302625026761e-06, + "loss": 0.72665131, + "num_input_tokens_seen": 36348865, + "router_z_loss_clip": 1.05419922, + "router_z_loss_mlp": 0.23999023, + "step": 1315, + "time_per_iteration": 2.3874592781066895 + }, + { + "auxiliary_loss_clip": 0.01164002, + "auxiliary_loss_mlp": 0.01078122, + "balance_loss_clip": 1.04745126, + "balance_loss_mlp": 1.04779506, + "epoch": 0.03818698856711741, + "flos": 16829287931520.0, + "grad_norm": 5.597584145864065, + "language_loss": 1.03356171, + "learning_rate": 3.999297652962975e-06, + "loss": 1.05598283, + "num_input_tokens_seen": 36362765, + "router_z_loss_clip": 1.16699219, + "router_z_loss_mlp": 0.30310059, + "step": 1316, + "time_per_iteration": 2.4393293857574463 + }, + { + "auxiliary_loss_clip": 0.01038762, + "auxiliary_loss_mlp": 0.01002274, + "balance_loss_clip": 1.01616037, + "balance_loss_mlp": 0.99944836, + "epoch": 0.03821600603563345, + "flos": 62584988469120.0, + "grad_norm": 0.6911624332764204, + "language_loss": 0.51624942, + "learning_rate": 3.999292663240584e-06, + "loss": 0.53665984, + "num_input_tokens_seen": 36425535, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.02819824, + "step": 1317, + "time_per_iteration": 3.0965423583984375 + }, + { + "auxiliary_loss_clip": 0.01036363, + "auxiliary_loss_mlp": 0.01001796, + "balance_loss_clip": 1.01358199, + "balance_loss_mlp": 0.99886316, + "epoch": 0.038245023504149496, + "flos": 60398203461120.0, + "grad_norm": 0.6900031147368164, + "language_loss": 0.52791357, + "learning_rate": 3.999287655859631e-06, + "loss": 0.54829514, + "num_input_tokens_seen": 36487280, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.02929688, + "step": 1318, + "time_per_iteration": 2.9508728981018066 + }, + { + "auxiliary_loss_clip": 0.01164007, + "auxiliary_loss_mlp": 0.01059065, + "balance_loss_clip": 1.04698324, + "balance_loss_mlp": 1.02872622, + "epoch": 0.03827404097266555, + "flos": 45657944997120.0, + "grad_norm": 4.00014728639902, + "language_loss": 0.91189724, + "learning_rate": 3.99928263082016e-06, + "loss": 0.93412793, + "num_input_tokens_seen": 36513100, + "router_z_loss_clip": 1.16894531, + "router_z_loss_mlp": 0.3034668, + "step": 1319, + "time_per_iteration": 2.787938356399536 + }, + { + "auxiliary_loss_clip": 0.0116361, + "auxiliary_loss_mlp": 0.0106113, + "balance_loss_clip": 1.04412079, + "balance_loss_mlp": 1.02990925, + "epoch": 0.03830305844118159, + "flos": 27080695336320.0, + "grad_norm": 2.5306895327772225, + "language_loss": 0.94051284, + "learning_rate": 3.999277588122215e-06, + "loss": 0.96276021, + "num_input_tokens_seen": 36530980, + "router_z_loss_clip": 1.1953125, + "router_z_loss_mlp": 0.31237793, + "step": 1320, + "time_per_iteration": 2.607597827911377 + }, + { + "auxiliary_loss_clip": 0.01149332, + "auxiliary_loss_mlp": 0.01053023, + "balance_loss_clip": 1.04169381, + "balance_loss_mlp": 1.02314949, + "epoch": 0.03833207590969764, + "flos": 22776812127360.0, + "grad_norm": 2.2492453320266534, + "language_loss": 0.86506104, + "learning_rate": 3.999272527765843e-06, + "loss": 0.8870846, + "num_input_tokens_seen": 36546405, + "router_z_loss_clip": 1.07617188, + "router_z_loss_mlp": 0.29882812, + "step": 1321, + "time_per_iteration": 2.7075729370117188 + }, + { + "auxiliary_loss_clip": 0.01027687, + "auxiliary_loss_mlp": 0.01008193, + "balance_loss_clip": 1.00552201, + "balance_loss_mlp": 1.00551093, + "epoch": 0.03836109337821368, + "flos": 69484838353920.0, + "grad_norm": 0.7350073375518746, + "language_loss": 0.57224995, + "learning_rate": 3.999267449751085e-06, + "loss": 0.59260875, + "num_input_tokens_seen": 36608175, + "router_z_loss_clip": 0.22167969, + "router_z_loss_mlp": 0.02685547, + "step": 1322, + "time_per_iteration": 3.0500996112823486 + }, + { + "auxiliary_loss_clip": 0.01169317, + "auxiliary_loss_mlp": 0.01084834, + "balance_loss_clip": 1.04806232, + "balance_loss_mlp": 1.05196798, + "epoch": 0.03839011084672973, + "flos": 39374870924160.0, + "grad_norm": 2.650297634400174, + "language_loss": 1.01519668, + "learning_rate": 3.99926235407799e-06, + "loss": 1.0377382, + "num_input_tokens_seen": 36625190, + "router_z_loss_clip": 1.21289062, + "router_z_loss_mlp": 0.32861328, + "step": 1323, + "time_per_iteration": 2.587031364440918 + }, + { + "auxiliary_loss_clip": 0.01166169, + "auxiliary_loss_mlp": 0.01058222, + "balance_loss_clip": 1.04871154, + "balance_loss_mlp": 1.02961159, + "epoch": 0.03841912831524578, + "flos": 12012460894080.0, + "grad_norm": 3.2446227014684013, + "language_loss": 0.65381205, + "learning_rate": 3.999257240746599e-06, + "loss": 0.67605591, + "num_input_tokens_seen": 36637880, + "router_z_loss_clip": 1.17382812, + "router_z_loss_mlp": 0.28564453, + "step": 1324, + "time_per_iteration": 2.3813273906707764 + }, + { + "auxiliary_loss_clip": 0.01157138, + "auxiliary_loss_mlp": 0.01062184, + "balance_loss_clip": 1.04794979, + "balance_loss_mlp": 1.03489661, + "epoch": 0.03844814578376182, + "flos": 55868120219520.0, + "grad_norm": 2.9447684818150712, + "language_loss": 0.74235034, + "learning_rate": 3.99925210975696e-06, + "loss": 0.76454353, + "num_input_tokens_seen": 36658220, + "router_z_loss_clip": 1.09423828, + "router_z_loss_mlp": 0.27282715, + "step": 1325, + "time_per_iteration": 2.660217046737671 + }, + { + "auxiliary_loss_clip": 0.01155832, + "auxiliary_loss_mlp": 0.01061212, + "balance_loss_clip": 1.04559803, + "balance_loss_mlp": 1.03317404, + "epoch": 0.038477163252277874, + "flos": 18145548299520.0, + "grad_norm": 2.655353246219437, + "language_loss": 0.83076179, + "learning_rate": 3.9992469611091175e-06, + "loss": 0.85293221, + "num_input_tokens_seen": 36670780, + "router_z_loss_clip": 1.10253906, + "router_z_loss_mlp": 0.28063965, + "step": 1326, + "time_per_iteration": 2.396671772003174 + }, + { + "auxiliary_loss_clip": 0.01036059, + "auxiliary_loss_mlp": 0.01008898, + "balance_loss_clip": 1.01340866, + "balance_loss_mlp": 1.00646567, + "epoch": 0.03850618072079392, + "flos": 74767756308480.0, + "grad_norm": 0.6651866107563937, + "language_loss": 0.50353652, + "learning_rate": 3.999241794803117e-06, + "loss": 0.5239861, + "num_input_tokens_seen": 36738790, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.02429199, + "step": 1327, + "time_per_iteration": 3.2538230419158936 + }, + { + "auxiliary_loss_clip": 0.01168783, + "auxiliary_loss_mlp": 0.01064699, + "balance_loss_clip": 1.04832649, + "balance_loss_mlp": 1.03363311, + "epoch": 0.03853519818930996, + "flos": 16832045928960.0, + "grad_norm": 3.369773470391444, + "language_loss": 0.85710132, + "learning_rate": 3.999236610839003e-06, + "loss": 0.87943608, + "num_input_tokens_seen": 36752300, + "router_z_loss_clip": 1.203125, + "router_z_loss_mlp": 0.31066895, + "step": 1328, + "time_per_iteration": 2.399304151535034 + }, + { + "auxiliary_loss_clip": 0.01034654, + "auxiliary_loss_mlp": 0.01015884, + "balance_loss_clip": 1.01141489, + "balance_loss_mlp": 1.01345205, + "epoch": 0.03856421565782601, + "flos": 60940544520960.0, + "grad_norm": 0.673706883228896, + "language_loss": 0.48714656, + "learning_rate": 3.999231409216823e-06, + "loss": 0.50765193, + "num_input_tokens_seen": 36815850, + "router_z_loss_clip": 0.23242188, + "router_z_loss_mlp": 0.02429199, + "step": 1329, + "time_per_iteration": 3.160860538482666 + }, + { + "auxiliary_loss_clip": 0.01032122, + "auxiliary_loss_mlp": 0.01005049, + "balance_loss_clip": 1.00931466, + "balance_loss_mlp": 1.00254571, + "epoch": 0.03859323312634206, + "flos": 64728935372160.0, + "grad_norm": 0.7032693577081722, + "language_loss": 0.53846216, + "learning_rate": 3.9992261899366226e-06, + "loss": 0.5588339, + "num_input_tokens_seen": 36873510, + "router_z_loss_clip": 0.22851562, + "router_z_loss_mlp": 0.02502441, + "step": 1330, + "time_per_iteration": 3.021531820297241 + }, + { + "auxiliary_loss_clip": 0.01158565, + "auxiliary_loss_mlp": 0.0105614, + "balance_loss_clip": 1.04441619, + "balance_loss_mlp": 1.02766156, + "epoch": 0.038622250594858104, + "flos": 12596906920320.0, + "grad_norm": 3.0831121002554935, + "language_loss": 0.93683839, + "learning_rate": 3.999220952998446e-06, + "loss": 0.95898533, + "num_input_tokens_seen": 36885815, + "router_z_loss_clip": 1.14355469, + "router_z_loss_mlp": 0.28479004, + "step": 1331, + "time_per_iteration": 2.491931915283203 + }, + { + "auxiliary_loss_clip": 0.01158793, + "auxiliary_loss_mlp": 0.01065384, + "balance_loss_clip": 1.04624248, + "balance_loss_mlp": 1.03967083, + "epoch": 0.03865126806337415, + "flos": 36021463102080.0, + "grad_norm": 1.9097061473602757, + "language_loss": 0.88224697, + "learning_rate": 3.999215698402342e-06, + "loss": 0.90448874, + "num_input_tokens_seen": 36911440, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.25708008, + "step": 1332, + "time_per_iteration": 2.8025200366973877 + }, + { + "auxiliary_loss_clip": 0.01161935, + "auxiliary_loss_mlp": 0.01062914, + "balance_loss_clip": 1.04899979, + "balance_loss_mlp": 1.0354836, + "epoch": 0.0386802855318902, + "flos": 19930552848000.0, + "grad_norm": 2.908463840628232, + "language_loss": 0.91646409, + "learning_rate": 3.999210426148356e-06, + "loss": 0.93871266, + "num_input_tokens_seen": 36924340, + "router_z_loss_clip": 1.12841797, + "router_z_loss_mlp": 0.27429199, + "step": 1333, + "time_per_iteration": 2.4551098346710205 + }, + { + "auxiliary_loss_clip": 0.01027244, + "auxiliary_loss_mlp": 0.01028493, + "balance_loss_clip": 1.00475001, + "balance_loss_mlp": 1.02582288, + "epoch": 0.038709303000406245, + "flos": 74790413781120.0, + "grad_norm": 0.6470589048584962, + "language_loss": 0.50268543, + "learning_rate": 3.9992051362365346e-06, + "loss": 0.52324283, + "num_input_tokens_seen": 36997080, + "router_z_loss_clip": 0.22460938, + "router_z_loss_mlp": 0.0267334, + "step": 1334, + "time_per_iteration": 3.255740165710449 + }, + { + "auxiliary_loss_clip": 0.01027362, + "auxiliary_loss_mlp": 0.01012224, + "balance_loss_clip": 1.00470185, + "balance_loss_mlp": 1.00967288, + "epoch": 0.03873832046892229, + "flos": 74766499499520.0, + "grad_norm": 0.7053730884721106, + "language_loss": 0.50683439, + "learning_rate": 3.9991998286669245e-06, + "loss": 0.52723026, + "num_input_tokens_seen": 37065425, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.0255127, + "step": 1335, + "time_per_iteration": 3.3223116397857666 + }, + { + "auxiliary_loss_clip": 0.01159344, + "auxiliary_loss_mlp": 0.01048619, + "balance_loss_clip": 1.0459609, + "balance_loss_mlp": 1.02196383, + "epoch": 0.03876733793743834, + "flos": 38065627739520.0, + "grad_norm": 1.6532439844427658, + "language_loss": 0.86709571, + "learning_rate": 3.999194503439572e-06, + "loss": 0.8891753, + "num_input_tokens_seen": 37096820, + "router_z_loss_clip": 1.13378906, + "router_z_loss_mlp": 0.26623535, + "step": 1336, + "time_per_iteration": 2.909201145172119 + }, + { + "auxiliary_loss_clip": 0.01153907, + "auxiliary_loss_mlp": 0.01050024, + "balance_loss_clip": 1.04730248, + "balance_loss_mlp": 1.02360773, + "epoch": 0.038796355405954386, + "flos": 15407345278080.0, + "grad_norm": 2.6837102110160256, + "language_loss": 0.94586277, + "learning_rate": 3.999189160554525e-06, + "loss": 0.96790206, + "num_input_tokens_seen": 37110115, + "router_z_loss_clip": 1.06591797, + "router_z_loss_mlp": 0.26379395, + "step": 1337, + "time_per_iteration": 2.421085834503174 + }, + { + "auxiliary_loss_clip": 0.01172007, + "auxiliary_loss_mlp": 0.0106611, + "balance_loss_clip": 1.05065274, + "balance_loss_mlp": 1.03346992, + "epoch": 0.03882537287447043, + "flos": 16499742808320.0, + "grad_norm": 3.3132310087702694, + "language_loss": 0.7866174, + "learning_rate": 3.99918380001183e-06, + "loss": 0.80899858, + "num_input_tokens_seen": 37123825, + "router_z_loss_clip": 1.21386719, + "router_z_loss_mlp": 0.32629395, + "step": 1338, + "time_per_iteration": 2.360137939453125 + }, + { + "auxiliary_loss_clip": 0.01060206, + "auxiliary_loss_mlp": 0.01030361, + "balance_loss_clip": 1.03558564, + "balance_loss_mlp": 1.0276432, + "epoch": 0.038854390342986475, + "flos": 74766220208640.0, + "grad_norm": 0.697414246028015, + "language_loss": 0.51785862, + "learning_rate": 3.999178421811535e-06, + "loss": 0.5387643, + "num_input_tokens_seen": 37185365, + "router_z_loss_clip": 0.24609375, + "router_z_loss_mlp": 0.02722168, + "step": 1339, + "time_per_iteration": 3.0426642894744873 + }, + { + "auxiliary_loss_clip": 0.01168739, + "auxiliary_loss_mlp": 0.01066232, + "balance_loss_clip": 1.0490799, + "balance_loss_mlp": 1.03724003, + "epoch": 0.03888340781150253, + "flos": 14933399304960.0, + "grad_norm": 2.900394186055911, + "language_loss": 0.86103517, + "learning_rate": 3.999173025953687e-06, + "loss": 0.88338488, + "num_input_tokens_seen": 37199420, + "router_z_loss_clip": 1.19628906, + "router_z_loss_mlp": 0.29003906, + "step": 1340, + "time_per_iteration": 2.391855239868164 + }, + { + "auxiliary_loss_clip": 0.0116113, + "auxiliary_loss_mlp": 0.01060305, + "balance_loss_clip": 1.04965663, + "balance_loss_mlp": 1.03161144, + "epoch": 0.03891242528001857, + "flos": 40949069483520.0, + "grad_norm": 2.1190114616150737, + "language_loss": 0.87611699, + "learning_rate": 3.999167612438333e-06, + "loss": 0.89833134, + "num_input_tokens_seen": 37222680, + "router_z_loss_clip": 1.11523438, + "router_z_loss_mlp": 0.28723145, + "step": 1341, + "time_per_iteration": 2.6992313861846924 + }, + { + "auxiliary_loss_clip": 0.01151631, + "auxiliary_loss_mlp": 0.01060787, + "balance_loss_clip": 1.04996729, + "balance_loss_mlp": 1.03607535, + "epoch": 0.038941442748534616, + "flos": 42988905112320.0, + "grad_norm": 6.4982003618631206, + "language_loss": 0.69057548, + "learning_rate": 3.999162181265523e-06, + "loss": 0.71269971, + "num_input_tokens_seen": 37241135, + "router_z_loss_clip": 1.01464844, + "router_z_loss_mlp": 0.24682617, + "step": 1342, + "time_per_iteration": 2.5644631385803223 + }, + { + "auxiliary_loss_clip": 0.01176862, + "auxiliary_loss_mlp": 0.0108064, + "balance_loss_clip": 1.05413735, + "balance_loss_mlp": 1.05144525, + "epoch": 0.03897046021705067, + "flos": 23071862960640.0, + "grad_norm": 2.4415503715416618, + "language_loss": 0.764575, + "learning_rate": 3.999156732435304e-06, + "loss": 0.78715003, + "num_input_tokens_seen": 37255850, + "router_z_loss_clip": 1.22753906, + "router_z_loss_mlp": 0.29211426, + "step": 1343, + "time_per_iteration": 2.535637617111206 + }, + { + "auxiliary_loss_clip": 0.01176728, + "auxiliary_loss_mlp": 0.01082869, + "balance_loss_clip": 1.04897571, + "balance_loss_mlp": 1.05082512, + "epoch": 0.03899947768556671, + "flos": 25767542079360.0, + "grad_norm": 8.357396531128696, + "language_loss": 0.89033401, + "learning_rate": 3.999151265947723e-06, + "loss": 0.91292995, + "num_input_tokens_seen": 37271465, + "router_z_loss_clip": 1.27929688, + "router_z_loss_mlp": 0.32019043, + "step": 1344, + "time_per_iteration": 2.7614197731018066 + }, + { + "auxiliary_loss_clip": 0.01153361, + "auxiliary_loss_mlp": 0.01064882, + "balance_loss_clip": 1.04504514, + "balance_loss_mlp": 1.0384413, + "epoch": 0.03902849515408276, + "flos": 10954976590080.0, + "grad_norm": 3.4957512241074955, + "language_loss": 0.82425106, + "learning_rate": 3.999145781802829e-06, + "loss": 0.84643352, + "num_input_tokens_seen": 37283240, + "router_z_loss_clip": 1.08349609, + "router_z_loss_mlp": 0.26428223, + "step": 1345, + "time_per_iteration": 2.43105411529541 + }, + { + "auxiliary_loss_clip": 0.01154281, + "auxiliary_loss_mlp": 0.01060452, + "balance_loss_clip": 1.04734325, + "balance_loss_mlp": 1.0352515, + "epoch": 0.0390575126225988, + "flos": 34014166727040.0, + "grad_norm": 2.705932082740036, + "language_loss": 0.76996732, + "learning_rate": 3.99914028000067e-06, + "loss": 0.79211462, + "num_input_tokens_seen": 37302295, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.25195312, + "step": 1346, + "time_per_iteration": 2.539652109146118 + }, + { + "auxiliary_loss_clip": 0.01155607, + "auxiliary_loss_mlp": 0.01051086, + "balance_loss_clip": 1.0459981, + "balance_loss_mlp": 1.02190351, + "epoch": 0.03908653009111485, + "flos": 18397481736960.0, + "grad_norm": 3.3288546832083554, + "language_loss": 1.03737712, + "learning_rate": 3.999134760541296e-06, + "loss": 1.05944407, + "num_input_tokens_seen": 37313935, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.29138184, + "step": 1347, + "time_per_iteration": 2.4582481384277344 + }, + { + "auxiliary_loss_clip": 0.01163853, + "auxiliary_loss_mlp": 0.01068188, + "balance_loss_clip": 1.04730725, + "balance_loss_mlp": 1.0393393, + "epoch": 0.0391155475596309, + "flos": 31058105621760.0, + "grad_norm": 2.753778788709793, + "language_loss": 0.96376634, + "learning_rate": 3.999129223424754e-06, + "loss": 0.98608673, + "num_input_tokens_seen": 37329250, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.28881836, + "step": 1348, + "time_per_iteration": 2.556020736694336 + }, + { + "auxiliary_loss_clip": 0.01147981, + "auxiliary_loss_mlp": 0.01054365, + "balance_loss_clip": 1.04548645, + "balance_loss_mlp": 1.03035653, + "epoch": 0.03914456502814694, + "flos": 23285357124480.0, + "grad_norm": 2.518977240967636, + "language_loss": 0.6325376, + "learning_rate": 3.999123668651094e-06, + "loss": 0.65456104, + "num_input_tokens_seen": 37343165, + "router_z_loss_clip": 1.02539062, + "router_z_loss_mlp": 0.2401123, + "step": 1349, + "time_per_iteration": 2.550731897354126 + }, + { + "auxiliary_loss_clip": 0.01158508, + "auxiliary_loss_mlp": 0.01060705, + "balance_loss_clip": 1.0468626, + "balance_loss_mlp": 1.0323329, + "epoch": 0.039173582496662994, + "flos": 15260814835200.0, + "grad_norm": 3.538680432080127, + "language_loss": 0.8850559, + "learning_rate": 3.999118096220366e-06, + "loss": 0.90724802, + "num_input_tokens_seen": 37356915, + "router_z_loss_clip": 1.1171875, + "router_z_loss_mlp": 0.28369141, + "step": 1350, + "time_per_iteration": 2.547382354736328 + }, + { + "auxiliary_loss_clip": 0.0115745, + "auxiliary_loss_mlp": 0.01064686, + "balance_loss_clip": 1.04824018, + "balance_loss_mlp": 1.03557563, + "epoch": 0.03920259996517904, + "flos": 16062385806720.0, + "grad_norm": 2.73500663463466, + "language_loss": 0.89056182, + "learning_rate": 3.999112506132616e-06, + "loss": 0.91278327, + "num_input_tokens_seen": 37370485, + "router_z_loss_clip": 1.09130859, + "router_z_loss_mlp": 0.29150391, + "step": 1351, + "time_per_iteration": 2.4470629692077637 + }, + { + "auxiliary_loss_clip": 0.01167824, + "auxiliary_loss_mlp": 0.01073195, + "balance_loss_clip": 1.04970431, + "balance_loss_mlp": 1.04444182, + "epoch": 0.03923161743369508, + "flos": 19968084426240.0, + "grad_norm": 2.923072644250933, + "language_loss": 0.96659631, + "learning_rate": 3.999106898387897e-06, + "loss": 0.98900652, + "num_input_tokens_seen": 37385310, + "router_z_loss_clip": 1.18261719, + "router_z_loss_mlp": 0.28759766, + "step": 1352, + "time_per_iteration": 2.4565534591674805 + }, + { + "auxiliary_loss_clip": 0.01154556, + "auxiliary_loss_mlp": 0.01074122, + "balance_loss_clip": 1.0474093, + "balance_loss_mlp": 1.04796767, + "epoch": 0.03926063490221113, + "flos": 25256972223360.0, + "grad_norm": 3.054278699589654, + "language_loss": 0.80264932, + "learning_rate": 3.999101272986256e-06, + "loss": 0.82493603, + "num_input_tokens_seen": 37398455, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.26159668, + "step": 1353, + "time_per_iteration": 2.515408992767334 + }, + { + "auxiliary_loss_clip": 0.01031784, + "auxiliary_loss_mlp": 0.01009225, + "balance_loss_clip": 1.00971627, + "balance_loss_mlp": 1.00682843, + "epoch": 0.03928965237072718, + "flos": 74775576453120.0, + "grad_norm": 0.6917551385540769, + "language_loss": 0.50763416, + "learning_rate": 3.999095629927744e-06, + "loss": 0.52804428, + "num_input_tokens_seen": 37462715, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.02392578, + "step": 1354, + "time_per_iteration": 3.269275188446045 + }, + { + "auxiliary_loss_clip": 0.01158591, + "auxiliary_loss_mlp": 0.01065931, + "balance_loss_clip": 1.04495239, + "balance_loss_mlp": 1.03733265, + "epoch": 0.039318669839243224, + "flos": 18654198030720.0, + "grad_norm": 3.758831167864224, + "language_loss": 0.92539704, + "learning_rate": 3.99908996921241e-06, + "loss": 0.94764221, + "num_input_tokens_seen": 37477995, + "router_z_loss_clip": 1.13525391, + "router_z_loss_mlp": 0.28588867, + "step": 1355, + "time_per_iteration": 2.404914379119873 + }, + { + "auxiliary_loss_clip": 0.01155148, + "auxiliary_loss_mlp": 0.01057277, + "balance_loss_clip": 1.0444572, + "balance_loss_mlp": 1.02983499, + "epoch": 0.03934768730775927, + "flos": 26029704545280.0, + "grad_norm": 2.3986395623523804, + "language_loss": 0.80822998, + "learning_rate": 3.999084290840305e-06, + "loss": 0.83035421, + "num_input_tokens_seen": 37496900, + "router_z_loss_clip": 1.10693359, + "router_z_loss_mlp": 0.27429199, + "step": 1356, + "time_per_iteration": 2.5362770557403564 + }, + { + "auxiliary_loss_clip": 0.01161619, + "auxiliary_loss_mlp": 0.01056252, + "balance_loss_clip": 1.04524732, + "balance_loss_mlp": 1.02816677, + "epoch": 0.03937670477627532, + "flos": 23869279480320.0, + "grad_norm": 5.16582658888393, + "language_loss": 0.80292046, + "learning_rate": 3.999078594811478e-06, + "loss": 0.82509911, + "num_input_tokens_seen": 37510390, + "router_z_loss_clip": 1.16308594, + "router_z_loss_mlp": 0.28063965, + "step": 1357, + "time_per_iteration": 2.418978691101074 + }, + { + "auxiliary_loss_clip": 0.01151264, + "auxiliary_loss_mlp": 0.01048811, + "balance_loss_clip": 1.04163027, + "balance_loss_mlp": 1.02166748, + "epoch": 0.039405722244791365, + "flos": 34322448965760.0, + "grad_norm": 1.6784458877528565, + "language_loss": 0.7731939, + "learning_rate": 3.999072881125981e-06, + "loss": 0.79519463, + "num_input_tokens_seen": 37535465, + "router_z_loss_clip": 1.09863281, + "router_z_loss_mlp": 0.27160645, + "step": 1358, + "time_per_iteration": 2.822127103805542 + }, + { + "auxiliary_loss_clip": 0.01168723, + "auxiliary_loss_mlp": 0.01065117, + "balance_loss_clip": 1.04769599, + "balance_loss_mlp": 1.03176236, + "epoch": 0.03943473971330741, + "flos": 39344251795200.0, + "grad_norm": 2.7405245430353906, + "language_loss": 0.86892474, + "learning_rate": 3.999067149783863e-06, + "loss": 0.89126313, + "num_input_tokens_seen": 37555345, + "router_z_loss_clip": 1.20996094, + "router_z_loss_mlp": 0.33349609, + "step": 1359, + "time_per_iteration": 2.5613579750061035 + }, + { + "auxiliary_loss_clip": 0.01176309, + "auxiliary_loss_mlp": 0.01067645, + "balance_loss_clip": 1.05053854, + "balance_loss_mlp": 1.03537536, + "epoch": 0.03946375718182346, + "flos": 16651579777920.0, + "grad_norm": 2.5181569019065893, + "language_loss": 0.82004756, + "learning_rate": 3.999061400785174e-06, + "loss": 0.8424871, + "num_input_tokens_seen": 37575710, + "router_z_loss_clip": 1.25878906, + "router_z_loss_mlp": 0.32263184, + "step": 1360, + "time_per_iteration": 2.496206283569336 + }, + { + "auxiliary_loss_clip": 0.01027238, + "auxiliary_loss_mlp": 0.01002889, + "balance_loss_clip": 1.00549459, + "balance_loss_mlp": 1.00077891, + "epoch": 0.039492774650339506, + "flos": 58276043112960.0, + "grad_norm": 0.7787515800721344, + "language_loss": 0.51475871, + "learning_rate": 3.999055634129966e-06, + "loss": 0.53505993, + "num_input_tokens_seen": 37629910, + "router_z_loss_clip": 0.21777344, + "router_z_loss_mlp": 0.02111816, + "step": 1361, + "time_per_iteration": 2.817131519317627 + }, + { + "auxiliary_loss_clip": 0.01154597, + "auxiliary_loss_mlp": 0.01072522, + "balance_loss_clip": 1.04593205, + "balance_loss_mlp": 1.04311323, + "epoch": 0.03952179211885555, + "flos": 74730331330560.0, + "grad_norm": 2.2970429750993264, + "language_loss": 0.72684771, + "learning_rate": 3.999049849818291e-06, + "loss": 0.74911892, + "num_input_tokens_seen": 37650935, + "router_z_loss_clip": 1.08740234, + "router_z_loss_mlp": 0.29418945, + "step": 1362, + "time_per_iteration": 2.856781244277954 + }, + { + "auxiliary_loss_clip": 0.01154649, + "auxiliary_loss_mlp": 0.01063417, + "balance_loss_clip": 1.04492068, + "balance_loss_mlp": 1.03520048, + "epoch": 0.039550809587371595, + "flos": 12012844919040.0, + "grad_norm": 2.900673865313459, + "language_loss": 0.74605942, + "learning_rate": 3.999044047850198e-06, + "loss": 0.76824009, + "num_input_tokens_seen": 37662615, + "router_z_loss_clip": 1.09765625, + "router_z_loss_mlp": 0.28210449, + "step": 1363, + "time_per_iteration": 2.372621774673462 + }, + { + "auxiliary_loss_clip": 0.01026482, + "auxiliary_loss_mlp": 0.01002404, + "balance_loss_clip": 1.00459349, + "balance_loss_mlp": 1.00012696, + "epoch": 0.03957982705588765, + "flos": 74770723774080.0, + "grad_norm": 0.7891616342230452, + "language_loss": 0.52064317, + "learning_rate": 3.999038228225739e-06, + "loss": 0.540932, + "num_input_tokens_seen": 37721980, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.02282715, + "step": 1364, + "time_per_iteration": 3.076324224472046 + }, + { + "auxiliary_loss_clip": 0.01025658, + "auxiliary_loss_mlp": 0.01003067, + "balance_loss_clip": 1.00363314, + "balance_loss_mlp": 1.00076663, + "epoch": 0.03960884452440369, + "flos": 59862286091520.0, + "grad_norm": 0.7015148546680481, + "language_loss": 0.49867484, + "learning_rate": 3.999032390944965e-06, + "loss": 0.51896214, + "num_input_tokens_seen": 37784685, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.02294922, + "step": 1365, + "time_per_iteration": 3.074134588241577 + }, + { + "auxiliary_loss_clip": 0.01162385, + "auxiliary_loss_mlp": 0.0106027, + "balance_loss_clip": 1.0501864, + "balance_loss_mlp": 1.0313499, + "epoch": 0.039637861992919736, + "flos": 16098590753280.0, + "grad_norm": 2.6171964493182434, + "language_loss": 0.74752599, + "learning_rate": 3.999026536007929e-06, + "loss": 0.76975256, + "num_input_tokens_seen": 37796360, + "router_z_loss_clip": 1.12255859, + "router_z_loss_mlp": 0.28894043, + "step": 1366, + "time_per_iteration": 2.4438040256500244 + }, + { + "auxiliary_loss_clip": 0.01023772, + "auxiliary_loss_mlp": 0.01002503, + "balance_loss_clip": 1.00173187, + "balance_loss_mlp": 1.00019026, + "epoch": 0.03966687946143579, + "flos": 60839086510080.0, + "grad_norm": 0.6574865654423689, + "language_loss": 0.51486415, + "learning_rate": 3.999020663414681e-06, + "loss": 0.53512686, + "num_input_tokens_seen": 37862740, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.02307129, + "step": 1367, + "time_per_iteration": 3.142613410949707 + }, + { + "auxiliary_loss_clip": 0.01023671, + "auxiliary_loss_mlp": 0.01000507, + "balance_loss_clip": 1.00161695, + "balance_loss_mlp": 0.99826604, + "epoch": 0.03969589692995183, + "flos": 67076913594240.0, + "grad_norm": 0.6034324540401352, + "language_loss": 0.46126056, + "learning_rate": 3.999014773165273e-06, + "loss": 0.48150235, + "num_input_tokens_seen": 37931500, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.02246094, + "step": 1368, + "time_per_iteration": 3.1367781162261963 + }, + { + "auxiliary_loss_clip": 0.01170134, + "auxiliary_loss_mlp": 0.01071767, + "balance_loss_clip": 1.04645324, + "balance_loss_mlp": 1.03838873, + "epoch": 0.03972491439846788, + "flos": 20075128254720.0, + "grad_norm": 3.006941997321811, + "language_loss": 1.06681848, + "learning_rate": 3.999008865259759e-06, + "loss": 1.08923757, + "num_input_tokens_seen": 37944340, + "router_z_loss_clip": 1.23535156, + "router_z_loss_mlp": 0.33398438, + "step": 1369, + "time_per_iteration": 5.231359481811523 + }, + { + "auxiliary_loss_clip": 0.01159585, + "auxiliary_loss_mlp": 0.01056302, + "balance_loss_clip": 1.04560089, + "balance_loss_mlp": 1.02834749, + "epoch": 0.03975393186698392, + "flos": 36727406259840.0, + "grad_norm": 2.6186312532866953, + "language_loss": 0.83574831, + "learning_rate": 3.999002939698189e-06, + "loss": 0.85790724, + "num_input_tokens_seen": 37961550, + "router_z_loss_clip": 1.13964844, + "router_z_loss_mlp": 0.27954102, + "step": 1370, + "time_per_iteration": 2.5272912979125977 + }, + { + "auxiliary_loss_clip": 0.01025677, + "auxiliary_loss_mlp": 0.01004959, + "balance_loss_clip": 1.00384998, + "balance_loss_mlp": 1.00269437, + "epoch": 0.03978294933549997, + "flos": 74773446860160.0, + "grad_norm": 0.6894095289304631, + "language_loss": 0.5250017, + "learning_rate": 3.9989969964806165e-06, + "loss": 0.54530805, + "num_input_tokens_seen": 38024885, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.02270508, + "step": 1371, + "time_per_iteration": 3.0543630123138428 + }, + { + "auxiliary_loss_clip": 0.01026111, + "auxiliary_loss_mlp": 0.01003921, + "balance_loss_clip": 1.00441754, + "balance_loss_mlp": 1.00172806, + "epoch": 0.03981196680401602, + "flos": 69516085887360.0, + "grad_norm": 0.7203766975390087, + "language_loss": 0.52570271, + "learning_rate": 3.998991035607093e-06, + "loss": 0.5460031, + "num_input_tokens_seen": 38087115, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.02197266, + "step": 1372, + "time_per_iteration": 3.003977060317993 + }, + { + "auxiliary_loss_clip": 0.01155218, + "auxiliary_loss_mlp": 0.01062064, + "balance_loss_clip": 1.04562283, + "balance_loss_mlp": 1.034729, + "epoch": 0.03984098427253206, + "flos": 12267466531200.0, + "grad_norm": 3.075832172407254, + "language_loss": 0.84643555, + "learning_rate": 3.9989850570776726e-06, + "loss": 0.86860836, + "num_input_tokens_seen": 38098110, + "router_z_loss_clip": 1.09619141, + "router_z_loss_mlp": 0.27368164, + "step": 1373, + "time_per_iteration": 2.3691554069519043 + }, + { + "auxiliary_loss_clip": 0.01164555, + "auxiliary_loss_mlp": 0.0106927, + "balance_loss_clip": 1.04980803, + "balance_loss_mlp": 1.03775144, + "epoch": 0.039870001741048114, + "flos": 28978678644480.0, + "grad_norm": 3.868900394322277, + "language_loss": 0.91637766, + "learning_rate": 3.998979060892407e-06, + "loss": 0.93871593, + "num_input_tokens_seen": 38113445, + "router_z_loss_clip": 1.14941406, + "router_z_loss_mlp": 0.31555176, + "step": 1374, + "time_per_iteration": 2.5401008129119873 + }, + { + "auxiliary_loss_clip": 0.01157358, + "auxiliary_loss_mlp": 0.01065365, + "balance_loss_clip": 1.04453516, + "balance_loss_mlp": 1.03485906, + "epoch": 0.03989901920956416, + "flos": 14532351984000.0, + "grad_norm": 2.6065608953305692, + "language_loss": 0.84949666, + "learning_rate": 3.998973047051349e-06, + "loss": 0.87172389, + "num_input_tokens_seen": 38126325, + "router_z_loss_clip": 1.12695312, + "router_z_loss_mlp": 0.30480957, + "step": 1375, + "time_per_iteration": 4.778738260269165 + }, + { + "auxiliary_loss_clip": 0.0102335, + "auxiliary_loss_mlp": 0.01009869, + "balance_loss_clip": 1.002316, + "balance_loss_mlp": 1.00756812, + "epoch": 0.0399280366780802, + "flos": 59222852807040.0, + "grad_norm": 0.7090008043497599, + "language_loss": 0.52697754, + "learning_rate": 3.998967015554552e-06, + "loss": 0.54730976, + "num_input_tokens_seen": 38182675, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.02294922, + "step": 1376, + "time_per_iteration": 2.8534514904022217 + }, + { + "auxiliary_loss_clip": 0.01023048, + "auxiliary_loss_mlp": 0.01003715, + "balance_loss_clip": 1.00203454, + "balance_loss_mlp": 1.0012238, + "epoch": 0.03995705414659625, + "flos": 74777356932480.0, + "grad_norm": 0.6362580261970744, + "language_loss": 0.52781236, + "learning_rate": 3.998960966402071e-06, + "loss": 0.54807997, + "num_input_tokens_seen": 38253010, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.02490234, + "step": 1377, + "time_per_iteration": 3.1808688640594482 + }, + { + "auxiliary_loss_clip": 0.01163177, + "auxiliary_loss_mlp": 0.01059464, + "balance_loss_clip": 1.04654491, + "balance_loss_mlp": 1.03084159, + "epoch": 0.0399860716151123, + "flos": 16717810360320.0, + "grad_norm": 5.004581787255686, + "language_loss": 1.06278419, + "learning_rate": 3.998954899593956e-06, + "loss": 1.08501041, + "num_input_tokens_seen": 38263990, + "router_z_loss_clip": 1.16601562, + "router_z_loss_mlp": 0.28637695, + "step": 1378, + "time_per_iteration": 4.808757543563843 + }, + { + "auxiliary_loss_clip": 0.01023684, + "auxiliary_loss_mlp": 0.01002037, + "balance_loss_clip": 1.00261092, + "balance_loss_mlp": 0.99995053, + "epoch": 0.040015089083628344, + "flos": 70713676944000.0, + "grad_norm": 0.6997726242762716, + "language_loss": 0.55647165, + "learning_rate": 3.998948815130263e-06, + "loss": 0.57672888, + "num_input_tokens_seen": 38332155, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.02087402, + "step": 1379, + "time_per_iteration": 5.91654109954834 + }, + { + "auxiliary_loss_clip": 0.01023893, + "auxiliary_loss_mlp": 0.01002034, + "balance_loss_clip": 1.0024358, + "balance_loss_mlp": 0.99991214, + "epoch": 0.04004410655214439, + "flos": 68664275602560.0, + "grad_norm": 0.6685042052050749, + "language_loss": 0.48895359, + "learning_rate": 3.9989427130110455e-06, + "loss": 0.50921285, + "num_input_tokens_seen": 38387570, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.02124023, + "step": 1380, + "time_per_iteration": 2.980682134628296 + }, + { + "auxiliary_loss_clip": 0.01167614, + "auxiliary_loss_mlp": 0.01069288, + "balance_loss_clip": 1.04841781, + "balance_loss_mlp": 1.03656495, + "epoch": 0.04007312402066044, + "flos": 21244962913920.0, + "grad_norm": 4.522920059356016, + "language_loss": 0.94921327, + "learning_rate": 3.998936593236356e-06, + "loss": 0.97158229, + "num_input_tokens_seen": 38403400, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.32702637, + "step": 1381, + "time_per_iteration": 2.435450792312622 + }, + { + "auxiliary_loss_clip": 0.010228, + "auxiliary_loss_mlp": 0.01002438, + "balance_loss_clip": 1.0021224, + "balance_loss_mlp": 1.00018489, + "epoch": 0.040102141489176485, + "flos": 68603246812800.0, + "grad_norm": 0.6620415388321779, + "language_loss": 0.46081316, + "learning_rate": 3.998930455806251e-06, + "loss": 0.48106551, + "num_input_tokens_seen": 38462125, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.02258301, + "step": 1382, + "time_per_iteration": 3.023695707321167 + }, + { + "auxiliary_loss_clip": 0.01162687, + "auxiliary_loss_mlp": 0.0106305, + "balance_loss_clip": 1.04663157, + "balance_loss_mlp": 1.03334284, + "epoch": 0.04013115895769253, + "flos": 34489963002240.0, + "grad_norm": 2.3016894235616943, + "language_loss": 0.93017566, + "learning_rate": 3.998924300720783e-06, + "loss": 0.95243311, + "num_input_tokens_seen": 38485450, + "router_z_loss_clip": 1.16113281, + "router_z_loss_mlp": 0.29699707, + "step": 1383, + "time_per_iteration": 2.5379037857055664 + }, + { + "auxiliary_loss_clip": 0.01158905, + "auxiliary_loss_mlp": 0.01063755, + "balance_loss_clip": 1.04813671, + "balance_loss_mlp": 1.03634882, + "epoch": 0.040160176426208574, + "flos": 74737278691200.0, + "grad_norm": 1.7735235540834466, + "language_loss": 0.80287468, + "learning_rate": 3.998918127980006e-06, + "loss": 0.82510132, + "num_input_tokens_seen": 38514815, + "router_z_loss_clip": 1.10693359, + "router_z_loss_mlp": 0.27380371, + "step": 1384, + "time_per_iteration": 2.833693504333496 + }, + { + "auxiliary_loss_clip": 0.01162471, + "auxiliary_loss_mlp": 0.01058957, + "balance_loss_clip": 1.05115581, + "balance_loss_mlp": 1.02976251, + "epoch": 0.040189193894724626, + "flos": 27811811450880.0, + "grad_norm": 2.40446085639815, + "language_loss": 0.86148977, + "learning_rate": 3.998911937583976e-06, + "loss": 0.88370395, + "num_input_tokens_seen": 38529120, + "router_z_loss_clip": 1.11279297, + "router_z_loss_mlp": 0.29187012, + "step": 1385, + "time_per_iteration": 2.4947738647460938 + }, + { + "auxiliary_loss_clip": 0.01151561, + "auxiliary_loss_mlp": 0.0105997, + "balance_loss_clip": 1.04269946, + "balance_loss_mlp": 1.0304184, + "epoch": 0.04021821136324067, + "flos": 22995019324800.0, + "grad_norm": 2.4134306388749267, + "language_loss": 0.81312025, + "learning_rate": 3.998905729532746e-06, + "loss": 0.8352356, + "num_input_tokens_seen": 38543860, + "router_z_loss_clip": 1.08886719, + "router_z_loss_mlp": 0.29528809, + "step": 1386, + "time_per_iteration": 2.4755542278289795 + }, + { + "auxiliary_loss_clip": 0.01165577, + "auxiliary_loss_mlp": 0.01067803, + "balance_loss_clip": 1.04423881, + "balance_loss_mlp": 1.03070521, + "epoch": 0.040247228831756715, + "flos": 31132365816960.0, + "grad_norm": 2.478576820552025, + "language_loss": 0.92280042, + "learning_rate": 3.998899503826373e-06, + "loss": 0.94513428, + "num_input_tokens_seen": 38559860, + "router_z_loss_clip": 1.21191406, + "router_z_loss_mlp": 0.37097168, + "step": 1387, + "time_per_iteration": 2.5086491107940674 + }, + { + "auxiliary_loss_clip": 0.01168038, + "auxiliary_loss_mlp": 0.01072484, + "balance_loss_clip": 1.04892147, + "balance_loss_mlp": 1.03821123, + "epoch": 0.04027624630027277, + "flos": 24452294140800.0, + "grad_norm": 3.232555300596667, + "language_loss": 0.87709445, + "learning_rate": 3.99889326046491e-06, + "loss": 0.89949965, + "num_input_tokens_seen": 38574205, + "router_z_loss_clip": 1.19238281, + "router_z_loss_mlp": 0.34301758, + "step": 1388, + "time_per_iteration": 2.462663412094116 + }, + { + "auxiliary_loss_clip": 0.0115501, + "auxiliary_loss_mlp": 0.0106331, + "balance_loss_clip": 1.04414511, + "balance_loss_mlp": 1.03549814, + "epoch": 0.04030526376878881, + "flos": 35946679236480.0, + "grad_norm": 2.526073530613747, + "language_loss": 0.79025364, + "learning_rate": 3.998886999448413e-06, + "loss": 0.81243682, + "num_input_tokens_seen": 38590095, + "router_z_loss_clip": 1.10839844, + "router_z_loss_mlp": 0.2779541, + "step": 1389, + "time_per_iteration": 2.586077928543091 + }, + { + "auxiliary_loss_clip": 0.01159441, + "auxiliary_loss_mlp": 0.01060999, + "balance_loss_clip": 1.04519987, + "balance_loss_mlp": 1.03199577, + "epoch": 0.040334281237304856, + "flos": 26285966991360.0, + "grad_norm": 2.507779159931937, + "language_loss": 1.00434303, + "learning_rate": 3.998880720776937e-06, + "loss": 1.02654755, + "num_input_tokens_seen": 38607095, + "router_z_loss_clip": 1.14550781, + "router_z_loss_mlp": 0.29003906, + "step": 1390, + "time_per_iteration": 2.563242197036743 + }, + { + "auxiliary_loss_clip": 0.01166688, + "auxiliary_loss_mlp": 0.01064357, + "balance_loss_clip": 1.0478251, + "balance_loss_mlp": 1.03346944, + "epoch": 0.04036329870582091, + "flos": 15261198860160.0, + "grad_norm": 2.69433093321462, + "language_loss": 0.91890597, + "learning_rate": 3.998874424450538e-06, + "loss": 0.94121647, + "num_input_tokens_seen": 38621090, + "router_z_loss_clip": 1.18652344, + "router_z_loss_mlp": 0.30895996, + "step": 1391, + "time_per_iteration": 2.41369366645813 + }, + { + "auxiliary_loss_clip": 0.01163427, + "auxiliary_loss_mlp": 0.0106758, + "balance_loss_clip": 1.04947162, + "balance_loss_mlp": 1.03571546, + "epoch": 0.04039231617433695, + "flos": 35289718583040.0, + "grad_norm": 2.3545278743898446, + "language_loss": 0.97130305, + "learning_rate": 3.99886811046927e-06, + "loss": 0.99361306, + "num_input_tokens_seen": 38640300, + "router_z_loss_clip": 1.140625, + "router_z_loss_mlp": 0.31848145, + "step": 1392, + "time_per_iteration": 2.6596782207489014 + }, + { + "auxiliary_loss_clip": 0.01153908, + "auxiliary_loss_mlp": 0.01059209, + "balance_loss_clip": 1.04711282, + "balance_loss_mlp": 1.03124309, + "epoch": 0.040421333642853, + "flos": 28869889248000.0, + "grad_norm": 2.5837402169846437, + "language_loss": 0.70157248, + "learning_rate": 3.998861778833192e-06, + "loss": 0.72370362, + "num_input_tokens_seen": 38662360, + "router_z_loss_clip": 1.06738281, + "router_z_loss_mlp": 0.27978516, + "step": 1393, + "time_per_iteration": 2.756094217300415 + }, + { + "auxiliary_loss_clip": 0.01164641, + "auxiliary_loss_mlp": 0.01066763, + "balance_loss_clip": 1.05132937, + "balance_loss_mlp": 1.04135942, + "epoch": 0.04045035111136904, + "flos": 16025796835200.0, + "grad_norm": 2.937766903134838, + "language_loss": 0.97282195, + "learning_rate": 3.998855429542357e-06, + "loss": 0.99513602, + "num_input_tokens_seen": 38672665, + "router_z_loss_clip": 1.13232422, + "router_z_loss_mlp": 0.25427246, + "step": 1394, + "time_per_iteration": 2.367281436920166 + }, + { + "auxiliary_loss_clip": 0.01029331, + "auxiliary_loss_mlp": 0.01004115, + "balance_loss_clip": 1.00828707, + "balance_loss_mlp": 1.00208843, + "epoch": 0.04047936857988509, + "flos": 57766695154560.0, + "grad_norm": 0.7031651897539702, + "language_loss": 0.50466359, + "learning_rate": 3.998849062596821e-06, + "loss": 0.52499807, + "num_input_tokens_seen": 38730725, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.02026367, + "step": 1395, + "time_per_iteration": 3.0288708209991455 + }, + { + "auxiliary_loss_clip": 0.01155254, + "auxiliary_loss_mlp": 0.01060382, + "balance_loss_clip": 1.04488444, + "balance_loss_mlp": 1.03266549, + "epoch": 0.04050838604840114, + "flos": 15589801376640.0, + "grad_norm": 2.9243955840003353, + "language_loss": 0.9649471, + "learning_rate": 3.998842677996642e-06, + "loss": 0.9871034, + "num_input_tokens_seen": 38743565, + "router_z_loss_clip": 1.10400391, + "router_z_loss_mlp": 0.27734375, + "step": 1396, + "time_per_iteration": 2.4578676223754883 + }, + { + "auxiliary_loss_clip": 0.01164411, + "auxiliary_loss_mlp": 0.01058426, + "balance_loss_clip": 1.04644811, + "balance_loss_mlp": 1.02731204, + "epoch": 0.04053740351691718, + "flos": 27921438720000.0, + "grad_norm": 2.311581694988709, + "language_loss": 0.905527, + "learning_rate": 3.9988362757418765e-06, + "loss": 0.92775536, + "num_input_tokens_seen": 38759655, + "router_z_loss_clip": 1.1796875, + "router_z_loss_mlp": 0.31091309, + "step": 1397, + "time_per_iteration": 2.51130747795105 + }, + { + "auxiliary_loss_clip": 0.01160498, + "auxiliary_loss_mlp": 0.01056371, + "balance_loss_clip": 1.0454123, + "balance_loss_mlp": 1.02453029, + "epoch": 0.040566420985433234, + "flos": 18943558312320.0, + "grad_norm": 2.759551187461977, + "language_loss": 0.88401276, + "learning_rate": 3.9988298558325785e-06, + "loss": 0.90618145, + "num_input_tokens_seen": 38772545, + "router_z_loss_clip": 1.15185547, + "router_z_loss_mlp": 0.31848145, + "step": 1398, + "time_per_iteration": 2.426201581954956 + }, + { + "auxiliary_loss_clip": 0.01152999, + "auxiliary_loss_mlp": 0.01051212, + "balance_loss_clip": 1.04388773, + "balance_loss_mlp": 1.02598691, + "epoch": 0.04059543845394928, + "flos": 25184771798400.0, + "grad_norm": 2.134447451070387, + "language_loss": 0.76246655, + "learning_rate": 3.998823418268807e-06, + "loss": 0.78450865, + "num_input_tokens_seen": 38789575, + "router_z_loss_clip": 1.09130859, + "router_z_loss_mlp": 0.2520752, + "step": 1399, + "time_per_iteration": 2.4994070529937744 + }, + { + "auxiliary_loss_clip": 0.01159982, + "auxiliary_loss_mlp": 0.01079812, + "balance_loss_clip": 1.04558086, + "balance_loss_mlp": 1.04725552, + "epoch": 0.04062445592246532, + "flos": 29603833182720.0, + "grad_norm": 1.969764362917468, + "language_loss": 1.08898139, + "learning_rate": 3.998816963050619e-06, + "loss": 1.11137927, + "num_input_tokens_seen": 38811850, + "router_z_loss_clip": 1.14453125, + "router_z_loss_mlp": 0.32568359, + "step": 1400, + "time_per_iteration": 2.5795247554779053 + }, + { + "auxiliary_loss_clip": 0.01144968, + "auxiliary_loss_mlp": 0.01050894, + "balance_loss_clip": 1.04123926, + "balance_loss_mlp": 1.0242269, + "epoch": 0.04065347339098137, + "flos": 17710949295360.0, + "grad_norm": 3.2534962369210016, + "language_loss": 0.91141605, + "learning_rate": 3.99881049017807e-06, + "loss": 0.9333747, + "num_input_tokens_seen": 38826270, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.26672363, + "step": 1401, + "time_per_iteration": 2.417257070541382 + }, + { + "auxiliary_loss_clip": 0.01140988, + "auxiliary_loss_mlp": 0.01044298, + "balance_loss_clip": 1.04089677, + "balance_loss_mlp": 1.0212307, + "epoch": 0.04068249085949742, + "flos": 27447422924160.0, + "grad_norm": 2.4525947818313534, + "language_loss": 1.02821839, + "learning_rate": 3.998803999651218e-06, + "loss": 1.05007124, + "num_input_tokens_seen": 38841340, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.23083496, + "step": 1402, + "time_per_iteration": 2.440746307373047 + }, + { + "auxiliary_loss_clip": 0.01025701, + "auxiliary_loss_mlp": 0.01016077, + "balance_loss_clip": 1.0045346, + "balance_loss_mlp": 1.01385927, + "epoch": 0.040711508328013464, + "flos": 57902542721280.0, + "grad_norm": 0.7372534166035092, + "language_loss": 0.47122079, + "learning_rate": 3.99879749147012e-06, + "loss": 0.49163857, + "num_input_tokens_seen": 38892920, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.0222168, + "step": 1403, + "time_per_iteration": 2.9857778549194336 + }, + { + "auxiliary_loss_clip": 0.01156942, + "auxiliary_loss_mlp": 0.01062517, + "balance_loss_clip": 1.04649544, + "balance_loss_mlp": 1.03319192, + "epoch": 0.04074052579652951, + "flos": 56085768835200.0, + "grad_norm": 2.326153438047791, + "language_loss": 0.80878127, + "learning_rate": 3.998790965634835e-06, + "loss": 0.83097583, + "num_input_tokens_seen": 38913995, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.29333496, + "step": 1404, + "time_per_iteration": 2.721970319747925 + }, + { + "auxiliary_loss_clip": 0.01146047, + "auxiliary_loss_mlp": 0.01052219, + "balance_loss_clip": 1.04009974, + "balance_loss_mlp": 1.02674389, + "epoch": 0.04076954326504556, + "flos": 40945787815680.0, + "grad_norm": 2.2688659969597387, + "language_loss": 0.861314, + "learning_rate": 3.998784422145418e-06, + "loss": 0.88329661, + "num_input_tokens_seen": 38930185, + "router_z_loss_clip": 1.06005859, + "router_z_loss_mlp": 0.25488281, + "step": 1405, + "time_per_iteration": 2.622920274734497 + }, + { + "auxiliary_loss_clip": 0.01162668, + "auxiliary_loss_mlp": 0.01058047, + "balance_loss_clip": 1.04652512, + "balance_loss_mlp": 1.02673078, + "epoch": 0.040798560733561605, + "flos": 47291601335040.0, + "grad_norm": 3.5673809955783167, + "language_loss": 1.00579238, + "learning_rate": 3.9987778610019285e-06, + "loss": 1.0279994, + "num_input_tokens_seen": 38944970, + "router_z_loss_clip": 1.16015625, + "router_z_loss_mlp": 0.31298828, + "step": 1406, + "time_per_iteration": 2.7838456630706787 + }, + { + "auxiliary_loss_clip": 0.01154325, + "auxiliary_loss_mlp": 0.01063692, + "balance_loss_clip": 1.04389989, + "balance_loss_mlp": 1.03547525, + "epoch": 0.04082757820207765, + "flos": 26794861102080.0, + "grad_norm": 2.4477087552661208, + "language_loss": 1.0599792, + "learning_rate": 3.998771282204425e-06, + "loss": 1.08215928, + "num_input_tokens_seen": 38962650, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.28210449, + "step": 1407, + "time_per_iteration": 2.5011775493621826 + }, + { + "auxiliary_loss_clip": 0.01140569, + "auxiliary_loss_mlp": 0.01050279, + "balance_loss_clip": 1.04236507, + "balance_loss_mlp": 1.02671182, + "epoch": 0.040856595670593694, + "flos": 12595789756800.0, + "grad_norm": 3.666849750689566, + "language_loss": 0.97034526, + "learning_rate": 3.9987646857529634e-06, + "loss": 0.99225378, + "num_input_tokens_seen": 38974215, + "router_z_loss_clip": 0.98291016, + "router_z_loss_mlp": 0.23571777, + "step": 1408, + "time_per_iteration": 2.3459970951080322 + }, + { + "auxiliary_loss_clip": 0.01158252, + "auxiliary_loss_mlp": 0.01071981, + "balance_loss_clip": 1.0450654, + "balance_loss_mlp": 1.04101002, + "epoch": 0.040885613139109746, + "flos": 14498346453120.0, + "grad_norm": 3.700498700703625, + "language_loss": 0.86095011, + "learning_rate": 3.998758071647604e-06, + "loss": 0.88325244, + "num_input_tokens_seen": 38989350, + "router_z_loss_clip": 1.13183594, + "router_z_loss_mlp": 0.30944824, + "step": 1409, + "time_per_iteration": 2.4537174701690674 + }, + { + "auxiliary_loss_clip": 0.01154918, + "auxiliary_loss_mlp": 0.0106845, + "balance_loss_clip": 1.04713392, + "balance_loss_mlp": 1.04149699, + "epoch": 0.04091463060762579, + "flos": 16391442170880.0, + "grad_norm": 3.0375043550276666, + "language_loss": 1.14071703, + "learning_rate": 3.998751439888404e-06, + "loss": 1.16295075, + "num_input_tokens_seen": 39002100, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.26953125, + "step": 1410, + "time_per_iteration": 2.473879814147949 + }, + { + "auxiliary_loss_clip": 0.0115354, + "auxiliary_loss_mlp": 0.01066505, + "balance_loss_clip": 1.04022205, + "balance_loss_mlp": 1.03806162, + "epoch": 0.040943648076141835, + "flos": 33283713928320.0, + "grad_norm": 2.2802742176578183, + "language_loss": 0.95480382, + "learning_rate": 3.998744790475423e-06, + "loss": 0.97700429, + "num_input_tokens_seen": 39022125, + "router_z_loss_clip": 1.1328125, + "router_z_loss_mlp": 0.2845459, + "step": 1411, + "time_per_iteration": 2.5605015754699707 + }, + { + "auxiliary_loss_clip": 0.01162171, + "auxiliary_loss_mlp": 0.01056775, + "balance_loss_clip": 1.04799998, + "balance_loss_mlp": 1.02572107, + "epoch": 0.04097266554465789, + "flos": 25985923833600.0, + "grad_norm": 5.149215146913239, + "language_loss": 0.94384015, + "learning_rate": 3.998738123408719e-06, + "loss": 0.96602958, + "num_input_tokens_seen": 39037250, + "router_z_loss_clip": 1.14111328, + "router_z_loss_mlp": 0.31066895, + "step": 1412, + "time_per_iteration": 2.451261520385742 + }, + { + "auxiliary_loss_clip": 0.01156169, + "auxiliary_loss_mlp": 0.01052849, + "balance_loss_clip": 1.04814827, + "balance_loss_mlp": 1.02664709, + "epoch": 0.04100168301317393, + "flos": 35766527287680.0, + "grad_norm": 2.2789985612561594, + "language_loss": 0.85627407, + "learning_rate": 3.998731438688351e-06, + "loss": 0.87836421, + "num_input_tokens_seen": 39057600, + "router_z_loss_clip": 1.08056641, + "router_z_loss_mlp": 0.26196289, + "step": 1413, + "time_per_iteration": 2.5703165531158447 + }, + { + "auxiliary_loss_clip": 0.01157273, + "auxiliary_loss_mlp": 0.01064984, + "balance_loss_clip": 1.04466176, + "balance_loss_mlp": 1.03650522, + "epoch": 0.041030700481689976, + "flos": 13908873191040.0, + "grad_norm": 3.297409853120061, + "language_loss": 0.96759033, + "learning_rate": 3.998724736314378e-06, + "loss": 0.98981285, + "num_input_tokens_seen": 39068410, + "router_z_loss_clip": 1.125, + "router_z_loss_mlp": 0.28479004, + "step": 1414, + "time_per_iteration": 2.346762180328369 + }, + { + "auxiliary_loss_clip": 0.0115444, + "auxiliary_loss_mlp": 0.01065249, + "balance_loss_clip": 1.04391336, + "balance_loss_mlp": 1.03483844, + "epoch": 0.04105971795020603, + "flos": 30952388424960.0, + "grad_norm": 3.015250019234904, + "language_loss": 0.99414802, + "learning_rate": 3.99871801628686e-06, + "loss": 1.01634479, + "num_input_tokens_seen": 39083135, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.30395508, + "step": 1415, + "time_per_iteration": 2.5953078269958496 + }, + { + "auxiliary_loss_clip": 0.01157785, + "auxiliary_loss_mlp": 0.01057747, + "balance_loss_clip": 1.04294848, + "balance_loss_mlp": 1.02877879, + "epoch": 0.04108873541872207, + "flos": 33541547385600.0, + "grad_norm": 2.672842620023637, + "language_loss": 0.98570162, + "learning_rate": 3.998711278605855e-06, + "loss": 1.00785685, + "num_input_tokens_seen": 39103315, + "router_z_loss_clip": 1.14892578, + "router_z_loss_mlp": 0.28967285, + "step": 1416, + "time_per_iteration": 2.5386669635772705 + }, + { + "auxiliary_loss_clip": 0.01026994, + "auxiliary_loss_mlp": 0.01006425, + "balance_loss_clip": 1.00522912, + "balance_loss_mlp": 1.00427878, + "epoch": 0.04111775288723812, + "flos": 68713467575040.0, + "grad_norm": 0.704712932329145, + "language_loss": 0.52495718, + "learning_rate": 3.998704523271423e-06, + "loss": 0.54529142, + "num_input_tokens_seen": 39164760, + "router_z_loss_clip": 0.21777344, + "router_z_loss_mlp": 0.02148438, + "step": 1417, + "time_per_iteration": 3.043391227722168 + }, + { + "auxiliary_loss_clip": 0.01024353, + "auxiliary_loss_mlp": 0.01002028, + "balance_loss_clip": 1.00293446, + "balance_loss_mlp": 0.9998824, + "epoch": 0.04114677035575416, + "flos": 63006808049280.0, + "grad_norm": 0.6600149528727375, + "language_loss": 0.52036345, + "learning_rate": 3.998697750283624e-06, + "loss": 0.54062724, + "num_input_tokens_seen": 39233720, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.02148438, + "step": 1418, + "time_per_iteration": 3.2004687786102295 + }, + { + "auxiliary_loss_clip": 0.01023972, + "auxiliary_loss_mlp": 0.01001237, + "balance_loss_clip": 1.00277948, + "balance_loss_mlp": 0.99906725, + "epoch": 0.04117578782427021, + "flos": 74329386877440.0, + "grad_norm": 0.6702398066677858, + "language_loss": 0.50444871, + "learning_rate": 3.998690959642519e-06, + "loss": 0.52470082, + "num_input_tokens_seen": 39299080, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.02172852, + "step": 1419, + "time_per_iteration": 3.2222115993499756 + }, + { + "auxiliary_loss_clip": 0.01165691, + "auxiliary_loss_mlp": 0.0107158, + "balance_loss_clip": 1.04780102, + "balance_loss_mlp": 1.0420996, + "epoch": 0.04120480529278626, + "flos": 56817862467840.0, + "grad_norm": 2.1359009714339403, + "language_loss": 0.72238314, + "learning_rate": 3.9986841513481646e-06, + "loss": 0.7447558, + "num_input_tokens_seen": 39321450, + "router_z_loss_clip": 1.1796875, + "router_z_loss_mlp": 0.2947998, + "step": 1420, + "time_per_iteration": 2.798877000808716 + }, + { + "auxiliary_loss_clip": 0.01157503, + "auxiliary_loss_mlp": 0.01062364, + "balance_loss_clip": 1.04310596, + "balance_loss_mlp": 1.03420651, + "epoch": 0.0412338227613023, + "flos": 45664543244160.0, + "grad_norm": 1.8890193587488633, + "language_loss": 0.61743593, + "learning_rate": 3.998677325400625e-06, + "loss": 0.63963455, + "num_input_tokens_seen": 39342110, + "router_z_loss_clip": 1.14599609, + "router_z_loss_mlp": 0.28161621, + "step": 1421, + "time_per_iteration": 2.550502061843872 + }, + { + "auxiliary_loss_clip": 0.01150627, + "auxiliary_loss_mlp": 0.01057452, + "balance_loss_clip": 1.04301751, + "balance_loss_mlp": 1.0320363, + "epoch": 0.041262840229818354, + "flos": 16172920771200.0, + "grad_norm": 2.854262758000337, + "language_loss": 0.97759962, + "learning_rate": 3.998670481799957e-06, + "loss": 0.9996804, + "num_input_tokens_seen": 39355520, + "router_z_loss_clip": 1.07568359, + "router_z_loss_mlp": 0.25427246, + "step": 1422, + "time_per_iteration": 2.4247584342956543 + }, + { + "auxiliary_loss_clip": 0.01146739, + "auxiliary_loss_mlp": 0.01054856, + "balance_loss_clip": 1.04245412, + "balance_loss_mlp": 1.0296309, + "epoch": 0.0412918576983344, + "flos": 18656083244160.0, + "grad_norm": 3.656082303570973, + "language_loss": 0.77096093, + "learning_rate": 3.998663620546223e-06, + "loss": 0.79297686, + "num_input_tokens_seen": 39369135, + "router_z_loss_clip": 1.04101562, + "router_z_loss_mlp": 0.2520752, + "step": 1423, + "time_per_iteration": 2.3771753311157227 + }, + { + "auxiliary_loss_clip": 0.01163861, + "auxiliary_loss_mlp": 0.01075156, + "balance_loss_clip": 1.04612815, + "balance_loss_mlp": 1.04456711, + "epoch": 0.04132087516685044, + "flos": 19781753166720.0, + "grad_norm": 3.63188534506624, + "language_loss": 1.26164293, + "learning_rate": 3.998656741639484e-06, + "loss": 1.28403306, + "num_input_tokens_seen": 39380005, + "router_z_loss_clip": 1.17773438, + "router_z_loss_mlp": 0.30554199, + "step": 1424, + "time_per_iteration": 2.45164155960083 + }, + { + "auxiliary_loss_clip": 0.01040633, + "auxiliary_loss_mlp": 0.01066514, + "balance_loss_clip": 1.01878119, + "balance_loss_mlp": 1.06405795, + "epoch": 0.04134989263536649, + "flos": 74291610919680.0, + "grad_norm": 0.7730497799593756, + "language_loss": 0.50164795, + "learning_rate": 3.9986498450797986e-06, + "loss": 0.52271938, + "num_input_tokens_seen": 39436230, + "router_z_loss_clip": 0.21875, + "router_z_loss_mlp": 0.02453613, + "step": 1425, + "time_per_iteration": 2.978487253189087 + }, + { + "auxiliary_loss_clip": 0.01031407, + "auxiliary_loss_mlp": 0.01028839, + "balance_loss_clip": 1.01021099, + "balance_loss_mlp": 1.02668166, + "epoch": 0.04137891010388254, + "flos": 73128549064320.0, + "grad_norm": 0.7563378917097714, + "language_loss": 0.50927913, + "learning_rate": 3.9986429308672286e-06, + "loss": 0.5298816, + "num_input_tokens_seen": 39495390, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.02160645, + "step": 1426, + "time_per_iteration": 3.034090042114258 + }, + { + "auxiliary_loss_clip": 0.01023828, + "auxiliary_loss_mlp": 0.01000651, + "balance_loss_clip": 1.00294495, + "balance_loss_mlp": 0.99865979, + "epoch": 0.041407927572398584, + "flos": 61760059931520.0, + "grad_norm": 0.6587893145725982, + "language_loss": 0.50649655, + "learning_rate": 3.998635999001837e-06, + "loss": 0.52674139, + "num_input_tokens_seen": 39560995, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.01989746, + "step": 1427, + "time_per_iteration": 3.13657283782959 + }, + { + "auxiliary_loss_clip": 0.0114622, + "auxiliary_loss_mlp": 0.01056637, + "balance_loss_clip": 1.04029107, + "balance_loss_mlp": 1.02820539, + "epoch": 0.04143694504091463, + "flos": 12194009297280.0, + "grad_norm": 3.593475110815141, + "language_loss": 0.70107484, + "learning_rate": 3.998629049483683e-06, + "loss": 0.7231034, + "num_input_tokens_seen": 39573280, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.28442383, + "step": 1428, + "time_per_iteration": 2.355280637741089 + }, + { + "auxiliary_loss_clip": 0.01035952, + "auxiliary_loss_mlp": 0.01016914, + "balance_loss_clip": 1.01390219, + "balance_loss_mlp": 1.01488793, + "epoch": 0.04146596250943068, + "flos": 66559361466240.0, + "grad_norm": 0.7493889896642862, + "language_loss": 0.56239426, + "learning_rate": 3.9986220823128275e-06, + "loss": 0.58292294, + "num_input_tokens_seen": 39633055, + "router_z_loss_clip": 0.22070312, + "router_z_loss_mlp": 0.02026367, + "step": 1429, + "time_per_iteration": 3.024284601211548 + }, + { + "auxiliary_loss_clip": 0.01154664, + "auxiliary_loss_mlp": 0.01058822, + "balance_loss_clip": 1.04402494, + "balance_loss_mlp": 1.0316422, + "epoch": 0.041494979977946725, + "flos": 23799348293760.0, + "grad_norm": 3.651578175294221, + "language_loss": 0.93674457, + "learning_rate": 3.998615097489334e-06, + "loss": 0.95887947, + "num_input_tokens_seen": 39649010, + "router_z_loss_clip": 1.10546875, + "router_z_loss_mlp": 0.27160645, + "step": 1430, + "time_per_iteration": 2.464176654815674 + }, + { + "auxiliary_loss_clip": 0.01150509, + "auxiliary_loss_mlp": 0.01050837, + "balance_loss_clip": 1.04341555, + "balance_loss_mlp": 1.02353787, + "epoch": 0.04152399744646277, + "flos": 74733857377920.0, + "grad_norm": 2.257198671480256, + "language_loss": 0.82767814, + "learning_rate": 3.998608095013262e-06, + "loss": 0.84969163, + "num_input_tokens_seen": 39673240, + "router_z_loss_clip": 1.07128906, + "router_z_loss_mlp": 0.27319336, + "step": 1431, + "time_per_iteration": 2.8615047931671143 + }, + { + "auxiliary_loss_clip": 0.01041719, + "auxiliary_loss_mlp": 0.01026415, + "balance_loss_clip": 1.01972413, + "balance_loss_mlp": 1.02441227, + "epoch": 0.041553014914978814, + "flos": 74779102500480.0, + "grad_norm": 0.7388080721667162, + "language_loss": 0.56196696, + "learning_rate": 3.998601074884676e-06, + "loss": 0.58264828, + "num_input_tokens_seen": 39743950, + "router_z_loss_clip": 0.21972656, + "router_z_loss_mlp": 0.02001953, + "step": 1432, + "time_per_iteration": 3.3393969535827637 + }, + { + "auxiliary_loss_clip": 0.01032431, + "auxiliary_loss_mlp": 0.01014428, + "balance_loss_clip": 1.01100612, + "balance_loss_mlp": 1.01246142, + "epoch": 0.041582032383494866, + "flos": 63287126288640.0, + "grad_norm": 0.7430281930635737, + "language_loss": 0.52700537, + "learning_rate": 3.998594037103637e-06, + "loss": 0.54747397, + "num_input_tokens_seen": 39803000, + "router_z_loss_clip": 0.21484375, + "router_z_loss_mlp": 0.01965332, + "step": 1433, + "time_per_iteration": 3.0130810737609863 + }, + { + "auxiliary_loss_clip": 0.01155637, + "auxiliary_loss_mlp": 0.01054205, + "balance_loss_clip": 1.04554248, + "balance_loss_mlp": 1.02865815, + "epoch": 0.04161104985201091, + "flos": 13290526368000.0, + "grad_norm": 4.606090505199015, + "language_loss": 0.85467786, + "learning_rate": 3.998586981670206e-06, + "loss": 0.87677622, + "num_input_tokens_seen": 39815630, + "router_z_loss_clip": 1.1015625, + "router_z_loss_mlp": 0.25549316, + "step": 1434, + "time_per_iteration": 2.3970422744750977 + }, + { + "auxiliary_loss_clip": 0.0115474, + "auxiliary_loss_mlp": 0.01057202, + "balance_loss_clip": 1.04164469, + "balance_loss_mlp": 1.02800763, + "epoch": 0.041640067320526955, + "flos": 30220993019520.0, + "grad_norm": 2.537267208880595, + "language_loss": 0.78458333, + "learning_rate": 3.998579908584445e-06, + "loss": 0.80670273, + "num_input_tokens_seen": 39833430, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.29211426, + "step": 1435, + "time_per_iteration": 2.5293684005737305 + }, + { + "auxiliary_loss_clip": 0.01153615, + "auxiliary_loss_mlp": 0.01055745, + "balance_loss_clip": 1.04842734, + "balance_loss_mlp": 1.02810049, + "epoch": 0.04166908478904301, + "flos": 34890765943680.0, + "grad_norm": 2.161217041644513, + "language_loss": 0.91202676, + "learning_rate": 3.998572817846419e-06, + "loss": 0.93412042, + "num_input_tokens_seen": 39853670, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.27600098, + "step": 1436, + "time_per_iteration": 2.554250478744507 + }, + { + "auxiliary_loss_clip": 0.01043495, + "auxiliary_loss_mlp": 0.01025307, + "balance_loss_clip": 1.02176619, + "balance_loss_mlp": 1.02326822, + "epoch": 0.04169810225755905, + "flos": 60437061671040.0, + "grad_norm": 0.6830371429028388, + "language_loss": 0.51854336, + "learning_rate": 3.998565709456188e-06, + "loss": 0.53923142, + "num_input_tokens_seen": 39917985, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.02038574, + "step": 1437, + "time_per_iteration": 3.0817747116088867 + }, + { + "auxiliary_loss_clip": 0.01161195, + "auxiliary_loss_mlp": 0.01065504, + "balance_loss_clip": 1.04762483, + "balance_loss_mlp": 1.03706038, + "epoch": 0.041727119726075096, + "flos": 20186431269120.0, + "grad_norm": 4.441489984303779, + "language_loss": 0.98949301, + "learning_rate": 3.998558583413817e-06, + "loss": 1.01176, + "num_input_tokens_seen": 39932210, + "router_z_loss_clip": 1.13671875, + "router_z_loss_mlp": 0.2845459, + "step": 1438, + "time_per_iteration": 2.53818416595459 + }, + { + "auxiliary_loss_clip": 0.01045002, + "auxiliary_loss_mlp": 0.01036936, + "balance_loss_clip": 1.0232687, + "balance_loss_mlp": 1.03469503, + "epoch": 0.04175613719459114, + "flos": 61522057992960.0, + "grad_norm": 0.7710296308244163, + "language_loss": 0.56600606, + "learning_rate": 3.998551439719367e-06, + "loss": 0.58682543, + "num_input_tokens_seen": 39993345, + "router_z_loss_clip": 0.21679688, + "router_z_loss_mlp": 0.02246094, + "step": 1439, + "time_per_iteration": 2.998626232147217 + }, + { + "auxiliary_loss_clip": 0.01037287, + "auxiliary_loss_mlp": 0.01018546, + "balance_loss_clip": 1.01617754, + "balance_loss_mlp": 1.01654363, + "epoch": 0.04178515466310719, + "flos": 69119262840960.0, + "grad_norm": 0.6823518042077553, + "language_loss": 0.53932232, + "learning_rate": 3.998544278372902e-06, + "loss": 0.55988067, + "num_input_tokens_seen": 40056155, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.02001953, + "step": 1440, + "time_per_iteration": 3.090810537338257 + }, + { + "auxiliary_loss_clip": 0.01163232, + "auxiliary_loss_mlp": 0.01067437, + "balance_loss_clip": 1.04630065, + "balance_loss_mlp": 1.03767014, + "epoch": 0.04181417213162324, + "flos": 13509082679040.0, + "grad_norm": 4.909547726677259, + "language_loss": 0.97034836, + "learning_rate": 3.998537099374486e-06, + "loss": 0.99265504, + "num_input_tokens_seen": 40069275, + "router_z_loss_clip": 1.171875, + "router_z_loss_mlp": 0.29736328, + "step": 1441, + "time_per_iteration": 2.3977158069610596 + }, + { + "auxiliary_loss_clip": 0.01023367, + "auxiliary_loss_mlp": 0.01003454, + "balance_loss_clip": 1.00299263, + "balance_loss_mlp": 1.00139201, + "epoch": 0.04184318960013928, + "flos": 63605709244800.0, + "grad_norm": 0.627018883302582, + "language_loss": 0.52341276, + "learning_rate": 3.99852990272418e-06, + "loss": 0.54368091, + "num_input_tokens_seen": 40133275, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.02062988, + "step": 1442, + "time_per_iteration": 3.122527837753296 + }, + { + "auxiliary_loss_clip": 0.01151754, + "auxiliary_loss_mlp": 0.01058139, + "balance_loss_clip": 1.04318595, + "balance_loss_mlp": 1.03023195, + "epoch": 0.04187220706865533, + "flos": 74730086951040.0, + "grad_norm": 6.452973649901908, + "language_loss": 0.70307547, + "learning_rate": 3.998522688422051e-06, + "loss": 0.72517443, + "num_input_tokens_seen": 40158460, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.27893066, + "step": 1443, + "time_per_iteration": 2.8850414752960205 + }, + { + "auxiliary_loss_clip": 0.01152369, + "auxiliary_loss_mlp": 0.01067317, + "balance_loss_clip": 1.04829311, + "balance_loss_mlp": 1.04126978, + "epoch": 0.04190122453717138, + "flos": 16903338658560.0, + "grad_norm": 2.2004059117401344, + "language_loss": 0.77551341, + "learning_rate": 3.99851545646816e-06, + "loss": 0.7977103, + "num_input_tokens_seen": 40172380, + "router_z_loss_clip": 1.04150391, + "router_z_loss_mlp": 0.26074219, + "step": 1444, + "time_per_iteration": 2.771448850631714 + }, + { + "auxiliary_loss_clip": 0.01158311, + "auxiliary_loss_mlp": 0.01064316, + "balance_loss_clip": 1.04950941, + "balance_loss_mlp": 1.03605163, + "epoch": 0.04193024200568742, + "flos": 29015337438720.0, + "grad_norm": 2.113492467155164, + "language_loss": 0.79743373, + "learning_rate": 3.998508206862572e-06, + "loss": 0.81966007, + "num_input_tokens_seen": 40187140, + "router_z_loss_clip": 1.09033203, + "router_z_loss_mlp": 0.28271484, + "step": 1445, + "time_per_iteration": 2.536740779876709 + }, + { + "auxiliary_loss_clip": 0.01160667, + "auxiliary_loss_mlp": 0.01051917, + "balance_loss_clip": 1.05771923, + "balance_loss_mlp": 1.02864695, + "epoch": 0.041959259474203474, + "flos": 43827553814400.0, + "grad_norm": 2.2355364327164886, + "language_loss": 0.94909894, + "learning_rate": 3.998500939605351e-06, + "loss": 0.97122484, + "num_input_tokens_seen": 40206690, + "router_z_loss_clip": 1.02929688, + "router_z_loss_mlp": 0.23242188, + "step": 1446, + "time_per_iteration": 5.008721590042114 + }, + { + "auxiliary_loss_clip": 0.01157122, + "auxiliary_loss_mlp": 0.01062126, + "balance_loss_clip": 1.0544467, + "balance_loss_mlp": 1.03797388, + "epoch": 0.04198827694271952, + "flos": 27698483577600.0, + "grad_norm": 2.793184894448862, + "language_loss": 0.92660761, + "learning_rate": 3.998493654696561e-06, + "loss": 0.94880009, + "num_input_tokens_seen": 40219295, + "router_z_loss_clip": 1.02636719, + "router_z_loss_mlp": 0.24169922, + "step": 1447, + "time_per_iteration": 2.465895891189575 + }, + { + "auxiliary_loss_clip": 0.0116243, + "auxiliary_loss_mlp": 0.01073657, + "balance_loss_clip": 1.05085301, + "balance_loss_mlp": 1.04665565, + "epoch": 0.04201729441123556, + "flos": 19207745637120.0, + "grad_norm": 2.128992977583077, + "language_loss": 0.8352741, + "learning_rate": 3.998486352136265e-06, + "loss": 0.85763502, + "num_input_tokens_seen": 40235505, + "router_z_loss_clip": 1.11767578, + "router_z_loss_mlp": 0.27050781, + "step": 1448, + "time_per_iteration": 2.480846881866455 + }, + { + "auxiliary_loss_clip": 0.01154201, + "auxiliary_loss_mlp": 0.01063685, + "balance_loss_clip": 1.05487657, + "balance_loss_mlp": 1.03993917, + "epoch": 0.04204631187975161, + "flos": 12233042064000.0, + "grad_norm": 3.198476860543928, + "language_loss": 0.88517886, + "learning_rate": 3.99847903192453e-06, + "loss": 0.90735769, + "num_input_tokens_seen": 40247745, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.23718262, + "step": 1449, + "time_per_iteration": 2.3957481384277344 + }, + { + "auxiliary_loss_clip": 0.01152717, + "auxiliary_loss_mlp": 0.01055025, + "balance_loss_clip": 1.04746711, + "balance_loss_mlp": 1.02882326, + "epoch": 0.04207532934826766, + "flos": 29312343308160.0, + "grad_norm": 1.9948052826749005, + "language_loss": 0.83379668, + "learning_rate": 3.99847169406142e-06, + "loss": 0.85587418, + "num_input_tokens_seen": 40267785, + "router_z_loss_clip": 1.05175781, + "router_z_loss_mlp": 0.26208496, + "step": 1450, + "time_per_iteration": 2.6781585216522217 + }, + { + "auxiliary_loss_clip": 0.01155712, + "auxiliary_loss_mlp": 0.01052384, + "balance_loss_clip": 1.05011058, + "balance_loss_mlp": 1.02662277, + "epoch": 0.042104346816783704, + "flos": 17886911880960.0, + "grad_norm": 3.07254424640421, + "language_loss": 0.88912362, + "learning_rate": 3.9984643385469986e-06, + "loss": 0.91120458, + "num_input_tokens_seen": 40280300, + "router_z_loss_clip": 1.05664062, + "router_z_loss_mlp": 0.25744629, + "step": 1451, + "time_per_iteration": 2.413957118988037 + }, + { + "auxiliary_loss_clip": 0.01151185, + "auxiliary_loss_mlp": 0.01047222, + "balance_loss_clip": 1.04722118, + "balance_loss_mlp": 1.02259338, + "epoch": 0.04213336428529975, + "flos": 34889544046080.0, + "grad_norm": 2.5936032868321783, + "language_loss": 0.71793079, + "learning_rate": 3.998456965381331e-06, + "loss": 0.73991483, + "num_input_tokens_seen": 40297640, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.24658203, + "step": 1452, + "time_per_iteration": 4.762778997421265 + }, + { + "auxiliary_loss_clip": 0.01156788, + "auxiliary_loss_mlp": 0.01048854, + "balance_loss_clip": 1.04735506, + "balance_loss_mlp": 1.02348578, + "epoch": 0.0421623817538158, + "flos": 11101053185280.0, + "grad_norm": 3.8850996392874513, + "language_loss": 0.76745987, + "learning_rate": 3.998449574564484e-06, + "loss": 0.78951627, + "num_input_tokens_seen": 40309930, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.25354004, + "step": 1453, + "time_per_iteration": 2.41801381111145 + }, + { + "auxiliary_loss_clip": 0.01153631, + "auxiliary_loss_mlp": 0.01062826, + "balance_loss_clip": 1.04329979, + "balance_loss_mlp": 1.03473985, + "epoch": 0.042191399222331845, + "flos": 30257337611520.0, + "grad_norm": 2.4612282485467296, + "language_loss": 1.06080604, + "learning_rate": 3.998442166096521e-06, + "loss": 1.08297062, + "num_input_tokens_seen": 40327730, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.28063965, + "step": 1454, + "time_per_iteration": 5.014498710632324 + }, + { + "auxiliary_loss_clip": 0.01154391, + "auxiliary_loss_mlp": 0.01053808, + "balance_loss_clip": 1.04944277, + "balance_loss_mlp": 1.0292747, + "epoch": 0.04222041669084789, + "flos": 15004342920960.0, + "grad_norm": 3.098738706911014, + "language_loss": 0.80425298, + "learning_rate": 3.998434739977508e-06, + "loss": 0.82633495, + "num_input_tokens_seen": 40338910, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.24536133, + "step": 1455, + "time_per_iteration": 5.217005014419556 + }, + { + "auxiliary_loss_clip": 0.01144767, + "auxiliary_loss_mlp": 0.01051524, + "balance_loss_clip": 1.04533267, + "balance_loss_mlp": 1.02774155, + "epoch": 0.042249434159363934, + "flos": 74731727784960.0, + "grad_norm": 2.1289962298510963, + "language_loss": 0.83227539, + "learning_rate": 3.99842729620751e-06, + "loss": 0.85423833, + "num_input_tokens_seen": 40362840, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.23803711, + "step": 1456, + "time_per_iteration": 2.8323378562927246 + }, + { + "auxiliary_loss_clip": 0.01166835, + "auxiliary_loss_mlp": 0.01053244, + "balance_loss_clip": 1.0489037, + "balance_loss_mlp": 1.02387094, + "epoch": 0.042278451627879986, + "flos": 31896544855680.0, + "grad_norm": 3.7912126699267508, + "language_loss": 0.86414772, + "learning_rate": 3.998419834786595e-06, + "loss": 0.88634849, + "num_input_tokens_seen": 40377525, + "router_z_loss_clip": 1.17871094, + "router_z_loss_mlp": 0.29370117, + "step": 1457, + "time_per_iteration": 2.628897190093994 + }, + { + "auxiliary_loss_clip": 0.01042169, + "auxiliary_loss_mlp": 0.01036102, + "balance_loss_clip": 1.0205934, + "balance_loss_mlp": 1.03398049, + "epoch": 0.04230746909639603, + "flos": 74771317267200.0, + "grad_norm": 0.7361168638075961, + "language_loss": 0.49160489, + "learning_rate": 3.998412355714826e-06, + "loss": 0.51238763, + "num_input_tokens_seen": 40442855, + "router_z_loss_clip": 0.21582031, + "router_z_loss_mlp": 0.02124023, + "step": 1458, + "time_per_iteration": 3.3697516918182373 + }, + { + "auxiliary_loss_clip": 0.01030666, + "auxiliary_loss_mlp": 0.01012293, + "balance_loss_clip": 1.0094285, + "balance_loss_mlp": 1.01029003, + "epoch": 0.042336486564912075, + "flos": 69226969985280.0, + "grad_norm": 0.7609094188461553, + "language_loss": 0.53146869, + "learning_rate": 3.998404858992271e-06, + "loss": 0.55189824, + "num_input_tokens_seen": 40507345, + "router_z_loss_clip": 0.21191406, + "router_z_loss_mlp": 0.02001953, + "step": 1459, + "time_per_iteration": 3.2109062671661377 + }, + { + "auxiliary_loss_clip": 0.01024099, + "auxiliary_loss_mlp": 0.01004492, + "balance_loss_clip": 1.00338721, + "balance_loss_mlp": 1.00238192, + "epoch": 0.04236550403342813, + "flos": 74781092448000.0, + "grad_norm": 0.6285769738679374, + "language_loss": 0.52234238, + "learning_rate": 3.998397344618996e-06, + "loss": 0.54262829, + "num_input_tokens_seen": 40577040, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.02111816, + "step": 1460, + "time_per_iteration": 3.185866594314575 + }, + { + "auxiliary_loss_clip": 0.01169176, + "auxiliary_loss_mlp": 0.01074486, + "balance_loss_clip": 1.05248833, + "balance_loss_mlp": 1.04755712, + "epoch": 0.04239452150194417, + "flos": 37923845241600.0, + "grad_norm": 2.4599863685126553, + "language_loss": 0.90866899, + "learning_rate": 3.9983898125950665e-06, + "loss": 0.93110561, + "num_input_tokens_seen": 40594925, + "router_z_loss_clip": 1.16650391, + "router_z_loss_mlp": 0.26916504, + "step": 1461, + "time_per_iteration": 2.6185905933380127 + }, + { + "auxiliary_loss_clip": 0.01038138, + "auxiliary_loss_mlp": 0.01023716, + "balance_loss_clip": 1.01742649, + "balance_loss_mlp": 1.02148712, + "epoch": 0.042423538970460216, + "flos": 64125425877120.0, + "grad_norm": 0.697879261083145, + "language_loss": 0.48883873, + "learning_rate": 3.998382262920549e-06, + "loss": 0.50945729, + "num_input_tokens_seen": 40660760, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.02233887, + "step": 1462, + "time_per_iteration": 3.1733028888702393 + }, + { + "auxiliary_loss_clip": 0.01163597, + "auxiliary_loss_mlp": 0.01091763, + "balance_loss_clip": 1.05676413, + "balance_loss_mlp": 1.06469023, + "epoch": 0.04245255643897626, + "flos": 23615600474880.0, + "grad_norm": 2.6135776722907367, + "language_loss": 0.88065422, + "learning_rate": 3.9983746955955115e-06, + "loss": 0.90320778, + "num_input_tokens_seen": 40673980, + "router_z_loss_clip": 1.06933594, + "router_z_loss_mlp": 0.27087402, + "step": 1463, + "time_per_iteration": 2.485689163208008 + }, + { + "auxiliary_loss_clip": 0.01171869, + "auxiliary_loss_mlp": 0.01091685, + "balance_loss_clip": 1.05532813, + "balance_loss_mlp": 1.06322992, + "epoch": 0.04248157390749231, + "flos": 29570840081280.0, + "grad_norm": 2.3873831020099576, + "language_loss": 0.81238413, + "learning_rate": 3.9983671106200205e-06, + "loss": 0.83501971, + "num_input_tokens_seen": 40689740, + "router_z_loss_clip": 1.16503906, + "router_z_loss_mlp": 0.2845459, + "step": 1464, + "time_per_iteration": 2.528719186782837 + }, + { + "auxiliary_loss_clip": 0.01167043, + "auxiliary_loss_mlp": 0.0108947, + "balance_loss_clip": 1.05848145, + "balance_loss_mlp": 1.06456685, + "epoch": 0.04251059137600836, + "flos": 15661897067520.0, + "grad_norm": 2.5633113863430332, + "language_loss": 0.75859189, + "learning_rate": 3.998359507994142e-06, + "loss": 0.78115708, + "num_input_tokens_seen": 40702180, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.24902344, + "step": 1465, + "time_per_iteration": 2.377399444580078 + }, + { + "auxiliary_loss_clip": 0.01051608, + "auxiliary_loss_mlp": 0.01051814, + "balance_loss_clip": 1.02902007, + "balance_loss_mlp": 1.04904819, + "epoch": 0.0425396088445244, + "flos": 57044062500480.0, + "grad_norm": 0.7323633237910941, + "language_loss": 0.52404213, + "learning_rate": 3.998351887717943e-06, + "loss": 0.54507625, + "num_input_tokens_seen": 40762105, + "router_z_loss_clip": 0.2265625, + "router_z_loss_mlp": 0.02770996, + "step": 1466, + "time_per_iteration": 3.2348709106445312 + }, + { + "auxiliary_loss_clip": 0.01171187, + "auxiliary_loss_mlp": 0.01085813, + "balance_loss_clip": 1.05419636, + "balance_loss_mlp": 1.05664301, + "epoch": 0.04256862631304045, + "flos": 14347905937920.0, + "grad_norm": 4.728193779595837, + "language_loss": 0.89715117, + "learning_rate": 3.998344249791492e-06, + "loss": 0.91972113, + "num_input_tokens_seen": 40773220, + "router_z_loss_clip": 1.17138672, + "router_z_loss_mlp": 0.29174805, + "step": 1467, + "time_per_iteration": 2.4233362674713135 + }, + { + "auxiliary_loss_clip": 0.01034559, + "auxiliary_loss_mlp": 0.01019025, + "balance_loss_clip": 1.01365924, + "balance_loss_mlp": 1.01639009, + "epoch": 0.0425976437815565, + "flos": 55068188215680.0, + "grad_norm": 0.663399587827061, + "language_loss": 0.47833362, + "learning_rate": 3.998336594214856e-06, + "loss": 0.49886945, + "num_input_tokens_seen": 40833030, + "router_z_loss_clip": 0.20898438, + "router_z_loss_mlp": 0.02636719, + "step": 1468, + "time_per_iteration": 2.9691269397735596 + }, + { + "auxiliary_loss_clip": 0.01149946, + "auxiliary_loss_mlp": 0.01068399, + "balance_loss_clip": 1.04561758, + "balance_loss_mlp": 1.04397297, + "epoch": 0.04262666125007254, + "flos": 14897159447040.0, + "grad_norm": 3.000843501756164, + "language_loss": 0.84538579, + "learning_rate": 3.998328920988102e-06, + "loss": 0.86756921, + "num_input_tokens_seen": 40845060, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.24414062, + "step": 1469, + "time_per_iteration": 2.3926520347595215 + }, + { + "auxiliary_loss_clip": 0.01162936, + "auxiliary_loss_mlp": 0.01072597, + "balance_loss_clip": 1.04674888, + "balance_loss_mlp": 1.0434978, + "epoch": 0.04265567871858859, + "flos": 27045153705600.0, + "grad_norm": 2.4810603199334222, + "language_loss": 1.10573459, + "learning_rate": 3.9983212301113e-06, + "loss": 1.12809002, + "num_input_tokens_seen": 40863115, + "router_z_loss_clip": 1.16210938, + "router_z_loss_mlp": 0.29101562, + "step": 1470, + "time_per_iteration": 2.556607723236084 + }, + { + "auxiliary_loss_clip": 0.01152794, + "auxiliary_loss_mlp": 0.01052163, + "balance_loss_clip": 1.04429364, + "balance_loss_mlp": 1.02491188, + "epoch": 0.04268469618710464, + "flos": 13874867660160.0, + "grad_norm": 3.1268589469731163, + "language_loss": 0.89049602, + "learning_rate": 3.998313521584514e-06, + "loss": 0.9125455, + "num_input_tokens_seen": 40877410, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.27246094, + "step": 1471, + "time_per_iteration": 2.403996229171753 + }, + { + "auxiliary_loss_clip": 0.01157329, + "auxiliary_loss_mlp": 0.0105426, + "balance_loss_clip": 1.04453576, + "balance_loss_mlp": 1.02601969, + "epoch": 0.04271371365562068, + "flos": 28467969143040.0, + "grad_norm": 2.5456264861391915, + "language_loss": 0.92632198, + "learning_rate": 3.998305795407816e-06, + "loss": 0.94843781, + "num_input_tokens_seen": 40895275, + "router_z_loss_clip": 1.12792969, + "router_z_loss_mlp": 0.28271484, + "step": 1472, + "time_per_iteration": 2.491021156311035 + }, + { + "auxiliary_loss_clip": 0.01150742, + "auxiliary_loss_mlp": 0.0105, + "balance_loss_clip": 1.0415231, + "balance_loss_mlp": 1.02383399, + "epoch": 0.04274273112413673, + "flos": 31897208171520.0, + "grad_norm": 3.8277357694787266, + "language_loss": 0.83132052, + "learning_rate": 3.998298051581272e-06, + "loss": 0.85332787, + "num_input_tokens_seen": 40911995, + "router_z_loss_clip": 1.09228516, + "router_z_loss_mlp": 0.26171875, + "step": 1473, + "time_per_iteration": 2.5457918643951416 + }, + { + "auxiliary_loss_clip": 0.01142981, + "auxiliary_loss_mlp": 0.01050053, + "balance_loss_clip": 1.04264975, + "balance_loss_mlp": 1.02568674, + "epoch": 0.04277174859265278, + "flos": 16538391550080.0, + "grad_norm": 2.503118416831276, + "language_loss": 0.67983031, + "learning_rate": 3.998290290104951e-06, + "loss": 0.70176065, + "num_input_tokens_seen": 40925860, + "router_z_loss_clip": 1.00341797, + "router_z_loss_mlp": 0.24377441, + "step": 1474, + "time_per_iteration": 2.503659248352051 + }, + { + "auxiliary_loss_clip": 0.01151394, + "auxiliary_loss_mlp": 0.01067708, + "balance_loss_clip": 1.04418254, + "balance_loss_mlp": 1.03988421, + "epoch": 0.042800766061168824, + "flos": 29892041389440.0, + "grad_norm": 2.3033281039288473, + "language_loss": 0.90635788, + "learning_rate": 3.998282510978922e-06, + "loss": 0.92854893, + "num_input_tokens_seen": 40942415, + "router_z_loss_clip": 1.07226562, + "router_z_loss_mlp": 0.27819824, + "step": 1475, + "time_per_iteration": 2.56528377532959 + }, + { + "auxiliary_loss_clip": 0.01073304, + "auxiliary_loss_mlp": 0.01026191, + "balance_loss_clip": 1.0503304, + "balance_loss_mlp": 1.02408099, + "epoch": 0.04282978352968487, + "flos": 55534348955520.0, + "grad_norm": 0.7826721606188839, + "language_loss": 0.56930864, + "learning_rate": 3.998274714203252e-06, + "loss": 0.59030366, + "num_input_tokens_seen": 40999330, + "router_z_loss_clip": 0.22949219, + "router_z_loss_mlp": 0.02111816, + "step": 1476, + "time_per_iteration": 2.9600818157196045 + }, + { + "auxiliary_loss_clip": 0.01159894, + "auxiliary_loss_mlp": 0.01067803, + "balance_loss_clip": 1.05034947, + "balance_loss_mlp": 1.03804874, + "epoch": 0.04285880099820092, + "flos": 33977542844160.0, + "grad_norm": 2.3224321978423155, + "language_loss": 0.81641239, + "learning_rate": 3.998266899778012e-06, + "loss": 0.83868933, + "num_input_tokens_seen": 41014725, + "router_z_loss_clip": 1.09472656, + "router_z_loss_mlp": 0.29748535, + "step": 1477, + "time_per_iteration": 2.570636034011841 + }, + { + "auxiliary_loss_clip": 0.0114604, + "auxiliary_loss_mlp": 0.01055511, + "balance_loss_clip": 1.04521728, + "balance_loss_mlp": 1.02955914, + "epoch": 0.042887818466716965, + "flos": 16135528838400.0, + "grad_norm": 3.1096783370124848, + "language_loss": 0.73583031, + "learning_rate": 3.9982590677032705e-06, + "loss": 0.75784576, + "num_input_tokens_seen": 41028050, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.25927734, + "step": 1478, + "time_per_iteration": 2.3709537982940674 + }, + { + "auxiliary_loss_clip": 0.01155521, + "auxiliary_loss_mlp": 0.01057885, + "balance_loss_clip": 1.04689956, + "balance_loss_mlp": 1.02995455, + "epoch": 0.04291683593523301, + "flos": 26641906968960.0, + "grad_norm": 2.8700250268604264, + "language_loss": 0.85444236, + "learning_rate": 3.998251217979095e-06, + "loss": 0.87657642, + "num_input_tokens_seen": 41041355, + "router_z_loss_clip": 1.08496094, + "router_z_loss_mlp": 0.27978516, + "step": 1479, + "time_per_iteration": 2.533339023590088 + }, + { + "auxiliary_loss_clip": 0.01032167, + "auxiliary_loss_mlp": 0.01007945, + "balance_loss_clip": 1.01105249, + "balance_loss_mlp": 1.00582337, + "epoch": 0.042945853403749054, + "flos": 74777531489280.0, + "grad_norm": 0.6619144713789078, + "language_loss": 0.55109167, + "learning_rate": 3.9982433506055574e-06, + "loss": 0.57149279, + "num_input_tokens_seen": 41110965, + "router_z_loss_clip": 0.2109375, + "router_z_loss_mlp": 0.02124023, + "step": 1480, + "time_per_iteration": 3.328178644180298 + }, + { + "auxiliary_loss_clip": 0.01153961, + "auxiliary_loss_mlp": 0.01061263, + "balance_loss_clip": 1.04380834, + "balance_loss_mlp": 1.02850413, + "epoch": 0.042974870872265106, + "flos": 15516483788160.0, + "grad_norm": 2.473041564570905, + "language_loss": 1.02290154, + "learning_rate": 3.998235465582726e-06, + "loss": 1.04505384, + "num_input_tokens_seen": 41124830, + "router_z_loss_clip": 1.10107422, + "router_z_loss_mlp": 0.32763672, + "step": 1481, + "time_per_iteration": 2.418264865875244 + }, + { + "auxiliary_loss_clip": 0.01027142, + "auxiliary_loss_mlp": 0.01004659, + "balance_loss_clip": 1.00649071, + "balance_loss_mlp": 1.00253749, + "epoch": 0.04300388834078115, + "flos": 63344943233280.0, + "grad_norm": 0.7227824736329652, + "language_loss": 0.51697081, + "learning_rate": 3.99822756291067e-06, + "loss": 0.53728884, + "num_input_tokens_seen": 41177110, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.02124023, + "step": 1482, + "time_per_iteration": 2.868655204772949 + }, + { + "auxiliary_loss_clip": 0.0115215, + "auxiliary_loss_mlp": 0.01058528, + "balance_loss_clip": 1.04334998, + "balance_loss_mlp": 1.03027511, + "epoch": 0.043032905809297195, + "flos": 74731867430400.0, + "grad_norm": 2.4419184433012435, + "language_loss": 0.93206227, + "learning_rate": 3.998219642589459e-06, + "loss": 0.95416903, + "num_input_tokens_seen": 41198220, + "router_z_loss_clip": 1.08789062, + "router_z_loss_mlp": 0.2824707, + "step": 1483, + "time_per_iteration": 2.836735486984253 + }, + { + "auxiliary_loss_clip": 0.01152423, + "auxiliary_loss_mlp": 0.01062111, + "balance_loss_clip": 1.04685044, + "balance_loss_mlp": 1.03462136, + "epoch": 0.04306192327781325, + "flos": 15333050171520.0, + "grad_norm": 3.375384508924648, + "language_loss": 0.79733604, + "learning_rate": 3.998211704619164e-06, + "loss": 0.81948137, + "num_input_tokens_seen": 41210770, + "router_z_loss_clip": 1.05712891, + "router_z_loss_mlp": 0.2746582, + "step": 1484, + "time_per_iteration": 2.3954203128814697 + }, + { + "auxiliary_loss_clip": 0.01033159, + "auxiliary_loss_mlp": 0.01010776, + "balance_loss_clip": 1.01257038, + "balance_loss_mlp": 1.00857019, + "epoch": 0.04309094074632929, + "flos": 57864941320320.0, + "grad_norm": 0.7178224406957866, + "language_loss": 0.47381791, + "learning_rate": 3.998203748999854e-06, + "loss": 0.49425721, + "num_input_tokens_seen": 41259835, + "router_z_loss_clip": 0.20605469, + "router_z_loss_mlp": 0.02209473, + "step": 1485, + "time_per_iteration": 2.7398788928985596 + }, + { + "auxiliary_loss_clip": 0.01161652, + "auxiliary_loss_mlp": 0.01062658, + "balance_loss_clip": 1.04767632, + "balance_loss_mlp": 1.03243887, + "epoch": 0.043119958214845336, + "flos": 27226632286080.0, + "grad_norm": 2.4012338170282805, + "language_loss": 0.87867081, + "learning_rate": 3.9981957757316015e-06, + "loss": 0.90091395, + "num_input_tokens_seen": 41274655, + "router_z_loss_clip": 1.14160156, + "router_z_loss_mlp": 0.30212402, + "step": 1486, + "time_per_iteration": 2.3966493606567383 + }, + { + "auxiliary_loss_clip": 0.01140334, + "auxiliary_loss_mlp": 0.01048984, + "balance_loss_clip": 1.04397869, + "balance_loss_mlp": 1.02750206, + "epoch": 0.04314897568336138, + "flos": 33503352491520.0, + "grad_norm": 3.5159558475765973, + "language_loss": 0.90286446, + "learning_rate": 3.998187784814474e-06, + "loss": 0.9247576, + "num_input_tokens_seen": 41288925, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.21472168, + "step": 1487, + "time_per_iteration": 2.5184717178344727 + }, + { + "auxiliary_loss_clip": 0.01026889, + "auxiliary_loss_mlp": 0.01009761, + "balance_loss_clip": 1.00649595, + "balance_loss_mlp": 1.00736487, + "epoch": 0.04317799315187743, + "flos": 70543509644160.0, + "grad_norm": 0.9708137336094561, + "language_loss": 0.52892274, + "learning_rate": 3.998179776248544e-06, + "loss": 0.54928923, + "num_input_tokens_seen": 41355420, + "router_z_loss_clip": 0.20410156, + "router_z_loss_mlp": 0.02392578, + "step": 1488, + "time_per_iteration": 3.0427870750427246 + }, + { + "auxiliary_loss_clip": 0.01157274, + "auxiliary_loss_mlp": 0.01069645, + "balance_loss_clip": 1.04529011, + "balance_loss_mlp": 1.0404985, + "epoch": 0.04320701062039348, + "flos": 18658666684800.0, + "grad_norm": 2.3244348600049514, + "language_loss": 0.84421998, + "learning_rate": 3.998171750033881e-06, + "loss": 0.86648917, + "num_input_tokens_seen": 41370535, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.29162598, + "step": 1489, + "time_per_iteration": 2.4571266174316406 + }, + { + "auxiliary_loss_clip": 0.01147026, + "auxiliary_loss_mlp": 0.01053004, + "balance_loss_clip": 1.04271913, + "balance_loss_mlp": 1.02820802, + "epoch": 0.04323602808890952, + "flos": 26789275284480.0, + "grad_norm": 2.647925971837745, + "language_loss": 0.76974118, + "learning_rate": 3.998163706170557e-06, + "loss": 0.79174143, + "num_input_tokens_seen": 41388665, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.24804688, + "step": 1490, + "time_per_iteration": 2.554766893386841 + }, + { + "auxiliary_loss_clip": 0.01022231, + "auxiliary_loss_mlp": 0.01006682, + "balance_loss_clip": 1.00189304, + "balance_loss_mlp": 1.00450075, + "epoch": 0.04326504555742557, + "flos": 62438143824000.0, + "grad_norm": 0.6389110305243949, + "language_loss": 0.48352864, + "learning_rate": 3.998155644658642e-06, + "loss": 0.5038178, + "num_input_tokens_seen": 41452970, + "router_z_loss_clip": 0.203125, + "router_z_loss_mlp": 0.02185059, + "step": 1491, + "time_per_iteration": 3.0527772903442383 + }, + { + "auxiliary_loss_clip": 0.01146071, + "auxiliary_loss_mlp": 0.01053672, + "balance_loss_clip": 1.04133213, + "balance_loss_mlp": 1.02825689, + "epoch": 0.04329406302594162, + "flos": 36718364217600.0, + "grad_norm": 1.9872449686009412, + "language_loss": 0.99765539, + "learning_rate": 3.998147565498208e-06, + "loss": 1.01965284, + "num_input_tokens_seen": 41472605, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.25415039, + "step": 1492, + "time_per_iteration": 2.657599687576294 + }, + { + "auxiliary_loss_clip": 0.01141394, + "auxiliary_loss_mlp": 0.01049594, + "balance_loss_clip": 1.04087448, + "balance_loss_mlp": 1.02326119, + "epoch": 0.04332308049445766, + "flos": 11693982672000.0, + "grad_norm": 2.9576916492932668, + "language_loss": 0.79427111, + "learning_rate": 3.998139468689327e-06, + "loss": 0.81618094, + "num_input_tokens_seen": 41485800, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.26293945, + "step": 1493, + "time_per_iteration": 2.7407302856445312 + }, + { + "auxiliary_loss_clip": 0.01145419, + "auxiliary_loss_mlp": 0.01048794, + "balance_loss_clip": 1.0430994, + "balance_loss_mlp": 1.02402246, + "epoch": 0.04335209796297371, + "flos": 39557850693120.0, + "grad_norm": 3.074535733289656, + "language_loss": 0.94643533, + "learning_rate": 3.998131354232069e-06, + "loss": 0.96837747, + "num_input_tokens_seen": 41502675, + "router_z_loss_clip": 1.02148438, + "router_z_loss_mlp": 0.24755859, + "step": 1494, + "time_per_iteration": 2.624504327774048 + }, + { + "auxiliary_loss_clip": 0.01150163, + "auxiliary_loss_mlp": 0.01048498, + "balance_loss_clip": 1.042099, + "balance_loss_mlp": 1.02226019, + "epoch": 0.04338111543148976, + "flos": 33619159071360.0, + "grad_norm": 1.9028951050186986, + "language_loss": 1.10392964, + "learning_rate": 3.998123222126506e-06, + "loss": 1.12591624, + "num_input_tokens_seen": 41525430, + "router_z_loss_clip": 1.08105469, + "router_z_loss_mlp": 0.2623291, + "step": 1495, + "time_per_iteration": 2.549949884414673 + }, + { + "auxiliary_loss_clip": 0.011592, + "auxiliary_loss_mlp": 0.01066559, + "balance_loss_clip": 1.04836833, + "balance_loss_mlp": 1.03697157, + "epoch": 0.0434101329000058, + "flos": 11905242508800.0, + "grad_norm": 9.890305629450777, + "language_loss": 0.89162749, + "learning_rate": 3.998115072372711e-06, + "loss": 0.91388512, + "num_input_tokens_seen": 41537700, + "router_z_loss_clip": 1.10791016, + "router_z_loss_mlp": 0.2956543, + "step": 1496, + "time_per_iteration": 2.4973061084747314 + }, + { + "auxiliary_loss_clip": 0.01141391, + "auxiliary_loss_mlp": 0.01054224, + "balance_loss_clip": 1.04104924, + "balance_loss_mlp": 1.02779555, + "epoch": 0.04343915036852185, + "flos": 39959735886720.0, + "grad_norm": 3.845688981543057, + "language_loss": 1.02074957, + "learning_rate": 3.998106904970754e-06, + "loss": 1.04270577, + "num_input_tokens_seen": 41554365, + "router_z_loss_clip": 1.00439453, + "router_z_loss_mlp": 0.2644043, + "step": 1497, + "time_per_iteration": 2.502089262008667 + }, + { + "auxiliary_loss_clip": 0.01146225, + "auxiliary_loss_mlp": 0.01057456, + "balance_loss_clip": 1.04548287, + "balance_loss_mlp": 1.03275585, + "epoch": 0.0434681678370379, + "flos": 35034119452800.0, + "grad_norm": 5.119489880420007, + "language_loss": 0.91491264, + "learning_rate": 3.9980987199207096e-06, + "loss": 0.93694949, + "num_input_tokens_seen": 41571635, + "router_z_loss_clip": 1.00732422, + "router_z_loss_mlp": 0.2467041, + "step": 1498, + "time_per_iteration": 2.589798927307129 + }, + { + "auxiliary_loss_clip": 0.01027867, + "auxiliary_loss_mlp": 0.0100423, + "balance_loss_clip": 1.00762534, + "balance_loss_mlp": 1.00213194, + "epoch": 0.043497185305553944, + "flos": 74776379414400.0, + "grad_norm": 0.6583417801709164, + "language_loss": 0.47284758, + "learning_rate": 3.998090517222648e-06, + "loss": 0.49316859, + "num_input_tokens_seen": 41636485, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.02099609, + "step": 1499, + "time_per_iteration": 3.099416494369507 + }, + { + "auxiliary_loss_clip": 0.01155875, + "auxiliary_loss_mlp": 0.01073148, + "balance_loss_clip": 1.04413104, + "balance_loss_mlp": 1.04362011, + "epoch": 0.04352620277406999, + "flos": 10774510439040.0, + "grad_norm": 3.647420444253279, + "language_loss": 0.98244876, + "learning_rate": 3.998082296876643e-06, + "loss": 1.00473905, + "num_input_tokens_seen": 41648245, + "router_z_loss_clip": 1.11816406, + "router_z_loss_mlp": 0.29528809, + "step": 1500, + "time_per_iteration": 2.5035243034362793 + }, + { + "auxiliary_loss_clip": 0.01025083, + "auxiliary_loss_mlp": 0.01002655, + "balance_loss_clip": 1.00538528, + "balance_loss_mlp": 1.00049758, + "epoch": 0.04355522024258604, + "flos": 65845805765760.0, + "grad_norm": 0.6805968345061282, + "language_loss": 0.52530324, + "learning_rate": 3.9980740588827655e-06, + "loss": 0.54558063, + "num_input_tokens_seen": 41716230, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.02160645, + "step": 1501, + "time_per_iteration": 3.0951881408691406 + }, + { + "auxiliary_loss_clip": 0.0115007, + "auxiliary_loss_mlp": 0.01057749, + "balance_loss_clip": 1.0427767, + "balance_loss_mlp": 1.02983069, + "epoch": 0.043584237711102085, + "flos": 28504348646400.0, + "grad_norm": 2.5009893523389652, + "language_loss": 0.83542639, + "learning_rate": 3.99806580324109e-06, + "loss": 0.85750461, + "num_input_tokens_seen": 41731535, + "router_z_loss_clip": 1.07177734, + "router_z_loss_mlp": 0.27929688, + "step": 1502, + "time_per_iteration": 2.5360050201416016 + }, + { + "auxiliary_loss_clip": 0.01151723, + "auxiliary_loss_mlp": 0.01055732, + "balance_loss_clip": 1.04342365, + "balance_loss_mlp": 1.0290525, + "epoch": 0.04361325517961813, + "flos": 31424553918720.0, + "grad_norm": 2.6674604258594723, + "language_loss": 1.15414751, + "learning_rate": 3.998057529951688e-06, + "loss": 1.17622209, + "num_input_tokens_seen": 41754425, + "router_z_loss_clip": 1.0859375, + "router_z_loss_mlp": 0.26672363, + "step": 1503, + "time_per_iteration": 2.613891363143921 + }, + { + "auxiliary_loss_clip": 0.0115088, + "auxiliary_loss_mlp": 0.0105516, + "balance_loss_clip": 1.04169488, + "balance_loss_mlp": 1.02632368, + "epoch": 0.043642272648134174, + "flos": 32085599201280.0, + "grad_norm": 2.4638180625754367, + "language_loss": 1.07591999, + "learning_rate": 3.998049239014634e-06, + "loss": 1.09798038, + "num_input_tokens_seen": 41772655, + "router_z_loss_clip": 1.09277344, + "router_z_loss_mlp": 0.28820801, + "step": 1504, + "time_per_iteration": 2.5318658351898193 + }, + { + "auxiliary_loss_clip": 0.01137778, + "auxiliary_loss_mlp": 0.01048667, + "balance_loss_clip": 1.04110861, + "balance_loss_mlp": 1.02490807, + "epoch": 0.043671290116650226, + "flos": 15661617776640.0, + "grad_norm": 2.8730984880128307, + "language_loss": 0.90980625, + "learning_rate": 3.998040930430001e-06, + "loss": 0.93167073, + "num_input_tokens_seen": 41785700, + "router_z_loss_clip": 0.96679688, + "router_z_loss_mlp": 0.23754883, + "step": 1505, + "time_per_iteration": 2.381957769393921 + }, + { + "auxiliary_loss_clip": 0.01136815, + "auxiliary_loss_mlp": 0.01046386, + "balance_loss_clip": 1.03921103, + "balance_loss_mlp": 1.02151942, + "epoch": 0.04370030758516627, + "flos": 13726766206080.0, + "grad_norm": 2.641187500156385, + "language_loss": 0.74591959, + "learning_rate": 3.998032604197862e-06, + "loss": 0.76775157, + "num_input_tokens_seen": 41799735, + "router_z_loss_clip": 0.97558594, + "router_z_loss_mlp": 0.24853516, + "step": 1506, + "time_per_iteration": 2.705390214920044 + }, + { + "auxiliary_loss_clip": 0.01142094, + "auxiliary_loss_mlp": 0.01043488, + "balance_loss_clip": 1.04223168, + "balance_loss_mlp": 1.01894236, + "epoch": 0.043729325053682315, + "flos": 33285494407680.0, + "grad_norm": 2.401421005530978, + "language_loss": 0.74835396, + "learning_rate": 3.99802426031829e-06, + "loss": 0.77020979, + "num_input_tokens_seen": 41816505, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.2454834, + "step": 1507, + "time_per_iteration": 2.591686725616455 + }, + { + "auxiliary_loss_clip": 0.01023583, + "auxiliary_loss_mlp": 0.01008715, + "balance_loss_clip": 1.00381756, + "balance_loss_mlp": 1.00659299, + "epoch": 0.04375834252219837, + "flos": 74772085317120.0, + "grad_norm": 0.8289380096539338, + "language_loss": 0.49881685, + "learning_rate": 3.9980158987913595e-06, + "loss": 0.51913977, + "num_input_tokens_seen": 41880970, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.02124023, + "step": 1508, + "time_per_iteration": 3.0819857120513916 + }, + { + "auxiliary_loss_clip": 0.01023683, + "auxiliary_loss_mlp": 0.01002195, + "balance_loss_clip": 1.00361598, + "balance_loss_mlp": 1.00015676, + "epoch": 0.04378735999071441, + "flos": 74763497122560.0, + "grad_norm": 0.7369109958205335, + "language_loss": 0.61000609, + "learning_rate": 3.998007519617144e-06, + "loss": 0.63026488, + "num_input_tokens_seen": 41936300, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.02038574, + "step": 1509, + "time_per_iteration": 3.003929853439331 + }, + { + "auxiliary_loss_clip": 0.01151321, + "auxiliary_loss_mlp": 0.01059292, + "balance_loss_clip": 1.04545856, + "balance_loss_mlp": 1.03307772, + "epoch": 0.043816377459230456, + "flos": 74731692873600.0, + "grad_norm": 3.3927948215581796, + "language_loss": 0.90699071, + "learning_rate": 3.997999122795718e-06, + "loss": 0.92909688, + "num_input_tokens_seen": 41959140, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.26196289, + "step": 1510, + "time_per_iteration": 2.822732448577881 + }, + { + "auxiliary_loss_clip": 0.01151425, + "auxiliary_loss_mlp": 0.01064003, + "balance_loss_clip": 1.04510951, + "balance_loss_mlp": 1.03584599, + "epoch": 0.0438453949277465, + "flos": 13324915923840.0, + "grad_norm": 4.608100156572601, + "language_loss": 0.83531845, + "learning_rate": 3.997990708327154e-06, + "loss": 0.85747272, + "num_input_tokens_seen": 41969400, + "router_z_loss_clip": 1.06298828, + "router_z_loss_mlp": 0.28149414, + "step": 1511, + "time_per_iteration": 2.4498159885406494 + }, + { + "auxiliary_loss_clip": 0.01139356, + "auxiliary_loss_mlp": 0.01040563, + "balance_loss_clip": 1.03977156, + "balance_loss_mlp": 1.01667297, + "epoch": 0.04387441239626255, + "flos": 15770895932160.0, + "grad_norm": 2.236364109637703, + "language_loss": 0.66971844, + "learning_rate": 3.997982276211529e-06, + "loss": 0.69151759, + "num_input_tokens_seen": 41983435, + "router_z_loss_clip": 0.99609375, + "router_z_loss_mlp": 0.2388916, + "step": 1512, + "time_per_iteration": 2.5479352474212646 + }, + { + "auxiliary_loss_clip": 0.01139093, + "auxiliary_loss_mlp": 0.0104251, + "balance_loss_clip": 1.04199564, + "balance_loss_mlp": 1.02026582, + "epoch": 0.0439034298647786, + "flos": 20625219636480.0, + "grad_norm": 3.0440149411722524, + "language_loss": 0.7442559, + "learning_rate": 3.997973826448915e-06, + "loss": 0.76607192, + "num_input_tokens_seen": 41998705, + "router_z_loss_clip": 0.97119141, + "router_z_loss_mlp": 0.22241211, + "step": 1513, + "time_per_iteration": 2.4705710411071777 + }, + { + "auxiliary_loss_clip": 0.01150689, + "auxiliary_loss_mlp": 0.01054701, + "balance_loss_clip": 1.0427022, + "balance_loss_mlp": 1.02796268, + "epoch": 0.04393244733329464, + "flos": 36313232267520.0, + "grad_norm": 2.315141642485015, + "language_loss": 0.8571316, + "learning_rate": 3.997965359039388e-06, + "loss": 0.8791855, + "num_input_tokens_seen": 42019220, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.26733398, + "step": 1514, + "time_per_iteration": 2.539665937423706 + }, + { + "auxiliary_loss_clip": 0.01024207, + "auxiliary_loss_mlp": 0.01020735, + "balance_loss_clip": 1.00401282, + "balance_loss_mlp": 1.01874399, + "epoch": 0.04396146480181069, + "flos": 70503464448000.0, + "grad_norm": 0.7075020155229766, + "language_loss": 0.53552377, + "learning_rate": 3.997956873983023e-06, + "loss": 0.55597317, + "num_input_tokens_seen": 42079550, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.01989746, + "step": 1515, + "time_per_iteration": 3.0266942977905273 + }, + { + "auxiliary_loss_clip": 0.01147167, + "auxiliary_loss_mlp": 0.0105165, + "balance_loss_clip": 1.04052961, + "balance_loss_mlp": 1.02318335, + "epoch": 0.04399048227032674, + "flos": 37371414798720.0, + "grad_norm": 2.6430344414123716, + "language_loss": 0.9331044, + "learning_rate": 3.997948371279894e-06, + "loss": 0.95509261, + "num_input_tokens_seen": 42098940, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.28491211, + "step": 1516, + "time_per_iteration": 2.554276466369629 + }, + { + "auxiliary_loss_clip": 0.01155397, + "auxiliary_loss_mlp": 0.01057338, + "balance_loss_clip": 1.04304695, + "balance_loss_mlp": 1.02661729, + "epoch": 0.04401949973884278, + "flos": 30658035818880.0, + "grad_norm": 2.5450106894230764, + "language_loss": 1.00580239, + "learning_rate": 3.997939850930076e-06, + "loss": 1.02792978, + "num_input_tokens_seen": 42119350, + "router_z_loss_clip": 1.12402344, + "router_z_loss_mlp": 0.30688477, + "step": 1517, + "time_per_iteration": 2.550222158432007 + }, + { + "auxiliary_loss_clip": 0.01022133, + "auxiliary_loss_mlp": 0.01012398, + "balance_loss_clip": 1.00245643, + "balance_loss_mlp": 1.0105505, + "epoch": 0.04404851720735883, + "flos": 57039698580480.0, + "grad_norm": 0.695344077278716, + "language_loss": 0.49964708, + "learning_rate": 3.997931312933645e-06, + "loss": 0.51999235, + "num_input_tokens_seen": 42174115, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.01843262, + "step": 1518, + "time_per_iteration": 2.9310035705566406 + }, + { + "auxiliary_loss_clip": 0.01021077, + "auxiliary_loss_mlp": 0.01005574, + "balance_loss_clip": 1.00173736, + "balance_loss_mlp": 1.00361919, + "epoch": 0.04407753467587488, + "flos": 47688457426560.0, + "grad_norm": 0.7272724272981181, + "language_loss": 0.49917489, + "learning_rate": 3.997922757290677e-06, + "loss": 0.51944137, + "num_input_tokens_seen": 42222380, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.01953125, + "step": 1519, + "time_per_iteration": 2.8617353439331055 + }, + { + "auxiliary_loss_clip": 0.01147309, + "auxiliary_loss_mlp": 0.01047166, + "balance_loss_clip": 1.04240632, + "balance_loss_mlp": 1.01922321, + "epoch": 0.04410655214439092, + "flos": 30036616796160.0, + "grad_norm": 3.5840440600280026, + "language_loss": 0.85105062, + "learning_rate": 3.997914184001246e-06, + "loss": 0.87299538, + "num_input_tokens_seen": 42238210, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.27966309, + "step": 1520, + "time_per_iteration": 2.5848441123962402 + }, + { + "auxiliary_loss_clip": 0.01140068, + "auxiliary_loss_mlp": 0.0104577, + "balance_loss_clip": 1.03791428, + "balance_loss_mlp": 1.01891196, + "epoch": 0.04413556961290697, + "flos": 36568028436480.0, + "grad_norm": 2.3000175519175197, + "language_loss": 0.95571125, + "learning_rate": 3.997905593065429e-06, + "loss": 0.9775697, + "num_input_tokens_seen": 42258205, + "router_z_loss_clip": 1.02099609, + "router_z_loss_mlp": 0.2689209, + "step": 1521, + "time_per_iteration": 2.5319318771362305 + }, + { + "auxiliary_loss_clip": 0.01022965, + "auxiliary_loss_mlp": 0.01008666, + "balance_loss_clip": 1.00276268, + "balance_loss_mlp": 1.0066638, + "epoch": 0.04416458708142302, + "flos": 55796127396480.0, + "grad_norm": 0.7379219498978813, + "language_loss": 0.4978292, + "learning_rate": 3.9978969844833e-06, + "loss": 0.51814556, + "num_input_tokens_seen": 42316345, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.02001953, + "step": 1522, + "time_per_iteration": 5.548024892807007 + }, + { + "auxiliary_loss_clip": 0.01134189, + "auxiliary_loss_mlp": 0.01047532, + "balance_loss_clip": 1.03749561, + "balance_loss_mlp": 1.02358937, + "epoch": 0.044193604549939064, + "flos": 24670780629120.0, + "grad_norm": 3.3173382483276317, + "language_loss": 0.86341965, + "learning_rate": 3.997888358254937e-06, + "loss": 0.88523686, + "num_input_tokens_seen": 42328330, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.23931885, + "step": 1523, + "time_per_iteration": 2.424879312515259 + }, + { + "auxiliary_loss_clip": 0.0102303, + "auxiliary_loss_mlp": 0.01008162, + "balance_loss_clip": 1.00288486, + "balance_loss_mlp": 1.00612354, + "epoch": 0.04422262201845511, + "flos": 74774214910080.0, + "grad_norm": 0.6873575863354413, + "language_loss": 0.52926362, + "learning_rate": 3.997879714380416e-06, + "loss": 0.54957551, + "num_input_tokens_seen": 42393765, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.02038574, + "step": 1524, + "time_per_iteration": 3.111471176147461 + }, + { + "auxiliary_loss_clip": 0.01147622, + "auxiliary_loss_mlp": 0.01055952, + "balance_loss_clip": 1.0455364, + "balance_loss_mlp": 1.0298686, + "epoch": 0.04425163948697115, + "flos": 13764611986560.0, + "grad_norm": 3.652635513056604, + "language_loss": 0.93412727, + "learning_rate": 3.997871052859813e-06, + "loss": 0.95616305, + "num_input_tokens_seen": 42405350, + "router_z_loss_clip": 1.02050781, + "router_z_loss_mlp": 0.26098633, + "step": 1525, + "time_per_iteration": 2.397064447402954 + }, + { + "auxiliary_loss_clip": 0.01147632, + "auxiliary_loss_mlp": 0.01048145, + "balance_loss_clip": 1.04118299, + "balance_loss_mlp": 1.02295613, + "epoch": 0.044280656955487205, + "flos": 17741079665280.0, + "grad_norm": 2.121919712090239, + "language_loss": 0.77315027, + "learning_rate": 3.997862373693203e-06, + "loss": 0.79510802, + "num_input_tokens_seen": 42418210, + "router_z_loss_clip": 1.06445312, + "router_z_loss_mlp": 0.2520752, + "step": 1526, + "time_per_iteration": 2.373171329498291 + }, + { + "auxiliary_loss_clip": 0.01139968, + "auxiliary_loss_mlp": 0.01051452, + "balance_loss_clip": 1.03698266, + "balance_loss_mlp": 1.02396202, + "epoch": 0.04430967442400325, + "flos": 16245435398400.0, + "grad_norm": 4.481551751105783, + "language_loss": 0.89588642, + "learning_rate": 3.9978536768806665e-06, + "loss": 0.91780066, + "num_input_tokens_seen": 42430275, + "router_z_loss_clip": 1.03076172, + "router_z_loss_mlp": 0.27490234, + "step": 1527, + "time_per_iteration": 2.404849052429199 + }, + { + "auxiliary_loss_clip": 0.01148182, + "auxiliary_loss_mlp": 0.01050308, + "balance_loss_clip": 1.03761566, + "balance_loss_mlp": 1.02380824, + "epoch": 0.044338691892519294, + "flos": 16755551406720.0, + "grad_norm": 2.2985437752439037, + "language_loss": 0.80167621, + "learning_rate": 3.997844962422277e-06, + "loss": 0.82366109, + "num_input_tokens_seen": 42442235, + "router_z_loss_clip": 1.10644531, + "router_z_loss_mlp": 0.26513672, + "step": 1528, + "time_per_iteration": 4.452122449874878 + }, + { + "auxiliary_loss_clip": 0.01021553, + "auxiliary_loss_mlp": 0.01005552, + "balance_loss_clip": 1.00198579, + "balance_loss_mlp": 1.00350165, + "epoch": 0.044367709361035346, + "flos": 67294911323520.0, + "grad_norm": 0.671769003356667, + "language_loss": 0.52583659, + "learning_rate": 3.997836230318111e-06, + "loss": 0.54610765, + "num_input_tokens_seen": 42506120, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.02050781, + "step": 1529, + "time_per_iteration": 5.612056493759155 + }, + { + "auxiliary_loss_clip": 0.01022086, + "auxiliary_loss_mlp": 0.01004221, + "balance_loss_clip": 1.00253558, + "balance_loss_mlp": 1.00223029, + "epoch": 0.04439672682955139, + "flos": 67795845644160.0, + "grad_norm": 0.7402157054589611, + "language_loss": 0.49070475, + "learning_rate": 3.997827480568248e-06, + "loss": 0.51096785, + "num_input_tokens_seen": 42565935, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.01989746, + "step": 1530, + "time_per_iteration": 5.441180944442749 + }, + { + "auxiliary_loss_clip": 0.01023313, + "auxiliary_loss_mlp": 0.01002778, + "balance_loss_clip": 1.00344872, + "balance_loss_mlp": 1.00078702, + "epoch": 0.044425744298067435, + "flos": 53967761072640.0, + "grad_norm": 0.7379979055562921, + "language_loss": 0.4976564, + "learning_rate": 3.997818713172764e-06, + "loss": 0.51791728, + "num_input_tokens_seen": 42618385, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.01989746, + "step": 1531, + "time_per_iteration": 2.8692729473114014 + }, + { + "auxiliary_loss_clip": 0.01146088, + "auxiliary_loss_mlp": 0.01058382, + "balance_loss_clip": 1.04203296, + "balance_loss_mlp": 1.03268087, + "epoch": 0.04445476176658349, + "flos": 25873224364800.0, + "grad_norm": 2.7871301022364294, + "language_loss": 0.89985347, + "learning_rate": 3.997809928131737e-06, + "loss": 0.92189813, + "num_input_tokens_seen": 42630755, + "router_z_loss_clip": 1.04003906, + "router_z_loss_mlp": 0.25720215, + "step": 1532, + "time_per_iteration": 2.4712305068969727 + }, + { + "auxiliary_loss_clip": 0.01024831, + "auxiliary_loss_mlp": 0.01009684, + "balance_loss_clip": 1.00492001, + "balance_loss_mlp": 1.00777638, + "epoch": 0.04448377923509953, + "flos": 65282552801280.0, + "grad_norm": 0.7222217225900279, + "language_loss": 0.52697372, + "learning_rate": 3.997801125445244e-06, + "loss": 0.54731894, + "num_input_tokens_seen": 42689745, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.01904297, + "step": 1533, + "time_per_iteration": 2.976585865020752 + }, + { + "auxiliary_loss_clip": 0.01151941, + "auxiliary_loss_mlp": 0.01053697, + "balance_loss_clip": 1.04762483, + "balance_loss_mlp": 1.02850795, + "epoch": 0.044512796703615576, + "flos": 16318334050560.0, + "grad_norm": 2.9456522084086956, + "language_loss": 0.64006972, + "learning_rate": 3.997792305113363e-06, + "loss": 0.66212612, + "num_input_tokens_seen": 42702550, + "router_z_loss_clip": 1.04394531, + "router_z_loss_mlp": 0.25195312, + "step": 1534, + "time_per_iteration": 2.4429807662963867 + }, + { + "auxiliary_loss_clip": 0.01136837, + "auxiliary_loss_mlp": 0.01053453, + "balance_loss_clip": 1.04006982, + "balance_loss_mlp": 1.02936077, + "epoch": 0.04454181417213162, + "flos": 12049538624640.0, + "grad_norm": 2.8281703448708675, + "language_loss": 0.77995503, + "learning_rate": 3.997783467136172e-06, + "loss": 0.80185795, + "num_input_tokens_seen": 42715200, + "router_z_loss_clip": 0.96582031, + "router_z_loss_mlp": 0.24084473, + "step": 1535, + "time_per_iteration": 2.4264121055603027 + }, + { + "auxiliary_loss_clip": 0.01135768, + "auxiliary_loss_mlp": 0.01052556, + "balance_loss_clip": 1.03727555, + "balance_loss_mlp": 1.02405334, + "epoch": 0.04457083164064767, + "flos": 44593338775680.0, + "grad_norm": 3.426578794305907, + "language_loss": 0.94364673, + "learning_rate": 3.99777461151375e-06, + "loss": 0.96552998, + "num_input_tokens_seen": 42734245, + "router_z_loss_clip": 0.98583984, + "router_z_loss_mlp": 0.28503418, + "step": 1536, + "time_per_iteration": 2.5304431915283203 + }, + { + "auxiliary_loss_clip": 0.0114292, + "auxiliary_loss_mlp": 0.01056797, + "balance_loss_clip": 1.0414989, + "balance_loss_mlp": 1.03178668, + "epoch": 0.04459984910916372, + "flos": 26351848460160.0, + "grad_norm": 2.455761274944113, + "language_loss": 0.92297345, + "learning_rate": 3.997765738246173e-06, + "loss": 0.94497073, + "num_input_tokens_seen": 42750425, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.24987793, + "step": 1537, + "time_per_iteration": 2.5128464698791504 + }, + { + "auxiliary_loss_clip": 0.0102216, + "auxiliary_loss_mlp": 0.01013067, + "balance_loss_clip": 1.00266898, + "balance_loss_mlp": 1.01114762, + "epoch": 0.04462886657767976, + "flos": 57843399144960.0, + "grad_norm": 0.687749291215957, + "language_loss": 0.47434735, + "learning_rate": 3.997756847333521e-06, + "loss": 0.49469963, + "num_input_tokens_seen": 42815870, + "router_z_loss_clip": 0.1953125, + "router_z_loss_mlp": 0.01916504, + "step": 1538, + "time_per_iteration": 3.094231605529785 + }, + { + "auxiliary_loss_clip": 0.01147924, + "auxiliary_loss_mlp": 0.01055506, + "balance_loss_clip": 1.04325938, + "balance_loss_mlp": 1.02887487, + "epoch": 0.04465788404619581, + "flos": 31787161966080.0, + "grad_norm": 2.799599744558474, + "language_loss": 1.18560553, + "learning_rate": 3.997747938775872e-06, + "loss": 1.20763993, + "num_input_tokens_seen": 42833515, + "router_z_loss_clip": 1.046875, + "router_z_loss_mlp": 0.26623535, + "step": 1539, + "time_per_iteration": 2.584771156311035 + }, + { + "auxiliary_loss_clip": 0.01139015, + "auxiliary_loss_mlp": 0.01045109, + "balance_loss_clip": 1.03803694, + "balance_loss_mlp": 1.01857328, + "epoch": 0.04468690151471186, + "flos": 32263202620800.0, + "grad_norm": 2.120830060758496, + "language_loss": 0.76338422, + "learning_rate": 3.997739012573305e-06, + "loss": 0.78522545, + "num_input_tokens_seen": 42852355, + "router_z_loss_clip": 1.00927734, + "router_z_loss_mlp": 0.26513672, + "step": 1540, + "time_per_iteration": 2.4978322982788086 + }, + { + "auxiliary_loss_clip": 0.01145087, + "auxiliary_loss_mlp": 0.01051049, + "balance_loss_clip": 1.04195809, + "balance_loss_mlp": 1.0236187, + "epoch": 0.0447159189832279, + "flos": 31423262198400.0, + "grad_norm": 3.0609343085124054, + "language_loss": 0.76455665, + "learning_rate": 3.997730068725898e-06, + "loss": 0.78651804, + "num_input_tokens_seen": 42867305, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.2746582, + "step": 1541, + "time_per_iteration": 2.843729257583618 + }, + { + "auxiliary_loss_clip": 0.01022309, + "auxiliary_loss_mlp": 0.01005243, + "balance_loss_clip": 1.0028882, + "balance_loss_mlp": 1.00339568, + "epoch": 0.04474493645174395, + "flos": 49591677438720.0, + "grad_norm": 0.698589583358036, + "language_loss": 0.51622164, + "learning_rate": 3.997721107233731e-06, + "loss": 0.53649724, + "num_input_tokens_seen": 42925860, + "router_z_loss_clip": 0.19433594, + "router_z_loss_mlp": 0.01843262, + "step": 1542, + "time_per_iteration": 3.0207905769348145 + }, + { + "auxiliary_loss_clip": 0.01023041, + "auxiliary_loss_mlp": 0.0100208, + "balance_loss_clip": 1.00374103, + "balance_loss_mlp": 1.00031614, + "epoch": 0.04477395392026, + "flos": 73359778199040.0, + "grad_norm": 0.6610334820978906, + "language_loss": 0.51210213, + "learning_rate": 3.9977121280968834e-06, + "loss": 0.53235334, + "num_input_tokens_seen": 42995855, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.0177002, + "step": 1543, + "time_per_iteration": 3.249654531478882 + }, + { + "auxiliary_loss_clip": 0.01021949, + "auxiliary_loss_mlp": 0.01001534, + "balance_loss_clip": 1.00261736, + "balance_loss_mlp": 0.99968606, + "epoch": 0.04480297138877604, + "flos": 57038162480640.0, + "grad_norm": 0.7154335545683684, + "language_loss": 0.51313818, + "learning_rate": 3.997703131315434e-06, + "loss": 0.533373, + "num_input_tokens_seen": 43051915, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.01843262, + "step": 1544, + "time_per_iteration": 2.879417896270752 + }, + { + "auxiliary_loss_clip": 0.01021625, + "auxiliary_loss_mlp": 0.0100263, + "balance_loss_clip": 1.00257623, + "balance_loss_mlp": 1.00086582, + "epoch": 0.04483198885729209, + "flos": 74768454535680.0, + "grad_norm": 0.6712932507526785, + "language_loss": 0.48234731, + "learning_rate": 3.997694116889461e-06, + "loss": 0.50258988, + "num_input_tokens_seen": 43114030, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.0177002, + "step": 1545, + "time_per_iteration": 3.0773346424102783 + }, + { + "auxiliary_loss_clip": 0.01020397, + "auxiliary_loss_mlp": 0.01003998, + "balance_loss_clip": 1.00156808, + "balance_loss_mlp": 1.00219834, + "epoch": 0.04486100632580814, + "flos": 63313416408960.0, + "grad_norm": 0.677311353398688, + "language_loss": 0.49824432, + "learning_rate": 3.997685084819046e-06, + "loss": 0.51848823, + "num_input_tokens_seen": 43175515, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.01794434, + "step": 1546, + "time_per_iteration": 3.169982671737671 + }, + { + "auxiliary_loss_clip": 0.01128549, + "auxiliary_loss_mlp": 0.01045332, + "balance_loss_clip": 1.03929424, + "balance_loss_mlp": 1.02407718, + "epoch": 0.044890023794324184, + "flos": 12560387771520.0, + "grad_norm": 2.481938824146989, + "language_loss": 0.68614334, + "learning_rate": 3.9976760351042675e-06, + "loss": 0.70788217, + "num_input_tokens_seen": 43187810, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.21252441, + "step": 1547, + "time_per_iteration": 2.3644840717315674 + }, + { + "auxiliary_loss_clip": 0.01021046, + "auxiliary_loss_mlp": 0.01003516, + "balance_loss_clip": 1.00221992, + "balance_loss_mlp": 1.00162077, + "epoch": 0.04491904126284023, + "flos": 74773027923840.0, + "grad_norm": 0.726316953896756, + "language_loss": 0.57022011, + "learning_rate": 3.997666967745206e-06, + "loss": 0.59046566, + "num_input_tokens_seen": 43255190, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.0189209, + "step": 1548, + "time_per_iteration": 3.0953872203826904 + }, + { + "auxiliary_loss_clip": 0.01132987, + "auxiliary_loss_mlp": 0.01043668, + "balance_loss_clip": 1.03731322, + "balance_loss_mlp": 1.02029073, + "epoch": 0.04494805873135627, + "flos": 31022459256960.0, + "grad_norm": 2.1346528408977683, + "language_loss": 0.8429296, + "learning_rate": 3.997657882741942e-06, + "loss": 0.86469615, + "num_input_tokens_seen": 43271420, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.23400879, + "step": 1549, + "time_per_iteration": 2.4862542152404785 + }, + { + "auxiliary_loss_clip": 0.01146827, + "auxiliary_loss_mlp": 0.01055288, + "balance_loss_clip": 1.041399, + "balance_loss_mlp": 1.02734542, + "epoch": 0.044977076199872325, + "flos": 24783410275200.0, + "grad_norm": 3.6994600849837727, + "language_loss": 1.01447976, + "learning_rate": 3.997648780094554e-06, + "loss": 1.03650093, + "num_input_tokens_seen": 43285405, + "router_z_loss_clip": 1.05371094, + "router_z_loss_mlp": 0.27941895, + "step": 1550, + "time_per_iteration": 2.512491464614868 + }, + { + "auxiliary_loss_clip": 0.01147061, + "auxiliary_loss_mlp": 0.01058636, + "balance_loss_clip": 1.04294479, + "balance_loss_mlp": 1.03153384, + "epoch": 0.04500609366838837, + "flos": 74735428389120.0, + "grad_norm": 4.953040801471976, + "language_loss": 0.65148395, + "learning_rate": 3.997639659803124e-06, + "loss": 0.67354089, + "num_input_tokens_seen": 43311145, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.27093506, + "step": 1551, + "time_per_iteration": 2.8844246864318848 + }, + { + "auxiliary_loss_clip": 0.01144699, + "auxiliary_loss_mlp": 0.01053867, + "balance_loss_clip": 1.04155505, + "balance_loss_mlp": 1.02711701, + "epoch": 0.045035111136904414, + "flos": 31604147285760.0, + "grad_norm": 2.808395891303946, + "language_loss": 0.92657101, + "learning_rate": 3.9976305218677324e-06, + "loss": 0.94855666, + "num_input_tokens_seen": 43326720, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.26733398, + "step": 1552, + "time_per_iteration": 2.4922308921813965 + }, + { + "auxiliary_loss_clip": 0.01030657, + "auxiliary_loss_mlp": 0.01013219, + "balance_loss_clip": 1.0112884, + "balance_loss_mlp": 1.01139486, + "epoch": 0.045064128605420466, + "flos": 63422624741760.0, + "grad_norm": 0.7393742795542035, + "language_loss": 0.52643865, + "learning_rate": 3.997621366288461e-06, + "loss": 0.54687738, + "num_input_tokens_seen": 43388330, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.01818848, + "step": 1553, + "time_per_iteration": 3.2787764072418213 + }, + { + "auxiliary_loss_clip": 0.01151845, + "auxiliary_loss_mlp": 0.01060159, + "balance_loss_clip": 1.04137063, + "balance_loss_mlp": 1.02971292, + "epoch": 0.04509314607393651, + "flos": 32300350174080.0, + "grad_norm": 2.1592089905805207, + "language_loss": 0.90191603, + "learning_rate": 3.997612193065388e-06, + "loss": 0.92403603, + "num_input_tokens_seen": 43410230, + "router_z_loss_clip": 1.10449219, + "router_z_loss_mlp": 0.3046875, + "step": 1554, + "time_per_iteration": 2.544456958770752 + }, + { + "auxiliary_loss_clip": 0.01142813, + "auxiliary_loss_mlp": 0.01055091, + "balance_loss_clip": 1.04158568, + "balance_loss_mlp": 1.02776861, + "epoch": 0.045122163542452555, + "flos": 14349616594560.0, + "grad_norm": 3.503983992624012, + "language_loss": 0.83022791, + "learning_rate": 3.9976030021985955e-06, + "loss": 0.85220695, + "num_input_tokens_seen": 43422210, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.27319336, + "step": 1555, + "time_per_iteration": 2.3905229568481445 + }, + { + "auxiliary_loss_clip": 0.01152903, + "auxiliary_loss_mlp": 0.01053524, + "balance_loss_clip": 1.04349971, + "balance_loss_mlp": 1.02642787, + "epoch": 0.04515118101096861, + "flos": 20368049495040.0, + "grad_norm": 6.926698261287042, + "language_loss": 0.70162886, + "learning_rate": 3.9975937936881655e-06, + "loss": 0.72369313, + "num_input_tokens_seen": 43435700, + "router_z_loss_clip": 1.09375, + "router_z_loss_mlp": 0.27087402, + "step": 1556, + "time_per_iteration": 2.4402823448181152 + }, + { + "auxiliary_loss_clip": 0.01026876, + "auxiliary_loss_mlp": 0.01010616, + "balance_loss_clip": 1.00759625, + "balance_loss_mlp": 1.00876868, + "epoch": 0.04518019847948465, + "flos": 74774284732800.0, + "grad_norm": 0.7339574198561936, + "language_loss": 0.47964531, + "learning_rate": 3.997584567534178e-06, + "loss": 0.50002021, + "num_input_tokens_seen": 43495935, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.01843262, + "step": 1557, + "time_per_iteration": 3.149887800216675 + }, + { + "auxiliary_loss_clip": 0.01140414, + "auxiliary_loss_mlp": 0.01066681, + "balance_loss_clip": 1.03762734, + "balance_loss_mlp": 1.0417062, + "epoch": 0.045209215948000696, + "flos": 29194686426240.0, + "grad_norm": 10.611344063112623, + "language_loss": 0.90761316, + "learning_rate": 3.997575323736717e-06, + "loss": 0.92968416, + "num_input_tokens_seen": 43512380, + "router_z_loss_clip": 1.02685547, + "router_z_loss_mlp": 0.24987793, + "step": 1558, + "time_per_iteration": 2.531628370285034 + }, + { + "auxiliary_loss_clip": 0.01150291, + "auxiliary_loss_mlp": 0.01066257, + "balance_loss_clip": 1.04068029, + "balance_loss_mlp": 1.034464, + "epoch": 0.04523823341651674, + "flos": 30114123747840.0, + "grad_norm": 2.395066543066777, + "language_loss": 0.82751393, + "learning_rate": 3.9975660622958605e-06, + "loss": 0.84967935, + "num_input_tokens_seen": 43528600, + "router_z_loss_clip": 1.09667969, + "router_z_loss_mlp": 0.31787109, + "step": 1559, + "time_per_iteration": 2.511592149734497 + }, + { + "auxiliary_loss_clip": 0.01148421, + "auxiliary_loss_mlp": 0.01047823, + "balance_loss_clip": 1.04232454, + "balance_loss_mlp": 1.02096558, + "epoch": 0.04526725088503279, + "flos": 10851249340800.0, + "grad_norm": 2.9382333528953435, + "language_loss": 0.94062138, + "learning_rate": 3.997556783211693e-06, + "loss": 0.96258384, + "num_input_tokens_seen": 43539370, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.26867676, + "step": 1560, + "time_per_iteration": 2.371797800064087 + }, + { + "auxiliary_loss_clip": 0.01136956, + "auxiliary_loss_mlp": 0.01055273, + "balance_loss_clip": 1.03808904, + "balance_loss_mlp": 1.02976227, + "epoch": 0.04529626835354884, + "flos": 10444197265920.0, + "grad_norm": 17.71836179120815, + "language_loss": 0.84234303, + "learning_rate": 3.997547486484296e-06, + "loss": 0.86426532, + "num_input_tokens_seen": 43550890, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.25512695, + "step": 1561, + "time_per_iteration": 2.404651641845703 + }, + { + "auxiliary_loss_clip": 0.01155143, + "auxiliary_loss_mlp": 0.01069293, + "balance_loss_clip": 1.04312706, + "balance_loss_mlp": 1.03771472, + "epoch": 0.04532528582206488, + "flos": 12012041957760.0, + "grad_norm": 3.1361637839335628, + "language_loss": 0.92207724, + "learning_rate": 3.997538172113751e-06, + "loss": 0.94432151, + "num_input_tokens_seen": 43561195, + "router_z_loss_clip": 1.12060547, + "router_z_loss_mlp": 0.3157959, + "step": 1562, + "time_per_iteration": 2.392444610595703 + }, + { + "auxiliary_loss_clip": 0.010207, + "auxiliary_loss_mlp": 0.01009332, + "balance_loss_clip": 1.00153065, + "balance_loss_mlp": 1.00747228, + "epoch": 0.04535430329058093, + "flos": 71486793290880.0, + "grad_norm": 0.7046530457570068, + "language_loss": 0.47816235, + "learning_rate": 3.99752884010014e-06, + "loss": 0.49846268, + "num_input_tokens_seen": 43624190, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.01855469, + "step": 1563, + "time_per_iteration": 3.037433624267578 + }, + { + "auxiliary_loss_clip": 0.01162019, + "auxiliary_loss_mlp": 0.01061643, + "balance_loss_clip": 1.04747939, + "balance_loss_mlp": 1.0308156, + "epoch": 0.04538332075909698, + "flos": 23032166878080.0, + "grad_norm": 4.393274344793944, + "language_loss": 0.72557092, + "learning_rate": 3.997519490443547e-06, + "loss": 0.7478075, + "num_input_tokens_seen": 43639145, + "router_z_loss_clip": 1.14257812, + "router_z_loss_mlp": 0.30847168, + "step": 1564, + "time_per_iteration": 2.378267765045166 + }, + { + "auxiliary_loss_clip": 0.01021672, + "auxiliary_loss_mlp": 0.01006043, + "balance_loss_clip": 1.00233817, + "balance_loss_mlp": 1.00423098, + "epoch": 0.04541233822761302, + "flos": 74778613741440.0, + "grad_norm": 0.6703404565431598, + "language_loss": 0.54036808, + "learning_rate": 3.997510123144053e-06, + "loss": 0.56064522, + "num_input_tokens_seen": 43708675, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.01806641, + "step": 1565, + "time_per_iteration": 3.1433894634246826 + }, + { + "auxiliary_loss_clip": 0.01152759, + "auxiliary_loss_mlp": 0.01061921, + "balance_loss_clip": 1.04670727, + "balance_loss_mlp": 1.03366828, + "epoch": 0.04544135569612907, + "flos": 23325751434240.0, + "grad_norm": 2.3750423853934124, + "language_loss": 0.98568869, + "learning_rate": 3.9975007382017406e-06, + "loss": 1.00783539, + "num_input_tokens_seen": 43727065, + "router_z_loss_clip": 1.06005859, + "router_z_loss_mlp": 0.28234863, + "step": 1566, + "time_per_iteration": 2.598203420639038 + }, + { + "auxiliary_loss_clip": 0.01023334, + "auxiliary_loss_mlp": 0.01001887, + "balance_loss_clip": 1.00406027, + "balance_loss_mlp": 0.99996811, + "epoch": 0.04547037316464512, + "flos": 70903534250880.0, + "grad_norm": 0.6132155259415601, + "language_loss": 0.49841648, + "learning_rate": 3.997491335616694e-06, + "loss": 0.51866871, + "num_input_tokens_seen": 43794795, + "router_z_loss_clip": 0.19238281, + "router_z_loss_mlp": 0.01916504, + "step": 1567, + "time_per_iteration": 3.330345630645752 + }, + { + "auxiliary_loss_clip": 0.01022953, + "auxiliary_loss_mlp": 0.01004819, + "balance_loss_clip": 1.00395453, + "balance_loss_mlp": 1.00305521, + "epoch": 0.04549939063316116, + "flos": 74765137956480.0, + "grad_norm": 0.7182449242605461, + "language_loss": 0.53422248, + "learning_rate": 3.997481915388996e-06, + "loss": 0.55450022, + "num_input_tokens_seen": 43854295, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.0177002, + "step": 1568, + "time_per_iteration": 3.070234775543213 + }, + { + "auxiliary_loss_clip": 0.01022191, + "auxiliary_loss_mlp": 0.01005438, + "balance_loss_clip": 1.00307119, + "balance_loss_mlp": 1.00353086, + "epoch": 0.04552840810167721, + "flos": 62894457694080.0, + "grad_norm": 0.7078573489913738, + "language_loss": 0.50519896, + "learning_rate": 3.997472477518729e-06, + "loss": 0.52547526, + "num_input_tokens_seen": 43920915, + "router_z_loss_clip": 0.19140625, + "router_z_loss_mlp": 0.01904297, + "step": 1569, + "time_per_iteration": 3.0878074169158936 + }, + { + "auxiliary_loss_clip": 0.0116006, + "auxiliary_loss_mlp": 0.0105951, + "balance_loss_clip": 1.04840732, + "balance_loss_mlp": 1.03144789, + "epoch": 0.04555742557019326, + "flos": 22593553067520.0, + "grad_norm": 2.636171931360623, + "language_loss": 0.74913818, + "learning_rate": 3.997463022005977e-06, + "loss": 0.77133387, + "num_input_tokens_seen": 43934475, + "router_z_loss_clip": 1.11669922, + "router_z_loss_mlp": 0.28039551, + "step": 1570, + "time_per_iteration": 2.4087746143341064 + }, + { + "auxiliary_loss_clip": 0.0102063, + "auxiliary_loss_mlp": 0.01008092, + "balance_loss_clip": 1.00187731, + "balance_loss_mlp": 1.00632751, + "epoch": 0.045586443038709304, + "flos": 62002916415360.0, + "grad_norm": 0.6593585075754386, + "language_loss": 0.49458817, + "learning_rate": 3.997453548850823e-06, + "loss": 0.51487535, + "num_input_tokens_seen": 43999015, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.0177002, + "step": 1571, + "time_per_iteration": 3.0341506004333496 + }, + { + "auxiliary_loss_clip": 0.01151925, + "auxiliary_loss_mlp": 0.01062145, + "balance_loss_clip": 1.04245937, + "balance_loss_mlp": 1.03308153, + "epoch": 0.04561546050722535, + "flos": 26606155870080.0, + "grad_norm": 5.456134929447543, + "language_loss": 0.84526378, + "learning_rate": 3.997444058053352e-06, + "loss": 0.86740446, + "num_input_tokens_seen": 44013305, + "router_z_loss_clip": 1.09570312, + "router_z_loss_mlp": 0.29064941, + "step": 1572, + "time_per_iteration": 2.445913314819336 + }, + { + "auxiliary_loss_clip": 0.01154059, + "auxiliary_loss_mlp": 0.01062241, + "balance_loss_clip": 1.04466689, + "balance_loss_mlp": 1.0343703, + "epoch": 0.04564447797574139, + "flos": 74729109432960.0, + "grad_norm": 2.3467755078378936, + "language_loss": 0.80469251, + "learning_rate": 3.997434549613646e-06, + "loss": 0.82685554, + "num_input_tokens_seen": 44035210, + "router_z_loss_clip": 1.09228516, + "router_z_loss_mlp": 0.27868652, + "step": 1573, + "time_per_iteration": 2.81290864944458 + }, + { + "auxiliary_loss_clip": 0.01134146, + "auxiliary_loss_mlp": 0.01049799, + "balance_loss_clip": 1.03760767, + "balance_loss_mlp": 1.02494943, + "epoch": 0.045673495444257445, + "flos": 74736824843520.0, + "grad_norm": 1.8574098872653129, + "language_loss": 0.83432567, + "learning_rate": 3.997425023531789e-06, + "loss": 0.85616505, + "num_input_tokens_seen": 44067440, + "router_z_loss_clip": 0.96533203, + "router_z_loss_mlp": 0.24865723, + "step": 1574, + "time_per_iteration": 2.804487943649292 + }, + { + "auxiliary_loss_clip": 0.0114325, + "auxiliary_loss_mlp": 0.01060201, + "balance_loss_clip": 1.04442716, + "balance_loss_mlp": 1.03413045, + "epoch": 0.04570251291277349, + "flos": 9641055283200.0, + "grad_norm": 5.562002063246805, + "language_loss": 0.85822964, + "learning_rate": 3.997415479807867e-06, + "loss": 0.88026416, + "num_input_tokens_seen": 44078225, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.2611084, + "step": 1575, + "time_per_iteration": 2.4042458534240723 + }, + { + "auxiliary_loss_clip": 0.0103018, + "auxiliary_loss_mlp": 0.01005286, + "balance_loss_clip": 1.01124024, + "balance_loss_mlp": 1.00339043, + "epoch": 0.045731530381289534, + "flos": 72117568558080.0, + "grad_norm": 0.6879727077665088, + "language_loss": 0.50676179, + "learning_rate": 3.997405918441963e-06, + "loss": 0.52711642, + "num_input_tokens_seen": 44146605, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.0189209, + "step": 1576, + "time_per_iteration": 3.106411933898926 + }, + { + "auxiliary_loss_clip": 0.01027529, + "auxiliary_loss_mlp": 0.01003046, + "balance_loss_clip": 1.0087769, + "balance_loss_mlp": 1.00135362, + "epoch": 0.045760547849805586, + "flos": 68720624403840.0, + "grad_norm": 0.6627212812322854, + "language_loss": 0.50325227, + "learning_rate": 3.9973963394341616e-06, + "loss": 0.52355802, + "num_input_tokens_seen": 44211840, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.01696777, + "step": 1577, + "time_per_iteration": 3.0889205932617188 + }, + { + "auxiliary_loss_clip": 0.01025513, + "auxiliary_loss_mlp": 0.0100152, + "balance_loss_clip": 1.00677812, + "balance_loss_mlp": 0.99982727, + "epoch": 0.04578956531832163, + "flos": 67176591125760.0, + "grad_norm": 0.7106708333670149, + "language_loss": 0.51673198, + "learning_rate": 3.997386742784547e-06, + "loss": 0.53700233, + "num_input_tokens_seen": 44271135, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.01696777, + "step": 1578, + "time_per_iteration": 2.9440248012542725 + }, + { + "auxiliary_loss_clip": 0.01147104, + "auxiliary_loss_mlp": 0.01052574, + "balance_loss_clip": 1.04156148, + "balance_loss_mlp": 1.02564466, + "epoch": 0.045818582786837675, + "flos": 41457020987520.0, + "grad_norm": 1.9633005498543425, + "language_loss": 0.82132697, + "learning_rate": 3.997377128493205e-06, + "loss": 0.84332383, + "num_input_tokens_seen": 44296225, + "router_z_loss_clip": 1.05615234, + "router_z_loss_mlp": 0.26953125, + "step": 1579, + "time_per_iteration": 2.892308235168457 + }, + { + "auxiliary_loss_clip": 0.01142206, + "auxiliary_loss_mlp": 0.0106863, + "balance_loss_clip": 1.04037368, + "balance_loss_mlp": 1.03789771, + "epoch": 0.04584760025535372, + "flos": 37587352757760.0, + "grad_norm": 2.356540069995543, + "language_loss": 0.94811344, + "learning_rate": 3.99736749656022e-06, + "loss": 0.97022182, + "num_input_tokens_seen": 44313700, + "router_z_loss_clip": 1.01855469, + "router_z_loss_mlp": 0.30725098, + "step": 1580, + "time_per_iteration": 2.5606472492218018 + }, + { + "auxiliary_loss_clip": 0.01020249, + "auxiliary_loss_mlp": 0.01006644, + "balance_loss_clip": 1.00148916, + "balance_loss_mlp": 1.00487971, + "epoch": 0.04587661772386977, + "flos": 74791216742400.0, + "grad_norm": 0.7070067166435834, + "language_loss": 0.50232786, + "learning_rate": 3.997357846985677e-06, + "loss": 0.52259678, + "num_input_tokens_seen": 44382450, + "router_z_loss_clip": 0.1875, + "router_z_loss_mlp": 0.0177002, + "step": 1581, + "time_per_iteration": 3.250602960586548 + }, + { + "auxiliary_loss_clip": 0.01137594, + "auxiliary_loss_mlp": 0.010584, + "balance_loss_clip": 1.04037571, + "balance_loss_mlp": 1.03279352, + "epoch": 0.045905635192385816, + "flos": 20040843432960.0, + "grad_norm": 4.355454768715232, + "language_loss": 1.09432614, + "learning_rate": 3.997348179769661e-06, + "loss": 1.11628604, + "num_input_tokens_seen": 44398985, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.25622559, + "step": 1582, + "time_per_iteration": 2.5315537452697754 + }, + { + "auxiliary_loss_clip": 0.01023165, + "auxiliary_loss_mlp": 0.0100357, + "balance_loss_clip": 1.00433612, + "balance_loss_mlp": 1.0017817, + "epoch": 0.04593465266090186, + "flos": 71804782753920.0, + "grad_norm": 0.7395540888128848, + "language_loss": 0.49569097, + "learning_rate": 3.997338494912258e-06, + "loss": 0.51595831, + "num_input_tokens_seen": 44452490, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.01782227, + "step": 1583, + "time_per_iteration": 2.864212989807129 + }, + { + "auxiliary_loss_clip": 0.01154156, + "auxiliary_loss_mlp": 0.01054031, + "balance_loss_clip": 1.04478276, + "balance_loss_mlp": 1.02375197, + "epoch": 0.04596367012941791, + "flos": 21358500255360.0, + "grad_norm": 2.8027671604121553, + "language_loss": 0.87818766, + "learning_rate": 3.997328792413552e-06, + "loss": 0.90026957, + "num_input_tokens_seen": 44467060, + "router_z_loss_clip": 1.09277344, + "router_z_loss_mlp": 0.30285645, + "step": 1584, + "time_per_iteration": 2.495262384414673 + }, + { + "auxiliary_loss_clip": 0.01149316, + "auxiliary_loss_mlp": 0.01061608, + "balance_loss_clip": 1.04137111, + "balance_loss_mlp": 1.03334391, + "epoch": 0.04599268759793396, + "flos": 29454125806080.0, + "grad_norm": 2.3583679641439566, + "language_loss": 0.99390793, + "learning_rate": 3.997319072273631e-06, + "loss": 1.0160172, + "num_input_tokens_seen": 44483245, + "router_z_loss_clip": 1.07910156, + "router_z_loss_mlp": 0.28271484, + "step": 1585, + "time_per_iteration": 2.4654579162597656 + }, + { + "auxiliary_loss_clip": 0.01145261, + "auxiliary_loss_mlp": 0.01064354, + "balance_loss_clip": 1.04433608, + "balance_loss_mlp": 1.03698313, + "epoch": 0.04602170506645, + "flos": 31971957125760.0, + "grad_norm": 2.8075494812157435, + "language_loss": 0.8765555, + "learning_rate": 3.997309334492579e-06, + "loss": 0.89865166, + "num_input_tokens_seen": 44499005, + "router_z_loss_clip": 1.00976562, + "router_z_loss_mlp": 0.27380371, + "step": 1586, + "time_per_iteration": 2.5360107421875 + }, + { + "auxiliary_loss_clip": 0.01036157, + "auxiliary_loss_mlp": 0.01007729, + "balance_loss_clip": 1.01591933, + "balance_loss_mlp": 1.00584602, + "epoch": 0.04605072253496605, + "flos": 74772888278400.0, + "grad_norm": 0.7287047794645887, + "language_loss": 0.49290603, + "learning_rate": 3.997299579070483e-06, + "loss": 0.51334488, + "num_input_tokens_seen": 44562570, + "router_z_loss_clip": 0.20214844, + "router_z_loss_mlp": 0.01879883, + "step": 1587, + "time_per_iteration": 3.059096336364746 + }, + { + "auxiliary_loss_clip": 0.01031647, + "auxiliary_loss_mlp": 0.01003646, + "balance_loss_clip": 1.01194143, + "balance_loss_mlp": 1.0017978, + "epoch": 0.0460797400034821, + "flos": 74780080018560.0, + "grad_norm": 0.6507913522255229, + "language_loss": 0.52622008, + "learning_rate": 3.99728980600743e-06, + "loss": 0.54657304, + "num_input_tokens_seen": 44633675, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.01843262, + "step": 1588, + "time_per_iteration": 3.130197286605835 + }, + { + "auxiliary_loss_clip": 0.01151524, + "auxiliary_loss_mlp": 0.01063569, + "balance_loss_clip": 1.04121268, + "balance_loss_mlp": 1.03232431, + "epoch": 0.04610875747199814, + "flos": 27854335353600.0, + "grad_norm": 4.661377901989525, + "language_loss": 0.95055264, + "learning_rate": 3.997280015303504e-06, + "loss": 0.97270358, + "num_input_tokens_seen": 44652845, + "router_z_loss_clip": 1.10400391, + "router_z_loss_mlp": 0.31274414, + "step": 1589, + "time_per_iteration": 2.4274935722351074 + }, + { + "auxiliary_loss_clip": 0.01143738, + "auxiliary_loss_mlp": 0.01053324, + "balance_loss_clip": 1.03988945, + "balance_loss_mlp": 1.02571487, + "epoch": 0.04613777494051419, + "flos": 24016438327680.0, + "grad_norm": 2.626151768858525, + "language_loss": 0.79182482, + "learning_rate": 3.997270206958793e-06, + "loss": 0.81379533, + "num_input_tokens_seen": 44666580, + "router_z_loss_clip": 1.0390625, + "router_z_loss_mlp": 0.27624512, + "step": 1590, + "time_per_iteration": 2.3981895446777344 + }, + { + "auxiliary_loss_clip": 0.01160992, + "auxiliary_loss_mlp": 0.01070599, + "balance_loss_clip": 1.0441339, + "balance_loss_mlp": 1.03770947, + "epoch": 0.04616679240903024, + "flos": 13031436101760.0, + "grad_norm": 3.579477280538452, + "language_loss": 0.93650544, + "learning_rate": 3.997260380973384e-06, + "loss": 0.9588213, + "num_input_tokens_seen": 44676330, + "router_z_loss_clip": 1.16992188, + "router_z_loss_mlp": 0.32897949, + "step": 1591, + "time_per_iteration": 2.3705837726593018 + }, + { + "auxiliary_loss_clip": 0.01022236, + "auxiliary_loss_mlp": 0.01002326, + "balance_loss_clip": 1.00386977, + "balance_loss_mlp": 1.0006094, + "epoch": 0.04619580987754628, + "flos": 74764823754240.0, + "grad_norm": 0.658500391975631, + "language_loss": 0.525433, + "learning_rate": 3.9972505373473626e-06, + "loss": 0.54567856, + "num_input_tokens_seen": 44737345, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.01721191, + "step": 1592, + "time_per_iteration": 3.106032371520996 + }, + { + "auxiliary_loss_clip": 0.01023515, + "auxiliary_loss_mlp": 0.01003064, + "balance_loss_clip": 1.00487995, + "balance_loss_mlp": 1.0013119, + "epoch": 0.04622482734606233, + "flos": 62223183383040.0, + "grad_norm": 0.6985187858769593, + "language_loss": 0.48788187, + "learning_rate": 3.997240676080816e-06, + "loss": 0.50814766, + "num_input_tokens_seen": 44799230, + "router_z_loss_clip": 0.18652344, + "router_z_loss_mlp": 0.01757812, + "step": 1593, + "time_per_iteration": 3.032240629196167 + }, + { + "auxiliary_loss_clip": 0.01023148, + "auxiliary_loss_mlp": 0.01001968, + "balance_loss_clip": 1.00470901, + "balance_loss_mlp": 1.00014436, + "epoch": 0.04625384481457838, + "flos": 68766850776960.0, + "grad_norm": 0.755207391983236, + "language_loss": 0.56634045, + "learning_rate": 3.997230797173831e-06, + "loss": 0.5865916, + "num_input_tokens_seen": 44849465, + "router_z_loss_clip": 0.18457031, + "router_z_loss_mlp": 0.01818848, + "step": 1594, + "time_per_iteration": 2.8492259979248047 + }, + { + "auxiliary_loss_clip": 0.01147012, + "auxiliary_loss_mlp": 0.01063779, + "balance_loss_clip": 1.04226458, + "balance_loss_mlp": 1.03630161, + "epoch": 0.046282862283094424, + "flos": 12451458729600.0, + "grad_norm": 2.9133017434164032, + "language_loss": 0.92979437, + "learning_rate": 3.9972209006264965e-06, + "loss": 0.95190227, + "num_input_tokens_seen": 44860745, + "router_z_loss_clip": 1.04638672, + "router_z_loss_mlp": 0.27478027, + "step": 1595, + "time_per_iteration": 2.4474425315856934 + }, + { + "auxiliary_loss_clip": 0.01147137, + "auxiliary_loss_mlp": 0.01053663, + "balance_loss_clip": 1.04047096, + "balance_loss_mlp": 1.02632844, + "epoch": 0.04631187975161047, + "flos": 12379642329600.0, + "grad_norm": 3.787952880748748, + "language_loss": 0.90541929, + "learning_rate": 3.997210986438898e-06, + "loss": 0.92742717, + "num_input_tokens_seen": 44874910, + "router_z_loss_clip": 1.06689453, + "router_z_loss_mlp": 0.2734375, + "step": 1596, + "time_per_iteration": 2.398467779159546 + }, + { + "auxiliary_loss_clip": 0.01159438, + "auxiliary_loss_mlp": 0.01066712, + "balance_loss_clip": 1.0457015, + "balance_loss_mlp": 1.03667128, + "epoch": 0.04634089722012651, + "flos": 24567053379840.0, + "grad_norm": 2.953608034452614, + "language_loss": 0.96146452, + "learning_rate": 3.997201054611124e-06, + "loss": 0.98372602, + "num_input_tokens_seen": 44890885, + "router_z_loss_clip": 1.13818359, + "router_z_loss_mlp": 0.30041504, + "step": 1597, + "time_per_iteration": 4.985163450241089 + }, + { + "auxiliary_loss_clip": 0.01151087, + "auxiliary_loss_mlp": 0.01057167, + "balance_loss_clip": 1.04410553, + "balance_loss_mlp": 1.02881873, + "epoch": 0.046369914688642565, + "flos": 29232811497600.0, + "grad_norm": 2.577379505436593, + "language_loss": 1.04701102, + "learning_rate": 3.997191105143263e-06, + "loss": 1.06909359, + "num_input_tokens_seen": 44905015, + "router_z_loss_clip": 1.06982422, + "router_z_loss_mlp": 0.2833252, + "step": 1598, + "time_per_iteration": 2.502250909805298 + }, + { + "auxiliary_loss_clip": 0.01147747, + "auxiliary_loss_mlp": 0.01059911, + "balance_loss_clip": 1.04168749, + "balance_loss_mlp": 1.02997792, + "epoch": 0.04639893215715861, + "flos": 29932784812800.0, + "grad_norm": 2.2680821777867197, + "language_loss": 0.99524915, + "learning_rate": 3.997181138035401e-06, + "loss": 1.01732576, + "num_input_tokens_seen": 44927515, + "router_z_loss_clip": 1.0625, + "router_z_loss_mlp": 0.29931641, + "step": 1599, + "time_per_iteration": 2.483307361602783 + }, + { + "auxiliary_loss_clip": 0.01145643, + "auxiliary_loss_mlp": 0.01059585, + "balance_loss_clip": 1.04144216, + "balance_loss_mlp": 1.02973461, + "epoch": 0.046427949625674654, + "flos": 29750852384640.0, + "grad_norm": 2.3510183494548444, + "language_loss": 0.88014776, + "learning_rate": 3.997171153287627e-06, + "loss": 0.90219998, + "num_input_tokens_seen": 44946205, + "router_z_loss_clip": 1.04394531, + "router_z_loss_mlp": 0.29846191, + "step": 1600, + "time_per_iteration": 2.609923839569092 + }, + { + "auxiliary_loss_clip": 0.01147626, + "auxiliary_loss_mlp": 0.0105439, + "balance_loss_clip": 1.04480052, + "balance_loss_mlp": 1.0251478, + "epoch": 0.046456967094190706, + "flos": 47366734314240.0, + "grad_norm": 2.5615874231611557, + "language_loss": 0.82420051, + "learning_rate": 3.9971611509000305e-06, + "loss": 0.84622067, + "num_input_tokens_seen": 44964950, + "router_z_loss_clip": 1.02734375, + "router_z_loss_mlp": 0.29223633, + "step": 1601, + "time_per_iteration": 2.668189764022827 + }, + { + "auxiliary_loss_clip": 0.01151629, + "auxiliary_loss_mlp": 0.01062266, + "balance_loss_clip": 1.04238462, + "balance_loss_mlp": 1.03154564, + "epoch": 0.04648598456270675, + "flos": 14567893614720.0, + "grad_norm": 3.7137581172870178, + "language_loss": 0.74406803, + "learning_rate": 3.997151130872697e-06, + "loss": 0.76620698, + "num_input_tokens_seen": 44977325, + "router_z_loss_clip": 1.09277344, + "router_z_loss_mlp": 0.30688477, + "step": 1602, + "time_per_iteration": 2.421712875366211 + }, + { + "auxiliary_loss_clip": 0.01150912, + "auxiliary_loss_mlp": 0.01077735, + "balance_loss_clip": 1.04346919, + "balance_loss_mlp": 1.04794502, + "epoch": 0.046515002031222795, + "flos": 12159689564160.0, + "grad_norm": 5.274353656436939, + "language_loss": 0.85817903, + "learning_rate": 3.997141093205717e-06, + "loss": 0.88046551, + "num_input_tokens_seen": 44990715, + "router_z_loss_clip": 1.07373047, + "router_z_loss_mlp": 0.29797363, + "step": 1603, + "time_per_iteration": 4.7749412059783936 + }, + { + "auxiliary_loss_clip": 0.01140499, + "auxiliary_loss_mlp": 0.01051717, + "balance_loss_clip": 1.04073417, + "balance_loss_mlp": 1.02568197, + "epoch": 0.04654401949973884, + "flos": 26205911510400.0, + "grad_norm": 2.3971312021644535, + "language_loss": 0.77603006, + "learning_rate": 3.997131037899179e-06, + "loss": 0.79795229, + "num_input_tokens_seen": 45006980, + "router_z_loss_clip": 0.99853516, + "router_z_loss_mlp": 0.26037598, + "step": 1604, + "time_per_iteration": 2.494150161743164 + }, + { + "auxiliary_loss_clip": 0.01029796, + "auxiliary_loss_mlp": 0.01023909, + "balance_loss_clip": 1.01012778, + "balance_loss_mlp": 1.02210879, + "epoch": 0.04657303696825489, + "flos": 67155921734400.0, + "grad_norm": 0.7636615694260824, + "language_loss": 0.52654248, + "learning_rate": 3.997120964953171e-06, + "loss": 0.5470795, + "num_input_tokens_seen": 45068160, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.01794434, + "step": 1605, + "time_per_iteration": 5.714563846588135 + }, + { + "auxiliary_loss_clip": 0.01027351, + "auxiliary_loss_mlp": 0.01018805, + "balance_loss_clip": 1.00809956, + "balance_loss_mlp": 1.01690936, + "epoch": 0.046602054436770936, + "flos": 60430603069440.0, + "grad_norm": 0.7846887505398792, + "language_loss": 0.52196109, + "learning_rate": 3.997110874367784e-06, + "loss": 0.54242265, + "num_input_tokens_seen": 45128660, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.0189209, + "step": 1606, + "time_per_iteration": 5.452507972717285 + }, + { + "auxiliary_loss_clip": 0.01149701, + "auxiliary_loss_mlp": 0.01064994, + "balance_loss_clip": 1.04163444, + "balance_loss_mlp": 1.03677702, + "epoch": 0.04663107190528698, + "flos": 16102186623360.0, + "grad_norm": 2.754878136983746, + "language_loss": 0.7985332, + "learning_rate": 3.997100766143104e-06, + "loss": 0.82068014, + "num_input_tokens_seen": 45142810, + "router_z_loss_clip": 1.08105469, + "router_z_loss_mlp": 0.28198242, + "step": 1607, + "time_per_iteration": 2.4917681217193604 + }, + { + "auxiliary_loss_clip": 0.01145803, + "auxiliary_loss_mlp": 0.01050721, + "balance_loss_clip": 1.04135942, + "balance_loss_mlp": 1.02381504, + "epoch": 0.04666008937380303, + "flos": 16358274512640.0, + "grad_norm": 2.7764399560023714, + "language_loss": 0.91236138, + "learning_rate": 3.997090640279222e-06, + "loss": 0.93432665, + "num_input_tokens_seen": 45157025, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.26928711, + "step": 1608, + "time_per_iteration": 2.347825765609741 + }, + { + "auxiliary_loss_clip": 0.01153329, + "auxiliary_loss_mlp": 0.01061211, + "balance_loss_clip": 1.04354692, + "balance_loss_mlp": 1.03293455, + "epoch": 0.04668910684231908, + "flos": 52847783568000.0, + "grad_norm": 2.072581738163816, + "language_loss": 0.89715809, + "learning_rate": 3.9970804967762276e-06, + "loss": 0.91930354, + "num_input_tokens_seen": 45185775, + "router_z_loss_clip": 1.09912109, + "router_z_loss_mlp": 0.28259277, + "step": 1609, + "time_per_iteration": 2.7259061336517334 + }, + { + "auxiliary_loss_clip": 0.01151882, + "auxiliary_loss_mlp": 0.01052877, + "balance_loss_clip": 1.04177654, + "balance_loss_mlp": 1.02253842, + "epoch": 0.04671812431083512, + "flos": 27190566984960.0, + "grad_norm": 2.623524595705323, + "language_loss": 1.03498983, + "learning_rate": 3.997070335634211e-06, + "loss": 1.05703747, + "num_input_tokens_seen": 45198860, + "router_z_loss_clip": 1.10107422, + "router_z_loss_mlp": 0.30371094, + "step": 1610, + "time_per_iteration": 2.4426050186157227 + }, + { + "auxiliary_loss_clip": 0.01148583, + "auxiliary_loss_mlp": 0.01053337, + "balance_loss_clip": 1.04207003, + "balance_loss_mlp": 1.02470326, + "epoch": 0.046747141779351166, + "flos": 18762289200000.0, + "grad_norm": 2.602829804070021, + "language_loss": 0.72767353, + "learning_rate": 3.99706015685326e-06, + "loss": 0.74969274, + "num_input_tokens_seen": 45212515, + "router_z_loss_clip": 1.06494141, + "router_z_loss_mlp": 0.28649902, + "step": 1611, + "time_per_iteration": 2.4888055324554443 + }, + { + "auxiliary_loss_clip": 0.0115151, + "auxiliary_loss_mlp": 0.01061001, + "balance_loss_clip": 1.04485893, + "balance_loss_mlp": 1.03016126, + "epoch": 0.04677615924786722, + "flos": 23691187301760.0, + "grad_norm": 2.108012679650007, + "language_loss": 0.76557791, + "learning_rate": 3.997049960433466e-06, + "loss": 0.78770304, + "num_input_tokens_seen": 45230125, + "router_z_loss_clip": 1.06787109, + "router_z_loss_mlp": 0.30834961, + "step": 1612, + "time_per_iteration": 2.4275481700897217 + }, + { + "auxiliary_loss_clip": 0.01157219, + "auxiliary_loss_mlp": 0.01055692, + "balance_loss_clip": 1.04885507, + "balance_loss_mlp": 1.02780867, + "epoch": 0.04680517671638326, + "flos": 26388716722560.0, + "grad_norm": 2.689057181779247, + "language_loss": 0.98466831, + "learning_rate": 3.997039746374918e-06, + "loss": 1.00679743, + "num_input_tokens_seen": 45244655, + "router_z_loss_clip": 1.08300781, + "router_z_loss_mlp": 0.27844238, + "step": 1613, + "time_per_iteration": 2.5611236095428467 + }, + { + "auxiliary_loss_clip": 0.01148039, + "auxiliary_loss_mlp": 0.01054686, + "balance_loss_clip": 1.04417634, + "balance_loss_mlp": 1.02733898, + "epoch": 0.04683419418489931, + "flos": 41273552459520.0, + "grad_norm": 2.8425469738950877, + "language_loss": 0.72084963, + "learning_rate": 3.997029514677708e-06, + "loss": 0.74287689, + "num_input_tokens_seen": 45263760, + "router_z_loss_clip": 1.03857422, + "router_z_loss_mlp": 0.2734375, + "step": 1614, + "time_per_iteration": 2.845703601837158 + }, + { + "auxiliary_loss_clip": 0.01141131, + "auxiliary_loss_mlp": 0.01048373, + "balance_loss_clip": 1.04209399, + "balance_loss_mlp": 1.02343404, + "epoch": 0.04686321165341536, + "flos": 15888029143680.0, + "grad_norm": 2.4854581511762888, + "language_loss": 0.68917608, + "learning_rate": 3.997019265341924e-06, + "loss": 0.71107113, + "num_input_tokens_seen": 45283035, + "router_z_loss_clip": 0.99072266, + "router_z_loss_mlp": 0.24938965, + "step": 1615, + "time_per_iteration": 2.5806171894073486 + }, + { + "auxiliary_loss_clip": 0.01148008, + "auxiliary_loss_mlp": 0.01048096, + "balance_loss_clip": 1.04484367, + "balance_loss_mlp": 1.02312124, + "epoch": 0.0468922291219314, + "flos": 32051454024960.0, + "grad_norm": 2.8778948756585097, + "language_loss": 0.98649585, + "learning_rate": 3.997008998367658e-06, + "loss": 1.00845683, + "num_input_tokens_seen": 45302530, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.24963379, + "step": 1616, + "time_per_iteration": 2.491119623184204 + }, + { + "auxiliary_loss_clip": 0.01034151, + "auxiliary_loss_mlp": 0.0103398, + "balance_loss_clip": 1.01345539, + "balance_loss_mlp": 1.03172672, + "epoch": 0.04692124659044745, + "flos": 74781546295680.0, + "grad_norm": 0.6788564744313346, + "language_loss": 0.46084476, + "learning_rate": 3.996998713755001e-06, + "loss": 0.48152602, + "num_input_tokens_seen": 45364585, + "router_z_loss_clip": 0.20703125, + "router_z_loss_mlp": 0.02258301, + "step": 1617, + "time_per_iteration": 3.1860313415527344 + }, + { + "auxiliary_loss_clip": 0.01033396, + "auxiliary_loss_mlp": 0.01022783, + "balance_loss_clip": 1.01286507, + "balance_loss_mlp": 1.02064908, + "epoch": 0.0469502640589635, + "flos": 65654028334080.0, + "grad_norm": 0.6441064964888098, + "language_loss": 0.51794839, + "learning_rate": 3.9969884115040435e-06, + "loss": 0.53851014, + "num_input_tokens_seen": 45429370, + "router_z_loss_clip": 0.20507812, + "router_z_loss_mlp": 0.0213623, + "step": 1618, + "time_per_iteration": 3.065910816192627 + }, + { + "auxiliary_loss_clip": 0.01142726, + "auxiliary_loss_mlp": 0.01064135, + "balance_loss_clip": 1.04144955, + "balance_loss_mlp": 1.03616834, + "epoch": 0.046979281527479544, + "flos": 25586133321600.0, + "grad_norm": 2.695559073682704, + "language_loss": 0.93003881, + "learning_rate": 3.996978091614875e-06, + "loss": 0.95210749, + "num_input_tokens_seen": 45446955, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.27954102, + "step": 1619, + "time_per_iteration": 2.448448657989502 + }, + { + "auxiliary_loss_clip": 0.01027894, + "auxiliary_loss_mlp": 0.0100159, + "balance_loss_clip": 1.00855041, + "balance_loss_mlp": 0.99982601, + "epoch": 0.04700829899599559, + "flos": 61661745809280.0, + "grad_norm": 0.731370311737159, + "language_loss": 0.56072342, + "learning_rate": 3.996967754087589e-06, + "loss": 0.58101821, + "num_input_tokens_seen": 45497800, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.0177002, + "step": 1620, + "time_per_iteration": 2.856804609298706 + }, + { + "auxiliary_loss_clip": 0.01142571, + "auxiliary_loss_mlp": 0.01062368, + "balance_loss_clip": 1.04127359, + "balance_loss_mlp": 1.03671384, + "epoch": 0.04703731646451163, + "flos": 33575448182400.0, + "grad_norm": 3.6753520781272284, + "language_loss": 0.94705087, + "learning_rate": 3.996957398922275e-06, + "loss": 0.96910024, + "num_input_tokens_seen": 45516155, + "router_z_loss_clip": 1.01269531, + "router_z_loss_mlp": 0.25646973, + "step": 1621, + "time_per_iteration": 2.513867139816284 + }, + { + "auxiliary_loss_clip": 0.01146925, + "auxiliary_loss_mlp": 0.0105721, + "balance_loss_clip": 1.04419315, + "balance_loss_mlp": 1.03206849, + "epoch": 0.047066333933027685, + "flos": 11282322297600.0, + "grad_norm": 3.0753970585869177, + "language_loss": 0.91523236, + "learning_rate": 3.996947026119026e-06, + "loss": 0.93727362, + "num_input_tokens_seen": 45528870, + "router_z_loss_clip": 1.02832031, + "router_z_loss_mlp": 0.2512207, + "step": 1622, + "time_per_iteration": 2.4957642555236816 + }, + { + "auxiliary_loss_clip": 0.010256, + "auxiliary_loss_mlp": 0.01021669, + "balance_loss_clip": 1.00674725, + "balance_loss_mlp": 1.01967859, + "epoch": 0.04709535140154373, + "flos": 74776693616640.0, + "grad_norm": 0.7330582539766592, + "language_loss": 0.49358183, + "learning_rate": 3.996936635677932e-06, + "loss": 0.51405448, + "num_input_tokens_seen": 45589075, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.01989746, + "step": 1623, + "time_per_iteration": 3.0615200996398926 + }, + { + "auxiliary_loss_clip": 0.01024682, + "auxiliary_loss_mlp": 0.01024022, + "balance_loss_clip": 1.00587034, + "balance_loss_mlp": 1.02213812, + "epoch": 0.047124368870059774, + "flos": 60319823725440.0, + "grad_norm": 0.7012311971647832, + "language_loss": 0.46996957, + "learning_rate": 3.996926227599085e-06, + "loss": 0.49045658, + "num_input_tokens_seen": 45645060, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.01879883, + "step": 1624, + "time_per_iteration": 2.9639782905578613 + }, + { + "auxiliary_loss_clip": 0.011344, + "auxiliary_loss_mlp": 0.01067583, + "balance_loss_clip": 1.03983021, + "balance_loss_mlp": 1.04259634, + "epoch": 0.047153386338575826, + "flos": 17230230518400.0, + "grad_norm": 3.485334356360042, + "language_loss": 0.76652712, + "learning_rate": 3.996915801882579e-06, + "loss": 0.78854692, + "num_input_tokens_seen": 45656125, + "router_z_loss_clip": 0.94580078, + "router_z_loss_mlp": 0.24975586, + "step": 1625, + "time_per_iteration": 2.3532018661499023 + }, + { + "auxiliary_loss_clip": 0.01141512, + "auxiliary_loss_mlp": 0.01069179, + "balance_loss_clip": 1.04017687, + "balance_loss_mlp": 1.04471707, + "epoch": 0.04718240380709187, + "flos": 11756373004800.0, + "grad_norm": 3.18957744241125, + "language_loss": 1.0035516, + "learning_rate": 3.996905358528504e-06, + "loss": 1.02565849, + "num_input_tokens_seen": 45666565, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.2442627, + "step": 1626, + "time_per_iteration": 2.442126989364624 + }, + { + "auxiliary_loss_clip": 0.01026692, + "auxiliary_loss_mlp": 0.01012605, + "balance_loss_clip": 1.00788498, + "balance_loss_mlp": 1.01056695, + "epoch": 0.047211421275607915, + "flos": 70263156493440.0, + "grad_norm": 0.7115371984952165, + "language_loss": 0.55196726, + "learning_rate": 3.996894897536953e-06, + "loss": 0.57236016, + "num_input_tokens_seen": 45735045, + "router_z_loss_clip": 0.18847656, + "router_z_loss_mlp": 0.02038574, + "step": 1627, + "time_per_iteration": 3.3369364738464355 + }, + { + "auxiliary_loss_clip": 0.01139354, + "auxiliary_loss_mlp": 0.0105883, + "balance_loss_clip": 1.03901803, + "balance_loss_mlp": 1.0339992, + "epoch": 0.04724043874412396, + "flos": 37369634319360.0, + "grad_norm": 3.037224172554929, + "language_loss": 0.93551755, + "learning_rate": 3.9968844189080174e-06, + "loss": 0.95749938, + "num_input_tokens_seen": 45750460, + "router_z_loss_clip": 1.00341797, + "router_z_loss_mlp": 0.24829102, + "step": 1628, + "time_per_iteration": 2.5497820377349854 + }, + { + "auxiliary_loss_clip": 0.01144338, + "auxiliary_loss_mlp": 0.0105145, + "balance_loss_clip": 1.04316449, + "balance_loss_mlp": 1.02605844, + "epoch": 0.04726945621264001, + "flos": 13917147183360.0, + "grad_norm": 3.0283505865143425, + "language_loss": 1.04044509, + "learning_rate": 3.996873922641791e-06, + "loss": 1.06240296, + "num_input_tokens_seen": 45763715, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.25366211, + "step": 1629, + "time_per_iteration": 2.4254860877990723 + }, + { + "auxiliary_loss_clip": 0.01128652, + "auxiliary_loss_mlp": 0.01045199, + "balance_loss_clip": 1.03915024, + "balance_loss_mlp": 1.02500463, + "epoch": 0.047298473681156056, + "flos": 10552009144320.0, + "grad_norm": 3.6310238208513805, + "language_loss": 0.97611284, + "learning_rate": 3.996863408738366e-06, + "loss": 0.99785137, + "num_input_tokens_seen": 45774190, + "router_z_loss_clip": 0.89404297, + "router_z_loss_mlp": 0.20202637, + "step": 1630, + "time_per_iteration": 2.348541498184204 + }, + { + "auxiliary_loss_clip": 0.01152354, + "auxiliary_loss_mlp": 0.01053396, + "balance_loss_clip": 1.04925609, + "balance_loss_mlp": 1.02722967, + "epoch": 0.0473274911496721, + "flos": 26098239277440.0, + "grad_norm": 1.853441339434317, + "language_loss": 0.82033122, + "learning_rate": 3.996852877197835e-06, + "loss": 0.84238875, + "num_input_tokens_seen": 45795905, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.26171875, + "step": 1631, + "time_per_iteration": 2.6489932537078857 + }, + { + "auxiliary_loss_clip": 0.01134314, + "auxiliary_loss_mlp": 0.0103924, + "balance_loss_clip": 1.04302001, + "balance_loss_mlp": 1.01726985, + "epoch": 0.04735650861818815, + "flos": 11026897724160.0, + "grad_norm": 3.7193747786280635, + "language_loss": 0.79072022, + "learning_rate": 3.996842328020292e-06, + "loss": 0.81245577, + "num_input_tokens_seen": 45809505, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.21972656, + "step": 1632, + "time_per_iteration": 2.3884482383728027 + }, + { + "auxiliary_loss_clip": 0.01047541, + "auxiliary_loss_mlp": 0.0101043, + "balance_loss_clip": 1.02749634, + "balance_loss_mlp": 1.00836742, + "epoch": 0.0473855260867042, + "flos": 59697322450560.0, + "grad_norm": 0.703245409759948, + "language_loss": 0.54176468, + "learning_rate": 3.99683176120583e-06, + "loss": 0.56234437, + "num_input_tokens_seen": 45866390, + "router_z_loss_clip": 0.20117188, + "router_z_loss_mlp": 0.02062988, + "step": 1633, + "time_per_iteration": 2.916517734527588 + }, + { + "auxiliary_loss_clip": 0.01158149, + "auxiliary_loss_mlp": 0.01056987, + "balance_loss_clip": 1.04881644, + "balance_loss_mlp": 1.02854371, + "epoch": 0.04741454355522024, + "flos": 22667499060480.0, + "grad_norm": 2.3844073003412714, + "language_loss": 0.99842477, + "learning_rate": 3.996821176754541e-06, + "loss": 1.02057612, + "num_input_tokens_seen": 45881810, + "router_z_loss_clip": 1.09326172, + "router_z_loss_mlp": 0.28466797, + "step": 1634, + "time_per_iteration": 2.4160871505737305 + }, + { + "auxiliary_loss_clip": 0.01038071, + "auxiliary_loss_mlp": 0.01013294, + "balance_loss_clip": 1.01874542, + "balance_loss_mlp": 1.01135123, + "epoch": 0.047443561023736286, + "flos": 74791216742400.0, + "grad_norm": 0.5926811563584969, + "language_loss": 0.50137269, + "learning_rate": 3.996810574666519e-06, + "loss": 0.52188635, + "num_input_tokens_seen": 45955650, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.01940918, + "step": 1635, + "time_per_iteration": 3.325268030166626 + }, + { + "auxiliary_loss_clip": 0.01138734, + "auxiliary_loss_mlp": 0.01052334, + "balance_loss_clip": 1.04339695, + "balance_loss_mlp": 1.02927876, + "epoch": 0.04747257849225234, + "flos": 43607985073920.0, + "grad_norm": 4.716153787532842, + "language_loss": 0.88964349, + "learning_rate": 3.996799954941859e-06, + "loss": 0.91155416, + "num_input_tokens_seen": 45973715, + "router_z_loss_clip": 0.953125, + "router_z_loss_mlp": 0.23022461, + "step": 1636, + "time_per_iteration": 2.518639087677002 + }, + { + "auxiliary_loss_clip": 0.01133017, + "auxiliary_loss_mlp": 0.01048091, + "balance_loss_clip": 1.03837872, + "balance_loss_mlp": 1.02343822, + "epoch": 0.04750159596076838, + "flos": 16321895009280.0, + "grad_norm": 2.4569520835962626, + "language_loss": 0.6972478, + "learning_rate": 3.9967893175806535e-06, + "loss": 0.71905887, + "num_input_tokens_seen": 45988600, + "router_z_loss_clip": 0.9453125, + "router_z_loss_mlp": 0.24658203, + "step": 1637, + "time_per_iteration": 2.4381487369537354 + }, + { + "auxiliary_loss_clip": 0.01142272, + "auxiliary_loss_mlp": 0.0104682, + "balance_loss_clip": 1.03998005, + "balance_loss_mlp": 1.02178597, + "epoch": 0.04753061342928443, + "flos": 30844646369280.0, + "grad_norm": 1.8546085547644562, + "language_loss": 0.85710466, + "learning_rate": 3.996778662582997e-06, + "loss": 0.87899566, + "num_input_tokens_seen": 46009030, + "router_z_loss_clip": 1.02148438, + "router_z_loss_mlp": 0.25048828, + "step": 1638, + "time_per_iteration": 2.5199108123779297 + }, + { + "auxiliary_loss_clip": 0.01023409, + "auxiliary_loss_mlp": 0.01004693, + "balance_loss_clip": 1.0043515, + "balance_loss_mlp": 1.00286889, + "epoch": 0.04755963089780048, + "flos": 74756305382400.0, + "grad_norm": 0.6567890581213015, + "language_loss": 0.4548285, + "learning_rate": 3.996767989948982e-06, + "loss": 0.47510952, + "num_input_tokens_seen": 46066725, + "router_z_loss_clip": 0.19042969, + "router_z_loss_mlp": 0.01818848, + "step": 1639, + "time_per_iteration": 3.0379881858825684 + }, + { + "auxiliary_loss_clip": 0.01023162, + "auxiliary_loss_mlp": 0.01002418, + "balance_loss_clip": 1.00418246, + "balance_loss_mlp": 1.00062954, + "epoch": 0.04758864836631652, + "flos": 73636884593280.0, + "grad_norm": 0.6552582493310104, + "language_loss": 0.50596923, + "learning_rate": 3.996757299678705e-06, + "loss": 0.52622503, + "num_input_tokens_seen": 46123730, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.01782227, + "step": 1640, + "time_per_iteration": 3.1388697624206543 + }, + { + "auxiliary_loss_clip": 0.01145809, + "auxiliary_loss_mlp": 0.01048991, + "balance_loss_clip": 1.04015088, + "balance_loss_mlp": 1.02388608, + "epoch": 0.04761766583483257, + "flos": 22702482109440.0, + "grad_norm": 2.016990750505263, + "language_loss": 0.80290425, + "learning_rate": 3.99674659177226e-06, + "loss": 0.82485235, + "num_input_tokens_seen": 46146960, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.25073242, + "step": 1641, + "time_per_iteration": 2.545466899871826 + }, + { + "auxiliary_loss_clip": 0.01027103, + "auxiliary_loss_mlp": 0.0100473, + "balance_loss_clip": 1.00719285, + "balance_loss_mlp": 1.0029062, + "epoch": 0.04764668330334862, + "flos": 60943442163840.0, + "grad_norm": 0.6932265960484447, + "language_loss": 0.50290203, + "learning_rate": 3.99673586622974e-06, + "loss": 0.5232203, + "num_input_tokens_seen": 46204605, + "router_z_loss_clip": 0.19921875, + "router_z_loss_mlp": 0.01818848, + "step": 1642, + "time_per_iteration": 2.8952770233154297 + }, + { + "auxiliary_loss_clip": 0.01029105, + "auxiliary_loss_mlp": 0.01002647, + "balance_loss_clip": 1.00944734, + "balance_loss_mlp": 1.00082314, + "epoch": 0.047675700771864664, + "flos": 67628680721280.0, + "grad_norm": 0.6807853869799756, + "language_loss": 0.50955677, + "learning_rate": 3.996725123051242e-06, + "loss": 0.52987427, + "num_input_tokens_seen": 46267920, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.01818848, + "step": 1643, + "time_per_iteration": 3.061972141265869 + }, + { + "auxiliary_loss_clip": 0.01029854, + "auxiliary_loss_mlp": 0.01003281, + "balance_loss_clip": 1.01012206, + "balance_loss_mlp": 1.0014807, + "epoch": 0.04770471824038071, + "flos": 74774703669120.0, + "grad_norm": 0.6216902436423629, + "language_loss": 0.53737605, + "learning_rate": 3.996714362236859e-06, + "loss": 0.55770743, + "num_input_tokens_seen": 46335070, + "router_z_loss_clip": 0.19726562, + "router_z_loss_mlp": 0.01794434, + "step": 1644, + "time_per_iteration": 3.0991289615631104 + }, + { + "auxiliary_loss_clip": 0.01141713, + "auxiliary_loss_mlp": 0.01049084, + "balance_loss_clip": 1.04743505, + "balance_loss_mlp": 1.0253967, + "epoch": 0.04773373570889675, + "flos": 11539178236800.0, + "grad_norm": 2.7239199404868537, + "language_loss": 0.7420854, + "learning_rate": 3.996703583786687e-06, + "loss": 0.76399338, + "num_input_tokens_seen": 46347560, + "router_z_loss_clip": 0.94189453, + "router_z_loss_mlp": 0.23657227, + "step": 1645, + "time_per_iteration": 2.413914918899536 + }, + { + "auxiliary_loss_clip": 0.01027681, + "auxiliary_loss_mlp": 0.0100597, + "balance_loss_clip": 1.00840127, + "balance_loss_mlp": 1.00418222, + "epoch": 0.047762753177412805, + "flos": 74781232093440.0, + "grad_norm": 0.6466000203520603, + "language_loss": 0.51336253, + "learning_rate": 3.996692787700821e-06, + "loss": 0.53369904, + "num_input_tokens_seen": 46415375, + "router_z_loss_clip": 0.19335938, + "router_z_loss_mlp": 0.01782227, + "step": 1646, + "time_per_iteration": 3.1654303073883057 + }, + { + "auxiliary_loss_clip": 0.01137406, + "auxiliary_loss_mlp": 0.01054454, + "balance_loss_clip": 1.03815937, + "balance_loss_mlp": 1.02735758, + "epoch": 0.04779177064592885, + "flos": 30618200090880.0, + "grad_norm": 2.6438389832985356, + "language_loss": 0.80340362, + "learning_rate": 3.996681973979356e-06, + "loss": 0.82532227, + "num_input_tokens_seen": 46430720, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.27099609, + "step": 1647, + "time_per_iteration": 2.423478364944458 + }, + { + "auxiliary_loss_clip": 0.0115793, + "auxiliary_loss_mlp": 0.01065032, + "balance_loss_clip": 1.04933393, + "balance_loss_mlp": 1.03766131, + "epoch": 0.047820788114444894, + "flos": 23213715281280.0, + "grad_norm": 2.661314329428724, + "language_loss": 0.80553472, + "learning_rate": 3.996671142622389e-06, + "loss": 0.82776433, + "num_input_tokens_seen": 46449005, + "router_z_loss_clip": 1.08447266, + "router_z_loss_mlp": 0.27355957, + "step": 1648, + "time_per_iteration": 2.5134122371673584 + }, + { + "auxiliary_loss_clip": 0.01145541, + "auxiliary_loss_mlp": 0.01065751, + "balance_loss_clip": 1.04184127, + "balance_loss_mlp": 1.03475666, + "epoch": 0.047849805582960946, + "flos": 32959335686400.0, + "grad_norm": 2.287452296749553, + "language_loss": 0.8720296, + "learning_rate": 3.996660293630013e-06, + "loss": 0.89414251, + "num_input_tokens_seen": 46466575, + "router_z_loss_clip": 1.03564453, + "router_z_loss_mlp": 0.30993652, + "step": 1649, + "time_per_iteration": 2.6255710124969482 + }, + { + "auxiliary_loss_clip": 0.01021087, + "auxiliary_loss_mlp": 0.01002387, + "balance_loss_clip": 1.00286055, + "balance_loss_mlp": 1.00064611, + "epoch": 0.04787882305147699, + "flos": 74773307214720.0, + "grad_norm": 0.657151971585364, + "language_loss": 0.4799208, + "learning_rate": 3.996649427002326e-06, + "loss": 0.50015557, + "num_input_tokens_seen": 46534210, + "router_z_loss_clip": 0.18261719, + "router_z_loss_mlp": 0.01745605, + "step": 1650, + "time_per_iteration": 3.0987401008605957 + }, + { + "auxiliary_loss_clip": 0.01139089, + "auxiliary_loss_mlp": 0.01051335, + "balance_loss_clip": 1.0430299, + "balance_loss_mlp": 1.02591968, + "epoch": 0.047907840519993035, + "flos": 45834047228160.0, + "grad_norm": 3.604559110177773, + "language_loss": 0.73490775, + "learning_rate": 3.996638542739423e-06, + "loss": 0.75681204, + "num_input_tokens_seen": 46551900, + "router_z_loss_clip": 0.96044922, + "router_z_loss_mlp": 0.25390625, + "step": 1651, + "time_per_iteration": 2.672273874282837 + }, + { + "auxiliary_loss_clip": 0.01024217, + "auxiliary_loss_mlp": 0.01002866, + "balance_loss_clip": 1.0055778, + "balance_loss_mlp": 1.00110197, + "epoch": 0.04793685798850908, + "flos": 70278030599040.0, + "grad_norm": 0.7422334958391371, + "language_loss": 0.54743063, + "learning_rate": 3.9966276408414005e-06, + "loss": 0.56770146, + "num_input_tokens_seen": 46611870, + "router_z_loss_clip": 0.18652344, + "router_z_loss_mlp": 0.0177002, + "step": 1652, + "time_per_iteration": 3.157158851623535 + }, + { + "auxiliary_loss_clip": 0.0114904, + "auxiliary_loss_mlp": 0.01058016, + "balance_loss_clip": 1.04379165, + "balance_loss_mlp": 1.02919149, + "epoch": 0.04796587545702513, + "flos": 26862523050240.0, + "grad_norm": 2.511584023497487, + "language_loss": 1.04072249, + "learning_rate": 3.996616721308355e-06, + "loss": 1.06279302, + "num_input_tokens_seen": 46629220, + "router_z_loss_clip": 1.05224609, + "router_z_loss_mlp": 0.28820801, + "step": 1653, + "time_per_iteration": 2.5803616046905518 + }, + { + "auxiliary_loss_clip": 0.01136767, + "auxiliary_loss_mlp": 0.01054615, + "balance_loss_clip": 1.03846955, + "balance_loss_mlp": 1.02624273, + "epoch": 0.047994892925541176, + "flos": 31132086526080.0, + "grad_norm": 3.0030939156951972, + "language_loss": 0.81491029, + "learning_rate": 3.996605784140383e-06, + "loss": 0.83682412, + "num_input_tokens_seen": 46643810, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.28381348, + "step": 1654, + "time_per_iteration": 2.4697375297546387 + }, + { + "auxiliary_loss_clip": 0.0102868, + "auxiliary_loss_mlp": 0.01001374, + "balance_loss_clip": 1.01009703, + "balance_loss_mlp": 0.99945503, + "epoch": 0.04802391039405722, + "flos": 63957669327360.0, + "grad_norm": 0.8379565826916433, + "language_loss": 0.49333656, + "learning_rate": 3.99659482933758e-06, + "loss": 0.51363707, + "num_input_tokens_seen": 46697025, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.01916504, + "step": 1655, + "time_per_iteration": 3.0021302700042725 + }, + { + "auxiliary_loss_clip": 0.01029732, + "auxiliary_loss_mlp": 0.01002499, + "balance_loss_clip": 1.01115191, + "balance_loss_mlp": 1.00074625, + "epoch": 0.04805292786257327, + "flos": 74784583584000.0, + "grad_norm": 0.7829952968888079, + "language_loss": 0.53405225, + "learning_rate": 3.9965838569000435e-06, + "loss": 0.55437458, + "num_input_tokens_seen": 46765570, + "router_z_loss_clip": 0.18554688, + "router_z_loss_mlp": 0.01757812, + "step": 1656, + "time_per_iteration": 3.1818809509277344 + }, + { + "auxiliary_loss_clip": 0.01141618, + "auxiliary_loss_mlp": 0.0105846, + "balance_loss_clip": 1.04293799, + "balance_loss_mlp": 1.03416491, + "epoch": 0.04808194533108932, + "flos": 18909064022400.0, + "grad_norm": 2.8087993735167727, + "language_loss": 0.71373647, + "learning_rate": 3.99657286682787e-06, + "loss": 0.7357372, + "num_input_tokens_seen": 46778620, + "router_z_loss_clip": 0.98681641, + "router_z_loss_mlp": 0.24304199, + "step": 1657, + "time_per_iteration": 2.3476898670196533 + }, + { + "auxiliary_loss_clip": 0.01159355, + "auxiliary_loss_mlp": 0.01065312, + "balance_loss_clip": 1.04622602, + "balance_loss_mlp": 1.03744054, + "epoch": 0.04811096279960536, + "flos": 31532226151680.0, + "grad_norm": 2.5589993411711727, + "language_loss": 0.9570058, + "learning_rate": 3.9965618591211585e-06, + "loss": 0.97925252, + "num_input_tokens_seen": 46796975, + "router_z_loss_clip": 1.13085938, + "router_z_loss_mlp": 0.27832031, + "step": 1658, + "time_per_iteration": 2.4994983673095703 + }, + { + "auxiliary_loss_clip": 0.01024519, + "auxiliary_loss_mlp": 0.01005906, + "balance_loss_clip": 1.00616336, + "balance_loss_mlp": 1.00408196, + "epoch": 0.048139980268121406, + "flos": 68572941886080.0, + "grad_norm": 0.6488227604480347, + "language_loss": 0.46586639, + "learning_rate": 3.996550833780004e-06, + "loss": 0.48617065, + "num_input_tokens_seen": 46863770, + "router_z_loss_clip": 0.18359375, + "router_z_loss_mlp": 0.01818848, + "step": 1659, + "time_per_iteration": 3.139000177383423 + }, + { + "auxiliary_loss_clip": 0.01142584, + "auxiliary_loss_mlp": 0.01053253, + "balance_loss_clip": 1.03943253, + "balance_loss_mlp": 1.02570391, + "epoch": 0.04816899773663746, + "flos": 13987322749440.0, + "grad_norm": 2.63741560418896, + "language_loss": 1.02153063, + "learning_rate": 3.996539790804505e-06, + "loss": 1.04348898, + "num_input_tokens_seen": 46876295, + "router_z_loss_clip": 1.03173828, + "router_z_loss_mlp": 0.27575684, + "step": 1660, + "time_per_iteration": 2.464866876602173 + }, + { + "auxiliary_loss_clip": 0.01142756, + "auxiliary_loss_mlp": 0.01062577, + "balance_loss_clip": 1.04056859, + "balance_loss_mlp": 1.03617179, + "epoch": 0.0481980152051535, + "flos": 11830493554560.0, + "grad_norm": 3.0736146540053393, + "language_loss": 0.90642166, + "learning_rate": 3.996528730194757e-06, + "loss": 0.92847496, + "num_input_tokens_seen": 46887810, + "router_z_loss_clip": 1.02246094, + "router_z_loss_mlp": 0.26403809, + "step": 1661, + "time_per_iteration": 2.3793928623199463 + }, + { + "auxiliary_loss_clip": 0.01142378, + "auxiliary_loss_mlp": 0.01059968, + "balance_loss_clip": 1.04103291, + "balance_loss_mlp": 1.03202558, + "epoch": 0.04822703267366955, + "flos": 24236810029440.0, + "grad_norm": 3.54357344550177, + "language_loss": 0.80474699, + "learning_rate": 3.996517651950861e-06, + "loss": 0.82677042, + "num_input_tokens_seen": 46903070, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.27966309, + "step": 1662, + "time_per_iteration": 2.5563488006591797 + }, + { + "auxiliary_loss_clip": 0.01145996, + "auxiliary_loss_mlp": 0.01061607, + "balance_loss_clip": 1.03963065, + "balance_loss_mlp": 1.03098249, + "epoch": 0.0482560501421856, + "flos": 16965238366080.0, + "grad_norm": 2.629816932822627, + "language_loss": 0.93918759, + "learning_rate": 3.996506556072913e-06, + "loss": 0.9612636, + "num_input_tokens_seen": 46922180, + "router_z_loss_clip": 1.06201172, + "router_z_loss_mlp": 0.30603027, + "step": 1663, + "time_per_iteration": 2.540196180343628 + }, + { + "auxiliary_loss_clip": 0.01153099, + "auxiliary_loss_mlp": 0.01065712, + "balance_loss_clip": 1.0493921, + "balance_loss_mlp": 1.0378412, + "epoch": 0.04828506761070164, + "flos": 51158476656000.0, + "grad_norm": 3.1713778872667233, + "language_loss": 0.96553117, + "learning_rate": 3.996495442561011e-06, + "loss": 0.9877193, + "num_input_tokens_seen": 46937330, + "router_z_loss_clip": 1.03759766, + "router_z_loss_mlp": 0.27844238, + "step": 1664, + "time_per_iteration": 2.7375965118408203 + }, + { + "auxiliary_loss_clip": 0.01129398, + "auxiliary_loss_mlp": 0.01061725, + "balance_loss_clip": 1.03875935, + "balance_loss_mlp": 1.03801405, + "epoch": 0.04831408507921769, + "flos": 33318766800000.0, + "grad_norm": 2.33774653917065, + "language_loss": 0.82183659, + "learning_rate": 3.996484311415254e-06, + "loss": 0.84374785, + "num_input_tokens_seen": 46957075, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.23706055, + "step": 1665, + "time_per_iteration": 2.6754846572875977 + }, + { + "auxiliary_loss_clip": 0.01143326, + "auxiliary_loss_mlp": 0.01046925, + "balance_loss_clip": 1.04327178, + "balance_loss_mlp": 1.01938808, + "epoch": 0.04834310254773373, + "flos": 37699004885760.0, + "grad_norm": 2.6986337591589473, + "language_loss": 0.72414392, + "learning_rate": 3.9964731626357385e-06, + "loss": 0.74604642, + "num_input_tokens_seen": 46975505, + "router_z_loss_clip": 1.00097656, + "router_z_loss_mlp": 0.2755127, + "step": 1666, + "time_per_iteration": 2.677769660949707 + }, + { + "auxiliary_loss_clip": 0.01149491, + "auxiliary_loss_mlp": 0.01063963, + "balance_loss_clip": 1.04217625, + "balance_loss_mlp": 1.03568649, + "epoch": 0.048372120016249784, + "flos": 32627346768000.0, + "grad_norm": 2.6117042301831357, + "language_loss": 1.04608965, + "learning_rate": 3.996461996222565e-06, + "loss": 1.06822419, + "num_input_tokens_seen": 46992010, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.28283691, + "step": 1667, + "time_per_iteration": 2.4680862426757812 + }, + { + "auxiliary_loss_clip": 0.01137066, + "auxiliary_loss_mlp": 0.01049649, + "balance_loss_clip": 1.04169941, + "balance_loss_mlp": 1.02645087, + "epoch": 0.04840113748476583, + "flos": 36311661256320.0, + "grad_norm": 2.881044717988082, + "language_loss": 0.91187912, + "learning_rate": 3.996450812175831e-06, + "loss": 0.93374634, + "num_input_tokens_seen": 47007570, + "router_z_loss_clip": 0.95410156, + "router_z_loss_mlp": 0.23217773, + "step": 1668, + "time_per_iteration": 2.5485758781433105 + }, + { + "auxiliary_loss_clip": 0.0114603, + "auxiliary_loss_mlp": 0.01053523, + "balance_loss_clip": 1.04394758, + "balance_loss_mlp": 1.02785707, + "epoch": 0.04843015495328187, + "flos": 36243266169600.0, + "grad_norm": 2.092649820083937, + "language_loss": 0.86627984, + "learning_rate": 3.9964396104956344e-06, + "loss": 0.88827538, + "num_input_tokens_seen": 47033600, + "router_z_loss_clip": 1.02099609, + "router_z_loss_mlp": 0.25671387, + "step": 1669, + "time_per_iteration": 2.598388910293579 + }, + { + "auxiliary_loss_clip": 0.0102449, + "auxiliary_loss_mlp": 0.01010048, + "balance_loss_clip": 1.00635219, + "balance_loss_mlp": 1.00833166, + "epoch": 0.048459172421797925, + "flos": 63309331779840.0, + "grad_norm": 0.7303271849168533, + "language_loss": 0.49558574, + "learning_rate": 3.996428391182077e-06, + "loss": 0.51593119, + "num_input_tokens_seen": 47091355, + "router_z_loss_clip": 0.18164062, + "router_z_loss_mlp": 0.01721191, + "step": 1670, + "time_per_iteration": 3.0253827571868896 + }, + { + "auxiliary_loss_clip": 0.01133108, + "auxiliary_loss_mlp": 0.01046353, + "balance_loss_clip": 1.03965998, + "balance_loss_mlp": 1.02342892, + "epoch": 0.04848818989031397, + "flos": 36877008902400.0, + "grad_norm": 2.5880526578563514, + "language_loss": 0.82351106, + "learning_rate": 3.9964171542352555e-06, + "loss": 0.84530568, + "num_input_tokens_seen": 47110510, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.22937012, + "step": 1671, + "time_per_iteration": 2.5669403076171875 + }, + { + "auxiliary_loss_clip": 0.01025003, + "auxiliary_loss_mlp": 0.01002116, + "balance_loss_clip": 1.00719333, + "balance_loss_mlp": 1.00036383, + "epoch": 0.048517207358830014, + "flos": 70060032869760.0, + "grad_norm": 0.7146051188608551, + "language_loss": 0.52154911, + "learning_rate": 3.9964058996552705e-06, + "loss": 0.54182029, + "num_input_tokens_seen": 47169785, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.01757812, + "step": 1672, + "time_per_iteration": 2.9591805934906006 + }, + { + "auxiliary_loss_clip": 0.0102203, + "auxiliary_loss_mlp": 0.01000701, + "balance_loss_clip": 1.00446045, + "balance_loss_mlp": 0.99893665, + "epoch": 0.048546224827346066, + "flos": 67572227185920.0, + "grad_norm": 0.7441820741728563, + "language_loss": 0.51838833, + "learning_rate": 3.9963946274422195e-06, + "loss": 0.53861564, + "num_input_tokens_seen": 47222165, + "router_z_loss_clip": 0.17578125, + "router_z_loss_mlp": 0.0177002, + "step": 1673, + "time_per_iteration": 2.8953957557678223 + }, + { + "auxiliary_loss_clip": 0.0102071, + "auxiliary_loss_mlp": 0.01001071, + "balance_loss_clip": 1.00315416, + "balance_loss_mlp": 0.99942547, + "epoch": 0.04857524229586211, + "flos": 72486251182080.0, + "grad_norm": 0.7321176112442745, + "language_loss": 0.49970061, + "learning_rate": 3.996383337596204e-06, + "loss": 0.51991844, + "num_input_tokens_seen": 47281300, + "router_z_loss_clip": 0.17578125, + "router_z_loss_mlp": 0.01647949, + "step": 1674, + "time_per_iteration": 5.518938302993774 + }, + { + "auxiliary_loss_clip": 0.01139021, + "auxiliary_loss_mlp": 0.01052238, + "balance_loss_clip": 1.04294181, + "balance_loss_mlp": 1.02626276, + "epoch": 0.048604259764378155, + "flos": 28614464674560.0, + "grad_norm": 3.041307416519769, + "language_loss": 0.85552889, + "learning_rate": 3.9963720301173225e-06, + "loss": 0.87744153, + "num_input_tokens_seen": 47299615, + "router_z_loss_clip": 0.95996094, + "router_z_loss_mlp": 0.25964355, + "step": 1675, + "time_per_iteration": 2.489943742752075 + }, + { + "auxiliary_loss_clip": 0.01019785, + "auxiliary_loss_mlp": 0.0100223, + "balance_loss_clip": 1.00247216, + "balance_loss_mlp": 1.00051379, + "epoch": 0.0486332772328942, + "flos": 52097185048320.0, + "grad_norm": 0.7126480473266744, + "language_loss": 0.5081827, + "learning_rate": 3.996360705005676e-06, + "loss": 0.52840286, + "num_input_tokens_seen": 47349885, + "router_z_loss_clip": 0.17382812, + "router_z_loss_mlp": 0.01721191, + "step": 1676, + "time_per_iteration": 2.8286728858947754 + }, + { + "auxiliary_loss_clip": 0.01018984, + "auxiliary_loss_mlp": 0.01002786, + "balance_loss_clip": 1.00170696, + "balance_loss_mlp": 1.00116479, + "epoch": 0.04866229470141025, + "flos": 74767651574400.0, + "grad_norm": 0.6538859015884672, + "language_loss": 0.54322815, + "learning_rate": 3.996349362261364e-06, + "loss": 0.56344581, + "num_input_tokens_seen": 47415390, + "router_z_loss_clip": 0.17285156, + "router_z_loss_mlp": 0.01623535, + "step": 1677, + "time_per_iteration": 3.213672399520874 + }, + { + "auxiliary_loss_clip": 0.01137047, + "auxiliary_loss_mlp": 0.01054318, + "balance_loss_clip": 1.04148519, + "balance_loss_mlp": 1.02809191, + "epoch": 0.048691312169926296, + "flos": 26026597434240.0, + "grad_norm": 2.149513258450314, + "language_loss": 0.80858535, + "learning_rate": 3.9963380018844865e-06, + "loss": 0.83049905, + "num_input_tokens_seen": 47432260, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.26245117, + "step": 1678, + "time_per_iteration": 2.4646294116973877 + }, + { + "auxiliary_loss_clip": 0.01139984, + "auxiliary_loss_mlp": 0.01057066, + "balance_loss_clip": 1.03965449, + "balance_loss_mlp": 1.0300051, + "epoch": 0.04872032963844234, + "flos": 13436253849600.0, + "grad_norm": 2.494533733754758, + "language_loss": 0.85707957, + "learning_rate": 3.996326623875143e-06, + "loss": 0.87905008, + "num_input_tokens_seen": 47443170, + "router_z_loss_clip": 1.00439453, + "router_z_loss_mlp": 0.27062988, + "step": 1679, + "time_per_iteration": 4.705786943435669 + }, + { + "auxiliary_loss_clip": 0.0102093, + "auxiliary_loss_mlp": 0.01002997, + "balance_loss_clip": 1.0032233, + "balance_loss_mlp": 1.00131595, + "epoch": 0.04874934710695839, + "flos": 72326384599680.0, + "grad_norm": 0.6182195918123665, + "language_loss": 0.54520255, + "learning_rate": 3.996315228233436e-06, + "loss": 0.56544185, + "num_input_tokens_seen": 47511555, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.0168457, + "step": 1680, + "time_per_iteration": 3.1601085662841797 + }, + { + "auxiliary_loss_clip": 0.01148904, + "auxiliary_loss_mlp": 0.01053746, + "balance_loss_clip": 1.04403377, + "balance_loss_mlp": 1.02625608, + "epoch": 0.04877836457547444, + "flos": 13728756153600.0, + "grad_norm": 2.918121207479106, + "language_loss": 0.96325481, + "learning_rate": 3.996303814959465e-06, + "loss": 0.98528135, + "num_input_tokens_seen": 47523650, + "router_z_loss_clip": 1.04931641, + "router_z_loss_mlp": 0.27502441, + "step": 1681, + "time_per_iteration": 2.4359934329986572 + }, + { + "auxiliary_loss_clip": 0.0102318, + "auxiliary_loss_mlp": 0.01001206, + "balance_loss_clip": 1.00533855, + "balance_loss_mlp": 0.99965608, + "epoch": 0.04880738204399048, + "flos": 74767581751680.0, + "grad_norm": 0.6635901720559814, + "language_loss": 0.4746626, + "learning_rate": 3.9962923840533305e-06, + "loss": 0.49490643, + "num_input_tokens_seen": 47585840, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.01544189, + "step": 1682, + "time_per_iteration": 5.542536497116089 + }, + { + "auxiliary_loss_clip": 0.01147381, + "auxiliary_loss_mlp": 0.01056764, + "balance_loss_clip": 1.04417491, + "balance_loss_mlp": 1.03156281, + "epoch": 0.048836399512506526, + "flos": 32304504625920.0, + "grad_norm": 2.1111330773640167, + "language_loss": 0.99337751, + "learning_rate": 3.996280935515134e-06, + "loss": 1.01541901, + "num_input_tokens_seen": 47603315, + "router_z_loss_clip": 1.03222656, + "router_z_loss_mlp": 0.25219727, + "step": 1683, + "time_per_iteration": 2.528834819793701 + }, + { + "auxiliary_loss_clip": 0.01134447, + "auxiliary_loss_mlp": 0.01055432, + "balance_loss_clip": 1.04105818, + "balance_loss_mlp": 1.03027928, + "epoch": 0.04886541698102258, + "flos": 15515436447360.0, + "grad_norm": 2.754335322640171, + "language_loss": 1.01105237, + "learning_rate": 3.9962694693449765e-06, + "loss": 1.03295112, + "num_input_tokens_seen": 47614980, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.25146484, + "step": 1684, + "time_per_iteration": 2.358793258666992 + }, + { + "auxiliary_loss_clip": 0.0115172, + "auxiliary_loss_mlp": 0.01068201, + "balance_loss_clip": 1.04701018, + "balance_loss_mlp": 1.03427446, + "epoch": 0.04889443444953862, + "flos": 13472074771200.0, + "grad_norm": 7.580263012221961, + "language_loss": 0.87317693, + "learning_rate": 3.996257985542959e-06, + "loss": 0.89537615, + "num_input_tokens_seen": 47628135, + "router_z_loss_clip": 1.04833984, + "router_z_loss_mlp": 0.33947754, + "step": 1685, + "time_per_iteration": 2.4430243968963623 + }, + { + "auxiliary_loss_clip": 0.01147233, + "auxiliary_loss_mlp": 0.01061025, + "balance_loss_clip": 1.04612982, + "balance_loss_mlp": 1.0352881, + "epoch": 0.04892345191805467, + "flos": 29864075523840.0, + "grad_norm": 2.6927370543064675, + "language_loss": 0.72735119, + "learning_rate": 3.996246484109184e-06, + "loss": 0.74943376, + "num_input_tokens_seen": 47644315, + "router_z_loss_clip": 1.01171875, + "router_z_loss_mlp": 0.25756836, + "step": 1686, + "time_per_iteration": 2.5415217876434326 + }, + { + "auxiliary_loss_clip": 0.01149884, + "auxiliary_loss_mlp": 0.01056287, + "balance_loss_clip": 1.04541874, + "balance_loss_mlp": 1.02805805, + "epoch": 0.04895246938657072, + "flos": 30219003072000.0, + "grad_norm": 2.768288945919998, + "language_loss": 0.924178, + "learning_rate": 3.9962349650437514e-06, + "loss": 0.94623965, + "num_input_tokens_seen": 47661035, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.28222656, + "step": 1687, + "time_per_iteration": 2.5160698890686035 + }, + { + "auxiliary_loss_clip": 0.01141851, + "auxiliary_loss_mlp": 0.01051575, + "balance_loss_clip": 1.04670978, + "balance_loss_mlp": 1.02899647, + "epoch": 0.04898148685508676, + "flos": 52401419435520.0, + "grad_norm": 1.9615651688128566, + "language_loss": 0.8236146, + "learning_rate": 3.996223428346764e-06, + "loss": 0.84554887, + "num_input_tokens_seen": 47682195, + "router_z_loss_clip": 0.95214844, + "router_z_loss_mlp": 0.22546387, + "step": 1688, + "time_per_iteration": 2.613492012023926 + }, + { + "auxiliary_loss_clip": 0.01023492, + "auxiliary_loss_mlp": 0.01006547, + "balance_loss_clip": 1.00618434, + "balance_loss_mlp": 1.00490177, + "epoch": 0.04901050432360281, + "flos": 67158819377280.0, + "grad_norm": 0.7027708739943764, + "language_loss": 0.50741374, + "learning_rate": 3.9962118740183235e-06, + "loss": 0.52771413, + "num_input_tokens_seen": 47747720, + "router_z_loss_clip": 0.17285156, + "router_z_loss_mlp": 0.01647949, + "step": 1689, + "time_per_iteration": 3.2025911808013916 + }, + { + "auxiliary_loss_clip": 0.01145214, + "auxiliary_loss_mlp": 0.01054999, + "balance_loss_clip": 1.04520559, + "balance_loss_mlp": 1.03113365, + "epoch": 0.04903952179211885, + "flos": 33139103610240.0, + "grad_norm": 2.4867078674731795, + "language_loss": 0.82403547, + "learning_rate": 3.996200302058532e-06, + "loss": 0.84603763, + "num_input_tokens_seen": 47768645, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.23852539, + "step": 1690, + "time_per_iteration": 2.5136303901672363 + }, + { + "auxiliary_loss_clip": 0.01144038, + "auxiliary_loss_mlp": 0.01055221, + "balance_loss_clip": 1.0426116, + "balance_loss_mlp": 1.03068733, + "epoch": 0.049068539260634904, + "flos": 16682862222720.0, + "grad_norm": 2.620128644065341, + "language_loss": 0.83120036, + "learning_rate": 3.9961887124674916e-06, + "loss": 0.85319293, + "num_input_tokens_seen": 47781785, + "router_z_loss_clip": 1.01318359, + "router_z_loss_mlp": 0.24523926, + "step": 1691, + "time_per_iteration": 2.4651100635528564 + }, + { + "auxiliary_loss_clip": 0.01145098, + "auxiliary_loss_mlp": 0.01053827, + "balance_loss_clip": 1.04533088, + "balance_loss_mlp": 1.02792311, + "epoch": 0.04909755672915095, + "flos": 25000744688640.0, + "grad_norm": 3.0236518791601688, + "language_loss": 0.72989011, + "learning_rate": 3.996177105245304e-06, + "loss": 0.75187933, + "num_input_tokens_seen": 47796315, + "router_z_loss_clip": 0.99804688, + "router_z_loss_mlp": 0.2590332, + "step": 1692, + "time_per_iteration": 2.509385585784912 + }, + { + "auxiliary_loss_clip": 0.01145502, + "auxiliary_loss_mlp": 0.01054122, + "balance_loss_clip": 1.04168248, + "balance_loss_mlp": 1.02737105, + "epoch": 0.04912657419766699, + "flos": 13327220073600.0, + "grad_norm": 3.5157210588888117, + "language_loss": 0.84278107, + "learning_rate": 3.996165480392074e-06, + "loss": 0.86477733, + "num_input_tokens_seen": 47808675, + "router_z_loss_clip": 1.03710938, + "router_z_loss_mlp": 0.26745605, + "step": 1693, + "time_per_iteration": 2.4298245906829834 + }, + { + "auxiliary_loss_clip": 0.011418, + "auxiliary_loss_mlp": 0.01061927, + "balance_loss_clip": 1.04018271, + "balance_loss_mlp": 1.03570104, + "epoch": 0.049155591666183045, + "flos": 10699028346240.0, + "grad_norm": 4.5562697729978785, + "language_loss": 0.81118035, + "learning_rate": 3.996153837907902e-06, + "loss": 0.83321756, + "num_input_tokens_seen": 47821645, + "router_z_loss_clip": 1.01464844, + "router_z_loss_mlp": 0.26220703, + "step": 1694, + "time_per_iteration": 2.34911847114563 + }, + { + "auxiliary_loss_clip": 0.0113889, + "auxiliary_loss_mlp": 0.01052721, + "balance_loss_clip": 1.04421532, + "balance_loss_mlp": 1.02568388, + "epoch": 0.04918460913469909, + "flos": 17704630339200.0, + "grad_norm": 3.589449422659764, + "language_loss": 0.99160081, + "learning_rate": 3.996142177792891e-06, + "loss": 1.0135169, + "num_input_tokens_seen": 47834530, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.27038574, + "step": 1695, + "time_per_iteration": 2.4168858528137207 + }, + { + "auxiliary_loss_clip": 0.01146153, + "auxiliary_loss_mlp": 0.0104583, + "balance_loss_clip": 1.04211259, + "balance_loss_mlp": 1.0199616, + "epoch": 0.049213626603215134, + "flos": 28869854336640.0, + "grad_norm": 2.332817681273199, + "language_loss": 0.92533374, + "learning_rate": 3.996130500047145e-06, + "loss": 0.94725347, + "num_input_tokens_seen": 47850805, + "router_z_loss_clip": 1.03857422, + "router_z_loss_mlp": 0.25842285, + "step": 1696, + "time_per_iteration": 2.442639112472534 + }, + { + "auxiliary_loss_clip": 0.01144312, + "auxiliary_loss_mlp": 0.01052308, + "balance_loss_clip": 1.04469657, + "balance_loss_mlp": 1.02447248, + "epoch": 0.049242644071731186, + "flos": 26462557981440.0, + "grad_norm": 2.722377655184809, + "language_loss": 0.74729019, + "learning_rate": 3.996118804670767e-06, + "loss": 0.76925635, + "num_input_tokens_seen": 47868295, + "router_z_loss_clip": 0.99707031, + "router_z_loss_mlp": 0.27807617, + "step": 1697, + "time_per_iteration": 2.5346319675445557 + }, + { + "auxiliary_loss_clip": 0.0101948, + "auxiliary_loss_mlp": 0.01001697, + "balance_loss_clip": 1.0019387, + "balance_loss_mlp": 0.99994481, + "epoch": 0.04927166154024723, + "flos": 48791642567040.0, + "grad_norm": 0.8020593822458081, + "language_loss": 0.51863635, + "learning_rate": 3.99610709166386e-06, + "loss": 0.53884816, + "num_input_tokens_seen": 47917785, + "router_z_loss_clip": 0.17578125, + "router_z_loss_mlp": 0.01757812, + "step": 1698, + "time_per_iteration": 2.7617580890655518 + }, + { + "auxiliary_loss_clip": 0.01020334, + "auxiliary_loss_mlp": 0.01001553, + "balance_loss_clip": 1.002684, + "balance_loss_mlp": 0.9997291, + "epoch": 0.049300679008763275, + "flos": 74768279978880.0, + "grad_norm": 0.7615437018728687, + "language_loss": 0.504269, + "learning_rate": 3.996095361026526e-06, + "loss": 0.52448785, + "num_input_tokens_seen": 47974730, + "router_z_loss_clip": 0.17578125, + "router_z_loss_mlp": 0.01818848, + "step": 1699, + "time_per_iteration": 3.0067973136901855 + }, + { + "auxiliary_loss_clip": 0.0113851, + "auxiliary_loss_mlp": 0.01046026, + "balance_loss_clip": 1.04164052, + "balance_loss_mlp": 1.02217209, + "epoch": 0.04932969647727932, + "flos": 28685094088320.0, + "grad_norm": 2.367364122174946, + "language_loss": 0.91679263, + "learning_rate": 3.996083612758871e-06, + "loss": 0.93863797, + "num_input_tokens_seen": 47991065, + "router_z_loss_clip": 0.96972656, + "router_z_loss_mlp": 0.23864746, + "step": 1700, + "time_per_iteration": 2.4911391735076904 + }, + { + "auxiliary_loss_clip": 0.01140513, + "auxiliary_loss_mlp": 0.01049037, + "balance_loss_clip": 1.04221749, + "balance_loss_mlp": 1.02481341, + "epoch": 0.04935871394579537, + "flos": 20951936939520.0, + "grad_norm": 3.013379819517853, + "language_loss": 0.73077512, + "learning_rate": 3.996071846860998e-06, + "loss": 0.75267065, + "num_input_tokens_seen": 48003495, + "router_z_loss_clip": 0.98291016, + "router_z_loss_mlp": 0.24206543, + "step": 1701, + "time_per_iteration": 2.362729072570801 + }, + { + "auxiliary_loss_clip": 0.01138702, + "auxiliary_loss_mlp": 0.01071982, + "balance_loss_clip": 1.04069424, + "balance_loss_mlp": 1.04594707, + "epoch": 0.049387731414311416, + "flos": 30731109027840.0, + "grad_norm": 3.0430035031476246, + "language_loss": 0.88374174, + "learning_rate": 3.996060063333011e-06, + "loss": 0.90584862, + "num_input_tokens_seen": 48023340, + "router_z_loss_clip": 0.97851562, + "router_z_loss_mlp": 0.26025391, + "step": 1702, + "time_per_iteration": 2.745818614959717 + }, + { + "auxiliary_loss_clip": 0.0101923, + "auxiliary_loss_mlp": 0.01002085, + "balance_loss_clip": 1.0015099, + "balance_loss_mlp": 1.00018954, + "epoch": 0.04941674888282746, + "flos": 66794395939200.0, + "grad_norm": 0.6877401303209815, + "language_loss": 0.55175787, + "learning_rate": 3.996048262175013e-06, + "loss": 0.571971, + "num_input_tokens_seen": 48089515, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.0189209, + "step": 1703, + "time_per_iteration": 3.120098114013672 + }, + { + "auxiliary_loss_clip": 0.01152707, + "auxiliary_loss_mlp": 0.01060528, + "balance_loss_clip": 1.04351139, + "balance_loss_mlp": 1.03109562, + "epoch": 0.04944576635134351, + "flos": 23177789625600.0, + "grad_norm": 3.267040568366122, + "language_loss": 1.00170898, + "learning_rate": 3.99603644338711e-06, + "loss": 1.02384138, + "num_input_tokens_seen": 48106025, + "router_z_loss_clip": 1.09179688, + "router_z_loss_mlp": 0.29443359, + "step": 1704, + "time_per_iteration": 2.418368339538574 + }, + { + "auxiliary_loss_clip": 0.01134355, + "auxiliary_loss_mlp": 0.01057644, + "balance_loss_clip": 1.04202533, + "balance_loss_mlp": 1.03383827, + "epoch": 0.04947478381985956, + "flos": 11319016003200.0, + "grad_norm": 3.15485957401378, + "language_loss": 0.82721287, + "learning_rate": 3.996024606969405e-06, + "loss": 0.8491329, + "num_input_tokens_seen": 48116120, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.23815918, + "step": 1705, + "time_per_iteration": 2.3953230381011963 + }, + { + "auxiliary_loss_clip": 0.01019105, + "auxiliary_loss_mlp": 0.01000816, + "balance_loss_clip": 1.00185013, + "balance_loss_mlp": 0.99906403, + "epoch": 0.0495038012883756, + "flos": 70793592779520.0, + "grad_norm": 0.6811661762961672, + "language_loss": 0.4968257, + "learning_rate": 3.996012752922002e-06, + "loss": 0.51702487, + "num_input_tokens_seen": 48173955, + "router_z_loss_clip": 0.17285156, + "router_z_loss_mlp": 0.01757812, + "step": 1706, + "time_per_iteration": 3.0079660415649414 + }, + { + "auxiliary_loss_clip": 0.01136809, + "auxiliary_loss_mlp": 0.01041154, + "balance_loss_clip": 1.03992295, + "balance_loss_mlp": 1.01759851, + "epoch": 0.049532818756891646, + "flos": 21351867096960.0, + "grad_norm": 2.7890780448306716, + "language_loss": 0.93117511, + "learning_rate": 3.996000881245008e-06, + "loss": 0.95295471, + "num_input_tokens_seen": 48187125, + "router_z_loss_clip": 0.96826172, + "router_z_loss_mlp": 0.2355957, + "step": 1707, + "time_per_iteration": 2.3655261993408203 + }, + { + "auxiliary_loss_clip": 0.01135829, + "auxiliary_loss_mlp": 0.01059848, + "balance_loss_clip": 1.04011536, + "balance_loss_mlp": 1.03394389, + "epoch": 0.0495618362254077, + "flos": 14566846273920.0, + "grad_norm": 3.33084184399351, + "language_loss": 0.96484566, + "learning_rate": 3.995988991938526e-06, + "loss": 0.98680246, + "num_input_tokens_seen": 48198350, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.25915527, + "step": 1708, + "time_per_iteration": 2.3471198081970215 + }, + { + "auxiliary_loss_clip": 0.01146534, + "auxiliary_loss_mlp": 0.01066697, + "balance_loss_clip": 1.04254198, + "balance_loss_mlp": 1.03732407, + "epoch": 0.04959085369392374, + "flos": 30218339756160.0, + "grad_norm": 2.2879810528306357, + "language_loss": 0.85565996, + "learning_rate": 3.9959770850026615e-06, + "loss": 0.87779224, + "num_input_tokens_seen": 48212790, + "router_z_loss_clip": 1.04003906, + "router_z_loss_mlp": 0.29382324, + "step": 1709, + "time_per_iteration": 2.500214099884033 + }, + { + "auxiliary_loss_clip": 0.01141034, + "auxiliary_loss_mlp": 0.01054391, + "balance_loss_clip": 1.04224658, + "balance_loss_mlp": 1.02898777, + "epoch": 0.04961987116243979, + "flos": 12932805911040.0, + "grad_norm": 2.515590200174524, + "language_loss": 0.81773072, + "learning_rate": 3.99596516043752e-06, + "loss": 0.83968496, + "num_input_tokens_seen": 48225655, + "router_z_loss_clip": 0.98876953, + "router_z_loss_mlp": 0.25402832, + "step": 1710, + "time_per_iteration": 2.3866517543792725 + }, + { + "auxiliary_loss_clip": 0.01149107, + "auxiliary_loss_mlp": 0.01066722, + "balance_loss_clip": 1.04314816, + "balance_loss_mlp": 1.03865981, + "epoch": 0.04964888863095584, + "flos": 31093193404800.0, + "grad_norm": 2.7982344560368704, + "language_loss": 1.07497239, + "learning_rate": 3.995953218243206e-06, + "loss": 1.09713066, + "num_input_tokens_seen": 48240665, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.28076172, + "step": 1711, + "time_per_iteration": 2.490082263946533 + }, + { + "auxiliary_loss_clip": 0.01149108, + "auxiliary_loss_mlp": 0.01059714, + "balance_loss_clip": 1.04336834, + "balance_loss_mlp": 1.03361905, + "epoch": 0.04967790609947188, + "flos": 16902535697280.0, + "grad_norm": 2.977706007805589, + "language_loss": 0.81514335, + "learning_rate": 3.995941258419826e-06, + "loss": 0.83723152, + "num_input_tokens_seen": 48254640, + "router_z_loss_clip": 1.05712891, + "router_z_loss_mlp": 0.26098633, + "step": 1712, + "time_per_iteration": 2.3435468673706055 + }, + { + "auxiliary_loss_clip": 0.0115292, + "auxiliary_loss_mlp": 0.01073011, + "balance_loss_clip": 1.04555953, + "balance_loss_mlp": 1.03947723, + "epoch": 0.04970692356798793, + "flos": 12160667082240.0, + "grad_norm": 3.137361258365096, + "language_loss": 0.78880739, + "learning_rate": 3.995929280967485e-06, + "loss": 0.81106663, + "num_input_tokens_seen": 48266310, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.33520508, + "step": 1713, + "time_per_iteration": 2.417691946029663 + }, + { + "auxiliary_loss_clip": 0.01019182, + "auxiliary_loss_mlp": 0.01006742, + "balance_loss_clip": 1.00194001, + "balance_loss_mlp": 1.00508475, + "epoch": 0.04973594103650397, + "flos": 64217248352640.0, + "grad_norm": 0.7880697833787375, + "language_loss": 0.55478656, + "learning_rate": 3.995917285886289e-06, + "loss": 0.57504582, + "num_input_tokens_seen": 48323180, + "router_z_loss_clip": 0.17285156, + "router_z_loss_mlp": 0.01660156, + "step": 1714, + "time_per_iteration": 2.8553929328918457 + }, + { + "auxiliary_loss_clip": 0.01143298, + "auxiliary_loss_mlp": 0.01057652, + "balance_loss_clip": 1.04323959, + "balance_loss_mlp": 1.02749181, + "epoch": 0.049764958505020024, + "flos": 15187986005760.0, + "grad_norm": 3.6663557449663413, + "language_loss": 0.77679074, + "learning_rate": 3.995905273176343e-06, + "loss": 0.79880029, + "num_input_tokens_seen": 48335835, + "router_z_loss_clip": 1.00146484, + "router_z_loss_mlp": 0.30151367, + "step": 1715, + "time_per_iteration": 2.4250807762145996 + }, + { + "auxiliary_loss_clip": 0.01141205, + "auxiliary_loss_mlp": 0.0104975, + "balance_loss_clip": 1.04296184, + "balance_loss_mlp": 1.02638459, + "epoch": 0.04979397597353607, + "flos": 26534583849600.0, + "grad_norm": 3.6161391614603353, + "language_loss": 0.73187506, + "learning_rate": 3.9958932428377545e-06, + "loss": 0.75378466, + "num_input_tokens_seen": 48350420, + "router_z_loss_clip": 0.98193359, + "router_z_loss_mlp": 0.23388672, + "step": 1716, + "time_per_iteration": 2.638857841491699 + }, + { + "auxiliary_loss_clip": 0.01137987, + "auxiliary_loss_mlp": 0.01057187, + "balance_loss_clip": 1.0407548, + "balance_loss_mlp": 1.03217733, + "epoch": 0.04982299344205211, + "flos": 16281465788160.0, + "grad_norm": 3.0199780566062926, + "language_loss": 0.67907441, + "learning_rate": 3.99588119487063e-06, + "loss": 0.7010262, + "num_input_tokens_seen": 48363590, + "router_z_loss_clip": 0.97119141, + "router_z_loss_mlp": 0.25012207, + "step": 1717, + "time_per_iteration": 2.482445001602173 + }, + { + "auxiliary_loss_clip": 0.0114771, + "auxiliary_loss_mlp": 0.01058674, + "balance_loss_clip": 1.04340935, + "balance_loss_mlp": 1.03026664, + "epoch": 0.049852010910568165, + "flos": 22266102625920.0, + "grad_norm": 2.6091842308199675, + "language_loss": 0.94072628, + "learning_rate": 3.995869129275074e-06, + "loss": 0.96279013, + "num_input_tokens_seen": 48379160, + "router_z_loss_clip": 1.04296875, + "router_z_loss_mlp": 0.28417969, + "step": 1718, + "time_per_iteration": 2.513075590133667 + }, + { + "auxiliary_loss_clip": 0.01136203, + "auxiliary_loss_mlp": 0.0105682, + "balance_loss_clip": 1.04066396, + "balance_loss_mlp": 1.03215575, + "epoch": 0.04988102837908421, + "flos": 18110495427840.0, + "grad_norm": 1.9464244438842437, + "language_loss": 0.6840359, + "learning_rate": 3.995857046051195e-06, + "loss": 0.70596606, + "num_input_tokens_seen": 48396205, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.24658203, + "step": 1719, + "time_per_iteration": 2.4848124980926514 + }, + { + "auxiliary_loss_clip": 0.01134054, + "auxiliary_loss_mlp": 0.01048552, + "balance_loss_clip": 1.03822291, + "balance_loss_mlp": 1.02522278, + "epoch": 0.049910045847600254, + "flos": 27455906384640.0, + "grad_norm": 2.883399897682582, + "language_loss": 0.94545305, + "learning_rate": 3.995844945199099e-06, + "loss": 0.9672792, + "num_input_tokens_seen": 48413235, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.23364258, + "step": 1720, + "time_per_iteration": 2.35676646232605 + }, + { + "auxiliary_loss_clip": 0.01018181, + "auxiliary_loss_mlp": 0.01003638, + "balance_loss_clip": 1.00135517, + "balance_loss_mlp": 1.00181437, + "epoch": 0.0499390633161163, + "flos": 74772399519360.0, + "grad_norm": 0.6525111153699313, + "language_loss": 0.50266743, + "learning_rate": 3.995832826718892e-06, + "loss": 0.52288562, + "num_input_tokens_seen": 48483785, + "router_z_loss_clip": 0.16796875, + "router_z_loss_mlp": 0.01818848, + "step": 1721, + "time_per_iteration": 3.335505723953247 + }, + { + "auxiliary_loss_clip": 0.01017807, + "auxiliary_loss_mlp": 0.01003894, + "balance_loss_clip": 1.00106573, + "balance_loss_mlp": 1.00206971, + "epoch": 0.04996808078463235, + "flos": 61474470076800.0, + "grad_norm": 0.6902129287693526, + "language_loss": 0.51808941, + "learning_rate": 3.995820690610682e-06, + "loss": 0.53830642, + "num_input_tokens_seen": 48550950, + "router_z_loss_clip": 0.16796875, + "router_z_loss_mlp": 0.01818848, + "step": 1722, + "time_per_iteration": 3.203348398208618 + }, + { + "auxiliary_loss_clip": 0.01146121, + "auxiliary_loss_mlp": 0.01060857, + "balance_loss_clip": 1.04180336, + "balance_loss_mlp": 1.03253269, + "epoch": 0.049997098253148395, + "flos": 13508279717760.0, + "grad_norm": 3.517870458373816, + "language_loss": 1.02760148, + "learning_rate": 3.995808536874577e-06, + "loss": 1.04967129, + "num_input_tokens_seen": 48562490, + "router_z_loss_clip": 1.04492188, + "router_z_loss_mlp": 0.28356934, + "step": 1723, + "time_per_iteration": 2.377363920211792 + }, + { + "auxiliary_loss_clip": 0.01142078, + "auxiliary_loss_mlp": 0.01064268, + "balance_loss_clip": 1.04267406, + "balance_loss_mlp": 1.03522849, + "epoch": 0.05002611572166444, + "flos": 18923936261760.0, + "grad_norm": 2.314495586014057, + "language_loss": 0.81570959, + "learning_rate": 3.995796365510682e-06, + "loss": 0.83777308, + "num_input_tokens_seen": 48578755, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.29040527, + "step": 1724, + "time_per_iteration": 2.4437360763549805 + }, + { + "auxiliary_loss_clip": 0.01140159, + "auxiliary_loss_mlp": 0.01058704, + "balance_loss_clip": 1.04199052, + "balance_loss_mlp": 1.03406358, + "epoch": 0.05005513319018049, + "flos": 27226527552000.0, + "grad_norm": 2.7550011982294547, + "language_loss": 0.88265526, + "learning_rate": 3.995784176519107e-06, + "loss": 0.90464389, + "num_input_tokens_seen": 48595135, + "router_z_loss_clip": 0.98193359, + "router_z_loss_mlp": 0.24633789, + "step": 1725, + "time_per_iteration": 2.49100661277771 + }, + { + "auxiliary_loss_clip": 0.01131575, + "auxiliary_loss_mlp": 0.01040123, + "balance_loss_clip": 1.03924561, + "balance_loss_mlp": 1.01691329, + "epoch": 0.050084150658696536, + "flos": 20587932437760.0, + "grad_norm": 2.680573155836852, + "language_loss": 0.87196368, + "learning_rate": 3.995771969899958e-06, + "loss": 0.89368069, + "num_input_tokens_seen": 48610115, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.23193359, + "step": 1726, + "time_per_iteration": 2.396298885345459 + }, + { + "auxiliary_loss_clip": 0.01144113, + "auxiliary_loss_mlp": 0.01046329, + "balance_loss_clip": 1.045439, + "balance_loss_mlp": 1.02152205, + "epoch": 0.05011316812721258, + "flos": 20806698216960.0, + "grad_norm": 2.859495284100451, + "language_loss": 0.86493897, + "learning_rate": 3.9957597456533435e-06, + "loss": 0.88684344, + "num_input_tokens_seen": 48625615, + "router_z_loss_clip": 0.98681641, + "router_z_loss_mlp": 0.24853516, + "step": 1727, + "time_per_iteration": 2.463676929473877 + }, + { + "auxiliary_loss_clip": 0.01143009, + "auxiliary_loss_mlp": 0.01053205, + "balance_loss_clip": 1.04140258, + "balance_loss_mlp": 1.02645421, + "epoch": 0.05014218559572863, + "flos": 21535789472640.0, + "grad_norm": 3.1330577445861754, + "language_loss": 1.03442931, + "learning_rate": 3.995747503779372e-06, + "loss": 1.05639148, + "num_input_tokens_seen": 48638880, + "router_z_loss_clip": 1.01611328, + "router_z_loss_mlp": 0.26782227, + "step": 1728, + "time_per_iteration": 2.4467358589172363 + }, + { + "auxiliary_loss_clip": 0.01142345, + "auxiliary_loss_mlp": 0.01057401, + "balance_loss_clip": 1.04304075, + "balance_loss_mlp": 1.02795684, + "epoch": 0.05017120306424468, + "flos": 23615146627200.0, + "grad_norm": 2.2214154316322112, + "language_loss": 0.856655, + "learning_rate": 3.9957352442781504e-06, + "loss": 0.87865245, + "num_input_tokens_seen": 48657505, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.2947998, + "step": 1729, + "time_per_iteration": 2.7256600856781006 + }, + { + "auxiliary_loss_clip": 0.01019413, + "auxiliary_loss_mlp": 0.01005196, + "balance_loss_clip": 1.00208008, + "balance_loss_mlp": 1.00324094, + "epoch": 0.05020022053276072, + "flos": 67582456214400.0, + "grad_norm": 0.6849630498439386, + "language_loss": 0.52832437, + "learning_rate": 3.995722967149787e-06, + "loss": 0.54857045, + "num_input_tokens_seen": 48717525, + "router_z_loss_clip": 0.17382812, + "router_z_loss_mlp": 0.01953125, + "step": 1730, + "time_per_iteration": 3.0095338821411133 + }, + { + "auxiliary_loss_clip": 0.01139913, + "auxiliary_loss_mlp": 0.01061724, + "balance_loss_clip": 1.03851879, + "balance_loss_mlp": 1.03547382, + "epoch": 0.050229238001276766, + "flos": 20808688164480.0, + "grad_norm": 3.047285467113173, + "language_loss": 0.92756891, + "learning_rate": 3.9957106723943915e-06, + "loss": 0.94958526, + "num_input_tokens_seen": 48735345, + "router_z_loss_clip": 1.01464844, + "router_z_loss_mlp": 0.26269531, + "step": 1731, + "time_per_iteration": 2.413710355758667 + }, + { + "auxiliary_loss_clip": 0.0113393, + "auxiliary_loss_mlp": 0.01047243, + "balance_loss_clip": 1.04257369, + "balance_loss_mlp": 1.02486718, + "epoch": 0.05025825546979282, + "flos": 40180177411200.0, + "grad_norm": 2.1418494964439896, + "language_loss": 0.77743763, + "learning_rate": 3.995698360012072e-06, + "loss": 0.79924935, + "num_input_tokens_seen": 48754600, + "router_z_loss_clip": 0.91357422, + "router_z_loss_mlp": 0.22375488, + "step": 1732, + "time_per_iteration": 2.585435152053833 + }, + { + "auxiliary_loss_clip": 0.01019735, + "auxiliary_loss_mlp": 0.01002705, + "balance_loss_clip": 1.0026176, + "balance_loss_mlp": 1.0009172, + "epoch": 0.05028727293830886, + "flos": 63318234176640.0, + "grad_norm": 0.6609435501606231, + "language_loss": 0.48456103, + "learning_rate": 3.995686030002936e-06, + "loss": 0.50478542, + "num_input_tokens_seen": 48816535, + "router_z_loss_clip": 0.17089844, + "router_z_loss_mlp": 0.01782227, + "step": 1733, + "time_per_iteration": 2.9932358264923096 + }, + { + "auxiliary_loss_clip": 0.01134237, + "auxiliary_loss_mlp": 0.01050125, + "balance_loss_clip": 1.04427898, + "balance_loss_mlp": 1.02829814, + "epoch": 0.05031629040682491, + "flos": 18142545922560.0, + "grad_norm": 2.572993921823496, + "language_loss": 0.92658764, + "learning_rate": 3.995673682367094e-06, + "loss": 0.94843131, + "num_input_tokens_seen": 48829095, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.21813965, + "step": 1734, + "time_per_iteration": 2.422652006149292 + }, + { + "auxiliary_loss_clip": 0.01018876, + "auxiliary_loss_mlp": 0.01002339, + "balance_loss_clip": 1.00185657, + "balance_loss_mlp": 1.00078976, + "epoch": 0.05034530787534096, + "flos": 74775332073600.0, + "grad_norm": 0.7196085268287433, + "language_loss": 0.53798449, + "learning_rate": 3.995661317104654e-06, + "loss": 0.5581966, + "num_input_tokens_seen": 48892975, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01544189, + "step": 1735, + "time_per_iteration": 3.1282458305358887 + }, + { + "auxiliary_loss_clip": 0.01019538, + "auxiliary_loss_mlp": 0.01005947, + "balance_loss_clip": 1.00250793, + "balance_loss_mlp": 1.00423014, + "epoch": 0.050374325343857, + "flos": 74482061719680.0, + "grad_norm": 0.6978854386728661, + "language_loss": 0.56033766, + "learning_rate": 3.995648934215726e-06, + "loss": 0.58059251, + "num_input_tokens_seen": 48959070, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01721191, + "step": 1736, + "time_per_iteration": 3.109553813934326 + }, + { + "auxiliary_loss_clip": 0.01140921, + "auxiliary_loss_mlp": 0.0105515, + "balance_loss_clip": 1.04229939, + "balance_loss_mlp": 1.0302, + "epoch": 0.05040334281237305, + "flos": 16543837722240.0, + "grad_norm": 2.163502159987134, + "language_loss": 0.70472276, + "learning_rate": 3.995636533700419e-06, + "loss": 0.7266835, + "num_input_tokens_seen": 48973280, + "router_z_loss_clip": 0.98632812, + "router_z_loss_mlp": 0.24951172, + "step": 1737, + "time_per_iteration": 2.505241870880127 + }, + { + "auxiliary_loss_clip": 0.01151623, + "auxiliary_loss_mlp": 0.01058414, + "balance_loss_clip": 1.04538429, + "balance_loss_mlp": 1.02764583, + "epoch": 0.05043236028088909, + "flos": 18106585355520.0, + "grad_norm": 2.526633425587941, + "language_loss": 1.03644657, + "learning_rate": 3.995624115558843e-06, + "loss": 1.05854702, + "num_input_tokens_seen": 48989085, + "router_z_loss_clip": 1.06201172, + "router_z_loss_mlp": 0.30773926, + "step": 1738, + "time_per_iteration": 2.413233757019043 + }, + { + "auxiliary_loss_clip": 0.01018495, + "auxiliary_loss_mlp": 0.01007442, + "balance_loss_clip": 1.0014497, + "balance_loss_mlp": 1.00573778, + "epoch": 0.050461377749405144, + "flos": 64719858418560.0, + "grad_norm": 0.7655279929386792, + "language_loss": 0.47692722, + "learning_rate": 3.995611679791107e-06, + "loss": 0.49718657, + "num_input_tokens_seen": 49036660, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01708984, + "step": 1739, + "time_per_iteration": 2.930530548095703 + }, + { + "auxiliary_loss_clip": 0.0114129, + "auxiliary_loss_mlp": 0.01057344, + "balance_loss_clip": 1.04089952, + "balance_loss_mlp": 1.02967548, + "epoch": 0.05049039521792119, + "flos": 11139632104320.0, + "grad_norm": 3.085555864769303, + "language_loss": 0.94060338, + "learning_rate": 3.995599226397321e-06, + "loss": 0.96258974, + "num_input_tokens_seen": 49049080, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.27709961, + "step": 1740, + "time_per_iteration": 2.37862491607666 + }, + { + "auxiliary_loss_clip": 0.01133503, + "auxiliary_loss_mlp": 0.01051733, + "balance_loss_clip": 1.04111886, + "balance_loss_mlp": 1.02646017, + "epoch": 0.05051941268643723, + "flos": 21902202858240.0, + "grad_norm": 3.7961211952886638, + "language_loss": 0.8453331, + "learning_rate": 3.995586755377595e-06, + "loss": 0.86718541, + "num_input_tokens_seen": 49063855, + "router_z_loss_clip": 0.92529297, + "router_z_loss_mlp": 0.25292969, + "step": 1741, + "time_per_iteration": 2.516958475112915 + }, + { + "auxiliary_loss_clip": 0.01020953, + "auxiliary_loss_mlp": 0.01002699, + "balance_loss_clip": 1.00365949, + "balance_loss_mlp": 1.00098252, + "epoch": 0.050548430154953285, + "flos": 63692851731840.0, + "grad_norm": 0.6253719102397172, + "language_loss": 0.53673518, + "learning_rate": 3.99557426673204e-06, + "loss": 0.55697167, + "num_input_tokens_seen": 49133965, + "router_z_loss_clip": 0.17382812, + "router_z_loss_mlp": 0.01721191, + "step": 1742, + "time_per_iteration": 3.446472644805908 + }, + { + "auxiliary_loss_clip": 0.01154684, + "auxiliary_loss_mlp": 0.0106167, + "balance_loss_clip": 1.04799116, + "balance_loss_mlp": 1.03453803, + "epoch": 0.05057744762346933, + "flos": 16058894670720.0, + "grad_norm": 2.7090396220072153, + "language_loss": 0.8007077, + "learning_rate": 3.9955617604607644e-06, + "loss": 0.82287121, + "num_input_tokens_seen": 49145200, + "router_z_loss_clip": 1.06640625, + "router_z_loss_mlp": 0.27148438, + "step": 1743, + "time_per_iteration": 2.426360607147217 + }, + { + "auxiliary_loss_clip": 0.01149893, + "auxiliary_loss_mlp": 0.01062748, + "balance_loss_clip": 1.04734886, + "balance_loss_mlp": 1.03625953, + "epoch": 0.050606465091985374, + "flos": 74730855000960.0, + "grad_norm": 2.2093325160980823, + "language_loss": 0.69718981, + "learning_rate": 3.99554923656388e-06, + "loss": 0.71931624, + "num_input_tokens_seen": 49169180, + "router_z_loss_clip": 1.02490234, + "router_z_loss_mlp": 0.26525879, + "step": 1744, + "time_per_iteration": 2.7855687141418457 + }, + { + "auxiliary_loss_clip": 0.01144691, + "auxiliary_loss_mlp": 0.0106821, + "balance_loss_clip": 1.045367, + "balance_loss_mlp": 1.04252017, + "epoch": 0.05063548256050142, + "flos": 18041087911680.0, + "grad_norm": 2.502861440568068, + "language_loss": 0.86856401, + "learning_rate": 3.995536695041499e-06, + "loss": 0.89069301, + "num_input_tokens_seen": 49183930, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.25708008, + "step": 1745, + "time_per_iteration": 2.4100492000579834 + }, + { + "auxiliary_loss_clip": 0.01137727, + "auxiliary_loss_mlp": 0.01053638, + "balance_loss_clip": 1.04288495, + "balance_loss_mlp": 1.02825809, + "epoch": 0.05066450002901747, + "flos": 31681444769280.0, + "grad_norm": 1.9567486702117922, + "language_loss": 0.86282355, + "learning_rate": 3.995524135893728e-06, + "loss": 0.88473725, + "num_input_tokens_seen": 49200215, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.25390625, + "step": 1746, + "time_per_iteration": 2.4719059467315674 + }, + { + "auxiliary_loss_clip": 0.01020679, + "auxiliary_loss_mlp": 0.01004399, + "balance_loss_clip": 1.0037446, + "balance_loss_mlp": 1.00245607, + "epoch": 0.050693517497533515, + "flos": 60105037841280.0, + "grad_norm": 0.676210124132498, + "language_loss": 0.50106072, + "learning_rate": 3.995511559120681e-06, + "loss": 0.52131146, + "num_input_tokens_seen": 49264335, + "router_z_loss_clip": 0.16894531, + "router_z_loss_mlp": 0.01940918, + "step": 1747, + "time_per_iteration": 3.127826690673828 + }, + { + "auxiliary_loss_clip": 0.01020239, + "auxiliary_loss_mlp": 0.01002727, + "balance_loss_clip": 1.00327587, + "balance_loss_mlp": 1.00098646, + "epoch": 0.05072253496604956, + "flos": 61370989073280.0, + "grad_norm": 0.7143618797023337, + "language_loss": 0.51031774, + "learning_rate": 3.995498964722469e-06, + "loss": 0.53054738, + "num_input_tokens_seen": 49319605, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01745605, + "step": 1748, + "time_per_iteration": 2.9796950817108154 + }, + { + "auxiliary_loss_clip": 0.01130482, + "auxiliary_loss_mlp": 0.0104967, + "balance_loss_clip": 1.04128146, + "balance_loss_mlp": 1.02636433, + "epoch": 0.05075155243456561, + "flos": 16391477082240.0, + "grad_norm": 2.846677927920875, + "language_loss": 0.81625384, + "learning_rate": 3.9954863526992026e-06, + "loss": 0.83805537, + "num_input_tokens_seen": 49333280, + "router_z_loss_clip": 0.89208984, + "router_z_loss_mlp": 0.23303223, + "step": 1749, + "time_per_iteration": 2.3494651317596436 + }, + { + "auxiliary_loss_clip": 0.01145708, + "auxiliary_loss_mlp": 0.0105277, + "balance_loss_clip": 1.04267049, + "balance_loss_mlp": 1.02458882, + "epoch": 0.050780569903081656, + "flos": 20074046002560.0, + "grad_norm": 2.5776857509871087, + "language_loss": 0.73799187, + "learning_rate": 3.995473723050993e-06, + "loss": 0.75997669, + "num_input_tokens_seen": 49347555, + "router_z_loss_clip": 1.03076172, + "router_z_loss_mlp": 0.28161621, + "step": 1750, + "time_per_iteration": 4.816134691238403 + }, + { + "auxiliary_loss_clip": 0.01135884, + "auxiliary_loss_mlp": 0.01049674, + "balance_loss_clip": 1.04137802, + "balance_loss_mlp": 1.02681017, + "epoch": 0.0508095873715977, + "flos": 10808795260800.0, + "grad_norm": 2.8109595086644514, + "language_loss": 0.78794932, + "learning_rate": 3.995461075777952e-06, + "loss": 0.80980486, + "num_input_tokens_seen": 49358300, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.22888184, + "step": 1751, + "time_per_iteration": 2.4239768981933594 + }, + { + "auxiliary_loss_clip": 0.0113167, + "auxiliary_loss_mlp": 0.01054284, + "balance_loss_clip": 1.04133534, + "balance_loss_mlp": 1.02978611, + "epoch": 0.05083860484011375, + "flos": 74729982216960.0, + "grad_norm": 2.0749271859445706, + "language_loss": 0.821172, + "learning_rate": 3.995448410880192e-06, + "loss": 0.84303153, + "num_input_tokens_seen": 49378550, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.24487305, + "step": 1752, + "time_per_iteration": 2.8372514247894287 + }, + { + "auxiliary_loss_clip": 0.01141325, + "auxiliary_loss_mlp": 0.0105422, + "balance_loss_clip": 1.04484856, + "balance_loss_mlp": 1.02905536, + "epoch": 0.0508676223086298, + "flos": 24893456480640.0, + "grad_norm": 2.457627439364449, + "language_loss": 0.77851093, + "learning_rate": 3.995435728357823e-06, + "loss": 0.80046636, + "num_input_tokens_seen": 49392820, + "router_z_loss_clip": 0.96289062, + "router_z_loss_mlp": 0.25146484, + "step": 1753, + "time_per_iteration": 2.5043442249298096 + }, + { + "auxiliary_loss_clip": 0.01149756, + "auxiliary_loss_mlp": 0.01066922, + "balance_loss_clip": 1.04707122, + "balance_loss_mlp": 1.03858626, + "epoch": 0.05089663977714584, + "flos": 16244737171200.0, + "grad_norm": 2.6847076896108564, + "language_loss": 0.74641263, + "learning_rate": 3.995423028210959e-06, + "loss": 0.76857942, + "num_input_tokens_seen": 49406565, + "router_z_loss_clip": 1.02636719, + "router_z_loss_mlp": 0.28320312, + "step": 1754, + "time_per_iteration": 2.634495496749878 + }, + { + "auxiliary_loss_clip": 0.01142079, + "auxiliary_loss_mlp": 0.01055091, + "balance_loss_clip": 1.04313254, + "balance_loss_mlp": 1.02938914, + "epoch": 0.050925657245661886, + "flos": 27232846508160.0, + "grad_norm": 1.9133812100362222, + "language_loss": 0.65105557, + "learning_rate": 3.995410310439711e-06, + "loss": 0.67302728, + "num_input_tokens_seen": 49430015, + "router_z_loss_clip": 0.98974609, + "router_z_loss_mlp": 0.25683594, + "step": 1755, + "time_per_iteration": 4.7568678855896 + }, + { + "auxiliary_loss_clip": 0.01147354, + "auxiliary_loss_mlp": 0.01053862, + "balance_loss_clip": 1.04215586, + "balance_loss_mlp": 1.02796936, + "epoch": 0.05095467471417794, + "flos": 13295379047040.0, + "grad_norm": 2.92774619272626, + "language_loss": 0.9971453, + "learning_rate": 3.9953975750441915e-06, + "loss": 1.01915741, + "num_input_tokens_seen": 49443105, + "router_z_loss_clip": 1.05078125, + "router_z_loss_mlp": 0.25891113, + "step": 1756, + "time_per_iteration": 2.415353775024414 + }, + { + "auxiliary_loss_clip": 0.0115111, + "auxiliary_loss_mlp": 0.01076673, + "balance_loss_clip": 1.04509926, + "balance_loss_mlp": 1.04718053, + "epoch": 0.05098369218269398, + "flos": 15734900453760.0, + "grad_norm": 2.48617053369031, + "language_loss": 0.78965318, + "learning_rate": 3.995384822024513e-06, + "loss": 0.81193101, + "num_input_tokens_seen": 49456990, + "router_z_loss_clip": 1.05957031, + "router_z_loss_mlp": 0.2947998, + "step": 1757, + "time_per_iteration": 2.3640990257263184 + }, + { + "auxiliary_loss_clip": 0.01125993, + "auxiliary_loss_mlp": 0.01048332, + "balance_loss_clip": 1.03990936, + "balance_loss_mlp": 1.02596855, + "epoch": 0.05101270965121003, + "flos": 25439463233280.0, + "grad_norm": 4.4495205155355615, + "language_loss": 0.92175674, + "learning_rate": 3.995372051380789e-06, + "loss": 0.94350004, + "num_input_tokens_seen": 49470115, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.22363281, + "step": 1758, + "time_per_iteration": 4.8764708042144775 + }, + { + "auxiliary_loss_clip": 0.0115316, + "auxiliary_loss_mlp": 0.01053209, + "balance_loss_clip": 1.04405165, + "balance_loss_mlp": 1.024122, + "epoch": 0.05104172711972608, + "flos": 15261198860160.0, + "grad_norm": 2.9777360977899683, + "language_loss": 0.82848942, + "learning_rate": 3.9953592631131315e-06, + "loss": 0.85055315, + "num_input_tokens_seen": 49483675, + "router_z_loss_clip": 1.09082031, + "router_z_loss_mlp": 0.2911377, + "step": 1759, + "time_per_iteration": 4.8489110469818115 + }, + { + "auxiliary_loss_clip": 0.01135065, + "auxiliary_loss_mlp": 0.01035951, + "balance_loss_clip": 1.04393888, + "balance_loss_mlp": 1.01632929, + "epoch": 0.05107074458824212, + "flos": 30218514312960.0, + "grad_norm": 2.074817961348676, + "language_loss": 0.76992106, + "learning_rate": 3.995346457221653e-06, + "loss": 0.79163122, + "num_input_tokens_seen": 49500015, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.19641113, + "step": 1760, + "time_per_iteration": 2.5062942504882812 + }, + { + "auxiliary_loss_clip": 0.0103127, + "auxiliary_loss_mlp": 0.01002313, + "balance_loss_clip": 1.01401925, + "balance_loss_mlp": 1.00047696, + "epoch": 0.05109976205675817, + "flos": 57107884199040.0, + "grad_norm": 0.7107969021202618, + "language_loss": 0.56112057, + "learning_rate": 3.995333633706468e-06, + "loss": 0.58145642, + "num_input_tokens_seen": 49551920, + "router_z_loss_clip": 0.17285156, + "router_z_loss_mlp": 0.01831055, + "step": 1761, + "time_per_iteration": 2.848525047302246 + }, + { + "auxiliary_loss_clip": 0.01029108, + "auxiliary_loss_mlp": 0.01001894, + "balance_loss_clip": 1.01207209, + "balance_loss_mlp": 0.99984342, + "epoch": 0.05112877952527421, + "flos": 74776274680320.0, + "grad_norm": 0.7318108818403097, + "language_loss": 0.51317412, + "learning_rate": 3.995320792567688e-06, + "loss": 0.53348416, + "num_input_tokens_seen": 49620250, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.02050781, + "step": 1762, + "time_per_iteration": 3.1881141662597656 + }, + { + "auxiliary_loss_clip": 0.01143137, + "auxiliary_loss_mlp": 0.01060833, + "balance_loss_clip": 1.04272962, + "balance_loss_mlp": 1.03447556, + "epoch": 0.051157796993790264, + "flos": 21172797400320.0, + "grad_norm": 8.725349477589067, + "language_loss": 0.85044086, + "learning_rate": 3.995307933805428e-06, + "loss": 0.87248057, + "num_input_tokens_seen": 49634965, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.2635498, + "step": 1763, + "time_per_iteration": 2.383406400680542 + }, + { + "auxiliary_loss_clip": 0.01020162, + "auxiliary_loss_mlp": 0.01003089, + "balance_loss_clip": 1.00354826, + "balance_loss_mlp": 1.00133646, + "epoch": 0.05118681446230631, + "flos": 74770479394560.0, + "grad_norm": 0.649424477724531, + "language_loss": 0.53192675, + "learning_rate": 3.9952950574198e-06, + "loss": 0.55215919, + "num_input_tokens_seen": 49697380, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01757812, + "step": 1764, + "time_per_iteration": 3.065216541290283 + }, + { + "auxiliary_loss_clip": 0.01142979, + "auxiliary_loss_mlp": 0.01065356, + "balance_loss_clip": 1.04248869, + "balance_loss_mlp": 1.03576779, + "epoch": 0.05121583193082235, + "flos": 15808427510400.0, + "grad_norm": 4.229715804950658, + "language_loss": 1.0544802, + "learning_rate": 3.99528216341092e-06, + "loss": 1.07656348, + "num_input_tokens_seen": 49712370, + "router_z_loss_clip": 1.00390625, + "router_z_loss_mlp": 0.29589844, + "step": 1765, + "time_per_iteration": 2.6432137489318848 + }, + { + "auxiliary_loss_clip": 0.01018679, + "auxiliary_loss_mlp": 0.01006476, + "balance_loss_clip": 1.00232518, + "balance_loss_mlp": 1.00462818, + "epoch": 0.051244849399338405, + "flos": 51319563269760.0, + "grad_norm": 0.7641420762418243, + "language_loss": 0.52516383, + "learning_rate": 3.9952692517789e-06, + "loss": 0.5454154, + "num_input_tokens_seen": 49771515, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01843262, + "step": 1766, + "time_per_iteration": 2.987337112426758 + }, + { + "auxiliary_loss_clip": 0.01020248, + "auxiliary_loss_mlp": 0.01002185, + "balance_loss_clip": 1.00372958, + "balance_loss_mlp": 1.00045633, + "epoch": 0.05127386686785445, + "flos": 71882987932800.0, + "grad_norm": 0.7154909605322282, + "language_loss": 0.49697796, + "learning_rate": 3.995256322523854e-06, + "loss": 0.51720226, + "num_input_tokens_seen": 49827015, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01733398, + "step": 1767, + "time_per_iteration": 3.037382125854492 + }, + { + "auxiliary_loss_clip": 0.01142879, + "auxiliary_loss_mlp": 0.01058772, + "balance_loss_clip": 1.04306769, + "balance_loss_mlp": 1.03324926, + "epoch": 0.051302884336370494, + "flos": 14164821434880.0, + "grad_norm": 4.037380098649873, + "language_loss": 1.04145515, + "learning_rate": 3.995243375645898e-06, + "loss": 1.06347167, + "num_input_tokens_seen": 49838015, + "router_z_loss_clip": 0.99853516, + "router_z_loss_mlp": 0.25537109, + "step": 1768, + "time_per_iteration": 2.4306886196136475 + }, + { + "auxiliary_loss_clip": 0.01024697, + "auxiliary_loss_mlp": 0.01005108, + "balance_loss_clip": 1.00785112, + "balance_loss_mlp": 1.00323665, + "epoch": 0.05133190180488654, + "flos": 62033849746560.0, + "grad_norm": 0.6470610710816844, + "language_loss": 0.48694134, + "learning_rate": 3.995230411145144e-06, + "loss": 0.50723934, + "num_input_tokens_seen": 49899960, + "router_z_loss_clip": 0.16796875, + "router_z_loss_mlp": 0.01867676, + "step": 1769, + "time_per_iteration": 3.056001901626587 + }, + { + "auxiliary_loss_clip": 0.01149528, + "auxiliary_loss_mlp": 0.01054178, + "balance_loss_clip": 1.04333699, + "balance_loss_mlp": 1.02537727, + "epoch": 0.05136091927340259, + "flos": 32699337724800.0, + "grad_norm": 2.4918344132804306, + "language_loss": 0.93663585, + "learning_rate": 3.995217429021708e-06, + "loss": 0.95867288, + "num_input_tokens_seen": 49918150, + "router_z_loss_clip": 1.06298828, + "router_z_loss_mlp": 0.28808594, + "step": 1770, + "time_per_iteration": 2.530487537384033 + }, + { + "auxiliary_loss_clip": 0.01137098, + "auxiliary_loss_mlp": 0.01069248, + "balance_loss_clip": 1.04089928, + "balance_loss_mlp": 1.04440498, + "epoch": 0.051389936741918635, + "flos": 16062176338560.0, + "grad_norm": 2.3045967771654428, + "language_loss": 0.73675656, + "learning_rate": 3.995204429275704e-06, + "loss": 0.75882, + "num_input_tokens_seen": 49932735, + "router_z_loss_clip": 0.96240234, + "router_z_loss_mlp": 0.24829102, + "step": 1771, + "time_per_iteration": 2.4003219604492188 + }, + { + "auxiliary_loss_clip": 0.01153226, + "auxiliary_loss_mlp": 0.01068567, + "balance_loss_clip": 1.0477922, + "balance_loss_mlp": 1.04176831, + "epoch": 0.05141895421043468, + "flos": 29487921868800.0, + "grad_norm": 2.372289551858586, + "language_loss": 0.85016352, + "learning_rate": 3.9951914119072466e-06, + "loss": 0.87238145, + "num_input_tokens_seen": 49947890, + "router_z_loss_clip": 1.05419922, + "router_z_loss_mlp": 0.26806641, + "step": 1772, + "time_per_iteration": 2.476341962814331 + }, + { + "auxiliary_loss_clip": 0.0114559, + "auxiliary_loss_mlp": 0.01067179, + "balance_loss_clip": 1.04630291, + "balance_loss_mlp": 1.03966534, + "epoch": 0.05144797167895073, + "flos": 14164646878080.0, + "grad_norm": 2.3936987696284615, + "language_loss": 0.80492598, + "learning_rate": 3.995178376916453e-06, + "loss": 0.82705373, + "num_input_tokens_seen": 49960110, + "router_z_loss_clip": 0.99316406, + "router_z_loss_mlp": 0.27539062, + "step": 1773, + "time_per_iteration": 2.500910997390747 + }, + { + "auxiliary_loss_clip": 0.01140275, + "auxiliary_loss_mlp": 0.01049783, + "balance_loss_clip": 1.04535067, + "balance_loss_mlp": 1.0277884, + "epoch": 0.051476989147466776, + "flos": 30475335340800.0, + "grad_norm": 2.7160041597328175, + "language_loss": 1.10948813, + "learning_rate": 3.9951653243034355e-06, + "loss": 1.13138866, + "num_input_tokens_seen": 49979035, + "router_z_loss_clip": 0.94970703, + "router_z_loss_mlp": 0.21984863, + "step": 1774, + "time_per_iteration": 2.467733144760132 + }, + { + "auxiliary_loss_clip": 0.01023497, + "auxiliary_loss_mlp": 0.01011837, + "balance_loss_clip": 1.00706792, + "balance_loss_mlp": 1.01000071, + "epoch": 0.05150600661598282, + "flos": 74791216742400.0, + "grad_norm": 0.6502988721281238, + "language_loss": 0.45660841, + "learning_rate": 3.99515225406831e-06, + "loss": 0.47696173, + "num_input_tokens_seen": 50040985, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01831055, + "step": 1775, + "time_per_iteration": 3.220428943634033 + }, + { + "auxiliary_loss_clip": 0.01141179, + "auxiliary_loss_mlp": 0.01046152, + "balance_loss_clip": 1.04286456, + "balance_loss_mlp": 1.01932955, + "epoch": 0.051535024084498865, + "flos": 25331930645760.0, + "grad_norm": 2.095518078605235, + "language_loss": 0.80499518, + "learning_rate": 3.995139166211193e-06, + "loss": 0.82686847, + "num_input_tokens_seen": 50058245, + "router_z_loss_clip": 0.98193359, + "router_z_loss_mlp": 0.26782227, + "step": 1776, + "time_per_iteration": 2.4681508541107178 + }, + { + "auxiliary_loss_clip": 0.01019004, + "auxiliary_loss_mlp": 0.0100265, + "balance_loss_clip": 1.00289333, + "balance_loss_mlp": 1.00092113, + "epoch": 0.05156404155301492, + "flos": 59533509018240.0, + "grad_norm": 0.7072406048850319, + "language_loss": 0.51976073, + "learning_rate": 3.9951260607322e-06, + "loss": 0.53997731, + "num_input_tokens_seen": 50120770, + "router_z_loss_clip": 0.16113281, + "router_z_loss_mlp": 0.01733398, + "step": 1777, + "time_per_iteration": 3.1717989444732666 + }, + { + "auxiliary_loss_clip": 0.0101791, + "auxiliary_loss_mlp": 0.01003461, + "balance_loss_clip": 1.00182199, + "balance_loss_mlp": 1.00167263, + "epoch": 0.05159305902153096, + "flos": 66637217531520.0, + "grad_norm": 0.6393627727915042, + "language_loss": 0.50662732, + "learning_rate": 3.995112937631446e-06, + "loss": 0.52684104, + "num_input_tokens_seen": 50186035, + "router_z_loss_clip": 0.16015625, + "router_z_loss_mlp": 0.01782227, + "step": 1778, + "time_per_iteration": 3.2732958793640137 + }, + { + "auxiliary_loss_clip": 0.0115575, + "auxiliary_loss_mlp": 0.01055974, + "balance_loss_clip": 1.04654646, + "balance_loss_mlp": 1.02906847, + "epoch": 0.051622076490047006, + "flos": 20406698236800.0, + "grad_norm": 1.9893697642607509, + "language_loss": 0.78831124, + "learning_rate": 3.9950997969090465e-06, + "loss": 0.8104285, + "num_input_tokens_seen": 50203225, + "router_z_loss_clip": 1.09326172, + "router_z_loss_mlp": 0.26904297, + "step": 1779, + "time_per_iteration": 2.4510035514831543 + }, + { + "auxiliary_loss_clip": 0.01139237, + "auxiliary_loss_mlp": 0.01048062, + "balance_loss_clip": 1.04260874, + "balance_loss_mlp": 1.02438736, + "epoch": 0.05165109395856306, + "flos": 10588982140800.0, + "grad_norm": 2.233125077899047, + "language_loss": 0.70757514, + "learning_rate": 3.995086638565119e-06, + "loss": 0.72944814, + "num_input_tokens_seen": 50214900, + "router_z_loss_clip": 0.96533203, + "router_z_loss_mlp": 0.23669434, + "step": 1780, + "time_per_iteration": 2.43485689163208 + }, + { + "auxiliary_loss_clip": 0.01133044, + "auxiliary_loss_mlp": 0.01037261, + "balance_loss_clip": 1.04068232, + "balance_loss_mlp": 1.01588678, + "epoch": 0.0516801114270791, + "flos": 12597046565760.0, + "grad_norm": 2.3039416986713332, + "language_loss": 0.79888737, + "learning_rate": 3.9950734625997795e-06, + "loss": 0.8205905, + "num_input_tokens_seen": 50227670, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.21374512, + "step": 1781, + "time_per_iteration": 2.449751853942871 + }, + { + "auxiliary_loss_clip": 0.01137154, + "auxiliary_loss_mlp": 0.01048814, + "balance_loss_clip": 1.03792703, + "balance_loss_mlp": 1.02212286, + "epoch": 0.05170912889559515, + "flos": 32189326450560.0, + "grad_norm": 4.332316861321214, + "language_loss": 0.9586544, + "learning_rate": 3.995060269013142e-06, + "loss": 0.98051405, + "num_input_tokens_seen": 50243495, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.26696777, + "step": 1782, + "time_per_iteration": 2.576129913330078 + }, + { + "auxiliary_loss_clip": 0.01138465, + "auxiliary_loss_mlp": 0.01050798, + "balance_loss_clip": 1.04123259, + "balance_loss_mlp": 1.02472663, + "epoch": 0.0517381463641112, + "flos": 18033791437440.0, + "grad_norm": 3.1084768206060653, + "language_loss": 0.91475737, + "learning_rate": 3.9950470578053265e-06, + "loss": 0.93664992, + "num_input_tokens_seen": 50259360, + "router_z_loss_clip": 0.97216797, + "router_z_loss_mlp": 0.26049805, + "step": 1783, + "time_per_iteration": 2.386350631713867 + }, + { + "auxiliary_loss_clip": 0.01133997, + "auxiliary_loss_mlp": 0.01058445, + "balance_loss_clip": 1.04069901, + "balance_loss_mlp": 1.03724945, + "epoch": 0.05176716383262724, + "flos": 36786689481600.0, + "grad_norm": 2.689423920827924, + "language_loss": 0.78105652, + "learning_rate": 3.995033828976448e-06, + "loss": 0.8029809, + "num_input_tokens_seen": 50279695, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.2121582, + "step": 1784, + "time_per_iteration": 2.479233503341675 + }, + { + "auxiliary_loss_clip": 0.01136093, + "auxiliary_loss_mlp": 0.01048346, + "balance_loss_clip": 1.04568124, + "balance_loss_mlp": 1.02597046, + "epoch": 0.05179618130114329, + "flos": 23615321184000.0, + "grad_norm": 3.7529154455865896, + "language_loss": 0.88026077, + "learning_rate": 3.995020582526623e-06, + "loss": 0.90210509, + "num_input_tokens_seen": 50292505, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.22387695, + "step": 1785, + "time_per_iteration": 2.420229196548462 + }, + { + "auxiliary_loss_clip": 0.01147566, + "auxiliary_loss_mlp": 0.01060436, + "balance_loss_clip": 1.04557562, + "balance_loss_mlp": 1.03381622, + "epoch": 0.05182519876965933, + "flos": 29633928641280.0, + "grad_norm": 2.9448537217453317, + "language_loss": 1.01339531, + "learning_rate": 3.995007318455968e-06, + "loss": 1.03547537, + "num_input_tokens_seen": 50306520, + "router_z_loss_clip": 1.02050781, + "router_z_loss_mlp": 0.26586914, + "step": 1786, + "time_per_iteration": 2.4924416542053223 + }, + { + "auxiliary_loss_clip": 0.010316, + "auxiliary_loss_mlp": 0.01011933, + "balance_loss_clip": 1.01445675, + "balance_loss_mlp": 1.00985837, + "epoch": 0.051854216238175384, + "flos": 57221351717760.0, + "grad_norm": 0.7132590468309685, + "language_loss": 0.50625503, + "learning_rate": 3.994994036764603e-06, + "loss": 0.52669036, + "num_input_tokens_seen": 50364155, + "router_z_loss_clip": 0.171875, + "router_z_loss_mlp": 0.02075195, + "step": 1787, + "time_per_iteration": 2.994961738586426 + }, + { + "auxiliary_loss_clip": 0.01028492, + "auxiliary_loss_mlp": 0.01006864, + "balance_loss_clip": 1.01179194, + "balance_loss_mlp": 1.00487304, + "epoch": 0.05188323370669143, + "flos": 74787411404160.0, + "grad_norm": 0.622582534548416, + "language_loss": 0.478066, + "learning_rate": 3.994980737452642e-06, + "loss": 0.49841955, + "num_input_tokens_seen": 50428965, + "router_z_loss_clip": 0.16699219, + "router_z_loss_mlp": 0.01989746, + "step": 1788, + "time_per_iteration": 3.2411205768585205 + }, + { + "auxiliary_loss_clip": 0.01142152, + "auxiliary_loss_mlp": 0.01043986, + "balance_loss_clip": 1.04343653, + "balance_loss_mlp": 1.01718807, + "epoch": 0.05191225117520747, + "flos": 19161940066560.0, + "grad_norm": 2.955650773447717, + "language_loss": 0.92402625, + "learning_rate": 3.994967420520204e-06, + "loss": 0.94588763, + "num_input_tokens_seen": 50441310, + "router_z_loss_clip": 0.98583984, + "router_z_loss_mlp": 0.26794434, + "step": 1789, + "time_per_iteration": 2.3953559398651123 + }, + { + "auxiliary_loss_clip": 0.01135836, + "auxiliary_loss_mlp": 0.01049154, + "balance_loss_clip": 1.04256749, + "balance_loss_mlp": 1.02693331, + "epoch": 0.051941268643723525, + "flos": 15589138060800.0, + "grad_norm": 3.5190411596434994, + "language_loss": 0.86168796, + "learning_rate": 3.994954085967407e-06, + "loss": 0.88353789, + "num_input_tokens_seen": 50453875, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.22241211, + "step": 1790, + "time_per_iteration": 2.423271656036377 + }, + { + "auxiliary_loss_clip": 0.01021586, + "auxiliary_loss_mlp": 0.01010011, + "balance_loss_clip": 1.00531411, + "balance_loss_mlp": 1.00799668, + "epoch": 0.05197028611223957, + "flos": 61379193242880.0, + "grad_norm": 0.68940451532403, + "language_loss": 0.51733404, + "learning_rate": 3.994940733794368e-06, + "loss": 0.53764999, + "num_input_tokens_seen": 50507665, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.0201416, + "step": 1791, + "time_per_iteration": 3.0078647136688232 + }, + { + "auxiliary_loss_clip": 0.01139495, + "auxiliary_loss_mlp": 0.01055272, + "balance_loss_clip": 1.04422843, + "balance_loss_mlp": 1.03321862, + "epoch": 0.051999303580755614, + "flos": 17229671936640.0, + "grad_norm": 7.0685867731860395, + "language_loss": 0.83425343, + "learning_rate": 3.9949273640012056e-06, + "loss": 0.85620111, + "num_input_tokens_seen": 50519505, + "router_z_loss_clip": 0.95166016, + "router_z_loss_mlp": 0.22058105, + "step": 1792, + "time_per_iteration": 2.3890089988708496 + }, + { + "auxiliary_loss_clip": 0.01146184, + "auxiliary_loss_mlp": 0.01057411, + "balance_loss_clip": 1.04109311, + "balance_loss_mlp": 1.02883673, + "epoch": 0.05202832104927166, + "flos": 25183200787200.0, + "grad_norm": 3.06167197221701, + "language_loss": 1.03320014, + "learning_rate": 3.994913976588036e-06, + "loss": 1.0552361, + "num_input_tokens_seen": 50535170, + "router_z_loss_clip": 1.05078125, + "router_z_loss_mlp": 0.28613281, + "step": 1793, + "time_per_iteration": 2.4259696006774902 + }, + { + "auxiliary_loss_clip": 0.01140773, + "auxiliary_loss_mlp": 0.01051553, + "balance_loss_clip": 1.04367042, + "balance_loss_mlp": 1.02755618, + "epoch": 0.05205733851778771, + "flos": 11682811036800.0, + "grad_norm": 3.663641885031386, + "language_loss": 0.9790349, + "learning_rate": 3.99490057155498e-06, + "loss": 1.0009582, + "num_input_tokens_seen": 50546945, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.23999023, + "step": 1794, + "time_per_iteration": 2.4521026611328125 + }, + { + "auxiliary_loss_clip": 0.01141946, + "auxiliary_loss_mlp": 0.0105311, + "balance_loss_clip": 1.04257834, + "balance_loss_mlp": 1.02882695, + "epoch": 0.052086355986303755, + "flos": 28576409425920.0, + "grad_norm": 2.1268750616970276, + "language_loss": 0.9056952, + "learning_rate": 3.994887148902155e-06, + "loss": 0.9276458, + "num_input_tokens_seen": 50562205, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.24304199, + "step": 1795, + "time_per_iteration": 2.4754953384399414 + }, + { + "auxiliary_loss_clip": 0.01023327, + "auxiliary_loss_mlp": 0.01014736, + "balance_loss_clip": 1.00659811, + "balance_loss_mlp": 1.01266229, + "epoch": 0.0521153734548198, + "flos": 67031596782720.0, + "grad_norm": 1.5711688910222021, + "language_loss": 0.54050398, + "learning_rate": 3.99487370862968e-06, + "loss": 0.56088459, + "num_input_tokens_seen": 50617275, + "router_z_loss_clip": 0.16796875, + "router_z_loss_mlp": 0.02075195, + "step": 1796, + "time_per_iteration": 3.057661533355713 + }, + { + "auxiliary_loss_clip": 0.01134775, + "auxiliary_loss_mlp": 0.01049109, + "balance_loss_clip": 1.03825402, + "balance_loss_mlp": 1.02587545, + "epoch": 0.05214439092333585, + "flos": 12706464366720.0, + "grad_norm": 11.822020158738384, + "language_loss": 0.87710559, + "learning_rate": 3.994860250737673e-06, + "loss": 0.89894438, + "num_input_tokens_seen": 50630635, + "router_z_loss_clip": 0.96582031, + "router_z_loss_mlp": 0.23242188, + "step": 1797, + "time_per_iteration": 2.379055976867676 + }, + { + "auxiliary_loss_clip": 0.01136265, + "auxiliary_loss_mlp": 0.01046325, + "balance_loss_clip": 1.04151297, + "balance_loss_mlp": 1.02336538, + "epoch": 0.052173408391851896, + "flos": 52220743816320.0, + "grad_norm": 1.999154994057806, + "language_loss": 0.84429407, + "learning_rate": 3.994846775226252e-06, + "loss": 0.86611998, + "num_input_tokens_seen": 50652455, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.22973633, + "step": 1798, + "time_per_iteration": 2.6734256744384766 + }, + { + "auxiliary_loss_clip": 0.01148716, + "auxiliary_loss_mlp": 0.01065801, + "balance_loss_clip": 1.04388762, + "balance_loss_mlp": 1.03837121, + "epoch": 0.05220242586036794, + "flos": 23688638772480.0, + "grad_norm": 4.722025137498424, + "language_loss": 1.17760575, + "learning_rate": 3.9948332820955365e-06, + "loss": 1.1997509, + "num_input_tokens_seen": 50668495, + "router_z_loss_clip": 1.04785156, + "router_z_loss_mlp": 0.27416992, + "step": 1799, + "time_per_iteration": 2.4764602184295654 + }, + { + "auxiliary_loss_clip": 0.01138468, + "auxiliary_loss_mlp": 0.01044399, + "balance_loss_clip": 1.04048097, + "balance_loss_mlp": 1.01835227, + "epoch": 0.052231443328883985, + "flos": 26936189752320.0, + "grad_norm": 3.762133488617657, + "language_loss": 0.80374849, + "learning_rate": 3.994819771345648e-06, + "loss": 0.82557714, + "num_input_tokens_seen": 50686755, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.26013184, + "step": 1800, + "time_per_iteration": 2.441899538040161 + }, + { + "auxiliary_loss_clip": 0.01141047, + "auxiliary_loss_mlp": 0.01053137, + "balance_loss_clip": 1.04112673, + "balance_loss_mlp": 1.02560008, + "epoch": 0.052260460797400037, + "flos": 19460132922240.0, + "grad_norm": 4.710300322647216, + "language_loss": 0.75572646, + "learning_rate": 3.994806242976703e-06, + "loss": 0.77766824, + "num_input_tokens_seen": 50703635, + "router_z_loss_clip": 0.99951172, + "router_z_loss_mlp": 0.27526855, + "step": 1801, + "time_per_iteration": 2.449686050415039 + }, + { + "auxiliary_loss_clip": 0.01022455, + "auxiliary_loss_mlp": 0.01006843, + "balance_loss_clip": 1.00593555, + "balance_loss_mlp": 1.0048281, + "epoch": 0.05228947826591608, + "flos": 57136408646400.0, + "grad_norm": 0.683003227942452, + "language_loss": 0.4885177, + "learning_rate": 3.994792696988822e-06, + "loss": 0.50881064, + "num_input_tokens_seen": 50759755, + "router_z_loss_clip": 0.16503906, + "router_z_loss_mlp": 0.0201416, + "step": 1802, + "time_per_iteration": 2.9621169567108154 + }, + { + "auxiliary_loss_clip": 0.0102083, + "auxiliary_loss_mlp": 0.01002829, + "balance_loss_clip": 1.00416946, + "balance_loss_mlp": 1.00094581, + "epoch": 0.052318495734432126, + "flos": 69938813162880.0, + "grad_norm": 0.674210635496312, + "language_loss": 0.51421404, + "learning_rate": 3.994779133382125e-06, + "loss": 0.53445059, + "num_input_tokens_seen": 50823880, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01879883, + "step": 1803, + "time_per_iteration": 3.199887275695801 + }, + { + "auxiliary_loss_clip": 0.01019683, + "auxiliary_loss_mlp": 0.01002085, + "balance_loss_clip": 1.00316763, + "balance_loss_mlp": 1.00009418, + "epoch": 0.05234751320294818, + "flos": 66095958723840.0, + "grad_norm": 0.6493896819572089, + "language_loss": 0.5220198, + "learning_rate": 3.994765552156731e-06, + "loss": 0.54223746, + "num_input_tokens_seen": 50888845, + "router_z_loss_clip": 0.16503906, + "router_z_loss_mlp": 0.01989746, + "step": 1804, + "time_per_iteration": 3.259584665298462 + }, + { + "auxiliary_loss_clip": 0.01153506, + "auxiliary_loss_mlp": 0.01058433, + "balance_loss_clip": 1.04695773, + "balance_loss_mlp": 1.03231382, + "epoch": 0.05237653067146422, + "flos": 32334844464000.0, + "grad_norm": 2.702625982005808, + "language_loss": 0.74472094, + "learning_rate": 3.994751953312762e-06, + "loss": 0.76684034, + "num_input_tokens_seen": 50903530, + "router_z_loss_clip": 1.06494141, + "router_z_loss_mlp": 0.26123047, + "step": 1805, + "time_per_iteration": 2.467315435409546 + }, + { + "auxiliary_loss_clip": 0.01017968, + "auxiliary_loss_mlp": 0.01005526, + "balance_loss_clip": 1.00130177, + "balance_loss_mlp": 1.00355911, + "epoch": 0.05240554813998027, + "flos": 72436849741440.0, + "grad_norm": 0.6187539515924906, + "language_loss": 0.51633602, + "learning_rate": 3.994738336850336e-06, + "loss": 0.53657097, + "num_input_tokens_seen": 50970050, + "router_z_loss_clip": 0.16699219, + "router_z_loss_mlp": 0.01965332, + "step": 1806, + "time_per_iteration": 3.0961761474609375 + }, + { + "auxiliary_loss_clip": 0.01126266, + "auxiliary_loss_mlp": 0.01048288, + "balance_loss_clip": 1.03980422, + "balance_loss_mlp": 1.02712882, + "epoch": 0.05243456560849631, + "flos": 30181715873280.0, + "grad_norm": 2.9197395112636837, + "language_loss": 0.8890022, + "learning_rate": 3.994724702769573e-06, + "loss": 0.91074777, + "num_input_tokens_seen": 50984425, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.21154785, + "step": 1807, + "time_per_iteration": 2.4745006561279297 + }, + { + "auxiliary_loss_clip": 0.01140846, + "auxiliary_loss_mlp": 0.01058289, + "balance_loss_clip": 1.03823948, + "balance_loss_mlp": 1.03040624, + "epoch": 0.05246358307701236, + "flos": 23651945066880.0, + "grad_norm": 2.424817158453865, + "language_loss": 0.75270438, + "learning_rate": 3.994711051070595e-06, + "loss": 0.77469575, + "num_input_tokens_seen": 51001550, + "router_z_loss_clip": 1.02587891, + "router_z_loss_mlp": 0.27880859, + "step": 1808, + "time_per_iteration": 2.4120140075683594 + }, + { + "auxiliary_loss_clip": 0.01020575, + "auxiliary_loss_mlp": 0.01003032, + "balance_loss_clip": 1.00370646, + "balance_loss_mlp": 1.00117254, + "epoch": 0.05249260054552841, + "flos": 65911477766400.0, + "grad_norm": 0.7410102832696666, + "language_loss": 0.54270995, + "learning_rate": 3.99469738175352e-06, + "loss": 0.56294608, + "num_input_tokens_seen": 51062220, + "router_z_loss_clip": 0.16796875, + "router_z_loss_mlp": 0.01855469, + "step": 1809, + "time_per_iteration": 2.990540027618408 + }, + { + "auxiliary_loss_clip": 0.01020363, + "auxiliary_loss_mlp": 0.01002008, + "balance_loss_clip": 1.00400996, + "balance_loss_mlp": 1.00010061, + "epoch": 0.05252161801404445, + "flos": 64667278177920.0, + "grad_norm": 0.647702478707502, + "language_loss": 0.51106095, + "learning_rate": 3.994683694818472e-06, + "loss": 0.53128469, + "num_input_tokens_seen": 51127685, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01904297, + "step": 1810, + "time_per_iteration": 3.087097644805908 + }, + { + "auxiliary_loss_clip": 0.01136582, + "auxiliary_loss_mlp": 0.01048031, + "balance_loss_clip": 1.03943479, + "balance_loss_mlp": 1.02359319, + "epoch": 0.052550635482560504, + "flos": 22267010321280.0, + "grad_norm": 3.000961183156839, + "language_loss": 0.95693183, + "learning_rate": 3.994669990265571e-06, + "loss": 0.97877795, + "num_input_tokens_seen": 51150190, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.24438477, + "step": 1811, + "time_per_iteration": 2.66988468170166 + }, + { + "auxiliary_loss_clip": 0.0115051, + "auxiliary_loss_mlp": 0.01056709, + "balance_loss_clip": 1.04254305, + "balance_loss_mlp": 1.0273478, + "epoch": 0.05257965295107655, + "flos": 34086820999680.0, + "grad_norm": 2.8405050444028, + "language_loss": 0.95123464, + "learning_rate": 3.994656268094937e-06, + "loss": 0.97330683, + "num_input_tokens_seen": 51164500, + "router_z_loss_clip": 1.08154297, + "router_z_loss_mlp": 0.29370117, + "step": 1812, + "time_per_iteration": 2.5466175079345703 + }, + { + "auxiliary_loss_clip": 0.01136027, + "auxiliary_loss_mlp": 0.01053008, + "balance_loss_clip": 1.04009891, + "balance_loss_mlp": 1.02866566, + "epoch": 0.05260867041959259, + "flos": 25947484560000.0, + "grad_norm": 3.3661121970874843, + "language_loss": 1.0808655, + "learning_rate": 3.994642528306691e-06, + "loss": 1.1027559, + "num_input_tokens_seen": 51176405, + "router_z_loss_clip": 0.96044922, + "router_z_loss_mlp": 0.2434082, + "step": 1813, + "time_per_iteration": 2.4215335845947266 + }, + { + "auxiliary_loss_clip": 0.0115131, + "auxiliary_loss_mlp": 0.01067722, + "balance_loss_clip": 1.04550028, + "balance_loss_mlp": 1.04062557, + "epoch": 0.052637687888108645, + "flos": 11463137562240.0, + "grad_norm": 3.1635030677290463, + "language_loss": 1.09294152, + "learning_rate": 3.994628770900956e-06, + "loss": 1.11513186, + "num_input_tokens_seen": 51186080, + "router_z_loss_clip": 1.05859375, + "router_z_loss_mlp": 0.27111816, + "step": 1814, + "time_per_iteration": 2.3839125633239746 + }, + { + "auxiliary_loss_clip": 0.01144823, + "auxiliary_loss_mlp": 0.01059496, + "balance_loss_clip": 1.04367197, + "balance_loss_mlp": 1.03104079, + "epoch": 0.05266670535662469, + "flos": 33179881944960.0, + "grad_norm": 2.583417185553397, + "language_loss": 0.8755188, + "learning_rate": 3.994614995877852e-06, + "loss": 0.89756203, + "num_input_tokens_seen": 51209075, + "router_z_loss_clip": 1.01074219, + "router_z_loss_mlp": 0.2845459, + "step": 1815, + "time_per_iteration": 2.491487741470337 + }, + { + "auxiliary_loss_clip": 0.01147924, + "auxiliary_loss_mlp": 0.0105158, + "balance_loss_clip": 1.04630661, + "balance_loss_mlp": 1.0255444, + "epoch": 0.052695722825140734, + "flos": 18909238579200.0, + "grad_norm": 2.813845002207085, + "language_loss": 0.89677787, + "learning_rate": 3.994601203237501e-06, + "loss": 0.91877294, + "num_input_tokens_seen": 51221510, + "router_z_loss_clip": 1.01708984, + "router_z_loss_mlp": 0.26025391, + "step": 1816, + "time_per_iteration": 2.396700859069824 + }, + { + "auxiliary_loss_clip": 0.01135288, + "auxiliary_loss_mlp": 0.01044902, + "balance_loss_clip": 1.03940034, + "balance_loss_mlp": 1.01995206, + "epoch": 0.05272474029365678, + "flos": 13108454294400.0, + "grad_norm": 2.706233085675434, + "language_loss": 0.87266612, + "learning_rate": 3.994587392980026e-06, + "loss": 0.89446807, + "num_input_tokens_seen": 51234150, + "router_z_loss_clip": 0.96044922, + "router_z_loss_mlp": 0.24963379, + "step": 1817, + "time_per_iteration": 2.5842795372009277 + }, + { + "auxiliary_loss_clip": 0.01149797, + "auxiliary_loss_mlp": 0.01062784, + "balance_loss_clip": 1.04518986, + "balance_loss_mlp": 1.03375649, + "epoch": 0.05275375776217283, + "flos": 14128756133760.0, + "grad_norm": 3.182116942338578, + "language_loss": 0.94358665, + "learning_rate": 3.9945735651055475e-06, + "loss": 0.96571249, + "num_input_tokens_seen": 51245270, + "router_z_loss_clip": 1.04589844, + "router_z_loss_mlp": 0.29040527, + "step": 1818, + "time_per_iteration": 2.539299964904785 + }, + { + "auxiliary_loss_clip": 0.01136567, + "auxiliary_loss_mlp": 0.01051056, + "balance_loss_clip": 1.04006124, + "balance_loss_mlp": 1.02466285, + "epoch": 0.052782775230688875, + "flos": 39017464669440.0, + "grad_norm": 2.5805473436891004, + "language_loss": 0.81615168, + "learning_rate": 3.994559719614189e-06, + "loss": 0.83802789, + "num_input_tokens_seen": 51270730, + "router_z_loss_clip": 0.96533203, + "router_z_loss_mlp": 0.26416016, + "step": 1819, + "time_per_iteration": 2.6288299560546875 + }, + { + "auxiliary_loss_clip": 0.01023387, + "auxiliary_loss_mlp": 0.01004822, + "balance_loss_clip": 1.00712967, + "balance_loss_mlp": 1.00324869, + "epoch": 0.05281179269920492, + "flos": 74787760517760.0, + "grad_norm": 0.6753641750368817, + "language_loss": 0.48112506, + "learning_rate": 3.99454585650607e-06, + "loss": 0.50140715, + "num_input_tokens_seen": 51339545, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.01574707, + "step": 1820, + "time_per_iteration": 3.2329885959625244 + }, + { + "auxiliary_loss_clip": 0.01021244, + "auxiliary_loss_mlp": 0.01002583, + "balance_loss_clip": 1.0050354, + "balance_loss_mlp": 1.00102162, + "epoch": 0.05284081016772097, + "flos": 67227039907200.0, + "grad_norm": 0.7177684833545321, + "language_loss": 0.52185524, + "learning_rate": 3.994531975781316e-06, + "loss": 0.54209352, + "num_input_tokens_seen": 51407870, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.015625, + "step": 1821, + "time_per_iteration": 3.119499683380127 + }, + { + "auxiliary_loss_clip": 0.01139848, + "auxiliary_loss_mlp": 0.01056913, + "balance_loss_clip": 1.04454052, + "balance_loss_mlp": 1.03313041, + "epoch": 0.052869827636237016, + "flos": 16390534475520.0, + "grad_norm": 2.8452434331062832, + "language_loss": 0.98522329, + "learning_rate": 3.994518077440049e-06, + "loss": 1.00719094, + "num_input_tokens_seen": 51419305, + "router_z_loss_clip": 0.95117188, + "router_z_loss_mlp": 0.23754883, + "step": 1822, + "time_per_iteration": 2.4310576915740967 + }, + { + "auxiliary_loss_clip": 0.01138103, + "auxiliary_loss_mlp": 0.01055417, + "balance_loss_clip": 1.03759205, + "balance_loss_mlp": 1.02799892, + "epoch": 0.05289884510475306, + "flos": 32263237532160.0, + "grad_norm": 2.2400616281671186, + "language_loss": 0.75722319, + "learning_rate": 3.99450416148239e-06, + "loss": 0.77915841, + "num_input_tokens_seen": 51435855, + "router_z_loss_clip": 1.00439453, + "router_z_loss_mlp": 0.27429199, + "step": 1823, + "time_per_iteration": 2.505462169647217 + }, + { + "auxiliary_loss_clip": 0.01144804, + "auxiliary_loss_mlp": 0.01062434, + "balance_loss_clip": 1.04301357, + "balance_loss_mlp": 1.03592181, + "epoch": 0.052927862573269105, + "flos": 42040943343360.0, + "grad_norm": 3.5436194300635857, + "language_loss": 1.03492856, + "learning_rate": 3.994490227908464e-06, + "loss": 1.05700099, + "num_input_tokens_seen": 51455110, + "router_z_loss_clip": 1.01757812, + "router_z_loss_mlp": 0.26525879, + "step": 1824, + "time_per_iteration": 2.5767152309417725 + }, + { + "auxiliary_loss_clip": 0.01020131, + "auxiliary_loss_mlp": 0.01003452, + "balance_loss_clip": 1.0040288, + "balance_loss_mlp": 1.00196159, + "epoch": 0.052956880041785157, + "flos": 58052319920640.0, + "grad_norm": 0.7093276195378693, + "language_loss": 0.51764941, + "learning_rate": 3.994476276718394e-06, + "loss": 0.53788531, + "num_input_tokens_seen": 51512655, + "router_z_loss_clip": 0.16113281, + "router_z_loss_mlp": 0.01489258, + "step": 1825, + "time_per_iteration": 2.8795793056488037 + }, + { + "auxiliary_loss_clip": 0.01150898, + "auxiliary_loss_mlp": 0.0105674, + "balance_loss_clip": 1.04321194, + "balance_loss_mlp": 1.02803397, + "epoch": 0.0529858975103012, + "flos": 33804582635520.0, + "grad_norm": 1.8567342791034347, + "language_loss": 0.99006611, + "learning_rate": 3.9944623079123004e-06, + "loss": 1.01214254, + "num_input_tokens_seen": 51534775, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.28710938, + "step": 1826, + "time_per_iteration": 5.072173833847046 + }, + { + "auxiliary_loss_clip": 0.01145481, + "auxiliary_loss_mlp": 0.01049574, + "balance_loss_clip": 1.04635096, + "balance_loss_mlp": 1.02294302, + "epoch": 0.053014914978817246, + "flos": 25840650199680.0, + "grad_norm": 1.9374284507746518, + "language_loss": 0.89593637, + "learning_rate": 3.99444832149031e-06, + "loss": 0.91788691, + "num_input_tokens_seen": 51553795, + "router_z_loss_clip": 0.99169922, + "router_z_loss_mlp": 0.26647949, + "step": 1827, + "time_per_iteration": 2.4939374923706055 + }, + { + "auxiliary_loss_clip": 0.01128607, + "auxiliary_loss_mlp": 0.01044691, + "balance_loss_clip": 1.04034936, + "balance_loss_mlp": 1.0230428, + "epoch": 0.0530439324473333, + "flos": 38466046656000.0, + "grad_norm": 2.287056054815108, + "language_loss": 1.05499852, + "learning_rate": 3.994434317452545e-06, + "loss": 1.07673156, + "num_input_tokens_seen": 51572940, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.21655273, + "step": 1828, + "time_per_iteration": 2.5405383110046387 + }, + { + "auxiliary_loss_clip": 0.01146379, + "auxiliary_loss_mlp": 0.01050805, + "balance_loss_clip": 1.04353023, + "balance_loss_mlp": 1.02646208, + "epoch": 0.05307294991584934, + "flos": 51086974458240.0, + "grad_norm": 3.205346718715488, + "language_loss": 1.01425278, + "learning_rate": 3.994420295799129e-06, + "loss": 1.0362246, + "num_input_tokens_seen": 51591620, + "router_z_loss_clip": 1.02734375, + "router_z_loss_mlp": 0.24353027, + "step": 1829, + "time_per_iteration": 2.916975975036621 + }, + { + "auxiliary_loss_clip": 0.01142894, + "auxiliary_loss_mlp": 0.01045485, + "balance_loss_clip": 1.04175067, + "balance_loss_mlp": 1.02121425, + "epoch": 0.05310196738436539, + "flos": 21432725539200.0, + "grad_norm": 1.9861741810461366, + "language_loss": 0.88942289, + "learning_rate": 3.994406256530185e-06, + "loss": 0.91130674, + "num_input_tokens_seen": 51607455, + "router_z_loss_clip": 1.01025391, + "router_z_loss_mlp": 0.24279785, + "step": 1830, + "time_per_iteration": 4.687759637832642 + }, + { + "auxiliary_loss_clip": 0.01135436, + "auxiliary_loss_mlp": 0.01050646, + "balance_loss_clip": 1.03895426, + "balance_loss_mlp": 1.02571917, + "epoch": 0.05313098485288143, + "flos": 17888552714880.0, + "grad_norm": 2.0244065989658355, + "language_loss": 0.7495271, + "learning_rate": 3.9943921996458385e-06, + "loss": 0.77138793, + "num_input_tokens_seen": 51623000, + "router_z_loss_clip": 0.96484375, + "router_z_loss_mlp": 0.24902344, + "step": 1831, + "time_per_iteration": 2.3456432819366455 + }, + { + "auxiliary_loss_clip": 0.01022062, + "auxiliary_loss_mlp": 0.01003611, + "balance_loss_clip": 1.00502181, + "balance_loss_mlp": 1.00195408, + "epoch": 0.05316000232139748, + "flos": 74777845691520.0, + "grad_norm": 0.6631426703563252, + "language_loss": 0.51137513, + "learning_rate": 3.9943781251462135e-06, + "loss": 0.53163189, + "num_input_tokens_seen": 51685760, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01660156, + "step": 1832, + "time_per_iteration": 3.1267430782318115 + }, + { + "auxiliary_loss_clip": 0.01020649, + "auxiliary_loss_mlp": 0.01001505, + "balance_loss_clip": 1.00366962, + "balance_loss_mlp": 0.99982411, + "epoch": 0.05318901978991353, + "flos": 62588130491520.0, + "grad_norm": 0.6771362166513926, + "language_loss": 0.54907846, + "learning_rate": 3.994364033031433e-06, + "loss": 0.5693, + "num_input_tokens_seen": 51750160, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.0168457, + "step": 1833, + "time_per_iteration": 3.0295891761779785 + }, + { + "auxiliary_loss_clip": 0.01018526, + "auxiliary_loss_mlp": 0.01001873, + "balance_loss_clip": 1.00203872, + "balance_loss_mlp": 1.00022769, + "epoch": 0.05321803725842957, + "flos": 71415151447680.0, + "grad_norm": 0.6839841298453379, + "language_loss": 0.52043927, + "learning_rate": 3.994349923301623e-06, + "loss": 0.54064322, + "num_input_tokens_seen": 51812750, + "router_z_loss_clip": 0.16503906, + "router_z_loss_mlp": 0.01647949, + "step": 1834, + "time_per_iteration": 8.022744178771973 + }, + { + "auxiliary_loss_clip": 0.01145858, + "auxiliary_loss_mlp": 0.01056341, + "balance_loss_clip": 1.04072881, + "balance_loss_mlp": 1.02804112, + "epoch": 0.053247054726945624, + "flos": 29488550273280.0, + "grad_norm": 2.170050168180209, + "language_loss": 0.96079838, + "learning_rate": 3.994335795956907e-06, + "loss": 0.98282039, + "num_input_tokens_seen": 51830770, + "router_z_loss_clip": 1.04980469, + "router_z_loss_mlp": 0.28308105, + "step": 1835, + "time_per_iteration": 2.5537450313568115 + }, + { + "auxiliary_loss_clip": 0.01141738, + "auxiliary_loss_mlp": 0.01061684, + "balance_loss_clip": 1.0429697, + "balance_loss_mlp": 1.03735352, + "epoch": 0.05327607219546167, + "flos": 20886020559360.0, + "grad_norm": 4.201103016129279, + "language_loss": 0.99972057, + "learning_rate": 3.9943216509974105e-06, + "loss": 1.02175486, + "num_input_tokens_seen": 51849715, + "router_z_loss_clip": 0.98779297, + "router_z_loss_mlp": 0.24353027, + "step": 1836, + "time_per_iteration": 2.386125087738037 + }, + { + "auxiliary_loss_clip": 0.01145252, + "auxiliary_loss_mlp": 0.01048405, + "balance_loss_clip": 1.04231203, + "balance_loss_mlp": 1.02126098, + "epoch": 0.05330508966397771, + "flos": 20077188024960.0, + "grad_norm": 3.1092331692370356, + "language_loss": 0.83092356, + "learning_rate": 3.994307488423258e-06, + "loss": 0.85286009, + "num_input_tokens_seen": 51862980, + "router_z_loss_clip": 1.03125, + "router_z_loss_mlp": 0.27148438, + "step": 1837, + "time_per_iteration": 2.4332852363586426 + }, + { + "auxiliary_loss_clip": 0.01147608, + "auxiliary_loss_mlp": 0.01051838, + "balance_loss_clip": 1.04191875, + "balance_loss_mlp": 1.02203596, + "epoch": 0.053334107132493765, + "flos": 30693298158720.0, + "grad_norm": 2.332393435821677, + "language_loss": 1.050722, + "learning_rate": 3.994293308234575e-06, + "loss": 1.07271647, + "num_input_tokens_seen": 51880780, + "router_z_loss_clip": 1.05761719, + "router_z_loss_mlp": 0.29821777, + "step": 1838, + "time_per_iteration": 2.4704437255859375 + }, + { + "auxiliary_loss_clip": 0.01149546, + "auxiliary_loss_mlp": 0.01061311, + "balance_loss_clip": 1.04221261, + "balance_loss_mlp": 1.03355908, + "epoch": 0.05336312460100981, + "flos": 22812807605760.0, + "grad_norm": 3.1885000585637284, + "language_loss": 1.04989386, + "learning_rate": 3.994279110431487e-06, + "loss": 1.07200241, + "num_input_tokens_seen": 51894595, + "router_z_loss_clip": 1.07324219, + "router_z_loss_mlp": 0.27783203, + "step": 1839, + "time_per_iteration": 2.7259271144866943 + }, + { + "auxiliary_loss_clip": 0.01031761, + "auxiliary_loss_mlp": 0.01017199, + "balance_loss_clip": 1.01384676, + "balance_loss_mlp": 1.01541102, + "epoch": 0.053392142069525854, + "flos": 74769850990080.0, + "grad_norm": 0.6618869763542368, + "language_loss": 0.54439974, + "learning_rate": 3.994264895014118e-06, + "loss": 0.56488937, + "num_input_tokens_seen": 51958645, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.01782227, + "step": 1840, + "time_per_iteration": 3.084192991256714 + }, + { + "auxiliary_loss_clip": 0.01147965, + "auxiliary_loss_mlp": 0.01067399, + "balance_loss_clip": 1.04341149, + "balance_loss_mlp": 1.04277015, + "epoch": 0.0534211595380419, + "flos": 37407619745280.0, + "grad_norm": 2.849909268758771, + "language_loss": 0.78420138, + "learning_rate": 3.994250661982594e-06, + "loss": 0.80635506, + "num_input_tokens_seen": 51976750, + "router_z_loss_clip": 1.04638672, + "router_z_loss_mlp": 0.24633789, + "step": 1841, + "time_per_iteration": 2.5519139766693115 + }, + { + "auxiliary_loss_clip": 0.01022453, + "auxiliary_loss_mlp": 0.01002353, + "balance_loss_clip": 1.00522912, + "balance_loss_mlp": 1.00076723, + "epoch": 0.05345017700655795, + "flos": 74777740957440.0, + "grad_norm": 0.6194912789332426, + "language_loss": 0.528566, + "learning_rate": 3.994236411337043e-06, + "loss": 0.54881406, + "num_input_tokens_seen": 52041235, + "router_z_loss_clip": 0.171875, + "router_z_loss_mlp": 0.01586914, + "step": 1842, + "time_per_iteration": 3.1311395168304443 + }, + { + "auxiliary_loss_clip": 0.01019407, + "auxiliary_loss_mlp": 0.01009585, + "balance_loss_clip": 1.002496, + "balance_loss_mlp": 1.00791574, + "epoch": 0.053479194475073995, + "flos": 65357231932800.0, + "grad_norm": 0.6686137237041861, + "language_loss": 0.50808954, + "learning_rate": 3.994222143077587e-06, + "loss": 0.52837944, + "num_input_tokens_seen": 52106420, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01672363, + "step": 1843, + "time_per_iteration": 3.0275349617004395 + }, + { + "auxiliary_loss_clip": 0.01132195, + "auxiliary_loss_mlp": 0.01048082, + "balance_loss_clip": 1.03933513, + "balance_loss_mlp": 1.02593303, + "epoch": 0.05350821194359004, + "flos": 11581038823680.0, + "grad_norm": 2.9259639874082084, + "language_loss": 0.86998081, + "learning_rate": 3.994207857204355e-06, + "loss": 0.8917836, + "num_input_tokens_seen": 52119330, + "router_z_loss_clip": 0.92871094, + "router_z_loss_mlp": 0.22143555, + "step": 1844, + "time_per_iteration": 2.44954514503479 + }, + { + "auxiliary_loss_clip": 0.01153505, + "auxiliary_loss_mlp": 0.01061505, + "balance_loss_clip": 1.047014, + "balance_loss_mlp": 1.03667402, + "epoch": 0.05353722941210609, + "flos": 11320028432640.0, + "grad_norm": 3.1338623647733823, + "language_loss": 0.67970407, + "learning_rate": 3.994193553717472e-06, + "loss": 0.70185423, + "num_input_tokens_seen": 52131915, + "router_z_loss_clip": 1.06494141, + "router_z_loss_mlp": 0.24804688, + "step": 1845, + "time_per_iteration": 2.34299373626709 + }, + { + "auxiliary_loss_clip": 0.01133197, + "auxiliary_loss_mlp": 0.01044436, + "balance_loss_clip": 1.04012156, + "balance_loss_mlp": 1.01996255, + "epoch": 0.053566246880622136, + "flos": 19421414357760.0, + "grad_norm": 2.5647726445983343, + "language_loss": 0.8140142, + "learning_rate": 3.994179232617065e-06, + "loss": 0.83579051, + "num_input_tokens_seen": 52145425, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.24487305, + "step": 1846, + "time_per_iteration": 2.421173095703125 + }, + { + "auxiliary_loss_clip": 0.01148747, + "auxiliary_loss_mlp": 0.01051763, + "balance_loss_clip": 1.04459214, + "balance_loss_mlp": 1.02432132, + "epoch": 0.05359526434913818, + "flos": 35818232878080.0, + "grad_norm": 5.123290249718465, + "language_loss": 0.82901335, + "learning_rate": 3.994164893903259e-06, + "loss": 0.85101849, + "num_input_tokens_seen": 52165995, + "router_z_loss_clip": 1.04199219, + "router_z_loss_mlp": 0.27441406, + "step": 1847, + "time_per_iteration": 2.5648584365844727 + }, + { + "auxiliary_loss_clip": 0.01142103, + "auxiliary_loss_mlp": 0.01053393, + "balance_loss_clip": 1.04629922, + "balance_loss_mlp": 1.02894306, + "epoch": 0.053624281817654225, + "flos": 14530082745600.0, + "grad_norm": 3.6985169995870826, + "language_loss": 0.90467256, + "learning_rate": 3.9941505375761826e-06, + "loss": 0.92662746, + "num_input_tokens_seen": 52179215, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.24450684, + "step": 1848, + "time_per_iteration": 2.4004766941070557 + }, + { + "auxiliary_loss_clip": 0.01143161, + "auxiliary_loss_mlp": 0.01049517, + "balance_loss_clip": 1.04593503, + "balance_loss_mlp": 1.02636647, + "epoch": 0.053653299286170276, + "flos": 27592766380800.0, + "grad_norm": 2.525297260758306, + "language_loss": 0.79957485, + "learning_rate": 3.994136163635962e-06, + "loss": 0.82150161, + "num_input_tokens_seen": 52195110, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.23156738, + "step": 1849, + "time_per_iteration": 2.4483070373535156 + }, + { + "auxiliary_loss_clip": 0.0104758, + "auxiliary_loss_mlp": 0.01007582, + "balance_loss_clip": 1.02982795, + "balance_loss_mlp": 1.00591278, + "epoch": 0.05368231675468632, + "flos": 74770723774080.0, + "grad_norm": 0.6832955930275053, + "language_loss": 0.52204227, + "learning_rate": 3.994121772082724e-06, + "loss": 0.5425939, + "num_input_tokens_seen": 52264025, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.01672363, + "step": 1850, + "time_per_iteration": 3.179323434829712 + }, + { + "auxiliary_loss_clip": 0.01149385, + "auxiliary_loss_mlp": 0.01061029, + "balance_loss_clip": 1.05366576, + "balance_loss_mlp": 1.03668678, + "epoch": 0.053711334223202366, + "flos": 31968640546560.0, + "grad_norm": 3.134154493886561, + "language_loss": 0.86888421, + "learning_rate": 3.9941073629165945e-06, + "loss": 0.89098835, + "num_input_tokens_seen": 52278120, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.24346924, + "step": 1851, + "time_per_iteration": 2.4912798404693604 + }, + { + "auxiliary_loss_clip": 0.01133867, + "auxiliary_loss_mlp": 0.01048359, + "balance_loss_clip": 1.04142034, + "balance_loss_mlp": 1.02367055, + "epoch": 0.05374035169171842, + "flos": 30546593159040.0, + "grad_norm": 2.821550358441472, + "language_loss": 0.89017522, + "learning_rate": 3.994092936137702e-06, + "loss": 0.91199738, + "num_input_tokens_seen": 52295000, + "router_z_loss_clip": 0.92626953, + "router_z_loss_mlp": 0.24682617, + "step": 1852, + "time_per_iteration": 2.8658201694488525 + }, + { + "auxiliary_loss_clip": 0.01029617, + "auxiliary_loss_mlp": 0.01003967, + "balance_loss_clip": 1.01215899, + "balance_loss_mlp": 1.00233352, + "epoch": 0.05376936916023446, + "flos": 64046452648320.0, + "grad_norm": 0.7389787122444851, + "language_loss": 0.55380511, + "learning_rate": 3.994078491746175e-06, + "loss": 0.57414103, + "num_input_tokens_seen": 52351105, + "router_z_loss_clip": 0.17382812, + "router_z_loss_mlp": 0.01635742, + "step": 1853, + "time_per_iteration": 2.9276084899902344 + }, + { + "auxiliary_loss_clip": 0.01024034, + "auxiliary_loss_mlp": 0.01003096, + "balance_loss_clip": 1.00695527, + "balance_loss_mlp": 1.0014149, + "epoch": 0.05379838662875051, + "flos": 64741992220800.0, + "grad_norm": 0.6804771245881662, + "language_loss": 0.49825576, + "learning_rate": 3.994064029742138e-06, + "loss": 0.51852703, + "num_input_tokens_seen": 52413490, + "router_z_loss_clip": 0.17089844, + "router_z_loss_mlp": 0.0168457, + "step": 1854, + "time_per_iteration": 3.071674108505249 + }, + { + "auxiliary_loss_clip": 0.01136046, + "auxiliary_loss_mlp": 0.0104864, + "balance_loss_clip": 1.04200244, + "balance_loss_mlp": 1.02364194, + "epoch": 0.05382740409726655, + "flos": 66814613349120.0, + "grad_norm": 3.3731892962586385, + "language_loss": 0.67901784, + "learning_rate": 3.994049550125722e-06, + "loss": 0.70086467, + "num_input_tokens_seen": 52435935, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.25, + "step": 1855, + "time_per_iteration": 2.8716447353363037 + }, + { + "auxiliary_loss_clip": 0.01125354, + "auxiliary_loss_mlp": 0.01043768, + "balance_loss_clip": 1.0381279, + "balance_loss_mlp": 1.02408636, + "epoch": 0.0538564215657826, + "flos": 28832427492480.0, + "grad_norm": 2.326705313061191, + "language_loss": 0.63823128, + "learning_rate": 3.994035052897053e-06, + "loss": 0.65992248, + "num_input_tokens_seen": 52449910, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.19689941, + "step": 1856, + "time_per_iteration": 2.468017101287842 + }, + { + "auxiliary_loss_clip": 0.01025311, + "auxiliary_loss_mlp": 0.01008402, + "balance_loss_clip": 1.00753736, + "balance_loss_mlp": 1.0067215, + "epoch": 0.05388543903429865, + "flos": 74776030300800.0, + "grad_norm": 0.7528014312375523, + "language_loss": 0.55759442, + "learning_rate": 3.99402053805626e-06, + "loss": 0.57793152, + "num_input_tokens_seen": 52512785, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.0168457, + "step": 1857, + "time_per_iteration": 3.077566146850586 + }, + { + "auxiliary_loss_clip": 0.01028626, + "auxiliary_loss_mlp": 0.01007706, + "balance_loss_clip": 1.01092625, + "balance_loss_mlp": 1.00597775, + "epoch": 0.05391445650281469, + "flos": 59375038890240.0, + "grad_norm": 0.63563363793272, + "language_loss": 0.49088493, + "learning_rate": 3.99400600560347e-06, + "loss": 0.51124823, + "num_input_tokens_seen": 52579040, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.01733398, + "step": 1858, + "time_per_iteration": 3.161036252975464 + }, + { + "auxiliary_loss_clip": 0.01141625, + "auxiliary_loss_mlp": 0.01056287, + "balance_loss_clip": 1.04318547, + "balance_loss_mlp": 1.03147936, + "epoch": 0.053943473971330744, + "flos": 11832029654400.0, + "grad_norm": 4.475797849690109, + "language_loss": 0.78916639, + "learning_rate": 3.993991455538812e-06, + "loss": 0.81114548, + "num_input_tokens_seen": 52590510, + "router_z_loss_clip": 0.984375, + "router_z_loss_mlp": 0.24780273, + "step": 1859, + "time_per_iteration": 2.426107168197632 + }, + { + "auxiliary_loss_clip": 0.01138013, + "auxiliary_loss_mlp": 0.01047035, + "balance_loss_clip": 1.04315555, + "balance_loss_mlp": 1.02493358, + "epoch": 0.05397249143984679, + "flos": 16280418447360.0, + "grad_norm": 2.4839538544582997, + "language_loss": 0.72285676, + "learning_rate": 3.993976887862415e-06, + "loss": 0.74470723, + "num_input_tokens_seen": 52602255, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.22106934, + "step": 1860, + "time_per_iteration": 2.457700490951538 + }, + { + "auxiliary_loss_clip": 0.01033088, + "auxiliary_loss_mlp": 0.01009867, + "balance_loss_clip": 1.01535916, + "balance_loss_mlp": 1.00825787, + "epoch": 0.05400150890836283, + "flos": 59706155024640.0, + "grad_norm": 0.6595417978516218, + "language_loss": 0.4996171, + "learning_rate": 3.993962302574407e-06, + "loss": 0.52004665, + "num_input_tokens_seen": 52664805, + "router_z_loss_clip": 0.17773438, + "router_z_loss_mlp": 0.01611328, + "step": 1861, + "time_per_iteration": 3.073176145553589 + }, + { + "auxiliary_loss_clip": 0.01140866, + "auxiliary_loss_mlp": 0.01055019, + "balance_loss_clip": 1.04214239, + "balance_loss_mlp": 1.02942514, + "epoch": 0.05403052637687888, + "flos": 16355167401600.0, + "grad_norm": 3.4280474975669177, + "language_loss": 0.83219784, + "learning_rate": 3.993947699674917e-06, + "loss": 0.85415673, + "num_input_tokens_seen": 52677270, + "router_z_loss_clip": 0.98583984, + "router_z_loss_mlp": 0.2557373, + "step": 1862, + "time_per_iteration": 2.473581314086914 + }, + { + "auxiliary_loss_clip": 0.01141846, + "auxiliary_loss_mlp": 0.0105615, + "balance_loss_clip": 1.04401171, + "balance_loss_mlp": 1.03074634, + "epoch": 0.05405954384539493, + "flos": 18435118049280.0, + "grad_norm": 2.9567859174241367, + "language_loss": 0.83839703, + "learning_rate": 3.993933079164075e-06, + "loss": 0.86037695, + "num_input_tokens_seen": 52690740, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.25390625, + "step": 1863, + "time_per_iteration": 2.626203775405884 + }, + { + "auxiliary_loss_clip": 0.01139463, + "auxiliary_loss_mlp": 0.01058233, + "balance_loss_clip": 1.03916192, + "balance_loss_mlp": 1.03116024, + "epoch": 0.054088561313910974, + "flos": 31714542604800.0, + "grad_norm": 2.371971933251819, + "language_loss": 0.83571464, + "learning_rate": 3.993918441042008e-06, + "loss": 0.85769153, + "num_input_tokens_seen": 52705105, + "router_z_loss_clip": 1.00341797, + "router_z_loss_mlp": 0.27087402, + "step": 1864, + "time_per_iteration": 2.4979631900787354 + }, + { + "auxiliary_loss_clip": 0.01130592, + "auxiliary_loss_mlp": 0.01039182, + "balance_loss_clip": 1.03995728, + "balance_loss_mlp": 1.01579309, + "epoch": 0.05411757878242702, + "flos": 20876873783040.0, + "grad_norm": 2.2205247857113934, + "language_loss": 0.58998191, + "learning_rate": 3.993903785308847e-06, + "loss": 0.61167967, + "num_input_tokens_seen": 52716485, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.23413086, + "step": 1865, + "time_per_iteration": 2.785456895828247 + }, + { + "auxiliary_loss_clip": 0.01138683, + "auxiliary_loss_mlp": 0.01047265, + "balance_loss_clip": 1.04118133, + "balance_loss_mlp": 1.02330363, + "epoch": 0.05414659625094307, + "flos": 11504893415040.0, + "grad_norm": 3.9068427725771238, + "language_loss": 0.8490082, + "learning_rate": 3.993889111964721e-06, + "loss": 0.87086761, + "num_input_tokens_seen": 52729885, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.23962402, + "step": 1866, + "time_per_iteration": 2.4469263553619385 + }, + { + "auxiliary_loss_clip": 0.01132532, + "auxiliary_loss_mlp": 0.01044732, + "balance_loss_clip": 1.04014826, + "balance_loss_mlp": 1.02011514, + "epoch": 0.054175613719459115, + "flos": 26095795482240.0, + "grad_norm": 3.384131749049059, + "language_loss": 0.92239839, + "learning_rate": 3.993874421009759e-06, + "loss": 0.94417107, + "num_input_tokens_seen": 52744265, + "router_z_loss_clip": 0.92333984, + "router_z_loss_mlp": 0.24621582, + "step": 1867, + "time_per_iteration": 2.463029623031616 + }, + { + "auxiliary_loss_clip": 0.01017793, + "auxiliary_loss_mlp": 0.01007152, + "balance_loss_clip": 1.00119615, + "balance_loss_mlp": 1.00539923, + "epoch": 0.05420463118797516, + "flos": 66425154733440.0, + "grad_norm": 0.6445642624385068, + "language_loss": 0.50376362, + "learning_rate": 3.993859712444092e-06, + "loss": 0.52401304, + "num_input_tokens_seen": 52810155, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01757812, + "step": 1868, + "time_per_iteration": 3.131380558013916 + }, + { + "auxiliary_loss_clip": 0.01133858, + "auxiliary_loss_mlp": 0.01055403, + "balance_loss_clip": 1.03804755, + "balance_loss_mlp": 1.02942777, + "epoch": 0.05423364865649121, + "flos": 24566040950400.0, + "grad_norm": 2.4100648481655034, + "language_loss": 0.90033031, + "learning_rate": 3.993844986267849e-06, + "loss": 0.92222291, + "num_input_tokens_seen": 52826775, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.2598877, + "step": 1869, + "time_per_iteration": 2.4549217224121094 + }, + { + "auxiliary_loss_clip": 0.01132548, + "auxiliary_loss_mlp": 0.01048458, + "balance_loss_clip": 1.03802216, + "balance_loss_mlp": 1.02446151, + "epoch": 0.054262666125007256, + "flos": 16755237204480.0, + "grad_norm": 3.562160683115815, + "language_loss": 0.88599968, + "learning_rate": 3.9938302424811605e-06, + "loss": 0.90780973, + "num_input_tokens_seen": 52838435, + "router_z_loss_clip": 0.94433594, + "router_z_loss_mlp": 0.2401123, + "step": 1870, + "time_per_iteration": 2.4485628604888916 + }, + { + "auxiliary_loss_clip": 0.01138189, + "auxiliary_loss_mlp": 0.01049309, + "balance_loss_clip": 1.0403595, + "balance_loss_mlp": 1.02425146, + "epoch": 0.0542916835935233, + "flos": 15950838412800.0, + "grad_norm": 2.5409515001466767, + "language_loss": 0.6086185, + "learning_rate": 3.993815481084156e-06, + "loss": 0.63049352, + "num_input_tokens_seen": 52850670, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.25085449, + "step": 1871, + "time_per_iteration": 2.3530189990997314 + }, + { + "auxiliary_loss_clip": 0.01139804, + "auxiliary_loss_mlp": 0.01052742, + "balance_loss_clip": 1.03969562, + "balance_loss_mlp": 1.02707624, + "epoch": 0.054320701062039345, + "flos": 31360662397440.0, + "grad_norm": 2.010320090811332, + "language_loss": 1.02169526, + "learning_rate": 3.9938007020769665e-06, + "loss": 1.04362071, + "num_input_tokens_seen": 52872325, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.25646973, + "step": 1872, + "time_per_iteration": 2.537409782409668 + }, + { + "auxiliary_loss_clip": 0.01018885, + "auxiliary_loss_mlp": 0.01001186, + "balance_loss_clip": 1.00245142, + "balance_loss_mlp": 0.99949354, + "epoch": 0.054349718530555396, + "flos": 74774110176000.0, + "grad_norm": 0.7187551688115489, + "language_loss": 0.49552792, + "learning_rate": 3.993785905459722e-06, + "loss": 0.51572865, + "num_input_tokens_seen": 52940855, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01696777, + "step": 1873, + "time_per_iteration": 3.137871265411377 + }, + { + "auxiliary_loss_clip": 0.01018687, + "auxiliary_loss_mlp": 0.01002244, + "balance_loss_clip": 1.00200415, + "balance_loss_mlp": 1.00050318, + "epoch": 0.05437873599907144, + "flos": 74771177621760.0, + "grad_norm": 0.693533431566823, + "language_loss": 0.5180077, + "learning_rate": 3.993771091232554e-06, + "loss": 0.53821701, + "num_input_tokens_seen": 53006680, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01745605, + "step": 1874, + "time_per_iteration": 3.1757962703704834 + }, + { + "auxiliary_loss_clip": 0.0101823, + "auxiliary_loss_mlp": 0.01002521, + "balance_loss_clip": 1.00156236, + "balance_loss_mlp": 1.00085199, + "epoch": 0.054407753467587486, + "flos": 70070925214080.0, + "grad_norm": 0.6698445365834448, + "language_loss": 0.5646379, + "learning_rate": 3.993756259395593e-06, + "loss": 0.58484542, + "num_input_tokens_seen": 53073775, + "router_z_loss_clip": 0.16699219, + "router_z_loss_mlp": 0.01672363, + "step": 1875, + "time_per_iteration": 3.058816432952881 + }, + { + "auxiliary_loss_clip": 0.01017661, + "auxiliary_loss_mlp": 0.01005115, + "balance_loss_clip": 1.00118268, + "balance_loss_mlp": 1.00345767, + "epoch": 0.05443677093610354, + "flos": 70761193171200.0, + "grad_norm": 0.6941376953438589, + "language_loss": 0.47173363, + "learning_rate": 3.993741409948969e-06, + "loss": 0.49196136, + "num_input_tokens_seen": 53134680, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01660156, + "step": 1876, + "time_per_iteration": 3.0295932292938232 + }, + { + "auxiliary_loss_clip": 0.01132323, + "auxiliary_loss_mlp": 0.0105669, + "balance_loss_clip": 1.03865671, + "balance_loss_mlp": 1.03320622, + "epoch": 0.05446578840461958, + "flos": 22013680429440.0, + "grad_norm": 2.3070870377392065, + "language_loss": 0.70795214, + "learning_rate": 3.993726542892815e-06, + "loss": 0.72984231, + "num_input_tokens_seen": 53150195, + "router_z_loss_clip": 0.93652344, + "router_z_loss_mlp": 0.23498535, + "step": 1877, + "time_per_iteration": 2.567310333251953 + }, + { + "auxiliary_loss_clip": 0.01018574, + "auxiliary_loss_mlp": 0.01003288, + "balance_loss_clip": 1.00208044, + "balance_loss_mlp": 1.00165498, + "epoch": 0.05449480587313563, + "flos": 62978355290880.0, + "grad_norm": 0.7238218451977743, + "language_loss": 0.51823175, + "learning_rate": 3.993711658227262e-06, + "loss": 0.53845036, + "num_input_tokens_seen": 53210565, + "router_z_loss_clip": 0.16503906, + "router_z_loss_mlp": 0.01635742, + "step": 1878, + "time_per_iteration": 3.11933970451355 + }, + { + "auxiliary_loss_clip": 0.0113282, + "auxiliary_loss_mlp": 0.01048762, + "balance_loss_clip": 1.03862154, + "balance_loss_mlp": 1.02253604, + "epoch": 0.05452382334165167, + "flos": 74739024259200.0, + "grad_norm": 2.1053482593712682, + "language_loss": 0.82426286, + "learning_rate": 3.99369675595244e-06, + "loss": 0.84607875, + "num_input_tokens_seen": 53234210, + "router_z_loss_clip": 0.94238281, + "router_z_loss_mlp": 0.26220703, + "step": 1879, + "time_per_iteration": 2.925760269165039 + }, + { + "auxiliary_loss_clip": 0.01128659, + "auxiliary_loss_mlp": 0.01045801, + "balance_loss_clip": 1.03927851, + "balance_loss_mlp": 1.02274609, + "epoch": 0.05455284081016772, + "flos": 11464534016640.0, + "grad_norm": 4.9484474232861, + "language_loss": 0.8463434, + "learning_rate": 3.993681836068481e-06, + "loss": 0.86808795, + "num_input_tokens_seen": 53245480, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.23059082, + "step": 1880, + "time_per_iteration": 2.384870767593384 + }, + { + "auxiliary_loss_clip": 0.01130567, + "auxiliary_loss_mlp": 0.01046544, + "balance_loss_clip": 1.03836823, + "balance_loss_mlp": 1.02333426, + "epoch": 0.05458185827868377, + "flos": 25475877648000.0, + "grad_norm": 2.6888214500793604, + "language_loss": 0.85940361, + "learning_rate": 3.993666898575518e-06, + "loss": 0.88117468, + "num_input_tokens_seen": 53259740, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.23205566, + "step": 1881, + "time_per_iteration": 2.5737926959991455 + }, + { + "auxiliary_loss_clip": 0.01120337, + "auxiliary_loss_mlp": 0.01042345, + "balance_loss_clip": 1.03587568, + "balance_loss_mlp": 1.02253222, + "epoch": 0.05461087574719981, + "flos": 17814292519680.0, + "grad_norm": 2.1784147798449696, + "language_loss": 0.78787875, + "learning_rate": 3.993651943473682e-06, + "loss": 0.80950558, + "num_input_tokens_seen": 53276345, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.19812012, + "step": 1882, + "time_per_iteration": 2.4619624614715576 + }, + { + "auxiliary_loss_clip": 0.01140456, + "auxiliary_loss_mlp": 0.01063791, + "balance_loss_clip": 1.04044425, + "balance_loss_mlp": 1.03778005, + "epoch": 0.054639893215715864, + "flos": 11866768323840.0, + "grad_norm": 2.199242082480213, + "language_loss": 0.815135, + "learning_rate": 3.993636970763106e-06, + "loss": 0.8371774, + "num_input_tokens_seen": 53288785, + "router_z_loss_clip": 1.00048828, + "router_z_loss_mlp": 0.26037598, + "step": 1883, + "time_per_iteration": 2.469564437866211 + }, + { + "auxiliary_loss_clip": 0.01136505, + "auxiliary_loss_mlp": 0.01050631, + "balance_loss_clip": 1.04043639, + "balance_loss_mlp": 1.02506089, + "epoch": 0.05466891068423191, + "flos": 34561919047680.0, + "grad_norm": 2.9064945566648377, + "language_loss": 1.04908037, + "learning_rate": 3.9936219804439205e-06, + "loss": 1.0709517, + "num_input_tokens_seen": 53309415, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.2557373, + "step": 1884, + "time_per_iteration": 2.5376315116882324 + }, + { + "auxiliary_loss_clip": 0.01135618, + "auxiliary_loss_mlp": 0.01051197, + "balance_loss_clip": 1.0407331, + "balance_loss_mlp": 1.02587652, + "epoch": 0.05469792815274795, + "flos": 35035585729920.0, + "grad_norm": 2.953646396284567, + "language_loss": 0.97764337, + "learning_rate": 3.9936069725162594e-06, + "loss": 0.99951148, + "num_input_tokens_seen": 53325620, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.25305176, + "step": 1885, + "time_per_iteration": 2.509556770324707 + }, + { + "auxiliary_loss_clip": 0.01133906, + "auxiliary_loss_mlp": 0.01057314, + "balance_loss_clip": 1.04056692, + "balance_loss_mlp": 1.03344798, + "epoch": 0.054726945621264, + "flos": 21572971937280.0, + "grad_norm": 2.8817428587067098, + "language_loss": 0.6052525, + "learning_rate": 3.993591946980255e-06, + "loss": 0.62716472, + "num_input_tokens_seen": 53339170, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.23852539, + "step": 1886, + "time_per_iteration": 2.398688554763794 + }, + { + "auxiliary_loss_clip": 0.01021835, + "auxiliary_loss_mlp": 0.01000948, + "balance_loss_clip": 1.00517321, + "balance_loss_mlp": 0.99929076, + "epoch": 0.05475596308978005, + "flos": 62987746446720.0, + "grad_norm": 0.7324071038660138, + "language_loss": 0.52525651, + "learning_rate": 3.993576903836039e-06, + "loss": 0.5454843, + "num_input_tokens_seen": 53399620, + "router_z_loss_clip": 0.16699219, + "router_z_loss_mlp": 0.01660156, + "step": 1887, + "time_per_iteration": 2.9865434169769287 + }, + { + "auxiliary_loss_clip": 0.01020827, + "auxiliary_loss_mlp": 0.01001138, + "balance_loss_clip": 1.00437045, + "balance_loss_mlp": 0.99949312, + "epoch": 0.054784980558296094, + "flos": 60079096834560.0, + "grad_norm": 0.7380919475116331, + "language_loss": 0.57644689, + "learning_rate": 3.993561843083745e-06, + "loss": 0.59666646, + "num_input_tokens_seen": 53464325, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01647949, + "step": 1888, + "time_per_iteration": 3.112744092941284 + }, + { + "auxiliary_loss_clip": 0.01019726, + "auxiliary_loss_mlp": 0.01002091, + "balance_loss_clip": 1.00333464, + "balance_loss_mlp": 1.00039804, + "epoch": 0.05481399802681214, + "flos": 74768768737920.0, + "grad_norm": 0.6759653194875369, + "language_loss": 0.54865026, + "learning_rate": 3.993546764723507e-06, + "loss": 0.5688684, + "num_input_tokens_seen": 53529965, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01696777, + "step": 1889, + "time_per_iteration": 3.158041477203369 + }, + { + "auxiliary_loss_clip": 0.01145195, + "auxiliary_loss_mlp": 0.01057525, + "balance_loss_clip": 1.04475164, + "balance_loss_mlp": 1.0321331, + "epoch": 0.05484301549532819, + "flos": 34125958500480.0, + "grad_norm": 2.31308480286224, + "language_loss": 0.90982521, + "learning_rate": 3.993531668755458e-06, + "loss": 0.9318524, + "num_input_tokens_seen": 53550155, + "router_z_loss_clip": 1.00439453, + "router_z_loss_mlp": 0.25402832, + "step": 1890, + "time_per_iteration": 2.873180389404297 + }, + { + "auxiliary_loss_clip": 0.01142015, + "auxiliary_loss_mlp": 0.0106076, + "balance_loss_clip": 1.04144859, + "balance_loss_mlp": 1.03272212, + "epoch": 0.054872032963844235, + "flos": 24893107367040.0, + "grad_norm": 2.1253366660435336, + "language_loss": 0.77506822, + "learning_rate": 3.993516555179729e-06, + "loss": 0.79709595, + "num_input_tokens_seen": 53565570, + "router_z_loss_clip": 1.00634766, + "router_z_loss_mlp": 0.28063965, + "step": 1891, + "time_per_iteration": 2.4419658184051514 + }, + { + "auxiliary_loss_clip": 0.01136824, + "auxiliary_loss_mlp": 0.01044143, + "balance_loss_clip": 1.03940964, + "balance_loss_mlp": 1.02031279, + "epoch": 0.05490105043236028, + "flos": 12852575873280.0, + "grad_norm": 2.9926714477940046, + "language_loss": 0.9128598, + "learning_rate": 3.993501423996456e-06, + "loss": 0.93466949, + "num_input_tokens_seen": 53577385, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.23840332, + "step": 1892, + "time_per_iteration": 2.5737547874450684 + }, + { + "auxiliary_loss_clip": 0.01144447, + "auxiliary_loss_mlp": 0.01064417, + "balance_loss_clip": 1.04362035, + "balance_loss_mlp": 1.03649783, + "epoch": 0.05493006790087633, + "flos": 20258666605440.0, + "grad_norm": 3.3471804629753303, + "language_loss": 0.91453058, + "learning_rate": 3.993486275205771e-06, + "loss": 0.93661922, + "num_input_tokens_seen": 53591810, + "router_z_loss_clip": 1.0078125, + "router_z_loss_mlp": 0.27893066, + "step": 1893, + "time_per_iteration": 2.397387981414795 + }, + { + "auxiliary_loss_clip": 0.01143245, + "auxiliary_loss_mlp": 0.01054207, + "balance_loss_clip": 1.04223037, + "balance_loss_mlp": 1.02675307, + "epoch": 0.054959085369392376, + "flos": 28613172954240.0, + "grad_norm": 2.374952360935981, + "language_loss": 0.88077509, + "learning_rate": 3.993471108807809e-06, + "loss": 0.9027496, + "num_input_tokens_seen": 53603890, + "router_z_loss_clip": 1.00927734, + "router_z_loss_mlp": 0.2746582, + "step": 1894, + "time_per_iteration": 2.5118865966796875 + }, + { + "auxiliary_loss_clip": 0.01141639, + "auxiliary_loss_mlp": 0.01046883, + "balance_loss_clip": 1.03915501, + "balance_loss_mlp": 1.0205853, + "epoch": 0.05498810283790842, + "flos": 17120952362880.0, + "grad_norm": 2.9433945118708396, + "language_loss": 0.74106485, + "learning_rate": 3.993455924802703e-06, + "loss": 0.76295006, + "num_input_tokens_seen": 53617160, + "router_z_loss_clip": 1.02587891, + "router_z_loss_mlp": 0.26306152, + "step": 1895, + "time_per_iteration": 2.3532681465148926 + }, + { + "auxiliary_loss_clip": 0.01141823, + "auxiliary_loss_mlp": 0.01062515, + "balance_loss_clip": 1.04398847, + "balance_loss_mlp": 1.03596663, + "epoch": 0.055017120306424465, + "flos": 35690975372160.0, + "grad_norm": 3.1727482163678404, + "language_loss": 0.813712, + "learning_rate": 3.9934407231905885e-06, + "loss": 0.83575535, + "num_input_tokens_seen": 53633550, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.26501465, + "step": 1896, + "time_per_iteration": 2.4911158084869385 + }, + { + "auxiliary_loss_clip": 0.01024433, + "auxiliary_loss_mlp": 0.0100483, + "balance_loss_clip": 1.00778747, + "balance_loss_mlp": 1.00319707, + "epoch": 0.055046137774940516, + "flos": 60580519914240.0, + "grad_norm": 0.649638459613251, + "language_loss": 0.50161082, + "learning_rate": 3.993425503971598e-06, + "loss": 0.52190346, + "num_input_tokens_seen": 53694370, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01635742, + "step": 1897, + "time_per_iteration": 2.960111141204834 + }, + { + "auxiliary_loss_clip": 0.01137561, + "auxiliary_loss_mlp": 0.01066751, + "balance_loss_clip": 1.04008031, + "balance_loss_mlp": 1.04054916, + "epoch": 0.05507515524345656, + "flos": 14033547256320.0, + "grad_norm": 2.793510686972853, + "language_loss": 0.92046952, + "learning_rate": 3.993410267145868e-06, + "loss": 0.94251263, + "num_input_tokens_seen": 53707870, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.26196289, + "step": 1898, + "time_per_iteration": 2.4047343730926514 + }, + { + "auxiliary_loss_clip": 0.01021704, + "auxiliary_loss_mlp": 0.01005236, + "balance_loss_clip": 1.00529742, + "balance_loss_mlp": 1.00367451, + "epoch": 0.055104172711972606, + "flos": 70357632232320.0, + "grad_norm": 0.6946986018645389, + "language_loss": 0.51575279, + "learning_rate": 3.99339501271353e-06, + "loss": 0.53602219, + "num_input_tokens_seen": 53772755, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.015625, + "step": 1899, + "time_per_iteration": 3.16163969039917 + }, + { + "auxiliary_loss_clip": 0.01127449, + "auxiliary_loss_mlp": 0.01047356, + "balance_loss_clip": 1.03850305, + "balance_loss_mlp": 1.02370536, + "epoch": 0.05513319018048866, + "flos": 31167802713600.0, + "grad_norm": 2.154039270914026, + "language_loss": 0.65086341, + "learning_rate": 3.993379740674722e-06, + "loss": 0.67261153, + "num_input_tokens_seen": 53788175, + "router_z_loss_clip": 0.890625, + "router_z_loss_mlp": 0.23669434, + "step": 1900, + "time_per_iteration": 2.5663394927978516 + }, + { + "auxiliary_loss_clip": 0.01018312, + "auxiliary_loss_mlp": 0.0100323, + "balance_loss_clip": 1.0021286, + "balance_loss_mlp": 1.00158536, + "epoch": 0.0551622076490047, + "flos": 59122442136960.0, + "grad_norm": 0.6998906641447908, + "language_loss": 0.52557629, + "learning_rate": 3.993364451029578e-06, + "loss": 0.54579169, + "num_input_tokens_seen": 53844825, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.01647949, + "step": 1901, + "time_per_iteration": 2.931478977203369 + }, + { + "auxiliary_loss_clip": 0.0113383, + "auxiliary_loss_mlp": 0.01048291, + "balance_loss_clip": 1.03888822, + "balance_loss_mlp": 1.0244019, + "epoch": 0.05519122511752075, + "flos": 15589103149440.0, + "grad_norm": 2.4959660071774485, + "language_loss": 0.77583933, + "learning_rate": 3.9933491437782314e-06, + "loss": 0.79766059, + "num_input_tokens_seen": 53859955, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.2388916, + "step": 1902, + "time_per_iteration": 5.0805041790008545 + }, + { + "auxiliary_loss_clip": 0.01139517, + "auxiliary_loss_mlp": 0.01064012, + "balance_loss_clip": 1.04143381, + "balance_loss_mlp": 1.03605747, + "epoch": 0.05522024258603679, + "flos": 29344079600640.0, + "grad_norm": 2.806879152761323, + "language_loss": 0.87950063, + "learning_rate": 3.993333818920819e-06, + "loss": 0.90153593, + "num_input_tokens_seen": 53881455, + "router_z_loss_clip": 0.98046875, + "router_z_loss_mlp": 0.27966309, + "step": 1903, + "time_per_iteration": 2.731797695159912 + }, + { + "auxiliary_loss_clip": 0.01144024, + "auxiliary_loss_mlp": 0.0105673, + "balance_loss_clip": 1.04273331, + "balance_loss_mlp": 1.02769113, + "epoch": 0.05524926005455284, + "flos": 25585609651200.0, + "grad_norm": 2.311272281018964, + "language_loss": 0.77769178, + "learning_rate": 3.993318476457476e-06, + "loss": 0.79969931, + "num_input_tokens_seen": 53898610, + "router_z_loss_clip": 1.01367188, + "router_z_loss_mlp": 0.29040527, + "step": 1904, + "time_per_iteration": 2.456064462661743 + }, + { + "auxiliary_loss_clip": 0.01123163, + "auxiliary_loss_mlp": 0.01046115, + "balance_loss_clip": 1.03359866, + "balance_loss_mlp": 1.02130806, + "epoch": 0.05527827752306889, + "flos": 11987671962240.0, + "grad_norm": 2.9358916994134843, + "language_loss": 0.68323892, + "learning_rate": 3.993303116388336e-06, + "loss": 0.70493174, + "num_input_tokens_seen": 53911925, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.24804688, + "step": 1905, + "time_per_iteration": 2.3995521068573 + }, + { + "auxiliary_loss_clip": 0.01139743, + "auxiliary_loss_mlp": 0.01048427, + "balance_loss_clip": 1.04086161, + "balance_loss_mlp": 1.02298832, + "epoch": 0.05530729499158493, + "flos": 11209598202240.0, + "grad_norm": 2.9132309932614993, + "language_loss": 0.88503861, + "learning_rate": 3.993287738713538e-06, + "loss": 0.90692031, + "num_input_tokens_seen": 53924455, + "router_z_loss_clip": 0.98828125, + "router_z_loss_mlp": 0.25463867, + "step": 1906, + "time_per_iteration": 2.3586928844451904 + }, + { + "auxiliary_loss_clip": 0.01129376, + "auxiliary_loss_mlp": 0.0105496, + "balance_loss_clip": 1.0388608, + "balance_loss_mlp": 1.0304749, + "epoch": 0.055336312460100984, + "flos": 74729633103360.0, + "grad_norm": 3.024957753170163, + "language_loss": 0.83684647, + "learning_rate": 3.9932723434332155e-06, + "loss": 0.85868979, + "num_input_tokens_seen": 53947145, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.24499512, + "step": 1907, + "time_per_iteration": 5.14106822013855 + }, + { + "auxiliary_loss_clip": 0.01136853, + "auxiliary_loss_mlp": 0.0104424, + "balance_loss_clip": 1.03916597, + "balance_loss_mlp": 1.02000487, + "epoch": 0.05536532992861703, + "flos": 46600286037120.0, + "grad_norm": 2.2245471919609248, + "language_loss": 0.9363451, + "learning_rate": 3.993256930547505e-06, + "loss": 0.95815605, + "num_input_tokens_seen": 53967215, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.2421875, + "step": 1908, + "time_per_iteration": 2.645275354385376 + }, + { + "auxiliary_loss_clip": 0.01141105, + "auxiliary_loss_mlp": 0.01055064, + "balance_loss_clip": 1.04091847, + "balance_loss_mlp": 1.02646577, + "epoch": 0.05539434739713307, + "flos": 30290924206080.0, + "grad_norm": 2.294211414440941, + "language_loss": 0.93857598, + "learning_rate": 3.993241500056543e-06, + "loss": 0.96053773, + "num_input_tokens_seen": 53985930, + "router_z_loss_clip": 1.0, + "router_z_loss_mlp": 0.28588867, + "step": 1909, + "time_per_iteration": 2.504643201828003 + }, + { + "auxiliary_loss_clip": 0.01132829, + "auxiliary_loss_mlp": 0.01047019, + "balance_loss_clip": 1.03849161, + "balance_loss_mlp": 1.02299833, + "epoch": 0.05542336486564912, + "flos": 16606228055040.0, + "grad_norm": 3.13139259525242, + "language_loss": 0.97907454, + "learning_rate": 3.993226051960465e-06, + "loss": 1.00087309, + "num_input_tokens_seen": 53995850, + "router_z_loss_clip": 0.94384766, + "router_z_loss_mlp": 0.2401123, + "step": 1910, + "time_per_iteration": 7.473978519439697 + }, + { + "auxiliary_loss_clip": 0.01145978, + "auxiliary_loss_mlp": 0.01057999, + "balance_loss_clip": 1.04201603, + "balance_loss_mlp": 1.02971065, + "epoch": 0.05545238233416517, + "flos": 11502833644800.0, + "grad_norm": 2.964920912890202, + "language_loss": 0.93335909, + "learning_rate": 3.993210586259408e-06, + "loss": 0.9553988, + "num_input_tokens_seen": 54007905, + "router_z_loss_clip": 1.04150391, + "router_z_loss_mlp": 0.28259277, + "step": 1911, + "time_per_iteration": 2.3873775005340576 + }, + { + "auxiliary_loss_clip": 0.01133014, + "auxiliary_loss_mlp": 0.01051394, + "balance_loss_clip": 1.04041052, + "balance_loss_mlp": 1.02500701, + "epoch": 0.055481399802681214, + "flos": 35692825674240.0, + "grad_norm": 2.5534887393852848, + "language_loss": 0.87585866, + "learning_rate": 3.993195102953509e-06, + "loss": 0.89770275, + "num_input_tokens_seen": 54027865, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.26379395, + "step": 1912, + "time_per_iteration": 2.65920352935791 + }, + { + "auxiliary_loss_clip": 0.01134678, + "auxiliary_loss_mlp": 0.01047, + "balance_loss_clip": 1.0413307, + "balance_loss_mlp": 1.02437353, + "epoch": 0.05551041727119726, + "flos": 25074167011200.0, + "grad_norm": 3.1799257869743185, + "language_loss": 0.93144476, + "learning_rate": 3.9931796020429036e-06, + "loss": 0.95326155, + "num_input_tokens_seen": 54043705, + "router_z_loss_clip": 0.93408203, + "router_z_loss_mlp": 0.22631836, + "step": 1913, + "time_per_iteration": 2.7818844318389893 + }, + { + "auxiliary_loss_clip": 0.01029539, + "auxiliary_loss_mlp": 0.01005341, + "balance_loss_clip": 1.01293457, + "balance_loss_mlp": 1.00384486, + "epoch": 0.05553943473971331, + "flos": 74766115474560.0, + "grad_norm": 0.686859242619144, + "language_loss": 0.49402666, + "learning_rate": 3.99316408352773e-06, + "loss": 0.51437545, + "num_input_tokens_seen": 54104640, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01495361, + "step": 1914, + "time_per_iteration": 3.0481185913085938 + }, + { + "auxiliary_loss_clip": 0.01136845, + "auxiliary_loss_mlp": 0.01056418, + "balance_loss_clip": 1.03994799, + "balance_loss_mlp": 1.0302279, + "epoch": 0.055568452208229355, + "flos": 54554478203520.0, + "grad_norm": 2.851856266621382, + "language_loss": 0.93163818, + "learning_rate": 3.993148547408124e-06, + "loss": 0.95357078, + "num_input_tokens_seen": 54122490, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.26208496, + "step": 1915, + "time_per_iteration": 2.650416612625122 + }, + { + "auxiliary_loss_clip": 0.01129957, + "auxiliary_loss_mlp": 0.01053435, + "balance_loss_clip": 1.03865075, + "balance_loss_mlp": 1.03009343, + "epoch": 0.0555974696767454, + "flos": 17046936547200.0, + "grad_norm": 3.3002066519314925, + "language_loss": 0.89770508, + "learning_rate": 3.993132993684224e-06, + "loss": 0.91953903, + "num_input_tokens_seen": 54134150, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.23339844, + "step": 1916, + "time_per_iteration": 2.3542628288269043 + }, + { + "auxiliary_loss_clip": 0.01021083, + "auxiliary_loss_mlp": 0.01000417, + "balance_loss_clip": 1.00480938, + "balance_loss_mlp": 0.99906957, + "epoch": 0.055626487145261444, + "flos": 60896030670720.0, + "grad_norm": 0.7442443260892001, + "language_loss": 0.55828631, + "learning_rate": 3.993117422356168e-06, + "loss": 0.57850128, + "num_input_tokens_seen": 54182225, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.01348877, + "step": 1917, + "time_per_iteration": 2.806532621383667 + }, + { + "auxiliary_loss_clip": 0.01137629, + "auxiliary_loss_mlp": 0.01050659, + "balance_loss_clip": 1.04116261, + "balance_loss_mlp": 1.02277565, + "epoch": 0.055655504613777496, + "flos": 20476385043840.0, + "grad_norm": 2.171219736721862, + "language_loss": 0.81555212, + "learning_rate": 3.993101833424091e-06, + "loss": 0.83743501, + "num_input_tokens_seen": 54196395, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.27868652, + "step": 1918, + "time_per_iteration": 2.3417725563049316 + }, + { + "auxiliary_loss_clip": 0.01019186, + "auxiliary_loss_mlp": 0.01003536, + "balance_loss_clip": 1.00305438, + "balance_loss_mlp": 1.002123, + "epoch": 0.05568452208229354, + "flos": 66779418965760.0, + "grad_norm": 0.744640882023177, + "language_loss": 0.57309532, + "learning_rate": 3.993086226888132e-06, + "loss": 0.59332252, + "num_input_tokens_seen": 54253085, + "router_z_loss_clip": 0.16113281, + "router_z_loss_mlp": 0.01409912, + "step": 1919, + "time_per_iteration": 3.0105786323547363 + }, + { + "auxiliary_loss_clip": 0.01135288, + "auxiliary_loss_mlp": 0.01049789, + "balance_loss_clip": 1.03860283, + "balance_loss_mlp": 1.02516031, + "epoch": 0.055713539550809585, + "flos": 16320463643520.0, + "grad_norm": 2.511430450944144, + "language_loss": 0.74940431, + "learning_rate": 3.993070602748429e-06, + "loss": 0.77125514, + "num_input_tokens_seen": 54268265, + "router_z_loss_clip": 0.96679688, + "router_z_loss_mlp": 0.24645996, + "step": 1920, + "time_per_iteration": 2.374995470046997 + }, + { + "auxiliary_loss_clip": 0.01144791, + "auxiliary_loss_mlp": 0.0106618, + "balance_loss_clip": 1.04360259, + "balance_loss_mlp": 1.03791595, + "epoch": 0.055742557019325636, + "flos": 28246026430080.0, + "grad_norm": 3.409489728481339, + "language_loss": 0.89279914, + "learning_rate": 3.993054961005121e-06, + "loss": 0.91490883, + "num_input_tokens_seen": 54280450, + "router_z_loss_clip": 1.01123047, + "router_z_loss_mlp": 0.28259277, + "step": 1921, + "time_per_iteration": 2.592360019683838 + }, + { + "auxiliary_loss_clip": 0.01019172, + "auxiliary_loss_mlp": 0.01007434, + "balance_loss_clip": 1.00254226, + "balance_loss_mlp": 1.00588477, + "epoch": 0.05577157448784168, + "flos": 59521185308160.0, + "grad_norm": 0.6338865026389265, + "language_loss": 0.53397924, + "learning_rate": 3.9930393016583435e-06, + "loss": 0.55424529, + "num_input_tokens_seen": 54340800, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01544189, + "step": 1922, + "time_per_iteration": 2.919583320617676 + }, + { + "auxiliary_loss_clip": 0.01019628, + "auxiliary_loss_mlp": 0.01006571, + "balance_loss_clip": 1.00307465, + "balance_loss_mlp": 1.00505137, + "epoch": 0.055800591956357726, + "flos": 52702228776960.0, + "grad_norm": 0.6737687816070629, + "language_loss": 0.51013219, + "learning_rate": 3.993023624708237e-06, + "loss": 0.5303942, + "num_input_tokens_seen": 54402060, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01519775, + "step": 1923, + "time_per_iteration": 3.069918632507324 + }, + { + "auxiliary_loss_clip": 0.01019282, + "auxiliary_loss_mlp": 0.01002838, + "balance_loss_clip": 1.00281882, + "balance_loss_mlp": 1.00147271, + "epoch": 0.05582960942487378, + "flos": 57483339626880.0, + "grad_norm": 0.723918693746104, + "language_loss": 0.52355242, + "learning_rate": 3.99300793015494e-06, + "loss": 0.54377365, + "num_input_tokens_seen": 54462010, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01367188, + "step": 1924, + "time_per_iteration": 2.9445931911468506 + }, + { + "auxiliary_loss_clip": 0.01135327, + "auxiliary_loss_mlp": 0.01058764, + "balance_loss_clip": 1.03951132, + "balance_loss_mlp": 1.0340997, + "epoch": 0.05585862689338982, + "flos": 17193327344640.0, + "grad_norm": 3.0726011553598767, + "language_loss": 0.76216793, + "learning_rate": 3.99299221799859e-06, + "loss": 0.78410888, + "num_input_tokens_seen": 54476830, + "router_z_loss_clip": 0.95898438, + "router_z_loss_mlp": 0.24658203, + "step": 1925, + "time_per_iteration": 2.4830212593078613 + }, + { + "auxiliary_loss_clip": 0.01127752, + "auxiliary_loss_mlp": 0.01043418, + "balance_loss_clip": 1.03688574, + "balance_loss_mlp": 1.02030373, + "epoch": 0.05588764436190587, + "flos": 20449780721280.0, + "grad_norm": 2.554882723674301, + "language_loss": 0.871539, + "learning_rate": 3.992976488239327e-06, + "loss": 0.8932507, + "num_input_tokens_seen": 54491145, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.23120117, + "step": 1926, + "time_per_iteration": 2.743598222732544 + }, + { + "auxiliary_loss_clip": 0.01140065, + "auxiliary_loss_mlp": 0.01046541, + "balance_loss_clip": 1.04235268, + "balance_loss_mlp": 1.02078021, + "epoch": 0.05591666183042191, + "flos": 34456481141760.0, + "grad_norm": 2.5137427531663494, + "language_loss": 0.92119247, + "learning_rate": 3.992960740877287e-06, + "loss": 0.94305855, + "num_input_tokens_seen": 54509670, + "router_z_loss_clip": 0.97753906, + "router_z_loss_mlp": 0.25756836, + "step": 1927, + "time_per_iteration": 2.6277072429656982 + }, + { + "auxiliary_loss_clip": 0.0113195, + "auxiliary_loss_mlp": 0.0104726, + "balance_loss_clip": 1.03870487, + "balance_loss_mlp": 1.02235746, + "epoch": 0.05594567929893796, + "flos": 28795873432320.0, + "grad_norm": 1.9505873076513642, + "language_loss": 0.68326461, + "learning_rate": 3.992944975912613e-06, + "loss": 0.70505667, + "num_input_tokens_seen": 54525320, + "router_z_loss_clip": 0.93212891, + "router_z_loss_mlp": 0.24853516, + "step": 1928, + "time_per_iteration": 2.5604310035705566 + }, + { + "auxiliary_loss_clip": 0.01133676, + "auxiliary_loss_mlp": 0.0104817, + "balance_loss_clip": 1.03956509, + "balance_loss_mlp": 1.02399492, + "epoch": 0.05597469676745401, + "flos": 27591649217280.0, + "grad_norm": 3.7822765396632185, + "language_loss": 0.99922442, + "learning_rate": 3.992929193345443e-06, + "loss": 1.02104282, + "num_input_tokens_seen": 54542240, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.24194336, + "step": 1929, + "time_per_iteration": 2.472874164581299 + }, + { + "auxiliary_loss_clip": 0.01137382, + "auxiliary_loss_mlp": 0.01062829, + "balance_loss_clip": 1.04392648, + "balance_loss_mlp": 1.03746092, + "epoch": 0.05600371423597005, + "flos": 17448856652160.0, + "grad_norm": 2.575887785364911, + "language_loss": 0.73604238, + "learning_rate": 3.9929133931759145e-06, + "loss": 0.75804454, + "num_input_tokens_seen": 54554740, + "router_z_loss_clip": 0.93457031, + "router_z_loss_mlp": 0.25390625, + "step": 1930, + "time_per_iteration": 2.3984811305999756 + }, + { + "auxiliary_loss_clip": 0.01144267, + "auxiliary_loss_mlp": 0.0105939, + "balance_loss_clip": 1.04056525, + "balance_loss_mlp": 1.03000498, + "epoch": 0.056032731704486104, + "flos": 25405283145600.0, + "grad_norm": 2.7426780933514263, + "language_loss": 1.03837204, + "learning_rate": 3.992897575404169e-06, + "loss": 1.06040871, + "num_input_tokens_seen": 54568675, + "router_z_loss_clip": 1.03759766, + "router_z_loss_mlp": 0.29418945, + "step": 1931, + "time_per_iteration": 2.4401068687438965 + }, + { + "auxiliary_loss_clip": 0.01031488, + "auxiliary_loss_mlp": 0.01028419, + "balance_loss_clip": 1.01460457, + "balance_loss_mlp": 1.02671385, + "epoch": 0.05606174917300215, + "flos": 58215433259520.0, + "grad_norm": 0.6813521324811045, + "language_loss": 0.51549369, + "learning_rate": 3.9928817400303456e-06, + "loss": 0.53609276, + "num_input_tokens_seen": 54630900, + "router_z_loss_clip": 0.16894531, + "router_z_loss_mlp": 0.01708984, + "step": 1932, + "time_per_iteration": 3.0484542846679688 + }, + { + "auxiliary_loss_clip": 0.01027439, + "auxiliary_loss_mlp": 0.01027312, + "balance_loss_clip": 1.01065111, + "balance_loss_mlp": 1.02558315, + "epoch": 0.05609076664151819, + "flos": 73066019086080.0, + "grad_norm": 0.7297261102109772, + "language_loss": 0.55018508, + "learning_rate": 3.9928658870545844e-06, + "loss": 0.57073259, + "num_input_tokens_seen": 54697855, + "router_z_loss_clip": 0.16796875, + "router_z_loss_mlp": 0.01733398, + "step": 1933, + "time_per_iteration": 3.142076015472412 + }, + { + "auxiliary_loss_clip": 0.01135636, + "auxiliary_loss_mlp": 0.0105661, + "balance_loss_clip": 1.04178023, + "balance_loss_mlp": 1.03084922, + "epoch": 0.05611978411003424, + "flos": 24784143413760.0, + "grad_norm": 2.866721576530481, + "language_loss": 0.84970111, + "learning_rate": 3.9928500164770255e-06, + "loss": 0.87162352, + "num_input_tokens_seen": 54713775, + "router_z_loss_clip": 0.93896484, + "router_z_loss_mlp": 0.25732422, + "step": 1934, + "time_per_iteration": 2.5013604164123535 + }, + { + "auxiliary_loss_clip": 0.01142781, + "auxiliary_loss_mlp": 0.010551, + "balance_loss_clip": 1.04431486, + "balance_loss_mlp": 1.02733588, + "epoch": 0.05614880157855029, + "flos": 22339699505280.0, + "grad_norm": 2.4646389213691045, + "language_loss": 1.00093436, + "learning_rate": 3.9928341282978086e-06, + "loss": 1.02291322, + "num_input_tokens_seen": 54733215, + "router_z_loss_clip": 0.98339844, + "router_z_loss_mlp": 0.27783203, + "step": 1935, + "time_per_iteration": 2.719108819961548 + }, + { + "auxiliary_loss_clip": 0.01138765, + "auxiliary_loss_mlp": 0.01052563, + "balance_loss_clip": 1.03889012, + "balance_loss_mlp": 1.02489412, + "epoch": 0.056177819047066334, + "flos": 30074811690240.0, + "grad_norm": 2.123931194886393, + "language_loss": 0.94596845, + "learning_rate": 3.992818222517074e-06, + "loss": 0.96788168, + "num_input_tokens_seen": 54750505, + "router_z_loss_clip": 0.99658203, + "router_z_loss_mlp": 0.27661133, + "step": 1936, + "time_per_iteration": 2.4700493812561035 + }, + { + "auxiliary_loss_clip": 0.01139049, + "auxiliary_loss_mlp": 0.01050377, + "balance_loss_clip": 1.04026318, + "balance_loss_mlp": 1.0232687, + "epoch": 0.05620683651558238, + "flos": 29451821656320.0, + "grad_norm": 2.1801981997722635, + "language_loss": 1.0212698, + "learning_rate": 3.992802299134963e-06, + "loss": 1.04316401, + "num_input_tokens_seen": 54765915, + "router_z_loss_clip": 0.98925781, + "router_z_loss_mlp": 0.27075195, + "step": 1937, + "time_per_iteration": 2.438690662384033 + }, + { + "auxiliary_loss_clip": 0.01028199, + "auxiliary_loss_mlp": 0.01002853, + "balance_loss_clip": 1.01062667, + "balance_loss_mlp": 1.00148761, + "epoch": 0.05623585398409843, + "flos": 59154038784000.0, + "grad_norm": 0.7095274330765847, + "language_loss": 0.52312636, + "learning_rate": 3.9927863581516155e-06, + "loss": 0.54343688, + "num_input_tokens_seen": 54824975, + "router_z_loss_clip": 0.17578125, + "router_z_loss_mlp": 0.01367188, + "step": 1938, + "time_per_iteration": 2.934415102005005 + }, + { + "auxiliary_loss_clip": 0.01136154, + "auxiliary_loss_mlp": 0.01052027, + "balance_loss_clip": 1.04027796, + "balance_loss_mlp": 1.02681434, + "epoch": 0.056264871452614475, + "flos": 24855505966080.0, + "grad_norm": 2.4411240201813498, + "language_loss": 0.96450305, + "learning_rate": 3.992770399567172e-06, + "loss": 0.98638487, + "num_input_tokens_seen": 54839840, + "router_z_loss_clip": 0.95751953, + "router_z_loss_mlp": 0.2520752, + "step": 1939, + "time_per_iteration": 2.815399646759033 + }, + { + "auxiliary_loss_clip": 0.01139573, + "auxiliary_loss_mlp": 0.01067716, + "balance_loss_clip": 1.04348755, + "balance_loss_mlp": 1.04133511, + "epoch": 0.05629388892113052, + "flos": 32663726271360.0, + "grad_norm": 2.755535193318366, + "language_loss": 0.73493963, + "learning_rate": 3.992754423381774e-06, + "loss": 0.75701249, + "num_input_tokens_seen": 54855545, + "router_z_loss_clip": 0.95996094, + "router_z_loss_mlp": 0.26403809, + "step": 1940, + "time_per_iteration": 2.4991583824157715 + }, + { + "auxiliary_loss_clip": 0.01144477, + "auxiliary_loss_mlp": 0.01057813, + "balance_loss_clip": 1.04737115, + "balance_loss_mlp": 1.03260016, + "epoch": 0.056322906389646564, + "flos": 20621519032320.0, + "grad_norm": 2.9334924539694494, + "language_loss": 0.95578909, + "learning_rate": 3.9927384295955636e-06, + "loss": 0.97781199, + "num_input_tokens_seen": 54868360, + "router_z_loss_clip": 0.97167969, + "router_z_loss_mlp": 0.25219727, + "step": 1941, + "time_per_iteration": 2.4540061950683594 + }, + { + "auxiliary_loss_clip": 0.01029113, + "auxiliary_loss_mlp": 0.01006769, + "balance_loss_clip": 1.01185536, + "balance_loss_mlp": 1.00538063, + "epoch": 0.056351923858162616, + "flos": 56547633611520.0, + "grad_norm": 0.7457891570209324, + "language_loss": 0.52753901, + "learning_rate": 3.992722418208681e-06, + "loss": 0.54789782, + "num_input_tokens_seen": 54918355, + "router_z_loss_clip": 0.171875, + "router_z_loss_mlp": 0.01391602, + "step": 1942, + "time_per_iteration": 2.815816879272461 + }, + { + "auxiliary_loss_clip": 0.01143457, + "auxiliary_loss_mlp": 0.010676, + "balance_loss_clip": 1.04209065, + "balance_loss_mlp": 1.03822708, + "epoch": 0.05638094132667866, + "flos": 15443166199680.0, + "grad_norm": 2.192900570984712, + "language_loss": 0.78639805, + "learning_rate": 3.992706389221266e-06, + "loss": 0.80850863, + "num_input_tokens_seen": 54932020, + "router_z_loss_clip": 1.01416016, + "router_z_loss_mlp": 0.29370117, + "step": 1943, + "time_per_iteration": 2.4365525245666504 + }, + { + "auxiliary_loss_clip": 0.01129119, + "auxiliary_loss_mlp": 0.01043798, + "balance_loss_clip": 1.0395956, + "balance_loss_mlp": 1.02188754, + "epoch": 0.056409958795194705, + "flos": 21717652078080.0, + "grad_norm": 2.2106205082419104, + "language_loss": 0.90738964, + "learning_rate": 3.992690342633463e-06, + "loss": 0.92911881, + "num_input_tokens_seen": 54946710, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.21899414, + "step": 1944, + "time_per_iteration": 2.445640802383423 + }, + { + "auxiliary_loss_clip": 0.01144855, + "auxiliary_loss_mlp": 0.01054413, + "balance_loss_clip": 1.0451802, + "balance_loss_mlp": 1.02573133, + "epoch": 0.056438976263710756, + "flos": 26424397998720.0, + "grad_norm": 2.224988943841425, + "language_loss": 0.9041118, + "learning_rate": 3.992674278445412e-06, + "loss": 0.92610443, + "num_input_tokens_seen": 54961465, + "router_z_loss_clip": 0.99707031, + "router_z_loss_mlp": 0.28686523, + "step": 1945, + "time_per_iteration": 2.4753000736236572 + }, + { + "auxiliary_loss_clip": 0.01148427, + "auxiliary_loss_mlp": 0.010603, + "balance_loss_clip": 1.04089606, + "balance_loss_mlp": 1.02714825, + "epoch": 0.0564679937322268, + "flos": 27008425088640.0, + "grad_norm": 3.8229393462904104, + "language_loss": 0.92471623, + "learning_rate": 3.992658196657256e-06, + "loss": 0.94680357, + "num_input_tokens_seen": 54978740, + "router_z_loss_clip": 1.07519531, + "router_z_loss_mlp": 0.3314209, + "step": 1946, + "time_per_iteration": 2.454608917236328 + }, + { + "auxiliary_loss_clip": 0.01017491, + "auxiliary_loss_mlp": 0.01005518, + "balance_loss_clip": 1.00152659, + "balance_loss_mlp": 1.00424254, + "epoch": 0.056497011200742846, + "flos": 74786119683840.0, + "grad_norm": 0.651305524593581, + "language_loss": 0.51157391, + "learning_rate": 3.992642097269136e-06, + "loss": 0.53180403, + "num_input_tokens_seen": 55046635, + "router_z_loss_clip": 0.15917969, + "router_z_loss_mlp": 0.01275635, + "step": 1947, + "time_per_iteration": 3.1803548336029053 + }, + { + "auxiliary_loss_clip": 0.01123333, + "auxiliary_loss_mlp": 0.01045846, + "balance_loss_clip": 1.0339092, + "balance_loss_mlp": 1.02403092, + "epoch": 0.0565260286692589, + "flos": 27191020832640.0, + "grad_norm": 2.3940238438571777, + "language_loss": 0.68613344, + "learning_rate": 3.992625980281195e-06, + "loss": 0.70782524, + "num_input_tokens_seen": 55060610, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.21826172, + "step": 1948, + "time_per_iteration": 2.4297139644622803 + }, + { + "auxiliary_loss_clip": 0.01135814, + "auxiliary_loss_mlp": 0.0105185, + "balance_loss_clip": 1.03959358, + "balance_loss_mlp": 1.02611232, + "epoch": 0.05655504613777494, + "flos": 24490454123520.0, + "grad_norm": 2.505620685661152, + "language_loss": 0.80087519, + "learning_rate": 3.992609845693574e-06, + "loss": 0.82275188, + "num_input_tokens_seen": 55074200, + "router_z_loss_clip": 0.96337891, + "router_z_loss_mlp": 0.25732422, + "step": 1949, + "time_per_iteration": 2.4998698234558105 + }, + { + "auxiliary_loss_clip": 0.01133245, + "auxiliary_loss_mlp": 0.01045889, + "balance_loss_clip": 1.03848171, + "balance_loss_mlp": 1.01900733, + "epoch": 0.05658406360629099, + "flos": 40797511804800.0, + "grad_norm": 2.40456911933454, + "language_loss": 0.84825987, + "learning_rate": 3.992593693506418e-06, + "loss": 0.87005121, + "num_input_tokens_seen": 55094800, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.2689209, + "step": 1950, + "time_per_iteration": 2.5996763706207275 + }, + { + "auxiliary_loss_clip": 0.01019165, + "auxiliary_loss_mlp": 0.01003297, + "balance_loss_clip": 1.00307572, + "balance_loss_mlp": 1.0020628, + "epoch": 0.05661308107480703, + "flos": 57772280972160.0, + "grad_norm": 0.7016678719775223, + "language_loss": 0.49833858, + "learning_rate": 3.9925775237198675e-06, + "loss": 0.51856321, + "num_input_tokens_seen": 55152805, + "router_z_loss_clip": 0.16015625, + "router_z_loss_mlp": 0.0123291, + "step": 1951, + "time_per_iteration": 2.938185930252075 + }, + { + "auxiliary_loss_clip": 0.0101886, + "auxiliary_loss_mlp": 0.01003333, + "balance_loss_clip": 1.00266886, + "balance_loss_mlp": 1.00204551, + "epoch": 0.05664209854332308, + "flos": 52223988706560.0, + "grad_norm": 0.7207010746903143, + "language_loss": 0.50852591, + "learning_rate": 3.992561336334066e-06, + "loss": 0.52874786, + "num_input_tokens_seen": 55215420, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.01287842, + "step": 1952, + "time_per_iteration": 3.2993004322052 + }, + { + "auxiliary_loss_clip": 0.01018291, + "auxiliary_loss_mlp": 0.0100152, + "balance_loss_clip": 1.00205088, + "balance_loss_mlp": 1.00012481, + "epoch": 0.05667111601183913, + "flos": 69375420552960.0, + "grad_norm": 0.6810998974419389, + "language_loss": 0.49425673, + "learning_rate": 3.992545131349156e-06, + "loss": 0.51445484, + "num_input_tokens_seen": 55279415, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.01397705, + "step": 1953, + "time_per_iteration": 3.0614726543426514 + }, + { + "auxiliary_loss_clip": 0.01128017, + "auxiliary_loss_mlp": 0.01047952, + "balance_loss_clip": 1.03887904, + "balance_loss_mlp": 1.02648199, + "epoch": 0.05670013348035517, + "flos": 17302535677440.0, + "grad_norm": 3.7310192172366015, + "language_loss": 0.87092191, + "learning_rate": 3.992528908765282e-06, + "loss": 0.8926816, + "num_input_tokens_seen": 55292170, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.21472168, + "step": 1954, + "time_per_iteration": 2.439631938934326 + }, + { + "auxiliary_loss_clip": 0.01130345, + "auxiliary_loss_mlp": 0.01054971, + "balance_loss_clip": 1.03899503, + "balance_loss_mlp": 1.03155887, + "epoch": 0.056729150948871224, + "flos": 26825724610560.0, + "grad_norm": 2.487171811784783, + "language_loss": 0.80084789, + "learning_rate": 3.992512668582586e-06, + "loss": 0.8227011, + "num_input_tokens_seen": 55307090, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.23449707, + "step": 1955, + "time_per_iteration": 2.4509987831115723 + }, + { + "auxiliary_loss_clip": 0.0113416, + "auxiliary_loss_mlp": 0.01050656, + "balance_loss_clip": 1.04028869, + "balance_loss_mlp": 1.02605152, + "epoch": 0.05675816841738727, + "flos": 23578662389760.0, + "grad_norm": 2.3365280990901485, + "language_loss": 0.76713407, + "learning_rate": 3.992496410801212e-06, + "loss": 0.78898221, + "num_input_tokens_seen": 55322060, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.24609375, + "step": 1956, + "time_per_iteration": 2.463214635848999 + }, + { + "auxiliary_loss_clip": 0.01016998, + "auxiliary_loss_mlp": 0.01000107, + "balance_loss_clip": 1.00111008, + "balance_loss_mlp": 0.99872446, + "epoch": 0.05678718588590331, + "flos": 70209286398720.0, + "grad_norm": 0.6955640423160652, + "language_loss": 0.49762207, + "learning_rate": 3.992480135421303e-06, + "loss": 0.51779312, + "num_input_tokens_seen": 55384235, + "router_z_loss_clip": 0.15917969, + "router_z_loss_mlp": 0.01385498, + "step": 1957, + "time_per_iteration": 3.038266658782959 + }, + { + "auxiliary_loss_clip": 0.01137388, + "auxiliary_loss_mlp": 0.01050003, + "balance_loss_clip": 1.04017186, + "balance_loss_mlp": 1.02513564, + "epoch": 0.05681620335441936, + "flos": 19819703681280.0, + "grad_norm": 4.55434914414141, + "language_loss": 0.91412497, + "learning_rate": 3.9924638424430035e-06, + "loss": 0.93599892, + "num_input_tokens_seen": 55395960, + "router_z_loss_clip": 0.97314453, + "router_z_loss_mlp": 0.24890137, + "step": 1958, + "time_per_iteration": 2.4105546474456787 + }, + { + "auxiliary_loss_clip": 0.011477, + "auxiliary_loss_mlp": 0.0106053, + "balance_loss_clip": 1.03980684, + "balance_loss_mlp": 1.03269482, + "epoch": 0.05684522082293541, + "flos": 28833405010560.0, + "grad_norm": 2.1801680072568366, + "language_loss": 1.01581395, + "learning_rate": 3.992447531866457e-06, + "loss": 1.03789616, + "num_input_tokens_seen": 55415990, + "router_z_loss_clip": 1.07714844, + "router_z_loss_mlp": 0.27844238, + "step": 1959, + "time_per_iteration": 2.4919068813323975 + }, + { + "auxiliary_loss_clip": 0.01135202, + "auxiliary_loss_mlp": 0.01056914, + "balance_loss_clip": 1.04108262, + "balance_loss_mlp": 1.03179693, + "epoch": 0.056874238291451454, + "flos": 40288373314560.0, + "grad_norm": 2.4560191051826936, + "language_loss": 0.76744652, + "learning_rate": 3.992431203691807e-06, + "loss": 0.78936768, + "num_input_tokens_seen": 55436675, + "router_z_loss_clip": 0.94140625, + "router_z_loss_mlp": 0.25073242, + "step": 1960, + "time_per_iteration": 2.6130363941192627 + }, + { + "auxiliary_loss_clip": 0.01138224, + "auxiliary_loss_mlp": 0.01067448, + "balance_loss_clip": 1.03879476, + "balance_loss_mlp": 1.0366087, + "epoch": 0.0569032557599675, + "flos": 12596208693120.0, + "grad_norm": 2.320899589071867, + "language_loss": 0.93028402, + "learning_rate": 3.992414857919199e-06, + "loss": 0.95234072, + "num_input_tokens_seen": 55449595, + "router_z_loss_clip": 0.99365234, + "router_z_loss_mlp": 0.30822754, + "step": 1961, + "time_per_iteration": 2.4236392974853516 + }, + { + "auxiliary_loss_clip": 0.01145965, + "auxiliary_loss_mlp": 0.01062367, + "balance_loss_clip": 1.04381251, + "balance_loss_mlp": 1.03466308, + "epoch": 0.05693227322848355, + "flos": 31568675477760.0, + "grad_norm": 2.7614817644164322, + "language_loss": 0.83525562, + "learning_rate": 3.992398494548777e-06, + "loss": 0.85733891, + "num_input_tokens_seen": 55467800, + "router_z_loss_clip": 1.02197266, + "router_z_loss_mlp": 0.27709961, + "step": 1962, + "time_per_iteration": 2.5415852069854736 + }, + { + "auxiliary_loss_clip": 0.01018978, + "auxiliary_loss_mlp": 0.01001896, + "balance_loss_clip": 1.00288522, + "balance_loss_mlp": 1.00025117, + "epoch": 0.056961290696999595, + "flos": 74003507447040.0, + "grad_norm": 0.6976644615336308, + "language_loss": 0.5347445, + "learning_rate": 3.992382113580685e-06, + "loss": 0.55495322, + "num_input_tokens_seen": 55528745, + "router_z_loss_clip": 0.16113281, + "router_z_loss_mlp": 0.01647949, + "step": 1963, + "time_per_iteration": 3.0131046772003174 + }, + { + "auxiliary_loss_clip": 0.01141123, + "auxiliary_loss_mlp": 0.01065548, + "balance_loss_clip": 1.04004979, + "balance_loss_mlp": 1.03616309, + "epoch": 0.05699030816551564, + "flos": 35728472039040.0, + "grad_norm": 2.7226043021180577, + "language_loss": 0.95606297, + "learning_rate": 3.992365715015068e-06, + "loss": 0.97812968, + "num_input_tokens_seen": 55544640, + "router_z_loss_clip": 1.01025391, + "router_z_loss_mlp": 0.2935791, + "step": 1964, + "time_per_iteration": 2.576111078262329 + }, + { + "auxiliary_loss_clip": 0.01018976, + "auxiliary_loss_mlp": 0.01004086, + "balance_loss_clip": 1.00257957, + "balance_loss_mlp": 1.00226176, + "epoch": 0.057019325634031684, + "flos": 58566904583040.0, + "grad_norm": 0.7490120341499665, + "language_loss": 0.56243646, + "learning_rate": 3.992349298852071e-06, + "loss": 0.58266711, + "num_input_tokens_seen": 55601315, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01818848, + "step": 1965, + "time_per_iteration": 3.1720292568206787 + }, + { + "auxiliary_loss_clip": 0.01134615, + "auxiliary_loss_mlp": 0.01052839, + "balance_loss_clip": 1.03757167, + "balance_loss_mlp": 1.02852035, + "epoch": 0.057048343102547736, + "flos": 19789678045440.0, + "grad_norm": 2.864203659789531, + "language_loss": 1.02829576, + "learning_rate": 3.992332865091838e-06, + "loss": 1.05017018, + "num_input_tokens_seen": 55615290, + "router_z_loss_clip": 0.97119141, + "router_z_loss_mlp": 0.2432251, + "step": 1966, + "time_per_iteration": 2.4525513648986816 + }, + { + "auxiliary_loss_clip": 0.01135848, + "auxiliary_loss_mlp": 0.01049314, + "balance_loss_clip": 1.03826654, + "balance_loss_mlp": 1.02221727, + "epoch": 0.05707736057106378, + "flos": 19054163099520.0, + "grad_norm": 2.439155547663603, + "language_loss": 0.9272058, + "learning_rate": 3.992316413734516e-06, + "loss": 0.9490574, + "num_input_tokens_seen": 55630915, + "router_z_loss_clip": 0.97460938, + "router_z_loss_mlp": 0.27087402, + "step": 1967, + "time_per_iteration": 2.4767372608184814 + }, + { + "auxiliary_loss_clip": 0.01136865, + "auxiliary_loss_mlp": 0.01057887, + "balance_loss_clip": 1.04132652, + "balance_loss_mlp": 1.03165531, + "epoch": 0.057106378039579825, + "flos": 25786778238720.0, + "grad_norm": 2.713498483779722, + "language_loss": 0.88141739, + "learning_rate": 3.992299944780248e-06, + "loss": 0.90336496, + "num_input_tokens_seen": 55651145, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.26202393, + "step": 1968, + "time_per_iteration": 2.4916841983795166 + }, + { + "auxiliary_loss_clip": 0.01134182, + "auxiliary_loss_mlp": 0.0105008, + "balance_loss_clip": 1.03868175, + "balance_loss_mlp": 1.02636933, + "epoch": 0.057135395508095876, + "flos": 11101576855680.0, + "grad_norm": 2.7635085641667048, + "language_loss": 0.89857119, + "learning_rate": 3.9922834582291815e-06, + "loss": 0.92041379, + "num_input_tokens_seen": 55662780, + "router_z_loss_clip": 0.95507812, + "router_z_loss_mlp": 0.23730469, + "step": 1969, + "time_per_iteration": 2.3529787063598633 + }, + { + "auxiliary_loss_clip": 0.01146489, + "auxiliary_loss_mlp": 0.01059835, + "balance_loss_clip": 1.04368317, + "balance_loss_mlp": 1.03069997, + "epoch": 0.05716441297661192, + "flos": 15296635756800.0, + "grad_norm": 3.2964197084125355, + "language_loss": 0.87141705, + "learning_rate": 3.992266954081461e-06, + "loss": 0.89348024, + "num_input_tokens_seen": 55675295, + "router_z_loss_clip": 1.02929688, + "router_z_loss_mlp": 0.29150391, + "step": 1970, + "time_per_iteration": 2.43532395362854 + }, + { + "auxiliary_loss_clip": 0.01144813, + "auxiliary_loss_mlp": 0.01055986, + "balance_loss_clip": 1.04308558, + "balance_loss_mlp": 1.02875829, + "epoch": 0.057193430445127966, + "flos": 34528995768960.0, + "grad_norm": 2.068027253113939, + "language_loss": 0.960989, + "learning_rate": 3.992250432337233e-06, + "loss": 0.98299694, + "num_input_tokens_seen": 55700635, + "router_z_loss_clip": 1.01708984, + "router_z_loss_mlp": 0.27233887, + "step": 1971, + "time_per_iteration": 2.679720401763916 + }, + { + "auxiliary_loss_clip": 0.01018117, + "auxiliary_loss_mlp": 0.01003513, + "balance_loss_clip": 1.00167286, + "balance_loss_mlp": 1.0020287, + "epoch": 0.05722244791364401, + "flos": 74769292408320.0, + "grad_norm": 0.6690553045915648, + "language_loss": 0.50875282, + "learning_rate": 3.992233892996642e-06, + "loss": 0.52896911, + "num_input_tokens_seen": 55765515, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01483154, + "step": 1972, + "time_per_iteration": 3.076676845550537 + }, + { + "auxiliary_loss_clip": 0.01017695, + "auxiliary_loss_mlp": 0.01002619, + "balance_loss_clip": 1.00152254, + "balance_loss_mlp": 1.00118244, + "epoch": 0.05725146538216006, + "flos": 62147072885760.0, + "grad_norm": 0.6508969371722261, + "language_loss": 0.44673747, + "learning_rate": 3.992217336059836e-06, + "loss": 0.46694064, + "num_input_tokens_seen": 55828275, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.01434326, + "step": 1973, + "time_per_iteration": 3.055285692214966 + }, + { + "auxiliary_loss_clip": 0.01017564, + "auxiliary_loss_mlp": 0.01003253, + "balance_loss_clip": 1.00141978, + "balance_loss_mlp": 1.00161934, + "epoch": 0.05728048285067611, + "flos": 63751401815040.0, + "grad_norm": 0.6785240003754097, + "language_loss": 0.52152336, + "learning_rate": 3.9922007615269606e-06, + "loss": 0.54173148, + "num_input_tokens_seen": 55887525, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.01635742, + "step": 1974, + "time_per_iteration": 3.0360891819000244 + }, + { + "auxiliary_loss_clip": 0.01133682, + "auxiliary_loss_mlp": 0.01064824, + "balance_loss_clip": 1.04003787, + "balance_loss_mlp": 1.04100585, + "epoch": 0.05730950031919215, + "flos": 52182967858560.0, + "grad_norm": 2.5151640861952083, + "language_loss": 1.1107657, + "learning_rate": 3.992184169398161e-06, + "loss": 1.13275075, + "num_input_tokens_seen": 55907895, + "router_z_loss_clip": 0.93798828, + "router_z_loss_mlp": 0.23828125, + "step": 1975, + "time_per_iteration": 2.7239975929260254 + }, + { + "auxiliary_loss_clip": 0.01144078, + "auxiliary_loss_mlp": 0.01064376, + "balance_loss_clip": 1.04117155, + "balance_loss_mlp": 1.03418016, + "epoch": 0.0573385177877082, + "flos": 38251295683200.0, + "grad_norm": 2.364697597111792, + "language_loss": 0.90947968, + "learning_rate": 3.992167559673585e-06, + "loss": 0.93156421, + "num_input_tokens_seen": 55929165, + "router_z_loss_clip": 1.02978516, + "router_z_loss_mlp": 0.30200195, + "step": 1976, + "time_per_iteration": 2.584494113922119 + }, + { + "auxiliary_loss_clip": 0.01136532, + "auxiliary_loss_mlp": 0.0106181, + "balance_loss_clip": 1.04248178, + "balance_loss_mlp": 1.03623927, + "epoch": 0.05736753525622425, + "flos": 21499375057920.0, + "grad_norm": 2.134765530887392, + "language_loss": 0.60657841, + "learning_rate": 3.9921509323533796e-06, + "loss": 0.62856185, + "num_input_tokens_seen": 55942155, + "router_z_loss_clip": 0.94042969, + "router_z_loss_mlp": 0.25585938, + "step": 1977, + "time_per_iteration": 2.486107349395752 + }, + { + "auxiliary_loss_clip": 0.01135433, + "auxiliary_loss_mlp": 0.01051963, + "balance_loss_clip": 1.04084527, + "balance_loss_mlp": 1.02869344, + "epoch": 0.05739655272474029, + "flos": 37662555559680.0, + "grad_norm": 2.2189878060836428, + "language_loss": 0.82371145, + "learning_rate": 3.9921342874376906e-06, + "loss": 0.84558535, + "num_input_tokens_seen": 55961730, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.23278809, + "step": 1978, + "time_per_iteration": 2.6764309406280518 + }, + { + "auxiliary_loss_clip": 0.01135441, + "auxiliary_loss_mlp": 0.01050652, + "balance_loss_clip": 1.04077029, + "balance_loss_mlp": 1.025213, + "epoch": 0.057425570193256344, + "flos": 51052829281920.0, + "grad_norm": 2.7512232521372604, + "language_loss": 0.7748037, + "learning_rate": 3.992117624926665e-06, + "loss": 0.79666466, + "num_input_tokens_seen": 55980090, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.25439453, + "step": 1979, + "time_per_iteration": 5.956955432891846 + }, + { + "auxiliary_loss_clip": 0.01137137, + "auxiliary_loss_mlp": 0.01046249, + "balance_loss_clip": 1.03932762, + "balance_loss_mlp": 1.02089357, + "epoch": 0.05745458766177239, + "flos": 12050236851840.0, + "grad_norm": 2.5979402848031237, + "language_loss": 0.77001381, + "learning_rate": 3.992100944820451e-06, + "loss": 0.79184771, + "num_input_tokens_seen": 55994030, + "router_z_loss_clip": 0.97949219, + "router_z_loss_mlp": 0.25366211, + "step": 1980, + "time_per_iteration": 2.415802478790283 + }, + { + "auxiliary_loss_clip": 0.01138443, + "auxiliary_loss_mlp": 0.01058946, + "balance_loss_clip": 1.04227054, + "balance_loss_mlp": 1.03232658, + "epoch": 0.05748360513028843, + "flos": 11137816713600.0, + "grad_norm": 2.686957406241139, + "language_loss": 0.96630788, + "learning_rate": 3.992084247119194e-06, + "loss": 0.98828173, + "num_input_tokens_seen": 56006445, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.26623535, + "step": 1981, + "time_per_iteration": 2.366823673248291 + }, + { + "auxiliary_loss_clip": 0.01126948, + "auxiliary_loss_mlp": 0.0103984, + "balance_loss_clip": 1.03768158, + "balance_loss_mlp": 1.0177027, + "epoch": 0.05751262259880448, + "flos": 20915836727040.0, + "grad_norm": 3.8092820456886685, + "language_loss": 0.79709947, + "learning_rate": 3.9920675318230445e-06, + "loss": 0.81876743, + "num_input_tokens_seen": 56019140, + "router_z_loss_clip": 0.89355469, + "router_z_loss_mlp": 0.22119141, + "step": 1982, + "time_per_iteration": 4.7380592823028564 + }, + { + "auxiliary_loss_clip": 0.01130449, + "auxiliary_loss_mlp": 0.01052808, + "balance_loss_clip": 1.03715587, + "balance_loss_mlp": 1.02694511, + "epoch": 0.05754164006732053, + "flos": 32116392887040.0, + "grad_norm": 2.235287330253791, + "language_loss": 0.73204267, + "learning_rate": 3.992050798932148e-06, + "loss": 0.75387526, + "num_input_tokens_seen": 56037580, + "router_z_loss_clip": 0.93261719, + "router_z_loss_mlp": 0.25866699, + "step": 1983, + "time_per_iteration": 2.6493780612945557 + }, + { + "auxiliary_loss_clip": 0.01133861, + "auxiliary_loss_mlp": 0.01048725, + "balance_loss_clip": 1.03984261, + "balance_loss_mlp": 1.02457273, + "epoch": 0.057570657535836574, + "flos": 20262122830080.0, + "grad_norm": 4.964502261984182, + "language_loss": 0.81170458, + "learning_rate": 3.992034048446652e-06, + "loss": 0.83353043, + "num_input_tokens_seen": 56056180, + "router_z_loss_clip": 0.94091797, + "router_z_loss_mlp": 0.24145508, + "step": 1984, + "time_per_iteration": 2.7032041549682617 + }, + { + "auxiliary_loss_clip": 0.01020466, + "auxiliary_loss_mlp": 0.01005817, + "balance_loss_clip": 1.00322318, + "balance_loss_mlp": 1.00419617, + "epoch": 0.05759967500435262, + "flos": 60141382433280.0, + "grad_norm": 0.7333859045702747, + "language_loss": 0.53125644, + "learning_rate": 3.992017280366706e-06, + "loss": 0.55151927, + "num_input_tokens_seen": 56114660, + "router_z_loss_clip": 0.171875, + "router_z_loss_mlp": 0.01623535, + "step": 1985, + "time_per_iteration": 5.351144790649414 + }, + { + "auxiliary_loss_clip": 0.01123094, + "auxiliary_loss_mlp": 0.01044261, + "balance_loss_clip": 1.03420818, + "balance_loss_mlp": 1.02105093, + "epoch": 0.05762869247286867, + "flos": 20183778005760.0, + "grad_norm": 2.428128396168138, + "language_loss": 1.00443566, + "learning_rate": 3.9920004946924574e-06, + "loss": 1.02610922, + "num_input_tokens_seen": 56128535, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.23217773, + "step": 1986, + "time_per_iteration": 4.947774887084961 + }, + { + "auxiliary_loss_clip": 0.01020092, + "auxiliary_loss_mlp": 0.01003193, + "balance_loss_clip": 1.00276744, + "balance_loss_mlp": 1.0015595, + "epoch": 0.057657709941384715, + "flos": 63792564174720.0, + "grad_norm": 0.7212952285061128, + "language_loss": 0.50562751, + "learning_rate": 3.991983691424054e-06, + "loss": 0.52586037, + "num_input_tokens_seen": 56189080, + "router_z_loss_clip": 0.17382812, + "router_z_loss_mlp": 0.01635742, + "step": 1987, + "time_per_iteration": 3.010291337966919 + }, + { + "auxiliary_loss_clip": 0.01135044, + "auxiliary_loss_mlp": 0.01066912, + "balance_loss_clip": 1.0415051, + "balance_loss_mlp": 1.04260492, + "epoch": 0.05768672740990076, + "flos": 13946928439680.0, + "grad_norm": 3.5930782998887256, + "language_loss": 1.03207016, + "learning_rate": 3.991966870561644e-06, + "loss": 1.05408955, + "num_input_tokens_seen": 56202505, + "router_z_loss_clip": 0.93603516, + "router_z_loss_mlp": 0.24291992, + "step": 1988, + "time_per_iteration": 2.7351174354553223 + }, + { + "auxiliary_loss_clip": 0.01128533, + "auxiliary_loss_mlp": 0.01055821, + "balance_loss_clip": 1.03909659, + "balance_loss_mlp": 1.03153849, + "epoch": 0.057715744878416804, + "flos": 45213710457600.0, + "grad_norm": 2.2408226356089633, + "language_loss": 1.02372003, + "learning_rate": 3.991950032105378e-06, + "loss": 1.04556358, + "num_input_tokens_seen": 56222880, + "router_z_loss_clip": 0.89501953, + "router_z_loss_mlp": 0.24279785, + "step": 1989, + "time_per_iteration": 2.6183059215545654 + }, + { + "auxiliary_loss_clip": 0.01134284, + "auxiliary_loss_mlp": 0.01054214, + "balance_loss_clip": 1.03572786, + "balance_loss_mlp": 1.02588964, + "epoch": 0.057744762346932856, + "flos": 12340469917440.0, + "grad_norm": 2.433124790050521, + "language_loss": 0.92830491, + "learning_rate": 3.991933176055402e-06, + "loss": 0.95018983, + "num_input_tokens_seen": 56234730, + "router_z_loss_clip": 0.98535156, + "router_z_loss_mlp": 0.28344727, + "step": 1990, + "time_per_iteration": 2.423211097717285 + }, + { + "auxiliary_loss_clip": 0.01138284, + "auxiliary_loss_mlp": 0.01054408, + "balance_loss_clip": 1.0412395, + "balance_loss_mlp": 1.02776504, + "epoch": 0.0577737798154489, + "flos": 25694817984000.0, + "grad_norm": 2.2311630013342105, + "language_loss": 0.9410708, + "learning_rate": 3.991916302411866e-06, + "loss": 0.96299767, + "num_input_tokens_seen": 56248735, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.26647949, + "step": 1991, + "time_per_iteration": 2.445559501647949 + }, + { + "auxiliary_loss_clip": 0.01020079, + "auxiliary_loss_mlp": 0.01002983, + "balance_loss_clip": 1.0030688, + "balance_loss_mlp": 1.00143301, + "epoch": 0.057802797283964945, + "flos": 58794991695360.0, + "grad_norm": 0.6720869170510317, + "language_loss": 0.53155738, + "learning_rate": 3.9918994111749194e-06, + "loss": 0.55178803, + "num_input_tokens_seen": 56310655, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01550293, + "step": 1992, + "time_per_iteration": 3.011256217956543 + }, + { + "auxiliary_loss_clip": 0.01150541, + "auxiliary_loss_mlp": 0.01074166, + "balance_loss_clip": 1.04453039, + "balance_loss_mlp": 1.04523432, + "epoch": 0.057831814752480996, + "flos": 27483174023040.0, + "grad_norm": 2.387305904277683, + "language_loss": 0.93779272, + "learning_rate": 3.991882502344712e-06, + "loss": 0.96003979, + "num_input_tokens_seen": 56328190, + "router_z_loss_clip": 1.06103516, + "router_z_loss_mlp": 0.28942871, + "step": 1993, + "time_per_iteration": 2.4865646362304688 + }, + { + "auxiliary_loss_clip": 0.01019674, + "auxiliary_loss_mlp": 0.01004673, + "balance_loss_clip": 1.00235295, + "balance_loss_mlp": 1.003088, + "epoch": 0.05786083222099704, + "flos": 63060575276160.0, + "grad_norm": 0.672022932915424, + "language_loss": 0.49331, + "learning_rate": 3.991865575921392e-06, + "loss": 0.5135535, + "num_input_tokens_seen": 56388670, + "router_z_loss_clip": 0.17285156, + "router_z_loss_mlp": 0.01586914, + "step": 1994, + "time_per_iteration": 2.94358229637146 + }, + { + "auxiliary_loss_clip": 0.01019666, + "auxiliary_loss_mlp": 0.01003017, + "balance_loss_clip": 1.00228667, + "balance_loss_mlp": 1.00153923, + "epoch": 0.057889849689513086, + "flos": 66787658046720.0, + "grad_norm": 0.6672165954769109, + "language_loss": 0.48233926, + "learning_rate": 3.9918486319051084e-06, + "loss": 0.5025661, + "num_input_tokens_seen": 56450730, + "router_z_loss_clip": 0.17382812, + "router_z_loss_mlp": 0.01477051, + "step": 1995, + "time_per_iteration": 3.112018346786499 + }, + { + "auxiliary_loss_clip": 0.01019075, + "auxiliary_loss_mlp": 0.01003045, + "balance_loss_clip": 1.00175238, + "balance_loss_mlp": 1.00147104, + "epoch": 0.05791886715802913, + "flos": 70829867548800.0, + "grad_norm": 0.663463755668712, + "language_loss": 0.46380451, + "learning_rate": 3.991831670296013e-06, + "loss": 0.48402572, + "num_input_tokens_seen": 56507510, + "router_z_loss_clip": 0.17285156, + "router_z_loss_mlp": 0.01574707, + "step": 1996, + "time_per_iteration": 3.0371625423431396 + }, + { + "auxiliary_loss_clip": 0.01145701, + "auxiliary_loss_mlp": 0.01063311, + "balance_loss_clip": 1.03959322, + "balance_loss_mlp": 1.03418851, + "epoch": 0.05794788462654518, + "flos": 22234436156160.0, + "grad_norm": 2.00403665531181, + "language_loss": 0.82933772, + "learning_rate": 3.991814691094253e-06, + "loss": 0.85142785, + "num_input_tokens_seen": 56524785, + "router_z_loss_clip": 1.06054688, + "router_z_loss_mlp": 0.29125977, + "step": 1997, + "time_per_iteration": 2.462944984436035 + }, + { + "auxiliary_loss_clip": 0.01018643, + "auxiliary_loss_mlp": 0.01000897, + "balance_loss_clip": 1.00155842, + "balance_loss_mlp": 0.99945438, + "epoch": 0.05797690209506123, + "flos": 70281382089600.0, + "grad_norm": 0.7793225603915847, + "language_loss": 0.50274479, + "learning_rate": 3.99179769429998e-06, + "loss": 0.52294016, + "num_input_tokens_seen": 56592375, + "router_z_loss_clip": 0.17089844, + "router_z_loss_mlp": 0.0144043, + "step": 1998, + "time_per_iteration": 3.1414287090301514 + }, + { + "auxiliary_loss_clip": 0.01138601, + "auxiliary_loss_mlp": 0.01056361, + "balance_loss_clip": 1.04086852, + "balance_loss_mlp": 1.03125525, + "epoch": 0.05800591956357727, + "flos": 30403518940800.0, + "grad_norm": 2.1583496189080966, + "language_loss": 0.8929137, + "learning_rate": 3.991780679913344e-06, + "loss": 0.91486329, + "num_input_tokens_seen": 56610465, + "router_z_loss_clip": 0.97607422, + "router_z_loss_mlp": 0.2512207, + "step": 1999, + "time_per_iteration": 2.5445353984832764 + }, + { + "auxiliary_loss_clip": 0.01018728, + "auxiliary_loss_mlp": 0.01001037, + "balance_loss_clip": 1.00169158, + "balance_loss_mlp": 0.99948704, + "epoch": 0.05803493703209332, + "flos": 74774354555520.0, + "grad_norm": 0.6963319601938718, + "language_loss": 0.52083051, + "learning_rate": 3.991763647934495e-06, + "loss": 0.54102814, + "num_input_tokens_seen": 56669135, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01544189, + "step": 2000, + "time_per_iteration": 3.135969400405884 + }, + { + "auxiliary_loss_clip": 0.01127637, + "auxiliary_loss_mlp": 0.01048068, + "balance_loss_clip": 1.03760004, + "balance_loss_mlp": 1.02392817, + "epoch": 0.05806395450060937, + "flos": 15594479498880.0, + "grad_norm": 2.7879719661133517, + "language_loss": 0.82067573, + "learning_rate": 3.991746598363583e-06, + "loss": 0.8424328, + "num_input_tokens_seen": 56683985, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.24133301, + "step": 2001, + "time_per_iteration": 2.4218335151672363 + }, + { + "auxiliary_loss_clip": 0.01123291, + "auxiliary_loss_mlp": 0.01041645, + "balance_loss_clip": 1.03453696, + "balance_loss_mlp": 1.01847053, + "epoch": 0.05809297196912541, + "flos": 16465492897920.0, + "grad_norm": 3.3499564403163617, + "language_loss": 0.77423531, + "learning_rate": 3.99172953120076e-06, + "loss": 0.79588461, + "num_input_tokens_seen": 56698400, + "router_z_loss_clip": 0.88720703, + "router_z_loss_mlp": 0.23181152, + "step": 2002, + "time_per_iteration": 2.3618383407592773 + }, + { + "auxiliary_loss_clip": 0.01122499, + "auxiliary_loss_mlp": 0.01048573, + "balance_loss_clip": 1.03703308, + "balance_loss_mlp": 1.02629232, + "epoch": 0.05812198943764146, + "flos": 31605753208320.0, + "grad_norm": 2.2470709534396995, + "language_loss": 0.72786796, + "learning_rate": 3.991712446446175e-06, + "loss": 0.74957871, + "num_input_tokens_seen": 56712595, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.22290039, + "step": 2003, + "time_per_iteration": 2.568464994430542 + }, + { + "auxiliary_loss_clip": 0.01017927, + "auxiliary_loss_mlp": 0.01000621, + "balance_loss_clip": 1.00113237, + "balance_loss_mlp": 0.99904734, + "epoch": 0.05815100690615751, + "flos": 61935463935360.0, + "grad_norm": 0.667655829853497, + "language_loss": 0.51029551, + "learning_rate": 3.991695344099981e-06, + "loss": 0.53048098, + "num_input_tokens_seen": 56782225, + "router_z_loss_clip": 0.16796875, + "router_z_loss_mlp": 0.01574707, + "step": 2004, + "time_per_iteration": 3.179860830307007 + }, + { + "auxiliary_loss_clip": 0.01120234, + "auxiliary_loss_mlp": 0.01043955, + "balance_loss_clip": 1.03706706, + "balance_loss_mlp": 1.02225876, + "epoch": 0.05818002437467355, + "flos": 32954378273280.0, + "grad_norm": 3.2773651399115877, + "language_loss": 0.95336962, + "learning_rate": 3.991678224162326e-06, + "loss": 0.97501147, + "num_input_tokens_seen": 56794750, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.21679688, + "step": 2005, + "time_per_iteration": 2.542644500732422 + }, + { + "auxiliary_loss_clip": 0.01017233, + "auxiliary_loss_mlp": 0.0100048, + "balance_loss_clip": 1.0008049, + "balance_loss_mlp": 0.99897814, + "epoch": 0.0582090418431896, + "flos": 59814595307520.0, + "grad_norm": 0.6863579927887808, + "language_loss": 0.51678473, + "learning_rate": 3.991661086633364e-06, + "loss": 0.53696185, + "num_input_tokens_seen": 56855465, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01501465, + "step": 2006, + "time_per_iteration": 2.974256753921509 + }, + { + "auxiliary_loss_clip": 0.01144439, + "auxiliary_loss_mlp": 0.01057609, + "balance_loss_clip": 1.04274869, + "balance_loss_mlp": 1.03050041, + "epoch": 0.05823805931170565, + "flos": 52402501687680.0, + "grad_norm": 2.928416188555958, + "language_loss": 0.85842687, + "learning_rate": 3.9916439315132455e-06, + "loss": 0.88044733, + "num_input_tokens_seen": 56873740, + "router_z_loss_clip": 1.015625, + "router_z_loss_mlp": 0.27087402, + "step": 2007, + "time_per_iteration": 2.614006519317627 + }, + { + "auxiliary_loss_clip": 0.01134364, + "auxiliary_loss_mlp": 0.01061031, + "balance_loss_clip": 1.04038358, + "balance_loss_mlp": 1.03593755, + "epoch": 0.058267076780221694, + "flos": 31276522287360.0, + "grad_norm": 2.5062119230419717, + "language_loss": 0.8965441, + "learning_rate": 3.9916267588021215e-06, + "loss": 0.91849804, + "num_input_tokens_seen": 56889550, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.2512207, + "step": 2008, + "time_per_iteration": 2.533311605453491 + }, + { + "auxiliary_loss_clip": 0.01135645, + "auxiliary_loss_mlp": 0.01051051, + "balance_loss_clip": 1.04407179, + "balance_loss_mlp": 1.0266248, + "epoch": 0.05829609424873774, + "flos": 74730924823680.0, + "grad_norm": 2.35885445461373, + "language_loss": 0.85756564, + "learning_rate": 3.991609568500144e-06, + "loss": 0.8794325, + "num_input_tokens_seen": 56914560, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.2442627, + "step": 2009, + "time_per_iteration": 2.811138391494751 + }, + { + "auxiliary_loss_clip": 0.01131605, + "auxiliary_loss_mlp": 0.01047231, + "balance_loss_clip": 1.03847933, + "balance_loss_mlp": 1.02326989, + "epoch": 0.05832511171725379, + "flos": 11683893288960.0, + "grad_norm": 2.945528591568276, + "language_loss": 1.04352117, + "learning_rate": 3.991592360607465e-06, + "loss": 1.06530952, + "num_input_tokens_seen": 56928070, + "router_z_loss_clip": 0.93115234, + "router_z_loss_mlp": 0.23974609, + "step": 2010, + "time_per_iteration": 2.4493565559387207 + }, + { + "auxiliary_loss_clip": 0.01134886, + "auxiliary_loss_mlp": 0.01058636, + "balance_loss_clip": 1.0401026, + "balance_loss_mlp": 1.0315156, + "epoch": 0.058354129185769835, + "flos": 10772485580160.0, + "grad_norm": 3.507893913258437, + "language_loss": 1.01805067, + "learning_rate": 3.991575135124236e-06, + "loss": 1.0399859, + "num_input_tokens_seen": 56938995, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.27124023, + "step": 2011, + "time_per_iteration": 2.428098201751709 + }, + { + "auxiliary_loss_clip": 0.0101895, + "auxiliary_loss_mlp": 0.0101152, + "balance_loss_clip": 1.00221109, + "balance_loss_mlp": 1.00999367, + "epoch": 0.05838314665428588, + "flos": 74769082940160.0, + "grad_norm": 0.7141026010762975, + "language_loss": 0.53977489, + "learning_rate": 3.9915578920506095e-06, + "loss": 0.56007957, + "num_input_tokens_seen": 57002925, + "router_z_loss_clip": 0.16796875, + "router_z_loss_mlp": 0.01525879, + "step": 2012, + "time_per_iteration": 3.0942819118499756 + }, + { + "auxiliary_loss_clip": 0.01138389, + "auxiliary_loss_mlp": 0.01056153, + "balance_loss_clip": 1.04059672, + "balance_loss_mlp": 1.02836514, + "epoch": 0.058412164122801924, + "flos": 48971761470720.0, + "grad_norm": 2.4321703084483635, + "language_loss": 0.76261795, + "learning_rate": 3.991540631386739e-06, + "loss": 0.7845633, + "num_input_tokens_seen": 57021115, + "router_z_loss_clip": 0.97851562, + "router_z_loss_mlp": 0.2779541, + "step": 2013, + "time_per_iteration": 2.8061928749084473 + }, + { + "auxiliary_loss_clip": 0.0101773, + "auxiliary_loss_mlp": 0.01007702, + "balance_loss_clip": 1.00117922, + "balance_loss_mlp": 1.00618768, + "epoch": 0.058441181591317976, + "flos": 74779975284480.0, + "grad_norm": 0.6276213732520662, + "language_loss": 0.47811326, + "learning_rate": 3.991523353132774e-06, + "loss": 0.49836758, + "num_input_tokens_seen": 57090410, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01513672, + "step": 2014, + "time_per_iteration": 3.2446858882904053 + }, + { + "auxiliary_loss_clip": 0.01131767, + "auxiliary_loss_mlp": 0.01047113, + "balance_loss_clip": 1.03920293, + "balance_loss_mlp": 1.02284169, + "epoch": 0.05847019905983402, + "flos": 39924089521920.0, + "grad_norm": 2.2592483737260873, + "language_loss": 1.00753403, + "learning_rate": 3.99150605728887e-06, + "loss": 1.02932274, + "num_input_tokens_seen": 57107745, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.24291992, + "step": 2015, + "time_per_iteration": 2.5759739875793457 + }, + { + "auxiliary_loss_clip": 0.0112138, + "auxiliary_loss_mlp": 0.01030666, + "balance_loss_clip": 1.03737402, + "balance_loss_mlp": 1.01209331, + "epoch": 0.058499216528350065, + "flos": 13837755018240.0, + "grad_norm": 2.1431081638151555, + "language_loss": 0.61394119, + "learning_rate": 3.991488743855178e-06, + "loss": 0.63546163, + "num_input_tokens_seen": 57120560, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.18579102, + "step": 2016, + "time_per_iteration": 2.3827481269836426 + }, + { + "auxiliary_loss_clip": 0.01129646, + "auxiliary_loss_mlp": 0.01051406, + "balance_loss_clip": 1.03586006, + "balance_loss_mlp": 1.02587116, + "epoch": 0.058528233996866116, + "flos": 15881465808000.0, + "grad_norm": 2.7498288748323865, + "language_loss": 0.66613638, + "learning_rate": 3.9914714128318515e-06, + "loss": 0.68794692, + "num_input_tokens_seen": 57139850, + "router_z_loss_clip": 0.9375, + "router_z_loss_mlp": 0.25561523, + "step": 2017, + "time_per_iteration": 2.480679512023926 + }, + { + "auxiliary_loss_clip": 0.01136102, + "auxiliary_loss_mlp": 0.01056467, + "balance_loss_clip": 1.04051518, + "balance_loss_mlp": 1.03176665, + "epoch": 0.05855725146538216, + "flos": 27269784593280.0, + "grad_norm": 1.8354593647058772, + "language_loss": 0.96036333, + "learning_rate": 3.9914540642190445e-06, + "loss": 0.98228896, + "num_input_tokens_seen": 57160660, + "router_z_loss_clip": 0.95654297, + "router_z_loss_mlp": 0.24731445, + "step": 2018, + "time_per_iteration": 2.500910520553589 + }, + { + "auxiliary_loss_clip": 0.01020539, + "auxiliary_loss_mlp": 0.01002332, + "balance_loss_clip": 1.00400031, + "balance_loss_mlp": 1.0006392, + "epoch": 0.058586268933898206, + "flos": 70351627478400.0, + "grad_norm": 0.7012583377456362, + "language_loss": 0.50281286, + "learning_rate": 3.991436698016909e-06, + "loss": 0.52304161, + "num_input_tokens_seen": 57216190, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01696777, + "step": 2019, + "time_per_iteration": 2.935405731201172 + }, + { + "auxiliary_loss_clip": 0.01020308, + "auxiliary_loss_mlp": 0.01002515, + "balance_loss_clip": 1.0036943, + "balance_loss_mlp": 1.00089347, + "epoch": 0.05861528640241425, + "flos": 74781790675200.0, + "grad_norm": 0.6878804888993736, + "language_loss": 0.51568794, + "learning_rate": 3.991419314225598e-06, + "loss": 0.53591621, + "num_input_tokens_seen": 57279550, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01623535, + "step": 2020, + "time_per_iteration": 3.114192485809326 + }, + { + "auxiliary_loss_clip": 0.01129645, + "auxiliary_loss_mlp": 0.01061624, + "balance_loss_clip": 1.04179013, + "balance_loss_mlp": 1.03902233, + "epoch": 0.0586443038709303, + "flos": 18652661930880.0, + "grad_norm": 2.510818291240637, + "language_loss": 0.9044373, + "learning_rate": 3.991401912845267e-06, + "loss": 0.92635, + "num_input_tokens_seen": 57293315, + "router_z_loss_clip": 0.87841797, + "router_z_loss_mlp": 0.22619629, + "step": 2021, + "time_per_iteration": 2.4196290969848633 + }, + { + "auxiliary_loss_clip": 0.01126512, + "auxiliary_loss_mlp": 0.01045906, + "balance_loss_clip": 1.03804827, + "balance_loss_mlp": 1.02350092, + "epoch": 0.05867332133944635, + "flos": 14712503932800.0, + "grad_norm": 2.989036706048114, + "language_loss": 0.71328551, + "learning_rate": 3.9913844938760675e-06, + "loss": 0.73500967, + "num_input_tokens_seen": 57304815, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.22442627, + "step": 2022, + "time_per_iteration": 2.3877155780792236 + }, + { + "auxiliary_loss_clip": 0.01139901, + "auxiliary_loss_mlp": 0.01071023, + "balance_loss_clip": 1.04197001, + "balance_loss_mlp": 1.04526234, + "epoch": 0.05870233880796239, + "flos": 37700750453760.0, + "grad_norm": 2.726741377822889, + "language_loss": 0.90771639, + "learning_rate": 3.991367057318155e-06, + "loss": 0.9298256, + "num_input_tokens_seen": 57324750, + "router_z_loss_clip": 0.97949219, + "router_z_loss_mlp": 0.2578125, + "step": 2023, + "time_per_iteration": 2.578159809112549 + }, + { + "auxiliary_loss_clip": 0.01019205, + "auxiliary_loss_mlp": 0.01003307, + "balance_loss_clip": 1.00218046, + "balance_loss_mlp": 1.00165045, + "epoch": 0.05873135627647844, + "flos": 67516749302400.0, + "grad_norm": 0.6374861662944579, + "language_loss": 0.54009873, + "learning_rate": 3.9913496031716816e-06, + "loss": 0.56032389, + "num_input_tokens_seen": 57389645, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01660156, + "step": 2024, + "time_per_iteration": 3.067861318588257 + }, + { + "auxiliary_loss_clip": 0.01018946, + "auxiliary_loss_mlp": 0.01004415, + "balance_loss_clip": 1.00190663, + "balance_loss_mlp": 1.00272202, + "epoch": 0.05876037374499449, + "flos": 63052755131520.0, + "grad_norm": 0.7105815038625306, + "language_loss": 0.50225061, + "learning_rate": 3.991332131436804e-06, + "loss": 0.52248424, + "num_input_tokens_seen": 57445510, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01696777, + "step": 2025, + "time_per_iteration": 2.9166061878204346 + }, + { + "auxiliary_loss_clip": 0.01018464, + "auxiliary_loss_mlp": 0.01002201, + "balance_loss_clip": 1.00154281, + "balance_loss_mlp": 1.00050855, + "epoch": 0.05878939121351053, + "flos": 59034950536320.0, + "grad_norm": 0.7280751445180286, + "language_loss": 0.5156545, + "learning_rate": 3.991314642113675e-06, + "loss": 0.53586113, + "num_input_tokens_seen": 57501545, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01696777, + "step": 2026, + "time_per_iteration": 3.2676174640655518 + }, + { + "auxiliary_loss_clip": 0.01018947, + "auxiliary_loss_mlp": 0.01001506, + "balance_loss_clip": 1.00232553, + "balance_loss_mlp": 0.99990827, + "epoch": 0.05881840868202658, + "flos": 62157790673280.0, + "grad_norm": 0.6331582936417179, + "language_loss": 0.47733748, + "learning_rate": 3.991297135202448e-06, + "loss": 0.49754202, + "num_input_tokens_seen": 57561480, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01599121, + "step": 2027, + "time_per_iteration": 3.0503089427948 + }, + { + "auxiliary_loss_clip": 0.01123012, + "auxiliary_loss_mlp": 0.01045149, + "balance_loss_clip": 1.03720593, + "balance_loss_mlp": 1.02137852, + "epoch": 0.05884742615054263, + "flos": 37742331749760.0, + "grad_norm": 4.228120328823215, + "language_loss": 0.92573315, + "learning_rate": 3.991279610703281e-06, + "loss": 0.94741476, + "num_input_tokens_seen": 57577105, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.23779297, + "step": 2028, + "time_per_iteration": 2.503897190093994 + }, + { + "auxiliary_loss_clip": 0.01138706, + "auxiliary_loss_mlp": 0.01049039, + "balance_loss_clip": 1.0416497, + "balance_loss_mlp": 1.02365887, + "epoch": 0.05887644361905867, + "flos": 39687377437440.0, + "grad_norm": 2.009694772453196, + "language_loss": 0.97875917, + "learning_rate": 3.991262068616325e-06, + "loss": 1.00063658, + "num_input_tokens_seen": 57602400, + "router_z_loss_clip": 0.97021484, + "router_z_loss_mlp": 0.25390625, + "step": 2029, + "time_per_iteration": 2.6165242195129395 + }, + { + "auxiliary_loss_clip": 0.01122247, + "auxiliary_loss_mlp": 0.01039018, + "balance_loss_clip": 1.03783631, + "balance_loss_mlp": 1.01722682, + "epoch": 0.05890546108757472, + "flos": 15915436427520.0, + "grad_norm": 2.4705082106217704, + "language_loss": 0.73244035, + "learning_rate": 3.991244508941737e-06, + "loss": 0.75405306, + "num_input_tokens_seen": 57616680, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.21789551, + "step": 2030, + "time_per_iteration": 2.424501419067383 + }, + { + "auxiliary_loss_clip": 0.01024518, + "auxiliary_loss_mlp": 0.01009523, + "balance_loss_clip": 1.00744677, + "balance_loss_mlp": 1.00768709, + "epoch": 0.05893447855609077, + "flos": 70471029928320.0, + "grad_norm": 0.6977743529529506, + "language_loss": 0.52345049, + "learning_rate": 3.991226931679672e-06, + "loss": 0.54379094, + "num_input_tokens_seen": 57680890, + "router_z_loss_clip": 0.16992188, + "router_z_loss_mlp": 0.01831055, + "step": 2031, + "time_per_iteration": 3.0454089641571045 + }, + { + "auxiliary_loss_clip": 0.01117119, + "auxiliary_loss_mlp": 0.01038582, + "balance_loss_clip": 1.03744864, + "balance_loss_mlp": 1.01801825, + "epoch": 0.058963496024606814, + "flos": 12158397843840.0, + "grad_norm": 3.510162314282441, + "language_loss": 0.76653343, + "learning_rate": 3.991209336830285e-06, + "loss": 0.78809047, + "num_input_tokens_seen": 57692135, + "router_z_loss_clip": 0.79638672, + "router_z_loss_mlp": 0.20568848, + "step": 2032, + "time_per_iteration": 2.406644582748413 + }, + { + "auxiliary_loss_clip": 0.01120841, + "auxiliary_loss_mlp": 0.01049775, + "balance_loss_clip": 1.03754187, + "balance_loss_mlp": 1.0301652, + "epoch": 0.05899251349312286, + "flos": 23213017054080.0, + "grad_norm": 2.0833998507732794, + "language_loss": 0.75792366, + "learning_rate": 3.991191724393732e-06, + "loss": 0.77962983, + "num_input_tokens_seen": 57705305, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.19592285, + "step": 2033, + "time_per_iteration": 2.4470951557159424 + }, + { + "auxiliary_loss_clip": 0.01135818, + "auxiliary_loss_mlp": 0.01050607, + "balance_loss_clip": 1.04236531, + "balance_loss_mlp": 1.02636039, + "epoch": 0.05902153096163891, + "flos": 23616054322560.0, + "grad_norm": 2.6115717905075195, + "language_loss": 0.72890979, + "learning_rate": 3.991174094370167e-06, + "loss": 0.75077403, + "num_input_tokens_seen": 57718525, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.24255371, + "step": 2034, + "time_per_iteration": 2.492889404296875 + }, + { + "auxiliary_loss_clip": 0.01130132, + "auxiliary_loss_mlp": 0.01045054, + "balance_loss_clip": 1.04427838, + "balance_loss_mlp": 1.02302444, + "epoch": 0.059050548430154955, + "flos": 24891431621760.0, + "grad_norm": 2.544581395058192, + "language_loss": 1.07618499, + "learning_rate": 3.991156446759747e-06, + "loss": 1.09793675, + "num_input_tokens_seen": 57734605, + "router_z_loss_clip": 0.859375, + "router_z_loss_mlp": 0.22058105, + "step": 2035, + "time_per_iteration": 2.4326834678649902 + }, + { + "auxiliary_loss_clip": 0.01021287, + "auxiliary_loss_mlp": 0.01000723, + "balance_loss_clip": 1.00470948, + "balance_loss_mlp": 0.99901789, + "epoch": 0.059079565898671, + "flos": 71048703150720.0, + "grad_norm": 0.6748812760791539, + "language_loss": 0.51248884, + "learning_rate": 3.991138781562627e-06, + "loss": 0.53270888, + "num_input_tokens_seen": 57795595, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01708984, + "step": 2036, + "time_per_iteration": 3.021651029586792 + }, + { + "auxiliary_loss_clip": 0.01136824, + "auxiliary_loss_mlp": 0.0104623, + "balance_loss_clip": 1.03736329, + "balance_loss_mlp": 1.02126741, + "epoch": 0.059108583367187044, + "flos": 74734346136960.0, + "grad_norm": 2.7914109937294773, + "language_loss": 0.93877935, + "learning_rate": 3.991121098778964e-06, + "loss": 0.96060991, + "num_input_tokens_seen": 57817550, + "router_z_loss_clip": 0.99462891, + "router_z_loss_mlp": 0.24951172, + "step": 2037, + "time_per_iteration": 2.835513114929199 + }, + { + "auxiliary_loss_clip": 0.01142581, + "auxiliary_loss_mlp": 0.01054995, + "balance_loss_clip": 1.04395843, + "balance_loss_mlp": 1.02538383, + "epoch": 0.059137600835703096, + "flos": 10915943823360.0, + "grad_norm": 2.926541790254463, + "language_loss": 0.99200517, + "learning_rate": 3.991103398408914e-06, + "loss": 1.01398098, + "num_input_tokens_seen": 57827940, + "router_z_loss_clip": 0.98681641, + "router_z_loss_mlp": 0.29614258, + "step": 2038, + "time_per_iteration": 2.400585651397705 + }, + { + "auxiliary_loss_clip": 0.01018804, + "auxiliary_loss_mlp": 0.01004686, + "balance_loss_clip": 1.00232053, + "balance_loss_mlp": 1.00306511, + "epoch": 0.05916661830421914, + "flos": 74767337372160.0, + "grad_norm": 0.7642371153024365, + "language_loss": 0.51360232, + "learning_rate": 3.991085680452633e-06, + "loss": 0.5338372, + "num_input_tokens_seen": 57891715, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01623535, + "step": 2039, + "time_per_iteration": 3.134756326675415 + }, + { + "auxiliary_loss_clip": 0.01018214, + "auxiliary_loss_mlp": 0.01001614, + "balance_loss_clip": 1.00193036, + "balance_loss_mlp": 1.00015378, + "epoch": 0.059195635772735185, + "flos": 63310030007040.0, + "grad_norm": 0.7086658137567403, + "language_loss": 0.54437369, + "learning_rate": 3.991067944910277e-06, + "loss": 0.56457198, + "num_input_tokens_seen": 57947455, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.0145874, + "step": 2040, + "time_per_iteration": 2.8778536319732666 + }, + { + "auxiliary_loss_clip": 0.01018162, + "auxiliary_loss_mlp": 0.01002424, + "balance_loss_clip": 1.00213981, + "balance_loss_mlp": 1.00085044, + "epoch": 0.059224653241251236, + "flos": 74771806026240.0, + "grad_norm": 0.7144994873902396, + "language_loss": 0.48804298, + "learning_rate": 3.991050191782004e-06, + "loss": 0.50824881, + "num_input_tokens_seen": 58006690, + "router_z_loss_clip": 0.16015625, + "router_z_loss_mlp": 0.01574707, + "step": 2041, + "time_per_iteration": 3.1078970432281494 + }, + { + "auxiliary_loss_clip": 0.01139643, + "auxiliary_loss_mlp": 0.01054507, + "balance_loss_clip": 1.04033506, + "balance_loss_mlp": 1.02855539, + "epoch": 0.05925367070976728, + "flos": 21353822133120.0, + "grad_norm": 2.5275527060096215, + "language_loss": 0.94825888, + "learning_rate": 3.9910324210679695e-06, + "loss": 0.97020042, + "num_input_tokens_seen": 58020330, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.25939941, + "step": 2042, + "time_per_iteration": 2.4119677543640137 + }, + { + "auxiliary_loss_clip": 0.01134303, + "auxiliary_loss_mlp": 0.01052563, + "balance_loss_clip": 1.03806901, + "balance_loss_mlp": 1.02464437, + "epoch": 0.059282688178283326, + "flos": 25621989154560.0, + "grad_norm": 9.01769112246619, + "language_loss": 0.9502843, + "learning_rate": 3.991014632768331e-06, + "loss": 0.97215301, + "num_input_tokens_seen": 58035585, + "router_z_loss_clip": 0.96289062, + "router_z_loss_mlp": 0.27905273, + "step": 2043, + "time_per_iteration": 2.548267126083374 + }, + { + "auxiliary_loss_clip": 0.01133158, + "auxiliary_loss_mlp": 0.01057195, + "balance_loss_clip": 1.04170716, + "balance_loss_mlp": 1.03230405, + "epoch": 0.05931170564679937, + "flos": 47804649897600.0, + "grad_norm": 2.2394866587517788, + "language_loss": 0.83008766, + "learning_rate": 3.990996826883246e-06, + "loss": 0.85199118, + "num_input_tokens_seen": 58056960, + "router_z_loss_clip": 0.91455078, + "router_z_loss_mlp": 0.24902344, + "step": 2044, + "time_per_iteration": 2.7218446731567383 + }, + { + "auxiliary_loss_clip": 0.01121025, + "auxiliary_loss_mlp": 0.01049749, + "balance_loss_clip": 1.03754878, + "balance_loss_mlp": 1.02700377, + "epoch": 0.05934072311531542, + "flos": 19496512425600.0, + "grad_norm": 2.135914527932684, + "language_loss": 0.76356792, + "learning_rate": 3.990979003412871e-06, + "loss": 0.7852757, + "num_input_tokens_seen": 58071820, + "router_z_loss_clip": 0.83496094, + "router_z_loss_mlp": 0.22741699, + "step": 2045, + "time_per_iteration": 2.4385297298431396 + }, + { + "auxiliary_loss_clip": 0.01129685, + "auxiliary_loss_mlp": 0.01038716, + "balance_loss_clip": 1.03962362, + "balance_loss_mlp": 1.01799786, + "epoch": 0.059369740583831467, + "flos": 58165195812480.0, + "grad_norm": 2.315385409058409, + "language_loss": 0.63338131, + "learning_rate": 3.990961162357363e-06, + "loss": 0.6550653, + "num_input_tokens_seen": 58092955, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.20715332, + "step": 2046, + "time_per_iteration": 2.650325059890747 + }, + { + "auxiliary_loss_clip": 0.01125553, + "auxiliary_loss_mlp": 0.01048285, + "balance_loss_clip": 1.03787899, + "balance_loss_mlp": 1.02631426, + "epoch": 0.05939875805234751, + "flos": 15185961146880.0, + "grad_norm": 2.8672035620527954, + "language_loss": 0.88239098, + "learning_rate": 3.9909433037168815e-06, + "loss": 0.90412933, + "num_input_tokens_seen": 58105050, + "router_z_loss_clip": 0.87744141, + "router_z_loss_mlp": 0.21984863, + "step": 2047, + "time_per_iteration": 2.4214887619018555 + }, + { + "auxiliary_loss_clip": 0.01133353, + "auxiliary_loss_mlp": 0.01048276, + "balance_loss_clip": 1.03853679, + "balance_loss_mlp": 1.02227676, + "epoch": 0.05942777552086356, + "flos": 51525762825600.0, + "grad_norm": 4.537788666980924, + "language_loss": 0.60579979, + "learning_rate": 3.990925427491583e-06, + "loss": 0.62761605, + "num_input_tokens_seen": 58122150, + "router_z_loss_clip": 0.94921875, + "router_z_loss_mlp": 0.2598877, + "step": 2048, + "time_per_iteration": 2.583911180496216 + }, + { + "auxiliary_loss_clip": 0.01125341, + "auxiliary_loss_mlp": 0.01050338, + "balance_loss_clip": 1.03549612, + "balance_loss_mlp": 1.02846348, + "epoch": 0.05945679298937961, + "flos": 20770318713600.0, + "grad_norm": 2.3601998937397357, + "language_loss": 0.92828727, + "learning_rate": 3.990907533681625e-06, + "loss": 0.95004416, + "num_input_tokens_seen": 58138765, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.21875, + "step": 2049, + "time_per_iteration": 2.5257487297058105 + }, + { + "auxiliary_loss_clip": 0.01138875, + "auxiliary_loss_mlp": 0.01048655, + "balance_loss_clip": 1.04420996, + "balance_loss_mlp": 1.02238083, + "epoch": 0.05948581045789565, + "flos": 26022896830080.0, + "grad_norm": 2.6427198560639753, + "language_loss": 0.94409764, + "learning_rate": 3.990889622287166e-06, + "loss": 0.9659729, + "num_input_tokens_seen": 58155710, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.2623291, + "step": 2050, + "time_per_iteration": 2.4752140045166016 + }, + { + "auxiliary_loss_clip": 0.01136832, + "auxiliary_loss_mlp": 0.01056529, + "balance_loss_clip": 1.04173541, + "balance_loss_mlp": 1.0310061, + "epoch": 0.0595148279264117, + "flos": 13435869824640.0, + "grad_norm": 2.4086957661108577, + "language_loss": 0.78906053, + "learning_rate": 3.990871693308365e-06, + "loss": 0.81099421, + "num_input_tokens_seen": 58168795, + "router_z_loss_clip": 0.94970703, + "router_z_loss_mlp": 0.25500488, + "step": 2051, + "time_per_iteration": 2.4601938724517822 + }, + { + "auxiliary_loss_clip": 0.01127816, + "auxiliary_loss_mlp": 0.01049035, + "balance_loss_clip": 1.03546941, + "balance_loss_mlp": 1.02552652, + "epoch": 0.05954384539492775, + "flos": 74728760319360.0, + "grad_norm": 2.180635196707833, + "language_loss": 1.04967332, + "learning_rate": 3.990853746745379e-06, + "loss": 1.07144189, + "num_input_tokens_seen": 58195535, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.23498535, + "step": 2052, + "time_per_iteration": 3.05525541305542 + }, + { + "auxiliary_loss_clip": 0.0112209, + "auxiliary_loss_mlp": 0.01043596, + "balance_loss_clip": 1.03766108, + "balance_loss_mlp": 1.02116084, + "epoch": 0.05957286286344379, + "flos": 12779712132480.0, + "grad_norm": 3.6749315917486665, + "language_loss": 0.84110284, + "learning_rate": 3.990835782598367e-06, + "loss": 0.86275971, + "num_input_tokens_seen": 58207930, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.2244873, + "step": 2053, + "time_per_iteration": 2.4215922355651855 + }, + { + "auxiliary_loss_clip": 0.01127201, + "auxiliary_loss_mlp": 0.01042558, + "balance_loss_clip": 1.03712678, + "balance_loss_mlp": 1.01966977, + "epoch": 0.05960188033195984, + "flos": 25803677203200.0, + "grad_norm": 3.2587680620832984, + "language_loss": 0.97106397, + "learning_rate": 3.990817800867488e-06, + "loss": 0.99276155, + "num_input_tokens_seen": 58224775, + "router_z_loss_clip": 0.90136719, + "router_z_loss_mlp": 0.22888184, + "step": 2054, + "time_per_iteration": 2.4493772983551025 + }, + { + "auxiliary_loss_clip": 0.01123341, + "auxiliary_loss_mlp": 0.01042624, + "balance_loss_clip": 1.03827465, + "balance_loss_mlp": 1.01891351, + "epoch": 0.05963089780047589, + "flos": 31461492003840.0, + "grad_norm": 2.126897367823402, + "language_loss": 0.86145347, + "learning_rate": 3.9907998015529e-06, + "loss": 0.88311315, + "num_input_tokens_seen": 58242425, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.23687744, + "step": 2055, + "time_per_iteration": 5.1737143993377686 + }, + { + "auxiliary_loss_clip": 0.01019952, + "auxiliary_loss_mlp": 0.0100134, + "balance_loss_clip": 1.00349021, + "balance_loss_mlp": 0.99962318, + "epoch": 0.059659915268991934, + "flos": 61740265190400.0, + "grad_norm": 0.7554209449630969, + "language_loss": 0.53816354, + "learning_rate": 3.990781784654763e-06, + "loss": 0.55837655, + "num_input_tokens_seen": 58300685, + "router_z_loss_clip": 0.16503906, + "router_z_loss_mlp": 0.01721191, + "step": 2056, + "time_per_iteration": 2.920725107192993 + }, + { + "auxiliary_loss_clip": 0.01019241, + "auxiliary_loss_mlp": 0.01002352, + "balance_loss_clip": 1.00297856, + "balance_loss_mlp": 1.00064707, + "epoch": 0.05968893273750798, + "flos": 64115929987200.0, + "grad_norm": 0.7188559774341606, + "language_loss": 0.5467605, + "learning_rate": 3.990763750173237e-06, + "loss": 0.56697643, + "num_input_tokens_seen": 58362210, + "router_z_loss_clip": 0.16308594, + "router_z_loss_mlp": 0.01708984, + "step": 2057, + "time_per_iteration": 3.011263847351074 + }, + { + "auxiliary_loss_clip": 0.01017711, + "auxiliary_loss_mlp": 0.01001547, + "balance_loss_clip": 1.0015347, + "balance_loss_mlp": 0.9999612, + "epoch": 0.05971795020602402, + "flos": 55430447149440.0, + "grad_norm": 0.7153717476733329, + "language_loss": 0.50035262, + "learning_rate": 3.990745698108478e-06, + "loss": 0.52054524, + "num_input_tokens_seen": 58419355, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.01586914, + "step": 2058, + "time_per_iteration": 5.301928758621216 + }, + { + "auxiliary_loss_clip": 0.01134789, + "auxiliary_loss_mlp": 0.01055958, + "balance_loss_clip": 1.03855848, + "balance_loss_mlp": 1.02967191, + "epoch": 0.059746967674540075, + "flos": 34584751077120.0, + "grad_norm": 2.4132619632279044, + "language_loss": 0.83053571, + "learning_rate": 3.990727628460648e-06, + "loss": 0.85244322, + "num_input_tokens_seen": 58441350, + "router_z_loss_clip": 0.96191406, + "router_z_loss_mlp": 0.26281738, + "step": 2059, + "time_per_iteration": 2.6035690307617188 + }, + { + "auxiliary_loss_clip": 0.01128378, + "auxiliary_loss_mlp": 0.01046464, + "balance_loss_clip": 1.03844452, + "balance_loss_mlp": 1.02331376, + "epoch": 0.05977598514305612, + "flos": 17231941175040.0, + "grad_norm": 6.1648158697576125, + "language_loss": 0.62792563, + "learning_rate": 3.990709541229906e-06, + "loss": 0.649674, + "num_input_tokens_seen": 58456620, + "router_z_loss_clip": 0.8984375, + "router_z_loss_mlp": 0.23132324, + "step": 2060, + "time_per_iteration": 2.378048896789551 + }, + { + "auxiliary_loss_clip": 0.01140091, + "auxiliary_loss_mlp": 0.01061093, + "balance_loss_clip": 1.04063177, + "balance_loss_mlp": 1.03328121, + "epoch": 0.059805002611572164, + "flos": 19310006609280.0, + "grad_norm": 3.0673496294376617, + "language_loss": 1.05208933, + "learning_rate": 3.990691436416412e-06, + "loss": 1.07410121, + "num_input_tokens_seen": 58471395, + "router_z_loss_clip": 0.99267578, + "router_z_loss_mlp": 0.27832031, + "step": 2061, + "time_per_iteration": 4.8693037033081055 + }, + { + "auxiliary_loss_clip": 0.01130865, + "auxiliary_loss_mlp": 0.01040127, + "balance_loss_clip": 1.03852916, + "balance_loss_mlp": 1.01686954, + "epoch": 0.059834020080088215, + "flos": 28284221324160.0, + "grad_norm": 2.422794454897921, + "language_loss": 0.94538128, + "learning_rate": 3.990673314020326e-06, + "loss": 0.96709132, + "num_input_tokens_seen": 58485380, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.23266602, + "step": 2062, + "time_per_iteration": 2.807918071746826 + }, + { + "auxiliary_loss_clip": 0.01017825, + "auxiliary_loss_mlp": 0.0100193, + "balance_loss_clip": 1.00224161, + "balance_loss_mlp": 1.0004282, + "epoch": 0.05986303754860426, + "flos": 63596527557120.0, + "grad_norm": 0.7227079776583637, + "language_loss": 0.51551795, + "learning_rate": 3.990655174041807e-06, + "loss": 0.53571552, + "num_input_tokens_seen": 58540830, + "router_z_loss_clip": 0.15625, + "router_z_loss_mlp": 0.01501465, + "step": 2063, + "time_per_iteration": 5.309752702713013 + }, + { + "auxiliary_loss_clip": 0.01130891, + "auxiliary_loss_mlp": 0.01050346, + "balance_loss_clip": 1.03905344, + "balance_loss_mlp": 1.02738667, + "epoch": 0.059892055017120305, + "flos": 18070240763520.0, + "grad_norm": 2.453614635194205, + "language_loss": 0.74752373, + "learning_rate": 3.9906370164810164e-06, + "loss": 0.76933616, + "num_input_tokens_seen": 58556105, + "router_z_loss_clip": 0.91748047, + "router_z_loss_mlp": 0.22961426, + "step": 2064, + "time_per_iteration": 2.68994140625 + }, + { + "auxiliary_loss_clip": 0.01017073, + "auxiliary_loss_mlp": 0.01002579, + "balance_loss_clip": 1.00158572, + "balance_loss_mlp": 1.0011009, + "epoch": 0.059921072485636356, + "flos": 65687126169600.0, + "grad_norm": 0.7113733173395603, + "language_loss": 0.49820963, + "learning_rate": 3.990618841338115e-06, + "loss": 0.51840615, + "num_input_tokens_seen": 58621085, + "router_z_loss_clip": 0.15429688, + "router_z_loss_mlp": 0.01477051, + "step": 2065, + "time_per_iteration": 3.0386948585510254 + }, + { + "auxiliary_loss_clip": 0.01131799, + "auxiliary_loss_mlp": 0.01051984, + "balance_loss_clip": 1.03924799, + "balance_loss_mlp": 1.02754605, + "epoch": 0.0599500899541524, + "flos": 25146053233920.0, + "grad_norm": 10.012197324497663, + "language_loss": 0.80793673, + "learning_rate": 3.990600648613261e-06, + "loss": 0.82977462, + "num_input_tokens_seen": 58636500, + "router_z_loss_clip": 0.92578125, + "router_z_loss_mlp": 0.24450684, + "step": 2066, + "time_per_iteration": 2.4596385955810547 + }, + { + "auxiliary_loss_clip": 0.01134987, + "auxiliary_loss_mlp": 0.0105963, + "balance_loss_clip": 1.03901803, + "balance_loss_mlp": 1.03291512, + "epoch": 0.059979107422668446, + "flos": 13799315744640.0, + "grad_norm": 15.89985659545935, + "language_loss": 1.02861047, + "learning_rate": 3.990582438306617e-06, + "loss": 1.05055666, + "num_input_tokens_seen": 58647155, + "router_z_loss_clip": 0.95800781, + "router_z_loss_mlp": 0.26745605, + "step": 2067, + "time_per_iteration": 2.385128974914551 + }, + { + "auxiliary_loss_clip": 0.01132832, + "auxiliary_loss_mlp": 0.01053761, + "balance_loss_clip": 1.04091334, + "balance_loss_mlp": 1.02890539, + "epoch": 0.06000812489118449, + "flos": 26460742590720.0, + "grad_norm": 2.225477970773724, + "language_loss": 1.10358393, + "learning_rate": 3.990564210418344e-06, + "loss": 1.1254499, + "num_input_tokens_seen": 58665710, + "router_z_loss_clip": 0.91943359, + "router_z_loss_mlp": 0.24890137, + "step": 2068, + "time_per_iteration": 2.5135183334350586 + }, + { + "auxiliary_loss_clip": 0.01140639, + "auxiliary_loss_mlp": 0.01051067, + "balance_loss_clip": 1.04014444, + "balance_loss_mlp": 1.02453136, + "epoch": 0.06003714235970054, + "flos": 36677760439680.0, + "grad_norm": 2.270100282447856, + "language_loss": 0.72768509, + "learning_rate": 3.990545964948602e-06, + "loss": 0.7496022, + "num_input_tokens_seen": 58686325, + "router_z_loss_clip": 1.00488281, + "router_z_loss_mlp": 0.26513672, + "step": 2069, + "time_per_iteration": 2.5529160499572754 + }, + { + "auxiliary_loss_clip": 0.0113054, + "auxiliary_loss_mlp": 0.0106251, + "balance_loss_clip": 1.03839231, + "balance_loss_mlp": 1.03435302, + "epoch": 0.060066159828216587, + "flos": 61498178622720.0, + "grad_norm": 2.1585402591464518, + "language_loss": 0.75144136, + "learning_rate": 3.990527701897552e-06, + "loss": 0.77337188, + "num_input_tokens_seen": 58706060, + "router_z_loss_clip": 0.921875, + "router_z_loss_mlp": 0.28125, + "step": 2070, + "time_per_iteration": 2.793473482131958 + }, + { + "auxiliary_loss_clip": 0.01140488, + "auxiliary_loss_mlp": 0.01064622, + "balance_loss_clip": 1.04057026, + "balance_loss_mlp": 1.03676295, + "epoch": 0.06009517729673263, + "flos": 30985625905920.0, + "grad_norm": 3.375370898126864, + "language_loss": 0.89643919, + "learning_rate": 3.990509421265356e-06, + "loss": 0.91849029, + "num_input_tokens_seen": 58720800, + "router_z_loss_clip": 0.99902344, + "router_z_loss_mlp": 0.27868652, + "step": 2071, + "time_per_iteration": 2.511652708053589 + }, + { + "auxiliary_loss_clip": 0.01131715, + "auxiliary_loss_mlp": 0.01053827, + "balance_loss_clip": 1.0368638, + "balance_loss_mlp": 1.02881646, + "epoch": 0.06012419476524868, + "flos": 49700922549120.0, + "grad_norm": 2.975645537159076, + "language_loss": 0.65610588, + "learning_rate": 3.990491123052176e-06, + "loss": 0.67796129, + "num_input_tokens_seen": 58738545, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.25048828, + "step": 2072, + "time_per_iteration": 2.7033650875091553 + }, + { + "auxiliary_loss_clip": 0.01124614, + "auxiliary_loss_mlp": 0.01052259, + "balance_loss_clip": 1.03948712, + "balance_loss_mlp": 1.02989507, + "epoch": 0.06015321223376473, + "flos": 38467792224000.0, + "grad_norm": 2.2368137852977314, + "language_loss": 1.03228986, + "learning_rate": 3.9904728072581726e-06, + "loss": 1.05405855, + "num_input_tokens_seen": 58755405, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.22338867, + "step": 2073, + "time_per_iteration": 2.6356654167175293 + }, + { + "auxiliary_loss_clip": 0.01131985, + "auxiliary_loss_mlp": 0.01056998, + "balance_loss_clip": 1.03735924, + "balance_loss_mlp": 1.03120112, + "epoch": 0.06018222970228077, + "flos": 33689928130560.0, + "grad_norm": 2.1297722157523955, + "language_loss": 0.67340225, + "learning_rate": 3.990454473883508e-06, + "loss": 0.695292, + "num_input_tokens_seen": 58774900, + "router_z_loss_clip": 0.94628906, + "router_z_loss_mlp": 0.25805664, + "step": 2074, + "time_per_iteration": 2.813235282897949 + }, + { + "auxiliary_loss_clip": 0.01124378, + "auxiliary_loss_mlp": 0.01042666, + "balance_loss_clip": 1.03786707, + "balance_loss_mlp": 1.02145898, + "epoch": 0.06021124717079682, + "flos": 31604705867520.0, + "grad_norm": 4.7542770843025774, + "language_loss": 0.9988113, + "learning_rate": 3.990436122928344e-06, + "loss": 1.02048171, + "num_input_tokens_seen": 58790835, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.2121582, + "step": 2075, + "time_per_iteration": 2.539015054702759 + }, + { + "auxiliary_loss_clip": 0.01134986, + "auxiliary_loss_mlp": 0.01055, + "balance_loss_clip": 1.04023123, + "balance_loss_mlp": 1.03075314, + "epoch": 0.06024026463931287, + "flos": 33391525806720.0, + "grad_norm": 2.9030676628319316, + "language_loss": 1.03651261, + "learning_rate": 3.990417754392843e-06, + "loss": 1.05841243, + "num_input_tokens_seen": 58805345, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.24267578, + "step": 2076, + "time_per_iteration": 2.5229713916778564 + }, + { + "auxiliary_loss_clip": 0.01123939, + "auxiliary_loss_mlp": 0.01042498, + "balance_loss_clip": 1.03837121, + "balance_loss_mlp": 1.01888227, + "epoch": 0.06026928210782891, + "flos": 25882196584320.0, + "grad_norm": 2.3888793801977704, + "language_loss": 0.65636218, + "learning_rate": 3.990399368277166e-06, + "loss": 0.67802656, + "num_input_tokens_seen": 58821830, + "router_z_loss_clip": 0.85546875, + "router_z_loss_mlp": 0.23571777, + "step": 2077, + "time_per_iteration": 2.4417972564697266 + }, + { + "auxiliary_loss_clip": 0.01128364, + "auxiliary_loss_mlp": 0.01044092, + "balance_loss_clip": 1.04006338, + "balance_loss_mlp": 1.02168703, + "epoch": 0.06029829957634496, + "flos": 37703019692160.0, + "grad_norm": 3.118376906343676, + "language_loss": 0.9639765, + "learning_rate": 3.990380964581477e-06, + "loss": 0.98570108, + "num_input_tokens_seen": 58837815, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.22393799, + "step": 2078, + "time_per_iteration": 2.5782618522644043 + }, + { + "auxiliary_loss_clip": 0.01134804, + "auxiliary_loss_mlp": 0.01053397, + "balance_loss_clip": 1.03994513, + "balance_loss_mlp": 1.02557325, + "epoch": 0.06032731704486101, + "flos": 17886632590080.0, + "grad_norm": 8.438748325360807, + "language_loss": 0.73332769, + "learning_rate": 3.990362543305938e-06, + "loss": 0.75520968, + "num_input_tokens_seen": 58850730, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.27807617, + "step": 2079, + "time_per_iteration": 2.4615533351898193 + }, + { + "auxiliary_loss_clip": 0.01131484, + "auxiliary_loss_mlp": 0.01054504, + "balance_loss_clip": 1.04029667, + "balance_loss_mlp": 1.02836096, + "epoch": 0.060356334513377054, + "flos": 30949246402560.0, + "grad_norm": 2.3236480491353646, + "language_loss": 0.86910468, + "learning_rate": 3.990344104450711e-06, + "loss": 0.89096451, + "num_input_tokens_seen": 58869985, + "router_z_loss_clip": 0.91113281, + "router_z_loss_mlp": 0.26147461, + "step": 2080, + "time_per_iteration": 2.5163323879241943 + }, + { + "auxiliary_loss_clip": 0.01134028, + "auxiliary_loss_mlp": 0.01047644, + "balance_loss_clip": 1.03976357, + "balance_loss_mlp": 1.02047634, + "epoch": 0.0603853519818931, + "flos": 32230104785280.0, + "grad_norm": 2.3221941390508953, + "language_loss": 0.81271821, + "learning_rate": 3.99032564801596e-06, + "loss": 0.83453494, + "num_input_tokens_seen": 58889220, + "router_z_loss_clip": 0.94287109, + "router_z_loss_mlp": 0.27185059, + "step": 2081, + "time_per_iteration": 2.53938627243042 + }, + { + "auxiliary_loss_clip": 0.0112495, + "auxiliary_loss_mlp": 0.01044732, + "balance_loss_clip": 1.03600121, + "balance_loss_mlp": 1.02050853, + "epoch": 0.06041436945040914, + "flos": 28069121237760.0, + "grad_norm": 2.7588968158048903, + "language_loss": 0.92018473, + "learning_rate": 3.990307174001848e-06, + "loss": 0.94188154, + "num_input_tokens_seen": 58906410, + "router_z_loss_clip": 0.88916016, + "router_z_loss_mlp": 0.24230957, + "step": 2082, + "time_per_iteration": 2.450387954711914 + }, + { + "auxiliary_loss_clip": 0.01019343, + "auxiliary_loss_mlp": 0.01001812, + "balance_loss_clip": 1.00402188, + "balance_loss_mlp": 1.00031579, + "epoch": 0.060443386918925195, + "flos": 54840659685120.0, + "grad_norm": 0.7527244930434304, + "language_loss": 0.50858963, + "learning_rate": 3.9902886824085375e-06, + "loss": 0.52880114, + "num_input_tokens_seen": 58962195, + "router_z_loss_clip": 0.15332031, + "router_z_loss_mlp": 0.01495361, + "step": 2083, + "time_per_iteration": 2.996371030807495 + }, + { + "auxiliary_loss_clip": 0.01133687, + "auxiliary_loss_mlp": 0.01057529, + "balance_loss_clip": 1.04258347, + "balance_loss_mlp": 1.03070712, + "epoch": 0.06047240438744124, + "flos": 20003137297920.0, + "grad_norm": 2.5572191551242947, + "language_loss": 0.85189927, + "learning_rate": 3.990270173236192e-06, + "loss": 0.87381148, + "num_input_tokens_seen": 58977855, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.26806641, + "step": 2084, + "time_per_iteration": 2.4242122173309326 + }, + { + "auxiliary_loss_clip": 0.01017597, + "auxiliary_loss_mlp": 0.01003092, + "balance_loss_clip": 1.00230002, + "balance_loss_mlp": 1.00154245, + "epoch": 0.060501421855957284, + "flos": 74780498954880.0, + "grad_norm": 0.6820321877870407, + "language_loss": 0.48608345, + "learning_rate": 3.990251646484974e-06, + "loss": 0.50629032, + "num_input_tokens_seen": 59046595, + "router_z_loss_clip": 0.15234375, + "router_z_loss_mlp": 0.01544189, + "step": 2085, + "time_per_iteration": 3.201591730117798 + }, + { + "auxiliary_loss_clip": 0.01127751, + "auxiliary_loss_mlp": 0.01046507, + "balance_loss_clip": 1.03866577, + "balance_loss_mlp": 1.02214026, + "epoch": 0.060530439324473335, + "flos": 56381308427520.0, + "grad_norm": 13.9589119976714, + "language_loss": 0.94447362, + "learning_rate": 3.990233102155048e-06, + "loss": 0.96621621, + "num_input_tokens_seen": 59068660, + "router_z_loss_clip": 0.88964844, + "router_z_loss_mlp": 0.24365234, + "step": 2086, + "time_per_iteration": 2.7397232055664062 + }, + { + "auxiliary_loss_clip": 0.01124058, + "auxiliary_loss_mlp": 0.01045026, + "balance_loss_clip": 1.03729057, + "balance_loss_mlp": 1.02086806, + "epoch": 0.06055945679298938, + "flos": 29459222864640.0, + "grad_norm": 2.770814194883483, + "language_loss": 0.71360964, + "learning_rate": 3.990214540246578e-06, + "loss": 0.73530042, + "num_input_tokens_seen": 59086640, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.24169922, + "step": 2087, + "time_per_iteration": 2.555388927459717 + }, + { + "auxiliary_loss_clip": 0.01125019, + "auxiliary_loss_mlp": 0.01046049, + "balance_loss_clip": 1.03825569, + "balance_loss_mlp": 1.02399576, + "epoch": 0.060588474261505425, + "flos": 17667762076800.0, + "grad_norm": 2.694690451857414, + "language_loss": 0.88017642, + "learning_rate": 3.9901959607597285e-06, + "loss": 0.90188712, + "num_input_tokens_seen": 59099445, + "router_z_loss_clip": 0.86865234, + "router_z_loss_mlp": 0.22058105, + "step": 2088, + "time_per_iteration": 2.5346620082855225 + }, + { + "auxiliary_loss_clip": 0.01126723, + "auxiliary_loss_mlp": 0.01049063, + "balance_loss_clip": 1.03914595, + "balance_loss_mlp": 1.02646649, + "epoch": 0.060617491730021476, + "flos": 41120840839680.0, + "grad_norm": 1.8018277663739026, + "language_loss": 0.87608379, + "learning_rate": 3.990177363694662e-06, + "loss": 0.89784163, + "num_input_tokens_seen": 59124720, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.22613525, + "step": 2089, + "time_per_iteration": 2.8486850261688232 + }, + { + "auxiliary_loss_clip": 0.01133111, + "auxiliary_loss_mlp": 0.01057541, + "balance_loss_clip": 1.03992939, + "balance_loss_mlp": 1.03261423, + "epoch": 0.06064650919853752, + "flos": 27704488331520.0, + "grad_norm": 3.0745255067728006, + "language_loss": 0.83361745, + "learning_rate": 3.990158749051545e-06, + "loss": 0.855524, + "num_input_tokens_seen": 59143295, + "router_z_loss_clip": 0.93310547, + "router_z_loss_mlp": 0.24938965, + "step": 2090, + "time_per_iteration": 2.4864981174468994 + }, + { + "auxiliary_loss_clip": 0.0113476, + "auxiliary_loss_mlp": 0.01052862, + "balance_loss_clip": 1.04198551, + "balance_loss_mlp": 1.02619505, + "epoch": 0.060675526667053566, + "flos": 15587357581440.0, + "grad_norm": 3.6052969702196562, + "language_loss": 0.8909694, + "learning_rate": 3.99014011683054e-06, + "loss": 0.91284549, + "num_input_tokens_seen": 59154760, + "router_z_loss_clip": 0.92822266, + "router_z_loss_mlp": 0.26708984, + "step": 2091, + "time_per_iteration": 2.392993688583374 + }, + { + "auxiliary_loss_clip": 0.01121589, + "auxiliary_loss_mlp": 0.01047277, + "balance_loss_clip": 1.03597164, + "balance_loss_mlp": 1.02337599, + "epoch": 0.06070454413556961, + "flos": 12122716567680.0, + "grad_norm": 2.8430583046557785, + "language_loss": 0.91934496, + "learning_rate": 3.990121467031812e-06, + "loss": 0.94103366, + "num_input_tokens_seen": 59166145, + "router_z_loss_clip": 0.85644531, + "router_z_loss_mlp": 0.23913574, + "step": 2092, + "time_per_iteration": 2.3748762607574463 + }, + { + "auxiliary_loss_clip": 0.01017391, + "auxiliary_loss_mlp": 0.01003513, + "balance_loss_clip": 1.00187707, + "balance_loss_mlp": 1.00206423, + "epoch": 0.06073356160408566, + "flos": 69006458638080.0, + "grad_norm": 0.6717326263528601, + "language_loss": 0.50913638, + "learning_rate": 3.990102799655526e-06, + "loss": 0.52934539, + "num_input_tokens_seen": 59230900, + "router_z_loss_clip": 0.15527344, + "router_z_loss_mlp": 0.01446533, + "step": 2093, + "time_per_iteration": 3.1427788734436035 + }, + { + "auxiliary_loss_clip": 0.0113969, + "auxiliary_loss_mlp": 0.01060405, + "balance_loss_clip": 1.04415727, + "balance_loss_mlp": 1.03441715, + "epoch": 0.060762579072601706, + "flos": 13581318015360.0, + "grad_norm": 2.8883135175297108, + "language_loss": 0.86525792, + "learning_rate": 3.990084114701847e-06, + "loss": 0.88725889, + "num_input_tokens_seen": 59242500, + "router_z_loss_clip": 0.95605469, + "router_z_loss_mlp": 0.26000977, + "step": 2094, + "time_per_iteration": 2.3798036575317383 + }, + { + "auxiliary_loss_clip": 0.01133813, + "auxiliary_loss_mlp": 0.01056814, + "balance_loss_clip": 1.04094529, + "balance_loss_mlp": 1.03061175, + "epoch": 0.06079159654111775, + "flos": 14166671736960.0, + "grad_norm": 2.9875375224931706, + "language_loss": 0.96948004, + "learning_rate": 3.990065412170939e-06, + "loss": 0.99138623, + "num_input_tokens_seen": 59254135, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.26208496, + "step": 2095, + "time_per_iteration": 2.377934694290161 + }, + { + "auxiliary_loss_clip": 0.01138181, + "auxiliary_loss_mlp": 0.01065366, + "balance_loss_clip": 1.0386796, + "balance_loss_mlp": 1.03611159, + "epoch": 0.0608206140096338, + "flos": 28265192766720.0, + "grad_norm": 22.378030755419065, + "language_loss": 0.90588892, + "learning_rate": 3.990046692062969e-06, + "loss": 0.92792439, + "num_input_tokens_seen": 59272665, + "router_z_loss_clip": 0.99560547, + "router_z_loss_mlp": 0.29284668, + "step": 2096, + "time_per_iteration": 2.556713342666626 + }, + { + "auxiliary_loss_clip": 0.01124306, + "auxiliary_loss_mlp": 0.01036666, + "balance_loss_clip": 1.03835189, + "balance_loss_mlp": 1.01703835, + "epoch": 0.06084963147814985, + "flos": 18405511349760.0, + "grad_norm": 3.0190322266495566, + "language_loss": 0.86088645, + "learning_rate": 3.990027954378101e-06, + "loss": 0.88249624, + "num_input_tokens_seen": 59287505, + "router_z_loss_clip": 0.85839844, + "router_z_loss_mlp": 0.1963501, + "step": 2097, + "time_per_iteration": 2.4461112022399902 + }, + { + "auxiliary_loss_clip": 0.01017361, + "auxiliary_loss_mlp": 0.01000907, + "balance_loss_clip": 1.00162351, + "balance_loss_mlp": 0.99947071, + "epoch": 0.06087864894666589, + "flos": 63652841447040.0, + "grad_norm": 0.6937291636477143, + "language_loss": 0.51157802, + "learning_rate": 3.990009199116501e-06, + "loss": 0.53176069, + "num_input_tokens_seen": 59354730, + "router_z_loss_clip": 0.15722656, + "router_z_loss_mlp": 0.01434326, + "step": 2098, + "time_per_iteration": 3.213775634765625 + }, + { + "auxiliary_loss_clip": 0.01137341, + "auxiliary_loss_mlp": 0.01057424, + "balance_loss_clip": 1.04343772, + "balance_loss_mlp": 1.03137648, + "epoch": 0.06090766641518194, + "flos": 13616580355200.0, + "grad_norm": 3.5801898517470527, + "language_loss": 1.04708695, + "learning_rate": 3.989990426278334e-06, + "loss": 1.0690347, + "num_input_tokens_seen": 59365210, + "router_z_loss_clip": 0.93994141, + "router_z_loss_mlp": 0.26074219, + "step": 2099, + "time_per_iteration": 2.444157838821411 + }, + { + "auxiliary_loss_clip": 0.0113124, + "auxiliary_loss_mlp": 0.01054981, + "balance_loss_clip": 1.04005218, + "balance_loss_mlp": 1.03073406, + "epoch": 0.06093668388369799, + "flos": 25146123056640.0, + "grad_norm": 2.086974087719002, + "language_loss": 0.84874773, + "learning_rate": 3.9899716358637665e-06, + "loss": 0.87061, + "num_input_tokens_seen": 59384610, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.24255371, + "step": 2100, + "time_per_iteration": 2.750629186630249 + }, + { + "auxiliary_loss_clip": 0.01122262, + "auxiliary_loss_mlp": 0.01044294, + "balance_loss_clip": 1.03615785, + "balance_loss_mlp": 1.02116704, + "epoch": 0.06096570135221403, + "flos": 16207799086080.0, + "grad_norm": 3.455376096803015, + "language_loss": 0.91184682, + "learning_rate": 3.989952827872964e-06, + "loss": 0.93351233, + "num_input_tokens_seen": 59396515, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.23120117, + "step": 2101, + "time_per_iteration": 2.7332684993743896 + }, + { + "auxiliary_loss_clip": 0.0112564, + "auxiliary_loss_mlp": 0.01052289, + "balance_loss_clip": 1.03744888, + "balance_loss_mlp": 1.0285306, + "epoch": 0.06099471882073008, + "flos": 29275544868480.0, + "grad_norm": 2.236527962726484, + "language_loss": 0.85708582, + "learning_rate": 3.989934002306094e-06, + "loss": 0.87886512, + "num_input_tokens_seen": 59421870, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.23754883, + "step": 2102, + "time_per_iteration": 2.706671953201294 + }, + { + "auxiliary_loss_clip": 0.01017438, + "auxiliary_loss_mlp": 0.01001481, + "balance_loss_clip": 1.00204229, + "balance_loss_mlp": 1.00005031, + "epoch": 0.06102373628924613, + "flos": 58719998361600.0, + "grad_norm": 0.676476643792868, + "language_loss": 0.53521848, + "learning_rate": 3.989915159163321e-06, + "loss": 0.5554077, + "num_input_tokens_seen": 59485280, + "router_z_loss_clip": 0.15429688, + "router_z_loss_mlp": 0.01428223, + "step": 2103, + "time_per_iteration": 3.059938907623291 + }, + { + "auxiliary_loss_clip": 0.01126775, + "auxiliary_loss_mlp": 0.01054192, + "balance_loss_clip": 1.03685951, + "balance_loss_mlp": 1.02840686, + "epoch": 0.061052753757762174, + "flos": 28067480403840.0, + "grad_norm": 2.3530609519768126, + "language_loss": 0.82611185, + "learning_rate": 3.9898962984448105e-06, + "loss": 0.84792149, + "num_input_tokens_seen": 59502910, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.25805664, + "step": 2104, + "time_per_iteration": 2.527127981185913 + }, + { + "auxiliary_loss_clip": 0.01119053, + "auxiliary_loss_mlp": 0.01051644, + "balance_loss_clip": 1.03280401, + "balance_loss_mlp": 1.02859545, + "epoch": 0.06108177122627822, + "flos": 24891745824000.0, + "grad_norm": 2.5845934115771443, + "language_loss": 0.86290431, + "learning_rate": 3.9898774201507324e-06, + "loss": 0.88461125, + "num_input_tokens_seen": 59517010, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.23052979, + "step": 2105, + "time_per_iteration": 2.423780918121338 + }, + { + "auxiliary_loss_clip": 0.01119709, + "auxiliary_loss_mlp": 0.0105134, + "balance_loss_clip": 1.03343308, + "balance_loss_mlp": 1.02902436, + "epoch": 0.06111078869479426, + "flos": 17710704915840.0, + "grad_norm": 2.2139700198487184, + "language_loss": 0.74037236, + "learning_rate": 3.989858524281252e-06, + "loss": 0.76208282, + "num_input_tokens_seen": 59533615, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.2232666, + "step": 2106, + "time_per_iteration": 2.4485981464385986 + }, + { + "auxiliary_loss_clip": 0.01129965, + "auxiliary_loss_mlp": 0.0105916, + "balance_loss_clip": 1.04073787, + "balance_loss_mlp": 1.03314829, + "epoch": 0.061139806163310315, + "flos": 70572664362240.0, + "grad_norm": 2.191820120441598, + "language_loss": 0.65963507, + "learning_rate": 3.989839610836535e-06, + "loss": 0.6815263, + "num_input_tokens_seen": 59557425, + "router_z_loss_clip": 0.89257812, + "router_z_loss_mlp": 0.26000977, + "step": 2107, + "time_per_iteration": 2.8293371200561523 + }, + { + "auxiliary_loss_clip": 0.01018344, + "auxiliary_loss_mlp": 0.01003578, + "balance_loss_clip": 1.00253975, + "balance_loss_mlp": 1.00211728, + "epoch": 0.06116882363182636, + "flos": 74157264541440.0, + "grad_norm": 0.6688552541884893, + "language_loss": 0.51193178, + "learning_rate": 3.9898206798167495e-06, + "loss": 0.53215098, + "num_input_tokens_seen": 59624055, + "router_z_loss_clip": 0.15820312, + "router_z_loss_mlp": 0.0145874, + "step": 2108, + "time_per_iteration": 3.179501533508301 + }, + { + "auxiliary_loss_clip": 0.01131707, + "auxiliary_loss_mlp": 0.01053513, + "balance_loss_clip": 1.03714585, + "balance_loss_mlp": 1.02614284, + "epoch": 0.061197841100342404, + "flos": 34195226371200.0, + "grad_norm": 2.3826200941774287, + "language_loss": 1.01276386, + "learning_rate": 3.989801731222062e-06, + "loss": 1.03461611, + "num_input_tokens_seen": 59643100, + "router_z_loss_clip": 0.94580078, + "router_z_loss_mlp": 0.27392578, + "step": 2109, + "time_per_iteration": 2.6158201694488525 + }, + { + "auxiliary_loss_clip": 0.01128928, + "auxiliary_loss_mlp": 0.01059492, + "balance_loss_clip": 1.03758192, + "balance_loss_mlp": 1.03420734, + "epoch": 0.061226858568858455, + "flos": 17157750802560.0, + "grad_norm": 3.025879568652029, + "language_loss": 1.00441539, + "learning_rate": 3.989782765052642e-06, + "loss": 1.0262996, + "num_input_tokens_seen": 59656035, + "router_z_loss_clip": 0.91308594, + "router_z_loss_mlp": 0.25292969, + "step": 2110, + "time_per_iteration": 2.4394123554229736 + }, + { + "auxiliary_loss_clip": 0.01121479, + "auxiliary_loss_mlp": 0.01050228, + "balance_loss_clip": 1.03830838, + "balance_loss_mlp": 1.02664852, + "epoch": 0.0612558760373745, + "flos": 32372829889920.0, + "grad_norm": 2.6921974574946765, + "language_loss": 0.82143527, + "learning_rate": 3.989763781308654e-06, + "loss": 0.84315234, + "num_input_tokens_seen": 59671595, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.23583984, + "step": 2111, + "time_per_iteration": 2.557558059692383 + }, + { + "auxiliary_loss_clip": 0.01149442, + "auxiliary_loss_mlp": 0.01070447, + "balance_loss_clip": 1.04156828, + "balance_loss_mlp": 1.03939331, + "epoch": 0.061284893505890545, + "flos": 27200307254400.0, + "grad_norm": 2.1295908942399278, + "language_loss": 0.89991474, + "learning_rate": 3.989744779990268e-06, + "loss": 0.92211366, + "num_input_tokens_seen": 59691365, + "router_z_loss_clip": 1.078125, + "router_z_loss_mlp": 0.3104248, + "step": 2112, + "time_per_iteration": 2.4844257831573486 + }, + { + "auxiliary_loss_clip": 0.01125835, + "auxiliary_loss_mlp": 0.0105538, + "balance_loss_clip": 1.03816533, + "balance_loss_mlp": 1.0316453, + "epoch": 0.06131391097440659, + "flos": 37808108484480.0, + "grad_norm": 5.869113828560416, + "language_loss": 0.82707596, + "learning_rate": 3.989725761097651e-06, + "loss": 0.8488881, + "num_input_tokens_seen": 59708665, + "router_z_loss_clip": 0.87695312, + "router_z_loss_mlp": 0.23730469, + "step": 2113, + "time_per_iteration": 2.620753288269043 + }, + { + "auxiliary_loss_clip": 0.010185, + "auxiliary_loss_mlp": 0.01002613, + "balance_loss_clip": 1.00264692, + "balance_loss_mlp": 1.001194, + "epoch": 0.06134292844292264, + "flos": 61152991344000.0, + "grad_norm": 0.6641436495313484, + "language_loss": 0.47201696, + "learning_rate": 3.98970672463097e-06, + "loss": 0.49222809, + "num_input_tokens_seen": 59767355, + "router_z_loss_clip": 0.15820312, + "router_z_loss_mlp": 0.01416016, + "step": 2114, + "time_per_iteration": 3.1377806663513184 + }, + { + "auxiliary_loss_clip": 0.01135679, + "auxiliary_loss_mlp": 0.01057512, + "balance_loss_clip": 1.0387361, + "balance_loss_mlp": 1.02939105, + "epoch": 0.061371945911438686, + "flos": 29307874654080.0, + "grad_norm": 2.395357146816408, + "language_loss": 1.01004887, + "learning_rate": 3.989687670590394e-06, + "loss": 1.03198075, + "num_input_tokens_seen": 59784495, + "router_z_loss_clip": 0.97070312, + "router_z_loss_mlp": 0.28137207, + "step": 2115, + "time_per_iteration": 2.5520131587982178 + }, + { + "auxiliary_loss_clip": 0.0101773, + "auxiliary_loss_mlp": 0.01001157, + "balance_loss_clip": 1.00208759, + "balance_loss_mlp": 0.99973804, + "epoch": 0.06140096337995473, + "flos": 57117380088960.0, + "grad_norm": 0.6975147559776116, + "language_loss": 0.48654604, + "learning_rate": 3.989668598976092e-06, + "loss": 0.50673491, + "num_input_tokens_seen": 59841615, + "router_z_loss_clip": 0.15625, + "router_z_loss_mlp": 0.01416016, + "step": 2116, + "time_per_iteration": 2.9590959548950195 + }, + { + "auxiliary_loss_clip": 0.01125538, + "auxiliary_loss_mlp": 0.01037821, + "balance_loss_clip": 1.03585172, + "balance_loss_mlp": 1.01507628, + "epoch": 0.06142998084847078, + "flos": 19419005473920.0, + "grad_norm": 3.1105586512132892, + "language_loss": 0.99200302, + "learning_rate": 3.989649509788232e-06, + "loss": 1.01363659, + "num_input_tokens_seen": 59854835, + "router_z_loss_clip": 0.89648438, + "router_z_loss_mlp": 0.2277832, + "step": 2117, + "time_per_iteration": 2.5339925289154053 + }, + { + "auxiliary_loss_clip": 0.01017142, + "auxiliary_loss_mlp": 0.01000703, + "balance_loss_clip": 1.00165355, + "balance_loss_mlp": 0.99923122, + "epoch": 0.061458998316986826, + "flos": 74768349801600.0, + "grad_norm": 0.6607532280279796, + "language_loss": 0.4948706, + "learning_rate": 3.9896304030269816e-06, + "loss": 0.51504904, + "num_input_tokens_seen": 59919840, + "router_z_loss_clip": 0.15429688, + "router_z_loss_mlp": 0.01470947, + "step": 2118, + "time_per_iteration": 3.1654558181762695 + }, + { + "auxiliary_loss_clip": 0.01017405, + "auxiliary_loss_mlp": 0.01001258, + "balance_loss_clip": 1.00195217, + "balance_loss_mlp": 0.99989897, + "epoch": 0.06148801578550287, + "flos": 74774982960000.0, + "grad_norm": 0.6481272414305329, + "language_loss": 0.53558046, + "learning_rate": 3.989611278692511e-06, + "loss": 0.55576706, + "num_input_tokens_seen": 59989600, + "router_z_loss_clip": 0.15429688, + "router_z_loss_mlp": 0.01361084, + "step": 2119, + "time_per_iteration": 3.157709836959839 + }, + { + "auxiliary_loss_clip": 0.01132034, + "auxiliary_loss_mlp": 0.01056397, + "balance_loss_clip": 1.043185, + "balance_loss_mlp": 1.03212571, + "epoch": 0.06151703325401892, + "flos": 26935037677440.0, + "grad_norm": 2.3401255096608615, + "language_loss": 0.88498688, + "learning_rate": 3.989592136784989e-06, + "loss": 0.90687114, + "num_input_tokens_seen": 60004390, + "router_z_loss_clip": 0.88818359, + "router_z_loss_mlp": 0.24267578, + "step": 2120, + "time_per_iteration": 2.564762830734253 + }, + { + "auxiliary_loss_clip": 0.01129702, + "auxiliary_loss_mlp": 0.01049238, + "balance_loss_clip": 1.03760684, + "balance_loss_mlp": 1.02392972, + "epoch": 0.06154605072253497, + "flos": 25987494844800.0, + "grad_norm": 2.8005429112076854, + "language_loss": 0.85183847, + "learning_rate": 3.9895729773045825e-06, + "loss": 0.8736279, + "num_input_tokens_seen": 60018875, + "router_z_loss_clip": 0.92236328, + "router_z_loss_mlp": 0.25305176, + "step": 2121, + "time_per_iteration": 2.463380813598633 + }, + { + "auxiliary_loss_clip": 0.01130333, + "auxiliary_loss_mlp": 0.01041925, + "balance_loss_clip": 1.03884161, + "balance_loss_mlp": 1.01793981, + "epoch": 0.06157506819105101, + "flos": 43426157379840.0, + "grad_norm": 3.462333041957345, + "language_loss": 0.86266935, + "learning_rate": 3.989553800251464e-06, + "loss": 0.88439184, + "num_input_tokens_seen": 60036590, + "router_z_loss_clip": 0.91357422, + "router_z_loss_mlp": 0.23974609, + "step": 2122, + "time_per_iteration": 2.688873529434204 + }, + { + "auxiliary_loss_clip": 0.01130718, + "auxiliary_loss_mlp": 0.01049526, + "balance_loss_clip": 1.03826904, + "balance_loss_mlp": 1.0255053, + "epoch": 0.06160408565956706, + "flos": 34452396512640.0, + "grad_norm": 2.0870723135204194, + "language_loss": 0.79710221, + "learning_rate": 3.9895346056258e-06, + "loss": 0.81890464, + "num_input_tokens_seen": 60055145, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.24023438, + "step": 2123, + "time_per_iteration": 2.556123733520508 + }, + { + "auxiliary_loss_clip": 0.0113652, + "auxiliary_loss_mlp": 0.01055855, + "balance_loss_clip": 1.03964913, + "balance_loss_mlp": 1.02991462, + "epoch": 0.06163310312808311, + "flos": 17231347681920.0, + "grad_norm": 2.393960967792906, + "language_loss": 0.87134141, + "learning_rate": 3.989515393427762e-06, + "loss": 0.89326513, + "num_input_tokens_seen": 60070500, + "router_z_loss_clip": 0.96826172, + "router_z_loss_mlp": 0.25964355, + "step": 2124, + "time_per_iteration": 2.4530105590820312 + }, + { + "auxiliary_loss_clip": 0.01128903, + "auxiliary_loss_mlp": 0.01056042, + "balance_loss_clip": 1.037256, + "balance_loss_mlp": 1.03193831, + "epoch": 0.06166212059659915, + "flos": 23360385369600.0, + "grad_norm": 2.831266423384728, + "language_loss": 0.80655122, + "learning_rate": 3.989496163657519e-06, + "loss": 0.82840067, + "num_input_tokens_seen": 60083605, + "router_z_loss_clip": 0.91796875, + "router_z_loss_mlp": 0.24121094, + "step": 2125, + "time_per_iteration": 2.3876848220825195 + }, + { + "auxiliary_loss_clip": 0.0102038, + "auxiliary_loss_mlp": 0.01006756, + "balance_loss_clip": 1.00403762, + "balance_loss_mlp": 1.00521779, + "epoch": 0.0616911380651152, + "flos": 70689062568960.0, + "grad_norm": 0.69006564198148, + "language_loss": 0.51884186, + "learning_rate": 3.9894769163152405e-06, + "loss": 0.53911316, + "num_input_tokens_seen": 60146745, + "router_z_loss_clip": 0.1640625, + "router_z_loss_mlp": 0.01531982, + "step": 2126, + "time_per_iteration": 3.2298152446746826 + }, + { + "auxiliary_loss_clip": 0.0112071, + "auxiliary_loss_mlp": 0.0104118, + "balance_loss_clip": 1.03536248, + "balance_loss_mlp": 1.0189594, + "epoch": 0.06172015553363125, + "flos": 26169357450240.0, + "grad_norm": 2.296368498033598, + "language_loss": 0.69081801, + "learning_rate": 3.9894576514010975e-06, + "loss": 0.71243685, + "num_input_tokens_seen": 60160315, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.22198486, + "step": 2127, + "time_per_iteration": 2.550816297531128 + }, + { + "auxiliary_loss_clip": 0.01137638, + "auxiliary_loss_mlp": 0.01053187, + "balance_loss_clip": 1.04122329, + "balance_loss_mlp": 1.02560222, + "epoch": 0.061749173002147294, + "flos": 33210466162560.0, + "grad_norm": 3.2361412324997088, + "language_loss": 1.01792729, + "learning_rate": 3.989438368915259e-06, + "loss": 1.03983557, + "num_input_tokens_seen": 60175960, + "router_z_loss_clip": 0.96386719, + "router_z_loss_mlp": 0.27563477, + "step": 2128, + "time_per_iteration": 2.5257232189178467 + }, + { + "auxiliary_loss_clip": 0.0101978, + "auxiliary_loss_mlp": 0.01007108, + "balance_loss_clip": 1.00321698, + "balance_loss_mlp": 1.00561225, + "epoch": 0.06177819047066334, + "flos": 58204436181120.0, + "grad_norm": 0.7082336502275386, + "language_loss": 0.52757502, + "learning_rate": 3.989419068857896e-06, + "loss": 0.54784393, + "num_input_tokens_seen": 60238290, + "router_z_loss_clip": 0.16601562, + "router_z_loss_mlp": 0.01495361, + "step": 2129, + "time_per_iteration": 2.9992542266845703 + }, + { + "auxiliary_loss_clip": 0.01125544, + "auxiliary_loss_mlp": 0.01045316, + "balance_loss_clip": 1.03894162, + "balance_loss_mlp": 1.0229404, + "epoch": 0.06180720793917938, + "flos": 13727185142400.0, + "grad_norm": 3.7414401780669073, + "language_loss": 1.02427411, + "learning_rate": 3.989399751229178e-06, + "loss": 1.04598284, + "num_input_tokens_seen": 60248770, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.22399902, + "step": 2130, + "time_per_iteration": 2.4196536540985107 + }, + { + "auxiliary_loss_clip": 0.01128746, + "auxiliary_loss_mlp": 0.01048451, + "balance_loss_clip": 1.03712034, + "balance_loss_mlp": 1.02415633, + "epoch": 0.061836225407695435, + "flos": 21135021442560.0, + "grad_norm": 2.6742499236845654, + "language_loss": 0.79212487, + "learning_rate": 3.989380416029276e-06, + "loss": 0.81389689, + "num_input_tokens_seen": 60263385, + "router_z_loss_clip": 0.91650391, + "router_z_loss_mlp": 0.24279785, + "step": 2131, + "time_per_iteration": 2.412468910217285 + }, + { + "auxiliary_loss_clip": 0.01137773, + "auxiliary_loss_mlp": 0.0105126, + "balance_loss_clip": 1.04083025, + "balance_loss_mlp": 1.02406836, + "epoch": 0.06186524287621148, + "flos": 15446238399360.0, + "grad_norm": 2.4220235671660997, + "language_loss": 0.85465956, + "learning_rate": 3.989361063258362e-06, + "loss": 0.87654984, + "num_input_tokens_seen": 60277190, + "router_z_loss_clip": 0.96875, + "router_z_loss_mlp": 0.27209473, + "step": 2132, + "time_per_iteration": 4.79742169380188 + }, + { + "auxiliary_loss_clip": 0.01128582, + "auxiliary_loss_mlp": 0.01047943, + "balance_loss_clip": 1.03822279, + "balance_loss_mlp": 1.02344513, + "epoch": 0.061894260344727524, + "flos": 45871508983680.0, + "grad_norm": 2.6127719484009204, + "language_loss": 0.73204982, + "learning_rate": 3.989341692916607e-06, + "loss": 0.75381505, + "num_input_tokens_seen": 60295525, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.24475098, + "step": 2133, + "time_per_iteration": 4.958388566970825 + }, + { + "auxiliary_loss_clip": 0.01124776, + "auxiliary_loss_mlp": 0.01043319, + "balance_loss_clip": 1.03503442, + "balance_loss_mlp": 1.0185349, + "epoch": 0.061923277813243575, + "flos": 44813361363840.0, + "grad_norm": 2.0424740346966854, + "language_loss": 0.90397441, + "learning_rate": 3.98932230500418e-06, + "loss": 0.92565531, + "num_input_tokens_seen": 60315785, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.24768066, + "step": 2134, + "time_per_iteration": 2.599627733230591 + }, + { + "auxiliary_loss_clip": 0.01136773, + "auxiliary_loss_mlp": 0.01061193, + "balance_loss_clip": 1.04201269, + "balance_loss_mlp": 1.03389382, + "epoch": 0.06195229528175962, + "flos": 12925718904960.0, + "grad_norm": 2.7084723259420143, + "language_loss": 1.06093085, + "learning_rate": 3.9893028995212544e-06, + "loss": 1.08291054, + "num_input_tokens_seen": 60327715, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.27294922, + "step": 2135, + "time_per_iteration": 2.3863632678985596 + }, + { + "auxiliary_loss_clip": 0.01124377, + "auxiliary_loss_mlp": 0.01048808, + "balance_loss_clip": 1.03530669, + "balance_loss_mlp": 1.02538323, + "epoch": 0.061981312750275665, + "flos": 21719118355200.0, + "grad_norm": 2.6879973474912906, + "language_loss": 0.84142947, + "learning_rate": 3.989283476467999e-06, + "loss": 0.86316133, + "num_input_tokens_seen": 60342010, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.23425293, + "step": 2136, + "time_per_iteration": 2.438763380050659 + }, + { + "auxiliary_loss_clip": 0.01137671, + "auxiliary_loss_mlp": 0.01053511, + "balance_loss_clip": 1.03878856, + "balance_loss_mlp": 1.02406669, + "epoch": 0.06201033021879171, + "flos": 27409961168640.0, + "grad_norm": 2.7614730400866594, + "language_loss": 0.90039563, + "learning_rate": 3.989264035844588e-06, + "loss": 0.92230749, + "num_input_tokens_seen": 60357235, + "router_z_loss_clip": 0.99023438, + "router_z_loss_mlp": 0.29431152, + "step": 2137, + "time_per_iteration": 5.217626571655273 + }, + { + "auxiliary_loss_clip": 0.0101941, + "auxiliary_loss_mlp": 0.01000982, + "balance_loss_clip": 1.00340104, + "balance_loss_mlp": 0.99951565, + "epoch": 0.06203934768730776, + "flos": 71051845173120.0, + "grad_norm": 0.6791437279479657, + "language_loss": 0.5339843, + "learning_rate": 3.989244577651192e-06, + "loss": 0.55418825, + "num_input_tokens_seen": 60425755, + "router_z_loss_clip": 0.16015625, + "router_z_loss_mlp": 0.01464844, + "step": 2138, + "time_per_iteration": 3.2327661514282227 + }, + { + "auxiliary_loss_clip": 0.01127387, + "auxiliary_loss_mlp": 0.01058685, + "balance_loss_clip": 1.03699064, + "balance_loss_mlp": 1.03290009, + "epoch": 0.062068365155823806, + "flos": 41607845527680.0, + "grad_norm": 2.202976396192131, + "language_loss": 0.8213616, + "learning_rate": 3.989225101887983e-06, + "loss": 0.84322238, + "num_input_tokens_seen": 60453515, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.2578125, + "step": 2139, + "time_per_iteration": 2.6732211112976074 + }, + { + "auxiliary_loss_clip": 0.01117069, + "auxiliary_loss_mlp": 0.01045146, + "balance_loss_clip": 1.03376746, + "balance_loss_mlp": 1.0223887, + "epoch": 0.06209738262433985, + "flos": 25557294672000.0, + "grad_norm": 2.9164078337109456, + "language_loss": 0.75660348, + "learning_rate": 3.9892056085551326e-06, + "loss": 0.77822566, + "num_input_tokens_seen": 60473675, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.22766113, + "step": 2140, + "time_per_iteration": 4.882779121398926 + }, + { + "auxiliary_loss_clip": 0.01018727, + "auxiliary_loss_mlp": 0.01002456, + "balance_loss_clip": 1.00269103, + "balance_loss_mlp": 1.00090623, + "epoch": 0.0621264000928559, + "flos": 68641022770560.0, + "grad_norm": 0.9110825461809732, + "language_loss": 0.51112634, + "learning_rate": 3.989186097652814e-06, + "loss": 0.53133816, + "num_input_tokens_seen": 60535590, + "router_z_loss_clip": 0.16015625, + "router_z_loss_mlp": 0.01550293, + "step": 2141, + "time_per_iteration": 3.245142698287964 + }, + { + "auxiliary_loss_clip": 0.01128613, + "auxiliary_loss_mlp": 0.01047293, + "balance_loss_clip": 1.03828788, + "balance_loss_mlp": 1.02181816, + "epoch": 0.062155417561371946, + "flos": 32079838826880.0, + "grad_norm": 2.501215976863665, + "language_loss": 0.92555505, + "learning_rate": 3.989166569181198e-06, + "loss": 0.94731414, + "num_input_tokens_seen": 60550825, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.25463867, + "step": 2142, + "time_per_iteration": 2.5286929607391357 + }, + { + "auxiliary_loss_clip": 0.01017348, + "auxiliary_loss_mlp": 0.0100206, + "balance_loss_clip": 1.00135064, + "balance_loss_mlp": 1.00052261, + "epoch": 0.06218443502988799, + "flos": 69162415148160.0, + "grad_norm": 0.6460941243140155, + "language_loss": 0.50378042, + "learning_rate": 3.989147023140458e-06, + "loss": 0.52397454, + "num_input_tokens_seen": 60614170, + "router_z_loss_clip": 0.16015625, + "router_z_loss_mlp": 0.01538086, + "step": 2143, + "time_per_iteration": 3.1622977256774902 + }, + { + "auxiliary_loss_clip": 0.01121634, + "auxiliary_loss_mlp": 0.01049687, + "balance_loss_clip": 1.03492796, + "balance_loss_mlp": 1.02708459, + "epoch": 0.06221345249840404, + "flos": 48573088122240.0, + "grad_norm": 3.457701160167326, + "language_loss": 0.95967221, + "learning_rate": 3.989127459530767e-06, + "loss": 0.98138541, + "num_input_tokens_seen": 60633555, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.22619629, + "step": 2144, + "time_per_iteration": 2.588108539581299 + }, + { + "auxiliary_loss_clip": 0.01017947, + "auxiliary_loss_mlp": 0.01001611, + "balance_loss_clip": 1.00175786, + "balance_loss_mlp": 1.00004888, + "epoch": 0.06224246996692009, + "flos": 74779207234560.0, + "grad_norm": 0.6402675084467003, + "language_loss": 0.5267067, + "learning_rate": 3.989107878352297e-06, + "loss": 0.5469023, + "num_input_tokens_seen": 60701960, + "router_z_loss_clip": 0.16210938, + "router_z_loss_mlp": 0.015625, + "step": 2145, + "time_per_iteration": 3.123812675476074 + }, + { + "auxiliary_loss_clip": 0.0113863, + "auxiliary_loss_mlp": 0.01061876, + "balance_loss_clip": 1.04138398, + "balance_loss_mlp": 1.03324151, + "epoch": 0.06227148743543613, + "flos": 74729772748800.0, + "grad_norm": 2.4413005381322423, + "language_loss": 0.90856296, + "learning_rate": 3.989088279605222e-06, + "loss": 0.93056804, + "num_input_tokens_seen": 60722135, + "router_z_loss_clip": 0.97412109, + "router_z_loss_mlp": 0.28662109, + "step": 2146, + "time_per_iteration": 2.8758983612060547 + }, + { + "auxiliary_loss_clip": 0.01117499, + "auxiliary_loss_mlp": 0.01046796, + "balance_loss_clip": 1.03495431, + "balance_loss_mlp": 1.02576745, + "epoch": 0.06230050490395218, + "flos": 17340556014720.0, + "grad_norm": 2.2055970509802045, + "language_loss": 0.82572573, + "learning_rate": 3.989068663289713e-06, + "loss": 0.84736872, + "num_input_tokens_seen": 60738100, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.21020508, + "step": 2147, + "time_per_iteration": 2.412337064743042 + }, + { + "auxiliary_loss_clip": 0.01134033, + "auxiliary_loss_mlp": 0.01054258, + "balance_loss_clip": 1.04122984, + "balance_loss_mlp": 1.02935529, + "epoch": 0.06232952237246823, + "flos": 28795768698240.0, + "grad_norm": 2.525602650199294, + "language_loss": 0.81132531, + "learning_rate": 3.989049029405947e-06, + "loss": 0.8332082, + "num_input_tokens_seen": 60752345, + "router_z_loss_clip": 0.92724609, + "router_z_loss_mlp": 0.2487793, + "step": 2148, + "time_per_iteration": 2.523732900619507 + }, + { + "auxiliary_loss_clip": 0.01017343, + "auxiliary_loss_mlp": 0.01004585, + "balance_loss_clip": 1.00160921, + "balance_loss_mlp": 1.00311303, + "epoch": 0.06235853984098427, + "flos": 74780568777600.0, + "grad_norm": 0.6940730333571707, + "language_loss": 0.5365591, + "learning_rate": 3.989029377954093e-06, + "loss": 0.55677843, + "num_input_tokens_seen": 60822275, + "router_z_loss_clip": 0.15722656, + "router_z_loss_mlp": 0.01470947, + "step": 2149, + "time_per_iteration": 3.317394256591797 + }, + { + "auxiliary_loss_clip": 0.0113023, + "auxiliary_loss_mlp": 0.01052663, + "balance_loss_clip": 1.04025388, + "balance_loss_mlp": 1.02890456, + "epoch": 0.06238755730950032, + "flos": 15589906110720.0, + "grad_norm": 2.265527486226636, + "language_loss": 0.85662514, + "learning_rate": 3.989009708934328e-06, + "loss": 0.87845403, + "num_input_tokens_seen": 60836645, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.23754883, + "step": 2150, + "time_per_iteration": 2.4353108406066895 + }, + { + "auxiliary_loss_clip": 0.01128221, + "auxiliary_loss_mlp": 0.01063383, + "balance_loss_clip": 1.03762162, + "balance_loss_mlp": 1.03853929, + "epoch": 0.06241657477801637, + "flos": 30510876971520.0, + "grad_norm": 1.9098137715530565, + "language_loss": 0.91026831, + "learning_rate": 3.9889900223468234e-06, + "loss": 0.93218428, + "num_input_tokens_seen": 60855285, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.24829102, + "step": 2151, + "time_per_iteration": 2.517507791519165 + }, + { + "auxiliary_loss_clip": 0.01135739, + "auxiliary_loss_mlp": 0.01056695, + "balance_loss_clip": 1.03886926, + "balance_loss_mlp": 1.03006387, + "epoch": 0.062445592246532414, + "flos": 28179132531840.0, + "grad_norm": 2.149821653443288, + "language_loss": 0.94530773, + "learning_rate": 3.988970318191753e-06, + "loss": 0.96723211, + "num_input_tokens_seen": 60875125, + "router_z_loss_clip": 0.96630859, + "router_z_loss_mlp": 0.26611328, + "step": 2152, + "time_per_iteration": 2.5553386211395264 + }, + { + "auxiliary_loss_clip": 0.01127271, + "auxiliary_loss_mlp": 0.01047025, + "balance_loss_clip": 1.03903079, + "balance_loss_mlp": 1.02443457, + "epoch": 0.06247460971504846, + "flos": 29856255379200.0, + "grad_norm": 2.5659100698890085, + "language_loss": 0.87012112, + "learning_rate": 3.9889505964692946e-06, + "loss": 0.89186406, + "num_input_tokens_seen": 60891545, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.22583008, + "step": 2153, + "time_per_iteration": 2.4646072387695312 + }, + { + "auxiliary_loss_clip": 0.01020698, + "auxiliary_loss_mlp": 0.01006533, + "balance_loss_clip": 1.00498986, + "balance_loss_mlp": 1.00497127, + "epoch": 0.0625036271835645, + "flos": 74770269926400.0, + "grad_norm": 0.7141729790228842, + "language_loss": 0.51608825, + "learning_rate": 3.988930857179618e-06, + "loss": 0.5363605, + "num_input_tokens_seen": 60951225, + "router_z_loss_clip": 0.15625, + "router_z_loss_mlp": 0.015625, + "step": 2154, + "time_per_iteration": 3.068878412246704 + }, + { + "auxiliary_loss_clip": 0.01135123, + "auxiliary_loss_mlp": 0.01063647, + "balance_loss_clip": 1.04083633, + "balance_loss_mlp": 1.03740871, + "epoch": 0.06253264465208055, + "flos": 14786973596160.0, + "grad_norm": 2.87871897480034, + "language_loss": 0.82075274, + "learning_rate": 3.988911100322899e-06, + "loss": 0.84274042, + "num_input_tokens_seen": 60966465, + "router_z_loss_clip": 0.94384766, + "router_z_loss_mlp": 0.26220703, + "step": 2155, + "time_per_iteration": 2.3924784660339355 + }, + { + "auxiliary_loss_clip": 0.01018079, + "auxiliary_loss_mlp": 0.01003183, + "balance_loss_clip": 1.00251031, + "balance_loss_mlp": 1.00179994, + "epoch": 0.06256166212059659, + "flos": 58312597173120.0, + "grad_norm": 0.6898169259821068, + "language_loss": 0.52416861, + "learning_rate": 3.988891325899313e-06, + "loss": 0.54438114, + "num_input_tokens_seen": 61022415, + "router_z_loss_clip": 0.15625, + "router_z_loss_mlp": 0.01385498, + "step": 2156, + "time_per_iteration": 2.886132001876831 + }, + { + "auxiliary_loss_clip": 0.01120194, + "auxiliary_loss_mlp": 0.01048302, + "balance_loss_clip": 1.0366714, + "balance_loss_mlp": 1.0272615, + "epoch": 0.06259067958911264, + "flos": 30256395004800.0, + "grad_norm": 2.262531311968228, + "language_loss": 0.7404145, + "learning_rate": 3.988871533909035e-06, + "loss": 0.76209939, + "num_input_tokens_seen": 61040645, + "router_z_loss_clip": 0.83544922, + "router_z_loss_mlp": 0.21057129, + "step": 2157, + "time_per_iteration": 2.530007839202881 + }, + { + "auxiliary_loss_clip": 0.01121433, + "auxiliary_loss_mlp": 0.01048905, + "balance_loss_clip": 1.03896904, + "balance_loss_mlp": 1.02613628, + "epoch": 0.0626196970576287, + "flos": 49118955229440.0, + "grad_norm": 2.340016761000148, + "language_loss": 0.72660172, + "learning_rate": 3.988851724352237e-06, + "loss": 0.74830514, + "num_input_tokens_seen": 61061750, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.2277832, + "step": 2158, + "time_per_iteration": 2.657329797744751 + }, + { + "auxiliary_loss_clip": 0.01129909, + "auxiliary_loss_mlp": 0.01051656, + "balance_loss_clip": 1.0410428, + "balance_loss_mlp": 1.02374959, + "epoch": 0.06264871452614473, + "flos": 11500913520000.0, + "grad_norm": 2.2977694321782325, + "language_loss": 0.80963039, + "learning_rate": 3.988831897229097e-06, + "loss": 0.83144611, + "num_input_tokens_seen": 61073395, + "router_z_loss_clip": 0.88867188, + "router_z_loss_mlp": 0.27880859, + "step": 2159, + "time_per_iteration": 2.3877203464508057 + }, + { + "auxiliary_loss_clip": 0.01121706, + "auxiliary_loss_mlp": 0.01044533, + "balance_loss_clip": 1.03781796, + "balance_loss_mlp": 1.02061939, + "epoch": 0.06267773199466078, + "flos": 74732321278080.0, + "grad_norm": 3.7791374415818026, + "language_loss": 0.78914255, + "learning_rate": 3.988812052539788e-06, + "loss": 0.8108049, + "num_input_tokens_seen": 61100390, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.23913574, + "step": 2160, + "time_per_iteration": 2.8339269161224365 + }, + { + "auxiliary_loss_clip": 0.01016871, + "auxiliary_loss_mlp": 0.01001383, + "balance_loss_clip": 1.00158215, + "balance_loss_mlp": 1.0000298, + "epoch": 0.06270674946317684, + "flos": 74783780622720.0, + "grad_norm": 0.6513054672607399, + "language_loss": 0.50725281, + "learning_rate": 3.988792190284487e-06, + "loss": 0.52743536, + "num_input_tokens_seen": 61166900, + "router_z_loss_clip": 0.15234375, + "router_z_loss_mlp": 0.0135498, + "step": 2161, + "time_per_iteration": 3.196429491043091 + }, + { + "auxiliary_loss_clip": 0.01122809, + "auxiliary_loss_mlp": 0.01043686, + "balance_loss_clip": 1.03381276, + "balance_loss_mlp": 1.02163196, + "epoch": 0.06273576693169287, + "flos": 16536366691200.0, + "grad_norm": 2.5279305262048304, + "language_loss": 0.69015282, + "learning_rate": 3.988772310463368e-06, + "loss": 0.7118178, + "num_input_tokens_seen": 61180240, + "router_z_loss_clip": 0.89013672, + "router_z_loss_mlp": 0.22058105, + "step": 2162, + "time_per_iteration": 2.5675289630889893 + }, + { + "auxiliary_loss_clip": 0.01135543, + "auxiliary_loss_mlp": 0.01046332, + "balance_loss_clip": 1.0407542, + "balance_loss_mlp": 1.01883006, + "epoch": 0.06276478440020893, + "flos": 30364067237760.0, + "grad_norm": 2.523595168659564, + "language_loss": 0.95280772, + "learning_rate": 3.988752413076607e-06, + "loss": 0.97462654, + "num_input_tokens_seen": 61194985, + "router_z_loss_clip": 0.94824219, + "router_z_loss_mlp": 0.27502441, + "step": 2163, + "time_per_iteration": 2.518529176712036 + }, + { + "auxiliary_loss_clip": 0.01123923, + "auxiliary_loss_mlp": 0.01049419, + "balance_loss_clip": 1.03691912, + "balance_loss_mlp": 1.02400374, + "epoch": 0.06279380186872498, + "flos": 43025878108800.0, + "grad_norm": 1.7490447404211718, + "language_loss": 0.84607261, + "learning_rate": 3.98873249812438e-06, + "loss": 0.86780608, + "num_input_tokens_seen": 61215305, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.25408936, + "step": 2164, + "time_per_iteration": 2.7722115516662598 + }, + { + "auxiliary_loss_clip": 0.011361, + "auxiliary_loss_mlp": 0.01052779, + "balance_loss_clip": 1.0415442, + "balance_loss_mlp": 1.02816212, + "epoch": 0.06282281933724101, + "flos": 20991109351680.0, + "grad_norm": 2.255733642866683, + "language_loss": 0.69127637, + "learning_rate": 3.988712565606864e-06, + "loss": 0.71316516, + "num_input_tokens_seen": 61231905, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.24597168, + "step": 2165, + "time_per_iteration": 2.447946310043335 + }, + { + "auxiliary_loss_clip": 0.01127851, + "auxiliary_loss_mlp": 0.01041976, + "balance_loss_clip": 1.04035807, + "balance_loss_mlp": 1.01739502, + "epoch": 0.06285183680575707, + "flos": 14823457833600.0, + "grad_norm": 2.319647849433049, + "language_loss": 0.66958064, + "learning_rate": 3.9886926155242325e-06, + "loss": 0.69127893, + "num_input_tokens_seen": 61244330, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.24572754, + "step": 2166, + "time_per_iteration": 2.390880584716797 + }, + { + "auxiliary_loss_clip": 0.01127004, + "auxiliary_loss_mlp": 0.01045463, + "balance_loss_clip": 1.03725922, + "balance_loss_mlp": 1.02002406, + "epoch": 0.06288085427427312, + "flos": 23797463080320.0, + "grad_norm": 5.0762641543500315, + "language_loss": 0.98506653, + "learning_rate": 3.988672647876664e-06, + "loss": 1.00679123, + "num_input_tokens_seen": 61259250, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.25439453, + "step": 2167, + "time_per_iteration": 2.4662587642669678 + }, + { + "auxiliary_loss_clip": 0.01136537, + "auxiliary_loss_mlp": 0.01063106, + "balance_loss_clip": 1.04058039, + "balance_loss_mlp": 1.03616428, + "epoch": 0.06290987174278916, + "flos": 47623206228480.0, + "grad_norm": 1.9965347954533303, + "language_loss": 0.80259371, + "learning_rate": 3.988652662664333e-06, + "loss": 0.82459021, + "num_input_tokens_seen": 61281860, + "router_z_loss_clip": 0.95947266, + "router_z_loss_mlp": 0.26965332, + "step": 2168, + "time_per_iteration": 2.63419771194458 + }, + { + "auxiliary_loss_clip": 0.01122818, + "auxiliary_loss_mlp": 0.01051035, + "balance_loss_clip": 1.03551221, + "balance_loss_mlp": 1.02565503, + "epoch": 0.06293888921130521, + "flos": 29927198995200.0, + "grad_norm": 2.1359870567318437, + "language_loss": 0.73109257, + "learning_rate": 3.988632659887417e-06, + "loss": 0.7528311, + "num_input_tokens_seen": 61296710, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.25378418, + "step": 2169, + "time_per_iteration": 2.5255300998687744 + }, + { + "auxiliary_loss_clip": 0.01017641, + "auxiliary_loss_mlp": 0.00999998, + "balance_loss_clip": 1.00265503, + "balance_loss_mlp": 0.99865735, + "epoch": 0.06296790667982126, + "flos": 74769781167360.0, + "grad_norm": 0.8099342190154618, + "language_loss": 0.53762186, + "learning_rate": 3.988612639546093e-06, + "loss": 0.55779827, + "num_input_tokens_seen": 61365055, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01342773, + "step": 2170, + "time_per_iteration": 3.1558268070220947 + }, + { + "auxiliary_loss_clip": 0.01123912, + "auxiliary_loss_mlp": 0.01051115, + "balance_loss_clip": 1.03581285, + "balance_loss_mlp": 1.02804828, + "epoch": 0.0629969241483373, + "flos": 12935284617600.0, + "grad_norm": 2.3681835342421156, + "language_loss": 0.84392595, + "learning_rate": 3.988592601640538e-06, + "loss": 0.86567616, + "num_input_tokens_seen": 61379445, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.23071289, + "step": 2171, + "time_per_iteration": 2.405836343765259 + }, + { + "auxiliary_loss_clip": 0.01016603, + "auxiliary_loss_mlp": 0.01001438, + "balance_loss_clip": 1.00169206, + "balance_loss_mlp": 1.00006723, + "epoch": 0.06302594161685335, + "flos": 67618591338240.0, + "grad_norm": 0.6615608726899272, + "language_loss": 0.50114465, + "learning_rate": 3.988572546170928e-06, + "loss": 0.52132499, + "num_input_tokens_seen": 61436370, + "router_z_loss_clip": 0.14941406, + "router_z_loss_mlp": 0.01373291, + "step": 2172, + "time_per_iteration": 2.9902288913726807 + }, + { + "auxiliary_loss_clip": 0.01016605, + "auxiliary_loss_mlp": 0.0100043, + "balance_loss_clip": 1.00169039, + "balance_loss_mlp": 0.99908888, + "epoch": 0.06305495908536939, + "flos": 62835665097600.0, + "grad_norm": 0.7556035769014084, + "language_loss": 0.53063858, + "learning_rate": 3.9885524731374405e-06, + "loss": 0.55080891, + "num_input_tokens_seen": 61498475, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01342773, + "step": 2173, + "time_per_iteration": 3.0282959938049316 + }, + { + "auxiliary_loss_clip": 0.01138403, + "auxiliary_loss_mlp": 0.01052768, + "balance_loss_clip": 1.04141128, + "balance_loss_mlp": 1.02662575, + "epoch": 0.06308397655388544, + "flos": 29671948978560.0, + "grad_norm": 1.9458301368121362, + "language_loss": 0.89716387, + "learning_rate": 3.988532382540253e-06, + "loss": 0.91907561, + "num_input_tokens_seen": 61518450, + "router_z_loss_clip": 0.96923828, + "router_z_loss_mlp": 0.26123047, + "step": 2174, + "time_per_iteration": 2.532010793685913 + }, + { + "auxiliary_loss_clip": 0.01119707, + "auxiliary_loss_mlp": 0.01046166, + "balance_loss_clip": 1.03557861, + "balance_loss_mlp": 1.02241921, + "epoch": 0.06311299402240149, + "flos": 29853986140800.0, + "grad_norm": 4.97537917557407, + "language_loss": 0.91164112, + "learning_rate": 3.988512274379543e-06, + "loss": 0.93329978, + "num_input_tokens_seen": 61532740, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.23706055, + "step": 2175, + "time_per_iteration": 2.722780227661133 + }, + { + "auxiliary_loss_clip": 0.01105529, + "auxiliary_loss_mlp": 0.01044501, + "balance_loss_clip": 1.03321826, + "balance_loss_mlp": 1.02404487, + "epoch": 0.06314201149091753, + "flos": 20549318607360.0, + "grad_norm": 2.748076871156604, + "language_loss": 0.80818975, + "learning_rate": 3.988492148655487e-06, + "loss": 0.82968998, + "num_input_tokens_seen": 61545645, + "router_z_loss_clip": 0.72216797, + "router_z_loss_mlp": 0.20471191, + "step": 2176, + "time_per_iteration": 2.493403673171997 + }, + { + "auxiliary_loss_clip": 0.01124311, + "auxiliary_loss_mlp": 0.01046696, + "balance_loss_clip": 1.03656292, + "balance_loss_mlp": 1.02278256, + "epoch": 0.06317102895943358, + "flos": 19457305102080.0, + "grad_norm": 3.5408478864261115, + "language_loss": 0.89236498, + "learning_rate": 3.9884720053682645e-06, + "loss": 0.91407508, + "num_input_tokens_seen": 61558855, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.23913574, + "step": 2177, + "time_per_iteration": 2.38291597366333 + }, + { + "auxiliary_loss_clip": 0.01128149, + "auxiliary_loss_mlp": 0.01044605, + "balance_loss_clip": 1.03817868, + "balance_loss_mlp": 1.02025032, + "epoch": 0.06320004642794963, + "flos": 25441069155840.0, + "grad_norm": 2.302205617633888, + "language_loss": 0.90328157, + "learning_rate": 3.988451844518052e-06, + "loss": 0.92500913, + "num_input_tokens_seen": 61572720, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.24377441, + "step": 2178, + "time_per_iteration": 2.4232394695281982 + }, + { + "auxiliary_loss_clip": 0.01019345, + "auxiliary_loss_mlp": 0.01008242, + "balance_loss_clip": 1.00428498, + "balance_loss_mlp": 1.00683522, + "epoch": 0.06322906389646567, + "flos": 60910693441920.0, + "grad_norm": 0.7643800012963894, + "language_loss": 0.49455222, + "learning_rate": 3.98843166610503e-06, + "loss": 0.51482809, + "num_input_tokens_seen": 61629855, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01403809, + "step": 2179, + "time_per_iteration": 3.074239730834961 + }, + { + "auxiliary_loss_clip": 0.01019033, + "auxiliary_loss_mlp": 0.01002854, + "balance_loss_clip": 1.00405955, + "balance_loss_mlp": 1.00132799, + "epoch": 0.06325808136498172, + "flos": 59561998554240.0, + "grad_norm": 0.6973102129718681, + "language_loss": 0.4969399, + "learning_rate": 3.9884114701293725e-06, + "loss": 0.51715875, + "num_input_tokens_seen": 61695015, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01525879, + "step": 2180, + "time_per_iteration": 3.1057488918304443 + }, + { + "auxiliary_loss_clip": 0.01116482, + "auxiliary_loss_mlp": 0.01042189, + "balance_loss_clip": 1.03452778, + "balance_loss_mlp": 1.02107656, + "epoch": 0.06328709883349777, + "flos": 21026581159680.0, + "grad_norm": 2.3346795646334386, + "language_loss": 0.89438152, + "learning_rate": 3.9883912565912614e-06, + "loss": 0.9159683, + "num_input_tokens_seen": 61714105, + "router_z_loss_clip": 0.82080078, + "router_z_loss_mlp": 0.21105957, + "step": 2181, + "time_per_iteration": 2.42598819732666 + }, + { + "auxiliary_loss_clip": 0.01016814, + "auxiliary_loss_mlp": 0.01002112, + "balance_loss_clip": 1.00219488, + "balance_loss_mlp": 1.0006274, + "epoch": 0.06331611630201381, + "flos": 66053818846080.0, + "grad_norm": 0.6956057731393864, + "language_loss": 0.50483656, + "learning_rate": 3.988371025490874e-06, + "loss": 0.52502584, + "num_input_tokens_seen": 61777675, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01483154, + "step": 2182, + "time_per_iteration": 3.0268468856811523 + }, + { + "auxiliary_loss_clip": 0.01130513, + "auxiliary_loss_mlp": 0.01046965, + "balance_loss_clip": 1.03910196, + "balance_loss_mlp": 1.02402949, + "epoch": 0.06334513377052986, + "flos": 36895583612160.0, + "grad_norm": 2.554213957970789, + "language_loss": 0.722911, + "learning_rate": 3.98835077682839e-06, + "loss": 0.74468583, + "num_input_tokens_seen": 61795555, + "router_z_loss_clip": 0.9140625, + "router_z_loss_mlp": 0.22937012, + "step": 2183, + "time_per_iteration": 2.5938379764556885 + }, + { + "auxiliary_loss_clip": 0.01123557, + "auxiliary_loss_mlp": 0.01055311, + "balance_loss_clip": 1.03426218, + "balance_loss_mlp": 1.0309689, + "epoch": 0.06337415123904591, + "flos": 15369464586240.0, + "grad_norm": 2.807029711276468, + "language_loss": 0.74794173, + "learning_rate": 3.988330510603986e-06, + "loss": 0.76973045, + "num_input_tokens_seen": 61807265, + "router_z_loss_clip": 0.89306641, + "router_z_loss_mlp": 0.2434082, + "step": 2184, + "time_per_iteration": 2.4412989616394043 + }, + { + "auxiliary_loss_clip": 0.01128229, + "auxiliary_loss_mlp": 0.01046276, + "balance_loss_clip": 1.0349015, + "balance_loss_mlp": 1.02263641, + "epoch": 0.06340316870756195, + "flos": 19420052814720.0, + "grad_norm": 3.4456453200828445, + "language_loss": 0.96529019, + "learning_rate": 3.9883102268178425e-06, + "loss": 0.98703521, + "num_input_tokens_seen": 61820340, + "router_z_loss_clip": 0.93359375, + "router_z_loss_mlp": 0.2364502, + "step": 2185, + "time_per_iteration": 2.3883724212646484 + }, + { + "auxiliary_loss_clip": 0.01129407, + "auxiliary_loss_mlp": 0.01064282, + "balance_loss_clip": 1.0371747, + "balance_loss_mlp": 1.03586292, + "epoch": 0.063432186176078, + "flos": 29895707082240.0, + "grad_norm": 2.12432008681369, + "language_loss": 0.88629377, + "learning_rate": 3.988289925470138e-06, + "loss": 0.90823072, + "num_input_tokens_seen": 61839485, + "router_z_loss_clip": 0.92285156, + "router_z_loss_mlp": 0.2845459, + "step": 2186, + "time_per_iteration": 2.505707025527954 + }, + { + "auxiliary_loss_clip": 0.01016227, + "auxiliary_loss_mlp": 0.01003471, + "balance_loss_clip": 1.00163043, + "balance_loss_mlp": 1.00202215, + "epoch": 0.06346120364459405, + "flos": 66749288595840.0, + "grad_norm": 0.6065143958488999, + "language_loss": 0.46423587, + "learning_rate": 3.988269606561054e-06, + "loss": 0.48443285, + "num_input_tokens_seen": 61907555, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01446533, + "step": 2187, + "time_per_iteration": 3.126533031463623 + }, + { + "auxiliary_loss_clip": 0.01127274, + "auxiliary_loss_mlp": 0.01051313, + "balance_loss_clip": 1.03692508, + "balance_loss_mlp": 1.02604103, + "epoch": 0.06349022111311009, + "flos": 27190706630400.0, + "grad_norm": 3.0373559841006914, + "language_loss": 0.86535108, + "learning_rate": 3.988249270090767e-06, + "loss": 0.88713694, + "num_input_tokens_seen": 61921040, + "router_z_loss_clip": 0.90380859, + "router_z_loss_mlp": 0.25268555, + "step": 2188, + "time_per_iteration": 2.705173969268799 + }, + { + "auxiliary_loss_clip": 0.01126926, + "auxiliary_loss_mlp": 0.01043314, + "balance_loss_clip": 1.03783607, + "balance_loss_mlp": 1.01865005, + "epoch": 0.06351923858162614, + "flos": 26293158731520.0, + "grad_norm": 3.491284004840228, + "language_loss": 1.01096833, + "learning_rate": 3.988228916059459e-06, + "loss": 1.03267074, + "num_input_tokens_seen": 61937520, + "router_z_loss_clip": 0.89111328, + "router_z_loss_mlp": 0.2467041, + "step": 2189, + "time_per_iteration": 2.4850594997406006 + }, + { + "auxiliary_loss_clip": 0.01127919, + "auxiliary_loss_mlp": 0.01048782, + "balance_loss_clip": 1.03697896, + "balance_loss_mlp": 1.02349758, + "epoch": 0.06354825605014218, + "flos": 25696179527040.0, + "grad_norm": 2.127149531093213, + "language_loss": 0.83435422, + "learning_rate": 3.988208544467307e-06, + "loss": 0.85612118, + "num_input_tokens_seen": 61954590, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.25292969, + "step": 2190, + "time_per_iteration": 2.5384020805358887 + }, + { + "auxiliary_loss_clip": 0.01122297, + "auxiliary_loss_mlp": 0.01056703, + "balance_loss_clip": 1.03611326, + "balance_loss_mlp": 1.0293088, + "epoch": 0.06357727351865823, + "flos": 12414660289920.0, + "grad_norm": 3.0697255145693365, + "language_loss": 0.97805071, + "learning_rate": 3.988188155314494e-06, + "loss": 0.99984062, + "num_input_tokens_seen": 61965445, + "router_z_loss_clip": 0.86230469, + "router_z_loss_mlp": 0.27404785, + "step": 2191, + "time_per_iteration": 2.356536626815796 + }, + { + "auxiliary_loss_clip": 0.01130884, + "auxiliary_loss_mlp": 0.01052476, + "balance_loss_clip": 1.03919518, + "balance_loss_mlp": 1.02667904, + "epoch": 0.06360629098717428, + "flos": 34278109672320.0, + "grad_norm": 2.0215520571599046, + "language_loss": 0.88521159, + "learning_rate": 3.988167748601198e-06, + "loss": 0.90704525, + "num_input_tokens_seen": 61986890, + "router_z_loss_clip": 0.91650391, + "router_z_loss_mlp": 0.2578125, + "step": 2192, + "time_per_iteration": 2.5490620136260986 + }, + { + "auxiliary_loss_clip": 0.01127011, + "auxiliary_loss_mlp": 0.01048602, + "balance_loss_clip": 1.04027653, + "balance_loss_mlp": 1.02464116, + "epoch": 0.06363530845569032, + "flos": 24893770682880.0, + "grad_norm": 2.610476432359899, + "language_loss": 0.84943402, + "learning_rate": 3.9881473243275994e-06, + "loss": 0.87119013, + "num_input_tokens_seen": 62001320, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.23974609, + "step": 2193, + "time_per_iteration": 2.465771198272705 + }, + { + "auxiliary_loss_clip": 0.01018733, + "auxiliary_loss_mlp": 0.01002366, + "balance_loss_clip": 1.00409913, + "balance_loss_mlp": 1.00091124, + "epoch": 0.06366432592420637, + "flos": 67875377454720.0, + "grad_norm": 0.7572086152947165, + "language_loss": 0.56695354, + "learning_rate": 3.98812688249388e-06, + "loss": 0.58716452, + "num_input_tokens_seen": 62064250, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01452637, + "step": 2194, + "time_per_iteration": 3.0490899085998535 + }, + { + "auxiliary_loss_clip": 0.01018572, + "auxiliary_loss_mlp": 0.01002612, + "balance_loss_clip": 1.00397921, + "balance_loss_mlp": 1.00122893, + "epoch": 0.06369334339272242, + "flos": 71850972349440.0, + "grad_norm": 0.8155378866524862, + "language_loss": 0.529836, + "learning_rate": 3.988106423100219e-06, + "loss": 0.55004781, + "num_input_tokens_seen": 62124240, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01385498, + "step": 2195, + "time_per_iteration": 3.0083587169647217 + }, + { + "auxiliary_loss_clip": 0.0112714, + "auxiliary_loss_mlp": 0.01057028, + "balance_loss_clip": 1.03994155, + "balance_loss_mlp": 1.03158903, + "epoch": 0.06372236086123846, + "flos": 31677709253760.0, + "grad_norm": 2.740276441923622, + "language_loss": 0.93371618, + "learning_rate": 3.988085946146798e-06, + "loss": 0.95555788, + "num_input_tokens_seen": 62140860, + "router_z_loss_clip": 0.87255859, + "router_z_loss_mlp": 0.25427246, + "step": 2196, + "time_per_iteration": 2.4974608421325684 + }, + { + "auxiliary_loss_clip": 0.01017238, + "auxiliary_loss_mlp": 0.01000553, + "balance_loss_clip": 1.0027976, + "balance_loss_mlp": 0.99925321, + "epoch": 0.06375137832975451, + "flos": 58862164884480.0, + "grad_norm": 0.7023130955886342, + "language_loss": 0.53654373, + "learning_rate": 3.988065451633798e-06, + "loss": 0.55672163, + "num_input_tokens_seen": 62195735, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01300049, + "step": 2197, + "time_per_iteration": 2.8959622383117676 + }, + { + "auxiliary_loss_clip": 0.01132178, + "auxiliary_loss_mlp": 0.01047766, + "balance_loss_clip": 1.03976572, + "balance_loss_mlp": 1.02267241, + "epoch": 0.06378039579827056, + "flos": 21937814311680.0, + "grad_norm": 3.110637222595542, + "language_loss": 0.78059077, + "learning_rate": 3.9880449395613984e-06, + "loss": 0.80239022, + "num_input_tokens_seen": 62212915, + "router_z_loss_clip": 0.92480469, + "router_z_loss_mlp": 0.25097656, + "step": 2198, + "time_per_iteration": 2.527750253677368 + }, + { + "auxiliary_loss_clip": 0.01015337, + "auxiliary_loss_mlp": 0.01003006, + "balance_loss_clip": 1.00081301, + "balance_loss_mlp": 1.00149834, + "epoch": 0.0638094132667866, + "flos": 74771701292160.0, + "grad_norm": 0.6972443812735443, + "language_loss": 0.48992974, + "learning_rate": 3.988024409929782e-06, + "loss": 0.51011318, + "num_input_tokens_seen": 62271555, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01507568, + "step": 2199, + "time_per_iteration": 3.0569186210632324 + }, + { + "auxiliary_loss_clip": 0.01128838, + "auxiliary_loss_mlp": 0.01053386, + "balance_loss_clip": 1.04045153, + "balance_loss_mlp": 1.02826881, + "epoch": 0.06383843073530265, + "flos": 12158397843840.0, + "grad_norm": 3.3712844603808816, + "language_loss": 1.00334334, + "learning_rate": 3.988003862739129e-06, + "loss": 1.02516556, + "num_input_tokens_seen": 62282560, + "router_z_loss_clip": 0.88525391, + "router_z_loss_mlp": 0.25134277, + "step": 2200, + "time_per_iteration": 2.3658313751220703 + }, + { + "auxiliary_loss_clip": 0.01137693, + "auxiliary_loss_mlp": 0.01064299, + "balance_loss_clip": 1.04297638, + "balance_loss_mlp": 1.04029036, + "epoch": 0.0638674482038187, + "flos": 70063665517440.0, + "grad_norm": 2.2107160723785313, + "language_loss": 0.91820097, + "learning_rate": 3.987983297989621e-06, + "loss": 0.94022095, + "num_input_tokens_seen": 62306780, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.23999023, + "step": 2201, + "time_per_iteration": 2.949467420578003 + }, + { + "auxiliary_loss_clip": 0.01124024, + "auxiliary_loss_mlp": 0.0105206, + "balance_loss_clip": 1.03717721, + "balance_loss_mlp": 1.02863574, + "epoch": 0.06389646567233474, + "flos": 18689774572800.0, + "grad_norm": 3.1670415717001137, + "language_loss": 0.76779842, + "learning_rate": 3.9879627156814415e-06, + "loss": 0.78955925, + "num_input_tokens_seen": 62320670, + "router_z_loss_clip": 0.86914062, + "router_z_loss_mlp": 0.23413086, + "step": 2202, + "time_per_iteration": 2.3763105869293213 + }, + { + "auxiliary_loss_clip": 0.01133705, + "auxiliary_loss_mlp": 0.01052043, + "balance_loss_clip": 1.04120922, + "balance_loss_mlp": 1.02691412, + "epoch": 0.0639254831408508, + "flos": 37302426218880.0, + "grad_norm": 2.475809896086195, + "language_loss": 0.82580316, + "learning_rate": 3.98794211581477e-06, + "loss": 0.8476606, + "num_input_tokens_seen": 62345265, + "router_z_loss_clip": 0.92529297, + "router_z_loss_mlp": 0.25109863, + "step": 2203, + "time_per_iteration": 2.6442742347717285 + }, + { + "auxiliary_loss_clip": 0.01019649, + "auxiliary_loss_mlp": 0.0101362, + "balance_loss_clip": 1.00480366, + "balance_loss_mlp": 1.0120821, + "epoch": 0.06395450060936683, + "flos": 74777426755200.0, + "grad_norm": 0.6545195555225692, + "language_loss": 0.53050733, + "learning_rate": 3.9879214983897896e-06, + "loss": 0.55084002, + "num_input_tokens_seen": 62414170, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01538086, + "step": 2204, + "time_per_iteration": 3.243216037750244 + }, + { + "auxiliary_loss_clip": 0.01126805, + "auxiliary_loss_mlp": 0.01049772, + "balance_loss_clip": 1.03969026, + "balance_loss_mlp": 1.02581143, + "epoch": 0.06398351807788288, + "flos": 33212002262400.0, + "grad_norm": 4.513521793206258, + "language_loss": 0.8433677, + "learning_rate": 3.9879008634066815e-06, + "loss": 0.86513352, + "num_input_tokens_seen": 62431365, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.23974609, + "step": 2205, + "time_per_iteration": 2.519645929336548 + }, + { + "auxiliary_loss_clip": 0.01115878, + "auxiliary_loss_mlp": 0.01048449, + "balance_loss_clip": 1.03510535, + "balance_loss_mlp": 1.02860641, + "epoch": 0.06401253554639894, + "flos": 27664862071680.0, + "grad_norm": 2.26622556098352, + "language_loss": 0.69847482, + "learning_rate": 3.987880210865629e-06, + "loss": 0.72011805, + "num_input_tokens_seen": 62446705, + "router_z_loss_clip": 0.80859375, + "router_z_loss_mlp": 0.19854736, + "step": 2206, + "time_per_iteration": 2.5210680961608887 + }, + { + "auxiliary_loss_clip": 0.01112377, + "auxiliary_loss_mlp": 0.01039883, + "balance_loss_clip": 1.03372383, + "balance_loss_mlp": 1.01815748, + "epoch": 0.06404155301491497, + "flos": 27921857656320.0, + "grad_norm": 2.861545183601994, + "language_loss": 0.97401011, + "learning_rate": 3.9878595407668144e-06, + "loss": 0.99553281, + "num_input_tokens_seen": 62461930, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.21716309, + "step": 2207, + "time_per_iteration": 2.4690942764282227 + }, + { + "auxiliary_loss_clip": 0.01124865, + "auxiliary_loss_mlp": 0.01050395, + "balance_loss_clip": 1.03850603, + "balance_loss_mlp": 1.02505064, + "epoch": 0.06407057048343102, + "flos": 28834766553600.0, + "grad_norm": 2.8808613225958566, + "language_loss": 0.89858764, + "learning_rate": 3.98783885311042e-06, + "loss": 0.9203403, + "num_input_tokens_seen": 62476810, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.25341797, + "step": 2208, + "time_per_iteration": 2.5202372074127197 + }, + { + "auxiliary_loss_clip": 0.01121313, + "auxiliary_loss_mlp": 0.010558, + "balance_loss_clip": 1.03595185, + "balance_loss_mlp": 1.03050399, + "epoch": 0.06409958795194708, + "flos": 18288098847360.0, + "grad_norm": 2.7224434135221363, + "language_loss": 0.92457891, + "learning_rate": 3.987818147896627e-06, + "loss": 0.9463501, + "num_input_tokens_seen": 62489925, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.25292969, + "step": 2209, + "time_per_iteration": 4.8620359897613525 + }, + { + "auxiliary_loss_clip": 0.01020885, + "auxiliary_loss_mlp": 0.01003024, + "balance_loss_clip": 1.00653744, + "balance_loss_mlp": 1.00170124, + "epoch": 0.06412860542046311, + "flos": 50871139367040.0, + "grad_norm": 0.7408095340432312, + "language_loss": 0.49560738, + "learning_rate": 3.987797425125621e-06, + "loss": 0.51584649, + "num_input_tokens_seen": 62548540, + "router_z_loss_clip": 0.14355469, + "router_z_loss_mlp": 0.01324463, + "step": 2210, + "time_per_iteration": 2.994415044784546 + }, + { + "auxiliary_loss_clip": 0.0112877, + "auxiliary_loss_mlp": 0.01043175, + "balance_loss_clip": 1.03855157, + "balance_loss_mlp": 1.01858234, + "epoch": 0.06415762288897917, + "flos": 19965954833280.0, + "grad_norm": 2.4178887735019314, + "language_loss": 0.90303826, + "learning_rate": 3.987776684797583e-06, + "loss": 0.92475778, + "num_input_tokens_seen": 62561040, + "router_z_loss_clip": 0.90234375, + "router_z_loss_mlp": 0.24584961, + "step": 2211, + "time_per_iteration": 2.4399912357330322 + }, + { + "auxiliary_loss_clip": 0.01017991, + "auxiliary_loss_mlp": 0.01002198, + "balance_loss_clip": 1.00372148, + "balance_loss_mlp": 1.00085092, + "epoch": 0.06418664035749522, + "flos": 71781529921920.0, + "grad_norm": 0.6780937020362681, + "language_loss": 0.51756424, + "learning_rate": 3.987755926912698e-06, + "loss": 0.53776616, + "num_input_tokens_seen": 62628715, + "router_z_loss_clip": 0.14257812, + "router_z_loss_mlp": 0.01348877, + "step": 2212, + "time_per_iteration": 3.0819900035858154 + }, + { + "auxiliary_loss_clip": 0.01126089, + "auxiliary_loss_mlp": 0.01055879, + "balance_loss_clip": 1.03880286, + "balance_loss_mlp": 1.0308094, + "epoch": 0.06421565782601125, + "flos": 45979984177920.0, + "grad_norm": 4.563502363744113, + "language_loss": 0.90671629, + "learning_rate": 3.987735151471148e-06, + "loss": 0.92853594, + "num_input_tokens_seen": 62645180, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.25061035, + "step": 2213, + "time_per_iteration": 5.259060621261597 + }, + { + "auxiliary_loss_clip": 0.01127633, + "auxiliary_loss_mlp": 0.01052655, + "balance_loss_clip": 1.03784776, + "balance_loss_mlp": 1.02878892, + "epoch": 0.0642446752945273, + "flos": 16028834123520.0, + "grad_norm": 2.3856622299157877, + "language_loss": 0.88352054, + "learning_rate": 3.987714358473116e-06, + "loss": 0.90532339, + "num_input_tokens_seen": 62658505, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.2388916, + "step": 2214, + "time_per_iteration": 2.4405577182769775 + }, + { + "auxiliary_loss_clip": 0.01015612, + "auxiliary_loss_mlp": 0.01001825, + "balance_loss_clip": 1.00146794, + "balance_loss_mlp": 1.0004127, + "epoch": 0.06427369276304336, + "flos": 62736511236480.0, + "grad_norm": 0.6663755155763654, + "language_loss": 0.51448655, + "learning_rate": 3.987693547918787e-06, + "loss": 0.53466088, + "num_input_tokens_seen": 62721085, + "router_z_loss_clip": 0.14160156, + "router_z_loss_mlp": 0.01409912, + "step": 2215, + "time_per_iteration": 5.443866968154907 + }, + { + "auxiliary_loss_clip": 0.01124716, + "auxiliary_loss_mlp": 0.01054711, + "balance_loss_clip": 1.03500915, + "balance_loss_mlp": 1.02935565, + "epoch": 0.0643027102315594, + "flos": 15880697758080.0, + "grad_norm": 3.1296323457476443, + "language_loss": 0.91670096, + "learning_rate": 3.9876727198083445e-06, + "loss": 0.93849528, + "num_input_tokens_seen": 62733965, + "router_z_loss_clip": 0.89697266, + "router_z_loss_mlp": 0.25366211, + "step": 2216, + "time_per_iteration": 2.3669469356536865 + }, + { + "auxiliary_loss_clip": 0.01123768, + "auxiliary_loss_mlp": 0.01046795, + "balance_loss_clip": 1.03639126, + "balance_loss_mlp": 1.02283406, + "epoch": 0.06433172770007545, + "flos": 12344973482880.0, + "grad_norm": 1.9267792537499184, + "language_loss": 0.65772378, + "learning_rate": 3.987651874141972e-06, + "loss": 0.67942941, + "num_input_tokens_seen": 62752530, + "router_z_loss_clip": 0.87402344, + "router_z_loss_mlp": 0.23974609, + "step": 2217, + "time_per_iteration": 2.475633144378662 + }, + { + "auxiliary_loss_clip": 0.01015376, + "auxiliary_loss_mlp": 0.01004503, + "balance_loss_clip": 1.00133109, + "balance_loss_mlp": 1.00319219, + "epoch": 0.0643607451685915, + "flos": 74772818455680.0, + "grad_norm": 0.686721336035766, + "language_loss": 0.59673285, + "learning_rate": 3.987631010919853e-06, + "loss": 0.61693168, + "num_input_tokens_seen": 62812975, + "router_z_loss_clip": 0.140625, + "router_z_loss_mlp": 0.01312256, + "step": 2218, + "time_per_iteration": 3.131227493286133 + }, + { + "auxiliary_loss_clip": 0.0101532, + "auxiliary_loss_mlp": 0.00998934, + "balance_loss_clip": 1.00124061, + "balance_loss_mlp": 0.99751514, + "epoch": 0.06438976263710754, + "flos": 74790518515200.0, + "grad_norm": 0.642661446284155, + "language_loss": 0.44509101, + "learning_rate": 3.9876101301421735e-06, + "loss": 0.46523356, + "num_input_tokens_seen": 62877450, + "router_z_loss_clip": 0.140625, + "router_z_loss_mlp": 0.01416016, + "step": 2219, + "time_per_iteration": 3.2366855144500732 + }, + { + "auxiliary_loss_clip": 0.01125337, + "auxiliary_loss_mlp": 0.01046419, + "balance_loss_clip": 1.03711998, + "balance_loss_mlp": 1.02050316, + "epoch": 0.06441878010562359, + "flos": 27082859840640.0, + "grad_norm": 2.3812804399638936, + "language_loss": 0.85862529, + "learning_rate": 3.987589231809117e-06, + "loss": 0.88034284, + "num_input_tokens_seen": 62894975, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.25915527, + "step": 2220, + "time_per_iteration": 2.483211040496826 + }, + { + "auxiliary_loss_clip": 0.01015838, + "auxiliary_loss_mlp": 0.01003435, + "balance_loss_clip": 1.00174308, + "balance_loss_mlp": 1.00204062, + "epoch": 0.06444779757413963, + "flos": 74776309591680.0, + "grad_norm": 0.6707599345775312, + "language_loss": 0.5382036, + "learning_rate": 3.987568315920868e-06, + "loss": 0.55839634, + "num_input_tokens_seen": 62964695, + "router_z_loss_clip": 0.140625, + "router_z_loss_mlp": 0.01397705, + "step": 2221, + "time_per_iteration": 3.1583445072174072 + }, + { + "auxiliary_loss_clip": 0.01115679, + "auxiliary_loss_mlp": 0.01048508, + "balance_loss_clip": 1.036201, + "balance_loss_mlp": 1.02752709, + "epoch": 0.06447681504265568, + "flos": 25111803323520.0, + "grad_norm": 2.96546594360633, + "language_loss": 0.98050559, + "learning_rate": 3.987547382477611e-06, + "loss": 1.00214744, + "num_input_tokens_seen": 62978280, + "router_z_loss_clip": 0.79443359, + "router_z_loss_mlp": 0.2097168, + "step": 2222, + "time_per_iteration": 2.4174365997314453 + }, + { + "auxiliary_loss_clip": 0.01127124, + "auxiliary_loss_mlp": 0.01053369, + "balance_loss_clip": 1.03753281, + "balance_loss_mlp": 1.02736974, + "epoch": 0.06450583251117173, + "flos": 26319518674560.0, + "grad_norm": 1.9128471793235553, + "language_loss": 0.83728671, + "learning_rate": 3.987526431479533e-06, + "loss": 0.85909164, + "num_input_tokens_seen": 62997420, + "router_z_loss_clip": 0.89599609, + "router_z_loss_mlp": 0.26000977, + "step": 2223, + "time_per_iteration": 2.4492931365966797 + }, + { + "auxiliary_loss_clip": 0.01126476, + "auxiliary_loss_mlp": 0.0105431, + "balance_loss_clip": 1.03825641, + "balance_loss_mlp": 1.02899003, + "epoch": 0.06453484997968777, + "flos": 12962238053760.0, + "grad_norm": 3.737556004321379, + "language_loss": 0.86762118, + "learning_rate": 3.987505462926815e-06, + "loss": 0.88942897, + "num_input_tokens_seen": 63009905, + "router_z_loss_clip": 0.8828125, + "router_z_loss_mlp": 0.25341797, + "step": 2224, + "time_per_iteration": 2.7480688095092773 + }, + { + "auxiliary_loss_clip": 0.0112264, + "auxiliary_loss_mlp": 0.01053279, + "balance_loss_clip": 1.0378139, + "balance_loss_mlp": 1.02869797, + "epoch": 0.06456386744820382, + "flos": 11867431639680.0, + "grad_norm": 2.40428733835726, + "language_loss": 0.81464761, + "learning_rate": 3.987484476819645e-06, + "loss": 0.83640677, + "num_input_tokens_seen": 63026240, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.24609375, + "step": 2225, + "time_per_iteration": 2.3630340099334717 + }, + { + "auxiliary_loss_clip": 0.01017391, + "auxiliary_loss_mlp": 0.01009764, + "balance_loss_clip": 1.00336647, + "balance_loss_mlp": 1.00832129, + "epoch": 0.06459288491671987, + "flos": 74770688862720.0, + "grad_norm": 0.6473318675548695, + "language_loss": 0.48283824, + "learning_rate": 3.987463473158208e-06, + "loss": 0.50310981, + "num_input_tokens_seen": 63094350, + "router_z_loss_clip": 0.140625, + "router_z_loss_mlp": 0.0144043, + "step": 2226, + "time_per_iteration": 3.2730674743652344 + }, + { + "auxiliary_loss_clip": 0.01016809, + "auxiliary_loss_mlp": 0.01003466, + "balance_loss_clip": 1.00276673, + "balance_loss_mlp": 1.00195158, + "epoch": 0.06462190238523591, + "flos": 74774214910080.0, + "grad_norm": 0.7339932391505001, + "language_loss": 0.48061192, + "learning_rate": 3.98744245194269e-06, + "loss": 0.50081468, + "num_input_tokens_seen": 63152460, + "router_z_loss_clip": 0.140625, + "router_z_loss_mlp": 0.01513672, + "step": 2227, + "time_per_iteration": 3.0823287963867188 + }, + { + "auxiliary_loss_clip": 0.01134527, + "auxiliary_loss_mlp": 0.0104879, + "balance_loss_clip": 1.03975606, + "balance_loss_mlp": 1.02313566, + "epoch": 0.06465091985375196, + "flos": 32560627426560.0, + "grad_norm": 1.945851259728202, + "language_loss": 0.78039467, + "learning_rate": 3.9874214131732765e-06, + "loss": 0.80222785, + "num_input_tokens_seen": 63169900, + "router_z_loss_clip": 0.94726562, + "router_z_loss_mlp": 0.25671387, + "step": 2228, + "time_per_iteration": 2.4847288131713867 + }, + { + "auxiliary_loss_clip": 0.01128296, + "auxiliary_loss_mlp": 0.01053039, + "balance_loss_clip": 1.03702652, + "balance_loss_mlp": 1.02874422, + "epoch": 0.06467993732226801, + "flos": 25949474507520.0, + "grad_norm": 2.7809539792871116, + "language_loss": 0.91416663, + "learning_rate": 3.987400356850152e-06, + "loss": 0.93597996, + "num_input_tokens_seen": 63183935, + "router_z_loss_clip": 0.91210938, + "router_z_loss_mlp": 0.24316406, + "step": 2229, + "time_per_iteration": 2.667182445526123 + }, + { + "auxiliary_loss_clip": 0.01124433, + "auxiliary_loss_mlp": 0.01039837, + "balance_loss_clip": 1.03841341, + "balance_loss_mlp": 1.01719856, + "epoch": 0.06470895479078405, + "flos": 19056816362880.0, + "grad_norm": 2.185962283382413, + "language_loss": 0.74066067, + "learning_rate": 3.987379282973503e-06, + "loss": 0.76230335, + "num_input_tokens_seen": 63197010, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.22631836, + "step": 2230, + "time_per_iteration": 2.465318202972412 + }, + { + "auxiliary_loss_clip": 0.01120652, + "auxiliary_loss_mlp": 0.01038431, + "balance_loss_clip": 1.03738439, + "balance_loss_mlp": 1.0176053, + "epoch": 0.0647379722593001, + "flos": 40186810569600.0, + "grad_norm": 1.7454059612523543, + "language_loss": 0.73117208, + "learning_rate": 3.987358191543516e-06, + "loss": 0.75276291, + "num_input_tokens_seen": 63223310, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.20800781, + "step": 2231, + "time_per_iteration": 2.6379384994506836 + }, + { + "auxiliary_loss_clip": 0.01125992, + "auxiliary_loss_mlp": 0.01052265, + "balance_loss_clip": 1.03729761, + "balance_loss_mlp": 1.02949595, + "epoch": 0.06476698972781615, + "flos": 31071022824960.0, + "grad_norm": 1.7507909316782608, + "language_loss": 0.87014067, + "learning_rate": 3.987337082560378e-06, + "loss": 0.89192319, + "num_input_tokens_seen": 63248040, + "router_z_loss_clip": 0.88623047, + "router_z_loss_mlp": 0.22753906, + "step": 2232, + "time_per_iteration": 2.5545079708099365 + }, + { + "auxiliary_loss_clip": 0.01125183, + "auxiliary_loss_mlp": 0.01046211, + "balance_loss_clip": 1.03815901, + "balance_loss_mlp": 1.02317357, + "epoch": 0.06479600719633219, + "flos": 30666449456640.0, + "grad_norm": 2.026157948500145, + "language_loss": 0.70719898, + "learning_rate": 3.987315956024273e-06, + "loss": 0.72891295, + "num_input_tokens_seen": 63269710, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.23065186, + "step": 2233, + "time_per_iteration": 2.5405890941619873 + }, + { + "auxiliary_loss_clip": 0.01116955, + "auxiliary_loss_mlp": 0.01045066, + "balance_loss_clip": 1.03380752, + "balance_loss_mlp": 1.02114081, + "epoch": 0.06482502466484824, + "flos": 32408266786560.0, + "grad_norm": 2.0701387200237313, + "language_loss": 0.78540814, + "learning_rate": 3.987294811935391e-06, + "loss": 0.80702829, + "num_input_tokens_seen": 63287420, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.23937988, + "step": 2234, + "time_per_iteration": 2.4984805583953857 + }, + { + "auxiliary_loss_clip": 0.01117321, + "auxiliary_loss_mlp": 0.01038656, + "balance_loss_clip": 1.03630638, + "balance_loss_mlp": 1.01750863, + "epoch": 0.06485404213336428, + "flos": 24235867422720.0, + "grad_norm": 2.7575035881301364, + "language_loss": 1.0133301, + "learning_rate": 3.987273650293917e-06, + "loss": 1.03488994, + "num_input_tokens_seen": 63305950, + "router_z_loss_clip": 0.81054688, + "router_z_loss_mlp": 0.21118164, + "step": 2235, + "time_per_iteration": 2.5565812587738037 + }, + { + "auxiliary_loss_clip": 0.01126354, + "auxiliary_loss_mlp": 0.01050205, + "balance_loss_clip": 1.03642941, + "balance_loss_mlp": 1.02654147, + "epoch": 0.06488305960188033, + "flos": 23542736734080.0, + "grad_norm": 2.344601197838917, + "language_loss": 0.89676797, + "learning_rate": 3.987252471100038e-06, + "loss": 0.91853356, + "num_input_tokens_seen": 63319245, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.23669434, + "step": 2236, + "time_per_iteration": 2.409674882888794 + }, + { + "auxiliary_loss_clip": 0.01121661, + "auxiliary_loss_mlp": 0.01054412, + "balance_loss_clip": 1.03605723, + "balance_loss_mlp": 1.03065395, + "epoch": 0.06491207707039638, + "flos": 25601040472320.0, + "grad_norm": 2.4801526013446544, + "language_loss": 0.73700047, + "learning_rate": 3.98723127435394e-06, + "loss": 0.75876117, + "num_input_tokens_seen": 63337360, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.2376709, + "step": 2237, + "time_per_iteration": 2.676957130432129 + }, + { + "auxiliary_loss_clip": 0.01125861, + "auxiliary_loss_mlp": 0.01051436, + "balance_loss_clip": 1.03909051, + "balance_loss_mlp": 1.03026462, + "epoch": 0.06494109453891242, + "flos": 26934618741120.0, + "grad_norm": 2.502016081815935, + "language_loss": 0.88305461, + "learning_rate": 3.987210060055812e-06, + "loss": 0.90482759, + "num_input_tokens_seen": 63349840, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.21191406, + "step": 2238, + "time_per_iteration": 2.450894355773926 + }, + { + "auxiliary_loss_clip": 0.01121079, + "auxiliary_loss_mlp": 0.01056222, + "balance_loss_clip": 1.03653407, + "balance_loss_mlp": 1.03236771, + "epoch": 0.06497011200742847, + "flos": 43974223902720.0, + "grad_norm": 3.9565078196750108, + "language_loss": 0.91341484, + "learning_rate": 3.98718882820584e-06, + "loss": 0.93518794, + "num_input_tokens_seen": 63365640, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.23852539, + "step": 2239, + "time_per_iteration": 2.5754153728485107 + }, + { + "auxiliary_loss_clip": 0.01127545, + "auxiliary_loss_mlp": 0.01051511, + "balance_loss_clip": 1.03908038, + "balance_loss_mlp": 1.02789533, + "epoch": 0.06499912947594452, + "flos": 66687807824640.0, + "grad_norm": 2.3073117350979158, + "language_loss": 0.84329033, + "learning_rate": 3.9871675788042125e-06, + "loss": 0.86508089, + "num_input_tokens_seen": 63391835, + "router_z_loss_clip": 0.88330078, + "router_z_loss_mlp": 0.23632812, + "step": 2240, + "time_per_iteration": 2.7668538093566895 + }, + { + "auxiliary_loss_clip": 0.01127983, + "auxiliary_loss_mlp": 0.01049172, + "balance_loss_clip": 1.03827548, + "balance_loss_mlp": 1.02293372, + "epoch": 0.06502814694446056, + "flos": 59809674672000.0, + "grad_norm": 2.3796162270759886, + "language_loss": 0.81338477, + "learning_rate": 3.987146311851118e-06, + "loss": 0.83515632, + "num_input_tokens_seen": 63413250, + "router_z_loss_clip": 0.89746094, + "router_z_loss_mlp": 0.26245117, + "step": 2241, + "time_per_iteration": 2.7630836963653564 + }, + { + "auxiliary_loss_clip": 0.01131114, + "auxiliary_loss_mlp": 0.01048986, + "balance_loss_clip": 1.04231405, + "balance_loss_mlp": 1.0233326, + "epoch": 0.06505716441297661, + "flos": 33098115807360.0, + "grad_norm": 4.10210542114097, + "language_loss": 0.92714131, + "learning_rate": 3.987125027346741e-06, + "loss": 0.94894236, + "num_input_tokens_seen": 63425980, + "router_z_loss_clip": 0.88671875, + "router_z_loss_mlp": 0.25671387, + "step": 2242, + "time_per_iteration": 2.5214762687683105 + }, + { + "auxiliary_loss_clip": 0.01123162, + "auxiliary_loss_mlp": 0.0104835, + "balance_loss_clip": 1.03719974, + "balance_loss_mlp": 1.02417397, + "epoch": 0.06508618188149266, + "flos": 28430367742080.0, + "grad_norm": 3.261142168706625, + "language_loss": 0.87931597, + "learning_rate": 3.987103725291273e-06, + "loss": 0.90103102, + "num_input_tokens_seen": 63441355, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.24182129, + "step": 2243, + "time_per_iteration": 2.49347186088562 + }, + { + "auxiliary_loss_clip": 0.01129961, + "auxiliary_loss_mlp": 0.01054578, + "balance_loss_clip": 1.04054737, + "balance_loss_mlp": 1.03146267, + "epoch": 0.0651151993500087, + "flos": 44120544877440.0, + "grad_norm": 2.2866360261646372, + "language_loss": 0.82587302, + "learning_rate": 3.9870824056849e-06, + "loss": 0.84771842, + "num_input_tokens_seen": 63458600, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.23095703, + "step": 2244, + "time_per_iteration": 2.648545742034912 + }, + { + "auxiliary_loss_clip": 0.01124419, + "auxiliary_loss_mlp": 0.01058468, + "balance_loss_clip": 1.03772426, + "balance_loss_mlp": 1.03401792, + "epoch": 0.06514421681852475, + "flos": 27699321450240.0, + "grad_norm": 2.3731842340526437, + "language_loss": 0.74939781, + "learning_rate": 3.987061068527812e-06, + "loss": 0.77122664, + "num_input_tokens_seen": 63472940, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.2444458, + "step": 2245, + "time_per_iteration": 2.52473783493042 + }, + { + "auxiliary_loss_clip": 0.01018487, + "auxiliary_loss_mlp": 0.01001432, + "balance_loss_clip": 1.00355744, + "balance_loss_mlp": 1.0001086, + "epoch": 0.0651732342870408, + "flos": 64448302930560.0, + "grad_norm": 0.7762265747853568, + "language_loss": 0.50626159, + "learning_rate": 3.987039713820196e-06, + "loss": 0.52646077, + "num_input_tokens_seen": 63534065, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01324463, + "step": 2246, + "time_per_iteration": 3.0785884857177734 + }, + { + "auxiliary_loss_clip": 0.01126544, + "auxiliary_loss_mlp": 0.01047424, + "balance_loss_clip": 1.03641415, + "balance_loss_mlp": 1.02346325, + "epoch": 0.06520225175555684, + "flos": 11357210897280.0, + "grad_norm": 2.8286135965351114, + "language_loss": 0.92935115, + "learning_rate": 3.9870183415622415e-06, + "loss": 0.95109081, + "num_input_tokens_seen": 63545540, + "router_z_loss_clip": 0.90039062, + "router_z_loss_mlp": 0.23950195, + "step": 2247, + "time_per_iteration": 2.436099052429199 + }, + { + "auxiliary_loss_clip": 0.01017319, + "auxiliary_loss_mlp": 0.01001163, + "balance_loss_clip": 1.0024004, + "balance_loss_mlp": 0.99977428, + "epoch": 0.0652312692240729, + "flos": 63010510519680.0, + "grad_norm": 0.7496578411968668, + "language_loss": 0.46543503, + "learning_rate": 3.986996951754136e-06, + "loss": 0.48561984, + "num_input_tokens_seen": 63598905, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01391602, + "step": 2248, + "time_per_iteration": 3.060136556625366 + }, + { + "auxiliary_loss_clip": 0.01120088, + "auxiliary_loss_mlp": 0.01054548, + "balance_loss_clip": 1.03701913, + "balance_loss_mlp": 1.02958512, + "epoch": 0.06526028669258895, + "flos": 27776304731520.0, + "grad_norm": 2.878209826810303, + "language_loss": 0.8847599, + "learning_rate": 3.98697554439607e-06, + "loss": 0.90650636, + "num_input_tokens_seen": 63617915, + "router_z_loss_clip": 0.82958984, + "router_z_loss_mlp": 0.24951172, + "step": 2249, + "time_per_iteration": 2.5080087184906006 + }, + { + "auxiliary_loss_clip": 0.01016137, + "auxiliary_loss_mlp": 0.010029, + "balance_loss_clip": 1.00169611, + "balance_loss_mlp": 1.00147557, + "epoch": 0.06528930416110498, + "flos": 74764160438400.0, + "grad_norm": 0.7321021496255589, + "language_loss": 0.56539595, + "learning_rate": 3.9869541194882326e-06, + "loss": 0.58558631, + "num_input_tokens_seen": 63677335, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01422119, + "step": 2250, + "time_per_iteration": 3.159499406814575 + }, + { + "auxiliary_loss_clip": 0.01120514, + "auxiliary_loss_mlp": 0.01062107, + "balance_loss_clip": 1.03413868, + "balance_loss_mlp": 1.03739512, + "epoch": 0.06531832162962103, + "flos": 24162689479680.0, + "grad_norm": 2.7206124308718094, + "language_loss": 0.9046219, + "learning_rate": 3.986932677030812e-06, + "loss": 0.92644811, + "num_input_tokens_seen": 63692445, + "router_z_loss_clip": 0.86474609, + "router_z_loss_mlp": 0.24707031, + "step": 2251, + "time_per_iteration": 2.4599809646606445 + }, + { + "auxiliary_loss_clip": 0.01118896, + "auxiliary_loss_mlp": 0.01054212, + "balance_loss_clip": 1.03533721, + "balance_loss_mlp": 1.03294539, + "epoch": 0.06534733909813707, + "flos": 21790376173440.0, + "grad_norm": 2.569532404630408, + "language_loss": 0.85432035, + "learning_rate": 3.9869112170239975e-06, + "loss": 0.87605143, + "num_input_tokens_seen": 63705245, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.21276855, + "step": 2252, + "time_per_iteration": 2.430448532104492 + }, + { + "auxiliary_loss_clip": 0.01120509, + "auxiliary_loss_mlp": 0.01045604, + "balance_loss_clip": 1.03381658, + "balance_loss_mlp": 1.02210772, + "epoch": 0.06537635656665312, + "flos": 28177317141120.0, + "grad_norm": 1.9417515403919645, + "language_loss": 0.81458807, + "learning_rate": 3.98688973946798e-06, + "loss": 0.83624917, + "num_input_tokens_seen": 63725945, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.23498535, + "step": 2253, + "time_per_iteration": 2.550792932510376 + }, + { + "auxiliary_loss_clip": 0.0112431, + "auxiliary_loss_mlp": 0.01051203, + "balance_loss_clip": 1.03784859, + "balance_loss_mlp": 1.02529883, + "epoch": 0.06540537403516918, + "flos": 11027770508160.0, + "grad_norm": 3.430198807735387, + "language_loss": 0.792521, + "learning_rate": 3.986868244362947e-06, + "loss": 0.81427616, + "num_input_tokens_seen": 63736825, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.2590332, + "step": 2254, + "time_per_iteration": 2.392158269882202 + }, + { + "auxiliary_loss_clip": 0.01138122, + "auxiliary_loss_mlp": 0.01055895, + "balance_loss_clip": 1.0420754, + "balance_loss_mlp": 1.02841735, + "epoch": 0.06543439150368521, + "flos": 34159126158720.0, + "grad_norm": 2.5138593005874657, + "language_loss": 1.00569916, + "learning_rate": 3.986846731709091e-06, + "loss": 1.02763927, + "num_input_tokens_seen": 63756660, + "router_z_loss_clip": 0.96142578, + "router_z_loss_mlp": 0.27478027, + "step": 2255, + "time_per_iteration": 2.505873203277588 + }, + { + "auxiliary_loss_clip": 0.01113277, + "auxiliary_loss_mlp": 0.01043974, + "balance_loss_clip": 1.03561676, + "balance_loss_mlp": 1.02456725, + "epoch": 0.06546340897220126, + "flos": 41819140275840.0, + "grad_norm": 2.413629148820032, + "language_loss": 0.8462795, + "learning_rate": 3.9868252015066e-06, + "loss": 0.86785203, + "num_input_tokens_seen": 63773305, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.1940918, + "step": 2256, + "time_per_iteration": 2.597013473510742 + }, + { + "auxiliary_loss_clip": 0.01131566, + "auxiliary_loss_mlp": 0.01054397, + "balance_loss_clip": 1.04324961, + "balance_loss_mlp": 1.02923155, + "epoch": 0.06549242644071732, + "flos": 24308696252160.0, + "grad_norm": 2.671885178618856, + "language_loss": 0.95014948, + "learning_rate": 3.9868036537556645e-06, + "loss": 0.97200906, + "num_input_tokens_seen": 63787955, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.25183105, + "step": 2257, + "time_per_iteration": 2.423057794570923 + }, + { + "auxiliary_loss_clip": 0.01122662, + "auxiliary_loss_mlp": 0.01048023, + "balance_loss_clip": 1.03756726, + "balance_loss_mlp": 1.02443123, + "epoch": 0.06552144390923335, + "flos": 35510544132480.0, + "grad_norm": 3.2304934442408926, + "language_loss": 0.75370753, + "learning_rate": 3.986782088456476e-06, + "loss": 0.77541441, + "num_input_tokens_seen": 63804110, + "router_z_loss_clip": 0.8515625, + "router_z_loss_mlp": 0.23583984, + "step": 2258, + "time_per_iteration": 2.4568052291870117 + }, + { + "auxiliary_loss_clip": 0.01131964, + "auxiliary_loss_mlp": 0.01059758, + "balance_loss_clip": 1.04101729, + "balance_loss_mlp": 1.03273368, + "epoch": 0.0655504613777494, + "flos": 11174091482880.0, + "grad_norm": 3.898046514198358, + "language_loss": 0.89651388, + "learning_rate": 3.986760505609224e-06, + "loss": 0.91843104, + "num_input_tokens_seen": 63815595, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.27038574, + "step": 2259, + "time_per_iteration": 2.37646222114563 + }, + { + "auxiliary_loss_clip": 0.01126067, + "auxiliary_loss_mlp": 0.01056606, + "balance_loss_clip": 1.03826797, + "balance_loss_mlp": 1.03028429, + "epoch": 0.06557947884626546, + "flos": 21937011350400.0, + "grad_norm": 2.2851304448360903, + "language_loss": 0.96561438, + "learning_rate": 3.986738905214099e-06, + "loss": 0.98744106, + "num_input_tokens_seen": 63832200, + "router_z_loss_clip": 0.87890625, + "router_z_loss_mlp": 0.26306152, + "step": 2260, + "time_per_iteration": 2.450166940689087 + }, + { + "auxiliary_loss_clip": 0.01020177, + "auxiliary_loss_mlp": 0.010029, + "balance_loss_clip": 1.00582004, + "balance_loss_mlp": 1.00152886, + "epoch": 0.0656084963147815, + "flos": 62434792333440.0, + "grad_norm": 0.7322143008055562, + "language_loss": 0.53094727, + "learning_rate": 3.986717287271291e-06, + "loss": 0.55117804, + "num_input_tokens_seen": 63894035, + "router_z_loss_clip": 0.14355469, + "router_z_loss_mlp": 0.01373291, + "step": 2261, + "time_per_iteration": 3.100433826446533 + }, + { + "auxiliary_loss_clip": 0.01123185, + "auxiliary_loss_mlp": 0.01058169, + "balance_loss_clip": 1.03896594, + "balance_loss_mlp": 1.03246713, + "epoch": 0.06563751378329755, + "flos": 23323936043520.0, + "grad_norm": 2.813504211421243, + "language_loss": 1.02260089, + "learning_rate": 3.986695651780994e-06, + "loss": 1.04441428, + "num_input_tokens_seen": 63909680, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.25720215, + "step": 2262, + "time_per_iteration": 2.4877800941467285 + }, + { + "auxiliary_loss_clip": 0.01017349, + "auxiliary_loss_mlp": 0.01005288, + "balance_loss_clip": 1.00315166, + "balance_loss_mlp": 1.0037142, + "epoch": 0.0656665312518136, + "flos": 68254463664000.0, + "grad_norm": 0.656002102545947, + "language_loss": 0.49698752, + "learning_rate": 3.986673998743396e-06, + "loss": 0.51721394, + "num_input_tokens_seen": 63971455, + "router_z_loss_clip": 0.14160156, + "router_z_loss_mlp": 0.01574707, + "step": 2263, + "time_per_iteration": 3.272783041000366 + }, + { + "auxiliary_loss_clip": 0.01117926, + "auxiliary_loss_mlp": 0.01050923, + "balance_loss_clip": 1.03293121, + "balance_loss_mlp": 1.0266881, + "epoch": 0.06569554872032964, + "flos": 28397479374720.0, + "grad_norm": 1.676695185047641, + "language_loss": 0.70381695, + "learning_rate": 3.986652328158688e-06, + "loss": 0.72550547, + "num_input_tokens_seen": 63997475, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.24255371, + "step": 2264, + "time_per_iteration": 2.899778366088867 + }, + { + "auxiliary_loss_clip": 0.01125557, + "auxiliary_loss_mlp": 0.01057162, + "balance_loss_clip": 1.03714395, + "balance_loss_mlp": 1.03059006, + "epoch": 0.06572456618884569, + "flos": 30035883657600.0, + "grad_norm": 3.9386681576987046, + "language_loss": 0.73958802, + "learning_rate": 3.986630640027065e-06, + "loss": 0.76141518, + "num_input_tokens_seen": 64011780, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.265625, + "step": 2265, + "time_per_iteration": 2.4319653511047363 + }, + { + "auxiliary_loss_clip": 0.01016229, + "auxiliary_loss_mlp": 0.01001391, + "balance_loss_clip": 1.00180542, + "balance_loss_mlp": 0.9999252, + "epoch": 0.06575358365736172, + "flos": 63173170010880.0, + "grad_norm": 0.7503878775878708, + "language_loss": 0.55124432, + "learning_rate": 3.9866089343487155e-06, + "loss": 0.57142055, + "num_input_tokens_seen": 64070275, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01464844, + "step": 2266, + "time_per_iteration": 2.9761664867401123 + }, + { + "auxiliary_loss_clip": 0.01125711, + "auxiliary_loss_mlp": 0.0105237, + "balance_loss_clip": 1.03938246, + "balance_loss_mlp": 1.02644253, + "epoch": 0.06578260112587778, + "flos": 16646273251200.0, + "grad_norm": 3.8028124981515288, + "language_loss": 0.7945385, + "learning_rate": 3.986587211123832e-06, + "loss": 0.81631935, + "num_input_tokens_seen": 64081275, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.25939941, + "step": 2267, + "time_per_iteration": 2.405552625656128 + }, + { + "auxiliary_loss_clip": 0.01124446, + "auxiliary_loss_mlp": 0.01048564, + "balance_loss_clip": 1.03432012, + "balance_loss_mlp": 1.01996541, + "epoch": 0.06581161859439383, + "flos": 33138649762560.0, + "grad_norm": 2.917756982951107, + "language_loss": 0.88681865, + "learning_rate": 3.986565470352606e-06, + "loss": 0.90854877, + "num_input_tokens_seen": 64097745, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.28601074, + "step": 2268, + "time_per_iteration": 2.5498099327087402 + }, + { + "auxiliary_loss_clip": 0.01125953, + "auxiliary_loss_mlp": 0.01048332, + "balance_loss_clip": 1.03851199, + "balance_loss_mlp": 1.02395368, + "epoch": 0.06584063606290987, + "flos": 23432306503680.0, + "grad_norm": 3.114073789541204, + "language_loss": 0.92645949, + "learning_rate": 3.986543712035231e-06, + "loss": 0.94820237, + "num_input_tokens_seen": 64111170, + "router_z_loss_clip": 0.87451172, + "router_z_loss_mlp": 0.24389648, + "step": 2269, + "time_per_iteration": 2.4813132286071777 + }, + { + "auxiliary_loss_clip": 0.0112697, + "auxiliary_loss_mlp": 0.01047826, + "balance_loss_clip": 1.03623819, + "balance_loss_mlp": 1.02221978, + "epoch": 0.06586965353142592, + "flos": 24637368591360.0, + "grad_norm": 2.411036679583238, + "language_loss": 0.90640551, + "learning_rate": 3.986521936171897e-06, + "loss": 0.92815346, + "num_input_tokens_seen": 64129810, + "router_z_loss_clip": 0.90673828, + "router_z_loss_mlp": 0.25610352, + "step": 2270, + "time_per_iteration": 2.462317705154419 + }, + { + "auxiliary_loss_clip": 0.01130389, + "auxiliary_loss_mlp": 0.010462, + "balance_loss_clip": 1.03825593, + "balance_loss_mlp": 1.01741123, + "epoch": 0.06589867099994197, + "flos": 31759265923200.0, + "grad_norm": 1.8866383925767616, + "language_loss": 0.72049356, + "learning_rate": 3.986500142762797e-06, + "loss": 0.74225944, + "num_input_tokens_seen": 64149590, + "router_z_loss_clip": 0.92041016, + "router_z_loss_mlp": 0.28808594, + "step": 2271, + "time_per_iteration": 2.532221555709839 + }, + { + "auxiliary_loss_clip": 0.01118252, + "auxiliary_loss_mlp": 0.01050775, + "balance_loss_clip": 1.03589964, + "balance_loss_mlp": 1.02513313, + "epoch": 0.065927688468458, + "flos": 17010138107520.0, + "grad_norm": 3.270439572783588, + "language_loss": 0.96608281, + "learning_rate": 3.986478331808125e-06, + "loss": 0.98777306, + "num_input_tokens_seen": 64160415, + "router_z_loss_clip": 0.82421875, + "router_z_loss_mlp": 0.25646973, + "step": 2272, + "time_per_iteration": 2.33227801322937 + }, + { + "auxiliary_loss_clip": 0.01121028, + "auxiliary_loss_mlp": 0.01051838, + "balance_loss_clip": 1.03654015, + "balance_loss_mlp": 1.0276866, + "epoch": 0.06595670593697406, + "flos": 13655648033280.0, + "grad_norm": 2.110766838768614, + "language_loss": 0.63722634, + "learning_rate": 3.986456503308072e-06, + "loss": 0.65895498, + "num_input_tokens_seen": 64172850, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.24145508, + "step": 2273, + "time_per_iteration": 2.410076379776001 + }, + { + "auxiliary_loss_clip": 0.01018523, + "auxiliary_loss_mlp": 0.01001477, + "balance_loss_clip": 1.00380254, + "balance_loss_mlp": 0.99993896, + "epoch": 0.06598572340549011, + "flos": 63284123911680.0, + "grad_norm": 0.6766765814367873, + "language_loss": 0.52461654, + "learning_rate": 3.98643465726283e-06, + "loss": 0.54481655, + "num_input_tokens_seen": 64235440, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01538086, + "step": 2274, + "time_per_iteration": 3.065901517868042 + }, + { + "auxiliary_loss_clip": 0.01135568, + "auxiliary_loss_mlp": 0.01061683, + "balance_loss_clip": 1.04089975, + "balance_loss_mlp": 1.03277493, + "epoch": 0.06601474087400615, + "flos": 21681307486080.0, + "grad_norm": 2.494283269923472, + "language_loss": 0.93830639, + "learning_rate": 3.986412793672596e-06, + "loss": 0.96027887, + "num_input_tokens_seen": 64249415, + "router_z_loss_clip": 0.94677734, + "router_z_loss_mlp": 0.28918457, + "step": 2275, + "time_per_iteration": 2.4457173347473145 + }, + { + "auxiliary_loss_clip": 0.01122131, + "auxiliary_loss_mlp": 0.01050352, + "balance_loss_clip": 1.03678823, + "balance_loss_mlp": 1.02752352, + "epoch": 0.0660437583425222, + "flos": 16537763145600.0, + "grad_norm": 2.2562553397127134, + "language_loss": 0.75711715, + "learning_rate": 3.986390912537558e-06, + "loss": 0.77884203, + "num_input_tokens_seen": 64262605, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.22839355, + "step": 2276, + "time_per_iteration": 2.3688266277313232 + }, + { + "auxiliary_loss_clip": 0.01127932, + "auxiliary_loss_mlp": 0.0105977, + "balance_loss_clip": 1.03799748, + "balance_loss_mlp": 1.0352366, + "epoch": 0.06607277581103825, + "flos": 34423872065280.0, + "grad_norm": 2.188674840553553, + "language_loss": 0.86414015, + "learning_rate": 3.986369013857914e-06, + "loss": 0.8860172, + "num_input_tokens_seen": 64286460, + "router_z_loss_clip": 0.89794922, + "router_z_loss_mlp": 0.24536133, + "step": 2277, + "time_per_iteration": 2.848813772201538 + }, + { + "auxiliary_loss_clip": 0.0112753, + "auxiliary_loss_mlp": 0.01050614, + "balance_loss_clip": 1.04055345, + "balance_loss_mlp": 1.02476931, + "epoch": 0.06610179327955429, + "flos": 11173567812480.0, + "grad_norm": 3.8971091733652528, + "language_loss": 0.77248609, + "learning_rate": 3.986347097633853e-06, + "loss": 0.79426754, + "num_input_tokens_seen": 64300125, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.25842285, + "step": 2278, + "time_per_iteration": 2.3454484939575195 + }, + { + "auxiliary_loss_clip": 0.01123988, + "auxiliary_loss_mlp": 0.01048897, + "balance_loss_clip": 1.04014814, + "balance_loss_mlp": 1.02429199, + "epoch": 0.06613081074807034, + "flos": 37625128715520.0, + "grad_norm": 2.823532646383033, + "language_loss": 0.98751307, + "learning_rate": 3.986325163865571e-06, + "loss": 1.00924194, + "num_input_tokens_seen": 64317400, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.24572754, + "step": 2279, + "time_per_iteration": 2.592470645904541 + }, + { + "auxiliary_loss_clip": 0.01137742, + "auxiliary_loss_mlp": 0.01070199, + "balance_loss_clip": 1.0409987, + "balance_loss_mlp": 1.04042101, + "epoch": 0.06615982821658639, + "flos": 29093507706240.0, + "grad_norm": 2.1604521962794885, + "language_loss": 1.01204991, + "learning_rate": 3.986303212553262e-06, + "loss": 1.03412926, + "num_input_tokens_seen": 64336125, + "router_z_loss_clip": 0.96728516, + "router_z_loss_mlp": 0.29797363, + "step": 2280, + "time_per_iteration": 2.495746374130249 + }, + { + "auxiliary_loss_clip": 0.01121241, + "auxiliary_loss_mlp": 0.01045794, + "balance_loss_clip": 1.03676403, + "balance_loss_mlp": 1.02065277, + "epoch": 0.06618884568510243, + "flos": 43066551709440.0, + "grad_norm": 2.2185240729007267, + "language_loss": 0.79025519, + "learning_rate": 3.986281243697119e-06, + "loss": 0.81192553, + "num_input_tokens_seen": 64354865, + "router_z_loss_clip": 0.84521484, + "router_z_loss_mlp": 0.25134277, + "step": 2281, + "time_per_iteration": 2.6245133876800537 + }, + { + "auxiliary_loss_clip": 0.01117967, + "auxiliary_loss_mlp": 0.01050965, + "balance_loss_clip": 1.03535569, + "balance_loss_mlp": 1.02713561, + "epoch": 0.06621786315361848, + "flos": 19275233028480.0, + "grad_norm": 2.888996999894517, + "language_loss": 0.82819378, + "learning_rate": 3.986259257297337e-06, + "loss": 0.84988314, + "num_input_tokens_seen": 64368545, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.23828125, + "step": 2282, + "time_per_iteration": 2.408390998840332 + }, + { + "auxiliary_loss_clip": 0.01017396, + "auxiliary_loss_mlp": 0.01005288, + "balance_loss_clip": 1.00285792, + "balance_loss_mlp": 1.00377989, + "epoch": 0.06624688062213452, + "flos": 74769746256000.0, + "grad_norm": 0.6510110659150371, + "language_loss": 0.45509726, + "learning_rate": 3.9862372533541085e-06, + "loss": 0.47532409, + "num_input_tokens_seen": 64430150, + "router_z_loss_clip": 0.14550781, + "router_z_loss_mlp": 0.01507568, + "step": 2283, + "time_per_iteration": 3.081639051437378 + }, + { + "auxiliary_loss_clip": 0.01124839, + "auxiliary_loss_mlp": 0.01050446, + "balance_loss_clip": 1.03770292, + "balance_loss_mlp": 1.02511406, + "epoch": 0.06627589809065057, + "flos": 36090905529600.0, + "grad_norm": 2.486467283917705, + "language_loss": 0.89283317, + "learning_rate": 3.986215231867629e-06, + "loss": 0.91458601, + "num_input_tokens_seen": 64448820, + "router_z_loss_clip": 0.87109375, + "router_z_loss_mlp": 0.25341797, + "step": 2284, + "time_per_iteration": 6.213549613952637 + }, + { + "auxiliary_loss_clip": 0.01124017, + "auxiliary_loss_mlp": 0.01062568, + "balance_loss_clip": 1.03859377, + "balance_loss_mlp": 1.03607988, + "epoch": 0.06630491555916662, + "flos": 33175064177280.0, + "grad_norm": 2.477172321627193, + "language_loss": 0.95391244, + "learning_rate": 3.986193192838093e-06, + "loss": 0.97577828, + "num_input_tokens_seen": 64464680, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.26501465, + "step": 2285, + "time_per_iteration": 4.825457811355591 + }, + { + "auxiliary_loss_clip": 0.01126131, + "auxiliary_loss_mlp": 0.01051994, + "balance_loss_clip": 1.03585172, + "balance_loss_mlp": 1.02377748, + "epoch": 0.06633393302768266, + "flos": 22340572289280.0, + "grad_norm": 2.5400279526833884, + "language_loss": 0.98823869, + "learning_rate": 3.986171136265695e-06, + "loss": 1.0100199, + "num_input_tokens_seen": 64481345, + "router_z_loss_clip": 0.90332031, + "router_z_loss_mlp": 0.28198242, + "step": 2286, + "time_per_iteration": 2.4615373611450195 + }, + { + "auxiliary_loss_clip": 0.01122858, + "auxiliary_loss_mlp": 0.01057764, + "balance_loss_clip": 1.03751171, + "balance_loss_mlp": 1.03399324, + "epoch": 0.06636295049619871, + "flos": 21170283782400.0, + "grad_norm": 1.940333322231702, + "language_loss": 0.73203915, + "learning_rate": 3.98614906215063e-06, + "loss": 0.75384533, + "num_input_tokens_seen": 64495465, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.23779297, + "step": 2287, + "time_per_iteration": 2.417694330215454 + }, + { + "auxiliary_loss_clip": 0.01127999, + "auxiliary_loss_mlp": 0.01058966, + "balance_loss_clip": 1.03999138, + "balance_loss_mlp": 1.03307366, + "epoch": 0.06639196796471476, + "flos": 39045884382720.0, + "grad_norm": 2.5945160416702024, + "language_loss": 0.9880178, + "learning_rate": 3.986126970493092e-06, + "loss": 1.00988746, + "num_input_tokens_seen": 64510355, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.25915527, + "step": 2288, + "time_per_iteration": 2.8285863399505615 + }, + { + "auxiliary_loss_clip": 0.0101709, + "auxiliary_loss_mlp": 0.01002798, + "balance_loss_clip": 1.00271726, + "balance_loss_mlp": 1.00142109, + "epoch": 0.0664209854332308, + "flos": 59303746160640.0, + "grad_norm": 0.703020319315091, + "language_loss": 0.52830178, + "learning_rate": 3.986104861293277e-06, + "loss": 0.54850066, + "num_input_tokens_seen": 64569395, + "router_z_loss_clip": 0.14355469, + "router_z_loss_mlp": 0.01379395, + "step": 2289, + "time_per_iteration": 5.400583505630493 + }, + { + "auxiliary_loss_clip": 0.01122067, + "auxiliary_loss_mlp": 0.01050343, + "balance_loss_clip": 1.03741574, + "balance_loss_mlp": 1.02431989, + "epoch": 0.06645000290174685, + "flos": 40142715655680.0, + "grad_norm": 2.9871417012668044, + "language_loss": 0.98189187, + "learning_rate": 3.986082734551381e-06, + "loss": 1.00361598, + "num_input_tokens_seen": 64585380, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.26000977, + "step": 2290, + "time_per_iteration": 2.600491762161255 + }, + { + "auxiliary_loss_clip": 0.0101653, + "auxiliary_loss_mlp": 0.01001868, + "balance_loss_clip": 1.00206709, + "balance_loss_mlp": 1.00040162, + "epoch": 0.0664790203702629, + "flos": 62987851180800.0, + "grad_norm": 0.637303342647678, + "language_loss": 0.48027834, + "learning_rate": 3.986060590267598e-06, + "loss": 0.50046229, + "num_input_tokens_seen": 64647895, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01464844, + "step": 2291, + "time_per_iteration": 3.1360814571380615 + }, + { + "auxiliary_loss_clip": 0.01128636, + "auxiliary_loss_mlp": 0.01061039, + "balance_loss_clip": 1.03705144, + "balance_loss_mlp": 1.03403783, + "epoch": 0.06650803783877894, + "flos": 28587476327040.0, + "grad_norm": 2.7648818366839416, + "language_loss": 1.07927513, + "learning_rate": 3.986038428442125e-06, + "loss": 1.10117185, + "num_input_tokens_seen": 64670735, + "router_z_loss_clip": 0.91503906, + "router_z_loss_mlp": 0.2701416, + "step": 2292, + "time_per_iteration": 4.87772536277771 + }, + { + "auxiliary_loss_clip": 0.01015773, + "auxiliary_loss_mlp": 0.01001644, + "balance_loss_clip": 1.00128961, + "balance_loss_mlp": 1.00020778, + "epoch": 0.06653705530729499, + "flos": 74773795973760.0, + "grad_norm": 0.7880915759481435, + "language_loss": 0.49416196, + "learning_rate": 3.986016249075156e-06, + "loss": 0.51433611, + "num_input_tokens_seen": 64739680, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01434326, + "step": 2293, + "time_per_iteration": 3.264768600463867 + }, + { + "auxiliary_loss_clip": 0.0101598, + "auxiliary_loss_mlp": 0.01001302, + "balance_loss_clip": 1.00163889, + "balance_loss_mlp": 0.99980557, + "epoch": 0.06656607277581104, + "flos": 73093705660800.0, + "grad_norm": 0.6556009964936712, + "language_loss": 0.4990969, + "learning_rate": 3.985994052166888e-06, + "loss": 0.5192697, + "num_input_tokens_seen": 64801605, + "router_z_loss_clip": 0.14257812, + "router_z_loss_mlp": 0.01495361, + "step": 2294, + "time_per_iteration": 3.0511884689331055 + }, + { + "auxiliary_loss_clip": 0.01015664, + "auxiliary_loss_mlp": 0.01001378, + "balance_loss_clip": 1.00132084, + "balance_loss_mlp": 0.99995375, + "epoch": 0.06659509024432708, + "flos": 74757981127680.0, + "grad_norm": 0.6752601142511192, + "language_loss": 0.48796669, + "learning_rate": 3.985971837717517e-06, + "loss": 0.50813711, + "num_input_tokens_seen": 64862580, + "router_z_loss_clip": 0.14355469, + "router_z_loss_mlp": 0.01422119, + "step": 2295, + "time_per_iteration": 3.022172451019287 + }, + { + "auxiliary_loss_clip": 0.01122225, + "auxiliary_loss_mlp": 0.01044104, + "balance_loss_clip": 1.0398953, + "balance_loss_mlp": 1.02228892, + "epoch": 0.06662410771284313, + "flos": 16498520910720.0, + "grad_norm": 3.6466107522477387, + "language_loss": 0.73763359, + "learning_rate": 3.985949605727239e-06, + "loss": 0.75929689, + "num_input_tokens_seen": 64874330, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.21850586, + "step": 2296, + "time_per_iteration": 2.3845465183258057 + }, + { + "auxiliary_loss_clip": 0.01127874, + "auxiliary_loss_mlp": 0.01047088, + "balance_loss_clip": 1.03974044, + "balance_loss_mlp": 1.0239495, + "epoch": 0.06665312518135917, + "flos": 74734939630080.0, + "grad_norm": 2.186420083627713, + "language_loss": 0.72856408, + "learning_rate": 3.9859273561962516e-06, + "loss": 0.7503137, + "num_input_tokens_seen": 64898580, + "router_z_loss_clip": 0.88134766, + "router_z_loss_mlp": 0.23132324, + "step": 2297, + "time_per_iteration": 2.8373138904571533 + }, + { + "auxiliary_loss_clip": 0.01116955, + "auxiliary_loss_mlp": 0.01047935, + "balance_loss_clip": 1.03549361, + "balance_loss_mlp": 1.02567863, + "epoch": 0.06668214264987522, + "flos": 31530969342720.0, + "grad_norm": 4.0385729626159765, + "language_loss": 0.78102815, + "learning_rate": 3.985905089124749e-06, + "loss": 0.80267704, + "num_input_tokens_seen": 64912900, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.22253418, + "step": 2298, + "time_per_iteration": 2.412980794906616 + }, + { + "auxiliary_loss_clip": 0.01016817, + "auxiliary_loss_mlp": 0.01003778, + "balance_loss_clip": 1.00259721, + "balance_loss_mlp": 1.00234759, + "epoch": 0.06671116011839127, + "flos": 60686656047360.0, + "grad_norm": 0.7505196210103791, + "language_loss": 0.50253081, + "learning_rate": 3.9858828045129285e-06, + "loss": 0.52273679, + "num_input_tokens_seen": 64970595, + "router_z_loss_clip": 0.14257812, + "router_z_loss_mlp": 0.01428223, + "step": 2299, + "time_per_iteration": 3.0008704662323 + }, + { + "auxiliary_loss_clip": 0.01127024, + "auxiliary_loss_mlp": 0.01049816, + "balance_loss_clip": 1.03852844, + "balance_loss_mlp": 1.02548575, + "epoch": 0.06674017758690731, + "flos": 21536208408960.0, + "grad_norm": 2.180580728471918, + "language_loss": 0.98576754, + "learning_rate": 3.985860502360988e-06, + "loss": 1.00753593, + "num_input_tokens_seen": 64985405, + "router_z_loss_clip": 0.88476562, + "router_z_loss_mlp": 0.24328613, + "step": 2300, + "time_per_iteration": 2.386322498321533 + }, + { + "auxiliary_loss_clip": 0.01130461, + "auxiliary_loss_mlp": 0.01055163, + "balance_loss_clip": 1.03764141, + "balance_loss_mlp": 1.02885342, + "epoch": 0.06676919505542336, + "flos": 28834801464960.0, + "grad_norm": 2.0332557801460185, + "language_loss": 1.08443141, + "learning_rate": 3.9858381826691245e-06, + "loss": 1.10628772, + "num_input_tokens_seen": 65002615, + "router_z_loss_clip": 0.9296875, + "router_z_loss_mlp": 0.26281738, + "step": 2301, + "time_per_iteration": 2.471649169921875 + }, + { + "auxiliary_loss_clip": 0.01128834, + "auxiliary_loss_mlp": 0.01055906, + "balance_loss_clip": 1.03822732, + "balance_loss_mlp": 1.02722466, + "epoch": 0.06679821252393942, + "flos": 74734695250560.0, + "grad_norm": 1.8286137820700599, + "language_loss": 0.87063086, + "learning_rate": 3.985815845437535e-06, + "loss": 0.89247823, + "num_input_tokens_seen": 65028225, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.28625488, + "step": 2302, + "time_per_iteration": 2.7955427169799805 + }, + { + "auxiliary_loss_clip": 0.0113084, + "auxiliary_loss_mlp": 0.01052327, + "balance_loss_clip": 1.04264867, + "balance_loss_mlp": 1.02790117, + "epoch": 0.06682722999245545, + "flos": 30109585271040.0, + "grad_norm": 2.1536916464668714, + "language_loss": 0.8529377, + "learning_rate": 3.985793490666415e-06, + "loss": 0.87476933, + "num_input_tokens_seen": 65042385, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.24438477, + "step": 2303, + "time_per_iteration": 2.51615309715271 + }, + { + "auxiliary_loss_clip": 0.01126856, + "auxiliary_loss_mlp": 0.01058093, + "balance_loss_clip": 1.03568316, + "balance_loss_mlp": 1.03062749, + "epoch": 0.0668562474609715, + "flos": 35071162272000.0, + "grad_norm": 1.937035435313975, + "language_loss": 0.79492033, + "learning_rate": 3.9857711183559636e-06, + "loss": 0.81676984, + "num_input_tokens_seen": 65059690, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.2746582, + "step": 2304, + "time_per_iteration": 2.535337209701538 + }, + { + "auxiliary_loss_clip": 0.01126978, + "auxiliary_loss_mlp": 0.01052835, + "balance_loss_clip": 1.03611588, + "balance_loss_mlp": 1.02901757, + "epoch": 0.06688526492948756, + "flos": 19383638400000.0, + "grad_norm": 2.7024282559380994, + "language_loss": 0.94973743, + "learning_rate": 3.985748728506379e-06, + "loss": 0.97153556, + "num_input_tokens_seen": 65074565, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.23828125, + "step": 2305, + "time_per_iteration": 2.4417057037353516 + }, + { + "auxiliary_loss_clip": 0.01018038, + "auxiliary_loss_mlp": 0.01007268, + "balance_loss_clip": 1.00339913, + "balance_loss_mlp": 1.00581396, + "epoch": 0.0669142823980036, + "flos": 74767092992640.0, + "grad_norm": 0.7203652214865878, + "language_loss": 0.49852574, + "learning_rate": 3.985726321117857e-06, + "loss": 0.5187788, + "num_input_tokens_seen": 65130105, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01452637, + "step": 2306, + "time_per_iteration": 3.0661964416503906 + }, + { + "auxiliary_loss_clip": 0.01136586, + "auxiliary_loss_mlp": 0.01053212, + "balance_loss_clip": 1.04139757, + "balance_loss_mlp": 1.02635443, + "epoch": 0.06694329986651965, + "flos": 31165673120640.0, + "grad_norm": 2.7004088846909813, + "language_loss": 0.99300611, + "learning_rate": 3.985703896190597e-06, + "loss": 1.01490402, + "num_input_tokens_seen": 65144705, + "router_z_loss_clip": 0.95263672, + "router_z_loss_mlp": 0.26867676, + "step": 2307, + "time_per_iteration": 2.4904699325561523 + }, + { + "auxiliary_loss_clip": 0.01113565, + "auxiliary_loss_mlp": 0.0104884, + "balance_loss_clip": 1.03549862, + "balance_loss_mlp": 1.02735853, + "epoch": 0.0669723173350357, + "flos": 39048886759680.0, + "grad_norm": 3.547409975339654, + "language_loss": 0.78922522, + "learning_rate": 3.985681453724797e-06, + "loss": 0.81084931, + "num_input_tokens_seen": 65160610, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.21484375, + "step": 2308, + "time_per_iteration": 2.537407636642456 + }, + { + "auxiliary_loss_clip": 0.01127404, + "auxiliary_loss_mlp": 0.0105416, + "balance_loss_clip": 1.03900182, + "balance_loss_mlp": 1.03042603, + "epoch": 0.06700133480355173, + "flos": 26023420500480.0, + "grad_norm": 2.3237999484398633, + "language_loss": 0.71779847, + "learning_rate": 3.985658993720655e-06, + "loss": 0.73961413, + "num_input_tokens_seen": 65177205, + "router_z_loss_clip": 0.88378906, + "router_z_loss_mlp": 0.23742676, + "step": 2309, + "time_per_iteration": 2.4445748329162598 + }, + { + "auxiliary_loss_clip": 0.01123324, + "auxiliary_loss_mlp": 0.01048189, + "balance_loss_clip": 1.03632331, + "balance_loss_mlp": 1.02336943, + "epoch": 0.06703035227206779, + "flos": 35582325621120.0, + "grad_norm": 2.262343353947789, + "language_loss": 0.850784, + "learning_rate": 3.9856365161783685e-06, + "loss": 0.87249911, + "num_input_tokens_seen": 65193585, + "router_z_loss_clip": 0.86962891, + "router_z_loss_mlp": 0.24829102, + "step": 2310, + "time_per_iteration": 2.4936015605926514 + }, + { + "auxiliary_loss_clip": 0.01017003, + "auxiliary_loss_mlp": 0.01002343, + "balance_loss_clip": 1.00223136, + "balance_loss_mlp": 1.00078106, + "epoch": 0.06705936974058384, + "flos": 68248074885120.0, + "grad_norm": 0.6315251739962935, + "language_loss": 0.49810937, + "learning_rate": 3.985614021098138e-06, + "loss": 0.5183028, + "num_input_tokens_seen": 65257105, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01556396, + "step": 2311, + "time_per_iteration": 3.082444906234741 + }, + { + "auxiliary_loss_clip": 0.01124345, + "auxiliary_loss_mlp": 0.01050062, + "balance_loss_clip": 1.03922701, + "balance_loss_mlp": 1.02494454, + "epoch": 0.06708838720909988, + "flos": 56853473921280.0, + "grad_norm": 1.719578232156557, + "language_loss": 0.84898049, + "learning_rate": 3.98559150848016e-06, + "loss": 0.87072444, + "num_input_tokens_seen": 65276950, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.25146484, + "step": 2312, + "time_per_iteration": 2.9661736488342285 + }, + { + "auxiliary_loss_clip": 0.01114823, + "auxiliary_loss_mlp": 0.01050862, + "balance_loss_clip": 1.03432691, + "balance_loss_mlp": 1.02750945, + "epoch": 0.06711740467761593, + "flos": 18802508952960.0, + "grad_norm": 2.199403491657279, + "language_loss": 0.72262037, + "learning_rate": 3.985568978324634e-06, + "loss": 0.74427718, + "num_input_tokens_seen": 65292740, + "router_z_loss_clip": 0.80419922, + "router_z_loss_mlp": 0.23352051, + "step": 2313, + "time_per_iteration": 2.391239643096924 + }, + { + "auxiliary_loss_clip": 0.01138527, + "auxiliary_loss_mlp": 0.01059587, + "balance_loss_clip": 1.04019344, + "balance_loss_mlp": 1.03128719, + "epoch": 0.06714642214613196, + "flos": 33434294088960.0, + "grad_norm": 2.9552545115490902, + "language_loss": 0.97756529, + "learning_rate": 3.98554643063176e-06, + "loss": 0.99954647, + "num_input_tokens_seen": 65309145, + "router_z_loss_clip": 0.98291016, + "router_z_loss_mlp": 0.28308105, + "step": 2314, + "time_per_iteration": 2.5669405460357666 + }, + { + "auxiliary_loss_clip": 0.01121944, + "auxiliary_loss_mlp": 0.01048965, + "balance_loss_clip": 1.03707457, + "balance_loss_mlp": 1.02416992, + "epoch": 0.06717543961464802, + "flos": 18144151845120.0, + "grad_norm": 2.670253412065317, + "language_loss": 0.83137286, + "learning_rate": 3.985523865401736e-06, + "loss": 0.85308194, + "num_input_tokens_seen": 65324540, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.2479248, + "step": 2315, + "time_per_iteration": 2.3915886878967285 + }, + { + "auxiliary_loss_clip": 0.01130556, + "auxiliary_loss_mlp": 0.010544, + "balance_loss_clip": 1.03927302, + "balance_loss_mlp": 1.02797103, + "epoch": 0.06720445708316407, + "flos": 11612391091200.0, + "grad_norm": 2.3032473898177117, + "language_loss": 0.85934591, + "learning_rate": 3.985501282634762e-06, + "loss": 0.88119555, + "num_input_tokens_seen": 65336085, + "router_z_loss_clip": 0.91259766, + "router_z_loss_mlp": 0.26416016, + "step": 2316, + "time_per_iteration": 2.4050979614257812 + }, + { + "auxiliary_loss_clip": 0.01115439, + "auxiliary_loss_mlp": 0.01055013, + "balance_loss_clip": 1.03705192, + "balance_loss_mlp": 1.0326854, + "epoch": 0.0672334745516801, + "flos": 13508943033600.0, + "grad_norm": 2.1739607220943347, + "language_loss": 0.73203266, + "learning_rate": 3.985478682331037e-06, + "loss": 0.75373709, + "num_input_tokens_seen": 65348915, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.22314453, + "step": 2317, + "time_per_iteration": 2.3387746810913086 + }, + { + "auxiliary_loss_clip": 0.01120045, + "auxiliary_loss_mlp": 0.01048241, + "balance_loss_clip": 1.03716135, + "balance_loss_mlp": 1.02497125, + "epoch": 0.06726249202019616, + "flos": 48900119627520.0, + "grad_norm": 1.8313000684256007, + "language_loss": 0.79371393, + "learning_rate": 3.985456064490761e-06, + "loss": 0.81539673, + "num_input_tokens_seen": 65369650, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.23291016, + "step": 2318, + "time_per_iteration": 2.702723741531372 + }, + { + "auxiliary_loss_clip": 0.01117538, + "auxiliary_loss_mlp": 0.01049641, + "balance_loss_clip": 1.03521776, + "balance_loss_mlp": 1.02725339, + "epoch": 0.06729150948871221, + "flos": 42660407329920.0, + "grad_norm": 2.0471729309240034, + "language_loss": 0.94037724, + "learning_rate": 3.9854334291141335e-06, + "loss": 0.96204901, + "num_input_tokens_seen": 65388550, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.22387695, + "step": 2319, + "time_per_iteration": 2.5378496646881104 + }, + { + "auxiliary_loss_clip": 0.01129414, + "auxiliary_loss_mlp": 0.010526, + "balance_loss_clip": 1.03999853, + "balance_loss_mlp": 1.02781665, + "epoch": 0.06732052695722825, + "flos": 25110441780480.0, + "grad_norm": 2.6932231581352686, + "language_loss": 0.87333977, + "learning_rate": 3.985410776201355e-06, + "loss": 0.89515984, + "num_input_tokens_seen": 65401865, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.24780273, + "step": 2320, + "time_per_iteration": 2.487584352493286 + }, + { + "auxiliary_loss_clip": 0.0101937, + "auxiliary_loss_mlp": 0.01014467, + "balance_loss_clip": 1.00436544, + "balance_loss_mlp": 1.01299453, + "epoch": 0.0673495444257443, + "flos": 74778229716480.0, + "grad_norm": 0.6844756163576237, + "language_loss": 0.48953247, + "learning_rate": 3.985388105752625e-06, + "loss": 0.50987083, + "num_input_tokens_seen": 65463570, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01470947, + "step": 2321, + "time_per_iteration": 3.096061944961548 + }, + { + "auxiliary_loss_clip": 0.01121752, + "auxiliary_loss_mlp": 0.01051219, + "balance_loss_clip": 1.03744054, + "balance_loss_mlp": 1.02811623, + "epoch": 0.06737856189426035, + "flos": 42184087384320.0, + "grad_norm": 3.039282755530393, + "language_loss": 0.80986047, + "learning_rate": 3.985365417768144e-06, + "loss": 0.83159018, + "num_input_tokens_seen": 65479125, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.23095703, + "step": 2322, + "time_per_iteration": 2.6313769817352295 + }, + { + "auxiliary_loss_clip": 0.01018449, + "auxiliary_loss_mlp": 0.0100389, + "balance_loss_clip": 1.00330758, + "balance_loss_mlp": 1.00230455, + "epoch": 0.06740757936277639, + "flos": 72248144509440.0, + "grad_norm": 0.6814829178650071, + "language_loss": 0.47311604, + "learning_rate": 3.985342712248112e-06, + "loss": 0.49333948, + "num_input_tokens_seen": 65541450, + "router_z_loss_clip": 0.15136719, + "router_z_loss_mlp": 0.01586914, + "step": 2323, + "time_per_iteration": 3.0379273891448975 + }, + { + "auxiliary_loss_clip": 0.01123966, + "auxiliary_loss_mlp": 0.01055645, + "balance_loss_clip": 1.03729546, + "balance_loss_mlp": 1.02989578, + "epoch": 0.06743659683129244, + "flos": 16136192154240.0, + "grad_norm": 2.8723623940686993, + "language_loss": 0.77742398, + "learning_rate": 3.985319989192729e-06, + "loss": 0.79922009, + "num_input_tokens_seen": 65555405, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.25769043, + "step": 2324, + "time_per_iteration": 2.4136970043182373 + }, + { + "auxiliary_loss_clip": 0.01121785, + "auxiliary_loss_mlp": 0.01043937, + "balance_loss_clip": 1.03665113, + "balance_loss_mlp": 1.02153718, + "epoch": 0.06746561429980849, + "flos": 17852173211520.0, + "grad_norm": 2.7264737701008457, + "language_loss": 0.82563186, + "learning_rate": 3.985297248602197e-06, + "loss": 0.84728903, + "num_input_tokens_seen": 65569855, + "router_z_loss_clip": 0.85107422, + "router_z_loss_mlp": 0.22424316, + "step": 2325, + "time_per_iteration": 2.7282168865203857 + }, + { + "auxiliary_loss_clip": 0.01018892, + "auxiliary_loss_mlp": 0.01002213, + "balance_loss_clip": 1.00383282, + "balance_loss_mlp": 1.00075817, + "epoch": 0.06749463176832453, + "flos": 74741256720000.0, + "grad_norm": 0.7022207294429911, + "language_loss": 0.47863996, + "learning_rate": 3.985274490476717e-06, + "loss": 0.498851, + "num_input_tokens_seen": 65631380, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01452637, + "step": 2326, + "time_per_iteration": 3.0963544845581055 + }, + { + "auxiliary_loss_clip": 0.01121576, + "auxiliary_loss_mlp": 0.01058253, + "balance_loss_clip": 1.03767538, + "balance_loss_mlp": 1.03370821, + "epoch": 0.06752364923684058, + "flos": 15737239514880.0, + "grad_norm": 2.744142564972516, + "language_loss": 0.87453443, + "learning_rate": 3.985251714816489e-06, + "loss": 0.89633262, + "num_input_tokens_seen": 65648075, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.2454834, + "step": 2327, + "time_per_iteration": 2.406663179397583 + }, + { + "auxiliary_loss_clip": 0.01128251, + "auxiliary_loss_mlp": 0.01058343, + "balance_loss_clip": 1.0407021, + "balance_loss_mlp": 1.03222406, + "epoch": 0.06755266670535663, + "flos": 20076070861440.0, + "grad_norm": 2.664843421730258, + "language_loss": 0.91267991, + "learning_rate": 3.985228921621714e-06, + "loss": 0.93454587, + "num_input_tokens_seen": 65663485, + "router_z_loss_clip": 0.87548828, + "router_z_loss_mlp": 0.26086426, + "step": 2328, + "time_per_iteration": 2.409417152404785 + }, + { + "auxiliary_loss_clip": 0.01129724, + "auxiliary_loss_mlp": 0.01064578, + "balance_loss_clip": 1.04180455, + "balance_loss_mlp": 1.03861475, + "epoch": 0.06758168417387267, + "flos": 33102933575040.0, + "grad_norm": 3.4017650056469373, + "language_loss": 1.03746498, + "learning_rate": 3.985206110892594e-06, + "loss": 1.05940795, + "num_input_tokens_seen": 65682665, + "router_z_loss_clip": 0.87939453, + "router_z_loss_mlp": 0.25927734, + "step": 2329, + "time_per_iteration": 2.467141628265381 + }, + { + "auxiliary_loss_clip": 0.01018153, + "auxiliary_loss_mlp": 0.01010691, + "balance_loss_clip": 1.00331736, + "balance_loss_mlp": 1.00923705, + "epoch": 0.06761070164238872, + "flos": 60358437555840.0, + "grad_norm": 0.7543413445503068, + "language_loss": 0.53330559, + "learning_rate": 3.985183282629331e-06, + "loss": 0.55359405, + "num_input_tokens_seen": 65737030, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01452637, + "step": 2330, + "time_per_iteration": 2.955308198928833 + }, + { + "auxiliary_loss_clip": 0.01120577, + "auxiliary_loss_mlp": 0.01050427, + "balance_loss_clip": 1.03690457, + "balance_loss_mlp": 1.02707362, + "epoch": 0.06763971911090476, + "flos": 19603311874560.0, + "grad_norm": 2.617980168484716, + "language_loss": 0.87826347, + "learning_rate": 3.985160436832126e-06, + "loss": 0.89997357, + "num_input_tokens_seen": 65751585, + "router_z_loss_clip": 0.83740234, + "router_z_loss_mlp": 0.23376465, + "step": 2331, + "time_per_iteration": 2.4954447746276855 + }, + { + "auxiliary_loss_clip": 0.01015977, + "auxiliary_loss_mlp": 0.01009035, + "balance_loss_clip": 1.00116372, + "balance_loss_mlp": 1.00761032, + "epoch": 0.06766873657942081, + "flos": 74774738580480.0, + "grad_norm": 0.6175900258717762, + "language_loss": 0.49344578, + "learning_rate": 3.985137573501179e-06, + "loss": 0.51369596, + "num_input_tokens_seen": 65818270, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01422119, + "step": 2332, + "time_per_iteration": 3.241398811340332 + }, + { + "auxiliary_loss_clip": 0.01016624, + "auxiliary_loss_mlp": 0.01005792, + "balance_loss_clip": 1.00181174, + "balance_loss_mlp": 1.00436699, + "epoch": 0.06769775404793686, + "flos": 74773830885120.0, + "grad_norm": 0.6925024803896689, + "language_loss": 0.52455479, + "learning_rate": 3.985114692636695e-06, + "loss": 0.54477894, + "num_input_tokens_seen": 65877890, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01422119, + "step": 2333, + "time_per_iteration": 3.172290325164795 + }, + { + "auxiliary_loss_clip": 0.0113003, + "auxiliary_loss_mlp": 0.01048786, + "balance_loss_clip": 1.03954792, + "balance_loss_mlp": 1.0243721, + "epoch": 0.0677267715164529, + "flos": 36424535281920.0, + "grad_norm": 2.7783463807854156, + "language_loss": 0.9343828, + "learning_rate": 3.985091794238875e-06, + "loss": 0.95617092, + "num_input_tokens_seen": 65899115, + "router_z_loss_clip": 0.90478516, + "router_z_loss_mlp": 0.24414062, + "step": 2334, + "time_per_iteration": 2.5209054946899414 + }, + { + "auxiliary_loss_clip": 0.011218, + "auxiliary_loss_mlp": 0.01047273, + "balance_loss_clip": 1.03876495, + "balance_loss_mlp": 1.02296031, + "epoch": 0.06775578898496895, + "flos": 32954727386880.0, + "grad_norm": 1.828931283452847, + "language_loss": 0.78823411, + "learning_rate": 3.98506887830792e-06, + "loss": 0.80992478, + "num_input_tokens_seen": 65919125, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.24291992, + "step": 2335, + "time_per_iteration": 2.551335096359253 + }, + { + "auxiliary_loss_clip": 0.01023181, + "auxiliary_loss_mlp": 0.01004359, + "balance_loss_clip": 1.00845301, + "balance_loss_mlp": 1.00297654, + "epoch": 0.067784806453485, + "flos": 67430444688000.0, + "grad_norm": 0.7157966251556115, + "language_loss": 0.53084409, + "learning_rate": 3.985045944844034e-06, + "loss": 0.55111957, + "num_input_tokens_seen": 65974245, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01385498, + "step": 2336, + "time_per_iteration": 2.9178521633148193 + }, + { + "auxiliary_loss_clip": 0.01022141, + "auxiliary_loss_mlp": 0.01003458, + "balance_loss_clip": 1.00738192, + "balance_loss_mlp": 1.00197387, + "epoch": 0.06781382392200104, + "flos": 66996578822400.0, + "grad_norm": 0.5849313736997434, + "language_loss": 0.46587282, + "learning_rate": 3.985022993847419e-06, + "loss": 0.48612881, + "num_input_tokens_seen": 66041205, + "router_z_loss_clip": 0.14746094, + "router_z_loss_mlp": 0.01483154, + "step": 2337, + "time_per_iteration": 3.1844418048858643 + }, + { + "auxiliary_loss_clip": 0.01114903, + "auxiliary_loss_mlp": 0.01036842, + "balance_loss_clip": 1.03408921, + "balance_loss_mlp": 1.01587856, + "epoch": 0.06784284139051709, + "flos": 20984476193280.0, + "grad_norm": 2.918330674325158, + "language_loss": 0.98684037, + "learning_rate": 3.985000025318277e-06, + "loss": 1.00835776, + "num_input_tokens_seen": 66052845, + "router_z_loss_clip": 0.80761719, + "router_z_loss_mlp": 0.20977783, + "step": 2338, + "time_per_iteration": 2.629704475402832 + }, + { + "auxiliary_loss_clip": 0.01124449, + "auxiliary_loss_mlp": 0.01047517, + "balance_loss_clip": 1.03512585, + "balance_loss_mlp": 1.0241046, + "epoch": 0.06787185885903314, + "flos": 33319150824960.0, + "grad_norm": 2.4243801230575706, + "language_loss": 0.7581467, + "learning_rate": 3.984977039256812e-06, + "loss": 0.77986634, + "num_input_tokens_seen": 66067730, + "router_z_loss_clip": 0.89379883, + "router_z_loss_mlp": 0.23413086, + "step": 2339, + "time_per_iteration": 2.513176202774048 + }, + { + "auxiliary_loss_clip": 0.01017144, + "auxiliary_loss_mlp": 0.01005484, + "balance_loss_clip": 1.00277972, + "balance_loss_mlp": 1.00407767, + "epoch": 0.06790087632754918, + "flos": 60288157255680.0, + "grad_norm": 0.6720120433413053, + "language_loss": 0.50833517, + "learning_rate": 3.984954035663227e-06, + "loss": 0.52856147, + "num_input_tokens_seen": 66131640, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01403809, + "step": 2340, + "time_per_iteration": 3.0069639682769775 + }, + { + "auxiliary_loss_clip": 0.01123735, + "auxiliary_loss_mlp": 0.01052863, + "balance_loss_clip": 1.03898096, + "balance_loss_mlp": 1.0277338, + "epoch": 0.06792989379606523, + "flos": 22923761506560.0, + "grad_norm": 2.717685020673443, + "language_loss": 0.86407757, + "learning_rate": 3.984931014537724e-06, + "loss": 0.88584358, + "num_input_tokens_seen": 66145470, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.25085449, + "step": 2341, + "time_per_iteration": 2.4283719062805176 + }, + { + "auxiliary_loss_clip": 0.01116922, + "auxiliary_loss_mlp": 0.01047975, + "balance_loss_clip": 1.03576541, + "balance_loss_mlp": 1.02546859, + "epoch": 0.06795891126458128, + "flos": 25732733587200.0, + "grad_norm": 2.2367070403633167, + "language_loss": 0.89696199, + "learning_rate": 3.984907975880508e-06, + "loss": 0.91861093, + "num_input_tokens_seen": 66163045, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.22521973, + "step": 2342, + "time_per_iteration": 2.437633514404297 + }, + { + "auxiliary_loss_clip": 0.01023755, + "auxiliary_loss_mlp": 0.01005604, + "balance_loss_clip": 1.00910974, + "balance_loss_mlp": 1.00435829, + "epoch": 0.06798792873309732, + "flos": 61680877234560.0, + "grad_norm": 0.6567023244313289, + "language_loss": 0.4603588, + "learning_rate": 3.984884919691781e-06, + "loss": 0.48065239, + "num_input_tokens_seen": 66222835, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01245117, + "step": 2343, + "time_per_iteration": 3.0382883548736572 + }, + { + "auxiliary_loss_clip": 0.01136162, + "auxiliary_loss_mlp": 0.01051445, + "balance_loss_clip": 1.04012346, + "balance_loss_mlp": 1.02545691, + "epoch": 0.06801694620161337, + "flos": 17594619045120.0, + "grad_norm": 3.515335935412393, + "language_loss": 0.91636467, + "learning_rate": 3.984861845971747e-06, + "loss": 0.93824077, + "num_input_tokens_seen": 66236400, + "router_z_loss_clip": 0.95996094, + "router_z_loss_mlp": 0.2598877, + "step": 2344, + "time_per_iteration": 2.396223783493042 + }, + { + "auxiliary_loss_clip": 0.01127138, + "auxiliary_loss_mlp": 0.01044837, + "balance_loss_clip": 1.04031456, + "balance_loss_mlp": 1.02156758, + "epoch": 0.06804596367012941, + "flos": 51980156570880.0, + "grad_norm": 1.9811881173513495, + "language_loss": 1.15504277, + "learning_rate": 3.98483875472061e-06, + "loss": 1.17676246, + "num_input_tokens_seen": 66261695, + "router_z_loss_clip": 0.86767578, + "router_z_loss_mlp": 0.23278809, + "step": 2345, + "time_per_iteration": 2.7493443489074707 + }, + { + "auxiliary_loss_clip": 0.01029067, + "auxiliary_loss_mlp": 0.01005011, + "balance_loss_clip": 1.01421261, + "balance_loss_mlp": 1.00349677, + "epoch": 0.06807498113864546, + "flos": 69264222272640.0, + "grad_norm": 0.7529693247557527, + "language_loss": 0.52694082, + "learning_rate": 3.984815645938575e-06, + "loss": 0.54728162, + "num_input_tokens_seen": 66317225, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01513672, + "step": 2346, + "time_per_iteration": 2.9726476669311523 + }, + { + "auxiliary_loss_clip": 0.01121401, + "auxiliary_loss_mlp": 0.01051647, + "balance_loss_clip": 1.04083538, + "balance_loss_mlp": 1.0276618, + "epoch": 0.06810399860716151, + "flos": 26792487129600.0, + "grad_norm": 2.2560402075522483, + "language_loss": 0.77479064, + "learning_rate": 3.984792519625844e-06, + "loss": 0.79652107, + "num_input_tokens_seen": 66337570, + "router_z_loss_clip": 0.80566406, + "router_z_loss_mlp": 0.23974609, + "step": 2347, + "time_per_iteration": 2.5064074993133545 + }, + { + "auxiliary_loss_clip": 0.01130291, + "auxiliary_loss_mlp": 0.01052492, + "balance_loss_clip": 1.04036927, + "balance_loss_mlp": 1.02949643, + "epoch": 0.06813301607567755, + "flos": 21062576638080.0, + "grad_norm": 2.5336888834562235, + "language_loss": 0.74646598, + "learning_rate": 3.984769375782622e-06, + "loss": 0.7682938, + "num_input_tokens_seen": 66352685, + "router_z_loss_clip": 0.89941406, + "router_z_loss_mlp": 0.22998047, + "step": 2348, + "time_per_iteration": 2.41288685798645 + }, + { + "auxiliary_loss_clip": 0.01118361, + "auxiliary_loss_mlp": 0.01047797, + "balance_loss_clip": 1.03850603, + "balance_loss_mlp": 1.02732897, + "epoch": 0.0681620335441936, + "flos": 21537500129280.0, + "grad_norm": 2.140827152084739, + "language_loss": 0.85678691, + "learning_rate": 3.984746214409114e-06, + "loss": 0.87844849, + "num_input_tokens_seen": 66368580, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.20471191, + "step": 2349, + "time_per_iteration": 2.385256290435791 + }, + { + "auxiliary_loss_clip": 0.01123559, + "auxiliary_loss_mlp": 0.01048395, + "balance_loss_clip": 1.04087162, + "balance_loss_mlp": 1.02647233, + "epoch": 0.06819105101270966, + "flos": 27081253918080.0, + "grad_norm": 3.8943530608235544, + "language_loss": 1.07015705, + "learning_rate": 3.9847230355055245e-06, + "loss": 1.09187651, + "num_input_tokens_seen": 66382370, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.21893311, + "step": 2350, + "time_per_iteration": 2.447244167327881 + }, + { + "auxiliary_loss_clip": 0.01120224, + "auxiliary_loss_mlp": 0.01061918, + "balance_loss_clip": 1.0419842, + "balance_loss_mlp": 1.04056728, + "epoch": 0.06822006848122569, + "flos": 28394267529600.0, + "grad_norm": 3.4825240114679277, + "language_loss": 0.89173436, + "learning_rate": 3.984699839072058e-06, + "loss": 0.9135558, + "num_input_tokens_seen": 66396740, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.21350098, + "step": 2351, + "time_per_iteration": 2.6681742668151855 + }, + { + "auxiliary_loss_clip": 0.01116565, + "auxiliary_loss_mlp": 0.01046059, + "balance_loss_clip": 1.03651977, + "balance_loss_mlp": 1.02492297, + "epoch": 0.06824908594974174, + "flos": 16316797950720.0, + "grad_norm": 2.466125014622119, + "language_loss": 0.85261494, + "learning_rate": 3.9846766251089195e-06, + "loss": 0.87424123, + "num_input_tokens_seen": 66409440, + "router_z_loss_clip": 0.79931641, + "router_z_loss_mlp": 0.21142578, + "step": 2352, + "time_per_iteration": 2.3862643241882324 + }, + { + "auxiliary_loss_clip": 0.01123557, + "auxiliary_loss_mlp": 0.01048028, + "balance_loss_clip": 1.03983521, + "balance_loss_mlp": 1.02463937, + "epoch": 0.0682781034182578, + "flos": 28176653825280.0, + "grad_norm": 2.617611307291406, + "language_loss": 0.92491418, + "learning_rate": 3.984653393616313e-06, + "loss": 0.94662994, + "num_input_tokens_seen": 66425485, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.23388672, + "step": 2353, + "time_per_iteration": 2.47420072555542 + }, + { + "auxiliary_loss_clip": 0.01126813, + "auxiliary_loss_mlp": 0.01052174, + "balance_loss_clip": 1.04054737, + "balance_loss_mlp": 1.0296911, + "epoch": 0.06830712088677383, + "flos": 12378036407040.0, + "grad_norm": 3.9952264615608932, + "language_loss": 0.8702057, + "learning_rate": 3.984630144594446e-06, + "loss": 0.89199555, + "num_input_tokens_seen": 66436315, + "router_z_loss_clip": 0.86132812, + "router_z_loss_mlp": 0.22485352, + "step": 2354, + "time_per_iteration": 2.372223138809204 + }, + { + "auxiliary_loss_clip": 0.01122256, + "auxiliary_loss_mlp": 0.01053948, + "balance_loss_clip": 1.03682101, + "balance_loss_mlp": 1.0286876, + "epoch": 0.06833613835528989, + "flos": 18835257674880.0, + "grad_norm": 3.5533740525260105, + "language_loss": 0.89115262, + "learning_rate": 3.984606878043522e-06, + "loss": 0.91291463, + "num_input_tokens_seen": 66448050, + "router_z_loss_clip": 0.85449219, + "router_z_loss_mlp": 0.25256348, + "step": 2355, + "time_per_iteration": 2.344653367996216 + }, + { + "auxiliary_loss_clip": 0.01118367, + "auxiliary_loss_mlp": 0.01047543, + "balance_loss_clip": 1.03761029, + "balance_loss_mlp": 1.02504206, + "epoch": 0.06836515582380594, + "flos": 27382518973440.0, + "grad_norm": 2.261318191098306, + "language_loss": 1.08078325, + "learning_rate": 3.984583593963747e-06, + "loss": 1.10244226, + "num_input_tokens_seen": 66466985, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.22497559, + "step": 2356, + "time_per_iteration": 2.5507898330688477 + }, + { + "auxiliary_loss_clip": 0.01123211, + "auxiliary_loss_mlp": 0.01056628, + "balance_loss_clip": 1.03700602, + "balance_loss_mlp": 1.03047371, + "epoch": 0.06839417329232197, + "flos": 25733222346240.0, + "grad_norm": 2.020476209715353, + "language_loss": 0.85234076, + "learning_rate": 3.984560292355326e-06, + "loss": 0.87413919, + "num_input_tokens_seen": 66484575, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.26147461, + "step": 2357, + "time_per_iteration": 2.479326009750366 + }, + { + "auxiliary_loss_clip": 0.01132768, + "auxiliary_loss_mlp": 0.0105134, + "balance_loss_clip": 1.03785849, + "balance_loss_mlp": 1.02466106, + "epoch": 0.06842319076083803, + "flos": 47477827860480.0, + "grad_norm": 7.870169826404653, + "language_loss": 0.98560375, + "learning_rate": 3.984536973218466e-06, + "loss": 1.00744486, + "num_input_tokens_seen": 66503110, + "router_z_loss_clip": 0.94873047, + "router_z_loss_mlp": 0.26672363, + "step": 2358, + "time_per_iteration": 2.6836767196655273 + }, + { + "auxiliary_loss_clip": 0.0112616, + "auxiliary_loss_mlp": 0.01044771, + "balance_loss_clip": 1.04262531, + "balance_loss_mlp": 1.02250218, + "epoch": 0.06845220822935408, + "flos": 11175208646400.0, + "grad_norm": 2.5952110160869832, + "language_loss": 0.78941989, + "learning_rate": 3.984513636553372e-06, + "loss": 0.81112921, + "num_input_tokens_seen": 66514520, + "router_z_loss_clip": 0.83447266, + "router_z_loss_mlp": 0.22290039, + "step": 2359, + "time_per_iteration": 2.38466215133667 + }, + { + "auxiliary_loss_clip": 0.01129412, + "auxiliary_loss_mlp": 0.01058908, + "balance_loss_clip": 1.0389986, + "balance_loss_mlp": 1.03311133, + "epoch": 0.06848122569787012, + "flos": 30398526616320.0, + "grad_norm": 2.8659649309216633, + "language_loss": 0.77506924, + "learning_rate": 3.984490282360251e-06, + "loss": 0.79695249, + "num_input_tokens_seen": 66527920, + "router_z_loss_clip": 0.90429688, + "router_z_loss_mlp": 0.25793457, + "step": 2360, + "time_per_iteration": 5.235094308853149 + }, + { + "auxiliary_loss_clip": 0.01120276, + "auxiliary_loss_mlp": 0.01045677, + "balance_loss_clip": 1.04017639, + "balance_loss_mlp": 1.02362359, + "epoch": 0.06851024316638617, + "flos": 58752679127040.0, + "grad_norm": 1.781592269397547, + "language_loss": 0.85168886, + "learning_rate": 3.984466910639309e-06, + "loss": 0.87334841, + "num_input_tokens_seen": 66552745, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.22058105, + "step": 2361, + "time_per_iteration": 4.96644926071167 + }, + { + "auxiliary_loss_clip": 0.01118943, + "auxiliary_loss_mlp": 0.01046789, + "balance_loss_clip": 1.03971493, + "balance_loss_mlp": 1.02523565, + "epoch": 0.0685392606349022, + "flos": 16136366711040.0, + "grad_norm": 3.5112973575163315, + "language_loss": 0.68336058, + "learning_rate": 3.984443521390752e-06, + "loss": 0.70501786, + "num_input_tokens_seen": 66565650, + "router_z_loss_clip": 0.79296875, + "router_z_loss_mlp": 0.21569824, + "step": 2362, + "time_per_iteration": 2.4850165843963623 + }, + { + "auxiliary_loss_clip": 0.01124049, + "auxiliary_loss_mlp": 0.01046292, + "balance_loss_clip": 1.039518, + "balance_loss_mlp": 1.02242589, + "epoch": 0.06856827810341826, + "flos": 29787860292480.0, + "grad_norm": 1.7647670788683176, + "language_loss": 0.84490013, + "learning_rate": 3.984420114614786e-06, + "loss": 0.86660349, + "num_input_tokens_seen": 66586585, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.23876953, + "step": 2363, + "time_per_iteration": 2.909181594848633 + }, + { + "auxiliary_loss_clip": 0.01119108, + "auxiliary_loss_mlp": 0.01050791, + "balance_loss_clip": 1.03868961, + "balance_loss_mlp": 1.0297389, + "epoch": 0.06859729557193431, + "flos": 16179833220480.0, + "grad_norm": 7.569769931432349, + "language_loss": 0.76763844, + "learning_rate": 3.984396690311619e-06, + "loss": 0.78933734, + "num_input_tokens_seen": 66600515, + "router_z_loss_clip": 0.80371094, + "router_z_loss_mlp": 0.21057129, + "step": 2364, + "time_per_iteration": 2.4984805583953857 + }, + { + "auxiliary_loss_clip": 0.0101876, + "auxiliary_loss_mlp": 0.0101332, + "balance_loss_clip": 1.00417519, + "balance_loss_mlp": 1.01143694, + "epoch": 0.06862631304045035, + "flos": 61271276630400.0, + "grad_norm": 0.7519941904391646, + "language_loss": 0.52291954, + "learning_rate": 3.9843732484814585e-06, + "loss": 0.54324031, + "num_input_tokens_seen": 66658005, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01879883, + "step": 2365, + "time_per_iteration": 5.3792712688446045 + }, + { + "auxiliary_loss_clip": 0.01019315, + "auxiliary_loss_mlp": 0.0101263, + "balance_loss_clip": 1.00475669, + "balance_loss_mlp": 1.0107106, + "epoch": 0.0686553305089664, + "flos": 74778543918720.0, + "grad_norm": 0.6771324797602501, + "language_loss": 0.48275557, + "learning_rate": 3.984349789124509e-06, + "loss": 0.503075, + "num_input_tokens_seen": 66726810, + "router_z_loss_clip": 0.14550781, + "router_z_loss_mlp": 0.01916504, + "step": 2366, + "time_per_iteration": 3.157770872116089 + }, + { + "auxiliary_loss_clip": 0.01123007, + "auxiliary_loss_mlp": 0.01039833, + "balance_loss_clip": 1.03700733, + "balance_loss_mlp": 1.01688552, + "epoch": 0.06868434797748245, + "flos": 25841522983680.0, + "grad_norm": 2.5884954283666057, + "language_loss": 0.77190268, + "learning_rate": 3.98432631224098e-06, + "loss": 0.79353112, + "num_input_tokens_seen": 66743380, + "router_z_loss_clip": 0.85986328, + "router_z_loss_mlp": 0.2298584, + "step": 2367, + "time_per_iteration": 2.4688196182250977 + }, + { + "auxiliary_loss_clip": 0.01124571, + "auxiliary_loss_mlp": 0.01057608, + "balance_loss_clip": 1.043015, + "balance_loss_mlp": 1.03487515, + "epoch": 0.06871336544599849, + "flos": 34086227506560.0, + "grad_norm": 6.631081446584609, + "language_loss": 0.99277377, + "learning_rate": 3.984302817831078e-06, + "loss": 1.01459551, + "num_input_tokens_seen": 66761635, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.22717285, + "step": 2368, + "time_per_iteration": 4.951500177383423 + }, + { + "auxiliary_loss_clip": 0.01021356, + "auxiliary_loss_mlp": 0.01001991, + "balance_loss_clip": 1.00678468, + "balance_loss_mlp": 1.00016677, + "epoch": 0.06874238291451454, + "flos": 74769850990080.0, + "grad_norm": 0.7207816227944187, + "language_loss": 0.48319897, + "learning_rate": 3.98427930589501e-06, + "loss": 0.50343239, + "num_input_tokens_seen": 66824315, + "router_z_loss_clip": 0.14550781, + "router_z_loss_mlp": 0.01818848, + "step": 2369, + "time_per_iteration": 3.0839452743530273 + }, + { + "auxiliary_loss_clip": 0.01119848, + "auxiliary_loss_mlp": 0.01056158, + "balance_loss_clip": 1.03792, + "balance_loss_mlp": 1.03229284, + "epoch": 0.06877140038303059, + "flos": 47344668468480.0, + "grad_norm": 2.4464869197121617, + "language_loss": 0.78424978, + "learning_rate": 3.984255776432984e-06, + "loss": 0.80600989, + "num_input_tokens_seen": 66843070, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.23876953, + "step": 2370, + "time_per_iteration": 2.7334980964660645 + }, + { + "auxiliary_loss_clip": 0.01134158, + "auxiliary_loss_mlp": 0.01067884, + "balance_loss_clip": 1.04426098, + "balance_loss_mlp": 1.04236174, + "epoch": 0.06880041785154663, + "flos": 14789487214080.0, + "grad_norm": 2.1986014462406827, + "language_loss": 0.82266414, + "learning_rate": 3.984232229445209e-06, + "loss": 0.8446846, + "num_input_tokens_seen": 66855350, + "router_z_loss_clip": 0.89892578, + "router_z_loss_mlp": 0.25537109, + "step": 2371, + "time_per_iteration": 2.513758659362793 + }, + { + "auxiliary_loss_clip": 0.01021623, + "auxiliary_loss_mlp": 0.01013718, + "balance_loss_clip": 1.00729287, + "balance_loss_mlp": 1.01207316, + "epoch": 0.06882943532006268, + "flos": 74773027923840.0, + "grad_norm": 0.7326563666310586, + "language_loss": 0.51602739, + "learning_rate": 3.984208664931891e-06, + "loss": 0.53638083, + "num_input_tokens_seen": 66918380, + "router_z_loss_clip": 0.14355469, + "router_z_loss_mlp": 0.01647949, + "step": 2372, + "time_per_iteration": 3.0422286987304688 + }, + { + "auxiliary_loss_clip": 0.01019667, + "auxiliary_loss_mlp": 0.01008849, + "balance_loss_clip": 1.00526476, + "balance_loss_mlp": 1.0072037, + "epoch": 0.06885845278857873, + "flos": 69963602094720.0, + "grad_norm": 0.6855479623079609, + "language_loss": 0.54090893, + "learning_rate": 3.984185082893241e-06, + "loss": 0.56119418, + "num_input_tokens_seen": 66986760, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01647949, + "step": 2373, + "time_per_iteration": 3.2211077213287354 + }, + { + "auxiliary_loss_clip": 0.01019485, + "auxiliary_loss_mlp": 0.01008288, + "balance_loss_clip": 1.00472498, + "balance_loss_mlp": 1.0067265, + "epoch": 0.06888747025709477, + "flos": 66049489837440.0, + "grad_norm": 0.6595006984802653, + "language_loss": 0.51367068, + "learning_rate": 3.984161483329464e-06, + "loss": 0.5339483, + "num_input_tokens_seen": 67047025, + "router_z_loss_clip": 0.14746094, + "router_z_loss_mlp": 0.015625, + "step": 2374, + "time_per_iteration": 3.169116497039795 + }, + { + "auxiliary_loss_clip": 0.01122809, + "auxiliary_loss_mlp": 0.01055868, + "balance_loss_clip": 1.04064441, + "balance_loss_mlp": 1.03435063, + "epoch": 0.06891648772561082, + "flos": 11976919263360.0, + "grad_norm": 3.179231270720942, + "language_loss": 0.90289104, + "learning_rate": 3.9841378662407696e-06, + "loss": 0.92467773, + "num_input_tokens_seen": 67059635, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.21533203, + "step": 2375, + "time_per_iteration": 2.3571109771728516 + }, + { + "auxiliary_loss_clip": 0.01122562, + "auxiliary_loss_mlp": 0.01056418, + "balance_loss_clip": 1.04178393, + "balance_loss_mlp": 1.03364909, + "epoch": 0.06894550519412686, + "flos": 51568461285120.0, + "grad_norm": 2.1353115581924706, + "language_loss": 0.90565342, + "learning_rate": 3.984114231627367e-06, + "loss": 0.92744327, + "num_input_tokens_seen": 67079745, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.22766113, + "step": 2376, + "time_per_iteration": 2.710341215133667 + }, + { + "auxiliary_loss_clip": 0.01114102, + "auxiliary_loss_mlp": 0.01049341, + "balance_loss_clip": 1.0406096, + "balance_loss_mlp": 1.03036261, + "epoch": 0.06897452266264291, + "flos": 19055803933440.0, + "grad_norm": 2.4489395060690518, + "language_loss": 0.79197842, + "learning_rate": 3.984090579489465e-06, + "loss": 0.81361282, + "num_input_tokens_seen": 67094325, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.18969727, + "step": 2377, + "time_per_iteration": 2.342372417449951 + }, + { + "auxiliary_loss_clip": 0.01130079, + "auxiliary_loss_mlp": 0.01044305, + "balance_loss_clip": 1.04063582, + "balance_loss_mlp": 1.02083254, + "epoch": 0.06900354013115896, + "flos": 17782137290880.0, + "grad_norm": 2.2507383547298603, + "language_loss": 0.81882286, + "learning_rate": 3.9840669098272715e-06, + "loss": 0.84056664, + "num_input_tokens_seen": 67110825, + "router_z_loss_clip": 0.89453125, + "router_z_loss_mlp": 0.23474121, + "step": 2378, + "time_per_iteration": 2.4326395988464355 + }, + { + "auxiliary_loss_clip": 0.01124679, + "auxiliary_loss_mlp": 0.01046401, + "balance_loss_clip": 1.04360127, + "balance_loss_mlp": 1.02418017, + "epoch": 0.069032557599675, + "flos": 25475772913920.0, + "grad_norm": 2.3521127782398445, + "language_loss": 0.96994734, + "learning_rate": 3.984043222640997e-06, + "loss": 0.99165815, + "num_input_tokens_seen": 67126200, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.22241211, + "step": 2379, + "time_per_iteration": 2.4205942153930664 + }, + { + "auxiliary_loss_clip": 0.01128694, + "auxiliary_loss_mlp": 0.01054516, + "balance_loss_clip": 1.04400373, + "balance_loss_mlp": 1.03073335, + "epoch": 0.06906157506819105, + "flos": 12851388887040.0, + "grad_norm": 3.2178120981491, + "language_loss": 0.84284252, + "learning_rate": 3.984019517930849e-06, + "loss": 0.86467457, + "num_input_tokens_seen": 67137415, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.23791504, + "step": 2380, + "time_per_iteration": 2.392119884490967 + }, + { + "auxiliary_loss_clip": 0.01120148, + "auxiliary_loss_mlp": 0.01037237, + "balance_loss_clip": 1.04508233, + "balance_loss_mlp": 1.01896191, + "epoch": 0.0690905925367071, + "flos": 16209649388160.0, + "grad_norm": 2.60512098304943, + "language_loss": 0.71825719, + "learning_rate": 3.983995795697038e-06, + "loss": 0.73983097, + "num_input_tokens_seen": 67152110, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.18286133, + "step": 2381, + "time_per_iteration": 2.3653573989868164 + }, + { + "auxiliary_loss_clip": 0.01136331, + "auxiliary_loss_mlp": 0.01064342, + "balance_loss_clip": 1.04549074, + "balance_loss_mlp": 1.03920031, + "epoch": 0.06911961000522314, + "flos": 16318473696000.0, + "grad_norm": 2.4657498438332457, + "language_loss": 0.78315759, + "learning_rate": 3.983972055939774e-06, + "loss": 0.80516422, + "num_input_tokens_seen": 67164450, + "router_z_loss_clip": 0.90917969, + "router_z_loss_mlp": 0.25146484, + "step": 2382, + "time_per_iteration": 2.392284870147705 + }, + { + "auxiliary_loss_clip": 0.01134652, + "auxiliary_loss_mlp": 0.01055654, + "balance_loss_clip": 1.04899144, + "balance_loss_mlp": 1.03138328, + "epoch": 0.06914862747373919, + "flos": 27444874394880.0, + "grad_norm": 2.673355911418145, + "language_loss": 0.99851501, + "learning_rate": 3.983948298659266e-06, + "loss": 1.02041805, + "num_input_tokens_seen": 67177580, + "router_z_loss_clip": 0.85595703, + "router_z_loss_mlp": 0.24255371, + "step": 2383, + "time_per_iteration": 2.440692186355591 + }, + { + "auxiliary_loss_clip": 0.0113267, + "auxiliary_loss_mlp": 0.01055993, + "balance_loss_clip": 1.04534173, + "balance_loss_mlp": 1.0315913, + "epoch": 0.06917764494225524, + "flos": 41600025383040.0, + "grad_norm": 3.0180426298628626, + "language_loss": 1.06345677, + "learning_rate": 3.983924523855723e-06, + "loss": 1.08534348, + "num_input_tokens_seen": 67192595, + "router_z_loss_clip": 0.87304688, + "router_z_loss_mlp": 0.24438477, + "step": 2384, + "time_per_iteration": 2.6925930976867676 + }, + { + "auxiliary_loss_clip": 0.0112763, + "auxiliary_loss_mlp": 0.01057526, + "balance_loss_clip": 1.04728258, + "balance_loss_mlp": 1.03433967, + "epoch": 0.06920666241077128, + "flos": 19966094478720.0, + "grad_norm": 3.432872047529656, + "language_loss": 0.74698514, + "learning_rate": 3.983900731529356e-06, + "loss": 0.76883674, + "num_input_tokens_seen": 67204425, + "router_z_loss_clip": 0.80371094, + "router_z_loss_mlp": 0.23205566, + "step": 2385, + "time_per_iteration": 2.38666033744812 + }, + { + "auxiliary_loss_clip": 0.01126447, + "auxiliary_loss_mlp": 0.01049088, + "balance_loss_clip": 1.04063046, + "balance_loss_mlp": 1.02451885, + "epoch": 0.06923567987928733, + "flos": 18216841029120.0, + "grad_norm": 2.2204621715946034, + "language_loss": 0.78268337, + "learning_rate": 3.983876921680375e-06, + "loss": 0.80443871, + "num_input_tokens_seen": 67220650, + "router_z_loss_clip": 0.85791016, + "router_z_loss_mlp": 0.24572754, + "step": 2386, + "time_per_iteration": 2.4656124114990234 + }, + { + "auxiliary_loss_clip": 0.01128983, + "auxiliary_loss_mlp": 0.01054941, + "balance_loss_clip": 1.04514742, + "balance_loss_mlp": 1.03008628, + "epoch": 0.06926469734780338, + "flos": 16427228181120.0, + "grad_norm": 2.762437941724509, + "language_loss": 0.85347223, + "learning_rate": 3.98385309430899e-06, + "loss": 0.87531149, + "num_input_tokens_seen": 67233180, + "router_z_loss_clip": 0.83789062, + "router_z_loss_mlp": 0.24829102, + "step": 2387, + "time_per_iteration": 2.4651095867156982 + }, + { + "auxiliary_loss_clip": 0.01132572, + "auxiliary_loss_mlp": 0.01052702, + "balance_loss_clip": 1.04585326, + "balance_loss_mlp": 1.02825189, + "epoch": 0.06929371481631942, + "flos": 29234173040640.0, + "grad_norm": 2.2623954620212823, + "language_loss": 1.00660729, + "learning_rate": 3.9838292494154125e-06, + "loss": 1.02846003, + "num_input_tokens_seen": 67250545, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.24438477, + "step": 2388, + "time_per_iteration": 2.5857810974121094 + }, + { + "auxiliary_loss_clip": 0.01053243, + "auxiliary_loss_mlp": 0.01028349, + "balance_loss_clip": 1.03764021, + "balance_loss_mlp": 1.02653742, + "epoch": 0.06932273228483547, + "flos": 74791216742400.0, + "grad_norm": 0.7440560127925595, + "language_loss": 0.5332098, + "learning_rate": 3.983805386999851e-06, + "loss": 0.55402571, + "num_input_tokens_seen": 67316275, + "router_z_loss_clip": 0.15625, + "router_z_loss_mlp": 0.01806641, + "step": 2389, + "time_per_iteration": 3.241790771484375 + }, + { + "auxiliary_loss_clip": 0.01116384, + "auxiliary_loss_mlp": 0.01043435, + "balance_loss_clip": 1.03965569, + "balance_loss_mlp": 1.02314544, + "epoch": 0.06935174975335152, + "flos": 18105852216960.0, + "grad_norm": 2.838420934826311, + "language_loss": 0.76139414, + "learning_rate": 3.9837815070625185e-06, + "loss": 0.78299236, + "num_input_tokens_seen": 67329310, + "router_z_loss_clip": 0.76708984, + "router_z_loss_mlp": 0.20288086, + "step": 2390, + "time_per_iteration": 2.4016549587249756 + }, + { + "auxiliary_loss_clip": 0.01124773, + "auxiliary_loss_mlp": 0.01049068, + "balance_loss_clip": 1.0369972, + "balance_loss_mlp": 1.02530932, + "epoch": 0.06938076722186756, + "flos": 32307332446080.0, + "grad_norm": 2.425167320005028, + "language_loss": 0.7847234, + "learning_rate": 3.983757609603625e-06, + "loss": 0.80646181, + "num_input_tokens_seen": 67352275, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.23754883, + "step": 2391, + "time_per_iteration": 2.4834442138671875 + }, + { + "auxiliary_loss_clip": 0.01106801, + "auxiliary_loss_mlp": 0.01044359, + "balance_loss_clip": 1.03592169, + "balance_loss_mlp": 1.02712703, + "epoch": 0.06940978469038361, + "flos": 28867515275520.0, + "grad_norm": 1.9625529026277537, + "language_loss": 0.63501298, + "learning_rate": 3.983733694623382e-06, + "loss": 0.6565246, + "num_input_tokens_seen": 67366460, + "router_z_loss_clip": 0.70996094, + "router_z_loss_mlp": 0.17211914, + "step": 2392, + "time_per_iteration": 2.51335072517395 + }, + { + "auxiliary_loss_clip": 0.01025167, + "auxiliary_loss_mlp": 0.01002108, + "balance_loss_clip": 1.0099318, + "balance_loss_mlp": 1.00040317, + "epoch": 0.06943880215889965, + "flos": 59587206422400.0, + "grad_norm": 0.7525507975528718, + "language_loss": 0.47775698, + "learning_rate": 3.983709762121999e-06, + "loss": 0.49802974, + "num_input_tokens_seen": 67419300, + "router_z_loss_clip": 0.15234375, + "router_z_loss_mlp": 0.01708984, + "step": 2393, + "time_per_iteration": 2.8691024780273438 + }, + { + "auxiliary_loss_clip": 0.01121852, + "auxiliary_loss_mlp": 0.0105353, + "balance_loss_clip": 1.03680313, + "balance_loss_mlp": 1.03035522, + "epoch": 0.0694678196274157, + "flos": 10990937157120.0, + "grad_norm": 2.8732872729899452, + "language_loss": 0.83085936, + "learning_rate": 3.983685812099689e-06, + "loss": 0.85261315, + "num_input_tokens_seen": 67429205, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.23181152, + "step": 2394, + "time_per_iteration": 2.4189209938049316 + }, + { + "auxiliary_loss_clip": 0.01119778, + "auxiliary_loss_mlp": 0.01055036, + "balance_loss_clip": 1.03713179, + "balance_loss_mlp": 1.03143239, + "epoch": 0.06949683709593175, + "flos": 54629471537280.0, + "grad_norm": 1.9938341693074335, + "language_loss": 0.92497909, + "learning_rate": 3.983661844556664e-06, + "loss": 0.94672722, + "num_input_tokens_seen": 67448635, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.23620605, + "step": 2395, + "time_per_iteration": 2.686403751373291 + }, + { + "auxiliary_loss_clip": 0.01019635, + "auxiliary_loss_mlp": 0.01002806, + "balance_loss_clip": 1.00456858, + "balance_loss_mlp": 1.00137508, + "epoch": 0.06952585456444779, + "flos": 74773097746560.0, + "grad_norm": 0.7083096680737909, + "language_loss": 0.52411467, + "learning_rate": 3.983637859493134e-06, + "loss": 0.54433906, + "num_input_tokens_seen": 67510405, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01428223, + "step": 2396, + "time_per_iteration": 3.1074838638305664 + }, + { + "auxiliary_loss_clip": 0.01128054, + "auxiliary_loss_mlp": 0.01054212, + "balance_loss_clip": 1.0409708, + "balance_loss_mlp": 1.03138375, + "epoch": 0.06955487203296384, + "flos": 21793622929920.0, + "grad_norm": 3.0872730435851747, + "language_loss": 0.83851492, + "learning_rate": 3.9836138569093125e-06, + "loss": 0.8603375, + "num_input_tokens_seen": 67525045, + "router_z_loss_clip": 0.87060547, + "router_z_loss_mlp": 0.22827148, + "step": 2397, + "time_per_iteration": 2.398298740386963 + }, + { + "auxiliary_loss_clip": 0.01130173, + "auxiliary_loss_mlp": 0.01056031, + "balance_loss_clip": 1.04419231, + "balance_loss_mlp": 1.03140211, + "epoch": 0.0695838895014799, + "flos": 37625966588160.0, + "grad_norm": 2.0940648061865934, + "language_loss": 0.85930318, + "learning_rate": 3.98358983680541e-06, + "loss": 0.88116521, + "num_input_tokens_seen": 67540305, + "router_z_loss_clip": 0.86035156, + "router_z_loss_mlp": 0.24621582, + "step": 2398, + "time_per_iteration": 2.5912160873413086 + }, + { + "auxiliary_loss_clip": 0.01129579, + "auxiliary_loss_mlp": 0.01053253, + "balance_loss_clip": 1.04528189, + "balance_loss_mlp": 1.02950656, + "epoch": 0.06961290696999593, + "flos": 23835204126720.0, + "grad_norm": 2.068971617769903, + "language_loss": 0.80315185, + "learning_rate": 3.98356579918164e-06, + "loss": 0.82498014, + "num_input_tokens_seen": 67558600, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.23742676, + "step": 2399, + "time_per_iteration": 2.427912473678589 + }, + { + "auxiliary_loss_clip": 0.01131895, + "auxiliary_loss_mlp": 0.01054429, + "balance_loss_clip": 1.04519534, + "balance_loss_mlp": 1.03030097, + "epoch": 0.06964192443851198, + "flos": 30701537239680.0, + "grad_norm": 1.8695680003836894, + "language_loss": 0.84521168, + "learning_rate": 3.983541744038214e-06, + "loss": 0.86707497, + "num_input_tokens_seen": 67579145, + "router_z_loss_clip": 0.8671875, + "router_z_loss_mlp": 0.24145508, + "step": 2400, + "time_per_iteration": 2.5123822689056396 + }, + { + "auxiliary_loss_clip": 0.01132184, + "auxiliary_loss_mlp": 0.01040686, + "balance_loss_clip": 1.04808795, + "balance_loss_mlp": 1.01856112, + "epoch": 0.06967094190702804, + "flos": 21536592433920.0, + "grad_norm": 2.47934162506578, + "language_loss": 0.77295917, + "learning_rate": 3.983517671375344e-06, + "loss": 0.79468787, + "num_input_tokens_seen": 67593515, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.22131348, + "step": 2401, + "time_per_iteration": 2.5613348484039307 + }, + { + "auxiliary_loss_clip": 0.01131346, + "auxiliary_loss_mlp": 0.01062274, + "balance_loss_clip": 1.04469538, + "balance_loss_mlp": 1.03672719, + "epoch": 0.06969995937554407, + "flos": 21281063126400.0, + "grad_norm": 6.261463538668208, + "language_loss": 1.02495193, + "learning_rate": 3.983493581193243e-06, + "loss": 1.04688811, + "num_input_tokens_seen": 67605750, + "router_z_loss_clip": 0.86669922, + "router_z_loss_mlp": 0.25549316, + "step": 2402, + "time_per_iteration": 2.4644229412078857 + }, + { + "auxiliary_loss_clip": 0.0112944, + "auxiliary_loss_mlp": 0.01045652, + "balance_loss_clip": 1.0468874, + "balance_loss_mlp": 1.02500522, + "epoch": 0.06972897684406013, + "flos": 11982470169600.0, + "grad_norm": 2.3199319502746523, + "language_loss": 0.73883152, + "learning_rate": 3.983469473492126e-06, + "loss": 0.76058245, + "num_input_tokens_seen": 67619705, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.2064209, + "step": 2403, + "time_per_iteration": 2.3558642864227295 + }, + { + "auxiliary_loss_clip": 0.01135951, + "auxiliary_loss_mlp": 0.01065079, + "balance_loss_clip": 1.05031037, + "balance_loss_mlp": 1.04008055, + "epoch": 0.06975799431257618, + "flos": 45651905331840.0, + "grad_norm": 2.700786187042496, + "language_loss": 0.907628, + "learning_rate": 3.983445348272203e-06, + "loss": 0.92963827, + "num_input_tokens_seen": 67639530, + "router_z_loss_clip": 0.85693359, + "router_z_loss_mlp": 0.25, + "step": 2404, + "time_per_iteration": 2.6299448013305664 + }, + { + "auxiliary_loss_clip": 0.01128623, + "auxiliary_loss_mlp": 0.0105113, + "balance_loss_clip": 1.04469872, + "balance_loss_mlp": 1.02693081, + "epoch": 0.06978701178109221, + "flos": 17924129256960.0, + "grad_norm": 3.060798883055964, + "language_loss": 0.75909984, + "learning_rate": 3.983421205533688e-06, + "loss": 0.78089738, + "num_input_tokens_seen": 67654340, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.2421875, + "step": 2405, + "time_per_iteration": 2.3709936141967773 + }, + { + "auxiliary_loss_clip": 0.01119648, + "auxiliary_loss_mlp": 0.01048159, + "balance_loss_clip": 1.03985476, + "balance_loss_mlp": 1.02670145, + "epoch": 0.06981602924960827, + "flos": 60974935943040.0, + "grad_norm": 1.8588979934137304, + "language_loss": 0.87490869, + "learning_rate": 3.9833970452767935e-06, + "loss": 0.89658684, + "num_input_tokens_seen": 67682505, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.21459961, + "step": 2406, + "time_per_iteration": 2.7624011039733887 + }, + { + "auxiliary_loss_clip": 0.01028174, + "auxiliary_loss_mlp": 0.01007943, + "balance_loss_clip": 1.01268101, + "balance_loss_mlp": 1.00599968, + "epoch": 0.0698450467181243, + "flos": 60825820193280.0, + "grad_norm": 0.7170203035910275, + "language_loss": 0.51003242, + "learning_rate": 3.9833728675017355e-06, + "loss": 0.5303936, + "num_input_tokens_seen": 67735405, + "router_z_loss_clip": 0.15429688, + "router_z_loss_mlp": 0.01940918, + "step": 2407, + "time_per_iteration": 2.850367546081543 + }, + { + "auxiliary_loss_clip": 0.01118871, + "auxiliary_loss_mlp": 0.01045173, + "balance_loss_clip": 1.03877199, + "balance_loss_mlp": 1.02346516, + "epoch": 0.06987406418664036, + "flos": 16356773324160.0, + "grad_norm": 2.2778110035276047, + "language_loss": 0.72615063, + "learning_rate": 3.983348672208724e-06, + "loss": 0.74779099, + "num_input_tokens_seen": 67750465, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.21691895, + "step": 2408, + "time_per_iteration": 2.4021992683410645 + }, + { + "auxiliary_loss_clip": 0.01123091, + "auxiliary_loss_mlp": 0.01050975, + "balance_loss_clip": 1.04007161, + "balance_loss_mlp": 1.02741969, + "epoch": 0.06990308165515641, + "flos": 74730086951040.0, + "grad_norm": 1.9283579978363001, + "language_loss": 0.81238651, + "learning_rate": 3.983324459397975e-06, + "loss": 0.83412719, + "num_input_tokens_seen": 67775430, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.23571777, + "step": 2409, + "time_per_iteration": 2.8827033042907715 + }, + { + "auxiliary_loss_clip": 0.01021897, + "auxiliary_loss_mlp": 0.01002364, + "balance_loss_clip": 1.0060941, + "balance_loss_mlp": 1.00038517, + "epoch": 0.06993209912367244, + "flos": 67186261572480.0, + "grad_norm": 0.7396288718190229, + "language_loss": 0.51739907, + "learning_rate": 3.983300229069703e-06, + "loss": 0.53764164, + "num_input_tokens_seen": 67835045, + "router_z_loss_clip": 0.15820312, + "router_z_loss_mlp": 0.01977539, + "step": 2410, + "time_per_iteration": 3.057316541671753 + }, + { + "auxiliary_loss_clip": 0.01116432, + "auxiliary_loss_mlp": 0.01042719, + "balance_loss_clip": 1.03750563, + "balance_loss_mlp": 1.02021229, + "epoch": 0.0699611165921885, + "flos": 16974771033600.0, + "grad_norm": 2.7390136577716, + "language_loss": 0.64412242, + "learning_rate": 3.9832759812241195e-06, + "loss": 0.66571391, + "num_input_tokens_seen": 67850030, + "router_z_loss_clip": 0.79003906, + "router_z_loss_mlp": 0.22509766, + "step": 2411, + "time_per_iteration": 2.414410352706909 + }, + { + "auxiliary_loss_clip": 0.0112056, + "auxiliary_loss_mlp": 0.01045988, + "balance_loss_clip": 1.03638434, + "balance_loss_mlp": 1.02317154, + "epoch": 0.06999013406070455, + "flos": 12195650131200.0, + "grad_norm": 2.662958687479808, + "language_loss": 0.83163738, + "learning_rate": 3.98325171586144e-06, + "loss": 0.85330284, + "num_input_tokens_seen": 67861730, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.22814941, + "step": 2412, + "time_per_iteration": 2.3674967288970947 + }, + { + "auxiliary_loss_clip": 0.01115399, + "auxiliary_loss_mlp": 0.01042565, + "balance_loss_clip": 1.03693318, + "balance_loss_mlp": 1.02216816, + "epoch": 0.07001915152922059, + "flos": 16865946725760.0, + "grad_norm": 3.562006576150382, + "language_loss": 0.81221724, + "learning_rate": 3.983227432981879e-06, + "loss": 0.8337968, + "num_input_tokens_seen": 67877200, + "router_z_loss_clip": 0.78466797, + "router_z_loss_mlp": 0.20397949, + "step": 2413, + "time_per_iteration": 2.39298939704895 + }, + { + "auxiliary_loss_clip": 0.0111478, + "auxiliary_loss_mlp": 0.01044863, + "balance_loss_clip": 1.03595841, + "balance_loss_mlp": 1.02141428, + "epoch": 0.07004816899773664, + "flos": 20553228679680.0, + "grad_norm": 2.2090020309842755, + "language_loss": 0.73445249, + "learning_rate": 3.9832031325856515e-06, + "loss": 0.75604892, + "num_input_tokens_seen": 67893830, + "router_z_loss_clip": 0.78857422, + "router_z_loss_mlp": 0.23449707, + "step": 2414, + "time_per_iteration": 2.629561424255371 + }, + { + "auxiliary_loss_clip": 0.01123245, + "auxiliary_loss_mlp": 0.0105111, + "balance_loss_clip": 1.03980923, + "balance_loss_mlp": 1.02797151, + "epoch": 0.07007718646625269, + "flos": 39820152804480.0, + "grad_norm": 2.0246807658031867, + "language_loss": 1.16114759, + "learning_rate": 3.98317881467297e-06, + "loss": 1.18289113, + "num_input_tokens_seen": 67917965, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.23144531, + "step": 2415, + "time_per_iteration": 2.6167407035827637 + }, + { + "auxiliary_loss_clip": 0.01115121, + "auxiliary_loss_mlp": 0.01048166, + "balance_loss_clip": 1.03512025, + "balance_loss_mlp": 1.02657723, + "epoch": 0.07010620393476873, + "flos": 24713548911360.0, + "grad_norm": 2.7510249995948373, + "language_loss": 0.94476974, + "learning_rate": 3.983154479244051e-06, + "loss": 0.96640259, + "num_input_tokens_seen": 67933780, + "router_z_loss_clip": 0.80029297, + "router_z_loss_mlp": 0.21594238, + "step": 2416, + "time_per_iteration": 2.4465813636779785 + }, + { + "auxiliary_loss_clip": 0.01118102, + "auxiliary_loss_mlp": 0.01045247, + "balance_loss_clip": 1.03738523, + "balance_loss_mlp": 1.02237105, + "epoch": 0.07013522140328478, + "flos": 15556598807040.0, + "grad_norm": 2.739237745094355, + "language_loss": 0.94380343, + "learning_rate": 3.9831301262991105e-06, + "loss": 0.96543694, + "num_input_tokens_seen": 67946980, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.22875977, + "step": 2417, + "time_per_iteration": 2.3944499492645264 + }, + { + "auxiliary_loss_clip": 0.01025217, + "auxiliary_loss_mlp": 0.01002894, + "balance_loss_clip": 1.00938141, + "balance_loss_mlp": 1.00114155, + "epoch": 0.07016423887180083, + "flos": 61093009895040.0, + "grad_norm": 0.734463933616755, + "language_loss": 0.4921138, + "learning_rate": 3.983105755838361e-06, + "loss": 0.51239491, + "num_input_tokens_seen": 68007435, + "router_z_loss_clip": 0.15820312, + "router_z_loss_mlp": 0.01757812, + "step": 2418, + "time_per_iteration": 2.9554786682128906 + }, + { + "auxiliary_loss_clip": 0.01024329, + "auxiliary_loss_mlp": 0.01002793, + "balance_loss_clip": 1.00831592, + "balance_loss_mlp": 1.00113559, + "epoch": 0.07019325634031687, + "flos": 66997486517760.0, + "grad_norm": 0.7026819140720837, + "language_loss": 0.52528638, + "learning_rate": 3.983081367862019e-06, + "loss": 0.54555762, + "num_input_tokens_seen": 68071090, + "router_z_loss_clip": 0.16015625, + "router_z_loss_mlp": 0.01660156, + "step": 2419, + "time_per_iteration": 3.0703399181365967 + }, + { + "auxiliary_loss_clip": 0.01023968, + "auxiliary_loss_mlp": 0.01003689, + "balance_loss_clip": 1.00817156, + "balance_loss_mlp": 1.00201976, + "epoch": 0.07022227380883292, + "flos": 74780359309440.0, + "grad_norm": 2.26909655659139, + "language_loss": 0.51736361, + "learning_rate": 3.983056962370301e-06, + "loss": 0.53764015, + "num_input_tokens_seen": 68142680, + "router_z_loss_clip": 0.15820312, + "router_z_loss_mlp": 0.01672363, + "step": 2420, + "time_per_iteration": 3.2180185317993164 + }, + { + "auxiliary_loss_clip": 0.01125457, + "auxiliary_loss_mlp": 0.0105362, + "balance_loss_clip": 1.03915477, + "balance_loss_mlp": 1.02886057, + "epoch": 0.07025129127734897, + "flos": 27593324962560.0, + "grad_norm": 2.767208401925969, + "language_loss": 1.11987805, + "learning_rate": 3.9830325393634205e-06, + "loss": 1.14166868, + "num_input_tokens_seen": 68159435, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.24743652, + "step": 2421, + "time_per_iteration": 2.486114025115967 + }, + { + "auxiliary_loss_clip": 0.01023081, + "auxiliary_loss_mlp": 0.01002216, + "balance_loss_clip": 1.00724542, + "balance_loss_mlp": 1.00040376, + "epoch": 0.07028030874586501, + "flos": 66208413813120.0, + "grad_norm": 0.7121032667208267, + "language_loss": 0.51996434, + "learning_rate": 3.983008098841594e-06, + "loss": 0.54021728, + "num_input_tokens_seen": 68226570, + "router_z_loss_clip": 0.15820312, + "router_z_loss_mlp": 0.01806641, + "step": 2422, + "time_per_iteration": 3.078472852706909 + }, + { + "auxiliary_loss_clip": 0.01023066, + "auxiliary_loss_mlp": 0.01001999, + "balance_loss_clip": 1.00709033, + "balance_loss_mlp": 1.00033045, + "epoch": 0.07030932621438106, + "flos": 63715371425280.0, + "grad_norm": 0.708153762125222, + "language_loss": 0.51627529, + "learning_rate": 3.9829836408050385e-06, + "loss": 0.53652596, + "num_input_tokens_seen": 68286765, + "router_z_loss_clip": 0.16015625, + "router_z_loss_mlp": 0.01672363, + "step": 2423, + "time_per_iteration": 2.9255940914154053 + }, + { + "auxiliary_loss_clip": 0.01112108, + "auxiliary_loss_mlp": 0.0104231, + "balance_loss_clip": 1.03664565, + "balance_loss_mlp": 1.02092397, + "epoch": 0.0703383436828971, + "flos": 15879720240000.0, + "grad_norm": 2.9173862109110584, + "language_loss": 0.66642761, + "learning_rate": 3.982959165253967e-06, + "loss": 0.68797177, + "num_input_tokens_seen": 68297915, + "router_z_loss_clip": 0.75488281, + "router_z_loss_mlp": 0.21398926, + "step": 2424, + "time_per_iteration": 2.424764633178711 + }, + { + "auxiliary_loss_clip": 0.01116143, + "auxiliary_loss_mlp": 0.01047417, + "balance_loss_clip": 1.0377996, + "balance_loss_mlp": 1.02582788, + "epoch": 0.07036736115141315, + "flos": 11646710824320.0, + "grad_norm": 3.7634633197331344, + "language_loss": 0.85262817, + "learning_rate": 3.9829346721886e-06, + "loss": 0.87426376, + "num_input_tokens_seen": 68308665, + "router_z_loss_clip": 0.78222656, + "router_z_loss_mlp": 0.21594238, + "step": 2425, + "time_per_iteration": 2.329908847808838 + }, + { + "auxiliary_loss_clip": 0.01018507, + "auxiliary_loss_mlp": 0.01001603, + "balance_loss_clip": 1.0028547, + "balance_loss_mlp": 0.99976766, + "epoch": 0.0703963786199292, + "flos": 70904476857600.0, + "grad_norm": 0.8065193310398712, + "language_loss": 0.58048731, + "learning_rate": 3.982910161609151e-06, + "loss": 0.6006884, + "num_input_tokens_seen": 68365385, + "router_z_loss_clip": 0.15625, + "router_z_loss_mlp": 0.01831055, + "step": 2426, + "time_per_iteration": 3.0337605476379395 + }, + { + "auxiliary_loss_clip": 0.01118906, + "auxiliary_loss_mlp": 0.01047085, + "balance_loss_clip": 1.0355494, + "balance_loss_mlp": 1.02575886, + "epoch": 0.07042539608844524, + "flos": 16157069147520.0, + "grad_norm": 4.963272690960948, + "language_loss": 0.8286705, + "learning_rate": 3.982885633515837e-06, + "loss": 0.85033047, + "num_input_tokens_seen": 68382125, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.21337891, + "step": 2427, + "time_per_iteration": 2.6670210361480713 + }, + { + "auxiliary_loss_clip": 0.01110947, + "auxiliary_loss_mlp": 0.01043994, + "balance_loss_clip": 1.03825331, + "balance_loss_mlp": 1.02438378, + "epoch": 0.07045441355696129, + "flos": 36896805509760.0, + "grad_norm": 2.3585336748952854, + "language_loss": 0.70958722, + "learning_rate": 3.982861087908874e-06, + "loss": 0.73113656, + "num_input_tokens_seen": 68398300, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.19604492, + "step": 2428, + "time_per_iteration": 2.5512731075286865 + }, + { + "auxiliary_loss_clip": 0.01130921, + "auxiliary_loss_mlp": 0.01051186, + "balance_loss_clip": 1.04441047, + "balance_loss_mlp": 1.02705836, + "epoch": 0.07048343102547734, + "flos": 30145301458560.0, + "grad_norm": 2.1229839232139334, + "language_loss": 0.9180128, + "learning_rate": 3.98283652478848e-06, + "loss": 0.93983388, + "num_input_tokens_seen": 68416270, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.24169922, + "step": 2429, + "time_per_iteration": 2.444275140762329 + }, + { + "auxiliary_loss_clip": 0.01121539, + "auxiliary_loss_mlp": 0.0105102, + "balance_loss_clip": 1.03641713, + "balance_loss_mlp": 1.02591491, + "epoch": 0.07051244849399338, + "flos": 18979449056640.0, + "grad_norm": 3.8410142338340867, + "language_loss": 0.86633408, + "learning_rate": 3.982811944154872e-06, + "loss": 0.88805968, + "num_input_tokens_seen": 68427270, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.25109863, + "step": 2430, + "time_per_iteration": 2.383622646331787 + }, + { + "auxiliary_loss_clip": 0.01023237, + "auxiliary_loss_mlp": 0.01005137, + "balance_loss_clip": 1.00766087, + "balance_loss_mlp": 1.0033133, + "epoch": 0.07054146596250943, + "flos": 74776833262080.0, + "grad_norm": 0.6996327127791938, + "language_loss": 0.58457696, + "learning_rate": 3.982787346008265e-06, + "loss": 0.60486078, + "num_input_tokens_seen": 68487990, + "router_z_loss_clip": 0.15625, + "router_z_loss_mlp": 0.01818848, + "step": 2431, + "time_per_iteration": 3.0951790809631348 + }, + { + "auxiliary_loss_clip": 0.01121048, + "auxiliary_loss_mlp": 0.01043736, + "balance_loss_clip": 1.03821051, + "balance_loss_mlp": 1.01982212, + "epoch": 0.07057048343102548, + "flos": 21062506815360.0, + "grad_norm": 2.2957381031839708, + "language_loss": 0.81896067, + "learning_rate": 3.98276273034888e-06, + "loss": 0.84060848, + "num_input_tokens_seen": 68502085, + "router_z_loss_clip": 0.82763672, + "router_z_loss_mlp": 0.2388916, + "step": 2432, + "time_per_iteration": 2.4143669605255127 + }, + { + "auxiliary_loss_clip": 0.01115747, + "auxiliary_loss_mlp": 0.01053754, + "balance_loss_clip": 1.04059398, + "balance_loss_mlp": 1.03456783, + "epoch": 0.07059950089954152, + "flos": 28611217918080.0, + "grad_norm": 2.514268332548287, + "language_loss": 0.88511759, + "learning_rate": 3.98273809717693e-06, + "loss": 0.90681255, + "num_input_tokens_seen": 68515850, + "router_z_loss_clip": 0.75146484, + "router_z_loss_mlp": 0.19189453, + "step": 2433, + "time_per_iteration": 2.441790819168091 + }, + { + "auxiliary_loss_clip": 0.01025492, + "auxiliary_loss_mlp": 0.0100664, + "balance_loss_clip": 1.00973177, + "balance_loss_mlp": 1.00479281, + "epoch": 0.07062851836805757, + "flos": 57112317941760.0, + "grad_norm": 0.6501787003124844, + "language_loss": 0.52884054, + "learning_rate": 3.982713446492637e-06, + "loss": 0.54916191, + "num_input_tokens_seen": 68573430, + "router_z_loss_clip": 0.15820312, + "router_z_loss_mlp": 0.01843262, + "step": 2434, + "time_per_iteration": 2.895941734313965 + }, + { + "auxiliary_loss_clip": 0.01127147, + "auxiliary_loss_mlp": 0.01055144, + "balance_loss_clip": 1.03915811, + "balance_loss_mlp": 1.02890623, + "epoch": 0.07065753583657362, + "flos": 25329382116480.0, + "grad_norm": 2.493802897173159, + "language_loss": 1.00185323, + "learning_rate": 3.982688778296215e-06, + "loss": 1.02367616, + "num_input_tokens_seen": 68588735, + "router_z_loss_clip": 0.87988281, + "router_z_loss_mlp": 0.2623291, + "step": 2435, + "time_per_iteration": 2.461374521255493 + }, + { + "auxiliary_loss_clip": 0.01118417, + "auxiliary_loss_mlp": 0.01053504, + "balance_loss_clip": 1.03974235, + "balance_loss_mlp": 1.0303477, + "epoch": 0.07068655330508966, + "flos": 29708153925120.0, + "grad_norm": 2.4287099090222535, + "language_loss": 0.70867878, + "learning_rate": 3.982664092587884e-06, + "loss": 0.730398, + "num_input_tokens_seen": 68603985, + "router_z_loss_clip": 0.78637695, + "router_z_loss_mlp": 0.23156738, + "step": 2436, + "time_per_iteration": 2.45782470703125 + }, + { + "auxiliary_loss_clip": 0.01120033, + "auxiliary_loss_mlp": 0.01043666, + "balance_loss_clip": 1.03640318, + "balance_loss_mlp": 1.02092087, + "epoch": 0.07071557077360571, + "flos": 16864969207680.0, + "grad_norm": 2.9386069783005007, + "language_loss": 0.93088567, + "learning_rate": 3.98263938936786e-06, + "loss": 0.95252264, + "num_input_tokens_seen": 68619415, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.22741699, + "step": 2437, + "time_per_iteration": 4.908581733703613 + }, + { + "auxiliary_loss_clip": 0.01116667, + "auxiliary_loss_mlp": 0.01035681, + "balance_loss_clip": 1.03675652, + "balance_loss_mlp": 1.0156002, + "epoch": 0.07074458824212176, + "flos": 39302077006080.0, + "grad_norm": 2.6663675760474495, + "language_loss": 0.96227181, + "learning_rate": 3.982614668636365e-06, + "loss": 0.98379534, + "num_input_tokens_seen": 68638860, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.20080566, + "step": 2438, + "time_per_iteration": 2.575517177581787 + }, + { + "auxiliary_loss_clip": 0.01024973, + "auxiliary_loss_mlp": 0.01001384, + "balance_loss_clip": 1.00930858, + "balance_loss_mlp": 0.99978679, + "epoch": 0.0707736057106378, + "flos": 67103832119040.0, + "grad_norm": 0.7274951212390275, + "language_loss": 0.53026545, + "learning_rate": 3.982589930393613e-06, + "loss": 0.550529, + "num_input_tokens_seen": 68694985, + "router_z_loss_clip": 0.15625, + "router_z_loss_mlp": 0.01599121, + "step": 2439, + "time_per_iteration": 3.1178550720214844 + }, + { + "auxiliary_loss_clip": 0.01023721, + "auxiliary_loss_mlp": 0.01003002, + "balance_loss_clip": 1.00811386, + "balance_loss_mlp": 1.00147641, + "epoch": 0.07080262317915385, + "flos": 59176209363840.0, + "grad_norm": 0.7353264288960866, + "language_loss": 0.4550364, + "learning_rate": 3.982565174639825e-06, + "loss": 0.47530365, + "num_input_tokens_seen": 68740160, + "router_z_loss_clip": 0.15625, + "router_z_loss_mlp": 0.01525879, + "step": 2440, + "time_per_iteration": 2.7287676334381104 + }, + { + "auxiliary_loss_clip": 0.01120231, + "auxiliary_loss_mlp": 0.01046797, + "balance_loss_clip": 1.03798831, + "balance_loss_mlp": 1.02703261, + "epoch": 0.07083164064766989, + "flos": 14931933027840.0, + "grad_norm": 2.4483470899615605, + "language_loss": 0.69867951, + "learning_rate": 3.982540401375219e-06, + "loss": 0.72034979, + "num_input_tokens_seen": 68753730, + "router_z_loss_clip": 0.82177734, + "router_z_loss_mlp": 0.19763184, + "step": 2441, + "time_per_iteration": 4.8403003215789795 + }, + { + "auxiliary_loss_clip": 0.01120494, + "auxiliary_loss_mlp": 0.0104613, + "balance_loss_clip": 1.03559923, + "balance_loss_mlp": 1.02122724, + "epoch": 0.07086065811618594, + "flos": 39815020834560.0, + "grad_norm": 2.9549885924861807, + "language_loss": 0.64630866, + "learning_rate": 3.982515610600015e-06, + "loss": 0.66797489, + "num_input_tokens_seen": 68769855, + "router_z_loss_clip": 0.84912109, + "router_z_loss_mlp": 0.24914551, + "step": 2442, + "time_per_iteration": 2.5122294425964355 + }, + { + "auxiliary_loss_clip": 0.011238, + "auxiliary_loss_mlp": 0.01048226, + "balance_loss_clip": 1.03752935, + "balance_loss_mlp": 1.02461088, + "epoch": 0.070889675584702, + "flos": 36091080086400.0, + "grad_norm": 3.80476637636194, + "language_loss": 0.99877131, + "learning_rate": 3.98249080231443e-06, + "loss": 1.02049148, + "num_input_tokens_seen": 68785715, + "router_z_loss_clip": 0.86376953, + "router_z_loss_mlp": 0.23620605, + "step": 2443, + "time_per_iteration": 2.5909876823425293 + }, + { + "auxiliary_loss_clip": 0.01017494, + "auxiliary_loss_mlp": 0.01010161, + "balance_loss_clip": 1.00242269, + "balance_loss_mlp": 1.00850368, + "epoch": 0.07091869305321803, + "flos": 74780498954880.0, + "grad_norm": 0.6856778119903857, + "language_loss": 0.58018696, + "learning_rate": 3.982465976518685e-06, + "loss": 0.60046351, + "num_input_tokens_seen": 68854945, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01660156, + "step": 2444, + "time_per_iteration": 3.167667865753174 + }, + { + "auxiliary_loss_clip": 0.01017672, + "auxiliary_loss_mlp": 0.01019966, + "balance_loss_clip": 1.00292039, + "balance_loss_mlp": 1.01842797, + "epoch": 0.07094771052173408, + "flos": 74768803649280.0, + "grad_norm": 0.6814926365430501, + "language_loss": 0.5267247, + "learning_rate": 3.982441133212997e-06, + "loss": 0.54710102, + "num_input_tokens_seen": 68918040, + "router_z_loss_clip": 0.14746094, + "router_z_loss_mlp": 0.01531982, + "step": 2445, + "time_per_iteration": 5.63121771812439 + }, + { + "auxiliary_loss_clip": 0.01128304, + "auxiliary_loss_mlp": 0.0104875, + "balance_loss_clip": 1.04008675, + "balance_loss_mlp": 1.02527809, + "epoch": 0.07097672799025014, + "flos": 16792629137280.0, + "grad_norm": 2.1514166176922807, + "language_loss": 0.73348957, + "learning_rate": 3.982416272397587e-06, + "loss": 0.75526011, + "num_input_tokens_seen": 68931285, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.23474121, + "step": 2446, + "time_per_iteration": 2.4440274238586426 + }, + { + "auxiliary_loss_clip": 0.01022911, + "auxiliary_loss_mlp": 0.01001772, + "balance_loss_clip": 1.00823593, + "balance_loss_mlp": 1.00022185, + "epoch": 0.07100574545876617, + "flos": 57329373064320.0, + "grad_norm": 0.6631788938110597, + "language_loss": 0.49313566, + "learning_rate": 3.982391394072675e-06, + "loss": 0.51338255, + "num_input_tokens_seen": 68991230, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01550293, + "step": 2447, + "time_per_iteration": 2.9926652908325195 + }, + { + "auxiliary_loss_clip": 0.01122676, + "auxiliary_loss_mlp": 0.01050491, + "balance_loss_clip": 1.04001284, + "balance_loss_mlp": 1.02701914, + "epoch": 0.07103476292728222, + "flos": 26714142305280.0, + "grad_norm": 3.2803576503511627, + "language_loss": 0.90729713, + "learning_rate": 3.982366498238478e-06, + "loss": 0.92902887, + "num_input_tokens_seen": 69004245, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.23486328, + "step": 2448, + "time_per_iteration": 2.5251171588897705 + }, + { + "auxiliary_loss_clip": 0.0112252, + "auxiliary_loss_mlp": 0.01048084, + "balance_loss_clip": 1.04562545, + "balance_loss_mlp": 1.02595854, + "epoch": 0.07106378039579828, + "flos": 28322939888640.0, + "grad_norm": 2.5519864554349954, + "language_loss": 0.78077072, + "learning_rate": 3.98234158489522e-06, + "loss": 0.80247676, + "num_input_tokens_seen": 69020540, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.22131348, + "step": 2449, + "time_per_iteration": 2.475677728652954 + }, + { + "auxiliary_loss_clip": 0.01032419, + "auxiliary_loss_mlp": 0.01010949, + "balance_loss_clip": 1.01734257, + "balance_loss_mlp": 1.00923264, + "epoch": 0.07109279786431431, + "flos": 69552186099840.0, + "grad_norm": 0.6795578262086945, + "language_loss": 0.5001651, + "learning_rate": 3.982316654043118e-06, + "loss": 0.52059877, + "num_input_tokens_seen": 69080925, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01721191, + "step": 2450, + "time_per_iteration": 3.158191442489624 + }, + { + "auxiliary_loss_clip": 0.0112703, + "auxiliary_loss_mlp": 0.01049023, + "balance_loss_clip": 1.04333055, + "balance_loss_mlp": 1.02484751, + "epoch": 0.07112181533283037, + "flos": 39302984701440.0, + "grad_norm": 2.320896116726961, + "language_loss": 0.70608294, + "learning_rate": 3.982291705682393e-06, + "loss": 0.7278434, + "num_input_tokens_seen": 69096295, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.24194336, + "step": 2451, + "time_per_iteration": 2.6489169597625732 + }, + { + "auxiliary_loss_clip": 0.01126833, + "auxiliary_loss_mlp": 0.01060911, + "balance_loss_clip": 1.04285192, + "balance_loss_mlp": 1.03611553, + "epoch": 0.07115083280134642, + "flos": 43074581322240.0, + "grad_norm": 1.982366722139882, + "language_loss": 0.87529069, + "learning_rate": 3.9822667398132665e-06, + "loss": 0.89716816, + "num_input_tokens_seen": 69121620, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.24780273, + "step": 2452, + "time_per_iteration": 2.612773895263672 + }, + { + "auxiliary_loss_clip": 0.01026563, + "auxiliary_loss_mlp": 0.01012019, + "balance_loss_clip": 1.01155281, + "balance_loss_mlp": 1.01019514, + "epoch": 0.07117985026986245, + "flos": 70977759534720.0, + "grad_norm": 0.7374521127900964, + "language_loss": 0.48323452, + "learning_rate": 3.982241756435958e-06, + "loss": 0.50362033, + "num_input_tokens_seen": 69185190, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01818848, + "step": 2453, + "time_per_iteration": 3.2004833221435547 + }, + { + "auxiliary_loss_clip": 0.01021665, + "auxiliary_loss_mlp": 0.01010844, + "balance_loss_clip": 1.00728774, + "balance_loss_mlp": 1.00905561, + "epoch": 0.0712088677383785, + "flos": 71966988397440.0, + "grad_norm": 0.6946893804702762, + "language_loss": 0.55889642, + "learning_rate": 3.982216755550687e-06, + "loss": 0.57922149, + "num_input_tokens_seen": 69249070, + "router_z_loss_clip": 0.14355469, + "router_z_loss_mlp": 0.01782227, + "step": 2454, + "time_per_iteration": 3.03792667388916 + }, + { + "auxiliary_loss_clip": 0.01016652, + "auxiliary_loss_mlp": 0.01004799, + "balance_loss_clip": 1.00236392, + "balance_loss_mlp": 1.00301051, + "epoch": 0.07123788520689454, + "flos": 54367865786880.0, + "grad_norm": 0.7656057125056088, + "language_loss": 0.55512494, + "learning_rate": 3.982191737157677e-06, + "loss": 0.57533938, + "num_input_tokens_seen": 69300100, + "router_z_loss_clip": 0.14257812, + "router_z_loss_mlp": 0.01782227, + "step": 2455, + "time_per_iteration": 2.7909622192382812 + }, + { + "auxiliary_loss_clip": 0.01124281, + "auxiliary_loss_mlp": 0.01047071, + "balance_loss_clip": 1.03603292, + "balance_loss_mlp": 1.02249002, + "epoch": 0.0712669026754106, + "flos": 30956473054080.0, + "grad_norm": 2.313716389949131, + "language_loss": 0.86510527, + "learning_rate": 3.982166701257146e-06, + "loss": 0.88681883, + "num_input_tokens_seen": 69322415, + "router_z_loss_clip": 0.88183594, + "router_z_loss_mlp": 0.24572754, + "step": 2456, + "time_per_iteration": 2.471219778060913 + }, + { + "auxiliary_loss_clip": 0.01114612, + "auxiliary_loss_mlp": 0.01039974, + "balance_loss_clip": 1.03416014, + "balance_loss_mlp": 1.01960146, + "epoch": 0.07129592014392665, + "flos": 28979935453440.0, + "grad_norm": 2.3990303457283555, + "language_loss": 0.69338965, + "learning_rate": 3.982141647849318e-06, + "loss": 0.71493548, + "num_input_tokens_seen": 69339720, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.20385742, + "step": 2457, + "time_per_iteration": 2.4743752479553223 + }, + { + "auxiliary_loss_clip": 0.01113435, + "auxiliary_loss_mlp": 0.01048011, + "balance_loss_clip": 1.03514171, + "balance_loss_mlp": 1.02353752, + "epoch": 0.07132493761244268, + "flos": 34414166707200.0, + "grad_norm": 2.847463593228489, + "language_loss": 1.07529736, + "learning_rate": 3.9821165769344115e-06, + "loss": 1.09691191, + "num_input_tokens_seen": 69355595, + "router_z_loss_clip": 0.78320312, + "router_z_loss_mlp": 0.24475098, + "step": 2458, + "time_per_iteration": 2.4948818683624268 + }, + { + "auxiliary_loss_clip": 0.01123732, + "auxiliary_loss_mlp": 0.01049267, + "balance_loss_clip": 1.03920841, + "balance_loss_mlp": 1.0239234, + "epoch": 0.07135395508095874, + "flos": 30767767822080.0, + "grad_norm": 3.686553749006841, + "language_loss": 0.94597304, + "learning_rate": 3.98209148851265e-06, + "loss": 0.96770298, + "num_input_tokens_seen": 69372860, + "router_z_loss_clip": 0.84570312, + "router_z_loss_mlp": 0.25366211, + "step": 2459, + "time_per_iteration": 2.431093692779541 + }, + { + "auxiliary_loss_clip": 0.0111785, + "auxiliary_loss_mlp": 0.01050735, + "balance_loss_clip": 1.04084361, + "balance_loss_mlp": 1.03061259, + "epoch": 0.07138297254947479, + "flos": 16791058126080.0, + "grad_norm": 2.578765074349787, + "language_loss": 0.92571396, + "learning_rate": 3.982066382584254e-06, + "loss": 0.94739991, + "num_input_tokens_seen": 69385140, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.20141602, + "step": 2460, + "time_per_iteration": 2.3721306324005127 + }, + { + "auxiliary_loss_clip": 0.01126766, + "auxiliary_loss_mlp": 0.01060163, + "balance_loss_clip": 1.04121935, + "balance_loss_mlp": 1.0357132, + "epoch": 0.07141199001799083, + "flos": 30876382661760.0, + "grad_norm": 2.406160035381754, + "language_loss": 1.1819973, + "learning_rate": 3.982041259149446e-06, + "loss": 1.2038666, + "num_input_tokens_seen": 69402850, + "router_z_loss_clip": 0.85498047, + "router_z_loss_mlp": 0.24462891, + "step": 2461, + "time_per_iteration": 2.50882625579834 + }, + { + "auxiliary_loss_clip": 0.01111745, + "auxiliary_loss_mlp": 0.01048076, + "balance_loss_clip": 1.03730989, + "balance_loss_mlp": 1.02841806, + "epoch": 0.07144100748650688, + "flos": 15917007438720.0, + "grad_norm": 3.2319196474062273, + "language_loss": 0.78655463, + "learning_rate": 3.982016118208448e-06, + "loss": 0.80815285, + "num_input_tokens_seen": 69415360, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.19677734, + "step": 2462, + "time_per_iteration": 2.3571486473083496 + }, + { + "auxiliary_loss_clip": 0.010429, + "auxiliary_loss_mlp": 0.01013311, + "balance_loss_clip": 1.02707505, + "balance_loss_mlp": 1.01167738, + "epoch": 0.07147002495502293, + "flos": 61416201150720.0, + "grad_norm": 0.7415876821622351, + "language_loss": 0.51584738, + "learning_rate": 3.98199095976148e-06, + "loss": 0.5364095, + "num_input_tokens_seen": 69474755, + "router_z_loss_clip": 0.15820312, + "router_z_loss_mlp": 0.01635742, + "step": 2463, + "time_per_iteration": 3.0812339782714844 + }, + { + "auxiliary_loss_clip": 0.011201, + "auxiliary_loss_mlp": 0.01048619, + "balance_loss_clip": 1.03870201, + "balance_loss_mlp": 1.0258981, + "epoch": 0.07149904242353897, + "flos": 39230330428800.0, + "grad_norm": 3.339061109876392, + "language_loss": 0.92425179, + "learning_rate": 3.981965783808768e-06, + "loss": 0.94593894, + "num_input_tokens_seen": 69491895, + "router_z_loss_clip": 0.81396484, + "router_z_loss_mlp": 0.22729492, + "step": 2464, + "time_per_iteration": 2.601099729537964 + }, + { + "auxiliary_loss_clip": 0.01120175, + "auxiliary_loss_mlp": 0.01042649, + "balance_loss_clip": 1.03860867, + "balance_loss_mlp": 1.02071452, + "epoch": 0.07152805989205502, + "flos": 27043827073920.0, + "grad_norm": 3.0308424869336044, + "language_loss": 0.81855106, + "learning_rate": 3.981940590350531e-06, + "loss": 0.84017932, + "num_input_tokens_seen": 69507155, + "router_z_loss_clip": 0.81445312, + "router_z_loss_mlp": 0.21936035, + "step": 2465, + "time_per_iteration": 2.3989362716674805 + }, + { + "auxiliary_loss_clip": 0.01026545, + "auxiliary_loss_mlp": 0.0100362, + "balance_loss_clip": 1.01200652, + "balance_loss_mlp": 1.00207043, + "epoch": 0.07155707736057107, + "flos": 55099715040000.0, + "grad_norm": 0.7212160025265903, + "language_loss": 0.52161336, + "learning_rate": 3.981915379386992e-06, + "loss": 0.541915, + "num_input_tokens_seen": 69558720, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01544189, + "step": 2466, + "time_per_iteration": 2.84279203414917 + }, + { + "auxiliary_loss_clip": 0.01115408, + "auxiliary_loss_mlp": 0.01040353, + "balance_loss_clip": 1.03629088, + "balance_loss_mlp": 1.01833463, + "epoch": 0.07158609482908711, + "flos": 15879859885440.0, + "grad_norm": 2.5864509076231315, + "language_loss": 0.95991045, + "learning_rate": 3.981890150918376e-06, + "loss": 0.98146808, + "num_input_tokens_seen": 69570470, + "router_z_loss_clip": 0.79101562, + "router_z_loss_mlp": 0.22021484, + "step": 2467, + "time_per_iteration": 2.459059953689575 + }, + { + "auxiliary_loss_clip": 0.01121841, + "auxiliary_loss_mlp": 0.01041081, + "balance_loss_clip": 1.03597879, + "balance_loss_mlp": 1.01604676, + "epoch": 0.07161511229760316, + "flos": 30437000801280.0, + "grad_norm": 2.6012024918326166, + "language_loss": 0.82241267, + "learning_rate": 3.981864904944903e-06, + "loss": 0.84404182, + "num_input_tokens_seen": 69585825, + "router_z_loss_clip": 0.85888672, + "router_z_loss_mlp": 0.25, + "step": 2468, + "time_per_iteration": 2.5259716510772705 + }, + { + "auxiliary_loss_clip": 0.01017486, + "auxiliary_loss_mlp": 0.01003751, + "balance_loss_clip": 1.00330126, + "balance_loss_mlp": 1.0020225, + "epoch": 0.07164412976611921, + "flos": 74768210156160.0, + "grad_norm": 0.6456314287739804, + "language_loss": 0.49196628, + "learning_rate": 3.981839641466798e-06, + "loss": 0.51217866, + "num_input_tokens_seen": 69648430, + "router_z_loss_clip": 0.14160156, + "router_z_loss_mlp": 0.01733398, + "step": 2469, + "time_per_iteration": 3.0432016849517822 + }, + { + "auxiliary_loss_clip": 0.01122399, + "auxiliary_loss_mlp": 0.01058101, + "balance_loss_clip": 1.0373944, + "balance_loss_mlp": 1.03257883, + "epoch": 0.07167314723463525, + "flos": 32298080935680.0, + "grad_norm": 3.6452344388852143, + "language_loss": 0.92925, + "learning_rate": 3.981814360484283e-06, + "loss": 0.95105505, + "num_input_tokens_seen": 69664895, + "router_z_loss_clip": 0.85058594, + "router_z_loss_mlp": 0.25537109, + "step": 2470, + "time_per_iteration": 2.5302319526672363 + }, + { + "auxiliary_loss_clip": 0.01120533, + "auxiliary_loss_mlp": 0.01041909, + "balance_loss_clip": 1.03705418, + "balance_loss_mlp": 1.02004623, + "epoch": 0.0717021647031513, + "flos": 22884414537600.0, + "grad_norm": 1.9043995104769007, + "language_loss": 0.84446669, + "learning_rate": 3.981789061997581e-06, + "loss": 0.86609113, + "num_input_tokens_seen": 69681975, + "router_z_loss_clip": 0.83349609, + "router_z_loss_mlp": 0.21862793, + "step": 2471, + "time_per_iteration": 2.3508031368255615 + }, + { + "auxiliary_loss_clip": 0.01124724, + "auxiliary_loss_mlp": 0.01046831, + "balance_loss_clip": 1.04002714, + "balance_loss_mlp": 1.02338219, + "epoch": 0.07173118217166734, + "flos": 23432585794560.0, + "grad_norm": 2.142058683601404, + "language_loss": 0.92227638, + "learning_rate": 3.981763746006916e-06, + "loss": 0.94399196, + "num_input_tokens_seen": 69698630, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.23474121, + "step": 2472, + "time_per_iteration": 2.4575724601745605 + }, + { + "auxiliary_loss_clip": 0.01124968, + "auxiliary_loss_mlp": 0.01056979, + "balance_loss_clip": 1.04450202, + "balance_loss_mlp": 1.03471041, + "epoch": 0.07176019964018339, + "flos": 16793257541760.0, + "grad_norm": 2.4969715961596393, + "language_loss": 0.81166464, + "learning_rate": 3.981738412512513e-06, + "loss": 0.83348411, + "num_input_tokens_seen": 69713210, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.22265625, + "step": 2473, + "time_per_iteration": 2.3758528232574463 + }, + { + "auxiliary_loss_clip": 0.01125572, + "auxiliary_loss_mlp": 0.01047658, + "balance_loss_clip": 1.04289937, + "balance_loss_mlp": 1.02441859, + "epoch": 0.07178921710869944, + "flos": 27481603011840.0, + "grad_norm": 2.6503447311461743, + "language_loss": 0.89096153, + "learning_rate": 3.981713061514593e-06, + "loss": 0.91269386, + "num_input_tokens_seen": 69732020, + "router_z_loss_clip": 0.82666016, + "router_z_loss_mlp": 0.23236084, + "step": 2474, + "time_per_iteration": 2.6560862064361572 + }, + { + "auxiliary_loss_clip": 0.01023729, + "auxiliary_loss_mlp": 0.01004877, + "balance_loss_clip": 1.0096699, + "balance_loss_mlp": 1.00324416, + "epoch": 0.07181823457721548, + "flos": 69792876213120.0, + "grad_norm": 0.6567155717214365, + "language_loss": 0.50354016, + "learning_rate": 3.981687693013383e-06, + "loss": 0.52382618, + "num_input_tokens_seen": 69799915, + "router_z_loss_clip": 0.140625, + "router_z_loss_mlp": 0.01635742, + "step": 2475, + "time_per_iteration": 3.1754202842712402 + }, + { + "auxiliary_loss_clip": 0.01110733, + "auxiliary_loss_mlp": 0.01038044, + "balance_loss_clip": 1.03884077, + "balance_loss_mlp": 1.01974487, + "epoch": 0.07184725204573153, + "flos": 42879906247680.0, + "grad_norm": 5.884837113199282, + "language_loss": 0.68604589, + "learning_rate": 3.981662307009104e-06, + "loss": 0.7075336, + "num_input_tokens_seen": 69819485, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.1829834, + "step": 2476, + "time_per_iteration": 2.7446038722991943 + }, + { + "auxiliary_loss_clip": 0.01109367, + "auxiliary_loss_mlp": 0.01042087, + "balance_loss_clip": 1.03615248, + "balance_loss_mlp": 1.02272749, + "epoch": 0.07187626951424758, + "flos": 30073554881280.0, + "grad_norm": 2.1487314498529475, + "language_loss": 0.76598251, + "learning_rate": 3.981636903501982e-06, + "loss": 0.78749704, + "num_input_tokens_seen": 69834970, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.19372559, + "step": 2477, + "time_per_iteration": 2.5148253440856934 + }, + { + "auxiliary_loss_clip": 0.01130152, + "auxiliary_loss_mlp": 0.01053471, + "balance_loss_clip": 1.04285622, + "balance_loss_mlp": 1.02799571, + "epoch": 0.07190528698276362, + "flos": 20622636195840.0, + "grad_norm": 2.7505218064611308, + "language_loss": 0.93316627, + "learning_rate": 3.9816114824922406e-06, + "loss": 0.95500255, + "num_input_tokens_seen": 69846675, + "router_z_loss_clip": 0.87207031, + "router_z_loss_mlp": 0.2545166, + "step": 2478, + "time_per_iteration": 2.4081876277923584 + }, + { + "auxiliary_loss_clip": 0.01117349, + "auxiliary_loss_mlp": 0.01045679, + "balance_loss_clip": 1.03596878, + "balance_loss_mlp": 1.02401912, + "epoch": 0.07193430445127967, + "flos": 27228517499520.0, + "grad_norm": 2.174359949748159, + "language_loss": 0.85763031, + "learning_rate": 3.981586043980106e-06, + "loss": 0.87926054, + "num_input_tokens_seen": 69864930, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.2166748, + "step": 2479, + "time_per_iteration": 2.449871301651001 + }, + { + "auxiliary_loss_clip": 0.01119654, + "auxiliary_loss_mlp": 0.0105012, + "balance_loss_clip": 1.04118383, + "balance_loss_mlp": 1.03011703, + "epoch": 0.07196332191979572, + "flos": 31680711630720.0, + "grad_norm": 2.002587212259717, + "language_loss": 0.82987785, + "learning_rate": 3.9815605879658e-06, + "loss": 0.85157549, + "num_input_tokens_seen": 69884395, + "router_z_loss_clip": 0.78466797, + "router_z_loss_mlp": 0.20031738, + "step": 2480, + "time_per_iteration": 2.522440195083618 + }, + { + "auxiliary_loss_clip": 0.01020466, + "auxiliary_loss_mlp": 0.01001552, + "balance_loss_clip": 1.00630963, + "balance_loss_mlp": 0.9999786, + "epoch": 0.07199233938831176, + "flos": 60061676065920.0, + "grad_norm": 0.6981715873696291, + "language_loss": 0.52428865, + "learning_rate": 3.98153511444955e-06, + "loss": 0.54450887, + "num_input_tokens_seen": 69941095, + "router_z_loss_clip": 0.14160156, + "router_z_loss_mlp": 0.01574707, + "step": 2481, + "time_per_iteration": 2.884773015975952 + }, + { + "auxiliary_loss_clip": 0.01115337, + "auxiliary_loss_mlp": 0.0104503, + "balance_loss_clip": 1.03732753, + "balance_loss_mlp": 1.02384615, + "epoch": 0.07202135685682781, + "flos": 38975953196160.0, + "grad_norm": 2.683197295693188, + "language_loss": 0.983383, + "learning_rate": 3.981509623431579e-06, + "loss": 1.00498676, + "num_input_tokens_seen": 69964405, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.21179199, + "step": 2482, + "time_per_iteration": 2.6772501468658447 + }, + { + "auxiliary_loss_clip": 0.01018243, + "auxiliary_loss_mlp": 0.0100239, + "balance_loss_clip": 1.00410438, + "balance_loss_mlp": 1.0007447, + "epoch": 0.07205037432534386, + "flos": 72430810076160.0, + "grad_norm": 0.7132303648201024, + "language_loss": 0.47483557, + "learning_rate": 3.981484114912114e-06, + "loss": 0.49504191, + "num_input_tokens_seen": 70024980, + "router_z_loss_clip": 0.14160156, + "router_z_loss_mlp": 0.01647949, + "step": 2483, + "time_per_iteration": 3.043616771697998 + }, + { + "auxiliary_loss_clip": 0.0111961, + "auxiliary_loss_mlp": 0.01042294, + "balance_loss_clip": 1.03830385, + "balance_loss_mlp": 1.01894104, + "epoch": 0.0720793917938599, + "flos": 24235483397760.0, + "grad_norm": 3.3865777971453848, + "language_loss": 0.93282187, + "learning_rate": 3.98145858889138e-06, + "loss": 0.95444095, + "num_input_tokens_seen": 70038345, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.23352051, + "step": 2484, + "time_per_iteration": 2.449073553085327 + }, + { + "auxiliary_loss_clip": 0.01115963, + "auxiliary_loss_mlp": 0.0104358, + "balance_loss_clip": 1.03295398, + "balance_loss_mlp": 1.01934516, + "epoch": 0.07210840926237595, + "flos": 20311978118400.0, + "grad_norm": 2.3743713262982977, + "language_loss": 0.98261333, + "learning_rate": 3.981433045369601e-06, + "loss": 1.0042088, + "num_input_tokens_seen": 70055065, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.24230957, + "step": 2485, + "time_per_iteration": 2.430527925491333 + }, + { + "auxiliary_loss_clip": 0.0112692, + "auxiliary_loss_mlp": 0.01050527, + "balance_loss_clip": 1.03634024, + "balance_loss_mlp": 1.02476573, + "epoch": 0.07213742673089199, + "flos": 18983324217600.0, + "grad_norm": 2.0385198436600462, + "language_loss": 0.95891184, + "learning_rate": 3.981407484347004e-06, + "loss": 0.98068631, + "num_input_tokens_seen": 70071940, + "router_z_loss_clip": 0.90527344, + "router_z_loss_mlp": 0.25756836, + "step": 2486, + "time_per_iteration": 2.4693377017974854 + }, + { + "auxiliary_loss_clip": 0.01111918, + "auxiliary_loss_mlp": 0.01041213, + "balance_loss_clip": 1.03371048, + "balance_loss_mlp": 1.0194695, + "epoch": 0.07216644419940804, + "flos": 18333380747520.0, + "grad_norm": 3.0381737264739823, + "language_loss": 0.82841325, + "learning_rate": 3.981381905823814e-06, + "loss": 0.84994459, + "num_input_tokens_seen": 70087875, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.2175293, + "step": 2487, + "time_per_iteration": 2.3968045711517334 + }, + { + "auxiliary_loss_clip": 0.0111703, + "auxiliary_loss_mlp": 0.01052737, + "balance_loss_clip": 1.03396177, + "balance_loss_mlp": 1.02965808, + "epoch": 0.0721954616679241, + "flos": 35764746808320.0, + "grad_norm": 2.3072015809309856, + "language_loss": 0.86522037, + "learning_rate": 3.981356309800257e-06, + "loss": 0.88691801, + "num_input_tokens_seen": 70106815, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.23083496, + "step": 2488, + "time_per_iteration": 2.5157198905944824 + }, + { + "auxiliary_loss_clip": 0.01120524, + "auxiliary_loss_mlp": 0.01044972, + "balance_loss_clip": 1.03894663, + "balance_loss_mlp": 1.02303767, + "epoch": 0.07222447913644013, + "flos": 21243915573120.0, + "grad_norm": 2.351795115939233, + "language_loss": 0.8341397, + "learning_rate": 3.98133069627656e-06, + "loss": 0.85579461, + "num_input_tokens_seen": 70120375, + "router_z_loss_clip": 0.81591797, + "router_z_loss_mlp": 0.21960449, + "step": 2489, + "time_per_iteration": 2.4801506996154785 + }, + { + "auxiliary_loss_clip": 0.01110677, + "auxiliary_loss_mlp": 0.0104374, + "balance_loss_clip": 1.03479719, + "balance_loss_mlp": 1.02323604, + "epoch": 0.07225349660495618, + "flos": 10774510439040.0, + "grad_norm": 2.837690736892048, + "language_loss": 0.82455528, + "learning_rate": 3.981305065252948e-06, + "loss": 0.84609944, + "num_input_tokens_seen": 70132445, + "router_z_loss_clip": 0.75976562, + "router_z_loss_mlp": 0.20495605, + "step": 2490, + "time_per_iteration": 2.465275764465332 + }, + { + "auxiliary_loss_clip": 0.01116186, + "auxiliary_loss_mlp": 0.0104556, + "balance_loss_clip": 1.03633332, + "balance_loss_mlp": 1.02255213, + "epoch": 0.07228251407347223, + "flos": 13107337130880.0, + "grad_norm": 12.179416887371977, + "language_loss": 0.95554644, + "learning_rate": 3.981279416729649e-06, + "loss": 0.97716391, + "num_input_tokens_seen": 70144855, + "router_z_loss_clip": 0.79833984, + "router_z_loss_mlp": 0.23022461, + "step": 2491, + "time_per_iteration": 2.3870327472686768 + }, + { + "auxiliary_loss_clip": 0.01017112, + "auxiliary_loss_mlp": 0.01003801, + "balance_loss_clip": 1.00255275, + "balance_loss_mlp": 1.00226283, + "epoch": 0.07231153154198827, + "flos": 54737665574400.0, + "grad_norm": 0.7502467209364345, + "language_loss": 0.56318057, + "learning_rate": 3.981253750706887e-06, + "loss": 0.5833897, + "num_input_tokens_seen": 70199510, + "router_z_loss_clip": 0.14550781, + "router_z_loss_mlp": 0.01538086, + "step": 2492, + "time_per_iteration": 2.943577766418457 + }, + { + "auxiliary_loss_clip": 0.01115625, + "auxiliary_loss_mlp": 0.01049169, + "balance_loss_clip": 1.03526425, + "balance_loss_mlp": 1.02676964, + "epoch": 0.07234054901050432, + "flos": 18033616880640.0, + "grad_norm": 2.10797945226124, + "language_loss": 0.71216017, + "learning_rate": 3.981228067184891e-06, + "loss": 0.7338081, + "num_input_tokens_seen": 70215605, + "router_z_loss_clip": 0.80322266, + "router_z_loss_mlp": 0.22424316, + "step": 2493, + "time_per_iteration": 2.418030023574829 + }, + { + "auxiliary_loss_clip": 0.01016073, + "auxiliary_loss_mlp": 0.01001891, + "balance_loss_clip": 1.00174904, + "balance_loss_mlp": 1.00026965, + "epoch": 0.07236956647902038, + "flos": 67922439834240.0, + "grad_norm": 0.7497213885181888, + "language_loss": 0.53583372, + "learning_rate": 3.981202366163886e-06, + "loss": 0.55601335, + "num_input_tokens_seen": 70281495, + "router_z_loss_clip": 0.14355469, + "router_z_loss_mlp": 0.01623535, + "step": 2494, + "time_per_iteration": 3.168882131576538 + }, + { + "auxiliary_loss_clip": 0.011175, + "auxiliary_loss_mlp": 0.01050064, + "balance_loss_clip": 1.03464031, + "balance_loss_mlp": 1.02456498, + "epoch": 0.07239858394753641, + "flos": 62981953027200.0, + "grad_norm": 3.210888429621638, + "language_loss": 0.86566907, + "learning_rate": 3.981176647644101e-06, + "loss": 0.88734466, + "num_input_tokens_seen": 70300985, + "router_z_loss_clip": 0.82910156, + "router_z_loss_mlp": 0.25524902, + "step": 2495, + "time_per_iteration": 2.691514253616333 + }, + { + "auxiliary_loss_clip": 0.01130157, + "auxiliary_loss_mlp": 0.01051536, + "balance_loss_clip": 1.03898799, + "balance_loss_mlp": 1.0266211, + "epoch": 0.07242760141605246, + "flos": 48133042945920.0, + "grad_norm": 2.1620847188870767, + "language_loss": 0.91155541, + "learning_rate": 3.981150911625762e-06, + "loss": 0.93337238, + "num_input_tokens_seen": 70321255, + "router_z_loss_clip": 0.91162109, + "router_z_loss_mlp": 0.24914551, + "step": 2496, + "time_per_iteration": 2.634310007095337 + }, + { + "auxiliary_loss_clip": 0.01131645, + "auxiliary_loss_mlp": 0.01076625, + "balance_loss_clip": 1.04114366, + "balance_loss_mlp": 1.04703736, + "epoch": 0.07245661888456852, + "flos": 62911637815680.0, + "grad_norm": 2.2326036796165, + "language_loss": 0.97772998, + "learning_rate": 3.981125158109096e-06, + "loss": 0.99981266, + "num_input_tokens_seen": 70345955, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.2956543, + "step": 2497, + "time_per_iteration": 2.9111998081207275 + }, + { + "auxiliary_loss_clip": 0.0112103, + "auxiliary_loss_mlp": 0.01053847, + "balance_loss_clip": 1.03679776, + "balance_loss_mlp": 1.02881289, + "epoch": 0.07248563635308455, + "flos": 39265243655040.0, + "grad_norm": 2.9457651448320212, + "language_loss": 0.81528282, + "learning_rate": 3.981099387094332e-06, + "loss": 0.83703154, + "num_input_tokens_seen": 70360510, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.25012207, + "step": 2498, + "time_per_iteration": 2.615436553955078 + }, + { + "auxiliary_loss_clip": 0.01121313, + "auxiliary_loss_mlp": 0.01049817, + "balance_loss_clip": 1.03751254, + "balance_loss_mlp": 1.02446103, + "epoch": 0.0725146538216006, + "flos": 60207265768320.0, + "grad_norm": 2.0011909815755122, + "language_loss": 0.76203936, + "learning_rate": 3.981073598581696e-06, + "loss": 0.78375065, + "num_input_tokens_seen": 70381760, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.2532959, + "step": 2499, + "time_per_iteration": 2.748425245285034 + }, + { + "auxiliary_loss_clip": 0.01016, + "auxiliary_loss_mlp": 0.01003598, + "balance_loss_clip": 1.00162625, + "balance_loss_mlp": 1.00197637, + "epoch": 0.07254367129011666, + "flos": 59727767022720.0, + "grad_norm": 0.7031445720799993, + "language_loss": 0.50847101, + "learning_rate": 3.9810477925714154e-06, + "loss": 0.52866697, + "num_input_tokens_seen": 70434125, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01623535, + "step": 2500, + "time_per_iteration": 2.7776877880096436 + }, + { + "auxiliary_loss_clip": 0.01016278, + "auxiliary_loss_mlp": 0.0100179, + "balance_loss_clip": 1.00161326, + "balance_loss_mlp": 1.00015664, + "epoch": 0.0725726887586327, + "flos": 67253120559360.0, + "grad_norm": 0.6812256539597031, + "language_loss": 0.52221054, + "learning_rate": 3.98102196906372e-06, + "loss": 0.54239112, + "num_input_tokens_seen": 70498705, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01635742, + "step": 2501, + "time_per_iteration": 3.086294174194336 + }, + { + "auxiliary_loss_clip": 0.01117, + "auxiliary_loss_mlp": 0.01047764, + "balance_loss_clip": 1.03503585, + "balance_loss_mlp": 1.02438724, + "epoch": 0.07260170622714875, + "flos": 74733333707520.0, + "grad_norm": 1.8767095329795733, + "language_loss": 0.70942348, + "learning_rate": 3.980996128058837e-06, + "loss": 0.73107111, + "num_input_tokens_seen": 70523090, + "router_z_loss_clip": 0.81982422, + "router_z_loss_mlp": 0.23388672, + "step": 2502, + "time_per_iteration": 2.806422710418701 + }, + { + "auxiliary_loss_clip": 0.01115453, + "auxiliary_loss_mlp": 0.01041231, + "balance_loss_clip": 1.03472233, + "balance_loss_mlp": 1.01865244, + "epoch": 0.07263072369566478, + "flos": 27336608668800.0, + "grad_norm": 2.3944075085336243, + "language_loss": 0.80817449, + "learning_rate": 3.980970269556994e-06, + "loss": 0.8297413, + "num_input_tokens_seen": 70536330, + "router_z_loss_clip": 0.80615234, + "router_z_loss_mlp": 0.22576904, + "step": 2503, + "time_per_iteration": 2.457223653793335 + }, + { + "auxiliary_loss_clip": 0.0111065, + "auxiliary_loss_mlp": 0.01042656, + "balance_loss_clip": 1.03575087, + "balance_loss_mlp": 1.02199745, + "epoch": 0.07265974116418084, + "flos": 41713004142720.0, + "grad_norm": 2.12633997176548, + "language_loss": 0.57166803, + "learning_rate": 3.98094439355842e-06, + "loss": 0.5932011, + "num_input_tokens_seen": 70555270, + "router_z_loss_clip": 0.74975586, + "router_z_loss_mlp": 0.20654297, + "step": 2504, + "time_per_iteration": 2.4929327964782715 + }, + { + "auxiliary_loss_clip": 0.01113313, + "auxiliary_loss_mlp": 0.01040721, + "balance_loss_clip": 1.03200543, + "balance_loss_mlp": 1.01790428, + "epoch": 0.07268875863269689, + "flos": 20263170170880.0, + "grad_norm": 2.1971872672522315, + "language_loss": 0.77889574, + "learning_rate": 3.980918500063344e-06, + "loss": 0.80043602, + "num_input_tokens_seen": 70575110, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.22839355, + "step": 2505, + "time_per_iteration": 2.410581588745117 + }, + { + "auxiliary_loss_clip": 0.01122379, + "auxiliary_loss_mlp": 0.01049748, + "balance_loss_clip": 1.03563285, + "balance_loss_mlp": 1.02414155, + "epoch": 0.07271777610121292, + "flos": 38062520628480.0, + "grad_norm": 2.775915352202887, + "language_loss": 0.89997256, + "learning_rate": 3.980892589071993e-06, + "loss": 0.92169386, + "num_input_tokens_seen": 70590185, + "router_z_loss_clip": 0.86572266, + "router_z_loss_mlp": 0.25610352, + "step": 2506, + "time_per_iteration": 2.574922561645508 + }, + { + "auxiliary_loss_clip": 0.01017048, + "auxiliary_loss_mlp": 0.01001729, + "balance_loss_clip": 1.00236487, + "balance_loss_mlp": 1.00001264, + "epoch": 0.07274679356972898, + "flos": 64416880840320.0, + "grad_norm": 0.6533392955053805, + "language_loss": 0.48582989, + "learning_rate": 3.9808666605845985e-06, + "loss": 0.50601768, + "num_input_tokens_seen": 70652365, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01721191, + "step": 2507, + "time_per_iteration": 3.0635173320770264 + }, + { + "auxiliary_loss_clip": 0.01119897, + "auxiliary_loss_mlp": 0.01050038, + "balance_loss_clip": 1.03651404, + "balance_loss_mlp": 1.02784157, + "epoch": 0.07277581103824503, + "flos": 16500161744640.0, + "grad_norm": 2.531701386000843, + "language_loss": 0.87891006, + "learning_rate": 3.980840714601388e-06, + "loss": 0.90060949, + "num_input_tokens_seen": 70664340, + "router_z_loss_clip": 0.83398438, + "router_z_loss_mlp": 0.22180176, + "step": 2508, + "time_per_iteration": 2.384377956390381 + }, + { + "auxiliary_loss_clip": 0.01016895, + "auxiliary_loss_mlp": 0.01001603, + "balance_loss_clip": 1.00191545, + "balance_loss_mlp": 0.99983847, + "epoch": 0.07280482850676107, + "flos": 74769501876480.0, + "grad_norm": 0.646647451940212, + "language_loss": 0.47211498, + "learning_rate": 3.98081475112259e-06, + "loss": 0.49229997, + "num_input_tokens_seen": 70723695, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.0177002, + "step": 2509, + "time_per_iteration": 3.0295703411102295 + }, + { + "auxiliary_loss_clip": 0.01017069, + "auxiliary_loss_mlp": 0.01001936, + "balance_loss_clip": 1.00201774, + "balance_loss_mlp": 1.0002197, + "epoch": 0.07283384597527712, + "flos": 65070105978240.0, + "grad_norm": 0.7619170713323737, + "language_loss": 0.53170258, + "learning_rate": 3.980788770148435e-06, + "loss": 0.55189264, + "num_input_tokens_seen": 70788860, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01721191, + "step": 2510, + "time_per_iteration": 3.090522527694702 + }, + { + "auxiliary_loss_clip": 0.01112824, + "auxiliary_loss_mlp": 0.01044775, + "balance_loss_clip": 1.03301263, + "balance_loss_mlp": 1.02105188, + "epoch": 0.07286286344379317, + "flos": 25439044296960.0, + "grad_norm": 4.140568313548901, + "language_loss": 0.84697407, + "learning_rate": 3.980762771679152e-06, + "loss": 0.86855006, + "num_input_tokens_seen": 70805550, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.23730469, + "step": 2511, + "time_per_iteration": 2.4470136165618896 + }, + { + "auxiliary_loss_clip": 0.01111209, + "auxiliary_loss_mlp": 0.01042731, + "balance_loss_clip": 1.03259933, + "balance_loss_mlp": 1.02041495, + "epoch": 0.0728918809123092, + "flos": 31679070796800.0, + "grad_norm": 2.9097100569176737, + "language_loss": 0.96463341, + "learning_rate": 3.980736755714971e-06, + "loss": 0.98617274, + "num_input_tokens_seen": 70821520, + "router_z_loss_clip": 0.78613281, + "router_z_loss_mlp": 0.2232666, + "step": 2512, + "time_per_iteration": 4.838717460632324 + }, + { + "auxiliary_loss_clip": 0.010177, + "auxiliary_loss_mlp": 0.01001731, + "balance_loss_clip": 1.00267315, + "balance_loss_mlp": 0.99997884, + "epoch": 0.07292089838082526, + "flos": 69485257290240.0, + "grad_norm": 1.3473427287098978, + "language_loss": 0.53177065, + "learning_rate": 3.98071072225612e-06, + "loss": 0.551965, + "num_input_tokens_seen": 70888150, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01757812, + "step": 2513, + "time_per_iteration": 5.630423545837402 + }, + { + "auxiliary_loss_clip": 0.01124602, + "auxiliary_loss_mlp": 0.01054948, + "balance_loss_clip": 1.03757954, + "balance_loss_mlp": 1.02892494, + "epoch": 0.07294991584934131, + "flos": 39231936351360.0, + "grad_norm": 5.03994145131045, + "language_loss": 0.96193612, + "learning_rate": 3.980684671302832e-06, + "loss": 0.98373163, + "num_input_tokens_seen": 70904640, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.26013184, + "step": 2514, + "time_per_iteration": 2.6140832901000977 + }, + { + "auxiliary_loss_clip": 0.01118025, + "auxiliary_loss_mlp": 0.01051479, + "balance_loss_clip": 1.03555393, + "balance_loss_mlp": 1.02781558, + "epoch": 0.07297893331785735, + "flos": 10298888720640.0, + "grad_norm": 3.087041673447646, + "language_loss": 0.86048502, + "learning_rate": 3.980658602855335e-06, + "loss": 0.88218009, + "num_input_tokens_seen": 70915955, + "router_z_loss_clip": 0.82519531, + "router_z_loss_mlp": 0.23669434, + "step": 2515, + "time_per_iteration": 2.4874050617218018 + }, + { + "auxiliary_loss_clip": 0.01116684, + "auxiliary_loss_mlp": 0.0104801, + "balance_loss_clip": 1.03262115, + "balance_loss_mlp": 1.02370369, + "epoch": 0.0730079507863734, + "flos": 29636616816000.0, + "grad_norm": 4.0020267590350995, + "language_loss": 0.85791981, + "learning_rate": 3.98063251691386e-06, + "loss": 0.87956679, + "num_input_tokens_seen": 70932235, + "router_z_loss_clip": 0.84033203, + "router_z_loss_mlp": 0.24316406, + "step": 2516, + "time_per_iteration": 2.444220781326294 + }, + { + "auxiliary_loss_clip": 0.01102024, + "auxiliary_loss_mlp": 0.01040779, + "balance_loss_clip": 1.03299117, + "balance_loss_mlp": 1.0216701, + "epoch": 0.07303696825488944, + "flos": 11038697763840.0, + "grad_norm": 3.152182223042465, + "language_loss": 0.92581046, + "learning_rate": 3.980606413478637e-06, + "loss": 0.94723845, + "num_input_tokens_seen": 70945635, + "router_z_loss_clip": 0.69042969, + "router_z_loss_mlp": 0.19104004, + "step": 2517, + "time_per_iteration": 4.784756422042847 + }, + { + "auxiliary_loss_clip": 0.01115148, + "auxiliary_loss_mlp": 0.01043293, + "balance_loss_clip": 1.03480041, + "balance_loss_mlp": 1.02143025, + "epoch": 0.07306598572340549, + "flos": 37590075843840.0, + "grad_norm": 4.233179322945313, + "language_loss": 0.82726771, + "learning_rate": 3.980580292549896e-06, + "loss": 0.84885216, + "num_input_tokens_seen": 70961890, + "router_z_loss_clip": 0.80322266, + "router_z_loss_mlp": 0.21875, + "step": 2518, + "time_per_iteration": 2.5419251918792725 + }, + { + "auxiliary_loss_clip": 0.01112487, + "auxiliary_loss_mlp": 0.0104324, + "balance_loss_clip": 1.03343451, + "balance_loss_mlp": 1.02144825, + "epoch": 0.07309500319192154, + "flos": 25948601723520.0, + "grad_norm": 2.627874589516318, + "language_loss": 0.96436691, + "learning_rate": 3.980554154127869e-06, + "loss": 0.98592424, + "num_input_tokens_seen": 70975425, + "router_z_loss_clip": 0.79199219, + "router_z_loss_mlp": 0.21777344, + "step": 2519, + "time_per_iteration": 2.3721048831939697 + }, + { + "auxiliary_loss_clip": 0.01128245, + "auxiliary_loss_mlp": 0.01051391, + "balance_loss_clip": 1.04120255, + "balance_loss_mlp": 1.02832377, + "epoch": 0.07312402066043758, + "flos": 33100734159360.0, + "grad_norm": 2.337202270377821, + "language_loss": 0.91042918, + "learning_rate": 3.980527998212786e-06, + "loss": 0.93222553, + "num_input_tokens_seen": 70998235, + "router_z_loss_clip": 0.87158203, + "router_z_loss_mlp": 0.23059082, + "step": 2520, + "time_per_iteration": 4.940293073654175 + }, + { + "auxiliary_loss_clip": 0.01017759, + "auxiliary_loss_mlp": 0.01003974, + "balance_loss_clip": 1.00274932, + "balance_loss_mlp": 1.00220954, + "epoch": 0.07315303812895363, + "flos": 56386054506240.0, + "grad_norm": 0.6780087462865418, + "language_loss": 0.50802451, + "learning_rate": 3.980501824804879e-06, + "loss": 0.52824187, + "num_input_tokens_seen": 71059830, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.0177002, + "step": 2521, + "time_per_iteration": 3.0167171955108643 + }, + { + "auxiliary_loss_clip": 0.01017696, + "auxiliary_loss_mlp": 0.01001846, + "balance_loss_clip": 1.00284052, + "balance_loss_mlp": 1.00020099, + "epoch": 0.07318205559746968, + "flos": 68760564865920.0, + "grad_norm": 0.6004344817879241, + "language_loss": 0.50568509, + "learning_rate": 3.980475633904378e-06, + "loss": 0.52588052, + "num_input_tokens_seen": 71128680, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01647949, + "step": 2522, + "time_per_iteration": 3.164663791656494 + }, + { + "auxiliary_loss_clip": 0.01115621, + "auxiliary_loss_mlp": 0.01044688, + "balance_loss_clip": 1.03547907, + "balance_loss_mlp": 1.02267575, + "epoch": 0.07321107306598572, + "flos": 68273005328640.0, + "grad_norm": 1.9077234672639756, + "language_loss": 0.71446353, + "learning_rate": 3.980449425511515e-06, + "loss": 0.73606664, + "num_input_tokens_seen": 71150085, + "router_z_loss_clip": 0.80273438, + "router_z_loss_mlp": 0.22033691, + "step": 2523, + "time_per_iteration": 3.044147253036499 + }, + { + "auxiliary_loss_clip": 0.01110648, + "auxiliary_loss_mlp": 0.0104064, + "balance_loss_clip": 1.03369653, + "balance_loss_mlp": 1.01979065, + "epoch": 0.07324009053450177, + "flos": 18146874931200.0, + "grad_norm": 3.025804619473074, + "language_loss": 0.76476324, + "learning_rate": 3.980423199626521e-06, + "loss": 0.78627616, + "num_input_tokens_seen": 71163065, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.20861816, + "step": 2524, + "time_per_iteration": 2.3722705841064453 + }, + { + "auxiliary_loss_clip": 0.01126545, + "auxiliary_loss_mlp": 0.01060016, + "balance_loss_clip": 1.03891158, + "balance_loss_mlp": 1.03510118, + "epoch": 0.07326910800301782, + "flos": 10406595864960.0, + "grad_norm": 2.988804843311375, + "language_loss": 0.93088984, + "learning_rate": 3.980396956249628e-06, + "loss": 0.95275545, + "num_input_tokens_seen": 71173325, + "router_z_loss_clip": 0.87597656, + "router_z_loss_mlp": 0.24926758, + "step": 2525, + "time_per_iteration": 2.4393999576568604 + }, + { + "auxiliary_loss_clip": 0.01125301, + "auxiliary_loss_mlp": 0.01053722, + "balance_loss_clip": 1.03990793, + "balance_loss_mlp": 1.02757978, + "epoch": 0.07329812547153386, + "flos": 13737972752640.0, + "grad_norm": 2.9304361541980057, + "language_loss": 0.95384884, + "learning_rate": 3.980370695381067e-06, + "loss": 0.97563905, + "num_input_tokens_seen": 71187550, + "router_z_loss_clip": 0.85400391, + "router_z_loss_mlp": 0.26123047, + "step": 2526, + "time_per_iteration": 2.365048408508301 + }, + { + "auxiliary_loss_clip": 0.01118488, + "auxiliary_loss_mlp": 0.010476, + "balance_loss_clip": 1.03872442, + "balance_loss_mlp": 1.0272032, + "epoch": 0.07332714294004991, + "flos": 25552476904320.0, + "grad_norm": 1.9277901865923994, + "language_loss": 0.77848911, + "learning_rate": 3.980344417021071e-06, + "loss": 0.80014998, + "num_input_tokens_seen": 71206815, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.20397949, + "step": 2527, + "time_per_iteration": 2.6423797607421875 + }, + { + "auxiliary_loss_clip": 0.01018099, + "auxiliary_loss_mlp": 0.0100344, + "balance_loss_clip": 1.00365758, + "balance_loss_mlp": 1.00183058, + "epoch": 0.07335616040856596, + "flos": 74772190051200.0, + "grad_norm": 0.8525524957986906, + "language_loss": 0.46998376, + "learning_rate": 3.980318121169872e-06, + "loss": 0.49019915, + "num_input_tokens_seen": 71269880, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01611328, + "step": 2528, + "time_per_iteration": 3.100681781768799 + }, + { + "auxiliary_loss_clip": 0.01017098, + "auxiliary_loss_mlp": 0.01003726, + "balance_loss_clip": 1.00249028, + "balance_loss_mlp": 1.00211716, + "epoch": 0.073385177877082, + "flos": 62589945882240.0, + "grad_norm": 0.7000252037774068, + "language_loss": 0.54179317, + "learning_rate": 3.980291807827702e-06, + "loss": 0.56200147, + "num_input_tokens_seen": 71332500, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01611328, + "step": 2529, + "time_per_iteration": 3.0984742641448975 + }, + { + "auxiliary_loss_clip": 0.01115481, + "auxiliary_loss_mlp": 0.01047709, + "balance_loss_clip": 1.03709912, + "balance_loss_mlp": 1.02625108, + "epoch": 0.07341419534559805, + "flos": 11100634248960.0, + "grad_norm": 4.247524410741794, + "language_loss": 0.87143803, + "learning_rate": 3.980265476994794e-06, + "loss": 0.89306986, + "num_input_tokens_seen": 71344260, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.21447754, + "step": 2530, + "time_per_iteration": 2.451777696609497 + }, + { + "auxiliary_loss_clip": 0.01119348, + "auxiliary_loss_mlp": 0.01050192, + "balance_loss_clip": 1.03448367, + "balance_loss_mlp": 1.0248605, + "epoch": 0.0734432128141141, + "flos": 20223858113280.0, + "grad_norm": 2.1948100274190514, + "language_loss": 0.85493523, + "learning_rate": 3.9802391286713796e-06, + "loss": 0.87663054, + "num_input_tokens_seen": 71362700, + "router_z_loss_clip": 0.85009766, + "router_z_loss_mlp": 0.25341797, + "step": 2531, + "time_per_iteration": 2.5847764015197754 + }, + { + "auxiliary_loss_clip": 0.01111552, + "auxiliary_loss_mlp": 0.01039498, + "balance_loss_clip": 1.03676784, + "balance_loss_mlp": 1.01953006, + "epoch": 0.07347223028263014, + "flos": 38755756051200.0, + "grad_norm": 2.233818861156868, + "language_loss": 0.91968346, + "learning_rate": 3.980212762857691e-06, + "loss": 0.94119394, + "num_input_tokens_seen": 71379280, + "router_z_loss_clip": 0.74804688, + "router_z_loss_mlp": 0.1998291, + "step": 2532, + "time_per_iteration": 2.604724168777466 + }, + { + "auxiliary_loss_clip": 0.01112616, + "auxiliary_loss_mlp": 0.01039983, + "balance_loss_clip": 1.03376138, + "balance_loss_mlp": 1.01749969, + "epoch": 0.07350124775114619, + "flos": 32117440227840.0, + "grad_norm": 3.087336796307236, + "language_loss": 1.004691, + "learning_rate": 3.980186379553963e-06, + "loss": 1.02621698, + "num_input_tokens_seen": 71393745, + "router_z_loss_clip": 0.78857422, + "router_z_loss_mlp": 0.22485352, + "step": 2533, + "time_per_iteration": 2.6108431816101074 + }, + { + "auxiliary_loss_clip": 0.01113344, + "auxiliary_loss_mlp": 0.01040303, + "balance_loss_clip": 1.03458714, + "balance_loss_mlp": 1.02007365, + "epoch": 0.07353026521966223, + "flos": 25586238055680.0, + "grad_norm": 2.9890552069274223, + "language_loss": 0.7675131, + "learning_rate": 3.980159978760427e-06, + "loss": 0.78904957, + "num_input_tokens_seen": 71409120, + "router_z_loss_clip": 0.78759766, + "router_z_loss_mlp": 0.20227051, + "step": 2534, + "time_per_iteration": 2.51739501953125 + }, + { + "auxiliary_loss_clip": 0.01111982, + "auxiliary_loss_mlp": 0.01042413, + "balance_loss_clip": 1.03236556, + "balance_loss_mlp": 1.01971531, + "epoch": 0.07355928268817828, + "flos": 22486299770880.0, + "grad_norm": 4.792724940614353, + "language_loss": 0.94957596, + "learning_rate": 3.9801335604773175e-06, + "loss": 0.97111988, + "num_input_tokens_seen": 71423795, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.22692871, + "step": 2535, + "time_per_iteration": 2.482250690460205 + }, + { + "auxiliary_loss_clip": 0.01109028, + "auxiliary_loss_mlp": 0.01046029, + "balance_loss_clip": 1.03341198, + "balance_loss_mlp": 1.02449989, + "epoch": 0.07358830015669433, + "flos": 11220420723840.0, + "grad_norm": 2.63838386601987, + "language_loss": 0.8478924, + "learning_rate": 3.980107124704866e-06, + "loss": 0.86944294, + "num_input_tokens_seen": 71435585, + "router_z_loss_clip": 0.75683594, + "router_z_loss_mlp": 0.21508789, + "step": 2536, + "time_per_iteration": 2.5823237895965576 + }, + { + "auxiliary_loss_clip": 0.0111745, + "auxiliary_loss_mlp": 0.01042215, + "balance_loss_clip": 1.03556991, + "balance_loss_mlp": 1.01700211, + "epoch": 0.07361731762521037, + "flos": 17814397253760.0, + "grad_norm": 2.8550808203686238, + "language_loss": 0.79234052, + "learning_rate": 3.980080671443308e-06, + "loss": 0.81393719, + "num_input_tokens_seen": 71452840, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.25219727, + "step": 2537, + "time_per_iteration": 2.8436155319213867 + }, + { + "auxiliary_loss_clip": 0.01107618, + "auxiliary_loss_mlp": 0.01040064, + "balance_loss_clip": 1.03258955, + "balance_loss_mlp": 1.02044773, + "epoch": 0.07364633509372642, + "flos": 17701907253120.0, + "grad_norm": 3.2497908806272986, + "language_loss": 0.79745108, + "learning_rate": 3.980054200692876e-06, + "loss": 0.81892788, + "num_input_tokens_seen": 71463575, + "router_z_loss_clip": 0.74975586, + "router_z_loss_mlp": 0.19616699, + "step": 2538, + "time_per_iteration": 2.503596305847168 + }, + { + "auxiliary_loss_clip": 0.01121407, + "auxiliary_loss_mlp": 0.01041919, + "balance_loss_clip": 1.03718162, + "balance_loss_mlp": 1.01891148, + "epoch": 0.07367535256224247, + "flos": 12048037436160.0, + "grad_norm": 3.199133627657077, + "language_loss": 1.00454533, + "learning_rate": 3.9800277124538036e-06, + "loss": 1.0261786, + "num_input_tokens_seen": 71476530, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.23010254, + "step": 2539, + "time_per_iteration": 2.4727487564086914 + }, + { + "auxiliary_loss_clip": 0.01105353, + "auxiliary_loss_mlp": 0.01044736, + "balance_loss_clip": 1.03242433, + "balance_loss_mlp": 1.02544785, + "epoch": 0.07370437003075851, + "flos": 34232688126720.0, + "grad_norm": 2.653341392005416, + "language_loss": 0.92403555, + "learning_rate": 3.980001206726326e-06, + "loss": 0.94553649, + "num_input_tokens_seen": 71493550, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.19287109, + "step": 2540, + "time_per_iteration": 2.7130849361419678 + }, + { + "auxiliary_loss_clip": 0.0111329, + "auxiliary_loss_mlp": 0.0104487, + "balance_loss_clip": 1.03581214, + "balance_loss_mlp": 1.0217433, + "epoch": 0.07373338749927456, + "flos": 38428619811840.0, + "grad_norm": 2.431942116612947, + "language_loss": 0.80051458, + "learning_rate": 3.979974683510676e-06, + "loss": 0.82209623, + "num_input_tokens_seen": 71513375, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.2310791, + "step": 2541, + "time_per_iteration": 2.7095861434936523 + }, + { + "auxiliary_loss_clip": 0.01113101, + "auxiliary_loss_mlp": 0.01041357, + "balance_loss_clip": 1.03561878, + "balance_loss_mlp": 1.02016187, + "epoch": 0.07376240496779062, + "flos": 32407987495680.0, + "grad_norm": 2.7194408191871786, + "language_loss": 0.62534404, + "learning_rate": 3.979948142807089e-06, + "loss": 0.64688861, + "num_input_tokens_seen": 71528245, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.21179199, + "step": 2542, + "time_per_iteration": 2.5709547996520996 + }, + { + "auxiliary_loss_clip": 0.01019838, + "auxiliary_loss_mlp": 0.01004655, + "balance_loss_clip": 1.00568247, + "balance_loss_mlp": 1.00296223, + "epoch": 0.07379142243630665, + "flos": 57476671557120.0, + "grad_norm": 0.6867168724146052, + "language_loss": 0.48165146, + "learning_rate": 3.979921584615798e-06, + "loss": 0.50189638, + "num_input_tokens_seen": 71588245, + "router_z_loss_clip": 0.14160156, + "router_z_loss_mlp": 0.01696777, + "step": 2543, + "time_per_iteration": 3.0249698162078857 + }, + { + "auxiliary_loss_clip": 0.01119703, + "auxiliary_loss_mlp": 0.01043687, + "balance_loss_clip": 1.03601432, + "balance_loss_mlp": 1.020239, + "epoch": 0.0738204399048227, + "flos": 44010009912960.0, + "grad_norm": 2.2979810232946485, + "language_loss": 0.82514262, + "learning_rate": 3.979895008937039e-06, + "loss": 0.84677649, + "num_input_tokens_seen": 71606775, + "router_z_loss_clip": 0.83642578, + "router_z_loss_mlp": 0.23461914, + "step": 2544, + "time_per_iteration": 2.7471649646759033 + }, + { + "auxiliary_loss_clip": 0.01117808, + "auxiliary_loss_mlp": 0.01042611, + "balance_loss_clip": 1.03814197, + "balance_loss_mlp": 1.02138019, + "epoch": 0.07384945737333876, + "flos": 27117214485120.0, + "grad_norm": 2.699506684560989, + "language_loss": 0.94646668, + "learning_rate": 3.979868415771046e-06, + "loss": 0.96807081, + "num_input_tokens_seen": 71621510, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.2121582, + "step": 2545, + "time_per_iteration": 2.546618938446045 + }, + { + "auxiliary_loss_clip": 0.0111455, + "auxiliary_loss_mlp": 0.01046828, + "balance_loss_clip": 1.03630972, + "balance_loss_mlp": 1.02472627, + "epoch": 0.0738784748418548, + "flos": 24784946375040.0, + "grad_norm": 2.153203082513383, + "language_loss": 0.84956133, + "learning_rate": 3.979841805118054e-06, + "loss": 0.87117505, + "num_input_tokens_seen": 71635925, + "router_z_loss_clip": 0.78271484, + "router_z_loss_mlp": 0.22106934, + "step": 2546, + "time_per_iteration": 2.5993852615356445 + }, + { + "auxiliary_loss_clip": 0.01015986, + "auxiliary_loss_mlp": 0.01003303, + "balance_loss_clip": 1.00175571, + "balance_loss_mlp": 1.00171745, + "epoch": 0.07390749231037085, + "flos": 63159729137280.0, + "grad_norm": 0.6787841183166787, + "language_loss": 0.4601416, + "learning_rate": 3.979815176978298e-06, + "loss": 0.48033446, + "num_input_tokens_seen": 71685165, + "router_z_loss_clip": 0.14257812, + "router_z_loss_mlp": 0.01586914, + "step": 2547, + "time_per_iteration": 2.8481361865997314 + }, + { + "auxiliary_loss_clip": 0.01117883, + "auxiliary_loss_mlp": 0.01048839, + "balance_loss_clip": 1.03661168, + "balance_loss_mlp": 1.02603447, + "epoch": 0.07393650977888688, + "flos": 17523605606400.0, + "grad_norm": 14.17221903159032, + "language_loss": 0.92906129, + "learning_rate": 3.979788531352013e-06, + "loss": 0.95072854, + "num_input_tokens_seen": 71698315, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.22814941, + "step": 2548, + "time_per_iteration": 2.408050775527954 + }, + { + "auxiliary_loss_clip": 0.0111784, + "auxiliary_loss_mlp": 0.01048108, + "balance_loss_clip": 1.03798127, + "balance_loss_mlp": 1.02522016, + "epoch": 0.07396552724740293, + "flos": 31860025706880.0, + "grad_norm": 2.2591089401986952, + "language_loss": 0.86264348, + "learning_rate": 3.979761868239434e-06, + "loss": 0.88430297, + "num_input_tokens_seen": 71714640, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.22888184, + "step": 2549, + "time_per_iteration": 2.5603249073028564 + }, + { + "auxiliary_loss_clip": 0.01015873, + "auxiliary_loss_mlp": 0.0100344, + "balance_loss_clip": 1.0017693, + "balance_loss_mlp": 1.00184298, + "epoch": 0.07399454471591899, + "flos": 74762659249920.0, + "grad_norm": 0.7179394789160791, + "language_loss": 0.53456831, + "learning_rate": 3.979735187640798e-06, + "loss": 0.55476141, + "num_input_tokens_seen": 71769100, + "router_z_loss_clip": 0.140625, + "router_z_loss_mlp": 0.01599121, + "step": 2550, + "time_per_iteration": 2.9607760906219482 + }, + { + "auxiliary_loss_clip": 0.01016443, + "auxiliary_loss_mlp": 0.01005737, + "balance_loss_clip": 1.00225639, + "balance_loss_mlp": 1.0041877, + "epoch": 0.07402356218443502, + "flos": 63968352203520.0, + "grad_norm": 0.6580779238214463, + "language_loss": 0.4606421, + "learning_rate": 3.97970848955634e-06, + "loss": 0.48086393, + "num_input_tokens_seen": 71831430, + "router_z_loss_clip": 0.14257812, + "router_z_loss_mlp": 0.01544189, + "step": 2551, + "time_per_iteration": 2.941026449203491 + }, + { + "auxiliary_loss_clip": 0.01126685, + "auxiliary_loss_mlp": 0.01045207, + "balance_loss_clip": 1.04010546, + "balance_loss_mlp": 1.02179396, + "epoch": 0.07405257965295108, + "flos": 20185279194240.0, + "grad_norm": 2.841344534463477, + "language_loss": 1.02631974, + "learning_rate": 3.9796817739862945e-06, + "loss": 1.04803872, + "num_input_tokens_seen": 71845075, + "router_z_loss_clip": 0.86523438, + "router_z_loss_mlp": 0.23431396, + "step": 2552, + "time_per_iteration": 2.3860023021698 + }, + { + "auxiliary_loss_clip": 0.01120439, + "auxiliary_loss_mlp": 0.01044948, + "balance_loss_clip": 1.03642201, + "balance_loss_mlp": 1.02256024, + "epoch": 0.07408159712146713, + "flos": 41565042334080.0, + "grad_norm": 2.19725979209766, + "language_loss": 0.99404073, + "learning_rate": 3.979655040930898e-06, + "loss": 1.01569462, + "num_input_tokens_seen": 71864950, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.22412109, + "step": 2553, + "time_per_iteration": 2.6268563270568848 + }, + { + "auxiliary_loss_clip": 0.01119548, + "auxiliary_loss_mlp": 0.01054477, + "balance_loss_clip": 1.03481174, + "balance_loss_mlp": 1.03016973, + "epoch": 0.07411061458998316, + "flos": 21827244435840.0, + "grad_norm": 2.6959759541945347, + "language_loss": 0.98600626, + "learning_rate": 3.979628290390389e-06, + "loss": 1.00774646, + "num_input_tokens_seen": 71879415, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.24328613, + "step": 2554, + "time_per_iteration": 2.3858890533447266 + }, + { + "auxiliary_loss_clip": 0.01112744, + "auxiliary_loss_mlp": 0.01037646, + "balance_loss_clip": 1.03322291, + "balance_loss_mlp": 1.01629591, + "epoch": 0.07413963205849922, + "flos": 12560108480640.0, + "grad_norm": 3.3333291853226084, + "language_loss": 1.12658048, + "learning_rate": 3.979601522365e-06, + "loss": 1.1480844, + "num_input_tokens_seen": 71890295, + "router_z_loss_clip": 0.79492188, + "router_z_loss_mlp": 0.21350098, + "step": 2555, + "time_per_iteration": 2.4186465740203857 + }, + { + "auxiliary_loss_clip": 0.01016686, + "auxiliary_loss_mlp": 0.01004164, + "balance_loss_clip": 1.00263453, + "balance_loss_mlp": 1.00272179, + "epoch": 0.07416864952701527, + "flos": 69947228666880.0, + "grad_norm": 0.647652290583372, + "language_loss": 0.50443602, + "learning_rate": 3.979574736854971e-06, + "loss": 0.52464449, + "num_input_tokens_seen": 71947230, + "router_z_loss_clip": 0.140625, + "router_z_loss_mlp": 0.0144043, + "step": 2556, + "time_per_iteration": 2.8998830318450928 + }, + { + "auxiliary_loss_clip": 0.01110282, + "auxiliary_loss_mlp": 0.01043356, + "balance_loss_clip": 1.03643417, + "balance_loss_mlp": 1.02430654, + "epoch": 0.0741976669955313, + "flos": 29855801531520.0, + "grad_norm": 1.6853501031197653, + "language_loss": 0.83575237, + "learning_rate": 3.979547933860535e-06, + "loss": 0.85728872, + "num_input_tokens_seen": 71968905, + "router_z_loss_clip": 0.73681641, + "router_z_loss_mlp": 0.19067383, + "step": 2557, + "time_per_iteration": 2.6439430713653564 + }, + { + "auxiliary_loss_clip": 0.01118335, + "auxiliary_loss_mlp": 0.01040876, + "balance_loss_clip": 1.03434229, + "balance_loss_mlp": 1.01634312, + "epoch": 0.07422668446404736, + "flos": 13983482499840.0, + "grad_norm": 3.3378652256224557, + "language_loss": 0.83943248, + "learning_rate": 3.979521113381932e-06, + "loss": 0.86102456, + "num_input_tokens_seen": 71981365, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.24536133, + "step": 2558, + "time_per_iteration": 2.4123620986938477 + }, + { + "auxiliary_loss_clip": 0.01122734, + "auxiliary_loss_mlp": 0.01051829, + "balance_loss_clip": 1.03899086, + "balance_loss_mlp": 1.0281539, + "epoch": 0.07425570193256341, + "flos": 28064373292800.0, + "grad_norm": 3.03873821378503, + "language_loss": 0.82350516, + "learning_rate": 3.979494275419398e-06, + "loss": 0.84525079, + "num_input_tokens_seen": 71994990, + "router_z_loss_clip": 0.83691406, + "router_z_loss_mlp": 0.23681641, + "step": 2559, + "time_per_iteration": 2.4889943599700928 + }, + { + "auxiliary_loss_clip": 0.01109255, + "auxiliary_loss_mlp": 0.01039398, + "balance_loss_clip": 1.0330615, + "balance_loss_mlp": 1.01798832, + "epoch": 0.07428471940107945, + "flos": 66778513136640.0, + "grad_norm": 2.316822252718243, + "language_loss": 0.73139167, + "learning_rate": 3.979467419973168e-06, + "loss": 0.75287819, + "num_input_tokens_seen": 72018165, + "router_z_loss_clip": 0.76196289, + "router_z_loss_mlp": 0.21398926, + "step": 2560, + "time_per_iteration": 2.7056374549865723 + }, + { + "auxiliary_loss_clip": 0.01116596, + "auxiliary_loss_mlp": 0.01051398, + "balance_loss_clip": 1.03535652, + "balance_loss_mlp": 1.02915335, + "epoch": 0.0743137368695955, + "flos": 43097450129280.0, + "grad_norm": 2.9848448465954176, + "language_loss": 0.83983022, + "learning_rate": 3.979440547043482e-06, + "loss": 0.86151016, + "num_input_tokens_seen": 72038680, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.22253418, + "step": 2561, + "time_per_iteration": 2.6212334632873535 + }, + { + "auxiliary_loss_clip": 0.01021004, + "auxiliary_loss_mlp": 0.01004154, + "balance_loss_clip": 1.0066514, + "balance_loss_mlp": 1.00272965, + "epoch": 0.07434275433811155, + "flos": 74774564023680.0, + "grad_norm": 0.7062397869619723, + "language_loss": 0.4976798, + "learning_rate": 3.979413656630575e-06, + "loss": 0.5179314, + "num_input_tokens_seen": 72104160, + "router_z_loss_clip": 0.14355469, + "router_z_loss_mlp": 0.01422119, + "step": 2562, + "time_per_iteration": 3.092369556427002 + }, + { + "auxiliary_loss_clip": 0.01117609, + "auxiliary_loss_mlp": 0.01048478, + "balance_loss_clip": 1.03829813, + "balance_loss_mlp": 1.02747965, + "epoch": 0.07437177180662759, + "flos": 25476226761600.0, + "grad_norm": 2.6023918202145735, + "language_loss": 0.85597968, + "learning_rate": 3.979386748734686e-06, + "loss": 0.8776406, + "num_input_tokens_seen": 72118480, + "router_z_loss_clip": 0.79296875, + "router_z_loss_mlp": 0.21002197, + "step": 2563, + "time_per_iteration": 2.5962748527526855 + }, + { + "auxiliary_loss_clip": 0.01122975, + "auxiliary_loss_mlp": 0.01040867, + "balance_loss_clip": 1.03516626, + "balance_loss_mlp": 1.01828933, + "epoch": 0.07440078927514364, + "flos": 16063712438400.0, + "grad_norm": 3.0245508313545155, + "language_loss": 0.89090449, + "learning_rate": 3.979359823356053e-06, + "loss": 0.91254294, + "num_input_tokens_seen": 72133135, + "router_z_loss_clip": 0.87792969, + "router_z_loss_mlp": 0.22595215, + "step": 2564, + "time_per_iteration": 2.4368057250976562 + }, + { + "auxiliary_loss_clip": 0.01114425, + "auxiliary_loss_mlp": 0.01044678, + "balance_loss_clip": 1.03503096, + "balance_loss_mlp": 1.02219486, + "epoch": 0.07442980674365968, + "flos": 17412162946560.0, + "grad_norm": 2.262735621626221, + "language_loss": 0.76558733, + "learning_rate": 3.979332880494912e-06, + "loss": 0.7871784, + "num_input_tokens_seen": 72144875, + "router_z_loss_clip": 0.79394531, + "router_z_loss_mlp": 0.22485352, + "step": 2565, + "time_per_iteration": 2.478309392929077 + }, + { + "auxiliary_loss_clip": 0.01118619, + "auxiliary_loss_mlp": 0.01045888, + "balance_loss_clip": 1.03688061, + "balance_loss_mlp": 1.02167678, + "epoch": 0.07445882421217573, + "flos": 42952176495360.0, + "grad_norm": 1.8525036647216258, + "language_loss": 0.82425839, + "learning_rate": 3.9793059201515025e-06, + "loss": 0.84590352, + "num_input_tokens_seen": 72179375, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.24212646, + "step": 2566, + "time_per_iteration": 3.148373603820801 + }, + { + "auxiliary_loss_clip": 0.01120055, + "auxiliary_loss_mlp": 0.0104236, + "balance_loss_clip": 1.03556681, + "balance_loss_mlp": 1.01967466, + "epoch": 0.07448784168069178, + "flos": 16901802558720.0, + "grad_norm": 2.8209487715714574, + "language_loss": 0.91809607, + "learning_rate": 3.979278942326062e-06, + "loss": 0.93972015, + "num_input_tokens_seen": 72192120, + "router_z_loss_clip": 0.84472656, + "router_z_loss_mlp": 0.22692871, + "step": 2567, + "time_per_iteration": 2.39019775390625 + }, + { + "auxiliary_loss_clip": 0.0102054, + "auxiliary_loss_mlp": 0.01004476, + "balance_loss_clip": 1.00586438, + "balance_loss_mlp": 1.00290215, + "epoch": 0.07451685914920782, + "flos": 74770409571840.0, + "grad_norm": 0.7940978107781878, + "language_loss": 0.50740463, + "learning_rate": 3.979251947018829e-06, + "loss": 0.52765477, + "num_input_tokens_seen": 72253810, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01574707, + "step": 2568, + "time_per_iteration": 3.0217349529266357 + }, + { + "auxiliary_loss_clip": 0.01120101, + "auxiliary_loss_mlp": 0.01053568, + "balance_loss_clip": 1.03781068, + "balance_loss_mlp": 1.0306257, + "epoch": 0.07454587661772387, + "flos": 39268979170560.0, + "grad_norm": 2.220857435327992, + "language_loss": 0.64776182, + "learning_rate": 3.979224934230043e-06, + "loss": 0.66949856, + "num_input_tokens_seen": 72271910, + "router_z_loss_clip": 0.82275391, + "router_z_loss_mlp": 0.22943115, + "step": 2569, + "time_per_iteration": 2.587099313735962 + }, + { + "auxiliary_loss_clip": 0.01018198, + "auxiliary_loss_mlp": 0.01002319, + "balance_loss_clip": 1.00330973, + "balance_loss_mlp": 1.00088298, + "epoch": 0.07457489408623992, + "flos": 63752658624000.0, + "grad_norm": 0.723812617324247, + "language_loss": 0.53971279, + "learning_rate": 3.9791979039599395e-06, + "loss": 0.55991799, + "num_input_tokens_seen": 72332235, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01434326, + "step": 2570, + "time_per_iteration": 3.011460781097412 + }, + { + "auxiliary_loss_clip": 0.01101845, + "auxiliary_loss_mlp": 0.01042408, + "balance_loss_clip": 1.03005433, + "balance_loss_mlp": 1.02214205, + "epoch": 0.07460391155475596, + "flos": 39816172909440.0, + "grad_norm": 2.0403414121882157, + "language_loss": 0.841995, + "learning_rate": 3.979170856208761e-06, + "loss": 0.86343759, + "num_input_tokens_seen": 72350020, + "router_z_loss_clip": 0.71850586, + "router_z_loss_mlp": 0.20263672, + "step": 2571, + "time_per_iteration": 2.561005115509033 + }, + { + "auxiliary_loss_clip": 0.01118224, + "auxiliary_loss_mlp": 0.01042859, + "balance_loss_clip": 1.03524685, + "balance_loss_mlp": 1.02007854, + "epoch": 0.07463292902327201, + "flos": 10919574604800.0, + "grad_norm": 2.867143476768628, + "language_loss": 0.83055031, + "learning_rate": 3.979143790976744e-06, + "loss": 0.85216117, + "num_input_tokens_seen": 72361700, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.22790527, + "step": 2572, + "time_per_iteration": 2.4224295616149902 + }, + { + "auxiliary_loss_clip": 0.01114218, + "auxiliary_loss_mlp": 0.01050069, + "balance_loss_clip": 1.03351867, + "balance_loss_mlp": 1.02665615, + "epoch": 0.07466194649178806, + "flos": 31897103437440.0, + "grad_norm": 2.2923163812913785, + "language_loss": 0.7758646, + "learning_rate": 3.9791167082641275e-06, + "loss": 0.79750741, + "num_input_tokens_seen": 72378065, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.234375, + "step": 2573, + "time_per_iteration": 2.4519522190093994 + }, + { + "auxiliary_loss_clip": 0.01112936, + "auxiliary_loss_mlp": 0.0104529, + "balance_loss_clip": 1.03380322, + "balance_loss_mlp": 1.02229476, + "epoch": 0.0746909639603041, + "flos": 28030751786880.0, + "grad_norm": 3.75870921626245, + "language_loss": 0.91719162, + "learning_rate": 3.979089608071152e-06, + "loss": 0.93877387, + "num_input_tokens_seen": 72392355, + "router_z_loss_clip": 0.79052734, + "router_z_loss_mlp": 0.2300415, + "step": 2574, + "time_per_iteration": 2.500239372253418 + }, + { + "auxiliary_loss_clip": 0.01016723, + "auxiliary_loss_mlp": 0.01001383, + "balance_loss_clip": 1.00205421, + "balance_loss_mlp": 1.00007725, + "epoch": 0.07471998142882015, + "flos": 59412291177600.0, + "grad_norm": 0.6774717942164219, + "language_loss": 0.48286653, + "learning_rate": 3.979062490398056e-06, + "loss": 0.50304759, + "num_input_tokens_seen": 72452110, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01306152, + "step": 2575, + "time_per_iteration": 2.9467873573303223 + }, + { + "auxiliary_loss_clip": 0.01119062, + "auxiliary_loss_mlp": 0.01056759, + "balance_loss_clip": 1.03740525, + "balance_loss_mlp": 1.03302455, + "epoch": 0.0747489988973362, + "flos": 34198333482240.0, + "grad_norm": 2.8152116711025723, + "language_loss": 0.87942421, + "learning_rate": 3.979035355245079e-06, + "loss": 0.90118247, + "num_input_tokens_seen": 72473345, + "router_z_loss_clip": 0.81640625, + "router_z_loss_mlp": 0.23754883, + "step": 2576, + "time_per_iteration": 2.523164749145508 + }, + { + "auxiliary_loss_clip": 0.01016695, + "auxiliary_loss_mlp": 0.01001568, + "balance_loss_clip": 1.00219822, + "balance_loss_mlp": 1.00026274, + "epoch": 0.07477801636585224, + "flos": 74778090071040.0, + "grad_norm": 0.6493685372026015, + "language_loss": 0.53141475, + "learning_rate": 3.979008202612461e-06, + "loss": 0.55159736, + "num_input_tokens_seen": 72539500, + "router_z_loss_clip": 0.14453125, + "router_z_loss_mlp": 0.01306152, + "step": 2577, + "time_per_iteration": 3.0705199241638184 + }, + { + "auxiliary_loss_clip": 0.01116938, + "auxiliary_loss_mlp": 0.0104799, + "balance_loss_clip": 1.03764915, + "balance_loss_mlp": 1.02592421, + "epoch": 0.07480703383436829, + "flos": 27372185210880.0, + "grad_norm": 2.2176318381232583, + "language_loss": 0.91430146, + "learning_rate": 3.9789810325004425e-06, + "loss": 0.9359507, + "num_input_tokens_seen": 72554415, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.2208252, + "step": 2578, + "time_per_iteration": 2.4695260524749756 + }, + { + "auxiliary_loss_clip": 0.01114282, + "auxiliary_loss_mlp": 0.01045569, + "balance_loss_clip": 1.03874755, + "balance_loss_mlp": 1.02563739, + "epoch": 0.07483605130288434, + "flos": 24166948665600.0, + "grad_norm": 3.2785989171130683, + "language_loss": 0.8181119, + "learning_rate": 3.978953844909262e-06, + "loss": 0.83971035, + "num_input_tokens_seen": 72571150, + "router_z_loss_clip": 0.75585938, + "router_z_loss_mlp": 0.19934082, + "step": 2579, + "time_per_iteration": 2.4254207611083984 + }, + { + "auxiliary_loss_clip": 0.01115366, + "auxiliary_loss_mlp": 0.01044201, + "balance_loss_clip": 1.03355551, + "balance_loss_mlp": 1.02296948, + "epoch": 0.07486506877140038, + "flos": 18944850032640.0, + "grad_norm": 2.5696872540029467, + "language_loss": 0.87915069, + "learning_rate": 3.9789266398391605e-06, + "loss": 0.90074641, + "num_input_tokens_seen": 72585230, + "router_z_loss_clip": 0.81787109, + "router_z_loss_mlp": 0.21240234, + "step": 2580, + "time_per_iteration": 2.3936336040496826 + }, + { + "auxiliary_loss_clip": 0.0101732, + "auxiliary_loss_mlp": 0.01002632, + "balance_loss_clip": 1.00249112, + "balance_loss_mlp": 1.00132084, + "epoch": 0.07489408623991643, + "flos": 56780887605120.0, + "grad_norm": 0.6228826994140442, + "language_loss": 0.50119531, + "learning_rate": 3.978899417290378e-06, + "loss": 0.52139485, + "num_input_tokens_seen": 72647280, + "router_z_loss_clip": 0.1484375, + "router_z_loss_mlp": 0.01312256, + "step": 2581, + "time_per_iteration": 3.010185480117798 + }, + { + "auxiliary_loss_clip": 0.01122717, + "auxiliary_loss_mlp": 0.01044751, + "balance_loss_clip": 1.03941572, + "balance_loss_mlp": 1.0208199, + "epoch": 0.07492310370843247, + "flos": 42806449013760.0, + "grad_norm": 2.7009985004022594, + "language_loss": 0.83328843, + "learning_rate": 3.978872177263156e-06, + "loss": 0.85496306, + "num_input_tokens_seen": 72665970, + "router_z_loss_clip": 0.83300781, + "router_z_loss_mlp": 0.23937988, + "step": 2582, + "time_per_iteration": 2.4944169521331787 + }, + { + "auxiliary_loss_clip": 0.01112008, + "auxiliary_loss_mlp": 0.01045454, + "balance_loss_clip": 1.03438389, + "balance_loss_mlp": 1.02364469, + "epoch": 0.07495212117694852, + "flos": 29871022884480.0, + "grad_norm": 2.135987242032664, + "language_loss": 0.92453772, + "learning_rate": 3.978844919757733e-06, + "loss": 0.94611239, + "num_input_tokens_seen": 72691320, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.21789551, + "step": 2583, + "time_per_iteration": 2.6054916381835938 + }, + { + "auxiliary_loss_clip": 0.01121128, + "auxiliary_loss_mlp": 0.01052663, + "balance_loss_clip": 1.04057837, + "balance_loss_mlp": 1.03121734, + "epoch": 0.07498113864546457, + "flos": 26315399134080.0, + "grad_norm": 12.799805619594302, + "language_loss": 0.74754041, + "learning_rate": 3.9788176447743516e-06, + "loss": 0.76927829, + "num_input_tokens_seen": 72707485, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.21466064, + "step": 2584, + "time_per_iteration": 2.453662157058716 + }, + { + "auxiliary_loss_clip": 0.01109539, + "auxiliary_loss_mlp": 0.01040803, + "balance_loss_clip": 1.03414559, + "balance_loss_mlp": 1.02091932, + "epoch": 0.07501015611398061, + "flos": 40733061701760.0, + "grad_norm": 2.4871934296335882, + "language_loss": 0.87097919, + "learning_rate": 3.978790352313251e-06, + "loss": 0.89248252, + "num_input_tokens_seen": 72729350, + "router_z_loss_clip": 0.75390625, + "router_z_loss_mlp": 0.19885254, + "step": 2585, + "time_per_iteration": 2.6685128211975098 + }, + { + "auxiliary_loss_clip": 0.01116308, + "auxiliary_loss_mlp": 0.01047378, + "balance_loss_clip": 1.03786016, + "balance_loss_mlp": 1.02384031, + "epoch": 0.07503917358249666, + "flos": 26717563618560.0, + "grad_norm": 2.427747646223599, + "language_loss": 0.79229957, + "learning_rate": 3.978763042374674e-06, + "loss": 0.81393641, + "num_input_tokens_seen": 72742020, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.23535156, + "step": 2586, + "time_per_iteration": 2.4979214668273926 + }, + { + "auxiliary_loss_clip": 0.01111159, + "auxiliary_loss_mlp": 0.01043775, + "balance_loss_clip": 1.0338769, + "balance_loss_mlp": 1.02230549, + "epoch": 0.07506819105101271, + "flos": 15952374512640.0, + "grad_norm": 3.1339154489545074, + "language_loss": 1.00167668, + "learning_rate": 3.978735714958861e-06, + "loss": 1.02322602, + "num_input_tokens_seen": 72754120, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.21459961, + "step": 2587, + "time_per_iteration": 2.423309564590454 + }, + { + "auxiliary_loss_clip": 0.01110175, + "auxiliary_loss_mlp": 0.01044249, + "balance_loss_clip": 1.0343281, + "balance_loss_mlp": 1.02254069, + "epoch": 0.07509720851952875, + "flos": 16426879067520.0, + "grad_norm": 4.487492423945067, + "language_loss": 0.94226867, + "learning_rate": 3.9787083700660535e-06, + "loss": 0.96381289, + "num_input_tokens_seen": 72766050, + "router_z_loss_clip": 0.75830078, + "router_z_loss_mlp": 0.21691895, + "step": 2588, + "time_per_iteration": 2.3725016117095947 + }, + { + "auxiliary_loss_clip": 0.01120759, + "auxiliary_loss_mlp": 0.010442, + "balance_loss_clip": 1.03693962, + "balance_loss_mlp": 1.02177691, + "epoch": 0.0751262259880448, + "flos": 31935123774720.0, + "grad_norm": 1.9881702855128118, + "language_loss": 0.94019073, + "learning_rate": 3.978681007696493e-06, + "loss": 0.96184027, + "num_input_tokens_seen": 72788195, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.22424316, + "step": 2589, + "time_per_iteration": 7.106634140014648 + }, + { + "auxiliary_loss_clip": 0.0111809, + "auxiliary_loss_mlp": 0.01054384, + "balance_loss_clip": 1.03771544, + "balance_loss_mlp": 1.03094745, + "epoch": 0.07515524345656086, + "flos": 11574510399360.0, + "grad_norm": 7.851629194295365, + "language_loss": 0.78786016, + "learning_rate": 3.978653627850422e-06, + "loss": 0.80958486, + "num_input_tokens_seen": 72799595, + "router_z_loss_clip": 0.80322266, + "router_z_loss_mlp": 0.23425293, + "step": 2590, + "time_per_iteration": 2.3991270065307617 + }, + { + "auxiliary_loss_clip": 0.01117597, + "auxiliary_loss_mlp": 0.01052765, + "balance_loss_clip": 1.03440678, + "balance_loss_mlp": 1.03026986, + "epoch": 0.07518426092507689, + "flos": 30912343228800.0, + "grad_norm": 2.62499048017397, + "language_loss": 0.92129695, + "learning_rate": 3.97862623052808e-06, + "loss": 0.94300056, + "num_input_tokens_seen": 72820365, + "router_z_loss_clip": 0.83154297, + "router_z_loss_mlp": 0.22485352, + "step": 2591, + "time_per_iteration": 2.548109292984009 + }, + { + "auxiliary_loss_clip": 0.0111408, + "auxiliary_loss_mlp": 0.01045927, + "balance_loss_clip": 1.03608286, + "balance_loss_mlp": 1.0245471, + "epoch": 0.07521327839359294, + "flos": 49920840403200.0, + "grad_norm": 2.3973197448373096, + "language_loss": 0.9044714, + "learning_rate": 3.978598815729711e-06, + "loss": 0.92607141, + "num_input_tokens_seen": 72837700, + "router_z_loss_clip": 0.7800293, + "router_z_loss_mlp": 0.21374512, + "step": 2592, + "time_per_iteration": 2.699009656906128 + }, + { + "auxiliary_loss_clip": 0.01117383, + "auxiliary_loss_mlp": 0.01041028, + "balance_loss_clip": 1.03975928, + "balance_loss_mlp": 1.01806855, + "epoch": 0.075242295862109, + "flos": 26316516297600.0, + "grad_norm": 2.0201269152367565, + "language_loss": 0.79745245, + "learning_rate": 3.978571383455557e-06, + "loss": 0.81903654, + "num_input_tokens_seen": 72854155, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.22973633, + "step": 2593, + "time_per_iteration": 2.460893392562866 + }, + { + "auxiliary_loss_clip": 0.01108372, + "auxiliary_loss_mlp": 0.01050541, + "balance_loss_clip": 1.03422999, + "balance_loss_mlp": 1.03060365, + "epoch": 0.07527131333062503, + "flos": 33099547173120.0, + "grad_norm": 2.0488779561420496, + "language_loss": 0.78971446, + "learning_rate": 3.978543933705859e-06, + "loss": 0.81130362, + "num_input_tokens_seen": 72868965, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.19934082, + "step": 2594, + "time_per_iteration": 4.852231025695801 + }, + { + "auxiliary_loss_clip": 0.01104767, + "auxiliary_loss_mlp": 0.01035337, + "balance_loss_clip": 1.03365827, + "balance_loss_mlp": 1.01716375, + "epoch": 0.07530033079914109, + "flos": 17266959135360.0, + "grad_norm": 2.2978649235675466, + "language_loss": 0.74217069, + "learning_rate": 3.978516466480862e-06, + "loss": 0.76357174, + "num_input_tokens_seen": 72881580, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.18182373, + "step": 2595, + "time_per_iteration": 2.4003453254699707 + }, + { + "auxiliary_loss_clip": 0.01115888, + "auxiliary_loss_mlp": 0.01038225, + "balance_loss_clip": 1.03664184, + "balance_loss_mlp": 1.01689863, + "epoch": 0.07532934826765712, + "flos": 22047022644480.0, + "grad_norm": 2.5550932575657894, + "language_loss": 0.84697223, + "learning_rate": 3.978488981780805e-06, + "loss": 0.86851346, + "num_input_tokens_seen": 72895465, + "router_z_loss_clip": 0.79199219, + "router_z_loss_mlp": 0.21337891, + "step": 2596, + "time_per_iteration": 4.952139616012573 + }, + { + "auxiliary_loss_clip": 0.01113927, + "auxiliary_loss_mlp": 0.01052331, + "balance_loss_clip": 1.03581476, + "balance_loss_mlp": 1.03075469, + "epoch": 0.07535836573617317, + "flos": 20623474068480.0, + "grad_norm": 3.2787420312621216, + "language_loss": 0.83109784, + "learning_rate": 3.978461479605933e-06, + "loss": 0.85276043, + "num_input_tokens_seen": 72907820, + "router_z_loss_clip": 0.78222656, + "router_z_loss_mlp": 0.21569824, + "step": 2597, + "time_per_iteration": 2.378948450088501 + }, + { + "auxiliary_loss_clip": 0.01018173, + "auxiliary_loss_mlp": 0.01003575, + "balance_loss_clip": 1.00309789, + "balance_loss_mlp": 1.00202572, + "epoch": 0.07538738320468923, + "flos": 71493042424320.0, + "grad_norm": 0.742938135018677, + "language_loss": 0.48631284, + "learning_rate": 3.97843395995649e-06, + "loss": 0.50653028, + "num_input_tokens_seen": 72972135, + "router_z_loss_clip": 0.15039062, + "router_z_loss_mlp": 0.01550293, + "step": 2598, + "time_per_iteration": 3.0661468505859375 + }, + { + "auxiliary_loss_clip": 0.01118094, + "auxiliary_loss_mlp": 0.01044311, + "balance_loss_clip": 1.03834391, + "balance_loss_mlp": 1.02119589, + "epoch": 0.07541640067320526, + "flos": 12022166252160.0, + "grad_norm": 11.874744547253211, + "language_loss": 0.86088502, + "learning_rate": 3.978406422832717e-06, + "loss": 0.88250905, + "num_input_tokens_seen": 72984960, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.23095703, + "step": 2599, + "time_per_iteration": 2.395866870880127 + }, + { + "auxiliary_loss_clip": 0.01115257, + "auxiliary_loss_mlp": 0.01056407, + "balance_loss_clip": 1.03558445, + "balance_loss_mlp": 1.03445423, + "epoch": 0.07544541814172132, + "flos": 19565885030400.0, + "grad_norm": 2.8861053926899882, + "language_loss": 0.85743719, + "learning_rate": 3.978378868234858e-06, + "loss": 0.87915379, + "num_input_tokens_seen": 72997870, + "router_z_loss_clip": 0.79541016, + "router_z_loss_mlp": 0.21942139, + "step": 2600, + "time_per_iteration": 2.3500263690948486 + }, + { + "auxiliary_loss_clip": 0.01126389, + "auxiliary_loss_mlp": 0.01055429, + "balance_loss_clip": 1.03674769, + "balance_loss_mlp": 1.02654469, + "epoch": 0.07547443561023737, + "flos": 35109950659200.0, + "grad_norm": 2.94863977486944, + "language_loss": 1.11370778, + "learning_rate": 3.978351296163156e-06, + "loss": 1.13552594, + "num_input_tokens_seen": 73019975, + "router_z_loss_clip": 0.89550781, + "router_z_loss_mlp": 0.28881836, + "step": 2601, + "time_per_iteration": 2.4977006912231445 + }, + { + "auxiliary_loss_clip": 0.01120264, + "auxiliary_loss_mlp": 0.01053852, + "balance_loss_clip": 1.03590059, + "balance_loss_mlp": 1.0299747, + "epoch": 0.0755034530787534, + "flos": 53283360090240.0, + "grad_norm": 3.184536705727729, + "language_loss": 0.9584614, + "learning_rate": 3.978323706617855e-06, + "loss": 0.98020256, + "num_input_tokens_seen": 73040055, + "router_z_loss_clip": 0.84326172, + "router_z_loss_mlp": 0.23901367, + "step": 2602, + "time_per_iteration": 2.7053475379943848 + }, + { + "auxiliary_loss_clip": 0.01017186, + "auxiliary_loss_mlp": 0.0100322, + "balance_loss_clip": 1.00255084, + "balance_loss_mlp": 1.00181341, + "epoch": 0.07553247054726946, + "flos": 73755449170560.0, + "grad_norm": 0.6242068729745464, + "language_loss": 0.51424801, + "learning_rate": 3.978296099599198e-06, + "loss": 0.53445208, + "num_input_tokens_seen": 73107940, + "router_z_loss_clip": 0.14648438, + "router_z_loss_mlp": 0.01403809, + "step": 2603, + "time_per_iteration": 3.1174535751342773 + }, + { + "auxiliary_loss_clip": 0.01105983, + "auxiliary_loss_mlp": 0.01040096, + "balance_loss_clip": 1.03391218, + "balance_loss_mlp": 1.02137995, + "epoch": 0.07556148801578551, + "flos": 34341407700480.0, + "grad_norm": 2.3654618594602166, + "language_loss": 0.88206565, + "learning_rate": 3.97826847510743e-06, + "loss": 0.90352643, + "num_input_tokens_seen": 73123475, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.18707275, + "step": 2604, + "time_per_iteration": 2.4934427738189697 + }, + { + "auxiliary_loss_clip": 0.01017104, + "auxiliary_loss_mlp": 0.01003382, + "balance_loss_clip": 1.00282335, + "balance_loss_mlp": 1.00191605, + "epoch": 0.07559050548430155, + "flos": 71598899266560.0, + "grad_norm": 0.6885054705739571, + "language_loss": 0.52583635, + "learning_rate": 3.978240833142794e-06, + "loss": 0.54604125, + "num_input_tokens_seen": 73184880, + "router_z_loss_clip": 0.14257812, + "router_z_loss_mlp": 0.01464844, + "step": 2605, + "time_per_iteration": 3.0231411457061768 + }, + { + "auxiliary_loss_clip": 0.01118114, + "auxiliary_loss_mlp": 0.01051322, + "balance_loss_clip": 1.03594303, + "balance_loss_mlp": 1.02927995, + "epoch": 0.0756195229528176, + "flos": 30476766706560.0, + "grad_norm": 2.5765849039887856, + "language_loss": 0.90680778, + "learning_rate": 3.978213173705534e-06, + "loss": 0.92850214, + "num_input_tokens_seen": 73202505, + "router_z_loss_clip": 0.82226562, + "router_z_loss_mlp": 0.22045898, + "step": 2606, + "time_per_iteration": 2.492361068725586 + }, + { + "auxiliary_loss_clip": 0.01113377, + "auxiliary_loss_mlp": 0.01049213, + "balance_loss_clip": 1.03467321, + "balance_loss_mlp": 1.02584839, + "epoch": 0.07564854042133365, + "flos": 16032395082240.0, + "grad_norm": 2.024286127879991, + "language_loss": 0.76563811, + "learning_rate": 3.978185496795896e-06, + "loss": 0.78726399, + "num_input_tokens_seen": 73216290, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.23376465, + "step": 2607, + "time_per_iteration": 2.488840341567993 + }, + { + "auxiliary_loss_clip": 0.0112058, + "auxiliary_loss_mlp": 0.01045371, + "balance_loss_clip": 1.03589404, + "balance_loss_mlp": 1.02192283, + "epoch": 0.07567755788984969, + "flos": 33867287170560.0, + "grad_norm": 2.2072078806756625, + "language_loss": 0.77357042, + "learning_rate": 3.978157802414122e-06, + "loss": 0.79522997, + "num_input_tokens_seen": 73231450, + "router_z_loss_clip": 0.84765625, + "router_z_loss_mlp": 0.23461914, + "step": 2608, + "time_per_iteration": 2.51339054107666 + }, + { + "auxiliary_loss_clip": 0.01016868, + "auxiliary_loss_mlp": 0.01001548, + "balance_loss_clip": 1.00267553, + "balance_loss_mlp": 1.00020659, + "epoch": 0.07570657535836574, + "flos": 74231873850240.0, + "grad_norm": 0.6768391413716975, + "language_loss": 0.53298241, + "learning_rate": 3.978130090560458e-06, + "loss": 0.55316657, + "num_input_tokens_seen": 73294030, + "router_z_loss_clip": 0.14160156, + "router_z_loss_mlp": 0.01342773, + "step": 2609, + "time_per_iteration": 3.1123156547546387 + }, + { + "auxiliary_loss_clip": 0.01114765, + "auxiliary_loss_mlp": 0.0105322, + "balance_loss_clip": 1.03609669, + "balance_loss_mlp": 1.02989078, + "epoch": 0.07573559282688179, + "flos": 23943469852800.0, + "grad_norm": 2.5724998994733532, + "language_loss": 0.91289878, + "learning_rate": 3.978102361235149e-06, + "loss": 0.93457866, + "num_input_tokens_seen": 73310480, + "router_z_loss_clip": 0.78662109, + "router_z_loss_mlp": 0.2331543, + "step": 2610, + "time_per_iteration": 2.636925220489502 + }, + { + "auxiliary_loss_clip": 0.01123616, + "auxiliary_loss_mlp": 0.01045632, + "balance_loss_clip": 1.03561878, + "balance_loss_mlp": 1.02185011, + "epoch": 0.07576461029539783, + "flos": 24748113024000.0, + "grad_norm": 3.15447549484375, + "language_loss": 0.95871609, + "learning_rate": 3.978074614438439e-06, + "loss": 0.98040867, + "num_input_tokens_seen": 73325480, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.23779297, + "step": 2611, + "time_per_iteration": 2.527252674102783 + }, + { + "auxiliary_loss_clip": 0.01131036, + "auxiliary_loss_mlp": 0.01055392, + "balance_loss_clip": 1.0403235, + "balance_loss_mlp": 1.02686572, + "epoch": 0.07579362776391388, + "flos": 15988963484160.0, + "grad_norm": 2.5066518328937994, + "language_loss": 0.83630186, + "learning_rate": 3.978046850170574e-06, + "loss": 0.8581661, + "num_input_tokens_seen": 73338820, + "router_z_loss_clip": 0.90625, + "router_z_loss_mlp": 0.28503418, + "step": 2612, + "time_per_iteration": 2.3809731006622314 + }, + { + "auxiliary_loss_clip": 0.01017762, + "auxiliary_loss_mlp": 0.01005006, + "balance_loss_clip": 1.00342584, + "balance_loss_mlp": 1.00366473, + "epoch": 0.07582264523242992, + "flos": 61532007730560.0, + "grad_norm": 0.6436479039453652, + "language_loss": 0.51144648, + "learning_rate": 3.978019068431799e-06, + "loss": 0.53167415, + "num_input_tokens_seen": 73403590, + "router_z_loss_clip": 0.14257812, + "router_z_loss_mlp": 0.01342773, + "step": 2613, + "time_per_iteration": 3.026398181915283 + }, + { + "auxiliary_loss_clip": 0.01108463, + "auxiliary_loss_mlp": 0.01042167, + "balance_loss_clip": 1.03315938, + "balance_loss_mlp": 1.0223428, + "epoch": 0.07585166270094597, + "flos": 18543907445760.0, + "grad_norm": 3.403454904892585, + "language_loss": 0.87223577, + "learning_rate": 3.977991269222358e-06, + "loss": 0.89374208, + "num_input_tokens_seen": 73415705, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.19824219, + "step": 2614, + "time_per_iteration": 2.406860113143921 + }, + { + "auxiliary_loss_clip": 0.01118265, + "auxiliary_loss_mlp": 0.01045139, + "balance_loss_clip": 1.03521419, + "balance_loss_mlp": 1.02285838, + "epoch": 0.07588068016946202, + "flos": 34015633004160.0, + "grad_norm": 1.7906832876845904, + "language_loss": 0.85772514, + "learning_rate": 3.977963452542499e-06, + "loss": 0.87935913, + "num_input_tokens_seen": 73437920, + "router_z_loss_clip": 0.83056641, + "router_z_loss_mlp": 0.22277832, + "step": 2615, + "time_per_iteration": 2.6955864429473877 + }, + { + "auxiliary_loss_clip": 0.01119158, + "auxiliary_loss_mlp": 0.01043824, + "balance_loss_clip": 1.03862166, + "balance_loss_mlp": 1.02377295, + "epoch": 0.07590969763797806, + "flos": 19271497512960.0, + "grad_norm": 3.8606392983853746, + "language_loss": 0.77301675, + "learning_rate": 3.977935618392466e-06, + "loss": 0.79464662, + "num_input_tokens_seen": 73448955, + "router_z_loss_clip": 0.80566406, + "router_z_loss_mlp": 0.20068359, + "step": 2616, + "time_per_iteration": 2.630324125289917 + }, + { + "auxiliary_loss_clip": 0.0110921, + "auxiliary_loss_mlp": 0.01053386, + "balance_loss_clip": 1.03467751, + "balance_loss_mlp": 1.0329895, + "epoch": 0.07593871510649411, + "flos": 29963508675840.0, + "grad_norm": 2.157508334905772, + "language_loss": 0.80152106, + "learning_rate": 3.977907766772505e-06, + "loss": 0.82314706, + "num_input_tokens_seen": 73464185, + "router_z_loss_clip": 0.74560547, + "router_z_loss_mlp": 0.20385742, + "step": 2617, + "time_per_iteration": 2.581841230392456 + }, + { + "auxiliary_loss_clip": 0.01114435, + "auxiliary_loss_mlp": 0.01042794, + "balance_loss_clip": 1.03366458, + "balance_loss_mlp": 1.02071655, + "epoch": 0.07596773257501016, + "flos": 61340651101440.0, + "grad_norm": 3.4097448745687764, + "language_loss": 1.06755888, + "learning_rate": 3.977879897682862e-06, + "loss": 1.08913124, + "num_input_tokens_seen": 73485365, + "router_z_loss_clip": 0.80761719, + "router_z_loss_mlp": 0.2208252, + "step": 2618, + "time_per_iteration": 2.8611676692962646 + }, + { + "auxiliary_loss_clip": 0.0111565, + "auxiliary_loss_mlp": 0.01042804, + "balance_loss_clip": 1.03626132, + "balance_loss_mlp": 1.02214503, + "epoch": 0.0759967500435262, + "flos": 12670538711040.0, + "grad_norm": 3.701392949521098, + "language_loss": 0.83174253, + "learning_rate": 3.977852011123784e-06, + "loss": 0.85332704, + "num_input_tokens_seen": 73498175, + "router_z_loss_clip": 0.79345703, + "router_z_loss_mlp": 0.20666504, + "step": 2619, + "time_per_iteration": 2.346513509750366 + }, + { + "auxiliary_loss_clip": 0.01104159, + "auxiliary_loss_mlp": 0.01034546, + "balance_loss_clip": 1.03460658, + "balance_loss_mlp": 1.01722443, + "epoch": 0.07602576751204225, + "flos": 41132468188800.0, + "grad_norm": 3.9214888339062557, + "language_loss": 0.70681036, + "learning_rate": 3.977824107095516e-06, + "loss": 0.72819746, + "num_input_tokens_seen": 73516305, + "router_z_loss_clip": 0.69433594, + "router_z_loss_mlp": 0.1730957, + "step": 2620, + "time_per_iteration": 2.63287353515625 + }, + { + "auxiliary_loss_clip": 0.0111342, + "auxiliary_loss_mlp": 0.01045317, + "balance_loss_clip": 1.03511715, + "balance_loss_mlp": 1.02289343, + "epoch": 0.0760547849805583, + "flos": 23285776060800.0, + "grad_norm": 2.825846717702483, + "language_loss": 0.69786477, + "learning_rate": 3.977796185598306e-06, + "loss": 0.71945214, + "num_input_tokens_seen": 73530400, + "router_z_loss_clip": 0.78271484, + "router_z_loss_mlp": 0.2244873, + "step": 2621, + "time_per_iteration": 2.3925557136535645 + }, + { + "auxiliary_loss_clip": 0.01016083, + "auxiliary_loss_mlp": 0.01001258, + "balance_loss_clip": 1.00201929, + "balance_loss_mlp": 1.00004792, + "epoch": 0.07608380244907434, + "flos": 69769625247360.0, + "grad_norm": 0.7655514953382381, + "language_loss": 0.49600583, + "learning_rate": 3.977768246632399e-06, + "loss": 0.51617926, + "num_input_tokens_seen": 73588970, + "router_z_loss_clip": 0.140625, + "router_z_loss_mlp": 0.01208496, + "step": 2622, + "time_per_iteration": 2.9478251934051514 + }, + { + "auxiliary_loss_clip": 0.01015211, + "auxiliary_loss_mlp": 0.01004017, + "balance_loss_clip": 1.00134397, + "balance_loss_mlp": 1.00267553, + "epoch": 0.07611281991759039, + "flos": 67252596888960.0, + "grad_norm": 0.726846806699017, + "language_loss": 0.55664772, + "learning_rate": 3.977740290198043e-06, + "loss": 0.57683998, + "num_input_tokens_seen": 73650380, + "router_z_loss_clip": 0.13867188, + "router_z_loss_mlp": 0.01342773, + "step": 2623, + "time_per_iteration": 2.9571170806884766 + }, + { + "auxiliary_loss_clip": 0.01128226, + "auxiliary_loss_mlp": 0.0104214, + "balance_loss_clip": 1.03912926, + "balance_loss_mlp": 1.01747572, + "epoch": 0.07614183738610644, + "flos": 31861422161280.0, + "grad_norm": 2.72596813813339, + "language_loss": 0.95043683, + "learning_rate": 3.977712316295484e-06, + "loss": 0.97214055, + "num_input_tokens_seen": 73669550, + "router_z_loss_clip": 0.89160156, + "router_z_loss_mlp": 0.24658203, + "step": 2624, + "time_per_iteration": 2.5444700717926025 + }, + { + "auxiliary_loss_clip": 0.01119782, + "auxiliary_loss_mlp": 0.01044715, + "balance_loss_clip": 1.03687739, + "balance_loss_mlp": 1.02150512, + "epoch": 0.07617085485462248, + "flos": 11357245808640.0, + "grad_norm": 3.052435687194369, + "language_loss": 0.80671161, + "learning_rate": 3.97768432492497e-06, + "loss": 0.82835656, + "num_input_tokens_seen": 73681775, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.23217773, + "step": 2625, + "time_per_iteration": 2.3231027126312256 + }, + { + "auxiliary_loss_clip": 0.01016057, + "auxiliary_loss_mlp": 0.01001869, + "balance_loss_clip": 1.00223422, + "balance_loss_mlp": 1.00055194, + "epoch": 0.07619987232313853, + "flos": 74761821377280.0, + "grad_norm": 0.640121507788924, + "language_loss": 0.51192689, + "learning_rate": 3.977656316086748e-06, + "loss": 0.53210616, + "num_input_tokens_seen": 73740250, + "router_z_loss_clip": 0.13867188, + "router_z_loss_mlp": 0.01318359, + "step": 2626, + "time_per_iteration": 3.074192523956299 + }, + { + "auxiliary_loss_clip": 0.01118166, + "auxiliary_loss_mlp": 0.01044069, + "balance_loss_clip": 1.03685534, + "balance_loss_mlp": 1.02352905, + "epoch": 0.07622888979165457, + "flos": 23834436076800.0, + "grad_norm": 1.9479161156124738, + "language_loss": 0.76889288, + "learning_rate": 3.977628289781064e-06, + "loss": 0.79051518, + "num_input_tokens_seen": 73761195, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.20544434, + "step": 2627, + "time_per_iteration": 2.5757334232330322 + }, + { + "auxiliary_loss_clip": 0.0111635, + "auxiliary_loss_mlp": 0.01043361, + "balance_loss_clip": 1.03413081, + "balance_loss_mlp": 1.01963866, + "epoch": 0.07625790726017062, + "flos": 23726135439360.0, + "grad_norm": 1.9801763360278697, + "language_loss": 0.95003271, + "learning_rate": 3.977600246008167e-06, + "loss": 0.97162986, + "num_input_tokens_seen": 73780540, + "router_z_loss_clip": 0.82128906, + "router_z_loss_mlp": 0.23706055, + "step": 2628, + "time_per_iteration": 2.606889009475708 + }, + { + "auxiliary_loss_clip": 0.01016054, + "auxiliary_loss_mlp": 0.01002763, + "balance_loss_clip": 1.00211537, + "balance_loss_mlp": 1.00134408, + "epoch": 0.07628692472868667, + "flos": 60322721368320.0, + "grad_norm": 0.6756901753694955, + "language_loss": 0.47661939, + "learning_rate": 3.977572184768305e-06, + "loss": 0.49680755, + "num_input_tokens_seen": 73837700, + "router_z_loss_clip": 0.13867188, + "router_z_loss_mlp": 0.01416016, + "step": 2629, + "time_per_iteration": 3.0106470584869385 + }, + { + "auxiliary_loss_clip": 0.01108254, + "auxiliary_loss_mlp": 0.01034647, + "balance_loss_clip": 1.03465915, + "balance_loss_mlp": 1.01563931, + "epoch": 0.07631594219720271, + "flos": 17849799239040.0, + "grad_norm": 3.6783070408406497, + "language_loss": 0.81084949, + "learning_rate": 3.977544106061725e-06, + "loss": 0.83227849, + "num_input_tokens_seen": 73849375, + "router_z_loss_clip": 0.73632812, + "router_z_loss_mlp": 0.18988037, + "step": 2630, + "time_per_iteration": 2.4216835498809814 + }, + { + "auxiliary_loss_clip": 0.01125225, + "auxiliary_loss_mlp": 0.01054661, + "balance_loss_clip": 1.04067075, + "balance_loss_mlp": 1.03171372, + "epoch": 0.07634495966571876, + "flos": 16209300274560.0, + "grad_norm": 2.2739927359543524, + "language_loss": 0.84021455, + "learning_rate": 3.977516009888675e-06, + "loss": 0.8620134, + "num_input_tokens_seen": 73863710, + "router_z_loss_clip": 0.84619141, + "router_z_loss_mlp": 0.22961426, + "step": 2631, + "time_per_iteration": 2.3826398849487305 + }, + { + "auxiliary_loss_clip": 0.01100352, + "auxiliary_loss_mlp": 0.01035452, + "balance_loss_clip": 1.03138018, + "balance_loss_mlp": 1.01778483, + "epoch": 0.07637397713423481, + "flos": 19056781451520.0, + "grad_norm": 2.1425457111079824, + "language_loss": 0.86470222, + "learning_rate": 3.977487896249404e-06, + "loss": 0.88606024, + "num_input_tokens_seen": 73878780, + "router_z_loss_clip": 0.68945312, + "router_z_loss_mlp": 0.17675781, + "step": 2632, + "time_per_iteration": 2.431598424911499 + }, + { + "auxiliary_loss_clip": 0.01110948, + "auxiliary_loss_mlp": 0.01044287, + "balance_loss_clip": 1.03297186, + "balance_loss_mlp": 1.02402091, + "epoch": 0.07640299460275085, + "flos": 53204561418240.0, + "grad_norm": 1.935614366252818, + "language_loss": 0.60498929, + "learning_rate": 3.977459765144159e-06, + "loss": 0.62654161, + "num_input_tokens_seen": 73907175, + "router_z_loss_clip": 0.77783203, + "router_z_loss_mlp": 0.20275879, + "step": 2633, + "time_per_iteration": 2.6853690147399902 + }, + { + "auxiliary_loss_clip": 0.01108826, + "auxiliary_loss_mlp": 0.01039133, + "balance_loss_clip": 1.03264594, + "balance_loss_mlp": 1.01681685, + "epoch": 0.0764320120712669, + "flos": 35435655532800.0, + "grad_norm": 4.640973239738044, + "language_loss": 1.12359202, + "learning_rate": 3.9774316165731895e-06, + "loss": 1.14507163, + "num_input_tokens_seen": 73922135, + "router_z_loss_clip": 0.76318359, + "router_z_loss_mlp": 0.22314453, + "step": 2634, + "time_per_iteration": 2.4366326332092285 + }, + { + "auxiliary_loss_clip": 0.01016402, + "auxiliary_loss_mlp": 0.01003374, + "balance_loss_clip": 1.00274026, + "balance_loss_mlp": 1.00208604, + "epoch": 0.07646102953978295, + "flos": 67395007791360.0, + "grad_norm": 0.6840733241948516, + "language_loss": 0.52554059, + "learning_rate": 3.977403450536744e-06, + "loss": 0.54573834, + "num_input_tokens_seen": 73983365, + "router_z_loss_clip": 0.13671875, + "router_z_loss_mlp": 0.01287842, + "step": 2635, + "time_per_iteration": 3.0076704025268555 + }, + { + "auxiliary_loss_clip": 0.01107925, + "auxiliary_loss_mlp": 0.01037348, + "balance_loss_clip": 1.03271747, + "balance_loss_mlp": 1.01519907, + "epoch": 0.07649004700829899, + "flos": 24343434921600.0, + "grad_norm": 2.7273100914323996, + "language_loss": 0.81996274, + "learning_rate": 3.977375267035071e-06, + "loss": 0.84141546, + "num_input_tokens_seen": 73997605, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.22155762, + "step": 2636, + "time_per_iteration": 2.4260711669921875 + }, + { + "auxiliary_loss_clip": 0.01015945, + "auxiliary_loss_mlp": 0.0100545, + "balance_loss_clip": 1.00276542, + "balance_loss_mlp": 1.00425184, + "epoch": 0.07651906447681504, + "flos": 62079166558080.0, + "grad_norm": 0.9045560273694172, + "language_loss": 0.48512691, + "learning_rate": 3.977347066068419e-06, + "loss": 0.50534081, + "num_input_tokens_seen": 74060975, + "router_z_loss_clip": 0.13183594, + "router_z_loss_mlp": 0.01196289, + "step": 2637, + "time_per_iteration": 3.0322563648223877 + }, + { + "auxiliary_loss_clip": 0.01115531, + "auxiliary_loss_mlp": 0.01040341, + "balance_loss_clip": 1.03670549, + "balance_loss_mlp": 1.02086258, + "epoch": 0.0765480819453311, + "flos": 13947452110080.0, + "grad_norm": 2.337807793491519, + "language_loss": 0.92162538, + "learning_rate": 3.977318847637038e-06, + "loss": 0.94318414, + "num_input_tokens_seen": 74072655, + "router_z_loss_clip": 0.78759766, + "router_z_loss_mlp": 0.19470215, + "step": 2638, + "time_per_iteration": 2.3495283126831055 + }, + { + "auxiliary_loss_clip": 0.01015171, + "auxiliary_loss_mlp": 0.010025, + "balance_loss_clip": 1.0016638, + "balance_loss_mlp": 1.00124276, + "epoch": 0.07657709941384713, + "flos": 69245160670080.0, + "grad_norm": 0.6940679619658945, + "language_loss": 0.4751032, + "learning_rate": 3.977290611741177e-06, + "loss": 0.49527991, + "num_input_tokens_seen": 74125000, + "router_z_loss_clip": 0.13476562, + "router_z_loss_mlp": 0.01257324, + "step": 2639, + "time_per_iteration": 2.9286952018737793 + }, + { + "auxiliary_loss_clip": 0.0101472, + "auxiliary_loss_mlp": 0.01001599, + "balance_loss_clip": 1.00149643, + "balance_loss_mlp": 1.00024033, + "epoch": 0.07660611688236318, + "flos": 61508023626240.0, + "grad_norm": 0.8053523872889499, + "language_loss": 0.50359088, + "learning_rate": 3.977262358381084e-06, + "loss": 0.52375406, + "num_input_tokens_seen": 74176480, + "router_z_loss_clip": 0.1328125, + "router_z_loss_mlp": 0.01361084, + "step": 2640, + "time_per_iteration": 3.080620050430298 + }, + { + "auxiliary_loss_clip": 0.01014597, + "auxiliary_loss_mlp": 0.01001646, + "balance_loss_clip": 1.00132251, + "balance_loss_mlp": 1.00044227, + "epoch": 0.07663513435087924, + "flos": 60129198368640.0, + "grad_norm": 0.666821380277016, + "language_loss": 0.479534, + "learning_rate": 3.977234087557011e-06, + "loss": 0.4996964, + "num_input_tokens_seen": 74232010, + "router_z_loss_clip": 0.1328125, + "router_z_loss_mlp": 0.01202393, + "step": 2641, + "time_per_iteration": 2.9062461853027344 + }, + { + "auxiliary_loss_clip": 0.01014761, + "auxiliary_loss_mlp": 0.01003342, + "balance_loss_clip": 1.00158715, + "balance_loss_mlp": 1.00211406, + "epoch": 0.07666415181939527, + "flos": 74771387089920.0, + "grad_norm": 0.6737604352516544, + "language_loss": 0.47745141, + "learning_rate": 3.977205799269206e-06, + "loss": 0.49763244, + "num_input_tokens_seen": 74298780, + "router_z_loss_clip": 0.13085938, + "router_z_loss_mlp": 0.01226807, + "step": 2642, + "time_per_iteration": 3.15490460395813 + }, + { + "auxiliary_loss_clip": 0.01109875, + "auxiliary_loss_mlp": 0.01040928, + "balance_loss_clip": 1.03414297, + "balance_loss_mlp": 1.02103174, + "epoch": 0.07669316928791133, + "flos": 74727538421760.0, + "grad_norm": 6.159161859531523, + "language_loss": 0.94409418, + "learning_rate": 3.977177493517919e-06, + "loss": 0.96560216, + "num_input_tokens_seen": 74321455, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.19891357, + "step": 2643, + "time_per_iteration": 2.792358875274658 + }, + { + "auxiliary_loss_clip": 0.01108802, + "auxiliary_loss_mlp": 0.01041549, + "balance_loss_clip": 1.03239238, + "balance_loss_mlp": 1.01995993, + "epoch": 0.07672218675642736, + "flos": 10553161219200.0, + "grad_norm": 3.09600757510985, + "language_loss": 0.81055689, + "learning_rate": 3.977149170303401e-06, + "loss": 0.83206034, + "num_input_tokens_seen": 74331995, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.21582031, + "step": 2644, + "time_per_iteration": 2.38818621635437 + }, + { + "auxiliary_loss_clip": 0.01013959, + "auxiliary_loss_mlp": 0.01001613, + "balance_loss_clip": 1.00081182, + "balance_loss_mlp": 1.00044489, + "epoch": 0.07675120422494341, + "flos": 62185197957120.0, + "grad_norm": 0.6754085532733483, + "language_loss": 0.5185442, + "learning_rate": 3.977120829625901e-06, + "loss": 0.53869998, + "num_input_tokens_seen": 74392960, + "router_z_loss_clip": 0.13085938, + "router_z_loss_mlp": 0.01165771, + "step": 2645, + "time_per_iteration": 2.950434923171997 + }, + { + "auxiliary_loss_clip": 0.01014325, + "auxiliary_loss_mlp": 0.01002068, + "balance_loss_clip": 1.00128746, + "balance_loss_mlp": 1.00090528, + "epoch": 0.07678022169345947, + "flos": 74772364608000.0, + "grad_norm": 0.6316698130384012, + "language_loss": 0.52678788, + "learning_rate": 3.97709247148567e-06, + "loss": 0.54695177, + "num_input_tokens_seen": 74457960, + "router_z_loss_clip": 0.13085938, + "router_z_loss_mlp": 0.01159668, + "step": 2646, + "time_per_iteration": 3.2739055156707764 + }, + { + "auxiliary_loss_clip": 0.0109429, + "auxiliary_loss_mlp": 0.01037305, + "balance_loss_clip": 1.02937102, + "balance_loss_mlp": 1.02141476, + "epoch": 0.0768092391619755, + "flos": 24090244675200.0, + "grad_norm": 2.6829270244064527, + "language_loss": 0.7509535, + "learning_rate": 3.977064095882958e-06, + "loss": 0.77226943, + "num_input_tokens_seen": 74472125, + "router_z_loss_clip": 0.64990234, + "router_z_loss_mlp": 0.15869141, + "step": 2647, + "time_per_iteration": 2.3709967136383057 + }, + { + "auxiliary_loss_clip": 0.01112543, + "auxiliary_loss_mlp": 0.01035997, + "balance_loss_clip": 1.03492427, + "balance_loss_mlp": 1.01417017, + "epoch": 0.07683825663049156, + "flos": 46162684656000.0, + "grad_norm": 2.1814396097005146, + "language_loss": 0.82312989, + "learning_rate": 3.977035702818016e-06, + "loss": 0.84461528, + "num_input_tokens_seen": 74490740, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.21838379, + "step": 2648, + "time_per_iteration": 2.5394036769866943 + }, + { + "auxiliary_loss_clip": 0.0110528, + "auxiliary_loss_mlp": 0.01042338, + "balance_loss_clip": 1.0318073, + "balance_loss_mlp": 1.02359843, + "epoch": 0.07686727409900761, + "flos": 18268546619520.0, + "grad_norm": 2.9782161439844144, + "language_loss": 0.85975337, + "learning_rate": 3.977007292291094e-06, + "loss": 0.88122952, + "num_input_tokens_seen": 74509400, + "router_z_loss_clip": 0.73535156, + "router_z_loss_mlp": 0.18713379, + "step": 2649, + "time_per_iteration": 2.3852875232696533 + }, + { + "auxiliary_loss_clip": 0.01116309, + "auxiliary_loss_mlp": 0.01047512, + "balance_loss_clip": 1.03543139, + "balance_loss_mlp": 1.02769899, + "epoch": 0.07689629156752364, + "flos": 26570823707520.0, + "grad_norm": 2.1054537323665223, + "language_loss": 0.89942485, + "learning_rate": 3.976978864302445e-06, + "loss": 0.92106307, + "num_input_tokens_seen": 74526865, + "router_z_loss_clip": 0.80908203, + "router_z_loss_mlp": 0.19805908, + "step": 2650, + "time_per_iteration": 2.509512186050415 + }, + { + "auxiliary_loss_clip": 0.01016671, + "auxiliary_loss_mlp": 0.01001669, + "balance_loss_clip": 1.00360274, + "balance_loss_mlp": 1.0005486, + "epoch": 0.0769253090360397, + "flos": 74773761062400.0, + "grad_norm": 0.6763016989708411, + "language_loss": 0.4498997, + "learning_rate": 3.976950418852317e-06, + "loss": 0.47008309, + "num_input_tokens_seen": 74592660, + "router_z_loss_clip": 0.13085938, + "router_z_loss_mlp": 0.01123047, + "step": 2651, + "time_per_iteration": 3.066040515899658 + }, + { + "auxiliary_loss_clip": 0.01016448, + "auxiliary_loss_mlp": 0.01003269, + "balance_loss_clip": 1.00344074, + "balance_loss_mlp": 1.00196946, + "epoch": 0.07695432650455575, + "flos": 69552465390720.0, + "grad_norm": 0.6688794376509885, + "language_loss": 0.53762251, + "learning_rate": 3.976921955940964e-06, + "loss": 0.55781966, + "num_input_tokens_seen": 74654495, + "router_z_loss_clip": 0.12988281, + "router_z_loss_mlp": 0.01300049, + "step": 2652, + "time_per_iteration": 3.0346627235412598 + }, + { + "auxiliary_loss_clip": 0.0111622, + "auxiliary_loss_mlp": 0.01049358, + "balance_loss_clip": 1.03626442, + "balance_loss_mlp": 1.02794254, + "epoch": 0.07698334397307179, + "flos": 28099321430400.0, + "grad_norm": 2.840692777917185, + "language_loss": 0.99368817, + "learning_rate": 3.976893475568636e-06, + "loss": 1.0153439, + "num_input_tokens_seen": 74666805, + "router_z_loss_clip": 0.79833984, + "router_z_loss_mlp": 0.21411133, + "step": 2653, + "time_per_iteration": 2.4774928092956543 + }, + { + "auxiliary_loss_clip": 0.01110051, + "auxiliary_loss_mlp": 0.01041046, + "balance_loss_clip": 1.03406608, + "balance_loss_mlp": 1.01956475, + "epoch": 0.07701236144158784, + "flos": 13510583867520.0, + "grad_norm": 3.1850630501169794, + "language_loss": 1.02168787, + "learning_rate": 3.976864977735585e-06, + "loss": 1.04319882, + "num_input_tokens_seen": 74678975, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.21472168, + "step": 2654, + "time_per_iteration": 2.356917142868042 + }, + { + "auxiliary_loss_clip": 0.01125256, + "auxiliary_loss_mlp": 0.01047777, + "balance_loss_clip": 1.03823876, + "balance_loss_mlp": 1.02404261, + "epoch": 0.07704137891010389, + "flos": 26031345379200.0, + "grad_norm": 2.1141844119231363, + "language_loss": 0.81453884, + "learning_rate": 3.976836462442062e-06, + "loss": 0.83626914, + "num_input_tokens_seen": 74695205, + "router_z_loss_clip": 0.87011719, + "router_z_loss_mlp": 0.23754883, + "step": 2655, + "time_per_iteration": 2.4241955280303955 + }, + { + "auxiliary_loss_clip": 0.01111461, + "auxiliary_loss_mlp": 0.01042419, + "balance_loss_clip": 1.03309202, + "balance_loss_mlp": 1.02160537, + "epoch": 0.07707039637861993, + "flos": 24527636588160.0, + "grad_norm": 2.968883437967796, + "language_loss": 0.91271156, + "learning_rate": 3.976807929688321e-06, + "loss": 0.93425047, + "num_input_tokens_seen": 74711340, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.20837402, + "step": 2656, + "time_per_iteration": 2.458146572113037 + }, + { + "auxiliary_loss_clip": 0.01112776, + "auxiliary_loss_mlp": 0.01046289, + "balance_loss_clip": 1.03402591, + "balance_loss_mlp": 1.02325153, + "epoch": 0.07709941384713598, + "flos": 48797125516800.0, + "grad_norm": 2.0862908247184024, + "language_loss": 0.87586451, + "learning_rate": 3.976779379474611e-06, + "loss": 0.8974551, + "num_input_tokens_seen": 74733100, + "router_z_loss_clip": 0.78808594, + "router_z_loss_mlp": 0.23034668, + "step": 2657, + "time_per_iteration": 2.640331268310547 + }, + { + "auxiliary_loss_clip": 0.01116298, + "auxiliary_loss_mlp": 0.01052833, + "balance_loss_clip": 1.03542984, + "balance_loss_mlp": 1.02841926, + "epoch": 0.07712843131565202, + "flos": 34272419120640.0, + "grad_norm": 2.0074506114899706, + "language_loss": 0.87262517, + "learning_rate": 3.976750811801186e-06, + "loss": 0.89431649, + "num_input_tokens_seen": 74755560, + "router_z_loss_clip": 0.80957031, + "router_z_loss_mlp": 0.24414062, + "step": 2658, + "time_per_iteration": 2.5366058349609375 + }, + { + "auxiliary_loss_clip": 0.01014598, + "auxiliary_loss_mlp": 0.0100431, + "balance_loss_clip": 1.00189853, + "balance_loss_mlp": 1.00314176, + "epoch": 0.07715744878416807, + "flos": 74772050405760.0, + "grad_norm": 0.6332445830392739, + "language_loss": 0.49196422, + "learning_rate": 3.9767222266682975e-06, + "loss": 0.51215327, + "num_input_tokens_seen": 74822545, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01165771, + "step": 2659, + "time_per_iteration": 3.261190891265869 + }, + { + "auxiliary_loss_clip": 0.01117112, + "auxiliary_loss_mlp": 0.01046826, + "balance_loss_clip": 1.03552783, + "balance_loss_mlp": 1.02468848, + "epoch": 0.07718646625268412, + "flos": 44740567445760.0, + "grad_norm": 2.3524842027067834, + "language_loss": 0.80316889, + "learning_rate": 3.976693624076199e-06, + "loss": 0.82480824, + "num_input_tokens_seen": 74841230, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.22125244, + "step": 2660, + "time_per_iteration": 2.6053004264831543 + }, + { + "auxiliary_loss_clip": 0.01115229, + "auxiliary_loss_mlp": 0.01043451, + "balance_loss_clip": 1.03466034, + "balance_loss_mlp": 1.02269673, + "epoch": 0.07721548372120016, + "flos": 13252401296640.0, + "grad_norm": 2.35158947487464, + "language_loss": 0.82687086, + "learning_rate": 3.9766650040251426e-06, + "loss": 0.84845769, + "num_input_tokens_seen": 74854345, + "router_z_loss_clip": 0.80615234, + "router_z_loss_mlp": 0.2076416, + "step": 2661, + "time_per_iteration": 2.393146276473999 + }, + { + "auxiliary_loss_clip": 0.0101456, + "auxiliary_loss_mlp": 0.01001048, + "balance_loss_clip": 1.00167096, + "balance_loss_mlp": 0.99986744, + "epoch": 0.07724450118971621, + "flos": 71531237318400.0, + "grad_norm": 0.6127646586080442, + "language_loss": 0.50381255, + "learning_rate": 3.976636366515381e-06, + "loss": 0.52396864, + "num_input_tokens_seen": 74924010, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01177979, + "step": 2662, + "time_per_iteration": 3.109760284423828 + }, + { + "auxiliary_loss_clip": 0.01113421, + "auxiliary_loss_mlp": 0.0105614, + "balance_loss_clip": 1.03400469, + "balance_loss_mlp": 1.0322032, + "epoch": 0.07727351865823226, + "flos": 25258543234560.0, + "grad_norm": 2.264122212242791, + "language_loss": 0.85707051, + "learning_rate": 3.976607711547166e-06, + "loss": 0.87876618, + "num_input_tokens_seen": 74939925, + "router_z_loss_clip": 0.79296875, + "router_z_loss_mlp": 0.23950195, + "step": 2663, + "time_per_iteration": 2.3981099128723145 + }, + { + "auxiliary_loss_clip": 0.01115366, + "auxiliary_loss_mlp": 0.01051821, + "balance_loss_clip": 1.03425086, + "balance_loss_mlp": 1.02770519, + "epoch": 0.0773025361267483, + "flos": 19639621555200.0, + "grad_norm": 2.4393124674464652, + "language_loss": 0.95218134, + "learning_rate": 3.976579039120753e-06, + "loss": 0.97385323, + "num_input_tokens_seen": 74955055, + "router_z_loss_clip": 0.81054688, + "router_z_loss_mlp": 0.24108887, + "step": 2664, + "time_per_iteration": 2.348769426345825 + }, + { + "auxiliary_loss_clip": 0.01106131, + "auxiliary_loss_mlp": 0.01041183, + "balance_loss_clip": 1.03056431, + "balance_loss_mlp": 1.02041709, + "epoch": 0.07733155359526435, + "flos": 32189501007360.0, + "grad_norm": 2.291614820442223, + "language_loss": 0.75922942, + "learning_rate": 3.976550349236394e-06, + "loss": 0.78070259, + "num_input_tokens_seen": 74971665, + "router_z_loss_clip": 0.75512695, + "router_z_loss_mlp": 0.20751953, + "step": 2665, + "time_per_iteration": 4.537835121154785 + }, + { + "auxiliary_loss_clip": 0.01112369, + "auxiliary_loss_mlp": 0.01046698, + "balance_loss_clip": 1.0343076, + "balance_loss_mlp": 1.02496624, + "epoch": 0.0773605710637804, + "flos": 11428014867840.0, + "grad_norm": 4.168724294817798, + "language_loss": 0.825028, + "learning_rate": 3.976521641894342e-06, + "loss": 0.84661871, + "num_input_tokens_seen": 74982995, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.21728516, + "step": 2666, + "time_per_iteration": 4.65468955039978 + }, + { + "auxiliary_loss_clip": 0.0111519, + "auxiliary_loss_mlp": 0.01047755, + "balance_loss_clip": 1.03554368, + "balance_loss_mlp": 1.02563012, + "epoch": 0.07738958853229644, + "flos": 29125104353280.0, + "grad_norm": 2.805709401964914, + "language_loss": 0.82739973, + "learning_rate": 3.976492917094851e-06, + "loss": 0.84902924, + "num_input_tokens_seen": 75000725, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.22119141, + "step": 2667, + "time_per_iteration": 2.480992555618286 + }, + { + "auxiliary_loss_clip": 0.01014842, + "auxiliary_loss_mlp": 0.01012496, + "balance_loss_clip": 1.00181866, + "balance_loss_mlp": 1.01108289, + "epoch": 0.07741860600081249, + "flos": 74773272303360.0, + "grad_norm": 0.6889606050413634, + "language_loss": 0.49397492, + "learning_rate": 3.976464174838175e-06, + "loss": 0.51424825, + "num_input_tokens_seen": 75061805, + "router_z_loss_clip": 0.13085938, + "router_z_loss_mlp": 0.01409912, + "step": 2668, + "time_per_iteration": 3.0427515506744385 + }, + { + "auxiliary_loss_clip": 0.01014123, + "auxiliary_loss_mlp": 0.01006421, + "balance_loss_clip": 1.00129914, + "balance_loss_mlp": 1.00514555, + "epoch": 0.07744762346932854, + "flos": 74595529238400.0, + "grad_norm": 0.6561450042115208, + "language_loss": 0.50632101, + "learning_rate": 3.976435415124568e-06, + "loss": 0.52652645, + "num_input_tokens_seen": 75131735, + "router_z_loss_clip": 0.12792969, + "router_z_loss_mlp": 0.01275635, + "step": 2669, + "time_per_iteration": 3.157111406326294 + }, + { + "auxiliary_loss_clip": 0.01105361, + "auxiliary_loss_mlp": 0.01037533, + "balance_loss_clip": 1.03151226, + "balance_loss_mlp": 1.01801872, + "epoch": 0.07747664093784458, + "flos": 11171298574080.0, + "grad_norm": 5.185907940098726, + "language_loss": 1.05749214, + "learning_rate": 3.976406637954283e-06, + "loss": 1.0789212, + "num_input_tokens_seen": 75141625, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.19519043, + "step": 2670, + "time_per_iteration": 4.868021011352539 + }, + { + "auxiliary_loss_clip": 0.01116547, + "auxiliary_loss_mlp": 0.0105359, + "balance_loss_clip": 1.03618884, + "balance_loss_mlp": 1.02917624, + "epoch": 0.07750565840636063, + "flos": 16354992844800.0, + "grad_norm": 2.5895114044647194, + "language_loss": 0.89783412, + "learning_rate": 3.9763778433275755e-06, + "loss": 0.91953552, + "num_input_tokens_seen": 75155325, + "router_z_loss_clip": 0.80371094, + "router_z_loss_mlp": 0.24401855, + "step": 2671, + "time_per_iteration": 2.341444969177246 + }, + { + "auxiliary_loss_clip": 0.01014649, + "auxiliary_loss_mlp": 0.01001983, + "balance_loss_clip": 1.00201499, + "balance_loss_mlp": 1.00064766, + "epoch": 0.07753467587487668, + "flos": 55074507171840.0, + "grad_norm": 0.7331879108382245, + "language_loss": 0.49071005, + "learning_rate": 3.976349031244699e-06, + "loss": 0.51087642, + "num_input_tokens_seen": 75213155, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.0133667, + "step": 2672, + "time_per_iteration": 5.370174884796143 + }, + { + "auxiliary_loss_clip": 0.01118307, + "auxiliary_loss_mlp": 0.01054388, + "balance_loss_clip": 1.03511357, + "balance_loss_mlp": 1.02973509, + "epoch": 0.07756369334339272, + "flos": 40327091879040.0, + "grad_norm": 1.6866770902962644, + "language_loss": 0.75346863, + "learning_rate": 3.976320201705908e-06, + "loss": 0.7751956, + "num_input_tokens_seen": 75233480, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.24645996, + "step": 2673, + "time_per_iteration": 2.558239698410034 + }, + { + "auxiliary_loss_clip": 0.01016848, + "auxiliary_loss_mlp": 0.01006042, + "balance_loss_clip": 1.00378132, + "balance_loss_mlp": 1.00478423, + "epoch": 0.07759271081190877, + "flos": 73167826210560.0, + "grad_norm": 0.6975728486612347, + "language_loss": 0.54062468, + "learning_rate": 3.976291354711457e-06, + "loss": 0.56085354, + "num_input_tokens_seen": 75295235, + "router_z_loss_clip": 0.13085938, + "router_z_loss_mlp": 0.01257324, + "step": 2674, + "time_per_iteration": 3.0189075469970703 + }, + { + "auxiliary_loss_clip": 0.01121748, + "auxiliary_loss_mlp": 0.01049462, + "balance_loss_clip": 1.04146445, + "balance_loss_mlp": 1.02634692, + "epoch": 0.07762172828042481, + "flos": 11720796462720.0, + "grad_norm": 2.8807317904701297, + "language_loss": 0.87515712, + "learning_rate": 3.9762624902616015e-06, + "loss": 0.89686924, + "num_input_tokens_seen": 75306415, + "router_z_loss_clip": 0.80322266, + "router_z_loss_mlp": 0.2310791, + "step": 2675, + "time_per_iteration": 2.4695491790771484 + }, + { + "auxiliary_loss_clip": 0.01113904, + "auxiliary_loss_mlp": 0.0104829, + "balance_loss_clip": 1.03595138, + "balance_loss_mlp": 1.02628386, + "epoch": 0.07765074574894086, + "flos": 34284917387520.0, + "grad_norm": 2.0194777762375176, + "language_loss": 0.88219237, + "learning_rate": 3.976233608356595e-06, + "loss": 0.90381426, + "num_input_tokens_seen": 75326270, + "router_z_loss_clip": 0.77978516, + "router_z_loss_mlp": 0.2199707, + "step": 2676, + "time_per_iteration": 2.503310203552246 + }, + { + "auxiliary_loss_clip": 0.01115045, + "auxiliary_loss_mlp": 0.01047625, + "balance_loss_clip": 1.03911006, + "balance_loss_mlp": 1.02658498, + "epoch": 0.07767976321745691, + "flos": 37809120913920.0, + "grad_norm": 5.51698544389006, + "language_loss": 0.82071602, + "learning_rate": 3.976204708996694e-06, + "loss": 0.84234273, + "num_input_tokens_seen": 75343840, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.21032715, + "step": 2677, + "time_per_iteration": 2.5865697860717773 + }, + { + "auxiliary_loss_clip": 0.01104469, + "auxiliary_loss_mlp": 0.01048401, + "balance_loss_clip": 1.0321449, + "balance_loss_mlp": 1.02823067, + "epoch": 0.07770878068597295, + "flos": 28580459143680.0, + "grad_norm": 1.9990692338422316, + "language_loss": 0.81313944, + "learning_rate": 3.976175792182154e-06, + "loss": 0.83466816, + "num_input_tokens_seen": 75361930, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.20166016, + "step": 2678, + "time_per_iteration": 2.449354887008667 + }, + { + "auxiliary_loss_clip": 0.01103946, + "auxiliary_loss_mlp": 0.01048675, + "balance_loss_clip": 1.03051138, + "balance_loss_mlp": 1.02727747, + "epoch": 0.077737798154489, + "flos": 11646955203840.0, + "grad_norm": 3.0157937507742596, + "language_loss": 1.02923465, + "learning_rate": 3.976146857913229e-06, + "loss": 1.05076087, + "num_input_tokens_seen": 75374260, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.21398926, + "step": 2679, + "time_per_iteration": 2.4489595890045166 + }, + { + "auxiliary_loss_clip": 0.01015034, + "auxiliary_loss_mlp": 0.01003973, + "balance_loss_clip": 1.00266695, + "balance_loss_mlp": 1.00280499, + "epoch": 0.07776681562300505, + "flos": 65849892261120.0, + "grad_norm": 0.7264973186339589, + "language_loss": 0.49486893, + "learning_rate": 3.976117906190176e-06, + "loss": 0.51505899, + "num_input_tokens_seen": 75428390, + "router_z_loss_clip": 0.12353516, + "router_z_loss_mlp": 0.01165771, + "step": 2680, + "time_per_iteration": 2.882451295852661 + }, + { + "auxiliary_loss_clip": 0.01112002, + "auxiliary_loss_mlp": 0.0103781, + "balance_loss_clip": 1.03496766, + "balance_loss_mlp": 1.01703167, + "epoch": 0.07779583309152109, + "flos": 16062734920320.0, + "grad_norm": 2.9747511852485933, + "language_loss": 0.71401489, + "learning_rate": 3.97608893701325e-06, + "loss": 0.73551303, + "num_input_tokens_seen": 75441375, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.20776367, + "step": 2681, + "time_per_iteration": 2.3671514987945557 + }, + { + "auxiliary_loss_clip": 0.01015599, + "auxiliary_loss_mlp": 0.01001599, + "balance_loss_clip": 1.00306153, + "balance_loss_mlp": 1.00028169, + "epoch": 0.07782485056003714, + "flos": 74779137411840.0, + "grad_norm": 0.6465590776813541, + "language_loss": 0.47791731, + "learning_rate": 3.976059950382706e-06, + "loss": 0.49808931, + "num_input_tokens_seen": 75506280, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.01318359, + "step": 2682, + "time_per_iteration": 3.1124107837677 + }, + { + "auxiliary_loss_clip": 0.01115558, + "auxiliary_loss_mlp": 0.01046679, + "balance_loss_clip": 1.03832388, + "balance_loss_mlp": 1.02450633, + "epoch": 0.0778538680285532, + "flos": 19359442961280.0, + "grad_norm": 3.230308195515834, + "language_loss": 0.82727897, + "learning_rate": 3.976030946298802e-06, + "loss": 0.84890139, + "num_input_tokens_seen": 75522010, + "router_z_loss_clip": 0.77197266, + "router_z_loss_mlp": 0.22167969, + "step": 2683, + "time_per_iteration": 2.3593881130218506 + }, + { + "auxiliary_loss_clip": 0.01015331, + "auxiliary_loss_mlp": 0.01001593, + "balance_loss_clip": 1.00292552, + "balance_loss_mlp": 1.00028217, + "epoch": 0.07788288549706923, + "flos": 74777671134720.0, + "grad_norm": 3.1046368194665352, + "language_loss": 0.48821843, + "learning_rate": 3.976001924761791e-06, + "loss": 0.50838768, + "num_input_tokens_seen": 75592465, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.01312256, + "step": 2684, + "time_per_iteration": 3.11946439743042 + }, + { + "auxiliary_loss_clip": 0.01107491, + "auxiliary_loss_mlp": 0.01038926, + "balance_loss_clip": 1.03231871, + "balance_loss_mlp": 1.01750398, + "epoch": 0.07791190296558528, + "flos": 25549509438720.0, + "grad_norm": 2.935923116857068, + "language_loss": 0.84958935, + "learning_rate": 3.975972885771934e-06, + "loss": 0.87105346, + "num_input_tokens_seen": 75606620, + "router_z_loss_clip": 0.75146484, + "router_z_loss_mlp": 0.21411133, + "step": 2685, + "time_per_iteration": 2.426114559173584 + }, + { + "auxiliary_loss_clip": 0.01111462, + "auxiliary_loss_mlp": 0.01043044, + "balance_loss_clip": 1.03351974, + "balance_loss_mlp": 1.02001238, + "epoch": 0.07794092043410134, + "flos": 28431380171520.0, + "grad_norm": 2.2517829288971267, + "language_loss": 0.80375123, + "learning_rate": 3.975943829329483e-06, + "loss": 0.82529634, + "num_input_tokens_seen": 75622240, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.23046875, + "step": 2686, + "time_per_iteration": 2.464318037033081 + }, + { + "auxiliary_loss_clip": 0.01115546, + "auxiliary_loss_mlp": 0.01052432, + "balance_loss_clip": 1.03367305, + "balance_loss_mlp": 1.02745712, + "epoch": 0.07796993790261737, + "flos": 44814792729600.0, + "grad_norm": 2.4785148179120995, + "language_loss": 0.75026745, + "learning_rate": 3.975914755434697e-06, + "loss": 0.77194715, + "num_input_tokens_seen": 75644675, + "router_z_loss_clip": 0.81884766, + "router_z_loss_mlp": 0.24975586, + "step": 2687, + "time_per_iteration": 2.582317352294922 + }, + { + "auxiliary_loss_clip": 0.01015865, + "auxiliary_loss_mlp": 0.01004183, + "balance_loss_clip": 1.00318408, + "balance_loss_mlp": 1.00300264, + "epoch": 0.07799895537113342, + "flos": 64405290268800.0, + "grad_norm": 0.648272703735156, + "language_loss": 0.50190103, + "learning_rate": 3.975885664087833e-06, + "loss": 0.52210152, + "num_input_tokens_seen": 75704125, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01177979, + "step": 2688, + "time_per_iteration": 3.0389082431793213 + }, + { + "auxiliary_loss_clip": 0.01110085, + "auxiliary_loss_mlp": 0.0104711, + "balance_loss_clip": 1.03372252, + "balance_loss_mlp": 1.02468634, + "epoch": 0.07802797283964946, + "flos": 15631906343040.0, + "grad_norm": 2.435822812237484, + "language_loss": 0.66423905, + "learning_rate": 3.975856555289146e-06, + "loss": 0.68581104, + "num_input_tokens_seen": 75720065, + "router_z_loss_clip": 0.76318359, + "router_z_loss_mlp": 0.22424316, + "step": 2689, + "time_per_iteration": 2.3711154460906982 + }, + { + "auxiliary_loss_clip": 0.01123334, + "auxiliary_loss_mlp": 0.01052121, + "balance_loss_clip": 1.03679705, + "balance_loss_mlp": 1.02706361, + "epoch": 0.07805699030816551, + "flos": 26058787574400.0, + "grad_norm": 2.205439187959615, + "language_loss": 0.71392763, + "learning_rate": 3.975827429038895e-06, + "loss": 0.73568219, + "num_input_tokens_seen": 75733235, + "router_z_loss_clip": 0.86621094, + "router_z_loss_mlp": 0.25036621, + "step": 2690, + "time_per_iteration": 2.3980603218078613 + }, + { + "auxiliary_loss_clip": 0.01014189, + "auxiliary_loss_mlp": 0.01000937, + "balance_loss_clip": 1.00162363, + "balance_loss_mlp": 0.99976885, + "epoch": 0.07808600777668157, + "flos": 70439922040320.0, + "grad_norm": 0.6697901122024461, + "language_loss": 0.58037245, + "learning_rate": 3.975798285337337e-06, + "loss": 0.60052371, + "num_input_tokens_seen": 75798305, + "router_z_loss_clip": 0.12597656, + "router_z_loss_mlp": 0.01165771, + "step": 2691, + "time_per_iteration": 3.1046366691589355 + }, + { + "auxiliary_loss_clip": 0.01108849, + "auxiliary_loss_mlp": 0.01045178, + "balance_loss_clip": 1.03246164, + "balance_loss_mlp": 1.02388716, + "epoch": 0.0781150252451976, + "flos": 25295097294720.0, + "grad_norm": 1.7505390448150486, + "language_loss": 0.6451152, + "learning_rate": 3.975769124184729e-06, + "loss": 0.66665542, + "num_input_tokens_seen": 75816115, + "router_z_loss_clip": 0.76367188, + "router_z_loss_mlp": 0.21313477, + "step": 2692, + "time_per_iteration": 2.458252429962158 + }, + { + "auxiliary_loss_clip": 0.01108225, + "auxiliary_loss_mlp": 0.01042712, + "balance_loss_clip": 1.03247213, + "balance_loss_mlp": 1.02273309, + "epoch": 0.07814404271371365, + "flos": 22885985548800.0, + "grad_norm": 2.3956562125059913, + "language_loss": 0.80623794, + "learning_rate": 3.975739945581328e-06, + "loss": 0.82774729, + "num_input_tokens_seen": 75830595, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.19970703, + "step": 2693, + "time_per_iteration": 2.4018635749816895 + }, + { + "auxiliary_loss_clip": 0.01115527, + "auxiliary_loss_mlp": 0.01058659, + "balance_loss_clip": 1.0346669, + "balance_loss_mlp": 1.03584194, + "epoch": 0.0781730601822297, + "flos": 17558483921280.0, + "grad_norm": 2.644908798128184, + "language_loss": 0.80002183, + "learning_rate": 3.975710749527393e-06, + "loss": 0.82176375, + "num_input_tokens_seen": 75843780, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.22827148, + "step": 2694, + "time_per_iteration": 2.4229800701141357 + }, + { + "auxiliary_loss_clip": 0.01117544, + "auxiliary_loss_mlp": 0.01050791, + "balance_loss_clip": 1.03514457, + "balance_loss_mlp": 1.02729499, + "epoch": 0.07820207765074574, + "flos": 15082722656640.0, + "grad_norm": 2.1474453326595904, + "language_loss": 0.74425626, + "learning_rate": 3.9756815360231814e-06, + "loss": 0.76593959, + "num_input_tokens_seen": 75858440, + "router_z_loss_clip": 0.82470703, + "router_z_loss_mlp": 0.23486328, + "step": 2695, + "time_per_iteration": 2.341642379760742 + }, + { + "auxiliary_loss_clip": 0.01113202, + "auxiliary_loss_mlp": 0.01042423, + "balance_loss_clip": 1.03553843, + "balance_loss_mlp": 1.02100134, + "epoch": 0.0782310951192618, + "flos": 74733717732480.0, + "grad_norm": 2.177794991974002, + "language_loss": 0.96058774, + "learning_rate": 3.97565230506895e-06, + "loss": 0.98214388, + "num_input_tokens_seen": 75881125, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.2144165, + "step": 2696, + "time_per_iteration": 2.825787305831909 + }, + { + "auxiliary_loss_clip": 0.01015024, + "auxiliary_loss_mlp": 0.01002062, + "balance_loss_clip": 1.00273299, + "balance_loss_mlp": 1.0009532, + "epoch": 0.07826011258777785, + "flos": 59992512929280.0, + "grad_norm": 0.6769970076693242, + "language_loss": 0.49701178, + "learning_rate": 3.9756230566649584e-06, + "loss": 0.51718271, + "num_input_tokens_seen": 75947155, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.0111084, + "step": 2697, + "time_per_iteration": 3.0598535537719727 + }, + { + "auxiliary_loss_clip": 0.01108368, + "auxiliary_loss_mlp": 0.01042766, + "balance_loss_clip": 1.03428543, + "balance_loss_mlp": 1.02368021, + "epoch": 0.07828913005629388, + "flos": 18689215991040.0, + "grad_norm": 2.3612199008626495, + "language_loss": 0.90058869, + "learning_rate": 3.9755937908114646e-06, + "loss": 0.92210007, + "num_input_tokens_seen": 75960355, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.19067383, + "step": 2698, + "time_per_iteration": 2.386894702911377 + }, + { + "auxiliary_loss_clip": 0.01017014, + "auxiliary_loss_mlp": 0.01001311, + "balance_loss_clip": 1.00450063, + "balance_loss_mlp": 1.00020862, + "epoch": 0.07831814752480994, + "flos": 57220444022400.0, + "grad_norm": 0.6717703836740867, + "language_loss": 0.47276723, + "learning_rate": 3.975564507508727e-06, + "loss": 0.4929505, + "num_input_tokens_seen": 76021415, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.01104736, + "step": 2699, + "time_per_iteration": 2.9709441661834717 + }, + { + "auxiliary_loss_clip": 0.01111358, + "auxiliary_loss_mlp": 0.01040027, + "balance_loss_clip": 1.03222573, + "balance_loss_mlp": 1.01815844, + "epoch": 0.07834716499332599, + "flos": 12012914741760.0, + "grad_norm": 2.7896783338625686, + "language_loss": 0.75609809, + "learning_rate": 3.975535206757004e-06, + "loss": 0.77761197, + "num_input_tokens_seen": 76035080, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.21856689, + "step": 2700, + "time_per_iteration": 2.3865649700164795 + }, + { + "auxiliary_loss_clip": 0.01108822, + "auxiliary_loss_mlp": 0.0105111, + "balance_loss_clip": 1.0327003, + "balance_loss_mlp": 1.02776861, + "epoch": 0.07837618246184203, + "flos": 13910025265920.0, + "grad_norm": 2.8977653649728414, + "language_loss": 0.8012774, + "learning_rate": 3.9755058885565545e-06, + "loss": 0.82287669, + "num_input_tokens_seen": 76046950, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.2333374, + "step": 2701, + "time_per_iteration": 2.4238083362579346 + }, + { + "auxiliary_loss_clip": 0.01112819, + "auxiliary_loss_mlp": 0.01043811, + "balance_loss_clip": 1.0320549, + "balance_loss_mlp": 1.02131665, + "epoch": 0.07840519993035808, + "flos": 54006167301120.0, + "grad_norm": 2.1729334185159916, + "language_loss": 0.76744449, + "learning_rate": 3.975476552907638e-06, + "loss": 0.78901082, + "num_input_tokens_seen": 76070880, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.22497559, + "step": 2702, + "time_per_iteration": 2.7266948223114014 + }, + { + "auxiliary_loss_clip": 0.01118622, + "auxiliary_loss_mlp": 0.01048181, + "balance_loss_clip": 1.03234947, + "balance_loss_mlp": 1.02417266, + "epoch": 0.07843421739887413, + "flos": 11247129780480.0, + "grad_norm": 2.906153720288753, + "language_loss": 0.90697128, + "learning_rate": 3.975447199810513e-06, + "loss": 0.92863929, + "num_input_tokens_seen": 76083240, + "router_z_loss_clip": 0.86181641, + "router_z_loss_mlp": 0.23999023, + "step": 2703, + "time_per_iteration": 2.415311574935913 + }, + { + "auxiliary_loss_clip": 0.01108793, + "auxiliary_loss_mlp": 0.01040547, + "balance_loss_clip": 1.03351963, + "balance_loss_mlp": 1.01999497, + "epoch": 0.07846323486739017, + "flos": 20147887261440.0, + "grad_norm": 2.326637772852106, + "language_loss": 0.79785693, + "learning_rate": 3.975417829265439e-06, + "loss": 0.81935036, + "num_input_tokens_seen": 76098920, + "router_z_loss_clip": 0.75292969, + "router_z_loss_mlp": 0.20568848, + "step": 2704, + "time_per_iteration": 2.379469394683838 + }, + { + "auxiliary_loss_clip": 0.01114518, + "auxiliary_loss_mlp": 0.01042378, + "balance_loss_clip": 1.03416729, + "balance_loss_mlp": 1.01910806, + "epoch": 0.07849225233590622, + "flos": 25950033089280.0, + "grad_norm": 2.2461552146352797, + "language_loss": 0.93619841, + "learning_rate": 3.975388441272676e-06, + "loss": 0.95776737, + "num_input_tokens_seen": 76117625, + "router_z_loss_clip": 0.80371094, + "router_z_loss_mlp": 0.23291016, + "step": 2705, + "time_per_iteration": 2.545067071914673 + }, + { + "auxiliary_loss_clip": 0.01112668, + "auxiliary_loss_mlp": 0.0103867, + "balance_loss_clip": 1.03191662, + "balance_loss_mlp": 1.01673532, + "epoch": 0.07852126980442226, + "flos": 39411948654720.0, + "grad_norm": 2.9212295441664162, + "language_loss": 0.9798072, + "learning_rate": 3.975359035832482e-06, + "loss": 1.0013206, + "num_input_tokens_seen": 76133145, + "router_z_loss_clip": 0.80761719, + "router_z_loss_mlp": 0.21960449, + "step": 2706, + "time_per_iteration": 2.464756727218628 + }, + { + "auxiliary_loss_clip": 0.01015438, + "auxiliary_loss_mlp": 0.01006024, + "balance_loss_clip": 1.00240374, + "balance_loss_mlp": 1.00490355, + "epoch": 0.07855028727293831, + "flos": 74771631469440.0, + "grad_norm": 0.8259460165874255, + "language_loss": 0.49510521, + "learning_rate": 3.975329612945118e-06, + "loss": 0.51531982, + "num_input_tokens_seen": 76196575, + "router_z_loss_clip": 0.13085938, + "router_z_loss_mlp": 0.01123047, + "step": 2707, + "time_per_iteration": 3.100116491317749 + }, + { + "auxiliary_loss_clip": 0.01108345, + "auxiliary_loss_mlp": 0.01044334, + "balance_loss_clip": 1.0313077, + "balance_loss_mlp": 1.0252248, + "epoch": 0.07857930474145436, + "flos": 17635432291200.0, + "grad_norm": 2.5373164709389346, + "language_loss": 0.72361314, + "learning_rate": 3.975300172610844e-06, + "loss": 0.74513996, + "num_input_tokens_seen": 76211780, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.19104004, + "step": 2708, + "time_per_iteration": 2.40100359916687 + }, + { + "auxiliary_loss_clip": 0.01101418, + "auxiliary_loss_mlp": 0.01041404, + "balance_loss_clip": 1.03281617, + "balance_loss_mlp": 1.02194881, + "epoch": 0.0786083222099704, + "flos": 34596448248960.0, + "grad_norm": 11.000029766587346, + "language_loss": 0.67389095, + "learning_rate": 3.975270714829919e-06, + "loss": 0.69531918, + "num_input_tokens_seen": 76228080, + "router_z_loss_clip": 0.68652344, + "router_z_loss_mlp": 0.19445801, + "step": 2709, + "time_per_iteration": 2.519883632659912 + }, + { + "auxiliary_loss_clip": 0.01120015, + "auxiliary_loss_mlp": 0.01043629, + "balance_loss_clip": 1.03451777, + "balance_loss_mlp": 1.01789188, + "epoch": 0.07863733967848645, + "flos": 10769587937280.0, + "grad_norm": 3.3897145976033385, + "language_loss": 1.00623703, + "learning_rate": 3.975241239602604e-06, + "loss": 1.02787352, + "num_input_tokens_seen": 76237385, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.25732422, + "step": 2710, + "time_per_iteration": 2.3512279987335205 + }, + { + "auxiliary_loss_clip": 0.01014615, + "auxiliary_loss_mlp": 0.01001043, + "balance_loss_clip": 1.0019449, + "balance_loss_mlp": 1.00008297, + "epoch": 0.0786663571470025, + "flos": 60515755608960.0, + "grad_norm": 0.7040991511417347, + "language_loss": 0.55169022, + "learning_rate": 3.975211746929158e-06, + "loss": 0.57184672, + "num_input_tokens_seen": 76303305, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.00958252, + "step": 2711, + "time_per_iteration": 3.154693365097046 + }, + { + "auxiliary_loss_clip": 0.01117436, + "auxiliary_loss_mlp": 0.01042307, + "balance_loss_clip": 1.03684974, + "balance_loss_mlp": 1.01908469, + "epoch": 0.07869537461551854, + "flos": 23906357210880.0, + "grad_norm": 2.078712454800377, + "language_loss": 0.81767172, + "learning_rate": 3.975182236809844e-06, + "loss": 0.83926916, + "num_input_tokens_seen": 76317690, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.23205566, + "step": 2712, + "time_per_iteration": 2.3890724182128906 + }, + { + "auxiliary_loss_clip": 0.01107817, + "auxiliary_loss_mlp": 0.01043661, + "balance_loss_clip": 1.03261435, + "balance_loss_mlp": 1.02171457, + "epoch": 0.07872439208403459, + "flos": 39523321491840.0, + "grad_norm": 2.1149861066053424, + "language_loss": 0.65356374, + "learning_rate": 3.97515270924492e-06, + "loss": 0.67507851, + "num_input_tokens_seen": 76335030, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.21948242, + "step": 2713, + "time_per_iteration": 2.559727668762207 + }, + { + "auxiliary_loss_clip": 0.01101117, + "auxiliary_loss_mlp": 0.01040951, + "balance_loss_clip": 1.03068829, + "balance_loss_mlp": 1.01992226, + "epoch": 0.07875340955255064, + "flos": 42478370167680.0, + "grad_norm": 3.671807535101549, + "language_loss": 0.90725285, + "learning_rate": 3.9751231642346485e-06, + "loss": 0.92867357, + "num_input_tokens_seen": 76351790, + "router_z_loss_clip": 0.70361328, + "router_z_loss_mlp": 0.21020508, + "step": 2714, + "time_per_iteration": 2.555182456970215 + }, + { + "auxiliary_loss_clip": 0.01113365, + "auxiliary_loss_mlp": 0.01047566, + "balance_loss_clip": 1.03426814, + "balance_loss_mlp": 1.02755082, + "epoch": 0.07878242702106668, + "flos": 32663481891840.0, + "grad_norm": 2.488223722365559, + "language_loss": 0.92114276, + "learning_rate": 3.97509360177929e-06, + "loss": 0.94275212, + "num_input_tokens_seen": 76366260, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.20019531, + "step": 2715, + "time_per_iteration": 2.5245749950408936 + }, + { + "auxiliary_loss_clip": 0.0101487, + "auxiliary_loss_mlp": 0.01002184, + "balance_loss_clip": 1.00254536, + "balance_loss_mlp": 1.00118852, + "epoch": 0.07881144448958273, + "flos": 59687547269760.0, + "grad_norm": 0.763198760162738, + "language_loss": 0.50965965, + "learning_rate": 3.975064021879106e-06, + "loss": 0.52983022, + "num_input_tokens_seen": 76419485, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.00994873, + "step": 2716, + "time_per_iteration": 2.871048927307129 + }, + { + "auxiliary_loss_clip": 0.01109114, + "auxiliary_loss_mlp": 0.01043721, + "balance_loss_clip": 1.03349721, + "balance_loss_mlp": 1.02155972, + "epoch": 0.07884046195809878, + "flos": 30256220448000.0, + "grad_norm": 2.2215749544688124, + "language_loss": 1.01775098, + "learning_rate": 3.975034424534358e-06, + "loss": 1.03927946, + "num_input_tokens_seen": 76436070, + "router_z_loss_clip": 0.75634766, + "router_z_loss_mlp": 0.22155762, + "step": 2717, + "time_per_iteration": 2.5171878337860107 + }, + { + "auxiliary_loss_clip": 0.01113528, + "auxiliary_loss_mlp": 0.01049477, + "balance_loss_clip": 1.03344631, + "balance_loss_mlp": 1.02772117, + "epoch": 0.07886947942661482, + "flos": 43723721831040.0, + "grad_norm": 2.0821416092532585, + "language_loss": 0.82124954, + "learning_rate": 3.975004809745305e-06, + "loss": 0.84287953, + "num_input_tokens_seen": 76458335, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.2175293, + "step": 2718, + "time_per_iteration": 2.6026766300201416 + }, + { + "auxiliary_loss_clip": 0.01014507, + "auxiliary_loss_mlp": 0.01001733, + "balance_loss_clip": 1.00223684, + "balance_loss_mlp": 1.0007081, + "epoch": 0.07889849689513087, + "flos": 74771666380800.0, + "grad_norm": 0.7196709401364365, + "language_loss": 0.53361738, + "learning_rate": 3.974975177512212e-06, + "loss": 0.55377978, + "num_input_tokens_seen": 76521570, + "router_z_loss_clip": 0.12255859, + "router_z_loss_mlp": 0.01025391, + "step": 2719, + "time_per_iteration": 3.1296136379241943 + }, + { + "auxiliary_loss_clip": 0.01118047, + "auxiliary_loss_mlp": 0.01052873, + "balance_loss_clip": 1.03566563, + "balance_loss_mlp": 1.02761269, + "epoch": 0.07892751436364692, + "flos": 29494205913600.0, + "grad_norm": 2.0406561084955595, + "language_loss": 1.01441205, + "learning_rate": 3.9749455278353375e-06, + "loss": 1.03612125, + "num_input_tokens_seen": 76544700, + "router_z_loss_clip": 0.82373047, + "router_z_loss_mlp": 0.25256348, + "step": 2720, + "time_per_iteration": 2.600407361984253 + }, + { + "auxiliary_loss_clip": 0.01108608, + "auxiliary_loss_mlp": 0.01032902, + "balance_loss_clip": 1.03420174, + "balance_loss_mlp": 1.01163507, + "epoch": 0.07895653183216296, + "flos": 32918312972160.0, + "grad_norm": 2.846135611200532, + "language_loss": 0.80627811, + "learning_rate": 3.974915860714946e-06, + "loss": 0.82769322, + "num_input_tokens_seen": 76561395, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.21252441, + "step": 2721, + "time_per_iteration": 2.533327579498291 + }, + { + "auxiliary_loss_clip": 0.01013938, + "auxiliary_loss_mlp": 0.01001304, + "balance_loss_clip": 1.00168371, + "balance_loss_mlp": 1.00024319, + "epoch": 0.07898554930067901, + "flos": 61878277261440.0, + "grad_norm": 0.7088387456094665, + "language_loss": 0.51911426, + "learning_rate": 3.9748861761512975e-06, + "loss": 0.53926671, + "num_input_tokens_seen": 76618945, + "router_z_loss_clip": 0.12255859, + "router_z_loss_mlp": 0.01062012, + "step": 2722, + "time_per_iteration": 2.9346115589141846 + }, + { + "auxiliary_loss_clip": 0.0101342, + "auxiliary_loss_mlp": 0.01001119, + "balance_loss_clip": 1.00115395, + "balance_loss_mlp": 1.00009394, + "epoch": 0.07901456676919505, + "flos": 70540332710400.0, + "grad_norm": 0.6908764129878675, + "language_loss": 0.49451551, + "learning_rate": 3.9748564741446556e-06, + "loss": 0.51466089, + "num_input_tokens_seen": 76685010, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01025391, + "step": 2723, + "time_per_iteration": 3.1056771278381348 + }, + { + "auxiliary_loss_clip": 0.01122287, + "auxiliary_loss_mlp": 0.01053104, + "balance_loss_clip": 1.03544724, + "balance_loss_mlp": 1.0248394, + "epoch": 0.0790435842377111, + "flos": 11757385434240.0, + "grad_norm": 2.5795697492454166, + "language_loss": 0.93158138, + "learning_rate": 3.974826754695283e-06, + "loss": 0.95333529, + "num_input_tokens_seen": 76697200, + "router_z_loss_clip": 0.86816406, + "router_z_loss_mlp": 0.28234863, + "step": 2724, + "time_per_iteration": 2.3746137619018555 + }, + { + "auxiliary_loss_clip": 0.01118413, + "auxiliary_loss_mlp": 0.01053709, + "balance_loss_clip": 1.03577065, + "balance_loss_mlp": 1.0300107, + "epoch": 0.07907260170622715, + "flos": 10917898859520.0, + "grad_norm": 3.4750637613753965, + "language_loss": 0.93276042, + "learning_rate": 3.97479701780344e-06, + "loss": 0.9544816, + "num_input_tokens_seen": 76706635, + "router_z_loss_clip": 0.82714844, + "router_z_loss_mlp": 0.23681641, + "step": 2725, + "time_per_iteration": 2.3520126342773438 + }, + { + "auxiliary_loss_clip": 0.01116087, + "auxiliary_loss_mlp": 0.01048276, + "balance_loss_clip": 1.03732443, + "balance_loss_mlp": 1.02538753, + "epoch": 0.07910161917474319, + "flos": 15664864533120.0, + "grad_norm": 4.057298722113381, + "language_loss": 0.8483842, + "learning_rate": 3.974767263469391e-06, + "loss": 0.87002784, + "num_input_tokens_seen": 76720415, + "router_z_loss_clip": 0.78613281, + "router_z_loss_mlp": 0.22888184, + "step": 2726, + "time_per_iteration": 2.378046989440918 + }, + { + "auxiliary_loss_clip": 0.01013499, + "auxiliary_loss_mlp": 0.01000934, + "balance_loss_clip": 1.00105762, + "balance_loss_mlp": 0.99986082, + "epoch": 0.07913063664325924, + "flos": 60036258729600.0, + "grad_norm": 0.6693584244624745, + "language_loss": 0.51611328, + "learning_rate": 3.974737491693399e-06, + "loss": 0.53625762, + "num_input_tokens_seen": 76785720, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.01074219, + "step": 2727, + "time_per_iteration": 2.99918794631958 + }, + { + "auxiliary_loss_clip": 0.01117154, + "auxiliary_loss_mlp": 0.01048971, + "balance_loss_clip": 1.03639388, + "balance_loss_mlp": 1.02512908, + "epoch": 0.0791596541117753, + "flos": 28540169568000.0, + "grad_norm": 2.9969479994414523, + "language_loss": 0.8783626, + "learning_rate": 3.9747077024757255e-06, + "loss": 0.90002382, + "num_input_tokens_seen": 76799525, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.23852539, + "step": 2728, + "time_per_iteration": 2.4855000972747803 + }, + { + "auxiliary_loss_clip": 0.01114405, + "auxiliary_loss_mlp": 0.0104183, + "balance_loss_clip": 1.03373027, + "balance_loss_mlp": 1.01828623, + "epoch": 0.07918867158029133, + "flos": 13436149115520.0, + "grad_norm": 2.6337576581118944, + "language_loss": 0.65646887, + "learning_rate": 3.974677895816636e-06, + "loss": 0.67803121, + "num_input_tokens_seen": 76811155, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.23535156, + "step": 2729, + "time_per_iteration": 2.32902193069458 + }, + { + "auxiliary_loss_clip": 0.01013163, + "auxiliary_loss_mlp": 0.01001407, + "balance_loss_clip": 1.00081253, + "balance_loss_mlp": 1.00040555, + "epoch": 0.07921768904880738, + "flos": 74763985881600.0, + "grad_norm": 0.6844429899230583, + "language_loss": 0.53567618, + "learning_rate": 3.974648071716391e-06, + "loss": 0.5558219, + "num_input_tokens_seen": 76870800, + "router_z_loss_clip": 0.12353516, + "router_z_loss_mlp": 0.01000977, + "step": 2730, + "time_per_iteration": 3.101943016052246 + }, + { + "auxiliary_loss_clip": 0.01013548, + "auxiliary_loss_mlp": 0.01000842, + "balance_loss_clip": 1.00126362, + "balance_loss_mlp": 0.99984688, + "epoch": 0.07924670651732343, + "flos": 72018414696960.0, + "grad_norm": 0.663511738867551, + "language_loss": 0.46102846, + "learning_rate": 3.974618230175255e-06, + "loss": 0.48117232, + "num_input_tokens_seen": 76936485, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.00994873, + "step": 2731, + "time_per_iteration": 3.2455389499664307 + }, + { + "auxiliary_loss_clip": 0.01112189, + "auxiliary_loss_mlp": 0.01041744, + "balance_loss_clip": 1.03481615, + "balance_loss_mlp": 1.02056026, + "epoch": 0.07927572398583947, + "flos": 33511242458880.0, + "grad_norm": 1.636955536333309, + "language_loss": 0.77736002, + "learning_rate": 3.974588371193492e-06, + "loss": 0.79889941, + "num_input_tokens_seen": 76958055, + "router_z_loss_clip": 0.77294922, + "router_z_loss_mlp": 0.21179199, + "step": 2732, + "time_per_iteration": 2.5593106746673584 + }, + { + "auxiliary_loss_clip": 0.01123153, + "auxiliary_loss_mlp": 0.01055558, + "balance_loss_clip": 1.03520012, + "balance_loss_mlp": 1.02853298, + "epoch": 0.07930474145435552, + "flos": 32664040473600.0, + "grad_norm": 2.081475198514667, + "language_loss": 1.06472778, + "learning_rate": 3.974558494771366e-06, + "loss": 1.08651483, + "num_input_tokens_seen": 76976190, + "router_z_loss_clip": 0.88037109, + "router_z_loss_mlp": 0.27038574, + "step": 2733, + "time_per_iteration": 2.4735162258148193 + }, + { + "auxiliary_loss_clip": 0.01112074, + "auxiliary_loss_mlp": 0.01039367, + "balance_loss_clip": 1.03557897, + "balance_loss_mlp": 1.01722991, + "epoch": 0.07933375892287158, + "flos": 31576670179200.0, + "grad_norm": 2.0132792496558607, + "language_loss": 0.68836617, + "learning_rate": 3.97452860090914e-06, + "loss": 0.70988059, + "num_input_tokens_seen": 76998640, + "router_z_loss_clip": 0.76464844, + "router_z_loss_mlp": 0.22131348, + "step": 2734, + "time_per_iteration": 2.536318063735962 + }, + { + "auxiliary_loss_clip": 0.01116465, + "auxiliary_loss_mlp": 0.01054238, + "balance_loss_clip": 1.03596449, + "balance_loss_mlp": 1.03031909, + "epoch": 0.07936277639138761, + "flos": 40616487072000.0, + "grad_norm": 3.6391832431613604, + "language_loss": 1.01712465, + "learning_rate": 3.974498689607078e-06, + "loss": 1.03883171, + "num_input_tokens_seen": 77015445, + "router_z_loss_clip": 0.80419922, + "router_z_loss_mlp": 0.23937988, + "step": 2735, + "time_per_iteration": 2.5813419818878174 + }, + { + "auxiliary_loss_clip": 0.01117477, + "auxiliary_loss_mlp": 0.01040593, + "balance_loss_clip": 1.03593707, + "balance_loss_mlp": 1.01812172, + "epoch": 0.07939179385990366, + "flos": 16248577420800.0, + "grad_norm": 2.4065132560619324, + "language_loss": 0.76533628, + "learning_rate": 3.974468760865446e-06, + "loss": 0.78691697, + "num_input_tokens_seen": 77028385, + "router_z_loss_clip": 0.81591797, + "router_z_loss_mlp": 0.22460938, + "step": 2736, + "time_per_iteration": 2.33974027633667 + }, + { + "auxiliary_loss_clip": 0.01111191, + "auxiliary_loss_mlp": 0.01041174, + "balance_loss_clip": 1.03465748, + "balance_loss_mlp": 1.02007413, + "epoch": 0.0794208113284197, + "flos": 69877543726080.0, + "grad_norm": 2.6537882570039955, + "language_loss": 0.81067848, + "learning_rate": 3.974438814684506e-06, + "loss": 0.83220208, + "num_input_tokens_seen": 77050635, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.21118164, + "step": 2737, + "time_per_iteration": 2.7757608890533447 + }, + { + "auxiliary_loss_clip": 0.01015203, + "auxiliary_loss_mlp": 0.0100169, + "balance_loss_clip": 1.00292635, + "balance_loss_mlp": 1.00044441, + "epoch": 0.07944982879693575, + "flos": 67373884552320.0, + "grad_norm": 0.6720105119564037, + "language_loss": 0.48461187, + "learning_rate": 3.974408851064523e-06, + "loss": 0.50478089, + "num_input_tokens_seen": 77117345, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01245117, + "step": 2738, + "time_per_iteration": 3.1623315811157227 + }, + { + "auxiliary_loss_clip": 0.01120251, + "auxiliary_loss_mlp": 0.01046498, + "balance_loss_clip": 1.0363735, + "balance_loss_mlp": 1.02169108, + "epoch": 0.0794788462654518, + "flos": 14787148152960.0, + "grad_norm": 2.902508647975109, + "language_loss": 0.9068886, + "learning_rate": 3.974378870005762e-06, + "loss": 0.92855608, + "num_input_tokens_seen": 77130245, + "router_z_loss_clip": 0.83837891, + "router_z_loss_mlp": 0.2479248, + "step": 2739, + "time_per_iteration": 2.4062654972076416 + }, + { + "auxiliary_loss_clip": 0.01014366, + "auxiliary_loss_mlp": 0.01001259, + "balance_loss_clip": 1.00207615, + "balance_loss_mlp": 1.00003088, + "epoch": 0.07950786373396784, + "flos": 70438874699520.0, + "grad_norm": 0.7162524043765084, + "language_loss": 0.52834213, + "learning_rate": 3.9743488715084884e-06, + "loss": 0.54849833, + "num_input_tokens_seen": 77198770, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01226807, + "step": 2740, + "time_per_iteration": 3.1538867950439453 + }, + { + "auxiliary_loss_clip": 0.01115064, + "auxiliary_loss_mlp": 0.01049515, + "balance_loss_clip": 1.03499126, + "balance_loss_mlp": 1.02625728, + "epoch": 0.0795368812024839, + "flos": 28942892634240.0, + "grad_norm": 1.9045060802197153, + "language_loss": 0.88605648, + "learning_rate": 3.974318855572967e-06, + "loss": 0.90770233, + "num_input_tokens_seen": 77216390, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.23266602, + "step": 2741, + "time_per_iteration": 7.278194189071655 + }, + { + "auxiliary_loss_clip": 0.01107588, + "auxiliary_loss_mlp": 0.01047633, + "balance_loss_clip": 1.03195429, + "balance_loss_mlp": 1.02556705, + "epoch": 0.07956589867099995, + "flos": 17739822856320.0, + "grad_norm": 2.5477579743822254, + "language_loss": 0.86948204, + "learning_rate": 3.9742888221994616e-06, + "loss": 0.89103425, + "num_input_tokens_seen": 77228980, + "router_z_loss_clip": 0.75634766, + "router_z_loss_mlp": 0.22058105, + "step": 2742, + "time_per_iteration": 2.3652071952819824 + }, + { + "auxiliary_loss_clip": 0.01113421, + "auxiliary_loss_mlp": 0.01050871, + "balance_loss_clip": 1.0370158, + "balance_loss_mlp": 1.02942562, + "epoch": 0.07959491613951598, + "flos": 55574919688320.0, + "grad_norm": 2.310901519851341, + "language_loss": 0.70660323, + "learning_rate": 3.974258771388239e-06, + "loss": 0.72824615, + "num_input_tokens_seen": 77247560, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.21435547, + "step": 2743, + "time_per_iteration": 2.6775972843170166 + }, + { + "auxiliary_loss_clip": 0.01115946, + "auxiliary_loss_mlp": 0.010444, + "balance_loss_clip": 1.03708482, + "balance_loss_mlp": 1.02365744, + "epoch": 0.07962393360803204, + "flos": 18906655138560.0, + "grad_norm": 2.252053268690254, + "language_loss": 0.99880087, + "learning_rate": 3.974228703139564e-06, + "loss": 1.02040434, + "num_input_tokens_seen": 77260075, + "router_z_loss_clip": 0.78857422, + "router_z_loss_mlp": 0.20751953, + "step": 2744, + "time_per_iteration": 2.380056858062744 + }, + { + "auxiliary_loss_clip": 0.01110542, + "auxiliary_loss_mlp": 0.01047957, + "balance_loss_clip": 1.03394413, + "balance_loss_mlp": 1.02676177, + "epoch": 0.07965295107654809, + "flos": 38242847134080.0, + "grad_norm": 2.845884323369105, + "language_loss": 0.95887125, + "learning_rate": 3.9741986174537026e-06, + "loss": 0.98045623, + "num_input_tokens_seen": 77274705, + "router_z_loss_clip": 0.76635742, + "router_z_loss_mlp": 0.21179199, + "step": 2745, + "time_per_iteration": 2.5883779525756836 + }, + { + "auxiliary_loss_clip": 0.01125586, + "auxiliary_loss_mlp": 0.01044819, + "balance_loss_clip": 1.03932738, + "balance_loss_mlp": 1.02054787, + "epoch": 0.07968196854506412, + "flos": 10588702849920.0, + "grad_norm": 2.729722380004703, + "language_loss": 0.75041914, + "learning_rate": 3.97416851433092e-06, + "loss": 0.77212322, + "num_input_tokens_seen": 77287630, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.24291992, + "step": 2746, + "time_per_iteration": 4.8840250968933105 + }, + { + "auxiliary_loss_clip": 0.01113466, + "auxiliary_loss_mlp": 0.01047329, + "balance_loss_clip": 1.03339636, + "balance_loss_mlp": 1.02236629, + "epoch": 0.07971098601358018, + "flos": 32592189162240.0, + "grad_norm": 2.0855303603445416, + "language_loss": 0.88092792, + "learning_rate": 3.974138393771481e-06, + "loss": 0.9025358, + "num_input_tokens_seen": 77307445, + "router_z_loss_clip": 0.80126953, + "router_z_loss_mlp": 0.24975586, + "step": 2747, + "time_per_iteration": 2.5007174015045166 + }, + { + "auxiliary_loss_clip": 0.01112926, + "auxiliary_loss_mlp": 0.01039997, + "balance_loss_clip": 1.03641951, + "balance_loss_mlp": 1.0178721, + "epoch": 0.07974000348209623, + "flos": 21608024808960.0, + "grad_norm": 2.6601153347718034, + "language_loss": 0.88634163, + "learning_rate": 3.974108255775654e-06, + "loss": 0.90787089, + "num_input_tokens_seen": 77322030, + "router_z_loss_clip": 0.76513672, + "router_z_loss_mlp": 0.22125244, + "step": 2748, + "time_per_iteration": 4.824861288070679 + }, + { + "auxiliary_loss_clip": 0.01110345, + "auxiliary_loss_mlp": 0.0104351, + "balance_loss_clip": 1.0340054, + "balance_loss_mlp": 1.02113473, + "epoch": 0.07976902095061227, + "flos": 29452031124480.0, + "grad_norm": 2.141365777223197, + "language_loss": 0.79645848, + "learning_rate": 3.9740781003437035e-06, + "loss": 0.81799698, + "num_input_tokens_seen": 77337660, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.22363281, + "step": 2749, + "time_per_iteration": 2.495596170425415 + }, + { + "auxiliary_loss_clip": 0.01013774, + "auxiliary_loss_mlp": 0.01003845, + "balance_loss_clip": 1.0014832, + "balance_loss_mlp": 1.00263476, + "epoch": 0.07979803841912832, + "flos": 63311880309120.0, + "grad_norm": 0.7203344556792848, + "language_loss": 0.49295098, + "learning_rate": 3.974047927475897e-06, + "loss": 0.51312715, + "num_input_tokens_seen": 77392040, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01208496, + "step": 2750, + "time_per_iteration": 2.8697526454925537 + }, + { + "auxiliary_loss_clip": 0.01108638, + "auxiliary_loss_mlp": 0.01055885, + "balance_loss_clip": 1.03192139, + "balance_loss_mlp": 1.03164959, + "epoch": 0.07982705588764437, + "flos": 54226329534720.0, + "grad_norm": 2.6080811147717444, + "language_loss": 0.95229691, + "learning_rate": 3.9740177371725e-06, + "loss": 0.9739421, + "num_input_tokens_seen": 77417975, + "router_z_loss_clip": 0.76708984, + "router_z_loss_mlp": 0.24206543, + "step": 2751, + "time_per_iteration": 2.7658772468566895 + }, + { + "auxiliary_loss_clip": 0.01111719, + "auxiliary_loss_mlp": 0.01041442, + "balance_loss_clip": 1.03371799, + "balance_loss_mlp": 1.0171113, + "epoch": 0.0798560733561604, + "flos": 42378552990720.0, + "grad_norm": 2.533349965928027, + "language_loss": 0.76353943, + "learning_rate": 3.9739875294337795e-06, + "loss": 0.78507102, + "num_input_tokens_seen": 77439400, + "router_z_loss_clip": 0.78076172, + "router_z_loss_mlp": 0.24328613, + "step": 2752, + "time_per_iteration": 2.609642744064331 + }, + { + "auxiliary_loss_clip": 0.01013639, + "auxiliary_loss_mlp": 0.01000709, + "balance_loss_clip": 1.00127864, + "balance_loss_mlp": 0.99952912, + "epoch": 0.07988509082467646, + "flos": 62985651765120.0, + "grad_norm": 0.779187413629134, + "language_loss": 0.51053852, + "learning_rate": 3.973957304260002e-06, + "loss": 0.53068197, + "num_input_tokens_seen": 77492195, + "router_z_loss_clip": 0.12353516, + "router_z_loss_mlp": 0.01177979, + "step": 2753, + "time_per_iteration": 2.9094789028167725 + }, + { + "auxiliary_loss_clip": 0.01096231, + "auxiliary_loss_mlp": 0.0103463, + "balance_loss_clip": 1.03117394, + "balance_loss_mlp": 1.01841807, + "epoch": 0.0799141082931925, + "flos": 16206053518080.0, + "grad_norm": 2.7265146153065225, + "language_loss": 0.74725187, + "learning_rate": 3.973927061651435e-06, + "loss": 0.76856047, + "num_input_tokens_seen": 77502680, + "router_z_loss_clip": 0.65087891, + "router_z_loss_mlp": 0.16223145, + "step": 2754, + "time_per_iteration": 2.365929126739502 + }, + { + "auxiliary_loss_clip": 0.01113786, + "auxiliary_loss_mlp": 0.01045405, + "balance_loss_clip": 1.03228831, + "balance_loss_mlp": 1.02077615, + "epoch": 0.07994312576170855, + "flos": 12962098408320.0, + "grad_norm": 2.7679112858047548, + "language_loss": 0.67192268, + "learning_rate": 3.973896801608347e-06, + "loss": 0.69351465, + "num_input_tokens_seen": 77516035, + "router_z_loss_clip": 0.81445312, + "router_z_loss_mlp": 0.24645996, + "step": 2755, + "time_per_iteration": 2.381070852279663 + }, + { + "auxiliary_loss_clip": 0.01113599, + "auxiliary_loss_mlp": 0.01047541, + "balance_loss_clip": 1.03407371, + "balance_loss_mlp": 1.02429485, + "epoch": 0.0799721432302246, + "flos": 20514649760640.0, + "grad_norm": 3.605938723927601, + "language_loss": 0.86256456, + "learning_rate": 3.9738665241310016e-06, + "loss": 0.88417602, + "num_input_tokens_seen": 77531290, + "router_z_loss_clip": 0.79541016, + "router_z_loss_mlp": 0.2322998, + "step": 2756, + "time_per_iteration": 2.4157207012176514 + }, + { + "auxiliary_loss_clip": 0.01106481, + "auxiliary_loss_mlp": 0.01042179, + "balance_loss_clip": 1.03249288, + "balance_loss_mlp": 1.02070987, + "epoch": 0.08000116069874064, + "flos": 39927301367040.0, + "grad_norm": 2.0984049395701927, + "language_loss": 0.98927402, + "learning_rate": 3.9738362292196695e-06, + "loss": 1.01076078, + "num_input_tokens_seen": 77553525, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.21459961, + "step": 2757, + "time_per_iteration": 2.567922592163086 + }, + { + "auxiliary_loss_clip": 0.01108785, + "auxiliary_loss_mlp": 0.01042966, + "balance_loss_clip": 1.03218961, + "balance_loss_mlp": 1.02115071, + "epoch": 0.08003017816725669, + "flos": 34342804154880.0, + "grad_norm": 2.2505268863687173, + "language_loss": 0.86039311, + "learning_rate": 3.973805916874616e-06, + "loss": 0.88191062, + "num_input_tokens_seen": 77570860, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.21813965, + "step": 2758, + "time_per_iteration": 2.509460926055908 + }, + { + "auxiliary_loss_clip": 0.01014815, + "auxiliary_loss_mlp": 0.01000842, + "balance_loss_clip": 1.00235415, + "balance_loss_mlp": 0.99964362, + "epoch": 0.08005919563577274, + "flos": 74768838560640.0, + "grad_norm": 0.7041240885647514, + "language_loss": 0.54301739, + "learning_rate": 3.973775587096112e-06, + "loss": 0.56317395, + "num_input_tokens_seen": 77631570, + "router_z_loss_clip": 0.12451172, + "router_z_loss_mlp": 0.01196289, + "step": 2759, + "time_per_iteration": 3.0176966190338135 + }, + { + "auxiliary_loss_clip": 0.01104562, + "auxiliary_loss_mlp": 0.01046005, + "balance_loss_clip": 1.02983046, + "balance_loss_mlp": 1.0233674, + "epoch": 0.08008821310428878, + "flos": 16172990593920.0, + "grad_norm": 2.502188182696102, + "language_loss": 0.7602185, + "learning_rate": 3.973745239884422e-06, + "loss": 0.78172421, + "num_input_tokens_seen": 77645030, + "router_z_loss_clip": 0.74804688, + "router_z_loss_mlp": 0.22644043, + "step": 2760, + "time_per_iteration": 2.3334262371063232 + }, + { + "auxiliary_loss_clip": 0.01114702, + "auxiliary_loss_mlp": 0.01041834, + "balance_loss_clip": 1.03521323, + "balance_loss_mlp": 1.01854086, + "epoch": 0.08011723057280483, + "flos": 30475056049920.0, + "grad_norm": 3.081647965705743, + "language_loss": 0.85784471, + "learning_rate": 3.973714875239815e-06, + "loss": 0.87941003, + "num_input_tokens_seen": 77661270, + "router_z_loss_clip": 0.79443359, + "router_z_loss_mlp": 0.23278809, + "step": 2761, + "time_per_iteration": 2.463449239730835 + }, + { + "auxiliary_loss_clip": 0.01109848, + "auxiliary_loss_mlp": 0.0104639, + "balance_loss_clip": 1.03405142, + "balance_loss_mlp": 1.02316821, + "epoch": 0.08014624804132088, + "flos": 29417467011840.0, + "grad_norm": 1.7556246725116924, + "language_loss": 0.81244624, + "learning_rate": 3.973684493162559e-06, + "loss": 0.83400857, + "num_input_tokens_seen": 77681110, + "router_z_loss_clip": 0.75878906, + "router_z_loss_mlp": 0.2322998, + "step": 2762, + "time_per_iteration": 2.4602231979370117 + }, + { + "auxiliary_loss_clip": 0.01111829, + "auxiliary_loss_mlp": 0.01048276, + "balance_loss_clip": 1.03467262, + "balance_loss_mlp": 1.02650857, + "epoch": 0.08017526550983692, + "flos": 70060314026880.0, + "grad_norm": 2.5208209150155065, + "language_loss": 1.0042336, + "learning_rate": 3.973654093652924e-06, + "loss": 1.02583456, + "num_input_tokens_seen": 77702940, + "router_z_loss_clip": 0.77197266, + "router_z_loss_mlp": 0.21765137, + "step": 2763, + "time_per_iteration": 2.7730634212493896 + }, + { + "auxiliary_loss_clip": 0.01113309, + "auxiliary_loss_mlp": 0.01043399, + "balance_loss_clip": 1.03317666, + "balance_loss_mlp": 1.01950979, + "epoch": 0.08020428297835297, + "flos": 29888899367040.0, + "grad_norm": 3.791311712031682, + "language_loss": 0.96742129, + "learning_rate": 3.973623676711178e-06, + "loss": 0.9889884, + "num_input_tokens_seen": 77720095, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.23901367, + "step": 2764, + "time_per_iteration": 2.5356640815734863 + }, + { + "auxiliary_loss_clip": 0.0101468, + "auxiliary_loss_mlp": 0.01000745, + "balance_loss_clip": 1.00220346, + "balance_loss_mlp": 0.99955308, + "epoch": 0.08023330044686902, + "flos": 63817772042880.0, + "grad_norm": 0.6583456937585833, + "language_loss": 0.46494338, + "learning_rate": 3.973593242337587e-06, + "loss": 0.48509765, + "num_input_tokens_seen": 77780995, + "router_z_loss_clip": 0.12451172, + "router_z_loss_mlp": 0.01190186, + "step": 2765, + "time_per_iteration": 3.1147661209106445 + }, + { + "auxiliary_loss_clip": 0.01100681, + "auxiliary_loss_mlp": 0.01034946, + "balance_loss_clip": 1.02954531, + "balance_loss_mlp": 1.01587296, + "epoch": 0.08026231791538506, + "flos": 15296007352320.0, + "grad_norm": 3.1100156768178233, + "language_loss": 0.86210519, + "learning_rate": 3.973562790532424e-06, + "loss": 0.88346148, + "num_input_tokens_seen": 77792880, + "router_z_loss_clip": 0.7109375, + "router_z_loss_mlp": 0.19049072, + "step": 2766, + "time_per_iteration": 2.36130690574646 + }, + { + "auxiliary_loss_clip": 0.01115056, + "auxiliary_loss_mlp": 0.01042296, + "balance_loss_clip": 1.03516233, + "balance_loss_mlp": 1.01753592, + "epoch": 0.08029133538390111, + "flos": 29489353234560.0, + "grad_norm": 2.5535532652650033, + "language_loss": 0.82055014, + "learning_rate": 3.973532321295955e-06, + "loss": 0.84212363, + "num_input_tokens_seen": 77807655, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.24743652, + "step": 2767, + "time_per_iteration": 2.4284236431121826 + }, + { + "auxiliary_loss_clip": 0.01110117, + "auxiliary_loss_mlp": 0.01044843, + "balance_loss_clip": 1.03079224, + "balance_loss_mlp": 1.02137089, + "epoch": 0.08032035285241715, + "flos": 45623660175360.0, + "grad_norm": 2.3497215360518826, + "language_loss": 0.92854762, + "learning_rate": 3.973501834628449e-06, + "loss": 0.9500972, + "num_input_tokens_seen": 77829060, + "router_z_loss_clip": 0.79248047, + "router_z_loss_mlp": 0.23486328, + "step": 2768, + "time_per_iteration": 2.663454294204712 + }, + { + "auxiliary_loss_clip": 0.01116264, + "auxiliary_loss_mlp": 0.01044553, + "balance_loss_clip": 1.03397584, + "balance_loss_mlp": 1.01929259, + "epoch": 0.0803493703209332, + "flos": 20478689193600.0, + "grad_norm": 2.2806668972197586, + "language_loss": 0.94208503, + "learning_rate": 3.9734713305301775e-06, + "loss": 0.96369314, + "num_input_tokens_seen": 77844735, + "router_z_loss_clip": 0.82324219, + "router_z_loss_mlp": 0.2520752, + "step": 2769, + "time_per_iteration": 2.3790643215179443 + }, + { + "auxiliary_loss_clip": 0.011198, + "auxiliary_loss_mlp": 0.010518, + "balance_loss_clip": 1.03506327, + "balance_loss_mlp": 1.02779114, + "epoch": 0.08037838778944925, + "flos": 27334967834880.0, + "grad_norm": 2.382575989972092, + "language_loss": 0.93572831, + "learning_rate": 3.973440809001408e-06, + "loss": 0.95744431, + "num_input_tokens_seen": 77858655, + "router_z_loss_clip": 0.84716797, + "router_z_loss_mlp": 0.23999023, + "step": 2770, + "time_per_iteration": 2.391655206680298 + }, + { + "auxiliary_loss_clip": 0.011209, + "auxiliary_loss_mlp": 0.01056218, + "balance_loss_clip": 1.03590059, + "balance_loss_mlp": 1.03217316, + "epoch": 0.08040740525796529, + "flos": 15770965754880.0, + "grad_norm": 2.47225221570217, + "language_loss": 0.84232992, + "learning_rate": 3.973410270042411e-06, + "loss": 0.86410111, + "num_input_tokens_seen": 77872280, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.24060059, + "step": 2771, + "time_per_iteration": 2.384553909301758 + }, + { + "auxiliary_loss_clip": 0.01014078, + "auxiliary_loss_mlp": 0.01004673, + "balance_loss_clip": 1.00187922, + "balance_loss_mlp": 1.00352299, + "epoch": 0.08043642272648134, + "flos": 62911286835840.0, + "grad_norm": 0.6846573695707564, + "language_loss": 0.48888367, + "learning_rate": 3.973379713653455e-06, + "loss": 0.50907117, + "num_input_tokens_seen": 77932040, + "router_z_loss_clip": 0.12207031, + "router_z_loss_mlp": 0.01147461, + "step": 2772, + "time_per_iteration": 2.9520022869110107 + }, + { + "auxiliary_loss_clip": 0.01112175, + "auxiliary_loss_mlp": 0.01037879, + "balance_loss_clip": 1.03407311, + "balance_loss_mlp": 1.01644492, + "epoch": 0.08046544019499739, + "flos": 32741652159360.0, + "grad_norm": 3.3740951599659543, + "language_loss": 1.0030663, + "learning_rate": 3.973349139834812e-06, + "loss": 1.02456689, + "num_input_tokens_seen": 77949865, + "router_z_loss_clip": 0.78125, + "router_z_loss_mlp": 0.21435547, + "step": 2773, + "time_per_iteration": 2.4571433067321777 + }, + { + "auxiliary_loss_clip": 0.01113471, + "auxiliary_loss_mlp": 0.01047049, + "balance_loss_clip": 1.03316689, + "balance_loss_mlp": 1.02523386, + "epoch": 0.08049445766351343, + "flos": 32080222851840.0, + "grad_norm": 3.8839361137569157, + "language_loss": 0.89770234, + "learning_rate": 3.97331854858675e-06, + "loss": 0.91930759, + "num_input_tokens_seen": 77964835, + "router_z_loss_clip": 0.80322266, + "router_z_loss_mlp": 0.21826172, + "step": 2774, + "time_per_iteration": 2.4936580657958984 + }, + { + "auxiliary_loss_clip": 0.01103485, + "auxiliary_loss_mlp": 0.01037595, + "balance_loss_clip": 1.03124428, + "balance_loss_mlp": 1.01765156, + "epoch": 0.08052347513202948, + "flos": 25182746939520.0, + "grad_norm": 2.8897619858393604, + "language_loss": 0.89673442, + "learning_rate": 3.9732879399095416e-06, + "loss": 0.91814518, + "num_input_tokens_seen": 77979095, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.19952393, + "step": 2775, + "time_per_iteration": 2.4053220748901367 + }, + { + "auxiliary_loss_clip": 0.01109585, + "auxiliary_loss_mlp": 0.01049876, + "balance_loss_clip": 1.03511381, + "balance_loss_mlp": 1.02770305, + "epoch": 0.08055249260054553, + "flos": 74733194062080.0, + "grad_norm": 1.960139446990703, + "language_loss": 0.79505873, + "learning_rate": 3.973257313803454e-06, + "loss": 0.81665337, + "num_input_tokens_seen": 78011060, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.22143555, + "step": 2776, + "time_per_iteration": 2.8937125205993652 + }, + { + "auxiliary_loss_clip": 0.01119049, + "auxiliary_loss_mlp": 0.0105233, + "balance_loss_clip": 1.03901887, + "balance_loss_mlp": 1.02970362, + "epoch": 0.08058151006906157, + "flos": 18181404132480.0, + "grad_norm": 4.155116004988664, + "language_loss": 0.79292476, + "learning_rate": 3.97322667026876e-06, + "loss": 0.81463861, + "num_input_tokens_seen": 78023655, + "router_z_loss_clip": 0.80029297, + "router_z_loss_mlp": 0.22619629, + "step": 2777, + "time_per_iteration": 2.422963857650757 + }, + { + "auxiliary_loss_clip": 0.01014568, + "auxiliary_loss_mlp": 0.01002291, + "balance_loss_clip": 1.00230742, + "balance_loss_mlp": 1.0010097, + "epoch": 0.08061052753757762, + "flos": 51328851557760.0, + "grad_norm": 0.7668776152182493, + "language_loss": 0.56656313, + "learning_rate": 3.973196009305729e-06, + "loss": 0.58673179, + "num_input_tokens_seen": 78068045, + "router_z_loss_clip": 0.12255859, + "router_z_loss_mlp": 0.01281738, + "step": 2778, + "time_per_iteration": 2.732379913330078 + }, + { + "auxiliary_loss_clip": 0.01118207, + "auxiliary_loss_mlp": 0.01045597, + "balance_loss_clip": 1.03618765, + "balance_loss_mlp": 1.02150488, + "epoch": 0.08063954500609367, + "flos": 15920568397440.0, + "grad_norm": 3.3739446897047274, + "language_loss": 0.90325922, + "learning_rate": 3.9731653309146335e-06, + "loss": 0.92489731, + "num_input_tokens_seen": 78081020, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.24084473, + "step": 2779, + "time_per_iteration": 2.3376805782318115 + }, + { + "auxiliary_loss_clip": 0.01119279, + "auxiliary_loss_mlp": 0.01046381, + "balance_loss_clip": 1.03639042, + "balance_loss_mlp": 1.02231252, + "epoch": 0.08066856247460971, + "flos": 30925819013760.0, + "grad_norm": 2.100540165458122, + "language_loss": 0.74452162, + "learning_rate": 3.973134635095742e-06, + "loss": 0.76617819, + "num_input_tokens_seen": 78101145, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.24072266, + "step": 2780, + "time_per_iteration": 2.511307954788208 + }, + { + "auxiliary_loss_clip": 0.01115599, + "auxiliary_loss_mlp": 0.0104361, + "balance_loss_clip": 1.03589439, + "balance_loss_mlp": 1.02212882, + "epoch": 0.08069757994312576, + "flos": 36827363082240.0, + "grad_norm": 2.244603082621978, + "language_loss": 0.74745202, + "learning_rate": 3.973103921849328e-06, + "loss": 0.7690441, + "num_input_tokens_seen": 78127865, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.21484375, + "step": 2781, + "time_per_iteration": 2.5697555541992188 + }, + { + "auxiliary_loss_clip": 0.01108597, + "auxiliary_loss_mlp": 0.01048561, + "balance_loss_clip": 1.03456986, + "balance_loss_mlp": 1.02800894, + "epoch": 0.08072659741164182, + "flos": 18549772554240.0, + "grad_norm": 1.8809899377510266, + "language_loss": 0.68097401, + "learning_rate": 3.973073191175661e-06, + "loss": 0.70254552, + "num_input_tokens_seen": 78143890, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.20532227, + "step": 2782, + "time_per_iteration": 2.489837169647217 + }, + { + "auxiliary_loss_clip": 0.01105612, + "auxiliary_loss_mlp": 0.01039583, + "balance_loss_clip": 1.03107417, + "balance_loss_mlp": 1.01897144, + "epoch": 0.08075561488015785, + "flos": 19310774659200.0, + "grad_norm": 2.1525383221404386, + "language_loss": 0.84403396, + "learning_rate": 3.973042443075013e-06, + "loss": 0.86548591, + "num_input_tokens_seen": 78161225, + "router_z_loss_clip": 0.74609375, + "router_z_loss_mlp": 0.20605469, + "step": 2783, + "time_per_iteration": 2.3817176818847656 + }, + { + "auxiliary_loss_clip": 0.01015306, + "auxiliary_loss_mlp": 0.01001612, + "balance_loss_clip": 1.00270438, + "balance_loss_mlp": 1.00038373, + "epoch": 0.0807846323486739, + "flos": 74790343958400.0, + "grad_norm": 0.6375583536996765, + "language_loss": 0.46977854, + "learning_rate": 3.973011677547657e-06, + "loss": 0.48994774, + "num_input_tokens_seen": 78227595, + "router_z_loss_clip": 0.12597656, + "router_z_loss_mlp": 0.01226807, + "step": 2784, + "time_per_iteration": 3.2676639556884766 + }, + { + "auxiliary_loss_clip": 0.01115043, + "auxiliary_loss_mlp": 0.01042769, + "balance_loss_clip": 1.03423762, + "balance_loss_mlp": 1.01955914, + "epoch": 0.08081364981718994, + "flos": 25840894579200.0, + "grad_norm": 3.2579120482358697, + "language_loss": 0.8631084, + "learning_rate": 3.972980894593863e-06, + "loss": 0.88468653, + "num_input_tokens_seen": 78245215, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.23205566, + "step": 2785, + "time_per_iteration": 2.4110946655273438 + }, + { + "auxiliary_loss_clip": 0.01103874, + "auxiliary_loss_mlp": 0.01035752, + "balance_loss_clip": 1.03035069, + "balance_loss_mlp": 1.01434195, + "epoch": 0.080842667285706, + "flos": 16281570522240.0, + "grad_norm": 3.4874866799929247, + "language_loss": 0.87551099, + "learning_rate": 3.9729500942139024e-06, + "loss": 0.89690727, + "num_input_tokens_seen": 78257420, + "router_z_loss_clip": 0.73486328, + "router_z_loss_mlp": 0.2142334, + "step": 2786, + "time_per_iteration": 2.4298343658447266 + }, + { + "auxiliary_loss_clip": 0.01118841, + "auxiliary_loss_mlp": 0.01041811, + "balance_loss_clip": 1.03753662, + "balance_loss_mlp": 1.01870823, + "epoch": 0.08087168475422205, + "flos": 29890889314560.0, + "grad_norm": 2.3440294242750723, + "language_loss": 0.7784946, + "learning_rate": 3.9729192764080485e-06, + "loss": 0.8001011, + "num_input_tokens_seen": 78277050, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.23095703, + "step": 2787, + "time_per_iteration": 2.443696975708008 + }, + { + "auxiliary_loss_clip": 0.01014573, + "auxiliary_loss_mlp": 0.01001236, + "balance_loss_clip": 1.00177407, + "balance_loss_mlp": 1.00004447, + "epoch": 0.08090070222273808, + "flos": 58428124462080.0, + "grad_norm": 0.6962307818308093, + "language_loss": 0.53228414, + "learning_rate": 3.972888441176574e-06, + "loss": 0.55244231, + "num_input_tokens_seen": 78337715, + "router_z_loss_clip": 0.12792969, + "router_z_loss_mlp": 0.01190186, + "step": 2788, + "time_per_iteration": 3.022117853164673 + }, + { + "auxiliary_loss_clip": 0.01109795, + "auxiliary_loss_mlp": 0.01040724, + "balance_loss_clip": 1.03380203, + "balance_loss_mlp": 1.02017188, + "epoch": 0.08092971969125413, + "flos": 22921562090880.0, + "grad_norm": 2.073017157691487, + "language_loss": 0.92956674, + "learning_rate": 3.97285758851975e-06, + "loss": 0.95107198, + "num_input_tokens_seen": 78352180, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.20556641, + "step": 2789, + "time_per_iteration": 2.3951210975646973 + }, + { + "auxiliary_loss_clip": 0.01118172, + "auxiliary_loss_mlp": 0.0104588, + "balance_loss_clip": 1.03391707, + "balance_loss_mlp": 1.02116799, + "epoch": 0.08095873715977019, + "flos": 14237370973440.0, + "grad_norm": 2.9765462298271763, + "language_loss": 0.80835724, + "learning_rate": 3.972826718437849e-06, + "loss": 0.82999778, + "num_input_tokens_seen": 78365905, + "router_z_loss_clip": 0.84277344, + "router_z_loss_mlp": 0.24719238, + "step": 2790, + "time_per_iteration": 2.454566717147827 + }, + { + "auxiliary_loss_clip": 0.0111257, + "auxiliary_loss_mlp": 0.01049821, + "balance_loss_clip": 1.03345418, + "balance_loss_mlp": 1.02632463, + "epoch": 0.08098775462828622, + "flos": 17486527875840.0, + "grad_norm": 4.687292881579179, + "language_loss": 0.94028378, + "learning_rate": 3.972795830931145e-06, + "loss": 0.96190763, + "num_input_tokens_seen": 78380245, + "router_z_loss_clip": 0.79101562, + "router_z_loss_mlp": 0.23522949, + "step": 2791, + "time_per_iteration": 2.345904588699341 + }, + { + "auxiliary_loss_clip": 0.01110206, + "auxiliary_loss_mlp": 0.01041063, + "balance_loss_clip": 1.03336406, + "balance_loss_mlp": 1.02150071, + "epoch": 0.08101677209680228, + "flos": 24674167031040.0, + "grad_norm": 2.360121902631626, + "language_loss": 0.87758058, + "learning_rate": 3.972764925999909e-06, + "loss": 0.89909327, + "num_input_tokens_seen": 78395665, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.19567871, + "step": 2792, + "time_per_iteration": 2.4407098293304443 + }, + { + "auxiliary_loss_clip": 0.01014544, + "auxiliary_loss_mlp": 0.01006781, + "balance_loss_clip": 1.00180888, + "balance_loss_mlp": 1.00544024, + "epoch": 0.08104578956531833, + "flos": 65203579572480.0, + "grad_norm": 0.6451025043080679, + "language_loss": 0.4540756, + "learning_rate": 3.972734003644415e-06, + "loss": 0.47428888, + "num_input_tokens_seen": 78455080, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01342773, + "step": 2793, + "time_per_iteration": 2.9403045177459717 + }, + { + "auxiliary_loss_clip": 0.01114788, + "auxiliary_loss_mlp": 0.01036944, + "balance_loss_clip": 1.03268564, + "balance_loss_mlp": 1.01303041, + "epoch": 0.08107480703383436, + "flos": 12814695181440.0, + "grad_norm": 3.3802119858454582, + "language_loss": 0.84451872, + "learning_rate": 3.9727030638649366e-06, + "loss": 0.86603606, + "num_input_tokens_seen": 78466485, + "router_z_loss_clip": 0.82080078, + "router_z_loss_mlp": 0.23925781, + "step": 2794, + "time_per_iteration": 2.372408866882324 + }, + { + "auxiliary_loss_clip": 0.01108235, + "auxiliary_loss_mlp": 0.01042342, + "balance_loss_clip": 1.035308, + "balance_loss_mlp": 1.02343547, + "epoch": 0.08110382450235042, + "flos": 31825182303360.0, + "grad_norm": 1.9325677220953799, + "language_loss": 0.61647171, + "learning_rate": 3.9726721066617465e-06, + "loss": 0.63797748, + "num_input_tokens_seen": 78483535, + "router_z_loss_clip": 0.72851562, + "router_z_loss_mlp": 0.18908691, + "step": 2795, + "time_per_iteration": 2.499075174331665 + }, + { + "auxiliary_loss_clip": 0.01109345, + "auxiliary_loss_mlp": 0.0104014, + "balance_loss_clip": 1.03266132, + "balance_loss_mlp": 1.0165484, + "epoch": 0.08113284197086647, + "flos": 33175867138560.0, + "grad_norm": 2.1714502991618407, + "language_loss": 0.82024342, + "learning_rate": 3.972641132035118e-06, + "loss": 0.84173822, + "num_input_tokens_seen": 78501735, + "router_z_loss_clip": 0.76660156, + "router_z_loss_mlp": 0.23596191, + "step": 2796, + "time_per_iteration": 2.5647313594818115 + }, + { + "auxiliary_loss_clip": 0.01013454, + "auxiliary_loss_mlp": 0.01001749, + "balance_loss_clip": 1.00124609, + "balance_loss_mlp": 1.00064063, + "epoch": 0.0811618594393825, + "flos": 68197486458240.0, + "grad_norm": 0.6542550572710054, + "language_loss": 0.48785192, + "learning_rate": 3.972610139985324e-06, + "loss": 0.50800395, + "num_input_tokens_seen": 78561620, + "router_z_loss_clip": 0.12207031, + "router_z_loss_mlp": 0.0111084, + "step": 2797, + "time_per_iteration": 2.9903743267059326 + }, + { + "auxiliary_loss_clip": 0.01106769, + "auxiliary_loss_mlp": 0.01040242, + "balance_loss_clip": 1.03219891, + "balance_loss_mlp": 1.01911807, + "epoch": 0.08119087690789856, + "flos": 20075128254720.0, + "grad_norm": 18.749008873623318, + "language_loss": 0.90179539, + "learning_rate": 3.97257913051264e-06, + "loss": 0.92326546, + "num_input_tokens_seen": 78578090, + "router_z_loss_clip": 0.74560547, + "router_z_loss_mlp": 0.21118164, + "step": 2798, + "time_per_iteration": 2.4563710689544678 + }, + { + "auxiliary_loss_clip": 0.0110684, + "auxiliary_loss_mlp": 0.01038266, + "balance_loss_clip": 1.03100431, + "balance_loss_mlp": 1.01687944, + "epoch": 0.0812198943764146, + "flos": 39230993744640.0, + "grad_norm": 2.505869888801791, + "language_loss": 0.71678799, + "learning_rate": 3.972548103617338e-06, + "loss": 0.73823905, + "num_input_tokens_seen": 78597045, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.21417236, + "step": 2799, + "time_per_iteration": 2.533085346221924 + }, + { + "auxiliary_loss_clip": 0.01116691, + "auxiliary_loss_mlp": 0.01045952, + "balance_loss_clip": 1.03664804, + "balance_loss_mlp": 1.02534044, + "epoch": 0.08124891184493065, + "flos": 33687030487680.0, + "grad_norm": 2.168963982439952, + "language_loss": 0.95358688, + "learning_rate": 3.972517059299694e-06, + "loss": 0.97521335, + "num_input_tokens_seen": 78615905, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.20593262, + "step": 2800, + "time_per_iteration": 2.506619930267334 + }, + { + "auxiliary_loss_clip": 0.01101249, + "auxiliary_loss_mlp": 0.01037316, + "balance_loss_clip": 1.03211665, + "balance_loss_mlp": 1.02004218, + "epoch": 0.0812779293134467, + "flos": 16247425345920.0, + "grad_norm": 1.9985657476126286, + "language_loss": 0.76281959, + "learning_rate": 3.972485997559981e-06, + "loss": 0.7842052, + "num_input_tokens_seen": 78631130, + "router_z_loss_clip": 0.69091797, + "router_z_loss_mlp": 0.17272949, + "step": 2801, + "time_per_iteration": 2.3465709686279297 + }, + { + "auxiliary_loss_clip": 0.01111263, + "auxiliary_loss_mlp": 0.01041542, + "balance_loss_clip": 1.03367734, + "balance_loss_mlp": 1.01842737, + "epoch": 0.08130694678196274, + "flos": 31935472888320.0, + "grad_norm": 2.194763139499809, + "language_loss": 0.97185349, + "learning_rate": 3.972454918398473e-06, + "loss": 0.99338162, + "num_input_tokens_seen": 78648255, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.23095703, + "step": 2802, + "time_per_iteration": 2.5287318229675293 + }, + { + "auxiliary_loss_clip": 0.01109215, + "auxiliary_loss_mlp": 0.0104873, + "balance_loss_clip": 1.03240633, + "balance_loss_mlp": 1.02768922, + "epoch": 0.08133596425047879, + "flos": 18361695726720.0, + "grad_norm": 2.2425388065159786, + "language_loss": 0.74350941, + "learning_rate": 3.972423821815445e-06, + "loss": 0.7650888, + "num_input_tokens_seen": 78661715, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.21044922, + "step": 2803, + "time_per_iteration": 2.33306884765625 + }, + { + "auxiliary_loss_clip": 0.01103714, + "auxiliary_loss_mlp": 0.01038425, + "balance_loss_clip": 1.03041255, + "balance_loss_mlp": 1.01866031, + "epoch": 0.08136498171899484, + "flos": 31394248992000.0, + "grad_norm": 1.9919347933223404, + "language_loss": 0.70702529, + "learning_rate": 3.9723927078111715e-06, + "loss": 0.72844672, + "num_input_tokens_seen": 78680170, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.19775391, + "step": 2804, + "time_per_iteration": 2.5419561862945557 + }, + { + "auxiliary_loss_clip": 0.01129915, + "auxiliary_loss_mlp": 0.0104895, + "balance_loss_clip": 1.03840947, + "balance_loss_mlp": 1.02301025, + "epoch": 0.08139399918751088, + "flos": 12086162507520.0, + "grad_norm": 2.863142714672402, + "language_loss": 0.90790451, + "learning_rate": 3.9723615763859275e-06, + "loss": 0.92969316, + "num_input_tokens_seen": 78692995, + "router_z_loss_clip": 0.91552734, + "router_z_loss_mlp": 0.25939941, + "step": 2805, + "time_per_iteration": 2.333620309829712 + }, + { + "auxiliary_loss_clip": 0.01013751, + "auxiliary_loss_mlp": 0.01001395, + "balance_loss_clip": 1.00133395, + "balance_loss_mlp": 1.00015533, + "epoch": 0.08142301665602693, + "flos": 54314447673600.0, + "grad_norm": 0.6727194997024125, + "language_loss": 0.45510697, + "learning_rate": 3.972330427539988e-06, + "loss": 0.47525841, + "num_input_tokens_seen": 78753245, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.01239014, + "step": 2806, + "time_per_iteration": 3.018225908279419 + }, + { + "auxiliary_loss_clip": 0.01119606, + "auxiliary_loss_mlp": 0.01045934, + "balance_loss_clip": 1.03543448, + "balance_loss_mlp": 1.02227068, + "epoch": 0.08145203412454298, + "flos": 28871704638720.0, + "grad_norm": 2.000841252723629, + "language_loss": 1.02158463, + "learning_rate": 3.972299261273628e-06, + "loss": 1.04324007, + "num_input_tokens_seen": 78770310, + "router_z_loss_clip": 0.84082031, + "router_z_loss_mlp": 0.2364502, + "step": 2807, + "time_per_iteration": 2.4682860374450684 + }, + { + "auxiliary_loss_clip": 0.01103637, + "auxiliary_loss_mlp": 0.01049311, + "balance_loss_clip": 1.03059173, + "balance_loss_mlp": 1.02737653, + "epoch": 0.08148105159305902, + "flos": 20661878430720.0, + "grad_norm": 2.331568643702562, + "language_loss": 0.79327393, + "learning_rate": 3.972268077587123e-06, + "loss": 0.81480336, + "num_input_tokens_seen": 78785440, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.21923828, + "step": 2808, + "time_per_iteration": 2.4111108779907227 + }, + { + "auxiliary_loss_clip": 0.01108519, + "auxiliary_loss_mlp": 0.01044488, + "balance_loss_clip": 1.03331614, + "balance_loss_mlp": 1.02443099, + "epoch": 0.08151006906157507, + "flos": 12342040928640.0, + "grad_norm": 3.639476354936706, + "language_loss": 0.78213251, + "learning_rate": 3.972236876480748e-06, + "loss": 0.80366254, + "num_input_tokens_seen": 78798000, + "router_z_loss_clip": 0.75292969, + "router_z_loss_mlp": 0.20050049, + "step": 2809, + "time_per_iteration": 2.3257339000701904 + }, + { + "auxiliary_loss_clip": 0.01013353, + "auxiliary_loss_mlp": 0.0100432, + "balance_loss_clip": 1.00092232, + "balance_loss_mlp": 1.00294304, + "epoch": 0.08153908653009112, + "flos": 71999772030720.0, + "grad_norm": 0.6815111299523796, + "language_loss": 0.52562737, + "learning_rate": 3.972205657954779e-06, + "loss": 0.54580414, + "num_input_tokens_seen": 78861820, + "router_z_loss_clip": 0.12451172, + "router_z_loss_mlp": 0.01379395, + "step": 2810, + "time_per_iteration": 3.1344571113586426 + }, + { + "auxiliary_loss_clip": 0.01113566, + "auxiliary_loss_mlp": 0.01041567, + "balance_loss_clip": 1.03321028, + "balance_loss_mlp": 1.01718879, + "epoch": 0.08156810399860716, + "flos": 24498029888640.0, + "grad_norm": 2.366733172796482, + "language_loss": 0.84490699, + "learning_rate": 3.972174422009492e-06, + "loss": 0.8664583, + "num_input_tokens_seen": 78876665, + "router_z_loss_clip": 0.80419922, + "router_z_loss_mlp": 0.24401855, + "step": 2811, + "time_per_iteration": 2.4597268104553223 + }, + { + "auxiliary_loss_clip": 0.0111515, + "auxiliary_loss_mlp": 0.01051164, + "balance_loss_clip": 1.03624249, + "balance_loss_mlp": 1.02840698, + "epoch": 0.08159712146712321, + "flos": 47146956105600.0, + "grad_norm": 2.5260083709079053, + "language_loss": 0.80903959, + "learning_rate": 3.972143168645162e-06, + "loss": 0.83070278, + "num_input_tokens_seen": 78896085, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.22729492, + "step": 2812, + "time_per_iteration": 2.523031711578369 + }, + { + "auxiliary_loss_clip": 0.01115951, + "auxiliary_loss_mlp": 0.01044431, + "balance_loss_clip": 1.03388202, + "balance_loss_mlp": 1.01900363, + "epoch": 0.08162613893563926, + "flos": 21755846972160.0, + "grad_norm": 5.092742849118235, + "language_loss": 0.97586298, + "learning_rate": 3.972111897862065e-06, + "loss": 0.9974668, + "num_input_tokens_seen": 78912310, + "router_z_loss_clip": 0.8203125, + "router_z_loss_mlp": 0.25415039, + "step": 2813, + "time_per_iteration": 2.4438488483428955 + }, + { + "auxiliary_loss_clip": 0.01113759, + "auxiliary_loss_mlp": 0.01044004, + "balance_loss_clip": 1.03541541, + "balance_loss_mlp": 1.02144957, + "epoch": 0.0816551564041553, + "flos": 23505624092160.0, + "grad_norm": 2.93165729590932, + "language_loss": 0.82764667, + "learning_rate": 3.972080609660478e-06, + "loss": 0.84922433, + "num_input_tokens_seen": 78926950, + "router_z_loss_clip": 0.78320312, + "router_z_loss_mlp": 0.22558594, + "step": 2814, + "time_per_iteration": 2.38787841796875 + }, + { + "auxiliary_loss_clip": 0.01107621, + "auxiliary_loss_mlp": 0.01037036, + "balance_loss_clip": 1.03291965, + "balance_loss_mlp": 1.01667464, + "epoch": 0.08168417387267135, + "flos": 21353926867200.0, + "grad_norm": 3.812380251305587, + "language_loss": 0.77541316, + "learning_rate": 3.972049304040678e-06, + "loss": 0.79685968, + "num_input_tokens_seen": 78941075, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.20373535, + "step": 2815, + "time_per_iteration": 2.400787830352783 + }, + { + "auxiliary_loss_clip": 0.01119252, + "auxiliary_loss_mlp": 0.01047005, + "balance_loss_clip": 1.03516507, + "balance_loss_mlp": 1.02371168, + "epoch": 0.08171319134118739, + "flos": 12778839348480.0, + "grad_norm": 2.38587709275865, + "language_loss": 0.87363112, + "learning_rate": 3.972017981002939e-06, + "loss": 0.89529371, + "num_input_tokens_seen": 78952065, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.23266602, + "step": 2816, + "time_per_iteration": 2.342170000076294 + }, + { + "auxiliary_loss_clip": 0.01108184, + "auxiliary_loss_mlp": 0.01049417, + "balance_loss_clip": 1.03365648, + "balance_loss_mlp": 1.02854931, + "epoch": 0.08174220880970344, + "flos": 25768694154240.0, + "grad_norm": 2.3165229988684213, + "language_loss": 0.81959379, + "learning_rate": 3.971986640547541e-06, + "loss": 0.84116977, + "num_input_tokens_seen": 78967760, + "router_z_loss_clip": 0.74511719, + "router_z_loss_mlp": 0.20874023, + "step": 2817, + "time_per_iteration": 2.427696943283081 + }, + { + "auxiliary_loss_clip": 0.01113535, + "auxiliary_loss_mlp": 0.01041907, + "balance_loss_clip": 1.03367782, + "balance_loss_mlp": 1.01959097, + "epoch": 0.08177122627821949, + "flos": 16975713640320.0, + "grad_norm": 2.704788911402479, + "language_loss": 0.85603982, + "learning_rate": 3.971955282674758e-06, + "loss": 0.87759429, + "num_input_tokens_seen": 78984445, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.2232666, + "step": 2818, + "time_per_iteration": 6.563819408416748 + }, + { + "auxiliary_loss_clip": 0.01116984, + "auxiliary_loss_mlp": 0.0104576, + "balance_loss_clip": 1.03300405, + "balance_loss_mlp": 1.02226365, + "epoch": 0.08180024374673553, + "flos": 32553505509120.0, + "grad_norm": 2.4491220640894626, + "language_loss": 0.88391447, + "learning_rate": 3.971923907384868e-06, + "loss": 0.9055419, + "num_input_tokens_seen": 78999140, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.23510742, + "step": 2819, + "time_per_iteration": 2.4633822441101074 + }, + { + "auxiliary_loss_clip": 0.01015281, + "auxiliary_loss_mlp": 0.01001539, + "balance_loss_clip": 1.00261509, + "balance_loss_mlp": 1.00035274, + "epoch": 0.08182926121525158, + "flos": 64370377042560.0, + "grad_norm": 0.7279757708069117, + "language_loss": 0.52889514, + "learning_rate": 3.971892514678147e-06, + "loss": 0.54906332, + "num_input_tokens_seen": 79055315, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01184082, + "step": 2820, + "time_per_iteration": 2.9314746856689453 + }, + { + "auxiliary_loss_clip": 0.01015359, + "auxiliary_loss_mlp": 0.0100142, + "balance_loss_clip": 1.00246954, + "balance_loss_mlp": 1.00018001, + "epoch": 0.08185827868376763, + "flos": 61715511169920.0, + "grad_norm": 0.7006653529318077, + "language_loss": 0.52330232, + "learning_rate": 3.971861104554876e-06, + "loss": 0.54347008, + "num_input_tokens_seen": 79121095, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01239014, + "step": 2821, + "time_per_iteration": 5.598997592926025 + }, + { + "auxiliary_loss_clip": 0.01113284, + "auxiliary_loss_mlp": 0.0104222, + "balance_loss_clip": 1.03495991, + "balance_loss_mlp": 1.02073884, + "epoch": 0.08188729615228367, + "flos": 65717293317120.0, + "grad_norm": 3.307394256569776, + "language_loss": 0.76100922, + "learning_rate": 3.971829677015328e-06, + "loss": 0.78256422, + "num_input_tokens_seen": 79140545, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.21484375, + "step": 2822, + "time_per_iteration": 2.7140769958496094 + }, + { + "auxiliary_loss_clip": 0.01115024, + "auxiliary_loss_mlp": 0.01048167, + "balance_loss_clip": 1.03513014, + "balance_loss_mlp": 1.02413464, + "epoch": 0.08191631362079972, + "flos": 11976779617920.0, + "grad_norm": 3.1667031173173505, + "language_loss": 0.9588362, + "learning_rate": 3.971798232059782e-06, + "loss": 0.98046809, + "num_input_tokens_seen": 79151650, + "router_z_loss_clip": 0.79931641, + "router_z_loss_mlp": 0.24023438, + "step": 2823, + "time_per_iteration": 4.89693808555603 + }, + { + "auxiliary_loss_clip": 0.01109415, + "auxiliary_loss_mlp": 0.01041728, + "balance_loss_clip": 1.03233004, + "balance_loss_mlp": 1.02086616, + "epoch": 0.08194533108931577, + "flos": 15625517564160.0, + "grad_norm": 2.297924898718754, + "language_loss": 0.73952675, + "learning_rate": 3.9717667696885165e-06, + "loss": 0.76103818, + "num_input_tokens_seen": 79166095, + "router_z_loss_clip": 0.77099609, + "router_z_loss_mlp": 0.20849609, + "step": 2824, + "time_per_iteration": 2.3427562713623047 + }, + { + "auxiliary_loss_clip": 0.01118429, + "auxiliary_loss_mlp": 0.01046558, + "balance_loss_clip": 1.03560328, + "balance_loss_mlp": 1.02236998, + "epoch": 0.08197434855783181, + "flos": 34124771514240.0, + "grad_norm": 2.1716692566776863, + "language_loss": 0.98600471, + "learning_rate": 3.97173528990181e-06, + "loss": 1.00765467, + "num_input_tokens_seen": 79186530, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.24206543, + "step": 2825, + "time_per_iteration": 2.4778099060058594 + }, + { + "auxiliary_loss_clip": 0.01019563, + "auxiliary_loss_mlp": 0.01001458, + "balance_loss_clip": 1.00678921, + "balance_loss_mlp": 1.00021207, + "epoch": 0.08200336602634786, + "flos": 63827896337280.0, + "grad_norm": 0.8145387166134179, + "language_loss": 0.54958689, + "learning_rate": 3.971703792699938e-06, + "loss": 0.56979704, + "num_input_tokens_seen": 79244290, + "router_z_loss_clip": 0.12792969, + "router_z_loss_mlp": 0.01245117, + "step": 2826, + "time_per_iteration": 2.9667797088623047 + }, + { + "auxiliary_loss_clip": 0.01109934, + "auxiliary_loss_mlp": 0.01043634, + "balance_loss_clip": 1.03344405, + "balance_loss_mlp": 1.01988792, + "epoch": 0.08203238349486391, + "flos": 13217522981760.0, + "grad_norm": 3.087101356243065, + "language_loss": 0.96559983, + "learning_rate": 3.971672278083181e-06, + "loss": 0.98713553, + "num_input_tokens_seen": 79256600, + "router_z_loss_clip": 0.76367188, + "router_z_loss_mlp": 0.23754883, + "step": 2827, + "time_per_iteration": 2.3593242168426514 + }, + { + "auxiliary_loss_clip": 0.01109506, + "auxiliary_loss_mlp": 0.01045995, + "balance_loss_clip": 1.03493977, + "balance_loss_mlp": 1.02450728, + "epoch": 0.08206140096337995, + "flos": 74729737837440.0, + "grad_norm": 2.5556098284687594, + "language_loss": 0.70676738, + "learning_rate": 3.971640746051817e-06, + "loss": 0.72832233, + "num_input_tokens_seen": 79277260, + "router_z_loss_clip": 0.74511719, + "router_z_loss_mlp": 0.21490479, + "step": 2828, + "time_per_iteration": 2.7703676223754883 + }, + { + "auxiliary_loss_clip": 0.0111916, + "auxiliary_loss_mlp": 0.01053275, + "balance_loss_clip": 1.03745294, + "balance_loss_mlp": 1.02956462, + "epoch": 0.082090418431896, + "flos": 19345862442240.0, + "grad_norm": 3.437956067965596, + "language_loss": 0.96708095, + "learning_rate": 3.971609196606123e-06, + "loss": 0.98880529, + "num_input_tokens_seen": 79289820, + "router_z_loss_clip": 0.81738281, + "router_z_loss_mlp": 0.23718262, + "step": 2829, + "time_per_iteration": 2.4881749153137207 + }, + { + "auxiliary_loss_clip": 0.01015986, + "auxiliary_loss_mlp": 0.01001214, + "balance_loss_clip": 1.00331116, + "balance_loss_mlp": 0.99996245, + "epoch": 0.08211943590041205, + "flos": 69298856208000.0, + "grad_norm": 0.7953340373508999, + "language_loss": 0.50072682, + "learning_rate": 3.97157762974638e-06, + "loss": 0.52089882, + "num_input_tokens_seen": 79348430, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01251221, + "step": 2830, + "time_per_iteration": 2.9679698944091797 + }, + { + "auxiliary_loss_clip": 0.01115481, + "auxiliary_loss_mlp": 0.010387, + "balance_loss_clip": 1.03205848, + "balance_loss_mlp": 1.01551342, + "epoch": 0.08214845336892809, + "flos": 24892478962560.0, + "grad_norm": 2.7648554993628522, + "language_loss": 0.78363371, + "learning_rate": 3.9715460454728655e-06, + "loss": 0.80517542, + "num_input_tokens_seen": 79361765, + "router_z_loss_clip": 0.83251953, + "router_z_loss_mlp": 0.23181152, + "step": 2831, + "time_per_iteration": 2.4596493244171143 + }, + { + "auxiliary_loss_clip": 0.01108991, + "auxiliary_loss_mlp": 0.01039323, + "balance_loss_clip": 1.03422534, + "balance_loss_mlp": 1.0177815, + "epoch": 0.08217747083744414, + "flos": 35035725375360.0, + "grad_norm": 2.113289416445381, + "language_loss": 0.88918757, + "learning_rate": 3.971514443785858e-06, + "loss": 0.91067064, + "num_input_tokens_seen": 79380390, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.21557617, + "step": 2832, + "time_per_iteration": 2.4740288257598877 + }, + { + "auxiliary_loss_clip": 0.01109335, + "auxiliary_loss_mlp": 0.01047714, + "balance_loss_clip": 1.03333187, + "balance_loss_mlp": 1.02646506, + "epoch": 0.08220648830596018, + "flos": 10805967440640.0, + "grad_norm": 2.605736018606615, + "language_loss": 0.79056299, + "learning_rate": 3.971482824685637e-06, + "loss": 0.81213343, + "num_input_tokens_seen": 79390070, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.21258545, + "step": 2833, + "time_per_iteration": 2.4650120735168457 + }, + { + "auxiliary_loss_clip": 0.01113088, + "auxiliary_loss_mlp": 0.01044222, + "balance_loss_clip": 1.03238285, + "balance_loss_mlp": 1.02135777, + "epoch": 0.08223550577447623, + "flos": 32415179235840.0, + "grad_norm": 2.1889191431290476, + "language_loss": 0.85319817, + "learning_rate": 3.971451188172482e-06, + "loss": 0.87477136, + "num_input_tokens_seen": 79408015, + "router_z_loss_clip": 0.80664062, + "router_z_loss_mlp": 0.22875977, + "step": 2834, + "time_per_iteration": 2.3943138122558594 + }, + { + "auxiliary_loss_clip": 0.01122464, + "auxiliary_loss_mlp": 0.01050059, + "balance_loss_clip": 1.03855646, + "balance_loss_mlp": 1.02229512, + "epoch": 0.08226452324299229, + "flos": 20074918786560.0, + "grad_norm": 3.7090395770492717, + "language_loss": 1.06374621, + "learning_rate": 3.971419534246673e-06, + "loss": 1.08547151, + "num_input_tokens_seen": 79419590, + "router_z_loss_clip": 0.83886719, + "router_z_loss_mlp": 0.27770996, + "step": 2835, + "time_per_iteration": 2.4401817321777344 + }, + { + "auxiliary_loss_clip": 0.01106257, + "auxiliary_loss_mlp": 0.01040811, + "balance_loss_clip": 1.02994442, + "balance_loss_mlp": 1.02116585, + "epoch": 0.08229354071150832, + "flos": 26205457662720.0, + "grad_norm": 2.6007615076427046, + "language_loss": 0.9063642, + "learning_rate": 3.971387862908488e-06, + "loss": 0.92783487, + "num_input_tokens_seen": 79433355, + "router_z_loss_clip": 0.76391602, + "router_z_loss_mlp": 0.19641113, + "step": 2836, + "time_per_iteration": 2.416846513748169 + }, + { + "auxiliary_loss_clip": 0.01105921, + "auxiliary_loss_mlp": 0.01043331, + "balance_loss_clip": 1.03246307, + "balance_loss_mlp": 1.0211525, + "epoch": 0.08232255818002437, + "flos": 31643494254720.0, + "grad_norm": 2.3629252668751213, + "language_loss": 0.9881795, + "learning_rate": 3.971356174158207e-06, + "loss": 1.00967193, + "num_input_tokens_seen": 79449785, + "router_z_loss_clip": 0.73388672, + "router_z_loss_mlp": 0.22186279, + "step": 2837, + "time_per_iteration": 2.52455997467041 + }, + { + "auxiliary_loss_clip": 0.0101677, + "auxiliary_loss_mlp": 0.01002645, + "balance_loss_clip": 1.0037744, + "balance_loss_mlp": 1.00143504, + "epoch": 0.08235157564854043, + "flos": 60789929448960.0, + "grad_norm": 0.7850145147445855, + "language_loss": 0.54529172, + "learning_rate": 3.971324467996112e-06, + "loss": 0.56548589, + "num_input_tokens_seen": 79501635, + "router_z_loss_clip": 0.12988281, + "router_z_loss_mlp": 0.01208496, + "step": 2838, + "time_per_iteration": 2.819887638092041 + }, + { + "auxiliary_loss_clip": 0.01015861, + "auxiliary_loss_mlp": 0.01003723, + "balance_loss_clip": 1.00283527, + "balance_loss_mlp": 1.00259078, + "epoch": 0.08238059311705646, + "flos": 65358104716800.0, + "grad_norm": 0.712672971002272, + "language_loss": 0.51423508, + "learning_rate": 3.971292744422481e-06, + "loss": 0.53443092, + "num_input_tokens_seen": 79560665, + "router_z_loss_clip": 0.12988281, + "router_z_loss_mlp": 0.01135254, + "step": 2839, + "time_per_iteration": 3.051694393157959 + }, + { + "auxiliary_loss_clip": 0.01108528, + "auxiliary_loss_mlp": 0.01039228, + "balance_loss_clip": 1.03319955, + "balance_loss_mlp": 1.01700771, + "epoch": 0.08240961058557252, + "flos": 21792470855040.0, + "grad_norm": 2.1010054025020386, + "language_loss": 0.8206166, + "learning_rate": 3.971261003437595e-06, + "loss": 0.84209424, + "num_input_tokens_seen": 79575580, + "router_z_loss_clip": 0.75341797, + "router_z_loss_mlp": 0.22229004, + "step": 2840, + "time_per_iteration": 2.3269801139831543 + }, + { + "auxiliary_loss_clip": 0.01015421, + "auxiliary_loss_mlp": 0.01001967, + "balance_loss_clip": 1.0024035, + "balance_loss_mlp": 1.00069177, + "epoch": 0.08243862805408857, + "flos": 62262111415680.0, + "grad_norm": 0.6485090830250111, + "language_loss": 0.50004387, + "learning_rate": 3.9712292450417345e-06, + "loss": 0.52021766, + "num_input_tokens_seen": 79639355, + "router_z_loss_clip": 0.13085938, + "router_z_loss_mlp": 0.01275635, + "step": 2841, + "time_per_iteration": 3.0446794033050537 + }, + { + "auxiliary_loss_clip": 0.01109636, + "auxiliary_loss_mlp": 0.01044432, + "balance_loss_clip": 1.03352869, + "balance_loss_mlp": 1.02293849, + "epoch": 0.0824676455226046, + "flos": 37188225561600.0, + "grad_norm": 2.141454758305472, + "language_loss": 0.58936584, + "learning_rate": 3.971197469235179e-06, + "loss": 0.61090654, + "num_input_tokens_seen": 79656910, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.21484375, + "step": 2842, + "time_per_iteration": 2.5459182262420654 + }, + { + "auxiliary_loss_clip": 0.01113673, + "auxiliary_loss_mlp": 0.01046455, + "balance_loss_clip": 1.03425145, + "balance_loss_mlp": 1.02424634, + "epoch": 0.08249666299112066, + "flos": 20876454846720.0, + "grad_norm": 2.230630644011925, + "language_loss": 0.86190856, + "learning_rate": 3.97116567601821e-06, + "loss": 0.88350987, + "num_input_tokens_seen": 79670550, + "router_z_loss_clip": 0.79394531, + "router_z_loss_mlp": 0.2220459, + "step": 2843, + "time_per_iteration": 2.362783670425415 + }, + { + "auxiliary_loss_clip": 0.01105905, + "auxiliary_loss_mlp": 0.01038708, + "balance_loss_clip": 1.03245807, + "balance_loss_mlp": 1.01823938, + "epoch": 0.08252568045963671, + "flos": 11428468715520.0, + "grad_norm": 2.3333185092444864, + "language_loss": 0.75449461, + "learning_rate": 3.971133865391108e-06, + "loss": 0.77594072, + "num_input_tokens_seen": 79683340, + "router_z_loss_clip": 0.73388672, + "router_z_loss_mlp": 0.20471191, + "step": 2844, + "time_per_iteration": 2.330998420715332 + }, + { + "auxiliary_loss_clip": 0.01109295, + "auxiliary_loss_mlp": 0.01040645, + "balance_loss_clip": 1.03085399, + "balance_loss_mlp": 1.01772106, + "epoch": 0.08255469792815275, + "flos": 23395158950400.0, + "grad_norm": 2.9672586686223505, + "language_loss": 0.91595751, + "learning_rate": 3.971102037354154e-06, + "loss": 0.93745685, + "num_input_tokens_seen": 79699075, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.22949219, + "step": 2845, + "time_per_iteration": 2.4109604358673096 + }, + { + "auxiliary_loss_clip": 0.01015866, + "auxiliary_loss_mlp": 0.01002026, + "balance_loss_clip": 1.00310397, + "balance_loss_mlp": 1.00070238, + "epoch": 0.0825837153966688, + "flos": 68677611742080.0, + "grad_norm": 0.7084361258157877, + "language_loss": 0.49931431, + "learning_rate": 3.97107019190763e-06, + "loss": 0.51949322, + "num_input_tokens_seen": 79764135, + "router_z_loss_clip": 0.12792969, + "router_z_loss_mlp": 0.01324463, + "step": 2846, + "time_per_iteration": 3.1206564903259277 + }, + { + "auxiliary_loss_clip": 0.01112281, + "auxiliary_loss_mlp": 0.01038334, + "balance_loss_clip": 1.0350461, + "balance_loss_mlp": 1.01630378, + "epoch": 0.08261273286518483, + "flos": 16501278908160.0, + "grad_norm": 3.0680720781911797, + "language_loss": 0.77038133, + "learning_rate": 3.971038329051816e-06, + "loss": 0.79188752, + "num_input_tokens_seen": 79776670, + "router_z_loss_clip": 0.77246094, + "router_z_loss_mlp": 0.22021484, + "step": 2847, + "time_per_iteration": 2.3784539699554443 + }, + { + "auxiliary_loss_clip": 0.01102852, + "auxiliary_loss_mlp": 0.01040511, + "balance_loss_clip": 1.03137708, + "balance_loss_mlp": 1.01937497, + "epoch": 0.08264175033370089, + "flos": 45105130529280.0, + "grad_norm": 2.2757462477016706, + "language_loss": 0.89623427, + "learning_rate": 3.971006448786993e-06, + "loss": 0.91766798, + "num_input_tokens_seen": 79796380, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.21130371, + "step": 2848, + "time_per_iteration": 2.6014771461486816 + }, + { + "auxiliary_loss_clip": 0.0111109, + "auxiliary_loss_mlp": 0.01050936, + "balance_loss_clip": 1.03383112, + "balance_loss_mlp": 1.02863181, + "epoch": 0.08267076780221694, + "flos": 41315657425920.0, + "grad_norm": 1.898485468891536, + "language_loss": 0.93329787, + "learning_rate": 3.970974551113444e-06, + "loss": 0.95491809, + "num_input_tokens_seen": 79818490, + "router_z_loss_clip": 0.77246094, + "router_z_loss_mlp": 0.22314453, + "step": 2849, + "time_per_iteration": 2.616746664047241 + }, + { + "auxiliary_loss_clip": 0.01116638, + "auxiliary_loss_mlp": 0.01044614, + "balance_loss_clip": 1.03688407, + "balance_loss_mlp": 1.02314413, + "epoch": 0.08269978527073298, + "flos": 10515420172800.0, + "grad_norm": 2.581780500574662, + "language_loss": 1.01030898, + "learning_rate": 3.970942636031451e-06, + "loss": 1.03192151, + "num_input_tokens_seen": 79829675, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.21472168, + "step": 2850, + "time_per_iteration": 2.4988608360290527 + }, + { + "auxiliary_loss_clip": 0.01112326, + "auxiliary_loss_mlp": 0.01049876, + "balance_loss_clip": 1.03486753, + "balance_loss_mlp": 1.02676177, + "epoch": 0.08272880273924903, + "flos": 29855173127040.0, + "grad_norm": 2.1628023858775776, + "language_loss": 0.81580162, + "learning_rate": 3.970910703541295e-06, + "loss": 0.83742368, + "num_input_tokens_seen": 79846265, + "router_z_loss_clip": 0.77490234, + "router_z_loss_mlp": 0.2310791, + "step": 2851, + "time_per_iteration": 2.5333685874938965 + }, + { + "auxiliary_loss_clip": 0.0110624, + "auxiliary_loss_mlp": 0.01045282, + "balance_loss_clip": 1.03228164, + "balance_loss_mlp": 1.02493286, + "epoch": 0.08275782020776508, + "flos": 41461594375680.0, + "grad_norm": 4.70342185380804, + "language_loss": 0.89107311, + "learning_rate": 3.970878753643257e-06, + "loss": 0.9125883, + "num_input_tokens_seen": 79872505, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.20349121, + "step": 2852, + "time_per_iteration": 2.558671712875366 + }, + { + "auxiliary_loss_clip": 0.0111902, + "auxiliary_loss_mlp": 0.01045068, + "balance_loss_clip": 1.03540635, + "balance_loss_mlp": 1.02234685, + "epoch": 0.08278683767628112, + "flos": 10881903381120.0, + "grad_norm": 2.662579804026585, + "language_loss": 0.74462295, + "learning_rate": 3.970846786337621e-06, + "loss": 0.76626378, + "num_input_tokens_seen": 79884075, + "router_z_loss_clip": 0.8359375, + "router_z_loss_mlp": 0.22705078, + "step": 2853, + "time_per_iteration": 2.3824541568756104 + }, + { + "auxiliary_loss_clip": 0.01115592, + "auxiliary_loss_mlp": 0.01042998, + "balance_loss_clip": 1.03653622, + "balance_loss_mlp": 1.02207661, + "epoch": 0.08281585514479717, + "flos": 16392070575360.0, + "grad_norm": 2.906208761993784, + "language_loss": 0.90444219, + "learning_rate": 3.970814801624668e-06, + "loss": 0.92602807, + "num_input_tokens_seen": 79896940, + "router_z_loss_clip": 0.79003906, + "router_z_loss_mlp": 0.20922852, + "step": 2854, + "time_per_iteration": 2.381115674972534 + }, + { + "auxiliary_loss_clip": 0.01016807, + "auxiliary_loss_mlp": 0.01009178, + "balance_loss_clip": 1.0040735, + "balance_loss_mlp": 1.00791442, + "epoch": 0.08284487261331322, + "flos": 59143425730560.0, + "grad_norm": 0.7504412906896574, + "language_loss": 0.52296829, + "learning_rate": 3.970782799504682e-06, + "loss": 0.54322815, + "num_input_tokens_seen": 79952015, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01263428, + "step": 2855, + "time_per_iteration": 3.018023729324341 + }, + { + "auxiliary_loss_clip": 0.01114059, + "auxiliary_loss_mlp": 0.01047346, + "balance_loss_clip": 1.03419471, + "balance_loss_mlp": 1.02467275, + "epoch": 0.08287389008182926, + "flos": 16795526780160.0, + "grad_norm": 2.9349940846680367, + "language_loss": 0.90284532, + "learning_rate": 3.970750779977944e-06, + "loss": 0.9244594, + "num_input_tokens_seen": 79965315, + "router_z_loss_clip": 0.79882812, + "router_z_loss_mlp": 0.22668457, + "step": 2856, + "time_per_iteration": 2.411261796951294 + }, + { + "auxiliary_loss_clip": 0.01106489, + "auxiliary_loss_mlp": 0.0103372, + "balance_loss_clip": 1.03110671, + "balance_loss_mlp": 1.01455092, + "epoch": 0.08290290755034531, + "flos": 24200709816960.0, + "grad_norm": 3.004601193600722, + "language_loss": 0.5833739, + "learning_rate": 3.9707187430447384e-06, + "loss": 0.60477597, + "num_input_tokens_seen": 79983950, + "router_z_loss_clip": 0.75390625, + "router_z_loss_mlp": 0.19171143, + "step": 2857, + "time_per_iteration": 2.3447868824005127 + }, + { + "auxiliary_loss_clip": 0.01107613, + "auxiliary_loss_mlp": 0.01041387, + "balance_loss_clip": 1.03504717, + "balance_loss_mlp": 1.01988196, + "epoch": 0.08293192501886136, + "flos": 20112555098880.0, + "grad_norm": 3.070251938661648, + "language_loss": 0.82827586, + "learning_rate": 3.970686688705347e-06, + "loss": 0.8497659, + "num_input_tokens_seen": 79997820, + "router_z_loss_clip": 0.72558594, + "router_z_loss_mlp": 0.21484375, + "step": 2858, + "time_per_iteration": 2.3682544231414795 + }, + { + "auxiliary_loss_clip": 0.01116268, + "auxiliary_loss_mlp": 0.01039979, + "balance_loss_clip": 1.03590536, + "balance_loss_mlp": 1.0175792, + "epoch": 0.0829609424873774, + "flos": 27993499499520.0, + "grad_norm": 2.3382947223561206, + "language_loss": 0.67023635, + "learning_rate": 3.970654616960054e-06, + "loss": 0.69179881, + "num_input_tokens_seen": 80011150, + "router_z_loss_clip": 0.80371094, + "router_z_loss_mlp": 0.22387695, + "step": 2859, + "time_per_iteration": 2.41036319732666 + }, + { + "auxiliary_loss_clip": 0.01107396, + "auxiliary_loss_mlp": 0.01042606, + "balance_loss_clip": 1.03353846, + "balance_loss_mlp": 1.02151799, + "epoch": 0.08298995995589345, + "flos": 11757978927360.0, + "grad_norm": 2.37889111820421, + "language_loss": 0.79459351, + "learning_rate": 3.970622527809142e-06, + "loss": 0.81609356, + "num_input_tokens_seen": 80022510, + "router_z_loss_clip": 0.73925781, + "router_z_loss_mlp": 0.2109375, + "step": 2860, + "time_per_iteration": 2.3345744609832764 + }, + { + "auxiliary_loss_clip": 0.01015331, + "auxiliary_loss_mlp": 0.01001645, + "balance_loss_clip": 1.00292122, + "balance_loss_mlp": 1.0004822, + "epoch": 0.0830189774244095, + "flos": 66554369141760.0, + "grad_norm": 0.6612859108205398, + "language_loss": 0.47982681, + "learning_rate": 3.970590421252893e-06, + "loss": 0.49999654, + "num_input_tokens_seen": 80079365, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.01159668, + "step": 2861, + "time_per_iteration": 3.001441240310669 + }, + { + "auxiliary_loss_clip": 0.01118118, + "auxiliary_loss_mlp": 0.0104569, + "balance_loss_clip": 1.03686452, + "balance_loss_mlp": 1.0208112, + "epoch": 0.08304799489292554, + "flos": 48906019647360.0, + "grad_norm": 2.744411720127663, + "language_loss": 0.8527863, + "learning_rate": 3.970558297291593e-06, + "loss": 0.87442434, + "num_input_tokens_seen": 80099900, + "router_z_loss_clip": 0.81298828, + "router_z_loss_mlp": 0.24853516, + "step": 2862, + "time_per_iteration": 2.6095998287200928 + }, + { + "auxiliary_loss_clip": 0.01113347, + "auxiliary_loss_mlp": 0.01041311, + "balance_loss_clip": 1.03354895, + "balance_loss_mlp": 1.01929927, + "epoch": 0.08307701236144159, + "flos": 21061075449600.0, + "grad_norm": 2.4749686039801913, + "language_loss": 1.00972676, + "learning_rate": 3.9705261559255246e-06, + "loss": 1.03127337, + "num_input_tokens_seen": 80112670, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.22015381, + "step": 2863, + "time_per_iteration": 2.3870084285736084 + }, + { + "auxiliary_loss_clip": 0.0111594, + "auxiliary_loss_mlp": 0.01046027, + "balance_loss_clip": 1.03388619, + "balance_loss_mlp": 1.02369905, + "epoch": 0.08310602982995763, + "flos": 74729772748800.0, + "grad_norm": 1.6385638110503944, + "language_loss": 0.84473002, + "learning_rate": 3.970493997154972e-06, + "loss": 0.8663497, + "num_input_tokens_seen": 80141800, + "router_z_loss_clip": 0.82080078, + "router_z_loss_mlp": 0.22338867, + "step": 2864, + "time_per_iteration": 2.834379196166992 + }, + { + "auxiliary_loss_clip": 0.01117365, + "auxiliary_loss_mlp": 0.01050385, + "balance_loss_clip": 1.03340292, + "balance_loss_mlp": 1.0241468, + "epoch": 0.08313504729847368, + "flos": 16246028891520.0, + "grad_norm": 7.829829178743152, + "language_loss": 0.8403008, + "learning_rate": 3.970461820980218e-06, + "loss": 0.86197835, + "num_input_tokens_seen": 80155880, + "router_z_loss_clip": 0.83984375, + "router_z_loss_mlp": 0.26245117, + "step": 2865, + "time_per_iteration": 2.3415496349334717 + }, + { + "auxiliary_loss_clip": 0.01105963, + "auxiliary_loss_mlp": 0.01043883, + "balance_loss_clip": 1.03211486, + "balance_loss_mlp": 1.02193666, + "epoch": 0.08316406476698973, + "flos": 16353247276800.0, + "grad_norm": 3.480478132625179, + "language_loss": 0.79358792, + "learning_rate": 3.97042962740155e-06, + "loss": 0.81508636, + "num_input_tokens_seen": 80165950, + "router_z_loss_clip": 0.73828125, + "router_z_loss_mlp": 0.21929932, + "step": 2866, + "time_per_iteration": 2.3742856979370117 + }, + { + "auxiliary_loss_clip": 0.01116786, + "auxiliary_loss_mlp": 0.01048536, + "balance_loss_clip": 1.03623128, + "balance_loss_mlp": 1.02581501, + "epoch": 0.08319308223550577, + "flos": 13689723386880.0, + "grad_norm": 3.240558775803687, + "language_loss": 0.84523553, + "learning_rate": 3.970397416419248e-06, + "loss": 0.8668887, + "num_input_tokens_seen": 80176790, + "router_z_loss_clip": 0.80615234, + "router_z_loss_mlp": 0.22729492, + "step": 2867, + "time_per_iteration": 2.3723416328430176 + }, + { + "auxiliary_loss_clip": 0.01013371, + "auxiliary_loss_mlp": 0.01001833, + "balance_loss_clip": 1.00102115, + "balance_loss_mlp": 1.00068295, + "epoch": 0.08322209970402182, + "flos": 65587064613120.0, + "grad_norm": 0.8503100531247292, + "language_loss": 0.50152695, + "learning_rate": 3.9703651880336e-06, + "loss": 0.52167904, + "num_input_tokens_seen": 80244220, + "router_z_loss_clip": 0.12353516, + "router_z_loss_mlp": 0.01147461, + "step": 2868, + "time_per_iteration": 3.0508241653442383 + }, + { + "auxiliary_loss_clip": 0.01107895, + "auxiliary_loss_mlp": 0.01042373, + "balance_loss_clip": 1.03591394, + "balance_loss_mlp": 1.02200055, + "epoch": 0.08325111717253787, + "flos": 31205229557760.0, + "grad_norm": 1.7995008325609045, + "language_loss": 0.83926541, + "learning_rate": 3.9703329422448884e-06, + "loss": 0.86076814, + "num_input_tokens_seen": 80264805, + "router_z_loss_clip": 0.71972656, + "router_z_loss_mlp": 0.20361328, + "step": 2869, + "time_per_iteration": 2.450917959213257 + }, + { + "auxiliary_loss_clip": 0.01111, + "auxiliary_loss_mlp": 0.0105225, + "balance_loss_clip": 1.03236878, + "balance_loss_mlp": 1.03223419, + "epoch": 0.08328013464105391, + "flos": 15662281092480.0, + "grad_norm": 2.3362108532902557, + "language_loss": 0.81637776, + "learning_rate": 3.970300679053399e-06, + "loss": 0.83801031, + "num_input_tokens_seen": 80279240, + "router_z_loss_clip": 0.78662109, + "router_z_loss_mlp": 0.20019531, + "step": 2870, + "time_per_iteration": 2.3841991424560547 + }, + { + "auxiliary_loss_clip": 0.01108132, + "auxiliary_loss_mlp": 0.01047113, + "balance_loss_clip": 1.03358114, + "balance_loss_mlp": 1.02643049, + "epoch": 0.08330915210956996, + "flos": 16061722490880.0, + "grad_norm": 3.2289308575169633, + "language_loss": 0.73736823, + "learning_rate": 3.970268398459417e-06, + "loss": 0.75892067, + "num_input_tokens_seen": 80291680, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.20678711, + "step": 2871, + "time_per_iteration": 2.330613136291504 + }, + { + "auxiliary_loss_clip": 0.01101996, + "auxiliary_loss_mlp": 0.01036488, + "balance_loss_clip": 1.03088856, + "balance_loss_mlp": 1.01785588, + "epoch": 0.08333816957808601, + "flos": 43098951317760.0, + "grad_norm": 3.0962726130313616, + "language_loss": 0.7637139, + "learning_rate": 3.970236100463228e-06, + "loss": 0.78509879, + "num_input_tokens_seen": 80309670, + "router_z_loss_clip": 0.70996094, + "router_z_loss_mlp": 0.18615723, + "step": 2872, + "time_per_iteration": 2.6138830184936523 + }, + { + "auxiliary_loss_clip": 0.01114807, + "auxiliary_loss_mlp": 0.01038346, + "balance_loss_clip": 1.03396678, + "balance_loss_mlp": 1.01678133, + "epoch": 0.08336718704660205, + "flos": 16610138127360.0, + "grad_norm": 3.290263422967554, + "language_loss": 0.73054361, + "learning_rate": 3.970203785065116e-06, + "loss": 0.75207514, + "num_input_tokens_seen": 80322605, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.21569824, + "step": 2873, + "time_per_iteration": 2.321110248565674 + }, + { + "auxiliary_loss_clip": 0.01111915, + "auxiliary_loss_mlp": 0.01042297, + "balance_loss_clip": 1.03217828, + "balance_loss_mlp": 1.01920569, + "epoch": 0.0833962045151181, + "flos": 18909657515520.0, + "grad_norm": 2.4537313728214163, + "language_loss": 0.72204697, + "learning_rate": 3.970171452265366e-06, + "loss": 0.74358916, + "num_input_tokens_seen": 80338735, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.23095703, + "step": 2874, + "time_per_iteration": 2.471444845199585 + }, + { + "auxiliary_loss_clip": 0.01014267, + "auxiliary_loss_mlp": 0.01002962, + "balance_loss_clip": 1.00196648, + "balance_loss_mlp": 1.00177634, + "epoch": 0.08342522198363415, + "flos": 66128916913920.0, + "grad_norm": 0.6407525891425349, + "language_loss": 0.46893013, + "learning_rate": 3.970139102064265e-06, + "loss": 0.48910239, + "num_input_tokens_seen": 80402755, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01184082, + "step": 2875, + "time_per_iteration": 3.147367000579834 + }, + { + "auxiliary_loss_clip": 0.01013931, + "auxiliary_loss_mlp": 0.01002168, + "balance_loss_clip": 1.00160694, + "balance_loss_mlp": 1.00098777, + "epoch": 0.08345423945215019, + "flos": 61080092691840.0, + "grad_norm": 0.7164452714712021, + "language_loss": 0.48851502, + "learning_rate": 3.970106734462099e-06, + "loss": 0.50867599, + "num_input_tokens_seen": 80460090, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01177979, + "step": 2876, + "time_per_iteration": 2.8701207637786865 + }, + { + "auxiliary_loss_clip": 0.01112349, + "auxiliary_loss_mlp": 0.01038785, + "balance_loss_clip": 1.03650022, + "balance_loss_mlp": 1.01890099, + "epoch": 0.08348325692066624, + "flos": 38465592808320.0, + "grad_norm": 2.2478626283422987, + "language_loss": 0.89126515, + "learning_rate": 3.970074349459152e-06, + "loss": 0.91277647, + "num_input_tokens_seen": 80481000, + "router_z_loss_clip": 0.75878906, + "router_z_loss_mlp": 0.19885254, + "step": 2877, + "time_per_iteration": 2.5122828483581543 + }, + { + "auxiliary_loss_clip": 0.01112777, + "auxiliary_loss_mlp": 0.01059378, + "balance_loss_clip": 1.03534365, + "balance_loss_mlp": 1.03541672, + "epoch": 0.08351227438918228, + "flos": 20113881730560.0, + "grad_norm": 2.231062084073646, + "language_loss": 0.73109102, + "learning_rate": 3.970041947055712e-06, + "loss": 0.75281256, + "num_input_tokens_seen": 80496045, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.23962402, + "step": 2878, + "time_per_iteration": 2.4654481410980225 + }, + { + "auxiliary_loss_clip": 0.0112522, + "auxiliary_loss_mlp": 0.01047274, + "balance_loss_clip": 1.03506982, + "balance_loss_mlp": 1.02345562, + "epoch": 0.08354129185769833, + "flos": 27632392640640.0, + "grad_norm": 2.179919908640117, + "language_loss": 1.09750581, + "learning_rate": 3.970009527252064e-06, + "loss": 1.11923087, + "num_input_tokens_seen": 80520330, + "router_z_loss_clip": 0.89990234, + "router_z_loss_mlp": 0.23828125, + "step": 2879, + "time_per_iteration": 2.5950372219085693 + }, + { + "auxiliary_loss_clip": 0.01119262, + "auxiliary_loss_mlp": 0.01048506, + "balance_loss_clip": 1.0349834, + "balance_loss_mlp": 1.02489638, + "epoch": 0.08357030932621438, + "flos": 25404235804800.0, + "grad_norm": 2.7387046517180234, + "language_loss": 1.04937053, + "learning_rate": 3.969977090048495e-06, + "loss": 1.07104826, + "num_input_tokens_seen": 80535250, + "router_z_loss_clip": 0.84228516, + "router_z_loss_mlp": 0.23638916, + "step": 2880, + "time_per_iteration": 2.514106273651123 + }, + { + "auxiliary_loss_clip": 0.01014807, + "auxiliary_loss_mlp": 0.0100244, + "balance_loss_clip": 1.00288391, + "balance_loss_mlp": 1.0014447, + "epoch": 0.08359932679473042, + "flos": 74767791219840.0, + "grad_norm": 0.7531210995760185, + "language_loss": 0.56350124, + "learning_rate": 3.9699446354452904e-06, + "loss": 0.58367372, + "num_input_tokens_seen": 80592605, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.00994873, + "step": 2881, + "time_per_iteration": 2.9580252170562744 + }, + { + "auxiliary_loss_clip": 0.01013371, + "auxiliary_loss_mlp": 0.0100197, + "balance_loss_clip": 1.0014379, + "balance_loss_mlp": 1.00079, + "epoch": 0.08362834426324647, + "flos": 68499694120320.0, + "grad_norm": 0.6192170512423878, + "language_loss": 0.47495487, + "learning_rate": 3.969912163442738e-06, + "loss": 0.49510828, + "num_input_tokens_seen": 80654020, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01177979, + "step": 2882, + "time_per_iteration": 3.0092341899871826 + }, + { + "auxiliary_loss_clip": 0.01013148, + "auxiliary_loss_mlp": 0.01001303, + "balance_loss_clip": 1.00119758, + "balance_loss_mlp": 1.00006962, + "epoch": 0.08365736173176253, + "flos": 74786084772480.0, + "grad_norm": 0.645954074613164, + "language_loss": 0.52937442, + "learning_rate": 3.969879674041125e-06, + "loss": 0.54951894, + "num_input_tokens_seen": 80721415, + "router_z_loss_clip": 0.11962891, + "router_z_loss_mlp": 0.0123291, + "step": 2883, + "time_per_iteration": 3.150560140609741 + }, + { + "auxiliary_loss_clip": 0.01105928, + "auxiliary_loss_mlp": 0.01036854, + "balance_loss_clip": 1.03363299, + "balance_loss_mlp": 1.01786375, + "epoch": 0.08368637920027856, + "flos": 40508256257280.0, + "grad_norm": 2.3464567003663053, + "language_loss": 0.83695853, + "learning_rate": 3.969847167240736e-06, + "loss": 0.8583864, + "num_input_tokens_seen": 80737960, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.18994141, + "step": 2884, + "time_per_iteration": 2.4496350288391113 + }, + { + "auxiliary_loss_clip": 0.01114867, + "auxiliary_loss_mlp": 0.01043731, + "balance_loss_clip": 1.03421104, + "balance_loss_mlp": 1.021415, + "epoch": 0.08371539666879461, + "flos": 34306738853760.0, + "grad_norm": 2.056557496686284, + "language_loss": 0.88741076, + "learning_rate": 3.969814643041861e-06, + "loss": 0.90899676, + "num_input_tokens_seen": 80755925, + "router_z_loss_clip": 0.80712891, + "router_z_loss_mlp": 0.22314453, + "step": 2885, + "time_per_iteration": 2.5275161266326904 + }, + { + "auxiliary_loss_clip": 0.01113634, + "auxiliary_loss_mlp": 0.01041098, + "balance_loss_clip": 1.03516746, + "balance_loss_mlp": 1.0196048, + "epoch": 0.08374441413731067, + "flos": 17779553850240.0, + "grad_norm": 2.3393004901795207, + "language_loss": 0.80371249, + "learning_rate": 3.969782101444785e-06, + "loss": 0.82525975, + "num_input_tokens_seen": 80770785, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.21484375, + "step": 2886, + "time_per_iteration": 2.3556175231933594 + }, + { + "auxiliary_loss_clip": 0.01116154, + "auxiliary_loss_mlp": 0.01051156, + "balance_loss_clip": 1.03544414, + "balance_loss_mlp": 1.02898335, + "epoch": 0.0837734316058267, + "flos": 26460986970240.0, + "grad_norm": 3.9711776734146293, + "language_loss": 0.82495129, + "learning_rate": 3.969749542449797e-06, + "loss": 0.84662437, + "num_input_tokens_seen": 80787430, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.22149658, + "step": 2887, + "time_per_iteration": 2.4694035053253174 + }, + { + "auxiliary_loss_clip": 0.0101404, + "auxiliary_loss_mlp": 0.0100231, + "balance_loss_clip": 1.00181317, + "balance_loss_mlp": 1.00128436, + "epoch": 0.08380244907434276, + "flos": 74776763439360.0, + "grad_norm": 0.6852484785396002, + "language_loss": 0.50380546, + "learning_rate": 3.969716966057184e-06, + "loss": 0.52396899, + "num_input_tokens_seen": 80850515, + "router_z_loss_clip": 0.12207031, + "router_z_loss_mlp": 0.01025391, + "step": 2888, + "time_per_iteration": 3.0646798610687256 + }, + { + "auxiliary_loss_clip": 0.01114293, + "auxiliary_loss_mlp": 0.01039322, + "balance_loss_clip": 1.03263271, + "balance_loss_mlp": 1.01563525, + "epoch": 0.0838314665428588, + "flos": 21099025964160.0, + "grad_norm": 2.6142555324166024, + "language_loss": 1.07799971, + "learning_rate": 3.969684372267235e-06, + "loss": 1.09953594, + "num_input_tokens_seen": 80864320, + "router_z_loss_clip": 0.81689453, + "router_z_loss_mlp": 0.23681641, + "step": 2889, + "time_per_iteration": 2.417722225189209 + }, + { + "auxiliary_loss_clip": 0.01111585, + "auxiliary_loss_mlp": 0.01044376, + "balance_loss_clip": 1.0318861, + "balance_loss_mlp": 1.02451563, + "epoch": 0.08386048401137484, + "flos": 17083769898240.0, + "grad_norm": 3.1834413252069704, + "language_loss": 1.00042129, + "learning_rate": 3.9696517610802345e-06, + "loss": 1.02198076, + "num_input_tokens_seen": 80877340, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.19873047, + "step": 2890, + "time_per_iteration": 2.371473789215088 + }, + { + "auxiliary_loss_clip": 0.01119888, + "auxiliary_loss_mlp": 0.01053764, + "balance_loss_clip": 1.03687096, + "balance_loss_mlp": 1.03013086, + "epoch": 0.0838895014798909, + "flos": 11764681908480.0, + "grad_norm": 3.357519805973125, + "language_loss": 0.88526499, + "learning_rate": 3.969619132496473e-06, + "loss": 0.9070015, + "num_input_tokens_seen": 80888470, + "router_z_loss_clip": 0.82861328, + "router_z_loss_mlp": 0.23638916, + "step": 2891, + "time_per_iteration": 2.404602527618408 + }, + { + "auxiliary_loss_clip": 0.01012591, + "auxiliary_loss_mlp": 0.01004929, + "balance_loss_clip": 1.00074637, + "balance_loss_mlp": 1.00396359, + "epoch": 0.08391851894840695, + "flos": 65797975336320.0, + "grad_norm": 0.7725585545916055, + "language_loss": 0.53194392, + "learning_rate": 3.969586486516239e-06, + "loss": 0.55211914, + "num_input_tokens_seen": 80946685, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.00964355, + "step": 2892, + "time_per_iteration": 2.9537501335144043 + }, + { + "auxiliary_loss_clip": 0.01117503, + "auxiliary_loss_mlp": 0.01044193, + "balance_loss_clip": 1.03560781, + "balance_loss_mlp": 1.02111459, + "epoch": 0.08394753641692299, + "flos": 26136573816960.0, + "grad_norm": 1.985582175383617, + "language_loss": 0.91010731, + "learning_rate": 3.96955382313982e-06, + "loss": 0.93172425, + "num_input_tokens_seen": 80966870, + "router_z_loss_clip": 0.81933594, + "router_z_loss_mlp": 0.23083496, + "step": 2893, + "time_per_iteration": 4.685545444488525 + }, + { + "auxiliary_loss_clip": 0.01107565, + "auxiliary_loss_mlp": 0.01039898, + "balance_loss_clip": 1.03161252, + "balance_loss_mlp": 1.01954877, + "epoch": 0.08397655388543904, + "flos": 17449973815680.0, + "grad_norm": 2.264280897247058, + "language_loss": 0.81428707, + "learning_rate": 3.969521142367504e-06, + "loss": 0.83576167, + "num_input_tokens_seen": 80984620, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.20361328, + "step": 2894, + "time_per_iteration": 2.373903512954712 + }, + { + "auxiliary_loss_clip": 0.01107479, + "auxiliary_loss_mlp": 0.01048023, + "balance_loss_clip": 1.0320828, + "balance_loss_mlp": 1.02504003, + "epoch": 0.08400557135395507, + "flos": 26132663744640.0, + "grad_norm": 2.125895788132434, + "language_loss": 0.80439037, + "learning_rate": 3.969488444199581e-06, + "loss": 0.82594538, + "num_input_tokens_seen": 81000215, + "router_z_loss_clip": 0.75390625, + "router_z_loss_mlp": 0.22998047, + "step": 2895, + "time_per_iteration": 4.723361253738403 + }, + { + "auxiliary_loss_clip": 0.01112208, + "auxiliary_loss_mlp": 0.01035604, + "balance_loss_clip": 1.03538215, + "balance_loss_mlp": 1.01691222, + "epoch": 0.08403458882247113, + "flos": 23542876379520.0, + "grad_norm": 2.592087333532796, + "language_loss": 0.88510013, + "learning_rate": 3.969455728636339e-06, + "loss": 0.9065783, + "num_input_tokens_seen": 81015230, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.18701172, + "step": 2896, + "time_per_iteration": 2.4262301921844482 + }, + { + "auxiliary_loss_clip": 0.01111228, + "auxiliary_loss_mlp": 0.01043288, + "balance_loss_clip": 1.03299189, + "balance_loss_mlp": 1.02076364, + "epoch": 0.08406360629098718, + "flos": 27555060245760.0, + "grad_norm": 2.647297477493107, + "language_loss": 0.76946032, + "learning_rate": 3.969422995678067e-06, + "loss": 0.79100549, + "num_input_tokens_seen": 81033775, + "router_z_loss_clip": 0.78320312, + "router_z_loss_mlp": 0.2253418, + "step": 2897, + "time_per_iteration": 2.4689230918884277 + }, + { + "auxiliary_loss_clip": 0.01103881, + "auxiliary_loss_mlp": 0.01038882, + "balance_loss_clip": 1.03107095, + "balance_loss_mlp": 1.01837778, + "epoch": 0.08409262375950322, + "flos": 16213768928640.0, + "grad_norm": 3.3372891619798044, + "language_loss": 0.62328851, + "learning_rate": 3.969390245325053e-06, + "loss": 0.6447162, + "num_input_tokens_seen": 81048230, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.2052002, + "step": 2898, + "time_per_iteration": 4.897232294082642 + }, + { + "auxiliary_loss_clip": 0.01105158, + "auxiliary_loss_mlp": 0.01037149, + "balance_loss_clip": 1.03327179, + "balance_loss_mlp": 1.01863599, + "epoch": 0.08412164122801927, + "flos": 44082454717440.0, + "grad_norm": 2.089060805895795, + "language_loss": 0.74684989, + "learning_rate": 3.969357477577589e-06, + "loss": 0.768273, + "num_input_tokens_seen": 81067855, + "router_z_loss_clip": 0.71923828, + "router_z_loss_mlp": 0.18493652, + "step": 2899, + "time_per_iteration": 5.122437477111816 + }, + { + "auxiliary_loss_clip": 0.01113515, + "auxiliary_loss_mlp": 0.01051784, + "balance_loss_clip": 1.03436732, + "balance_loss_mlp": 1.02975392, + "epoch": 0.08415065869653532, + "flos": 74729039610240.0, + "grad_norm": 4.702346963689112, + "language_loss": 0.7625705, + "learning_rate": 3.969324692435962e-06, + "loss": 0.78422356, + "num_input_tokens_seen": 81089800, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.22033691, + "step": 2900, + "time_per_iteration": 2.7802886962890625 + }, + { + "auxiliary_loss_clip": 0.01115867, + "auxiliary_loss_mlp": 0.01048534, + "balance_loss_clip": 1.03536141, + "balance_loss_mlp": 1.02706397, + "epoch": 0.08417967616505136, + "flos": 25511908037760.0, + "grad_norm": 2.5967311492255463, + "language_loss": 0.98246753, + "learning_rate": 3.969291889900463e-06, + "loss": 1.00411153, + "num_input_tokens_seen": 81105500, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.21472168, + "step": 2901, + "time_per_iteration": 2.45947003364563 + }, + { + "auxiliary_loss_clip": 0.01014423, + "auxiliary_loss_mlp": 0.01001388, + "balance_loss_clip": 1.00255179, + "balance_loss_mlp": 1.00029731, + "epoch": 0.08420869363356741, + "flos": 59117449812480.0, + "grad_norm": 0.661339153633707, + "language_loss": 0.48312122, + "learning_rate": 3.969259069971381e-06, + "loss": 0.50327933, + "num_input_tokens_seen": 81168915, + "router_z_loss_clip": 0.11865234, + "router_z_loss_mlp": 0.01092529, + "step": 2902, + "time_per_iteration": 3.040364980697632 + }, + { + "auxiliary_loss_clip": 0.01112188, + "auxiliary_loss_mlp": 0.0103826, + "balance_loss_clip": 1.03464079, + "balance_loss_mlp": 1.0164448, + "epoch": 0.08423771110208346, + "flos": 30037105555200.0, + "grad_norm": 2.520294992545679, + "language_loss": 0.87090558, + "learning_rate": 3.9692262326490054e-06, + "loss": 0.89241004, + "num_input_tokens_seen": 81184685, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.21838379, + "step": 2903, + "time_per_iteration": 2.4781723022460938 + }, + { + "auxiliary_loss_clip": 0.01108691, + "auxiliary_loss_mlp": 0.01042055, + "balance_loss_clip": 1.03508008, + "balance_loss_mlp": 1.02108562, + "epoch": 0.0842667285705995, + "flos": 11611483395840.0, + "grad_norm": 3.800332743798573, + "language_loss": 0.90523899, + "learning_rate": 3.969193377933628e-06, + "loss": 0.92674643, + "num_input_tokens_seen": 81196130, + "router_z_loss_clip": 0.73632812, + "router_z_loss_mlp": 0.20977783, + "step": 2904, + "time_per_iteration": 2.3363852500915527 + }, + { + "auxiliary_loss_clip": 0.01109516, + "auxiliary_loss_mlp": 0.01045034, + "balance_loss_clip": 1.03197646, + "balance_loss_mlp": 1.02461326, + "epoch": 0.08429574603911555, + "flos": 22884449448960.0, + "grad_norm": 2.459277668561083, + "language_loss": 0.9279356, + "learning_rate": 3.969160505825536e-06, + "loss": 0.94948107, + "num_input_tokens_seen": 81209205, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.20410156, + "step": 2905, + "time_per_iteration": 2.4292609691619873 + }, + { + "auxiliary_loss_clip": 0.01109725, + "auxiliary_loss_mlp": 0.01045244, + "balance_loss_clip": 1.03372145, + "balance_loss_mlp": 1.02389407, + "epoch": 0.0843247635076316, + "flos": 32626648540800.0, + "grad_norm": 2.0175375645129017, + "language_loss": 0.86062658, + "learning_rate": 3.9691276163250235e-06, + "loss": 0.88217628, + "num_input_tokens_seen": 81231660, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.21350098, + "step": 2906, + "time_per_iteration": 2.580383539199829 + }, + { + "auxiliary_loss_clip": 0.01106185, + "auxiliary_loss_mlp": 0.01037474, + "balance_loss_clip": 1.0315001, + "balance_loss_mlp": 1.01656461, + "epoch": 0.08435378097614764, + "flos": 74734346136960.0, + "grad_norm": 1.7440519288800636, + "language_loss": 0.83991671, + "learning_rate": 3.969094709432378e-06, + "loss": 0.86135334, + "num_input_tokens_seen": 81258125, + "router_z_loss_clip": 0.74707031, + "router_z_loss_mlp": 0.20922852, + "step": 2907, + "time_per_iteration": 2.8346500396728516 + }, + { + "auxiliary_loss_clip": 0.0110595, + "auxiliary_loss_mlp": 0.01047992, + "balance_loss_clip": 1.03060126, + "balance_loss_mlp": 1.02498496, + "epoch": 0.08438279844466369, + "flos": 25366529669760.0, + "grad_norm": 2.4073598668768192, + "language_loss": 0.76404643, + "learning_rate": 3.9690617851478915e-06, + "loss": 0.78558588, + "num_input_tokens_seen": 81273645, + "router_z_loss_clip": 0.75292969, + "router_z_loss_mlp": 0.22998047, + "step": 2908, + "time_per_iteration": 2.3819143772125244 + }, + { + "auxiliary_loss_clip": 0.01118608, + "auxiliary_loss_mlp": 0.01051021, + "balance_loss_clip": 1.03557897, + "balance_loss_mlp": 1.027215, + "epoch": 0.08441181591317973, + "flos": 50141910332160.0, + "grad_norm": 2.1520191773984085, + "language_loss": 1.05332029, + "learning_rate": 3.969028843471854e-06, + "loss": 1.0750165, + "num_input_tokens_seen": 81298960, + "router_z_loss_clip": 0.83007812, + "router_z_loss_mlp": 0.23803711, + "step": 2909, + "time_per_iteration": 2.669613838195801 + }, + { + "auxiliary_loss_clip": 0.01014526, + "auxiliary_loss_mlp": 0.01002323, + "balance_loss_clip": 1.00264132, + "balance_loss_mlp": 1.00113094, + "epoch": 0.08444083338169578, + "flos": 61342464625920.0, + "grad_norm": 0.6944322975836631, + "language_loss": 0.5009377, + "learning_rate": 3.968995884404558e-06, + "loss": 0.52110612, + "num_input_tokens_seen": 81355070, + "router_z_loss_clip": 0.11865234, + "router_z_loss_mlp": 0.01190186, + "step": 2910, + "time_per_iteration": 2.9432737827301025 + }, + { + "auxiliary_loss_clip": 0.01115641, + "auxiliary_loss_mlp": 0.01061565, + "balance_loss_clip": 1.03565693, + "balance_loss_mlp": 1.03527975, + "epoch": 0.08446985085021183, + "flos": 10370879677440.0, + "grad_norm": 3.751919102985839, + "language_loss": 1.02433431, + "learning_rate": 3.968962907946293e-06, + "loss": 1.04610646, + "num_input_tokens_seen": 81364915, + "router_z_loss_clip": 0.79931641, + "router_z_loss_mlp": 0.26281738, + "step": 2911, + "time_per_iteration": 2.3995561599731445 + }, + { + "auxiliary_loss_clip": 0.01014008, + "auxiliary_loss_mlp": 0.01002366, + "balance_loss_clip": 1.00197005, + "balance_loss_mlp": 1.00126958, + "epoch": 0.08449886831872787, + "flos": 74776030300800.0, + "grad_norm": 0.72997072098109, + "language_loss": 0.45242137, + "learning_rate": 3.968929914097351e-06, + "loss": 0.47258508, + "num_input_tokens_seen": 81431215, + "router_z_loss_clip": 0.12011719, + "router_z_loss_mlp": 0.01098633, + "step": 2912, + "time_per_iteration": 3.053807020187378 + }, + { + "auxiliary_loss_clip": 0.01013523, + "auxiliary_loss_mlp": 0.01002181, + "balance_loss_clip": 1.00159478, + "balance_loss_mlp": 1.00092363, + "epoch": 0.08452788578724392, + "flos": 74777845691520.0, + "grad_norm": 0.7064530304618512, + "language_loss": 0.53292584, + "learning_rate": 3.968896902858023e-06, + "loss": 0.55308282, + "num_input_tokens_seen": 81501990, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01257324, + "step": 2913, + "time_per_iteration": 3.175983428955078 + }, + { + "auxiliary_loss_clip": 0.01013615, + "auxiliary_loss_mlp": 0.01001272, + "balance_loss_clip": 1.00159979, + "balance_loss_mlp": 0.9999969, + "epoch": 0.08455690325575997, + "flos": 74775122605440.0, + "grad_norm": 0.6995476875337099, + "language_loss": 0.53645968, + "learning_rate": 3.968863874228601e-06, + "loss": 0.55660856, + "num_input_tokens_seen": 81569450, + "router_z_loss_clip": 0.12011719, + "router_z_loss_mlp": 0.01275635, + "step": 2914, + "time_per_iteration": 3.0939345359802246 + }, + { + "auxiliary_loss_clip": 0.01107386, + "auxiliary_loss_mlp": 0.01041715, + "balance_loss_clip": 1.03318191, + "balance_loss_mlp": 1.02099609, + "epoch": 0.08458592072427601, + "flos": 25074760504320.0, + "grad_norm": 2.867158487837187, + "language_loss": 0.854877, + "learning_rate": 3.968830828209377e-06, + "loss": 0.87636805, + "num_input_tokens_seen": 81585420, + "router_z_loss_clip": 0.74169922, + "router_z_loss_mlp": 0.20727539, + "step": 2915, + "time_per_iteration": 2.407895565032959 + }, + { + "auxiliary_loss_clip": 0.01108299, + "auxiliary_loss_mlp": 0.01046028, + "balance_loss_clip": 1.03327084, + "balance_loss_mlp": 1.02355695, + "epoch": 0.08461493819279206, + "flos": 33503596871040.0, + "grad_norm": 2.2680668952153424, + "language_loss": 0.81015182, + "learning_rate": 3.968797764800642e-06, + "loss": 0.83169508, + "num_input_tokens_seen": 81603545, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.22473145, + "step": 2916, + "time_per_iteration": 2.527401924133301 + }, + { + "auxiliary_loss_clip": 0.0110899, + "auxiliary_loss_mlp": 0.01044269, + "balance_loss_clip": 1.03293121, + "balance_loss_mlp": 1.02094018, + "epoch": 0.08464395566130811, + "flos": 32703282708480.0, + "grad_norm": 2.557574959529587, + "language_loss": 0.89477301, + "learning_rate": 3.968764684002688e-06, + "loss": 0.9163056, + "num_input_tokens_seen": 81619200, + "router_z_loss_clip": 0.76171875, + "router_z_loss_mlp": 0.23327637, + "step": 2917, + "time_per_iteration": 2.463477849960327 + }, + { + "auxiliary_loss_clip": 0.0111178, + "auxiliary_loss_mlp": 0.01051194, + "balance_loss_clip": 1.03286564, + "balance_loss_mlp": 1.0279603, + "epoch": 0.08467297312982415, + "flos": 24200046501120.0, + "grad_norm": 2.5323080033072305, + "language_loss": 0.96315646, + "learning_rate": 3.968731585815808e-06, + "loss": 0.98478621, + "num_input_tokens_seen": 81634375, + "router_z_loss_clip": 0.7890625, + "router_z_loss_mlp": 0.23205566, + "step": 2918, + "time_per_iteration": 2.4515674114227295 + }, + { + "auxiliary_loss_clip": 0.01113414, + "auxiliary_loss_mlp": 0.01047515, + "balance_loss_clip": 1.03371942, + "balance_loss_mlp": 1.02492547, + "epoch": 0.0847019905983402, + "flos": 30477534756480.0, + "grad_norm": 2.496858841847892, + "language_loss": 0.9099918, + "learning_rate": 3.968698470240294e-06, + "loss": 0.93160111, + "num_input_tokens_seen": 81650875, + "router_z_loss_clip": 0.79638672, + "router_z_loss_mlp": 0.22583008, + "step": 2919, + "time_per_iteration": 2.4358572959899902 + }, + { + "auxiliary_loss_clip": 0.01106163, + "auxiliary_loss_mlp": 0.01044496, + "balance_loss_clip": 1.03240466, + "balance_loss_mlp": 1.02263284, + "epoch": 0.08473100806685625, + "flos": 22630107127680.0, + "grad_norm": 2.9367373494307274, + "language_loss": 0.81108546, + "learning_rate": 3.968665337276439e-06, + "loss": 0.83259201, + "num_input_tokens_seen": 81662870, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.21881104, + "step": 2920, + "time_per_iteration": 2.4195220470428467 + }, + { + "auxiliary_loss_clip": 0.0111329, + "auxiliary_loss_mlp": 0.01040244, + "balance_loss_clip": 1.03351665, + "balance_loss_mlp": 1.01686764, + "epoch": 0.08476002553537229, + "flos": 31939313137920.0, + "grad_norm": 3.0352206368291013, + "language_loss": 0.81243491, + "learning_rate": 3.968632186924534e-06, + "loss": 0.83397025, + "num_input_tokens_seen": 81687295, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.23400879, + "step": 2921, + "time_per_iteration": 2.4951188564300537 + }, + { + "auxiliary_loss_clip": 0.01015558, + "auxiliary_loss_mlp": 0.01002082, + "balance_loss_clip": 1.0035243, + "balance_loss_mlp": 1.00077653, + "epoch": 0.08478904300388834, + "flos": 74774913137280.0, + "grad_norm": 0.6310468290547204, + "language_loss": 0.48865139, + "learning_rate": 3.968599019184874e-06, + "loss": 0.50882775, + "num_input_tokens_seen": 81756140, + "router_z_loss_clip": 0.12011719, + "router_z_loss_mlp": 0.01306152, + "step": 2922, + "time_per_iteration": 3.2305245399475098 + }, + { + "auxiliary_loss_clip": 0.01014777, + "auxiliary_loss_mlp": 0.01001496, + "balance_loss_clip": 1.00280666, + "balance_loss_mlp": 1.00008917, + "epoch": 0.0848180604724044, + "flos": 59484526513920.0, + "grad_norm": 0.6507227341441056, + "language_loss": 0.44095623, + "learning_rate": 3.96856583405775e-06, + "loss": 0.461119, + "num_input_tokens_seen": 81815875, + "router_z_loss_clip": 0.11962891, + "router_z_loss_mlp": 0.01403809, + "step": 2923, + "time_per_iteration": 3.0067548751831055 + }, + { + "auxiliary_loss_clip": 0.01013376, + "auxiliary_loss_mlp": 0.0100117, + "balance_loss_clip": 1.0014441, + "balance_loss_mlp": 0.99987638, + "epoch": 0.08484707794092043, + "flos": 58348906853760.0, + "grad_norm": 0.6096458875570194, + "language_loss": 0.46671462, + "learning_rate": 3.968532631543457e-06, + "loss": 0.4868601, + "num_input_tokens_seen": 81879675, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01293945, + "step": 2924, + "time_per_iteration": 3.060215950012207 + }, + { + "auxiliary_loss_clip": 0.01012988, + "auxiliary_loss_mlp": 0.01000553, + "balance_loss_clip": 1.00116134, + "balance_loss_mlp": 0.99929559, + "epoch": 0.08487609540943648, + "flos": 53230779849600.0, + "grad_norm": 0.8228602170528517, + "language_loss": 0.49211764, + "learning_rate": 3.9684994116422855e-06, + "loss": 0.51225305, + "num_input_tokens_seen": 81928060, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.01257324, + "step": 2925, + "time_per_iteration": 2.76523494720459 + }, + { + "auxiliary_loss_clip": 0.01111554, + "auxiliary_loss_mlp": 0.01044555, + "balance_loss_clip": 1.03451037, + "balance_loss_mlp": 1.02011752, + "epoch": 0.08490511287795252, + "flos": 24380931588480.0, + "grad_norm": 2.97376781618042, + "language_loss": 0.83070105, + "learning_rate": 3.968466174354532e-06, + "loss": 0.8522622, + "num_input_tokens_seen": 81940890, + "router_z_loss_clip": 0.77050781, + "router_z_loss_mlp": 0.24475098, + "step": 2926, + "time_per_iteration": 2.514211654663086 + }, + { + "auxiliary_loss_clip": 0.01103634, + "auxiliary_loss_mlp": 0.01035004, + "balance_loss_clip": 1.0321753, + "balance_loss_mlp": 1.0162226, + "epoch": 0.08493413034646857, + "flos": 14827053703680.0, + "grad_norm": 4.730546122121636, + "language_loss": 0.86849737, + "learning_rate": 3.968432919680489e-06, + "loss": 0.88988376, + "num_input_tokens_seen": 81954135, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.1875, + "step": 2927, + "time_per_iteration": 2.3448734283447266 + }, + { + "auxiliary_loss_clip": 0.0101782, + "auxiliary_loss_mlp": 0.01004003, + "balance_loss_clip": 1.00559187, + "balance_loss_mlp": 1.00268555, + "epoch": 0.08496314781498462, + "flos": 74767372283520.0, + "grad_norm": 0.6912654813173056, + "language_loss": 0.5082013, + "learning_rate": 3.968399647620449e-06, + "loss": 0.52841949, + "num_input_tokens_seen": 82015780, + "router_z_loss_clip": 0.12207031, + "router_z_loss_mlp": 0.01318359, + "step": 2928, + "time_per_iteration": 3.0145275592803955 + }, + { + "auxiliary_loss_clip": 0.01017414, + "auxiliary_loss_mlp": 0.01003762, + "balance_loss_clip": 1.00548685, + "balance_loss_mlp": 1.00250411, + "epoch": 0.08499216528350066, + "flos": 63826011123840.0, + "grad_norm": 0.684987284963948, + "language_loss": 0.49348319, + "learning_rate": 3.9683663581747075e-06, + "loss": 0.513695, + "num_input_tokens_seen": 82075535, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01257324, + "step": 2929, + "time_per_iteration": 2.946953773498535 + }, + { + "auxiliary_loss_clip": 0.01113574, + "auxiliary_loss_mlp": 0.01046806, + "balance_loss_clip": 1.03405845, + "balance_loss_mlp": 1.02189147, + "epoch": 0.08502118275201671, + "flos": 22412214132480.0, + "grad_norm": 2.387957949338557, + "language_loss": 0.72368455, + "learning_rate": 3.968333051343557e-06, + "loss": 0.74528837, + "num_input_tokens_seen": 82091210, + "router_z_loss_clip": 0.79492188, + "router_z_loss_mlp": 0.24902344, + "step": 2930, + "time_per_iteration": 2.3096351623535156 + }, + { + "auxiliary_loss_clip": 0.01015368, + "auxiliary_loss_mlp": 0.01005393, + "balance_loss_clip": 1.00339198, + "balance_loss_mlp": 1.00417161, + "epoch": 0.08505020022053277, + "flos": 51960010849920.0, + "grad_norm": 0.7136080289672131, + "language_loss": 0.51634747, + "learning_rate": 3.968299727127292e-06, + "loss": 0.53655505, + "num_input_tokens_seen": 82140880, + "router_z_loss_clip": 0.12011719, + "router_z_loss_mlp": 0.01220703, + "step": 2931, + "time_per_iteration": 2.7243869304656982 + }, + { + "auxiliary_loss_clip": 0.01106436, + "auxiliary_loss_mlp": 0.01041202, + "balance_loss_clip": 1.03266335, + "balance_loss_mlp": 1.01964927, + "epoch": 0.0850792176890488, + "flos": 16391616727680.0, + "grad_norm": 3.2892949454419114, + "language_loss": 0.69772118, + "learning_rate": 3.968266385526209e-06, + "loss": 0.71919757, + "num_input_tokens_seen": 82154165, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.2154541, + "step": 2932, + "time_per_iteration": 2.3072221279144287 + }, + { + "auxiliary_loss_clip": 0.0111742, + "auxiliary_loss_mlp": 0.01042561, + "balance_loss_clip": 1.03659511, + "balance_loss_mlp": 1.02062607, + "epoch": 0.08510823515756485, + "flos": 25807901477760.0, + "grad_norm": 2.114974607812209, + "language_loss": 0.89607137, + "learning_rate": 3.9682330265406e-06, + "loss": 0.9176712, + "num_input_tokens_seen": 82170685, + "router_z_loss_clip": 0.80908203, + "router_z_loss_mlp": 0.21936035, + "step": 2933, + "time_per_iteration": 2.5257413387298584 + }, + { + "auxiliary_loss_clip": 0.01015042, + "auxiliary_loss_mlp": 0.01001452, + "balance_loss_clip": 1.00306475, + "balance_loss_mlp": 1.00017679, + "epoch": 0.0851372526260809, + "flos": 74774773491840.0, + "grad_norm": 0.7818823174808663, + "language_loss": 0.48349112, + "learning_rate": 3.96819965017076e-06, + "loss": 0.50365609, + "num_input_tokens_seen": 82233585, + "router_z_loss_clip": 0.11962891, + "router_z_loss_mlp": 0.01275635, + "step": 2934, + "time_per_iteration": 3.049414873123169 + }, + { + "auxiliary_loss_clip": 0.01110527, + "auxiliary_loss_mlp": 0.01043853, + "balance_loss_clip": 1.03612566, + "balance_loss_mlp": 1.02179909, + "epoch": 0.08516627009459694, + "flos": 25074411390720.0, + "grad_norm": 3.3870127326709154, + "language_loss": 0.83729267, + "learning_rate": 3.968166256416985e-06, + "loss": 0.85883653, + "num_input_tokens_seen": 82246235, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.22058105, + "step": 2935, + "time_per_iteration": 2.489893913269043 + }, + { + "auxiliary_loss_clip": 0.01113427, + "auxiliary_loss_mlp": 0.01047539, + "balance_loss_clip": 1.03310633, + "balance_loss_mlp": 1.02372134, + "epoch": 0.085195287563113, + "flos": 21827803017600.0, + "grad_norm": 2.940528619292482, + "language_loss": 0.93828344, + "learning_rate": 3.968132845279569e-06, + "loss": 0.95989311, + "num_input_tokens_seen": 82259175, + "router_z_loss_clip": 0.80273438, + "router_z_loss_mlp": 0.23840332, + "step": 2936, + "time_per_iteration": 2.3490095138549805 + }, + { + "auxiliary_loss_clip": 0.01100873, + "auxiliary_loss_mlp": 0.01040406, + "balance_loss_clip": 1.03329301, + "balance_loss_mlp": 1.02153492, + "epoch": 0.08522430503162905, + "flos": 19163476166400.0, + "grad_norm": 2.8614446953289123, + "language_loss": 0.82202256, + "learning_rate": 3.968099416758807e-06, + "loss": 0.84343541, + "num_input_tokens_seen": 82272095, + "router_z_loss_clip": 0.67626953, + "router_z_loss_mlp": 0.18884277, + "step": 2937, + "time_per_iteration": 2.4223389625549316 + }, + { + "auxiliary_loss_clip": 0.01112822, + "auxiliary_loss_mlp": 0.01047442, + "balance_loss_clip": 1.03343511, + "balance_loss_mlp": 1.02565098, + "epoch": 0.08525332250014508, + "flos": 20550854707200.0, + "grad_norm": 2.1450510926958475, + "language_loss": 0.8159734, + "learning_rate": 3.968065970854994e-06, + "loss": 0.83757603, + "num_input_tokens_seen": 82290455, + "router_z_loss_clip": 0.79443359, + "router_z_loss_mlp": 0.21801758, + "step": 2938, + "time_per_iteration": 2.487910509109497 + }, + { + "auxiliary_loss_clip": 0.01116391, + "auxiliary_loss_mlp": 0.01055186, + "balance_loss_clip": 1.03479886, + "balance_loss_mlp": 1.03078389, + "epoch": 0.08528233996866114, + "flos": 23286299731200.0, + "grad_norm": 2.2629571763635563, + "language_loss": 0.80549169, + "learning_rate": 3.968032507568427e-06, + "loss": 0.82720745, + "num_input_tokens_seen": 82304355, + "router_z_loss_clip": 0.81494141, + "router_z_loss_mlp": 0.24414062, + "step": 2939, + "time_per_iteration": 2.538604259490967 + }, + { + "auxiliary_loss_clip": 0.01114592, + "auxiliary_loss_mlp": 0.01051567, + "balance_loss_clip": 1.03477955, + "balance_loss_mlp": 1.02820158, + "epoch": 0.08531135743717717, + "flos": 24307718734080.0, + "grad_norm": 3.9465532862807624, + "language_loss": 0.9028483, + "learning_rate": 3.9679990268994e-06, + "loss": 0.92450988, + "num_input_tokens_seen": 82323635, + "router_z_loss_clip": 0.79833984, + "router_z_loss_mlp": 0.23376465, + "step": 2940, + "time_per_iteration": 2.524836301803589 + }, + { + "auxiliary_loss_clip": 0.01111803, + "auxiliary_loss_mlp": 0.01044128, + "balance_loss_clip": 1.03343177, + "balance_loss_mlp": 1.02113295, + "epoch": 0.08534037490569323, + "flos": 21866347025280.0, + "grad_norm": 2.569143359897354, + "language_loss": 0.88301659, + "learning_rate": 3.96796552884821e-06, + "loss": 0.90457588, + "num_input_tokens_seen": 82337660, + "router_z_loss_clip": 0.78320312, + "router_z_loss_mlp": 0.22998047, + "step": 2941, + "time_per_iteration": 2.4660282135009766 + }, + { + "auxiliary_loss_clip": 0.01105034, + "auxiliary_loss_mlp": 0.01040985, + "balance_loss_clip": 1.03231299, + "balance_loss_mlp": 1.01969433, + "epoch": 0.08536939237420928, + "flos": 25840650199680.0, + "grad_norm": 2.2352895385037423, + "language_loss": 0.6927402, + "learning_rate": 3.967932013415151e-06, + "loss": 0.71420044, + "num_input_tokens_seen": 82351460, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.21313477, + "step": 2942, + "time_per_iteration": 2.437232255935669 + }, + { + "auxiliary_loss_clip": 0.01017344, + "auxiliary_loss_mlp": 0.01004947, + "balance_loss_clip": 1.00498986, + "balance_loss_mlp": 1.00361145, + "epoch": 0.08539840984272531, + "flos": 65435122909440.0, + "grad_norm": 0.7476421021109306, + "language_loss": 0.52725154, + "learning_rate": 3.967898480600521e-06, + "loss": 0.54747444, + "num_input_tokens_seen": 82412940, + "router_z_loss_clip": 0.12353516, + "router_z_loss_mlp": 0.0133667, + "step": 2943, + "time_per_iteration": 2.9965317249298096 + }, + { + "auxiliary_loss_clip": 0.01113338, + "auxiliary_loss_mlp": 0.01049274, + "balance_loss_clip": 1.03363621, + "balance_loss_mlp": 1.02624321, + "epoch": 0.08542742731124137, + "flos": 32335088843520.0, + "grad_norm": 2.0164058966497405, + "language_loss": 0.9719736, + "learning_rate": 3.967864930404615e-06, + "loss": 0.99359965, + "num_input_tokens_seen": 82432090, + "router_z_loss_clip": 0.79736328, + "router_z_loss_mlp": 0.23034668, + "step": 2944, + "time_per_iteration": 2.468201160430908 + }, + { + "auxiliary_loss_clip": 0.01104157, + "auxiliary_loss_mlp": 0.010402, + "balance_loss_clip": 1.03030539, + "balance_loss_mlp": 1.01771712, + "epoch": 0.08545644477975742, + "flos": 14311631168640.0, + "grad_norm": 2.011231028664456, + "language_loss": 0.86522907, + "learning_rate": 3.9678313628277295e-06, + "loss": 0.88667268, + "num_input_tokens_seen": 82445605, + "router_z_loss_clip": 0.73925781, + "router_z_loss_mlp": 0.22473145, + "step": 2945, + "time_per_iteration": 2.44335675239563 + }, + { + "auxiliary_loss_clip": 0.01103604, + "auxiliary_loss_mlp": 0.01038923, + "balance_loss_clip": 1.03199291, + "balance_loss_mlp": 1.01931357, + "epoch": 0.08548546224827346, + "flos": 25769182913280.0, + "grad_norm": 3.0941231272641283, + "language_loss": 0.90607816, + "learning_rate": 3.967797777870161e-06, + "loss": 0.92750347, + "num_input_tokens_seen": 82462585, + "router_z_loss_clip": 0.71582031, + "router_z_loss_mlp": 0.19604492, + "step": 2946, + "time_per_iteration": 2.390655040740967 + }, + { + "auxiliary_loss_clip": 0.01125371, + "auxiliary_loss_mlp": 0.01052952, + "balance_loss_clip": 1.03721499, + "balance_loss_mlp": 1.02472281, + "epoch": 0.08551447971678951, + "flos": 13148708958720.0, + "grad_norm": 4.849839876328743, + "language_loss": 0.95895576, + "learning_rate": 3.967764175532207e-06, + "loss": 0.98073888, + "num_input_tokens_seen": 82475850, + "router_z_loss_clip": 0.88232422, + "router_z_loss_mlp": 0.28234863, + "step": 2947, + "time_per_iteration": 2.4649784564971924 + }, + { + "auxiliary_loss_clip": 0.01013888, + "auxiliary_loss_mlp": 0.0100468, + "balance_loss_clip": 1.00209165, + "balance_loss_mlp": 1.00338614, + "epoch": 0.08554349718530556, + "flos": 54448130736000.0, + "grad_norm": 0.6876543797443572, + "language_loss": 0.44303185, + "learning_rate": 3.967730555814162e-06, + "loss": 0.4632175, + "num_input_tokens_seen": 82538950, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.01293945, + "step": 2948, + "time_per_iteration": 3.199681043624878 + }, + { + "auxiliary_loss_clip": 0.01104175, + "auxiliary_loss_mlp": 0.01044294, + "balance_loss_clip": 1.03410268, + "balance_loss_mlp": 1.02320635, + "epoch": 0.0855725146538216, + "flos": 13980689591040.0, + "grad_norm": 2.809275767944441, + "language_loss": 1.00346947, + "learning_rate": 3.967696918716326e-06, + "loss": 1.02495432, + "num_input_tokens_seen": 82549720, + "router_z_loss_clip": 0.70166016, + "router_z_loss_mlp": 0.2109375, + "step": 2949, + "time_per_iteration": 2.4011051654815674 + }, + { + "auxiliary_loss_clip": 0.01106632, + "auxiliary_loss_mlp": 0.01047653, + "balance_loss_clip": 1.03477883, + "balance_loss_mlp": 1.02549219, + "epoch": 0.08560153212233765, + "flos": 16324338804480.0, + "grad_norm": 2.791789572466702, + "language_loss": 0.75691068, + "learning_rate": 3.967663264238994e-06, + "loss": 0.77845353, + "num_input_tokens_seen": 82564730, + "router_z_loss_clip": 0.71923828, + "router_z_loss_mlp": 0.22167969, + "step": 2950, + "time_per_iteration": 2.3301281929016113 + }, + { + "auxiliary_loss_clip": 0.01115346, + "auxiliary_loss_mlp": 0.01050811, + "balance_loss_clip": 1.03608179, + "balance_loss_mlp": 1.02829242, + "epoch": 0.0856305495908537, + "flos": 29671669687680.0, + "grad_norm": 2.2914190412280138, + "language_loss": 0.92569703, + "learning_rate": 3.967629592382463e-06, + "loss": 0.94735861, + "num_input_tokens_seen": 82582765, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.22521973, + "step": 2951, + "time_per_iteration": 2.4695117473602295 + }, + { + "auxiliary_loss_clip": 0.01111327, + "auxiliary_loss_mlp": 0.01043263, + "balance_loss_clip": 1.03583241, + "balance_loss_mlp": 1.02047002, + "epoch": 0.08565956705936974, + "flos": 74731588139520.0, + "grad_norm": 2.250498375676529, + "language_loss": 0.76033866, + "learning_rate": 3.967595903147033e-06, + "loss": 0.78188455, + "num_input_tokens_seen": 82604750, + "router_z_loss_clip": 0.75488281, + "router_z_loss_mlp": 0.22814941, + "step": 2952, + "time_per_iteration": 2.7818150520324707 + }, + { + "auxiliary_loss_clip": 0.01112512, + "auxiliary_loss_mlp": 0.01050329, + "balance_loss_clip": 1.03377533, + "balance_loss_mlp": 1.02763128, + "epoch": 0.08568858452788579, + "flos": 51125972313600.0, + "grad_norm": 1.9822064921845373, + "language_loss": 0.6977582, + "learning_rate": 3.9675621965329985e-06, + "loss": 0.71938664, + "num_input_tokens_seen": 82623855, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.22680664, + "step": 2953, + "time_per_iteration": 2.5373446941375732 + }, + { + "auxiliary_loss_clip": 0.01113925, + "auxiliary_loss_mlp": 0.0104059, + "balance_loss_clip": 1.03399789, + "balance_loss_mlp": 1.01799989, + "epoch": 0.08571760199640184, + "flos": 11616615365760.0, + "grad_norm": 2.8553397605793673, + "language_loss": 0.90933913, + "learning_rate": 3.967528472540658e-06, + "loss": 0.93088436, + "num_input_tokens_seen": 82636850, + "router_z_loss_clip": 0.80029297, + "router_z_loss_mlp": 0.22595215, + "step": 2954, + "time_per_iteration": 2.3122098445892334 + }, + { + "auxiliary_loss_clip": 0.01125008, + "auxiliary_loss_mlp": 0.01059345, + "balance_loss_clip": 1.03882432, + "balance_loss_mlp": 1.03296363, + "epoch": 0.08574661946491788, + "flos": 15882582971520.0, + "grad_norm": 3.197594840585645, + "language_loss": 0.79777557, + "learning_rate": 3.967494731170311e-06, + "loss": 0.81961906, + "num_input_tokens_seen": 82651110, + "router_z_loss_clip": 0.86279297, + "router_z_loss_mlp": 0.26391602, + "step": 2955, + "time_per_iteration": 2.3956217765808105 + }, + { + "auxiliary_loss_clip": 0.01112153, + "auxiliary_loss_mlp": 0.01051761, + "balance_loss_clip": 1.03065681, + "balance_loss_mlp": 1.02660751, + "epoch": 0.08577563693343393, + "flos": 27045118794240.0, + "grad_norm": 2.2121821443659955, + "language_loss": 0.94759655, + "learning_rate": 3.967460972422254e-06, + "loss": 0.96923578, + "num_input_tokens_seen": 82666475, + "router_z_loss_clip": 0.81542969, + "router_z_loss_mlp": 0.25170898, + "step": 2956, + "time_per_iteration": 2.437793731689453 + }, + { + "auxiliary_loss_clip": 0.01014432, + "auxiliary_loss_mlp": 0.01001482, + "balance_loss_clip": 1.00291002, + "balance_loss_mlp": 1.00014687, + "epoch": 0.08580465440194997, + "flos": 62472323911680.0, + "grad_norm": 0.6618889101891466, + "language_loss": 0.49627829, + "learning_rate": 3.967427196296785e-06, + "loss": 0.51643741, + "num_input_tokens_seen": 82725945, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.0133667, + "step": 2957, + "time_per_iteration": 2.944389820098877 + }, + { + "auxiliary_loss_clip": 0.01107098, + "auxiliary_loss_mlp": 0.01043682, + "balance_loss_clip": 1.03101563, + "balance_loss_mlp": 1.02314198, + "epoch": 0.08583367187046602, + "flos": 44265155195520.0, + "grad_norm": 2.023751574568256, + "language_loss": 0.69732952, + "learning_rate": 3.967393402794204e-06, + "loss": 0.71883726, + "num_input_tokens_seen": 82744660, + "router_z_loss_clip": 0.76025391, + "router_z_loss_mlp": 0.20556641, + "step": 2958, + "time_per_iteration": 2.4918835163116455 + }, + { + "auxiliary_loss_clip": 0.01012827, + "auxiliary_loss_mlp": 0.01001346, + "balance_loss_clip": 1.00126421, + "balance_loss_mlp": 0.99998707, + "epoch": 0.08586268933898207, + "flos": 63339985820160.0, + "grad_norm": 0.681615525950424, + "language_loss": 0.47445524, + "learning_rate": 3.967359591914807e-06, + "loss": 0.49459699, + "num_input_tokens_seen": 82799420, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01361084, + "step": 2959, + "time_per_iteration": 2.885444164276123 + }, + { + "auxiliary_loss_clip": 0.0110474, + "auxiliary_loss_mlp": 0.01046008, + "balance_loss_clip": 1.03226256, + "balance_loss_mlp": 1.02242827, + "epoch": 0.08589170680749811, + "flos": 21975729914880.0, + "grad_norm": 2.3516160613426482, + "language_loss": 0.76824886, + "learning_rate": 3.9673257636588956e-06, + "loss": 0.78975642, + "num_input_tokens_seen": 82815605, + "router_z_loss_clip": 0.72460938, + "router_z_loss_mlp": 0.23608398, + "step": 2960, + "time_per_iteration": 2.405132532119751 + }, + { + "auxiliary_loss_clip": 0.0111698, + "auxiliary_loss_mlp": 0.01048779, + "balance_loss_clip": 1.03413105, + "balance_loss_mlp": 1.02533102, + "epoch": 0.08592072427601416, + "flos": 20953996709760.0, + "grad_norm": 2.258745021085984, + "language_loss": 0.84586, + "learning_rate": 3.967291918026766e-06, + "loss": 0.86751765, + "num_input_tokens_seen": 82832975, + "router_z_loss_clip": 0.82763672, + "router_z_loss_mlp": 0.23449707, + "step": 2961, + "time_per_iteration": 2.4651167392730713 + }, + { + "auxiliary_loss_clip": 0.01109229, + "auxiliary_loss_mlp": 0.01041911, + "balance_loss_clip": 1.03413033, + "balance_loss_mlp": 1.02062011, + "epoch": 0.08594974174453021, + "flos": 35155895875200.0, + "grad_norm": 1.9200575312046582, + "language_loss": 0.89264941, + "learning_rate": 3.967258055018719e-06, + "loss": 0.91416085, + "num_input_tokens_seen": 82855390, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.21289062, + "step": 2962, + "time_per_iteration": 2.527693510055542 + }, + { + "auxiliary_loss_clip": 0.01113898, + "auxiliary_loss_mlp": 0.01048081, + "balance_loss_clip": 1.03344274, + "balance_loss_mlp": 1.02488327, + "epoch": 0.08597875921304625, + "flos": 29971154263680.0, + "grad_norm": 1.8327821861916551, + "language_loss": 0.92896593, + "learning_rate": 3.967224174635052e-06, + "loss": 0.95058578, + "num_input_tokens_seen": 82874040, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.23181152, + "step": 2963, + "time_per_iteration": 2.4572227001190186 + }, + { + "auxiliary_loss_clip": 0.01107871, + "auxiliary_loss_mlp": 0.01040534, + "balance_loss_clip": 1.0351721, + "balance_loss_mlp": 1.02076948, + "epoch": 0.0860077766815623, + "flos": 26059834915200.0, + "grad_norm": 2.3691767167198794, + "language_loss": 0.67886972, + "learning_rate": 3.967190276876065e-06, + "loss": 0.70035386, + "num_input_tokens_seen": 82888640, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.19787598, + "step": 2964, + "time_per_iteration": 2.420105218887329 + }, + { + "auxiliary_loss_clip": 0.01117199, + "auxiliary_loss_mlp": 0.01043947, + "balance_loss_clip": 1.03456903, + "balance_loss_mlp": 1.01776838, + "epoch": 0.08603679415007835, + "flos": 16755935431680.0, + "grad_norm": 3.103810650830711, + "language_loss": 0.89782333, + "learning_rate": 3.967156361742057e-06, + "loss": 0.91943479, + "num_input_tokens_seen": 82901695, + "router_z_loss_clip": 0.82617188, + "router_z_loss_mlp": 0.26184082, + "step": 2965, + "time_per_iteration": 2.367971897125244 + }, + { + "auxiliary_loss_clip": 0.01111817, + "auxiliary_loss_mlp": 0.01051904, + "balance_loss_clip": 1.03440142, + "balance_loss_mlp": 1.02683461, + "epoch": 0.08606581161859439, + "flos": 17052487453440.0, + "grad_norm": 2.1974952526388916, + "language_loss": 0.71181703, + "learning_rate": 3.967122429233328e-06, + "loss": 0.73345423, + "num_input_tokens_seen": 82913445, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.25073242, + "step": 2966, + "time_per_iteration": 2.3227779865264893 + }, + { + "auxiliary_loss_clip": 0.0101421, + "auxiliary_loss_mlp": 0.01002876, + "balance_loss_clip": 1.002702, + "balance_loss_mlp": 1.00165462, + "epoch": 0.08609482908711044, + "flos": 62479271272320.0, + "grad_norm": 0.6804613089448655, + "language_loss": 0.49073815, + "learning_rate": 3.967088479350179e-06, + "loss": 0.51090896, + "num_input_tokens_seen": 82974995, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01220703, + "step": 2967, + "time_per_iteration": 2.9942915439605713 + }, + { + "auxiliary_loss_clip": 0.01013243, + "auxiliary_loss_mlp": 0.01002419, + "balance_loss_clip": 1.00170934, + "balance_loss_mlp": 1.00128675, + "epoch": 0.0861238465556265, + "flos": 69372802200960.0, + "grad_norm": 0.6331890730917915, + "language_loss": 0.5093081, + "learning_rate": 3.9670545120929075e-06, + "loss": 0.52946472, + "num_input_tokens_seen": 83040845, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01135254, + "step": 2968, + "time_per_iteration": 3.0802180767059326 + }, + { + "auxiliary_loss_clip": 0.01106839, + "auxiliary_loss_mlp": 0.01041244, + "balance_loss_clip": 1.03187048, + "balance_loss_mlp": 1.01846337, + "epoch": 0.08615286402414253, + "flos": 23505938294400.0, + "grad_norm": 2.895474893389994, + "language_loss": 0.95540625, + "learning_rate": 3.967020527461815e-06, + "loss": 0.97688711, + "num_input_tokens_seen": 83055765, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.22802734, + "step": 2969, + "time_per_iteration": 2.4065868854522705 + }, + { + "auxiliary_loss_clip": 0.01109115, + "auxiliary_loss_mlp": 0.01042212, + "balance_loss_clip": 1.03338981, + "balance_loss_mlp": 1.02139187, + "epoch": 0.08618188149265858, + "flos": 20550854707200.0, + "grad_norm": 3.960861917839127, + "language_loss": 1.04905629, + "learning_rate": 3.966986525457201e-06, + "loss": 1.07056952, + "num_input_tokens_seen": 83068405, + "router_z_loss_clip": 0.75683594, + "router_z_loss_mlp": 0.20837402, + "step": 2970, + "time_per_iteration": 4.564685106277466 + }, + { + "auxiliary_loss_clip": 0.01112202, + "auxiliary_loss_mlp": 0.01050259, + "balance_loss_clip": 1.03508103, + "balance_loss_mlp": 1.02810407, + "epoch": 0.08621089896117463, + "flos": 44046459239040.0, + "grad_norm": 2.209502647997082, + "language_loss": 0.87805796, + "learning_rate": 3.966952506079366e-06, + "loss": 0.89968264, + "num_input_tokens_seen": 83086835, + "router_z_loss_clip": 0.77099609, + "router_z_loss_mlp": 0.22137451, + "step": 2971, + "time_per_iteration": 2.6181581020355225 + }, + { + "auxiliary_loss_clip": 0.01107602, + "auxiliary_loss_mlp": 0.01049391, + "balance_loss_clip": 1.03229463, + "balance_loss_mlp": 1.02671158, + "epoch": 0.08623991642969067, + "flos": 29242621589760.0, + "grad_norm": 2.445095290251979, + "language_loss": 0.90445822, + "learning_rate": 3.96691846932861e-06, + "loss": 0.92602813, + "num_input_tokens_seen": 83103805, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.22662354, + "step": 2972, + "time_per_iteration": 2.467780590057373 + }, + { + "auxiliary_loss_clip": 0.01106756, + "auxiliary_loss_mlp": 0.01047041, + "balance_loss_clip": 1.0327127, + "balance_loss_mlp": 1.02482057, + "epoch": 0.08626893389820672, + "flos": 37114105011840.0, + "grad_norm": 2.968770014347855, + "language_loss": 0.94931036, + "learning_rate": 3.966884415205234e-06, + "loss": 0.97084832, + "num_input_tokens_seen": 83119235, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.22241211, + "step": 2973, + "time_per_iteration": 2.586610794067383 + }, + { + "auxiliary_loss_clip": 0.01012635, + "auxiliary_loss_mlp": 0.01001049, + "balance_loss_clip": 1.00127888, + "balance_loss_mlp": 0.99999368, + "epoch": 0.08629795136672276, + "flos": 68531674792320.0, + "grad_norm": 0.661292175088796, + "language_loss": 0.46914411, + "learning_rate": 3.966850343709541e-06, + "loss": 0.48928097, + "num_input_tokens_seen": 83183645, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01055908, + "step": 2974, + "time_per_iteration": 5.436224699020386 + }, + { + "auxiliary_loss_clip": 0.01121612, + "auxiliary_loss_mlp": 0.0105381, + "balance_loss_clip": 1.03551602, + "balance_loss_mlp": 1.02987885, + "epoch": 0.08632696883523881, + "flos": 43463968248960.0, + "grad_norm": 3.0895836939209262, + "language_loss": 0.93457401, + "learning_rate": 3.966816254841828e-06, + "loss": 0.95632827, + "num_input_tokens_seen": 83200595, + "router_z_loss_clip": 0.86083984, + "router_z_loss_mlp": 0.23950195, + "step": 2975, + "time_per_iteration": 2.580528497695923 + }, + { + "auxiliary_loss_clip": 0.01111536, + "auxiliary_loss_mlp": 0.01052765, + "balance_loss_clip": 1.03338861, + "balance_loss_mlp": 1.02960229, + "epoch": 0.08635598630375486, + "flos": 12974910877440.0, + "grad_norm": 3.0615051730331997, + "language_loss": 0.99915922, + "learning_rate": 3.966782148602399e-06, + "loss": 1.02080226, + "num_input_tokens_seen": 83213930, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.23144531, + "step": 2976, + "time_per_iteration": 4.801189184188843 + }, + { + "auxiliary_loss_clip": 0.01106517, + "auxiliary_loss_mlp": 0.01048032, + "balance_loss_clip": 1.03272283, + "balance_loss_mlp": 1.02734935, + "epoch": 0.0863850037722709, + "flos": 36238238933760.0, + "grad_norm": 2.3358466795057717, + "language_loss": 0.88808191, + "learning_rate": 3.966748024991553e-06, + "loss": 0.90962732, + "num_input_tokens_seen": 83229760, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.20690918, + "step": 2977, + "time_per_iteration": 2.563852548599243 + }, + { + "auxiliary_loss_clip": 0.01013024, + "auxiliary_loss_mlp": 0.01001609, + "balance_loss_clip": 1.00160384, + "balance_loss_mlp": 1.00056612, + "epoch": 0.08641402124078695, + "flos": 58791011800320.0, + "grad_norm": 0.6562090247625729, + "language_loss": 0.49852568, + "learning_rate": 3.966713884009594e-06, + "loss": 0.51867199, + "num_input_tokens_seen": 83292035, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01043701, + "step": 2978, + "time_per_iteration": 2.9438860416412354 + }, + { + "auxiliary_loss_clip": 0.01013676, + "auxiliary_loss_mlp": 0.01001249, + "balance_loss_clip": 1.00221801, + "balance_loss_mlp": 1.00008631, + "epoch": 0.086443038709303, + "flos": 74765661626880.0, + "grad_norm": 0.706234286505572, + "language_loss": 0.54110283, + "learning_rate": 3.96667972565682e-06, + "loss": 0.56125212, + "num_input_tokens_seen": 83345420, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01159668, + "step": 2979, + "time_per_iteration": 2.9924683570861816 + }, + { + "auxiliary_loss_clip": 0.01108636, + "auxiliary_loss_mlp": 0.01042616, + "balance_loss_clip": 1.03518271, + "balance_loss_mlp": 1.02202857, + "epoch": 0.08647205617781904, + "flos": 19021554023040.0, + "grad_norm": 2.0146292813133635, + "language_loss": 0.72338951, + "learning_rate": 3.966645549933537e-06, + "loss": 0.74490201, + "num_input_tokens_seen": 83365005, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.20617676, + "step": 2980, + "time_per_iteration": 2.497459888458252 + }, + { + "auxiliary_loss_clip": 0.0111393, + "auxiliary_loss_mlp": 0.01048377, + "balance_loss_clip": 1.03713119, + "balance_loss_mlp": 1.02524996, + "epoch": 0.0865010736463351, + "flos": 12085219900800.0, + "grad_norm": 4.169606894920878, + "language_loss": 1.17981791, + "learning_rate": 3.966611356840044e-06, + "loss": 1.20144093, + "num_input_tokens_seen": 83376525, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.23120117, + "step": 2981, + "time_per_iteration": 2.4098923206329346 + }, + { + "auxiliary_loss_clip": 0.01113351, + "auxiliary_loss_mlp": 0.01042936, + "balance_loss_clip": 1.03545165, + "balance_loss_mlp": 1.02218151, + "epoch": 0.08653009111485115, + "flos": 11392752528000.0, + "grad_norm": 2.6690212417144643, + "language_loss": 0.88974267, + "learning_rate": 3.966577146376644e-06, + "loss": 0.91130543, + "num_input_tokens_seen": 83387550, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.20751953, + "step": 2982, + "time_per_iteration": 2.3118958473205566 + }, + { + "auxiliary_loss_clip": 0.01110115, + "auxiliary_loss_mlp": 0.01045839, + "balance_loss_clip": 1.0325948, + "balance_loss_mlp": 1.02438188, + "epoch": 0.08655910858336718, + "flos": 15542632396800.0, + "grad_norm": 2.1436722217830115, + "language_loss": 0.80550605, + "learning_rate": 3.966542918543638e-06, + "loss": 0.82706559, + "num_input_tokens_seen": 83404265, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.21459961, + "step": 2983, + "time_per_iteration": 2.513519525527954 + }, + { + "auxiliary_loss_clip": 0.01120644, + "auxiliary_loss_mlp": 0.01053027, + "balance_loss_clip": 1.03745568, + "balance_loss_mlp": 1.02946508, + "epoch": 0.08658812605188324, + "flos": 70134469488000.0, + "grad_norm": 2.21093398138965, + "language_loss": 0.94074512, + "learning_rate": 3.966508673341329e-06, + "loss": 0.96248186, + "num_input_tokens_seen": 83430270, + "router_z_loss_clip": 0.83203125, + "router_z_loss_mlp": 0.23571777, + "step": 2984, + "time_per_iteration": 2.733515501022339 + }, + { + "auxiliary_loss_clip": 0.01105137, + "auxiliary_loss_mlp": 0.01044328, + "balance_loss_clip": 1.03375185, + "balance_loss_mlp": 1.02324009, + "epoch": 0.08661714352039929, + "flos": 12381143518080.0, + "grad_norm": 3.325341346864267, + "language_loss": 0.78009588, + "learning_rate": 3.966474410770021e-06, + "loss": 0.80159056, + "num_input_tokens_seen": 83444340, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.21081543, + "step": 2985, + "time_per_iteration": 2.38578462600708 + }, + { + "auxiliary_loss_clip": 0.01109222, + "auxiliary_loss_mlp": 0.01044801, + "balance_loss_clip": 1.03446603, + "balance_loss_mlp": 1.0235337, + "epoch": 0.08664616098891532, + "flos": 17048821760640.0, + "grad_norm": 2.4438438901725243, + "language_loss": 0.89421749, + "learning_rate": 3.966440130830015e-06, + "loss": 0.91575772, + "num_input_tokens_seen": 83457715, + "router_z_loss_clip": 0.74707031, + "router_z_loss_mlp": 0.21276855, + "step": 2986, + "time_per_iteration": 2.3220064640045166 + }, + { + "auxiliary_loss_clip": 0.01113891, + "auxiliary_loss_mlp": 0.01050241, + "balance_loss_clip": 1.03427744, + "balance_loss_mlp": 1.02579129, + "epoch": 0.08667517845743138, + "flos": 28869435400320.0, + "grad_norm": 2.1728199634899172, + "language_loss": 0.97777903, + "learning_rate": 3.966405833521613e-06, + "loss": 0.99942029, + "num_input_tokens_seen": 83475890, + "router_z_loss_clip": 0.79589844, + "router_z_loss_mlp": 0.24432373, + "step": 2987, + "time_per_iteration": 2.497601270675659 + }, + { + "auxiliary_loss_clip": 0.01109988, + "auxiliary_loss_mlp": 0.0103708, + "balance_loss_clip": 1.03358102, + "balance_loss_mlp": 1.01566422, + "epoch": 0.08670419592594741, + "flos": 30292041369600.0, + "grad_norm": 10.87111011220729, + "language_loss": 1.13091636, + "learning_rate": 3.9663715188451196e-06, + "loss": 1.1523869, + "num_input_tokens_seen": 83495060, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.21411133, + "step": 2988, + "time_per_iteration": 2.4747819900512695 + }, + { + "auxiliary_loss_clip": 0.0111659, + "auxiliary_loss_mlp": 0.01053756, + "balance_loss_clip": 1.03955507, + "balance_loss_mlp": 1.03223896, + "epoch": 0.08673321339446347, + "flos": 31900105814400.0, + "grad_norm": 2.2409677283014955, + "language_loss": 1.09048319, + "learning_rate": 3.966337186800837e-06, + "loss": 1.11218667, + "num_input_tokens_seen": 83510665, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.21520996, + "step": 2989, + "time_per_iteration": 2.490459442138672 + }, + { + "auxiliary_loss_clip": 0.01014922, + "auxiliary_loss_mlp": 0.01013512, + "balance_loss_clip": 1.00330043, + "balance_loss_mlp": 1.01254046, + "epoch": 0.08676223086297952, + "flos": 60063805658880.0, + "grad_norm": 0.725394760700777, + "language_loss": 0.53535044, + "learning_rate": 3.966302837389069e-06, + "loss": 0.55563474, + "num_input_tokens_seen": 83570370, + "router_z_loss_clip": 0.11621094, + "router_z_loss_mlp": 0.00970459, + "step": 2990, + "time_per_iteration": 2.9886984825134277 + }, + { + "auxiliary_loss_clip": 0.01106811, + "auxiliary_loss_mlp": 0.01049126, + "balance_loss_clip": 1.0350008, + "balance_loss_mlp": 1.02905142, + "epoch": 0.08679124833149555, + "flos": 20048035173120.0, + "grad_norm": 1.8169362125206237, + "language_loss": 0.68604267, + "learning_rate": 3.9662684706101185e-06, + "loss": 0.70760202, + "num_input_tokens_seen": 83585960, + "router_z_loss_clip": 0.71826172, + "router_z_loss_mlp": 0.20068359, + "step": 2991, + "time_per_iteration": 2.432452917098999 + }, + { + "auxiliary_loss_clip": 0.01111883, + "auxiliary_loss_mlp": 0.0105005, + "balance_loss_clip": 1.03486753, + "balance_loss_mlp": 1.0275079, + "epoch": 0.0868202658000116, + "flos": 26547466141440.0, + "grad_norm": 2.272973953831414, + "language_loss": 0.85042274, + "learning_rate": 3.966234086464289e-06, + "loss": 0.87204206, + "num_input_tokens_seen": 83602445, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.22509766, + "step": 2992, + "time_per_iteration": 2.4396979808807373 + }, + { + "auxiliary_loss_clip": 0.01114762, + "auxiliary_loss_mlp": 0.0105219, + "balance_loss_clip": 1.0335809, + "balance_loss_mlp": 1.02808595, + "epoch": 0.08684928326852766, + "flos": 35873431470720.0, + "grad_norm": 2.6397874534966284, + "language_loss": 0.98094308, + "learning_rate": 3.966199684951885e-06, + "loss": 1.00261259, + "num_input_tokens_seen": 83617205, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.2409668, + "step": 2993, + "time_per_iteration": 2.505547046661377 + }, + { + "auxiliary_loss_clip": 0.01013255, + "auxiliary_loss_mlp": 0.01001122, + "balance_loss_clip": 1.00155735, + "balance_loss_mlp": 1.00014448, + "epoch": 0.0868783007370437, + "flos": 74773795973760.0, + "grad_norm": 0.6305231384669662, + "language_loss": 0.53092879, + "learning_rate": 3.9661652660732085e-06, + "loss": 0.5510726, + "num_input_tokens_seen": 83685800, + "router_z_loss_clip": 0.11669922, + "router_z_loss_mlp": 0.00976562, + "step": 2994, + "time_per_iteration": 3.1039552688598633 + }, + { + "auxiliary_loss_clip": 0.01103516, + "auxiliary_loss_mlp": 0.0103877, + "balance_loss_clip": 1.03185701, + "balance_loss_mlp": 1.01696658, + "epoch": 0.08690731820555975, + "flos": 16435746552960.0, + "grad_norm": 2.3628022100737125, + "language_loss": 0.78052253, + "learning_rate": 3.966130829828566e-06, + "loss": 0.80194539, + "num_input_tokens_seen": 83701440, + "router_z_loss_clip": 0.71630859, + "router_z_loss_mlp": 0.21801758, + "step": 2995, + "time_per_iteration": 2.4932680130004883 + }, + { + "auxiliary_loss_clip": 0.01106286, + "auxiliary_loss_mlp": 0.01042876, + "balance_loss_clip": 1.03435659, + "balance_loss_mlp": 1.02417827, + "epoch": 0.0869363356740758, + "flos": 45074965248000.0, + "grad_norm": 1.9595238178630137, + "language_loss": 0.75271487, + "learning_rate": 3.9660963762182605e-06, + "loss": 0.77420652, + "num_input_tokens_seen": 83722085, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.18707275, + "step": 2996, + "time_per_iteration": 2.5906496047973633 + }, + { + "auxiliary_loss_clip": 0.01111336, + "auxiliary_loss_mlp": 0.01048725, + "balance_loss_clip": 1.03452516, + "balance_loss_mlp": 1.02512157, + "epoch": 0.08696535314259184, + "flos": 25914561281280.0, + "grad_norm": 3.0644376318077544, + "language_loss": 0.90058208, + "learning_rate": 3.9660619052425955e-06, + "loss": 0.92218268, + "num_input_tokens_seen": 83736680, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.23583984, + "step": 2997, + "time_per_iteration": 2.4263124465942383 + }, + { + "auxiliary_loss_clip": 0.01110521, + "auxiliary_loss_mlp": 0.010423, + "balance_loss_clip": 1.03220093, + "balance_loss_mlp": 1.01824331, + "epoch": 0.08699437061110789, + "flos": 40950256469760.0, + "grad_norm": 2.150555452025101, + "language_loss": 0.85497928, + "learning_rate": 3.966027416901876e-06, + "loss": 0.87650758, + "num_input_tokens_seen": 83754960, + "router_z_loss_clip": 0.78271484, + "router_z_loss_mlp": 0.24072266, + "step": 2998, + "time_per_iteration": 2.5299224853515625 + }, + { + "auxiliary_loss_clip": 0.01104335, + "auxiliary_loss_mlp": 0.01037051, + "balance_loss_clip": 1.0306989, + "balance_loss_mlp": 1.01735771, + "epoch": 0.08702338807962394, + "flos": 16757995201920.0, + "grad_norm": 1.906105820391166, + "language_loss": 0.66352761, + "learning_rate": 3.965992911196407e-06, + "loss": 0.68494147, + "num_input_tokens_seen": 83771945, + "router_z_loss_clip": 0.73486328, + "router_z_loss_mlp": 0.19689941, + "step": 2999, + "time_per_iteration": 2.3862948417663574 + }, + { + "auxiliary_loss_clip": 0.01105146, + "auxiliary_loss_mlp": 0.01044083, + "balance_loss_clip": 1.03286469, + "balance_loss_mlp": 1.0232929, + "epoch": 0.08705240554813998, + "flos": 18325211489280.0, + "grad_norm": 3.0982200080265905, + "language_loss": 0.69757986, + "learning_rate": 3.965958388126493e-06, + "loss": 0.7190721, + "num_input_tokens_seen": 83784165, + "router_z_loss_clip": 0.72265625, + "router_z_loss_mlp": 0.20776367, + "step": 3000, + "time_per_iteration": 2.3106720447540283 + }, + { + "auxiliary_loss_clip": 0.01108912, + "auxiliary_loss_mlp": 0.0104384, + "balance_loss_clip": 1.03157187, + "balance_loss_mlp": 1.01889002, + "epoch": 0.08708142301665603, + "flos": 17596050410880.0, + "grad_norm": 3.403044686547517, + "language_loss": 0.98085368, + "learning_rate": 3.9659238476924395e-06, + "loss": 1.00238109, + "num_input_tokens_seen": 83797305, + "router_z_loss_clip": 0.77294922, + "router_z_loss_mlp": 0.24951172, + "step": 3001, + "time_per_iteration": 2.359910011291504 + }, + { + "auxiliary_loss_clip": 0.01112228, + "auxiliary_loss_mlp": 0.01056722, + "balance_loss_clip": 1.03572154, + "balance_loss_mlp": 1.03496075, + "epoch": 0.08711044048517208, + "flos": 14676892479360.0, + "grad_norm": 2.492984335984422, + "language_loss": 0.79155231, + "learning_rate": 3.965889289894551e-06, + "loss": 0.81324172, + "num_input_tokens_seen": 83809840, + "router_z_loss_clip": 0.76464844, + "router_z_loss_mlp": 0.2177124, + "step": 3002, + "time_per_iteration": 2.4043984413146973 + }, + { + "auxiliary_loss_clip": 0.01014864, + "auxiliary_loss_mlp": 0.01007124, + "balance_loss_clip": 1.00327802, + "balance_loss_mlp": 1.00609922, + "epoch": 0.08713945795368812, + "flos": 63791097897600.0, + "grad_norm": 0.6725092414644457, + "language_loss": 0.46417052, + "learning_rate": 3.965854714733132e-06, + "loss": 0.48439038, + "num_input_tokens_seen": 83867970, + "router_z_loss_clip": 0.11572266, + "router_z_loss_mlp": 0.01025391, + "step": 3003, + "time_per_iteration": 2.929743528366089 + }, + { + "auxiliary_loss_clip": 0.01110246, + "auxiliary_loss_mlp": 0.01041787, + "balance_loss_clip": 1.03465426, + "balance_loss_mlp": 1.02128267, + "epoch": 0.08716847542220417, + "flos": 46819889688960.0, + "grad_norm": 2.3072953979075956, + "language_loss": 0.77671897, + "learning_rate": 3.96582012220849e-06, + "loss": 0.79823923, + "num_input_tokens_seen": 83886625, + "router_z_loss_clip": 0.75634766, + "router_z_loss_mlp": 0.20507812, + "step": 3004, + "time_per_iteration": 2.5086753368377686 + }, + { + "auxiliary_loss_clip": 0.01013793, + "auxiliary_loss_mlp": 0.01006187, + "balance_loss_clip": 1.00216722, + "balance_loss_mlp": 1.00516808, + "epoch": 0.08719749289072021, + "flos": 64190225093760.0, + "grad_norm": 0.6330944722126577, + "language_loss": 0.47983447, + "learning_rate": 3.965785512320928e-06, + "loss": 0.50003421, + "num_input_tokens_seen": 83947440, + "router_z_loss_clip": 0.11621094, + "router_z_loss_mlp": 0.01019287, + "step": 3005, + "time_per_iteration": 2.9950942993164062 + }, + { + "auxiliary_loss_clip": 0.01111223, + "auxiliary_loss_mlp": 0.01046081, + "balance_loss_clip": 1.03562164, + "balance_loss_mlp": 1.02457595, + "epoch": 0.08722651035923626, + "flos": 11355779531520.0, + "grad_norm": 3.8692999037015015, + "language_loss": 1.04486799, + "learning_rate": 3.965750885070753e-06, + "loss": 1.06644106, + "num_input_tokens_seen": 83956845, + "router_z_loss_clip": 0.75537109, + "router_z_loss_mlp": 0.21508789, + "step": 3006, + "time_per_iteration": 2.3696093559265137 + }, + { + "auxiliary_loss_clip": 0.01012306, + "auxiliary_loss_mlp": 0.01003431, + "balance_loss_clip": 1.00079441, + "balance_loss_mlp": 1.0025605, + "epoch": 0.08725552782775231, + "flos": 71479182614400.0, + "grad_norm": 0.6947694138434846, + "language_loss": 0.50475204, + "learning_rate": 3.965716240458271e-06, + "loss": 0.52490938, + "num_input_tokens_seen": 84013015, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.00872803, + "step": 3007, + "time_per_iteration": 2.9298524856567383 + }, + { + "auxiliary_loss_clip": 0.01012862, + "auxiliary_loss_mlp": 0.01001339, + "balance_loss_clip": 1.00139523, + "balance_loss_mlp": 1.0003916, + "epoch": 0.08728454529626835, + "flos": 67622955258240.0, + "grad_norm": 0.5990730870017765, + "language_loss": 0.41236809, + "learning_rate": 3.965681578483788e-06, + "loss": 0.43251011, + "num_input_tokens_seen": 84080210, + "router_z_loss_clip": 0.11474609, + "router_z_loss_mlp": 0.00946045, + "step": 3008, + "time_per_iteration": 3.160613775253296 + }, + { + "auxiliary_loss_clip": 0.01115007, + "auxiliary_loss_mlp": 0.01047505, + "balance_loss_clip": 1.03514433, + "balance_loss_mlp": 1.02679861, + "epoch": 0.0873135627647844, + "flos": 37892667530880.0, + "grad_norm": 1.785182053810844, + "language_loss": 0.9004513, + "learning_rate": 3.965646899147609e-06, + "loss": 0.92207646, + "num_input_tokens_seen": 84102595, + "router_z_loss_clip": 0.79833984, + "router_z_loss_mlp": 0.20690918, + "step": 3009, + "time_per_iteration": 2.5053086280822754 + }, + { + "auxiliary_loss_clip": 0.01113548, + "auxiliary_loss_mlp": 0.0104426, + "balance_loss_clip": 1.03167892, + "balance_loss_mlp": 1.02001286, + "epoch": 0.08734258023330045, + "flos": 30552667735680.0, + "grad_norm": 2.303544401441082, + "language_loss": 1.10063207, + "learning_rate": 3.965612202450042e-06, + "loss": 1.12221026, + "num_input_tokens_seen": 84123360, + "router_z_loss_clip": 0.81835938, + "router_z_loss_mlp": 0.24249268, + "step": 3010, + "time_per_iteration": 2.5700840950012207 + }, + { + "auxiliary_loss_clip": 0.01114658, + "auxiliary_loss_mlp": 0.01045642, + "balance_loss_clip": 1.03620672, + "balance_loss_mlp": 1.023422, + "epoch": 0.08737159770181649, + "flos": 20075791570560.0, + "grad_norm": 2.1434617956267843, + "language_loss": 0.81283379, + "learning_rate": 3.965577488391393e-06, + "loss": 0.83443683, + "num_input_tokens_seen": 84138665, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.22192383, + "step": 3011, + "time_per_iteration": 2.3802216053009033 + }, + { + "auxiliary_loss_clip": 0.01110826, + "auxiliary_loss_mlp": 0.01037145, + "balance_loss_clip": 1.03352809, + "balance_loss_mlp": 1.01607442, + "epoch": 0.08740061517033254, + "flos": 30474357822720.0, + "grad_norm": 2.7688127156250064, + "language_loss": 0.75870788, + "learning_rate": 3.965542756971967e-06, + "loss": 0.78018761, + "num_input_tokens_seen": 84155010, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.21075439, + "step": 3012, + "time_per_iteration": 2.4603817462921143 + }, + { + "auxiliary_loss_clip": 0.01015409, + "auxiliary_loss_mlp": 0.01004934, + "balance_loss_clip": 1.00412726, + "balance_loss_mlp": 1.00397992, + "epoch": 0.08742963263884859, + "flos": 64884123832320.0, + "grad_norm": 0.7679965054384889, + "language_loss": 0.51869571, + "learning_rate": 3.965508008192072e-06, + "loss": 0.53889912, + "num_input_tokens_seen": 84220900, + "router_z_loss_clip": 0.11279297, + "router_z_loss_mlp": 0.00952148, + "step": 3013, + "time_per_iteration": 3.0405869483947754 + }, + { + "auxiliary_loss_clip": 0.01104747, + "auxiliary_loss_mlp": 0.01032113, + "balance_loss_clip": 1.03279066, + "balance_loss_mlp": 1.01445818, + "epoch": 0.08745865010736463, + "flos": 12926068018560.0, + "grad_norm": 2.4503602580120334, + "language_loss": 0.69769329, + "learning_rate": 3.965473242052016e-06, + "loss": 0.71906185, + "num_input_tokens_seen": 84237145, + "router_z_loss_clip": 0.71972656, + "router_z_loss_mlp": 0.17657471, + "step": 3014, + "time_per_iteration": 2.3950674533843994 + }, + { + "auxiliary_loss_clip": 0.0111267, + "auxiliary_loss_mlp": 0.01053032, + "balance_loss_clip": 1.03600001, + "balance_loss_mlp": 1.03071606, + "epoch": 0.08748766757588068, + "flos": 11501995772160.0, + "grad_norm": 3.0817410964578875, + "language_loss": 0.88573766, + "learning_rate": 3.965438458552104e-06, + "loss": 0.90739471, + "num_input_tokens_seen": 84247905, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.22314453, + "step": 3015, + "time_per_iteration": 2.317376136779785 + }, + { + "auxiliary_loss_clip": 0.01110258, + "auxiliary_loss_mlp": 0.01045435, + "balance_loss_clip": 1.03210843, + "balance_loss_mlp": 1.02207017, + "epoch": 0.08751668504439673, + "flos": 30328700163840.0, + "grad_norm": 2.543521973803548, + "language_loss": 0.92039341, + "learning_rate": 3.965403657692645e-06, + "loss": 0.94195038, + "num_input_tokens_seen": 84262340, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.23364258, + "step": 3016, + "time_per_iteration": 2.5123403072357178 + }, + { + "auxiliary_loss_clip": 0.01103649, + "auxiliary_loss_mlp": 0.01038679, + "balance_loss_clip": 1.03367019, + "balance_loss_mlp": 1.02034998, + "epoch": 0.08754570251291277, + "flos": 23177964182400.0, + "grad_norm": 2.5607035935097655, + "language_loss": 0.79756451, + "learning_rate": 3.965368839473946e-06, + "loss": 0.81898773, + "num_input_tokens_seen": 84280545, + "router_z_loss_clip": 0.69970703, + "router_z_loss_mlp": 0.18322754, + "step": 3017, + "time_per_iteration": 2.411144495010376 + }, + { + "auxiliary_loss_clip": 0.01113009, + "auxiliary_loss_mlp": 0.01044492, + "balance_loss_clip": 1.03349018, + "balance_loss_mlp": 1.02199769, + "epoch": 0.08757471998142882, + "flos": 24892060026240.0, + "grad_norm": 2.466318068248292, + "language_loss": 0.95984644, + "learning_rate": 3.965334003896313e-06, + "loss": 0.98142147, + "num_input_tokens_seen": 84296170, + "router_z_loss_clip": 0.79492188, + "router_z_loss_mlp": 0.22497559, + "step": 3018, + "time_per_iteration": 2.481722593307495 + }, + { + "auxiliary_loss_clip": 0.01019028, + "auxiliary_loss_mlp": 0.01004492, + "balance_loss_clip": 1.00742006, + "balance_loss_mlp": 1.00340676, + "epoch": 0.08760373744994486, + "flos": 55616184915840.0, + "grad_norm": 0.6573945715823039, + "language_loss": 0.46689147, + "learning_rate": 3.965299150960055e-06, + "loss": 0.48712665, + "num_input_tokens_seen": 84355125, + "router_z_loss_clip": 0.11621094, + "router_z_loss_mlp": 0.01086426, + "step": 3019, + "time_per_iteration": 2.8928515911102295 + }, + { + "auxiliary_loss_clip": 0.01110607, + "auxiliary_loss_mlp": 0.01046319, + "balance_loss_clip": 1.03658462, + "balance_loss_mlp": 1.02691221, + "epoch": 0.08763275491846091, + "flos": 16938147150720.0, + "grad_norm": 2.713471885587767, + "language_loss": 0.77133948, + "learning_rate": 3.96526428066548e-06, + "loss": 0.79290873, + "num_input_tokens_seen": 84367125, + "router_z_loss_clip": 0.7409668, + "router_z_loss_mlp": 0.19396973, + "step": 3020, + "time_per_iteration": 2.394428253173828 + }, + { + "auxiliary_loss_clip": 0.01111192, + "auxiliary_loss_mlp": 0.01046624, + "balance_loss_clip": 1.03358865, + "balance_loss_mlp": 1.02476072, + "epoch": 0.08766177238697696, + "flos": 16359915346560.0, + "grad_norm": 3.185180495559537, + "language_loss": 0.80989987, + "learning_rate": 3.965229393012895e-06, + "loss": 0.831478, + "num_input_tokens_seen": 84380835, + "router_z_loss_clip": 0.77636719, + "router_z_loss_mlp": 0.21887207, + "step": 3021, + "time_per_iteration": 2.301684617996216 + }, + { + "auxiliary_loss_clip": 0.01113911, + "auxiliary_loss_mlp": 0.01043609, + "balance_loss_clip": 1.0354476, + "balance_loss_mlp": 1.02268744, + "epoch": 0.087690789855493, + "flos": 74734939630080.0, + "grad_norm": 2.1512047604723894, + "language_loss": 0.75836927, + "learning_rate": 3.96519448800261e-06, + "loss": 0.77994448, + "num_input_tokens_seen": 84406050, + "router_z_loss_clip": 0.78369141, + "router_z_loss_mlp": 0.20922852, + "step": 3022, + "time_per_iteration": 2.8744707107543945 + }, + { + "auxiliary_loss_clip": 0.01107002, + "auxiliary_loss_mlp": 0.01048346, + "balance_loss_clip": 1.03585899, + "balance_loss_mlp": 1.03044116, + "epoch": 0.08771980732400905, + "flos": 27155269733760.0, + "grad_norm": 2.128917055694475, + "language_loss": 0.99839747, + "learning_rate": 3.96515956563493e-06, + "loss": 1.01995099, + "num_input_tokens_seen": 84424980, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.17907715, + "step": 3023, + "time_per_iteration": 2.4896390438079834 + }, + { + "auxiliary_loss_clip": 0.01121208, + "auxiliary_loss_mlp": 0.01042356, + "balance_loss_clip": 1.03638458, + "balance_loss_mlp": 1.01697671, + "epoch": 0.0877488247925251, + "flos": 23871443984640.0, + "grad_norm": 2.1112863594204088, + "language_loss": 0.9767487, + "learning_rate": 3.965124625910168e-06, + "loss": 0.99838436, + "num_input_tokens_seen": 84442980, + "router_z_loss_clip": 0.84814453, + "router_z_loss_mlp": 0.25378418, + "step": 3024, + "time_per_iteration": 2.4290435314178467 + }, + { + "auxiliary_loss_clip": 0.01115724, + "auxiliary_loss_mlp": 0.01058097, + "balance_loss_clip": 1.03455436, + "balance_loss_mlp": 1.03256214, + "epoch": 0.08777784226104114, + "flos": 30584892787200.0, + "grad_norm": 2.366582677941765, + "language_loss": 0.79455191, + "learning_rate": 3.965089668828628e-06, + "loss": 0.81629014, + "num_input_tokens_seen": 84458860, + "router_z_loss_clip": 0.81103516, + "router_z_loss_mlp": 0.25524902, + "step": 3025, + "time_per_iteration": 2.4689841270446777 + }, + { + "auxiliary_loss_clip": 0.01014045, + "auxiliary_loss_mlp": 0.0100533, + "balance_loss_clip": 1.002352, + "balance_loss_mlp": 1.0042336, + "epoch": 0.0878068597295572, + "flos": 55629976769280.0, + "grad_norm": 0.7472005143291721, + "language_loss": 0.45573437, + "learning_rate": 3.965054694390622e-06, + "loss": 0.47592813, + "num_input_tokens_seen": 84501200, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01098633, + "step": 3026, + "time_per_iteration": 2.7719130516052246 + }, + { + "auxiliary_loss_clip": 0.01102248, + "auxiliary_loss_mlp": 0.01039488, + "balance_loss_clip": 1.03169048, + "balance_loss_mlp": 1.02042639, + "epoch": 0.08783587719807325, + "flos": 15515645915520.0, + "grad_norm": 2.646191230340404, + "language_loss": 0.8630355, + "learning_rate": 3.9650197025964576e-06, + "loss": 0.88445282, + "num_input_tokens_seen": 84514895, + "router_z_loss_clip": 0.70556641, + "router_z_loss_mlp": 0.1907959, + "step": 3027, + "time_per_iteration": 2.3132801055908203 + }, + { + "auxiliary_loss_clip": 0.01014305, + "auxiliary_loss_mlp": 0.01001606, + "balance_loss_clip": 1.00236869, + "balance_loss_mlp": 1.00058079, + "epoch": 0.08786489466658928, + "flos": 74768210156160.0, + "grad_norm": 0.7594620808804378, + "language_loss": 0.5101729, + "learning_rate": 3.9649846934464435e-06, + "loss": 0.53033203, + "num_input_tokens_seen": 84577080, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01025391, + "step": 3028, + "time_per_iteration": 2.9895877838134766 + }, + { + "auxiliary_loss_clip": 0.01113172, + "auxiliary_loss_mlp": 0.01044461, + "balance_loss_clip": 1.033113, + "balance_loss_mlp": 1.02200162, + "epoch": 0.08789391213510533, + "flos": 28211846342400.0, + "grad_norm": 2.8051916374266486, + "language_loss": 0.78025556, + "learning_rate": 3.9649496669408904e-06, + "loss": 0.80183196, + "num_input_tokens_seen": 84597240, + "router_z_loss_clip": 0.80029297, + "router_z_loss_mlp": 0.22460938, + "step": 3029, + "time_per_iteration": 2.477947235107422 + }, + { + "auxiliary_loss_clip": 0.01014183, + "auxiliary_loss_mlp": 0.01001523, + "balance_loss_clip": 1.00268936, + "balance_loss_mlp": 1.00058079, + "epoch": 0.08792292960362139, + "flos": 74462858605440.0, + "grad_norm": 0.6251642564067652, + "language_loss": 0.47917077, + "learning_rate": 3.964914623080106e-06, + "loss": 0.49932784, + "num_input_tokens_seen": 84670190, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.00939941, + "step": 3030, + "time_per_iteration": 3.2656936645507812 + }, + { + "auxiliary_loss_clip": 0.01013307, + "auxiliary_loss_mlp": 0.0100181, + "balance_loss_clip": 1.00194812, + "balance_loss_mlp": 1.00079083, + "epoch": 0.08795194707213742, + "flos": 74786084772480.0, + "grad_norm": 0.6108717583045379, + "language_loss": 0.46978638, + "learning_rate": 3.9648795618644e-06, + "loss": 0.48993754, + "num_input_tokens_seen": 84735105, + "router_z_loss_clip": 0.11376953, + "router_z_loss_mlp": 0.01019287, + "step": 3031, + "time_per_iteration": 3.1700148582458496 + }, + { + "auxiliary_loss_clip": 0.01013228, + "auxiliary_loss_mlp": 0.01001391, + "balance_loss_clip": 1.00181007, + "balance_loss_mlp": 1.00027621, + "epoch": 0.08798096454065348, + "flos": 74772643898880.0, + "grad_norm": 0.6529616505087474, + "language_loss": 0.48773891, + "learning_rate": 3.964844483294084e-06, + "loss": 0.5078851, + "num_input_tokens_seen": 84792095, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01116943, + "step": 3032, + "time_per_iteration": 3.031139850616455 + }, + { + "auxiliary_loss_clip": 0.01014231, + "auxiliary_loss_mlp": 0.01000926, + "balance_loss_clip": 1.00280857, + "balance_loss_mlp": 0.99991268, + "epoch": 0.08800998200916953, + "flos": 74512017532800.0, + "grad_norm": 0.6515611459279338, + "language_loss": 0.53973806, + "learning_rate": 3.964809387369466e-06, + "loss": 0.55988967, + "num_input_tokens_seen": 84857135, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01013184, + "step": 3033, + "time_per_iteration": 3.2291228771209717 + }, + { + "auxiliary_loss_clip": 0.01110648, + "auxiliary_loss_mlp": 0.01045541, + "balance_loss_clip": 1.03532791, + "balance_loss_mlp": 1.02512074, + "epoch": 0.08803899947768556, + "flos": 12632692930560.0, + "grad_norm": 2.7604486528355134, + "language_loss": 0.96213996, + "learning_rate": 3.9647742740908555e-06, + "loss": 0.98370188, + "num_input_tokens_seen": 84868400, + "router_z_loss_clip": 0.75390625, + "router_z_loss_mlp": 0.20422363, + "step": 3034, + "time_per_iteration": 2.332275390625 + }, + { + "auxiliary_loss_clip": 0.01108016, + "auxiliary_loss_mlp": 0.01035139, + "balance_loss_clip": 1.0330857, + "balance_loss_mlp": 1.01465845, + "epoch": 0.08806801694620162, + "flos": 21061808588160.0, + "grad_norm": 2.507579306307104, + "language_loss": 0.68098527, + "learning_rate": 3.964739143458564e-06, + "loss": 0.70241684, + "num_input_tokens_seen": 84881615, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.20483398, + "step": 3035, + "time_per_iteration": 2.436967611312866 + }, + { + "auxiliary_loss_clip": 0.01108428, + "auxiliary_loss_mlp": 0.01045554, + "balance_loss_clip": 1.03372908, + "balance_loss_mlp": 1.02147341, + "epoch": 0.08809703441471765, + "flos": 12634333764480.0, + "grad_norm": 3.981622645016487, + "language_loss": 0.83242255, + "learning_rate": 3.964703995472902e-06, + "loss": 0.85396242, + "num_input_tokens_seen": 84894280, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.24072266, + "step": 3036, + "time_per_iteration": 2.3204753398895264 + }, + { + "auxiliary_loss_clip": 0.01112547, + "auxiliary_loss_mlp": 0.01047208, + "balance_loss_clip": 1.03409743, + "balance_loss_mlp": 1.02497494, + "epoch": 0.0881260518832337, + "flos": 13109990394240.0, + "grad_norm": 2.1008028849395077, + "language_loss": 0.68553495, + "learning_rate": 3.964668830134179e-06, + "loss": 0.70713246, + "num_input_tokens_seen": 84908825, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.22229004, + "step": 3037, + "time_per_iteration": 2.437350273132324 + }, + { + "auxiliary_loss_clip": 0.01023139, + "auxiliary_loss_mlp": 0.01001742, + "balance_loss_clip": 1.01180887, + "balance_loss_mlp": 1.00065136, + "epoch": 0.08815506935174976, + "flos": 64657607731200.0, + "grad_norm": 0.7500074015025405, + "language_loss": 0.50470376, + "learning_rate": 3.964633647442706e-06, + "loss": 0.52495253, + "num_input_tokens_seen": 84958690, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01092529, + "step": 3038, + "time_per_iteration": 2.8416554927825928 + }, + { + "auxiliary_loss_clip": 0.01107073, + "auxiliary_loss_mlp": 0.01043963, + "balance_loss_clip": 1.03372288, + "balance_loss_mlp": 1.02294636, + "epoch": 0.0881840868202658, + "flos": 25003397952000.0, + "grad_norm": 2.2429586424861787, + "language_loss": 0.78194076, + "learning_rate": 3.9645984473987925e-06, + "loss": 0.80345118, + "num_input_tokens_seen": 84975220, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.20983887, + "step": 3039, + "time_per_iteration": 2.4359359741210938 + }, + { + "auxiliary_loss_clip": 0.01020701, + "auxiliary_loss_mlp": 0.01004662, + "balance_loss_clip": 1.00939941, + "balance_loss_mlp": 1.00362504, + "epoch": 0.08821310428878185, + "flos": 65474260410240.0, + "grad_norm": 0.6401265280796894, + "language_loss": 0.50540382, + "learning_rate": 3.964563230002751e-06, + "loss": 0.52565742, + "num_input_tokens_seen": 85038360, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01037598, + "step": 3040, + "time_per_iteration": 3.030602216720581 + }, + { + "auxiliary_loss_clip": 0.0110784, + "auxiliary_loss_mlp": 0.01047689, + "balance_loss_clip": 1.0321455, + "balance_loss_mlp": 1.02313161, + "epoch": 0.0882421217572979, + "flos": 22850548652160.0, + "grad_norm": 2.192454734565222, + "language_loss": 0.83431011, + "learning_rate": 3.964527995254893e-06, + "loss": 0.85586536, + "num_input_tokens_seen": 85051770, + "router_z_loss_clip": 0.75683594, + "router_z_loss_mlp": 0.2454834, + "step": 3041, + "time_per_iteration": 2.357708692550659 + }, + { + "auxiliary_loss_clip": 0.01101078, + "auxiliary_loss_mlp": 0.01046737, + "balance_loss_clip": 1.03054321, + "balance_loss_mlp": 1.02678084, + "epoch": 0.08827113922581394, + "flos": 33146225527680.0, + "grad_norm": 2.337399049262414, + "language_loss": 0.91592669, + "learning_rate": 3.964492743155528e-06, + "loss": 0.93740481, + "num_input_tokens_seen": 85072565, + "router_z_loss_clip": 0.70458984, + "router_z_loss_mlp": 0.19934082, + "step": 3042, + "time_per_iteration": 2.5480453968048096 + }, + { + "auxiliary_loss_clip": 0.01107371, + "auxiliary_loss_mlp": 0.01052733, + "balance_loss_clip": 1.03522253, + "balance_loss_mlp": 1.0313822, + "epoch": 0.08830015669432999, + "flos": 32737113682560.0, + "grad_norm": 3.046431948951061, + "language_loss": 0.90767872, + "learning_rate": 3.964457473704969e-06, + "loss": 0.9292798, + "num_input_tokens_seen": 85091290, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.21350098, + "step": 3043, + "time_per_iteration": 2.4733822345733643 + }, + { + "auxiliary_loss_clip": 0.01109823, + "auxiliary_loss_mlp": 0.01044056, + "balance_loss_clip": 1.03584051, + "balance_loss_mlp": 1.02544749, + "epoch": 0.08832917416284604, + "flos": 25812509777280.0, + "grad_norm": 2.9436798795834, + "language_loss": 0.87133503, + "learning_rate": 3.964422186903525e-06, + "loss": 0.89287376, + "num_input_tokens_seen": 85110335, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.18603516, + "step": 3044, + "time_per_iteration": 2.4229397773742676 + }, + { + "auxiliary_loss_clip": 0.01113608, + "auxiliary_loss_mlp": 0.01043995, + "balance_loss_clip": 1.03569245, + "balance_loss_mlp": 1.01972413, + "epoch": 0.08835819163136208, + "flos": 14055368722560.0, + "grad_norm": 2.8883266247036983, + "language_loss": 1.04363966, + "learning_rate": 3.964386882751511e-06, + "loss": 1.06521559, + "num_input_tokens_seen": 85120620, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.24267578, + "step": 3045, + "time_per_iteration": 2.3778748512268066 + }, + { + "auxiliary_loss_clip": 0.0111128, + "auxiliary_loss_mlp": 0.01050429, + "balance_loss_clip": 1.03600228, + "balance_loss_mlp": 1.02804756, + "epoch": 0.08838720909987813, + "flos": 21534148638720.0, + "grad_norm": 2.490167264113758, + "language_loss": 0.84158623, + "learning_rate": 3.964351561249236e-06, + "loss": 0.86320329, + "num_input_tokens_seen": 85133700, + "router_z_loss_clip": 0.75268555, + "router_z_loss_mlp": 0.22381592, + "step": 3046, + "time_per_iteration": 6.804587364196777 + }, + { + "auxiliary_loss_clip": 0.0102126, + "auxiliary_loss_mlp": 0.01002118, + "balance_loss_clip": 1.00941396, + "balance_loss_mlp": 1.00099778, + "epoch": 0.08841622656839418, + "flos": 63349970469120.0, + "grad_norm": 0.8494226558243568, + "language_loss": 0.48401263, + "learning_rate": 3.964316222397014e-06, + "loss": 0.50424641, + "num_input_tokens_seen": 85190265, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.01123047, + "step": 3047, + "time_per_iteration": 2.985027313232422 + }, + { + "auxiliary_loss_clip": 0.01114181, + "auxiliary_loss_mlp": 0.01043715, + "balance_loss_clip": 1.03651452, + "balance_loss_mlp": 1.01779914, + "epoch": 0.08844524403691022, + "flos": 20185802864640.0, + "grad_norm": 5.206455563895591, + "language_loss": 0.79011941, + "learning_rate": 3.964280866195156e-06, + "loss": 0.81169844, + "num_input_tokens_seen": 85204655, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.25939941, + "step": 3048, + "time_per_iteration": 2.3914124965667725 + }, + { + "auxiliary_loss_clip": 0.0111716, + "auxiliary_loss_mlp": 0.01050893, + "balance_loss_clip": 1.03913164, + "balance_loss_mlp": 1.02784944, + "epoch": 0.08847426150542627, + "flos": 31971642923520.0, + "grad_norm": 2.0997983562429376, + "language_loss": 0.95264339, + "learning_rate": 3.964245492643974e-06, + "loss": 0.97432393, + "num_input_tokens_seen": 85223090, + "router_z_loss_clip": 0.78027344, + "router_z_loss_mlp": 0.23059082, + "step": 3049, + "time_per_iteration": 2.514299154281616 + }, + { + "auxiliary_loss_clip": 0.01015706, + "auxiliary_loss_mlp": 0.01003285, + "balance_loss_clip": 1.00396347, + "balance_loss_mlp": 1.00219381, + "epoch": 0.0885032789739423, + "flos": 64856891105280.0, + "grad_norm": 0.7296614972600296, + "language_loss": 0.48790026, + "learning_rate": 3.964210101743781e-06, + "loss": 0.5080902, + "num_input_tokens_seen": 85288030, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01092529, + "step": 3050, + "time_per_iteration": 5.444076776504517 + }, + { + "auxiliary_loss_clip": 0.0101433, + "auxiliary_loss_mlp": 0.0100095, + "balance_loss_clip": 1.00266075, + "balance_loss_mlp": 0.99986523, + "epoch": 0.08853229644245836, + "flos": 68017509066240.0, + "grad_norm": 0.6668000760807177, + "language_loss": 0.48577899, + "learning_rate": 3.96417469349489e-06, + "loss": 0.50593179, + "num_input_tokens_seen": 85347875, + "router_z_loss_clip": 0.11669922, + "router_z_loss_mlp": 0.01086426, + "step": 3051, + "time_per_iteration": 5.478471279144287 + }, + { + "auxiliary_loss_clip": 0.01108548, + "auxiliary_loss_mlp": 0.0104432, + "balance_loss_clip": 1.03461385, + "balance_loss_mlp": 1.02139568, + "epoch": 0.08856131391097441, + "flos": 48533322216960.0, + "grad_norm": 10.11266606000266, + "language_loss": 0.85780203, + "learning_rate": 3.964139267897613e-06, + "loss": 0.87933064, + "num_input_tokens_seen": 85365790, + "router_z_loss_clip": 0.73925781, + "router_z_loss_mlp": 0.22937012, + "step": 3052, + "time_per_iteration": 2.6532301902770996 + }, + { + "auxiliary_loss_clip": 0.01099669, + "auxiliary_loss_mlp": 0.01037407, + "balance_loss_clip": 1.03173101, + "balance_loss_mlp": 1.01853025, + "epoch": 0.08859033137949045, + "flos": 46162614833280.0, + "grad_norm": 5.70611791753675, + "language_loss": 0.6880765, + "learning_rate": 3.9641038249522634e-06, + "loss": 0.70944726, + "num_input_tokens_seen": 85384705, + "router_z_loss_clip": 0.67919922, + "router_z_loss_mlp": 0.18878174, + "step": 3053, + "time_per_iteration": 2.4898486137390137 + }, + { + "auxiliary_loss_clip": 0.01104584, + "auxiliary_loss_mlp": 0.01042759, + "balance_loss_clip": 1.03253436, + "balance_loss_mlp": 1.0235188, + "epoch": 0.0886193488480065, + "flos": 28941880204800.0, + "grad_norm": 1.8738067372779452, + "language_loss": 0.70789516, + "learning_rate": 3.964068364659154e-06, + "loss": 0.72936857, + "num_input_tokens_seen": 85401310, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.19250488, + "step": 3054, + "time_per_iteration": 2.453101873397827 + }, + { + "auxiliary_loss_clip": 0.01014971, + "auxiliary_loss_mlp": 0.01006658, + "balance_loss_clip": 1.00325155, + "balance_loss_mlp": 1.00557292, + "epoch": 0.08864836631652255, + "flos": 61741521999360.0, + "grad_norm": 0.6686824442493005, + "language_loss": 0.45161363, + "learning_rate": 3.964032887018598e-06, + "loss": 0.47182992, + "num_input_tokens_seen": 85464090, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01086426, + "step": 3055, + "time_per_iteration": 3.1184849739074707 + }, + { + "auxiliary_loss_clip": 0.01015418, + "auxiliary_loss_mlp": 0.01004157, + "balance_loss_clip": 1.00391197, + "balance_loss_mlp": 1.00308967, + "epoch": 0.08867738378503859, + "flos": 60427495958400.0, + "grad_norm": 0.7364614453135365, + "language_loss": 0.48428392, + "learning_rate": 3.963997392030909e-06, + "loss": 0.50447971, + "num_input_tokens_seen": 85521015, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01068115, + "step": 3056, + "time_per_iteration": 2.8845839500427246 + }, + { + "auxiliary_loss_clip": 0.01014893, + "auxiliary_loss_mlp": 0.01004903, + "balance_loss_clip": 1.00352263, + "balance_loss_mlp": 1.00376451, + "epoch": 0.08870640125355464, + "flos": 74779277057280.0, + "grad_norm": 0.6774804673975371, + "language_loss": 0.49544194, + "learning_rate": 3.9639618796964e-06, + "loss": 0.5156399, + "num_input_tokens_seen": 85589205, + "router_z_loss_clip": 0.11376953, + "router_z_loss_mlp": 0.01141357, + "step": 3057, + "time_per_iteration": 3.1153512001037598 + }, + { + "auxiliary_loss_clip": 0.01108053, + "auxiliary_loss_mlp": 0.01041009, + "balance_loss_clip": 1.03509295, + "balance_loss_mlp": 1.02069604, + "epoch": 0.08873541872207069, + "flos": 23250129696000.0, + "grad_norm": 2.4916118926385282, + "language_loss": 0.96406984, + "learning_rate": 3.963926350015385e-06, + "loss": 0.98556048, + "num_input_tokens_seen": 85604055, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.20324707, + "step": 3058, + "time_per_iteration": 2.43048357963562 + }, + { + "auxiliary_loss_clip": 0.01015761, + "auxiliary_loss_mlp": 0.0100064, + "balance_loss_clip": 1.00433874, + "balance_loss_mlp": 0.99961483, + "epoch": 0.08876443619058673, + "flos": 74390276021760.0, + "grad_norm": 0.7300450826795247, + "language_loss": 0.55734336, + "learning_rate": 3.963890802988178e-06, + "loss": 0.57750738, + "num_input_tokens_seen": 85656140, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01025391, + "step": 3059, + "time_per_iteration": 2.965346574783325 + }, + { + "auxiliary_loss_clip": 0.01016356, + "auxiliary_loss_mlp": 0.01001185, + "balance_loss_clip": 1.0050838, + "balance_loss_mlp": 1.00007033, + "epoch": 0.08879345365910278, + "flos": 72067433978880.0, + "grad_norm": 0.6681173085529762, + "language_loss": 0.51571649, + "learning_rate": 3.963855238615092e-06, + "loss": 0.53589189, + "num_input_tokens_seen": 85716500, + "router_z_loss_clip": 0.11279297, + "router_z_loss_mlp": 0.01116943, + "step": 3060, + "time_per_iteration": 3.0169618129730225 + }, + { + "auxiliary_loss_clip": 0.01015882, + "auxiliary_loss_mlp": 0.01002136, + "balance_loss_clip": 1.00458348, + "balance_loss_mlp": 1.00095034, + "epoch": 0.08882247112761883, + "flos": 73804328807040.0, + "grad_norm": 0.6965228676129448, + "language_loss": 0.47620007, + "learning_rate": 3.963819656896443e-06, + "loss": 0.49638027, + "num_input_tokens_seen": 85768580, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01184082, + "step": 3061, + "time_per_iteration": 2.915726661682129 + }, + { + "auxiliary_loss_clip": 0.01015062, + "auxiliary_loss_mlp": 0.01003467, + "balance_loss_clip": 1.00382113, + "balance_loss_mlp": 1.00235271, + "epoch": 0.08885148859613487, + "flos": 74780149841280.0, + "grad_norm": 0.6569794165485078, + "language_loss": 0.53493714, + "learning_rate": 3.963784057832543e-06, + "loss": 0.55512238, + "num_input_tokens_seen": 85832920, + "router_z_loss_clip": 0.11230469, + "router_z_loss_mlp": 0.01116943, + "step": 3062, + "time_per_iteration": 3.088529109954834 + }, + { + "auxiliary_loss_clip": 0.0110585, + "auxiliary_loss_mlp": 0.01047543, + "balance_loss_clip": 1.03316367, + "balance_loss_mlp": 1.02491689, + "epoch": 0.08888050606465092, + "flos": 14421502817280.0, + "grad_norm": 4.076475950605199, + "language_loss": 0.84765643, + "learning_rate": 3.963748441423708e-06, + "loss": 0.86919034, + "num_input_tokens_seen": 85846430, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.22631836, + "step": 3063, + "time_per_iteration": 2.3821351528167725 + }, + { + "auxiliary_loss_clip": 0.01108012, + "auxiliary_loss_mlp": 0.01047039, + "balance_loss_clip": 1.03401899, + "balance_loss_mlp": 1.02503347, + "epoch": 0.08890952353316697, + "flos": 13364262892800.0, + "grad_norm": 4.523124297273277, + "language_loss": 0.69152588, + "learning_rate": 3.963712807670252e-06, + "loss": 0.71307635, + "num_input_tokens_seen": 85859975, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.2199707, + "step": 3064, + "time_per_iteration": 2.3343167304992676 + }, + { + "auxiliary_loss_clip": 0.01105893, + "auxiliary_loss_mlp": 0.01047252, + "balance_loss_clip": 1.03349268, + "balance_loss_mlp": 1.0262655, + "epoch": 0.08893854100168301, + "flos": 31569164236800.0, + "grad_norm": 2.322225666641229, + "language_loss": 0.87711298, + "learning_rate": 3.96367715657249e-06, + "loss": 0.89864445, + "num_input_tokens_seen": 85876465, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.2097168, + "step": 3065, + "time_per_iteration": 2.490902900695801 + }, + { + "auxiliary_loss_clip": 0.01111792, + "auxiliary_loss_mlp": 0.01046524, + "balance_loss_clip": 1.03581369, + "balance_loss_mlp": 1.02627063, + "epoch": 0.08896755847019906, + "flos": 12470659977600.0, + "grad_norm": 2.205444081806757, + "language_loss": 0.65846777, + "learning_rate": 3.963641488130736e-06, + "loss": 0.68005097, + "num_input_tokens_seen": 85893830, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.20251465, + "step": 3066, + "time_per_iteration": 2.3891148567199707 + }, + { + "auxiliary_loss_clip": 0.0111745, + "auxiliary_loss_mlp": 0.0104374, + "balance_loss_clip": 1.03903937, + "balance_loss_mlp": 1.01999307, + "epoch": 0.0889965759387151, + "flos": 50505635543040.0, + "grad_norm": 2.875807418779486, + "language_loss": 0.8816663, + "learning_rate": 3.9636058023453075e-06, + "loss": 0.90327823, + "num_input_tokens_seen": 85914835, + "router_z_loss_clip": 0.78417969, + "router_z_loss_mlp": 0.23779297, + "step": 3067, + "time_per_iteration": 2.7800941467285156 + }, + { + "auxiliary_loss_clip": 0.01114369, + "auxiliary_loss_mlp": 0.01050023, + "balance_loss_clip": 1.0358547, + "balance_loss_mlp": 1.02684915, + "epoch": 0.08902559340723115, + "flos": 17192664028800.0, + "grad_norm": 2.570381678682949, + "language_loss": 0.932688, + "learning_rate": 3.9635700992165166e-06, + "loss": 0.95433187, + "num_input_tokens_seen": 85927515, + "router_z_loss_clip": 0.78515625, + "router_z_loss_mlp": 0.23181152, + "step": 3068, + "time_per_iteration": 2.392040491104126 + }, + { + "auxiliary_loss_clip": 0.01104043, + "auxiliary_loss_mlp": 0.01042192, + "balance_loss_clip": 1.03482461, + "balance_loss_mlp": 1.02038884, + "epoch": 0.0890546108757472, + "flos": 28651263114240.0, + "grad_norm": 2.853666024303307, + "language_loss": 0.94530344, + "learning_rate": 3.96353437874468e-06, + "loss": 0.96676582, + "num_input_tokens_seen": 85942040, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.21789551, + "step": 3069, + "time_per_iteration": 2.4663658142089844 + }, + { + "auxiliary_loss_clip": 0.01110487, + "auxiliary_loss_mlp": 0.01038846, + "balance_loss_clip": 1.03532755, + "balance_loss_mlp": 1.01840162, + "epoch": 0.08908362834426324, + "flos": 22375066579200.0, + "grad_norm": 2.195484457660597, + "language_loss": 0.88363928, + "learning_rate": 3.963498640930114e-06, + "loss": 0.90513259, + "num_input_tokens_seen": 85958910, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.20422363, + "step": 3070, + "time_per_iteration": 2.4651243686676025 + }, + { + "auxiliary_loss_clip": 0.01024645, + "auxiliary_loss_mlp": 0.010143, + "balance_loss_clip": 1.0127604, + "balance_loss_mlp": 1.0131793, + "epoch": 0.08911264581277929, + "flos": 53752172227200.0, + "grad_norm": 0.7425423388917992, + "language_loss": 0.56084967, + "learning_rate": 3.963462885773133e-06, + "loss": 0.58123916, + "num_input_tokens_seen": 86014985, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01123047, + "step": 3071, + "time_per_iteration": 2.919916868209839 + }, + { + "auxiliary_loss_clip": 0.01103319, + "auxiliary_loss_mlp": 0.01034123, + "balance_loss_clip": 1.03646886, + "balance_loss_mlp": 1.01594353, + "epoch": 0.08914166328129534, + "flos": 22374019238400.0, + "grad_norm": 2.206000190095, + "language_loss": 0.73158181, + "learning_rate": 3.963427113274054e-06, + "loss": 0.75295627, + "num_input_tokens_seen": 86030130, + "router_z_loss_clip": 0.66894531, + "router_z_loss_mlp": 0.1817627, + "step": 3072, + "time_per_iteration": 2.432861089706421 + }, + { + "auxiliary_loss_clip": 0.01107283, + "auxiliary_loss_mlp": 0.01048455, + "balance_loss_clip": 1.03614414, + "balance_loss_mlp": 1.0261271, + "epoch": 0.08917068074981138, + "flos": 34268194846080.0, + "grad_norm": 2.122722271493029, + "language_loss": 0.6730839, + "learning_rate": 3.9633913234331904e-06, + "loss": 0.69464123, + "num_input_tokens_seen": 86044820, + "router_z_loss_clip": 0.71142578, + "router_z_loss_mlp": 0.22338867, + "step": 3073, + "time_per_iteration": 2.4752907752990723 + }, + { + "auxiliary_loss_clip": 0.01117659, + "auxiliary_loss_mlp": 0.01055806, + "balance_loss_clip": 1.03909731, + "balance_loss_mlp": 1.03278685, + "epoch": 0.08919969821832743, + "flos": 18477956154240.0, + "grad_norm": 3.4427977040886932, + "language_loss": 0.9603529, + "learning_rate": 3.9633555162508615e-06, + "loss": 0.98208749, + "num_input_tokens_seen": 86059850, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.23022461, + "step": 3074, + "time_per_iteration": 2.341956853866577 + }, + { + "auxiliary_loss_clip": 0.01125355, + "auxiliary_loss_mlp": 0.01057318, + "balance_loss_clip": 1.04126239, + "balance_loss_mlp": 1.03130674, + "epoch": 0.08922871568684348, + "flos": 15700196695680.0, + "grad_norm": 2.463614377229126, + "language_loss": 0.89422494, + "learning_rate": 3.963319691727382e-06, + "loss": 0.91605163, + "num_input_tokens_seen": 86074575, + "router_z_loss_clip": 0.84130859, + "router_z_loss_mlp": 0.26013184, + "step": 3075, + "time_per_iteration": 2.4014179706573486 + }, + { + "auxiliary_loss_clip": 0.0111272, + "auxiliary_loss_mlp": 0.01043331, + "balance_loss_clip": 1.03687596, + "balance_loss_mlp": 1.02167058, + "epoch": 0.08925773315535952, + "flos": 13144205393280.0, + "grad_norm": 2.242996848516286, + "language_loss": 0.78446358, + "learning_rate": 3.963283849863069e-06, + "loss": 0.80602407, + "num_input_tokens_seen": 86087570, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.21655273, + "step": 3076, + "time_per_iteration": 2.3416404724121094 + }, + { + "auxiliary_loss_clip": 0.0102787, + "auxiliary_loss_mlp": 0.01001365, + "balance_loss_clip": 1.01537097, + "balance_loss_mlp": 1.00007164, + "epoch": 0.08928675062387557, + "flos": 61025871617280.0, + "grad_norm": 0.6281450084314729, + "language_loss": 0.49898943, + "learning_rate": 3.963247990658238e-06, + "loss": 0.51928174, + "num_input_tokens_seen": 86152120, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.01293945, + "step": 3077, + "time_per_iteration": 3.0565152168273926 + }, + { + "auxiliary_loss_clip": 0.01111061, + "auxiliary_loss_mlp": 0.01051149, + "balance_loss_clip": 1.03514957, + "balance_loss_mlp": 1.02903605, + "epoch": 0.08931576809239163, + "flos": 25914037610880.0, + "grad_norm": 2.5055676401334286, + "language_loss": 0.92347801, + "learning_rate": 3.963212114113206e-06, + "loss": 0.94510013, + "num_input_tokens_seen": 86170530, + "router_z_loss_clip": 0.75976562, + "router_z_loss_mlp": 0.22119141, + "step": 3078, + "time_per_iteration": 2.3957085609436035 + }, + { + "auxiliary_loss_clip": 0.01026508, + "auxiliary_loss_mlp": 0.0100817, + "balance_loss_clip": 1.01414919, + "balance_loss_mlp": 1.00678718, + "epoch": 0.08934478556090766, + "flos": 63167723838720.0, + "grad_norm": 0.6871516789239462, + "language_loss": 0.48132044, + "learning_rate": 3.96317622022829e-06, + "loss": 0.5016672, + "num_input_tokens_seen": 86229355, + "router_z_loss_clip": 0.12353516, + "router_z_loss_mlp": 0.01385498, + "step": 3079, + "time_per_iteration": 2.884514808654785 + }, + { + "auxiliary_loss_clip": 0.01020797, + "auxiliary_loss_mlp": 0.01008571, + "balance_loss_clip": 1.00876319, + "balance_loss_mlp": 1.00726557, + "epoch": 0.08937380302942372, + "flos": 65619918069120.0, + "grad_norm": 0.613969502195551, + "language_loss": 0.46454632, + "learning_rate": 3.963140309003808e-06, + "loss": 0.48484001, + "num_input_tokens_seen": 86296450, + "router_z_loss_clip": 0.12011719, + "router_z_loss_mlp": 0.01306152, + "step": 3080, + "time_per_iteration": 3.103694438934326 + }, + { + "auxiliary_loss_clip": 0.01110718, + "auxiliary_loss_mlp": 0.01053765, + "balance_loss_clip": 1.03236604, + "balance_loss_mlp": 1.03034019, + "epoch": 0.08940282049793975, + "flos": 20076070861440.0, + "grad_norm": 3.4146260064847227, + "language_loss": 0.90200186, + "learning_rate": 3.963104380440076e-06, + "loss": 0.92364675, + "num_input_tokens_seen": 86310125, + "router_z_loss_clip": 0.78271484, + "router_z_loss_mlp": 0.23425293, + "step": 3081, + "time_per_iteration": 2.350935220718384 + }, + { + "auxiliary_loss_clip": 0.01110352, + "auxiliary_loss_mlp": 0.01049955, + "balance_loss_clip": 1.03613448, + "balance_loss_mlp": 1.02878904, + "epoch": 0.0894318379664558, + "flos": 17704700161920.0, + "grad_norm": 2.481695569665215, + "language_loss": 0.80250371, + "learning_rate": 3.963068434537413e-06, + "loss": 0.82410681, + "num_input_tokens_seen": 86322465, + "router_z_loss_clip": 0.74169922, + "router_z_loss_mlp": 0.21173096, + "step": 3082, + "time_per_iteration": 2.3785784244537354 + }, + { + "auxiliary_loss_clip": 0.01015536, + "auxiliary_loss_mlp": 0.01004287, + "balance_loss_clip": 1.00359321, + "balance_loss_mlp": 1.00319016, + "epoch": 0.08946085543497186, + "flos": 65534276770560.0, + "grad_norm": 0.8578062459984345, + "language_loss": 0.53359145, + "learning_rate": 3.9630324712961335e-06, + "loss": 0.55378968, + "num_input_tokens_seen": 86379895, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01098633, + "step": 3083, + "time_per_iteration": 2.869349479675293 + }, + { + "auxiliary_loss_clip": 0.01108553, + "auxiliary_loss_mlp": 0.01037345, + "balance_loss_clip": 1.03575647, + "balance_loss_mlp": 1.01557708, + "epoch": 0.0894898729034879, + "flos": 15457514768640.0, + "grad_norm": 2.5438886710010578, + "language_loss": 0.81313145, + "learning_rate": 3.9629964907165575e-06, + "loss": 0.83459038, + "num_input_tokens_seen": 86394860, + "router_z_loss_clip": 0.72851562, + "router_z_loss_mlp": 0.21777344, + "step": 3084, + "time_per_iteration": 2.397671937942505 + }, + { + "auxiliary_loss_clip": 0.01120273, + "auxiliary_loss_mlp": 0.01046076, + "balance_loss_clip": 1.04064798, + "balance_loss_mlp": 1.02237689, + "epoch": 0.08951889037200395, + "flos": 22375625160960.0, + "grad_norm": 2.340875257170885, + "language_loss": 0.81765759, + "learning_rate": 3.962960492799002e-06, + "loss": 0.83932114, + "num_input_tokens_seen": 86409735, + "router_z_loss_clip": 0.79541016, + "router_z_loss_mlp": 0.23718262, + "step": 3085, + "time_per_iteration": 2.3646812438964844 + }, + { + "auxiliary_loss_clip": 0.011166, + "auxiliary_loss_mlp": 0.01042974, + "balance_loss_clip": 1.04157376, + "balance_loss_mlp": 1.02227914, + "epoch": 0.08954790784052, + "flos": 63349448664960.0, + "grad_norm": 6.328698650886426, + "language_loss": 0.94440496, + "learning_rate": 3.9629244775437845e-06, + "loss": 0.96600068, + "num_input_tokens_seen": 86435935, + "router_z_loss_clip": 0.74902344, + "router_z_loss_mlp": 0.20703125, + "step": 3086, + "time_per_iteration": 2.7455368041992188 + }, + { + "auxiliary_loss_clip": 0.01124973, + "auxiliary_loss_mlp": 0.01046245, + "balance_loss_clip": 1.04525232, + "balance_loss_mlp": 1.02432215, + "epoch": 0.08957692530903603, + "flos": 16682757488640.0, + "grad_norm": 2.6019321579450185, + "language_loss": 0.68355227, + "learning_rate": 3.9628884449512246e-06, + "loss": 0.70526439, + "num_input_tokens_seen": 86453265, + "router_z_loss_clip": 0.796875, + "router_z_loss_mlp": 0.21936035, + "step": 3087, + "time_per_iteration": 2.5008604526519775 + }, + { + "auxiliary_loss_clip": 0.01123079, + "auxiliary_loss_mlp": 0.0104273, + "balance_loss_clip": 1.04737842, + "balance_loss_mlp": 1.01985335, + "epoch": 0.08960594277755209, + "flos": 19015968205440.0, + "grad_norm": 2.3649246451197596, + "language_loss": 0.849388, + "learning_rate": 3.96285239502164e-06, + "loss": 0.87104607, + "num_input_tokens_seen": 86467870, + "router_z_loss_clip": 0.75683594, + "router_z_loss_mlp": 0.22900391, + "step": 3088, + "time_per_iteration": 2.4297821521759033 + }, + { + "auxiliary_loss_clip": 0.01123291, + "auxiliary_loss_mlp": 0.01044776, + "balance_loss_clip": 1.04856122, + "balance_loss_mlp": 1.02212584, + "epoch": 0.08963496024606814, + "flos": 30693018867840.0, + "grad_norm": 6.205055170466368, + "language_loss": 0.7222051, + "learning_rate": 3.9628163277553484e-06, + "loss": 0.74388576, + "num_input_tokens_seen": 86481695, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.2265625, + "step": 3089, + "time_per_iteration": 2.466732978820801 + }, + { + "auxiliary_loss_clip": 0.01115889, + "auxiliary_loss_mlp": 0.0103755, + "balance_loss_clip": 1.04746497, + "balance_loss_mlp": 1.01919174, + "epoch": 0.08966397771458418, + "flos": 13765449859200.0, + "grad_norm": 2.6492405364924863, + "language_loss": 0.6986087, + "learning_rate": 3.962780243152668e-06, + "loss": 0.72014308, + "num_input_tokens_seen": 86494805, + "router_z_loss_clip": 0.68457031, + "router_z_loss_mlp": 0.18347168, + "step": 3090, + "time_per_iteration": 2.3964178562164307 + }, + { + "auxiliary_loss_clip": 0.01123929, + "auxiliary_loss_mlp": 0.01040681, + "balance_loss_clip": 1.04918385, + "balance_loss_mlp": 1.0213697, + "epoch": 0.08969299518310023, + "flos": 17120952362880.0, + "grad_norm": 2.325830830968102, + "language_loss": 0.86233228, + "learning_rate": 3.962744141213919e-06, + "loss": 0.88397837, + "num_input_tokens_seen": 86507990, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.1932373, + "step": 3091, + "time_per_iteration": 2.391122579574585 + }, + { + "auxiliary_loss_clip": 0.01125181, + "auxiliary_loss_mlp": 0.01051586, + "balance_loss_clip": 1.05159616, + "balance_loss_mlp": 1.02775574, + "epoch": 0.08972201265161628, + "flos": 12893424030720.0, + "grad_norm": 2.3305162224374367, + "language_loss": 0.62697017, + "learning_rate": 3.962708021939419e-06, + "loss": 0.64873785, + "num_input_tokens_seen": 86521415, + "router_z_loss_clip": 0.73535156, + "router_z_loss_mlp": 0.23852539, + "step": 3092, + "time_per_iteration": 2.591876983642578 + }, + { + "auxiliary_loss_clip": 0.01127772, + "auxiliary_loss_mlp": 0.01047588, + "balance_loss_clip": 1.05294585, + "balance_loss_mlp": 1.02750111, + "epoch": 0.08975103012013232, + "flos": 11178734693760.0, + "grad_norm": 2.7904431471945013, + "language_loss": 0.86538243, + "learning_rate": 3.962671885329488e-06, + "loss": 0.88713598, + "num_input_tokens_seen": 86535700, + "router_z_loss_clip": 0.74902344, + "router_z_loss_mlp": 0.2010498, + "step": 3093, + "time_per_iteration": 2.3755576610565186 + }, + { + "auxiliary_loss_clip": 0.0112348, + "auxiliary_loss_mlp": 0.01050201, + "balance_loss_clip": 1.04663038, + "balance_loss_mlp": 1.02720535, + "epoch": 0.08978004758864837, + "flos": 15807310346880.0, + "grad_norm": 3.572503372872116, + "language_loss": 0.85293943, + "learning_rate": 3.962635731384444e-06, + "loss": 0.87467629, + "num_input_tokens_seen": 86548790, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.22998047, + "step": 3094, + "time_per_iteration": 2.3797647953033447 + }, + { + "auxiliary_loss_clip": 0.01055854, + "auxiliary_loss_mlp": 0.01012217, + "balance_loss_clip": 1.04301083, + "balance_loss_mlp": 1.01112664, + "epoch": 0.08980906505716442, + "flos": 72875917399680.0, + "grad_norm": 0.7350898647615206, + "language_loss": 0.53045744, + "learning_rate": 3.962599560104608e-06, + "loss": 0.55113816, + "num_input_tokens_seen": 86616390, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01092529, + "step": 3095, + "time_per_iteration": 3.1093461513519287 + }, + { + "auxiliary_loss_clip": 0.01117134, + "auxiliary_loss_mlp": 0.01045377, + "balance_loss_clip": 1.04126489, + "balance_loss_mlp": 1.02331161, + "epoch": 0.08983808252568046, + "flos": 19821763451520.0, + "grad_norm": 2.521960087818998, + "language_loss": 0.79209447, + "learning_rate": 3.9625633714902984e-06, + "loss": 0.81371957, + "num_input_tokens_seen": 86631400, + "router_z_loss_clip": 0.75878906, + "router_z_loss_mlp": 0.22058105, + "step": 3096, + "time_per_iteration": 2.428079605102539 + }, + { + "auxiliary_loss_clip": 0.01030519, + "auxiliary_loss_mlp": 0.01003853, + "balance_loss_clip": 1.01849699, + "balance_loss_mlp": 1.0027982, + "epoch": 0.08986709999419651, + "flos": 64965855058560.0, + "grad_norm": 0.6723299726985956, + "language_loss": 0.45987585, + "learning_rate": 3.962527165541834e-06, + "loss": 0.48021954, + "num_input_tokens_seen": 86693760, + "router_z_loss_clip": 0.12011719, + "router_z_loss_mlp": 0.01055908, + "step": 3097, + "time_per_iteration": 3.076152801513672 + }, + { + "auxiliary_loss_clip": 0.01115168, + "auxiliary_loss_mlp": 0.01040641, + "balance_loss_clip": 1.03819597, + "balance_loss_mlp": 1.01838446, + "epoch": 0.08989611746271255, + "flos": 10772625225600.0, + "grad_norm": 2.4508644805639053, + "language_loss": 0.90757346, + "learning_rate": 3.962490942259536e-06, + "loss": 0.92913157, + "num_input_tokens_seen": 86705440, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.22253418, + "step": 3098, + "time_per_iteration": 2.3627982139587402 + }, + { + "auxiliary_loss_clip": 0.01018452, + "auxiliary_loss_mlp": 0.01009109, + "balance_loss_clip": 1.00624752, + "balance_loss_mlp": 1.0080421, + "epoch": 0.0899251349312286, + "flos": 59377308128640.0, + "grad_norm": 0.6621689321576565, + "language_loss": 0.53712571, + "learning_rate": 3.962454701643724e-06, + "loss": 0.5574013, + "num_input_tokens_seen": 86768490, + "router_z_loss_clip": 0.12207031, + "router_z_loss_mlp": 0.01068115, + "step": 3099, + "time_per_iteration": 2.984283924102783 + }, + { + "auxiliary_loss_clip": 0.01020907, + "auxiliary_loss_mlp": 0.01004296, + "balance_loss_clip": 1.00883794, + "balance_loss_mlp": 1.0032891, + "epoch": 0.08995415239974465, + "flos": 74771840937600.0, + "grad_norm": 0.6546149972283256, + "language_loss": 0.53484774, + "learning_rate": 3.962418443694717e-06, + "loss": 0.55509973, + "num_input_tokens_seen": 86827760, + "router_z_loss_clip": 0.12109375, + "router_z_loss_mlp": 0.0100708, + "step": 3100, + "time_per_iteration": 3.069844961166382 + }, + { + "auxiliary_loss_clip": 0.01107912, + "auxiliary_loss_mlp": 0.01039702, + "balance_loss_clip": 1.03797948, + "balance_loss_mlp": 1.01913834, + "epoch": 0.08998316986826069, + "flos": 20075896304640.0, + "grad_norm": 3.327421934133379, + "language_loss": 0.85000539, + "learning_rate": 3.962382168412838e-06, + "loss": 0.87148154, + "num_input_tokens_seen": 86840645, + "router_z_loss_clip": 0.69970703, + "router_z_loss_mlp": 0.20556641, + "step": 3101, + "time_per_iteration": 2.3714911937713623 + }, + { + "auxiliary_loss_clip": 0.01120643, + "auxiliary_loss_mlp": 0.01046972, + "balance_loss_clip": 1.03993964, + "balance_loss_mlp": 1.02191412, + "epoch": 0.09001218733677674, + "flos": 15625308096000.0, + "grad_norm": 2.404628853335641, + "language_loss": 0.79807353, + "learning_rate": 3.962345875798405e-06, + "loss": 0.81974971, + "num_input_tokens_seen": 86854340, + "router_z_loss_clip": 0.80810547, + "router_z_loss_mlp": 0.25097656, + "step": 3102, + "time_per_iteration": 2.3952836990356445 + }, + { + "auxiliary_loss_clip": 0.01038605, + "auxiliary_loss_mlp": 0.01005007, + "balance_loss_clip": 1.02616429, + "balance_loss_mlp": 1.00383258, + "epoch": 0.09004120480529279, + "flos": 60801066172800.0, + "grad_norm": 0.7113925045386074, + "language_loss": 0.5101645, + "learning_rate": 3.962309565851738e-06, + "loss": 0.53060055, + "num_input_tokens_seen": 86918005, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.01171875, + "step": 3103, + "time_per_iteration": 2.9544591903686523 + }, + { + "auxiliary_loss_clip": 0.01124786, + "auxiliary_loss_mlp": 0.01049913, + "balance_loss_clip": 1.04633021, + "balance_loss_mlp": 1.02632201, + "epoch": 0.09007022227380883, + "flos": 23106217605120.0, + "grad_norm": 2.7774543105782157, + "language_loss": 0.69933128, + "learning_rate": 3.96227323857316e-06, + "loss": 0.72107828, + "num_input_tokens_seen": 86932280, + "router_z_loss_clip": 0.78466797, + "router_z_loss_mlp": 0.23596191, + "step": 3104, + "time_per_iteration": 2.4245834350585938 + }, + { + "auxiliary_loss_clip": 0.01125529, + "auxiliary_loss_mlp": 0.01038884, + "balance_loss_clip": 1.04852188, + "balance_loss_mlp": 1.01765895, + "epoch": 0.09009923974232488, + "flos": 74738919525120.0, + "grad_norm": 1.8844160824847267, + "language_loss": 0.86686754, + "learning_rate": 3.962236893962991e-06, + "loss": 0.88851178, + "num_input_tokens_seen": 86959015, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.21221924, + "step": 3105, + "time_per_iteration": 2.831350088119507 + }, + { + "auxiliary_loss_clip": 0.01121844, + "auxiliary_loss_mlp": 0.01047382, + "balance_loss_clip": 1.04782963, + "balance_loss_mlp": 1.02565026, + "epoch": 0.09012825721084093, + "flos": 11584285580160.0, + "grad_norm": 2.9112454233438836, + "language_loss": 0.7590996, + "learning_rate": 3.962200532021551e-06, + "loss": 0.78079188, + "num_input_tokens_seen": 86971795, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.21716309, + "step": 3106, + "time_per_iteration": 2.3941781520843506 + }, + { + "auxiliary_loss_clip": 0.01126771, + "auxiliary_loss_mlp": 0.0105703, + "balance_loss_clip": 1.04950595, + "balance_loss_mlp": 1.03503549, + "epoch": 0.09015727467935697, + "flos": 19859818700160.0, + "grad_norm": 2.397445413893872, + "language_loss": 0.87925828, + "learning_rate": 3.962164152749162e-06, + "loss": 0.90109634, + "num_input_tokens_seen": 86984595, + "router_z_loss_clip": 0.7734375, + "router_z_loss_mlp": 0.2199707, + "step": 3107, + "time_per_iteration": 2.398898124694824 + }, + { + "auxiliary_loss_clip": 0.01138123, + "auxiliary_loss_mlp": 0.01052457, + "balance_loss_clip": 1.05285394, + "balance_loss_mlp": 1.02773261, + "epoch": 0.09018629214787302, + "flos": 39384681016320.0, + "grad_norm": 3.741015530717058, + "language_loss": 0.85440814, + "learning_rate": 3.962127756146145e-06, + "loss": 0.87631392, + "num_input_tokens_seen": 87006585, + "router_z_loss_clip": 0.85351562, + "router_z_loss_mlp": 0.24719238, + "step": 3108, + "time_per_iteration": 2.5813186168670654 + }, + { + "auxiliary_loss_clip": 0.0112154, + "auxiliary_loss_mlp": 0.01047998, + "balance_loss_clip": 1.04679918, + "balance_loss_mlp": 1.02682662, + "epoch": 0.09021530961638907, + "flos": 19708994160000.0, + "grad_norm": 3.698928498101077, + "language_loss": 0.95578218, + "learning_rate": 3.962091342212822e-06, + "loss": 0.97747761, + "num_input_tokens_seen": 87018950, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.21179199, + "step": 3109, + "time_per_iteration": 2.367969512939453 + }, + { + "auxiliary_loss_clip": 0.01057117, + "auxiliary_loss_mlp": 0.00999617, + "balance_loss_clip": 1.04334116, + "balance_loss_mlp": 0.99840689, + "epoch": 0.09024432708490511, + "flos": 70887405202560.0, + "grad_norm": 0.8288772350981254, + "language_loss": 0.45852184, + "learning_rate": 3.9620549109495145e-06, + "loss": 0.4790892, + "num_input_tokens_seen": 87072385, + "router_z_loss_clip": 0.13769531, + "router_z_loss_mlp": 0.01208496, + "step": 3110, + "time_per_iteration": 2.979039192199707 + }, + { + "auxiliary_loss_clip": 0.01052797, + "auxiliary_loss_mlp": 0.01001798, + "balance_loss_clip": 1.03928804, + "balance_loss_mlp": 1.00059986, + "epoch": 0.09027334455342116, + "flos": 64320240597120.0, + "grad_norm": 1.4891094911000573, + "language_loss": 0.50447047, + "learning_rate": 3.962018462356543e-06, + "loss": 0.52501643, + "num_input_tokens_seen": 87136750, + "router_z_loss_clip": 0.13476562, + "router_z_loss_mlp": 0.01196289, + "step": 3111, + "time_per_iteration": 3.207510471343994 + }, + { + "auxiliary_loss_clip": 0.01135732, + "auxiliary_loss_mlp": 0.01051463, + "balance_loss_clip": 1.0471642, + "balance_loss_mlp": 1.0253917, + "epoch": 0.09030236202193721, + "flos": 21500841335040.0, + "grad_norm": 1.8358772479560805, + "language_loss": 0.85153925, + "learning_rate": 3.961981996434231e-06, + "loss": 0.87341118, + "num_input_tokens_seen": 87156225, + "router_z_loss_clip": 0.88574219, + "router_z_loss_mlp": 0.26074219, + "step": 3112, + "time_per_iteration": 2.4396634101867676 + }, + { + "auxiliary_loss_clip": 0.01125733, + "auxiliary_loss_mlp": 0.01052279, + "balance_loss_clip": 1.04403877, + "balance_loss_mlp": 1.02966487, + "epoch": 0.09033137949045325, + "flos": 16427437649280.0, + "grad_norm": 2.750714475974573, + "language_loss": 0.79830629, + "learning_rate": 3.9619455131829e-06, + "loss": 0.82008648, + "num_input_tokens_seen": 87169495, + "router_z_loss_clip": 0.81689453, + "router_z_loss_mlp": 0.22607422, + "step": 3113, + "time_per_iteration": 2.468979597091675 + }, + { + "auxiliary_loss_clip": 0.01037568, + "auxiliary_loss_mlp": 0.01006653, + "balance_loss_clip": 1.02465677, + "balance_loss_mlp": 1.00544858, + "epoch": 0.0903603969589693, + "flos": 74775820832640.0, + "grad_norm": 0.6559278970508357, + "language_loss": 0.4821794, + "learning_rate": 3.961909012602873e-06, + "loss": 0.50262165, + "num_input_tokens_seen": 87237000, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01202393, + "step": 3114, + "time_per_iteration": 3.12431001663208 + }, + { + "auxiliary_loss_clip": 0.01033021, + "auxiliary_loss_mlp": 0.01007092, + "balance_loss_clip": 1.02070045, + "balance_loss_mlp": 1.00591218, + "epoch": 0.09038941442748534, + "flos": 74776798350720.0, + "grad_norm": 0.6492415077503084, + "language_loss": 0.50525761, + "learning_rate": 3.961872494694472e-06, + "loss": 0.52565873, + "num_input_tokens_seen": 87301920, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01177979, + "step": 3115, + "time_per_iteration": 3.1230711936950684 + }, + { + "auxiliary_loss_clip": 0.01115463, + "auxiliary_loss_mlp": 0.01048459, + "balance_loss_clip": 1.0354495, + "balance_loss_mlp": 1.02354383, + "epoch": 0.09041843189600139, + "flos": 17596085322240.0, + "grad_norm": 2.2254321838925555, + "language_loss": 0.84774709, + "learning_rate": 3.961835959458018e-06, + "loss": 0.8693862, + "num_input_tokens_seen": 87316030, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.24914551, + "step": 3116, + "time_per_iteration": 2.3421993255615234 + }, + { + "auxiliary_loss_clip": 0.01110563, + "auxiliary_loss_mlp": 0.01042231, + "balance_loss_clip": 1.0351907, + "balance_loss_mlp": 1.02120209, + "epoch": 0.09044744936451744, + "flos": 36021218722560.0, + "grad_norm": 2.1075896905303786, + "language_loss": 0.99875051, + "learning_rate": 3.961799406893836e-06, + "loss": 1.02027845, + "num_input_tokens_seen": 87334660, + "router_z_loss_clip": 0.75488281, + "router_z_loss_mlp": 0.21008301, + "step": 3117, + "time_per_iteration": 2.512504816055298 + }, + { + "auxiliary_loss_clip": 0.01111307, + "auxiliary_loss_mlp": 0.01041969, + "balance_loss_clip": 1.03591979, + "balance_loss_mlp": 1.02078509, + "epoch": 0.09047646683303348, + "flos": 30256360093440.0, + "grad_norm": 2.180804572909841, + "language_loss": 0.77557814, + "learning_rate": 3.961762837002247e-06, + "loss": 0.79711092, + "num_input_tokens_seen": 87350205, + "router_z_loss_clip": 0.75292969, + "router_z_loss_mlp": 0.21203613, + "step": 3118, + "time_per_iteration": 2.480617046356201 + }, + { + "auxiliary_loss_clip": 0.01115998, + "auxiliary_loss_mlp": 0.01048509, + "balance_loss_clip": 1.03847218, + "balance_loss_mlp": 1.0266341, + "epoch": 0.09050548430154953, + "flos": 26388297786240.0, + "grad_norm": 2.5008086210134315, + "language_loss": 1.02197409, + "learning_rate": 3.961726249783575e-06, + "loss": 1.04361916, + "num_input_tokens_seen": 87367660, + "router_z_loss_clip": 0.77441406, + "router_z_loss_mlp": 0.21862793, + "step": 3119, + "time_per_iteration": 2.450326442718506 + }, + { + "auxiliary_loss_clip": 0.01113317, + "auxiliary_loss_mlp": 0.01041849, + "balance_loss_clip": 1.03995025, + "balance_loss_mlp": 1.02072549, + "epoch": 0.09053450177006558, + "flos": 19345827530880.0, + "grad_norm": 2.9955520360440926, + "language_loss": 0.9389208, + "learning_rate": 3.961689645238143e-06, + "loss": 0.96047252, + "num_input_tokens_seen": 87380060, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.21118164, + "step": 3120, + "time_per_iteration": 2.3529176712036133 + }, + { + "auxiliary_loss_clip": 0.01028082, + "auxiliary_loss_mlp": 0.01003888, + "balance_loss_clip": 1.01561868, + "balance_loss_mlp": 1.00284505, + "epoch": 0.09056351923858162, + "flos": 67305140352000.0, + "grad_norm": 0.6729959371883615, + "language_loss": 0.46209237, + "learning_rate": 3.961653023366274e-06, + "loss": 0.48241207, + "num_input_tokens_seen": 87442370, + "router_z_loss_clip": 0.12451172, + "router_z_loss_mlp": 0.01043701, + "step": 3121, + "time_per_iteration": 3.1165504455566406 + }, + { + "auxiliary_loss_clip": 0.01123482, + "auxiliary_loss_mlp": 0.01036805, + "balance_loss_clip": 1.04261255, + "balance_loss_mlp": 1.01461387, + "epoch": 0.09059253670709767, + "flos": 13363704311040.0, + "grad_norm": 3.0302988633611996, + "language_loss": 0.75294459, + "learning_rate": 3.9616163841682915e-06, + "loss": 0.77454746, + "num_input_tokens_seen": 87455765, + "router_z_loss_clip": 0.81005859, + "router_z_loss_mlp": 0.22198486, + "step": 3122, + "time_per_iteration": 6.756641149520874 + }, + { + "auxiliary_loss_clip": 0.01035727, + "auxiliary_loss_mlp": 0.01003901, + "balance_loss_clip": 1.02282679, + "balance_loss_mlp": 1.00285232, + "epoch": 0.09062155417561372, + "flos": 54734209349760.0, + "grad_norm": 0.6793082606570704, + "language_loss": 0.46059132, + "learning_rate": 3.96157972764452e-06, + "loss": 0.48098761, + "num_input_tokens_seen": 87511035, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01049805, + "step": 3123, + "time_per_iteration": 2.89870285987854 + }, + { + "auxiliary_loss_clip": 0.01112535, + "auxiliary_loss_mlp": 0.01041683, + "balance_loss_clip": 1.04232073, + "balance_loss_mlp": 1.02383757, + "epoch": 0.09065057164412976, + "flos": 25623176140800.0, + "grad_norm": 2.259469961376783, + "language_loss": 0.81969476, + "learning_rate": 3.961543053795283e-06, + "loss": 0.84123695, + "num_input_tokens_seen": 87525145, + "router_z_loss_clip": 0.70214844, + "router_z_loss_mlp": 0.1784668, + "step": 3124, + "time_per_iteration": 2.4318175315856934 + }, + { + "auxiliary_loss_clip": 0.01110747, + "auxiliary_loss_mlp": 0.01044168, + "balance_loss_clip": 1.04395127, + "balance_loss_mlp": 1.02689409, + "epoch": 0.09067958911264581, + "flos": 24964330273920.0, + "grad_norm": 2.6491169559341605, + "language_loss": 0.84742391, + "learning_rate": 3.961506362620903e-06, + "loss": 0.86897308, + "num_input_tokens_seen": 87537940, + "router_z_loss_clip": 0.66699219, + "router_z_loss_mlp": 0.17285156, + "step": 3125, + "time_per_iteration": 2.529043436050415 + }, + { + "auxiliary_loss_clip": 0.0104092, + "auxiliary_loss_mlp": 0.01002291, + "balance_loss_clip": 1.02833915, + "balance_loss_mlp": 1.00125396, + "epoch": 0.09070860658116187, + "flos": 62843031394560.0, + "grad_norm": 0.6745267065787611, + "language_loss": 0.52245355, + "learning_rate": 3.9614696541217054e-06, + "loss": 0.54288566, + "num_input_tokens_seen": 87600230, + "router_z_loss_clip": 0.12597656, + "router_z_loss_mlp": 0.01037598, + "step": 3126, + "time_per_iteration": 5.4396796226501465 + }, + { + "auxiliary_loss_clip": 0.01121369, + "auxiliary_loss_mlp": 0.01050318, + "balance_loss_clip": 1.04641032, + "balance_loss_mlp": 1.03026676, + "epoch": 0.0907376240496779, + "flos": 22701958439040.0, + "grad_norm": 2.0764165013415097, + "language_loss": 0.7493242, + "learning_rate": 3.961432928298014e-06, + "loss": 0.77104115, + "num_input_tokens_seen": 87614430, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.20056152, + "step": 3127, + "time_per_iteration": 4.737607002258301 + }, + { + "auxiliary_loss_clip": 0.01117514, + "auxiliary_loss_mlp": 0.01054724, + "balance_loss_clip": 1.04237342, + "balance_loss_mlp": 1.031955, + "epoch": 0.09076664151819396, + "flos": 33539487615360.0, + "grad_norm": 2.192056518184785, + "language_loss": 0.90722382, + "learning_rate": 3.9613961851501534e-06, + "loss": 0.92894626, + "num_input_tokens_seen": 87631080, + "router_z_loss_clip": 0.75146484, + "router_z_loss_mlp": 0.2277832, + "step": 3128, + "time_per_iteration": 2.4929239749908447 + }, + { + "auxiliary_loss_clip": 0.01113089, + "auxiliary_loss_mlp": 0.01052929, + "balance_loss_clip": 1.03978217, + "balance_loss_mlp": 1.03246069, + "epoch": 0.09079565898670999, + "flos": 20076943645440.0, + "grad_norm": 2.448447863770528, + "language_loss": 0.96199006, + "learning_rate": 3.961359424678448e-06, + "loss": 0.98365021, + "num_input_tokens_seen": 87644160, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.20471191, + "step": 3129, + "time_per_iteration": 2.3844709396362305 + }, + { + "auxiliary_loss_clip": 0.01112388, + "auxiliary_loss_mlp": 0.01058768, + "balance_loss_clip": 1.03616619, + "balance_loss_mlp": 1.03633296, + "epoch": 0.09082467645522604, + "flos": 25404515095680.0, + "grad_norm": 2.8749629272424624, + "language_loss": 0.94808173, + "learning_rate": 3.961322646883222e-06, + "loss": 0.96979326, + "num_input_tokens_seen": 87656830, + "router_z_loss_clip": 0.76220703, + "router_z_loss_mlp": 0.22412109, + "step": 3130, + "time_per_iteration": 2.3935530185699463 + }, + { + "auxiliary_loss_clip": 0.01028472, + "auxiliary_loss_mlp": 0.01006152, + "balance_loss_clip": 1.01588202, + "balance_loss_mlp": 1.00515032, + "epoch": 0.0908536939237421, + "flos": 62465271816960.0, + "grad_norm": 0.7407673950177955, + "language_loss": 0.55398476, + "learning_rate": 3.961285851764801e-06, + "loss": 0.57433105, + "num_input_tokens_seen": 87715725, + "router_z_loss_clip": 0.12597656, + "router_z_loss_mlp": 0.01000977, + "step": 3131, + "time_per_iteration": 3.114870071411133 + }, + { + "auxiliary_loss_clip": 0.01111271, + "auxiliary_loss_mlp": 0.01046791, + "balance_loss_clip": 1.03580964, + "balance_loss_mlp": 1.02503562, + "epoch": 0.09088271139225813, + "flos": 31240317340800.0, + "grad_norm": 2.937701778450188, + "language_loss": 1.0311799, + "learning_rate": 3.96124903932351e-06, + "loss": 1.05276048, + "num_input_tokens_seen": 87734830, + "router_z_loss_clip": 0.75390625, + "router_z_loss_mlp": 0.2175293, + "step": 3132, + "time_per_iteration": 2.567871570587158 + }, + { + "auxiliary_loss_clip": 0.01110885, + "auxiliary_loss_mlp": 0.01059976, + "balance_loss_clip": 1.03341854, + "balance_loss_mlp": 1.03782701, + "epoch": 0.09091172886077419, + "flos": 26098483656960.0, + "grad_norm": 3.701007648321734, + "language_loss": 0.82687694, + "learning_rate": 3.961212209559674e-06, + "loss": 0.84858555, + "num_input_tokens_seen": 87749915, + "router_z_loss_clip": 0.77490234, + "router_z_loss_mlp": 0.22143555, + "step": 3133, + "time_per_iteration": 2.376882553100586 + }, + { + "auxiliary_loss_clip": 0.01024329, + "auxiliary_loss_mlp": 0.01003458, + "balance_loss_clip": 1.01194668, + "balance_loss_mlp": 1.00248027, + "epoch": 0.09094074632929024, + "flos": 72655440963840.0, + "grad_norm": 0.6849528426482753, + "language_loss": 0.53819799, + "learning_rate": 3.961175362473618e-06, + "loss": 0.55847585, + "num_input_tokens_seen": 87812910, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.00976562, + "step": 3134, + "time_per_iteration": 3.0495359897613525 + }, + { + "auxiliary_loss_clip": 0.01023142, + "auxiliary_loss_mlp": 0.01001983, + "balance_loss_clip": 1.01082623, + "balance_loss_mlp": 1.00099909, + "epoch": 0.09096976379780627, + "flos": 62362591908480.0, + "grad_norm": 0.6781340870859384, + "language_loss": 0.47379842, + "learning_rate": 3.961138498065667e-06, + "loss": 0.49404967, + "num_input_tokens_seen": 87875070, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.00982666, + "step": 3135, + "time_per_iteration": 3.0105645656585693 + }, + { + "auxiliary_loss_clip": 0.01024491, + "auxiliary_loss_mlp": 0.01001827, + "balance_loss_clip": 1.01250625, + "balance_loss_mlp": 1.00081968, + "epoch": 0.09099878126632233, + "flos": 67608187752960.0, + "grad_norm": 0.7440135518149452, + "language_loss": 0.51857042, + "learning_rate": 3.9611016163361476e-06, + "loss": 0.53883362, + "num_input_tokens_seen": 87926810, + "router_z_loss_clip": 0.12011719, + "router_z_loss_mlp": 0.0100708, + "step": 3136, + "time_per_iteration": 2.8494300842285156 + }, + { + "auxiliary_loss_clip": 0.01022992, + "auxiliary_loss_mlp": 0.01002379, + "balance_loss_clip": 1.01106119, + "balance_loss_mlp": 1.00133622, + "epoch": 0.09102779873483838, + "flos": 65097792552960.0, + "grad_norm": 0.6859241862462021, + "language_loss": 0.51709068, + "learning_rate": 3.961064717285386e-06, + "loss": 0.53734446, + "num_input_tokens_seen": 87988920, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01043701, + "step": 3137, + "time_per_iteration": 3.0420899391174316 + }, + { + "auxiliary_loss_clip": 0.01130586, + "auxiliary_loss_mlp": 0.01058107, + "balance_loss_clip": 1.04415011, + "balance_loss_mlp": 1.03188062, + "epoch": 0.09105681620335442, + "flos": 25660184048640.0, + "grad_norm": 2.113110239533854, + "language_loss": 0.94586802, + "learning_rate": 3.961027800913706e-06, + "loss": 0.9677549, + "num_input_tokens_seen": 88009845, + "router_z_loss_clip": 0.86425781, + "router_z_loss_mlp": 0.26220703, + "step": 3138, + "time_per_iteration": 2.5451414585113525 + }, + { + "auxiliary_loss_clip": 0.01114749, + "auxiliary_loss_mlp": 0.01054496, + "balance_loss_clip": 1.04144681, + "balance_loss_mlp": 1.03424191, + "epoch": 0.09108583367187047, + "flos": 37923147014400.0, + "grad_norm": 2.01289355494838, + "language_loss": 0.70764452, + "learning_rate": 3.9609908672214355e-06, + "loss": 0.72933698, + "num_input_tokens_seen": 88028080, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.20239258, + "step": 3139, + "time_per_iteration": 2.5304172039031982 + }, + { + "auxiliary_loss_clip": 0.01114643, + "auxiliary_loss_mlp": 0.01043192, + "balance_loss_clip": 1.04264796, + "balance_loss_mlp": 1.02359366, + "epoch": 0.09111485114038652, + "flos": 20623927916160.0, + "grad_norm": 3.085939058929713, + "language_loss": 0.76860583, + "learning_rate": 3.9609539162088995e-06, + "loss": 0.79018414, + "num_input_tokens_seen": 88041190, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.19592285, + "step": 3140, + "time_per_iteration": 2.4283933639526367 + }, + { + "auxiliary_loss_clip": 0.01119578, + "auxiliary_loss_mlp": 0.01053711, + "balance_loss_clip": 1.04763937, + "balance_loss_mlp": 1.03467906, + "epoch": 0.09114386860890256, + "flos": 18142301543040.0, + "grad_norm": 2.3309735441621684, + "language_loss": 0.7837857, + "learning_rate": 3.960916947876426e-06, + "loss": 0.80551863, + "num_input_tokens_seen": 88059210, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.19024658, + "step": 3141, + "time_per_iteration": 2.4830799102783203 + }, + { + "auxiliary_loss_clip": 0.01120147, + "auxiliary_loss_mlp": 0.01050176, + "balance_loss_clip": 1.04522753, + "balance_loss_mlp": 1.02660823, + "epoch": 0.09117288607741861, + "flos": 11428084690560.0, + "grad_norm": 3.4649499717042507, + "language_loss": 0.85385406, + "learning_rate": 3.96087996222434e-06, + "loss": 0.87555736, + "num_input_tokens_seen": 88069885, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.2355957, + "step": 3142, + "time_per_iteration": 2.381890058517456 + }, + { + "auxiliary_loss_clip": 0.01114808, + "auxiliary_loss_mlp": 0.0104288, + "balance_loss_clip": 1.04208016, + "balance_loss_mlp": 1.02303147, + "epoch": 0.09120190354593466, + "flos": 12305626513920.0, + "grad_norm": 2.153817425854068, + "language_loss": 0.7719239, + "learning_rate": 3.960842959252969e-06, + "loss": 0.79350078, + "num_input_tokens_seen": 88083135, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.19848633, + "step": 3143, + "time_per_iteration": 2.3385860919952393 + }, + { + "auxiliary_loss_clip": 0.0111509, + "auxiliary_loss_mlp": 0.0104064, + "balance_loss_clip": 1.04184246, + "balance_loss_mlp": 1.02156663, + "epoch": 0.0912309210144507, + "flos": 39451819294080.0, + "grad_norm": 2.308814707031164, + "language_loss": 1.10545206, + "learning_rate": 3.960805938962639e-06, + "loss": 1.12700939, + "num_input_tokens_seen": 88103805, + "router_z_loss_clip": 0.73193359, + "router_z_loss_mlp": 0.19067383, + "step": 3144, + "time_per_iteration": 2.5902154445648193 + }, + { + "auxiliary_loss_clip": 0.01120717, + "auxiliary_loss_mlp": 0.0105517, + "balance_loss_clip": 1.04293144, + "balance_loss_mlp": 1.03252006, + "epoch": 0.09125993848296675, + "flos": 26538249542400.0, + "grad_norm": 2.451547261748973, + "language_loss": 0.79286337, + "learning_rate": 3.960768901353678e-06, + "loss": 0.81462222, + "num_input_tokens_seen": 88119810, + "router_z_loss_clip": 0.77783203, + "router_z_loss_mlp": 0.2265625, + "step": 3145, + "time_per_iteration": 2.4192769527435303 + }, + { + "auxiliary_loss_clip": 0.01115525, + "auxiliary_loss_mlp": 0.01042858, + "balance_loss_clip": 1.04043293, + "balance_loss_mlp": 1.02210355, + "epoch": 0.09128895595148279, + "flos": 21864461811840.0, + "grad_norm": 2.3273773840165193, + "language_loss": 0.70610821, + "learning_rate": 3.960731846426411e-06, + "loss": 0.72769201, + "num_input_tokens_seen": 88135180, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.20751953, + "step": 3146, + "time_per_iteration": 2.404881715774536 + }, + { + "auxiliary_loss_clip": 0.01118256, + "auxiliary_loss_mlp": 0.01047293, + "balance_loss_clip": 1.03884685, + "balance_loss_mlp": 1.02359414, + "epoch": 0.09131797341999884, + "flos": 21286299830400.0, + "grad_norm": 2.1671290574811417, + "language_loss": 1.06562757, + "learning_rate": 3.960694774181169e-06, + "loss": 1.08728313, + "num_input_tokens_seen": 88154590, + "router_z_loss_clip": 0.79345703, + "router_z_loss_mlp": 0.23706055, + "step": 3147, + "time_per_iteration": 2.4137096405029297 + }, + { + "auxiliary_loss_clip": 0.0112416, + "auxiliary_loss_mlp": 0.01047123, + "balance_loss_clip": 1.04555774, + "balance_loss_mlp": 1.023615, + "epoch": 0.09134699088851489, + "flos": 68565123607680.0, + "grad_norm": 2.882575811883854, + "language_loss": 0.95652187, + "learning_rate": 3.960657684618277e-06, + "loss": 0.97823471, + "num_input_tokens_seen": 88173435, + "router_z_loss_clip": 0.78710938, + "router_z_loss_mlp": 0.23510742, + "step": 3148, + "time_per_iteration": 2.7913362979888916 + }, + { + "auxiliary_loss_clip": 0.01119363, + "auxiliary_loss_mlp": 0.01045831, + "balance_loss_clip": 1.04024494, + "balance_loss_mlp": 1.0227282, + "epoch": 0.09137600835703093, + "flos": 12415463251200.0, + "grad_norm": 2.288646037934519, + "language_loss": 0.75296056, + "learning_rate": 3.960620577738062e-06, + "loss": 0.77461249, + "num_input_tokens_seen": 88188855, + "router_z_loss_clip": 0.79101562, + "router_z_loss_mlp": 0.23120117, + "step": 3149, + "time_per_iteration": 2.395673990249634 + }, + { + "auxiliary_loss_clip": 0.01036976, + "auxiliary_loss_mlp": 0.01001457, + "balance_loss_clip": 1.0235393, + "balance_loss_mlp": 1.00031865, + "epoch": 0.09140502582554698, + "flos": 55622992631040.0, + "grad_norm": 0.710991813471319, + "language_loss": 0.48637861, + "learning_rate": 3.960583453540853e-06, + "loss": 0.50676298, + "num_input_tokens_seen": 88250740, + "router_z_loss_clip": 0.13476562, + "router_z_loss_mlp": 0.01141357, + "step": 3150, + "time_per_iteration": 3.004586696624756 + }, + { + "auxiliary_loss_clip": 0.01111147, + "auxiliary_loss_mlp": 0.01052588, + "balance_loss_clip": 1.03539729, + "balance_loss_mlp": 1.03014088, + "epoch": 0.09143404329406303, + "flos": 30586428887040.0, + "grad_norm": 1.9179032882117648, + "language_loss": 0.81594622, + "learning_rate": 3.960546312026978e-06, + "loss": 0.8375836, + "num_input_tokens_seen": 88276040, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.22473145, + "step": 3151, + "time_per_iteration": 2.5887534618377686 + }, + { + "auxiliary_loss_clip": 0.01110014, + "auxiliary_loss_mlp": 0.01045204, + "balance_loss_clip": 1.03658104, + "balance_loss_mlp": 1.02361536, + "epoch": 0.09146306076257907, + "flos": 13070154666240.0, + "grad_norm": 3.0004291768641496, + "language_loss": 1.02030635, + "learning_rate": 3.960509153196764e-06, + "loss": 1.04185843, + "num_input_tokens_seen": 88286350, + "router_z_loss_clip": 0.73388672, + "router_z_loss_mlp": 0.21582031, + "step": 3152, + "time_per_iteration": 2.382892608642578 + }, + { + "auxiliary_loss_clip": 0.01101869, + "auxiliary_loss_mlp": 0.01039075, + "balance_loss_clip": 1.03270376, + "balance_loss_mlp": 1.01945281, + "epoch": 0.09149207823109512, + "flos": 15114703328640.0, + "grad_norm": 2.4523764960286614, + "language_loss": 0.79119396, + "learning_rate": 3.960471977050541e-06, + "loss": 0.81260335, + "num_input_tokens_seen": 88300595, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.19641113, + "step": 3153, + "time_per_iteration": 2.3235011100769043 + }, + { + "auxiliary_loss_clip": 0.01020219, + "auxiliary_loss_mlp": 0.01005238, + "balance_loss_clip": 1.00831389, + "balance_loss_mlp": 1.00414157, + "epoch": 0.09152109569961117, + "flos": 74781790675200.0, + "grad_norm": 0.6817225382241158, + "language_loss": 0.49239588, + "learning_rate": 3.960434783588635e-06, + "loss": 0.51265049, + "num_input_tokens_seen": 88371290, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01098633, + "step": 3154, + "time_per_iteration": 3.1517975330352783 + }, + { + "auxiliary_loss_clip": 0.01108191, + "auxiliary_loss_mlp": 0.01043117, + "balance_loss_clip": 1.03485096, + "balance_loss_mlp": 1.02436495, + "epoch": 0.09155011316812721, + "flos": 24602979035520.0, + "grad_norm": 2.1968495214025765, + "language_loss": 0.77342093, + "learning_rate": 3.9603975728113766e-06, + "loss": 0.79493397, + "num_input_tokens_seen": 88388385, + "router_z_loss_clip": 0.73242188, + "router_z_loss_mlp": 0.18737793, + "step": 3155, + "time_per_iteration": 2.4598453044891357 + }, + { + "auxiliary_loss_clip": 0.01108777, + "auxiliary_loss_mlp": 0.0104229, + "balance_loss_clip": 1.03558087, + "balance_loss_mlp": 1.022156, + "epoch": 0.09157913063664326, + "flos": 15114319303680.0, + "grad_norm": 3.295587692231728, + "language_loss": 0.83360159, + "learning_rate": 3.960360344719092e-06, + "loss": 0.85511231, + "num_input_tokens_seen": 88401655, + "router_z_loss_clip": 0.73242188, + "router_z_loss_mlp": 0.20147705, + "step": 3156, + "time_per_iteration": 2.3420770168304443 + }, + { + "auxiliary_loss_clip": 0.01018993, + "auxiliary_loss_mlp": 0.01009808, + "balance_loss_clip": 1.00756764, + "balance_loss_mlp": 1.00879443, + "epoch": 0.09160814810515931, + "flos": 52010075606400.0, + "grad_norm": 0.7256168469768873, + "language_loss": 0.48173767, + "learning_rate": 3.960323099312113e-06, + "loss": 0.50202572, + "num_input_tokens_seen": 88459675, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01013184, + "step": 3157, + "time_per_iteration": 2.935141086578369 + }, + { + "auxiliary_loss_clip": 0.01119467, + "auxiliary_loss_mlp": 0.01050831, + "balance_loss_clip": 1.03899455, + "balance_loss_mlp": 1.0269773, + "epoch": 0.09163716557367535, + "flos": 18508435637760.0, + "grad_norm": 2.6357531198003787, + "language_loss": 0.82579178, + "learning_rate": 3.960285836590767e-06, + "loss": 0.84749478, + "num_input_tokens_seen": 88474220, + "router_z_loss_clip": 0.80419922, + "router_z_loss_mlp": 0.23840332, + "step": 3158, + "time_per_iteration": 2.3633065223693848 + }, + { + "auxiliary_loss_clip": 0.01021087, + "auxiliary_loss_mlp": 0.01005984, + "balance_loss_clip": 1.00969648, + "balance_loss_mlp": 1.00506043, + "epoch": 0.0916661830421914, + "flos": 63607419901440.0, + "grad_norm": 0.6036916850415645, + "language_loss": 0.47537652, + "learning_rate": 3.960248556555383e-06, + "loss": 0.49564722, + "num_input_tokens_seen": 88538380, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.00921631, + "step": 3159, + "time_per_iteration": 3.0252132415771484 + }, + { + "auxiliary_loss_clip": 0.01119342, + "auxiliary_loss_mlp": 0.01045966, + "balance_loss_clip": 1.04624164, + "balance_loss_mlp": 1.02330399, + "epoch": 0.09169520051070744, + "flos": 30547989613440.0, + "grad_norm": 1.9989042180275618, + "language_loss": 0.7181226, + "learning_rate": 3.96021125920629e-06, + "loss": 0.73977566, + "num_input_tokens_seen": 88557240, + "router_z_loss_clip": 0.73144531, + "router_z_loss_mlp": 0.22680664, + "step": 3160, + "time_per_iteration": 2.4604270458221436 + }, + { + "auxiliary_loss_clip": 0.01121055, + "auxiliary_loss_mlp": 0.01049917, + "balance_loss_clip": 1.04481149, + "balance_loss_mlp": 1.02689159, + "epoch": 0.09172421797922349, + "flos": 30949665338880.0, + "grad_norm": 6.536712871577775, + "language_loss": 0.79519862, + "learning_rate": 3.960173944543819e-06, + "loss": 0.8169083, + "num_input_tokens_seen": 88574140, + "router_z_loss_clip": 0.76318359, + "router_z_loss_mlp": 0.23022461, + "step": 3161, + "time_per_iteration": 2.491046667098999 + }, + { + "auxiliary_loss_clip": 0.01123096, + "auxiliary_loss_mlp": 0.01043812, + "balance_loss_clip": 1.04265189, + "balance_loss_mlp": 1.02119792, + "epoch": 0.09175323544773954, + "flos": 27155164999680.0, + "grad_norm": 2.1867242007959544, + "language_loss": 0.9380641, + "learning_rate": 3.960136612568298e-06, + "loss": 0.95973319, + "num_input_tokens_seen": 88593970, + "router_z_loss_clip": 0.8046875, + "router_z_loss_mlp": 0.22607422, + "step": 3162, + "time_per_iteration": 2.5793182849884033 + }, + { + "auxiliary_loss_clip": 0.01124194, + "auxiliary_loss_mlp": 0.01054004, + "balance_loss_clip": 1.04322195, + "balance_loss_mlp": 1.03068709, + "epoch": 0.09178225291625558, + "flos": 11579048876160.0, + "grad_norm": 3.0383481364384544, + "language_loss": 0.89986444, + "learning_rate": 3.960099263280057e-06, + "loss": 0.92164648, + "num_input_tokens_seen": 88606315, + "router_z_loss_clip": 0.80908203, + "router_z_loss_mlp": 0.2331543, + "step": 3163, + "time_per_iteration": 2.39345383644104 + }, + { + "auxiliary_loss_clip": 0.01124991, + "auxiliary_loss_mlp": 0.01052635, + "balance_loss_clip": 1.04504776, + "balance_loss_mlp": 1.02982986, + "epoch": 0.09181127038477163, + "flos": 28585311822720.0, + "grad_norm": 1.9467904158830729, + "language_loss": 0.8946259, + "learning_rate": 3.960061896679426e-06, + "loss": 0.91640216, + "num_input_tokens_seen": 88629375, + "router_z_loss_clip": 0.79931641, + "router_z_loss_mlp": 0.22814941, + "step": 3164, + "time_per_iteration": 2.488755226135254 + }, + { + "auxiliary_loss_clip": 0.0111585, + "auxiliary_loss_mlp": 0.01041978, + "balance_loss_clip": 1.04328847, + "balance_loss_mlp": 1.02288079, + "epoch": 0.09184028785328768, + "flos": 12309606408960.0, + "grad_norm": 2.680105716034242, + "language_loss": 0.96457207, + "learning_rate": 3.960024512766736e-06, + "loss": 0.98615026, + "num_input_tokens_seen": 88642175, + "router_z_loss_clip": 0.72558594, + "router_z_loss_mlp": 0.19110107, + "step": 3165, + "time_per_iteration": 2.467393159866333 + }, + { + "auxiliary_loss_clip": 0.01032515, + "auxiliary_loss_mlp": 0.01000779, + "balance_loss_clip": 1.02067351, + "balance_loss_mlp": 0.99978966, + "epoch": 0.09186930532180372, + "flos": 66959012332800.0, + "grad_norm": 0.754401884802611, + "language_loss": 0.49390909, + "learning_rate": 3.959987111542316e-06, + "loss": 0.51424205, + "num_input_tokens_seen": 88697525, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.0098877, + "step": 3166, + "time_per_iteration": 2.914644956588745 + }, + { + "auxiliary_loss_clip": 0.01026496, + "auxiliary_loss_mlp": 0.01002564, + "balance_loss_clip": 1.01465869, + "balance_loss_mlp": 1.00166392, + "epoch": 0.09189832279031977, + "flos": 73612514597760.0, + "grad_norm": 0.802324766475095, + "language_loss": 0.46486497, + "learning_rate": 3.9599496930064965e-06, + "loss": 0.48515552, + "num_input_tokens_seen": 88757840, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.00897217, + "step": 3167, + "time_per_iteration": 3.1406939029693604 + }, + { + "auxiliary_loss_clip": 0.01119742, + "auxiliary_loss_mlp": 0.01056219, + "balance_loss_clip": 1.04005647, + "balance_loss_mlp": 1.03231716, + "epoch": 0.09192734025883582, + "flos": 16026634707840.0, + "grad_norm": 2.8149683647510075, + "language_loss": 0.87740839, + "learning_rate": 3.959912257159608e-06, + "loss": 0.89916795, + "num_input_tokens_seen": 88770885, + "router_z_loss_clip": 0.79736328, + "router_z_loss_mlp": 0.23901367, + "step": 3168, + "time_per_iteration": 2.364375352859497 + }, + { + "auxiliary_loss_clip": 0.01019926, + "auxiliary_loss_mlp": 0.01003684, + "balance_loss_clip": 1.00858212, + "balance_loss_mlp": 1.00282025, + "epoch": 0.09195635772735186, + "flos": 68865269633280.0, + "grad_norm": 0.6583360834267106, + "language_loss": 0.47074112, + "learning_rate": 3.959874804001982e-06, + "loss": 0.4909772, + "num_input_tokens_seen": 88837170, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.00866699, + "step": 3169, + "time_per_iteration": 3.0778586864471436 + }, + { + "auxiliary_loss_clip": 0.01017391, + "auxiliary_loss_mlp": 0.01004734, + "balance_loss_clip": 1.00627518, + "balance_loss_mlp": 1.00386989, + "epoch": 0.09198537519586791, + "flos": 66519630472320.0, + "grad_norm": 0.683134048013976, + "language_loss": 0.51350343, + "learning_rate": 3.959837333533948e-06, + "loss": 0.53372473, + "num_input_tokens_seen": 88896175, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.00866699, + "step": 3170, + "time_per_iteration": 3.0035064220428467 + }, + { + "auxiliary_loss_clip": 0.01017331, + "auxiliary_loss_mlp": 0.01003111, + "balance_loss_clip": 1.00629663, + "balance_loss_mlp": 1.00224721, + "epoch": 0.09201439266438396, + "flos": 60717903580800.0, + "grad_norm": 0.7664809947749314, + "language_loss": 0.46577388, + "learning_rate": 3.959799845755838e-06, + "loss": 0.48597831, + "num_input_tokens_seen": 88952660, + "router_z_loss_clip": 0.11035156, + "router_z_loss_mlp": 0.00866699, + "step": 3171, + "time_per_iteration": 2.837892770767212 + }, + { + "auxiliary_loss_clip": 0.01119664, + "auxiliary_loss_mlp": 0.01045291, + "balance_loss_clip": 1.03848028, + "balance_loss_mlp": 1.02271307, + "epoch": 0.0920434101329, + "flos": 30984508742400.0, + "grad_norm": 2.7309076381081554, + "language_loss": 0.81432593, + "learning_rate": 3.959762340667983e-06, + "loss": 0.83597547, + "num_input_tokens_seen": 88970200, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.22583008, + "step": 3172, + "time_per_iteration": 2.5870327949523926 + }, + { + "auxiliary_loss_clip": 0.01109679, + "auxiliary_loss_mlp": 0.01055343, + "balance_loss_clip": 1.03635585, + "balance_loss_mlp": 1.03276467, + "epoch": 0.09207242760141605, + "flos": 16907143996800.0, + "grad_norm": 2.380810878999906, + "language_loss": 0.73680413, + "learning_rate": 3.959724818270713e-06, + "loss": 0.75845432, + "num_input_tokens_seen": 88983390, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.22558594, + "step": 3173, + "time_per_iteration": 2.3660335540771484 + }, + { + "auxiliary_loss_clip": 0.01029961, + "auxiliary_loss_mlp": 0.01000216, + "balance_loss_clip": 1.01807666, + "balance_loss_mlp": 0.99926245, + "epoch": 0.0921014450699321, + "flos": 61562103189120.0, + "grad_norm": 0.7091300317138163, + "language_loss": 0.5110386, + "learning_rate": 3.959687278564361e-06, + "loss": 0.53134036, + "num_input_tokens_seen": 89042285, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.00952148, + "step": 3174, + "time_per_iteration": 2.908458709716797 + }, + { + "auxiliary_loss_clip": 0.01111317, + "auxiliary_loss_mlp": 0.01038595, + "balance_loss_clip": 1.03814697, + "balance_loss_mlp": 1.01807952, + "epoch": 0.09213046253844814, + "flos": 25039183962240.0, + "grad_norm": 2.578007563537855, + "language_loss": 1.02294397, + "learning_rate": 3.959649721549258e-06, + "loss": 1.04444313, + "num_input_tokens_seen": 89057525, + "router_z_loss_clip": 0.73095703, + "router_z_loss_mlp": 0.20495605, + "step": 3175, + "time_per_iteration": 2.4238598346710205 + }, + { + "auxiliary_loss_clip": 0.01114135, + "auxiliary_loss_mlp": 0.01043241, + "balance_loss_clip": 1.04074621, + "balance_loss_mlp": 1.02075839, + "epoch": 0.0921594800069642, + "flos": 33101781500160.0, + "grad_norm": 2.438173587793095, + "language_loss": 0.81563675, + "learning_rate": 3.959612147225735e-06, + "loss": 0.83721054, + "num_input_tokens_seen": 89077120, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.22485352, + "step": 3176, + "time_per_iteration": 2.5621516704559326 + }, + { + "auxiliary_loss_clip": 0.01041929, + "auxiliary_loss_mlp": 0.00999227, + "balance_loss_clip": 1.02906322, + "balance_loss_mlp": 0.99808848, + "epoch": 0.09218849747548023, + "flos": 67812184160640.0, + "grad_norm": 0.7099605891263493, + "language_loss": 0.48889697, + "learning_rate": 3.959574555594126e-06, + "loss": 0.50930858, + "num_input_tokens_seen": 89142025, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01141357, + "step": 3177, + "time_per_iteration": 3.0659852027893066 + }, + { + "auxiliary_loss_clip": 0.01116654, + "auxiliary_loss_mlp": 0.01043834, + "balance_loss_clip": 1.04104996, + "balance_loss_mlp": 1.02280498, + "epoch": 0.09221751494399628, + "flos": 12059418539520.0, + "grad_norm": 2.537869325796042, + "language_loss": 0.82281983, + "learning_rate": 3.959536946654761e-06, + "loss": 0.84442472, + "num_input_tokens_seen": 89157130, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.21020508, + "step": 3178, + "time_per_iteration": 2.413411855697632 + }, + { + "auxiliary_loss_clip": 0.0111766, + "auxiliary_loss_mlp": 0.01049345, + "balance_loss_clip": 1.04245186, + "balance_loss_mlp": 1.02803075, + "epoch": 0.09224653241251234, + "flos": 23832690508800.0, + "grad_norm": 2.2623194756745364, + "language_loss": 0.75940168, + "learning_rate": 3.959499320407972e-06, + "loss": 0.78107178, + "num_input_tokens_seen": 89169975, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.21350098, + "step": 3179, + "time_per_iteration": 2.376497983932495 + }, + { + "auxiliary_loss_clip": 0.01112458, + "auxiliary_loss_mlp": 0.01039988, + "balance_loss_clip": 1.0411489, + "balance_loss_mlp": 1.02222621, + "epoch": 0.09227554988102837, + "flos": 19820716110720.0, + "grad_norm": 2.561777929972142, + "language_loss": 0.85354137, + "learning_rate": 3.959461676854092e-06, + "loss": 0.87506586, + "num_input_tokens_seen": 89184900, + "router_z_loss_clip": 0.71337891, + "router_z_loss_mlp": 0.1774292, + "step": 3180, + "time_per_iteration": 2.4698710441589355 + }, + { + "auxiliary_loss_clip": 0.01125055, + "auxiliary_loss_mlp": 0.01054081, + "balance_loss_clip": 1.04248977, + "balance_loss_mlp": 1.03054893, + "epoch": 0.09230456734954443, + "flos": 23029199412480.0, + "grad_norm": 2.8539039211528996, + "language_loss": 1.13607717, + "learning_rate": 3.959424015993455e-06, + "loss": 1.1578685, + "num_input_tokens_seen": 89197425, + "router_z_loss_clip": 0.82568359, + "router_z_loss_mlp": 0.23547363, + "step": 3181, + "time_per_iteration": 2.41558575630188 + }, + { + "auxiliary_loss_clip": 0.01114255, + "auxiliary_loss_mlp": 0.01043515, + "balance_loss_clip": 1.04126215, + "balance_loss_mlp": 1.02261758, + "epoch": 0.09233358481806048, + "flos": 16584755702400.0, + "grad_norm": 2.513322652865331, + "language_loss": 0.71754134, + "learning_rate": 3.959386337826391e-06, + "loss": 0.73911905, + "num_input_tokens_seen": 89211430, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.20910645, + "step": 3182, + "time_per_iteration": 2.472635269165039 + }, + { + "auxiliary_loss_clip": 0.01040265, + "auxiliary_loss_mlp": 0.0100824, + "balance_loss_clip": 1.02731371, + "balance_loss_mlp": 1.00701213, + "epoch": 0.09236260228657651, + "flos": 60729738531840.0, + "grad_norm": 0.755172321248207, + "language_loss": 0.54867738, + "learning_rate": 3.959348642353234e-06, + "loss": 0.56916243, + "num_input_tokens_seen": 89272660, + "router_z_loss_clip": 0.12988281, + "router_z_loss_mlp": 0.01226807, + "step": 3183, + "time_per_iteration": 3.0754005908966064 + }, + { + "auxiliary_loss_clip": 0.01105241, + "auxiliary_loss_mlp": 0.01044396, + "balance_loss_clip": 1.03655505, + "balance_loss_mlp": 1.02467871, + "epoch": 0.09239161975509257, + "flos": 28505186519040.0, + "grad_norm": 2.197444212366098, + "language_loss": 0.90689385, + "learning_rate": 3.959310929574317e-06, + "loss": 0.92839032, + "num_input_tokens_seen": 89294255, + "router_z_loss_clip": 0.68652344, + "router_z_loss_mlp": 0.19714355, + "step": 3184, + "time_per_iteration": 2.5027928352355957 + }, + { + "auxiliary_loss_clip": 0.01113504, + "auxiliary_loss_mlp": 0.01051791, + "balance_loss_clip": 1.03845859, + "balance_loss_mlp": 1.02936745, + "epoch": 0.09242063722360862, + "flos": 27373092906240.0, + "grad_norm": 1.9891104216767885, + "language_loss": 0.73492664, + "learning_rate": 3.959273199489974e-06, + "loss": 0.75657964, + "num_input_tokens_seen": 89310450, + "router_z_loss_clip": 0.75146484, + "router_z_loss_mlp": 0.2244873, + "step": 3185, + "time_per_iteration": 2.3702991008758545 + }, + { + "auxiliary_loss_clip": 0.01113865, + "auxiliary_loss_mlp": 0.01048339, + "balance_loss_clip": 1.03745627, + "balance_loss_mlp": 1.02866983, + "epoch": 0.09244965469212466, + "flos": 20987722949760.0, + "grad_norm": 2.856113750959902, + "language_loss": 0.84799826, + "learning_rate": 3.959235452100536e-06, + "loss": 0.86962032, + "num_input_tokens_seen": 89322520, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.19677734, + "step": 3186, + "time_per_iteration": 2.418060302734375 + }, + { + "auxiliary_loss_clip": 0.01116856, + "auxiliary_loss_mlp": 0.01061037, + "balance_loss_clip": 1.03757513, + "balance_loss_mlp": 1.03926945, + "epoch": 0.09247867216064071, + "flos": 31824204785280.0, + "grad_norm": 2.3752894957525257, + "language_loss": 1.02399099, + "learning_rate": 3.9591976874063385e-06, + "loss": 1.04576993, + "num_input_tokens_seen": 89340060, + "router_z_loss_clip": 0.79150391, + "router_z_loss_mlp": 0.2175293, + "step": 3187, + "time_per_iteration": 2.4825847148895264 + }, + { + "auxiliary_loss_clip": 0.01105564, + "auxiliary_loss_mlp": 0.01037645, + "balance_loss_clip": 1.03284454, + "balance_loss_mlp": 1.01686692, + "epoch": 0.09250768962915676, + "flos": 29201668698240.0, + "grad_norm": 11.127857793763718, + "language_loss": 0.63303888, + "learning_rate": 3.959159905407713e-06, + "loss": 0.65447098, + "num_input_tokens_seen": 89356130, + "router_z_loss_clip": 0.72607422, + "router_z_loss_mlp": 0.2076416, + "step": 3188, + "time_per_iteration": 2.502688407897949 + }, + { + "auxiliary_loss_clip": 0.01119476, + "auxiliary_loss_mlp": 0.01046537, + "balance_loss_clip": 1.03976583, + "balance_loss_mlp": 1.02404189, + "epoch": 0.0925367070976728, + "flos": 28102987123200.0, + "grad_norm": 2.0882961688229855, + "language_loss": 0.83949339, + "learning_rate": 3.959122106104996e-06, + "loss": 0.86115348, + "num_input_tokens_seen": 89376080, + "router_z_loss_clip": 0.79736328, + "router_z_loss_mlp": 0.22485352, + "step": 3189, + "time_per_iteration": 2.4908790588378906 + }, + { + "auxiliary_loss_clip": 0.01098851, + "auxiliary_loss_mlp": 0.01043858, + "balance_loss_clip": 1.03187275, + "balance_loss_mlp": 1.02430725, + "epoch": 0.09256572456618885, + "flos": 36724368971520.0, + "grad_norm": 2.956652056532513, + "language_loss": 0.72063887, + "learning_rate": 3.959084289498519e-06, + "loss": 0.74206597, + "num_input_tokens_seen": 89392510, + "router_z_loss_clip": 0.66992188, + "router_z_loss_mlp": 0.19543457, + "step": 3190, + "time_per_iteration": 2.437180280685425 + }, + { + "auxiliary_loss_clip": 0.01110001, + "auxiliary_loss_mlp": 0.01047728, + "balance_loss_clip": 1.03329206, + "balance_loss_mlp": 1.02798116, + "epoch": 0.09259474203470489, + "flos": 21171226389120.0, + "grad_norm": 1.939268413505635, + "language_loss": 0.691535, + "learning_rate": 3.959046455588617e-06, + "loss": 0.71311224, + "num_input_tokens_seen": 89407370, + "router_z_loss_clip": 0.7668457, + "router_z_loss_mlp": 0.19763184, + "step": 3191, + "time_per_iteration": 2.332169532775879 + }, + { + "auxiliary_loss_clip": 0.01117319, + "auxiliary_loss_mlp": 0.0105054, + "balance_loss_clip": 1.03828466, + "balance_loss_mlp": 1.0268178, + "epoch": 0.09262375950322094, + "flos": 23358465244800.0, + "grad_norm": 2.3357944850094765, + "language_loss": 1.07250428, + "learning_rate": 3.9590086043756235e-06, + "loss": 1.09418297, + "num_input_tokens_seen": 89420665, + "router_z_loss_clip": 0.79052734, + "router_z_loss_mlp": 0.23730469, + "step": 3192, + "time_per_iteration": 2.4587695598602295 + }, + { + "auxiliary_loss_clip": 0.01106406, + "auxiliary_loss_mlp": 0.0104618, + "balance_loss_clip": 1.03516519, + "balance_loss_mlp": 1.02589083, + "epoch": 0.09265277697173699, + "flos": 14128022995200.0, + "grad_norm": 2.35897538781564, + "language_loss": 0.86139542, + "learning_rate": 3.958970735859874e-06, + "loss": 0.88292122, + "num_input_tokens_seen": 89433875, + "router_z_loss_clip": 0.71240234, + "router_z_loss_mlp": 0.203125, + "step": 3193, + "time_per_iteration": 2.347973346710205 + }, + { + "auxiliary_loss_clip": 0.01107563, + "auxiliary_loss_mlp": 0.01041051, + "balance_loss_clip": 1.03464818, + "balance_loss_mlp": 1.01965272, + "epoch": 0.09268179444025303, + "flos": 21282215201280.0, + "grad_norm": 2.584207418246578, + "language_loss": 0.84963322, + "learning_rate": 3.958932850041702e-06, + "loss": 0.87111938, + "num_input_tokens_seen": 89446795, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.21398926, + "step": 3194, + "time_per_iteration": 2.475141763687134 + }, + { + "auxiliary_loss_clip": 0.01107593, + "auxiliary_loss_mlp": 0.01038638, + "balance_loss_clip": 1.03837967, + "balance_loss_mlp": 1.01982105, + "epoch": 0.09271081190876908, + "flos": 32117754430080.0, + "grad_norm": 2.1121198127094223, + "language_loss": 0.82992661, + "learning_rate": 3.958894946921443e-06, + "loss": 0.85138893, + "num_input_tokens_seen": 89465550, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.18811035, + "step": 3195, + "time_per_iteration": 2.470963478088379 + }, + { + "auxiliary_loss_clip": 0.01021971, + "auxiliary_loss_mlp": 0.01010071, + "balance_loss_clip": 1.01051521, + "balance_loss_mlp": 1.0090394, + "epoch": 0.09273982937728513, + "flos": 60344822125440.0, + "grad_norm": 0.7183544185442067, + "language_loss": 0.5180186, + "learning_rate": 3.958857026499429e-06, + "loss": 0.53833902, + "num_input_tokens_seen": 89520390, + "router_z_loss_clip": 0.11474609, + "router_z_loss_mlp": 0.01031494, + "step": 3196, + "time_per_iteration": 2.9149997234344482 + }, + { + "auxiliary_loss_clip": 0.01112103, + "auxiliary_loss_mlp": 0.0104198, + "balance_loss_clip": 1.03917623, + "balance_loss_mlp": 1.02153599, + "epoch": 0.09276884684580117, + "flos": 10406246751360.0, + "grad_norm": 2.566745182178783, + "language_loss": 0.85763532, + "learning_rate": 3.958819088775999e-06, + "loss": 0.87917614, + "num_input_tokens_seen": 89531055, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.2043457, + "step": 3197, + "time_per_iteration": 2.320331335067749 + }, + { + "auxiliary_loss_clip": 0.01103283, + "auxiliary_loss_mlp": 0.01041292, + "balance_loss_clip": 1.0358479, + "balance_loss_mlp": 1.02195048, + "epoch": 0.09279786431431722, + "flos": 34816435925760.0, + "grad_norm": 2.399133482994175, + "language_loss": 0.84509474, + "learning_rate": 3.958781133751486e-06, + "loss": 0.86654043, + "num_input_tokens_seen": 89547495, + "router_z_loss_clip": 0.67431641, + "router_z_loss_mlp": 0.19342041, + "step": 3198, + "time_per_iteration": 4.641801357269287 + }, + { + "auxiliary_loss_clip": 0.01113655, + "auxiliary_loss_mlp": 0.01045617, + "balance_loss_clip": 1.03791487, + "balance_loss_mlp": 1.0245291, + "epoch": 0.09282688178283327, + "flos": 48024218638080.0, + "grad_norm": 2.1510279121805755, + "language_loss": 0.95385522, + "learning_rate": 3.9587431614262245e-06, + "loss": 0.97544789, + "num_input_tokens_seen": 89567925, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.21051025, + "step": 3199, + "time_per_iteration": 4.8707544803619385 + }, + { + "auxiliary_loss_clip": 0.01019404, + "auxiliary_loss_mlp": 0.010083, + "balance_loss_clip": 1.00815105, + "balance_loss_mlp": 1.0072515, + "epoch": 0.09285589925134931, + "flos": 74768698915200.0, + "grad_norm": 0.6545636396882076, + "language_loss": 0.51809692, + "learning_rate": 3.958705171800551e-06, + "loss": 0.53837401, + "num_input_tokens_seen": 89630450, + "router_z_loss_clip": 0.11230469, + "router_z_loss_mlp": 0.01049805, + "step": 3200, + "time_per_iteration": 3.092639923095703 + }, + { + "auxiliary_loss_clip": 0.01111156, + "auxiliary_loss_mlp": 0.01044478, + "balance_loss_clip": 1.039814, + "balance_loss_mlp": 1.0254997, + "epoch": 0.09288491671986536, + "flos": 28286281094400.0, + "grad_norm": 8.870514531293642, + "language_loss": 1.0747149, + "learning_rate": 3.958667164874802e-06, + "loss": 1.09627116, + "num_input_tokens_seen": 89648655, + "router_z_loss_clip": 0.71386719, + "router_z_loss_mlp": 0.18969727, + "step": 3201, + "time_per_iteration": 2.4373693466186523 + }, + { + "auxiliary_loss_clip": 0.01018945, + "auxiliary_loss_mlp": 0.01002585, + "balance_loss_clip": 1.00799441, + "balance_loss_mlp": 1.00161374, + "epoch": 0.09291393418838141, + "flos": 63963255144960.0, + "grad_norm": 0.7467692405133212, + "language_loss": 0.54966342, + "learning_rate": 3.958629140649311e-06, + "loss": 0.5698787, + "num_input_tokens_seen": 89701070, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.00970459, + "step": 3202, + "time_per_iteration": 2.851064443588257 + }, + { + "auxiliary_loss_clip": 0.01114924, + "auxiliary_loss_mlp": 0.01050016, + "balance_loss_clip": 1.03907371, + "balance_loss_mlp": 1.02719903, + "epoch": 0.09294295165689745, + "flos": 16136261976960.0, + "grad_norm": 3.7660272582358703, + "language_loss": 0.82960653, + "learning_rate": 3.958591099124415e-06, + "loss": 0.85125589, + "num_input_tokens_seen": 89714040, + "router_z_loss_clip": 0.75878906, + "router_z_loss_mlp": 0.22827148, + "step": 3203, + "time_per_iteration": 4.8590192794799805 + }, + { + "auxiliary_loss_clip": 0.01017657, + "auxiliary_loss_mlp": 0.0100229, + "balance_loss_clip": 1.00672817, + "balance_loss_mlp": 1.0013181, + "epoch": 0.0929719691254135, + "flos": 61509280435200.0, + "grad_norm": 0.8024039501612793, + "language_loss": 0.47712433, + "learning_rate": 3.95855304030045e-06, + "loss": 0.49732378, + "num_input_tokens_seen": 89763855, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.00970459, + "step": 3204, + "time_per_iteration": 5.1852686405181885 + }, + { + "auxiliary_loss_clip": 0.01119106, + "auxiliary_loss_mlp": 0.01047902, + "balance_loss_clip": 1.04057252, + "balance_loss_mlp": 1.02580047, + "epoch": 0.09300098659392955, + "flos": 33324212972160.0, + "grad_norm": 2.3472468387154786, + "language_loss": 0.83083117, + "learning_rate": 3.9585149641777515e-06, + "loss": 0.85250115, + "num_input_tokens_seen": 89786740, + "router_z_loss_clip": 0.78564453, + "router_z_loss_mlp": 0.22106934, + "step": 3205, + "time_per_iteration": 2.4815564155578613 + }, + { + "auxiliary_loss_clip": 0.01112781, + "auxiliary_loss_mlp": 0.01045706, + "balance_loss_clip": 1.04077506, + "balance_loss_mlp": 1.02626276, + "epoch": 0.09303000406244559, + "flos": 15116867832960.0, + "grad_norm": 2.480497677198515, + "language_loss": 0.73144567, + "learning_rate": 3.958476870756657e-06, + "loss": 0.75303048, + "num_input_tokens_seen": 89798675, + "router_z_loss_clip": 0.71972656, + "router_z_loss_mlp": 0.19445801, + "step": 3206, + "time_per_iteration": 2.393794536590576 + }, + { + "auxiliary_loss_clip": 0.01105634, + "auxiliary_loss_mlp": 0.01039442, + "balance_loss_clip": 1.04002964, + "balance_loss_mlp": 1.02145934, + "epoch": 0.09305902153096164, + "flos": 18799052728320.0, + "grad_norm": 2.221033699989699, + "language_loss": 0.81134045, + "learning_rate": 3.958438760037502e-06, + "loss": 0.83279121, + "num_input_tokens_seen": 89813525, + "router_z_loss_clip": 0.65625, + "router_z_loss_mlp": 0.17974854, + "step": 3207, + "time_per_iteration": 2.3534693717956543 + }, + { + "auxiliary_loss_clip": 0.01020381, + "auxiliary_loss_mlp": 0.01001447, + "balance_loss_clip": 1.00912797, + "balance_loss_mlp": 1.0004636, + "epoch": 0.09308803899947768, + "flos": 60903955549440.0, + "grad_norm": 0.6937811288168536, + "language_loss": 0.47518399, + "learning_rate": 3.9584006320206225e-06, + "loss": 0.49540228, + "num_input_tokens_seen": 89873600, + "router_z_loss_clip": 0.11230469, + "router_z_loss_mlp": 0.00982666, + "step": 3208, + "time_per_iteration": 2.9304986000061035 + }, + { + "auxiliary_loss_clip": 0.01019615, + "auxiliary_loss_mlp": 0.01001072, + "balance_loss_clip": 1.00852942, + "balance_loss_mlp": 1.00017178, + "epoch": 0.09311705646799373, + "flos": 69811939681920.0, + "grad_norm": 0.7298561718488314, + "language_loss": 0.55531275, + "learning_rate": 3.9583624867063575e-06, + "loss": 0.57551956, + "num_input_tokens_seen": 89941210, + "router_z_loss_clip": 0.11083984, + "router_z_loss_mlp": 0.00897217, + "step": 3209, + "time_per_iteration": 3.242631673812866 + }, + { + "auxiliary_loss_clip": 0.01109428, + "auxiliary_loss_mlp": 0.01043038, + "balance_loss_clip": 1.03478014, + "balance_loss_mlp": 1.02141321, + "epoch": 0.09314607393650978, + "flos": 36644104022400.0, + "grad_norm": 2.2427040295170904, + "language_loss": 0.73185533, + "learning_rate": 3.958324324095042e-06, + "loss": 0.75338, + "num_input_tokens_seen": 89959630, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.21630859, + "step": 3210, + "time_per_iteration": 2.518173933029175 + }, + { + "auxiliary_loss_clip": 0.011082, + "auxiliary_loss_mlp": 0.01047051, + "balance_loss_clip": 1.03637004, + "balance_loss_mlp": 1.02779841, + "epoch": 0.09317509140502582, + "flos": 74729598192000.0, + "grad_norm": 2.290115886809261, + "language_loss": 0.88690561, + "learning_rate": 3.9582861441870134e-06, + "loss": 0.90845811, + "num_input_tokens_seen": 89981295, + "router_z_loss_clip": 0.71826172, + "router_z_loss_mlp": 0.19226074, + "step": 3211, + "time_per_iteration": 2.8352901935577393 + }, + { + "auxiliary_loss_clip": 0.01019334, + "auxiliary_loss_mlp": 0.01001258, + "balance_loss_clip": 1.00807786, + "balance_loss_mlp": 1.0003345, + "epoch": 0.09320410887354187, + "flos": 70617944396160.0, + "grad_norm": 0.7660104849541195, + "language_loss": 0.57188976, + "learning_rate": 3.95824794698261e-06, + "loss": 0.59209567, + "num_input_tokens_seen": 90047275, + "router_z_loss_clip": 0.11279297, + "router_z_loss_mlp": 0.00921631, + "step": 3212, + "time_per_iteration": 3.0192642211914062 + }, + { + "auxiliary_loss_clip": 0.011066, + "auxiliary_loss_mlp": 0.01042316, + "balance_loss_clip": 1.0332458, + "balance_loss_mlp": 1.02182412, + "epoch": 0.09323312634205792, + "flos": 27225480211200.0, + "grad_norm": 2.221281351292524, + "language_loss": 1.03274035, + "learning_rate": 3.958209732482167e-06, + "loss": 1.0542295, + "num_input_tokens_seen": 90062895, + "router_z_loss_clip": 0.73242188, + "router_z_loss_mlp": 0.20507812, + "step": 3213, + "time_per_iteration": 2.4694972038269043 + }, + { + "auxiliary_loss_clip": 0.01113236, + "auxiliary_loss_mlp": 0.01048197, + "balance_loss_clip": 1.03712821, + "balance_loss_mlp": 1.02460527, + "epoch": 0.09326214381057396, + "flos": 32047858154880.0, + "grad_norm": 2.1004582067717115, + "language_loss": 0.92093217, + "learning_rate": 3.958171500686024e-06, + "loss": 0.94254649, + "num_input_tokens_seen": 90078900, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.23596191, + "step": 3214, + "time_per_iteration": 2.509491443634033 + }, + { + "auxiliary_loss_clip": 0.01109749, + "auxiliary_loss_mlp": 0.01050973, + "balance_loss_clip": 1.03432441, + "balance_loss_mlp": 1.02901495, + "epoch": 0.09329116127909001, + "flos": 22958604910080.0, + "grad_norm": 2.1965477476327426, + "language_loss": 0.71371686, + "learning_rate": 3.958133251594518e-06, + "loss": 0.73532414, + "num_input_tokens_seen": 90094855, + "router_z_loss_clip": 0.75439453, + "router_z_loss_mlp": 0.21984863, + "step": 3215, + "time_per_iteration": 2.456042528152466 + }, + { + "auxiliary_loss_clip": 0.0111009, + "auxiliary_loss_mlp": 0.01043887, + "balance_loss_clip": 1.03607464, + "balance_loss_mlp": 1.02163076, + "epoch": 0.09332017874760606, + "flos": 16500720326400.0, + "grad_norm": 4.839701875786942, + "language_loss": 0.74993378, + "learning_rate": 3.958094985207987e-06, + "loss": 0.77147365, + "num_input_tokens_seen": 90108770, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.22253418, + "step": 3216, + "time_per_iteration": 2.3274636268615723 + }, + { + "auxiliary_loss_clip": 0.01105624, + "auxiliary_loss_mlp": 0.01036113, + "balance_loss_clip": 1.03547263, + "balance_loss_mlp": 1.01768899, + "epoch": 0.0933491962161221, + "flos": 16428380256000.0, + "grad_norm": 2.534293014864353, + "language_loss": 0.75117528, + "learning_rate": 3.958056701526768e-06, + "loss": 0.77259266, + "num_input_tokens_seen": 90122315, + "router_z_loss_clip": 0.70166016, + "router_z_loss_mlp": 0.18408203, + "step": 3217, + "time_per_iteration": 2.3959667682647705 + }, + { + "auxiliary_loss_clip": 0.01111476, + "auxiliary_loss_mlp": 0.01044498, + "balance_loss_clip": 1.03686142, + "balance_loss_mlp": 1.02326691, + "epoch": 0.09337821368463815, + "flos": 14458021966080.0, + "grad_norm": 1.9621550392694884, + "language_loss": 0.91322422, + "learning_rate": 3.9580184005512e-06, + "loss": 0.934784, + "num_input_tokens_seen": 90137370, + "router_z_loss_clip": 0.74511719, + "router_z_loss_mlp": 0.21228027, + "step": 3218, + "time_per_iteration": 2.3683173656463623 + }, + { + "auxiliary_loss_clip": 0.01113967, + "auxiliary_loss_mlp": 0.0104796, + "balance_loss_clip": 1.03820717, + "balance_loss_mlp": 1.0255487, + "epoch": 0.0934072311531542, + "flos": 27014918601600.0, + "grad_norm": 1.7942530502606044, + "language_loss": 0.84849209, + "learning_rate": 3.957980082281621e-06, + "loss": 0.87011135, + "num_input_tokens_seen": 90159530, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.22412109, + "step": 3219, + "time_per_iteration": 2.508178234100342 + }, + { + "auxiliary_loss_clip": 0.0110837, + "auxiliary_loss_mlp": 0.01044332, + "balance_loss_clip": 1.03843999, + "balance_loss_mlp": 1.02599716, + "epoch": 0.09343624862167024, + "flos": 29567872615680.0, + "grad_norm": 1.515975509542529, + "language_loss": 0.81431055, + "learning_rate": 3.957941746718371e-06, + "loss": 0.8358376, + "num_input_tokens_seen": 90190565, + "router_z_loss_clip": 0.70019531, + "router_z_loss_mlp": 0.18334961, + "step": 3220, + "time_per_iteration": 2.698349952697754 + }, + { + "auxiliary_loss_clip": 0.01120432, + "auxiliary_loss_mlp": 0.01052506, + "balance_loss_clip": 1.03961658, + "balance_loss_mlp": 1.02893806, + "epoch": 0.0934652660901863, + "flos": 11213438451840.0, + "grad_norm": 2.185759190857939, + "language_loss": 0.7051965, + "learning_rate": 3.9579033938617855e-06, + "loss": 0.72692585, + "num_input_tokens_seen": 90203065, + "router_z_loss_clip": 0.80908203, + "router_z_loss_mlp": 0.23583984, + "step": 3221, + "time_per_iteration": 2.3916146755218506 + }, + { + "auxiliary_loss_clip": 0.01112148, + "auxiliary_loss_mlp": 0.0105881, + "balance_loss_clip": 1.03691721, + "balance_loss_mlp": 1.03328776, + "epoch": 0.09349428355870233, + "flos": 11173463078400.0, + "grad_norm": 3.4302947507099772, + "language_loss": 0.75942314, + "learning_rate": 3.957865023712205e-06, + "loss": 0.7811327, + "num_input_tokens_seen": 90213275, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.25524902, + "step": 3222, + "time_per_iteration": 2.3563783168792725 + }, + { + "auxiliary_loss_clip": 0.01108097, + "auxiliary_loss_mlp": 0.01043861, + "balance_loss_clip": 1.03661191, + "balance_loss_mlp": 1.02488279, + "epoch": 0.09352330102721838, + "flos": 74729283989760.0, + "grad_norm": 3.9138708081841376, + "language_loss": 0.89159632, + "learning_rate": 3.957826636269969e-06, + "loss": 0.91311586, + "num_input_tokens_seen": 90240170, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.18988037, + "step": 3223, + "time_per_iteration": 2.8415470123291016 + }, + { + "auxiliary_loss_clip": 0.01107399, + "auxiliary_loss_mlp": 0.01044548, + "balance_loss_clip": 1.03409147, + "balance_loss_mlp": 1.02244639, + "epoch": 0.09355231849573444, + "flos": 13764158138880.0, + "grad_norm": 3.6631211272946724, + "language_loss": 0.88551438, + "learning_rate": 3.957788231535416e-06, + "loss": 0.90703392, + "num_input_tokens_seen": 90250905, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.22106934, + "step": 3224, + "time_per_iteration": 2.3665566444396973 + }, + { + "auxiliary_loss_clip": 0.01118303, + "auxiliary_loss_mlp": 0.01053467, + "balance_loss_clip": 1.03802395, + "balance_loss_mlp": 1.02829003, + "epoch": 0.09358133596425047, + "flos": 31022249788800.0, + "grad_norm": 3.6844308155624335, + "language_loss": 1.0420804, + "learning_rate": 3.9577498095088855e-06, + "loss": 1.06379807, + "num_input_tokens_seen": 90267325, + "router_z_loss_clip": 0.80273438, + "router_z_loss_mlp": 0.25183105, + "step": 3225, + "time_per_iteration": 2.507065534591675 + }, + { + "auxiliary_loss_clip": 0.01031511, + "auxiliary_loss_mlp": 0.01000549, + "balance_loss_clip": 1.01917863, + "balance_loss_mlp": 0.99927962, + "epoch": 0.09361035343276652, + "flos": 72276284931840.0, + "grad_norm": 0.7219205186355253, + "language_loss": 0.48110566, + "learning_rate": 3.957711370190715e-06, + "loss": 0.50142622, + "num_input_tokens_seen": 90318240, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01269531, + "step": 3226, + "time_per_iteration": 2.9319117069244385 + }, + { + "auxiliary_loss_clip": 0.01117561, + "auxiliary_loss_mlp": 0.01042944, + "balance_loss_clip": 1.03760552, + "balance_loss_mlp": 1.02074742, + "epoch": 0.09363937090128258, + "flos": 17047739508480.0, + "grad_norm": 2.696890670999953, + "language_loss": 1.00084591, + "learning_rate": 3.957672913581247e-06, + "loss": 1.02245092, + "num_input_tokens_seen": 90330885, + "router_z_loss_clip": 0.79980469, + "router_z_loss_mlp": 0.22192383, + "step": 3227, + "time_per_iteration": 2.402125597000122 + }, + { + "auxiliary_loss_clip": 0.01114102, + "auxiliary_loss_mlp": 0.01053791, + "balance_loss_clip": 1.03433704, + "balance_loss_mlp": 1.02955532, + "epoch": 0.09366838836979861, + "flos": 12157943996160.0, + "grad_norm": 2.444589404264395, + "language_loss": 0.80020344, + "learning_rate": 3.957634439680819e-06, + "loss": 0.82188237, + "num_input_tokens_seen": 90343200, + "router_z_loss_clip": 0.79785156, + "router_z_loss_mlp": 0.24206543, + "step": 3228, + "time_per_iteration": 2.3282110691070557 + }, + { + "auxiliary_loss_clip": 0.01106875, + "auxiliary_loss_mlp": 0.01048523, + "balance_loss_clip": 1.03254175, + "balance_loss_mlp": 1.02792382, + "epoch": 0.09369740583831467, + "flos": 32117160936960.0, + "grad_norm": 2.273575415127573, + "language_loss": 0.70786929, + "learning_rate": 3.9575959484897715e-06, + "loss": 0.72942328, + "num_input_tokens_seen": 90359770, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.20617676, + "step": 3229, + "time_per_iteration": 2.5081937313079834 + }, + { + "auxiliary_loss_clip": 0.01108819, + "auxiliary_loss_mlp": 0.01047589, + "balance_loss_clip": 1.03397965, + "balance_loss_mlp": 1.02365196, + "epoch": 0.09372642330683072, + "flos": 10662194995200.0, + "grad_norm": 3.0299160781340353, + "language_loss": 0.91680574, + "learning_rate": 3.957557440008444e-06, + "loss": 0.93836975, + "num_input_tokens_seen": 90369560, + "router_z_loss_clip": 0.74804688, + "router_z_loss_mlp": 0.23937988, + "step": 3230, + "time_per_iteration": 2.338116407394409 + }, + { + "auxiliary_loss_clip": 0.01020701, + "auxiliary_loss_mlp": 0.01003187, + "balance_loss_clip": 1.00883949, + "balance_loss_mlp": 1.00177991, + "epoch": 0.09375544077534675, + "flos": 69342043426560.0, + "grad_norm": 0.7045702364703668, + "language_loss": 0.51977265, + "learning_rate": 3.957518914237177e-06, + "loss": 0.54001153, + "num_input_tokens_seen": 90436565, + "router_z_loss_clip": 0.11865234, + "router_z_loss_mlp": 0.01403809, + "step": 3231, + "time_per_iteration": 3.17284893989563 + }, + { + "auxiliary_loss_clip": 0.01098964, + "auxiliary_loss_mlp": 0.01042601, + "balance_loss_clip": 1.03142095, + "balance_loss_mlp": 1.02337229, + "epoch": 0.0937844582438628, + "flos": 26206086067200.0, + "grad_norm": 1.7029143209699613, + "language_loss": 0.64734119, + "learning_rate": 3.957480371176312e-06, + "loss": 0.66875684, + "num_input_tokens_seen": 90453005, + "router_z_loss_clip": 0.67529297, + "router_z_loss_mlp": 0.19213867, + "step": 3232, + "time_per_iteration": 2.4122328758239746 + }, + { + "auxiliary_loss_clip": 0.01118375, + "auxiliary_loss_mlp": 0.01051207, + "balance_loss_clip": 1.03797925, + "balance_loss_mlp": 1.02634001, + "epoch": 0.09381347571237886, + "flos": 18362393953920.0, + "grad_norm": 2.4955093664837866, + "language_loss": 0.88459647, + "learning_rate": 3.957441810826188e-06, + "loss": 0.90629232, + "num_input_tokens_seen": 90467695, + "router_z_loss_clip": 0.80322266, + "router_z_loss_mlp": 0.24865723, + "step": 3233, + "time_per_iteration": 2.3767971992492676 + }, + { + "auxiliary_loss_clip": 0.01116983, + "auxiliary_loss_mlp": 0.01056093, + "balance_loss_clip": 1.03589344, + "balance_loss_mlp": 1.02948523, + "epoch": 0.0938424931808949, + "flos": 25219161354240.0, + "grad_norm": 3.153245658258429, + "language_loss": 1.09087372, + "learning_rate": 3.957403233187145e-06, + "loss": 1.1126045, + "num_input_tokens_seen": 90480090, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.26623535, + "step": 3234, + "time_per_iteration": 2.4232966899871826 + }, + { + "auxiliary_loss_clip": 0.01016642, + "auxiliary_loss_mlp": 0.01005714, + "balance_loss_clip": 1.00458217, + "balance_loss_mlp": 1.00445604, + "epoch": 0.09387151064941095, + "flos": 61853627975040.0, + "grad_norm": 0.829857994629427, + "language_loss": 0.50134653, + "learning_rate": 3.957364638259524e-06, + "loss": 0.52157009, + "num_input_tokens_seen": 90540995, + "router_z_loss_clip": 0.12060547, + "router_z_loss_mlp": 0.01257324, + "step": 3235, + "time_per_iteration": 2.988847494125366 + }, + { + "auxiliary_loss_clip": 0.01114191, + "auxiliary_loss_mlp": 0.01043723, + "balance_loss_clip": 1.03479695, + "balance_loss_mlp": 1.0197854, + "epoch": 0.093900528117927, + "flos": 11904299902080.0, + "grad_norm": 3.2198541649450068, + "language_loss": 1.02081108, + "learning_rate": 3.957326026043668e-06, + "loss": 1.04239011, + "num_input_tokens_seen": 90553155, + "router_z_loss_clip": 0.79443359, + "router_z_loss_mlp": 0.23950195, + "step": 3236, + "time_per_iteration": 2.34391713142395 + }, + { + "auxiliary_loss_clip": 0.01015474, + "auxiliary_loss_mlp": 0.01005166, + "balance_loss_clip": 1.00323164, + "balance_loss_mlp": 1.00408101, + "epoch": 0.09392954558644304, + "flos": 59445668304000.0, + "grad_norm": 0.6916535375129561, + "language_loss": 0.50549847, + "learning_rate": 3.957287396539916e-06, + "loss": 0.52570486, + "num_input_tokens_seen": 90611195, + "router_z_loss_clip": 0.12255859, + "router_z_loss_mlp": 0.01086426, + "step": 3237, + "time_per_iteration": 2.933785915374756 + }, + { + "auxiliary_loss_clip": 0.01015576, + "auxiliary_loss_mlp": 0.01004479, + "balance_loss_clip": 1.00305796, + "balance_loss_mlp": 1.00324476, + "epoch": 0.09395856305495909, + "flos": 58359519907200.0, + "grad_norm": 0.7807828450407358, + "language_loss": 0.53646982, + "learning_rate": 3.95724874974861e-06, + "loss": 0.55667037, + "num_input_tokens_seen": 90669840, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.0123291, + "step": 3238, + "time_per_iteration": 2.9674794673919678 + }, + { + "auxiliary_loss_clip": 0.01111258, + "auxiliary_loss_mlp": 0.01049822, + "balance_loss_clip": 1.03763735, + "balance_loss_mlp": 1.02819705, + "epoch": 0.09398758052347513, + "flos": 13476159400320.0, + "grad_norm": 2.352566867708559, + "language_loss": 0.75079888, + "learning_rate": 3.95721008567009e-06, + "loss": 0.77240968, + "num_input_tokens_seen": 90685990, + "router_z_loss_clip": 0.73632812, + "router_z_loss_mlp": 0.21630859, + "step": 3239, + "time_per_iteration": 2.3782472610473633 + }, + { + "auxiliary_loss_clip": 0.01108862, + "auxiliary_loss_mlp": 0.01039642, + "balance_loss_clip": 1.03435469, + "balance_loss_mlp": 1.01653934, + "epoch": 0.09401659799199118, + "flos": 15442537795200.0, + "grad_norm": 2.5044405233408398, + "language_loss": 0.68712497, + "learning_rate": 3.9571714043047e-06, + "loss": 0.70861006, + "num_input_tokens_seen": 90698820, + "router_z_loss_clip": 0.74560547, + "router_z_loss_mlp": 0.2310791, + "step": 3240, + "time_per_iteration": 2.331400156021118 + }, + { + "auxiliary_loss_clip": 0.01109564, + "auxiliary_loss_mlp": 0.01042568, + "balance_loss_clip": 1.03550458, + "balance_loss_mlp": 1.02179003, + "epoch": 0.09404561546050723, + "flos": 20840599013760.0, + "grad_norm": 2.7398848373520033, + "language_loss": 0.82441372, + "learning_rate": 3.957132705652778e-06, + "loss": 0.84593511, + "num_input_tokens_seen": 90716860, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.2076416, + "step": 3241, + "time_per_iteration": 2.586247682571411 + }, + { + "auxiliary_loss_clip": 0.01120483, + "auxiliary_loss_mlp": 0.01055586, + "balance_loss_clip": 1.04036808, + "balance_loss_mlp": 1.03158927, + "epoch": 0.09407463292902327, + "flos": 14021153723520.0, + "grad_norm": 3.3125213266795526, + "language_loss": 1.07770205, + "learning_rate": 3.9570939897146695e-06, + "loss": 1.09946275, + "num_input_tokens_seen": 90729760, + "router_z_loss_clip": 0.80126953, + "router_z_loss_mlp": 0.23999023, + "step": 3242, + "time_per_iteration": 2.3576126098632812 + }, + { + "auxiliary_loss_clip": 0.01114485, + "auxiliary_loss_mlp": 0.01052485, + "balance_loss_clip": 1.03897536, + "balance_loss_mlp": 1.02939439, + "epoch": 0.09410365039753932, + "flos": 19862402140800.0, + "grad_norm": 2.34368324495517, + "language_loss": 0.75027329, + "learning_rate": 3.957055256490715e-06, + "loss": 0.77194297, + "num_input_tokens_seen": 90745215, + "router_z_loss_clip": 0.75537109, + "router_z_loss_mlp": 0.23083496, + "step": 3243, + "time_per_iteration": 2.42634654045105 + }, + { + "auxiliary_loss_clip": 0.01023363, + "auxiliary_loss_mlp": 0.01008163, + "balance_loss_clip": 1.01096511, + "balance_loss_mlp": 1.00685799, + "epoch": 0.09413266786605537, + "flos": 61347038014080.0, + "grad_norm": 0.6097187072281809, + "language_loss": 0.4416821, + "learning_rate": 3.957016505981256e-06, + "loss": 0.46199739, + "num_input_tokens_seen": 90812625, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.01306152, + "step": 3244, + "time_per_iteration": 3.1729793548583984 + }, + { + "auxiliary_loss_clip": 0.01108519, + "auxiliary_loss_mlp": 0.01044664, + "balance_loss_clip": 1.03804755, + "balance_loss_mlp": 1.02506602, + "epoch": 0.09416168533457141, + "flos": 11575697385600.0, + "grad_norm": 4.065759928466915, + "language_loss": 0.89608496, + "learning_rate": 3.956977738186636e-06, + "loss": 0.91761684, + "num_input_tokens_seen": 90824320, + "router_z_loss_clip": 0.70458984, + "router_z_loss_mlp": 0.19592285, + "step": 3245, + "time_per_iteration": 2.3704686164855957 + }, + { + "auxiliary_loss_clip": 0.01114296, + "auxiliary_loss_mlp": 0.01047562, + "balance_loss_clip": 1.03996551, + "balance_loss_mlp": 1.02462566, + "epoch": 0.09419070280308746, + "flos": 26425096225920.0, + "grad_norm": 3.1157985869526175, + "language_loss": 0.93052983, + "learning_rate": 3.956938953107196e-06, + "loss": 0.95214844, + "num_input_tokens_seen": 90840395, + "router_z_loss_clip": 0.74267578, + "router_z_loss_mlp": 0.22949219, + "step": 3246, + "time_per_iteration": 2.4824163913726807 + }, + { + "auxiliary_loss_clip": 0.01124019, + "auxiliary_loss_mlp": 0.01052673, + "balance_loss_clip": 1.03970766, + "balance_loss_mlp": 1.02713883, + "epoch": 0.09421972027160351, + "flos": 17047809331200.0, + "grad_norm": 3.0553921745445987, + "language_loss": 0.90803784, + "learning_rate": 3.956900150743279e-06, + "loss": 0.9298048, + "num_input_tokens_seen": 90854320, + "router_z_loss_clip": 0.84423828, + "router_z_loss_mlp": 0.25561523, + "step": 3247, + "time_per_iteration": 2.361999273300171 + }, + { + "auxiliary_loss_clip": 0.01019394, + "auxiliary_loss_mlp": 0.01002451, + "balance_loss_clip": 1.00679195, + "balance_loss_mlp": 1.00112736, + "epoch": 0.09424873774011955, + "flos": 61639679963520.0, + "grad_norm": 0.7329141188102559, + "language_loss": 0.48616844, + "learning_rate": 3.956861331095229e-06, + "loss": 0.50638694, + "num_input_tokens_seen": 90910080, + "router_z_loss_clip": 0.12597656, + "router_z_loss_mlp": 0.01324463, + "step": 3248, + "time_per_iteration": 2.9447500705718994 + }, + { + "auxiliary_loss_clip": 0.01117376, + "auxiliary_loss_mlp": 0.01044032, + "balance_loss_clip": 1.03713846, + "balance_loss_mlp": 1.02159631, + "epoch": 0.0942777552086356, + "flos": 35254526065920.0, + "grad_norm": 2.085352608590256, + "language_loss": 0.80774212, + "learning_rate": 3.956822494163387e-06, + "loss": 0.82935619, + "num_input_tokens_seen": 90926805, + "router_z_loss_clip": 0.80273438, + "router_z_loss_mlp": 0.22424316, + "step": 3249, + "time_per_iteration": 2.5119030475616455 + }, + { + "auxiliary_loss_clip": 0.01104659, + "auxiliary_loss_mlp": 0.01036514, + "balance_loss_clip": 1.03141212, + "balance_loss_mlp": 1.01574779, + "epoch": 0.09430677267715165, + "flos": 15223597459200.0, + "grad_norm": 2.177275349625173, + "language_loss": 0.8143729, + "learning_rate": 3.956783639948098e-06, + "loss": 0.83578455, + "num_input_tokens_seen": 90941690, + "router_z_loss_clip": 0.73242188, + "router_z_loss_mlp": 0.20751953, + "step": 3250, + "time_per_iteration": 2.411375045776367 + }, + { + "auxiliary_loss_clip": 0.01102039, + "auxiliary_loss_mlp": 0.01043545, + "balance_loss_clip": 1.03286862, + "balance_loss_mlp": 1.02335095, + "epoch": 0.09433579014566769, + "flos": 23766878862720.0, + "grad_norm": 1.8930183474187658, + "language_loss": 0.62842566, + "learning_rate": 3.956744768449703e-06, + "loss": 0.64988148, + "num_input_tokens_seen": 90958880, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.20202637, + "step": 3251, + "time_per_iteration": 2.4142770767211914 + }, + { + "auxiliary_loss_clip": 0.011043, + "auxiliary_loss_mlp": 0.01046615, + "balance_loss_clip": 1.03259015, + "balance_loss_mlp": 1.02395308, + "epoch": 0.09436480761418374, + "flos": 18764488615680.0, + "grad_norm": 2.2975008397405885, + "language_loss": 0.77851093, + "learning_rate": 3.956705879668547e-06, + "loss": 0.80001998, + "num_input_tokens_seen": 90974000, + "router_z_loss_clip": 0.71679688, + "router_z_loss_mlp": 0.2265625, + "step": 3252, + "time_per_iteration": 2.4272758960723877 + }, + { + "auxiliary_loss_clip": 0.01107678, + "auxiliary_loss_mlp": 0.01037099, + "balance_loss_clip": 1.03530097, + "balance_loss_mlp": 1.01617217, + "epoch": 0.09439382508269979, + "flos": 43906736511360.0, + "grad_norm": 2.200663707532145, + "language_loss": 0.69648534, + "learning_rate": 3.956666973604972e-06, + "loss": 0.71793312, + "num_input_tokens_seen": 90996900, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.20904541, + "step": 3253, + "time_per_iteration": 2.5704989433288574 + }, + { + "auxiliary_loss_clip": 0.01017448, + "auxiliary_loss_mlp": 0.01003682, + "balance_loss_clip": 1.0047543, + "balance_loss_mlp": 1.0024842, + "epoch": 0.09442284255121583, + "flos": 63454116654720.0, + "grad_norm": 0.7194397074073292, + "language_loss": 0.49594253, + "learning_rate": 3.956628050259323e-06, + "loss": 0.51615387, + "num_input_tokens_seen": 91049720, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01196289, + "step": 3254, + "time_per_iteration": 2.823937177658081 + }, + { + "auxiliary_loss_clip": 0.01017724, + "auxiliary_loss_mlp": 0.01002446, + "balance_loss_clip": 1.00546169, + "balance_loss_mlp": 1.00135493, + "epoch": 0.09445186001973188, + "flos": 74778718475520.0, + "grad_norm": 0.6844214573511318, + "language_loss": 0.49987099, + "learning_rate": 3.956589109631944e-06, + "loss": 0.5200727, + "num_input_tokens_seen": 91116165, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01092529, + "step": 3255, + "time_per_iteration": 3.1081464290618896 + }, + { + "auxiliary_loss_clip": 0.01114066, + "auxiliary_loss_mlp": 0.01047025, + "balance_loss_clip": 1.03505445, + "balance_loss_mlp": 1.02085829, + "epoch": 0.09448087748824792, + "flos": 65319981511680.0, + "grad_norm": 1.881686433159673, + "language_loss": 0.93317717, + "learning_rate": 3.956550151723178e-06, + "loss": 0.95478809, + "num_input_tokens_seen": 91140755, + "router_z_loss_clip": 0.79003906, + "router_z_loss_mlp": 0.26171875, + "step": 3256, + "time_per_iteration": 2.805098295211792 + }, + { + "auxiliary_loss_clip": 0.0101983, + "auxiliary_loss_mlp": 0.01001169, + "balance_loss_clip": 1.00708842, + "balance_loss_mlp": 0.99983948, + "epoch": 0.09450989495676397, + "flos": 67790469294720.0, + "grad_norm": 0.6623337641359902, + "language_loss": 0.46694055, + "learning_rate": 3.956511176533368e-06, + "loss": 0.48715055, + "num_input_tokens_seen": 91192920, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01330566, + "step": 3257, + "time_per_iteration": 2.883549690246582 + }, + { + "auxiliary_loss_clip": 0.01105063, + "auxiliary_loss_mlp": 0.01042137, + "balance_loss_clip": 1.03282928, + "balance_loss_mlp": 1.02059627, + "epoch": 0.09453891242528002, + "flos": 21061005626880.0, + "grad_norm": 2.6415793058251658, + "language_loss": 0.73710841, + "learning_rate": 3.956472184062861e-06, + "loss": 0.75858039, + "num_input_tokens_seen": 91206595, + "router_z_loss_clip": 0.72216797, + "router_z_loss_mlp": 0.2154541, + "step": 3258, + "time_per_iteration": 2.39789080619812 + }, + { + "auxiliary_loss_clip": 0.01017801, + "auxiliary_loss_mlp": 0.01003018, + "balance_loss_clip": 1.00525498, + "balance_loss_mlp": 1.00186157, + "epoch": 0.09456792989379606, + "flos": 70348834569600.0, + "grad_norm": 0.6847008356803447, + "language_loss": 0.47891074, + "learning_rate": 3.956433174312e-06, + "loss": 0.49911892, + "num_input_tokens_seen": 91263780, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.01153564, + "step": 3259, + "time_per_iteration": 2.9240875244140625 + }, + { + "auxiliary_loss_clip": 0.01018485, + "auxiliary_loss_mlp": 0.01002191, + "balance_loss_clip": 1.00566125, + "balance_loss_mlp": 1.00093341, + "epoch": 0.09459694736231211, + "flos": 69963951208320.0, + "grad_norm": 0.6549634618446005, + "language_loss": 0.4904471, + "learning_rate": 3.9563941472811285e-06, + "loss": 0.51065385, + "num_input_tokens_seen": 91331495, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01257324, + "step": 3260, + "time_per_iteration": 3.1363954544067383 + }, + { + "auxiliary_loss_clip": 0.0110544, + "auxiliary_loss_mlp": 0.01042313, + "balance_loss_clip": 1.03156614, + "balance_loss_mlp": 1.02226198, + "epoch": 0.09462596483082816, + "flos": 13655543299200.0, + "grad_norm": 3.2813371319628954, + "language_loss": 1.00926137, + "learning_rate": 3.956355102970593e-06, + "loss": 1.03073883, + "num_input_tokens_seen": 91342720, + "router_z_loss_clip": 0.73876953, + "router_z_loss_mlp": 0.20068359, + "step": 3261, + "time_per_iteration": 2.4118196964263916 + }, + { + "auxiliary_loss_clip": 0.01016964, + "auxiliary_loss_mlp": 0.01004025, + "balance_loss_clip": 1.00411332, + "balance_loss_mlp": 1.00290394, + "epoch": 0.0946549822993442, + "flos": 74763287654400.0, + "grad_norm": 0.8211886929324997, + "language_loss": 0.51169485, + "learning_rate": 3.956316041380737e-06, + "loss": 0.5319047, + "num_input_tokens_seen": 91401140, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01123047, + "step": 3262, + "time_per_iteration": 2.9952220916748047 + }, + { + "auxiliary_loss_clip": 0.01104621, + "auxiliary_loss_mlp": 0.01047954, + "balance_loss_clip": 1.03099275, + "balance_loss_mlp": 1.02655578, + "epoch": 0.09468399976786025, + "flos": 11397186270720.0, + "grad_norm": 2.6248251360139987, + "language_loss": 0.95688379, + "learning_rate": 3.956276962511907e-06, + "loss": 0.97840953, + "num_input_tokens_seen": 91413785, + "router_z_loss_clip": 0.73583984, + "router_z_loss_mlp": 0.21398926, + "step": 3263, + "time_per_iteration": 2.3756091594696045 + }, + { + "auxiliary_loss_clip": 0.01103908, + "auxiliary_loss_mlp": 0.01048569, + "balance_loss_clip": 1.03326297, + "balance_loss_mlp": 1.02805269, + "epoch": 0.0947130172363763, + "flos": 17851544807040.0, + "grad_norm": 7.155522375819373, + "language_loss": 0.90741408, + "learning_rate": 3.956237866364446e-06, + "loss": 0.92893898, + "num_input_tokens_seen": 91428145, + "router_z_loss_clip": 0.70751953, + "router_z_loss_mlp": 0.20495605, + "step": 3264, + "time_per_iteration": 2.332200765609741 + }, + { + "auxiliary_loss_clip": 0.01103089, + "auxiliary_loss_mlp": 0.0104118, + "balance_loss_clip": 1.03134418, + "balance_loss_mlp": 1.02125967, + "epoch": 0.09474203470489234, + "flos": 53721520053120.0, + "grad_norm": 2.1865738806394917, + "language_loss": 0.87948126, + "learning_rate": 3.9561987529387014e-06, + "loss": 0.90092397, + "num_input_tokens_seen": 91450925, + "router_z_loss_clip": 0.71777344, + "router_z_loss_mlp": 0.19909668, + "step": 3265, + "time_per_iteration": 2.7394683361053467 + }, + { + "auxiliary_loss_clip": 0.01110077, + "auxiliary_loss_mlp": 0.01042641, + "balance_loss_clip": 1.03679276, + "balance_loss_mlp": 1.0212189, + "epoch": 0.0947710521734084, + "flos": 32225810688000.0, + "grad_norm": 2.5444775312201315, + "language_loss": 0.84087104, + "learning_rate": 3.9561596222350175e-06, + "loss": 0.86239821, + "num_input_tokens_seen": 91466895, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.21411133, + "step": 3266, + "time_per_iteration": 2.4581167697906494 + }, + { + "auxiliary_loss_clip": 0.01102311, + "auxiliary_loss_mlp": 0.01044809, + "balance_loss_clip": 1.03399646, + "balance_loss_mlp": 1.02473402, + "epoch": 0.09480006964192444, + "flos": 39632564736000.0, + "grad_norm": 3.0119514642312817, + "language_loss": 0.83350641, + "learning_rate": 3.95612047425374e-06, + "loss": 0.85497761, + "num_input_tokens_seen": 91481815, + "router_z_loss_clip": 0.68237305, + "router_z_loss_mlp": 0.20074463, + "step": 3267, + "time_per_iteration": 2.5197391510009766 + }, + { + "auxiliary_loss_clip": 0.01018476, + "auxiliary_loss_mlp": 0.01002306, + "balance_loss_clip": 1.00642371, + "balance_loss_mlp": 1.00123882, + "epoch": 0.09482908711044048, + "flos": 62726770967040.0, + "grad_norm": 0.7281828667498513, + "language_loss": 0.49721926, + "learning_rate": 3.956081308995216e-06, + "loss": 0.51742709, + "num_input_tokens_seen": 91534345, + "router_z_loss_clip": 0.12060547, + "router_z_loss_mlp": 0.01068115, + "step": 3268, + "time_per_iteration": 2.8750083446502686 + }, + { + "auxiliary_loss_clip": 0.01113537, + "auxiliary_loss_mlp": 0.01047987, + "balance_loss_clip": 1.03920817, + "balance_loss_mlp": 1.02699435, + "epoch": 0.09485810457895653, + "flos": 28284849728640.0, + "grad_norm": 2.8493117990947763, + "language_loss": 0.80447865, + "learning_rate": 3.9560421264597894e-06, + "loss": 0.82609385, + "num_input_tokens_seen": 91548675, + "router_z_loss_clip": 0.74243164, + "router_z_loss_mlp": 0.20996094, + "step": 3269, + "time_per_iteration": 2.5027618408203125 + }, + { + "auxiliary_loss_clip": 0.01019954, + "auxiliary_loss_mlp": 0.01002402, + "balance_loss_clip": 1.00771737, + "balance_loss_mlp": 1.00133502, + "epoch": 0.09488712204747257, + "flos": 74772259873920.0, + "grad_norm": 1.4383999839680952, + "language_loss": 0.53310359, + "learning_rate": 3.956002926647807e-06, + "loss": 0.55332708, + "num_input_tokens_seen": 91609770, + "router_z_loss_clip": 0.12255859, + "router_z_loss_mlp": 0.01068115, + "step": 3270, + "time_per_iteration": 3.009967565536499 + }, + { + "auxiliary_loss_clip": 0.01019827, + "auxiliary_loss_mlp": 0.01001323, + "balance_loss_clip": 1.00772929, + "balance_loss_mlp": 1.00029218, + "epoch": 0.09491613951598862, + "flos": 63201903926400.0, + "grad_norm": 0.6532697859184095, + "language_loss": 0.50668949, + "learning_rate": 3.9559637095596155e-06, + "loss": 0.52690101, + "num_input_tokens_seen": 91669870, + "router_z_loss_clip": 0.12109375, + "router_z_loss_mlp": 0.01031494, + "step": 3271, + "time_per_iteration": 3.003584861755371 + }, + { + "auxiliary_loss_clip": 0.01115647, + "auxiliary_loss_mlp": 0.01054007, + "balance_loss_clip": 1.03546619, + "balance_loss_mlp": 1.02992654, + "epoch": 0.09494515698450468, + "flos": 31496230673280.0, + "grad_norm": 2.967239463219184, + "language_loss": 0.81177717, + "learning_rate": 3.955924475195562e-06, + "loss": 0.83347374, + "num_input_tokens_seen": 91687635, + "router_z_loss_clip": 0.80175781, + "router_z_loss_mlp": 0.24084473, + "step": 3272, + "time_per_iteration": 2.484459161758423 + }, + { + "auxiliary_loss_clip": 0.01102505, + "auxiliary_loss_mlp": 0.01042412, + "balance_loss_clip": 1.03359914, + "balance_loss_mlp": 1.02314758, + "epoch": 0.09497417445302071, + "flos": 29562810468480.0, + "grad_norm": 2.137001035247052, + "language_loss": 0.87826174, + "learning_rate": 3.955885223555991e-06, + "loss": 0.89971089, + "num_input_tokens_seen": 91703845, + "router_z_loss_clip": 0.68896484, + "router_z_loss_mlp": 0.19281006, + "step": 3273, + "time_per_iteration": 2.4673078060150146 + }, + { + "auxiliary_loss_clip": 0.01106082, + "auxiliary_loss_mlp": 0.01042563, + "balance_loss_clip": 1.0348897, + "balance_loss_mlp": 1.02034211, + "epoch": 0.09500319192153676, + "flos": 27920251733760.0, + "grad_norm": 2.784220868659076, + "language_loss": 0.85213161, + "learning_rate": 3.9558459546412505e-06, + "loss": 0.87361801, + "num_input_tokens_seen": 91721240, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.2220459, + "step": 3274, + "time_per_iteration": 6.805707216262817 + }, + { + "auxiliary_loss_clip": 0.01109001, + "auxiliary_loss_mlp": 0.01045319, + "balance_loss_clip": 1.03708816, + "balance_loss_mlp": 1.02442789, + "epoch": 0.09503220939005282, + "flos": 38064196373760.0, + "grad_norm": 1.855681947521249, + "language_loss": 0.64236045, + "learning_rate": 3.955806668451687e-06, + "loss": 0.66390371, + "num_input_tokens_seen": 91741780, + "router_z_loss_clip": 0.71972656, + "router_z_loss_mlp": 0.20892334, + "step": 3275, + "time_per_iteration": 2.5559473037719727 + }, + { + "auxiliary_loss_clip": 0.01015585, + "auxiliary_loss_mlp": 0.01003996, + "balance_loss_clip": 1.00385797, + "balance_loss_mlp": 1.00294709, + "epoch": 0.09506122685856885, + "flos": 53857645044480.0, + "grad_norm": 0.7421386480009038, + "language_loss": 0.49758533, + "learning_rate": 3.955767364987648e-06, + "loss": 0.51778114, + "num_input_tokens_seen": 91790610, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01049805, + "step": 3276, + "time_per_iteration": 2.718393087387085 + }, + { + "auxiliary_loss_clip": 0.01015232, + "auxiliary_loss_mlp": 0.01002157, + "balance_loss_clip": 1.00371099, + "balance_loss_mlp": 1.00107265, + "epoch": 0.0950902443270849, + "flos": 63102470774400.0, + "grad_norm": 0.6762547526144401, + "language_loss": 0.46052831, + "learning_rate": 3.955728044249479e-06, + "loss": 0.48070222, + "num_input_tokens_seen": 91850135, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01086426, + "step": 3277, + "time_per_iteration": 3.010038137435913 + }, + { + "auxiliary_loss_clip": 0.01107666, + "auxiliary_loss_mlp": 0.01044461, + "balance_loss_clip": 1.03483701, + "balance_loss_mlp": 1.02327764, + "epoch": 0.09511926179560096, + "flos": 31387860213120.0, + "grad_norm": 2.093165268632401, + "language_loss": 0.81477153, + "learning_rate": 3.95568870623753e-06, + "loss": 0.83629286, + "num_input_tokens_seen": 91867730, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.21179199, + "step": 3278, + "time_per_iteration": 2.4591026306152344 + }, + { + "auxiliary_loss_clip": 0.01109243, + "auxiliary_loss_mlp": 0.01036973, + "balance_loss_clip": 1.03301573, + "balance_loss_mlp": 1.01717281, + "epoch": 0.095148279264117, + "flos": 10845942814080.0, + "grad_norm": 2.507623311896267, + "language_loss": 0.8886174, + "learning_rate": 3.955649350952147e-06, + "loss": 0.91007966, + "num_input_tokens_seen": 91878740, + "router_z_loss_clip": 0.76171875, + "router_z_loss_mlp": 0.19799805, + "step": 3279, + "time_per_iteration": 2.3579719066619873 + }, + { + "auxiliary_loss_clip": 0.01112444, + "auxiliary_loss_mlp": 0.01044632, + "balance_loss_clip": 1.03562462, + "balance_loss_mlp": 1.02198207, + "epoch": 0.09517729673263305, + "flos": 35874827925120.0, + "grad_norm": 2.1147624493930746, + "language_loss": 0.90226561, + "learning_rate": 3.955609978393676e-06, + "loss": 0.92383635, + "num_input_tokens_seen": 91895525, + "router_z_loss_clip": 0.76806641, + "router_z_loss_mlp": 0.22668457, + "step": 3280, + "time_per_iteration": 7.445435285568237 + }, + { + "auxiliary_loss_clip": 0.01019024, + "auxiliary_loss_mlp": 0.01004454, + "balance_loss_clip": 1.00756383, + "balance_loss_mlp": 1.00348222, + "epoch": 0.0952063142011491, + "flos": 62512822955520.0, + "grad_norm": 0.6717400856018075, + "language_loss": 0.48936647, + "learning_rate": 3.9555705885624675e-06, + "loss": 0.50960124, + "num_input_tokens_seen": 91956735, + "router_z_loss_clip": 0.11474609, + "router_z_loss_mlp": 0.00970459, + "step": 3281, + "time_per_iteration": 3.0083441734313965 + }, + { + "auxiliary_loss_clip": 0.01097877, + "auxiliary_loss_mlp": 0.01037341, + "balance_loss_clip": 1.03313506, + "balance_loss_mlp": 1.01897049, + "epoch": 0.09523533166966514, + "flos": 23184771897600.0, + "grad_norm": 1.8191492298133412, + "language_loss": 0.6783042, + "learning_rate": 3.955531181458868e-06, + "loss": 0.69965637, + "num_input_tokens_seen": 91973680, + "router_z_loss_clip": 0.64746094, + "router_z_loss_mlp": 0.18371582, + "step": 3282, + "time_per_iteration": 2.524670362472534 + }, + { + "auxiliary_loss_clip": 0.01090286, + "auxiliary_loss_mlp": 0.01043023, + "balance_loss_clip": 1.02986395, + "balance_loss_mlp": 1.02553546, + "epoch": 0.09526434913818119, + "flos": 35983023828480.0, + "grad_norm": 2.3037997361378855, + "language_loss": 0.72266519, + "learning_rate": 3.955491757083225e-06, + "loss": 0.74399829, + "num_input_tokens_seen": 91989470, + "router_z_loss_clip": 0.60449219, + "router_z_loss_mlp": 0.17492676, + "step": 3283, + "time_per_iteration": 2.4977686405181885 + }, + { + "auxiliary_loss_clip": 0.01113376, + "auxiliary_loss_mlp": 0.01045887, + "balance_loss_clip": 1.03861439, + "balance_loss_mlp": 1.02429831, + "epoch": 0.09529336660669724, + "flos": 32263586645760.0, + "grad_norm": 2.136369148316699, + "language_loss": 0.7469629, + "learning_rate": 3.955452315435889e-06, + "loss": 0.76855558, + "num_input_tokens_seen": 92004225, + "router_z_loss_clip": 0.74804688, + "router_z_loss_mlp": 0.21569824, + "step": 3284, + "time_per_iteration": 2.504570245742798 + }, + { + "auxiliary_loss_clip": 0.01112526, + "auxiliary_loss_mlp": 0.01046217, + "balance_loss_clip": 1.03625643, + "balance_loss_mlp": 1.0238775, + "epoch": 0.09532238407521328, + "flos": 16501488376320.0, + "grad_norm": 2.4983566702749296, + "language_loss": 0.71268928, + "learning_rate": 3.955412856517205e-06, + "loss": 0.73427671, + "num_input_tokens_seen": 92018580, + "router_z_loss_clip": 0.76220703, + "router_z_loss_mlp": 0.22351074, + "step": 3285, + "time_per_iteration": 2.361846923828125 + }, + { + "auxiliary_loss_clip": 0.01018786, + "auxiliary_loss_mlp": 0.01001409, + "balance_loss_clip": 1.00722504, + "balance_loss_mlp": 1.0003655, + "epoch": 0.09535140154372933, + "flos": 60321464559360.0, + "grad_norm": 0.67006846235749, + "language_loss": 0.44464773, + "learning_rate": 3.9553733803275255e-06, + "loss": 0.46484965, + "num_input_tokens_seen": 92075475, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01043701, + "step": 3286, + "time_per_iteration": 2.891467809677124 + }, + { + "auxiliary_loss_clip": 0.01110232, + "auxiliary_loss_mlp": 0.01045144, + "balance_loss_clip": 1.03527236, + "balance_loss_mlp": 1.02419877, + "epoch": 0.09538041901224537, + "flos": 30040003198080.0, + "grad_norm": 2.514645787761841, + "language_loss": 0.98255163, + "learning_rate": 3.955333886867196e-06, + "loss": 1.00410533, + "num_input_tokens_seen": 92101755, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.20959473, + "step": 3287, + "time_per_iteration": 2.613111972808838 + }, + { + "auxiliary_loss_clip": 0.01016434, + "auxiliary_loss_mlp": 0.0100086, + "balance_loss_clip": 1.00490999, + "balance_loss_mlp": 0.99984646, + "epoch": 0.09540943648076142, + "flos": 70686444216960.0, + "grad_norm": 0.6794742875967629, + "language_loss": 0.54497474, + "learning_rate": 3.955294376136566e-06, + "loss": 0.5651477, + "num_input_tokens_seen": 92166985, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01013184, + "step": 3288, + "time_per_iteration": 3.066020965576172 + }, + { + "auxiliary_loss_clip": 0.01104516, + "auxiliary_loss_mlp": 0.01048788, + "balance_loss_clip": 1.03309214, + "balance_loss_mlp": 1.02840328, + "epoch": 0.09543845394927747, + "flos": 12160806727680.0, + "grad_norm": 3.0237305232069907, + "language_loss": 0.74345589, + "learning_rate": 3.955254848135985e-06, + "loss": 0.76498896, + "num_input_tokens_seen": 92178560, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.20397949, + "step": 3289, + "time_per_iteration": 2.337785482406616 + }, + { + "auxiliary_loss_clip": 0.0111111, + "auxiliary_loss_mlp": 0.01047146, + "balance_loss_clip": 1.03325701, + "balance_loss_mlp": 1.02208185, + "epoch": 0.0954674714177935, + "flos": 33140011305600.0, + "grad_norm": 2.847943747991736, + "language_loss": 1.57954168, + "learning_rate": 3.955215302865802e-06, + "loss": 1.60112441, + "num_input_tokens_seen": 92194390, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.25073242, + "step": 3290, + "time_per_iteration": 2.5415163040161133 + }, + { + "auxiliary_loss_clip": 0.01106572, + "auxiliary_loss_mlp": 0.01042316, + "balance_loss_clip": 1.03215897, + "balance_loss_mlp": 1.02338576, + "epoch": 0.09549648888630956, + "flos": 19245486683520.0, + "grad_norm": 2.228009915734447, + "language_loss": 0.80657506, + "learning_rate": 3.955175740326367e-06, + "loss": 0.82806385, + "num_input_tokens_seen": 92209870, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.18933105, + "step": 3291, + "time_per_iteration": 2.3696632385253906 + }, + { + "auxiliary_loss_clip": 0.01107124, + "auxiliary_loss_mlp": 0.01047289, + "balance_loss_clip": 1.03412414, + "balance_loss_mlp": 1.0258199, + "epoch": 0.09552550635482561, + "flos": 16610103216000.0, + "grad_norm": 2.086476490490394, + "language_loss": 0.51642519, + "learning_rate": 3.955136160518029e-06, + "loss": 0.53796935, + "num_input_tokens_seen": 92222050, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.21459961, + "step": 3292, + "time_per_iteration": 2.389608144760132 + }, + { + "auxiliary_loss_clip": 0.0110576, + "auxiliary_loss_mlp": 0.01046624, + "balance_loss_clip": 1.03453875, + "balance_loss_mlp": 1.02463031, + "epoch": 0.09555452382334165, + "flos": 24383410295040.0, + "grad_norm": 2.457755554471519, + "language_loss": 0.93581927, + "learning_rate": 3.9550965634411356e-06, + "loss": 0.9573431, + "num_input_tokens_seen": 92238645, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.2199707, + "step": 3293, + "time_per_iteration": 2.4546988010406494 + }, + { + "auxiliary_loss_clip": 0.01108195, + "auxiliary_loss_mlp": 0.01042944, + "balance_loss_clip": 1.03615093, + "balance_loss_mlp": 1.02278543, + "epoch": 0.0955835412918577, + "flos": 19019598986880.0, + "grad_norm": 2.3001633041768677, + "language_loss": 0.80657536, + "learning_rate": 3.955056949096039e-06, + "loss": 0.82808673, + "num_input_tokens_seen": 92253970, + "router_z_loss_clip": 0.72021484, + "router_z_loss_mlp": 0.20178223, + "step": 3294, + "time_per_iteration": 2.4355239868164062 + }, + { + "auxiliary_loss_clip": 0.01093526, + "auxiliary_loss_mlp": 0.01036242, + "balance_loss_clip": 1.0295949, + "balance_loss_mlp": 1.0198921, + "epoch": 0.09561255876037375, + "flos": 14529140138880.0, + "grad_norm": 2.2792004229179654, + "language_loss": 0.7089743, + "learning_rate": 3.955017317483089e-06, + "loss": 0.73027194, + "num_input_tokens_seen": 92266600, + "router_z_loss_clip": 0.63964844, + "router_z_loss_mlp": 0.16345215, + "step": 3295, + "time_per_iteration": 2.3577423095703125 + }, + { + "auxiliary_loss_clip": 0.01015741, + "auxiliary_loss_mlp": 0.01002561, + "balance_loss_clip": 1.0037837, + "balance_loss_mlp": 1.00133955, + "epoch": 0.09564157622888979, + "flos": 58567707544320.0, + "grad_norm": 0.6815238698988847, + "language_loss": 0.54303348, + "learning_rate": 3.954977668602634e-06, + "loss": 0.56321657, + "num_input_tokens_seen": 92325645, + "router_z_loss_clip": 0.11962891, + "router_z_loss_mlp": 0.01220703, + "step": 3296, + "time_per_iteration": 2.91129469871521 + }, + { + "auxiliary_loss_clip": 0.01110821, + "auxiliary_loss_mlp": 0.01047309, + "balance_loss_clip": 1.03437686, + "balance_loss_mlp": 1.02192903, + "epoch": 0.09567059369740584, + "flos": 43499789170560.0, + "grad_norm": 1.8951935373958975, + "language_loss": 0.79329419, + "learning_rate": 3.954938002455025e-06, + "loss": 0.81487548, + "num_input_tokens_seen": 92345125, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.25390625, + "step": 3297, + "time_per_iteration": 2.5616159439086914 + }, + { + "auxiliary_loss_clip": 0.01114251, + "auxiliary_loss_mlp": 0.01046835, + "balance_loss_clip": 1.03565884, + "balance_loss_mlp": 1.02363682, + "epoch": 0.09569961116592189, + "flos": 48827186064000.0, + "grad_norm": 2.2472674766749527, + "language_loss": 0.87917769, + "learning_rate": 3.954898319040613e-06, + "loss": 0.90078861, + "num_input_tokens_seen": 92367500, + "router_z_loss_clip": 0.78613281, + "router_z_loss_mlp": 0.23205566, + "step": 3298, + "time_per_iteration": 2.649815082550049 + }, + { + "auxiliary_loss_clip": 0.0111691, + "auxiliary_loss_mlp": 0.01051866, + "balance_loss_clip": 1.03523171, + "balance_loss_mlp": 1.02729654, + "epoch": 0.09572862863443793, + "flos": 31276976135040.0, + "grad_norm": 2.6695798379371234, + "language_loss": 1.1309917, + "learning_rate": 3.954858618359748e-06, + "loss": 1.15267944, + "num_input_tokens_seen": 92387710, + "router_z_loss_clip": 0.81689453, + "router_z_loss_mlp": 0.24560547, + "step": 3299, + "time_per_iteration": 2.448984146118164 + }, + { + "auxiliary_loss_clip": 0.01109483, + "auxiliary_loss_mlp": 0.01051451, + "balance_loss_clip": 1.03286195, + "balance_loss_mlp": 1.02677441, + "epoch": 0.09575764610295398, + "flos": 36862904712960.0, + "grad_norm": 2.5603510886947083, + "language_loss": 0.89013314, + "learning_rate": 3.9548189004127805e-06, + "loss": 0.91174245, + "num_input_tokens_seen": 92404830, + "router_z_loss_clip": 0.76611328, + "router_z_loss_mlp": 0.24694824, + "step": 3300, + "time_per_iteration": 2.547609806060791 + }, + { + "auxiliary_loss_clip": 0.0110545, + "auxiliary_loss_mlp": 0.01037892, + "balance_loss_clip": 1.03573859, + "balance_loss_mlp": 1.02114868, + "epoch": 0.09578666357147002, + "flos": 27957783312000.0, + "grad_norm": 2.954585754826865, + "language_loss": 0.93728077, + "learning_rate": 3.954779165200061e-06, + "loss": 0.95871425, + "num_input_tokens_seen": 92419385, + "router_z_loss_clip": 0.6965332, + "router_z_loss_mlp": 0.16748047, + "step": 3301, + "time_per_iteration": 2.424027442932129 + }, + { + "auxiliary_loss_clip": 0.01015262, + "auxiliary_loss_mlp": 0.01004255, + "balance_loss_clip": 1.00315738, + "balance_loss_mlp": 1.00302124, + "epoch": 0.09581568103998607, + "flos": 57508093647360.0, + "grad_norm": 0.7213013597437079, + "language_loss": 0.52994072, + "learning_rate": 3.954739412721942e-06, + "loss": 0.55013591, + "num_input_tokens_seen": 92473655, + "router_z_loss_clip": 0.12109375, + "router_z_loss_mlp": 0.0123291, + "step": 3302, + "time_per_iteration": 2.9445078372955322 + }, + { + "auxiliary_loss_clip": 0.01109289, + "auxiliary_loss_mlp": 0.01040807, + "balance_loss_clip": 1.03523564, + "balance_loss_mlp": 1.01959968, + "epoch": 0.09584469850850212, + "flos": 26682685303680.0, + "grad_norm": 2.3615297523876566, + "language_loss": 0.74532747, + "learning_rate": 3.954699642978773e-06, + "loss": 0.76682842, + "num_input_tokens_seen": 92489155, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.2121582, + "step": 3303, + "time_per_iteration": 2.4165685176849365 + }, + { + "auxiliary_loss_clip": 0.01104685, + "auxiliary_loss_mlp": 0.01037089, + "balance_loss_clip": 1.03488791, + "balance_loss_mlp": 1.01709747, + "epoch": 0.09587371597701816, + "flos": 70246959488640.0, + "grad_norm": 2.1708671533525017, + "language_loss": 0.65475798, + "learning_rate": 3.954659855970905e-06, + "loss": 0.67617565, + "num_input_tokens_seen": 92512685, + "router_z_loss_clip": 0.69726562, + "router_z_loss_mlp": 0.20007324, + "step": 3304, + "time_per_iteration": 2.729491949081421 + }, + { + "auxiliary_loss_clip": 0.01103775, + "auxiliary_loss_mlp": 0.01046013, + "balance_loss_clip": 1.03293478, + "balance_loss_mlp": 1.02607512, + "epoch": 0.09590273344553421, + "flos": 42160625084160.0, + "grad_norm": 1.932778355123675, + "language_loss": 1.02156258, + "learning_rate": 3.954620051698691e-06, + "loss": 1.04306054, + "num_input_tokens_seen": 92535245, + "router_z_loss_clip": 0.70825195, + "router_z_loss_mlp": 0.19958496, + "step": 3305, + "time_per_iteration": 2.6309194564819336 + }, + { + "auxiliary_loss_clip": 0.01098472, + "auxiliary_loss_mlp": 0.0103904, + "balance_loss_clip": 1.03089082, + "balance_loss_mlp": 1.01962137, + "epoch": 0.09593175091405026, + "flos": 13143088229760.0, + "grad_norm": 3.4789302409880403, + "language_loss": 0.95730293, + "learning_rate": 3.954580230162482e-06, + "loss": 0.97867799, + "num_input_tokens_seen": 92545905, + "router_z_loss_clip": 0.67578125, + "router_z_loss_mlp": 0.19433594, + "step": 3306, + "time_per_iteration": 2.3691413402557373 + }, + { + "auxiliary_loss_clip": 0.01104862, + "auxiliary_loss_mlp": 0.01044774, + "balance_loss_clip": 1.03187847, + "balance_loss_mlp": 1.02360201, + "epoch": 0.0959607683825663, + "flos": 51499228325760.0, + "grad_norm": 1.8431465826366886, + "language_loss": 0.92780232, + "learning_rate": 3.954540391362629e-06, + "loss": 0.9492988, + "num_input_tokens_seen": 92571900, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.21179199, + "step": 3307, + "time_per_iteration": 2.673718214035034 + }, + { + "auxiliary_loss_clip": 0.01107573, + "auxiliary_loss_mlp": 0.01045103, + "balance_loss_clip": 1.03142083, + "balance_loss_mlp": 1.02352643, + "epoch": 0.09598978585108235, + "flos": 16571733765120.0, + "grad_norm": 2.693754789831086, + "language_loss": 0.83277792, + "learning_rate": 3.954500535299484e-06, + "loss": 0.85430473, + "num_input_tokens_seen": 92585070, + "router_z_loss_clip": 0.76171875, + "router_z_loss_mlp": 0.21582031, + "step": 3308, + "time_per_iteration": 2.35038161277771 + }, + { + "auxiliary_loss_clip": 0.01106471, + "auxiliary_loss_mlp": 0.01039431, + "balance_loss_clip": 1.03203988, + "balance_loss_mlp": 1.01868844, + "epoch": 0.0960188033195984, + "flos": 26425096225920.0, + "grad_norm": 2.233278837024707, + "language_loss": 0.91517377, + "learning_rate": 3.9544606619734e-06, + "loss": 0.93663275, + "num_input_tokens_seen": 92601405, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.20751953, + "step": 3309, + "time_per_iteration": 2.413638114929199 + }, + { + "auxiliary_loss_clip": 0.0110802, + "auxiliary_loss_mlp": 0.01049958, + "balance_loss_clip": 1.03582048, + "balance_loss_mlp": 1.02966845, + "epoch": 0.09604782078811444, + "flos": 27637000940160.0, + "grad_norm": 1.9745410719379313, + "language_loss": 0.79384321, + "learning_rate": 3.954420771384728e-06, + "loss": 0.81542301, + "num_input_tokens_seen": 92620815, + "router_z_loss_clip": 0.72216797, + "router_z_loss_mlp": 0.20281982, + "step": 3310, + "time_per_iteration": 2.5184831619262695 + }, + { + "auxiliary_loss_clip": 0.01015599, + "auxiliary_loss_mlp": 0.0100465, + "balance_loss_clip": 1.00387526, + "balance_loss_mlp": 1.00354683, + "epoch": 0.09607683825663049, + "flos": 72181460079360.0, + "grad_norm": 0.6409427159302641, + "language_loss": 0.54350632, + "learning_rate": 3.954380863533821e-06, + "loss": 0.56370878, + "num_input_tokens_seen": 92684385, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01104736, + "step": 3311, + "time_per_iteration": 3.0969724655151367 + }, + { + "auxiliary_loss_clip": 0.01108361, + "auxiliary_loss_mlp": 0.0104948, + "balance_loss_clip": 1.03216064, + "balance_loss_mlp": 1.02545905, + "epoch": 0.09610585572514654, + "flos": 38541249457920.0, + "grad_norm": 2.8888545593886232, + "language_loss": 1.03122377, + "learning_rate": 3.954340938421032e-06, + "loss": 1.05280209, + "num_input_tokens_seen": 92707340, + "router_z_loss_clip": 0.76220703, + "router_z_loss_mlp": 0.24029541, + "step": 3312, + "time_per_iteration": 2.615671396255493 + }, + { + "auxiliary_loss_clip": 0.01015429, + "auxiliary_loss_mlp": 0.01001474, + "balance_loss_clip": 1.00375211, + "balance_loss_mlp": 1.00040734, + "epoch": 0.09613487319366258, + "flos": 66124971930240.0, + "grad_norm": 0.6678909959057865, + "language_loss": 0.50857282, + "learning_rate": 3.954300996046712e-06, + "loss": 0.52874184, + "num_input_tokens_seen": 92766060, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01068115, + "step": 3313, + "time_per_iteration": 2.97345232963562 + }, + { + "auxiliary_loss_clip": 0.01104762, + "auxiliary_loss_mlp": 0.01045444, + "balance_loss_clip": 1.03403676, + "balance_loss_mlp": 1.02169728, + "epoch": 0.09616389066217863, + "flos": 34819508125440.0, + "grad_norm": 2.521836268058633, + "language_loss": 0.81242955, + "learning_rate": 3.954261036411215e-06, + "loss": 0.83393162, + "num_input_tokens_seen": 92782525, + "router_z_loss_clip": 0.70751953, + "router_z_loss_mlp": 0.23754883, + "step": 3314, + "time_per_iteration": 2.511469602584839 + }, + { + "auxiliary_loss_clip": 0.01013793, + "auxiliary_loss_mlp": 0.01001117, + "balance_loss_clip": 1.00206804, + "balance_loss_mlp": 1.00001431, + "epoch": 0.09619290813069468, + "flos": 74777531489280.0, + "grad_norm": 0.6394884100573163, + "language_loss": 0.5010314, + "learning_rate": 3.954221059514895e-06, + "loss": 0.52118051, + "num_input_tokens_seen": 92844850, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01104736, + "step": 3315, + "time_per_iteration": 3.070981740951538 + }, + { + "auxiliary_loss_clip": 0.01013681, + "auxiliary_loss_mlp": 0.01001735, + "balance_loss_clip": 1.00201035, + "balance_loss_mlp": 1.00065053, + "epoch": 0.09622192559921072, + "flos": 69620057516160.0, + "grad_norm": 0.5945772170077223, + "language_loss": 0.4399111, + "learning_rate": 3.954181065358102e-06, + "loss": 0.46006528, + "num_input_tokens_seen": 92910270, + "router_z_loss_clip": 0.11669922, + "router_z_loss_mlp": 0.01086426, + "step": 3316, + "time_per_iteration": 3.2372612953186035 + }, + { + "auxiliary_loss_clip": 0.01109862, + "auxiliary_loss_mlp": 0.01045809, + "balance_loss_clip": 1.03421891, + "balance_loss_mlp": 1.02318299, + "epoch": 0.09625094306772677, + "flos": 47839982060160.0, + "grad_norm": 1.787231814883488, + "language_loss": 0.82222891, + "learning_rate": 3.954141053941192e-06, + "loss": 0.84378564, + "num_input_tokens_seen": 92930045, + "router_z_loss_clip": 0.75683594, + "router_z_loss_mlp": 0.22631836, + "step": 3317, + "time_per_iteration": 2.613241672515869 + }, + { + "auxiliary_loss_clip": 0.01104045, + "auxiliary_loss_mlp": 0.01047699, + "balance_loss_clip": 1.03137827, + "balance_loss_mlp": 1.02483416, + "epoch": 0.09627996053624281, + "flos": 54152278807680.0, + "grad_norm": 2.666805333119253, + "language_loss": 0.77663833, + "learning_rate": 3.954101025264517e-06, + "loss": 0.79815578, + "num_input_tokens_seen": 92949285, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.22851562, + "step": 3318, + "time_per_iteration": 2.5797574520111084 + }, + { + "auxiliary_loss_clip": 0.01109746, + "auxiliary_loss_mlp": 0.01051514, + "balance_loss_clip": 1.03198767, + "balance_loss_mlp": 1.02800596, + "epoch": 0.09630897800475886, + "flos": 19237107957120.0, + "grad_norm": 2.335880684407148, + "language_loss": 0.84062922, + "learning_rate": 3.954060979328432e-06, + "loss": 0.86224186, + "num_input_tokens_seen": 92963975, + "router_z_loss_clip": 0.77832031, + "router_z_loss_mlp": 0.23510742, + "step": 3319, + "time_per_iteration": 2.417003870010376 + }, + { + "auxiliary_loss_clip": 0.01012778, + "auxiliary_loss_mlp": 0.01001949, + "balance_loss_clip": 1.00141478, + "balance_loss_mlp": 1.00102496, + "epoch": 0.09633799547327491, + "flos": 72257989512960.0, + "grad_norm": 0.6137810051109168, + "language_loss": 0.49373138, + "learning_rate": 3.954020916133289e-06, + "loss": 0.5138787, + "num_input_tokens_seen": 93028700, + "router_z_loss_clip": 0.11376953, + "router_z_loss_mlp": 0.00921631, + "step": 3320, + "time_per_iteration": 3.0546934604644775 + }, + { + "auxiliary_loss_clip": 0.01107092, + "auxiliary_loss_mlp": 0.01044078, + "balance_loss_clip": 1.03267539, + "balance_loss_mlp": 1.02196503, + "epoch": 0.09636701294179095, + "flos": 18836095547520.0, + "grad_norm": 4.224077213907138, + "language_loss": 0.94609845, + "learning_rate": 3.953980835679442e-06, + "loss": 0.96761012, + "num_input_tokens_seen": 93043220, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.22106934, + "step": 3321, + "time_per_iteration": 2.4218671321868896 + }, + { + "auxiliary_loss_clip": 0.01013519, + "auxiliary_loss_mlp": 0.01001524, + "balance_loss_clip": 1.00197554, + "balance_loss_mlp": 1.00051677, + "epoch": 0.096396030410307, + "flos": 70362065975040.0, + "grad_norm": 0.6764086723538701, + "language_loss": 0.52133876, + "learning_rate": 3.953940737967247e-06, + "loss": 0.54148918, + "num_input_tokens_seen": 93111250, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.0100708, + "step": 3322, + "time_per_iteration": 3.129873037338257 + }, + { + "auxiliary_loss_clip": 0.01116346, + "auxiliary_loss_mlp": 0.0105087, + "balance_loss_clip": 1.03516269, + "balance_loss_mlp": 1.02606273, + "epoch": 0.09642504787882306, + "flos": 28432497335040.0, + "grad_norm": 2.5801817391122874, + "language_loss": 1.01373506, + "learning_rate": 3.9539006229970555e-06, + "loss": 1.03540719, + "num_input_tokens_seen": 93131135, + "router_z_loss_clip": 0.8125, + "router_z_loss_mlp": 0.2479248, + "step": 3323, + "time_per_iteration": 2.461642265319824 + }, + { + "auxiliary_loss_clip": 0.01102801, + "auxiliary_loss_mlp": 0.01050453, + "balance_loss_clip": 1.03251314, + "balance_loss_mlp": 1.03109312, + "epoch": 0.0964540653473391, + "flos": 32124178120320.0, + "grad_norm": 1.8708436257787742, + "language_loss": 0.61379278, + "learning_rate": 3.953860490769224e-06, + "loss": 0.63532531, + "num_input_tokens_seen": 93149115, + "router_z_loss_clip": 0.70361328, + "router_z_loss_mlp": 0.19372559, + "step": 3324, + "time_per_iteration": 2.503978967666626 + }, + { + "auxiliary_loss_clip": 0.01105968, + "auxiliary_loss_mlp": 0.01039781, + "balance_loss_clip": 1.03367758, + "balance_loss_mlp": 1.01894319, + "epoch": 0.09648308281585515, + "flos": 30443040466560.0, + "grad_norm": 2.415284291995839, + "language_loss": 0.77332383, + "learning_rate": 3.953820341284105e-06, + "loss": 0.79478133, + "num_input_tokens_seen": 93166555, + "router_z_loss_clip": 0.72314453, + "router_z_loss_mlp": 0.20812988, + "step": 3325, + "time_per_iteration": 2.41957426071167 + }, + { + "auxiliary_loss_clip": 0.01114456, + "auxiliary_loss_mlp": 0.01053738, + "balance_loss_clip": 1.03433824, + "balance_loss_mlp": 1.02802467, + "epoch": 0.0965121002843712, + "flos": 37662415914240.0, + "grad_norm": 2.0464400492954615, + "language_loss": 0.96743, + "learning_rate": 3.953780174542054e-06, + "loss": 0.98911202, + "num_input_tokens_seen": 93185165, + "router_z_loss_clip": 0.80078125, + "router_z_loss_mlp": 0.25720215, + "step": 3326, + "time_per_iteration": 2.574725866317749 + }, + { + "auxiliary_loss_clip": 0.01019324, + "auxiliary_loss_mlp": 0.01001494, + "balance_loss_clip": 1.00739479, + "balance_loss_mlp": 1.0005343, + "epoch": 0.09654111775288723, + "flos": 58968999244800.0, + "grad_norm": 0.7379052650381902, + "language_loss": 0.51569545, + "learning_rate": 3.953739990543427e-06, + "loss": 0.53590369, + "num_input_tokens_seen": 93243850, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.00958252, + "step": 3327, + "time_per_iteration": 2.8520967960357666 + }, + { + "auxiliary_loss_clip": 0.01019727, + "auxiliary_loss_mlp": 0.01001768, + "balance_loss_clip": 1.00785446, + "balance_loss_mlp": 1.0007906, + "epoch": 0.09657013522140329, + "flos": 62152100121600.0, + "grad_norm": 0.625189552624857, + "language_loss": 0.42986265, + "learning_rate": 3.953699789288576e-06, + "loss": 0.45007762, + "num_input_tokens_seen": 93311805, + "router_z_loss_clip": 0.11865234, + "router_z_loss_mlp": 0.00976562, + "step": 3328, + "time_per_iteration": 3.1239185333251953 + }, + { + "auxiliary_loss_clip": 0.01099622, + "auxiliary_loss_mlp": 0.01051917, + "balance_loss_clip": 1.03236353, + "balance_loss_mlp": 1.03101349, + "epoch": 0.09659915268991934, + "flos": 23215460849280.0, + "grad_norm": 3.098745589734057, + "language_loss": 0.87489986, + "learning_rate": 3.9536595707778605e-06, + "loss": 0.89641523, + "num_input_tokens_seen": 93326580, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.20916748, + "step": 3329, + "time_per_iteration": 2.3670034408569336 + }, + { + "auxiliary_loss_clip": 0.01017995, + "auxiliary_loss_mlp": 0.01000438, + "balance_loss_clip": 1.00619435, + "balance_loss_mlp": 0.99937081, + "epoch": 0.09662817015843538, + "flos": 58097113061760.0, + "grad_norm": 0.6207007466875056, + "language_loss": 0.42924356, + "learning_rate": 3.9536193350116315e-06, + "loss": 0.4494279, + "num_input_tokens_seen": 93386880, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.01068115, + "step": 3330, + "time_per_iteration": 2.9728188514709473 + }, + { + "auxiliary_loss_clip": 0.01108283, + "auxiliary_loss_mlp": 0.01044711, + "balance_loss_clip": 1.03362048, + "balance_loss_mlp": 1.02376592, + "epoch": 0.09665718762695143, + "flos": 31642621470720.0, + "grad_norm": 2.8472263887837603, + "language_loss": 0.75763863, + "learning_rate": 3.953579081990246e-06, + "loss": 0.77916861, + "num_input_tokens_seen": 93403175, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.20922852, + "step": 3331, + "time_per_iteration": 2.5873355865478516 + }, + { + "auxiliary_loss_clip": 0.01016904, + "auxiliary_loss_mlp": 0.01002034, + "balance_loss_clip": 1.00515342, + "balance_loss_mlp": 1.00097299, + "epoch": 0.09668620509546746, + "flos": 74772329696640.0, + "grad_norm": 0.676292520194858, + "language_loss": 0.4831138, + "learning_rate": 3.95353881171406e-06, + "loss": 0.50330317, + "num_input_tokens_seen": 93468200, + "router_z_loss_clip": 0.11767578, + "router_z_loss_mlp": 0.01062012, + "step": 3332, + "time_per_iteration": 3.0844130516052246 + }, + { + "auxiliary_loss_clip": 0.01107785, + "auxiliary_loss_mlp": 0.01039877, + "balance_loss_clip": 1.03355134, + "balance_loss_mlp": 1.0176686, + "epoch": 0.09671522256398352, + "flos": 36531858401280.0, + "grad_norm": 3.74200982701645, + "language_loss": 0.80132508, + "learning_rate": 3.953498524183429e-06, + "loss": 0.82280171, + "num_input_tokens_seen": 93486025, + "router_z_loss_clip": 0.74267578, + "router_z_loss_mlp": 0.2220459, + "step": 3333, + "time_per_iteration": 2.4969711303710938 + }, + { + "auxiliary_loss_clip": 0.01115942, + "auxiliary_loss_mlp": 0.01047806, + "balance_loss_clip": 1.0355823, + "balance_loss_mlp": 1.02576375, + "epoch": 0.09674424003249957, + "flos": 35395296134400.0, + "grad_norm": 4.783216457528765, + "language_loss": 1.11473799, + "learning_rate": 3.953458219398707e-06, + "loss": 1.13637555, + "num_input_tokens_seen": 93499540, + "router_z_loss_clip": 0.80419922, + "router_z_loss_mlp": 0.22045898, + "step": 3334, + "time_per_iteration": 2.5065691471099854 + }, + { + "auxiliary_loss_clip": 0.01106447, + "auxiliary_loss_mlp": 0.01049013, + "balance_loss_clip": 1.03423262, + "balance_loss_mlp": 1.02729356, + "epoch": 0.0967732575010156, + "flos": 31021830852480.0, + "grad_norm": 2.592302170413913, + "language_loss": 0.79836506, + "learning_rate": 3.953417897360253e-06, + "loss": 0.81991965, + "num_input_tokens_seen": 93516265, + "router_z_loss_clip": 0.72167969, + "router_z_loss_mlp": 0.21716309, + "step": 3335, + "time_per_iteration": 2.462000846862793 + }, + { + "auxiliary_loss_clip": 0.01111484, + "auxiliary_loss_mlp": 0.010563, + "balance_loss_clip": 1.03545141, + "balance_loss_mlp": 1.03380513, + "epoch": 0.09680227496953166, + "flos": 28105605475200.0, + "grad_norm": 2.066686297316839, + "language_loss": 0.903337, + "learning_rate": 3.953377558068421e-06, + "loss": 0.92501485, + "num_input_tokens_seen": 93534020, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.22485352, + "step": 3336, + "time_per_iteration": 2.4714109897613525 + }, + { + "auxiliary_loss_clip": 0.01018271, + "auxiliary_loss_mlp": 0.01022732, + "balance_loss_clip": 1.00597632, + "balance_loss_mlp": 1.0217067, + "epoch": 0.09683129243804771, + "flos": 74779102500480.0, + "grad_norm": 0.6438231236144981, + "language_loss": 0.50693572, + "learning_rate": 3.9533372015235685e-06, + "loss": 0.52734578, + "num_input_tokens_seen": 93605390, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01025391, + "step": 3337, + "time_per_iteration": 3.231182098388672 + }, + { + "auxiliary_loss_clip": 0.01108438, + "auxiliary_loss_mlp": 0.01044276, + "balance_loss_clip": 1.03513753, + "balance_loss_mlp": 1.02343798, + "epoch": 0.09686030990656375, + "flos": 42300976216320.0, + "grad_norm": 2.0612386289017715, + "language_loss": 0.93492162, + "learning_rate": 3.953296827726051e-06, + "loss": 0.95644879, + "num_input_tokens_seen": 93630345, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.20812988, + "step": 3338, + "time_per_iteration": 2.618983030319214 + }, + { + "auxiliary_loss_clip": 0.01101819, + "auxiliary_loss_mlp": 0.01040585, + "balance_loss_clip": 1.03343368, + "balance_loss_mlp": 1.01988983, + "epoch": 0.0968893273750798, + "flos": 40617604235520.0, + "grad_norm": 2.0965701019011886, + "language_loss": 0.9163838, + "learning_rate": 3.953256436676225e-06, + "loss": 0.9378078, + "num_input_tokens_seen": 93649140, + "router_z_loss_clip": 0.68457031, + "router_z_loss_mlp": 0.20715332, + "step": 3339, + "time_per_iteration": 2.636404514312744 + }, + { + "auxiliary_loss_clip": 0.01022141, + "auxiliary_loss_mlp": 0.01018007, + "balance_loss_clip": 1.0094254, + "balance_loss_mlp": 1.01698804, + "epoch": 0.09691834484359585, + "flos": 72545534403840.0, + "grad_norm": 0.6397230039567023, + "language_loss": 0.48018223, + "learning_rate": 3.9532160283744485e-06, + "loss": 0.50058371, + "num_input_tokens_seen": 93717825, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01019287, + "step": 3340, + "time_per_iteration": 3.1450295448303223 + }, + { + "auxiliary_loss_clip": 0.01099078, + "auxiliary_loss_mlp": 0.01034833, + "balance_loss_clip": 1.03234267, + "balance_loss_mlp": 1.01556921, + "epoch": 0.09694736231211189, + "flos": 74730121862400.0, + "grad_norm": 2.1362128577119797, + "language_loss": 0.67584497, + "learning_rate": 3.953175602821077e-06, + "loss": 0.69718409, + "num_input_tokens_seen": 93741825, + "router_z_loss_clip": 0.66772461, + "router_z_loss_mlp": 0.19238281, + "step": 3341, + "time_per_iteration": 2.791144847869873 + }, + { + "auxiliary_loss_clip": 0.01114376, + "auxiliary_loss_mlp": 0.01049663, + "balance_loss_clip": 1.03764248, + "balance_loss_mlp": 1.02567792, + "epoch": 0.09697637978062794, + "flos": 74729353812480.0, + "grad_norm": 2.653267671137036, + "language_loss": 0.83490759, + "learning_rate": 3.95313516001647e-06, + "loss": 0.85654795, + "num_input_tokens_seen": 93764820, + "router_z_loss_clip": 0.76660156, + "router_z_loss_mlp": 0.23986816, + "step": 3342, + "time_per_iteration": 2.7742457389831543 + }, + { + "auxiliary_loss_clip": 0.01108901, + "auxiliary_loss_mlp": 0.01037702, + "balance_loss_clip": 1.03712511, + "balance_loss_mlp": 1.01624393, + "epoch": 0.09700539724914399, + "flos": 25439393410560.0, + "grad_norm": 2.6674457548823987, + "language_loss": 0.88892186, + "learning_rate": 3.953094699960981e-06, + "loss": 0.91038787, + "num_input_tokens_seen": 93781590, + "router_z_loss_clip": 0.71777344, + "router_z_loss_mlp": 0.21435547, + "step": 3343, + "time_per_iteration": 2.4267024993896484 + }, + { + "auxiliary_loss_clip": 0.01035741, + "auxiliary_loss_mlp": 0.01002427, + "balance_loss_clip": 1.02224731, + "balance_loss_mlp": 1.0013597, + "epoch": 0.09703441471766003, + "flos": 74768314890240.0, + "grad_norm": 0.6673939993501593, + "language_loss": 0.47826961, + "learning_rate": 3.9530542226549696e-06, + "loss": 0.49865127, + "num_input_tokens_seen": 93843855, + "router_z_loss_clip": 0.13476562, + "router_z_loss_mlp": 0.01068115, + "step": 3344, + "time_per_iteration": 3.0286874771118164 + }, + { + "auxiliary_loss_clip": 0.01113379, + "auxiliary_loss_mlp": 0.01055424, + "balance_loss_clip": 1.0397408, + "balance_loss_mlp": 1.03216672, + "epoch": 0.09706343218617608, + "flos": 15369988256640.0, + "grad_norm": 2.7634483357581963, + "language_loss": 0.87169957, + "learning_rate": 3.953013728098793e-06, + "loss": 0.89338756, + "num_input_tokens_seen": 93857705, + "router_z_loss_clip": 0.73681641, + "router_z_loss_mlp": 0.23266602, + "step": 3345, + "time_per_iteration": 2.394225835800171 + }, + { + "auxiliary_loss_clip": 0.01107733, + "auxiliary_loss_mlp": 0.01044403, + "balance_loss_clip": 1.03928435, + "balance_loss_mlp": 1.02405357, + "epoch": 0.09709244965469213, + "flos": 34965864011520.0, + "grad_norm": 1.9543375473587183, + "language_loss": 0.74622607, + "learning_rate": 3.9529732162928095e-06, + "loss": 0.76774746, + "num_input_tokens_seen": 93877710, + "router_z_loss_clip": 0.68481445, + "router_z_loss_mlp": 0.20336914, + "step": 3346, + "time_per_iteration": 2.4987571239471436 + }, + { + "auxiliary_loss_clip": 0.01115013, + "auxiliary_loss_mlp": 0.01060224, + "balance_loss_clip": 1.0411818, + "balance_loss_mlp": 1.03828943, + "epoch": 0.09712146712320817, + "flos": 17812721508480.0, + "grad_norm": 2.3328895867751096, + "language_loss": 0.89075977, + "learning_rate": 3.9529326872373755e-06, + "loss": 0.91251218, + "num_input_tokens_seen": 93892715, + "router_z_loss_clip": 0.73925781, + "router_z_loss_mlp": 0.21936035, + "step": 3347, + "time_per_iteration": 2.4163801670074463 + }, + { + "auxiliary_loss_clip": 0.01027802, + "auxiliary_loss_mlp": 0.0101491, + "balance_loss_clip": 1.01502919, + "balance_loss_mlp": 1.01390314, + "epoch": 0.09715048459172422, + "flos": 74772085317120.0, + "grad_norm": 0.7195377082647838, + "language_loss": 0.49703825, + "learning_rate": 3.952892140932851e-06, + "loss": 0.51746535, + "num_input_tokens_seen": 93958110, + "router_z_loss_clip": 0.12792969, + "router_z_loss_mlp": 0.0100708, + "step": 3348, + "time_per_iteration": 3.1218583583831787 + }, + { + "auxiliary_loss_clip": 0.01110981, + "auxiliary_loss_mlp": 0.01058045, + "balance_loss_clip": 1.03520799, + "balance_loss_mlp": 1.03535914, + "epoch": 0.09717950206024026, + "flos": 15004692034560.0, + "grad_norm": 3.2437663351783885, + "language_loss": 0.97762626, + "learning_rate": 3.952851577379591e-06, + "loss": 0.99931639, + "num_input_tokens_seen": 93971550, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.22705078, + "step": 3349, + "time_per_iteration": 4.6393821239471436 + }, + { + "auxiliary_loss_clip": 0.01115304, + "auxiliary_loss_mlp": 0.01050548, + "balance_loss_clip": 1.03612208, + "balance_loss_mlp": 1.02548993, + "epoch": 0.09720851952875631, + "flos": 11809614695040.0, + "grad_norm": 6.461918551527609, + "language_loss": 0.97317994, + "learning_rate": 3.952810996577957e-06, + "loss": 0.99483836, + "num_input_tokens_seen": 93985550, + "router_z_loss_clip": 0.79199219, + "router_z_loss_mlp": 0.25061035, + "step": 3350, + "time_per_iteration": 2.4556007385253906 + }, + { + "auxiliary_loss_clip": 0.01117461, + "auxiliary_loss_mlp": 0.01060029, + "balance_loss_clip": 1.04097998, + "balance_loss_mlp": 1.0379281, + "epoch": 0.09723753699727236, + "flos": 12231994723200.0, + "grad_norm": 3.0426329599536506, + "language_loss": 0.96137786, + "learning_rate": 3.9527703985283055e-06, + "loss": 0.98315287, + "num_input_tokens_seen": 93997240, + "router_z_loss_clip": 0.76513672, + "router_z_loss_mlp": 0.22119141, + "step": 3351, + "time_per_iteration": 4.665435791015625 + }, + { + "auxiliary_loss_clip": 0.01022848, + "auxiliary_loss_mlp": 0.01008497, + "balance_loss_clip": 1.01031888, + "balance_loss_mlp": 1.00739443, + "epoch": 0.0972665544657884, + "flos": 67725320964480.0, + "grad_norm": 0.6230551961383622, + "language_loss": 0.46520925, + "learning_rate": 3.952729783230996e-06, + "loss": 0.48552269, + "num_input_tokens_seen": 94057605, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.01104736, + "step": 3352, + "time_per_iteration": 2.9758529663085938 + }, + { + "auxiliary_loss_clip": 0.01114187, + "auxiliary_loss_mlp": 0.01046704, + "balance_loss_clip": 1.04047024, + "balance_loss_mlp": 1.02588999, + "epoch": 0.09729557193430445, + "flos": 25919274314880.0, + "grad_norm": 1.500273607524027, + "language_loss": 0.80333722, + "learning_rate": 3.9526891506863865e-06, + "loss": 0.82494617, + "num_input_tokens_seen": 94082725, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.20800781, + "step": 3353, + "time_per_iteration": 2.5531914234161377 + }, + { + "auxiliary_loss_clip": 0.01123057, + "auxiliary_loss_mlp": 0.0104255, + "balance_loss_clip": 1.04188061, + "balance_loss_mlp": 1.01692033, + "epoch": 0.0973245894028205, + "flos": 33866938056960.0, + "grad_norm": 2.661465550545743, + "language_loss": 0.59761673, + "learning_rate": 3.952648500894836e-06, + "loss": 0.61927283, + "num_input_tokens_seen": 94098765, + "router_z_loss_clip": 0.81152344, + "router_z_loss_mlp": 0.25646973, + "step": 3354, + "time_per_iteration": 2.4555702209472656 + }, + { + "auxiliary_loss_clip": 0.01108752, + "auxiliary_loss_mlp": 0.0104844, + "balance_loss_clip": 1.03767598, + "balance_loss_mlp": 1.02649331, + "epoch": 0.09735360687133654, + "flos": 27775292302080.0, + "grad_norm": 2.256336129201585, + "language_loss": 0.68421853, + "learning_rate": 3.952607833856704e-06, + "loss": 0.70579046, + "num_input_tokens_seen": 94114050, + "router_z_loss_clip": 0.71142578, + "router_z_loss_mlp": 0.21972656, + "step": 3355, + "time_per_iteration": 2.374830961227417 + }, + { + "auxiliary_loss_clip": 0.01030527, + "auxiliary_loss_mlp": 0.01002911, + "balance_loss_clip": 1.0180968, + "balance_loss_mlp": 1.00168324, + "epoch": 0.09738262433985259, + "flos": 56853332409600.0, + "grad_norm": 0.7268297508297819, + "language_loss": 0.54955781, + "learning_rate": 3.95256714957235e-06, + "loss": 0.56989217, + "num_input_tokens_seen": 94165965, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.01226807, + "step": 3356, + "time_per_iteration": 5.187344074249268 + }, + { + "auxiliary_loss_clip": 0.01118389, + "auxiliary_loss_mlp": 0.01043111, + "balance_loss_clip": 1.04373455, + "balance_loss_mlp": 1.02178478, + "epoch": 0.09741164180836864, + "flos": 23104925884800.0, + "grad_norm": 6.877447616065933, + "language_loss": 1.05386317, + "learning_rate": 3.952526448042132e-06, + "loss": 1.07547808, + "num_input_tokens_seen": 94181455, + "router_z_loss_clip": 0.74609375, + "router_z_loss_mlp": 0.21313477, + "step": 3357, + "time_per_iteration": 4.783688306808472 + }, + { + "auxiliary_loss_clip": 0.01112575, + "auxiliary_loss_mlp": 0.01037917, + "balance_loss_clip": 1.04489982, + "balance_loss_mlp": 1.02010679, + "epoch": 0.09744065927688468, + "flos": 33576809725440.0, + "grad_norm": 2.3689303749329786, + "language_loss": 0.85549664, + "learning_rate": 3.952485729266411e-06, + "loss": 0.87700158, + "num_input_tokens_seen": 94198660, + "router_z_loss_clip": 0.67626953, + "router_z_loss_mlp": 0.17822266, + "step": 3358, + "time_per_iteration": 2.543905019760132 + }, + { + "auxiliary_loss_clip": 0.01118315, + "auxiliary_loss_mlp": 0.01047609, + "balance_loss_clip": 1.04204261, + "balance_loss_mlp": 1.02331388, + "epoch": 0.09746967674540073, + "flos": 27995803649280.0, + "grad_norm": 2.0858088334337097, + "language_loss": 0.96478975, + "learning_rate": 3.952444993245546e-06, + "loss": 0.98644888, + "num_input_tokens_seen": 94219485, + "router_z_loss_clip": 0.76269531, + "router_z_loss_mlp": 0.24304199, + "step": 3359, + "time_per_iteration": 2.618138074874878 + }, + { + "auxiliary_loss_clip": 0.01120159, + "auxiliary_loss_mlp": 0.01051685, + "balance_loss_clip": 1.04534197, + "balance_loss_mlp": 1.03028703, + "epoch": 0.09749869421391678, + "flos": 16317181975680.0, + "grad_norm": 3.2999059036559526, + "language_loss": 0.84896767, + "learning_rate": 3.952404239979896e-06, + "loss": 0.87068611, + "num_input_tokens_seen": 94232035, + "router_z_loss_clip": 0.74804688, + "router_z_loss_mlp": 0.21411133, + "step": 3360, + "time_per_iteration": 2.384371757507324 + }, + { + "auxiliary_loss_clip": 0.01123477, + "auxiliary_loss_mlp": 0.01050832, + "balance_loss_clip": 1.04651427, + "balance_loss_mlp": 1.0276221, + "epoch": 0.09752771168243282, + "flos": 29234173040640.0, + "grad_norm": 2.514998317810507, + "language_loss": 0.9264369, + "learning_rate": 3.952363469469823e-06, + "loss": 0.94817996, + "num_input_tokens_seen": 94248280, + "router_z_loss_clip": 0.77099609, + "router_z_loss_mlp": 0.23193359, + "step": 3361, + "time_per_iteration": 2.442263126373291 + }, + { + "auxiliary_loss_clip": 0.01116258, + "auxiliary_loss_mlp": 0.01048774, + "balance_loss_clip": 1.04230464, + "balance_loss_mlp": 1.02750707, + "epoch": 0.09755672915094887, + "flos": 25110755982720.0, + "grad_norm": 2.548732852348562, + "language_loss": 0.8919524, + "learning_rate": 3.952322681715685e-06, + "loss": 0.91360277, + "num_input_tokens_seen": 94263490, + "router_z_loss_clip": 0.73974609, + "router_z_loss_mlp": 0.21264648, + "step": 3362, + "time_per_iteration": 2.399507761001587 + }, + { + "auxiliary_loss_clip": 0.01120994, + "auxiliary_loss_mlp": 0.01050745, + "balance_loss_clip": 1.04684114, + "balance_loss_mlp": 1.02926338, + "epoch": 0.09758574661946492, + "flos": 26607203210880.0, + "grad_norm": 2.6888104369929553, + "language_loss": 0.70905328, + "learning_rate": 3.952281876717843e-06, + "loss": 0.73077065, + "num_input_tokens_seen": 94277790, + "router_z_loss_clip": 0.7421875, + "router_z_loss_mlp": 0.21472168, + "step": 3363, + "time_per_iteration": 2.4092061519622803 + }, + { + "auxiliary_loss_clip": 0.01118097, + "auxiliary_loss_mlp": 0.01054612, + "balance_loss_clip": 1.04172003, + "balance_loss_mlp": 1.03265405, + "epoch": 0.09761476408798096, + "flos": 16574421939840.0, + "grad_norm": 2.795172679104417, + "language_loss": 0.87174076, + "learning_rate": 3.952241054476658e-06, + "loss": 0.89346778, + "num_input_tokens_seen": 94290890, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.21960449, + "step": 3364, + "time_per_iteration": 2.3434994220733643 + }, + { + "auxiliary_loss_clip": 0.01037739, + "auxiliary_loss_mlp": 0.01011023, + "balance_loss_clip": 1.02517629, + "balance_loss_mlp": 1.00958014, + "epoch": 0.09764378155649701, + "flos": 56849946007680.0, + "grad_norm": 0.7260277190840381, + "language_loss": 0.54424614, + "learning_rate": 3.952200214992489e-06, + "loss": 0.56473374, + "num_input_tokens_seen": 94347280, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.0144043, + "step": 3365, + "time_per_iteration": 2.8579747676849365 + }, + { + "auxiliary_loss_clip": 0.0103532, + "auxiliary_loss_mlp": 0.01005595, + "balance_loss_clip": 1.02283669, + "balance_loss_mlp": 1.0042417, + "epoch": 0.09767279902501305, + "flos": 60280825870080.0, + "grad_norm": 0.6735012815076895, + "language_loss": 0.47150069, + "learning_rate": 3.9521593582656975e-06, + "loss": 0.49190986, + "num_input_tokens_seen": 94404235, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.0135498, + "step": 3366, + "time_per_iteration": 2.905251979827881 + }, + { + "auxiliary_loss_clip": 0.01115301, + "auxiliary_loss_mlp": 0.01052967, + "balance_loss_clip": 1.03739738, + "balance_loss_mlp": 1.02745628, + "epoch": 0.0977018164935291, + "flos": 38756454278400.0, + "grad_norm": 3.4451829437094066, + "language_loss": 0.75518936, + "learning_rate": 3.952118484296646e-06, + "loss": 0.77687204, + "num_input_tokens_seen": 94420050, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.25524902, + "step": 3367, + "time_per_iteration": 2.537679433822632 + }, + { + "auxiliary_loss_clip": 0.0110732, + "auxiliary_loss_mlp": 0.01047101, + "balance_loss_clip": 1.03429294, + "balance_loss_mlp": 1.02536869, + "epoch": 0.09773083396204515, + "flos": 32298220581120.0, + "grad_norm": 2.175669960855852, + "language_loss": 0.94773114, + "learning_rate": 3.952077593085694e-06, + "loss": 0.9692753, + "num_input_tokens_seen": 94437860, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.21728516, + "step": 3368, + "time_per_iteration": 2.5070478916168213 + }, + { + "auxiliary_loss_clip": 0.0111344, + "auxiliary_loss_mlp": 0.01051611, + "balance_loss_clip": 1.03667128, + "balance_loss_mlp": 1.0281744, + "epoch": 0.09775985143056119, + "flos": 30621586492800.0, + "grad_norm": 2.141613083489615, + "language_loss": 0.91157734, + "learning_rate": 3.952036684633201e-06, + "loss": 0.93322784, + "num_input_tokens_seen": 94459605, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.234375, + "step": 3369, + "time_per_iteration": 2.5155932903289795 + }, + { + "auxiliary_loss_clip": 0.01022889, + "auxiliary_loss_mlp": 0.01009866, + "balance_loss_clip": 1.01057577, + "balance_loss_mlp": 1.0086863, + "epoch": 0.09778886889907724, + "flos": 71813405859840.0, + "grad_norm": 0.6616531539975372, + "language_loss": 0.524351, + "learning_rate": 3.951995758939532e-06, + "loss": 0.54467845, + "num_input_tokens_seen": 94524810, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01177979, + "step": 3370, + "time_per_iteration": 3.0782182216644287 + }, + { + "auxiliary_loss_clip": 0.01103184, + "auxiliary_loss_mlp": 0.01051808, + "balance_loss_clip": 1.03587484, + "balance_loss_mlp": 1.03325939, + "epoch": 0.0978178863675933, + "flos": 27263360903040.0, + "grad_norm": 2.1381278004790762, + "language_loss": 0.72186553, + "learning_rate": 3.951954816005046e-06, + "loss": 0.74341547, + "num_input_tokens_seen": 94540165, + "router_z_loss_clip": 0.67285156, + "router_z_loss_mlp": 0.18554688, + "step": 3371, + "time_per_iteration": 2.474102258682251 + }, + { + "auxiliary_loss_clip": 0.01114873, + "auxiliary_loss_mlp": 0.01055012, + "balance_loss_clip": 1.03522873, + "balance_loss_mlp": 1.03232598, + "epoch": 0.09784690383610933, + "flos": 21828221953920.0, + "grad_norm": 2.9105894953012057, + "language_loss": 0.92339826, + "learning_rate": 3.951913855830104e-06, + "loss": 0.94509709, + "num_input_tokens_seen": 94553455, + "router_z_loss_clip": 0.79638672, + "router_z_loss_mlp": 0.22680664, + "step": 3372, + "time_per_iteration": 2.439373731613159 + }, + { + "auxiliary_loss_clip": 0.01110625, + "auxiliary_loss_mlp": 0.01047399, + "balance_loss_clip": 1.03514099, + "balance_loss_mlp": 1.02590585, + "epoch": 0.09787592130462539, + "flos": 30219177628800.0, + "grad_norm": 2.301779530244768, + "language_loss": 0.79845679, + "learning_rate": 3.951872878415071e-06, + "loss": 0.82003701, + "num_input_tokens_seen": 94571310, + "router_z_loss_clip": 0.75439453, + "router_z_loss_mlp": 0.21508789, + "step": 3373, + "time_per_iteration": 2.469477891921997 + }, + { + "auxiliary_loss_clip": 0.01100906, + "auxiliary_loss_mlp": 0.01037821, + "balance_loss_clip": 1.03680491, + "balance_loss_mlp": 1.02011204, + "epoch": 0.09790493877314144, + "flos": 35363175816960.0, + "grad_norm": 1.9949817742204747, + "language_loss": 0.7566039, + "learning_rate": 3.951831883760306e-06, + "loss": 0.77799118, + "num_input_tokens_seen": 94587680, + "router_z_loss_clip": 0.64160156, + "router_z_loss_mlp": 0.17694092, + "step": 3374, + "time_per_iteration": 2.579895496368408 + }, + { + "auxiliary_loss_clip": 0.01107786, + "auxiliary_loss_mlp": 0.01041684, + "balance_loss_clip": 1.0367192, + "balance_loss_mlp": 1.02212214, + "epoch": 0.09793395624165747, + "flos": 42258661781760.0, + "grad_norm": 1.8988097402624613, + "language_loss": 0.77715921, + "learning_rate": 3.951790871866172e-06, + "loss": 0.79865396, + "num_input_tokens_seen": 94603940, + "router_z_loss_clip": 0.7109375, + "router_z_loss_mlp": 0.19567871, + "step": 3375, + "time_per_iteration": 2.564103603363037 + }, + { + "auxiliary_loss_clip": 0.01020733, + "auxiliary_loss_mlp": 0.01014972, + "balance_loss_clip": 1.00858688, + "balance_loss_mlp": 1.0137862, + "epoch": 0.09796297371017353, + "flos": 66379244428800.0, + "grad_norm": 0.7092375679756757, + "language_loss": 0.48545679, + "learning_rate": 3.951749842733031e-06, + "loss": 0.50581384, + "num_input_tokens_seen": 94664875, + "router_z_loss_clip": 0.12109375, + "router_z_loss_mlp": 0.01184082, + "step": 3376, + "time_per_iteration": 3.007734537124634 + }, + { + "auxiliary_loss_clip": 0.01022755, + "auxiliary_loss_mlp": 0.01009958, + "balance_loss_clip": 1.01050615, + "balance_loss_mlp": 1.00871825, + "epoch": 0.09799199117868958, + "flos": 63645300593280.0, + "grad_norm": 0.6819687808916981, + "language_loss": 0.52578413, + "learning_rate": 3.951708796361245e-06, + "loss": 0.54611123, + "num_input_tokens_seen": 94728080, + "router_z_loss_clip": 0.12255859, + "router_z_loss_mlp": 0.01239014, + "step": 3377, + "time_per_iteration": 2.9795167446136475 + }, + { + "auxiliary_loss_clip": 0.01105937, + "auxiliary_loss_mlp": 0.01042174, + "balance_loss_clip": 1.03738427, + "balance_loss_mlp": 1.02125311, + "epoch": 0.09802100864720562, + "flos": 22266835764480.0, + "grad_norm": 2.408178010704885, + "language_loss": 0.86428916, + "learning_rate": 3.9516677327511785e-06, + "loss": 0.88577026, + "num_input_tokens_seen": 94745225, + "router_z_loss_clip": 0.68554688, + "router_z_loss_mlp": 0.20935059, + "step": 3378, + "time_per_iteration": 2.4942665100097656 + }, + { + "auxiliary_loss_clip": 0.01028087, + "auxiliary_loss_mlp": 0.01002591, + "balance_loss_clip": 1.01549983, + "balance_loss_mlp": 1.00143445, + "epoch": 0.09805002611572167, + "flos": 68681521814400.0, + "grad_norm": 0.7515653242529633, + "language_loss": 0.48739702, + "learning_rate": 3.951626651903192e-06, + "loss": 0.50770378, + "num_input_tokens_seen": 94807780, + "router_z_loss_clip": 0.12597656, + "router_z_loss_mlp": 0.01153564, + "step": 3379, + "time_per_iteration": 3.0657708644866943 + }, + { + "auxiliary_loss_clip": 0.01118546, + "auxiliary_loss_mlp": 0.01053523, + "balance_loss_clip": 1.04218078, + "balance_loss_mlp": 1.03157616, + "epoch": 0.0980790435842377, + "flos": 37562389269120.0, + "grad_norm": 2.9026072405964984, + "language_loss": 0.80181587, + "learning_rate": 3.951585553817649e-06, + "loss": 0.82353657, + "num_input_tokens_seen": 94826975, + "router_z_loss_clip": 0.76416016, + "router_z_loss_mlp": 0.21960449, + "step": 3380, + "time_per_iteration": 2.5462934970855713 + }, + { + "auxiliary_loss_clip": 0.01115816, + "auxiliary_loss_mlp": 0.01052716, + "balance_loss_clip": 1.04300773, + "balance_loss_mlp": 1.03041148, + "epoch": 0.09810806105275376, + "flos": 36129484448640.0, + "grad_norm": 2.854892790043739, + "language_loss": 0.90020216, + "learning_rate": 3.9515444384949136e-06, + "loss": 0.92188752, + "num_input_tokens_seen": 94848370, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.22302246, + "step": 3381, + "time_per_iteration": 2.554224967956543 + }, + { + "auxiliary_loss_clip": 0.01118333, + "auxiliary_loss_mlp": 0.01060302, + "balance_loss_clip": 1.04134095, + "balance_loss_mlp": 1.03628087, + "epoch": 0.09813707852126981, + "flos": 16317147064320.0, + "grad_norm": 2.5510552999614977, + "language_loss": 0.86953902, + "learning_rate": 3.951503305935347e-06, + "loss": 0.89132541, + "num_input_tokens_seen": 94861275, + "router_z_loss_clip": 0.77001953, + "router_z_loss_mlp": 0.23999023, + "step": 3382, + "time_per_iteration": 2.34978985786438 + }, + { + "auxiliary_loss_clip": 0.01113385, + "auxiliary_loss_mlp": 0.01060656, + "balance_loss_clip": 1.0397501, + "balance_loss_mlp": 1.03739786, + "epoch": 0.09816609598978585, + "flos": 24563597155200.0, + "grad_norm": 2.5681630799814066, + "language_loss": 0.88689888, + "learning_rate": 3.951462156139314e-06, + "loss": 0.90863931, + "num_input_tokens_seen": 94876090, + "router_z_loss_clip": 0.73632812, + "router_z_loss_mlp": 0.23266602, + "step": 3383, + "time_per_iteration": 2.5075252056121826 + }, + { + "auxiliary_loss_clip": 0.01027716, + "auxiliary_loss_mlp": 0.01021528, + "balance_loss_clip": 1.01530361, + "balance_loss_mlp": 1.02046156, + "epoch": 0.0981951134583019, + "flos": 48160867299840.0, + "grad_norm": 0.7403631724041037, + "language_loss": 0.46325827, + "learning_rate": 3.951420989107178e-06, + "loss": 0.4837507, + "num_input_tokens_seen": 94920730, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.01068115, + "step": 3384, + "time_per_iteration": 2.680600643157959 + }, + { + "auxiliary_loss_clip": 0.01025095, + "auxiliary_loss_mlp": 0.01016716, + "balance_loss_clip": 1.01258826, + "balance_loss_mlp": 1.01563132, + "epoch": 0.09822413092681795, + "flos": 71336946268800.0, + "grad_norm": 0.6609649558901728, + "language_loss": 0.50381511, + "learning_rate": 3.951379804839301e-06, + "loss": 0.52423316, + "num_input_tokens_seen": 94983110, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.01086426, + "step": 3385, + "time_per_iteration": 2.951669216156006 + }, + { + "auxiliary_loss_clip": 0.01021445, + "auxiliary_loss_mlp": 0.01008807, + "balance_loss_clip": 1.00955296, + "balance_loss_mlp": 1.00764441, + "epoch": 0.09825314839533399, + "flos": 74769432053760.0, + "grad_norm": 0.6427863855175056, + "language_loss": 0.50736356, + "learning_rate": 3.9513386033360494e-06, + "loss": 0.52766603, + "num_input_tokens_seen": 95052360, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01159668, + "step": 3386, + "time_per_iteration": 3.1457533836364746 + }, + { + "auxiliary_loss_clip": 0.01020757, + "auxiliary_loss_mlp": 0.01002402, + "balance_loss_clip": 1.00926626, + "balance_loss_mlp": 1.00139499, + "epoch": 0.09828216586385004, + "flos": 62502384458880.0, + "grad_norm": 0.782289522112054, + "language_loss": 0.49433395, + "learning_rate": 3.951297384597785e-06, + "loss": 0.51456559, + "num_input_tokens_seen": 95103805, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.0100708, + "step": 3387, + "time_per_iteration": 2.8535878658294678 + }, + { + "auxiliary_loss_clip": 0.01108057, + "auxiliary_loss_mlp": 0.01044466, + "balance_loss_clip": 1.03307295, + "balance_loss_mlp": 1.02143455, + "epoch": 0.09831118333236609, + "flos": 34231291672320.0, + "grad_norm": 2.2220184504644203, + "language_loss": 0.84929132, + "learning_rate": 3.951256148624872e-06, + "loss": 0.87081659, + "num_input_tokens_seen": 95120895, + "router_z_loss_clip": 0.74902344, + "router_z_loss_mlp": 0.23022461, + "step": 3388, + "time_per_iteration": 2.5051233768463135 + }, + { + "auxiliary_loss_clip": 0.01024367, + "auxiliary_loss_mlp": 0.01003737, + "balance_loss_clip": 1.01205361, + "balance_loss_mlp": 1.00264657, + "epoch": 0.09834020080088213, + "flos": 73091855358720.0, + "grad_norm": 0.6323788687481829, + "language_loss": 0.5041213, + "learning_rate": 3.951214895417675e-06, + "loss": 0.52440226, + "num_input_tokens_seen": 95189375, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01092529, + "step": 3389, + "time_per_iteration": 3.0938150882720947 + }, + { + "auxiliary_loss_clip": 0.01111105, + "auxiliary_loss_mlp": 0.01040694, + "balance_loss_clip": 1.03767145, + "balance_loss_mlp": 1.01780534, + "epoch": 0.09836921826939818, + "flos": 29784543713280.0, + "grad_norm": 2.317064568686936, + "language_loss": 0.82563061, + "learning_rate": 3.951173624976559e-06, + "loss": 0.84714866, + "num_input_tokens_seen": 95206140, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.22888184, + "step": 3390, + "time_per_iteration": 2.4453887939453125 + }, + { + "auxiliary_loss_clip": 0.01029942, + "auxiliary_loss_mlp": 0.01006084, + "balance_loss_clip": 1.01703107, + "balance_loss_mlp": 1.00493324, + "epoch": 0.09839823573791423, + "flos": 74773097746560.0, + "grad_norm": 0.6612049722197852, + "language_loss": 0.47473574, + "learning_rate": 3.951132337301888e-06, + "loss": 0.49509603, + "num_input_tokens_seen": 95271020, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01147461, + "step": 3391, + "time_per_iteration": 3.1011109352111816 + }, + { + "auxiliary_loss_clip": 0.0111513, + "auxiliary_loss_mlp": 0.01042602, + "balance_loss_clip": 1.03871429, + "balance_loss_mlp": 1.02133501, + "epoch": 0.09842725320643027, + "flos": 27740518721280.0, + "grad_norm": 2.2431915068825203, + "language_loss": 0.673527, + "learning_rate": 3.951091032394027e-06, + "loss": 0.6951043, + "num_input_tokens_seen": 95288055, + "router_z_loss_clip": 0.76367188, + "router_z_loss_mlp": 0.21252441, + "step": 3392, + "time_per_iteration": 2.466578245162964 + }, + { + "auxiliary_loss_clip": 0.01109386, + "auxiliary_loss_mlp": 0.01041399, + "balance_loss_clip": 1.03673518, + "balance_loss_mlp": 1.02087069, + "epoch": 0.09845627067494632, + "flos": 28285862158080.0, + "grad_norm": 2.047024586847378, + "language_loss": 0.8855828, + "learning_rate": 3.95104971025334e-06, + "loss": 0.90709066, + "num_input_tokens_seen": 95304820, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.20544434, + "step": 3393, + "time_per_iteration": 2.4236881732940674 + }, + { + "auxiliary_loss_clip": 0.01119882, + "auxiliary_loss_mlp": 0.01055009, + "balance_loss_clip": 1.03931594, + "balance_loss_mlp": 1.02983177, + "epoch": 0.09848528814346237, + "flos": 18873592214400.0, + "grad_norm": 2.7628446954033037, + "language_loss": 0.94055039, + "learning_rate": 3.951008370880192e-06, + "loss": 0.96229929, + "num_input_tokens_seen": 95319580, + "router_z_loss_clip": 0.80517578, + "router_z_loss_mlp": 0.2520752, + "step": 3394, + "time_per_iteration": 2.466905355453491 + }, + { + "auxiliary_loss_clip": 0.01042167, + "auxiliary_loss_mlp": 0.01015724, + "balance_loss_clip": 1.028723, + "balance_loss_mlp": 1.01443624, + "epoch": 0.09851430561197841, + "flos": 62694929940480.0, + "grad_norm": 0.6695706999599664, + "language_loss": 0.48629674, + "learning_rate": 3.950967014274949e-06, + "loss": 0.50687563, + "num_input_tokens_seen": 95382455, + "router_z_loss_clip": 0.13476562, + "router_z_loss_mlp": 0.01287842, + "step": 3395, + "time_per_iteration": 3.0026865005493164 + }, + { + "auxiliary_loss_clip": 0.01038163, + "auxiliary_loss_mlp": 0.01010894, + "balance_loss_clip": 1.02524424, + "balance_loss_mlp": 1.00951171, + "epoch": 0.09854332308049446, + "flos": 55835753656320.0, + "grad_norm": 3.54596687439498, + "language_loss": 0.51433527, + "learning_rate": 3.950925640437976e-06, + "loss": 0.5348258, + "num_input_tokens_seen": 95440210, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01385498, + "step": 3396, + "time_per_iteration": 2.95998215675354 + }, + { + "auxiliary_loss_clip": 0.01115788, + "auxiliary_loss_mlp": 0.01063326, + "balance_loss_clip": 1.03890896, + "balance_loss_mlp": 1.04023504, + "epoch": 0.0985723405490105, + "flos": 18471427729920.0, + "grad_norm": 2.1775260529465497, + "language_loss": 0.7442857, + "learning_rate": 3.950884249369638e-06, + "loss": 0.76607692, + "num_input_tokens_seen": 95455670, + "router_z_loss_clip": 0.76855469, + "router_z_loss_mlp": 0.23120117, + "step": 3397, + "time_per_iteration": 2.382784128189087 + }, + { + "auxiliary_loss_clip": 0.01114693, + "auxiliary_loss_mlp": 0.01050376, + "balance_loss_clip": 1.03877604, + "balance_loss_mlp": 1.02862012, + "epoch": 0.09860135801752655, + "flos": 26243024152320.0, + "grad_norm": 2.672463681962324, + "language_loss": 0.87183583, + "learning_rate": 3.950842841070301e-06, + "loss": 0.89348656, + "num_input_tokens_seen": 95469555, + "router_z_loss_clip": 0.75927734, + "router_z_loss_mlp": 0.21740723, + "step": 3398, + "time_per_iteration": 2.406933307647705 + }, + { + "auxiliary_loss_clip": 0.01110241, + "auxiliary_loss_mlp": 0.01045516, + "balance_loss_clip": 1.03913677, + "balance_loss_mlp": 1.0244813, + "epoch": 0.0986303754860426, + "flos": 34633281600000.0, + "grad_norm": 2.2986871167113994, + "language_loss": 0.77028894, + "learning_rate": 3.950801415540331e-06, + "loss": 0.79184651, + "num_input_tokens_seen": 95485415, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.21051025, + "step": 3399, + "time_per_iteration": 2.4908039569854736 + }, + { + "auxiliary_loss_clip": 0.01111064, + "auxiliary_loss_mlp": 0.01052615, + "balance_loss_clip": 1.03831637, + "balance_loss_mlp": 1.03106213, + "epoch": 0.09865939295455864, + "flos": 15953107651200.0, + "grad_norm": 3.3314558825109595, + "language_loss": 0.80815697, + "learning_rate": 3.950759972780093e-06, + "loss": 0.82979375, + "num_input_tokens_seen": 95498375, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.21582031, + "step": 3400, + "time_per_iteration": 2.444890022277832 + }, + { + "auxiliary_loss_clip": 0.01110451, + "auxiliary_loss_mlp": 0.01044611, + "balance_loss_clip": 1.03800905, + "balance_loss_mlp": 1.02308202, + "epoch": 0.09868841042307469, + "flos": 27889597693440.0, + "grad_norm": 2.5294493862644236, + "language_loss": 0.9189837, + "learning_rate": 3.9507185127899535e-06, + "loss": 0.94053435, + "num_input_tokens_seen": 95513710, + "router_z_loss_clip": 0.72460938, + "router_z_loss_mlp": 0.21533203, + "step": 3401, + "time_per_iteration": 2.4648780822753906 + }, + { + "auxiliary_loss_clip": 0.01025102, + "auxiliary_loss_mlp": 0.0100102, + "balance_loss_clip": 1.01355791, + "balance_loss_mlp": 0.9998998, + "epoch": 0.09871742789159074, + "flos": 62580031056000.0, + "grad_norm": 0.7283220908054872, + "language_loss": 0.50429702, + "learning_rate": 3.950677035570279e-06, + "loss": 0.52455825, + "num_input_tokens_seen": 95570645, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01123047, + "step": 3402, + "time_per_iteration": 2.93097186088562 + }, + { + "auxiliary_loss_clip": 0.01017553, + "auxiliary_loss_mlp": 0.0100142, + "balance_loss_clip": 1.00622129, + "balance_loss_mlp": 1.00034142, + "epoch": 0.09874644536010678, + "flos": 71159517406080.0, + "grad_norm": 0.7228718521975022, + "language_loss": 0.54185164, + "learning_rate": 3.950635541121436e-06, + "loss": 0.56204128, + "num_input_tokens_seen": 95632775, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01080322, + "step": 3403, + "time_per_iteration": 2.9983701705932617 + }, + { + "auxiliary_loss_clip": 0.01103364, + "auxiliary_loss_mlp": 0.01037125, + "balance_loss_clip": 1.0326755, + "balance_loss_mlp": 1.01734841, + "epoch": 0.09877546282862283, + "flos": 26206330446720.0, + "grad_norm": 2.629347495618548, + "language_loss": 0.86345708, + "learning_rate": 3.95059402944379e-06, + "loss": 0.88486195, + "num_input_tokens_seen": 95647375, + "router_z_loss_clip": 0.70703125, + "router_z_loss_mlp": 0.19787598, + "step": 3404, + "time_per_iteration": 2.4264371395111084 + }, + { + "auxiliary_loss_clip": 0.01123746, + "auxiliary_loss_mlp": 0.01056972, + "balance_loss_clip": 1.0374372, + "balance_loss_mlp": 1.0299474, + "epoch": 0.09880448029713888, + "flos": 16318508607360.0, + "grad_norm": 2.2178000953533763, + "language_loss": 0.83333755, + "learning_rate": 3.950552500537708e-06, + "loss": 0.85514474, + "num_input_tokens_seen": 95662935, + "router_z_loss_clip": 0.86328125, + "router_z_loss_mlp": 0.27050781, + "step": 3405, + "time_per_iteration": 2.3931021690368652 + }, + { + "auxiliary_loss_clip": 0.01104676, + "auxiliary_loss_mlp": 0.01041599, + "balance_loss_clip": 1.03319883, + "balance_loss_mlp": 1.02014756, + "epoch": 0.09883349776565492, + "flos": 46528295080320.0, + "grad_norm": 2.3906134479631795, + "language_loss": 0.76583892, + "learning_rate": 3.950510954403557e-06, + "loss": 0.78730166, + "num_input_tokens_seen": 95680995, + "router_z_loss_clip": 0.71459961, + "router_z_loss_mlp": 0.21466064, + "step": 3406, + "time_per_iteration": 2.6131808757781982 + }, + { + "auxiliary_loss_clip": 0.0101902, + "auxiliary_loss_mlp": 0.0100974, + "balance_loss_clip": 1.00730848, + "balance_loss_mlp": 1.00866735, + "epoch": 0.09886251523417097, + "flos": 61413827178240.0, + "grad_norm": 0.7108251745237796, + "language_loss": 0.43866149, + "learning_rate": 3.950469391041705e-06, + "loss": 0.45894909, + "num_input_tokens_seen": 95740850, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01074219, + "step": 3407, + "time_per_iteration": 2.9966721534729004 + }, + { + "auxiliary_loss_clip": 0.01106319, + "auxiliary_loss_mlp": 0.01042975, + "balance_loss_clip": 1.03266728, + "balance_loss_mlp": 1.02027726, + "epoch": 0.09889153270268702, + "flos": 11170321056000.0, + "grad_norm": 4.113448768669767, + "language_loss": 0.80891013, + "learning_rate": 3.9504278104525165e-06, + "loss": 0.83040309, + "num_input_tokens_seen": 95748675, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.22711182, + "step": 3408, + "time_per_iteration": 2.380214214324951 + }, + { + "auxiliary_loss_clip": 0.01104631, + "auxiliary_loss_mlp": 0.01039557, + "balance_loss_clip": 1.03604245, + "balance_loss_mlp": 1.02043581, + "epoch": 0.09892055017120306, + "flos": 28577386944000.0, + "grad_norm": 3.1305934687572683, + "language_loss": 0.96421778, + "learning_rate": 3.950386212636361e-06, + "loss": 0.98565978, + "num_input_tokens_seen": 95764180, + "router_z_loss_clip": 0.68652344, + "router_z_loss_mlp": 0.19128418, + "step": 3409, + "time_per_iteration": 2.42758846282959 + }, + { + "auxiliary_loss_clip": 0.0102285, + "auxiliary_loss_mlp": 0.01005386, + "balance_loss_clip": 1.01072526, + "balance_loss_mlp": 1.00412226, + "epoch": 0.09894956763971911, + "flos": 74782453991040.0, + "grad_norm": 0.6626418698256041, + "language_loss": 0.49075121, + "learning_rate": 3.950344597593606e-06, + "loss": 0.51103359, + "num_input_tokens_seen": 95833580, + "router_z_loss_clip": 0.12109375, + "router_z_loss_mlp": 0.01263428, + "step": 3410, + "time_per_iteration": 3.1570627689361572 + }, + { + "auxiliary_loss_clip": 0.01111105, + "auxiliary_loss_mlp": 0.01047562, + "balance_loss_clip": 1.03867602, + "balance_loss_mlp": 1.02550828, + "epoch": 0.09897858510823515, + "flos": 15589487174400.0, + "grad_norm": 3.5422413348520134, + "language_loss": 0.98220956, + "learning_rate": 3.9503029653246175e-06, + "loss": 1.0037961, + "num_input_tokens_seen": 95846295, + "router_z_loss_clip": 0.72412109, + "router_z_loss_mlp": 0.22058105, + "step": 3411, + "time_per_iteration": 2.3386356830596924 + }, + { + "auxiliary_loss_clip": 0.01104188, + "auxiliary_loss_mlp": 0.01042357, + "balance_loss_clip": 1.03762436, + "balance_loss_mlp": 1.02029133, + "epoch": 0.0990076025767512, + "flos": 32664599055360.0, + "grad_norm": 1.9235396345021742, + "language_loss": 0.79377425, + "learning_rate": 3.950261315829764e-06, + "loss": 0.81523979, + "num_input_tokens_seen": 95864395, + "router_z_loss_clip": 0.66601562, + "router_z_loss_mlp": 0.22058105, + "step": 3412, + "time_per_iteration": 2.4824059009552 + }, + { + "auxiliary_loss_clip": 0.01116209, + "auxiliary_loss_mlp": 0.01045743, + "balance_loss_clip": 1.03960991, + "balance_loss_mlp": 1.02360594, + "epoch": 0.09903662004526725, + "flos": 21209979864960.0, + "grad_norm": 2.6578125143662086, + "language_loss": 0.83972001, + "learning_rate": 3.950219649109414e-06, + "loss": 0.86133957, + "num_input_tokens_seen": 95881070, + "router_z_loss_clip": 0.76611328, + "router_z_loss_mlp": 0.22131348, + "step": 3413, + "time_per_iteration": 2.406534433364868 + }, + { + "auxiliary_loss_clip": 0.01106445, + "auxiliary_loss_mlp": 0.01045431, + "balance_loss_clip": 1.0396409, + "balance_loss_mlp": 1.02706718, + "epoch": 0.09906563751378329, + "flos": 11429620790400.0, + "grad_norm": 3.785387626003143, + "language_loss": 0.8924855, + "learning_rate": 3.950177965163934e-06, + "loss": 0.91400427, + "num_input_tokens_seen": 95891380, + "router_z_loss_clip": 0.66748047, + "router_z_loss_mlp": 0.18365479, + "step": 3414, + "time_per_iteration": 2.377272844314575 + }, + { + "auxiliary_loss_clip": 0.01116733, + "auxiliary_loss_mlp": 0.01044592, + "balance_loss_clip": 1.0403161, + "balance_loss_mlp": 1.02224016, + "epoch": 0.09909465498229934, + "flos": 17303583018240.0, + "grad_norm": 2.657721706052334, + "language_loss": 0.82203853, + "learning_rate": 3.950136263993694e-06, + "loss": 0.84365177, + "num_input_tokens_seen": 95905645, + "router_z_loss_clip": 0.76318359, + "router_z_loss_mlp": 0.22363281, + "step": 3415, + "time_per_iteration": 2.3406474590301514 + }, + { + "auxiliary_loss_clip": 0.0111601, + "auxiliary_loss_mlp": 0.01052138, + "balance_loss_clip": 1.04165208, + "balance_loss_mlp": 1.029953, + "epoch": 0.0991236724508154, + "flos": 24862069301760.0, + "grad_norm": 1.8377615205706181, + "language_loss": 1.01840103, + "learning_rate": 3.95009454559906e-06, + "loss": 1.04008245, + "num_input_tokens_seen": 95925500, + "router_z_loss_clip": 0.74316406, + "router_z_loss_mlp": 0.22180176, + "step": 3416, + "time_per_iteration": 2.431274175643921 + }, + { + "auxiliary_loss_clip": 0.01111263, + "auxiliary_loss_mlp": 0.0104657, + "balance_loss_clip": 1.04073119, + "balance_loss_mlp": 1.02432549, + "epoch": 0.09915268991933143, + "flos": 22521247908480.0, + "grad_norm": 2.7864236089251415, + "language_loss": 0.80123246, + "learning_rate": 3.950052809980403e-06, + "loss": 0.82281077, + "num_input_tokens_seen": 95939250, + "router_z_loss_clip": 0.70556641, + "router_z_loss_mlp": 0.22241211, + "step": 3417, + "time_per_iteration": 2.3832740783691406 + }, + { + "auxiliary_loss_clip": 0.01027174, + "auxiliary_loss_mlp": 0.01001682, + "balance_loss_clip": 1.01427221, + "balance_loss_mlp": 1.00047231, + "epoch": 0.09918170738784748, + "flos": 65336808787200.0, + "grad_norm": 0.7713807535556131, + "language_loss": 0.52240127, + "learning_rate": 3.95001105713809e-06, + "loss": 0.5426898, + "num_input_tokens_seen": 95985410, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01208496, + "step": 3418, + "time_per_iteration": 2.77740740776062 + }, + { + "auxiliary_loss_clip": 0.01114585, + "auxiliary_loss_mlp": 0.01040963, + "balance_loss_clip": 1.04107487, + "balance_loss_mlp": 1.02063775, + "epoch": 0.09921072485636354, + "flos": 27883174003200.0, + "grad_norm": 2.242161141050669, + "language_loss": 0.77360469, + "learning_rate": 3.949969287072491e-06, + "loss": 0.79516017, + "num_input_tokens_seen": 95999710, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.20324707, + "step": 3419, + "time_per_iteration": 2.4708611965179443 + }, + { + "auxiliary_loss_clip": 0.01113609, + "auxiliary_loss_mlp": 0.01053201, + "balance_loss_clip": 1.03682756, + "balance_loss_mlp": 1.02859664, + "epoch": 0.09923974232487957, + "flos": 43900627023360.0, + "grad_norm": 2.391534353465719, + "language_loss": 0.71535987, + "learning_rate": 3.949927499783973e-06, + "loss": 0.73702794, + "num_input_tokens_seen": 96017785, + "router_z_loss_clip": 0.76806641, + "router_z_loss_mlp": 0.24597168, + "step": 3420, + "time_per_iteration": 2.530086040496826 + }, + { + "auxiliary_loss_clip": 0.01102143, + "auxiliary_loss_mlp": 0.010383, + "balance_loss_clip": 1.03364301, + "balance_loss_mlp": 1.01698577, + "epoch": 0.09926875979339563, + "flos": 16100301409920.0, + "grad_norm": 3.1835907610093774, + "language_loss": 0.96064496, + "learning_rate": 3.949885695272908e-06, + "loss": 0.98204935, + "num_input_tokens_seen": 96030455, + "router_z_loss_clip": 0.68505859, + "router_z_loss_mlp": 0.21313477, + "step": 3421, + "time_per_iteration": 2.344426393508911 + }, + { + "auxiliary_loss_clip": 0.01021824, + "auxiliary_loss_mlp": 0.01002008, + "balance_loss_clip": 1.00925636, + "balance_loss_mlp": 1.00082803, + "epoch": 0.09929777726191168, + "flos": 74767267549440.0, + "grad_norm": 0.6882214322635143, + "language_loss": 0.4803105, + "learning_rate": 3.949843873539662e-06, + "loss": 0.50054884, + "num_input_tokens_seen": 96091435, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.01177979, + "step": 3422, + "time_per_iteration": 3.012939214706421 + }, + { + "auxiliary_loss_clip": 0.01108411, + "auxiliary_loss_mlp": 0.01047983, + "balance_loss_clip": 1.03461564, + "balance_loss_mlp": 1.0254643, + "epoch": 0.09932679473042771, + "flos": 11830912490880.0, + "grad_norm": 6.402275689996039, + "language_loss": 0.89339185, + "learning_rate": 3.949802034584606e-06, + "loss": 0.91495574, + "num_input_tokens_seen": 96105620, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.22509766, + "step": 3423, + "time_per_iteration": 2.4081480503082275 + }, + { + "auxiliary_loss_clip": 0.01107373, + "auxiliary_loss_mlp": 0.01047616, + "balance_loss_clip": 1.0346725, + "balance_loss_mlp": 1.02527583, + "epoch": 0.09935581219894377, + "flos": 13218395765760.0, + "grad_norm": 3.7337984336937295, + "language_loss": 0.87918651, + "learning_rate": 3.94976017840811e-06, + "loss": 0.90073639, + "num_input_tokens_seen": 96117765, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.22351074, + "step": 3424, + "time_per_iteration": 4.771291494369507 + }, + { + "auxiliary_loss_clip": 0.01016552, + "auxiliary_loss_mlp": 0.01001206, + "balance_loss_clip": 1.00458503, + "balance_loss_mlp": 1.00001943, + "epoch": 0.09938482966745982, + "flos": 62624230704000.0, + "grad_norm": 0.6341711604210452, + "language_loss": 0.48960552, + "learning_rate": 3.9497183050105425e-06, + "loss": 0.50978309, + "num_input_tokens_seen": 96181040, + "router_z_loss_clip": 0.12011719, + "router_z_loss_mlp": 0.01184082, + "step": 3425, + "time_per_iteration": 3.039429187774658 + }, + { + "auxiliary_loss_clip": 0.01107512, + "auxiliary_loss_mlp": 0.01044495, + "balance_loss_clip": 1.0341531, + "balance_loss_mlp": 1.02223837, + "epoch": 0.09941384713597586, + "flos": 23543539695360.0, + "grad_norm": 1.859310501444483, + "language_loss": 0.7907325, + "learning_rate": 3.949676414392274e-06, + "loss": 0.81225258, + "num_input_tokens_seen": 96199450, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.22253418, + "step": 3426, + "time_per_iteration": 2.5858869552612305 + }, + { + "auxiliary_loss_clip": 0.01014907, + "auxiliary_loss_mlp": 0.01001692, + "balance_loss_clip": 1.00298977, + "balance_loss_mlp": 1.00064898, + "epoch": 0.0994428646044919, + "flos": 58678382154240.0, + "grad_norm": 0.6710689712050676, + "language_loss": 0.50795346, + "learning_rate": 3.949634506553675e-06, + "loss": 0.52811944, + "num_input_tokens_seen": 96261305, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01043701, + "step": 3427, + "time_per_iteration": 5.129628658294678 + }, + { + "auxiliary_loss_clip": 0.01014513, + "auxiliary_loss_mlp": 0.01001306, + "balance_loss_clip": 1.00277913, + "balance_loss_mlp": 1.00005448, + "epoch": 0.09947188207300794, + "flos": 74769781167360.0, + "grad_norm": 0.6626802624012884, + "language_loss": 0.49880621, + "learning_rate": 3.949592581495115e-06, + "loss": 0.51896441, + "num_input_tokens_seen": 96325650, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01251221, + "step": 3428, + "time_per_iteration": 3.1672677993774414 + }, + { + "auxiliary_loss_clip": 0.0110669, + "auxiliary_loss_mlp": 0.01039939, + "balance_loss_clip": 1.03285873, + "balance_loss_mlp": 1.01734924, + "epoch": 0.099500899541524, + "flos": 32532172801920.0, + "grad_norm": 2.384188942587375, + "language_loss": 0.75478333, + "learning_rate": 3.949550639216964e-06, + "loss": 0.77624959, + "num_input_tokens_seen": 96342530, + "router_z_loss_clip": 0.73876953, + "router_z_loss_mlp": 0.22583008, + "step": 3429, + "time_per_iteration": 2.545584201812744 + }, + { + "auxiliary_loss_clip": 0.01014904, + "auxiliary_loss_mlp": 0.01003271, + "balance_loss_clip": 1.0030272, + "balance_loss_mlp": 1.00201952, + "epoch": 0.09952991701004005, + "flos": 63229695235200.0, + "grad_norm": 0.7070356581012839, + "language_loss": 0.47434103, + "learning_rate": 3.949508679719593e-06, + "loss": 0.49452281, + "num_input_tokens_seen": 96391330, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01251221, + "step": 3430, + "time_per_iteration": 2.816681385040283 + }, + { + "auxiliary_loss_clip": 0.01107148, + "auxiliary_loss_mlp": 0.01046721, + "balance_loss_clip": 1.03620982, + "balance_loss_mlp": 1.0266943, + "epoch": 0.09955893447855609, + "flos": 54809763131520.0, + "grad_norm": 2.4713106210538376, + "language_loss": 1.20969033, + "learning_rate": 3.949466703003373e-06, + "loss": 1.23122907, + "num_input_tokens_seen": 96413550, + "router_z_loss_clip": 0.70947266, + "router_z_loss_mlp": 0.20031738, + "step": 3431, + "time_per_iteration": 2.599336624145508 + }, + { + "auxiliary_loss_clip": 0.01105861, + "auxiliary_loss_mlp": 0.01039421, + "balance_loss_clip": 1.03490806, + "balance_loss_mlp": 1.01896477, + "epoch": 0.09958795194707214, + "flos": 74729353812480.0, + "grad_norm": 1.957392843910239, + "language_loss": 0.77058536, + "learning_rate": 3.949424709068674e-06, + "loss": 0.79203814, + "num_input_tokens_seen": 96438410, + "router_z_loss_clip": 0.70996094, + "router_z_loss_mlp": 0.20458984, + "step": 3432, + "time_per_iteration": 5.236513614654541 + }, + { + "auxiliary_loss_clip": 0.01109586, + "auxiliary_loss_mlp": 0.01046325, + "balance_loss_clip": 1.03482997, + "balance_loss_mlp": 1.02316213, + "epoch": 0.09961696941558819, + "flos": 17559147237120.0, + "grad_norm": 2.426010182631245, + "language_loss": 0.82623458, + "learning_rate": 3.949382697915866e-06, + "loss": 0.8477937, + "num_input_tokens_seen": 96452550, + "router_z_loss_clip": 0.74658203, + "router_z_loss_mlp": 0.23144531, + "step": 3433, + "time_per_iteration": 4.728010416030884 + }, + { + "auxiliary_loss_clip": 0.01021513, + "auxiliary_loss_mlp": 0.0100098, + "balance_loss_clip": 1.0093298, + "balance_loss_mlp": 0.9997167, + "epoch": 0.09964598688410423, + "flos": 63240413022720.0, + "grad_norm": 0.6994919216721416, + "language_loss": 0.51340711, + "learning_rate": 3.949340669545323e-06, + "loss": 0.53363204, + "num_input_tokens_seen": 96515705, + "router_z_loss_clip": 0.12207031, + "router_z_loss_mlp": 0.01263428, + "step": 3434, + "time_per_iteration": 3.166301727294922 + }, + { + "auxiliary_loss_clip": 0.01104581, + "auxiliary_loss_mlp": 0.01039403, + "balance_loss_clip": 1.035393, + "balance_loss_mlp": 1.01997137, + "epoch": 0.09967500435262028, + "flos": 31205264469120.0, + "grad_norm": 2.6644869806754885, + "language_loss": 0.78456444, + "learning_rate": 3.9492986239574134e-06, + "loss": 0.80600435, + "num_input_tokens_seen": 96535760, + "router_z_loss_clip": 0.69091797, + "router_z_loss_mlp": 0.19445801, + "step": 3435, + "time_per_iteration": 2.533510208129883 + }, + { + "auxiliary_loss_clip": 0.01019474, + "auxiliary_loss_mlp": 0.0100196, + "balance_loss_clip": 1.00809216, + "balance_loss_mlp": 1.00064909, + "epoch": 0.09970402182113633, + "flos": 71349619092480.0, + "grad_norm": 0.6455324132490485, + "language_loss": 0.46363026, + "learning_rate": 3.949256561152509e-06, + "loss": 0.48384464, + "num_input_tokens_seen": 96602780, + "router_z_loss_clip": 0.11376953, + "router_z_loss_mlp": 0.01312256, + "step": 3436, + "time_per_iteration": 3.095123291015625 + }, + { + "auxiliary_loss_clip": 0.01018254, + "auxiliary_loss_mlp": 0.01000654, + "balance_loss_clip": 1.00651026, + "balance_loss_mlp": 0.99935442, + "epoch": 0.09973303928965237, + "flos": 74767372283520.0, + "grad_norm": 0.6602022226652098, + "language_loss": 0.52196199, + "learning_rate": 3.949214481130983e-06, + "loss": 0.54215103, + "num_input_tokens_seen": 96668775, + "router_z_loss_clip": 0.11767578, + "router_z_loss_mlp": 0.01300049, + "step": 3437, + "time_per_iteration": 3.1537578105926514 + }, + { + "auxiliary_loss_clip": 0.01018083, + "auxiliary_loss_mlp": 0.01000594, + "balance_loss_clip": 1.00620723, + "balance_loss_mlp": 0.99941969, + "epoch": 0.09976205675816842, + "flos": 58344019263360.0, + "grad_norm": 0.7684959122845456, + "language_loss": 0.52351284, + "learning_rate": 3.949172383893205e-06, + "loss": 0.54369962, + "num_input_tokens_seen": 96721320, + "router_z_loss_clip": 0.11865234, + "router_z_loss_mlp": 0.01171875, + "step": 3438, + "time_per_iteration": 2.849104404449463 + }, + { + "auxiliary_loss_clip": 0.0101636, + "auxiliary_loss_mlp": 0.0099977, + "balance_loss_clip": 1.00467968, + "balance_loss_mlp": 0.99858969, + "epoch": 0.09979107422668447, + "flos": 54665430238080.0, + "grad_norm": 0.634184783555992, + "language_loss": 0.49551305, + "learning_rate": 3.949130269439549e-06, + "loss": 0.51567435, + "num_input_tokens_seen": 96783485, + "router_z_loss_clip": 0.11669922, + "router_z_loss_mlp": 0.01177979, + "step": 3439, + "time_per_iteration": 3.0590500831604004 + }, + { + "auxiliary_loss_clip": 0.0101848, + "auxiliary_loss_mlp": 0.01000859, + "balance_loss_clip": 1.00679898, + "balance_loss_mlp": 0.99964958, + "epoch": 0.09982009169520051, + "flos": 61121813633280.0, + "grad_norm": 0.6752752758247051, + "language_loss": 0.49960625, + "learning_rate": 3.949088137770385e-06, + "loss": 0.51979965, + "num_input_tokens_seen": 96839995, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01208496, + "step": 3440, + "time_per_iteration": 2.903383255004883 + }, + { + "auxiliary_loss_clip": 0.01021536, + "auxiliary_loss_mlp": 0.01005599, + "balance_loss_clip": 1.00975657, + "balance_loss_mlp": 1.00424647, + "epoch": 0.09984910916371656, + "flos": 74765347424640.0, + "grad_norm": 0.6864508225474046, + "language_loss": 0.47018364, + "learning_rate": 3.949045988886086e-06, + "loss": 0.49045497, + "num_input_tokens_seen": 96894965, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.0135498, + "step": 3441, + "time_per_iteration": 2.967292070388794 + }, + { + "auxiliary_loss_clip": 0.01019786, + "auxiliary_loss_mlp": 0.01020328, + "balance_loss_clip": 1.00814855, + "balance_loss_mlp": 1.01900458, + "epoch": 0.0998781266322326, + "flos": 72177794386560.0, + "grad_norm": 0.6627432093734029, + "language_loss": 0.45811284, + "learning_rate": 3.9490038227870235e-06, + "loss": 0.47851396, + "num_input_tokens_seen": 96953490, + "router_z_loss_clip": 0.11621094, + "router_z_loss_mlp": 0.01324463, + "step": 3442, + "time_per_iteration": 3.061732292175293 + }, + { + "auxiliary_loss_clip": 0.01016069, + "auxiliary_loss_mlp": 0.01019918, + "balance_loss_clip": 1.00470138, + "balance_loss_mlp": 1.01876199, + "epoch": 0.09990714410074865, + "flos": 64517047130880.0, + "grad_norm": 0.6664165466659318, + "language_loss": 0.47258404, + "learning_rate": 3.94896163947357e-06, + "loss": 0.49294391, + "num_input_tokens_seen": 97019635, + "router_z_loss_clip": 0.11376953, + "router_z_loss_mlp": 0.01153564, + "step": 3443, + "time_per_iteration": 3.040410041809082 + }, + { + "auxiliary_loss_clip": 0.01111806, + "auxiliary_loss_mlp": 0.01050011, + "balance_loss_clip": 1.03511119, + "balance_loss_mlp": 1.0254184, + "epoch": 0.0999361615692647, + "flos": 28542368983680.0, + "grad_norm": 2.599423119527375, + "language_loss": 1.13495159, + "learning_rate": 3.948919438946101e-06, + "loss": 1.15656972, + "num_input_tokens_seen": 97041470, + "router_z_loss_clip": 0.76660156, + "router_z_loss_mlp": 0.24584961, + "step": 3444, + "time_per_iteration": 2.5795788764953613 + }, + { + "auxiliary_loss_clip": 0.0109798, + "auxiliary_loss_mlp": 0.01038842, + "balance_loss_clip": 1.03136539, + "balance_loss_mlp": 1.02060306, + "epoch": 0.09996517903778074, + "flos": 11647862899200.0, + "grad_norm": 4.8435000174026674, + "language_loss": 0.7972945, + "learning_rate": 3.948877221204984e-06, + "loss": 0.8186627, + "num_input_tokens_seen": 97052950, + "router_z_loss_clip": 0.66601562, + "router_z_loss_mlp": 0.18243408, + "step": 3445, + "time_per_iteration": 2.5039334297180176 + }, + { + "auxiliary_loss_clip": 0.01114364, + "auxiliary_loss_mlp": 0.01057016, + "balance_loss_clip": 1.03677154, + "balance_loss_mlp": 1.03443754, + "epoch": 0.09999419650629679, + "flos": 20186780382720.0, + "grad_norm": 2.3139374502239805, + "language_loss": 0.92776966, + "learning_rate": 3.948834986250597e-06, + "loss": 0.94948351, + "num_input_tokens_seen": 97070740, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.22583008, + "step": 3446, + "time_per_iteration": 2.529811382293701 + }, + { + "auxiliary_loss_clip": 0.01098736, + "auxiliary_loss_mlp": 0.01034267, + "balance_loss_clip": 1.03399825, + "balance_loss_mlp": 1.01661801, + "epoch": 0.10002321397481284, + "flos": 74731937253120.0, + "grad_norm": 2.9195839297165116, + "language_loss": 0.83165395, + "learning_rate": 3.94879273408331e-06, + "loss": 0.85298395, + "num_input_tokens_seen": 97094640, + "router_z_loss_clip": 0.64697266, + "router_z_loss_mlp": 0.1762085, + "step": 3447, + "time_per_iteration": 2.8047640323638916 + }, + { + "auxiliary_loss_clip": 0.01099811, + "auxiliary_loss_mlp": 0.01036708, + "balance_loss_clip": 1.03647041, + "balance_loss_mlp": 1.01998329, + "epoch": 0.10005223144332888, + "flos": 29053183219200.0, + "grad_norm": 2.0107788915011326, + "language_loss": 0.67787486, + "learning_rate": 3.948750464703497e-06, + "loss": 0.69924009, + "num_input_tokens_seen": 97112150, + "router_z_loss_clip": 0.63330078, + "router_z_loss_mlp": 0.16711426, + "step": 3448, + "time_per_iteration": 2.4882938861846924 + }, + { + "auxiliary_loss_clip": 0.01119174, + "auxiliary_loss_mlp": 0.01043259, + "balance_loss_clip": 1.04155707, + "balance_loss_mlp": 1.02057302, + "epoch": 0.10008124891184493, + "flos": 14748918347520.0, + "grad_norm": 2.579434313716164, + "language_loss": 0.8429392, + "learning_rate": 3.948708178111531e-06, + "loss": 0.86456358, + "num_input_tokens_seen": 97124200, + "router_z_loss_clip": 0.77734375, + "router_z_loss_mlp": 0.22692871, + "step": 3449, + "time_per_iteration": 2.415786027908325 + }, + { + "auxiliary_loss_clip": 0.0111385, + "auxiliary_loss_mlp": 0.01044494, + "balance_loss_clip": 1.04048371, + "balance_loss_mlp": 1.02314377, + "epoch": 0.10011026638036098, + "flos": 14931130066560.0, + "grad_norm": 4.1493607544051105, + "language_loss": 0.98683363, + "learning_rate": 3.948665874307787e-06, + "loss": 1.00841713, + "num_input_tokens_seen": 97134175, + "router_z_loss_clip": 0.73388672, + "router_z_loss_mlp": 0.21356201, + "step": 3450, + "time_per_iteration": 2.4466030597686768 + }, + { + "auxiliary_loss_clip": 0.01119983, + "auxiliary_loss_mlp": 0.01043506, + "balance_loss_clip": 1.04630697, + "balance_loss_mlp": 1.02070093, + "epoch": 0.10013928384887702, + "flos": 25624432949760.0, + "grad_norm": 2.966362366403428, + "language_loss": 0.83970511, + "learning_rate": 3.948623553292636e-06, + "loss": 0.86133999, + "num_input_tokens_seen": 97152730, + "router_z_loss_clip": 0.73583984, + "router_z_loss_mlp": 0.22802734, + "step": 3451, + "time_per_iteration": 2.482478141784668 + }, + { + "auxiliary_loss_clip": 0.01115873, + "auxiliary_loss_mlp": 0.01047423, + "balance_loss_clip": 1.04158068, + "balance_loss_mlp": 1.02530956, + "epoch": 0.10016830131739307, + "flos": 34017762597120.0, + "grad_norm": 1.9864456564259343, + "language_loss": 0.84349233, + "learning_rate": 3.948581215066454e-06, + "loss": 0.86512524, + "num_input_tokens_seen": 97174390, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.22131348, + "step": 3452, + "time_per_iteration": 2.5660932064056396 + }, + { + "auxiliary_loss_clip": 0.01119455, + "auxiliary_loss_mlp": 0.01045149, + "balance_loss_clip": 1.04155064, + "balance_loss_mlp": 1.02074683, + "epoch": 0.10019731878590912, + "flos": 12854461086720.0, + "grad_norm": 2.5029347390240044, + "language_loss": 0.75695479, + "learning_rate": 3.948538859629614e-06, + "loss": 0.77860081, + "num_input_tokens_seen": 97186215, + "router_z_loss_clip": 0.77929688, + "router_z_loss_mlp": 0.24414062, + "step": 3453, + "time_per_iteration": 2.3764712810516357 + }, + { + "auxiliary_loss_clip": 0.01067261, + "auxiliary_loss_mlp": 0.01005322, + "balance_loss_clip": 1.0540545, + "balance_loss_mlp": 1.00435042, + "epoch": 0.10022633625442516, + "flos": 62106853132800.0, + "grad_norm": 0.7594632486226656, + "language_loss": 0.54299998, + "learning_rate": 3.948496486982491e-06, + "loss": 0.56372583, + "num_input_tokens_seen": 97248085, + "router_z_loss_clip": 0.13183594, + "router_z_loss_mlp": 0.00970459, + "step": 3454, + "time_per_iteration": 2.9612882137298584 + }, + { + "auxiliary_loss_clip": 0.01052123, + "auxiliary_loss_mlp": 0.01001501, + "balance_loss_clip": 1.03890824, + "balance_loss_mlp": 1.00045204, + "epoch": 0.10025535372294121, + "flos": 65866126043520.0, + "grad_norm": 0.6930683155160066, + "language_loss": 0.45734906, + "learning_rate": 3.948454097125458e-06, + "loss": 0.47788531, + "num_input_tokens_seen": 97302770, + "router_z_loss_clip": 0.1328125, + "router_z_loss_mlp": 0.01049805, + "step": 3455, + "time_per_iteration": 2.9619250297546387 + }, + { + "auxiliary_loss_clip": 0.01115125, + "auxiliary_loss_mlp": 0.01048026, + "balance_loss_clip": 1.0378859, + "balance_loss_mlp": 1.02482772, + "epoch": 0.10028437119145726, + "flos": 23511035352960.0, + "grad_norm": 2.363679730831352, + "language_loss": 0.87023425, + "learning_rate": 3.948411690058889e-06, + "loss": 0.89186573, + "num_input_tokens_seen": 97317655, + "router_z_loss_clip": 0.77197266, + "router_z_loss_mlp": 0.23205566, + "step": 3456, + "time_per_iteration": 2.4897851943969727 + }, + { + "auxiliary_loss_clip": 0.01110471, + "auxiliary_loss_mlp": 0.01044318, + "balance_loss_clip": 1.03975499, + "balance_loss_mlp": 1.0233134, + "epoch": 0.1003133886599733, + "flos": 24345704160000.0, + "grad_norm": 2.1250950895487537, + "language_loss": 0.63011885, + "learning_rate": 3.948369265783161e-06, + "loss": 0.6516667, + "num_input_tokens_seen": 97339495, + "router_z_loss_clip": 0.70751953, + "router_z_loss_mlp": 0.2097168, + "step": 3457, + "time_per_iteration": 2.6511435508728027 + }, + { + "auxiliary_loss_clip": 0.01107011, + "auxiliary_loss_mlp": 0.01042184, + "balance_loss_clip": 1.03397632, + "balance_loss_mlp": 1.0206666, + "epoch": 0.10034240612848935, + "flos": 21098292825600.0, + "grad_norm": 2.234023159164607, + "language_loss": 0.93319148, + "learning_rate": 3.948326824298646e-06, + "loss": 0.95468342, + "num_input_tokens_seen": 97355320, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.21533203, + "step": 3458, + "time_per_iteration": 2.454486131668091 + }, + { + "auxiliary_loss_clip": 0.01110935, + "auxiliary_loss_mlp": 0.01034676, + "balance_loss_clip": 1.03768611, + "balance_loss_mlp": 1.01426697, + "epoch": 0.10037142359700539, + "flos": 14420280919680.0, + "grad_norm": 2.9820099406621106, + "language_loss": 0.86095577, + "learning_rate": 3.948284365605721e-06, + "loss": 0.88241196, + "num_input_tokens_seen": 97368000, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.20422363, + "step": 3459, + "time_per_iteration": 2.34749436378479 + }, + { + "auxiliary_loss_clip": 0.01111348, + "auxiliary_loss_mlp": 0.01048614, + "balance_loss_clip": 1.03839052, + "balance_loss_mlp": 1.0252018, + "epoch": 0.10040044106552144, + "flos": 23687661254400.0, + "grad_norm": 2.9404047743925643, + "language_loss": 0.82254684, + "learning_rate": 3.94824188970476e-06, + "loss": 0.84414649, + "num_input_tokens_seen": 97382320, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.23425293, + "step": 3460, + "time_per_iteration": 2.5055723190307617 + }, + { + "auxiliary_loss_clip": 0.01112559, + "auxiliary_loss_mlp": 0.01042777, + "balance_loss_clip": 1.04065514, + "balance_loss_mlp": 1.02149189, + "epoch": 0.1004294585340375, + "flos": 67032506344320.0, + "grad_norm": 2.777552912071321, + "language_loss": 0.8580386, + "learning_rate": 3.948199396596138e-06, + "loss": 0.87959194, + "num_input_tokens_seen": 97406935, + "router_z_loss_clip": 0.7199707, + "router_z_loss_mlp": 0.21307373, + "step": 3461, + "time_per_iteration": 2.712371349334717 + }, + { + "auxiliary_loss_clip": 0.01033964, + "auxiliary_loss_mlp": 0.01008128, + "balance_loss_clip": 1.02204263, + "balance_loss_mlp": 1.00704968, + "epoch": 0.10045847600255353, + "flos": 74774145087360.0, + "grad_norm": 0.6778053124575817, + "language_loss": 0.54160094, + "learning_rate": 3.94815688628023e-06, + "loss": 0.56202185, + "num_input_tokens_seen": 97468345, + "router_z_loss_clip": 0.11914062, + "router_z_loss_mlp": 0.01080322, + "step": 3462, + "time_per_iteration": 3.0826807022094727 + }, + { + "auxiliary_loss_clip": 0.01035855, + "auxiliary_loss_mlp": 0.01003071, + "balance_loss_clip": 1.02379084, + "balance_loss_mlp": 1.00198579, + "epoch": 0.10048749347106958, + "flos": 65324832324480.0, + "grad_norm": 0.7055706986685782, + "language_loss": 0.50045764, + "learning_rate": 3.948114358757414e-06, + "loss": 0.5208469, + "num_input_tokens_seen": 97526525, + "router_z_loss_clip": 0.12060547, + "router_z_loss_mlp": 0.01086426, + "step": 3463, + "time_per_iteration": 2.975640058517456 + }, + { + "auxiliary_loss_clip": 0.01036739, + "auxiliary_loss_mlp": 0.01001156, + "balance_loss_clip": 1.02465391, + "balance_loss_mlp": 0.99999964, + "epoch": 0.10051651093958563, + "flos": 74772085317120.0, + "grad_norm": 0.654900361275301, + "language_loss": 0.45884621, + "learning_rate": 3.948071814028061e-06, + "loss": 0.47922516, + "num_input_tokens_seen": 97589950, + "router_z_loss_clip": 0.12109375, + "router_z_loss_mlp": 0.01153564, + "step": 3464, + "time_per_iteration": 3.090657949447632 + }, + { + "auxiliary_loss_clip": 0.01119838, + "auxiliary_loss_mlp": 0.010483, + "balance_loss_clip": 1.04196274, + "balance_loss_mlp": 1.02544141, + "epoch": 0.10054552840810167, + "flos": 42733096513920.0, + "grad_norm": 4.667917515134305, + "language_loss": 0.8119067, + "learning_rate": 3.948029252092551e-06, + "loss": 0.83358812, + "num_input_tokens_seen": 97607580, + "router_z_loss_clip": 0.77954102, + "router_z_loss_mlp": 0.2286377, + "step": 3465, + "time_per_iteration": 2.5881779193878174 + }, + { + "auxiliary_loss_clip": 0.01107256, + "auxiliary_loss_mlp": 0.01039384, + "balance_loss_clip": 1.0403614, + "balance_loss_mlp": 1.01989365, + "epoch": 0.10057454587661772, + "flos": 15625866677760.0, + "grad_norm": 2.4937287530131993, + "language_loss": 0.67929637, + "learning_rate": 3.947986672951258e-06, + "loss": 0.70076275, + "num_input_tokens_seen": 97621440, + "router_z_loss_clip": 0.66845703, + "router_z_loss_mlp": 0.19506836, + "step": 3466, + "time_per_iteration": 2.4263713359832764 + }, + { + "auxiliary_loss_clip": 0.01028604, + "auxiliary_loss_mlp": 0.01001004, + "balance_loss_clip": 1.01646638, + "balance_loss_mlp": 0.99984217, + "epoch": 0.10060356334513378, + "flos": 63727241287680.0, + "grad_norm": 0.6984864242074945, + "language_loss": 0.55315286, + "learning_rate": 3.947944076604559e-06, + "loss": 0.57344896, + "num_input_tokens_seen": 97684785, + "router_z_loss_clip": 0.12158203, + "router_z_loss_mlp": 0.01159668, + "step": 3467, + "time_per_iteration": 3.0989058017730713 + }, + { + "auxiliary_loss_clip": 0.01113883, + "auxiliary_loss_mlp": 0.01045048, + "balance_loss_clip": 1.0430907, + "balance_loss_mlp": 1.02516448, + "epoch": 0.10063258081364981, + "flos": 16790080608000.0, + "grad_norm": 3.7655102688072537, + "language_loss": 0.84825289, + "learning_rate": 3.947901463052829e-06, + "loss": 0.86984223, + "num_input_tokens_seen": 97696155, + "router_z_loss_clip": 0.70800781, + "router_z_loss_mlp": 0.19897461, + "step": 3468, + "time_per_iteration": 2.3658201694488525 + }, + { + "auxiliary_loss_clip": 0.01121465, + "auxiliary_loss_mlp": 0.01049386, + "balance_loss_clip": 1.04152465, + "balance_loss_mlp": 1.02472138, + "epoch": 0.10066159828216587, + "flos": 19789363843200.0, + "grad_norm": 2.9325054088429257, + "language_loss": 0.77208757, + "learning_rate": 3.947858832296445e-06, + "loss": 0.79379606, + "num_input_tokens_seen": 97711315, + "router_z_loss_clip": 0.79931641, + "router_z_loss_mlp": 0.24682617, + "step": 3469, + "time_per_iteration": 2.4259369373321533 + }, + { + "auxiliary_loss_clip": 0.01119041, + "auxiliary_loss_mlp": 0.01048981, + "balance_loss_clip": 1.0424819, + "balance_loss_mlp": 1.02753484, + "epoch": 0.10069061575068192, + "flos": 19783568557440.0, + "grad_norm": 2.896430652851407, + "language_loss": 0.72589886, + "learning_rate": 3.947816184335784e-06, + "loss": 0.7475791, + "num_input_tokens_seen": 97723990, + "router_z_loss_clip": 0.76464844, + "router_z_loss_mlp": 0.2142334, + "step": 3470, + "time_per_iteration": 2.411127805709839 + }, + { + "auxiliary_loss_clip": 0.01036495, + "auxiliary_loss_mlp": 0.00999407, + "balance_loss_clip": 1.02361572, + "balance_loss_mlp": 0.99812603, + "epoch": 0.10071963321919795, + "flos": 74765312513280.0, + "grad_norm": 0.7493031218589425, + "language_loss": 0.50483835, + "learning_rate": 3.947773519171222e-06, + "loss": 0.52519739, + "num_input_tokens_seen": 97781205, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01281738, + "step": 3471, + "time_per_iteration": 2.996035575866699 + }, + { + "auxiliary_loss_clip": 0.01124223, + "auxiliary_loss_mlp": 0.01052377, + "balance_loss_clip": 1.04465234, + "balance_loss_mlp": 1.02616239, + "epoch": 0.100748650687714, + "flos": 32627975172480.0, + "grad_norm": 1.8786287531552996, + "language_loss": 0.88858044, + "learning_rate": 3.947730836803137e-06, + "loss": 0.91034645, + "num_input_tokens_seen": 97802440, + "router_z_loss_clip": 0.79492188, + "router_z_loss_mlp": 0.26220703, + "step": 3472, + "time_per_iteration": 2.526304244995117 + }, + { + "auxiliary_loss_clip": 0.01112906, + "auxiliary_loss_mlp": 0.01048189, + "balance_loss_clip": 1.03792715, + "balance_loss_mlp": 1.02530122, + "epoch": 0.10077766815623004, + "flos": 24928788643200.0, + "grad_norm": 2.508216587804593, + "language_loss": 0.85853469, + "learning_rate": 3.947688137231904e-06, + "loss": 0.88014567, + "num_input_tokens_seen": 97817815, + "router_z_loss_clip": 0.74951172, + "router_z_loss_mlp": 0.22875977, + "step": 3473, + "time_per_iteration": 2.45745849609375 + }, + { + "auxiliary_loss_clip": 0.01029839, + "auxiliary_loss_mlp": 0.01002437, + "balance_loss_clip": 1.01722193, + "balance_loss_mlp": 1.00119126, + "epoch": 0.1008066856247461, + "flos": 72257535665280.0, + "grad_norm": 0.617603524396709, + "language_loss": 0.51582015, + "learning_rate": 3.947645420457901e-06, + "loss": 0.53614289, + "num_input_tokens_seen": 97889575, + "router_z_loss_clip": 0.12597656, + "router_z_loss_mlp": 0.01245117, + "step": 3474, + "time_per_iteration": 3.157000780105591 + }, + { + "auxiliary_loss_clip": 0.01102974, + "auxiliary_loss_mlp": 0.01050549, + "balance_loss_clip": 1.03573906, + "balance_loss_mlp": 1.03148103, + "epoch": 0.10083570309326215, + "flos": 39377454364800.0, + "grad_norm": 2.3549675414136826, + "language_loss": 0.86929107, + "learning_rate": 3.947602686481507e-06, + "loss": 0.89082634, + "num_input_tokens_seen": 97908785, + "router_z_loss_clip": 0.67285156, + "router_z_loss_mlp": 0.19067383, + "step": 3475, + "time_per_iteration": 2.611593723297119 + }, + { + "auxiliary_loss_clip": 0.01107921, + "auxiliary_loss_mlp": 0.01045067, + "balance_loss_clip": 1.03344476, + "balance_loss_mlp": 1.02354944, + "epoch": 0.10086472056177818, + "flos": 37191856343040.0, + "grad_norm": 3.1765987786035366, + "language_loss": 0.72297496, + "learning_rate": 3.9475599353030965e-06, + "loss": 0.74450481, + "num_input_tokens_seen": 97935000, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.21520996, + "step": 3476, + "time_per_iteration": 2.6013879776000977 + }, + { + "auxiliary_loss_clip": 0.01020822, + "auxiliary_loss_mlp": 0.01003597, + "balance_loss_clip": 1.00848866, + "balance_loss_mlp": 1.00239348, + "epoch": 0.10089373803029424, + "flos": 60254500838400.0, + "grad_norm": 0.696349094330054, + "language_loss": 0.45162839, + "learning_rate": 3.947517166923049e-06, + "loss": 0.4718726, + "num_input_tokens_seen": 98004655, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01202393, + "step": 3477, + "time_per_iteration": 3.221595048904419 + }, + { + "auxiliary_loss_clip": 0.0102027, + "auxiliary_loss_mlp": 0.01004769, + "balance_loss_clip": 1.00807834, + "balance_loss_mlp": 1.00367808, + "epoch": 0.10092275549881029, + "flos": 70573814570880.0, + "grad_norm": 0.6396303333095442, + "language_loss": 0.5017364, + "learning_rate": 3.947474381341741e-06, + "loss": 0.52198672, + "num_input_tokens_seen": 98066750, + "router_z_loss_clip": 0.12207031, + "router_z_loss_mlp": 0.01092529, + "step": 3478, + "time_per_iteration": 3.02439284324646 + }, + { + "auxiliary_loss_clip": 0.01101313, + "auxiliary_loss_mlp": 0.01044692, + "balance_loss_clip": 1.03432393, + "balance_loss_mlp": 1.02498031, + "epoch": 0.10095177296732633, + "flos": 26867899399680.0, + "grad_norm": 3.8546500440994667, + "language_loss": 0.8286317, + "learning_rate": 3.947431578559553e-06, + "loss": 0.85009176, + "num_input_tokens_seen": 98087100, + "router_z_loss_clip": 0.67089844, + "router_z_loss_mlp": 0.19714355, + "step": 3479, + "time_per_iteration": 2.4752724170684814 + }, + { + "auxiliary_loss_clip": 0.01113585, + "auxiliary_loss_mlp": 0.01052698, + "balance_loss_clip": 1.03647602, + "balance_loss_mlp": 1.0306567, + "epoch": 0.10098079043584238, + "flos": 38106231517440.0, + "grad_norm": 2.1742026557321785, + "language_loss": 1.07097316, + "learning_rate": 3.94738875857686e-06, + "loss": 1.09263599, + "num_input_tokens_seen": 98112000, + "router_z_loss_clip": 0.77197266, + "router_z_loss_mlp": 0.22033691, + "step": 3480, + "time_per_iteration": 2.5608718395233154 + }, + { + "auxiliary_loss_clip": 0.0102199, + "auxiliary_loss_mlp": 0.01013215, + "balance_loss_clip": 1.00974822, + "balance_loss_mlp": 1.01217186, + "epoch": 0.10100980790435843, + "flos": 74775262250880.0, + "grad_norm": 0.7040040094017541, + "language_loss": 0.49240351, + "learning_rate": 3.947345921394042e-06, + "loss": 0.51275557, + "num_input_tokens_seen": 98177410, + "router_z_loss_clip": 0.12255859, + "router_z_loss_mlp": 0.01043701, + "step": 3481, + "time_per_iteration": 3.087739944458008 + }, + { + "auxiliary_loss_clip": 0.010239, + "auxiliary_loss_mlp": 0.01018351, + "balance_loss_clip": 1.01140738, + "balance_loss_mlp": 1.01721239, + "epoch": 0.10103882537287447, + "flos": 67620197260800.0, + "grad_norm": 0.690890972565348, + "language_loss": 0.47042385, + "learning_rate": 3.947303067011477e-06, + "loss": 0.49084634, + "num_input_tokens_seen": 98235730, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.01141357, + "step": 3482, + "time_per_iteration": 3.05619478225708 + }, + { + "auxiliary_loss_clip": 0.01116374, + "auxiliary_loss_mlp": 0.01054511, + "balance_loss_clip": 1.0373466, + "balance_loss_mlp": 1.03072894, + "epoch": 0.10106784284139052, + "flos": 24964330273920.0, + "grad_norm": 2.6851953029325766, + "language_loss": 1.08044827, + "learning_rate": 3.947260195429542e-06, + "loss": 1.10215712, + "num_input_tokens_seen": 98250320, + "router_z_loss_clip": 0.79003906, + "router_z_loss_mlp": 0.23815918, + "step": 3483, + "time_per_iteration": 2.428802013397217 + }, + { + "auxiliary_loss_clip": 0.01111497, + "auxiliary_loss_mlp": 0.01047589, + "balance_loss_clip": 1.03631139, + "balance_loss_mlp": 1.02367544, + "epoch": 0.10109686030990657, + "flos": 24089616270720.0, + "grad_norm": 2.952986494874401, + "language_loss": 0.9287855, + "learning_rate": 3.947217306648619e-06, + "loss": 0.95037639, + "num_input_tokens_seen": 98263665, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.23913574, + "step": 3484, + "time_per_iteration": 2.448819398880005 + }, + { + "auxiliary_loss_clip": 0.01031988, + "auxiliary_loss_mlp": 0.01003169, + "balance_loss_clip": 1.01902544, + "balance_loss_mlp": 1.00207782, + "epoch": 0.10112587777842261, + "flos": 74648842617600.0, + "grad_norm": 0.7503654810087006, + "language_loss": 0.51988173, + "learning_rate": 3.947174400669083e-06, + "loss": 0.54023337, + "num_input_tokens_seen": 98318110, + "router_z_loss_clip": 0.12988281, + "router_z_loss_mlp": 0.01092529, + "step": 3485, + "time_per_iteration": 2.8908510208129883 + }, + { + "auxiliary_loss_clip": 0.01104419, + "auxiliary_loss_mlp": 0.01042838, + "balance_loss_clip": 1.04009426, + "balance_loss_mlp": 1.02574944, + "epoch": 0.10115489524693866, + "flos": 26897052251520.0, + "grad_norm": 2.3730898455301337, + "language_loss": 0.86664587, + "learning_rate": 3.947131477491315e-06, + "loss": 0.88811839, + "num_input_tokens_seen": 98332110, + "router_z_loss_clip": 0.64355469, + "router_z_loss_mlp": 0.17077637, + "step": 3486, + "time_per_iteration": 2.417299270629883 + }, + { + "auxiliary_loss_clip": 0.01109119, + "auxiliary_loss_mlp": 0.01041959, + "balance_loss_clip": 1.04069757, + "balance_loss_mlp": 1.0233804, + "epoch": 0.10118391271545471, + "flos": 30844611457920.0, + "grad_norm": 1.7654995654325507, + "language_loss": 0.68392336, + "learning_rate": 3.947088537115695e-06, + "loss": 0.70543414, + "num_input_tokens_seen": 98356955, + "router_z_loss_clip": 0.68481445, + "router_z_loss_mlp": 0.18597412, + "step": 3487, + "time_per_iteration": 2.6481127738952637 + }, + { + "auxiliary_loss_clip": 0.01114609, + "auxiliary_loss_mlp": 0.01051911, + "balance_loss_clip": 1.04012942, + "balance_loss_mlp": 1.02960145, + "epoch": 0.10121293018397075, + "flos": 35217203955840.0, + "grad_norm": 2.2740576783697963, + "language_loss": 1.00311065, + "learning_rate": 3.947045579542601e-06, + "loss": 1.02477574, + "num_input_tokens_seen": 98374795, + "router_z_loss_clip": 0.74365234, + "router_z_loss_mlp": 0.22290039, + "step": 3488, + "time_per_iteration": 2.5621156692504883 + }, + { + "auxiliary_loss_clip": 0.01115112, + "auxiliary_loss_mlp": 0.01041293, + "balance_loss_clip": 1.04394817, + "balance_loss_mlp": 1.02102745, + "epoch": 0.1012419476524868, + "flos": 16536261957120.0, + "grad_norm": 3.1306764374239493, + "language_loss": 0.85534257, + "learning_rate": 3.947002604772411e-06, + "loss": 0.87690663, + "num_input_tokens_seen": 98387880, + "router_z_loss_clip": 0.7109375, + "router_z_loss_mlp": 0.20263672, + "step": 3489, + "time_per_iteration": 2.339465856552124 + }, + { + "auxiliary_loss_clip": 0.01109845, + "auxiliary_loss_mlp": 0.01048793, + "balance_loss_clip": 1.04185653, + "balance_loss_mlp": 1.02869415, + "epoch": 0.10127096512100284, + "flos": 11209318911360.0, + "grad_norm": 3.408186810435871, + "language_loss": 1.00016916, + "learning_rate": 3.946959612805507e-06, + "loss": 1.02175558, + "num_input_tokens_seen": 98400095, + "router_z_loss_clip": 0.68017578, + "router_z_loss_mlp": 0.20092773, + "step": 3490, + "time_per_iteration": 2.43788743019104 + }, + { + "auxiliary_loss_clip": 0.01121263, + "auxiliary_loss_mlp": 0.01053136, + "balance_loss_clip": 1.04539418, + "balance_loss_mlp": 1.0314455, + "epoch": 0.10129998258951889, + "flos": 29822948075520.0, + "grad_norm": 2.3334733353983776, + "language_loss": 0.83684576, + "learning_rate": 3.946916603642268e-06, + "loss": 0.85858977, + "num_input_tokens_seen": 98417980, + "router_z_loss_clip": 0.75830078, + "router_z_loss_mlp": 0.21673584, + "step": 3491, + "time_per_iteration": 2.500444173812866 + }, + { + "auxiliary_loss_clip": 0.01112627, + "auxiliary_loss_mlp": 0.01053512, + "balance_loss_clip": 1.04069877, + "balance_loss_mlp": 1.03238869, + "epoch": 0.10132900005803494, + "flos": 17303443372800.0, + "grad_norm": 2.3664584353362432, + "language_loss": 0.82072973, + "learning_rate": 3.946873577283074e-06, + "loss": 0.84239113, + "num_input_tokens_seen": 98431265, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.21130371, + "step": 3492, + "time_per_iteration": 2.401271104812622 + }, + { + "auxiliary_loss_clip": 0.01114646, + "auxiliary_loss_mlp": 0.01058567, + "balance_loss_clip": 1.0411787, + "balance_loss_mlp": 1.03596497, + "epoch": 0.10135801752655098, + "flos": 36859134286080.0, + "grad_norm": 4.389980792265147, + "language_loss": 1.01471174, + "learning_rate": 3.946830533728304e-06, + "loss": 1.03644395, + "num_input_tokens_seen": 98447820, + "router_z_loss_clip": 0.73510742, + "router_z_loss_mlp": 0.22601318, + "step": 3493, + "time_per_iteration": 2.5151495933532715 + }, + { + "auxiliary_loss_clip": 0.01112993, + "auxiliary_loss_mlp": 0.01045468, + "balance_loss_clip": 1.04241908, + "balance_loss_mlp": 1.0250833, + "epoch": 0.10138703499506703, + "flos": 35779339756800.0, + "grad_norm": 2.1155868778722597, + "language_loss": 0.88164318, + "learning_rate": 3.9467874729783395e-06, + "loss": 0.90322781, + "num_input_tokens_seen": 98470995, + "router_z_loss_clip": 0.70605469, + "router_z_loss_mlp": 0.20385742, + "step": 3494, + "time_per_iteration": 2.624697685241699 + }, + { + "auxiliary_loss_clip": 0.011134, + "auxiliary_loss_mlp": 0.01050308, + "balance_loss_clip": 1.04129624, + "balance_loss_mlp": 1.03020966, + "epoch": 0.10141605246358308, + "flos": 16465283429760.0, + "grad_norm": 2.5795912400445853, + "language_loss": 0.97989362, + "learning_rate": 3.94674439503356e-06, + "loss": 1.00153077, + "num_input_tokens_seen": 98486550, + "router_z_loss_clip": 0.72167969, + "router_z_loss_mlp": 0.20092773, + "step": 3495, + "time_per_iteration": 2.3386080265045166 + }, + { + "auxiliary_loss_clip": 0.01104467, + "auxiliary_loss_mlp": 0.01049299, + "balance_loss_clip": 1.03796458, + "balance_loss_mlp": 1.03017747, + "epoch": 0.10144506993209912, + "flos": 26571766314240.0, + "grad_norm": 1.8148092901962471, + "language_loss": 0.7719841, + "learning_rate": 3.946701299894347e-06, + "loss": 0.79352176, + "num_input_tokens_seen": 98508275, + "router_z_loss_clip": 0.66503906, + "router_z_loss_mlp": 0.19128418, + "step": 3496, + "time_per_iteration": 2.5826025009155273 + }, + { + "auxiliary_loss_clip": 0.01104409, + "auxiliary_loss_mlp": 0.01036582, + "balance_loss_clip": 1.0368576, + "balance_loss_mlp": 1.01780653, + "epoch": 0.10147408740061517, + "flos": 40624830887040.0, + "grad_norm": 2.4455813478700934, + "language_loss": 0.64212453, + "learning_rate": 3.94665818756108e-06, + "loss": 0.66353446, + "num_input_tokens_seen": 98529480, + "router_z_loss_clip": 0.67578125, + "router_z_loss_mlp": 0.18786621, + "step": 3497, + "time_per_iteration": 2.577029228210449 + }, + { + "auxiliary_loss_clip": 0.01106416, + "auxiliary_loss_mlp": 0.01039125, + "balance_loss_clip": 1.03748727, + "balance_loss_mlp": 1.01982474, + "epoch": 0.10150310486913122, + "flos": 22738896524160.0, + "grad_norm": 2.3703735427351496, + "language_loss": 0.90337712, + "learning_rate": 3.9466150580341395e-06, + "loss": 0.92483258, + "num_input_tokens_seen": 98543400, + "router_z_loss_clip": 0.68945312, + "router_z_loss_mlp": 0.1932373, + "step": 3498, + "time_per_iteration": 2.4716691970825195 + }, + { + "auxiliary_loss_clip": 0.01107093, + "auxiliary_loss_mlp": 0.01044637, + "balance_loss_clip": 1.03567493, + "balance_loss_mlp": 1.02391839, + "epoch": 0.10153212233764726, + "flos": 16321022225280.0, + "grad_norm": 3.4995422813848345, + "language_loss": 0.85374534, + "learning_rate": 3.946571911313907e-06, + "loss": 0.87526262, + "num_input_tokens_seen": 98558500, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.20703125, + "step": 3499, + "time_per_iteration": 2.3874518871307373 + }, + { + "auxiliary_loss_clip": 0.01107809, + "auxiliary_loss_mlp": 0.01041986, + "balance_loss_clip": 1.03798723, + "balance_loss_mlp": 1.02108908, + "epoch": 0.10156113980616331, + "flos": 22194251314560.0, + "grad_norm": 2.8934342498513326, + "language_loss": 0.54934943, + "learning_rate": 3.946528747400765e-06, + "loss": 0.57084739, + "num_input_tokens_seen": 98572245, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.20892334, + "step": 3500, + "time_per_iteration": 2.5398612022399902 + }, + { + "auxiliary_loss_clip": 0.01021336, + "auxiliary_loss_mlp": 0.01017687, + "balance_loss_clip": 1.00965452, + "balance_loss_mlp": 1.01669145, + "epoch": 0.10159015727467936, + "flos": 74763287654400.0, + "grad_norm": 0.7429327420883677, + "language_loss": 0.47629702, + "learning_rate": 3.9464855662950925e-06, + "loss": 0.49668723, + "num_input_tokens_seen": 98631945, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.00994873, + "step": 3501, + "time_per_iteration": 5.483505964279175 + }, + { + "auxiliary_loss_clip": 0.01106711, + "auxiliary_loss_mlp": 0.01046907, + "balance_loss_clip": 1.0359534, + "balance_loss_mlp": 1.02701139, + "epoch": 0.1016191747431954, + "flos": 14865353331840.0, + "grad_norm": 2.0553479790068514, + "language_loss": 0.67080748, + "learning_rate": 3.946442367997272e-06, + "loss": 0.69234365, + "num_input_tokens_seen": 98649125, + "router_z_loss_clip": 0.70751953, + "router_z_loss_mlp": 0.19909668, + "step": 3502, + "time_per_iteration": 4.761424541473389 + }, + { + "auxiliary_loss_clip": 0.01019935, + "auxiliary_loss_mlp": 0.01003434, + "balance_loss_clip": 1.00859845, + "balance_loss_mlp": 1.00223565, + "epoch": 0.10164819221171145, + "flos": 63838299922560.0, + "grad_norm": 0.7180763604424215, + "language_loss": 0.57137215, + "learning_rate": 3.946399152507685e-06, + "loss": 0.59160578, + "num_input_tokens_seen": 98711760, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01196289, + "step": 3503, + "time_per_iteration": 3.0485172271728516 + }, + { + "auxiliary_loss_clip": 0.01114135, + "auxiliary_loss_mlp": 0.01040788, + "balance_loss_clip": 1.04317641, + "balance_loss_mlp": 1.02095151, + "epoch": 0.1016772096802275, + "flos": 26898658174080.0, + "grad_norm": 2.2645294179741264, + "language_loss": 0.91076672, + "learning_rate": 3.9463559198267125e-06, + "loss": 0.93231589, + "num_input_tokens_seen": 98727140, + "router_z_loss_clip": 0.70898438, + "router_z_loss_mlp": 0.19836426, + "step": 3504, + "time_per_iteration": 2.5432755947113037 + }, + { + "auxiliary_loss_clip": 0.01117243, + "auxiliary_loss_mlp": 0.01055992, + "balance_loss_clip": 1.04573059, + "balance_loss_mlp": 1.03638196, + "epoch": 0.10170622714874354, + "flos": 19927899584640.0, + "grad_norm": 2.143509840834836, + "language_loss": 0.82857871, + "learning_rate": 3.946312669954737e-06, + "loss": 0.85031104, + "num_input_tokens_seen": 98740555, + "router_z_loss_clip": 0.71533203, + "router_z_loss_mlp": 0.19616699, + "step": 3505, + "time_per_iteration": 2.3842251300811768 + }, + { + "auxiliary_loss_clip": 0.01109948, + "auxiliary_loss_mlp": 0.01053406, + "balance_loss_clip": 1.04124701, + "balance_loss_mlp": 1.03413045, + "epoch": 0.1017352446172596, + "flos": 35660635534080.0, + "grad_norm": 2.4487238852172553, + "language_loss": 0.84351915, + "learning_rate": 3.946269402892141e-06, + "loss": 0.86515272, + "num_input_tokens_seen": 98760755, + "router_z_loss_clip": 0.68603516, + "router_z_loss_mlp": 0.19274902, + "step": 3506, + "time_per_iteration": 2.5384681224823 + }, + { + "auxiliary_loss_clip": 0.01114543, + "auxiliary_loss_mlp": 0.01059533, + "balance_loss_clip": 1.04274654, + "balance_loss_mlp": 1.03916013, + "epoch": 0.10176426208577563, + "flos": 18440703866880.0, + "grad_norm": 2.4155380694571718, + "language_loss": 0.70196342, + "learning_rate": 3.946226118639305e-06, + "loss": 0.72370422, + "num_input_tokens_seen": 98778370, + "router_z_loss_clip": 0.71679688, + "router_z_loss_mlp": 0.20385742, + "step": 3507, + "time_per_iteration": 2.5176830291748047 + }, + { + "auxiliary_loss_clip": 0.01022665, + "auxiliary_loss_mlp": 0.01024061, + "balance_loss_clip": 1.01124573, + "balance_loss_mlp": 1.02279115, + "epoch": 0.10179327955429168, + "flos": 67214227438080.0, + "grad_norm": 0.6773170022156573, + "language_loss": 0.52517205, + "learning_rate": 3.9461828171966135e-06, + "loss": 0.54563934, + "num_input_tokens_seen": 98839770, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01269531, + "step": 3508, + "time_per_iteration": 5.4435715675354 + }, + { + "auxiliary_loss_clip": 0.01113181, + "auxiliary_loss_mlp": 0.0104816, + "balance_loss_clip": 1.04356718, + "balance_loss_mlp": 1.02748895, + "epoch": 0.10182229702280773, + "flos": 28619422087680.0, + "grad_norm": 2.7844485218337436, + "language_loss": 0.77990514, + "learning_rate": 3.946139498564448e-06, + "loss": 0.80151856, + "num_input_tokens_seen": 98858745, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.20690918, + "step": 3509, + "time_per_iteration": 4.739392042160034 + }, + { + "auxiliary_loss_clip": 0.01022865, + "auxiliary_loss_mlp": 0.01004238, + "balance_loss_clip": 1.01118302, + "balance_loss_mlp": 1.00313497, + "epoch": 0.10185131449132377, + "flos": 71084279692800.0, + "grad_norm": 0.8381946255103705, + "language_loss": 0.50751531, + "learning_rate": 3.94609616274319e-06, + "loss": 0.52778637, + "num_input_tokens_seen": 98923750, + "router_z_loss_clip": 0.11669922, + "router_z_loss_mlp": 0.01104736, + "step": 3510, + "time_per_iteration": 3.1043176651000977 + }, + { + "auxiliary_loss_clip": 0.01108902, + "auxiliary_loss_mlp": 0.01044643, + "balance_loss_clip": 1.03989983, + "balance_loss_mlp": 1.02624857, + "epoch": 0.10188033195983982, + "flos": 15844632456960.0, + "grad_norm": 2.8743187784326802, + "language_loss": 0.72340846, + "learning_rate": 3.9460528097332235e-06, + "loss": 0.74494392, + "num_input_tokens_seen": 98935470, + "router_z_loss_clip": 0.68994141, + "router_z_loss_mlp": 0.18371582, + "step": 3511, + "time_per_iteration": 2.3637406826019287 + }, + { + "auxiliary_loss_clip": 0.01019251, + "auxiliary_loss_mlp": 0.01000711, + "balance_loss_clip": 1.00745392, + "balance_loss_mlp": 0.99944168, + "epoch": 0.10190934942835587, + "flos": 53966504263680.0, + "grad_norm": 0.7434102122729396, + "language_loss": 0.54162395, + "learning_rate": 3.946009439534931e-06, + "loss": 0.56182355, + "num_input_tokens_seen": 98989080, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.01269531, + "step": 3512, + "time_per_iteration": 2.8368136882781982 + }, + { + "auxiliary_loss_clip": 0.01107965, + "auxiliary_loss_mlp": 0.01046429, + "balance_loss_clip": 1.03730547, + "balance_loss_mlp": 1.02594304, + "epoch": 0.10193836689687191, + "flos": 31386044822400.0, + "grad_norm": 3.092838278065705, + "language_loss": 1.14879608, + "learning_rate": 3.945966052148696e-06, + "loss": 1.17033994, + "num_input_tokens_seen": 99002480, + "router_z_loss_clip": 0.70654297, + "router_z_loss_mlp": 0.20507812, + "step": 3513, + "time_per_iteration": 2.5187597274780273 + }, + { + "auxiliary_loss_clip": 0.01017361, + "auxiliary_loss_mlp": 0.01002236, + "balance_loss_clip": 1.00573337, + "balance_loss_mlp": 1.0009371, + "epoch": 0.10196738436538796, + "flos": 52808958403200.0, + "grad_norm": 0.664675196389377, + "language_loss": 0.51102763, + "learning_rate": 3.945922647574901e-06, + "loss": 0.5312236, + "num_input_tokens_seen": 99062160, + "router_z_loss_clip": 0.11621094, + "router_z_loss_mlp": 0.01300049, + "step": 3514, + "time_per_iteration": 2.884122133255005 + }, + { + "auxiliary_loss_clip": 0.01105467, + "auxiliary_loss_mlp": 0.01046719, + "balance_loss_clip": 1.03714585, + "balance_loss_mlp": 1.02822971, + "epoch": 0.10199640183390402, + "flos": 41785832972160.0, + "grad_norm": 2.2252533578500016, + "language_loss": 0.72612804, + "learning_rate": 3.94587922581393e-06, + "loss": 0.74764991, + "num_input_tokens_seen": 99083210, + "router_z_loss_clip": 0.68310547, + "router_z_loss_mlp": 0.18505859, + "step": 3515, + "time_per_iteration": 2.600816249847412 + }, + { + "auxiliary_loss_clip": 0.01109637, + "auxiliary_loss_mlp": 0.01048592, + "balance_loss_clip": 1.03563547, + "balance_loss_mlp": 1.02900648, + "epoch": 0.10202541930242005, + "flos": 17922837536640.0, + "grad_norm": 3.1993626612630552, + "language_loss": 0.96686637, + "learning_rate": 3.945835786866166e-06, + "loss": 0.98844868, + "num_input_tokens_seen": 99094380, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.19592285, + "step": 3516, + "time_per_iteration": 2.340155839920044 + }, + { + "auxiliary_loss_clip": 0.01110389, + "auxiliary_loss_mlp": 0.01045433, + "balance_loss_clip": 1.03539658, + "balance_loss_mlp": 1.02359366, + "epoch": 0.1020544367709361, + "flos": 34086681354240.0, + "grad_norm": 1.890181644844361, + "language_loss": 0.92940032, + "learning_rate": 3.9457923307319935e-06, + "loss": 0.95095849, + "num_input_tokens_seen": 99114625, + "router_z_loss_clip": 0.75, + "router_z_loss_mlp": 0.21862793, + "step": 3517, + "time_per_iteration": 2.5595269203186035 + }, + { + "auxiliary_loss_clip": 0.01107353, + "auxiliary_loss_mlp": 0.01045716, + "balance_loss_clip": 1.03462458, + "balance_loss_mlp": 1.02621365, + "epoch": 0.10208345423945216, + "flos": 28252066095360.0, + "grad_norm": 1.9042550225226, + "language_loss": 0.80841231, + "learning_rate": 3.945748857411796e-06, + "loss": 0.82994294, + "num_input_tokens_seen": 99130675, + "router_z_loss_clip": 0.72680664, + "router_z_loss_mlp": 0.19494629, + "step": 3518, + "time_per_iteration": 2.476638078689575 + }, + { + "auxiliary_loss_clip": 0.01110823, + "auxiliary_loss_mlp": 0.01050552, + "balance_loss_clip": 1.03364205, + "balance_loss_mlp": 1.02598262, + "epoch": 0.1021124717079682, + "flos": 13625343106560.0, + "grad_norm": 2.5820122220987973, + "language_loss": 0.80956, + "learning_rate": 3.9457053669059555e-06, + "loss": 0.83117372, + "num_input_tokens_seen": 99144450, + "router_z_loss_clip": 0.77246094, + "router_z_loss_mlp": 0.24560547, + "step": 3519, + "time_per_iteration": 2.4333746433258057 + }, + { + "auxiliary_loss_clip": 0.0111928, + "auxiliary_loss_mlp": 0.01049608, + "balance_loss_clip": 1.03446984, + "balance_loss_mlp": 1.02419305, + "epoch": 0.10214148917648425, + "flos": 33575727473280.0, + "grad_norm": 4.348841129241996, + "language_loss": 0.9877004, + "learning_rate": 3.945661859214859e-06, + "loss": 1.00938928, + "num_input_tokens_seen": 99159685, + "router_z_loss_clip": 0.84863281, + "router_z_loss_mlp": 0.25415039, + "step": 3520, + "time_per_iteration": 2.477477550506592 + }, + { + "auxiliary_loss_clip": 0.01107678, + "auxiliary_loss_mlp": 0.01044979, + "balance_loss_clip": 1.03491962, + "balance_loss_mlp": 1.02563143, + "epoch": 0.10217050664500028, + "flos": 17266854401280.0, + "grad_norm": 2.958167866137248, + "language_loss": 0.89706147, + "learning_rate": 3.94561833433889e-06, + "loss": 0.91858804, + "num_input_tokens_seen": 99172800, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.19348145, + "step": 3521, + "time_per_iteration": 2.385510206222534 + }, + { + "auxiliary_loss_clip": 0.01108813, + "auxiliary_loss_mlp": 0.01039165, + "balance_loss_clip": 1.03651547, + "balance_loss_mlp": 1.01775479, + "epoch": 0.10219952411351634, + "flos": 28650983823360.0, + "grad_norm": 2.083925686635021, + "language_loss": 0.84331441, + "learning_rate": 3.9455747922784324e-06, + "loss": 0.86479425, + "num_input_tokens_seen": 99188085, + "router_z_loss_clip": 0.72412109, + "router_z_loss_mlp": 0.2142334, + "step": 3522, + "time_per_iteration": 2.4668712615966797 + }, + { + "auxiliary_loss_clip": 0.01102892, + "auxiliary_loss_mlp": 0.01039623, + "balance_loss_clip": 1.03604662, + "balance_loss_mlp": 1.02066827, + "epoch": 0.10222854158203239, + "flos": 35509042944000.0, + "grad_norm": 2.608311098184278, + "language_loss": 0.73193753, + "learning_rate": 3.94553123303387e-06, + "loss": 0.75336266, + "num_input_tokens_seen": 99204030, + "router_z_loss_clip": 0.66796875, + "router_z_loss_mlp": 0.18969727, + "step": 3523, + "time_per_iteration": 2.558417320251465 + }, + { + "auxiliary_loss_clip": 0.01112239, + "auxiliary_loss_mlp": 0.01041947, + "balance_loss_clip": 1.03704977, + "balance_loss_mlp": 1.01893985, + "epoch": 0.10225755905054842, + "flos": 30036581884800.0, + "grad_norm": 1.8854220833105921, + "language_loss": 0.86445558, + "learning_rate": 3.9454876566055895e-06, + "loss": 0.88599747, + "num_input_tokens_seen": 99222915, + "router_z_loss_clip": 0.75195312, + "router_z_loss_mlp": 0.2298584, + "step": 3524, + "time_per_iteration": 2.4448864459991455 + }, + { + "auxiliary_loss_clip": 0.01101904, + "auxiliary_loss_mlp": 0.01043218, + "balance_loss_clip": 1.03340626, + "balance_loss_mlp": 1.02230918, + "epoch": 0.10228657651906448, + "flos": 29199958041600.0, + "grad_norm": 2.8631108198714865, + "language_loss": 0.81631541, + "learning_rate": 3.945444062993975e-06, + "loss": 0.83776659, + "num_input_tokens_seen": 99239685, + "router_z_loss_clip": 0.68457031, + "router_z_loss_mlp": 0.20898438, + "step": 3525, + "time_per_iteration": 2.5102784633636475 + }, + { + "auxiliary_loss_clip": 0.01101558, + "auxiliary_loss_mlp": 0.01040011, + "balance_loss_clip": 1.03589237, + "balance_loss_mlp": 1.02031207, + "epoch": 0.10231559398758053, + "flos": 34233246708480.0, + "grad_norm": 2.2762446811005232, + "language_loss": 0.87366116, + "learning_rate": 3.94540045219941e-06, + "loss": 0.89507687, + "num_input_tokens_seen": 99258785, + "router_z_loss_clip": 0.65673828, + "router_z_loss_mlp": 0.19677734, + "step": 3526, + "time_per_iteration": 2.524273633956909 + }, + { + "auxiliary_loss_clip": 0.01106618, + "auxiliary_loss_mlp": 0.01035792, + "balance_loss_clip": 1.03772068, + "balance_loss_mlp": 1.01718354, + "epoch": 0.10234461145609657, + "flos": 31242342199680.0, + "grad_norm": 2.1841215863026155, + "language_loss": 0.93946904, + "learning_rate": 3.945356824222282e-06, + "loss": 0.96089315, + "num_input_tokens_seen": 99277965, + "router_z_loss_clip": 0.68896484, + "router_z_loss_mlp": 0.18591309, + "step": 3527, + "time_per_iteration": 2.533653497695923 + }, + { + "auxiliary_loss_clip": 0.01102547, + "auxiliary_loss_mlp": 0.01045654, + "balance_loss_clip": 1.03496087, + "balance_loss_mlp": 1.02701616, + "epoch": 0.10237362892461262, + "flos": 12195545397120.0, + "grad_norm": 2.109147417370676, + "language_loss": 0.76791763, + "learning_rate": 3.945313179062975e-06, + "loss": 0.78939962, + "num_input_tokens_seen": 99289835, + "router_z_loss_clip": 0.67724609, + "router_z_loss_mlp": 0.18634033, + "step": 3528, + "time_per_iteration": 2.353287696838379 + }, + { + "auxiliary_loss_clip": 0.01099836, + "auxiliary_loss_mlp": 0.01033313, + "balance_loss_clip": 1.0355134, + "balance_loss_mlp": 1.01531255, + "epoch": 0.10240264639312867, + "flos": 10659157706880.0, + "grad_norm": 3.129870207623067, + "language_loss": 0.97802246, + "learning_rate": 3.945269516721875e-06, + "loss": 0.999354, + "num_input_tokens_seen": 99298085, + "router_z_loss_clip": 0.64306641, + "router_z_loss_mlp": 0.18005371, + "step": 3529, + "time_per_iteration": 2.431807279586792 + }, + { + "auxiliary_loss_clip": 0.01024566, + "auxiliary_loss_mlp": 0.01013267, + "balance_loss_clip": 1.01283979, + "balance_loss_mlp": 1.01180112, + "epoch": 0.1024316638616447, + "flos": 58901791144320.0, + "grad_norm": 0.6867035249822128, + "language_loss": 0.49938077, + "learning_rate": 3.945225837199367e-06, + "loss": 0.51975912, + "num_input_tokens_seen": 99358565, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01464844, + "step": 3530, + "time_per_iteration": 2.938283681869507 + }, + { + "auxiliary_loss_clip": 0.01111632, + "auxiliary_loss_mlp": 0.01050663, + "balance_loss_clip": 1.03483295, + "balance_loss_mlp": 1.02826357, + "epoch": 0.10246068133016076, + "flos": 29347710382080.0, + "grad_norm": 1.9398367104827825, + "language_loss": 0.93308872, + "learning_rate": 3.945182140495838e-06, + "loss": 0.95471168, + "num_input_tokens_seen": 99378885, + "router_z_loss_clip": 0.76806641, + "router_z_loss_mlp": 0.22387695, + "step": 3531, + "time_per_iteration": 2.508542060852051 + }, + { + "auxiliary_loss_clip": 0.01095986, + "auxiliary_loss_mlp": 0.0103963, + "balance_loss_clip": 1.03109932, + "balance_loss_mlp": 1.02108145, + "epoch": 0.10248969879867681, + "flos": 23762864056320.0, + "grad_norm": 1.9821998603300277, + "language_loss": 0.92496651, + "learning_rate": 3.945138426611672e-06, + "loss": 0.94632268, + "num_input_tokens_seen": 99394075, + "router_z_loss_clip": 0.64892578, + "router_z_loss_mlp": 0.18518066, + "step": 3532, + "time_per_iteration": 2.407025098800659 + }, + { + "auxiliary_loss_clip": 0.01106043, + "auxiliary_loss_mlp": 0.01041828, + "balance_loss_clip": 1.03418183, + "balance_loss_mlp": 1.01985765, + "epoch": 0.10251871626719285, + "flos": 32190792727680.0, + "grad_norm": 3.3554153830790145, + "language_loss": 1.04589927, + "learning_rate": 3.945094695547258e-06, + "loss": 1.06737804, + "num_input_tokens_seen": 99413140, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.21984863, + "step": 3533, + "time_per_iteration": 2.5051918029785156 + }, + { + "auxiliary_loss_clip": 0.01103826, + "auxiliary_loss_mlp": 0.0104176, + "balance_loss_clip": 1.03303301, + "balance_loss_mlp": 1.02242434, + "epoch": 0.1025477337357089, + "flos": 25147414776960.0, + "grad_norm": 2.934722046578235, + "language_loss": 0.89707989, + "learning_rate": 3.945050947302979e-06, + "loss": 0.91853583, + "num_input_tokens_seen": 99427310, + "router_z_loss_clip": 0.70849609, + "router_z_loss_mlp": 0.19348145, + "step": 3534, + "time_per_iteration": 2.4079267978668213 + }, + { + "auxiliary_loss_clip": 0.01101608, + "auxiliary_loss_mlp": 0.01038422, + "balance_loss_clip": 1.03441882, + "balance_loss_mlp": 1.02056444, + "epoch": 0.10257675120422495, + "flos": 24124145472000.0, + "grad_norm": 2.0087984924577262, + "language_loss": 0.75638801, + "learning_rate": 3.945007181879224e-06, + "loss": 0.77778828, + "num_input_tokens_seen": 99441375, + "router_z_loss_clip": 0.67138672, + "router_z_loss_mlp": 0.1784668, + "step": 3535, + "time_per_iteration": 2.467965841293335 + }, + { + "auxiliary_loss_clip": 0.01106778, + "auxiliary_loss_mlp": 0.01043637, + "balance_loss_clip": 1.03477311, + "balance_loss_mlp": 1.0231086, + "epoch": 0.10260576867274099, + "flos": 15842293395840.0, + "grad_norm": 3.307574874717922, + "language_loss": 1.03558052, + "learning_rate": 3.944963399276378e-06, + "loss": 1.0570848, + "num_input_tokens_seen": 99453280, + "router_z_loss_clip": 0.71972656, + "router_z_loss_mlp": 0.20532227, + "step": 3536, + "time_per_iteration": 2.3306922912597656 + }, + { + "auxiliary_loss_clip": 0.01019274, + "auxiliary_loss_mlp": 0.01004795, + "balance_loss_clip": 1.00753617, + "balance_loss_mlp": 1.00345421, + "epoch": 0.10263478614125704, + "flos": 70690808136960.0, + "grad_norm": 0.6247325578225976, + "language_loss": 0.47049171, + "learning_rate": 3.944919599494828e-06, + "loss": 0.4907324, + "num_input_tokens_seen": 99520100, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01342773, + "step": 3537, + "time_per_iteration": 3.093066930770874 + }, + { + "auxiliary_loss_clip": 0.01101096, + "auxiliary_loss_mlp": 0.01038522, + "balance_loss_clip": 1.03117096, + "balance_loss_mlp": 1.01907921, + "epoch": 0.10266380360977308, + "flos": 14823911681280.0, + "grad_norm": 2.3545728825167465, + "language_loss": 0.76169252, + "learning_rate": 3.944875782534962e-06, + "loss": 0.78308868, + "num_input_tokens_seen": 99532345, + "router_z_loss_clip": 0.69921875, + "router_z_loss_mlp": 0.19445801, + "step": 3538, + "time_per_iteration": 2.3348968029022217 + }, + { + "auxiliary_loss_clip": 0.01016954, + "auxiliary_loss_mlp": 0.01003523, + "balance_loss_clip": 1.00523174, + "balance_loss_mlp": 1.00227749, + "epoch": 0.10269282107828913, + "flos": 63574601356800.0, + "grad_norm": 0.6690972131371807, + "language_loss": 0.49227342, + "learning_rate": 3.9448319483971655e-06, + "loss": 0.51247823, + "num_input_tokens_seen": 99593660, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01245117, + "step": 3539, + "time_per_iteration": 3.068122625350952 + }, + { + "auxiliary_loss_clip": 0.01100919, + "auxiliary_loss_mlp": 0.01040928, + "balance_loss_clip": 1.033391, + "balance_loss_mlp": 1.02133572, + "epoch": 0.10272183854680518, + "flos": 29344568359680.0, + "grad_norm": 2.313932784074866, + "language_loss": 1.02085328, + "learning_rate": 3.944788097081826e-06, + "loss": 1.04227185, + "num_input_tokens_seen": 99614345, + "router_z_loss_clip": 0.67529297, + "router_z_loss_mlp": 0.19592285, + "step": 3540, + "time_per_iteration": 2.4719417095184326 + }, + { + "auxiliary_loss_clip": 0.01098527, + "auxiliary_loss_mlp": 0.01035612, + "balance_loss_clip": 1.03202438, + "balance_loss_mlp": 1.01736081, + "epoch": 0.10275085601532122, + "flos": 24928963200000.0, + "grad_norm": 2.4267927364142112, + "language_loss": 0.76628339, + "learning_rate": 3.944744228589331e-06, + "loss": 0.78762472, + "num_input_tokens_seen": 99629355, + "router_z_loss_clip": 0.66503906, + "router_z_loss_mlp": 0.18249512, + "step": 3541, + "time_per_iteration": 2.426124095916748 + }, + { + "auxiliary_loss_clip": 0.01106687, + "auxiliary_loss_mlp": 0.01049929, + "balance_loss_clip": 1.03500414, + "balance_loss_mlp": 1.02920997, + "epoch": 0.10277987348383727, + "flos": 48059166775680.0, + "grad_norm": 1.9979822594150536, + "language_loss": 0.85442823, + "learning_rate": 3.944700342920069e-06, + "loss": 0.87599444, + "num_input_tokens_seen": 99648845, + "router_z_loss_clip": 0.71630859, + "router_z_loss_mlp": 0.20733643, + "step": 3542, + "time_per_iteration": 2.6224143505096436 + }, + { + "auxiliary_loss_clip": 0.01016159, + "auxiliary_loss_mlp": 0.01002239, + "balance_loss_clip": 1.00457609, + "balance_loss_mlp": 1.00099325, + "epoch": 0.10280889095235332, + "flos": 68533455271680.0, + "grad_norm": 0.6772859345007819, + "language_loss": 0.4886196, + "learning_rate": 3.9446564400744255e-06, + "loss": 0.50880355, + "num_input_tokens_seen": 99710180, + "router_z_loss_clip": 0.11621094, + "router_z_loss_mlp": 0.01245117, + "step": 3543, + "time_per_iteration": 2.981843948364258 + }, + { + "auxiliary_loss_clip": 0.01015723, + "auxiliary_loss_mlp": 0.01002966, + "balance_loss_clip": 1.00417173, + "balance_loss_mlp": 1.00157154, + "epoch": 0.10283790842086936, + "flos": 60468553584000.0, + "grad_norm": 0.6576225812917045, + "language_loss": 0.48136297, + "learning_rate": 3.94461252005279e-06, + "loss": 0.50154984, + "num_input_tokens_seen": 99767705, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01397705, + "step": 3544, + "time_per_iteration": 2.9363033771514893 + }, + { + "auxiliary_loss_clip": 0.01104259, + "auxiliary_loss_mlp": 0.01043304, + "balance_loss_clip": 1.03250086, + "balance_loss_mlp": 1.02300298, + "epoch": 0.10286692588938541, + "flos": 33941372808960.0, + "grad_norm": 2.469086043266352, + "language_loss": 0.73915941, + "learning_rate": 3.944568582855549e-06, + "loss": 0.76063502, + "num_input_tokens_seen": 99783540, + "router_z_loss_clip": 0.71679688, + "router_z_loss_mlp": 0.20324707, + "step": 3545, + "time_per_iteration": 2.428203821182251 + }, + { + "auxiliary_loss_clip": 0.0111282, + "auxiliary_loss_mlp": 0.01047483, + "balance_loss_clip": 1.03517294, + "balance_loss_mlp": 1.02318835, + "epoch": 0.10289594335790146, + "flos": 27776688756480.0, + "grad_norm": 2.477356545401687, + "language_loss": 0.87900186, + "learning_rate": 3.944524628483093e-06, + "loss": 0.90060496, + "num_input_tokens_seen": 99801740, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.24291992, + "step": 3546, + "time_per_iteration": 2.4574766159057617 + }, + { + "auxiliary_loss_clip": 0.01100786, + "auxiliary_loss_mlp": 0.01042821, + "balance_loss_clip": 1.03184032, + "balance_loss_mlp": 1.02248383, + "epoch": 0.1029249608264175, + "flos": 10405583435520.0, + "grad_norm": 3.2880629248980897, + "language_loss": 0.87347913, + "learning_rate": 3.944480656935807e-06, + "loss": 0.89491522, + "num_input_tokens_seen": 99813705, + "router_z_loss_clip": 0.68896484, + "router_z_loss_mlp": 0.20343018, + "step": 3547, + "time_per_iteration": 2.3340988159179688 + }, + { + "auxiliary_loss_clip": 0.01014477, + "auxiliary_loss_mlp": 0.01001219, + "balance_loss_clip": 1.00320506, + "balance_loss_mlp": 0.99970472, + "epoch": 0.10295397829493355, + "flos": 74779032677760.0, + "grad_norm": 0.5962161752941036, + "language_loss": 0.50775325, + "learning_rate": 3.944436668214082e-06, + "loss": 0.52791017, + "num_input_tokens_seen": 99882985, + "router_z_loss_clip": 0.11279297, + "router_z_loss_mlp": 0.01513672, + "step": 3548, + "time_per_iteration": 3.145132303237915 + }, + { + "auxiliary_loss_clip": 0.01109382, + "auxiliary_loss_mlp": 0.01050244, + "balance_loss_clip": 1.03586817, + "balance_loss_mlp": 1.02861357, + "epoch": 0.1029829957634496, + "flos": 28249273186560.0, + "grad_norm": 3.43345542633292, + "language_loss": 0.93463099, + "learning_rate": 3.9443926623183045e-06, + "loss": 0.9562273, + "num_input_tokens_seen": 99898050, + "router_z_loss_clip": 0.73510742, + "router_z_loss_mlp": 0.21612549, + "step": 3549, + "time_per_iteration": 2.4588420391082764 + }, + { + "auxiliary_loss_clip": 0.01097382, + "auxiliary_loss_mlp": 0.01037377, + "balance_loss_clip": 1.03288102, + "balance_loss_mlp": 1.01993692, + "epoch": 0.10301201323196564, + "flos": 11355500240640.0, + "grad_norm": 2.4689431163210447, + "language_loss": 0.694574, + "learning_rate": 3.944348639248865e-06, + "loss": 0.71592164, + "num_input_tokens_seen": 99909700, + "router_z_loss_clip": 0.64453125, + "router_z_loss_mlp": 0.17437744, + "step": 3550, + "time_per_iteration": 2.45284104347229 + }, + { + "auxiliary_loss_clip": 0.01014637, + "auxiliary_loss_mlp": 0.01002213, + "balance_loss_clip": 1.00349283, + "balance_loss_mlp": 1.00080609, + "epoch": 0.10304103070048169, + "flos": 74768105422080.0, + "grad_norm": 0.6605411846623508, + "language_loss": 0.53083169, + "learning_rate": 3.944304599006151e-06, + "loss": 0.55100024, + "num_input_tokens_seen": 99973710, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.01403809, + "step": 3551, + "time_per_iteration": 3.1143481731414795 + }, + { + "auxiliary_loss_clip": 0.01014712, + "auxiliary_loss_mlp": 0.01002233, + "balance_loss_clip": 1.00345528, + "balance_loss_mlp": 1.00095129, + "epoch": 0.10307004816899773, + "flos": 74777740957440.0, + "grad_norm": 0.6740459333002509, + "language_loss": 0.50124007, + "learning_rate": 3.944260541590553e-06, + "loss": 0.52140951, + "num_input_tokens_seen": 100036985, + "router_z_loss_clip": 0.11279297, + "router_z_loss_mlp": 0.01281738, + "step": 3552, + "time_per_iteration": 3.1171956062316895 + }, + { + "auxiliary_loss_clip": 0.01100078, + "auxiliary_loss_mlp": 0.01040573, + "balance_loss_clip": 1.03290224, + "balance_loss_mlp": 1.02120197, + "epoch": 0.10309906563751378, + "flos": 24054214285440.0, + "grad_norm": 2.3457257732327745, + "language_loss": 0.88563353, + "learning_rate": 3.944216467002458e-06, + "loss": 0.90704006, + "num_input_tokens_seen": 100051455, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.19372559, + "step": 3553, + "time_per_iteration": 2.3900246620178223 + }, + { + "auxiliary_loss_clip": 0.01114026, + "auxiliary_loss_mlp": 0.0105177, + "balance_loss_clip": 1.03699291, + "balance_loss_mlp": 1.02757061, + "epoch": 0.10312808310602983, + "flos": 36675142087680.0, + "grad_norm": 3.1031383091372895, + "language_loss": 1.0330447, + "learning_rate": 3.944172375242258e-06, + "loss": 1.05470264, + "num_input_tokens_seen": 100065375, + "router_z_loss_clip": 0.77050781, + "router_z_loss_mlp": 0.24194336, + "step": 3554, + "time_per_iteration": 2.526801109313965 + }, + { + "auxiliary_loss_clip": 0.01109096, + "auxiliary_loss_mlp": 0.0104604, + "balance_loss_clip": 1.03593862, + "balance_loss_mlp": 1.02433181, + "epoch": 0.10315710057454587, + "flos": 33321350240640.0, + "grad_norm": 2.1036257955041004, + "language_loss": 0.71010756, + "learning_rate": 3.944128266310339e-06, + "loss": 0.73165888, + "num_input_tokens_seen": 100084000, + "router_z_loss_clip": 0.73242188, + "router_z_loss_mlp": 0.21704102, + "step": 3555, + "time_per_iteration": 2.4892969131469727 + }, + { + "auxiliary_loss_clip": 0.01104483, + "auxiliary_loss_mlp": 0.01047709, + "balance_loss_clip": 1.03466845, + "balance_loss_mlp": 1.02725244, + "epoch": 0.10318611804306192, + "flos": 32226020156160.0, + "grad_norm": 2.1422675931318924, + "language_loss": 0.95256197, + "learning_rate": 3.944084140207093e-06, + "loss": 0.9740839, + "num_input_tokens_seen": 100105350, + "router_z_loss_clip": 0.69824219, + "router_z_loss_mlp": 0.20446777, + "step": 3556, + "time_per_iteration": 2.4926819801330566 + }, + { + "auxiliary_loss_clip": 0.01103353, + "auxiliary_loss_mlp": 0.01032902, + "balance_loss_clip": 1.03490269, + "balance_loss_mlp": 1.01355481, + "epoch": 0.10321513551157797, + "flos": 36827572550400.0, + "grad_norm": 2.378676817095263, + "language_loss": 1.1800369, + "learning_rate": 3.944039996932909e-06, + "loss": 1.20139956, + "num_input_tokens_seen": 100134310, + "router_z_loss_clip": 0.68359375, + "router_z_loss_mlp": 0.19354248, + "step": 3557, + "time_per_iteration": 2.8141562938690186 + }, + { + "auxiliary_loss_clip": 0.0111055, + "auxiliary_loss_mlp": 0.01045062, + "balance_loss_clip": 1.03706527, + "balance_loss_mlp": 1.02244806, + "epoch": 0.10324415298009401, + "flos": 24124529496960.0, + "grad_norm": 2.7233235122780255, + "language_loss": 0.76788908, + "learning_rate": 3.9439958364881785e-06, + "loss": 0.78944522, + "num_input_tokens_seen": 100149735, + "router_z_loss_clip": 0.734375, + "router_z_loss_mlp": 0.22619629, + "step": 3558, + "time_per_iteration": 2.4865009784698486 + }, + { + "auxiliary_loss_clip": 0.01108004, + "auxiliary_loss_mlp": 0.01040699, + "balance_loss_clip": 1.03525615, + "balance_loss_mlp": 1.01999211, + "epoch": 0.10327317044861006, + "flos": 15734481517440.0, + "grad_norm": 2.1794541580191322, + "language_loss": 0.88775885, + "learning_rate": 3.943951658873289e-06, + "loss": 0.90924591, + "num_input_tokens_seen": 100163390, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.20727539, + "step": 3559, + "time_per_iteration": 2.4212169647216797 + }, + { + "auxiliary_loss_clip": 0.010219, + "auxiliary_loss_mlp": 0.01015501, + "balance_loss_clip": 1.01061368, + "balance_loss_mlp": 1.01411223, + "epoch": 0.10330218791712611, + "flos": 74763497122560.0, + "grad_norm": 0.6991408380585884, + "language_loss": 0.46530634, + "learning_rate": 3.9439074640886314e-06, + "loss": 0.48568034, + "num_input_tokens_seen": 100221550, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01391602, + "step": 3560, + "time_per_iteration": 2.948697328567505 + }, + { + "auxiliary_loss_clip": 0.01021187, + "auxiliary_loss_mlp": 0.01011029, + "balance_loss_clip": 1.00988221, + "balance_loss_mlp": 1.00952685, + "epoch": 0.10333120538564215, + "flos": 65036065536000.0, + "grad_norm": 0.6311156961193104, + "language_loss": 0.49445468, + "learning_rate": 3.9438632521345975e-06, + "loss": 0.51477683, + "num_input_tokens_seen": 100283810, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01501465, + "step": 3561, + "time_per_iteration": 3.018866539001465 + }, + { + "auxiliary_loss_clip": 0.0111103, + "auxiliary_loss_mlp": 0.01045257, + "balance_loss_clip": 1.03802252, + "balance_loss_mlp": 1.02619565, + "epoch": 0.1033602228541582, + "flos": 39340900304640.0, + "grad_norm": 2.066193042481146, + "language_loss": 0.88719451, + "learning_rate": 3.943819023011576e-06, + "loss": 0.90875745, + "num_input_tokens_seen": 100303220, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.19067383, + "step": 3562, + "time_per_iteration": 2.5258920192718506 + }, + { + "auxiliary_loss_clip": 0.01110037, + "auxiliary_loss_mlp": 0.01045706, + "balance_loss_clip": 1.03693128, + "balance_loss_mlp": 1.02398658, + "epoch": 0.10338924032267426, + "flos": 25731406955520.0, + "grad_norm": 3.2014969031872886, + "language_loss": 1.00998771, + "learning_rate": 3.943774776719959e-06, + "loss": 1.03154516, + "num_input_tokens_seen": 100316540, + "router_z_loss_clip": 0.73095703, + "router_z_loss_mlp": 0.21734619, + "step": 3563, + "time_per_iteration": 2.4421169757843018 + }, + { + "auxiliary_loss_clip": 0.01108886, + "auxiliary_loss_mlp": 0.01047032, + "balance_loss_clip": 1.03632772, + "balance_loss_mlp": 1.02716017, + "epoch": 0.1034182577911903, + "flos": 30473345393280.0, + "grad_norm": 1.901312249699209, + "language_loss": 0.81223786, + "learning_rate": 3.943730513260136e-06, + "loss": 0.83379698, + "num_input_tokens_seen": 100333835, + "router_z_loss_clip": 0.72509766, + "router_z_loss_mlp": 0.19873047, + "step": 3564, + "time_per_iteration": 2.4443295001983643 + }, + { + "auxiliary_loss_clip": 0.01094063, + "auxiliary_loss_mlp": 0.0103372, + "balance_loss_clip": 1.03008318, + "balance_loss_mlp": 1.01555872, + "epoch": 0.10344727525970634, + "flos": 31496475052800.0, + "grad_norm": 2.5842232630731385, + "language_loss": 0.83377051, + "learning_rate": 3.943686232632498e-06, + "loss": 0.85504842, + "num_input_tokens_seen": 100349975, + "router_z_loss_clip": 0.63989258, + "router_z_loss_mlp": 0.18170166, + "step": 3565, + "time_per_iteration": 2.5160086154937744 + }, + { + "auxiliary_loss_clip": 0.01012274, + "auxiliary_loss_mlp": 0.01007074, + "balance_loss_clip": 1.0010674, + "balance_loss_mlp": 1.00570941, + "epoch": 0.1034762927282224, + "flos": 64738431262080.0, + "grad_norm": 0.6484987463006112, + "language_loss": 0.49546099, + "learning_rate": 3.943641934837438e-06, + "loss": 0.51565444, + "num_input_tokens_seen": 100410670, + "router_z_loss_clip": 0.11230469, + "router_z_loss_mlp": 0.01367188, + "step": 3566, + "time_per_iteration": 3.048530340194702 + }, + { + "auxiliary_loss_clip": 0.01099027, + "auxiliary_loss_mlp": 0.01039274, + "balance_loss_clip": 1.0339222, + "balance_loss_mlp": 1.02121973, + "epoch": 0.10350531019673843, + "flos": 14457288827520.0, + "grad_norm": 2.3706455946995755, + "language_loss": 0.81076908, + "learning_rate": 3.943597619875345e-06, + "loss": 0.83215201, + "num_input_tokens_seen": 100423595, + "router_z_loss_clip": 0.65087891, + "router_z_loss_mlp": 0.18054199, + "step": 3567, + "time_per_iteration": 2.385159969329834 + }, + { + "auxiliary_loss_clip": 0.01110547, + "auxiliary_loss_mlp": 0.01045825, + "balance_loss_clip": 1.03264618, + "balance_loss_mlp": 1.02358007, + "epoch": 0.10353432766525449, + "flos": 29797705296000.0, + "grad_norm": 2.16899394387255, + "language_loss": 0.93512642, + "learning_rate": 3.9435532877466116e-06, + "loss": 0.95669019, + "num_input_tokens_seen": 100446720, + "router_z_loss_clip": 0.77880859, + "router_z_loss_mlp": 0.22277832, + "step": 3568, + "time_per_iteration": 2.4818501472473145 + }, + { + "auxiliary_loss_clip": 0.01099926, + "auxiliary_loss_mlp": 0.01040344, + "balance_loss_clip": 1.03208256, + "balance_loss_mlp": 1.02183104, + "epoch": 0.10356334513377052, + "flos": 21394356088320.0, + "grad_norm": 1.8860729294321117, + "language_loss": 0.76213896, + "learning_rate": 3.94350893845163e-06, + "loss": 0.78354162, + "num_input_tokens_seen": 100463885, + "router_z_loss_clip": 0.67871094, + "router_z_loss_mlp": 0.18518066, + "step": 3569, + "time_per_iteration": 2.54707932472229 + }, + { + "auxiliary_loss_clip": 0.01104412, + "auxiliary_loss_mlp": 0.01041866, + "balance_loss_clip": 1.03401053, + "balance_loss_mlp": 1.02099276, + "epoch": 0.10359236260228658, + "flos": 33758113749120.0, + "grad_norm": 2.4650947007776316, + "language_loss": 0.80145001, + "learning_rate": 3.94346457199079e-06, + "loss": 0.82291281, + "num_input_tokens_seen": 100479820, + "router_z_loss_clip": 0.70410156, + "router_z_loss_mlp": 0.20874023, + "step": 3570, + "time_per_iteration": 2.523042678833008 + }, + { + "auxiliary_loss_clip": 0.01106871, + "auxiliary_loss_mlp": 0.01046833, + "balance_loss_clip": 1.03282487, + "balance_loss_mlp": 1.02543473, + "epoch": 0.10362138007080263, + "flos": 34233491088000.0, + "grad_norm": 2.2211156064359265, + "language_loss": 0.91242629, + "learning_rate": 3.943420188364484e-06, + "loss": 0.93396336, + "num_input_tokens_seen": 100498050, + "router_z_loss_clip": 0.74121094, + "router_z_loss_mlp": 0.21386719, + "step": 3571, + "time_per_iteration": 2.5704236030578613 + }, + { + "auxiliary_loss_clip": 0.01107042, + "auxiliary_loss_mlp": 0.01039589, + "balance_loss_clip": 1.03344238, + "balance_loss_mlp": 1.01772594, + "epoch": 0.10365039753931866, + "flos": 18470380389120.0, + "grad_norm": 1.8784269570401702, + "language_loss": 0.65475637, + "learning_rate": 3.943375787573106e-06, + "loss": 0.67622268, + "num_input_tokens_seen": 100511910, + "router_z_loss_clip": 0.73632812, + "router_z_loss_mlp": 0.21844482, + "step": 3572, + "time_per_iteration": 2.362654447555542 + }, + { + "auxiliary_loss_clip": 0.01014896, + "auxiliary_loss_mlp": 0.01005421, + "balance_loss_clip": 1.00357008, + "balance_loss_mlp": 1.00411618, + "epoch": 0.10367941500783472, + "flos": 74771945671680.0, + "grad_norm": 0.674301392503831, + "language_loss": 0.4714151, + "learning_rate": 3.943331369617045e-06, + "loss": 0.49161828, + "num_input_tokens_seen": 100569920, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01306152, + "step": 3573, + "time_per_iteration": 3.0659878253936768 + }, + { + "auxiliary_loss_clip": 0.01015835, + "auxiliary_loss_mlp": 0.01003323, + "balance_loss_clip": 1.00428236, + "balance_loss_mlp": 1.00205326, + "epoch": 0.10370843247635077, + "flos": 61742569340160.0, + "grad_norm": 0.7115883039387342, + "language_loss": 0.46647686, + "learning_rate": 3.943286934496695e-06, + "loss": 0.48666847, + "num_input_tokens_seen": 100631400, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01269531, + "step": 3574, + "time_per_iteration": 2.951510190963745 + }, + { + "auxiliary_loss_clip": 0.01014663, + "auxiliary_loss_mlp": 0.01001987, + "balance_loss_clip": 1.00325239, + "balance_loss_mlp": 1.00060999, + "epoch": 0.1037374499448668, + "flos": 53348227263360.0, + "grad_norm": 0.757911870404808, + "language_loss": 0.51526499, + "learning_rate": 3.94324248221245e-06, + "loss": 0.5354315, + "num_input_tokens_seen": 100684225, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01379395, + "step": 3575, + "time_per_iteration": 2.7845242023468018 + }, + { + "auxiliary_loss_clip": 0.01013933, + "auxiliary_loss_mlp": 0.01002443, + "balance_loss_clip": 1.00257993, + "balance_loss_mlp": 1.00120342, + "epoch": 0.10376646741338286, + "flos": 70067189698560.0, + "grad_norm": 0.6415688758492589, + "language_loss": 0.49472916, + "learning_rate": 3.9431980127647e-06, + "loss": 0.51489294, + "num_input_tokens_seen": 100750595, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01239014, + "step": 3576, + "time_per_iteration": 3.1426539421081543 + }, + { + "auxiliary_loss_clip": 0.01097796, + "auxiliary_loss_mlp": 0.01038796, + "balance_loss_clip": 1.02942872, + "balance_loss_mlp": 1.0173856, + "epoch": 0.10379548488189891, + "flos": 33791944723200.0, + "grad_norm": 11.473365574015457, + "language_loss": 0.92626792, + "learning_rate": 3.943153526153839e-06, + "loss": 0.94763386, + "num_input_tokens_seen": 100765890, + "router_z_loss_clip": 0.68359375, + "router_z_loss_mlp": 0.21411133, + "step": 3577, + "time_per_iteration": 4.684573173522949 + }, + { + "auxiliary_loss_clip": 0.01101885, + "auxiliary_loss_mlp": 0.01042266, + "balance_loss_clip": 1.03269827, + "balance_loss_mlp": 1.02067733, + "epoch": 0.10382450235041495, + "flos": 21354380714880.0, + "grad_norm": 2.683211983951529, + "language_loss": 0.86005521, + "learning_rate": 3.94310902238026e-06, + "loss": 0.88149673, + "num_input_tokens_seen": 100782720, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.21618652, + "step": 3578, + "time_per_iteration": 4.666961669921875 + }, + { + "auxiliary_loss_clip": 0.01106688, + "auxiliary_loss_mlp": 0.01041969, + "balance_loss_clip": 1.03441954, + "balance_loss_mlp": 1.02116668, + "epoch": 0.103853519818931, + "flos": 66121482660480.0, + "grad_norm": 1.8099759677143548, + "language_loss": 0.86398405, + "learning_rate": 3.943064501444355e-06, + "loss": 0.88547063, + "num_input_tokens_seen": 100806760, + "router_z_loss_clip": 0.72216797, + "router_z_loss_mlp": 0.20800781, + "step": 3579, + "time_per_iteration": 2.703439235687256 + }, + { + "auxiliary_loss_clip": 0.01115633, + "auxiliary_loss_mlp": 0.01046885, + "balance_loss_clip": 1.0380218, + "balance_loss_mlp": 1.02384186, + "epoch": 0.10388253728744705, + "flos": 37632320455680.0, + "grad_norm": 2.5399604950926133, + "language_loss": 0.75658327, + "learning_rate": 3.9430199633465185e-06, + "loss": 0.77820849, + "num_input_tokens_seen": 100827400, + "router_z_loss_clip": 0.77539062, + "router_z_loss_mlp": 0.23034668, + "step": 3580, + "time_per_iteration": 2.551011800765991 + }, + { + "auxiliary_loss_clip": 0.01114862, + "auxiliary_loss_mlp": 0.01043378, + "balance_loss_clip": 1.0391593, + "balance_loss_mlp": 1.02165806, + "epoch": 0.10391155475596309, + "flos": 25619580270720.0, + "grad_norm": 2.7492783865062083, + "language_loss": 1.09559679, + "learning_rate": 3.942975408087144e-06, + "loss": 1.11717927, + "num_input_tokens_seen": 100840280, + "router_z_loss_clip": 0.75683594, + "router_z_loss_mlp": 0.2175293, + "step": 3581, + "time_per_iteration": 2.4525489807128906 + }, + { + "auxiliary_loss_clip": 0.01102621, + "auxiliary_loss_mlp": 0.01032906, + "balance_loss_clip": 1.0346508, + "balance_loss_mlp": 1.01441717, + "epoch": 0.10394057222447914, + "flos": 39959526418560.0, + "grad_norm": 2.180620540318514, + "language_loss": 0.80443203, + "learning_rate": 3.9429308356666235e-06, + "loss": 0.82578731, + "num_input_tokens_seen": 100862655, + "router_z_loss_clip": 0.6796875, + "router_z_loss_mlp": 0.18493652, + "step": 3582, + "time_per_iteration": 2.5884902477264404 + }, + { + "auxiliary_loss_clip": 0.01017179, + "auxiliary_loss_mlp": 0.01003496, + "balance_loss_clip": 1.00599754, + "balance_loss_mlp": 1.00244093, + "epoch": 0.10396958969299518, + "flos": 74775681187200.0, + "grad_norm": 0.6979430223731926, + "language_loss": 0.47237679, + "learning_rate": 3.942886246085352e-06, + "loss": 0.49258351, + "num_input_tokens_seen": 100925115, + "router_z_loss_clip": 0.11230469, + "router_z_loss_mlp": 0.01055908, + "step": 3583, + "time_per_iteration": 3.119891405105591 + }, + { + "auxiliary_loss_clip": 0.01104025, + "auxiliary_loss_mlp": 0.01040652, + "balance_loss_clip": 1.03321183, + "balance_loss_mlp": 1.01833653, + "epoch": 0.10399860716151123, + "flos": 17888727271680.0, + "grad_norm": 3.430661453713274, + "language_loss": 0.79386413, + "learning_rate": 3.942841639343723e-06, + "loss": 0.8153109, + "num_input_tokens_seen": 100938170, + "router_z_loss_clip": 0.70800781, + "router_z_loss_mlp": 0.22290039, + "step": 3584, + "time_per_iteration": 4.793038606643677 + }, + { + "auxiliary_loss_clip": 0.0101744, + "auxiliary_loss_mlp": 0.01003159, + "balance_loss_clip": 1.00646687, + "balance_loss_mlp": 1.00202096, + "epoch": 0.10402762463002728, + "flos": 57688280507520.0, + "grad_norm": 1.243738042903247, + "language_loss": 0.49055898, + "learning_rate": 3.942797015442131e-06, + "loss": 0.51076496, + "num_input_tokens_seen": 100991815, + "router_z_loss_clip": 0.10986328, + "router_z_loss_mlp": 0.01141357, + "step": 3585, + "time_per_iteration": 5.266366004943848 + }, + { + "auxiliary_loss_clip": 0.01105158, + "auxiliary_loss_mlp": 0.01041949, + "balance_loss_clip": 1.03319263, + "balance_loss_mlp": 1.02025306, + "epoch": 0.10405664209854332, + "flos": 15442712352000.0, + "grad_norm": 2.2791530280502688, + "language_loss": 0.76753056, + "learning_rate": 3.942752374380969e-06, + "loss": 0.78900158, + "num_input_tokens_seen": 101008430, + "router_z_loss_clip": 0.71972656, + "router_z_loss_mlp": 0.21704102, + "step": 3586, + "time_per_iteration": 2.4824330806732178 + }, + { + "auxiliary_loss_clip": 0.01016361, + "auxiliary_loss_mlp": 0.01002264, + "balance_loss_clip": 1.00522625, + "balance_loss_mlp": 1.00108349, + "epoch": 0.10408565956705937, + "flos": 64745448445440.0, + "grad_norm": 0.6177629695170932, + "language_loss": 0.48926443, + "learning_rate": 3.942707716160632e-06, + "loss": 0.50945067, + "num_input_tokens_seen": 101076560, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.01177979, + "step": 3587, + "time_per_iteration": 3.0728352069854736 + }, + { + "auxiliary_loss_clip": 0.01105353, + "auxiliary_loss_mlp": 0.0103683, + "balance_loss_clip": 1.03317714, + "balance_loss_mlp": 1.01655245, + "epoch": 0.10411467703557542, + "flos": 27373791133440.0, + "grad_norm": 1.8512517114454186, + "language_loss": 0.85611928, + "learning_rate": 3.942663040781514e-06, + "loss": 0.87754112, + "num_input_tokens_seen": 101095995, + "router_z_loss_clip": 0.72216797, + "router_z_loss_mlp": 0.20275879, + "step": 3588, + "time_per_iteration": 2.532973051071167 + }, + { + "auxiliary_loss_clip": 0.01096495, + "auxiliary_loss_mlp": 0.01035903, + "balance_loss_clip": 1.03170383, + "balance_loss_mlp": 1.01799786, + "epoch": 0.10414369450409146, + "flos": 37444383273600.0, + "grad_norm": 2.0873658314993997, + "language_loss": 0.79919076, + "learning_rate": 3.942618348244011e-06, + "loss": 0.8205148, + "num_input_tokens_seen": 101113395, + "router_z_loss_clip": 0.64794922, + "router_z_loss_mlp": 0.17907715, + "step": 3589, + "time_per_iteration": 2.528568983078003 + }, + { + "auxiliary_loss_clip": 0.01090813, + "auxiliary_loss_mlp": 0.0103643, + "balance_loss_clip": 1.02856112, + "balance_loss_mlp": 1.01907921, + "epoch": 0.10417271197260751, + "flos": 28396432033920.0, + "grad_norm": 2.042635468474376, + "language_loss": 0.77026951, + "learning_rate": 3.942573638548515e-06, + "loss": 0.79154199, + "num_input_tokens_seen": 101128915, + "router_z_loss_clip": 0.62304688, + "router_z_loss_mlp": 0.17352295, + "step": 3590, + "time_per_iteration": 2.4808926582336426 + }, + { + "auxiliary_loss_clip": 0.01100441, + "auxiliary_loss_mlp": 0.01040332, + "balance_loss_clip": 1.03305876, + "balance_loss_mlp": 1.0206511, + "epoch": 0.10420172944112356, + "flos": 27335875530240.0, + "grad_norm": 2.371446598102662, + "language_loss": 0.6807344, + "learning_rate": 3.9425289116954245e-06, + "loss": 0.70214212, + "num_input_tokens_seen": 101147695, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.19689941, + "step": 3591, + "time_per_iteration": 2.4441306591033936 + }, + { + "auxiliary_loss_clip": 0.01102268, + "auxiliary_loss_mlp": 0.01042147, + "balance_loss_clip": 1.0308404, + "balance_loss_mlp": 1.02276921, + "epoch": 0.1042307469096396, + "flos": 12926068018560.0, + "grad_norm": 2.2189264797392543, + "language_loss": 0.75214934, + "learning_rate": 3.942484167685131e-06, + "loss": 0.77359354, + "num_input_tokens_seen": 101160305, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.19378662, + "step": 3592, + "time_per_iteration": 2.382225275039673 + }, + { + "auxiliary_loss_clip": 0.01103419, + "auxiliary_loss_mlp": 0.01042372, + "balance_loss_clip": 1.03337526, + "balance_loss_mlp": 1.02205873, + "epoch": 0.10425976437815565, + "flos": 15151850881920.0, + "grad_norm": 2.6167033096820576, + "language_loss": 0.87788951, + "learning_rate": 3.9424394065180315e-06, + "loss": 0.89934742, + "num_input_tokens_seen": 101175425, + "router_z_loss_clip": 0.70068359, + "router_z_loss_mlp": 0.20324707, + "step": 3593, + "time_per_iteration": 2.445310115814209 + }, + { + "auxiliary_loss_clip": 0.01102999, + "auxiliary_loss_mlp": 0.01042414, + "balance_loss_clip": 1.03571141, + "balance_loss_mlp": 1.02278066, + "epoch": 0.1042887818466717, + "flos": 19086981644160.0, + "grad_norm": 4.033285262496092, + "language_loss": 1.04126883, + "learning_rate": 3.942394628194522e-06, + "loss": 1.06272292, + "num_input_tokens_seen": 101184420, + "router_z_loss_clip": 0.67285156, + "router_z_loss_mlp": 0.1963501, + "step": 3594, + "time_per_iteration": 2.4477179050445557 + }, + { + "auxiliary_loss_clip": 0.01014777, + "auxiliary_loss_mlp": 0.01004646, + "balance_loss_clip": 1.00318396, + "balance_loss_mlp": 1.0033468, + "epoch": 0.10431779931518774, + "flos": 67693095912960.0, + "grad_norm": 0.7536656731841987, + "language_loss": 0.53578085, + "learning_rate": 3.9423498327149965e-06, + "loss": 0.55597502, + "num_input_tokens_seen": 101248950, + "router_z_loss_clip": 0.11621094, + "router_z_loss_mlp": 0.01300049, + "step": 3595, + "time_per_iteration": 3.073000907897949 + }, + { + "auxiliary_loss_clip": 0.01109009, + "auxiliary_loss_mlp": 0.01039852, + "balance_loss_clip": 1.03412259, + "balance_loss_mlp": 1.01814365, + "epoch": 0.10434681678370379, + "flos": 29416978252800.0, + "grad_norm": 2.252519683920283, + "language_loss": 0.8979606, + "learning_rate": 3.942305020079852e-06, + "loss": 0.91944921, + "num_input_tokens_seen": 101266210, + "router_z_loss_clip": 0.74902344, + "router_z_loss_mlp": 0.21716309, + "step": 3596, + "time_per_iteration": 2.5213515758514404 + }, + { + "auxiliary_loss_clip": 0.01104377, + "auxiliary_loss_mlp": 0.01044811, + "balance_loss_clip": 1.03364253, + "balance_loss_mlp": 1.02651227, + "epoch": 0.10437583425221984, + "flos": 33685005628800.0, + "grad_norm": 2.740490546889822, + "language_loss": 0.80139369, + "learning_rate": 3.942260190289483e-06, + "loss": 0.82288563, + "num_input_tokens_seen": 101284895, + "router_z_loss_clip": 0.70678711, + "router_z_loss_mlp": 0.18304443, + "step": 3597, + "time_per_iteration": 2.485682249069214 + }, + { + "auxiliary_loss_clip": 0.01102186, + "auxiliary_loss_mlp": 0.01040005, + "balance_loss_clip": 1.03125167, + "balance_loss_mlp": 1.01902378, + "epoch": 0.10440485172073588, + "flos": 74731762696320.0, + "grad_norm": 2.2560972513065285, + "language_loss": 0.70289969, + "learning_rate": 3.9422153433442854e-06, + "loss": 0.7243216, + "num_input_tokens_seen": 101307575, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.20983887, + "step": 3598, + "time_per_iteration": 2.802116870880127 + }, + { + "auxiliary_loss_clip": 0.01014246, + "auxiliary_loss_mlp": 0.01001887, + "balance_loss_clip": 1.00329268, + "balance_loss_mlp": 1.00069451, + "epoch": 0.10443386918925193, + "flos": 69537174215040.0, + "grad_norm": 0.8492973450568202, + "language_loss": 0.50472462, + "learning_rate": 3.9421704792446565e-06, + "loss": 0.52488595, + "num_input_tokens_seen": 101364095, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01190186, + "step": 3599, + "time_per_iteration": 2.947714328765869 + }, + { + "auxiliary_loss_clip": 0.01109377, + "auxiliary_loss_mlp": 0.01051404, + "balance_loss_clip": 1.03498983, + "balance_loss_mlp": 1.02751446, + "epoch": 0.10446288665776797, + "flos": 34744060944000.0, + "grad_norm": 3.3709077368070304, + "language_loss": 0.88682514, + "learning_rate": 3.9421255979909925e-06, + "loss": 0.90843296, + "num_input_tokens_seen": 101381965, + "router_z_loss_clip": 0.74438477, + "router_z_loss_mlp": 0.23913574, + "step": 3600, + "time_per_iteration": 2.5791702270507812 + }, + { + "auxiliary_loss_clip": 0.01106958, + "auxiliary_loss_mlp": 0.01045144, + "balance_loss_clip": 1.03318346, + "balance_loss_mlp": 1.02239847, + "epoch": 0.10449190412628402, + "flos": 25512606264960.0, + "grad_norm": 2.61759860136843, + "language_loss": 0.90407044, + "learning_rate": 3.942080699583689e-06, + "loss": 0.92559147, + "num_input_tokens_seen": 101401785, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.22741699, + "step": 3601, + "time_per_iteration": 2.5365066528320312 + }, + { + "auxiliary_loss_clip": 0.01012406, + "auxiliary_loss_mlp": 0.01000829, + "balance_loss_clip": 1.00142193, + "balance_loss_mlp": 0.99966049, + "epoch": 0.10452092159480007, + "flos": 58710607205760.0, + "grad_norm": 0.7464775187249914, + "language_loss": 0.48622844, + "learning_rate": 3.9420357840231425e-06, + "loss": 0.50636083, + "num_input_tokens_seen": 101460735, + "router_z_loss_clip": 0.10986328, + "router_z_loss_mlp": 0.01165771, + "step": 3602, + "time_per_iteration": 3.107257127761841 + }, + { + "auxiliary_loss_clip": 0.01012334, + "auxiliary_loss_mlp": 0.01001208, + "balance_loss_clip": 1.0013938, + "balance_loss_mlp": 1.00007534, + "epoch": 0.10454993906331611, + "flos": 74778369361920.0, + "grad_norm": 0.6718717991839542, + "language_loss": 0.47296694, + "learning_rate": 3.94199085130975e-06, + "loss": 0.49310234, + "num_input_tokens_seen": 101530365, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01135254, + "step": 3603, + "time_per_iteration": 3.0900728702545166 + }, + { + "auxiliary_loss_clip": 0.01011821, + "auxiliary_loss_mlp": 0.01000313, + "balance_loss_clip": 1.0012362, + "balance_loss_mlp": 0.99914479, + "epoch": 0.10457895653183216, + "flos": 66523365987840.0, + "grad_norm": 0.6135333034282499, + "language_loss": 0.44693494, + "learning_rate": 3.9419459014439095e-06, + "loss": 0.46705624, + "num_input_tokens_seen": 101593505, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01165771, + "step": 3604, + "time_per_iteration": 3.0271902084350586 + }, + { + "auxiliary_loss_clip": 0.01102961, + "auxiliary_loss_mlp": 0.01040273, + "balance_loss_clip": 1.0327661, + "balance_loss_mlp": 1.02050853, + "epoch": 0.10460797400034821, + "flos": 58349641858560.0, + "grad_norm": 1.9398214523988646, + "language_loss": 0.62189126, + "learning_rate": 3.941900934426017e-06, + "loss": 0.64332354, + "num_input_tokens_seen": 101615285, + "router_z_loss_clip": 0.70214844, + "router_z_loss_mlp": 0.19763184, + "step": 3605, + "time_per_iteration": 2.6365973949432373 + }, + { + "auxiliary_loss_clip": 0.01105451, + "auxiliary_loss_mlp": 0.01040358, + "balance_loss_clip": 1.03402495, + "balance_loss_mlp": 1.01872182, + "epoch": 0.10463699146886425, + "flos": 30292320660480.0, + "grad_norm": 1.8739982720384292, + "language_loss": 0.71720576, + "learning_rate": 3.941855950256468e-06, + "loss": 0.73866391, + "num_input_tokens_seen": 101631805, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.21643066, + "step": 3606, + "time_per_iteration": 2.5017778873443604 + }, + { + "auxiliary_loss_clip": 0.01102974, + "auxiliary_loss_mlp": 0.01037202, + "balance_loss_clip": 1.03434885, + "balance_loss_mlp": 1.01999402, + "epoch": 0.1046660089373803, + "flos": 13071830411520.0, + "grad_norm": 2.117272352399885, + "language_loss": 0.62383604, + "learning_rate": 3.941810948935663e-06, + "loss": 0.6452378, + "num_input_tokens_seen": 101644270, + "router_z_loss_clip": 0.68652344, + "router_z_loss_mlp": 0.17230225, + "step": 3607, + "time_per_iteration": 2.4382829666137695 + }, + { + "auxiliary_loss_clip": 0.01104705, + "auxiliary_loss_mlp": 0.01046534, + "balance_loss_clip": 1.03409541, + "balance_loss_mlp": 1.02454591, + "epoch": 0.10469502640589635, + "flos": 54626713539840.0, + "grad_norm": 2.260477027620164, + "language_loss": 0.71931869, + "learning_rate": 3.941765930463997e-06, + "loss": 0.74083114, + "num_input_tokens_seen": 101664870, + "router_z_loss_clip": 0.70507812, + "router_z_loss_mlp": 0.21984863, + "step": 3608, + "time_per_iteration": 2.6878175735473633 + }, + { + "auxiliary_loss_clip": 0.01107774, + "auxiliary_loss_mlp": 0.01048323, + "balance_loss_clip": 1.03471982, + "balance_loss_mlp": 1.02465391, + "epoch": 0.10472404387441239, + "flos": 70243782554880.0, + "grad_norm": 2.3537929786516396, + "language_loss": 0.87141991, + "learning_rate": 3.941720894841869e-06, + "loss": 0.89298081, + "num_input_tokens_seen": 101688945, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.23669434, + "step": 3609, + "time_per_iteration": 2.7984538078308105 + }, + { + "auxiliary_loss_clip": 0.01103272, + "auxiliary_loss_mlp": 0.0103572, + "balance_loss_clip": 1.03129983, + "balance_loss_mlp": 1.01596153, + "epoch": 0.10475306134292844, + "flos": 43355562877440.0, + "grad_norm": 8.084485020381479, + "language_loss": 0.75025964, + "learning_rate": 3.941675842069676e-06, + "loss": 0.7716496, + "num_input_tokens_seen": 101709115, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.19781494, + "step": 3610, + "time_per_iteration": 2.605419874191284 + }, + { + "auxiliary_loss_clip": 0.01105034, + "auxiliary_loss_mlp": 0.01044841, + "balance_loss_clip": 1.0329355, + "balance_loss_mlp": 1.02324069, + "epoch": 0.1047820788114445, + "flos": 29162286817920.0, + "grad_norm": 2.2182929508472156, + "language_loss": 0.95544034, + "learning_rate": 3.9416307721478165e-06, + "loss": 0.97693908, + "num_input_tokens_seen": 101728005, + "router_z_loss_clip": 0.72216797, + "router_z_loss_mlp": 0.21582031, + "step": 3611, + "time_per_iteration": 2.498008966445923 + }, + { + "auxiliary_loss_clip": 0.01109269, + "auxiliary_loss_mlp": 0.01043064, + "balance_loss_clip": 1.03502202, + "balance_loss_mlp": 1.02171397, + "epoch": 0.10481109627996053, + "flos": 27081009538560.0, + "grad_norm": 2.7293734755954024, + "language_loss": 1.00882983, + "learning_rate": 3.941585685076689e-06, + "loss": 1.03035319, + "num_input_tokens_seen": 101743045, + "router_z_loss_clip": 0.74243164, + "router_z_loss_mlp": 0.21337891, + "step": 3612, + "time_per_iteration": 2.451763391494751 + }, + { + "auxiliary_loss_clip": 0.01017678, + "auxiliary_loss_mlp": 0.01001897, + "balance_loss_clip": 1.00670457, + "balance_loss_mlp": 1.00074029, + "epoch": 0.10484011374847658, + "flos": 64009130538240.0, + "grad_norm": 0.844958902812242, + "language_loss": 0.48991001, + "learning_rate": 3.9415405808566905e-06, + "loss": 0.51010573, + "num_input_tokens_seen": 101807645, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01153564, + "step": 3613, + "time_per_iteration": 3.0620932579040527 + }, + { + "auxiliary_loss_clip": 0.01102239, + "auxiliary_loss_mlp": 0.01038473, + "balance_loss_clip": 1.03604531, + "balance_loss_mlp": 1.01972175, + "epoch": 0.10486913121699262, + "flos": 15699184266240.0, + "grad_norm": 2.035311003895027, + "language_loss": 0.61223054, + "learning_rate": 3.94149545948822e-06, + "loss": 0.63363767, + "num_input_tokens_seen": 101821330, + "router_z_loss_clip": 0.66210938, + "router_z_loss_mlp": 0.1875, + "step": 3614, + "time_per_iteration": 2.4825587272644043 + }, + { + "auxiliary_loss_clip": 0.01098762, + "auxiliary_loss_mlp": 0.01040454, + "balance_loss_clip": 1.03229427, + "balance_loss_mlp": 1.0203557, + "epoch": 0.10489814868550867, + "flos": 33869312029440.0, + "grad_norm": 2.3877305245094633, + "language_loss": 0.7593388, + "learning_rate": 3.941450320971675e-06, + "loss": 0.78073096, + "num_input_tokens_seen": 101838860, + "router_z_loss_clip": 0.66381836, + "router_z_loss_mlp": 0.2010498, + "step": 3615, + "time_per_iteration": 2.5416414737701416 + }, + { + "auxiliary_loss_clip": 0.01107815, + "auxiliary_loss_mlp": 0.01046423, + "balance_loss_clip": 1.03322887, + "balance_loss_mlp": 1.02211618, + "epoch": 0.10492716615402473, + "flos": 32808022387200.0, + "grad_norm": 2.687875919364466, + "language_loss": 0.88792467, + "learning_rate": 3.941405165307456e-06, + "loss": 0.90946716, + "num_input_tokens_seen": 101854245, + "router_z_loss_clip": 0.74560547, + "router_z_loss_mlp": 0.24291992, + "step": 3616, + "time_per_iteration": 2.652904510498047 + }, + { + "auxiliary_loss_clip": 0.01014899, + "auxiliary_loss_mlp": 0.01002034, + "balance_loss_clip": 1.00394404, + "balance_loss_mlp": 1.00081825, + "epoch": 0.10495618362254076, + "flos": 67367425950720.0, + "grad_norm": 0.6753935834688587, + "language_loss": 0.51402521, + "learning_rate": 3.941359992495961e-06, + "loss": 0.53419459, + "num_input_tokens_seen": 101915255, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.012146, + "step": 3617, + "time_per_iteration": 2.9827871322631836 + }, + { + "auxiliary_loss_clip": 0.01108799, + "auxiliary_loss_mlp": 0.01044083, + "balance_loss_clip": 1.03637552, + "balance_loss_mlp": 1.02289963, + "epoch": 0.10498520109105682, + "flos": 25370335008000.0, + "grad_norm": 1.8422926854208752, + "language_loss": 0.7355203, + "learning_rate": 3.941314802537589e-06, + "loss": 0.75704914, + "num_input_tokens_seen": 101934760, + "router_z_loss_clip": 0.72412109, + "router_z_loss_mlp": 0.21166992, + "step": 3618, + "time_per_iteration": 2.489325761795044 + }, + { + "auxiliary_loss_clip": 0.0101323, + "auxiliary_loss_mlp": 0.01005711, + "balance_loss_clip": 1.00212765, + "balance_loss_mlp": 1.0044651, + "epoch": 0.10501421855957287, + "flos": 68568927079680.0, + "grad_norm": 0.7022234236476611, + "language_loss": 0.48817855, + "learning_rate": 3.941269595432739e-06, + "loss": 0.50836796, + "num_input_tokens_seen": 101995855, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.01245117, + "step": 3619, + "time_per_iteration": 3.0172197818756104 + }, + { + "auxiliary_loss_clip": 0.01106771, + "auxiliary_loss_mlp": 0.01039214, + "balance_loss_clip": 1.03278005, + "balance_loss_mlp": 1.01761317, + "epoch": 0.1050432360280889, + "flos": 14896112106240.0, + "grad_norm": 2.8858466254840085, + "language_loss": 0.71671522, + "learning_rate": 3.941224371181811e-06, + "loss": 0.73817503, + "num_input_tokens_seen": 102009510, + "router_z_loss_clip": 0.74023438, + "router_z_loss_mlp": 0.21606445, + "step": 3620, + "time_per_iteration": 2.4131979942321777 + }, + { + "auxiliary_loss_clip": 0.01101161, + "auxiliary_loss_mlp": 0.01046212, + "balance_loss_clip": 1.03181708, + "balance_loss_mlp": 1.0250349, + "epoch": 0.10507225349660496, + "flos": 13363599576960.0, + "grad_norm": 3.1641555076980263, + "language_loss": 0.70560086, + "learning_rate": 3.9411791297852026e-06, + "loss": 0.72707456, + "num_input_tokens_seen": 102022060, + "router_z_loss_clip": 0.69335938, + "router_z_loss_mlp": 0.21173096, + "step": 3621, + "time_per_iteration": 2.3370935916900635 + }, + { + "auxiliary_loss_clip": 0.01098844, + "auxiliary_loss_mlp": 0.01040725, + "balance_loss_clip": 1.03251636, + "balance_loss_mlp": 1.02235532, + "epoch": 0.10510127096512101, + "flos": 45871613717760.0, + "grad_norm": 2.502845462315552, + "language_loss": 0.73263234, + "learning_rate": 3.941133871243315e-06, + "loss": 0.75402796, + "num_input_tokens_seen": 102040600, + "router_z_loss_clip": 0.6640625, + "router_z_loss_mlp": 0.18371582, + "step": 3622, + "time_per_iteration": 2.597456693649292 + }, + { + "auxiliary_loss_clip": 0.01105148, + "auxiliary_loss_mlp": 0.01049092, + "balance_loss_clip": 1.0328536, + "balance_loss_mlp": 1.02730036, + "epoch": 0.10513028843363705, + "flos": 22412179221120.0, + "grad_norm": 2.615145731632477, + "language_loss": 0.81636417, + "learning_rate": 3.941088595556548e-06, + "loss": 0.8379066, + "num_input_tokens_seen": 102057220, + "router_z_loss_clip": 0.72314453, + "router_z_loss_mlp": 0.21777344, + "step": 3623, + "time_per_iteration": 2.4003961086273193 + }, + { + "auxiliary_loss_clip": 0.01012089, + "auxiliary_loss_mlp": 0.01001247, + "balance_loss_clip": 1.00125551, + "balance_loss_mlp": 1.0000608, + "epoch": 0.1051593059021531, + "flos": 50576716938240.0, + "grad_norm": 0.7109918539516952, + "language_loss": 0.50019914, + "learning_rate": 3.9410433027253005e-06, + "loss": 0.52033246, + "num_input_tokens_seen": 102105840, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01184082, + "step": 3624, + "time_per_iteration": 2.839620351791382 + }, + { + "auxiliary_loss_clip": 0.01011994, + "auxiliary_loss_mlp": 0.01003285, + "balance_loss_clip": 1.00114107, + "balance_loss_mlp": 1.00194347, + "epoch": 0.10518832337066915, + "flos": 67397416675200.0, + "grad_norm": 0.6886646791200841, + "language_loss": 0.48976859, + "learning_rate": 3.9409979927499735e-06, + "loss": 0.50992137, + "num_input_tokens_seen": 102166410, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01342773, + "step": 3625, + "time_per_iteration": 3.05059552192688 + }, + { + "auxiliary_loss_clip": 0.01099914, + "auxiliary_loss_mlp": 0.01041334, + "balance_loss_clip": 1.03389645, + "balance_loss_mlp": 1.02307153, + "epoch": 0.10521734083918519, + "flos": 21610014756480.0, + "grad_norm": 33.34488656558145, + "language_loss": 0.84412509, + "learning_rate": 3.9409526656309665e-06, + "loss": 0.86553752, + "num_input_tokens_seen": 102181115, + "router_z_loss_clip": 0.66015625, + "router_z_loss_mlp": 0.18280029, + "step": 3626, + "time_per_iteration": 2.419398546218872 + }, + { + "auxiliary_loss_clip": 0.01100578, + "auxiliary_loss_mlp": 0.01048773, + "balance_loss_clip": 1.03245568, + "balance_loss_mlp": 1.02894866, + "epoch": 0.10524635830770124, + "flos": 15844318254720.0, + "grad_norm": 2.80389632016729, + "language_loss": 0.91795838, + "learning_rate": 3.94090732136868e-06, + "loss": 0.93945187, + "num_input_tokens_seen": 102192920, + "router_z_loss_clip": 0.68066406, + "router_z_loss_mlp": 0.19812012, + "step": 3627, + "time_per_iteration": 2.4000120162963867 + }, + { + "auxiliary_loss_clip": 0.01110162, + "auxiliary_loss_mlp": 0.01048121, + "balance_loss_clip": 1.03509951, + "balance_loss_mlp": 1.0263176, + "epoch": 0.10527537577621729, + "flos": 10699970952960.0, + "grad_norm": 6.281481859388844, + "language_loss": 0.86492157, + "learning_rate": 3.940861959963516e-06, + "loss": 0.88650441, + "num_input_tokens_seen": 102206195, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.21838379, + "step": 3628, + "time_per_iteration": 2.3617701530456543 + }, + { + "auxiliary_loss_clip": 0.01011699, + "auxiliary_loss_mlp": 0.01003503, + "balance_loss_clip": 1.00071669, + "balance_loss_mlp": 1.00228107, + "epoch": 0.10530439324473333, + "flos": 58465900419840.0, + "grad_norm": 0.6406199373353664, + "language_loss": 0.51469076, + "learning_rate": 3.940816581415872e-06, + "loss": 0.53484285, + "num_input_tokens_seen": 102266180, + "router_z_loss_clip": 0.10986328, + "router_z_loss_mlp": 0.01220703, + "step": 3629, + "time_per_iteration": 2.89958119392395 + }, + { + "auxiliary_loss_clip": 0.01106711, + "auxiliary_loss_mlp": 0.01052229, + "balance_loss_clip": 1.03398669, + "balance_loss_mlp": 1.02947187, + "epoch": 0.10533341071324938, + "flos": 33940185822720.0, + "grad_norm": 2.6195772206641217, + "language_loss": 0.86378497, + "learning_rate": 3.940771185726152e-06, + "loss": 0.88537443, + "num_input_tokens_seen": 102283595, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.22729492, + "step": 3630, + "time_per_iteration": 2.5102741718292236 + }, + { + "auxiliary_loss_clip": 0.01104578, + "auxiliary_loss_mlp": 0.01049184, + "balance_loss_clip": 1.03243303, + "balance_loss_mlp": 1.02701139, + "epoch": 0.10536242818176542, + "flos": 12779293196160.0, + "grad_norm": 3.602762696580264, + "language_loss": 1.01147377, + "learning_rate": 3.940725772894754e-06, + "loss": 1.03301144, + "num_input_tokens_seen": 102294810, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.22180176, + "step": 3631, + "time_per_iteration": 2.316710948944092 + }, + { + "auxiliary_loss_clip": 0.0110632, + "auxiliary_loss_mlp": 0.01048284, + "balance_loss_clip": 1.03502941, + "balance_loss_mlp": 1.02758944, + "epoch": 0.10539144565028147, + "flos": 22522260337920.0, + "grad_norm": 4.2805870522105405, + "language_loss": 0.87811899, + "learning_rate": 3.940680342922081e-06, + "loss": 0.899665, + "num_input_tokens_seen": 102309360, + "router_z_loss_clip": 0.71386719, + "router_z_loss_mlp": 0.20678711, + "step": 3632, + "time_per_iteration": 2.4182791709899902 + }, + { + "auxiliary_loss_clip": 0.01012495, + "auxiliary_loss_mlp": 0.01003335, + "balance_loss_clip": 1.00154233, + "balance_loss_mlp": 1.00204182, + "epoch": 0.10542046311879752, + "flos": 70437129131520.0, + "grad_norm": 0.6685283400499157, + "language_loss": 0.47936243, + "learning_rate": 3.9406348958085345e-06, + "loss": 0.49952072, + "num_input_tokens_seen": 102372010, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01293945, + "step": 3633, + "time_per_iteration": 3.0566763877868652 + }, + { + "auxiliary_loss_clip": 0.01096625, + "auxiliary_loss_mlp": 0.01035453, + "balance_loss_clip": 1.02962828, + "balance_loss_mlp": 1.01812553, + "epoch": 0.10544948058731356, + "flos": 42701255487360.0, + "grad_norm": 2.1342507028381656, + "language_loss": 0.71496791, + "learning_rate": 3.9405894315545155e-06, + "loss": 0.73628867, + "num_input_tokens_seen": 102391550, + "router_z_loss_clip": 0.67138672, + "router_z_loss_mlp": 0.17327881, + "step": 3634, + "time_per_iteration": 2.618180513381958 + }, + { + "auxiliary_loss_clip": 0.01105694, + "auxiliary_loss_mlp": 0.01043671, + "balance_loss_clip": 1.03326499, + "balance_loss_mlp": 1.02193272, + "epoch": 0.10547849805582961, + "flos": 33215633043840.0, + "grad_norm": 2.28479779315373, + "language_loss": 0.9201324, + "learning_rate": 3.940543950160426e-06, + "loss": 0.94162601, + "num_input_tokens_seen": 102408190, + "router_z_loss_clip": 0.72558594, + "router_z_loss_mlp": 0.21734619, + "step": 3635, + "time_per_iteration": 2.5596604347229004 + }, + { + "auxiliary_loss_clip": 0.01099507, + "auxiliary_loss_mlp": 0.01041296, + "balance_loss_clip": 1.03218842, + "balance_loss_mlp": 1.02022552, + "epoch": 0.10550751552434566, + "flos": 46162579921920.0, + "grad_norm": 1.9240229745386839, + "language_loss": 0.87371337, + "learning_rate": 3.940498451626666e-06, + "loss": 0.89512146, + "num_input_tokens_seen": 102432195, + "router_z_loss_clip": 0.67285156, + "router_z_loss_mlp": 0.21063232, + "step": 3636, + "time_per_iteration": 2.6271450519561768 + }, + { + "auxiliary_loss_clip": 0.010133, + "auxiliary_loss_mlp": 0.01001326, + "balance_loss_clip": 1.00256705, + "balance_loss_mlp": 1.0002172, + "epoch": 0.1055365329928617, + "flos": 58819012577280.0, + "grad_norm": 0.6832495431144995, + "language_loss": 0.46656287, + "learning_rate": 3.940452935953639e-06, + "loss": 0.48670912, + "num_input_tokens_seen": 102488430, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.0111084, + "step": 3637, + "time_per_iteration": 2.8312079906463623 + }, + { + "auxiliary_loss_clip": 0.01105411, + "auxiliary_loss_mlp": 0.0104405, + "balance_loss_clip": 1.03489208, + "balance_loss_mlp": 1.02362955, + "epoch": 0.10556555046137775, + "flos": 74730785178240.0, + "grad_norm": 2.0898756870959643, + "language_loss": 0.84807158, + "learning_rate": 3.940407403141745e-06, + "loss": 0.86956614, + "num_input_tokens_seen": 102512410, + "router_z_loss_clip": 0.70556641, + "router_z_loss_mlp": 0.2043457, + "step": 3638, + "time_per_iteration": 2.7713735103607178 + }, + { + "auxiliary_loss_clip": 0.01097434, + "auxiliary_loss_mlp": 0.01041347, + "balance_loss_clip": 1.02981138, + "balance_loss_mlp": 1.0222261, + "epoch": 0.1055945679298938, + "flos": 36722518669440.0, + "grad_norm": 2.515577510774472, + "language_loss": 0.87628597, + "learning_rate": 3.940361853191389e-06, + "loss": 0.89767373, + "num_input_tokens_seen": 102534150, + "router_z_loss_clip": 0.67626953, + "router_z_loss_mlp": 0.19116211, + "step": 3639, + "time_per_iteration": 2.595547914505005 + }, + { + "auxiliary_loss_clip": 0.0110281, + "auxiliary_loss_mlp": 0.01041995, + "balance_loss_clip": 1.03017926, + "balance_loss_mlp": 1.02052569, + "epoch": 0.10562358539840984, + "flos": 34964013709440.0, + "grad_norm": 1.9651952769117946, + "language_loss": 0.93719572, + "learning_rate": 3.9403162861029715e-06, + "loss": 0.95864373, + "num_input_tokens_seen": 102556800, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.21496582, + "step": 3640, + "time_per_iteration": 2.5818021297454834 + }, + { + "auxiliary_loss_clip": 0.01101567, + "auxiliary_loss_mlp": 0.01037296, + "balance_loss_clip": 1.03062284, + "balance_loss_mlp": 1.01728034, + "epoch": 0.10565260286692589, + "flos": 15407415100800.0, + "grad_norm": 2.323949530064403, + "language_loss": 0.84762716, + "learning_rate": 3.940270701876896e-06, + "loss": 0.86901581, + "num_input_tokens_seen": 102570985, + "router_z_loss_clip": 0.70996094, + "router_z_loss_mlp": 0.20001221, + "step": 3641, + "time_per_iteration": 2.402644634246826 + }, + { + "auxiliary_loss_clip": 0.01106119, + "auxiliary_loss_mlp": 0.01042589, + "balance_loss_clip": 1.03351068, + "balance_loss_mlp": 1.02085137, + "epoch": 0.10568162033544194, + "flos": 40216522003200.0, + "grad_norm": 1.9781055606511015, + "language_loss": 0.751149, + "learning_rate": 3.940225100513564e-06, + "loss": 0.77263606, + "num_input_tokens_seen": 102593440, + "router_z_loss_clip": 0.72607422, + "router_z_loss_mlp": 0.21716309, + "step": 3642, + "time_per_iteration": 2.5605244636535645 + }, + { + "auxiliary_loss_clip": 0.01014292, + "auxiliary_loss_mlp": 0.01002783, + "balance_loss_clip": 1.00351512, + "balance_loss_mlp": 1.00167406, + "epoch": 0.10571063780395798, + "flos": 74771806026240.0, + "grad_norm": 0.7067722645869067, + "language_loss": 0.47712103, + "learning_rate": 3.940179482013378e-06, + "loss": 0.49729174, + "num_input_tokens_seen": 102652835, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.0111084, + "step": 3643, + "time_per_iteration": 3.0961575508117676 + }, + { + "auxiliary_loss_clip": 0.01014394, + "auxiliary_loss_mlp": 0.01001488, + "balance_loss_clip": 1.00339448, + "balance_loss_mlp": 1.00015295, + "epoch": 0.10573965527247403, + "flos": 61448321468160.0, + "grad_norm": 0.6938558569682767, + "language_loss": 0.50274879, + "learning_rate": 3.940133846376742e-06, + "loss": 0.52290761, + "num_input_tokens_seen": 102710055, + "router_z_loss_clip": 0.10986328, + "router_z_loss_mlp": 0.0133667, + "step": 3644, + "time_per_iteration": 2.909332036972046 + }, + { + "auxiliary_loss_clip": 0.01013658, + "auxiliary_loss_mlp": 0.01002408, + "balance_loss_clip": 1.00251758, + "balance_loss_mlp": 1.00110292, + "epoch": 0.10576867274099008, + "flos": 74776379414400.0, + "grad_norm": 0.6755098986066799, + "language_loss": 0.52084637, + "learning_rate": 3.94008819360406e-06, + "loss": 0.54100704, + "num_input_tokens_seen": 102775945, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.01306152, + "step": 3645, + "time_per_iteration": 3.0828311443328857 + }, + { + "auxiliary_loss_clip": 0.01106314, + "auxiliary_loss_mlp": 0.01042151, + "balance_loss_clip": 1.03300524, + "balance_loss_mlp": 1.02032387, + "epoch": 0.10579769020950612, + "flos": 21316534934400.0, + "grad_norm": 2.4449665581664815, + "language_loss": 0.8709954, + "learning_rate": 3.940042523695733e-06, + "loss": 0.89248002, + "num_input_tokens_seen": 102790230, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.21826172, + "step": 3646, + "time_per_iteration": 2.422440528869629 + }, + { + "auxiliary_loss_clip": 0.01101268, + "auxiliary_loss_mlp": 0.01041015, + "balance_loss_clip": 1.03177035, + "balance_loss_mlp": 1.02271616, + "epoch": 0.10582670767802217, + "flos": 24271653432960.0, + "grad_norm": 2.437160679475157, + "language_loss": 0.74260986, + "learning_rate": 3.939996836652166e-06, + "loss": 0.76403272, + "num_input_tokens_seen": 102804170, + "router_z_loss_clip": 0.69580078, + "router_z_loss_mlp": 0.18310547, + "step": 3647, + "time_per_iteration": 2.430814743041992 + }, + { + "auxiliary_loss_clip": 0.01103038, + "auxiliary_loss_mlp": 0.01038661, + "balance_loss_clip": 1.03367317, + "balance_loss_mlp": 1.01856232, + "epoch": 0.10585572514653821, + "flos": 17669263265280.0, + "grad_norm": 1.8064382580859888, + "language_loss": 0.76677895, + "learning_rate": 3.939951132473761e-06, + "loss": 0.78819591, + "num_input_tokens_seen": 102819415, + "router_z_loss_clip": 0.69384766, + "router_z_loss_mlp": 0.20117188, + "step": 3648, + "time_per_iteration": 2.3829567432403564 + }, + { + "auxiliary_loss_clip": 0.01102134, + "auxiliary_loss_mlp": 0.01037236, + "balance_loss_clip": 1.0313127, + "balance_loss_mlp": 1.015993, + "epoch": 0.10588474261505426, + "flos": 29817851016960.0, + "grad_norm": 2.0814469527662776, + "language_loss": 1.0522716, + "learning_rate": 3.939905411160923e-06, + "loss": 1.07366538, + "num_input_tokens_seen": 102837610, + "router_z_loss_clip": 0.70800781, + "router_z_loss_mlp": 0.21276855, + "step": 3649, + "time_per_iteration": 2.4786064624786377 + }, + { + "auxiliary_loss_clip": 0.01100968, + "auxiliary_loss_mlp": 0.01042915, + "balance_loss_clip": 1.03074455, + "balance_loss_mlp": 1.02347183, + "epoch": 0.10591376008357031, + "flos": 29124999619200.0, + "grad_norm": 2.0376560845387828, + "language_loss": 0.91832858, + "learning_rate": 3.939859672714056e-06, + "loss": 0.93976742, + "num_input_tokens_seen": 102854605, + "router_z_loss_clip": 0.70288086, + "router_z_loss_mlp": 0.19464111, + "step": 3650, + "time_per_iteration": 2.4471182823181152 + }, + { + "auxiliary_loss_clip": 0.0110319, + "auxiliary_loss_mlp": 0.0103591, + "balance_loss_clip": 1.0346396, + "balance_loss_mlp": 1.01668096, + "epoch": 0.10594277755208635, + "flos": 27664687514880.0, + "grad_norm": 2.5188761228675833, + "language_loss": 0.91452014, + "learning_rate": 3.939813917133563e-06, + "loss": 0.93591112, + "num_input_tokens_seen": 102869930, + "router_z_loss_clip": 0.68554688, + "router_z_loss_mlp": 0.19250488, + "step": 3651, + "time_per_iteration": 2.4664816856384277 + }, + { + "auxiliary_loss_clip": 0.01100475, + "auxiliary_loss_mlp": 0.01038454, + "balance_loss_clip": 1.03221965, + "balance_loss_mlp": 1.01953602, + "epoch": 0.1059717950206024, + "flos": 38390878765440.0, + "grad_norm": 2.8579837421695364, + "language_loss": 0.99990863, + "learning_rate": 3.939768144419848e-06, + "loss": 1.02129793, + "num_input_tokens_seen": 102884440, + "router_z_loss_clip": 0.68310547, + "router_z_loss_mlp": 0.18933105, + "step": 3652, + "time_per_iteration": 2.532029867172241 + }, + { + "auxiliary_loss_clip": 0.01092829, + "auxiliary_loss_mlp": 0.01038451, + "balance_loss_clip": 1.02790928, + "balance_loss_mlp": 1.01943755, + "epoch": 0.10600081248911845, + "flos": 21206942576640.0, + "grad_norm": 3.4124254166484675, + "language_loss": 0.7534138, + "learning_rate": 3.939722354573317e-06, + "loss": 0.77472663, + "num_input_tokens_seen": 102897590, + "router_z_loss_clip": 0.64892578, + "router_z_loss_mlp": 0.19018555, + "step": 3653, + "time_per_iteration": 4.814787864685059 + }, + { + "auxiliary_loss_clip": 0.01012811, + "auxiliary_loss_mlp": 0.01005357, + "balance_loss_clip": 1.00167716, + "balance_loss_mlp": 1.00407553, + "epoch": 0.10602982995763449, + "flos": 59773642416000.0, + "grad_norm": 0.6714768534623762, + "language_loss": 0.49627355, + "learning_rate": 3.939676547594373e-06, + "loss": 0.51645523, + "num_input_tokens_seen": 102960385, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.01281738, + "step": 3654, + "time_per_iteration": 3.1066460609436035 + }, + { + "auxiliary_loss_clip": 0.01098375, + "auxiliary_loss_mlp": 0.01043779, + "balance_loss_clip": 1.03096259, + "balance_loss_mlp": 1.02565885, + "epoch": 0.10605884742615054, + "flos": 15302151751680.0, + "grad_norm": 2.3066254652454106, + "language_loss": 0.55374807, + "learning_rate": 3.93963072348342e-06, + "loss": 0.57516956, + "num_input_tokens_seen": 102976680, + "router_z_loss_clip": 0.67431641, + "router_z_loss_mlp": 0.18109131, + "step": 3655, + "time_per_iteration": 2.426767587661743 + }, + { + "auxiliary_loss_clip": 0.01013143, + "auxiliary_loss_mlp": 0.01004075, + "balance_loss_clip": 1.00192761, + "balance_loss_mlp": 1.00284696, + "epoch": 0.1060878648946666, + "flos": 57544542973440.0, + "grad_norm": 0.724211822559959, + "language_loss": 0.51331246, + "learning_rate": 3.9395848822408635e-06, + "loss": 0.53348464, + "num_input_tokens_seen": 103032610, + "router_z_loss_clip": 0.11230469, + "router_z_loss_mlp": 0.01226807, + "step": 3656, + "time_per_iteration": 2.9063351154327393 + }, + { + "auxiliary_loss_clip": 0.01101903, + "auxiliary_loss_mlp": 0.01039651, + "balance_loss_clip": 1.03266144, + "balance_loss_mlp": 1.02016068, + "epoch": 0.10611688236318263, + "flos": 34159754563200.0, + "grad_norm": 1.7442856036966197, + "language_loss": 0.73427492, + "learning_rate": 3.939539023867109e-06, + "loss": 0.75569046, + "num_input_tokens_seen": 103051910, + "router_z_loss_clip": 0.69287109, + "router_z_loss_mlp": 0.19494629, + "step": 3657, + "time_per_iteration": 2.5699431896209717 + }, + { + "auxiliary_loss_clip": 0.01097795, + "auxiliary_loss_mlp": 0.01033679, + "balance_loss_clip": 1.03354883, + "balance_loss_mlp": 1.01633394, + "epoch": 0.10614589983169868, + "flos": 26934758386560.0, + "grad_norm": 2.563180209070422, + "language_loss": 0.85864675, + "learning_rate": 3.939493148362561e-06, + "loss": 0.87996155, + "num_input_tokens_seen": 103065945, + "router_z_loss_clip": 0.64355469, + "router_z_loss_mlp": 0.17346191, + "step": 3658, + "time_per_iteration": 2.434373378753662 + }, + { + "auxiliary_loss_clip": 0.01101496, + "auxiliary_loss_mlp": 0.01033133, + "balance_loss_clip": 1.03273225, + "balance_loss_mlp": 1.01396382, + "epoch": 0.10617491730021474, + "flos": 10734011395200.0, + "grad_norm": 5.745646037958796, + "language_loss": 0.70761216, + "learning_rate": 3.939447255727624e-06, + "loss": 0.72895843, + "num_input_tokens_seen": 103076500, + "router_z_loss_clip": 0.68847656, + "router_z_loss_mlp": 0.19165039, + "step": 3659, + "time_per_iteration": 2.4157588481903076 + }, + { + "auxiliary_loss_clip": 0.0110249, + "auxiliary_loss_mlp": 0.01038027, + "balance_loss_clip": 1.03142142, + "balance_loss_mlp": 1.01738012, + "epoch": 0.10620393476873077, + "flos": 22265160019200.0, + "grad_norm": 3.5786289697625486, + "language_loss": 0.57866901, + "learning_rate": 3.939401345962705e-06, + "loss": 0.60007417, + "num_input_tokens_seen": 103093190, + "router_z_loss_clip": 0.71142578, + "router_z_loss_mlp": 0.2064209, + "step": 3660, + "time_per_iteration": 2.4342234134674072 + }, + { + "auxiliary_loss_clip": 0.01013565, + "auxiliary_loss_mlp": 0.01001588, + "balance_loss_clip": 1.00231135, + "balance_loss_mlp": 1.00049162, + "epoch": 0.10623295223724682, + "flos": 70617176346240.0, + "grad_norm": 0.6449809223647059, + "language_loss": 0.52534699, + "learning_rate": 3.939355419068208e-06, + "loss": 0.54549849, + "num_input_tokens_seen": 103161880, + "router_z_loss_clip": 0.11230469, + "router_z_loss_mlp": 0.01098633, + "step": 3661, + "time_per_iteration": 7.988888263702393 + }, + { + "auxiliary_loss_clip": 0.0110032, + "auxiliary_loss_mlp": 0.01043453, + "balance_loss_clip": 1.03299427, + "balance_loss_mlp": 1.02420115, + "epoch": 0.10626196970576286, + "flos": 27262767409920.0, + "grad_norm": 2.1888004592161483, + "language_loss": 0.92870045, + "learning_rate": 3.939309475044539e-06, + "loss": 0.95013821, + "num_input_tokens_seen": 103178095, + "router_z_loss_clip": 0.67333984, + "router_z_loss_mlp": 0.19238281, + "step": 3662, + "time_per_iteration": 2.4496774673461914 + }, + { + "auxiliary_loss_clip": 0.01105186, + "auxiliary_loss_mlp": 0.01039139, + "balance_loss_clip": 1.03350317, + "balance_loss_mlp": 1.01759815, + "epoch": 0.10629098717427891, + "flos": 17705433300480.0, + "grad_norm": 2.07902032768663, + "language_loss": 0.69898152, + "learning_rate": 3.939263513892105e-06, + "loss": 0.72042477, + "num_input_tokens_seen": 103192020, + "router_z_loss_clip": 0.71630859, + "router_z_loss_mlp": 0.2154541, + "step": 3663, + "time_per_iteration": 2.4047727584838867 + }, + { + "auxiliary_loss_clip": 0.01094213, + "auxiliary_loss_mlp": 0.01037073, + "balance_loss_clip": 1.02950406, + "balance_loss_mlp": 1.01950169, + "epoch": 0.10632000464279497, + "flos": 24710092686720.0, + "grad_norm": 2.0900893214652716, + "language_loss": 0.82783675, + "learning_rate": 3.93921753561131e-06, + "loss": 0.84914964, + "num_input_tokens_seen": 103207820, + "router_z_loss_clip": 0.64648438, + "router_z_loss_mlp": 0.17565918, + "step": 3664, + "time_per_iteration": 2.4195802211761475 + }, + { + "auxiliary_loss_clip": 0.01095852, + "auxiliary_loss_mlp": 0.01044299, + "balance_loss_clip": 1.02875471, + "balance_loss_mlp": 1.02508235, + "epoch": 0.106349022111311, + "flos": 28910143912320.0, + "grad_norm": 1.8496680314233096, + "language_loss": 0.85038626, + "learning_rate": 3.93917154020256e-06, + "loss": 0.87178779, + "num_input_tokens_seen": 103230010, + "router_z_loss_clip": 0.67041016, + "router_z_loss_mlp": 0.19226074, + "step": 3665, + "time_per_iteration": 2.5629379749298096 + }, + { + "auxiliary_loss_clip": 0.01012538, + "auxiliary_loss_mlp": 0.01002905, + "balance_loss_clip": 1.00180078, + "balance_loss_mlp": 1.00190377, + "epoch": 0.10637803957982706, + "flos": 59258568994560.0, + "grad_norm": 0.744757471411713, + "language_loss": 0.52912217, + "learning_rate": 3.939125527666264e-06, + "loss": 0.54927665, + "num_input_tokens_seen": 103286485, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01000977, + "step": 3666, + "time_per_iteration": 2.879460096359253 + }, + { + "auxiliary_loss_clip": 0.0101293, + "auxiliary_loss_mlp": 0.01001861, + "balance_loss_clip": 1.00212789, + "balance_loss_mlp": 1.00067484, + "epoch": 0.1064070570483431, + "flos": 59963325166080.0, + "grad_norm": 0.6571140233854557, + "language_loss": 0.50511229, + "learning_rate": 3.939079498002826e-06, + "loss": 0.52526009, + "num_input_tokens_seen": 103350660, + "router_z_loss_clip": 0.10791016, + "router_z_loss_mlp": 0.01184082, + "step": 3667, + "time_per_iteration": 3.0616767406463623 + }, + { + "auxiliary_loss_clip": 0.01104404, + "auxiliary_loss_mlp": 0.01047916, + "balance_loss_clip": 1.0321523, + "balance_loss_mlp": 1.02526617, + "epoch": 0.10643607451685914, + "flos": 27300333899520.0, + "grad_norm": 3.2163149624827807, + "language_loss": 0.8648349, + "learning_rate": 3.939033451212654e-06, + "loss": 0.88635802, + "num_input_tokens_seen": 103365310, + "router_z_loss_clip": 0.72265625, + "router_z_loss_mlp": 0.2265625, + "step": 3668, + "time_per_iteration": 2.3626821041107178 + }, + { + "auxiliary_loss_clip": 0.01012255, + "auxiliary_loss_mlp": 0.01001687, + "balance_loss_clip": 1.00167, + "balance_loss_mlp": 1.00069785, + "epoch": 0.1064650919853752, + "flos": 67042733506560.0, + "grad_norm": 0.6495479845402587, + "language_loss": 0.51086837, + "learning_rate": 3.938987387296152e-06, + "loss": 0.53100777, + "num_input_tokens_seen": 103430990, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.0098877, + "step": 3669, + "time_per_iteration": 3.0431792736053467 + }, + { + "auxiliary_loss_clip": 0.01110762, + "auxiliary_loss_mlp": 0.0104643, + "balance_loss_clip": 1.03631902, + "balance_loss_mlp": 1.02376819, + "epoch": 0.10649410945389125, + "flos": 15638853703680.0, + "grad_norm": 2.1615605406669935, + "language_loss": 0.88464898, + "learning_rate": 3.938941306253731e-06, + "loss": 0.90622091, + "num_input_tokens_seen": 103445470, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.2265625, + "step": 3670, + "time_per_iteration": 2.4182791709899902 + }, + { + "auxiliary_loss_clip": 0.01013692, + "auxiliary_loss_mlp": 0.01001155, + "balance_loss_clip": 1.00308394, + "balance_loss_mlp": 1.00023115, + "epoch": 0.10652312692240729, + "flos": 65623513939200.0, + "grad_norm": 0.6666083409760041, + "language_loss": 0.48093867, + "learning_rate": 3.938895208085794e-06, + "loss": 0.50108719, + "num_input_tokens_seen": 103508490, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.00921631, + "step": 3671, + "time_per_iteration": 3.0726540088653564 + }, + { + "auxiliary_loss_clip": 0.01105872, + "auxiliary_loss_mlp": 0.01042973, + "balance_loss_clip": 1.03472149, + "balance_loss_mlp": 1.02310669, + "epoch": 0.10655214439092334, + "flos": 17815444594560.0, + "grad_norm": 2.8828420223763422, + "language_loss": 0.71045339, + "learning_rate": 3.938849092792751e-06, + "loss": 0.73194188, + "num_input_tokens_seen": 103522475, + "router_z_loss_clip": 0.71240234, + "router_z_loss_mlp": 0.19854736, + "step": 3672, + "time_per_iteration": 2.4088711738586426 + }, + { + "auxiliary_loss_clip": 0.01110212, + "auxiliary_loss_mlp": 0.0104239, + "balance_loss_clip": 1.03544271, + "balance_loss_mlp": 1.01971698, + "epoch": 0.10658116185943939, + "flos": 16586815472640.0, + "grad_norm": 2.8922515919797895, + "language_loss": 0.80134869, + "learning_rate": 3.938802960375008e-06, + "loss": 0.82287478, + "num_input_tokens_seen": 103538580, + "router_z_loss_clip": 0.74804688, + "router_z_loss_mlp": 0.22668457, + "step": 3673, + "time_per_iteration": 2.3908514976501465 + }, + { + "auxiliary_loss_clip": 0.01014199, + "auxiliary_loss_mlp": 0.01001155, + "balance_loss_clip": 1.00350177, + "balance_loss_mlp": 1.00009954, + "epoch": 0.10661017932795543, + "flos": 71441860504320.0, + "grad_norm": 0.71181703144459, + "language_loss": 0.44827008, + "learning_rate": 3.938756810832972e-06, + "loss": 0.46842363, + "num_input_tokens_seen": 103589745, + "router_z_loss_clip": 0.10693359, + "router_z_loss_mlp": 0.01055908, + "step": 3674, + "time_per_iteration": 2.9227294921875 + }, + { + "auxiliary_loss_clip": 0.0101376, + "auxiliary_loss_mlp": 0.01001354, + "balance_loss_clip": 1.00314164, + "balance_loss_mlp": 1.00014377, + "epoch": 0.10663919679647148, + "flos": 69453520997760.0, + "grad_norm": 0.6338667999834467, + "language_loss": 0.53073132, + "learning_rate": 3.938710644167052e-06, + "loss": 0.55088246, + "num_input_tokens_seen": 103656555, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01208496, + "step": 3675, + "time_per_iteration": 3.070180654525757 + }, + { + "auxiliary_loss_clip": 0.01109423, + "auxiliary_loss_mlp": 0.01055808, + "balance_loss_clip": 1.0350194, + "balance_loss_mlp": 1.03476763, + "epoch": 0.10666821426498753, + "flos": 29345441143680.0, + "grad_norm": 2.139388561316166, + "language_loss": 1.07246447, + "learning_rate": 3.938664460377655e-06, + "loss": 1.09411681, + "num_input_tokens_seen": 103674730, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.21044922, + "step": 3676, + "time_per_iteration": 2.5052437782287598 + }, + { + "auxiliary_loss_clip": 0.01099574, + "auxiliary_loss_mlp": 0.01032117, + "balance_loss_clip": 1.03142858, + "balance_loss_mlp": 1.01384187, + "epoch": 0.10669723173350357, + "flos": 33173318609280.0, + "grad_norm": 5.976732240789033, + "language_loss": 0.8820523, + "learning_rate": 3.93861825946519e-06, + "loss": 0.90336925, + "num_input_tokens_seen": 103690645, + "router_z_loss_clip": 0.68139648, + "router_z_loss_mlp": 0.18267822, + "step": 3677, + "time_per_iteration": 2.4852850437164307 + }, + { + "auxiliary_loss_clip": 0.0110441, + "auxiliary_loss_mlp": 0.01046795, + "balance_loss_clip": 1.03471375, + "balance_loss_mlp": 1.02701831, + "epoch": 0.10672624920201962, + "flos": 39049515164160.0, + "grad_norm": 2.59731163432483, + "language_loss": 0.83906186, + "learning_rate": 3.938572041430063e-06, + "loss": 0.86057389, + "num_input_tokens_seen": 103706385, + "router_z_loss_clip": 0.69628906, + "router_z_loss_mlp": 0.19763184, + "step": 3678, + "time_per_iteration": 2.588120460510254 + }, + { + "auxiliary_loss_clip": 0.01111909, + "auxiliary_loss_mlp": 0.01047728, + "balance_loss_clip": 1.03647876, + "balance_loss_mlp": 1.02489901, + "epoch": 0.10675526667053566, + "flos": 29891901744000.0, + "grad_norm": 1.969505026782508, + "language_loss": 0.91923416, + "learning_rate": 3.938525806272682e-06, + "loss": 0.94083047, + "num_input_tokens_seen": 103722620, + "router_z_loss_clip": 0.75341797, + "router_z_loss_mlp": 0.22802734, + "step": 3679, + "time_per_iteration": 2.523604393005371 + }, + { + "auxiliary_loss_clip": 0.01101707, + "auxiliary_loss_mlp": 0.01038272, + "balance_loss_clip": 1.03036714, + "balance_loss_mlp": 1.01808989, + "epoch": 0.10678428413905171, + "flos": 19783847848320.0, + "grad_norm": 2.3982553508540545, + "language_loss": 0.83709693, + "learning_rate": 3.938479553993458e-06, + "loss": 0.85849679, + "num_input_tokens_seen": 103735605, + "router_z_loss_clip": 0.71337891, + "router_z_loss_mlp": 0.20178223, + "step": 3680, + "time_per_iteration": 2.518972635269165 + }, + { + "auxiliary_loss_clip": 0.01110391, + "auxiliary_loss_mlp": 0.01038659, + "balance_loss_clip": 1.03652906, + "balance_loss_mlp": 1.01803589, + "epoch": 0.10681330160756776, + "flos": 26793080622720.0, + "grad_norm": 2.372626823871684, + "language_loss": 0.90031403, + "learning_rate": 3.938433284592799e-06, + "loss": 0.92180455, + "num_input_tokens_seen": 103750330, + "router_z_loss_clip": 0.73828125, + "router_z_loss_mlp": 0.20617676, + "step": 3681, + "time_per_iteration": 2.435804605484009 + }, + { + "auxiliary_loss_clip": 0.01014862, + "auxiliary_loss_mlp": 0.01003492, + "balance_loss_clip": 1.00436151, + "balance_loss_mlp": 1.00235331, + "epoch": 0.1068423190760838, + "flos": 65284822039680.0, + "grad_norm": 0.6810630079242095, + "language_loss": 0.54310715, + "learning_rate": 3.938386998071112e-06, + "loss": 0.56329072, + "num_input_tokens_seen": 103813120, + "router_z_loss_clip": 0.10498047, + "router_z_loss_mlp": 0.01141357, + "step": 3682, + "time_per_iteration": 3.0560224056243896 + }, + { + "auxiliary_loss_clip": 0.01094391, + "auxiliary_loss_mlp": 0.01040824, + "balance_loss_clip": 1.0303185, + "balance_loss_mlp": 1.02380085, + "epoch": 0.10687133654459985, + "flos": 26681184115200.0, + "grad_norm": 2.205636913549662, + "language_loss": 0.58520734, + "learning_rate": 3.938340694428806e-06, + "loss": 0.60655951, + "num_input_tokens_seen": 103828230, + "router_z_loss_clip": 0.640625, + "router_z_loss_mlp": 0.17010498, + "step": 3683, + "time_per_iteration": 2.455976724624634 + }, + { + "auxiliary_loss_clip": 0.01101698, + "auxiliary_loss_mlp": 0.010443, + "balance_loss_clip": 1.0326426, + "balance_loss_mlp": 1.02343869, + "epoch": 0.1069003540131159, + "flos": 25221011656320.0, + "grad_norm": 2.6619544970411972, + "language_loss": 0.84474391, + "learning_rate": 3.938294373666291e-06, + "loss": 0.8662039, + "num_input_tokens_seen": 103842790, + "router_z_loss_clip": 0.69091797, + "router_z_loss_mlp": 0.20861816, + "step": 3684, + "time_per_iteration": 2.482245683670044 + }, + { + "auxiliary_loss_clip": 0.01103164, + "auxiliary_loss_mlp": 0.01045655, + "balance_loss_clip": 1.03194451, + "balance_loss_mlp": 1.02386379, + "epoch": 0.10692937148163194, + "flos": 16319521036800.0, + "grad_norm": 2.4119673535705424, + "language_loss": 0.89008456, + "learning_rate": 3.938248035783976e-06, + "loss": 0.91157269, + "num_input_tokens_seen": 103855400, + "router_z_loss_clip": 0.71240234, + "router_z_loss_mlp": 0.21777344, + "step": 3685, + "time_per_iteration": 2.40329909324646 + }, + { + "auxiliary_loss_clip": 0.01102767, + "auxiliary_loss_mlp": 0.01041718, + "balance_loss_clip": 1.03235388, + "balance_loss_mlp": 1.02138066, + "epoch": 0.10695838895014799, + "flos": 16793187719040.0, + "grad_norm": 3.503272306647544, + "language_loss": 0.7753737, + "learning_rate": 3.9382016807822705e-06, + "loss": 0.79681855, + "num_input_tokens_seen": 103869190, + "router_z_loss_clip": 0.70361328, + "router_z_loss_mlp": 0.20349121, + "step": 3686, + "time_per_iteration": 2.4253005981445312 + }, + { + "auxiliary_loss_clip": 0.01097115, + "auxiliary_loss_mlp": 0.01036257, + "balance_loss_clip": 1.03040385, + "balance_loss_mlp": 1.01696873, + "epoch": 0.10698740641866404, + "flos": 16502675362560.0, + "grad_norm": 2.4828116705617784, + "language_loss": 0.8098821, + "learning_rate": 3.938155308661583e-06, + "loss": 0.8312158, + "num_input_tokens_seen": 103882870, + "router_z_loss_clip": 0.66650391, + "router_z_loss_mlp": 0.19287109, + "step": 3687, + "time_per_iteration": 2.343838930130005 + }, + { + "auxiliary_loss_clip": 0.01108419, + "auxiliary_loss_mlp": 0.01049922, + "balance_loss_clip": 1.03186405, + "balance_loss_mlp": 1.02758265, + "epoch": 0.10701642388718008, + "flos": 24091187281920.0, + "grad_norm": 2.2019854567867694, + "language_loss": 0.90091711, + "learning_rate": 3.938108919422323e-06, + "loss": 0.92250055, + "num_input_tokens_seen": 103899560, + "router_z_loss_clip": 0.765625, + "router_z_loss_mlp": 0.22338867, + "step": 3688, + "time_per_iteration": 2.473390817642212 + }, + { + "auxiliary_loss_clip": 0.0109944, + "auxiliary_loss_mlp": 0.01031645, + "balance_loss_clip": 1.03292501, + "balance_loss_mlp": 1.01265514, + "epoch": 0.10704544135569613, + "flos": 27263535459840.0, + "grad_norm": 2.5499913368077243, + "language_loss": 0.91189015, + "learning_rate": 3.938062513064902e-06, + "loss": 0.93320096, + "num_input_tokens_seen": 103912745, + "router_z_loss_clip": 0.66601562, + "router_z_loss_mlp": 0.18994141, + "step": 3689, + "time_per_iteration": 2.4279592037200928 + }, + { + "auxiliary_loss_clip": 0.01099605, + "auxiliary_loss_mlp": 0.01040382, + "balance_loss_clip": 1.029935, + "balance_loss_mlp": 1.02033067, + "epoch": 0.10707445882421218, + "flos": 25257286425600.0, + "grad_norm": 2.8575536061151596, + "language_loss": 1.00057936, + "learning_rate": 3.938016089589727e-06, + "loss": 1.02197933, + "num_input_tokens_seen": 103930425, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.20043945, + "step": 3690, + "time_per_iteration": 2.5179295539855957 + }, + { + "auxiliary_loss_clip": 0.0110565, + "auxiliary_loss_mlp": 0.01040952, + "balance_loss_clip": 1.03289843, + "balance_loss_mlp": 1.02063847, + "epoch": 0.10710347629272822, + "flos": 37152823576320.0, + "grad_norm": 3.0662564299883406, + "language_loss": 0.84967512, + "learning_rate": 3.9379696489972105e-06, + "loss": 0.87114108, + "num_input_tokens_seen": 103948995, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.20300293, + "step": 3691, + "time_per_iteration": 2.523879289627075 + }, + { + "auxiliary_loss_clip": 0.01109801, + "auxiliary_loss_mlp": 0.01038186, + "balance_loss_clip": 1.03285789, + "balance_loss_mlp": 1.01675224, + "epoch": 0.10713249376124427, + "flos": 14567335032960.0, + "grad_norm": 2.2244940960679678, + "language_loss": 0.77238911, + "learning_rate": 3.937923191287762e-06, + "loss": 0.79386902, + "num_input_tokens_seen": 103964050, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.21447754, + "step": 3692, + "time_per_iteration": 2.4131994247436523 + }, + { + "auxiliary_loss_clip": 0.0101309, + "auxiliary_loss_mlp": 0.01001124, + "balance_loss_clip": 1.0023855, + "balance_loss_mlp": 1.00011063, + "epoch": 0.10716151122976031, + "flos": 60757425106560.0, + "grad_norm": 0.7175998779515271, + "language_loss": 0.46975297, + "learning_rate": 3.937876716461792e-06, + "loss": 0.48989508, + "num_input_tokens_seen": 104016635, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01013184, + "step": 3693, + "time_per_iteration": 2.8739466667175293 + }, + { + "auxiliary_loss_clip": 0.0110525, + "auxiliary_loss_mlp": 0.01041354, + "balance_loss_clip": 1.03128839, + "balance_loss_mlp": 1.01916921, + "epoch": 0.10719052869827636, + "flos": 29015442172800.0, + "grad_norm": 3.034641714199061, + "language_loss": 0.86023664, + "learning_rate": 3.93783022451971e-06, + "loss": 0.88170272, + "num_input_tokens_seen": 104030960, + "router_z_loss_clip": 0.73925781, + "router_z_loss_mlp": 0.22192383, + "step": 3694, + "time_per_iteration": 2.488245725631714 + }, + { + "auxiliary_loss_clip": 0.01101189, + "auxiliary_loss_mlp": 0.01046347, + "balance_loss_clip": 1.0335896, + "balance_loss_mlp": 1.02718985, + "epoch": 0.10721954616679241, + "flos": 45083204328960.0, + "grad_norm": 2.1568129065868598, + "language_loss": 0.62628847, + "learning_rate": 3.937783715461927e-06, + "loss": 0.64776385, + "num_input_tokens_seen": 104054930, + "router_z_loss_clip": 0.67529297, + "router_z_loss_mlp": 0.19152832, + "step": 3695, + "time_per_iteration": 2.5904085636138916 + }, + { + "auxiliary_loss_clip": 0.01103583, + "auxiliary_loss_mlp": 0.01035868, + "balance_loss_clip": 1.03329146, + "balance_loss_mlp": 1.01653218, + "epoch": 0.10724856363530845, + "flos": 21793937132160.0, + "grad_norm": 2.317407463022426, + "language_loss": 0.81398404, + "learning_rate": 3.937737189288855e-06, + "loss": 0.83537853, + "num_input_tokens_seen": 104068060, + "router_z_loss_clip": 0.70263672, + "router_z_loss_mlp": 0.19348145, + "step": 3696, + "time_per_iteration": 2.463609218597412 + }, + { + "auxiliary_loss_clip": 0.01102802, + "auxiliary_loss_mlp": 0.01036318, + "balance_loss_clip": 1.03382158, + "balance_loss_mlp": 1.01736426, + "epoch": 0.1072775811038245, + "flos": 33539382881280.0, + "grad_norm": 1.8331044312367337, + "language_loss": 0.73447788, + "learning_rate": 3.9376906460009035e-06, + "loss": 0.75586909, + "num_input_tokens_seen": 104083525, + "router_z_loss_clip": 0.68994141, + "router_z_loss_mlp": 0.1895752, + "step": 3697, + "time_per_iteration": 2.476818561553955 + }, + { + "auxiliary_loss_clip": 0.01098424, + "auxiliary_loss_mlp": 0.0103674, + "balance_loss_clip": 1.03124785, + "balance_loss_mlp": 1.0197649, + "epoch": 0.10730659857234055, + "flos": 21827768106240.0, + "grad_norm": 1.9759351223650363, + "language_loss": 0.8177774, + "learning_rate": 3.937644085598485e-06, + "loss": 0.83912903, + "num_input_tokens_seen": 104098820, + "router_z_loss_clip": 0.67285156, + "router_z_loss_mlp": 0.16967773, + "step": 3698, + "time_per_iteration": 2.4601378440856934 + }, + { + "auxiliary_loss_clip": 0.01109193, + "auxiliary_loss_mlp": 0.01053255, + "balance_loss_clip": 1.03618193, + "balance_loss_mlp": 1.03183317, + "epoch": 0.10733561604085659, + "flos": 18665858424960.0, + "grad_norm": 2.3140638595063887, + "language_loss": 0.65768915, + "learning_rate": 3.937597508082008e-06, + "loss": 0.67931366, + "num_input_tokens_seen": 104114770, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.2142334, + "step": 3699, + "time_per_iteration": 2.4270167350769043 + }, + { + "auxiliary_loss_clip": 0.01103083, + "auxiliary_loss_mlp": 0.01045215, + "balance_loss_clip": 1.03141069, + "balance_loss_mlp": 1.02250552, + "epoch": 0.10736463350937264, + "flos": 29561553659520.0, + "grad_norm": 3.5416145696963413, + "language_loss": 0.90009612, + "learning_rate": 3.937550913451887e-06, + "loss": 0.92157912, + "num_input_tokens_seen": 104131865, + "router_z_loss_clip": 0.71728516, + "router_z_loss_mlp": 0.22680664, + "step": 3700, + "time_per_iteration": 2.5328633785247803 + }, + { + "auxiliary_loss_clip": 0.01097846, + "auxiliary_loss_mlp": 0.0104677, + "balance_loss_clip": 1.03237808, + "balance_loss_mlp": 1.02953231, + "epoch": 0.1073936509778887, + "flos": 11901821195520.0, + "grad_norm": 2.2885568219759467, + "language_loss": 0.6525231, + "learning_rate": 3.937504301708532e-06, + "loss": 0.67396933, + "num_input_tokens_seen": 104143640, + "router_z_loss_clip": 0.65380859, + "router_z_loss_mlp": 0.17236328, + "step": 3701, + "time_per_iteration": 2.355984687805176 + }, + { + "auxiliary_loss_clip": 0.01013292, + "auxiliary_loss_mlp": 0.01004397, + "balance_loss_clip": 1.0025717, + "balance_loss_mlp": 1.00313342, + "epoch": 0.10742266844640473, + "flos": 69302905925760.0, + "grad_norm": 0.670032372166832, + "language_loss": 0.49656451, + "learning_rate": 3.9374576728523555e-06, + "loss": 0.51674139, + "num_input_tokens_seen": 104207295, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01263428, + "step": 3702, + "time_per_iteration": 3.0839788913726807 + }, + { + "auxiliary_loss_clip": 0.01108994, + "auxiliary_loss_mlp": 0.01047656, + "balance_loss_clip": 1.03413928, + "balance_loss_mlp": 1.02511406, + "epoch": 0.10745168591492078, + "flos": 35299144650240.0, + "grad_norm": 1.9956305018733336, + "language_loss": 0.69624448, + "learning_rate": 3.937411026883768e-06, + "loss": 0.71781099, + "num_input_tokens_seen": 104226510, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.2253418, + "step": 3703, + "time_per_iteration": 2.5116310119628906 + }, + { + "auxiliary_loss_clip": 0.01095714, + "auxiliary_loss_mlp": 0.01042225, + "balance_loss_clip": 1.03179955, + "balance_loss_mlp": 1.02483284, + "epoch": 0.10748070338343683, + "flos": 32990827599360.0, + "grad_norm": 2.3953847754286843, + "language_loss": 0.94505584, + "learning_rate": 3.937364363803182e-06, + "loss": 0.96643519, + "num_input_tokens_seen": 104243580, + "router_z_loss_clip": 0.63842773, + "router_z_loss_mlp": 0.17376709, + "step": 3704, + "time_per_iteration": 2.4869091510772705 + }, + { + "auxiliary_loss_clip": 0.01013706, + "auxiliary_loss_mlp": 0.010024, + "balance_loss_clip": 1.00280499, + "balance_loss_mlp": 1.00136292, + "epoch": 0.10750972085195287, + "flos": 62399914018560.0, + "grad_norm": 0.7912708152047273, + "language_loss": 0.49419275, + "learning_rate": 3.937317683611012e-06, + "loss": 0.51435381, + "num_input_tokens_seen": 104303375, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01037598, + "step": 3705, + "time_per_iteration": 2.920006036758423 + }, + { + "auxiliary_loss_clip": 0.01098994, + "auxiliary_loss_mlp": 0.01038212, + "balance_loss_clip": 1.03231907, + "balance_loss_mlp": 1.02004421, + "epoch": 0.10753873832046892, + "flos": 23986936362240.0, + "grad_norm": 2.09014577182343, + "language_loss": 0.92500883, + "learning_rate": 3.937270986307666e-06, + "loss": 0.94638097, + "num_input_tokens_seen": 104319360, + "router_z_loss_clip": 0.66650391, + "router_z_loss_mlp": 0.18170166, + "step": 3706, + "time_per_iteration": 2.49560284614563 + }, + { + "auxiliary_loss_clip": 0.01107008, + "auxiliary_loss_mlp": 0.0103763, + "balance_loss_clip": 1.03392577, + "balance_loss_mlp": 1.01663113, + "epoch": 0.10756775578898498, + "flos": 43388486156160.0, + "grad_norm": 2.196982144220368, + "language_loss": 0.89778566, + "learning_rate": 3.93722427189356e-06, + "loss": 0.91923213, + "num_input_tokens_seen": 104335200, + "router_z_loss_clip": 0.73095703, + "router_z_loss_mlp": 0.2098999, + "step": 3707, + "time_per_iteration": 2.668046712875366 + }, + { + "auxiliary_loss_clip": 0.01012344, + "auxiliary_loss_mlp": 0.01002849, + "balance_loss_clip": 1.00158441, + "balance_loss_mlp": 1.00148988, + "epoch": 0.10759677325750101, + "flos": 74776623793920.0, + "grad_norm": 0.622446818141014, + "language_loss": 0.47759944, + "learning_rate": 3.937177540369105e-06, + "loss": 0.49775138, + "num_input_tokens_seen": 104402575, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01361084, + "step": 3708, + "time_per_iteration": 3.1703176498413086 + }, + { + "auxiliary_loss_clip": 0.01102713, + "auxiliary_loss_mlp": 0.01050719, + "balance_loss_clip": 1.03455639, + "balance_loss_mlp": 1.0309186, + "epoch": 0.10762579072601706, + "flos": 36895339232640.0, + "grad_norm": 2.1492668651645523, + "language_loss": 0.82463712, + "learning_rate": 3.937130791734714e-06, + "loss": 0.8461715, + "num_input_tokens_seen": 104422470, + "router_z_loss_clip": 0.68115234, + "router_z_loss_mlp": 0.19824219, + "step": 3709, + "time_per_iteration": 2.605585813522339 + }, + { + "auxiliary_loss_clip": 0.01098342, + "auxiliary_loss_mlp": 0.01042373, + "balance_loss_clip": 1.03119528, + "balance_loss_mlp": 1.0231328, + "epoch": 0.1076548081945331, + "flos": 32080886167680.0, + "grad_norm": 2.2594615946350873, + "language_loss": 0.91297197, + "learning_rate": 3.937084025990801e-06, + "loss": 0.9343791, + "num_input_tokens_seen": 104440370, + "router_z_loss_clip": 0.67089844, + "router_z_loss_mlp": 0.19244385, + "step": 3710, + "time_per_iteration": 2.4552924633026123 + }, + { + "auxiliary_loss_clip": 0.01011984, + "auxiliary_loss_mlp": 0.01001559, + "balance_loss_clip": 1.00109124, + "balance_loss_mlp": 1.0003494, + "epoch": 0.10768382566304915, + "flos": 74078256401280.0, + "grad_norm": 0.75582544585713, + "language_loss": 0.50951749, + "learning_rate": 3.937037243137776e-06, + "loss": 0.52965295, + "num_input_tokens_seen": 104508565, + "router_z_loss_clip": 0.10888672, + "router_z_loss_mlp": 0.01208496, + "step": 3711, + "time_per_iteration": 3.3279876708984375 + }, + { + "auxiliary_loss_clip": 0.01110909, + "auxiliary_loss_mlp": 0.01047168, + "balance_loss_clip": 1.03509533, + "balance_loss_mlp": 1.0228132, + "epoch": 0.1077128431315652, + "flos": 11794114051200.0, + "grad_norm": 2.539907458592236, + "language_loss": 0.84375799, + "learning_rate": 3.936990443176056e-06, + "loss": 0.8653388, + "num_input_tokens_seen": 104520605, + "router_z_loss_clip": 0.7578125, + "router_z_loss_mlp": 0.2434082, + "step": 3712, + "time_per_iteration": 2.4768524169921875 + }, + { + "auxiliary_loss_clip": 0.01106363, + "auxiliary_loss_mlp": 0.01056465, + "balance_loss_clip": 1.03407323, + "balance_loss_mlp": 1.03535342, + "epoch": 0.10774186060008124, + "flos": 11975767188480.0, + "grad_norm": 2.631855658134626, + "language_loss": 1.02210021, + "learning_rate": 3.936943626106052e-06, + "loss": 1.04372847, + "num_input_tokens_seen": 104532375, + "router_z_loss_clip": 0.72265625, + "router_z_loss_mlp": 0.21105957, + "step": 3713, + "time_per_iteration": 2.4109857082366943 + }, + { + "auxiliary_loss_clip": 0.01098101, + "auxiliary_loss_mlp": 0.0103725, + "balance_loss_clip": 1.03131413, + "balance_loss_mlp": 1.01737809, + "epoch": 0.1077708780685973, + "flos": 36056725441920.0, + "grad_norm": 2.927940997578681, + "language_loss": 0.90777886, + "learning_rate": 3.936896791928178e-06, + "loss": 0.92913234, + "num_input_tokens_seen": 104551995, + "router_z_loss_clip": 0.66845703, + "router_z_loss_mlp": 0.1986084, + "step": 3714, + "time_per_iteration": 2.5174851417541504 + }, + { + "auxiliary_loss_clip": 0.01095453, + "auxiliary_loss_mlp": 0.01040613, + "balance_loss_clip": 1.03234363, + "balance_loss_mlp": 1.02167022, + "epoch": 0.10779989553711335, + "flos": 46855431319680.0, + "grad_norm": 3.0057873539054953, + "language_loss": 0.76777279, + "learning_rate": 3.936849940642848e-06, + "loss": 0.78913343, + "num_input_tokens_seen": 104569205, + "router_z_loss_clip": 0.63085938, + "router_z_loss_mlp": 0.18933105, + "step": 3715, + "time_per_iteration": 2.65950083732605 + }, + { + "auxiliary_loss_clip": 0.01097881, + "auxiliary_loss_mlp": 0.01040529, + "balance_loss_clip": 1.03211319, + "balance_loss_mlp": 1.02213538, + "epoch": 0.10782891300562938, + "flos": 74730785178240.0, + "grad_norm": 4.054507579593436, + "language_loss": 0.74151731, + "learning_rate": 3.936803072250475e-06, + "loss": 0.76290143, + "num_input_tokens_seen": 104590840, + "router_z_loss_clip": 0.65673828, + "router_z_loss_mlp": 0.184021, + "step": 3716, + "time_per_iteration": 2.793048143386841 + }, + { + "auxiliary_loss_clip": 0.01012865, + "auxiliary_loss_mlp": 0.01002055, + "balance_loss_clip": 1.00228369, + "balance_loss_mlp": 1.00057137, + "epoch": 0.10785793047414544, + "flos": 74770619040000.0, + "grad_norm": 0.6769041510463264, + "language_loss": 0.51689816, + "learning_rate": 3.9367561867514735e-06, + "loss": 0.53704739, + "num_input_tokens_seen": 104647695, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.01483154, + "step": 3717, + "time_per_iteration": 3.0096049308776855 + }, + { + "auxiliary_loss_clip": 0.0110623, + "auxiliary_loss_mlp": 0.01041754, + "balance_loss_clip": 1.03560162, + "balance_loss_mlp": 1.0208385, + "epoch": 0.10788694794266149, + "flos": 16172152721280.0, + "grad_norm": 2.720500065083921, + "language_loss": 0.79492539, + "learning_rate": 3.936709284146258e-06, + "loss": 0.81640524, + "num_input_tokens_seen": 104659490, + "router_z_loss_clip": 0.70678711, + "router_z_loss_mlp": 0.20916748, + "step": 3718, + "time_per_iteration": 2.3956351280212402 + }, + { + "auxiliary_loss_clip": 0.01096794, + "auxiliary_loss_mlp": 0.01037286, + "balance_loss_clip": 1.03079963, + "balance_loss_mlp": 1.01801038, + "epoch": 0.10791596541117753, + "flos": 17814187785600.0, + "grad_norm": 2.858813053457193, + "language_loss": 0.77757746, + "learning_rate": 3.936662364435243e-06, + "loss": 0.79891825, + "num_input_tokens_seen": 104675430, + "router_z_loss_clip": 0.65966797, + "router_z_loss_mlp": 0.19287109, + "step": 3719, + "time_per_iteration": 2.4093332290649414 + }, + { + "auxiliary_loss_clip": 0.01109318, + "auxiliary_loss_mlp": 0.01034238, + "balance_loss_clip": 1.0368464, + "balance_loss_mlp": 1.01583254, + "epoch": 0.10794498287969358, + "flos": 26937411649920.0, + "grad_norm": 4.40414755478474, + "language_loss": 0.82134557, + "learning_rate": 3.936615427618841e-06, + "loss": 0.84278113, + "num_input_tokens_seen": 104691890, + "router_z_loss_clip": 0.72509766, + "router_z_loss_mlp": 0.18395996, + "step": 3720, + "time_per_iteration": 2.4216511249542236 + }, + { + "auxiliary_loss_clip": 0.0111036, + "auxiliary_loss_mlp": 0.01050967, + "balance_loss_clip": 1.03506231, + "balance_loss_mlp": 1.0294261, + "epoch": 0.10797400034820963, + "flos": 28174314764160.0, + "grad_norm": 2.760434159748618, + "language_loss": 0.98121083, + "learning_rate": 3.936568473697469e-06, + "loss": 1.00282407, + "num_input_tokens_seen": 104707060, + "router_z_loss_clip": 0.75244141, + "router_z_loss_mlp": 0.21557617, + "step": 3721, + "time_per_iteration": 2.465550661087036 + }, + { + "auxiliary_loss_clip": 0.0109937, + "auxiliary_loss_mlp": 0.01033738, + "balance_loss_clip": 1.02978384, + "balance_loss_mlp": 1.01322198, + "epoch": 0.10800301781672567, + "flos": 13836602943360.0, + "grad_norm": 2.8413982202787804, + "language_loss": 0.94576454, + "learning_rate": 3.936521502671539e-06, + "loss": 0.96709561, + "num_input_tokens_seen": 104719720, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.2052002, + "step": 3722, + "time_per_iteration": 2.3495945930480957 + }, + { + "auxiliary_loss_clip": 0.01098078, + "auxiliary_loss_mlp": 0.01043871, + "balance_loss_clip": 1.03219414, + "balance_loss_mlp": 1.02362299, + "epoch": 0.10803203528524172, + "flos": 19132473012480.0, + "grad_norm": 1.5498838660667866, + "language_loss": 0.52180564, + "learning_rate": 3.936474514541469e-06, + "loss": 0.54322511, + "num_input_tokens_seen": 104738840, + "router_z_loss_clip": 0.65795898, + "router_z_loss_mlp": 0.20257568, + "step": 3723, + "time_per_iteration": 2.467754364013672 + }, + { + "auxiliary_loss_clip": 0.01107591, + "auxiliary_loss_mlp": 0.01051677, + "balance_loss_clip": 1.03407776, + "balance_loss_mlp": 1.02982593, + "epoch": 0.10806105275375776, + "flos": 30369269030400.0, + "grad_norm": 2.7041538507045875, + "language_loss": 0.73403633, + "learning_rate": 3.936427509307673e-06, + "loss": 0.755629, + "num_input_tokens_seen": 104754365, + "router_z_loss_clip": 0.73535156, + "router_z_loss_mlp": 0.21850586, + "step": 3724, + "time_per_iteration": 2.4401819705963135 + }, + { + "auxiliary_loss_clip": 0.01013277, + "auxiliary_loss_mlp": 0.0100184, + "balance_loss_clip": 1.00274408, + "balance_loss_mlp": 1.00041556, + "epoch": 0.1080900702222738, + "flos": 74777322021120.0, + "grad_norm": 0.6774868451314954, + "language_loss": 0.54488939, + "learning_rate": 3.936380486970564e-06, + "loss": 0.56504059, + "num_input_tokens_seen": 104820535, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01422119, + "step": 3725, + "time_per_iteration": 3.103224754333496 + }, + { + "auxiliary_loss_clip": 0.01012734, + "auxiliary_loss_mlp": 0.01001438, + "balance_loss_clip": 1.00212383, + "balance_loss_mlp": 0.99998319, + "epoch": 0.10811908769078986, + "flos": 68426236886400.0, + "grad_norm": 0.621410303746233, + "language_loss": 0.48744819, + "learning_rate": 3.93633344753056e-06, + "loss": 0.50758994, + "num_input_tokens_seen": 104882720, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.01452637, + "step": 3726, + "time_per_iteration": 3.0227534770965576 + }, + { + "auxiliary_loss_clip": 0.01104194, + "auxiliary_loss_mlp": 0.01046121, + "balance_loss_clip": 1.03482449, + "balance_loss_mlp": 1.02519941, + "epoch": 0.1081481051593059, + "flos": 11320691748480.0, + "grad_norm": 4.093040516469623, + "language_loss": 0.81508845, + "learning_rate": 3.936286390988076e-06, + "loss": 0.83659166, + "num_input_tokens_seen": 104893095, + "router_z_loss_clip": 0.69360352, + "router_z_loss_mlp": 0.20910645, + "step": 3727, + "time_per_iteration": 2.3471667766571045 + }, + { + "auxiliary_loss_clip": 0.0111024, + "auxiliary_loss_mlp": 0.01051444, + "balance_loss_clip": 1.03602517, + "balance_loss_mlp": 1.02801991, + "epoch": 0.10817712262782195, + "flos": 15553107671040.0, + "grad_norm": 2.3000245525369207, + "language_loss": 0.88500047, + "learning_rate": 3.936239317343525e-06, + "loss": 0.90661728, + "num_input_tokens_seen": 104911540, + "router_z_loss_clip": 0.7421875, + "router_z_loss_mlp": 0.23425293, + "step": 3728, + "time_per_iteration": 2.589759588241577 + }, + { + "auxiliary_loss_clip": 0.01012288, + "auxiliary_loss_mlp": 0.01001193, + "balance_loss_clip": 1.00164366, + "balance_loss_mlp": 0.99992371, + "epoch": 0.108206140096338, + "flos": 63460645079040.0, + "grad_norm": 0.6796408125099356, + "language_loss": 0.49641603, + "learning_rate": 3.936192226597327e-06, + "loss": 0.51655084, + "num_input_tokens_seen": 104969050, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01269531, + "step": 3729, + "time_per_iteration": 7.402519702911377 + }, + { + "auxiliary_loss_clip": 0.01101774, + "auxiliary_loss_mlp": 0.01037157, + "balance_loss_clip": 1.03215933, + "balance_loss_mlp": 1.01702285, + "epoch": 0.10823515756485404, + "flos": 12962028585600.0, + "grad_norm": 2.727239292634211, + "language_loss": 0.8789109, + "learning_rate": 3.936145118749894e-06, + "loss": 0.9003002, + "num_input_tokens_seen": 104982655, + "router_z_loss_clip": 0.69628906, + "router_z_loss_mlp": 0.20153809, + "step": 3730, + "time_per_iteration": 2.411757469177246 + }, + { + "auxiliary_loss_clip": 0.01104629, + "auxiliary_loss_mlp": 0.01036209, + "balance_loss_clip": 1.03263378, + "balance_loss_mlp": 1.01464438, + "epoch": 0.10826417503337009, + "flos": 72835001285760.0, + "grad_norm": 1.7702036527376284, + "language_loss": 0.72781342, + "learning_rate": 3.936097993801645e-06, + "loss": 0.7492218, + "num_input_tokens_seen": 105008100, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.21582031, + "step": 3731, + "time_per_iteration": 2.7775585651397705 + }, + { + "auxiliary_loss_clip": 0.01106216, + "auxiliary_loss_mlp": 0.01047568, + "balance_loss_clip": 1.03389072, + "balance_loss_mlp": 1.02594399, + "epoch": 0.10829319250188614, + "flos": 30729049257600.0, + "grad_norm": 2.4934832702142415, + "language_loss": 0.84836394, + "learning_rate": 3.9360508517529945e-06, + "loss": 0.86990184, + "num_input_tokens_seen": 105022325, + "router_z_loss_clip": 0.72314453, + "router_z_loss_mlp": 0.21630859, + "step": 3732, + "time_per_iteration": 2.491567850112915 + }, + { + "auxiliary_loss_clip": 0.01097552, + "auxiliary_loss_mlp": 0.01041005, + "balance_loss_clip": 1.0326165, + "balance_loss_mlp": 1.01864147, + "epoch": 0.10832220997040218, + "flos": 16719590839680.0, + "grad_norm": 2.5415251378762647, + "language_loss": 0.941791, + "learning_rate": 3.93600369260436e-06, + "loss": 0.96317655, + "num_input_tokens_seen": 105034070, + "router_z_loss_clip": 0.64892578, + "router_z_loss_mlp": 0.22351074, + "step": 3733, + "time_per_iteration": 2.341484546661377 + }, + { + "auxiliary_loss_clip": 0.01014036, + "auxiliary_loss_mlp": 0.01003132, + "balance_loss_clip": 1.00337291, + "balance_loss_mlp": 1.0018028, + "epoch": 0.10835122743891823, + "flos": 70136248101120.0, + "grad_norm": 0.7072318864223045, + "language_loss": 0.48535225, + "learning_rate": 3.9359565163561565e-06, + "loss": 0.50552392, + "num_input_tokens_seen": 105096565, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01330566, + "step": 3734, + "time_per_iteration": 2.9733619689941406 + }, + { + "auxiliary_loss_clip": 0.01104385, + "auxiliary_loss_mlp": 0.01041642, + "balance_loss_clip": 1.03274465, + "balance_loss_mlp": 1.01902843, + "epoch": 0.10838024490743428, + "flos": 13254461066880.0, + "grad_norm": 2.8624828612155926, + "language_loss": 0.75672925, + "learning_rate": 3.935909323008803e-06, + "loss": 0.77818954, + "num_input_tokens_seen": 105109375, + "router_z_loss_clip": 0.71777344, + "router_z_loss_mlp": 0.22595215, + "step": 3735, + "time_per_iteration": 2.321333169937134 + }, + { + "auxiliary_loss_clip": 0.01113662, + "auxiliary_loss_mlp": 0.01045436, + "balance_loss_clip": 1.03681302, + "balance_loss_mlp": 1.02303612, + "epoch": 0.10840926237595032, + "flos": 13254042130560.0, + "grad_norm": 2.8302188953970746, + "language_loss": 0.82693452, + "learning_rate": 3.935862112562714e-06, + "loss": 0.84852552, + "num_input_tokens_seen": 105121895, + "router_z_loss_clip": 0.76757812, + "router_z_loss_mlp": 0.22399902, + "step": 3736, + "time_per_iteration": 4.795709609985352 + }, + { + "auxiliary_loss_clip": 0.01101839, + "auxiliary_loss_mlp": 0.01045667, + "balance_loss_clip": 1.03160977, + "balance_loss_mlp": 1.02541304, + "epoch": 0.10843827984446637, + "flos": 25889667615360.0, + "grad_norm": 1.8861047607391968, + "language_loss": 0.81331754, + "learning_rate": 3.935814885018308e-06, + "loss": 0.83479255, + "num_input_tokens_seen": 105143185, + "router_z_loss_clip": 0.70166016, + "router_z_loss_mlp": 0.20263672, + "step": 3737, + "time_per_iteration": 2.498257875442505 + }, + { + "auxiliary_loss_clip": 0.010966, + "auxiliary_loss_mlp": 0.01042265, + "balance_loss_clip": 1.02911675, + "balance_loss_mlp": 1.02244079, + "epoch": 0.10846729731298242, + "flos": 13437789949440.0, + "grad_norm": 2.3166400546767827, + "language_loss": 0.79453492, + "learning_rate": 3.935767640376001e-06, + "loss": 0.81592351, + "num_input_tokens_seen": 105157720, + "router_z_loss_clip": 0.67529297, + "router_z_loss_mlp": 0.19824219, + "step": 3738, + "time_per_iteration": 4.852168560028076 + }, + { + "auxiliary_loss_clip": 0.01014854, + "auxiliary_loss_mlp": 0.01006922, + "balance_loss_clip": 1.00420415, + "balance_loss_mlp": 1.00566411, + "epoch": 0.10849631478149846, + "flos": 62626255562880.0, + "grad_norm": 0.6523990965689072, + "language_loss": 0.53021407, + "learning_rate": 3.935720378636211e-06, + "loss": 0.55043179, + "num_input_tokens_seen": 105224310, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01257324, + "step": 3739, + "time_per_iteration": 3.113370418548584 + }, + { + "auxiliary_loss_clip": 0.01109028, + "auxiliary_loss_mlp": 0.01044657, + "balance_loss_clip": 1.03331852, + "balance_loss_mlp": 1.02348542, + "epoch": 0.10852533225001451, + "flos": 16026425239680.0, + "grad_norm": 3.2964466017758576, + "language_loss": 0.80852866, + "learning_rate": 3.935673099799355e-06, + "loss": 0.83006555, + "num_input_tokens_seen": 105237730, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.21179199, + "step": 3740, + "time_per_iteration": 2.473311185836792 + }, + { + "auxiliary_loss_clip": 0.01014344, + "auxiliary_loss_mlp": 0.01001637, + "balance_loss_clip": 1.00397909, + "balance_loss_mlp": 1.00035, + "epoch": 0.10855434971853055, + "flos": 62808432370560.0, + "grad_norm": 0.6408479891429175, + "language_loss": 0.48276204, + "learning_rate": 3.935625803865852e-06, + "loss": 0.50292182, + "num_input_tokens_seen": 105303805, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01287842, + "step": 3741, + "time_per_iteration": 3.0345733165740967 + }, + { + "auxiliary_loss_clip": 0.01102007, + "auxiliary_loss_mlp": 0.01043485, + "balance_loss_clip": 1.03266239, + "balance_loss_mlp": 1.02307594, + "epoch": 0.1085833671870466, + "flos": 35077585962240.0, + "grad_norm": 1.7901666452528202, + "language_loss": 0.74068522, + "learning_rate": 3.935578490836118e-06, + "loss": 0.7621401, + "num_input_tokens_seen": 105325460, + "router_z_loss_clip": 0.69335938, + "router_z_loss_mlp": 0.20410156, + "step": 3742, + "time_per_iteration": 2.62735652923584 + }, + { + "auxiliary_loss_clip": 0.01099615, + "auxiliary_loss_mlp": 0.01036981, + "balance_loss_clip": 1.03212166, + "balance_loss_mlp": 1.01755023, + "epoch": 0.10861238465556265, + "flos": 34451942664960.0, + "grad_norm": 1.8711832157558133, + "language_loss": 0.84817052, + "learning_rate": 3.935531160710572e-06, + "loss": 0.8695364, + "num_input_tokens_seen": 105344780, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.1942749, + "step": 3743, + "time_per_iteration": 2.514151096343994 + }, + { + "auxiliary_loss_clip": 0.01107457, + "auxiliary_loss_mlp": 0.01048037, + "balance_loss_clip": 1.03316188, + "balance_loss_mlp": 1.02334845, + "epoch": 0.10864140212407869, + "flos": 25590008482560.0, + "grad_norm": 2.523980779382946, + "language_loss": 0.98396659, + "learning_rate": 3.93548381348963e-06, + "loss": 1.00552154, + "num_input_tokens_seen": 105361105, + "router_z_loss_clip": 0.74316406, + "router_z_loss_mlp": 0.24658203, + "step": 3744, + "time_per_iteration": 2.5231635570526123 + }, + { + "auxiliary_loss_clip": 0.0101248, + "auxiliary_loss_mlp": 0.01001032, + "balance_loss_clip": 1.00206089, + "balance_loss_mlp": 0.99976212, + "epoch": 0.10867041959259474, + "flos": 49670860135680.0, + "grad_norm": 0.6818346430085638, + "language_loss": 0.46604773, + "learning_rate": 3.935436449173713e-06, + "loss": 0.48618281, + "num_input_tokens_seen": 105419590, + "router_z_loss_clip": 0.10400391, + "router_z_loss_mlp": 0.01269531, + "step": 3745, + "time_per_iteration": 2.8734967708587646 + }, + { + "auxiliary_loss_clip": 0.0110498, + "auxiliary_loss_mlp": 0.01036508, + "balance_loss_clip": 1.03394783, + "balance_loss_mlp": 1.01550317, + "epoch": 0.10869943706111079, + "flos": 28397025527040.0, + "grad_norm": 1.8308644304412336, + "language_loss": 0.76249284, + "learning_rate": 3.935389067763238e-06, + "loss": 0.78390777, + "num_input_tokens_seen": 105442125, + "router_z_loss_clip": 0.71044922, + "router_z_loss_mlp": 0.20996094, + "step": 3746, + "time_per_iteration": 2.576371192932129 + }, + { + "auxiliary_loss_clip": 0.01108186, + "auxiliary_loss_mlp": 0.01042059, + "balance_loss_clip": 1.03173232, + "balance_loss_mlp": 1.0207088, + "epoch": 0.10872845452962683, + "flos": 20007047370240.0, + "grad_norm": 2.6353659201091992, + "language_loss": 0.83345997, + "learning_rate": 3.935341669258624e-06, + "loss": 0.85496241, + "num_input_tokens_seen": 105457120, + "router_z_loss_clip": 0.76464844, + "router_z_loss_mlp": 0.21350098, + "step": 3747, + "time_per_iteration": 2.375080108642578 + }, + { + "auxiliary_loss_clip": 0.01105628, + "auxiliary_loss_mlp": 0.01049638, + "balance_loss_clip": 1.03391171, + "balance_loss_mlp": 1.02751303, + "epoch": 0.10875747199814288, + "flos": 35872279395840.0, + "grad_norm": 2.7398991020501113, + "language_loss": 0.8705318, + "learning_rate": 3.935294253660289e-06, + "loss": 0.89208448, + "num_input_tokens_seen": 105472985, + "router_z_loss_clip": 0.71777344, + "router_z_loss_mlp": 0.22119141, + "step": 3748, + "time_per_iteration": 2.5101816654205322 + }, + { + "auxiliary_loss_clip": 0.01103397, + "auxiliary_loss_mlp": 0.01046819, + "balance_loss_clip": 1.0328995, + "balance_loss_mlp": 1.02545655, + "epoch": 0.10878648946665893, + "flos": 20843741036160.0, + "grad_norm": 2.6179866741046447, + "language_loss": 0.80846584, + "learning_rate": 3.935246820968652e-06, + "loss": 0.82996798, + "num_input_tokens_seen": 105487805, + "router_z_loss_clip": 0.70410156, + "router_z_loss_mlp": 0.21368408, + "step": 3749, + "time_per_iteration": 2.3750972747802734 + }, + { + "auxiliary_loss_clip": 0.01102098, + "auxiliary_loss_mlp": 0.01038207, + "balance_loss_clip": 1.03164041, + "balance_loss_mlp": 1.01741648, + "epoch": 0.10881550693517497, + "flos": 10040810883840.0, + "grad_norm": 3.3289034687058052, + "language_loss": 1.12037206, + "learning_rate": 3.935199371184131e-06, + "loss": 1.14177513, + "num_input_tokens_seen": 105498220, + "router_z_loss_clip": 0.70410156, + "router_z_loss_mlp": 0.20788574, + "step": 3750, + "time_per_iteration": 2.3819897174835205 + }, + { + "auxiliary_loss_clip": 0.01097842, + "auxiliary_loss_mlp": 0.01041139, + "balance_loss_clip": 1.03224206, + "balance_loss_mlp": 1.02198172, + "epoch": 0.10884452440369102, + "flos": 16611150556800.0, + "grad_norm": 2.7911821126842575, + "language_loss": 0.67912173, + "learning_rate": 3.935151904307148e-06, + "loss": 0.70051157, + "num_input_tokens_seen": 105517035, + "router_z_loss_clip": 0.65576172, + "router_z_loss_mlp": 0.19152832, + "step": 3751, + "time_per_iteration": 2.5462286472320557 + }, + { + "auxiliary_loss_clip": 0.01098807, + "auxiliary_loss_mlp": 0.01042169, + "balance_loss_clip": 1.03194332, + "balance_loss_mlp": 1.02313149, + "epoch": 0.10887354187220707, + "flos": 25658368657920.0, + "grad_norm": 1.9576558363646692, + "language_loss": 0.72143841, + "learning_rate": 3.935104420338118e-06, + "loss": 0.74284822, + "num_input_tokens_seen": 105532425, + "router_z_loss_clip": 0.66821289, + "router_z_loss_mlp": 0.19024658, + "step": 3752, + "time_per_iteration": 2.542868137359619 + }, + { + "auxiliary_loss_clip": 0.01013218, + "auxiliary_loss_mlp": 0.01004117, + "balance_loss_clip": 1.00272584, + "balance_loss_mlp": 1.00299692, + "epoch": 0.10890255934072311, + "flos": 56967917091840.0, + "grad_norm": 0.6587435621620232, + "language_loss": 0.48163629, + "learning_rate": 3.935056919277464e-06, + "loss": 0.50180966, + "num_input_tokens_seen": 105596165, + "router_z_loss_clip": 0.10498047, + "router_z_loss_mlp": 0.01123047, + "step": 3753, + "time_per_iteration": 3.1932008266448975 + }, + { + "auxiliary_loss_clip": 0.01012368, + "auxiliary_loss_mlp": 0.01003236, + "balance_loss_clip": 1.0020107, + "balance_loss_mlp": 1.00211537, + "epoch": 0.10893157680923916, + "flos": 74774249821440.0, + "grad_norm": 0.6561249506468065, + "language_loss": 0.48832977, + "learning_rate": 3.935009401125604e-06, + "loss": 0.50848579, + "num_input_tokens_seen": 105661025, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01123047, + "step": 3754, + "time_per_iteration": 3.174896001815796 + }, + { + "auxiliary_loss_clip": 0.01102377, + "auxiliary_loss_mlp": 0.01037377, + "balance_loss_clip": 1.03299522, + "balance_loss_mlp": 1.01804125, + "epoch": 0.10896059427775522, + "flos": 38249096267520.0, + "grad_norm": 1.7263222976939818, + "language_loss": 0.73752534, + "learning_rate": 3.934961865882959e-06, + "loss": 0.75892282, + "num_input_tokens_seen": 105684205, + "router_z_loss_clip": 0.69384766, + "router_z_loss_mlp": 0.19348145, + "step": 3755, + "time_per_iteration": 2.5588443279266357 + }, + { + "auxiliary_loss_clip": 0.01104246, + "auxiliary_loss_mlp": 0.01038369, + "balance_loss_clip": 1.03229547, + "balance_loss_mlp": 1.01747179, + "epoch": 0.10898961174627125, + "flos": 12415044314880.0, + "grad_norm": 4.1219761067338245, + "language_loss": 0.77211392, + "learning_rate": 3.934914313549946e-06, + "loss": 0.79354012, + "num_input_tokens_seen": 105697765, + "router_z_loss_clip": 0.71826172, + "router_z_loss_mlp": 0.20898438, + "step": 3756, + "time_per_iteration": 2.342247724533081 + }, + { + "auxiliary_loss_clip": 0.01013116, + "auxiliary_loss_mlp": 0.01005573, + "balance_loss_clip": 1.00288439, + "balance_loss_mlp": 1.00445271, + "epoch": 0.1090186292147873, + "flos": 71450309053440.0, + "grad_norm": 0.6350948718832157, + "language_loss": 0.48775893, + "learning_rate": 3.934866744126988e-06, + "loss": 0.50794578, + "num_input_tokens_seen": 105755825, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01123047, + "step": 3757, + "time_per_iteration": 3.0062921047210693 + }, + { + "auxiliary_loss_clip": 0.0101233, + "auxiliary_loss_mlp": 0.01004472, + "balance_loss_clip": 1.00194037, + "balance_loss_mlp": 1.00342274, + "epoch": 0.10904764668330334, + "flos": 56969488103040.0, + "grad_norm": 0.6787670332032657, + "language_loss": 0.5098598, + "learning_rate": 3.934819157614504e-06, + "loss": 0.53002781, + "num_input_tokens_seen": 105817065, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01049805, + "step": 3758, + "time_per_iteration": 3.023026704788208 + }, + { + "auxiliary_loss_clip": 0.01100703, + "auxiliary_loss_mlp": 0.0103741, + "balance_loss_clip": 1.03195047, + "balance_loss_mlp": 1.01807451, + "epoch": 0.1090766641518194, + "flos": 16937553657600.0, + "grad_norm": 2.0099302092981977, + "language_loss": 0.70288241, + "learning_rate": 3.934771554012913e-06, + "loss": 0.72426355, + "num_input_tokens_seen": 105832095, + "router_z_loss_clip": 0.68652344, + "router_z_loss_mlp": 0.19335938, + "step": 3759, + "time_per_iteration": 2.4079837799072266 + }, + { + "auxiliary_loss_clip": 0.01110346, + "auxiliary_loss_mlp": 0.01054756, + "balance_loss_clip": 1.03536153, + "balance_loss_mlp": 1.03357208, + "epoch": 0.10910568162033545, + "flos": 13396976703360.0, + "grad_norm": 3.8245638048384976, + "language_loss": 0.90253091, + "learning_rate": 3.9347239333226375e-06, + "loss": 0.92418182, + "num_input_tokens_seen": 105843880, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.21203613, + "step": 3760, + "time_per_iteration": 2.326235055923462 + }, + { + "auxiliary_loss_clip": 0.01106132, + "auxiliary_loss_mlp": 0.01043929, + "balance_loss_clip": 1.0324012, + "balance_loss_mlp": 1.02305532, + "epoch": 0.10913469908885148, + "flos": 23870082441600.0, + "grad_norm": 3.3442323621169874, + "language_loss": 0.78819346, + "learning_rate": 3.934676295544098e-06, + "loss": 0.80969411, + "num_input_tokens_seen": 105859485, + "router_z_loss_clip": 0.73828125, + "router_z_loss_mlp": 0.20849609, + "step": 3761, + "time_per_iteration": 2.4495327472686768 + }, + { + "auxiliary_loss_clip": 0.01015272, + "auxiliary_loss_mlp": 0.01001069, + "balance_loss_clip": 1.00470233, + "balance_loss_mlp": 1.00003815, + "epoch": 0.10916371655736753, + "flos": 70821977581440.0, + "grad_norm": 0.6716446114016031, + "language_loss": 0.49911335, + "learning_rate": 3.934628640677714e-06, + "loss": 0.51927674, + "num_input_tokens_seen": 105917145, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01031494, + "step": 3762, + "time_per_iteration": 2.919025182723999 + }, + { + "auxiliary_loss_clip": 0.01099483, + "auxiliary_loss_mlp": 0.0103761, + "balance_loss_clip": 1.03181291, + "balance_loss_mlp": 1.01892436, + "epoch": 0.10919273402588359, + "flos": 13728791064960.0, + "grad_norm": 2.2048749682568767, + "language_loss": 0.66393268, + "learning_rate": 3.9345809687239065e-06, + "loss": 0.68530369, + "num_input_tokens_seen": 105930425, + "router_z_loss_clip": 0.67675781, + "router_z_loss_mlp": 0.18695068, + "step": 3763, + "time_per_iteration": 2.402919054031372 + }, + { + "auxiliary_loss_clip": 0.0110958, + "auxiliary_loss_mlp": 0.01048881, + "balance_loss_clip": 1.03421092, + "balance_loss_mlp": 1.02830553, + "epoch": 0.10922175149439962, + "flos": 74732879859840.0, + "grad_norm": 2.352858764272264, + "language_loss": 0.89720672, + "learning_rate": 3.934533279683098e-06, + "loss": 0.91879129, + "num_input_tokens_seen": 105952955, + "router_z_loss_clip": 0.75439453, + "router_z_loss_mlp": 0.20587158, + "step": 3764, + "time_per_iteration": 2.793088436126709 + }, + { + "auxiliary_loss_clip": 0.01109112, + "auxiliary_loss_mlp": 0.01050935, + "balance_loss_clip": 1.03545082, + "balance_loss_mlp": 1.02929235, + "epoch": 0.10925076896291568, + "flos": 28540658327040.0, + "grad_norm": 2.610313622468146, + "language_loss": 0.84216034, + "learning_rate": 3.934485573555708e-06, + "loss": 0.86376077, + "num_input_tokens_seen": 105966815, + "router_z_loss_clip": 0.73730469, + "router_z_loss_mlp": 0.21655273, + "step": 3765, + "time_per_iteration": 2.430006980895996 + }, + { + "auxiliary_loss_clip": 0.01093478, + "auxiliary_loss_mlp": 0.01042038, + "balance_loss_clip": 1.03193581, + "balance_loss_mlp": 1.02248764, + "epoch": 0.10927978643143173, + "flos": 23686509179520.0, + "grad_norm": 1.8949799318119076, + "language_loss": 0.7537998, + "learning_rate": 3.934437850342159e-06, + "loss": 0.77515501, + "num_input_tokens_seen": 105980520, + "router_z_loss_clip": 0.61499023, + "router_z_loss_mlp": 0.19543457, + "step": 3766, + "time_per_iteration": 2.4090332984924316 + }, + { + "auxiliary_loss_clip": 0.01015589, + "auxiliary_loss_mlp": 0.01001001, + "balance_loss_clip": 1.00486267, + "balance_loss_mlp": 0.99978536, + "epoch": 0.10930880389994777, + "flos": 73762221974400.0, + "grad_norm": 0.6557546789543942, + "language_loss": 0.49389023, + "learning_rate": 3.934390110042872e-06, + "loss": 0.51405609, + "num_input_tokens_seen": 106044065, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.012146, + "step": 3767, + "time_per_iteration": 3.147676706314087 + }, + { + "auxiliary_loss_clip": 0.01098303, + "auxiliary_loss_mlp": 0.01047276, + "balance_loss_clip": 1.03055334, + "balance_loss_mlp": 1.02761841, + "epoch": 0.10933782136846382, + "flos": 19274115864960.0, + "grad_norm": 2.5519223176299226, + "language_loss": 0.84686619, + "learning_rate": 3.934342352658268e-06, + "loss": 0.86832201, + "num_input_tokens_seen": 106057580, + "router_z_loss_clip": 0.67724609, + "router_z_loss_mlp": 0.19641113, + "step": 3768, + "time_per_iteration": 2.420863389968872 + }, + { + "auxiliary_loss_clip": 0.01014157, + "auxiliary_loss_mlp": 0.01001594, + "balance_loss_clip": 1.00356555, + "balance_loss_mlp": 1.0004791, + "epoch": 0.10936683883697987, + "flos": 71156340472320.0, + "grad_norm": 0.6685147478235162, + "language_loss": 0.48994899, + "learning_rate": 3.934294578188771e-06, + "loss": 0.51010656, + "num_input_tokens_seen": 106123215, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.01116943, + "step": 3769, + "time_per_iteration": 3.0918376445770264 + }, + { + "auxiliary_loss_clip": 0.01096816, + "auxiliary_loss_mlp": 0.01040091, + "balance_loss_clip": 1.03333664, + "balance_loss_mlp": 1.02182794, + "epoch": 0.1093958563054959, + "flos": 15770337350400.0, + "grad_norm": 2.9280373031803486, + "language_loss": 0.82200152, + "learning_rate": 3.934246786634801e-06, + "loss": 0.84337062, + "num_input_tokens_seen": 106134085, + "router_z_loss_clip": 0.63378906, + "router_z_loss_mlp": 0.18261719, + "step": 3770, + "time_per_iteration": 2.38337779045105 + }, + { + "auxiliary_loss_clip": 0.01095968, + "auxiliary_loss_mlp": 0.01036748, + "balance_loss_clip": 1.0314008, + "balance_loss_mlp": 1.01711416, + "epoch": 0.10942487377401196, + "flos": 19017713773440.0, + "grad_norm": 3.0750049865477003, + "language_loss": 0.91902304, + "learning_rate": 3.93419897799678e-06, + "loss": 0.94035017, + "num_input_tokens_seen": 106144795, + "router_z_loss_clip": 0.64550781, + "router_z_loss_mlp": 0.19641113, + "step": 3771, + "time_per_iteration": 2.366586208343506 + }, + { + "auxiliary_loss_clip": 0.01099926, + "auxiliary_loss_mlp": 0.01038061, + "balance_loss_clip": 1.03354716, + "balance_loss_mlp": 1.02019131, + "epoch": 0.109453891242528, + "flos": 21900701669760.0, + "grad_norm": 1.857914378864833, + "language_loss": 0.68494487, + "learning_rate": 3.934151152275132e-06, + "loss": 0.70632476, + "num_input_tokens_seen": 106160585, + "router_z_loss_clip": 0.66308594, + "router_z_loss_mlp": 0.17883301, + "step": 3772, + "time_per_iteration": 2.444931983947754 + }, + { + "auxiliary_loss_clip": 0.01013343, + "auxiliary_loss_mlp": 0.01007367, + "balance_loss_clip": 1.00271583, + "balance_loss_mlp": 1.0061152, + "epoch": 0.10948290871104405, + "flos": 64996264719360.0, + "grad_norm": 0.6439815320035027, + "language_loss": 0.47469258, + "learning_rate": 3.934103309470278e-06, + "loss": 0.49489966, + "num_input_tokens_seen": 106226915, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01251221, + "step": 3773, + "time_per_iteration": 3.021609306335449 + }, + { + "auxiliary_loss_clip": 0.01115511, + "auxiliary_loss_mlp": 0.01052257, + "balance_loss_clip": 1.03865886, + "balance_loss_mlp": 1.02836716, + "epoch": 0.1095119261795601, + "flos": 40983144837120.0, + "grad_norm": 2.481455734918587, + "language_loss": 0.81777781, + "learning_rate": 3.934055449582641e-06, + "loss": 0.83945554, + "num_input_tokens_seen": 106243620, + "router_z_loss_clip": 0.76953125, + "router_z_loss_mlp": 0.23901367, + "step": 3774, + "time_per_iteration": 2.6409764289855957 + }, + { + "auxiliary_loss_clip": 0.01014659, + "auxiliary_loss_mlp": 0.01004156, + "balance_loss_clip": 1.00370646, + "balance_loss_mlp": 1.00295222, + "epoch": 0.10954094364807614, + "flos": 67702382334720.0, + "grad_norm": 0.622600486486628, + "language_loss": 0.48867655, + "learning_rate": 3.934007572612643e-06, + "loss": 0.5088647, + "num_input_tokens_seen": 106311970, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01202393, + "step": 3775, + "time_per_iteration": 3.1550140380859375 + }, + { + "auxiliary_loss_clip": 0.01104388, + "auxiliary_loss_mlp": 0.01040648, + "balance_loss_clip": 1.03626537, + "balance_loss_mlp": 1.02149665, + "epoch": 0.10956996111659219, + "flos": 12230912471040.0, + "grad_norm": 3.04639645890858, + "language_loss": 0.79346764, + "learning_rate": 3.9339596785607074e-06, + "loss": 0.81491798, + "num_input_tokens_seen": 106322875, + "router_z_loss_clip": 0.68164062, + "router_z_loss_mlp": 0.19158936, + "step": 3776, + "time_per_iteration": 2.3668878078460693 + }, + { + "auxiliary_loss_clip": 0.01013886, + "auxiliary_loss_mlp": 0.0100128, + "balance_loss_clip": 1.00321746, + "balance_loss_mlp": 1.0001055, + "epoch": 0.10959897858510824, + "flos": 59917694152320.0, + "grad_norm": 0.7063923414184031, + "language_loss": 0.52244663, + "learning_rate": 3.933911767427258e-06, + "loss": 0.54259837, + "num_input_tokens_seen": 106382930, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01171875, + "step": 3777, + "time_per_iteration": 2.984170436859131 + }, + { + "auxiliary_loss_clip": 0.01102136, + "auxiliary_loss_mlp": 0.01042304, + "balance_loss_clip": 1.03269529, + "balance_loss_mlp": 1.02163887, + "epoch": 0.10962799605362428, + "flos": 30693298158720.0, + "grad_norm": 2.720386399887685, + "language_loss": 0.97165394, + "learning_rate": 3.9338638392127174e-06, + "loss": 0.99309838, + "num_input_tokens_seen": 106399915, + "router_z_loss_clip": 0.69555664, + "router_z_loss_mlp": 0.20666504, + "step": 3778, + "time_per_iteration": 2.450202465057373 + }, + { + "auxiliary_loss_clip": 0.01015422, + "auxiliary_loss_mlp": 0.01011383, + "balance_loss_clip": 1.00470293, + "balance_loss_mlp": 1.01025069, + "epoch": 0.10965701352214033, + "flos": 74770304837760.0, + "grad_norm": 0.6490429584098634, + "language_loss": 0.47460133, + "learning_rate": 3.933815893917509e-06, + "loss": 0.49486935, + "num_input_tokens_seen": 106464465, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01135254, + "step": 3779, + "time_per_iteration": 3.250304698944092 + }, + { + "auxiliary_loss_clip": 0.0101492, + "auxiliary_loss_mlp": 0.01006607, + "balance_loss_clip": 1.00404286, + "balance_loss_mlp": 1.00546813, + "epoch": 0.10968603099065638, + "flos": 57474786343680.0, + "grad_norm": 0.7229987076113082, + "language_loss": 0.49276423, + "learning_rate": 3.9337679315420555e-06, + "loss": 0.51297951, + "num_input_tokens_seen": 106514820, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01141357, + "step": 3780, + "time_per_iteration": 2.8384628295898438 + }, + { + "auxiliary_loss_clip": 0.01099784, + "auxiliary_loss_mlp": 0.01040425, + "balance_loss_clip": 1.03222537, + "balance_loss_mlp": 1.021137, + "epoch": 0.10971504845917242, + "flos": 12670643445120.0, + "grad_norm": 3.1546165926765273, + "language_loss": 0.84736496, + "learning_rate": 3.9337199520867816e-06, + "loss": 0.86876702, + "num_input_tokens_seen": 106525610, + "router_z_loss_clip": 0.67626953, + "router_z_loss_mlp": 0.19274902, + "step": 3781, + "time_per_iteration": 2.3866567611694336 + }, + { + "auxiliary_loss_clip": 0.01100794, + "auxiliary_loss_mlp": 0.01038554, + "balance_loss_clip": 1.03285062, + "balance_loss_mlp": 1.01946878, + "epoch": 0.10974406592768847, + "flos": 14493214483200.0, + "grad_norm": 2.495143582252482, + "language_loss": 0.82135403, + "learning_rate": 3.93367195555211e-06, + "loss": 0.84274757, + "num_input_tokens_seen": 106537545, + "router_z_loss_clip": 0.67919922, + "router_z_loss_mlp": 0.19091797, + "step": 3782, + "time_per_iteration": 2.3472068309783936 + }, + { + "auxiliary_loss_clip": 0.01102566, + "auxiliary_loss_mlp": 0.01045305, + "balance_loss_clip": 1.03530884, + "balance_loss_mlp": 1.02495575, + "epoch": 0.10977308339620452, + "flos": 10481135351040.0, + "grad_norm": 3.1952650357200985, + "language_loss": 0.94932806, + "learning_rate": 3.933623941938465e-06, + "loss": 0.97080672, + "num_input_tokens_seen": 106549970, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.20361328, + "step": 3783, + "time_per_iteration": 2.3530678749084473 + }, + { + "auxiliary_loss_clip": 0.0109555, + "auxiliary_loss_mlp": 0.01039434, + "balance_loss_clip": 1.03110361, + "balance_loss_mlp": 1.01999116, + "epoch": 0.10980210086472056, + "flos": 29307874654080.0, + "grad_norm": 2.2491385817828498, + "language_loss": 0.97688776, + "learning_rate": 3.933575911246272e-06, + "loss": 0.99823761, + "num_input_tokens_seen": 106565935, + "router_z_loss_clip": 0.64477539, + "router_z_loss_mlp": 0.19433594, + "step": 3784, + "time_per_iteration": 2.457639455795288 + }, + { + "auxiliary_loss_clip": 0.01103175, + "auxiliary_loss_mlp": 0.01041239, + "balance_loss_clip": 1.03124285, + "balance_loss_mlp": 1.02030563, + "epoch": 0.10983111833323661, + "flos": 42296751941760.0, + "grad_norm": 2.1640420471855375, + "language_loss": 0.89898956, + "learning_rate": 3.933527863475953e-06, + "loss": 0.9204337, + "num_input_tokens_seen": 106586710, + "router_z_loss_clip": 0.72021484, + "router_z_loss_mlp": 0.20922852, + "step": 3785, + "time_per_iteration": 2.6092162132263184 + }, + { + "auxiliary_loss_clip": 0.01107151, + "auxiliary_loss_mlp": 0.01055938, + "balance_loss_clip": 1.03502393, + "balance_loss_mlp": 1.03465891, + "epoch": 0.10986013580175266, + "flos": 12047653411200.0, + "grad_norm": 2.50221625364436, + "language_loss": 0.95897889, + "learning_rate": 3.933479798627935e-06, + "loss": 0.98060977, + "num_input_tokens_seen": 106596275, + "router_z_loss_clip": 0.72070312, + "router_z_loss_mlp": 0.21289062, + "step": 3786, + "time_per_iteration": 2.36224102973938 + }, + { + "auxiliary_loss_clip": 0.01109332, + "auxiliary_loss_mlp": 0.01043675, + "balance_loss_clip": 1.03640509, + "balance_loss_mlp": 1.02062035, + "epoch": 0.1098891532702687, + "flos": 14458161611520.0, + "grad_norm": 3.4553588982339436, + "language_loss": 0.87868512, + "learning_rate": 3.933431716702639e-06, + "loss": 0.90021527, + "num_input_tokens_seen": 106608435, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.23071289, + "step": 3787, + "time_per_iteration": 2.3653459548950195 + }, + { + "auxiliary_loss_clip": 0.01097217, + "auxiliary_loss_mlp": 0.01034155, + "balance_loss_clip": 1.03288007, + "balance_loss_mlp": 1.01691079, + "epoch": 0.10991817073878475, + "flos": 22666451719680.0, + "grad_norm": 1.946277479031412, + "language_loss": 0.58681637, + "learning_rate": 3.933383617700493e-06, + "loss": 0.6081301, + "num_input_tokens_seen": 106623665, + "router_z_loss_clip": 0.64257812, + "router_z_loss_mlp": 0.17260742, + "step": 3788, + "time_per_iteration": 2.4439964294433594 + }, + { + "auxiliary_loss_clip": 0.01102362, + "auxiliary_loss_mlp": 0.01044019, + "balance_loss_clip": 1.03383565, + "balance_loss_mlp": 1.02309811, + "epoch": 0.10994718820730079, + "flos": 22374717465600.0, + "grad_norm": 2.1729122548099933, + "language_loss": 0.92734087, + "learning_rate": 3.93333550162192e-06, + "loss": 0.94880468, + "num_input_tokens_seen": 106639960, + "router_z_loss_clip": 0.68554688, + "router_z_loss_mlp": 0.20910645, + "step": 3789, + "time_per_iteration": 2.423234224319458 + }, + { + "auxiliary_loss_clip": 0.01100507, + "auxiliary_loss_mlp": 0.01033078, + "balance_loss_clip": 1.03316474, + "balance_loss_mlp": 1.01475525, + "epoch": 0.10997620567581684, + "flos": 23543504784000.0, + "grad_norm": 1.9383669872823934, + "language_loss": 0.73018229, + "learning_rate": 3.933287368467346e-06, + "loss": 0.75151807, + "num_input_tokens_seen": 106656115, + "router_z_loss_clip": 0.67333984, + "router_z_loss_mlp": 0.18322754, + "step": 3790, + "time_per_iteration": 2.4562478065490723 + }, + { + "auxiliary_loss_clip": 0.01103905, + "auxiliary_loss_mlp": 0.01039291, + "balance_loss_clip": 1.03392267, + "balance_loss_mlp": 1.01877511, + "epoch": 0.11000522314433289, + "flos": 31491657285120.0, + "grad_norm": 2.4991980251153993, + "language_loss": 0.94548386, + "learning_rate": 3.933239218237196e-06, + "loss": 0.96691585, + "num_input_tokens_seen": 106670460, + "router_z_loss_clip": 0.69970703, + "router_z_loss_mlp": 0.20507812, + "step": 3791, + "time_per_iteration": 2.457284450531006 + }, + { + "auxiliary_loss_clip": 0.01026305, + "auxiliary_loss_mlp": 0.01008698, + "balance_loss_clip": 1.01559782, + "balance_loss_mlp": 1.00741649, + "epoch": 0.11003424061284893, + "flos": 59151071318400.0, + "grad_norm": 0.7864372181090035, + "language_loss": 0.50352567, + "learning_rate": 3.933191050931894e-06, + "loss": 0.52387571, + "num_input_tokens_seen": 106728915, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01281738, + "step": 3792, + "time_per_iteration": 2.9089553356170654 + }, + { + "auxiliary_loss_clip": 0.01025686, + "auxiliary_loss_mlp": 0.01006053, + "balance_loss_clip": 1.01488686, + "balance_loss_mlp": 1.00469422, + "epoch": 0.11006325808136498, + "flos": 71040568803840.0, + "grad_norm": 0.6734939353266061, + "language_loss": 0.46081823, + "learning_rate": 3.9331428665518665e-06, + "loss": 0.48113567, + "num_input_tokens_seen": 106787720, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01361084, + "step": 3793, + "time_per_iteration": 3.0450494289398193 + }, + { + "auxiliary_loss_clip": 0.01022664, + "auxiliary_loss_mlp": 0.01002182, + "balance_loss_clip": 1.01205313, + "balance_loss_mlp": 1.00097227, + "epoch": 0.11009227554988103, + "flos": 74764788842880.0, + "grad_norm": 0.8680775335882666, + "language_loss": 0.53760469, + "learning_rate": 3.933094665097539e-06, + "loss": 0.5578531, + "num_input_tokens_seen": 106848365, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01208496, + "step": 3794, + "time_per_iteration": 3.036532163619995 + }, + { + "auxiliary_loss_clip": 0.01102733, + "auxiliary_loss_mlp": 0.01045319, + "balance_loss_clip": 1.03183007, + "balance_loss_mlp": 1.02458787, + "epoch": 0.11012129301839707, + "flos": 16755341938560.0, + "grad_norm": 3.061598155866814, + "language_loss": 0.74281585, + "learning_rate": 3.933046446569338e-06, + "loss": 0.76429635, + "num_input_tokens_seen": 106860410, + "router_z_loss_clip": 0.70849609, + "router_z_loss_mlp": 0.20727539, + "step": 3795, + "time_per_iteration": 2.456111192703247 + }, + { + "auxiliary_loss_clip": 0.0101882, + "auxiliary_loss_mlp": 0.01002635, + "balance_loss_clip": 1.00778222, + "balance_loss_mlp": 1.0013113, + "epoch": 0.11015031048691312, + "flos": 51448288919040.0, + "grad_norm": 0.6854562729878428, + "language_loss": 0.50867373, + "learning_rate": 3.932998210967687e-06, + "loss": 0.52888823, + "num_input_tokens_seen": 106910970, + "router_z_loss_clip": 0.11035156, + "router_z_loss_mlp": 0.01324463, + "step": 3796, + "time_per_iteration": 2.7813000679016113 + }, + { + "auxiliary_loss_clip": 0.01101011, + "auxiliary_loss_mlp": 0.01045505, + "balance_loss_clip": 1.03154802, + "balance_loss_mlp": 1.02378464, + "epoch": 0.11017932795542917, + "flos": 32006486327040.0, + "grad_norm": 2.645848864211415, + "language_loss": 0.93474174, + "learning_rate": 3.932949958293015e-06, + "loss": 0.95620692, + "num_input_tokens_seen": 106928265, + "router_z_loss_clip": 0.69482422, + "router_z_loss_mlp": 0.21704102, + "step": 3797, + "time_per_iteration": 2.596949815750122 + }, + { + "auxiliary_loss_clip": 0.01101307, + "auxiliary_loss_mlp": 0.01052837, + "balance_loss_clip": 1.03459156, + "balance_loss_mlp": 1.03087258, + "epoch": 0.11020834542394521, + "flos": 54992184318720.0, + "grad_norm": 2.4096359089941846, + "language_loss": 0.86528486, + "learning_rate": 3.932901688545746e-06, + "loss": 0.88682634, + "num_input_tokens_seen": 106947835, + "router_z_loss_clip": 0.66748047, + "router_z_loss_mlp": 0.21942139, + "step": 3798, + "time_per_iteration": 2.584848165512085 + }, + { + "auxiliary_loss_clip": 0.01108898, + "auxiliary_loss_mlp": 0.01049514, + "balance_loss_clip": 1.03252554, + "balance_loss_mlp": 1.02743614, + "epoch": 0.11023736289246126, + "flos": 23985120971520.0, + "grad_norm": 2.421085105712883, + "language_loss": 0.84540206, + "learning_rate": 3.932853401726308e-06, + "loss": 0.86698622, + "num_input_tokens_seen": 106963900, + "router_z_loss_clip": 0.76269531, + "router_z_loss_mlp": 0.2208252, + "step": 3799, + "time_per_iteration": 2.478203773498535 + }, + { + "auxiliary_loss_clip": 0.01094916, + "auxiliary_loss_mlp": 0.01046011, + "balance_loss_clip": 1.03228664, + "balance_loss_mlp": 1.02851081, + "epoch": 0.11026638036097731, + "flos": 30473554861440.0, + "grad_norm": 2.1674092909109084, + "language_loss": 0.85752118, + "learning_rate": 3.932805097835125e-06, + "loss": 0.87893033, + "num_input_tokens_seen": 106978790, + "router_z_loss_clip": 0.62670898, + "router_z_loss_mlp": 0.17486572, + "step": 3800, + "time_per_iteration": 2.438952922821045 + }, + { + "auxiliary_loss_clip": 0.01098984, + "auxiliary_loss_mlp": 0.01040612, + "balance_loss_clip": 1.03399801, + "balance_loss_mlp": 1.02195621, + "epoch": 0.11029539782949335, + "flos": 49991993487360.0, + "grad_norm": 2.73995483711284, + "language_loss": 0.92451286, + "learning_rate": 3.932756776872627e-06, + "loss": 0.94590878, + "num_input_tokens_seen": 106995460, + "router_z_loss_clip": 0.65039062, + "router_z_loss_mlp": 0.18652344, + "step": 3801, + "time_per_iteration": 2.6485087871551514 + }, + { + "auxiliary_loss_clip": 0.01100306, + "auxiliary_loss_mlp": 0.01044685, + "balance_loss_clip": 1.0324719, + "balance_loss_mlp": 1.02546227, + "epoch": 0.1103244152980094, + "flos": 19055175528960.0, + "grad_norm": 2.3661216957170086, + "language_loss": 0.78177845, + "learning_rate": 3.9327084388392385e-06, + "loss": 0.80322838, + "num_input_tokens_seen": 107011535, + "router_z_loss_clip": 0.67871094, + "router_z_loss_mlp": 0.19232178, + "step": 3802, + "time_per_iteration": 2.3609466552734375 + }, + { + "auxiliary_loss_clip": 0.01016597, + "auxiliary_loss_mlp": 0.01014109, + "balance_loss_clip": 1.00583053, + "balance_loss_mlp": 1.01290536, + "epoch": 0.11035343276652544, + "flos": 68571370874880.0, + "grad_norm": 2.173756149570518, + "language_loss": 0.5016951, + "learning_rate": 3.932660083735387e-06, + "loss": 0.52200216, + "num_input_tokens_seen": 107077805, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01202393, + "step": 3803, + "time_per_iteration": 3.0863964557647705 + }, + { + "auxiliary_loss_clip": 0.01015617, + "auxiliary_loss_mlp": 0.01016619, + "balance_loss_clip": 1.00468278, + "balance_loss_mlp": 1.01534951, + "epoch": 0.1103824502350415, + "flos": 74770793596800.0, + "grad_norm": 1.0814822232202181, + "language_loss": 0.50663042, + "learning_rate": 3.932611711561499e-06, + "loss": 0.52695274, + "num_input_tokens_seen": 107138635, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01269531, + "step": 3804, + "time_per_iteration": 3.0971243381500244 + }, + { + "auxiliary_loss_clip": 0.01013951, + "auxiliary_loss_mlp": 0.0100639, + "balance_loss_clip": 1.00364041, + "balance_loss_mlp": 1.00518632, + "epoch": 0.11041146770355754, + "flos": 74774075264640.0, + "grad_norm": 0.6607584059661857, + "language_loss": 0.45264322, + "learning_rate": 3.932563322318002e-06, + "loss": 0.47284663, + "num_input_tokens_seen": 107202275, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01202393, + "step": 3805, + "time_per_iteration": 7.607380628585815 + }, + { + "auxiliary_loss_clip": 0.0101354, + "auxiliary_loss_mlp": 0.0100391, + "balance_loss_clip": 1.00293517, + "balance_loss_mlp": 1.00272942, + "epoch": 0.11044048517207358, + "flos": 71223862775040.0, + "grad_norm": 0.6692008797146404, + "language_loss": 0.49754369, + "learning_rate": 3.932514916005325e-06, + "loss": 0.5177182, + "num_input_tokens_seen": 107265980, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01177979, + "step": 3806, + "time_per_iteration": 3.2379305362701416 + }, + { + "auxiliary_loss_clip": 0.01092661, + "auxiliary_loss_mlp": 0.01036138, + "balance_loss_clip": 1.02992678, + "balance_loss_mlp": 1.0170821, + "epoch": 0.11046950264058963, + "flos": 16389801336960.0, + "grad_norm": 2.539234030540694, + "language_loss": 0.82369626, + "learning_rate": 3.932466492623894e-06, + "loss": 0.84498423, + "num_input_tokens_seen": 107277875, + "router_z_loss_clip": 0.62719727, + "router_z_loss_mlp": 0.19042969, + "step": 3807, + "time_per_iteration": 2.399102210998535 + }, + { + "auxiliary_loss_clip": 0.01100696, + "auxiliary_loss_mlp": 0.01038575, + "balance_loss_clip": 1.03317714, + "balance_loss_mlp": 1.01864338, + "epoch": 0.11049852010910569, + "flos": 27888445618560.0, + "grad_norm": 2.0135334547192913, + "language_loss": 0.79141706, + "learning_rate": 3.932418052174136e-06, + "loss": 0.81280977, + "num_input_tokens_seen": 107297210, + "router_z_loss_clip": 0.67578125, + "router_z_loss_mlp": 0.19909668, + "step": 3808, + "time_per_iteration": 2.5465915203094482 + }, + { + "auxiliary_loss_clip": 0.01104608, + "auxiliary_loss_mlp": 0.01043968, + "balance_loss_clip": 1.03797543, + "balance_loss_mlp": 1.02552629, + "epoch": 0.11052753757762172, + "flos": 13069910286720.0, + "grad_norm": 2.479579776796168, + "language_loss": 0.74157774, + "learning_rate": 3.9323695946564805e-06, + "loss": 0.76306343, + "num_input_tokens_seen": 107308680, + "router_z_loss_clip": 0.66601562, + "router_z_loss_mlp": 0.18469238, + "step": 3809, + "time_per_iteration": 2.338622570037842 + }, + { + "auxiliary_loss_clip": 0.01114058, + "auxiliary_loss_mlp": 0.01057254, + "balance_loss_clip": 1.04029489, + "balance_loss_mlp": 1.03541517, + "epoch": 0.11055655504613777, + "flos": 38653844192640.0, + "grad_norm": 2.154864066707298, + "language_loss": 0.809421, + "learning_rate": 3.932321120071355e-06, + "loss": 0.83113414, + "num_input_tokens_seen": 107330750, + "router_z_loss_clip": 0.73828125, + "router_z_loss_mlp": 0.21850586, + "step": 3810, + "time_per_iteration": 2.600876808166504 + }, + { + "auxiliary_loss_clip": 0.01110691, + "auxiliary_loss_mlp": 0.01051231, + "balance_loss_clip": 1.03752208, + "balance_loss_mlp": 1.03021455, + "epoch": 0.11058557251465383, + "flos": 20919083483520.0, + "grad_norm": 2.194009343393006, + "language_loss": 0.83647239, + "learning_rate": 3.932272628419187e-06, + "loss": 0.85809159, + "num_input_tokens_seen": 107344410, + "router_z_loss_clip": 0.73193359, + "router_z_loss_mlp": 0.20996094, + "step": 3811, + "time_per_iteration": 2.368356466293335 + }, + { + "auxiliary_loss_clip": 0.01019491, + "auxiliary_loss_mlp": 0.01031337, + "balance_loss_clip": 1.0080682, + "balance_loss_mlp": 1.02995396, + "epoch": 0.11061458998316986, + "flos": 67437392048640.0, + "grad_norm": 0.6953122185432394, + "language_loss": 0.45839342, + "learning_rate": 3.932224119700406e-06, + "loss": 0.47890168, + "num_input_tokens_seen": 107408730, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01385498, + "step": 3812, + "time_per_iteration": 5.545523405075073 + }, + { + "auxiliary_loss_clip": 0.0110734, + "auxiliary_loss_mlp": 0.01045978, + "balance_loss_clip": 1.03616643, + "balance_loss_mlp": 1.02556956, + "epoch": 0.11064360745168592, + "flos": 28395698895360.0, + "grad_norm": 2.1440794723650316, + "language_loss": 0.64384168, + "learning_rate": 3.932175593915439e-06, + "loss": 0.66537488, + "num_input_tokens_seen": 107435865, + "router_z_loss_clip": 0.71289062, + "router_z_loss_mlp": 0.20410156, + "step": 3813, + "time_per_iteration": 2.849813938140869 + }, + { + "auxiliary_loss_clip": 0.01104132, + "auxiliary_loss_mlp": 0.01037086, + "balance_loss_clip": 1.03685915, + "balance_loss_mlp": 1.01827455, + "epoch": 0.11067262492020197, + "flos": 27116760637440.0, + "grad_norm": 2.850184488794805, + "language_loss": 1.06004214, + "learning_rate": 3.932127051064714e-06, + "loss": 1.08145428, + "num_input_tokens_seen": 107449955, + "router_z_loss_clip": 0.67285156, + "router_z_loss_mlp": 0.18823242, + "step": 3814, + "time_per_iteration": 4.994329929351807 + }, + { + "auxiliary_loss_clip": 0.01018674, + "auxiliary_loss_mlp": 0.01024346, + "balance_loss_clip": 1.00738943, + "balance_loss_mlp": 1.02299285, + "epoch": 0.110701642388718, + "flos": 66670001164800.0, + "grad_norm": 0.7886576806332385, + "language_loss": 0.50390208, + "learning_rate": 3.932078491148663e-06, + "loss": 0.52433228, + "num_input_tokens_seen": 107505585, + "router_z_loss_clip": 0.11279297, + "router_z_loss_mlp": 0.0135498, + "step": 3815, + "time_per_iteration": 2.924440860748291 + }, + { + "auxiliary_loss_clip": 0.0101667, + "auxiliary_loss_mlp": 0.01014402, + "balance_loss_clip": 1.00557876, + "balance_loss_mlp": 1.0132637, + "epoch": 0.11073065985723406, + "flos": 65976032603520.0, + "grad_norm": 0.7299592896792849, + "language_loss": 0.50389028, + "learning_rate": 3.932029914167712e-06, + "loss": 0.52420104, + "num_input_tokens_seen": 107566995, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.01141357, + "step": 3816, + "time_per_iteration": 3.1516823768615723 + }, + { + "auxiliary_loss_clip": 0.0110437, + "auxiliary_loss_mlp": 0.01037105, + "balance_loss_clip": 1.03462625, + "balance_loss_mlp": 1.01784074, + "epoch": 0.11075967732575011, + "flos": 34707471972480.0, + "grad_norm": 2.1810754436584916, + "language_loss": 0.83161247, + "learning_rate": 3.931981320122292e-06, + "loss": 0.85302722, + "num_input_tokens_seen": 107582890, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.19274902, + "step": 3817, + "time_per_iteration": 2.5783915519714355 + }, + { + "auxiliary_loss_clip": 0.01098349, + "auxiliary_loss_mlp": 0.01037098, + "balance_loss_clip": 1.03413868, + "balance_loss_mlp": 1.02003908, + "epoch": 0.11078869479426615, + "flos": 23693875476480.0, + "grad_norm": 2.1483296040590774, + "language_loss": 0.71567047, + "learning_rate": 3.93193270901283e-06, + "loss": 0.73702496, + "num_input_tokens_seen": 107597725, + "router_z_loss_clip": 0.64208984, + "router_z_loss_mlp": 0.1706543, + "step": 3818, + "time_per_iteration": 2.3942785263061523 + }, + { + "auxiliary_loss_clip": 0.01102412, + "auxiliary_loss_mlp": 0.01040631, + "balance_loss_clip": 1.03501499, + "balance_loss_mlp": 1.02209401, + "epoch": 0.1108177122627822, + "flos": 10042032781440.0, + "grad_norm": 2.4575495790312996, + "language_loss": 0.93269747, + "learning_rate": 3.931884080839757e-06, + "loss": 0.95412791, + "num_input_tokens_seen": 107607315, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.18530273, + "step": 3819, + "time_per_iteration": 2.3686771392822266 + }, + { + "auxiliary_loss_clip": 0.01016455, + "auxiliary_loss_mlp": 0.01014374, + "balance_loss_clip": 1.00596559, + "balance_loss_mlp": 1.01313996, + "epoch": 0.11084672973129824, + "flos": 58976400453120.0, + "grad_norm": 0.8235151992633186, + "language_loss": 0.51988578, + "learning_rate": 3.931835435603502e-06, + "loss": 0.54019403, + "num_input_tokens_seen": 107664805, + "router_z_loss_clip": 0.10498047, + "router_z_loss_mlp": 0.0123291, + "step": 3820, + "time_per_iteration": 2.947117805480957 + }, + { + "auxiliary_loss_clip": 0.01104982, + "auxiliary_loss_mlp": 0.01049525, + "balance_loss_clip": 1.03204775, + "balance_loss_mlp": 1.02873516, + "epoch": 0.11087574719981429, + "flos": 23432446149120.0, + "grad_norm": 2.571791580931644, + "language_loss": 0.94489682, + "learning_rate": 3.931786773304494e-06, + "loss": 0.96644187, + "num_input_tokens_seen": 107678655, + "router_z_loss_clip": 0.72827148, + "router_z_loss_mlp": 0.20782471, + "step": 3821, + "time_per_iteration": 2.4061474800109863 + }, + { + "auxiliary_loss_clip": 0.01017393, + "auxiliary_loss_mlp": 0.01025094, + "balance_loss_clip": 1.00715983, + "balance_loss_mlp": 1.0239737, + "epoch": 0.11090476466833034, + "flos": 66780256838400.0, + "grad_norm": 0.6418585845466847, + "language_loss": 0.46163267, + "learning_rate": 3.931738093943165e-06, + "loss": 0.48205757, + "num_input_tokens_seen": 107735380, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01123047, + "step": 3822, + "time_per_iteration": 2.941627025604248 + }, + { + "auxiliary_loss_clip": 0.01090274, + "auxiliary_loss_mlp": 0.01041506, + "balance_loss_clip": 1.02945638, + "balance_loss_mlp": 1.02470994, + "epoch": 0.11093378213684638, + "flos": 21170842364160.0, + "grad_norm": 4.058954419113409, + "language_loss": 0.90077376, + "learning_rate": 3.931689397519943e-06, + "loss": 0.92209166, + "num_input_tokens_seen": 107748560, + "router_z_loss_clip": 0.60717773, + "router_z_loss_mlp": 0.16784668, + "step": 3823, + "time_per_iteration": 2.4194374084472656 + }, + { + "auxiliary_loss_clip": 0.01101084, + "auxiliary_loss_mlp": 0.01043911, + "balance_loss_clip": 1.03230524, + "balance_loss_mlp": 1.0233947, + "epoch": 0.11096279960536243, + "flos": 21097629509760.0, + "grad_norm": 2.6471491880280604, + "language_loss": 0.9112801, + "learning_rate": 3.931640684035258e-06, + "loss": 0.93273002, + "num_input_tokens_seen": 107761190, + "router_z_loss_clip": 0.68798828, + "router_z_loss_mlp": 0.20526123, + "step": 3824, + "time_per_iteration": 2.3781042098999023 + }, + { + "auxiliary_loss_clip": 0.01105081, + "auxiliary_loss_mlp": 0.01053868, + "balance_loss_clip": 1.03483272, + "balance_loss_mlp": 1.032565, + "epoch": 0.11099181707387848, + "flos": 22703599272960.0, + "grad_norm": 2.365930983019167, + "language_loss": 0.76974094, + "learning_rate": 3.9315919534895415e-06, + "loss": 0.7913304, + "num_input_tokens_seen": 107776280, + "router_z_loss_clip": 0.70263672, + "router_z_loss_mlp": 0.21313477, + "step": 3825, + "time_per_iteration": 2.451756000518799 + }, + { + "auxiliary_loss_clip": 0.01099851, + "auxiliary_loss_mlp": 0.01040376, + "balance_loss_clip": 1.03166187, + "balance_loss_mlp": 1.0214572, + "epoch": 0.11102083454239452, + "flos": 16611185468160.0, + "grad_norm": 3.026812019606791, + "language_loss": 0.77697545, + "learning_rate": 3.931543205883223e-06, + "loss": 0.79837775, + "num_input_tokens_seen": 107790805, + "router_z_loss_clip": 0.68115234, + "router_z_loss_mlp": 0.18920898, + "step": 3826, + "time_per_iteration": 2.3454694747924805 + }, + { + "auxiliary_loss_clip": 0.0110431, + "auxiliary_loss_mlp": 0.0104419, + "balance_loss_clip": 1.03376019, + "balance_loss_mlp": 1.02402592, + "epoch": 0.11104985201091057, + "flos": 16250462634240.0, + "grad_norm": 2.051062463774624, + "language_loss": 0.73097444, + "learning_rate": 3.931494441216733e-06, + "loss": 0.75245941, + "num_input_tokens_seen": 107805655, + "router_z_loss_clip": 0.70605469, + "router_z_loss_mlp": 0.20147705, + "step": 3827, + "time_per_iteration": 2.3772897720336914 + }, + { + "auxiliary_loss_clip": 0.01103452, + "auxiliary_loss_mlp": 0.01041034, + "balance_loss_clip": 1.03322613, + "balance_loss_mlp": 1.01945722, + "epoch": 0.11107886947942662, + "flos": 26360576300160.0, + "grad_norm": 2.0558528596583017, + "language_loss": 0.79359806, + "learning_rate": 3.931445659490502e-06, + "loss": 0.81504297, + "num_input_tokens_seen": 107825105, + "router_z_loss_clip": 0.703125, + "router_z_loss_mlp": 0.21569824, + "step": 3828, + "time_per_iteration": 2.4331820011138916 + }, + { + "auxiliary_loss_clip": 0.01093929, + "auxiliary_loss_mlp": 0.01043086, + "balance_loss_clip": 1.03070712, + "balance_loss_mlp": 1.02433395, + "epoch": 0.11110788694794266, + "flos": 34706319897600.0, + "grad_norm": 2.3359912650683277, + "language_loss": 0.80091965, + "learning_rate": 3.931396860704963e-06, + "loss": 0.82228976, + "num_input_tokens_seen": 107842550, + "router_z_loss_clip": 0.63256836, + "router_z_loss_mlp": 0.18762207, + "step": 3829, + "time_per_iteration": 2.547792673110962 + }, + { + "auxiliary_loss_clip": 0.01103381, + "auxiliary_loss_mlp": 0.01043861, + "balance_loss_clip": 1.03519201, + "balance_loss_mlp": 1.02383351, + "epoch": 0.11113690441645871, + "flos": 36127948348800.0, + "grad_norm": 2.6028905531089785, + "language_loss": 0.87898016, + "learning_rate": 3.931348044860544e-06, + "loss": 0.90045273, + "num_input_tokens_seen": 107858815, + "router_z_loss_clip": 0.68212891, + "router_z_loss_mlp": 0.20043945, + "step": 3830, + "time_per_iteration": 2.527750015258789 + }, + { + "auxiliary_loss_clip": 0.01102343, + "auxiliary_loss_mlp": 0.01038891, + "balance_loss_clip": 1.03367245, + "balance_loss_mlp": 1.01972198, + "epoch": 0.11116592188497476, + "flos": 33214027121280.0, + "grad_norm": 6.543278628560888, + "language_loss": 0.804582, + "learning_rate": 3.931299211957678e-06, + "loss": 0.82599437, + "num_input_tokens_seen": 107878770, + "router_z_loss_clip": 0.68725586, + "router_z_loss_mlp": 0.19152832, + "step": 3831, + "time_per_iteration": 2.5529515743255615 + }, + { + "auxiliary_loss_clip": 0.01024213, + "auxiliary_loss_mlp": 0.01005519, + "balance_loss_clip": 1.01330948, + "balance_loss_mlp": 1.00454128, + "epoch": 0.1111949393534908, + "flos": 63174950490240.0, + "grad_norm": 0.6700183089965209, + "language_loss": 0.51619482, + "learning_rate": 3.931250361996796e-06, + "loss": 0.53649211, + "num_input_tokens_seen": 107940600, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.00976562, + "step": 3832, + "time_per_iteration": 2.9944698810577393 + }, + { + "auxiliary_loss_clip": 0.0102106, + "auxiliary_loss_mlp": 0.01003728, + "balance_loss_clip": 1.01027513, + "balance_loss_mlp": 1.00270832, + "epoch": 0.11122395682200685, + "flos": 55984343869440.0, + "grad_norm": 0.7503009774018943, + "language_loss": 0.57278419, + "learning_rate": 3.931201494978329e-06, + "loss": 0.593032, + "num_input_tokens_seen": 107997445, + "router_z_loss_clip": 0.10791016, + "router_z_loss_mlp": 0.01019287, + "step": 3833, + "time_per_iteration": 2.9569356441497803 + }, + { + "auxiliary_loss_clip": 0.01020167, + "auxiliary_loss_mlp": 0.01000518, + "balance_loss_clip": 1.00956225, + "balance_loss_mlp": 0.99953496, + "epoch": 0.11125297429052289, + "flos": 64800647038080.0, + "grad_norm": 0.7124662928826853, + "language_loss": 0.53390896, + "learning_rate": 3.931152610902709e-06, + "loss": 0.55411577, + "num_input_tokens_seen": 108054745, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.00982666, + "step": 3834, + "time_per_iteration": 2.9534130096435547 + }, + { + "auxiliary_loss_clip": 0.01095076, + "auxiliary_loss_mlp": 0.01041745, + "balance_loss_clip": 1.03232145, + "balance_loss_mlp": 1.024019, + "epoch": 0.11128199175903894, + "flos": 38134476673920.0, + "grad_norm": 2.352571503825622, + "language_loss": 0.75573331, + "learning_rate": 3.931103709770367e-06, + "loss": 0.77710152, + "num_input_tokens_seen": 108069225, + "router_z_loss_clip": 0.62841797, + "router_z_loss_mlp": 0.17724609, + "step": 3835, + "time_per_iteration": 2.438084602355957 + }, + { + "auxiliary_loss_clip": 0.01017415, + "auxiliary_loss_mlp": 0.01001146, + "balance_loss_clip": 1.00689209, + "balance_loss_mlp": 1.00005519, + "epoch": 0.11131100922755499, + "flos": 68061499246080.0, + "grad_norm": 0.6892284958969922, + "language_loss": 0.52575892, + "learning_rate": 3.931054791581737e-06, + "loss": 0.54594451, + "num_input_tokens_seen": 108131370, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01092529, + "step": 3836, + "time_per_iteration": 3.0512819290161133 + }, + { + "auxiliary_loss_clip": 0.01101735, + "auxiliary_loss_mlp": 0.01037023, + "balance_loss_clip": 1.034657, + "balance_loss_mlp": 1.01719892, + "epoch": 0.11134002669607103, + "flos": 16794235059840.0, + "grad_norm": 2.4248090341956625, + "language_loss": 0.83077025, + "learning_rate": 3.931005856337249e-06, + "loss": 0.85215783, + "num_input_tokens_seen": 108144760, + "router_z_loss_clip": 0.67138672, + "router_z_loss_mlp": 0.19799805, + "step": 3837, + "time_per_iteration": 2.525991678237915 + }, + { + "auxiliary_loss_clip": 0.01104309, + "auxiliary_loss_mlp": 0.01036664, + "balance_loss_clip": 1.03198099, + "balance_loss_mlp": 1.01561141, + "epoch": 0.11136904416458708, + "flos": 20587513501440.0, + "grad_norm": 2.241870333551526, + "language_loss": 0.91774517, + "learning_rate": 3.930956904037335e-06, + "loss": 0.93915492, + "num_input_tokens_seen": 108159845, + "router_z_loss_clip": 0.72314453, + "router_z_loss_mlp": 0.21057129, + "step": 3838, + "time_per_iteration": 2.3710081577301025 + }, + { + "auxiliary_loss_clip": 0.01015755, + "auxiliary_loss_mlp": 0.0100463, + "balance_loss_clip": 1.0054121, + "balance_loss_mlp": 1.00356936, + "epoch": 0.11139806163310313, + "flos": 74766220208640.0, + "grad_norm": 0.6534967397391233, + "language_loss": 0.45815885, + "learning_rate": 3.930907934682429e-06, + "loss": 0.47836268, + "num_input_tokens_seen": 108221375, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01062012, + "step": 3839, + "time_per_iteration": 3.0516538619995117 + }, + { + "auxiliary_loss_clip": 0.01014731, + "auxiliary_loss_mlp": 0.01005566, + "balance_loss_clip": 1.00448585, + "balance_loss_mlp": 1.00455272, + "epoch": 0.11142707910161917, + "flos": 61455967056000.0, + "grad_norm": 0.6928783504303193, + "language_loss": 0.48209831, + "learning_rate": 3.930858948272964e-06, + "loss": 0.50230134, + "num_input_tokens_seen": 108285490, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01013184, + "step": 3840, + "time_per_iteration": 3.0839107036590576 + }, + { + "auxiliary_loss_clip": 0.01101136, + "auxiliary_loss_mlp": 0.01038692, + "balance_loss_clip": 1.03238523, + "balance_loss_mlp": 1.01846218, + "epoch": 0.11145609657013522, + "flos": 36969285225600.0, + "grad_norm": 2.2653643999983917, + "language_loss": 0.72737336, + "learning_rate": 3.93080994480937e-06, + "loss": 0.74877167, + "num_input_tokens_seen": 108302125, + "router_z_loss_clip": 0.6875, + "router_z_loss_mlp": 0.20239258, + "step": 3841, + "time_per_iteration": 2.5000998973846436 + }, + { + "auxiliary_loss_clip": 0.0109121, + "auxiliary_loss_mlp": 0.0103151, + "balance_loss_clip": 1.03189278, + "balance_loss_mlp": 1.01463592, + "epoch": 0.11148511403865127, + "flos": 9237983103360.0, + "grad_norm": 4.856194059640731, + "language_loss": 0.94527376, + "learning_rate": 3.930760924292081e-06, + "loss": 0.966501, + "num_input_tokens_seen": 108312200, + "router_z_loss_clip": 0.59228516, + "router_z_loss_mlp": 0.16876221, + "step": 3842, + "time_per_iteration": 2.4021832942962646 + }, + { + "auxiliary_loss_clip": 0.01017751, + "auxiliary_loss_mlp": 0.01003454, + "balance_loss_clip": 1.00739992, + "balance_loss_mlp": 1.00244117, + "epoch": 0.11151413150716731, + "flos": 65467417783680.0, + "grad_norm": 0.6418993301566057, + "language_loss": 0.47326282, + "learning_rate": 3.930711886721531e-06, + "loss": 0.49347487, + "num_input_tokens_seen": 108375655, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01013184, + "step": 3843, + "time_per_iteration": 2.9608778953552246 + }, + { + "auxiliary_loss_clip": 0.01112921, + "auxiliary_loss_mlp": 0.01048547, + "balance_loss_clip": 1.03714275, + "balance_loss_mlp": 1.02632594, + "epoch": 0.11154314897568336, + "flos": 53094166099200.0, + "grad_norm": 3.1220745166900863, + "language_loss": 0.99664956, + "learning_rate": 3.930662832098153e-06, + "loss": 1.01826417, + "num_input_tokens_seen": 108398380, + "router_z_loss_clip": 0.75878906, + "router_z_loss_mlp": 0.22216797, + "step": 3844, + "time_per_iteration": 2.7381718158721924 + }, + { + "auxiliary_loss_clip": 0.01022412, + "auxiliary_loss_mlp": 0.01001481, + "balance_loss_clip": 1.01155722, + "balance_loss_mlp": 1.000283, + "epoch": 0.11157216644419941, + "flos": 69591567980160.0, + "grad_norm": 0.6393742340854768, + "language_loss": 0.45209879, + "learning_rate": 3.930613760422378e-06, + "loss": 0.47233772, + "num_input_tokens_seen": 108456785, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01196289, + "step": 3845, + "time_per_iteration": 2.9765641689300537 + }, + { + "auxiliary_loss_clip": 0.01023871, + "auxiliary_loss_mlp": 0.01000201, + "balance_loss_clip": 1.01278377, + "balance_loss_mlp": 0.9990145, + "epoch": 0.11160118391271545, + "flos": 61820983987200.0, + "grad_norm": 0.6500807699339239, + "language_loss": 0.41494805, + "learning_rate": 3.930564671694641e-06, + "loss": 0.43518877, + "num_input_tokens_seen": 108519430, + "router_z_loss_clip": 0.11083984, + "router_z_loss_mlp": 0.01184082, + "step": 3846, + "time_per_iteration": 2.939427614212036 + }, + { + "auxiliary_loss_clip": 0.01102321, + "auxiliary_loss_mlp": 0.01048595, + "balance_loss_clip": 1.03479743, + "balance_loss_mlp": 1.02887797, + "epoch": 0.1116302013812315, + "flos": 16245749600640.0, + "grad_norm": 2.5384964961771015, + "language_loss": 0.73320329, + "learning_rate": 3.930515565915377e-06, + "loss": 0.75471246, + "num_input_tokens_seen": 108531290, + "router_z_loss_clip": 0.67578125, + "router_z_loss_mlp": 0.19714355, + "step": 3847, + "time_per_iteration": 2.3397979736328125 + }, + { + "auxiliary_loss_clip": 0.01116947, + "auxiliary_loss_mlp": 0.01056554, + "balance_loss_clip": 1.03836536, + "balance_loss_mlp": 1.03249764, + "epoch": 0.11165921884974755, + "flos": 22812249024000.0, + "grad_norm": 2.624594378001188, + "language_loss": 1.01466751, + "learning_rate": 3.930466443085018e-06, + "loss": 1.03640246, + "num_input_tokens_seen": 108545640, + "router_z_loss_clip": 0.78564453, + "router_z_loss_mlp": 0.24072266, + "step": 3848, + "time_per_iteration": 2.449455738067627 + }, + { + "auxiliary_loss_clip": 0.01107664, + "auxiliary_loss_mlp": 0.01053144, + "balance_loss_clip": 1.03697217, + "balance_loss_mlp": 1.03151965, + "epoch": 0.11168823631826359, + "flos": 20410678131840.0, + "grad_norm": 2.5220034446083655, + "language_loss": 0.82113135, + "learning_rate": 3.930417303203997e-06, + "loss": 0.84273946, + "num_input_tokens_seen": 108560995, + "router_z_loss_clip": 0.70751953, + "router_z_loss_mlp": 0.21618652, + "step": 3849, + "time_per_iteration": 2.3734235763549805 + }, + { + "auxiliary_loss_clip": 0.01108875, + "auxiliary_loss_mlp": 0.01053592, + "balance_loss_clip": 1.03661537, + "balance_loss_mlp": 1.0321579, + "epoch": 0.11171725378677964, + "flos": 19930203734400.0, + "grad_norm": 3.0716746389604705, + "language_loss": 0.90120649, + "learning_rate": 3.9303681462727505e-06, + "loss": 0.92283106, + "num_input_tokens_seen": 108574765, + "router_z_loss_clip": 0.72314453, + "router_z_loss_mlp": 0.21435547, + "step": 3850, + "time_per_iteration": 2.3946495056152344 + }, + { + "auxiliary_loss_clip": 0.01022339, + "auxiliary_loss_mlp": 0.01032959, + "balance_loss_clip": 1.01147199, + "balance_loss_mlp": 1.03182077, + "epoch": 0.11174627125529568, + "flos": 61039942761600.0, + "grad_norm": 0.7962230129142249, + "language_loss": 0.49541578, + "learning_rate": 3.9303189722917115e-06, + "loss": 0.5159688, + "num_input_tokens_seen": 108625990, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01141357, + "step": 3851, + "time_per_iteration": 2.8088629245758057 + }, + { + "auxiliary_loss_clip": 0.01108147, + "auxiliary_loss_mlp": 0.01051759, + "balance_loss_clip": 1.03714859, + "balance_loss_mlp": 1.02927661, + "epoch": 0.11177528872381173, + "flos": 25621779686400.0, + "grad_norm": 2.2925630214640216, + "language_loss": 0.85796785, + "learning_rate": 3.930269781261313e-06, + "loss": 0.87956691, + "num_input_tokens_seen": 108644670, + "router_z_loss_clip": 0.70996094, + "router_z_loss_mlp": 0.22485352, + "step": 3852, + "time_per_iteration": 2.6810429096221924 + }, + { + "auxiliary_loss_clip": 0.01020479, + "auxiliary_loss_mlp": 0.01038697, + "balance_loss_clip": 1.0096643, + "balance_loss_mlp": 1.03756499, + "epoch": 0.11180430619232778, + "flos": 52284738205440.0, + "grad_norm": 0.6924032501607907, + "language_loss": 0.48590618, + "learning_rate": 3.930220573181992e-06, + "loss": 0.50649792, + "num_input_tokens_seen": 108693975, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01135254, + "step": 3853, + "time_per_iteration": 2.7731692790985107 + }, + { + "auxiliary_loss_clip": 0.01020104, + "auxiliary_loss_mlp": 0.01027784, + "balance_loss_clip": 1.00919878, + "balance_loss_mlp": 1.02665162, + "epoch": 0.11183332366084382, + "flos": 64615048917120.0, + "grad_norm": 0.6886013159862232, + "language_loss": 0.46894962, + "learning_rate": 3.930171348054181e-06, + "loss": 0.48942852, + "num_input_tokens_seen": 108754200, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01135254, + "step": 3854, + "time_per_iteration": 3.0743894577026367 + }, + { + "auxiliary_loss_clip": 0.01023841, + "auxiliary_loss_mlp": 0.0101712, + "balance_loss_clip": 1.01269317, + "balance_loss_mlp": 1.0160774, + "epoch": 0.11186234112935987, + "flos": 73931481578880.0, + "grad_norm": 0.6501921940731602, + "language_loss": 0.50058103, + "learning_rate": 3.9301221058783155e-06, + "loss": 0.52099067, + "num_input_tokens_seen": 108817245, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.01043701, + "step": 3855, + "time_per_iteration": 3.0740792751312256 + }, + { + "auxiliary_loss_clip": 0.01110308, + "auxiliary_loss_mlp": 0.01043237, + "balance_loss_clip": 1.03838658, + "balance_loss_mlp": 1.01960957, + "epoch": 0.11189135859787593, + "flos": 20368258963200.0, + "grad_norm": 2.320869515421762, + "language_loss": 0.92427123, + "learning_rate": 3.930072846654831e-06, + "loss": 0.94580674, + "num_input_tokens_seen": 108830430, + "router_z_loss_clip": 0.71875, + "router_z_loss_mlp": 0.23620605, + "step": 3856, + "time_per_iteration": 2.3900089263916016 + }, + { + "auxiliary_loss_clip": 0.01100112, + "auxiliary_loss_mlp": 0.01036467, + "balance_loss_clip": 1.03961158, + "balance_loss_mlp": 1.02048099, + "epoch": 0.11192037606639196, + "flos": 34048800662400.0, + "grad_norm": 1.955781914888725, + "language_loss": 0.82950795, + "learning_rate": 3.930023570384162e-06, + "loss": 0.85087371, + "num_input_tokens_seen": 108847140, + "router_z_loss_clip": 0.60522461, + "router_z_loss_mlp": 0.16003418, + "step": 3857, + "time_per_iteration": 2.603360414505005 + }, + { + "auxiliary_loss_clip": 0.01044497, + "auxiliary_loss_mlp": 0.01001931, + "balance_loss_clip": 1.03124428, + "balance_loss_mlp": 1.00076854, + "epoch": 0.11194939353490801, + "flos": 74765242690560.0, + "grad_norm": 0.6338144365437134, + "language_loss": 0.47410572, + "learning_rate": 3.929974277066744e-06, + "loss": 0.49456996, + "num_input_tokens_seen": 108914345, + "router_z_loss_clip": 0.1328125, + "router_z_loss_mlp": 0.01159668, + "step": 3858, + "time_per_iteration": 3.255509376525879 + }, + { + "auxiliary_loss_clip": 0.01042998, + "auxiliary_loss_mlp": 0.01011367, + "balance_loss_clip": 1.03028631, + "balance_loss_mlp": 1.01025271, + "epoch": 0.11197841100342407, + "flos": 63354439123200.0, + "grad_norm": 0.6572509083461396, + "language_loss": 0.4808141, + "learning_rate": 3.9299249667030115e-06, + "loss": 0.50135773, + "num_input_tokens_seen": 108975070, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01116943, + "step": 3859, + "time_per_iteration": 2.987898588180542 + }, + { + "auxiliary_loss_clip": 0.01106339, + "auxiliary_loss_mlp": 0.01052439, + "balance_loss_clip": 1.03945196, + "balance_loss_mlp": 1.03181553, + "epoch": 0.1120074284719401, + "flos": 21574298568960.0, + "grad_norm": 1.937725562846395, + "language_loss": 0.87129271, + "learning_rate": 3.929875639293401e-06, + "loss": 0.89288044, + "num_input_tokens_seen": 108992945, + "router_z_loss_clip": 0.66943359, + "router_z_loss_mlp": 0.20593262, + "step": 3860, + "time_per_iteration": 2.4320414066314697 + }, + { + "auxiliary_loss_clip": 0.01107966, + "auxiliary_loss_mlp": 0.0104911, + "balance_loss_clip": 1.03920257, + "balance_loss_mlp": 1.0291779, + "epoch": 0.11203644594045616, + "flos": 29675509937280.0, + "grad_norm": 5.395184074445895, + "language_loss": 0.72250712, + "learning_rate": 3.929826294838348e-06, + "loss": 0.74407786, + "num_input_tokens_seen": 109010140, + "router_z_loss_clip": 0.6875, + "router_z_loss_mlp": 0.19934082, + "step": 3861, + "time_per_iteration": 2.493868112564087 + }, + { + "auxiliary_loss_clip": 0.01026803, + "auxiliary_loss_mlp": 0.0104748, + "balance_loss_clip": 1.0154624, + "balance_loss_mlp": 1.04626966, + "epoch": 0.11206546340897221, + "flos": 58814264632320.0, + "grad_norm": 0.7177877167564833, + "language_loss": 0.47655171, + "learning_rate": 3.929776933338289e-06, + "loss": 0.49729455, + "num_input_tokens_seen": 109065545, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01208496, + "step": 3862, + "time_per_iteration": 2.933314800262451 + }, + { + "auxiliary_loss_clip": 0.01106999, + "auxiliary_loss_mlp": 0.01059507, + "balance_loss_clip": 1.03742957, + "balance_loss_mlp": 1.03787041, + "epoch": 0.11209448087748825, + "flos": 29087956800000.0, + "grad_norm": 2.8646383817158005, + "language_loss": 0.91820133, + "learning_rate": 3.9297275547936585e-06, + "loss": 0.93986642, + "num_input_tokens_seen": 109079885, + "router_z_loss_clip": 0.69580078, + "router_z_loss_mlp": 0.2164917, + "step": 3863, + "time_per_iteration": 2.4821317195892334 + }, + { + "auxiliary_loss_clip": 0.01023866, + "auxiliary_loss_mlp": 0.01019681, + "balance_loss_clip": 1.01283336, + "balance_loss_mlp": 1.01854229, + "epoch": 0.1121234983460043, + "flos": 70980028773120.0, + "grad_norm": 0.6451537039183899, + "language_loss": 0.48415375, + "learning_rate": 3.929678159204894e-06, + "loss": 0.5045892, + "num_input_tokens_seen": 109150060, + "router_z_loss_clip": 0.11035156, + "router_z_loss_mlp": 0.01141357, + "step": 3864, + "time_per_iteration": 3.0891475677490234 + }, + { + "auxiliary_loss_clip": 0.01104847, + "auxiliary_loss_mlp": 0.01049387, + "balance_loss_clip": 1.0383538, + "balance_loss_mlp": 1.03011036, + "epoch": 0.11215251581452033, + "flos": 33904120521600.0, + "grad_norm": 2.1548027512543673, + "language_loss": 0.89025545, + "learning_rate": 3.9296287465724294e-06, + "loss": 0.91179776, + "num_input_tokens_seen": 109165405, + "router_z_loss_clip": 0.66503906, + "router_z_loss_mlp": 0.19293213, + "step": 3865, + "time_per_iteration": 2.503089666366577 + }, + { + "auxiliary_loss_clip": 0.01035299, + "auxiliary_loss_mlp": 0.01001147, + "balance_loss_clip": 1.02348661, + "balance_loss_mlp": 1.00011623, + "epoch": 0.11218153328303639, + "flos": 73389105607680.0, + "grad_norm": 0.7689197128976446, + "language_loss": 0.46953142, + "learning_rate": 3.929579316896705e-06, + "loss": 0.48989585, + "num_input_tokens_seen": 109225100, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.01031494, + "step": 3866, + "time_per_iteration": 3.085164785385132 + }, + { + "auxiliary_loss_clip": 0.01115996, + "auxiliary_loss_mlp": 0.01041888, + "balance_loss_clip": 1.04497075, + "balance_loss_mlp": 1.02169383, + "epoch": 0.11221055075155244, + "flos": 36564327832320.0, + "grad_norm": 1.9432748429858882, + "language_loss": 0.8624624, + "learning_rate": 3.9295298701781534e-06, + "loss": 0.88404131, + "num_input_tokens_seen": 109245515, + "router_z_loss_clip": 0.70996094, + "router_z_loss_mlp": 0.2020874, + "step": 3867, + "time_per_iteration": 2.6328365802764893 + }, + { + "auxiliary_loss_clip": 0.01115004, + "auxiliary_loss_mlp": 0.01042251, + "balance_loss_clip": 1.04805434, + "balance_loss_mlp": 1.02318931, + "epoch": 0.11223956822006848, + "flos": 12085394457600.0, + "grad_norm": 2.68007994703619, + "language_loss": 0.81491005, + "learning_rate": 3.929480406417215e-06, + "loss": 0.83648264, + "num_input_tokens_seen": 109258195, + "router_z_loss_clip": 0.66943359, + "router_z_loss_mlp": 0.1907959, + "step": 3868, + "time_per_iteration": 2.448482036590576 + }, + { + "auxiliary_loss_clip": 0.01123487, + "auxiliary_loss_mlp": 0.01045546, + "balance_loss_clip": 1.05423474, + "balance_loss_mlp": 1.02560234, + "epoch": 0.11226858568858453, + "flos": 16315925166720.0, + "grad_norm": 4.51669231512051, + "language_loss": 0.77244079, + "learning_rate": 3.929430925614324e-06, + "loss": 0.79413116, + "num_input_tokens_seen": 109271110, + "router_z_loss_clip": 0.69238281, + "router_z_loss_mlp": 0.19946289, + "step": 3869, + "time_per_iteration": 2.352830410003662 + }, + { + "auxiliary_loss_clip": 0.01121881, + "auxiliary_loss_mlp": 0.01040123, + "balance_loss_clip": 1.0532198, + "balance_loss_mlp": 1.02165747, + "epoch": 0.11229760315710058, + "flos": 14092655921280.0, + "grad_norm": 2.510233897283114, + "language_loss": 0.75541055, + "learning_rate": 3.929381427769918e-06, + "loss": 0.77703059, + "num_input_tokens_seen": 109282785, + "router_z_loss_clip": 0.68603516, + "router_z_loss_mlp": 0.18457031, + "step": 3870, + "time_per_iteration": 2.4436604976654053 + }, + { + "auxiliary_loss_clip": 0.01133253, + "auxiliary_loss_mlp": 0.01047443, + "balance_loss_clip": 1.05823326, + "balance_loss_mlp": 1.02604461, + "epoch": 0.11232662062561662, + "flos": 35107506864000.0, + "grad_norm": 1.710751332466951, + "language_loss": 0.73821259, + "learning_rate": 3.929331912884435e-06, + "loss": 0.7600196, + "num_input_tokens_seen": 109306165, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.2142334, + "step": 3871, + "time_per_iteration": 2.5173184871673584 + }, + { + "auxiliary_loss_clip": 0.01087196, + "auxiliary_loss_mlp": 0.01037456, + "balance_loss_clip": 1.06818056, + "balance_loss_mlp": 1.03582323, + "epoch": 0.11235563809413267, + "flos": 64044637257600.0, + "grad_norm": 0.6991196841516909, + "language_loss": 0.51495987, + "learning_rate": 3.929282380958311e-06, + "loss": 0.53620636, + "num_input_tokens_seen": 109368290, + "router_z_loss_clip": 0.18945312, + "router_z_loss_mlp": 0.01635742, + "step": 3872, + "time_per_iteration": 3.029186964035034 + }, + { + "auxiliary_loss_clip": 0.01079933, + "auxiliary_loss_mlp": 0.01042011, + "balance_loss_clip": 1.06190217, + "balance_loss_mlp": 1.0403657, + "epoch": 0.11238465556264872, + "flos": 71816896995840.0, + "grad_norm": 0.6832656532492429, + "language_loss": 0.50906074, + "learning_rate": 3.9292328319919855e-06, + "loss": 0.53028023, + "num_input_tokens_seen": 109434930, + "router_z_loss_clip": 0.1796875, + "router_z_loss_mlp": 0.01647949, + "step": 3873, + "time_per_iteration": 3.073045253753662 + }, + { + "auxiliary_loss_clip": 0.0112685, + "auxiliary_loss_mlp": 0.01049481, + "balance_loss_clip": 1.05245805, + "balance_loss_mlp": 1.02876234, + "epoch": 0.11241367303116476, + "flos": 38901867557760.0, + "grad_norm": 2.4280664288600606, + "language_loss": 1.01287317, + "learning_rate": 3.929183265985894e-06, + "loss": 1.03463638, + "num_input_tokens_seen": 109452645, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.20727539, + "step": 3874, + "time_per_iteration": 2.5888967514038086 + }, + { + "auxiliary_loss_clip": 0.01112435, + "auxiliary_loss_mlp": 0.01040027, + "balance_loss_clip": 1.0469346, + "balance_loss_mlp": 1.02144861, + "epoch": 0.11244269049968081, + "flos": 30218793603840.0, + "grad_norm": 1.800654719267266, + "language_loss": 0.68155682, + "learning_rate": 3.929133682940476e-06, + "loss": 0.70308149, + "num_input_tokens_seen": 109473015, + "router_z_loss_clip": 0.65478516, + "router_z_loss_mlp": 0.18572998, + "step": 3875, + "time_per_iteration": 2.6672613620758057 + }, + { + "auxiliary_loss_clip": 0.01056976, + "auxiliary_loss_mlp": 0.0102397, + "balance_loss_clip": 1.04340553, + "balance_loss_mlp": 1.0224973, + "epoch": 0.11247170796819686, + "flos": 62512334196480.0, + "grad_norm": 0.635889942737062, + "language_loss": 0.47465223, + "learning_rate": 3.929084082856167e-06, + "loss": 0.4954617, + "num_input_tokens_seen": 109536095, + "router_z_loss_clip": 0.13574219, + "router_z_loss_mlp": 0.01470947, + "step": 3876, + "time_per_iteration": 2.9436588287353516 + }, + { + "auxiliary_loss_clip": 0.01116573, + "auxiliary_loss_mlp": 0.01042602, + "balance_loss_clip": 1.04519713, + "balance_loss_mlp": 1.02169263, + "epoch": 0.1125007254367129, + "flos": 29562147152640.0, + "grad_norm": 2.161549001421268, + "language_loss": 0.77230465, + "learning_rate": 3.9290344657334085e-06, + "loss": 0.79389638, + "num_input_tokens_seen": 109554585, + "router_z_loss_clip": 0.71386719, + "router_z_loss_mlp": 0.20910645, + "step": 3877, + "time_per_iteration": 2.5009515285491943 + }, + { + "auxiliary_loss_clip": 0.01114316, + "auxiliary_loss_mlp": 0.01041454, + "balance_loss_clip": 1.0433569, + "balance_loss_mlp": 1.01997209, + "epoch": 0.11252974290522895, + "flos": 32553470597760.0, + "grad_norm": 1.9495454797779692, + "language_loss": 0.75737679, + "learning_rate": 3.928984831572637e-06, + "loss": 0.77893448, + "num_input_tokens_seen": 109572990, + "router_z_loss_clip": 0.70996094, + "router_z_loss_mlp": 0.21484375, + "step": 3878, + "time_per_iteration": 2.493391513824463 + }, + { + "auxiliary_loss_clip": 0.01107372, + "auxiliary_loss_mlp": 0.01037082, + "balance_loss_clip": 1.04177475, + "balance_loss_mlp": 1.01717448, + "epoch": 0.112558760373745, + "flos": 33283679016960.0, + "grad_norm": 2.20904972012084, + "language_loss": 0.68318063, + "learning_rate": 3.92893518037429e-06, + "loss": 0.70462519, + "num_input_tokens_seen": 109589855, + "router_z_loss_clip": 0.65625, + "router_z_loss_mlp": 0.19909668, + "step": 3879, + "time_per_iteration": 2.5483171939849854 + }, + { + "auxiliary_loss_clip": 0.01107192, + "auxiliary_loss_mlp": 0.01051525, + "balance_loss_clip": 1.0396837, + "balance_loss_mlp": 1.03079438, + "epoch": 0.11258777784226104, + "flos": 17854023513600.0, + "grad_norm": 2.22016053917716, + "language_loss": 0.86921299, + "learning_rate": 3.928885512138808e-06, + "loss": 0.89080012, + "num_input_tokens_seen": 109611180, + "router_z_loss_clip": 0.67529297, + "router_z_loss_mlp": 0.20715332, + "step": 3880, + "time_per_iteration": 2.5592093467712402 + }, + { + "auxiliary_loss_clip": 0.01032508, + "auxiliary_loss_mlp": 0.00999581, + "balance_loss_clip": 1.02121997, + "balance_loss_mlp": 0.99851978, + "epoch": 0.11261679531077709, + "flos": 46060002881280.0, + "grad_norm": 0.654643494509161, + "language_loss": 0.4084585, + "learning_rate": 3.928835826866628e-06, + "loss": 0.42877939, + "num_input_tokens_seen": 109655680, + "router_z_loss_clip": 0.11279297, + "router_z_loss_mlp": 0.01062012, + "step": 3881, + "time_per_iteration": 2.7995059490203857 + }, + { + "auxiliary_loss_clip": 0.01095346, + "auxiliary_loss_mlp": 0.01042349, + "balance_loss_clip": 1.03656566, + "balance_loss_mlp": 1.02660775, + "epoch": 0.11264581277929313, + "flos": 39960434113920.0, + "grad_norm": 2.095286004861891, + "language_loss": 0.85496807, + "learning_rate": 3.928786124558189e-06, + "loss": 0.87634504, + "num_input_tokens_seen": 109673585, + "router_z_loss_clip": 0.58740234, + "router_z_loss_mlp": 0.15734863, + "step": 3882, + "time_per_iteration": 4.932114362716675 + }, + { + "auxiliary_loss_clip": 0.01021436, + "auxiliary_loss_mlp": 0.01004942, + "balance_loss_clip": 1.01094067, + "balance_loss_mlp": 1.00386953, + "epoch": 0.11267483024780918, + "flos": 74765452158720.0, + "grad_norm": 0.6662785603980501, + "language_loss": 0.48258734, + "learning_rate": 3.928736405213931e-06, + "loss": 0.50285113, + "num_input_tokens_seen": 109736060, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01074219, + "step": 3883, + "time_per_iteration": 3.0338528156280518 + }, + { + "auxiliary_loss_clip": 0.0101787, + "auxiliary_loss_mlp": 0.01004038, + "balance_loss_clip": 1.00724626, + "balance_loss_mlp": 1.00301266, + "epoch": 0.11270384771632523, + "flos": 53132289304320.0, + "grad_norm": 0.7113803548831378, + "language_loss": 0.53347802, + "learning_rate": 3.928686668834292e-06, + "loss": 0.55369711, + "num_input_tokens_seen": 109793745, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01025391, + "step": 3884, + "time_per_iteration": 2.9503355026245117 + }, + { + "auxiliary_loss_clip": 0.01112154, + "auxiliary_loss_mlp": 0.01052539, + "balance_loss_clip": 1.03670311, + "balance_loss_mlp": 1.02956748, + "epoch": 0.11273286518484127, + "flos": 36240403438080.0, + "grad_norm": 1.8455899334856132, + "language_loss": 0.83613241, + "learning_rate": 3.928636915419713e-06, + "loss": 0.85777938, + "num_input_tokens_seen": 109817305, + "router_z_loss_clip": 0.75488281, + "router_z_loss_mlp": 0.22998047, + "step": 3885, + "time_per_iteration": 2.5577123165130615 + }, + { + "auxiliary_loss_clip": 0.01105014, + "auxiliary_loss_mlp": 0.01043562, + "balance_loss_clip": 1.03936791, + "balance_loss_mlp": 1.02547836, + "epoch": 0.11276188265335732, + "flos": 17231138213760.0, + "grad_norm": 5.189148139513087, + "language_loss": 0.78018624, + "learning_rate": 3.928587144970631e-06, + "loss": 0.80167198, + "num_input_tokens_seen": 109830035, + "router_z_loss_clip": 0.65625, + "router_z_loss_mlp": 0.18103027, + "step": 3886, + "time_per_iteration": 2.41437029838562 + }, + { + "auxiliary_loss_clip": 0.01105961, + "auxiliary_loss_mlp": 0.01053704, + "balance_loss_clip": 1.03783154, + "balance_loss_mlp": 1.0334506, + "epoch": 0.11279090012187337, + "flos": 74730121862400.0, + "grad_norm": 2.340660661157167, + "language_loss": 0.85866368, + "learning_rate": 3.928537357487487e-06, + "loss": 0.88026035, + "num_input_tokens_seen": 109851310, + "router_z_loss_clip": 0.68017578, + "router_z_loss_mlp": 0.20251465, + "step": 3887, + "time_per_iteration": 2.831784963607788 + }, + { + "auxiliary_loss_clip": 0.01106437, + "auxiliary_loss_mlp": 0.01047063, + "balance_loss_clip": 1.03724992, + "balance_loss_mlp": 1.02614129, + "epoch": 0.11281991759038941, + "flos": 34012246602240.0, + "grad_norm": 2.2377349019506565, + "language_loss": 0.82112002, + "learning_rate": 3.928487552970722e-06, + "loss": 0.84265506, + "num_input_tokens_seen": 109866615, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.20928955, + "step": 3888, + "time_per_iteration": 4.892369270324707 + }, + { + "auxiliary_loss_clip": 0.0110953, + "auxiliary_loss_mlp": 0.01043184, + "balance_loss_clip": 1.041803, + "balance_loss_mlp": 1.02331233, + "epoch": 0.11284893505890546, + "flos": 12449538604800.0, + "grad_norm": 3.84699027700077, + "language_loss": 1.00003016, + "learning_rate": 3.928437731420774e-06, + "loss": 1.02155721, + "num_input_tokens_seen": 109877135, + "router_z_loss_clip": 0.67724609, + "router_z_loss_mlp": 0.19897461, + "step": 3889, + "time_per_iteration": 2.3944990634918213 + }, + { + "auxiliary_loss_clip": 0.01109385, + "auxiliary_loss_mlp": 0.0103984, + "balance_loss_clip": 1.04282236, + "balance_loss_mlp": 1.02234006, + "epoch": 0.11287795252742151, + "flos": 26107525699200.0, + "grad_norm": 2.4767880115554477, + "language_loss": 0.71687126, + "learning_rate": 3.928387892838083e-06, + "loss": 0.7383635, + "num_input_tokens_seen": 109893545, + "router_z_loss_clip": 0.66577148, + "router_z_loss_mlp": 0.17492676, + "step": 3890, + "time_per_iteration": 2.3795430660247803 + }, + { + "auxiliary_loss_clip": 0.01033133, + "auxiliary_loss_mlp": 0.0100777, + "balance_loss_clip": 1.02085948, + "balance_loss_mlp": 1.00632203, + "epoch": 0.11290696999593755, + "flos": 63649559779200.0, + "grad_norm": 0.6589424949493426, + "language_loss": 0.49093446, + "learning_rate": 3.928338037223091e-06, + "loss": 0.51134348, + "num_input_tokens_seen": 109959035, + "router_z_loss_clip": 0.12255859, + "router_z_loss_mlp": 0.01446533, + "step": 3891, + "time_per_iteration": 5.494143724441528 + }, + { + "auxiliary_loss_clip": 0.01117118, + "auxiliary_loss_mlp": 0.01042241, + "balance_loss_clip": 1.04547071, + "balance_loss_mlp": 1.02058041, + "epoch": 0.1129359874644536, + "flos": 58532831095680.0, + "grad_norm": 3.023813688795536, + "language_loss": 0.79023045, + "learning_rate": 3.9282881645762365e-06, + "loss": 0.81182408, + "num_input_tokens_seen": 109981830, + "router_z_loss_clip": 0.71606445, + "router_z_loss_mlp": 0.21655273, + "step": 3892, + "time_per_iteration": 2.653855085372925 + }, + { + "auxiliary_loss_clip": 0.01033924, + "auxiliary_loss_mlp": 0.01004364, + "balance_loss_clip": 1.02146816, + "balance_loss_mlp": 1.00299346, + "epoch": 0.11296500493296965, + "flos": 74787690695040.0, + "grad_norm": 0.6353247376612737, + "language_loss": 0.50196654, + "learning_rate": 3.9282382748979604e-06, + "loss": 0.52234942, + "num_input_tokens_seen": 110052370, + "router_z_loss_clip": 0.12451172, + "router_z_loss_mlp": 0.01373291, + "step": 3893, + "time_per_iteration": 3.224666118621826 + }, + { + "auxiliary_loss_clip": 0.01115062, + "auxiliary_loss_mlp": 0.01042228, + "balance_loss_clip": 1.0438205, + "balance_loss_mlp": 1.02031708, + "epoch": 0.11299402240148569, + "flos": 22271234595840.0, + "grad_norm": 2.2670668787747634, + "language_loss": 0.76919639, + "learning_rate": 3.928188368188704e-06, + "loss": 0.79076934, + "num_input_tokens_seen": 110068625, + "router_z_loss_clip": 0.71142578, + "router_z_loss_mlp": 0.21899414, + "step": 3894, + "time_per_iteration": 2.433964252471924 + }, + { + "auxiliary_loss_clip": 0.01128958, + "auxiliary_loss_mlp": 0.01051964, + "balance_loss_clip": 1.04837084, + "balance_loss_mlp": 1.02356863, + "epoch": 0.11302303987000174, + "flos": 22483541773440.0, + "grad_norm": 2.3410242267004118, + "language_loss": 0.97581124, + "learning_rate": 3.928138444448906e-06, + "loss": 0.99762046, + "num_input_tokens_seen": 110083775, + "router_z_loss_clip": 0.80615234, + "router_z_loss_mlp": 0.28381348, + "step": 3895, + "time_per_iteration": 2.458254814147949 + }, + { + "auxiliary_loss_clip": 0.01029813, + "auxiliary_loss_mlp": 0.01001624, + "balance_loss_clip": 1.01638579, + "balance_loss_mlp": 1.00049746, + "epoch": 0.1130520573385178, + "flos": 61957843983360.0, + "grad_norm": 0.7028876920559144, + "language_loss": 0.5064404, + "learning_rate": 3.928088503679011e-06, + "loss": 0.52675486, + "num_input_tokens_seen": 110139160, + "router_z_loss_clip": 0.13378906, + "router_z_loss_mlp": 0.0112915, + "step": 3896, + "time_per_iteration": 2.847005605697632 + }, + { + "auxiliary_loss_clip": 0.01108986, + "auxiliary_loss_mlp": 0.01048968, + "balance_loss_clip": 1.04285777, + "balance_loss_mlp": 1.02730799, + "epoch": 0.11308107480703383, + "flos": 19564942423680.0, + "grad_norm": 2.2735750505215444, + "language_loss": 0.89293551, + "learning_rate": 3.928038545879457e-06, + "loss": 0.91451508, + "num_input_tokens_seen": 110152525, + "router_z_loss_clip": 0.66113281, + "router_z_loss_mlp": 0.21643066, + "step": 3897, + "time_per_iteration": 2.4324066638946533 + }, + { + "auxiliary_loss_clip": 0.01024992, + "auxiliary_loss_mlp": 0.01003796, + "balance_loss_clip": 1.01181817, + "balance_loss_mlp": 1.0027349, + "epoch": 0.11311009227554988, + "flos": 60332531460480.0, + "grad_norm": 0.6522764950788227, + "language_loss": 0.50712603, + "learning_rate": 3.927988571050688e-06, + "loss": 0.5274139, + "num_input_tokens_seen": 110215930, + "router_z_loss_clip": 0.13183594, + "router_z_loss_mlp": 0.01062012, + "step": 3898, + "time_per_iteration": 3.011500835418701 + }, + { + "auxiliary_loss_clip": 0.01110802, + "auxiliary_loss_mlp": 0.01048712, + "balance_loss_clip": 1.03799367, + "balance_loss_mlp": 1.02946019, + "epoch": 0.11313910974406592, + "flos": 19602159799680.0, + "grad_norm": 2.281589507163641, + "language_loss": 0.94386905, + "learning_rate": 3.927938579193142e-06, + "loss": 0.96546423, + "num_input_tokens_seen": 110230465, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.19238281, + "step": 3899, + "time_per_iteration": 2.422330856323242 + }, + { + "auxiliary_loss_clip": 0.01105021, + "auxiliary_loss_mlp": 0.01045142, + "balance_loss_clip": 1.03348446, + "balance_loss_mlp": 1.02443528, + "epoch": 0.11316812721258197, + "flos": 45764081130240.0, + "grad_norm": 2.475092457108727, + "language_loss": 0.83223665, + "learning_rate": 3.927888570307263e-06, + "loss": 0.85373831, + "num_input_tokens_seen": 110247270, + "router_z_loss_clip": 0.71582031, + "router_z_loss_mlp": 0.20709229, + "step": 3900, + "time_per_iteration": 2.6699626445770264 + }, + { + "auxiliary_loss_clip": 0.01016418, + "auxiliary_loss_mlp": 0.01004642, + "balance_loss_clip": 1.00479662, + "balance_loss_mlp": 1.00364625, + "epoch": 0.11319714468109802, + "flos": 67251025877760.0, + "grad_norm": 0.6858065428826106, + "language_loss": 0.46722722, + "learning_rate": 3.927838544393492e-06, + "loss": 0.48743784, + "num_input_tokens_seen": 110301960, + "router_z_loss_clip": 0.11621094, + "router_z_loss_mlp": 0.00994873, + "step": 3901, + "time_per_iteration": 2.9104716777801514 + }, + { + "auxiliary_loss_clip": 0.01101197, + "auxiliary_loss_mlp": 0.01043083, + "balance_loss_clip": 1.03484833, + "balance_loss_mlp": 1.0234139, + "epoch": 0.11322616214961406, + "flos": 36717945281280.0, + "grad_norm": 2.2588437825472534, + "language_loss": 0.92697906, + "learning_rate": 3.92778850145227e-06, + "loss": 0.94842196, + "num_input_tokens_seen": 110318455, + "router_z_loss_clip": 0.6640625, + "router_z_loss_mlp": 0.19689941, + "step": 3902, + "time_per_iteration": 2.459237813949585 + }, + { + "auxiliary_loss_clip": 0.01100291, + "auxiliary_loss_mlp": 0.01037614, + "balance_loss_clip": 1.03114986, + "balance_loss_mlp": 1.01766992, + "epoch": 0.11325517961813011, + "flos": 26766057363840.0, + "grad_norm": 2.341114649479055, + "language_loss": 0.81581205, + "learning_rate": 3.927738441484042e-06, + "loss": 0.8371911, + "num_input_tokens_seen": 110334150, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.19946289, + "step": 3903, + "time_per_iteration": 2.542588233947754 + }, + { + "auxiliary_loss_clip": 0.01109995, + "auxiliary_loss_mlp": 0.01038269, + "balance_loss_clip": 1.03519726, + "balance_loss_mlp": 1.01675129, + "epoch": 0.11328419708664617, + "flos": 27486281134080.0, + "grad_norm": 2.3171563531286665, + "language_loss": 0.81657076, + "learning_rate": 3.927688364489246e-06, + "loss": 0.83805335, + "num_input_tokens_seen": 110355810, + "router_z_loss_clip": 0.74755859, + "router_z_loss_mlp": 0.21508789, + "step": 3904, + "time_per_iteration": 2.702981472015381 + }, + { + "auxiliary_loss_clip": 0.01104221, + "auxiliary_loss_mlp": 0.0103973, + "balance_loss_clip": 1.03383172, + "balance_loss_mlp": 1.01764047, + "epoch": 0.1133132145551622, + "flos": 12596348338560.0, + "grad_norm": 3.1854235188383293, + "language_loss": 0.86453354, + "learning_rate": 3.927638270468327e-06, + "loss": 0.88597298, + "num_input_tokens_seen": 110366885, + "router_z_loss_clip": 0.70410156, + "router_z_loss_mlp": 0.2208252, + "step": 3905, + "time_per_iteration": 2.3765504360198975 + }, + { + "auxiliary_loss_clip": 0.0110313, + "auxiliary_loss_mlp": 0.01042926, + "balance_loss_clip": 1.03221631, + "balance_loss_mlp": 1.02119446, + "epoch": 0.11334223202367825, + "flos": 33540430222080.0, + "grad_norm": 3.999700469089822, + "language_loss": 0.82410252, + "learning_rate": 3.927588159421727e-06, + "loss": 0.84556311, + "num_input_tokens_seen": 110385580, + "router_z_loss_clip": 0.70898438, + "router_z_loss_mlp": 0.21728516, + "step": 3906, + "time_per_iteration": 2.476166248321533 + }, + { + "auxiliary_loss_clip": 0.01098414, + "auxiliary_loss_mlp": 0.01040547, + "balance_loss_clip": 1.03117061, + "balance_loss_mlp": 1.01967359, + "epoch": 0.1133712494921943, + "flos": 28575711198720.0, + "grad_norm": 2.9039298764001664, + "language_loss": 0.88777399, + "learning_rate": 3.927538031349888e-06, + "loss": 0.90916365, + "num_input_tokens_seen": 110399305, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.2088623, + "step": 3907, + "time_per_iteration": 2.4470930099487305 + }, + { + "auxiliary_loss_clip": 0.01095211, + "auxiliary_loss_mlp": 0.01040697, + "balance_loss_clip": 1.03090143, + "balance_loss_mlp": 1.02125406, + "epoch": 0.11340026696071034, + "flos": 30328700163840.0, + "grad_norm": 2.308585029788415, + "language_loss": 0.84685969, + "learning_rate": 3.927487886253253e-06, + "loss": 0.86821878, + "num_input_tokens_seen": 110413070, + "router_z_loss_clip": 0.64306641, + "router_z_loss_mlp": 0.19445801, + "step": 3908, + "time_per_iteration": 2.4619314670562744 + }, + { + "auxiliary_loss_clip": 0.01107864, + "auxiliary_loss_mlp": 0.01045745, + "balance_loss_clip": 1.03656316, + "balance_loss_mlp": 1.02434683, + "epoch": 0.1134292844292264, + "flos": 74730959735040.0, + "grad_norm": 3.517353522049342, + "language_loss": 0.73370391, + "learning_rate": 3.927437724132265e-06, + "loss": 0.75524002, + "num_input_tokens_seen": 110434435, + "router_z_loss_clip": 0.71337891, + "router_z_loss_mlp": 0.21411133, + "step": 3909, + "time_per_iteration": 2.8044593334198 + }, + { + "auxiliary_loss_clip": 0.01104431, + "auxiliary_loss_mlp": 0.01043114, + "balance_loss_clip": 1.03219485, + "balance_loss_mlp": 1.02109623, + "epoch": 0.11345830189774245, + "flos": 20367560736000.0, + "grad_norm": 2.707900704017053, + "language_loss": 0.94398487, + "learning_rate": 3.927387544987367e-06, + "loss": 0.9654603, + "num_input_tokens_seen": 110446750, + "router_z_loss_clip": 0.72167969, + "router_z_loss_mlp": 0.22009277, + "step": 3910, + "time_per_iteration": 2.3793344497680664 + }, + { + "auxiliary_loss_clip": 0.01020763, + "auxiliary_loss_mlp": 0.01005192, + "balance_loss_clip": 1.00976872, + "balance_loss_mlp": 1.00408292, + "epoch": 0.11348731936625849, + "flos": 65190346300800.0, + "grad_norm": 0.6506794908549595, + "language_loss": 0.49536943, + "learning_rate": 3.927337348819003e-06, + "loss": 0.51562899, + "num_input_tokens_seen": 110513395, + "router_z_loss_clip": 0.10986328, + "router_z_loss_mlp": 0.0111084, + "step": 3911, + "time_per_iteration": 3.1930935382843018 + }, + { + "auxiliary_loss_clip": 0.01021193, + "auxiliary_loss_mlp": 0.01002991, + "balance_loss_clip": 1.01026988, + "balance_loss_mlp": 1.00179255, + "epoch": 0.11351633683477454, + "flos": 71130224908800.0, + "grad_norm": 0.6013753663551076, + "language_loss": 0.48828393, + "learning_rate": 3.927287135627615e-06, + "loss": 0.50852579, + "num_input_tokens_seen": 110582165, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01196289, + "step": 3912, + "time_per_iteration": 3.1317849159240723 + }, + { + "auxiliary_loss_clip": 0.01019406, + "auxiliary_loss_mlp": 0.01002488, + "balance_loss_clip": 1.00873196, + "balance_loss_mlp": 1.00131989, + "epoch": 0.11354535430329057, + "flos": 61895942409600.0, + "grad_norm": 0.681001540962612, + "language_loss": 0.50496948, + "learning_rate": 3.9272369054136475e-06, + "loss": 0.52518845, + "num_input_tokens_seen": 110645815, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01165771, + "step": 3913, + "time_per_iteration": 2.9698657989501953 + }, + { + "auxiliary_loss_clip": 0.01101388, + "auxiliary_loss_mlp": 0.01039873, + "balance_loss_clip": 1.03519046, + "balance_loss_mlp": 1.02017403, + "epoch": 0.11357437177180663, + "flos": 20185139548800.0, + "grad_norm": 2.8435320359903433, + "language_loss": 0.71927744, + "learning_rate": 3.927186658177544e-06, + "loss": 0.74069011, + "num_input_tokens_seen": 110660280, + "router_z_loss_clip": 0.66137695, + "router_z_loss_mlp": 0.19714355, + "step": 3914, + "time_per_iteration": 2.418215274810791 + }, + { + "auxiliary_loss_clip": 0.01019322, + "auxiliary_loss_mlp": 0.01001688, + "balance_loss_clip": 1.00895882, + "balance_loss_mlp": 1.00059724, + "epoch": 0.11360338924032268, + "flos": 65321306277120.0, + "grad_norm": 0.6501246865897867, + "language_loss": 0.52143502, + "learning_rate": 3.927136393919748e-06, + "loss": 0.54164517, + "num_input_tokens_seen": 110723815, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01092529, + "step": 3915, + "time_per_iteration": 3.0127129554748535 + }, + { + "auxiliary_loss_clip": 0.01113893, + "auxiliary_loss_mlp": 0.01038457, + "balance_loss_clip": 1.03707993, + "balance_loss_mlp": 1.01689243, + "epoch": 0.11363240670883872, + "flos": 15880907226240.0, + "grad_norm": 3.2314325141708324, + "language_loss": 1.03578246, + "learning_rate": 3.927086112640703e-06, + "loss": 1.05730605, + "num_input_tokens_seen": 110735890, + "router_z_loss_clip": 0.76904297, + "router_z_loss_mlp": 0.21569824, + "step": 3916, + "time_per_iteration": 2.3853158950805664 + }, + { + "auxiliary_loss_clip": 0.01096434, + "auxiliary_loss_mlp": 0.01032095, + "balance_loss_clip": 1.03248215, + "balance_loss_mlp": 1.01447606, + "epoch": 0.11366142417735477, + "flos": 20223788290560.0, + "grad_norm": 2.976613802506393, + "language_loss": 0.76842904, + "learning_rate": 3.927035814340854e-06, + "loss": 0.78971428, + "num_input_tokens_seen": 110750810, + "router_z_loss_clip": 0.63964844, + "router_z_loss_mlp": 0.17614746, + "step": 3917, + "time_per_iteration": 2.409939765930176 + }, + { + "auxiliary_loss_clip": 0.01105938, + "auxiliary_loss_mlp": 0.010388, + "balance_loss_clip": 1.03552628, + "balance_loss_mlp": 1.01796198, + "epoch": 0.11369044164587082, + "flos": 30583496332800.0, + "grad_norm": 4.837321561511571, + "language_loss": 0.88309813, + "learning_rate": 3.926985499020645e-06, + "loss": 0.90454549, + "num_input_tokens_seen": 110765030, + "router_z_loss_clip": 0.70410156, + "router_z_loss_mlp": 0.20837402, + "step": 3918, + "time_per_iteration": 2.4829883575439453 + }, + { + "auxiliary_loss_clip": 0.01103864, + "auxiliary_loss_mlp": 0.01039369, + "balance_loss_clip": 1.0347513, + "balance_loss_mlp": 1.01825762, + "epoch": 0.11371945911438686, + "flos": 27920216822400.0, + "grad_norm": 2.7531845258313883, + "language_loss": 0.90705359, + "learning_rate": 3.926935166680519e-06, + "loss": 0.92848593, + "num_input_tokens_seen": 110781500, + "router_z_loss_clip": 0.69042969, + "router_z_loss_mlp": 0.21105957, + "step": 3919, + "time_per_iteration": 2.449335813522339 + }, + { + "auxiliary_loss_clip": 0.01101471, + "auxiliary_loss_mlp": 0.01041778, + "balance_loss_clip": 1.03439689, + "balance_loss_mlp": 1.02096391, + "epoch": 0.11374847658290291, + "flos": 36712848222720.0, + "grad_norm": 2.2228231433900985, + "language_loss": 0.8593182, + "learning_rate": 3.926884817320924e-06, + "loss": 0.88075078, + "num_input_tokens_seen": 110798115, + "router_z_loss_clip": 0.67114258, + "router_z_loss_mlp": 0.20800781, + "step": 3920, + "time_per_iteration": 2.521552085876465 + }, + { + "auxiliary_loss_clip": 0.01109363, + "auxiliary_loss_mlp": 0.01042866, + "balance_loss_clip": 1.03493047, + "balance_loss_mlp": 1.01984656, + "epoch": 0.11377749405141896, + "flos": 10260693826560.0, + "grad_norm": 3.8836023325357125, + "language_loss": 0.78902638, + "learning_rate": 3.926834450942301e-06, + "loss": 0.81054866, + "num_input_tokens_seen": 110810035, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.23022461, + "step": 3921, + "time_per_iteration": 2.3305370807647705 + }, + { + "auxiliary_loss_clip": 0.01017692, + "auxiliary_loss_mlp": 0.01005288, + "balance_loss_clip": 1.00749922, + "balance_loss_mlp": 1.00416136, + "epoch": 0.113806511519935, + "flos": 65027128227840.0, + "grad_norm": 0.7029433058575085, + "language_loss": 0.47901747, + "learning_rate": 3.926784067545097e-06, + "loss": 0.49924725, + "num_input_tokens_seen": 110867155, + "router_z_loss_clip": 0.10205078, + "router_z_loss_mlp": 0.0112915, + "step": 3922, + "time_per_iteration": 2.888172149658203 + }, + { + "auxiliary_loss_clip": 0.01100756, + "auxiliary_loss_mlp": 0.01039608, + "balance_loss_clip": 1.03317344, + "balance_loss_mlp": 1.01910448, + "epoch": 0.11383552898845105, + "flos": 11726033166720.0, + "grad_norm": 3.39312217381678, + "language_loss": 0.97101855, + "learning_rate": 3.926733667129756e-06, + "loss": 0.99242222, + "num_input_tokens_seen": 110882750, + "router_z_loss_clip": 0.67553711, + "router_z_loss_mlp": 0.20495605, + "step": 3923, + "time_per_iteration": 2.398016929626465 + }, + { + "auxiliary_loss_clip": 0.01096733, + "auxiliary_loss_mlp": 0.0103541, + "balance_loss_clip": 1.03257143, + "balance_loss_mlp": 1.01750493, + "epoch": 0.1138645464569671, + "flos": 18439551792000.0, + "grad_norm": 2.2241506901253496, + "language_loss": 0.68633735, + "learning_rate": 3.926683249696724e-06, + "loss": 0.70765883, + "num_input_tokens_seen": 110899195, + "router_z_loss_clip": 0.64111328, + "router_z_loss_mlp": 0.17907715, + "step": 3924, + "time_per_iteration": 2.3910794258117676 + }, + { + "auxiliary_loss_clip": 0.01017363, + "auxiliary_loss_mlp": 0.01001917, + "balance_loss_clip": 1.00702882, + "balance_loss_mlp": 1.00090945, + "epoch": 0.11389356392548314, + "flos": 63088785521280.0, + "grad_norm": 0.6911436920337142, + "language_loss": 0.45424676, + "learning_rate": 3.926632815246446e-06, + "loss": 0.47443956, + "num_input_tokens_seen": 110959765, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.0100708, + "step": 3925, + "time_per_iteration": 2.938743829727173 + }, + { + "auxiliary_loss_clip": 0.01099873, + "auxiliary_loss_mlp": 0.01041843, + "balance_loss_clip": 1.03356051, + "balance_loss_mlp": 1.02164865, + "epoch": 0.11392258139399919, + "flos": 28685513024640.0, + "grad_norm": 4.449716310331184, + "language_loss": 0.85652053, + "learning_rate": 3.926582363779367e-06, + "loss": 0.87793773, + "num_input_tokens_seen": 110974035, + "router_z_loss_clip": 0.66308594, + "router_z_loss_mlp": 0.20202637, + "step": 3926, + "time_per_iteration": 2.4567415714263916 + }, + { + "auxiliary_loss_clip": 0.01100745, + "auxiliary_loss_mlp": 0.01034525, + "balance_loss_clip": 1.031142, + "balance_loss_mlp": 1.01506984, + "epoch": 0.11395159886251524, + "flos": 26097680695680.0, + "grad_norm": 2.5184183792952126, + "language_loss": 0.9053784, + "learning_rate": 3.926531895295934e-06, + "loss": 0.92673111, + "num_input_tokens_seen": 110989295, + "router_z_loss_clip": 0.69628906, + "router_z_loss_mlp": 0.19445801, + "step": 3927, + "time_per_iteration": 2.4381208419799805 + }, + { + "auxiliary_loss_clip": 0.01012813, + "auxiliary_loss_mlp": 0.01001699, + "balance_loss_clip": 1.00237107, + "balance_loss_mlp": 1.00062597, + "epoch": 0.11398061633103128, + "flos": 65436659009280.0, + "grad_norm": 0.7211901475115328, + "language_loss": 0.51939356, + "learning_rate": 3.92648140979659e-06, + "loss": 0.53953868, + "num_input_tokens_seen": 111056260, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01074219, + "step": 3928, + "time_per_iteration": 3.0818960666656494 + }, + { + "auxiliary_loss_clip": 0.01097048, + "auxiliary_loss_mlp": 0.01043259, + "balance_loss_clip": 1.03063869, + "balance_loss_mlp": 1.02367234, + "epoch": 0.11400963379954733, + "flos": 16466226036480.0, + "grad_norm": 3.2043307940848447, + "language_loss": 0.78758425, + "learning_rate": 3.926430907281784e-06, + "loss": 0.80898726, + "num_input_tokens_seen": 111070430, + "router_z_loss_clip": 0.66455078, + "router_z_loss_mlp": 0.19592285, + "step": 3929, + "time_per_iteration": 2.4821741580963135 + }, + { + "auxiliary_loss_clip": 0.01095094, + "auxiliary_loss_mlp": 0.01034422, + "balance_loss_clip": 1.030406, + "balance_loss_mlp": 1.01624274, + "epoch": 0.11403865126806337, + "flos": 30043145220480.0, + "grad_norm": 1.8682360039391586, + "language_loss": 0.81074077, + "learning_rate": 3.92638038775196e-06, + "loss": 0.83203596, + "num_input_tokens_seen": 111090160, + "router_z_loss_clip": 0.64648438, + "router_z_loss_mlp": 0.1817627, + "step": 3930, + "time_per_iteration": 2.456798791885376 + }, + { + "auxiliary_loss_clip": 0.01111492, + "auxiliary_loss_mlp": 0.01049532, + "balance_loss_clip": 1.03543043, + "balance_loss_mlp": 1.02477264, + "epoch": 0.11406766873657942, + "flos": 16179903043200.0, + "grad_norm": 2.459788778719829, + "language_loss": 0.85261965, + "learning_rate": 3.926329851207565e-06, + "loss": 0.87422991, + "num_input_tokens_seen": 111104025, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.24755859, + "step": 3931, + "time_per_iteration": 2.344517469406128 + }, + { + "auxiliary_loss_clip": 0.01107918, + "auxiliary_loss_mlp": 0.01043486, + "balance_loss_clip": 1.033391, + "balance_loss_mlp": 1.02161121, + "epoch": 0.11409668620509547, + "flos": 45617445953280.0, + "grad_norm": 2.014704492344848, + "language_loss": 0.83708417, + "learning_rate": 3.9262792976490455e-06, + "loss": 0.85859823, + "num_input_tokens_seen": 111127170, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.21862793, + "step": 3932, + "time_per_iteration": 2.616436719894409 + }, + { + "auxiliary_loss_clip": 0.01098025, + "auxiliary_loss_mlp": 0.01043104, + "balance_loss_clip": 1.03084219, + "balance_loss_mlp": 1.02447152, + "epoch": 0.11412570367361151, + "flos": 29235185470080.0, + "grad_norm": 2.1495499387431067, + "language_loss": 0.64936829, + "learning_rate": 3.926228727076847e-06, + "loss": 0.67077959, + "num_input_tokens_seen": 111148405, + "router_z_loss_clip": 0.67236328, + "router_z_loss_mlp": 0.18640137, + "step": 3933, + "time_per_iteration": 2.597716808319092 + }, + { + "auxiliary_loss_clip": 0.0110162, + "auxiliary_loss_mlp": 0.01047679, + "balance_loss_clip": 1.03436613, + "balance_loss_mlp": 1.02668631, + "epoch": 0.11415472114212756, + "flos": 11904090433920.0, + "grad_norm": 3.2077622342946706, + "language_loss": 0.94367576, + "learning_rate": 3.926178139491418e-06, + "loss": 0.96516871, + "num_input_tokens_seen": 111161070, + "router_z_loss_clip": 0.67236328, + "router_z_loss_mlp": 0.20996094, + "step": 3934, + "time_per_iteration": 2.316598415374756 + }, + { + "auxiliary_loss_clip": 0.01014526, + "auxiliary_loss_mlp": 0.01001502, + "balance_loss_clip": 1.00369525, + "balance_loss_mlp": 1.00046444, + "epoch": 0.11418373861064361, + "flos": 61523349713280.0, + "grad_norm": 0.6748266975953542, + "language_loss": 0.53044474, + "learning_rate": 3.9261275348932036e-06, + "loss": 0.55060506, + "num_input_tokens_seen": 111225735, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01037598, + "step": 3935, + "time_per_iteration": 3.167388677597046 + }, + { + "auxiliary_loss_clip": 0.01013758, + "auxiliary_loss_mlp": 0.01001789, + "balance_loss_clip": 1.00328684, + "balance_loss_mlp": 1.00076389, + "epoch": 0.11421275607915965, + "flos": 67412149269120.0, + "grad_norm": 0.6810351446801922, + "language_loss": 0.49337187, + "learning_rate": 3.9260769132826515e-06, + "loss": 0.51352733, + "num_input_tokens_seen": 111290520, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01025391, + "step": 3936, + "time_per_iteration": 3.0680553913116455 + }, + { + "auxiliary_loss_clip": 0.01105906, + "auxiliary_loss_mlp": 0.01041589, + "balance_loss_clip": 1.03343022, + "balance_loss_mlp": 1.02172875, + "epoch": 0.1142417735476757, + "flos": 12233565734400.0, + "grad_norm": 2.0052535498163078, + "language_loss": 0.68470633, + "learning_rate": 3.926026274660208e-06, + "loss": 0.70618129, + "num_input_tokens_seen": 111304770, + "router_z_loss_clip": 0.72509766, + "router_z_loss_mlp": 0.19848633, + "step": 3937, + "time_per_iteration": 2.4032161235809326 + }, + { + "auxiliary_loss_clip": 0.01097974, + "auxiliary_loss_mlp": 0.0103907, + "balance_loss_clip": 1.03253996, + "balance_loss_mlp": 1.02106917, + "epoch": 0.11427079101619175, + "flos": 20514719583360.0, + "grad_norm": 2.2647712025815854, + "language_loss": 0.70869559, + "learning_rate": 3.925975619026322e-06, + "loss": 0.73006606, + "num_input_tokens_seen": 111319740, + "router_z_loss_clip": 0.65429688, + "router_z_loss_mlp": 0.17999268, + "step": 3938, + "time_per_iteration": 2.337952136993408 + }, + { + "auxiliary_loss_clip": 0.01012879, + "auxiliary_loss_mlp": 0.0100163, + "balance_loss_clip": 1.00254035, + "balance_loss_mlp": 1.00058675, + "epoch": 0.11429980848470779, + "flos": 56935517483520.0, + "grad_norm": 0.7883037345717748, + "language_loss": 0.48698777, + "learning_rate": 3.9259249463814406e-06, + "loss": 0.50713289, + "num_input_tokens_seen": 111376500, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01043701, + "step": 3939, + "time_per_iteration": 2.9666240215301514 + }, + { + "auxiliary_loss_clip": 0.01013211, + "auxiliary_loss_mlp": 0.01001508, + "balance_loss_clip": 1.0028168, + "balance_loss_mlp": 1.000489, + "epoch": 0.11432882595322384, + "flos": 68934851706240.0, + "grad_norm": 0.6495587507448709, + "language_loss": 0.50640374, + "learning_rate": 3.9258742567260095e-06, + "loss": 0.52655095, + "num_input_tokens_seen": 111440920, + "router_z_loss_clip": 0.10400391, + "router_z_loss_mlp": 0.01019287, + "step": 3940, + "time_per_iteration": 3.1995770931243896 + }, + { + "auxiliary_loss_clip": 0.01102231, + "auxiliary_loss_mlp": 0.01041938, + "balance_loss_clip": 1.03264713, + "balance_loss_mlp": 1.02256608, + "epoch": 0.1143578434217399, + "flos": 35442567982080.0, + "grad_norm": 2.4810817257555917, + "language_loss": 0.88826656, + "learning_rate": 3.925823550060478e-06, + "loss": 0.9097082, + "num_input_tokens_seen": 111459425, + "router_z_loss_clip": 0.69580078, + "router_z_loss_mlp": 0.19372559, + "step": 3941, + "time_per_iteration": 2.556486129760742 + }, + { + "auxiliary_loss_clip": 0.01093851, + "auxiliary_loss_mlp": 0.01043119, + "balance_loss_clip": 1.03004348, + "balance_loss_mlp": 1.02480853, + "epoch": 0.11438686089025593, + "flos": 18506724981120.0, + "grad_norm": 3.4493813000410913, + "language_loss": 0.77510625, + "learning_rate": 3.925772826385293e-06, + "loss": 0.79647601, + "num_input_tokens_seen": 111472385, + "router_z_loss_clip": 0.63818359, + "router_z_loss_mlp": 0.18310547, + "step": 3942, + "time_per_iteration": 2.386749029159546 + }, + { + "auxiliary_loss_clip": 0.01092832, + "auxiliary_loss_mlp": 0.01044576, + "balance_loss_clip": 1.0315609, + "balance_loss_mlp": 1.02663493, + "epoch": 0.11441587835877198, + "flos": 10773532920960.0, + "grad_norm": 2.2378022224422933, + "language_loss": 0.66346943, + "learning_rate": 3.9257220857009044e-06, + "loss": 0.68484354, + "num_input_tokens_seen": 111483810, + "router_z_loss_clip": 0.61230469, + "router_z_loss_mlp": 0.17956543, + "step": 3943, + "time_per_iteration": 2.375913619995117 + }, + { + "auxiliary_loss_clip": 0.01014812, + "auxiliary_loss_mlp": 0.01007169, + "balance_loss_clip": 1.00454271, + "balance_loss_mlp": 1.00611949, + "epoch": 0.11444489582728802, + "flos": 58827530949120.0, + "grad_norm": 0.7116948317498252, + "language_loss": 0.54438901, + "learning_rate": 3.9256713280077585e-06, + "loss": 0.56460881, + "num_input_tokens_seen": 111538990, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01049805, + "step": 3944, + "time_per_iteration": 2.909815549850464 + }, + { + "auxiliary_loss_clip": 0.01013687, + "auxiliary_loss_mlp": 0.010056, + "balance_loss_clip": 1.00323844, + "balance_loss_mlp": 1.00461078, + "epoch": 0.11447391329580407, + "flos": 69670471386240.0, + "grad_norm": 0.6517607961424455, + "language_loss": 0.46527719, + "learning_rate": 3.925620553306304e-06, + "loss": 0.48547006, + "num_input_tokens_seen": 111603210, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.0098877, + "step": 3945, + "time_per_iteration": 3.0581820011138916 + }, + { + "auxiliary_loss_clip": 0.0109689, + "auxiliary_loss_mlp": 0.0103801, + "balance_loss_clip": 1.03374064, + "balance_loss_mlp": 1.01844144, + "epoch": 0.11450293076432012, + "flos": 16468146161280.0, + "grad_norm": 2.728942005125054, + "language_loss": 0.75248367, + "learning_rate": 3.92556976159699e-06, + "loss": 0.77383268, + "num_input_tokens_seen": 111617580, + "router_z_loss_clip": 0.6315918, + "router_z_loss_mlp": 0.19573975, + "step": 3946, + "time_per_iteration": 2.4384725093841553 + }, + { + "auxiliary_loss_clip": 0.01106348, + "auxiliary_loss_mlp": 0.01047371, + "balance_loss_clip": 1.03311801, + "balance_loss_mlp": 1.02511489, + "epoch": 0.11453194823283616, + "flos": 29307211338240.0, + "grad_norm": 1.8815693894383885, + "language_loss": 0.79090536, + "learning_rate": 3.925518952880264e-06, + "loss": 0.81244254, + "num_input_tokens_seen": 111634320, + "router_z_loss_clip": 0.73242188, + "router_z_loss_mlp": 0.22253418, + "step": 3947, + "time_per_iteration": 2.4686334133148193 + }, + { + "auxiliary_loss_clip": 0.01100463, + "auxiliary_loss_mlp": 0.01044562, + "balance_loss_clip": 1.03324771, + "balance_loss_mlp": 1.02500021, + "epoch": 0.11456096570135221, + "flos": 45363697125120.0, + "grad_norm": 1.6035480403552211, + "language_loss": 0.88114381, + "learning_rate": 3.925468127156576e-06, + "loss": 0.90259409, + "num_input_tokens_seen": 111657025, + "router_z_loss_clip": 0.67236328, + "router_z_loss_mlp": 0.19567871, + "step": 3948, + "time_per_iteration": 2.6958703994750977 + }, + { + "auxiliary_loss_clip": 0.01098445, + "auxiliary_loss_mlp": 0.01046757, + "balance_loss_clip": 1.03478754, + "balance_loss_mlp": 1.02996588, + "epoch": 0.11458998316986826, + "flos": 11247618539520.0, + "grad_norm": 2.2764201206408825, + "language_loss": 0.69774365, + "learning_rate": 3.9254172844263745e-06, + "loss": 0.71919572, + "num_input_tokens_seen": 111667980, + "router_z_loss_clip": 0.63720703, + "router_z_loss_mlp": 0.16784668, + "step": 3949, + "time_per_iteration": 2.382190465927124 + }, + { + "auxiliary_loss_clip": 0.01014268, + "auxiliary_loss_mlp": 0.01008734, + "balance_loss_clip": 1.00344443, + "balance_loss_mlp": 1.00786424, + "epoch": 0.1146190006383843, + "flos": 63539059726080.0, + "grad_norm": 0.6917092470689671, + "language_loss": 0.4837836, + "learning_rate": 3.925366424690107e-06, + "loss": 0.50401366, + "num_input_tokens_seen": 111736400, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.00872803, + "step": 3950, + "time_per_iteration": 3.3113183975219727 + }, + { + "auxiliary_loss_clip": 0.01013149, + "auxiliary_loss_mlp": 0.01008772, + "balance_loss_clip": 1.00264037, + "balance_loss_mlp": 1.00798523, + "epoch": 0.11464801810690035, + "flos": 64304181371520.0, + "grad_norm": 0.7025556178481979, + "language_loss": 0.53418696, + "learning_rate": 3.9253155479482255e-06, + "loss": 0.55440617, + "num_input_tokens_seen": 111797420, + "router_z_loss_clip": 0.10498047, + "router_z_loss_mlp": 0.00787354, + "step": 3951, + "time_per_iteration": 2.949873208999634 + }, + { + "auxiliary_loss_clip": 0.01100688, + "auxiliary_loss_mlp": 0.01044522, + "balance_loss_clip": 1.032763, + "balance_loss_mlp": 1.02180076, + "epoch": 0.1146770355754164, + "flos": 16135773217920.0, + "grad_norm": 2.810716675118624, + "language_loss": 0.9212147, + "learning_rate": 3.925264654201178e-06, + "loss": 0.94266689, + "num_input_tokens_seen": 111811490, + "router_z_loss_clip": 0.67822266, + "router_z_loss_mlp": 0.22741699, + "step": 3952, + "time_per_iteration": 2.4154398441314697 + }, + { + "auxiliary_loss_clip": 0.0110067, + "auxiliary_loss_mlp": 0.01037318, + "balance_loss_clip": 1.03418136, + "balance_loss_mlp": 1.01902485, + "epoch": 0.11470605304393244, + "flos": 32227207142400.0, + "grad_norm": 2.8008260076311573, + "language_loss": 0.76612151, + "learning_rate": 3.925213743449413e-06, + "loss": 0.78750134, + "num_input_tokens_seen": 111827825, + "router_z_loss_clip": 0.66455078, + "router_z_loss_mlp": 0.18292236, + "step": 3953, + "time_per_iteration": 2.448838949203491 + }, + { + "auxiliary_loss_clip": 0.01104244, + "auxiliary_loss_mlp": 0.01041598, + "balance_loss_clip": 1.03422916, + "balance_loss_mlp": 1.02092123, + "epoch": 0.1147350705124485, + "flos": 20805999989760.0, + "grad_norm": 4.294237182738007, + "language_loss": 1.02763844, + "learning_rate": 3.9251628156933825e-06, + "loss": 1.04909682, + "num_input_tokens_seen": 111840895, + "router_z_loss_clip": 0.70019531, + "router_z_loss_mlp": 0.206604, + "step": 3954, + "time_per_iteration": 2.3933064937591553 + }, + { + "auxiliary_loss_clip": 0.01015065, + "auxiliary_loss_mlp": 0.01003659, + "balance_loss_clip": 1.00462651, + "balance_loss_mlp": 1.00270545, + "epoch": 0.11476408798096455, + "flos": 59507430232320.0, + "grad_norm": 0.6714499134099037, + "language_loss": 0.4894557, + "learning_rate": 3.9251118709335345e-06, + "loss": 0.50964296, + "num_input_tokens_seen": 111893540, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.00952148, + "step": 3955, + "time_per_iteration": 2.7821640968322754 + }, + { + "auxiliary_loss_clip": 0.010937, + "auxiliary_loss_mlp": 0.01039851, + "balance_loss_clip": 1.03099501, + "balance_loss_mlp": 1.0202713, + "epoch": 0.11479310544948058, + "flos": 15808567155840.0, + "grad_norm": 2.1358215612567255, + "language_loss": 0.58855057, + "learning_rate": 3.925060909170318e-06, + "loss": 0.60988605, + "num_input_tokens_seen": 111908995, + "router_z_loss_clip": 0.62670898, + "router_z_loss_mlp": 0.19567871, + "step": 3956, + "time_per_iteration": 2.400707483291626 + }, + { + "auxiliary_loss_clip": 0.01016868, + "auxiliary_loss_mlp": 0.01000899, + "balance_loss_clip": 1.00651574, + "balance_loss_mlp": 0.99998719, + "epoch": 0.11482212291799664, + "flos": 62805744195840.0, + "grad_norm": 0.6741129212431699, + "language_loss": 0.48617062, + "learning_rate": 3.925009930404186e-06, + "loss": 0.50634825, + "num_input_tokens_seen": 111970470, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.00909424, + "step": 3957, + "time_per_iteration": 2.9593732357025146 + }, + { + "auxiliary_loss_clip": 0.01104525, + "auxiliary_loss_mlp": 0.01045935, + "balance_loss_clip": 1.03069437, + "balance_loss_mlp": 1.02415526, + "epoch": 0.11485114038651269, + "flos": 18837945849600.0, + "grad_norm": 2.412391487892027, + "language_loss": 0.8159132, + "learning_rate": 3.924958934635587e-06, + "loss": 0.83741778, + "num_input_tokens_seen": 111985070, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.21789551, + "step": 3958, + "time_per_iteration": 4.656138896942139 + }, + { + "auxiliary_loss_clip": 0.01100797, + "auxiliary_loss_mlp": 0.01041828, + "balance_loss_clip": 1.03226519, + "balance_loss_mlp": 1.02256382, + "epoch": 0.11488015785502873, + "flos": 12925439614080.0, + "grad_norm": 2.7537436903366244, + "language_loss": 0.79486632, + "learning_rate": 3.924907921864973e-06, + "loss": 0.81629252, + "num_input_tokens_seen": 111999115, + "router_z_loss_clip": 0.68554688, + "router_z_loss_mlp": 0.19262695, + "step": 3959, + "time_per_iteration": 4.504788160324097 + }, + { + "auxiliary_loss_clip": 0.01106616, + "auxiliary_loss_mlp": 0.01042599, + "balance_loss_clip": 1.03729153, + "balance_loss_mlp": 1.02207077, + "epoch": 0.11490917532354478, + "flos": 45474581203200.0, + "grad_norm": 1.654438037739855, + "language_loss": 0.63653117, + "learning_rate": 3.924856892092792e-06, + "loss": 0.6580233, + "num_input_tokens_seen": 112022605, + "router_z_loss_clip": 0.69335938, + "router_z_loss_mlp": 0.20532227, + "step": 3960, + "time_per_iteration": 2.6512115001678467 + }, + { + "auxiliary_loss_clip": 0.01101106, + "auxiliary_loss_mlp": 0.01041236, + "balance_loss_clip": 1.03522658, + "balance_loss_mlp": 1.02108943, + "epoch": 0.11493819279206081, + "flos": 16100231587200.0, + "grad_norm": 2.7361896953702542, + "language_loss": 0.80152035, + "learning_rate": 3.924805845319496e-06, + "loss": 0.82294369, + "num_input_tokens_seen": 112035750, + "router_z_loss_clip": 0.65917969, + "router_z_loss_mlp": 0.20166016, + "step": 3961, + "time_per_iteration": 2.345860242843628 + }, + { + "auxiliary_loss_clip": 0.0102194, + "auxiliary_loss_mlp": 0.01007696, + "balance_loss_clip": 1.01104617, + "balance_loss_mlp": 1.00676608, + "epoch": 0.11496721026057687, + "flos": 59543111508480.0, + "grad_norm": 0.7154164697599551, + "language_loss": 0.49229318, + "learning_rate": 3.924754781545536e-06, + "loss": 0.51258957, + "num_input_tokens_seen": 112085960, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.00927734, + "step": 3962, + "time_per_iteration": 2.8487417697906494 + }, + { + "auxiliary_loss_clip": 0.01103713, + "auxiliary_loss_mlp": 0.01039573, + "balance_loss_clip": 1.0343585, + "balance_loss_mlp": 1.01753759, + "epoch": 0.11499622772909292, + "flos": 16684119031680.0, + "grad_norm": 3.5885129155762017, + "language_loss": 0.80372268, + "learning_rate": 3.9247037007713634e-06, + "loss": 0.82515556, + "num_input_tokens_seen": 112099350, + "router_z_loss_clip": 0.69287109, + "router_z_loss_mlp": 0.22039795, + "step": 3963, + "time_per_iteration": 2.3660337924957275 + }, + { + "auxiliary_loss_clip": 0.01017941, + "auxiliary_loss_mlp": 0.01005577, + "balance_loss_clip": 1.0072155, + "balance_loss_mlp": 1.00451565, + "epoch": 0.11502524519760896, + "flos": 61272777818880.0, + "grad_norm": 0.6966432028622427, + "language_loss": 0.4794274, + "learning_rate": 3.924652602997428e-06, + "loss": 0.49966258, + "num_input_tokens_seen": 112159430, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01062012, + "step": 3964, + "time_per_iteration": 5.349365234375 + }, + { + "auxiliary_loss_clip": 0.01106139, + "auxiliary_loss_mlp": 0.01049849, + "balance_loss_clip": 1.03252864, + "balance_loss_mlp": 1.02935719, + "epoch": 0.115054262666125, + "flos": 15516239408640.0, + "grad_norm": 3.2493732281470953, + "language_loss": 0.86496282, + "learning_rate": 3.9246014882241825e-06, + "loss": 0.88652277, + "num_input_tokens_seen": 112172395, + "router_z_loss_clip": 0.73583984, + "router_z_loss_mlp": 0.20495605, + "step": 3965, + "time_per_iteration": 2.3897511959075928 + }, + { + "auxiliary_loss_clip": 0.01104211, + "auxiliary_loss_mlp": 0.01045631, + "balance_loss_clip": 1.0341177, + "balance_loss_mlp": 1.02599764, + "epoch": 0.11508328013464106, + "flos": 24309813415680.0, + "grad_norm": 2.3632822015416464, + "language_loss": 1.04246354, + "learning_rate": 3.924550356452078e-06, + "loss": 1.0639621, + "num_input_tokens_seen": 112189715, + "router_z_loss_clip": 0.70068359, + "router_z_loss_mlp": 0.1965332, + "step": 3966, + "time_per_iteration": 2.412088632583618 + }, + { + "auxiliary_loss_clip": 0.01015599, + "auxiliary_loss_mlp": 0.01001625, + "balance_loss_clip": 1.00533533, + "balance_loss_mlp": 1.00072491, + "epoch": 0.1151122976031571, + "flos": 68078117053440.0, + "grad_norm": 0.6960457107896436, + "language_loss": 0.46749693, + "learning_rate": 3.9244992076815655e-06, + "loss": 0.48766917, + "num_input_tokens_seen": 112253870, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.00897217, + "step": 3967, + "time_per_iteration": 5.580856800079346 + }, + { + "auxiliary_loss_clip": 0.01100479, + "auxiliary_loss_mlp": 0.01039978, + "balance_loss_clip": 1.03197026, + "balance_loss_mlp": 1.0206902, + "epoch": 0.11514131507167315, + "flos": 15229672035840.0, + "grad_norm": 3.7041858973218273, + "language_loss": 0.5324229, + "learning_rate": 3.9244480419130974e-06, + "loss": 0.55382746, + "num_input_tokens_seen": 112270460, + "router_z_loss_clip": 0.68408203, + "router_z_loss_mlp": 0.19311523, + "step": 3968, + "time_per_iteration": 2.369737148284912 + }, + { + "auxiliary_loss_clip": 0.01099942, + "auxiliary_loss_mlp": 0.01045348, + "balance_loss_clip": 1.03023994, + "balance_loss_mlp": 1.02431941, + "epoch": 0.1151703325401892, + "flos": 25548182807040.0, + "grad_norm": 2.5808245534754817, + "language_loss": 0.96887106, + "learning_rate": 3.924396859147126e-06, + "loss": 0.9903239, + "num_input_tokens_seen": 112283700, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.21008301, + "step": 3969, + "time_per_iteration": 2.4562673568725586 + }, + { + "auxiliary_loss_clip": 0.01092607, + "auxiliary_loss_mlp": 0.01042525, + "balance_loss_clip": 1.02975225, + "balance_loss_mlp": 1.02340341, + "epoch": 0.11519935000870524, + "flos": 26212405023360.0, + "grad_norm": 2.1559451466871464, + "language_loss": 0.87651122, + "learning_rate": 3.924345659384103e-06, + "loss": 0.89786255, + "num_input_tokens_seen": 112299215, + "router_z_loss_clip": 0.62890625, + "router_z_loss_mlp": 0.19128418, + "step": 3970, + "time_per_iteration": 2.4820899963378906 + }, + { + "auxiliary_loss_clip": 0.01107715, + "auxiliary_loss_mlp": 0.01049447, + "balance_loss_clip": 1.03443277, + "balance_loss_mlp": 1.02813208, + "epoch": 0.11522836747722129, + "flos": 31788732977280.0, + "grad_norm": 2.378541250430382, + "language_loss": 0.93123227, + "learning_rate": 3.924294442624479e-06, + "loss": 0.95280385, + "num_input_tokens_seen": 112320055, + "router_z_loss_clip": 0.73339844, + "router_z_loss_mlp": 0.21325684, + "step": 3971, + "time_per_iteration": 2.4806199073791504 + }, + { + "auxiliary_loss_clip": 0.01011538, + "auxiliary_loss_mlp": 0.01001032, + "balance_loss_clip": 1.0013361, + "balance_loss_mlp": 0.99995923, + "epoch": 0.11525738494573734, + "flos": 69267469029120.0, + "grad_norm": 0.6280735883581442, + "language_loss": 0.48249462, + "learning_rate": 3.924243208868708e-06, + "loss": 0.50262034, + "num_input_tokens_seen": 112381190, + "router_z_loss_clip": 0.10205078, + "router_z_loss_mlp": 0.01074219, + "step": 3972, + "time_per_iteration": 3.045677661895752 + }, + { + "auxiliary_loss_clip": 0.01105636, + "auxiliary_loss_mlp": 0.01038101, + "balance_loss_clip": 1.03439307, + "balance_loss_mlp": 1.01753736, + "epoch": 0.11528640241425338, + "flos": 24491885489280.0, + "grad_norm": 3.066488248922369, + "language_loss": 0.74579889, + "learning_rate": 3.924191958117243e-06, + "loss": 0.76723623, + "num_input_tokens_seen": 112398815, + "router_z_loss_clip": 0.71142578, + "router_z_loss_mlp": 0.20568848, + "step": 3973, + "time_per_iteration": 2.481597661972046 + }, + { + "auxiliary_loss_clip": 0.01101623, + "auxiliary_loss_mlp": 0.01036093, + "balance_loss_clip": 1.03253686, + "balance_loss_mlp": 1.01684034, + "epoch": 0.11531541988276943, + "flos": 29781296956800.0, + "grad_norm": 1.9931570091934059, + "language_loss": 0.72515404, + "learning_rate": 3.9241406903705365e-06, + "loss": 0.74653119, + "num_input_tokens_seen": 112418150, + "router_z_loss_clip": 0.69042969, + "router_z_loss_mlp": 0.19244385, + "step": 3974, + "time_per_iteration": 2.4754912853240967 + }, + { + "auxiliary_loss_clip": 0.010886, + "auxiliary_loss_mlp": 0.01030449, + "balance_loss_clip": 1.02983069, + "balance_loss_mlp": 1.01366436, + "epoch": 0.11534443735128547, + "flos": 16610626886400.0, + "grad_norm": 1.9605195753416664, + "language_loss": 0.68377542, + "learning_rate": 3.9240894056290395e-06, + "loss": 0.70496589, + "num_input_tokens_seen": 112432280, + "router_z_loss_clip": 0.58789062, + "router_z_loss_mlp": 0.16796875, + "step": 3975, + "time_per_iteration": 2.3989205360412598 + }, + { + "auxiliary_loss_clip": 0.01102671, + "auxiliary_loss_mlp": 0.01038671, + "balance_loss_clip": 1.03351951, + "balance_loss_mlp": 1.01767826, + "epoch": 0.11537345481980152, + "flos": 22920654395520.0, + "grad_norm": 3.0050903189943368, + "language_loss": 0.78377521, + "learning_rate": 3.924038103893208e-06, + "loss": 0.80518866, + "num_input_tokens_seen": 112445850, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.2097168, + "step": 3976, + "time_per_iteration": 2.496860980987549 + }, + { + "auxiliary_loss_clip": 0.01103207, + "auxiliary_loss_mlp": 0.01039186, + "balance_loss_clip": 1.03215909, + "balance_loss_mlp": 1.01982665, + "epoch": 0.11540247228831757, + "flos": 42621409474560.0, + "grad_norm": 2.645036107940912, + "language_loss": 0.72832155, + "learning_rate": 3.9239867851634925e-06, + "loss": 0.74974543, + "num_input_tokens_seen": 112462520, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.19348145, + "step": 3977, + "time_per_iteration": 2.5038645267486572 + }, + { + "auxiliary_loss_clip": 0.01093636, + "auxiliary_loss_mlp": 0.01035581, + "balance_loss_clip": 1.03153753, + "balance_loss_mlp": 1.01830781, + "epoch": 0.11543148975683361, + "flos": 46892509050240.0, + "grad_norm": 2.057724254521195, + "language_loss": 0.75571036, + "learning_rate": 3.923935449440347e-06, + "loss": 0.77700257, + "num_input_tokens_seen": 112480655, + "router_z_loss_clip": 0.62109375, + "router_z_loss_mlp": 0.17272949, + "step": 3978, + "time_per_iteration": 2.606133222579956 + }, + { + "auxiliary_loss_clip": 0.01099293, + "auxiliary_loss_mlp": 0.01039465, + "balance_loss_clip": 1.03080368, + "balance_loss_mlp": 1.01869929, + "epoch": 0.11546050722534966, + "flos": 15259942051200.0, + "grad_norm": 3.216995420339907, + "language_loss": 0.9395172, + "learning_rate": 3.923884096724225e-06, + "loss": 0.96090478, + "num_input_tokens_seen": 112492690, + "router_z_loss_clip": 0.68505859, + "router_z_loss_mlp": 0.20788574, + "step": 3979, + "time_per_iteration": 2.357626438140869 + }, + { + "auxiliary_loss_clip": 0.0101425, + "auxiliary_loss_mlp": 0.01001789, + "balance_loss_clip": 1.00388765, + "balance_loss_mlp": 1.00080562, + "epoch": 0.11548952469386571, + "flos": 74777147464320.0, + "grad_norm": 0.6235776608313601, + "language_loss": 0.5387367, + "learning_rate": 3.92383272701558e-06, + "loss": 0.55889714, + "num_input_tokens_seen": 112557940, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.00982666, + "step": 3980, + "time_per_iteration": 3.126453161239624 + }, + { + "auxiliary_loss_clip": 0.01101634, + "auxiliary_loss_mlp": 0.01036558, + "balance_loss_clip": 1.03325689, + "balance_loss_mlp": 1.0150888, + "epoch": 0.11551854216238175, + "flos": 44666586541440.0, + "grad_norm": 2.304304668701509, + "language_loss": 0.92443693, + "learning_rate": 3.923781340314866e-06, + "loss": 0.9458189, + "num_input_tokens_seen": 112576120, + "router_z_loss_clip": 0.68408203, + "router_z_loss_mlp": 0.21472168, + "step": 3981, + "time_per_iteration": 2.6161551475524902 + }, + { + "auxiliary_loss_clip": 0.01097101, + "auxiliary_loss_mlp": 0.01041778, + "balance_loss_clip": 1.03028345, + "balance_loss_mlp": 1.02151251, + "epoch": 0.1155475596308978, + "flos": 24273154621440.0, + "grad_norm": 2.2335050413885567, + "language_loss": 1.02608299, + "learning_rate": 3.923729936622537e-06, + "loss": 1.04747188, + "num_input_tokens_seen": 112591455, + "router_z_loss_clip": 0.66748047, + "router_z_loss_mlp": 0.20269775, + "step": 3982, + "time_per_iteration": 2.383105993270874 + }, + { + "auxiliary_loss_clip": 0.01014398, + "auxiliary_loss_mlp": 0.0100116, + "balance_loss_clip": 1.00390792, + "balance_loss_mlp": 1.00016463, + "epoch": 0.11557657709941385, + "flos": 57798012510720.0, + "grad_norm": 0.6440102187634195, + "language_loss": 0.45777628, + "learning_rate": 3.9236785159390465e-06, + "loss": 0.47793189, + "num_input_tokens_seen": 112649195, + "router_z_loss_clip": 0.10498047, + "router_z_loss_mlp": 0.00994873, + "step": 3983, + "time_per_iteration": 2.8942999839782715 + }, + { + "auxiliary_loss_clip": 0.01090688, + "auxiliary_loss_mlp": 0.01027629, + "balance_loss_clip": 1.02971148, + "balance_loss_mlp": 1.01067173, + "epoch": 0.11560559456792989, + "flos": 15040896981120.0, + "grad_norm": 3.5092130077215176, + "language_loss": 0.7651695, + "learning_rate": 3.92362707826485e-06, + "loss": 0.78635263, + "num_input_tokens_seen": 112663830, + "router_z_loss_clip": 0.60888672, + "router_z_loss_mlp": 0.1696167, + "step": 3984, + "time_per_iteration": 2.354809522628784 + }, + { + "auxiliary_loss_clip": 0.01013781, + "auxiliary_loss_mlp": 0.0100089, + "balance_loss_clip": 1.00334787, + "balance_loss_mlp": 0.99997836, + "epoch": 0.11563461203644594, + "flos": 52180242906240.0, + "grad_norm": 0.7415258883106854, + "language_loss": 0.55334336, + "learning_rate": 3.923575623600399e-06, + "loss": 0.57349008, + "num_input_tokens_seen": 112720170, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.00909424, + "step": 3985, + "time_per_iteration": 2.893705368041992 + }, + { + "auxiliary_loss_clip": 0.01012523, + "auxiliary_loss_mlp": 0.01000895, + "balance_loss_clip": 1.00244188, + "balance_loss_mlp": 0.99999499, + "epoch": 0.11566362950496199, + "flos": 74771526735360.0, + "grad_norm": 0.5868423284877091, + "language_loss": 0.44995916, + "learning_rate": 3.92352415194615e-06, + "loss": 0.47009334, + "num_input_tokens_seen": 112785350, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.00897217, + "step": 3986, + "time_per_iteration": 3.043294668197632 + }, + { + "auxiliary_loss_clip": 0.010985, + "auxiliary_loss_mlp": 0.01034742, + "balance_loss_clip": 1.03089166, + "balance_loss_mlp": 1.01503694, + "epoch": 0.11569264697347803, + "flos": 16246727118720.0, + "grad_norm": 2.848940919637173, + "language_loss": 0.75362372, + "learning_rate": 3.923472663302558e-06, + "loss": 0.77495623, + "num_input_tokens_seen": 112797010, + "router_z_loss_clip": 0.67578125, + "router_z_loss_mlp": 0.19702148, + "step": 3987, + "time_per_iteration": 2.3789310455322266 + }, + { + "auxiliary_loss_clip": 0.01011163, + "auxiliary_loss_mlp": 0.01001772, + "balance_loss_clip": 1.00077319, + "balance_loss_mlp": 1.00093722, + "epoch": 0.11572166444199408, + "flos": 55323124030080.0, + "grad_norm": 0.6670336456483866, + "language_loss": 0.42428768, + "learning_rate": 3.923421157670077e-06, + "loss": 0.444417, + "num_input_tokens_seen": 112854955, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.00836182, + "step": 3988, + "time_per_iteration": 3.05397629737854 + }, + { + "auxiliary_loss_clip": 0.01010561, + "auxiliary_loss_mlp": 0.01000543, + "balance_loss_clip": 1.00079513, + "balance_loss_mlp": 0.99968511, + "epoch": 0.11575068191051013, + "flos": 63883129841280.0, + "grad_norm": 0.7223007856634951, + "language_loss": 0.49923092, + "learning_rate": 3.9233696350491614e-06, + "loss": 0.51934195, + "num_input_tokens_seen": 112905210, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.00860596, + "step": 3989, + "time_per_iteration": 2.849198579788208 + }, + { + "auxiliary_loss_clip": 0.01098159, + "auxiliary_loss_mlp": 0.01046102, + "balance_loss_clip": 1.03063226, + "balance_loss_mlp": 1.02494216, + "epoch": 0.11577969937902617, + "flos": 54372406129920.0, + "grad_norm": 1.9376092632301174, + "language_loss": 0.8799057, + "learning_rate": 3.923318095440268e-06, + "loss": 0.90134829, + "num_input_tokens_seen": 112926635, + "router_z_loss_clip": 0.67529297, + "router_z_loss_mlp": 0.21191406, + "step": 3990, + "time_per_iteration": 2.723769426345825 + }, + { + "auxiliary_loss_clip": 0.01097848, + "auxiliary_loss_mlp": 0.01041227, + "balance_loss_clip": 1.03002381, + "balance_loss_mlp": 1.02104485, + "epoch": 0.11580871684754222, + "flos": 17047983888000.0, + "grad_norm": 2.171809172311224, + "language_loss": 0.83772421, + "learning_rate": 3.92326653884385e-06, + "loss": 0.85911494, + "num_input_tokens_seen": 112940215, + "router_z_loss_clip": 0.67822266, + "router_z_loss_mlp": 0.20178223, + "step": 3991, + "time_per_iteration": 2.356548547744751 + }, + { + "auxiliary_loss_clip": 0.01094706, + "auxiliary_loss_mlp": 0.01034633, + "balance_loss_clip": 1.0321095, + "balance_loss_mlp": 1.01749659, + "epoch": 0.11583773431605826, + "flos": 19970702778240.0, + "grad_norm": 1.8994110438167702, + "language_loss": 0.79114759, + "learning_rate": 3.9232149652603635e-06, + "loss": 0.81244099, + "num_input_tokens_seen": 112956740, + "router_z_loss_clip": 0.62646484, + "router_z_loss_mlp": 0.17150879, + "step": 3992, + "time_per_iteration": 2.4369680881500244 + }, + { + "auxiliary_loss_clip": 0.01093436, + "auxiliary_loss_mlp": 0.01036621, + "balance_loss_clip": 1.02951169, + "balance_loss_mlp": 1.01805949, + "epoch": 0.11586675178457431, + "flos": 53242477021440.0, + "grad_norm": 2.2249564158531587, + "language_loss": 0.79468971, + "learning_rate": 3.923163374690265e-06, + "loss": 0.81599033, + "num_input_tokens_seen": 112975445, + "router_z_loss_clip": 0.63964844, + "router_z_loss_mlp": 0.18579102, + "step": 3993, + "time_per_iteration": 2.674774646759033 + }, + { + "auxiliary_loss_clip": 0.01100581, + "auxiliary_loss_mlp": 0.01043106, + "balance_loss_clip": 1.03326106, + "balance_loss_mlp": 1.02158856, + "epoch": 0.11589576925309036, + "flos": 11503357315200.0, + "grad_norm": 2.545812847904438, + "language_loss": 0.74410653, + "learning_rate": 3.923111767134009e-06, + "loss": 0.7655434, + "num_input_tokens_seen": 112987485, + "router_z_loss_clip": 0.67431641, + "router_z_loss_mlp": 0.21533203, + "step": 3994, + "time_per_iteration": 2.3627774715423584 + }, + { + "auxiliary_loss_clip": 0.01015743, + "auxiliary_loss_mlp": 0.01003941, + "balance_loss_clip": 1.00574374, + "balance_loss_mlp": 1.00305855, + "epoch": 0.1159247867216064, + "flos": 61123210087680.0, + "grad_norm": 0.6783599016763108, + "language_loss": 0.49248397, + "learning_rate": 3.923060142592052e-06, + "loss": 0.51268077, + "num_input_tokens_seen": 113043450, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.0088501, + "step": 3995, + "time_per_iteration": 2.875253915786743 + }, + { + "auxiliary_loss_clip": 0.01100731, + "auxiliary_loss_mlp": 0.01038665, + "balance_loss_clip": 1.03527403, + "balance_loss_mlp": 1.02019334, + "epoch": 0.11595380419012245, + "flos": 52552523266560.0, + "grad_norm": 1.7611542218483403, + "language_loss": 0.93781084, + "learning_rate": 3.9230085010648495e-06, + "loss": 0.95920485, + "num_input_tokens_seen": 113069505, + "router_z_loss_clip": 0.65478516, + "router_z_loss_mlp": 0.18475342, + "step": 3996, + "time_per_iteration": 2.755741596221924 + }, + { + "auxiliary_loss_clip": 0.01106085, + "auxiliary_loss_mlp": 0.01043325, + "balance_loss_clip": 1.03243804, + "balance_loss_mlp": 1.02035367, + "epoch": 0.1159828216586385, + "flos": 74735009452800.0, + "grad_norm": 1.773505788500052, + "language_loss": 0.87165672, + "learning_rate": 3.922956842552857e-06, + "loss": 0.89315081, + "num_input_tokens_seen": 113095235, + "router_z_loss_clip": 0.73632812, + "router_z_loss_mlp": 0.22961426, + "step": 3997, + "time_per_iteration": 2.7693395614624023 + }, + { + "auxiliary_loss_clip": 0.01102751, + "auxiliary_loss_mlp": 0.01040468, + "balance_loss_clip": 1.03192878, + "balance_loss_mlp": 1.02076268, + "epoch": 0.11601183912715454, + "flos": 30443110289280.0, + "grad_norm": 2.641628765252904, + "language_loss": 1.03172958, + "learning_rate": 3.922905167056532e-06, + "loss": 1.05316174, + "num_input_tokens_seen": 113115130, + "router_z_loss_clip": 0.70800781, + "router_z_loss_mlp": 0.19714355, + "step": 3998, + "time_per_iteration": 2.4965686798095703 + }, + { + "auxiliary_loss_clip": 0.01103567, + "auxiliary_loss_mlp": 0.01040301, + "balance_loss_clip": 1.03426254, + "balance_loss_mlp": 1.01880765, + "epoch": 0.1160408565956706, + "flos": 12341342701440.0, + "grad_norm": 3.221020830097361, + "language_loss": 0.80434453, + "learning_rate": 3.92285347457633e-06, + "loss": 0.82578319, + "num_input_tokens_seen": 113126570, + "router_z_loss_clip": 0.69287109, + "router_z_loss_mlp": 0.21496582, + "step": 3999, + "time_per_iteration": 2.3365631103515625 + }, + { + "auxiliary_loss_clip": 0.01014876, + "auxiliary_loss_mlp": 0.01001192, + "balance_loss_clip": 1.00484097, + "balance_loss_mlp": 1.00028634, + "epoch": 0.11606987406418665, + "flos": 64656246188160.0, + "grad_norm": 0.708492725539548, + "language_loss": 0.4877418, + "learning_rate": 3.922801765112709e-06, + "loss": 0.5079025, + "num_input_tokens_seen": 113181885, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.0090332, + "step": 4000, + "time_per_iteration": 2.9382858276367188 + }, + { + "auxiliary_loss_clip": 0.01098592, + "auxiliary_loss_mlp": 0.01041682, + "balance_loss_clip": 1.03070748, + "balance_loss_mlp": 1.02057552, + "epoch": 0.11609889153270268, + "flos": 30770490908160.0, + "grad_norm": 2.3723774054027853, + "language_loss": 0.87075222, + "learning_rate": 3.922750038666124e-06, + "loss": 0.89215493, + "num_input_tokens_seen": 113201595, + "router_z_loss_clip": 0.67871094, + "router_z_loss_mlp": 0.21099854, + "step": 4001, + "time_per_iteration": 2.4686169624328613 + }, + { + "auxiliary_loss_clip": 0.01103717, + "auxiliary_loss_mlp": 0.0104369, + "balance_loss_clip": 1.03419304, + "balance_loss_mlp": 1.02343678, + "epoch": 0.11612790900121873, + "flos": 23613715261440.0, + "grad_norm": 2.4585097380306835, + "language_loss": 1.07762265, + "learning_rate": 3.922698295237034e-06, + "loss": 1.09909678, + "num_input_tokens_seen": 113215800, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.20263672, + "step": 4002, + "time_per_iteration": 2.42273211479187 + }, + { + "auxiliary_loss_clip": 0.01013625, + "auxiliary_loss_mlp": 0.01002038, + "balance_loss_clip": 1.0036459, + "balance_loss_mlp": 1.0012511, + "epoch": 0.11615692646973479, + "flos": 63241565097600.0, + "grad_norm": 0.6771600181967367, + "language_loss": 0.53720808, + "learning_rate": 3.922646534825893e-06, + "loss": 0.5573647, + "num_input_tokens_seen": 113278060, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.00787354, + "step": 4003, + "time_per_iteration": 2.956754446029663 + }, + { + "auxiliary_loss_clip": 0.01105507, + "auxiliary_loss_mlp": 0.01042012, + "balance_loss_clip": 1.03429902, + "balance_loss_mlp": 1.0199703, + "epoch": 0.11618594393825082, + "flos": 17487470482560.0, + "grad_norm": 1.9046761506126744, + "language_loss": 0.93740112, + "learning_rate": 3.92259475743316e-06, + "loss": 0.95887631, + "num_input_tokens_seen": 113297275, + "router_z_loss_clip": 0.71240234, + "router_z_loss_mlp": 0.22058105, + "step": 4004, + "time_per_iteration": 2.49824595451355 + }, + { + "auxiliary_loss_clip": 0.0101177, + "auxiliary_loss_mlp": 0.01002539, + "balance_loss_clip": 1.00200212, + "balance_loss_mlp": 1.00173438, + "epoch": 0.11621496140676688, + "flos": 60721150337280.0, + "grad_norm": 0.6794547334803652, + "language_loss": 0.48548466, + "learning_rate": 3.9225429630592925e-06, + "loss": 0.50562775, + "num_input_tokens_seen": 113352460, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.00805664, + "step": 4005, + "time_per_iteration": 2.8805270195007324 + }, + { + "auxiliary_loss_clip": 0.01102186, + "auxiliary_loss_mlp": 0.01048947, + "balance_loss_clip": 1.03292584, + "balance_loss_mlp": 1.02861047, + "epoch": 0.11624397887528291, + "flos": 46820273713920.0, + "grad_norm": 2.5837971420244608, + "language_loss": 0.70278442, + "learning_rate": 3.922491151704747e-06, + "loss": 0.72429574, + "num_input_tokens_seen": 113375690, + "router_z_loss_clip": 0.69384766, + "router_z_loss_mlp": 0.20306396, + "step": 4006, + "time_per_iteration": 2.6417694091796875 + }, + { + "auxiliary_loss_clip": 0.01011345, + "auxiliary_loss_mlp": 0.01001057, + "balance_loss_clip": 1.00162292, + "balance_loss_mlp": 1.00025845, + "epoch": 0.11627299634379896, + "flos": 74768349801600.0, + "grad_norm": 0.6677718520043004, + "language_loss": 0.5129022, + "learning_rate": 3.922439323369983e-06, + "loss": 0.53302622, + "num_input_tokens_seen": 113442120, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.00799561, + "step": 4007, + "time_per_iteration": 3.066848039627075 + }, + { + "auxiliary_loss_clip": 0.0101152, + "auxiliary_loss_mlp": 0.01003044, + "balance_loss_clip": 1.00163484, + "balance_loss_mlp": 1.00218022, + "epoch": 0.11630201381231502, + "flos": 67626865330560.0, + "grad_norm": 0.6314905087830861, + "language_loss": 0.48902571, + "learning_rate": 3.922387478055456e-06, + "loss": 0.50917137, + "num_input_tokens_seen": 113503990, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.00866699, + "step": 4008, + "time_per_iteration": 3.0194900035858154 + }, + { + "auxiliary_loss_clip": 0.01101682, + "auxiliary_loss_mlp": 0.01046431, + "balance_loss_clip": 1.03396785, + "balance_loss_mlp": 1.02519441, + "epoch": 0.11633103128083105, + "flos": 38757222328320.0, + "grad_norm": 2.7339939649032425, + "language_loss": 0.95371819, + "learning_rate": 3.922335615761625e-06, + "loss": 0.97519934, + "num_input_tokens_seen": 113520880, + "router_z_loss_clip": 0.67675781, + "router_z_loss_mlp": 0.21234131, + "step": 4009, + "time_per_iteration": 2.689431667327881 + }, + { + "auxiliary_loss_clip": 0.01093775, + "auxiliary_loss_mlp": 0.01035437, + "balance_loss_clip": 1.03037167, + "balance_loss_mlp": 1.01595831, + "epoch": 0.1163600487493471, + "flos": 16537239475200.0, + "grad_norm": 2.7326790501484757, + "language_loss": 0.78501934, + "learning_rate": 3.922283736488947e-06, + "loss": 0.80631143, + "num_input_tokens_seen": 113535160, + "router_z_loss_clip": 0.63427734, + "router_z_loss_mlp": 0.19476318, + "step": 4010, + "time_per_iteration": 2.3636162281036377 + }, + { + "auxiliary_loss_clip": 0.01098911, + "auxiliary_loss_mlp": 0.01036554, + "balance_loss_clip": 1.02961957, + "balance_loss_mlp": 1.0172658, + "epoch": 0.11638906621786316, + "flos": 18251579698560.0, + "grad_norm": 3.034481919644722, + "language_loss": 0.63185734, + "learning_rate": 3.922231840237883e-06, + "loss": 0.65321195, + "num_input_tokens_seen": 113549810, + "router_z_loss_clip": 0.69335938, + "router_z_loss_mlp": 0.19293213, + "step": 4011, + "time_per_iteration": 2.378909111022949 + }, + { + "auxiliary_loss_clip": 0.01099595, + "auxiliary_loss_mlp": 0.01037824, + "balance_loss_clip": 1.03083587, + "balance_loss_mlp": 1.01695061, + "epoch": 0.1164180836863792, + "flos": 18690088775040.0, + "grad_norm": 2.251449900866168, + "language_loss": 0.8288371, + "learning_rate": 3.922179927008888e-06, + "loss": 0.85021126, + "num_input_tokens_seen": 113562975, + "router_z_loss_clip": 0.68652344, + "router_z_loss_mlp": 0.20898438, + "step": 4012, + "time_per_iteration": 2.338468551635742 + }, + { + "auxiliary_loss_clip": 0.01107213, + "auxiliary_loss_mlp": 0.01043091, + "balance_loss_clip": 1.03409493, + "balance_loss_mlp": 1.02246797, + "epoch": 0.11644710115489525, + "flos": 25843896956160.0, + "grad_norm": 1.9504221213235555, + "language_loss": 0.87005293, + "learning_rate": 3.9221279968024236e-06, + "loss": 0.89155602, + "num_input_tokens_seen": 113582995, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.2064209, + "step": 4013, + "time_per_iteration": 2.6035144329071045 + }, + { + "auxiliary_loss_clip": 0.01101543, + "auxiliary_loss_mlp": 0.01039999, + "balance_loss_clip": 1.03010011, + "balance_loss_mlp": 1.01943552, + "epoch": 0.1164761186234113, + "flos": 24926554316160.0, + "grad_norm": 2.608276174963472, + "language_loss": 0.93488508, + "learning_rate": 3.9220760496189455e-06, + "loss": 0.9563005, + "num_input_tokens_seen": 113599260, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.20568848, + "step": 4014, + "time_per_iteration": 2.405101776123047 + }, + { + "auxiliary_loss_clip": 0.01095268, + "auxiliary_loss_mlp": 0.01036029, + "balance_loss_clip": 1.03140116, + "balance_loss_mlp": 1.01583481, + "epoch": 0.11650513609192734, + "flos": 30267357171840.0, + "grad_norm": 1.9187476087952586, + "language_loss": 0.88242042, + "learning_rate": 3.922024085458915e-06, + "loss": 0.90373337, + "num_input_tokens_seen": 113618710, + "router_z_loss_clip": 0.63867188, + "router_z_loss_mlp": 0.20202637, + "step": 4015, + "time_per_iteration": 2.4805262088775635 + }, + { + "auxiliary_loss_clip": 0.01100129, + "auxiliary_loss_mlp": 0.01042606, + "balance_loss_clip": 1.03191853, + "balance_loss_mlp": 1.02019429, + "epoch": 0.11653415356044339, + "flos": 29927617931520.0, + "grad_norm": 2.9708560100009356, + "language_loss": 0.92548645, + "learning_rate": 3.9219721043227885e-06, + "loss": 0.94691372, + "num_input_tokens_seen": 113633685, + "router_z_loss_clip": 0.68212891, + "router_z_loss_mlp": 0.22412109, + "step": 4016, + "time_per_iteration": 2.4381024837493896 + }, + { + "auxiliary_loss_clip": 0.0110792, + "auxiliary_loss_mlp": 0.01043956, + "balance_loss_clip": 1.03634572, + "balance_loss_mlp": 1.02380943, + "epoch": 0.11656317102895944, + "flos": 33323968592640.0, + "grad_norm": 2.144054349617807, + "language_loss": 0.8793478, + "learning_rate": 3.9219201062110285e-06, + "loss": 0.90086651, + "num_input_tokens_seen": 113656670, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.20153809, + "step": 4017, + "time_per_iteration": 2.6459124088287354 + }, + { + "auxiliary_loss_clip": 0.01092752, + "auxiliary_loss_mlp": 0.0103843, + "balance_loss_clip": 1.03086472, + "balance_loss_mlp": 1.02088237, + "epoch": 0.11659218849747548, + "flos": 27008320354560.0, + "grad_norm": 2.144692254137961, + "language_loss": 0.70839131, + "learning_rate": 3.921868091124091e-06, + "loss": 0.72970313, + "num_input_tokens_seen": 113671255, + "router_z_loss_clip": 0.61938477, + "router_z_loss_mlp": 0.17559814, + "step": 4018, + "time_per_iteration": 2.3459091186523438 + }, + { + "auxiliary_loss_clip": 0.01091222, + "auxiliary_loss_mlp": 0.01039702, + "balance_loss_clip": 1.02866435, + "balance_loss_mlp": 1.02173722, + "epoch": 0.11662120596599153, + "flos": 22410468564480.0, + "grad_norm": 2.459216815104009, + "language_loss": 0.91653776, + "learning_rate": 3.9218160590624376e-06, + "loss": 0.9378469, + "num_input_tokens_seen": 113685305, + "router_z_loss_clip": 0.62548828, + "router_z_loss_mlp": 0.1796875, + "step": 4019, + "time_per_iteration": 2.4237074851989746 + }, + { + "auxiliary_loss_clip": 0.01012139, + "auxiliary_loss_mlp": 0.01003714, + "balance_loss_clip": 1.00209785, + "balance_loss_mlp": 1.00275421, + "epoch": 0.11665022343450758, + "flos": 58201713095040.0, + "grad_norm": 0.7800990489818521, + "language_loss": 0.52762336, + "learning_rate": 3.9217640100265265e-06, + "loss": 0.54778188, + "num_input_tokens_seen": 113750560, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.00958252, + "step": 4020, + "time_per_iteration": 3.0564942359924316 + }, + { + "auxiliary_loss_clip": 0.0101282, + "auxiliary_loss_mlp": 0.01002031, + "balance_loss_clip": 1.00272667, + "balance_loss_mlp": 1.00092804, + "epoch": 0.11667924090302362, + "flos": 71013021874560.0, + "grad_norm": 0.6273084472515156, + "language_loss": 0.49389261, + "learning_rate": 3.921711944016819e-06, + "loss": 0.51404113, + "num_input_tokens_seen": 113813025, + "router_z_loss_clip": 0.10107422, + "router_z_loss_mlp": 0.01104736, + "step": 4021, + "time_per_iteration": 3.016819477081299 + }, + { + "auxiliary_loss_clip": 0.01102668, + "auxiliary_loss_mlp": 0.01044492, + "balance_loss_clip": 1.03394842, + "balance_loss_mlp": 1.02382088, + "epoch": 0.11670825837153967, + "flos": 42184227029760.0, + "grad_norm": 2.341916892710388, + "language_loss": 0.71478975, + "learning_rate": 3.921659861033773e-06, + "loss": 0.73626137, + "num_input_tokens_seen": 113829925, + "router_z_loss_clip": 0.6875, + "router_z_loss_mlp": 0.20678711, + "step": 4022, + "time_per_iteration": 2.557988405227661 + }, + { + "auxiliary_loss_clip": 0.01111591, + "auxiliary_loss_mlp": 0.01063367, + "balance_loss_clip": 1.03576529, + "balance_loss_mlp": 1.04006147, + "epoch": 0.11673727584005571, + "flos": 12378141141120.0, + "grad_norm": 3.2960604969422658, + "language_loss": 0.86741418, + "learning_rate": 3.92160776107785e-06, + "loss": 0.88916373, + "num_input_tokens_seen": 113840845, + "router_z_loss_clip": 0.75732422, + "router_z_loss_mlp": 0.23303223, + "step": 4023, + "time_per_iteration": 2.3756353855133057 + }, + { + "auxiliary_loss_clip": 0.01098226, + "auxiliary_loss_mlp": 0.01042388, + "balance_loss_clip": 1.03174722, + "balance_loss_mlp": 1.02375519, + "epoch": 0.11676629330857176, + "flos": 29818269953280.0, + "grad_norm": 2.0860742134490824, + "language_loss": 0.91613352, + "learning_rate": 3.921555644149509e-06, + "loss": 0.9375397, + "num_input_tokens_seen": 113859055, + "router_z_loss_clip": 0.6640625, + "router_z_loss_mlp": 0.1862793, + "step": 4024, + "time_per_iteration": 2.4470012187957764 + }, + { + "auxiliary_loss_clip": 0.01011924, + "auxiliary_loss_mlp": 0.0100065, + "balance_loss_clip": 1.00202918, + "balance_loss_mlp": 0.99975014, + "epoch": 0.11679531077708781, + "flos": 69550405620480.0, + "grad_norm": 0.6351155893230127, + "language_loss": 0.45502463, + "learning_rate": 3.921503510249212e-06, + "loss": 0.47515035, + "num_input_tokens_seen": 113917345, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.00897217, + "step": 4025, + "time_per_iteration": 2.9750781059265137 + }, + { + "auxiliary_loss_clip": 0.01103921, + "auxiliary_loss_mlp": 0.01041397, + "balance_loss_clip": 1.03381443, + "balance_loss_mlp": 1.02063036, + "epoch": 0.11682432824560385, + "flos": 26534374381440.0, + "grad_norm": 2.0794715883860486, + "language_loss": 0.94311488, + "learning_rate": 3.9214513593774175e-06, + "loss": 0.96456814, + "num_input_tokens_seen": 113936020, + "router_z_loss_clip": 0.70117188, + "router_z_loss_mlp": 0.2076416, + "step": 4026, + "time_per_iteration": 2.43837571144104 + }, + { + "auxiliary_loss_clip": 0.01096985, + "auxiliary_loss_mlp": 0.01041455, + "balance_loss_clip": 1.03261805, + "balance_loss_mlp": 1.02201223, + "epoch": 0.1168533457141199, + "flos": 22411201703040.0, + "grad_norm": 2.4960236634053397, + "language_loss": 0.86840326, + "learning_rate": 3.921399191534588e-06, + "loss": 0.88978767, + "num_input_tokens_seen": 113951040, + "router_z_loss_clip": 0.64404297, + "router_z_loss_mlp": 0.19445801, + "step": 4027, + "time_per_iteration": 2.4012045860290527 + }, + { + "auxiliary_loss_clip": 0.0101167, + "auxiliary_loss_mlp": 0.01002675, + "balance_loss_clip": 1.00170422, + "balance_loss_mlp": 1.00175738, + "epoch": 0.11688236318263595, + "flos": 58688227157760.0, + "grad_norm": 0.6871113376925567, + "language_loss": 0.50968003, + "learning_rate": 3.921347006721182e-06, + "loss": 0.52982354, + "num_input_tokens_seen": 114012100, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.00915527, + "step": 4028, + "time_per_iteration": 2.924294948577881 + }, + { + "auxiliary_loss_clip": 0.01011846, + "auxiliary_loss_mlp": 0.01001871, + "balance_loss_clip": 1.00204635, + "balance_loss_mlp": 1.00085759, + "epoch": 0.11691138065115199, + "flos": 60829485886080.0, + "grad_norm": 0.713787728576603, + "language_loss": 0.49448222, + "learning_rate": 3.921294804937663e-06, + "loss": 0.51461941, + "num_input_tokens_seen": 114069340, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.01013184, + "step": 4029, + "time_per_iteration": 2.924527406692505 + }, + { + "auxiliary_loss_clip": 0.01095286, + "auxiliary_loss_mlp": 0.01040138, + "balance_loss_clip": 1.03458691, + "balance_loss_mlp": 1.02289486, + "epoch": 0.11694039811966804, + "flos": 47585220802560.0, + "grad_norm": 2.497522741703602, + "language_loss": 0.86370814, + "learning_rate": 3.9212425861844905e-06, + "loss": 0.88506234, + "num_input_tokens_seen": 114086090, + "router_z_loss_clip": 0.60693359, + "router_z_loss_mlp": 0.17248535, + "step": 4030, + "time_per_iteration": 2.687852382659912 + }, + { + "auxiliary_loss_clip": 0.01011516, + "auxiliary_loss_mlp": 0.01000973, + "balance_loss_clip": 1.00142705, + "balance_loss_mlp": 1.00013876, + "epoch": 0.11696941558818409, + "flos": 66592319656320.0, + "grad_norm": 0.7073447590401147, + "language_loss": 0.51931411, + "learning_rate": 3.921190350462126e-06, + "loss": 0.53943896, + "num_input_tokens_seen": 114137590, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.00836182, + "step": 4031, + "time_per_iteration": 2.870173215866089 + }, + { + "auxiliary_loss_clip": 0.01011564, + "auxiliary_loss_mlp": 0.01001013, + "balance_loss_clip": 1.00166941, + "balance_loss_mlp": 1.00005972, + "epoch": 0.11699843305670013, + "flos": 67986261532800.0, + "grad_norm": 0.6701897419371033, + "language_loss": 0.5277909, + "learning_rate": 3.921138097771031e-06, + "loss": 0.54791665, + "num_input_tokens_seen": 114197110, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.00952148, + "step": 4032, + "time_per_iteration": 3.010056257247925 + }, + { + "auxiliary_loss_clip": 0.01011558, + "auxiliary_loss_mlp": 0.01001316, + "balance_loss_clip": 1.00168633, + "balance_loss_mlp": 1.00038636, + "epoch": 0.11702745052521618, + "flos": 65471327856000.0, + "grad_norm": 0.6755203397930764, + "language_loss": 0.50928092, + "learning_rate": 3.921085828111667e-06, + "loss": 0.52940965, + "num_input_tokens_seen": 114259645, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.00927734, + "step": 4033, + "time_per_iteration": 2.95180344581604 + }, + { + "auxiliary_loss_clip": 0.01011923, + "auxiliary_loss_mlp": 0.01000918, + "balance_loss_clip": 1.00209451, + "balance_loss_mlp": 0.99997026, + "epoch": 0.11705646799373223, + "flos": 58241234620800.0, + "grad_norm": 0.6910541307077787, + "language_loss": 0.48207229, + "learning_rate": 3.921033541484495e-06, + "loss": 0.50220066, + "num_input_tokens_seen": 114315160, + "router_z_loss_clip": 0.09814453, + "router_z_loss_mlp": 0.00946045, + "step": 4034, + "time_per_iteration": 5.06709885597229 + }, + { + "auxiliary_loss_clip": 0.01103868, + "auxiliary_loss_mlp": 0.01047383, + "balance_loss_clip": 1.03244925, + "balance_loss_mlp": 1.02791619, + "epoch": 0.11708548546224827, + "flos": 40107767518080.0, + "grad_norm": 2.3082321784388276, + "language_loss": 0.92985487, + "learning_rate": 3.920981237889978e-06, + "loss": 0.95136744, + "num_input_tokens_seen": 114335045, + "router_z_loss_clip": 0.71484375, + "router_z_loss_mlp": 0.19494629, + "step": 4035, + "time_per_iteration": 4.807422399520874 + }, + { + "auxiliary_loss_clip": 0.01105241, + "auxiliary_loss_mlp": 0.01046838, + "balance_loss_clip": 1.03222895, + "balance_loss_mlp": 1.02483249, + "epoch": 0.11711450293076432, + "flos": 29937358200960.0, + "grad_norm": 2.3291345420904133, + "language_loss": 0.90693408, + "learning_rate": 3.9209289173285766e-06, + "loss": 0.92845488, + "num_input_tokens_seen": 114354260, + "router_z_loss_clip": 0.72998047, + "router_z_loss_mlp": 0.2199707, + "step": 4036, + "time_per_iteration": 2.4362165927886963 + }, + { + "auxiliary_loss_clip": 0.01012016, + "auxiliary_loss_mlp": 0.01006597, + "balance_loss_clip": 1.00222898, + "balance_loss_mlp": 1.00565493, + "epoch": 0.11714352039928037, + "flos": 56676043192320.0, + "grad_norm": 0.6936810219178211, + "language_loss": 0.5360719, + "learning_rate": 3.920876579800754e-06, + "loss": 0.55625802, + "num_input_tokens_seen": 114416515, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.00939941, + "step": 4037, + "time_per_iteration": 2.9368603229522705 + }, + { + "auxiliary_loss_clip": 0.01012794, + "auxiliary_loss_mlp": 0.0100249, + "balance_loss_clip": 1.00319505, + "balance_loss_mlp": 1.00161421, + "epoch": 0.11717253786779641, + "flos": 74779067589120.0, + "grad_norm": 0.649204262789625, + "language_loss": 0.53966653, + "learning_rate": 3.920824225306973e-06, + "loss": 0.5598194, + "num_input_tokens_seen": 114486560, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.00878906, + "step": 4038, + "time_per_iteration": 3.1570940017700195 + }, + { + "auxiliary_loss_clip": 0.01094463, + "auxiliary_loss_mlp": 0.0104529, + "balance_loss_clip": 1.03155041, + "balance_loss_mlp": 1.02616286, + "epoch": 0.11720155533631246, + "flos": 15369813699840.0, + "grad_norm": 2.588667355243422, + "language_loss": 0.70765281, + "learning_rate": 3.9207718538476946e-06, + "loss": 0.72905034, + "num_input_tokens_seen": 114502860, + "router_z_loss_clip": 0.62939453, + "router_z_loss_mlp": 0.19134521, + "step": 4039, + "time_per_iteration": 2.4884252548217773 + }, + { + "auxiliary_loss_clip": 0.01099891, + "auxiliary_loss_mlp": 0.0104113, + "balance_loss_clip": 1.03370881, + "balance_loss_mlp": 1.02171063, + "epoch": 0.1172305728048285, + "flos": 14822934163200.0, + "grad_norm": 2.664101250888947, + "language_loss": 0.87599558, + "learning_rate": 3.920719465423381e-06, + "loss": 0.8974058, + "num_input_tokens_seen": 114517000, + "router_z_loss_clip": 0.66162109, + "router_z_loss_mlp": 0.1940918, + "step": 4040, + "time_per_iteration": 4.899071931838989 + }, + { + "auxiliary_loss_clip": 0.01014889, + "auxiliary_loss_mlp": 0.01001325, + "balance_loss_clip": 1.00525057, + "balance_loss_mlp": 1.00035357, + "epoch": 0.11725959027334455, + "flos": 74771980583040.0, + "grad_norm": 0.6831772517893705, + "language_loss": 0.54361641, + "learning_rate": 3.920667060034497e-06, + "loss": 0.56377852, + "num_input_tokens_seen": 114578505, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.00970459, + "step": 4041, + "time_per_iteration": 3.0519096851348877 + }, + { + "auxiliary_loss_clip": 0.01099328, + "auxiliary_loss_mlp": 0.0104917, + "balance_loss_clip": 1.03269506, + "balance_loss_mlp": 1.02861845, + "epoch": 0.1172886077418606, + "flos": 38028026338560.0, + "grad_norm": 1.8237708898701548, + "language_loss": 0.74048889, + "learning_rate": 3.920614637681505e-06, + "loss": 0.76197392, + "num_input_tokens_seen": 114597185, + "router_z_loss_clip": 0.66699219, + "router_z_loss_mlp": 0.20544434, + "step": 4042, + "time_per_iteration": 2.5811734199523926 + }, + { + "auxiliary_loss_clip": 0.01095506, + "auxiliary_loss_mlp": 0.01038929, + "balance_loss_clip": 1.03177047, + "balance_loss_mlp": 1.02158356, + "epoch": 0.11731762521037664, + "flos": 28617152849280.0, + "grad_norm": 2.5760053945767645, + "language_loss": 0.69023293, + "learning_rate": 3.920562198364866e-06, + "loss": 0.7115773, + "num_input_tokens_seen": 114611725, + "router_z_loss_clip": 0.63720703, + "router_z_loss_mlp": 0.17333984, + "step": 4043, + "time_per_iteration": 4.8641815185546875 + }, + { + "auxiliary_loss_clip": 0.01097197, + "auxiliary_loss_mlp": 0.01041182, + "balance_loss_clip": 1.03381419, + "balance_loss_mlp": 1.02247238, + "epoch": 0.1173466426788927, + "flos": 15516483788160.0, + "grad_norm": 2.250214763157537, + "language_loss": 0.90255916, + "learning_rate": 3.920509742085045e-06, + "loss": 0.92394298, + "num_input_tokens_seen": 114624900, + "router_z_loss_clip": 0.63378906, + "router_z_loss_mlp": 0.18701172, + "step": 4044, + "time_per_iteration": 2.337505578994751 + }, + { + "auxiliary_loss_clip": 0.01018061, + "auxiliary_loss_mlp": 0.00999888, + "balance_loss_clip": 1.00835037, + "balance_loss_mlp": 0.99892223, + "epoch": 0.11737566014740874, + "flos": 74779032677760.0, + "grad_norm": 0.6119397262425924, + "language_loss": 0.50124973, + "learning_rate": 3.920457268842504e-06, + "loss": 0.52142918, + "num_input_tokens_seen": 114693545, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.00964355, + "step": 4045, + "time_per_iteration": 3.1452841758728027 + }, + { + "auxiliary_loss_clip": 0.01101168, + "auxiliary_loss_mlp": 0.01045917, + "balance_loss_clip": 1.03251839, + "balance_loss_mlp": 1.02634287, + "epoch": 0.11740467761592478, + "flos": 33911905754880.0, + "grad_norm": 1.901066701624027, + "language_loss": 0.92742938, + "learning_rate": 3.920404778637708e-06, + "loss": 0.94890022, + "num_input_tokens_seen": 114715810, + "router_z_loss_clip": 0.68652344, + "router_z_loss_mlp": 0.19580078, + "step": 4046, + "time_per_iteration": 2.482412338256836 + }, + { + "auxiliary_loss_clip": 0.01098707, + "auxiliary_loss_mlp": 0.01042607, + "balance_loss_clip": 1.03158975, + "balance_loss_mlp": 1.02327132, + "epoch": 0.11743369508444083, + "flos": 13400886775680.0, + "grad_norm": 2.8364128813667016, + "language_loss": 0.86643171, + "learning_rate": 3.92035227147112e-06, + "loss": 0.8878448, + "num_input_tokens_seen": 114730260, + "router_z_loss_clip": 0.67089844, + "router_z_loss_mlp": 0.1932373, + "step": 4047, + "time_per_iteration": 2.369480609893799 + }, + { + "auxiliary_loss_clip": 0.01101034, + "auxiliary_loss_mlp": 0.01042135, + "balance_loss_clip": 1.03500342, + "balance_loss_mlp": 1.02287126, + "epoch": 0.11746271255295689, + "flos": 20040075383040.0, + "grad_norm": 2.1524010153491115, + "language_loss": 0.86310494, + "learning_rate": 3.920299747343203e-06, + "loss": 0.88453656, + "num_input_tokens_seen": 114743855, + "router_z_loss_clip": 0.66064453, + "router_z_loss_mlp": 0.19287109, + "step": 4048, + "time_per_iteration": 2.3648033142089844 + }, + { + "auxiliary_loss_clip": 0.01093687, + "auxiliary_loss_mlp": 0.01038551, + "balance_loss_clip": 1.03187644, + "balance_loss_mlp": 1.02187395, + "epoch": 0.11749173002147292, + "flos": 24782397845760.0, + "grad_norm": 2.2203386600815773, + "language_loss": 0.69748151, + "learning_rate": 3.920247206254422e-06, + "loss": 0.71880388, + "num_input_tokens_seen": 114759735, + "router_z_loss_clip": 0.61816406, + "router_z_loss_mlp": 0.16656494, + "step": 4049, + "time_per_iteration": 2.446277379989624 + }, + { + "auxiliary_loss_clip": 0.01108799, + "auxiliary_loss_mlp": 0.0104663, + "balance_loss_clip": 1.03563142, + "balance_loss_mlp": 1.0239327, + "epoch": 0.11752074748998897, + "flos": 27189554555520.0, + "grad_norm": 2.8357141231510186, + "language_loss": 0.95573819, + "learning_rate": 3.9201946482052406e-06, + "loss": 0.97729242, + "num_input_tokens_seen": 114776035, + "router_z_loss_clip": 0.73095703, + "router_z_loss_mlp": 0.22692871, + "step": 4050, + "time_per_iteration": 2.4790263175964355 + }, + { + "auxiliary_loss_clip": 0.01016149, + "auxiliary_loss_mlp": 0.01001847, + "balance_loss_clip": 1.0060544, + "balance_loss_mlp": 1.00084579, + "epoch": 0.11754976495850503, + "flos": 74188442252160.0, + "grad_norm": 0.6375322793836535, + "language_loss": 0.47538537, + "learning_rate": 3.920142073196123e-06, + "loss": 0.49556533, + "num_input_tokens_seen": 114839130, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.01000977, + "step": 4051, + "time_per_iteration": 3.1046578884124756 + }, + { + "auxiliary_loss_clip": 0.01106306, + "auxiliary_loss_mlp": 0.01051793, + "balance_loss_clip": 1.03373945, + "balance_loss_mlp": 1.03174162, + "epoch": 0.11757878242702106, + "flos": 20551448200320.0, + "grad_norm": 2.046309730934561, + "language_loss": 0.83835459, + "learning_rate": 3.920089481227534e-06, + "loss": 0.85993558, + "num_input_tokens_seen": 114856485, + "router_z_loss_clip": 0.7265625, + "router_z_loss_mlp": 0.20050049, + "step": 4052, + "time_per_iteration": 2.3722448348999023 + }, + { + "auxiliary_loss_clip": 0.01016623, + "auxiliary_loss_mlp": 0.01004781, + "balance_loss_clip": 1.00625169, + "balance_loss_mlp": 1.00385678, + "epoch": 0.11760779989553712, + "flos": 56478121361280.0, + "grad_norm": 0.7666319731727007, + "language_loss": 0.49863163, + "learning_rate": 3.920036872299937e-06, + "loss": 0.51884562, + "num_input_tokens_seen": 114909740, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.00921631, + "step": 4053, + "time_per_iteration": 2.858264207839966 + }, + { + "auxiliary_loss_clip": 0.01103018, + "auxiliary_loss_mlp": 0.0104874, + "balance_loss_clip": 1.03346336, + "balance_loss_mlp": 1.02755618, + "epoch": 0.11763681736405315, + "flos": 19675058451840.0, + "grad_norm": 2.1070570109702724, + "language_loss": 0.82227612, + "learning_rate": 3.919984246413798e-06, + "loss": 0.84379375, + "num_input_tokens_seen": 114925680, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.21179199, + "step": 4054, + "time_per_iteration": 2.4033749103546143 + }, + { + "auxiliary_loss_clip": 0.01102331, + "auxiliary_loss_mlp": 0.01040701, + "balance_loss_clip": 1.03116095, + "balance_loss_mlp": 1.01904106, + "epoch": 0.1176658348325692, + "flos": 27121508582400.0, + "grad_norm": 2.4301820302814177, + "language_loss": 1.04428768, + "learning_rate": 3.919931603569582e-06, + "loss": 1.06571794, + "num_input_tokens_seen": 114944515, + "router_z_loss_clip": 0.71240234, + "router_z_loss_mlp": 0.2166748, + "step": 4055, + "time_per_iteration": 2.4446306228637695 + }, + { + "auxiliary_loss_clip": 0.01014724, + "auxiliary_loss_mlp": 0.01003148, + "balance_loss_clip": 1.00496495, + "balance_loss_mlp": 1.0021646, + "epoch": 0.11769485230108526, + "flos": 74774110176000.0, + "grad_norm": 0.6687833758363328, + "language_loss": 0.51844978, + "learning_rate": 3.919878943767751e-06, + "loss": 0.53862852, + "num_input_tokens_seen": 115013190, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.00982666, + "step": 4056, + "time_per_iteration": 3.1533336639404297 + }, + { + "auxiliary_loss_clip": 0.01096389, + "auxiliary_loss_mlp": 0.01037587, + "balance_loss_clip": 1.03057468, + "balance_loss_mlp": 1.01932395, + "epoch": 0.1177238697696013, + "flos": 34120721796480.0, + "grad_norm": 2.326674388689628, + "language_loss": 0.86293733, + "learning_rate": 3.919826267008774e-06, + "loss": 0.88427711, + "num_input_tokens_seen": 115031705, + "router_z_loss_clip": 0.65771484, + "router_z_loss_mlp": 0.18280029, + "step": 4057, + "time_per_iteration": 2.5014877319335938 + }, + { + "auxiliary_loss_clip": 0.01107155, + "auxiliary_loss_mlp": 0.01038653, + "balance_loss_clip": 1.0355835, + "balance_loss_mlp": 1.0171479, + "epoch": 0.11775288723811735, + "flos": 40638167026560.0, + "grad_norm": 2.362214181231486, + "language_loss": 0.97536683, + "learning_rate": 3.919773573293114e-06, + "loss": 0.99682486, + "num_input_tokens_seen": 115053335, + "router_z_loss_clip": 0.71679688, + "router_z_loss_mlp": 0.21508789, + "step": 4058, + "time_per_iteration": 2.5808448791503906 + }, + { + "auxiliary_loss_clip": 0.01093068, + "auxiliary_loss_mlp": 0.01041515, + "balance_loss_clip": 1.02937102, + "balance_loss_mlp": 1.0231092, + "epoch": 0.1177819047066334, + "flos": 38172252631680.0, + "grad_norm": 2.7762532703793443, + "language_loss": 0.74284446, + "learning_rate": 3.919720862621237e-06, + "loss": 0.76419032, + "num_input_tokens_seen": 115069255, + "router_z_loss_clip": 0.63720703, + "router_z_loss_mlp": 0.18395996, + "step": 4059, + "time_per_iteration": 2.5195789337158203 + }, + { + "auxiliary_loss_clip": 0.01103522, + "auxiliary_loss_mlp": 0.01044976, + "balance_loss_clip": 1.03797245, + "balance_loss_mlp": 1.02612948, + "epoch": 0.11781092217514944, + "flos": 22994670211200.0, + "grad_norm": 3.341612375649468, + "language_loss": 0.83400363, + "learning_rate": 3.919668134993608e-06, + "loss": 0.85548854, + "num_input_tokens_seen": 115085900, + "router_z_loss_clip": 0.65478516, + "router_z_loss_mlp": 0.18847656, + "step": 4060, + "time_per_iteration": 2.413033962249756 + }, + { + "auxiliary_loss_clip": 0.01013078, + "auxiliary_loss_mlp": 0.01006214, + "balance_loss_clip": 1.00354385, + "balance_loss_mlp": 1.00506353, + "epoch": 0.11783993964366549, + "flos": 64267382931840.0, + "grad_norm": 0.6000103706908123, + "language_loss": 0.50114888, + "learning_rate": 3.919615390410694e-06, + "loss": 0.5213418, + "num_input_tokens_seen": 115149450, + "router_z_loss_clip": 0.09521484, + "router_z_loss_mlp": 0.01147461, + "step": 4061, + "time_per_iteration": 3.042811870574951 + }, + { + "auxiliary_loss_clip": 0.01013512, + "auxiliary_loss_mlp": 0.01014326, + "balance_loss_clip": 1.00401592, + "balance_loss_mlp": 1.0132767, + "epoch": 0.11786895711218154, + "flos": 69446957662080.0, + "grad_norm": 0.6828915683500579, + "language_loss": 0.51253319, + "learning_rate": 3.91956262887296e-06, + "loss": 0.53281158, + "num_input_tokens_seen": 115209255, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01049805, + "step": 4062, + "time_per_iteration": 3.018808364868164 + }, + { + "auxiliary_loss_clip": 0.01012628, + "auxiliary_loss_mlp": 0.01011834, + "balance_loss_clip": 1.00314426, + "balance_loss_mlp": 1.01082635, + "epoch": 0.11789797458069758, + "flos": 65209689060480.0, + "grad_norm": 0.6518800848883174, + "language_loss": 0.5408892, + "learning_rate": 3.919509850380872e-06, + "loss": 0.56113386, + "num_input_tokens_seen": 115270680, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.0100708, + "step": 4063, + "time_per_iteration": 2.986020565032959 + }, + { + "auxiliary_loss_clip": 0.01012139, + "auxiliary_loss_mlp": 0.01019855, + "balance_loss_clip": 1.00268316, + "balance_loss_mlp": 1.01890779, + "epoch": 0.11792699204921363, + "flos": 70792894552320.0, + "grad_norm": 0.6284816552269559, + "language_loss": 0.47589499, + "learning_rate": 3.919457054934896e-06, + "loss": 0.49621493, + "num_input_tokens_seen": 115333050, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.00946045, + "step": 4064, + "time_per_iteration": 3.0278096199035645 + }, + { + "auxiliary_loss_clip": 0.01100914, + "auxiliary_loss_mlp": 0.01040779, + "balance_loss_clip": 1.03464174, + "balance_loss_mlp": 1.0223552, + "epoch": 0.11795600951772968, + "flos": 38209958766720.0, + "grad_norm": 2.9615926266258468, + "language_loss": 0.72670794, + "learning_rate": 3.9194042425354985e-06, + "loss": 0.74812496, + "num_input_tokens_seen": 115351685, + "router_z_loss_clip": 0.66308594, + "router_z_loss_mlp": 0.18438721, + "step": 4065, + "time_per_iteration": 2.534801959991455 + }, + { + "auxiliary_loss_clip": 0.01016002, + "auxiliary_loss_mlp": 0.0102173, + "balance_loss_clip": 1.00628293, + "balance_loss_mlp": 1.02070487, + "epoch": 0.11798502698624572, + "flos": 74775087694080.0, + "grad_norm": 0.6558605192229366, + "language_loss": 0.5020324, + "learning_rate": 3.919351413183146e-06, + "loss": 0.5224098, + "num_input_tokens_seen": 115418440, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.01025391, + "step": 4066, + "time_per_iteration": 3.1414759159088135 + }, + { + "auxiliary_loss_clip": 0.01109841, + "auxiliary_loss_mlp": 0.01044579, + "balance_loss_clip": 1.03568935, + "balance_loss_mlp": 1.02400374, + "epoch": 0.11801404445476177, + "flos": 26132419365120.0, + "grad_norm": 2.0267363145387645, + "language_loss": 0.96200633, + "learning_rate": 3.919298566878306e-06, + "loss": 0.98355055, + "num_input_tokens_seen": 115432880, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.20556641, + "step": 4067, + "time_per_iteration": 2.4255800247192383 + }, + { + "auxiliary_loss_clip": 0.01111824, + "auxiliary_loss_mlp": 0.01046724, + "balance_loss_clip": 1.04046631, + "balance_loss_mlp": 1.02599335, + "epoch": 0.11804306192327782, + "flos": 50723144513280.0, + "grad_norm": 2.3242903752266946, + "language_loss": 0.86029983, + "learning_rate": 3.9192457036214435e-06, + "loss": 0.88188529, + "num_input_tokens_seen": 115451015, + "router_z_loss_clip": 0.71313477, + "router_z_loss_mlp": 0.20727539, + "step": 4068, + "time_per_iteration": 2.6660585403442383 + }, + { + "auxiliary_loss_clip": 0.01105438, + "auxiliary_loss_mlp": 0.01043435, + "balance_loss_clip": 1.03967166, + "balance_loss_mlp": 1.02380157, + "epoch": 0.11807207939179386, + "flos": 15988614370560.0, + "grad_norm": 4.6705650289487854, + "language_loss": 0.90267825, + "learning_rate": 3.919192823413026e-06, + "loss": 0.92416698, + "num_input_tokens_seen": 115461350, + "router_z_loss_clip": 0.65820312, + "router_z_loss_mlp": 0.19628906, + "step": 4069, + "time_per_iteration": 2.3626511096954346 + }, + { + "auxiliary_loss_clip": 0.01033755, + "auxiliary_loss_mlp": 0.01009869, + "balance_loss_clip": 1.02295256, + "balance_loss_mlp": 1.00890338, + "epoch": 0.11810109686030991, + "flos": 66718320353280.0, + "grad_norm": 0.6247850820169155, + "language_loss": 0.48007482, + "learning_rate": 3.919139926253522e-06, + "loss": 0.50051105, + "num_input_tokens_seen": 115527770, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.00964355, + "step": 4070, + "time_per_iteration": 3.158235549926758 + }, + { + "auxiliary_loss_clip": 0.01030625, + "auxiliary_loss_mlp": 0.01020159, + "balance_loss_clip": 1.01949966, + "balance_loss_mlp": 1.01919317, + "epoch": 0.11813011432882595, + "flos": 48864052460160.0, + "grad_norm": 0.7184667877623272, + "language_loss": 0.4877758, + "learning_rate": 3.919087012143398e-06, + "loss": 0.50828362, + "num_input_tokens_seen": 115582695, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.00964355, + "step": 4071, + "time_per_iteration": 2.827831745147705 + }, + { + "auxiliary_loss_clip": 0.01095372, + "auxiliary_loss_mlp": 0.01056602, + "balance_loss_clip": 1.03404307, + "balance_loss_mlp": 1.04047918, + "epoch": 0.118159131797342, + "flos": 12489618712320.0, + "grad_norm": 2.7435400538907686, + "language_loss": 0.84634423, + "learning_rate": 3.919034081083119e-06, + "loss": 0.86786401, + "num_input_tokens_seen": 115595475, + "router_z_loss_clip": 0.61230469, + "router_z_loss_mlp": 0.16125488, + "step": 4072, + "time_per_iteration": 2.3631350994110107 + }, + { + "auxiliary_loss_clip": 0.01106012, + "auxiliary_loss_mlp": 0.01054638, + "balance_loss_clip": 1.03623152, + "balance_loss_mlp": 1.034814, + "epoch": 0.11818814926585805, + "flos": 74735882236800.0, + "grad_norm": 2.685782359043624, + "language_loss": 0.69619071, + "learning_rate": 3.918981133073156e-06, + "loss": 0.71779716, + "num_input_tokens_seen": 115620695, + "router_z_loss_clip": 0.69824219, + "router_z_loss_mlp": 0.19812012, + "step": 4073, + "time_per_iteration": 2.813300371170044 + }, + { + "auxiliary_loss_clip": 0.01099895, + "auxiliary_loss_mlp": 0.01065274, + "balance_loss_clip": 1.03627682, + "balance_loss_mlp": 1.04645634, + "epoch": 0.11821716673437409, + "flos": 29454125806080.0, + "grad_norm": 3.168624457102334, + "language_loss": 0.7437923, + "learning_rate": 3.918928168113974e-06, + "loss": 0.76544398, + "num_input_tokens_seen": 115641470, + "router_z_loss_clip": 0.63525391, + "router_z_loss_mlp": 0.18823242, + "step": 4074, + "time_per_iteration": 2.474740505218506 + }, + { + "auxiliary_loss_clip": 0.01094685, + "auxiliary_loss_mlp": 0.01054417, + "balance_loss_clip": 1.03543758, + "balance_loss_mlp": 1.03769231, + "epoch": 0.11824618420289014, + "flos": 15186519728640.0, + "grad_norm": 5.901314324012359, + "language_loss": 0.76651841, + "learning_rate": 3.918875186206042e-06, + "loss": 0.78800946, + "num_input_tokens_seen": 115653870, + "router_z_loss_clip": 0.59301758, + "router_z_loss_mlp": 0.16723633, + "step": 4075, + "time_per_iteration": 2.4226739406585693 + }, + { + "auxiliary_loss_clip": 0.01032072, + "auxiliary_loss_mlp": 0.01027132, + "balance_loss_clip": 1.0215019, + "balance_loss_mlp": 1.02617812, + "epoch": 0.11827520167140619, + "flos": 63501528147840.0, + "grad_norm": 0.7311033393615339, + "language_loss": 0.5118981, + "learning_rate": 3.918822187349829e-06, + "loss": 0.53249013, + "num_input_tokens_seen": 115713885, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.00952148, + "step": 4076, + "time_per_iteration": 2.9699084758758545 + }, + { + "auxiliary_loss_clip": 0.0103008, + "auxiliary_loss_mlp": 0.01023314, + "balance_loss_clip": 1.01941085, + "balance_loss_mlp": 1.022277, + "epoch": 0.11830421913992223, + "flos": 67354229456640.0, + "grad_norm": 0.6324421802141657, + "language_loss": 0.46343631, + "learning_rate": 3.918769171545801e-06, + "loss": 0.48397025, + "num_input_tokens_seen": 115772080, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01037598, + "step": 4077, + "time_per_iteration": 3.00376558303833 + }, + { + "auxiliary_loss_clip": 0.01104284, + "auxiliary_loss_mlp": 0.01037515, + "balance_loss_clip": 1.03759456, + "balance_loss_mlp": 1.01943135, + "epoch": 0.11833323660843828, + "flos": 16574107737600.0, + "grad_norm": 6.96559475147114, + "language_loss": 0.78073186, + "learning_rate": 3.918716138794427e-06, + "loss": 0.80214989, + "num_input_tokens_seen": 115784590, + "router_z_loss_clip": 0.66699219, + "router_z_loss_mlp": 0.1809082, + "step": 4078, + "time_per_iteration": 2.345510721206665 + }, + { + "auxiliary_loss_clip": 0.01103132, + "auxiliary_loss_mlp": 0.0104445, + "balance_loss_clip": 1.03774357, + "balance_loss_mlp": 1.02469707, + "epoch": 0.11836225407695433, + "flos": 18690333154560.0, + "grad_norm": 2.175525253801101, + "language_loss": 0.79126811, + "learning_rate": 3.918663089096177e-06, + "loss": 0.81274396, + "num_input_tokens_seen": 115798510, + "router_z_loss_clip": 0.65380859, + "router_z_loss_mlp": 0.19726562, + "step": 4079, + "time_per_iteration": 2.4058303833007812 + }, + { + "auxiliary_loss_clip": 0.01038012, + "auxiliary_loss_mlp": 0.01008669, + "balance_loss_clip": 1.02721465, + "balance_loss_mlp": 1.00722647, + "epoch": 0.11839127154547037, + "flos": 52061536817280.0, + "grad_norm": 0.7088007664577481, + "language_loss": 0.49041197, + "learning_rate": 3.918610022451517e-06, + "loss": 0.5108788, + "num_input_tokens_seen": 115858975, + "router_z_loss_clip": 0.10791016, + "router_z_loss_mlp": 0.0144043, + "step": 4080, + "time_per_iteration": 3.0873215198516846 + }, + { + "auxiliary_loss_clip": 0.01110927, + "auxiliary_loss_mlp": 0.010374, + "balance_loss_clip": 1.04375267, + "balance_loss_mlp": 1.01764727, + "epoch": 0.11842028901398642, + "flos": 29963822878080.0, + "grad_norm": 2.0048457737586522, + "language_loss": 0.83292925, + "learning_rate": 3.918556938860917e-06, + "loss": 0.85441256, + "num_input_tokens_seen": 115886140, + "router_z_loss_clip": 0.67138672, + "router_z_loss_mlp": 0.19732666, + "step": 4081, + "time_per_iteration": 2.811976671218872 + }, + { + "auxiliary_loss_clip": 0.01115757, + "auxiliary_loss_mlp": 0.01054778, + "balance_loss_clip": 1.04448855, + "balance_loss_mlp": 1.0329504, + "epoch": 0.11844930648250247, + "flos": 13621502856960.0, + "grad_norm": 4.4972451883243325, + "language_loss": 0.82588923, + "learning_rate": 3.918503838324846e-06, + "loss": 0.84759456, + "num_input_tokens_seen": 115899685, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.21850586, + "step": 4082, + "time_per_iteration": 2.358410120010376 + }, + { + "auxiliary_loss_clip": 0.01113439, + "auxiliary_loss_mlp": 0.01043415, + "balance_loss_clip": 1.04533792, + "balance_loss_mlp": 1.02415085, + "epoch": 0.11847832395101851, + "flos": 14312119927680.0, + "grad_norm": 2.5551210464881584, + "language_loss": 0.86995959, + "learning_rate": 3.9184507208437725e-06, + "loss": 0.89152813, + "num_input_tokens_seen": 115911165, + "router_z_loss_clip": 0.68115234, + "router_z_loss_mlp": 0.19274902, + "step": 4083, + "time_per_iteration": 2.349902391433716 + }, + { + "auxiliary_loss_clip": 0.01114633, + "auxiliary_loss_mlp": 0.01044003, + "balance_loss_clip": 1.04547691, + "balance_loss_mlp": 1.02261662, + "epoch": 0.11850734141953456, + "flos": 21612039615360.0, + "grad_norm": 2.2827036299175987, + "language_loss": 0.92257845, + "learning_rate": 3.918397586418167e-06, + "loss": 0.94416475, + "num_input_tokens_seen": 115928845, + "router_z_loss_clip": 0.69238281, + "router_z_loss_mlp": 0.21362305, + "step": 4084, + "time_per_iteration": 2.431382179260254 + }, + { + "auxiliary_loss_clip": 0.01114139, + "auxiliary_loss_mlp": 0.01046372, + "balance_loss_clip": 1.04629862, + "balance_loss_mlp": 1.02723885, + "epoch": 0.1185363588880506, + "flos": 20771994458880.0, + "grad_norm": 3.179258157309048, + "language_loss": 0.82054424, + "learning_rate": 3.918344435048496e-06, + "loss": 0.84214938, + "num_input_tokens_seen": 115943210, + "router_z_loss_clip": 0.67773438, + "router_z_loss_mlp": 0.19140625, + "step": 4085, + "time_per_iteration": 2.416649341583252 + }, + { + "auxiliary_loss_clip": 0.01124915, + "auxiliary_loss_mlp": 0.0105112, + "balance_loss_clip": 1.04974496, + "balance_loss_mlp": 1.02915025, + "epoch": 0.11856537635656665, + "flos": 47440366104960.0, + "grad_norm": 2.0828204745000214, + "language_loss": 0.83015466, + "learning_rate": 3.918291266735232e-06, + "loss": 0.85191506, + "num_input_tokens_seen": 115962070, + "router_z_loss_clip": 0.75097656, + "router_z_loss_mlp": 0.21942139, + "step": 4086, + "time_per_iteration": 2.6223411560058594 + }, + { + "auxiliary_loss_clip": 0.0106163, + "auxiliary_loss_mlp": 0.01000283, + "balance_loss_clip": 1.04687023, + "balance_loss_mlp": 0.99876279, + "epoch": 0.1185943938250827, + "flos": 74776030300800.0, + "grad_norm": 0.6574291176384263, + "language_loss": 0.50696689, + "learning_rate": 3.9182380814788425e-06, + "loss": 0.52758604, + "num_input_tokens_seen": 116027130, + "router_z_loss_clip": 0.14746094, + "router_z_loss_mlp": 0.01519775, + "step": 4087, + "time_per_iteration": 3.113823890686035 + }, + { + "auxiliary_loss_clip": 0.01110233, + "auxiliary_loss_mlp": 0.01049967, + "balance_loss_clip": 1.04497039, + "balance_loss_mlp": 1.03081059, + "epoch": 0.11862341129359874, + "flos": 35259658035840.0, + "grad_norm": 2.157333984111194, + "language_loss": 0.77330446, + "learning_rate": 3.918184879279799e-06, + "loss": 0.79490644, + "num_input_tokens_seen": 116050625, + "router_z_loss_clip": 0.65332031, + "router_z_loss_mlp": 0.19177246, + "step": 4088, + "time_per_iteration": 2.581829071044922 + }, + { + "auxiliary_loss_clip": 0.01111366, + "auxiliary_loss_mlp": 0.01058321, + "balance_loss_clip": 1.04133701, + "balance_loss_mlp": 1.03742385, + "epoch": 0.11865242876211479, + "flos": 34646513005440.0, + "grad_norm": 1.933687026539112, + "language_loss": 0.67145562, + "learning_rate": 3.918131660138569e-06, + "loss": 0.69315255, + "num_input_tokens_seen": 116070080, + "router_z_loss_clip": 0.70117188, + "router_z_loss_mlp": 0.20898438, + "step": 4089, + "time_per_iteration": 2.5274059772491455 + }, + { + "auxiliary_loss_clip": 0.01119688, + "auxiliary_loss_mlp": 0.01067328, + "balance_loss_clip": 1.0457468, + "balance_loss_mlp": 1.04689574, + "epoch": 0.11868144623063084, + "flos": 35289928051200.0, + "grad_norm": 2.5108348848967257, + "language_loss": 0.8013016, + "learning_rate": 3.918078424055626e-06, + "loss": 0.82317168, + "num_input_tokens_seen": 116084980, + "router_z_loss_clip": 0.73925781, + "router_z_loss_mlp": 0.20446777, + "step": 4090, + "time_per_iteration": 2.574246883392334 + }, + { + "auxiliary_loss_clip": 0.01113159, + "auxiliary_loss_mlp": 0.01064259, + "balance_loss_clip": 1.04309511, + "balance_loss_mlp": 1.04455948, + "epoch": 0.11871046369914688, + "flos": 19820681199360.0, + "grad_norm": 2.4992453920776803, + "language_loss": 0.67927605, + "learning_rate": 3.918025171031436e-06, + "loss": 0.70105022, + "num_input_tokens_seen": 116098505, + "router_z_loss_clip": 0.70043945, + "router_z_loss_mlp": 0.19720459, + "step": 4091, + "time_per_iteration": 2.4318246841430664 + }, + { + "auxiliary_loss_clip": 0.01044807, + "auxiliary_loss_mlp": 0.01007562, + "balance_loss_clip": 1.03051376, + "balance_loss_mlp": 1.00607145, + "epoch": 0.11873948116766293, + "flos": 74770793596800.0, + "grad_norm": 0.6326155331909367, + "language_loss": 0.49949491, + "learning_rate": 3.917971901066473e-06, + "loss": 0.52001864, + "num_input_tokens_seen": 116162680, + "router_z_loss_clip": 0.14257812, + "router_z_loss_mlp": 0.01489258, + "step": 4092, + "time_per_iteration": 3.102231502532959 + }, + { + "auxiliary_loss_clip": 0.01113485, + "auxiliary_loss_mlp": 0.01051302, + "balance_loss_clip": 1.04074061, + "balance_loss_mlp": 1.02946258, + "epoch": 0.11876849863617898, + "flos": 14128441931520.0, + "grad_norm": 5.189019461875811, + "language_loss": 0.91721296, + "learning_rate": 3.917918614161206e-06, + "loss": 0.93886077, + "num_input_tokens_seen": 116174190, + "router_z_loss_clip": 0.72753906, + "router_z_loss_mlp": 0.21850586, + "step": 4093, + "time_per_iteration": 2.3641433715820312 + }, + { + "auxiliary_loss_clip": 0.01037677, + "auxiliary_loss_mlp": 0.01002128, + "balance_loss_clip": 1.02496481, + "balance_loss_mlp": 1.00084019, + "epoch": 0.11879751610469502, + "flos": 67037112777600.0, + "grad_norm": 0.6818052666914973, + "language_loss": 0.49609739, + "learning_rate": 3.917865310316105e-06, + "loss": 0.51649541, + "num_input_tokens_seen": 116237345, + "router_z_loss_clip": 0.12695312, + "router_z_loss_mlp": 0.01287842, + "step": 4094, + "time_per_iteration": 3.165323495864868 + }, + { + "auxiliary_loss_clip": 0.0110375, + "auxiliary_loss_mlp": 0.01046607, + "balance_loss_clip": 1.03932524, + "balance_loss_mlp": 1.02804041, + "epoch": 0.11882653357321107, + "flos": 35399799699840.0, + "grad_norm": 2.615541645818323, + "language_loss": 0.78511405, + "learning_rate": 3.917811989531642e-06, + "loss": 0.80661756, + "num_input_tokens_seen": 116253210, + "router_z_loss_clip": 0.64428711, + "router_z_loss_mlp": 0.18566895, + "step": 4095, + "time_per_iteration": 2.5669620037078857 + }, + { + "auxiliary_loss_clip": 0.01032736, + "auxiliary_loss_mlp": 0.01007691, + "balance_loss_clip": 1.02102566, + "balance_loss_mlp": 1.00641525, + "epoch": 0.11885555104172713, + "flos": 53351578753920.0, + "grad_norm": 0.7478383076330665, + "language_loss": 0.51943552, + "learning_rate": 3.917758651808287e-06, + "loss": 0.53983974, + "num_input_tokens_seen": 116308440, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01275635, + "step": 4096, + "time_per_iteration": 2.8922529220581055 + }, + { + "auxiliary_loss_clip": 0.01118818, + "auxiliary_loss_mlp": 0.01056476, + "balance_loss_clip": 1.0410955, + "balance_loss_mlp": 1.03246737, + "epoch": 0.11888456851024316, + "flos": 46717558894080.0, + "grad_norm": 2.4333535266894213, + "language_loss": 0.86821973, + "learning_rate": 3.917705297146511e-06, + "loss": 0.88997263, + "num_input_tokens_seen": 116329735, + "router_z_loss_clip": 0.77685547, + "router_z_loss_mlp": 0.2401123, + "step": 4097, + "time_per_iteration": 2.6109983921051025 + }, + { + "auxiliary_loss_clip": 0.01026848, + "auxiliary_loss_mlp": 0.01006964, + "balance_loss_clip": 1.01599264, + "balance_loss_mlp": 1.00569487, + "epoch": 0.11891358597875921, + "flos": 70787169089280.0, + "grad_norm": 0.6840974662551429, + "language_loss": 0.46526915, + "learning_rate": 3.9176519255467875e-06, + "loss": 0.48560727, + "num_input_tokens_seen": 116385180, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01269531, + "step": 4098, + "time_per_iteration": 3.0131235122680664 + }, + { + "auxiliary_loss_clip": 0.01109956, + "auxiliary_loss_mlp": 0.01044974, + "balance_loss_clip": 1.04210711, + "balance_loss_mlp": 1.02486372, + "epoch": 0.11894260344727527, + "flos": 17485585269120.0, + "grad_norm": 2.2815576435037914, + "language_loss": 0.99816024, + "learning_rate": 3.917598537009585e-06, + "loss": 1.01970959, + "num_input_tokens_seen": 116400840, + "router_z_loss_clip": 0.67822266, + "router_z_loss_mlp": 0.2010498, + "step": 4099, + "time_per_iteration": 2.435131072998047 + }, + { + "auxiliary_loss_clip": 0.01122991, + "auxiliary_loss_mlp": 0.01060074, + "balance_loss_clip": 1.04474115, + "balance_loss_mlp": 1.036852, + "epoch": 0.1189716209157913, + "flos": 14207519894400.0, + "grad_norm": 2.349854424703329, + "language_loss": 0.86620992, + "learning_rate": 3.917545131535377e-06, + "loss": 0.88804054, + "num_input_tokens_seen": 116414015, + "router_z_loss_clip": 0.78173828, + "router_z_loss_mlp": 0.23217773, + "step": 4100, + "time_per_iteration": 2.34672474861145 + }, + { + "auxiliary_loss_clip": 0.01111666, + "auxiliary_loss_mlp": 0.01041734, + "balance_loss_clip": 1.04540277, + "balance_loss_mlp": 1.02223134, + "epoch": 0.11900063838430736, + "flos": 23762619676800.0, + "grad_norm": 2.387068881615143, + "language_loss": 1.08723342, + "learning_rate": 3.917491709124634e-06, + "loss": 1.10876739, + "num_input_tokens_seen": 116430320, + "router_z_loss_clip": 0.66259766, + "router_z_loss_mlp": 0.19519043, + "step": 4101, + "time_per_iteration": 2.4317049980163574 + }, + { + "auxiliary_loss_clip": 0.01108329, + "auxiliary_loss_mlp": 0.01033981, + "balance_loss_clip": 1.04765701, + "balance_loss_mlp": 1.01636779, + "epoch": 0.1190296558528234, + "flos": 36714105031680.0, + "grad_norm": 2.392285236858854, + "language_loss": 0.65836746, + "learning_rate": 3.9174382697778284e-06, + "loss": 0.6797905, + "num_input_tokens_seen": 116448875, + "router_z_loss_clip": 0.60742188, + "router_z_loss_mlp": 0.17614746, + "step": 4102, + "time_per_iteration": 2.5486602783203125 + }, + { + "auxiliary_loss_clip": 0.01125497, + "auxiliary_loss_mlp": 0.01051251, + "balance_loss_clip": 1.04779863, + "balance_loss_mlp": 1.02779055, + "epoch": 0.11905867332133944, + "flos": 30440107912320.0, + "grad_norm": 1.8836410209549603, + "language_loss": 0.81827557, + "learning_rate": 3.917384813495431e-06, + "loss": 0.84004301, + "num_input_tokens_seen": 116472115, + "router_z_loss_clip": 0.77734375, + "router_z_loss_mlp": 0.23474121, + "step": 4103, + "time_per_iteration": 2.656792640686035 + }, + { + "auxiliary_loss_clip": 0.01129717, + "auxiliary_loss_mlp": 0.01040003, + "balance_loss_clip": 1.05338645, + "balance_loss_mlp": 1.01642346, + "epoch": 0.1190876907898555, + "flos": 11646850469760.0, + "grad_norm": 2.7686589433700837, + "language_loss": 0.89245987, + "learning_rate": 3.917331340277917e-06, + "loss": 0.91415703, + "num_input_tokens_seen": 116483385, + "router_z_loss_clip": 0.76269531, + "router_z_loss_mlp": 0.23583984, + "step": 4104, + "time_per_iteration": 2.349808692932129 + }, + { + "auxiliary_loss_clip": 0.01125017, + "auxiliary_loss_mlp": 0.01043031, + "balance_loss_clip": 1.0574975, + "balance_loss_mlp": 1.02160895, + "epoch": 0.11911670825837153, + "flos": 21351517983360.0, + "grad_norm": 2.321546575663678, + "language_loss": 0.85091728, + "learning_rate": 3.917277850125755e-06, + "loss": 0.87259781, + "num_input_tokens_seen": 116498020, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.21411133, + "step": 4105, + "time_per_iteration": 2.4321694374084473 + }, + { + "auxiliary_loss_clip": 0.01119908, + "auxiliary_loss_mlp": 0.01042511, + "balance_loss_clip": 1.05105591, + "balance_loss_mlp": 1.02199507, + "epoch": 0.11914572572688759, + "flos": 17228799152640.0, + "grad_norm": 2.3689575615832257, + "language_loss": 0.94787586, + "learning_rate": 3.91722434303942e-06, + "loss": 0.96950001, + "num_input_tokens_seen": 116510550, + "router_z_loss_clip": 0.68945312, + "router_z_loss_mlp": 0.20507812, + "step": 4106, + "time_per_iteration": 2.3975775241851807 + }, + { + "auxiliary_loss_clip": 0.01039943, + "auxiliary_loss_mlp": 0.01028476, + "balance_loss_clip": 1.02650547, + "balance_loss_mlp": 1.02711678, + "epoch": 0.11917474319540364, + "flos": 65899433347200.0, + "grad_norm": 0.7387596700560433, + "language_loss": 0.50310427, + "learning_rate": 3.917170819019384e-06, + "loss": 0.52378851, + "num_input_tokens_seen": 116573130, + "router_z_loss_clip": 0.13476562, + "router_z_loss_mlp": 0.01361084, + "step": 4107, + "time_per_iteration": 2.9989702701568604 + }, + { + "auxiliary_loss_clip": 0.0103797, + "auxiliary_loss_mlp": 0.01020248, + "balance_loss_clip": 1.02428222, + "balance_loss_mlp": 1.01888871, + "epoch": 0.11920376066391968, + "flos": 74757038520960.0, + "grad_norm": 0.6713319702907387, + "language_loss": 0.49513859, + "learning_rate": 3.91711727806612e-06, + "loss": 0.51572078, + "num_input_tokens_seen": 116627850, + "router_z_loss_clip": 0.13671875, + "router_z_loss_mlp": 0.01361084, + "step": 4108, + "time_per_iteration": 2.9362223148345947 + }, + { + "auxiliary_loss_clip": 0.0111401, + "auxiliary_loss_mlp": 0.01043864, + "balance_loss_clip": 1.04899716, + "balance_loss_mlp": 1.02384877, + "epoch": 0.11923277813243573, + "flos": 11974999138560.0, + "grad_norm": 4.256572564611885, + "language_loss": 0.99005079, + "learning_rate": 3.917063720180099e-06, + "loss": 1.01162958, + "num_input_tokens_seen": 116638420, + "router_z_loss_clip": 0.65087891, + "router_z_loss_mlp": 0.20031738, + "step": 4109, + "time_per_iteration": 2.388159990310669 + }, + { + "auxiliary_loss_clip": 0.01118063, + "auxiliary_loss_mlp": 0.01055942, + "balance_loss_clip": 1.05090666, + "balance_loss_mlp": 1.03462696, + "epoch": 0.11926179560095178, + "flos": 28869121198080.0, + "grad_norm": 2.114777296455376, + "language_loss": 0.95721519, + "learning_rate": 3.917010145361796e-06, + "loss": 0.97895515, + "num_input_tokens_seen": 116655610, + "router_z_loss_clip": 0.67236328, + "router_z_loss_mlp": 0.21313477, + "step": 4110, + "time_per_iteration": 2.456023931503296 + }, + { + "auxiliary_loss_clip": 0.01129285, + "auxiliary_loss_mlp": 0.01062108, + "balance_loss_clip": 1.05152285, + "balance_loss_mlp": 1.03782463, + "epoch": 0.11929081306946782, + "flos": 17379414224640.0, + "grad_norm": 2.4461220679991094, + "language_loss": 0.73628676, + "learning_rate": 3.916956553611684e-06, + "loss": 0.75820071, + "num_input_tokens_seen": 116669440, + "router_z_loss_clip": 0.77783203, + "router_z_loss_mlp": 0.24291992, + "step": 4111, + "time_per_iteration": 7.019374132156372 + }, + { + "auxiliary_loss_clip": 0.01117989, + "auxiliary_loss_mlp": 0.01055052, + "balance_loss_clip": 1.04629874, + "balance_loss_mlp": 1.03403568, + "epoch": 0.11931983053798387, + "flos": 35764956276480.0, + "grad_norm": 3.4007194333205204, + "language_loss": 0.63398796, + "learning_rate": 3.9169029449302355e-06, + "loss": 0.65571839, + "num_input_tokens_seen": 116685695, + "router_z_loss_clip": 0.71679688, + "router_z_loss_mlp": 0.21020508, + "step": 4112, + "time_per_iteration": 2.5300521850585938 + }, + { + "auxiliary_loss_clip": 0.01036167, + "auxiliary_loss_mlp": 0.01001796, + "balance_loss_clip": 1.02337027, + "balance_loss_mlp": 1.00041318, + "epoch": 0.11934884800649992, + "flos": 58605797704320.0, + "grad_norm": 0.6420022385614498, + "language_loss": 0.47734261, + "learning_rate": 3.9168493193179256e-06, + "loss": 0.49772224, + "num_input_tokens_seen": 116743190, + "router_z_loss_clip": 0.12792969, + "router_z_loss_mlp": 0.01385498, + "step": 4113, + "time_per_iteration": 2.8682379722595215 + }, + { + "auxiliary_loss_clip": 0.01107531, + "auxiliary_loss_mlp": 0.0104741, + "balance_loss_clip": 1.04740858, + "balance_loss_mlp": 1.03024411, + "epoch": 0.11937786547501596, + "flos": 20915068677120.0, + "grad_norm": 2.4248678446449086, + "language_loss": 0.79919064, + "learning_rate": 3.916795676775225e-06, + "loss": 0.82073998, + "num_input_tokens_seen": 116756015, + "router_z_loss_clip": 0.6015625, + "router_z_loss_mlp": 0.17175293, + "step": 4114, + "time_per_iteration": 2.411555290222168 + }, + { + "auxiliary_loss_clip": 0.0112254, + "auxiliary_loss_mlp": 0.01071859, + "balance_loss_clip": 1.04640365, + "balance_loss_mlp": 1.04529953, + "epoch": 0.11940688294353201, + "flos": 29160157224960.0, + "grad_norm": 2.383184279105901, + "language_loss": 0.98333079, + "learning_rate": 3.9167420173026105e-06, + "loss": 1.00527477, + "num_input_tokens_seen": 116769995, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.26525879, + "step": 4115, + "time_per_iteration": 2.514658212661743 + }, + { + "auxiliary_loss_clip": 0.01113407, + "auxiliary_loss_mlp": 0.01058548, + "balance_loss_clip": 1.04211259, + "balance_loss_mlp": 1.03701901, + "epoch": 0.11943590041204805, + "flos": 29454195628800.0, + "grad_norm": 2.0936604876102582, + "language_loss": 0.63668656, + "learning_rate": 3.916688340900555e-06, + "loss": 0.65840614, + "num_input_tokens_seen": 116785950, + "router_z_loss_clip": 0.71337891, + "router_z_loss_mlp": 0.2154541, + "step": 4116, + "time_per_iteration": 4.968485116958618 + }, + { + "auxiliary_loss_clip": 0.01105026, + "auxiliary_loss_mlp": 0.01068041, + "balance_loss_clip": 1.04077506, + "balance_loss_mlp": 1.04983747, + "epoch": 0.1194649178805641, + "flos": 16100091941760.0, + "grad_norm": 2.7079974311347055, + "language_loss": 0.72399282, + "learning_rate": 3.916634647569533e-06, + "loss": 0.74572349, + "num_input_tokens_seen": 116800110, + "router_z_loss_clip": 0.64208984, + "router_z_loss_mlp": 0.18200684, + "step": 4117, + "time_per_iteration": 2.3972420692443848 + }, + { + "auxiliary_loss_clip": 0.01102782, + "auxiliary_loss_mlp": 0.0105701, + "balance_loss_clip": 1.03773618, + "balance_loss_mlp": 1.03779376, + "epoch": 0.11949393534908015, + "flos": 16828903906560.0, + "grad_norm": 2.2696258287339353, + "language_loss": 0.62860388, + "learning_rate": 3.916580937310017e-06, + "loss": 0.6502018, + "num_input_tokens_seen": 116816185, + "router_z_loss_clip": 0.64941406, + "router_z_loss_mlp": 0.19226074, + "step": 4118, + "time_per_iteration": 2.468414783477783 + }, + { + "auxiliary_loss_clip": 0.01106585, + "auxiliary_loss_mlp": 0.01057356, + "balance_loss_clip": 1.03962839, + "balance_loss_mlp": 1.03880155, + "epoch": 0.11952295281759619, + "flos": 29816978232960.0, + "grad_norm": 2.346290214086321, + "language_loss": 0.83192801, + "learning_rate": 3.9165272101224834e-06, + "loss": 0.85356748, + "num_input_tokens_seen": 116831840, + "router_z_loss_clip": 0.66992188, + "router_z_loss_mlp": 0.18554688, + "step": 4119, + "time_per_iteration": 4.923456430435181 + }, + { + "auxiliary_loss_clip": 0.01103531, + "auxiliary_loss_mlp": 0.01060933, + "balance_loss_clip": 1.03946841, + "balance_loss_mlp": 1.04182386, + "epoch": 0.11955197028611224, + "flos": 31897452551040.0, + "grad_norm": 1.665728885931038, + "language_loss": 0.78018844, + "learning_rate": 3.916473466007405e-06, + "loss": 0.80183315, + "num_input_tokens_seen": 116853015, + "router_z_loss_clip": 0.640625, + "router_z_loss_mlp": 0.19116211, + "step": 4120, + "time_per_iteration": 2.535219430923462 + }, + { + "auxiliary_loss_clip": 0.01104082, + "auxiliary_loss_mlp": 0.01050255, + "balance_loss_clip": 1.03688526, + "balance_loss_mlp": 1.02975094, + "epoch": 0.11958098775462829, + "flos": 44667214945920.0, + "grad_norm": 2.3830192123268907, + "language_loss": 0.89412904, + "learning_rate": 3.916419704965259e-06, + "loss": 0.91567248, + "num_input_tokens_seen": 116873985, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.20507812, + "step": 4121, + "time_per_iteration": 2.623995304107666 + }, + { + "auxiliary_loss_clip": 0.0110354, + "auxiliary_loss_mlp": 0.01048833, + "balance_loss_clip": 1.03969204, + "balance_loss_mlp": 1.0296402, + "epoch": 0.11961000522314433, + "flos": 21937395375360.0, + "grad_norm": 2.1859122702527003, + "language_loss": 0.8913312, + "learning_rate": 3.916365926996517e-06, + "loss": 0.91285491, + "num_input_tokens_seen": 116890795, + "router_z_loss_clip": 0.63818359, + "router_z_loss_mlp": 0.19177246, + "step": 4122, + "time_per_iteration": 2.398693799972534 + }, + { + "auxiliary_loss_clip": 0.0103157, + "auxiliary_loss_mlp": 0.01016587, + "balance_loss_clip": 1.02005601, + "balance_loss_mlp": 1.01532304, + "epoch": 0.11963902269166038, + "flos": 74788842769920.0, + "grad_norm": 0.6362534444483663, + "language_loss": 0.50017273, + "learning_rate": 3.916312132101657e-06, + "loss": 0.52065432, + "num_input_tokens_seen": 116961340, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01263428, + "step": 4123, + "time_per_iteration": 3.2738425731658936 + }, + { + "auxiliary_loss_clip": 0.01029909, + "auxiliary_loss_mlp": 0.01016834, + "balance_loss_clip": 1.01814127, + "balance_loss_mlp": 1.01559985, + "epoch": 0.11966804016017643, + "flos": 74782069966080.0, + "grad_norm": 0.6658286026748367, + "language_loss": 0.48819077, + "learning_rate": 3.916258320281152e-06, + "loss": 0.50865817, + "num_input_tokens_seen": 117028615, + "router_z_loss_clip": 0.11767578, + "router_z_loss_mlp": 0.0123291, + "step": 4124, + "time_per_iteration": 3.2119784355163574 + }, + { + "auxiliary_loss_clip": 0.01028964, + "auxiliary_loss_mlp": 0.01012584, + "balance_loss_clip": 1.01744604, + "balance_loss_mlp": 1.01151073, + "epoch": 0.11969705762869247, + "flos": 61703850775680.0, + "grad_norm": 0.7157135829700514, + "language_loss": 0.53606826, + "learning_rate": 3.916204491535478e-06, + "loss": 0.55648375, + "num_input_tokens_seen": 117083115, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01074219, + "step": 4125, + "time_per_iteration": 2.847158193588257 + }, + { + "auxiliary_loss_clip": 0.01115872, + "auxiliary_loss_mlp": 0.01044353, + "balance_loss_clip": 1.04458261, + "balance_loss_mlp": 1.02364588, + "epoch": 0.11972607509720852, + "flos": 21572378444160.0, + "grad_norm": 2.507541532562126, + "language_loss": 0.7545675, + "learning_rate": 3.9161506458651115e-06, + "loss": 0.77616966, + "num_input_tokens_seen": 117099230, + "router_z_loss_clip": 0.71191406, + "router_z_loss_mlp": 0.20721436, + "step": 4126, + "time_per_iteration": 2.426302194595337 + }, + { + "auxiliary_loss_clip": 0.01116262, + "auxiliary_loss_mlp": 0.01049575, + "balance_loss_clip": 1.04380846, + "balance_loss_mlp": 1.02872515, + "epoch": 0.11975509256572457, + "flos": 14857009516800.0, + "grad_norm": 3.08936877046033, + "language_loss": 0.96826327, + "learning_rate": 3.916096783270526e-06, + "loss": 0.98992163, + "num_input_tokens_seen": 117109955, + "router_z_loss_clip": 0.72509766, + "router_z_loss_mlp": 0.20861816, + "step": 4127, + "time_per_iteration": 2.4179704189300537 + }, + { + "auxiliary_loss_clip": 0.01034741, + "auxiliary_loss_mlp": 0.01004525, + "balance_loss_clip": 1.02240252, + "balance_loss_mlp": 1.00340414, + "epoch": 0.11978411003424061, + "flos": 48977170865280.0, + "grad_norm": 0.6751679715367144, + "language_loss": 0.50038028, + "learning_rate": 3.916042903752199e-06, + "loss": 0.52077293, + "num_input_tokens_seen": 117167285, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01123047, + "step": 4128, + "time_per_iteration": 2.926647901535034 + }, + { + "auxiliary_loss_clip": 0.01116976, + "auxiliary_loss_mlp": 0.01057686, + "balance_loss_clip": 1.05089784, + "balance_loss_mlp": 1.03748035, + "epoch": 0.11981312750275666, + "flos": 32631396485760.0, + "grad_norm": 1.8680888009376517, + "language_loss": 0.9235611, + "learning_rate": 3.915989007310605e-06, + "loss": 0.94530773, + "num_input_tokens_seen": 117187545, + "router_z_loss_clip": 0.66137695, + "router_z_loss_mlp": 0.20233154, + "step": 4129, + "time_per_iteration": 2.5313379764556885 + }, + { + "auxiliary_loss_clip": 0.01120404, + "auxiliary_loss_mlp": 0.01055518, + "balance_loss_clip": 1.05002546, + "balance_loss_mlp": 1.03396463, + "epoch": 0.11984214497127271, + "flos": 32567435141760.0, + "grad_norm": 2.7951665355437347, + "language_loss": 0.88815641, + "learning_rate": 3.9159350939462216e-06, + "loss": 0.90991563, + "num_input_tokens_seen": 117207415, + "router_z_loss_clip": 0.70410156, + "router_z_loss_mlp": 0.21533203, + "step": 4130, + "time_per_iteration": 2.5303003787994385 + }, + { + "auxiliary_loss_clip": 0.01114006, + "auxiliary_loss_mlp": 0.01063621, + "balance_loss_clip": 1.04805362, + "balance_loss_mlp": 1.04258084, + "epoch": 0.11987116243978875, + "flos": 30693298158720.0, + "grad_norm": 2.0946640108000687, + "language_loss": 0.81179166, + "learning_rate": 3.915881163659524e-06, + "loss": 0.83356786, + "num_input_tokens_seen": 117223735, + "router_z_loss_clip": 0.65917969, + "router_z_loss_mlp": 0.21044922, + "step": 4131, + "time_per_iteration": 2.5128607749938965 + }, + { + "auxiliary_loss_clip": 0.01043535, + "auxiliary_loss_mlp": 0.01005458, + "balance_loss_clip": 1.03035855, + "balance_loss_mlp": 1.00408745, + "epoch": 0.1199001799083048, + "flos": 56822887837440.0, + "grad_norm": 0.6788593350279516, + "language_loss": 0.49945915, + "learning_rate": 3.915827216450989e-06, + "loss": 0.51994908, + "num_input_tokens_seen": 117285645, + "router_z_loss_clip": 0.13183594, + "router_z_loss_mlp": 0.01373291, + "step": 4132, + "time_per_iteration": 3.048222303390503 + }, + { + "auxiliary_loss_clip": 0.01116731, + "auxiliary_loss_mlp": 0.0106045, + "balance_loss_clip": 1.04748368, + "balance_loss_mlp": 1.03906345, + "epoch": 0.11992919737682084, + "flos": 31901048421120.0, + "grad_norm": 1.7336807612896603, + "language_loss": 0.83482933, + "learning_rate": 3.915773252321091e-06, + "loss": 0.85660118, + "num_input_tokens_seen": 117304535, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.21386719, + "step": 4133, + "time_per_iteration": 2.4815664291381836 + }, + { + "auxiliary_loss_clip": 0.01119025, + "auxiliary_loss_mlp": 0.01072426, + "balance_loss_clip": 1.04543734, + "balance_loss_mlp": 1.04947829, + "epoch": 0.11995821484533689, + "flos": 48022054133760.0, + "grad_norm": 3.224945259361045, + "language_loss": 0.94189602, + "learning_rate": 3.91571927127031e-06, + "loss": 0.9638105, + "num_input_tokens_seen": 117319740, + "router_z_loss_clip": 0.73583984, + "router_z_loss_mlp": 0.22961426, + "step": 4134, + "time_per_iteration": 2.6171083450317383 + }, + { + "auxiliary_loss_clip": 0.01111069, + "auxiliary_loss_mlp": 0.01075918, + "balance_loss_clip": 1.04437184, + "balance_loss_mlp": 1.05245805, + "epoch": 0.11998723231385294, + "flos": 36093209679360.0, + "grad_norm": 2.2708122684660856, + "language_loss": 1.01260495, + "learning_rate": 3.915665273299121e-06, + "loss": 1.03447485, + "num_input_tokens_seen": 117338995, + "router_z_loss_clip": 0.66601562, + "router_z_loss_mlp": 0.234375, + "step": 4135, + "time_per_iteration": 2.5242526531219482 + }, + { + "auxiliary_loss_clip": 0.01115503, + "auxiliary_loss_mlp": 0.01071954, + "balance_loss_clip": 1.0436995, + "balance_loss_mlp": 1.04873157, + "epoch": 0.12001624978236898, + "flos": 27451193351040.0, + "grad_norm": 2.3185762260560514, + "language_loss": 0.89535248, + "learning_rate": 3.915611258408002e-06, + "loss": 0.91722709, + "num_input_tokens_seen": 117354265, + "router_z_loss_clip": 0.71826172, + "router_z_loss_mlp": 0.23205566, + "step": 4136, + "time_per_iteration": 2.501422643661499 + }, + { + "auxiliary_loss_clip": 0.01110065, + "auxiliary_loss_mlp": 0.01062349, + "balance_loss_clip": 1.04043651, + "balance_loss_mlp": 1.03935313, + "epoch": 0.12004526725088503, + "flos": 18035257714560.0, + "grad_norm": 2.406173746523251, + "language_loss": 0.85826504, + "learning_rate": 3.9155572265974275e-06, + "loss": 0.87998915, + "num_input_tokens_seen": 117368480, + "router_z_loss_clip": 0.69677734, + "router_z_loss_mlp": 0.22998047, + "step": 4137, + "time_per_iteration": 2.3998606204986572 + }, + { + "auxiliary_loss_clip": 0.01103992, + "auxiliary_loss_mlp": 0.01062421, + "balance_loss_clip": 1.03878057, + "balance_loss_mlp": 1.04138041, + "epoch": 0.12007428471940108, + "flos": 39085440819840.0, + "grad_norm": 2.1311265792274408, + "language_loss": 0.72492206, + "learning_rate": 3.915503177867877e-06, + "loss": 0.7465862, + "num_input_tokens_seen": 117387655, + "router_z_loss_clip": 0.65283203, + "router_z_loss_mlp": 0.21044922, + "step": 4138, + "time_per_iteration": 2.610748291015625 + }, + { + "auxiliary_loss_clip": 0.01103758, + "auxiliary_loss_mlp": 0.01059087, + "balance_loss_clip": 1.03886425, + "balance_loss_mlp": 1.03907776, + "epoch": 0.12010330218791712, + "flos": 30951585463680.0, + "grad_norm": 1.7552800407862112, + "language_loss": 0.80237389, + "learning_rate": 3.915449112219828e-06, + "loss": 0.82400239, + "num_input_tokens_seen": 117407030, + "router_z_loss_clip": 0.6484375, + "router_z_loss_mlp": 0.20013428, + "step": 4139, + "time_per_iteration": 2.5520975589752197 + }, + { + "auxiliary_loss_clip": 0.01108072, + "auxiliary_loss_mlp": 0.01057225, + "balance_loss_clip": 1.03986835, + "balance_loss_mlp": 1.03501606, + "epoch": 0.12013231965643317, + "flos": 28687747351680.0, + "grad_norm": 32.07928241416786, + "language_loss": 0.87884188, + "learning_rate": 3.9153950296537564e-06, + "loss": 0.90049481, + "num_input_tokens_seen": 117422445, + "router_z_loss_clip": 0.68164062, + "router_z_loss_mlp": 0.22192383, + "step": 4140, + "time_per_iteration": 2.5197865962982178 + }, + { + "auxiliary_loss_clip": 0.01108327, + "auxiliary_loss_mlp": 0.0104709, + "balance_loss_clip": 1.03761315, + "balance_loss_mlp": 1.02379692, + "epoch": 0.12016133712494922, + "flos": 39523216757760.0, + "grad_norm": 1.8894433995201212, + "language_loss": 0.83276069, + "learning_rate": 3.9153409301701414e-06, + "loss": 0.8543148, + "num_input_tokens_seen": 117443810, + "router_z_loss_clip": 0.70800781, + "router_z_loss_mlp": 0.23303223, + "step": 4141, + "time_per_iteration": 2.5515661239624023 + }, + { + "auxiliary_loss_clip": 0.01107005, + "auxiliary_loss_mlp": 0.01047186, + "balance_loss_clip": 1.03840637, + "balance_loss_mlp": 1.02590752, + "epoch": 0.12019035459346526, + "flos": 12814311156480.0, + "grad_norm": 2.343392836007381, + "language_loss": 0.55615753, + "learning_rate": 3.91528681376946e-06, + "loss": 0.57769942, + "num_input_tokens_seen": 117456665, + "router_z_loss_clip": 0.68603516, + "router_z_loss_mlp": 0.21289062, + "step": 4142, + "time_per_iteration": 2.4736669063568115 + }, + { + "auxiliary_loss_clip": 0.01108955, + "auxiliary_loss_mlp": 0.01041514, + "balance_loss_clip": 1.03562403, + "balance_loss_mlp": 1.01828051, + "epoch": 0.12021937206198131, + "flos": 16463363304960.0, + "grad_norm": 3.1776496315315907, + "language_loss": 0.75833577, + "learning_rate": 3.91523268045219e-06, + "loss": 0.77984047, + "num_input_tokens_seen": 117468690, + "router_z_loss_clip": 0.73291016, + "router_z_loss_mlp": 0.23242188, + "step": 4143, + "time_per_iteration": 2.3467085361480713 + }, + { + "auxiliary_loss_clip": 0.01095381, + "auxiliary_loss_mlp": 0.01037553, + "balance_loss_clip": 1.03431356, + "balance_loss_mlp": 1.01878905, + "epoch": 0.12024838953049737, + "flos": 17522872467840.0, + "grad_norm": 2.3409330846980936, + "language_loss": 0.75821877, + "learning_rate": 3.915178530218811e-06, + "loss": 0.77954805, + "num_input_tokens_seen": 117482890, + "router_z_loss_clip": 0.61035156, + "router_z_loss_mlp": 0.1875, + "step": 4144, + "time_per_iteration": 2.432853937149048 + }, + { + "auxiliary_loss_clip": 0.01024323, + "auxiliary_loss_mlp": 0.00998798, + "balance_loss_clip": 1.0127728, + "balance_loss_mlp": 0.99755865, + "epoch": 0.1202774069990134, + "flos": 68707392998400.0, + "grad_norm": 0.6737725833720466, + "language_loss": 0.48541474, + "learning_rate": 3.915124363069799e-06, + "loss": 0.50564599, + "num_input_tokens_seen": 117541705, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01239014, + "step": 4145, + "time_per_iteration": 3.0277915000915527 + }, + { + "auxiliary_loss_clip": 0.01105304, + "auxiliary_loss_mlp": 0.0104832, + "balance_loss_clip": 1.03458285, + "balance_loss_mlp": 1.02526522, + "epoch": 0.12030642446752945, + "flos": 18178297021440.0, + "grad_norm": 5.652131753358485, + "language_loss": 0.77853262, + "learning_rate": 3.915070179005635e-06, + "loss": 0.80006886, + "num_input_tokens_seen": 117556605, + "router_z_loss_clip": 0.70800781, + "router_z_loss_mlp": 0.23059082, + "step": 4146, + "time_per_iteration": 2.3922247886657715 + }, + { + "auxiliary_loss_clip": 0.01100151, + "auxiliary_loss_mlp": 0.0104634, + "balance_loss_clip": 1.03683674, + "balance_loss_mlp": 1.02711725, + "epoch": 0.1203354419360455, + "flos": 23469035120640.0, + "grad_norm": 2.45156566459716, + "language_loss": 0.9352051, + "learning_rate": 3.915015978026795e-06, + "loss": 0.95666993, + "num_input_tokens_seen": 117570440, + "router_z_loss_clip": 0.63378906, + "router_z_loss_mlp": 0.19232178, + "step": 4147, + "time_per_iteration": 2.4119956493377686 + }, + { + "auxiliary_loss_clip": 0.0110639, + "auxiliary_loss_mlp": 0.01050486, + "balance_loss_clip": 1.03748977, + "balance_loss_mlp": 1.02626312, + "epoch": 0.12036445940456154, + "flos": 15662455649280.0, + "grad_norm": 2.966181499582223, + "language_loss": 0.99451649, + "learning_rate": 3.91496176013376e-06, + "loss": 1.01608515, + "num_input_tokens_seen": 117587570, + "router_z_loss_clip": 0.69091797, + "router_z_loss_mlp": 0.24206543, + "step": 4148, + "time_per_iteration": 2.546365976333618 + }, + { + "auxiliary_loss_clip": 0.01111819, + "auxiliary_loss_mlp": 0.01046277, + "balance_loss_clip": 1.03945875, + "balance_loss_mlp": 1.02428293, + "epoch": 0.1203934768730776, + "flos": 19636793735040.0, + "grad_norm": 2.8294827553047046, + "language_loss": 0.89908969, + "learning_rate": 3.914907525327007e-06, + "loss": 0.92067069, + "num_input_tokens_seen": 117598485, + "router_z_loss_clip": 0.72363281, + "router_z_loss_mlp": 0.21984863, + "step": 4149, + "time_per_iteration": 2.3750033378601074 + }, + { + "auxiliary_loss_clip": 0.01107561, + "auxiliary_loss_mlp": 0.01040916, + "balance_loss_clip": 1.03759742, + "balance_loss_mlp": 1.0206387, + "epoch": 0.12042249434159363, + "flos": 13509745994880.0, + "grad_norm": 2.2042658971680615, + "language_loss": 0.9481591, + "learning_rate": 3.914853273607017e-06, + "loss": 0.96964389, + "num_input_tokens_seen": 117613075, + "router_z_loss_clip": 0.69897461, + "router_z_loss_mlp": 0.20269775, + "step": 4150, + "time_per_iteration": 2.464897871017456 + }, + { + "auxiliary_loss_clip": 0.01108657, + "auxiliary_loss_mlp": 0.01045679, + "balance_loss_clip": 1.03925836, + "balance_loss_mlp": 1.02407217, + "epoch": 0.12045151181010968, + "flos": 32188663134720.0, + "grad_norm": 2.5813175811893463, + "language_loss": 0.89073622, + "learning_rate": 3.914799004974266e-06, + "loss": 0.91227961, + "num_input_tokens_seen": 117628400, + "router_z_loss_clip": 0.69482422, + "router_z_loss_mlp": 0.21600342, + "step": 4151, + "time_per_iteration": 2.489713191986084 + }, + { + "auxiliary_loss_clip": 0.01105202, + "auxiliary_loss_mlp": 0.01053356, + "balance_loss_clip": 1.03741217, + "balance_loss_mlp": 1.03045535, + "epoch": 0.12048052927862574, + "flos": 16659295188480.0, + "grad_norm": 2.3587680352253684, + "language_loss": 0.70571786, + "learning_rate": 3.9147447194292374e-06, + "loss": 0.72730345, + "num_input_tokens_seen": 117643370, + "router_z_loss_clip": 0.67749023, + "router_z_loss_mlp": 0.22888184, + "step": 4152, + "time_per_iteration": 2.4097347259521484 + }, + { + "auxiliary_loss_clip": 0.01103855, + "auxiliary_loss_mlp": 0.01042598, + "balance_loss_clip": 1.03680313, + "balance_loss_mlp": 1.02149212, + "epoch": 0.12050954674714177, + "flos": 27337446541440.0, + "grad_norm": 2.4217297135535962, + "language_loss": 0.59998882, + "learning_rate": 3.914690416972408e-06, + "loss": 0.6214534, + "num_input_tokens_seen": 117659040, + "router_z_loss_clip": 0.67041016, + "router_z_loss_mlp": 0.21142578, + "step": 4153, + "time_per_iteration": 2.360643148422241 + }, + { + "auxiliary_loss_clip": 0.01020686, + "auxiliary_loss_mlp": 0.01024814, + "balance_loss_clip": 1.00902462, + "balance_loss_mlp": 1.02346075, + "epoch": 0.12053856421565783, + "flos": 73112906908800.0, + "grad_norm": 0.7029333698035858, + "language_loss": 0.51744831, + "learning_rate": 3.914636097604258e-06, + "loss": 0.53790331, + "num_input_tokens_seen": 117726060, + "router_z_loss_clip": 0.11669922, + "router_z_loss_mlp": 0.0135498, + "step": 4154, + "time_per_iteration": 3.223970890045166 + }, + { + "auxiliary_loss_clip": 0.01021588, + "auxiliary_loss_mlp": 0.01020218, + "balance_loss_clip": 1.01008213, + "balance_loss_mlp": 1.01884735, + "epoch": 0.12056758168417388, + "flos": 59623969950720.0, + "grad_norm": 0.6959423802940048, + "language_loss": 0.50104898, + "learning_rate": 3.9145817613252666e-06, + "loss": 0.52146703, + "num_input_tokens_seen": 117784295, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01373291, + "step": 4155, + "time_per_iteration": 2.886871814727783 + }, + { + "auxiliary_loss_clip": 0.01109442, + "auxiliary_loss_mlp": 0.01042539, + "balance_loss_clip": 1.039271, + "balance_loss_mlp": 1.02062809, + "epoch": 0.12059659915268992, + "flos": 16717112133120.0, + "grad_norm": 2.834430818873492, + "language_loss": 0.79417026, + "learning_rate": 3.914527408135915e-06, + "loss": 0.81569004, + "num_input_tokens_seen": 117796050, + "router_z_loss_clip": 0.70141602, + "router_z_loss_mlp": 0.21923828, + "step": 4156, + "time_per_iteration": 2.4353692531585693 + }, + { + "auxiliary_loss_clip": 0.01113695, + "auxiliary_loss_mlp": 0.01044535, + "balance_loss_clip": 1.04232001, + "balance_loss_mlp": 1.02311301, + "epoch": 0.12062561662120597, + "flos": 18106620266880.0, + "grad_norm": 2.6083227151898454, + "language_loss": 0.97676688, + "learning_rate": 3.914473038036682e-06, + "loss": 0.99834919, + "num_input_tokens_seen": 117808785, + "router_z_loss_clip": 0.71337891, + "router_z_loss_mlp": 0.21447754, + "step": 4157, + "time_per_iteration": 2.3854353427886963 + }, + { + "auxiliary_loss_clip": 0.01109271, + "auxiliary_loss_mlp": 0.01044729, + "balance_loss_clip": 1.0387857, + "balance_loss_mlp": 1.02037442, + "epoch": 0.12065463408972202, + "flos": 25697855272320.0, + "grad_norm": 2.448187588573903, + "language_loss": 0.76958966, + "learning_rate": 3.914418651028049e-06, + "loss": 0.79112965, + "num_input_tokens_seen": 117824955, + "router_z_loss_clip": 0.70507812, + "router_z_loss_mlp": 0.24353027, + "step": 4158, + "time_per_iteration": 2.4557063579559326 + }, + { + "auxiliary_loss_clip": 0.01017592, + "auxiliary_loss_mlp": 0.01027282, + "balance_loss_clip": 1.0062598, + "balance_loss_mlp": 1.02586973, + "epoch": 0.12068365155823806, + "flos": 68752325784960.0, + "grad_norm": 0.6674032471577873, + "language_loss": 0.48175865, + "learning_rate": 3.914364247110495e-06, + "loss": 0.5022074, + "num_input_tokens_seen": 117882130, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01409912, + "step": 4159, + "time_per_iteration": 2.9672300815582275 + }, + { + "auxiliary_loss_clip": 0.01018273, + "auxiliary_loss_mlp": 0.01014595, + "balance_loss_clip": 1.00644219, + "balance_loss_mlp": 1.01326585, + "epoch": 0.12071266902675411, + "flos": 64429693309440.0, + "grad_norm": 0.6818975420869167, + "language_loss": 0.49934539, + "learning_rate": 3.914309826284502e-06, + "loss": 0.51967406, + "num_input_tokens_seen": 117943075, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.01330566, + "step": 4160, + "time_per_iteration": 3.133185863494873 + }, + { + "auxiliary_loss_clip": 0.01020682, + "auxiliary_loss_mlp": 0.01005829, + "balance_loss_clip": 1.00908816, + "balance_loss_mlp": 1.00439298, + "epoch": 0.12074168649527016, + "flos": 63348992951040.0, + "grad_norm": 0.8138154838464169, + "language_loss": 0.45389134, + "learning_rate": 3.91425538855055e-06, + "loss": 0.47415644, + "num_input_tokens_seen": 117998880, + "router_z_loss_clip": 0.11621094, + "router_z_loss_mlp": 0.01434326, + "step": 4161, + "time_per_iteration": 2.9706785678863525 + }, + { + "auxiliary_loss_clip": 0.01106452, + "auxiliary_loss_mlp": 0.01049126, + "balance_loss_clip": 1.03807461, + "balance_loss_mlp": 1.02843153, + "epoch": 0.1207707039637862, + "flos": 56056336692480.0, + "grad_norm": 2.0973749058412072, + "language_loss": 0.7619797, + "learning_rate": 3.91420093390912e-06, + "loss": 0.78353548, + "num_input_tokens_seen": 118022025, + "router_z_loss_clip": 0.68359375, + "router_z_loss_mlp": 0.20672607, + "step": 4162, + "time_per_iteration": 2.7253918647766113 + }, + { + "auxiliary_loss_clip": 0.01118624, + "auxiliary_loss_mlp": 0.0106661, + "balance_loss_clip": 1.04067945, + "balance_loss_mlp": 1.0407418, + "epoch": 0.12079972143230225, + "flos": 19492672176000.0, + "grad_norm": 2.419666529074791, + "language_loss": 0.90373194, + "learning_rate": 3.914146462360693e-06, + "loss": 0.92558426, + "num_input_tokens_seen": 118037895, + "router_z_loss_clip": 0.78076172, + "router_z_loss_mlp": 0.25878906, + "step": 4163, + "time_per_iteration": 2.4876160621643066 + }, + { + "auxiliary_loss_clip": 0.01110605, + "auxiliary_loss_mlp": 0.01060786, + "balance_loss_clip": 1.04173338, + "balance_loss_mlp": 1.04061532, + "epoch": 0.12082873890081829, + "flos": 74735114186880.0, + "grad_norm": 1.9290890140063774, + "language_loss": 0.89090234, + "learning_rate": 3.914091973905748e-06, + "loss": 0.91261625, + "num_input_tokens_seen": 118070220, + "router_z_loss_clip": 0.68847656, + "router_z_loss_mlp": 0.2019043, + "step": 4164, + "time_per_iteration": 2.8067076206207275 + }, + { + "auxiliary_loss_clip": 0.01108348, + "auxiliary_loss_mlp": 0.01070585, + "balance_loss_clip": 1.03852677, + "balance_loss_mlp": 1.04696953, + "epoch": 0.12085775636933434, + "flos": 20879212844160.0, + "grad_norm": 3.0522757890999985, + "language_loss": 0.87056446, + "learning_rate": 3.91403746854477e-06, + "loss": 0.89235377, + "num_input_tokens_seen": 118085060, + "router_z_loss_clip": 0.69775391, + "router_z_loss_mlp": 0.23620605, + "step": 4165, + "time_per_iteration": 2.4841089248657227 + }, + { + "auxiliary_loss_clip": 0.01102453, + "auxiliary_loss_mlp": 0.0105619, + "balance_loss_clip": 1.03647852, + "balance_loss_mlp": 1.03638971, + "epoch": 0.12088677383785039, + "flos": 25405457702400.0, + "grad_norm": 2.171860007850356, + "language_loss": 0.80613905, + "learning_rate": 3.9139829462782375e-06, + "loss": 0.82772547, + "num_input_tokens_seen": 118102780, + "router_z_loss_clip": 0.65966797, + "router_z_loss_mlp": 0.19805908, + "step": 4166, + "time_per_iteration": 2.4216012954711914 + }, + { + "auxiliary_loss_clip": 0.01020864, + "auxiliary_loss_mlp": 0.01015614, + "balance_loss_clip": 1.0095489, + "balance_loss_mlp": 1.01441634, + "epoch": 0.12091579130636643, + "flos": 62289797990400.0, + "grad_norm": 0.7282833805909591, + "language_loss": 0.54615378, + "learning_rate": 3.913928407106634e-06, + "loss": 0.56651866, + "num_input_tokens_seen": 118164150, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01196289, + "step": 4167, + "time_per_iteration": 3.140509605407715 + }, + { + "auxiliary_loss_clip": 0.01101053, + "auxiliary_loss_mlp": 0.01071314, + "balance_loss_clip": 1.03456712, + "balance_loss_mlp": 1.05082798, + "epoch": 0.12094480877488248, + "flos": 36896316750720.0, + "grad_norm": 2.6465228964865983, + "language_loss": 0.95850945, + "learning_rate": 3.913873851030441e-06, + "loss": 0.98023313, + "num_input_tokens_seen": 118183600, + "router_z_loss_clip": 0.66430664, + "router_z_loss_mlp": 0.20489502, + "step": 4168, + "time_per_iteration": 2.5155251026153564 + }, + { + "auxiliary_loss_clip": 0.01018137, + "auxiliary_loss_mlp": 0.01028534, + "balance_loss_clip": 1.00704026, + "balance_loss_mlp": 1.02740765, + "epoch": 0.12097382624339853, + "flos": 74772993012480.0, + "grad_norm": 0.6888301018282855, + "language_loss": 0.50911689, + "learning_rate": 3.913819278050138e-06, + "loss": 0.52958357, + "num_input_tokens_seen": 118248060, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.0112915, + "step": 4169, + "time_per_iteration": 3.0650506019592285 + }, + { + "auxiliary_loss_clip": 0.01105423, + "auxiliary_loss_mlp": 0.01057965, + "balance_loss_clip": 1.03422642, + "balance_loss_mlp": 1.03544629, + "epoch": 0.12100284371191457, + "flos": 17158344295680.0, + "grad_norm": 2.2422416697435295, + "language_loss": 0.85385495, + "learning_rate": 3.9137646881662085e-06, + "loss": 0.87548876, + "num_input_tokens_seen": 118260825, + "router_z_loss_clip": 0.71240234, + "router_z_loss_mlp": 0.2253418, + "step": 4170, + "time_per_iteration": 2.358092784881592 + }, + { + "auxiliary_loss_clip": 0.01101773, + "auxiliary_loss_mlp": 0.01052826, + "balance_loss_clip": 1.03198171, + "balance_loss_mlp": 1.03190434, + "epoch": 0.12103186118043062, + "flos": 20295534867840.0, + "grad_norm": 3.625781889090758, + "language_loss": 0.79196733, + "learning_rate": 3.913710081379136e-06, + "loss": 0.81351328, + "num_input_tokens_seen": 118274650, + "router_z_loss_clip": 0.69775391, + "router_z_loss_mlp": 0.20910645, + "step": 4171, + "time_per_iteration": 2.412139654159546 + }, + { + "auxiliary_loss_clip": 0.01100569, + "auxiliary_loss_mlp": 0.01044768, + "balance_loss_clip": 1.0370748, + "balance_loss_mlp": 1.02519393, + "epoch": 0.12106087864894667, + "flos": 22739140903680.0, + "grad_norm": 2.1065796673045276, + "language_loss": 0.81165385, + "learning_rate": 3.913655457689401e-06, + "loss": 0.83310723, + "num_input_tokens_seen": 118294680, + "router_z_loss_clip": 0.63427734, + "router_z_loss_mlp": 0.19580078, + "step": 4172, + "time_per_iteration": 2.56070613861084 + }, + { + "auxiliary_loss_clip": 0.01107333, + "auxiliary_loss_mlp": 0.01050407, + "balance_loss_clip": 1.03812933, + "balance_loss_mlp": 1.02790082, + "epoch": 0.12108989611746271, + "flos": 25220034138240.0, + "grad_norm": 3.009836446801361, + "language_loss": 0.9092654, + "learning_rate": 3.913600817097487e-06, + "loss": 0.93084282, + "num_input_tokens_seen": 118310490, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.22509766, + "step": 4173, + "time_per_iteration": 2.448596715927124 + }, + { + "auxiliary_loss_clip": 0.01030314, + "auxiliary_loss_mlp": 0.01001361, + "balance_loss_clip": 1.01873136, + "balance_loss_mlp": 1.00026393, + "epoch": 0.12111891358597876, + "flos": 74764265172480.0, + "grad_norm": 0.6726430429765871, + "language_loss": 0.44258586, + "learning_rate": 3.913546159603877e-06, + "loss": 0.46290261, + "num_input_tokens_seen": 118370190, + "router_z_loss_clip": 0.11572266, + "router_z_loss_mlp": 0.01098633, + "step": 4174, + "time_per_iteration": 3.1178040504455566 + }, + { + "auxiliary_loss_clip": 0.01107109, + "auxiliary_loss_mlp": 0.01049403, + "balance_loss_clip": 1.03916347, + "balance_loss_mlp": 1.02783799, + "epoch": 0.12114793105449481, + "flos": 19456537052160.0, + "grad_norm": 2.9252326430544824, + "language_loss": 0.82391119, + "learning_rate": 3.913491485209052e-06, + "loss": 0.84547627, + "num_input_tokens_seen": 118384550, + "router_z_loss_clip": 0.67871094, + "router_z_loss_mlp": 0.21557617, + "step": 4175, + "time_per_iteration": 2.3928918838500977 + }, + { + "auxiliary_loss_clip": 0.01034392, + "auxiliary_loss_mlp": 0.01000991, + "balance_loss_clip": 1.02240682, + "balance_loss_mlp": 1.00001371, + "epoch": 0.12117694852301085, + "flos": 60409759121280.0, + "grad_norm": 0.6980013550955884, + "language_loss": 0.48453066, + "learning_rate": 3.913436793913496e-06, + "loss": 0.50488448, + "num_input_tokens_seen": 118448535, + "router_z_loss_clip": 0.11962891, + "router_z_loss_mlp": 0.00976562, + "step": 4176, + "time_per_iteration": 3.0879759788513184 + }, + { + "auxiliary_loss_clip": 0.0103495, + "auxiliary_loss_mlp": 0.01000401, + "balance_loss_clip": 1.02287078, + "balance_loss_mlp": 0.99933434, + "epoch": 0.1212059659915269, + "flos": 52397507496960.0, + "grad_norm": 0.801320585884712, + "language_loss": 0.52290773, + "learning_rate": 3.913382085717692e-06, + "loss": 0.54326129, + "num_input_tokens_seen": 118498555, + "router_z_loss_clip": 0.12060547, + "router_z_loss_mlp": 0.01068115, + "step": 4177, + "time_per_iteration": 2.7409117221832275 + }, + { + "auxiliary_loss_clip": 0.01109281, + "auxiliary_loss_mlp": 0.01044977, + "balance_loss_clip": 1.0408361, + "balance_loss_mlp": 1.0240438, + "epoch": 0.12123498346004295, + "flos": 25914596192640.0, + "grad_norm": 1.888651932695792, + "language_loss": 0.77039659, + "learning_rate": 3.913327360622123e-06, + "loss": 0.7919392, + "num_input_tokens_seen": 118516975, + "router_z_loss_clip": 0.68359375, + "router_z_loss_mlp": 0.20922852, + "step": 4178, + "time_per_iteration": 2.506105899810791 + }, + { + "auxiliary_loss_clip": 0.01104048, + "auxiliary_loss_mlp": 0.0104063, + "balance_loss_clip": 1.03996217, + "balance_loss_mlp": 1.0203402, + "epoch": 0.12126400092855899, + "flos": 22119467448960.0, + "grad_norm": 2.4975606481437986, + "language_loss": 0.95029503, + "learning_rate": 3.913272618627273e-06, + "loss": 0.9717418, + "num_input_tokens_seen": 118531775, + "router_z_loss_clip": 0.640625, + "router_z_loss_mlp": 0.20288086, + "step": 4179, + "time_per_iteration": 2.406487226486206 + }, + { + "auxiliary_loss_clip": 0.01097897, + "auxiliary_loss_mlp": 0.01041131, + "balance_loss_clip": 1.03984904, + "balance_loss_mlp": 1.02458501, + "epoch": 0.12129301839707504, + "flos": 17486248584960.0, + "grad_norm": 2.5152396912102475, + "language_loss": 0.58796823, + "learning_rate": 3.913217859733624e-06, + "loss": 0.60935849, + "num_input_tokens_seen": 118545025, + "router_z_loss_clip": 0.58056641, + "router_z_loss_mlp": 0.16552734, + "step": 4180, + "time_per_iteration": 2.4354448318481445 + }, + { + "auxiliary_loss_clip": 0.01111822, + "auxiliary_loss_mlp": 0.01043616, + "balance_loss_clip": 1.04180181, + "balance_loss_mlp": 1.02201545, + "epoch": 0.12132203586559108, + "flos": 21099060875520.0, + "grad_norm": 2.158053169863901, + "language_loss": 0.78548479, + "learning_rate": 3.913163083941661e-06, + "loss": 0.80703914, + "num_input_tokens_seen": 118559500, + "router_z_loss_clip": 0.69970703, + "router_z_loss_mlp": 0.21606445, + "step": 4181, + "time_per_iteration": 2.356825351715088 + }, + { + "auxiliary_loss_clip": 0.01111863, + "auxiliary_loss_mlp": 0.01055743, + "balance_loss_clip": 1.04259372, + "balance_loss_mlp": 1.0331769, + "epoch": 0.12135105333410713, + "flos": 16758414138240.0, + "grad_norm": 3.0917102815221167, + "language_loss": 0.82564551, + "learning_rate": 3.913108291251868e-06, + "loss": 0.84732157, + "num_input_tokens_seen": 118574475, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.22558594, + "step": 4182, + "time_per_iteration": 2.3857781887054443 + }, + { + "auxiliary_loss_clip": 0.0110402, + "auxiliary_loss_mlp": 0.01045254, + "balance_loss_clip": 1.04135537, + "balance_loss_mlp": 1.02538216, + "epoch": 0.12138007080262318, + "flos": 43827868016640.0, + "grad_norm": 2.31299768121081, + "language_loss": 0.7334913, + "learning_rate": 3.9130534816647286e-06, + "loss": 0.75498402, + "num_input_tokens_seen": 118591820, + "router_z_loss_clip": 0.62695312, + "router_z_loss_mlp": 0.19848633, + "step": 4183, + "time_per_iteration": 2.7089641094207764 + }, + { + "auxiliary_loss_clip": 0.01105691, + "auxiliary_loss_mlp": 0.01044592, + "balance_loss_clip": 1.04000688, + "balance_loss_mlp": 1.02544117, + "epoch": 0.12140908827113922, + "flos": 22013401138560.0, + "grad_norm": 1.935859763210635, + "language_loss": 0.76781541, + "learning_rate": 3.912998655180727e-06, + "loss": 0.7893182, + "num_input_tokens_seen": 118609555, + "router_z_loss_clip": 0.65722656, + "router_z_loss_mlp": 0.19146729, + "step": 4184, + "time_per_iteration": 2.4832823276519775 + }, + { + "auxiliary_loss_clip": 0.01110119, + "auxiliary_loss_mlp": 0.01048949, + "balance_loss_clip": 1.03991413, + "balance_loss_mlp": 1.02494025, + "epoch": 0.12143810573965527, + "flos": 32374470723840.0, + "grad_norm": 2.2495007962465383, + "language_loss": 0.85757577, + "learning_rate": 3.912943811800346e-06, + "loss": 0.87916648, + "num_input_tokens_seen": 118627890, + "router_z_loss_clip": 0.70117188, + "router_z_loss_mlp": 0.2401123, + "step": 4185, + "time_per_iteration": 2.5258901119232178 + }, + { + "auxiliary_loss_clip": 0.01036308, + "auxiliary_loss_mlp": 0.01000642, + "balance_loss_clip": 1.02387762, + "balance_loss_mlp": 0.99943256, + "epoch": 0.12146712320817132, + "flos": 59408276371200.0, + "grad_norm": 0.6791978320368424, + "language_loss": 0.49905276, + "learning_rate": 3.912888951524072e-06, + "loss": 0.51942229, + "num_input_tokens_seen": 118689050, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.01208496, + "step": 4186, + "time_per_iteration": 5.230626583099365 + }, + { + "auxiliary_loss_clip": 0.01104025, + "auxiliary_loss_mlp": 0.01046381, + "balance_loss_clip": 1.03620601, + "balance_loss_mlp": 1.02535224, + "epoch": 0.12149614067668736, + "flos": 22850583563520.0, + "grad_norm": 2.731888378835021, + "language_loss": 1.05220795, + "learning_rate": 3.912834074352388e-06, + "loss": 1.07371187, + "num_input_tokens_seen": 118705460, + "router_z_loss_clip": 0.67895508, + "router_z_loss_mlp": 0.21020508, + "step": 4187, + "time_per_iteration": 2.5060629844665527 + }, + { + "auxiliary_loss_clip": 0.01092522, + "auxiliary_loss_mlp": 0.01043225, + "balance_loss_clip": 1.03253317, + "balance_loss_mlp": 1.02462292, + "epoch": 0.12152515814520341, + "flos": 17230265429760.0, + "grad_norm": 2.6977191380044996, + "language_loss": 0.97151387, + "learning_rate": 3.912779180285779e-06, + "loss": 0.9928714, + "num_input_tokens_seen": 118717210, + "router_z_loss_clip": 0.59960938, + "router_z_loss_mlp": 0.1862793, + "step": 4188, + "time_per_iteration": 4.494508266448975 + }, + { + "auxiliary_loss_clip": 0.01029593, + "auxiliary_loss_mlp": 0.0099922, + "balance_loss_clip": 1.01803303, + "balance_loss_mlp": 0.99804574, + "epoch": 0.12155417561371946, + "flos": 61041199570560.0, + "grad_norm": 0.6538875915864963, + "language_loss": 0.47307885, + "learning_rate": 3.912724269324732e-06, + "loss": 0.49336699, + "num_input_tokens_seen": 118777045, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01171875, + "step": 4189, + "time_per_iteration": 2.9692816734313965 + }, + { + "auxiliary_loss_clip": 0.01112961, + "auxiliary_loss_mlp": 0.01047863, + "balance_loss_clip": 1.03882527, + "balance_loss_mlp": 1.02297258, + "epoch": 0.1215831930822355, + "flos": 74729109432960.0, + "grad_norm": 2.6526114127438802, + "language_loss": 0.80169374, + "learning_rate": 3.912669341469729e-06, + "loss": 0.82330203, + "num_input_tokens_seen": 118802345, + "router_z_loss_clip": 0.74072266, + "router_z_loss_mlp": 0.24890137, + "step": 4190, + "time_per_iteration": 2.8275673389434814 + }, + { + "auxiliary_loss_clip": 0.01022274, + "auxiliary_loss_mlp": 0.00999161, + "balance_loss_clip": 1.01151299, + "balance_loss_mlp": 0.99790895, + "epoch": 0.12161221055075155, + "flos": 74769920812800.0, + "grad_norm": 0.6655708680232238, + "language_loss": 0.49334735, + "learning_rate": 3.912614396721257e-06, + "loss": 0.51356167, + "num_input_tokens_seen": 118862585, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01251221, + "step": 4191, + "time_per_iteration": 3.014409065246582 + }, + { + "auxiliary_loss_clip": 0.01103291, + "auxiliary_loss_mlp": 0.01048364, + "balance_loss_clip": 1.03485799, + "balance_loss_mlp": 1.02641773, + "epoch": 0.1216412280192676, + "flos": 12741936174720.0, + "grad_norm": 2.79178287431409, + "language_loss": 0.93162918, + "learning_rate": 3.912559435079801e-06, + "loss": 0.95314574, + "num_input_tokens_seen": 118873805, + "router_z_loss_clip": 0.68457031, + "router_z_loss_mlp": 0.21960449, + "step": 4192, + "time_per_iteration": 4.7412333488464355 + }, + { + "auxiliary_loss_clip": 0.01109168, + "auxiliary_loss_mlp": 0.01044879, + "balance_loss_clip": 1.03556943, + "balance_loss_mlp": 1.02252746, + "epoch": 0.12167024548778364, + "flos": 34742769223680.0, + "grad_norm": 2.6742846502945268, + "language_loss": 0.93331647, + "learning_rate": 3.9125044565458444e-06, + "loss": 0.95485699, + "num_input_tokens_seen": 118889295, + "router_z_loss_clip": 0.73632812, + "router_z_loss_mlp": 0.22351074, + "step": 4193, + "time_per_iteration": 2.495375871658325 + }, + { + "auxiliary_loss_clip": 0.01020764, + "auxiliary_loss_mlp": 0.0101267, + "balance_loss_clip": 1.01016724, + "balance_loss_mlp": 1.01146603, + "epoch": 0.1216992629562997, + "flos": 74228941296000.0, + "grad_norm": 0.6232078880019752, + "language_loss": 0.51854122, + "learning_rate": 3.912449461119876e-06, + "loss": 0.53887558, + "num_input_tokens_seen": 118959685, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.01202393, + "step": 4194, + "time_per_iteration": 3.2704455852508545 + }, + { + "auxiliary_loss_clip": 0.01113653, + "auxiliary_loss_mlp": 0.01059836, + "balance_loss_clip": 1.03920794, + "balance_loss_mlp": 1.03670907, + "epoch": 0.12172828042481573, + "flos": 43897869025920.0, + "grad_norm": 2.2894450975765426, + "language_loss": 0.77417552, + "learning_rate": 3.91239444880238e-06, + "loss": 0.79591036, + "num_input_tokens_seen": 118975815, + "router_z_loss_clip": 0.74462891, + "router_z_loss_mlp": 0.2310791, + "step": 4195, + "time_per_iteration": 2.5987231731414795 + }, + { + "auxiliary_loss_clip": 0.01019102, + "auxiliary_loss_mlp": 0.0101662, + "balance_loss_clip": 1.00867295, + "balance_loss_mlp": 1.01525509, + "epoch": 0.12175729789333178, + "flos": 68868062542080.0, + "grad_norm": 0.6622258443039072, + "language_loss": 0.49999219, + "learning_rate": 3.912339419593843e-06, + "loss": 0.52034944, + "num_input_tokens_seen": 119038055, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01367188, + "step": 4196, + "time_per_iteration": 5.409262657165527 + }, + { + "auxiliary_loss_clip": 0.01017597, + "auxiliary_loss_mlp": 0.01014124, + "balance_loss_clip": 1.00697541, + "balance_loss_mlp": 1.01282454, + "epoch": 0.12178631536184784, + "flos": 66238893296640.0, + "grad_norm": 0.6962117793237513, + "language_loss": 0.52782607, + "learning_rate": 3.912284373494748e-06, + "loss": 0.54814327, + "num_input_tokens_seen": 119102215, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.01300049, + "step": 4197, + "time_per_iteration": 3.007418394088745 + }, + { + "auxiliary_loss_clip": 0.01099953, + "auxiliary_loss_mlp": 0.01041447, + "balance_loss_clip": 1.03589368, + "balance_loss_mlp": 1.02237976, + "epoch": 0.12181533283036387, + "flos": 17266714755840.0, + "grad_norm": 2.0582210130232, + "language_loss": 0.63588274, + "learning_rate": 3.912229310505586e-06, + "loss": 0.65729672, + "num_input_tokens_seen": 119115375, + "router_z_loss_clip": 0.64111328, + "router_z_loss_mlp": 0.1907959, + "step": 4198, + "time_per_iteration": 2.357793092727661 + }, + { + "auxiliary_loss_clip": 0.01105059, + "auxiliary_loss_mlp": 0.01051456, + "balance_loss_clip": 1.0349468, + "balance_loss_mlp": 1.02971864, + "epoch": 0.12184435029887992, + "flos": 39631447572480.0, + "grad_norm": 2.319086806973323, + "language_loss": 0.9020949, + "learning_rate": 3.91217423062684e-06, + "loss": 0.92366004, + "num_input_tokens_seen": 119133080, + "router_z_loss_clip": 0.70068359, + "router_z_loss_mlp": 0.21728516, + "step": 4199, + "time_per_iteration": 2.601803779602051 + }, + { + "auxiliary_loss_clip": 0.01117353, + "auxiliary_loss_mlp": 0.01046848, + "balance_loss_clip": 1.04433703, + "balance_loss_mlp": 1.02478194, + "epoch": 0.12187336776739598, + "flos": 15406577228160.0, + "grad_norm": 2.4413624908842477, + "language_loss": 0.73758841, + "learning_rate": 3.912119133858997e-06, + "loss": 0.75923043, + "num_input_tokens_seen": 119145950, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.22070312, + "step": 4200, + "time_per_iteration": 2.4056689739227295 + }, + { + "auxiliary_loss_clip": 0.01092264, + "auxiliary_loss_mlp": 0.01036279, + "balance_loss_clip": 1.0348649, + "balance_loss_mlp": 1.01961327, + "epoch": 0.12190238523591201, + "flos": 23105170264320.0, + "grad_norm": 2.2540904285805774, + "language_loss": 0.97369719, + "learning_rate": 3.912064020202545e-06, + "loss": 0.99498254, + "num_input_tokens_seen": 119162405, + "router_z_loss_clip": 0.57397461, + "router_z_loss_mlp": 0.16680908, + "step": 4201, + "time_per_iteration": 2.4722626209259033 + }, + { + "auxiliary_loss_clip": 0.01110393, + "auxiliary_loss_mlp": 0.01042651, + "balance_loss_clip": 1.04160452, + "balance_loss_mlp": 1.02024007, + "epoch": 0.12193140270442807, + "flos": 28686909479040.0, + "grad_norm": 3.3725436276480822, + "language_loss": 0.82363808, + "learning_rate": 3.91200888965797e-06, + "loss": 0.84516847, + "num_input_tokens_seen": 119182360, + "router_z_loss_clip": 0.68798828, + "router_z_loss_mlp": 0.22399902, + "step": 4202, + "time_per_iteration": 2.5554966926574707 + }, + { + "auxiliary_loss_clip": 0.01114137, + "auxiliary_loss_mlp": 0.01042812, + "balance_loss_clip": 1.04121852, + "balance_loss_mlp": 1.02075779, + "epoch": 0.12196042017294412, + "flos": 16792419669120.0, + "grad_norm": 3.314462819426613, + "language_loss": 0.77260739, + "learning_rate": 3.911953742225757e-06, + "loss": 0.79417688, + "num_input_tokens_seen": 119196845, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.22058105, + "step": 4203, + "time_per_iteration": 2.4091808795928955 + }, + { + "auxiliary_loss_clip": 0.01110269, + "auxiliary_loss_mlp": 0.01052178, + "balance_loss_clip": 1.04235196, + "balance_loss_mlp": 1.02993393, + "epoch": 0.12198943764146016, + "flos": 36494536291200.0, + "grad_norm": 2.3013458465588243, + "language_loss": 0.88521647, + "learning_rate": 3.911898577906396e-06, + "loss": 0.90684104, + "num_input_tokens_seen": 119213065, + "router_z_loss_clip": 0.67822266, + "router_z_loss_mlp": 0.22241211, + "step": 4204, + "time_per_iteration": 2.5193212032318115 + }, + { + "auxiliary_loss_clip": 0.01034927, + "auxiliary_loss_mlp": 0.01003645, + "balance_loss_clip": 1.02104664, + "balance_loss_mlp": 1.0023632, + "epoch": 0.1220184551099762, + "flos": 74627614644480.0, + "grad_norm": 0.5956060521015291, + "language_loss": 0.46375942, + "learning_rate": 3.911843396700373e-06, + "loss": 0.48414513, + "num_input_tokens_seen": 119283680, + "router_z_loss_clip": 0.13867188, + "router_z_loss_mlp": 0.01281738, + "step": 4205, + "time_per_iteration": 3.222639322280884 + }, + { + "auxiliary_loss_clip": 0.01119021, + "auxiliary_loss_mlp": 0.01044931, + "balance_loss_clip": 1.04412293, + "balance_loss_mlp": 1.02234077, + "epoch": 0.12204747257849226, + "flos": 22198405766400.0, + "grad_norm": 1.9929851776851368, + "language_loss": 0.74414837, + "learning_rate": 3.911788198608176e-06, + "loss": 0.76578784, + "num_input_tokens_seen": 119297805, + "router_z_loss_clip": 0.75048828, + "router_z_loss_mlp": 0.22595215, + "step": 4206, + "time_per_iteration": 2.3668301105499268 + }, + { + "auxiliary_loss_clip": 0.01034303, + "auxiliary_loss_mlp": 0.01004758, + "balance_loss_clip": 1.02115941, + "balance_loss_mlp": 1.00354218, + "epoch": 0.1220764900470083, + "flos": 65616007996800.0, + "grad_norm": 0.6259956798178822, + "language_loss": 0.49867791, + "learning_rate": 3.911732983630292e-06, + "loss": 0.51906848, + "num_input_tokens_seen": 119364465, + "router_z_loss_clip": 0.13183594, + "router_z_loss_mlp": 0.012146, + "step": 4207, + "time_per_iteration": 3.142115354537964 + }, + { + "auxiliary_loss_clip": 0.01031264, + "auxiliary_loss_mlp": 0.01001567, + "balance_loss_clip": 1.01822007, + "balance_loss_mlp": 1.00030339, + "epoch": 0.12210550751552435, + "flos": 60983661916800.0, + "grad_norm": 0.6799262567735603, + "language_loss": 0.49085414, + "learning_rate": 3.911677751767208e-06, + "loss": 0.51118249, + "num_input_tokens_seen": 119429640, + "router_z_loss_clip": 0.13085938, + "router_z_loss_mlp": 0.01263428, + "step": 4208, + "time_per_iteration": 3.1122875213623047 + }, + { + "auxiliary_loss_clip": 0.01026321, + "auxiliary_loss_mlp": 0.01001374, + "balance_loss_clip": 1.01400113, + "balance_loss_mlp": 1.00018167, + "epoch": 0.1221345249840404, + "flos": 51055620324480.0, + "grad_norm": 0.7236536206147516, + "language_loss": 0.52455008, + "learning_rate": 3.911622503019413e-06, + "loss": 0.54482704, + "num_input_tokens_seen": 119482875, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01190186, + "step": 4209, + "time_per_iteration": 2.8680670261383057 + }, + { + "auxiliary_loss_clip": 0.01022355, + "auxiliary_loss_mlp": 0.01007644, + "balance_loss_clip": 1.01055741, + "balance_loss_mlp": 1.00652909, + "epoch": 0.12216354245255644, + "flos": 74768489447040.0, + "grad_norm": 0.6899197879927262, + "language_loss": 0.52075958, + "learning_rate": 3.911567237387394e-06, + "loss": 0.54105955, + "num_input_tokens_seen": 119547265, + "router_z_loss_clip": 0.11816406, + "router_z_loss_mlp": 0.01116943, + "step": 4210, + "time_per_iteration": 3.0046818256378174 + }, + { + "auxiliary_loss_clip": 0.01104733, + "auxiliary_loss_mlp": 0.01054934, + "balance_loss_clip": 1.03517914, + "balance_loss_mlp": 1.03190875, + "epoch": 0.12219255992107249, + "flos": 33031815402240.0, + "grad_norm": 1.8675020931837063, + "language_loss": 0.77069896, + "learning_rate": 3.91151195487164e-06, + "loss": 0.79229563, + "num_input_tokens_seen": 119567220, + "router_z_loss_clip": 0.69555664, + "router_z_loss_mlp": 0.23034668, + "step": 4211, + "time_per_iteration": 2.4762916564941406 + }, + { + "auxiliary_loss_clip": 0.01109282, + "auxiliary_loss_mlp": 0.01063406, + "balance_loss_clip": 1.03388238, + "balance_loss_mlp": 1.03722787, + "epoch": 0.12222157738958853, + "flos": 28284675171840.0, + "grad_norm": 2.9737439100509753, + "language_loss": 0.85527086, + "learning_rate": 3.911456655472639e-06, + "loss": 0.87699783, + "num_input_tokens_seen": 119580765, + "router_z_loss_clip": 0.75341797, + "router_z_loss_mlp": 0.26196289, + "step": 4212, + "time_per_iteration": 2.4659879207611084 + }, + { + "auxiliary_loss_clip": 0.01103288, + "auxiliary_loss_mlp": 0.01062527, + "balance_loss_clip": 1.03486323, + "balance_loss_mlp": 1.03994942, + "epoch": 0.12225059485810458, + "flos": 34450895324160.0, + "grad_norm": 4.111747731456289, + "language_loss": 0.88564014, + "learning_rate": 3.911401339190879e-06, + "loss": 0.90729827, + "num_input_tokens_seen": 119594915, + "router_z_loss_clip": 0.68457031, + "router_z_loss_mlp": 0.22595215, + "step": 4213, + "time_per_iteration": 2.502371311187744 + }, + { + "auxiliary_loss_clip": 0.01019173, + "auxiliary_loss_mlp": 0.01023146, + "balance_loss_clip": 1.00831985, + "balance_loss_mlp": 1.02208495, + "epoch": 0.12227961232662063, + "flos": 74778962855040.0, + "grad_norm": 0.6874127023162226, + "language_loss": 0.48244995, + "learning_rate": 3.911346006026849e-06, + "loss": 0.50287312, + "num_input_tokens_seen": 119663540, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01062012, + "step": 4214, + "time_per_iteration": 3.2063913345336914 + }, + { + "auxiliary_loss_clip": 0.0102124, + "auxiliary_loss_mlp": 0.01012527, + "balance_loss_clip": 1.01011598, + "balance_loss_mlp": 1.01144838, + "epoch": 0.12230862979513667, + "flos": 56179402968960.0, + "grad_norm": 0.7130645779669716, + "language_loss": 0.46029478, + "learning_rate": 3.911290655981038e-06, + "loss": 0.48063248, + "num_input_tokens_seen": 119722285, + "router_z_loss_clip": 0.11132812, + "router_z_loss_mlp": 0.01080322, + "step": 4215, + "time_per_iteration": 3.0257294178009033 + }, + { + "auxiliary_loss_clip": 0.01108285, + "auxiliary_loss_mlp": 0.01069256, + "balance_loss_clip": 1.03647065, + "balance_loss_mlp": 1.04361451, + "epoch": 0.12233764726365272, + "flos": 30514403018880.0, + "grad_norm": 1.8885895795098138, + "language_loss": 0.96899343, + "learning_rate": 3.911235289053934e-06, + "loss": 0.99076879, + "num_input_tokens_seen": 119744700, + "router_z_loss_clip": 0.71826172, + "router_z_loss_mlp": 0.25646973, + "step": 4216, + "time_per_iteration": 2.6534950733184814 + }, + { + "auxiliary_loss_clip": 0.01028517, + "auxiliary_loss_mlp": 0.01003169, + "balance_loss_clip": 1.01706362, + "balance_loss_mlp": 1.00197697, + "epoch": 0.12236666473216877, + "flos": 74756794141440.0, + "grad_norm": 0.6758583074392538, + "language_loss": 0.47739571, + "learning_rate": 3.911179905246027e-06, + "loss": 0.49771255, + "num_input_tokens_seen": 119798410, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01190186, + "step": 4217, + "time_per_iteration": 2.91831636428833 + }, + { + "auxiliary_loss_clip": 0.01122106, + "auxiliary_loss_mlp": 0.01058925, + "balance_loss_clip": 1.04770923, + "balance_loss_mlp": 1.03451073, + "epoch": 0.12239568220068481, + "flos": 22994530565760.0, + "grad_norm": 2.9284786418707216, + "language_loss": 0.72752661, + "learning_rate": 3.911124504557806e-06, + "loss": 0.74933696, + "num_input_tokens_seen": 119813855, + "router_z_loss_clip": 0.74414062, + "router_z_loss_mlp": 0.2442627, + "step": 4218, + "time_per_iteration": 2.3859541416168213 + }, + { + "auxiliary_loss_clip": 0.01107086, + "auxiliary_loss_mlp": 0.01039992, + "balance_loss_clip": 1.04025793, + "balance_loss_mlp": 1.02072799, + "epoch": 0.12242469966920086, + "flos": 31095288086400.0, + "grad_norm": 2.4881795436582315, + "language_loss": 0.85959411, + "learning_rate": 3.9110690869897584e-06, + "loss": 0.88106483, + "num_input_tokens_seen": 119831595, + "router_z_loss_clip": 0.66748047, + "router_z_loss_mlp": 0.19274902, + "step": 4219, + "time_per_iteration": 2.5357651710510254 + }, + { + "auxiliary_loss_clip": 0.01113885, + "auxiliary_loss_mlp": 0.01052801, + "balance_loss_clip": 1.04383278, + "balance_loss_mlp": 1.03140283, + "epoch": 0.12245371713771691, + "flos": 20375520526080.0, + "grad_norm": 2.542034058450755, + "language_loss": 0.6041643, + "learning_rate": 3.911013652542377e-06, + "loss": 0.62583113, + "num_input_tokens_seen": 119845415, + "router_z_loss_clip": 0.70019531, + "router_z_loss_mlp": 0.21398926, + "step": 4220, + "time_per_iteration": 2.4234519004821777 + }, + { + "auxiliary_loss_clip": 0.0104755, + "auxiliary_loss_mlp": 0.00999578, + "balance_loss_clip": 1.03299546, + "balance_loss_mlp": 0.9983089, + "epoch": 0.12248273460623295, + "flos": 74775646275840.0, + "grad_norm": 0.6175453950112322, + "language_loss": 0.49265561, + "learning_rate": 3.910958201216149e-06, + "loss": 0.51312691, + "num_input_tokens_seen": 119911025, + "router_z_loss_clip": 0.14550781, + "router_z_loss_mlp": 0.01269531, + "step": 4221, + "time_per_iteration": 3.1084964275360107 + }, + { + "auxiliary_loss_clip": 0.01118216, + "auxiliary_loss_mlp": 0.01040684, + "balance_loss_clip": 1.04623473, + "balance_loss_mlp": 1.01740289, + "epoch": 0.122511752074749, + "flos": 28363892780160.0, + "grad_norm": 4.103712144486523, + "language_loss": 0.73884761, + "learning_rate": 3.910902733011565e-06, + "loss": 0.76043665, + "num_input_tokens_seen": 119937475, + "router_z_loss_clip": 0.71972656, + "router_z_loss_mlp": 0.23291016, + "step": 4222, + "time_per_iteration": 2.807508707046509 + }, + { + "auxiliary_loss_clip": 0.01117286, + "auxiliary_loss_mlp": 0.0104214, + "balance_loss_clip": 1.04807484, + "balance_loss_mlp": 1.02038467, + "epoch": 0.12254076954326505, + "flos": 15077695420800.0, + "grad_norm": 3.718982120084113, + "language_loss": 0.9083187, + "learning_rate": 3.9108472479291145e-06, + "loss": 0.92991292, + "num_input_tokens_seen": 119950245, + "router_z_loss_clip": 0.69238281, + "router_z_loss_mlp": 0.21740723, + "step": 4223, + "time_per_iteration": 2.4197423458099365 + }, + { + "auxiliary_loss_clip": 0.0111967, + "auxiliary_loss_mlp": 0.01039698, + "balance_loss_clip": 1.04766655, + "balance_loss_mlp": 1.01763201, + "epoch": 0.12256978701178109, + "flos": 33101188007040.0, + "grad_norm": 3.4591663679105435, + "language_loss": 0.81290352, + "learning_rate": 3.9107917459692885e-06, + "loss": 0.83449721, + "num_input_tokens_seen": 119969630, + "router_z_loss_clip": 0.72094727, + "router_z_loss_mlp": 0.22070312, + "step": 4224, + "time_per_iteration": 2.5050137042999268 + }, + { + "auxiliary_loss_clip": 0.01111591, + "auxiliary_loss_mlp": 0.01043086, + "balance_loss_clip": 1.04569852, + "balance_loss_mlp": 1.02393472, + "epoch": 0.12259880448029714, + "flos": 11063451784320.0, + "grad_norm": 2.0401644263091505, + "language_loss": 0.78824198, + "learning_rate": 3.910736227132577e-06, + "loss": 0.8097887, + "num_input_tokens_seen": 119981565, + "router_z_loss_clip": 0.65820312, + "router_z_loss_mlp": 0.19171143, + "step": 4225, + "time_per_iteration": 2.4121150970458984 + }, + { + "auxiliary_loss_clip": 0.01118077, + "auxiliary_loss_mlp": 0.01055389, + "balance_loss_clip": 1.04594886, + "balance_loss_mlp": 1.03225017, + "epoch": 0.12262782194881318, + "flos": 30804810641280.0, + "grad_norm": 3.1212533843413306, + "language_loss": 0.94761795, + "learning_rate": 3.9106806914194685e-06, + "loss": 0.9693526, + "num_input_tokens_seen": 119999950, + "router_z_loss_clip": 0.72167969, + "router_z_loss_mlp": 0.23168945, + "step": 4226, + "time_per_iteration": 2.4917426109313965 + }, + { + "auxiliary_loss_clip": 0.010535, + "auxiliary_loss_mlp": 0.01006259, + "balance_loss_clip": 1.03809023, + "balance_loss_mlp": 1.00482869, + "epoch": 0.12265683941732923, + "flos": 59692330126080.0, + "grad_norm": 0.6811619515749978, + "language_loss": 0.46154809, + "learning_rate": 3.9106251388304555e-06, + "loss": 0.4821457, + "num_input_tokens_seen": 120054615, + "router_z_loss_clip": 0.15429688, + "router_z_loss_mlp": 0.01428223, + "step": 4227, + "time_per_iteration": 2.904445171356201 + }, + { + "auxiliary_loss_clip": 0.01114328, + "auxiliary_loss_mlp": 0.01060683, + "balance_loss_clip": 1.04716551, + "balance_loss_mlp": 1.03631639, + "epoch": 0.12268585688584528, + "flos": 14712469021440.0, + "grad_norm": 2.633340915559712, + "language_loss": 0.8602066, + "learning_rate": 3.910569569366029e-06, + "loss": 0.8819567, + "num_input_tokens_seen": 120066950, + "router_z_loss_clip": 0.671875, + "router_z_loss_mlp": 0.24365234, + "step": 4228, + "time_per_iteration": 2.427985191345215 + }, + { + "auxiliary_loss_clip": 0.01101243, + "auxiliary_loss_mlp": 0.01029968, + "balance_loss_clip": 1.04290199, + "balance_loss_mlp": 1.01370788, + "epoch": 0.12271487435436132, + "flos": 41693382092160.0, + "grad_norm": 2.612091568256145, + "language_loss": 1.03287053, + "learning_rate": 3.910513983026678e-06, + "loss": 1.05418253, + "num_input_tokens_seen": 120086150, + "router_z_loss_clip": 0.58447266, + "router_z_loss_mlp": 0.16271973, + "step": 4229, + "time_per_iteration": 2.7223079204559326 + }, + { + "auxiliary_loss_clip": 0.01107611, + "auxiliary_loss_mlp": 0.01042548, + "balance_loss_clip": 1.03875756, + "balance_loss_mlp": 1.01961231, + "epoch": 0.12274389182287737, + "flos": 15367719018240.0, + "grad_norm": 3.1541538312014556, + "language_loss": 0.71056378, + "learning_rate": 3.910458379812894e-06, + "loss": 0.73206532, + "num_input_tokens_seen": 120099655, + "router_z_loss_clip": 0.68896484, + "router_z_loss_mlp": 0.22924805, + "step": 4230, + "time_per_iteration": 2.356297492980957 + }, + { + "auxiliary_loss_clip": 0.01113596, + "auxiliary_loss_mlp": 0.01042452, + "balance_loss_clip": 1.04188752, + "balance_loss_mlp": 1.02039862, + "epoch": 0.12277290929139342, + "flos": 34451907753600.0, + "grad_norm": 1.7879193863981593, + "language_loss": 0.84938216, + "learning_rate": 3.910402759725169e-06, + "loss": 0.87094259, + "num_input_tokens_seen": 120119735, + "router_z_loss_clip": 0.71630859, + "router_z_loss_mlp": 0.22058105, + "step": 4231, + "time_per_iteration": 2.5637266635894775 + }, + { + "auxiliary_loss_clip": 0.01101038, + "auxiliary_loss_mlp": 0.01041823, + "balance_loss_clip": 1.03947318, + "balance_loss_mlp": 1.02367878, + "epoch": 0.12280192675990946, + "flos": 15953212385280.0, + "grad_norm": 2.0590819918801326, + "language_loss": 0.65932572, + "learning_rate": 3.910347122763994e-06, + "loss": 0.6807543, + "num_input_tokens_seen": 120133105, + "router_z_loss_clip": 0.61547852, + "router_z_loss_mlp": 0.18145752, + "step": 4232, + "time_per_iteration": 2.3694021701812744 + }, + { + "auxiliary_loss_clip": 0.01099982, + "auxiliary_loss_mlp": 0.01041118, + "balance_loss_clip": 1.03674126, + "balance_loss_mlp": 1.02141261, + "epoch": 0.12283094422842551, + "flos": 15771035577600.0, + "grad_norm": 3.368361034644538, + "language_loss": 0.98209798, + "learning_rate": 3.9102914689298605e-06, + "loss": 1.00350904, + "num_input_tokens_seen": 120145225, + "router_z_loss_clip": 0.6328125, + "router_z_loss_mlp": 0.19683838, + "step": 4233, + "time_per_iteration": 2.5629544258117676 + }, + { + "auxiliary_loss_clip": 0.01033993, + "auxiliary_loss_mlp": 0.01001289, + "balance_loss_clip": 1.02224517, + "balance_loss_mlp": 1.00001347, + "epoch": 0.12285996169694156, + "flos": 72283616317440.0, + "grad_norm": 0.6517276115726974, + "language_loss": 0.48028189, + "learning_rate": 3.910235798223259e-06, + "loss": 0.50063467, + "num_input_tokens_seen": 120207180, + "router_z_loss_clip": 0.1171875, + "router_z_loss_mlp": 0.01275635, + "step": 4234, + "time_per_iteration": 3.199559450149536 + }, + { + "auxiliary_loss_clip": 0.01095464, + "auxiliary_loss_mlp": 0.01040771, + "balance_loss_clip": 1.03276575, + "balance_loss_mlp": 1.02121496, + "epoch": 0.1228889791654576, + "flos": 28869470311680.0, + "grad_norm": 2.748290425732532, + "language_loss": 0.87197566, + "learning_rate": 3.910180110644682e-06, + "loss": 0.89333797, + "num_input_tokens_seen": 120221825, + "router_z_loss_clip": 0.62719727, + "router_z_loss_mlp": 0.19573975, + "step": 4235, + "time_per_iteration": 2.4547908306121826 + }, + { + "auxiliary_loss_clip": 0.01096581, + "auxiliary_loss_mlp": 0.01034073, + "balance_loss_clip": 1.03371322, + "balance_loss_mlp": 1.01600063, + "epoch": 0.12291799663397365, + "flos": 21681831156480.0, + "grad_norm": 2.08491253143732, + "language_loss": 0.68515676, + "learning_rate": 3.910124406194623e-06, + "loss": 0.70646334, + "num_input_tokens_seen": 120236640, + "router_z_loss_clip": 0.62890625, + "router_z_loss_mlp": 0.1809082, + "step": 4236, + "time_per_iteration": 2.3747293949127197 + }, + { + "auxiliary_loss_clip": 0.01099374, + "auxiliary_loss_mlp": 0.01044614, + "balance_loss_clip": 1.03320074, + "balance_loss_mlp": 1.02241147, + "epoch": 0.1229470141024897, + "flos": 16552181537280.0, + "grad_norm": 2.7455860684395854, + "language_loss": 0.85515898, + "learning_rate": 3.91006868487357e-06, + "loss": 0.87659889, + "num_input_tokens_seen": 120251915, + "router_z_loss_clip": 0.66162109, + "router_z_loss_mlp": 0.22186279, + "step": 4237, + "time_per_iteration": 2.3679351806640625 + }, + { + "auxiliary_loss_clip": 0.01022786, + "auxiliary_loss_mlp": 0.00999647, + "balance_loss_clip": 1.01230931, + "balance_loss_mlp": 0.99840122, + "epoch": 0.12297603157100574, + "flos": 74773481771520.0, + "grad_norm": 0.6480901001324949, + "language_loss": 0.49845344, + "learning_rate": 3.910012946682018e-06, + "loss": 0.51867777, + "num_input_tokens_seen": 120314075, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01245117, + "step": 4238, + "time_per_iteration": 3.0458755493164062 + }, + { + "auxiliary_loss_clip": 0.01107422, + "auxiliary_loss_mlp": 0.01039722, + "balance_loss_clip": 1.03602672, + "balance_loss_mlp": 1.01846671, + "epoch": 0.1230050490395218, + "flos": 27707769999360.0, + "grad_norm": 3.102290216321778, + "language_loss": 0.8878876, + "learning_rate": 3.909957191620459e-06, + "loss": 0.90935904, + "num_input_tokens_seen": 120332855, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.21240234, + "step": 4239, + "time_per_iteration": 2.46980619430542 + }, + { + "auxiliary_loss_clip": 0.010907, + "auxiliary_loss_mlp": 0.01036547, + "balance_loss_clip": 1.03049242, + "balance_loss_mlp": 1.01805162, + "epoch": 0.12303406650803785, + "flos": 28578329550720.0, + "grad_norm": 1.9314575825406999, + "language_loss": 0.66337252, + "learning_rate": 3.9099014196893855e-06, + "loss": 0.684645, + "num_input_tokens_seen": 120350035, + "router_z_loss_clip": 0.60229492, + "router_z_loss_mlp": 0.18481445, + "step": 4240, + "time_per_iteration": 2.493189811706543 + }, + { + "auxiliary_loss_clip": 0.01018048, + "auxiliary_loss_mlp": 0.01001717, + "balance_loss_clip": 1.00746369, + "balance_loss_mlp": 1.00037551, + "epoch": 0.12306308397655388, + "flos": 64664799471360.0, + "grad_norm": 0.6581873539253035, + "language_loss": 0.47972369, + "learning_rate": 3.90984563088929e-06, + "loss": 0.49992132, + "num_input_tokens_seen": 120405465, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01342773, + "step": 4241, + "time_per_iteration": 2.9321839809417725 + }, + { + "auxiliary_loss_clip": 0.01104279, + "auxiliary_loss_mlp": 0.01050508, + "balance_loss_clip": 1.03452885, + "balance_loss_mlp": 1.02802527, + "epoch": 0.12309210144506993, + "flos": 26170649170560.0, + "grad_norm": 2.9073768550481702, + "language_loss": 0.81269628, + "learning_rate": 3.909789825220664e-06, + "loss": 0.83424413, + "num_input_tokens_seen": 120419735, + "router_z_loss_clip": 0.69824219, + "router_z_loss_mlp": 0.22473145, + "step": 4242, + "time_per_iteration": 2.354515314102173 + }, + { + "auxiliary_loss_clip": 0.01097055, + "auxiliary_loss_mlp": 0.01044386, + "balance_loss_clip": 1.03374457, + "balance_loss_mlp": 1.02620661, + "epoch": 0.12312111891358597, + "flos": 26865490515840.0, + "grad_norm": 1.7295374844206872, + "language_loss": 0.77879494, + "learning_rate": 3.909734002684002e-06, + "loss": 0.8002094, + "num_input_tokens_seen": 120440465, + "router_z_loss_clip": 0.63330078, + "router_z_loss_mlp": 0.18188477, + "step": 4243, + "time_per_iteration": 2.4765400886535645 + }, + { + "auxiliary_loss_clip": 0.01104349, + "auxiliary_loss_mlp": 0.01046322, + "balance_loss_clip": 1.03520286, + "balance_loss_mlp": 1.02279031, + "epoch": 0.12315013638210202, + "flos": 21717896457600.0, + "grad_norm": 2.4120828436944652, + "language_loss": 0.90080428, + "learning_rate": 3.909678163279797e-06, + "loss": 0.92231095, + "num_input_tokens_seen": 120453595, + "router_z_loss_clip": 0.69140625, + "router_z_loss_mlp": 0.2355957, + "step": 4244, + "time_per_iteration": 2.410734176635742 + }, + { + "auxiliary_loss_clip": 0.01107269, + "auxiliary_loss_mlp": 0.01054203, + "balance_loss_clip": 1.03605103, + "balance_loss_mlp": 1.03192317, + "epoch": 0.12317915385061808, + "flos": 27815546966400.0, + "grad_norm": 2.5722145880958296, + "language_loss": 0.96913832, + "learning_rate": 3.909622307008541e-06, + "loss": 0.99075305, + "num_input_tokens_seen": 120474755, + "router_z_loss_clip": 0.71240234, + "router_z_loss_mlp": 0.22265625, + "step": 4245, + "time_per_iteration": 2.527224540710449 + }, + { + "auxiliary_loss_clip": 0.01102234, + "auxiliary_loss_mlp": 0.0104651, + "balance_loss_clip": 1.03565729, + "balance_loss_mlp": 1.02211952, + "epoch": 0.12320817131913411, + "flos": 20513218394880.0, + "grad_norm": 2.4920979297326684, + "language_loss": 0.78003752, + "learning_rate": 3.909566433870728e-06, + "loss": 0.80152494, + "num_input_tokens_seen": 120489960, + "router_z_loss_clip": 0.66601562, + "router_z_loss_mlp": 0.24414062, + "step": 4246, + "time_per_iteration": 2.369797945022583 + }, + { + "auxiliary_loss_clip": 0.01098021, + "auxiliary_loss_mlp": 0.01040715, + "balance_loss_clip": 1.03575826, + "balance_loss_mlp": 1.02272606, + "epoch": 0.12323718878765016, + "flos": 24088952954880.0, + "grad_norm": 2.2094210228474664, + "language_loss": 0.76125771, + "learning_rate": 3.909510543866852e-06, + "loss": 0.78264511, + "num_input_tokens_seen": 120504515, + "router_z_loss_clip": 0.62304688, + "router_z_loss_mlp": 0.17980957, + "step": 4247, + "time_per_iteration": 2.46346116065979 + }, + { + "auxiliary_loss_clip": 0.01116798, + "auxiliary_loss_mlp": 0.01062646, + "balance_loss_clip": 1.03930342, + "balance_loss_mlp": 1.03776693, + "epoch": 0.12326620625616622, + "flos": 48462238955520.0, + "grad_norm": 2.739266145471977, + "language_loss": 0.82755959, + "learning_rate": 3.909454636997406e-06, + "loss": 0.84935403, + "num_input_tokens_seen": 120522525, + "router_z_loss_clip": 0.77587891, + "router_z_loss_mlp": 0.24890137, + "step": 4248, + "time_per_iteration": 2.6145009994506836 + }, + { + "auxiliary_loss_clip": 0.01106123, + "auxiliary_loss_mlp": 0.01041984, + "balance_loss_clip": 1.0368427, + "balance_loss_mlp": 1.02127743, + "epoch": 0.12329522372468225, + "flos": 33102165525120.0, + "grad_norm": 1.9470048113362972, + "language_loss": 0.85034621, + "learning_rate": 3.909398713262884e-06, + "loss": 0.87182724, + "num_input_tokens_seen": 120543140, + "router_z_loss_clip": 0.69238281, + "router_z_loss_mlp": 0.20703125, + "step": 4249, + "time_per_iteration": 2.550910472869873 + }, + { + "auxiliary_loss_clip": 0.01016285, + "auxiliary_loss_mlp": 0.01038029, + "balance_loss_clip": 1.0054034, + "balance_loss_mlp": 1.03646743, + "epoch": 0.1233242411931983, + "flos": 74770409571840.0, + "grad_norm": 0.6642685355808803, + "language_loss": 0.47864261, + "learning_rate": 3.90934277266378e-06, + "loss": 0.49918574, + "num_input_tokens_seen": 120605625, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01556396, + "step": 4250, + "time_per_iteration": 3.096395492553711 + }, + { + "auxiliary_loss_clip": 0.01106249, + "auxiliary_loss_mlp": 0.01042324, + "balance_loss_clip": 1.03941512, + "balance_loss_mlp": 1.023072, + "epoch": 0.12335325866171436, + "flos": 17195107824000.0, + "grad_norm": 2.2271282906977823, + "language_loss": 0.75889617, + "learning_rate": 3.909286815200588e-06, + "loss": 0.78038192, + "num_input_tokens_seen": 120619175, + "router_z_loss_clip": 0.66943359, + "router_z_loss_mlp": 0.19250488, + "step": 4251, + "time_per_iteration": 2.3455698490142822 + }, + { + "auxiliary_loss_clip": 0.01016635, + "auxiliary_loss_mlp": 0.01019834, + "balance_loss_clip": 1.00515795, + "balance_loss_mlp": 1.0182246, + "epoch": 0.1233822761302304, + "flos": 74786817911040.0, + "grad_norm": 0.6480024707004045, + "language_loss": 0.47715068, + "learning_rate": 3.909230840873802e-06, + "loss": 0.49751535, + "num_input_tokens_seen": 120692490, + "router_z_loss_clip": 0.11474609, + "router_z_loss_mlp": 0.01611328, + "step": 4252, + "time_per_iteration": 3.2096025943756104 + }, + { + "auxiliary_loss_clip": 0.01016936, + "auxiliary_loss_mlp": 0.01006493, + "balance_loss_clip": 1.00546801, + "balance_loss_mlp": 1.00510383, + "epoch": 0.12341129359874645, + "flos": 65215728725760.0, + "grad_norm": 0.6726356177735242, + "language_loss": 0.49197689, + "learning_rate": 3.909174849683917e-06, + "loss": 0.51221114, + "num_input_tokens_seen": 120753605, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01391602, + "step": 4253, + "time_per_iteration": 3.0478127002716064 + }, + { + "auxiliary_loss_clip": 0.01103866, + "auxiliary_loss_mlp": 0.01046305, + "balance_loss_clip": 1.037444, + "balance_loss_mlp": 1.02617681, + "epoch": 0.1234403110672625, + "flos": 18105642748800.0, + "grad_norm": 2.5492120030436554, + "language_loss": 0.80060399, + "learning_rate": 3.909118841631427e-06, + "loss": 0.82210565, + "num_input_tokens_seen": 120767665, + "router_z_loss_clip": 0.66357422, + "router_z_loss_mlp": 0.20117188, + "step": 4254, + "time_per_iteration": 2.414759397506714 + }, + { + "auxiliary_loss_clip": 0.01022878, + "auxiliary_loss_mlp": 0.01002026, + "balance_loss_clip": 1.0103724, + "balance_loss_mlp": 1.00068486, + "epoch": 0.12346932853577854, + "flos": 60426448617600.0, + "grad_norm": 0.6367876942865408, + "language_loss": 0.47456598, + "learning_rate": 3.909062816716827e-06, + "loss": 0.49481499, + "num_input_tokens_seen": 120828340, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.01342773, + "step": 4255, + "time_per_iteration": 2.9307377338409424 + }, + { + "auxiliary_loss_clip": 0.01120375, + "auxiliary_loss_mlp": 0.01057145, + "balance_loss_clip": 1.04174697, + "balance_loss_mlp": 1.03231335, + "epoch": 0.12349834600429459, + "flos": 12780096157440.0, + "grad_norm": 2.4844442494204966, + "language_loss": 0.73285824, + "learning_rate": 3.909006774940611e-06, + "loss": 0.75463343, + "num_input_tokens_seen": 120839385, + "router_z_loss_clip": 0.78613281, + "router_z_loss_mlp": 0.2479248, + "step": 4256, + "time_per_iteration": 2.3719327449798584 + }, + { + "auxiliary_loss_clip": 0.01115142, + "auxiliary_loss_mlp": 0.01067171, + "balance_loss_clip": 1.03905272, + "balance_loss_mlp": 1.042328, + "epoch": 0.12352736347281063, + "flos": 27048993955200.0, + "grad_norm": 2.4083011209823306, + "language_loss": 0.85269046, + "learning_rate": 3.908950716303275e-06, + "loss": 0.87451363, + "num_input_tokens_seen": 120857565, + "router_z_loss_clip": 0.76074219, + "router_z_loss_mlp": 0.24853516, + "step": 4257, + "time_per_iteration": 2.4265074729919434 + }, + { + "auxiliary_loss_clip": 0.01115641, + "auxiliary_loss_mlp": 0.01057904, + "balance_loss_clip": 1.04241729, + "balance_loss_mlp": 1.03282213, + "epoch": 0.12355638094132668, + "flos": 30292215926400.0, + "grad_norm": 3.401873003731107, + "language_loss": 0.75343132, + "learning_rate": 3.908894640805315e-06, + "loss": 0.77516681, + "num_input_tokens_seen": 120873765, + "router_z_loss_clip": 0.73095703, + "router_z_loss_mlp": 0.25085449, + "step": 4258, + "time_per_iteration": 2.486628293991089 + }, + { + "auxiliary_loss_clip": 0.01097987, + "auxiliary_loss_mlp": 0.01053917, + "balance_loss_clip": 1.03644943, + "balance_loss_mlp": 1.0348556, + "epoch": 0.12358539840984273, + "flos": 19455978470400.0, + "grad_norm": 2.527624265100935, + "language_loss": 0.78829372, + "learning_rate": 3.9088385484472235e-06, + "loss": 0.80981278, + "num_input_tokens_seen": 120888725, + "router_z_loss_clip": 0.61572266, + "router_z_loss_mlp": 0.19049072, + "step": 4259, + "time_per_iteration": 2.3667831420898438 + }, + { + "auxiliary_loss_clip": 0.01104378, + "auxiliary_loss_mlp": 0.0104908, + "balance_loss_clip": 1.03877878, + "balance_loss_mlp": 1.02966058, + "epoch": 0.12361441587835877, + "flos": 21462716263680.0, + "grad_norm": 2.4992788077694237, + "language_loss": 0.7524271, + "learning_rate": 3.908782439229498e-06, + "loss": 0.77396166, + "num_input_tokens_seen": 120902830, + "router_z_loss_clip": 0.65673828, + "router_z_loss_mlp": 0.19421387, + "step": 4260, + "time_per_iteration": 2.3759920597076416 + }, + { + "auxiliary_loss_clip": 0.01018631, + "auxiliary_loss_mlp": 0.01032309, + "balance_loss_clip": 1.00622392, + "balance_loss_mlp": 1.0310632, + "epoch": 0.12364343334687482, + "flos": 64783643339520.0, + "grad_norm": 0.6454593397317252, + "language_loss": 0.50261092, + "learning_rate": 3.908726313152633e-06, + "loss": 0.52312034, + "num_input_tokens_seen": 120970225, + "router_z_loss_clip": 0.12402344, + "router_z_loss_mlp": 0.01245117, + "step": 4261, + "time_per_iteration": 3.112119674682617 + }, + { + "auxiliary_loss_clip": 0.01104429, + "auxiliary_loss_mlp": 0.01050023, + "balance_loss_clip": 1.03827703, + "balance_loss_mlp": 1.02988899, + "epoch": 0.12367245081539087, + "flos": 55028878024320.0, + "grad_norm": 2.3451777486071816, + "language_loss": 0.86931443, + "learning_rate": 3.908670170217126e-06, + "loss": 0.89085901, + "num_input_tokens_seen": 120988295, + "router_z_loss_clip": 0.66162109, + "router_z_loss_mlp": 0.20153809, + "step": 4262, + "time_per_iteration": 2.634194850921631 + }, + { + "auxiliary_loss_clip": 0.01014982, + "auxiliary_loss_mlp": 0.01017139, + "balance_loss_clip": 1.00381398, + "balance_loss_mlp": 1.01587582, + "epoch": 0.1237014682839069, + "flos": 52840450316160.0, + "grad_norm": 0.6136372448909141, + "language_loss": 0.4219574, + "learning_rate": 3.908614010423471e-06, + "loss": 0.44227859, + "num_input_tokens_seen": 121044190, + "router_z_loss_clip": 0.11181641, + "router_z_loss_mlp": 0.01263428, + "step": 4263, + "time_per_iteration": 5.077678918838501 + }, + { + "auxiliary_loss_clip": 0.01102192, + "auxiliary_loss_mlp": 0.01054455, + "balance_loss_clip": 1.0309279, + "balance_loss_mlp": 1.03043461, + "epoch": 0.12373048575242296, + "flos": 14748988170240.0, + "grad_norm": 2.3556074190244063, + "language_loss": 0.92796761, + "learning_rate": 3.908557833772165e-06, + "loss": 0.949534, + "num_input_tokens_seen": 121057500, + "router_z_loss_clip": 0.71289062, + "router_z_loss_mlp": 0.24035645, + "step": 4264, + "time_per_iteration": 4.635498285293579 + }, + { + "auxiliary_loss_clip": 0.01100986, + "auxiliary_loss_mlp": 0.0105458, + "balance_loss_clip": 1.03228068, + "balance_loss_mlp": 1.03386128, + "epoch": 0.12375950322093901, + "flos": 26095690748160.0, + "grad_norm": 2.3513983770064812, + "language_loss": 0.80613351, + "learning_rate": 3.908501640263704e-06, + "loss": 0.82768911, + "num_input_tokens_seen": 121071570, + "router_z_loss_clip": 0.68701172, + "router_z_loss_mlp": 0.20727539, + "step": 4265, + "time_per_iteration": 2.4367175102233887 + }, + { + "auxiliary_loss_clip": 0.01100918, + "auxiliary_loss_mlp": 0.01045614, + "balance_loss_clip": 1.03334951, + "balance_loss_mlp": 1.02345252, + "epoch": 0.12378852068945505, + "flos": 36385397781120.0, + "grad_norm": 3.0588659434198813, + "language_loss": 0.80816031, + "learning_rate": 3.9084454298985834e-06, + "loss": 0.82962561, + "num_input_tokens_seen": 121088730, + "router_z_loss_clip": 0.67626953, + "router_z_loss_mlp": 0.22192383, + "step": 4266, + "time_per_iteration": 2.4560048580169678 + }, + { + "auxiliary_loss_clip": 0.01103698, + "auxiliary_loss_mlp": 0.0104138, + "balance_loss_clip": 1.03290653, + "balance_loss_mlp": 1.01888561, + "epoch": 0.1238175381579711, + "flos": 23615006981760.0, + "grad_norm": 3.0049871495853524, + "language_loss": 0.8865664, + "learning_rate": 3.908389202677301e-06, + "loss": 0.90801722, + "num_input_tokens_seen": 121102835, + "router_z_loss_clip": 0.70800781, + "router_z_loss_mlp": 0.22497559, + "step": 4267, + "time_per_iteration": 2.4268250465393066 + }, + { + "auxiliary_loss_clip": 0.01104933, + "auxiliary_loss_mlp": 0.01039541, + "balance_loss_clip": 1.03430521, + "balance_loss_mlp": 1.01628304, + "epoch": 0.12384655562648715, + "flos": 33721978625280.0, + "grad_norm": 1.9151801014700116, + "language_loss": 0.86731249, + "learning_rate": 3.908332958600353e-06, + "loss": 0.88875723, + "num_input_tokens_seen": 121118780, + "router_z_loss_clip": 0.70556641, + "router_z_loss_mlp": 0.23254395, + "step": 4268, + "time_per_iteration": 2.5149853229522705 + }, + { + "auxiliary_loss_clip": 0.01095262, + "auxiliary_loss_mlp": 0.01033206, + "balance_loss_clip": 1.03556108, + "balance_loss_mlp": 1.01418078, + "epoch": 0.12387557309500319, + "flos": 30951341084160.0, + "grad_norm": 2.1906724672542937, + "language_loss": 1.02022147, + "learning_rate": 3.908276697668237e-06, + "loss": 1.04150617, + "num_input_tokens_seen": 121134030, + "router_z_loss_clip": 0.59667969, + "router_z_loss_mlp": 0.19018555, + "step": 4269, + "time_per_iteration": 4.897105932235718 + }, + { + "auxiliary_loss_clip": 0.01106593, + "auxiliary_loss_mlp": 0.01049572, + "balance_loss_clip": 1.03791451, + "balance_loss_mlp": 1.02742326, + "epoch": 0.12390459056351924, + "flos": 11610052030080.0, + "grad_norm": 2.7741314342791847, + "language_loss": 0.78728724, + "learning_rate": 3.908220419881448e-06, + "loss": 0.80884886, + "num_input_tokens_seen": 121145765, + "router_z_loss_clip": 0.68701172, + "router_z_loss_mlp": 0.22143555, + "step": 4270, + "time_per_iteration": 2.3803768157958984 + }, + { + "auxiliary_loss_clip": 0.01034298, + "auxiliary_loss_mlp": 0.01007131, + "balance_loss_clip": 1.02181697, + "balance_loss_mlp": 1.00579584, + "epoch": 0.12393360803203529, + "flos": 71963322704640.0, + "grad_norm": 0.6288392671780185, + "language_loss": 0.44600487, + "learning_rate": 3.908164125240484e-06, + "loss": 0.46641916, + "num_input_tokens_seen": 121209420, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.0133667, + "step": 4271, + "time_per_iteration": 3.041513204574585 + }, + { + "auxiliary_loss_clip": 0.01033666, + "auxiliary_loss_mlp": 0.01002617, + "balance_loss_clip": 1.02108788, + "balance_loss_mlp": 1.0013181, + "epoch": 0.12396262550055133, + "flos": 74774249821440.0, + "grad_norm": 0.6491590860756268, + "language_loss": 0.54001224, + "learning_rate": 3.908107813745842e-06, + "loss": 0.56037509, + "num_input_tokens_seen": 121275380, + "router_z_loss_clip": 0.125, + "router_z_loss_mlp": 0.01300049, + "step": 4272, + "time_per_iteration": 5.633620023727417 + }, + { + "auxiliary_loss_clip": 0.01104358, + "auxiliary_loss_mlp": 0.01043418, + "balance_loss_clip": 1.03815579, + "balance_loss_mlp": 1.02253211, + "epoch": 0.12399164296906738, + "flos": 13107022928640.0, + "grad_norm": 2.679422973155005, + "language_loss": 0.74072587, + "learning_rate": 3.908051485398021e-06, + "loss": 0.76220363, + "num_input_tokens_seen": 121287060, + "router_z_loss_clip": 0.66210938, + "router_z_loss_mlp": 0.2088623, + "step": 4273, + "time_per_iteration": 2.418896436691284 + }, + { + "auxiliary_loss_clip": 0.01102039, + "auxiliary_loss_mlp": 0.01043632, + "balance_loss_clip": 1.03725982, + "balance_loss_mlp": 1.02390313, + "epoch": 0.12402066043758342, + "flos": 26825934078720.0, + "grad_norm": 2.135611057281504, + "language_loss": 0.66862905, + "learning_rate": 3.9079951401975165e-06, + "loss": 0.69008577, + "num_input_tokens_seen": 121301820, + "router_z_loss_clip": 0.64892578, + "router_z_loss_mlp": 0.19726562, + "step": 4274, + "time_per_iteration": 2.485469102859497 + }, + { + "auxiliary_loss_clip": 0.01097662, + "auxiliary_loss_mlp": 0.01040441, + "balance_loss_clip": 1.0351491, + "balance_loss_mlp": 1.02032447, + "epoch": 0.12404967790609947, + "flos": 27300578279040.0, + "grad_norm": 1.9211686230954224, + "language_loss": 0.74570101, + "learning_rate": 3.907938778144827e-06, + "loss": 0.7670821, + "num_input_tokens_seen": 121316325, + "router_z_loss_clip": 0.625, + "router_z_loss_mlp": 0.20117188, + "step": 4275, + "time_per_iteration": 2.5645956993103027 + }, + { + "auxiliary_loss_clip": 0.01103815, + "auxiliary_loss_mlp": 0.01042806, + "balance_loss_clip": 1.03678918, + "balance_loss_mlp": 1.02141976, + "epoch": 0.12407869537461552, + "flos": 31023506597760.0, + "grad_norm": 2.3042160122020725, + "language_loss": 0.84138674, + "learning_rate": 3.9078823992404495e-06, + "loss": 0.86285293, + "num_input_tokens_seen": 121332475, + "router_z_loss_clip": 0.66992188, + "router_z_loss_mlp": 0.21398926, + "step": 4276, + "time_per_iteration": 2.600614547729492 + }, + { + "auxiliary_loss_clip": 0.01027726, + "auxiliary_loss_mlp": 0.01001272, + "balance_loss_clip": 1.01635861, + "balance_loss_mlp": 0.9999308, + "epoch": 0.12410771284313156, + "flos": 68894213016960.0, + "grad_norm": 0.688333647349342, + "language_loss": 0.45730928, + "learning_rate": 3.907826003484883e-06, + "loss": 0.47759929, + "num_input_tokens_seen": 121393170, + "router_z_loss_clip": 0.11376953, + "router_z_loss_mlp": 0.01342773, + "step": 4277, + "time_per_iteration": 3.136293888092041 + }, + { + "auxiliary_loss_clip": 0.01097087, + "auxiliary_loss_mlp": 0.01039218, + "balance_loss_clip": 1.03479898, + "balance_loss_mlp": 1.02039504, + "epoch": 0.12413673031164761, + "flos": 15224470243200.0, + "grad_norm": 2.0999162756027876, + "language_loss": 0.59569907, + "learning_rate": 3.907769590878625e-06, + "loss": 0.61706209, + "num_input_tokens_seen": 121407655, + "router_z_loss_clip": 0.62280273, + "router_z_loss_mlp": 0.18823242, + "step": 4278, + "time_per_iteration": 2.344078779220581 + }, + { + "auxiliary_loss_clip": 0.01023512, + "auxiliary_loss_mlp": 0.01000688, + "balance_loss_clip": 1.01264858, + "balance_loss_mlp": 0.99937057, + "epoch": 0.12416574778016366, + "flos": 66339583257600.0, + "grad_norm": 0.7228949033580208, + "language_loss": 0.50120556, + "learning_rate": 3.907713161422174e-06, + "loss": 0.52144754, + "num_input_tokens_seen": 121467400, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01318359, + "step": 4279, + "time_per_iteration": 2.947073459625244 + }, + { + "auxiliary_loss_clip": 0.01103923, + "auxiliary_loss_mlp": 0.01040823, + "balance_loss_clip": 1.03508639, + "balance_loss_mlp": 1.01916313, + "epoch": 0.1241947652486797, + "flos": 14057253936000.0, + "grad_norm": 2.641317332251218, + "language_loss": 0.76230991, + "learning_rate": 3.907656715116028e-06, + "loss": 0.78375733, + "num_input_tokens_seen": 121480440, + "router_z_loss_clip": 0.68896484, + "router_z_loss_mlp": 0.21679688, + "step": 4280, + "time_per_iteration": 2.34898042678833 + }, + { + "auxiliary_loss_clip": 0.01096509, + "auxiliary_loss_mlp": 0.01044664, + "balance_loss_clip": 1.03290093, + "balance_loss_mlp": 1.02513123, + "epoch": 0.12422378271719575, + "flos": 37226664835200.0, + "grad_norm": 2.0389086912435785, + "language_loss": 0.63924038, + "learning_rate": 3.907600251960687e-06, + "loss": 0.6606521, + "num_input_tokens_seen": 121501525, + "router_z_loss_clip": 0.63574219, + "router_z_loss_mlp": 0.19549561, + "step": 4281, + "time_per_iteration": 2.542577028274536 + }, + { + "auxiliary_loss_clip": 0.01017776, + "auxiliary_loss_mlp": 0.01001107, + "balance_loss_clip": 1.007653, + "balance_loss_mlp": 0.99974233, + "epoch": 0.1242528001857118, + "flos": 61932985228800.0, + "grad_norm": 0.6736167572016509, + "language_loss": 0.50726873, + "learning_rate": 3.907543771956647e-06, + "loss": 0.52745754, + "num_input_tokens_seen": 121565085, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.01367188, + "step": 4282, + "time_per_iteration": 2.9833502769470215 + }, + { + "auxiliary_loss_clip": 0.01093817, + "auxiliary_loss_mlp": 0.01046931, + "balance_loss_clip": 1.0308001, + "balance_loss_mlp": 1.02765489, + "epoch": 0.12428181765422784, + "flos": 14387776577280.0, + "grad_norm": 6.957192762069437, + "language_loss": 0.9288497, + "learning_rate": 3.90748727510441e-06, + "loss": 0.95025718, + "num_input_tokens_seen": 121578245, + "router_z_loss_clip": 0.63037109, + "router_z_loss_mlp": 0.19274902, + "step": 4283, + "time_per_iteration": 2.390331268310547 + }, + { + "auxiliary_loss_clip": 0.01014595, + "auxiliary_loss_mlp": 0.01003368, + "balance_loss_clip": 1.00471878, + "balance_loss_mlp": 1.00206876, + "epoch": 0.12431083512274389, + "flos": 74791216742400.0, + "grad_norm": 0.6417823786139399, + "language_loss": 0.45083684, + "learning_rate": 3.907430761404474e-06, + "loss": 0.47101647, + "num_input_tokens_seen": 121637845, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01300049, + "step": 4284, + "time_per_iteration": 3.170628786087036 + }, + { + "auxiliary_loss_clip": 0.01014356, + "auxiliary_loss_mlp": 0.01002039, + "balance_loss_clip": 1.00452793, + "balance_loss_mlp": 1.00087106, + "epoch": 0.12433985259125994, + "flos": 48969455454720.0, + "grad_norm": 0.6999446178570652, + "language_loss": 0.49930209, + "learning_rate": 3.907374230857336e-06, + "loss": 0.51946604, + "num_input_tokens_seen": 121689750, + "router_z_loss_clip": 0.09814453, + "router_z_loss_mlp": 0.01165771, + "step": 4285, + "time_per_iteration": 2.7904443740844727 + }, + { + "auxiliary_loss_clip": 0.01094468, + "auxiliary_loss_mlp": 0.01040089, + "balance_loss_clip": 1.02961302, + "balance_loss_mlp": 1.01838124, + "epoch": 0.12436887005977598, + "flos": 23325402320640.0, + "grad_norm": 2.602142277428242, + "language_loss": 0.83171391, + "learning_rate": 3.907317683463498e-06, + "loss": 0.85305947, + "num_input_tokens_seen": 121706815, + "router_z_loss_clip": 0.64794922, + "router_z_loss_mlp": 0.21704102, + "step": 4286, + "time_per_iteration": 2.3980584144592285 + }, + { + "auxiliary_loss_clip": 0.0109993, + "auxiliary_loss_mlp": 0.01051424, + "balance_loss_clip": 1.03178358, + "balance_loss_mlp": 1.03036523, + "epoch": 0.12439788752829203, + "flos": 39593217767040.0, + "grad_norm": 2.3642155903563555, + "language_loss": 0.79637212, + "learning_rate": 3.907261119223458e-06, + "loss": 0.81788564, + "num_input_tokens_seen": 121722690, + "router_z_loss_clip": 0.68066406, + "router_z_loss_mlp": 0.21063232, + "step": 4287, + "time_per_iteration": 2.559140205383301 + }, + { + "auxiliary_loss_clip": 0.01013547, + "auxiliary_loss_mlp": 0.01005132, + "balance_loss_clip": 1.00336349, + "balance_loss_mlp": 1.00386274, + "epoch": 0.12442690499680809, + "flos": 60266267832960.0, + "grad_norm": 0.7231114199182233, + "language_loss": 0.46549302, + "learning_rate": 3.907204538137716e-06, + "loss": 0.48567981, + "num_input_tokens_seen": 121773065, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.01269531, + "step": 4288, + "time_per_iteration": 2.9782090187072754 + }, + { + "auxiliary_loss_clip": 0.01108423, + "auxiliary_loss_mlp": 0.01048095, + "balance_loss_clip": 1.03315604, + "balance_loss_mlp": 1.02385998, + "epoch": 0.12445592246532412, + "flos": 27115224537600.0, + "grad_norm": 3.2200583671365712, + "language_loss": 1.04224598, + "learning_rate": 3.907147940206773e-06, + "loss": 1.06381118, + "num_input_tokens_seen": 121786215, + "router_z_loss_clip": 0.75292969, + "router_z_loss_mlp": 0.24230957, + "step": 4289, + "time_per_iteration": 2.4607560634613037 + }, + { + "auxiliary_loss_clip": 0.01095898, + "auxiliary_loss_mlp": 0.0104438, + "balance_loss_clip": 1.03269601, + "balance_loss_mlp": 1.02440059, + "epoch": 0.12448493993384017, + "flos": 55361076410880.0, + "grad_norm": 3.6233221931128874, + "language_loss": 0.52497578, + "learning_rate": 3.907091325431125e-06, + "loss": 0.54637861, + "num_input_tokens_seen": 121809115, + "router_z_loss_clip": 0.63232422, + "router_z_loss_mlp": 0.1998291, + "step": 4290, + "time_per_iteration": 2.9042487144470215 + }, + { + "auxiliary_loss_clip": 0.01015917, + "auxiliary_loss_mlp": 0.01004416, + "balance_loss_clip": 1.00510955, + "balance_loss_mlp": 1.00298595, + "epoch": 0.12451395740235621, + "flos": 63208990932480.0, + "grad_norm": 0.6413603576117884, + "language_loss": 0.49601215, + "learning_rate": 3.907034693811277e-06, + "loss": 0.5162155, + "num_input_tokens_seen": 121872220, + "router_z_loss_clip": 0.10791016, + "router_z_loss_mlp": 0.01428223, + "step": 4291, + "time_per_iteration": 3.028677225112915 + }, + { + "auxiliary_loss_clip": 0.01095466, + "auxiliary_loss_mlp": 0.01047792, + "balance_loss_clip": 1.03273869, + "balance_loss_mlp": 1.02852821, + "epoch": 0.12454297487087226, + "flos": 11538864034560.0, + "grad_norm": 2.6704982255134064, + "language_loss": 0.74108815, + "learning_rate": 3.906978045347726e-06, + "loss": 0.76252079, + "num_input_tokens_seen": 121884990, + "router_z_loss_clip": 0.62744141, + "router_z_loss_mlp": 0.19274902, + "step": 4292, + "time_per_iteration": 2.5576961040496826 + }, + { + "auxiliary_loss_clip": 0.0101632, + "auxiliary_loss_mlp": 0.01007156, + "balance_loss_clip": 1.00544131, + "balance_loss_mlp": 1.00582731, + "epoch": 0.12457199233938832, + "flos": 66492293011200.0, + "grad_norm": 0.6312530662782818, + "language_loss": 0.48284918, + "learning_rate": 3.906921380040973e-06, + "loss": 0.50308394, + "num_input_tokens_seen": 121951440, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01330566, + "step": 4293, + "time_per_iteration": 3.057857036590576 + }, + { + "auxiliary_loss_clip": 0.01091407, + "auxiliary_loss_mlp": 0.0104271, + "balance_loss_clip": 1.03077769, + "balance_loss_mlp": 1.02301645, + "epoch": 0.12460100980790435, + "flos": 31902863811840.0, + "grad_norm": 1.7784047805202488, + "language_loss": 0.87111372, + "learning_rate": 3.90686469789152e-06, + "loss": 0.89245486, + "num_input_tokens_seen": 121976390, + "router_z_loss_clip": 0.60693359, + "router_z_loss_mlp": 0.19689941, + "step": 4294, + "time_per_iteration": 2.681356906890869 + }, + { + "auxiliary_loss_clip": 0.0110294, + "auxiliary_loss_mlp": 0.01049274, + "balance_loss_clip": 1.03479433, + "balance_loss_mlp": 1.02812684, + "epoch": 0.1246300272764204, + "flos": 34122816478080.0, + "grad_norm": 2.5597808758588685, + "language_loss": 0.9312737, + "learning_rate": 3.906807998899866e-06, + "loss": 0.95279586, + "num_input_tokens_seen": 121993740, + "router_z_loss_clip": 0.68017578, + "router_z_loss_mlp": 0.21154785, + "step": 4295, + "time_per_iteration": 2.635049343109131 + }, + { + "auxiliary_loss_clip": 0.01017031, + "auxiliary_loss_mlp": 0.01001376, + "balance_loss_clip": 1.0060513, + "balance_loss_mlp": 1.00018406, + "epoch": 0.12465904474493646, + "flos": 55975685852160.0, + "grad_norm": 0.6874907417805202, + "language_loss": 0.48252451, + "learning_rate": 3.906751283066511e-06, + "loss": 0.50270855, + "num_input_tokens_seen": 122055320, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01190186, + "step": 4296, + "time_per_iteration": 3.1127545833587646 + }, + { + "auxiliary_loss_clip": 0.01092019, + "auxiliary_loss_mlp": 0.0104519, + "balance_loss_clip": 1.03053248, + "balance_loss_mlp": 1.02643847, + "epoch": 0.1246880622134525, + "flos": 13221153763200.0, + "grad_norm": 2.304784221459676, + "language_loss": 0.70510542, + "learning_rate": 3.906694550391958e-06, + "loss": 0.72647762, + "num_input_tokens_seen": 122068445, + "router_z_loss_clip": 0.61474609, + "router_z_loss_mlp": 0.1875, + "step": 4297, + "time_per_iteration": 2.3727667331695557 + }, + { + "auxiliary_loss_clip": 0.01016827, + "auxiliary_loss_mlp": 0.01002035, + "balance_loss_clip": 1.00594687, + "balance_loss_mlp": 1.00071228, + "epoch": 0.12471707968196855, + "flos": 60647518546560.0, + "grad_norm": 0.6849394750951607, + "language_loss": 0.48402983, + "learning_rate": 3.906637800876706e-06, + "loss": 0.50421846, + "num_input_tokens_seen": 122127275, + "router_z_loss_clip": 0.10888672, + "router_z_loss_mlp": 0.01324463, + "step": 4298, + "time_per_iteration": 2.927659511566162 + }, + { + "auxiliary_loss_clip": 0.01095423, + "auxiliary_loss_mlp": 0.01044656, + "balance_loss_clip": 1.03412187, + "balance_loss_mlp": 1.02404451, + "epoch": 0.1247460971504846, + "flos": 11358572440320.0, + "grad_norm": 2.7169984922478916, + "language_loss": 0.90490091, + "learning_rate": 3.906581034521259e-06, + "loss": 0.92630166, + "num_input_tokens_seen": 122138040, + "router_z_loss_clip": 0.61230469, + "router_z_loss_mlp": 0.20617676, + "step": 4299, + "time_per_iteration": 2.374307632446289 + }, + { + "auxiliary_loss_clip": 0.01015459, + "auxiliary_loss_mlp": 0.01001452, + "balance_loss_clip": 1.004614, + "balance_loss_mlp": 1.00003386, + "epoch": 0.12477511461900063, + "flos": 62481715067520.0, + "grad_norm": 0.6981619364852875, + "language_loss": 0.53293288, + "learning_rate": 3.906524251326116e-06, + "loss": 0.55310202, + "num_input_tokens_seen": 122201460, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01416016, + "step": 4300, + "time_per_iteration": 2.9821043014526367 + }, + { + "auxiliary_loss_clip": 0.01013856, + "auxiliary_loss_mlp": 0.01001521, + "balance_loss_clip": 1.00301647, + "balance_loss_mlp": 1.0000962, + "epoch": 0.12480413208751669, + "flos": 74791216742400.0, + "grad_norm": 0.592471081418853, + "language_loss": 0.39914605, + "learning_rate": 3.906467451291779e-06, + "loss": 0.41929984, + "num_input_tokens_seen": 122272725, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01422119, + "step": 4301, + "time_per_iteration": 3.2696237564086914 + }, + { + "auxiliary_loss_clip": 0.01014395, + "auxiliary_loss_mlp": 0.01002062, + "balance_loss_clip": 1.00363266, + "balance_loss_mlp": 1.00057781, + "epoch": 0.12483314955603274, + "flos": 74343595800960.0, + "grad_norm": 0.709531278910636, + "language_loss": 0.54273713, + "learning_rate": 3.90641063441875e-06, + "loss": 0.56290168, + "num_input_tokens_seen": 122341065, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01483154, + "step": 4302, + "time_per_iteration": 3.1523067951202393 + }, + { + "auxiliary_loss_clip": 0.01093905, + "auxiliary_loss_mlp": 0.01043339, + "balance_loss_clip": 1.03045559, + "balance_loss_mlp": 1.02313304, + "epoch": 0.12486216702454878, + "flos": 25548881034240.0, + "grad_norm": 2.429883554991909, + "language_loss": 0.98453724, + "learning_rate": 3.906353800707532e-06, + "loss": 1.00590968, + "num_input_tokens_seen": 122355840, + "router_z_loss_clip": 0.63525391, + "router_z_loss_mlp": 0.20214844, + "step": 4303, + "time_per_iteration": 2.4261887073516846 + }, + { + "auxiliary_loss_clip": 0.01100813, + "auxiliary_loss_mlp": 0.01045598, + "balance_loss_clip": 1.03334749, + "balance_loss_mlp": 1.02415228, + "epoch": 0.12489118449306483, + "flos": 22594740053760.0, + "grad_norm": 1.976980934394955, + "language_loss": 0.81693667, + "learning_rate": 3.906296950158625e-06, + "loss": 0.83840078, + "num_input_tokens_seen": 122374215, + "router_z_loss_clip": 0.67431641, + "router_z_loss_mlp": 0.21472168, + "step": 4304, + "time_per_iteration": 2.4203133583068848 + }, + { + "auxiliary_loss_clip": 0.01087971, + "auxiliary_loss_mlp": 0.01040684, + "balance_loss_clip": 1.02920723, + "balance_loss_mlp": 1.02183723, + "epoch": 0.12492020196158087, + "flos": 12632518373760.0, + "grad_norm": 2.9247099062751936, + "language_loss": 0.74873179, + "learning_rate": 3.9062400827725325e-06, + "loss": 0.77001834, + "num_input_tokens_seen": 122390905, + "router_z_loss_clip": 0.5871582, + "router_z_loss_mlp": 0.18847656, + "step": 4305, + "time_per_iteration": 2.392711877822876 + }, + { + "auxiliary_loss_clip": 0.01083, + "auxiliary_loss_mlp": 0.01038667, + "balance_loss_clip": 1.02746773, + "balance_loss_mlp": 1.02223945, + "epoch": 0.12494921943009692, + "flos": 32482387336320.0, + "grad_norm": 2.1363678487735434, + "language_loss": 0.80565596, + "learning_rate": 3.906183198549755e-06, + "loss": 0.82687271, + "num_input_tokens_seen": 122406360, + "router_z_loss_clip": 0.55541992, + "router_z_loss_mlp": 0.16418457, + "step": 4306, + "time_per_iteration": 2.4641716480255127 + }, + { + "auxiliary_loss_clip": 0.01097071, + "auxiliary_loss_mlp": 0.0103988, + "balance_loss_clip": 1.03061855, + "balance_loss_mlp": 1.01954877, + "epoch": 0.12497823689861297, + "flos": 41680115775360.0, + "grad_norm": 1.9489044329026137, + "language_loss": 0.87523639, + "learning_rate": 3.906126297490797e-06, + "loss": 0.89660585, + "num_input_tokens_seen": 122426770, + "router_z_loss_clip": 0.6640625, + "router_z_loss_mlp": 0.20306396, + "step": 4307, + "time_per_iteration": 2.5912575721740723 + }, + { + "auxiliary_loss_clip": 0.01099206, + "auxiliary_loss_mlp": 0.01041875, + "balance_loss_clip": 1.03275359, + "balance_loss_mlp": 1.02042937, + "epoch": 0.125007254367129, + "flos": 28943625772800.0, + "grad_norm": 2.6471305737693753, + "language_loss": 0.6621055, + "learning_rate": 3.90606937959616e-06, + "loss": 0.68351626, + "num_input_tokens_seen": 122440570, + "router_z_loss_clip": 0.66503906, + "router_z_loss_mlp": 0.21435547, + "step": 4308, + "time_per_iteration": 2.3713536262512207 + }, + { + "auxiliary_loss_clip": 0.0109404, + "auxiliary_loss_mlp": 0.01037761, + "balance_loss_clip": 1.03110528, + "balance_loss_mlp": 1.0178175, + "epoch": 0.12503627183564506, + "flos": 11321739089280.0, + "grad_norm": 2.0637390836543084, + "language_loss": 0.74688649, + "learning_rate": 3.906012444866346e-06, + "loss": 0.76820445, + "num_input_tokens_seen": 122454440, + "router_z_loss_clip": 0.62890625, + "router_z_loss_mlp": 0.1998291, + "step": 4309, + "time_per_iteration": 2.672299861907959 + }, + { + "auxiliary_loss_clip": 0.01016243, + "auxiliary_loss_mlp": 0.01006071, + "balance_loss_clip": 1.00618243, + "balance_loss_mlp": 1.00475943, + "epoch": 0.1250652893041611, + "flos": 67362608183040.0, + "grad_norm": 0.7453819999017881, + "language_loss": 0.50560278, + "learning_rate": 3.905955493301861e-06, + "loss": 0.52582586, + "num_input_tokens_seen": 122518250, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.01312256, + "step": 4310, + "time_per_iteration": 2.9692726135253906 + }, + { + "auxiliary_loss_clip": 0.01100734, + "auxiliary_loss_mlp": 0.01041145, + "balance_loss_clip": 1.03402591, + "balance_loss_mlp": 1.02003336, + "epoch": 0.12509430677267716, + "flos": 17962359062400.0, + "grad_norm": 2.3344947544113426, + "language_loss": 0.79594904, + "learning_rate": 3.905898524903204e-06, + "loss": 0.81736779, + "num_input_tokens_seen": 122532440, + "router_z_loss_clip": 0.66723633, + "router_z_loss_mlp": 0.21118164, + "step": 4311, + "time_per_iteration": 2.360891342163086 + }, + { + "auxiliary_loss_clip": 0.0110104, + "auxiliary_loss_mlp": 0.01034839, + "balance_loss_clip": 1.03429151, + "balance_loss_mlp": 1.01700509, + "epoch": 0.12512332424119318, + "flos": 10441893116160.0, + "grad_norm": 13.256777452515, + "language_loss": 0.74448645, + "learning_rate": 3.9058415396708805e-06, + "loss": 0.76584524, + "num_input_tokens_seen": 122542705, + "router_z_loss_clip": 0.66748047, + "router_z_loss_mlp": 0.17828369, + "step": 4312, + "time_per_iteration": 2.3595194816589355 + }, + { + "auxiliary_loss_clip": 0.01016304, + "auxiliary_loss_mlp": 0.0100144, + "balance_loss_clip": 1.00601053, + "balance_loss_mlp": 1.00017679, + "epoch": 0.12515234170970924, + "flos": 59222922629760.0, + "grad_norm": 0.6804588323005181, + "language_loss": 0.48698559, + "learning_rate": 3.905784537605394e-06, + "loss": 0.50716305, + "num_input_tokens_seen": 122600665, + "router_z_loss_clip": 0.10302734, + "router_z_loss_mlp": 0.01263428, + "step": 4313, + "time_per_iteration": 2.831491231918335 + }, + { + "auxiliary_loss_clip": 0.01106004, + "auxiliary_loss_mlp": 0.01057468, + "balance_loss_clip": 1.03790975, + "balance_loss_mlp": 1.03653514, + "epoch": 0.1251813591782253, + "flos": 20262157741440.0, + "grad_norm": 3.861393875441388, + "language_loss": 0.83366573, + "learning_rate": 3.905727518707247e-06, + "loss": 0.85530037, + "num_input_tokens_seen": 122614070, + "router_z_loss_clip": 0.68017578, + "router_z_loss_mlp": 0.20922852, + "step": 4314, + "time_per_iteration": 2.4075613021850586 + }, + { + "auxiliary_loss_clip": 0.01111438, + "auxiliary_loss_mlp": 0.01052399, + "balance_loss_clip": 1.03760839, + "balance_loss_mlp": 1.0287596, + "epoch": 0.12521037664674134, + "flos": 18543488509440.0, + "grad_norm": 2.332871022030549, + "language_loss": 0.73145515, + "learning_rate": 3.905670482976942e-06, + "loss": 0.75309348, + "num_input_tokens_seen": 122626425, + "router_z_loss_clip": 0.73779297, + "router_z_loss_mlp": 0.23632812, + "step": 4315, + "time_per_iteration": 2.3566341400146484 + }, + { + "auxiliary_loss_clip": 0.01096331, + "auxiliary_loss_mlp": 0.0103993, + "balance_loss_clip": 1.03060246, + "balance_loss_mlp": 1.01969969, + "epoch": 0.1252393941152574, + "flos": 26169217804800.0, + "grad_norm": 2.136854399568716, + "language_loss": 0.90196693, + "learning_rate": 3.905613430414986e-06, + "loss": 0.92332953, + "num_input_tokens_seen": 122644825, + "router_z_loss_clip": 0.65820312, + "router_z_loss_mlp": 0.20239258, + "step": 4316, + "time_per_iteration": 2.5222294330596924 + }, + { + "auxiliary_loss_clip": 0.01017448, + "auxiliary_loss_mlp": 0.01002064, + "balance_loss_clip": 1.00716853, + "balance_loss_mlp": 1.00077701, + "epoch": 0.12526841158377344, + "flos": 74775681187200.0, + "grad_norm": 0.620982220124819, + "language_loss": 0.45212701, + "learning_rate": 3.9055563610218805e-06, + "loss": 0.47232217, + "num_input_tokens_seen": 122714850, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01287842, + "step": 4317, + "time_per_iteration": 3.204464912414551 + }, + { + "auxiliary_loss_clip": 0.01017295, + "auxiliary_loss_mlp": 0.01000938, + "balance_loss_clip": 1.00679255, + "balance_loss_mlp": 0.9996624, + "epoch": 0.12529742905228947, + "flos": 60773065395840.0, + "grad_norm": 0.6534591614405072, + "language_loss": 0.48835209, + "learning_rate": 3.905499274798129e-06, + "loss": 0.50853443, + "num_input_tokens_seen": 122776475, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01275635, + "step": 4318, + "time_per_iteration": 3.0461788177490234 + }, + { + "auxiliary_loss_clip": 0.01100854, + "auxiliary_loss_mlp": 0.01042534, + "balance_loss_clip": 1.03239512, + "balance_loss_mlp": 1.01968181, + "epoch": 0.12532644652080552, + "flos": 16905852276480.0, + "grad_norm": 2.4743866018530483, + "language_loss": 0.87436759, + "learning_rate": 3.905442171744238e-06, + "loss": 0.89580142, + "num_input_tokens_seen": 122789080, + "router_z_loss_clip": 0.68505859, + "router_z_loss_mlp": 0.2286377, + "step": 4319, + "time_per_iteration": 2.3338494300842285 + }, + { + "auxiliary_loss_clip": 0.01016432, + "auxiliary_loss_mlp": 0.01001723, + "balance_loss_clip": 1.00641632, + "balance_loss_mlp": 1.00051308, + "epoch": 0.12535546398932157, + "flos": 57139445934720.0, + "grad_norm": 0.7007133872146317, + "language_loss": 0.48712909, + "learning_rate": 3.905385051860711e-06, + "loss": 0.50731063, + "num_input_tokens_seen": 122845285, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.01208496, + "step": 4320, + "time_per_iteration": 2.845053195953369 + }, + { + "auxiliary_loss_clip": 0.01100911, + "auxiliary_loss_mlp": 0.01043229, + "balance_loss_clip": 1.03278089, + "balance_loss_mlp": 1.0211035, + "epoch": 0.12538448145783762, + "flos": 38465767365120.0, + "grad_norm": 2.7882547082941294, + "language_loss": 0.90114683, + "learning_rate": 3.9053279151480515e-06, + "loss": 0.92258823, + "num_input_tokens_seen": 122861700, + "router_z_loss_clip": 0.68115234, + "router_z_loss_mlp": 0.22119141, + "step": 4321, + "time_per_iteration": 2.4213807582855225 + }, + { + "auxiliary_loss_clip": 0.01104075, + "auxiliary_loss_mlp": 0.01052091, + "balance_loss_clip": 1.03295588, + "balance_loss_mlp": 1.0305022, + "epoch": 0.12541349892635367, + "flos": 20441262349440.0, + "grad_norm": 2.568578044929946, + "language_loss": 0.83769512, + "learning_rate": 3.905270761606765e-06, + "loss": 0.8592568, + "num_input_tokens_seen": 122874620, + "router_z_loss_clip": 0.7109375, + "router_z_loss_mlp": 0.21582031, + "step": 4322, + "time_per_iteration": 2.7030205726623535 + }, + { + "auxiliary_loss_clip": 0.01099583, + "auxiliary_loss_mlp": 0.01039923, + "balance_loss_clip": 1.03096151, + "balance_loss_mlp": 1.01641512, + "epoch": 0.12544251639486972, + "flos": 12495588554880.0, + "grad_norm": 4.81291272375921, + "language_loss": 0.63121718, + "learning_rate": 3.905213591237356e-06, + "loss": 0.65261221, + "num_input_tokens_seen": 122892190, + "router_z_loss_clip": 0.68603516, + "router_z_loss_mlp": 0.23522949, + "step": 4323, + "time_per_iteration": 2.3787646293640137 + }, + { + "auxiliary_loss_clip": 0.01014578, + "auxiliary_loss_mlp": 0.01001803, + "balance_loss_clip": 1.00441051, + "balance_loss_mlp": 1.00060534, + "epoch": 0.12547153386338575, + "flos": 65049925345920.0, + "grad_norm": 0.6902195381346239, + "language_loss": 0.47724342, + "learning_rate": 3.90515640404033e-06, + "loss": 0.49740723, + "num_input_tokens_seen": 122958200, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.01196289, + "step": 4324, + "time_per_iteration": 3.1510396003723145 + }, + { + "auxiliary_loss_clip": 0.01105383, + "auxiliary_loss_mlp": 0.01045692, + "balance_loss_clip": 1.03409791, + "balance_loss_mlp": 1.02345943, + "epoch": 0.1255005513319018, + "flos": 34896386672640.0, + "grad_norm": 1.9944584133514092, + "language_loss": 0.85783303, + "learning_rate": 3.905099200016192e-06, + "loss": 0.87934375, + "num_input_tokens_seen": 122976955, + "router_z_loss_clip": 0.71337891, + "router_z_loss_mlp": 0.22216797, + "step": 4325, + "time_per_iteration": 2.48555850982666 + }, + { + "auxiliary_loss_clip": 0.01014069, + "auxiliary_loss_mlp": 0.01004809, + "balance_loss_clip": 1.00395477, + "balance_loss_mlp": 1.00362301, + "epoch": 0.12552956880041785, + "flos": 68060521728000.0, + "grad_norm": 0.7104720827722599, + "language_loss": 0.50379497, + "learning_rate": 3.905041979165446e-06, + "loss": 0.52398378, + "num_input_tokens_seen": 123038310, + "router_z_loss_clip": 0.10107422, + "router_z_loss_mlp": 0.01184082, + "step": 4326, + "time_per_iteration": 2.9946351051330566 + }, + { + "auxiliary_loss_clip": 0.01092558, + "auxiliary_loss_mlp": 0.01041435, + "balance_loss_clip": 1.03084052, + "balance_loss_mlp": 1.02342296, + "epoch": 0.1255585862689339, + "flos": 27955095137280.0, + "grad_norm": 2.547127306670034, + "language_loss": 0.78519416, + "learning_rate": 3.904984741488598e-06, + "loss": 0.80653417, + "num_input_tokens_seen": 123053145, + "router_z_loss_clip": 0.61816406, + "router_z_loss_mlp": 0.18005371, + "step": 4327, + "time_per_iteration": 2.379086494445801 + }, + { + "auxiliary_loss_clip": 0.01090788, + "auxiliary_loss_mlp": 0.01043565, + "balance_loss_clip": 1.03118074, + "balance_loss_mlp": 1.02530849, + "epoch": 0.12558760373744995, + "flos": 22959372960000.0, + "grad_norm": 2.20342372635157, + "language_loss": 0.78158575, + "learning_rate": 3.904927486986155e-06, + "loss": 0.80292934, + "num_input_tokens_seen": 123068395, + "router_z_loss_clip": 0.59619141, + "router_z_loss_mlp": 0.18273926, + "step": 4328, + "time_per_iteration": 2.3784432411193848 + }, + { + "auxiliary_loss_clip": 0.01100807, + "auxiliary_loss_mlp": 0.01053687, + "balance_loss_clip": 1.0332948, + "balance_loss_mlp": 1.03101373, + "epoch": 0.12561662120596598, + "flos": 17639551831680.0, + "grad_norm": 2.345067068208995, + "language_loss": 0.71912515, + "learning_rate": 3.904870215658621e-06, + "loss": 0.74067003, + "num_input_tokens_seen": 123085315, + "router_z_loss_clip": 0.67578125, + "router_z_loss_mlp": 0.2265625, + "step": 4329, + "time_per_iteration": 2.5184051990509033 + }, + { + "auxiliary_loss_clip": 0.01100805, + "auxiliary_loss_mlp": 0.01045603, + "balance_loss_clip": 1.03109002, + "balance_loss_mlp": 1.02328706, + "epoch": 0.12564563867448203, + "flos": 16754364420480.0, + "grad_norm": 4.069080707124241, + "language_loss": 0.90808219, + "learning_rate": 3.904812927506503e-06, + "loss": 0.92954624, + "num_input_tokens_seen": 123096210, + "router_z_loss_clip": 0.69726562, + "router_z_loss_mlp": 0.22290039, + "step": 4330, + "time_per_iteration": 2.3538312911987305 + }, + { + "auxiliary_loss_clip": 0.01096041, + "auxiliary_loss_mlp": 0.01055576, + "balance_loss_clip": 1.02968192, + "balance_loss_mlp": 1.03187704, + "epoch": 0.12567465614299808, + "flos": 37113790809600.0, + "grad_norm": 2.173362284441944, + "language_loss": 0.77778208, + "learning_rate": 3.904755622530306e-06, + "loss": 0.79929817, + "num_input_tokens_seen": 123112735, + "router_z_loss_clip": 0.6640625, + "router_z_loss_mlp": 0.23693848, + "step": 4331, + "time_per_iteration": 2.510401964187622 + }, + { + "auxiliary_loss_clip": 0.01098351, + "auxiliary_loss_mlp": 0.01042971, + "balance_loss_clip": 1.03213787, + "balance_loss_mlp": 1.02200186, + "epoch": 0.12570367361151413, + "flos": 26211392593920.0, + "grad_norm": 1.7972172126940649, + "language_loss": 0.67251122, + "learning_rate": 3.904698300730537e-06, + "loss": 0.69392443, + "num_input_tokens_seen": 123133345, + "router_z_loss_clip": 0.66113281, + "router_z_loss_mlp": 0.20983887, + "step": 4332, + "time_per_iteration": 2.4205806255340576 + }, + { + "auxiliary_loss_clip": 0.01097315, + "auxiliary_loss_mlp": 0.01048012, + "balance_loss_clip": 1.03129804, + "balance_loss_mlp": 1.02883148, + "epoch": 0.12573269108003018, + "flos": 12378176052480.0, + "grad_norm": 2.5930884729962944, + "language_loss": 0.7830863, + "learning_rate": 3.904640962107701e-06, + "loss": 0.80453962, + "num_input_tokens_seen": 123146445, + "router_z_loss_clip": 0.66015625, + "router_z_loss_mlp": 0.19177246, + "step": 4333, + "time_per_iteration": 2.343015670776367 + }, + { + "auxiliary_loss_clip": 0.01093963, + "auxiliary_loss_mlp": 0.01043123, + "balance_loss_clip": 1.02985096, + "balance_loss_mlp": 1.02416849, + "epoch": 0.12576170854854624, + "flos": 31897662019200.0, + "grad_norm": 3.372753888274729, + "language_loss": 0.82839519, + "learning_rate": 3.904583606662306e-06, + "loss": 0.84976602, + "num_input_tokens_seen": 123163500, + "router_z_loss_clip": 0.64111328, + "router_z_loss_mlp": 0.1895752, + "step": 4334, + "time_per_iteration": 2.477788209915161 + }, + { + "auxiliary_loss_clip": 0.01093825, + "auxiliary_loss_mlp": 0.01045325, + "balance_loss_clip": 1.03085375, + "balance_loss_mlp": 1.02489257, + "epoch": 0.12579072601706226, + "flos": 30255906245760.0, + "grad_norm": 2.301719108168513, + "language_loss": 0.89240527, + "learning_rate": 3.904526234394858e-06, + "loss": 0.91379678, + "num_input_tokens_seen": 123179145, + "router_z_loss_clip": 0.62939453, + "router_z_loss_mlp": 0.20410156, + "step": 4335, + "time_per_iteration": 2.4536263942718506 + }, + { + "auxiliary_loss_clip": 0.01012979, + "auxiliary_loss_mlp": 0.01007834, + "balance_loss_clip": 1.00275207, + "balance_loss_mlp": 1.00640333, + "epoch": 0.1258197434855783, + "flos": 63983503733760.0, + "grad_norm": 0.6701815867612464, + "language_loss": 0.5006249, + "learning_rate": 3.904468845305863e-06, + "loss": 0.52083302, + "num_input_tokens_seen": 123244260, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01428223, + "step": 4336, + "time_per_iteration": 3.212601900100708 + }, + { + "auxiliary_loss_clip": 0.01094961, + "auxiliary_loss_mlp": 0.01041434, + "balance_loss_clip": 1.03145504, + "balance_loss_mlp": 1.02093017, + "epoch": 0.12584876095409436, + "flos": 33795191479680.0, + "grad_norm": 2.2903202787517847, + "language_loss": 0.74368083, + "learning_rate": 3.904411439395829e-06, + "loss": 0.76504481, + "num_input_tokens_seen": 123261995, + "router_z_loss_clip": 0.63525391, + "router_z_loss_mlp": 0.20532227, + "step": 4337, + "time_per_iteration": 2.550039768218994 + }, + { + "auxiliary_loss_clip": 0.01013467, + "auxiliary_loss_mlp": 0.01001369, + "balance_loss_clip": 1.00366759, + "balance_loss_mlp": 0.99997419, + "epoch": 0.12587777842261041, + "flos": 74773342126080.0, + "grad_norm": 0.6285329595729536, + "language_loss": 0.44849867, + "learning_rate": 3.9043540166652625e-06, + "loss": 0.46864706, + "num_input_tokens_seen": 123329190, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.01397705, + "step": 4338, + "time_per_iteration": 3.0747671127319336 + }, + { + "auxiliary_loss_clip": 0.0110176, + "auxiliary_loss_mlp": 0.01042012, + "balance_loss_clip": 1.03406036, + "balance_loss_mlp": 1.01943386, + "epoch": 0.12590679589112647, + "flos": 23616263790720.0, + "grad_norm": 2.567034682841579, + "language_loss": 0.93459451, + "learning_rate": 3.90429657711467e-06, + "loss": 0.95603216, + "num_input_tokens_seen": 123343300, + "router_z_loss_clip": 0.67724609, + "router_z_loss_mlp": 0.22583008, + "step": 4339, + "time_per_iteration": 4.59054708480835 + }, + { + "auxiliary_loss_clip": 0.01014649, + "auxiliary_loss_mlp": 0.01004938, + "balance_loss_clip": 1.00450873, + "balance_loss_mlp": 1.00341237, + "epoch": 0.12593581335964252, + "flos": 57698893560960.0, + "grad_norm": 0.7022205761718657, + "language_loss": 0.49931383, + "learning_rate": 3.90423912074456e-06, + "loss": 0.51950973, + "num_input_tokens_seen": 123407145, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.01525879, + "step": 4340, + "time_per_iteration": 5.119565963745117 + }, + { + "auxiliary_loss_clip": 0.01099554, + "auxiliary_loss_mlp": 0.01047794, + "balance_loss_clip": 1.03360963, + "balance_loss_mlp": 1.02701616, + "epoch": 0.12596483082815854, + "flos": 19528039249920.0, + "grad_norm": 2.5586617121320265, + "language_loss": 1.01082563, + "learning_rate": 3.90418164755544e-06, + "loss": 1.03229916, + "num_input_tokens_seen": 123419395, + "router_z_loss_clip": 0.65917969, + "router_z_loss_mlp": 0.20788574, + "step": 4341, + "time_per_iteration": 2.4113144874572754 + }, + { + "auxiliary_loss_clip": 0.01095872, + "auxiliary_loss_mlp": 0.01034039, + "balance_loss_clip": 1.03208923, + "balance_loss_mlp": 1.01641989, + "epoch": 0.1259938482966746, + "flos": 26753349628800.0, + "grad_norm": 2.624971284701397, + "language_loss": 0.7393409, + "learning_rate": 3.904124157547817e-06, + "loss": 0.76064003, + "num_input_tokens_seen": 123433890, + "router_z_loss_clip": 0.63818359, + "router_z_loss_mlp": 0.17633057, + "step": 4342, + "time_per_iteration": 2.4215691089630127 + }, + { + "auxiliary_loss_clip": 0.01105464, + "auxiliary_loss_mlp": 0.01044599, + "balance_loss_clip": 1.03595328, + "balance_loss_mlp": 1.02223516, + "epoch": 0.12602286576519064, + "flos": 15842607598080.0, + "grad_norm": 2.5060114025987263, + "language_loss": 0.71519339, + "learning_rate": 3.9040666507221985e-06, + "loss": 0.73669404, + "num_input_tokens_seen": 123446425, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.22387695, + "step": 4343, + "time_per_iteration": 2.368110179901123 + }, + { + "auxiliary_loss_clip": 0.01013528, + "auxiliary_loss_mlp": 0.0100142, + "balance_loss_clip": 1.00306356, + "balance_loss_mlp": 1.00003123, + "epoch": 0.1260518832337067, + "flos": 55219606248960.0, + "grad_norm": 0.7157343149204622, + "language_loss": 0.52314413, + "learning_rate": 3.904009127079093e-06, + "loss": 0.5432936, + "num_input_tokens_seen": 123509040, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01391602, + "step": 4344, + "time_per_iteration": 2.965240716934204 + }, + { + "auxiliary_loss_clip": 0.01103592, + "auxiliary_loss_mlp": 0.01046563, + "balance_loss_clip": 1.032722, + "balance_loss_mlp": 1.0243305, + "epoch": 0.12608090070222275, + "flos": 34195959509760.0, + "grad_norm": 2.2855625070438794, + "language_loss": 1.03360856, + "learning_rate": 3.903951586619009e-06, + "loss": 1.0551101, + "num_input_tokens_seen": 123525075, + "router_z_loss_clip": 0.70849609, + "router_z_loss_mlp": 0.22253418, + "step": 4345, + "time_per_iteration": 2.6063742637634277 + }, + { + "auxiliary_loss_clip": 0.01101943, + "auxiliary_loss_mlp": 0.01037492, + "balance_loss_clip": 1.03432977, + "balance_loss_mlp": 1.01882386, + "epoch": 0.12610991817073877, + "flos": 46965023677440.0, + "grad_norm": 1.905670066732293, + "language_loss": 0.80892617, + "learning_rate": 3.903894029342453e-06, + "loss": 0.83032054, + "num_input_tokens_seen": 123545165, + "router_z_loss_clip": 0.67675781, + "router_z_loss_mlp": 0.18652344, + "step": 4346, + "time_per_iteration": 5.212854385375977 + }, + { + "auxiliary_loss_clip": 0.01012151, + "auxiliary_loss_mlp": 0.01001158, + "balance_loss_clip": 1.00213146, + "balance_loss_mlp": 0.99980539, + "epoch": 0.12613893563925482, + "flos": 64145397041280.0, + "grad_norm": 0.7211979718992851, + "language_loss": 0.50285274, + "learning_rate": 3.903836455249935e-06, + "loss": 0.52298588, + "num_input_tokens_seen": 123601105, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.0135498, + "step": 4347, + "time_per_iteration": 2.857924461364746 + }, + { + "auxiliary_loss_clip": 0.01011963, + "auxiliary_loss_mlp": 0.01001652, + "balance_loss_clip": 1.00208879, + "balance_loss_mlp": 1.00026917, + "epoch": 0.12616795310777087, + "flos": 74768838560640.0, + "grad_norm": 0.7179374590617676, + "language_loss": 0.48638329, + "learning_rate": 3.9037788643419635e-06, + "loss": 0.50651944, + "num_input_tokens_seen": 123654925, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01385498, + "step": 4348, + "time_per_iteration": 5.4704272747039795 + }, + { + "auxiliary_loss_clip": 0.01011978, + "auxiliary_loss_mlp": 0.01001337, + "balance_loss_clip": 1.00200653, + "balance_loss_mlp": 1.00003731, + "epoch": 0.12619697057628693, + "flos": 62402532370560.0, + "grad_norm": 0.7143697669307489, + "language_loss": 0.53032827, + "learning_rate": 3.903721256619046e-06, + "loss": 0.55046141, + "num_input_tokens_seen": 123716735, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01300049, + "step": 4349, + "time_per_iteration": 2.999443292617798 + }, + { + "auxiliary_loss_clip": 0.01091933, + "auxiliary_loss_mlp": 0.0104061, + "balance_loss_clip": 1.03047609, + "balance_loss_mlp": 1.02255595, + "epoch": 0.12622598804480298, + "flos": 11868548803200.0, + "grad_norm": 2.563468651693003, + "language_loss": 0.84290797, + "learning_rate": 3.903663632081693e-06, + "loss": 0.86423343, + "num_input_tokens_seen": 123729050, + "router_z_loss_clip": 0.61474609, + "router_z_loss_mlp": 0.18054199, + "step": 4350, + "time_per_iteration": 2.3336877822875977 + }, + { + "auxiliary_loss_clip": 0.01097816, + "auxiliary_loss_mlp": 0.0105344, + "balance_loss_clip": 1.03310013, + "balance_loss_mlp": 1.03486681, + "epoch": 0.12625500551331903, + "flos": 19273627105920.0, + "grad_norm": 2.1457636470787587, + "language_loss": 0.57357848, + "learning_rate": 3.903605990730411e-06, + "loss": 0.5950911, + "num_input_tokens_seen": 123743560, + "router_z_loss_clip": 0.64746094, + "router_z_loss_mlp": 0.18572998, + "step": 4351, + "time_per_iteration": 2.3625032901763916 + }, + { + "auxiliary_loss_clip": 0.01012509, + "auxiliary_loss_mlp": 0.01004635, + "balance_loss_clip": 1.0028069, + "balance_loss_mlp": 1.0033952, + "epoch": 0.12628402298183505, + "flos": 70001412963840.0, + "grad_norm": 0.6284655801995269, + "language_loss": 0.51007736, + "learning_rate": 3.903548332565712e-06, + "loss": 0.53024876, + "num_input_tokens_seen": 123807385, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.01239014, + "step": 4352, + "time_per_iteration": 3.1643941402435303 + }, + { + "auxiliary_loss_clip": 0.01101862, + "auxiliary_loss_mlp": 0.01044435, + "balance_loss_clip": 1.03352571, + "balance_loss_mlp": 1.02345467, + "epoch": 0.1263130404503511, + "flos": 24926938341120.0, + "grad_norm": 3.105556684030964, + "language_loss": 0.97100949, + "learning_rate": 3.903490657588103e-06, + "loss": 0.99247241, + "num_input_tokens_seen": 123821490, + "router_z_loss_clip": 0.68359375, + "router_z_loss_mlp": 0.20996094, + "step": 4353, + "time_per_iteration": 2.7272863388061523 + }, + { + "auxiliary_loss_clip": 0.01093722, + "auxiliary_loss_mlp": 0.01040531, + "balance_loss_clip": 1.0305059, + "balance_loss_mlp": 1.02179134, + "epoch": 0.12634205791886716, + "flos": 28869854336640.0, + "grad_norm": 2.1195980056530095, + "language_loss": 0.82111526, + "learning_rate": 3.9034329657980946e-06, + "loss": 0.84245783, + "num_input_tokens_seen": 123836975, + "router_z_loss_clip": 0.63183594, + "router_z_loss_mlp": 0.18737793, + "step": 4354, + "time_per_iteration": 2.559799909591675 + }, + { + "auxiliary_loss_clip": 0.01100975, + "auxiliary_loss_mlp": 0.01046638, + "balance_loss_clip": 1.03286207, + "balance_loss_mlp": 1.02640796, + "epoch": 0.1263710753873832, + "flos": 35186794295040.0, + "grad_norm": 2.321574147310663, + "language_loss": 0.85492247, + "learning_rate": 3.903375257196195e-06, + "loss": 0.87639856, + "num_input_tokens_seen": 123855600, + "router_z_loss_clip": 0.68164062, + "router_z_loss_mlp": 0.20214844, + "step": 4355, + "time_per_iteration": 2.7065138816833496 + }, + { + "auxiliary_loss_clip": 0.01101878, + "auxiliary_loss_mlp": 0.01048277, + "balance_loss_clip": 1.0334394, + "balance_loss_mlp": 1.0282259, + "epoch": 0.12640009285589926, + "flos": 37665732493440.0, + "grad_norm": 1.7009647918072148, + "language_loss": 0.9466958, + "learning_rate": 3.9033175317829165e-06, + "loss": 0.96819735, + "num_input_tokens_seen": 123879730, + "router_z_loss_clip": 0.68457031, + "router_z_loss_mlp": 0.20056152, + "step": 4356, + "time_per_iteration": 2.5938265323638916 + }, + { + "auxiliary_loss_clip": 0.01095594, + "auxiliary_loss_mlp": 0.01042204, + "balance_loss_clip": 1.03380752, + "balance_loss_mlp": 1.02401245, + "epoch": 0.1264291103244153, + "flos": 25075179440640.0, + "grad_norm": 2.318443659147682, + "language_loss": 0.79065633, + "learning_rate": 3.9032597895587666e-06, + "loss": 0.81203437, + "num_input_tokens_seen": 123894475, + "router_z_loss_clip": 0.61767578, + "router_z_loss_mlp": 0.18170166, + "step": 4357, + "time_per_iteration": 2.400771141052246 + }, + { + "auxiliary_loss_clip": 0.01097617, + "auxiliary_loss_mlp": 0.01042362, + "balance_loss_clip": 1.03371072, + "balance_loss_mlp": 1.02203703, + "epoch": 0.12645812779293134, + "flos": 11757141054720.0, + "grad_norm": 2.784118210241092, + "language_loss": 0.91201961, + "learning_rate": 3.903202030524256e-06, + "loss": 0.93341935, + "num_input_tokens_seen": 123905720, + "router_z_loss_clip": 0.63916016, + "router_z_loss_mlp": 0.20300293, + "step": 4358, + "time_per_iteration": 2.4470553398132324 + }, + { + "auxiliary_loss_clip": 0.01016376, + "auxiliary_loss_mlp": 0.01004382, + "balance_loss_clip": 1.00630236, + "balance_loss_mlp": 1.0032016, + "epoch": 0.1264871452614474, + "flos": 63719770256640.0, + "grad_norm": 0.7324333241582586, + "language_loss": 0.4816193, + "learning_rate": 3.903144254679895e-06, + "loss": 0.50182688, + "num_input_tokens_seen": 123968350, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.01177979, + "step": 4359, + "time_per_iteration": 2.989309310913086 + }, + { + "auxiliary_loss_clip": 0.01102329, + "auxiliary_loss_mlp": 0.01035601, + "balance_loss_clip": 1.03382099, + "balance_loss_mlp": 1.0140475, + "epoch": 0.12651616272996344, + "flos": 23870187175680.0, + "grad_norm": 2.3112968464983368, + "language_loss": 0.87169778, + "learning_rate": 3.903086462026194e-06, + "loss": 0.89307708, + "num_input_tokens_seen": 123984265, + "router_z_loss_clip": 0.68554688, + "router_z_loss_mlp": 0.21539307, + "step": 4360, + "time_per_iteration": 2.565140724182129 + }, + { + "auxiliary_loss_clip": 0.0101795, + "auxiliary_loss_mlp": 0.01002443, + "balance_loss_clip": 1.00768924, + "balance_loss_mlp": 1.0012573, + "epoch": 0.1265451801984795, + "flos": 74791216742400.0, + "grad_norm": 0.6103931775446889, + "language_loss": 0.44196481, + "learning_rate": 3.903028652563663e-06, + "loss": 0.46216872, + "num_input_tokens_seen": 124050510, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01184082, + "step": 4361, + "time_per_iteration": 3.242807388305664 + }, + { + "auxiliary_loss_clip": 0.01105817, + "auxiliary_loss_mlp": 0.01050041, + "balance_loss_clip": 1.03275084, + "balance_loss_mlp": 1.02755785, + "epoch": 0.12657419766699554, + "flos": 27848854270080.0, + "grad_norm": 2.003068330743857, + "language_loss": 0.82601869, + "learning_rate": 3.902970826292814e-06, + "loss": 0.84757721, + "num_input_tokens_seen": 124067260, + "router_z_loss_clip": 0.73046875, + "router_z_loss_mlp": 0.22485352, + "step": 4362, + "time_per_iteration": 2.5262606143951416 + }, + { + "auxiliary_loss_clip": 0.01109933, + "auxiliary_loss_mlp": 0.01045024, + "balance_loss_clip": 1.03729296, + "balance_loss_mlp": 1.0221951, + "epoch": 0.12660321513551157, + "flos": 31749560565120.0, + "grad_norm": 2.61223478557369, + "language_loss": 1.00628233, + "learning_rate": 3.902912983214155e-06, + "loss": 1.02783191, + "num_input_tokens_seen": 124086600, + "router_z_loss_clip": 0.72509766, + "router_z_loss_mlp": 0.22827148, + "step": 4363, + "time_per_iteration": 2.519972801208496 + }, + { + "auxiliary_loss_clip": 0.01020002, + "auxiliary_loss_mlp": 0.01002527, + "balance_loss_clip": 1.0094049, + "balance_loss_mlp": 1.00153208, + "epoch": 0.12663223260402762, + "flos": 65798568829440.0, + "grad_norm": 0.7032343774999478, + "language_loss": 0.5314489, + "learning_rate": 3.9028551233281985e-06, + "loss": 0.55167419, + "num_input_tokens_seen": 124150145, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.00994873, + "step": 4364, + "time_per_iteration": 3.10261869430542 + }, + { + "auxiliary_loss_clip": 0.0110029, + "auxiliary_loss_mlp": 0.01046979, + "balance_loss_clip": 1.03515244, + "balance_loss_mlp": 1.02720213, + "epoch": 0.12666125007254367, + "flos": 22265125107840.0, + "grad_norm": 2.2960927428285025, + "language_loss": 0.91231561, + "learning_rate": 3.9027972466354565e-06, + "loss": 0.9337883, + "num_input_tokens_seen": 124165865, + "router_z_loss_clip": 0.65087891, + "router_z_loss_mlp": 0.19775391, + "step": 4365, + "time_per_iteration": 2.4870169162750244 + }, + { + "auxiliary_loss_clip": 0.01091914, + "auxiliary_loss_mlp": 0.01035165, + "balance_loss_clip": 1.03114057, + "balance_loss_mlp": 1.01568592, + "epoch": 0.12669026754105972, + "flos": 18799820778240.0, + "grad_norm": 1.7819963366335476, + "language_loss": 0.63206601, + "learning_rate": 3.902739353136439e-06, + "loss": 0.65333682, + "num_input_tokens_seen": 124181365, + "router_z_loss_clip": 0.60791016, + "router_z_loss_mlp": 0.19470215, + "step": 4366, + "time_per_iteration": 2.3881425857543945 + }, + { + "auxiliary_loss_clip": 0.01094622, + "auxiliary_loss_mlp": 0.0103524, + "balance_loss_clip": 1.03227973, + "balance_loss_mlp": 1.01665545, + "epoch": 0.12671928500957577, + "flos": 16135109902080.0, + "grad_norm": 2.7046891782889713, + "language_loss": 0.99741328, + "learning_rate": 3.902681442831658e-06, + "loss": 1.01871192, + "num_input_tokens_seen": 124193845, + "router_z_loss_clip": 0.62255859, + "router_z_loss_mlp": 0.18603516, + "step": 4367, + "time_per_iteration": 2.4457411766052246 + }, + { + "auxiliary_loss_clip": 0.01096634, + "auxiliary_loss_mlp": 0.01041173, + "balance_loss_clip": 1.0324986, + "balance_loss_mlp": 1.02112174, + "epoch": 0.12674830247809182, + "flos": 27082824929280.0, + "grad_norm": 2.70420342183033, + "language_loss": 0.90132248, + "learning_rate": 3.902623515721623e-06, + "loss": 0.92270046, + "num_input_tokens_seen": 124210385, + "router_z_loss_clip": 0.640625, + "router_z_loss_mlp": 0.20056152, + "step": 4368, + "time_per_iteration": 2.5991780757904053 + }, + { + "auxiliary_loss_clip": 0.01104066, + "auxiliary_loss_mlp": 0.01048812, + "balance_loss_clip": 1.03223896, + "balance_loss_mlp": 1.02550638, + "epoch": 0.12677731994660785, + "flos": 16317705646080.0, + "grad_norm": 3.264230578742761, + "language_loss": 0.90050906, + "learning_rate": 3.902565571806849e-06, + "loss": 0.92203784, + "num_input_tokens_seen": 124222565, + "router_z_loss_clip": 0.71728516, + "router_z_loss_mlp": 0.23303223, + "step": 4369, + "time_per_iteration": 2.4368982315063477 + }, + { + "auxiliary_loss_clip": 0.01015654, + "auxiliary_loss_mlp": 0.01005985, + "balance_loss_clip": 1.00571918, + "balance_loss_mlp": 1.00475168, + "epoch": 0.1268063374151239, + "flos": 63864694776960.0, + "grad_norm": 0.6524946532314484, + "language_loss": 0.48711234, + "learning_rate": 3.902507611087845e-06, + "loss": 0.50732875, + "num_input_tokens_seen": 124283845, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.0123291, + "step": 4370, + "time_per_iteration": 2.9935035705566406 + }, + { + "auxiliary_loss_clip": 0.01093443, + "auxiliary_loss_mlp": 0.01044893, + "balance_loss_clip": 1.030316, + "balance_loss_mlp": 1.02364969, + "epoch": 0.12683535488363995, + "flos": 14533608792960.0, + "grad_norm": 2.6164608567381755, + "language_loss": 0.88032782, + "learning_rate": 3.902449633565124e-06, + "loss": 0.90171111, + "num_input_tokens_seen": 124296535, + "router_z_loss_clip": 0.63134766, + "router_z_loss_mlp": 0.21240234, + "step": 4371, + "time_per_iteration": 2.4169695377349854 + }, + { + "auxiliary_loss_clip": 0.01092659, + "auxiliary_loss_mlp": 0.01045572, + "balance_loss_clip": 1.02983665, + "balance_loss_mlp": 1.02664113, + "epoch": 0.126864372352156, + "flos": 24965133235200.0, + "grad_norm": 2.1427706712155574, + "language_loss": 0.8477, + "learning_rate": 3.902391639239199e-06, + "loss": 0.86908233, + "num_input_tokens_seen": 124311450, + "router_z_loss_clip": 0.62768555, + "router_z_loss_mlp": 0.18945312, + "step": 4372, + "time_per_iteration": 2.4542438983917236 + }, + { + "auxiliary_loss_clip": 0.01013631, + "auxiliary_loss_mlp": 0.01002828, + "balance_loss_clip": 1.00378084, + "balance_loss_mlp": 1.00181508, + "epoch": 0.12689338982067205, + "flos": 68201815466880.0, + "grad_norm": 0.6990232953171562, + "language_loss": 0.52976137, + "learning_rate": 3.90233362811058e-06, + "loss": 0.54992598, + "num_input_tokens_seen": 124372815, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01013184, + "step": 4373, + "time_per_iteration": 3.0623779296875 + }, + { + "auxiliary_loss_clip": 0.01093376, + "auxiliary_loss_mlp": 0.01037956, + "balance_loss_clip": 1.02891231, + "balance_loss_mlp": 1.01798892, + "epoch": 0.1269224072891881, + "flos": 30948897288960.0, + "grad_norm": 1.9860828249098883, + "language_loss": 0.90224397, + "learning_rate": 3.9022756001797805e-06, + "loss": 0.92355728, + "num_input_tokens_seen": 124391845, + "router_z_loss_clip": 0.64404297, + "router_z_loss_mlp": 0.1998291, + "step": 4374, + "time_per_iteration": 2.5268404483795166 + }, + { + "auxiliary_loss_clip": 0.01092389, + "auxiliary_loss_mlp": 0.01035579, + "balance_loss_clip": 1.03008986, + "balance_loss_mlp": 1.01596916, + "epoch": 0.12695142475770413, + "flos": 22156580090880.0, + "grad_norm": 2.5808287365438267, + "language_loss": 0.83635592, + "learning_rate": 3.902217555447314e-06, + "loss": 0.85763562, + "num_input_tokens_seen": 124405435, + "router_z_loss_clip": 0.62255859, + "router_z_loss_mlp": 0.19604492, + "step": 4375, + "time_per_iteration": 2.4451825618743896 + }, + { + "auxiliary_loss_clip": 0.01099377, + "auxiliary_loss_mlp": 0.01043095, + "balance_loss_clip": 1.03377151, + "balance_loss_mlp": 1.02225685, + "epoch": 0.12698044222622018, + "flos": 52365039932160.0, + "grad_norm": 2.253416580018117, + "language_loss": 0.80265272, + "learning_rate": 3.902159493913692e-06, + "loss": 0.82407743, + "num_input_tokens_seen": 124426975, + "router_z_loss_clip": 0.65625, + "router_z_loss_mlp": 0.20861816, + "step": 4376, + "time_per_iteration": 2.6450181007385254 + }, + { + "auxiliary_loss_clip": 0.01107781, + "auxiliary_loss_mlp": 0.01045667, + "balance_loss_clip": 1.03504038, + "balance_loss_mlp": 1.0221231, + "epoch": 0.12700945969473623, + "flos": 36932451874560.0, + "grad_norm": 1.8225999113707654, + "language_loss": 0.84933329, + "learning_rate": 3.902101415579427e-06, + "loss": 0.87086779, + "num_input_tokens_seen": 124451650, + "router_z_loss_clip": 0.72802734, + "router_z_loss_mlp": 0.23547363, + "step": 4377, + "time_per_iteration": 2.6647677421569824 + }, + { + "auxiliary_loss_clip": 0.01098567, + "auxiliary_loss_mlp": 0.01042092, + "balance_loss_clip": 1.0335772, + "balance_loss_mlp": 1.0228399, + "epoch": 0.12703847716325228, + "flos": 15515750649600.0, + "grad_norm": 2.1592710339825727, + "language_loss": 0.74051005, + "learning_rate": 3.902043320445033e-06, + "loss": 0.76191664, + "num_input_tokens_seen": 124466675, + "router_z_loss_clip": 0.64941406, + "router_z_loss_mlp": 0.19226074, + "step": 4378, + "time_per_iteration": 2.3477792739868164 + }, + { + "auxiliary_loss_clip": 0.01092061, + "auxiliary_loss_mlp": 0.01041136, + "balance_loss_clip": 1.02978551, + "balance_loss_mlp": 1.02113914, + "epoch": 0.12706749463176834, + "flos": 24900892600320.0, + "grad_norm": 2.324121132240665, + "language_loss": 0.77420431, + "learning_rate": 3.901985208511023e-06, + "loss": 0.79553628, + "num_input_tokens_seen": 124486990, + "router_z_loss_clip": 0.62304688, + "router_z_loss_mlp": 0.19976807, + "step": 4379, + "time_per_iteration": 2.4606733322143555 + }, + { + "auxiliary_loss_clip": 0.01092787, + "auxiliary_loss_mlp": 0.01041732, + "balance_loss_clip": 1.03078389, + "balance_loss_mlp": 1.02190113, + "epoch": 0.12709651210028436, + "flos": 31204042571520.0, + "grad_norm": 3.105141854105697, + "language_loss": 0.91839314, + "learning_rate": 3.90192707977791e-06, + "loss": 0.93973827, + "num_input_tokens_seen": 124501870, + "router_z_loss_clip": 0.61962891, + "router_z_loss_mlp": 0.19836426, + "step": 4380, + "time_per_iteration": 2.4772889614105225 + }, + { + "auxiliary_loss_clip": 0.01013371, + "auxiliary_loss_mlp": 0.01003547, + "balance_loss_clip": 1.00323129, + "balance_loss_mlp": 1.00243795, + "epoch": 0.1271255295688004, + "flos": 60074383800960.0, + "grad_norm": 0.6912881271290572, + "language_loss": 0.54766876, + "learning_rate": 3.901868934246208e-06, + "loss": 0.56783789, + "num_input_tokens_seen": 124562555, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.0111084, + "step": 4381, + "time_per_iteration": 3.090219020843506 + }, + { + "auxiliary_loss_clip": 0.01092007, + "auxiliary_loss_mlp": 0.01040567, + "balance_loss_clip": 1.03031898, + "balance_loss_mlp": 1.02121925, + "epoch": 0.12715454703731646, + "flos": 26351569169280.0, + "grad_norm": 2.7537009557319903, + "language_loss": 0.90747911, + "learning_rate": 3.9018107719164285e-06, + "loss": 0.92880487, + "num_input_tokens_seen": 124577830, + "router_z_loss_clip": 0.61621094, + "router_z_loss_mlp": 0.19335938, + "step": 4382, + "time_per_iteration": 2.538644552230835 + }, + { + "auxiliary_loss_clip": 0.01012995, + "auxiliary_loss_mlp": 0.01005866, + "balance_loss_clip": 1.00285411, + "balance_loss_mlp": 1.00485253, + "epoch": 0.1271835645058325, + "flos": 57776435424000.0, + "grad_norm": 0.6792391817320612, + "language_loss": 0.52226043, + "learning_rate": 3.901752592789088e-06, + "loss": 0.54244906, + "num_input_tokens_seen": 124639895, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.01013184, + "step": 4383, + "time_per_iteration": 3.1200942993164062 + }, + { + "auxiliary_loss_clip": 0.0101207, + "auxiliary_loss_mlp": 0.01001582, + "balance_loss_clip": 1.00194478, + "balance_loss_mlp": 1.00037849, + "epoch": 0.12721258197434857, + "flos": 57503310791040.0, + "grad_norm": 0.7461659816545194, + "language_loss": 0.46690339, + "learning_rate": 3.901694396864698e-06, + "loss": 0.48703989, + "num_input_tokens_seen": 124689070, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.01202393, + "step": 4384, + "time_per_iteration": 2.7807459831237793 + }, + { + "auxiliary_loss_clip": 0.0109813, + "auxiliary_loss_mlp": 0.0105023, + "balance_loss_clip": 1.03235614, + "balance_loss_mlp": 1.02749646, + "epoch": 0.12724159944286462, + "flos": 23107823527680.0, + "grad_norm": 1.9855965599785816, + "language_loss": 0.68147129, + "learning_rate": 3.901636184143774e-06, + "loss": 0.70295489, + "num_input_tokens_seen": 124703560, + "router_z_loss_clip": 0.65917969, + "router_z_loss_mlp": 0.22717285, + "step": 4385, + "time_per_iteration": 2.5495429039001465 + }, + { + "auxiliary_loss_clip": 0.01099973, + "auxiliary_loss_mlp": 0.01035736, + "balance_loss_clip": 1.03346539, + "balance_loss_mlp": 1.01651382, + "epoch": 0.12727061691138064, + "flos": 32901763121280.0, + "grad_norm": 2.32961688282291, + "language_loss": 0.92058933, + "learning_rate": 3.901577954626829e-06, + "loss": 0.94194639, + "num_input_tokens_seen": 124731940, + "router_z_loss_clip": 0.6640625, + "router_z_loss_mlp": 0.1920166, + "step": 4386, + "time_per_iteration": 2.756829023361206 + }, + { + "auxiliary_loss_clip": 0.01090293, + "auxiliary_loss_mlp": 0.01046788, + "balance_loss_clip": 1.02956688, + "balance_loss_mlp": 1.02771461, + "epoch": 0.1272996343798967, + "flos": 19783324177920.0, + "grad_norm": 2.356818114623098, + "language_loss": 0.92580092, + "learning_rate": 3.901519708314379e-06, + "loss": 0.94717175, + "num_input_tokens_seen": 124745735, + "router_z_loss_clip": 0.60717773, + "router_z_loss_mlp": 0.1907959, + "step": 4387, + "time_per_iteration": 2.372591018676758 + }, + { + "auxiliary_loss_clip": 0.01095671, + "auxiliary_loss_mlp": 0.01040141, + "balance_loss_clip": 1.03118873, + "balance_loss_mlp": 1.01994753, + "epoch": 0.12732865184841274, + "flos": 29088236090880.0, + "grad_norm": 3.169310256836186, + "language_loss": 0.92250121, + "learning_rate": 3.901461445206937e-06, + "loss": 0.94385934, + "num_input_tokens_seen": 124761225, + "router_z_loss_clip": 0.64501953, + "router_z_loss_mlp": 0.2019043, + "step": 4388, + "time_per_iteration": 2.463315725326538 + }, + { + "auxiliary_loss_clip": 0.01089958, + "auxiliary_loss_mlp": 0.01039537, + "balance_loss_clip": 1.02816701, + "balance_loss_mlp": 1.02131569, + "epoch": 0.1273576693169288, + "flos": 18726188987520.0, + "grad_norm": 3.372420152405163, + "language_loss": 1.09168744, + "learning_rate": 3.901403165305018e-06, + "loss": 1.11298227, + "num_input_tokens_seen": 124773850, + "router_z_loss_clip": 0.6184082, + "router_z_loss_mlp": 0.18218994, + "step": 4389, + "time_per_iteration": 2.36201548576355 + }, + { + "auxiliary_loss_clip": 0.01097552, + "auxiliary_loss_mlp": 0.01047768, + "balance_loss_clip": 1.03215003, + "balance_loss_mlp": 1.02759767, + "epoch": 0.12738668678544485, + "flos": 26974978139520.0, + "grad_norm": 2.3329013563291805, + "language_loss": 0.90168881, + "learning_rate": 3.901344868609138e-06, + "loss": 0.92314208, + "num_input_tokens_seen": 124792030, + "router_z_loss_clip": 0.65380859, + "router_z_loss_mlp": 0.20172119, + "step": 4390, + "time_per_iteration": 2.520395278930664 + }, + { + "auxiliary_loss_clip": 0.01087557, + "auxiliary_loss_mlp": 0.01037281, + "balance_loss_clip": 1.02892184, + "balance_loss_mlp": 1.01952493, + "epoch": 0.12741570425396087, + "flos": 22155812040960.0, + "grad_norm": 2.7078774996139194, + "language_loss": 0.76074016, + "learning_rate": 3.90128655511981e-06, + "loss": 0.78198856, + "num_input_tokens_seen": 124804840, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.1776123, + "step": 4391, + "time_per_iteration": 2.404634714126587 + }, + { + "auxiliary_loss_clip": 0.01097452, + "auxiliary_loss_mlp": 0.01041931, + "balance_loss_clip": 1.03032804, + "balance_loss_mlp": 1.02115262, + "epoch": 0.12744472172247692, + "flos": 11318352687360.0, + "grad_norm": 3.3241207847125365, + "language_loss": 0.93803239, + "learning_rate": 3.901228224837549e-06, + "loss": 0.95942622, + "num_input_tokens_seen": 124814995, + "router_z_loss_clip": 0.67138672, + "router_z_loss_mlp": 0.2076416, + "step": 4392, + "time_per_iteration": 2.359025239944458 + }, + { + "auxiliary_loss_clip": 0.01013896, + "auxiliary_loss_mlp": 0.01001274, + "balance_loss_clip": 1.00384128, + "balance_loss_mlp": 0.99998075, + "epoch": 0.12747373919099297, + "flos": 64199616249600.0, + "grad_norm": 0.6874349496625387, + "language_loss": 0.53462279, + "learning_rate": 3.901169877762872e-06, + "loss": 0.55477452, + "num_input_tokens_seen": 124876650, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.01293945, + "step": 4393, + "time_per_iteration": 3.0397846698760986 + }, + { + "auxiliary_loss_clip": 0.01093877, + "auxiliary_loss_mlp": 0.01035553, + "balance_loss_clip": 1.03160417, + "balance_loss_mlp": 1.01755285, + "epoch": 0.12750275665950903, + "flos": 25038555557760.0, + "grad_norm": 3.6835721231952196, + "language_loss": 0.79512143, + "learning_rate": 3.9011115138962925e-06, + "loss": 0.81641579, + "num_input_tokens_seen": 124890835, + "router_z_loss_clip": 0.62280273, + "router_z_loss_mlp": 0.18005371, + "step": 4394, + "time_per_iteration": 2.3626487255096436 + }, + { + "auxiliary_loss_clip": 0.0101366, + "auxiliary_loss_mlp": 0.01001725, + "balance_loss_clip": 1.00366068, + "balance_loss_mlp": 1.00053263, + "epoch": 0.12753177412802508, + "flos": 61314812962560.0, + "grad_norm": 0.6889329972509616, + "language_loss": 0.51732647, + "learning_rate": 3.901053133238327e-06, + "loss": 0.53748035, + "num_input_tokens_seen": 124951770, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.01190186, + "step": 4395, + "time_per_iteration": 3.1454086303710938 + }, + { + "auxiliary_loss_clip": 0.01087949, + "auxiliary_loss_mlp": 0.01045269, + "balance_loss_clip": 1.02920723, + "balance_loss_mlp": 1.02615929, + "epoch": 0.12756079159654113, + "flos": 25549614172800.0, + "grad_norm": 3.4088798945594765, + "language_loss": 0.67446685, + "learning_rate": 3.900994735789491e-06, + "loss": 0.69579899, + "num_input_tokens_seen": 124967740, + "router_z_loss_clip": 0.58642578, + "router_z_loss_mlp": 0.19110107, + "step": 4396, + "time_per_iteration": 2.477802038192749 + }, + { + "auxiliary_loss_clip": 0.01096772, + "auxiliary_loss_mlp": 0.01047307, + "balance_loss_clip": 1.02937174, + "balance_loss_mlp": 1.02434754, + "epoch": 0.12758980906505715, + "flos": 9970181470080.0, + "grad_norm": 5.278866632449552, + "language_loss": 1.02664399, + "learning_rate": 3.9009363215503005e-06, + "loss": 1.04808486, + "num_input_tokens_seen": 124977200, + "router_z_loss_clip": 0.67382812, + "router_z_loss_mlp": 0.22949219, + "step": 4397, + "time_per_iteration": 2.3231375217437744 + }, + { + "auxiliary_loss_clip": 0.0109447, + "auxiliary_loss_mlp": 0.01042451, + "balance_loss_clip": 1.02964807, + "balance_loss_mlp": 1.02293634, + "epoch": 0.1276188265335732, + "flos": 28649308078080.0, + "grad_norm": 1.8991222946513244, + "language_loss": 0.81567711, + "learning_rate": 3.900877890521271e-06, + "loss": 0.83704638, + "num_input_tokens_seen": 124991355, + "router_z_loss_clip": 0.64794922, + "router_z_loss_mlp": 0.19506836, + "step": 4398, + "time_per_iteration": 2.4260685443878174 + }, + { + "auxiliary_loss_clip": 0.01011799, + "auxiliary_loss_mlp": 0.01005439, + "balance_loss_clip": 1.00203872, + "balance_loss_mlp": 1.00428236, + "epoch": 0.12764784400208926, + "flos": 60457102657920.0, + "grad_norm": 0.718690243680847, + "language_loss": 0.48795134, + "learning_rate": 3.90081944270292e-06, + "loss": 0.50812376, + "num_input_tokens_seen": 125045630, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.01153564, + "step": 4399, + "time_per_iteration": 2.8700218200683594 + }, + { + "auxiliary_loss_clip": 0.01011587, + "auxiliary_loss_mlp": 0.01003811, + "balance_loss_clip": 1.00186539, + "balance_loss_mlp": 1.00266111, + "epoch": 0.1276768614706053, + "flos": 49881910504320.0, + "grad_norm": 0.6927735614410697, + "language_loss": 0.49650258, + "learning_rate": 3.900760978095761e-06, + "loss": 0.51665658, + "num_input_tokens_seen": 125105310, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.01147461, + "step": 4400, + "time_per_iteration": 3.078169345855713 + }, + { + "auxiliary_loss_clip": 0.0101112, + "auxiliary_loss_mlp": 0.01003957, + "balance_loss_clip": 1.00154901, + "balance_loss_mlp": 1.00282407, + "epoch": 0.12770587893912136, + "flos": 64445335464960.0, + "grad_norm": 0.6556329862298955, + "language_loss": 0.4781422, + "learning_rate": 3.900702496700312e-06, + "loss": 0.49829298, + "num_input_tokens_seen": 125161330, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01135254, + "step": 4401, + "time_per_iteration": 2.9058940410614014 + }, + { + "auxiliary_loss_clip": 0.01096851, + "auxiliary_loss_mlp": 0.01044776, + "balance_loss_clip": 1.03115308, + "balance_loss_mlp": 1.02436781, + "epoch": 0.1277348964076374, + "flos": 23871374161920.0, + "grad_norm": 2.0771411458793456, + "language_loss": 0.77098399, + "learning_rate": 3.90064399851709e-06, + "loss": 0.79240024, + "num_input_tokens_seen": 125178490, + "router_z_loss_clip": 0.65722656, + "router_z_loss_mlp": 0.20410156, + "step": 4402, + "time_per_iteration": 2.4193661212921143 + }, + { + "auxiliary_loss_clip": 0.01090018, + "auxiliary_loss_mlp": 0.0103552, + "balance_loss_clip": 1.02656376, + "balance_loss_mlp": 1.01655364, + "epoch": 0.12776391387615343, + "flos": 20441995488000.0, + "grad_norm": 2.4918392279775365, + "language_loss": 0.84436977, + "learning_rate": 3.90058548354661e-06, + "loss": 0.86562514, + "num_input_tokens_seen": 125194175, + "router_z_loss_clip": 0.63476562, + "router_z_loss_mlp": 0.18969727, + "step": 4403, + "time_per_iteration": 2.3952338695526123 + }, + { + "auxiliary_loss_clip": 0.01085476, + "auxiliary_loss_mlp": 0.01040632, + "balance_loss_clip": 1.02847672, + "balance_loss_mlp": 1.02341807, + "epoch": 0.12779293134466949, + "flos": 33393131729280.0, + "grad_norm": 2.081777021901478, + "language_loss": 0.80956972, + "learning_rate": 3.900526951789391e-06, + "loss": 0.83083075, + "num_input_tokens_seen": 125212290, + "router_z_loss_clip": 0.56982422, + "router_z_loss_mlp": 0.17224121, + "step": 4404, + "time_per_iteration": 2.5395925045013428 + }, + { + "auxiliary_loss_clip": 0.01100546, + "auxiliary_loss_mlp": 0.01041341, + "balance_loss_clip": 1.03129768, + "balance_loss_mlp": 1.02099204, + "epoch": 0.12782194881318554, + "flos": 13215358477440.0, + "grad_norm": 3.2367482991697374, + "language_loss": 0.78414667, + "learning_rate": 3.900468403245949e-06, + "loss": 0.80556554, + "num_input_tokens_seen": 125226525, + "router_z_loss_clip": 0.69262695, + "router_z_loss_mlp": 0.20349121, + "step": 4405, + "time_per_iteration": 2.459221601486206 + }, + { + "auxiliary_loss_clip": 0.01090896, + "auxiliary_loss_mlp": 0.0103529, + "balance_loss_clip": 1.03041744, + "balance_loss_mlp": 1.01745057, + "epoch": 0.1278509662817016, + "flos": 35509636437120.0, + "grad_norm": 1.6530918385825848, + "language_loss": 0.69354475, + "learning_rate": 3.9004098379168e-06, + "loss": 0.71480662, + "num_input_tokens_seen": 125246285, + "router_z_loss_clip": 0.60424805, + "router_z_loss_mlp": 0.1784668, + "step": 4406, + "time_per_iteration": 2.566375732421875 + }, + { + "auxiliary_loss_clip": 0.01092999, + "auxiliary_loss_mlp": 0.01037222, + "balance_loss_clip": 1.02970624, + "balance_loss_mlp": 1.01827955, + "epoch": 0.12787998375021764, + "flos": 11174789710080.0, + "grad_norm": 4.052746176215178, + "language_loss": 0.65964234, + "learning_rate": 3.900351255802463e-06, + "loss": 0.68094456, + "num_input_tokens_seen": 125257820, + "router_z_loss_clip": 0.6328125, + "router_z_loss_mlp": 0.18945312, + "step": 4407, + "time_per_iteration": 2.4128832817077637 + }, + { + "auxiliary_loss_clip": 0.01090897, + "auxiliary_loss_mlp": 0.01030057, + "balance_loss_clip": 1.02908611, + "balance_loss_mlp": 1.01347554, + "epoch": 0.12790900121873366, + "flos": 15624644780160.0, + "grad_norm": 2.515793869417904, + "language_loss": 0.91105747, + "learning_rate": 3.900292656903454e-06, + "loss": 0.93226695, + "num_input_tokens_seen": 125269835, + "router_z_loss_clip": 0.61767578, + "router_z_loss_mlp": 0.16583252, + "step": 4408, + "time_per_iteration": 2.6803138256073 + }, + { + "auxiliary_loss_clip": 0.01095739, + "auxiliary_loss_mlp": 0.01037322, + "balance_loss_clip": 1.0336585, + "balance_loss_mlp": 1.01738477, + "epoch": 0.12793801868724972, + "flos": 16682233818240.0, + "grad_norm": 2.646860187698926, + "language_loss": 0.75177509, + "learning_rate": 3.900234041220292e-06, + "loss": 0.77310574, + "num_input_tokens_seen": 125281530, + "router_z_loss_clip": 0.62109375, + "router_z_loss_mlp": 0.19946289, + "step": 4409, + "time_per_iteration": 2.3426969051361084 + }, + { + "auxiliary_loss_clip": 0.01089239, + "auxiliary_loss_mlp": 0.01037064, + "balance_loss_clip": 1.02842784, + "balance_loss_mlp": 1.0197072, + "epoch": 0.12796703615576577, + "flos": 14640408241920.0, + "grad_norm": 4.586141636621308, + "language_loss": 0.95352149, + "learning_rate": 3.900175408753494e-06, + "loss": 0.97478461, + "num_input_tokens_seen": 125296315, + "router_z_loss_clip": 0.60791016, + "router_z_loss_mlp": 0.17346191, + "step": 4410, + "time_per_iteration": 2.3906192779541016 + }, + { + "auxiliary_loss_clip": 0.01092108, + "auxiliary_loss_mlp": 0.01040348, + "balance_loss_clip": 1.02969098, + "balance_loss_mlp": 1.02104235, + "epoch": 0.12799605362428182, + "flos": 74729598192000.0, + "grad_norm": 2.127950198915589, + "language_loss": 0.99533492, + "learning_rate": 3.900116759503578e-06, + "loss": 1.0166595, + "num_input_tokens_seen": 125321410, + "router_z_loss_clip": 0.62451172, + "router_z_loss_mlp": 0.19293213, + "step": 4411, + "time_per_iteration": 2.943726062774658 + }, + { + "auxiliary_loss_clip": 0.01103881, + "auxiliary_loss_mlp": 0.0104432, + "balance_loss_clip": 1.03360343, + "balance_loss_mlp": 1.02326727, + "epoch": 0.12802507109279787, + "flos": 29271076214400.0, + "grad_norm": 6.890967012002592, + "language_loss": 0.95477295, + "learning_rate": 3.900058093471062e-06, + "loss": 0.976255, + "num_input_tokens_seen": 125339625, + "router_z_loss_clip": 0.70361328, + "router_z_loss_mlp": 0.21057129, + "step": 4412, + "time_per_iteration": 2.5207815170288086 + }, + { + "auxiliary_loss_clip": 0.01102948, + "auxiliary_loss_mlp": 0.01041957, + "balance_loss_clip": 1.03348303, + "balance_loss_mlp": 1.02073741, + "epoch": 0.12805408856131392, + "flos": 38757955466880.0, + "grad_norm": 3.1880581498349647, + "language_loss": 1.02063084, + "learning_rate": 3.899999410656463e-06, + "loss": 1.04207993, + "num_input_tokens_seen": 125359605, + "router_z_loss_clip": 0.6953125, + "router_z_loss_mlp": 0.21203613, + "step": 4413, + "time_per_iteration": 2.4445738792419434 + }, + { + "auxiliary_loss_clip": 0.01105468, + "auxiliary_loss_mlp": 0.01043667, + "balance_loss_clip": 1.03409982, + "balance_loss_mlp": 1.02185154, + "epoch": 0.12808310602982995, + "flos": 14346125458560.0, + "grad_norm": 2.7920041657210324, + "language_loss": 1.06283784, + "learning_rate": 3.899940711060301e-06, + "loss": 1.08432913, + "num_input_tokens_seen": 125371675, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.21801758, + "step": 4414, + "time_per_iteration": 2.3783018589019775 + }, + { + "auxiliary_loss_clip": 0.01018285, + "auxiliary_loss_mlp": 0.01003874, + "balance_loss_clip": 1.00754917, + "balance_loss_mlp": 1.00290871, + "epoch": 0.128112123498346, + "flos": 62655129123840.0, + "grad_norm": 0.66500627723951, + "language_loss": 0.41121137, + "learning_rate": 3.899881994683094e-06, + "loss": 0.43143296, + "num_input_tokens_seen": 125433970, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.00964355, + "step": 4415, + "time_per_iteration": 5.258975028991699 + }, + { + "auxiliary_loss_clip": 0.01094401, + "auxiliary_loss_mlp": 0.01035877, + "balance_loss_clip": 1.03222394, + "balance_loss_mlp": 1.01759636, + "epoch": 0.12814114096686205, + "flos": 24875195973120.0, + "grad_norm": 3.532100802884555, + "language_loss": 0.8046068, + "learning_rate": 3.89982326152536e-06, + "loss": 0.8259095, + "num_input_tokens_seen": 125452455, + "router_z_loss_clip": 0.62158203, + "router_z_loss_mlp": 0.18286133, + "step": 4416, + "time_per_iteration": 4.789100885391235 + }, + { + "auxiliary_loss_clip": 0.01015786, + "auxiliary_loss_mlp": 0.01001517, + "balance_loss_clip": 1.00567591, + "balance_loss_mlp": 1.00042629, + "epoch": 0.1281701584353781, + "flos": 65683111363200.0, + "grad_norm": 0.749760640936035, + "language_loss": 0.49155006, + "learning_rate": 3.899764511587618e-06, + "loss": 0.5117231, + "num_input_tokens_seen": 125511045, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.01092529, + "step": 4417, + "time_per_iteration": 2.9909274578094482 + }, + { + "auxiliary_loss_clip": 0.01093054, + "auxiliary_loss_mlp": 0.01037733, + "balance_loss_clip": 1.02912509, + "balance_loss_mlp": 1.01911259, + "epoch": 0.12819917590389415, + "flos": 17959147217280.0, + "grad_norm": 2.3526809351025926, + "language_loss": 0.80023372, + "learning_rate": 3.899705744870388e-06, + "loss": 0.82154161, + "num_input_tokens_seen": 125524145, + "router_z_loss_clip": 0.63989258, + "router_z_loss_mlp": 0.18615723, + "step": 4418, + "time_per_iteration": 2.380620002746582 + }, + { + "auxiliary_loss_clip": 0.01103255, + "auxiliary_loss_mlp": 0.0105192, + "balance_loss_clip": 1.03176832, + "balance_loss_mlp": 1.02828074, + "epoch": 0.1282281933724102, + "flos": 28872193397760.0, + "grad_norm": 3.7521675336618134, + "language_loss": 0.96736217, + "learning_rate": 3.899646961374188e-06, + "loss": 0.98891395, + "num_input_tokens_seen": 125543915, + "router_z_loss_clip": 0.71582031, + "router_z_loss_mlp": 0.23651123, + "step": 4419, + "time_per_iteration": 2.447946548461914 + }, + { + "auxiliary_loss_clip": 0.0108757, + "auxiliary_loss_mlp": 0.01033291, + "balance_loss_clip": 1.02866292, + "balance_loss_mlp": 1.01585031, + "epoch": 0.12825721084092623, + "flos": 37223522812800.0, + "grad_norm": 3.217546233528597, + "language_loss": 0.66473281, + "learning_rate": 3.899588161099537e-06, + "loss": 0.68594146, + "num_input_tokens_seen": 125560745, + "router_z_loss_clip": 0.58911133, + "router_z_loss_mlp": 0.17443848, + "step": 4420, + "time_per_iteration": 2.772113800048828 + }, + { + "auxiliary_loss_clip": 0.01083287, + "auxiliary_loss_mlp": 0.01031345, + "balance_loss_clip": 1.0264945, + "balance_loss_mlp": 1.01493609, + "epoch": 0.12828622830944228, + "flos": 17487121368960.0, + "grad_norm": 2.450277751131522, + "language_loss": 0.62633717, + "learning_rate": 3.899529344046955e-06, + "loss": 0.64748347, + "num_input_tokens_seen": 125575595, + "router_z_loss_clip": 0.56835938, + "router_z_loss_mlp": 0.1640625, + "step": 4421, + "time_per_iteration": 2.3669772148132324 + }, + { + "auxiliary_loss_clip": 0.01083832, + "auxiliary_loss_mlp": 0.0103642, + "balance_loss_clip": 1.02811718, + "balance_loss_mlp": 1.01929569, + "epoch": 0.12831524577795833, + "flos": 15918124602240.0, + "grad_norm": 2.497543082685936, + "language_loss": 0.72161925, + "learning_rate": 3.89947051021696e-06, + "loss": 0.74282181, + "num_input_tokens_seen": 125589115, + "router_z_loss_clip": 0.55664062, + "router_z_loss_mlp": 0.17138672, + "step": 4422, + "time_per_iteration": 4.936917066574097 + }, + { + "auxiliary_loss_clip": 0.01013987, + "auxiliary_loss_mlp": 0.01002, + "balance_loss_clip": 1.00406337, + "balance_loss_mlp": 1.00091553, + "epoch": 0.12834426324647438, + "flos": 74766255120000.0, + "grad_norm": 0.6763536624551522, + "language_loss": 0.47381684, + "learning_rate": 3.899411659610075e-06, + "loss": 0.49397674, + "num_input_tokens_seen": 125650970, + "router_z_loss_clip": 0.09912109, + "router_z_loss_mlp": 0.01086426, + "step": 4423, + "time_per_iteration": 3.0469019412994385 + }, + { + "auxiliary_loss_clip": 0.01089972, + "auxiliary_loss_mlp": 0.01041718, + "balance_loss_clip": 1.03021371, + "balance_loss_mlp": 1.02328253, + "epoch": 0.12837328071499043, + "flos": 14056276417920.0, + "grad_norm": 2.6545552069448597, + "language_loss": 0.89046121, + "learning_rate": 3.899352792226815e-06, + "loss": 0.91177809, + "num_input_tokens_seen": 125664460, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.18432617, + "step": 4424, + "time_per_iteration": 4.8171727657318115 + }, + { + "auxiliary_loss_clip": 0.01085747, + "auxiliary_loss_mlp": 0.01030959, + "balance_loss_clip": 1.02906561, + "balance_loss_mlp": 1.01453221, + "epoch": 0.12840229818350646, + "flos": 23030805335040.0, + "grad_norm": 2.0550093791644812, + "language_loss": 0.78362048, + "learning_rate": 3.899293908067705e-06, + "loss": 0.80478758, + "num_input_tokens_seen": 125677580, + "router_z_loss_clip": 0.56640625, + "router_z_loss_mlp": 0.16418457, + "step": 4425, + "time_per_iteration": 2.435692071914673 + }, + { + "auxiliary_loss_clip": 0.01011991, + "auxiliary_loss_mlp": 0.01001606, + "balance_loss_clip": 1.00241256, + "balance_loss_mlp": 1.00040185, + "epoch": 0.1284313156520225, + "flos": 68957161931520.0, + "grad_norm": 0.6889070879033514, + "language_loss": 0.49686301, + "learning_rate": 3.899235007133261e-06, + "loss": 0.51699901, + "num_input_tokens_seen": 125738505, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01202393, + "step": 4426, + "time_per_iteration": 3.0611624717712402 + }, + { + "auxiliary_loss_clip": 0.0101253, + "auxiliary_loss_mlp": 0.01002121, + "balance_loss_clip": 1.00286853, + "balance_loss_mlp": 1.00088155, + "epoch": 0.12846033312053856, + "flos": 74771561646720.0, + "grad_norm": 0.6279006240523697, + "language_loss": 0.4571811, + "learning_rate": 3.899176089424005e-06, + "loss": 0.47732762, + "num_input_tokens_seen": 125802835, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.01239014, + "step": 4427, + "time_per_iteration": 3.086170196533203 + }, + { + "auxiliary_loss_clip": 0.01095032, + "auxiliary_loss_mlp": 0.01045245, + "balance_loss_clip": 1.03054583, + "balance_loss_mlp": 1.02463341, + "epoch": 0.1284893505890546, + "flos": 25476296584320.0, + "grad_norm": 2.5251192965313782, + "language_loss": 0.80010629, + "learning_rate": 3.899117154940458e-06, + "loss": 0.82150912, + "num_input_tokens_seen": 125815590, + "router_z_loss_clip": 0.64501953, + "router_z_loss_mlp": 0.20629883, + "step": 4428, + "time_per_iteration": 2.3995542526245117 + }, + { + "auxiliary_loss_clip": 0.01012322, + "auxiliary_loss_mlp": 0.01002104, + "balance_loss_clip": 1.00241876, + "balance_loss_mlp": 1.00088263, + "epoch": 0.12851836805757066, + "flos": 74759552138880.0, + "grad_norm": 0.6241606937801127, + "language_loss": 0.47308326, + "learning_rate": 3.8990582036831395e-06, + "loss": 0.49322748, + "num_input_tokens_seen": 125877510, + "router_z_loss_clip": 0.09912109, + "router_z_loss_mlp": 0.01220703, + "step": 4429, + "time_per_iteration": 3.228362560272217 + }, + { + "auxiliary_loss_clip": 0.01088505, + "auxiliary_loss_mlp": 0.01034544, + "balance_loss_clip": 1.02900672, + "balance_loss_mlp": 1.01836181, + "epoch": 0.12854738552608672, + "flos": 29926186565760.0, + "grad_norm": 3.3933924947424123, + "language_loss": 0.84462303, + "learning_rate": 3.8989992356525704e-06, + "loss": 0.86585355, + "num_input_tokens_seen": 125890850, + "router_z_loss_clip": 0.59472656, + "router_z_loss_mlp": 0.16174316, + "step": 4430, + "time_per_iteration": 2.435692071914673 + }, + { + "auxiliary_loss_clip": 0.01092443, + "auxiliary_loss_mlp": 0.01037237, + "balance_loss_clip": 1.03049994, + "balance_loss_mlp": 1.01844382, + "epoch": 0.12857640299460274, + "flos": 25766983497600.0, + "grad_norm": 2.360661912693787, + "language_loss": 0.96251541, + "learning_rate": 3.898940250849272e-06, + "loss": 0.98381233, + "num_input_tokens_seen": 125905155, + "router_z_loss_clip": 0.62011719, + "router_z_loss_mlp": 0.18780518, + "step": 4431, + "time_per_iteration": 2.6241424083709717 + }, + { + "auxiliary_loss_clip": 0.01011791, + "auxiliary_loss_mlp": 0.01001768, + "balance_loss_clip": 1.00188541, + "balance_loss_mlp": 1.0005877, + "epoch": 0.1286054204631188, + "flos": 65244602286720.0, + "grad_norm": 0.6938882861662643, + "language_loss": 0.49511969, + "learning_rate": 3.898881249273764e-06, + "loss": 0.51525527, + "num_input_tokens_seen": 125968210, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01177979, + "step": 4432, + "time_per_iteration": 3.0279130935668945 + }, + { + "auxiliary_loss_clip": 0.01098964, + "auxiliary_loss_mlp": 0.01040424, + "balance_loss_clip": 1.03286076, + "balance_loss_mlp": 1.01927686, + "epoch": 0.12863443793163484, + "flos": 28065211165440.0, + "grad_norm": 2.58594973722622, + "language_loss": 0.92654049, + "learning_rate": 3.898822230926569e-06, + "loss": 0.94793439, + "num_input_tokens_seen": 125981235, + "router_z_loss_clip": 0.66113281, + "router_z_loss_mlp": 0.21142578, + "step": 4433, + "time_per_iteration": 2.4578402042388916 + }, + { + "auxiliary_loss_clip": 0.01110121, + "auxiliary_loss_mlp": 0.01047969, + "balance_loss_clip": 1.03522801, + "balance_loss_mlp": 1.02424681, + "epoch": 0.1286634554001509, + "flos": 17663677447680.0, + "grad_norm": 5.458320565208579, + "language_loss": 1.04184365, + "learning_rate": 3.898763195808208e-06, + "loss": 1.06342459, + "num_input_tokens_seen": 125995830, + "router_z_loss_clip": 0.74902344, + "router_z_loss_mlp": 0.23742676, + "step": 4434, + "time_per_iteration": 2.4589154720306396 + }, + { + "auxiliary_loss_clip": 0.01100641, + "auxiliary_loss_mlp": 0.01039056, + "balance_loss_clip": 1.03116846, + "balance_loss_mlp": 1.01778889, + "epoch": 0.12869247286866695, + "flos": 24634889884800.0, + "grad_norm": 2.6712560911071583, + "language_loss": 0.96253157, + "learning_rate": 3.898704143919201e-06, + "loss": 0.98392856, + "num_input_tokens_seen": 126012255, + "router_z_loss_clip": 0.69384766, + "router_z_loss_mlp": 0.21276855, + "step": 4435, + "time_per_iteration": 2.45330548286438 + }, + { + "auxiliary_loss_clip": 0.01102399, + "auxiliary_loss_mlp": 0.01042921, + "balance_loss_clip": 1.03264904, + "balance_loss_mlp": 1.022012, + "epoch": 0.128721490337183, + "flos": 16394025611520.0, + "grad_norm": 2.2826893826237944, + "language_loss": 0.69765687, + "learning_rate": 3.898645075260071e-06, + "loss": 0.71911007, + "num_input_tokens_seen": 126031665, + "router_z_loss_clip": 0.69726562, + "router_z_loss_mlp": 0.20935059, + "step": 4436, + "time_per_iteration": 2.5328781604766846 + }, + { + "auxiliary_loss_clip": 0.01095101, + "auxiliary_loss_mlp": 0.01038238, + "balance_loss_clip": 1.03210378, + "balance_loss_mlp": 1.01922977, + "epoch": 0.12875050780569902, + "flos": 32371957105920.0, + "grad_norm": 1.8136137486832715, + "language_loss": 0.8130002, + "learning_rate": 3.89858598983134e-06, + "loss": 0.8343336, + "num_input_tokens_seen": 126054270, + "router_z_loss_clip": 0.63012695, + "router_z_loss_mlp": 0.19030762, + "step": 4437, + "time_per_iteration": 2.591404676437378 + }, + { + "auxiliary_loss_clip": 0.01095032, + "auxiliary_loss_mlp": 0.01032431, + "balance_loss_clip": 1.03195214, + "balance_loss_mlp": 1.01451957, + "epoch": 0.12877952527421507, + "flos": 14638697585280.0, + "grad_norm": 7.165945035955552, + "language_loss": 0.8321923, + "learning_rate": 3.898526887633529e-06, + "loss": 0.85346693, + "num_input_tokens_seen": 126064720, + "router_z_loss_clip": 0.63110352, + "router_z_loss_mlp": 0.17913818, + "step": 4438, + "time_per_iteration": 2.3438704013824463 + }, + { + "auxiliary_loss_clip": 0.01014044, + "auxiliary_loss_mlp": 0.01001884, + "balance_loss_clip": 1.00415325, + "balance_loss_mlp": 1.00061429, + "epoch": 0.12880854274273112, + "flos": 74772504253440.0, + "grad_norm": 0.7216691263208528, + "language_loss": 0.46880811, + "learning_rate": 3.89846776866716e-06, + "loss": 0.48896736, + "num_input_tokens_seen": 126112780, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01269531, + "step": 4439, + "time_per_iteration": 3.034043788909912 + }, + { + "auxiliary_loss_clip": 0.01013339, + "auxiliary_loss_mlp": 0.01003092, + "balance_loss_clip": 1.00362706, + "balance_loss_mlp": 1.00191224, + "epoch": 0.12883756021124718, + "flos": 72577829278080.0, + "grad_norm": 0.6760693258650747, + "language_loss": 0.46540585, + "learning_rate": 3.898408632932756e-06, + "loss": 0.48557016, + "num_input_tokens_seen": 126179160, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.01177979, + "step": 4440, + "time_per_iteration": 3.1484286785125732 + }, + { + "auxiliary_loss_clip": 0.01097266, + "auxiliary_loss_mlp": 0.01039866, + "balance_loss_clip": 1.03223681, + "balance_loss_mlp": 1.02041113, + "epoch": 0.12886657767976323, + "flos": 62474036434560.0, + "grad_norm": 2.211659435850821, + "language_loss": 0.87678957, + "learning_rate": 3.898349480430839e-06, + "loss": 0.89816093, + "num_input_tokens_seen": 126203005, + "router_z_loss_clip": 0.65039062, + "router_z_loss_mlp": 0.19458008, + "step": 4441, + "time_per_iteration": 2.6738462448120117 + }, + { + "auxiliary_loss_clip": 0.01013111, + "auxiliary_loss_mlp": 0.01003901, + "balance_loss_clip": 1.00345802, + "balance_loss_mlp": 1.00254238, + "epoch": 0.12889559514827925, + "flos": 74773621416960.0, + "grad_norm": 0.755191226928894, + "language_loss": 0.55537736, + "learning_rate": 3.89829031116193e-06, + "loss": 0.57554746, + "num_input_tokens_seen": 126264685, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.01361084, + "step": 4442, + "time_per_iteration": 3.018846035003662 + }, + { + "auxiliary_loss_clip": 0.01100052, + "auxiliary_loss_mlp": 0.01044064, + "balance_loss_clip": 1.03245163, + "balance_loss_mlp": 1.02224278, + "epoch": 0.1289246126167953, + "flos": 23797847105280.0, + "grad_norm": 2.7384563710272647, + "language_loss": 0.8434515, + "learning_rate": 3.898231125126553e-06, + "loss": 0.86489272, + "num_input_tokens_seen": 126279200, + "router_z_loss_clip": 0.67626953, + "router_z_loss_mlp": 0.21826172, + "step": 4443, + "time_per_iteration": 2.406120538711548 + }, + { + "auxiliary_loss_clip": 0.0109022, + "auxiliary_loss_mlp": 0.01034354, + "balance_loss_clip": 1.02872014, + "balance_loss_mlp": 1.01617491, + "epoch": 0.12895363008531135, + "flos": 25840580376960.0, + "grad_norm": 2.5069121042935363, + "language_loss": 0.7077173, + "learning_rate": 3.898171922325232e-06, + "loss": 0.72896302, + "num_input_tokens_seen": 126294840, + "router_z_loss_clip": 0.61499023, + "router_z_loss_mlp": 0.18164062, + "step": 4444, + "time_per_iteration": 2.6066713333129883 + }, + { + "auxiliary_loss_clip": 0.01088628, + "auxiliary_loss_mlp": 0.01040246, + "balance_loss_clip": 1.02973402, + "balance_loss_mlp": 1.02093375, + "epoch": 0.1289826475538274, + "flos": 18071218281600.0, + "grad_norm": 3.1237648988274884, + "language_loss": 0.74122787, + "learning_rate": 3.898112702758487e-06, + "loss": 0.76251656, + "num_input_tokens_seen": 126310265, + "router_z_loss_clip": 0.58862305, + "router_z_loss_mlp": 0.19299316, + "step": 4445, + "time_per_iteration": 2.368262529373169 + }, + { + "auxiliary_loss_clip": 0.01092333, + "auxiliary_loss_mlp": 0.01032745, + "balance_loss_clip": 1.03037035, + "balance_loss_mlp": 1.01443422, + "epoch": 0.12901166502234346, + "flos": 14933154925440.0, + "grad_norm": 2.9850739257284955, + "language_loss": 0.79596019, + "learning_rate": 3.898053466426843e-06, + "loss": 0.81721091, + "num_input_tokens_seen": 126325110, + "router_z_loss_clip": 0.61987305, + "router_z_loss_mlp": 0.18310547, + "step": 4446, + "time_per_iteration": 2.40181565284729 + }, + { + "auxiliary_loss_clip": 0.01087773, + "auxiliary_loss_mlp": 0.01040956, + "balance_loss_clip": 1.02990353, + "balance_loss_mlp": 1.02290177, + "epoch": 0.1290406824908595, + "flos": 26060812433280.0, + "grad_norm": 3.3787367477545955, + "language_loss": 1.08749747, + "learning_rate": 3.897994213330823e-06, + "loss": 1.10878468, + "num_input_tokens_seen": 126338895, + "router_z_loss_clip": 0.57861328, + "router_z_loss_mlp": 0.18054199, + "step": 4447, + "time_per_iteration": 2.4154984951019287 + }, + { + "auxiliary_loss_clip": 0.01012837, + "auxiliary_loss_mlp": 0.01007393, + "balance_loss_clip": 1.0034827, + "balance_loss_mlp": 1.00609922, + "epoch": 0.12906969995937553, + "flos": 69701753831040.0, + "grad_norm": 0.6635735777922176, + "language_loss": 0.48203582, + "learning_rate": 3.89793494347095e-06, + "loss": 0.50223809, + "num_input_tokens_seen": 126402480, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.01293945, + "step": 4448, + "time_per_iteration": 3.0413577556610107 + }, + { + "auxiliary_loss_clip": 0.01090947, + "auxiliary_loss_mlp": 0.01034674, + "balance_loss_clip": 1.02878404, + "balance_loss_mlp": 1.01643503, + "epoch": 0.12909871742789158, + "flos": 16866994066560.0, + "grad_norm": 2.3408674121423307, + "language_loss": 0.79063332, + "learning_rate": 3.897875656847747e-06, + "loss": 0.81188953, + "num_input_tokens_seen": 126416585, + "router_z_loss_clip": 0.62207031, + "router_z_loss_mlp": 0.18237305, + "step": 4449, + "time_per_iteration": 2.3625779151916504 + }, + { + "auxiliary_loss_clip": 0.01013979, + "auxiliary_loss_mlp": 0.01004956, + "balance_loss_clip": 1.00452709, + "balance_loss_mlp": 1.00369799, + "epoch": 0.12912773489640764, + "flos": 74775471719040.0, + "grad_norm": 0.6504764716544711, + "language_loss": 0.48263314, + "learning_rate": 3.897816353461739e-06, + "loss": 0.50282252, + "num_input_tokens_seen": 126484415, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01257324, + "step": 4450, + "time_per_iteration": 3.127139091491699 + }, + { + "auxiliary_loss_clip": 0.01015031, + "auxiliary_loss_mlp": 0.01003813, + "balance_loss_clip": 1.00547957, + "balance_loss_mlp": 1.0026089, + "epoch": 0.1291567523649237, + "flos": 65438614045440.0, + "grad_norm": 0.6852434965547871, + "language_loss": 0.51833117, + "learning_rate": 3.8977570333134484e-06, + "loss": 0.53851962, + "num_input_tokens_seen": 126551925, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01202393, + "step": 4451, + "time_per_iteration": 3.099097728729248 + }, + { + "auxiliary_loss_clip": 0.01104371, + "auxiliary_loss_mlp": 0.01042981, + "balance_loss_clip": 1.03237689, + "balance_loss_mlp": 1.01977062, + "epoch": 0.12918576983343974, + "flos": 35876782961280.0, + "grad_norm": 1.9952708995445594, + "language_loss": 0.92208713, + "learning_rate": 3.8976976964034e-06, + "loss": 0.9435606, + "num_input_tokens_seen": 126572960, + "router_z_loss_clip": 0.71972656, + "router_z_loss_mlp": 0.23205566, + "step": 4452, + "time_per_iteration": 2.517399311065674 + }, + { + "auxiliary_loss_clip": 0.01105157, + "auxiliary_loss_mlp": 0.01042022, + "balance_loss_clip": 1.03407359, + "balance_loss_mlp": 1.01906276, + "epoch": 0.12921478730195576, + "flos": 10991670295680.0, + "grad_norm": 2.69590458390524, + "language_loss": 0.81740129, + "learning_rate": 3.897638342732118e-06, + "loss": 0.83887303, + "num_input_tokens_seen": 126584815, + "router_z_loss_clip": 0.7109375, + "router_z_loss_mlp": 0.22949219, + "step": 4453, + "time_per_iteration": 2.3639140129089355 + }, + { + "auxiliary_loss_clip": 0.01100474, + "auxiliary_loss_mlp": 0.01041442, + "balance_loss_clip": 1.03349519, + "balance_loss_mlp": 1.02041328, + "epoch": 0.12924380477047182, + "flos": 31970595582720.0, + "grad_norm": 2.3866721847152865, + "language_loss": 0.77882874, + "learning_rate": 3.897578972300126e-06, + "loss": 0.80024785, + "num_input_tokens_seen": 126607485, + "router_z_loss_clip": 0.66918945, + "router_z_loss_mlp": 0.21014404, + "step": 4454, + "time_per_iteration": 2.588637590408325 + }, + { + "auxiliary_loss_clip": 0.01017695, + "auxiliary_loss_mlp": 0.01000823, + "balance_loss_clip": 1.0079875, + "balance_loss_mlp": 0.9995594, + "epoch": 0.12927282223898787, + "flos": 71412498184320.0, + "grad_norm": 0.6719167640839953, + "language_loss": 0.50700629, + "learning_rate": 3.897519585107948e-06, + "loss": 0.52719146, + "num_input_tokens_seen": 126671435, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.01263428, + "step": 4455, + "time_per_iteration": 3.0204684734344482 + }, + { + "auxiliary_loss_clip": 0.01100704, + "auxiliary_loss_mlp": 0.01040324, + "balance_loss_clip": 1.03203893, + "balance_loss_mlp": 1.01854467, + "epoch": 0.12930183970750392, + "flos": 41381224692480.0, + "grad_norm": 3.1325346151258366, + "language_loss": 0.8999083, + "learning_rate": 3.89746018115611e-06, + "loss": 0.92131853, + "num_input_tokens_seen": 126686620, + "router_z_loss_clip": 0.6875, + "router_z_loss_mlp": 0.21777344, + "step": 4456, + "time_per_iteration": 2.5349931716918945 + }, + { + "auxiliary_loss_clip": 0.01091707, + "auxiliary_loss_mlp": 0.01037913, + "balance_loss_clip": 1.02863193, + "balance_loss_mlp": 1.01903033, + "epoch": 0.12933085717601997, + "flos": 20405720718720.0, + "grad_norm": 2.0736799486556627, + "language_loss": 0.82177609, + "learning_rate": 3.897400760445136e-06, + "loss": 0.8430723, + "num_input_tokens_seen": 126702535, + "router_z_loss_clip": 0.63037109, + "router_z_loss_mlp": 0.18884277, + "step": 4457, + "time_per_iteration": 2.6428587436676025 + }, + { + "auxiliary_loss_clip": 0.01017214, + "auxiliary_loss_mlp": 0.0100692, + "balance_loss_clip": 1.00694764, + "balance_loss_mlp": 1.00572205, + "epoch": 0.12935987464453602, + "flos": 65393820904320.0, + "grad_norm": 0.8028451567451723, + "language_loss": 0.48628804, + "learning_rate": 3.8973413229755496e-06, + "loss": 0.50652933, + "num_input_tokens_seen": 126762320, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01196289, + "step": 4458, + "time_per_iteration": 2.9765050411224365 + }, + { + "auxiliary_loss_clip": 0.01090022, + "auxiliary_loss_mlp": 0.01042299, + "balance_loss_clip": 1.03057051, + "balance_loss_mlp": 1.02339828, + "epoch": 0.12938889211305205, + "flos": 28324371254400.0, + "grad_norm": 2.6385107346723635, + "language_loss": 0.7443763, + "learning_rate": 3.897281868747877e-06, + "loss": 0.76569951, + "num_input_tokens_seen": 126777830, + "router_z_loss_clip": 0.59423828, + "router_z_loss_mlp": 0.18896484, + "step": 4459, + "time_per_iteration": 2.466609001159668 + }, + { + "auxiliary_loss_clip": 0.01094651, + "auxiliary_loss_mlp": 0.01036011, + "balance_loss_clip": 1.03223157, + "balance_loss_mlp": 1.01715183, + "epoch": 0.1294179095815681, + "flos": 16099044600960.0, + "grad_norm": 2.5216847230036734, + "language_loss": 0.78647822, + "learning_rate": 3.897222397762644e-06, + "loss": 0.80778486, + "num_input_tokens_seen": 126790320, + "router_z_loss_clip": 0.62402344, + "router_z_loss_mlp": 0.18878174, + "step": 4460, + "time_per_iteration": 2.3233909606933594 + }, + { + "auxiliary_loss_clip": 0.01014929, + "auxiliary_loss_mlp": 0.01005204, + "balance_loss_clip": 1.00521123, + "balance_loss_mlp": 1.00392222, + "epoch": 0.12944692705008415, + "flos": 63177917955840.0, + "grad_norm": 0.5905892241668171, + "language_loss": 0.44155902, + "learning_rate": 3.8971629100203754e-06, + "loss": 0.46176034, + "num_input_tokens_seen": 126859375, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.01281738, + "step": 4461, + "time_per_iteration": 3.2501022815704346 + }, + { + "auxiliary_loss_clip": 0.01092416, + "auxiliary_loss_mlp": 0.01043895, + "balance_loss_clip": 1.02887797, + "balance_loss_mlp": 1.0237608, + "epoch": 0.1294759445186002, + "flos": 29671111105920.0, + "grad_norm": 4.147746174505456, + "language_loss": 0.90630084, + "learning_rate": 3.897103405521595e-06, + "loss": 0.92766392, + "num_input_tokens_seen": 126873155, + "router_z_loss_clip": 0.63549805, + "router_z_loss_mlp": 0.20153809, + "step": 4462, + "time_per_iteration": 2.4857492446899414 + }, + { + "auxiliary_loss_clip": 0.01014953, + "auxiliary_loss_mlp": 0.01006522, + "balance_loss_clip": 1.00541162, + "balance_loss_mlp": 1.00530016, + "epoch": 0.12950496198711625, + "flos": 64890512611200.0, + "grad_norm": 0.7002254490472535, + "language_loss": 0.5498665, + "learning_rate": 3.89704388426683e-06, + "loss": 0.57008123, + "num_input_tokens_seen": 126937975, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01220703, + "step": 4463, + "time_per_iteration": 3.0478129386901855 + }, + { + "auxiliary_loss_clip": 0.0101285, + "auxiliary_loss_mlp": 0.01005874, + "balance_loss_clip": 1.00360489, + "balance_loss_mlp": 1.00453329, + "epoch": 0.1295339794556323, + "flos": 74780324398080.0, + "grad_norm": 0.6279459143464173, + "language_loss": 0.48189658, + "learning_rate": 3.896984346256606e-06, + "loss": 0.50208384, + "num_input_tokens_seen": 127006760, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01342773, + "step": 4464, + "time_per_iteration": 3.163670063018799 + }, + { + "auxiliary_loss_clip": 0.01011745, + "auxiliary_loss_mlp": 0.01010965, + "balance_loss_clip": 1.00246739, + "balance_loss_mlp": 1.00962985, + "epoch": 0.12956299692414833, + "flos": 69235069420800.0, + "grad_norm": 0.7626126447118281, + "language_loss": 0.47598708, + "learning_rate": 3.896924791491449e-06, + "loss": 0.49621418, + "num_input_tokens_seen": 127069445, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.0133667, + "step": 4465, + "time_per_iteration": 3.0623934268951416 + }, + { + "auxiliary_loss_clip": 0.0101181, + "auxiliary_loss_mlp": 0.01005514, + "balance_loss_clip": 1.00248075, + "balance_loss_mlp": 1.00420821, + "epoch": 0.12959201439266438, + "flos": 64109645942400.0, + "grad_norm": 0.7092893125091858, + "language_loss": 0.48839444, + "learning_rate": 3.896865219971884e-06, + "loss": 0.50856763, + "num_input_tokens_seen": 127124670, + "router_z_loss_clip": 0.09326172, + "router_z_loss_mlp": 0.01306152, + "step": 4466, + "time_per_iteration": 2.8562841415405273 + }, + { + "auxiliary_loss_clip": 0.01104604, + "auxiliary_loss_mlp": 0.0105186, + "balance_loss_clip": 1.03262782, + "balance_loss_mlp": 1.02845979, + "epoch": 0.12962103186118043, + "flos": 20223439176960.0, + "grad_norm": 3.1898226925310817, + "language_loss": 0.7858367, + "learning_rate": 3.896805631698438e-06, + "loss": 0.8074013, + "num_input_tokens_seen": 127137495, + "router_z_loss_clip": 0.72021484, + "router_z_loss_mlp": 0.23364258, + "step": 4467, + "time_per_iteration": 2.3500187397003174 + }, + { + "auxiliary_loss_clip": 0.0110709, + "auxiliary_loss_mlp": 0.01041174, + "balance_loss_clip": 1.03719509, + "balance_loss_mlp": 1.01951337, + "epoch": 0.12965004932969648, + "flos": 39959491507200.0, + "grad_norm": 2.0117534254303617, + "language_loss": 0.70700467, + "learning_rate": 3.896746026671637e-06, + "loss": 0.72848731, + "num_input_tokens_seen": 127153460, + "router_z_loss_clip": 0.69873047, + "router_z_loss_mlp": 0.2166748, + "step": 4468, + "time_per_iteration": 2.5039565563201904 + }, + { + "auxiliary_loss_clip": 0.01093917, + "auxiliary_loss_mlp": 0.01032852, + "balance_loss_clip": 1.03254747, + "balance_loss_mlp": 1.01468492, + "epoch": 0.12967906679821253, + "flos": 30657093212160.0, + "grad_norm": 2.6935574979970323, + "language_loss": 0.61842096, + "learning_rate": 3.896686404892008e-06, + "loss": 0.63968873, + "num_input_tokens_seen": 127169395, + "router_z_loss_clip": 0.61425781, + "router_z_loss_mlp": 0.18182373, + "step": 4469, + "time_per_iteration": 2.466891288757324 + }, + { + "auxiliary_loss_clip": 0.01105713, + "auxiliary_loss_mlp": 0.0104226, + "balance_loss_clip": 1.03866971, + "balance_loss_mlp": 1.02216172, + "epoch": 0.12970808426672856, + "flos": 17048193356160.0, + "grad_norm": 3.4668518309665495, + "language_loss": 0.967731, + "learning_rate": 3.896626766360077e-06, + "loss": 0.98921078, + "num_input_tokens_seen": 127181220, + "router_z_loss_clip": 0.67089844, + "router_z_loss_mlp": 0.2010498, + "step": 4470, + "time_per_iteration": 2.652592420578003 + }, + { + "auxiliary_loss_clip": 0.01021864, + "auxiliary_loss_mlp": 0.01003571, + "balance_loss_clip": 1.01151156, + "balance_loss_mlp": 1.00227797, + "epoch": 0.1297371017352446, + "flos": 61891578489600.0, + "grad_norm": 0.6427991639609948, + "language_loss": 0.50987786, + "learning_rate": 3.896567111076371e-06, + "loss": 0.53013217, + "num_input_tokens_seen": 127240905, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01293945, + "step": 4471, + "time_per_iteration": 2.938729763031006 + }, + { + "auxiliary_loss_clip": 0.01104044, + "auxiliary_loss_mlp": 0.01040592, + "balance_loss_clip": 1.03558517, + "balance_loss_mlp": 1.02006412, + "epoch": 0.12976611920376066, + "flos": 30848207328000.0, + "grad_norm": 2.207606245557766, + "language_loss": 0.82973444, + "learning_rate": 3.896507439041417e-06, + "loss": 0.85118085, + "num_input_tokens_seen": 127263770, + "router_z_loss_clip": 0.68505859, + "router_z_loss_mlp": 0.20532227, + "step": 4472, + "time_per_iteration": 2.550671100616455 + }, + { + "auxiliary_loss_clip": 0.01100583, + "auxiliary_loss_mlp": 0.01049797, + "balance_loss_clip": 1.03807855, + "balance_loss_mlp": 1.03090203, + "epoch": 0.1297951366722767, + "flos": 11285394497280.0, + "grad_norm": 2.756345569353692, + "language_loss": 0.74207211, + "learning_rate": 3.896447750255741e-06, + "loss": 0.76357591, + "num_input_tokens_seen": 127275295, + "router_z_loss_clip": 0.62548828, + "router_z_loss_mlp": 0.18896484, + "step": 4473, + "time_per_iteration": 2.4313013553619385 + }, + { + "auxiliary_loss_clip": 0.01023388, + "auxiliary_loss_mlp": 0.01008449, + "balance_loss_clip": 1.01262116, + "balance_loss_mlp": 1.00704277, + "epoch": 0.12982415414079276, + "flos": 62983766551680.0, + "grad_norm": 0.6677467677213996, + "language_loss": 0.49918807, + "learning_rate": 3.896388044719872e-06, + "loss": 0.51950645, + "num_input_tokens_seen": 127341495, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01403809, + "step": 4474, + "time_per_iteration": 3.057884931564331 + }, + { + "auxiliary_loss_clip": 0.01090914, + "auxiliary_loss_mlp": 0.01037792, + "balance_loss_clip": 1.03150773, + "balance_loss_mlp": 1.02026188, + "epoch": 0.12985317160930882, + "flos": 24017311111680.0, + "grad_norm": 3.339143144022326, + "language_loss": 0.79417336, + "learning_rate": 3.896328322434335e-06, + "loss": 0.81546044, + "num_input_tokens_seen": 127355630, + "router_z_loss_clip": 0.59350586, + "router_z_loss_mlp": 0.17541504, + "step": 4475, + "time_per_iteration": 2.4368302822113037 + }, + { + "auxiliary_loss_clip": 0.01102724, + "auxiliary_loss_mlp": 0.01042166, + "balance_loss_clip": 1.03769767, + "balance_loss_mlp": 1.0226748, + "epoch": 0.12988218907782484, + "flos": 22192505746560.0, + "grad_norm": 3.2011413181180526, + "language_loss": 0.72226906, + "learning_rate": 3.896268583399661e-06, + "loss": 0.74371791, + "num_input_tokens_seen": 127368980, + "router_z_loss_clip": 0.65039062, + "router_z_loss_mlp": 0.19494629, + "step": 4476, + "time_per_iteration": 2.391957998275757 + }, + { + "auxiliary_loss_clip": 0.0101782, + "auxiliary_loss_mlp": 0.01009281, + "balance_loss_clip": 1.00750232, + "balance_loss_mlp": 1.0079937, + "epoch": 0.1299112065463409, + "flos": 60396143690880.0, + "grad_norm": 0.7165451420156406, + "language_loss": 0.52098465, + "learning_rate": 3.896208827616374e-06, + "loss": 0.54125571, + "num_input_tokens_seen": 127430725, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01287842, + "step": 4477, + "time_per_iteration": 2.996147871017456 + }, + { + "auxiliary_loss_clip": 0.01094669, + "auxiliary_loss_mlp": 0.01047787, + "balance_loss_clip": 1.03295398, + "balance_loss_mlp": 1.02700925, + "epoch": 0.12994022401485694, + "flos": 18144570781440.0, + "grad_norm": 2.639950451201006, + "language_loss": 0.7968719, + "learning_rate": 3.896149055085004e-06, + "loss": 0.81829649, + "num_input_tokens_seen": 127445575, + "router_z_loss_clip": 0.61669922, + "router_z_loss_mlp": 0.20788574, + "step": 4478, + "time_per_iteration": 2.3588759899139404 + }, + { + "auxiliary_loss_clip": 0.01098152, + "auxiliary_loss_mlp": 0.01049434, + "balance_loss_clip": 1.03131449, + "balance_loss_mlp": 1.02943087, + "epoch": 0.129969241483373, + "flos": 27013417413120.0, + "grad_norm": 5.125951434666074, + "language_loss": 0.75601214, + "learning_rate": 3.896089265806077e-06, + "loss": 0.77748799, + "num_input_tokens_seen": 127465060, + "router_z_loss_clip": 0.66870117, + "router_z_loss_mlp": 0.19995117, + "step": 4479, + "time_per_iteration": 2.4624783992767334 + }, + { + "auxiliary_loss_clip": 0.01014226, + "auxiliary_loss_mlp": 0.01002964, + "balance_loss_clip": 1.00410914, + "balance_loss_mlp": 1.00151539, + "epoch": 0.12999825895188905, + "flos": 74772608987520.0, + "grad_norm": 0.7232093760768102, + "language_loss": 0.48475724, + "learning_rate": 3.896029459780124e-06, + "loss": 0.50492918, + "num_input_tokens_seen": 127532040, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.01446533, + "step": 4480, + "time_per_iteration": 3.159836769104004 + }, + { + "auxiliary_loss_clip": 0.01106526, + "auxiliary_loss_mlp": 0.01044613, + "balance_loss_clip": 1.03717613, + "balance_loss_mlp": 1.02363276, + "epoch": 0.1300272764204051, + "flos": 26168554488960.0, + "grad_norm": 2.156352603634555, + "language_loss": 0.89517194, + "learning_rate": 3.895969637007671e-06, + "loss": 0.91668332, + "num_input_tokens_seen": 127547225, + "router_z_loss_clip": 0.69433594, + "router_z_loss_mlp": 0.2097168, + "step": 4481, + "time_per_iteration": 2.4679391384124756 + }, + { + "auxiliary_loss_clip": 0.01015655, + "auxiliary_loss_mlp": 0.01002702, + "balance_loss_clip": 1.00571203, + "balance_loss_mlp": 1.00136113, + "epoch": 0.13005629388892112, + "flos": 62184534641280.0, + "grad_norm": 0.6869844050427553, + "language_loss": 0.48749942, + "learning_rate": 3.895909797489246e-06, + "loss": 0.50768304, + "num_input_tokens_seen": 127608645, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01342773, + "step": 4482, + "time_per_iteration": 2.896836042404175 + }, + { + "auxiliary_loss_clip": 0.01093239, + "auxiliary_loss_mlp": 0.01041493, + "balance_loss_clip": 1.03008783, + "balance_loss_mlp": 1.02057195, + "epoch": 0.13008531135743717, + "flos": 31940604858240.0, + "grad_norm": 3.572749937555342, + "language_loss": 0.8166281, + "learning_rate": 3.89584994122538e-06, + "loss": 0.83797538, + "num_input_tokens_seen": 127625460, + "router_z_loss_clip": 0.63183594, + "router_z_loss_mlp": 0.20910645, + "step": 4483, + "time_per_iteration": 2.716981887817383 + }, + { + "auxiliary_loss_clip": 0.01091525, + "auxiliary_loss_mlp": 0.01043017, + "balance_loss_clip": 1.03191972, + "balance_loss_mlp": 1.02567244, + "epoch": 0.13011432882595322, + "flos": 24599592633600.0, + "grad_norm": 6.745218370738543, + "language_loss": 0.86301309, + "learning_rate": 3.895790068216599e-06, + "loss": 0.88435853, + "num_input_tokens_seen": 127640695, + "router_z_loss_clip": 0.59619141, + "router_z_loss_mlp": 0.17358398, + "step": 4484, + "time_per_iteration": 2.4704957008361816 + }, + { + "auxiliary_loss_clip": 0.01101387, + "auxiliary_loss_mlp": 0.01037551, + "balance_loss_clip": 1.03613067, + "balance_loss_mlp": 1.01746416, + "epoch": 0.13014334629446928, + "flos": 12742634401920.0, + "grad_norm": 2.278906348147616, + "language_loss": 0.88328207, + "learning_rate": 3.8957301784634336e-06, + "loss": 0.90467143, + "num_input_tokens_seen": 127653135, + "router_z_loss_clip": 0.65283203, + "router_z_loss_mlp": 0.20092773, + "step": 4485, + "time_per_iteration": 2.377182960510254 + }, + { + "auxiliary_loss_clip": 0.01091062, + "auxiliary_loss_mlp": 0.01030744, + "balance_loss_clip": 1.03282118, + "balance_loss_mlp": 1.01399469, + "epoch": 0.13017236376298533, + "flos": 20734113767040.0, + "grad_norm": 1.9126756774488265, + "language_loss": 0.71840346, + "learning_rate": 3.895670271966412e-06, + "loss": 0.73962152, + "num_input_tokens_seen": 127668445, + "router_z_loss_clip": 0.58300781, + "router_z_loss_mlp": 0.1673584, + "step": 4486, + "time_per_iteration": 2.418581962585449 + }, + { + "auxiliary_loss_clip": 0.01025989, + "auxiliary_loss_mlp": 0.01003634, + "balance_loss_clip": 1.01577365, + "balance_loss_mlp": 1.00226331, + "epoch": 0.13020138123150135, + "flos": 53688141060480.0, + "grad_norm": 0.7086214519626531, + "language_loss": 0.4854731, + "learning_rate": 3.895610348726063e-06, + "loss": 0.50576931, + "num_input_tokens_seen": 127733515, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01373291, + "step": 4487, + "time_per_iteration": 3.119011878967285 + }, + { + "auxiliary_loss_clip": 0.0110178, + "auxiliary_loss_mlp": 0.01044698, + "balance_loss_clip": 1.03527141, + "balance_loss_mlp": 1.02309728, + "epoch": 0.1302303987000174, + "flos": 30402506511360.0, + "grad_norm": 2.646602537357973, + "language_loss": 1.10035992, + "learning_rate": 3.8955504087429175e-06, + "loss": 1.12182474, + "num_input_tokens_seen": 127755760, + "router_z_loss_clip": 0.66455078, + "router_z_loss_mlp": 0.21582031, + "step": 4488, + "time_per_iteration": 2.755805253982544 + }, + { + "auxiliary_loss_clip": 0.01108635, + "auxiliary_loss_mlp": 0.01042779, + "balance_loss_clip": 1.03934526, + "balance_loss_mlp": 1.0205586, + "epoch": 0.13025941616853345, + "flos": 29053113396480.0, + "grad_norm": 2.9074486561384245, + "language_loss": 0.8326683, + "learning_rate": 3.895490452017503e-06, + "loss": 0.85418242, + "num_input_tokens_seen": 127769350, + "router_z_loss_clip": 0.69335938, + "router_z_loss_mlp": 0.22216797, + "step": 4489, + "time_per_iteration": 2.473597764968872 + }, + { + "auxiliary_loss_clip": 0.01024989, + "auxiliary_loss_mlp": 0.01000716, + "balance_loss_clip": 1.01440072, + "balance_loss_mlp": 0.99952418, + "epoch": 0.1302884336370495, + "flos": 69958539947520.0, + "grad_norm": 0.6742838852318384, + "language_loss": 0.53441972, + "learning_rate": 3.895430478550349e-06, + "loss": 0.55467677, + "num_input_tokens_seen": 127829640, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.01190186, + "step": 4490, + "time_per_iteration": 3.026629686355591 + }, + { + "auxiliary_loss_clip": 0.0102493, + "auxiliary_loss_mlp": 0.01006924, + "balance_loss_clip": 1.01456499, + "balance_loss_mlp": 1.00569606, + "epoch": 0.13031745110556556, + "flos": 59084212331520.0, + "grad_norm": 0.6858620695914016, + "language_loss": 0.48362786, + "learning_rate": 3.8953704883419875e-06, + "loss": 0.50394636, + "num_input_tokens_seen": 127889880, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01226807, + "step": 4491, + "time_per_iteration": 5.18927264213562 + }, + { + "auxiliary_loss_clip": 0.01101341, + "auxiliary_loss_mlp": 0.01045625, + "balance_loss_clip": 1.03299546, + "balance_loss_mlp": 1.02459645, + "epoch": 0.1303464685740816, + "flos": 31972306239360.0, + "grad_norm": 3.232993743860932, + "language_loss": 0.88653862, + "learning_rate": 3.895310481392946e-06, + "loss": 0.90800828, + "num_input_tokens_seen": 127911220, + "router_z_loss_clip": 0.68408203, + "router_z_loss_mlp": 0.21032715, + "step": 4492, + "time_per_iteration": 2.617366313934326 + }, + { + "auxiliary_loss_clip": 0.01102263, + "auxiliary_loss_mlp": 0.01041986, + "balance_loss_clip": 1.03674352, + "balance_loss_mlp": 1.02230418, + "epoch": 0.13037548604259763, + "flos": 32116253241600.0, + "grad_norm": 3.003718854297871, + "language_loss": 1.1117003, + "learning_rate": 3.895250457703756e-06, + "loss": 1.13314283, + "num_input_tokens_seen": 127927425, + "router_z_loss_clip": 0.65478516, + "router_z_loss_mlp": 0.19677734, + "step": 4493, + "time_per_iteration": 4.80561375617981 + }, + { + "auxiliary_loss_clip": 0.0102278, + "auxiliary_loss_mlp": 0.01001101, + "balance_loss_clip": 1.01224351, + "balance_loss_mlp": 0.99979568, + "epoch": 0.13040450351111368, + "flos": 65540665549440.0, + "grad_norm": 0.7020050981138065, + "language_loss": 0.52586538, + "learning_rate": 3.895190417274947e-06, + "loss": 0.54610419, + "num_input_tokens_seen": 127986005, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01306152, + "step": 4494, + "time_per_iteration": 3.2032699584960938 + }, + { + "auxiliary_loss_clip": 0.0110459, + "auxiliary_loss_mlp": 0.01046203, + "balance_loss_clip": 1.0387342, + "balance_loss_mlp": 1.02644992, + "epoch": 0.13043352097962974, + "flos": 19310635013760.0, + "grad_norm": 4.0215741558510185, + "language_loss": 1.00381267, + "learning_rate": 3.895130360107048e-06, + "loss": 1.02532053, + "num_input_tokens_seen": 127999415, + "router_z_loss_clip": 0.65917969, + "router_z_loss_mlp": 0.19744873, + "step": 4495, + "time_per_iteration": 2.397193670272827 + }, + { + "auxiliary_loss_clip": 0.01024139, + "auxiliary_loss_mlp": 0.01003369, + "balance_loss_clip": 1.01365566, + "balance_loss_mlp": 1.00218308, + "epoch": 0.1304625384481458, + "flos": 63788968304640.0, + "grad_norm": 0.6849106230754145, + "language_loss": 0.45635512, + "learning_rate": 3.895070286200592e-06, + "loss": 0.47663021, + "num_input_tokens_seen": 128061475, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01184082, + "step": 4496, + "time_per_iteration": 3.0486271381378174 + }, + { + "auxiliary_loss_clip": 0.01100477, + "auxiliary_loss_mlp": 0.01051041, + "balance_loss_clip": 1.03373432, + "balance_loss_mlp": 1.03053665, + "epoch": 0.13049155591666184, + "flos": 33578031623040.0, + "grad_norm": 2.4154334650435993, + "language_loss": 0.9836846, + "learning_rate": 3.895010195556108e-06, + "loss": 1.00519979, + "num_input_tokens_seen": 128079455, + "router_z_loss_clip": 0.66699219, + "router_z_loss_mlp": 0.20507812, + "step": 4497, + "time_per_iteration": 2.6701345443725586 + }, + { + "auxiliary_loss_clip": 0.01102127, + "auxiliary_loss_mlp": 0.01047662, + "balance_loss_clip": 1.03366089, + "balance_loss_mlp": 1.02578163, + "epoch": 0.1305205733851779, + "flos": 11613508254720.0, + "grad_norm": 2.361775956081625, + "language_loss": 0.7372759, + "learning_rate": 3.894950088174127e-06, + "loss": 0.7587738, + "num_input_tokens_seen": 128091135, + "router_z_loss_clip": 0.68457031, + "router_z_loss_mlp": 0.21893311, + "step": 4498, + "time_per_iteration": 4.85748815536499 + }, + { + "auxiliary_loss_clip": 0.0110332, + "auxiliary_loss_mlp": 0.01050695, + "balance_loss_clip": 1.03751159, + "balance_loss_mlp": 1.03095365, + "epoch": 0.13054959085369391, + "flos": 14172013175040.0, + "grad_norm": 2.7847944655666743, + "language_loss": 0.78617305, + "learning_rate": 3.89488996405518e-06, + "loss": 0.80771321, + "num_input_tokens_seen": 128103725, + "router_z_loss_clip": 0.65820312, + "router_z_loss_mlp": 0.19750977, + "step": 4499, + "time_per_iteration": 2.362063407897949 + }, + { + "auxiliary_loss_clip": 0.01094586, + "auxiliary_loss_mlp": 0.01042097, + "balance_loss_clip": 1.03225958, + "balance_loss_mlp": 1.02184391, + "epoch": 0.13057860832220997, + "flos": 26097226848000.0, + "grad_norm": 2.4787497338284274, + "language_loss": 0.92257643, + "learning_rate": 3.894829823199799e-06, + "loss": 0.9439432, + "num_input_tokens_seen": 128121515, + "router_z_loss_clip": 0.62304688, + "router_z_loss_mlp": 0.20251465, + "step": 4500, + "time_per_iteration": 4.935269832611084 + }, + { + "auxiliary_loss_clip": 0.01095134, + "auxiliary_loss_mlp": 0.01040131, + "balance_loss_clip": 1.03340387, + "balance_loss_mlp": 1.02060485, + "epoch": 0.13060762579072602, + "flos": 28796152723200.0, + "grad_norm": 2.382953612587127, + "language_loss": 0.74098158, + "learning_rate": 3.8947696656085135e-06, + "loss": 0.76233423, + "num_input_tokens_seen": 128136800, + "router_z_loss_clip": 0.6171875, + "router_z_loss_mlp": 0.19543457, + "step": 4501, + "time_per_iteration": 2.5091989040374756 + }, + { + "auxiliary_loss_clip": 0.01097633, + "auxiliary_loss_mlp": 0.01044033, + "balance_loss_clip": 1.03365898, + "balance_loss_mlp": 1.02397037, + "epoch": 0.13063664325924207, + "flos": 17305572965760.0, + "grad_norm": 2.8887863545388646, + "language_loss": 0.73840582, + "learning_rate": 3.894709491281855e-06, + "loss": 0.75982255, + "num_input_tokens_seen": 128150485, + "router_z_loss_clip": 0.64013672, + "router_z_loss_mlp": 0.20031738, + "step": 4502, + "time_per_iteration": 2.497772216796875 + }, + { + "auxiliary_loss_clip": 0.01018562, + "auxiliary_loss_mlp": 0.00999711, + "balance_loss_clip": 1.00845981, + "balance_loss_mlp": 0.99838746, + "epoch": 0.13066566072775812, + "flos": 51168529261440.0, + "grad_norm": 0.7146953140483665, + "language_loss": 0.55266064, + "learning_rate": 3.894649300220356e-06, + "loss": 0.57284337, + "num_input_tokens_seen": 128210900, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.01324463, + "step": 4503, + "time_per_iteration": 3.00870418548584 + }, + { + "auxiliary_loss_clip": 0.01100412, + "auxiliary_loss_mlp": 0.01036677, + "balance_loss_clip": 1.03341579, + "balance_loss_mlp": 1.01648927, + "epoch": 0.13069467819627414, + "flos": 28908468167040.0, + "grad_norm": 2.3678074340493054, + "language_loss": 0.79751945, + "learning_rate": 3.894589092424549e-06, + "loss": 0.81889033, + "num_input_tokens_seen": 128227930, + "router_z_loss_clip": 0.67041016, + "router_z_loss_mlp": 0.20214844, + "step": 4504, + "time_per_iteration": 2.4470977783203125 + }, + { + "auxiliary_loss_clip": 0.01097522, + "auxiliary_loss_mlp": 0.01050566, + "balance_loss_clip": 1.03159153, + "balance_loss_mlp": 1.02979934, + "epoch": 0.1307236956647902, + "flos": 25548008250240.0, + "grad_norm": 3.503322785110711, + "language_loss": 0.84030241, + "learning_rate": 3.894528867894963e-06, + "loss": 0.86178327, + "num_input_tokens_seen": 128242245, + "router_z_loss_clip": 0.65917969, + "router_z_loss_mlp": 0.20776367, + "step": 4505, + "time_per_iteration": 2.4388184547424316 + }, + { + "auxiliary_loss_clip": 0.01092805, + "auxiliary_loss_mlp": 0.01045485, + "balance_loss_clip": 1.03263998, + "balance_loss_mlp": 1.02476025, + "epoch": 0.13075271313330625, + "flos": 28470063824640.0, + "grad_norm": 1.8298049090176376, + "language_loss": 0.75065124, + "learning_rate": 3.8944686266321314e-06, + "loss": 0.77203417, + "num_input_tokens_seen": 128260510, + "router_z_loss_clip": 0.60083008, + "router_z_loss_mlp": 0.20697021, + "step": 4506, + "time_per_iteration": 2.576658010482788 + }, + { + "auxiliary_loss_clip": 0.01094796, + "auxiliary_loss_mlp": 0.01042985, + "balance_loss_clip": 1.0330472, + "balance_loss_mlp": 1.02249312, + "epoch": 0.1307817306018223, + "flos": 26936259575040.0, + "grad_norm": 2.6718047640926708, + "language_loss": 0.7988978, + "learning_rate": 3.894408368636586e-06, + "loss": 0.82027566, + "num_input_tokens_seen": 128274985, + "router_z_loss_clip": 0.6171875, + "router_z_loss_mlp": 0.20495605, + "step": 4507, + "time_per_iteration": 2.4164323806762695 + }, + { + "auxiliary_loss_clip": 0.01102205, + "auxiliary_loss_mlp": 0.01046165, + "balance_loss_clip": 1.03482437, + "balance_loss_mlp": 1.02527988, + "epoch": 0.13081074807033835, + "flos": 23213645458560.0, + "grad_norm": 2.318329090700599, + "language_loss": 0.87152606, + "learning_rate": 3.89434809390886e-06, + "loss": 0.89300972, + "num_input_tokens_seen": 128289605, + "router_z_loss_clip": 0.67407227, + "router_z_loss_mlp": 0.2088623, + "step": 4508, + "time_per_iteration": 2.461906909942627 + }, + { + "auxiliary_loss_clip": 0.01107387, + "auxiliary_loss_mlp": 0.01049124, + "balance_loss_clip": 1.03448081, + "balance_loss_mlp": 1.0277741, + "epoch": 0.1308397655388544, + "flos": 21499025944320.0, + "grad_norm": 2.9310145001736205, + "language_loss": 0.86354268, + "learning_rate": 3.894287802449485e-06, + "loss": 0.88510776, + "num_input_tokens_seen": 128303010, + "router_z_loss_clip": 0.72949219, + "router_z_loss_mlp": 0.21350098, + "step": 4509, + "time_per_iteration": 2.36212158203125 + }, + { + "auxiliary_loss_clip": 0.01017741, + "auxiliary_loss_mlp": 0.01005819, + "balance_loss_clip": 1.00709784, + "balance_loss_mlp": 1.00413787, + "epoch": 0.13086878300737043, + "flos": 66601326787200.0, + "grad_norm": 0.7508154581638191, + "language_loss": 0.51985204, + "learning_rate": 3.894227494258995e-06, + "loss": 0.5400877, + "num_input_tokens_seen": 128359000, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.0168457, + "step": 4510, + "time_per_iteration": 3.0131676197052 + }, + { + "auxiliary_loss_clip": 0.01017711, + "auxiliary_loss_mlp": 0.01007478, + "balance_loss_clip": 1.00713134, + "balance_loss_mlp": 1.00558293, + "epoch": 0.13089780047588648, + "flos": 71968699054080.0, + "grad_norm": 0.6532704684711602, + "language_loss": 0.48906362, + "learning_rate": 3.894167169337919e-06, + "loss": 0.50931561, + "num_input_tokens_seen": 128419425, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.0189209, + "step": 4511, + "time_per_iteration": 3.054389238357544 + }, + { + "auxiliary_loss_clip": 0.01016584, + "auxiliary_loss_mlp": 0.0100515, + "balance_loss_clip": 1.00559032, + "balance_loss_mlp": 1.00362456, + "epoch": 0.13092681794440253, + "flos": 55217965415040.0, + "grad_norm": 0.6341205640687341, + "language_loss": 0.45924464, + "learning_rate": 3.894106827686793e-06, + "loss": 0.47946197, + "num_input_tokens_seen": 128479480, + "router_z_loss_clip": 0.11035156, + "router_z_loss_mlp": 0.01525879, + "step": 4512, + "time_per_iteration": 2.93023943901062 + }, + { + "auxiliary_loss_clip": 0.01017273, + "auxiliary_loss_mlp": 0.01003721, + "balance_loss_clip": 1.00642586, + "balance_loss_mlp": 1.00213563, + "epoch": 0.13095583541291858, + "flos": 66054726541440.0, + "grad_norm": 0.6737991672927551, + "language_loss": 0.52343214, + "learning_rate": 3.8940464693061484e-06, + "loss": 0.54364204, + "num_input_tokens_seen": 128545160, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01586914, + "step": 4513, + "time_per_iteration": 3.111896514892578 + }, + { + "auxiliary_loss_clip": 0.01016486, + "auxiliary_loss_mlp": 0.01006949, + "balance_loss_clip": 1.00590122, + "balance_loss_mlp": 1.00538695, + "epoch": 0.13098485288143463, + "flos": 74774598935040.0, + "grad_norm": 0.7077647645584223, + "language_loss": 0.54181343, + "learning_rate": 3.893986094196519e-06, + "loss": 0.56204778, + "num_input_tokens_seen": 128612160, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.01556396, + "step": 4514, + "time_per_iteration": 3.240156412124634 + }, + { + "auxiliary_loss_clip": 0.01017858, + "auxiliary_loss_mlp": 0.01005144, + "balance_loss_clip": 1.00712752, + "balance_loss_mlp": 1.00361836, + "epoch": 0.13101387034995068, + "flos": 59470666704000.0, + "grad_norm": 0.7043219406807943, + "language_loss": 0.49664909, + "learning_rate": 3.893925702358439e-06, + "loss": 0.51687908, + "num_input_tokens_seen": 128661235, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01525879, + "step": 4515, + "time_per_iteration": 2.759230375289917 + }, + { + "auxiliary_loss_clip": 0.01102552, + "auxiliary_loss_mlp": 0.0104179, + "balance_loss_clip": 1.03548884, + "balance_loss_mlp": 1.0193553, + "epoch": 0.1310428878184667, + "flos": 19236898488960.0, + "grad_norm": 2.50825148774103, + "language_loss": 0.82162285, + "learning_rate": 3.893865293792441e-06, + "loss": 0.84306628, + "num_input_tokens_seen": 128674475, + "router_z_loss_clip": 0.67016602, + "router_z_loss_mlp": 0.2244873, + "step": 4516, + "time_per_iteration": 2.469165563583374 + }, + { + "auxiliary_loss_clip": 0.01018147, + "auxiliary_loss_mlp": 0.0100182, + "balance_loss_clip": 1.00697553, + "balance_loss_mlp": 1.00027072, + "epoch": 0.13107190528698276, + "flos": 67144296251520.0, + "grad_norm": 0.6732785761946304, + "language_loss": 0.50702584, + "learning_rate": 3.893804868499058e-06, + "loss": 0.52722549, + "num_input_tokens_seen": 128735720, + "router_z_loss_clip": 0.11181641, + "router_z_loss_mlp": 0.01550293, + "step": 4517, + "time_per_iteration": 2.944666862487793 + }, + { + "auxiliary_loss_clip": 0.01108584, + "auxiliary_loss_mlp": 0.01045993, + "balance_loss_clip": 1.04146576, + "balance_loss_mlp": 1.0258112, + "epoch": 0.1311009227554988, + "flos": 30257372522880.0, + "grad_norm": 2.404140844827668, + "language_loss": 0.87948382, + "learning_rate": 3.893744426478823e-06, + "loss": 0.90102965, + "num_input_tokens_seen": 128752380, + "router_z_loss_clip": 0.67041016, + "router_z_loss_mlp": 0.20166016, + "step": 4518, + "time_per_iteration": 2.803813934326172 + }, + { + "auxiliary_loss_clip": 0.01103363, + "auxiliary_loss_mlp": 0.01036873, + "balance_loss_clip": 1.03734207, + "balance_loss_mlp": 1.01483166, + "epoch": 0.13112994022401486, + "flos": 24379395488640.0, + "grad_norm": 2.8361028160528376, + "language_loss": 0.78760189, + "learning_rate": 3.8936839677322715e-06, + "loss": 0.80900425, + "num_input_tokens_seen": 128764735, + "router_z_loss_clip": 0.66064453, + "router_z_loss_mlp": 0.22058105, + "step": 4519, + "time_per_iteration": 2.419874429702759 + }, + { + "auxiliary_loss_clip": 0.01021227, + "auxiliary_loss_mlp": 0.01006758, + "balance_loss_clip": 1.0102005, + "balance_loss_mlp": 1.00535095, + "epoch": 0.13115895769253091, + "flos": 69259369593600.0, + "grad_norm": 0.6135398566742074, + "language_loss": 0.49141443, + "learning_rate": 3.893623492259937e-06, + "loss": 0.51169431, + "num_input_tokens_seen": 128827115, + "router_z_loss_clip": 0.11035156, + "router_z_loss_mlp": 0.01403809, + "step": 4520, + "time_per_iteration": 3.0552165508270264 + }, + { + "auxiliary_loss_clip": 0.01020637, + "auxiliary_loss_mlp": 0.01017, + "balance_loss_clip": 1.01012468, + "balance_loss_mlp": 1.01573062, + "epoch": 0.13118797516104694, + "flos": 56162645516160.0, + "grad_norm": 0.7260971524646511, + "language_loss": 0.44912902, + "learning_rate": 3.893563000062354e-06, + "loss": 0.46950537, + "num_input_tokens_seen": 128877715, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01269531, + "step": 4521, + "time_per_iteration": 2.870774507522583 + }, + { + "auxiliary_loss_clip": 0.01016618, + "auxiliary_loss_mlp": 0.01006364, + "balance_loss_clip": 1.00669694, + "balance_loss_mlp": 1.0051893, + "epoch": 0.131216992629563, + "flos": 51164828657280.0, + "grad_norm": 0.6747617682732928, + "language_loss": 0.49729946, + "learning_rate": 3.893502491140055e-06, + "loss": 0.51752925, + "num_input_tokens_seen": 128934735, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01171875, + "step": 4522, + "time_per_iteration": 2.8705332279205322 + }, + { + "auxiliary_loss_clip": 0.01086718, + "auxiliary_loss_mlp": 0.01053253, + "balance_loss_clip": 1.02821279, + "balance_loss_mlp": 1.0337975, + "epoch": 0.13124601009807904, + "flos": 12814031865600.0, + "grad_norm": 3.7929535540323847, + "language_loss": 1.10912013, + "learning_rate": 3.8934419654935775e-06, + "loss": 1.13051975, + "num_input_tokens_seen": 128944255, + "router_z_loss_clip": 0.58496094, + "router_z_loss_mlp": 0.19470215, + "step": 4523, + "time_per_iteration": 2.3435401916503906 + }, + { + "auxiliary_loss_clip": 0.01091647, + "auxiliary_loss_mlp": 0.01042948, + "balance_loss_clip": 1.03130889, + "balance_loss_mlp": 1.02379072, + "epoch": 0.1312750275665951, + "flos": 38283730202880.0, + "grad_norm": 2.412245381309537, + "language_loss": 0.64477229, + "learning_rate": 3.893381423123453e-06, + "loss": 0.66611826, + "num_input_tokens_seen": 128961260, + "router_z_loss_clip": 0.60375977, + "router_z_loss_mlp": 0.19152832, + "step": 4524, + "time_per_iteration": 2.5578877925872803 + }, + { + "auxiliary_loss_clip": 0.01020344, + "auxiliary_loss_mlp": 0.01003911, + "balance_loss_clip": 1.01037729, + "balance_loss_mlp": 1.00258195, + "epoch": 0.13130404503511114, + "flos": 61329756890880.0, + "grad_norm": 0.7362610346189915, + "language_loss": 0.48330039, + "learning_rate": 3.893320864030219e-06, + "loss": 0.5035429, + "num_input_tokens_seen": 129013985, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01330566, + "step": 4525, + "time_per_iteration": 2.782029867172241 + }, + { + "auxiliary_loss_clip": 0.0109321, + "auxiliary_loss_mlp": 0.01042224, + "balance_loss_clip": 1.03219092, + "balance_loss_mlp": 1.0204922, + "epoch": 0.1313330625036272, + "flos": 16391302525440.0, + "grad_norm": 2.5968741631994887, + "language_loss": 0.91689789, + "learning_rate": 3.893260288214407e-06, + "loss": 0.93825221, + "num_input_tokens_seen": 129026725, + "router_z_loss_clip": 0.60986328, + "router_z_loss_mlp": 0.21722412, + "step": 4526, + "time_per_iteration": 2.3809759616851807 + }, + { + "auxiliary_loss_clip": 0.0109874, + "auxiliary_loss_mlp": 0.01051947, + "balance_loss_clip": 1.03739572, + "balance_loss_mlp": 1.03456032, + "epoch": 0.13136207997214322, + "flos": 32117789341440.0, + "grad_norm": 1.9329903530488608, + "language_loss": 0.65677088, + "learning_rate": 3.893199695676555e-06, + "loss": 0.67827773, + "num_input_tokens_seen": 129044085, + "router_z_loss_clip": 0.61328125, + "router_z_loss_mlp": 0.1739502, + "step": 4527, + "time_per_iteration": 2.470684766769409 + }, + { + "auxiliary_loss_clip": 0.01105398, + "auxiliary_loss_mlp": 0.01048939, + "balance_loss_clip": 1.03711677, + "balance_loss_mlp": 1.02627134, + "epoch": 0.13139109744065927, + "flos": 24928928288640.0, + "grad_norm": 2.345778970819389, + "language_loss": 1.03067505, + "learning_rate": 3.893139086417198e-06, + "loss": 1.05221844, + "num_input_tokens_seen": 129059590, + "router_z_loss_clip": 0.68334961, + "router_z_loss_mlp": 0.22662354, + "step": 4528, + "time_per_iteration": 2.4424071311950684 + }, + { + "auxiliary_loss_clip": 0.01105347, + "auxiliary_loss_mlp": 0.01039041, + "balance_loss_clip": 1.0378716, + "balance_loss_mlp": 1.0177387, + "epoch": 0.13142011490917532, + "flos": 50869360753920.0, + "grad_norm": 4.395422561973628, + "language_loss": 0.80595434, + "learning_rate": 3.89307846043687e-06, + "loss": 0.82739818, + "num_input_tokens_seen": 129078745, + "router_z_loss_clip": 0.67480469, + "router_z_loss_mlp": 0.21313477, + "step": 4529, + "time_per_iteration": 2.7208640575408936 + }, + { + "auxiliary_loss_clip": 0.01102947, + "auxiliary_loss_mlp": 0.01046567, + "balance_loss_clip": 1.0374496, + "balance_loss_mlp": 1.02609873, + "epoch": 0.13144913237769137, + "flos": 31200830726400.0, + "grad_norm": 2.4767936788621148, + "language_loss": 0.87219238, + "learning_rate": 3.893017817736107e-06, + "loss": 0.89368755, + "num_input_tokens_seen": 129094070, + "router_z_loss_clip": 0.65527344, + "router_z_loss_mlp": 0.20471191, + "step": 4530, + "time_per_iteration": 2.479480028152466 + }, + { + "auxiliary_loss_clip": 0.01101435, + "auxiliary_loss_mlp": 0.01039045, + "balance_loss_clip": 1.03991663, + "balance_loss_mlp": 1.0211401, + "epoch": 0.13147814984620743, + "flos": 15732386835840.0, + "grad_norm": 4.336356115703832, + "language_loss": 0.86100703, + "learning_rate": 3.892957158315444e-06, + "loss": 0.88241184, + "num_input_tokens_seen": 129104315, + "router_z_loss_clip": 0.61572266, + "router_z_loss_mlp": 0.17907715, + "step": 4531, + "time_per_iteration": 2.590381383895874 + }, + { + "auxiliary_loss_clip": 0.0104467, + "auxiliary_loss_mlp": 0.01003746, + "balance_loss_clip": 1.0313189, + "balance_loss_mlp": 1.00239861, + "epoch": 0.13150716731472345, + "flos": 53542902337920.0, + "grad_norm": 0.6587749724799976, + "language_loss": 0.47994018, + "learning_rate": 3.892896482175418e-06, + "loss": 0.50042439, + "num_input_tokens_seen": 129162430, + "router_z_loss_clip": 0.1328125, + "router_z_loss_mlp": 0.01348877, + "step": 4532, + "time_per_iteration": 2.961604595184326 + }, + { + "auxiliary_loss_clip": 0.01104153, + "auxiliary_loss_mlp": 0.01041027, + "balance_loss_clip": 1.03881788, + "balance_loss_mlp": 1.02274013, + "epoch": 0.1315361847832395, + "flos": 30583217041920.0, + "grad_norm": 2.691402057385783, + "language_loss": 0.86728281, + "learning_rate": 3.8928357893165645e-06, + "loss": 0.88873458, + "num_input_tokens_seen": 129177395, + "router_z_loss_clip": 0.65356445, + "router_z_loss_mlp": 0.18280029, + "step": 4533, + "time_per_iteration": 2.554072141647339 + }, + { + "auxiliary_loss_clip": 0.01106508, + "auxiliary_loss_mlp": 0.01033781, + "balance_loss_clip": 1.04163158, + "balance_loss_mlp": 1.01445735, + "epoch": 0.13156520225175555, + "flos": 11866314476160.0, + "grad_norm": 2.4439813154859378, + "language_loss": 0.85194194, + "learning_rate": 3.892775079739418e-06, + "loss": 0.87334484, + "num_input_tokens_seen": 129189225, + "router_z_loss_clip": 0.6484375, + "router_z_loss_mlp": 0.1932373, + "step": 4534, + "time_per_iteration": 2.404583692550659 + }, + { + "auxiliary_loss_clip": 0.01108802, + "auxiliary_loss_mlp": 0.01044619, + "balance_loss_clip": 1.03878355, + "balance_loss_mlp": 1.02202868, + "epoch": 0.1315942197202716, + "flos": 30109759827840.0, + "grad_norm": 2.288595397068352, + "language_loss": 0.84877717, + "learning_rate": 3.892714353444518e-06, + "loss": 0.87031132, + "num_input_tokens_seen": 129205630, + "router_z_loss_clip": 0.70019531, + "router_z_loss_mlp": 0.22619629, + "step": 4535, + "time_per_iteration": 2.4948763847351074 + }, + { + "auxiliary_loss_clip": 0.01096089, + "auxiliary_loss_mlp": 0.01039185, + "balance_loss_clip": 1.03644264, + "balance_loss_mlp": 1.02144623, + "epoch": 0.13162323718878766, + "flos": 28945057138560.0, + "grad_norm": 2.2183353232265346, + "language_loss": 0.89804357, + "learning_rate": 3.892653610432398e-06, + "loss": 0.91939628, + "num_input_tokens_seen": 129224715, + "router_z_loss_clip": 0.59643555, + "router_z_loss_mlp": 0.17749023, + "step": 4536, + "time_per_iteration": 2.523699998855591 + }, + { + "auxiliary_loss_clip": 0.01097607, + "auxiliary_loss_mlp": 0.01034659, + "balance_loss_clip": 1.0339663, + "balance_loss_mlp": 1.01379716, + "epoch": 0.1316522546573037, + "flos": 11392927084800.0, + "grad_norm": 2.6953375801385464, + "language_loss": 0.75353801, + "learning_rate": 3.892592850703595e-06, + "loss": 0.77486068, + "num_input_tokens_seen": 129235630, + "router_z_loss_clip": 0.63671875, + "router_z_loss_mlp": 0.20874023, + "step": 4537, + "time_per_iteration": 2.407270669937134 + }, + { + "auxiliary_loss_clip": 0.01040187, + "auxiliary_loss_mlp": 0.01006361, + "balance_loss_clip": 1.02736783, + "balance_loss_mlp": 1.00488329, + "epoch": 0.13168127212581973, + "flos": 68889046135680.0, + "grad_norm": 0.6603615978453374, + "language_loss": 0.48298424, + "learning_rate": 3.892532074258647e-06, + "loss": 0.50344974, + "num_input_tokens_seen": 129298350, + "router_z_loss_clip": 0.12890625, + "router_z_loss_mlp": 0.01477051, + "step": 4538, + "time_per_iteration": 3.0795536041259766 + }, + { + "auxiliary_loss_clip": 0.01034413, + "auxiliary_loss_mlp": 0.01001648, + "balance_loss_clip": 1.02275896, + "balance_loss_mlp": 1.00013375, + "epoch": 0.13171028959433578, + "flos": 62623078629120.0, + "grad_norm": 0.6624358799777688, + "language_loss": 0.49325323, + "learning_rate": 3.892471281098089e-06, + "loss": 0.51361382, + "num_input_tokens_seen": 129361005, + "router_z_loss_clip": 0.11621094, + "router_z_loss_mlp": 0.01513672, + "step": 4539, + "time_per_iteration": 3.144362688064575 + }, + { + "auxiliary_loss_clip": 0.01096897, + "auxiliary_loss_mlp": 0.01042135, + "balance_loss_clip": 1.03375864, + "balance_loss_mlp": 1.02226281, + "epoch": 0.13173930706285183, + "flos": 16755097559040.0, + "grad_norm": 2.5522813327271083, + "language_loss": 0.60039687, + "learning_rate": 3.892410471222459e-06, + "loss": 0.62178731, + "num_input_tokens_seen": 129376220, + "router_z_loss_clip": 0.63134766, + "router_z_loss_mlp": 0.19885254, + "step": 4540, + "time_per_iteration": 2.365953207015991 + }, + { + "auxiliary_loss_clip": 0.01025633, + "auxiliary_loss_mlp": 0.01004131, + "balance_loss_clip": 1.01529157, + "balance_loss_mlp": 1.00277781, + "epoch": 0.1317683245313679, + "flos": 63094441161600.0, + "grad_norm": 0.6518849932358053, + "language_loss": 0.45585221, + "learning_rate": 3.892349644632295e-06, + "loss": 0.47614986, + "num_input_tokens_seen": 129437580, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.0135498, + "step": 4541, + "time_per_iteration": 3.110562562942505 + }, + { + "auxiliary_loss_clip": 0.01020804, + "auxiliary_loss_mlp": 0.01001326, + "balance_loss_clip": 1.01136065, + "balance_loss_mlp": 0.99990112, + "epoch": 0.13179734199988394, + "flos": 70246154661120.0, + "grad_norm": 0.6665441424593854, + "language_loss": 0.48650843, + "learning_rate": 3.8922888013281324e-06, + "loss": 0.50672966, + "num_input_tokens_seen": 129496185, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01422119, + "step": 4542, + "time_per_iteration": 3.0151047706604004 + }, + { + "auxiliary_loss_clip": 0.01102573, + "auxiliary_loss_mlp": 0.01036929, + "balance_loss_clip": 1.03588712, + "balance_loss_mlp": 1.01646042, + "epoch": 0.1318263594684, + "flos": 34197635255040.0, + "grad_norm": 3.156676043248439, + "language_loss": 1.065745, + "learning_rate": 3.892227941310509e-06, + "loss": 1.08713996, + "num_input_tokens_seen": 129513435, + "router_z_loss_clip": 0.66650391, + "router_z_loss_mlp": 0.20471191, + "step": 4543, + "time_per_iteration": 2.5134122371673584 + }, + { + "auxiliary_loss_clip": 0.01092985, + "auxiliary_loss_mlp": 0.01034608, + "balance_loss_clip": 1.03402686, + "balance_loss_mlp": 1.01759124, + "epoch": 0.131855376936916, + "flos": 24125681571840.0, + "grad_norm": 2.7435107476199403, + "language_loss": 0.81871831, + "learning_rate": 3.892167064579963e-06, + "loss": 0.83999431, + "num_input_tokens_seen": 129525900, + "router_z_loss_clip": 0.59008789, + "router_z_loss_mlp": 0.17016602, + "step": 4544, + "time_per_iteration": 2.71865177154541 + }, + { + "auxiliary_loss_clip": 0.01101845, + "auxiliary_loss_mlp": 0.01041397, + "balance_loss_clip": 1.03463936, + "balance_loss_mlp": 1.02139318, + "epoch": 0.13188439440543206, + "flos": 25549998197760.0, + "grad_norm": 3.927610121247547, + "language_loss": 0.90443194, + "learning_rate": 3.892106171137032e-06, + "loss": 0.92586428, + "num_input_tokens_seen": 129539915, + "router_z_loss_clip": 0.67138672, + "router_z_loss_mlp": 0.20007324, + "step": 4545, + "time_per_iteration": 2.388869047164917 + }, + { + "auxiliary_loss_clip": 0.01100627, + "auxiliary_loss_mlp": 0.01042092, + "balance_loss_clip": 1.03944767, + "balance_loss_mlp": 1.02454412, + "epoch": 0.13191341187394812, + "flos": 27883418382720.0, + "grad_norm": 2.2491639895400843, + "language_loss": 0.9092412, + "learning_rate": 3.892045260982254e-06, + "loss": 0.93066841, + "num_input_tokens_seen": 129556010, + "router_z_loss_clip": 0.61230469, + "router_z_loss_mlp": 0.17553711, + "step": 4546, + "time_per_iteration": 2.503086566925049 + }, + { + "auxiliary_loss_clip": 0.01095956, + "auxiliary_loss_mlp": 0.01034934, + "balance_loss_clip": 1.03613138, + "balance_loss_mlp": 1.0166831, + "epoch": 0.13194242934246417, + "flos": 27301067038080.0, + "grad_norm": 1.6516466920016566, + "language_loss": 0.69470716, + "learning_rate": 3.891984334116166e-06, + "loss": 0.71601605, + "num_input_tokens_seen": 129575295, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.18237305, + "step": 4547, + "time_per_iteration": 2.4455695152282715 + }, + { + "auxiliary_loss_clip": 0.01024794, + "auxiliary_loss_mlp": 0.01011298, + "balance_loss_clip": 1.01424432, + "balance_loss_mlp": 1.00984371, + "epoch": 0.13197144681098022, + "flos": 64817858338560.0, + "grad_norm": 0.6219586623508455, + "language_loss": 0.46689236, + "learning_rate": 3.891923390539307e-06, + "loss": 0.48725331, + "num_input_tokens_seen": 129643220, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01452637, + "step": 4548, + "time_per_iteration": 3.0815181732177734 + }, + { + "auxiliary_loss_clip": 0.01096245, + "auxiliary_loss_mlp": 0.01030144, + "balance_loss_clip": 1.03904569, + "balance_loss_mlp": 1.01351488, + "epoch": 0.13200046427949624, + "flos": 23798231130240.0, + "grad_norm": 1.6816817831786077, + "language_loss": 0.6371401, + "learning_rate": 3.8918624302522145e-06, + "loss": 0.65840399, + "num_input_tokens_seen": 129658650, + "router_z_loss_clip": 0.57202148, + "router_z_loss_mlp": 0.1663208, + "step": 4549, + "time_per_iteration": 2.450085401535034 + }, + { + "auxiliary_loss_clip": 0.01102178, + "auxiliary_loss_mlp": 0.01033085, + "balance_loss_clip": 1.03870547, + "balance_loss_mlp": 1.0134151, + "epoch": 0.1320294817480123, + "flos": 43720370340480.0, + "grad_norm": 2.2911008212900437, + "language_loss": 0.79132152, + "learning_rate": 3.891801453255428e-06, + "loss": 0.81267416, + "num_input_tokens_seen": 129677810, + "router_z_loss_clip": 0.63427734, + "router_z_loss_mlp": 0.19665527, + "step": 4550, + "time_per_iteration": 2.6041858196258545 + }, + { + "auxiliary_loss_clip": 0.01025111, + "auxiliary_loss_mlp": 0.01005547, + "balance_loss_clip": 1.01377511, + "balance_loss_mlp": 1.00422966, + "epoch": 0.13205849921652835, + "flos": 65613145265280.0, + "grad_norm": 0.6764813286226634, + "language_loss": 0.4830468, + "learning_rate": 3.891740459549485e-06, + "loss": 0.50335336, + "num_input_tokens_seen": 129735500, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01318359, + "step": 4551, + "time_per_iteration": 2.9399356842041016 + }, + { + "auxiliary_loss_clip": 0.0110086, + "auxiliary_loss_mlp": 0.01030911, + "balance_loss_clip": 1.03879631, + "balance_loss_mlp": 1.01325023, + "epoch": 0.1320875166850444, + "flos": 16610696709120.0, + "grad_norm": 3.0448717254793887, + "language_loss": 0.77640212, + "learning_rate": 3.891679449134925e-06, + "loss": 0.79771984, + "num_input_tokens_seen": 129747200, + "router_z_loss_clip": 0.61962891, + "router_z_loss_mlp": 0.17657471, + "step": 4552, + "time_per_iteration": 2.455352783203125 + }, + { + "auxiliary_loss_clip": 0.01098906, + "auxiliary_loss_mlp": 0.01038682, + "balance_loss_clip": 1.04223824, + "balance_loss_mlp": 1.02102733, + "epoch": 0.13211653415356045, + "flos": 15914877845760.0, + "grad_norm": 2.751021876468583, + "language_loss": 0.73044479, + "learning_rate": 3.891618422012287e-06, + "loss": 0.75182068, + "num_input_tokens_seen": 129758665, + "router_z_loss_clip": 0.56665039, + "router_z_loss_mlp": 0.1763916, + "step": 4553, + "time_per_iteration": 2.383070468902588 + }, + { + "auxiliary_loss_clip": 0.01108068, + "auxiliary_loss_mlp": 0.01032873, + "balance_loss_clip": 1.04194057, + "balance_loss_mlp": 1.01341772, + "epoch": 0.1321455516220765, + "flos": 22306531847040.0, + "grad_norm": 4.167973157841701, + "language_loss": 1.05960679, + "learning_rate": 3.89155737818211e-06, + "loss": 1.08101606, + "num_input_tokens_seen": 129778180, + "router_z_loss_clip": 0.66113281, + "router_z_loss_mlp": 0.19458008, + "step": 4554, + "time_per_iteration": 2.477989673614502 + }, + { + "auxiliary_loss_clip": 0.0110038, + "auxiliary_loss_mlp": 0.01040556, + "balance_loss_clip": 1.04088306, + "balance_loss_mlp": 1.02365756, + "epoch": 0.13217456909059253, + "flos": 35400148813440.0, + "grad_norm": 1.8402128683874104, + "language_loss": 0.62529516, + "learning_rate": 3.891496317644932e-06, + "loss": 0.64670449, + "num_input_tokens_seen": 129800635, + "router_z_loss_clip": 0.59643555, + "router_z_loss_mlp": 0.16906738, + "step": 4555, + "time_per_iteration": 2.5218186378479004 + }, + { + "auxiliary_loss_clip": 0.01104009, + "auxiliary_loss_mlp": 0.01044281, + "balance_loss_clip": 1.03679538, + "balance_loss_mlp": 1.02331233, + "epoch": 0.13220358655910858, + "flos": 12705067912320.0, + "grad_norm": 2.8362696004501955, + "language_loss": 0.90569258, + "learning_rate": 3.891435240401293e-06, + "loss": 0.92717552, + "num_input_tokens_seen": 129813140, + "router_z_loss_clip": 0.67138672, + "router_z_loss_mlp": 0.20959473, + "step": 4556, + "time_per_iteration": 2.4339888095855713 + }, + { + "auxiliary_loss_clip": 0.01108454, + "auxiliary_loss_mlp": 0.01055431, + "balance_loss_clip": 1.039554, + "balance_loss_mlp": 1.03275764, + "epoch": 0.13223260402762463, + "flos": 35444418284160.0, + "grad_norm": 1.9569806937476908, + "language_loss": 0.88323581, + "learning_rate": 3.891374146451733e-06, + "loss": 0.90487468, + "num_input_tokens_seen": 129836005, + "router_z_loss_clip": 0.68847656, + "router_z_loss_mlp": 0.22668457, + "step": 4557, + "time_per_iteration": 2.526367425918579 + }, + { + "auxiliary_loss_clip": 0.01100887, + "auxiliary_loss_mlp": 0.01038786, + "balance_loss_clip": 1.04007721, + "balance_loss_mlp": 1.02178669, + "epoch": 0.13226162149614068, + "flos": 34567644510720.0, + "grad_norm": 2.1584404082699433, + "language_loss": 0.8037281, + "learning_rate": 3.8913130357967915e-06, + "loss": 0.82512486, + "num_input_tokens_seen": 129856165, + "router_z_loss_clip": 0.60791016, + "router_z_loss_mlp": 0.16992188, + "step": 4558, + "time_per_iteration": 2.728001832962036 + }, + { + "auxiliary_loss_clip": 0.01020005, + "auxiliary_loss_mlp": 0.01004522, + "balance_loss_clip": 1.00851548, + "balance_loss_mlp": 1.00324667, + "epoch": 0.13229063896465673, + "flos": 59153410379520.0, + "grad_norm": 0.6917594785878357, + "language_loss": 0.48824632, + "learning_rate": 3.891251908437008e-06, + "loss": 0.50849164, + "num_input_tokens_seen": 129910560, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01275635, + "step": 4559, + "time_per_iteration": 2.8256890773773193 + }, + { + "auxiliary_loss_clip": 0.01094777, + "auxiliary_loss_mlp": 0.01032111, + "balance_loss_clip": 1.03430355, + "balance_loss_mlp": 1.01436114, + "epoch": 0.13231965643317278, + "flos": 21066870735360.0, + "grad_norm": 1.6715762962058842, + "language_loss": 0.62463069, + "learning_rate": 3.8911907643729216e-06, + "loss": 0.64589953, + "num_input_tokens_seen": 129929280, + "router_z_loss_clip": 0.60449219, + "router_z_loss_mlp": 0.17755127, + "step": 4560, + "time_per_iteration": 2.4900593757629395 + }, + { + "auxiliary_loss_clip": 0.01098927, + "auxiliary_loss_mlp": 0.01039695, + "balance_loss_clip": 1.03524256, + "balance_loss_mlp": 1.02037108, + "epoch": 0.1323486739016888, + "flos": 32226578737920.0, + "grad_norm": 2.3640059253049364, + "language_loss": 0.92182869, + "learning_rate": 3.8911296036050736e-06, + "loss": 0.94321501, + "num_input_tokens_seen": 129944165, + "router_z_loss_clip": 0.63720703, + "router_z_loss_mlp": 0.1932373, + "step": 4561, + "time_per_iteration": 2.5501928329467773 + }, + { + "auxiliary_loss_clip": 0.01097633, + "auxiliary_loss_mlp": 0.010362, + "balance_loss_clip": 1.0331279, + "balance_loss_mlp": 1.01524329, + "epoch": 0.13237769137020486, + "flos": 17193187699200.0, + "grad_norm": 2.9378084463879675, + "language_loss": 0.85985625, + "learning_rate": 3.8910684261340035e-06, + "loss": 0.88119459, + "num_input_tokens_seen": 129956605, + "router_z_loss_clip": 0.64453125, + "router_z_loss_mlp": 0.20959473, + "step": 4562, + "time_per_iteration": 2.387925863265991 + }, + { + "auxiliary_loss_clip": 0.01016197, + "auxiliary_loss_mlp": 0.01004005, + "balance_loss_clip": 1.00535893, + "balance_loss_mlp": 1.0027473, + "epoch": 0.1324067088387209, + "flos": 63602916336000.0, + "grad_norm": 0.7541329375022036, + "language_loss": 0.52046692, + "learning_rate": 3.891007231960252e-06, + "loss": 0.54066896, + "num_input_tokens_seen": 130006270, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01257324, + "step": 4563, + "time_per_iteration": 2.8756637573242188 + }, + { + "auxiliary_loss_clip": 0.01095114, + "auxiliary_loss_mlp": 0.01037112, + "balance_loss_clip": 1.03293169, + "balance_loss_mlp": 1.01877809, + "epoch": 0.13243572630723696, + "flos": 38976092841600.0, + "grad_norm": 5.410750787720246, + "language_loss": 0.90525973, + "learning_rate": 3.890946021084359e-06, + "loss": 0.92658192, + "num_input_tokens_seen": 130022560, + "router_z_loss_clip": 0.62109375, + "router_z_loss_mlp": 0.18347168, + "step": 4564, + "time_per_iteration": 2.634753465652466 + }, + { + "auxiliary_loss_clip": 0.01096676, + "auxiliary_loss_mlp": 0.01039867, + "balance_loss_clip": 1.03163171, + "balance_loss_mlp": 1.01912439, + "epoch": 0.132464743775753, + "flos": 31569757729920.0, + "grad_norm": 2.076444535598723, + "language_loss": 0.86810398, + "learning_rate": 3.890884793506865e-06, + "loss": 0.88946939, + "num_input_tokens_seen": 130042530, + "router_z_loss_clip": 0.65136719, + "router_z_loss_mlp": 0.20727539, + "step": 4565, + "time_per_iteration": 2.5124528408050537 + }, + { + "auxiliary_loss_clip": 0.0101421, + "auxiliary_loss_mlp": 0.01001373, + "balance_loss_clip": 1.00424147, + "balance_loss_mlp": 1.00028837, + "epoch": 0.13249376124426904, + "flos": 74765801272320.0, + "grad_norm": 0.6467710733245262, + "language_loss": 0.44873688, + "learning_rate": 3.8908235492283125e-06, + "loss": 0.46889269, + "num_input_tokens_seen": 130107465, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01086426, + "step": 4566, + "time_per_iteration": 5.381329774856567 + }, + { + "auxiliary_loss_clip": 0.01013884, + "auxiliary_loss_mlp": 0.01002005, + "balance_loss_clip": 1.00367403, + "balance_loss_mlp": 1.00084841, + "epoch": 0.1325227787127851, + "flos": 71020457994240.0, + "grad_norm": 0.6662614165787697, + "language_loss": 0.54674828, + "learning_rate": 3.890762288249241e-06, + "loss": 0.56690717, + "num_input_tokens_seen": 130171750, + "router_z_loss_clip": 0.10205078, + "router_z_loss_mlp": 0.01153564, + "step": 4567, + "time_per_iteration": 3.1134417057037354 + }, + { + "auxiliary_loss_clip": 0.01013577, + "auxiliary_loss_mlp": 0.01001444, + "balance_loss_clip": 1.00357604, + "balance_loss_mlp": 1.00032902, + "epoch": 0.13255179618130114, + "flos": 74768245067520.0, + "grad_norm": 0.7088756367720923, + "language_loss": 0.54303133, + "learning_rate": 3.890701010570192e-06, + "loss": 0.56318152, + "num_input_tokens_seen": 130229705, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.01116943, + "step": 4568, + "time_per_iteration": 3.009357452392578 + }, + { + "auxiliary_loss_clip": 0.01098892, + "auxiliary_loss_mlp": 0.01041401, + "balance_loss_clip": 1.0314548, + "balance_loss_mlp": 1.02034283, + "epoch": 0.1325808136498172, + "flos": 74730226596480.0, + "grad_norm": 2.264551927856311, + "language_loss": 0.8397069, + "learning_rate": 3.890639716191706e-06, + "loss": 0.86110985, + "num_input_tokens_seen": 130251850, + "router_z_loss_clip": 0.67456055, + "router_z_loss_mlp": 0.21044922, + "step": 4569, + "time_per_iteration": 5.098620414733887 + }, + { + "auxiliary_loss_clip": 0.01093839, + "auxiliary_loss_mlp": 0.01038273, + "balance_loss_clip": 1.03152871, + "balance_loss_mlp": 1.01785254, + "epoch": 0.13260983111833324, + "flos": 32885843541120.0, + "grad_norm": 2.442458227342697, + "language_loss": 0.74055457, + "learning_rate": 3.890578405114325e-06, + "loss": 0.76187569, + "num_input_tokens_seen": 130269885, + "router_z_loss_clip": 0.62426758, + "router_z_loss_mlp": 0.20422363, + "step": 4570, + "time_per_iteration": 2.4871578216552734 + }, + { + "auxiliary_loss_clip": 0.01014095, + "auxiliary_loss_mlp": 0.01003226, + "balance_loss_clip": 1.00419378, + "balance_loss_mlp": 1.00211096, + "epoch": 0.1326388485868493, + "flos": 69338517379200.0, + "grad_norm": 0.6984813535520423, + "language_loss": 0.46738315, + "learning_rate": 3.890517077338591e-06, + "loss": 0.4875564, + "num_input_tokens_seen": 130329670, + "router_z_loss_clip": 0.09912109, + "router_z_loss_mlp": 0.01116943, + "step": 4571, + "time_per_iteration": 3.020453929901123 + }, + { + "auxiliary_loss_clip": 0.0109113, + "auxiliary_loss_mlp": 0.01037235, + "balance_loss_clip": 1.03208899, + "balance_loss_mlp": 1.01934719, + "epoch": 0.13266786605536532, + "flos": 39812786507520.0, + "grad_norm": 4.885295432643655, + "language_loss": 0.57876188, + "learning_rate": 3.890455732865045e-06, + "loss": 0.6000455, + "num_input_tokens_seen": 130350170, + "router_z_loss_clip": 0.59082031, + "router_z_loss_mlp": 0.17871094, + "step": 4572, + "time_per_iteration": 2.5664315223693848 + }, + { + "auxiliary_loss_clip": 0.01014237, + "auxiliary_loss_mlp": 0.0100102, + "balance_loss_clip": 1.00439036, + "balance_loss_mlp": 0.99992883, + "epoch": 0.13269688352388137, + "flos": 74770584128640.0, + "grad_norm": 0.6751546797031075, + "language_loss": 0.44739792, + "learning_rate": 3.890394371694228e-06, + "loss": 0.46755046, + "num_input_tokens_seen": 130416000, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01092529, + "step": 4573, + "time_per_iteration": 3.181532859802246 + }, + { + "auxiliary_loss_clip": 0.01088757, + "auxiliary_loss_mlp": 0.01044843, + "balance_loss_clip": 1.03123164, + "balance_loss_mlp": 1.02818966, + "epoch": 0.13272590099239742, + "flos": 22850304272640.0, + "grad_norm": 2.237249998617417, + "language_loss": 0.67681277, + "learning_rate": 3.890332993826685e-06, + "loss": 0.69814879, + "num_input_tokens_seen": 130430320, + "router_z_loss_clip": 0.57568359, + "router_z_loss_mlp": 0.16668701, + "step": 4574, + "time_per_iteration": 4.844549179077148 + }, + { + "auxiliary_loss_clip": 0.01014667, + "auxiliary_loss_mlp": 0.0100049, + "balance_loss_clip": 1.00495744, + "balance_loss_mlp": 0.99944103, + "epoch": 0.13275491846091347, + "flos": 69693584572800.0, + "grad_norm": 0.6747572296489043, + "language_loss": 0.49965683, + "learning_rate": 3.890271599262955e-06, + "loss": 0.51980841, + "num_input_tokens_seen": 130486335, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.01049805, + "step": 4575, + "time_per_iteration": 2.9346084594726562 + }, + { + "auxiliary_loss_clip": 0.01093467, + "auxiliary_loss_mlp": 0.01049161, + "balance_loss_clip": 1.03390336, + "balance_loss_mlp": 1.03014731, + "epoch": 0.13278393592942953, + "flos": 35981103703680.0, + "grad_norm": 4.492525598896902, + "language_loss": 0.7705093, + "learning_rate": 3.890210188003581e-06, + "loss": 0.79193556, + "num_input_tokens_seen": 130503095, + "router_z_loss_clip": 0.59619141, + "router_z_loss_mlp": 0.19018555, + "step": 4576, + "time_per_iteration": 4.94526219367981 + }, + { + "auxiliary_loss_clip": 0.01103206, + "auxiliary_loss_mlp": 0.0104807, + "balance_loss_clip": 1.03509128, + "balance_loss_mlp": 1.02714896, + "epoch": 0.13281295339794558, + "flos": 36166562179200.0, + "grad_norm": 2.3402798236262807, + "language_loss": 0.87263203, + "learning_rate": 3.890148760049106e-06, + "loss": 0.89414477, + "num_input_tokens_seen": 130529340, + "router_z_loss_clip": 0.68164062, + "router_z_loss_mlp": 0.20910645, + "step": 4577, + "time_per_iteration": 2.622607707977295 + }, + { + "auxiliary_loss_clip": 0.01103752, + "auxiliary_loss_mlp": 0.0105019, + "balance_loss_clip": 1.0347681, + "balance_loss_mlp": 1.02878022, + "epoch": 0.1328419708664616, + "flos": 16976795892480.0, + "grad_norm": 2.137994538646954, + "language_loss": 0.81596661, + "learning_rate": 3.890087315400072e-06, + "loss": 0.837506, + "num_input_tokens_seen": 130546925, + "router_z_loss_clip": 0.68945312, + "router_z_loss_mlp": 0.21411133, + "step": 4578, + "time_per_iteration": 2.518782138824463 + }, + { + "auxiliary_loss_clip": 0.01099845, + "auxiliary_loss_mlp": 0.01051215, + "balance_loss_clip": 1.03669071, + "balance_loss_mlp": 1.03103328, + "epoch": 0.13287098833497765, + "flos": 28801319604480.0, + "grad_norm": 2.1606087683266066, + "language_loss": 0.8881321, + "learning_rate": 3.890025854057022e-06, + "loss": 0.90964264, + "num_input_tokens_seen": 130562385, + "router_z_loss_clip": 0.63134766, + "router_z_loss_mlp": 0.2019043, + "step": 4579, + "time_per_iteration": 2.4922306537628174 + }, + { + "auxiliary_loss_clip": 0.01016638, + "auxiliary_loss_mlp": 0.0100477, + "balance_loss_clip": 1.00683212, + "balance_loss_mlp": 1.00364983, + "epoch": 0.1329000058034937, + "flos": 58173537761280.0, + "grad_norm": 0.6679271896594007, + "language_loss": 0.48113298, + "learning_rate": 3.8899643760204994e-06, + "loss": 0.50134706, + "num_input_tokens_seen": 130624070, + "router_z_loss_clip": 0.09814453, + "router_z_loss_mlp": 0.01123047, + "step": 4580, + "time_per_iteration": 3.1425235271453857 + }, + { + "auxiliary_loss_clip": 0.01094289, + "auxiliary_loss_mlp": 0.01038239, + "balance_loss_clip": 1.03308451, + "balance_loss_mlp": 1.01835477, + "epoch": 0.13292902327200976, + "flos": 17121650590080.0, + "grad_norm": 2.884613706261707, + "language_loss": 0.69609076, + "learning_rate": 3.889902881291046e-06, + "loss": 0.71741599, + "num_input_tokens_seen": 130636005, + "router_z_loss_clip": 0.61181641, + "router_z_loss_mlp": 0.19873047, + "step": 4581, + "time_per_iteration": 2.348398447036743 + }, + { + "auxiliary_loss_clip": 0.01102496, + "auxiliary_loss_mlp": 0.01041274, + "balance_loss_clip": 1.03457427, + "balance_loss_mlp": 1.02148557, + "epoch": 0.1329580407405258, + "flos": 32376844696320.0, + "grad_norm": 2.2418793721002577, + "language_loss": 0.78777587, + "learning_rate": 3.889841369869207e-06, + "loss": 0.80921352, + "num_input_tokens_seen": 130653015, + "router_z_loss_clip": 0.67919922, + "router_z_loss_mlp": 0.19799805, + "step": 4582, + "time_per_iteration": 2.4452857971191406 + }, + { + "auxiliary_loss_clip": 0.01097612, + "auxiliary_loss_mlp": 0.01046367, + "balance_loss_clip": 1.03123379, + "balance_loss_mlp": 1.026793, + "epoch": 0.13298705820904183, + "flos": 30583566155520.0, + "grad_norm": 1.8314703048912584, + "language_loss": 0.88289154, + "learning_rate": 3.8897798417555225e-06, + "loss": 0.90433139, + "num_input_tokens_seen": 130676655, + "router_z_loss_clip": 0.66357422, + "router_z_loss_mlp": 0.19580078, + "step": 4583, + "time_per_iteration": 2.693908929824829 + }, + { + "auxiliary_loss_clip": 0.01101307, + "auxiliary_loss_mlp": 0.0103962, + "balance_loss_clip": 1.03289711, + "balance_loss_mlp": 1.01873493, + "epoch": 0.13301607567755788, + "flos": 16317181975680.0, + "grad_norm": 3.014362296296968, + "language_loss": 1.04441559, + "learning_rate": 3.889718296950539e-06, + "loss": 1.06582487, + "num_input_tokens_seen": 130690240, + "router_z_loss_clip": 0.68408203, + "router_z_loss_mlp": 0.2088623, + "step": 4584, + "time_per_iteration": 2.42276930809021 + }, + { + "auxiliary_loss_clip": 0.01088711, + "auxiliary_loss_mlp": 0.01032581, + "balance_loss_clip": 1.03216219, + "balance_loss_mlp": 1.01670861, + "epoch": 0.13304509314607393, + "flos": 15662141447040.0, + "grad_norm": 2.0980448015455813, + "language_loss": 0.6404919, + "learning_rate": 3.889656735454798e-06, + "loss": 0.66170484, + "num_input_tokens_seen": 130702495, + "router_z_loss_clip": 0.56591797, + "router_z_loss_mlp": 0.15875244, + "step": 4585, + "time_per_iteration": 2.3548951148986816 + }, + { + "auxiliary_loss_clip": 0.01105185, + "auxiliary_loss_mlp": 0.01045131, + "balance_loss_clip": 1.03463697, + "balance_loss_mlp": 1.02252936, + "epoch": 0.13307411061458999, + "flos": 32809139550720.0, + "grad_norm": 2.1756823112693318, + "language_loss": 0.96244901, + "learning_rate": 3.889595157268844e-06, + "loss": 0.98395216, + "num_input_tokens_seen": 130719900, + "router_z_loss_clip": 0.70556641, + "router_z_loss_mlp": 0.22595215, + "step": 4586, + "time_per_iteration": 2.5118534564971924 + }, + { + "auxiliary_loss_clip": 0.01090425, + "auxiliary_loss_mlp": 0.01035292, + "balance_loss_clip": 1.03177047, + "balance_loss_mlp": 1.01693976, + "epoch": 0.13310312808310604, + "flos": 35107297395840.0, + "grad_norm": 2.0616089958679202, + "language_loss": 0.73352242, + "learning_rate": 3.889533562393222e-06, + "loss": 0.75477964, + "num_input_tokens_seen": 130735815, + "router_z_loss_clip": 0.58666992, + "router_z_loss_mlp": 0.18365479, + "step": 4587, + "time_per_iteration": 2.522709608078003 + }, + { + "auxiliary_loss_clip": 0.01097219, + "auxiliary_loss_mlp": 0.01042772, + "balance_loss_clip": 1.03307211, + "balance_loss_mlp": 1.02414584, + "epoch": 0.1331321455516221, + "flos": 28830926304000.0, + "grad_norm": 2.6129328394095372, + "language_loss": 0.96746784, + "learning_rate": 3.8894719508284735e-06, + "loss": 0.98886776, + "num_input_tokens_seen": 130751590, + "router_z_loss_clip": 0.64160156, + "router_z_loss_mlp": 0.18609619, + "step": 4588, + "time_per_iteration": 2.4810986518859863 + }, + { + "auxiliary_loss_clip": 0.01016212, + "auxiliary_loss_mlp": 0.0100045, + "balance_loss_clip": 1.00707507, + "balance_loss_mlp": 0.9994669, + "epoch": 0.1331611630201381, + "flos": 60467226952320.0, + "grad_norm": 0.6422531052891308, + "language_loss": 0.46408814, + "learning_rate": 3.889410322575145e-06, + "loss": 0.48425478, + "num_input_tokens_seen": 130817530, + "router_z_loss_clip": 0.09130859, + "router_z_loss_mlp": 0.00982666, + "step": 4589, + "time_per_iteration": 3.058823823928833 + }, + { + "auxiliary_loss_clip": 0.01016876, + "auxiliary_loss_mlp": 0.00999724, + "balance_loss_clip": 1.00802374, + "balance_loss_mlp": 0.99861503, + "epoch": 0.13319018048865416, + "flos": 74766429676800.0, + "grad_norm": 0.7299142743676454, + "language_loss": 0.45268697, + "learning_rate": 3.88934867763378e-06, + "loss": 0.47285298, + "num_input_tokens_seen": 130875410, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.0111084, + "step": 4590, + "time_per_iteration": 3.0463898181915283 + }, + { + "auxiliary_loss_clip": 0.01090874, + "auxiliary_loss_mlp": 0.01034846, + "balance_loss_clip": 1.0312624, + "balance_loss_mlp": 1.01649356, + "epoch": 0.13321919795717022, + "flos": 20879352489600.0, + "grad_norm": 2.620694456231806, + "language_loss": 0.77135837, + "learning_rate": 3.889287016004923e-06, + "loss": 0.79261559, + "num_input_tokens_seen": 130888580, + "router_z_loss_clip": 0.59570312, + "router_z_loss_mlp": 0.18353271, + "step": 4591, + "time_per_iteration": 2.3756494522094727 + }, + { + "auxiliary_loss_clip": 0.01092236, + "auxiliary_loss_mlp": 0.01037994, + "balance_loss_clip": 1.03190041, + "balance_loss_mlp": 1.02039266, + "epoch": 0.13324821542568627, + "flos": 30885180324480.0, + "grad_norm": 2.2138272252151214, + "language_loss": 0.73572111, + "learning_rate": 3.889225337689118e-06, + "loss": 0.75702339, + "num_input_tokens_seen": 130908065, + "router_z_loss_clip": 0.60302734, + "router_z_loss_mlp": 0.17596436, + "step": 4592, + "time_per_iteration": 2.5515987873077393 + }, + { + "auxiliary_loss_clip": 0.01087529, + "auxiliary_loss_mlp": 0.01044366, + "balance_loss_clip": 1.02886963, + "balance_loss_mlp": 1.02728343, + "epoch": 0.13327723289420232, + "flos": 16754748445440.0, + "grad_norm": 2.6843390472897033, + "language_loss": 0.85111427, + "learning_rate": 3.889163642686911e-06, + "loss": 0.87243325, + "num_input_tokens_seen": 130921805, + "router_z_loss_clip": 0.58691406, + "router_z_loss_mlp": 0.17089844, + "step": 4593, + "time_per_iteration": 2.5845251083374023 + }, + { + "auxiliary_loss_clip": 0.01091245, + "auxiliary_loss_mlp": 0.01039097, + "balance_loss_clip": 1.03228939, + "balance_loss_mlp": 1.02223492, + "epoch": 0.13330625036271834, + "flos": 40143902641920.0, + "grad_norm": 2.3230353523008054, + "language_loss": 0.91150469, + "learning_rate": 3.8891019309988456e-06, + "loss": 0.9328081, + "num_input_tokens_seen": 130938955, + "router_z_loss_clip": 0.59008789, + "router_z_loss_mlp": 0.16864014, + "step": 4594, + "time_per_iteration": 2.561457395553589 + }, + { + "auxiliary_loss_clip": 0.0101408, + "auxiliary_loss_mlp": 0.01004787, + "balance_loss_clip": 1.00528848, + "balance_loss_mlp": 1.00373173, + "epoch": 0.1333352678312344, + "flos": 65616426933120.0, + "grad_norm": 0.644581884342694, + "language_loss": 0.5213964, + "learning_rate": 3.889040202625468e-06, + "loss": 0.54158503, + "num_input_tokens_seen": 130999005, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01055908, + "step": 4595, + "time_per_iteration": 2.9943182468414307 + }, + { + "auxiliary_loss_clip": 0.01013882, + "auxiliary_loss_mlp": 0.01002465, + "balance_loss_clip": 1.00488997, + "balance_loss_mlp": 1.00123763, + "epoch": 0.13336428529975045, + "flos": 61314394026240.0, + "grad_norm": 0.6587733043436198, + "language_loss": 0.467621, + "learning_rate": 3.888978457567322e-06, + "loss": 0.48778448, + "num_input_tokens_seen": 131058630, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01226807, + "step": 4596, + "time_per_iteration": 2.9802165031433105 + }, + { + "auxiliary_loss_clip": 0.01092352, + "auxiliary_loss_mlp": 0.01042697, + "balance_loss_clip": 1.03083718, + "balance_loss_mlp": 1.02109623, + "epoch": 0.1333933027682665, + "flos": 33794213961600.0, + "grad_norm": 2.8762781340172303, + "language_loss": 0.92614651, + "learning_rate": 3.8889166958249544e-06, + "loss": 0.94749701, + "num_input_tokens_seen": 131076075, + "router_z_loss_clip": 0.61499023, + "router_z_loss_mlp": 0.21600342, + "step": 4597, + "time_per_iteration": 2.5256693363189697 + }, + { + "auxiliary_loss_clip": 0.01085498, + "auxiliary_loss_mlp": 0.0103303, + "balance_loss_clip": 1.02808022, + "balance_loss_mlp": 1.01693666, + "epoch": 0.13342232023678255, + "flos": 25478251620480.0, + "grad_norm": 2.5364480360797974, + "language_loss": 0.75393045, + "learning_rate": 3.888854917398911e-06, + "loss": 0.77511573, + "num_input_tokens_seen": 131095435, + "router_z_loss_clip": 0.57421875, + "router_z_loss_mlp": 0.16101074, + "step": 4598, + "time_per_iteration": 2.4287800788879395 + }, + { + "auxiliary_loss_clip": 0.01091133, + "auxiliary_loss_mlp": 0.01039206, + "balance_loss_clip": 1.0292263, + "balance_loss_mlp": 1.02074075, + "epoch": 0.1334513377052986, + "flos": 16065283449600.0, + "grad_norm": 3.370767017180422, + "language_loss": 0.930457, + "learning_rate": 3.888793122289736e-06, + "loss": 0.95176035, + "num_input_tokens_seen": 131107250, + "router_z_loss_clip": 0.61865234, + "router_z_loss_mlp": 0.18469238, + "step": 4599, + "time_per_iteration": 2.41852068901062 + }, + { + "auxiliary_loss_clip": 0.01096401, + "auxiliary_loss_mlp": 0.01043768, + "balance_loss_clip": 1.03049302, + "balance_loss_mlp": 1.02301383, + "epoch": 0.13348035517381462, + "flos": 58025228705280.0, + "grad_norm": 2.4790465364816785, + "language_loss": 0.82516456, + "learning_rate": 3.888731310497976e-06, + "loss": 0.8465662, + "num_input_tokens_seen": 131127980, + "router_z_loss_clip": 0.65820312, + "router_z_loss_mlp": 0.2076416, + "step": 4600, + "time_per_iteration": 2.662785768508911 + }, + { + "auxiliary_loss_clip": 0.01095135, + "auxiliary_loss_mlp": 0.01043035, + "balance_loss_clip": 1.03077471, + "balance_loss_mlp": 1.02312672, + "epoch": 0.13350937264233068, + "flos": 12341237967360.0, + "grad_norm": 2.9127260974939584, + "language_loss": 1.00481713, + "learning_rate": 3.888669482024176e-06, + "loss": 1.02619886, + "num_input_tokens_seen": 131139465, + "router_z_loss_clip": 0.64404297, + "router_z_loss_mlp": 0.19909668, + "step": 4601, + "time_per_iteration": 2.3753764629364014 + }, + { + "auxiliary_loss_clip": 0.01095413, + "auxiliary_loss_mlp": 0.010463, + "balance_loss_clip": 1.03047407, + "balance_loss_mlp": 1.02778649, + "epoch": 0.13353839011084673, + "flos": 14057533226880.0, + "grad_norm": 2.566456941269389, + "language_loss": 0.83787352, + "learning_rate": 3.888607636868884e-06, + "loss": 0.85929066, + "num_input_tokens_seen": 131153165, + "router_z_loss_clip": 0.64990234, + "router_z_loss_mlp": 0.18518066, + "step": 4602, + "time_per_iteration": 2.3350260257720947 + }, + { + "auxiliary_loss_clip": 0.01095667, + "auxiliary_loss_mlp": 0.01040227, + "balance_loss_clip": 1.03243995, + "balance_loss_mlp": 1.02046204, + "epoch": 0.13356740757936278, + "flos": 26900403742080.0, + "grad_norm": 1.9184100753421003, + "language_loss": 0.77135873, + "learning_rate": 3.8885457750326445e-06, + "loss": 0.7927177, + "num_input_tokens_seen": 131170420, + "router_z_loss_clip": 0.6328125, + "router_z_loss_mlp": 0.19763184, + "step": 4603, + "time_per_iteration": 2.4690656661987305 + }, + { + "auxiliary_loss_clip": 0.01093556, + "auxiliary_loss_mlp": 0.01037891, + "balance_loss_clip": 1.03228235, + "balance_loss_mlp": 1.01934218, + "epoch": 0.13359642504787883, + "flos": 16649450184960.0, + "grad_norm": 2.2677619892762837, + "language_loss": 0.62009567, + "learning_rate": 3.888483896516004e-06, + "loss": 0.64141011, + "num_input_tokens_seen": 131185085, + "router_z_loss_clip": 0.61352539, + "router_z_loss_mlp": 0.1854248, + "step": 4604, + "time_per_iteration": 2.522289991378784 + }, + { + "auxiliary_loss_clip": 0.01095614, + "auxiliary_loss_mlp": 0.01037282, + "balance_loss_clip": 1.02978253, + "balance_loss_mlp": 1.01738572, + "epoch": 0.13362544251639488, + "flos": 26502952291200.0, + "grad_norm": 2.4523158393169684, + "language_loss": 0.69216442, + "learning_rate": 3.8884220013195106e-06, + "loss": 0.71349347, + "num_input_tokens_seen": 131206015, + "router_z_loss_clip": 0.65869141, + "router_z_loss_mlp": 0.19909668, + "step": 4605, + "time_per_iteration": 2.464299201965332 + }, + { + "auxiliary_loss_clip": 0.01016549, + "auxiliary_loss_mlp": 0.01000408, + "balance_loss_clip": 1.00771654, + "balance_loss_mlp": 0.99920446, + "epoch": 0.1336544599849109, + "flos": 74772888278400.0, + "grad_norm": 0.6914120883558855, + "language_loss": 0.48344111, + "learning_rate": 3.88836008944371e-06, + "loss": 0.50361073, + "num_input_tokens_seen": 131269770, + "router_z_loss_clip": 0.08837891, + "router_z_loss_mlp": 0.01202393, + "step": 4606, + "time_per_iteration": 3.0853800773620605 + }, + { + "auxiliary_loss_clip": 0.01093718, + "auxiliary_loss_mlp": 0.01041709, + "balance_loss_clip": 1.02839649, + "balance_loss_mlp": 1.0227195, + "epoch": 0.13368347745342696, + "flos": 23360490103680.0, + "grad_norm": 2.4191660035186042, + "language_loss": 0.96428275, + "learning_rate": 3.888298160889148e-06, + "loss": 0.98563707, + "num_input_tokens_seen": 131286120, + "router_z_loss_clip": 0.65307617, + "router_z_loss_mlp": 0.18994141, + "step": 4607, + "time_per_iteration": 2.6091701984405518 + }, + { + "auxiliary_loss_clip": 0.01100277, + "auxiliary_loss_mlp": 0.01038258, + "balance_loss_clip": 1.03194106, + "balance_loss_mlp": 1.01691961, + "epoch": 0.133712494921943, + "flos": 29562426443520.0, + "grad_norm": 1.8804105878944135, + "language_loss": 0.87384081, + "learning_rate": 3.888236215656373e-06, + "loss": 0.89522612, + "num_input_tokens_seen": 131305715, + "router_z_loss_clip": 0.68359375, + "router_z_loss_mlp": 0.21337891, + "step": 4608, + "time_per_iteration": 2.4571869373321533 + }, + { + "auxiliary_loss_clip": 0.01097544, + "auxiliary_loss_mlp": 0.01041689, + "balance_loss_clip": 1.03076351, + "balance_loss_mlp": 1.02166176, + "epoch": 0.13374151239045906, + "flos": 14676229163520.0, + "grad_norm": 3.2583582840468837, + "language_loss": 0.97063291, + "learning_rate": 3.888174253745931e-06, + "loss": 0.99202526, + "num_input_tokens_seen": 131318220, + "router_z_loss_clip": 0.66748047, + "router_z_loss_mlp": 0.20019531, + "step": 4609, + "time_per_iteration": 2.414766311645508 + }, + { + "auxiliary_loss_clip": 0.01091258, + "auxiliary_loss_mlp": 0.01041205, + "balance_loss_clip": 1.02896762, + "balance_loss_mlp": 1.02316856, + "epoch": 0.1337705298589751, + "flos": 37888408344960.0, + "grad_norm": 2.1769988333264605, + "language_loss": 0.99756068, + "learning_rate": 3.888112275158371e-06, + "loss": 1.01888537, + "num_input_tokens_seen": 131342095, + "router_z_loss_clip": 0.62353516, + "router_z_loss_mlp": 0.18041992, + "step": 4610, + "time_per_iteration": 2.5676136016845703 + }, + { + "auxiliary_loss_clip": 0.01085856, + "auxiliary_loss_mlp": 0.01033566, + "balance_loss_clip": 1.02882993, + "balance_loss_mlp": 1.01797926, + "epoch": 0.13379954732749114, + "flos": 31496195761920.0, + "grad_norm": 2.343200574977531, + "language_loss": 0.9117071, + "learning_rate": 3.888050279894239e-06, + "loss": 0.93290126, + "num_input_tokens_seen": 131356475, + "router_z_loss_clip": 0.57006836, + "router_z_loss_mlp": 0.15594482, + "step": 4611, + "time_per_iteration": 2.4961814880371094 + }, + { + "auxiliary_loss_clip": 0.01089124, + "auxiliary_loss_mlp": 0.01039456, + "balance_loss_clip": 1.03106117, + "balance_loss_mlp": 1.02210486, + "epoch": 0.1338285647960072, + "flos": 27997758685440.0, + "grad_norm": 2.1592494311757338, + "language_loss": 0.85503566, + "learning_rate": 3.8879882679540824e-06, + "loss": 0.87632143, + "num_input_tokens_seen": 131379440, + "router_z_loss_clip": 0.58105469, + "router_z_loss_mlp": 0.17352295, + "step": 4612, + "time_per_iteration": 2.5576581954956055 + }, + { + "auxiliary_loss_clip": 0.01095237, + "auxiliary_loss_mlp": 0.01034456, + "balance_loss_clip": 1.03023624, + "balance_loss_mlp": 1.01427341, + "epoch": 0.13385758226452324, + "flos": 29744603251200.0, + "grad_norm": 3.0458018883442937, + "language_loss": 0.69575661, + "learning_rate": 3.88792623933845e-06, + "loss": 0.71705353, + "num_input_tokens_seen": 131395510, + "router_z_loss_clip": 0.6496582, + "router_z_loss_mlp": 0.20178223, + "step": 4613, + "time_per_iteration": 2.5709922313690186 + }, + { + "auxiliary_loss_clip": 0.01014408, + "auxiliary_loss_mlp": 0.010036, + "balance_loss_clip": 1.005198, + "balance_loss_mlp": 1.00223458, + "epoch": 0.1338865997330393, + "flos": 69510081133440.0, + "grad_norm": 0.7093846333902981, + "language_loss": 0.49677083, + "learning_rate": 3.887864194047889e-06, + "loss": 0.51695085, + "num_input_tokens_seen": 131451495, + "router_z_loss_clip": 0.09228516, + "router_z_loss_mlp": 0.01367188, + "step": 4614, + "time_per_iteration": 2.9099972248077393 + }, + { + "auxiliary_loss_clip": 0.01013259, + "auxiliary_loss_mlp": 0.01001948, + "balance_loss_clip": 1.00404811, + "balance_loss_mlp": 1.00055969, + "epoch": 0.13391561720155534, + "flos": 70248354076800.0, + "grad_norm": 0.7722968071245678, + "language_loss": 0.52488101, + "learning_rate": 3.887802132082947e-06, + "loss": 0.5450331, + "num_input_tokens_seen": 131516760, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01391602, + "step": 4615, + "time_per_iteration": 3.094606399536133 + }, + { + "auxiliary_loss_clip": 0.0109505, + "auxiliary_loss_mlp": 0.0104456, + "balance_loss_clip": 1.02991533, + "balance_loss_mlp": 1.02361488, + "epoch": 0.1339446346700714, + "flos": 21500003462400.0, + "grad_norm": 3.8179955114154196, + "language_loss": 1.0553112, + "learning_rate": 3.8877400534441735e-06, + "loss": 1.07670736, + "num_input_tokens_seen": 131537805, + "router_z_loss_clip": 0.65087891, + "router_z_loss_mlp": 0.20959473, + "step": 4616, + "time_per_iteration": 2.545448064804077 + }, + { + "auxiliary_loss_clip": 0.01013471, + "auxiliary_loss_mlp": 0.01005403, + "balance_loss_clip": 1.00393724, + "balance_loss_mlp": 1.00388336, + "epoch": 0.13397365213858742, + "flos": 62330576325120.0, + "grad_norm": 0.7711402470191752, + "language_loss": 0.51613712, + "learning_rate": 3.887677958132115e-06, + "loss": 0.53632581, + "num_input_tokens_seen": 131598015, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01519775, + "step": 4617, + "time_per_iteration": 2.927299737930298 + }, + { + "auxiliary_loss_clip": 0.01013952, + "auxiliary_loss_mlp": 0.01001334, + "balance_loss_clip": 1.00470018, + "balance_loss_mlp": 1.00015354, + "epoch": 0.13400266960710347, + "flos": 67328637563520.0, + "grad_norm": 0.6255599678263841, + "language_loss": 0.46164113, + "learning_rate": 3.887615846147322e-06, + "loss": 0.48179403, + "num_input_tokens_seen": 131662050, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01177979, + "step": 4618, + "time_per_iteration": 3.0448591709136963 + }, + { + "auxiliary_loss_clip": 0.01093215, + "auxiliary_loss_mlp": 0.01041165, + "balance_loss_clip": 1.03176963, + "balance_loss_mlp": 1.02253282, + "epoch": 0.13403168707561952, + "flos": 26607796704000.0, + "grad_norm": 2.108099658882139, + "language_loss": 0.89236426, + "learning_rate": 3.887553717490341e-06, + "loss": 0.91370809, + "num_input_tokens_seen": 131678830, + "router_z_loss_clip": 0.61474609, + "router_z_loss_mlp": 0.1862793, + "step": 4619, + "time_per_iteration": 2.466749668121338 + }, + { + "auxiliary_loss_clip": 0.01087373, + "auxiliary_loss_mlp": 0.01034325, + "balance_loss_clip": 1.02949476, + "balance_loss_mlp": 1.01762366, + "epoch": 0.13406070454413557, + "flos": 31424763386880.0, + "grad_norm": 1.773109678678373, + "language_loss": 0.8265245, + "learning_rate": 3.887491572161722e-06, + "loss": 0.84774154, + "num_input_tokens_seen": 131699490, + "router_z_loss_clip": 0.57910156, + "router_z_loss_mlp": 0.16711426, + "step": 4620, + "time_per_iteration": 2.6448490619659424 + }, + { + "auxiliary_loss_clip": 0.01088315, + "auxiliary_loss_mlp": 0.01043577, + "balance_loss_clip": 1.02793694, + "balance_loss_mlp": 1.02595186, + "epoch": 0.13408972201265162, + "flos": 17230265429760.0, + "grad_norm": 2.5350685744772607, + "language_loss": 0.77524072, + "learning_rate": 3.8874294101620145e-06, + "loss": 0.79655963, + "num_input_tokens_seen": 131711440, + "router_z_loss_clip": 0.60449219, + "router_z_loss_mlp": 0.17626953, + "step": 4621, + "time_per_iteration": 2.373257875442505 + }, + { + "auxiliary_loss_clip": 0.01084853, + "auxiliary_loss_mlp": 0.01027949, + "balance_loss_clip": 1.03087902, + "balance_loss_mlp": 1.01154017, + "epoch": 0.13411873948116768, + "flos": 26242500481920.0, + "grad_norm": 1.9660953666726568, + "language_loss": 0.77193987, + "learning_rate": 3.887367231491765e-06, + "loss": 0.79306793, + "num_input_tokens_seen": 131726205, + "router_z_loss_clip": 0.53979492, + "router_z_loss_mlp": 0.1640625, + "step": 4622, + "time_per_iteration": 2.45178484916687 + }, + { + "auxiliary_loss_clip": 0.01092657, + "auxiliary_loss_mlp": 0.0104304, + "balance_loss_clip": 1.02913392, + "balance_loss_mlp": 1.02292895, + "epoch": 0.1341477569496837, + "flos": 11648665860480.0, + "grad_norm": 2.7621513059063907, + "language_loss": 0.8226732, + "learning_rate": 3.887305036151526e-06, + "loss": 0.84403014, + "num_input_tokens_seen": 131737755, + "router_z_loss_clip": 0.63525391, + "router_z_loss_mlp": 0.20117188, + "step": 4623, + "time_per_iteration": 2.325585126876831 + }, + { + "auxiliary_loss_clip": 0.01089392, + "auxiliary_loss_mlp": 0.0103771, + "balance_loss_clip": 1.02811062, + "balance_loss_mlp": 1.01924467, + "epoch": 0.13417677441819975, + "flos": 44668367020800.0, + "grad_norm": 2.5389239536635118, + "language_loss": 0.91507602, + "learning_rate": 3.887242824141845e-06, + "loss": 0.93634707, + "num_input_tokens_seen": 131754615, + "router_z_loss_clip": 0.61303711, + "router_z_loss_mlp": 0.18457031, + "step": 4624, + "time_per_iteration": 2.616710901260376 + }, + { + "auxiliary_loss_clip": 0.01014965, + "auxiliary_loss_mlp": 0.01001654, + "balance_loss_clip": 1.00589144, + "balance_loss_mlp": 1.00065291, + "epoch": 0.1342057918867158, + "flos": 74779346880000.0, + "grad_norm": 0.6304570679315123, + "language_loss": 0.46786124, + "learning_rate": 3.887180595463271e-06, + "loss": 0.48802742, + "num_input_tokens_seen": 131819725, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01000977, + "step": 4625, + "time_per_iteration": 3.1282715797424316 + }, + { + "auxiliary_loss_clip": 0.01091606, + "auxiliary_loss_mlp": 0.01035118, + "balance_loss_clip": 1.03132343, + "balance_loss_mlp": 1.01581156, + "epoch": 0.13423480935523185, + "flos": 31827975212160.0, + "grad_norm": 1.9093426127626838, + "language_loss": 0.84122324, + "learning_rate": 3.887118350116355e-06, + "loss": 0.86249048, + "num_input_tokens_seen": 131839875, + "router_z_loss_clip": 0.60253906, + "router_z_loss_mlp": 0.19281006, + "step": 4626, + "time_per_iteration": 2.5177879333496094 + }, + { + "auxiliary_loss_clip": 0.01013508, + "auxiliary_loss_mlp": 0.01003972, + "balance_loss_clip": 1.00426245, + "balance_loss_mlp": 1.00282121, + "epoch": 0.1342638268237479, + "flos": 72113658485760.0, + "grad_norm": 0.6128899504890299, + "language_loss": 0.44170779, + "learning_rate": 3.887056088101645e-06, + "loss": 0.46188253, + "num_input_tokens_seen": 131905310, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01147461, + "step": 4627, + "time_per_iteration": 3.0989110469818115 + }, + { + "auxiliary_loss_clip": 0.01013761, + "auxiliary_loss_mlp": 0.01000265, + "balance_loss_clip": 1.00449443, + "balance_loss_mlp": 0.99916786, + "epoch": 0.13429284429226393, + "flos": 69301090535040.0, + "grad_norm": 0.6805949546229013, + "language_loss": 0.46903974, + "learning_rate": 3.886993809419693e-06, + "loss": 0.48918, + "num_input_tokens_seen": 131962840, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01098633, + "step": 4628, + "time_per_iteration": 2.999526023864746 + }, + { + "auxiliary_loss_clip": 0.01098892, + "auxiliary_loss_mlp": 0.01048979, + "balance_loss_clip": 1.03114021, + "balance_loss_mlp": 1.02779567, + "epoch": 0.13432186176077998, + "flos": 42414548469120.0, + "grad_norm": 7.177741383708294, + "language_loss": 0.83319962, + "learning_rate": 3.886931514071047e-06, + "loss": 0.85467827, + "num_input_tokens_seen": 131983575, + "router_z_loss_clip": 0.67675781, + "router_z_loss_mlp": 0.21191406, + "step": 4629, + "time_per_iteration": 2.5044403076171875 + }, + { + "auxiliary_loss_clip": 0.01093992, + "auxiliary_loss_mlp": 0.01045411, + "balance_loss_clip": 1.03193069, + "balance_loss_mlp": 1.02439404, + "epoch": 0.13435087922929603, + "flos": 29853811584000.0, + "grad_norm": 2.0286759248965294, + "language_loss": 0.60354215, + "learning_rate": 3.88686920205626e-06, + "loss": 0.62493622, + "num_input_tokens_seen": 131999200, + "router_z_loss_clip": 0.62011719, + "router_z_loss_mlp": 0.21020508, + "step": 4630, + "time_per_iteration": 2.4463727474212646 + }, + { + "auxiliary_loss_clip": 0.01095044, + "auxiliary_loss_mlp": 0.01042931, + "balance_loss_clip": 1.02918923, + "balance_loss_mlp": 1.02078235, + "epoch": 0.13437989669781208, + "flos": 24600779619840.0, + "grad_norm": 2.3491495445467474, + "language_loss": 0.7520175, + "learning_rate": 3.88680687337588e-06, + "loss": 0.77339721, + "num_input_tokens_seen": 132012310, + "router_z_loss_clip": 0.65869141, + "router_z_loss_mlp": 0.22167969, + "step": 4631, + "time_per_iteration": 2.3429882526397705 + }, + { + "auxiliary_loss_clip": 0.01102263, + "auxiliary_loss_mlp": 0.0104948, + "balance_loss_clip": 1.03449941, + "balance_loss_mlp": 1.02867818, + "epoch": 0.13440891416632814, + "flos": 28979481605760.0, + "grad_norm": 3.4223275636540054, + "language_loss": 0.92589968, + "learning_rate": 3.886744528030458e-06, + "loss": 0.94741708, + "num_input_tokens_seen": 132027875, + "router_z_loss_clip": 0.67724609, + "router_z_loss_mlp": 0.20800781, + "step": 4632, + "time_per_iteration": 2.464627265930176 + }, + { + "auxiliary_loss_clip": 0.01092454, + "auxiliary_loss_mlp": 0.01038485, + "balance_loss_clip": 1.030604, + "balance_loss_mlp": 1.01919734, + "epoch": 0.1344379316348442, + "flos": 27517703224320.0, + "grad_norm": 3.1656800681740327, + "language_loss": 1.07762778, + "learning_rate": 3.886682166020544e-06, + "loss": 1.09893715, + "num_input_tokens_seen": 132043125, + "router_z_loss_clip": 0.61914062, + "router_z_loss_mlp": 0.1930542, + "step": 4633, + "time_per_iteration": 2.6596577167510986 + }, + { + "auxiliary_loss_clip": 0.01100938, + "auxiliary_loss_mlp": 0.01050405, + "balance_loss_clip": 1.03295994, + "balance_loss_mlp": 1.02858973, + "epoch": 0.1344669491033602, + "flos": 12084661319040.0, + "grad_norm": 2.940629870115769, + "language_loss": 0.71605074, + "learning_rate": 3.8866197873466915e-06, + "loss": 0.73756421, + "num_input_tokens_seen": 132053885, + "router_z_loss_clip": 0.68017578, + "router_z_loss_mlp": 0.21838379, + "step": 4634, + "time_per_iteration": 2.3648366928100586 + }, + { + "auxiliary_loss_clip": 0.0109204, + "auxiliary_loss_mlp": 0.01046411, + "balance_loss_clip": 1.03233159, + "balance_loss_mlp": 1.02891088, + "epoch": 0.13449596657187626, + "flos": 28503790064640.0, + "grad_norm": 2.602309292679582, + "language_loss": 0.85611665, + "learning_rate": 3.8865573920094484e-06, + "loss": 0.87750119, + "num_input_tokens_seen": 132070890, + "router_z_loss_clip": 0.59716797, + "router_z_loss_mlp": 0.17504883, + "step": 4635, + "time_per_iteration": 2.490874767303467 + }, + { + "auxiliary_loss_clip": 0.01092326, + "auxiliary_loss_mlp": 0.01056814, + "balance_loss_clip": 1.03000665, + "balance_loss_mlp": 1.03629839, + "epoch": 0.13452498404039231, + "flos": 15443480401920.0, + "grad_norm": 2.903933905287014, + "language_loss": 0.83324784, + "learning_rate": 3.8864949800093665e-06, + "loss": 0.85473919, + "num_input_tokens_seen": 132084110, + "router_z_loss_clip": 0.62353516, + "router_z_loss_mlp": 0.2052002, + "step": 4636, + "time_per_iteration": 2.37208890914917 + }, + { + "auxiliary_loss_clip": 0.01014526, + "auxiliary_loss_mlp": 0.01018064, + "balance_loss_clip": 1.00528395, + "balance_loss_mlp": 1.01686049, + "epoch": 0.13455400150890837, + "flos": 65170726116480.0, + "grad_norm": 0.7716891787187579, + "language_loss": 0.41373444, + "learning_rate": 3.886432551346998e-06, + "loss": 0.43406034, + "num_input_tokens_seen": 132139935, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01202393, + "step": 4637, + "time_per_iteration": 2.8829991817474365 + }, + { + "auxiliary_loss_clip": 0.01097393, + "auxiliary_loss_mlp": 0.01052321, + "balance_loss_clip": 1.03236127, + "balance_loss_mlp": 1.03166211, + "epoch": 0.13458301897742442, + "flos": 17198145112320.0, + "grad_norm": 5.882964886041869, + "language_loss": 0.79746234, + "learning_rate": 3.886370106022895e-06, + "loss": 0.81895947, + "num_input_tokens_seen": 132158385, + "router_z_loss_clip": 0.65039062, + "router_z_loss_mlp": 0.20666504, + "step": 4638, + "time_per_iteration": 2.418360948562622 + }, + { + "auxiliary_loss_clip": 0.01091769, + "auxiliary_loss_mlp": 0.01036396, + "balance_loss_clip": 1.0302099, + "balance_loss_mlp": 1.01842546, + "epoch": 0.13461203644594047, + "flos": 14423143651200.0, + "grad_norm": 2.758843698509738, + "language_loss": 0.86203951, + "learning_rate": 3.886307644037606e-06, + "loss": 0.88332117, + "num_input_tokens_seen": 132171260, + "router_z_loss_clip": 0.61572266, + "router_z_loss_mlp": 0.17987061, + "step": 4639, + "time_per_iteration": 2.4215087890625 + }, + { + "auxiliary_loss_clip": 0.01095792, + "auxiliary_loss_mlp": 0.01039818, + "balance_loss_clip": 1.03540897, + "balance_loss_mlp": 1.02075624, + "epoch": 0.1346410539144565, + "flos": 25259381107200.0, + "grad_norm": 3.746459162562876, + "language_loss": 0.97727799, + "learning_rate": 3.886245165391686e-06, + "loss": 0.99863404, + "num_input_tokens_seen": 132185145, + "router_z_loss_clip": 0.60473633, + "router_z_loss_mlp": 0.19055176, + "step": 4640, + "time_per_iteration": 2.463397979736328 + }, + { + "auxiliary_loss_clip": 0.01091994, + "auxiliary_loss_mlp": 0.01034625, + "balance_loss_clip": 1.03237665, + "balance_loss_mlp": 1.01619506, + "epoch": 0.13467007138297254, + "flos": 31095183352320.0, + "grad_norm": 2.2102522151806316, + "language_loss": 0.81183583, + "learning_rate": 3.886182670085685e-06, + "loss": 0.83310199, + "num_input_tokens_seen": 132203275, + "router_z_loss_clip": 0.59619141, + "router_z_loss_mlp": 0.18438721, + "step": 4641, + "time_per_iteration": 2.490792751312256 + }, + { + "auxiliary_loss_clip": 0.01094531, + "auxiliary_loss_mlp": 0.0103849, + "balance_loss_clip": 1.0358609, + "balance_loss_mlp": 1.01938057, + "epoch": 0.1346990888514886, + "flos": 44085526917120.0, + "grad_norm": 2.0461972584080006, + "language_loss": 0.70985103, + "learning_rate": 3.8861201581201554e-06, + "loss": 0.73118126, + "num_input_tokens_seen": 132222920, + "router_z_loss_clip": 0.5871582, + "router_z_loss_mlp": 0.19104004, + "step": 4642, + "time_per_iteration": 4.940409898757935 + }, + { + "auxiliary_loss_clip": 0.01094159, + "auxiliary_loss_mlp": 0.01034911, + "balance_loss_clip": 1.03471422, + "balance_loss_mlp": 1.01737499, + "epoch": 0.13472810632000465, + "flos": 11104020650880.0, + "grad_norm": 2.2637062304974873, + "language_loss": 0.73317015, + "learning_rate": 3.886057629495649e-06, + "loss": 0.75446081, + "num_input_tokens_seen": 132234410, + "router_z_loss_clip": 0.59423828, + "router_z_loss_mlp": 0.17541504, + "step": 4643, + "time_per_iteration": 2.4328835010528564 + }, + { + "auxiliary_loss_clip": 0.01103343, + "auxiliary_loss_mlp": 0.01043834, + "balance_loss_clip": 1.03703809, + "balance_loss_mlp": 1.02235258, + "epoch": 0.1347571237885207, + "flos": 27664792248960.0, + "grad_norm": 2.517360353739535, + "language_loss": 0.82272828, + "learning_rate": 3.885995084212719e-06, + "loss": 0.84420002, + "num_input_tokens_seen": 132249020, + "router_z_loss_clip": 0.66259766, + "router_z_loss_mlp": 0.21484375, + "step": 4644, + "time_per_iteration": 4.652031183242798 + }, + { + "auxiliary_loss_clip": 0.01018681, + "auxiliary_loss_mlp": 0.01000879, + "balance_loss_clip": 1.00925887, + "balance_loss_mlp": 0.99988937, + "epoch": 0.13478614125703672, + "flos": 74766080563200.0, + "grad_norm": 0.7052523044631062, + "language_loss": 0.53437698, + "learning_rate": 3.8859325222719174e-06, + "loss": 0.55457258, + "num_input_tokens_seen": 132308525, + "router_z_loss_clip": 0.09423828, + "router_z_loss_mlp": 0.0098877, + "step": 4645, + "time_per_iteration": 3.1716818809509277 + }, + { + "auxiliary_loss_clip": 0.01018465, + "auxiliary_loss_mlp": 0.01000277, + "balance_loss_clip": 1.00878942, + "balance_loss_mlp": 0.99931771, + "epoch": 0.13481515872555278, + "flos": 60219971637120.0, + "grad_norm": 0.6841846909517061, + "language_loss": 0.49970618, + "learning_rate": 3.8858699436737955e-06, + "loss": 0.51989359, + "num_input_tokens_seen": 132369715, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.00958252, + "step": 4646, + "time_per_iteration": 3.0319018363952637 + }, + { + "auxiliary_loss_clip": 0.01093896, + "auxiliary_loss_mlp": 0.01042632, + "balance_loss_clip": 1.03361058, + "balance_loss_mlp": 1.02552497, + "epoch": 0.13484417619406883, + "flos": 19054198010880.0, + "grad_norm": 2.058106318030898, + "language_loss": 0.66937447, + "learning_rate": 3.885807348418908e-06, + "loss": 0.69073975, + "num_input_tokens_seen": 132384790, + "router_z_loss_clip": 0.60351562, + "router_z_loss_mlp": 0.17114258, + "step": 4647, + "time_per_iteration": 2.382897138595581 + }, + { + "auxiliary_loss_clip": 0.0109835, + "auxiliary_loss_mlp": 0.0103729, + "balance_loss_clip": 1.03628469, + "balance_loss_mlp": 1.01847827, + "epoch": 0.13487319366258488, + "flos": 22120759169280.0, + "grad_norm": 2.30641068558417, + "language_loss": 0.72922015, + "learning_rate": 3.885744736507807e-06, + "loss": 0.75057656, + "num_input_tokens_seen": 132399530, + "router_z_loss_clip": 0.62109375, + "router_z_loss_mlp": 0.18811035, + "step": 4648, + "time_per_iteration": 2.4103035926818848 + }, + { + "auxiliary_loss_clip": 0.01095603, + "auxiliary_loss_mlp": 0.01042414, + "balance_loss_clip": 1.03284943, + "balance_loss_mlp": 1.02123034, + "epoch": 0.13490221113110093, + "flos": 72612430168320.0, + "grad_norm": 3.3726914336547944, + "language_loss": 0.93591595, + "learning_rate": 3.885682107941045e-06, + "loss": 0.95729613, + "num_input_tokens_seen": 132418300, + "router_z_loss_clip": 0.62744141, + "router_z_loss_mlp": 0.21191406, + "step": 4649, + "time_per_iteration": 2.830972194671631 + }, + { + "auxiliary_loss_clip": 0.01101841, + "auxiliary_loss_mlp": 0.01047581, + "balance_loss_clip": 1.03356612, + "balance_loss_mlp": 1.02642167, + "epoch": 0.13493122859961698, + "flos": 25110721071360.0, + "grad_norm": 1.9757065777438843, + "language_loss": 0.72338414, + "learning_rate": 3.885619462719175e-06, + "loss": 0.74487829, + "num_input_tokens_seen": 132433290, + "router_z_loss_clip": 0.68212891, + "router_z_loss_mlp": 0.21179199, + "step": 4650, + "time_per_iteration": 4.88140344619751 + }, + { + "auxiliary_loss_clip": 0.01096795, + "auxiliary_loss_mlp": 0.01039559, + "balance_loss_clip": 1.0342319, + "balance_loss_mlp": 1.01890051, + "epoch": 0.134960246068133, + "flos": 28614674142720.0, + "grad_norm": 2.9288813263138187, + "language_loss": 0.64966005, + "learning_rate": 3.885556800842753e-06, + "loss": 0.67102355, + "num_input_tokens_seen": 132448625, + "router_z_loss_clip": 0.62597656, + "router_z_loss_mlp": 0.20666504, + "step": 4651, + "time_per_iteration": 2.3722503185272217 + }, + { + "auxiliary_loss_clip": 0.01092391, + "auxiliary_loss_mlp": 0.01043554, + "balance_loss_clip": 1.03027248, + "balance_loss_mlp": 1.0239799, + "epoch": 0.13498926353664906, + "flos": 13144345038720.0, + "grad_norm": 2.4745226101504643, + "language_loss": 0.8441807, + "learning_rate": 3.885494122312327e-06, + "loss": 0.86554021, + "num_input_tokens_seen": 132460660, + "router_z_loss_clip": 0.62158203, + "router_z_loss_mlp": 0.19580078, + "step": 4652, + "time_per_iteration": 4.74937105178833 + }, + { + "auxiliary_loss_clip": 0.01088205, + "auxiliary_loss_mlp": 0.01039166, + "balance_loss_clip": 1.03010225, + "balance_loss_mlp": 1.02093935, + "epoch": 0.1350182810051651, + "flos": 13326556757760.0, + "grad_norm": 2.74012334275818, + "language_loss": 0.74575508, + "learning_rate": 3.885431427128457e-06, + "loss": 0.76702881, + "num_input_tokens_seen": 132472395, + "router_z_loss_clip": 0.58105469, + "router_z_loss_mlp": 0.18225098, + "step": 4653, + "time_per_iteration": 2.356858491897583 + }, + { + "auxiliary_loss_clip": 0.01088301, + "auxiliary_loss_mlp": 0.01037604, + "balance_loss_clip": 1.02799404, + "balance_loss_mlp": 1.018924, + "epoch": 0.13504729847368116, + "flos": 12707302239360.0, + "grad_norm": 2.1024537359067863, + "language_loss": 0.72546405, + "learning_rate": 3.885368715291692e-06, + "loss": 0.74672306, + "num_input_tokens_seen": 132487380, + "router_z_loss_clip": 0.60302734, + "router_z_loss_mlp": 0.18676758, + "step": 4654, + "time_per_iteration": 2.3959550857543945 + }, + { + "auxiliary_loss_clip": 0.01013771, + "auxiliary_loss_mlp": 0.01002467, + "balance_loss_clip": 1.00467622, + "balance_loss_mlp": 1.00142992, + "epoch": 0.1350763159421972, + "flos": 72472112081280.0, + "grad_norm": 0.6694785304567492, + "language_loss": 0.50440055, + "learning_rate": 3.8853059868025885e-06, + "loss": 0.52456295, + "num_input_tokens_seen": 132550370, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01037598, + "step": 4655, + "time_per_iteration": 3.066361427307129 + }, + { + "auxiliary_loss_clip": 0.01012926, + "auxiliary_loss_mlp": 0.01001719, + "balance_loss_clip": 1.00379789, + "balance_loss_mlp": 1.00054491, + "epoch": 0.13510533341071326, + "flos": 66450746626560.0, + "grad_norm": 0.7508879770587599, + "language_loss": 0.48714346, + "learning_rate": 3.885243241661699e-06, + "loss": 0.50728989, + "num_input_tokens_seen": 132611460, + "router_z_loss_clip": 0.09130859, + "router_z_loss_mlp": 0.01171875, + "step": 4656, + "time_per_iteration": 3.1312642097473145 + }, + { + "auxiliary_loss_clip": 0.01012506, + "auxiliary_loss_mlp": 0.0100239, + "balance_loss_clip": 1.0033679, + "balance_loss_mlp": 1.00119162, + "epoch": 0.1351343508792293, + "flos": 64736266757760.0, + "grad_norm": 0.7082348117979889, + "language_loss": 0.49889314, + "learning_rate": 3.885180479869578e-06, + "loss": 0.51904202, + "num_input_tokens_seen": 132671710, + "router_z_loss_clip": 0.09130859, + "router_z_loss_mlp": 0.01196289, + "step": 4657, + "time_per_iteration": 2.9956130981445312 + }, + { + "auxiliary_loss_clip": 0.01011419, + "auxiliary_loss_mlp": 0.00999959, + "balance_loss_clip": 1.00229526, + "balance_loss_mlp": 0.99893951, + "epoch": 0.13516336834774534, + "flos": 74291471274240.0, + "grad_norm": 0.6949513322355466, + "language_loss": 0.53912896, + "learning_rate": 3.885117701426781e-06, + "loss": 0.55924273, + "num_input_tokens_seen": 132734250, + "router_z_loss_clip": 0.09130859, + "router_z_loss_mlp": 0.01019287, + "step": 4658, + "time_per_iteration": 3.000319480895996 + }, + { + "auxiliary_loss_clip": 0.01011868, + "auxiliary_loss_mlp": 0.0100261, + "balance_loss_clip": 1.0027585, + "balance_loss_mlp": 1.00155509, + "epoch": 0.1351923858162614, + "flos": 57477614163840.0, + "grad_norm": 0.6229688602418854, + "language_loss": 0.46521556, + "learning_rate": 3.885054906333861e-06, + "loss": 0.48536032, + "num_input_tokens_seen": 132795370, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01055908, + "step": 4659, + "time_per_iteration": 3.036803722381592 + }, + { + "auxiliary_loss_clip": 0.01088738, + "auxiliary_loss_mlp": 0.01042353, + "balance_loss_clip": 1.02846622, + "balance_loss_mlp": 1.02391124, + "epoch": 0.13522140328477744, + "flos": 25298553519360.0, + "grad_norm": 1.9542628393111592, + "language_loss": 1.06263661, + "learning_rate": 3.884992094591373e-06, + "loss": 1.08394742, + "num_input_tokens_seen": 132815895, + "router_z_loss_clip": 0.6027832, + "router_z_loss_mlp": 0.18444824, + "step": 4660, + "time_per_iteration": 2.478153944015503 + }, + { + "auxiliary_loss_clip": 0.01011978, + "auxiliary_loss_mlp": 0.01002151, + "balance_loss_clip": 1.0029335, + "balance_loss_mlp": 1.00116122, + "epoch": 0.1352504207532935, + "flos": 62687284352640.0, + "grad_norm": 0.765293826140871, + "language_loss": 0.5396533, + "learning_rate": 3.8849292661998734e-06, + "loss": 0.55979455, + "num_input_tokens_seen": 132871560, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.0098877, + "step": 4661, + "time_per_iteration": 2.8726983070373535 + }, + { + "auxiliary_loss_clip": 0.01087943, + "auxiliary_loss_mlp": 0.01043239, + "balance_loss_clip": 1.02788162, + "balance_loss_mlp": 1.02449918, + "epoch": 0.13527943822180952, + "flos": 10953440490240.0, + "grad_norm": 5.350617487320023, + "language_loss": 1.02073765, + "learning_rate": 3.884866421159915e-06, + "loss": 1.04204941, + "num_input_tokens_seen": 132881595, + "router_z_loss_clip": 0.60009766, + "router_z_loss_mlp": 0.18743896, + "step": 4662, + "time_per_iteration": 2.342573404312134 + }, + { + "auxiliary_loss_clip": 0.01011568, + "auxiliary_loss_mlp": 0.0100305, + "balance_loss_clip": 1.00252581, + "balance_loss_mlp": 1.00191188, + "epoch": 0.13530845569032557, + "flos": 71774373093120.0, + "grad_norm": 0.7321141270043975, + "language_loss": 0.47923684, + "learning_rate": 3.8848035594720535e-06, + "loss": 0.49938309, + "num_input_tokens_seen": 132937290, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01141357, + "step": 4663, + "time_per_iteration": 2.965764045715332 + }, + { + "auxiliary_loss_clip": 0.01086771, + "auxiliary_loss_mlp": 0.01040217, + "balance_loss_clip": 1.02991581, + "balance_loss_mlp": 1.02314615, + "epoch": 0.13533747315884162, + "flos": 25001966586240.0, + "grad_norm": 4.391331875743003, + "language_loss": 0.71847588, + "learning_rate": 3.884740681136844e-06, + "loss": 0.73974574, + "num_input_tokens_seen": 132953845, + "router_z_loss_clip": 0.56787109, + "router_z_loss_mlp": 0.17053223, + "step": 4664, + "time_per_iteration": 2.4774131774902344 + }, + { + "auxiliary_loss_clip": 0.01014111, + "auxiliary_loss_mlp": 0.01001401, + "balance_loss_clip": 1.00457227, + "balance_loss_mlp": 1.00038135, + "epoch": 0.13536649062735767, + "flos": 72344643240960.0, + "grad_norm": 0.6632742732027563, + "language_loss": 0.51082015, + "learning_rate": 3.884677786154843e-06, + "loss": 0.53097522, + "num_input_tokens_seen": 133022550, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01019287, + "step": 4665, + "time_per_iteration": 3.174272060394287 + }, + { + "auxiliary_loss_clip": 0.01014158, + "auxiliary_loss_mlp": 0.0100128, + "balance_loss_clip": 1.00478649, + "balance_loss_mlp": 1.00017703, + "epoch": 0.13539550809587372, + "flos": 66418661220480.0, + "grad_norm": 0.6907802370071003, + "language_loss": 0.48148119, + "learning_rate": 3.884614874526604e-06, + "loss": 0.50163555, + "num_input_tokens_seen": 133085495, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.01104736, + "step": 4666, + "time_per_iteration": 3.027543783187866 + }, + { + "auxiliary_loss_clip": 0.01013596, + "auxiliary_loss_mlp": 0.01003664, + "balance_loss_clip": 1.00423145, + "balance_loss_mlp": 1.00264454, + "epoch": 0.13542452556438977, + "flos": 64228419987840.0, + "grad_norm": 0.7613729618044952, + "language_loss": 0.53959441, + "learning_rate": 3.884551946252684e-06, + "loss": 0.55976701, + "num_input_tokens_seen": 133144345, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.01019287, + "step": 4667, + "time_per_iteration": 2.9736289978027344 + }, + { + "auxiliary_loss_clip": 0.01091693, + "auxiliary_loss_mlp": 0.01039513, + "balance_loss_clip": 1.03103793, + "balance_loss_mlp": 1.0202843, + "epoch": 0.1354535430329058, + "flos": 23723796378240.0, + "grad_norm": 2.576307674520749, + "language_loss": 1.14538383, + "learning_rate": 3.88448900133364e-06, + "loss": 1.16669583, + "num_input_tokens_seen": 133159600, + "router_z_loss_clip": 0.60595703, + "router_z_loss_mlp": 0.1920166, + "step": 4668, + "time_per_iteration": 2.4473204612731934 + }, + { + "auxiliary_loss_clip": 0.01014543, + "auxiliary_loss_mlp": 0.01001901, + "balance_loss_clip": 1.00514674, + "balance_loss_mlp": 1.00086403, + "epoch": 0.13548256050142185, + "flos": 64739932450560.0, + "grad_norm": 0.6301502041930449, + "language_loss": 0.45885676, + "learning_rate": 3.8844260397700255e-06, + "loss": 0.47902119, + "num_input_tokens_seen": 133224710, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.01037598, + "step": 4669, + "time_per_iteration": 3.2943124771118164 + }, + { + "auxiliary_loss_clip": 0.01094913, + "auxiliary_loss_mlp": 0.01036049, + "balance_loss_clip": 1.0316658, + "balance_loss_mlp": 1.01656413, + "epoch": 0.1355115779699379, + "flos": 16099393714560.0, + "grad_norm": 2.633408169636056, + "language_loss": 0.91401082, + "learning_rate": 3.884363061562397e-06, + "loss": 0.93532044, + "num_input_tokens_seen": 133237425, + "router_z_loss_clip": 0.6328125, + "router_z_loss_mlp": 0.19512939, + "step": 4670, + "time_per_iteration": 2.3734025955200195 + }, + { + "auxiliary_loss_clip": 0.0108962, + "auxiliary_loss_mlp": 0.01036839, + "balance_loss_clip": 1.02925634, + "balance_loss_mlp": 1.01719904, + "epoch": 0.13554059543845395, + "flos": 16500825060480.0, + "grad_norm": 2.3289560359944663, + "language_loss": 0.7572915, + "learning_rate": 3.884300066711313e-06, + "loss": 0.77855611, + "num_input_tokens_seen": 133250970, + "router_z_loss_clip": 0.60375977, + "router_z_loss_mlp": 0.19647217, + "step": 4671, + "time_per_iteration": 2.3506126403808594 + }, + { + "auxiliary_loss_clip": 0.01015405, + "auxiliary_loss_mlp": 0.01006476, + "balance_loss_clip": 1.00568223, + "balance_loss_mlp": 1.00535583, + "epoch": 0.13556961290697, + "flos": 65863891716480.0, + "grad_norm": 0.6788319740348894, + "language_loss": 0.4800058, + "learning_rate": 3.884237055217327e-06, + "loss": 0.50022465, + "num_input_tokens_seen": 133307290, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.01123047, + "step": 4672, + "time_per_iteration": 2.903615951538086 + }, + { + "auxiliary_loss_clip": 0.0101423, + "auxiliary_loss_mlp": 0.01007877, + "balance_loss_clip": 1.00478566, + "balance_loss_mlp": 1.0067867, + "epoch": 0.13559863037548603, + "flos": 62775928028160.0, + "grad_norm": 0.6224808057650232, + "language_loss": 0.49089122, + "learning_rate": 3.8841740270809974e-06, + "loss": 0.51111233, + "num_input_tokens_seen": 133374450, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01092529, + "step": 4673, + "time_per_iteration": 3.0943636894226074 + }, + { + "auxiliary_loss_clip": 0.01086029, + "auxiliary_loss_mlp": 0.01034337, + "balance_loss_clip": 1.02988863, + "balance_loss_mlp": 1.01809478, + "epoch": 0.13562764784400208, + "flos": 40697520071040.0, + "grad_norm": 1.8666248313110925, + "language_loss": 1.04712021, + "learning_rate": 3.88411098230288e-06, + "loss": 1.06832385, + "num_input_tokens_seen": 133403210, + "router_z_loss_clip": 0.56054688, + "router_z_loss_mlp": 0.16235352, + "step": 4674, + "time_per_iteration": 2.63100528717041 + }, + { + "auxiliary_loss_clip": 0.01090276, + "auxiliary_loss_mlp": 0.01036555, + "balance_loss_clip": 1.0286473, + "balance_loss_mlp": 1.01661706, + "epoch": 0.13565666531251813, + "flos": 15115820492160.0, + "grad_norm": 4.600848693827226, + "language_loss": 0.81447613, + "learning_rate": 3.884047920883532e-06, + "loss": 0.83574444, + "num_input_tokens_seen": 133417000, + "router_z_loss_clip": 0.61621094, + "router_z_loss_mlp": 0.19927979, + "step": 4675, + "time_per_iteration": 2.387747287750244 + }, + { + "auxiliary_loss_clip": 0.01011615, + "auxiliary_loss_mlp": 0.01012949, + "balance_loss_clip": 1.00270104, + "balance_loss_mlp": 1.01164365, + "epoch": 0.13568568278103418, + "flos": 74764823754240.0, + "grad_norm": 0.6875755220144912, + "language_loss": 0.45917559, + "learning_rate": 3.883984842823512e-06, + "loss": 0.4794212, + "num_input_tokens_seen": 133475575, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.01306152, + "step": 4676, + "time_per_iteration": 2.98575758934021 + }, + { + "auxiliary_loss_clip": 0.01089948, + "auxiliary_loss_mlp": 0.01038635, + "balance_loss_clip": 1.03102517, + "balance_loss_mlp": 1.02120638, + "epoch": 0.13571470024955024, + "flos": 17195107824000.0, + "grad_norm": 2.1446348594564353, + "language_loss": 0.77133071, + "learning_rate": 3.883921748123374e-06, + "loss": 0.79261655, + "num_input_tokens_seen": 133490005, + "router_z_loss_clip": 0.58984375, + "router_z_loss_mlp": 0.17437744, + "step": 4677, + "time_per_iteration": 2.402590274810791 + }, + { + "auxiliary_loss_clip": 0.01085381, + "auxiliary_loss_mlp": 0.01040943, + "balance_loss_clip": 1.02879453, + "balance_loss_mlp": 1.02370501, + "epoch": 0.1357437177180663, + "flos": 29562601000320.0, + "grad_norm": 1.9694699731985685, + "language_loss": 0.86082554, + "learning_rate": 3.883858636783676e-06, + "loss": 0.88208872, + "num_input_tokens_seen": 133513735, + "router_z_loss_clip": 0.56640625, + "router_z_loss_mlp": 0.17236328, + "step": 4678, + "time_per_iteration": 2.804335355758667 + }, + { + "auxiliary_loss_clip": 0.01085698, + "auxiliary_loss_mlp": 0.01038293, + "balance_loss_clip": 1.03095531, + "balance_loss_mlp": 1.02213979, + "epoch": 0.1357727351865823, + "flos": 10919818984320.0, + "grad_norm": 3.59460520887831, + "language_loss": 0.85540158, + "learning_rate": 3.883795508804978e-06, + "loss": 0.87664145, + "num_input_tokens_seen": 133523660, + "router_z_loss_clip": 0.54638672, + "router_z_loss_mlp": 0.16162109, + "step": 4679, + "time_per_iteration": 2.360386610031128 + }, + { + "auxiliary_loss_clip": 0.01090424, + "auxiliary_loss_mlp": 0.01050595, + "balance_loss_clip": 1.03074765, + "balance_loss_mlp": 1.03161693, + "epoch": 0.13580175265509836, + "flos": 16353770947200.0, + "grad_norm": 3.945780708367295, + "language_loss": 0.97192347, + "learning_rate": 3.8837323641878345e-06, + "loss": 0.9933337, + "num_input_tokens_seen": 133534385, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.18988037, + "step": 4680, + "time_per_iteration": 2.3255937099456787 + }, + { + "auxiliary_loss_clip": 0.01019061, + "auxiliary_loss_mlp": 0.01004189, + "balance_loss_clip": 1.00922012, + "balance_loss_mlp": 1.00314581, + "epoch": 0.1358307701236144, + "flos": 70100427179520.0, + "grad_norm": 0.7176911502510416, + "language_loss": 0.51263559, + "learning_rate": 3.883669202932805e-06, + "loss": 0.53286803, + "num_input_tokens_seen": 133595740, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01043701, + "step": 4681, + "time_per_iteration": 2.9924685955047607 + }, + { + "auxiliary_loss_clip": 0.01093872, + "auxiliary_loss_mlp": 0.01041496, + "balance_loss_clip": 1.02947044, + "balance_loss_mlp": 1.02217174, + "epoch": 0.13585978759213047, + "flos": 25110790894080.0, + "grad_norm": 3.1923569877918774, + "language_loss": 0.8213383, + "learning_rate": 3.883606025040447e-06, + "loss": 0.84269202, + "num_input_tokens_seen": 133612145, + "router_z_loss_clip": 0.64453125, + "router_z_loss_mlp": 0.19317627, + "step": 4682, + "time_per_iteration": 2.5653791427612305 + }, + { + "auxiliary_loss_clip": 0.01084161, + "auxiliary_loss_mlp": 0.01031756, + "balance_loss_clip": 1.03197575, + "balance_loss_mlp": 1.01622272, + "epoch": 0.13588880506064652, + "flos": 26207796723840.0, + "grad_norm": 3.7361322502909826, + "language_loss": 0.90243125, + "learning_rate": 3.883542830511318e-06, + "loss": 0.92359042, + "num_input_tokens_seen": 133632015, + "router_z_loss_clip": 0.5222168, + "router_z_loss_mlp": 0.15539551, + "step": 4683, + "time_per_iteration": 2.4437499046325684 + }, + { + "auxiliary_loss_clip": 0.01020579, + "auxiliary_loss_mlp": 0.0100169, + "balance_loss_clip": 1.01021922, + "balance_loss_mlp": 1.00058746, + "epoch": 0.13591782252916257, + "flos": 69299414789760.0, + "grad_norm": 0.699140064521986, + "language_loss": 0.4685947, + "learning_rate": 3.8834796193459766e-06, + "loss": 0.48881739, + "num_input_tokens_seen": 133688230, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01104736, + "step": 4684, + "time_per_iteration": 2.9881436824798584 + }, + { + "auxiliary_loss_clip": 0.01090933, + "auxiliary_loss_mlp": 0.01033018, + "balance_loss_clip": 1.03044724, + "balance_loss_mlp": 1.01499367, + "epoch": 0.1359468399976786, + "flos": 18762359022720.0, + "grad_norm": 2.8547873173121436, + "language_loss": 0.84095967, + "learning_rate": 3.883416391544981e-06, + "loss": 0.86219919, + "num_input_tokens_seen": 133700095, + "router_z_loss_clip": 0.60449219, + "router_z_loss_mlp": 0.18029785, + "step": 4685, + "time_per_iteration": 2.4382081031799316 + }, + { + "auxiliary_loss_clip": 0.01020097, + "auxiliary_loss_mlp": 0.01001094, + "balance_loss_clip": 1.01007414, + "balance_loss_mlp": 0.99991369, + "epoch": 0.13597585746619464, + "flos": 74778369361920.0, + "grad_norm": 0.6458305208553664, + "language_loss": 0.50691861, + "learning_rate": 3.88335314710889e-06, + "loss": 0.52713054, + "num_input_tokens_seen": 133768045, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.01177979, + "step": 4686, + "time_per_iteration": 3.1111719608306885 + }, + { + "auxiliary_loss_clip": 0.01018877, + "auxiliary_loss_mlp": 0.00999906, + "balance_loss_clip": 1.00924981, + "balance_loss_mlp": 0.99876732, + "epoch": 0.1360048749347107, + "flos": 74765521981440.0, + "grad_norm": 0.6388149274827246, + "language_loss": 0.44751772, + "learning_rate": 3.883289886038262e-06, + "loss": 0.46770555, + "num_input_tokens_seen": 133833890, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.01141357, + "step": 4687, + "time_per_iteration": 3.2390236854553223 + }, + { + "auxiliary_loss_clip": 0.01016925, + "auxiliary_loss_mlp": 0.0100075, + "balance_loss_clip": 1.00739741, + "balance_loss_mlp": 0.99952257, + "epoch": 0.13603389240322675, + "flos": 56166834879360.0, + "grad_norm": 0.7356066920544723, + "language_loss": 0.51561785, + "learning_rate": 3.883226608333655e-06, + "loss": 0.53579456, + "num_input_tokens_seen": 133891460, + "router_z_loss_clip": 0.09521484, + "router_z_loss_mlp": 0.01226807, + "step": 4688, + "time_per_iteration": 3.0607409477233887 + }, + { + "auxiliary_loss_clip": 0.01016354, + "auxiliary_loss_mlp": 0.01000243, + "balance_loss_clip": 1.00679755, + "balance_loss_mlp": 0.99909234, + "epoch": 0.1360629098717428, + "flos": 59227496017920.0, + "grad_norm": 0.6423317914993296, + "language_loss": 0.52687764, + "learning_rate": 3.883163313995629e-06, + "loss": 0.54704362, + "num_input_tokens_seen": 133953480, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01147461, + "step": 4689, + "time_per_iteration": 3.1118876934051514 + }, + { + "auxiliary_loss_clip": 0.01082862, + "auxiliary_loss_mlp": 0.01031537, + "balance_loss_clip": 1.02840352, + "balance_loss_mlp": 1.01471043, + "epoch": 0.13609192734025882, + "flos": 23981525101440.0, + "grad_norm": 3.6480186876615424, + "language_loss": 0.78561252, + "learning_rate": 3.883100003024743e-06, + "loss": 0.80675656, + "num_input_tokens_seen": 133969580, + "router_z_loss_clip": 0.54467773, + "router_z_loss_mlp": 0.168396, + "step": 4690, + "time_per_iteration": 2.497812032699585 + }, + { + "auxiliary_loss_clip": 0.01014324, + "auxiliary_loss_mlp": 0.01001124, + "balance_loss_clip": 1.00538158, + "balance_loss_mlp": 0.99994349, + "epoch": 0.13612094480877487, + "flos": 74771736203520.0, + "grad_norm": 0.6293072743330068, + "language_loss": 0.48408955, + "learning_rate": 3.883036675421555e-06, + "loss": 0.50424403, + "num_input_tokens_seen": 134038295, + "router_z_loss_clip": 0.08935547, + "router_z_loss_mlp": 0.01177979, + "step": 4691, + "time_per_iteration": 3.2256665229797363 + }, + { + "auxiliary_loss_clip": 0.01087148, + "auxiliary_loss_mlp": 0.01041201, + "balance_loss_clip": 1.02754807, + "balance_loss_mlp": 1.02393973, + "epoch": 0.13614996227729093, + "flos": 13434054433920.0, + "grad_norm": 3.180516092506244, + "language_loss": 0.88533813, + "learning_rate": 3.882973331186625e-06, + "loss": 0.90662158, + "num_input_tokens_seen": 134049135, + "router_z_loss_clip": 0.59521484, + "router_z_loss_mlp": 0.17279053, + "step": 4692, + "time_per_iteration": 2.381603240966797 + }, + { + "auxiliary_loss_clip": 0.01012164, + "auxiliary_loss_mlp": 0.01003993, + "balance_loss_clip": 1.00308847, + "balance_loss_mlp": 1.00286031, + "epoch": 0.13617897974580698, + "flos": 70715841448320.0, + "grad_norm": 0.6446865933079656, + "language_loss": 0.42485297, + "learning_rate": 3.882909970320513e-06, + "loss": 0.44501454, + "num_input_tokens_seen": 134102070, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01135254, + "step": 4693, + "time_per_iteration": 2.945035934448242 + }, + { + "auxiliary_loss_clip": 0.01094132, + "auxiliary_loss_mlp": 0.01035902, + "balance_loss_clip": 1.02938509, + "balance_loss_mlp": 1.01514804, + "epoch": 0.13620799721432303, + "flos": 18034350019200.0, + "grad_norm": 2.2794630853869027, + "language_loss": 0.80153859, + "learning_rate": 3.8828465928237784e-06, + "loss": 0.82283896, + "num_input_tokens_seen": 134117085, + "router_z_loss_clip": 0.64648438, + "router_z_loss_mlp": 0.20739746, + "step": 4694, + "time_per_iteration": 2.5866870880126953 + }, + { + "auxiliary_loss_clip": 0.01086279, + "auxiliary_loss_mlp": 0.01039907, + "balance_loss_clip": 1.0295434, + "balance_loss_mlp": 1.02049947, + "epoch": 0.13623701468283908, + "flos": 16318124582400.0, + "grad_norm": 2.3113462781928287, + "language_loss": 0.74520588, + "learning_rate": 3.88278319869698e-06, + "loss": 0.76646781, + "num_input_tokens_seen": 134129670, + "router_z_loss_clip": 0.56689453, + "router_z_loss_mlp": 0.19433594, + "step": 4695, + "time_per_iteration": 2.4465208053588867 + }, + { + "auxiliary_loss_clip": 0.01084926, + "auxiliary_loss_mlp": 0.0103968, + "balance_loss_clip": 1.02874207, + "balance_loss_mlp": 1.02382553, + "epoch": 0.1362660321513551, + "flos": 18068949043200.0, + "grad_norm": 2.6884412233195136, + "language_loss": 0.75380147, + "learning_rate": 3.882719787940679e-06, + "loss": 0.77504754, + "num_input_tokens_seen": 134141495, + "router_z_loss_clip": 0.56176758, + "router_z_loss_mlp": 0.15869141, + "step": 4696, + "time_per_iteration": 2.3271026611328125 + }, + { + "auxiliary_loss_clip": 0.01086976, + "auxiliary_loss_mlp": 0.01042357, + "balance_loss_clip": 1.02995038, + "balance_loss_mlp": 1.02519119, + "epoch": 0.13629504961987116, + "flos": 20113008946560.0, + "grad_norm": 2.4047603105763353, + "language_loss": 0.85210842, + "learning_rate": 3.882656360555435e-06, + "loss": 0.87340176, + "num_input_tokens_seen": 134154030, + "router_z_loss_clip": 0.56982422, + "router_z_loss_mlp": 0.17144775, + "step": 4697, + "time_per_iteration": 2.427304983139038 + }, + { + "auxiliary_loss_clip": 0.01088892, + "auxiliary_loss_mlp": 0.01046888, + "balance_loss_clip": 1.02812517, + "balance_loss_mlp": 1.02274811, + "epoch": 0.1363240670883872, + "flos": 25076715540480.0, + "grad_norm": 2.501158252370298, + "language_loss": 0.68812001, + "learning_rate": 3.882592916541808e-06, + "loss": 0.70947784, + "num_input_tokens_seen": 134171320, + "router_z_loss_clip": 0.60791016, + "router_z_loss_mlp": 0.24151611, + "step": 4698, + "time_per_iteration": 2.4219207763671875 + }, + { + "auxiliary_loss_clip": 0.0108946, + "auxiliary_loss_mlp": 0.01034016, + "balance_loss_clip": 1.02926481, + "balance_loss_mlp": 1.01605153, + "epoch": 0.13635308455690326, + "flos": 11356477758720.0, + "grad_norm": 2.9713717173869076, + "language_loss": 0.75746131, + "learning_rate": 3.882529455900359e-06, + "loss": 0.77869612, + "num_input_tokens_seen": 134182020, + "router_z_loss_clip": 0.60107422, + "router_z_loss_mlp": 0.17956543, + "step": 4699, + "time_per_iteration": 2.357727289199829 + }, + { + "auxiliary_loss_clip": 0.01016206, + "auxiliary_loss_mlp": 0.01005294, + "balance_loss_clip": 1.00670731, + "balance_loss_mlp": 1.0039829, + "epoch": 0.1363821020254193, + "flos": 74771282355840.0, + "grad_norm": 0.6225399677870497, + "language_loss": 0.43988019, + "learning_rate": 3.8824659786316474e-06, + "loss": 0.4600952, + "num_input_tokens_seen": 134250615, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01312256, + "step": 4700, + "time_per_iteration": 3.236222743988037 + }, + { + "auxiliary_loss_clip": 0.0109201, + "auxiliary_loss_mlp": 0.01046639, + "balance_loss_clip": 1.03171599, + "balance_loss_mlp": 1.02754784, + "epoch": 0.13641111949393536, + "flos": 14054426115840.0, + "grad_norm": 3.088758112452515, + "language_loss": 0.7455579, + "learning_rate": 3.882402484736235e-06, + "loss": 0.76694441, + "num_input_tokens_seen": 134262230, + "router_z_loss_clip": 0.60253906, + "router_z_loss_mlp": 0.19110107, + "step": 4701, + "time_per_iteration": 2.3918302059173584 + }, + { + "auxiliary_loss_clip": 0.01085703, + "auxiliary_loss_mlp": 0.01034223, + "balance_loss_clip": 1.03008699, + "balance_loss_mlp": 1.01826096, + "epoch": 0.13644013696245139, + "flos": 20549807366400.0, + "grad_norm": 2.3915070913235654, + "language_loss": 0.74180543, + "learning_rate": 3.8823389742146816e-06, + "loss": 0.76300466, + "num_input_tokens_seen": 134274630, + "router_z_loss_clip": 0.5559082, + "router_z_loss_mlp": 0.15960693, + "step": 4702, + "time_per_iteration": 2.3395586013793945 + }, + { + "auxiliary_loss_clip": 0.01016564, + "auxiliary_loss_mlp": 0.01007022, + "balance_loss_clip": 1.00698352, + "balance_loss_mlp": 1.00562739, + "epoch": 0.13646915443096744, + "flos": 66159152017920.0, + "grad_norm": 0.676404055230391, + "language_loss": 0.50158864, + "learning_rate": 3.88227544706755e-06, + "loss": 0.52182448, + "num_input_tokens_seen": 134337495, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01397705, + "step": 4703, + "time_per_iteration": 3.018831491470337 + }, + { + "auxiliary_loss_clip": 0.01100936, + "auxiliary_loss_mlp": 0.01039277, + "balance_loss_clip": 1.03556991, + "balance_loss_mlp": 1.01728296, + "epoch": 0.1364981718994835, + "flos": 25329172648320.0, + "grad_norm": 2.9770402664970064, + "language_loss": 1.04749167, + "learning_rate": 3.882211903295399e-06, + "loss": 1.06889379, + "num_input_tokens_seen": 134350945, + "router_z_loss_clip": 0.65332031, + "router_z_loss_mlp": 0.22009277, + "step": 4704, + "time_per_iteration": 2.495537042617798 + }, + { + "auxiliary_loss_clip": 0.01095156, + "auxiliary_loss_mlp": 0.01037255, + "balance_loss_clip": 1.03246212, + "balance_loss_mlp": 1.01780057, + "epoch": 0.13652718936799954, + "flos": 30621516670080.0, + "grad_norm": 2.5223309082846956, + "language_loss": 0.93960971, + "learning_rate": 3.882148342898791e-06, + "loss": 0.96093386, + "num_input_tokens_seen": 134370750, + "router_z_loss_clip": 0.62695312, + "router_z_loss_mlp": 0.19464111, + "step": 4705, + "time_per_iteration": 2.4845998287200928 + }, + { + "auxiliary_loss_clip": 0.01017878, + "auxiliary_loss_mlp": 0.01005802, + "balance_loss_clip": 1.00749338, + "balance_loss_mlp": 1.00441372, + "epoch": 0.1365562068365156, + "flos": 74630512287360.0, + "grad_norm": 0.655829091124065, + "language_loss": 0.52590549, + "learning_rate": 3.882084765878287e-06, + "loss": 0.54614234, + "num_input_tokens_seen": 134437385, + "router_z_loss_clip": 0.10400391, + "router_z_loss_mlp": 0.01391602, + "step": 4706, + "time_per_iteration": 3.1155879497528076 + }, + { + "auxiliary_loss_clip": 0.01090547, + "auxiliary_loss_mlp": 0.01038806, + "balance_loss_clip": 1.03280711, + "balance_loss_mlp": 1.02065659, + "epoch": 0.13658522430503162, + "flos": 27191265212160.0, + "grad_norm": 2.1391696968188603, + "language_loss": 0.75008178, + "learning_rate": 3.882021172234449e-06, + "loss": 0.7713753, + "num_input_tokens_seen": 134453295, + "router_z_loss_clip": 0.57714844, + "router_z_loss_mlp": 0.18170166, + "step": 4707, + "time_per_iteration": 2.782820463180542 + }, + { + "auxiliary_loss_clip": 0.01093548, + "auxiliary_loss_mlp": 0.01037576, + "balance_loss_clip": 1.03368759, + "balance_loss_mlp": 1.01963472, + "epoch": 0.13661424177354767, + "flos": 28247806909440.0, + "grad_norm": 2.286151212617775, + "language_loss": 0.87980855, + "learning_rate": 3.8819575619678384e-06, + "loss": 0.90111977, + "num_input_tokens_seen": 134476635, + "router_z_loss_clip": 0.59912109, + "router_z_loss_mlp": 0.17932129, + "step": 4708, + "time_per_iteration": 2.8173441886901855 + }, + { + "auxiliary_loss_clip": 0.01093118, + "auxiliary_loss_mlp": 0.01039584, + "balance_loss_clip": 1.03277016, + "balance_loss_mlp": 1.02015853, + "epoch": 0.13664325924206372, + "flos": 31898639537280.0, + "grad_norm": 2.005214205349342, + "language_loss": 0.90427959, + "learning_rate": 3.881893935079017e-06, + "loss": 0.92560661, + "num_input_tokens_seen": 134493600, + "router_z_loss_clip": 0.60400391, + "router_z_loss_mlp": 0.1942749, + "step": 4709, + "time_per_iteration": 2.4984207153320312 + }, + { + "auxiliary_loss_clip": 0.01018857, + "auxiliary_loss_mlp": 0.0100899, + "balance_loss_clip": 1.00849128, + "balance_loss_mlp": 1.00780964, + "epoch": 0.13667227671057977, + "flos": 59231859937920.0, + "grad_norm": 0.6669568298826707, + "language_loss": 0.48582137, + "learning_rate": 3.881830291568546e-06, + "loss": 0.50609982, + "num_input_tokens_seen": 134556420, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01177979, + "step": 4710, + "time_per_iteration": 3.035173177719116 + }, + { + "auxiliary_loss_clip": 0.01092237, + "auxiliary_loss_mlp": 0.0104112, + "balance_loss_clip": 1.03159022, + "balance_loss_mlp": 1.0229764, + "epoch": 0.13670129417909582, + "flos": 23140362781440.0, + "grad_norm": 3.1941161359391743, + "language_loss": 0.88609111, + "learning_rate": 3.88176663143699e-06, + "loss": 0.90742469, + "num_input_tokens_seen": 134568695, + "router_z_loss_clip": 0.60644531, + "router_z_loss_mlp": 0.18151855, + "step": 4711, + "time_per_iteration": 2.3935000896453857 + }, + { + "auxiliary_loss_clip": 0.01091666, + "auxiliary_loss_mlp": 0.01047223, + "balance_loss_clip": 1.02995205, + "balance_loss_mlp": 1.02881169, + "epoch": 0.13673031164761187, + "flos": 28867585098240.0, + "grad_norm": 3.302613933226164, + "language_loss": 0.80129594, + "learning_rate": 3.881702954684908e-06, + "loss": 0.82268488, + "num_input_tokens_seen": 134582285, + "router_z_loss_clip": 0.61645508, + "router_z_loss_mlp": 0.18414307, + "step": 4712, + "time_per_iteration": 2.4999921321868896 + }, + { + "auxiliary_loss_clip": 0.01016752, + "auxiliary_loss_mlp": 0.01010009, + "balance_loss_clip": 1.00611758, + "balance_loss_mlp": 1.00866771, + "epoch": 0.1367593291161279, + "flos": 67728602632320.0, + "grad_norm": 0.688972810842279, + "language_loss": 0.47320768, + "learning_rate": 3.8816392613128654e-06, + "loss": 0.49347526, + "num_input_tokens_seen": 134641820, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01342773, + "step": 4713, + "time_per_iteration": 2.9615819454193115 + }, + { + "auxiliary_loss_clip": 0.01090633, + "auxiliary_loss_mlp": 0.01045736, + "balance_loss_clip": 1.03016901, + "balance_loss_mlp": 1.02473164, + "epoch": 0.13678834658464395, + "flos": 11719434919680.0, + "grad_norm": 3.2502293063531256, + "language_loss": 0.89740789, + "learning_rate": 3.881575551321423e-06, + "loss": 0.91877162, + "num_input_tokens_seen": 134651690, + "router_z_loss_clip": 0.60522461, + "router_z_loss_mlp": 0.21008301, + "step": 4714, + "time_per_iteration": 2.4765357971191406 + }, + { + "auxiliary_loss_clip": 0.01091178, + "auxiliary_loss_mlp": 0.01049489, + "balance_loss_clip": 1.02965927, + "balance_loss_mlp": 1.02939057, + "epoch": 0.13681736405316, + "flos": 13290351811200.0, + "grad_norm": 3.0960372970992913, + "language_loss": 0.84794772, + "learning_rate": 3.881511824711143e-06, + "loss": 0.86935443, + "num_input_tokens_seen": 134663315, + "router_z_loss_clip": 0.61499023, + "router_z_loss_mlp": 0.20092773, + "step": 4715, + "time_per_iteration": 2.3520493507385254 + }, + { + "auxiliary_loss_clip": 0.01087559, + "auxiliary_loss_mlp": 0.01038612, + "balance_loss_clip": 1.02871108, + "balance_loss_mlp": 1.01969385, + "epoch": 0.13684638152167605, + "flos": 57291529150080.0, + "grad_norm": 2.40227750647163, + "language_loss": 0.74584579, + "learning_rate": 3.88144808148259e-06, + "loss": 0.76710749, + "num_input_tokens_seen": 134687420, + "router_z_loss_clip": 0.58837891, + "router_z_loss_mlp": 0.18933105, + "step": 4716, + "time_per_iteration": 2.7865264415740967 + }, + { + "auxiliary_loss_clip": 0.0109001, + "auxiliary_loss_mlp": 0.01052627, + "balance_loss_clip": 1.02772522, + "balance_loss_mlp": 1.03414905, + "epoch": 0.1368753989901921, + "flos": 51238427402880.0, + "grad_norm": 2.171209011062441, + "language_loss": 0.78002244, + "learning_rate": 3.881384321636327e-06, + "loss": 0.80144882, + "num_input_tokens_seen": 134718355, + "router_z_loss_clip": 0.62353516, + "router_z_loss_mlp": 0.18481445, + "step": 4717, + "time_per_iteration": 2.780945301055908 + }, + { + "auxiliary_loss_clip": 0.01097315, + "auxiliary_loss_mlp": 0.01054795, + "balance_loss_clip": 1.03223634, + "balance_loss_mlp": 1.03457737, + "epoch": 0.13690441645870816, + "flos": 15917147084160.0, + "grad_norm": 4.520669583121826, + "language_loss": 0.78549194, + "learning_rate": 3.881320545172915e-06, + "loss": 0.80701303, + "num_input_tokens_seen": 134732355, + "router_z_loss_clip": 0.65063477, + "router_z_loss_mlp": 0.20214844, + "step": 4718, + "time_per_iteration": 4.541724681854248 + }, + { + "auxiliary_loss_clip": 0.01085823, + "auxiliary_loss_mlp": 0.01037519, + "balance_loss_clip": 1.02823341, + "balance_loss_mlp": 1.01830232, + "epoch": 0.13693343392722418, + "flos": 28834138149120.0, + "grad_norm": 2.2857505515474545, + "language_loss": 0.72441977, + "learning_rate": 3.88125675209292e-06, + "loss": 0.74565321, + "num_input_tokens_seen": 134749280, + "router_z_loss_clip": 0.57617188, + "router_z_loss_mlp": 0.19238281, + "step": 4719, + "time_per_iteration": 4.822210311889648 + }, + { + "auxiliary_loss_clip": 0.01019557, + "auxiliary_loss_mlp": 0.01002145, + "balance_loss_clip": 1.00974965, + "balance_loss_mlp": 1.00075042, + "epoch": 0.13696245139574023, + "flos": 74776309591680.0, + "grad_norm": 0.7376960783948492, + "language_loss": 0.45924163, + "learning_rate": 3.881192942396903e-06, + "loss": 0.47945866, + "num_input_tokens_seen": 134817295, + "router_z_loss_clip": 0.09814453, + "router_z_loss_mlp": 0.01397705, + "step": 4720, + "time_per_iteration": 3.08807110786438 + }, + { + "auxiliary_loss_clip": 0.01095599, + "auxiliary_loss_mlp": 0.01035245, + "balance_loss_clip": 1.03079033, + "balance_loss_mlp": 1.01582551, + "epoch": 0.13699146886425628, + "flos": 29233789015680.0, + "grad_norm": 2.983546581363283, + "language_loss": 0.79539102, + "learning_rate": 3.8811291160854285e-06, + "loss": 0.8166995, + "num_input_tokens_seen": 134831285, + "router_z_loss_clip": 0.64794922, + "router_z_loss_mlp": 0.19421387, + "step": 4721, + "time_per_iteration": 2.362093687057495 + }, + { + "auxiliary_loss_clip": 0.01090611, + "auxiliary_loss_mlp": 0.01038272, + "balance_loss_clip": 1.03174412, + "balance_loss_mlp": 1.01938939, + "epoch": 0.13702048633277233, + "flos": 12013752614400.0, + "grad_norm": 2.4840867375383073, + "language_loss": 0.74952197, + "learning_rate": 3.8810652731590615e-06, + "loss": 0.77081084, + "num_input_tokens_seen": 134843360, + "router_z_loss_clip": 0.58862305, + "router_z_loss_mlp": 0.18884277, + "step": 4722, + "time_per_iteration": 2.363279104232788 + }, + { + "auxiliary_loss_clip": 0.01091151, + "auxiliary_loss_mlp": 0.01032491, + "balance_loss_clip": 1.03073299, + "balance_loss_mlp": 1.01228523, + "epoch": 0.13704950380128839, + "flos": 18176900567040.0, + "grad_norm": 2.3661257712574937, + "language_loss": 0.78878075, + "learning_rate": 3.881001413618364e-06, + "loss": 0.81001717, + "num_input_tokens_seen": 134854710, + "router_z_loss_clip": 0.60302734, + "router_z_loss_mlp": 0.20214844, + "step": 4723, + "time_per_iteration": 2.376284122467041 + }, + { + "auxiliary_loss_clip": 0.01015804, + "auxiliary_loss_mlp": 0.01001236, + "balance_loss_clip": 1.00660253, + "balance_loss_mlp": 0.99986547, + "epoch": 0.1370785212698044, + "flos": 64031545497600.0, + "grad_norm": 0.7278738168792711, + "language_loss": 0.49852979, + "learning_rate": 3.880937537463901e-06, + "loss": 0.51870018, + "num_input_tokens_seen": 134902590, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01373291, + "step": 4724, + "time_per_iteration": 2.79606032371521 + }, + { + "auxiliary_loss_clip": 0.0109928, + "auxiliary_loss_mlp": 0.01044146, + "balance_loss_clip": 1.03550816, + "balance_loss_mlp": 1.02452445, + "epoch": 0.13710753873832046, + "flos": 14969674074240.0, + "grad_norm": 2.3326617850080966, + "language_loss": 0.82992387, + "learning_rate": 3.880873644696237e-06, + "loss": 0.85135818, + "num_input_tokens_seen": 134915285, + "router_z_loss_clip": 0.63720703, + "router_z_loss_mlp": 0.19628906, + "step": 4725, + "time_per_iteration": 2.365342378616333 + }, + { + "auxiliary_loss_clip": 0.01015497, + "auxiliary_loss_mlp": 0.01002249, + "balance_loss_clip": 1.00638628, + "balance_loss_mlp": 1.00104547, + "epoch": 0.1371365562068365, + "flos": 67397556320640.0, + "grad_norm": 0.6730386354634422, + "language_loss": 0.44345209, + "learning_rate": 3.880809735315935e-06, + "loss": 0.46362954, + "num_input_tokens_seen": 134972375, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01202393, + "step": 4726, + "time_per_iteration": 5.38751482963562 + }, + { + "auxiliary_loss_clip": 0.0110045, + "auxiliary_loss_mlp": 0.01039515, + "balance_loss_clip": 1.03303218, + "balance_loss_mlp": 1.01688945, + "epoch": 0.13716557367535256, + "flos": 38319830415360.0, + "grad_norm": 2.4567575716349754, + "language_loss": 0.94810367, + "learning_rate": 3.880745809323561e-06, + "loss": 0.96950328, + "num_input_tokens_seen": 134992315, + "router_z_loss_clip": 0.67529297, + "router_z_loss_mlp": 0.22631836, + "step": 4727, + "time_per_iteration": 4.9823431968688965 + }, + { + "auxiliary_loss_clip": 0.01090458, + "auxiliary_loss_mlp": 0.01039843, + "balance_loss_clip": 1.0299859, + "balance_loss_mlp": 1.01941025, + "epoch": 0.13719459114386862, + "flos": 21170877275520.0, + "grad_norm": 3.605355588116995, + "language_loss": 0.72515547, + "learning_rate": 3.880681866719679e-06, + "loss": 0.74645853, + "num_input_tokens_seen": 135005075, + "router_z_loss_clip": 0.60449219, + "router_z_loss_mlp": 0.2043457, + "step": 4728, + "time_per_iteration": 2.3975234031677246 + }, + { + "auxiliary_loss_clip": 0.01090214, + "auxiliary_loss_mlp": 0.01040509, + "balance_loss_clip": 1.03039753, + "balance_loss_mlp": 1.02209103, + "epoch": 0.13722360861238467, + "flos": 9422987731200.0, + "grad_norm": 2.7982252984603364, + "language_loss": 0.82925767, + "learning_rate": 3.880617907504854e-06, + "loss": 0.8505649, + "num_input_tokens_seen": 135015710, + "router_z_loss_clip": 0.59838867, + "router_z_loss_mlp": 0.1842041, + "step": 4729, + "time_per_iteration": 2.333828926086426 + }, + { + "auxiliary_loss_clip": 0.01084764, + "auxiliary_loss_mlp": 0.01039464, + "balance_loss_clip": 1.02876318, + "balance_loss_mlp": 1.02262568, + "epoch": 0.1372526260809007, + "flos": 22631433759360.0, + "grad_norm": 9.837220374419918, + "language_loss": 0.79801834, + "learning_rate": 3.88055393167965e-06, + "loss": 0.8192606, + "num_input_tokens_seen": 135031490, + "router_z_loss_clip": 0.56005859, + "router_z_loss_mlp": 0.16851807, + "step": 4730, + "time_per_iteration": 2.4550178050994873 + }, + { + "auxiliary_loss_clip": 0.01092764, + "auxiliary_loss_mlp": 0.01039552, + "balance_loss_clip": 1.03241682, + "balance_loss_mlp": 1.02053857, + "epoch": 0.13728164354941674, + "flos": 13909815797760.0, + "grad_norm": 2.7610818503440444, + "language_loss": 0.94061542, + "learning_rate": 3.880489939244633e-06, + "loss": 0.96193862, + "num_input_tokens_seen": 135043795, + "router_z_loss_clip": 0.60302734, + "router_z_loss_mlp": 0.19000244, + "step": 4731, + "time_per_iteration": 2.523017644882202 + }, + { + "auxiliary_loss_clip": 0.01106929, + "auxiliary_loss_mlp": 0.0105605, + "balance_loss_clip": 1.03450298, + "balance_loss_mlp": 1.03155196, + "epoch": 0.1373106610179328, + "flos": 11537083555200.0, + "grad_norm": 2.073117178863227, + "language_loss": 0.74293041, + "learning_rate": 3.880425930200368e-06, + "loss": 0.7645601, + "num_input_tokens_seen": 135055140, + "router_z_loss_clip": 0.72460938, + "router_z_loss_mlp": 0.24493408, + "step": 4732, + "time_per_iteration": 2.4475722312927246 + }, + { + "auxiliary_loss_clip": 0.01093535, + "auxiliary_loss_mlp": 0.01046667, + "balance_loss_clip": 1.03246415, + "balance_loss_mlp": 1.02755785, + "epoch": 0.13733967848644885, + "flos": 31900315282560.0, + "grad_norm": 2.3910740581432695, + "language_loss": 1.02894282, + "learning_rate": 3.88036190454742e-06, + "loss": 1.05034471, + "num_input_tokens_seen": 135073165, + "router_z_loss_clip": 0.61083984, + "router_z_loss_mlp": 0.19116211, + "step": 4733, + "time_per_iteration": 2.4850263595581055 + }, + { + "auxiliary_loss_clip": 0.01095267, + "auxiliary_loss_mlp": 0.01036888, + "balance_loss_clip": 1.03294981, + "balance_loss_mlp": 1.01667571, + "epoch": 0.1373686959549649, + "flos": 16622287280640.0, + "grad_norm": 2.4443319243057653, + "language_loss": 0.67266285, + "learning_rate": 3.880297862286355e-06, + "loss": 0.69398439, + "num_input_tokens_seen": 135088555, + "router_z_loss_clip": 0.62353516, + "router_z_loss_mlp": 0.2020874, + "step": 4734, + "time_per_iteration": 2.544631242752075 + }, + { + "auxiliary_loss_clip": 0.01090546, + "auxiliary_loss_mlp": 0.01032726, + "balance_loss_clip": 1.03152442, + "balance_loss_mlp": 1.01455855, + "epoch": 0.13739771342348092, + "flos": 13513970269440.0, + "grad_norm": 3.0838505689136193, + "language_loss": 0.71738064, + "learning_rate": 3.880233803417738e-06, + "loss": 0.73861337, + "num_input_tokens_seen": 135102305, + "router_z_loss_clip": 0.59033203, + "router_z_loss_mlp": 0.18188477, + "step": 4735, + "time_per_iteration": 2.357090950012207 + }, + { + "auxiliary_loss_clip": 0.01014454, + "auxiliary_loss_mlp": 0.01010375, + "balance_loss_clip": 1.00535393, + "balance_loss_mlp": 1.00918889, + "epoch": 0.13742673089199697, + "flos": 64743179207040.0, + "grad_norm": 0.6111326277482156, + "language_loss": 0.47619903, + "learning_rate": 3.880169727942135e-06, + "loss": 0.49644732, + "num_input_tokens_seen": 135167735, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01184082, + "step": 4736, + "time_per_iteration": 3.16904354095459 + }, + { + "auxiliary_loss_clip": 0.01098257, + "auxiliary_loss_mlp": 0.01041688, + "balance_loss_clip": 1.03248215, + "balance_loss_mlp": 1.02058756, + "epoch": 0.13745574836051302, + "flos": 15989696622720.0, + "grad_norm": 3.8605141089646997, + "language_loss": 1.21354532, + "learning_rate": 3.8801056358601125e-06, + "loss": 1.23494482, + "num_input_tokens_seen": 135178365, + "router_z_loss_clip": 0.65820312, + "router_z_loss_mlp": 0.21105957, + "step": 4737, + "time_per_iteration": 2.3561222553253174 + }, + { + "auxiliary_loss_clip": 0.01092027, + "auxiliary_loss_mlp": 0.01046927, + "balance_loss_clip": 1.03126645, + "balance_loss_mlp": 1.02750802, + "epoch": 0.13748476582902908, + "flos": 17127236407680.0, + "grad_norm": 2.9498283535043455, + "language_loss": 0.75362885, + "learning_rate": 3.880041527172237e-06, + "loss": 0.77501845, + "num_input_tokens_seen": 135193470, + "router_z_loss_clip": 0.60791016, + "router_z_loss_mlp": 0.19433594, + "step": 4738, + "time_per_iteration": 2.3578450679779053 + }, + { + "auxiliary_loss_clip": 0.01082791, + "auxiliary_loss_mlp": 0.0103198, + "balance_loss_clip": 1.02890909, + "balance_loss_mlp": 1.01558304, + "epoch": 0.13751378329754513, + "flos": 15588544567680.0, + "grad_norm": 2.996189551293049, + "language_loss": 0.94543731, + "learning_rate": 3.879977401879073e-06, + "loss": 0.96658504, + "num_input_tokens_seen": 135205510, + "router_z_loss_clip": 0.5390625, + "router_z_loss_mlp": 0.16387939, + "step": 4739, + "time_per_iteration": 2.3222529888153076 + }, + { + "auxiliary_loss_clip": 0.01096069, + "auxiliary_loss_mlp": 0.01043518, + "balance_loss_clip": 1.03155434, + "balance_loss_mlp": 1.0224061, + "epoch": 0.13754280076606118, + "flos": 32048626204800.0, + "grad_norm": 2.2784195189557863, + "language_loss": 0.8175208, + "learning_rate": 3.8799132599811875e-06, + "loss": 0.83891666, + "num_input_tokens_seen": 135224380, + "router_z_loss_clip": 0.64501953, + "router_z_loss_mlp": 0.21118164, + "step": 4740, + "time_per_iteration": 2.486950635910034 + }, + { + "auxiliary_loss_clip": 0.01092049, + "auxiliary_loss_mlp": 0.01043145, + "balance_loss_clip": 1.02931249, + "balance_loss_mlp": 1.02242637, + "epoch": 0.1375718182345772, + "flos": 16574003003520.0, + "grad_norm": 4.021122904685371, + "language_loss": 0.94265479, + "learning_rate": 3.879849101479148e-06, + "loss": 0.96400678, + "num_input_tokens_seen": 135238255, + "router_z_loss_clip": 0.62695312, + "router_z_loss_mlp": 0.20727539, + "step": 4741, + "time_per_iteration": 2.399068832397461 + }, + { + "auxiliary_loss_clip": 0.01098204, + "auxiliary_loss_mlp": 0.01045372, + "balance_loss_clip": 1.03106987, + "balance_loss_mlp": 1.02230549, + "epoch": 0.13760083570309325, + "flos": 16612058252160.0, + "grad_norm": 2.59817213974627, + "language_loss": 0.85971427, + "learning_rate": 3.879784926373521e-06, + "loss": 0.88115007, + "num_input_tokens_seen": 135253140, + "router_z_loss_clip": 0.67089844, + "router_z_loss_mlp": 0.23071289, + "step": 4742, + "time_per_iteration": 2.3971245288848877 + }, + { + "auxiliary_loss_clip": 0.01091941, + "auxiliary_loss_mlp": 0.01042749, + "balance_loss_clip": 1.02932358, + "balance_loss_mlp": 1.02145791, + "epoch": 0.1376298531716093, + "flos": 19975415811840.0, + "grad_norm": 2.2728825103398163, + "language_loss": 0.89997214, + "learning_rate": 3.879720734664872e-06, + "loss": 0.92131901, + "num_input_tokens_seen": 135268845, + "router_z_loss_clip": 0.62597656, + "router_z_loss_mlp": 0.21289062, + "step": 4743, + "time_per_iteration": 2.3831870555877686 + }, + { + "auxiliary_loss_clip": 0.01091486, + "auxiliary_loss_mlp": 0.01044605, + "balance_loss_clip": 1.02927613, + "balance_loss_mlp": 1.02518606, + "epoch": 0.13765887064012536, + "flos": 44775934519680.0, + "grad_norm": 2.3978377678133103, + "language_loss": 0.76853496, + "learning_rate": 3.879656526353769e-06, + "loss": 0.78989589, + "num_input_tokens_seen": 135285445, + "router_z_loss_clip": 0.62280273, + "router_z_loss_mlp": 0.19433594, + "step": 4744, + "time_per_iteration": 2.626538038253784 + }, + { + "auxiliary_loss_clip": 0.01014094, + "auxiliary_loss_mlp": 0.01001265, + "balance_loss_clip": 1.0051192, + "balance_loss_mlp": 0.99991202, + "epoch": 0.1376878881086414, + "flos": 63159659314560.0, + "grad_norm": 0.7471251467839425, + "language_loss": 0.50217128, + "learning_rate": 3.879592301440779e-06, + "loss": 0.52232492, + "num_input_tokens_seen": 135338580, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.0135498, + "step": 4745, + "time_per_iteration": 2.980947971343994 + }, + { + "auxiliary_loss_clip": 0.01085583, + "auxiliary_loss_mlp": 0.01044625, + "balance_loss_clip": 1.02967179, + "balance_loss_mlp": 1.02744651, + "epoch": 0.13771690557715746, + "flos": 15287103089280.0, + "grad_norm": 2.2843395865043687, + "language_loss": 0.78771734, + "learning_rate": 3.8795280599264695e-06, + "loss": 0.80901933, + "num_input_tokens_seen": 135355230, + "router_z_loss_clip": 0.55908203, + "router_z_loss_mlp": 0.17181396, + "step": 4746, + "time_per_iteration": 2.4678642749786377 + }, + { + "auxiliary_loss_clip": 0.01011355, + "auxiliary_loss_mlp": 0.01004186, + "balance_loss_clip": 1.00263095, + "balance_loss_mlp": 1.00272572, + "epoch": 0.13774592304567349, + "flos": 52550880566400.0, + "grad_norm": 0.6163218854833942, + "language_loss": 0.46288866, + "learning_rate": 3.879463801811408e-06, + "loss": 0.48304409, + "num_input_tokens_seen": 135414820, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.0145874, + "step": 4747, + "time_per_iteration": 2.9946630001068115 + }, + { + "auxiliary_loss_clip": 0.01086821, + "auxiliary_loss_mlp": 0.01033538, + "balance_loss_clip": 1.02657688, + "balance_loss_mlp": 1.0150485, + "epoch": 0.13777494051418954, + "flos": 16755656140800.0, + "grad_norm": 2.585387268291355, + "language_loss": 0.66283309, + "learning_rate": 3.87939952709616e-06, + "loss": 0.68403667, + "num_input_tokens_seen": 135427055, + "router_z_loss_clip": 0.60253906, + "router_z_loss_mlp": 0.18487549, + "step": 4748, + "time_per_iteration": 2.4012415409088135 + }, + { + "auxiliary_loss_clip": 0.01084255, + "auxiliary_loss_mlp": 0.01038507, + "balance_loss_clip": 1.02697515, + "balance_loss_mlp": 1.02168655, + "epoch": 0.1378039579827056, + "flos": 17229741759360.0, + "grad_norm": 3.857006010152664, + "language_loss": 0.81248671, + "learning_rate": 3.879335235781297e-06, + "loss": 0.83371437, + "num_input_tokens_seen": 135439870, + "router_z_loss_clip": 0.57226562, + "router_z_loss_mlp": 0.16845703, + "step": 4749, + "time_per_iteration": 2.3577160835266113 + }, + { + "auxiliary_loss_clip": 0.01093604, + "auxiliary_loss_mlp": 0.01039869, + "balance_loss_clip": 1.02886915, + "balance_loss_mlp": 1.01903129, + "epoch": 0.13783297545122164, + "flos": 25337970311040.0, + "grad_norm": 1.5152883191310802, + "language_loss": 0.85689747, + "learning_rate": 3.8792709278673824e-06, + "loss": 0.87823218, + "num_input_tokens_seen": 135465265, + "router_z_loss_clip": 0.64697266, + "router_z_loss_mlp": 0.20849609, + "step": 4750, + "time_per_iteration": 2.5998542308807373 + }, + { + "auxiliary_loss_clip": 0.01101922, + "auxiliary_loss_mlp": 0.01046834, + "balance_loss_clip": 1.03309095, + "balance_loss_mlp": 1.02413678, + "epoch": 0.1378619929197377, + "flos": 29488375716480.0, + "grad_norm": 2.560382976448682, + "language_loss": 0.99053568, + "learning_rate": 3.8792066033549885e-06, + "loss": 1.01202321, + "num_input_tokens_seen": 135485220, + "router_z_loss_clip": 0.68847656, + "router_z_loss_mlp": 0.22705078, + "step": 4751, + "time_per_iteration": 2.460102081298828 + }, + { + "auxiliary_loss_clip": 0.01012176, + "auxiliary_loss_mlp": 0.01006049, + "balance_loss_clip": 1.00314569, + "balance_loss_mlp": 1.00467801, + "epoch": 0.13789101038825372, + "flos": 73567721456640.0, + "grad_norm": 0.6508928627918528, + "language_loss": 0.49124026, + "learning_rate": 3.879142262244681e-06, + "loss": 0.51142251, + "num_input_tokens_seen": 135549555, + "router_z_loss_clip": 0.09033203, + "router_z_loss_mlp": 0.01373291, + "step": 4752, + "time_per_iteration": 3.080665349960327 + }, + { + "auxiliary_loss_clip": 0.01095261, + "auxiliary_loss_mlp": 0.0104485, + "balance_loss_clip": 1.03094673, + "balance_loss_mlp": 1.02244508, + "epoch": 0.13792002785676977, + "flos": 23473364129280.0, + "grad_norm": 2.0148925154970514, + "language_loss": 0.59211314, + "learning_rate": 3.8790779045370275e-06, + "loss": 0.6135143, + "num_input_tokens_seen": 135564205, + "router_z_loss_clip": 0.64404297, + "router_z_loss_mlp": 0.22375488, + "step": 4753, + "time_per_iteration": 2.436016082763672 + }, + { + "auxiliary_loss_clip": 0.01012191, + "auxiliary_loss_mlp": 0.01007598, + "balance_loss_clip": 1.00334072, + "balance_loss_mlp": 1.00626242, + "epoch": 0.13794904532528582, + "flos": 68282429529600.0, + "grad_norm": 1.0837295749755773, + "language_loss": 0.4709723, + "learning_rate": 3.879013530232599e-06, + "loss": 0.4911702, + "num_input_tokens_seen": 135628705, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.0133667, + "step": 4754, + "time_per_iteration": 3.0784354209899902 + }, + { + "auxiliary_loss_clip": 0.01088458, + "auxiliary_loss_mlp": 0.01039335, + "balance_loss_clip": 1.02857351, + "balance_loss_mlp": 1.01915276, + "epoch": 0.13797806279380187, + "flos": 12231924900480.0, + "grad_norm": 2.8612834025348213, + "language_loss": 0.76132393, + "learning_rate": 3.878949139331961e-06, + "loss": 0.78260189, + "num_input_tokens_seen": 135640790, + "router_z_loss_clip": 0.59887695, + "router_z_loss_mlp": 0.20202637, + "step": 4755, + "time_per_iteration": 2.4387197494506836 + }, + { + "auxiliary_loss_clip": 0.01087122, + "auxiliary_loss_mlp": 0.01036228, + "balance_loss_clip": 1.02859271, + "balance_loss_mlp": 1.01688004, + "epoch": 0.13800708026231792, + "flos": 16683455715840.0, + "grad_norm": 2.7320066563158414, + "language_loss": 0.8059386, + "learning_rate": 3.878884731835686e-06, + "loss": 0.82717204, + "num_input_tokens_seen": 135653715, + "router_z_loss_clip": 0.58544922, + "router_z_loss_mlp": 0.19348145, + "step": 4756, + "time_per_iteration": 2.3687241077423096 + }, + { + "auxiliary_loss_clip": 0.01091202, + "auxiliary_loss_mlp": 0.01039855, + "balance_loss_clip": 1.03173745, + "balance_loss_mlp": 1.01962519, + "epoch": 0.13803609773083397, + "flos": 23504821130880.0, + "grad_norm": 3.3910900039406453, + "language_loss": 0.95476097, + "learning_rate": 3.87882030774434e-06, + "loss": 0.97607148, + "num_input_tokens_seen": 135666495, + "router_z_loss_clip": 0.59570312, + "router_z_loss_mlp": 0.20239258, + "step": 4757, + "time_per_iteration": 2.4932966232299805 + }, + { + "auxiliary_loss_clip": 0.01091867, + "auxiliary_loss_mlp": 0.01034088, + "balance_loss_clip": 1.03272676, + "balance_loss_mlp": 1.01452589, + "epoch": 0.13806511519935, + "flos": 25156701198720.0, + "grad_norm": 1.7494179479716208, + "language_loss": 0.76652706, + "learning_rate": 3.878755867058492e-06, + "loss": 0.7877866, + "num_input_tokens_seen": 135683785, + "router_z_loss_clip": 0.59130859, + "router_z_loss_mlp": 0.19580078, + "step": 4758, + "time_per_iteration": 2.622619152069092 + }, + { + "auxiliary_loss_clip": 0.01095125, + "auxiliary_loss_mlp": 0.01040808, + "balance_loss_clip": 1.03081882, + "balance_loss_mlp": 1.02199674, + "epoch": 0.13809413266786605, + "flos": 24126624178560.0, + "grad_norm": 1.9806907287473765, + "language_loss": 0.86116588, + "learning_rate": 3.878691409778712e-06, + "loss": 0.88252527, + "num_input_tokens_seen": 135699135, + "router_z_loss_clip": 0.64355469, + "router_z_loss_mlp": 0.18823242, + "step": 4759, + "time_per_iteration": 2.4223954677581787 + }, + { + "auxiliary_loss_clip": 0.01092656, + "auxiliary_loss_mlp": 0.01039387, + "balance_loss_clip": 1.03242242, + "balance_loss_mlp": 1.0210886, + "epoch": 0.1381231501363821, + "flos": 29635220361600.0, + "grad_norm": 1.9406466500471524, + "language_loss": 0.7494576, + "learning_rate": 3.87862693590557e-06, + "loss": 0.77077806, + "num_input_tokens_seen": 135716815, + "router_z_loss_clip": 0.60205078, + "router_z_loss_mlp": 0.1829834, + "step": 4760, + "time_per_iteration": 2.4541993141174316 + }, + { + "auxiliary_loss_clip": 0.01094091, + "auxiliary_loss_mlp": 0.01044048, + "balance_loss_clip": 1.03245378, + "balance_loss_mlp": 1.02307892, + "epoch": 0.13815216760489815, + "flos": 38608422647040.0, + "grad_norm": 2.279596622017855, + "language_loss": 0.93847942, + "learning_rate": 3.878562445439634e-06, + "loss": 0.9598608, + "num_input_tokens_seen": 135735765, + "router_z_loss_clip": 0.61621094, + "router_z_loss_mlp": 0.20947266, + "step": 4761, + "time_per_iteration": 2.5935769081115723 + }, + { + "auxiliary_loss_clip": 0.0109067, + "auxiliary_loss_mlp": 0.01034607, + "balance_loss_clip": 1.03051639, + "balance_loss_mlp": 1.01662982, + "epoch": 0.1381811850734142, + "flos": 24350207725440.0, + "grad_norm": 2.1203753967802146, + "language_loss": 0.88894898, + "learning_rate": 3.878497938381475e-06, + "loss": 0.91020179, + "num_input_tokens_seen": 135754305, + "router_z_loss_clip": 0.60205078, + "router_z_loss_mlp": 0.17980957, + "step": 4762, + "time_per_iteration": 2.427415370941162 + }, + { + "auxiliary_loss_clip": 0.0101377, + "auxiliary_loss_mlp": 0.01000799, + "balance_loss_clip": 1.00417316, + "balance_loss_mlp": 0.9995892, + "epoch": 0.13821020254193025, + "flos": 64807000905600.0, + "grad_norm": 0.6860651336406179, + "language_loss": 0.53630251, + "learning_rate": 3.8784334147316614e-06, + "loss": 0.55644816, + "num_input_tokens_seen": 135815535, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01208496, + "step": 4763, + "time_per_iteration": 2.9969851970672607 + }, + { + "auxiliary_loss_clip": 0.01082709, + "auxiliary_loss_mlp": 0.01039464, + "balance_loss_clip": 1.02871394, + "balance_loss_mlp": 1.02188647, + "epoch": 0.13823922001044628, + "flos": 11647409051520.0, + "grad_norm": 2.4868065334587337, + "language_loss": 0.7561202, + "learning_rate": 3.8783688744907645e-06, + "loss": 0.77734196, + "num_input_tokens_seen": 135826500, + "router_z_loss_clip": 0.5402832, + "router_z_loss_mlp": 0.17590332, + "step": 4764, + "time_per_iteration": 2.309192657470703 + }, + { + "auxiliary_loss_clip": 0.01097893, + "auxiliary_loss_mlp": 0.0105535, + "balance_loss_clip": 1.03205132, + "balance_loss_mlp": 1.03184223, + "epoch": 0.13826823747896233, + "flos": 20586745451520.0, + "grad_norm": 2.46542957928518, + "language_loss": 0.94624639, + "learning_rate": 3.8783043176593526e-06, + "loss": 0.96777886, + "num_input_tokens_seen": 135840870, + "router_z_loss_clip": 0.65795898, + "router_z_loss_mlp": 0.23498535, + "step": 4765, + "time_per_iteration": 2.4109442234039307 + }, + { + "auxiliary_loss_clip": 0.01096532, + "auxiliary_loss_mlp": 0.01058226, + "balance_loss_clip": 1.03226924, + "balance_loss_mlp": 1.03612494, + "epoch": 0.13829725494747838, + "flos": 26060184028800.0, + "grad_norm": 2.2192521396712914, + "language_loss": 0.6463145, + "learning_rate": 3.878239744237997e-06, + "loss": 0.66786212, + "num_input_tokens_seen": 135856830, + "router_z_loss_clip": 0.64306641, + "router_z_loss_mlp": 0.22106934, + "step": 4766, + "time_per_iteration": 2.4218432903289795 + }, + { + "auxiliary_loss_clip": 0.01098371, + "auxiliary_loss_mlp": 0.01045063, + "balance_loss_clip": 1.03316343, + "balance_loss_mlp": 1.02297354, + "epoch": 0.13832627241599443, + "flos": 20004010081920.0, + "grad_norm": 2.564129727865489, + "language_loss": 0.76595455, + "learning_rate": 3.878175154227269e-06, + "loss": 0.78738892, + "num_input_tokens_seen": 135869795, + "router_z_loss_clip": 0.65234375, + "router_z_loss_mlp": 0.22064209, + "step": 4767, + "time_per_iteration": 2.3848021030426025 + }, + { + "auxiliary_loss_clip": 0.01094216, + "auxiliary_loss_mlp": 0.01046629, + "balance_loss_clip": 1.02864885, + "balance_loss_mlp": 1.0244441, + "epoch": 0.13835528988451049, + "flos": 13982574804480.0, + "grad_norm": 3.347220410148727, + "language_loss": 0.86168969, + "learning_rate": 3.878110547627737e-06, + "loss": 0.88309813, + "num_input_tokens_seen": 135880500, + "router_z_loss_clip": 0.65576172, + "router_z_loss_mlp": 0.22167969, + "step": 4768, + "time_per_iteration": 2.3491034507751465 + }, + { + "auxiliary_loss_clip": 0.01013252, + "auxiliary_loss_mlp": 0.01004122, + "balance_loss_clip": 1.00376797, + "balance_loss_mlp": 1.00280428, + "epoch": 0.1383843073530265, + "flos": 69110011330560.0, + "grad_norm": 0.6364757306561241, + "language_loss": 0.45895493, + "learning_rate": 3.878045924439974e-06, + "loss": 0.47912872, + "num_input_tokens_seen": 135937085, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01318359, + "step": 4769, + "time_per_iteration": 2.9475038051605225 + }, + { + "auxiliary_loss_clip": 0.01089884, + "auxiliary_loss_mlp": 0.01033, + "balance_loss_clip": 1.02957702, + "balance_loss_mlp": 1.01448703, + "epoch": 0.13841332482154256, + "flos": 36676992389760.0, + "grad_norm": 1.7906145688686215, + "language_loss": 0.72327513, + "learning_rate": 3.877981284664548e-06, + "loss": 0.74450397, + "num_input_tokens_seen": 135955760, + "router_z_loss_clip": 0.60302734, + "router_z_loss_mlp": 0.18505859, + "step": 4770, + "time_per_iteration": 2.5276200771331787 + }, + { + "auxiliary_loss_clip": 0.01087372, + "auxiliary_loss_mlp": 0.0103842, + "balance_loss_clip": 1.02896404, + "balance_loss_mlp": 1.0196743, + "epoch": 0.1384423422900586, + "flos": 41462990830080.0, + "grad_norm": 2.7290680758975925, + "language_loss": 0.76518434, + "learning_rate": 3.877916628302031e-06, + "loss": 0.78644222, + "num_input_tokens_seen": 135974400, + "router_z_loss_clip": 0.58447266, + "router_z_loss_mlp": 0.18731689, + "step": 4771, + "time_per_iteration": 2.5903351306915283 + }, + { + "auxiliary_loss_clip": 0.01107891, + "auxiliary_loss_mlp": 0.01050914, + "balance_loss_clip": 1.03495193, + "balance_loss_mlp": 1.02675056, + "epoch": 0.13847135975857466, + "flos": 32225321928960.0, + "grad_norm": 2.9652404910468015, + "language_loss": 0.85581475, + "learning_rate": 3.877851955352996e-06, + "loss": 0.87740278, + "num_input_tokens_seen": 135989515, + "router_z_loss_clip": 0.72900391, + "router_z_loss_mlp": 0.24169922, + "step": 4772, + "time_per_iteration": 2.5490455627441406 + }, + { + "auxiliary_loss_clip": 0.01014273, + "auxiliary_loss_mlp": 0.01002016, + "balance_loss_clip": 1.00500655, + "balance_loss_mlp": 1.00086617, + "epoch": 0.13850037722709072, + "flos": 74767232638080.0, + "grad_norm": 0.7025018614975794, + "language_loss": 0.50775015, + "learning_rate": 3.877787265818011e-06, + "loss": 0.52791309, + "num_input_tokens_seen": 136055365, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01147461, + "step": 4773, + "time_per_iteration": 3.0871481895446777 + }, + { + "auxiliary_loss_clip": 0.01094195, + "auxiliary_loss_mlp": 0.01041636, + "balance_loss_clip": 1.03159094, + "balance_loss_mlp": 1.01940334, + "epoch": 0.13852939469560677, + "flos": 28248993895680.0, + "grad_norm": 3.5750374327626733, + "language_loss": 0.8980298, + "learning_rate": 3.8777225596976506e-06, + "loss": 0.91938806, + "num_input_tokens_seen": 136075035, + "router_z_loss_clip": 0.62597656, + "router_z_loss_mlp": 0.22216797, + "step": 4774, + "time_per_iteration": 2.4515058994293213 + }, + { + "auxiliary_loss_clip": 0.01094944, + "auxiliary_loss_mlp": 0.0103789, + "balance_loss_clip": 1.03235722, + "balance_loss_mlp": 1.01878667, + "epoch": 0.1385584121641228, + "flos": 74729109432960.0, + "grad_norm": 2.2318438620667327, + "language_loss": 0.65902531, + "learning_rate": 3.877657836992484e-06, + "loss": 0.68035364, + "num_input_tokens_seen": 136096115, + "router_z_loss_clip": 0.62573242, + "router_z_loss_mlp": 0.190979, + "step": 4775, + "time_per_iteration": 2.7980058193206787 + }, + { + "auxiliary_loss_clip": 0.0101374, + "auxiliary_loss_mlp": 0.01000781, + "balance_loss_clip": 1.00468206, + "balance_loss_mlp": 0.99967229, + "epoch": 0.13858742963263884, + "flos": 72329177508480.0, + "grad_norm": 0.6579794836782572, + "language_loss": 0.48716849, + "learning_rate": 3.877593097703084e-06, + "loss": 0.50731373, + "num_input_tokens_seen": 136160945, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.0111084, + "step": 4776, + "time_per_iteration": 3.098165512084961 + }, + { + "auxiliary_loss_clip": 0.01089426, + "auxiliary_loss_mlp": 0.01034854, + "balance_loss_clip": 1.03265941, + "balance_loss_mlp": 1.01712203, + "epoch": 0.1386164471011549, + "flos": 16939473782400.0, + "grad_norm": 2.6877941008780555, + "language_loss": 0.66500801, + "learning_rate": 3.877528341830021e-06, + "loss": 0.68625081, + "num_input_tokens_seen": 136176325, + "router_z_loss_clip": 0.56738281, + "router_z_loss_mlp": 0.17724609, + "step": 4777, + "time_per_iteration": 2.3653533458709717 + }, + { + "auxiliary_loss_clip": 0.01087598, + "auxiliary_loss_mlp": 0.01036218, + "balance_loss_clip": 1.03175604, + "balance_loss_mlp": 1.01928985, + "epoch": 0.13864546456967095, + "flos": 11430179372160.0, + "grad_norm": 2.320116505261485, + "language_loss": 0.85809493, + "learning_rate": 3.8774635693738685e-06, + "loss": 0.87933314, + "num_input_tokens_seen": 136188395, + "router_z_loss_clip": 0.55786133, + "router_z_loss_mlp": 0.16931152, + "step": 4778, + "time_per_iteration": 2.356506824493408 + }, + { + "auxiliary_loss_clip": 0.01013035, + "auxiliary_loss_mlp": 0.01001216, + "balance_loss_clip": 1.00433135, + "balance_loss_mlp": 0.99992251, + "epoch": 0.138674482038187, + "flos": 72290528766720.0, + "grad_norm": 1.180014112363289, + "language_loss": 0.4809393, + "learning_rate": 3.877398780335199e-06, + "loss": 0.50108182, + "num_input_tokens_seen": 136254080, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.01293945, + "step": 4779, + "time_per_iteration": 3.0945799350738525 + }, + { + "auxiliary_loss_clip": 0.01087645, + "auxiliary_loss_mlp": 0.01035932, + "balance_loss_clip": 1.02877736, + "balance_loss_mlp": 1.01687014, + "epoch": 0.13870349950670305, + "flos": 31021935586560.0, + "grad_norm": 2.3086685603905286, + "language_loss": 0.83769476, + "learning_rate": 3.877333974714582e-06, + "loss": 0.85893059, + "num_input_tokens_seen": 136271430, + "router_z_loss_clip": 0.58886719, + "router_z_loss_mlp": 0.19067383, + "step": 4780, + "time_per_iteration": 2.492403268814087 + }, + { + "auxiliary_loss_clip": 0.01095723, + "auxiliary_loss_mlp": 0.0104803, + "balance_loss_clip": 1.03293014, + "balance_loss_mlp": 1.02785969, + "epoch": 0.13873251697521907, + "flos": 40139957658240.0, + "grad_norm": 2.311485020230874, + "language_loss": 0.69953334, + "learning_rate": 3.877269152512593e-06, + "loss": 0.72097081, + "num_input_tokens_seen": 136287685, + "router_z_loss_clip": 0.62719727, + "router_z_loss_mlp": 0.20153809, + "step": 4781, + "time_per_iteration": 2.5223538875579834 + }, + { + "auxiliary_loss_clip": 0.01013066, + "auxiliary_loss_mlp": 0.01003188, + "balance_loss_clip": 1.00385606, + "balance_loss_mlp": 1.00186455, + "epoch": 0.13876153444373512, + "flos": 71593313448960.0, + "grad_norm": 0.6527910531599662, + "language_loss": 0.47850114, + "learning_rate": 3.877204313729802e-06, + "loss": 0.49866369, + "num_input_tokens_seen": 136355700, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01324463, + "step": 4782, + "time_per_iteration": 3.176799774169922 + }, + { + "auxiliary_loss_clip": 0.01011912, + "auxiliary_loss_mlp": 0.01001287, + "balance_loss_clip": 1.00318396, + "balance_loss_mlp": 1.00024354, + "epoch": 0.13879055191225118, + "flos": 61559799039360.0, + "grad_norm": 0.7195496186569439, + "language_loss": 0.5001514, + "learning_rate": 3.877139458366783e-06, + "loss": 0.5202834, + "num_input_tokens_seen": 136416835, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.01043701, + "step": 4783, + "time_per_iteration": 2.9502203464508057 + }, + { + "auxiliary_loss_clip": 0.0101147, + "auxiliary_loss_mlp": 0.01003798, + "balance_loss_clip": 1.00247216, + "balance_loss_mlp": 1.00253987, + "epoch": 0.13881956938076723, + "flos": 65876250337920.0, + "grad_norm": 0.6696425346211523, + "language_loss": 0.46721217, + "learning_rate": 3.87707458642411e-06, + "loss": 0.48736486, + "num_input_tokens_seen": 136483870, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01257324, + "step": 4784, + "time_per_iteration": 3.2250003814697266 + }, + { + "auxiliary_loss_clip": 0.01011211, + "auxiliary_loss_mlp": 0.01001329, + "balance_loss_clip": 1.00240982, + "balance_loss_mlp": 1.0002259, + "epoch": 0.13884858684928328, + "flos": 73596525194880.0, + "grad_norm": 0.7240012302313981, + "language_loss": 0.51330125, + "learning_rate": 3.877009697902354e-06, + "loss": 0.53342664, + "num_input_tokens_seen": 136541115, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01104736, + "step": 4785, + "time_per_iteration": 3.032153844833374 + }, + { + "auxiliary_loss_clip": 0.01010796, + "auxiliary_loss_mlp": 0.01000153, + "balance_loss_clip": 1.00199151, + "balance_loss_mlp": 0.99896079, + "epoch": 0.1388776043177993, + "flos": 71476808641920.0, + "grad_norm": 0.7431953232362091, + "language_loss": 0.5215019, + "learning_rate": 3.8769447928020885e-06, + "loss": 0.54161143, + "num_input_tokens_seen": 136596710, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01190186, + "step": 4786, + "time_per_iteration": 2.8954761028289795 + }, + { + "auxiliary_loss_clip": 0.01099291, + "auxiliary_loss_mlp": 0.01045342, + "balance_loss_clip": 1.03305256, + "balance_loss_mlp": 1.02505207, + "epoch": 0.13890662178631535, + "flos": 40288617694080.0, + "grad_norm": 1.9955531796582604, + "language_loss": 0.8239429, + "learning_rate": 3.8768798711238875e-06, + "loss": 0.84538925, + "num_input_tokens_seen": 136612115, + "router_z_loss_clip": 0.66259766, + "router_z_loss_mlp": 0.20300293, + "step": 4787, + "time_per_iteration": 2.612082004547119 + }, + { + "auxiliary_loss_clip": 0.01078882, + "auxiliary_loss_mlp": 0.01039872, + "balance_loss_clip": 1.02654362, + "balance_loss_mlp": 1.02348053, + "epoch": 0.1389356392548314, + "flos": 42515482809600.0, + "grad_norm": 2.0852124861319896, + "language_loss": 0.68777937, + "learning_rate": 3.876814932868323e-06, + "loss": 0.70896691, + "num_input_tokens_seen": 136628895, + "router_z_loss_clip": 0.52368164, + "router_z_loss_mlp": 0.16381836, + "step": 4788, + "time_per_iteration": 2.565906524658203 + }, + { + "auxiliary_loss_clip": 0.01093415, + "auxiliary_loss_mlp": 0.01044175, + "balance_loss_clip": 1.03045416, + "balance_loss_mlp": 1.02350414, + "epoch": 0.13896465672334746, + "flos": 20513008926720.0, + "grad_norm": 2.4650528496550828, + "language_loss": 0.79193646, + "learning_rate": 3.8767499780359704e-06, + "loss": 0.81331235, + "num_input_tokens_seen": 136642490, + "router_z_loss_clip": 0.63037109, + "router_z_loss_mlp": 0.20678711, + "step": 4789, + "time_per_iteration": 2.400541305541992 + }, + { + "auxiliary_loss_clip": 0.01095872, + "auxiliary_loss_mlp": 0.01040365, + "balance_loss_clip": 1.0308733, + "balance_loss_mlp": 1.01987314, + "epoch": 0.1389936741918635, + "flos": 31465227519360.0, + "grad_norm": 2.1640173163772993, + "language_loss": 0.80913109, + "learning_rate": 3.876685006627403e-06, + "loss": 0.83049351, + "num_input_tokens_seen": 136658795, + "router_z_loss_clip": 0.64941406, + "router_z_loss_mlp": 0.20483398, + "step": 4790, + "time_per_iteration": 2.45334529876709 + }, + { + "auxiliary_loss_clip": 0.01012469, + "auxiliary_loss_mlp": 0.01000437, + "balance_loss_clip": 1.00353861, + "balance_loss_mlp": 0.99923301, + "epoch": 0.13902269166037956, + "flos": 74771247444480.0, + "grad_norm": 0.5881076831538542, + "language_loss": 0.44403082, + "learning_rate": 3.8766200186431935e-06, + "loss": 0.46415985, + "num_input_tokens_seen": 136727835, + "router_z_loss_clip": 0.08935547, + "router_z_loss_mlp": 0.01202393, + "step": 4791, + "time_per_iteration": 3.19653058052063 + }, + { + "auxiliary_loss_clip": 0.01091747, + "auxiliary_loss_mlp": 0.0104227, + "balance_loss_clip": 1.03025138, + "balance_loss_mlp": 1.02169466, + "epoch": 0.13905170912889558, + "flos": 28359144835200.0, + "grad_norm": 2.4828172384109335, + "language_loss": 0.83171618, + "learning_rate": 3.876555014083916e-06, + "loss": 0.85305631, + "num_input_tokens_seen": 136743840, + "router_z_loss_clip": 0.61572266, + "router_z_loss_mlp": 0.20581055, + "step": 4792, + "time_per_iteration": 2.444492816925049 + }, + { + "auxiliary_loss_clip": 0.01090086, + "auxiliary_loss_mlp": 0.01043384, + "balance_loss_clip": 1.02892172, + "balance_loss_mlp": 1.02373886, + "epoch": 0.13908072659741164, + "flos": 26648295747840.0, + "grad_norm": 1.9638175765351897, + "language_loss": 0.85250181, + "learning_rate": 3.876489992950147e-06, + "loss": 0.87383646, + "num_input_tokens_seen": 136764625, + "router_z_loss_clip": 0.61132812, + "router_z_loss_mlp": 0.19641113, + "step": 4793, + "time_per_iteration": 2.481579303741455 + }, + { + "auxiliary_loss_clip": 0.01089651, + "auxiliary_loss_mlp": 0.01035775, + "balance_loss_clip": 1.03000641, + "balance_loss_mlp": 1.01668334, + "epoch": 0.1391097440659277, + "flos": 21135056353920.0, + "grad_norm": 2.518896673597655, + "language_loss": 0.91407335, + "learning_rate": 3.876424955242458e-06, + "loss": 0.93532765, + "num_input_tokens_seen": 136779125, + "router_z_loss_clip": 0.59667969, + "router_z_loss_mlp": 0.19104004, + "step": 4794, + "time_per_iteration": 4.668737411499023 + }, + { + "auxiliary_loss_clip": 0.01012688, + "auxiliary_loss_mlp": 0.01004342, + "balance_loss_clip": 1.00367367, + "balance_loss_mlp": 1.00334668, + "epoch": 0.13913876153444374, + "flos": 72798445359360.0, + "grad_norm": 0.6687424675259639, + "language_loss": 0.4367401, + "learning_rate": 3.876359900961424e-06, + "loss": 0.45691037, + "num_input_tokens_seen": 136840325, + "router_z_loss_clip": 0.09033203, + "router_z_loss_mlp": 0.00994873, + "step": 4795, + "time_per_iteration": 3.0685858726501465 + }, + { + "auxiliary_loss_clip": 0.01098196, + "auxiliary_loss_mlp": 0.01044209, + "balance_loss_clip": 1.03128695, + "balance_loss_mlp": 1.02278686, + "epoch": 0.1391677790029598, + "flos": 33214166766720.0, + "grad_norm": 2.2370747345144997, + "language_loss": 0.78957003, + "learning_rate": 3.876294830107621e-06, + "loss": 0.81099415, + "num_input_tokens_seen": 136861125, + "router_z_loss_clip": 0.66894531, + "router_z_loss_mlp": 0.2142334, + "step": 4796, + "time_per_iteration": 4.997823476791382 + }, + { + "auxiliary_loss_clip": 0.01089162, + "auxiliary_loss_mlp": 0.010399, + "balance_loss_clip": 1.02903116, + "balance_loss_mlp": 1.02006316, + "epoch": 0.13919679647147584, + "flos": 18618167640960.0, + "grad_norm": 2.069006411379236, + "language_loss": 0.88467044, + "learning_rate": 3.876229742681622e-06, + "loss": 0.9059611, + "num_input_tokens_seen": 136878275, + "router_z_loss_clip": 0.60205078, + "router_z_loss_mlp": 0.19836426, + "step": 4797, + "time_per_iteration": 2.4197349548339844 + }, + { + "auxiliary_loss_clip": 0.01013636, + "auxiliary_loss_mlp": 0.01006027, + "balance_loss_clip": 1.00473428, + "balance_loss_mlp": 1.00473952, + "epoch": 0.13922581393999187, + "flos": 51748366988160.0, + "grad_norm": 0.6767206377864046, + "language_loss": 0.46593004, + "learning_rate": 3.876164638684004e-06, + "loss": 0.48612666, + "num_input_tokens_seen": 136937245, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.01287842, + "step": 4798, + "time_per_iteration": 2.9467828273773193 + }, + { + "auxiliary_loss_clip": 0.01090256, + "auxiliary_loss_mlp": 0.01036809, + "balance_loss_clip": 1.02831793, + "balance_loss_mlp": 1.01651978, + "epoch": 0.13925483140850792, + "flos": 20300457369600.0, + "grad_norm": 5.299500369943255, + "language_loss": 0.75951731, + "learning_rate": 3.87609951811534e-06, + "loss": 0.780788, + "num_input_tokens_seen": 136953605, + "router_z_loss_clip": 0.62011719, + "router_z_loss_mlp": 0.20281982, + "step": 4799, + "time_per_iteration": 2.4003710746765137 + }, + { + "auxiliary_loss_clip": 0.01091299, + "auxiliary_loss_mlp": 0.01039738, + "balance_loss_clip": 1.0288651, + "balance_loss_mlp": 1.02128482, + "epoch": 0.13928384887702397, + "flos": 38137304494080.0, + "grad_norm": 1.941754918885656, + "language_loss": 0.91572618, + "learning_rate": 3.876034380976205e-06, + "loss": 0.93703657, + "num_input_tokens_seen": 136974105, + "router_z_loss_clip": 0.62402344, + "router_z_loss_mlp": 0.18469238, + "step": 4800, + "time_per_iteration": 2.547868251800537 + }, + { + "auxiliary_loss_clip": 0.01075475, + "auxiliary_loss_mlp": 0.01030527, + "balance_loss_clip": 1.02791071, + "balance_loss_mlp": 1.01537538, + "epoch": 0.13931286634554002, + "flos": 37187736802560.0, + "grad_norm": 1.8743175619970622, + "language_loss": 0.636612, + "learning_rate": 3.875969227267176e-06, + "loss": 0.65767193, + "num_input_tokens_seen": 136992400, + "router_z_loss_clip": 0.47607422, + "router_z_loss_mlp": 0.15161133, + "step": 4801, + "time_per_iteration": 2.530055284500122 + }, + { + "auxiliary_loss_clip": 0.01094488, + "auxiliary_loss_mlp": 0.01041695, + "balance_loss_clip": 1.03147483, + "balance_loss_mlp": 1.02135742, + "epoch": 0.13934188381405607, + "flos": 15699044620800.0, + "grad_norm": 2.9221501430067915, + "language_loss": 0.99721813, + "learning_rate": 3.875904056988828e-06, + "loss": 1.01857996, + "num_input_tokens_seen": 137004530, + "router_z_loss_clip": 0.62939453, + "router_z_loss_mlp": 0.20349121, + "step": 4802, + "time_per_iteration": 4.818722486495972 + }, + { + "auxiliary_loss_clip": 0.01017475, + "auxiliary_loss_mlp": 0.0100476, + "balance_loss_clip": 1.00757527, + "balance_loss_mlp": 1.00347209, + "epoch": 0.1393709012825721, + "flos": 62008886257920.0, + "grad_norm": 0.6545017374199233, + "language_loss": 0.4519383, + "learning_rate": 3.875838870141735e-06, + "loss": 0.47216064, + "num_input_tokens_seen": 137072705, + "router_z_loss_clip": 0.09912109, + "router_z_loss_mlp": 0.01287842, + "step": 4803, + "time_per_iteration": 5.59528660774231 + }, + { + "auxiliary_loss_clip": 0.01103327, + "auxiliary_loss_mlp": 0.01048278, + "balance_loss_clip": 1.03407073, + "balance_loss_mlp": 1.02484155, + "epoch": 0.13939991875108815, + "flos": 16610766531840.0, + "grad_norm": 4.001836606504472, + "language_loss": 0.74662721, + "learning_rate": 3.875773666726475e-06, + "loss": 0.76814324, + "num_input_tokens_seen": 137086220, + "router_z_loss_clip": 0.69189453, + "router_z_loss_mlp": 0.23425293, + "step": 4804, + "time_per_iteration": 2.3400473594665527 + }, + { + "auxiliary_loss_clip": 0.01017303, + "auxiliary_loss_mlp": 0.0100159, + "balance_loss_clip": 1.00743294, + "balance_loss_mlp": 1.00057626, + "epoch": 0.1394289362196042, + "flos": 66997661074560.0, + "grad_norm": 0.6782484395842806, + "language_loss": 0.47492981, + "learning_rate": 3.875708446743623e-06, + "loss": 0.49511874, + "num_input_tokens_seen": 137148555, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01013184, + "step": 4805, + "time_per_iteration": 3.009934186935425 + }, + { + "auxiliary_loss_clip": 0.01087192, + "auxiliary_loss_mlp": 0.01037318, + "balance_loss_clip": 1.03024125, + "balance_loss_mlp": 1.02019382, + "epoch": 0.13945795368812025, + "flos": 30439270039680.0, + "grad_norm": 1.9938749187612819, + "language_loss": 0.91010714, + "learning_rate": 3.875643210193755e-06, + "loss": 0.93135226, + "num_input_tokens_seen": 137166505, + "router_z_loss_clip": 0.56982422, + "router_z_loss_mlp": 0.17108154, + "step": 4806, + "time_per_iteration": 2.507237195968628 + }, + { + "auxiliary_loss_clip": 0.01016064, + "auxiliary_loss_mlp": 0.01002442, + "balance_loss_clip": 1.00650871, + "balance_loss_mlp": 1.00138068, + "epoch": 0.1394869711566363, + "flos": 62161072341120.0, + "grad_norm": 0.642246615470657, + "language_loss": 0.49046683, + "learning_rate": 3.875577957077447e-06, + "loss": 0.51065195, + "num_input_tokens_seen": 137227595, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01062012, + "step": 4807, + "time_per_iteration": 3.096362829208374 + }, + { + "auxiliary_loss_clip": 0.01094799, + "auxiliary_loss_mlp": 0.01045966, + "balance_loss_clip": 1.02991283, + "balance_loss_mlp": 1.02614117, + "epoch": 0.13951598862515235, + "flos": 27710248705920.0, + "grad_norm": 2.3709715237384716, + "language_loss": 0.95272338, + "learning_rate": 3.875512687395275e-06, + "loss": 0.97413111, + "num_input_tokens_seen": 137248075, + "router_z_loss_clip": 0.6484375, + "router_z_loss_mlp": 0.19836426, + "step": 4808, + "time_per_iteration": 2.5108354091644287 + }, + { + "auxiliary_loss_clip": 0.01015111, + "auxiliary_loss_mlp": 0.01003853, + "balance_loss_clip": 1.00548768, + "balance_loss_mlp": 1.00270236, + "epoch": 0.13954500609366838, + "flos": 71860119125760.0, + "grad_norm": 0.7159085339622169, + "language_loss": 0.54060531, + "learning_rate": 3.875447401147817e-06, + "loss": 0.56079495, + "num_input_tokens_seen": 137313275, + "router_z_loss_clip": 0.09619141, + "router_z_loss_mlp": 0.01147461, + "step": 4809, + "time_per_iteration": 3.0790822505950928 + }, + { + "auxiliary_loss_clip": 0.01013047, + "auxiliary_loss_mlp": 0.01009273, + "balance_loss_clip": 1.00358343, + "balance_loss_mlp": 1.00821781, + "epoch": 0.13957402356218443, + "flos": 74775785921280.0, + "grad_norm": 0.5883561449850729, + "language_loss": 0.47384793, + "learning_rate": 3.875382098335648e-06, + "loss": 0.49407113, + "num_input_tokens_seen": 137389890, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01055908, + "step": 4810, + "time_per_iteration": 3.221883773803711 + }, + { + "auxiliary_loss_clip": 0.0109337, + "auxiliary_loss_mlp": 0.01044753, + "balance_loss_clip": 1.02857161, + "balance_loss_mlp": 1.02285433, + "epoch": 0.13960304103070048, + "flos": 25694678338560.0, + "grad_norm": 2.791658049504191, + "language_loss": 0.86473227, + "learning_rate": 3.875316778959346e-06, + "loss": 0.88611346, + "num_input_tokens_seen": 137404365, + "router_z_loss_clip": 0.64794922, + "router_z_loss_mlp": 0.21923828, + "step": 4811, + "time_per_iteration": 2.445157289505005 + }, + { + "auxiliary_loss_clip": 0.01089491, + "auxiliary_loss_mlp": 0.01039237, + "balance_loss_clip": 1.02956021, + "balance_loss_mlp": 1.01937675, + "epoch": 0.13963205849921653, + "flos": 27008844024960.0, + "grad_norm": 2.0061298321560717, + "language_loss": 0.78718626, + "learning_rate": 3.875251443019486e-06, + "loss": 0.80847359, + "num_input_tokens_seen": 137423170, + "router_z_loss_clip": 0.59912109, + "router_z_loss_mlp": 0.19848633, + "step": 4812, + "time_per_iteration": 2.6314234733581543 + }, + { + "auxiliary_loss_clip": 0.01088948, + "auxiliary_loss_mlp": 0.01038514, + "balance_loss_clip": 1.03050256, + "balance_loss_mlp": 1.02082384, + "epoch": 0.13966107596773258, + "flos": 15333504019200.0, + "grad_norm": 4.966373035114528, + "language_loss": 0.74928606, + "learning_rate": 3.875186090516648e-06, + "loss": 0.77056068, + "num_input_tokens_seen": 137435045, + "router_z_loss_clip": 0.58447266, + "router_z_loss_mlp": 0.17675781, + "step": 4813, + "time_per_iteration": 2.4063756465911865 + }, + { + "auxiliary_loss_clip": 0.01099365, + "auxiliary_loss_mlp": 0.01041232, + "balance_loss_clip": 1.03162324, + "balance_loss_mlp": 1.02000105, + "epoch": 0.1396900934362486, + "flos": 15624958982400.0, + "grad_norm": 3.082874510860287, + "language_loss": 0.89055866, + "learning_rate": 3.875120721451406e-06, + "loss": 0.91196465, + "num_input_tokens_seen": 137448110, + "router_z_loss_clip": 0.67724609, + "router_z_loss_mlp": 0.2121582, + "step": 4814, + "time_per_iteration": 2.3982009887695312 + }, + { + "auxiliary_loss_clip": 0.01019888, + "auxiliary_loss_mlp": 0.01003035, + "balance_loss_clip": 1.01078594, + "balance_loss_mlp": 1.00205755, + "epoch": 0.13971911090476466, + "flos": 61381811594880.0, + "grad_norm": 0.652621085426324, + "language_loss": 0.48014575, + "learning_rate": 3.87505533582434e-06, + "loss": 0.50037497, + "num_input_tokens_seen": 137506595, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.00976562, + "step": 4815, + "time_per_iteration": 2.9639127254486084 + }, + { + "auxiliary_loss_clip": 0.01101433, + "auxiliary_loss_mlp": 0.0103901, + "balance_loss_clip": 1.0336113, + "balance_loss_mlp": 1.01895285, + "epoch": 0.1397481283732807, + "flos": 17487784684800.0, + "grad_norm": 2.9989841827773063, + "language_loss": 0.92988038, + "learning_rate": 3.874989933636027e-06, + "loss": 0.95128477, + "num_input_tokens_seen": 137520995, + "router_z_loss_clip": 0.67773438, + "router_z_loss_mlp": 0.20062256, + "step": 4816, + "time_per_iteration": 2.3983421325683594 + }, + { + "auxiliary_loss_clip": 0.0108898, + "auxiliary_loss_mlp": 0.01032285, + "balance_loss_clip": 1.03253615, + "balance_loss_mlp": 1.01542318, + "epoch": 0.13977714584179676, + "flos": 24525716463360.0, + "grad_norm": 2.803644575053425, + "language_loss": 0.87699056, + "learning_rate": 3.874924514887043e-06, + "loss": 0.89820325, + "num_input_tokens_seen": 137535430, + "router_z_loss_clip": 0.5637207, + "router_z_loss_mlp": 0.16882324, + "step": 4817, + "time_per_iteration": 2.466991662979126 + }, + { + "auxiliary_loss_clip": 0.01094223, + "auxiliary_loss_mlp": 0.01037662, + "balance_loss_clip": 1.03560805, + "balance_loss_mlp": 1.01852906, + "epoch": 0.13980616331031281, + "flos": 25693107327360.0, + "grad_norm": 2.0377429205996425, + "language_loss": 0.86200881, + "learning_rate": 3.874859079577968e-06, + "loss": 0.8833276, + "num_input_tokens_seen": 137552295, + "router_z_loss_clip": 0.58618164, + "router_z_loss_mlp": 0.19128418, + "step": 4818, + "time_per_iteration": 2.4480268955230713 + }, + { + "auxiliary_loss_clip": 0.01088857, + "auxiliary_loss_mlp": 0.01034506, + "balance_loss_clip": 1.03199899, + "balance_loss_mlp": 1.01632667, + "epoch": 0.13983518077882887, + "flos": 29453776692480.0, + "grad_norm": 2.0054284618008005, + "language_loss": 0.72654033, + "learning_rate": 3.874793627709379e-06, + "loss": 0.74777395, + "num_input_tokens_seen": 137567470, + "router_z_loss_clip": 0.5690918, + "router_z_loss_mlp": 0.18182373, + "step": 4819, + "time_per_iteration": 2.4535553455352783 + }, + { + "auxiliary_loss_clip": 0.01088702, + "auxiliary_loss_mlp": 0.01033889, + "balance_loss_clip": 1.03481746, + "balance_loss_mlp": 1.01758099, + "epoch": 0.1398641982473449, + "flos": 19127236308480.0, + "grad_norm": 1.8755721952818354, + "language_loss": 0.78461748, + "learning_rate": 3.874728159281853e-06, + "loss": 0.80584341, + "num_input_tokens_seen": 137581025, + "router_z_loss_clip": 0.5390625, + "router_z_loss_mlp": 0.1630249, + "step": 4820, + "time_per_iteration": 2.6516811847686768 + }, + { + "auxiliary_loss_clip": 0.0109022, + "auxiliary_loss_mlp": 0.0103402, + "balance_loss_clip": 1.03365088, + "balance_loss_mlp": 1.01688957, + "epoch": 0.13989321571586094, + "flos": 21718943798400.0, + "grad_norm": 2.916072565549533, + "language_loss": 0.68024212, + "learning_rate": 3.8746626742959705e-06, + "loss": 0.70148456, + "num_input_tokens_seen": 137598045, + "router_z_loss_clip": 0.56542969, + "router_z_loss_mlp": 0.17138672, + "step": 4821, + "time_per_iteration": 2.369152307510376 + }, + { + "auxiliary_loss_clip": 0.01097388, + "auxiliary_loss_mlp": 0.01041348, + "balance_loss_clip": 1.03639722, + "balance_loss_mlp": 1.02250111, + "epoch": 0.139922233184377, + "flos": 20441157615360.0, + "grad_norm": 3.418927857443234, + "language_loss": 0.8078444, + "learning_rate": 3.874597172752308e-06, + "loss": 0.82923186, + "num_input_tokens_seen": 137611355, + "router_z_loss_clip": 0.60986328, + "router_z_loss_mlp": 0.18835449, + "step": 4822, + "time_per_iteration": 2.4232983589172363 + }, + { + "auxiliary_loss_clip": 0.01024839, + "auxiliary_loss_mlp": 0.01001023, + "balance_loss_clip": 1.01483679, + "balance_loss_mlp": 0.99994463, + "epoch": 0.13995125065289304, + "flos": 50356415059200.0, + "grad_norm": 0.720976094275732, + "language_loss": 0.51776522, + "learning_rate": 3.874531654651444e-06, + "loss": 0.53802389, + "num_input_tokens_seen": 137659970, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01080322, + "step": 4823, + "time_per_iteration": 2.803466320037842 + }, + { + "auxiliary_loss_clip": 0.0109815, + "auxiliary_loss_mlp": 0.01037332, + "balance_loss_clip": 1.03658843, + "balance_loss_mlp": 1.01886606, + "epoch": 0.1399802681214091, + "flos": 29489108855040.0, + "grad_norm": 4.704383260887796, + "language_loss": 0.75147223, + "learning_rate": 3.874466119993959e-06, + "loss": 0.77282703, + "num_input_tokens_seen": 137674305, + "router_z_loss_clip": 0.61645508, + "router_z_loss_mlp": 0.18463135, + "step": 4824, + "time_per_iteration": 2.4470534324645996 + }, + { + "auxiliary_loss_clip": 0.0109459, + "auxiliary_loss_mlp": 0.0104436, + "balance_loss_clip": 1.03452086, + "balance_loss_mlp": 1.0253346, + "epoch": 0.14000928558992515, + "flos": 28505954568960.0, + "grad_norm": 1.9998736361979588, + "language_loss": 0.76504636, + "learning_rate": 3.87440056878043e-06, + "loss": 0.7864359, + "num_input_tokens_seen": 137692280, + "router_z_loss_clip": 0.60083008, + "router_z_loss_mlp": 0.19018555, + "step": 4825, + "time_per_iteration": 2.4525699615478516 + }, + { + "auxiliary_loss_clip": 0.01100942, + "auxiliary_loss_mlp": 0.01043934, + "balance_loss_clip": 1.03510773, + "balance_loss_mlp": 1.02341259, + "epoch": 0.14003830305844117, + "flos": 19602788204160.0, + "grad_norm": 1.9651116712185202, + "language_loss": 0.83034283, + "learning_rate": 3.874335001011437e-06, + "loss": 0.85179162, + "num_input_tokens_seen": 137709770, + "router_z_loss_clip": 0.65795898, + "router_z_loss_mlp": 0.20501709, + "step": 4826, + "time_per_iteration": 2.398000955581665 + }, + { + "auxiliary_loss_clip": 0.01022961, + "auxiliary_loss_mlp": 0.01000563, + "balance_loss_clip": 1.01316261, + "balance_loss_mlp": 0.99943012, + "epoch": 0.14006732052695722, + "flos": 72946058054400.0, + "grad_norm": 0.6433431743092548, + "language_loss": 0.49371886, + "learning_rate": 3.874269416687559e-06, + "loss": 0.51395416, + "num_input_tokens_seen": 137774005, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.01135254, + "step": 4827, + "time_per_iteration": 3.0520541667938232 + }, + { + "auxiliary_loss_clip": 0.01092516, + "auxiliary_loss_mlp": 0.01039384, + "balance_loss_clip": 1.03121626, + "balance_loss_mlp": 1.02270031, + "epoch": 0.14009633799547327, + "flos": 42368044671360.0, + "grad_norm": 2.08358121502858, + "language_loss": 0.75307822, + "learning_rate": 3.874203815809375e-06, + "loss": 0.77439719, + "num_input_tokens_seen": 137792685, + "router_z_loss_clip": 0.61303711, + "router_z_loss_mlp": 0.16680908, + "step": 4828, + "time_per_iteration": 2.5644683837890625 + }, + { + "auxiliary_loss_clip": 0.01091253, + "auxiliary_loss_mlp": 0.01040893, + "balance_loss_clip": 1.03024042, + "balance_loss_mlp": 1.0225352, + "epoch": 0.14012535546398933, + "flos": 34963071102720.0, + "grad_norm": 2.14729659776374, + "language_loss": 0.80026412, + "learning_rate": 3.874138198377465e-06, + "loss": 0.8215856, + "num_input_tokens_seen": 137808845, + "router_z_loss_clip": 0.61035156, + "router_z_loss_mlp": 0.18353271, + "step": 4829, + "time_per_iteration": 2.5137412548065186 + }, + { + "auxiliary_loss_clip": 0.01091107, + "auxiliary_loss_mlp": 0.01044422, + "balance_loss_clip": 1.03097939, + "balance_loss_mlp": 1.02493167, + "epoch": 0.14015437293250538, + "flos": 30405578711040.0, + "grad_norm": 1.9734805733417267, + "language_loss": 0.73067892, + "learning_rate": 3.874072564392407e-06, + "loss": 0.75203419, + "num_input_tokens_seen": 137828915, + "router_z_loss_clip": 0.60107422, + "router_z_loss_mlp": 0.19506836, + "step": 4830, + "time_per_iteration": 2.4896068572998047 + }, + { + "auxiliary_loss_clip": 0.01084641, + "auxiliary_loss_mlp": 0.01047975, + "balance_loss_clip": 1.02942801, + "balance_loss_mlp": 1.0299089, + "epoch": 0.1401833904010214, + "flos": 17779449116160.0, + "grad_norm": 2.8299453683705047, + "language_loss": 0.86177766, + "learning_rate": 3.874006913854782e-06, + "loss": 0.88310385, + "num_input_tokens_seen": 137843705, + "router_z_loss_clip": 0.55175781, + "router_z_loss_mlp": 0.18054199, + "step": 4831, + "time_per_iteration": 2.411914348602295 + }, + { + "auxiliary_loss_clip": 0.01092922, + "auxiliary_loss_mlp": 0.01040914, + "balance_loss_clip": 1.03057861, + "balance_loss_mlp": 1.02056479, + "epoch": 0.14021240786953745, + "flos": 74729982216960.0, + "grad_norm": 3.029600665814788, + "language_loss": 0.89357555, + "learning_rate": 3.87394124676517e-06, + "loss": 0.91491395, + "num_input_tokens_seen": 137868145, + "router_z_loss_clip": 0.62255859, + "router_z_loss_mlp": 0.20361328, + "step": 4832, + "time_per_iteration": 2.787290573120117 + }, + { + "auxiliary_loss_clip": 0.01015448, + "auxiliary_loss_mlp": 0.01000859, + "balance_loss_clip": 1.00627375, + "balance_loss_mlp": 0.99969101, + "epoch": 0.1402414253380535, + "flos": 65104879559040.0, + "grad_norm": 0.6350015774353475, + "language_loss": 0.47303408, + "learning_rate": 3.87387556312415e-06, + "loss": 0.4931972, + "num_input_tokens_seen": 137931400, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01165771, + "step": 4833, + "time_per_iteration": 3.0840871334075928 + }, + { + "auxiliary_loss_clip": 0.01081932, + "auxiliary_loss_mlp": 0.01037002, + "balance_loss_clip": 1.02858305, + "balance_loss_mlp": 1.02067029, + "epoch": 0.14027044280656956, + "flos": 17270589916800.0, + "grad_norm": 3.2946621742528954, + "language_loss": 0.82342041, + "learning_rate": 3.873809862932303e-06, + "loss": 0.84460974, + "num_input_tokens_seen": 137946265, + "router_z_loss_clip": 0.53393555, + "router_z_loss_mlp": 0.16308594, + "step": 4834, + "time_per_iteration": 2.44645619392395 + }, + { + "auxiliary_loss_clip": 0.01089926, + "auxiliary_loss_mlp": 0.01036644, + "balance_loss_clip": 1.03072274, + "balance_loss_mlp": 1.0185957, + "epoch": 0.1402994602750856, + "flos": 26789414929920.0, + "grad_norm": 2.1454427671604095, + "language_loss": 0.74431467, + "learning_rate": 3.873744146190209e-06, + "loss": 0.76558036, + "num_input_tokens_seen": 137961195, + "router_z_loss_clip": 0.59228516, + "router_z_loss_mlp": 0.18054199, + "step": 4835, + "time_per_iteration": 2.485445261001587 + }, + { + "auxiliary_loss_clip": 0.01099359, + "auxiliary_loss_mlp": 0.0104364, + "balance_loss_clip": 1.03456104, + "balance_loss_mlp": 1.02271831, + "epoch": 0.14032847774360166, + "flos": 25110371957760.0, + "grad_norm": 2.6266959755124675, + "language_loss": 1.05689216, + "learning_rate": 3.8736784128984494e-06, + "loss": 1.07832217, + "num_input_tokens_seen": 137977000, + "router_z_loss_clip": 0.64770508, + "router_z_loss_mlp": 0.20922852, + "step": 4836, + "time_per_iteration": 2.4145026206970215 + }, + { + "auxiliary_loss_clip": 0.01098194, + "auxiliary_loss_mlp": 0.01040804, + "balance_loss_clip": 1.03288698, + "balance_loss_mlp": 1.02026427, + "epoch": 0.14035749521211768, + "flos": 40908954464640.0, + "grad_norm": 3.2564295891491173, + "language_loss": 0.92257065, + "learning_rate": 3.873612663057603e-06, + "loss": 0.94396061, + "num_input_tokens_seen": 137994130, + "router_z_loss_clip": 0.65234375, + "router_z_loss_mlp": 0.20501709, + "step": 4837, + "time_per_iteration": 2.6004204750061035 + }, + { + "auxiliary_loss_clip": 0.0101457, + "auxiliary_loss_mlp": 0.01000851, + "balance_loss_clip": 1.00532222, + "balance_loss_mlp": 0.99988592, + "epoch": 0.14038651268063373, + "flos": 68504930824320.0, + "grad_norm": 0.6498604030595575, + "language_loss": 0.51145047, + "learning_rate": 3.8735468966682515e-06, + "loss": 0.53160471, + "num_input_tokens_seen": 138060150, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.00964355, + "step": 4838, + "time_per_iteration": 3.057817220687866 + }, + { + "auxiliary_loss_clip": 0.0109885, + "auxiliary_loss_mlp": 0.01041643, + "balance_loss_clip": 1.03231311, + "balance_loss_mlp": 1.02149701, + "epoch": 0.1404155301491498, + "flos": 9678412304640.0, + "grad_norm": 2.6419125497646374, + "language_loss": 0.91811931, + "learning_rate": 3.873481113730976e-06, + "loss": 0.93952429, + "num_input_tokens_seen": 138071850, + "router_z_loss_clip": 0.66601562, + "router_z_loss_mlp": 0.20129395, + "step": 4839, + "time_per_iteration": 2.3731026649475098 + }, + { + "auxiliary_loss_clip": 0.01092901, + "auxiliary_loss_mlp": 0.01037551, + "balance_loss_clip": 1.03346586, + "balance_loss_mlp": 1.0182631, + "epoch": 0.14044454761766584, + "flos": 48423485479680.0, + "grad_norm": 2.866650498351165, + "language_loss": 0.8633579, + "learning_rate": 3.8734153142463565e-06, + "loss": 0.88466245, + "num_input_tokens_seen": 138090460, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.19299316, + "step": 4840, + "time_per_iteration": 2.644407272338867 + }, + { + "auxiliary_loss_clip": 0.01013561, + "auxiliary_loss_mlp": 0.01000607, + "balance_loss_clip": 1.00438511, + "balance_loss_mlp": 0.99965292, + "epoch": 0.1404735650861819, + "flos": 56232471968640.0, + "grad_norm": 0.6493525000554051, + "language_loss": 0.43615732, + "learning_rate": 3.873349498214975e-06, + "loss": 0.45629901, + "num_input_tokens_seen": 138149905, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.00952148, + "step": 4841, + "time_per_iteration": 2.955688238143921 + }, + { + "auxiliary_loss_clip": 0.01089736, + "auxiliary_loss_mlp": 0.01037721, + "balance_loss_clip": 1.03418422, + "balance_loss_mlp": 1.02140117, + "epoch": 0.14050258255469794, + "flos": 31278896259840.0, + "grad_norm": 1.8542809667885172, + "language_loss": 0.63192749, + "learning_rate": 3.873283665637414e-06, + "loss": 0.65320206, + "num_input_tokens_seen": 138166930, + "router_z_loss_clip": 0.55615234, + "router_z_loss_mlp": 0.16314697, + "step": 4842, + "time_per_iteration": 2.4659502506256104 + }, + { + "auxiliary_loss_clip": 0.01089017, + "auxiliary_loss_mlp": 0.01035457, + "balance_loss_clip": 1.03191352, + "balance_loss_mlp": 1.01837397, + "epoch": 0.14053160002321397, + "flos": 17339508673920.0, + "grad_norm": 2.825110167190833, + "language_loss": 0.70005131, + "learning_rate": 3.873217816514251e-06, + "loss": 0.72129607, + "num_input_tokens_seen": 138179785, + "router_z_loss_clip": 0.57177734, + "router_z_loss_mlp": 0.1706543, + "step": 4843, + "time_per_iteration": 2.441622018814087 + }, + { + "auxiliary_loss_clip": 0.01101541, + "auxiliary_loss_mlp": 0.01045276, + "balance_loss_clip": 1.0357213, + "balance_loss_mlp": 1.0223577, + "epoch": 0.14056061749173002, + "flos": 16209125717760.0, + "grad_norm": 3.4209226398778303, + "language_loss": 0.76396757, + "learning_rate": 3.873151950846072e-06, + "loss": 0.7854358, + "num_input_tokens_seen": 138194625, + "router_z_loss_clip": 0.65771484, + "router_z_loss_mlp": 0.22918701, + "step": 4844, + "time_per_iteration": 2.3733105659484863 + }, + { + "auxiliary_loss_clip": 0.01097353, + "auxiliary_loss_mlp": 0.0104388, + "balance_loss_clip": 1.03792429, + "balance_loss_mlp": 1.02475834, + "epoch": 0.14058963496024607, + "flos": 17921685461760.0, + "grad_norm": 2.6441078601889494, + "language_loss": 0.8159517, + "learning_rate": 3.873086068633457e-06, + "loss": 0.83736402, + "num_input_tokens_seen": 138205845, + "router_z_loss_clip": 0.59472656, + "router_z_loss_mlp": 0.19116211, + "step": 4845, + "time_per_iteration": 2.440626859664917 + }, + { + "auxiliary_loss_clip": 0.0109226, + "auxiliary_loss_mlp": 0.01046166, + "balance_loss_clip": 1.03519654, + "balance_loss_mlp": 1.02761102, + "epoch": 0.14061865242876212, + "flos": 30913006544640.0, + "grad_norm": 2.114774889237323, + "language_loss": 0.74697924, + "learning_rate": 3.873020169876988e-06, + "loss": 0.76836348, + "num_input_tokens_seen": 138222690, + "router_z_loss_clip": 0.57128906, + "router_z_loss_mlp": 0.18554688, + "step": 4846, + "time_per_iteration": 2.477691650390625 + }, + { + "auxiliary_loss_clip": 0.0109082, + "auxiliary_loss_mlp": 0.01033677, + "balance_loss_clip": 1.03446651, + "balance_loss_mlp": 1.01755357, + "epoch": 0.14064766989727817, + "flos": 18579658544640.0, + "grad_norm": 2.8665562780402087, + "language_loss": 0.83807003, + "learning_rate": 3.8729542545772465e-06, + "loss": 0.85931504, + "num_input_tokens_seen": 138234370, + "router_z_loss_clip": 0.56347656, + "router_z_loss_mlp": 0.16125488, + "step": 4847, + "time_per_iteration": 2.607304334640503 + }, + { + "auxiliary_loss_clip": 0.01096613, + "auxiliary_loss_mlp": 0.01043096, + "balance_loss_clip": 1.03167117, + "balance_loss_mlp": 1.0214715, + "epoch": 0.1406766873657942, + "flos": 16466575150080.0, + "grad_norm": 3.4432783008091556, + "language_loss": 0.87810189, + "learning_rate": 3.872888322734815e-06, + "loss": 0.899499, + "num_input_tokens_seen": 138251555, + "router_z_loss_clip": 0.64941406, + "router_z_loss_mlp": 0.21606445, + "step": 4848, + "time_per_iteration": 2.3649919033050537 + }, + { + "auxiliary_loss_clip": 0.01100552, + "auxiliary_loss_mlp": 0.0103827, + "balance_loss_clip": 1.03544903, + "balance_loss_mlp": 1.01933968, + "epoch": 0.14070570483431025, + "flos": 30292285749120.0, + "grad_norm": 2.3436686906932653, + "language_loss": 0.77377421, + "learning_rate": 3.8728223743502766e-06, + "loss": 0.79516244, + "num_input_tokens_seen": 138268145, + "router_z_loss_clip": 0.65087891, + "router_z_loss_mlp": 0.18933105, + "step": 4849, + "time_per_iteration": 2.458134412765503 + }, + { + "auxiliary_loss_clip": 0.01099464, + "auxiliary_loss_mlp": 0.01040009, + "balance_loss_clip": 1.0356679, + "balance_loss_mlp": 1.02215719, + "epoch": 0.1407347223028263, + "flos": 13655578210560.0, + "grad_norm": 2.908234993187214, + "language_loss": 0.67479277, + "learning_rate": 3.872756409424212e-06, + "loss": 0.6961875, + "num_input_tokens_seen": 138280320, + "router_z_loss_clip": 0.63867188, + "router_z_loss_mlp": 0.17852783, + "step": 4850, + "time_per_iteration": 2.3370041847229004 + }, + { + "auxiliary_loss_clip": 0.01096374, + "auxiliary_loss_mlp": 0.01035918, + "balance_loss_clip": 1.03407907, + "balance_loss_mlp": 1.01738048, + "epoch": 0.14076373977134235, + "flos": 21536313143040.0, + "grad_norm": 2.641110730744073, + "language_loss": 0.90431714, + "learning_rate": 3.872690427957206e-06, + "loss": 0.92564005, + "num_input_tokens_seen": 138295305, + "router_z_loss_clip": 0.62231445, + "router_z_loss_mlp": 0.1854248, + "step": 4851, + "time_per_iteration": 2.4820849895477295 + }, + { + "auxiliary_loss_clip": 0.01013455, + "auxiliary_loss_mlp": 0.0102544, + "balance_loss_clip": 1.00393534, + "balance_loss_mlp": 1.02439702, + "epoch": 0.1407927572398584, + "flos": 58608381144960.0, + "grad_norm": 0.6981294581805667, + "language_loss": 0.45653844, + "learning_rate": 3.8726244299498394e-06, + "loss": 0.4769274, + "num_input_tokens_seen": 138357115, + "router_z_loss_clip": 0.09521484, + "router_z_loss_mlp": 0.01043701, + "step": 4852, + "time_per_iteration": 3.051569700241089 + }, + { + "auxiliary_loss_clip": 0.01085821, + "auxiliary_loss_mlp": 0.01039519, + "balance_loss_clip": 1.02941978, + "balance_loss_mlp": 1.02235866, + "epoch": 0.14082177470837445, + "flos": 22776044077440.0, + "grad_norm": 2.8020923214769726, + "language_loss": 0.83544767, + "learning_rate": 3.872558415402697e-06, + "loss": 0.85670114, + "num_input_tokens_seen": 138370255, + "router_z_loss_clip": 0.56494141, + "router_z_loss_mlp": 0.17163086, + "step": 4853, + "time_per_iteration": 2.4763264656066895 + }, + { + "auxiliary_loss_clip": 0.01013959, + "auxiliary_loss_mlp": 0.01007532, + "balance_loss_clip": 1.00484061, + "balance_loss_mlp": 1.00644732, + "epoch": 0.14085079217689048, + "flos": 62835001781760.0, + "grad_norm": 0.6886846280871843, + "language_loss": 0.47214258, + "learning_rate": 3.87249238431636e-06, + "loss": 0.49235749, + "num_input_tokens_seen": 138426110, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01086426, + "step": 4854, + "time_per_iteration": 2.9021213054656982 + }, + { + "auxiliary_loss_clip": 0.0109297, + "auxiliary_loss_mlp": 0.01033353, + "balance_loss_clip": 1.03289533, + "balance_loss_mlp": 1.01562095, + "epoch": 0.14087980964540653, + "flos": 15704595527040.0, + "grad_norm": 2.503270233063171, + "language_loss": 0.66809618, + "learning_rate": 3.872426336691413e-06, + "loss": 0.68935949, + "num_input_tokens_seen": 138441855, + "router_z_loss_clip": 0.60009766, + "router_z_loss_mlp": 0.1774292, + "step": 4855, + "time_per_iteration": 2.3684637546539307 + }, + { + "auxiliary_loss_clip": 0.01100588, + "auxiliary_loss_mlp": 0.01046657, + "balance_loss_clip": 1.03565979, + "balance_loss_mlp": 1.02537823, + "epoch": 0.14090882711392258, + "flos": 25074865238400.0, + "grad_norm": 2.7350441741152296, + "language_loss": 0.80167145, + "learning_rate": 3.8723602725284396e-06, + "loss": 0.8231439, + "num_input_tokens_seen": 138456145, + "router_z_loss_clip": 0.6484375, + "router_z_loss_mlp": 0.21252441, + "step": 4856, + "time_per_iteration": 2.435067653656006 + }, + { + "auxiliary_loss_clip": 0.01015575, + "auxiliary_loss_mlp": 0.0100492, + "balance_loss_clip": 1.00576329, + "balance_loss_mlp": 1.00380588, + "epoch": 0.14093784458243863, + "flos": 72998077847040.0, + "grad_norm": 0.6436369854364951, + "language_loss": 0.45649081, + "learning_rate": 3.872294191828022e-06, + "loss": 0.47669572, + "num_input_tokens_seen": 138526920, + "router_z_loss_clip": 0.09814453, + "router_z_loss_mlp": 0.01116943, + "step": 4857, + "time_per_iteration": 3.2041139602661133 + }, + { + "auxiliary_loss_clip": 0.01098401, + "auxiliary_loss_mlp": 0.01051462, + "balance_loss_clip": 1.03589702, + "balance_loss_mlp": 1.03192377, + "epoch": 0.14096686205095468, + "flos": 43097938888320.0, + "grad_norm": 1.8549102226433625, + "language_loss": 0.73131967, + "learning_rate": 3.872228094590745e-06, + "loss": 0.75281835, + "num_input_tokens_seen": 138547950, + "router_z_loss_clip": 0.625, + "router_z_loss_mlp": 0.1953125, + "step": 4858, + "time_per_iteration": 2.6333131790161133 + }, + { + "auxiliary_loss_clip": 0.01093956, + "auxiliary_loss_mlp": 0.01051681, + "balance_loss_clip": 1.03580832, + "balance_loss_mlp": 1.03210068, + "epoch": 0.14099587951947073, + "flos": 34392065950080.0, + "grad_norm": 1.6923407124286514, + "language_loss": 0.7685076, + "learning_rate": 3.872161980817191e-06, + "loss": 0.78996384, + "num_input_tokens_seen": 138568180, + "router_z_loss_clip": 0.58178711, + "router_z_loss_mlp": 0.19580078, + "step": 4859, + "time_per_iteration": 2.6241695880889893 + }, + { + "auxiliary_loss_clip": 0.01091854, + "auxiliary_loss_mlp": 0.01039895, + "balance_loss_clip": 1.0323118, + "balance_loss_mlp": 1.02185273, + "epoch": 0.14102489698798676, + "flos": 36685510761600.0, + "grad_norm": 2.578103005012181, + "language_loss": 0.81733978, + "learning_rate": 3.872095850507945e-06, + "loss": 0.83865726, + "num_input_tokens_seen": 138584465, + "router_z_loss_clip": 0.59545898, + "router_z_loss_mlp": 0.18054199, + "step": 4860, + "time_per_iteration": 2.8382723331451416 + }, + { + "auxiliary_loss_clip": 0.01087522, + "auxiliary_loss_mlp": 0.01052291, + "balance_loss_clip": 1.03166282, + "balance_loss_mlp": 1.03476775, + "epoch": 0.1410539144565028, + "flos": 31241539238400.0, + "grad_norm": 1.8216775186600114, + "language_loss": 0.78643262, + "learning_rate": 3.87202970366359e-06, + "loss": 0.80783081, + "num_input_tokens_seen": 138600570, + "router_z_loss_clip": 0.55932617, + "router_z_loss_mlp": 0.175354, + "step": 4861, + "time_per_iteration": 2.5833938121795654 + }, + { + "auxiliary_loss_clip": 0.01091224, + "auxiliary_loss_mlp": 0.0105175, + "balance_loss_clip": 1.03203893, + "balance_loss_mlp": 1.03388655, + "epoch": 0.14108293192501886, + "flos": 27812858791680.0, + "grad_norm": 1.8092323954215197, + "language_loss": 0.80124319, + "learning_rate": 3.871963540284713e-06, + "loss": 0.82267296, + "num_input_tokens_seen": 138618145, + "router_z_loss_clip": 0.59179688, + "router_z_loss_mlp": 0.17889404, + "step": 4862, + "time_per_iteration": 2.530183792114258 + }, + { + "auxiliary_loss_clip": 0.01097947, + "auxiliary_loss_mlp": 0.01044551, + "balance_loss_clip": 1.03271985, + "balance_loss_mlp": 1.02414203, + "epoch": 0.1411119493935349, + "flos": 41683606911360.0, + "grad_norm": 2.420432897375999, + "language_loss": 0.90771079, + "learning_rate": 3.871897360371896e-06, + "loss": 0.9291358, + "num_input_tokens_seen": 138642405, + "router_z_loss_clip": 0.65185547, + "router_z_loss_mlp": 0.2043457, + "step": 4863, + "time_per_iteration": 2.5636401176452637 + }, + { + "auxiliary_loss_clip": 0.01013513, + "auxiliary_loss_mlp": 0.01005754, + "balance_loss_clip": 1.00457704, + "balance_loss_mlp": 1.00468075, + "epoch": 0.14114096686205096, + "flos": 59077439527680.0, + "grad_norm": 0.655085138680005, + "language_loss": 0.50749278, + "learning_rate": 3.871831163925724e-06, + "loss": 0.5276854, + "num_input_tokens_seen": 138702555, + "router_z_loss_clip": 0.08935547, + "router_z_loss_mlp": 0.01074219, + "step": 4864, + "time_per_iteration": 2.9131174087524414 + }, + { + "auxiliary_loss_clip": 0.01090051, + "auxiliary_loss_mlp": 0.01043634, + "balance_loss_clip": 1.03148127, + "balance_loss_mlp": 1.02705741, + "epoch": 0.141169984330567, + "flos": 30181925341440.0, + "grad_norm": 2.14821878143609, + "language_loss": 0.79604042, + "learning_rate": 3.8717649509467804e-06, + "loss": 0.81737721, + "num_input_tokens_seen": 138719480, + "router_z_loss_clip": 0.58642578, + "router_z_loss_mlp": 0.16577148, + "step": 4865, + "time_per_iteration": 2.4696571826934814 + }, + { + "auxiliary_loss_clip": 0.01090754, + "auxiliary_loss_mlp": 0.01043906, + "balance_loss_clip": 1.03213239, + "balance_loss_mlp": 1.0273478, + "epoch": 0.14119900179908304, + "flos": 15442433061120.0, + "grad_norm": 2.6416230523651243, + "language_loss": 0.76131374, + "learning_rate": 3.871698721435652e-06, + "loss": 0.78266037, + "num_input_tokens_seen": 138731635, + "router_z_loss_clip": 0.58642578, + "router_z_loss_mlp": 0.16564941, + "step": 4866, + "time_per_iteration": 2.3698911666870117 + }, + { + "auxiliary_loss_clip": 0.01087397, + "auxiliary_loss_mlp": 0.01035463, + "balance_loss_clip": 1.03067422, + "balance_loss_mlp": 1.01931, + "epoch": 0.1412280192675991, + "flos": 29122555824000.0, + "grad_norm": 2.8094604303342536, + "language_loss": 0.67113173, + "learning_rate": 3.871632475392924e-06, + "loss": 0.69236028, + "num_input_tokens_seen": 138745750, + "router_z_loss_clip": 0.56738281, + "router_z_loss_mlp": 0.16149902, + "step": 4867, + "time_per_iteration": 2.4536921977996826 + }, + { + "auxiliary_loss_clip": 0.01095884, + "auxiliary_loss_mlp": 0.01041183, + "balance_loss_clip": 1.03552806, + "balance_loss_mlp": 1.02152503, + "epoch": 0.14125703673611514, + "flos": 38720458800000.0, + "grad_norm": 2.1054880460712835, + "language_loss": 0.84035707, + "learning_rate": 3.87156621281918e-06, + "loss": 0.86172771, + "num_input_tokens_seen": 138761630, + "router_z_loss_clip": 0.60327148, + "router_z_loss_mlp": 0.19641113, + "step": 4868, + "time_per_iteration": 2.5220179557800293 + }, + { + "auxiliary_loss_clip": 0.01087435, + "auxiliary_loss_mlp": 0.01034867, + "balance_loss_clip": 1.03170896, + "balance_loss_mlp": 1.01804018, + "epoch": 0.1412860542046312, + "flos": 16609509722880.0, + "grad_norm": 2.3442607173745587, + "language_loss": 0.69267392, + "learning_rate": 3.871499933715006e-06, + "loss": 0.71389693, + "num_input_tokens_seen": 138774850, + "router_z_loss_clip": 0.55712891, + "router_z_loss_mlp": 0.168396, + "step": 4869, + "time_per_iteration": 2.3790204524993896 + }, + { + "auxiliary_loss_clip": 0.0108958, + "auxiliary_loss_mlp": 0.01031876, + "balance_loss_clip": 1.03237295, + "balance_loss_mlp": 1.01504397, + "epoch": 0.14131507167314725, + "flos": 29975064336000.0, + "grad_norm": 1.562566562709928, + "language_loss": 0.76673049, + "learning_rate": 3.8714336380809875e-06, + "loss": 0.78794509, + "num_input_tokens_seen": 138798380, + "router_z_loss_clip": 0.57275391, + "router_z_loss_mlp": 0.16833496, + "step": 4870, + "time_per_iteration": 4.7623395919799805 + }, + { + "auxiliary_loss_clip": 0.01086507, + "auxiliary_loss_mlp": 0.01037889, + "balance_loss_clip": 1.03336608, + "balance_loss_mlp": 1.0230062, + "epoch": 0.14134408914166327, + "flos": 71629624995840.0, + "grad_norm": 1.9877217277531218, + "language_loss": 0.86238551, + "learning_rate": 3.871367325917709e-06, + "loss": 0.88362938, + "num_input_tokens_seen": 138824920, + "router_z_loss_clip": 0.53100586, + "router_z_loss_mlp": 0.14886475, + "step": 4871, + "time_per_iteration": 2.755035400390625 + }, + { + "auxiliary_loss_clip": 0.0109161, + "auxiliary_loss_mlp": 0.01039092, + "balance_loss_clip": 1.03186166, + "balance_loss_mlp": 1.02162766, + "epoch": 0.14137310661017932, + "flos": 45251137301760.0, + "grad_norm": 2.1386181408907805, + "language_loss": 0.76432776, + "learning_rate": 3.871300997225758e-06, + "loss": 0.78563482, + "num_input_tokens_seen": 138843630, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.17468262, + "step": 4872, + "time_per_iteration": 2.5208580493927 + }, + { + "auxiliary_loss_clip": 0.01021238, + "auxiliary_loss_mlp": 0.01009579, + "balance_loss_clip": 1.01143301, + "balance_loss_mlp": 1.00847018, + "epoch": 0.14140212407869537, + "flos": 74774668757760.0, + "grad_norm": 0.7934341466082853, + "language_loss": 0.5338372, + "learning_rate": 3.8712346520057185e-06, + "loss": 0.55414534, + "num_input_tokens_seen": 138909345, + "router_z_loss_clip": 0.09814453, + "router_z_loss_mlp": 0.0111084, + "step": 4873, + "time_per_iteration": 5.260589361190796 + }, + { + "auxiliary_loss_clip": 0.01019988, + "auxiliary_loss_mlp": 0.01004922, + "balance_loss_clip": 1.0100615, + "balance_loss_mlp": 1.00386655, + "epoch": 0.14143114154721143, + "flos": 74774424378240.0, + "grad_norm": 0.6861922357487694, + "language_loss": 0.50099891, + "learning_rate": 3.871168290258178e-06, + "loss": 0.52124798, + "num_input_tokens_seen": 138970240, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01055908, + "step": 4874, + "time_per_iteration": 3.0631325244903564 + }, + { + "auxiliary_loss_clip": 0.01093367, + "auxiliary_loss_mlp": 0.01041934, + "balance_loss_clip": 1.03303409, + "balance_loss_mlp": 1.02314687, + "epoch": 0.14146015901572748, + "flos": 34525260253440.0, + "grad_norm": 2.4553632398648872, + "language_loss": 0.79764086, + "learning_rate": 3.871101911983722e-06, + "loss": 0.81899387, + "num_input_tokens_seen": 138986040, + "router_z_loss_clip": 0.60253906, + "router_z_loss_mlp": 0.18786621, + "step": 4875, + "time_per_iteration": 2.6047229766845703 + }, + { + "auxiliary_loss_clip": 0.01092874, + "auxiliary_loss_mlp": 0.01036906, + "balance_loss_clip": 1.03314519, + "balance_loss_mlp": 1.01870275, + "epoch": 0.14148917648424353, + "flos": 15988195434240.0, + "grad_norm": 3.5560303469285244, + "language_loss": 0.89258569, + "learning_rate": 3.871035517182936e-06, + "loss": 0.91388351, + "num_input_tokens_seen": 138998735, + "router_z_loss_clip": 0.59790039, + "router_z_loss_mlp": 0.18225098, + "step": 4876, + "time_per_iteration": 2.398033618927002 + }, + { + "auxiliary_loss_clip": 0.01091796, + "auxiliary_loss_mlp": 0.01037819, + "balance_loss_clip": 1.0347631, + "balance_loss_mlp": 1.01977134, + "epoch": 0.14151819395275955, + "flos": 32079454801920.0, + "grad_norm": 2.26890898055072, + "language_loss": 0.75507486, + "learning_rate": 3.870969105856408e-06, + "loss": 0.77637112, + "num_input_tokens_seen": 139014785, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.18060303, + "step": 4877, + "time_per_iteration": 4.940696716308594 + }, + { + "auxiliary_loss_clip": 0.01088705, + "auxiliary_loss_mlp": 0.01036366, + "balance_loss_clip": 1.03320074, + "balance_loss_mlp": 1.02021861, + "epoch": 0.1415472114212756, + "flos": 19127515599360.0, + "grad_norm": 2.5992273621240587, + "language_loss": 0.77826279, + "learning_rate": 3.8709026780047225e-06, + "loss": 0.79951346, + "num_input_tokens_seen": 139028520, + "router_z_loss_clip": 0.55517578, + "router_z_loss_mlp": 0.16143799, + "step": 4878, + "time_per_iteration": 2.363992214202881 + }, + { + "auxiliary_loss_clip": 0.01019781, + "auxiliary_loss_mlp": 0.0100124, + "balance_loss_clip": 1.00983584, + "balance_loss_mlp": 1.00007808, + "epoch": 0.14157622888979166, + "flos": 62698979658240.0, + "grad_norm": 0.6788785598981255, + "language_loss": 0.53932309, + "learning_rate": 3.870836233628469e-06, + "loss": 0.55953324, + "num_input_tokens_seen": 139093780, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01159668, + "step": 4879, + "time_per_iteration": 3.0876946449279785 + }, + { + "auxiliary_loss_clip": 0.01087319, + "auxiliary_loss_mlp": 0.01046044, + "balance_loss_clip": 1.03161967, + "balance_loss_mlp": 1.02918744, + "epoch": 0.1416052463583077, + "flos": 18660726455040.0, + "grad_norm": 3.04650016868972, + "language_loss": 0.89802337, + "learning_rate": 3.870769772728232e-06, + "loss": 0.919357, + "num_input_tokens_seen": 139107700, + "router_z_loss_clip": 0.55737305, + "router_z_loss_mlp": 0.16845703, + "step": 4880, + "time_per_iteration": 4.8208677768707275 + }, + { + "auxiliary_loss_clip": 0.01017294, + "auxiliary_loss_mlp": 0.01004037, + "balance_loss_clip": 1.00785303, + "balance_loss_mlp": 1.00298214, + "epoch": 0.14163426382682376, + "flos": 64368387095040.0, + "grad_norm": 0.7235758425774093, + "language_loss": 0.52937305, + "learning_rate": 3.8707032953046e-06, + "loss": 0.54958642, + "num_input_tokens_seen": 139169030, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01055908, + "step": 4881, + "time_per_iteration": 2.9577767848968506 + }, + { + "auxiliary_loss_clip": 0.01078125, + "auxiliary_loss_mlp": 0.01043542, + "balance_loss_clip": 1.02820325, + "balance_loss_mlp": 1.02831888, + "epoch": 0.14166328129533978, + "flos": 32372585510400.0, + "grad_norm": 1.9162274973873332, + "language_loss": 0.88525462, + "learning_rate": 3.870636801358158e-06, + "loss": 0.90647125, + "num_input_tokens_seen": 139188470, + "router_z_loss_clip": 0.4987793, + "router_z_loss_mlp": 0.15222168, + "step": 4882, + "time_per_iteration": 2.4811408519744873 + }, + { + "auxiliary_loss_clip": 0.01013999, + "auxiliary_loss_mlp": 0.01003618, + "balance_loss_clip": 1.00490665, + "balance_loss_mlp": 1.00256276, + "epoch": 0.14169229876385583, + "flos": 66049978596480.0, + "grad_norm": 0.6617023137319507, + "language_loss": 0.4555636, + "learning_rate": 3.870570290889496e-06, + "loss": 0.47573978, + "num_input_tokens_seen": 139246410, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01055908, + "step": 4883, + "time_per_iteration": 3.0906524658203125 + }, + { + "auxiliary_loss_clip": 0.01090999, + "auxiliary_loss_mlp": 0.01045755, + "balance_loss_clip": 1.02941835, + "balance_loss_mlp": 1.0271945, + "epoch": 0.14172131623237189, + "flos": 10593904642560.0, + "grad_norm": 3.0102426469010384, + "language_loss": 0.94711792, + "learning_rate": 3.870503763899201e-06, + "loss": 0.96848547, + "num_input_tokens_seen": 139258725, + "router_z_loss_clip": 0.61621094, + "router_z_loss_mlp": 0.18560791, + "step": 4884, + "time_per_iteration": 2.3253114223480225 + }, + { + "auxiliary_loss_clip": 0.01086072, + "auxiliary_loss_mlp": 0.01045064, + "balance_loss_clip": 1.02734971, + "balance_loss_mlp": 1.02656317, + "epoch": 0.14175033370088794, + "flos": 22850269361280.0, + "grad_norm": 2.305638479855892, + "language_loss": 0.77421504, + "learning_rate": 3.870437220387858e-06, + "loss": 0.79552639, + "num_input_tokens_seen": 139275895, + "router_z_loss_clip": 0.58740234, + "router_z_loss_mlp": 0.18493652, + "step": 4885, + "time_per_iteration": 2.4153196811676025 + }, + { + "auxiliary_loss_clip": 0.01089063, + "auxiliary_loss_mlp": 0.01047462, + "balance_loss_clip": 1.02904344, + "balance_loss_mlp": 1.02886581, + "epoch": 0.141779351169404, + "flos": 18949772534400.0, + "grad_norm": 3.762048236117725, + "language_loss": 0.84442127, + "learning_rate": 3.870370660356058e-06, + "loss": 0.86578649, + "num_input_tokens_seen": 139295140, + "router_z_loss_clip": 0.60009766, + "router_z_loss_mlp": 0.18603516, + "step": 4886, + "time_per_iteration": 2.570732831954956 + }, + { + "auxiliary_loss_clip": 0.01096413, + "auxiliary_loss_mlp": 0.01057928, + "balance_loss_clip": 1.02954721, + "balance_loss_mlp": 1.03704286, + "epoch": 0.14180836863792004, + "flos": 33245798325120.0, + "grad_norm": 2.246205122819286, + "language_loss": 0.75500715, + "learning_rate": 3.870304083804387e-06, + "loss": 0.77655059, + "num_input_tokens_seen": 139313020, + "router_z_loss_clip": 0.66748047, + "router_z_loss_mlp": 0.20874023, + "step": 4887, + "time_per_iteration": 2.6466238498687744 + }, + { + "auxiliary_loss_clip": 0.01085293, + "auxiliary_loss_mlp": 0.01041659, + "balance_loss_clip": 1.02706122, + "balance_loss_mlp": 1.02449298, + "epoch": 0.14183738610643606, + "flos": 15192873596160.0, + "grad_norm": 2.1038356007987122, + "language_loss": 0.77489626, + "learning_rate": 3.870237490733433e-06, + "loss": 0.79616576, + "num_input_tokens_seen": 139326510, + "router_z_loss_clip": 0.58227539, + "router_z_loss_mlp": 0.17169189, + "step": 4888, + "time_per_iteration": 2.3940343856811523 + }, + { + "auxiliary_loss_clip": 0.0108751, + "auxiliary_loss_mlp": 0.01038738, + "balance_loss_clip": 1.03046882, + "balance_loss_mlp": 1.02130342, + "epoch": 0.14186640357495212, + "flos": 24893072455680.0, + "grad_norm": 1.7877077772500511, + "language_loss": 0.72488558, + "learning_rate": 3.870170881143785e-06, + "loss": 0.74614811, + "num_input_tokens_seen": 139342750, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.17443848, + "step": 4889, + "time_per_iteration": 2.484304904937744 + }, + { + "auxiliary_loss_clip": 0.01020058, + "auxiliary_loss_mlp": 0.01006853, + "balance_loss_clip": 1.01066279, + "balance_loss_mlp": 1.00574481, + "epoch": 0.14189542104346817, + "flos": 62040971664000.0, + "grad_norm": 0.7166064002267898, + "language_loss": 0.49756682, + "learning_rate": 3.870104255036031e-06, + "loss": 0.51783597, + "num_input_tokens_seen": 139402785, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.0111084, + "step": 4890, + "time_per_iteration": 2.960599422454834 + }, + { + "auxiliary_loss_clip": 0.01017947, + "auxiliary_loss_mlp": 0.01008447, + "balance_loss_clip": 1.00864935, + "balance_loss_mlp": 1.00735629, + "epoch": 0.14192443851198422, + "flos": 74768873472000.0, + "grad_norm": 0.704189087781073, + "language_loss": 0.5028699, + "learning_rate": 3.87003761241076e-06, + "loss": 0.52313387, + "num_input_tokens_seen": 139462290, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01092529, + "step": 4891, + "time_per_iteration": 2.991252899169922 + }, + { + "auxiliary_loss_clip": 0.01089987, + "auxiliary_loss_mlp": 0.01031088, + "balance_loss_clip": 1.03195608, + "balance_loss_mlp": 1.01356983, + "epoch": 0.14195345598050027, + "flos": 23615425918080.0, + "grad_norm": 3.861658670658814, + "language_loss": 0.83614874, + "learning_rate": 3.8699709532685605e-06, + "loss": 0.85735953, + "num_input_tokens_seen": 139476790, + "router_z_loss_clip": 0.58056641, + "router_z_loss_mlp": 0.17529297, + "step": 4892, + "time_per_iteration": 2.411689281463623 + }, + { + "auxiliary_loss_clip": 0.01089523, + "auxiliary_loss_mlp": 0.01034313, + "balance_loss_clip": 1.03069949, + "balance_loss_mlp": 1.01584101, + "epoch": 0.1419824734490163, + "flos": 22666591365120.0, + "grad_norm": 3.2832698273919525, + "language_loss": 1.01929474, + "learning_rate": 3.869904277610019e-06, + "loss": 1.04053307, + "num_input_tokens_seen": 139491180, + "router_z_loss_clip": 0.58789062, + "router_z_loss_mlp": 0.18475342, + "step": 4893, + "time_per_iteration": 2.428811550140381 + }, + { + "auxiliary_loss_clip": 0.01093303, + "auxiliary_loss_mlp": 0.01038289, + "balance_loss_clip": 1.03130245, + "balance_loss_mlp": 1.01846421, + "epoch": 0.14201149091753235, + "flos": 17010766512000.0, + "grad_norm": 3.120222650931243, + "language_loss": 0.83876234, + "learning_rate": 3.869837585435727e-06, + "loss": 0.86007828, + "num_input_tokens_seen": 139504000, + "router_z_loss_clip": 0.62109375, + "router_z_loss_mlp": 0.19836426, + "step": 4894, + "time_per_iteration": 2.348374843597412 + }, + { + "auxiliary_loss_clip": 0.01025681, + "auxiliary_loss_mlp": 0.01001057, + "balance_loss_clip": 1.01590848, + "balance_loss_mlp": 1.00001407, + "epoch": 0.1420405083860484, + "flos": 74010594453120.0, + "grad_norm": 0.7104515557270076, + "language_loss": 0.52204734, + "learning_rate": 3.869770876746274e-06, + "loss": 0.54231477, + "num_input_tokens_seen": 139569210, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.01043701, + "step": 4895, + "time_per_iteration": 3.0616586208343506 + }, + { + "auxiliary_loss_clip": 0.01095243, + "auxiliary_loss_mlp": 0.01036795, + "balance_loss_clip": 1.03257585, + "balance_loss_mlp": 1.01793587, + "epoch": 0.14206952585456445, + "flos": 24602280808320.0, + "grad_norm": 1.956968141152986, + "language_loss": 0.73133773, + "learning_rate": 3.869704151542247e-06, + "loss": 0.75265807, + "num_input_tokens_seen": 139586180, + "router_z_loss_clip": 0.62646484, + "router_z_loss_mlp": 0.18865967, + "step": 4896, + "time_per_iteration": 2.6907880306243896 + }, + { + "auxiliary_loss_clip": 0.01092488, + "auxiliary_loss_mlp": 0.01043641, + "balance_loss_clip": 1.03162384, + "balance_loss_mlp": 1.02459097, + "epoch": 0.1420985433230805, + "flos": 24964714298880.0, + "grad_norm": 2.362036646560335, + "language_loss": 0.8047682, + "learning_rate": 3.869637409824237e-06, + "loss": 0.8261295, + "num_input_tokens_seen": 139601060, + "router_z_loss_clip": 0.60888672, + "router_z_loss_mlp": 0.19042969, + "step": 4897, + "time_per_iteration": 2.440213203430176 + }, + { + "auxiliary_loss_clip": 0.01094766, + "auxiliary_loss_mlp": 0.01041252, + "balance_loss_clip": 1.03308868, + "balance_loss_mlp": 1.02249479, + "epoch": 0.14212756079159655, + "flos": 16610242861440.0, + "grad_norm": 2.097938071122553, + "language_loss": 0.86268139, + "learning_rate": 3.869570651592831e-06, + "loss": 0.88404155, + "num_input_tokens_seen": 139614830, + "router_z_loss_clip": 0.61767578, + "router_z_loss_mlp": 0.18756104, + "step": 4898, + "time_per_iteration": 2.3936445713043213 + }, + { + "auxiliary_loss_clip": 0.01096995, + "auxiliary_loss_mlp": 0.01044102, + "balance_loss_clip": 1.03552103, + "balance_loss_mlp": 1.02380073, + "epoch": 0.14215657826011258, + "flos": 33649498909440.0, + "grad_norm": 2.417841751793342, + "language_loss": 0.77001482, + "learning_rate": 3.869503876848623e-06, + "loss": 0.79142582, + "num_input_tokens_seen": 139632640, + "router_z_loss_clip": 0.61474609, + "router_z_loss_mlp": 0.20300293, + "step": 4899, + "time_per_iteration": 2.5534632205963135 + }, + { + "auxiliary_loss_clip": 0.01022106, + "auxiliary_loss_mlp": 0.01026698, + "balance_loss_clip": 1.01225436, + "balance_loss_mlp": 1.02552938, + "epoch": 0.14218559572862863, + "flos": 74768698915200.0, + "grad_norm": 0.6514757024757653, + "language_loss": 0.51560444, + "learning_rate": 3.869437085592198e-06, + "loss": 0.53609246, + "num_input_tokens_seen": 139693945, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01165771, + "step": 4900, + "time_per_iteration": 3.0627081394195557 + }, + { + "auxiliary_loss_clip": 0.0102152, + "auxiliary_loss_mlp": 0.01015116, + "balance_loss_clip": 1.01180148, + "balance_loss_mlp": 1.01405549, + "epoch": 0.14221461319714468, + "flos": 70799737178880.0, + "grad_norm": 0.6278162339518647, + "language_loss": 0.45697674, + "learning_rate": 3.86937027782415e-06, + "loss": 0.47734311, + "num_input_tokens_seen": 139764910, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.01062012, + "step": 4901, + "time_per_iteration": 3.12349796295166 + }, + { + "auxiliary_loss_clip": 0.01082804, + "auxiliary_loss_mlp": 0.01036974, + "balance_loss_clip": 1.02853084, + "balance_loss_mlp": 1.02124405, + "epoch": 0.14224363066566073, + "flos": 28107874713600.0, + "grad_norm": 1.8082599602523262, + "language_loss": 0.81825531, + "learning_rate": 3.869303453545066e-06, + "loss": 0.83945304, + "num_input_tokens_seen": 139782930, + "router_z_loss_clip": 0.54345703, + "router_z_loss_mlp": 0.15716553, + "step": 4902, + "time_per_iteration": 2.5632076263427734 + }, + { + "auxiliary_loss_clip": 0.01085425, + "auxiliary_loss_mlp": 0.01033202, + "balance_loss_clip": 1.02830172, + "balance_loss_mlp": 1.01327062, + "epoch": 0.14227264813417678, + "flos": 16360892864640.0, + "grad_norm": 2.5028132618312164, + "language_loss": 0.86391914, + "learning_rate": 3.869236612755538e-06, + "loss": 0.88510537, + "num_input_tokens_seen": 139795940, + "router_z_loss_clip": 0.57128906, + "router_z_loss_mlp": 0.19909668, + "step": 4903, + "time_per_iteration": 2.3218178749084473 + }, + { + "auxiliary_loss_clip": 0.01086956, + "auxiliary_loss_mlp": 0.01038716, + "balance_loss_clip": 1.02970004, + "balance_loss_mlp": 1.02146673, + "epoch": 0.14230166560269283, + "flos": 25695097274880.0, + "grad_norm": 2.7389178102909932, + "language_loss": 0.74995375, + "learning_rate": 3.869169755456156e-06, + "loss": 0.77121043, + "num_input_tokens_seen": 139810465, + "router_z_loss_clip": 0.57202148, + "router_z_loss_mlp": 0.17254639, + "step": 4904, + "time_per_iteration": 2.536121368408203 + }, + { + "auxiliary_loss_clip": 0.0108955, + "auxiliary_loss_mlp": 0.01031752, + "balance_loss_clip": 1.03090394, + "balance_loss_mlp": 1.01385832, + "epoch": 0.14233068307120886, + "flos": 26719029895680.0, + "grad_norm": 1.5927275418423077, + "language_loss": 0.67412484, + "learning_rate": 3.86910288164751e-06, + "loss": 0.69533783, + "num_input_tokens_seen": 139826505, + "router_z_loss_clip": 0.58642578, + "router_z_loss_mlp": 0.17895508, + "step": 4905, + "time_per_iteration": 2.4286386966705322 + }, + { + "auxiliary_loss_clip": 0.01024432, + "auxiliary_loss_mlp": 0.01003618, + "balance_loss_clip": 1.01546347, + "balance_loss_mlp": 1.00259233, + "epoch": 0.1423597005397249, + "flos": 71772068943360.0, + "grad_norm": 0.7215455722386277, + "language_loss": 0.49755484, + "learning_rate": 3.869035991330192e-06, + "loss": 0.51783532, + "num_input_tokens_seen": 139883105, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01025391, + "step": 4906, + "time_per_iteration": 2.9950010776519775 + }, + { + "auxiliary_loss_clip": 0.01024697, + "auxiliary_loss_mlp": 0.01001317, + "balance_loss_clip": 1.01549053, + "balance_loss_mlp": 1.00033343, + "epoch": 0.14238871800824096, + "flos": 74772958101120.0, + "grad_norm": 0.6800201835872023, + "language_loss": 0.49486065, + "learning_rate": 3.86896908450479e-06, + "loss": 0.5151208, + "num_input_tokens_seen": 139949495, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.00982666, + "step": 4907, + "time_per_iteration": 3.0657765865325928 + }, + { + "auxiliary_loss_clip": 0.01096537, + "auxiliary_loss_mlp": 0.01044518, + "balance_loss_clip": 1.03448844, + "balance_loss_mlp": 1.02562916, + "epoch": 0.142417735476757, + "flos": 18288029024640.0, + "grad_norm": 3.6533801755462725, + "language_loss": 0.96505326, + "learning_rate": 3.868902161171897e-06, + "loss": 0.9864639, + "num_input_tokens_seen": 139960970, + "router_z_loss_clip": 0.62084961, + "router_z_loss_mlp": 0.18896484, + "step": 4908, + "time_per_iteration": 2.370354652404785 + }, + { + "auxiliary_loss_clip": 0.01090741, + "auxiliary_loss_mlp": 0.01036262, + "balance_loss_clip": 1.03242254, + "balance_loss_mlp": 1.01896501, + "epoch": 0.14244675294527306, + "flos": 36713267159040.0, + "grad_norm": 2.505858453516241, + "language_loss": 0.85437208, + "learning_rate": 3.868835221332105e-06, + "loss": 0.87564206, + "num_input_tokens_seen": 139980940, + "router_z_loss_clip": 0.58374023, + "router_z_loss_mlp": 0.17315674, + "step": 4909, + "time_per_iteration": 2.7722818851470947 + }, + { + "auxiliary_loss_clip": 0.01022998, + "auxiliary_loss_mlp": 0.01005058, + "balance_loss_clip": 1.01290393, + "balance_loss_mlp": 1.00396681, + "epoch": 0.1424757704137891, + "flos": 64002706848000.0, + "grad_norm": 0.6741450784239484, + "language_loss": 0.50801766, + "learning_rate": 3.868768264986004e-06, + "loss": 0.52829826, + "num_input_tokens_seen": 140042780, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.01092529, + "step": 4910, + "time_per_iteration": 2.9923415184020996 + }, + { + "auxiliary_loss_clip": 0.01020605, + "auxiliary_loss_mlp": 0.01002248, + "balance_loss_clip": 1.01066959, + "balance_loss_mlp": 1.00106215, + "epoch": 0.14250478788230514, + "flos": 70827319019520.0, + "grad_norm": 0.620453408651257, + "language_loss": 0.46433389, + "learning_rate": 3.868701292134185e-06, + "loss": 0.4845624, + "num_input_tokens_seen": 140105470, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01184082, + "step": 4911, + "time_per_iteration": 3.0547895431518555 + }, + { + "auxiliary_loss_clip": 0.01096572, + "auxiliary_loss_mlp": 0.01041396, + "balance_loss_clip": 1.03604686, + "balance_loss_mlp": 1.02146435, + "epoch": 0.1425338053508212, + "flos": 13144484684160.0, + "grad_norm": 2.6440871663655514, + "language_loss": 0.76585674, + "learning_rate": 3.86863430277724e-06, + "loss": 0.78723639, + "num_input_tokens_seen": 140118160, + "router_z_loss_clip": 0.60498047, + "router_z_loss_mlp": 0.19921875, + "step": 4912, + "time_per_iteration": 2.349280595779419 + }, + { + "auxiliary_loss_clip": 0.01096103, + "auxiliary_loss_mlp": 0.01045458, + "balance_loss_clip": 1.03527629, + "balance_loss_mlp": 1.02534747, + "epoch": 0.14256282281933724, + "flos": 24967158094080.0, + "grad_norm": 1.9118954445446679, + "language_loss": 0.85346359, + "learning_rate": 3.868567296915761e-06, + "loss": 0.87487918, + "num_input_tokens_seen": 140138420, + "router_z_loss_clip": 0.60742188, + "router_z_loss_mlp": 0.20117188, + "step": 4913, + "time_per_iteration": 2.4777402877807617 + }, + { + "auxiliary_loss_clip": 0.01024076, + "auxiliary_loss_mlp": 0.01001771, + "balance_loss_clip": 1.01431489, + "balance_loss_mlp": 1.000597, + "epoch": 0.1425918402878533, + "flos": 61856525617920.0, + "grad_norm": 0.9175034896162114, + "language_loss": 0.46385598, + "learning_rate": 3.868500274550339e-06, + "loss": 0.48411447, + "num_input_tokens_seen": 140198775, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.01171875, + "step": 4914, + "time_per_iteration": 2.933823585510254 + }, + { + "auxiliary_loss_clip": 0.01023135, + "auxiliary_loss_mlp": 0.01001564, + "balance_loss_clip": 1.01318169, + "balance_loss_mlp": 1.0004611, + "epoch": 0.14262085775636935, + "flos": 74776379414400.0, + "grad_norm": 0.6098573124871391, + "language_loss": 0.45797139, + "learning_rate": 3.868433235681566e-06, + "loss": 0.47821844, + "num_input_tokens_seen": 140265290, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01104736, + "step": 4915, + "time_per_iteration": 3.111370801925659 + }, + { + "auxiliary_loss_clip": 0.01086245, + "auxiliary_loss_mlp": 0.01048876, + "balance_loss_clip": 1.03080392, + "balance_loss_mlp": 1.02911139, + "epoch": 0.14264987522488537, + "flos": 14676962302080.0, + "grad_norm": 3.1132154224110646, + "language_loss": 0.88858986, + "learning_rate": 3.868366180310036e-06, + "loss": 0.90994108, + "num_input_tokens_seen": 140276615, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.19744873, + "step": 4916, + "time_per_iteration": 2.3284664154052734 + }, + { + "auxiliary_loss_clip": 0.01017326, + "auxiliary_loss_mlp": 0.01001375, + "balance_loss_clip": 1.00823545, + "balance_loss_mlp": 1.00037336, + "epoch": 0.14267889269340142, + "flos": 65865043791360.0, + "grad_norm": 0.6568931500210752, + "language_loss": 0.46552402, + "learning_rate": 3.868299108436339e-06, + "loss": 0.48571104, + "num_input_tokens_seen": 140333190, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01000977, + "step": 4917, + "time_per_iteration": 2.890259265899658 + }, + { + "auxiliary_loss_clip": 0.01097276, + "auxiliary_loss_mlp": 0.01040371, + "balance_loss_clip": 1.03238034, + "balance_loss_mlp": 1.01760244, + "epoch": 0.14270791016191747, + "flos": 29379272117760.0, + "grad_norm": 4.184453818063242, + "language_loss": 0.89567691, + "learning_rate": 3.868232020061068e-06, + "loss": 0.91705346, + "num_input_tokens_seen": 140350165, + "router_z_loss_clip": 0.64941406, + "router_z_loss_mlp": 0.22790527, + "step": 4918, + "time_per_iteration": 2.443570137023926 + }, + { + "auxiliary_loss_clip": 0.01014072, + "auxiliary_loss_mlp": 0.01001649, + "balance_loss_clip": 1.00512171, + "balance_loss_mlp": 1.00053394, + "epoch": 0.14273692763043352, + "flos": 60799041313920.0, + "grad_norm": 0.6115627643808991, + "language_loss": 0.45979181, + "learning_rate": 3.868164915184817e-06, + "loss": 0.479949, + "num_input_tokens_seen": 140412370, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01116943, + "step": 4919, + "time_per_iteration": 3.0723161697387695 + }, + { + "auxiliary_loss_clip": 0.0109217, + "auxiliary_loss_mlp": 0.01044449, + "balance_loss_clip": 1.02888083, + "balance_loss_mlp": 1.02426672, + "epoch": 0.14276594509894958, + "flos": 26863744947840.0, + "grad_norm": 2.93246797280003, + "language_loss": 0.93402541, + "learning_rate": 3.868097793808176e-06, + "loss": 0.95539153, + "num_input_tokens_seen": 140428275, + "router_z_loss_clip": 0.63330078, + "router_z_loss_mlp": 0.20178223, + "step": 4920, + "time_per_iteration": 2.3616316318511963 + }, + { + "auxiliary_loss_clip": 0.01087718, + "auxiliary_loss_mlp": 0.01043532, + "balance_loss_clip": 1.02972329, + "balance_loss_mlp": 1.0266757, + "epoch": 0.14279496256746563, + "flos": 22264391969280.0, + "grad_norm": 2.207220122566672, + "language_loss": 0.78925323, + "learning_rate": 3.8680306559317405e-06, + "loss": 0.81056571, + "num_input_tokens_seen": 140442500, + "router_z_loss_clip": 0.57958984, + "router_z_loss_mlp": 0.16851807, + "step": 4921, + "time_per_iteration": 2.4337282180786133 + }, + { + "auxiliary_loss_clip": 0.0108487, + "auxiliary_loss_mlp": 0.01038084, + "balance_loss_clip": 1.0287931, + "balance_loss_mlp": 1.01963055, + "epoch": 0.14282398003598165, + "flos": 26371468644480.0, + "grad_norm": 1.7811342199000435, + "language_loss": 0.56509328, + "learning_rate": 3.867963501556102e-06, + "loss": 0.58632284, + "num_input_tokens_seen": 140462010, + "router_z_loss_clip": 0.56152344, + "router_z_loss_mlp": 0.18469238, + "step": 4922, + "time_per_iteration": 2.644685745239258 + }, + { + "auxiliary_loss_clip": 0.010167, + "auxiliary_loss_mlp": 0.01001393, + "balance_loss_clip": 1.00763249, + "balance_loss_mlp": 1.00044513, + "epoch": 0.1428529975044977, + "flos": 55535815232640.0, + "grad_norm": 0.7214807491564493, + "language_loss": 0.50229621, + "learning_rate": 3.867896330681854e-06, + "loss": 0.52247715, + "num_input_tokens_seen": 140516450, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.00946045, + "step": 4923, + "time_per_iteration": 2.9474470615386963 + }, + { + "auxiliary_loss_clip": 0.01093242, + "auxiliary_loss_mlp": 0.01044403, + "balance_loss_clip": 1.03431773, + "balance_loss_mlp": 1.02629519, + "epoch": 0.14288201497301375, + "flos": 59372247847680.0, + "grad_norm": 1.4658805346514396, + "language_loss": 0.66695082, + "learning_rate": 3.86782914330959e-06, + "loss": 0.68832725, + "num_input_tokens_seen": 140544355, + "router_z_loss_clip": 0.59033203, + "router_z_loss_mlp": 0.18115234, + "step": 4924, + "time_per_iteration": 2.8110952377319336 + }, + { + "auxiliary_loss_clip": 0.01087937, + "auxiliary_loss_mlp": 0.01049061, + "balance_loss_clip": 1.03065372, + "balance_loss_mlp": 1.03250885, + "epoch": 0.1429110324415298, + "flos": 13474518566400.0, + "grad_norm": 2.36585613849852, + "language_loss": 0.66767383, + "learning_rate": 3.867761939439902e-06, + "loss": 0.68904388, + "num_input_tokens_seen": 140562625, + "router_z_loss_clip": 0.57250977, + "router_z_loss_mlp": 0.16552734, + "step": 4925, + "time_per_iteration": 2.394462823867798 + }, + { + "auxiliary_loss_clip": 0.01083848, + "auxiliary_loss_mlp": 0.01031762, + "balance_loss_clip": 1.02997839, + "balance_loss_mlp": 1.01652753, + "epoch": 0.14294004991004586, + "flos": 29711505415680.0, + "grad_norm": 1.8594729478094392, + "language_loss": 0.70443428, + "learning_rate": 3.8676947190733855e-06, + "loss": 0.72559041, + "num_input_tokens_seen": 140580935, + "router_z_loss_clip": 0.53808594, + "router_z_loss_mlp": 0.15222168, + "step": 4926, + "time_per_iteration": 2.5141115188598633 + }, + { + "auxiliary_loss_clip": 0.01093717, + "auxiliary_loss_mlp": 0.01039181, + "balance_loss_clip": 1.03411388, + "balance_loss_mlp": 1.02132332, + "epoch": 0.14296906737856188, + "flos": 33870114990720.0, + "grad_norm": 2.133663618323686, + "language_loss": 0.7665776, + "learning_rate": 3.867627482210634e-06, + "loss": 0.78790653, + "num_input_tokens_seen": 140597075, + "router_z_loss_clip": 0.59619141, + "router_z_loss_mlp": 0.17877197, + "step": 4927, + "time_per_iteration": 2.517810583114624 + }, + { + "auxiliary_loss_clip": 0.01094528, + "auxiliary_loss_mlp": 0.01042353, + "balance_loss_clip": 1.03447151, + "balance_loss_mlp": 1.02428079, + "epoch": 0.14299808484707793, + "flos": 24016787441280.0, + "grad_norm": 2.227490471283074, + "language_loss": 0.86172366, + "learning_rate": 3.86756022885224e-06, + "loss": 0.8830924, + "num_input_tokens_seen": 140611865, + "router_z_loss_clip": 0.60058594, + "router_z_loss_mlp": 0.18054199, + "step": 4928, + "time_per_iteration": 2.430691957473755 + }, + { + "auxiliary_loss_clip": 0.01094249, + "auxiliary_loss_mlp": 0.0103444, + "balance_loss_clip": 1.03519773, + "balance_loss_mlp": 1.01493776, + "epoch": 0.14302710231559398, + "flos": 25622582647680.0, + "grad_norm": 2.2133229755037944, + "language_loss": 0.8585484, + "learning_rate": 3.867492958998799e-06, + "loss": 0.87983525, + "num_input_tokens_seen": 140627065, + "router_z_loss_clip": 0.58984375, + "router_z_loss_mlp": 0.19494629, + "step": 4929, + "time_per_iteration": 2.4160537719726562 + }, + { + "auxiliary_loss_clip": 0.01093868, + "auxiliary_loss_mlp": 0.01034974, + "balance_loss_clip": 1.03633857, + "balance_loss_mlp": 1.01887465, + "epoch": 0.14305611978411004, + "flos": 11392124123520.0, + "grad_norm": 3.7907915660124805, + "language_loss": 0.74769986, + "learning_rate": 3.867425672650904e-06, + "loss": 0.76898825, + "num_input_tokens_seen": 140638110, + "router_z_loss_clip": 0.57568359, + "router_z_loss_mlp": 0.16101074, + "step": 4930, + "time_per_iteration": 2.376713514328003 + }, + { + "auxiliary_loss_clip": 0.01089561, + "auxiliary_loss_mlp": 0.01031429, + "balance_loss_clip": 1.0344069, + "balance_loss_mlp": 1.01460266, + "epoch": 0.1430851372526261, + "flos": 74730610621440.0, + "grad_norm": 1.981264290615809, + "language_loss": 0.71451962, + "learning_rate": 3.86735836980915e-06, + "loss": 0.73572958, + "num_input_tokens_seen": 140660835, + "router_z_loss_clip": 0.55224609, + "router_z_loss_mlp": 0.168396, + "step": 4931, + "time_per_iteration": 2.8008992671966553 + }, + { + "auxiliary_loss_clip": 0.01097216, + "auxiliary_loss_mlp": 0.01034982, + "balance_loss_clip": 1.03546131, + "balance_loss_mlp": 1.01540816, + "epoch": 0.14311415472114214, + "flos": 16572885840000.0, + "grad_norm": 2.6888799601164974, + "language_loss": 0.84231877, + "learning_rate": 3.8672910504741315e-06, + "loss": 0.86364079, + "num_input_tokens_seen": 140673115, + "router_z_loss_clip": 0.6171875, + "router_z_loss_mlp": 0.19567871, + "step": 4932, + "time_per_iteration": 2.39605450630188 + }, + { + "auxiliary_loss_clip": 0.01029708, + "auxiliary_loss_mlp": 0.0100404, + "balance_loss_clip": 1.0196147, + "balance_loss_mlp": 1.00314045, + "epoch": 0.14314317218965816, + "flos": 60753968881920.0, + "grad_norm": 0.7294402091573642, + "language_loss": 0.52440542, + "learning_rate": 3.867223714646442e-06, + "loss": 0.54474294, + "num_input_tokens_seen": 140725625, + "router_z_loss_clip": 0.10107422, + "router_z_loss_mlp": 0.00897217, + "step": 4933, + "time_per_iteration": 2.8258132934570312 + }, + { + "auxiliary_loss_clip": 0.01100863, + "auxiliary_loss_mlp": 0.01041933, + "balance_loss_clip": 1.03771424, + "balance_loss_mlp": 1.02288938, + "epoch": 0.14317218965817421, + "flos": 34085494368000.0, + "grad_norm": 2.780942066307583, + "language_loss": 0.94297153, + "learning_rate": 3.867156362326678e-06, + "loss": 0.96439946, + "num_input_tokens_seen": 140748190, + "router_z_loss_clip": 0.6315918, + "router_z_loss_mlp": 0.19055176, + "step": 4934, + "time_per_iteration": 2.742098808288574 + }, + { + "auxiliary_loss_clip": 0.01026888, + "auxiliary_loss_mlp": 0.01001846, + "balance_loss_clip": 1.01683688, + "balance_loss_mlp": 1.00074363, + "epoch": 0.14320120712669027, + "flos": 60620320730880.0, + "grad_norm": 0.6890489146426798, + "language_loss": 0.46839964, + "learning_rate": 3.867088993515432e-06, + "loss": 0.48868698, + "num_input_tokens_seen": 140813230, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.01104736, + "step": 4935, + "time_per_iteration": 3.195516586303711 + }, + { + "auxiliary_loss_clip": 0.01087102, + "auxiliary_loss_mlp": 0.01037257, + "balance_loss_clip": 1.0292201, + "balance_loss_mlp": 1.01872563, + "epoch": 0.14323022459520632, + "flos": 37043301041280.0, + "grad_norm": 1.9310756951055326, + "language_loss": 1.00317454, + "learning_rate": 3.867021608213302e-06, + "loss": 1.02441812, + "num_input_tokens_seen": 140834150, + "router_z_loss_clip": 0.57885742, + "router_z_loss_mlp": 0.18536377, + "step": 4936, + "time_per_iteration": 2.5728955268859863 + }, + { + "auxiliary_loss_clip": 0.01082143, + "auxiliary_loss_mlp": 0.01035406, + "balance_loss_clip": 1.02773976, + "balance_loss_mlp": 1.01833487, + "epoch": 0.14325924206372237, + "flos": 22995787374720.0, + "grad_norm": 2.612829004058317, + "language_loss": 0.98767704, + "learning_rate": 3.866954206420881e-06, + "loss": 1.00885248, + "num_input_tokens_seen": 140849965, + "router_z_loss_clip": 0.54370117, + "router_z_loss_mlp": 0.1706543, + "step": 4937, + "time_per_iteration": 2.3984591960906982 + }, + { + "auxiliary_loss_clip": 0.01100102, + "auxiliary_loss_mlp": 0.01052022, + "balance_loss_clip": 1.03224897, + "balance_loss_mlp": 1.03211379, + "epoch": 0.14328825953223842, + "flos": 15663468078720.0, + "grad_norm": 2.5109406842289332, + "language_loss": 0.98569846, + "learning_rate": 3.866886788138765e-06, + "loss": 1.00721955, + "num_input_tokens_seen": 140865175, + "router_z_loss_clip": 0.67871094, + "router_z_loss_mlp": 0.19909668, + "step": 4938, + "time_per_iteration": 2.4226067066192627 + }, + { + "auxiliary_loss_clip": 0.01090423, + "auxiliary_loss_mlp": 0.0104035, + "balance_loss_clip": 1.03111827, + "balance_loss_mlp": 1.0210855, + "epoch": 0.14331727700075444, + "flos": 15005145882240.0, + "grad_norm": 3.0747043674169654, + "language_loss": 0.7812447, + "learning_rate": 3.86681935336755e-06, + "loss": 0.80255246, + "num_input_tokens_seen": 140877400, + "router_z_loss_clip": 0.5925293, + "router_z_loss_mlp": 0.19274902, + "step": 4939, + "time_per_iteration": 2.4035332202911377 + }, + { + "auxiliary_loss_clip": 0.01088739, + "auxiliary_loss_mlp": 0.01037688, + "balance_loss_clip": 1.03019202, + "balance_loss_mlp": 1.01910341, + "epoch": 0.1433462944692705, + "flos": 74737557982080.0, + "grad_norm": 1.6346720197825582, + "language_loss": 0.84955299, + "learning_rate": 3.866751902107831e-06, + "loss": 0.87081724, + "num_input_tokens_seen": 140907505, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.18560791, + "step": 4940, + "time_per_iteration": 2.8945796489715576 + }, + { + "auxiliary_loss_clip": 0.01018665, + "auxiliary_loss_mlp": 0.0100384, + "balance_loss_clip": 1.0090549, + "balance_loss_mlp": 1.00279653, + "epoch": 0.14337531193778655, + "flos": 74784059913600.0, + "grad_norm": 0.6986341524337393, + "language_loss": 0.52963609, + "learning_rate": 3.866684434360203e-06, + "loss": 0.54986113, + "num_input_tokens_seen": 140978730, + "router_z_loss_clip": 0.09619141, + "router_z_loss_mlp": 0.01043701, + "step": 4941, + "time_per_iteration": 3.2025797367095947 + }, + { + "auxiliary_loss_clip": 0.01091902, + "auxiliary_loss_mlp": 0.01033284, + "balance_loss_clip": 1.02887666, + "balance_loss_mlp": 1.01500916, + "epoch": 0.1434043294063026, + "flos": 31788453686400.0, + "grad_norm": 2.344788247322154, + "language_loss": 0.80875409, + "learning_rate": 3.866616950125263e-06, + "loss": 0.830006, + "num_input_tokens_seen": 141000495, + "router_z_loss_clip": 0.63012695, + "router_z_loss_mlp": 0.18286133, + "step": 4942, + "time_per_iteration": 2.5491368770599365 + }, + { + "auxiliary_loss_clip": 0.01085219, + "auxiliary_loss_mlp": 0.01033447, + "balance_loss_clip": 1.02845752, + "balance_loss_mlp": 1.01687694, + "epoch": 0.14343334687481865, + "flos": 14023527696000.0, + "grad_norm": 2.0045217076744453, + "language_loss": 0.67042869, + "learning_rate": 3.866549449403607e-06, + "loss": 0.69161534, + "num_input_tokens_seen": 141016315, + "router_z_loss_clip": 0.56762695, + "router_z_loss_mlp": 0.16552734, + "step": 4943, + "time_per_iteration": 2.4003424644470215 + }, + { + "auxiliary_loss_clip": 0.01083932, + "auxiliary_loss_mlp": 0.01037268, + "balance_loss_clip": 1.02859306, + "balance_loss_mlp": 1.02047181, + "epoch": 0.14346236434333468, + "flos": 15405948823680.0, + "grad_norm": 2.9103207084029235, + "language_loss": 0.7507, + "learning_rate": 3.866481932195831e-06, + "loss": 0.77191198, + "num_input_tokens_seen": 141029335, + "router_z_loss_clip": 0.55371094, + "router_z_loss_mlp": 0.16796875, + "step": 4944, + "time_per_iteration": 2.3698391914367676 + }, + { + "auxiliary_loss_clip": 0.01092085, + "auxiliary_loss_mlp": 0.01040977, + "balance_loss_clip": 1.03212857, + "balance_loss_mlp": 1.02130699, + "epoch": 0.14349138181185073, + "flos": 13472772998400.0, + "grad_norm": 2.3139174002806677, + "language_loss": 0.70426506, + "learning_rate": 3.866414398502531e-06, + "loss": 0.72559571, + "num_input_tokens_seen": 141042830, + "router_z_loss_clip": 0.59960938, + "router_z_loss_mlp": 0.19677734, + "step": 4945, + "time_per_iteration": 2.3457953929901123 + }, + { + "auxiliary_loss_clip": 0.01084954, + "auxiliary_loss_mlp": 0.01031453, + "balance_loss_clip": 1.03094876, + "balance_loss_mlp": 1.01504397, + "epoch": 0.14352039928036678, + "flos": 37406886606720.0, + "grad_norm": 1.975631420019996, + "language_loss": 0.84982854, + "learning_rate": 3.866346848324304e-06, + "loss": 0.8709926, + "num_input_tokens_seen": 141059870, + "router_z_loss_clip": 0.54003906, + "router_z_loss_mlp": 0.16424561, + "step": 4946, + "time_per_iteration": 4.674873352050781 + }, + { + "auxiliary_loss_clip": 0.01088105, + "auxiliary_loss_mlp": 0.01033887, + "balance_loss_clip": 1.03192675, + "balance_loss_mlp": 1.0163393, + "epoch": 0.14354941674888283, + "flos": 18617783616000.0, + "grad_norm": 3.3883655268708757, + "language_loss": 0.85400581, + "learning_rate": 3.8662792816617465e-06, + "loss": 0.87522572, + "num_input_tokens_seen": 141073325, + "router_z_loss_clip": 0.56225586, + "router_z_loss_mlp": 0.17541504, + "step": 4947, + "time_per_iteration": 2.3744359016418457 + }, + { + "auxiliary_loss_clip": 0.01088897, + "auxiliary_loss_mlp": 0.01033374, + "balance_loss_clip": 1.03225946, + "balance_loss_mlp": 1.01647007, + "epoch": 0.14357843421739888, + "flos": 14166427357440.0, + "grad_norm": 2.355265428301162, + "language_loss": 0.90536952, + "learning_rate": 3.8662116985154545e-06, + "loss": 0.92659223, + "num_input_tokens_seen": 141086265, + "router_z_loss_clip": 0.56616211, + "router_z_loss_mlp": 0.16882324, + "step": 4948, + "time_per_iteration": 2.545245409011841 + }, + { + "auxiliary_loss_clip": 0.01095948, + "auxiliary_loss_mlp": 0.01044271, + "balance_loss_clip": 1.03380251, + "balance_loss_mlp": 1.02530444, + "epoch": 0.14360745168591493, + "flos": 25514735857920.0, + "grad_norm": 2.683085622567546, + "language_loss": 0.87710881, + "learning_rate": 3.866144098886027e-06, + "loss": 0.89851105, + "num_input_tokens_seen": 141102335, + "router_z_loss_clip": 0.62207031, + "router_z_loss_mlp": 0.18945312, + "step": 4949, + "time_per_iteration": 4.8945136070251465 + }, + { + "auxiliary_loss_clip": 0.0108861, + "auxiliary_loss_mlp": 0.01040431, + "balance_loss_clip": 1.03279495, + "balance_loss_mlp": 1.02334261, + "epoch": 0.14363646915443096, + "flos": 12631366298880.0, + "grad_norm": 2.722560496439674, + "language_loss": 0.87592626, + "learning_rate": 3.866076482774058e-06, + "loss": 0.89721668, + "num_input_tokens_seen": 141114270, + "router_z_loss_clip": 0.55883789, + "router_z_loss_mlp": 0.1708374, + "step": 4950, + "time_per_iteration": 2.3868117332458496 + }, + { + "auxiliary_loss_clip": 0.01086845, + "auxiliary_loss_mlp": 0.01039836, + "balance_loss_clip": 1.02993786, + "balance_loss_mlp": 1.02122164, + "epoch": 0.143665486622947, + "flos": 15921301536000.0, + "grad_norm": 3.7702166804371116, + "language_loss": 0.79631138, + "learning_rate": 3.8660088501801474e-06, + "loss": 0.8175782, + "num_input_tokens_seen": 141128730, + "router_z_loss_clip": 0.56884766, + "router_z_loss_mlp": 0.18621826, + "step": 4951, + "time_per_iteration": 2.3404057025909424 + }, + { + "auxiliary_loss_clip": 0.01092387, + "auxiliary_loss_mlp": 0.0104061, + "balance_loss_clip": 1.03350711, + "balance_loss_mlp": 1.02233565, + "epoch": 0.14369450409146306, + "flos": 26385469966080.0, + "grad_norm": 2.5752700298550075, + "language_loss": 0.95234418, + "learning_rate": 3.865941201104891e-06, + "loss": 0.97367418, + "num_input_tokens_seen": 141142240, + "router_z_loss_clip": 0.58837891, + "router_z_loss_mlp": 0.18249512, + "step": 4952, + "time_per_iteration": 2.4604246616363525 + }, + { + "auxiliary_loss_clip": 0.01096662, + "auxiliary_loss_mlp": 0.01041149, + "balance_loss_clip": 1.03272557, + "balance_loss_mlp": 1.02051377, + "epoch": 0.1437235215599791, + "flos": 41529291235200.0, + "grad_norm": 2.3460299268707256, + "language_loss": 0.93264616, + "learning_rate": 3.8658735355488875e-06, + "loss": 0.95402426, + "num_input_tokens_seen": 141159210, + "router_z_loss_clip": 0.64038086, + "router_z_loss_mlp": 0.20629883, + "step": 4953, + "time_per_iteration": 2.5646796226501465 + }, + { + "auxiliary_loss_clip": 0.01015174, + "auxiliary_loss_mlp": 0.0100158, + "balance_loss_clip": 1.00645328, + "balance_loss_mlp": 1.00061989, + "epoch": 0.14375253902849516, + "flos": 62501406940800.0, + "grad_norm": 0.7617674840537569, + "language_loss": 0.4888708, + "learning_rate": 3.865805853512733e-06, + "loss": 0.50903833, + "num_input_tokens_seen": 141211385, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.00958252, + "step": 4954, + "time_per_iteration": 5.211331605911255 + }, + { + "auxiliary_loss_clip": 0.01014895, + "auxiliary_loss_mlp": 0.01002353, + "balance_loss_clip": 1.00604582, + "balance_loss_mlp": 1.00141096, + "epoch": 0.1437815564970112, + "flos": 62988654142080.0, + "grad_norm": 0.6929146774728772, + "language_loss": 0.49641913, + "learning_rate": 3.865738154997027e-06, + "loss": 0.51659161, + "num_input_tokens_seen": 141274490, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.00939941, + "step": 4955, + "time_per_iteration": 5.4056556224823 + }, + { + "auxiliary_loss_clip": 0.0109365, + "auxiliary_loss_mlp": 0.01035112, + "balance_loss_clip": 1.03522408, + "balance_loss_mlp": 1.01664627, + "epoch": 0.14381057396552724, + "flos": 42012418896000.0, + "grad_norm": 1.996962634887495, + "language_loss": 0.92354393, + "learning_rate": 3.865670440002366e-06, + "loss": 0.94483161, + "num_input_tokens_seen": 141299290, + "router_z_loss_clip": 0.58398438, + "router_z_loss_mlp": 0.18469238, + "step": 4956, + "time_per_iteration": 2.6333372592926025 + }, + { + "auxiliary_loss_clip": 0.01094049, + "auxiliary_loss_mlp": 0.01046645, + "balance_loss_clip": 1.03304648, + "balance_loss_mlp": 1.02502608, + "epoch": 0.1438395914340433, + "flos": 36090661150080.0, + "grad_norm": 3.0278306692366854, + "language_loss": 0.8164503, + "learning_rate": 3.86560270852935e-06, + "loss": 0.83785725, + "num_input_tokens_seen": 141314325, + "router_z_loss_clip": 0.61035156, + "router_z_loss_mlp": 0.21636963, + "step": 4957, + "time_per_iteration": 2.498667001724243 + }, + { + "auxiliary_loss_clip": 0.01089082, + "auxiliary_loss_mlp": 0.01037525, + "balance_loss_clip": 1.03209198, + "balance_loss_mlp": 1.02038836, + "epoch": 0.14386860890255934, + "flos": 11503112935680.0, + "grad_norm": 3.541040295458522, + "language_loss": 0.76278532, + "learning_rate": 3.865534960578574e-06, + "loss": 0.78405142, + "num_input_tokens_seen": 141325265, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.17132568, + "step": 4958, + "time_per_iteration": 2.368053674697876 + }, + { + "auxiliary_loss_clip": 0.01012212, + "auxiliary_loss_mlp": 0.01005203, + "balance_loss_clip": 1.00328875, + "balance_loss_mlp": 1.00424349, + "epoch": 0.1438976263710754, + "flos": 58531677154560.0, + "grad_norm": 0.7047205009357179, + "language_loss": 0.51142335, + "learning_rate": 3.865467196150639e-06, + "loss": 0.5315975, + "num_input_tokens_seen": 141385290, + "router_z_loss_clip": 0.08935547, + "router_z_loss_mlp": 0.00958252, + "step": 4959, + "time_per_iteration": 3.076901435852051 + }, + { + "auxiliary_loss_clip": 0.01011334, + "auxiliary_loss_mlp": 0.0100127, + "balance_loss_clip": 1.00270689, + "balance_loss_mlp": 1.00026321, + "epoch": 0.14392664383959144, + "flos": 64842263245440.0, + "grad_norm": 0.7936764909267503, + "language_loss": 0.51433837, + "learning_rate": 3.865399415246144e-06, + "loss": 0.53446448, + "num_input_tokens_seen": 141442665, + "router_z_loss_clip": 0.0859375, + "router_z_loss_mlp": 0.0100708, + "step": 4960, + "time_per_iteration": 2.857747793197632 + }, + { + "auxiliary_loss_clip": 0.01086413, + "auxiliary_loss_mlp": 0.01040513, + "balance_loss_clip": 1.02834821, + "balance_loss_mlp": 1.021088, + "epoch": 0.14395566130810747, + "flos": 35727703989120.0, + "grad_norm": 2.682508379853128, + "language_loss": 1.01319695, + "learning_rate": 3.865331617865686e-06, + "loss": 1.03446615, + "num_input_tokens_seen": 141458015, + "router_z_loss_clip": 0.58081055, + "router_z_loss_mlp": 0.1942749, + "step": 4961, + "time_per_iteration": 2.528716802597046 + }, + { + "auxiliary_loss_clip": 0.01014378, + "auxiliary_loss_mlp": 0.01006521, + "balance_loss_clip": 1.00499868, + "balance_loss_mlp": 1.00504291, + "epoch": 0.14398467877662352, + "flos": 74772224962560.0, + "grad_norm": 0.7007283218276275, + "language_loss": 0.50252038, + "learning_rate": 3.865263804009863e-06, + "loss": 0.5227294, + "num_input_tokens_seen": 141520365, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.01477051, + "step": 4962, + "time_per_iteration": 3.030174970626831 + }, + { + "auxiliary_loss_clip": 0.01085143, + "auxiliary_loss_mlp": 0.01038402, + "balance_loss_clip": 1.02826154, + "balance_loss_mlp": 1.02016354, + "epoch": 0.14401369624513957, + "flos": 17232255377280.0, + "grad_norm": 3.8102942102978705, + "language_loss": 0.8373543, + "learning_rate": 3.865195973679277e-06, + "loss": 0.85858977, + "num_input_tokens_seen": 141534935, + "router_z_loss_clip": 0.56835938, + "router_z_loss_mlp": 0.18237305, + "step": 4963, + "time_per_iteration": 2.4093010425567627 + }, + { + "auxiliary_loss_clip": 0.01011432, + "auxiliary_loss_mlp": 0.01003204, + "balance_loss_clip": 1.00271273, + "balance_loss_mlp": 1.0022862, + "epoch": 0.14404271371365562, + "flos": 60856334588160.0, + "grad_norm": 0.6985921867253472, + "language_loss": 0.45812273, + "learning_rate": 3.8651281268745245e-06, + "loss": 0.4782691, + "num_input_tokens_seen": 141586535, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.00915527, + "step": 4964, + "time_per_iteration": 2.884340286254883 + }, + { + "auxiliary_loss_clip": 0.01087677, + "auxiliary_loss_mlp": 0.01040393, + "balance_loss_clip": 1.02865493, + "balance_loss_mlp": 1.0227499, + "epoch": 0.14407173118217168, + "flos": 40950849962880.0, + "grad_norm": 1.656460204659552, + "language_loss": 0.79118007, + "learning_rate": 3.865060263596206e-06, + "loss": 0.81246078, + "num_input_tokens_seen": 141607730, + "router_z_loss_clip": 0.58984375, + "router_z_loss_mlp": 0.17657471, + "step": 4965, + "time_per_iteration": 2.579681396484375 + }, + { + "auxiliary_loss_clip": 0.01091618, + "auxiliary_loss_mlp": 0.0104167, + "balance_loss_clip": 1.03249598, + "balance_loss_mlp": 1.02392006, + "epoch": 0.14410074865068773, + "flos": 35069870551680.0, + "grad_norm": 3.100583761388054, + "language_loss": 0.70477557, + "learning_rate": 3.864992383844921e-06, + "loss": 0.72610843, + "num_input_tokens_seen": 141624060, + "router_z_loss_clip": 0.59130859, + "router_z_loss_mlp": 0.1776123, + "step": 4966, + "time_per_iteration": 2.517404079437256 + }, + { + "auxiliary_loss_clip": 0.01087584, + "auxiliary_loss_mlp": 0.01046419, + "balance_loss_clip": 1.02891397, + "balance_loss_mlp": 1.02987242, + "epoch": 0.14412976611920375, + "flos": 29051507473920.0, + "grad_norm": 2.003576135021638, + "language_loss": 1.029755, + "learning_rate": 3.864924487621268e-06, + "loss": 1.05109501, + "num_input_tokens_seen": 141645265, + "router_z_loss_clip": 0.58740234, + "router_z_loss_mlp": 0.16540527, + "step": 4967, + "time_per_iteration": 2.5567259788513184 + }, + { + "auxiliary_loss_clip": 0.01087245, + "auxiliary_loss_mlp": 0.01044867, + "balance_loss_clip": 1.02882659, + "balance_loss_mlp": 1.02680683, + "epoch": 0.1441587835877198, + "flos": 13800258351360.0, + "grad_norm": 2.483794297545693, + "language_loss": 0.85355604, + "learning_rate": 3.864856574925847e-06, + "loss": 0.87487715, + "num_input_tokens_seen": 141656570, + "router_z_loss_clip": 0.5847168, + "router_z_loss_mlp": 0.18060303, + "step": 4968, + "time_per_iteration": 2.326721668243408 + }, + { + "auxiliary_loss_clip": 0.01086272, + "auxiliary_loss_mlp": 0.0103721, + "balance_loss_clip": 1.02879238, + "balance_loss_mlp": 1.01979351, + "epoch": 0.14418780105623585, + "flos": 14931688648320.0, + "grad_norm": 2.7438165284236424, + "language_loss": 0.92497301, + "learning_rate": 3.864788645759258e-06, + "loss": 0.94620788, + "num_input_tokens_seen": 141670560, + "router_z_loss_clip": 0.57519531, + "router_z_loss_mlp": 0.17407227, + "step": 4969, + "time_per_iteration": 2.3609566688537598 + }, + { + "auxiliary_loss_clip": 0.01086785, + "auxiliary_loss_mlp": 0.01055848, + "balance_loss_clip": 1.02759624, + "balance_loss_mlp": 1.03757906, + "epoch": 0.1442168185247519, + "flos": 27190811364480.0, + "grad_norm": 2.319438693604036, + "language_loss": 0.97204071, + "learning_rate": 3.864720700122101e-06, + "loss": 0.99346697, + "num_input_tokens_seen": 141687495, + "router_z_loss_clip": 0.59277344, + "router_z_loss_mlp": 0.18261719, + "step": 4970, + "time_per_iteration": 2.5091161727905273 + }, + { + "auxiliary_loss_clip": 0.01081772, + "auxiliary_loss_mlp": 0.01035723, + "balance_loss_clip": 1.02709913, + "balance_loss_mlp": 1.01811028, + "epoch": 0.14424583599326796, + "flos": 24600465417600.0, + "grad_norm": 1.9207446017546788, + "language_loss": 0.73058361, + "learning_rate": 3.864652738014977e-06, + "loss": 0.75175858, + "num_input_tokens_seen": 141703160, + "router_z_loss_clip": 0.54736328, + "router_z_loss_mlp": 0.17602539, + "step": 4971, + "time_per_iteration": 2.4525623321533203 + }, + { + "auxiliary_loss_clip": 0.01013158, + "auxiliary_loss_mlp": 0.01018512, + "balance_loss_clip": 1.00457311, + "balance_loss_mlp": 1.01758242, + "epoch": 0.14427485346178398, + "flos": 63239889352320.0, + "grad_norm": 0.6688533513989173, + "language_loss": 0.44632593, + "learning_rate": 3.864584759438485e-06, + "loss": 0.46664265, + "num_input_tokens_seen": 141760610, + "router_z_loss_clip": 0.0859375, + "router_z_loss_mlp": 0.00927734, + "step": 4972, + "time_per_iteration": 2.996342897415161 + }, + { + "auxiliary_loss_clip": 0.01079908, + "auxiliary_loss_mlp": 0.01037635, + "balance_loss_clip": 1.02740049, + "balance_loss_mlp": 1.02119565, + "epoch": 0.14430387093030003, + "flos": 20114579957760.0, + "grad_norm": 1.9759396233131976, + "language_loss": 0.74127692, + "learning_rate": 3.864516764393226e-06, + "loss": 0.7624523, + "num_input_tokens_seen": 141774080, + "router_z_loss_clip": 0.52587891, + "router_z_loss_mlp": 0.16436768, + "step": 4973, + "time_per_iteration": 2.380265235900879 + }, + { + "auxiliary_loss_clip": 0.01084754, + "auxiliary_loss_mlp": 0.01038933, + "balance_loss_clip": 1.02793217, + "balance_loss_mlp": 1.02127171, + "epoch": 0.14433288839881608, + "flos": 16171489405440.0, + "grad_norm": 2.8013870778326377, + "language_loss": 0.82272398, + "learning_rate": 3.864448752879801e-06, + "loss": 0.84396082, + "num_input_tokens_seen": 141787155, + "router_z_loss_clip": 0.56835938, + "router_z_loss_mlp": 0.17663574, + "step": 4974, + "time_per_iteration": 2.342679977416992 + }, + { + "auxiliary_loss_clip": 0.01087928, + "auxiliary_loss_mlp": 0.01041896, + "balance_loss_clip": 1.03017759, + "balance_loss_mlp": 1.02401495, + "epoch": 0.14436190586733214, + "flos": 31690207520640.0, + "grad_norm": 2.481357285825206, + "language_loss": 0.93590105, + "learning_rate": 3.864380724898809e-06, + "loss": 0.95719928, + "num_input_tokens_seen": 141805525, + "router_z_loss_clip": 0.57739258, + "router_z_loss_mlp": 0.17883301, + "step": 4975, + "time_per_iteration": 2.536362409591675 + }, + { + "auxiliary_loss_clip": 0.01088529, + "auxiliary_loss_mlp": 0.01035577, + "balance_loss_clip": 1.03061974, + "balance_loss_mlp": 1.01714754, + "epoch": 0.1443909233358482, + "flos": 28175397016320.0, + "grad_norm": 2.4169264882707786, + "language_loss": 0.72750419, + "learning_rate": 3.864312680450853e-06, + "loss": 0.74874532, + "num_input_tokens_seen": 141821925, + "router_z_loss_clip": 0.57910156, + "router_z_loss_mlp": 0.18408203, + "step": 4976, + "time_per_iteration": 2.4235117435455322 + }, + { + "auxiliary_loss_clip": 0.01019174, + "auxiliary_loss_mlp": 0.01002415, + "balance_loss_clip": 1.01008439, + "balance_loss_mlp": 1.00155663, + "epoch": 0.14441994080436424, + "flos": 74773900707840.0, + "grad_norm": 0.7264727808879762, + "language_loss": 0.4982509, + "learning_rate": 3.864244619536532e-06, + "loss": 0.51846683, + "num_input_tokens_seen": 141887380, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.00860596, + "step": 4977, + "time_per_iteration": 3.0985846519470215 + }, + { + "auxiliary_loss_clip": 0.01089533, + "auxiliary_loss_mlp": 0.01037676, + "balance_loss_clip": 1.03005981, + "balance_loss_mlp": 1.01935911, + "epoch": 0.14444895827288026, + "flos": 37003430401920.0, + "grad_norm": 2.5821407987520706, + "language_loss": 0.90833485, + "learning_rate": 3.8641765421564496e-06, + "loss": 0.92960691, + "num_input_tokens_seen": 141901820, + "router_z_loss_clip": 0.5949707, + "router_z_loss_mlp": 0.1831665, + "step": 4978, + "time_per_iteration": 2.52079439163208 + }, + { + "auxiliary_loss_clip": 0.01019369, + "auxiliary_loss_mlp": 0.00999985, + "balance_loss_clip": 1.01020479, + "balance_loss_mlp": 0.99912107, + "epoch": 0.14447797574139631, + "flos": 65170446825600.0, + "grad_norm": 0.7495845589850967, + "language_loss": 0.50673234, + "learning_rate": 3.864108448311205e-06, + "loss": 0.52692592, + "num_input_tokens_seen": 141961100, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.00866699, + "step": 4979, + "time_per_iteration": 2.9168612957000732 + }, + { + "auxiliary_loss_clip": 0.01091513, + "auxiliary_loss_mlp": 0.01040925, + "balance_loss_clip": 1.02855992, + "balance_loss_mlp": 1.02129078, + "epoch": 0.14450699320991237, + "flos": 15880348644480.0, + "grad_norm": 3.0870381115555583, + "language_loss": 0.93119735, + "learning_rate": 3.8640403380013995e-06, + "loss": 0.95252168, + "num_input_tokens_seen": 141973525, + "router_z_loss_clip": 0.62890625, + "router_z_loss_mlp": 0.1965332, + "step": 4980, + "time_per_iteration": 2.3599116802215576 + }, + { + "auxiliary_loss_clip": 0.01086533, + "auxiliary_loss_mlp": 0.0103717, + "balance_loss_clip": 1.03217459, + "balance_loss_mlp": 1.01853728, + "epoch": 0.14453601067842842, + "flos": 25043443148160.0, + "grad_norm": 1.873876998360615, + "language_loss": 0.68720365, + "learning_rate": 3.863972211227636e-06, + "loss": 0.70844066, + "num_input_tokens_seen": 141989935, + "router_z_loss_clip": 0.54345703, + "router_z_loss_mlp": 0.18640137, + "step": 4981, + "time_per_iteration": 2.4198877811431885 + }, + { + "auxiliary_loss_clip": 0.01021702, + "auxiliary_loss_mlp": 0.01014235, + "balance_loss_clip": 1.01243806, + "balance_loss_mlp": 1.01326311, + "epoch": 0.14456502814694447, + "flos": 64259108939520.0, + "grad_norm": 0.6728555904119881, + "language_loss": 0.51562506, + "learning_rate": 3.863904067990516e-06, + "loss": 0.5359844, + "num_input_tokens_seen": 142052330, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.00970459, + "step": 4982, + "time_per_iteration": 3.1082515716552734 + }, + { + "auxiliary_loss_clip": 0.01094485, + "auxiliary_loss_mlp": 0.01049126, + "balance_loss_clip": 1.03215325, + "balance_loss_mlp": 1.02967727, + "epoch": 0.14459404561546052, + "flos": 31714123668480.0, + "grad_norm": 2.604777688519566, + "language_loss": 0.83885336, + "learning_rate": 3.86383590829064e-06, + "loss": 0.86028945, + "num_input_tokens_seen": 142067210, + "router_z_loss_clip": 0.62353516, + "router_z_loss_mlp": 0.19451904, + "step": 4983, + "time_per_iteration": 2.4477365016937256 + }, + { + "auxiliary_loss_clip": 0.0108049, + "auxiliary_loss_mlp": 0.01040501, + "balance_loss_clip": 1.0261507, + "balance_loss_mlp": 1.02481318, + "epoch": 0.14462306308397654, + "flos": 48023729879040.0, + "grad_norm": 2.5925223804053155, + "language_loss": 0.72001034, + "learning_rate": 3.863767732128612e-06, + "loss": 0.74122024, + "num_input_tokens_seen": 142084600, + "router_z_loss_clip": 0.54467773, + "router_z_loss_mlp": 0.15686035, + "step": 4984, + "time_per_iteration": 2.5489532947540283 + }, + { + "auxiliary_loss_clip": 0.01016004, + "auxiliary_loss_mlp": 0.01020251, + "balance_loss_clip": 1.0069319, + "balance_loss_mlp": 1.01933336, + "epoch": 0.1446520805524926, + "flos": 74076231542400.0, + "grad_norm": 0.6410289603263004, + "language_loss": 0.49229571, + "learning_rate": 3.863699539505033e-06, + "loss": 0.51265824, + "num_input_tokens_seen": 142150715, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.00915527, + "step": 4985, + "time_per_iteration": 3.22477388381958 + }, + { + "auxiliary_loss_clip": 0.01087919, + "auxiliary_loss_mlp": 0.01041363, + "balance_loss_clip": 1.02954721, + "balance_loss_mlp": 1.02303505, + "epoch": 0.14468109802100865, + "flos": 19383708222720.0, + "grad_norm": 2.7544541938465126, + "language_loss": 0.78180635, + "learning_rate": 3.863631330420505e-06, + "loss": 0.80309927, + "num_input_tokens_seen": 142164305, + "router_z_loss_clip": 0.58398438, + "router_z_loss_mlp": 0.18328857, + "step": 4986, + "time_per_iteration": 2.367062568664551 + }, + { + "auxiliary_loss_clip": 0.01090422, + "auxiliary_loss_mlp": 0.01051849, + "balance_loss_clip": 1.02915776, + "balance_loss_mlp": 1.03344297, + "epoch": 0.1447101154895247, + "flos": 27883523116800.0, + "grad_norm": 2.5743541018704255, + "language_loss": 1.15838158, + "learning_rate": 3.863563104875632e-06, + "loss": 1.17980433, + "num_input_tokens_seen": 142184100, + "router_z_loss_clip": 0.61303711, + "router_z_loss_mlp": 0.18395996, + "step": 4987, + "time_per_iteration": 2.426362991333008 + }, + { + "auxiliary_loss_clip": 0.01090844, + "auxiliary_loss_mlp": 0.01050339, + "balance_loss_clip": 1.02947402, + "balance_loss_mlp": 1.02853584, + "epoch": 0.14473913295804075, + "flos": 14093109768960.0, + "grad_norm": 2.4599728067700073, + "language_loss": 0.92937338, + "learning_rate": 3.863494862871015e-06, + "loss": 0.95078522, + "num_input_tokens_seen": 142195920, + "router_z_loss_clip": 0.61425781, + "router_z_loss_mlp": 0.21813965, + "step": 4988, + "time_per_iteration": 2.3811991214752197 + }, + { + "auxiliary_loss_clip": 0.0108986, + "auxiliary_loss_mlp": 0.01040543, + "balance_loss_clip": 1.03163981, + "balance_loss_mlp": 1.02233934, + "epoch": 0.14476815042655677, + "flos": 19236619198080.0, + "grad_norm": 2.956756520833106, + "language_loss": 1.02613223, + "learning_rate": 3.863426604407257e-06, + "loss": 1.04743624, + "num_input_tokens_seen": 142209085, + "router_z_loss_clip": 0.58203125, + "router_z_loss_mlp": 0.18225098, + "step": 4989, + "time_per_iteration": 2.3600285053253174 + }, + { + "auxiliary_loss_clip": 0.01102272, + "auxiliary_loss_mlp": 0.01041912, + "balance_loss_clip": 1.03483963, + "balance_loss_mlp": 1.01826108, + "epoch": 0.14479716789507283, + "flos": 46782288288000.0, + "grad_norm": 2.401922901167095, + "language_loss": 1.1649524, + "learning_rate": 3.863358329484961e-06, + "loss": 1.18639421, + "num_input_tokens_seen": 142229605, + "router_z_loss_clip": 0.67480469, + "router_z_loss_mlp": 0.23632812, + "step": 4990, + "time_per_iteration": 2.690631866455078 + }, + { + "auxiliary_loss_clip": 0.01096278, + "auxiliary_loss_mlp": 0.01034162, + "balance_loss_clip": 1.03341079, + "balance_loss_mlp": 1.01418221, + "epoch": 0.14482618536358888, + "flos": 19056397426560.0, + "grad_norm": 2.472750603532757, + "language_loss": 0.89865351, + "learning_rate": 3.863290038104731e-06, + "loss": 0.91995794, + "num_input_tokens_seen": 142245485, + "router_z_loss_clip": 0.62890625, + "router_z_loss_mlp": 0.1998291, + "step": 4991, + "time_per_iteration": 2.3833916187286377 + }, + { + "auxiliary_loss_clip": 0.01098337, + "auxiliary_loss_mlp": 0.01036531, + "balance_loss_clip": 1.0345149, + "balance_loss_mlp": 1.01563418, + "epoch": 0.14485520283210493, + "flos": 30437245180800.0, + "grad_norm": 2.0265773966944587, + "language_loss": 0.76803619, + "learning_rate": 3.863221730267169e-06, + "loss": 0.78938484, + "num_input_tokens_seen": 142263885, + "router_z_loss_clip": 0.63769531, + "router_z_loss_mlp": 0.20898438, + "step": 4992, + "time_per_iteration": 2.5184648036956787 + }, + { + "auxiliary_loss_clip": 0.01090394, + "auxiliary_loss_mlp": 0.01031189, + "balance_loss_clip": 1.03431773, + "balance_loss_mlp": 1.01274753, + "epoch": 0.14488422030062098, + "flos": 36203395530240.0, + "grad_norm": 1.9589021152268706, + "language_loss": 0.73839259, + "learning_rate": 3.863153405972879e-06, + "loss": 0.75960839, + "num_input_tokens_seen": 142283255, + "router_z_loss_clip": 0.56054688, + "router_z_loss_mlp": 0.18444824, + "step": 4993, + "time_per_iteration": 2.567739486694336 + }, + { + "auxiliary_loss_clip": 0.01028521, + "auxiliary_loss_mlp": 0.01003219, + "balance_loss_clip": 1.01955509, + "balance_loss_mlp": 1.00227118, + "epoch": 0.14491323776913703, + "flos": 74772329696640.0, + "grad_norm": 0.7120335117747487, + "language_loss": 0.49620861, + "learning_rate": 3.863085065222464e-06, + "loss": 0.51652598, + "num_input_tokens_seen": 142343915, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.00946045, + "step": 4994, + "time_per_iteration": 3.1504695415496826 + }, + { + "auxiliary_loss_clip": 0.01094767, + "auxiliary_loss_mlp": 0.01045602, + "balance_loss_clip": 1.03629959, + "balance_loss_mlp": 1.02722585, + "epoch": 0.14494225523765306, + "flos": 26169392361600.0, + "grad_norm": 3.0376377092031825, + "language_loss": 0.92274594, + "learning_rate": 3.863016708016527e-06, + "loss": 0.94414967, + "num_input_tokens_seen": 142359890, + "router_z_loss_clip": 0.58496094, + "router_z_loss_mlp": 0.18389893, + "step": 4995, + "time_per_iteration": 2.4378268718719482 + }, + { + "auxiliary_loss_clip": 0.01025751, + "auxiliary_loss_mlp": 0.0099995, + "balance_loss_clip": 1.01620722, + "balance_loss_mlp": 0.99893105, + "epoch": 0.1449712727061691, + "flos": 74777566400640.0, + "grad_norm": 0.6789114516699285, + "language_loss": 0.46496004, + "learning_rate": 3.8629483343556735e-06, + "loss": 0.48521703, + "num_input_tokens_seen": 142426770, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01019287, + "step": 4996, + "time_per_iteration": 3.1019082069396973 + }, + { + "auxiliary_loss_clip": 0.01024009, + "auxiliary_loss_mlp": 0.01000998, + "balance_loss_clip": 1.0142529, + "balance_loss_mlp": 1.00000834, + "epoch": 0.14500029017468516, + "flos": 57440257142400.0, + "grad_norm": 0.6036801435221141, + "language_loss": 0.47179282, + "learning_rate": 3.862879944240506e-06, + "loss": 0.49204284, + "num_input_tokens_seen": 142488090, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.0098877, + "step": 4997, + "time_per_iteration": 3.0246055126190186 + }, + { + "auxiliary_loss_clip": 0.01091505, + "auxiliary_loss_mlp": 0.01041087, + "balance_loss_clip": 1.03339124, + "balance_loss_mlp": 1.02320528, + "epoch": 0.1450293076432012, + "flos": 30842656421760.0, + "grad_norm": 1.9000634027443406, + "language_loss": 0.80225897, + "learning_rate": 3.86281153767163e-06, + "loss": 0.82358485, + "num_input_tokens_seen": 142506990, + "router_z_loss_clip": 0.58178711, + "router_z_loss_mlp": 0.17883301, + "step": 4998, + "time_per_iteration": 2.5888383388519287 + }, + { + "auxiliary_loss_clip": 0.01094065, + "auxiliary_loss_mlp": 0.0104041, + "balance_loss_clip": 1.03318715, + "balance_loss_mlp": 1.02087116, + "epoch": 0.14505832511171726, + "flos": 16977214828800.0, + "grad_norm": 1.9932883767489056, + "language_loss": 0.73870468, + "learning_rate": 3.862743114649647e-06, + "loss": 0.76004946, + "num_input_tokens_seen": 142522395, + "router_z_loss_clip": 0.609375, + "router_z_loss_mlp": 0.19525146, + "step": 4999, + "time_per_iteration": 2.5696051120758057 + }, + { + "auxiliary_loss_clip": 0.01105107, + "auxiliary_loss_mlp": 0.01044725, + "balance_loss_clip": 1.03354979, + "balance_loss_mlp": 1.02003634, + "epoch": 0.1450873425802333, + "flos": 15045575103360.0, + "grad_norm": 2.6063239564914205, + "language_loss": 0.84108615, + "learning_rate": 3.862674675175164e-06, + "loss": 0.86258447, + "num_input_tokens_seen": 142534995, + "router_z_loss_clip": 0.71582031, + "router_z_loss_mlp": 0.24707031, + "step": 5000, + "time_per_iteration": 2.345125198364258 + }, + { + "auxiliary_loss_clip": 0.01101386, + "auxiliary_loss_mlp": 0.01044447, + "balance_loss_clip": 1.03553665, + "balance_loss_mlp": 1.02242911, + "epoch": 0.14511636004874934, + "flos": 74731483405440.0, + "grad_norm": 2.3768640321122034, + "language_loss": 0.91384858, + "learning_rate": 3.8626062192487845e-06, + "loss": 0.93530685, + "num_input_tokens_seen": 142559210, + "router_z_loss_clip": 0.65820312, + "router_z_loss_mlp": 0.22003174, + "step": 5001, + "time_per_iteration": 2.820756435394287 + }, + { + "auxiliary_loss_clip": 0.01022429, + "auxiliary_loss_mlp": 0.01011722, + "balance_loss_clip": 1.01236868, + "balance_loss_mlp": 1.0108155, + "epoch": 0.1451453775172654, + "flos": 74489602573440.0, + "grad_norm": 0.6060421933687732, + "language_loss": 0.44839635, + "learning_rate": 3.862537746871113e-06, + "loss": 0.46873787, + "num_input_tokens_seen": 142628375, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.0090332, + "step": 5002, + "time_per_iteration": 3.1334433555603027 + }, + { + "auxiliary_loss_clip": 0.01102457, + "auxiliary_loss_mlp": 0.01051029, + "balance_loss_clip": 1.03499079, + "balance_loss_mlp": 1.02939224, + "epoch": 0.14517439498578144, + "flos": 28067969162880.0, + "grad_norm": 2.994388958402436, + "language_loss": 0.97132564, + "learning_rate": 3.862469258042755e-06, + "loss": 0.99286056, + "num_input_tokens_seen": 142645535, + "router_z_loss_clip": 0.67480469, + "router_z_loss_mlp": 0.21630859, + "step": 5003, + "time_per_iteration": 2.464160919189453 + }, + { + "auxiliary_loss_clip": 0.01020273, + "auxiliary_loss_mlp": 0.01000985, + "balance_loss_clip": 1.01116085, + "balance_loss_mlp": 1.00009108, + "epoch": 0.1452034124542975, + "flos": 56927383136640.0, + "grad_norm": 0.7251157130742835, + "language_loss": 0.50508517, + "learning_rate": 3.862400752764314e-06, + "loss": 0.52529776, + "num_input_tokens_seen": 142704175, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.00891113, + "step": 5004, + "time_per_iteration": 2.9535140991210938 + }, + { + "auxiliary_loss_clip": 0.01017593, + "auxiliary_loss_mlp": 0.01000905, + "balance_loss_clip": 1.00875783, + "balance_loss_mlp": 0.99988014, + "epoch": 0.14523242992281354, + "flos": 66416845829760.0, + "grad_norm": 0.6769051018773515, + "language_loss": 0.50934768, + "learning_rate": 3.862332231036396e-06, + "loss": 0.52953267, + "num_input_tokens_seen": 142769150, + "router_z_loss_clip": 0.08837891, + "router_z_loss_mlp": 0.01025391, + "step": 5005, + "time_per_iteration": 3.048841953277588 + }, + { + "auxiliary_loss_clip": 0.01096115, + "auxiliary_loss_mlp": 0.01039907, + "balance_loss_clip": 1.03112268, + "balance_loss_mlp": 1.01918828, + "epoch": 0.14526144739132957, + "flos": 12267222151680.0, + "grad_norm": 2.8224083322672646, + "language_loss": 0.87836272, + "learning_rate": 3.862263692859607e-06, + "loss": 0.89972293, + "num_input_tokens_seen": 142783330, + "router_z_loss_clip": 0.65014648, + "router_z_loss_mlp": 0.20727539, + "step": 5006, + "time_per_iteration": 2.328031063079834 + }, + { + "auxiliary_loss_clip": 0.01091777, + "auxiliary_loss_mlp": 0.01041376, + "balance_loss_clip": 1.02930021, + "balance_loss_mlp": 1.02156305, + "epoch": 0.14529046485984562, + "flos": 22118594664960.0, + "grad_norm": 2.172034937029469, + "language_loss": 0.85544264, + "learning_rate": 3.86219513823455e-06, + "loss": 0.87677419, + "num_input_tokens_seen": 142798345, + "router_z_loss_clip": 0.62451172, + "router_z_loss_mlp": 0.19799805, + "step": 5007, + "time_per_iteration": 2.3902359008789062 + }, + { + "auxiliary_loss_clip": 0.01087526, + "auxiliary_loss_mlp": 0.01041862, + "balance_loss_clip": 1.03006339, + "balance_loss_mlp": 1.02308619, + "epoch": 0.14531948232836167, + "flos": 16027612225920.0, + "grad_norm": 6.310742643557623, + "language_loss": 0.82147294, + "learning_rate": 3.8621265671618334e-06, + "loss": 0.84276682, + "num_input_tokens_seen": 142811095, + "router_z_loss_clip": 0.57543945, + "router_z_loss_mlp": 0.18774414, + "step": 5008, + "time_per_iteration": 2.345332384109497 + }, + { + "auxiliary_loss_clip": 0.01091595, + "auxiliary_loss_mlp": 0.01043025, + "balance_loss_clip": 1.03296018, + "balance_loss_mlp": 1.02395129, + "epoch": 0.14534849979687772, + "flos": 20768573145600.0, + "grad_norm": 2.2219522475184927, + "language_loss": 0.74311602, + "learning_rate": 3.862057979642061e-06, + "loss": 0.76446223, + "num_input_tokens_seen": 142823575, + "router_z_loss_clip": 0.58642578, + "router_z_loss_mlp": 0.19055176, + "step": 5009, + "time_per_iteration": 2.3916726112365723 + }, + { + "auxiliary_loss_clip": 0.01094799, + "auxiliary_loss_mlp": 0.01040155, + "balance_loss_clip": 1.03188431, + "balance_loss_mlp": 1.0191381, + "epoch": 0.14537751726539377, + "flos": 48753449539200.0, + "grad_norm": 2.4111791084145944, + "language_loss": 0.84405071, + "learning_rate": 3.861989375675839e-06, + "loss": 0.8654002, + "num_input_tokens_seen": 142842025, + "router_z_loss_clip": 0.62988281, + "router_z_loss_mlp": 0.21014404, + "step": 5010, + "time_per_iteration": 2.644834518432617 + }, + { + "auxiliary_loss_clip": 0.01017467, + "auxiliary_loss_mlp": 0.01007384, + "balance_loss_clip": 1.00822639, + "balance_loss_mlp": 1.00641251, + "epoch": 0.14540653473390983, + "flos": 69653469553920.0, + "grad_norm": 0.748956408089507, + "language_loss": 0.50268441, + "learning_rate": 3.861920755263774e-06, + "loss": 0.52293295, + "num_input_tokens_seen": 142898880, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.00970459, + "step": 5011, + "time_per_iteration": 2.979703187942505 + }, + { + "auxiliary_loss_clip": 0.01095594, + "auxiliary_loss_mlp": 0.01038463, + "balance_loss_clip": 1.03372979, + "balance_loss_mlp": 1.0195682, + "epoch": 0.14543555220242585, + "flos": 26500194293760.0, + "grad_norm": 1.9359406331501237, + "language_loss": 0.79920971, + "learning_rate": 3.86185211840647e-06, + "loss": 0.82055026, + "num_input_tokens_seen": 142920875, + "router_z_loss_clip": 0.61938477, + "router_z_loss_mlp": 0.18908691, + "step": 5012, + "time_per_iteration": 2.6759185791015625 + }, + { + "auxiliary_loss_clip": 0.01098855, + "auxiliary_loss_mlp": 0.01042031, + "balance_loss_clip": 1.03374445, + "balance_loss_mlp": 1.02073979, + "epoch": 0.1454645696709419, + "flos": 50321782990080.0, + "grad_norm": 1.9976344013884062, + "language_loss": 0.88990283, + "learning_rate": 3.861783465104536e-06, + "loss": 0.91131169, + "num_input_tokens_seen": 142939925, + "router_z_loss_clip": 0.65087891, + "router_z_loss_mlp": 0.21289062, + "step": 5013, + "time_per_iteration": 2.561481237411499 + }, + { + "auxiliary_loss_clip": 0.0110037, + "auxiliary_loss_mlp": 0.01041884, + "balance_loss_clip": 1.03698444, + "balance_loss_mlp": 1.02326322, + "epoch": 0.14549358713945795, + "flos": 33684097933440.0, + "grad_norm": 2.585643725182761, + "language_loss": 0.81614447, + "learning_rate": 3.861714795358576e-06, + "loss": 0.83756709, + "num_input_tokens_seen": 142954940, + "router_z_loss_clip": 0.63378906, + "router_z_loss_mlp": 0.18640137, + "step": 5014, + "time_per_iteration": 2.550337314605713 + }, + { + "auxiliary_loss_clip": 0.01093244, + "auxiliary_loss_mlp": 0.01033839, + "balance_loss_clip": 1.03585124, + "balance_loss_mlp": 1.01515913, + "epoch": 0.145522604607974, + "flos": 40762563667200.0, + "grad_norm": 3.42977470745067, + "language_loss": 0.88476998, + "learning_rate": 3.861646109169198e-06, + "loss": 0.90604079, + "num_input_tokens_seen": 142973255, + "router_z_loss_clip": 0.57373047, + "router_z_loss_mlp": 0.18682861, + "step": 5015, + "time_per_iteration": 2.592332124710083 + }, + { + "auxiliary_loss_clip": 0.01026884, + "auxiliary_loss_mlp": 0.01000933, + "balance_loss_clip": 1.016536, + "balance_loss_mlp": 1.00003302, + "epoch": 0.14555162207649006, + "flos": 74780114929920.0, + "grad_norm": 0.6422895162808686, + "language_loss": 0.50340664, + "learning_rate": 3.861577406537009e-06, + "loss": 0.52368474, + "num_input_tokens_seen": 143038680, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.00897217, + "step": 5016, + "time_per_iteration": 3.1224770545959473 + }, + { + "auxiliary_loss_clip": 0.01098223, + "auxiliary_loss_mlp": 0.01042052, + "balance_loss_clip": 1.03839231, + "balance_loss_mlp": 1.02321696, + "epoch": 0.1455806395450061, + "flos": 30364311617280.0, + "grad_norm": 2.7875605437829627, + "language_loss": 0.99411637, + "learning_rate": 3.861508687462615e-06, + "loss": 1.01551914, + "num_input_tokens_seen": 143055410, + "router_z_loss_clip": 0.59814453, + "router_z_loss_mlp": 0.18835449, + "step": 5017, + "time_per_iteration": 2.498842716217041 + }, + { + "auxiliary_loss_clip": 0.01099553, + "auxiliary_loss_mlp": 0.01030349, + "balance_loss_clip": 1.03978598, + "balance_loss_mlp": 1.01238418, + "epoch": 0.14560965701352213, + "flos": 25987913781120.0, + "grad_norm": 1.8407562086494835, + "language_loss": 0.77397263, + "learning_rate": 3.861439951946622e-06, + "loss": 0.79527164, + "num_input_tokens_seen": 143071390, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.17980957, + "step": 5018, + "time_per_iteration": 2.4779577255249023 + }, + { + "auxiliary_loss_clip": 0.01032142, + "auxiliary_loss_mlp": 0.01005634, + "balance_loss_clip": 1.02148294, + "balance_loss_mlp": 1.00469816, + "epoch": 0.14563867448203818, + "flos": 61787467215360.0, + "grad_norm": 0.6026778783980848, + "language_loss": 0.4912917, + "learning_rate": 3.861371199989638e-06, + "loss": 0.51166946, + "num_input_tokens_seen": 143138490, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.00933838, + "step": 5019, + "time_per_iteration": 3.1677303314208984 + }, + { + "auxiliary_loss_clip": 0.01100414, + "auxiliary_loss_mlp": 0.01040844, + "balance_loss_clip": 1.036201, + "balance_loss_mlp": 1.02104354, + "epoch": 0.14566769195055423, + "flos": 19929226216320.0, + "grad_norm": 2.7334323222272707, + "language_loss": 0.90664798, + "learning_rate": 3.861302431592271e-06, + "loss": 0.92806053, + "num_input_tokens_seen": 143151160, + "router_z_loss_clip": 0.64160156, + "router_z_loss_mlp": 0.19787598, + "step": 5020, + "time_per_iteration": 2.437685012817383 + }, + { + "auxiliary_loss_clip": 0.01102823, + "auxiliary_loss_mlp": 0.01049092, + "balance_loss_clip": 1.03698337, + "balance_loss_mlp": 1.03000665, + "epoch": 0.14569670941907029, + "flos": 30983705781120.0, + "grad_norm": 2.419067219253819, + "language_loss": 0.8068921, + "learning_rate": 3.861233646755127e-06, + "loss": 0.82841122, + "num_input_tokens_seen": 143175920, + "router_z_loss_clip": 0.65820312, + "router_z_loss_mlp": 0.19067383, + "step": 5021, + "time_per_iteration": 2.7794156074523926 + }, + { + "auxiliary_loss_clip": 0.01028733, + "auxiliary_loss_mlp": 0.01006375, + "balance_loss_clip": 1.01839209, + "balance_loss_mlp": 1.00546288, + "epoch": 0.14572572688758634, + "flos": 74766604233600.0, + "grad_norm": 0.6682315307603345, + "language_loss": 0.5305813, + "learning_rate": 3.861164845478815e-06, + "loss": 0.55093241, + "num_input_tokens_seen": 143238255, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.00909424, + "step": 5022, + "time_per_iteration": 5.285508871078491 + }, + { + "auxiliary_loss_clip": 0.01085687, + "auxiliary_loss_mlp": 0.01034618, + "balance_loss_clip": 1.03218436, + "balance_loss_mlp": 1.01900721, + "epoch": 0.14575474435610236, + "flos": 28325209127040.0, + "grad_norm": 2.616791070111907, + "language_loss": 0.68416834, + "learning_rate": 3.861096027763942e-06, + "loss": 0.70537138, + "num_input_tokens_seen": 143252165, + "router_z_loss_clip": 0.53417969, + "router_z_loss_mlp": 0.15612793, + "step": 5023, + "time_per_iteration": 2.3701910972595215 + }, + { + "auxiliary_loss_clip": 0.01086309, + "auxiliary_loss_mlp": 0.01036951, + "balance_loss_clip": 1.02970934, + "balance_loss_mlp": 1.01964808, + "epoch": 0.1457837618246184, + "flos": 26753000515200.0, + "grad_norm": 2.1472658907672364, + "language_loss": 0.8909806, + "learning_rate": 3.8610271936111155e-06, + "loss": 0.91221321, + "num_input_tokens_seen": 143267320, + "router_z_loss_clip": 0.56616211, + "router_z_loss_mlp": 0.1730957, + "step": 5024, + "time_per_iteration": 2.6076242923736572 + }, + { + "auxiliary_loss_clip": 0.01093163, + "auxiliary_loss_mlp": 0.0104257, + "balance_loss_clip": 1.03274274, + "balance_loss_mlp": 1.02361, + "epoch": 0.14581277929313446, + "flos": 45067703685120.0, + "grad_norm": 2.1408806602942945, + "language_loss": 0.78415269, + "learning_rate": 3.860958343020944e-06, + "loss": 0.80550998, + "num_input_tokens_seen": 143286695, + "router_z_loss_clip": 0.60498047, + "router_z_loss_mlp": 0.18963623, + "step": 5025, + "time_per_iteration": 4.705872535705566 + }, + { + "auxiliary_loss_clip": 0.01092988, + "auxiliary_loss_mlp": 0.01043155, + "balance_loss_clip": 1.032197, + "balance_loss_mlp": 1.02304399, + "epoch": 0.14584179676165052, + "flos": 16245924157440.0, + "grad_norm": 2.4925767169248605, + "language_loss": 0.7758249, + "learning_rate": 3.860889475994035e-06, + "loss": 0.79718632, + "num_input_tokens_seen": 143299190, + "router_z_loss_clip": 0.60742188, + "router_z_loss_mlp": 0.2010498, + "step": 5026, + "time_per_iteration": 2.3565566539764404 + }, + { + "auxiliary_loss_clip": 0.01023216, + "auxiliary_loss_mlp": 0.01016325, + "balance_loss_clip": 1.01352799, + "balance_loss_mlp": 1.0154314, + "epoch": 0.14587081423016657, + "flos": 48866705723520.0, + "grad_norm": 0.7436209906798095, + "language_loss": 0.47991872, + "learning_rate": 3.860820592530997e-06, + "loss": 0.50031412, + "num_input_tokens_seen": 143345855, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.00891113, + "step": 5027, + "time_per_iteration": 2.7659173011779785 + }, + { + "auxiliary_loss_clip": 0.01021091, + "auxiliary_loss_mlp": 0.01008993, + "balance_loss_clip": 1.01182628, + "balance_loss_mlp": 1.00806332, + "epoch": 0.14589983169868262, + "flos": 69046294366080.0, + "grad_norm": 0.6435901108558448, + "language_loss": 0.46756932, + "learning_rate": 3.860751692632439e-06, + "loss": 0.48787019, + "num_input_tokens_seen": 143406805, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.00927734, + "step": 5028, + "time_per_iteration": 2.9969699382781982 + }, + { + "auxiliary_loss_clip": 0.01091244, + "auxiliary_loss_mlp": 0.01042427, + "balance_loss_clip": 1.02912569, + "balance_loss_mlp": 1.02243543, + "epoch": 0.14592884916719864, + "flos": 33393620488320.0, + "grad_norm": 2.312713076623502, + "language_loss": 0.83500308, + "learning_rate": 3.860682776298968e-06, + "loss": 0.85633975, + "num_input_tokens_seen": 143422145, + "router_z_loss_clip": 0.62060547, + "router_z_loss_mlp": 0.20007324, + "step": 5029, + "time_per_iteration": 5.01012396812439 + }, + { + "auxiliary_loss_clip": 0.01093974, + "auxiliary_loss_mlp": 0.01043246, + "balance_loss_clip": 1.030707, + "balance_loss_mlp": 1.02200329, + "epoch": 0.1459578666357147, + "flos": 29235220381440.0, + "grad_norm": 2.3138698596028804, + "language_loss": 0.79961729, + "learning_rate": 3.860613843531196e-06, + "loss": 0.82098949, + "num_input_tokens_seen": 143441885, + "router_z_loss_clip": 0.63183594, + "router_z_loss_mlp": 0.2121582, + "step": 5030, + "time_per_iteration": 2.46859073638916 + }, + { + "auxiliary_loss_clip": 0.01090888, + "auxiliary_loss_mlp": 0.01041241, + "balance_loss_clip": 1.03003216, + "balance_loss_mlp": 1.02250147, + "epoch": 0.14598688410423075, + "flos": 14276613208320.0, + "grad_norm": 2.5488306331450623, + "language_loss": 0.63439971, + "learning_rate": 3.860544894329728e-06, + "loss": 0.65572095, + "num_input_tokens_seen": 143454655, + "router_z_loss_clip": 0.60791016, + "router_z_loss_mlp": 0.18762207, + "step": 5031, + "time_per_iteration": 4.96521782875061 + }, + { + "auxiliary_loss_clip": 0.01019119, + "auxiliary_loss_mlp": 0.01003058, + "balance_loss_clip": 1.01025486, + "balance_loss_mlp": 1.00208092, + "epoch": 0.1460159015727468, + "flos": 64626604577280.0, + "grad_norm": 0.6556011079287822, + "language_loss": 0.49312574, + "learning_rate": 3.860475928695175e-06, + "loss": 0.51334757, + "num_input_tokens_seen": 143517825, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.00976562, + "step": 5032, + "time_per_iteration": 3.016710042953491 + }, + { + "auxiliary_loss_clip": 0.01105142, + "auxiliary_loss_mlp": 0.01040416, + "balance_loss_clip": 1.03871322, + "balance_loss_mlp": 1.01898181, + "epoch": 0.14604491904126285, + "flos": 21759407930880.0, + "grad_norm": 3.5786927563980426, + "language_loss": 0.82745028, + "learning_rate": 3.860406946628146e-06, + "loss": 0.84890592, + "num_input_tokens_seen": 143535860, + "router_z_loss_clip": 0.66455078, + "router_z_loss_mlp": 0.2142334, + "step": 5033, + "time_per_iteration": 2.461045980453491 + }, + { + "auxiliary_loss_clip": 0.01094477, + "auxiliary_loss_mlp": 0.01044718, + "balance_loss_clip": 1.03037214, + "balance_loss_mlp": 1.02569222, + "epoch": 0.14607393650977887, + "flos": 26862697607040.0, + "grad_norm": 1.8872512451544554, + "language_loss": 0.77826148, + "learning_rate": 3.860337948129249e-06, + "loss": 0.79965341, + "num_input_tokens_seen": 143551355, + "router_z_loss_clip": 0.64086914, + "router_z_loss_mlp": 0.19030762, + "step": 5034, + "time_per_iteration": 2.6836729049682617 + }, + { + "auxiliary_loss_clip": 0.01098484, + "auxiliary_loss_mlp": 0.01036451, + "balance_loss_clip": 1.03793013, + "balance_loss_mlp": 1.01794434, + "epoch": 0.14610295397829492, + "flos": 34708624047360.0, + "grad_norm": 2.0707546547046047, + "language_loss": 0.83541787, + "learning_rate": 3.860268933199095e-06, + "loss": 0.85676718, + "num_input_tokens_seen": 143571900, + "router_z_loss_clip": 0.60546875, + "router_z_loss_mlp": 0.18511963, + "step": 5035, + "time_per_iteration": 2.5461230278015137 + }, + { + "auxiliary_loss_clip": 0.01095547, + "auxiliary_loss_mlp": 0.01038355, + "balance_loss_clip": 1.03559864, + "balance_loss_mlp": 1.0191977, + "epoch": 0.14613197144681098, + "flos": 26572429630080.0, + "grad_norm": 1.7728177476241185, + "language_loss": 0.70481604, + "learning_rate": 3.8601999018382935e-06, + "loss": 0.72615504, + "num_input_tokens_seen": 143590530, + "router_z_loss_clip": 0.59960938, + "router_z_loss_mlp": 0.19158936, + "step": 5036, + "time_per_iteration": 2.4520297050476074 + }, + { + "auxiliary_loss_clip": 0.01023965, + "auxiliary_loss_mlp": 0.01001623, + "balance_loss_clip": 1.01410258, + "balance_loss_mlp": 1.0006398, + "epoch": 0.14616098891532703, + "flos": 73418048991360.0, + "grad_norm": 0.6882473099999038, + "language_loss": 0.51920378, + "learning_rate": 3.860130854047453e-06, + "loss": 0.53945965, + "num_input_tokens_seen": 143649390, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.00982666, + "step": 5037, + "time_per_iteration": 3.037036895751953 + }, + { + "auxiliary_loss_clip": 0.01097277, + "auxiliary_loss_mlp": 0.01046318, + "balance_loss_clip": 1.03760171, + "balance_loss_mlp": 1.02810252, + "epoch": 0.14619000638384308, + "flos": 23324040777600.0, + "grad_norm": 2.1077774115342214, + "language_loss": 0.77953005, + "learning_rate": 3.860061789827185e-06, + "loss": 0.80096596, + "num_input_tokens_seen": 143664475, + "router_z_loss_clip": 0.59667969, + "router_z_loss_mlp": 0.18212891, + "step": 5038, + "time_per_iteration": 2.401108980178833 + }, + { + "auxiliary_loss_clip": 0.01099093, + "auxiliary_loss_mlp": 0.01042944, + "balance_loss_clip": 1.03713679, + "balance_loss_mlp": 1.02362657, + "epoch": 0.14621902385235913, + "flos": 23032655637120.0, + "grad_norm": 1.8859321614928086, + "language_loss": 0.68980253, + "learning_rate": 3.859992709178097e-06, + "loss": 0.71122289, + "num_input_tokens_seen": 143679075, + "router_z_loss_clip": 0.61987305, + "router_z_loss_mlp": 0.19317627, + "step": 5039, + "time_per_iteration": 2.5044288635253906 + }, + { + "auxiliary_loss_clip": 0.01097567, + "auxiliary_loss_mlp": 0.01051862, + "balance_loss_clip": 1.03492975, + "balance_loss_mlp": 1.03018951, + "epoch": 0.14624804132087516, + "flos": 29927233906560.0, + "grad_norm": 2.1460014053053507, + "language_loss": 0.78091681, + "learning_rate": 3.859923612100803e-06, + "loss": 0.80241108, + "num_input_tokens_seen": 143695695, + "router_z_loss_clip": 0.62646484, + "router_z_loss_mlp": 0.21679688, + "step": 5040, + "time_per_iteration": 2.4601237773895264 + }, + { + "auxiliary_loss_clip": 0.01107029, + "auxiliary_loss_mlp": 0.01042406, + "balance_loss_clip": 1.03920329, + "balance_loss_mlp": 1.01963735, + "epoch": 0.1462770587893912, + "flos": 16025412810240.0, + "grad_norm": 2.488779613364329, + "language_loss": 0.85758656, + "learning_rate": 3.859854498595909e-06, + "loss": 0.87908095, + "num_input_tokens_seen": 143708480, + "router_z_loss_clip": 0.67773438, + "router_z_loss_mlp": 0.22790527, + "step": 5041, + "time_per_iteration": 2.480071783065796 + }, + { + "auxiliary_loss_clip": 0.01095369, + "auxiliary_loss_mlp": 0.0104576, + "balance_loss_clip": 1.03592849, + "balance_loss_mlp": 1.02603662, + "epoch": 0.14630607625790726, + "flos": 26716551189120.0, + "grad_norm": 1.8097904285265172, + "language_loss": 0.60656989, + "learning_rate": 3.859785368664028e-06, + "loss": 0.62798113, + "num_input_tokens_seen": 143726695, + "router_z_loss_clip": 0.59423828, + "router_z_loss_mlp": 0.19732666, + "step": 5042, + "time_per_iteration": 2.433854341506958 + }, + { + "auxiliary_loss_clip": 0.01086609, + "auxiliary_loss_mlp": 0.01029777, + "balance_loss_clip": 1.02930009, + "balance_loss_mlp": 1.01258731, + "epoch": 0.1463350937264233, + "flos": 16320638200320.0, + "grad_norm": 2.982952960860862, + "language_loss": 0.93024451, + "learning_rate": 3.859716222305771e-06, + "loss": 0.95140833, + "num_input_tokens_seen": 143739125, + "router_z_loss_clip": 0.57324219, + "router_z_loss_mlp": 0.17211914, + "step": 5043, + "time_per_iteration": 2.3700790405273438 + }, + { + "auxiliary_loss_clip": 0.01091179, + "auxiliary_loss_mlp": 0.01035614, + "balance_loss_clip": 1.03263366, + "balance_loss_mlp": 1.01770306, + "epoch": 0.14636411119493936, + "flos": 20368119317760.0, + "grad_norm": 2.28230211418453, + "language_loss": 0.72835588, + "learning_rate": 3.859647059521747e-06, + "loss": 0.74962378, + "num_input_tokens_seen": 143753705, + "router_z_loss_clip": 0.58642578, + "router_z_loss_mlp": 0.17901611, + "step": 5044, + "time_per_iteration": 2.3759562969207764 + }, + { + "auxiliary_loss_clip": 0.01018164, + "auxiliary_loss_mlp": 0.01003004, + "balance_loss_clip": 1.00829887, + "balance_loss_mlp": 1.00186539, + "epoch": 0.1463931286634554, + "flos": 74767965776640.0, + "grad_norm": 0.6949420636508966, + "language_loss": 0.47532746, + "learning_rate": 3.8595778803125675e-06, + "loss": 0.49553916, + "num_input_tokens_seen": 143808190, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01141357, + "step": 5045, + "time_per_iteration": 2.990565299987793 + }, + { + "auxiliary_loss_clip": 0.01085185, + "auxiliary_loss_mlp": 0.0103405, + "balance_loss_clip": 1.03053284, + "balance_loss_mlp": 1.01588285, + "epoch": 0.14642214613197144, + "flos": 12780445271040.0, + "grad_norm": 2.5960049427745924, + "language_loss": 0.88772738, + "learning_rate": 3.859508684678844e-06, + "loss": 0.90891975, + "num_input_tokens_seen": 143821235, + "router_z_loss_clip": 0.54663086, + "router_z_loss_mlp": 0.1817627, + "step": 5046, + "time_per_iteration": 2.335407018661499 + }, + { + "auxiliary_loss_clip": 0.01015665, + "auxiliary_loss_mlp": 0.01004436, + "balance_loss_clip": 1.00623536, + "balance_loss_mlp": 1.00341725, + "epoch": 0.1464511636004875, + "flos": 67637094359040.0, + "grad_norm": 0.8824491033276328, + "language_loss": 0.46886396, + "learning_rate": 3.859439472621188e-06, + "loss": 0.48906502, + "num_input_tokens_seen": 143895230, + "router_z_loss_clip": 0.09423828, + "router_z_loss_mlp": 0.01019287, + "step": 5047, + "time_per_iteration": 3.252454996109009 + }, + { + "auxiliary_loss_clip": 0.01012847, + "auxiliary_loss_mlp": 0.01001223, + "balance_loss_clip": 1.00370836, + "balance_loss_mlp": 1.00030553, + "epoch": 0.14648018106900354, + "flos": 72511109936640.0, + "grad_norm": 0.6723807289599598, + "language_loss": 0.47440234, + "learning_rate": 3.859370244140208e-06, + "loss": 0.49454305, + "num_input_tokens_seen": 143961165, + "router_z_loss_clip": 0.09130859, + "router_z_loss_mlp": 0.00915527, + "step": 5048, + "time_per_iteration": 3.1054494380950928 + }, + { + "auxiliary_loss_clip": 0.01011385, + "auxiliary_loss_mlp": 0.01004419, + "balance_loss_clip": 1.00254869, + "balance_loss_mlp": 1.00338781, + "epoch": 0.1465091985375196, + "flos": 67548555417600.0, + "grad_norm": 0.6945372916069381, + "language_loss": 0.49788576, + "learning_rate": 3.859300999236519e-06, + "loss": 0.51804382, + "num_input_tokens_seen": 144020665, + "router_z_loss_clip": 0.08837891, + "router_z_loss_mlp": 0.01031494, + "step": 5049, + "time_per_iteration": 2.9570727348327637 + }, + { + "auxiliary_loss_clip": 0.010114, + "auxiliary_loss_mlp": 0.01002654, + "balance_loss_clip": 1.00230861, + "balance_loss_mlp": 1.0016526, + "epoch": 0.14653821600603564, + "flos": 58863142402560.0, + "grad_norm": 0.6837389110086513, + "language_loss": 0.4915145, + "learning_rate": 3.859231737910732e-06, + "loss": 0.51165503, + "num_input_tokens_seen": 144080745, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01000977, + "step": 5050, + "time_per_iteration": 2.9628701210021973 + }, + { + "auxiliary_loss_clip": 0.01097125, + "auxiliary_loss_mlp": 0.01042663, + "balance_loss_clip": 1.03067005, + "balance_loss_mlp": 1.02041864, + "epoch": 0.14656723347455167, + "flos": 28249447743360.0, + "grad_norm": 2.884672306512607, + "language_loss": 0.88941342, + "learning_rate": 3.859162460163457e-06, + "loss": 0.91081131, + "num_input_tokens_seen": 144097910, + "router_z_loss_clip": 0.66455078, + "router_z_loss_mlp": 0.22241211, + "step": 5051, + "time_per_iteration": 2.433478593826294 + }, + { + "auxiliary_loss_clip": 0.01085091, + "auxiliary_loss_mlp": 0.0103596, + "balance_loss_clip": 1.02984643, + "balance_loss_mlp": 1.01792383, + "epoch": 0.14659625094306772, + "flos": 24215861347200.0, + "grad_norm": 2.717704897844671, + "language_loss": 0.74342966, + "learning_rate": 3.859093165995307e-06, + "loss": 0.76464021, + "num_input_tokens_seen": 144115095, + "router_z_loss_clip": 0.5534668, + "router_z_loss_mlp": 0.18029785, + "step": 5052, + "time_per_iteration": 2.443005323410034 + }, + { + "auxiliary_loss_clip": 0.01084734, + "auxiliary_loss_mlp": 0.0103194, + "balance_loss_clip": 1.0306294, + "balance_loss_mlp": 1.01607931, + "epoch": 0.14662526841158377, + "flos": 17119800288000.0, + "grad_norm": 2.358117488690729, + "language_loss": 0.89692914, + "learning_rate": 3.859023855406893e-06, + "loss": 0.91809583, + "num_input_tokens_seen": 144127845, + "router_z_loss_clip": 0.54174805, + "router_z_loss_mlp": 0.15856934, + "step": 5053, + "time_per_iteration": 2.376591682434082 + }, + { + "auxiliary_loss_clip": 0.01090036, + "auxiliary_loss_mlp": 0.01038889, + "balance_loss_clip": 1.03503168, + "balance_loss_mlp": 1.02255106, + "epoch": 0.14665428588009982, + "flos": 46161323112960.0, + "grad_norm": 2.3061833871861976, + "language_loss": 0.79386938, + "learning_rate": 3.858954528398829e-06, + "loss": 0.81515861, + "num_input_tokens_seen": 144147005, + "router_z_loss_clip": 0.54980469, + "router_z_loss_mlp": 0.16333008, + "step": 5054, + "time_per_iteration": 2.6389362812042236 + }, + { + "auxiliary_loss_clip": 0.01096045, + "auxiliary_loss_mlp": 0.01038457, + "balance_loss_clip": 1.0327394, + "balance_loss_mlp": 1.01683259, + "epoch": 0.14668330334861587, + "flos": 25547275111680.0, + "grad_norm": 2.8168727990558065, + "language_loss": 0.83430302, + "learning_rate": 3.858885184971726e-06, + "loss": 0.8556481, + "num_input_tokens_seen": 144161075, + "router_z_loss_clip": 0.63427734, + "router_z_loss_mlp": 0.21630859, + "step": 5055, + "time_per_iteration": 2.449871301651001 + }, + { + "auxiliary_loss_clip": 0.01028221, + "auxiliary_loss_mlp": 0.01009342, + "balance_loss_clip": 1.01798427, + "balance_loss_mlp": 1.00838792, + "epoch": 0.14671232081713192, + "flos": 74763497122560.0, + "grad_norm": 0.6543044180753177, + "language_loss": 0.45667976, + "learning_rate": 3.858815825126197e-06, + "loss": 0.47705543, + "num_input_tokens_seen": 144225520, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.00952148, + "step": 5056, + "time_per_iteration": 3.2400882244110107 + }, + { + "auxiliary_loss_clip": 0.01099402, + "auxiliary_loss_mlp": 0.01048148, + "balance_loss_clip": 1.03578544, + "balance_loss_mlp": 1.02592731, + "epoch": 0.14674133828564795, + "flos": 26571801225600.0, + "grad_norm": 4.715436791501982, + "language_loss": 0.9704318, + "learning_rate": 3.8587464488628555e-06, + "loss": 0.9919073, + "num_input_tokens_seen": 144244700, + "router_z_loss_clip": 0.63574219, + "router_z_loss_mlp": 0.22229004, + "step": 5057, + "time_per_iteration": 2.4403369426727295 + }, + { + "auxiliary_loss_clip": 0.01098884, + "auxiliary_loss_mlp": 0.01034012, + "balance_loss_clip": 1.03753638, + "balance_loss_mlp": 1.01657176, + "epoch": 0.146770355754164, + "flos": 32444332087680.0, + "grad_norm": 2.1696482975763325, + "language_loss": 1.00671172, + "learning_rate": 3.858677056182312e-06, + "loss": 1.02804065, + "num_input_tokens_seen": 144261550, + "router_z_loss_clip": 0.61328125, + "router_z_loss_mlp": 0.17431641, + "step": 5058, + "time_per_iteration": 2.5382559299468994 + }, + { + "auxiliary_loss_clip": 0.01102864, + "auxiliary_loss_mlp": 0.01042714, + "balance_loss_clip": 1.03984499, + "balance_loss_mlp": 1.02158999, + "epoch": 0.14679937322268005, + "flos": 11243673555840.0, + "grad_norm": 3.102040964469668, + "language_loss": 0.84299242, + "learning_rate": 3.85860764708518e-06, + "loss": 0.86444825, + "num_input_tokens_seen": 144270325, + "router_z_loss_clip": 0.63037109, + "router_z_loss_mlp": 0.21124268, + "step": 5059, + "time_per_iteration": 2.3490610122680664 + }, + { + "auxiliary_loss_clip": 0.01098158, + "auxiliary_loss_mlp": 0.01048503, + "balance_loss_clip": 1.03845119, + "balance_loss_mlp": 1.02999032, + "epoch": 0.1468283906911961, + "flos": 24859520772480.0, + "grad_norm": 2.2134759586809367, + "language_loss": 0.79306233, + "learning_rate": 3.858538221572074e-06, + "loss": 0.81452894, + "num_input_tokens_seen": 144287040, + "router_z_loss_clip": 0.59667969, + "router_z_loss_mlp": 0.18511963, + "step": 5060, + "time_per_iteration": 2.5208358764648438 + }, + { + "auxiliary_loss_clip": 0.01095001, + "auxiliary_loss_mlp": 0.01042777, + "balance_loss_clip": 1.03565395, + "balance_loss_mlp": 1.0232501, + "epoch": 0.14685740815971216, + "flos": 17632953584640.0, + "grad_norm": 2.296970247516557, + "language_loss": 0.99827802, + "learning_rate": 3.858468779643607e-06, + "loss": 1.01965582, + "num_input_tokens_seen": 144301450, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.1953125, + "step": 5061, + "time_per_iteration": 2.626889228820801 + }, + { + "auxiliary_loss_clip": 0.01089226, + "auxiliary_loss_mlp": 0.01039213, + "balance_loss_clip": 1.03530002, + "balance_loss_mlp": 1.02192783, + "epoch": 0.1468864256282282, + "flos": 58457279180160.0, + "grad_norm": 2.5558693762628626, + "language_loss": 0.70383418, + "learning_rate": 3.858399321300391e-06, + "loss": 0.72511864, + "num_input_tokens_seen": 144322565, + "router_z_loss_clip": 0.53955078, + "router_z_loss_mlp": 0.17272949, + "step": 5062, + "time_per_iteration": 2.7736949920654297 + }, + { + "auxiliary_loss_clip": 0.01093255, + "auxiliary_loss_mlp": 0.01039478, + "balance_loss_clip": 1.03594685, + "balance_loss_mlp": 1.02190065, + "epoch": 0.14691544309674423, + "flos": 34488112700160.0, + "grad_norm": 2.094937712521259, + "language_loss": 0.81663871, + "learning_rate": 3.85832984654304e-06, + "loss": 0.83796602, + "num_input_tokens_seen": 144340350, + "router_z_loss_clip": 0.57348633, + "router_z_loss_mlp": 0.17572021, + "step": 5063, + "time_per_iteration": 2.5341618061065674 + }, + { + "auxiliary_loss_clip": 0.01098521, + "auxiliary_loss_mlp": 0.01035897, + "balance_loss_clip": 1.03807199, + "balance_loss_mlp": 1.01800966, + "epoch": 0.14694446056526028, + "flos": 39450108637440.0, + "grad_norm": 73.59135939452878, + "language_loss": 0.88871646, + "learning_rate": 3.858260355372168e-06, + "loss": 0.91006058, + "num_input_tokens_seen": 144356980, + "router_z_loss_clip": 0.60424805, + "router_z_loss_mlp": 0.17889404, + "step": 5064, + "time_per_iteration": 2.6037683486938477 + }, + { + "auxiliary_loss_clip": 0.01096926, + "auxiliary_loss_mlp": 0.01040841, + "balance_loss_clip": 1.0360055, + "balance_loss_mlp": 1.01960993, + "epoch": 0.14697347803377633, + "flos": 28942892634240.0, + "grad_norm": 2.313152285844668, + "language_loss": 0.78697526, + "learning_rate": 3.858190847788388e-06, + "loss": 0.80835295, + "num_input_tokens_seen": 144373950, + "router_z_loss_clip": 0.609375, + "router_z_loss_mlp": 0.21240234, + "step": 5065, + "time_per_iteration": 2.473670482635498 + }, + { + "auxiliary_loss_clip": 0.01096944, + "auxiliary_loss_mlp": 0.01041993, + "balance_loss_clip": 1.03237355, + "balance_loss_mlp": 1.02057076, + "epoch": 0.14700249550229239, + "flos": 15333538930560.0, + "grad_norm": 2.268930737351503, + "language_loss": 0.72757161, + "learning_rate": 3.858121323792315e-06, + "loss": 0.74896103, + "num_input_tokens_seen": 144386550, + "router_z_loss_clip": 0.64599609, + "router_z_loss_mlp": 0.21435547, + "step": 5066, + "time_per_iteration": 2.5025956630706787 + }, + { + "auxiliary_loss_clip": 0.01030197, + "auxiliary_loss_mlp": 0.01017529, + "balance_loss_clip": 1.0202024, + "balance_loss_mlp": 1.01644468, + "epoch": 0.14703151297080844, + "flos": 59670857773440.0, + "grad_norm": 0.7290554884140185, + "language_loss": 0.52048671, + "learning_rate": 3.858051783384563e-06, + "loss": 0.54096395, + "num_input_tokens_seen": 144449640, + "router_z_loss_clip": 0.09960938, + "router_z_loss_mlp": 0.01086426, + "step": 5067, + "time_per_iteration": 3.0625171661376953 + }, + { + "auxiliary_loss_clip": 0.01093354, + "auxiliary_loss_mlp": 0.01038296, + "balance_loss_clip": 1.02941728, + "balance_loss_mlp": 1.01952016, + "epoch": 0.14706053043932446, + "flos": 16718543498880.0, + "grad_norm": 3.214179007727418, + "language_loss": 0.75535703, + "learning_rate": 3.857982226565745e-06, + "loss": 0.77667344, + "num_input_tokens_seen": 144461960, + "router_z_loss_clip": 0.63964844, + "router_z_loss_mlp": 0.18786621, + "step": 5068, + "time_per_iteration": 2.3894670009613037 + }, + { + "auxiliary_loss_clip": 0.01084136, + "auxiliary_loss_mlp": 0.01034747, + "balance_loss_clip": 1.02774215, + "balance_loss_mlp": 1.01645994, + "epoch": 0.1470895479078405, + "flos": 11940016089600.0, + "grad_norm": 2.34419790022664, + "language_loss": 0.74664205, + "learning_rate": 3.857912653336477e-06, + "loss": 0.76783085, + "num_input_tokens_seen": 144474120, + "router_z_loss_clip": 0.56396484, + "router_z_loss_mlp": 0.18286133, + "step": 5069, + "time_per_iteration": 2.3653810024261475 + }, + { + "auxiliary_loss_clip": 0.01084946, + "auxiliary_loss_mlp": 0.01027351, + "balance_loss_clip": 1.02913547, + "balance_loss_mlp": 1.00923681, + "epoch": 0.14711856537635656, + "flos": 15880313733120.0, + "grad_norm": 2.0033590785375646, + "language_loss": 0.58593255, + "learning_rate": 3.857843063697372e-06, + "loss": 0.60705554, + "num_input_tokens_seen": 144487805, + "router_z_loss_clip": 0.55834961, + "router_z_loss_mlp": 0.18109131, + "step": 5070, + "time_per_iteration": 2.370570659637451 + }, + { + "auxiliary_loss_clip": 0.0108416, + "auxiliary_loss_mlp": 0.01033561, + "balance_loss_clip": 1.02949929, + "balance_loss_mlp": 1.0159179, + "epoch": 0.14714758284487262, + "flos": 37444488007680.0, + "grad_norm": 2.371623141688724, + "language_loss": 0.82553691, + "learning_rate": 3.857773457649045e-06, + "loss": 0.84671408, + "num_input_tokens_seen": 144505670, + "router_z_loss_clip": 0.54614258, + "router_z_loss_mlp": 0.17657471, + "step": 5071, + "time_per_iteration": 2.507873058319092 + }, + { + "auxiliary_loss_clip": 0.01099488, + "auxiliary_loss_mlp": 0.01043522, + "balance_loss_clip": 1.03272748, + "balance_loss_mlp": 1.02022862, + "epoch": 0.14717660031338867, + "flos": 30219840944640.0, + "grad_norm": 2.515038148409386, + "language_loss": 0.96652985, + "learning_rate": 3.857703835192112e-06, + "loss": 0.98795998, + "num_input_tokens_seen": 144520500, + "router_z_loss_clip": 0.66699219, + "router_z_loss_mlp": 0.23303223, + "step": 5072, + "time_per_iteration": 2.4880621433258057 + }, + { + "auxiliary_loss_clip": 0.01021156, + "auxiliary_loss_mlp": 0.01004247, + "balance_loss_clip": 1.01169562, + "balance_loss_mlp": 1.00317371, + "epoch": 0.14720561778190472, + "flos": 74778124982400.0, + "grad_norm": 0.655890310010018, + "language_loss": 0.43341279, + "learning_rate": 3.857634196327187e-06, + "loss": 0.45366681, + "num_input_tokens_seen": 144589870, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01074219, + "step": 5073, + "time_per_iteration": 3.2126731872558594 + }, + { + "auxiliary_loss_clip": 0.01091433, + "auxiliary_loss_mlp": 0.01046286, + "balance_loss_clip": 1.03013361, + "balance_loss_mlp": 1.02691436, + "epoch": 0.14723463525042074, + "flos": 26243722379520.0, + "grad_norm": 2.2020740554100198, + "language_loss": 0.9139514, + "learning_rate": 3.8575645410548845e-06, + "loss": 0.93532854, + "num_input_tokens_seen": 144610505, + "router_z_loss_clip": 0.61328125, + "router_z_loss_mlp": 0.19384766, + "step": 5074, + "time_per_iteration": 2.8116455078125 + }, + { + "auxiliary_loss_clip": 0.01016353, + "auxiliary_loss_mlp": 0.01001011, + "balance_loss_clip": 1.00716209, + "balance_loss_mlp": 0.99999195, + "epoch": 0.1472636527189368, + "flos": 69952709750400.0, + "grad_norm": 0.6715555141753021, + "language_loss": 0.48335338, + "learning_rate": 3.85749486937582e-06, + "loss": 0.50352705, + "num_input_tokens_seen": 144672970, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01019287, + "step": 5075, + "time_per_iteration": 2.9889087677001953 + }, + { + "auxiliary_loss_clip": 0.01083461, + "auxiliary_loss_mlp": 0.01037979, + "balance_loss_clip": 1.0270772, + "balance_loss_mlp": 1.02046764, + "epoch": 0.14729267018745285, + "flos": 22083576704640.0, + "grad_norm": 2.574048927768998, + "language_loss": 0.9057461, + "learning_rate": 3.85742518129061e-06, + "loss": 0.92696053, + "num_input_tokens_seen": 144687560, + "router_z_loss_clip": 0.56445312, + "router_z_loss_mlp": 0.17510986, + "step": 5076, + "time_per_iteration": 2.3836264610290527 + }, + { + "auxiliary_loss_clip": 0.01093616, + "auxiliary_loss_mlp": 0.010372, + "balance_loss_clip": 1.02961147, + "balance_loss_mlp": 1.01689863, + "epoch": 0.1473216876559689, + "flos": 32042516716800.0, + "grad_norm": 2.045411637903165, + "language_loss": 0.61206764, + "learning_rate": 3.857355476799868e-06, + "loss": 0.63337582, + "num_input_tokens_seen": 144704625, + "router_z_loss_clip": 0.63891602, + "router_z_loss_mlp": 0.20281982, + "step": 5077, + "time_per_iteration": 2.53534197807312 + }, + { + "auxiliary_loss_clip": 0.01014685, + "auxiliary_loss_mlp": 0.01003053, + "balance_loss_clip": 1.00520873, + "balance_loss_mlp": 1.00206327, + "epoch": 0.14735070512448495, + "flos": 63866859281280.0, + "grad_norm": 0.675586537881202, + "language_loss": 0.47622764, + "learning_rate": 3.857285755904212e-06, + "loss": 0.49640501, + "num_input_tokens_seen": 144762385, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.0098877, + "step": 5078, + "time_per_iteration": 2.9482133388519287 + }, + { + "auxiliary_loss_clip": 0.01015947, + "auxiliary_loss_mlp": 0.01001323, + "balance_loss_clip": 1.00643337, + "balance_loss_mlp": 1.00030994, + "epoch": 0.147379722593001, + "flos": 74772748632960.0, + "grad_norm": 0.721525176993916, + "language_loss": 0.5294382, + "learning_rate": 3.857216018604256e-06, + "loss": 0.54961091, + "num_input_tokens_seen": 144823505, + "router_z_loss_clip": 0.09521484, + "router_z_loss_mlp": 0.01013184, + "step": 5079, + "time_per_iteration": 3.0697104930877686 + }, + { + "auxiliary_loss_clip": 0.01017227, + "auxiliary_loss_mlp": 0.01006221, + "balance_loss_clip": 1.00751114, + "balance_loss_mlp": 1.00535727, + "epoch": 0.14740874006151702, + "flos": 54624616992000.0, + "grad_norm": 0.7368551714606013, + "language_loss": 0.4803654, + "learning_rate": 3.857146264900617e-06, + "loss": 0.50059986, + "num_input_tokens_seen": 144879920, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.00866699, + "step": 5080, + "time_per_iteration": 2.874641180038452 + }, + { + "auxiliary_loss_clip": 0.01087211, + "auxiliary_loss_mlp": 0.0103732, + "balance_loss_clip": 1.03078282, + "balance_loss_mlp": 1.0209291, + "epoch": 0.14743775753003308, + "flos": 68418837544320.0, + "grad_norm": 2.7380611110202326, + "language_loss": 0.6167227, + "learning_rate": 3.857076494793911e-06, + "loss": 0.637968, + "num_input_tokens_seen": 144901945, + "router_z_loss_clip": 0.56347656, + "router_z_loss_mlp": 0.16387939, + "step": 5081, + "time_per_iteration": 2.7625885009765625 + }, + { + "auxiliary_loss_clip": 0.01089286, + "auxiliary_loss_mlp": 0.01036701, + "balance_loss_clip": 1.02948594, + "balance_loss_mlp": 1.018116, + "epoch": 0.14746677499854913, + "flos": 10151136380160.0, + "grad_norm": 3.41744155656627, + "language_loss": 0.86235607, + "learning_rate": 3.857006708284753e-06, + "loss": 0.88361597, + "num_input_tokens_seen": 144913195, + "router_z_loss_clip": 0.59814453, + "router_z_loss_mlp": 0.18591309, + "step": 5082, + "time_per_iteration": 2.3441734313964844 + }, + { + "auxiliary_loss_clip": 0.01093872, + "auxiliary_loss_mlp": 0.01040003, + "balance_loss_clip": 1.03324687, + "balance_loss_mlp": 1.02040493, + "epoch": 0.14749579246706518, + "flos": 40037173015680.0, + "grad_norm": 2.461450348899954, + "language_loss": 0.70103288, + "learning_rate": 3.856936905373761e-06, + "loss": 0.7223717, + "num_input_tokens_seen": 144933445, + "router_z_loss_clip": 0.60546875, + "router_z_loss_mlp": 0.19598389, + "step": 5083, + "time_per_iteration": 2.615468978881836 + }, + { + "auxiliary_loss_clip": 0.0109475, + "auxiliary_loss_mlp": 0.01049242, + "balance_loss_clip": 1.03052807, + "balance_loss_mlp": 1.02951312, + "epoch": 0.14752480993558123, + "flos": 27337306896000.0, + "grad_norm": 1.7922609172501198, + "language_loss": 0.80957007, + "learning_rate": 3.85686708606155e-06, + "loss": 0.83100998, + "num_input_tokens_seen": 144950185, + "router_z_loss_clip": 0.64208984, + "router_z_loss_mlp": 0.19714355, + "step": 5084, + "time_per_iteration": 2.4419167041778564 + }, + { + "auxiliary_loss_clip": 0.01087342, + "auxiliary_loss_mlp": 0.01041381, + "balance_loss_clip": 1.03043842, + "balance_loss_mlp": 1.02322578, + "epoch": 0.14755382740409725, + "flos": 21427139721600.0, + "grad_norm": 2.3550071904786125, + "language_loss": 0.82013333, + "learning_rate": 3.856797250348738e-06, + "loss": 0.84142047, + "num_input_tokens_seen": 144965775, + "router_z_loss_clip": 0.56982422, + "router_z_loss_mlp": 0.18151855, + "step": 5085, + "time_per_iteration": 2.4262421131134033 + }, + { + "auxiliary_loss_clip": 0.0109081, + "auxiliary_loss_mlp": 0.01044706, + "balance_loss_clip": 1.03177881, + "balance_loss_mlp": 1.02721226, + "epoch": 0.1475828448726133, + "flos": 22703564361600.0, + "grad_norm": 2.3357142270344644, + "language_loss": 0.76569062, + "learning_rate": 3.856727398235941e-06, + "loss": 0.78704578, + "num_input_tokens_seen": 144980110, + "router_z_loss_clip": 0.58984375, + "router_z_loss_mlp": 0.17486572, + "step": 5086, + "time_per_iteration": 2.389129161834717 + }, + { + "auxiliary_loss_clip": 0.0109541, + "auxiliary_loss_mlp": 0.01047106, + "balance_loss_clip": 1.0334419, + "balance_loss_mlp": 1.02750254, + "epoch": 0.14761186234112936, + "flos": 74729912394240.0, + "grad_norm": 2.162072641009884, + "language_loss": 0.89042693, + "learning_rate": 3.856657529723777e-06, + "loss": 0.91185212, + "num_input_tokens_seen": 145003225, + "router_z_loss_clip": 0.62036133, + "router_z_loss_mlp": 0.19616699, + "step": 5087, + "time_per_iteration": 2.9932634830474854 + }, + { + "auxiliary_loss_clip": 0.01084953, + "auxiliary_loss_mlp": 0.01037843, + "balance_loss_clip": 1.02793097, + "balance_loss_mlp": 1.02160668, + "epoch": 0.1476408798096454, + "flos": 34306529385600.0, + "grad_norm": 2.603785049516157, + "language_loss": 0.95061755, + "learning_rate": 3.856587644812862e-06, + "loss": 0.97184551, + "num_input_tokens_seen": 145021380, + "router_z_loss_clip": 0.56982422, + "router_z_loss_mlp": 0.16241455, + "step": 5088, + "time_per_iteration": 2.5287158489227295 + }, + { + "auxiliary_loss_clip": 0.01085997, + "auxiliary_loss_mlp": 0.01036222, + "balance_loss_clip": 1.02775633, + "balance_loss_mlp": 1.0187937, + "epoch": 0.14766989727816146, + "flos": 14823492744960.0, + "grad_norm": 2.7529943182743115, + "language_loss": 0.74569893, + "learning_rate": 3.8565177435038134e-06, + "loss": 0.7669211, + "num_input_tokens_seen": 145035575, + "router_z_loss_clip": 0.58398438, + "router_z_loss_mlp": 0.17443848, + "step": 5089, + "time_per_iteration": 2.4420478343963623 + }, + { + "auxiliary_loss_clip": 0.0108927, + "auxiliary_loss_mlp": 0.01044996, + "balance_loss_clip": 1.03017116, + "balance_loss_mlp": 1.02529109, + "epoch": 0.1476989147466775, + "flos": 23507334748800.0, + "grad_norm": 2.1236414693366794, + "language_loss": 0.88734025, + "learning_rate": 3.85644782579725e-06, + "loss": 0.90868288, + "num_input_tokens_seen": 145052955, + "router_z_loss_clip": 0.59130859, + "router_z_loss_mlp": 0.19689941, + "step": 5090, + "time_per_iteration": 2.4499423503875732 + }, + { + "auxiliary_loss_clip": 0.01090392, + "auxiliary_loss_mlp": 0.01042409, + "balance_loss_clip": 1.03275394, + "balance_loss_mlp": 1.02402115, + "epoch": 0.14772793221519354, + "flos": 36458121876480.0, + "grad_norm": 3.3649828941261255, + "language_loss": 0.7772783, + "learning_rate": 3.8563778916937865e-06, + "loss": 0.7986064, + "num_input_tokens_seen": 145068495, + "router_z_loss_clip": 0.57666016, + "router_z_loss_mlp": 0.18389893, + "step": 5091, + "time_per_iteration": 2.5381340980529785 + }, + { + "auxiliary_loss_clip": 0.01090661, + "auxiliary_loss_mlp": 0.01042722, + "balance_loss_clip": 1.0312376, + "balance_loss_mlp": 1.02489448, + "epoch": 0.1477569496837096, + "flos": 31275195655680.0, + "grad_norm": 3.4551014233943294, + "language_loss": 1.00938916, + "learning_rate": 3.856307941194042e-06, + "loss": 1.03072298, + "num_input_tokens_seen": 145082205, + "router_z_loss_clip": 0.59423828, + "router_z_loss_mlp": 0.17840576, + "step": 5092, + "time_per_iteration": 2.4567980766296387 + }, + { + "auxiliary_loss_clip": 0.0108594, + "auxiliary_loss_mlp": 0.0103493, + "balance_loss_clip": 1.0304296, + "balance_loss_mlp": 1.01843143, + "epoch": 0.14778596715222564, + "flos": 13180515073920.0, + "grad_norm": 2.3658398088200867, + "language_loss": 0.86429036, + "learning_rate": 3.856237974298636e-06, + "loss": 0.885499, + "num_input_tokens_seen": 145094755, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.16491699, + "step": 5093, + "time_per_iteration": 2.4228296279907227 + }, + { + "auxiliary_loss_clip": 0.01087024, + "auxiliary_loss_mlp": 0.01042305, + "balance_loss_clip": 1.02994013, + "balance_loss_mlp": 1.02331495, + "epoch": 0.1478149846207417, + "flos": 12815149029120.0, + "grad_norm": 3.189650989366145, + "language_loss": 0.86896157, + "learning_rate": 3.856167991008185e-06, + "loss": 0.89025486, + "num_input_tokens_seen": 145105365, + "router_z_loss_clip": 0.56982422, + "router_z_loss_mlp": 0.19006348, + "step": 5094, + "time_per_iteration": 2.343498945236206 + }, + { + "auxiliary_loss_clip": 0.01017902, + "auxiliary_loss_mlp": 0.01006651, + "balance_loss_clip": 1.00798237, + "balance_loss_mlp": 1.00558364, + "epoch": 0.14784400208925774, + "flos": 60158209708800.0, + "grad_norm": 0.7708611244875032, + "language_loss": 0.47671866, + "learning_rate": 3.856097991323307e-06, + "loss": 0.49696422, + "num_input_tokens_seen": 145167355, + "router_z_loss_clip": 0.09912109, + "router_z_loss_mlp": 0.01068115, + "step": 5095, + "time_per_iteration": 3.1010985374450684 + }, + { + "auxiliary_loss_clip": 0.01016698, + "auxiliary_loss_mlp": 0.01008004, + "balance_loss_clip": 1.00718331, + "balance_loss_mlp": 1.00699663, + "epoch": 0.14787301955777377, + "flos": 74791216742400.0, + "grad_norm": 0.6813048576824146, + "language_loss": 0.46052182, + "learning_rate": 3.856027975244621e-06, + "loss": 0.48076886, + "num_input_tokens_seen": 145232100, + "router_z_loss_clip": 0.09521484, + "router_z_loss_mlp": 0.0100708, + "step": 5096, + "time_per_iteration": 3.2409586906433105 + }, + { + "auxiliary_loss_clip": 0.01017091, + "auxiliary_loss_mlp": 0.01004701, + "balance_loss_clip": 1.0074985, + "balance_loss_mlp": 1.00374722, + "epoch": 0.14790203702628982, + "flos": 74769850990080.0, + "grad_norm": 0.6442034341132645, + "language_loss": 0.47773927, + "learning_rate": 3.855957942772743e-06, + "loss": 0.49795723, + "num_input_tokens_seen": 145298040, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.00952148, + "step": 5097, + "time_per_iteration": 3.140589714050293 + }, + { + "auxiliary_loss_clip": 0.0109268, + "auxiliary_loss_mlp": 0.01052611, + "balance_loss_clip": 1.03196621, + "balance_loss_mlp": 1.03223228, + "epoch": 0.14793105449480587, + "flos": 12524147913600.0, + "grad_norm": 2.517981540490465, + "language_loss": 0.77616471, + "learning_rate": 3.855887893908295e-06, + "loss": 0.79761767, + "num_input_tokens_seen": 145310535, + "router_z_loss_clip": 0.60742188, + "router_z_loss_mlp": 0.20379639, + "step": 5098, + "time_per_iteration": 4.557384729385376 + }, + { + "auxiliary_loss_clip": 0.01091674, + "auxiliary_loss_mlp": 0.01039153, + "balance_loss_clip": 1.03372252, + "balance_loss_mlp": 1.01992464, + "epoch": 0.14796007196332192, + "flos": 32699267902080.0, + "grad_norm": 2.18645149404108, + "language_loss": 0.72973418, + "learning_rate": 3.855817828651894e-06, + "loss": 0.75104243, + "num_input_tokens_seen": 145327840, + "router_z_loss_clip": 0.5793457, + "router_z_loss_mlp": 0.19238281, + "step": 5099, + "time_per_iteration": 2.6546432971954346 + }, + { + "auxiliary_loss_clip": 0.01093511, + "auxiliary_loss_mlp": 0.01050122, + "balance_loss_clip": 1.03531599, + "balance_loss_mlp": 1.02973711, + "epoch": 0.14798908943183797, + "flos": 13691643511680.0, + "grad_norm": 2.6109045432557307, + "language_loss": 0.71561456, + "learning_rate": 3.855747747004159e-06, + "loss": 0.73705089, + "num_input_tokens_seen": 145339350, + "router_z_loss_clip": 0.58203125, + "router_z_loss_mlp": 0.20373535, + "step": 5100, + "time_per_iteration": 4.797016382217407 + }, + { + "auxiliary_loss_clip": 0.01090841, + "auxiliary_loss_mlp": 0.01040523, + "balance_loss_clip": 1.03429019, + "balance_loss_mlp": 1.02332735, + "epoch": 0.14801810690035402, + "flos": 15880558112640.0, + "grad_norm": 2.219149218495358, + "language_loss": 0.52357972, + "learning_rate": 3.855677648965709e-06, + "loss": 0.54489326, + "num_input_tokens_seen": 145352450, + "router_z_loss_clip": 0.56518555, + "router_z_loss_mlp": 0.17205811, + "step": 5101, + "time_per_iteration": 2.35137939453125 + }, + { + "auxiliary_loss_clip": 0.01097215, + "auxiliary_loss_mlp": 0.01051172, + "balance_loss_clip": 1.03479588, + "balance_loss_mlp": 1.0304122, + "epoch": 0.14804712436887005, + "flos": 41894587457280.0, + "grad_norm": 3.9406248910322996, + "language_loss": 0.86656797, + "learning_rate": 3.855607534537162e-06, + "loss": 0.88805193, + "num_input_tokens_seen": 145371780, + "router_z_loss_clip": 0.62402344, + "router_z_loss_mlp": 0.2074585, + "step": 5102, + "time_per_iteration": 2.6140153408050537 + }, + { + "auxiliary_loss_clip": 0.01091877, + "auxiliary_loss_mlp": 0.01050187, + "balance_loss_clip": 1.03350067, + "balance_loss_mlp": 1.03203177, + "epoch": 0.1480761418373861, + "flos": 18474499929600.0, + "grad_norm": 2.5384654468103816, + "language_loss": 0.89458537, + "learning_rate": 3.8555374037191395e-06, + "loss": 0.91600603, + "num_input_tokens_seen": 145385675, + "router_z_loss_clip": 0.58422852, + "router_z_loss_mlp": 0.1817627, + "step": 5103, + "time_per_iteration": 2.3843860626220703 + }, + { + "auxiliary_loss_clip": 0.01100045, + "auxiliary_loss_mlp": 0.01064819, + "balance_loss_clip": 1.03556991, + "balance_loss_mlp": 1.04448199, + "epoch": 0.14810515930590215, + "flos": 52074073728000.0, + "grad_norm": 2.0923689684151525, + "language_loss": 0.74929458, + "learning_rate": 3.855467256512259e-06, + "loss": 0.77094322, + "num_input_tokens_seen": 145404905, + "router_z_loss_clip": 0.64453125, + "router_z_loss_mlp": 0.20361328, + "step": 5104, + "time_per_iteration": 2.718258857727051 + }, + { + "auxiliary_loss_clip": 0.01099484, + "auxiliary_loss_mlp": 0.01046968, + "balance_loss_clip": 1.039258, + "balance_loss_mlp": 1.02587986, + "epoch": 0.1481341767744182, + "flos": 21571366014720.0, + "grad_norm": 2.3792343731624928, + "language_loss": 0.88141197, + "learning_rate": 3.8553970929171414e-06, + "loss": 0.90287644, + "num_input_tokens_seen": 145417545, + "router_z_loss_clip": 0.60205078, + "router_z_loss_mlp": 0.21081543, + "step": 5105, + "time_per_iteration": 4.920596599578857 + }, + { + "auxiliary_loss_clip": 0.01090869, + "auxiliary_loss_mlp": 0.01052552, + "balance_loss_clip": 1.03535032, + "balance_loss_mlp": 1.03466439, + "epoch": 0.14816319424293425, + "flos": 20952495521280.0, + "grad_norm": 2.310433019519251, + "language_loss": 0.78409147, + "learning_rate": 3.855326912934406e-06, + "loss": 0.80552566, + "num_input_tokens_seen": 145431050, + "router_z_loss_clip": 0.55517578, + "router_z_loss_mlp": 0.17901611, + "step": 5106, + "time_per_iteration": 2.425278425216675 + }, + { + "auxiliary_loss_clip": 0.01089937, + "auxiliary_loss_mlp": 0.0104163, + "balance_loss_clip": 1.03420651, + "balance_loss_mlp": 1.02500582, + "epoch": 0.1481922117114503, + "flos": 35911905655680.0, + "grad_norm": 2.0403993294833347, + "language_loss": 0.61424398, + "learning_rate": 3.855256716564672e-06, + "loss": 0.63555962, + "num_input_tokens_seen": 145447895, + "router_z_loss_clip": 0.55664062, + "router_z_loss_mlp": 0.16625977, + "step": 5107, + "time_per_iteration": 5.057919979095459 + }, + { + "auxiliary_loss_clip": 0.01098968, + "auxiliary_loss_mlp": 0.01050486, + "balance_loss_clip": 1.03737938, + "balance_loss_mlp": 1.0323782, + "epoch": 0.14822122917996633, + "flos": 11101751412480.0, + "grad_norm": 3.9857058455502417, + "language_loss": 0.79650438, + "learning_rate": 3.8551865038085605e-06, + "loss": 0.81799901, + "num_input_tokens_seen": 145459170, + "router_z_loss_clip": 0.61572266, + "router_z_loss_mlp": 0.18103027, + "step": 5108, + "time_per_iteration": 2.386277437210083 + }, + { + "auxiliary_loss_clip": 0.01088724, + "auxiliary_loss_mlp": 0.01037379, + "balance_loss_clip": 1.03512859, + "balance_loss_mlp": 1.02260339, + "epoch": 0.14825024664848238, + "flos": 22739664574080.0, + "grad_norm": 2.65915060916311, + "language_loss": 0.79503715, + "learning_rate": 3.8551162746666904e-06, + "loss": 0.81629825, + "num_input_tokens_seen": 145471930, + "router_z_loss_clip": 0.53637695, + "router_z_loss_mlp": 0.14764404, + "step": 5109, + "time_per_iteration": 2.3963119983673096 + }, + { + "auxiliary_loss_clip": 0.01033886, + "auxiliary_loss_mlp": 0.01033441, + "balance_loss_clip": 1.02314818, + "balance_loss_mlp": 1.0323261, + "epoch": 0.14827926411699843, + "flos": 60794991596160.0, + "grad_norm": 0.7282136007658392, + "language_loss": 0.56665564, + "learning_rate": 3.855046029139683e-06, + "loss": 0.58732891, + "num_input_tokens_seen": 145532300, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01116943, + "step": 5110, + "time_per_iteration": 3.1998796463012695 + }, + { + "auxiliary_loss_clip": 0.01096395, + "auxiliary_loss_mlp": 0.01038359, + "balance_loss_clip": 1.03555632, + "balance_loss_mlp": 1.01880288, + "epoch": 0.14830828158551448, + "flos": 23871094871040.0, + "grad_norm": 3.9085138824523082, + "language_loss": 0.69815195, + "learning_rate": 3.854975767228159e-06, + "loss": 0.71949953, + "num_input_tokens_seen": 145547105, + "router_z_loss_clip": 0.6081543, + "router_z_loss_mlp": 0.19561768, + "step": 5111, + "time_per_iteration": 2.4129083156585693 + }, + { + "auxiliary_loss_clip": 0.01097017, + "auxiliary_loss_mlp": 0.01049123, + "balance_loss_clip": 1.03420281, + "balance_loss_mlp": 1.0283922, + "epoch": 0.14833729905403054, + "flos": 46236281535360.0, + "grad_norm": 2.0283902029559697, + "language_loss": 0.85368419, + "learning_rate": 3.854905488932738e-06, + "loss": 0.87514561, + "num_input_tokens_seen": 145568290, + "router_z_loss_clip": 0.62792969, + "router_z_loss_mlp": 0.20697021, + "step": 5112, + "time_per_iteration": 2.736980676651001 + }, + { + "auxiliary_loss_clip": 0.01095708, + "auxiliary_loss_mlp": 0.01055134, + "balance_loss_clip": 1.03417182, + "balance_loss_mlp": 1.03367579, + "epoch": 0.14836631652254656, + "flos": 21862367130240.0, + "grad_norm": 2.452741924954678, + "language_loss": 0.77255225, + "learning_rate": 3.854835194254041e-06, + "loss": 0.79406065, + "num_input_tokens_seen": 145581085, + "router_z_loss_clip": 0.61474609, + "router_z_loss_mlp": 0.21435547, + "step": 5113, + "time_per_iteration": 2.3895342350006104 + }, + { + "auxiliary_loss_clip": 0.01098041, + "auxiliary_loss_mlp": 0.01044623, + "balance_loss_clip": 1.03521907, + "balance_loss_mlp": 1.02317703, + "epoch": 0.1483953339910626, + "flos": 30590827718400.0, + "grad_norm": 2.0498359062186866, + "language_loss": 0.92212498, + "learning_rate": 3.854764883192689e-06, + "loss": 0.94355166, + "num_input_tokens_seen": 145598440, + "router_z_loss_clip": 0.62841797, + "router_z_loss_mlp": 0.21453857, + "step": 5114, + "time_per_iteration": 2.4896974563598633 + }, + { + "auxiliary_loss_clip": 0.01098341, + "auxiliary_loss_mlp": 0.0103917, + "balance_loss_clip": 1.03692412, + "balance_loss_mlp": 1.02041805, + "epoch": 0.14842435145957866, + "flos": 22959198403200.0, + "grad_norm": 2.5184816273248583, + "language_loss": 0.84209555, + "learning_rate": 3.854694555749303e-06, + "loss": 0.86347061, + "num_input_tokens_seen": 145615035, + "router_z_loss_clip": 0.61376953, + "router_z_loss_mlp": 0.18774414, + "step": 5115, + "time_per_iteration": 2.4178502559661865 + }, + { + "auxiliary_loss_clip": 0.01095869, + "auxiliary_loss_mlp": 0.01036711, + "balance_loss_clip": 1.03566241, + "balance_loss_mlp": 1.01801872, + "epoch": 0.14845336892809471, + "flos": 28431065969280.0, + "grad_norm": 2.4527314454945777, + "language_loss": 0.78422523, + "learning_rate": 3.854624211924504e-06, + "loss": 0.80555105, + "num_input_tokens_seen": 145630095, + "router_z_loss_clip": 0.6015625, + "router_z_loss_mlp": 0.18713379, + "step": 5116, + "time_per_iteration": 2.4662890434265137 + }, + { + "auxiliary_loss_clip": 0.01093474, + "auxiliary_loss_mlp": 0.01038197, + "balance_loss_clip": 1.03499031, + "balance_loss_mlp": 1.02131653, + "epoch": 0.14848238639661077, + "flos": 24855889991040.0, + "grad_norm": 2.0176641271347013, + "language_loss": 1.10470819, + "learning_rate": 3.854553851718913e-06, + "loss": 1.12602484, + "num_input_tokens_seen": 145648145, + "router_z_loss_clip": 0.5847168, + "router_z_loss_mlp": 0.16894531, + "step": 5117, + "time_per_iteration": 2.4625375270843506 + }, + { + "auxiliary_loss_clip": 0.01026559, + "auxiliary_loss_mlp": 0.0100794, + "balance_loss_clip": 1.01609564, + "balance_loss_mlp": 1.00673044, + "epoch": 0.14851140386512682, + "flos": 62041006575360.0, + "grad_norm": 0.7044717252371888, + "language_loss": 0.49558222, + "learning_rate": 3.854483475133153e-06, + "loss": 0.5159272, + "num_input_tokens_seen": 145707785, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01208496, + "step": 5118, + "time_per_iteration": 2.9634287357330322 + }, + { + "auxiliary_loss_clip": 0.01085785, + "auxiliary_loss_mlp": 0.01040361, + "balance_loss_clip": 1.02960479, + "balance_loss_mlp": 1.02191925, + "epoch": 0.14854042133364284, + "flos": 22592680283520.0, + "grad_norm": 1.8671519395552032, + "language_loss": 0.8290199, + "learning_rate": 3.854413082167844e-06, + "loss": 0.85028148, + "num_input_tokens_seen": 145724840, + "router_z_loss_clip": 0.56103516, + "router_z_loss_mlp": 0.18432617, + "step": 5119, + "time_per_iteration": 2.4383926391601562 + }, + { + "auxiliary_loss_clip": 0.01093439, + "auxiliary_loss_mlp": 0.0105519, + "balance_loss_clip": 1.03182983, + "balance_loss_mlp": 1.03509092, + "epoch": 0.1485694388021589, + "flos": 16137553697280.0, + "grad_norm": 2.940267828958122, + "language_loss": 0.7683503, + "learning_rate": 3.8543426728236086e-06, + "loss": 0.78983665, + "num_input_tokens_seen": 145738475, + "router_z_loss_clip": 0.61621094, + "router_z_loss_mlp": 0.20080566, + "step": 5120, + "time_per_iteration": 2.4139456748962402 + }, + { + "auxiliary_loss_clip": 0.01092216, + "auxiliary_loss_mlp": 0.01044398, + "balance_loss_clip": 1.03175807, + "balance_loss_mlp": 1.02517581, + "epoch": 0.14859845627067494, + "flos": 13435939647360.0, + "grad_norm": 2.1857309429139744, + "language_loss": 0.86896169, + "learning_rate": 3.8542722471010674e-06, + "loss": 0.89032787, + "num_input_tokens_seen": 145751075, + "router_z_loss_clip": 0.60424805, + "router_z_loss_mlp": 0.19232178, + "step": 5121, + "time_per_iteration": 2.3551599979400635 + }, + { + "auxiliary_loss_clip": 0.01086903, + "auxiliary_loss_mlp": 0.01043396, + "balance_loss_clip": 1.03056777, + "balance_loss_mlp": 1.02500784, + "epoch": 0.148627473739191, + "flos": 31168116915840.0, + "grad_norm": 2.472386726353938, + "language_loss": 0.90954947, + "learning_rate": 3.8542018050008445e-06, + "loss": 0.93085241, + "num_input_tokens_seen": 145766705, + "router_z_loss_clip": 0.5637207, + "router_z_loss_mlp": 0.18395996, + "step": 5122, + "time_per_iteration": 2.4771366119384766 + }, + { + "auxiliary_loss_clip": 0.01091997, + "auxiliary_loss_mlp": 0.01049338, + "balance_loss_clip": 1.03205335, + "balance_loss_mlp": 1.02976346, + "epoch": 0.14865649120770705, + "flos": 15991512013440.0, + "grad_norm": 2.3412418036606404, + "language_loss": 0.7069934, + "learning_rate": 3.8541313465235605e-06, + "loss": 0.72840679, + "num_input_tokens_seen": 145779545, + "router_z_loss_clip": 0.59912109, + "router_z_loss_mlp": 0.19580078, + "step": 5123, + "time_per_iteration": 2.386584758758545 + }, + { + "auxiliary_loss_clip": 0.01088418, + "auxiliary_loss_mlp": 0.01038508, + "balance_loss_clip": 1.0328064, + "balance_loss_mlp": 1.02057886, + "epoch": 0.1486855086762231, + "flos": 13351029621120.0, + "grad_norm": 2.3681979275256886, + "language_loss": 0.69501913, + "learning_rate": 3.854060871669838e-06, + "loss": 0.71628833, + "num_input_tokens_seen": 145795485, + "router_z_loss_clip": 0.55639648, + "router_z_loss_mlp": 0.17913818, + "step": 5124, + "time_per_iteration": 2.7155556678771973 + }, + { + "auxiliary_loss_clip": 0.01082411, + "auxiliary_loss_mlp": 0.01038591, + "balance_loss_clip": 1.02966642, + "balance_loss_mlp": 1.02220559, + "epoch": 0.14871452614473912, + "flos": 19641471857280.0, + "grad_norm": 2.9778097959253613, + "language_loss": 0.89550966, + "learning_rate": 3.8539903804403e-06, + "loss": 0.91671968, + "num_input_tokens_seen": 145810060, + "router_z_loss_clip": 0.52832031, + "router_z_loss_mlp": 0.16387939, + "step": 5125, + "time_per_iteration": 2.4370930194854736 + }, + { + "auxiliary_loss_clip": 0.0101821, + "auxiliary_loss_mlp": 0.01002905, + "balance_loss_clip": 1.00760758, + "balance_loss_mlp": 1.00167751, + "epoch": 0.14874354361325517, + "flos": 71730348001920.0, + "grad_norm": 0.6868234508585672, + "language_loss": 0.48829234, + "learning_rate": 3.853919872835568e-06, + "loss": 0.5085035, + "num_input_tokens_seen": 145867190, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.01226807, + "step": 5126, + "time_per_iteration": 2.911524534225464 + }, + { + "auxiliary_loss_clip": 0.01091785, + "auxiliary_loss_mlp": 0.01036932, + "balance_loss_clip": 1.03421509, + "balance_loss_mlp": 1.02074337, + "epoch": 0.14877256108177123, + "flos": 32297347797120.0, + "grad_norm": 2.1243071735721792, + "language_loss": 0.82453924, + "learning_rate": 3.853849348856267e-06, + "loss": 0.84582639, + "num_input_tokens_seen": 145881805, + "router_z_loss_clip": 0.57568359, + "router_z_loss_mlp": 0.16186523, + "step": 5127, + "time_per_iteration": 2.5779736042022705 + }, + { + "auxiliary_loss_clip": 0.01098402, + "auxiliary_loss_mlp": 0.01044029, + "balance_loss_clip": 1.03371239, + "balance_loss_mlp": 1.02317977, + "epoch": 0.14880157855028728, + "flos": 14533504058880.0, + "grad_norm": 2.3147289979365935, + "language_loss": 0.82550216, + "learning_rate": 3.853778808503017e-06, + "loss": 0.84692645, + "num_input_tokens_seen": 145895125, + "router_z_loss_clip": 0.64746094, + "router_z_loss_mlp": 0.20861816, + "step": 5128, + "time_per_iteration": 2.394585609436035 + }, + { + "auxiliary_loss_clip": 0.01018577, + "auxiliary_loss_mlp": 0.01006788, + "balance_loss_clip": 1.00755823, + "balance_loss_mlp": 1.00551796, + "epoch": 0.14883059601880333, + "flos": 70208623082880.0, + "grad_norm": 0.7129266381988297, + "language_loss": 0.51017696, + "learning_rate": 3.8537082517764425e-06, + "loss": 0.53043056, + "num_input_tokens_seen": 145961350, + "router_z_loss_clip": 0.11035156, + "router_z_loss_mlp": 0.01269531, + "step": 5129, + "time_per_iteration": 3.1657118797302246 + }, + { + "auxiliary_loss_clip": 0.01018158, + "auxiliary_loss_mlp": 0.01002232, + "balance_loss_clip": 1.00726748, + "balance_loss_mlp": 1.00090277, + "epoch": 0.14885961348731935, + "flos": 72423723070080.0, + "grad_norm": 0.6806804369265246, + "language_loss": 0.50551164, + "learning_rate": 3.853637678677167e-06, + "loss": 0.52571559, + "num_input_tokens_seen": 146022580, + "router_z_loss_clip": 0.10888672, + "router_z_loss_mlp": 0.01330566, + "step": 5130, + "time_per_iteration": 3.048602819442749 + }, + { + "auxiliary_loss_clip": 0.01088469, + "auxiliary_loss_mlp": 0.01041616, + "balance_loss_clip": 1.03301954, + "balance_loss_mlp": 1.02419949, + "epoch": 0.1488886309558354, + "flos": 11538794211840.0, + "grad_norm": 2.552330157886203, + "language_loss": 0.68244815, + "learning_rate": 3.853567089205813e-06, + "loss": 0.70374894, + "num_input_tokens_seen": 146034575, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.17407227, + "step": 5131, + "time_per_iteration": 2.385911703109741 + }, + { + "auxiliary_loss_clip": 0.01092622, + "auxiliary_loss_mlp": 0.01041464, + "balance_loss_clip": 1.0318892, + "balance_loss_mlp": 1.02178204, + "epoch": 0.14891764842435146, + "flos": 25367646833280.0, + "grad_norm": 1.7867806879698116, + "language_loss": 0.70302427, + "learning_rate": 3.853496483363005e-06, + "loss": 0.72436512, + "num_input_tokens_seen": 146050335, + "router_z_loss_clip": 0.60668945, + "router_z_loss_mlp": 0.19689941, + "step": 5132, + "time_per_iteration": 2.418926477432251 + }, + { + "auxiliary_loss_clip": 0.01086942, + "auxiliary_loss_mlp": 0.01040764, + "balance_loss_clip": 1.03217244, + "balance_loss_mlp": 1.02455783, + "epoch": 0.1489466658928675, + "flos": 20445277155840.0, + "grad_norm": 2.0673115998827836, + "language_loss": 0.68932724, + "learning_rate": 3.853425861149366e-06, + "loss": 0.71060431, + "num_input_tokens_seen": 146070985, + "router_z_loss_clip": 0.54736328, + "router_z_loss_mlp": 0.16217041, + "step": 5133, + "time_per_iteration": 2.5948095321655273 + }, + { + "auxiliary_loss_clip": 0.01088654, + "auxiliary_loss_mlp": 0.0104424, + "balance_loss_clip": 1.03136325, + "balance_loss_mlp": 1.02647817, + "epoch": 0.14897568336138356, + "flos": 17704804896000.0, + "grad_norm": 2.4227925831648287, + "language_loss": 0.71889699, + "learning_rate": 3.85335522256552e-06, + "loss": 0.74022591, + "num_input_tokens_seen": 146084475, + "router_z_loss_clip": 0.57324219, + "router_z_loss_mlp": 0.17736816, + "step": 5134, + "time_per_iteration": 2.369694709777832 + }, + { + "auxiliary_loss_clip": 0.01088406, + "auxiliary_loss_mlp": 0.01043236, + "balance_loss_clip": 1.03129399, + "balance_loss_mlp": 1.02505636, + "epoch": 0.1490047008298996, + "flos": 30328944543360.0, + "grad_norm": 2.2609272210987403, + "language_loss": 0.94580936, + "learning_rate": 3.853284567612089e-06, + "loss": 0.96712571, + "num_input_tokens_seen": 146102110, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.1819458, + "step": 5135, + "time_per_iteration": 2.4986441135406494 + }, + { + "auxiliary_loss_clip": 0.01013931, + "auxiliary_loss_mlp": 0.01017533, + "balance_loss_clip": 1.00351477, + "balance_loss_mlp": 1.01627493, + "epoch": 0.14903371829841564, + "flos": 64588549328640.0, + "grad_norm": 0.7314284247777245, + "language_loss": 0.50013292, + "learning_rate": 3.8532138962897e-06, + "loss": 0.52044755, + "num_input_tokens_seen": 146160200, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01257324, + "step": 5136, + "time_per_iteration": 2.92938232421875 + }, + { + "auxiliary_loss_clip": 0.01084718, + "auxiliary_loss_mlp": 0.01043881, + "balance_loss_clip": 1.02785444, + "balance_loss_mlp": 1.02503419, + "epoch": 0.1490627357669317, + "flos": 29416559316480.0, + "grad_norm": 2.381100245570692, + "language_loss": 0.78615403, + "learning_rate": 3.8531432085989764e-06, + "loss": 0.80743998, + "num_input_tokens_seen": 146175330, + "router_z_loss_clip": 0.56811523, + "router_z_loss_mlp": 0.18835449, + "step": 5137, + "time_per_iteration": 2.840623140335083 + }, + { + "auxiliary_loss_clip": 0.01088897, + "auxiliary_loss_mlp": 0.01044752, + "balance_loss_clip": 1.02951491, + "balance_loss_mlp": 1.02619112, + "epoch": 0.14909175323544774, + "flos": 74744121317760.0, + "grad_norm": 2.461891050818544, + "language_loss": 0.85218096, + "learning_rate": 3.8530725045405415e-06, + "loss": 0.87351739, + "num_input_tokens_seen": 146201930, + "router_z_loss_clip": 0.59375, + "router_z_loss_mlp": 0.1854248, + "step": 5138, + "time_per_iteration": 2.7912986278533936 + }, + { + "auxiliary_loss_clip": 0.01014322, + "auxiliary_loss_mlp": 0.01010578, + "balance_loss_clip": 1.00387359, + "balance_loss_mlp": 1.0092783, + "epoch": 0.1491207707039638, + "flos": 74769606610560.0, + "grad_norm": 0.701957033005356, + "language_loss": 0.52370441, + "learning_rate": 3.853001784115021e-06, + "loss": 0.54395342, + "num_input_tokens_seen": 146264080, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01300049, + "step": 5139, + "time_per_iteration": 3.1642677783966064 + }, + { + "auxiliary_loss_clip": 0.01091427, + "auxiliary_loss_mlp": 0.01047568, + "balance_loss_clip": 1.03158104, + "balance_loss_mlp": 1.02963877, + "epoch": 0.14914978817247984, + "flos": 13144519595520.0, + "grad_norm": 2.3487869584831724, + "language_loss": 0.83052343, + "learning_rate": 3.8529310473230385e-06, + "loss": 0.85191333, + "num_input_tokens_seen": 146276855, + "router_z_loss_clip": 0.59863281, + "router_z_loss_mlp": 0.17932129, + "step": 5140, + "time_per_iteration": 2.388540029525757 + }, + { + "auxiliary_loss_clip": 0.01084662, + "auxiliary_loss_mlp": 0.01036055, + "balance_loss_clip": 1.027143, + "balance_loss_mlp": 1.01779199, + "epoch": 0.1491788056409959, + "flos": 55250471623680.0, + "grad_norm": 1.6020075761114922, + "language_loss": 0.81765914, + "learning_rate": 3.852860294165219e-06, + "loss": 0.83886629, + "num_input_tokens_seen": 146300830, + "router_z_loss_clip": 0.57543945, + "router_z_loss_mlp": 0.18261719, + "step": 5141, + "time_per_iteration": 2.7233564853668213 + }, + { + "auxiliary_loss_clip": 0.01081221, + "auxiliary_loss_mlp": 0.01038786, + "balance_loss_clip": 1.02869201, + "balance_loss_mlp": 1.02196574, + "epoch": 0.14920782310951192, + "flos": 33432269230080.0, + "grad_norm": 2.186497411934809, + "language_loss": 0.95494026, + "learning_rate": 3.852789524642188e-06, + "loss": 0.97614032, + "num_input_tokens_seen": 146317850, + "router_z_loss_clip": 0.52441406, + "router_z_loss_mlp": 0.16833496, + "step": 5142, + "time_per_iteration": 2.5365333557128906 + }, + { + "auxiliary_loss_clip": 0.01088696, + "auxiliary_loss_mlp": 0.01038242, + "balance_loss_clip": 1.03126264, + "balance_loss_mlp": 1.021541, + "epoch": 0.14923684057802797, + "flos": 71738170012800.0, + "grad_norm": 2.077148683468503, + "language_loss": 0.68010795, + "learning_rate": 3.8527187387545695e-06, + "loss": 0.70137727, + "num_input_tokens_seen": 146350005, + "router_z_loss_clip": 0.57446289, + "router_z_loss_mlp": 0.16699219, + "step": 5143, + "time_per_iteration": 2.795603036880493 + }, + { + "auxiliary_loss_clip": 0.01095024, + "auxiliary_loss_mlp": 0.0104068, + "balance_loss_clip": 1.03190351, + "balance_loss_mlp": 1.02059281, + "epoch": 0.14926585804654402, + "flos": 36647455512960.0, + "grad_norm": 2.347172471442363, + "language_loss": 0.7722888, + "learning_rate": 3.85264793650299e-06, + "loss": 0.7936458, + "num_input_tokens_seen": 146370360, + "router_z_loss_clip": 0.63085938, + "router_z_loss_mlp": 0.20068359, + "step": 5144, + "time_per_iteration": 2.578758478164673 + }, + { + "auxiliary_loss_clip": 0.01092601, + "auxiliary_loss_mlp": 0.01036681, + "balance_loss_clip": 1.03415489, + "balance_loss_mlp": 1.01961064, + "epoch": 0.14929487551506007, + "flos": 30583601066880.0, + "grad_norm": 2.78577218007036, + "language_loss": 0.79115963, + "learning_rate": 3.8525771178880735e-06, + "loss": 0.81245244, + "num_input_tokens_seen": 146386680, + "router_z_loss_clip": 0.5847168, + "router_z_loss_mlp": 0.17071533, + "step": 5145, + "time_per_iteration": 2.4870007038116455 + }, + { + "auxiliary_loss_clip": 0.0108605, + "auxiliary_loss_mlp": 0.01035578, + "balance_loss_clip": 1.02874732, + "balance_loss_mlp": 1.0180546, + "epoch": 0.14932389298357612, + "flos": 21828361599360.0, + "grad_norm": 2.5984657331785996, + "language_loss": 0.84524393, + "learning_rate": 3.852506282910447e-06, + "loss": 0.86646026, + "num_input_tokens_seen": 146403400, + "router_z_loss_clip": 0.57299805, + "router_z_loss_mlp": 0.1751709, + "step": 5146, + "time_per_iteration": 2.4777727127075195 + }, + { + "auxiliary_loss_clip": 0.0101978, + "auxiliary_loss_mlp": 0.01010678, + "balance_loss_clip": 1.00950432, + "balance_loss_mlp": 1.00939691, + "epoch": 0.14935291045209215, + "flos": 50866391422080.0, + "grad_norm": 0.6833401517663894, + "language_loss": 0.50944984, + "learning_rate": 3.852435431570735e-06, + "loss": 0.52975446, + "num_input_tokens_seen": 146456605, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01281738, + "step": 5147, + "time_per_iteration": 2.809997081756592 + }, + { + "auxiliary_loss_clip": 0.01095133, + "auxiliary_loss_mlp": 0.01048425, + "balance_loss_clip": 1.03232121, + "balance_loss_mlp": 1.02781391, + "epoch": 0.1493819279206082, + "flos": 16829113374720.0, + "grad_norm": 2.303408014257408, + "language_loss": 0.78397357, + "learning_rate": 3.852364563869564e-06, + "loss": 0.80540907, + "num_input_tokens_seen": 146470630, + "router_z_loss_clip": 0.62890625, + "router_z_loss_mlp": 0.20617676, + "step": 5148, + "time_per_iteration": 2.419173002243042 + }, + { + "auxiliary_loss_clip": 0.01087658, + "auxiliary_loss_mlp": 0.01035478, + "balance_loss_clip": 1.02901375, + "balance_loss_mlp": 1.01826417, + "epoch": 0.14941094538912425, + "flos": 55879117297920.0, + "grad_norm": 1.8972587024242626, + "language_loss": 0.78545541, + "learning_rate": 3.8522936798075595e-06, + "loss": 0.8066867, + "num_input_tokens_seen": 146491775, + "router_z_loss_clip": 0.58642578, + "router_z_loss_mlp": 0.17218018, + "step": 5149, + "time_per_iteration": 2.5981271266937256 + }, + { + "auxiliary_loss_clip": 0.01020494, + "auxiliary_loss_mlp": 0.01005261, + "balance_loss_clip": 1.01037169, + "balance_loss_mlp": 1.00406313, + "epoch": 0.1494399628576403, + "flos": 57578618327040.0, + "grad_norm": 0.6687173928248559, + "language_loss": 0.45547906, + "learning_rate": 3.852222779385347e-06, + "loss": 0.47573662, + "num_input_tokens_seen": 146547960, + "router_z_loss_clip": 0.10107422, + "router_z_loss_mlp": 0.01196289, + "step": 5150, + "time_per_iteration": 3.1588737964630127 + }, + { + "auxiliary_loss_clip": 0.01018474, + "auxiliary_loss_mlp": 0.01001317, + "balance_loss_clip": 1.00802147, + "balance_loss_mlp": 0.9999758, + "epoch": 0.14946898032615635, + "flos": 74662632604800.0, + "grad_norm": 0.6801619367380125, + "language_loss": 0.46945554, + "learning_rate": 3.852151862603554e-06, + "loss": 0.48965347, + "num_input_tokens_seen": 146615260, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01342773, + "step": 5151, + "time_per_iteration": 3.179306983947754 + }, + { + "auxiliary_loss_clip": 0.01019065, + "auxiliary_loss_mlp": 0.01000811, + "balance_loss_clip": 1.00865841, + "balance_loss_mlp": 0.99964863, + "epoch": 0.1494979977946724, + "flos": 72321985768320.0, + "grad_norm": 0.8308734867648371, + "language_loss": 0.53280234, + "learning_rate": 3.852080929462807e-06, + "loss": 0.55300105, + "num_input_tokens_seen": 146672820, + "router_z_loss_clip": 0.10400391, + "router_z_loss_mlp": 0.01159668, + "step": 5152, + "time_per_iteration": 2.9975905418395996 + }, + { + "auxiliary_loss_clip": 0.01084991, + "auxiliary_loss_mlp": 0.01035706, + "balance_loss_clip": 1.03033006, + "balance_loss_mlp": 1.01881969, + "epoch": 0.14952701526318843, + "flos": 16208951160960.0, + "grad_norm": 2.4523690911235563, + "language_loss": 0.85226238, + "learning_rate": 3.852009979963731e-06, + "loss": 0.87346935, + "num_input_tokens_seen": 146685325, + "router_z_loss_clip": 0.54638672, + "router_z_loss_mlp": 0.16912842, + "step": 5153, + "time_per_iteration": 2.441096782684326 + }, + { + "auxiliary_loss_clip": 0.01015851, + "auxiliary_loss_mlp": 0.01009043, + "balance_loss_clip": 1.00556946, + "balance_loss_mlp": 1.00780296, + "epoch": 0.14955603273170448, + "flos": 64123121727360.0, + "grad_norm": 0.6248640148954525, + "language_loss": 0.46475846, + "learning_rate": 3.851939014106954e-06, + "loss": 0.48500741, + "num_input_tokens_seen": 146747815, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01239014, + "step": 5154, + "time_per_iteration": 3.000499963760376 + }, + { + "auxiliary_loss_clip": 0.01094543, + "auxiliary_loss_mlp": 0.01046017, + "balance_loss_clip": 1.03007329, + "balance_loss_mlp": 1.02563214, + "epoch": 0.14958505020022053, + "flos": 28470133647360.0, + "grad_norm": 4.233682480128776, + "language_loss": 1.20278668, + "learning_rate": 3.851868031893101e-06, + "loss": 1.22419238, + "num_input_tokens_seen": 146762335, + "router_z_loss_clip": 0.64453125, + "router_z_loss_mlp": 0.20397949, + "step": 5155, + "time_per_iteration": 2.523554563522339 + }, + { + "auxiliary_loss_clip": 0.01088977, + "auxiliary_loss_mlp": 0.01047819, + "balance_loss_clip": 1.02981651, + "balance_loss_mlp": 1.02978826, + "epoch": 0.14961406766873658, + "flos": 58640328771840.0, + "grad_norm": 2.8017406828374734, + "language_loss": 0.91622233, + "learning_rate": 3.851797033322801e-06, + "loss": 0.9375903, + "num_input_tokens_seen": 146780740, + "router_z_loss_clip": 0.59228516, + "router_z_loss_mlp": 0.18017578, + "step": 5156, + "time_per_iteration": 2.648486614227295 + }, + { + "auxiliary_loss_clip": 0.0109663, + "auxiliary_loss_mlp": 0.01045906, + "balance_loss_clip": 1.03178656, + "balance_loss_mlp": 1.02659416, + "epoch": 0.14964308513725263, + "flos": 16538391550080.0, + "grad_norm": 2.7460433201811276, + "language_loss": 0.94794047, + "learning_rate": 3.85172601839668e-06, + "loss": 0.96936578, + "num_input_tokens_seen": 146794685, + "router_z_loss_clip": 0.64892578, + "router_z_loss_mlp": 0.1932373, + "step": 5157, + "time_per_iteration": 2.4053232669830322 + }, + { + "auxiliary_loss_clip": 0.01084003, + "auxiliary_loss_mlp": 0.01036402, + "balance_loss_clip": 1.02866364, + "balance_loss_mlp": 1.01943898, + "epoch": 0.1496721026057687, + "flos": 23433668046720.0, + "grad_norm": 3.719920205328222, + "language_loss": 0.91420853, + "learning_rate": 3.851654987115365e-06, + "loss": 0.93541265, + "num_input_tokens_seen": 146809910, + "router_z_loss_clip": 0.55297852, + "router_z_loss_mlp": 0.16943359, + "step": 5158, + "time_per_iteration": 2.3905701637268066 + }, + { + "auxiliary_loss_clip": 0.01096758, + "auxiliary_loss_mlp": 0.01042353, + "balance_loss_clip": 1.03155911, + "balance_loss_mlp": 1.02270079, + "epoch": 0.1497011200742847, + "flos": 26750382163200.0, + "grad_norm": 2.388924000764724, + "language_loss": 0.92816281, + "learning_rate": 3.851583939479485e-06, + "loss": 0.94955397, + "num_input_tokens_seen": 146821945, + "router_z_loss_clip": 0.65185547, + "router_z_loss_mlp": 0.19647217, + "step": 5159, + "time_per_iteration": 2.535996913909912 + }, + { + "auxiliary_loss_clip": 0.0109032, + "auxiliary_loss_mlp": 0.01037013, + "balance_loss_clip": 1.03236222, + "balance_loss_mlp": 1.01898885, + "epoch": 0.14973013754280076, + "flos": 13217034222720.0, + "grad_norm": 2.381608038367611, + "language_loss": 0.84100056, + "learning_rate": 3.851512875489666e-06, + "loss": 0.86227393, + "num_input_tokens_seen": 146835940, + "router_z_loss_clip": 0.57983398, + "router_z_loss_mlp": 0.18029785, + "step": 5160, + "time_per_iteration": 2.3550031185150146 + }, + { + "auxiliary_loss_clip": 0.0108825, + "auxiliary_loss_mlp": 0.01038709, + "balance_loss_clip": 1.0312717, + "balance_loss_mlp": 1.01993394, + "epoch": 0.1497591550113168, + "flos": 23468930386560.0, + "grad_norm": 2.215436450961944, + "language_loss": 0.98155248, + "learning_rate": 3.851441795146535e-06, + "loss": 1.00282216, + "num_input_tokens_seen": 146850805, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.18780518, + "step": 5161, + "time_per_iteration": 2.4669525623321533 + }, + { + "auxiliary_loss_clip": 0.01019102, + "auxiliary_loss_mlp": 0.01013421, + "balance_loss_clip": 1.00884581, + "balance_loss_mlp": 1.01245582, + "epoch": 0.14978817247983287, + "flos": 60291543657600.0, + "grad_norm": 0.6357036595980972, + "language_loss": 0.46119595, + "learning_rate": 3.851370698450722e-06, + "loss": 0.48152119, + "num_input_tokens_seen": 146912870, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.00964355, + "step": 5162, + "time_per_iteration": 3.045029640197754 + }, + { + "auxiliary_loss_clip": 0.0101887, + "auxiliary_loss_mlp": 0.01008417, + "balance_loss_clip": 1.00847769, + "balance_loss_mlp": 1.00745785, + "epoch": 0.14981718994834892, + "flos": 61063438106880.0, + "grad_norm": 0.6529745254068721, + "language_loss": 0.47772464, + "learning_rate": 3.851299585402854e-06, + "loss": 0.49799752, + "num_input_tokens_seen": 146977635, + "router_z_loss_clip": 0.10400391, + "router_z_loss_mlp": 0.00958252, + "step": 5163, + "time_per_iteration": 3.2569401264190674 + }, + { + "auxiliary_loss_clip": 0.01096017, + "auxiliary_loss_mlp": 0.01036639, + "balance_loss_clip": 1.03377724, + "balance_loss_mlp": 1.01643848, + "epoch": 0.14984620741686494, + "flos": 33102514638720.0, + "grad_norm": 2.4359115883445943, + "language_loss": 1.07606435, + "learning_rate": 3.851228456003558e-06, + "loss": 1.09739089, + "num_input_tokens_seen": 146995155, + "router_z_loss_clip": 0.62304688, + "router_z_loss_mlp": 0.20214844, + "step": 5164, + "time_per_iteration": 2.48885440826416 + }, + { + "auxiliary_loss_clip": 0.0108797, + "auxiliary_loss_mlp": 0.01037032, + "balance_loss_clip": 1.03029764, + "balance_loss_mlp": 1.01928222, + "epoch": 0.149875224885381, + "flos": 21972587892480.0, + "grad_norm": 2.188929475347017, + "language_loss": 0.90687537, + "learning_rate": 3.8511573102534645e-06, + "loss": 0.92812538, + "num_input_tokens_seen": 147010330, + "router_z_loss_clip": 0.57739258, + "router_z_loss_mlp": 0.1776123, + "step": 5165, + "time_per_iteration": 2.4496374130249023 + }, + { + "auxiliary_loss_clip": 0.01090521, + "auxiliary_loss_mlp": 0.0103405, + "balance_loss_clip": 1.03192413, + "balance_loss_mlp": 1.01639533, + "epoch": 0.14990424235389704, + "flos": 17777040232320.0, + "grad_norm": 2.4624151069346514, + "language_loss": 0.80697024, + "learning_rate": 3.851086148153199e-06, + "loss": 0.82821596, + "num_input_tokens_seen": 147025005, + "router_z_loss_clip": 0.58642578, + "router_z_loss_mlp": 0.17663574, + "step": 5166, + "time_per_iteration": 2.3701343536376953 + }, + { + "auxiliary_loss_clip": 0.01091924, + "auxiliary_loss_mlp": 0.01037601, + "balance_loss_clip": 1.03104138, + "balance_loss_mlp": 1.01839089, + "epoch": 0.1499332598224131, + "flos": 16389836248320.0, + "grad_norm": 2.5580945888669233, + "language_loss": 0.88909572, + "learning_rate": 3.851014969703393e-06, + "loss": 0.91039097, + "num_input_tokens_seen": 147037045, + "router_z_loss_clip": 0.60913086, + "router_z_loss_mlp": 0.19207764, + "step": 5167, + "time_per_iteration": 2.376648187637329 + }, + { + "auxiliary_loss_clip": 0.01021069, + "auxiliary_loss_mlp": 0.01007949, + "balance_loss_clip": 1.01054466, + "balance_loss_mlp": 1.00685799, + "epoch": 0.14996227729092915, + "flos": 60974654785920.0, + "grad_norm": 0.6947093888527677, + "language_loss": 0.47785914, + "learning_rate": 3.850943774904672e-06, + "loss": 0.49814934, + "num_input_tokens_seen": 147094255, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01092529, + "step": 5168, + "time_per_iteration": 2.843290090560913 + }, + { + "auxiliary_loss_clip": 0.01018402, + "auxiliary_loss_mlp": 0.01004315, + "balance_loss_clip": 1.00801182, + "balance_loss_mlp": 1.00321805, + "epoch": 0.1499912947594452, + "flos": 62911775594880.0, + "grad_norm": 0.6987168349214327, + "language_loss": 0.50144005, + "learning_rate": 3.850872563757669e-06, + "loss": 0.52166724, + "num_input_tokens_seen": 147153650, + "router_z_loss_clip": 0.10400391, + "router_z_loss_mlp": 0.01098633, + "step": 5169, + "time_per_iteration": 2.9418222904205322 + }, + { + "auxiliary_loss_clip": 0.01087964, + "auxiliary_loss_mlp": 0.01042562, + "balance_loss_clip": 1.02979243, + "balance_loss_mlp": 1.02549171, + "epoch": 0.15002031222796122, + "flos": 22886369573760.0, + "grad_norm": 2.6123915195099623, + "language_loss": 0.66068637, + "learning_rate": 3.850801336263008e-06, + "loss": 0.68199158, + "num_input_tokens_seen": 147170410, + "router_z_loss_clip": 0.58129883, + "router_z_loss_mlp": 0.17071533, + "step": 5170, + "time_per_iteration": 2.4063353538513184 + }, + { + "auxiliary_loss_clip": 0.01092625, + "auxiliary_loss_mlp": 0.0103818, + "balance_loss_clip": 1.0300492, + "balance_loss_mlp": 1.01891613, + "epoch": 0.15004932969647727, + "flos": 25257111868800.0, + "grad_norm": 2.1475492519719492, + "language_loss": 0.7664237, + "learning_rate": 3.850730092421322e-06, + "loss": 0.78773177, + "num_input_tokens_seen": 147186820, + "router_z_loss_clip": 0.625, + "router_z_loss_mlp": 0.19274902, + "step": 5171, + "time_per_iteration": 2.4465363025665283 + }, + { + "auxiliary_loss_clip": 0.01094509, + "auxiliary_loss_mlp": 0.01040313, + "balance_loss_clip": 1.03233242, + "balance_loss_mlp": 1.02234817, + "epoch": 0.15007834716499333, + "flos": 24269628574080.0, + "grad_norm": 2.4464899970879244, + "language_loss": 0.98558301, + "learning_rate": 3.850658832233239e-06, + "loss": 1.00693119, + "num_input_tokens_seen": 147202715, + "router_z_loss_clip": 0.62158203, + "router_z_loss_mlp": 0.17974854, + "step": 5172, + "time_per_iteration": 2.452178955078125 + }, + { + "auxiliary_loss_clip": 0.01093634, + "auxiliary_loss_mlp": 0.0104054, + "balance_loss_clip": 1.03093493, + "balance_loss_mlp": 1.02146697, + "epoch": 0.15010736463350938, + "flos": 16282617863040.0, + "grad_norm": 2.6673754485725367, + "language_loss": 0.82232118, + "learning_rate": 3.850587555699388e-06, + "loss": 0.84366292, + "num_input_tokens_seen": 147217940, + "router_z_loss_clip": 0.62646484, + "router_z_loss_mlp": 0.19067383, + "step": 5173, + "time_per_iteration": 4.813257694244385 + }, + { + "auxiliary_loss_clip": 0.01017044, + "auxiliary_loss_mlp": 0.01002404, + "balance_loss_clip": 1.00720572, + "balance_loss_mlp": 1.00131357, + "epoch": 0.15013638210202543, + "flos": 74782628547840.0, + "grad_norm": 0.6690509088198788, + "language_loss": 0.49306351, + "learning_rate": 3.8505162628203986e-06, + "loss": 0.51325804, + "num_input_tokens_seen": 147285345, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01092529, + "step": 5174, + "time_per_iteration": 3.1552157402038574 + }, + { + "auxiliary_loss_clip": 0.01088831, + "auxiliary_loss_mlp": 0.01049495, + "balance_loss_clip": 1.03157711, + "balance_loss_mlp": 1.03238201, + "epoch": 0.15016539957054145, + "flos": 13252436208000.0, + "grad_norm": 3.430756735748146, + "language_loss": 1.01004267, + "learning_rate": 3.850444953596902e-06, + "loss": 1.03142583, + "num_input_tokens_seen": 147297045, + "router_z_loss_clip": 0.57202148, + "router_z_loss_mlp": 0.17114258, + "step": 5175, + "time_per_iteration": 2.3345298767089844 + }, + { + "auxiliary_loss_clip": 0.01082342, + "auxiliary_loss_mlp": 0.01034741, + "balance_loss_clip": 1.02805841, + "balance_loss_mlp": 1.02034092, + "epoch": 0.1501944170390575, + "flos": 26861440798080.0, + "grad_norm": 2.2895117503204108, + "language_loss": 0.92587268, + "learning_rate": 3.850373628029525e-06, + "loss": 0.9470436, + "num_input_tokens_seen": 147311040, + "router_z_loss_clip": 0.54248047, + "router_z_loss_mlp": 0.14398193, + "step": 5176, + "time_per_iteration": 4.764918565750122 + }, + { + "auxiliary_loss_clip": 0.01090551, + "auxiliary_loss_mlp": 0.01040692, + "balance_loss_clip": 1.03106701, + "balance_loss_mlp": 1.02128434, + "epoch": 0.15022343450757356, + "flos": 25183549900800.0, + "grad_norm": 2.1730404832389527, + "language_loss": 0.7466054, + "learning_rate": 3.850302286118901e-06, + "loss": 0.76791787, + "num_input_tokens_seen": 147326720, + "router_z_loss_clip": 0.59423828, + "router_z_loss_mlp": 0.19396973, + "step": 5177, + "time_per_iteration": 2.4354605674743652 + }, + { + "auxiliary_loss_clip": 0.01080189, + "auxiliary_loss_mlp": 0.01040747, + "balance_loss_clip": 1.02773273, + "balance_loss_mlp": 1.02489829, + "epoch": 0.1502524519760896, + "flos": 22011865038720.0, + "grad_norm": 2.5178800470575915, + "language_loss": 0.8245796, + "learning_rate": 3.8502309278656576e-06, + "loss": 0.84578896, + "num_input_tokens_seen": 147344865, + "router_z_loss_clip": 0.52441406, + "router_z_loss_mlp": 0.1583252, + "step": 5178, + "time_per_iteration": 2.3819215297698975 + }, + { + "auxiliary_loss_clip": 0.01080768, + "auxiliary_loss_mlp": 0.01046529, + "balance_loss_clip": 1.02820158, + "balance_loss_mlp": 1.03153265, + "epoch": 0.15028146944460566, + "flos": 26352686332800.0, + "grad_norm": 2.4051192632049627, + "language_loss": 0.75705773, + "learning_rate": 3.8501595532704256e-06, + "loss": 0.77833068, + "num_input_tokens_seen": 147359405, + "router_z_loss_clip": 0.52563477, + "router_z_loss_mlp": 0.14990234, + "step": 5179, + "time_per_iteration": 2.3532509803771973 + }, + { + "auxiliary_loss_clip": 0.01084601, + "auxiliary_loss_mlp": 0.01038437, + "balance_loss_clip": 1.02882075, + "balance_loss_mlp": 1.02130628, + "epoch": 0.1503104869131217, + "flos": 19746805029120.0, + "grad_norm": 3.975944982334316, + "language_loss": 0.76299924, + "learning_rate": 3.850088162333837e-06, + "loss": 0.78422958, + "num_input_tokens_seen": 147369800, + "router_z_loss_clip": 0.55761719, + "router_z_loss_mlp": 0.17138672, + "step": 5180, + "time_per_iteration": 2.3539891242980957 + }, + { + "auxiliary_loss_clip": 0.01091366, + "auxiliary_loss_mlp": 0.01050007, + "balance_loss_clip": 1.03084326, + "balance_loss_mlp": 1.03197682, + "epoch": 0.15033950438163773, + "flos": 35726761382400.0, + "grad_norm": 2.053019218120088, + "language_loss": 0.8950305, + "learning_rate": 3.8500167550565194e-06, + "loss": 0.9164443, + "num_input_tokens_seen": 147389580, + "router_z_loss_clip": 0.60546875, + "router_z_loss_mlp": 0.18029785, + "step": 5181, + "time_per_iteration": 5.055524587631226 + }, + { + "auxiliary_loss_clip": 0.01087412, + "auxiliary_loss_mlp": 0.0103911, + "balance_loss_clip": 1.03029597, + "balance_loss_mlp": 1.0215807, + "epoch": 0.15036852185015379, + "flos": 30802471580160.0, + "grad_norm": 2.2088756118111883, + "language_loss": 0.73460013, + "learning_rate": 3.8499453314391065e-06, + "loss": 0.75586534, + "num_input_tokens_seen": 147405005, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.175354, + "step": 5182, + "time_per_iteration": 2.4779858589172363 + }, + { + "auxiliary_loss_clip": 0.01099503, + "auxiliary_loss_mlp": 0.01048037, + "balance_loss_clip": 1.03094697, + "balance_loss_mlp": 1.02668643, + "epoch": 0.15039753931866984, + "flos": 26862488138880.0, + "grad_norm": 3.8643713401588387, + "language_loss": 0.90887934, + "learning_rate": 3.849873891482227e-06, + "loss": 0.93035477, + "num_input_tokens_seen": 147421210, + "router_z_loss_clip": 0.68603516, + "router_z_loss_mlp": 0.21350098, + "step": 5183, + "time_per_iteration": 2.413177251815796 + }, + { + "auxiliary_loss_clip": 0.01086749, + "auxiliary_loss_mlp": 0.01043884, + "balance_loss_clip": 1.03069401, + "balance_loss_mlp": 1.02782607, + "epoch": 0.1504265567871859, + "flos": 12450969970560.0, + "grad_norm": 2.6582970747821038, + "language_loss": 0.788297, + "learning_rate": 3.849802435186513e-06, + "loss": 0.80960333, + "num_input_tokens_seen": 147433300, + "router_z_loss_clip": 0.56054688, + "router_z_loss_mlp": 0.16046143, + "step": 5184, + "time_per_iteration": 4.934529542922974 + }, + { + "auxiliary_loss_clip": 0.01016086, + "auxiliary_loss_mlp": 0.01021651, + "balance_loss_clip": 1.00643229, + "balance_loss_mlp": 1.02070975, + "epoch": 0.15045557425570194, + "flos": 74766359854080.0, + "grad_norm": 0.6958461093359849, + "language_loss": 0.47897917, + "learning_rate": 3.849730962552596e-06, + "loss": 0.49935654, + "num_input_tokens_seen": 147493310, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.00939941, + "step": 5185, + "time_per_iteration": 3.0402894020080566 + }, + { + "auxiliary_loss_clip": 0.01091051, + "auxiliary_loss_mlp": 0.01037152, + "balance_loss_clip": 1.030532, + "balance_loss_mlp": 1.01989603, + "epoch": 0.150484591724218, + "flos": 16428380256000.0, + "grad_norm": 2.274530482141872, + "language_loss": 0.74003488, + "learning_rate": 3.849659473581106e-06, + "loss": 0.7613169, + "num_input_tokens_seen": 147507690, + "router_z_loss_clip": 0.60595703, + "router_z_loss_mlp": 0.17260742, + "step": 5186, + "time_per_iteration": 2.3561553955078125 + }, + { + "auxiliary_loss_clip": 0.01086547, + "auxiliary_loss_mlp": 0.01031795, + "balance_loss_clip": 1.03001237, + "balance_loss_mlp": 1.01580906, + "epoch": 0.15051360919273402, + "flos": 14238278668800.0, + "grad_norm": 3.9709379196465546, + "language_loss": 0.75380975, + "learning_rate": 3.849587968272675e-06, + "loss": 0.77499318, + "num_input_tokens_seen": 147521145, + "router_z_loss_clip": 0.56542969, + "router_z_loss_mlp": 0.15966797, + "step": 5187, + "time_per_iteration": 2.6493303775787354 + }, + { + "auxiliary_loss_clip": 0.01085437, + "auxiliary_loss_mlp": 0.01031883, + "balance_loss_clip": 1.03371489, + "balance_loss_mlp": 1.01734495, + "epoch": 0.15054262666125007, + "flos": 40507976966400.0, + "grad_norm": 1.9946250675324226, + "language_loss": 0.71856827, + "learning_rate": 3.849516446627935e-06, + "loss": 0.7397415, + "num_input_tokens_seen": 147544010, + "router_z_loss_clip": 0.51733398, + "router_z_loss_mlp": 0.14520264, + "step": 5188, + "time_per_iteration": 2.584662675857544 + }, + { + "auxiliary_loss_clip": 0.01018068, + "auxiliary_loss_mlp": 0.01011328, + "balance_loss_clip": 1.00816417, + "balance_loss_mlp": 1.01014781, + "epoch": 0.15057164412976612, + "flos": 66599441573760.0, + "grad_norm": 0.6758471370789816, + "language_loss": 0.51803416, + "learning_rate": 3.849444908647517e-06, + "loss": 0.53832811, + "num_input_tokens_seen": 147603900, + "router_z_loss_clip": 0.09912109, + "router_z_loss_mlp": 0.01177979, + "step": 5189, + "time_per_iteration": 2.977813959121704 + }, + { + "auxiliary_loss_clip": 0.01094418, + "auxiliary_loss_mlp": 0.01032653, + "balance_loss_clip": 1.03383565, + "balance_loss_mlp": 1.01539195, + "epoch": 0.15060066159828217, + "flos": 9201114840960.0, + "grad_norm": 3.834239088899798, + "language_loss": 0.83694524, + "learning_rate": 3.8493733543320535e-06, + "loss": 0.85821593, + "num_input_tokens_seen": 147614415, + "router_z_loss_clip": 0.60546875, + "router_z_loss_mlp": 0.17285156, + "step": 5190, + "time_per_iteration": 2.352587938308716 + }, + { + "auxiliary_loss_clip": 0.01098682, + "auxiliary_loss_mlp": 0.01049103, + "balance_loss_clip": 1.03640449, + "balance_loss_mlp": 1.02980351, + "epoch": 0.15062967906679822, + "flos": 14531234820480.0, + "grad_norm": 2.397762063443923, + "language_loss": 0.87968183, + "learning_rate": 3.849301783682176e-06, + "loss": 0.90115964, + "num_input_tokens_seen": 147626310, + "router_z_loss_clip": 0.62255859, + "router_z_loss_mlp": 0.19281006, + "step": 5191, + "time_per_iteration": 2.372591972351074 + }, + { + "auxiliary_loss_clip": 0.01020457, + "auxiliary_loss_mlp": 0.01002472, + "balance_loss_clip": 1.01040936, + "balance_loss_mlp": 1.00141692, + "epoch": 0.15065869653531425, + "flos": 67943213959680.0, + "grad_norm": 0.6838155440196001, + "language_loss": 0.51227307, + "learning_rate": 3.849230196698516e-06, + "loss": 0.53250235, + "num_input_tokens_seen": 147687625, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.01055908, + "step": 5192, + "time_per_iteration": 3.1828300952911377 + }, + { + "auxiliary_loss_clip": 0.01093151, + "auxiliary_loss_mlp": 0.01064876, + "balance_loss_clip": 1.03424311, + "balance_loss_mlp": 1.04365671, + "epoch": 0.1506877140038303, + "flos": 12158886602880.0, + "grad_norm": 4.071025415350418, + "language_loss": 0.83991373, + "learning_rate": 3.849158593381707e-06, + "loss": 0.86149395, + "num_input_tokens_seen": 147699515, + "router_z_loss_clip": 0.58959961, + "router_z_loss_mlp": 0.21228027, + "step": 5193, + "time_per_iteration": 2.3563215732574463 + }, + { + "auxiliary_loss_clip": 0.01097412, + "auxiliary_loss_mlp": 0.01062372, + "balance_loss_clip": 1.03468573, + "balance_loss_mlp": 1.04102182, + "epoch": 0.15071673147234635, + "flos": 31569897375360.0, + "grad_norm": 2.1342343045433005, + "language_loss": 0.90070045, + "learning_rate": 3.849086973732382e-06, + "loss": 0.92229831, + "num_input_tokens_seen": 147719435, + "router_z_loss_clip": 0.62695312, + "router_z_loss_mlp": 0.21350098, + "step": 5194, + "time_per_iteration": 2.5598554611206055 + }, + { + "auxiliary_loss_clip": 0.01099292, + "auxiliary_loss_mlp": 0.01055654, + "balance_loss_clip": 1.03609288, + "balance_loss_mlp": 1.03481638, + "epoch": 0.1507457489408624, + "flos": 22738687056000.0, + "grad_norm": 2.304119208926594, + "language_loss": 0.74326885, + "learning_rate": 3.8490153377511725e-06, + "loss": 0.76481831, + "num_input_tokens_seen": 147735075, + "router_z_loss_clip": 0.63232422, + "router_z_loss_mlp": 0.20849609, + "step": 5195, + "time_per_iteration": 2.4279873371124268 + }, + { + "auxiliary_loss_clip": 0.01021467, + "auxiliary_loss_mlp": 0.01014784, + "balance_loss_clip": 1.01076794, + "balance_loss_mlp": 1.01352608, + "epoch": 0.15077476640937845, + "flos": 61958018540160.0, + "grad_norm": 0.6752883713617808, + "language_loss": 0.49453026, + "learning_rate": 3.84894368543871e-06, + "loss": 0.51489276, + "num_input_tokens_seen": 147793500, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01257324, + "step": 5196, + "time_per_iteration": 2.9251348972320557 + }, + { + "auxiliary_loss_clip": 0.01095624, + "auxiliary_loss_mlp": 0.01061902, + "balance_loss_clip": 1.03630686, + "balance_loss_mlp": 1.04362679, + "epoch": 0.1508037838778945, + "flos": 35402487874560.0, + "grad_norm": 2.1780728986636624, + "language_loss": 0.97308075, + "learning_rate": 3.84887201679563e-06, + "loss": 0.99465609, + "num_input_tokens_seen": 147813355, + "router_z_loss_clip": 0.59277344, + "router_z_loss_mlp": 0.18286133, + "step": 5197, + "time_per_iteration": 2.529886245727539 + }, + { + "auxiliary_loss_clip": 0.01019122, + "auxiliary_loss_mlp": 0.01014251, + "balance_loss_clip": 1.00875139, + "balance_loss_mlp": 1.0130291, + "epoch": 0.15083280134641053, + "flos": 55838406919680.0, + "grad_norm": 0.6789666766956484, + "language_loss": 0.51885772, + "learning_rate": 3.848800331822563e-06, + "loss": 0.53919148, + "num_input_tokens_seen": 147871345, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01220703, + "step": 5198, + "time_per_iteration": 2.95247483253479 + }, + { + "auxiliary_loss_clip": 0.01018169, + "auxiliary_loss_mlp": 0.01012144, + "balance_loss_clip": 1.00783443, + "balance_loss_mlp": 1.01108897, + "epoch": 0.15086181881492658, + "flos": 56894494769280.0, + "grad_norm": 0.6796258346636038, + "language_loss": 0.49548072, + "learning_rate": 3.848728630520144e-06, + "loss": 0.51578385, + "num_input_tokens_seen": 147927610, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01055908, + "step": 5199, + "time_per_iteration": 2.895528554916382 + }, + { + "auxiliary_loss_clip": 0.01086907, + "auxiliary_loss_mlp": 0.01053027, + "balance_loss_clip": 1.02967966, + "balance_loss_mlp": 1.03604019, + "epoch": 0.15089083628344263, + "flos": 14676124429440.0, + "grad_norm": 2.2784295025220525, + "language_loss": 0.73702395, + "learning_rate": 3.8486569128890065e-06, + "loss": 0.75842321, + "num_input_tokens_seen": 147940990, + "router_z_loss_clip": 0.57128906, + "router_z_loss_mlp": 0.16992188, + "step": 5200, + "time_per_iteration": 2.633674383163452 + }, + { + "auxiliary_loss_clip": 0.01097704, + "auxiliary_loss_mlp": 0.01051813, + "balance_loss_clip": 1.03306627, + "balance_loss_mlp": 1.03039122, + "epoch": 0.15091985375195868, + "flos": 23870466466560.0, + "grad_norm": 2.467636656836881, + "language_loss": 0.95569581, + "learning_rate": 3.848585178929782e-06, + "loss": 0.97719097, + "num_input_tokens_seen": 147954550, + "router_z_loss_clip": 0.64648438, + "router_z_loss_mlp": 0.2142334, + "step": 5201, + "time_per_iteration": 2.3992838859558105 + }, + { + "auxiliary_loss_clip": 0.01090747, + "auxiliary_loss_mlp": 0.01042535, + "balance_loss_clip": 1.03082442, + "balance_loss_mlp": 1.02312803, + "epoch": 0.15094887122047473, + "flos": 26787809007360.0, + "grad_norm": 2.771086692736799, + "language_loss": 0.91904509, + "learning_rate": 3.848513428643105e-06, + "loss": 0.94037795, + "num_input_tokens_seen": 147967735, + "router_z_loss_clip": 0.59960938, + "router_z_loss_mlp": 0.19403076, + "step": 5202, + "time_per_iteration": 2.612290859222412 + }, + { + "auxiliary_loss_clip": 0.01088291, + "auxiliary_loss_mlp": 0.01040123, + "balance_loss_clip": 1.02946854, + "balance_loss_mlp": 1.02206922, + "epoch": 0.15097788868899079, + "flos": 24526379779200.0, + "grad_norm": 2.2488334453325933, + "language_loss": 0.84713548, + "learning_rate": 3.84844166202961e-06, + "loss": 0.86841965, + "num_input_tokens_seen": 147982070, + "router_z_loss_clip": 0.58789062, + "router_z_loss_mlp": 0.18035889, + "step": 5203, + "time_per_iteration": 2.592477321624756 + }, + { + "auxiliary_loss_clip": 0.01090426, + "auxiliary_loss_mlp": 0.01042217, + "balance_loss_clip": 1.03180063, + "balance_loss_mlp": 1.02421093, + "epoch": 0.1510069061575068, + "flos": 18945652993920.0, + "grad_norm": 3.2066901937753682, + "language_loss": 0.71554434, + "learning_rate": 3.8483698790899295e-06, + "loss": 0.73687083, + "num_input_tokens_seen": 147997340, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.17999268, + "step": 5204, + "time_per_iteration": 2.6385293006896973 + }, + { + "auxiliary_loss_clip": 0.0102494, + "auxiliary_loss_mlp": 0.01026126, + "balance_loss_clip": 1.01409483, + "balance_loss_mlp": 1.02504122, + "epoch": 0.15103592362602286, + "flos": 74770549217280.0, + "grad_norm": 0.6818622018984538, + "language_loss": 0.46697861, + "learning_rate": 3.8482980798247e-06, + "loss": 0.48748925, + "num_input_tokens_seen": 148055800, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01086426, + "step": 5205, + "time_per_iteration": 27.221853017807007 + }, + { + "auxiliary_loss_clip": 0.01105466, + "auxiliary_loss_mlp": 0.0105229, + "balance_loss_clip": 1.0379324, + "balance_loss_mlp": 1.03133321, + "epoch": 0.1510649410945389, + "flos": 44852952712320.0, + "grad_norm": 2.025605349865809, + "language_loss": 0.80398011, + "learning_rate": 3.848226264234552e-06, + "loss": 0.82555765, + "num_input_tokens_seen": 148077505, + "router_z_loss_clip": 0.67480469, + "router_z_loss_mlp": 0.20935059, + "step": 5206, + "time_per_iteration": 2.619401216506958 + }, + { + "auxiliary_loss_clip": 0.01025029, + "auxiliary_loss_mlp": 0.01009133, + "balance_loss_clip": 1.01430976, + "balance_loss_mlp": 1.00816751, + "epoch": 0.15109395856305496, + "flos": 58056229992960.0, + "grad_norm": 0.7153847162617327, + "language_loss": 0.49266788, + "learning_rate": 3.848154432320122e-06, + "loss": 0.51300949, + "num_input_tokens_seen": 148136010, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.00964355, + "step": 5207, + "time_per_iteration": 2.9038641452789307 + }, + { + "auxiliary_loss_clip": 0.01024028, + "auxiliary_loss_mlp": 0.01003698, + "balance_loss_clip": 1.01277077, + "balance_loss_mlp": 1.00270855, + "epoch": 0.15112297603157102, + "flos": 68583310560000.0, + "grad_norm": 0.6607347194111574, + "language_loss": 0.50353527, + "learning_rate": 3.8480825840820444e-06, + "loss": 0.52381253, + "num_input_tokens_seen": 148205460, + "router_z_loss_clip": 0.11230469, + "router_z_loss_mlp": 0.0098877, + "step": 5208, + "time_per_iteration": 3.231921434402466 + }, + { + "auxiliary_loss_clip": 0.01024501, + "auxiliary_loss_mlp": 0.0100335, + "balance_loss_clip": 1.01363564, + "balance_loss_mlp": 1.00239646, + "epoch": 0.15115199350008704, + "flos": 66639067833600.0, + "grad_norm": 0.6524132143698032, + "language_loss": 0.50000763, + "learning_rate": 3.848010719520954e-06, + "loss": 0.52028608, + "num_input_tokens_seen": 148269135, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.00952148, + "step": 5209, + "time_per_iteration": 3.0722713470458984 + }, + { + "auxiliary_loss_clip": 0.01096054, + "auxiliary_loss_mlp": 0.01046072, + "balance_loss_clip": 1.03498161, + "balance_loss_mlp": 1.0283215, + "epoch": 0.1511810109686031, + "flos": 20006628433920.0, + "grad_norm": 1.8557136956147164, + "language_loss": 0.70400065, + "learning_rate": 3.847938838637485e-06, + "loss": 0.72542191, + "num_input_tokens_seen": 148284100, + "router_z_loss_clip": 0.61132812, + "router_z_loss_mlp": 0.1774292, + "step": 5210, + "time_per_iteration": 2.3693485260009766 + }, + { + "auxiliary_loss_clip": 0.0108625, + "auxiliary_loss_mlp": 0.01036591, + "balance_loss_clip": 1.03226805, + "balance_loss_mlp": 1.02067709, + "epoch": 0.15121002843711914, + "flos": 18472475070720.0, + "grad_norm": 2.3031778174242215, + "language_loss": 0.87750566, + "learning_rate": 3.8478669414322725e-06, + "loss": 0.89873409, + "num_input_tokens_seen": 148299795, + "router_z_loss_clip": 0.54052734, + "router_z_loss_mlp": 0.15905762, + "step": 5211, + "time_per_iteration": 2.390702724456787 + }, + { + "auxiliary_loss_clip": 0.01092623, + "auxiliary_loss_mlp": 0.01039314, + "balance_loss_clip": 1.03289294, + "balance_loss_mlp": 1.02208805, + "epoch": 0.1512390459056352, + "flos": 15953421853440.0, + "grad_norm": 1.9895065835471961, + "language_loss": 0.69794571, + "learning_rate": 3.847795027905951e-06, + "loss": 0.71926504, + "num_input_tokens_seen": 148312880, + "router_z_loss_clip": 0.59741211, + "router_z_loss_mlp": 0.17236328, + "step": 5212, + "time_per_iteration": 2.357952356338501 + }, + { + "auxiliary_loss_clip": 0.01098565, + "auxiliary_loss_mlp": 0.01045254, + "balance_loss_clip": 1.03408885, + "balance_loss_mlp": 1.02663362, + "epoch": 0.15126806337415125, + "flos": 17013280129920.0, + "grad_norm": 2.9268874970289125, + "language_loss": 0.8563115, + "learning_rate": 3.847723098059156e-06, + "loss": 0.87774968, + "num_input_tokens_seen": 148326010, + "router_z_loss_clip": 0.64550781, + "router_z_loss_mlp": 0.1862793, + "step": 5213, + "time_per_iteration": 2.4224672317504883 + }, + { + "auxiliary_loss_clip": 0.01028355, + "auxiliary_loss_mlp": 0.01019786, + "balance_loss_clip": 1.01518166, + "balance_loss_mlp": 1.01872516, + "epoch": 0.1512970808426673, + "flos": 63419901655680.0, + "grad_norm": 0.727322383001997, + "language_loss": 0.48893499, + "learning_rate": 3.847651151892524e-06, + "loss": 0.50941646, + "num_input_tokens_seen": 148385245, + "router_z_loss_clip": 0.13183594, + "router_z_loss_mlp": 0.01062012, + "step": 5214, + "time_per_iteration": 2.9568123817443848 + }, + { + "auxiliary_loss_clip": 0.01025114, + "auxiliary_loss_mlp": 0.01013573, + "balance_loss_clip": 1.0128231, + "balance_loss_mlp": 1.01236939, + "epoch": 0.15132609831118332, + "flos": 66271607107200.0, + "grad_norm": 0.6591246130087716, + "language_loss": 0.5075227, + "learning_rate": 3.847579189406688e-06, + "loss": 0.52790952, + "num_input_tokens_seen": 148452720, + "router_z_loss_clip": 0.12304688, + "router_z_loss_mlp": 0.01202393, + "step": 5215, + "time_per_iteration": 3.055396318435669 + }, + { + "auxiliary_loss_clip": 0.01096167, + "auxiliary_loss_mlp": 0.01039269, + "balance_loss_clip": 1.03273261, + "balance_loss_mlp": 1.01946831, + "epoch": 0.15135511577969937, + "flos": 30109899473280.0, + "grad_norm": 2.009734091057892, + "language_loss": 0.94197053, + "learning_rate": 3.847507210602286e-06, + "loss": 0.9633249, + "num_input_tokens_seen": 148476500, + "router_z_loss_clip": 0.63427734, + "router_z_loss_mlp": 0.19799805, + "step": 5216, + "time_per_iteration": 2.6617584228515625 + }, + { + "auxiliary_loss_clip": 0.0102083, + "auxiliary_loss_mlp": 0.01002409, + "balance_loss_clip": 1.00934994, + "balance_loss_mlp": 1.00134206, + "epoch": 0.15138413324821542, + "flos": 64218365516160.0, + "grad_norm": 0.6815675484348185, + "language_loss": 0.45859939, + "learning_rate": 3.847435215479952e-06, + "loss": 0.47883177, + "num_input_tokens_seen": 148531920, + "router_z_loss_clip": 0.11474609, + "router_z_loss_mlp": 0.01068115, + "step": 5217, + "time_per_iteration": 2.998030424118042 + }, + { + "auxiliary_loss_clip": 0.01099239, + "auxiliary_loss_mlp": 0.01058716, + "balance_loss_clip": 1.03631258, + "balance_loss_mlp": 1.03881979, + "epoch": 0.15141315071673148, + "flos": 34817797468800.0, + "grad_norm": 1.9035402467220943, + "language_loss": 0.77715325, + "learning_rate": 3.847363204040323e-06, + "loss": 0.79873282, + "num_input_tokens_seen": 148553070, + "router_z_loss_clip": 0.62841797, + "router_z_loss_mlp": 0.19909668, + "step": 5218, + "time_per_iteration": 2.5126333236694336 + }, + { + "auxiliary_loss_clip": 0.01016092, + "auxiliary_loss_mlp": 0.01012547, + "balance_loss_clip": 1.00572085, + "balance_loss_mlp": 1.01146221, + "epoch": 0.15144216818524753, + "flos": 64915999770240.0, + "grad_norm": 0.6961959026340669, + "language_loss": 0.49321234, + "learning_rate": 3.8472911762840345e-06, + "loss": 0.51349872, + "num_input_tokens_seen": 148614505, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01086426, + "step": 5219, + "time_per_iteration": 3.081536293029785 + }, + { + "auxiliary_loss_clip": 0.01078923, + "auxiliary_loss_mlp": 0.0103546, + "balance_loss_clip": 1.02704728, + "balance_loss_mlp": 1.01894319, + "epoch": 0.15147118565376358, + "flos": 24273329178240.0, + "grad_norm": 2.9576637735862468, + "language_loss": 0.79169762, + "learning_rate": 3.847219132211723e-06, + "loss": 0.81284142, + "num_input_tokens_seen": 148631590, + "router_z_loss_clip": 0.51831055, + "router_z_loss_mlp": 0.16503906, + "step": 5220, + "time_per_iteration": 2.4411420822143555 + }, + { + "auxiliary_loss_clip": 0.0109642, + "auxiliary_loss_mlp": 0.01052224, + "balance_loss_clip": 1.03242183, + "balance_loss_mlp": 1.03160644, + "epoch": 0.1515002031222796, + "flos": 27117912712320.0, + "grad_norm": 2.157625464037471, + "language_loss": 0.81868327, + "learning_rate": 3.847147071824024e-06, + "loss": 0.84016967, + "num_input_tokens_seen": 148647015, + "router_z_loss_clip": 0.64013672, + "router_z_loss_mlp": 0.20635986, + "step": 5221, + "time_per_iteration": 2.4313080310821533 + }, + { + "auxiliary_loss_clip": 0.01017354, + "auxiliary_loss_mlp": 0.01014248, + "balance_loss_clip": 1.00728083, + "balance_loss_mlp": 1.01319289, + "epoch": 0.15152922059079565, + "flos": 64274679406080.0, + "grad_norm": 0.6392415767207784, + "language_loss": 0.46244127, + "learning_rate": 3.8470749951215755e-06, + "loss": 0.48275727, + "num_input_tokens_seen": 148712555, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.01055908, + "step": 5222, + "time_per_iteration": 3.0830929279327393 + }, + { + "auxiliary_loss_clip": 0.01086533, + "auxiliary_loss_mlp": 0.01033026, + "balance_loss_clip": 1.03136849, + "balance_loss_mlp": 1.01681936, + "epoch": 0.1515582380593117, + "flos": 15625936500480.0, + "grad_norm": 2.0192905525140166, + "language_loss": 0.80328596, + "learning_rate": 3.847002902105013e-06, + "loss": 0.82448155, + "num_input_tokens_seen": 148727460, + "router_z_loss_clip": 0.55224609, + "router_z_loss_mlp": 0.16204834, + "step": 5223, + "time_per_iteration": 2.425610303878784 + }, + { + "auxiliary_loss_clip": 0.01085914, + "auxiliary_loss_mlp": 0.01038973, + "balance_loss_clip": 1.0298779, + "balance_loss_mlp": 1.02271283, + "epoch": 0.15158725552782776, + "flos": 11137153397760.0, + "grad_norm": 2.0171460320265138, + "language_loss": 0.71763575, + "learning_rate": 3.846930792774973e-06, + "loss": 0.73888457, + "num_input_tokens_seen": 148739400, + "router_z_loss_clip": 0.55957031, + "router_z_loss_mlp": 0.16271973, + "step": 5224, + "time_per_iteration": 2.424239158630371 + }, + { + "auxiliary_loss_clip": 0.01025187, + "auxiliary_loss_mlp": 0.01012069, + "balance_loss_clip": 1.0144192, + "balance_loss_mlp": 1.01096594, + "epoch": 0.1516162729963438, + "flos": 68458147735680.0, + "grad_norm": 1.349395245939376, + "language_loss": 0.467659, + "learning_rate": 3.846858667132093e-06, + "loss": 0.48803157, + "num_input_tokens_seen": 148802720, + "router_z_loss_clip": 0.10742188, + "router_z_loss_mlp": 0.01104736, + "step": 5225, + "time_per_iteration": 2.998217821121216 + }, + { + "auxiliary_loss_clip": 0.0109548, + "auxiliary_loss_mlp": 0.01039234, + "balance_loss_clip": 1.03621292, + "balance_loss_mlp": 1.02109003, + "epoch": 0.15164529046485983, + "flos": 25256029616640.0, + "grad_norm": 2.944890595910711, + "language_loss": 0.87728691, + "learning_rate": 3.84678652517701e-06, + "loss": 0.89863408, + "num_input_tokens_seen": 148816750, + "router_z_loss_clip": 0.59301758, + "router_z_loss_mlp": 0.18139648, + "step": 5226, + "time_per_iteration": 2.476979970932007 + }, + { + "auxiliary_loss_clip": 0.01091803, + "auxiliary_loss_mlp": 0.01038014, + "balance_loss_clip": 1.03531802, + "balance_loss_mlp": 1.02110982, + "epoch": 0.15167430793337588, + "flos": 30730829736960.0, + "grad_norm": 1.7309180586089064, + "language_loss": 0.68013144, + "learning_rate": 3.846714366910361e-06, + "loss": 0.70142961, + "num_input_tokens_seen": 148837610, + "router_z_loss_clip": 0.56494141, + "router_z_loss_mlp": 0.16912842, + "step": 5227, + "time_per_iteration": 2.456674337387085 + }, + { + "auxiliary_loss_clip": 0.01099893, + "auxiliary_loss_mlp": 0.01045415, + "balance_loss_clip": 1.03834319, + "balance_loss_mlp": 1.02463651, + "epoch": 0.15170332540189194, + "flos": 16135214636160.0, + "grad_norm": 3.3112514346648765, + "language_loss": 0.74297768, + "learning_rate": 3.8466421923327835e-06, + "loss": 0.76443076, + "num_input_tokens_seen": 148849635, + "router_z_loss_clip": 0.61572266, + "router_z_loss_mlp": 0.20776367, + "step": 5228, + "time_per_iteration": 2.3576853275299072 + }, + { + "auxiliary_loss_clip": 0.01093108, + "auxiliary_loss_mlp": 0.01039149, + "balance_loss_clip": 1.03479886, + "balance_loss_mlp": 1.02257323, + "epoch": 0.151732342870408, + "flos": 16063258590720.0, + "grad_norm": 2.4480754666283375, + "language_loss": 0.76974702, + "learning_rate": 3.846570001444915e-06, + "loss": 0.79106963, + "num_input_tokens_seen": 148861775, + "router_z_loss_clip": 0.58325195, + "router_z_loss_mlp": 0.16577148, + "step": 5229, + "time_per_iteration": 2.3495888710021973 + }, + { + "auxiliary_loss_clip": 0.01100319, + "auxiliary_loss_mlp": 0.01045024, + "balance_loss_clip": 1.0402689, + "balance_loss_mlp": 1.02788186, + "epoch": 0.15176136033892404, + "flos": 33175448202240.0, + "grad_norm": 1.7850126352072144, + "language_loss": 0.76492012, + "learning_rate": 3.846497794247393e-06, + "loss": 0.7863735, + "num_input_tokens_seen": 148879175, + "router_z_loss_clip": 0.60180664, + "router_z_loss_mlp": 0.17150879, + "step": 5230, + "time_per_iteration": 2.7478771209716797 + }, + { + "auxiliary_loss_clip": 0.0103368, + "auxiliary_loss_mlp": 0.01017266, + "balance_loss_clip": 1.02230799, + "balance_loss_mlp": 1.01603258, + "epoch": 0.1517903778074401, + "flos": 74773202480640.0, + "grad_norm": 0.6942435020289255, + "language_loss": 0.48392558, + "learning_rate": 3.846425570740855e-06, + "loss": 0.50443506, + "num_input_tokens_seen": 148939010, + "router_z_loss_clip": 0.11376953, + "router_z_loss_mlp": 0.0123291, + "step": 5231, + "time_per_iteration": 3.067035675048828 + }, + { + "auxiliary_loss_clip": 0.01033748, + "auxiliary_loss_mlp": 0.01003732, + "balance_loss_clip": 1.02238536, + "balance_loss_mlp": 1.00267148, + "epoch": 0.15181939527595611, + "flos": 60603598189440.0, + "grad_norm": 0.6877018456944582, + "language_loss": 0.49817792, + "learning_rate": 3.846353330925939e-06, + "loss": 0.51855266, + "num_input_tokens_seen": 148990485, + "router_z_loss_clip": 0.11376953, + "router_z_loss_mlp": 0.01062012, + "step": 5232, + "time_per_iteration": 2.861107110977173 + }, + { + "auxiliary_loss_clip": 0.01031119, + "auxiliary_loss_mlp": 0.01006157, + "balance_loss_clip": 1.01962328, + "balance_loss_mlp": 1.00502491, + "epoch": 0.15184841274447217, + "flos": 74776658705280.0, + "grad_norm": 0.6599358120331782, + "language_loss": 0.52331758, + "learning_rate": 3.846281074803283e-06, + "loss": 0.54369032, + "num_input_tokens_seen": 149054945, + "router_z_loss_clip": 0.11523438, + "router_z_loss_mlp": 0.01135254, + "step": 5233, + "time_per_iteration": 3.1006557941436768 + }, + { + "auxiliary_loss_clip": 0.0109135, + "auxiliary_loss_mlp": 0.01046524, + "balance_loss_clip": 1.03307104, + "balance_loss_mlp": 1.02810669, + "epoch": 0.15187743021298822, + "flos": 30182134809600.0, + "grad_norm": 2.2985716010588924, + "language_loss": 0.70873582, + "learning_rate": 3.846208802373527e-06, + "loss": 0.73011458, + "num_input_tokens_seen": 149069900, + "router_z_loss_clip": 0.58300781, + "router_z_loss_mlp": 0.1842041, + "step": 5234, + "time_per_iteration": 2.4886956214904785 + }, + { + "auxiliary_loss_clip": 0.01092106, + "auxiliary_loss_mlp": 0.01041794, + "balance_loss_clip": 1.03186297, + "balance_loss_mlp": 1.02408004, + "epoch": 0.15190644768150427, + "flos": 15114738240000.0, + "grad_norm": 2.6189919732280647, + "language_loss": 0.8294329, + "learning_rate": 3.846136513637307e-06, + "loss": 0.8507719, + "num_input_tokens_seen": 149082220, + "router_z_loss_clip": 0.60253906, + "router_z_loss_mlp": 0.17706299, + "step": 5235, + "time_per_iteration": 2.3883512020111084 + }, + { + "auxiliary_loss_clip": 0.01093091, + "auxiliary_loss_mlp": 0.01039542, + "balance_loss_clip": 1.03563392, + "balance_loss_mlp": 1.02296007, + "epoch": 0.15193546515002032, + "flos": 28504558114560.0, + "grad_norm": 1.6929754930102199, + "language_loss": 0.79051727, + "learning_rate": 3.846064208595262e-06, + "loss": 0.81184363, + "num_input_tokens_seen": 149102100, + "router_z_loss_clip": 0.57470703, + "router_z_loss_mlp": 0.16577148, + "step": 5236, + "time_per_iteration": 2.4587461948394775 + }, + { + "auxiliary_loss_clip": 0.01092202, + "auxiliary_loss_mlp": 0.01041444, + "balance_loss_clip": 1.0339179, + "balance_loss_mlp": 1.02424169, + "epoch": 0.15196448261853635, + "flos": 16068111269760.0, + "grad_norm": 1.8690914607934312, + "language_loss": 0.71715915, + "learning_rate": 3.845991887248031e-06, + "loss": 0.73849553, + "num_input_tokens_seen": 149116850, + "router_z_loss_clip": 0.58300781, + "router_z_loss_mlp": 0.17199707, + "step": 5237, + "time_per_iteration": 2.4442646503448486 + }, + { + "auxiliary_loss_clip": 0.01084612, + "auxiliary_loss_mlp": 0.01038883, + "balance_loss_clip": 1.03221369, + "balance_loss_mlp": 1.02301645, + "epoch": 0.1519935000870524, + "flos": 22226581100160.0, + "grad_norm": 2.098627571009212, + "language_loss": 0.78037453, + "learning_rate": 3.845919549596252e-06, + "loss": 0.80160952, + "num_input_tokens_seen": 149129200, + "router_z_loss_clip": 0.52392578, + "router_z_loss_mlp": 0.15863037, + "step": 5238, + "time_per_iteration": 2.4213249683380127 + }, + { + "auxiliary_loss_clip": 0.01088217, + "auxiliary_loss_mlp": 0.01037725, + "balance_loss_clip": 1.03209257, + "balance_loss_mlp": 1.0204339, + "epoch": 0.15202251755556845, + "flos": 25440894599040.0, + "grad_norm": 1.8343823750557735, + "language_loss": 0.81580591, + "learning_rate": 3.845847195640566e-06, + "loss": 0.83706534, + "num_input_tokens_seen": 149148845, + "router_z_loss_clip": 0.56079102, + "router_z_loss_mlp": 0.17303467, + "step": 5239, + "time_per_iteration": 2.464616298675537 + }, + { + "auxiliary_loss_clip": 0.01096962, + "auxiliary_loss_mlp": 0.01047687, + "balance_loss_clip": 1.03700495, + "balance_loss_mlp": 1.02975833, + "epoch": 0.1520515350240845, + "flos": 24382537511040.0, + "grad_norm": 1.7890529352548405, + "language_loss": 0.88841242, + "learning_rate": 3.84577482538161e-06, + "loss": 0.90985888, + "num_input_tokens_seen": 149166385, + "router_z_loss_clip": 0.59912109, + "router_z_loss_mlp": 0.17932129, + "step": 5240, + "time_per_iteration": 2.476067066192627 + }, + { + "auxiliary_loss_clip": 0.01093474, + "auxiliary_loss_mlp": 0.01035648, + "balance_loss_clip": 1.03389859, + "balance_loss_mlp": 1.02006698, + "epoch": 0.15208055249260055, + "flos": 14639535457920.0, + "grad_norm": 2.34773802994492, + "language_loss": 0.82943833, + "learning_rate": 3.845702438820023e-06, + "loss": 0.85072953, + "num_input_tokens_seen": 149178420, + "router_z_loss_clip": 0.59594727, + "router_z_loss_mlp": 0.15576172, + "step": 5241, + "time_per_iteration": 2.3648996353149414 + }, + { + "auxiliary_loss_clip": 0.01088163, + "auxiliary_loss_mlp": 0.01041787, + "balance_loss_clip": 1.0324285, + "balance_loss_mlp": 1.02330327, + "epoch": 0.1521095699611166, + "flos": 35512080232320.0, + "grad_norm": 2.0412298931918342, + "language_loss": 0.834584, + "learning_rate": 3.845630035956447e-06, + "loss": 0.85588354, + "num_input_tokens_seen": 149196130, + "router_z_loss_clip": 0.55737305, + "router_z_loss_mlp": 0.18487549, + "step": 5242, + "time_per_iteration": 2.5544626712799072 + }, + { + "auxiliary_loss_clip": 0.01086257, + "auxiliary_loss_mlp": 0.01033613, + "balance_loss_clip": 1.03148854, + "balance_loss_mlp": 1.0176928, + "epoch": 0.15213858742963263, + "flos": 25080590701440.0, + "grad_norm": 2.2577219750641526, + "language_loss": 0.68014419, + "learning_rate": 3.845557616791517e-06, + "loss": 0.70134288, + "num_input_tokens_seen": 149213565, + "router_z_loss_clip": 0.54785156, + "router_z_loss_mlp": 0.15917969, + "step": 5243, + "time_per_iteration": 2.6860432624816895 + }, + { + "auxiliary_loss_clip": 0.01098182, + "auxiliary_loss_mlp": 0.01041547, + "balance_loss_clip": 1.03284359, + "balance_loss_mlp": 1.02218711, + "epoch": 0.15216760489814868, + "flos": 28869400488960.0, + "grad_norm": 2.1741898798895907, + "language_loss": 0.91838753, + "learning_rate": 3.8454851813258775e-06, + "loss": 0.93978482, + "num_input_tokens_seen": 149229375, + "router_z_loss_clip": 0.65332031, + "router_z_loss_mlp": 0.19360352, + "step": 5244, + "time_per_iteration": 2.4655773639678955 + }, + { + "auxiliary_loss_clip": 0.01025815, + "auxiliary_loss_mlp": 0.01007952, + "balance_loss_clip": 1.01596618, + "balance_loss_mlp": 1.00676537, + "epoch": 0.15219662236666473, + "flos": 74767861042560.0, + "grad_norm": 0.6910584155193733, + "language_loss": 0.49418244, + "learning_rate": 3.845412729560165e-06, + "loss": 0.51452011, + "num_input_tokens_seen": 149294480, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01184082, + "step": 5245, + "time_per_iteration": 3.0974032878875732 + }, + { + "auxiliary_loss_clip": 0.01088209, + "auxiliary_loss_mlp": 0.0103195, + "balance_loss_clip": 1.03182697, + "balance_loss_mlp": 1.01531434, + "epoch": 0.15222563983518078, + "flos": 14967684126720.0, + "grad_norm": 2.2079713842798245, + "language_loss": 0.79412889, + "learning_rate": 3.845340261495021e-06, + "loss": 0.81533045, + "num_input_tokens_seen": 149307540, + "router_z_loss_clip": 0.56445312, + "router_z_loss_mlp": 0.16650391, + "step": 5246, + "time_per_iteration": 2.389484167098999 + }, + { + "auxiliary_loss_clip": 0.01091372, + "auxiliary_loss_mlp": 0.01037029, + "balance_loss_clip": 1.03400993, + "balance_loss_mlp": 1.01913536, + "epoch": 0.15225465730369683, + "flos": 27592277621760.0, + "grad_norm": 2.007914828713808, + "language_loss": 0.60934252, + "learning_rate": 3.845267777131086e-06, + "loss": 0.63062656, + "num_input_tokens_seen": 149325240, + "router_z_loss_clip": 0.57421875, + "router_z_loss_mlp": 0.17877197, + "step": 5247, + "time_per_iteration": 2.4903502464294434 + }, + { + "auxiliary_loss_clip": 0.01021266, + "auxiliary_loss_mlp": 0.01005122, + "balance_loss_clip": 1.01170135, + "balance_loss_mlp": 1.00384617, + "epoch": 0.15228367477221288, + "flos": 69732931541760.0, + "grad_norm": 0.7224493542585251, + "language_loss": 0.48958656, + "learning_rate": 3.845195276468998e-06, + "loss": 0.50985038, + "num_input_tokens_seen": 149383125, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01275635, + "step": 5248, + "time_per_iteration": 2.954962730407715 + }, + { + "auxiliary_loss_clip": 0.01094143, + "auxiliary_loss_mlp": 0.01036768, + "balance_loss_clip": 1.03457808, + "balance_loss_mlp": 1.02000761, + "epoch": 0.1523126922407289, + "flos": 13946788794240.0, + "grad_norm": 2.313991209299613, + "language_loss": 0.63726473, + "learning_rate": 3.845122759509399e-06, + "loss": 0.65857387, + "num_input_tokens_seen": 149395645, + "router_z_loss_clip": 0.59545898, + "router_z_loss_mlp": 0.16760254, + "step": 5249, + "time_per_iteration": 4.632505178451538 + }, + { + "auxiliary_loss_clip": 0.01089616, + "auxiliary_loss_mlp": 0.01034943, + "balance_loss_clip": 1.03368592, + "balance_loss_mlp": 1.01914155, + "epoch": 0.15234170970924496, + "flos": 26934304538880.0, + "grad_norm": 2.5162031471094446, + "language_loss": 0.83849448, + "learning_rate": 3.845050226252929e-06, + "loss": 0.85974008, + "num_input_tokens_seen": 149410345, + "router_z_loss_clip": 0.55908203, + "router_z_loss_mlp": 0.15802002, + "step": 5250, + "time_per_iteration": 2.4387426376342773 + }, + { + "auxiliary_loss_clip": 0.01093533, + "auxiliary_loss_mlp": 0.01038741, + "balance_loss_clip": 1.0349586, + "balance_loss_mlp": 1.02078176, + "epoch": 0.152370727177761, + "flos": 14604657143040.0, + "grad_norm": 2.890691982773694, + "language_loss": 0.75082397, + "learning_rate": 3.844977676700229e-06, + "loss": 0.7721467, + "num_input_tokens_seen": 149423675, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.17956543, + "step": 5251, + "time_per_iteration": 2.4099175930023193 + }, + { + "auxiliary_loss_clip": 0.01101171, + "auxiliary_loss_mlp": 0.01042332, + "balance_loss_clip": 1.0378058, + "balance_loss_mlp": 1.02343702, + "epoch": 0.15239974464627706, + "flos": 14784774180480.0, + "grad_norm": 3.1178227972829915, + "language_loss": 1.0442481, + "learning_rate": 3.844905110851939e-06, + "loss": 1.06568313, + "num_input_tokens_seen": 149436075, + "router_z_loss_clip": 0.63330078, + "router_z_loss_mlp": 0.18908691, + "step": 5252, + "time_per_iteration": 4.653187274932861 + }, + { + "auxiliary_loss_clip": 0.01093541, + "auxiliary_loss_mlp": 0.01050919, + "balance_loss_clip": 1.03142715, + "balance_loss_mlp": 1.03175008, + "epoch": 0.15242876211479311, + "flos": 16574456851200.0, + "grad_norm": 3.8277230138039284, + "language_loss": 0.73740286, + "learning_rate": 3.844832528708702e-06, + "loss": 0.75884748, + "num_input_tokens_seen": 149450525, + "router_z_loss_clip": 0.62133789, + "router_z_loss_mlp": 0.19152832, + "step": 5253, + "time_per_iteration": 2.434938430786133 + }, + { + "auxiliary_loss_clip": 0.01100099, + "auxiliary_loss_mlp": 0.01044977, + "balance_loss_clip": 1.03734517, + "balance_loss_mlp": 1.02726233, + "epoch": 0.15245777958330914, + "flos": 36604896698880.0, + "grad_norm": 2.03853283778829, + "language_loss": 0.74494612, + "learning_rate": 3.844759930271156e-06, + "loss": 0.76639688, + "num_input_tokens_seen": 149470005, + "router_z_loss_clip": 0.62792969, + "router_z_loss_mlp": 0.17724609, + "step": 5254, + "time_per_iteration": 2.5411643981933594 + }, + { + "auxiliary_loss_clip": 0.0109732, + "auxiliary_loss_mlp": 0.01040916, + "balance_loss_clip": 1.03590596, + "balance_loss_mlp": 1.02352333, + "epoch": 0.1524867970518252, + "flos": 16684642702080.0, + "grad_norm": 2.8331447927347524, + "language_loss": 0.78461701, + "learning_rate": 3.844687315539944e-06, + "loss": 0.8059994, + "num_input_tokens_seen": 149487030, + "router_z_loss_clip": 0.61425781, + "router_z_loss_mlp": 0.17388916, + "step": 5255, + "time_per_iteration": 2.6487364768981934 + }, + { + "auxiliary_loss_clip": 0.01022679, + "auxiliary_loss_mlp": 0.0100492, + "balance_loss_clip": 1.01265287, + "balance_loss_mlp": 1.00378108, + "epoch": 0.15251581452034124, + "flos": 57777133651200.0, + "grad_norm": 0.6344846339356057, + "language_loss": 0.51454228, + "learning_rate": 3.844614684515708e-06, + "loss": 0.53481829, + "num_input_tokens_seen": 149548885, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.01141357, + "step": 5256, + "time_per_iteration": 3.041940689086914 + }, + { + "auxiliary_loss_clip": 0.01107126, + "auxiliary_loss_mlp": 0.01056632, + "balance_loss_clip": 1.03617191, + "balance_loss_mlp": 1.03534126, + "epoch": 0.1525448319888573, + "flos": 35183442804480.0, + "grad_norm": 8.293710226827594, + "language_loss": 0.83043903, + "learning_rate": 3.844542037199088e-06, + "loss": 0.85207665, + "num_input_tokens_seen": 149563085, + "router_z_loss_clip": 0.70996094, + "router_z_loss_mlp": 0.2130127, + "step": 5257, + "time_per_iteration": 4.947717905044556 + }, + { + "auxiliary_loss_clip": 0.01020483, + "auxiliary_loss_mlp": 0.0100206, + "balance_loss_clip": 1.01034594, + "balance_loss_mlp": 1.00076616, + "epoch": 0.15257384945737335, + "flos": 59048251764480.0, + "grad_norm": 0.6302141486519798, + "language_loss": 0.47546333, + "learning_rate": 3.844469373590727e-06, + "loss": 0.49568874, + "num_input_tokens_seen": 149622295, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.01293945, + "step": 5258, + "time_per_iteration": 2.908583164215088 + }, + { + "auxiliary_loss_clip": 0.01087348, + "auxiliary_loss_mlp": 0.01040244, + "balance_loss_clip": 1.0343945, + "balance_loss_mlp": 1.02515793, + "epoch": 0.1526028669258894, + "flos": 12780096157440.0, + "grad_norm": 2.829140977791191, + "language_loss": 0.82422805, + "learning_rate": 3.844396693691265e-06, + "loss": 0.84550393, + "num_input_tokens_seen": 149634485, + "router_z_loss_clip": 0.52880859, + "router_z_loss_mlp": 0.1506958, + "step": 5259, + "time_per_iteration": 4.772944211959839 + }, + { + "auxiliary_loss_clip": 0.01094612, + "auxiliary_loss_mlp": 0.0105172, + "balance_loss_clip": 1.03255486, + "balance_loss_mlp": 1.03461981, + "epoch": 0.15263188439440542, + "flos": 17267866830720.0, + "grad_norm": 2.4734990645837103, + "language_loss": 0.95158947, + "learning_rate": 3.8443239975013456e-06, + "loss": 0.9730528, + "num_input_tokens_seen": 149649105, + "router_z_loss_clip": 0.62011719, + "router_z_loss_mlp": 0.17108154, + "step": 5260, + "time_per_iteration": 2.346336841583252 + }, + { + "auxiliary_loss_clip": 0.01090475, + "auxiliary_loss_mlp": 0.01035595, + "balance_loss_clip": 1.03260827, + "balance_loss_mlp": 1.01724839, + "epoch": 0.15266090186292147, + "flos": 26096214418560.0, + "grad_norm": 2.1849238387334076, + "language_loss": 0.86955428, + "learning_rate": 3.84425128502161e-06, + "loss": 0.89081496, + "num_input_tokens_seen": 149665025, + "router_z_loss_clip": 0.57910156, + "router_z_loss_mlp": 0.18359375, + "step": 5261, + "time_per_iteration": 2.556906223297119 + }, + { + "auxiliary_loss_clip": 0.01085166, + "auxiliary_loss_mlp": 0.01038902, + "balance_loss_clip": 1.02960014, + "balance_loss_mlp": 1.02308857, + "epoch": 0.15268991933143752, + "flos": 33148320209280.0, + "grad_norm": 2.861735717566746, + "language_loss": 0.96712613, + "learning_rate": 3.844178556252702e-06, + "loss": 0.98836684, + "num_input_tokens_seen": 149686610, + "router_z_loss_clip": 0.55517578, + "router_z_loss_mlp": 0.15814209, + "step": 5262, + "time_per_iteration": 2.5224549770355225 + }, + { + "auxiliary_loss_clip": 0.01089454, + "auxiliary_loss_mlp": 0.01040305, + "balance_loss_clip": 1.03163552, + "balance_loss_mlp": 1.02228045, + "epoch": 0.15271893679995358, + "flos": 26097645784320.0, + "grad_norm": 2.3160132278376446, + "language_loss": 0.90161675, + "learning_rate": 3.844105811195262e-06, + "loss": 0.92291433, + "num_input_tokens_seen": 149702500, + "router_z_loss_clip": 0.578125, + "router_z_loss_mlp": 0.18017578, + "step": 5263, + "time_per_iteration": 2.4788310527801514 + }, + { + "auxiliary_loss_clip": 0.01084553, + "auxiliary_loss_mlp": 0.01030941, + "balance_loss_clip": 1.02862406, + "balance_loss_mlp": 1.01535487, + "epoch": 0.15274795426846963, + "flos": 22593762535680.0, + "grad_norm": 2.996129388051031, + "language_loss": 0.91559517, + "learning_rate": 3.844033049849933e-06, + "loss": 0.93675017, + "num_input_tokens_seen": 149716190, + "router_z_loss_clip": 0.55908203, + "router_z_loss_mlp": 0.15588379, + "step": 5264, + "time_per_iteration": 2.3925607204437256 + }, + { + "auxiliary_loss_clip": 0.01093818, + "auxiliary_loss_mlp": 0.01043848, + "balance_loss_clip": 1.03035378, + "balance_loss_mlp": 1.02440453, + "epoch": 0.15277697173698568, + "flos": 45913613950080.0, + "grad_norm": 2.3501231911151956, + "language_loss": 0.81234568, + "learning_rate": 3.843960272217358e-06, + "loss": 0.83372229, + "num_input_tokens_seen": 149739165, + "router_z_loss_clip": 0.63427734, + "router_z_loss_mlp": 0.19451904, + "step": 5265, + "time_per_iteration": 2.662504196166992 + }, + { + "auxiliary_loss_clip": 0.01093211, + "auxiliary_loss_mlp": 0.01041856, + "balance_loss_clip": 1.03006697, + "balance_loss_mlp": 1.02279425, + "epoch": 0.1528059892055017, + "flos": 31971712746240.0, + "grad_norm": 3.704525179429842, + "language_loss": 0.80242264, + "learning_rate": 3.8438874782981804e-06, + "loss": 0.82377338, + "num_input_tokens_seen": 149756160, + "router_z_loss_clip": 0.63183594, + "router_z_loss_mlp": 0.19067383, + "step": 5266, + "time_per_iteration": 2.4597220420837402 + }, + { + "auxiliary_loss_clip": 0.01081986, + "auxiliary_loss_mlp": 0.01035323, + "balance_loss_clip": 1.02690494, + "balance_loss_mlp": 1.01721466, + "epoch": 0.15283500667401775, + "flos": 16720568357760.0, + "grad_norm": 2.8984137338386944, + "language_loss": 0.85251844, + "learning_rate": 3.843814668093041e-06, + "loss": 0.87369156, + "num_input_tokens_seen": 149768480, + "router_z_loss_clip": 0.55126953, + "router_z_loss_mlp": 0.18103027, + "step": 5267, + "time_per_iteration": 2.5326077938079834 + }, + { + "auxiliary_loss_clip": 0.01102508, + "auxiliary_loss_mlp": 0.01043751, + "balance_loss_clip": 1.03105843, + "balance_loss_mlp": 1.02033842, + "epoch": 0.1528640241425338, + "flos": 74731064469120.0, + "grad_norm": 1.636860026478573, + "language_loss": 0.80336905, + "learning_rate": 3.843741841602585e-06, + "loss": 0.82483166, + "num_input_tokens_seen": 149791540, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.23400879, + "step": 5268, + "time_per_iteration": 2.7813000679016113 + }, + { + "auxiliary_loss_clip": 0.01086338, + "auxiliary_loss_mlp": 0.01029751, + "balance_loss_clip": 1.02966619, + "balance_loss_mlp": 1.0122869, + "epoch": 0.15289304161104986, + "flos": 20186570914560.0, + "grad_norm": 2.1349125025688545, + "language_loss": 0.75315869, + "learning_rate": 3.843668998827455e-06, + "loss": 0.77431965, + "num_input_tokens_seen": 149808655, + "router_z_loss_clip": 0.56640625, + "router_z_loss_mlp": 0.17462158, + "step": 5269, + "time_per_iteration": 2.429354429244995 + }, + { + "auxiliary_loss_clip": 0.01085458, + "auxiliary_loss_mlp": 0.01035328, + "balance_loss_clip": 1.02972674, + "balance_loss_mlp": 1.01704121, + "epoch": 0.1529220590795659, + "flos": 35289788405760.0, + "grad_norm": 2.418035085087601, + "language_loss": 0.78431904, + "learning_rate": 3.843596139768295e-06, + "loss": 0.80552691, + "num_input_tokens_seen": 149824870, + "router_z_loss_clip": 0.55761719, + "router_z_loss_mlp": 0.1829834, + "step": 5270, + "time_per_iteration": 2.548570394515991 + }, + { + "auxiliary_loss_clip": 0.01096415, + "auxiliary_loss_mlp": 0.01040614, + "balance_loss_clip": 1.03336239, + "balance_loss_mlp": 1.02069378, + "epoch": 0.15295107654808193, + "flos": 15479999550720.0, + "grad_norm": 22.813126034559495, + "language_loss": 0.86724961, + "learning_rate": 3.843523264425747e-06, + "loss": 0.8886199, + "num_input_tokens_seen": 149838865, + "router_z_loss_clip": 0.63012695, + "router_z_loss_mlp": 0.19934082, + "step": 5271, + "time_per_iteration": 2.399726152420044 + }, + { + "auxiliary_loss_clip": 0.01086135, + "auxiliary_loss_mlp": 0.01037093, + "balance_loss_clip": 1.03029895, + "balance_loss_mlp": 1.02010548, + "epoch": 0.15298009401659798, + "flos": 33246985311360.0, + "grad_norm": 1.898260871980992, + "language_loss": 0.68618774, + "learning_rate": 3.843450372800456e-06, + "loss": 0.70741999, + "num_input_tokens_seen": 149855390, + "router_z_loss_clip": 0.55810547, + "router_z_loss_mlp": 0.16992188, + "step": 5272, + "time_per_iteration": 2.4783287048339844 + }, + { + "auxiliary_loss_clip": 0.01088409, + "auxiliary_loss_mlp": 0.01034393, + "balance_loss_clip": 1.02964187, + "balance_loss_mlp": 1.01540244, + "epoch": 0.15300911148511404, + "flos": 40470515210880.0, + "grad_norm": 2.5972576872323088, + "language_loss": 0.86224627, + "learning_rate": 3.843377464893066e-06, + "loss": 0.88347435, + "num_input_tokens_seen": 149872310, + "router_z_loss_clip": 0.5871582, + "router_z_loss_mlp": 0.18988037, + "step": 5273, + "time_per_iteration": 2.5807535648345947 + }, + { + "auxiliary_loss_clip": 0.01017716, + "auxiliary_loss_mlp": 0.01003175, + "balance_loss_clip": 1.00753486, + "balance_loss_mlp": 1.00204241, + "epoch": 0.1530381289536301, + "flos": 63022659672960.0, + "grad_norm": 0.7464060865600668, + "language_loss": 0.49787855, + "learning_rate": 3.843304540704219e-06, + "loss": 0.51808745, + "num_input_tokens_seen": 149935145, + "router_z_loss_clip": 0.1015625, + "router_z_loss_mlp": 0.01135254, + "step": 5274, + "time_per_iteration": 3.0440824031829834 + }, + { + "auxiliary_loss_clip": 0.01017276, + "auxiliary_loss_mlp": 0.0100837, + "balance_loss_clip": 1.00694513, + "balance_loss_mlp": 1.00729144, + "epoch": 0.15306714642214614, + "flos": 55766869810560.0, + "grad_norm": 0.6934753213263191, + "language_loss": 0.48978418, + "learning_rate": 3.8432316002345605e-06, + "loss": 0.51004058, + "num_input_tokens_seen": 149995325, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01080322, + "step": 5275, + "time_per_iteration": 2.9590115547180176 + }, + { + "auxiliary_loss_clip": 0.01081212, + "auxiliary_loss_mlp": 0.01037655, + "balance_loss_clip": 1.02701271, + "balance_loss_mlp": 1.02166295, + "epoch": 0.1530961638906622, + "flos": 12961923851520.0, + "grad_norm": 4.908626245683616, + "language_loss": 0.89912462, + "learning_rate": 3.843158643484736e-06, + "loss": 0.92031324, + "num_input_tokens_seen": 150005595, + "router_z_loss_clip": 0.54150391, + "router_z_loss_mlp": 0.15997314, + "step": 5276, + "time_per_iteration": 2.3245959281921387 + }, + { + "auxiliary_loss_clip": 0.01081961, + "auxiliary_loss_mlp": 0.01039946, + "balance_loss_clip": 1.02782142, + "balance_loss_mlp": 1.0238111, + "epoch": 0.15312518135917821, + "flos": 15587287758720.0, + "grad_norm": 3.4379488816802004, + "language_loss": 0.86860919, + "learning_rate": 3.8430856704553865e-06, + "loss": 0.88982821, + "num_input_tokens_seen": 150018230, + "router_z_loss_clip": 0.54077148, + "router_z_loss_mlp": 0.16131592, + "step": 5277, + "time_per_iteration": 2.367194175720215 + }, + { + "auxiliary_loss_clip": 0.01094463, + "auxiliary_loss_mlp": 0.01040257, + "balance_loss_clip": 1.03195453, + "balance_loss_mlp": 1.02235222, + "epoch": 0.15315419882769427, + "flos": 13837789929600.0, + "grad_norm": 5.958244555298734, + "language_loss": 0.87622917, + "learning_rate": 3.843012681147159e-06, + "loss": 0.89757633, + "num_input_tokens_seen": 150030560, + "router_z_loss_clip": 0.625, + "router_z_loss_mlp": 0.17895508, + "step": 5278, + "time_per_iteration": 2.376744031906128 + }, + { + "auxiliary_loss_clip": 0.01089881, + "auxiliary_loss_mlp": 0.0103893, + "balance_loss_clip": 1.0300715, + "balance_loss_mlp": 1.02048206, + "epoch": 0.15318321629621032, + "flos": 27995140333440.0, + "grad_norm": 2.339965727194188, + "language_loss": 0.93521297, + "learning_rate": 3.8429396755606995e-06, + "loss": 0.95650101, + "num_input_tokens_seen": 150047200, + "router_z_loss_clip": 0.59814453, + "router_z_loss_mlp": 0.18457031, + "step": 5279, + "time_per_iteration": 2.4684464931488037 + }, + { + "auxiliary_loss_clip": 0.01093509, + "auxiliary_loss_mlp": 0.01039053, + "balance_loss_clip": 1.03218985, + "balance_loss_mlp": 1.02261448, + "epoch": 0.15321223376472637, + "flos": 35729170266240.0, + "grad_norm": 2.1676228884645767, + "language_loss": 1.01977181, + "learning_rate": 3.842866653696649e-06, + "loss": 1.04109752, + "num_input_tokens_seen": 150067710, + "router_z_loss_clip": 0.61328125, + "router_z_loss_mlp": 0.16430664, + "step": 5280, + "time_per_iteration": 2.735365629196167 + }, + { + "auxiliary_loss_clip": 0.01020717, + "auxiliary_loss_mlp": 0.01002561, + "balance_loss_clip": 1.01030111, + "balance_loss_mlp": 1.00135124, + "epoch": 0.15324125123324242, + "flos": 66628105666560.0, + "grad_norm": 0.6923062565067429, + "language_loss": 0.49382463, + "learning_rate": 3.842793615555656e-06, + "loss": 0.5140574, + "num_input_tokens_seen": 150126730, + "router_z_loss_clip": 0.10400391, + "router_z_loss_mlp": 0.01208496, + "step": 5281, + "time_per_iteration": 2.9567840099334717 + }, + { + "auxiliary_loss_clip": 0.01018557, + "auxiliary_loss_mlp": 0.01006768, + "balance_loss_clip": 1.00814581, + "balance_loss_mlp": 1.00546217, + "epoch": 0.15327026870175847, + "flos": 74772713721600.0, + "grad_norm": 0.6880702970301326, + "language_loss": 0.46163023, + "learning_rate": 3.842720561138363e-06, + "loss": 0.48188347, + "num_input_tokens_seen": 150188880, + "router_z_loss_clip": 0.10449219, + "router_z_loss_mlp": 0.01306152, + "step": 5282, + "time_per_iteration": 3.070110321044922 + }, + { + "auxiliary_loss_clip": 0.01095705, + "auxiliary_loss_mlp": 0.01051226, + "balance_loss_clip": 1.03320086, + "balance_loss_mlp": 1.03320122, + "epoch": 0.1532992861702745, + "flos": 60177622291200.0, + "grad_norm": 2.1454293428923785, + "language_loss": 1.03797174, + "learning_rate": 3.842647490445417e-06, + "loss": 1.05944109, + "num_input_tokens_seen": 150216475, + "router_z_loss_clip": 0.62548828, + "router_z_loss_mlp": 0.18017578, + "step": 5283, + "time_per_iteration": 2.9016425609588623 + }, + { + "auxiliary_loss_clip": 0.01104859, + "auxiliary_loss_mlp": 0.01044946, + "balance_loss_clip": 1.03336692, + "balance_loss_mlp": 1.02441788, + "epoch": 0.15332830363879055, + "flos": 11062509177600.0, + "grad_norm": 4.746371032189309, + "language_loss": 0.97465146, + "learning_rate": 3.842574403477463e-06, + "loss": 0.99614954, + "num_input_tokens_seen": 150227945, + "router_z_loss_clip": 0.71435547, + "router_z_loss_mlp": 0.20532227, + "step": 5284, + "time_per_iteration": 2.3775882720947266 + }, + { + "auxiliary_loss_clip": 0.01095277, + "auxiliary_loss_mlp": 0.01061092, + "balance_loss_clip": 1.03072751, + "balance_loss_mlp": 1.04083848, + "epoch": 0.1533573211073066, + "flos": 44997179005440.0, + "grad_norm": 2.1853175727507406, + "language_loss": 0.86454046, + "learning_rate": 3.842501300235146e-06, + "loss": 0.88610423, + "num_input_tokens_seen": 150249560, + "router_z_loss_clip": 0.64697266, + "router_z_loss_mlp": 0.20233154, + "step": 5285, + "time_per_iteration": 2.5832252502441406 + }, + { + "auxiliary_loss_clip": 0.0108655, + "auxiliary_loss_mlp": 0.01041723, + "balance_loss_clip": 1.0311271, + "balance_loss_mlp": 1.02446151, + "epoch": 0.15338633857582265, + "flos": 19272649587840.0, + "grad_norm": 2.9136714250654534, + "language_loss": 0.832322, + "learning_rate": 3.842428180719111e-06, + "loss": 0.85360467, + "num_input_tokens_seen": 150262115, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.17254639, + "step": 5286, + "time_per_iteration": 2.455550193786621 + }, + { + "auxiliary_loss_clip": 0.01090926, + "auxiliary_loss_mlp": 0.01035351, + "balance_loss_clip": 1.03023636, + "balance_loss_mlp": 1.01648021, + "epoch": 0.1534153560443387, + "flos": 27123463618560.0, + "grad_norm": 1.9076838826836873, + "language_loss": 0.65736026, + "learning_rate": 3.8423550449300056e-06, + "loss": 0.67862302, + "num_input_tokens_seen": 150280535, + "router_z_loss_clip": 0.60693359, + "router_z_loss_mlp": 0.18847656, + "step": 5287, + "time_per_iteration": 2.403236150741577 + }, + { + "auxiliary_loss_clip": 0.01022871, + "auxiliary_loss_mlp": 0.01002864, + "balance_loss_clip": 1.01186609, + "balance_loss_mlp": 1.00139761, + "epoch": 0.15344437351285473, + "flos": 74765417247360.0, + "grad_norm": 0.7183976223031759, + "language_loss": 0.49792969, + "learning_rate": 3.842281892868474e-06, + "loss": 0.51818705, + "num_input_tokens_seen": 150334525, + "router_z_loss_clip": 0.10986328, + "router_z_loss_mlp": 0.01464844, + "step": 5288, + "time_per_iteration": 2.9773142337799072 + }, + { + "auxiliary_loss_clip": 0.01097563, + "auxiliary_loss_mlp": 0.0103794, + "balance_loss_clip": 1.03576756, + "balance_loss_mlp": 1.019665, + "epoch": 0.15347339098137078, + "flos": 18468983934720.0, + "grad_norm": 2.6067902164039882, + "language_loss": 0.82248807, + "learning_rate": 3.842208724535164e-06, + "loss": 0.8438431, + "num_input_tokens_seen": 150346590, + "router_z_loss_clip": 0.6171875, + "router_z_loss_mlp": 0.18286133, + "step": 5289, + "time_per_iteration": 2.317091703414917 + }, + { + "auxiliary_loss_clip": 0.01085667, + "auxiliary_loss_mlp": 0.01033904, + "balance_loss_clip": 1.03070772, + "balance_loss_mlp": 1.01811457, + "epoch": 0.15350240844988683, + "flos": 23469698436480.0, + "grad_norm": 1.9183528936815863, + "language_loss": 0.83585292, + "learning_rate": 3.842135539930721e-06, + "loss": 0.85704863, + "num_input_tokens_seen": 150367315, + "router_z_loss_clip": 0.55029297, + "router_z_loss_mlp": 0.15783691, + "step": 5290, + "time_per_iteration": 2.5877864360809326 + }, + { + "auxiliary_loss_clip": 0.01085361, + "auxiliary_loss_mlp": 0.01044687, + "balance_loss_clip": 1.03074932, + "balance_loss_mlp": 1.02820623, + "epoch": 0.15353142591840288, + "flos": 18178855603200.0, + "grad_norm": 3.5901960977757663, + "language_loss": 0.93955755, + "learning_rate": 3.842062339055791e-06, + "loss": 0.96085799, + "num_input_tokens_seen": 150380365, + "router_z_loss_clip": 0.54589844, + "router_z_loss_mlp": 0.16491699, + "step": 5291, + "time_per_iteration": 2.3292462825775146 + }, + { + "auxiliary_loss_clip": 0.01017997, + "auxiliary_loss_mlp": 0.01004168, + "balance_loss_clip": 1.00632131, + "balance_loss_mlp": 1.0027076, + "epoch": 0.15356044338691893, + "flos": 70363567163520.0, + "grad_norm": 0.7503730497089884, + "language_loss": 0.52315068, + "learning_rate": 3.8419891219110225e-06, + "loss": 0.54337239, + "num_input_tokens_seen": 150442250, + "router_z_loss_clip": 0.11669922, + "router_z_loss_mlp": 0.0145874, + "step": 5292, + "time_per_iteration": 3.0475916862487793 + }, + { + "auxiliary_loss_clip": 0.01089384, + "auxiliary_loss_mlp": 0.01030198, + "balance_loss_clip": 1.0315578, + "balance_loss_mlp": 1.01245952, + "epoch": 0.15358946085543498, + "flos": 17118334010880.0, + "grad_norm": 4.3699793876178905, + "language_loss": 0.88696957, + "learning_rate": 3.84191588849706e-06, + "loss": 0.90816534, + "num_input_tokens_seen": 150453265, + "router_z_loss_clip": 0.57861328, + "router_z_loss_mlp": 0.17724609, + "step": 5293, + "time_per_iteration": 2.6398510932922363 + }, + { + "auxiliary_loss_clip": 0.01091271, + "auxiliary_loss_mlp": 0.0103177, + "balance_loss_clip": 1.03180814, + "balance_loss_mlp": 1.01415074, + "epoch": 0.153618478323951, + "flos": 29634138109440.0, + "grad_norm": 1.935455221308653, + "language_loss": 0.77127397, + "learning_rate": 3.84184263881455e-06, + "loss": 0.79250443, + "num_input_tokens_seen": 150470255, + "router_z_loss_clip": 0.59472656, + "router_z_loss_mlp": 0.1762085, + "step": 5294, + "time_per_iteration": 2.519973039627075 + }, + { + "auxiliary_loss_clip": 0.01017679, + "auxiliary_loss_mlp": 0.01004429, + "balance_loss_clip": 1.00625873, + "balance_loss_mlp": 1.00313568, + "epoch": 0.15364749579246706, + "flos": 64777394206080.0, + "grad_norm": 0.7351696719005226, + "language_loss": 0.46371168, + "learning_rate": 3.841769372864141e-06, + "loss": 0.4839327, + "num_input_tokens_seen": 150530675, + "router_z_loss_clip": 0.11425781, + "router_z_loss_mlp": 0.01293945, + "step": 5295, + "time_per_iteration": 2.970205545425415 + }, + { + "auxiliary_loss_clip": 0.01015816, + "auxiliary_loss_mlp": 0.01006082, + "balance_loss_clip": 1.00475502, + "balance_loss_mlp": 1.00471151, + "epoch": 0.1536765132609831, + "flos": 68541624529920.0, + "grad_norm": 0.6336906964841599, + "language_loss": 0.47849074, + "learning_rate": 3.841696090646481e-06, + "loss": 0.49870974, + "num_input_tokens_seen": 150596975, + "router_z_loss_clip": 0.11035156, + "router_z_loss_mlp": 0.01373291, + "step": 5296, + "time_per_iteration": 3.1023223400115967 + }, + { + "auxiliary_loss_clip": 0.01014081, + "auxiliary_loss_mlp": 0.0100028, + "balance_loss_clip": 1.00314879, + "balance_loss_mlp": 0.99914211, + "epoch": 0.15370553072949916, + "flos": 60927766963200.0, + "grad_norm": 0.7266260530636481, + "language_loss": 0.50071335, + "learning_rate": 3.841622792162214e-06, + "loss": 0.52085698, + "num_input_tokens_seen": 150655175, + "router_z_loss_clip": 0.109375, + "router_z_loss_mlp": 0.01141357, + "step": 5297, + "time_per_iteration": 2.990030527114868 + }, + { + "auxiliary_loss_clip": 0.01088456, + "auxiliary_loss_mlp": 0.01035709, + "balance_loss_clip": 1.03017402, + "balance_loss_mlp": 1.01772022, + "epoch": 0.15373454819801521, + "flos": 53427621294720.0, + "grad_norm": 2.7483947262178243, + "language_loss": 0.78556883, + "learning_rate": 3.84154947741199e-06, + "loss": 0.8068105, + "num_input_tokens_seen": 150678155, + "router_z_loss_clip": 0.58300781, + "router_z_loss_mlp": 0.17980957, + "step": 5298, + "time_per_iteration": 2.601888418197632 + }, + { + "auxiliary_loss_clip": 0.01012985, + "auxiliary_loss_mlp": 0.01002445, + "balance_loss_clip": 1.00243628, + "balance_loss_mlp": 1.00128853, + "epoch": 0.15376356566653127, + "flos": 60180415200000.0, + "grad_norm": 0.6331006805182009, + "language_loss": 0.46824318, + "learning_rate": 3.8414761463964555e-06, + "loss": 0.48839748, + "num_input_tokens_seen": 150738410, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01153564, + "step": 5299, + "time_per_iteration": 2.968247413635254 + }, + { + "auxiliary_loss_clip": 0.01078551, + "auxiliary_loss_mlp": 0.01040671, + "balance_loss_clip": 1.0272336, + "balance_loss_mlp": 1.02556729, + "epoch": 0.1537925831350473, + "flos": 42625563926400.0, + "grad_norm": 1.8896834777833864, + "language_loss": 0.60379088, + "learning_rate": 3.841402799116259e-06, + "loss": 0.62498307, + "num_input_tokens_seen": 150756725, + "router_z_loss_clip": 0.51318359, + "router_z_loss_mlp": 0.15118408, + "step": 5300, + "time_per_iteration": 2.5026748180389404 + }, + { + "auxiliary_loss_clip": 0.01011679, + "auxiliary_loss_mlp": 0.0100079, + "balance_loss_clip": 1.00197887, + "balance_loss_mlp": 0.99965769, + "epoch": 0.15382160060356334, + "flos": 63216287406720.0, + "grad_norm": 0.6373410596647077, + "language_loss": 0.54280567, + "learning_rate": 3.841329435572048e-06, + "loss": 0.56293035, + "num_input_tokens_seen": 150824370, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.01135254, + "step": 5301, + "time_per_iteration": 3.1366748809814453 + }, + { + "auxiliary_loss_clip": 0.01084748, + "auxiliary_loss_mlp": 0.01035609, + "balance_loss_clip": 1.02898085, + "balance_loss_mlp": 1.0188303, + "epoch": 0.1538506180720794, + "flos": 34232967417600.0, + "grad_norm": 11.093543805852876, + "language_loss": 0.84213936, + "learning_rate": 3.84125605576447e-06, + "loss": 0.86334288, + "num_input_tokens_seen": 150843555, + "router_z_loss_clip": 0.55712891, + "router_z_loss_mlp": 0.16778564, + "step": 5302, + "time_per_iteration": 2.4729206562042236 + }, + { + "auxiliary_loss_clip": 0.0110131, + "auxiliary_loss_mlp": 0.01043155, + "balance_loss_clip": 1.03275275, + "balance_loss_mlp": 1.02174473, + "epoch": 0.15387963554059544, + "flos": 41129361077760.0, + "grad_norm": 1.6293401693414356, + "language_loss": 0.79157943, + "learning_rate": 3.841182659694174e-06, + "loss": 0.81302404, + "num_input_tokens_seen": 150865280, + "router_z_loss_clip": 0.68603516, + "router_z_loss_mlp": 0.21398926, + "step": 5303, + "time_per_iteration": 2.557373523712158 + }, + { + "auxiliary_loss_clip": 0.01096034, + "auxiliary_loss_mlp": 0.01038419, + "balance_loss_clip": 1.0335412, + "balance_loss_mlp": 1.01979256, + "epoch": 0.1539086530091115, + "flos": 16500336301440.0, + "grad_norm": 3.457445720355042, + "language_loss": 0.85277641, + "learning_rate": 3.8411092473618065e-06, + "loss": 0.87412095, + "num_input_tokens_seen": 150879405, + "router_z_loss_clip": 0.625, + "router_z_loss_mlp": 0.18634033, + "step": 5304, + "time_per_iteration": 2.320791721343994 + }, + { + "auxiliary_loss_clip": 0.01012113, + "auxiliary_loss_mlp": 0.01001408, + "balance_loss_clip": 1.00243163, + "balance_loss_mlp": 1.00007844, + "epoch": 0.15393767047762752, + "flos": 74771806026240.0, + "grad_norm": 0.6520329420900285, + "language_loss": 0.48580772, + "learning_rate": 3.841035818768018e-06, + "loss": 0.50594288, + "num_input_tokens_seen": 150943615, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.01330566, + "step": 5305, + "time_per_iteration": 3.061418294906616 + }, + { + "auxiliary_loss_clip": 0.01090416, + "auxiliary_loss_mlp": 0.01031381, + "balance_loss_clip": 1.03093815, + "balance_loss_mlp": 1.01405954, + "epoch": 0.15396668794614357, + "flos": 42590476143360.0, + "grad_norm": 2.245439157976846, + "language_loss": 0.74500638, + "learning_rate": 3.8409623739134555e-06, + "loss": 0.76622438, + "num_input_tokens_seen": 150963870, + "router_z_loss_clip": 0.59423828, + "router_z_loss_mlp": 0.17321777, + "step": 5306, + "time_per_iteration": 2.7586779594421387 + }, + { + "auxiliary_loss_clip": 0.01090323, + "auxiliary_loss_mlp": 0.0104109, + "balance_loss_clip": 1.02851796, + "balance_loss_mlp": 1.0214982, + "epoch": 0.15399570541465962, + "flos": 37771275133440.0, + "grad_norm": 2.6916801037397833, + "language_loss": 0.95981836, + "learning_rate": 3.840888912798769e-06, + "loss": 0.98113245, + "num_input_tokens_seen": 150979985, + "router_z_loss_clip": 0.61865234, + "router_z_loss_mlp": 0.19604492, + "step": 5307, + "time_per_iteration": 2.5212807655334473 + }, + { + "auxiliary_loss_clip": 0.01084937, + "auxiliary_loss_mlp": 0.01041221, + "balance_loss_clip": 1.0292958, + "balance_loss_mlp": 1.02400708, + "epoch": 0.15402472288317567, + "flos": 32700838913280.0, + "grad_norm": 1.7371051009873255, + "language_loss": 0.84684503, + "learning_rate": 3.8408154354246065e-06, + "loss": 0.86810648, + "num_input_tokens_seen": 151000335, + "router_z_loss_clip": 0.55688477, + "router_z_loss_mlp": 0.17211914, + "step": 5308, + "time_per_iteration": 2.4802372455596924 + }, + { + "auxiliary_loss_clip": 0.01014065, + "auxiliary_loss_mlp": 0.01001796, + "balance_loss_clip": 1.00407743, + "balance_loss_mlp": 1.00059199, + "epoch": 0.15405374035169173, + "flos": 64336685713920.0, + "grad_norm": 0.6840490852222403, + "language_loss": 0.49561679, + "learning_rate": 3.8407419417916174e-06, + "loss": 0.51577538, + "num_input_tokens_seen": 151063600, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.01202393, + "step": 5309, + "time_per_iteration": 3.0390400886535645 + }, + { + "auxiliary_loss_clip": 0.01014597, + "auxiliary_loss_mlp": 0.01002823, + "balance_loss_clip": 1.00425649, + "balance_loss_mlp": 1.0015831, + "epoch": 0.15408275782020778, + "flos": 64114673178240.0, + "grad_norm": 0.6466400252703232, + "language_loss": 0.46701208, + "learning_rate": 3.84066843190045e-06, + "loss": 0.48718625, + "num_input_tokens_seen": 151128035, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01239014, + "step": 5310, + "time_per_iteration": 3.0602874755859375 + }, + { + "auxiliary_loss_clip": 0.01081468, + "auxiliary_loss_mlp": 0.01032726, + "balance_loss_clip": 1.03000355, + "balance_loss_mlp": 1.01685333, + "epoch": 0.1541117752887238, + "flos": 21389643054720.0, + "grad_norm": 2.0080819963739627, + "language_loss": 0.89775836, + "learning_rate": 3.840594905751754e-06, + "loss": 0.91890025, + "num_input_tokens_seen": 151144270, + "router_z_loss_clip": 0.51416016, + "router_z_loss_mlp": 0.15869141, + "step": 5311, + "time_per_iteration": 2.543123483657837 + }, + { + "auxiliary_loss_clip": 0.01089837, + "auxiliary_loss_mlp": 0.01031958, + "balance_loss_clip": 1.03198564, + "balance_loss_mlp": 1.01507831, + "epoch": 0.15414079275723985, + "flos": 30693996385920.0, + "grad_norm": 1.9270963919782897, + "language_loss": 0.8324312, + "learning_rate": 3.84052136334618e-06, + "loss": 0.85364908, + "num_input_tokens_seen": 151164740, + "router_z_loss_clip": 0.57836914, + "router_z_loss_mlp": 0.16882324, + "step": 5312, + "time_per_iteration": 2.5393459796905518 + }, + { + "auxiliary_loss_clip": 0.01013643, + "auxiliary_loss_mlp": 0.01001221, + "balance_loss_clip": 1.00376678, + "balance_loss_mlp": 1.00000489, + "epoch": 0.1541698102257559, + "flos": 74775890655360.0, + "grad_norm": 0.6079144040914988, + "language_loss": 0.438205, + "learning_rate": 3.840447804684376e-06, + "loss": 0.45835364, + "num_input_tokens_seen": 151233285, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.012146, + "step": 5313, + "time_per_iteration": 3.055614948272705 + }, + { + "auxiliary_loss_clip": 0.01083498, + "auxiliary_loss_mlp": 0.01039018, + "balance_loss_clip": 1.02751398, + "balance_loss_mlp": 1.02167964, + "epoch": 0.15419882769427196, + "flos": 31387336542720.0, + "grad_norm": 1.8970895253611249, + "language_loss": 0.91858935, + "learning_rate": 3.840374229766993e-06, + "loss": 0.93981451, + "num_input_tokens_seen": 151254855, + "router_z_loss_clip": 0.55981445, + "router_z_loss_mlp": 0.17364502, + "step": 5314, + "time_per_iteration": 2.5930676460266113 + }, + { + "auxiliary_loss_clip": 0.01013428, + "auxiliary_loss_mlp": 0.01003628, + "balance_loss_clip": 1.00341654, + "balance_loss_mlp": 1.002442, + "epoch": 0.154227845162788, + "flos": 67254412279680.0, + "grad_norm": 0.7264682680090616, + "language_loss": 0.47864681, + "learning_rate": 3.840300638594678e-06, + "loss": 0.49881738, + "num_input_tokens_seen": 151309375, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.01184082, + "step": 5315, + "time_per_iteration": 2.9415664672851562 + }, + { + "auxiliary_loss_clip": 0.01084776, + "auxiliary_loss_mlp": 0.0103155, + "balance_loss_clip": 1.02700949, + "balance_loss_mlp": 1.01362062, + "epoch": 0.15425686263130403, + "flos": 25840440731520.0, + "grad_norm": 2.194119116644531, + "language_loss": 0.78284931, + "learning_rate": 3.840227031168086e-06, + "loss": 0.80401266, + "num_input_tokens_seen": 151322415, + "router_z_loss_clip": 0.57714844, + "router_z_loss_mlp": 0.17932129, + "step": 5316, + "time_per_iteration": 2.4420619010925293 + }, + { + "auxiliary_loss_clip": 0.01089666, + "auxiliary_loss_mlp": 0.0103547, + "balance_loss_clip": 1.0296768, + "balance_loss_mlp": 1.01858985, + "epoch": 0.15428588009982008, + "flos": 26278146846720.0, + "grad_norm": 2.348218645605596, + "language_loss": 0.97252101, + "learning_rate": 3.8401534074878615e-06, + "loss": 0.99377239, + "num_input_tokens_seen": 151337855, + "router_z_loss_clip": 0.59960938, + "router_z_loss_mlp": 0.16894531, + "step": 5317, + "time_per_iteration": 2.4532885551452637 + }, + { + "auxiliary_loss_clip": 0.01083719, + "auxiliary_loss_mlp": 0.01041305, + "balance_loss_clip": 1.02794194, + "balance_loss_mlp": 1.024014, + "epoch": 0.15431489756833613, + "flos": 35660635534080.0, + "grad_norm": 1.8104382416698466, + "language_loss": 0.7528708, + "learning_rate": 3.840079767554659e-06, + "loss": 0.77412105, + "num_input_tokens_seen": 151358955, + "router_z_loss_clip": 0.55664062, + "router_z_loss_mlp": 0.17303467, + "step": 5318, + "time_per_iteration": 2.5314507484436035 + }, + { + "auxiliary_loss_clip": 0.01085039, + "auxiliary_loss_mlp": 0.0103693, + "balance_loss_clip": 1.02796578, + "balance_loss_mlp": 1.01897669, + "epoch": 0.1543439150368522, + "flos": 29020574142720.0, + "grad_norm": 1.7676864031664885, + "language_loss": 0.90532976, + "learning_rate": 3.840006111369127e-06, + "loss": 0.92654938, + "num_input_tokens_seen": 151379815, + "router_z_loss_clip": 0.57128906, + "router_z_loss_mlp": 0.17950439, + "step": 5319, + "time_per_iteration": 2.7098186016082764 + }, + { + "auxiliary_loss_clip": 0.01012475, + "auxiliary_loss_mlp": 0.01002085, + "balance_loss_clip": 1.00255394, + "balance_loss_mlp": 1.00094628, + "epoch": 0.15437293250536824, + "flos": 70143474752640.0, + "grad_norm": 0.6578404856755256, + "language_loss": 0.51501179, + "learning_rate": 3.839932438931916e-06, + "loss": 0.53515732, + "num_input_tokens_seen": 151444045, + "router_z_loss_clip": 0.09912109, + "router_z_loss_mlp": 0.01141357, + "step": 5320, + "time_per_iteration": 3.0795931816101074 + }, + { + "auxiliary_loss_clip": 0.01081939, + "auxiliary_loss_mlp": 0.01031818, + "balance_loss_clip": 1.02534366, + "balance_loss_mlp": 1.01390719, + "epoch": 0.1544019499738843, + "flos": 24748741428480.0, + "grad_norm": 1.9830953160272273, + "language_loss": 0.77677798, + "learning_rate": 3.839858750243678e-06, + "loss": 0.79791552, + "num_input_tokens_seen": 151461275, + "router_z_loss_clip": 0.56591797, + "router_z_loss_mlp": 0.17913818, + "step": 5321, + "time_per_iteration": 2.477278470993042 + }, + { + "auxiliary_loss_clip": 0.01081042, + "auxiliary_loss_mlp": 0.01032444, + "balance_loss_clip": 1.02974319, + "balance_loss_mlp": 1.01757908, + "epoch": 0.1544309674424003, + "flos": 48754880904960.0, + "grad_norm": 2.274881038634335, + "language_loss": 0.81674659, + "learning_rate": 3.839785045305062e-06, + "loss": 0.83788139, + "num_input_tokens_seen": 151484520, + "router_z_loss_clip": 0.51342773, + "router_z_loss_mlp": 0.14880371, + "step": 5322, + "time_per_iteration": 2.6542952060699463 + }, + { + "auxiliary_loss_clip": 0.01096088, + "auxiliary_loss_mlp": 0.01043501, + "balance_loss_clip": 1.03305721, + "balance_loss_mlp": 1.02315235, + "epoch": 0.15445998491091636, + "flos": 34271546336640.0, + "grad_norm": 3.394708429647051, + "language_loss": 0.88313943, + "learning_rate": 3.839711324116721e-06, + "loss": 0.90453535, + "num_input_tokens_seen": 151499665, + "router_z_loss_clip": 0.62988281, + "router_z_loss_mlp": 0.20336914, + "step": 5323, + "time_per_iteration": 2.543290376663208 + }, + { + "auxiliary_loss_clip": 0.01012173, + "auxiliary_loss_mlp": 0.01002429, + "balance_loss_clip": 1.00196195, + "balance_loss_mlp": 1.00137424, + "epoch": 0.15448900237943242, + "flos": 64922667840000.0, + "grad_norm": 0.6833296695676911, + "language_loss": 0.54753011, + "learning_rate": 3.8396375866793046e-06, + "loss": 0.56767613, + "num_input_tokens_seen": 151567395, + "router_z_loss_clip": 0.10205078, + "router_z_loss_mlp": 0.01055908, + "step": 5324, + "time_per_iteration": 5.169343709945679 + }, + { + "auxiliary_loss_clip": 0.01089578, + "auxiliary_loss_mlp": 0.01036583, + "balance_loss_clip": 1.02980447, + "balance_loss_mlp": 1.0170213, + "epoch": 0.15451801984794847, + "flos": 48539501527680.0, + "grad_norm": 2.4436965940335034, + "language_loss": 0.62484145, + "learning_rate": 3.839563832993465e-06, + "loss": 0.64610308, + "num_input_tokens_seen": 151586705, + "router_z_loss_clip": 0.59790039, + "router_z_loss_mlp": 0.19567871, + "step": 5325, + "time_per_iteration": 2.5482876300811768 + }, + { + "auxiliary_loss_clip": 0.01011891, + "auxiliary_loss_mlp": 0.01002466, + "balance_loss_clip": 1.00153399, + "balance_loss_mlp": 1.00128043, + "epoch": 0.15454703731646452, + "flos": 72356375324160.0, + "grad_norm": 0.6571005648942795, + "language_loss": 0.46345431, + "learning_rate": 3.8394900630598525e-06, + "loss": 0.48359787, + "num_input_tokens_seen": 151640880, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01184082, + "step": 5326, + "time_per_iteration": 2.933641195297241 + }, + { + "auxiliary_loss_clip": 0.01100383, + "auxiliary_loss_mlp": 0.01057179, + "balance_loss_clip": 1.03228331, + "balance_loss_mlp": 1.03302717, + "epoch": 0.15457605478498057, + "flos": 20150331056640.0, + "grad_norm": 2.503677876991596, + "language_loss": 1.02023923, + "learning_rate": 3.8394162768791205e-06, + "loss": 1.04181492, + "num_input_tokens_seen": 151654885, + "router_z_loss_clip": 0.68115234, + "router_z_loss_mlp": 0.24145508, + "step": 5327, + "time_per_iteration": 4.694795846939087 + }, + { + "auxiliary_loss_clip": 0.01102015, + "auxiliary_loss_mlp": 0.01041026, + "balance_loss_clip": 1.03150022, + "balance_loss_mlp": 1.01847136, + "epoch": 0.1546050722534966, + "flos": 22703040691200.0, + "grad_norm": 2.7404154678108767, + "language_loss": 0.92301464, + "learning_rate": 3.839342474451919e-06, + "loss": 0.94444501, + "num_input_tokens_seen": 151667895, + "router_z_loss_clip": 0.70507812, + "router_z_loss_mlp": 0.22546387, + "step": 5328, + "time_per_iteration": 2.377978563308716 + }, + { + "auxiliary_loss_clip": 0.01086282, + "auxiliary_loss_mlp": 0.01032676, + "balance_loss_clip": 1.02911544, + "balance_loss_mlp": 1.01559329, + "epoch": 0.15463408972201265, + "flos": 16026564885120.0, + "grad_norm": 2.451400451321569, + "language_loss": 0.72552937, + "learning_rate": 3.8392686557789e-06, + "loss": 0.74671894, + "num_input_tokens_seen": 151680935, + "router_z_loss_clip": 0.57177734, + "router_z_loss_mlp": 0.17089844, + "step": 5329, + "time_per_iteration": 2.429532527923584 + }, + { + "auxiliary_loss_clip": 0.01086184, + "auxiliary_loss_mlp": 0.01036927, + "balance_loss_clip": 1.02681327, + "balance_loss_mlp": 1.01955223, + "epoch": 0.1546631071905287, + "flos": 15993571783680.0, + "grad_norm": 2.057366808988635, + "language_loss": 0.63958967, + "learning_rate": 3.839194820860716e-06, + "loss": 0.66082072, + "num_input_tokens_seen": 151697605, + "router_z_loss_clip": 0.59350586, + "router_z_loss_mlp": 0.17370605, + "step": 5330, + "time_per_iteration": 2.6159024238586426 + }, + { + "auxiliary_loss_clip": 0.01011823, + "auxiliary_loss_mlp": 0.01000023, + "balance_loss_clip": 1.00158715, + "balance_loss_mlp": 0.99914634, + "epoch": 0.15469212465904475, + "flos": 74771631469440.0, + "grad_norm": 0.6741449431039345, + "language_loss": 0.43715069, + "learning_rate": 3.83912096969802e-06, + "loss": 0.45726916, + "num_input_tokens_seen": 151759735, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.00878906, + "step": 5331, + "time_per_iteration": 3.049622058868408 + }, + { + "auxiliary_loss_clip": 0.01092072, + "auxiliary_loss_mlp": 0.01035951, + "balance_loss_clip": 1.02963996, + "balance_loss_mlp": 1.01736605, + "epoch": 0.1547211421275608, + "flos": 30072891565440.0, + "grad_norm": 2.0021776615032336, + "language_loss": 0.74708354, + "learning_rate": 3.839047102291463e-06, + "loss": 0.76836371, + "num_input_tokens_seen": 151775845, + "router_z_loss_clip": 0.6237793, + "router_z_loss_mlp": 0.18585205, + "step": 5332, + "time_per_iteration": 2.4419608116149902 + }, + { + "auxiliary_loss_clip": 0.01094219, + "auxiliary_loss_mlp": 0.01038742, + "balance_loss_clip": 1.03107226, + "balance_loss_mlp": 1.01799953, + "epoch": 0.15475015959607683, + "flos": 49809642122880.0, + "grad_norm": 2.051455398588655, + "language_loss": 0.95279789, + "learning_rate": 3.838973218641698e-06, + "loss": 0.97412753, + "num_input_tokens_seen": 151795645, + "router_z_loss_clip": 0.6315918, + "router_z_loss_mlp": 0.20739746, + "step": 5333, + "time_per_iteration": 5.297952175140381 + }, + { + "auxiliary_loss_clip": 0.01089091, + "auxiliary_loss_mlp": 0.01034886, + "balance_loss_clip": 1.02980101, + "balance_loss_mlp": 1.01558626, + "epoch": 0.15477917706459288, + "flos": 15917600931840.0, + "grad_norm": 1.868649243608746, + "language_loss": 0.75297058, + "learning_rate": 3.838899318749377e-06, + "loss": 0.77421033, + "num_input_tokens_seen": 151811520, + "router_z_loss_clip": 0.59301758, + "router_z_loss_mlp": 0.19311523, + "step": 5334, + "time_per_iteration": 2.4707789421081543 + }, + { + "auxiliary_loss_clip": 0.01084972, + "auxiliary_loss_mlp": 0.01037516, + "balance_loss_clip": 1.02929533, + "balance_loss_mlp": 1.01944423, + "epoch": 0.15480819453310893, + "flos": 47256583374720.0, + "grad_norm": 2.020673989245762, + "language_loss": 0.72112209, + "learning_rate": 3.838825402615153e-06, + "loss": 0.742347, + "num_input_tokens_seen": 151829315, + "router_z_loss_clip": 0.55664062, + "router_z_loss_mlp": 0.18084717, + "step": 5335, + "time_per_iteration": 4.9712371826171875 + }, + { + "auxiliary_loss_clip": 0.0108831, + "auxiliary_loss_mlp": 0.01035823, + "balance_loss_clip": 1.02748537, + "balance_loss_mlp": 1.01652944, + "epoch": 0.15483721200162498, + "flos": 23545355086080.0, + "grad_norm": 2.3387823172414137, + "language_loss": 0.90111828, + "learning_rate": 3.838751470239679e-06, + "loss": 0.92235959, + "num_input_tokens_seen": 151847055, + "router_z_loss_clip": 0.6081543, + "router_z_loss_mlp": 0.19268799, + "step": 5336, + "time_per_iteration": 2.4059524536132812 + }, + { + "auxiliary_loss_clip": 0.01085718, + "auxiliary_loss_mlp": 0.01037412, + "balance_loss_clip": 1.02902675, + "balance_loss_mlp": 1.01764059, + "epoch": 0.15486622947014103, + "flos": 22264775994240.0, + "grad_norm": 2.3860636540367355, + "language_loss": 0.61519414, + "learning_rate": 3.838677521623608e-06, + "loss": 0.63642544, + "num_input_tokens_seen": 151860460, + "router_z_loss_clip": 0.56689453, + "router_z_loss_mlp": 0.19769287, + "step": 5337, + "time_per_iteration": 2.421351432800293 + }, + { + "auxiliary_loss_clip": 0.01010004, + "auxiliary_loss_mlp": 0.01001327, + "balance_loss_clip": 1.00031102, + "balance_loss_mlp": 1.00021255, + "epoch": 0.15489524693865708, + "flos": 60900534236160.0, + "grad_norm": 0.6999882940914005, + "language_loss": 0.49449939, + "learning_rate": 3.838603556767593e-06, + "loss": 0.51461267, + "num_input_tokens_seen": 151919815, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.01116943, + "step": 5338, + "time_per_iteration": 2.9788084030151367 + }, + { + "auxiliary_loss_clip": 0.01010261, + "auxiliary_loss_mlp": 0.0100089, + "balance_loss_clip": 1.0006144, + "balance_loss_mlp": 0.99997842, + "epoch": 0.1549242644071731, + "flos": 74773202480640.0, + "grad_norm": 0.6543401015407392, + "language_loss": 0.44089308, + "learning_rate": 3.8385295756722875e-06, + "loss": 0.46100461, + "num_input_tokens_seen": 151988620, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.00909424, + "step": 5339, + "time_per_iteration": 3.0818498134613037 + }, + { + "auxiliary_loss_clip": 0.01010344, + "auxiliary_loss_mlp": 0.01001816, + "balance_loss_clip": 1.00048184, + "balance_loss_mlp": 1.00076127, + "epoch": 0.15495328187568916, + "flos": 74773761062400.0, + "grad_norm": 0.6471011787372227, + "language_loss": 0.49202216, + "learning_rate": 3.838455578338345e-06, + "loss": 0.51214379, + "num_input_tokens_seen": 152054610, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01055908, + "step": 5340, + "time_per_iteration": 3.133155345916748 + }, + { + "auxiliary_loss_clip": 0.0101022, + "auxiliary_loss_mlp": 0.01001249, + "balance_loss_clip": 1.000386, + "balance_loss_mlp": 1.00028968, + "epoch": 0.1549822993442052, + "flos": 74772015494400.0, + "grad_norm": 0.6654750225843055, + "language_loss": 0.44657665, + "learning_rate": 3.838381564766418e-06, + "loss": 0.46669137, + "num_input_tokens_seen": 152119915, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.00958252, + "step": 5341, + "time_per_iteration": 3.375601053237915 + }, + { + "auxiliary_loss_clip": 0.01089066, + "auxiliary_loss_mlp": 0.01036959, + "balance_loss_clip": 1.02834582, + "balance_loss_mlp": 1.01684833, + "epoch": 0.15501131681272126, + "flos": 23906880881280.0, + "grad_norm": 2.233251550743868, + "language_loss": 0.77703118, + "learning_rate": 3.838307534957162e-06, + "loss": 0.79829144, + "num_input_tokens_seen": 152137190, + "router_z_loss_clip": 0.60668945, + "router_z_loss_mlp": 0.20111084, + "step": 5342, + "time_per_iteration": 2.4460270404815674 + }, + { + "auxiliary_loss_clip": 0.01010696, + "auxiliary_loss_mlp": 0.01001708, + "balance_loss_clip": 1.00097537, + "balance_loss_mlp": 1.00065947, + "epoch": 0.1550403342812373, + "flos": 58787450841600.0, + "grad_norm": 0.7057620620043191, + "language_loss": 0.5062691, + "learning_rate": 3.83823348891123e-06, + "loss": 0.52639318, + "num_input_tokens_seen": 152191495, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.01049805, + "step": 5343, + "time_per_iteration": 2.857945680618286 + }, + { + "auxiliary_loss_clip": 0.01089015, + "auxiliary_loss_mlp": 0.01041284, + "balance_loss_clip": 1.02995098, + "balance_loss_mlp": 1.02285433, + "epoch": 0.15506935174975336, + "flos": 25768414863360.0, + "grad_norm": 3.2267229759074376, + "language_loss": 0.64522719, + "learning_rate": 3.838159426629276e-06, + "loss": 0.66653013, + "num_input_tokens_seen": 152204460, + "router_z_loss_clip": 0.59155273, + "router_z_loss_mlp": 0.1842041, + "step": 5344, + "time_per_iteration": 2.426445245742798 + }, + { + "auxiliary_loss_clip": 0.01082747, + "auxiliary_loss_mlp": 0.01029723, + "balance_loss_clip": 1.02666748, + "balance_loss_mlp": 1.01225257, + "epoch": 0.1550983692182694, + "flos": 19312101290880.0, + "grad_norm": 2.4741176285224986, + "language_loss": 0.79256314, + "learning_rate": 3.8380853481119536e-06, + "loss": 0.81368786, + "num_input_tokens_seen": 152218755, + "router_z_loss_clip": 0.56103516, + "router_z_loss_mlp": 0.17449951, + "step": 5345, + "time_per_iteration": 2.3420846462249756 + }, + { + "auxiliary_loss_clip": 0.01011698, + "auxiliary_loss_mlp": 0.01002536, + "balance_loss_clip": 1.00220704, + "balance_loss_mlp": 1.00149846, + "epoch": 0.15512738668678544, + "flos": 74769850990080.0, + "grad_norm": 0.6693333637986569, + "language_loss": 0.45469421, + "learning_rate": 3.838011253359918e-06, + "loss": 0.47483659, + "num_input_tokens_seen": 152281510, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01037598, + "step": 5346, + "time_per_iteration": 3.1595094203948975 + }, + { + "auxiliary_loss_clip": 0.01010955, + "auxiliary_loss_mlp": 0.01002522, + "balance_loss_clip": 1.00140119, + "balance_loss_mlp": 1.00149643, + "epoch": 0.1551564041553015, + "flos": 74775332073600.0, + "grad_norm": 0.6398725304365916, + "language_loss": 0.50931323, + "learning_rate": 3.837937142373823e-06, + "loss": 0.52944797, + "num_input_tokens_seen": 152346185, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01025391, + "step": 5347, + "time_per_iteration": 3.086622476577759 + }, + { + "auxiliary_loss_clip": 0.01080609, + "auxiliary_loss_mlp": 0.01038713, + "balance_loss_clip": 1.02653575, + "balance_loss_mlp": 1.02184498, + "epoch": 0.15518542162381754, + "flos": 11463381941760.0, + "grad_norm": 3.1577721158354564, + "language_loss": 0.70711815, + "learning_rate": 3.837863015154324e-06, + "loss": 0.72831136, + "num_input_tokens_seen": 152355605, + "router_z_loss_clip": 0.54125977, + "router_z_loss_mlp": 0.16870117, + "step": 5348, + "time_per_iteration": 2.3403587341308594 + }, + { + "auxiliary_loss_clip": 0.01011364, + "auxiliary_loss_mlp": 0.01001795, + "balance_loss_clip": 1.00199711, + "balance_loss_mlp": 1.00073385, + "epoch": 0.1552144390923336, + "flos": 60646750496640.0, + "grad_norm": 0.730900528236134, + "language_loss": 0.51156008, + "learning_rate": 3.837788871702074e-06, + "loss": 0.53169167, + "num_input_tokens_seen": 152414360, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.01062012, + "step": 5349, + "time_per_iteration": 2.9242217540740967 + }, + { + "auxiliary_loss_clip": 0.01010854, + "auxiliary_loss_mlp": 0.01001613, + "balance_loss_clip": 1.00156713, + "balance_loss_mlp": 1.00039697, + "epoch": 0.15524345656084962, + "flos": 54447711799680.0, + "grad_norm": 0.7129225471478812, + "language_loss": 0.50821102, + "learning_rate": 3.837714712017731e-06, + "loss": 0.52833569, + "num_input_tokens_seen": 152473815, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.012146, + "step": 5350, + "time_per_iteration": 2.9343950748443604 + }, + { + "auxiliary_loss_clip": 0.01081952, + "auxiliary_loss_mlp": 0.01040729, + "balance_loss_clip": 1.02786744, + "balance_loss_mlp": 1.02460575, + "epoch": 0.15527247402936567, + "flos": 14676194252160.0, + "grad_norm": 2.432685390365806, + "language_loss": 0.81258005, + "learning_rate": 3.837640536101946e-06, + "loss": 0.83380687, + "num_input_tokens_seen": 152487350, + "router_z_loss_clip": 0.54003906, + "router_z_loss_mlp": 0.16101074, + "step": 5351, + "time_per_iteration": 2.413377046585083 + }, + { + "auxiliary_loss_clip": 0.01094893, + "auxiliary_loss_mlp": 0.01048108, + "balance_loss_clip": 1.03094482, + "balance_loss_mlp": 1.0261972, + "epoch": 0.15530149149788172, + "flos": 36754220050560.0, + "grad_norm": 1.6285552364843228, + "language_loss": 0.86093348, + "learning_rate": 3.837566343955377e-06, + "loss": 0.8823635, + "num_input_tokens_seen": 152510030, + "router_z_loss_clip": 0.63916016, + "router_z_loss_mlp": 0.21911621, + "step": 5352, + "time_per_iteration": 2.5064609050750732 + }, + { + "auxiliary_loss_clip": 0.01089131, + "auxiliary_loss_mlp": 0.01035919, + "balance_loss_clip": 1.02857327, + "balance_loss_mlp": 1.01685119, + "epoch": 0.15533050896639777, + "flos": 37262834870400.0, + "grad_norm": 2.0540888757898346, + "language_loss": 0.77647936, + "learning_rate": 3.8374921355786786e-06, + "loss": 0.79772985, + "num_input_tokens_seen": 152530320, + "router_z_loss_clip": 0.60644531, + "router_z_loss_mlp": 0.19061279, + "step": 5353, + "time_per_iteration": 2.547468662261963 + }, + { + "auxiliary_loss_clip": 0.01079874, + "auxiliary_loss_mlp": 0.0102948, + "balance_loss_clip": 1.02465689, + "balance_loss_mlp": 1.01320183, + "epoch": 0.15535952643491382, + "flos": 16609335166080.0, + "grad_norm": 2.425497315299106, + "language_loss": 0.8776058, + "learning_rate": 3.8374179109725055e-06, + "loss": 0.89869928, + "num_input_tokens_seen": 152543035, + "router_z_loss_clip": 0.55249023, + "router_z_loss_mlp": 0.16265869, + "step": 5354, + "time_per_iteration": 2.5113167762756348 + }, + { + "auxiliary_loss_clip": 0.0108688, + "auxiliary_loss_mlp": 0.01033011, + "balance_loss_clip": 1.02797556, + "balance_loss_mlp": 1.01725125, + "epoch": 0.15538854390342988, + "flos": 23651700687360.0, + "grad_norm": 2.186631904770158, + "language_loss": 0.88510007, + "learning_rate": 3.837343670137515e-06, + "loss": 0.90629888, + "num_input_tokens_seen": 152557730, + "router_z_loss_clip": 0.58886719, + "router_z_loss_mlp": 0.15771484, + "step": 5355, + "time_per_iteration": 2.449326515197754 + }, + { + "auxiliary_loss_clip": 0.01084536, + "auxiliary_loss_mlp": 0.01036098, + "balance_loss_clip": 1.02734327, + "balance_loss_mlp": 1.01887226, + "epoch": 0.1554175613719459, + "flos": 36532242426240.0, + "grad_norm": 1.7167715354476185, + "language_loss": 0.45098096, + "learning_rate": 3.837269413074361e-06, + "loss": 0.47218734, + "num_input_tokens_seen": 152575735, + "router_z_loss_clip": 0.57275391, + "router_z_loss_mlp": 0.17230225, + "step": 5356, + "time_per_iteration": 2.4186081886291504 + }, + { + "auxiliary_loss_clip": 0.010836, + "auxiliary_loss_mlp": 0.01031791, + "balance_loss_clip": 1.02561665, + "balance_loss_mlp": 1.01277757, + "epoch": 0.15544657884046195, + "flos": 30697836635520.0, + "grad_norm": 2.171548639437312, + "language_loss": 0.9398877, + "learning_rate": 3.837195139783699e-06, + "loss": 0.96104169, + "num_input_tokens_seen": 152593720, + "router_z_loss_clip": 0.58007812, + "router_z_loss_mlp": 0.19018555, + "step": 5357, + "time_per_iteration": 2.5393247604370117 + }, + { + "auxiliary_loss_clip": 0.01011489, + "auxiliary_loss_mlp": 0.01000745, + "balance_loss_clip": 1.00234294, + "balance_loss_mlp": 0.99980313, + "epoch": 0.155475596308978, + "flos": 69223409026560.0, + "grad_norm": 0.6397076036472913, + "language_loss": 0.47569412, + "learning_rate": 3.837120850266188e-06, + "loss": 0.49581647, + "num_input_tokens_seen": 152656455, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.00939941, + "step": 5358, + "time_per_iteration": 2.953395366668701 + }, + { + "auxiliary_loss_clip": 0.01085649, + "auxiliary_loss_mlp": 0.01035101, + "balance_loss_clip": 1.0276258, + "balance_loss_mlp": 1.01741004, + "epoch": 0.15550461377749406, + "flos": 18506620247040.0, + "grad_norm": 2.286012750838109, + "language_loss": 0.84797794, + "learning_rate": 3.837046544522481e-06, + "loss": 0.86918545, + "num_input_tokens_seen": 152672630, + "router_z_loss_clip": 0.58007812, + "router_z_loss_mlp": 0.17681885, + "step": 5359, + "time_per_iteration": 2.520167350769043 + }, + { + "auxiliary_loss_clip": 0.01091203, + "auxiliary_loss_mlp": 0.01039174, + "balance_loss_clip": 1.02940249, + "balance_loss_mlp": 1.0204941, + "epoch": 0.1555336312460101, + "flos": 14313306913920.0, + "grad_norm": 2.228852482054366, + "language_loss": 0.79301351, + "learning_rate": 3.836972222553236e-06, + "loss": 0.81431729, + "num_input_tokens_seen": 152686995, + "router_z_loss_clip": 0.61767578, + "router_z_loss_mlp": 0.18688965, + "step": 5360, + "time_per_iteration": 2.358360767364502 + }, + { + "auxiliary_loss_clip": 0.01082022, + "auxiliary_loss_mlp": 0.01027705, + "balance_loss_clip": 1.0279994, + "balance_loss_mlp": 1.01184464, + "epoch": 0.15556264871452616, + "flos": 22229897679360.0, + "grad_norm": 1.816531717115845, + "language_loss": 0.79419351, + "learning_rate": 3.836897884359109e-06, + "loss": 0.81529081, + "num_input_tokens_seen": 152702335, + "router_z_loss_clip": 0.53955078, + "router_z_loss_mlp": 0.1585083, + "step": 5361, + "time_per_iteration": 2.41377329826355 + }, + { + "auxiliary_loss_clip": 0.01092245, + "auxiliary_loss_mlp": 0.01041536, + "balance_loss_clip": 1.02897644, + "balance_loss_mlp": 1.02146077, + "epoch": 0.15559166618304218, + "flos": 37259204088960.0, + "grad_norm": 2.508280284397845, + "language_loss": 0.78528398, + "learning_rate": 3.836823529940757e-06, + "loss": 0.80662173, + "num_input_tokens_seen": 152717690, + "router_z_loss_clip": 0.63256836, + "router_z_loss_mlp": 0.20062256, + "step": 5362, + "time_per_iteration": 2.5052380561828613 + }, + { + "auxiliary_loss_clip": 0.01012057, + "auxiliary_loss_mlp": 0.01001522, + "balance_loss_clip": 1.00287509, + "balance_loss_mlp": 1.0004077, + "epoch": 0.15562068365155823, + "flos": 74765137956480.0, + "grad_norm": 0.6373834376751892, + "language_loss": 0.47412676, + "learning_rate": 3.836749159298835e-06, + "loss": 0.49426252, + "num_input_tokens_seen": 152781315, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01116943, + "step": 5363, + "time_per_iteration": 3.0478570461273193 + }, + { + "auxiliary_loss_clip": 0.0108802, + "auxiliary_loss_mlp": 0.01041035, + "balance_loss_clip": 1.02797222, + "balance_loss_mlp": 1.02191377, + "epoch": 0.15564970112007429, + "flos": 16871986391040.0, + "grad_norm": 1.8805949002162334, + "language_loss": 0.71354485, + "learning_rate": 3.836674772434002e-06, + "loss": 0.73483539, + "num_input_tokens_seen": 152795310, + "router_z_loss_clip": 0.60058594, + "router_z_loss_mlp": 0.19128418, + "step": 5364, + "time_per_iteration": 2.3283259868621826 + }, + { + "auxiliary_loss_clip": 0.01011985, + "auxiliary_loss_mlp": 0.01001013, + "balance_loss_clip": 1.00267863, + "balance_loss_mlp": 1.00002968, + "epoch": 0.15567871858859034, + "flos": 58748173695360.0, + "grad_norm": 0.746419331540189, + "language_loss": 0.4701373, + "learning_rate": 3.836600369346915e-06, + "loss": 0.49026728, + "num_input_tokens_seen": 152845250, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.00982666, + "step": 5365, + "time_per_iteration": 2.812272310256958 + }, + { + "auxiliary_loss_clip": 0.01011425, + "auxiliary_loss_mlp": 0.01000705, + "balance_loss_clip": 1.00219893, + "balance_loss_mlp": 0.99960816, + "epoch": 0.1557077360571064, + "flos": 74774319644160.0, + "grad_norm": 0.6551197464388475, + "language_loss": 0.4864634, + "learning_rate": 3.836525950038229e-06, + "loss": 0.50658476, + "num_input_tokens_seen": 152910570, + "router_z_loss_clip": 0.09228516, + "router_z_loss_mlp": 0.01098633, + "step": 5366, + "time_per_iteration": 3.0852439403533936 + }, + { + "auxiliary_loss_clip": 0.01011165, + "auxiliary_loss_mlp": 0.01001327, + "balance_loss_clip": 1.00213265, + "balance_loss_mlp": 1.00024772, + "epoch": 0.1557367535256224, + "flos": 69322493064960.0, + "grad_norm": 0.7103419356644606, + "language_loss": 0.47814915, + "learning_rate": 3.836451514508603e-06, + "loss": 0.49827406, + "num_input_tokens_seen": 152967555, + "router_z_loss_clip": 0.09033203, + "router_z_loss_mlp": 0.01080322, + "step": 5367, + "time_per_iteration": 3.0498223304748535 + }, + { + "auxiliary_loss_clip": 0.01078241, + "auxiliary_loss_mlp": 0.01044165, + "balance_loss_clip": 1.02571702, + "balance_loss_mlp": 1.02670121, + "epoch": 0.15576577099413846, + "flos": 22485915745920.0, + "grad_norm": 2.080278725698997, + "language_loss": 0.71924877, + "learning_rate": 3.8363770627586944e-06, + "loss": 0.74047279, + "num_input_tokens_seen": 152983750, + "router_z_loss_clip": 0.5246582, + "router_z_loss_mlp": 0.17468262, + "step": 5368, + "time_per_iteration": 2.4067540168762207 + }, + { + "auxiliary_loss_clip": 0.01093634, + "auxiliary_loss_mlp": 0.01055842, + "balance_loss_clip": 1.03008556, + "balance_loss_mlp": 1.03483772, + "epoch": 0.15579478846265452, + "flos": 33172934584320.0, + "grad_norm": 4.091383789665663, + "language_loss": 0.77223289, + "learning_rate": 3.83630259478916e-06, + "loss": 0.79372764, + "num_input_tokens_seen": 152997715, + "router_z_loss_clip": 0.63525391, + "router_z_loss_mlp": 0.20996094, + "step": 5369, + "time_per_iteration": 2.460225820541382 + }, + { + "auxiliary_loss_clip": 0.01086652, + "auxiliary_loss_mlp": 0.01035264, + "balance_loss_clip": 1.03013062, + "balance_loss_mlp": 1.01899207, + "epoch": 0.15582380593117057, + "flos": 16024924051200.0, + "grad_norm": 2.265455433144531, + "language_loss": 0.76580906, + "learning_rate": 3.8362281106006585e-06, + "loss": 0.78702819, + "num_input_tokens_seen": 153009495, + "router_z_loss_clip": 0.56469727, + "router_z_loss_mlp": 0.16278076, + "step": 5370, + "time_per_iteration": 2.3667898178100586 + }, + { + "auxiliary_loss_clip": 0.01093165, + "auxiliary_loss_mlp": 0.01037244, + "balance_loss_clip": 1.0307827, + "balance_loss_mlp": 1.01857638, + "epoch": 0.15585282339968662, + "flos": 10590064392960.0, + "grad_norm": 3.5741241205465903, + "language_loss": 0.81042188, + "learning_rate": 3.836153610193848e-06, + "loss": 0.83172596, + "num_input_tokens_seen": 153020275, + "router_z_loss_clip": 0.62353516, + "router_z_loss_mlp": 0.18664551, + "step": 5371, + "time_per_iteration": 2.3331923484802246 + }, + { + "auxiliary_loss_clip": 0.01011411, + "auxiliary_loss_mlp": 0.01005632, + "balance_loss_clip": 1.00246382, + "balance_loss_mlp": 1.00454152, + "epoch": 0.15588184086820267, + "flos": 60965996768640.0, + "grad_norm": 0.666023884257727, + "language_loss": 0.40609479, + "learning_rate": 3.836079093569384e-06, + "loss": 0.42626524, + "num_input_tokens_seen": 153081410, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01092529, + "step": 5372, + "time_per_iteration": 3.2013235092163086 + }, + { + "auxiliary_loss_clip": 0.01091185, + "auxiliary_loss_mlp": 0.01041544, + "balance_loss_clip": 1.02960801, + "balance_loss_mlp": 1.01875734, + "epoch": 0.1559108583367187, + "flos": 30765673140480.0, + "grad_norm": 14.999071978664706, + "language_loss": 0.88141567, + "learning_rate": 3.836004560727927e-06, + "loss": 0.90274292, + "num_input_tokens_seen": 153096350, + "router_z_loss_clip": 0.61547852, + "router_z_loss_mlp": 0.22796631, + "step": 5373, + "time_per_iteration": 2.4477787017822266 + }, + { + "auxiliary_loss_clip": 0.01012145, + "auxiliary_loss_mlp": 0.01003626, + "balance_loss_clip": 1.00298512, + "balance_loss_mlp": 1.0025171, + "epoch": 0.15593987580523475, + "flos": 65327380853760.0, + "grad_norm": 0.6278111624641315, + "language_loss": 0.44670305, + "learning_rate": 3.835930011670136e-06, + "loss": 0.46686077, + "num_input_tokens_seen": 153163015, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.0111084, + "step": 5374, + "time_per_iteration": 3.199111223220825 + }, + { + "auxiliary_loss_clip": 0.01080014, + "auxiliary_loss_mlp": 0.01030144, + "balance_loss_clip": 1.02811038, + "balance_loss_mlp": 1.01439619, + "epoch": 0.1559688932737508, + "flos": 10771926998400.0, + "grad_norm": 3.2764606915642376, + "language_loss": 0.81297457, + "learning_rate": 3.835855446396667e-06, + "loss": 0.83407617, + "num_input_tokens_seen": 153173570, + "router_z_loss_clip": 0.51953125, + "router_z_loss_mlp": 0.1574707, + "step": 5375, + "time_per_iteration": 2.343033790588379 + }, + { + "auxiliary_loss_clip": 0.01012341, + "auxiliary_loss_mlp": 0.01002182, + "balance_loss_clip": 1.00327492, + "balance_loss_mlp": 1.00119281, + "epoch": 0.15599791074226685, + "flos": 65941922338560.0, + "grad_norm": 0.6571943582934444, + "language_loss": 0.47529429, + "learning_rate": 3.83578086490818e-06, + "loss": 0.49543953, + "num_input_tokens_seen": 153236170, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.0098877, + "step": 5376, + "time_per_iteration": 3.029538154602051 + }, + { + "auxiliary_loss_clip": 0.01095551, + "auxiliary_loss_mlp": 0.01045621, + "balance_loss_clip": 1.03102493, + "balance_loss_mlp": 1.02571321, + "epoch": 0.1560269282107829, + "flos": 29524196638080.0, + "grad_norm": 2.4398248108053195, + "language_loss": 0.88169885, + "learning_rate": 3.835706267205334e-06, + "loss": 0.90311056, + "num_input_tokens_seen": 153249690, + "router_z_loss_clip": 0.64453125, + "router_z_loss_mlp": 0.19909668, + "step": 5377, + "time_per_iteration": 2.4331307411193848 + }, + { + "auxiliary_loss_clip": 0.010121, + "auxiliary_loss_mlp": 0.01000977, + "balance_loss_clip": 1.0029304, + "balance_loss_mlp": 0.99994034, + "epoch": 0.15605594567929892, + "flos": 68759901550080.0, + "grad_norm": 0.6575173908647264, + "language_loss": 0.52836722, + "learning_rate": 3.835631653288787e-06, + "loss": 0.54849797, + "num_input_tokens_seen": 153315500, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01037598, + "step": 5378, + "time_per_iteration": 3.063345432281494 + }, + { + "auxiliary_loss_clip": 0.01084705, + "auxiliary_loss_mlp": 0.0103348, + "balance_loss_clip": 1.02757931, + "balance_loss_mlp": 1.01699305, + "epoch": 0.15608496314781498, + "flos": 24056937371520.0, + "grad_norm": 2.413395802803985, + "language_loss": 0.97914308, + "learning_rate": 3.835557023159199e-06, + "loss": 1.00032496, + "num_input_tokens_seen": 153330165, + "router_z_loss_clip": 0.5715332, + "router_z_loss_mlp": 0.16473389, + "step": 5379, + "time_per_iteration": 2.401952028274536 + }, + { + "auxiliary_loss_clip": 0.01091148, + "auxiliary_loss_mlp": 0.01032803, + "balance_loss_clip": 1.03053331, + "balance_loss_mlp": 1.01541591, + "epoch": 0.15611398061633103, + "flos": 44448030230400.0, + "grad_norm": 1.7909405643242622, + "language_loss": 0.71683681, + "learning_rate": 3.835482376817228e-06, + "loss": 0.73807633, + "num_input_tokens_seen": 153349285, + "router_z_loss_clip": 0.60644531, + "router_z_loss_mlp": 0.17388916, + "step": 5380, + "time_per_iteration": 2.872206211090088 + }, + { + "auxiliary_loss_clip": 0.01085582, + "auxiliary_loss_mlp": 0.01032694, + "balance_loss_clip": 1.02738416, + "balance_loss_mlp": 1.01478267, + "epoch": 0.15614299808484708, + "flos": 12049189511040.0, + "grad_norm": 3.056564883053295, + "language_loss": 0.86643386, + "learning_rate": 3.8354077142635335e-06, + "loss": 0.88761663, + "num_input_tokens_seen": 153360750, + "router_z_loss_clip": 0.58178711, + "router_z_loss_mlp": 0.17919922, + "step": 5381, + "time_per_iteration": 2.4085681438446045 + }, + { + "auxiliary_loss_clip": 0.01091248, + "auxiliary_loss_mlp": 0.0103757, + "balance_loss_clip": 1.0302192, + "balance_loss_mlp": 1.01740003, + "epoch": 0.15617201555336313, + "flos": 26753594008320.0, + "grad_norm": 2.57226472325107, + "language_loss": 0.83511138, + "learning_rate": 3.835333035498776e-06, + "loss": 0.85639954, + "num_input_tokens_seen": 153374720, + "router_z_loss_clip": 0.60986328, + "router_z_loss_mlp": 0.20184326, + "step": 5382, + "time_per_iteration": 2.4641456604003906 + }, + { + "auxiliary_loss_clip": 0.01088268, + "auxiliary_loss_mlp": 0.01037972, + "balance_loss_clip": 1.02971828, + "balance_loss_mlp": 1.01953626, + "epoch": 0.15620103302187918, + "flos": 31423716046080.0, + "grad_norm": 2.5649228210851494, + "language_loss": 0.73976535, + "learning_rate": 3.835258340523614e-06, + "loss": 0.76102781, + "num_input_tokens_seen": 153391245, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.18438721, + "step": 5383, + "time_per_iteration": 2.5899946689605713 + }, + { + "auxiliary_loss_clip": 0.01088882, + "auxiliary_loss_mlp": 0.01038084, + "balance_loss_clip": 1.028543, + "balance_loss_mlp": 1.01958275, + "epoch": 0.1562300504903952, + "flos": 16172152721280.0, + "grad_norm": 2.8886131503011425, + "language_loss": 0.70849073, + "learning_rate": 3.835183629338709e-06, + "loss": 0.72976035, + "num_input_tokens_seen": 153404575, + "router_z_loss_clip": 0.60205078, + "router_z_loss_mlp": 0.18481445, + "step": 5384, + "time_per_iteration": 2.3463008403778076 + }, + { + "auxiliary_loss_clip": 0.01089273, + "auxiliary_loss_mlp": 0.01039319, + "balance_loss_clip": 1.03060627, + "balance_loss_mlp": 1.02189016, + "epoch": 0.15625906795891126, + "flos": 28284395880960.0, + "grad_norm": 13.381976258510635, + "language_loss": 1.0213902, + "learning_rate": 3.835108901944719e-06, + "loss": 1.04267621, + "num_input_tokens_seen": 153418335, + "router_z_loss_clip": 0.5859375, + "router_z_loss_mlp": 0.17443848, + "step": 5385, + "time_per_iteration": 2.4456348419189453 + }, + { + "auxiliary_loss_clip": 0.01089035, + "auxiliary_loss_mlp": 0.01035109, + "balance_loss_clip": 1.02816892, + "balance_loss_mlp": 1.01471186, + "epoch": 0.1562880854274273, + "flos": 26716655923200.0, + "grad_norm": 2.02984040602739, + "language_loss": 0.72212809, + "learning_rate": 3.835034158342303e-06, + "loss": 0.74336952, + "num_input_tokens_seen": 153434940, + "router_z_loss_clip": 0.60839844, + "router_z_loss_mlp": 0.20385742, + "step": 5386, + "time_per_iteration": 2.5045077800750732 + }, + { + "auxiliary_loss_clip": 0.01082727, + "auxiliary_loss_mlp": 0.01033894, + "balance_loss_clip": 1.02678752, + "balance_loss_mlp": 1.01570868, + "epoch": 0.15631710289594336, + "flos": 34962966368640.0, + "grad_norm": 2.009433198811909, + "language_loss": 0.98028648, + "learning_rate": 3.834959398532125e-06, + "loss": 1.00145268, + "num_input_tokens_seen": 153454355, + "router_z_loss_clip": 0.56005859, + "router_z_loss_mlp": 0.1819458, + "step": 5387, + "time_per_iteration": 2.5309691429138184 + }, + { + "auxiliary_loss_clip": 0.01085713, + "auxiliary_loss_mlp": 0.01035902, + "balance_loss_clip": 1.02786827, + "balance_loss_mlp": 1.01718593, + "epoch": 0.1563461203644594, + "flos": 41272190916480.0, + "grad_norm": 2.779385031313092, + "language_loss": 1.15364981, + "learning_rate": 3.834884622514842e-06, + "loss": 1.17486596, + "num_input_tokens_seen": 153468645, + "router_z_loss_clip": 0.578125, + "router_z_loss_mlp": 0.18725586, + "step": 5388, + "time_per_iteration": 2.5770483016967773 + }, + { + "auxiliary_loss_clip": 0.01012401, + "auxiliary_loss_mlp": 0.01004543, + "balance_loss_clip": 1.00320065, + "balance_loss_mlp": 1.00344586, + "epoch": 0.15637513783297546, + "flos": 66737279088000.0, + "grad_norm": 1.2426072498499012, + "language_loss": 0.47029394, + "learning_rate": 3.834809830291115e-06, + "loss": 0.49046338, + "num_input_tokens_seen": 153519480, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01098633, + "step": 5389, + "time_per_iteration": 2.8590283393859863 + }, + { + "auxiliary_loss_clip": 0.01088512, + "auxiliary_loss_mlp": 0.01041026, + "balance_loss_clip": 1.0287323, + "balance_loss_mlp": 1.02165473, + "epoch": 0.1564041553014915, + "flos": 30957904419840.0, + "grad_norm": 2.2998807755951187, + "language_loss": 0.89743382, + "learning_rate": 3.834735021861605e-06, + "loss": 0.91872919, + "num_input_tokens_seen": 153538725, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.19372559, + "step": 5390, + "time_per_iteration": 2.4581587314605713 + }, + { + "auxiliary_loss_clip": 0.01090991, + "auxiliary_loss_mlp": 0.010461, + "balance_loss_clip": 1.0290885, + "balance_loss_mlp": 1.02565527, + "epoch": 0.15643317277000754, + "flos": 50141665952640.0, + "grad_norm": 2.5641946387586456, + "language_loss": 0.88713169, + "learning_rate": 3.834660197226974e-06, + "loss": 0.9085027, + "num_input_tokens_seen": 153559940, + "router_z_loss_clip": 0.61938477, + "router_z_loss_mlp": 0.2043457, + "step": 5391, + "time_per_iteration": 2.6566789150238037 + }, + { + "auxiliary_loss_clip": 0.0101291, + "auxiliary_loss_mlp": 0.01002803, + "balance_loss_clip": 1.0037303, + "balance_loss_mlp": 1.00158095, + "epoch": 0.1564621902385236, + "flos": 66592598947200.0, + "grad_norm": 0.6978874973853302, + "language_loss": 0.49399668, + "learning_rate": 3.834585356387881e-06, + "loss": 0.51415384, + "num_input_tokens_seen": 153616820, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01220703, + "step": 5392, + "time_per_iteration": 2.9080286026000977 + }, + { + "auxiliary_loss_clip": 0.0108284, + "auxiliary_loss_mlp": 0.01038469, + "balance_loss_clip": 1.02802145, + "balance_loss_mlp": 1.01954436, + "epoch": 0.15649120770703964, + "flos": 40733026790400.0, + "grad_norm": 2.1124936762653523, + "language_loss": 0.72818285, + "learning_rate": 3.8345104993449884e-06, + "loss": 0.74939597, + "num_input_tokens_seen": 153643095, + "router_z_loss_clip": 0.54882812, + "router_z_loss_mlp": 0.18920898, + "step": 5393, + "time_per_iteration": 2.5832712650299072 + }, + { + "auxiliary_loss_clip": 0.01011584, + "auxiliary_loss_mlp": 0.01001185, + "balance_loss_clip": 1.00228572, + "balance_loss_mlp": 1.00018346, + "epoch": 0.1565202251755557, + "flos": 74783466420480.0, + "grad_norm": 0.6418123570541358, + "language_loss": 0.45430309, + "learning_rate": 3.834435626098956e-06, + "loss": 0.47443077, + "num_input_tokens_seen": 153708760, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01000977, + "step": 5394, + "time_per_iteration": 3.2008421421051025 + }, + { + "auxiliary_loss_clip": 0.01087597, + "auxiliary_loss_mlp": 0.01040717, + "balance_loss_clip": 1.03056192, + "balance_loss_mlp": 1.02143478, + "epoch": 0.15654924264407172, + "flos": 23322993436800.0, + "grad_norm": 3.281650203343549, + "language_loss": 0.78724289, + "learning_rate": 3.834360736650447e-06, + "loss": 0.80852592, + "num_input_tokens_seen": 153723440, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.19274902, + "step": 5395, + "time_per_iteration": 2.406796455383301 + }, + { + "auxiliary_loss_clip": 0.01084311, + "auxiliary_loss_mlp": 0.01046469, + "balance_loss_clip": 1.02901304, + "balance_loss_mlp": 1.02898645, + "epoch": 0.15657826011258777, + "flos": 10115420192640.0, + "grad_norm": 1.9191151185571993, + "language_loss": 0.60188067, + "learning_rate": 3.83428583100012e-06, + "loss": 0.6231885, + "num_input_tokens_seen": 153737020, + "router_z_loss_clip": 0.55273438, + "router_z_loss_mlp": 0.17474365, + "step": 5396, + "time_per_iteration": 2.326289415359497 + }, + { + "auxiliary_loss_clip": 0.01080748, + "auxiliary_loss_mlp": 0.01037836, + "balance_loss_clip": 1.02567303, + "balance_loss_mlp": 1.02109897, + "epoch": 0.15660727758110382, + "flos": 16718648232960.0, + "grad_norm": 2.60316402535081, + "language_loss": 0.88554913, + "learning_rate": 3.834210909148639e-06, + "loss": 0.90673494, + "num_input_tokens_seen": 153749515, + "router_z_loss_clip": 0.55151367, + "router_z_loss_mlp": 0.1673584, + "step": 5397, + "time_per_iteration": 2.3557193279266357 + }, + { + "auxiliary_loss_clip": 0.01086239, + "auxiliary_loss_mlp": 0.01037849, + "balance_loss_clip": 1.0282104, + "balance_loss_mlp": 1.02101088, + "epoch": 0.15663629504961987, + "flos": 30181366759680.0, + "grad_norm": 2.7299321747354894, + "language_loss": 0.73502016, + "learning_rate": 3.8341359710966655e-06, + "loss": 0.75626111, + "num_input_tokens_seen": 153768435, + "router_z_loss_clip": 0.57958984, + "router_z_loss_mlp": 0.16833496, + "step": 5398, + "time_per_iteration": 2.5581235885620117 + }, + { + "auxiliary_loss_clip": 0.01090814, + "auxiliary_loss_mlp": 0.0104197, + "balance_loss_clip": 1.03029931, + "balance_loss_mlp": 1.02241945, + "epoch": 0.15666531251813592, + "flos": 27851018774400.0, + "grad_norm": 1.8645097173685063, + "language_loss": 0.87778896, + "learning_rate": 3.834061016844861e-06, + "loss": 0.89911681, + "num_input_tokens_seen": 153797585, + "router_z_loss_clip": 0.60498047, + "router_z_loss_mlp": 0.19543457, + "step": 5399, + "time_per_iteration": 2.703878402709961 + }, + { + "auxiliary_loss_clip": 0.01010308, + "auxiliary_loss_mlp": 0.01001976, + "balance_loss_clip": 1.00112677, + "balance_loss_mlp": 1.00080764, + "epoch": 0.15669432998665198, + "flos": 74760320188800.0, + "grad_norm": 0.6878202191669209, + "language_loss": 0.46604821, + "learning_rate": 3.833986046393886e-06, + "loss": 0.48617107, + "num_input_tokens_seen": 153855665, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01165771, + "step": 5400, + "time_per_iteration": 5.235064268112183 + }, + { + "auxiliary_loss_clip": 0.01010625, + "auxiliary_loss_mlp": 0.01001547, + "balance_loss_clip": 1.00126255, + "balance_loss_mlp": 1.00042009, + "epoch": 0.156723347455168, + "flos": 66529335830400.0, + "grad_norm": 0.6399099173008197, + "language_loss": 0.47883892, + "learning_rate": 3.833911059744405e-06, + "loss": 0.49896061, + "num_input_tokens_seen": 153924695, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.0112915, + "step": 5401, + "time_per_iteration": 3.271044969558716 + }, + { + "auxiliary_loss_clip": 0.01010947, + "auxiliary_loss_mlp": 0.01000395, + "balance_loss_clip": 1.00145102, + "balance_loss_mlp": 0.99944699, + "epoch": 0.15675236492368405, + "flos": 74002425194880.0, + "grad_norm": 0.6608772028392369, + "language_loss": 0.47876036, + "learning_rate": 3.83383605689708e-06, + "loss": 0.49887377, + "num_input_tokens_seen": 153984580, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.00946045, + "step": 5402, + "time_per_iteration": 2.9923837184906006 + }, + { + "auxiliary_loss_clip": 0.01010618, + "auxiliary_loss_mlp": 0.01000803, + "balance_loss_clip": 1.00126088, + "balance_loss_mlp": 0.99986738, + "epoch": 0.1567813823922001, + "flos": 66035280913920.0, + "grad_norm": 0.7103844538111059, + "language_loss": 0.48333102, + "learning_rate": 3.833761037852572e-06, + "loss": 0.50344527, + "num_input_tokens_seen": 154037205, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.00933838, + "step": 5403, + "time_per_iteration": 4.962772846221924 + }, + { + "auxiliary_loss_clip": 0.01010432, + "auxiliary_loss_mlp": 0.01001233, + "balance_loss_clip": 1.00149369, + "balance_loss_mlp": 1.00039816, + "epoch": 0.15681039986071615, + "flos": 71162310314880.0, + "grad_norm": 0.6548122441018571, + "language_loss": 0.50889355, + "learning_rate": 3.833686002611545e-06, + "loss": 0.52901018, + "num_input_tokens_seen": 154100915, + "router_z_loss_clip": 0.08935547, + "router_z_loss_mlp": 0.00836182, + "step": 5404, + "time_per_iteration": 3.113929033279419 + }, + { + "auxiliary_loss_clip": 0.01084196, + "auxiliary_loss_mlp": 0.01034394, + "balance_loss_clip": 1.02839041, + "balance_loss_mlp": 1.01721001, + "epoch": 0.1568394173292322, + "flos": 14384041061760.0, + "grad_norm": 4.825066822313837, + "language_loss": 0.76653814, + "learning_rate": 3.833610951174661e-06, + "loss": 0.78772402, + "num_input_tokens_seen": 154114080, + "router_z_loss_clip": 0.55786133, + "router_z_loss_mlp": 0.171875, + "step": 5405, + "time_per_iteration": 2.490241289138794 + }, + { + "auxiliary_loss_clip": 0.01090514, + "auxiliary_loss_mlp": 0.01041765, + "balance_loss_clip": 1.02957702, + "balance_loss_mlp": 1.023031, + "epoch": 0.15686843479774826, + "flos": 32882561873280.0, + "grad_norm": 2.5739657021331417, + "language_loss": 0.86409879, + "learning_rate": 3.8335358835425835e-06, + "loss": 0.88542151, + "num_input_tokens_seen": 154130755, + "router_z_loss_clip": 0.609375, + "router_z_loss_mlp": 0.18725586, + "step": 5406, + "time_per_iteration": 2.6024231910705566 + }, + { + "auxiliary_loss_clip": 0.01011021, + "auxiliary_loss_mlp": 0.01000619, + "balance_loss_clip": 1.00207913, + "balance_loss_mlp": 0.99956387, + "epoch": 0.15689745226626428, + "flos": 61123768669440.0, + "grad_norm": 0.7563206896792328, + "language_loss": 0.49903828, + "learning_rate": 3.8334607997159745e-06, + "loss": 0.51915467, + "num_input_tokens_seen": 154193105, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01055908, + "step": 5407, + "time_per_iteration": 2.9640560150146484 + }, + { + "auxiliary_loss_clip": 0.0101148, + "auxiliary_loss_mlp": 0.01001479, + "balance_loss_clip": 1.002316, + "balance_loss_mlp": 1.0005132, + "epoch": 0.15692646973478033, + "flos": 70057833454080.0, + "grad_norm": 0.7083660052445061, + "language_loss": 0.48670334, + "learning_rate": 3.833385699695497e-06, + "loss": 0.50683296, + "num_input_tokens_seen": 154254745, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.00964355, + "step": 5408, + "time_per_iteration": 3.1393275260925293 + }, + { + "auxiliary_loss_clip": 0.01011209, + "auxiliary_loss_mlp": 0.01001247, + "balance_loss_clip": 1.00214911, + "balance_loss_mlp": 1.00012672, + "epoch": 0.15695548720329638, + "flos": 63129110008320.0, + "grad_norm": 0.6844669789126466, + "language_loss": 0.43346047, + "learning_rate": 3.833310583481817e-06, + "loss": 0.45358503, + "num_input_tokens_seen": 154309580, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01123047, + "step": 5409, + "time_per_iteration": 2.888092279434204 + }, + { + "auxiliary_loss_clip": 0.010885, + "auxiliary_loss_mlp": 0.01042538, + "balance_loss_clip": 1.02783465, + "balance_loss_mlp": 1.02342892, + "epoch": 0.15698450467181244, + "flos": 41091934233600.0, + "grad_norm": 2.4546273312148257, + "language_loss": 0.91968262, + "learning_rate": 3.833235451075596e-06, + "loss": 0.94099301, + "num_input_tokens_seen": 154327585, + "router_z_loss_clip": 0.60620117, + "router_z_loss_mlp": 0.19122314, + "step": 5410, + "time_per_iteration": 5.105243921279907 + }, + { + "auxiliary_loss_clip": 0.01089336, + "auxiliary_loss_mlp": 0.01047348, + "balance_loss_clip": 1.03267348, + "balance_loss_mlp": 1.03074861, + "epoch": 0.1570135221403285, + "flos": 40033297854720.0, + "grad_norm": 1.6188086581219634, + "language_loss": 0.73505396, + "learning_rate": 3.833160302477496e-06, + "loss": 0.75642079, + "num_input_tokens_seen": 154346230, + "router_z_loss_clip": 0.56665039, + "router_z_loss_mlp": 0.16601562, + "step": 5411, + "time_per_iteration": 5.115052938461304 + }, + { + "auxiliary_loss_clip": 0.01082725, + "auxiliary_loss_mlp": 0.01049161, + "balance_loss_clip": 1.03001237, + "balance_loss_mlp": 1.0342536, + "epoch": 0.1570425396088445, + "flos": 18981962674560.0, + "grad_norm": 1.9530370331499673, + "language_loss": 0.79790783, + "learning_rate": 3.833085137688183e-06, + "loss": 0.81922662, + "num_input_tokens_seen": 154360660, + "router_z_loss_clip": 0.52661133, + "router_z_loss_mlp": 0.14904785, + "step": 5412, + "time_per_iteration": 2.3703019618988037 + }, + { + "auxiliary_loss_clip": 0.01087256, + "auxiliary_loss_mlp": 0.01034479, + "balance_loss_clip": 1.02828598, + "balance_loss_mlp": 1.01654983, + "epoch": 0.15707155707736056, + "flos": 19127725067520.0, + "grad_norm": 2.642297696879231, + "language_loss": 0.84535468, + "learning_rate": 3.833009956708321e-06, + "loss": 0.86657202, + "num_input_tokens_seen": 154373305, + "router_z_loss_clip": 0.58984375, + "router_z_loss_mlp": 0.17944336, + "step": 5413, + "time_per_iteration": 2.360095262527466 + }, + { + "auxiliary_loss_clip": 0.01085806, + "auxiliary_loss_mlp": 0.0103338, + "balance_loss_clip": 1.03135753, + "balance_loss_mlp": 1.01644051, + "epoch": 0.15710057454587661, + "flos": 15479056944000.0, + "grad_norm": 3.9440106154672057, + "language_loss": 0.98406827, + "learning_rate": 3.832934759538573e-06, + "loss": 1.00526023, + "num_input_tokens_seen": 154385315, + "router_z_loss_clip": 0.54418945, + "router_z_loss_mlp": 0.16925049, + "step": 5414, + "time_per_iteration": 2.3404695987701416 + }, + { + "auxiliary_loss_clip": 0.01089222, + "auxiliary_loss_mlp": 0.01033107, + "balance_loss_clip": 1.02917421, + "balance_loss_mlp": 1.01510668, + "epoch": 0.15712959201439267, + "flos": 59187836712960.0, + "grad_norm": 1.9054447951263782, + "language_loss": 0.86069888, + "learning_rate": 3.832859546179604e-06, + "loss": 0.88192213, + "num_input_tokens_seen": 154413220, + "router_z_loss_clip": 0.60083008, + "router_z_loss_mlp": 0.18005371, + "step": 5415, + "time_per_iteration": 2.7425217628479004 + }, + { + "auxiliary_loss_clip": 0.0108826, + "auxiliary_loss_mlp": 0.01037559, + "balance_loss_clip": 1.03049278, + "balance_loss_mlp": 1.01915324, + "epoch": 0.15715860948290872, + "flos": 31607149662720.0, + "grad_norm": 2.327586998365913, + "language_loss": 0.82737923, + "learning_rate": 3.8327843166320766e-06, + "loss": 0.84863734, + "num_input_tokens_seen": 154430595, + "router_z_loss_clip": 0.57788086, + "router_z_loss_mlp": 0.18408203, + "step": 5416, + "time_per_iteration": 2.6646499633789062 + }, + { + "auxiliary_loss_clip": 0.0107862, + "auxiliary_loss_mlp": 0.01028053, + "balance_loss_clip": 1.0301609, + "balance_loss_mlp": 1.01418293, + "epoch": 0.15718762695142477, + "flos": 15224574977280.0, + "grad_norm": 2.0075516150058785, + "language_loss": 0.65313679, + "learning_rate": 3.832709070896657e-06, + "loss": 0.67420352, + "num_input_tokens_seen": 154444470, + "router_z_loss_clip": 0.484375, + "router_z_loss_mlp": 0.13873291, + "step": 5417, + "time_per_iteration": 2.5045907497406006 + }, + { + "auxiliary_loss_clip": 0.01083392, + "auxiliary_loss_mlp": 0.0104434, + "balance_loss_clip": 1.0310179, + "balance_loss_mlp": 1.02757955, + "epoch": 0.1572166444199408, + "flos": 30786096286080.0, + "grad_norm": 2.408594798341383, + "language_loss": 1.05363226, + "learning_rate": 3.832633808974009e-06, + "loss": 1.07490957, + "num_input_tokens_seen": 154466245, + "router_z_loss_clip": 0.52416992, + "router_z_loss_mlp": 0.16760254, + "step": 5418, + "time_per_iteration": 2.5768215656280518 + }, + { + "auxiliary_loss_clip": 0.01017944, + "auxiliary_loss_mlp": 0.0100008, + "balance_loss_clip": 1.0084331, + "balance_loss_mlp": 0.99897116, + "epoch": 0.15724566188845684, + "flos": 61229625511680.0, + "grad_norm": 0.6599576928413985, + "language_loss": 0.48313922, + "learning_rate": 3.832558530864798e-06, + "loss": 0.5033195, + "num_input_tokens_seen": 154526850, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.0111084, + "step": 5419, + "time_per_iteration": 2.984501838684082 + }, + { + "auxiliary_loss_clip": 0.01082893, + "auxiliary_loss_mlp": 0.01029373, + "balance_loss_clip": 1.02953458, + "balance_loss_mlp": 1.0128808, + "epoch": 0.1572746793569729, + "flos": 24926659050240.0, + "grad_norm": 2.237451508648342, + "language_loss": 0.86333799, + "learning_rate": 3.832483236569689e-06, + "loss": 0.88446069, + "num_input_tokens_seen": 154540125, + "router_z_loss_clip": 0.53344727, + "router_z_loss_mlp": 0.16503906, + "step": 5420, + "time_per_iteration": 2.4055562019348145 + }, + { + "auxiliary_loss_clip": 0.01089645, + "auxiliary_loss_mlp": 0.0103991, + "balance_loss_clip": 1.03023696, + "balance_loss_mlp": 1.02009702, + "epoch": 0.15730369682548895, + "flos": 11282671411200.0, + "grad_norm": 2.481663699052959, + "language_loss": 0.79203868, + "learning_rate": 3.832407926089345e-06, + "loss": 0.81333423, + "num_input_tokens_seen": 154551175, + "router_z_loss_clip": 0.59472656, + "router_z_loss_mlp": 0.19812012, + "step": 5421, + "time_per_iteration": 2.3713865280151367 + }, + { + "auxiliary_loss_clip": 0.01084644, + "auxiliary_loss_mlp": 0.01031466, + "balance_loss_clip": 1.0290941, + "balance_loss_mlp": 1.01381159, + "epoch": 0.157332714294005, + "flos": 74733089328000.0, + "grad_norm": 1.860583328514696, + "language_loss": 0.65397364, + "learning_rate": 3.8323325994244346e-06, + "loss": 0.67513472, + "num_input_tokens_seen": 154579510, + "router_z_loss_clip": 0.55444336, + "router_z_loss_mlp": 0.17633057, + "step": 5422, + "time_per_iteration": 2.837268114089966 + }, + { + "auxiliary_loss_clip": 0.01089899, + "auxiliary_loss_mlp": 0.01037937, + "balance_loss_clip": 1.02962148, + "balance_loss_mlp": 1.01930428, + "epoch": 0.15736173176252105, + "flos": 25403747045760.0, + "grad_norm": 2.4011804470998475, + "language_loss": 0.8770901, + "learning_rate": 3.8322572565756195e-06, + "loss": 0.89836842, + "num_input_tokens_seen": 154597015, + "router_z_loss_clip": 0.60302734, + "router_z_loss_mlp": 0.1862793, + "step": 5423, + "time_per_iteration": 2.4920523166656494 + }, + { + "auxiliary_loss_clip": 0.01083057, + "auxiliary_loss_mlp": 0.01034941, + "balance_loss_clip": 1.02934456, + "balance_loss_mlp": 1.0175488, + "epoch": 0.15739074923103707, + "flos": 13946823705600.0, + "grad_norm": 2.449257236781709, + "language_loss": 0.67037898, + "learning_rate": 3.832181897543568e-06, + "loss": 0.69155896, + "num_input_tokens_seen": 154609670, + "router_z_loss_clip": 0.53759766, + "router_z_loss_mlp": 0.17401123, + "step": 5424, + "time_per_iteration": 2.388054609298706 + }, + { + "auxiliary_loss_clip": 0.01082129, + "auxiliary_loss_mlp": 0.01035188, + "balance_loss_clip": 1.02846599, + "balance_loss_mlp": 1.01867795, + "epoch": 0.15741976669955313, + "flos": 25697610892800.0, + "grad_norm": 1.9906926092562767, + "language_loss": 0.73897159, + "learning_rate": 3.832106522328944e-06, + "loss": 0.76014477, + "num_input_tokens_seen": 154625150, + "router_z_loss_clip": 0.53637695, + "router_z_loss_mlp": 0.16516113, + "step": 5425, + "time_per_iteration": 2.528820753097534 + }, + { + "auxiliary_loss_clip": 0.01077498, + "auxiliary_loss_mlp": 0.01028828, + "balance_loss_clip": 1.02634668, + "balance_loss_mlp": 1.01389074, + "epoch": 0.15744878416806918, + "flos": 19712799498240.0, + "grad_norm": 1.9952307727823566, + "language_loss": 0.69858772, + "learning_rate": 3.832031130932415e-06, + "loss": 0.71965098, + "num_input_tokens_seen": 154640635, + "router_z_loss_clip": 0.51171875, + "router_z_loss_mlp": 0.14929199, + "step": 5426, + "time_per_iteration": 2.4024791717529297 + }, + { + "auxiliary_loss_clip": 0.01012443, + "auxiliary_loss_mlp": 0.01002206, + "balance_loss_clip": 1.00338578, + "balance_loss_mlp": 1.00099623, + "epoch": 0.15747780163658523, + "flos": 69082324755840.0, + "grad_norm": 0.7074072510157905, + "language_loss": 0.46140546, + "learning_rate": 3.8319557233546446e-06, + "loss": 0.48155195, + "num_input_tokens_seen": 154699175, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01208496, + "step": 5427, + "time_per_iteration": 3.024247646331787 + }, + { + "auxiliary_loss_clip": 0.01081957, + "auxiliary_loss_mlp": 0.01035631, + "balance_loss_clip": 1.02784514, + "balance_loss_mlp": 1.01851869, + "epoch": 0.15750681910510128, + "flos": 24199767210240.0, + "grad_norm": 2.1891515374710995, + "language_loss": 0.82556307, + "learning_rate": 3.8318802995963e-06, + "loss": 0.84673893, + "num_input_tokens_seen": 154713945, + "router_z_loss_clip": 0.54150391, + "router_z_loss_mlp": 0.17108154, + "step": 5428, + "time_per_iteration": 2.4126176834106445 + }, + { + "auxiliary_loss_clip": 0.01011752, + "auxiliary_loss_mlp": 0.01002047, + "balance_loss_clip": 1.00249124, + "balance_loss_mlp": 1.0009321, + "epoch": 0.1575358365736173, + "flos": 74781546295680.0, + "grad_norm": 0.6259199464702614, + "language_loss": 0.46342587, + "learning_rate": 3.831804859658047e-06, + "loss": 0.48356387, + "num_input_tokens_seen": 154781365, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01116943, + "step": 5429, + "time_per_iteration": 3.2101492881774902 + }, + { + "auxiliary_loss_clip": 0.01087382, + "auxiliary_loss_mlp": 0.01037992, + "balance_loss_clip": 1.02694786, + "balance_loss_mlp": 1.01968169, + "epoch": 0.15756485404213336, + "flos": 17861704012800.0, + "grad_norm": 2.8219591170365184, + "language_loss": 0.70643485, + "learning_rate": 3.831729403540553e-06, + "loss": 0.72768855, + "num_input_tokens_seen": 154796105, + "router_z_loss_clip": 0.60498047, + "router_z_loss_mlp": 0.1829834, + "step": 5430, + "time_per_iteration": 2.3845062255859375 + }, + { + "auxiliary_loss_clip": 0.01083429, + "auxiliary_loss_mlp": 0.01035204, + "balance_loss_clip": 1.02726078, + "balance_loss_mlp": 1.01798379, + "epoch": 0.1575938715106494, + "flos": 14676717922560.0, + "grad_norm": 3.0725843664164496, + "language_loss": 0.73794854, + "learning_rate": 3.831653931244483e-06, + "loss": 0.75913489, + "num_input_tokens_seen": 154807700, + "router_z_loss_clip": 0.56103516, + "router_z_loss_mlp": 0.17218018, + "step": 5431, + "time_per_iteration": 2.376068115234375 + }, + { + "auxiliary_loss_clip": 0.01011771, + "auxiliary_loss_mlp": 0.01006086, + "balance_loss_clip": 1.00228441, + "balance_loss_mlp": 1.00498927, + "epoch": 0.15762288897916546, + "flos": 63539548485120.0, + "grad_norm": 0.6268129900862052, + "language_loss": 0.47163725, + "learning_rate": 3.831578442770505e-06, + "loss": 0.49181581, + "num_input_tokens_seen": 154873925, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01098633, + "step": 5432, + "time_per_iteration": 3.0508415699005127 + }, + { + "auxiliary_loss_clip": 0.01083419, + "auxiliary_loss_mlp": 0.01038368, + "balance_loss_clip": 1.02787602, + "balance_loss_mlp": 1.02092719, + "epoch": 0.1576519064476815, + "flos": 10588283913600.0, + "grad_norm": 3.4397481891364903, + "language_loss": 1.09105909, + "learning_rate": 3.831502938119284e-06, + "loss": 1.11227679, + "num_input_tokens_seen": 154884125, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.17443848, + "step": 5433, + "time_per_iteration": 2.3864550590515137 + }, + { + "auxiliary_loss_clip": 0.01087559, + "auxiliary_loss_mlp": 0.01041199, + "balance_loss_clip": 1.0298593, + "balance_loss_mlp": 1.02241135, + "epoch": 0.15768092391619756, + "flos": 11360562387840.0, + "grad_norm": 4.339418388538165, + "language_loss": 1.00992608, + "learning_rate": 3.831427417291489e-06, + "loss": 1.03121376, + "num_input_tokens_seen": 154894315, + "router_z_loss_clip": 0.57714844, + "router_z_loss_mlp": 0.18798828, + "step": 5434, + "time_per_iteration": 2.3393046855926514 + }, + { + "auxiliary_loss_clip": 0.01086841, + "auxiliary_loss_mlp": 0.01034702, + "balance_loss_clip": 1.02803826, + "balance_loss_mlp": 1.01623642, + "epoch": 0.1577099413847136, + "flos": 13473820339200.0, + "grad_norm": 1.9528512646800895, + "language_loss": 0.67005479, + "learning_rate": 3.831351880287786e-06, + "loss": 0.69127023, + "num_input_tokens_seen": 154906480, + "router_z_loss_clip": 0.58740234, + "router_z_loss_mlp": 0.18469238, + "step": 5435, + "time_per_iteration": 2.373438596725464 + }, + { + "auxiliary_loss_clip": 0.01086618, + "auxiliary_loss_mlp": 0.01034446, + "balance_loss_clip": 1.02908874, + "balance_loss_mlp": 1.01700616, + "epoch": 0.15773895885322964, + "flos": 19930587759360.0, + "grad_norm": 1.668800168416903, + "language_loss": 0.72381926, + "learning_rate": 3.8312763271088415e-06, + "loss": 0.74502993, + "num_input_tokens_seen": 154921025, + "router_z_loss_clip": 0.57617188, + "router_z_loss_mlp": 0.17443848, + "step": 5436, + "time_per_iteration": 2.4303958415985107 + }, + { + "auxiliary_loss_clip": 0.0101307, + "auxiliary_loss_mlp": 0.01002297, + "balance_loss_clip": 1.00389528, + "balance_loss_mlp": 1.00111723, + "epoch": 0.1577679763217457, + "flos": 69950964182400.0, + "grad_norm": 0.627407029441657, + "language_loss": 0.48497686, + "learning_rate": 3.831200757755323e-06, + "loss": 0.50513053, + "num_input_tokens_seen": 154981375, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01177979, + "step": 5437, + "time_per_iteration": 3.0241031646728516 + }, + { + "auxiliary_loss_clip": 0.01094322, + "auxiliary_loss_mlp": 0.01042286, + "balance_loss_clip": 1.03094721, + "balance_loss_mlp": 1.02192509, + "epoch": 0.15779699379026174, + "flos": 39157676156160.0, + "grad_norm": 3.035195534696625, + "language_loss": 0.8897683, + "learning_rate": 3.831125172227899e-06, + "loss": 0.91113436, + "num_input_tokens_seen": 154997115, + "router_z_loss_clip": 0.63427734, + "router_z_loss_mlp": 0.20336914, + "step": 5438, + "time_per_iteration": 2.62519907951355 + }, + { + "auxiliary_loss_clip": 0.01084152, + "auxiliary_loss_mlp": 0.0103457, + "balance_loss_clip": 1.03053117, + "balance_loss_mlp": 1.0194838, + "epoch": 0.1578260112587778, + "flos": 27592906026240.0, + "grad_norm": 2.0636980456757965, + "language_loss": 0.75131798, + "learning_rate": 3.831049570527236e-06, + "loss": 0.77250522, + "num_input_tokens_seen": 155016990, + "router_z_loss_clip": 0.53613281, + "router_z_loss_mlp": 0.15100098, + "step": 5439, + "time_per_iteration": 2.5327022075653076 + }, + { + "auxiliary_loss_clip": 0.01087938, + "auxiliary_loss_mlp": 0.01038956, + "balance_loss_clip": 1.02948248, + "balance_loss_mlp": 1.02151585, + "epoch": 0.15785502872729384, + "flos": 27702009624960.0, + "grad_norm": 2.3086285384794354, + "language_loss": 0.87109667, + "learning_rate": 3.830973952654002e-06, + "loss": 0.89236563, + "num_input_tokens_seen": 155031860, + "router_z_loss_clip": 0.58447266, + "router_z_loss_mlp": 0.17431641, + "step": 5440, + "time_per_iteration": 2.5857090950012207 + }, + { + "auxiliary_loss_clip": 0.01077397, + "auxiliary_loss_mlp": 0.01036826, + "balance_loss_clip": 1.02860355, + "balance_loss_mlp": 1.02275944, + "epoch": 0.15788404619580987, + "flos": 23470187195520.0, + "grad_norm": 2.108380864334658, + "language_loss": 0.6408186, + "learning_rate": 3.830898318608867e-06, + "loss": 0.66196084, + "num_input_tokens_seen": 155046005, + "router_z_loss_clip": 0.48779297, + "router_z_loss_mlp": 0.14074707, + "step": 5441, + "time_per_iteration": 2.435073137283325 + }, + { + "auxiliary_loss_clip": 0.01014641, + "auxiliary_loss_mlp": 0.01004265, + "balance_loss_clip": 1.00493526, + "balance_loss_mlp": 1.0030489, + "epoch": 0.15791306366432592, + "flos": 62979786656640.0, + "grad_norm": 0.7102380171507173, + "language_loss": 0.47737771, + "learning_rate": 3.830822668392496e-06, + "loss": 0.49756682, + "num_input_tokens_seen": 155101105, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.012146, + "step": 5442, + "time_per_iteration": 2.8921680450439453 + }, + { + "auxiliary_loss_clip": 0.01093096, + "auxiliary_loss_mlp": 0.01043696, + "balance_loss_clip": 1.03079915, + "balance_loss_mlp": 1.02369285, + "epoch": 0.15794208113284197, + "flos": 10441683648000.0, + "grad_norm": 2.3426755332741136, + "language_loss": 0.86056972, + "learning_rate": 3.830747002005559e-06, + "loss": 0.88193762, + "num_input_tokens_seen": 155113315, + "router_z_loss_clip": 0.62304688, + "router_z_loss_mlp": 0.20007324, + "step": 5443, + "time_per_iteration": 2.5437660217285156 + }, + { + "auxiliary_loss_clip": 0.01016024, + "auxiliary_loss_mlp": 0.01001134, + "balance_loss_clip": 1.00590479, + "balance_loss_mlp": 0.99991852, + "epoch": 0.15797109860135802, + "flos": 63130576285440.0, + "grad_norm": 0.648958751546658, + "language_loss": 0.50958997, + "learning_rate": 3.830671319448722e-06, + "loss": 0.52976161, + "num_input_tokens_seen": 155174625, + "router_z_loss_clip": 0.10107422, + "router_z_loss_mlp": 0.012146, + "step": 5444, + "time_per_iteration": 3.029649496078491 + }, + { + "auxiliary_loss_clip": 0.01085522, + "auxiliary_loss_mlp": 0.0103266, + "balance_loss_clip": 1.0319438, + "balance_loss_mlp": 1.01679897, + "epoch": 0.15800011606987407, + "flos": 11648945151360.0, + "grad_norm": 2.383949590544654, + "language_loss": 0.71680051, + "learning_rate": 3.830595620722656e-06, + "loss": 0.73798227, + "num_input_tokens_seen": 155187260, + "router_z_loss_clip": 0.53515625, + "router_z_loss_mlp": 0.15869141, + "step": 5445, + "time_per_iteration": 2.3644630908966064 + }, + { + "auxiliary_loss_clip": 0.01097086, + "auxiliary_loss_mlp": 0.01042548, + "balance_loss_clip": 1.03243232, + "balance_loss_mlp": 1.02112603, + "epoch": 0.1580291335383901, + "flos": 24199103894400.0, + "grad_norm": 4.083231152246533, + "language_loss": 0.98170322, + "learning_rate": 3.8305199058280294e-06, + "loss": 1.00309956, + "num_input_tokens_seen": 155200975, + "router_z_loss_clip": 0.64648438, + "router_z_loss_mlp": 0.21398926, + "step": 5446, + "time_per_iteration": 2.453381299972534 + }, + { + "auxiliary_loss_clip": 0.01094894, + "auxiliary_loss_mlp": 0.01036819, + "balance_loss_clip": 1.03362226, + "balance_loss_mlp": 1.01824045, + "epoch": 0.15805815100690615, + "flos": 26863395834240.0, + "grad_norm": 2.1141097764094403, + "language_loss": 0.93097484, + "learning_rate": 3.8304441747655096e-06, + "loss": 0.95229197, + "num_input_tokens_seen": 155219505, + "router_z_loss_clip": 0.61279297, + "router_z_loss_mlp": 0.18572998, + "step": 5447, + "time_per_iteration": 2.4637744426727295 + }, + { + "auxiliary_loss_clip": 0.01015002, + "auxiliary_loss_mlp": 0.01001499, + "balance_loss_clip": 1.00575829, + "balance_loss_mlp": 1.00054514, + "epoch": 0.1580871684754222, + "flos": 66123086716800.0, + "grad_norm": 0.7593924725551184, + "language_loss": 0.52475643, + "learning_rate": 3.830368427535766e-06, + "loss": 0.5449214, + "num_input_tokens_seen": 155274495, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.00952148, + "step": 5448, + "time_per_iteration": 2.940983295440674 + }, + { + "auxiliary_loss_clip": 0.01089422, + "auxiliary_loss_mlp": 0.01045289, + "balance_loss_clip": 1.02976251, + "balance_loss_mlp": 1.02611399, + "epoch": 0.15811618594393825, + "flos": 25950591671040.0, + "grad_norm": 2.133620659783253, + "language_loss": 0.80747533, + "learning_rate": 3.830292664139468e-06, + "loss": 0.82882249, + "num_input_tokens_seen": 155291645, + "router_z_loss_clip": 0.59716797, + "router_z_loss_mlp": 0.19165039, + "step": 5449, + "time_per_iteration": 2.4372503757476807 + }, + { + "auxiliary_loss_clip": 0.01090093, + "auxiliary_loss_mlp": 0.01044887, + "balance_loss_clip": 1.03126884, + "balance_loss_mlp": 1.02718449, + "epoch": 0.1581452034124543, + "flos": 22884135246720.0, + "grad_norm": 2.2907809341375205, + "language_loss": 0.74753064, + "learning_rate": 3.830216884577284e-06, + "loss": 0.76888043, + "num_input_tokens_seen": 155305145, + "router_z_loss_clip": 0.58886719, + "router_z_loss_mlp": 0.17712402, + "step": 5450, + "time_per_iteration": 2.4727120399475098 + }, + { + "auxiliary_loss_clip": 0.01092971, + "auxiliary_loss_mlp": 0.01039247, + "balance_loss_clip": 1.03163981, + "balance_loss_mlp": 1.02031672, + "epoch": 0.15817422088097036, + "flos": 20183743094400.0, + "grad_norm": 2.8131471657163774, + "language_loss": 0.85275793, + "learning_rate": 3.830141088849885e-06, + "loss": 0.87408012, + "num_input_tokens_seen": 155317720, + "router_z_loss_clip": 0.61254883, + "router_z_loss_mlp": 0.18933105, + "step": 5451, + "time_per_iteration": 2.347378969192505 + }, + { + "auxiliary_loss_clip": 0.01090252, + "auxiliary_loss_mlp": 0.0104889, + "balance_loss_clip": 1.02924049, + "balance_loss_mlp": 1.02855325, + "epoch": 0.15820323834948638, + "flos": 10484277373440.0, + "grad_norm": 2.6640799249183846, + "language_loss": 0.82888281, + "learning_rate": 3.830065276957939e-06, + "loss": 0.85027421, + "num_input_tokens_seen": 155331865, + "router_z_loss_clip": 0.61010742, + "router_z_loss_mlp": 0.20336914, + "step": 5452, + "time_per_iteration": 2.473543405532837 + }, + { + "auxiliary_loss_clip": 0.01084088, + "auxiliary_loss_mlp": 0.0105151, + "balance_loss_clip": 1.02893519, + "balance_loss_mlp": 1.03397417, + "epoch": 0.15823225581800243, + "flos": 29194581692160.0, + "grad_norm": 2.6149111177523006, + "language_loss": 0.80494773, + "learning_rate": 3.829989448902116e-06, + "loss": 0.82630372, + "num_input_tokens_seen": 155345575, + "router_z_loss_clip": 0.55175781, + "router_z_loss_mlp": 0.17541504, + "step": 5453, + "time_per_iteration": 2.4492523670196533 + }, + { + "auxiliary_loss_clip": 0.01012495, + "auxiliary_loss_mlp": 0.01012349, + "balance_loss_clip": 1.0039705, + "balance_loss_mlp": 1.01128769, + "epoch": 0.15826127328651848, + "flos": 61493708102400.0, + "grad_norm": 0.6788993540662535, + "language_loss": 0.4769932, + "learning_rate": 3.829913604683085e-06, + "loss": 0.49724162, + "num_input_tokens_seen": 155407290, + "router_z_loss_clip": 0.08496094, + "router_z_loss_mlp": 0.01062012, + "step": 5454, + "time_per_iteration": 2.9941246509552 + }, + { + "auxiliary_loss_clip": 0.0108762, + "auxiliary_loss_mlp": 0.01045333, + "balance_loss_clip": 1.03019202, + "balance_loss_mlp": 1.02737403, + "epoch": 0.15829029075503454, + "flos": 42076100949120.0, + "grad_norm": 2.01865753289026, + "language_loss": 0.93088007, + "learning_rate": 3.8298377443015165e-06, + "loss": 0.95220959, + "num_input_tokens_seen": 155427570, + "router_z_loss_clip": 0.57421875, + "router_z_loss_mlp": 0.1796875, + "step": 5455, + "time_per_iteration": 2.570726156234741 + }, + { + "auxiliary_loss_clip": 0.01085945, + "auxiliary_loss_mlp": 0.01039627, + "balance_loss_clip": 1.02998412, + "balance_loss_mlp": 1.02231169, + "epoch": 0.1583193082235506, + "flos": 22775345850240.0, + "grad_norm": 2.17908237582308, + "language_loss": 0.91657746, + "learning_rate": 3.829761867758081e-06, + "loss": 0.93783319, + "num_input_tokens_seen": 155440070, + "router_z_loss_clip": 0.55883789, + "router_z_loss_mlp": 0.17333984, + "step": 5456, + "time_per_iteration": 2.6210367679595947 + }, + { + "auxiliary_loss_clip": 0.01090955, + "auxiliary_loss_mlp": 0.01042042, + "balance_loss_clip": 1.03111935, + "balance_loss_mlp": 1.0233382, + "epoch": 0.1583483256920666, + "flos": 12412076849280.0, + "grad_norm": 2.4674251699512153, + "language_loss": 0.89474589, + "learning_rate": 3.829685975053448e-06, + "loss": 0.91607583, + "num_input_tokens_seen": 155455055, + "router_z_loss_clip": 0.59863281, + "router_z_loss_mlp": 0.18676758, + "step": 5457, + "time_per_iteration": 2.3676486015319824 + }, + { + "auxiliary_loss_clip": 0.01091132, + "auxiliary_loss_mlp": 0.01041692, + "balance_loss_clip": 1.03208399, + "balance_loss_mlp": 1.02408445, + "epoch": 0.15837734316058266, + "flos": 26899565869440.0, + "grad_norm": 2.266996219341584, + "language_loss": 0.99201989, + "learning_rate": 3.829610066188288e-06, + "loss": 1.0133481, + "num_input_tokens_seen": 155469645, + "router_z_loss_clip": 0.59082031, + "router_z_loss_mlp": 0.17608643, + "step": 5458, + "time_per_iteration": 2.460080146789551 + }, + { + "auxiliary_loss_clip": 0.0108369, + "auxiliary_loss_mlp": 0.01038291, + "balance_loss_clip": 1.02704012, + "balance_loss_mlp": 1.02012992, + "epoch": 0.1584063606290987, + "flos": 15406088469120.0, + "grad_norm": 2.6009384056406244, + "language_loss": 0.66796279, + "learning_rate": 3.829534141163273e-06, + "loss": 0.68918264, + "num_input_tokens_seen": 155484005, + "router_z_loss_clip": 0.56591797, + "router_z_loss_mlp": 0.18170166, + "step": 5459, + "time_per_iteration": 2.3516812324523926 + }, + { + "auxiliary_loss_clip": 0.01012784, + "auxiliary_loss_mlp": 0.01007508, + "balance_loss_clip": 1.00397086, + "balance_loss_mlp": 1.00638151, + "epoch": 0.15843537809761477, + "flos": 61961020917120.0, + "grad_norm": 0.6839688379017814, + "language_loss": 0.45610645, + "learning_rate": 3.82945819997907e-06, + "loss": 0.47630939, + "num_input_tokens_seen": 155540640, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.0112915, + "step": 5460, + "time_per_iteration": 2.9034907817840576 + }, + { + "auxiliary_loss_clip": 0.01084455, + "auxiliary_loss_mlp": 0.01033521, + "balance_loss_clip": 1.02884662, + "balance_loss_mlp": 1.01593733, + "epoch": 0.15846439556613082, + "flos": 20514440292480.0, + "grad_norm": 2.22228228404738, + "language_loss": 0.69730532, + "learning_rate": 3.829382242636354e-06, + "loss": 0.71848506, + "num_input_tokens_seen": 155554915, + "router_z_loss_clip": 0.55566406, + "router_z_loss_mlp": 0.17590332, + "step": 5461, + "time_per_iteration": 2.383988380432129 + }, + { + "auxiliary_loss_clip": 0.01014589, + "auxiliary_loss_mlp": 0.01000192, + "balance_loss_clip": 1.00532937, + "balance_loss_mlp": 0.99907148, + "epoch": 0.15849341303464687, + "flos": 64664205978240.0, + "grad_norm": 0.6487329073081151, + "language_loss": 0.50833791, + "learning_rate": 3.829306269135792e-06, + "loss": 0.52848577, + "num_input_tokens_seen": 155615830, + "router_z_loss_clip": 0.09228516, + "router_z_loss_mlp": 0.01123047, + "step": 5462, + "time_per_iteration": 2.9521586894989014 + }, + { + "auxiliary_loss_clip": 0.01089736, + "auxiliary_loss_mlp": 0.01038562, + "balance_loss_clip": 1.03199601, + "balance_loss_mlp": 1.02144337, + "epoch": 0.1585224305031629, + "flos": 12705766139520.0, + "grad_norm": 1.97380394538921, + "language_loss": 0.79049683, + "learning_rate": 3.829230279478058e-06, + "loss": 0.8117798, + "num_input_tokens_seen": 155628010, + "router_z_loss_clip": 0.57666016, + "router_z_loss_mlp": 0.17126465, + "step": 5463, + "time_per_iteration": 2.3589279651641846 + }, + { + "auxiliary_loss_clip": 0.01091101, + "auxiliary_loss_mlp": 0.01036813, + "balance_loss_clip": 1.03207886, + "balance_loss_mlp": 1.01969421, + "epoch": 0.15855144797167894, + "flos": 13253762839680.0, + "grad_norm": 3.0691630012447377, + "language_loss": 0.7774896, + "learning_rate": 3.829154273663821e-06, + "loss": 0.79876876, + "num_input_tokens_seen": 155640200, + "router_z_loss_clip": 0.59033203, + "router_z_loss_mlp": 0.17102051, + "step": 5464, + "time_per_iteration": 2.4216527938842773 + }, + { + "auxiliary_loss_clip": 0.01087597, + "auxiliary_loss_mlp": 0.01041165, + "balance_loss_clip": 1.03177142, + "balance_loss_mlp": 1.02428496, + "epoch": 0.158580465440195, + "flos": 32190967284480.0, + "grad_norm": 2.0383679423610475, + "language_loss": 0.9414252, + "learning_rate": 3.829078251693753e-06, + "loss": 0.96271276, + "num_input_tokens_seen": 155661665, + "router_z_loss_clip": 0.55786133, + "router_z_loss_mlp": 0.16900635, + "step": 5465, + "time_per_iteration": 2.5956788063049316 + }, + { + "auxiliary_loss_clip": 0.01088082, + "auxiliary_loss_mlp": 0.0103387, + "balance_loss_clip": 1.03312707, + "balance_loss_mlp": 1.0166204, + "epoch": 0.15860948290871105, + "flos": 38682159171840.0, + "grad_norm": 2.2761718751172255, + "language_loss": 0.82951754, + "learning_rate": 3.829002213568526e-06, + "loss": 0.85073709, + "num_input_tokens_seen": 155676705, + "router_z_loss_clip": 0.54980469, + "router_z_loss_mlp": 0.17254639, + "step": 5466, + "time_per_iteration": 2.5609731674194336 + }, + { + "auxiliary_loss_clip": 0.01088637, + "auxiliary_loss_mlp": 0.01045743, + "balance_loss_clip": 1.03351521, + "balance_loss_mlp": 1.02943492, + "epoch": 0.1586385003772271, + "flos": 24963946248960.0, + "grad_norm": 2.7630269117909174, + "language_loss": 0.94318914, + "learning_rate": 3.828926159288812e-06, + "loss": 0.96453285, + "num_input_tokens_seen": 155691290, + "router_z_loss_clip": 0.55151367, + "router_z_loss_mlp": 0.16308594, + "step": 5467, + "time_per_iteration": 2.418247938156128 + }, + { + "auxiliary_loss_clip": 0.01088476, + "auxiliary_loss_mlp": 0.0103834, + "balance_loss_clip": 1.03282952, + "balance_loss_mlp": 1.02095914, + "epoch": 0.15866751784574315, + "flos": 12667012663680.0, + "grad_norm": 2.9988938149727864, + "language_loss": 0.78615797, + "learning_rate": 3.828850088855282e-06, + "loss": 0.8074261, + "num_input_tokens_seen": 155701230, + "router_z_loss_clip": 0.55664062, + "router_z_loss_mlp": 0.17370605, + "step": 5468, + "time_per_iteration": 2.350708246231079 + }, + { + "auxiliary_loss_clip": 0.01021544, + "auxiliary_loss_mlp": 0.01002411, + "balance_loss_clip": 1.01095891, + "balance_loss_mlp": 1.00124884, + "epoch": 0.15869653531425917, + "flos": 55789005479040.0, + "grad_norm": 0.6942549787346866, + "language_loss": 0.46752295, + "learning_rate": 3.828774002268608e-06, + "loss": 0.48776251, + "num_input_tokens_seen": 155754945, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.01159668, + "step": 5469, + "time_per_iteration": 2.8907856941223145 + }, + { + "auxiliary_loss_clip": 0.01019727, + "auxiliary_loss_mlp": 0.01003212, + "balance_loss_clip": 1.00941563, + "balance_loss_mlp": 1.00217533, + "epoch": 0.15872555278277523, + "flos": 74768210156160.0, + "grad_norm": 0.6282902174388334, + "language_loss": 0.47246575, + "learning_rate": 3.828697899529461e-06, + "loss": 0.49269515, + "num_input_tokens_seen": 155822315, + "router_z_loss_clip": 0.10302734, + "router_z_loss_mlp": 0.01037598, + "step": 5470, + "time_per_iteration": 3.147270441055298 + }, + { + "auxiliary_loss_clip": 0.01084764, + "auxiliary_loss_mlp": 0.01051081, + "balance_loss_clip": 1.03129911, + "balance_loss_mlp": 1.03464174, + "epoch": 0.15875457025129128, + "flos": 49371133046400.0, + "grad_norm": 2.4108264802980464, + "language_loss": 0.66873932, + "learning_rate": 3.828621780638515e-06, + "loss": 0.69009781, + "num_input_tokens_seen": 155838145, + "router_z_loss_clip": 0.53491211, + "router_z_loss_mlp": 0.16448975, + "step": 5471, + "time_per_iteration": 2.6873619556427 + }, + { + "auxiliary_loss_clip": 0.01015983, + "auxiliary_loss_mlp": 0.01001605, + "balance_loss_clip": 1.00560713, + "balance_loss_mlp": 1.00047278, + "epoch": 0.15878358771980733, + "flos": 58349395612800.0, + "grad_norm": 0.6996229482712404, + "language_loss": 0.46326682, + "learning_rate": 3.828545645596442e-06, + "loss": 0.48344266, + "num_input_tokens_seen": 155891600, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01135254, + "step": 5472, + "time_per_iteration": 2.822795867919922 + }, + { + "auxiliary_loss_clip": 0.01088196, + "auxiliary_loss_mlp": 0.01045383, + "balance_loss_clip": 1.02929258, + "balance_loss_mlp": 1.02790713, + "epoch": 0.15881260518832338, + "flos": 12488361903360.0, + "grad_norm": 2.4340577625509177, + "language_loss": 0.81673229, + "learning_rate": 3.828469494403913e-06, + "loss": 0.83806813, + "num_input_tokens_seen": 155904770, + "router_z_loss_clip": 0.58911133, + "router_z_loss_mlp": 0.17480469, + "step": 5473, + "time_per_iteration": 2.415804147720337 + }, + { + "auxiliary_loss_clip": 0.01089439, + "auxiliary_loss_mlp": 0.01035805, + "balance_loss_clip": 1.03138733, + "balance_loss_mlp": 1.01836491, + "epoch": 0.1588416226568394, + "flos": 40399781063040.0, + "grad_norm": 2.1671121859655886, + "language_loss": 0.79574031, + "learning_rate": 3.828393327061602e-06, + "loss": 0.81699276, + "num_input_tokens_seen": 155921130, + "router_z_loss_clip": 0.58007812, + "router_z_loss_mlp": 0.17468262, + "step": 5474, + "time_per_iteration": 2.537808895111084 + }, + { + "auxiliary_loss_clip": 0.01013557, + "auxiliary_loss_mlp": 0.01009477, + "balance_loss_clip": 1.00416565, + "balance_loss_mlp": 1.00839269, + "epoch": 0.15887064012535546, + "flos": 57040780832640.0, + "grad_norm": 0.6352945281599622, + "language_loss": 0.45278829, + "learning_rate": 3.8283171435701805e-06, + "loss": 0.47301859, + "num_input_tokens_seen": 155980125, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.01086426, + "step": 5475, + "time_per_iteration": 2.902068614959717 + }, + { + "auxiliary_loss_clip": 0.01096067, + "auxiliary_loss_mlp": 0.01046112, + "balance_loss_clip": 1.03414416, + "balance_loss_mlp": 1.02724123, + "epoch": 0.1588996575938715, + "flos": 30329188922880.0, + "grad_norm": 3.656423582755904, + "language_loss": 0.88898659, + "learning_rate": 3.828240943930323e-06, + "loss": 0.91040838, + "num_input_tokens_seen": 155995950, + "router_z_loss_clip": 0.61865234, + "router_z_loss_mlp": 0.18884277, + "step": 5476, + "time_per_iteration": 4.931640148162842 + }, + { + "auxiliary_loss_clip": 0.01089755, + "auxiliary_loss_mlp": 0.01044077, + "balance_loss_clip": 1.03279245, + "balance_loss_mlp": 1.02847242, + "epoch": 0.15892867506238756, + "flos": 32049952836480.0, + "grad_norm": 1.71014691568485, + "language_loss": 0.83230865, + "learning_rate": 3.828164728142701e-06, + "loss": 0.85364699, + "num_input_tokens_seen": 156017995, + "router_z_loss_clip": 0.56982422, + "router_z_loss_mlp": 0.15600586, + "step": 5477, + "time_per_iteration": 2.477957010269165 + }, + { + "auxiliary_loss_clip": 0.01015168, + "auxiliary_loss_mlp": 0.00999708, + "balance_loss_clip": 1.00632656, + "balance_loss_mlp": 0.9986589, + "epoch": 0.1589576925309036, + "flos": 65181758106240.0, + "grad_norm": 0.6980256889088287, + "language_loss": 0.46435085, + "learning_rate": 3.8280884962079885e-06, + "loss": 0.4844996, + "num_input_tokens_seen": 156078535, + "router_z_loss_clip": 0.08837891, + "router_z_loss_mlp": 0.01049805, + "step": 5478, + "time_per_iteration": 2.9828529357910156 + }, + { + "auxiliary_loss_clip": 0.01080528, + "auxiliary_loss_mlp": 0.01034926, + "balance_loss_clip": 1.02811766, + "balance_loss_mlp": 1.01978612, + "epoch": 0.15898670999941966, + "flos": 12486337044480.0, + "grad_norm": 1.9030113438616747, + "language_loss": 0.71393895, + "learning_rate": 3.828012248126859e-06, + "loss": 0.73509347, + "num_input_tokens_seen": 156090020, + "router_z_loss_clip": 0.5234375, + "router_z_loss_mlp": 0.15136719, + "step": 5479, + "time_per_iteration": 4.6610987186431885 + }, + { + "auxiliary_loss_clip": 0.01089516, + "auxiliary_loss_mlp": 0.010449, + "balance_loss_clip": 1.02883017, + "balance_loss_mlp": 1.02778769, + "epoch": 0.15901572746793569, + "flos": 19456362495360.0, + "grad_norm": 3.188722045744154, + "language_loss": 0.82300377, + "learning_rate": 3.827935983899985e-06, + "loss": 0.84434795, + "num_input_tokens_seen": 156103295, + "router_z_loss_clip": 0.60644531, + "router_z_loss_mlp": 0.17120361, + "step": 5480, + "time_per_iteration": 2.3303768634796143 + }, + { + "auxiliary_loss_clip": 0.01086224, + "auxiliary_loss_mlp": 0.01045942, + "balance_loss_clip": 1.03085184, + "balance_loss_mlp": 1.02906179, + "epoch": 0.15904474493645174, + "flos": 13179747024000.0, + "grad_norm": 2.806030379772304, + "language_loss": 0.75304079, + "learning_rate": 3.827859703528042e-06, + "loss": 0.77436244, + "num_input_tokens_seen": 156113795, + "router_z_loss_clip": 0.55419922, + "router_z_loss_mlp": 0.16870117, + "step": 5481, + "time_per_iteration": 2.5740127563476562 + }, + { + "auxiliary_loss_clip": 0.01017847, + "auxiliary_loss_mlp": 0.01001514, + "balance_loss_clip": 1.00842464, + "balance_loss_mlp": 1.00040495, + "epoch": 0.1590737624049678, + "flos": 63813093920640.0, + "grad_norm": 0.6514599938476394, + "language_loss": 0.46133178, + "learning_rate": 3.827783407011701e-06, + "loss": 0.48152536, + "num_input_tokens_seen": 156174840, + "router_z_loss_clip": 0.09423828, + "router_z_loss_mlp": 0.0111084, + "step": 5482, + "time_per_iteration": 3.0030124187469482 + }, + { + "auxiliary_loss_clip": 0.01096531, + "auxiliary_loss_mlp": 0.01041165, + "balance_loss_clip": 1.03335464, + "balance_loss_mlp": 1.02117336, + "epoch": 0.15910277987348384, + "flos": 29234103217920.0, + "grad_norm": 2.209246597863261, + "language_loss": 0.90074062, + "learning_rate": 3.8277070943516384e-06, + "loss": 0.92211759, + "num_input_tokens_seen": 156199035, + "router_z_loss_clip": 0.63183594, + "router_z_loss_mlp": 0.20001221, + "step": 5483, + "time_per_iteration": 2.799506664276123 + }, + { + "auxiliary_loss_clip": 0.01019624, + "auxiliary_loss_mlp": 0.01008224, + "balance_loss_clip": 1.01007199, + "balance_loss_mlp": 1.00721693, + "epoch": 0.1591317973419999, + "flos": 67365575648640.0, + "grad_norm": 0.8591176617244957, + "language_loss": 0.50037646, + "learning_rate": 3.827630765548527e-06, + "loss": 0.52065492, + "num_input_tokens_seen": 156259805, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.0100708, + "step": 5484, + "time_per_iteration": 3.089501142501831 + }, + { + "auxiliary_loss_clip": 0.01018631, + "auxiliary_loss_mlp": 0.01002581, + "balance_loss_clip": 1.00914669, + "balance_loss_mlp": 1.00154388, + "epoch": 0.15916081481051594, + "flos": 59633081815680.0, + "grad_norm": 0.6836640265382269, + "language_loss": 0.53835976, + "learning_rate": 3.827554420603041e-06, + "loss": 0.55857182, + "num_input_tokens_seen": 156319440, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01037598, + "step": 5485, + "time_per_iteration": 3.059277296066284 + }, + { + "auxiliary_loss_clip": 0.01017692, + "auxiliary_loss_mlp": 0.01004741, + "balance_loss_clip": 1.00833225, + "balance_loss_mlp": 1.00374532, + "epoch": 0.15918983227903197, + "flos": 72400749528960.0, + "grad_norm": 0.6546626179987799, + "language_loss": 0.48459142, + "learning_rate": 3.827478059515854e-06, + "loss": 0.50481576, + "num_input_tokens_seen": 156390040, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.00994873, + "step": 5486, + "time_per_iteration": 3.0953657627105713 + }, + { + "auxiliary_loss_clip": 0.01089703, + "auxiliary_loss_mlp": 0.01034172, + "balance_loss_clip": 1.03159416, + "balance_loss_mlp": 1.01401341, + "epoch": 0.15921884974754802, + "flos": 24458019603840.0, + "grad_norm": 1.9974852933787042, + "language_loss": 0.88129604, + "learning_rate": 3.827401682287642e-06, + "loss": 0.9025349, + "num_input_tokens_seen": 156407655, + "router_z_loss_clip": 0.58081055, + "router_z_loss_mlp": 0.20178223, + "step": 5487, + "time_per_iteration": 7.269043922424316 + }, + { + "auxiliary_loss_clip": 0.01017094, + "auxiliary_loss_mlp": 0.01003543, + "balance_loss_clip": 1.00757384, + "balance_loss_mlp": 1.0024823, + "epoch": 0.15924786721606407, + "flos": 66126193827840.0, + "grad_norm": 0.6446724919166433, + "language_loss": 0.45959753, + "learning_rate": 3.827325288919079e-06, + "loss": 0.47980392, + "num_input_tokens_seen": 156468495, + "router_z_loss_clip": 0.09521484, + "router_z_loss_mlp": 0.01062012, + "step": 5488, + "time_per_iteration": 3.009526252746582 + }, + { + "auxiliary_loss_clip": 0.01089875, + "auxiliary_loss_mlp": 0.01038799, + "balance_loss_clip": 1.02840018, + "balance_loss_mlp": 1.01797318, + "epoch": 0.15927688468458012, + "flos": 23032027232640.0, + "grad_norm": 2.1659087658370937, + "language_loss": 0.96974814, + "learning_rate": 3.827248879410839e-06, + "loss": 0.99103487, + "num_input_tokens_seen": 156484090, + "router_z_loss_clip": 0.61425781, + "router_z_loss_mlp": 0.20812988, + "step": 5489, + "time_per_iteration": 2.4125239849090576 + }, + { + "auxiliary_loss_clip": 0.01015201, + "auxiliary_loss_mlp": 0.01000853, + "balance_loss_clip": 1.00561786, + "balance_loss_mlp": 0.99992925, + "epoch": 0.15930590215309617, + "flos": 74775052782720.0, + "grad_norm": 0.6511748256322799, + "language_loss": 0.47604886, + "learning_rate": 3.827172453763598e-06, + "loss": 0.49620941, + "num_input_tokens_seen": 156551520, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.00921631, + "step": 5490, + "time_per_iteration": 3.108614921569824 + }, + { + "auxiliary_loss_clip": 0.0101388, + "auxiliary_loss_mlp": 0.01001114, + "balance_loss_clip": 1.00444674, + "balance_loss_mlp": 1.0001725, + "epoch": 0.1593349196216122, + "flos": 73419550179840.0, + "grad_norm": 0.6648328182859106, + "language_loss": 0.48527721, + "learning_rate": 3.82709601197803e-06, + "loss": 0.50542712, + "num_input_tokens_seen": 156613155, + "router_z_loss_clip": 0.09423828, + "router_z_loss_mlp": 0.00939941, + "step": 5491, + "time_per_iteration": 3.1972546577453613 + }, + { + "auxiliary_loss_clip": 0.01012913, + "auxiliary_loss_mlp": 0.0100362, + "balance_loss_clip": 1.00377119, + "balance_loss_mlp": 1.00263643, + "epoch": 0.15936393709012825, + "flos": 71418188736000.0, + "grad_norm": 0.6548384234680211, + "language_loss": 0.50180197, + "learning_rate": 3.827019554054811e-06, + "loss": 0.52196729, + "num_input_tokens_seen": 156675235, + "router_z_loss_clip": 0.09130859, + "router_z_loss_mlp": 0.00982666, + "step": 5492, + "time_per_iteration": 3.0853397846221924 + }, + { + "auxiliary_loss_clip": 0.01012561, + "auxiliary_loss_mlp": 0.0100481, + "balance_loss_clip": 1.0035094, + "balance_loss_mlp": 1.00376081, + "epoch": 0.1593929545586443, + "flos": 63935952595200.0, + "grad_norm": 0.6822611507288145, + "language_loss": 0.4868499, + "learning_rate": 3.826943079994616e-06, + "loss": 0.50702357, + "num_input_tokens_seen": 156736975, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01049805, + "step": 5493, + "time_per_iteration": 3.0352957248687744 + }, + { + "auxiliary_loss_clip": 0.01012948, + "auxiliary_loss_mlp": 0.01004049, + "balance_loss_clip": 1.00380969, + "balance_loss_mlp": 1.00299978, + "epoch": 0.15942197202716035, + "flos": 74771212533120.0, + "grad_norm": 0.6570192274745462, + "language_loss": 0.47884333, + "learning_rate": 3.826866589798121e-06, + "loss": 0.4990133, + "num_input_tokens_seen": 156798550, + "router_z_loss_clip": 0.09130859, + "router_z_loss_mlp": 0.01049805, + "step": 5494, + "time_per_iteration": 3.034020185470581 + }, + { + "auxiliary_loss_clip": 0.0101291, + "auxiliary_loss_mlp": 0.01003528, + "balance_loss_clip": 1.00394058, + "balance_loss_mlp": 1.00243723, + "epoch": 0.1594509894956764, + "flos": 63089693216640.0, + "grad_norm": 0.7424257045296618, + "language_loss": 0.5077126, + "learning_rate": 3.826790083466e-06, + "loss": 0.52787697, + "num_input_tokens_seen": 156858150, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01092529, + "step": 5495, + "time_per_iteration": 2.9439029693603516 + }, + { + "auxiliary_loss_clip": 0.0101361, + "auxiliary_loss_mlp": 0.01000982, + "balance_loss_clip": 1.00477028, + "balance_loss_mlp": 0.99998695, + "epoch": 0.15948000696419246, + "flos": 74771980583040.0, + "grad_norm": 0.6280849020165093, + "language_loss": 0.46009433, + "learning_rate": 3.826713560998931e-06, + "loss": 0.48024029, + "num_input_tokens_seen": 156918615, + "router_z_loss_clip": 0.08837891, + "router_z_loss_mlp": 0.00994873, + "step": 5496, + "time_per_iteration": 3.0456435680389404 + }, + { + "auxiliary_loss_clip": 0.01015799, + "auxiliary_loss_mlp": 0.01001838, + "balance_loss_clip": 1.00672936, + "balance_loss_mlp": 1.00073576, + "epoch": 0.15950902443270848, + "flos": 74788039808640.0, + "grad_norm": 0.6948116021995487, + "language_loss": 0.50861061, + "learning_rate": 3.826637022397588e-06, + "loss": 0.52878702, + "num_input_tokens_seen": 156991100, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01104736, + "step": 5497, + "time_per_iteration": 3.2199878692626953 + }, + { + "auxiliary_loss_clip": 0.0101673, + "auxiliary_loss_mlp": 0.01002211, + "balance_loss_clip": 1.00738072, + "balance_loss_mlp": 1.00114405, + "epoch": 0.15953804190122453, + "flos": 65319735265920.0, + "grad_norm": 0.7095374116060521, + "language_loss": 0.45639443, + "learning_rate": 3.826560467662647e-06, + "loss": 0.47658384, + "num_input_tokens_seen": 157055700, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.01068115, + "step": 5498, + "time_per_iteration": 3.085287570953369 + }, + { + "auxiliary_loss_clip": 0.01087288, + "auxiliary_loss_mlp": 0.01046083, + "balance_loss_clip": 1.03309679, + "balance_loss_mlp": 1.0300194, + "epoch": 0.15956705936974058, + "flos": 29307316072320.0, + "grad_norm": 3.0234261110366853, + "language_loss": 0.89727783, + "learning_rate": 3.826483896794785e-06, + "loss": 0.91861159, + "num_input_tokens_seen": 157070395, + "router_z_loss_clip": 0.54150391, + "router_z_loss_mlp": 0.1607666, + "step": 5499, + "time_per_iteration": 2.463684558868408 + }, + { + "auxiliary_loss_clip": 0.01020597, + "auxiliary_loss_mlp": 0.01003714, + "balance_loss_clip": 1.01101387, + "balance_loss_mlp": 1.00262368, + "epoch": 0.15959607683825663, + "flos": 67409600739840.0, + "grad_norm": 0.6280733712306082, + "language_loss": 0.50074726, + "learning_rate": 3.826407309794678e-06, + "loss": 0.52099037, + "num_input_tokens_seen": 157134590, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01092529, + "step": 5500, + "time_per_iteration": 3.041750907897949 + }, + { + "auxiliary_loss_clip": 0.01090095, + "auxiliary_loss_mlp": 0.01050363, + "balance_loss_clip": 1.03480816, + "balance_loss_mlp": 1.03245759, + "epoch": 0.15962509430677269, + "flos": 11536350416640.0, + "grad_norm": 3.084039991993527, + "language_loss": 0.86621445, + "learning_rate": 3.8263307066630035e-06, + "loss": 0.88761902, + "num_input_tokens_seen": 157145410, + "router_z_loss_clip": 0.55224609, + "router_z_loss_mlp": 0.17907715, + "step": 5501, + "time_per_iteration": 2.362125873565674 + }, + { + "auxiliary_loss_clip": 0.01089126, + "auxiliary_loss_mlp": 0.01039646, + "balance_loss_clip": 1.03384423, + "balance_loss_mlp": 1.02220607, + "epoch": 0.15965411177528874, + "flos": 13837859752320.0, + "grad_norm": 2.794307609064102, + "language_loss": 0.85956109, + "learning_rate": 3.826254087400437e-06, + "loss": 0.88084882, + "num_input_tokens_seen": 157157190, + "router_z_loss_clip": 0.55297852, + "router_z_loss_mlp": 0.17431641, + "step": 5502, + "time_per_iteration": 2.3883769512176514 + }, + { + "auxiliary_loss_clip": 0.0109481, + "auxiliary_loss_mlp": 0.0104555, + "balance_loss_clip": 1.0341661, + "balance_loss_mlp": 1.02723908, + "epoch": 0.15968312924380476, + "flos": 25914456547200.0, + "grad_norm": 2.121485858676699, + "language_loss": 0.7676897, + "learning_rate": 3.8261774520076545e-06, + "loss": 0.78909338, + "num_input_tokens_seen": 157174015, + "router_z_loss_clip": 0.60693359, + "router_z_loss_mlp": 0.18310547, + "step": 5503, + "time_per_iteration": 2.4441044330596924 + }, + { + "auxiliary_loss_clip": 0.01088423, + "auxiliary_loss_mlp": 0.01039614, + "balance_loss_clip": 1.0335021, + "balance_loss_mlp": 1.02252567, + "epoch": 0.1597121467123208, + "flos": 32481758931840.0, + "grad_norm": 2.2116767013645457, + "language_loss": 0.89055336, + "learning_rate": 3.826100800485335e-06, + "loss": 0.91183376, + "num_input_tokens_seen": 157194165, + "router_z_loss_clip": 0.54907227, + "router_z_loss_mlp": 0.17089844, + "step": 5504, + "time_per_iteration": 2.713972806930542 + }, + { + "auxiliary_loss_clip": 0.01035691, + "auxiliary_loss_mlp": 0.00999158, + "balance_loss_clip": 1.02528191, + "balance_loss_mlp": 0.99800122, + "epoch": 0.15974116418083686, + "flos": 67288313076480.0, + "grad_norm": 0.7115241154329304, + "language_loss": 0.49137095, + "learning_rate": 3.826024132834153e-06, + "loss": 0.51171941, + "num_input_tokens_seen": 157254490, + "router_z_loss_clip": 0.10400391, + "router_z_loss_mlp": 0.01153564, + "step": 5505, + "time_per_iteration": 3.2489068508148193 + }, + { + "auxiliary_loss_clip": 0.01094517, + "auxiliary_loss_mlp": 0.0104337, + "balance_loss_clip": 1.03709364, + "balance_loss_mlp": 1.02563775, + "epoch": 0.15977018164935292, + "flos": 34599345891840.0, + "grad_norm": 1.9616755925620795, + "language_loss": 0.98096895, + "learning_rate": 3.8259474490547875e-06, + "loss": 1.00234795, + "num_input_tokens_seen": 157272890, + "router_z_loss_clip": 0.57446289, + "router_z_loss_mlp": 0.17736816, + "step": 5506, + "time_per_iteration": 2.51411771774292 + }, + { + "auxiliary_loss_clip": 0.0103363, + "auxiliary_loss_mlp": 0.00998528, + "balance_loss_clip": 1.02307296, + "balance_loss_mlp": 0.99724019, + "epoch": 0.15979919911786897, + "flos": 63653016003840.0, + "grad_norm": 0.6310982709456362, + "language_loss": 0.48189285, + "learning_rate": 3.825870749147915e-06, + "loss": 0.50221437, + "num_input_tokens_seen": 157340100, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01287842, + "step": 5507, + "time_per_iteration": 3.1261277198791504 + }, + { + "auxiliary_loss_clip": 0.01091673, + "auxiliary_loss_mlp": 0.0103942, + "balance_loss_clip": 1.0348047, + "balance_loss_mlp": 1.02128804, + "epoch": 0.159828216586385, + "flos": 31423925514240.0, + "grad_norm": 1.8132425227620639, + "language_loss": 0.87540019, + "learning_rate": 3.825794033114214e-06, + "loss": 0.89671117, + "num_input_tokens_seen": 157360340, + "router_z_loss_clip": 0.57006836, + "router_z_loss_mlp": 0.18118286, + "step": 5508, + "time_per_iteration": 2.498326539993286 + }, + { + "auxiliary_loss_clip": 0.01029211, + "auxiliary_loss_mlp": 0.01010751, + "balance_loss_clip": 1.01888978, + "balance_loss_mlp": 1.00968432, + "epoch": 0.15985723405490104, + "flos": 70274644197120.0, + "grad_norm": 0.804304951039257, + "language_loss": 0.49857396, + "learning_rate": 3.825717300954361e-06, + "loss": 0.51897359, + "num_input_tokens_seen": 157418565, + "router_z_loss_clip": 0.10302734, + "router_z_loss_mlp": 0.01068115, + "step": 5509, + "time_per_iteration": 2.9220714569091797 + }, + { + "auxiliary_loss_clip": 0.01025818, + "auxiliary_loss_mlp": 0.01013965, + "balance_loss_clip": 1.01556551, + "balance_loss_mlp": 1.01274896, + "epoch": 0.1598862515234171, + "flos": 63266249295360.0, + "grad_norm": 0.7717670951988531, + "language_loss": 0.51735079, + "learning_rate": 3.825640552669034e-06, + "loss": 0.53774863, + "num_input_tokens_seen": 157474625, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.012146, + "step": 5510, + "time_per_iteration": 2.8838768005371094 + }, + { + "auxiliary_loss_clip": 0.01022989, + "auxiliary_loss_mlp": 0.01017581, + "balance_loss_clip": 1.01279807, + "balance_loss_mlp": 1.01640093, + "epoch": 0.15991526899193315, + "flos": 65072130837120.0, + "grad_norm": 0.6417041132835969, + "language_loss": 0.45275384, + "learning_rate": 3.825563788258911e-06, + "loss": 0.47315955, + "num_input_tokens_seen": 157536375, + "router_z_loss_clip": 0.10205078, + "router_z_loss_mlp": 0.01177979, + "step": 5511, + "time_per_iteration": 3.1382064819335938 + }, + { + "auxiliary_loss_clip": 0.01098329, + "auxiliary_loss_mlp": 0.01047445, + "balance_loss_clip": 1.0349797, + "balance_loss_mlp": 1.02735233, + "epoch": 0.1599442864604492, + "flos": 27152965584000.0, + "grad_norm": 5.492092324439336, + "language_loss": 0.95681202, + "learning_rate": 3.825487007724669e-06, + "loss": 0.97826976, + "num_input_tokens_seen": 157550625, + "router_z_loss_clip": 0.6328125, + "router_z_loss_mlp": 0.2008667, + "step": 5512, + "time_per_iteration": 2.6066341400146484 + }, + { + "auxiliary_loss_clip": 0.01023278, + "auxiliary_loss_mlp": 0.01012116, + "balance_loss_clip": 1.0138979, + "balance_loss_mlp": 1.01072764, + "epoch": 0.15997330392896525, + "flos": 74765207779200.0, + "grad_norm": 0.6823440400427386, + "language_loss": 0.51438981, + "learning_rate": 3.825410211066987e-06, + "loss": 0.53474379, + "num_input_tokens_seen": 157612330, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.01391602, + "step": 5513, + "time_per_iteration": 3.032057285308838 + }, + { + "auxiliary_loss_clip": 0.01023291, + "auxiliary_loss_mlp": 0.01009912, + "balance_loss_clip": 1.01378191, + "balance_loss_mlp": 1.00857711, + "epoch": 0.16000232139748127, + "flos": 68972732398080.0, + "grad_norm": 0.6754461469948735, + "language_loss": 0.50777858, + "learning_rate": 3.825333398286544e-06, + "loss": 0.52811062, + "num_input_tokens_seen": 157678455, + "router_z_loss_clip": 0.09521484, + "router_z_loss_mlp": 0.0133667, + "step": 5514, + "time_per_iteration": 3.2143795490264893 + }, + { + "auxiliary_loss_clip": 0.01097597, + "auxiliary_loss_mlp": 0.01037334, + "balance_loss_clip": 1.03537107, + "balance_loss_mlp": 1.01706815, + "epoch": 0.16003133886599732, + "flos": 29599015415040.0, + "grad_norm": 2.0933778434383923, + "language_loss": 0.65324974, + "learning_rate": 3.825256569384018e-06, + "loss": 0.67459905, + "num_input_tokens_seen": 157695005, + "router_z_loss_clip": 0.62133789, + "router_z_loss_mlp": 0.20275879, + "step": 5515, + "time_per_iteration": 2.4535341262817383 + }, + { + "auxiliary_loss_clip": 0.01084659, + "auxiliary_loss_mlp": 0.01038529, + "balance_loss_clip": 1.03434741, + "balance_loss_mlp": 1.02320433, + "epoch": 0.16006035633451338, + "flos": 36277132055040.0, + "grad_norm": 1.8435604602314122, + "language_loss": 0.86137891, + "learning_rate": 3.825179724360087e-06, + "loss": 0.8826108, + "num_input_tokens_seen": 157712575, + "router_z_loss_clip": 0.50268555, + "router_z_loss_mlp": 0.15332031, + "step": 5516, + "time_per_iteration": 2.754145860671997 + }, + { + "auxiliary_loss_clip": 0.01094371, + "auxiliary_loss_mlp": 0.01032254, + "balance_loss_clip": 1.03542066, + "balance_loss_mlp": 1.01312053, + "epoch": 0.16008937380302943, + "flos": 41129046875520.0, + "grad_norm": 1.7766606471967732, + "language_loss": 0.76252246, + "learning_rate": 3.825102863215431e-06, + "loss": 0.78378868, + "num_input_tokens_seen": 157733665, + "router_z_loss_clip": 0.58911133, + "router_z_loss_mlp": 0.19116211, + "step": 5517, + "time_per_iteration": 2.555241346359253 + }, + { + "auxiliary_loss_clip": 0.01101629, + "auxiliary_loss_mlp": 0.01042661, + "balance_loss_clip": 1.04005194, + "balance_loss_mlp": 1.02430248, + "epoch": 0.16011839127154548, + "flos": 33068090171520.0, + "grad_norm": 1.7099844910240762, + "language_loss": 0.94191009, + "learning_rate": 3.825025985950727e-06, + "loss": 0.96335304, + "num_input_tokens_seen": 157759790, + "router_z_loss_clip": 0.61572266, + "router_z_loss_mlp": 0.18347168, + "step": 5518, + "time_per_iteration": 2.5463945865631104 + }, + { + "auxiliary_loss_clip": 0.01099784, + "auxiliary_loss_mlp": 0.01039494, + "balance_loss_clip": 1.04321599, + "balance_loss_mlp": 1.02193427, + "epoch": 0.1601474087400615, + "flos": 28685897049600.0, + "grad_norm": 2.378218513804989, + "language_loss": 0.90553212, + "learning_rate": 3.824949092566655e-06, + "loss": 0.92692488, + "num_input_tokens_seen": 157774880, + "router_z_loss_clip": 0.56567383, + "router_z_loss_mlp": 0.17559814, + "step": 5519, + "time_per_iteration": 2.456545829772949 + }, + { + "auxiliary_loss_clip": 0.01105514, + "auxiliary_loss_mlp": 0.010412, + "balance_loss_clip": 1.04353189, + "balance_loss_mlp": 1.02179265, + "epoch": 0.16017642620857755, + "flos": 27264547889280.0, + "grad_norm": 2.190759807793442, + "language_loss": 0.79897171, + "learning_rate": 3.824872183063894e-06, + "loss": 0.82043886, + "num_input_tokens_seen": 157791620, + "router_z_loss_clip": 0.62011719, + "router_z_loss_mlp": 0.19421387, + "step": 5520, + "time_per_iteration": 2.4726412296295166 + }, + { + "auxiliary_loss_clip": 0.01105386, + "auxiliary_loss_mlp": 0.01042798, + "balance_loss_clip": 1.04643846, + "balance_loss_mlp": 1.02528572, + "epoch": 0.1602054436770936, + "flos": 17778087573120.0, + "grad_norm": 2.574848464175428, + "language_loss": 0.86841834, + "learning_rate": 3.824795257443125e-06, + "loss": 0.88990015, + "num_input_tokens_seen": 157805055, + "router_z_loss_clip": 0.58935547, + "router_z_loss_mlp": 0.17529297, + "step": 5521, + "time_per_iteration": 2.3709588050842285 + }, + { + "auxiliary_loss_clip": 0.01054796, + "auxiliary_loss_mlp": 0.01004829, + "balance_loss_clip": 1.04116154, + "balance_loss_mlp": 1.00338066, + "epoch": 0.16023446114560966, + "flos": 74760669302400.0, + "grad_norm": 0.6577445854838027, + "language_loss": 0.43131435, + "learning_rate": 3.824718315705023e-06, + "loss": 0.45191061, + "num_input_tokens_seen": 157860530, + "router_z_loss_clip": 0.13671875, + "router_z_loss_mlp": 0.01446533, + "step": 5522, + "time_per_iteration": 2.9338433742523193 + }, + { + "auxiliary_loss_clip": 0.01052644, + "auxiliary_loss_mlp": 0.01004727, + "balance_loss_clip": 1.03914857, + "balance_loss_mlp": 1.00326061, + "epoch": 0.1602634786141257, + "flos": 74768175244800.0, + "grad_norm": 0.6720417497709779, + "language_loss": 0.46985096, + "learning_rate": 3.824641357850273e-06, + "loss": 0.49042463, + "num_input_tokens_seen": 157924425, + "router_z_loss_clip": 0.13476562, + "router_z_loss_mlp": 0.01464844, + "step": 5523, + "time_per_iteration": 3.1793606281280518 + }, + { + "auxiliary_loss_clip": 0.01114568, + "auxiliary_loss_mlp": 0.01044956, + "balance_loss_clip": 1.05005157, + "balance_loss_mlp": 1.02323568, + "epoch": 0.16029249608264176, + "flos": 36384071149440.0, + "grad_norm": 2.390481170694356, + "language_loss": 0.87362862, + "learning_rate": 3.824564383879551e-06, + "loss": 0.89522386, + "num_input_tokens_seen": 157942305, + "router_z_loss_clip": 0.64453125, + "router_z_loss_mlp": 0.21728516, + "step": 5524, + "time_per_iteration": 2.524935483932495 + }, + { + "auxiliary_loss_clip": 0.01112483, + "auxiliary_loss_mlp": 0.01053126, + "balance_loss_clip": 1.04901373, + "balance_loss_mlp": 1.03562617, + "epoch": 0.16032151355115778, + "flos": 24465839748480.0, + "grad_norm": 2.2255720250423106, + "language_loss": 0.96757275, + "learning_rate": 3.8244873937935385e-06, + "loss": 0.98922884, + "num_input_tokens_seen": 157962280, + "router_z_loss_clip": 0.63476562, + "router_z_loss_mlp": 0.17492676, + "step": 5525, + "time_per_iteration": 2.5079967975616455 + }, + { + "auxiliary_loss_clip": 0.0110333, + "auxiliary_loss_mlp": 0.01046175, + "balance_loss_clip": 1.04275715, + "balance_loss_mlp": 1.02782249, + "epoch": 0.16035053101967384, + "flos": 34706843568000.0, + "grad_norm": 2.198113952395104, + "language_loss": 0.8323577, + "learning_rate": 3.8244103875929144e-06, + "loss": 0.85385275, + "num_input_tokens_seen": 157980460, + "router_z_loss_clip": 0.60595703, + "router_z_loss_mlp": 0.18322754, + "step": 5526, + "time_per_iteration": 2.5315613746643066 + }, + { + "auxiliary_loss_clip": 0.01104134, + "auxiliary_loss_mlp": 0.01049153, + "balance_loss_clip": 1.04302299, + "balance_loss_mlp": 1.03236818, + "epoch": 0.1603795484881899, + "flos": 21973844701440.0, + "grad_norm": 2.001559718835555, + "language_loss": 0.61603361, + "learning_rate": 3.82433336527836e-06, + "loss": 0.63756645, + "num_input_tokens_seen": 158000630, + "router_z_loss_clip": 0.61083984, + "router_z_loss_mlp": 0.16784668, + "step": 5527, + "time_per_iteration": 2.730118989944458 + }, + { + "auxiliary_loss_clip": 0.01105742, + "auxiliary_loss_mlp": 0.01044284, + "balance_loss_clip": 1.04241395, + "balance_loss_mlp": 1.02483475, + "epoch": 0.16040856595670594, + "flos": 26862802341120.0, + "grad_norm": 2.578414225003016, + "language_loss": 1.0202204, + "learning_rate": 3.824256326850555e-06, + "loss": 1.04172063, + "num_input_tokens_seen": 158015625, + "router_z_loss_clip": 0.63330078, + "router_z_loss_mlp": 0.19451904, + "step": 5528, + "time_per_iteration": 2.4471113681793213 + }, + { + "auxiliary_loss_clip": 0.01093818, + "auxiliary_loss_mlp": 0.01039633, + "balance_loss_clip": 1.04178441, + "balance_loss_mlp": 1.02517319, + "epoch": 0.160437583425222, + "flos": 20405336693760.0, + "grad_norm": 2.2757658433023353, + "language_loss": 0.804169, + "learning_rate": 3.824179272310181e-06, + "loss": 0.82550347, + "num_input_tokens_seen": 158028275, + "router_z_loss_clip": 0.52050781, + "router_z_loss_mlp": 0.14453125, + "step": 5529, + "time_per_iteration": 2.6432619094848633 + }, + { + "auxiliary_loss_clip": 0.0104427, + "auxiliary_loss_mlp": 0.01003845, + "balance_loss_clip": 1.03004122, + "balance_loss_mlp": 1.00260568, + "epoch": 0.16046660089373804, + "flos": 58057032954240.0, + "grad_norm": 0.6707000884326592, + "language_loss": 0.46876121, + "learning_rate": 3.824102201657916e-06, + "loss": 0.48924232, + "num_input_tokens_seen": 158082425, + "router_z_loss_clip": 0.14257812, + "router_z_loss_mlp": 0.01239014, + "step": 5530, + "time_per_iteration": 2.8300819396972656 + }, + { + "auxiliary_loss_clip": 0.01088539, + "auxiliary_loss_mlp": 0.01047877, + "balance_loss_clip": 1.03626347, + "balance_loss_mlp": 1.0315392, + "epoch": 0.16049561836225407, + "flos": 19675442476800.0, + "grad_norm": 2.4965235480530086, + "language_loss": 0.8114537, + "learning_rate": 3.824025114894443e-06, + "loss": 0.83281785, + "num_input_tokens_seen": 158096030, + "router_z_loss_clip": 0.5222168, + "router_z_loss_mlp": 0.16339111, + "step": 5531, + "time_per_iteration": 2.44477915763855 + }, + { + "auxiliary_loss_clip": 0.01034844, + "auxiliary_loss_mlp": 0.01005965, + "balance_loss_clip": 1.02172852, + "balance_loss_mlp": 1.0046891, + "epoch": 0.16052463583077012, + "flos": 63392145258240.0, + "grad_norm": 0.7051116665011677, + "language_loss": 0.49054015, + "learning_rate": 3.823948012020441e-06, + "loss": 0.5109483, + "num_input_tokens_seen": 158155745, + "router_z_loss_clip": 0.13085938, + "router_z_loss_mlp": 0.01275635, + "step": 5532, + "time_per_iteration": 2.950605630874634 + }, + { + "auxiliary_loss_clip": 0.01083536, + "auxiliary_loss_mlp": 0.01035508, + "balance_loss_clip": 1.03126073, + "balance_loss_mlp": 1.01990902, + "epoch": 0.16055365329928617, + "flos": 32992363699200.0, + "grad_norm": 1.8163586689280748, + "language_loss": 0.70395988, + "learning_rate": 3.823870893036594e-06, + "loss": 0.72515035, + "num_input_tokens_seen": 158185620, + "router_z_loss_clip": 0.52294922, + "router_z_loss_mlp": 0.15606689, + "step": 5533, + "time_per_iteration": 2.7848525047302246 + }, + { + "auxiliary_loss_clip": 0.01092601, + "auxiliary_loss_mlp": 0.01041493, + "balance_loss_clip": 1.03389096, + "balance_loss_mlp": 1.02238357, + "epoch": 0.16058267076780222, + "flos": 14384355264000.0, + "grad_norm": 2.8360543468896693, + "language_loss": 0.91936809, + "learning_rate": 3.823793757943579e-06, + "loss": 0.94070899, + "num_input_tokens_seen": 158197945, + "router_z_loss_clip": 0.58691406, + "router_z_loss_mlp": 0.19134521, + "step": 5534, + "time_per_iteration": 2.3361666202545166 + }, + { + "auxiliary_loss_clip": 0.01095064, + "auxiliary_loss_mlp": 0.0104389, + "balance_loss_clip": 1.03523111, + "balance_loss_mlp": 1.02510881, + "epoch": 0.16061168823631827, + "flos": 18874185707520.0, + "grad_norm": 2.4136003214935844, + "language_loss": 0.91289359, + "learning_rate": 3.82371660674208e-06, + "loss": 0.93428308, + "num_input_tokens_seen": 158211695, + "router_z_loss_clip": 0.59838867, + "router_z_loss_mlp": 0.18786621, + "step": 5535, + "time_per_iteration": 2.41882586479187 + }, + { + "auxiliary_loss_clip": 0.01088083, + "auxiliary_loss_mlp": 0.01042371, + "balance_loss_clip": 1.03154325, + "balance_loss_mlp": 1.02545536, + "epoch": 0.1606407057048343, + "flos": 34413154277760.0, + "grad_norm": 2.669169254385593, + "language_loss": 0.90881467, + "learning_rate": 3.8236394394327785e-06, + "loss": 0.93011916, + "num_input_tokens_seen": 158227980, + "router_z_loss_clip": 0.56542969, + "router_z_loss_mlp": 0.16900635, + "step": 5536, + "time_per_iteration": 2.4834678173065186 + }, + { + "auxiliary_loss_clip": 0.01098249, + "auxiliary_loss_mlp": 0.01050907, + "balance_loss_clip": 1.03545547, + "balance_loss_mlp": 1.03084421, + "epoch": 0.16066972317335035, + "flos": 16536645982080.0, + "grad_norm": 2.149681375836493, + "language_loss": 0.78345674, + "learning_rate": 3.823562256016357e-06, + "loss": 0.80494827, + "num_input_tokens_seen": 158241780, + "router_z_loss_clip": 0.62841797, + "router_z_loss_mlp": 0.20056152, + "step": 5537, + "time_per_iteration": 2.3924505710601807 + }, + { + "auxiliary_loss_clip": 0.01024694, + "auxiliary_loss_mlp": 0.01012693, + "balance_loss_clip": 1.01553893, + "balance_loss_mlp": 1.01144159, + "epoch": 0.1606987406418664, + "flos": 70510935479040.0, + "grad_norm": 0.6493813031166082, + "language_loss": 0.48643231, + "learning_rate": 3.823485056493493e-06, + "loss": 0.50680614, + "num_input_tokens_seen": 158307870, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01251221, + "step": 5538, + "time_per_iteration": 3.0545105934143066 + }, + { + "auxiliary_loss_clip": 0.01094026, + "auxiliary_loss_mlp": 0.01036437, + "balance_loss_clip": 1.03812933, + "balance_loss_mlp": 1.01946187, + "epoch": 0.16072775811038245, + "flos": 16575469280640.0, + "grad_norm": 2.8488736850578014, + "language_loss": 0.94162989, + "learning_rate": 3.823407840864873e-06, + "loss": 0.96293455, + "num_input_tokens_seen": 158323055, + "router_z_loss_clip": 0.55859375, + "router_z_loss_mlp": 0.1697998, + "step": 5539, + "time_per_iteration": 2.4266669750213623 + }, + { + "auxiliary_loss_clip": 0.01095778, + "auxiliary_loss_mlp": 0.01034293, + "balance_loss_clip": 1.03788424, + "balance_loss_mlp": 1.01778245, + "epoch": 0.1607567755788985, + "flos": 22266137537280.0, + "grad_norm": 2.6543074632316355, + "language_loss": 0.82898319, + "learning_rate": 3.8233306091311765e-06, + "loss": 0.85028386, + "num_input_tokens_seen": 158336765, + "router_z_loss_clip": 0.57885742, + "router_z_loss_mlp": 0.16503906, + "step": 5540, + "time_per_iteration": 2.4203977584838867 + }, + { + "auxiliary_loss_clip": 0.01088356, + "auxiliary_loss_mlp": 0.01030254, + "balance_loss_clip": 1.0358144, + "balance_loss_mlp": 1.01603222, + "epoch": 0.16078579304741455, + "flos": 18360753120000.0, + "grad_norm": 2.6144577756573706, + "language_loss": 0.92834985, + "learning_rate": 3.823253361293086e-06, + "loss": 0.94953597, + "num_input_tokens_seen": 158350190, + "router_z_loss_clip": 0.52636719, + "router_z_loss_mlp": 0.14227295, + "step": 5541, + "time_per_iteration": 2.4417152404785156 + }, + { + "auxiliary_loss_clip": 0.01107039, + "auxiliary_loss_mlp": 0.01038091, + "balance_loss_clip": 1.04375851, + "balance_loss_mlp": 1.01900578, + "epoch": 0.16081481051593058, + "flos": 16063328413440.0, + "grad_norm": 2.6380881598559975, + "language_loss": 0.89596987, + "learning_rate": 3.823176097351284e-06, + "loss": 0.91742116, + "num_input_tokens_seen": 158363585, + "router_z_loss_clip": 0.63256836, + "router_z_loss_mlp": 0.19104004, + "step": 5542, + "time_per_iteration": 2.371546745300293 + }, + { + "auxiliary_loss_clip": 0.01100921, + "auxiliary_loss_mlp": 0.01045278, + "balance_loss_clip": 1.04316473, + "balance_loss_mlp": 1.02749252, + "epoch": 0.16084382798444663, + "flos": 13873820319360.0, + "grad_norm": 3.1005592981329424, + "language_loss": 0.97765315, + "learning_rate": 3.823098817306453e-06, + "loss": 0.99911517, + "num_input_tokens_seen": 158376035, + "router_z_loss_clip": 0.57763672, + "router_z_loss_mlp": 0.17785645, + "step": 5543, + "time_per_iteration": 2.586190700531006 + }, + { + "auxiliary_loss_clip": 0.01021572, + "auxiliary_loss_mlp": 0.01006671, + "balance_loss_clip": 1.01137578, + "balance_loss_mlp": 1.00553262, + "epoch": 0.16087284545296268, + "flos": 67076878682880.0, + "grad_norm": 0.6381283168910775, + "language_loss": 0.5014894, + "learning_rate": 3.823021521159276e-06, + "loss": 0.52177179, + "num_input_tokens_seen": 158438465, + "router_z_loss_clip": 0.10205078, + "router_z_loss_mlp": 0.01141357, + "step": 5544, + "time_per_iteration": 3.007964849472046 + }, + { + "auxiliary_loss_clip": 0.01022885, + "auxiliary_loss_mlp": 0.01003978, + "balance_loss_clip": 1.01233768, + "balance_loss_mlp": 1.00286329, + "epoch": 0.16090186292147873, + "flos": 69916295335680.0, + "grad_norm": 3.321510179334596, + "language_loss": 0.46158606, + "learning_rate": 3.822944208910435e-06, + "loss": 0.48185471, + "num_input_tokens_seen": 158496845, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01116943, + "step": 5545, + "time_per_iteration": 2.982630968093872 + }, + { + "auxiliary_loss_clip": 0.01108269, + "auxiliary_loss_mlp": 0.0103491, + "balance_loss_clip": 1.04926658, + "balance_loss_mlp": 1.01753557, + "epoch": 0.16093088038999478, + "flos": 68162889300480.0, + "grad_norm": 2.3178158212449986, + "language_loss": 0.68548727, + "learning_rate": 3.822866880560613e-06, + "loss": 0.70691907, + "num_input_tokens_seen": 158518430, + "router_z_loss_clip": 0.59008789, + "router_z_loss_mlp": 0.17376709, + "step": 5546, + "time_per_iteration": 2.8523218631744385 + }, + { + "auxiliary_loss_clip": 0.01097774, + "auxiliary_loss_mlp": 0.01029068, + "balance_loss_clip": 1.04626822, + "balance_loss_mlp": 1.01394081, + "epoch": 0.16095989785851084, + "flos": 16428834103680.0, + "grad_norm": 2.0064767890569537, + "language_loss": 0.71262586, + "learning_rate": 3.822789536110493e-06, + "loss": 0.73389435, + "num_input_tokens_seen": 158532480, + "router_z_loss_clip": 0.5144043, + "router_z_loss_mlp": 0.15124512, + "step": 5547, + "time_per_iteration": 2.386564016342163 + }, + { + "auxiliary_loss_clip": 0.01027074, + "auxiliary_loss_mlp": 0.01001736, + "balance_loss_clip": 1.0157721, + "balance_loss_mlp": 1.0005976, + "epoch": 0.16098891532702686, + "flos": 61390050675840.0, + "grad_norm": 0.6570792232086291, + "language_loss": 0.50050819, + "learning_rate": 3.822712175560759e-06, + "loss": 0.52079624, + "num_input_tokens_seen": 158597220, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.01141357, + "step": 5548, + "time_per_iteration": 3.042262315750122 + }, + { + "auxiliary_loss_clip": 0.01107119, + "auxiliary_loss_mlp": 0.01044573, + "balance_loss_clip": 1.04403615, + "balance_loss_mlp": 1.02636433, + "epoch": 0.1610179327955429, + "flos": 37952404600320.0, + "grad_norm": 2.6380508725772653, + "language_loss": 0.82349527, + "learning_rate": 3.8226347989120926e-06, + "loss": 0.84501219, + "num_input_tokens_seen": 158613170, + "router_z_loss_clip": 0.63037109, + "router_z_loss_mlp": 0.18206787, + "step": 5549, + "time_per_iteration": 2.5297272205352783 + }, + { + "auxiliary_loss_clip": 0.01103533, + "auxiliary_loss_mlp": 0.01045284, + "balance_loss_clip": 1.04702079, + "balance_loss_mlp": 1.02906537, + "epoch": 0.16104695026405896, + "flos": 29232497295360.0, + "grad_norm": 3.799844184772974, + "language_loss": 0.85260516, + "learning_rate": 3.822557406165178e-06, + "loss": 0.87409329, + "num_input_tokens_seen": 158627810, + "router_z_loss_clip": 0.56591797, + "router_z_loss_mlp": 0.16210938, + "step": 5550, + "time_per_iteration": 2.511244773864746 + }, + { + "auxiliary_loss_clip": 0.01104672, + "auxiliary_loss_mlp": 0.01044129, + "balance_loss_clip": 1.04422998, + "balance_loss_mlp": 1.02473402, + "epoch": 0.16107596773257501, + "flos": 35400707395200.0, + "grad_norm": 2.756822774798387, + "language_loss": 0.89436054, + "learning_rate": 3.822479997320699e-06, + "loss": 0.91584861, + "num_input_tokens_seen": 158641495, + "router_z_loss_clip": 0.60400391, + "router_z_loss_mlp": 0.19396973, + "step": 5551, + "time_per_iteration": 2.454808473587036 + }, + { + "auxiliary_loss_clip": 0.01101753, + "auxiliary_loss_mlp": 0.01042444, + "balance_loss_clip": 1.04166567, + "balance_loss_mlp": 1.02449703, + "epoch": 0.16110498520109107, + "flos": 24674690701440.0, + "grad_norm": 2.5091407840195745, + "language_loss": 0.9246155, + "learning_rate": 3.82240257237934e-06, + "loss": 0.9460575, + "num_input_tokens_seen": 158657980, + "router_z_loss_clip": 0.60205078, + "router_z_loss_mlp": 0.17950439, + "step": 5552, + "time_per_iteration": 4.840656280517578 + }, + { + "auxiliary_loss_clip": 0.01099739, + "auxiliary_loss_mlp": 0.01046056, + "balance_loss_clip": 1.04012513, + "balance_loss_mlp": 1.02767944, + "epoch": 0.1611340026696071, + "flos": 11941272898560.0, + "grad_norm": 2.6820187435055813, + "language_loss": 0.84048128, + "learning_rate": 3.8223251313417825e-06, + "loss": 0.86193925, + "num_input_tokens_seen": 158670465, + "router_z_loss_clip": 0.59570312, + "router_z_loss_mlp": 0.18377686, + "step": 5553, + "time_per_iteration": 2.488130569458008 + }, + { + "auxiliary_loss_clip": 0.0109995, + "auxiliary_loss_mlp": 0.01044467, + "balance_loss_clip": 1.04229736, + "balance_loss_mlp": 1.02768242, + "epoch": 0.16116302013812314, + "flos": 10517759233920.0, + "grad_norm": 2.039231679762128, + "language_loss": 0.71554673, + "learning_rate": 3.8222476742087135e-06, + "loss": 0.73699093, + "num_input_tokens_seen": 158682600, + "router_z_loss_clip": 0.57617188, + "router_z_loss_mlp": 0.16784668, + "step": 5554, + "time_per_iteration": 2.41546368598938 + }, + { + "auxiliary_loss_clip": 0.01025595, + "auxiliary_loss_mlp": 0.0100504, + "balance_loss_clip": 1.0147438, + "balance_loss_mlp": 1.00392556, + "epoch": 0.1611920376066392, + "flos": 62517361432320.0, + "grad_norm": 0.6550043691801445, + "language_loss": 0.50685441, + "learning_rate": 3.822170200980815e-06, + "loss": 0.52716076, + "num_input_tokens_seen": 158740720, + "router_z_loss_clip": 0.10839844, + "router_z_loss_mlp": 0.01116943, + "step": 5555, + "time_per_iteration": 5.294307231903076 + }, + { + "auxiliary_loss_clip": 0.01090225, + "auxiliary_loss_mlp": 0.01034006, + "balance_loss_clip": 1.03805542, + "balance_loss_mlp": 1.01843691, + "epoch": 0.16122105507515525, + "flos": 15261094126080.0, + "grad_norm": 2.2138366751770606, + "language_loss": 0.72151947, + "learning_rate": 3.822092711658772e-06, + "loss": 0.74276173, + "num_input_tokens_seen": 158753115, + "router_z_loss_clip": 0.52099609, + "router_z_loss_mlp": 0.15557861, + "step": 5556, + "time_per_iteration": 2.4049768447875977 + }, + { + "auxiliary_loss_clip": 0.01021561, + "auxiliary_loss_mlp": 0.01001868, + "balance_loss_clip": 1.01127028, + "balance_loss_mlp": 1.00086629, + "epoch": 0.1612500725436713, + "flos": 74771980583040.0, + "grad_norm": 0.6494016184581406, + "language_loss": 0.45529377, + "learning_rate": 3.822015206243269e-06, + "loss": 0.47552806, + "num_input_tokens_seen": 158816270, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01000977, + "step": 5557, + "time_per_iteration": 3.1554222106933594 + }, + { + "auxiliary_loss_clip": 0.01093868, + "auxiliary_loss_mlp": 0.01047071, + "balance_loss_clip": 1.03482354, + "balance_loss_mlp": 1.02774751, + "epoch": 0.16127909001218735, + "flos": 28833160631040.0, + "grad_norm": 1.9586533763544385, + "language_loss": 0.77146804, + "learning_rate": 3.82193768473499e-06, + "loss": 0.79287744, + "num_input_tokens_seen": 158833175, + "router_z_loss_clip": 0.59082031, + "router_z_loss_mlp": 0.19342041, + "step": 5558, + "time_per_iteration": 2.502465009689331 + }, + { + "auxiliary_loss_clip": 0.01098051, + "auxiliary_loss_mlp": 0.01037413, + "balance_loss_clip": 1.03371799, + "balance_loss_mlp": 1.01637256, + "epoch": 0.16130810748070337, + "flos": 18360648385920.0, + "grad_norm": 2.1297614516815075, + "language_loss": 0.89676321, + "learning_rate": 3.82186014713462e-06, + "loss": 0.91811782, + "num_input_tokens_seen": 158848295, + "router_z_loss_clip": 0.64306641, + "router_z_loss_mlp": 0.21044922, + "step": 5559, + "time_per_iteration": 2.342958688735962 + }, + { + "auxiliary_loss_clip": 0.01018683, + "auxiliary_loss_mlp": 0.01000815, + "balance_loss_clip": 1.00915456, + "balance_loss_mlp": 0.9998787, + "epoch": 0.16133712494921942, + "flos": 62108109941760.0, + "grad_norm": 0.7617173889973943, + "language_loss": 0.48767751, + "learning_rate": 3.821782593442844e-06, + "loss": 0.50787246, + "num_input_tokens_seen": 158905200, + "router_z_loss_clip": 0.09521484, + "router_z_loss_mlp": 0.00933838, + "step": 5560, + "time_per_iteration": 2.960719347000122 + }, + { + "auxiliary_loss_clip": 0.01079992, + "auxiliary_loss_mlp": 0.01035768, + "balance_loss_clip": 1.02763522, + "balance_loss_mlp": 1.02131438, + "epoch": 0.16136614241773548, + "flos": 24746646746880.0, + "grad_norm": 2.089923914419767, + "language_loss": 0.83223325, + "learning_rate": 3.821705023660348e-06, + "loss": 0.85339093, + "num_input_tokens_seen": 158919015, + "router_z_loss_clip": 0.52416992, + "router_z_loss_mlp": 0.14447021, + "step": 5561, + "time_per_iteration": 2.432196617126465 + }, + { + "auxiliary_loss_clip": 0.01099242, + "auxiliary_loss_mlp": 0.01046391, + "balance_loss_clip": 1.03561878, + "balance_loss_mlp": 1.02731156, + "epoch": 0.16139515988625153, + "flos": 28691029019520.0, + "grad_norm": 2.468199410140819, + "language_loss": 0.85585713, + "learning_rate": 3.8216274377878155e-06, + "loss": 0.87731349, + "num_input_tokens_seen": 158936830, + "router_z_loss_clip": 0.63574219, + "router_z_loss_mlp": 0.190979, + "step": 5562, + "time_per_iteration": 2.4549825191497803 + }, + { + "auxiliary_loss_clip": 0.01016545, + "auxiliary_loss_mlp": 0.01002799, + "balance_loss_clip": 1.00765193, + "balance_loss_mlp": 1.00159466, + "epoch": 0.16142417735476758, + "flos": 61540875216000.0, + "grad_norm": 0.6540470727441822, + "language_loss": 0.47398061, + "learning_rate": 3.821549835825932e-06, + "loss": 0.49417406, + "num_input_tokens_seen": 159003245, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.01202393, + "step": 5563, + "time_per_iteration": 5.5570642948150635 + }, + { + "auxiliary_loss_clip": 0.0101633, + "auxiliary_loss_mlp": 0.01003506, + "balance_loss_clip": 1.00752473, + "balance_loss_mlp": 1.00247526, + "epoch": 0.16145319482328363, + "flos": 63354858059520.0, + "grad_norm": 0.6119019540795724, + "language_loss": 0.48070642, + "learning_rate": 3.821472217775383e-06, + "loss": 0.5009048, + "num_input_tokens_seen": 159066310, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01031494, + "step": 5564, + "time_per_iteration": 5.393697738647461 + }, + { + "auxiliary_loss_clip": 0.01082337, + "auxiliary_loss_mlp": 0.01032548, + "balance_loss_clip": 1.02779973, + "balance_loss_mlp": 1.01695514, + "epoch": 0.16148221229179965, + "flos": 21097978623360.0, + "grad_norm": 2.4479202782153693, + "language_loss": 0.76380289, + "learning_rate": 3.821394583636855e-06, + "loss": 0.78495181, + "num_input_tokens_seen": 159081575, + "router_z_loss_clip": 0.5456543, + "router_z_loss_mlp": 0.15600586, + "step": 5565, + "time_per_iteration": 2.5832743644714355 + }, + { + "auxiliary_loss_clip": 0.01084796, + "auxiliary_loss_mlp": 0.01032427, + "balance_loss_clip": 1.03052175, + "balance_loss_mlp": 1.01619625, + "epoch": 0.1615112297603157, + "flos": 37734546516480.0, + "grad_norm": 3.322529422596984, + "language_loss": 0.88845503, + "learning_rate": 3.8213169334110325e-06, + "loss": 0.90962726, + "num_input_tokens_seen": 159099525, + "router_z_loss_clip": 0.54321289, + "router_z_loss_mlp": 0.16229248, + "step": 5566, + "time_per_iteration": 2.536196231842041 + }, + { + "auxiliary_loss_clip": 0.01092445, + "auxiliary_loss_mlp": 0.01039903, + "balance_loss_clip": 1.03045237, + "balance_loss_mlp": 1.02031672, + "epoch": 0.16154024722883176, + "flos": 32895932924160.0, + "grad_norm": 1.876039968392616, + "language_loss": 0.77228236, + "learning_rate": 3.821239267098602e-06, + "loss": 0.7936058, + "num_input_tokens_seen": 159118605, + "router_z_loss_clip": 0.61962891, + "router_z_loss_mlp": 0.19604492, + "step": 5567, + "time_per_iteration": 2.524954319000244 + }, + { + "auxiliary_loss_clip": 0.0108792, + "auxiliary_loss_mlp": 0.01031946, + "balance_loss_clip": 1.03177977, + "balance_loss_mlp": 1.01598442, + "epoch": 0.1615692646973478, + "flos": 31351620355200.0, + "grad_norm": 1.8082265522659153, + "language_loss": 0.76931489, + "learning_rate": 3.821161584700249e-06, + "loss": 0.79051358, + "num_input_tokens_seen": 159135420, + "router_z_loss_clip": 0.5612793, + "router_z_loss_mlp": 0.15948486, + "step": 5568, + "time_per_iteration": 2.4942002296447754 + }, + { + "auxiliary_loss_clip": 0.01094915, + "auxiliary_loss_mlp": 0.01047965, + "balance_loss_clip": 1.03220797, + "balance_loss_mlp": 1.02833676, + "epoch": 0.16159828216586386, + "flos": 25147554422400.0, + "grad_norm": 2.258840929825066, + "language_loss": 0.8357023, + "learning_rate": 3.821083886216661e-06, + "loss": 0.85713112, + "num_input_tokens_seen": 159150650, + "router_z_loss_clip": 0.62695312, + "router_z_loss_mlp": 0.19610596, + "step": 5569, + "time_per_iteration": 2.466015338897705 + }, + { + "auxiliary_loss_clip": 0.01017517, + "auxiliary_loss_mlp": 0.0100189, + "balance_loss_clip": 1.00841331, + "balance_loss_mlp": 1.00109088, + "epoch": 0.16162729963437988, + "flos": 74776798350720.0, + "grad_norm": 0.6240947119437382, + "language_loss": 0.47251707, + "learning_rate": 3.821006171648522e-06, + "loss": 0.49271113, + "num_input_tokens_seen": 159217835, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.00799561, + "step": 5570, + "time_per_iteration": 3.0904085636138916 + }, + { + "auxiliary_loss_clip": 0.01017194, + "auxiliary_loss_mlp": 0.01001126, + "balance_loss_clip": 1.0074389, + "balance_loss_mlp": 1.00001121, + "epoch": 0.16165631710289594, + "flos": 74186975975040.0, + "grad_norm": 0.6560706794590261, + "language_loss": 0.56010783, + "learning_rate": 3.820928440996521e-06, + "loss": 0.58029103, + "num_input_tokens_seen": 159283610, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.01116943, + "step": 5571, + "time_per_iteration": 3.1353745460510254 + }, + { + "auxiliary_loss_clip": 0.0108781, + "auxiliary_loss_mlp": 0.01033264, + "balance_loss_clip": 1.03057981, + "balance_loss_mlp": 1.01661086, + "epoch": 0.161685334571412, + "flos": 16974317185920.0, + "grad_norm": 2.3353686782874123, + "language_loss": 0.67470491, + "learning_rate": 3.820850694261342e-06, + "loss": 0.69591564, + "num_input_tokens_seen": 159296375, + "router_z_loss_clip": 0.57226562, + "router_z_loss_mlp": 0.16638184, + "step": 5572, + "time_per_iteration": 2.3522160053253174 + }, + { + "auxiliary_loss_clip": 0.01016764, + "auxiliary_loss_mlp": 0.01000187, + "balance_loss_clip": 1.00700831, + "balance_loss_mlp": 0.99913198, + "epoch": 0.16171435203992804, + "flos": 67058201105280.0, + "grad_norm": 0.6641478365924595, + "language_loss": 0.4803569, + "learning_rate": 3.8207729314436725e-06, + "loss": 0.50052643, + "num_input_tokens_seen": 159352470, + "router_z_loss_clip": 0.09765625, + "router_z_loss_mlp": 0.01055908, + "step": 5573, + "time_per_iteration": 2.945974588394165 + }, + { + "auxiliary_loss_clip": 0.01081882, + "auxiliary_loss_mlp": 0.0103371, + "balance_loss_clip": 1.02994311, + "balance_loss_mlp": 1.018821, + "epoch": 0.1617433695084441, + "flos": 16537309297920.0, + "grad_norm": 2.0677789467341086, + "language_loss": 0.69937056, + "learning_rate": 3.8206951525442e-06, + "loss": 0.72052646, + "num_input_tokens_seen": 159365130, + "router_z_loss_clip": 0.51904297, + "router_z_loss_mlp": 0.14892578, + "step": 5574, + "time_per_iteration": 2.3499374389648438 + }, + { + "auxiliary_loss_clip": 0.01015931, + "auxiliary_loss_mlp": 0.01001204, + "balance_loss_clip": 1.00622308, + "balance_loss_mlp": 1.00030351, + "epoch": 0.16177238697696014, + "flos": 70790590402560.0, + "grad_norm": 0.6432336998819611, + "language_loss": 0.47417799, + "learning_rate": 3.820617357563612e-06, + "loss": 0.49434933, + "num_input_tokens_seen": 159425255, + "router_z_loss_clip": 0.09716797, + "router_z_loss_mlp": 0.00897217, + "step": 5575, + "time_per_iteration": 3.005133867263794 + }, + { + "auxiliary_loss_clip": 0.01085597, + "auxiliary_loss_mlp": 0.01036191, + "balance_loss_clip": 1.02910757, + "balance_loss_mlp": 1.01899481, + "epoch": 0.16180140444547617, + "flos": 16862944348800.0, + "grad_norm": 2.931153437789391, + "language_loss": 0.83814651, + "learning_rate": 3.820539546502594e-06, + "loss": 0.85936439, + "num_input_tokens_seen": 159439420, + "router_z_loss_clip": 0.56445312, + "router_z_loss_mlp": 0.17199707, + "step": 5576, + "time_per_iteration": 2.350292682647705 + }, + { + "auxiliary_loss_clip": 0.01091555, + "auxiliary_loss_mlp": 0.01035464, + "balance_loss_clip": 1.03146267, + "balance_loss_mlp": 1.01798177, + "epoch": 0.16183042191399222, + "flos": 23252573491200.0, + "grad_norm": 2.592800883232217, + "language_loss": 0.93574297, + "learning_rate": 3.820461719361834e-06, + "loss": 0.95701313, + "num_input_tokens_seen": 159457200, + "router_z_loss_clip": 0.60083008, + "router_z_loss_mlp": 0.17480469, + "step": 5577, + "time_per_iteration": 2.413274049758911 + }, + { + "auxiliary_loss_clip": 0.01083493, + "auxiliary_loss_mlp": 0.01037571, + "balance_loss_clip": 1.02916837, + "balance_loss_mlp": 1.02153778, + "epoch": 0.16185943938250827, + "flos": 33319150824960.0, + "grad_norm": 2.2317691206582024, + "language_loss": 0.831622, + "learning_rate": 3.82038387614202e-06, + "loss": 0.85283262, + "num_input_tokens_seen": 159474880, + "router_z_loss_clip": 0.54272461, + "router_z_loss_mlp": 0.16040039, + "step": 5578, + "time_per_iteration": 2.752631425857544 + }, + { + "auxiliary_loss_clip": 0.01083911, + "auxiliary_loss_mlp": 0.01034316, + "balance_loss_clip": 1.03013039, + "balance_loss_mlp": 1.01945043, + "epoch": 0.16188845685102432, + "flos": 26825445319680.0, + "grad_norm": 2.076244659169123, + "language_loss": 0.86799049, + "learning_rate": 3.820306016843838e-06, + "loss": 0.88917279, + "num_input_tokens_seen": 159492310, + "router_z_loss_clip": 0.53857422, + "router_z_loss_mlp": 0.14855957, + "step": 5579, + "time_per_iteration": 2.4483869075775146 + }, + { + "auxiliary_loss_clip": 0.01071335, + "auxiliary_loss_mlp": 0.01031273, + "balance_loss_clip": 1.02450156, + "balance_loss_mlp": 1.01798701, + "epoch": 0.16191747431954037, + "flos": 16391162880000.0, + "grad_norm": 2.253920911388637, + "language_loss": 0.72111845, + "learning_rate": 3.820228141467978e-06, + "loss": 0.74214458, + "num_input_tokens_seen": 159504050, + "router_z_loss_clip": 0.46875, + "router_z_loss_mlp": 0.13293457, + "step": 5580, + "time_per_iteration": 2.3340675830841064 + }, + { + "auxiliary_loss_clip": 0.01083734, + "auxiliary_loss_mlp": 0.01044595, + "balance_loss_clip": 1.02731395, + "balance_loss_mlp": 1.02655268, + "epoch": 0.16194649178805642, + "flos": 16901243976960.0, + "grad_norm": 2.487873333265049, + "language_loss": 0.75935119, + "learning_rate": 3.8201502500151255e-06, + "loss": 0.78063446, + "num_input_tokens_seen": 159516685, + "router_z_loss_clip": 0.56396484, + "router_z_loss_mlp": 0.18035889, + "step": 5581, + "time_per_iteration": 2.3662660121917725 + }, + { + "auxiliary_loss_clip": 0.01088287, + "auxiliary_loss_mlp": 0.01038394, + "balance_loss_clip": 1.02854991, + "balance_loss_mlp": 1.02081084, + "epoch": 0.16197550925657245, + "flos": 12268025112960.0, + "grad_norm": 3.2300862224929787, + "language_loss": 0.9304105, + "learning_rate": 3.82007234248597e-06, + "loss": 0.95167732, + "num_input_tokens_seen": 159527455, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.17602539, + "step": 5582, + "time_per_iteration": 2.423707962036133 + }, + { + "auxiliary_loss_clip": 0.01089379, + "auxiliary_loss_mlp": 0.01041717, + "balance_loss_clip": 1.02955198, + "balance_loss_mlp": 1.02426517, + "epoch": 0.1620045267250885, + "flos": 16902535697280.0, + "grad_norm": 2.672750460115435, + "language_loss": 0.82591099, + "learning_rate": 3.819994418881199e-06, + "loss": 0.84722197, + "num_input_tokens_seen": 159542235, + "router_z_loss_clip": 0.59814453, + "router_z_loss_mlp": 0.17468262, + "step": 5583, + "time_per_iteration": 2.377570867538452 + }, + { + "auxiliary_loss_clip": 0.01088076, + "auxiliary_loss_mlp": 0.01044563, + "balance_loss_clip": 1.02851832, + "balance_loss_mlp": 1.02545929, + "epoch": 0.16203354419360455, + "flos": 33539103590400.0, + "grad_norm": 1.9865780287284422, + "language_loss": 0.81132704, + "learning_rate": 3.8199164792015e-06, + "loss": 0.83265346, + "num_input_tokens_seen": 159561265, + "router_z_loss_clip": 0.59643555, + "router_z_loss_mlp": 0.19110107, + "step": 5584, + "time_per_iteration": 2.484574794769287 + }, + { + "auxiliary_loss_clip": 0.01087435, + "auxiliary_loss_mlp": 0.01040326, + "balance_loss_clip": 1.02947462, + "balance_loss_mlp": 1.02283847, + "epoch": 0.1620625616621206, + "flos": 29927617931520.0, + "grad_norm": 1.9937823498588731, + "language_loss": 0.82633996, + "learning_rate": 3.819838523447563e-06, + "loss": 0.84761763, + "num_input_tokens_seen": 159577985, + "router_z_loss_clip": 0.57885742, + "router_z_loss_mlp": 0.17480469, + "step": 5585, + "time_per_iteration": 2.450204372406006 + }, + { + "auxiliary_loss_clip": 0.0101297, + "auxiliary_loss_mlp": 0.0100798, + "balance_loss_clip": 1.00377607, + "balance_loss_mlp": 1.0067879, + "epoch": 0.16209157913063665, + "flos": 67583328998400.0, + "grad_norm": 0.7328246694473217, + "language_loss": 0.46336412, + "learning_rate": 3.8197605516200755e-06, + "loss": 0.48357362, + "num_input_tokens_seen": 159640755, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01190186, + "step": 5586, + "time_per_iteration": 2.9731101989746094 + }, + { + "auxiliary_loss_clip": 0.01086654, + "auxiliary_loss_mlp": 0.01034693, + "balance_loss_clip": 1.02923751, + "balance_loss_mlp": 1.01762843, + "epoch": 0.16212059659915268, + "flos": 11028329089920.0, + "grad_norm": 2.849345020413484, + "language_loss": 1.00255668, + "learning_rate": 3.819682563719727e-06, + "loss": 1.02377009, + "num_input_tokens_seen": 159651710, + "router_z_loss_clip": 0.57421875, + "router_z_loss_mlp": 0.17053223, + "step": 5587, + "time_per_iteration": 2.367818593978882 + }, + { + "auxiliary_loss_clip": 0.01089072, + "auxiliary_loss_mlp": 0.01040835, + "balance_loss_clip": 1.0292356, + "balance_loss_mlp": 1.02258468, + "epoch": 0.16214961406766873, + "flos": 34524980962560.0, + "grad_norm": 1.7281614418358486, + "language_loss": 0.77103269, + "learning_rate": 3.819604559747205e-06, + "loss": 0.7923317, + "num_input_tokens_seen": 159674125, + "router_z_loss_clip": 0.59863281, + "router_z_loss_mlp": 0.18273926, + "step": 5588, + "time_per_iteration": 2.5448079109191895 + }, + { + "auxiliary_loss_clip": 0.01085804, + "auxiliary_loss_mlp": 0.01035397, + "balance_loss_clip": 1.02921546, + "balance_loss_mlp": 1.01994729, + "epoch": 0.16217863153618478, + "flos": 16062630186240.0, + "grad_norm": 3.1833142631886555, + "language_loss": 0.85674578, + "learning_rate": 3.819526539703199e-06, + "loss": 0.87795782, + "num_input_tokens_seen": 159686700, + "router_z_loss_clip": 0.56518555, + "router_z_loss_mlp": 0.15447998, + "step": 5589, + "time_per_iteration": 2.3774733543395996 + }, + { + "auxiliary_loss_clip": 0.01086486, + "auxiliary_loss_mlp": 0.01034534, + "balance_loss_clip": 1.02854991, + "balance_loss_mlp": 1.01583576, + "epoch": 0.16220764900470083, + "flos": 33247404247680.0, + "grad_norm": 2.035186198858452, + "language_loss": 0.83061171, + "learning_rate": 3.819448503588399e-06, + "loss": 0.8518219, + "num_input_tokens_seen": 159707130, + "router_z_loss_clip": 0.57861328, + "router_z_loss_mlp": 0.18701172, + "step": 5590, + "time_per_iteration": 2.4931488037109375 + }, + { + "auxiliary_loss_clip": 0.01086452, + "auxiliary_loss_mlp": 0.01034712, + "balance_loss_clip": 1.02821755, + "balance_loss_mlp": 1.01901221, + "epoch": 0.16223666647321688, + "flos": 12892481424000.0, + "grad_norm": 1.9690324973895268, + "language_loss": 0.84886658, + "learning_rate": 3.819370451403493e-06, + "loss": 0.87007821, + "num_input_tokens_seen": 159721455, + "router_z_loss_clip": 0.58251953, + "router_z_loss_mlp": 0.15692139, + "step": 5591, + "time_per_iteration": 2.3575668334960938 + }, + { + "auxiliary_loss_clip": 0.01081545, + "auxiliary_loss_mlp": 0.01034089, + "balance_loss_clip": 1.02724218, + "balance_loss_mlp": 1.01946211, + "epoch": 0.16226568394173294, + "flos": 32555216165760.0, + "grad_norm": 2.9508224274523656, + "language_loss": 0.89886343, + "learning_rate": 3.819292383149172e-06, + "loss": 0.92001981, + "num_input_tokens_seen": 159736490, + "router_z_loss_clip": 0.54248047, + "router_z_loss_mlp": 0.14642334, + "step": 5592, + "time_per_iteration": 2.7661960124969482 + }, + { + "auxiliary_loss_clip": 0.01082897, + "auxiliary_loss_mlp": 0.01032611, + "balance_loss_clip": 1.03058958, + "balance_loss_mlp": 1.01855612, + "epoch": 0.16229470141024896, + "flos": 21319397665920.0, + "grad_norm": 2.059567286176059, + "language_loss": 0.60402232, + "learning_rate": 3.819214298826124e-06, + "loss": 0.62517738, + "num_input_tokens_seen": 159750850, + "router_z_loss_clip": 0.5234375, + "router_z_loss_mlp": 0.14056396, + "step": 5593, + "time_per_iteration": 2.4471051692962646 + }, + { + "auxiliary_loss_clip": 0.01010813, + "auxiliary_loss_mlp": 0.01001104, + "balance_loss_clip": 1.00206339, + "balance_loss_mlp": 0.99993616, + "epoch": 0.162323718878765, + "flos": 64745902293120.0, + "grad_norm": 0.6347657462539285, + "language_loss": 0.48097306, + "learning_rate": 3.8191361984350385e-06, + "loss": 0.50109226, + "num_input_tokens_seen": 159816625, + "router_z_loss_clip": 0.08740234, + "router_z_loss_mlp": 0.01165771, + "step": 5594, + "time_per_iteration": 3.05560040473938 + }, + { + "auxiliary_loss_clip": 0.01075782, + "auxiliary_loss_mlp": 0.0103383, + "balance_loss_clip": 1.02602243, + "balance_loss_mlp": 1.02031755, + "epoch": 0.16235273634728106, + "flos": 16210976019840.0, + "grad_norm": 2.1778414790920113, + "language_loss": 0.81971085, + "learning_rate": 3.819058081976606e-06, + "loss": 0.8408069, + "num_input_tokens_seen": 159834430, + "router_z_loss_clip": 0.49804688, + "router_z_loss_mlp": 0.13513184, + "step": 5595, + "time_per_iteration": 2.3693737983703613 + }, + { + "auxiliary_loss_clip": 0.01011188, + "auxiliary_loss_mlp": 0.01000853, + "balance_loss_clip": 1.00227356, + "balance_loss_mlp": 0.99979806, + "epoch": 0.16238175381579711, + "flos": 74770269926400.0, + "grad_norm": 0.7025090630780952, + "language_loss": 0.4884181, + "learning_rate": 3.818979949451517e-06, + "loss": 0.50853848, + "num_input_tokens_seen": 159891985, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.01055908, + "step": 5596, + "time_per_iteration": 3.001894235610962 + }, + { + "auxiliary_loss_clip": 0.01091989, + "auxiliary_loss_mlp": 0.01042844, + "balance_loss_clip": 1.0308708, + "balance_loss_mlp": 1.02221525, + "epoch": 0.16241077128431317, + "flos": 17704141580160.0, + "grad_norm": 2.634026585242982, + "language_loss": 0.9034102, + "learning_rate": 3.818901800860461e-06, + "loss": 0.92475855, + "num_input_tokens_seen": 159903915, + "router_z_loss_clip": 0.61181641, + "router_z_loss_mlp": 0.2064209, + "step": 5597, + "time_per_iteration": 2.357567071914673 + }, + { + "auxiliary_loss_clip": 0.01094355, + "auxiliary_loss_mlp": 0.01032532, + "balance_loss_clip": 1.03329039, + "balance_loss_mlp": 1.01473403, + "epoch": 0.1624397887528292, + "flos": 31313844397440.0, + "grad_norm": 5.760630105340001, + "language_loss": 0.88327122, + "learning_rate": 3.818823636204128e-06, + "loss": 0.90454006, + "num_input_tokens_seen": 159919525, + "router_z_loss_clip": 0.61132812, + "router_z_loss_mlp": 0.17803955, + "step": 5598, + "time_per_iteration": 2.466320753097534 + }, + { + "auxiliary_loss_clip": 0.01093139, + "auxiliary_loss_mlp": 0.01043935, + "balance_loss_clip": 1.03221965, + "balance_loss_mlp": 1.0272634, + "epoch": 0.16246880622134524, + "flos": 16901802558720.0, + "grad_norm": 2.4342173906426043, + "language_loss": 0.71208167, + "learning_rate": 3.818745455483209e-06, + "loss": 0.73345238, + "num_input_tokens_seen": 159934550, + "router_z_loss_clip": 0.60888672, + "router_z_loss_mlp": 0.16668701, + "step": 5599, + "time_per_iteration": 2.365739345550537 + }, + { + "auxiliary_loss_clip": 0.01087929, + "auxiliary_loss_mlp": 0.01031586, + "balance_loss_clip": 1.0285238, + "balance_loss_mlp": 1.01276255, + "epoch": 0.1624978236898613, + "flos": 18215723865600.0, + "grad_norm": 2.469759396720381, + "language_loss": 0.98194307, + "learning_rate": 3.818667258698394e-06, + "loss": 1.00313818, + "num_input_tokens_seen": 159947900, + "router_z_loss_clip": 0.59448242, + "router_z_loss_mlp": 0.18823242, + "step": 5600, + "time_per_iteration": 2.4804165363311768 + }, + { + "auxiliary_loss_clip": 0.01011661, + "auxiliary_loss_mlp": 0.01002597, + "balance_loss_clip": 1.00266576, + "balance_loss_mlp": 1.00156605, + "epoch": 0.16252684115837734, + "flos": 74787201936000.0, + "grad_norm": 0.6017774771662467, + "language_loss": 0.45969975, + "learning_rate": 3.818589045850373e-06, + "loss": 0.47984231, + "num_input_tokens_seen": 160015275, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01031494, + "step": 5601, + "time_per_iteration": 3.1920642852783203 + }, + { + "auxiliary_loss_clip": 0.01083515, + "auxiliary_loss_mlp": 0.0103525, + "balance_loss_clip": 1.02688122, + "balance_loss_mlp": 1.01850688, + "epoch": 0.1625558586268934, + "flos": 21939175854720.0, + "grad_norm": 1.7787738209468678, + "language_loss": 0.81604922, + "learning_rate": 3.818510816939839e-06, + "loss": 0.83723688, + "num_input_tokens_seen": 160035810, + "router_z_loss_clip": 0.56616211, + "router_z_loss_mlp": 0.16741943, + "step": 5602, + "time_per_iteration": 2.588176727294922 + }, + { + "auxiliary_loss_clip": 0.01084052, + "auxiliary_loss_mlp": 0.01037316, + "balance_loss_clip": 1.03118086, + "balance_loss_mlp": 1.02041221, + "epoch": 0.16258487609540945, + "flos": 29852764243200.0, + "grad_norm": 2.8145487909768234, + "language_loss": 0.74824578, + "learning_rate": 3.8184325719674804e-06, + "loss": 0.76945949, + "num_input_tokens_seen": 160048860, + "router_z_loss_clip": 0.52929688, + "router_z_loss_mlp": 0.16900635, + "step": 5603, + "time_per_iteration": 2.429750680923462 + }, + { + "auxiliary_loss_clip": 0.01086437, + "auxiliary_loss_mlp": 0.01035994, + "balance_loss_clip": 1.02988386, + "balance_loss_mlp": 1.01885748, + "epoch": 0.16261389356392547, + "flos": 9163722908160.0, + "grad_norm": 3.0866365019761433, + "language_loss": 0.78135496, + "learning_rate": 3.81835431093399e-06, + "loss": 0.80257928, + "num_input_tokens_seen": 160057555, + "router_z_loss_clip": 0.56542969, + "router_z_loss_mlp": 0.17126465, + "step": 5604, + "time_per_iteration": 2.3617103099823 + }, + { + "auxiliary_loss_clip": 0.0101216, + "auxiliary_loss_mlp": 0.01004141, + "balance_loss_clip": 1.00296474, + "balance_loss_mlp": 1.00296712, + "epoch": 0.16264291103244152, + "flos": 61821263278080.0, + "grad_norm": 0.6588130443726595, + "language_loss": 0.47911263, + "learning_rate": 3.818276033840059e-06, + "loss": 0.49927562, + "num_input_tokens_seen": 160116530, + "router_z_loss_clip": 0.09179688, + "router_z_loss_mlp": 0.01171875, + "step": 5605, + "time_per_iteration": 3.1263587474823 + }, + { + "auxiliary_loss_clip": 0.01010934, + "auxiliary_loss_mlp": 0.01003886, + "balance_loss_clip": 1.00191569, + "balance_loss_mlp": 1.00277162, + "epoch": 0.16267192850095757, + "flos": 67603821966720.0, + "grad_norm": 0.6388701782298999, + "language_loss": 0.45220894, + "learning_rate": 3.818197740686378e-06, + "loss": 0.47235715, + "num_input_tokens_seen": 160179475, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01116943, + "step": 5606, + "time_per_iteration": 3.265291929244995 + }, + { + "auxiliary_loss_clip": 0.01081042, + "auxiliary_loss_mlp": 0.01033611, + "balance_loss_clip": 1.0283761, + "balance_loss_mlp": 1.01744652, + "epoch": 0.16270094596947363, + "flos": 11828433784320.0, + "grad_norm": 3.32259053686091, + "language_loss": 0.78691864, + "learning_rate": 3.818119431473639e-06, + "loss": 0.80806512, + "num_input_tokens_seen": 160190225, + "router_z_loss_clip": 0.52685547, + "router_z_loss_mlp": 0.16174316, + "step": 5607, + "time_per_iteration": 2.3200714588165283 + }, + { + "auxiliary_loss_clip": 0.01010438, + "auxiliary_loss_mlp": 0.01001835, + "balance_loss_clip": 1.00155926, + "balance_loss_mlp": 1.00062513, + "epoch": 0.16272996343798968, + "flos": 60319998282240.0, + "grad_norm": 0.669231731010803, + "language_loss": 0.48534584, + "learning_rate": 3.818041106202533e-06, + "loss": 0.50546849, + "num_input_tokens_seen": 160247810, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.01208496, + "step": 5608, + "time_per_iteration": 2.8512187004089355 + }, + { + "auxiliary_loss_clip": 0.01086496, + "auxiliary_loss_mlp": 0.01032977, + "balance_loss_clip": 1.02691841, + "balance_loss_mlp": 1.01499999, + "epoch": 0.16275898090650573, + "flos": 16100790168960.0, + "grad_norm": 2.986714929172062, + "language_loss": 0.81038857, + "learning_rate": 3.817962764873752e-06, + "loss": 0.83158338, + "num_input_tokens_seen": 160261100, + "router_z_loss_clip": 0.59570312, + "router_z_loss_mlp": 0.17993164, + "step": 5609, + "time_per_iteration": 2.328927993774414 + }, + { + "auxiliary_loss_clip": 0.01084833, + "auxiliary_loss_mlp": 0.01043125, + "balance_loss_clip": 1.02690172, + "balance_loss_mlp": 1.02519631, + "epoch": 0.16278799837502175, + "flos": 25768345040640.0, + "grad_norm": 1.8841030733184712, + "language_loss": 0.94813639, + "learning_rate": 3.8178844074879894e-06, + "loss": 0.96941602, + "num_input_tokens_seen": 160277445, + "router_z_loss_clip": 0.57958984, + "router_z_loss_mlp": 0.17907715, + "step": 5610, + "time_per_iteration": 2.4284558296203613 + }, + { + "auxiliary_loss_clip": 0.01010149, + "auxiliary_loss_mlp": 0.01003249, + "balance_loss_clip": 1.00151324, + "balance_loss_mlp": 1.00215864, + "epoch": 0.1628170158435378, + "flos": 65937244216320.0, + "grad_norm": 0.6569845668677693, + "language_loss": 0.47116649, + "learning_rate": 3.817806034045935e-06, + "loss": 0.49130046, + "num_input_tokens_seen": 160339845, + "router_z_loss_clip": 0.08642578, + "router_z_loss_mlp": 0.01092529, + "step": 5611, + "time_per_iteration": 3.1051769256591797 + }, + { + "auxiliary_loss_clip": 0.01010519, + "auxiliary_loss_mlp": 0.01002892, + "balance_loss_clip": 1.0018791, + "balance_loss_mlp": 1.00180721, + "epoch": 0.16284603331205386, + "flos": 74065129729920.0, + "grad_norm": 0.7059640390901345, + "language_loss": 0.45802307, + "learning_rate": 3.8177276445482825e-06, + "loss": 0.47815716, + "num_input_tokens_seen": 160396030, + "router_z_loss_clip": 0.0859375, + "router_z_loss_mlp": 0.01086426, + "step": 5612, + "time_per_iteration": 2.887218713760376 + }, + { + "auxiliary_loss_clip": 0.01095078, + "auxiliary_loss_mlp": 0.01037769, + "balance_loss_clip": 1.03007412, + "balance_loss_mlp": 1.01830769, + "epoch": 0.1628750507805699, + "flos": 26311663618560.0, + "grad_norm": 4.637568040034162, + "language_loss": 0.81888705, + "learning_rate": 3.817649238995723e-06, + "loss": 0.84021556, + "num_input_tokens_seen": 160407955, + "router_z_loss_clip": 0.65014648, + "router_z_loss_mlp": 0.19476318, + "step": 5613, + "time_per_iteration": 2.451162815093994 + }, + { + "auxiliary_loss_clip": 0.01009933, + "auxiliary_loss_mlp": 0.01004535, + "balance_loss_clip": 1.00136566, + "balance_loss_mlp": 1.00343192, + "epoch": 0.16290406824908596, + "flos": 74767791219840.0, + "grad_norm": 0.6688696609480977, + "language_loss": 0.48212722, + "learning_rate": 3.817570817388952e-06, + "loss": 0.50227189, + "num_input_tokens_seen": 160467250, + "router_z_loss_clip": 0.08544922, + "router_z_loss_mlp": 0.01104736, + "step": 5614, + "time_per_iteration": 3.012791156768799 + }, + { + "auxiliary_loss_clip": 0.01083957, + "auxiliary_loss_mlp": 0.01033979, + "balance_loss_clip": 1.02965593, + "balance_loss_mlp": 1.01746833, + "epoch": 0.16293308571760198, + "flos": 12705417025920.0, + "grad_norm": 2.2194760264777216, + "language_loss": 0.81514859, + "learning_rate": 3.817492379728657e-06, + "loss": 0.83632791, + "num_input_tokens_seen": 160479340, + "router_z_loss_clip": 0.54296875, + "router_z_loss_mlp": 0.16516113, + "step": 5615, + "time_per_iteration": 2.433396100997925 + }, + { + "auxiliary_loss_clip": 0.01093661, + "auxiliary_loss_mlp": 0.01043501, + "balance_loss_clip": 1.02948511, + "balance_loss_mlp": 1.02294922, + "epoch": 0.16296210318611803, + "flos": 36277132055040.0, + "grad_norm": 1.5853747740084918, + "language_loss": 0.92035258, + "learning_rate": 3.817413926015537e-06, + "loss": 0.94172424, + "num_input_tokens_seen": 160504510, + "router_z_loss_clip": 0.64111328, + "router_z_loss_mlp": 0.20556641, + "step": 5616, + "time_per_iteration": 2.6665456295013428 + }, + { + "auxiliary_loss_clip": 0.01010713, + "auxiliary_loss_mlp": 0.01004319, + "balance_loss_clip": 1.00221848, + "balance_loss_mlp": 1.00330591, + "epoch": 0.1629911206546341, + "flos": 65870769254400.0, + "grad_norm": 0.7253833708858406, + "language_loss": 0.4683919, + "learning_rate": 3.81733545625028e-06, + "loss": 0.48854223, + "num_input_tokens_seen": 160561035, + "router_z_loss_clip": 0.08496094, + "router_z_loss_mlp": 0.01013184, + "step": 5617, + "time_per_iteration": 3.0247347354888916 + }, + { + "auxiliary_loss_clip": 0.01011758, + "auxiliary_loss_mlp": 0.01002029, + "balance_loss_clip": 1.00321102, + "balance_loss_mlp": 1.00100386, + "epoch": 0.16302013812315014, + "flos": 57187241452800.0, + "grad_norm": 0.6438006535587724, + "language_loss": 0.43875474, + "learning_rate": 3.817256970433581e-06, + "loss": 0.45889261, + "num_input_tokens_seen": 160621305, + "router_z_loss_clip": 0.08544922, + "router_z_loss_mlp": 0.01025391, + "step": 5618, + "time_per_iteration": 3.0852086544036865 + }, + { + "auxiliary_loss_clip": 0.01011578, + "auxiliary_loss_mlp": 0.01001223, + "balance_loss_clip": 1.00301409, + "balance_loss_mlp": 1.00022149, + "epoch": 0.1630491555916662, + "flos": 74776100123520.0, + "grad_norm": 0.6417957582873102, + "language_loss": 0.47281057, + "learning_rate": 3.817178468566134e-06, + "loss": 0.49293858, + "num_input_tokens_seen": 160686515, + "router_z_loss_clip": 0.08544922, + "router_z_loss_mlp": 0.01000977, + "step": 5619, + "time_per_iteration": 3.1210272312164307 + }, + { + "auxiliary_loss_clip": 0.01012331, + "auxiliary_loss_mlp": 0.01001197, + "balance_loss_clip": 1.00369883, + "balance_loss_mlp": 1.0001961, + "epoch": 0.16307817306018224, + "flos": 73454847431040.0, + "grad_norm": 0.9530521226493887, + "language_loss": 0.5022397, + "learning_rate": 3.81709995064863e-06, + "loss": 0.52237499, + "num_input_tokens_seen": 160745365, + "router_z_loss_clip": 0.08642578, + "router_z_loss_mlp": 0.01000977, + "step": 5620, + "time_per_iteration": 3.0253398418426514 + }, + { + "auxiliary_loss_clip": 0.01068667, + "auxiliary_loss_mlp": 0.01024347, + "balance_loss_clip": 1.02346468, + "balance_loss_mlp": 1.01225924, + "epoch": 0.16310719052869826, + "flos": 23542562177280.0, + "grad_norm": 6.197095510741992, + "language_loss": 0.79435289, + "learning_rate": 3.817021416681765e-06, + "loss": 0.815283, + "num_input_tokens_seen": 160759550, + "router_z_loss_clip": 0.4519043, + "router_z_loss_mlp": 0.12078857, + "step": 5621, + "time_per_iteration": 2.45296573638916 + }, + { + "auxiliary_loss_clip": 0.01095634, + "auxiliary_loss_mlp": 0.01046113, + "balance_loss_clip": 1.02967095, + "balance_loss_mlp": 1.02478695, + "epoch": 0.16313620799721432, + "flos": 48425126313600.0, + "grad_norm": 2.5219622689008605, + "language_loss": 0.89751071, + "learning_rate": 3.816942866666231e-06, + "loss": 0.91892821, + "num_input_tokens_seen": 160779310, + "router_z_loss_clip": 0.65869141, + "router_z_loss_mlp": 0.21313477, + "step": 5622, + "time_per_iteration": 2.6127922534942627 + }, + { + "auxiliary_loss_clip": 0.010835, + "auxiliary_loss_mlp": 0.010388, + "balance_loss_clip": 1.0280261, + "balance_loss_mlp": 1.02172887, + "epoch": 0.16316522546573037, + "flos": 21789747768960.0, + "grad_norm": 2.688719973961756, + "language_loss": 0.68139195, + "learning_rate": 3.816864300602723e-06, + "loss": 0.70261502, + "num_input_tokens_seen": 160792455, + "router_z_loss_clip": 0.55541992, + "router_z_loss_mlp": 0.1706543, + "step": 5623, + "time_per_iteration": 2.31734299659729 + }, + { + "auxiliary_loss_clip": 0.01013551, + "auxiliary_loss_mlp": 0.01002045, + "balance_loss_clip": 1.00503874, + "balance_loss_mlp": 1.0010494, + "epoch": 0.16319424293424642, + "flos": 63610911037440.0, + "grad_norm": 0.621499685409275, + "language_loss": 0.46804821, + "learning_rate": 3.8167857184919335e-06, + "loss": 0.48820418, + "num_input_tokens_seen": 160855110, + "router_z_loss_clip": 0.08496094, + "router_z_loss_mlp": 0.00994873, + "step": 5624, + "time_per_iteration": 2.998885154724121 + }, + { + "auxiliary_loss_clip": 0.01093468, + "auxiliary_loss_mlp": 0.01041089, + "balance_loss_clip": 1.03143167, + "balance_loss_mlp": 1.0213362, + "epoch": 0.16322326040276247, + "flos": 46678491216000.0, + "grad_norm": 2.1396661356004483, + "language_loss": 0.93187213, + "learning_rate": 3.816707120334558e-06, + "loss": 0.95321774, + "num_input_tokens_seen": 160876605, + "router_z_loss_clip": 0.62109375, + "router_z_loss_mlp": 0.19787598, + "step": 5625, + "time_per_iteration": 2.5787277221679688 + }, + { + "auxiliary_loss_clip": 0.01082316, + "auxiliary_loss_mlp": 0.01033986, + "balance_loss_clip": 1.02857757, + "balance_loss_mlp": 1.01854897, + "epoch": 0.16325227787127852, + "flos": 15661582865280.0, + "grad_norm": 2.220687765700408, + "language_loss": 0.70626748, + "learning_rate": 3.81662850613129e-06, + "loss": 0.72743046, + "num_input_tokens_seen": 160888710, + "router_z_loss_clip": 0.53759766, + "router_z_loss_mlp": 0.15441895, + "step": 5626, + "time_per_iteration": 2.390105724334717 + }, + { + "auxiliary_loss_clip": 0.01011923, + "auxiliary_loss_mlp": 0.01010741, + "balance_loss_clip": 1.00342464, + "balance_loss_mlp": 1.00974584, + "epoch": 0.16328129533979455, + "flos": 70576782036480.0, + "grad_norm": 0.6518395381527224, + "language_loss": 0.4853062, + "learning_rate": 3.816549875882824e-06, + "loss": 0.50553286, + "num_input_tokens_seen": 160950890, + "router_z_loss_clip": 0.08496094, + "router_z_loss_mlp": 0.00994873, + "step": 5627, + "time_per_iteration": 2.999920129776001 + }, + { + "auxiliary_loss_clip": 0.01012288, + "auxiliary_loss_mlp": 0.01013467, + "balance_loss_clip": 1.00388384, + "balance_loss_mlp": 1.01256144, + "epoch": 0.1633103128083106, + "flos": 62505421747200.0, + "grad_norm": 0.6699626944332538, + "language_loss": 0.47748435, + "learning_rate": 3.816471229589854e-06, + "loss": 0.49774194, + "num_input_tokens_seen": 161006895, + "router_z_loss_clip": 0.08398438, + "router_z_loss_mlp": 0.0090332, + "step": 5628, + "time_per_iteration": 2.9111173152923584 + }, + { + "auxiliary_loss_clip": 0.01076707, + "auxiliary_loss_mlp": 0.01033716, + "balance_loss_clip": 1.02595544, + "balance_loss_mlp": 1.01850462, + "epoch": 0.16333933027682665, + "flos": 18143209238400.0, + "grad_norm": 2.043173559629744, + "language_loss": 0.76141858, + "learning_rate": 3.816392567253075e-06, + "loss": 0.7825228, + "num_input_tokens_seen": 161021620, + "router_z_loss_clip": 0.50732422, + "router_z_loss_mlp": 0.15216064, + "step": 5629, + "time_per_iteration": 4.55604362487793 + }, + { + "auxiliary_loss_clip": 0.0100964, + "auxiliary_loss_mlp": 0.010131, + "balance_loss_clip": 1.00156188, + "balance_loss_mlp": 1.01213455, + "epoch": 0.1633683477453427, + "flos": 63869756924160.0, + "grad_norm": 0.652210465450128, + "language_loss": 0.46114025, + "learning_rate": 3.816313888873182e-06, + "loss": 0.48136762, + "num_input_tokens_seen": 161083280, + "router_z_loss_clip": 0.08105469, + "router_z_loss_mlp": 0.00964355, + "step": 5630, + "time_per_iteration": 3.2101166248321533 + }, + { + "auxiliary_loss_clip": 0.01087134, + "auxiliary_loss_mlp": 0.01040327, + "balance_loss_clip": 1.02800822, + "balance_loss_mlp": 1.02193332, + "epoch": 0.16339736521385875, + "flos": 12124217756160.0, + "grad_norm": 2.636019105675391, + "language_loss": 0.76429927, + "learning_rate": 3.81623519445087e-06, + "loss": 0.7855739, + "num_input_tokens_seen": 161095980, + "router_z_loss_clip": 0.59106445, + "router_z_loss_mlp": 0.18377686, + "step": 5631, + "time_per_iteration": 2.3764774799346924 + }, + { + "auxiliary_loss_clip": 0.01080813, + "auxiliary_loss_mlp": 0.01032293, + "balance_loss_clip": 1.02796555, + "balance_loss_mlp": 1.01611066, + "epoch": 0.16342638268237478, + "flos": 16243096337280.0, + "grad_norm": 2.4563235075995573, + "language_loss": 0.61585099, + "learning_rate": 3.816156483986834e-06, + "loss": 0.63698208, + "num_input_tokens_seen": 161107775, + "router_z_loss_clip": 0.52758789, + "router_z_loss_mlp": 0.16168213, + "step": 5632, + "time_per_iteration": 4.650836229324341 + }, + { + "auxiliary_loss_clip": 0.01081042, + "auxiliary_loss_mlp": 0.0103308, + "balance_loss_clip": 1.02630043, + "balance_loss_mlp": 1.01597381, + "epoch": 0.16345540015089083, + "flos": 14785821521280.0, + "grad_norm": 3.0567218882371443, + "language_loss": 0.9002744, + "learning_rate": 3.816077757481768e-06, + "loss": 0.92141557, + "num_input_tokens_seen": 161120440, + "router_z_loss_clip": 0.54736328, + "router_z_loss_mlp": 0.17102051, + "step": 5633, + "time_per_iteration": 2.361279249191284 + }, + { + "auxiliary_loss_clip": 0.0108646, + "auxiliary_loss_mlp": 0.0104855, + "balance_loss_clip": 1.0289582, + "balance_loss_mlp": 1.03089547, + "epoch": 0.16348441761940688, + "flos": 31603728349440.0, + "grad_norm": 2.3335824973029413, + "language_loss": 0.93434501, + "learning_rate": 3.815999014936369e-06, + "loss": 0.95569509, + "num_input_tokens_seen": 161134405, + "router_z_loss_clip": 0.57446289, + "router_z_loss_mlp": 0.17663574, + "step": 5634, + "time_per_iteration": 2.482924461364746 + }, + { + "auxiliary_loss_clip": 0.01013075, + "auxiliary_loss_mlp": 0.01003843, + "balance_loss_clip": 1.00434113, + "balance_loss_mlp": 1.00288892, + "epoch": 0.16351343508792293, + "flos": 74778683564160.0, + "grad_norm": 0.5908087106636121, + "language_loss": 0.52159309, + "learning_rate": 3.815920256351332e-06, + "loss": 0.54176223, + "num_input_tokens_seen": 161204830, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.00952148, + "step": 5635, + "time_per_iteration": 3.1990091800689697 + }, + { + "auxiliary_loss_clip": 0.01087303, + "auxiliary_loss_mlp": 0.01033127, + "balance_loss_clip": 1.02888203, + "balance_loss_mlp": 1.01579452, + "epoch": 0.16354245255643898, + "flos": 18945932284800.0, + "grad_norm": 2.027793028001712, + "language_loss": 0.84581512, + "learning_rate": 3.815841481727352e-06, + "loss": 0.86701941, + "num_input_tokens_seen": 161218870, + "router_z_loss_clip": 0.58496094, + "router_z_loss_mlp": 0.17333984, + "step": 5636, + "time_per_iteration": 2.3718652725219727 + }, + { + "auxiliary_loss_clip": 0.0108959, + "auxiliary_loss_mlp": 0.01046861, + "balance_loss_clip": 1.03085995, + "balance_loss_mlp": 1.02865815, + "epoch": 0.16357147002495503, + "flos": 15952130133120.0, + "grad_norm": 3.106076413958342, + "language_loss": 0.93990719, + "learning_rate": 3.815762691065126e-06, + "loss": 0.96127176, + "num_input_tokens_seen": 161230220, + "router_z_loss_clip": 0.58740234, + "router_z_loss_mlp": 0.18200684, + "step": 5637, + "time_per_iteration": 2.365511178970337 + }, + { + "auxiliary_loss_clip": 0.01087176, + "auxiliary_loss_mlp": 0.01044064, + "balance_loss_clip": 1.02920961, + "balance_loss_mlp": 1.02664793, + "epoch": 0.16360048749347106, + "flos": 16540381497600.0, + "grad_norm": 3.934125613944766, + "language_loss": 0.75453651, + "learning_rate": 3.815683884365348e-06, + "loss": 0.77584887, + "num_input_tokens_seen": 161242745, + "router_z_loss_clip": 0.58007812, + "router_z_loss_mlp": 0.17407227, + "step": 5638, + "time_per_iteration": 2.3950376510620117 + }, + { + "auxiliary_loss_clip": 0.01091444, + "auxiliary_loss_mlp": 0.01049215, + "balance_loss_clip": 1.02985477, + "balance_loss_mlp": 1.03052282, + "epoch": 0.1636295049619871, + "flos": 56631496296960.0, + "grad_norm": 3.547022285353407, + "language_loss": 0.86629856, + "learning_rate": 3.815605061628716e-06, + "loss": 0.88770521, + "num_input_tokens_seen": 161260630, + "router_z_loss_clip": 0.61621094, + "router_z_loss_mlp": 0.18676758, + "step": 5639, + "time_per_iteration": 5.0790696144104 + }, + { + "auxiliary_loss_clip": 0.01015142, + "auxiliary_loss_mlp": 0.01014327, + "balance_loss_clip": 1.00629067, + "balance_loss_mlp": 1.01328385, + "epoch": 0.16365852243050316, + "flos": 74775751009920.0, + "grad_norm": 0.6155990546871621, + "language_loss": 0.44203478, + "learning_rate": 3.815526222855926e-06, + "loss": 0.46232948, + "num_input_tokens_seen": 161324025, + "router_z_loss_clip": 0.08837891, + "router_z_loss_mlp": 0.01043701, + "step": 5640, + "time_per_iteration": 5.440369606018066 + }, + { + "auxiliary_loss_clip": 0.0109391, + "auxiliary_loss_mlp": 0.01043731, + "balance_loss_clip": 1.03173923, + "balance_loss_mlp": 1.02363789, + "epoch": 0.1636875398990192, + "flos": 19607989996800.0, + "grad_norm": 2.299088997345995, + "language_loss": 1.00309968, + "learning_rate": 3.8154473680476725e-06, + "loss": 1.02447605, + "num_input_tokens_seen": 161339465, + "router_z_loss_clip": 0.62207031, + "router_z_loss_mlp": 0.20092773, + "step": 5641, + "time_per_iteration": 2.555509090423584 + }, + { + "auxiliary_loss_clip": 0.01083843, + "auxiliary_loss_mlp": 0.01046361, + "balance_loss_clip": 1.0291357, + "balance_loss_mlp": 1.02924275, + "epoch": 0.16371655736753526, + "flos": 15698102014080.0, + "grad_norm": 1.8597265172378457, + "language_loss": 0.72640216, + "learning_rate": 3.815368497204654e-06, + "loss": 0.74770415, + "num_input_tokens_seen": 161351985, + "router_z_loss_clip": 0.54711914, + "router_z_loss_mlp": 0.17120361, + "step": 5642, + "time_per_iteration": 2.3882997035980225 + }, + { + "auxiliary_loss_clip": 0.01081392, + "auxiliary_loss_mlp": 0.01034721, + "balance_loss_clip": 1.0272975, + "balance_loss_mlp": 1.01908696, + "epoch": 0.16374557483605132, + "flos": 31495741914240.0, + "grad_norm": 1.7629474793011741, + "language_loss": 0.65489835, + "learning_rate": 3.815289610327566e-06, + "loss": 0.67605948, + "num_input_tokens_seen": 161368360, + "router_z_loss_clip": 0.54150391, + "router_z_loss_mlp": 0.15637207, + "step": 5643, + "time_per_iteration": 2.535090684890747 + }, + { + "auxiliary_loss_clip": 0.01089569, + "auxiliary_loss_mlp": 0.01042476, + "balance_loss_clip": 1.03149843, + "balance_loss_mlp": 1.02542877, + "epoch": 0.16377459230456734, + "flos": 74732321278080.0, + "grad_norm": 1.6649011635226953, + "language_loss": 0.7377584, + "learning_rate": 3.815210707417106e-06, + "loss": 0.75907886, + "num_input_tokens_seen": 161395940, + "router_z_loss_clip": 0.58081055, + "router_z_loss_mlp": 0.17034912, + "step": 5644, + "time_per_iteration": 2.811312675476074 + }, + { + "auxiliary_loss_clip": 0.01012124, + "auxiliary_loss_mlp": 0.01003457, + "balance_loss_clip": 1.00372481, + "balance_loss_mlp": 1.002491, + "epoch": 0.1638036097730834, + "flos": 64593820944000.0, + "grad_norm": 0.7375007199841176, + "language_loss": 0.50338268, + "learning_rate": 3.81513178847397e-06, + "loss": 0.52353847, + "num_input_tokens_seen": 161457300, + "router_z_loss_clip": 0.08398438, + "router_z_loss_mlp": 0.00964355, + "step": 5645, + "time_per_iteration": 2.982008695602417 + }, + { + "auxiliary_loss_clip": 0.01012081, + "auxiliary_loss_mlp": 0.01008652, + "balance_loss_clip": 1.00391734, + "balance_loss_mlp": 1.00754893, + "epoch": 0.16383262724159944, + "flos": 74777810780160.0, + "grad_norm": 0.6330655165594974, + "language_loss": 0.48720306, + "learning_rate": 3.815052853498855e-06, + "loss": 0.50741041, + "num_input_tokens_seen": 161526250, + "router_z_loss_clip": 0.08154297, + "router_z_loss_mlp": 0.01104736, + "step": 5646, + "time_per_iteration": 3.126283645629883 + }, + { + "auxiliary_loss_clip": 0.01089286, + "auxiliary_loss_mlp": 0.01033705, + "balance_loss_clip": 1.02912915, + "balance_loss_mlp": 1.01560879, + "epoch": 0.1638616447101155, + "flos": 32197565531520.0, + "grad_norm": 2.779892412420322, + "language_loss": 0.8652094, + "learning_rate": 3.81497390249246e-06, + "loss": 0.88643932, + "num_input_tokens_seen": 161542895, + "router_z_loss_clip": 0.60205078, + "router_z_loss_mlp": 0.1809082, + "step": 5647, + "time_per_iteration": 2.438446283340454 + }, + { + "auxiliary_loss_clip": 0.01082919, + "auxiliary_loss_mlp": 0.01034614, + "balance_loss_clip": 1.0280571, + "balance_loss_mlp": 1.01870584, + "epoch": 0.16389066217863155, + "flos": 17485271066880.0, + "grad_norm": 2.6499561945900765, + "language_loss": 0.79880071, + "learning_rate": 3.814894935455481e-06, + "loss": 0.81997597, + "num_input_tokens_seen": 161554750, + "router_z_loss_clip": 0.54882812, + "router_z_loss_mlp": 0.15917969, + "step": 5648, + "time_per_iteration": 2.370985507965088 + }, + { + "auxiliary_loss_clip": 0.01095042, + "auxiliary_loss_mlp": 0.01038768, + "balance_loss_clip": 1.02991891, + "balance_loss_mlp": 1.0191586, + "epoch": 0.16391967964714757, + "flos": 21352460590080.0, + "grad_norm": 2.0960685036264626, + "language_loss": 0.86854029, + "learning_rate": 3.814815952388614e-06, + "loss": 0.88987833, + "num_input_tokens_seen": 161568810, + "router_z_loss_clip": 0.65136719, + "router_z_loss_mlp": 0.19616699, + "step": 5649, + "time_per_iteration": 2.4359164237976074 + }, + { + "auxiliary_loss_clip": 0.01083861, + "auxiliary_loss_mlp": 0.01028718, + "balance_loss_clip": 1.02836609, + "balance_loss_mlp": 1.01228476, + "epoch": 0.16394869711566362, + "flos": 13032902378880.0, + "grad_norm": 4.315891867545376, + "language_loss": 1.01129639, + "learning_rate": 3.814736953292559e-06, + "loss": 1.03242218, + "num_input_tokens_seen": 161579495, + "router_z_loss_clip": 0.55517578, + "router_z_loss_mlp": 0.16418457, + "step": 5650, + "time_per_iteration": 2.3293347358703613 + }, + { + "auxiliary_loss_clip": 0.01086941, + "auxiliary_loss_mlp": 0.01043085, + "balance_loss_clip": 1.03039134, + "balance_loss_mlp": 1.02452433, + "epoch": 0.16397771458417967, + "flos": 32553016750080.0, + "grad_norm": 2.1804607049765745, + "language_loss": 0.67068458, + "learning_rate": 3.8146579381680134e-06, + "loss": 0.69198489, + "num_input_tokens_seen": 161594395, + "router_z_loss_clip": 0.56542969, + "router_z_loss_mlp": 0.18566895, + "step": 5651, + "time_per_iteration": 2.537501573562622 + }, + { + "auxiliary_loss_clip": 0.01093894, + "auxiliary_loss_mlp": 0.01040439, + "balance_loss_clip": 1.03095567, + "balance_loss_mlp": 1.01918364, + "epoch": 0.16400673205269573, + "flos": 44119776827520.0, + "grad_norm": 2.379746963051056, + "language_loss": 1.03186274, + "learning_rate": 3.814578907015674e-06, + "loss": 1.05320597, + "num_input_tokens_seen": 161614170, + "router_z_loss_clip": 0.62890625, + "router_z_loss_mlp": 0.21276855, + "step": 5652, + "time_per_iteration": 2.6075985431671143 + }, + { + "auxiliary_loss_clip": 0.01088855, + "auxiliary_loss_mlp": 0.0103352, + "balance_loss_clip": 1.02857673, + "balance_loss_mlp": 1.01566279, + "epoch": 0.16403574952121178, + "flos": 29671704599040.0, + "grad_norm": 2.683399973774888, + "language_loss": 1.03016269, + "learning_rate": 3.8144998598362397e-06, + "loss": 1.05138648, + "num_input_tokens_seen": 161630635, + "router_z_loss_clip": 0.60205078, + "router_z_loss_mlp": 0.17858887, + "step": 5653, + "time_per_iteration": 2.4801793098449707 + }, + { + "auxiliary_loss_clip": 0.01016639, + "auxiliary_loss_mlp": 0.01002968, + "balance_loss_clip": 1.00806987, + "balance_loss_mlp": 1.00194252, + "epoch": 0.16406476698972783, + "flos": 72537679347840.0, + "grad_norm": 0.6414905892283106, + "language_loss": 0.45374057, + "learning_rate": 3.8144207966304084e-06, + "loss": 0.47393665, + "num_input_tokens_seen": 161691765, + "router_z_loss_clip": 0.0859375, + "router_z_loss_mlp": 0.01025391, + "step": 5654, + "time_per_iteration": 3.2420740127563477 + }, + { + "auxiliary_loss_clip": 0.01089994, + "auxiliary_loss_mlp": 0.01041411, + "balance_loss_clip": 1.02904534, + "balance_loss_mlp": 1.02237296, + "epoch": 0.16409378445824385, + "flos": 28326081911040.0, + "grad_norm": 1.9408493108066434, + "language_loss": 0.89713424, + "learning_rate": 3.814341717398878e-06, + "loss": 0.91844827, + "num_input_tokens_seen": 161708955, + "router_z_loss_clip": 0.60986328, + "router_z_loss_mlp": 0.19055176, + "step": 5655, + "time_per_iteration": 2.469325542449951 + }, + { + "auxiliary_loss_clip": 0.01015775, + "auxiliary_loss_mlp": 0.01001536, + "balance_loss_clip": 1.00696909, + "balance_loss_mlp": 1.00046897, + "epoch": 0.1641228019267599, + "flos": 64933001602560.0, + "grad_norm": 0.6518082445235077, + "language_loss": 0.50562608, + "learning_rate": 3.8142626221423475e-06, + "loss": 0.52579921, + "num_input_tokens_seen": 161771950, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01068115, + "step": 5656, + "time_per_iteration": 3.126267910003662 + }, + { + "auxiliary_loss_clip": 0.01015103, + "auxiliary_loss_mlp": 0.01005392, + "balance_loss_clip": 1.00653481, + "balance_loss_mlp": 1.00433087, + "epoch": 0.16415181939527596, + "flos": 74775192428160.0, + "grad_norm": 0.6679174153184961, + "language_loss": 0.53310347, + "learning_rate": 3.8141835108615155e-06, + "loss": 0.55330843, + "num_input_tokens_seen": 161839550, + "router_z_loss_clip": 0.0859375, + "router_z_loss_mlp": 0.01062012, + "step": 5657, + "time_per_iteration": 3.176922082901001 + }, + { + "auxiliary_loss_clip": 0.01015267, + "auxiliary_loss_mlp": 0.01002711, + "balance_loss_clip": 1.00665498, + "balance_loss_mlp": 1.0017575, + "epoch": 0.164180836863792, + "flos": 53899261251840.0, + "grad_norm": 0.7174244751880956, + "language_loss": 0.5091995, + "learning_rate": 3.8141043835570804e-06, + "loss": 0.52937937, + "num_input_tokens_seen": 161900760, + "router_z_loss_clip": 0.0859375, + "router_z_loss_mlp": 0.00952148, + "step": 5658, + "time_per_iteration": 3.115730047225952 + }, + { + "auxiliary_loss_clip": 0.01085383, + "auxiliary_loss_mlp": 0.01042532, + "balance_loss_clip": 1.02968156, + "balance_loss_mlp": 1.0254786, + "epoch": 0.16420985433230806, + "flos": 31277604539520.0, + "grad_norm": 2.621481551550422, + "language_loss": 0.67491591, + "learning_rate": 3.8140252402297415e-06, + "loss": 0.69619513, + "num_input_tokens_seen": 161916895, + "router_z_loss_clip": 0.55664062, + "router_z_loss_mlp": 0.17053223, + "step": 5659, + "time_per_iteration": 2.4371750354766846 + }, + { + "auxiliary_loss_clip": 0.01014775, + "auxiliary_loss_mlp": 0.01001513, + "balance_loss_clip": 1.00587416, + "balance_loss_mlp": 1.00046432, + "epoch": 0.1642388718008241, + "flos": 71820597600000.0, + "grad_norm": 0.7073903169537471, + "language_loss": 0.50003493, + "learning_rate": 3.813946080880198e-06, + "loss": 0.52019781, + "num_input_tokens_seen": 161981025, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.01049805, + "step": 5660, + "time_per_iteration": 3.089326858520508 + }, + { + "auxiliary_loss_clip": 0.01084584, + "auxiliary_loss_mlp": 0.01036262, + "balance_loss_clip": 1.02929759, + "balance_loss_mlp": 1.02082407, + "epoch": 0.16426788926934013, + "flos": 27628692036480.0, + "grad_norm": 2.0875439361795074, + "language_loss": 0.97799528, + "learning_rate": 3.8138669055091483e-06, + "loss": 0.99920368, + "num_input_tokens_seen": 161995510, + "router_z_loss_clip": 0.55224609, + "router_z_loss_mlp": 0.15447998, + "step": 5661, + "time_per_iteration": 2.4620137214660645 + }, + { + "auxiliary_loss_clip": 0.01013575, + "auxiliary_loss_mlp": 0.01003755, + "balance_loss_clip": 1.00483489, + "balance_loss_mlp": 1.00278366, + "epoch": 0.16429690673785619, + "flos": 63827302844160.0, + "grad_norm": 0.6505440136630396, + "language_loss": 0.51536548, + "learning_rate": 3.813787714117292e-06, + "loss": 0.53553879, + "num_input_tokens_seen": 162058640, + "router_z_loss_clip": 0.08740234, + "router_z_loss_mlp": 0.00970459, + "step": 5662, + "time_per_iteration": 2.962773561477661 + }, + { + "auxiliary_loss_clip": 0.010795, + "auxiliary_loss_mlp": 0.01035209, + "balance_loss_clip": 1.02604675, + "balance_loss_mlp": 1.01916981, + "epoch": 0.16432592420637224, + "flos": 12923310021120.0, + "grad_norm": 2.8064702449512553, + "language_loss": 0.85975802, + "learning_rate": 3.8137085067053287e-06, + "loss": 0.88090515, + "num_input_tokens_seen": 162069905, + "router_z_loss_clip": 0.53442383, + "router_z_loss_mlp": 0.16033936, + "step": 5663, + "time_per_iteration": 2.3472747802734375 + }, + { + "auxiliary_loss_clip": 0.01011792, + "auxiliary_loss_mlp": 0.01006161, + "balance_loss_clip": 1.00335681, + "balance_loss_mlp": 1.00508237, + "epoch": 0.1643549416748883, + "flos": 52153114913280.0, + "grad_norm": 0.6757353367885912, + "language_loss": 0.48334765, + "learning_rate": 3.8136292832739582e-06, + "loss": 0.50352716, + "num_input_tokens_seen": 162127965, + "router_z_loss_clip": 0.08447266, + "router_z_loss_mlp": 0.01080322, + "step": 5664, + "time_per_iteration": 2.9145243167877197 + }, + { + "auxiliary_loss_clip": 0.01010687, + "auxiliary_loss_mlp": 0.0100313, + "balance_loss_clip": 1.00234866, + "balance_loss_mlp": 1.00211668, + "epoch": 0.16438395914340434, + "flos": 64591586616960.0, + "grad_norm": 0.7542131537086315, + "language_loss": 0.5159862, + "learning_rate": 3.8135500438238797e-06, + "loss": 0.53612435, + "num_input_tokens_seen": 162185575, + "router_z_loss_clip": 0.08300781, + "router_z_loss_mlp": 0.01013184, + "step": 5665, + "time_per_iteration": 2.9561309814453125 + }, + { + "auxiliary_loss_clip": 0.01011053, + "auxiliary_loss_mlp": 0.01010068, + "balance_loss_clip": 1.0026505, + "balance_loss_mlp": 1.00895357, + "epoch": 0.16441297661192036, + "flos": 70790241288960.0, + "grad_norm": 0.6253956080038836, + "language_loss": 0.47690159, + "learning_rate": 3.8134707883557936e-06, + "loss": 0.49711278, + "num_input_tokens_seen": 162248505, + "router_z_loss_clip": 0.08398438, + "router_z_loss_mlp": 0.01116943, + "step": 5666, + "time_per_iteration": 3.0723745822906494 + }, + { + "auxiliary_loss_clip": 0.01085972, + "auxiliary_loss_mlp": 0.01043241, + "balance_loss_clip": 1.03113222, + "balance_loss_mlp": 1.02571726, + "epoch": 0.16444199408043642, + "flos": 26424467821440.0, + "grad_norm": 2.27441711411702, + "language_loss": 0.71040738, + "learning_rate": 3.813391516870399e-06, + "loss": 0.73169953, + "num_input_tokens_seen": 162265490, + "router_z_loss_clip": 0.54882812, + "router_z_loss_mlp": 0.17529297, + "step": 5667, + "time_per_iteration": 2.4940264225006104 + }, + { + "auxiliary_loss_clip": 0.01086056, + "auxiliary_loss_mlp": 0.01035971, + "balance_loss_clip": 1.03089046, + "balance_loss_mlp": 1.01924574, + "epoch": 0.16447101154895247, + "flos": 12342424953600.0, + "grad_norm": 2.9284697135092608, + "language_loss": 0.79373533, + "learning_rate": 3.8133122293683977e-06, + "loss": 0.81495559, + "num_input_tokens_seen": 162277630, + "router_z_loss_clip": 0.55126953, + "router_z_loss_mlp": 0.1673584, + "step": 5668, + "time_per_iteration": 2.359708070755005 + }, + { + "auxiliary_loss_clip": 0.01091984, + "auxiliary_loss_mlp": 0.01039207, + "balance_loss_clip": 1.03217721, + "balance_loss_mlp": 1.02167702, + "epoch": 0.16450002901746852, + "flos": 27627714518400.0, + "grad_norm": 2.148570393729902, + "language_loss": 0.81091475, + "learning_rate": 3.813232925850488e-06, + "loss": 0.83222663, + "num_input_tokens_seen": 162294125, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.1751709, + "step": 5669, + "time_per_iteration": 2.485888719558716 + }, + { + "auxiliary_loss_clip": 0.01082625, + "auxiliary_loss_mlp": 0.01033465, + "balance_loss_clip": 1.03181481, + "balance_loss_mlp": 1.01734221, + "epoch": 0.16452904648598457, + "flos": 20845416781440.0, + "grad_norm": 2.5767314354763826, + "language_loss": 0.90711904, + "learning_rate": 3.813153606317372e-06, + "loss": 0.92827988, + "num_input_tokens_seen": 162307815, + "router_z_loss_clip": 0.50878906, + "router_z_loss_mlp": 0.16101074, + "step": 5670, + "time_per_iteration": 2.4394195079803467 + }, + { + "auxiliary_loss_clip": 0.010845, + "auxiliary_loss_mlp": 0.01036258, + "balance_loss_clip": 1.02941036, + "balance_loss_mlp": 1.01915717, + "epoch": 0.16455806395450062, + "flos": 26131511669760.0, + "grad_norm": 2.520604973101931, + "language_loss": 0.9127115, + "learning_rate": 3.8130742707697497e-06, + "loss": 0.93391907, + "num_input_tokens_seen": 162323345, + "router_z_loss_clip": 0.55053711, + "router_z_loss_mlp": 0.17089844, + "step": 5671, + "time_per_iteration": 2.5402321815490723 + }, + { + "auxiliary_loss_clip": 0.01087867, + "auxiliary_loss_mlp": 0.01034976, + "balance_loss_clip": 1.03350389, + "balance_loss_mlp": 1.01862073, + "epoch": 0.16458708142301665, + "flos": 30842062928640.0, + "grad_norm": 1.9103058881965977, + "language_loss": 0.65546739, + "learning_rate": 3.8129949192083215e-06, + "loss": 0.67669582, + "num_input_tokens_seen": 162339210, + "router_z_loss_clip": 0.54321289, + "router_z_loss_mlp": 0.16345215, + "step": 5672, + "time_per_iteration": 2.499467372894287 + }, + { + "auxiliary_loss_clip": 0.01091822, + "auxiliary_loss_mlp": 0.01037347, + "balance_loss_clip": 1.03764868, + "balance_loss_mlp": 1.02120018, + "epoch": 0.1646160988915327, + "flos": 37735838236800.0, + "grad_norm": 2.237425092256141, + "language_loss": 1.0297358, + "learning_rate": 3.8129155516337887e-06, + "loss": 1.05102742, + "num_input_tokens_seen": 162362150, + "router_z_loss_clip": 0.54125977, + "router_z_loss_mlp": 0.16149902, + "step": 5673, + "time_per_iteration": 2.7067196369171143 + }, + { + "auxiliary_loss_clip": 0.01025691, + "auxiliary_loss_mlp": 0.01006082, + "balance_loss_clip": 1.01564801, + "balance_loss_mlp": 1.00492597, + "epoch": 0.16464511636004875, + "flos": 74776344503040.0, + "grad_norm": 0.6482805508150019, + "language_loss": 0.46050203, + "learning_rate": 3.8128361680468516e-06, + "loss": 0.48081976, + "num_input_tokens_seen": 162428305, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.01153564, + "step": 5674, + "time_per_iteration": 3.097740888595581 + }, + { + "auxiliary_loss_clip": 0.0109635, + "auxiliary_loss_mlp": 0.01037758, + "balance_loss_clip": 1.03477955, + "balance_loss_mlp": 1.01808834, + "epoch": 0.1646741338285648, + "flos": 21463100288640.0, + "grad_norm": 2.206074206189653, + "language_loss": 0.88655823, + "learning_rate": 3.8127567684482126e-06, + "loss": 0.90789932, + "num_input_tokens_seen": 162443055, + "router_z_loss_clip": 0.61523438, + "router_z_loss_mlp": 0.19671631, + "step": 5675, + "time_per_iteration": 2.510939598083496 + }, + { + "auxiliary_loss_clip": 0.01091722, + "auxiliary_loss_mlp": 0.01037848, + "balance_loss_clip": 1.03768671, + "balance_loss_mlp": 1.02104533, + "epoch": 0.16470315129708085, + "flos": 74729318901120.0, + "grad_norm": 2.3513060360027644, + "language_loss": 0.93479329, + "learning_rate": 3.8126773528385723e-06, + "loss": 0.95608902, + "num_input_tokens_seen": 162468485, + "router_z_loss_clip": 0.5402832, + "router_z_loss_mlp": 0.16815186, + "step": 5676, + "time_per_iteration": 2.826333522796631 + }, + { + "auxiliary_loss_clip": 0.01098699, + "auxiliary_loss_mlp": 0.01040671, + "balance_loss_clip": 1.0369308, + "balance_loss_mlp": 1.02046514, + "epoch": 0.16473216876559688, + "flos": 27775920706560.0, + "grad_norm": 2.586812182299591, + "language_loss": 0.87714624, + "learning_rate": 3.8125979212186316e-06, + "loss": 0.89853996, + "num_input_tokens_seen": 162485545, + "router_z_loss_clip": 0.61791992, + "router_z_loss_mlp": 0.20214844, + "step": 5677, + "time_per_iteration": 2.4829134941101074 + }, + { + "auxiliary_loss_clip": 0.0109216, + "auxiliary_loss_mlp": 0.01036896, + "balance_loss_clip": 1.03537595, + "balance_loss_mlp": 1.01877606, + "epoch": 0.16476118623411293, + "flos": 30183112327680.0, + "grad_norm": 2.1182639379960952, + "language_loss": 0.76793897, + "learning_rate": 3.812518473589093e-06, + "loss": 0.78922951, + "num_input_tokens_seen": 162501620, + "router_z_loss_clip": 0.56811523, + "router_z_loss_mlp": 0.18139648, + "step": 5678, + "time_per_iteration": 2.4706499576568604 + }, + { + "auxiliary_loss_clip": 0.01028415, + "auxiliary_loss_mlp": 0.01001459, + "balance_loss_clip": 1.01879764, + "balance_loss_mlp": 1.00049388, + "epoch": 0.16479020370262898, + "flos": 74766778790400.0, + "grad_norm": 0.6584775488875411, + "language_loss": 0.45244253, + "learning_rate": 3.8124390099506573e-06, + "loss": 0.47274125, + "num_input_tokens_seen": 162560385, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.00964355, + "step": 5679, + "time_per_iteration": 3.242511749267578 + }, + { + "auxiliary_loss_clip": 0.0108775, + "auxiliary_loss_mlp": 0.01045229, + "balance_loss_clip": 1.0341599, + "balance_loss_mlp": 1.028301, + "epoch": 0.16481922117114503, + "flos": 31936974076800.0, + "grad_norm": 2.202869654360721, + "language_loss": 0.94896913, + "learning_rate": 3.812359530304027e-06, + "loss": 0.97029895, + "num_input_tokens_seen": 162575985, + "router_z_loss_clip": 0.53564453, + "router_z_loss_mlp": 0.16931152, + "step": 5680, + "time_per_iteration": 2.538832187652588 + }, + { + "auxiliary_loss_clip": 0.01085173, + "auxiliary_loss_mlp": 0.01033258, + "balance_loss_clip": 1.0316596, + "balance_loss_mlp": 1.01824951, + "epoch": 0.16484823863966108, + "flos": 16683036779520.0, + "grad_norm": 2.769110388003526, + "language_loss": 0.56237316, + "learning_rate": 3.8122800346499044e-06, + "loss": 0.58355749, + "num_input_tokens_seen": 162590770, + "router_z_loss_clip": 0.53515625, + "router_z_loss_mlp": 0.15002441, + "step": 5681, + "time_per_iteration": 2.3702456951141357 + }, + { + "auxiliary_loss_clip": 0.01024534, + "auxiliary_loss_mlp": 0.01004556, + "balance_loss_clip": 1.01530671, + "balance_loss_mlp": 1.00353134, + "epoch": 0.16487725610817713, + "flos": 63459458092800.0, + "grad_norm": 0.7278455088099748, + "language_loss": 0.53533852, + "learning_rate": 3.8122005229889907e-06, + "loss": 0.55562943, + "num_input_tokens_seen": 162650945, + "router_z_loss_clip": 0.09228516, + "router_z_loss_mlp": 0.01025391, + "step": 5682, + "time_per_iteration": 3.0109426975250244 + }, + { + "auxiliary_loss_clip": 0.01092957, + "auxiliary_loss_mlp": 0.01042147, + "balance_loss_clip": 1.03338039, + "balance_loss_mlp": 1.02474809, + "epoch": 0.16490627357669316, + "flos": 32626229604480.0, + "grad_norm": 2.856079885724088, + "language_loss": 0.85140359, + "learning_rate": 3.812120995321989e-06, + "loss": 0.87275463, + "num_input_tokens_seen": 162665420, + "router_z_loss_clip": 0.59619141, + "router_z_loss_mlp": 0.17388916, + "step": 5683, + "time_per_iteration": 2.5076088905334473 + }, + { + "auxiliary_loss_clip": 0.01020993, + "auxiliary_loss_mlp": 0.01008455, + "balance_loss_clip": 1.01209879, + "balance_loss_mlp": 1.00757897, + "epoch": 0.1649352910452092, + "flos": 61888576112640.0, + "grad_norm": 0.7174092239950031, + "language_loss": 0.42631376, + "learning_rate": 3.812041451649601e-06, + "loss": 0.44660819, + "num_input_tokens_seen": 162721645, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.00878906, + "step": 5684, + "time_per_iteration": 2.916971206665039 + }, + { + "auxiliary_loss_clip": 0.01018363, + "auxiliary_loss_mlp": 0.01021048, + "balance_loss_clip": 1.00969923, + "balance_loss_mlp": 1.01997471, + "epoch": 0.16496430851372526, + "flos": 59385547209600.0, + "grad_norm": 0.6778760732973098, + "language_loss": 0.48165807, + "learning_rate": 3.8119618919725302e-06, + "loss": 0.50205219, + "num_input_tokens_seen": 162786680, + "router_z_loss_clip": 0.08642578, + "router_z_loss_mlp": 0.01074219, + "step": 5685, + "time_per_iteration": 3.2125563621520996 + }, + { + "auxiliary_loss_clip": 0.0108009, + "auxiliary_loss_mlp": 0.01035651, + "balance_loss_clip": 1.02667677, + "balance_loss_mlp": 1.01955175, + "epoch": 0.1649933259822413, + "flos": 16757576265600.0, + "grad_norm": 2.299325914678552, + "language_loss": 0.80889738, + "learning_rate": 3.811882316291478e-06, + "loss": 0.83005482, + "num_input_tokens_seen": 162800670, + "router_z_loss_clip": 0.53417969, + "router_z_loss_mlp": 0.16101074, + "step": 5686, + "time_per_iteration": 2.4240193367004395 + }, + { + "auxiliary_loss_clip": 0.0108433, + "auxiliary_loss_mlp": 0.01039207, + "balance_loss_clip": 1.02922726, + "balance_loss_mlp": 1.02175438, + "epoch": 0.16502234345075736, + "flos": 18080365057920.0, + "grad_norm": 2.168164219720967, + "language_loss": 0.75052559, + "learning_rate": 3.8118027246071484e-06, + "loss": 0.77176094, + "num_input_tokens_seen": 162817905, + "router_z_loss_clip": 0.55126953, + "router_z_loss_mlp": 0.17456055, + "step": 5687, + "time_per_iteration": 2.3880112171173096 + }, + { + "auxiliary_loss_clip": 0.0108351, + "auxiliary_loss_mlp": 0.0103536, + "balance_loss_clip": 1.0279541, + "balance_loss_mlp": 1.01882005, + "epoch": 0.16505136091927342, + "flos": 28943032279680.0, + "grad_norm": 1.913935848549865, + "language_loss": 0.77900302, + "learning_rate": 3.811723116920244e-06, + "loss": 0.80019176, + "num_input_tokens_seen": 162833955, + "router_z_loss_clip": 0.55541992, + "router_z_loss_mlp": 0.1652832, + "step": 5688, + "time_per_iteration": 2.48913836479187 + }, + { + "auxiliary_loss_clip": 0.01081688, + "auxiliary_loss_mlp": 0.0103936, + "balance_loss_clip": 1.02986336, + "balance_loss_mlp": 1.02396965, + "epoch": 0.16508037838778944, + "flos": 11976709795200.0, + "grad_norm": 6.294785591247222, + "language_loss": 0.98486638, + "learning_rate": 3.811643493231468e-06, + "loss": 1.00607681, + "num_input_tokens_seen": 162846135, + "router_z_loss_clip": 0.51757812, + "router_z_loss_mlp": 0.15380859, + "step": 5689, + "time_per_iteration": 2.3479502201080322 + }, + { + "auxiliary_loss_clip": 0.01095422, + "auxiliary_loss_mlp": 0.01043761, + "balance_loss_clip": 1.03417468, + "balance_loss_mlp": 1.02365029, + "epoch": 0.1651093958563055, + "flos": 30546907361280.0, + "grad_norm": 1.851394785863671, + "language_loss": 0.95288825, + "learning_rate": 3.8115638535415235e-06, + "loss": 0.97428018, + "num_input_tokens_seen": 162866390, + "router_z_loss_clip": 0.61181641, + "router_z_loss_mlp": 0.20117188, + "step": 5690, + "time_per_iteration": 2.582644462585449 + }, + { + "auxiliary_loss_clip": 0.01022013, + "auxiliary_loss_mlp": 0.01005234, + "balance_loss_clip": 1.01303148, + "balance_loss_mlp": 1.00402367, + "epoch": 0.16513841332482154, + "flos": 72768559368960.0, + "grad_norm": 0.6633613030013041, + "language_loss": 0.5078336, + "learning_rate": 3.811484197851114e-06, + "loss": 0.52810609, + "num_input_tokens_seen": 162929835, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01208496, + "step": 5691, + "time_per_iteration": 3.065636157989502 + }, + { + "auxiliary_loss_clip": 0.01099628, + "auxiliary_loss_mlp": 0.01053214, + "balance_loss_clip": 1.03614116, + "balance_loss_mlp": 1.03124356, + "epoch": 0.1651674307933376, + "flos": 14603819270400.0, + "grad_norm": 2.5996675497494026, + "language_loss": 0.78877133, + "learning_rate": 3.8114045261609428e-06, + "loss": 0.81029975, + "num_input_tokens_seen": 162942410, + "router_z_loss_clip": 0.63525391, + "router_z_loss_mlp": 0.21972656, + "step": 5692, + "time_per_iteration": 2.4887778759002686 + }, + { + "auxiliary_loss_clip": 0.01025355, + "auxiliary_loss_mlp": 0.01011792, + "balance_loss_clip": 1.01604199, + "balance_loss_mlp": 1.01060605, + "epoch": 0.16519644826185365, + "flos": 74729980350720.0, + "grad_norm": 0.730357450425492, + "language_loss": 0.47743964, + "learning_rate": 3.811324838471714e-06, + "loss": 0.49781111, + "num_input_tokens_seen": 162997310, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01184082, + "step": 5693, + "time_per_iteration": 3.0539052486419678 + }, + { + "auxiliary_loss_clip": 0.01026022, + "auxiliary_loss_mlp": 0.01008396, + "balance_loss_clip": 1.01657748, + "balance_loss_mlp": 1.00712621, + "epoch": 0.16522546573036967, + "flos": 74777217287040.0, + "grad_norm": 0.6646404960640849, + "language_loss": 0.45867717, + "learning_rate": 3.811245134784131e-06, + "loss": 0.47902134, + "num_input_tokens_seen": 163058800, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.01269531, + "step": 5694, + "time_per_iteration": 3.0730795860290527 + }, + { + "auxiliary_loss_clip": 0.01086918, + "auxiliary_loss_mlp": 0.01036456, + "balance_loss_clip": 1.03552747, + "balance_loss_mlp": 1.01960564, + "epoch": 0.16525448319888572, + "flos": 28760541269760.0, + "grad_norm": 1.959908240229389, + "language_loss": 0.80872238, + "learning_rate": 3.8111654150988983e-06, + "loss": 0.82995611, + "num_input_tokens_seen": 163077395, + "router_z_loss_clip": 0.51293945, + "router_z_loss_mlp": 0.16842651, + "step": 5695, + "time_per_iteration": 2.6028735637664795 + }, + { + "auxiliary_loss_clip": 0.0110437, + "auxiliary_loss_mlp": 0.01039271, + "balance_loss_clip": 1.04300392, + "balance_loss_mlp": 1.01998258, + "epoch": 0.16528350066740177, + "flos": 74731623050880.0, + "grad_norm": 1.5894495515134115, + "language_loss": 0.77824402, + "learning_rate": 3.81108567941672e-06, + "loss": 0.79968041, + "num_input_tokens_seen": 163106130, + "router_z_loss_clip": 0.61328125, + "router_z_loss_mlp": 0.19287109, + "step": 5696, + "time_per_iteration": 2.841789960861206 + }, + { + "auxiliary_loss_clip": 0.01098471, + "auxiliary_loss_mlp": 0.01039608, + "balance_loss_clip": 1.03995419, + "balance_loss_mlp": 1.02196527, + "epoch": 0.16531251813591782, + "flos": 12451982400000.0, + "grad_norm": 2.7395854732116, + "language_loss": 0.79319614, + "learning_rate": 3.8110059277382998e-06, + "loss": 0.81457686, + "num_input_tokens_seen": 163119230, + "router_z_loss_clip": 0.58447266, + "router_z_loss_mlp": 0.17651367, + "step": 5697, + "time_per_iteration": 2.4532525539398193 + }, + { + "auxiliary_loss_clip": 0.01104668, + "auxiliary_loss_mlp": 0.0104972, + "balance_loss_clip": 1.04360986, + "balance_loss_mlp": 1.03022933, + "epoch": 0.16534153560443388, + "flos": 38287640275200.0, + "grad_norm": 2.352276805620025, + "language_loss": 0.94704145, + "learning_rate": 3.810926160064342e-06, + "loss": 0.96858531, + "num_input_tokens_seen": 163141040, + "router_z_loss_clip": 0.61132812, + "router_z_loss_mlp": 0.19494629, + "step": 5698, + "time_per_iteration": 2.5687308311462402 + }, + { + "auxiliary_loss_clip": 0.01096401, + "auxiliary_loss_mlp": 0.01048753, + "balance_loss_clip": 1.03784347, + "balance_loss_mlp": 1.03097928, + "epoch": 0.16537055307294993, + "flos": 15663363344640.0, + "grad_norm": 2.278651670344374, + "language_loss": 0.81633025, + "learning_rate": 3.8108463763955526e-06, + "loss": 0.83778179, + "num_input_tokens_seen": 163155495, + "router_z_loss_clip": 0.58447266, + "router_z_loss_mlp": 0.17779541, + "step": 5699, + "time_per_iteration": 2.453037977218628 + }, + { + "auxiliary_loss_clip": 0.01034058, + "auxiliary_loss_mlp": 0.00999691, + "balance_loss_clip": 1.02273381, + "balance_loss_mlp": 0.99875504, + "epoch": 0.16539957054146595, + "flos": 50212537879680.0, + "grad_norm": 0.717124226622072, + "language_loss": 0.44754642, + "learning_rate": 3.8107665767326343e-06, + "loss": 0.46788388, + "num_input_tokens_seen": 163212515, + "router_z_loss_clip": 0.11328125, + "router_z_loss_mlp": 0.00933838, + "step": 5700, + "time_per_iteration": 2.8746683597564697 + }, + { + "auxiliary_loss_clip": 0.0108329, + "auxiliary_loss_mlp": 0.01033296, + "balance_loss_clip": 1.03255296, + "balance_loss_mlp": 1.01955676, + "epoch": 0.165428588009982, + "flos": 30729572928000.0, + "grad_norm": 2.7015883695970944, + "language_loss": 0.92520505, + "learning_rate": 3.8106867610762935e-06, + "loss": 0.9463709, + "num_input_tokens_seen": 163227465, + "router_z_loss_clip": 0.50683594, + "router_z_loss_mlp": 0.13739014, + "step": 5701, + "time_per_iteration": 2.501473903656006 + }, + { + "auxiliary_loss_clip": 0.01099978, + "auxiliary_loss_mlp": 0.01044946, + "balance_loss_clip": 1.03915501, + "balance_loss_mlp": 1.02696872, + "epoch": 0.16545760547849805, + "flos": 15479161678080.0, + "grad_norm": 2.768066968629799, + "language_loss": 0.96837556, + "learning_rate": 3.810606929427234e-06, + "loss": 0.98982477, + "num_input_tokens_seen": 163240740, + "router_z_loss_clip": 0.60839844, + "router_z_loss_mlp": 0.1796875, + "step": 5702, + "time_per_iteration": 2.373077869415283 + }, + { + "auxiliary_loss_clip": 0.0108937, + "auxiliary_loss_mlp": 0.01039431, + "balance_loss_clip": 1.03350234, + "balance_loss_mlp": 1.02291489, + "epoch": 0.1654866229470141, + "flos": 39674599879680.0, + "grad_norm": 2.3151528722817263, + "language_loss": 0.84300756, + "learning_rate": 3.810527081786162e-06, + "loss": 0.86429548, + "num_input_tokens_seen": 163262645, + "router_z_loss_clip": 0.55810547, + "router_z_loss_mlp": 0.1651001, + "step": 5703, + "time_per_iteration": 2.6524765491485596 + }, + { + "auxiliary_loss_clip": 0.01088896, + "auxiliary_loss_mlp": 0.01051165, + "balance_loss_clip": 1.03086376, + "balance_loss_mlp": 1.03293753, + "epoch": 0.16551564041553016, + "flos": 27918261786240.0, + "grad_norm": 4.171452991299013, + "language_loss": 0.97964501, + "learning_rate": 3.8104472181537813e-06, + "loss": 1.0010457, + "num_input_tokens_seen": 163275950, + "router_z_loss_clip": 0.58056641, + "router_z_loss_mlp": 0.18212891, + "step": 5704, + "time_per_iteration": 2.490086793899536 + }, + { + "auxiliary_loss_clip": 0.01083799, + "auxiliary_loss_mlp": 0.01045757, + "balance_loss_clip": 1.03122473, + "balance_loss_mlp": 1.02850175, + "epoch": 0.1655446578840462, + "flos": 24710232332160.0, + "grad_norm": 2.44633319751539, + "language_loss": 0.91613495, + "learning_rate": 3.810367338530799e-06, + "loss": 0.9374305, + "num_input_tokens_seen": 163292635, + "router_z_loss_clip": 0.52587891, + "router_z_loss_mlp": 0.17266846, + "step": 5705, + "time_per_iteration": 2.471712827682495 + }, + { + "auxiliary_loss_clip": 0.01086693, + "auxiliary_loss_mlp": 0.01045148, + "balance_loss_clip": 1.03222728, + "balance_loss_mlp": 1.0287503, + "epoch": 0.16557367535256223, + "flos": 12013962082560.0, + "grad_norm": 4.307637181994143, + "language_loss": 1.00814354, + "learning_rate": 3.810287442917919e-06, + "loss": 1.02946198, + "num_input_tokens_seen": 163303900, + "router_z_loss_clip": 0.54394531, + "router_z_loss_mlp": 0.16400146, + "step": 5706, + "time_per_iteration": 4.627732276916504 + }, + { + "auxiliary_loss_clip": 0.01091007, + "auxiliary_loss_mlp": 0.01044647, + "balance_loss_clip": 1.031479, + "balance_loss_mlp": 1.02743268, + "epoch": 0.16560269282107828, + "flos": 11211308858880.0, + "grad_norm": 2.593891374748864, + "language_loss": 0.82396007, + "learning_rate": 3.8102075313158487e-06, + "loss": 0.84531659, + "num_input_tokens_seen": 163316165, + "router_z_loss_clip": 0.59521484, + "router_z_loss_mlp": 0.17224121, + "step": 5707, + "time_per_iteration": 4.510273694992065 + }, + { + "auxiliary_loss_clip": 0.01090106, + "auxiliary_loss_mlp": 0.01043105, + "balance_loss_clip": 1.02980208, + "balance_loss_mlp": 1.02466369, + "epoch": 0.16563171028959434, + "flos": 19236619198080.0, + "grad_norm": 2.8030037381775443, + "language_loss": 0.82228595, + "learning_rate": 3.8101276037252923e-06, + "loss": 0.84361804, + "num_input_tokens_seen": 163329610, + "router_z_loss_clip": 0.60375977, + "router_z_loss_mlp": 0.18432617, + "step": 5708, + "time_per_iteration": 2.363462209701538 + }, + { + "auxiliary_loss_clip": 0.01084055, + "auxiliary_loss_mlp": 0.01044292, + "balance_loss_clip": 1.02806258, + "balance_loss_mlp": 1.02752566, + "epoch": 0.1656607277581104, + "flos": 56053229581440.0, + "grad_norm": 2.3546423944791517, + "language_loss": 0.78247762, + "learning_rate": 3.8100476601469564e-06, + "loss": 0.80376112, + "num_input_tokens_seen": 163350825, + "router_z_loss_clip": 0.55957031, + "router_z_loss_mlp": 0.16760254, + "step": 5709, + "time_per_iteration": 2.7290618419647217 + }, + { + "auxiliary_loss_clip": 0.01087319, + "auxiliary_loss_mlp": 0.01047509, + "balance_loss_clip": 1.03148627, + "balance_loss_mlp": 1.02969897, + "epoch": 0.16568974522662644, + "flos": 35910893226240.0, + "grad_norm": 1.982565068935312, + "language_loss": 0.84297895, + "learning_rate": 3.8099677005815475e-06, + "loss": 0.86432719, + "num_input_tokens_seen": 163367640, + "router_z_loss_clip": 0.55908203, + "router_z_loss_mlp": 0.17810059, + "step": 5710, + "time_per_iteration": 2.4986400604248047 + }, + { + "auxiliary_loss_clip": 0.0108656, + "auxiliary_loss_mlp": 0.01040867, + "balance_loss_clip": 1.03042936, + "balance_loss_mlp": 1.02209198, + "epoch": 0.16571876269514246, + "flos": 11137642156800.0, + "grad_norm": 2.5320138237789553, + "language_loss": 0.84903204, + "learning_rate": 3.809887725029771e-06, + "loss": 0.87030631, + "num_input_tokens_seen": 163380420, + "router_z_loss_clip": 0.5612793, + "router_z_loss_mlp": 0.18798828, + "step": 5711, + "time_per_iteration": 2.4506242275238037 + }, + { + "auxiliary_loss_clip": 0.010903, + "auxiliary_loss_mlp": 0.01041492, + "balance_loss_clip": 1.02908421, + "balance_loss_mlp": 1.02282381, + "epoch": 0.16574778016365851, + "flos": 20916814245120.0, + "grad_norm": 2.0251521874863156, + "language_loss": 0.91892898, + "learning_rate": 3.8098077334923344e-06, + "loss": 0.94024694, + "num_input_tokens_seen": 163395670, + "router_z_loss_clip": 0.61230469, + "router_z_loss_mlp": 0.18664551, + "step": 5712, + "time_per_iteration": 2.4039270877838135 + }, + { + "auxiliary_loss_clip": 0.01093188, + "auxiliary_loss_mlp": 0.01043977, + "balance_loss_clip": 1.0320648, + "balance_loss_mlp": 1.02484417, + "epoch": 0.16577679763217457, + "flos": 25806295555200.0, + "grad_norm": 2.684737010064151, + "language_loss": 1.15407848, + "learning_rate": 3.809727725969943e-06, + "loss": 1.17545009, + "num_input_tokens_seen": 163415450, + "router_z_loss_clip": 0.61083984, + "router_z_loss_mlp": 0.19122314, + "step": 5713, + "time_per_iteration": 2.4701108932495117 + }, + { + "auxiliary_loss_clip": 0.01017073, + "auxiliary_loss_mlp": 0.010048, + "balance_loss_clip": 1.00816226, + "balance_loss_mlp": 1.00384057, + "epoch": 0.16580581510069062, + "flos": 69328393084800.0, + "grad_norm": 0.9075293326972322, + "language_loss": 0.48756957, + "learning_rate": 3.809647702463304e-06, + "loss": 0.5077883, + "num_input_tokens_seen": 163469760, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.00958252, + "step": 5714, + "time_per_iteration": 2.920474052429199 + }, + { + "auxiliary_loss_clip": 0.01085223, + "auxiliary_loss_mlp": 0.0103747, + "balance_loss_clip": 1.02897036, + "balance_loss_mlp": 1.02038789, + "epoch": 0.16583483256920667, + "flos": 32919115933440.0, + "grad_norm": 3.5852839800041654, + "language_loss": 0.87405419, + "learning_rate": 3.8095676629731245e-06, + "loss": 0.89528108, + "num_input_tokens_seen": 163485215, + "router_z_loss_clip": 0.56274414, + "router_z_loss_mlp": 0.1706543, + "step": 5715, + "time_per_iteration": 5.015303611755371 + }, + { + "auxiliary_loss_clip": 0.01094068, + "auxiliary_loss_mlp": 0.010441, + "balance_loss_clip": 1.0316968, + "balance_loss_mlp": 1.02431166, + "epoch": 0.16586385003772272, + "flos": 37330776109440.0, + "grad_norm": 2.1899296372476424, + "language_loss": 0.99994493, + "learning_rate": 3.8094876075001113e-06, + "loss": 1.02132666, + "num_input_tokens_seen": 163503220, + "router_z_loss_clip": 0.6237793, + "router_z_loss_mlp": 0.19775391, + "step": 5716, + "time_per_iteration": 5.082784175872803 + }, + { + "auxiliary_loss_clip": 0.01084088, + "auxiliary_loss_mlp": 0.01036156, + "balance_loss_clip": 1.0305109, + "balance_loss_mlp": 1.019943, + "epoch": 0.16589286750623874, + "flos": 30989012307840.0, + "grad_norm": 2.5585968369765046, + "language_loss": 0.59780896, + "learning_rate": 3.809407536044971e-06, + "loss": 0.6190114, + "num_input_tokens_seen": 163521080, + "router_z_loss_clip": 0.53564453, + "router_z_loss_mlp": 0.16223145, + "step": 5717, + "time_per_iteration": 2.440462589263916 + }, + { + "auxiliary_loss_clip": 0.01081247, + "auxiliary_loss_mlp": 0.01036348, + "balance_loss_clip": 1.02764869, + "balance_loss_mlp": 1.01987922, + "epoch": 0.1659218849747548, + "flos": 25742788058880.0, + "grad_norm": 2.4475183187799288, + "language_loss": 0.75455022, + "learning_rate": 3.8093274486084108e-06, + "loss": 0.7757262, + "num_input_tokens_seen": 163543855, + "router_z_loss_clip": 0.53637695, + "router_z_loss_mlp": 0.16479492, + "step": 5718, + "time_per_iteration": 2.4914462566375732 + }, + { + "auxiliary_loss_clip": 0.01087836, + "auxiliary_loss_mlp": 0.01036868, + "balance_loss_clip": 1.03001285, + "balance_loss_mlp": 1.01609588, + "epoch": 0.16595090244327085, + "flos": 29194197667200.0, + "grad_norm": 2.498865420303197, + "language_loss": 0.89927256, + "learning_rate": 3.8092473451911385e-06, + "loss": 0.92051959, + "num_input_tokens_seen": 163556875, + "router_z_loss_clip": 0.57788086, + "router_z_loss_mlp": 0.20776367, + "step": 5719, + "time_per_iteration": 2.5391838550567627 + }, + { + "auxiliary_loss_clip": 0.01088435, + "auxiliary_loss_mlp": 0.01037195, + "balance_loss_clip": 1.03052449, + "balance_loss_mlp": 1.01721549, + "epoch": 0.1659799199117869, + "flos": 45028077425280.0, + "grad_norm": 2.4096181971954187, + "language_loss": 1.10893536, + "learning_rate": 3.809167225793862e-06, + "loss": 1.13019156, + "num_input_tokens_seen": 163570895, + "router_z_loss_clip": 0.57958984, + "router_z_loss_mlp": 0.19995117, + "step": 5720, + "time_per_iteration": 2.596534490585327 + }, + { + "auxiliary_loss_clip": 0.01093977, + "auxiliary_loss_mlp": 0.01040306, + "balance_loss_clip": 1.03312445, + "balance_loss_mlp": 1.02024293, + "epoch": 0.16600893738030295, + "flos": 27776304731520.0, + "grad_norm": 2.48389014365656, + "language_loss": 0.86953455, + "learning_rate": 3.8090870904172883e-06, + "loss": 0.89087737, + "num_input_tokens_seen": 163587150, + "router_z_loss_clip": 0.60864258, + "router_z_loss_mlp": 0.20080566, + "step": 5721, + "time_per_iteration": 2.51400089263916 + }, + { + "auxiliary_loss_clip": 0.01085524, + "auxiliary_loss_mlp": 0.0103422, + "balance_loss_clip": 1.03048313, + "balance_loss_mlp": 1.01717877, + "epoch": 0.166037954848819, + "flos": 31681270212480.0, + "grad_norm": 1.7260101759917792, + "language_loss": 0.87000704, + "learning_rate": 3.809006939062126e-06, + "loss": 0.89120448, + "num_input_tokens_seen": 163605820, + "router_z_loss_clip": 0.55078125, + "router_z_loss_mlp": 0.1706543, + "step": 5722, + "time_per_iteration": 2.4877688884735107 + }, + { + "auxiliary_loss_clip": 0.01088782, + "auxiliary_loss_mlp": 0.0103378, + "balance_loss_clip": 1.03303623, + "balance_loss_mlp": 1.01671517, + "epoch": 0.16606697231733503, + "flos": 25878635625600.0, + "grad_norm": 1.7697867339784352, + "language_loss": 0.7593146, + "learning_rate": 3.808926771729081e-06, + "loss": 0.78054023, + "num_input_tokens_seen": 163619870, + "router_z_loss_clip": 0.55688477, + "router_z_loss_mlp": 0.17071533, + "step": 5723, + "time_per_iteration": 2.4593026638031006 + }, + { + "auxiliary_loss_clip": 0.0101667, + "auxiliary_loss_mlp": 0.01013233, + "balance_loss_clip": 1.0072732, + "balance_loss_mlp": 1.01216018, + "epoch": 0.16609598978585108, + "flos": 73099221655680.0, + "grad_norm": 0.6367906710027215, + "language_loss": 0.47636563, + "learning_rate": 3.8088465884188636e-06, + "loss": 0.49666464, + "num_input_tokens_seen": 163685955, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.01074219, + "step": 5724, + "time_per_iteration": 3.144761562347412 + }, + { + "auxiliary_loss_clip": 0.01093776, + "auxiliary_loss_mlp": 0.01044655, + "balance_loss_clip": 1.03175414, + "balance_loss_mlp": 1.02471173, + "epoch": 0.16612500725436713, + "flos": 39522762910080.0, + "grad_norm": 2.2914789689441437, + "language_loss": 0.80954725, + "learning_rate": 3.808766389132181e-06, + "loss": 0.83093154, + "num_input_tokens_seen": 163706430, + "router_z_loss_clip": 0.62060547, + "router_z_loss_mlp": 0.19946289, + "step": 5725, + "time_per_iteration": 2.557767868041992 + }, + { + "auxiliary_loss_clip": 0.01087041, + "auxiliary_loss_mlp": 0.01034253, + "balance_loss_clip": 1.03222024, + "balance_loss_mlp": 1.01742697, + "epoch": 0.16615402472288318, + "flos": 28031170723200.0, + "grad_norm": 2.1235319907178676, + "language_loss": 0.96177113, + "learning_rate": 3.808686173869742e-06, + "loss": 0.98298407, + "num_input_tokens_seen": 163724880, + "router_z_loss_clip": 0.5480957, + "router_z_loss_mlp": 0.168396, + "step": 5726, + "time_per_iteration": 2.4514412879943848 + }, + { + "auxiliary_loss_clip": 0.01091533, + "auxiliary_loss_mlp": 0.0103749, + "balance_loss_clip": 1.03511357, + "balance_loss_mlp": 1.02028847, + "epoch": 0.16618304219139923, + "flos": 30512517805440.0, + "grad_norm": 3.0241701806201258, + "language_loss": 0.84361207, + "learning_rate": 3.8086059426322546e-06, + "loss": 0.86490232, + "num_input_tokens_seen": 163740370, + "router_z_loss_clip": 0.56542969, + "router_z_loss_mlp": 0.171875, + "step": 5727, + "time_per_iteration": 2.485534429550171 + }, + { + "auxiliary_loss_clip": 0.01094429, + "auxiliary_loss_mlp": 0.01050172, + "balance_loss_clip": 1.03565335, + "balance_loss_mlp": 1.03118193, + "epoch": 0.16621205965991526, + "flos": 16900371192960.0, + "grad_norm": 2.668471656624073, + "language_loss": 0.91714776, + "learning_rate": 3.808525695420427e-06, + "loss": 0.93859375, + "num_input_tokens_seen": 163751635, + "router_z_loss_clip": 0.58837891, + "router_z_loss_mlp": 0.18981934, + "step": 5728, + "time_per_iteration": 2.3444058895111084 + }, + { + "auxiliary_loss_clip": 0.01094253, + "auxiliary_loss_mlp": 0.01041713, + "balance_loss_clip": 1.03371787, + "balance_loss_mlp": 1.02345014, + "epoch": 0.1662410771284313, + "flos": 53897552461440.0, + "grad_norm": 2.4681787134021804, + "language_loss": 0.88426769, + "learning_rate": 3.8084454322349698e-06, + "loss": 0.90562737, + "num_input_tokens_seen": 163769320, + "router_z_loss_clip": 0.60571289, + "router_z_loss_mlp": 0.18249512, + "step": 5729, + "time_per_iteration": 2.898538827896118 + }, + { + "auxiliary_loss_clip": 0.01015806, + "auxiliary_loss_mlp": 0.01009329, + "balance_loss_clip": 1.00679946, + "balance_loss_mlp": 1.00798833, + "epoch": 0.16627009459694736, + "flos": 74772748632960.0, + "grad_norm": 0.6516084235482243, + "language_loss": 0.45553586, + "learning_rate": 3.80836515307659e-06, + "loss": 0.47578722, + "num_input_tokens_seen": 163833655, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01342773, + "step": 5730, + "time_per_iteration": 3.0667078495025635 + }, + { + "auxiliary_loss_clip": 0.0109159, + "auxiliary_loss_mlp": 0.01047393, + "balance_loss_clip": 1.03286517, + "balance_loss_mlp": 1.02811718, + "epoch": 0.1662991120654634, + "flos": 43537460394240.0, + "grad_norm": 2.2455981825344304, + "language_loss": 0.79224318, + "learning_rate": 3.808284857945998e-06, + "loss": 0.81363302, + "num_input_tokens_seen": 163855490, + "router_z_loss_clip": 0.58691406, + "router_z_loss_mlp": 0.19281006, + "step": 5731, + "time_per_iteration": 2.581514835357666 + }, + { + "auxiliary_loss_clip": 0.01096771, + "auxiliary_loss_mlp": 0.01045239, + "balance_loss_clip": 1.03552675, + "balance_loss_mlp": 1.02667856, + "epoch": 0.16632812953397946, + "flos": 11647478874240.0, + "grad_norm": 2.885988019545161, + "language_loss": 0.8726396, + "learning_rate": 3.8082045468439015e-06, + "loss": 0.89405966, + "num_input_tokens_seen": 163866505, + "router_z_loss_clip": 0.61230469, + "router_z_loss_mlp": 0.18566895, + "step": 5732, + "time_per_iteration": 2.399362325668335 + }, + { + "auxiliary_loss_clip": 0.01016286, + "auxiliary_loss_mlp": 0.01000737, + "balance_loss_clip": 1.00717497, + "balance_loss_mlp": 0.9995327, + "epoch": 0.16635714700249551, + "flos": 52499347666560.0, + "grad_norm": 0.6653492801255781, + "language_loss": 0.49503875, + "learning_rate": 3.808124219771011e-06, + "loss": 0.51520896, + "num_input_tokens_seen": 163926465, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01202393, + "step": 5733, + "time_per_iteration": 3.065969705581665 + }, + { + "auxiliary_loss_clip": 0.01089717, + "auxiliary_loss_mlp": 0.01041445, + "balance_loss_clip": 1.0316714, + "balance_loss_mlp": 1.02472019, + "epoch": 0.16638616447101154, + "flos": 22997114006400.0, + "grad_norm": 1.6225591416544776, + "language_loss": 0.6964432, + "learning_rate": 3.8080438767280364e-06, + "loss": 0.71775472, + "num_input_tokens_seen": 163949090, + "router_z_loss_clip": 0.58007812, + "router_z_loss_mlp": 0.16705322, + "step": 5734, + "time_per_iteration": 2.681565046310425 + }, + { + "auxiliary_loss_clip": 0.01014915, + "auxiliary_loss_mlp": 0.01001051, + "balance_loss_clip": 1.00558591, + "balance_loss_mlp": 0.99972761, + "epoch": 0.1664151819395276, + "flos": 66716155848960.0, + "grad_norm": 0.6043987633697624, + "language_loss": 0.49094704, + "learning_rate": 3.807963517715686e-06, + "loss": 0.51110673, + "num_input_tokens_seen": 164017345, + "router_z_loss_clip": 0.09326172, + "router_z_loss_mlp": 0.01324463, + "step": 5735, + "time_per_iteration": 3.0839200019836426 + }, + { + "auxiliary_loss_clip": 0.01089673, + "auxiliary_loss_mlp": 0.01047318, + "balance_loss_clip": 1.03123891, + "balance_loss_mlp": 1.02833378, + "epoch": 0.16644419940804364, + "flos": 49008455176320.0, + "grad_norm": 2.2353163584704316, + "language_loss": 0.96693134, + "learning_rate": 3.8078831427346707e-06, + "loss": 0.98830128, + "num_input_tokens_seen": 164039625, + "router_z_loss_clip": 0.58447266, + "router_z_loss_mlp": 0.19006348, + "step": 5736, + "time_per_iteration": 2.651337146759033 + }, + { + "auxiliary_loss_clip": 0.01088025, + "auxiliary_loss_mlp": 0.01045091, + "balance_loss_clip": 1.0281651, + "balance_loss_mlp": 1.02787733, + "epoch": 0.1664732168765597, + "flos": 36420555386880.0, + "grad_norm": 1.8461135430698177, + "language_loss": 0.8381663, + "learning_rate": 3.8078027517857e-06, + "loss": 0.85949743, + "num_input_tokens_seen": 164057405, + "router_z_loss_clip": 0.59912109, + "router_z_loss_mlp": 0.17205811, + "step": 5737, + "time_per_iteration": 2.5519537925720215 + }, + { + "auxiliary_loss_clip": 0.0108676, + "auxiliary_loss_mlp": 0.01060531, + "balance_loss_clip": 1.03019869, + "balance_loss_mlp": 1.04002142, + "epoch": 0.16650223434507574, + "flos": 20659329901440.0, + "grad_norm": 3.2422798224959712, + "language_loss": 0.90193045, + "learning_rate": 3.8077223448694833e-06, + "loss": 0.92340338, + "num_input_tokens_seen": 164070905, + "router_z_loss_clip": 0.56567383, + "router_z_loss_mlp": 0.2052002, + "step": 5738, + "time_per_iteration": 2.4417951107025146 + }, + { + "auxiliary_loss_clip": 0.01077763, + "auxiliary_loss_mlp": 0.01036888, + "balance_loss_clip": 1.02726483, + "balance_loss_mlp": 1.02261901, + "epoch": 0.16653125181359177, + "flos": 16537553677440.0, + "grad_norm": 2.0355971716044894, + "language_loss": 0.62288177, + "learning_rate": 3.8076419219867317e-06, + "loss": 0.64402831, + "num_input_tokens_seen": 164085130, + "router_z_loss_clip": 0.50537109, + "router_z_loss_mlp": 0.14276123, + "step": 5739, + "time_per_iteration": 2.346346855163574 + }, + { + "auxiliary_loss_clip": 0.01014471, + "auxiliary_loss_mlp": 0.01009383, + "balance_loss_clip": 1.00576806, + "balance_loss_mlp": 1.00813127, + "epoch": 0.16656026928210782, + "flos": 74775227339520.0, + "grad_norm": 0.6912817501493855, + "language_loss": 0.49448788, + "learning_rate": 3.807561483138155e-06, + "loss": 0.5147264, + "num_input_tokens_seen": 164152530, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.01251221, + "step": 5740, + "time_per_iteration": 3.1348984241485596 + }, + { + "auxiliary_loss_clip": 0.01085268, + "auxiliary_loss_mlp": 0.01042849, + "balance_loss_clip": 1.03057504, + "balance_loss_mlp": 1.02642775, + "epoch": 0.16658928675062387, + "flos": 10770774923520.0, + "grad_norm": 2.781048541090919, + "language_loss": 0.79382896, + "learning_rate": 3.8074810283244638e-06, + "loss": 0.81511009, + "num_input_tokens_seen": 164163090, + "router_z_loss_clip": 0.54638672, + "router_z_loss_mlp": 0.16442871, + "step": 5741, + "time_per_iteration": 2.533172845840454 + }, + { + "auxiliary_loss_clip": 0.01082264, + "auxiliary_loss_mlp": 0.01050764, + "balance_loss_clip": 1.0289216, + "balance_loss_mlp": 1.03341925, + "epoch": 0.16661830421913992, + "flos": 59005066412160.0, + "grad_norm": 2.4194433751140965, + "language_loss": 1.0229702, + "learning_rate": 3.8074005575463684e-06, + "loss": 1.04430044, + "num_input_tokens_seen": 164181850, + "router_z_loss_clip": 0.53320312, + "router_z_loss_mlp": 0.17333984, + "step": 5742, + "time_per_iteration": 2.8905396461486816 + }, + { + "auxiliary_loss_clip": 0.01090844, + "auxiliary_loss_mlp": 0.01039571, + "balance_loss_clip": 1.03241694, + "balance_loss_mlp": 1.02177858, + "epoch": 0.16664732168765597, + "flos": 28538284354560.0, + "grad_norm": 4.009498984116169, + "language_loss": 0.97641093, + "learning_rate": 3.8073200708045806e-06, + "loss": 0.99771506, + "num_input_tokens_seen": 164196085, + "router_z_loss_clip": 0.58374023, + "router_z_loss_mlp": 0.17785645, + "step": 5743, + "time_per_iteration": 2.4662821292877197 + }, + { + "auxiliary_loss_clip": 0.01013274, + "auxiliary_loss_mlp": 0.01004386, + "balance_loss_clip": 1.00501871, + "balance_loss_mlp": 1.0030328, + "epoch": 0.16667633915617203, + "flos": 73921811132160.0, + "grad_norm": 0.6663398426881042, + "language_loss": 0.47964674, + "learning_rate": 3.80723956809981e-06, + "loss": 0.49982333, + "num_input_tokens_seen": 164251340, + "router_z_loss_clip": 0.08251953, + "router_z_loss_mlp": 0.0135498, + "step": 5744, + "time_per_iteration": 2.9634673595428467 + }, + { + "auxiliary_loss_clip": 0.0109136, + "auxiliary_loss_mlp": 0.01039119, + "balance_loss_clip": 1.03003526, + "balance_loss_mlp": 1.01836455, + "epoch": 0.16670535662468805, + "flos": 35802836968320.0, + "grad_norm": 2.8668552113078323, + "language_loss": 1.0233053, + "learning_rate": 3.8071590494327683e-06, + "loss": 1.04461002, + "num_input_tokens_seen": 164267900, + "router_z_loss_clip": 0.61328125, + "router_z_loss_mlp": 0.20727539, + "step": 5745, + "time_per_iteration": 2.562903881072998 + }, + { + "auxiliary_loss_clip": 0.01092376, + "auxiliary_loss_mlp": 0.01044824, + "balance_loss_clip": 1.03117383, + "balance_loss_mlp": 1.02421284, + "epoch": 0.1667343740932041, + "flos": 25840405820160.0, + "grad_norm": 2.277593259858919, + "language_loss": 0.83979332, + "learning_rate": 3.807078514804167e-06, + "loss": 0.86116529, + "num_input_tokens_seen": 164283295, + "router_z_loss_clip": 0.61181641, + "router_z_loss_mlp": 0.20605469, + "step": 5746, + "time_per_iteration": 2.4495456218719482 + }, + { + "auxiliary_loss_clip": 0.01087177, + "auxiliary_loss_mlp": 0.0103909, + "balance_loss_clip": 1.02855635, + "balance_loss_mlp": 1.02054071, + "epoch": 0.16676339156172015, + "flos": 15990778874880.0, + "grad_norm": 3.125024656633712, + "language_loss": 0.97825438, + "learning_rate": 3.806997964214717e-06, + "loss": 0.99951702, + "num_input_tokens_seen": 164295815, + "router_z_loss_clip": 0.58618164, + "router_z_loss_mlp": 0.18554688, + "step": 5747, + "time_per_iteration": 2.4224612712860107 + }, + { + "auxiliary_loss_clip": 0.01087656, + "auxiliary_loss_mlp": 0.01039265, + "balance_loss_clip": 1.03243089, + "balance_loss_mlp": 1.02191377, + "epoch": 0.1667924090302362, + "flos": 20004498840960.0, + "grad_norm": 1.963843975365576, + "language_loss": 0.80404353, + "learning_rate": 3.8069173976651295e-06, + "loss": 0.82531267, + "num_input_tokens_seen": 164311750, + "router_z_loss_clip": 0.55224609, + "router_z_loss_mlp": 0.17340088, + "step": 5748, + "time_per_iteration": 2.393583297729492 + }, + { + "auxiliary_loss_clip": 0.01088128, + "auxiliary_loss_mlp": 0.01038155, + "balance_loss_clip": 1.03306651, + "balance_loss_mlp": 1.02143025, + "epoch": 0.16682142649875226, + "flos": 24561711941760.0, + "grad_norm": 2.27624022511719, + "language_loss": 0.87656808, + "learning_rate": 3.806836815156116e-06, + "loss": 0.89783096, + "num_input_tokens_seen": 164324380, + "router_z_loss_clip": 0.55078125, + "router_z_loss_mlp": 0.16705322, + "step": 5749, + "time_per_iteration": 2.5232694149017334 + }, + { + "auxiliary_loss_clip": 0.0108481, + "auxiliary_loss_mlp": 0.01033383, + "balance_loss_clip": 1.02915382, + "balance_loss_mlp": 1.01551366, + "epoch": 0.1668504439672683, + "flos": 39860267823360.0, + "grad_norm": 1.751252535600212, + "language_loss": 0.69730496, + "learning_rate": 3.806756216688389e-06, + "loss": 0.71848691, + "num_input_tokens_seen": 164344480, + "router_z_loss_clip": 0.55639648, + "router_z_loss_mlp": 0.17883301, + "step": 5750, + "time_per_iteration": 2.456519842147827 + }, + { + "auxiliary_loss_clip": 0.01092652, + "auxiliary_loss_mlp": 0.01044877, + "balance_loss_clip": 1.03233027, + "balance_loss_mlp": 1.02421761, + "epoch": 0.16687946143578433, + "flos": 32342210760960.0, + "grad_norm": 1.833082481845657, + "language_loss": 1.11378014, + "learning_rate": 3.8066756022626604e-06, + "loss": 1.13515544, + "num_input_tokens_seen": 164367315, + "router_z_loss_clip": 0.60400391, + "router_z_loss_mlp": 0.20654297, + "step": 5751, + "time_per_iteration": 2.5446763038635254 + }, + { + "auxiliary_loss_clip": 0.01086298, + "auxiliary_loss_mlp": 0.0103284, + "balance_loss_clip": 1.03166389, + "balance_loss_mlp": 1.01478589, + "epoch": 0.16690847890430038, + "flos": 11830807756800.0, + "grad_norm": 2.0836935393572062, + "language_loss": 0.75128698, + "learning_rate": 3.806594971879641e-06, + "loss": 0.77247834, + "num_input_tokens_seen": 164380495, + "router_z_loss_clip": 0.54638672, + "router_z_loss_mlp": 0.18035889, + "step": 5752, + "time_per_iteration": 2.3549394607543945 + }, + { + "auxiliary_loss_clip": 0.01091783, + "auxiliary_loss_mlp": 0.01038867, + "balance_loss_clip": 1.03248751, + "balance_loss_mlp": 1.01973391, + "epoch": 0.16693749637281644, + "flos": 36247385710080.0, + "grad_norm": 1.644756875047352, + "language_loss": 0.79236817, + "learning_rate": 3.806514325540044e-06, + "loss": 0.81367469, + "num_input_tokens_seen": 164402680, + "router_z_loss_clip": 0.59399414, + "router_z_loss_mlp": 0.19128418, + "step": 5753, + "time_per_iteration": 2.568737030029297 + }, + { + "auxiliary_loss_clip": 0.01015689, + "auxiliary_loss_mlp": 0.01006075, + "balance_loss_clip": 1.00704598, + "balance_loss_mlp": 1.0049957, + "epoch": 0.1669665138413325, + "flos": 60145990732800.0, + "grad_norm": 0.6484875067881246, + "language_loss": 0.46744904, + "learning_rate": 3.806433663244582e-06, + "loss": 0.48766667, + "num_input_tokens_seen": 164467445, + "router_z_loss_clip": 0.08642578, + "router_z_loss_mlp": 0.01080322, + "step": 5754, + "time_per_iteration": 3.1233367919921875 + }, + { + "auxiliary_loss_clip": 0.01090996, + "auxiliary_loss_mlp": 0.01038776, + "balance_loss_clip": 1.02980924, + "balance_loss_mlp": 1.0197742, + "epoch": 0.16699553130984854, + "flos": 36644557870080.0, + "grad_norm": 2.821961359210981, + "language_loss": 0.78533518, + "learning_rate": 3.8063529849939663e-06, + "loss": 0.80663288, + "num_input_tokens_seen": 164488160, + "router_z_loss_clip": 0.61230469, + "router_z_loss_mlp": 0.18994141, + "step": 5755, + "time_per_iteration": 2.6469480991363525 + }, + { + "auxiliary_loss_clip": 0.01087337, + "auxiliary_loss_mlp": 0.01039614, + "balance_loss_clip": 1.03268242, + "balance_loss_mlp": 1.02120197, + "epoch": 0.16702454877836456, + "flos": 16756808215680.0, + "grad_norm": 2.328471137023566, + "language_loss": 0.82134116, + "learning_rate": 3.80627229078891e-06, + "loss": 0.84261072, + "num_input_tokens_seen": 164501670, + "router_z_loss_clip": 0.54638672, + "router_z_loss_mlp": 0.18408203, + "step": 5756, + "time_per_iteration": 2.439953327178955 + }, + { + "auxiliary_loss_clip": 0.01095169, + "auxiliary_loss_mlp": 0.01043212, + "balance_loss_clip": 1.03319931, + "balance_loss_mlp": 1.02435899, + "epoch": 0.1670535662468806, + "flos": 47621111546880.0, + "grad_norm": 2.5299251128613665, + "language_loss": 0.89165616, + "learning_rate": 3.806191580630126e-06, + "loss": 0.91303992, + "num_input_tokens_seen": 164528155, + "router_z_loss_clip": 0.61914062, + "router_z_loss_mlp": 0.1885376, + "step": 5757, + "time_per_iteration": 2.6785972118377686 + }, + { + "auxiliary_loss_clip": 0.01095424, + "auxiliary_loss_mlp": 0.01051046, + "balance_loss_clip": 1.033319, + "balance_loss_mlp": 1.02945757, + "epoch": 0.16708258371539667, + "flos": 12342075840000.0, + "grad_norm": 2.7628006984654316, + "language_loss": 0.86944807, + "learning_rate": 3.8061108545183275e-06, + "loss": 0.89091277, + "num_input_tokens_seen": 164541700, + "router_z_loss_clip": 0.62255859, + "router_z_loss_mlp": 0.21594238, + "step": 5758, + "time_per_iteration": 2.3501718044281006 + }, + { + "auxiliary_loss_clip": 0.01084972, + "auxiliary_loss_mlp": 0.01032573, + "balance_loss_clip": 1.03034103, + "balance_loss_mlp": 1.01645005, + "epoch": 0.16711160118391272, + "flos": 34201929352320.0, + "grad_norm": 1.8138673308651934, + "language_loss": 0.82359999, + "learning_rate": 3.806030112454227e-06, + "loss": 0.84477544, + "num_input_tokens_seen": 164560520, + "router_z_loss_clip": 0.54638672, + "router_z_loss_mlp": 0.16119385, + "step": 5759, + "time_per_iteration": 2.534554958343506 + }, + { + "auxiliary_loss_clip": 0.0101655, + "auxiliary_loss_mlp": 0.01002597, + "balance_loss_clip": 1.00750732, + "balance_loss_mlp": 1.00155365, + "epoch": 0.16714061865242877, + "flos": 59006530823040.0, + "grad_norm": 0.6657887505540322, + "language_loss": 0.48575413, + "learning_rate": 3.8059493544385373e-06, + "loss": 0.50594556, + "num_input_tokens_seen": 164624595, + "router_z_loss_clip": 0.09033203, + "router_z_loss_mlp": 0.01043701, + "step": 5760, + "time_per_iteration": 3.14518666267395 + }, + { + "auxiliary_loss_clip": 0.0108452, + "auxiliary_loss_mlp": 0.01048729, + "balance_loss_clip": 1.03006089, + "balance_loss_mlp": 1.03161049, + "epoch": 0.16716963612094482, + "flos": 21572203887360.0, + "grad_norm": 2.0286876947402805, + "language_loss": 0.85306162, + "learning_rate": 3.805868580471973e-06, + "loss": 0.87439418, + "num_input_tokens_seen": 164638785, + "router_z_loss_clip": 0.54443359, + "router_z_loss_mlp": 0.17114258, + "step": 5761, + "time_per_iteration": 2.433192014694214 + }, + { + "auxiliary_loss_clip": 0.01081025, + "auxiliary_loss_mlp": 0.01045436, + "balance_loss_clip": 1.02717698, + "balance_loss_mlp": 1.02708936, + "epoch": 0.16719865358946084, + "flos": 32007149642880.0, + "grad_norm": 2.5334633950443672, + "language_loss": 0.66156787, + "learning_rate": 3.8057877905552454e-06, + "loss": 0.68283248, + "num_input_tokens_seen": 164658040, + "router_z_loss_clip": 0.53857422, + "router_z_loss_mlp": 0.18334961, + "step": 5762, + "time_per_iteration": 2.5064773559570312 + }, + { + "auxiliary_loss_clip": 0.01014457, + "auxiliary_loss_mlp": 0.01001625, + "balance_loss_clip": 1.00545096, + "balance_loss_mlp": 1.0005281, + "epoch": 0.1672276710579769, + "flos": 64806128121600.0, + "grad_norm": 0.6678772870676369, + "language_loss": 0.48448521, + "learning_rate": 3.8057069846890704e-06, + "loss": 0.504646, + "num_input_tokens_seen": 164712740, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01098633, + "step": 5763, + "time_per_iteration": 2.910562753677368 + }, + { + "auxiliary_loss_clip": 0.01075867, + "auxiliary_loss_mlp": 0.01039774, + "balance_loss_clip": 1.02638614, + "balance_loss_mlp": 1.02618384, + "epoch": 0.16725668852649295, + "flos": 11649364087680.0, + "grad_norm": 3.719021852769471, + "language_loss": 0.75739908, + "learning_rate": 3.8056261628741595e-06, + "loss": 0.77855551, + "num_input_tokens_seen": 164724520, + "router_z_loss_clip": 0.49511719, + "router_z_loss_mlp": 0.13592529, + "step": 5764, + "time_per_iteration": 2.3382318019866943 + }, + { + "auxiliary_loss_clip": 0.01086251, + "auxiliary_loss_mlp": 0.01036665, + "balance_loss_clip": 1.02908945, + "balance_loss_mlp": 1.02007163, + "epoch": 0.167285705995009, + "flos": 15369045649920.0, + "grad_norm": 3.0029578094788687, + "language_loss": 0.60377467, + "learning_rate": 3.8055453251112288e-06, + "loss": 0.62500387, + "num_input_tokens_seen": 164736580, + "router_z_loss_clip": 0.57128906, + "router_z_loss_mlp": 0.16589355, + "step": 5765, + "time_per_iteration": 2.398167610168457 + }, + { + "auxiliary_loss_clip": 0.01012408, + "auxiliary_loss_mlp": 0.01006481, + "balance_loss_clip": 1.00362527, + "balance_loss_mlp": 1.00528896, + "epoch": 0.16731472346352505, + "flos": 74767511928960.0, + "grad_norm": 0.627505370233294, + "language_loss": 0.48553687, + "learning_rate": 3.8054644714009907e-06, + "loss": 0.5057258, + "num_input_tokens_seen": 164800840, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01190186, + "step": 5766, + "time_per_iteration": 3.0904979705810547 + }, + { + "auxiliary_loss_clip": 0.01087061, + "auxiliary_loss_mlp": 0.01054283, + "balance_loss_clip": 1.02741122, + "balance_loss_mlp": 1.03610373, + "epoch": 0.1673437409320411, + "flos": 44815316400000.0, + "grad_norm": 1.9986044779648149, + "language_loss": 0.96997434, + "learning_rate": 3.8053836017441597e-06, + "loss": 0.99138772, + "num_input_tokens_seen": 164822490, + "router_z_loss_clip": 0.59765625, + "router_z_loss_mlp": 0.18151855, + "step": 5767, + "time_per_iteration": 2.593055009841919 + }, + { + "auxiliary_loss_clip": 0.01082553, + "auxiliary_loss_mlp": 0.01047799, + "balance_loss_clip": 1.02755249, + "balance_loss_mlp": 1.03098464, + "epoch": 0.16737275840055713, + "flos": 52509301136640.0, + "grad_norm": 2.704067811223228, + "language_loss": 0.98614687, + "learning_rate": 3.80530271614145e-06, + "loss": 1.00745046, + "num_input_tokens_seen": 164839300, + "router_z_loss_clip": 0.55053711, + "router_z_loss_mlp": 0.16821289, + "step": 5768, + "time_per_iteration": 2.6775200366973877 + }, + { + "auxiliary_loss_clip": 0.0108755, + "auxiliary_loss_mlp": 0.01040989, + "balance_loss_clip": 1.02990401, + "balance_loss_mlp": 1.02398419, + "epoch": 0.16740177586907318, + "flos": 38175778679040.0, + "grad_norm": 1.8730993277132366, + "language_loss": 0.96025813, + "learning_rate": 3.8052218145935767e-06, + "loss": 0.98154354, + "num_input_tokens_seen": 164870590, + "router_z_loss_clip": 0.57641602, + "router_z_loss_mlp": 0.17004395, + "step": 5769, + "time_per_iteration": 2.866255044937134 + }, + { + "auxiliary_loss_clip": 0.0101173, + "auxiliary_loss_mlp": 0.01008115, + "balance_loss_clip": 1.00262547, + "balance_loss_mlp": 1.00692296, + "epoch": 0.16743079333758923, + "flos": 67260417033600.0, + "grad_norm": 0.7032737712019526, + "language_loss": 0.49432388, + "learning_rate": 3.8051408971012533e-06, + "loss": 0.51452231, + "num_input_tokens_seen": 164935405, + "router_z_loss_clip": 0.09082031, + "router_z_loss_mlp": 0.01190186, + "step": 5770, + "time_per_iteration": 3.0948145389556885 + }, + { + "auxiliary_loss_clip": 0.01081114, + "auxiliary_loss_mlp": 0.01042021, + "balance_loss_clip": 1.02759767, + "balance_loss_mlp": 1.02520084, + "epoch": 0.16745981080610528, + "flos": 14276229183360.0, + "grad_norm": 2.5101594253313393, + "language_loss": 0.79669774, + "learning_rate": 3.8050599636651952e-06, + "loss": 0.81792909, + "num_input_tokens_seen": 164949055, + "router_z_loss_clip": 0.53540039, + "router_z_loss_mlp": 0.16827393, + "step": 5771, + "time_per_iteration": 2.358210563659668 + }, + { + "auxiliary_loss_clip": 0.01011955, + "auxiliary_loss_mlp": 0.01004287, + "balance_loss_clip": 1.00302362, + "balance_loss_mlp": 1.00304127, + "epoch": 0.16748882827462133, + "flos": 61081279678080.0, + "grad_norm": 0.6402447133530366, + "language_loss": 0.4775455, + "learning_rate": 3.8049790142861175e-06, + "loss": 0.49770793, + "num_input_tokens_seen": 165008915, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.01245117, + "step": 5772, + "time_per_iteration": 2.952284097671509 + }, + { + "auxiliary_loss_clip": 0.0108728, + "auxiliary_loss_mlp": 0.01045762, + "balance_loss_clip": 1.03210425, + "balance_loss_mlp": 1.02916193, + "epoch": 0.16751784574313736, + "flos": 11283928220160.0, + "grad_norm": 3.474885749940699, + "language_loss": 0.93563116, + "learning_rate": 3.804898048964734e-06, + "loss": 0.95696157, + "num_input_tokens_seen": 165020825, + "router_z_loss_clip": 0.55224609, + "router_z_loss_mlp": 0.16589355, + "step": 5773, + "time_per_iteration": 2.3301796913146973 + }, + { + "auxiliary_loss_clip": 0.01082308, + "auxiliary_loss_mlp": 0.01032737, + "balance_loss_clip": 1.03076577, + "balance_loss_mlp": 1.01748967, + "epoch": 0.1675468632116534, + "flos": 10115071079040.0, + "grad_norm": 2.5116350712867126, + "language_loss": 0.79257464, + "learning_rate": 3.8048170677017615e-06, + "loss": 0.81372511, + "num_input_tokens_seen": 165031875, + "router_z_loss_clip": 0.51513672, + "router_z_loss_mlp": 0.15246582, + "step": 5774, + "time_per_iteration": 2.4056804180145264 + }, + { + "auxiliary_loss_clip": 0.01082927, + "auxiliary_loss_mlp": 0.01032524, + "balance_loss_clip": 1.03121841, + "balance_loss_mlp": 1.01823688, + "epoch": 0.16757588068016946, + "flos": 27666537816960.0, + "grad_norm": 2.3974738745196316, + "language_loss": 0.66029179, + "learning_rate": 3.8047360704979134e-06, + "loss": 0.68144631, + "num_input_tokens_seen": 165047685, + "router_z_loss_clip": 0.51708984, + "router_z_loss_mlp": 0.14294434, + "step": 5775, + "time_per_iteration": 2.4284369945526123 + }, + { + "auxiliary_loss_clip": 0.01085394, + "auxiliary_loss_mlp": 0.01028514, + "balance_loss_clip": 1.03144789, + "balance_loss_mlp": 1.01189566, + "epoch": 0.1676048981486855, + "flos": 20922190594560.0, + "grad_norm": 2.1147447389237803, + "language_loss": 0.96114331, + "learning_rate": 3.8046550573539066e-06, + "loss": 0.9822824, + "num_input_tokens_seen": 165064700, + "router_z_loss_clip": 0.53881836, + "router_z_loss_mlp": 0.16625977, + "step": 5776, + "time_per_iteration": 2.4584810733795166 + }, + { + "auxiliary_loss_clip": 0.01083097, + "auxiliary_loss_mlp": 0.01027763, + "balance_loss_clip": 1.03121221, + "balance_loss_mlp": 1.01262903, + "epoch": 0.16763391561720156, + "flos": 23177091398400.0, + "grad_norm": 6.070430918887086, + "language_loss": 0.66402423, + "learning_rate": 3.8045740282704557e-06, + "loss": 0.68513286, + "num_input_tokens_seen": 165081715, + "router_z_loss_clip": 0.51855469, + "router_z_loss_mlp": 0.15124512, + "step": 5777, + "time_per_iteration": 2.4002509117126465 + }, + { + "auxiliary_loss_clip": 0.01087458, + "auxiliary_loss_mlp": 0.01030345, + "balance_loss_clip": 1.03264666, + "balance_loss_mlp": 1.01465118, + "epoch": 0.1676629330857176, + "flos": 17346072009600.0, + "grad_norm": 1.8895307659359788, + "language_loss": 0.70399946, + "learning_rate": 3.804492983248277e-06, + "loss": 0.72517753, + "num_input_tokens_seen": 165097375, + "router_z_loss_clip": 0.5480957, + "router_z_loss_mlp": 0.15686035, + "step": 5778, + "time_per_iteration": 2.439761161804199 + }, + { + "auxiliary_loss_clip": 0.01088816, + "auxiliary_loss_mlp": 0.01034473, + "balance_loss_clip": 1.03295028, + "balance_loss_mlp": 1.01739049, + "epoch": 0.16769195055423364, + "flos": 10771054214400.0, + "grad_norm": 4.116037026043713, + "language_loss": 0.9090693, + "learning_rate": 3.804411922288086e-06, + "loss": 0.93030226, + "num_input_tokens_seen": 165107310, + "router_z_loss_clip": 0.55883789, + "router_z_loss_mlp": 0.17089844, + "step": 5779, + "time_per_iteration": 2.3321478366851807 + }, + { + "auxiliary_loss_clip": 0.01091114, + "auxiliary_loss_mlp": 0.01041907, + "balance_loss_clip": 1.03426242, + "balance_loss_mlp": 1.02327478, + "epoch": 0.1677209680227497, + "flos": 33321943733760.0, + "grad_norm": 1.8560583621682674, + "language_loss": 0.87905109, + "learning_rate": 3.8043308453905984e-06, + "loss": 0.90038127, + "num_input_tokens_seen": 165126455, + "router_z_loss_clip": 0.56933594, + "router_z_loss_mlp": 0.1862793, + "step": 5780, + "time_per_iteration": 2.516725540161133 + }, + { + "auxiliary_loss_clip": 0.01022234, + "auxiliary_loss_mlp": 0.01002279, + "balance_loss_clip": 1.01195693, + "balance_loss_mlp": 1.00114107, + "epoch": 0.16774998549126574, + "flos": 60789196310400.0, + "grad_norm": 0.6606196405955925, + "language_loss": 0.46495789, + "learning_rate": 3.804249752556531e-06, + "loss": 0.48520303, + "num_input_tokens_seen": 165182925, + "router_z_loss_clip": 0.10253906, + "router_z_loss_mlp": 0.01141357, + "step": 5781, + "time_per_iteration": 2.845734119415283 + }, + { + "auxiliary_loss_clip": 0.01093657, + "auxiliary_loss_mlp": 0.01027247, + "balance_loss_clip": 1.03545451, + "balance_loss_mlp": 1.00994956, + "epoch": 0.1677790029597818, + "flos": 44594176648320.0, + "grad_norm": 2.2132048563559317, + "language_loss": 0.74740767, + "learning_rate": 3.8041686437865995e-06, + "loss": 0.7686168, + "num_input_tokens_seen": 165200070, + "router_z_loss_clip": 0.58178711, + "router_z_loss_mlp": 0.17297363, + "step": 5782, + "time_per_iteration": 4.969680070877075 + }, + { + "auxiliary_loss_clip": 0.01089653, + "auxiliary_loss_mlp": 0.01044329, + "balance_loss_clip": 1.0326786, + "balance_loss_mlp": 1.02655506, + "epoch": 0.16780802042829784, + "flos": 32370874853760.0, + "grad_norm": 4.316664397400445, + "language_loss": 0.79616344, + "learning_rate": 3.804087519081521e-06, + "loss": 0.81750321, + "num_input_tokens_seen": 165215085, + "router_z_loss_clip": 0.56884766, + "router_z_loss_mlp": 0.17773438, + "step": 5783, + "time_per_iteration": 4.707519054412842 + }, + { + "auxiliary_loss_clip": 0.01093723, + "auxiliary_loss_mlp": 0.01039341, + "balance_loss_clip": 1.03463089, + "balance_loss_mlp": 1.02074385, + "epoch": 0.1678370378968139, + "flos": 18192994704000.0, + "grad_norm": 3.3085543189206494, + "language_loss": 0.83600771, + "learning_rate": 3.804006378442011e-06, + "loss": 0.85733831, + "num_input_tokens_seen": 165228615, + "router_z_loss_clip": 0.59057617, + "router_z_loss_mlp": 0.18609619, + "step": 5784, + "time_per_iteration": 2.4078705310821533 + }, + { + "auxiliary_loss_clip": 0.01088499, + "auxiliary_loss_mlp": 0.01034899, + "balance_loss_clip": 1.03039312, + "balance_loss_mlp": 1.01758993, + "epoch": 0.16786605536532992, + "flos": 74748415415040.0, + "grad_norm": 2.5968641091829445, + "language_loss": 0.74155706, + "learning_rate": 3.803925221868787e-06, + "loss": 0.76279104, + "num_input_tokens_seen": 165254750, + "router_z_loss_clip": 0.58056641, + "router_z_loss_mlp": 0.17321777, + "step": 5785, + "time_per_iteration": 2.816746950149536 + }, + { + "auxiliary_loss_clip": 0.01018284, + "auxiliary_loss_mlp": 0.01000571, + "balance_loss_clip": 1.0087198, + "balance_loss_mlp": 0.99953419, + "epoch": 0.16789507283384597, + "flos": 63357650968320.0, + "grad_norm": 0.7036510250727959, + "language_loss": 0.48704892, + "learning_rate": 3.803844049362565e-06, + "loss": 0.50723749, + "num_input_tokens_seen": 165320525, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01037598, + "step": 5786, + "time_per_iteration": 3.1571218967437744 + }, + { + "auxiliary_loss_clip": 0.01093912, + "auxiliary_loss_mlp": 0.01040087, + "balance_loss_clip": 1.03121865, + "balance_loss_mlp": 1.01996398, + "epoch": 0.16792409030236202, + "flos": 29672158446720.0, + "grad_norm": 2.2313275809586552, + "language_loss": 0.96045518, + "learning_rate": 3.803762860924063e-06, + "loss": 0.98179519, + "num_input_tokens_seen": 165338780, + "router_z_loss_clip": 0.62695312, + "router_z_loss_mlp": 0.20141602, + "step": 5787, + "time_per_iteration": 2.444936990737915 + }, + { + "auxiliary_loss_clip": 0.01085179, + "auxiliary_loss_mlp": 0.01037986, + "balance_loss_clip": 1.02953291, + "balance_loss_mlp": 1.02077258, + "epoch": 0.16795310777087807, + "flos": 16281710167680.0, + "grad_norm": 2.0353280781863528, + "language_loss": 0.7522831, + "learning_rate": 3.803681656553997e-06, + "loss": 0.77351475, + "num_input_tokens_seen": 165354045, + "router_z_loss_clip": 0.55639648, + "router_z_loss_mlp": 0.17205811, + "step": 5788, + "time_per_iteration": 2.4184486865997314 + }, + { + "auxiliary_loss_clip": 0.01014982, + "auxiliary_loss_mlp": 0.01000318, + "balance_loss_clip": 1.00595415, + "balance_loss_mlp": 0.99912572, + "epoch": 0.16798212523939413, + "flos": 68423548711680.0, + "grad_norm": 0.6270231805172711, + "language_loss": 0.48144227, + "learning_rate": 3.8036004362530847e-06, + "loss": 0.50159526, + "num_input_tokens_seen": 165418350, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01190186, + "step": 5789, + "time_per_iteration": 3.133805751800537 + }, + { + "auxiliary_loss_clip": 0.010885, + "auxiliary_loss_mlp": 0.01039146, + "balance_loss_clip": 1.02781808, + "balance_loss_mlp": 1.01892853, + "epoch": 0.16801114270791015, + "flos": 28175676307200.0, + "grad_norm": 2.5297402422547592, + "language_loss": 0.81489426, + "learning_rate": 3.803519200022044e-06, + "loss": 0.83617067, + "num_input_tokens_seen": 165434475, + "router_z_loss_clip": 0.60644531, + "router_z_loss_mlp": 0.20214844, + "step": 5790, + "time_per_iteration": 2.454083204269409 + }, + { + "auxiliary_loss_clip": 0.0108496, + "auxiliary_loss_mlp": 0.01034323, + "balance_loss_clip": 1.02893019, + "balance_loss_mlp": 1.0179491, + "epoch": 0.1680401601764262, + "flos": 16354818288000.0, + "grad_norm": 2.798960424998099, + "language_loss": 0.52650714, + "learning_rate": 3.8034379478615913e-06, + "loss": 0.54769993, + "num_input_tokens_seen": 165447440, + "router_z_loss_clip": 0.56005859, + "router_z_loss_mlp": 0.16369629, + "step": 5791, + "time_per_iteration": 2.416616439819336 + }, + { + "auxiliary_loss_clip": 0.01081021, + "auxiliary_loss_mlp": 0.01040429, + "balance_loss_clip": 1.02746236, + "balance_loss_mlp": 1.02254152, + "epoch": 0.16806917764494225, + "flos": 26204445233280.0, + "grad_norm": 2.0823125994531813, + "language_loss": 0.95610917, + "learning_rate": 3.8033566797724453e-06, + "loss": 0.97732359, + "num_input_tokens_seen": 165463330, + "router_z_loss_clip": 0.53637695, + "router_z_loss_mlp": 0.17895508, + "step": 5792, + "time_per_iteration": 7.5712034702301025 + }, + { + "auxiliary_loss_clip": 0.01081449, + "auxiliary_loss_mlp": 0.01043355, + "balance_loss_clip": 1.02699804, + "balance_loss_mlp": 1.02502704, + "epoch": 0.1680981951134583, + "flos": 30656534630400.0, + "grad_norm": 1.9556981018280728, + "language_loss": 0.64264274, + "learning_rate": 3.8032753957553233e-06, + "loss": 0.66389084, + "num_input_tokens_seen": 165482380, + "router_z_loss_clip": 0.54443359, + "router_z_loss_mlp": 0.18347168, + "step": 5793, + "time_per_iteration": 2.536806106567383 + }, + { + "auxiliary_loss_clip": 0.01084061, + "auxiliary_loss_mlp": 0.01039701, + "balance_loss_clip": 1.0270555, + "balance_loss_mlp": 1.02212977, + "epoch": 0.16812721258197436, + "flos": 19345164215040.0, + "grad_norm": 2.994449979732788, + "language_loss": 0.90401453, + "learning_rate": 3.8031940958109436e-06, + "loss": 0.9252522, + "num_input_tokens_seen": 165493520, + "router_z_loss_clip": 0.57080078, + "router_z_loss_mlp": 0.17578125, + "step": 5794, + "time_per_iteration": 2.3669819831848145 + }, + { + "auxiliary_loss_clip": 0.01083008, + "auxiliary_loss_mlp": 0.01037503, + "balance_loss_clip": 1.02775431, + "balance_loss_mlp": 1.02099836, + "epoch": 0.1681562300504904, + "flos": 34268753427840.0, + "grad_norm": 2.016448948875545, + "language_loss": 1.04685879, + "learning_rate": 3.803112779940023e-06, + "loss": 1.06806386, + "num_input_tokens_seen": 165510885, + "router_z_loss_clip": 0.55249023, + "router_z_loss_mlp": 0.16503906, + "step": 5795, + "time_per_iteration": 2.465362787246704 + }, + { + "auxiliary_loss_clip": 0.01090054, + "auxiliary_loss_mlp": 0.01038313, + "balance_loss_clip": 1.028826, + "balance_loss_mlp": 1.02000213, + "epoch": 0.16818524751900643, + "flos": 37515117421440.0, + "grad_norm": 2.690609503685327, + "language_loss": 0.93776143, + "learning_rate": 3.8030314481432815e-06, + "loss": 0.95904505, + "num_input_tokens_seen": 165527090, + "router_z_loss_clip": 0.61254883, + "router_z_loss_mlp": 0.18322754, + "step": 5796, + "time_per_iteration": 2.4521167278289795 + }, + { + "auxiliary_loss_clip": 0.01087173, + "auxiliary_loss_mlp": 0.01038658, + "balance_loss_clip": 1.02961922, + "balance_loss_mlp": 1.02146769, + "epoch": 0.16821426498752248, + "flos": 25630647171840.0, + "grad_norm": 2.158143472820507, + "language_loss": 1.1022439, + "learning_rate": 3.8029501004214363e-06, + "loss": 1.12350225, + "num_input_tokens_seen": 165548625, + "router_z_loss_clip": 0.57641602, + "router_z_loss_mlp": 0.171875, + "step": 5797, + "time_per_iteration": 2.447521448135376 + }, + { + "auxiliary_loss_clip": 0.01085516, + "auxiliary_loss_mlp": 0.01033999, + "balance_loss_clip": 1.02954483, + "balance_loss_mlp": 1.01610589, + "epoch": 0.16824328245603853, + "flos": 16246657296000.0, + "grad_norm": 2.513586122961207, + "language_loss": 0.7453475, + "learning_rate": 3.8028687367752064e-06, + "loss": 0.76654267, + "num_input_tokens_seen": 165562355, + "router_z_loss_clip": 0.55957031, + "router_z_loss_mlp": 0.1786499, + "step": 5798, + "time_per_iteration": 2.373420238494873 + }, + { + "auxiliary_loss_clip": 0.01014121, + "auxiliary_loss_mlp": 0.01000825, + "balance_loss_clip": 1.00545239, + "balance_loss_mlp": 0.99958569, + "epoch": 0.16827229992455459, + "flos": 60438632682240.0, + "grad_norm": 0.6314297206670892, + "language_loss": 0.44869027, + "learning_rate": 3.8027873572053106e-06, + "loss": 0.46883973, + "num_input_tokens_seen": 165629105, + "router_z_loss_clip": 0.08642578, + "router_z_loss_mlp": 0.01239014, + "step": 5799, + "time_per_iteration": 3.1512205600738525 + }, + { + "auxiliary_loss_clip": 0.01014996, + "auxiliary_loss_mlp": 0.01000606, + "balance_loss_clip": 1.00621378, + "balance_loss_mlp": 0.99946749, + "epoch": 0.16830131739307064, + "flos": 63245998840320.0, + "grad_norm": 0.6177357174744069, + "language_loss": 0.47758588, + "learning_rate": 3.8027059617124673e-06, + "loss": 0.49774191, + "num_input_tokens_seen": 165692505, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01141357, + "step": 5800, + "time_per_iteration": 3.0809948444366455 + }, + { + "auxiliary_loss_clip": 0.01088758, + "auxiliary_loss_mlp": 0.0104107, + "balance_loss_clip": 1.03097129, + "balance_loss_mlp": 1.02202034, + "epoch": 0.1683303348615867, + "flos": 12199874405760.0, + "grad_norm": 2.436968038424556, + "language_loss": 0.7367394, + "learning_rate": 3.8026245502973947e-06, + "loss": 0.75803769, + "num_input_tokens_seen": 165705290, + "router_z_loss_clip": 0.578125, + "router_z_loss_mlp": 0.19055176, + "step": 5801, + "time_per_iteration": 2.4716155529022217 + }, + { + "auxiliary_loss_clip": 0.01014526, + "auxiliary_loss_mlp": 0.01002785, + "balance_loss_clip": 1.00582409, + "balance_loss_mlp": 1.00172377, + "epoch": 0.1683593523301027, + "flos": 74769397142400.0, + "grad_norm": 0.6404153174946742, + "language_loss": 0.44693011, + "learning_rate": 3.8025431229608127e-06, + "loss": 0.46710321, + "num_input_tokens_seen": 165771815, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.01062012, + "step": 5802, + "time_per_iteration": 3.085287094116211 + }, + { + "auxiliary_loss_clip": 0.01079315, + "auxiliary_loss_mlp": 0.01028315, + "balance_loss_clip": 1.02900732, + "balance_loss_mlp": 1.01421881, + "epoch": 0.16838836979861876, + "flos": 29890994048640.0, + "grad_norm": 2.589062297994071, + "language_loss": 0.79476875, + "learning_rate": 3.8024616797034414e-06, + "loss": 0.81584507, + "num_input_tokens_seen": 165785130, + "router_z_loss_clip": 0.50268555, + "router_z_loss_mlp": 0.14099121, + "step": 5803, + "time_per_iteration": 2.5043787956237793 + }, + { + "auxiliary_loss_clip": 0.01087891, + "auxiliary_loss_mlp": 0.01036564, + "balance_loss_clip": 1.03308737, + "balance_loss_mlp": 1.02026749, + "epoch": 0.16841738726713482, + "flos": 29855347683840.0, + "grad_norm": 1.8075890593694357, + "language_loss": 0.75329006, + "learning_rate": 3.8023802205259986e-06, + "loss": 0.77453458, + "num_input_tokens_seen": 165802470, + "router_z_loss_clip": 0.5480957, + "router_z_loss_mlp": 0.16290283, + "step": 5804, + "time_per_iteration": 2.487773895263672 + }, + { + "auxiliary_loss_clip": 0.01097339, + "auxiliary_loss_mlp": 0.01042624, + "balance_loss_clip": 1.03226304, + "balance_loss_mlp": 1.02169085, + "epoch": 0.16844640473565087, + "flos": 18220925658240.0, + "grad_norm": 2.3403044608779697, + "language_loss": 0.82694817, + "learning_rate": 3.8022987454292043e-06, + "loss": 0.84834778, + "num_input_tokens_seen": 165821490, + "router_z_loss_clip": 0.65039062, + "router_z_loss_mlp": 0.20935059, + "step": 5805, + "time_per_iteration": 2.6794745922088623 + }, + { + "auxiliary_loss_clip": 0.01096128, + "auxiliary_loss_mlp": 0.01052986, + "balance_loss_clip": 1.03019619, + "balance_loss_mlp": 1.03100431, + "epoch": 0.16847542220416692, + "flos": 13106010499200.0, + "grad_norm": 3.1484968374517144, + "language_loss": 0.9654507, + "learning_rate": 3.8022172544137785e-06, + "loss": 0.98694181, + "num_input_tokens_seen": 165831950, + "router_z_loss_clip": 0.65917969, + "router_z_loss_mlp": 0.22009277, + "step": 5806, + "time_per_iteration": 2.3651294708251953 + }, + { + "auxiliary_loss_clip": 0.01092239, + "auxiliary_loss_mlp": 0.01031814, + "balance_loss_clip": 1.02858508, + "balance_loss_mlp": 1.01251435, + "epoch": 0.16850443967268294, + "flos": 13582260622080.0, + "grad_norm": 2.004451622523034, + "language_loss": 0.7129575, + "learning_rate": 3.80213574748044e-06, + "loss": 0.73419803, + "num_input_tokens_seen": 165846945, + "router_z_loss_clip": 0.63720703, + "router_z_loss_mlp": 0.19287109, + "step": 5807, + "time_per_iteration": 2.4027881622314453 + }, + { + "auxiliary_loss_clip": 0.01090158, + "auxiliary_loss_mlp": 0.01040827, + "balance_loss_clip": 1.03198993, + "balance_loss_mlp": 1.02322555, + "epoch": 0.168533457141199, + "flos": 11977338199680.0, + "grad_norm": 4.134005493099177, + "language_loss": 1.01679599, + "learning_rate": 3.8020542246299096e-06, + "loss": 1.03810596, + "num_input_tokens_seen": 165858120, + "router_z_loss_clip": 0.58178711, + "router_z_loss_mlp": 0.17614746, + "step": 5808, + "time_per_iteration": 2.3457584381103516 + }, + { + "auxiliary_loss_clip": 0.01013617, + "auxiliary_loss_mlp": 0.01001489, + "balance_loss_clip": 1.00516772, + "balance_loss_mlp": 1.00045145, + "epoch": 0.16856247460971505, + "flos": 69402792925440.0, + "grad_norm": 0.7356705590615072, + "language_loss": 0.49483174, + "learning_rate": 3.8019726858629073e-06, + "loss": 0.51498282, + "num_input_tokens_seen": 165915995, + "router_z_loss_clip": 0.08447266, + "router_z_loss_mlp": 0.01037598, + "step": 5809, + "time_per_iteration": 2.959120988845825 + }, + { + "auxiliary_loss_clip": 0.01087241, + "auxiliary_loss_mlp": 0.01040573, + "balance_loss_clip": 1.03022885, + "balance_loss_mlp": 1.02331209, + "epoch": 0.1685914920782311, + "flos": 28905989460480.0, + "grad_norm": 2.978660880458471, + "language_loss": 0.96956843, + "learning_rate": 3.801891131180153e-06, + "loss": 0.99084657, + "num_input_tokens_seen": 165929190, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.17272949, + "step": 5810, + "time_per_iteration": 2.4857499599456787 + }, + { + "auxiliary_loss_clip": 0.01088094, + "auxiliary_loss_mlp": 0.01038617, + "balance_loss_clip": 1.03050733, + "balance_loss_mlp": 1.0194602, + "epoch": 0.16862050954674715, + "flos": 26425375516800.0, + "grad_norm": 2.2608053536359867, + "language_loss": 0.78117913, + "learning_rate": 3.8018095605823666e-06, + "loss": 0.80244625, + "num_input_tokens_seen": 165944625, + "router_z_loss_clip": 0.57568359, + "router_z_loss_mlp": 0.19165039, + "step": 5811, + "time_per_iteration": 2.4523351192474365 + }, + { + "auxiliary_loss_clip": 0.01082737, + "auxiliary_loss_mlp": 0.01035702, + "balance_loss_clip": 1.02850771, + "balance_loss_mlp": 1.01981091, + "epoch": 0.1686495270152632, + "flos": 15452662089600.0, + "grad_norm": 3.110972034592692, + "language_loss": 0.75674003, + "learning_rate": 3.80172797407027e-06, + "loss": 0.77792442, + "num_input_tokens_seen": 165959700, + "router_z_loss_clip": 0.54199219, + "router_z_loss_mlp": 0.15887451, + "step": 5812, + "time_per_iteration": 2.3472297191619873 + }, + { + "auxiliary_loss_clip": 0.01078885, + "auxiliary_loss_mlp": 0.01039242, + "balance_loss_clip": 1.0272249, + "balance_loss_mlp": 1.02227306, + "epoch": 0.16867854448377922, + "flos": 30511435553280.0, + "grad_norm": 1.9568264540683082, + "language_loss": 0.85348463, + "learning_rate": 3.801646371644582e-06, + "loss": 0.87466586, + "num_input_tokens_seen": 165979685, + "router_z_loss_clip": 0.51660156, + "router_z_loss_mlp": 0.1696167, + "step": 5813, + "time_per_iteration": 2.5482757091522217 + }, + { + "auxiliary_loss_clip": 0.01078437, + "auxiliary_loss_mlp": 0.01033826, + "balance_loss_clip": 1.02636981, + "balance_loss_mlp": 1.01816225, + "epoch": 0.16870756195229528, + "flos": 37772462119680.0, + "grad_norm": 6.264218384159602, + "language_loss": 0.85858274, + "learning_rate": 3.8015647533060246e-06, + "loss": 0.87970531, + "num_input_tokens_seen": 166001715, + "router_z_loss_clip": 0.52001953, + "router_z_loss_mlp": 0.15667725, + "step": 5814, + "time_per_iteration": 2.5219733715057373 + }, + { + "auxiliary_loss_clip": 0.01084046, + "auxiliary_loss_mlp": 0.0103663, + "balance_loss_clip": 1.02727151, + "balance_loss_mlp": 1.01837349, + "epoch": 0.16873657942081133, + "flos": 35254700622720.0, + "grad_norm": 1.7495669846508735, + "language_loss": 0.84995854, + "learning_rate": 3.8014831190553182e-06, + "loss": 0.87116528, + "num_input_tokens_seen": 166022175, + "router_z_loss_clip": 0.56835938, + "router_z_loss_mlp": 0.18249512, + "step": 5815, + "time_per_iteration": 2.599551200866699 + }, + { + "auxiliary_loss_clip": 0.01088676, + "auxiliary_loss_mlp": 0.01039324, + "balance_loss_clip": 1.02866316, + "balance_loss_mlp": 1.02108502, + "epoch": 0.16876559688932738, + "flos": 28545126981120.0, + "grad_norm": 1.7838266850822708, + "language_loss": 0.84714925, + "learning_rate": 3.801401468893184e-06, + "loss": 0.8684293, + "num_input_tokens_seen": 166043360, + "router_z_loss_clip": 0.60009766, + "router_z_loss_mlp": 0.18237305, + "step": 5816, + "time_per_iteration": 2.4949581623077393 + }, + { + "auxiliary_loss_clip": 0.01078086, + "auxiliary_loss_mlp": 0.010365, + "balance_loss_clip": 1.02697182, + "balance_loss_mlp": 1.02174807, + "epoch": 0.16879461435784343, + "flos": 33430872775680.0, + "grad_norm": 1.9187122940594707, + "language_loss": 0.69444466, + "learning_rate": 3.801319802820343e-06, + "loss": 0.71559048, + "num_input_tokens_seen": 166066215, + "router_z_loss_clip": 0.51147461, + "router_z_loss_mlp": 0.14746094, + "step": 5817, + "time_per_iteration": 2.6802878379821777 + }, + { + "auxiliary_loss_clip": 0.01010834, + "auxiliary_loss_mlp": 0.01001193, + "balance_loss_clip": 1.00197971, + "balance_loss_mlp": 1.00001299, + "epoch": 0.16882363182635945, + "flos": 53606968416000.0, + "grad_norm": 0.6636597095345942, + "language_loss": 0.48507452, + "learning_rate": 3.8012381208375165e-06, + "loss": 0.50519478, + "num_input_tokens_seen": 166124090, + "router_z_loss_clip": 0.08837891, + "router_z_loss_mlp": 0.01177979, + "step": 5818, + "time_per_iteration": 2.872133493423462 + }, + { + "auxiliary_loss_clip": 0.01086839, + "auxiliary_loss_mlp": 0.01037213, + "balance_loss_clip": 1.02891946, + "balance_loss_mlp": 1.01900971, + "epoch": 0.1688526492948755, + "flos": 32481095616000.0, + "grad_norm": 2.1110740383240887, + "language_loss": 0.75962782, + "learning_rate": 3.801156422945426e-06, + "loss": 0.78086841, + "num_input_tokens_seen": 166142885, + "router_z_loss_clip": 0.57836914, + "router_z_loss_mlp": 0.18212891, + "step": 5819, + "time_per_iteration": 2.680044174194336 + }, + { + "auxiliary_loss_clip": 0.01089114, + "auxiliary_loss_mlp": 0.01041716, + "balance_loss_clip": 1.03026056, + "balance_loss_mlp": 1.02289867, + "epoch": 0.16888166676339156, + "flos": 28401075244800.0, + "grad_norm": 2.430907476226409, + "language_loss": 1.07580853, + "learning_rate": 3.8010747091447926e-06, + "loss": 1.09711683, + "num_input_tokens_seen": 166160600, + "router_z_loss_clip": 0.58886719, + "router_z_loss_mlp": 0.18829346, + "step": 5820, + "time_per_iteration": 2.4494221210479736 + }, + { + "auxiliary_loss_clip": 0.01081892, + "auxiliary_loss_mlp": 0.01033712, + "balance_loss_clip": 1.02540755, + "balance_loss_mlp": 1.01544881, + "epoch": 0.1689106842319076, + "flos": 21249466479360.0, + "grad_norm": 1.9319256960652236, + "language_loss": 0.79557598, + "learning_rate": 3.8009929794363394e-06, + "loss": 0.81673193, + "num_input_tokens_seen": 166177365, + "router_z_loss_clip": 0.56445312, + "router_z_loss_mlp": 0.18286133, + "step": 5821, + "time_per_iteration": 2.333202362060547 + }, + { + "auxiliary_loss_clip": 0.0101082, + "auxiliary_loss_mlp": 0.01001244, + "balance_loss_clip": 1.00196552, + "balance_loss_mlp": 1.00018322, + "epoch": 0.16893970170042366, + "flos": 74776344503040.0, + "grad_norm": 0.6892155209507229, + "language_loss": 0.48428565, + "learning_rate": 3.800911233820787e-06, + "loss": 0.50440633, + "num_input_tokens_seen": 166247295, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.01062012, + "step": 5822, + "time_per_iteration": 3.202421188354492 + }, + { + "auxiliary_loss_clip": 0.01083585, + "auxiliary_loss_mlp": 0.01031927, + "balance_loss_clip": 1.02882886, + "balance_loss_mlp": 1.01635277, + "epoch": 0.1689687191689397, + "flos": 30445414439040.0, + "grad_norm": 7.531221382921991, + "language_loss": 0.83627385, + "learning_rate": 3.800829472298858e-06, + "loss": 0.85742897, + "num_input_tokens_seen": 166267035, + "router_z_loss_clip": 0.54760742, + "router_z_loss_mlp": 0.15557861, + "step": 5823, + "time_per_iteration": 2.4777991771698 + }, + { + "auxiliary_loss_clip": 0.01088087, + "auxiliary_loss_mlp": 0.01041703, + "balance_loss_clip": 1.02624321, + "balance_loss_mlp": 1.02081203, + "epoch": 0.16899773663745574, + "flos": 32626404161280.0, + "grad_norm": 2.2757656389736574, + "language_loss": 0.81971622, + "learning_rate": 3.8007476948712745e-06, + "loss": 0.84101409, + "num_input_tokens_seen": 166281890, + "router_z_loss_clip": 0.61865234, + "router_z_loss_mlp": 0.20898438, + "step": 5824, + "time_per_iteration": 2.480330467224121 + }, + { + "auxiliary_loss_clip": 0.01012139, + "auxiliary_loss_mlp": 0.01001614, + "balance_loss_clip": 1.00337946, + "balance_loss_mlp": 1.00052965, + "epoch": 0.1690267541059718, + "flos": 74733401664000.0, + "grad_norm": 0.661691768646085, + "language_loss": 0.48473001, + "learning_rate": 3.8006659015387584e-06, + "loss": 0.50486755, + "num_input_tokens_seen": 166341265, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01086426, + "step": 5825, + "time_per_iteration": 3.0425095558166504 + }, + { + "auxiliary_loss_clip": 0.01011887, + "auxiliary_loss_mlp": 0.01001198, + "balance_loss_clip": 1.00314772, + "balance_loss_mlp": 1.00006533, + "epoch": 0.16905577157448784, + "flos": 70326070496640.0, + "grad_norm": 0.6223792213311462, + "language_loss": 0.46145228, + "learning_rate": 3.8005840923020324e-06, + "loss": 0.48158312, + "num_input_tokens_seen": 166405385, + "router_z_loss_clip": 0.08740234, + "router_z_loss_mlp": 0.01135254, + "step": 5826, + "time_per_iteration": 3.0965023040771484 + }, + { + "auxiliary_loss_clip": 0.01090043, + "auxiliary_loss_mlp": 0.01037611, + "balance_loss_clip": 1.02799058, + "balance_loss_mlp": 1.01804876, + "epoch": 0.1690847890430039, + "flos": 38355651336960.0, + "grad_norm": 1.9125249447408617, + "language_loss": 0.83923972, + "learning_rate": 3.8005022671618194e-06, + "loss": 0.86051631, + "num_input_tokens_seen": 166422040, + "router_z_loss_clip": 0.62011719, + "router_z_loss_mlp": 0.19567871, + "step": 5827, + "time_per_iteration": 2.5406785011291504 + }, + { + "auxiliary_loss_clip": 0.01084485, + "auxiliary_loss_mlp": 0.01041179, + "balance_loss_clip": 1.02761638, + "balance_loss_mlp": 1.02161694, + "epoch": 0.16911380651151994, + "flos": 18031173085440.0, + "grad_norm": 2.5742196590248, + "language_loss": 0.99787849, + "learning_rate": 3.8004204261188415e-06, + "loss": 1.019135, + "num_input_tokens_seen": 166432185, + "router_z_loss_clip": 0.5690918, + "router_z_loss_mlp": 0.19537354, + "step": 5828, + "time_per_iteration": 2.4236531257629395 + }, + { + "auxiliary_loss_clip": 0.0108211, + "auxiliary_loss_mlp": 0.01038909, + "balance_loss_clip": 1.02589262, + "balance_loss_mlp": 1.02066374, + "epoch": 0.169142823980036, + "flos": 19273277992320.0, + "grad_norm": 2.6388365633832747, + "language_loss": 0.98128825, + "learning_rate": 3.800338569173822e-06, + "loss": 1.00249851, + "num_input_tokens_seen": 166445915, + "router_z_loss_clip": 0.56225586, + "router_z_loss_mlp": 0.18249512, + "step": 5829, + "time_per_iteration": 2.3654913902282715 + }, + { + "auxiliary_loss_clip": 0.0101196, + "auxiliary_loss_mlp": 0.01002601, + "balance_loss_clip": 1.00314403, + "balance_loss_mlp": 1.00149834, + "epoch": 0.16917184144855202, + "flos": 74768908383360.0, + "grad_norm": 0.6870112398844095, + "language_loss": 0.49903572, + "learning_rate": 3.8002566963274836e-06, + "loss": 0.51918137, + "num_input_tokens_seen": 166507545, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01104736, + "step": 5830, + "time_per_iteration": 2.988234281539917 + }, + { + "auxiliary_loss_clip": 0.01088854, + "auxiliary_loss_mlp": 0.01042802, + "balance_loss_clip": 1.02756715, + "balance_loss_mlp": 1.02224445, + "epoch": 0.16920085891706807, + "flos": 16182975242880.0, + "grad_norm": 1.8298891813114266, + "language_loss": 0.67022759, + "learning_rate": 3.80017480758055e-06, + "loss": 0.69154418, + "num_input_tokens_seen": 166524965, + "router_z_loss_clip": 0.61279297, + "router_z_loss_mlp": 0.20562744, + "step": 5831, + "time_per_iteration": 2.3330981731414795 + }, + { + "auxiliary_loss_clip": 0.01082597, + "auxiliary_loss_mlp": 0.01034548, + "balance_loss_clip": 1.02748394, + "balance_loss_mlp": 1.01561761, + "epoch": 0.16922987638558412, + "flos": 16026599796480.0, + "grad_norm": 2.8017375119274486, + "language_loss": 0.84874564, + "learning_rate": 3.800092902933744e-06, + "loss": 0.86991709, + "num_input_tokens_seen": 166537770, + "router_z_loss_clip": 0.55053711, + "router_z_loss_mlp": 0.18933105, + "step": 5832, + "time_per_iteration": 2.6721935272216797 + }, + { + "auxiliary_loss_clip": 0.01082294, + "auxiliary_loss_mlp": 0.01038051, + "balance_loss_clip": 1.02722311, + "balance_loss_mlp": 1.01971626, + "epoch": 0.16925889385410017, + "flos": 24055156892160.0, + "grad_norm": 2.1962540762021767, + "language_loss": 0.9192946, + "learning_rate": 3.800010982387788e-06, + "loss": 0.94049811, + "num_input_tokens_seen": 166554020, + "router_z_loss_clip": 0.55029297, + "router_z_loss_mlp": 0.18328857, + "step": 5833, + "time_per_iteration": 2.4055962562561035 + }, + { + "auxiliary_loss_clip": 0.01076692, + "auxiliary_loss_mlp": 0.01033618, + "balance_loss_clip": 1.02631438, + "balance_loss_mlp": 1.01701283, + "epoch": 0.16928791132261622, + "flos": 15585646924800.0, + "grad_norm": 2.4269450634975454, + "language_loss": 0.61786389, + "learning_rate": 3.7999290459434076e-06, + "loss": 0.63896704, + "num_input_tokens_seen": 166564430, + "router_z_loss_clip": 0.50341797, + "router_z_loss_mlp": 0.16601562, + "step": 5834, + "time_per_iteration": 2.4129419326782227 + }, + { + "auxiliary_loss_clip": 0.01080832, + "auxiliary_loss_mlp": 0.0103894, + "balance_loss_clip": 1.02582717, + "balance_loss_mlp": 1.02372861, + "epoch": 0.16931692879113225, + "flos": 17630195587200.0, + "grad_norm": 2.715143704366516, + "language_loss": 0.96774852, + "learning_rate": 3.7998470936013253e-06, + "loss": 0.9889462, + "num_input_tokens_seen": 166575925, + "router_z_loss_clip": 0.55029297, + "router_z_loss_mlp": 0.1519165, + "step": 5835, + "time_per_iteration": 2.381040096282959 + }, + { + "auxiliary_loss_clip": 0.01083306, + "auxiliary_loss_mlp": 0.0103699, + "balance_loss_clip": 1.02883744, + "balance_loss_mlp": 1.01955533, + "epoch": 0.1693459462596483, + "flos": 74728969787520.0, + "grad_norm": 2.2639024955023714, + "language_loss": 0.93388116, + "learning_rate": 3.799765125362265e-06, + "loss": 0.9550842, + "num_input_tokens_seen": 166600065, + "router_z_loss_clip": 0.54418945, + "router_z_loss_mlp": 0.17449951, + "step": 5836, + "time_per_iteration": 2.7905750274658203 + }, + { + "auxiliary_loss_clip": 0.01086753, + "auxiliary_loss_mlp": 0.01041063, + "balance_loss_clip": 1.03104448, + "balance_loss_mlp": 1.02430224, + "epoch": 0.16937496372816435, + "flos": 27738703330560.0, + "grad_norm": 2.099218242831769, + "language_loss": 0.94439423, + "learning_rate": 3.7996831412269514e-06, + "loss": 0.96567249, + "num_input_tokens_seen": 166618370, + "router_z_loss_clip": 0.55810547, + "router_z_loss_mlp": 0.16766357, + "step": 5837, + "time_per_iteration": 2.415957450866699 + }, + { + "auxiliary_loss_clip": 0.0101322, + "auxiliary_loss_mlp": 0.010028, + "balance_loss_clip": 1.00422943, + "balance_loss_mlp": 1.00160789, + "epoch": 0.1694039811966804, + "flos": 61309471524480.0, + "grad_norm": 0.6798323848244868, + "language_loss": 0.51941037, + "learning_rate": 3.799601141196107e-06, + "loss": 0.53957057, + "num_input_tokens_seen": 166679065, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.01190186, + "step": 5838, + "time_per_iteration": 2.9861509799957275 + }, + { + "auxiliary_loss_clip": 0.01087189, + "auxiliary_loss_mlp": 0.01038698, + "balance_loss_clip": 1.02948546, + "balance_loss_mlp": 1.02167475, + "epoch": 0.16943299866519645, + "flos": 26752162642560.0, + "grad_norm": 2.087792860203079, + "language_loss": 0.78008664, + "learning_rate": 3.799519125270458e-06, + "loss": 0.80134547, + "num_input_tokens_seen": 166692725, + "router_z_loss_clip": 0.57714844, + "router_z_loss_mlp": 0.17041016, + "step": 5839, + "time_per_iteration": 2.414459228515625 + }, + { + "auxiliary_loss_clip": 0.01076226, + "auxiliary_loss_mlp": 0.01030976, + "balance_loss_clip": 1.02529383, + "balance_loss_mlp": 1.01379168, + "epoch": 0.1694620161337125, + "flos": 16133224688640.0, + "grad_norm": 3.677456334559039, + "language_loss": 0.97284102, + "learning_rate": 3.7994370934507276e-06, + "loss": 0.99391305, + "num_input_tokens_seen": 166702790, + "router_z_loss_clip": 0.50952148, + "router_z_loss_mlp": 0.17181396, + "step": 5840, + "time_per_iteration": 2.443953514099121 + }, + { + "auxiliary_loss_clip": 0.01077162, + "auxiliary_loss_mlp": 0.01031597, + "balance_loss_clip": 1.02578509, + "balance_loss_mlp": 1.01534271, + "epoch": 0.16949103360222853, + "flos": 17631033459840.0, + "grad_norm": 2.155638802936966, + "language_loss": 0.69291115, + "learning_rate": 3.7993550457376406e-06, + "loss": 0.71399868, + "num_input_tokens_seen": 166717190, + "router_z_loss_clip": 0.51391602, + "router_z_loss_mlp": 0.16259766, + "step": 5841, + "time_per_iteration": 2.3479113578796387 + }, + { + "auxiliary_loss_clip": 0.01012227, + "auxiliary_loss_mlp": 0.01002447, + "balance_loss_clip": 1.00335371, + "balance_loss_mlp": 1.00121284, + "epoch": 0.16952005107074458, + "flos": 70797118826880.0, + "grad_norm": 0.7984988684063421, + "language_loss": 0.4993405, + "learning_rate": 3.799272982131922e-06, + "loss": 0.51948726, + "num_input_tokens_seen": 166784350, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.0123291, + "step": 5842, + "time_per_iteration": 3.079296112060547 + }, + { + "auxiliary_loss_clip": 0.01076771, + "auxiliary_loss_mlp": 0.01030209, + "balance_loss_clip": 1.02458358, + "balance_loss_mlp": 1.01378202, + "epoch": 0.16954906853926063, + "flos": 16177110134400.0, + "grad_norm": 2.32842371376837, + "language_loss": 0.70913225, + "learning_rate": 3.799190902634296e-06, + "loss": 0.73020208, + "num_input_tokens_seen": 166799780, + "router_z_loss_clip": 0.52172852, + "router_z_loss_mlp": 0.16430664, + "step": 5843, + "time_per_iteration": 2.371098279953003 + }, + { + "auxiliary_loss_clip": 0.01011854, + "auxiliary_loss_mlp": 0.01001365, + "balance_loss_clip": 1.00317943, + "balance_loss_mlp": 1.00013745, + "epoch": 0.16957808600777668, + "flos": 60570011594880.0, + "grad_norm": 0.6894530092926138, + "language_loss": 0.49609795, + "learning_rate": 3.799108807245488e-06, + "loss": 0.51623011, + "num_input_tokens_seen": 166855080, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.01226807, + "step": 5844, + "time_per_iteration": 2.835474729537964 + }, + { + "auxiliary_loss_clip": 0.01011582, + "auxiliary_loss_mlp": 0.01001894, + "balance_loss_clip": 1.00277972, + "balance_loss_mlp": 1.0006845, + "epoch": 0.16960710347629274, + "flos": 55393404330240.0, + "grad_norm": 0.7856032912234258, + "language_loss": 0.51902652, + "learning_rate": 3.7990266959662227e-06, + "loss": 0.53916121, + "num_input_tokens_seen": 166915120, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01208496, + "step": 5845, + "time_per_iteration": 3.1191585063934326 + }, + { + "auxiliary_loss_clip": 0.01076254, + "auxiliary_loss_mlp": 0.01041307, + "balance_loss_clip": 1.02674866, + "balance_loss_mlp": 1.02460003, + "epoch": 0.1696361209448088, + "flos": 15332735969280.0, + "grad_norm": 4.069023439487062, + "language_loss": 0.9712671, + "learning_rate": 3.798944568797226e-06, + "loss": 0.99244273, + "num_input_tokens_seen": 166926430, + "router_z_loss_clip": 0.49511719, + "router_z_loss_mlp": 0.1673584, + "step": 5846, + "time_per_iteration": 2.369448661804199 + }, + { + "auxiliary_loss_clip": 0.01010856, + "auxiliary_loss_mlp": 0.01001847, + "balance_loss_clip": 1.00202274, + "balance_loss_mlp": 1.00074387, + "epoch": 0.1696651384133248, + "flos": 72296742988800.0, + "grad_norm": 0.5875351640593438, + "language_loss": 0.43543029, + "learning_rate": 3.798862425739223e-06, + "loss": 0.45555735, + "num_input_tokens_seen": 166997215, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01104736, + "step": 5847, + "time_per_iteration": 3.2869086265563965 + }, + { + "auxiliary_loss_clip": 0.01081717, + "auxiliary_loss_mlp": 0.01028646, + "balance_loss_clip": 1.0279367, + "balance_loss_mlp": 1.01342249, + "epoch": 0.16969415588184086, + "flos": 12453553411200.0, + "grad_norm": 2.076884848086282, + "language_loss": 0.77393532, + "learning_rate": 3.798780266792939e-06, + "loss": 0.79503894, + "num_input_tokens_seen": 167009195, + "router_z_loss_clip": 0.53710938, + "router_z_loss_mlp": 0.15203857, + "step": 5848, + "time_per_iteration": 2.353876829147339 + }, + { + "auxiliary_loss_clip": 0.01080254, + "auxiliary_loss_mlp": 0.01033033, + "balance_loss_clip": 1.02576089, + "balance_loss_mlp": 1.01654649, + "epoch": 0.16972317335035692, + "flos": 24417869673600.0, + "grad_norm": 1.924077171731098, + "language_loss": 0.73435241, + "learning_rate": 3.7986980919590998e-06, + "loss": 0.7554853, + "num_input_tokens_seen": 167027385, + "router_z_loss_clip": 0.54541016, + "router_z_loss_mlp": 0.16491699, + "step": 5849, + "time_per_iteration": 2.516139030456543 + }, + { + "auxiliary_loss_clip": 0.01091233, + "auxiliary_loss_mlp": 0.01034174, + "balance_loss_clip": 1.03087223, + "balance_loss_mlp": 1.01506531, + "epoch": 0.16975219081887297, + "flos": 17632395002880.0, + "grad_norm": 2.7789821330800124, + "language_loss": 0.77879542, + "learning_rate": 3.7986159012384312e-06, + "loss": 0.80004954, + "num_input_tokens_seen": 167041060, + "router_z_loss_clip": 0.60400391, + "router_z_loss_mlp": 0.19104004, + "step": 5850, + "time_per_iteration": 2.3458008766174316 + }, + { + "auxiliary_loss_clip": 0.01085584, + "auxiliary_loss_mlp": 0.01033688, + "balance_loss_clip": 1.02858424, + "balance_loss_mlp": 1.01584232, + "epoch": 0.16978120828738902, + "flos": 29307804831360.0, + "grad_norm": 3.609119691930068, + "language_loss": 0.7289722, + "learning_rate": 3.7985336946316585e-06, + "loss": 0.75016493, + "num_input_tokens_seen": 167060555, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.17858887, + "step": 5851, + "time_per_iteration": 2.5520293712615967 + }, + { + "auxiliary_loss_clip": 0.01081263, + "auxiliary_loss_mlp": 0.01040585, + "balance_loss_clip": 1.02833223, + "balance_loss_mlp": 1.02292418, + "epoch": 0.16981022575590504, + "flos": 26276331456000.0, + "grad_norm": 2.5916288048066236, + "language_loss": 0.88715714, + "learning_rate": 3.7984514721395096e-06, + "loss": 0.90837556, + "num_input_tokens_seen": 167073395, + "router_z_loss_clip": 0.52929688, + "router_z_loss_mlp": 0.17663574, + "step": 5852, + "time_per_iteration": 2.452138662338257 + }, + { + "auxiliary_loss_clip": 0.01011229, + "auxiliary_loss_mlp": 0.0100419, + "balance_loss_clip": 1.00251389, + "balance_loss_mlp": 1.00308108, + "epoch": 0.1698392432244211, + "flos": 62587152973440.0, + "grad_norm": 0.8051296312063544, + "language_loss": 0.53607166, + "learning_rate": 3.7983692337627087e-06, + "loss": 0.5562259, + "num_input_tokens_seen": 167137485, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.0111084, + "step": 5853, + "time_per_iteration": 3.0374464988708496 + }, + { + "auxiliary_loss_clip": 0.01086211, + "auxiliary_loss_mlp": 0.01040166, + "balance_loss_clip": 1.02843082, + "balance_loss_mlp": 1.02074671, + "epoch": 0.16986826069293715, + "flos": 45808036398720.0, + "grad_norm": 1.8566652633978153, + "language_loss": 0.917642, + "learning_rate": 3.7982869795019835e-06, + "loss": 0.93890572, + "num_input_tokens_seen": 167159265, + "router_z_loss_clip": 0.57763672, + "router_z_loss_mlp": 0.19421387, + "step": 5854, + "time_per_iteration": 2.595775842666626 + }, + { + "auxiliary_loss_clip": 0.0107838, + "auxiliary_loss_mlp": 0.01033104, + "balance_loss_clip": 1.02732372, + "balance_loss_mlp": 1.01643896, + "epoch": 0.1698972781614532, + "flos": 18801636168960.0, + "grad_norm": 2.4242432136421805, + "language_loss": 0.61359507, + "learning_rate": 3.7982047093580594e-06, + "loss": 0.63470989, + "num_input_tokens_seen": 167172395, + "router_z_loss_clip": 0.51074219, + "router_z_loss_mlp": 0.16662598, + "step": 5855, + "time_per_iteration": 2.4011900424957275 + }, + { + "auxiliary_loss_clip": 0.01086268, + "auxiliary_loss_mlp": 0.01038083, + "balance_loss_clip": 1.02870357, + "balance_loss_mlp": 1.01929545, + "epoch": 0.16992629562996925, + "flos": 39008771740800.0, + "grad_norm": 2.7418349146501813, + "language_loss": 1.03566885, + "learning_rate": 3.798122423331664e-06, + "loss": 1.0569123, + "num_input_tokens_seen": 167186715, + "router_z_loss_clip": 0.57543945, + "router_z_loss_mlp": 0.18792725, + "step": 5856, + "time_per_iteration": 2.555342435836792 + }, + { + "auxiliary_loss_clip": 0.01011755, + "auxiliary_loss_mlp": 0.01001617, + "balance_loss_clip": 1.00315738, + "balance_loss_mlp": 1.00031757, + "epoch": 0.1699553130984853, + "flos": 74134500468480.0, + "grad_norm": 0.6366320756621017, + "language_loss": 0.46467382, + "learning_rate": 3.7980401214235237e-06, + "loss": 0.48480752, + "num_input_tokens_seen": 167254625, + "router_z_loss_clip": 0.0859375, + "router_z_loss_mlp": 0.01300049, + "step": 5857, + "time_per_iteration": 5.520477294921875 + }, + { + "auxiliary_loss_clip": 0.0101117, + "auxiliary_loss_mlp": 0.01001081, + "balance_loss_clip": 1.00277603, + "balance_loss_mlp": 0.99988335, + "epoch": 0.16998433056700132, + "flos": 59470212856320.0, + "grad_norm": 0.6729586188553811, + "language_loss": 0.44592997, + "learning_rate": 3.7979578036343652e-06, + "loss": 0.4660525, + "num_input_tokens_seen": 167303015, + "router_z_loss_clip": 0.08398438, + "router_z_loss_mlp": 0.01196289, + "step": 5858, + "time_per_iteration": 2.8543283939361572 + }, + { + "auxiliary_loss_clip": 0.01091702, + "auxiliary_loss_mlp": 0.01048261, + "balance_loss_clip": 1.03125608, + "balance_loss_mlp": 1.02796018, + "epoch": 0.17001334803551738, + "flos": 17593327324800.0, + "grad_norm": 2.278404367413332, + "language_loss": 0.84681761, + "learning_rate": 3.7978754699649166e-06, + "loss": 0.86821723, + "num_input_tokens_seen": 167315840, + "router_z_loss_clip": 0.60449219, + "router_z_loss_mlp": 0.203125, + "step": 5859, + "time_per_iteration": 4.4947710037231445 + }, + { + "auxiliary_loss_clip": 0.01083126, + "auxiliary_loss_mlp": 0.01032788, + "balance_loss_clip": 1.03037965, + "balance_loss_mlp": 1.01656401, + "epoch": 0.17004236550403343, + "flos": 16208567136000.0, + "grad_norm": 2.269211995561638, + "language_loss": 0.75904369, + "learning_rate": 3.7977931204159037e-06, + "loss": 0.78020287, + "num_input_tokens_seen": 167327310, + "router_z_loss_clip": 0.52685547, + "router_z_loss_mlp": 0.16217041, + "step": 5860, + "time_per_iteration": 2.3629589080810547 + }, + { + "auxiliary_loss_clip": 0.01078662, + "auxiliary_loss_mlp": 0.01033672, + "balance_loss_clip": 1.02760839, + "balance_loss_mlp": 1.01692343, + "epoch": 0.17007138297254948, + "flos": 11719679299200.0, + "grad_norm": 3.155362274130065, + "language_loss": 0.92308569, + "learning_rate": 3.7977107549880545e-06, + "loss": 0.94420904, + "num_input_tokens_seen": 167337135, + "router_z_loss_clip": 0.51025391, + "router_z_loss_mlp": 0.16766357, + "step": 5861, + "time_per_iteration": 2.359877586364746 + }, + { + "auxiliary_loss_clip": 0.01085775, + "auxiliary_loss_mlp": 0.01038433, + "balance_loss_clip": 1.03005123, + "balance_loss_mlp": 1.01909757, + "epoch": 0.17010040044106553, + "flos": 10988912298240.0, + "grad_norm": 2.777747238223128, + "language_loss": 0.73835516, + "learning_rate": 3.7976283736820968e-06, + "loss": 0.7595973, + "num_input_tokens_seen": 167347905, + "router_z_loss_clip": 0.55761719, + "router_z_loss_mlp": 0.19342041, + "step": 5862, + "time_per_iteration": 2.365312337875366 + }, + { + "auxiliary_loss_clip": 0.01090664, + "auxiliary_loss_mlp": 0.01035959, + "balance_loss_clip": 1.03173649, + "balance_loss_mlp": 1.01729083, + "epoch": 0.17012941790958158, + "flos": 17157262043520.0, + "grad_norm": 4.395930410116148, + "language_loss": 0.93576157, + "learning_rate": 3.7975459764987575e-06, + "loss": 0.95702779, + "num_input_tokens_seen": 167359915, + "router_z_loss_clip": 0.58935547, + "router_z_loss_mlp": 0.18652344, + "step": 5863, + "time_per_iteration": 2.3839616775512695 + }, + { + "auxiliary_loss_clip": 0.0109116, + "auxiliary_loss_mlp": 0.01041502, + "balance_loss_clip": 1.03397465, + "balance_loss_mlp": 1.02197528, + "epoch": 0.1701584353780976, + "flos": 36309601486080.0, + "grad_norm": 2.8243752290539708, + "language_loss": 0.6779139, + "learning_rate": 3.797463563438765e-06, + "loss": 0.69924051, + "num_input_tokens_seen": 167375490, + "router_z_loss_clip": 0.57128906, + "router_z_loss_mlp": 0.1953125, + "step": 5864, + "time_per_iteration": 2.4250240325927734 + }, + { + "auxiliary_loss_clip": 0.01081713, + "auxiliary_loss_mlp": 0.01033141, + "balance_loss_clip": 1.03039777, + "balance_loss_mlp": 1.01714885, + "epoch": 0.17018745284661366, + "flos": 11134011375360.0, + "grad_norm": 2.563661400764351, + "language_loss": 0.8395685, + "learning_rate": 3.7973811345028464e-06, + "loss": 0.86071706, + "num_input_tokens_seen": 167385620, + "router_z_loss_clip": 0.51245117, + "router_z_loss_mlp": 0.15997314, + "step": 5865, + "time_per_iteration": 2.4925007820129395 + }, + { + "auxiliary_loss_clip": 0.0101479, + "auxiliary_loss_mlp": 0.01001282, + "balance_loss_clip": 1.00634837, + "balance_loss_mlp": 1.0000186, + "epoch": 0.1702164703151297, + "flos": 69153896776320.0, + "grad_norm": 0.7207887912993797, + "language_loss": 0.49563831, + "learning_rate": 3.7972986896917306e-06, + "loss": 0.51579905, + "num_input_tokens_seen": 167449985, + "router_z_loss_clip": 0.08447266, + "router_z_loss_mlp": 0.01263428, + "step": 5866, + "time_per_iteration": 3.1497299671173096 + }, + { + "auxiliary_loss_clip": 0.01014347, + "auxiliary_loss_mlp": 0.0100099, + "balance_loss_clip": 1.0063138, + "balance_loss_mlp": 0.99971998, + "epoch": 0.17024548778364576, + "flos": 57616673575680.0, + "grad_norm": 0.7111148868480891, + "language_loss": 0.54590839, + "learning_rate": 3.7972162290061462e-06, + "loss": 0.56606174, + "num_input_tokens_seen": 167507145, + "router_z_loss_clip": 0.08007812, + "router_z_loss_mlp": 0.01269531, + "step": 5867, + "time_per_iteration": 5.619910955429077 + }, + { + "auxiliary_loss_clip": 0.01013643, + "auxiliary_loss_mlp": 0.01000466, + "balance_loss_clip": 1.00545454, + "balance_loss_mlp": 0.99931532, + "epoch": 0.1702745052521618, + "flos": 74780464043520.0, + "grad_norm": 0.6435690032451441, + "language_loss": 0.51392257, + "learning_rate": 3.7971337524468197e-06, + "loss": 0.53406364, + "num_input_tokens_seen": 167573420, + "router_z_loss_clip": 0.08203125, + "router_z_loss_mlp": 0.01147461, + "step": 5868, + "time_per_iteration": 5.691675186157227 + }, + { + "auxiliary_loss_clip": 0.01012195, + "auxiliary_loss_mlp": 0.01000998, + "balance_loss_clip": 1.0042069, + "balance_loss_mlp": 0.99991298, + "epoch": 0.17030352272067784, + "flos": 74776833262080.0, + "grad_norm": 0.7142029793259028, + "language_loss": 0.44367129, + "learning_rate": 3.7970512600144816e-06, + "loss": 0.4638032, + "num_input_tokens_seen": 167634810, + "router_z_loss_clip": 0.08007812, + "router_z_loss_mlp": 0.01086426, + "step": 5869, + "time_per_iteration": 3.0596442222595215 + }, + { + "auxiliary_loss_clip": 0.0108471, + "auxiliary_loss_mlp": 0.01033741, + "balance_loss_clip": 1.0292381, + "balance_loss_mlp": 1.01631212, + "epoch": 0.1703325401891939, + "flos": 26137621157760.0, + "grad_norm": 1.9207377950362379, + "language_loss": 0.71595192, + "learning_rate": 3.796968751709859e-06, + "loss": 0.73713642, + "num_input_tokens_seen": 167650695, + "router_z_loss_clip": 0.55493164, + "router_z_loss_mlp": 0.17431641, + "step": 5870, + "time_per_iteration": 2.5438785552978516 + }, + { + "auxiliary_loss_clip": 0.01009922, + "auxiliary_loss_mlp": 0.0100273, + "balance_loss_clip": 1.00183654, + "balance_loss_mlp": 1.00156784, + "epoch": 0.17036155765770994, + "flos": 74780464043520.0, + "grad_norm": 0.6033105569080083, + "language_loss": 0.45992577, + "learning_rate": 3.7968862275336813e-06, + "loss": 0.48005229, + "num_input_tokens_seen": 167720695, + "router_z_loss_clip": 0.08105469, + "router_z_loss_mlp": 0.01159668, + "step": 5871, + "time_per_iteration": 3.176400899887085 + }, + { + "auxiliary_loss_clip": 0.01090021, + "auxiliary_loss_mlp": 0.01042502, + "balance_loss_clip": 1.0300355, + "balance_loss_mlp": 1.02409577, + "epoch": 0.170390575126226, + "flos": 16211010931200.0, + "grad_norm": 8.38641111461279, + "language_loss": 0.7244153, + "learning_rate": 3.7968036874866778e-06, + "loss": 0.74574053, + "num_input_tokens_seen": 167737125, + "router_z_loss_clip": 0.59960938, + "router_z_loss_mlp": 0.1842041, + "step": 5872, + "time_per_iteration": 2.3976528644561768 + }, + { + "auxiliary_loss_clip": 0.01011096, + "auxiliary_loss_mlp": 0.01003467, + "balance_loss_clip": 1.00287914, + "balance_loss_mlp": 1.00222683, + "epoch": 0.17041959259474204, + "flos": 71399930094720.0, + "grad_norm": 0.7110656039718718, + "language_loss": 0.4802106, + "learning_rate": 3.796721131569577e-06, + "loss": 0.50035626, + "num_input_tokens_seen": 167796325, + "router_z_loss_clip": 0.08203125, + "router_z_loss_mlp": 0.01239014, + "step": 5873, + "time_per_iteration": 3.012770175933838 + }, + { + "auxiliary_loss_clip": 0.01091713, + "auxiliary_loss_mlp": 0.01053447, + "balance_loss_clip": 1.03083205, + "balance_loss_mlp": 1.0315963, + "epoch": 0.1704486100632581, + "flos": 26133152503680.0, + "grad_norm": 2.2579372046237234, + "language_loss": 0.8738153, + "learning_rate": 3.7966385597831074e-06, + "loss": 0.89526689, + "num_input_tokens_seen": 167814060, + "router_z_loss_clip": 0.609375, + "router_z_loss_mlp": 0.21850586, + "step": 5874, + "time_per_iteration": 2.4475011825561523 + }, + { + "auxiliary_loss_clip": 0.01090049, + "auxiliary_loss_mlp": 0.01040852, + "balance_loss_clip": 1.03138518, + "balance_loss_mlp": 1.02308416, + "epoch": 0.17047762753177412, + "flos": 28687293504000.0, + "grad_norm": 2.399287591901061, + "language_loss": 0.89181572, + "learning_rate": 3.7965559721279995e-06, + "loss": 0.91312474, + "num_input_tokens_seen": 167829370, + "router_z_loss_clip": 0.58740234, + "router_z_loss_mlp": 0.17755127, + "step": 5875, + "time_per_iteration": 2.516388177871704 + }, + { + "auxiliary_loss_clip": 0.0108199, + "auxiliary_loss_mlp": 0.01035507, + "balance_loss_clip": 1.02846777, + "balance_loss_mlp": 1.01934826, + "epoch": 0.17050664500029017, + "flos": 35034922414080.0, + "grad_norm": 7.579780793745293, + "language_loss": 0.74035174, + "learning_rate": 3.796473368604982e-06, + "loss": 0.7615267, + "num_input_tokens_seen": 167846005, + "router_z_loss_clip": 0.53491211, + "router_z_loss_mlp": 0.16143799, + "step": 5876, + "time_per_iteration": 2.5466933250427246 + }, + { + "auxiliary_loss_clip": 0.01085138, + "auxiliary_loss_mlp": 0.01043111, + "balance_loss_clip": 1.02850211, + "balance_loss_mlp": 1.02375174, + "epoch": 0.17053566246880622, + "flos": 12157315591680.0, + "grad_norm": 3.1135979129732902, + "language_loss": 0.78595912, + "learning_rate": 3.7963907492147847e-06, + "loss": 0.80724156, + "num_input_tokens_seen": 167856425, + "router_z_loss_clip": 0.56591797, + "router_z_loss_mlp": 0.19348145, + "step": 5877, + "time_per_iteration": 2.398733615875244 + }, + { + "auxiliary_loss_clip": 0.01013486, + "auxiliary_loss_mlp": 0.01002581, + "balance_loss_clip": 1.00495231, + "balance_loss_mlp": 1.00137091, + "epoch": 0.17056467993732227, + "flos": 63025068556800.0, + "grad_norm": 0.690019913754318, + "language_loss": 0.49527931, + "learning_rate": 3.7963081139581375e-06, + "loss": 0.51543999, + "num_input_tokens_seen": 167914435, + "router_z_loss_clip": 0.08496094, + "router_z_loss_mlp": 0.01208496, + "step": 5878, + "time_per_iteration": 2.9622955322265625 + }, + { + "auxiliary_loss_clip": 0.0108941, + "auxiliary_loss_mlp": 0.01041924, + "balance_loss_clip": 1.02983451, + "balance_loss_mlp": 1.02221918, + "epoch": 0.17059369740583832, + "flos": 11097841340160.0, + "grad_norm": 4.913045365556424, + "language_loss": 0.83240747, + "learning_rate": 3.7962254628357704e-06, + "loss": 0.85372078, + "num_input_tokens_seen": 167924330, + "router_z_loss_clip": 0.59667969, + "router_z_loss_mlp": 0.19702148, + "step": 5879, + "time_per_iteration": 2.3988845348358154 + }, + { + "auxiliary_loss_clip": 0.01088936, + "auxiliary_loss_mlp": 0.01032157, + "balance_loss_clip": 1.02873695, + "balance_loss_mlp": 1.01342916, + "epoch": 0.17062271487435435, + "flos": 15625692120960.0, + "grad_norm": 3.096334307258647, + "language_loss": 0.85052836, + "learning_rate": 3.7961427958484135e-06, + "loss": 0.87173933, + "num_input_tokens_seen": 167938535, + "router_z_loss_clip": 0.6015625, + "router_z_loss_mlp": 0.18737793, + "step": 5880, + "time_per_iteration": 2.4222779273986816 + }, + { + "auxiliary_loss_clip": 0.01012983, + "auxiliary_loss_mlp": 0.01001356, + "balance_loss_clip": 1.00467682, + "balance_loss_mlp": 1.00007451, + "epoch": 0.1706517323428704, + "flos": 74767895953920.0, + "grad_norm": 0.7177502991637353, + "language_loss": 0.47374475, + "learning_rate": 3.7960601129967957e-06, + "loss": 0.49388811, + "num_input_tokens_seen": 168003245, + "router_z_loss_clip": 0.08300781, + "router_z_loss_mlp": 0.01281738, + "step": 5881, + "time_per_iteration": 3.314727544784546 + }, + { + "auxiliary_loss_clip": 0.01082898, + "auxiliary_loss_mlp": 0.01044947, + "balance_loss_clip": 1.02911818, + "balance_loss_mlp": 1.02521753, + "epoch": 0.17068074981138645, + "flos": 27848016397440.0, + "grad_norm": 1.8469598151460285, + "language_loss": 0.93784916, + "learning_rate": 3.7959774142816484e-06, + "loss": 0.95912766, + "num_input_tokens_seen": 168019610, + "router_z_loss_clip": 0.53833008, + "router_z_loss_mlp": 0.19726562, + "step": 5882, + "time_per_iteration": 2.5106067657470703 + }, + { + "auxiliary_loss_clip": 0.01011036, + "auxiliary_loss_mlp": 0.01001932, + "balance_loss_clip": 1.00283158, + "balance_loss_mlp": 1.00066209, + "epoch": 0.1707097672799025, + "flos": 57218698454400.0, + "grad_norm": 0.6934407701874571, + "language_loss": 0.5167973, + "learning_rate": 3.7958946997037026e-06, + "loss": 0.53692693, + "num_input_tokens_seen": 168080495, + "router_z_loss_clip": 0.08203125, + "router_z_loss_mlp": 0.01269531, + "step": 5883, + "time_per_iteration": 2.9939050674438477 + }, + { + "auxiliary_loss_clip": 0.01084002, + "auxiliary_loss_mlp": 0.01034411, + "balance_loss_clip": 1.02564204, + "balance_loss_mlp": 1.01546884, + "epoch": 0.17073878474841855, + "flos": 17813175356160.0, + "grad_norm": 3.087746648104332, + "language_loss": 0.85841751, + "learning_rate": 3.795811969263687e-06, + "loss": 0.8796016, + "num_input_tokens_seen": 168092615, + "router_z_loss_clip": 0.58325195, + "router_z_loss_mlp": 0.1895752, + "step": 5884, + "time_per_iteration": 2.3635079860687256 + }, + { + "auxiliary_loss_clip": 0.01088087, + "auxiliary_loss_mlp": 0.01032974, + "balance_loss_clip": 1.02884126, + "balance_loss_mlp": 1.01411486, + "epoch": 0.1707678022169346, + "flos": 16247041320960.0, + "grad_norm": 3.7421619792172165, + "language_loss": 0.8790642, + "learning_rate": 3.795729222962334e-06, + "loss": 0.90027487, + "num_input_tokens_seen": 168104960, + "router_z_loss_clip": 0.59228516, + "router_z_loss_mlp": 0.18859863, + "step": 5885, + "time_per_iteration": 2.4042246341705322 + }, + { + "auxiliary_loss_clip": 0.01009129, + "auxiliary_loss_mlp": 0.0100372, + "balance_loss_clip": 1.00117743, + "balance_loss_mlp": 1.00264096, + "epoch": 0.17079681968545063, + "flos": 64196334581760.0, + "grad_norm": 0.6750054960196524, + "language_loss": 0.49334419, + "learning_rate": 3.795646460800374e-06, + "loss": 0.51347268, + "num_input_tokens_seen": 168167295, + "router_z_loss_clip": 0.07958984, + "router_z_loss_mlp": 0.01080322, + "step": 5886, + "time_per_iteration": 2.994081974029541 + }, + { + "auxiliary_loss_clip": 0.01085787, + "auxiliary_loss_mlp": 0.01039799, + "balance_loss_clip": 1.02827227, + "balance_loss_mlp": 1.02160788, + "epoch": 0.17082583715396668, + "flos": 26497226828160.0, + "grad_norm": 2.121927839016553, + "language_loss": 0.95129538, + "learning_rate": 3.795563682778537e-06, + "loss": 0.97255123, + "num_input_tokens_seen": 168184215, + "router_z_loss_clip": 0.57568359, + "router_z_loss_mlp": 0.18188477, + "step": 5887, + "time_per_iteration": 2.453022003173828 + }, + { + "auxiliary_loss_clip": 0.0108581, + "auxiliary_loss_mlp": 0.01047639, + "balance_loss_clip": 1.03017426, + "balance_loss_mlp": 1.02783811, + "epoch": 0.17085485462248273, + "flos": 25113199777920.0, + "grad_norm": 2.4436778659020306, + "language_loss": 0.91502959, + "learning_rate": 3.795480888897556e-06, + "loss": 0.93636405, + "num_input_tokens_seen": 168202200, + "router_z_loss_clip": 0.5559082, + "router_z_loss_mlp": 0.19805908, + "step": 5888, + "time_per_iteration": 2.4514377117156982 + }, + { + "auxiliary_loss_clip": 0.01009046, + "auxiliary_loss_mlp": 0.01004032, + "balance_loss_clip": 1.00106764, + "balance_loss_mlp": 1.00300682, + "epoch": 0.17088387209099878, + "flos": 59002655662080.0, + "grad_norm": 0.7039803550231933, + "language_loss": 0.53252608, + "learning_rate": 3.79539807915816e-06, + "loss": 0.55265689, + "num_input_tokens_seen": 168255920, + "router_z_loss_clip": 0.08007812, + "router_z_loss_mlp": 0.01025391, + "step": 5889, + "time_per_iteration": 2.832817792892456 + }, + { + "auxiliary_loss_clip": 0.01009437, + "auxiliary_loss_mlp": 0.01000198, + "balance_loss_clip": 1.00158572, + "balance_loss_mlp": 0.99916095, + "epoch": 0.17091288955951484, + "flos": 61467348159360.0, + "grad_norm": 0.6685516185320548, + "language_loss": 0.48747885, + "learning_rate": 3.795315253561083e-06, + "loss": 0.50757521, + "num_input_tokens_seen": 168319750, + "router_z_loss_clip": 0.07861328, + "router_z_loss_mlp": 0.01037598, + "step": 5890, + "time_per_iteration": 3.093170404434204 + }, + { + "auxiliary_loss_clip": 0.010097, + "auxiliary_loss_mlp": 0.01000909, + "balance_loss_clip": 1.00180626, + "balance_loss_mlp": 0.99983573, + "epoch": 0.1709419070280309, + "flos": 58716123200640.0, + "grad_norm": 0.6757296946546638, + "language_loss": 0.52105236, + "learning_rate": 3.7952324121070543e-06, + "loss": 0.54115844, + "num_input_tokens_seen": 168379485, + "router_z_loss_clip": 0.07910156, + "router_z_loss_mlp": 0.01074219, + "step": 5891, + "time_per_iteration": 2.945755958557129 + }, + { + "auxiliary_loss_clip": 0.01070805, + "auxiliary_loss_mlp": 0.01029638, + "balance_loss_clip": 1.02393866, + "balance_loss_mlp": 1.01551735, + "epoch": 0.1709709244965469, + "flos": 11762028645120.0, + "grad_norm": 2.72707484687817, + "language_loss": 0.66850054, + "learning_rate": 3.7951495547968067e-06, + "loss": 0.68950498, + "num_input_tokens_seen": 168391740, + "router_z_loss_clip": 0.46923828, + "router_z_loss_mlp": 0.14135742, + "step": 5892, + "time_per_iteration": 2.425410270690918 + }, + { + "auxiliary_loss_clip": 0.01010634, + "auxiliary_loss_mlp": 0.01003242, + "balance_loss_clip": 1.00245857, + "balance_loss_mlp": 1.00212145, + "epoch": 0.17099994196506296, + "flos": 74295765371520.0, + "grad_norm": 0.6378092809914448, + "language_loss": 0.4894377, + "learning_rate": 3.7950666816310726e-06, + "loss": 0.50957644, + "num_input_tokens_seen": 168455950, + "router_z_loss_clip": 0.08203125, + "router_z_loss_mlp": 0.01123047, + "step": 5893, + "time_per_iteration": 3.0799503326416016 + }, + { + "auxiliary_loss_clip": 0.01092954, + "auxiliary_loss_mlp": 0.01045416, + "balance_loss_clip": 1.03135133, + "balance_loss_mlp": 1.02487612, + "epoch": 0.17102895943357901, + "flos": 39961446543360.0, + "grad_norm": 1.8549214215675107, + "language_loss": 1.02417946, + "learning_rate": 3.7949837926105826e-06, + "loss": 1.04556322, + "num_input_tokens_seen": 168477530, + "router_z_loss_clip": 0.61572266, + "router_z_loss_mlp": 0.20544434, + "step": 5894, + "time_per_iteration": 2.7645201683044434 + }, + { + "auxiliary_loss_clip": 0.01011897, + "auxiliary_loss_mlp": 0.01005596, + "balance_loss_clip": 1.00391114, + "balance_loss_mlp": 1.004529, + "epoch": 0.17105797690209507, + "flos": 74772608987520.0, + "grad_norm": 0.6549664650349575, + "language_loss": 0.45004103, + "learning_rate": 3.794900887736069e-06, + "loss": 0.47021595, + "num_input_tokens_seen": 168538945, + "router_z_loss_clip": 0.08007812, + "router_z_loss_mlp": 0.01068115, + "step": 5895, + "time_per_iteration": 3.042917490005493 + }, + { + "auxiliary_loss_clip": 0.01011478, + "auxiliary_loss_mlp": 0.01005665, + "balance_loss_clip": 1.00347829, + "balance_loss_mlp": 1.00471711, + "epoch": 0.17108699437061112, + "flos": 64594868284800.0, + "grad_norm": 0.6646309609609055, + "language_loss": 0.46806729, + "learning_rate": 3.7948179670082646e-06, + "loss": 0.48823875, + "num_input_tokens_seen": 168601545, + "router_z_loss_clip": 0.08007812, + "router_z_loss_mlp": 0.00946045, + "step": 5896, + "time_per_iteration": 3.0024054050445557 + }, + { + "auxiliary_loss_clip": 0.01083041, + "auxiliary_loss_mlp": 0.01036418, + "balance_loss_clip": 1.02788413, + "balance_loss_mlp": 1.01884651, + "epoch": 0.17111601183912714, + "flos": 30986359044480.0, + "grad_norm": 2.719342022244323, + "language_loss": 0.72353345, + "learning_rate": 3.794735030427902e-06, + "loss": 0.74472809, + "num_input_tokens_seen": 168621200, + "router_z_loss_clip": 0.55102539, + "router_z_loss_mlp": 0.17596436, + "step": 5897, + "time_per_iteration": 2.533376932144165 + }, + { + "auxiliary_loss_clip": 0.01011294, + "auxiliary_loss_mlp": 0.01002233, + "balance_loss_clip": 1.00350404, + "balance_loss_mlp": 1.00126183, + "epoch": 0.1711450293076432, + "flos": 69230042184960.0, + "grad_norm": 0.6187149025063104, + "language_loss": 0.51315761, + "learning_rate": 3.794652077995713e-06, + "loss": 0.53329289, + "num_input_tokens_seen": 168686480, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.00970459, + "step": 5898, + "time_per_iteration": 3.0586369037628174 + }, + { + "auxiliary_loss_clip": 0.01091392, + "auxiliary_loss_mlp": 0.01046319, + "balance_loss_clip": 1.03144121, + "balance_loss_mlp": 1.02544522, + "epoch": 0.17117404677615924, + "flos": 32081305104000.0, + "grad_norm": 2.931253948319027, + "language_loss": 0.86375034, + "learning_rate": 3.794569109712431e-06, + "loss": 0.88512743, + "num_input_tokens_seen": 168702015, + "router_z_loss_clip": 0.60009766, + "router_z_loss_mlp": 0.20874023, + "step": 5899, + "time_per_iteration": 2.5030405521392822 + }, + { + "auxiliary_loss_clip": 0.01010561, + "auxiliary_loss_mlp": 0.01001576, + "balance_loss_clip": 1.00290966, + "balance_loss_mlp": 1.00056279, + "epoch": 0.1712030642446753, + "flos": 74769013117440.0, + "grad_norm": 0.6630296904681603, + "language_loss": 0.48685524, + "learning_rate": 3.794486125578788e-06, + "loss": 0.5069766, + "num_input_tokens_seen": 168764760, + "router_z_loss_clip": 0.07666016, + "router_z_loss_mlp": 0.01013184, + "step": 5900, + "time_per_iteration": 2.999669313430786 + }, + { + "auxiliary_loss_clip": 0.01081689, + "auxiliary_loss_mlp": 0.01029555, + "balance_loss_clip": 1.02764225, + "balance_loss_mlp": 1.01426077, + "epoch": 0.17123208171319135, + "flos": 34200707454720.0, + "grad_norm": 1.9668952235664352, + "language_loss": 1.00773346, + "learning_rate": 3.7944031255955178e-06, + "loss": 1.02884591, + "num_input_tokens_seen": 168789380, + "router_z_loss_clip": 0.54077148, + "router_z_loss_mlp": 0.15307617, + "step": 5901, + "time_per_iteration": 2.5808258056640625 + }, + { + "auxiliary_loss_clip": 0.01009864, + "auxiliary_loss_mlp": 0.01003909, + "balance_loss_clip": 1.00220585, + "balance_loss_mlp": 1.0027591, + "epoch": 0.1712610991817074, + "flos": 72391079082240.0, + "grad_norm": 0.6622928601940683, + "language_loss": 0.46423307, + "learning_rate": 3.7943201097633527e-06, + "loss": 0.48437077, + "num_input_tokens_seen": 168843170, + "router_z_loss_clip": 0.07666016, + "router_z_loss_mlp": 0.01147461, + "step": 5902, + "time_per_iteration": 2.9458796977996826 + }, + { + "auxiliary_loss_clip": 0.01082589, + "auxiliary_loss_mlp": 0.01032946, + "balance_loss_clip": 1.02976441, + "balance_loss_mlp": 1.01731813, + "epoch": 0.17129011665022342, + "flos": 15222201004800.0, + "grad_norm": 2.4870198131383696, + "language_loss": 0.95469469, + "learning_rate": 3.794237078083026e-06, + "loss": 0.97584999, + "num_input_tokens_seen": 168853980, + "router_z_loss_clip": 0.52856445, + "router_z_loss_mlp": 0.15625, + "step": 5903, + "time_per_iteration": 2.4137930870056152 + }, + { + "auxiliary_loss_clip": 0.01009459, + "auxiliary_loss_mlp": 0.01000532, + "balance_loss_clip": 1.00193417, + "balance_loss_mlp": 0.99948895, + "epoch": 0.17131913411873947, + "flos": 61428978708480.0, + "grad_norm": 0.6970418315976361, + "language_loss": 0.50524163, + "learning_rate": 3.7941540305552724e-06, + "loss": 0.52534151, + "num_input_tokens_seen": 168915210, + "router_z_loss_clip": 0.07519531, + "router_z_loss_mlp": 0.01043701, + "step": 5904, + "time_per_iteration": 3.0326812267303467 + }, + { + "auxiliary_loss_clip": 0.0108139, + "auxiliary_loss_mlp": 0.01031851, + "balance_loss_clip": 1.02492499, + "balance_loss_mlp": 1.01537609, + "epoch": 0.17134815158725553, + "flos": 18798843260160.0, + "grad_norm": 2.8306216672412288, + "language_loss": 0.83464897, + "learning_rate": 3.794070967180824e-06, + "loss": 0.85578132, + "num_input_tokens_seen": 168927295, + "router_z_loss_clip": 0.56567383, + "router_z_loss_mlp": 0.16491699, + "step": 5905, + "time_per_iteration": 2.5200819969177246 + }, + { + "auxiliary_loss_clip": 0.01080777, + "auxiliary_loss_mlp": 0.01029149, + "balance_loss_clip": 1.02847695, + "balance_loss_mlp": 1.01310396, + "epoch": 0.17137716905577158, + "flos": 23945564534400.0, + "grad_norm": 2.749682007310115, + "language_loss": 0.73972023, + "learning_rate": 3.793987887960414e-06, + "loss": 0.76081944, + "num_input_tokens_seen": 168944685, + "router_z_loss_clip": 0.52172852, + "router_z_loss_mlp": 0.16040039, + "step": 5906, + "time_per_iteration": 2.664672374725342 + }, + { + "auxiliary_loss_clip": 0.01092335, + "auxiliary_loss_mlp": 0.01039398, + "balance_loss_clip": 1.0300262, + "balance_loss_mlp": 1.01884639, + "epoch": 0.17140618652428763, + "flos": 29859886160640.0, + "grad_norm": 1.909308091659244, + "language_loss": 0.80441535, + "learning_rate": 3.7939047928947775e-06, + "loss": 0.82573271, + "num_input_tokens_seen": 168968445, + "router_z_loss_clip": 0.62353516, + "router_z_loss_mlp": 0.20562744, + "step": 5907, + "time_per_iteration": 2.571314811706543 + }, + { + "auxiliary_loss_clip": 0.01082452, + "auxiliary_loss_mlp": 0.01030005, + "balance_loss_clip": 1.02775383, + "balance_loss_mlp": 1.01246953, + "epoch": 0.17143520399280368, + "flos": 17047948976640.0, + "grad_norm": 2.130998872801905, + "language_loss": 0.71244574, + "learning_rate": 3.7938216819846485e-06, + "loss": 0.73357028, + "num_input_tokens_seen": 168982190, + "router_z_loss_clip": 0.54638672, + "router_z_loss_mlp": 0.175354, + "step": 5908, + "time_per_iteration": 2.38517427444458 + }, + { + "auxiliary_loss_clip": 0.01074161, + "auxiliary_loss_mlp": 0.01034231, + "balance_loss_clip": 1.02583933, + "balance_loss_mlp": 1.01894236, + "epoch": 0.1714642214613197, + "flos": 74741921902080.0, + "grad_norm": 1.5229993834549478, + "language_loss": 0.89051139, + "learning_rate": 3.79373855523076e-06, + "loss": 0.91159534, + "num_input_tokens_seen": 169011425, + "router_z_loss_clip": 0.48266602, + "router_z_loss_mlp": 0.152771, + "step": 5909, + "time_per_iteration": 2.848390817642212 + }, + { + "auxiliary_loss_clip": 0.01080284, + "auxiliary_loss_mlp": 0.01035988, + "balance_loss_clip": 1.0279274, + "balance_loss_mlp": 1.01913142, + "epoch": 0.17149323892983576, + "flos": 31536310780800.0, + "grad_norm": 2.2049443549899554, + "language_loss": 0.75445735, + "learning_rate": 3.7936554126338473e-06, + "loss": 0.77562004, + "num_input_tokens_seen": 169028505, + "router_z_loss_clip": 0.52392578, + "router_z_loss_mlp": 0.16864014, + "step": 5910, + "time_per_iteration": 2.481428861618042 + }, + { + "auxiliary_loss_clip": 0.01085514, + "auxiliary_loss_mlp": 0.01035438, + "balance_loss_clip": 1.02944422, + "balance_loss_mlp": 1.01685274, + "epoch": 0.1715222563983518, + "flos": 15222689763840.0, + "grad_norm": 2.868992232044461, + "language_loss": 0.84712791, + "learning_rate": 3.793572254194643e-06, + "loss": 0.86833739, + "num_input_tokens_seen": 169041460, + "router_z_loss_clip": 0.56030273, + "router_z_loss_mlp": 0.18591309, + "step": 5911, + "time_per_iteration": 2.4012224674224854 + }, + { + "auxiliary_loss_clip": 0.01089153, + "auxiliary_loss_mlp": 0.01038714, + "balance_loss_clip": 1.0296092, + "balance_loss_mlp": 1.0200932, + "epoch": 0.17155127386686786, + "flos": 74729807660160.0, + "grad_norm": 2.078847993141057, + "language_loss": 0.84924924, + "learning_rate": 3.793489079913884e-06, + "loss": 0.87052786, + "num_input_tokens_seen": 169064675, + "router_z_loss_clip": 0.59570312, + "router_z_loss_mlp": 0.1862793, + "step": 5912, + "time_per_iteration": 2.854926586151123 + }, + { + "auxiliary_loss_clip": 0.01090758, + "auxiliary_loss_mlp": 0.01038421, + "balance_loss_clip": 1.03138375, + "balance_loss_mlp": 1.02235174, + "epoch": 0.1715802913353839, + "flos": 65428002858240.0, + "grad_norm": 2.078040482399237, + "language_loss": 0.92052245, + "learning_rate": 3.7934058897923032e-06, + "loss": 0.94181418, + "num_input_tokens_seen": 169091725, + "router_z_loss_clip": 0.59399414, + "router_z_loss_mlp": 0.1607666, + "step": 5913, + "time_per_iteration": 2.729536533355713 + }, + { + "auxiliary_loss_clip": 0.01082444, + "auxiliary_loss_mlp": 0.01034164, + "balance_loss_clip": 1.02814484, + "balance_loss_mlp": 1.0185535, + "epoch": 0.17160930880389993, + "flos": 47118117456000.0, + "grad_norm": 1.63326102981393, + "language_loss": 0.77217656, + "learning_rate": 3.7933226838306356e-06, + "loss": 0.79334259, + "num_input_tokens_seen": 169116930, + "router_z_loss_clip": 0.54296875, + "router_z_loss_mlp": 0.15606689, + "step": 5914, + "time_per_iteration": 2.553727388381958 + }, + { + "auxiliary_loss_clip": 0.0100979, + "auxiliary_loss_mlp": 0.01004127, + "balance_loss_clip": 1.00235999, + "balance_loss_mlp": 1.00311339, + "epoch": 0.171638326272416, + "flos": 60036852222720.0, + "grad_norm": 0.703873813483271, + "language_loss": 0.5059067, + "learning_rate": 3.7932394620296167e-06, + "loss": 0.52604586, + "num_input_tokens_seen": 169174730, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.01013184, + "step": 5915, + "time_per_iteration": 2.930753469467163 + }, + { + "auxiliary_loss_clip": 0.01091188, + "auxiliary_loss_mlp": 0.01041474, + "balance_loss_clip": 1.02871513, + "balance_loss_mlp": 1.02017689, + "epoch": 0.17166734374093204, + "flos": 49117838065920.0, + "grad_norm": 2.1691548719773097, + "language_loss": 0.86620283, + "learning_rate": 3.7931562243899816e-06, + "loss": 0.88752949, + "num_input_tokens_seen": 169194995, + "router_z_loss_clip": 0.625, + "router_z_loss_mlp": 0.21270752, + "step": 5916, + "time_per_iteration": 2.719513416290283 + }, + { + "auxiliary_loss_clip": 0.01079876, + "auxiliary_loss_mlp": 0.01029301, + "balance_loss_clip": 1.0271908, + "balance_loss_mlp": 1.01426315, + "epoch": 0.1716963612094481, + "flos": 49701376396800.0, + "grad_norm": 2.433554380035348, + "language_loss": 0.73302138, + "learning_rate": 3.7930729709124643e-06, + "loss": 0.75411314, + "num_input_tokens_seen": 169214065, + "router_z_loss_clip": 0.52636719, + "router_z_loss_mlp": 0.15039062, + "step": 5917, + "time_per_iteration": 2.646383285522461 + }, + { + "auxiliary_loss_clip": 0.01082505, + "auxiliary_loss_mlp": 0.01033085, + "balance_loss_clip": 1.02662826, + "balance_loss_mlp": 1.01619864, + "epoch": 0.17172537867796414, + "flos": 27336364289280.0, + "grad_norm": 3.134786992420515, + "language_loss": 0.72172868, + "learning_rate": 3.7929897015978013e-06, + "loss": 0.74288458, + "num_input_tokens_seen": 169228850, + "router_z_loss_clip": 0.55883789, + "router_z_loss_mlp": 0.16888428, + "step": 5918, + "time_per_iteration": 2.456639051437378 + }, + { + "auxiliary_loss_clip": 0.0100883, + "auxiliary_loss_mlp": 0.0100191, + "balance_loss_clip": 1.00156832, + "balance_loss_mlp": 1.00095034, + "epoch": 0.1717543961464802, + "flos": 74769013117440.0, + "grad_norm": 0.6400455059059358, + "language_loss": 0.49466613, + "learning_rate": 3.792906416446728e-06, + "loss": 0.51477361, + "num_input_tokens_seen": 169294270, + "router_z_loss_clip": 0.07275391, + "router_z_loss_mlp": 0.00958252, + "step": 5919, + "time_per_iteration": 3.185598373413086 + }, + { + "auxiliary_loss_clip": 0.0100849, + "auxiliary_loss_mlp": 0.01000941, + "balance_loss_clip": 1.00109816, + "balance_loss_mlp": 0.99991024, + "epoch": 0.17178341361499622, + "flos": 73019934224640.0, + "grad_norm": 0.678993979671545, + "language_loss": 0.48540646, + "learning_rate": 3.7928231154599796e-06, + "loss": 0.50550073, + "num_input_tokens_seen": 169356315, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.01031494, + "step": 5920, + "time_per_iteration": 3.0713083744049072 + }, + { + "auxiliary_loss_clip": 0.01008386, + "auxiliary_loss_mlp": 0.0100352, + "balance_loss_clip": 1.00101519, + "balance_loss_mlp": 1.00254798, + "epoch": 0.17181243108351227, + "flos": 63210666677760.0, + "grad_norm": 0.6163599156836668, + "language_loss": 0.47019082, + "learning_rate": 3.7927397986382913e-06, + "loss": 0.49030989, + "num_input_tokens_seen": 169420745, + "router_z_loss_clip": 0.07373047, + "router_z_loss_mlp": 0.00970459, + "step": 5921, + "time_per_iteration": 3.0188674926757812 + }, + { + "auxiliary_loss_clip": 0.0108771, + "auxiliary_loss_mlp": 0.01034771, + "balance_loss_clip": 1.03135908, + "balance_loss_mlp": 1.01779604, + "epoch": 0.17184144855202832, + "flos": 74590189666560.0, + "grad_norm": 2.0140144387043883, + "language_loss": 0.81236398, + "learning_rate": 3.7926564659824003e-06, + "loss": 0.83358878, + "num_input_tokens_seen": 169445415, + "router_z_loss_clip": 0.5637207, + "router_z_loss_mlp": 0.16973877, + "step": 5922, + "time_per_iteration": 2.7783703804016113 + }, + { + "auxiliary_loss_clip": 0.0100849, + "auxiliary_loss_mlp": 0.01002033, + "balance_loss_clip": 1.00111139, + "balance_loss_mlp": 1.00116909, + "epoch": 0.17187046602054437, + "flos": 72406125878400.0, + "grad_norm": 0.66556850152212, + "language_loss": 0.48409575, + "learning_rate": 3.792573117493042e-06, + "loss": 0.50420099, + "num_input_tokens_seen": 169512865, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00866699, + "step": 5923, + "time_per_iteration": 3.174569845199585 + }, + { + "auxiliary_loss_clip": 0.01079148, + "auxiliary_loss_mlp": 0.01037255, + "balance_loss_clip": 1.0255605, + "balance_loss_mlp": 1.01933837, + "epoch": 0.17189948348906042, + "flos": 43974433370880.0, + "grad_norm": 2.1480960885477183, + "language_loss": 0.85092998, + "learning_rate": 3.792489753170953e-06, + "loss": 0.87209404, + "num_input_tokens_seen": 169534260, + "router_z_loss_clip": 0.53564453, + "router_z_loss_mlp": 0.17932129, + "step": 5924, + "time_per_iteration": 2.5910561084747314 + }, + { + "auxiliary_loss_clip": 0.01083777, + "auxiliary_loss_mlp": 0.01044434, + "balance_loss_clip": 1.02867675, + "balance_loss_mlp": 1.02583122, + "epoch": 0.17192850095757647, + "flos": 18431592001920.0, + "grad_norm": 3.08545435786216, + "language_loss": 1.14249802, + "learning_rate": 3.792406373016868e-06, + "loss": 1.16378021, + "num_input_tokens_seen": 169546285, + "router_z_loss_clip": 0.55078125, + "router_z_loss_mlp": 0.18621826, + "step": 5925, + "time_per_iteration": 2.381782293319702 + }, + { + "auxiliary_loss_clip": 0.01078785, + "auxiliary_loss_mlp": 0.01032235, + "balance_loss_clip": 1.02650285, + "balance_loss_mlp": 1.01637471, + "epoch": 0.1719575184260925, + "flos": 16356528944640.0, + "grad_norm": 2.4766834968533815, + "language_loss": 0.74067795, + "learning_rate": 3.792322977031525e-06, + "loss": 0.76178813, + "num_input_tokens_seen": 169563250, + "router_z_loss_clip": 0.5234375, + "router_z_loss_mlp": 0.15844727, + "step": 5926, + "time_per_iteration": 2.375640392303467 + }, + { + "auxiliary_loss_clip": 0.01082957, + "auxiliary_loss_mlp": 0.0103788, + "balance_loss_clip": 1.0287261, + "balance_loss_mlp": 1.02132773, + "epoch": 0.17198653589460855, + "flos": 33394144158720.0, + "grad_norm": 2.136368105167085, + "language_loss": 0.9030062, + "learning_rate": 3.7922395652156607e-06, + "loss": 0.92421454, + "num_input_tokens_seen": 169581370, + "router_z_loss_clip": 0.54296875, + "router_z_loss_mlp": 0.16546631, + "step": 5927, + "time_per_iteration": 2.506521463394165 + }, + { + "auxiliary_loss_clip": 0.01010053, + "auxiliary_loss_mlp": 0.01004857, + "balance_loss_clip": 1.002226, + "balance_loss_mlp": 1.00394487, + "epoch": 0.1720155533631246, + "flos": 67727694936960.0, + "grad_norm": 0.6986568151916226, + "language_loss": 0.46247691, + "learning_rate": 3.7921561375700107e-06, + "loss": 0.48262602, + "num_input_tokens_seen": 169641765, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.00909424, + "step": 5928, + "time_per_iteration": 3.0088560581207275 + }, + { + "auxiliary_loss_clip": 0.01080322, + "auxiliary_loss_mlp": 0.01032855, + "balance_loss_clip": 1.02790833, + "balance_loss_mlp": 1.01677322, + "epoch": 0.17204457083164065, + "flos": 35983826789760.0, + "grad_norm": 1.8933503579123871, + "language_loss": 0.87739003, + "learning_rate": 3.7920726940953127e-06, + "loss": 0.89852184, + "num_input_tokens_seen": 169661490, + "router_z_loss_clip": 0.52416992, + "router_z_loss_mlp": 0.16088867, + "step": 5929, + "time_per_iteration": 2.5253982543945312 + }, + { + "auxiliary_loss_clip": 0.01009298, + "auxiliary_loss_mlp": 0.01001259, + "balance_loss_clip": 1.00189376, + "balance_loss_mlp": 1.00043058, + "epoch": 0.1720735883001567, + "flos": 58714168164480.0, + "grad_norm": 0.7207094143346283, + "language_loss": 0.48457131, + "learning_rate": 3.7919892347923036e-06, + "loss": 0.50467682, + "num_input_tokens_seen": 169723845, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00830078, + "step": 5930, + "time_per_iteration": 3.0561435222625732 + }, + { + "auxiliary_loss_clip": 0.01082129, + "auxiliary_loss_mlp": 0.01041568, + "balance_loss_clip": 1.02762389, + "balance_loss_mlp": 1.02365673, + "epoch": 0.17210260576867273, + "flos": 14311491523200.0, + "grad_norm": 2.644445781855447, + "language_loss": 0.80910033, + "learning_rate": 3.7919057596617207e-06, + "loss": 0.83033729, + "num_input_tokens_seen": 169736185, + "router_z_loss_clip": 0.54467773, + "router_z_loss_mlp": 0.17926025, + "step": 5931, + "time_per_iteration": 2.3920648097991943 + }, + { + "auxiliary_loss_clip": 0.01083603, + "auxiliary_loss_mlp": 0.01039872, + "balance_loss_clip": 1.03022575, + "balance_loss_mlp": 1.0228374, + "epoch": 0.17213162323718878, + "flos": 28979551428480.0, + "grad_norm": 1.8484292128897717, + "language_loss": 0.81874484, + "learning_rate": 3.791822268704301e-06, + "loss": 0.83997953, + "num_input_tokens_seen": 169755475, + "router_z_loss_clip": 0.53344727, + "router_z_loss_mlp": 0.17053223, + "step": 5932, + "time_per_iteration": 2.7642509937286377 + }, + { + "auxiliary_loss_clip": 0.01008196, + "auxiliary_loss_mlp": 0.01002024, + "balance_loss_clip": 1.00082994, + "balance_loss_mlp": 1.00118411, + "epoch": 0.17216064070570483, + "flos": 65623897964160.0, + "grad_norm": 0.6682579149172391, + "language_loss": 0.47805965, + "learning_rate": 3.791738761920781e-06, + "loss": 0.49816185, + "num_input_tokens_seen": 169823805, + "router_z_loss_clip": 0.07373047, + "router_z_loss_mlp": 0.00842285, + "step": 5933, + "time_per_iteration": 3.116401433944702 + }, + { + "auxiliary_loss_clip": 0.0108322, + "auxiliary_loss_mlp": 0.01027914, + "balance_loss_clip": 1.02724636, + "balance_loss_mlp": 1.01012206, + "epoch": 0.17218965817422088, + "flos": 32298534783360.0, + "grad_norm": 2.527316611101611, + "language_loss": 0.80692279, + "learning_rate": 3.7916552393119004e-06, + "loss": 0.8280341, + "num_input_tokens_seen": 169840350, + "router_z_loss_clip": 0.55957031, + "router_z_loss_mlp": 0.17779541, + "step": 5934, + "time_per_iteration": 7.109787702560425 + }, + { + "auxiliary_loss_clip": 0.01008941, + "auxiliary_loss_mlp": 0.01000634, + "balance_loss_clip": 1.00148296, + "balance_loss_mlp": 0.99970996, + "epoch": 0.17221867564273693, + "flos": 58679988076800.0, + "grad_norm": 0.6792233492983785, + "language_loss": 0.50762552, + "learning_rate": 3.791571700878395e-06, + "loss": 0.52772129, + "num_input_tokens_seen": 169902070, + "router_z_loss_clip": 0.07470703, + "router_z_loss_mlp": 0.00921631, + "step": 5935, + "time_per_iteration": 3.014580726623535 + }, + { + "auxiliary_loss_clip": 0.01077398, + "auxiliary_loss_mlp": 0.0103262, + "balance_loss_clip": 1.02386856, + "balance_loss_mlp": 1.01657474, + "epoch": 0.172247693111253, + "flos": 34596902096640.0, + "grad_norm": 2.0145460385110407, + "language_loss": 0.83537716, + "learning_rate": 3.7914881466210035e-06, + "loss": 0.85647732, + "num_input_tokens_seen": 169921545, + "router_z_loss_clip": 0.53637695, + "router_z_loss_mlp": 0.16046143, + "step": 5936, + "time_per_iteration": 2.53751802444458 + }, + { + "auxiliary_loss_clip": 0.01009377, + "auxiliary_loss_mlp": 0.01000157, + "balance_loss_clip": 1.00171518, + "balance_loss_mlp": 0.99929887, + "epoch": 0.172276710579769, + "flos": 62327888150400.0, + "grad_norm": 0.7051952039256442, + "language_loss": 0.46049255, + "learning_rate": 3.791404576540464e-06, + "loss": 0.4805879, + "num_input_tokens_seen": 169982565, + "router_z_loss_clip": 0.07666016, + "router_z_loss_mlp": 0.00860596, + "step": 5937, + "time_per_iteration": 2.9550893306732178 + }, + { + "auxiliary_loss_clip": 0.01010927, + "auxiliary_loss_mlp": 0.01001534, + "balance_loss_clip": 1.00339866, + "balance_loss_mlp": 1.0006038, + "epoch": 0.17230572804828506, + "flos": 63947892280320.0, + "grad_norm": 0.642327304885992, + "language_loss": 0.49007666, + "learning_rate": 3.791320990637514e-06, + "loss": 0.51020122, + "num_input_tokens_seen": 170046125, + "router_z_loss_clip": 0.07519531, + "router_z_loss_mlp": 0.00927734, + "step": 5938, + "time_per_iteration": 3.0529704093933105 + }, + { + "auxiliary_loss_clip": 0.01078367, + "auxiliary_loss_mlp": 0.01039388, + "balance_loss_clip": 1.02464211, + "balance_loss_mlp": 1.0218879, + "epoch": 0.1723347455168011, + "flos": 32007428933760.0, + "grad_norm": 2.1651611123029566, + "language_loss": 0.80505705, + "learning_rate": 3.7912373889128926e-06, + "loss": 0.82623458, + "num_input_tokens_seen": 170062945, + "router_z_loss_clip": 0.53735352, + "router_z_loss_mlp": 0.17492676, + "step": 5939, + "time_per_iteration": 2.481372356414795 + }, + { + "auxiliary_loss_clip": 0.01082199, + "auxiliary_loss_mlp": 0.01029926, + "balance_loss_clip": 1.02631283, + "balance_loss_mlp": 1.0118655, + "epoch": 0.17236376298531716, + "flos": 28325383683840.0, + "grad_norm": 2.212948273231646, + "language_loss": 0.96537143, + "learning_rate": 3.7911537713673374e-06, + "loss": 0.98649269, + "num_input_tokens_seen": 170079885, + "router_z_loss_clip": 0.55834961, + "router_z_loss_mlp": 0.18066406, + "step": 5940, + "time_per_iteration": 2.4790003299713135 + }, + { + "auxiliary_loss_clip": 0.01083417, + "auxiliary_loss_mlp": 0.01035984, + "balance_loss_clip": 1.02757335, + "balance_loss_mlp": 1.01816237, + "epoch": 0.17239278045383322, + "flos": 18727759998720.0, + "grad_norm": 3.08713458370244, + "language_loss": 0.80477619, + "learning_rate": 3.7910701380015872e-06, + "loss": 0.82597017, + "num_input_tokens_seen": 170094190, + "router_z_loss_clip": 0.55859375, + "router_z_loss_mlp": 0.17810059, + "step": 5941, + "time_per_iteration": 2.3434507846832275 + }, + { + "auxiliary_loss_clip": 0.01084484, + "auxiliary_loss_mlp": 0.01036189, + "balance_loss_clip": 1.02736878, + "balance_loss_mlp": 1.01968479, + "epoch": 0.17242179792234927, + "flos": 11609842561920.0, + "grad_norm": 4.485858327948909, + "language_loss": 0.9573524, + "learning_rate": 3.790986488816381e-06, + "loss": 0.97855914, + "num_input_tokens_seen": 170104810, + "router_z_loss_clip": 0.57080078, + "router_z_loss_mlp": 0.1651001, + "step": 5942, + "time_per_iteration": 2.3718175888061523 + }, + { + "auxiliary_loss_clip": 0.01011689, + "auxiliary_loss_mlp": 0.0100101, + "balance_loss_clip": 1.00427294, + "balance_loss_mlp": 1.00011027, + "epoch": 0.1724508153908653, + "flos": 55547789829120.0, + "grad_norm": 0.6761575240694566, + "language_loss": 0.47925946, + "learning_rate": 3.7909028238124572e-06, + "loss": 0.49938643, + "num_input_tokens_seen": 170166425, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00897217, + "step": 5943, + "time_per_iteration": 3.2590456008911133 + }, + { + "auxiliary_loss_clip": 0.01079441, + "auxiliary_loss_mlp": 0.01035933, + "balance_loss_clip": 1.02633834, + "balance_loss_mlp": 1.01939321, + "epoch": 0.17247983285938134, + "flos": 20988106974720.0, + "grad_norm": 2.31809441940397, + "language_loss": 0.66238242, + "learning_rate": 3.790819142990555e-06, + "loss": 0.68353617, + "num_input_tokens_seen": 170178940, + "router_z_loss_clip": 0.53051758, + "router_z_loss_mlp": 0.16534424, + "step": 5944, + "time_per_iteration": 7.478569507598877 + }, + { + "auxiliary_loss_clip": 0.01086918, + "auxiliary_loss_mlp": 0.01027993, + "balance_loss_clip": 1.03105354, + "balance_loss_mlp": 1.01160216, + "epoch": 0.1725088503278974, + "flos": 30330201352320.0, + "grad_norm": 1.8842510173749802, + "language_loss": 0.73603809, + "learning_rate": 3.7907354463514137e-06, + "loss": 0.75718719, + "num_input_tokens_seen": 170199570, + "router_z_loss_clip": 0.55761719, + "router_z_loss_mlp": 0.1640625, + "step": 5945, + "time_per_iteration": 2.5095274448394775 + }, + { + "auxiliary_loss_clip": 0.01077265, + "auxiliary_loss_mlp": 0.0103383, + "balance_loss_clip": 1.02651191, + "balance_loss_mlp": 1.01689041, + "epoch": 0.17253786779641345, + "flos": 15113830544640.0, + "grad_norm": 2.7872724745890967, + "language_loss": 0.74395633, + "learning_rate": 3.7906517338957718e-06, + "loss": 0.76506734, + "num_input_tokens_seen": 170212160, + "router_z_loss_clip": 0.50756836, + "router_z_loss_mlp": 0.16931152, + "step": 5946, + "time_per_iteration": 2.3757996559143066 + }, + { + "auxiliary_loss_clip": 0.01081304, + "auxiliary_loss_mlp": 0.0103434, + "balance_loss_clip": 1.02914011, + "balance_loss_mlp": 1.01748919, + "epoch": 0.1725668852649295, + "flos": 25622303356800.0, + "grad_norm": 2.099828268537412, + "language_loss": 0.86159861, + "learning_rate": 3.7905680056243696e-06, + "loss": 0.88275504, + "num_input_tokens_seen": 170228050, + "router_z_loss_clip": 0.52172852, + "router_z_loss_mlp": 0.16864014, + "step": 5947, + "time_per_iteration": 2.4885640144348145 + }, + { + "auxiliary_loss_clip": 0.01080286, + "auxiliary_loss_mlp": 0.01028763, + "balance_loss_clip": 1.02583694, + "balance_loss_mlp": 1.01221704, + "epoch": 0.17259590273344552, + "flos": 12560108480640.0, + "grad_norm": 2.0020010431969415, + "language_loss": 0.58121669, + "learning_rate": 3.790484261537946e-06, + "loss": 0.6023072, + "num_input_tokens_seen": 170239915, + "router_z_loss_clip": 0.54467773, + "router_z_loss_mlp": 0.16534424, + "step": 5948, + "time_per_iteration": 2.360893726348877 + }, + { + "auxiliary_loss_clip": 0.01081835, + "auxiliary_loss_mlp": 0.01034899, + "balance_loss_clip": 1.0260973, + "balance_loss_mlp": 1.01730394, + "epoch": 0.17262492020196157, + "flos": 41894203432320.0, + "grad_norm": 1.8756236251745488, + "language_loss": 0.72434109, + "learning_rate": 3.7904005016372413e-06, + "loss": 0.74550843, + "num_input_tokens_seen": 170258735, + "router_z_loss_clip": 0.55712891, + "router_z_loss_mlp": 0.17578125, + "step": 5949, + "time_per_iteration": 2.584057092666626 + }, + { + "auxiliary_loss_clip": 0.01087591, + "auxiliary_loss_mlp": 0.01036359, + "balance_loss_clip": 1.02907038, + "balance_loss_mlp": 1.01674879, + "epoch": 0.17265393767047763, + "flos": 44927073262080.0, + "grad_norm": 2.1041842585762187, + "language_loss": 0.93881291, + "learning_rate": 3.7903167259229944e-06, + "loss": 0.96005237, + "num_input_tokens_seen": 170279315, + "router_z_loss_clip": 0.58496094, + "router_z_loss_mlp": 0.19604492, + "step": 5950, + "time_per_iteration": 2.6050360202789307 + }, + { + "auxiliary_loss_clip": 0.01079976, + "auxiliary_loss_mlp": 0.01043069, + "balance_loss_clip": 1.02694857, + "balance_loss_mlp": 1.02640915, + "epoch": 0.17268295513899368, + "flos": 34091464210560.0, + "grad_norm": 3.4447778001878993, + "language_loss": 0.78068221, + "learning_rate": 3.790232934395946e-06, + "loss": 0.80191267, + "num_input_tokens_seen": 170299645, + "router_z_loss_clip": 0.52978516, + "router_z_loss_mlp": 0.16662598, + "step": 5951, + "time_per_iteration": 2.583496332168579 + }, + { + "auxiliary_loss_clip": 0.01082082, + "auxiliary_loss_mlp": 0.01033013, + "balance_loss_clip": 1.02693081, + "balance_loss_mlp": 1.01618695, + "epoch": 0.17271197260750973, + "flos": 38512934478720.0, + "grad_norm": 1.5926746645054688, + "language_loss": 0.67258418, + "learning_rate": 3.7901491270568354e-06, + "loss": 0.69373512, + "num_input_tokens_seen": 170320760, + "router_z_loss_clip": 0.55053711, + "router_z_loss_mlp": 0.16833496, + "step": 5952, + "time_per_iteration": 2.5558888912200928 + }, + { + "auxiliary_loss_clip": 0.01076999, + "auxiliary_loss_mlp": 0.01032224, + "balance_loss_clip": 1.02654648, + "balance_loss_mlp": 1.01648855, + "epoch": 0.17274099007602578, + "flos": 27777805920000.0, + "grad_norm": 1.6968712842223013, + "language_loss": 0.75772202, + "learning_rate": 3.790065303906404e-06, + "loss": 0.77881426, + "num_input_tokens_seen": 170346050, + "router_z_loss_clip": 0.50390625, + "router_z_loss_mlp": 0.15734863, + "step": 5953, + "time_per_iteration": 2.6451094150543213 + }, + { + "auxiliary_loss_clip": 0.01010178, + "auxiliary_loss_mlp": 0.01001105, + "balance_loss_clip": 1.00309324, + "balance_loss_mlp": 1.00016928, + "epoch": 0.1727700075445418, + "flos": 74769850990080.0, + "grad_norm": 0.6430564396052326, + "language_loss": 0.46245119, + "learning_rate": 3.7899814649453915e-06, + "loss": 0.482564, + "num_input_tokens_seen": 170412375, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00933838, + "step": 5954, + "time_per_iteration": 3.275791645050049 + }, + { + "auxiliary_loss_clip": 0.01082966, + "auxiliary_loss_mlp": 0.01037923, + "balance_loss_clip": 1.02857542, + "balance_loss_mlp": 1.02032769, + "epoch": 0.17279902501305786, + "flos": 18143872554240.0, + "grad_norm": 2.9181699405551136, + "language_loss": 0.94167697, + "learning_rate": 3.7898976101745383e-06, + "loss": 0.96288586, + "num_input_tokens_seen": 170428060, + "router_z_loss_clip": 0.54370117, + "router_z_loss_mlp": 0.17590332, + "step": 5955, + "time_per_iteration": 2.47979474067688 + }, + { + "auxiliary_loss_clip": 0.01085879, + "auxiliary_loss_mlp": 0.01039646, + "balance_loss_clip": 1.02712727, + "balance_loss_mlp": 1.02031636, + "epoch": 0.1728280424815739, + "flos": 38431447632000.0, + "grad_norm": 4.787711759914733, + "language_loss": 0.92291278, + "learning_rate": 3.789813739594584e-06, + "loss": 0.94416797, + "num_input_tokens_seen": 170446315, + "router_z_loss_clip": 0.58740234, + "router_z_loss_mlp": 0.19342041, + "step": 5956, + "time_per_iteration": 2.552238702774048 + }, + { + "auxiliary_loss_clip": 0.01083745, + "auxiliary_loss_mlp": 0.01032789, + "balance_loss_clip": 1.02682352, + "balance_loss_mlp": 1.01575446, + "epoch": 0.17285705995008996, + "flos": 62326563384960.0, + "grad_norm": 2.0060720269894703, + "language_loss": 0.89602256, + "learning_rate": 3.789729853206272e-06, + "loss": 0.91718787, + "num_input_tokens_seen": 170469490, + "router_z_loss_clip": 0.56933594, + "router_z_loss_mlp": 0.17028809, + "step": 5957, + "time_per_iteration": 2.696261167526245 + }, + { + "auxiliary_loss_clip": 0.01076628, + "auxiliary_loss_mlp": 0.01033028, + "balance_loss_clip": 1.02550483, + "balance_loss_mlp": 1.01649356, + "epoch": 0.172886077418606, + "flos": 31647823263360.0, + "grad_norm": 1.8011909608326284, + "language_loss": 0.77541238, + "learning_rate": 3.7896459510103406e-06, + "loss": 0.79650891, + "num_input_tokens_seen": 170491935, + "router_z_loss_clip": 0.51123047, + "router_z_loss_mlp": 0.16534424, + "step": 5958, + "time_per_iteration": 2.556297540664673 + }, + { + "auxiliary_loss_clip": 0.01077906, + "auxiliary_loss_mlp": 0.01030226, + "balance_loss_clip": 1.02426839, + "balance_loss_mlp": 1.01344132, + "epoch": 0.17291509488712203, + "flos": 25297855292160.0, + "grad_norm": 1.9763514397164201, + "language_loss": 0.79278791, + "learning_rate": 3.7895620330075326e-06, + "loss": 0.81386924, + "num_input_tokens_seen": 170509955, + "router_z_loss_clip": 0.53662109, + "router_z_loss_mlp": 0.16784668, + "step": 5959, + "time_per_iteration": 2.4818859100341797 + }, + { + "auxiliary_loss_clip": 0.01075539, + "auxiliary_loss_mlp": 0.01030629, + "balance_loss_clip": 1.02516246, + "balance_loss_mlp": 1.0154779, + "epoch": 0.17294411235563809, + "flos": 18325141666560.0, + "grad_norm": 1.960990760770095, + "language_loss": 0.69708496, + "learning_rate": 3.7894780991985887e-06, + "loss": 0.71814668, + "num_input_tokens_seen": 170523850, + "router_z_loss_clip": 0.50317383, + "router_z_loss_mlp": 0.15136719, + "step": 5960, + "time_per_iteration": 2.370969772338867 + }, + { + "auxiliary_loss_clip": 0.01009863, + "auxiliary_loss_mlp": 0.00999951, + "balance_loss_clip": 1.00238442, + "balance_loss_mlp": 0.99904478, + "epoch": 0.17297312982415414, + "flos": 74772783544320.0, + "grad_norm": 0.6338560762575113, + "language_loss": 0.42835033, + "learning_rate": 3.7893941495842494e-06, + "loss": 0.44844848, + "num_input_tokens_seen": 170592165, + "router_z_loss_clip": 0.07470703, + "router_z_loss_mlp": 0.0090332, + "step": 5961, + "time_per_iteration": 3.1355953216552734 + }, + { + "auxiliary_loss_clip": 0.01010135, + "auxiliary_loss_mlp": 0.00999953, + "balance_loss_clip": 1.00282884, + "balance_loss_mlp": 0.99904734, + "epoch": 0.1730021472926702, + "flos": 63785512080000.0, + "grad_norm": 0.6541181301350103, + "language_loss": 0.4873513, + "learning_rate": 3.7893101841652574e-06, + "loss": 0.50745225, + "num_input_tokens_seen": 170655785, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.0090332, + "step": 5962, + "time_per_iteration": 3.1002535820007324 + }, + { + "auxiliary_loss_clip": 0.01079652, + "auxiliary_loss_mlp": 0.01031392, + "balance_loss_clip": 1.02683473, + "balance_loss_mlp": 1.01535869, + "epoch": 0.17303116476118624, + "flos": 11210540808960.0, + "grad_norm": 3.005312298729852, + "language_loss": 0.73241282, + "learning_rate": 3.7892262029423534e-06, + "loss": 0.75352329, + "num_input_tokens_seen": 170668735, + "router_z_loss_clip": 0.52832031, + "router_z_loss_mlp": 0.16040039, + "step": 5963, + "time_per_iteration": 2.3620128631591797 + }, + { + "auxiliary_loss_clip": 0.0107897, + "auxiliary_loss_mlp": 0.01033703, + "balance_loss_clip": 1.02588534, + "balance_loss_mlp": 1.01718688, + "epoch": 0.1730601822297023, + "flos": 22413785143680.0, + "grad_norm": 2.5259893725097617, + "language_loss": 0.89081997, + "learning_rate": 3.7891422059162804e-06, + "loss": 0.91194671, + "num_input_tokens_seen": 170684035, + "router_z_loss_clip": 0.53100586, + "router_z_loss_mlp": 0.16503906, + "step": 5964, + "time_per_iteration": 2.3975348472595215 + }, + { + "auxiliary_loss_clip": 0.01080395, + "auxiliary_loss_mlp": 0.01032607, + "balance_loss_clip": 1.02536356, + "balance_loss_mlp": 1.0144279, + "epoch": 0.17308919969821832, + "flos": 74734101757440.0, + "grad_norm": 1.6399841748249617, + "language_loss": 0.82077819, + "learning_rate": 3.789058193087778e-06, + "loss": 0.84190822, + "num_input_tokens_seen": 170714490, + "router_z_loss_clip": 0.54980469, + "router_z_loss_mlp": 0.18200684, + "step": 5965, + "time_per_iteration": 2.804190158843994 + }, + { + "auxiliary_loss_clip": 0.0107508, + "auxiliary_loss_mlp": 0.01031248, + "balance_loss_clip": 1.0256474, + "balance_loss_mlp": 1.01565504, + "epoch": 0.17311821716673437, + "flos": 17666121242880.0, + "grad_norm": 2.054280761532021, + "language_loss": 0.72503698, + "learning_rate": 3.788974164457591e-06, + "loss": 0.74610025, + "num_input_tokens_seen": 170729325, + "router_z_loss_clip": 0.49462891, + "router_z_loss_mlp": 0.15594482, + "step": 5966, + "time_per_iteration": 2.393505096435547 + }, + { + "auxiliary_loss_clip": 0.0100883, + "auxiliary_loss_mlp": 0.01001028, + "balance_loss_clip": 1.00144315, + "balance_loss_mlp": 1.00018752, + "epoch": 0.17314723463525042, + "flos": 55495770036480.0, + "grad_norm": 0.7070614070758446, + "language_loss": 0.48337877, + "learning_rate": 3.78889012002646e-06, + "loss": 0.50347733, + "num_input_tokens_seen": 170782530, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00842285, + "step": 5967, + "time_per_iteration": 2.8897645473480225 + }, + { + "auxiliary_loss_clip": 0.01008915, + "auxiliary_loss_mlp": 0.01001198, + "balance_loss_clip": 1.00165987, + "balance_loss_mlp": 1.00033998, + "epoch": 0.17317625210376647, + "flos": 63131937828480.0, + "grad_norm": 0.6609647887078282, + "language_loss": 0.53718108, + "learning_rate": 3.788806059795127e-06, + "loss": 0.55728221, + "num_input_tokens_seen": 170841870, + "router_z_loss_clip": 0.07275391, + "router_z_loss_mlp": 0.00860596, + "step": 5968, + "time_per_iteration": 3.0356311798095703 + }, + { + "auxiliary_loss_clip": 0.01009421, + "auxiliary_loss_mlp": 0.01000611, + "balance_loss_clip": 1.00197184, + "balance_loss_mlp": 0.99981225, + "epoch": 0.17320526957228252, + "flos": 69318581126400.0, + "grad_norm": 0.7262438406905016, + "language_loss": 0.49537441, + "learning_rate": 3.7887219837643355e-06, + "loss": 0.5154748, + "num_input_tokens_seen": 170905640, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00799561, + "step": 5969, + "time_per_iteration": 3.1593613624572754 + }, + { + "auxiliary_loss_clip": 0.01082351, + "auxiliary_loss_mlp": 0.01038021, + "balance_loss_clip": 1.02666712, + "balance_loss_mlp": 1.02155781, + "epoch": 0.17323428704079857, + "flos": 17121301476480.0, + "grad_norm": 5.439712125522173, + "language_loss": 0.71356195, + "learning_rate": 3.7886378919348274e-06, + "loss": 0.73476565, + "num_input_tokens_seen": 170920345, + "router_z_loss_clip": 0.55664062, + "router_z_loss_mlp": 0.16461182, + "step": 5970, + "time_per_iteration": 2.4164466857910156 + }, + { + "auxiliary_loss_clip": 0.01009172, + "auxiliary_loss_mlp": 0.01001732, + "balance_loss_clip": 1.00163102, + "balance_loss_mlp": 1.00086164, + "epoch": 0.1732633045093146, + "flos": 74770933242240.0, + "grad_norm": 0.647064904313085, + "language_loss": 0.48765945, + "learning_rate": 3.7885537843073464e-06, + "loss": 0.50776845, + "num_input_tokens_seen": 170984390, + "router_z_loss_clip": 0.07568359, + "router_z_loss_mlp": 0.00872803, + "step": 5971, + "time_per_iteration": 3.0322813987731934 + }, + { + "auxiliary_loss_clip": 0.01073909, + "auxiliary_loss_mlp": 0.01030792, + "balance_loss_clip": 1.02435744, + "balance_loss_mlp": 1.01556873, + "epoch": 0.17329232197783065, + "flos": 28614080649600.0, + "grad_norm": 1.8501170738839603, + "language_loss": 0.76617169, + "learning_rate": 3.788469660882634e-06, + "loss": 0.78721869, + "num_input_tokens_seen": 171001155, + "router_z_loss_clip": 0.49536133, + "router_z_loss_mlp": 0.15234375, + "step": 5972, + "time_per_iteration": 2.4336886405944824 + }, + { + "auxiliary_loss_clip": 0.01079819, + "auxiliary_loss_mlp": 0.01035568, + "balance_loss_clip": 1.02513969, + "balance_loss_mlp": 1.01804423, + "epoch": 0.1733213394463467, + "flos": 22026004139520.0, + "grad_norm": 2.6771379148125725, + "language_loss": 0.83224607, + "learning_rate": 3.7883855216614335e-06, + "loss": 0.85339993, + "num_input_tokens_seen": 171017045, + "router_z_loss_clip": 0.54711914, + "router_z_loss_mlp": 0.17529297, + "step": 5973, + "time_per_iteration": 2.382383346557617 + }, + { + "auxiliary_loss_clip": 0.01010268, + "auxiliary_loss_mlp": 0.01002498, + "balance_loss_clip": 1.00282288, + "balance_loss_mlp": 1.00150883, + "epoch": 0.17335035691486275, + "flos": 56019850588800.0, + "grad_norm": 0.665046181214339, + "language_loss": 0.49343687, + "learning_rate": 3.7883013666444886e-06, + "loss": 0.51356453, + "num_input_tokens_seen": 171073410, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.0098877, + "step": 5974, + "time_per_iteration": 2.891352891921997 + }, + { + "auxiliary_loss_clip": 0.01078696, + "auxiliary_loss_mlp": 0.01032863, + "balance_loss_clip": 1.0245254, + "balance_loss_mlp": 1.01508927, + "epoch": 0.1733793743833788, + "flos": 15918787918080.0, + "grad_norm": 3.5665240814850896, + "language_loss": 0.71626747, + "learning_rate": 3.7882171958325426e-06, + "loss": 0.73738313, + "num_input_tokens_seen": 171088920, + "router_z_loss_clip": 0.54125977, + "router_z_loss_mlp": 0.17785645, + "step": 5975, + "time_per_iteration": 2.346560001373291 + }, + { + "auxiliary_loss_clip": 0.01012306, + "auxiliary_loss_mlp": 0.01001252, + "balance_loss_clip": 1.00483823, + "balance_loss_mlp": 1.00025666, + "epoch": 0.17340839185189483, + "flos": 74771561646720.0, + "grad_norm": 0.6123946095176716, + "language_loss": 0.47555852, + "learning_rate": 3.7881330092263386e-06, + "loss": 0.49569413, + "num_input_tokens_seen": 171153495, + "router_z_loss_clip": 0.07519531, + "router_z_loss_mlp": 0.00994873, + "step": 5976, + "time_per_iteration": 3.1545181274414062 + }, + { + "auxiliary_loss_clip": 0.01089845, + "auxiliary_loss_mlp": 0.01039273, + "balance_loss_clip": 1.02963805, + "balance_loss_mlp": 1.02054572, + "epoch": 0.17343740932041088, + "flos": 46310262439680.0, + "grad_norm": 2.222610191301845, + "language_loss": 0.76221639, + "learning_rate": 3.7880488068266205e-06, + "loss": 0.78350759, + "num_input_tokens_seen": 171170025, + "router_z_loss_clip": 0.6015625, + "router_z_loss_mlp": 0.18737793, + "step": 5977, + "time_per_iteration": 2.5712811946868896 + }, + { + "auxiliary_loss_clip": 0.01082934, + "auxiliary_loss_mlp": 0.01041071, + "balance_loss_clip": 1.02745593, + "balance_loss_mlp": 1.02285624, + "epoch": 0.17346642678892693, + "flos": 35327145427200.0, + "grad_norm": 2.8821333006535377, + "language_loss": 0.84012753, + "learning_rate": 3.787964588634131e-06, + "loss": 0.86136758, + "num_input_tokens_seen": 171186580, + "router_z_loss_clip": 0.55444336, + "router_z_loss_mlp": 0.18225098, + "step": 5978, + "time_per_iteration": 2.4351539611816406 + }, + { + "auxiliary_loss_clip": 0.01086147, + "auxiliary_loss_mlp": 0.01037744, + "balance_loss_clip": 1.02763677, + "balance_loss_mlp": 1.02069116, + "epoch": 0.17349544425744298, + "flos": 28942508609280.0, + "grad_norm": 3.4137339151979518, + "language_loss": 0.97379553, + "learning_rate": 3.7878803546496157e-06, + "loss": 0.99503446, + "num_input_tokens_seen": 171200445, + "router_z_loss_clip": 0.58447266, + "router_z_loss_mlp": 0.17059326, + "step": 5979, + "time_per_iteration": 2.5003061294555664 + }, + { + "auxiliary_loss_clip": 0.01080668, + "auxiliary_loss_mlp": 0.01034611, + "balance_loss_clip": 1.02795649, + "balance_loss_mlp": 1.01651549, + "epoch": 0.17352446172595903, + "flos": 17121161831040.0, + "grad_norm": 2.732402034320747, + "language_loss": 0.74375117, + "learning_rate": 3.7877961048738172e-06, + "loss": 0.7649039, + "num_input_tokens_seen": 171212690, + "router_z_loss_clip": 0.52758789, + "router_z_loss_mlp": 0.18103027, + "step": 5980, + "time_per_iteration": 2.3438456058502197 + }, + { + "auxiliary_loss_clip": 0.01092098, + "auxiliary_loss_mlp": 0.01034941, + "balance_loss_clip": 1.03153658, + "balance_loss_mlp": 1.01667833, + "epoch": 0.17355347919447509, + "flos": 40981643648640.0, + "grad_norm": 2.321854796763994, + "language_loss": 0.88539022, + "learning_rate": 3.78771183930748e-06, + "loss": 0.90666062, + "num_input_tokens_seen": 171230875, + "router_z_loss_clip": 0.60546875, + "router_z_loss_mlp": 0.18249512, + "step": 5981, + "time_per_iteration": 2.6284234523773193 + }, + { + "auxiliary_loss_clip": 0.01084501, + "auxiliary_loss_mlp": 0.01039292, + "balance_loss_clip": 1.0263927, + "balance_loss_mlp": 1.02036214, + "epoch": 0.1735824966629911, + "flos": 10955255880960.0, + "grad_norm": 4.964136655027494, + "language_loss": 0.86524868, + "learning_rate": 3.7876275579513487e-06, + "loss": 0.88648659, + "num_input_tokens_seen": 171241535, + "router_z_loss_clip": 0.58105469, + "router_z_loss_mlp": 0.18933105, + "step": 5982, + "time_per_iteration": 2.3361260890960693 + }, + { + "auxiliary_loss_clip": 0.01082358, + "auxiliary_loss_mlp": 0.01036124, + "balance_loss_clip": 1.02664495, + "balance_loss_mlp": 1.01770592, + "epoch": 0.17361151413150716, + "flos": 27044979148800.0, + "grad_norm": 2.326272313974951, + "language_loss": 0.84182358, + "learning_rate": 3.787543260806167e-06, + "loss": 0.86300844, + "num_input_tokens_seen": 171258625, + "router_z_loss_clip": 0.55639648, + "router_z_loss_mlp": 0.1842041, + "step": 5983, + "time_per_iteration": 2.4452011585235596 + }, + { + "auxiliary_loss_clip": 0.01089184, + "auxiliary_loss_mlp": 0.01044718, + "balance_loss_clip": 1.02822387, + "balance_loss_mlp": 1.02435756, + "epoch": 0.1736405316000232, + "flos": 19636374798720.0, + "grad_norm": 2.518645296385297, + "language_loss": 0.95138478, + "learning_rate": 3.7874589478726807e-06, + "loss": 0.97272378, + "num_input_tokens_seen": 171271675, + "router_z_loss_clip": 0.61035156, + "router_z_loss_mlp": 0.20349121, + "step": 5984, + "time_per_iteration": 2.381456136703491 + }, + { + "auxiliary_loss_clip": 0.01083266, + "auxiliary_loss_mlp": 0.01034544, + "balance_loss_clip": 1.02754104, + "balance_loss_mlp": 1.01562548, + "epoch": 0.17366954906853926, + "flos": 27849552497280.0, + "grad_norm": 2.6070412217868486, + "language_loss": 0.95691907, + "learning_rate": 3.7873746191516328e-06, + "loss": 0.97809714, + "num_input_tokens_seen": 171285130, + "router_z_loss_clip": 0.55712891, + "router_z_loss_mlp": 0.18920898, + "step": 5985, + "time_per_iteration": 2.4141035079956055 + }, + { + "auxiliary_loss_clip": 0.01079543, + "auxiliary_loss_mlp": 0.01036337, + "balance_loss_clip": 1.02522302, + "balance_loss_mlp": 1.01799679, + "epoch": 0.17369856653705532, + "flos": 19163580900480.0, + "grad_norm": 2.1125475806445073, + "language_loss": 0.88029027, + "learning_rate": 3.7872902746437694e-06, + "loss": 0.90144914, + "num_input_tokens_seen": 171297485, + "router_z_loss_clip": 0.54272461, + "router_z_loss_mlp": 0.18341064, + "step": 5986, + "time_per_iteration": 2.339245557785034 + }, + { + "auxiliary_loss_clip": 0.0108548, + "auxiliary_loss_mlp": 0.01036671, + "balance_loss_clip": 1.02724981, + "balance_loss_mlp": 1.01762152, + "epoch": 0.17372758400557137, + "flos": 16392280043520.0, + "grad_norm": 2.490416616998206, + "language_loss": 0.83303571, + "learning_rate": 3.7872059143498348e-06, + "loss": 0.85425723, + "num_input_tokens_seen": 171311350, + "router_z_loss_clip": 0.58203125, + "router_z_loss_mlp": 0.19030762, + "step": 5987, + "time_per_iteration": 2.3143060207366943 + }, + { + "auxiliary_loss_clip": 0.01078811, + "auxiliary_loss_mlp": 0.01036346, + "balance_loss_clip": 1.02681839, + "balance_loss_mlp": 1.01913249, + "epoch": 0.1737566014740874, + "flos": 20185698130560.0, + "grad_norm": 3.449815029116986, + "language_loss": 1.00443625, + "learning_rate": 3.7871215382705746e-06, + "loss": 1.0255878, + "num_input_tokens_seen": 171324615, + "router_z_loss_clip": 0.52050781, + "router_z_loss_mlp": 0.17224121, + "step": 5988, + "time_per_iteration": 2.34315824508667 + }, + { + "auxiliary_loss_clip": 0.01077949, + "auxiliary_loss_mlp": 0.01038186, + "balance_loss_clip": 1.02582717, + "balance_loss_mlp": 1.02148461, + "epoch": 0.17378561894260344, + "flos": 10553161219200.0, + "grad_norm": 2.947901964536528, + "language_loss": 0.93148285, + "learning_rate": 3.7870371464067336e-06, + "loss": 0.95264417, + "num_input_tokens_seen": 171336265, + "router_z_loss_clip": 0.52148438, + "router_z_loss_mlp": 0.16711426, + "step": 5989, + "time_per_iteration": 2.304248809814453 + }, + { + "auxiliary_loss_clip": 0.01078316, + "auxiliary_loss_mlp": 0.01032461, + "balance_loss_clip": 1.02454317, + "balance_loss_mlp": 1.01560497, + "epoch": 0.1738146364111195, + "flos": 21864950570880.0, + "grad_norm": 2.6868854587259, + "language_loss": 1.04882121, + "learning_rate": 3.7869527387590578e-06, + "loss": 1.069929, + "num_input_tokens_seen": 171349790, + "router_z_loss_clip": 0.5378418, + "router_z_loss_mlp": 0.1685791, + "step": 5990, + "time_per_iteration": 2.3832991123199463 + }, + { + "auxiliary_loss_clip": 0.01079505, + "auxiliary_loss_mlp": 0.01034255, + "balance_loss_clip": 1.02544773, + "balance_loss_mlp": 1.01600385, + "epoch": 0.17384365387963555, + "flos": 28687468060800.0, + "grad_norm": 2.514554044906746, + "language_loss": 0.8055768, + "learning_rate": 3.786868315328292e-06, + "loss": 0.8267144, + "num_input_tokens_seen": 171363555, + "router_z_loss_clip": 0.5402832, + "router_z_loss_mlp": 0.18255615, + "step": 5991, + "time_per_iteration": 2.3904290199279785 + }, + { + "auxiliary_loss_clip": 0.01078926, + "auxiliary_loss_mlp": 0.01035518, + "balance_loss_clip": 1.02516103, + "balance_loss_mlp": 1.01689124, + "epoch": 0.1738726713481516, + "flos": 12487628764800.0, + "grad_norm": 2.1293076231725903, + "language_loss": 0.8104434, + "learning_rate": 3.786783876115182e-06, + "loss": 0.83158779, + "num_input_tokens_seen": 171374770, + "router_z_loss_clip": 0.53881836, + "router_z_loss_mlp": 0.18621826, + "step": 5992, + "time_per_iteration": 2.3881642818450928 + }, + { + "auxiliary_loss_clip": 0.01011913, + "auxiliary_loss_mlp": 0.01000101, + "balance_loss_clip": 1.00460768, + "balance_loss_mlp": 0.99925506, + "epoch": 0.17390168881666762, + "flos": 63533823022080.0, + "grad_norm": 0.7664173166660907, + "language_loss": 0.54183453, + "learning_rate": 3.786699421120474e-06, + "loss": 0.56195468, + "num_input_tokens_seen": 171430065, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00848389, + "step": 5993, + "time_per_iteration": 2.904921770095825 + }, + { + "auxiliary_loss_clip": 0.01077886, + "auxiliary_loss_mlp": 0.0102707, + "balance_loss_clip": 1.02473235, + "balance_loss_mlp": 1.01139402, + "epoch": 0.17393070628518367, + "flos": 16828624615680.0, + "grad_norm": 4.383790834011802, + "language_loss": 0.87542057, + "learning_rate": 3.786614950344914e-06, + "loss": 0.89647007, + "num_input_tokens_seen": 171443460, + "router_z_loss_clip": 0.53027344, + "router_z_loss_mlp": 0.15679932, + "step": 5994, + "time_per_iteration": 2.576460599899292 + }, + { + "auxiliary_loss_clip": 0.01082106, + "auxiliary_loss_mlp": 0.01041253, + "balance_loss_clip": 1.02636838, + "balance_loss_mlp": 1.02095222, + "epoch": 0.17395972375369972, + "flos": 17885585249280.0, + "grad_norm": 2.3798883541373668, + "language_loss": 0.89641714, + "learning_rate": 3.786530463789247e-06, + "loss": 0.91765076, + "num_input_tokens_seen": 171456305, + "router_z_loss_clip": 0.55786133, + "router_z_loss_mlp": 0.20288086, + "step": 5995, + "time_per_iteration": 2.3408427238464355 + }, + { + "auxiliary_loss_clip": 0.01090528, + "auxiliary_loss_mlp": 0.01036943, + "balance_loss_clip": 1.03040743, + "balance_loss_mlp": 1.01705885, + "epoch": 0.17398874122221578, + "flos": 27556491611520.0, + "grad_norm": 2.1207861083568456, + "language_loss": 0.90212291, + "learning_rate": 3.7864459614542206e-06, + "loss": 0.92339772, + "num_input_tokens_seen": 171469815, + "router_z_loss_clip": 0.60107422, + "router_z_loss_mlp": 0.19885254, + "step": 5996, + "time_per_iteration": 2.4438416957855225 + }, + { + "auxiliary_loss_clip": 0.01077768, + "auxiliary_loss_mlp": 0.01031524, + "balance_loss_clip": 1.0259769, + "balance_loss_mlp": 1.01470947, + "epoch": 0.17401775869073183, + "flos": 30948268884480.0, + "grad_norm": 18.9448869169377, + "language_loss": 0.74922693, + "learning_rate": 3.7863614433405804e-06, + "loss": 0.77031994, + "num_input_tokens_seen": 171489380, + "router_z_loss_clip": 0.51782227, + "router_z_loss_mlp": 0.16827393, + "step": 5997, + "time_per_iteration": 2.5196759700775146 + }, + { + "auxiliary_loss_clip": 0.01009442, + "auxiliary_loss_mlp": 0.01001715, + "balance_loss_clip": 1.002177, + "balance_loss_mlp": 1.00075567, + "epoch": 0.17404677615924788, + "flos": 59049613307520.0, + "grad_norm": 0.6966185883681897, + "language_loss": 0.51095533, + "learning_rate": 3.786276909449073e-06, + "loss": 0.53106689, + "num_input_tokens_seen": 171548035, + "router_z_loss_clip": 0.07275391, + "router_z_loss_mlp": 0.00958252, + "step": 5998, + "time_per_iteration": 2.9085047245025635 + }, + { + "auxiliary_loss_clip": 0.01078697, + "auxiliary_loss_mlp": 0.01038806, + "balance_loss_clip": 1.02555013, + "balance_loss_mlp": 1.02241468, + "epoch": 0.1740757936277639, + "flos": 30875370232320.0, + "grad_norm": 2.582302815803452, + "language_loss": 0.69135123, + "learning_rate": 3.786192359780445e-06, + "loss": 0.7125262, + "num_input_tokens_seen": 171562540, + "router_z_loss_clip": 0.53198242, + "router_z_loss_mlp": 0.16381836, + "step": 5999, + "time_per_iteration": 2.375819206237793 + }, + { + "auxiliary_loss_clip": 0.0107719, + "auxiliary_loss_mlp": 0.01035, + "balance_loss_clip": 1.02515662, + "balance_loss_mlp": 1.01787543, + "epoch": 0.17410481109627995, + "flos": 16683036779520.0, + "grad_norm": 2.8365118959657187, + "language_loss": 0.72674561, + "learning_rate": 3.786107794335444e-06, + "loss": 0.74786747, + "num_input_tokens_seen": 171574435, + "router_z_loss_clip": 0.52148438, + "router_z_loss_mlp": 0.17132568, + "step": 6000, + "time_per_iteration": 2.359877824783325 + }, + { + "auxiliary_loss_clip": 0.01077829, + "auxiliary_loss_mlp": 0.01037047, + "balance_loss_clip": 1.02542126, + "balance_loss_mlp": 1.02045929, + "epoch": 0.174133828564796, + "flos": 11536874087040.0, + "grad_norm": 2.8859514768367345, + "language_loss": 0.80971992, + "learning_rate": 3.7860232131148154e-06, + "loss": 0.8308686, + "num_input_tokens_seen": 171583860, + "router_z_loss_clip": 0.52441406, + "router_z_loss_mlp": 0.16564941, + "step": 6001, + "time_per_iteration": 2.3186302185058594 + }, + { + "auxiliary_loss_clip": 0.01081286, + "auxiliary_loss_mlp": 0.01034507, + "balance_loss_clip": 1.02700615, + "balance_loss_mlp": 1.01732934, + "epoch": 0.17416284603331206, + "flos": 21617381053440.0, + "grad_norm": 1.976922641919038, + "language_loss": 0.70643497, + "learning_rate": 3.785938616119307e-06, + "loss": 0.72759283, + "num_input_tokens_seen": 171602675, + "router_z_loss_clip": 0.54248047, + "router_z_loss_mlp": 0.17199707, + "step": 6002, + "time_per_iteration": 2.5060548782348633 + }, + { + "auxiliary_loss_clip": 0.01074838, + "auxiliary_loss_mlp": 0.0103391, + "balance_loss_clip": 1.02362096, + "balance_loss_mlp": 1.01872325, + "epoch": 0.1741918635018281, + "flos": 16464026620800.0, + "grad_norm": 5.05638029301599, + "language_loss": 0.64892083, + "learning_rate": 3.785854003349667e-06, + "loss": 0.6700083, + "num_input_tokens_seen": 171615115, + "router_z_loss_clip": 0.51245117, + "router_z_loss_mlp": 0.15185547, + "step": 6003, + "time_per_iteration": 2.3201968669891357 + }, + { + "auxiliary_loss_clip": 0.01008992, + "auxiliary_loss_mlp": 0.00999018, + "balance_loss_clip": 1.00172389, + "balance_loss_mlp": 0.99811774, + "epoch": 0.17422088097034416, + "flos": 65096465921280.0, + "grad_norm": 0.6521063555573202, + "language_loss": 0.4649303, + "learning_rate": 3.785769374806641e-06, + "loss": 0.48501039, + "num_input_tokens_seen": 171678265, + "router_z_loss_clip": 0.07275391, + "router_z_loss_mlp": 0.00897217, + "step": 6004, + "time_per_iteration": 3.141225814819336 + }, + { + "auxiliary_loss_clip": 0.01009001, + "auxiliary_loss_mlp": 0.00999724, + "balance_loss_clip": 1.00159454, + "balance_loss_mlp": 0.9988302, + "epoch": 0.17424989843886018, + "flos": 55067140874880.0, + "grad_norm": 0.6287555429700501, + "language_loss": 0.47999239, + "learning_rate": 3.7856847304909775e-06, + "loss": 0.50007963, + "num_input_tokens_seen": 171736745, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00891113, + "step": 6005, + "time_per_iteration": 2.987313747406006 + }, + { + "auxiliary_loss_clip": 0.0107352, + "auxiliary_loss_mlp": 0.0103017, + "balance_loss_clip": 1.02387023, + "balance_loss_mlp": 1.01370728, + "epoch": 0.17427891590737624, + "flos": 15990429761280.0, + "grad_norm": 2.3375869971532057, + "language_loss": 0.7684567, + "learning_rate": 3.785600070403424e-06, + "loss": 0.78949362, + "num_input_tokens_seen": 171750050, + "router_z_loss_clip": 0.49633789, + "router_z_loss_mlp": 0.16455078, + "step": 6006, + "time_per_iteration": 2.365496873855591 + }, + { + "auxiliary_loss_clip": 0.01007819, + "auxiliary_loss_mlp": 0.01003549, + "balance_loss_clip": 1.00065756, + "balance_loss_mlp": 1.00256002, + "epoch": 0.1743079333758923, + "flos": 59448949971840.0, + "grad_norm": 0.7302128926121184, + "language_loss": 0.52113557, + "learning_rate": 3.7855153945447275e-06, + "loss": 0.54124928, + "num_input_tokens_seen": 171810775, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.0098877, + "step": 6007, + "time_per_iteration": 2.972553253173828 + }, + { + "auxiliary_loss_clip": 0.01075541, + "auxiliary_loss_mlp": 0.01032573, + "balance_loss_clip": 1.02652478, + "balance_loss_mlp": 1.01708138, + "epoch": 0.17433695084440834, + "flos": 12819896974080.0, + "grad_norm": 2.7158271857665923, + "language_loss": 0.84713399, + "learning_rate": 3.7854307029156375e-06, + "loss": 0.8682152, + "num_input_tokens_seen": 171824120, + "router_z_loss_clip": 0.49047852, + "router_z_loss_mlp": 0.15484619, + "step": 6008, + "time_per_iteration": 2.3200483322143555 + }, + { + "auxiliary_loss_clip": 0.01008022, + "auxiliary_loss_mlp": 0.01002778, + "balance_loss_clip": 1.00090742, + "balance_loss_mlp": 1.00186014, + "epoch": 0.1743659683129244, + "flos": 63606756585600.0, + "grad_norm": 0.6585027945670527, + "language_loss": 0.46318445, + "learning_rate": 3.7853459955169002e-06, + "loss": 0.48329246, + "num_input_tokens_seen": 171882995, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00915527, + "step": 6009, + "time_per_iteration": 2.963459014892578 + }, + { + "auxiliary_loss_clip": 0.01076671, + "auxiliary_loss_mlp": 0.01029861, + "balance_loss_clip": 1.02492499, + "balance_loss_mlp": 1.01516223, + "epoch": 0.17439498578144041, + "flos": 31065437007360.0, + "grad_norm": 2.9060606276071757, + "language_loss": 0.86695421, + "learning_rate": 3.785261272349265e-06, + "loss": 0.8880195, + "num_input_tokens_seen": 171903255, + "router_z_loss_clip": 0.51708984, + "router_z_loss_mlp": 0.14691162, + "step": 6010, + "time_per_iteration": 6.763833522796631 + }, + { + "auxiliary_loss_clip": 0.01070854, + "auxiliary_loss_mlp": 0.01029021, + "balance_loss_clip": 1.02265, + "balance_loss_mlp": 1.01426888, + "epoch": 0.17442400324995647, + "flos": 20190271518720.0, + "grad_norm": 2.425453456272244, + "language_loss": 0.80599421, + "learning_rate": 3.7851765334134792e-06, + "loss": 0.82699299, + "num_input_tokens_seen": 171917135, + "router_z_loss_clip": 0.48266602, + "router_z_loss_mlp": 0.14746094, + "step": 6011, + "time_per_iteration": 2.3208508491516113 + }, + { + "auxiliary_loss_clip": 0.01008912, + "auxiliary_loss_mlp": 0.01004374, + "balance_loss_clip": 1.00156987, + "balance_loss_mlp": 1.00345051, + "epoch": 0.17445302071847252, + "flos": 74781092448000.0, + "grad_norm": 0.6305155769325427, + "language_loss": 0.48225111, + "learning_rate": 3.785091778710293e-06, + "loss": 0.50238401, + "num_input_tokens_seen": 171984555, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00921631, + "step": 6012, + "time_per_iteration": 3.199615240097046 + }, + { + "auxiliary_loss_clip": 0.01080216, + "auxiliary_loss_mlp": 0.01039216, + "balance_loss_clip": 1.02636099, + "balance_loss_mlp": 1.0225445, + "epoch": 0.17448203818698857, + "flos": 18186291722880.0, + "grad_norm": 2.25248159388084, + "language_loss": 0.82506067, + "learning_rate": 3.785007008240453e-06, + "loss": 0.84625494, + "num_input_tokens_seen": 172001960, + "router_z_loss_clip": 0.53881836, + "router_z_loss_mlp": 0.16662598, + "step": 6013, + "time_per_iteration": 2.5136008262634277 + }, + { + "auxiliary_loss_clip": 0.01083899, + "auxiliary_loss_mlp": 0.010381, + "balance_loss_clip": 1.02725065, + "balance_loss_mlp": 1.01951528, + "epoch": 0.17451105565550462, + "flos": 35073291864960.0, + "grad_norm": 3.8418397509830924, + "language_loss": 0.71796399, + "learning_rate": 3.784922222004709e-06, + "loss": 0.73918402, + "num_input_tokens_seen": 172018890, + "router_z_loss_clip": 0.56689453, + "router_z_loss_mlp": 0.18603516, + "step": 6014, + "time_per_iteration": 2.635629415512085 + }, + { + "auxiliary_loss_clip": 0.01078608, + "auxiliary_loss_mlp": 0.01034654, + "balance_loss_clip": 1.02544498, + "balance_loss_mlp": 1.0184412, + "epoch": 0.17454007312402067, + "flos": 27956666148480.0, + "grad_norm": 2.8011766288834936, + "language_loss": 0.7963028, + "learning_rate": 3.78483742000381e-06, + "loss": 0.81743532, + "num_input_tokens_seen": 172033995, + "router_z_loss_clip": 0.53222656, + "router_z_loss_mlp": 0.16210938, + "step": 6015, + "time_per_iteration": 2.4834096431732178 + }, + { + "auxiliary_loss_clip": 0.01011521, + "auxiliary_loss_mlp": 0.01002368, + "balance_loss_clip": 1.00426579, + "balance_loss_mlp": 1.00144982, + "epoch": 0.1745690905925367, + "flos": 60761544647040.0, + "grad_norm": 0.6278540079666983, + "language_loss": 0.47201118, + "learning_rate": 3.7847526022385045e-06, + "loss": 0.49215004, + "num_input_tokens_seen": 172104910, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.00915527, + "step": 6016, + "time_per_iteration": 3.312506914138794 + }, + { + "auxiliary_loss_clip": 0.01011964, + "auxiliary_loss_mlp": 0.01008979, + "balance_loss_clip": 1.00481462, + "balance_loss_mlp": 1.00806677, + "epoch": 0.17459810806105275, + "flos": 74772120228480.0, + "grad_norm": 0.6153352169191967, + "language_loss": 0.47642362, + "learning_rate": 3.7846677687095408e-06, + "loss": 0.49663305, + "num_input_tokens_seen": 172172240, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00909424, + "step": 6017, + "time_per_iteration": 3.3207571506500244 + }, + { + "auxiliary_loss_clip": 0.01076454, + "auxiliary_loss_mlp": 0.01031926, + "balance_loss_clip": 1.02484775, + "balance_loss_mlp": 1.01655424, + "epoch": 0.1746271255295688, + "flos": 30154203855360.0, + "grad_norm": 1.966860394715991, + "language_loss": 0.74286389, + "learning_rate": 3.784582919417671e-06, + "loss": 0.76394773, + "num_input_tokens_seen": 172191295, + "router_z_loss_clip": 0.51611328, + "router_z_loss_mlp": 0.15368652, + "step": 6018, + "time_per_iteration": 2.5252597332000732 + }, + { + "auxiliary_loss_clip": 0.01069087, + "auxiliary_loss_mlp": 0.01030658, + "balance_loss_clip": 1.02356601, + "balance_loss_mlp": 1.01729429, + "epoch": 0.17465614299808485, + "flos": 15773898309120.0, + "grad_norm": 2.2605799384172167, + "language_loss": 0.67585438, + "learning_rate": 3.7844980543636417e-06, + "loss": 0.69685185, + "num_input_tokens_seen": 172205730, + "router_z_loss_clip": 0.45483398, + "router_z_loss_mlp": 0.13348389, + "step": 6019, + "time_per_iteration": 2.439821481704712 + }, + { + "auxiliary_loss_clip": 0.01074387, + "auxiliary_loss_mlp": 0.01031437, + "balance_loss_clip": 1.02479482, + "balance_loss_mlp": 1.0173409, + "epoch": 0.1746851604666009, + "flos": 29052415169280.0, + "grad_norm": 10.203984573894225, + "language_loss": 0.65400809, + "learning_rate": 3.784413173548203e-06, + "loss": 0.67506635, + "num_input_tokens_seen": 172223885, + "router_z_loss_clip": 0.49584961, + "router_z_loss_mlp": 0.14074707, + "step": 6020, + "time_per_iteration": 5.231232404708862 + }, + { + "auxiliary_loss_clip": 0.01082033, + "auxiliary_loss_mlp": 0.01048023, + "balance_loss_clip": 1.02820373, + "balance_loss_mlp": 1.03186965, + "epoch": 0.17471417793511693, + "flos": 12740923745280.0, + "grad_norm": 2.7514273898800914, + "language_loss": 0.8484717, + "learning_rate": 3.784328276972106e-06, + "loss": 0.86977226, + "num_input_tokens_seen": 172234455, + "router_z_loss_clip": 0.53857422, + "router_z_loss_mlp": 0.16149902, + "step": 6021, + "time_per_iteration": 2.475977659225464 + }, + { + "auxiliary_loss_clip": 0.01077847, + "auxiliary_loss_mlp": 0.010409, + "balance_loss_clip": 1.0261327, + "balance_loss_mlp": 1.02469385, + "epoch": 0.17474319540363298, + "flos": 28248854250240.0, + "grad_norm": 2.297483874954544, + "language_loss": 0.88766086, + "learning_rate": 3.7842433646360988e-06, + "loss": 0.90884835, + "num_input_tokens_seen": 172249335, + "router_z_loss_clip": 0.5168457, + "router_z_loss_mlp": 0.16223145, + "step": 6022, + "time_per_iteration": 2.8396644592285156 + }, + { + "auxiliary_loss_clip": 0.01081944, + "auxiliary_loss_mlp": 0.0103569, + "balance_loss_clip": 1.02675736, + "balance_loss_mlp": 1.01885796, + "epoch": 0.17477221287214903, + "flos": 15555202352640.0, + "grad_norm": 4.207873153155727, + "language_loss": 0.95427823, + "learning_rate": 3.7841584365409327e-06, + "loss": 0.97545457, + "num_input_tokens_seen": 172261790, + "router_z_loss_clip": 0.55200195, + "router_z_loss_mlp": 0.168396, + "step": 6023, + "time_per_iteration": 2.459939479827881 + }, + { + "auxiliary_loss_clip": 0.01011319, + "auxiliary_loss_mlp": 0.01002349, + "balance_loss_clip": 1.00419784, + "balance_loss_mlp": 1.00153852, + "epoch": 0.17480123034066508, + "flos": 66602583596160.0, + "grad_norm": 0.666672976666966, + "language_loss": 0.48246354, + "learning_rate": 3.7840734926873574e-06, + "loss": 0.50260019, + "num_input_tokens_seen": 172323435, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00811768, + "step": 6024, + "time_per_iteration": 3.165313243865967 + }, + { + "auxiliary_loss_clip": 0.01010616, + "auxiliary_loss_mlp": 0.0100535, + "balance_loss_clip": 1.00343966, + "balance_loss_mlp": 1.00445628, + "epoch": 0.17483024780918113, + "flos": 74773062835200.0, + "grad_norm": 0.6685376655538482, + "language_loss": 0.50615472, + "learning_rate": 3.7839885330761223e-06, + "loss": 0.52631438, + "num_input_tokens_seen": 172384080, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.00891113, + "step": 6025, + "time_per_iteration": 3.14408540725708 + }, + { + "auxiliary_loss_clip": 0.01080981, + "auxiliary_loss_mlp": 0.01036169, + "balance_loss_clip": 1.02639127, + "balance_loss_mlp": 1.02032578, + "epoch": 0.17485926527769718, + "flos": 35149542007680.0, + "grad_norm": 2.209663961551995, + "language_loss": 0.89335942, + "learning_rate": 3.7839035577079798e-06, + "loss": 0.91453087, + "num_input_tokens_seen": 172404595, + "router_z_loss_clip": 0.54589844, + "router_z_loss_mlp": 0.1583252, + "step": 6026, + "time_per_iteration": 2.5986289978027344 + }, + { + "auxiliary_loss_clip": 0.01012092, + "auxiliary_loss_mlp": 0.01000654, + "balance_loss_clip": 1.00474465, + "balance_loss_mlp": 0.99980754, + "epoch": 0.1748882827462132, + "flos": 56342972021760.0, + "grad_norm": 0.6481437603708845, + "language_loss": 0.475925, + "learning_rate": 3.7838185665836784e-06, + "loss": 0.49605247, + "num_input_tokens_seen": 172462340, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00848389, + "step": 6027, + "time_per_iteration": 2.9346866607666016 + }, + { + "auxiliary_loss_clip": 0.01010162, + "auxiliary_loss_mlp": 0.01001159, + "balance_loss_clip": 1.00294983, + "balance_loss_mlp": 1.00030112, + "epoch": 0.17491730021472926, + "flos": 54779037402240.0, + "grad_norm": 0.6263377841565431, + "language_loss": 0.44837132, + "learning_rate": 3.78373355970397e-06, + "loss": 0.46848452, + "num_input_tokens_seen": 172519500, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.00860596, + "step": 6028, + "time_per_iteration": 2.962909460067749 + }, + { + "auxiliary_loss_clip": 0.0107539, + "auxiliary_loss_mlp": 0.01035939, + "balance_loss_clip": 1.02558994, + "balance_loss_mlp": 1.02044749, + "epoch": 0.1749463176832453, + "flos": 35001859489920.0, + "grad_norm": 1.9950546627975145, + "language_loss": 0.98452353, + "learning_rate": 3.7836485370696044e-06, + "loss": 1.00563681, + "num_input_tokens_seen": 172540120, + "router_z_loss_clip": 0.49731445, + "router_z_loss_mlp": 0.15484619, + "step": 6029, + "time_per_iteration": 2.5185282230377197 + }, + { + "auxiliary_loss_clip": 0.01008618, + "auxiliary_loss_mlp": 0.01003373, + "balance_loss_clip": 1.00131989, + "balance_loss_mlp": 1.00252628, + "epoch": 0.17497533515176136, + "flos": 58862269618560.0, + "grad_norm": 0.6531591287697508, + "language_loss": 0.46369201, + "learning_rate": 3.783563498681334e-06, + "loss": 0.48381191, + "num_input_tokens_seen": 172599840, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00848389, + "step": 6030, + "time_per_iteration": 2.8941075801849365 + }, + { + "auxiliary_loss_clip": 0.01008479, + "auxiliary_loss_mlp": 0.01002227, + "balance_loss_clip": 1.00136304, + "balance_loss_mlp": 1.00135064, + "epoch": 0.17500435262027741, + "flos": 74767023169920.0, + "grad_norm": 0.6605196146178008, + "language_loss": 0.50147521, + "learning_rate": 3.7834784445399086e-06, + "loss": 0.52158237, + "num_input_tokens_seen": 172658185, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00878906, + "step": 6031, + "time_per_iteration": 3.002842664718628 + }, + { + "auxiliary_loss_clip": 0.01008234, + "auxiliary_loss_mlp": 0.01002309, + "balance_loss_clip": 1.00099874, + "balance_loss_mlp": 1.0015099, + "epoch": 0.17503337008879347, + "flos": 74767232638080.0, + "grad_norm": 0.6827203185017424, + "language_loss": 0.46767426, + "learning_rate": 3.7833933746460794e-06, + "loss": 0.48777971, + "num_input_tokens_seen": 172715750, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.00799561, + "step": 6032, + "time_per_iteration": 3.046060085296631 + }, + { + "auxiliary_loss_clip": 0.01008293, + "auxiliary_loss_mlp": 0.01001927, + "balance_loss_clip": 1.00082529, + "balance_loss_mlp": 1.00102103, + "epoch": 0.1750623875573095, + "flos": 63762329070720.0, + "grad_norm": 0.6434105455959975, + "language_loss": 0.48590404, + "learning_rate": 3.783308289000599e-06, + "loss": 0.50600624, + "num_input_tokens_seen": 172779315, + "router_z_loss_clip": 0.07470703, + "router_z_loss_mlp": 0.0090332, + "step": 6033, + "time_per_iteration": 3.1235804557800293 + }, + { + "auxiliary_loss_clip": 0.01077411, + "auxiliary_loss_mlp": 0.01035636, + "balance_loss_clip": 1.02673125, + "balance_loss_mlp": 1.02040684, + "epoch": 0.17509140502582554, + "flos": 34164921444480.0, + "grad_norm": 1.8815296904963792, + "language_loss": 0.83621711, + "learning_rate": 3.783223187604218e-06, + "loss": 0.85734761, + "num_input_tokens_seen": 172802010, + "router_z_loss_clip": 0.50708008, + "router_z_loss_mlp": 0.15228271, + "step": 6034, + "time_per_iteration": 2.694453239440918 + }, + { + "auxiliary_loss_clip": 0.01086166, + "auxiliary_loss_mlp": 0.01034102, + "balance_loss_clip": 1.02957058, + "balance_loss_mlp": 1.01722145, + "epoch": 0.1751204224943416, + "flos": 33287030507520.0, + "grad_norm": 1.9807048832598477, + "language_loss": 0.88201487, + "learning_rate": 3.7831380704576875e-06, + "loss": 0.90321755, + "num_input_tokens_seen": 172820570, + "router_z_loss_clip": 0.56567383, + "router_z_loss_mlp": 0.16894531, + "step": 6035, + "time_per_iteration": 2.5174524784088135 + }, + { + "auxiliary_loss_clip": 0.01073998, + "auxiliary_loss_mlp": 0.01042039, + "balance_loss_clip": 1.02617991, + "balance_loss_mlp": 1.02756119, + "epoch": 0.17514943996285764, + "flos": 11465651180160.0, + "grad_norm": 2.1982384700862747, + "language_loss": 0.75456595, + "learning_rate": 3.783052937561761e-06, + "loss": 0.77572626, + "num_input_tokens_seen": 172833630, + "router_z_loss_clip": 0.4777832, + "router_z_loss_mlp": 0.14471436, + "step": 6036, + "time_per_iteration": 2.396091938018799 + }, + { + "auxiliary_loss_clip": 0.01079906, + "auxiliary_loss_mlp": 0.01046764, + "balance_loss_clip": 1.02635479, + "balance_loss_mlp": 1.03062344, + "epoch": 0.1751784574313737, + "flos": 11207957368320.0, + "grad_norm": 3.2283515013818964, + "language_loss": 0.97641695, + "learning_rate": 3.782967788917189e-06, + "loss": 0.9976837, + "num_input_tokens_seen": 172843570, + "router_z_loss_clip": 0.53540039, + "router_z_loss_mlp": 0.16143799, + "step": 6037, + "time_per_iteration": 2.440065383911133 + }, + { + "auxiliary_loss_clip": 0.01080706, + "auxiliary_loss_mlp": 0.01038449, + "balance_loss_clip": 1.02702117, + "balance_loss_mlp": 1.02178907, + "epoch": 0.17520747489988972, + "flos": 23112990408960.0, + "grad_norm": 2.1861847097806164, + "language_loss": 0.87274212, + "learning_rate": 3.782882624524724e-06, + "loss": 0.89393365, + "num_input_tokens_seen": 172860850, + "router_z_loss_clip": 0.53710938, + "router_z_loss_mlp": 0.16674805, + "step": 6038, + "time_per_iteration": 2.5155694484710693 + }, + { + "auxiliary_loss_clip": 0.01081629, + "auxiliary_loss_mlp": 0.01040042, + "balance_loss_clip": 1.02941966, + "balance_loss_mlp": 1.02393699, + "epoch": 0.17523649236840577, + "flos": 17124722789760.0, + "grad_norm": 2.218906571127518, + "language_loss": 0.6728881, + "learning_rate": 3.7827974443851184e-06, + "loss": 0.69410479, + "num_input_tokens_seen": 172875855, + "router_z_loss_clip": 0.52197266, + "router_z_loss_mlp": 0.16101074, + "step": 6039, + "time_per_iteration": 2.36567759513855 + }, + { + "auxiliary_loss_clip": 0.01091576, + "auxiliary_loss_mlp": 0.01047441, + "balance_loss_clip": 1.03087735, + "balance_loss_mlp": 1.02913022, + "epoch": 0.17526550983692182, + "flos": 19425638632320.0, + "grad_norm": 1.902226741800918, + "language_loss": 0.80776709, + "learning_rate": 3.7827122484991237e-06, + "loss": 0.82915735, + "num_input_tokens_seen": 172893015, + "router_z_loss_clip": 0.60693359, + "router_z_loss_mlp": 0.18322754, + "step": 6040, + "time_per_iteration": 2.404326915740967 + }, + { + "auxiliary_loss_clip": 0.01014238, + "auxiliary_loss_mlp": 0.01003056, + "balance_loss_clip": 1.00684452, + "balance_loss_mlp": 1.00225699, + "epoch": 0.17529452730543787, + "flos": 66001449939840.0, + "grad_norm": 0.7017177305295677, + "language_loss": 0.50315911, + "learning_rate": 3.7826270368674937e-06, + "loss": 0.52333206, + "num_input_tokens_seen": 172945500, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00799561, + "step": 6041, + "time_per_iteration": 2.923515796661377 + }, + { + "auxiliary_loss_clip": 0.0108513, + "auxiliary_loss_mlp": 0.01035919, + "balance_loss_clip": 1.02867186, + "balance_loss_mlp": 1.01863372, + "epoch": 0.17532354477395393, + "flos": 13947452110080.0, + "grad_norm": 2.4747139974654515, + "language_loss": 0.76644224, + "learning_rate": 3.78254180949098e-06, + "loss": 0.78765273, + "num_input_tokens_seen": 172959760, + "router_z_loss_clip": 0.56420898, + "router_z_loss_mlp": 0.17285156, + "step": 6042, + "time_per_iteration": 2.373833179473877 + }, + { + "auxiliary_loss_clip": 0.01092837, + "auxiliary_loss_mlp": 0.01032493, + "balance_loss_clip": 1.03100526, + "balance_loss_mlp": 1.01364613, + "epoch": 0.17535256224246998, + "flos": 23323796398080.0, + "grad_norm": 2.5864656103628905, + "language_loss": 0.97572297, + "learning_rate": 3.782456566370336e-06, + "loss": 0.99697626, + "num_input_tokens_seen": 172971920, + "router_z_loss_clip": 0.61914062, + "router_z_loss_mlp": 0.18841553, + "step": 6043, + "time_per_iteration": 2.4038403034210205 + }, + { + "auxiliary_loss_clip": 0.01088526, + "auxiliary_loss_mlp": 0.01038215, + "balance_loss_clip": 1.03233528, + "balance_loss_mlp": 1.01959443, + "epoch": 0.175381579710986, + "flos": 35331404613120.0, + "grad_norm": 2.3215231758715134, + "language_loss": 0.89952064, + "learning_rate": 3.782371307506314e-06, + "loss": 0.92078805, + "num_input_tokens_seen": 172995480, + "router_z_loss_clip": 0.56201172, + "router_z_loss_mlp": 0.18621826, + "step": 6044, + "time_per_iteration": 2.570253610610962 + }, + { + "auxiliary_loss_clip": 0.01093094, + "auxiliary_loss_mlp": 0.01037266, + "balance_loss_clip": 1.03022599, + "balance_loss_mlp": 1.01757264, + "epoch": 0.17541059717950205, + "flos": 24748392314880.0, + "grad_norm": 2.7156950533605286, + "language_loss": 0.96005356, + "learning_rate": 3.782286032899668e-06, + "loss": 0.9813571, + "num_input_tokens_seen": 173016530, + "router_z_loss_clip": 0.62792969, + "router_z_loss_mlp": 0.19689941, + "step": 6045, + "time_per_iteration": 2.5054216384887695 + }, + { + "auxiliary_loss_clip": 0.01079271, + "auxiliary_loss_mlp": 0.01029378, + "balance_loss_clip": 1.03014839, + "balance_loss_mlp": 1.01512003, + "epoch": 0.1754396146480181, + "flos": 37844208696960.0, + "grad_norm": 1.8027193829089918, + "language_loss": 0.58719909, + "learning_rate": 3.78220074255115e-06, + "loss": 0.60828555, + "num_input_tokens_seen": 173032935, + "router_z_loss_clip": 0.4909668, + "router_z_loss_mlp": 0.14251709, + "step": 6046, + "time_per_iteration": 2.545279026031494 + }, + { + "auxiliary_loss_clip": 0.01087384, + "auxiliary_loss_mlp": 0.01039068, + "balance_loss_clip": 1.03289795, + "balance_loss_mlp": 1.02210498, + "epoch": 0.17546863211653416, + "flos": 24998754741120.0, + "grad_norm": 4.099550612490562, + "language_loss": 0.54579365, + "learning_rate": 3.782115436461514e-06, + "loss": 0.56705821, + "num_input_tokens_seen": 173045545, + "router_z_loss_clip": 0.54541016, + "router_z_loss_mlp": 0.16967773, + "step": 6047, + "time_per_iteration": 2.4411330223083496 + }, + { + "auxiliary_loss_clip": 0.01082699, + "auxiliary_loss_mlp": 0.01038554, + "balance_loss_clip": 1.02892983, + "balance_loss_mlp": 1.02246726, + "epoch": 0.1754976495850502, + "flos": 12815847256320.0, + "grad_norm": 2.3612627212440125, + "language_loss": 0.84434557, + "learning_rate": 3.782030114631513e-06, + "loss": 0.86555815, + "num_input_tokens_seen": 173056890, + "router_z_loss_clip": 0.5378418, + "router_z_loss_mlp": 0.16094971, + "step": 6048, + "time_per_iteration": 2.3701374530792236 + }, + { + "auxiliary_loss_clip": 0.01015641, + "auxiliary_loss_mlp": 0.01000186, + "balance_loss_clip": 1.00794649, + "balance_loss_mlp": 0.99933332, + "epoch": 0.17552666705356626, + "flos": 70392510547200.0, + "grad_norm": 0.6511160433237001, + "language_loss": 0.50001788, + "learning_rate": 3.781944777061901e-06, + "loss": 0.52017611, + "num_input_tokens_seen": 173115155, + "router_z_loss_clip": 0.07714844, + "router_z_loss_mlp": 0.00854492, + "step": 6049, + "time_per_iteration": 3.0220775604248047 + }, + { + "auxiliary_loss_clip": 0.01078604, + "auxiliary_loss_mlp": 0.01031441, + "balance_loss_clip": 1.02911305, + "balance_loss_mlp": 1.01794672, + "epoch": 0.17555568452208228, + "flos": 32588698026240.0, + "grad_norm": 2.3062125437647425, + "language_loss": 0.78315651, + "learning_rate": 3.781859423753432e-06, + "loss": 0.80425692, + "num_input_tokens_seen": 173128535, + "router_z_loss_clip": 0.49462891, + "router_z_loss_mlp": 0.13500977, + "step": 6050, + "time_per_iteration": 2.4748189449310303 + }, + { + "auxiliary_loss_clip": 0.01087414, + "auxiliary_loss_mlp": 0.01037894, + "balance_loss_clip": 1.03042841, + "balance_loss_mlp": 1.01972651, + "epoch": 0.17558470199059834, + "flos": 14351816010240.0, + "grad_norm": 2.989671841050811, + "language_loss": 0.90278995, + "learning_rate": 3.7817740547068596e-06, + "loss": 0.92404306, + "num_input_tokens_seen": 173142920, + "router_z_loss_clip": 0.56982422, + "router_z_loss_mlp": 0.18157959, + "step": 6051, + "time_per_iteration": 2.3931703567504883 + }, + { + "auxiliary_loss_clip": 0.01074079, + "auxiliary_loss_mlp": 0.01030234, + "balance_loss_clip": 1.02470231, + "balance_loss_mlp": 1.01509428, + "epoch": 0.1756137194591144, + "flos": 13691224575360.0, + "grad_norm": 1.9537310139238315, + "language_loss": 0.71109307, + "learning_rate": 3.7816886699229373e-06, + "loss": 0.73213613, + "num_input_tokens_seen": 173156005, + "router_z_loss_clip": 0.49414062, + "router_z_loss_mlp": 0.15130615, + "step": 6052, + "time_per_iteration": 2.3771145343780518 + }, + { + "auxiliary_loss_clip": 0.01014745, + "auxiliary_loss_mlp": 0.01006254, + "balance_loss_clip": 1.00730491, + "balance_loss_mlp": 1.00541353, + "epoch": 0.17564273692763044, + "flos": 72432974580480.0, + "grad_norm": 0.6880449982500113, + "language_loss": 0.49093735, + "learning_rate": 3.7816032694024197e-06, + "loss": 0.51114738, + "num_input_tokens_seen": 173214675, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00842285, + "step": 6053, + "time_per_iteration": 3.00150990486145 + }, + { + "auxiliary_loss_clip": 0.0107786, + "auxiliary_loss_mlp": 0.01031897, + "balance_loss_clip": 1.02482533, + "balance_loss_mlp": 1.01687098, + "epoch": 0.1756717543961465, + "flos": 16721336407680.0, + "grad_norm": 4.591762036826113, + "language_loss": 0.81136358, + "learning_rate": 3.7815178531460615e-06, + "loss": 0.83246124, + "num_input_tokens_seen": 173230215, + "router_z_loss_clip": 0.53027344, + "router_z_loss_mlp": 0.15020752, + "step": 6054, + "time_per_iteration": 2.3426363468170166 + }, + { + "auxiliary_loss_clip": 0.0101368, + "auxiliary_loss_mlp": 0.01009736, + "balance_loss_clip": 1.00634027, + "balance_loss_mlp": 1.00893688, + "epoch": 0.1757007718646625, + "flos": 66954089831040.0, + "grad_norm": 0.6947552399129753, + "language_loss": 0.49640751, + "learning_rate": 3.7814324211546166e-06, + "loss": 0.51664162, + "num_input_tokens_seen": 173286990, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00799561, + "step": 6055, + "time_per_iteration": 2.986269235610962 + }, + { + "auxiliary_loss_clip": 0.01076288, + "auxiliary_loss_mlp": 0.01035966, + "balance_loss_clip": 1.02652812, + "balance_loss_mlp": 1.02082682, + "epoch": 0.17572978933317857, + "flos": 13471446366720.0, + "grad_norm": 2.7790563753329582, + "language_loss": 0.79917872, + "learning_rate": 3.78134697342884e-06, + "loss": 0.82030129, + "num_input_tokens_seen": 173298645, + "router_z_loss_clip": 0.49707031, + "router_z_loss_mlp": 0.15136719, + "step": 6056, + "time_per_iteration": 2.3570361137390137 + }, + { + "auxiliary_loss_clip": 0.01077365, + "auxiliary_loss_mlp": 0.0103151, + "balance_loss_clip": 1.02860057, + "balance_loss_mlp": 1.0179503, + "epoch": 0.17575880680169462, + "flos": 11940295380480.0, + "grad_norm": 2.268168892051274, + "language_loss": 0.73960239, + "learning_rate": 3.7812615099694853e-06, + "loss": 0.76069117, + "num_input_tokens_seen": 173310600, + "router_z_loss_clip": 0.48779297, + "router_z_loss_mlp": 0.13555908, + "step": 6057, + "time_per_iteration": 2.403186082839966 + }, + { + "auxiliary_loss_clip": 0.01076975, + "auxiliary_loss_mlp": 0.01031123, + "balance_loss_clip": 1.02612782, + "balance_loss_mlp": 1.0159657, + "epoch": 0.17578782427021067, + "flos": 16791930910080.0, + "grad_norm": 2.5140098645946702, + "language_loss": 0.68293792, + "learning_rate": 3.781176030777309e-06, + "loss": 0.70401889, + "num_input_tokens_seen": 173323215, + "router_z_loss_clip": 0.5090332, + "router_z_loss_mlp": 0.15148926, + "step": 6058, + "time_per_iteration": 2.3710620403289795 + }, + { + "auxiliary_loss_clip": 0.01079505, + "auxiliary_loss_mlp": 0.01033501, + "balance_loss_clip": 1.02644396, + "balance_loss_mlp": 1.01709819, + "epoch": 0.17581684173872672, + "flos": 28724336323200.0, + "grad_norm": 2.1188810293336715, + "language_loss": 0.79274988, + "learning_rate": 3.781090535853065e-06, + "loss": 0.81387997, + "num_input_tokens_seen": 173339625, + "router_z_loss_clip": 0.53076172, + "router_z_loss_mlp": 0.1640625, + "step": 6059, + "time_per_iteration": 2.545445442199707 + }, + { + "auxiliary_loss_clip": 0.01081312, + "auxiliary_loss_mlp": 0.01031025, + "balance_loss_clip": 1.02870846, + "balance_loss_mlp": 1.01468766, + "epoch": 0.17584585920724277, + "flos": 18580007658240.0, + "grad_norm": 1.708355677868553, + "language_loss": 0.81032968, + "learning_rate": 3.781005025197508e-06, + "loss": 0.83145308, + "num_input_tokens_seen": 173356015, + "router_z_loss_clip": 0.52612305, + "router_z_loss_mlp": 0.16357422, + "step": 6060, + "time_per_iteration": 2.4170258045196533 + }, + { + "auxiliary_loss_clip": 0.01016522, + "auxiliary_loss_mlp": 0.01004204, + "balance_loss_clip": 1.00923252, + "balance_loss_mlp": 1.00330353, + "epoch": 0.1758748766757588, + "flos": 58749465415680.0, + "grad_norm": 0.6916762307514852, + "language_loss": 0.50260401, + "learning_rate": 3.7809194988113943e-06, + "loss": 0.52281123, + "num_input_tokens_seen": 173414275, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00897217, + "step": 6061, + "time_per_iteration": 2.958104133605957 + }, + { + "auxiliary_loss_clip": 0.01081915, + "auxiliary_loss_mlp": 0.01028762, + "balance_loss_clip": 1.02929044, + "balance_loss_mlp": 1.0123235, + "epoch": 0.17590389414427485, + "flos": 50942119760640.0, + "grad_norm": 5.568972506672689, + "language_loss": 0.89843154, + "learning_rate": 3.7808339566954786e-06, + "loss": 0.91953832, + "num_input_tokens_seen": 173432185, + "router_z_loss_clip": 0.52612305, + "router_z_loss_mlp": 0.16442871, + "step": 6062, + "time_per_iteration": 2.621640920639038 + }, + { + "auxiliary_loss_clip": 0.01081209, + "auxiliary_loss_mlp": 0.0103172, + "balance_loss_clip": 1.02857053, + "balance_loss_mlp": 1.01551938, + "epoch": 0.1759329116127909, + "flos": 36311731079040.0, + "grad_norm": 2.377721224619858, + "language_loss": 0.8056401, + "learning_rate": 3.7807483988505173e-06, + "loss": 0.82676935, + "num_input_tokens_seen": 173449820, + "router_z_loss_clip": 0.52612305, + "router_z_loss_mlp": 0.16210938, + "step": 6063, + "time_per_iteration": 2.526115894317627 + }, + { + "auxiliary_loss_clip": 0.01018376, + "auxiliary_loss_mlp": 0.01004591, + "balance_loss_clip": 1.01104546, + "balance_loss_mlp": 1.00369072, + "epoch": 0.17596192908130695, + "flos": 73311459010560.0, + "grad_norm": 0.7168220043873667, + "language_loss": 0.50120282, + "learning_rate": 3.7806628252772654e-06, + "loss": 0.52143252, + "num_input_tokens_seen": 173514635, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00897217, + "step": 6064, + "time_per_iteration": 3.016899824142456 + }, + { + "auxiliary_loss_clip": 0.01081663, + "auxiliary_loss_mlp": 0.01032679, + "balance_loss_clip": 1.02947581, + "balance_loss_mlp": 1.01847529, + "epoch": 0.175990946549823, + "flos": 33906913430400.0, + "grad_norm": 1.8942307281085182, + "language_loss": 0.76173449, + "learning_rate": 3.780577235976479e-06, + "loss": 0.78287804, + "num_input_tokens_seen": 173535110, + "router_z_loss_clip": 0.52197266, + "router_z_loss_mlp": 0.14196777, + "step": 6065, + "time_per_iteration": 2.5251970291137695 + }, + { + "auxiliary_loss_clip": 0.01019016, + "auxiliary_loss_mlp": 0.01001299, + "balance_loss_clip": 1.01128197, + "balance_loss_mlp": 1.00033295, + "epoch": 0.17601996401833905, + "flos": 65938082088960.0, + "grad_norm": 0.6952574923770289, + "language_loss": 0.4625091, + "learning_rate": 3.780491630948914e-06, + "loss": 0.48271221, + "num_input_tokens_seen": 173595105, + "router_z_loss_clip": 0.07763672, + "router_z_loss_mlp": 0.00964355, + "step": 6066, + "time_per_iteration": 2.9419422149658203 + }, + { + "auxiliary_loss_clip": 0.01091943, + "auxiliary_loss_mlp": 0.01039799, + "balance_loss_clip": 1.03275609, + "balance_loss_mlp": 1.02154803, + "epoch": 0.17604898148685508, + "flos": 14127185122560.0, + "grad_norm": 14.781277955173518, + "language_loss": 0.9005177, + "learning_rate": 3.780406010195326e-06, + "loss": 0.92183518, + "num_input_tokens_seen": 173605935, + "router_z_loss_clip": 0.59179688, + "router_z_loss_mlp": 0.18249512, + "step": 6067, + "time_per_iteration": 2.350186824798584 + }, + { + "auxiliary_loss_clip": 0.01087993, + "auxiliary_loss_mlp": 0.01039222, + "balance_loss_clip": 1.03350413, + "balance_loss_mlp": 1.02215683, + "epoch": 0.17607799895537113, + "flos": 10660763629440.0, + "grad_norm": 3.085626141282717, + "language_loss": 0.81680179, + "learning_rate": 3.7803203737164714e-06, + "loss": 0.83807397, + "num_input_tokens_seen": 173617090, + "router_z_loss_clip": 0.54541016, + "router_z_loss_mlp": 0.17059326, + "step": 6068, + "time_per_iteration": 2.350041627883911 + }, + { + "auxiliary_loss_clip": 0.01079462, + "auxiliary_loss_mlp": 0.01029056, + "balance_loss_clip": 1.0305388, + "balance_loss_mlp": 1.01487613, + "epoch": 0.17610701642388718, + "flos": 27012265338240.0, + "grad_norm": 2.1745536791385867, + "language_loss": 0.89558017, + "learning_rate": 3.780234721513108e-06, + "loss": 0.91666538, + "num_input_tokens_seen": 173635555, + "router_z_loss_clip": 0.48950195, + "router_z_loss_mlp": 0.14178467, + "step": 6069, + "time_per_iteration": 2.4808638095855713 + }, + { + "auxiliary_loss_clip": 0.01086805, + "auxiliary_loss_mlp": 0.01038209, + "balance_loss_clip": 1.03188515, + "balance_loss_mlp": 1.02091169, + "epoch": 0.17613603389240323, + "flos": 20844613820160.0, + "grad_norm": 2.9147602698092787, + "language_loss": 0.89515048, + "learning_rate": 3.7801490535859905e-06, + "loss": 0.91640061, + "num_input_tokens_seen": 173652970, + "router_z_loss_clip": 0.54882812, + "router_z_loss_mlp": 0.17285156, + "step": 6070, + "time_per_iteration": 2.413679361343384 + }, + { + "auxiliary_loss_clip": 0.01019472, + "auxiliary_loss_mlp": 0.01001161, + "balance_loss_clip": 1.01111162, + "balance_loss_mlp": 1.00029647, + "epoch": 0.17616505136091928, + "flos": 64374496583040.0, + "grad_norm": 0.6717476876129145, + "language_loss": 0.54247963, + "learning_rate": 3.7800633699358757e-06, + "loss": 0.56268597, + "num_input_tokens_seen": 173714530, + "router_z_loss_clip": 0.08398438, + "router_z_loss_mlp": 0.00866699, + "step": 6071, + "time_per_iteration": 3.0134809017181396 + }, + { + "auxiliary_loss_clip": 0.0108267, + "auxiliary_loss_mlp": 0.01039076, + "balance_loss_clip": 1.03174102, + "balance_loss_mlp": 1.02419853, + "epoch": 0.1761940688294353, + "flos": 12048211992960.0, + "grad_norm": 2.6345828843414907, + "language_loss": 0.79221898, + "learning_rate": 3.7799776705635216e-06, + "loss": 0.81343639, + "num_input_tokens_seen": 173726120, + "router_z_loss_clip": 0.50927734, + "router_z_loss_mlp": 0.14868164, + "step": 6072, + "time_per_iteration": 2.33738374710083 + }, + { + "auxiliary_loss_clip": 0.01079267, + "auxiliary_loss_mlp": 0.01036394, + "balance_loss_clip": 1.02984738, + "balance_loss_mlp": 1.02178526, + "epoch": 0.17622308629795136, + "flos": 16682373463680.0, + "grad_norm": 2.1088580122952827, + "language_loss": 0.74539721, + "learning_rate": 3.779891955469684e-06, + "loss": 0.76655382, + "num_input_tokens_seen": 173739255, + "router_z_loss_clip": 0.49438477, + "router_z_loss_mlp": 0.14605713, + "step": 6073, + "time_per_iteration": 2.4413766860961914 + }, + { + "auxiliary_loss_clip": 0.01078721, + "auxiliary_loss_mlp": 0.01034812, + "balance_loss_clip": 1.0282464, + "balance_loss_mlp": 1.02063191, + "epoch": 0.1762521037664674, + "flos": 35727773811840.0, + "grad_norm": 1.918109453809641, + "language_loss": 0.75529152, + "learning_rate": 3.7798062246551206e-06, + "loss": 0.77642691, + "num_input_tokens_seen": 173756220, + "router_z_loss_clip": 0.50439453, + "router_z_loss_mlp": 0.1416626, + "step": 6074, + "time_per_iteration": 2.5015974044799805 + }, + { + "auxiliary_loss_clip": 0.01088069, + "auxiliary_loss_mlp": 0.01043948, + "balance_loss_clip": 1.02895117, + "balance_loss_mlp": 1.02632332, + "epoch": 0.17628112123498346, + "flos": 36440245393920.0, + "grad_norm": 2.7315825617062504, + "language_loss": 0.73636377, + "learning_rate": 3.7797204781205886e-06, + "loss": 0.75768387, + "num_input_tokens_seen": 173776285, + "router_z_loss_clip": 0.59008789, + "router_z_loss_mlp": 0.17626953, + "step": 6075, + "time_per_iteration": 2.525787115097046 + }, + { + "auxiliary_loss_clip": 0.0101515, + "auxiliary_loss_mlp": 0.01005179, + "balance_loss_clip": 1.00736451, + "balance_loss_mlp": 1.00436282, + "epoch": 0.1763101387034995, + "flos": 52342727840640.0, + "grad_norm": 0.6631117718887043, + "language_loss": 0.48249996, + "learning_rate": 3.7796347158668455e-06, + "loss": 0.50270319, + "num_input_tokens_seen": 173835275, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.00817871, + "step": 6076, + "time_per_iteration": 3.051666736602783 + }, + { + "auxiliary_loss_clip": 0.01083422, + "auxiliary_loss_mlp": 0.01036446, + "balance_loss_clip": 1.02805758, + "balance_loss_mlp": 1.02128291, + "epoch": 0.17633915617201557, + "flos": 21028047436800.0, + "grad_norm": 1.8302071315115087, + "language_loss": 0.73860395, + "learning_rate": 3.779548937894647e-06, + "loss": 0.75980264, + "num_input_tokens_seen": 173853540, + "router_z_loss_clip": 0.55419922, + "router_z_loss_mlp": 0.1517334, + "step": 6077, + "time_per_iteration": 2.455887794494629 + }, + { + "auxiliary_loss_clip": 0.01012373, + "auxiliary_loss_mlp": 0.01010502, + "balance_loss_clip": 1.00490475, + "balance_loss_mlp": 1.00966191, + "epoch": 0.1763681736405316, + "flos": 71160390190080.0, + "grad_norm": 0.6547089176411705, + "language_loss": 0.46486121, + "learning_rate": 3.7794631442047534e-06, + "loss": 0.48508999, + "num_input_tokens_seen": 173918690, + "router_z_loss_clip": 0.07470703, + "router_z_loss_mlp": 0.00842285, + "step": 6078, + "time_per_iteration": 3.0743348598480225 + }, + { + "auxiliary_loss_clip": 0.01081302, + "auxiliary_loss_mlp": 0.01043824, + "balance_loss_clip": 1.02699232, + "balance_loss_mlp": 1.0274024, + "epoch": 0.17639719110904764, + "flos": 29672333003520.0, + "grad_norm": 3.4102850475499165, + "language_loss": 0.83432192, + "learning_rate": 3.779377334797921e-06, + "loss": 0.85557318, + "num_input_tokens_seen": 173937460, + "router_z_loss_clip": 0.54370117, + "router_z_loss_mlp": 0.16418457, + "step": 6079, + "time_per_iteration": 2.443948745727539 + }, + { + "auxiliary_loss_clip": 0.01010729, + "auxiliary_loss_mlp": 0.01002324, + "balance_loss_clip": 1.00329471, + "balance_loss_mlp": 1.00136411, + "epoch": 0.1764262085775637, + "flos": 66415868311680.0, + "grad_norm": 0.8039414201902242, + "language_loss": 0.51740229, + "learning_rate": 3.779291509674908e-06, + "loss": 0.53753281, + "num_input_tokens_seen": 173992715, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00958252, + "step": 6080, + "time_per_iteration": 2.9623067378997803 + }, + { + "auxiliary_loss_clip": 0.01077838, + "auxiliary_loss_mlp": 0.01039329, + "balance_loss_clip": 1.02604783, + "balance_loss_mlp": 1.02386117, + "epoch": 0.17645522604607974, + "flos": 31131143919360.0, + "grad_norm": 3.1676681697347657, + "language_loss": 0.72370172, + "learning_rate": 3.7792056688364725e-06, + "loss": 0.7448734, + "num_input_tokens_seen": 174009065, + "router_z_loss_clip": 0.51782227, + "router_z_loss_mlp": 0.15460205, + "step": 6081, + "time_per_iteration": 2.479753255844116 + }, + { + "auxiliary_loss_clip": 0.01079552, + "auxiliary_loss_mlp": 0.01040319, + "balance_loss_clip": 1.02655375, + "balance_loss_mlp": 1.02260458, + "epoch": 0.1764842435145958, + "flos": 41308675153920.0, + "grad_norm": 2.8640994818083416, + "language_loss": 0.8342154, + "learning_rate": 3.779119812283372e-06, + "loss": 0.85541415, + "num_input_tokens_seen": 174024975, + "router_z_loss_clip": 0.5300293, + "router_z_loss_mlp": 0.17718506, + "step": 6082, + "time_per_iteration": 2.6035420894622803 + }, + { + "auxiliary_loss_clip": 0.01085932, + "auxiliary_loss_mlp": 0.01048794, + "balance_loss_clip": 1.02801275, + "balance_loss_mlp": 1.0286243, + "epoch": 0.17651326098311185, + "flos": 28688375756160.0, + "grad_norm": 1.985847659152708, + "language_loss": 1.01572132, + "learning_rate": 3.779033940016366e-06, + "loss": 1.03706861, + "num_input_tokens_seen": 174044125, + "router_z_loss_clip": 0.57910156, + "router_z_loss_mlp": 0.20166016, + "step": 6083, + "time_per_iteration": 2.4543416500091553 + }, + { + "auxiliary_loss_clip": 0.01083606, + "auxiliary_loss_mlp": 0.01035524, + "balance_loss_clip": 1.03216076, + "balance_loss_mlp": 1.01862597, + "epoch": 0.17654227845162787, + "flos": 22885357144320.0, + "grad_norm": 2.928802235503082, + "language_loss": 0.74319375, + "learning_rate": 3.7789480520362117e-06, + "loss": 0.76438504, + "num_input_tokens_seen": 174056960, + "router_z_loss_clip": 0.51416016, + "router_z_loss_mlp": 0.16906738, + "step": 6084, + "time_per_iteration": 2.471120595932007 + }, + { + "auxiliary_loss_clip": 0.01088011, + "auxiliary_loss_mlp": 0.01040095, + "balance_loss_clip": 1.03320527, + "balance_loss_mlp": 1.02406716, + "epoch": 0.17657129592014392, + "flos": 31788209306880.0, + "grad_norm": 2.9598845337170947, + "language_loss": 0.81777167, + "learning_rate": 3.778862148343669e-06, + "loss": 0.8390528, + "num_input_tokens_seen": 174072830, + "router_z_loss_clip": 0.5480957, + "router_z_loss_mlp": 0.16046143, + "step": 6085, + "time_per_iteration": 2.4802663326263428 + }, + { + "auxiliary_loss_clip": 0.01085937, + "auxiliary_loss_mlp": 0.01035769, + "balance_loss_clip": 1.03268552, + "balance_loss_mlp": 1.0200274, + "epoch": 0.17660031338865997, + "flos": 29377002879360.0, + "grad_norm": 2.7201145174939914, + "language_loss": 0.98726046, + "learning_rate": 3.7787762289394954e-06, + "loss": 1.00847745, + "num_input_tokens_seen": 174087100, + "router_z_loss_clip": 0.5324707, + "router_z_loss_mlp": 0.1574707, + "step": 6086, + "time_per_iteration": 4.718475103378296 + }, + { + "auxiliary_loss_clip": 0.01091169, + "auxiliary_loss_mlp": 0.01042044, + "balance_loss_clip": 1.03690994, + "balance_loss_mlp": 1.02543807, + "epoch": 0.17662933085717603, + "flos": 21536836813440.0, + "grad_norm": 2.0647034443647923, + "language_loss": 0.82563186, + "learning_rate": 3.778690293824451e-06, + "loss": 0.846964, + "num_input_tokens_seen": 174103210, + "router_z_loss_clip": 0.54272461, + "router_z_loss_mlp": 0.16607666, + "step": 6087, + "time_per_iteration": 4.573576211929321 + }, + { + "auxiliary_loss_clip": 0.01080756, + "auxiliary_loss_mlp": 0.01026529, + "balance_loss_clip": 1.03140593, + "balance_loss_mlp": 1.01167607, + "epoch": 0.17665834832569208, + "flos": 19928283609600.0, + "grad_norm": 2.9236737893532574, + "language_loss": 1.0413307, + "learning_rate": 3.7786043429992935e-06, + "loss": 1.06240356, + "num_input_tokens_seen": 174115255, + "router_z_loss_clip": 0.49365234, + "router_z_loss_mlp": 0.14862061, + "step": 6088, + "time_per_iteration": 2.404207468032837 + }, + { + "auxiliary_loss_clip": 0.01092709, + "auxiliary_loss_mlp": 0.01039314, + "balance_loss_clip": 1.03535569, + "balance_loss_mlp": 1.02143312, + "epoch": 0.1766873657942081, + "flos": 33211234212480.0, + "grad_norm": 2.227671023607196, + "language_loss": 0.85758549, + "learning_rate": 3.7785183764647827e-06, + "loss": 0.87890577, + "num_input_tokens_seen": 174135190, + "router_z_loss_clip": 0.57299805, + "router_z_loss_mlp": 0.17895508, + "step": 6089, + "time_per_iteration": 2.492689847946167 + }, + { + "auxiliary_loss_clip": 0.01084396, + "auxiliary_loss_mlp": 0.01029938, + "balance_loss_clip": 1.03316331, + "balance_loss_mlp": 1.01446521, + "epoch": 0.17671638326272415, + "flos": 12266209722240.0, + "grad_norm": 3.3930954517189345, + "language_loss": 0.93148386, + "learning_rate": 3.7784323942216788e-06, + "loss": 0.95262718, + "num_input_tokens_seen": 174143995, + "router_z_loss_clip": 0.51293945, + "router_z_loss_mlp": 0.15472412, + "step": 6090, + "time_per_iteration": 2.373478889465332 + }, + { + "auxiliary_loss_clip": 0.01024081, + "auxiliary_loss_mlp": 0.01005022, + "balance_loss_clip": 1.01552844, + "balance_loss_mlp": 1.00414586, + "epoch": 0.1767454007312402, + "flos": 57211751093760.0, + "grad_norm": 0.7762960192278783, + "language_loss": 0.44699606, + "learning_rate": 3.7783463962707397e-06, + "loss": 0.46728706, + "num_input_tokens_seen": 174198785, + "router_z_loss_clip": 0.0859375, + "router_z_loss_mlp": 0.00878906, + "step": 6091, + "time_per_iteration": 2.8428521156311035 + }, + { + "auxiliary_loss_clip": 0.01024691, + "auxiliary_loss_mlp": 0.00999501, + "balance_loss_clip": 1.01603568, + "balance_loss_mlp": 0.99861926, + "epoch": 0.17677441819975626, + "flos": 61046890122240.0, + "grad_norm": 0.7631017039905613, + "language_loss": 0.48799515, + "learning_rate": 3.778260382612726e-06, + "loss": 0.508237, + "num_input_tokens_seen": 174258665, + "router_z_loss_clip": 0.08642578, + "router_z_loss_mlp": 0.0088501, + "step": 6092, + "time_per_iteration": 2.9940438270568848 + }, + { + "auxiliary_loss_clip": 0.01025404, + "auxiliary_loss_mlp": 0.0099948, + "balance_loss_clip": 1.01645732, + "balance_loss_mlp": 0.99857372, + "epoch": 0.1768034356682723, + "flos": 65872724290560.0, + "grad_norm": 0.7322676521181299, + "language_loss": 0.55189061, + "learning_rate": 3.778174353248396e-06, + "loss": 0.5721395, + "num_input_tokens_seen": 174317125, + "router_z_loss_clip": 0.08935547, + "router_z_loss_mlp": 0.0090332, + "step": 6093, + "time_per_iteration": 2.9587953090667725 + }, + { + "auxiliary_loss_clip": 0.01088049, + "auxiliary_loss_mlp": 0.0103361, + "balance_loss_clip": 1.03395534, + "balance_loss_mlp": 1.01730227, + "epoch": 0.17683245313678836, + "flos": 20551692579840.0, + "grad_norm": 2.2629767298004175, + "language_loss": 0.74374986, + "learning_rate": 3.778088308178511e-06, + "loss": 0.76496649, + "num_input_tokens_seen": 174334160, + "router_z_loss_clip": 0.54125977, + "router_z_loss_mlp": 0.16296387, + "step": 6094, + "time_per_iteration": 2.4487879276275635 + }, + { + "auxiliary_loss_clip": 0.01089711, + "auxiliary_loss_mlp": 0.01035488, + "balance_loss_clip": 1.03674436, + "balance_loss_mlp": 1.01958585, + "epoch": 0.17686147060530438, + "flos": 32078896220160.0, + "grad_norm": 2.3966300355966372, + "language_loss": 1.00271666, + "learning_rate": 3.7780022474038313e-06, + "loss": 1.0239687, + "num_input_tokens_seen": 174350485, + "router_z_loss_clip": 0.52954102, + "router_z_loss_mlp": 0.15905762, + "step": 6095, + "time_per_iteration": 2.485875129699707 + }, + { + "auxiliary_loss_clip": 0.0109003, + "auxiliary_loss_mlp": 0.01042298, + "balance_loss_clip": 1.03404284, + "balance_loss_mlp": 1.02518225, + "epoch": 0.17689048807382043, + "flos": 27153908190720.0, + "grad_norm": 2.5877090459925425, + "language_loss": 0.82813847, + "learning_rate": 3.7779161709251157e-06, + "loss": 0.84946167, + "num_input_tokens_seen": 174365630, + "router_z_loss_clip": 0.55932617, + "router_z_loss_mlp": 0.17120361, + "step": 6096, + "time_per_iteration": 4.919497966766357 + }, + { + "auxiliary_loss_clip": 0.01081826, + "auxiliary_loss_mlp": 0.0104625, + "balance_loss_clip": 1.02816081, + "balance_loss_mlp": 1.03046703, + "epoch": 0.17691950554233649, + "flos": 32153330972160.0, + "grad_norm": 2.091807328075077, + "language_loss": 0.69674253, + "learning_rate": 3.777830078743125e-06, + "loss": 0.7180233, + "num_input_tokens_seen": 174382355, + "router_z_loss_clip": 0.53662109, + "router_z_loss_mlp": 0.15777588, + "step": 6097, + "time_per_iteration": 2.512556552886963 + }, + { + "auxiliary_loss_clip": 0.01088044, + "auxiliary_loss_mlp": 0.01042852, + "balance_loss_clip": 1.03227603, + "balance_loss_mlp": 1.02646101, + "epoch": 0.17694852301085254, + "flos": 11536245682560.0, + "grad_norm": 3.1863986425529465, + "language_loss": 0.83603942, + "learning_rate": 3.77774397085862e-06, + "loss": 0.85734844, + "num_input_tokens_seen": 174391735, + "router_z_loss_clip": 0.55810547, + "router_z_loss_mlp": 0.16394043, + "step": 6098, + "time_per_iteration": 2.322679042816162 + }, + { + "auxiliary_loss_clip": 0.01082556, + "auxiliary_loss_mlp": 0.01043173, + "balance_loss_clip": 1.02915072, + "balance_loss_mlp": 1.02719891, + "epoch": 0.1769775404793686, + "flos": 31172026988160.0, + "grad_norm": 2.7495364409900693, + "language_loss": 0.86106586, + "learning_rate": 3.77765784727236e-06, + "loss": 0.88232309, + "num_input_tokens_seen": 174408855, + "router_z_loss_clip": 0.53466797, + "router_z_loss_mlp": 0.159729, + "step": 6099, + "time_per_iteration": 2.4370791912078857 + }, + { + "auxiliary_loss_clip": 0.01076027, + "auxiliary_loss_mlp": 0.01040861, + "balance_loss_clip": 1.0286696, + "balance_loss_mlp": 1.02690196, + "epoch": 0.1770065579478846, + "flos": 22422268604160.0, + "grad_norm": 2.174790298981699, + "language_loss": 0.82694417, + "learning_rate": 3.777571707985108e-06, + "loss": 0.84811306, + "num_input_tokens_seen": 174424265, + "router_z_loss_clip": 0.47363281, + "router_z_loss_mlp": 0.13964844, + "step": 6100, + "time_per_iteration": 2.443969964981079 + }, + { + "auxiliary_loss_clip": 0.01081445, + "auxiliary_loss_mlp": 0.01047768, + "balance_loss_clip": 1.02729642, + "balance_loss_mlp": 1.03228283, + "epoch": 0.17703557541640066, + "flos": 25545773923200.0, + "grad_norm": 2.9589113389066393, + "language_loss": 0.88023418, + "learning_rate": 3.7774855529976222e-06, + "loss": 0.90152633, + "num_input_tokens_seen": 174435695, + "router_z_loss_clip": 0.54101562, + "router_z_loss_mlp": 0.15472412, + "step": 6101, + "time_per_iteration": 2.4701688289642334 + }, + { + "auxiliary_loss_clip": 0.01082796, + "auxiliary_loss_mlp": 0.01048095, + "balance_loss_clip": 1.02920163, + "balance_loss_mlp": 1.03113174, + "epoch": 0.17706459288491672, + "flos": 44519357871360.0, + "grad_norm": 2.363219166285868, + "language_loss": 0.89805925, + "learning_rate": 3.7773993823106647e-06, + "loss": 0.91936815, + "num_input_tokens_seen": 174451675, + "router_z_loss_clip": 0.53588867, + "router_z_loss_mlp": 0.16955566, + "step": 6102, + "time_per_iteration": 2.591325044631958 + }, + { + "auxiliary_loss_clip": 0.01018485, + "auxiliary_loss_mlp": 0.01028971, + "balance_loss_clip": 1.01067483, + "balance_loss_mlp": 1.02821136, + "epoch": 0.17709361035343277, + "flos": 63573484193280.0, + "grad_norm": 0.7697681999607947, + "language_loss": 0.48035288, + "learning_rate": 3.777313195924997e-06, + "loss": 0.50082743, + "num_input_tokens_seen": 174515320, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.00759888, + "step": 6103, + "time_per_iteration": 3.0052056312561035 + }, + { + "auxiliary_loss_clip": 0.01016838, + "auxiliary_loss_mlp": 0.01021433, + "balance_loss_clip": 1.0092845, + "balance_loss_mlp": 1.0205214, + "epoch": 0.17712262782194882, + "flos": 61708179784320.0, + "grad_norm": 0.769589698445512, + "language_loss": 0.44376373, + "learning_rate": 3.77722699384138e-06, + "loss": 0.46414644, + "num_input_tokens_seen": 174579105, + "router_z_loss_clip": 0.07568359, + "router_z_loss_mlp": 0.00909424, + "step": 6104, + "time_per_iteration": 3.1816065311431885 + }, + { + "auxiliary_loss_clip": 0.01079208, + "auxiliary_loss_mlp": 0.01030146, + "balance_loss_clip": 1.02809715, + "balance_loss_mlp": 1.01537013, + "epoch": 0.17715164529046487, + "flos": 37300925030400.0, + "grad_norm": 1.6213488575615949, + "language_loss": 0.76542699, + "learning_rate": 3.777140776060575e-06, + "loss": 0.78652054, + "num_input_tokens_seen": 174607770, + "router_z_loss_clip": 0.51074219, + "router_z_loss_mlp": 0.14776611, + "step": 6105, + "time_per_iteration": 2.8382294178009033 + }, + { + "auxiliary_loss_clip": 0.01084045, + "auxiliary_loss_mlp": 0.01044964, + "balance_loss_clip": 1.0311861, + "balance_loss_mlp": 1.02765453, + "epoch": 0.1771806627589809, + "flos": 26936050106880.0, + "grad_norm": 2.462625874615041, + "language_loss": 0.95241809, + "learning_rate": 3.777054542583343e-06, + "loss": 0.97370815, + "num_input_tokens_seen": 174623240, + "router_z_loss_clip": 0.52880859, + "router_z_loss_mlp": 0.17315674, + "step": 6106, + "time_per_iteration": 2.4437830448150635 + }, + { + "auxiliary_loss_clip": 0.01084226, + "auxiliary_loss_mlp": 0.01032952, + "balance_loss_clip": 1.03109396, + "balance_loss_mlp": 1.01638174, + "epoch": 0.17720968022749695, + "flos": 74728376294400.0, + "grad_norm": 2.304576973278666, + "language_loss": 0.860165, + "learning_rate": 3.776968293410447e-06, + "loss": 0.88133681, + "num_input_tokens_seen": 174643855, + "router_z_loss_clip": 0.53198242, + "router_z_loss_mlp": 0.16589355, + "step": 6107, + "time_per_iteration": 2.795287847518921 + }, + { + "auxiliary_loss_clip": 0.01019557, + "auxiliary_loss_mlp": 0.01001139, + "balance_loss_clip": 1.01170659, + "balance_loss_mlp": 1.00029862, + "epoch": 0.177238697696013, + "flos": 74773411948800.0, + "grad_norm": 0.6751486774903673, + "language_loss": 0.46229106, + "learning_rate": 3.7768820285426477e-06, + "loss": 0.48249802, + "num_input_tokens_seen": 174709680, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.00842285, + "step": 6108, + "time_per_iteration": 3.054631471633911 + }, + { + "auxiliary_loss_clip": 0.01087257, + "auxiliary_loss_mlp": 0.01035226, + "balance_loss_clip": 1.02861214, + "balance_loss_mlp": 1.01716578, + "epoch": 0.17726771516452905, + "flos": 31317230799360.0, + "grad_norm": 2.4376114353124576, + "language_loss": 0.96477365, + "learning_rate": 3.7767957479807074e-06, + "loss": 0.98599851, + "num_input_tokens_seen": 174727330, + "router_z_loss_clip": 0.58618164, + "router_z_loss_mlp": 0.18060303, + "step": 6109, + "time_per_iteration": 2.449504852294922 + }, + { + "auxiliary_loss_clip": 0.01081741, + "auxiliary_loss_mlp": 0.0103891, + "balance_loss_clip": 1.03092897, + "balance_loss_mlp": 1.02379441, + "epoch": 0.1772967326330451, + "flos": 14837143086720.0, + "grad_norm": 2.0809595420652194, + "language_loss": 0.70079887, + "learning_rate": 3.7767094517253874e-06, + "loss": 0.72200537, + "num_input_tokens_seen": 174742645, + "router_z_loss_clip": 0.50952148, + "router_z_loss_mlp": 0.15106201, + "step": 6110, + "time_per_iteration": 2.3972527980804443 + }, + { + "auxiliary_loss_clip": 0.01087836, + "auxiliary_loss_mlp": 0.01037973, + "balance_loss_clip": 1.03492868, + "balance_loss_mlp": 1.02246356, + "epoch": 0.17732575010156115, + "flos": 22341549807360.0, + "grad_norm": 2.289752065334264, + "language_loss": 0.73826361, + "learning_rate": 3.776623139777451e-06, + "loss": 0.7595216, + "num_input_tokens_seen": 174758795, + "router_z_loss_clip": 0.52856445, + "router_z_loss_mlp": 0.1550293, + "step": 6111, + "time_per_iteration": 2.3974926471710205 + }, + { + "auxiliary_loss_clip": 0.01087694, + "auxiliary_loss_mlp": 0.01029463, + "balance_loss_clip": 1.03108156, + "balance_loss_mlp": 1.01402581, + "epoch": 0.17735476757007718, + "flos": 26062697646720.0, + "grad_norm": 1.6878546142909137, + "language_loss": 0.69917113, + "learning_rate": 3.77653681213766e-06, + "loss": 0.7203427, + "num_input_tokens_seen": 174777715, + "router_z_loss_clip": 0.56640625, + "router_z_loss_mlp": 0.15423584, + "step": 6112, + "time_per_iteration": 2.4152350425720215 + }, + { + "auxiliary_loss_clip": 0.01083263, + "auxiliary_loss_mlp": 0.01031599, + "balance_loss_clip": 1.03131628, + "balance_loss_mlp": 1.01659036, + "epoch": 0.17738378503859323, + "flos": 13946753882880.0, + "grad_norm": 2.5699555422640294, + "language_loss": 0.71955824, + "learning_rate": 3.7764504688067774e-06, + "loss": 0.74070692, + "num_input_tokens_seen": 174791500, + "router_z_loss_clip": 0.51904297, + "router_z_loss_mlp": 0.15026855, + "step": 6113, + "time_per_iteration": 2.3475568294525146 + }, + { + "auxiliary_loss_clip": 0.01026086, + "auxiliary_loss_mlp": 0.01004579, + "balance_loss_clip": 1.01622605, + "balance_loss_mlp": 1.00373244, + "epoch": 0.17741280250710928, + "flos": 67485955616640.0, + "grad_norm": 0.6584397241409994, + "language_loss": 0.47550225, + "learning_rate": 3.776364109785565e-06, + "loss": 0.49580884, + "num_input_tokens_seen": 174859110, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.00848389, + "step": 6114, + "time_per_iteration": 3.098867654800415 + }, + { + "auxiliary_loss_clip": 0.01080971, + "auxiliary_loss_mlp": 0.01034245, + "balance_loss_clip": 1.03008699, + "balance_loss_mlp": 1.02044654, + "epoch": 0.17744181997562533, + "flos": 16792908428160.0, + "grad_norm": 3.171428941341653, + "language_loss": 0.71131909, + "learning_rate": 3.776277735074786e-06, + "loss": 0.73247123, + "num_input_tokens_seen": 174874420, + "router_z_loss_clip": 0.5090332, + "router_z_loss_mlp": 0.13800049, + "step": 6115, + "time_per_iteration": 2.3287460803985596 + }, + { + "auxiliary_loss_clip": 0.01084112, + "auxiliary_loss_mlp": 0.0102824, + "balance_loss_clip": 1.03255773, + "balance_loss_mlp": 1.01267707, + "epoch": 0.17747083744414138, + "flos": 50147600883840.0, + "grad_norm": 6.952004819426966, + "language_loss": 0.66527146, + "learning_rate": 3.7761913446752037e-06, + "loss": 0.68639499, + "num_input_tokens_seen": 174893120, + "router_z_loss_clip": 0.51489258, + "router_z_loss_mlp": 0.15539551, + "step": 6116, + "time_per_iteration": 2.592925548553467 + }, + { + "auxiliary_loss_clip": 0.01087086, + "auxiliary_loss_mlp": 0.01041961, + "balance_loss_clip": 1.03330266, + "balance_loss_mlp": 1.02467561, + "epoch": 0.1774998549126574, + "flos": 14791407338880.0, + "grad_norm": 3.2248335454191643, + "language_loss": 0.92660135, + "learning_rate": 3.77610493858758e-06, + "loss": 0.94789183, + "num_input_tokens_seen": 174905470, + "router_z_loss_clip": 0.53808594, + "router_z_loss_mlp": 0.17285156, + "step": 6117, + "time_per_iteration": 2.409921407699585 + }, + { + "auxiliary_loss_clip": 0.01020124, + "auxiliary_loss_mlp": 0.01001537, + "balance_loss_clip": 1.01111054, + "balance_loss_mlp": 1.00058961, + "epoch": 0.17752887238117346, + "flos": 72546267542400.0, + "grad_norm": 0.6511795585733143, + "language_loss": 0.47176072, + "learning_rate": 3.776018516812679e-06, + "loss": 0.4919773, + "num_input_tokens_seen": 174968590, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.00946045, + "step": 6118, + "time_per_iteration": 3.1129369735717773 + }, + { + "auxiliary_loss_clip": 0.0101813, + "auxiliary_loss_mlp": 0.01001248, + "balance_loss_clip": 1.00941849, + "balance_loss_mlp": 1.00034821, + "epoch": 0.1775578898496895, + "flos": 68471344229760.0, + "grad_norm": 0.6486205174011404, + "language_loss": 0.47409502, + "learning_rate": 3.7759320793512643e-06, + "loss": 0.49428883, + "num_input_tokens_seen": 175034465, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.00897217, + "step": 6119, + "time_per_iteration": 3.1082236766815186 + }, + { + "auxiliary_loss_clip": 0.01080608, + "auxiliary_loss_mlp": 0.01036159, + "balance_loss_clip": 1.02909505, + "balance_loss_mlp": 1.02035236, + "epoch": 0.17758690731820556, + "flos": 14894645829120.0, + "grad_norm": 2.9684362138807985, + "language_loss": 0.87338936, + "learning_rate": 3.7758456262040986e-06, + "loss": 0.894557, + "num_input_tokens_seen": 175045945, + "router_z_loss_clip": 0.51538086, + "router_z_loss_mlp": 0.15814209, + "step": 6120, + "time_per_iteration": 2.358541488647461 + }, + { + "auxiliary_loss_clip": 0.01084681, + "auxiliary_loss_mlp": 0.01031466, + "balance_loss_clip": 1.02723742, + "balance_loss_mlp": 1.01496768, + "epoch": 0.1776159247867216, + "flos": 47414320364160.0, + "grad_norm": 1.9219279938162672, + "language_loss": 0.75674558, + "learning_rate": 3.7757591573719456e-06, + "loss": 0.77790701, + "num_input_tokens_seen": 175068035, + "router_z_loss_clip": 0.57421875, + "router_z_loss_mlp": 0.16516113, + "step": 6121, + "time_per_iteration": 2.624842405319214 + }, + { + "auxiliary_loss_clip": 0.01084815, + "auxiliary_loss_mlp": 0.01032171, + "balance_loss_clip": 1.02810752, + "balance_loss_mlp": 1.01474309, + "epoch": 0.17764494225523766, + "flos": 32664354675840.0, + "grad_norm": 2.229430551822806, + "language_loss": 0.98187983, + "learning_rate": 3.7756726728555686e-06, + "loss": 1.00304961, + "num_input_tokens_seen": 175086630, + "router_z_loss_clip": 0.56713867, + "router_z_loss_mlp": 0.17425537, + "step": 6122, + "time_per_iteration": 2.5168814659118652 + }, + { + "auxiliary_loss_clip": 0.0107761, + "auxiliary_loss_mlp": 0.01036888, + "balance_loss_clip": 1.0272857, + "balance_loss_mlp": 1.02131307, + "epoch": 0.1776739597237537, + "flos": 26975850923520.0, + "grad_norm": 1.9027889638914686, + "language_loss": 1.0851059, + "learning_rate": 3.775586172655733e-06, + "loss": 1.10625088, + "num_input_tokens_seen": 175105735, + "router_z_loss_clip": 0.50292969, + "router_z_loss_mlp": 0.15582275, + "step": 6123, + "time_per_iteration": 2.4516875743865967 + }, + { + "auxiliary_loss_clip": 0.01013745, + "auxiliary_loss_mlp": 0.01001258, + "balance_loss_clip": 1.0059135, + "balance_loss_mlp": 1.0003643, + "epoch": 0.17770297719226974, + "flos": 74774075264640.0, + "grad_norm": 0.6473561724218148, + "language_loss": 0.41866511, + "learning_rate": 3.775499656773201e-06, + "loss": 0.43881518, + "num_input_tokens_seen": 175169920, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.00891113, + "step": 6124, + "time_per_iteration": 3.061779737472534 + }, + { + "auxiliary_loss_clip": 0.01081547, + "auxiliary_loss_mlp": 0.01032033, + "balance_loss_clip": 1.02699828, + "balance_loss_mlp": 1.01511693, + "epoch": 0.1777319946607858, + "flos": 14493807976320.0, + "grad_norm": 2.715423871794057, + "language_loss": 0.79004443, + "learning_rate": 3.7754131252087377e-06, + "loss": 0.81118023, + "num_input_tokens_seen": 175183315, + "router_z_loss_clip": 0.54516602, + "router_z_loss_mlp": 0.16912842, + "step": 6125, + "time_per_iteration": 2.3507723808288574 + }, + { + "auxiliary_loss_clip": 0.01076249, + "auxiliary_loss_mlp": 0.01031514, + "balance_loss_clip": 1.02561426, + "balance_loss_mlp": 1.01550448, + "epoch": 0.17776101212930184, + "flos": 30295707062400.0, + "grad_norm": 2.0366621519628088, + "language_loss": 0.82628119, + "learning_rate": 3.7753265779631076e-06, + "loss": 0.84735888, + "num_input_tokens_seen": 175200455, + "router_z_loss_clip": 0.50610352, + "router_z_loss_mlp": 0.15997314, + "step": 6126, + "time_per_iteration": 2.4761602878570557 + }, + { + "auxiliary_loss_clip": 0.01085285, + "auxiliary_loss_mlp": 0.01038695, + "balance_loss_clip": 1.02990961, + "balance_loss_mlp": 1.02163601, + "epoch": 0.1777900295978179, + "flos": 35986689521280.0, + "grad_norm": 2.1025326856636117, + "language_loss": 0.81561321, + "learning_rate": 3.7752400150370745e-06, + "loss": 0.83685297, + "num_input_tokens_seen": 175224690, + "router_z_loss_clip": 0.55322266, + "router_z_loss_mlp": 0.17053223, + "step": 6127, + "time_per_iteration": 2.558598041534424 + }, + { + "auxiliary_loss_clip": 0.01082858, + "auxiliary_loss_mlp": 0.01034922, + "balance_loss_clip": 1.02660739, + "balance_loss_mlp": 1.01808333, + "epoch": 0.17781904706633395, + "flos": 16428589724160.0, + "grad_norm": 2.22719658001873, + "language_loss": 0.78433996, + "learning_rate": 3.775153436431403e-06, + "loss": 0.80551779, + "num_input_tokens_seen": 175238140, + "router_z_loss_clip": 0.56298828, + "router_z_loss_mlp": 0.16851807, + "step": 6128, + "time_per_iteration": 2.36249041557312 + }, + { + "auxiliary_loss_clip": 0.01014767, + "auxiliary_loss_mlp": 0.01002099, + "balance_loss_clip": 1.00691569, + "balance_loss_mlp": 1.00107336, + "epoch": 0.17784806453484997, + "flos": 56090724382080.0, + "grad_norm": 0.7234990677280776, + "language_loss": 0.50780708, + "learning_rate": 3.7750668421468584e-06, + "loss": 0.52797574, + "num_input_tokens_seen": 175293005, + "router_z_loss_clip": 0.07861328, + "router_z_loss_mlp": 0.01025391, + "step": 6129, + "time_per_iteration": 2.9519476890563965 + }, + { + "auxiliary_loss_clip": 0.01080086, + "auxiliary_loss_mlp": 0.01034568, + "balance_loss_clip": 1.02792192, + "balance_loss_mlp": 1.01730037, + "epoch": 0.17787708200336602, + "flos": 19201356858240.0, + "grad_norm": 2.9437949753693227, + "language_loss": 0.98220432, + "learning_rate": 3.7749802321842052e-06, + "loss": 1.00335085, + "num_input_tokens_seen": 175306490, + "router_z_loss_clip": 0.52124023, + "router_z_loss_mlp": 0.17254639, + "step": 6130, + "time_per_iteration": 2.3615970611572266 + }, + { + "auxiliary_loss_clip": 0.01083156, + "auxiliary_loss_mlp": 0.0104136, + "balance_loss_clip": 1.02766109, + "balance_loss_mlp": 1.02427101, + "epoch": 0.17790609947188207, + "flos": 24965796551040.0, + "grad_norm": 1.8323304843829484, + "language_loss": 0.80229318, + "learning_rate": 3.7748936065442085e-06, + "loss": 0.8235383, + "num_input_tokens_seen": 175325270, + "router_z_loss_clip": 0.55517578, + "router_z_loss_mlp": 0.1708374, + "step": 6131, + "time_per_iteration": 2.417088747024536 + }, + { + "auxiliary_loss_clip": 0.01082602, + "auxiliary_loss_mlp": 0.01038037, + "balance_loss_clip": 1.02670455, + "balance_loss_mlp": 1.02021527, + "epoch": 0.17793511694039812, + "flos": 42040280027520.0, + "grad_norm": 1.8796204395242029, + "language_loss": 0.79035932, + "learning_rate": 3.7748069652276325e-06, + "loss": 0.8115657, + "num_input_tokens_seen": 175347245, + "router_z_loss_clip": 0.55981445, + "router_z_loss_mlp": 0.17828369, + "step": 6132, + "time_per_iteration": 2.5672619342803955 + }, + { + "auxiliary_loss_clip": 0.01014795, + "auxiliary_loss_mlp": 0.01003334, + "balance_loss_clip": 1.00713718, + "balance_loss_mlp": 1.00224972, + "epoch": 0.17796413440891418, + "flos": 54925637667840.0, + "grad_norm": 0.7080321553105743, + "language_loss": 0.52466893, + "learning_rate": 3.774720308235244e-06, + "loss": 0.54485023, + "num_input_tokens_seen": 175411825, + "router_z_loss_clip": 0.07666016, + "router_z_loss_mlp": 0.01086426, + "step": 6133, + "time_per_iteration": 3.0446817874908447 + }, + { + "auxiliary_loss_clip": 0.01013756, + "auxiliary_loss_mlp": 0.01003388, + "balance_loss_clip": 1.00594735, + "balance_loss_mlp": 1.00235105, + "epoch": 0.1779931518774302, + "flos": 74765068133760.0, + "grad_norm": 0.6651079807537346, + "language_loss": 0.43420285, + "learning_rate": 3.7746336355678072e-06, + "loss": 0.45437428, + "num_input_tokens_seen": 175475600, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.01037598, + "step": 6134, + "time_per_iteration": 3.130976676940918 + }, + { + "auxiliary_loss_clip": 0.01013431, + "auxiliary_loss_mlp": 0.01006423, + "balance_loss_clip": 1.00568724, + "balance_loss_mlp": 1.00546908, + "epoch": 0.17802216934594625, + "flos": 60504828353280.0, + "grad_norm": 0.6618106199685712, + "language_loss": 0.50594187, + "learning_rate": 3.7745469472260885e-06, + "loss": 0.52614039, + "num_input_tokens_seen": 175534310, + "router_z_loss_clip": 0.07714844, + "router_z_loss_mlp": 0.00952148, + "step": 6135, + "time_per_iteration": 2.9690332412719727 + }, + { + "auxiliary_loss_clip": 0.01088135, + "auxiliary_loss_mlp": 0.0103524, + "balance_loss_clip": 1.02966142, + "balance_loss_mlp": 1.01683354, + "epoch": 0.1780511868144623, + "flos": 15079510811520.0, + "grad_norm": 2.4345642316374825, + "language_loss": 0.91342688, + "learning_rate": 3.7744602432108527e-06, + "loss": 0.93466055, + "num_input_tokens_seen": 175552130, + "router_z_loss_clip": 0.58398438, + "router_z_loss_mlp": 0.18408203, + "step": 6136, + "time_per_iteration": 2.3764405250549316 + }, + { + "auxiliary_loss_clip": 0.01088274, + "auxiliary_loss_mlp": 0.01043484, + "balance_loss_clip": 1.03027534, + "balance_loss_mlp": 1.02442241, + "epoch": 0.17808020428297835, + "flos": 44702372551680.0, + "grad_norm": 2.121947623881522, + "language_loss": 0.85764027, + "learning_rate": 3.7743735235228654e-06, + "loss": 0.87895787, + "num_input_tokens_seen": 175568115, + "router_z_loss_clip": 0.57958984, + "router_z_loss_mlp": 0.1907959, + "step": 6137, + "time_per_iteration": 2.584601402282715 + }, + { + "auxiliary_loss_clip": 0.01086907, + "auxiliary_loss_mlp": 0.01036644, + "balance_loss_clip": 1.02755833, + "balance_loss_mlp": 1.01953721, + "epoch": 0.1781092217514944, + "flos": 17960089824000.0, + "grad_norm": 6.144568340097058, + "language_loss": 0.82736009, + "learning_rate": 3.774286788162894e-06, + "loss": 0.8485955, + "num_input_tokens_seen": 175581160, + "router_z_loss_clip": 0.59423828, + "router_z_loss_mlp": 0.17102051, + "step": 6138, + "time_per_iteration": 2.360625982284546 + }, + { + "auxiliary_loss_clip": 0.01016735, + "auxiliary_loss_mlp": 0.01005827, + "balance_loss_clip": 1.00876784, + "balance_loss_mlp": 1.0049212, + "epoch": 0.17813823922001046, + "flos": 74773062835200.0, + "grad_norm": 0.6493827624315345, + "language_loss": 0.47383443, + "learning_rate": 3.7742000371317033e-06, + "loss": 0.49406007, + "num_input_tokens_seen": 175646350, + "router_z_loss_clip": 0.07958984, + "router_z_loss_mlp": 0.0090332, + "step": 6139, + "time_per_iteration": 3.0929148197174072 + }, + { + "auxiliary_loss_clip": 0.01081557, + "auxiliary_loss_mlp": 0.0104288, + "balance_loss_clip": 1.0281961, + "balance_loss_mlp": 1.02535081, + "epoch": 0.17816725668852648, + "flos": 35769180551040.0, + "grad_norm": 2.266546753494751, + "language_loss": 0.76204634, + "learning_rate": 3.77411327043006e-06, + "loss": 0.78329062, + "num_input_tokens_seen": 175662105, + "router_z_loss_clip": 0.53417969, + "router_z_loss_mlp": 0.17529297, + "step": 6140, + "time_per_iteration": 2.511009454727173 + }, + { + "auxiliary_loss_clip": 0.01070473, + "auxiliary_loss_mlp": 0.01027102, + "balance_loss_clip": 1.02509522, + "balance_loss_mlp": 1.01448941, + "epoch": 0.17819627415704253, + "flos": 15407310366720.0, + "grad_norm": 1.9504840359484998, + "language_loss": 0.5080362, + "learning_rate": 3.7740264880587305e-06, + "loss": 0.52901196, + "num_input_tokens_seen": 175674670, + "router_z_loss_clip": 0.45410156, + "router_z_loss_mlp": 0.12615967, + "step": 6141, + "time_per_iteration": 2.431535005569458 + }, + { + "auxiliary_loss_clip": 0.01089333, + "auxiliary_loss_mlp": 0.01043466, + "balance_loss_clip": 1.03026462, + "balance_loss_mlp": 1.02402878, + "epoch": 0.17822529162555859, + "flos": 26057984613120.0, + "grad_norm": 3.879277290396402, + "language_loss": 0.82317549, + "learning_rate": 3.7739396900184807e-06, + "loss": 0.84450346, + "num_input_tokens_seen": 175689255, + "router_z_loss_clip": 0.59057617, + "router_z_loss_mlp": 0.19445801, + "step": 6142, + "time_per_iteration": 2.408031463623047 + }, + { + "auxiliary_loss_clip": 0.01085369, + "auxiliary_loss_mlp": 0.01046334, + "balance_loss_clip": 1.02913308, + "balance_loss_mlp": 1.02906013, + "epoch": 0.17825430909407464, + "flos": 26024363107200.0, + "grad_norm": 4.007087918040536, + "language_loss": 0.72399735, + "learning_rate": 3.773852876310078e-06, + "loss": 0.74531448, + "num_input_tokens_seen": 175703565, + "router_z_loss_clip": 0.56201172, + "router_z_loss_mlp": 0.17272949, + "step": 6143, + "time_per_iteration": 2.4200327396392822 + }, + { + "auxiliary_loss_clip": 0.01078816, + "auxiliary_loss_mlp": 0.01040475, + "balance_loss_clip": 1.02890897, + "balance_loss_mlp": 1.02419138, + "epoch": 0.1782833265625907, + "flos": 15881186517120.0, + "grad_norm": 2.0496312120763, + "language_loss": 0.8133539, + "learning_rate": 3.7737660469342893e-06, + "loss": 0.8345468, + "num_input_tokens_seen": 175717800, + "router_z_loss_clip": 0.49853516, + "router_z_loss_mlp": 0.1628418, + "step": 6144, + "time_per_iteration": 2.350224733352661 + }, + { + "auxiliary_loss_clip": 0.01083189, + "auxiliary_loss_mlp": 0.01037944, + "balance_loss_clip": 1.03050959, + "balance_loss_mlp": 1.02235162, + "epoch": 0.17831234403110674, + "flos": 13873925053440.0, + "grad_norm": 2.0423264708505506, + "language_loss": 0.66561949, + "learning_rate": 3.77367920189188e-06, + "loss": 0.68683076, + "num_input_tokens_seen": 175730040, + "router_z_loss_clip": 0.52587891, + "router_z_loss_mlp": 0.15600586, + "step": 6145, + "time_per_iteration": 2.477653980255127 + }, + { + "auxiliary_loss_clip": 0.01090122, + "auxiliary_loss_mlp": 0.01048462, + "balance_loss_clip": 1.03279746, + "balance_loss_mlp": 1.03029466, + "epoch": 0.17834136149962276, + "flos": 26351324789760.0, + "grad_norm": 2.1437740984622016, + "language_loss": 0.76876152, + "learning_rate": 3.7735923411836196e-06, + "loss": 0.7901473, + "num_input_tokens_seen": 175745125, + "router_z_loss_clip": 0.57348633, + "router_z_loss_mlp": 0.1817627, + "step": 6146, + "time_per_iteration": 2.5280919075012207 + }, + { + "auxiliary_loss_clip": 0.01076479, + "auxiliary_loss_mlp": 0.01036045, + "balance_loss_clip": 1.02675128, + "balance_loss_mlp": 1.02085221, + "epoch": 0.17837037896813882, + "flos": 22302237749760.0, + "grad_norm": 1.851578608977929, + "language_loss": 0.73226768, + "learning_rate": 3.7735054648102733e-06, + "loss": 0.75339293, + "num_input_tokens_seen": 175762355, + "router_z_loss_clip": 0.49755859, + "router_z_loss_mlp": 0.15203857, + "step": 6147, + "time_per_iteration": 2.4179508686065674 + }, + { + "auxiliary_loss_clip": 0.01079859, + "auxiliary_loss_mlp": 0.01033757, + "balance_loss_clip": 1.02806759, + "balance_loss_mlp": 1.01856351, + "epoch": 0.17839939643665487, + "flos": 19747817458560.0, + "grad_norm": 3.0209517883927837, + "language_loss": 0.78620219, + "learning_rate": 3.773418572772609e-06, + "loss": 0.80733836, + "num_input_tokens_seen": 175774915, + "router_z_loss_clip": 0.51782227, + "router_z_loss_mlp": 0.1519165, + "step": 6148, + "time_per_iteration": 2.377979278564453 + }, + { + "auxiliary_loss_clip": 0.01079358, + "auxiliary_loss_mlp": 0.01037168, + "balance_loss_clip": 1.02763796, + "balance_loss_mlp": 1.02185583, + "epoch": 0.17842841390517092, + "flos": 48059271509760.0, + "grad_norm": 2.478305771780165, + "language_loss": 0.93227541, + "learning_rate": 3.773331665071395e-06, + "loss": 0.95344061, + "num_input_tokens_seen": 175793720, + "router_z_loss_clip": 0.5168457, + "router_z_loss_mlp": 0.15307617, + "step": 6149, + "time_per_iteration": 2.606157064437866 + }, + { + "auxiliary_loss_clip": 0.01077149, + "auxiliary_loss_mlp": 0.01035578, + "balance_loss_clip": 1.02545691, + "balance_loss_mlp": 1.02028346, + "epoch": 0.17845743137368697, + "flos": 29641364760960.0, + "grad_norm": 2.262845863217617, + "language_loss": 0.77266788, + "learning_rate": 3.773244741707397e-06, + "loss": 0.79379511, + "num_input_tokens_seen": 175813690, + "router_z_loss_clip": 0.515625, + "router_z_loss_mlp": 0.1529541, + "step": 6150, + "time_per_iteration": 2.422405958175659 + }, + { + "auxiliary_loss_clip": 0.01071498, + "auxiliary_loss_mlp": 0.01030785, + "balance_loss_clip": 1.02417755, + "balance_loss_mlp": 1.01731431, + "epoch": 0.178486448842203, + "flos": 30000621317760.0, + "grad_norm": 2.0904368751650373, + "language_loss": 0.73147714, + "learning_rate": 3.773157802681385e-06, + "loss": 0.7525, + "num_input_tokens_seen": 175829075, + "router_z_loss_clip": 0.47363281, + "router_z_loss_mlp": 0.13482666, + "step": 6151, + "time_per_iteration": 2.4625377655029297 + }, + { + "auxiliary_loss_clip": 0.01079509, + "auxiliary_loss_mlp": 0.0103415, + "balance_loss_clip": 1.0267961, + "balance_loss_mlp": 1.01726437, + "epoch": 0.17851546631071905, + "flos": 29890959137280.0, + "grad_norm": 1.9879727566506409, + "language_loss": 0.96224141, + "learning_rate": 3.7730708479941246e-06, + "loss": 0.98337793, + "num_input_tokens_seen": 175846885, + "router_z_loss_clip": 0.52685547, + "router_z_loss_mlp": 0.16882324, + "step": 6152, + "time_per_iteration": 2.439702272415161 + }, + { + "auxiliary_loss_clip": 0.01078343, + "auxiliary_loss_mlp": 0.01033395, + "balance_loss_clip": 1.02626514, + "balance_loss_mlp": 1.01767707, + "epoch": 0.1785444837792351, + "flos": 20624277029760.0, + "grad_norm": 2.7602693777899248, + "language_loss": 0.98176128, + "learning_rate": 3.7729838776463856e-06, + "loss": 1.00287867, + "num_input_tokens_seen": 175861245, + "router_z_loss_clip": 0.52001953, + "router_z_loss_mlp": 0.1572876, + "step": 6153, + "time_per_iteration": 2.4261810779571533 + }, + { + "auxiliary_loss_clip": 0.01012038, + "auxiliary_loss_mlp": 0.0100575, + "balance_loss_clip": 1.00440884, + "balance_loss_mlp": 1.0046953, + "epoch": 0.17857350124775115, + "flos": 56082520212480.0, + "grad_norm": 0.742401446580146, + "language_loss": 0.47098711, + "learning_rate": 3.7728968916389358e-06, + "loss": 0.49116498, + "num_input_tokens_seen": 175913410, + "router_z_loss_clip": 0.07617188, + "router_z_loss_mlp": 0.01055908, + "step": 6154, + "time_per_iteration": 2.8327364921569824 + }, + { + "auxiliary_loss_clip": 0.01011346, + "auxiliary_loss_mlp": 0.01011231, + "balance_loss_clip": 1.00365555, + "balance_loss_mlp": 1.01031351, + "epoch": 0.1786025187162672, + "flos": 63056036799360.0, + "grad_norm": 0.723110620189258, + "language_loss": 0.50655568, + "learning_rate": 3.7728098899725428e-06, + "loss": 0.52678144, + "num_input_tokens_seen": 175969820, + "router_z_loss_clip": 0.07714844, + "router_z_loss_mlp": 0.00915527, + "step": 6155, + "time_per_iteration": 2.8987369537353516 + }, + { + "auxiliary_loss_clip": 0.01076396, + "auxiliary_loss_mlp": 0.01034989, + "balance_loss_clip": 1.0267024, + "balance_loss_mlp": 1.01849055, + "epoch": 0.17863153618478325, + "flos": 36530147744640.0, + "grad_norm": 2.4072168530371183, + "language_loss": 0.92993629, + "learning_rate": 3.772722872647976e-06, + "loss": 0.95105016, + "num_input_tokens_seen": 175985715, + "router_z_loss_clip": 0.49658203, + "router_z_loss_mlp": 0.16522217, + "step": 6156, + "time_per_iteration": 2.4976799488067627 + }, + { + "auxiliary_loss_clip": 0.0101147, + "auxiliary_loss_mlp": 0.01001074, + "balance_loss_clip": 1.00415206, + "balance_loss_mlp": 1.00012636, + "epoch": 0.17866055365329928, + "flos": 55288839208320.0, + "grad_norm": 0.7214536190289857, + "language_loss": 0.50802976, + "learning_rate": 3.7726358396660027e-06, + "loss": 0.52815521, + "num_input_tokens_seen": 176046290, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00946045, + "step": 6157, + "time_per_iteration": 2.9492006301879883 + }, + { + "auxiliary_loss_clip": 0.01079391, + "auxiliary_loss_mlp": 0.01032969, + "balance_loss_clip": 1.02799344, + "balance_loss_mlp": 1.01649404, + "epoch": 0.17868957112181533, + "flos": 23145180549120.0, + "grad_norm": 2.118256108218907, + "language_loss": 0.77598989, + "learning_rate": 3.7725487910273926e-06, + "loss": 0.79711354, + "num_input_tokens_seen": 176063220, + "router_z_loss_clip": 0.51367188, + "router_z_loss_mlp": 0.16467285, + "step": 6158, + "time_per_iteration": 2.4378397464752197 + }, + { + "auxiliary_loss_clip": 0.01010837, + "auxiliary_loss_mlp": 0.01002199, + "balance_loss_clip": 1.00359321, + "balance_loss_mlp": 1.00125158, + "epoch": 0.17871858859033138, + "flos": 63089379014400.0, + "grad_norm": 0.7018197884777555, + "language_loss": 0.52192664, + "learning_rate": 3.7724617267329145e-06, + "loss": 0.54205704, + "num_input_tokens_seen": 176122185, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.00946045, + "step": 6159, + "time_per_iteration": 3.1312880516052246 + }, + { + "auxiliary_loss_clip": 0.01084684, + "auxiliary_loss_mlp": 0.01032842, + "balance_loss_clip": 1.03055096, + "balance_loss_mlp": 1.0155865, + "epoch": 0.17874760605884743, + "flos": 11249608487040.0, + "grad_norm": 4.494084294788935, + "language_loss": 0.68387353, + "learning_rate": 3.772374646783337e-06, + "loss": 0.7050488, + "num_input_tokens_seen": 176135070, + "router_z_loss_clip": 0.54174805, + "router_z_loss_mlp": 0.17266846, + "step": 6160, + "time_per_iteration": 2.40934157371521 + }, + { + "auxiliary_loss_clip": 0.01011129, + "auxiliary_loss_mlp": 0.01001602, + "balance_loss_clip": 1.00384712, + "balance_loss_mlp": 1.00061238, + "epoch": 0.17877662352736348, + "flos": 74783955179520.0, + "grad_norm": 0.6059994867210232, + "language_loss": 0.45087376, + "learning_rate": 3.7722875511794292e-06, + "loss": 0.47100109, + "num_input_tokens_seen": 176204470, + "router_z_loss_clip": 0.07275391, + "router_z_loss_mlp": 0.0098877, + "step": 6161, + "time_per_iteration": 3.1799371242523193 + }, + { + "auxiliary_loss_clip": 0.0107731, + "auxiliary_loss_mlp": 0.01031838, + "balance_loss_clip": 1.02683604, + "balance_loss_mlp": 1.01558399, + "epoch": 0.1788056409958795, + "flos": 14495658278400.0, + "grad_norm": 2.2127546903369057, + "language_loss": 0.68159032, + "learning_rate": 3.7722004399219616e-06, + "loss": 0.70268178, + "num_input_tokens_seen": 176220130, + "router_z_loss_clip": 0.50439453, + "router_z_loss_mlp": 0.16259766, + "step": 6162, + "time_per_iteration": 4.7111570835113525 + }, + { + "auxiliary_loss_clip": 0.01077845, + "auxiliary_loss_mlp": 0.01033354, + "balance_loss_clip": 1.02803624, + "balance_loss_mlp": 1.01786315, + "epoch": 0.17883465846439556, + "flos": 31787964927360.0, + "grad_norm": 2.006981191429775, + "language_loss": 0.70979536, + "learning_rate": 3.772113313011702e-06, + "loss": 0.73090732, + "num_input_tokens_seen": 176236370, + "router_z_loss_clip": 0.49853516, + "router_z_loss_mlp": 0.1550293, + "step": 6163, + "time_per_iteration": 2.511469602584839 + }, + { + "auxiliary_loss_clip": 0.0100987, + "auxiliary_loss_mlp": 0.01004311, + "balance_loss_clip": 1.00225639, + "balance_loss_mlp": 1.00348258, + "epoch": 0.1788636759329116, + "flos": 74767965776640.0, + "grad_norm": 0.7245054242598244, + "language_loss": 0.49291694, + "learning_rate": 3.77202617044942e-06, + "loss": 0.51305872, + "num_input_tokens_seen": 176294155, + "router_z_loss_clip": 0.07617188, + "router_z_loss_mlp": 0.00830078, + "step": 6164, + "time_per_iteration": 3.036088705062866 + }, + { + "auxiliary_loss_clip": 0.01010192, + "auxiliary_loss_mlp": 0.01003544, + "balance_loss_clip": 1.00261784, + "balance_loss_mlp": 1.00262034, + "epoch": 0.17889269340142766, + "flos": 66378860403840.0, + "grad_norm": 0.7082121533829571, + "language_loss": 0.52377295, + "learning_rate": 3.7719390122358867e-06, + "loss": 0.54391038, + "num_input_tokens_seen": 176351850, + "router_z_loss_clip": 0.07568359, + "router_z_loss_mlp": 0.00921631, + "step": 6165, + "time_per_iteration": 2.939084768295288 + }, + { + "auxiliary_loss_clip": 0.01077463, + "auxiliary_loss_mlp": 0.0103328, + "balance_loss_clip": 1.02725685, + "balance_loss_mlp": 1.01828349, + "epoch": 0.1789217108699437, + "flos": 25986726794880.0, + "grad_norm": 1.8418641192001544, + "language_loss": 0.63792431, + "learning_rate": 3.77185183837187e-06, + "loss": 0.65903181, + "num_input_tokens_seen": 176370315, + "router_z_loss_clip": 0.50146484, + "router_z_loss_mlp": 0.15002441, + "step": 6166, + "time_per_iteration": 2.46370005607605 + }, + { + "auxiliary_loss_clip": 0.01085217, + "auxiliary_loss_mlp": 0.01035321, + "balance_loss_clip": 1.02707481, + "balance_loss_mlp": 1.01540124, + "epoch": 0.17895072833845976, + "flos": 42477287915520.0, + "grad_norm": 3.1089094643822484, + "language_loss": 0.85500699, + "learning_rate": 3.7717646488581415e-06, + "loss": 0.87621236, + "num_input_tokens_seen": 176389365, + "router_z_loss_clip": 0.58154297, + "router_z_loss_mlp": 0.19909668, + "step": 6167, + "time_per_iteration": 2.5432965755462646 + }, + { + "auxiliary_loss_clip": 0.0107339, + "auxiliary_loss_mlp": 0.01037241, + "balance_loss_clip": 1.02551508, + "balance_loss_mlp": 1.02164245, + "epoch": 0.1789797458069758, + "flos": 32409558506880.0, + "grad_norm": 1.7187081360019336, + "language_loss": 0.77847254, + "learning_rate": 3.7716774436954706e-06, + "loss": 0.79957891, + "num_input_tokens_seen": 176408085, + "router_z_loss_clip": 0.47900391, + "router_z_loss_mlp": 0.15582275, + "step": 6168, + "time_per_iteration": 2.4806010723114014 + }, + { + "auxiliary_loss_clip": 0.01074861, + "auxiliary_loss_mlp": 0.01027579, + "balance_loss_clip": 1.02782536, + "balance_loss_mlp": 1.01355958, + "epoch": 0.17900876327549184, + "flos": 61484702837760.0, + "grad_norm": 1.8605540736000787, + "language_loss": 0.65652692, + "learning_rate": 3.7715902228846276e-06, + "loss": 0.67755139, + "num_input_tokens_seen": 176428395, + "router_z_loss_clip": 0.47021484, + "router_z_loss_mlp": 0.14001465, + "step": 6169, + "time_per_iteration": 2.64003586769104 + }, + { + "auxiliary_loss_clip": 0.01076221, + "auxiliary_loss_mlp": 0.01031039, + "balance_loss_clip": 1.02577662, + "balance_loss_mlp": 1.01589382, + "epoch": 0.1790377807440079, + "flos": 41273936484480.0, + "grad_norm": 2.5906014577871814, + "language_loss": 0.89725703, + "learning_rate": 3.7715029864263827e-06, + "loss": 0.91832966, + "num_input_tokens_seen": 176446355, + "router_z_loss_clip": 0.50463867, + "router_z_loss_mlp": 0.15130615, + "step": 6170, + "time_per_iteration": 2.539868116378784 + }, + { + "auxiliary_loss_clip": 0.0107085, + "auxiliary_loss_mlp": 0.01031863, + "balance_loss_clip": 1.02523303, + "balance_loss_mlp": 1.01690245, + "epoch": 0.17906679821252394, + "flos": 26002122704640.0, + "grad_norm": 2.69665648412085, + "language_loss": 0.7505672, + "learning_rate": 3.7714157343215067e-06, + "loss": 0.77159429, + "num_input_tokens_seen": 176466480, + "router_z_loss_clip": 0.45654297, + "router_z_loss_mlp": 0.14978027, + "step": 6171, + "time_per_iteration": 2.5330114364624023 + }, + { + "auxiliary_loss_clip": 0.01081412, + "auxiliary_loss_mlp": 0.01041161, + "balance_loss_clip": 1.02607417, + "balance_loss_mlp": 1.02258778, + "epoch": 0.17909581568104, + "flos": 19201496503680.0, + "grad_norm": 2.5659882255178865, + "language_loss": 0.94077832, + "learning_rate": 3.7713284665707697e-06, + "loss": 0.96200401, + "num_input_tokens_seen": 176481065, + "router_z_loss_clip": 0.5534668, + "router_z_loss_mlp": 0.18566895, + "step": 6172, + "time_per_iteration": 4.855993747711182 + }, + { + "auxiliary_loss_clip": 0.01080768, + "auxiliary_loss_mlp": 0.01032132, + "balance_loss_clip": 1.02915752, + "balance_loss_mlp": 1.01729083, + "epoch": 0.17912483314955605, + "flos": 19492811821440.0, + "grad_norm": 2.2908620816826475, + "language_loss": 1.07652926, + "learning_rate": 3.771241183174943e-06, + "loss": 1.09765816, + "num_input_tokens_seen": 176493880, + "router_z_loss_clip": 0.51635742, + "router_z_loss_mlp": 0.1484375, + "step": 6173, + "time_per_iteration": 2.4272854328155518 + }, + { + "auxiliary_loss_clip": 0.01011294, + "auxiliary_loss_mlp": 0.01001357, + "balance_loss_clip": 1.00394893, + "balance_loss_mlp": 1.00052822, + "epoch": 0.17915385061807207, + "flos": 66350650158720.0, + "grad_norm": 0.6150236649286782, + "language_loss": 0.5196867, + "learning_rate": 3.7711538841347985e-06, + "loss": 0.53981316, + "num_input_tokens_seen": 176561780, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00830078, + "step": 6174, + "time_per_iteration": 3.0941097736358643 + }, + { + "auxiliary_loss_clip": 0.01078857, + "auxiliary_loss_mlp": 0.01035745, + "balance_loss_clip": 1.02601838, + "balance_loss_mlp": 1.01826894, + "epoch": 0.17918286808658812, + "flos": 35983722055680.0, + "grad_norm": 2.1117637505044304, + "language_loss": 0.68652719, + "learning_rate": 3.771066569451105e-06, + "loss": 0.70767319, + "num_input_tokens_seen": 176578135, + "router_z_loss_clip": 0.52758789, + "router_z_loss_mlp": 0.17486572, + "step": 6175, + "time_per_iteration": 2.532449245452881 + }, + { + "auxiliary_loss_clip": 0.0107786, + "auxiliary_loss_mlp": 0.01036379, + "balance_loss_clip": 1.02618623, + "balance_loss_mlp": 1.02004099, + "epoch": 0.17921188555510417, + "flos": 35435201685120.0, + "grad_norm": 2.7704789869249953, + "language_loss": 0.6550948, + "learning_rate": 3.7709792391246356e-06, + "loss": 0.67623723, + "num_input_tokens_seen": 176594965, + "router_z_loss_clip": 0.5168457, + "router_z_loss_mlp": 0.16345215, + "step": 6176, + "time_per_iteration": 2.404924154281616 + }, + { + "auxiliary_loss_clip": 0.0107892, + "auxiliary_loss_mlp": 0.01039209, + "balance_loss_clip": 1.02527928, + "balance_loss_mlp": 1.02235329, + "epoch": 0.17924090302362022, + "flos": 15735843060480.0, + "grad_norm": 2.5771049506070742, + "language_loss": 0.74089658, + "learning_rate": 3.7708918931561606e-06, + "loss": 0.76207787, + "num_input_tokens_seen": 176608305, + "router_z_loss_clip": 0.53613281, + "router_z_loss_mlp": 0.1685791, + "step": 6177, + "time_per_iteration": 2.3466649055480957 + }, + { + "auxiliary_loss_clip": 0.01082594, + "auxiliary_loss_mlp": 0.01041294, + "balance_loss_clip": 1.02765334, + "balance_loss_mlp": 1.02443206, + "epoch": 0.17926992049213628, + "flos": 16500301390080.0, + "grad_norm": 3.326597283919937, + "language_loss": 0.6866613, + "learning_rate": 3.770804531546452e-06, + "loss": 0.70790017, + "num_input_tokens_seen": 176623185, + "router_z_loss_clip": 0.54858398, + "router_z_loss_mlp": 0.16851807, + "step": 6178, + "time_per_iteration": 2.3607242107391357 + }, + { + "auxiliary_loss_clip": 0.01074766, + "auxiliary_loss_mlp": 0.01028727, + "balance_loss_clip": 1.0242579, + "balance_loss_mlp": 1.01329565, + "epoch": 0.1792989379606523, + "flos": 23761397779200.0, + "grad_norm": 2.648720880919092, + "language_loss": 0.84813166, + "learning_rate": 3.7707171542962806e-06, + "loss": 0.86916655, + "num_input_tokens_seen": 176636225, + "router_z_loss_clip": 0.50512695, + "router_z_loss_mlp": 0.1541748, + "step": 6179, + "time_per_iteration": 2.4071664810180664 + }, + { + "auxiliary_loss_clip": 0.01012383, + "auxiliary_loss_mlp": 0.01004231, + "balance_loss_clip": 1.00484276, + "balance_loss_mlp": 1.00327098, + "epoch": 0.17932795542916835, + "flos": 59520591815040.0, + "grad_norm": 0.645828902872191, + "language_loss": 0.47616041, + "learning_rate": 3.7706297614064193e-06, + "loss": 0.49632657, + "num_input_tokens_seen": 176696010, + "router_z_loss_clip": 0.07519531, + "router_z_loss_mlp": 0.00958252, + "step": 6180, + "time_per_iteration": 3.0475730895996094 + }, + { + "auxiliary_loss_clip": 0.01082383, + "auxiliary_loss_mlp": 0.01033555, + "balance_loss_clip": 1.02718782, + "balance_loss_mlp": 1.01606739, + "epoch": 0.1793569728976844, + "flos": 17632534648320.0, + "grad_norm": 2.415693031316386, + "language_loss": 0.80186677, + "learning_rate": 3.7705423528776397e-06, + "loss": 0.82302612, + "num_input_tokens_seen": 176710660, + "router_z_loss_clip": 0.55224609, + "router_z_loss_mlp": 0.17480469, + "step": 6181, + "time_per_iteration": 2.3787004947662354 + }, + { + "auxiliary_loss_clip": 0.01012337, + "auxiliary_loss_mlp": 0.01002519, + "balance_loss_clip": 1.00468731, + "balance_loss_mlp": 1.0016427, + "epoch": 0.17938599036620045, + "flos": 60182614615680.0, + "grad_norm": 0.6526529210473614, + "language_loss": 0.53183532, + "learning_rate": 3.770454928710713e-06, + "loss": 0.55198395, + "num_input_tokens_seen": 176776175, + "router_z_loss_clip": 0.07666016, + "router_z_loss_mlp": 0.00878906, + "step": 6182, + "time_per_iteration": 2.99828839302063 + }, + { + "auxiliary_loss_clip": 0.01076397, + "auxiliary_loss_mlp": 0.01031874, + "balance_loss_clip": 1.02318168, + "balance_loss_mlp": 1.01608467, + "epoch": 0.1794150078347165, + "flos": 18433721594880.0, + "grad_norm": 2.340811758021144, + "language_loss": 0.80567324, + "learning_rate": 3.7703674889064122e-06, + "loss": 0.826756, + "num_input_tokens_seen": 176788445, + "router_z_loss_clip": 0.53173828, + "router_z_loss_mlp": 0.15783691, + "step": 6183, + "time_per_iteration": 2.414612293243408 + }, + { + "auxiliary_loss_clip": 0.01075093, + "auxiliary_loss_mlp": 0.01031065, + "balance_loss_clip": 1.02709627, + "balance_loss_mlp": 1.0153296, + "epoch": 0.17944402530323256, + "flos": 10333278276480.0, + "grad_norm": 2.4380049369193277, + "language_loss": 0.85170841, + "learning_rate": 3.770280033465509e-06, + "loss": 0.87277001, + "num_input_tokens_seen": 176798475, + "router_z_loss_clip": 0.48022461, + "router_z_loss_mlp": 0.15740967, + "step": 6184, + "time_per_iteration": 2.3101136684417725 + }, + { + "auxiliary_loss_clip": 0.01011858, + "auxiliary_loss_mlp": 0.01001204, + "balance_loss_clip": 1.00440061, + "balance_loss_mlp": 1.00029802, + "epoch": 0.17947304277174858, + "flos": 58354806873600.0, + "grad_norm": 0.6676948752621196, + "language_loss": 0.51747912, + "learning_rate": 3.770192562388777e-06, + "loss": 0.5376097, + "num_input_tokens_seen": 176861175, + "router_z_loss_clip": 0.07470703, + "router_z_loss_mlp": 0.0090332, + "step": 6185, + "time_per_iteration": 2.9778122901916504 + }, + { + "auxiliary_loss_clip": 0.01011221, + "auxiliary_loss_mlp": 0.01002552, + "balance_loss_clip": 1.00378478, + "balance_loss_mlp": 1.00168777, + "epoch": 0.17950206024026463, + "flos": 54008504496000.0, + "grad_norm": 0.6517648122449249, + "language_loss": 0.47362643, + "learning_rate": 3.7701050756769873e-06, + "loss": 0.49376419, + "num_input_tokens_seen": 176920120, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00866699, + "step": 6186, + "time_per_iteration": 2.9229562282562256 + }, + { + "auxiliary_loss_clip": 0.01010647, + "auxiliary_loss_mlp": 0.01001696, + "balance_loss_clip": 1.00294876, + "balance_loss_mlp": 1.00078428, + "epoch": 0.17953107770878068, + "flos": 68897075748480.0, + "grad_norm": 0.6596609291449096, + "language_loss": 0.4496842, + "learning_rate": 3.7700175733309133e-06, + "loss": 0.46980762, + "num_input_tokens_seen": 176983185, + "router_z_loss_clip": 0.07714844, + "router_z_loss_mlp": 0.00909424, + "step": 6187, + "time_per_iteration": 3.016838312149048 + }, + { + "auxiliary_loss_clip": 0.01074241, + "auxiliary_loss_mlp": 0.01036837, + "balance_loss_clip": 1.02455044, + "balance_loss_mlp": 1.02077365, + "epoch": 0.17956009517729674, + "flos": 16609544634240.0, + "grad_norm": 2.3793362593007865, + "language_loss": 0.91192055, + "learning_rate": 3.7699300553513276e-06, + "loss": 0.93303132, + "num_input_tokens_seen": 176999020, + "router_z_loss_clip": 0.49658203, + "router_z_loss_mlp": 0.16070557, + "step": 6188, + "time_per_iteration": 2.3628342151641846 + }, + { + "auxiliary_loss_clip": 0.01010544, + "auxiliary_loss_mlp": 0.01000758, + "balance_loss_clip": 1.00329375, + "balance_loss_mlp": 0.99987561, + "epoch": 0.1795891126458128, + "flos": 74786992467840.0, + "grad_norm": 0.6170427603995946, + "language_loss": 0.49089199, + "learning_rate": 3.7698425217390044e-06, + "loss": 0.51100504, + "num_input_tokens_seen": 177075540, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.0088501, + "step": 6189, + "time_per_iteration": 3.21339750289917 + }, + { + "auxiliary_loss_clip": 0.01010412, + "auxiliary_loss_mlp": 0.01002968, + "balance_loss_clip": 1.00309455, + "balance_loss_mlp": 1.00212145, + "epoch": 0.17961813011432884, + "flos": 74772294785280.0, + "grad_norm": 0.60873382189471, + "language_loss": 0.47352448, + "learning_rate": 3.769754972494715e-06, + "loss": 0.49365824, + "num_input_tokens_seen": 177142345, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00848389, + "step": 6190, + "time_per_iteration": 3.157207727432251 + }, + { + "auxiliary_loss_clip": 0.01009602, + "auxiliary_loss_mlp": 0.01001952, + "balance_loss_clip": 1.00243592, + "balance_loss_mlp": 1.00111794, + "epoch": 0.17964714758284486, + "flos": 74773446860160.0, + "grad_norm": 0.6769194542280268, + "language_loss": 0.49658957, + "learning_rate": 3.7696674076192337e-06, + "loss": 0.5167051, + "num_input_tokens_seen": 177205015, + "router_z_loss_clip": 0.07177734, + "router_z_loss_mlp": 0.00836182, + "step": 6191, + "time_per_iteration": 3.0582876205444336 + }, + { + "auxiliary_loss_clip": 0.01083816, + "auxiliary_loss_mlp": 0.01037808, + "balance_loss_clip": 1.02666402, + "balance_loss_mlp": 1.01899695, + "epoch": 0.17967616505136091, + "flos": 36675700669440.0, + "grad_norm": 2.52574423667373, + "language_loss": 0.99074137, + "learning_rate": 3.7695798271133343e-06, + "loss": 1.01195765, + "num_input_tokens_seen": 177222030, + "router_z_loss_clip": 0.5715332, + "router_z_loss_mlp": 0.18798828, + "step": 6192, + "time_per_iteration": 2.5493946075439453 + }, + { + "auxiliary_loss_clip": 0.01088164, + "auxiliary_loss_mlp": 0.01035678, + "balance_loss_clip": 1.02953434, + "balance_loss_mlp": 1.01848245, + "epoch": 0.17970518251987697, + "flos": 18286737304320.0, + "grad_norm": 2.2991223305359867, + "language_loss": 0.94112515, + "learning_rate": 3.769492230977789e-06, + "loss": 0.9623636, + "num_input_tokens_seen": 177236170, + "router_z_loss_clip": 0.58618164, + "router_z_loss_mlp": 0.17199707, + "step": 6193, + "time_per_iteration": 2.34883189201355 + }, + { + "auxiliary_loss_clip": 0.01082067, + "auxiliary_loss_mlp": 0.01033081, + "balance_loss_clip": 1.02516317, + "balance_loss_mlp": 1.01540852, + "epoch": 0.17973419998839302, + "flos": 52727194131840.0, + "grad_norm": 3.1364882152021076, + "language_loss": 0.92230475, + "learning_rate": 3.7694046192133725e-06, + "loss": 0.94345617, + "num_input_tokens_seen": 177253105, + "router_z_loss_clip": 0.56933594, + "router_z_loss_mlp": 0.17681885, + "step": 6194, + "time_per_iteration": 2.7307627201080322 + }, + { + "auxiliary_loss_clip": 0.01081874, + "auxiliary_loss_mlp": 0.01035837, + "balance_loss_clip": 1.02479708, + "balance_loss_mlp": 1.01770508, + "epoch": 0.17976321745690907, + "flos": 74730331330560.0, + "grad_norm": 3.0515486313112152, + "language_loss": 0.8760252, + "learning_rate": 3.7693169918208588e-06, + "loss": 0.89720231, + "num_input_tokens_seen": 177277155, + "router_z_loss_clip": 0.57080078, + "router_z_loss_mlp": 0.18121338, + "step": 6195, + "time_per_iteration": 2.7610228061676025 + }, + { + "auxiliary_loss_clip": 0.01081976, + "auxiliary_loss_mlp": 0.01039748, + "balance_loss_clip": 1.02573264, + "balance_loss_mlp": 1.02080607, + "epoch": 0.1797922349254251, + "flos": 59005555171200.0, + "grad_norm": 8.029453779797528, + "language_loss": 0.81739491, + "learning_rate": 3.769229348801021e-06, + "loss": 0.8386122, + "num_input_tokens_seen": 177297170, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.18939209, + "step": 6196, + "time_per_iteration": 2.732349157333374 + }, + { + "auxiliary_loss_clip": 0.01090574, + "auxiliary_loss_mlp": 0.01046039, + "balance_loss_clip": 1.02702045, + "balance_loss_mlp": 1.02459383, + "epoch": 0.17982125239394114, + "flos": 16502430983040.0, + "grad_norm": 2.737924954650966, + "language_loss": 0.95979494, + "learning_rate": 3.769141690154634e-06, + "loss": 0.981161, + "num_input_tokens_seen": 177312830, + "router_z_loss_clip": 0.63525391, + "router_z_loss_mlp": 0.21447754, + "step": 6197, + "time_per_iteration": 2.359790086746216 + }, + { + "auxiliary_loss_clip": 0.01079192, + "auxiliary_loss_mlp": 0.01036685, + "balance_loss_clip": 1.02562213, + "balance_loss_mlp": 1.02001345, + "epoch": 0.1798502698624572, + "flos": 16537518766080.0, + "grad_norm": 2.2695082190573257, + "language_loss": 0.91717106, + "learning_rate": 3.7690540158824717e-06, + "loss": 0.93832982, + "num_input_tokens_seen": 177325760, + "router_z_loss_clip": 0.53637695, + "router_z_loss_mlp": 0.16674805, + "step": 6198, + "time_per_iteration": 2.409038782119751 + }, + { + "auxiliary_loss_clip": 0.01080906, + "auxiliary_loss_mlp": 0.01034445, + "balance_loss_clip": 1.0261147, + "balance_loss_mlp": 1.01780295, + "epoch": 0.17987928733097325, + "flos": 21536592433920.0, + "grad_norm": 2.3206676404884163, + "language_loss": 0.88731956, + "learning_rate": 3.768966325985308e-06, + "loss": 0.90847313, + "num_input_tokens_seen": 177340490, + "router_z_loss_clip": 0.5480957, + "router_z_loss_mlp": 0.16644287, + "step": 6199, + "time_per_iteration": 2.36950421333313 + }, + { + "auxiliary_loss_clip": 0.01082068, + "auxiliary_loss_mlp": 0.0103556, + "balance_loss_clip": 1.02735782, + "balance_loss_mlp": 1.01862693, + "epoch": 0.1799083047994893, + "flos": 16683525538560.0, + "grad_norm": 2.474966166481425, + "language_loss": 0.77389807, + "learning_rate": 3.7688786204639182e-06, + "loss": 0.79507428, + "num_input_tokens_seen": 177354790, + "router_z_loss_clip": 0.546875, + "router_z_loss_mlp": 0.16937256, + "step": 6200, + "time_per_iteration": 2.378710985183716 + }, + { + "auxiliary_loss_clip": 0.01080138, + "auxiliary_loss_mlp": 0.01035583, + "balance_loss_clip": 1.02682567, + "balance_loss_mlp": 1.01916802, + "epoch": 0.17993732226800535, + "flos": 27263395814400.0, + "grad_norm": 2.062318201774845, + "language_loss": 0.79423809, + "learning_rate": 3.768790899319077e-06, + "loss": 0.81539536, + "num_input_tokens_seen": 177373085, + "router_z_loss_clip": 0.53320312, + "router_z_loss_mlp": 0.16418457, + "step": 6201, + "time_per_iteration": 2.4325857162475586 + }, + { + "auxiliary_loss_clip": 0.01082971, + "auxiliary_loss_mlp": 0.01039756, + "balance_loss_clip": 1.02609563, + "balance_loss_mlp": 1.02121878, + "epoch": 0.17996633973652137, + "flos": 11320377546240.0, + "grad_norm": 2.5153787086288713, + "language_loss": 0.80845839, + "learning_rate": 3.768703162551558e-06, + "loss": 0.82968569, + "num_input_tokens_seen": 177384055, + "router_z_loss_clip": 0.56835938, + "router_z_loss_mlp": 0.18530273, + "step": 6202, + "time_per_iteration": 2.4071898460388184 + }, + { + "auxiliary_loss_clip": 0.01013145, + "auxiliary_loss_mlp": 0.01001128, + "balance_loss_clip": 1.00576317, + "balance_loss_mlp": 1.00017416, + "epoch": 0.17999535720503743, + "flos": 60902175070080.0, + "grad_norm": 0.7129240446812514, + "language_loss": 0.50143689, + "learning_rate": 3.7686154101621374e-06, + "loss": 0.52157962, + "num_input_tokens_seen": 177444265, + "router_z_loss_clip": 0.07373047, + "router_z_loss_mlp": 0.00952148, + "step": 6203, + "time_per_iteration": 2.976806402206421 + }, + { + "auxiliary_loss_clip": 0.01077698, + "auxiliary_loss_mlp": 0.01034457, + "balance_loss_clip": 1.02478933, + "balance_loss_mlp": 1.01613474, + "epoch": 0.18002437467355348, + "flos": 38792938515840.0, + "grad_norm": 2.5883570621787326, + "language_loss": 0.94411486, + "learning_rate": 3.76852764215159e-06, + "loss": 0.96523643, + "num_input_tokens_seen": 177460245, + "router_z_loss_clip": 0.5300293, + "router_z_loss_mlp": 0.1831665, + "step": 6204, + "time_per_iteration": 2.5592689514160156 + }, + { + "auxiliary_loss_clip": 0.0107902, + "auxiliary_loss_mlp": 0.01037261, + "balance_loss_clip": 1.025491, + "balance_loss_mlp": 1.01908207, + "epoch": 0.18005339214206953, + "flos": 11683928200320.0, + "grad_norm": 2.1275971910784146, + "language_loss": 0.88934612, + "learning_rate": 3.768439858520691e-06, + "loss": 0.91050899, + "num_input_tokens_seen": 177471920, + "router_z_loss_clip": 0.53515625, + "router_z_loss_mlp": 0.18164062, + "step": 6205, + "time_per_iteration": 2.3408970832824707 + }, + { + "auxiliary_loss_clip": 0.01082245, + "auxiliary_loss_mlp": 0.01041783, + "balance_loss_clip": 1.02618802, + "balance_loss_mlp": 1.02263212, + "epoch": 0.18008240961058558, + "flos": 26972883457920.0, + "grad_norm": 9.523924911631404, + "language_loss": 0.88731551, + "learning_rate": 3.768352059270215e-06, + "loss": 0.90855581, + "num_input_tokens_seen": 177487840, + "router_z_loss_clip": 0.5604248, + "router_z_loss_mlp": 0.19143677, + "step": 6206, + "time_per_iteration": 2.4753668308258057 + }, + { + "auxiliary_loss_clip": 0.01082867, + "auxiliary_loss_mlp": 0.01035996, + "balance_loss_clip": 1.02872205, + "balance_loss_mlp": 1.01804876, + "epoch": 0.18011142707910163, + "flos": 41126952193920.0, + "grad_norm": 2.3249515919211516, + "language_loss": 0.96597314, + "learning_rate": 3.7682642444009383e-06, + "loss": 0.98716164, + "num_input_tokens_seen": 177504210, + "router_z_loss_clip": 0.54174805, + "router_z_loss_mlp": 0.17932129, + "step": 6207, + "time_per_iteration": 2.5425477027893066 + }, + { + "auxiliary_loss_clip": 0.01011237, + "auxiliary_loss_mlp": 0.01003476, + "balance_loss_clip": 1.00422692, + "balance_loss_mlp": 1.00257635, + "epoch": 0.18014044454761766, + "flos": 63171389531520.0, + "grad_norm": 0.8022945631113811, + "language_loss": 0.48364705, + "learning_rate": 3.768176413913636e-06, + "loss": 0.50379419, + "num_input_tokens_seen": 177565315, + "router_z_loss_clip": 0.0703125, + "router_z_loss_mlp": 0.00897217, + "step": 6208, + "time_per_iteration": 2.9515364170074463 + }, + { + "auxiliary_loss_clip": 0.01085407, + "auxiliary_loss_mlp": 0.01039947, + "balance_loss_clip": 1.02880001, + "balance_loss_mlp": 1.02051568, + "epoch": 0.1801694620161337, + "flos": 31894973844480.0, + "grad_norm": 2.834765260172189, + "language_loss": 0.94087189, + "learning_rate": 3.7680885678090847e-06, + "loss": 0.96212542, + "num_input_tokens_seen": 177578495, + "router_z_loss_clip": 0.56591797, + "router_z_loss_mlp": 0.19433594, + "step": 6209, + "time_per_iteration": 2.486485481262207 + }, + { + "auxiliary_loss_clip": 0.01009734, + "auxiliary_loss_mlp": 0.01002413, + "balance_loss_clip": 1.00279212, + "balance_loss_mlp": 1.00138736, + "epoch": 0.18019847948464976, + "flos": 74767756308480.0, + "grad_norm": 0.6263204074872792, + "language_loss": 0.45212317, + "learning_rate": 3.768000706088059e-06, + "loss": 0.47224462, + "num_input_tokens_seen": 177646060, + "router_z_loss_clip": 0.06933594, + "router_z_loss_mlp": 0.01025391, + "step": 6210, + "time_per_iteration": 3.1087517738342285 + }, + { + "auxiliary_loss_clip": 0.0100851, + "auxiliary_loss_mlp": 0.01001797, + "balance_loss_clip": 1.00164044, + "balance_loss_mlp": 1.00095701, + "epoch": 0.1802274969531658, + "flos": 65869687002240.0, + "grad_norm": 0.6620709109087326, + "language_loss": 0.47705746, + "learning_rate": 3.767912828751336e-06, + "loss": 0.49716052, + "num_input_tokens_seen": 177706775, + "router_z_loss_clip": 0.06884766, + "router_z_loss_mlp": 0.00842285, + "step": 6211, + "time_per_iteration": 3.0997800827026367 + }, + { + "auxiliary_loss_clip": 0.01080066, + "auxiliary_loss_mlp": 0.01033718, + "balance_loss_clip": 1.02474988, + "balance_loss_mlp": 1.01551461, + "epoch": 0.18025651442168186, + "flos": 16536925272960.0, + "grad_norm": 3.0448060473227243, + "language_loss": 0.73290944, + "learning_rate": 3.7678249357996915e-06, + "loss": 0.75404727, + "num_input_tokens_seen": 177717830, + "router_z_loss_clip": 0.55297852, + "router_z_loss_mlp": 0.18212891, + "step": 6212, + "time_per_iteration": 2.324336051940918 + }, + { + "auxiliary_loss_clip": 0.01082823, + "auxiliary_loss_mlp": 0.01044589, + "balance_loss_clip": 1.02441943, + "balance_loss_mlp": 1.02569449, + "epoch": 0.1802855318901979, + "flos": 18470101098240.0, + "grad_norm": 2.8080929939764454, + "language_loss": 0.82018858, + "learning_rate": 3.7677370272339015e-06, + "loss": 0.84146273, + "num_input_tokens_seen": 177730415, + "router_z_loss_clip": 0.58398438, + "router_z_loss_mlp": 0.18896484, + "step": 6213, + "time_per_iteration": 2.34366512298584 + }, + { + "auxiliary_loss_clip": 0.0107983, + "auxiliary_loss_mlp": 0.01037179, + "balance_loss_clip": 1.02411103, + "balance_loss_mlp": 1.01975703, + "epoch": 0.18031454935871394, + "flos": 23724738984960.0, + "grad_norm": 3.6591392604082276, + "language_loss": 0.82204735, + "learning_rate": 3.767649103054743e-06, + "loss": 0.84321749, + "num_input_tokens_seen": 177743490, + "router_z_loss_clip": 0.55688477, + "router_z_loss_mlp": 0.17431641, + "step": 6214, + "time_per_iteration": 2.3992340564727783 + }, + { + "auxiliary_loss_clip": 0.01008511, + "auxiliary_loss_mlp": 0.01002973, + "balance_loss_clip": 1.00169027, + "balance_loss_mlp": 1.00212085, + "epoch": 0.18034356682723, + "flos": 65064904185600.0, + "grad_norm": 0.6570019845542465, + "language_loss": 0.46622568, + "learning_rate": 3.7675611632629923e-06, + "loss": 0.48634052, + "num_input_tokens_seen": 177803610, + "router_z_loss_clip": 0.06835938, + "router_z_loss_mlp": 0.00854492, + "step": 6215, + "time_per_iteration": 2.9454476833343506 + }, + { + "auxiliary_loss_clip": 0.01082053, + "auxiliary_loss_mlp": 0.01041071, + "balance_loss_clip": 1.02608395, + "balance_loss_mlp": 1.02278435, + "epoch": 0.18037258429574604, + "flos": 28175850864000.0, + "grad_norm": 2.2293123387447644, + "language_loss": 0.89636302, + "learning_rate": 3.767473207859426e-06, + "loss": 0.91759425, + "num_input_tokens_seen": 177820825, + "router_z_loss_clip": 0.56005859, + "router_z_loss_mlp": 0.18292236, + "step": 6216, + "time_per_iteration": 2.4479317665100098 + }, + { + "auxiliary_loss_clip": 0.01085247, + "auxiliary_loss_mlp": 0.01042152, + "balance_loss_clip": 1.0284915, + "balance_loss_mlp": 1.02446091, + "epoch": 0.1804016017642621, + "flos": 19868860742400.0, + "grad_norm": 2.156871488842802, + "language_loss": 0.88112509, + "learning_rate": 3.7673852368448217e-06, + "loss": 0.90239912, + "num_input_tokens_seen": 177837370, + "router_z_loss_clip": 0.56738281, + "router_z_loss_mlp": 0.17687988, + "step": 6217, + "time_per_iteration": 2.4280033111572266 + }, + { + "auxiliary_loss_clip": 0.01088397, + "auxiliary_loss_mlp": 0.01048459, + "balance_loss_clip": 1.02694416, + "balance_loss_mlp": 1.02729964, + "epoch": 0.18043061923277814, + "flos": 24963876426240.0, + "grad_norm": 3.283478209205767, + "language_loss": 0.93848825, + "learning_rate": 3.767297250219955e-06, + "loss": 0.95985681, + "num_input_tokens_seen": 177851580, + "router_z_loss_clip": 0.61523438, + "router_z_loss_mlp": 0.21154785, + "step": 6218, + "time_per_iteration": 2.4238390922546387 + }, + { + "auxiliary_loss_clip": 0.01009731, + "auxiliary_loss_mlp": 0.01004022, + "balance_loss_clip": 1.00272608, + "balance_loss_mlp": 1.0031755, + "epoch": 0.18045963670129417, + "flos": 59456770116480.0, + "grad_norm": 0.7038617295287177, + "language_loss": 0.49798471, + "learning_rate": 3.7672092479856045e-06, + "loss": 0.51812226, + "num_input_tokens_seen": 177907410, + "router_z_loss_clip": 0.0703125, + "router_z_loss_mlp": 0.00848389, + "step": 6219, + "time_per_iteration": 3.0131258964538574 + }, + { + "auxiliary_loss_clip": 0.01075904, + "auxiliary_loss_mlp": 0.01030532, + "balance_loss_clip": 1.02558017, + "balance_loss_mlp": 1.0153091, + "epoch": 0.18048865416981022, + "flos": 32483364854400.0, + "grad_norm": 2.5488041394674243, + "language_loss": 0.66765004, + "learning_rate": 3.767121230142546e-06, + "loss": 0.68871439, + "num_input_tokens_seen": 177924770, + "router_z_loss_clip": 0.50341797, + "router_z_loss_mlp": 0.15222168, + "step": 6220, + "time_per_iteration": 2.4661433696746826 + }, + { + "auxiliary_loss_clip": 0.01074054, + "auxiliary_loss_mlp": 0.01028508, + "balance_loss_clip": 1.02569366, + "balance_loss_mlp": 1.01426268, + "epoch": 0.18051767163832627, + "flos": 19456781431680.0, + "grad_norm": 3.3162077444904345, + "language_loss": 0.78100848, + "learning_rate": 3.7670331966915586e-06, + "loss": 0.80203414, + "num_input_tokens_seen": 177937550, + "router_z_loss_clip": 0.48388672, + "router_z_loss_mlp": 0.14257812, + "step": 6221, + "time_per_iteration": 2.430654764175415 + }, + { + "auxiliary_loss_clip": 0.01084607, + "auxiliary_loss_mlp": 0.01040793, + "balance_loss_clip": 1.02744508, + "balance_loss_mlp": 1.022053, + "epoch": 0.18054668910684232, + "flos": 13726696383360.0, + "grad_norm": 2.402873024430685, + "language_loss": 0.85821891, + "learning_rate": 3.7669451476334187e-06, + "loss": 0.87947297, + "num_input_tokens_seen": 177949185, + "router_z_loss_clip": 0.57177734, + "router_z_loss_mlp": 0.18725586, + "step": 6222, + "time_per_iteration": 2.3323488235473633 + }, + { + "auxiliary_loss_clip": 0.01088691, + "auxiliary_loss_mlp": 0.01041663, + "balance_loss_clip": 1.02833652, + "balance_loss_mlp": 1.02214789, + "epoch": 0.18057570657535837, + "flos": 29635360007040.0, + "grad_norm": 2.3211818898504095, + "language_loss": 0.99372935, + "learning_rate": 3.7668570829689043e-06, + "loss": 1.01503277, + "num_input_tokens_seen": 177965635, + "router_z_loss_clip": 0.60327148, + "router_z_loss_mlp": 0.19537354, + "step": 6223, + "time_per_iteration": 2.58270001411438 + }, + { + "auxiliary_loss_clip": 0.01076935, + "auxiliary_loss_mlp": 0.01029771, + "balance_loss_clip": 1.02661586, + "balance_loss_mlp": 1.01396942, + "epoch": 0.18060472404387443, + "flos": 15152060350080.0, + "grad_norm": 2.0829208480258385, + "language_loss": 0.70361221, + "learning_rate": 3.766769002698793e-06, + "loss": 0.72467923, + "num_input_tokens_seen": 177981025, + "router_z_loss_clip": 0.50317383, + "router_z_loss_mlp": 0.15795898, + "step": 6224, + "time_per_iteration": 2.394455909729004 + }, + { + "auxiliary_loss_clip": 0.01069837, + "auxiliary_loss_mlp": 0.01032452, + "balance_loss_clip": 1.02385581, + "balance_loss_mlp": 1.01827836, + "epoch": 0.18063374151239045, + "flos": 34832669708160.0, + "grad_norm": 2.0284277111380598, + "language_loss": 0.62956941, + "learning_rate": 3.766680906823863e-06, + "loss": 0.65059233, + "num_input_tokens_seen": 177997365, + "router_z_loss_clip": 0.46020508, + "router_z_loss_mlp": 0.14172363, + "step": 6225, + "time_per_iteration": 2.5533618927001953 + }, + { + "auxiliary_loss_clip": 0.01010338, + "auxiliary_loss_mlp": 0.01010296, + "balance_loss_clip": 1.00325203, + "balance_loss_mlp": 1.00931847, + "epoch": 0.1806627589809065, + "flos": 58725863470080.0, + "grad_norm": 0.7045415783084891, + "language_loss": 0.51204312, + "learning_rate": 3.766592795344892e-06, + "loss": 0.53224951, + "num_input_tokens_seen": 178057810, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00976562, + "step": 6226, + "time_per_iteration": 2.97404146194458 + }, + { + "auxiliary_loss_clip": 0.01078999, + "auxiliary_loss_mlp": 0.01029143, + "balance_loss_clip": 1.02629995, + "balance_loss_mlp": 1.0117619, + "epoch": 0.18069177644942255, + "flos": 28210589533440.0, + "grad_norm": 1.7032392238744696, + "language_loss": 0.99519324, + "learning_rate": 3.766504668262659e-06, + "loss": 1.01627469, + "num_input_tokens_seen": 178077585, + "router_z_loss_clip": 0.52758789, + "router_z_loss_mlp": 0.17388916, + "step": 6227, + "time_per_iteration": 2.548611879348755 + }, + { + "auxiliary_loss_clip": 0.01074431, + "auxiliary_loss_mlp": 0.01031708, + "balance_loss_clip": 1.02546227, + "balance_loss_mlp": 1.01588249, + "epoch": 0.1807207939179386, + "flos": 38098760486400.0, + "grad_norm": 2.2514669347166363, + "language_loss": 0.75896406, + "learning_rate": 3.7664165255779413e-06, + "loss": 0.78002542, + "num_input_tokens_seen": 178093170, + "router_z_loss_clip": 0.48999023, + "router_z_loss_mlp": 0.15826416, + "step": 6228, + "time_per_iteration": 2.418421506881714 + }, + { + "auxiliary_loss_clip": 0.0101053, + "auxiliary_loss_mlp": 0.01008051, + "balance_loss_clip": 1.00348258, + "balance_loss_mlp": 1.00723481, + "epoch": 0.18074981138645466, + "flos": 60790313473920.0, + "grad_norm": 0.7191958157435485, + "language_loss": 0.52621293, + "learning_rate": 3.766328367291519e-06, + "loss": 0.54639876, + "num_input_tokens_seen": 178151655, + "router_z_loss_clip": 0.0703125, + "router_z_loss_mlp": 0.00817871, + "step": 6229, + "time_per_iteration": 2.947016716003418 + }, + { + "auxiliary_loss_clip": 0.01010064, + "auxiliary_loss_mlp": 0.01003025, + "balance_loss_clip": 1.00315309, + "balance_loss_mlp": 1.00217891, + "epoch": 0.18077882885497068, + "flos": 64513521083520.0, + "grad_norm": 0.7239668605694579, + "language_loss": 0.46166888, + "learning_rate": 3.766240193404169e-06, + "loss": 0.48179975, + "num_input_tokens_seen": 178203670, + "router_z_loss_clip": 0.06933594, + "router_z_loss_mlp": 0.00848389, + "step": 6230, + "time_per_iteration": 2.870788335800171 + }, + { + "auxiliary_loss_clip": 0.01082944, + "auxiliary_loss_mlp": 0.0104046, + "balance_loss_clip": 1.02673578, + "balance_loss_mlp": 1.02190542, + "epoch": 0.18080784632348673, + "flos": 28472123594880.0, + "grad_norm": 2.357388347961662, + "language_loss": 0.92394078, + "learning_rate": 3.766152003916671e-06, + "loss": 0.94517481, + "num_input_tokens_seen": 178225155, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.18566895, + "step": 6231, + "time_per_iteration": 2.557218074798584 + }, + { + "auxiliary_loss_clip": 0.01079695, + "auxiliary_loss_mlp": 0.01032416, + "balance_loss_clip": 1.02653146, + "balance_loss_mlp": 1.01552367, + "epoch": 0.18083686379200278, + "flos": 29928490715520.0, + "grad_norm": 2.1862297365886882, + "language_loss": 0.90229505, + "learning_rate": 3.7660637988298047e-06, + "loss": 0.9234162, + "num_input_tokens_seen": 178244820, + "router_z_loss_clip": 0.53173828, + "router_z_loss_mlp": 0.16882324, + "step": 6232, + "time_per_iteration": 2.459398031234741 + }, + { + "auxiliary_loss_clip": 0.01081961, + "auxiliary_loss_mlp": 0.01043617, + "balance_loss_clip": 1.02578557, + "balance_loss_mlp": 1.02661765, + "epoch": 0.18086588126051883, + "flos": 21974054169600.0, + "grad_norm": 1.9976236621027241, + "language_loss": 0.7504465, + "learning_rate": 3.7659755781443473e-06, + "loss": 0.77170223, + "num_input_tokens_seen": 178257730, + "router_z_loss_clip": 0.56176758, + "router_z_loss_mlp": 0.16998291, + "step": 6233, + "time_per_iteration": 2.3681845664978027 + }, + { + "auxiliary_loss_clip": 0.01009584, + "auxiliary_loss_mlp": 0.01007177, + "balance_loss_clip": 1.00262117, + "balance_loss_mlp": 1.00632441, + "epoch": 0.1808948987290349, + "flos": 74779172323200.0, + "grad_norm": 0.6382707754036907, + "language_loss": 0.48223132, + "learning_rate": 3.7658873418610797e-06, + "loss": 0.50239891, + "num_input_tokens_seen": 178326940, + "router_z_loss_clip": 0.06933594, + "router_z_loss_mlp": 0.00854492, + "step": 6234, + "time_per_iteration": 3.143378496170044 + }, + { + "auxiliary_loss_clip": 0.01082391, + "auxiliary_loss_mlp": 0.0104171, + "balance_loss_clip": 1.02472293, + "balance_loss_mlp": 1.02267194, + "epoch": 0.18092391619755094, + "flos": 16317181975680.0, + "grad_norm": 21.178400926471646, + "language_loss": 0.84710753, + "learning_rate": 3.76579908998078e-06, + "loss": 0.8683486, + "num_input_tokens_seen": 178339300, + "router_z_loss_clip": 0.57617188, + "router_z_loss_mlp": 0.19055176, + "step": 6235, + "time_per_iteration": 2.3305492401123047 + }, + { + "auxiliary_loss_clip": 0.01009422, + "auxiliary_loss_mlp": 0.0101336, + "balance_loss_clip": 1.00266767, + "balance_loss_mlp": 1.01259136, + "epoch": 0.18095293366606696, + "flos": 57355558450560.0, + "grad_norm": 0.6928776390432053, + "language_loss": 0.45021766, + "learning_rate": 3.7657108225042284e-06, + "loss": 0.47044548, + "num_input_tokens_seen": 178387785, + "router_z_loss_clip": 0.06738281, + "router_z_loss_mlp": 0.00765991, + "step": 6236, + "time_per_iteration": 2.8219099044799805 + }, + { + "auxiliary_loss_clip": 0.01079363, + "auxiliary_loss_mlp": 0.0103939, + "balance_loss_clip": 1.02517438, + "balance_loss_mlp": 1.02121055, + "epoch": 0.180981951134583, + "flos": 39449619878400.0, + "grad_norm": 3.9912200992052793, + "language_loss": 0.73941696, + "learning_rate": 3.765622539432204e-06, + "loss": 0.76060456, + "num_input_tokens_seen": 178403970, + "router_z_loss_clip": 0.54223633, + "router_z_loss_mlp": 0.1817627, + "step": 6237, + "time_per_iteration": 2.5222973823547363 + }, + { + "auxiliary_loss_clip": 0.01009375, + "auxiliary_loss_mlp": 0.01002595, + "balance_loss_clip": 1.00247324, + "balance_loss_mlp": 1.00179076, + "epoch": 0.18101096860309906, + "flos": 60807629508480.0, + "grad_norm": 0.6737138961544759, + "language_loss": 0.50503623, + "learning_rate": 3.7655342407654873e-06, + "loss": 0.5251559, + "num_input_tokens_seen": 178460070, + "router_z_loss_clip": 0.06933594, + "router_z_loss_mlp": 0.00805664, + "step": 6238, + "time_per_iteration": 2.9967541694641113 + }, + { + "auxiliary_loss_clip": 0.01071174, + "auxiliary_loss_mlp": 0.01027069, + "balance_loss_clip": 1.02354789, + "balance_loss_mlp": 1.01282394, + "epoch": 0.18103998607161512, + "flos": 23288115121920.0, + "grad_norm": 1.9042678172096195, + "language_loss": 0.84279585, + "learning_rate": 3.7654459265048574e-06, + "loss": 0.86377823, + "num_input_tokens_seen": 178481805, + "router_z_loss_clip": 0.47607422, + "router_z_loss_mlp": 0.14251709, + "step": 6239, + "time_per_iteration": 6.972848892211914 + }, + { + "auxiliary_loss_clip": 0.01085372, + "auxiliary_loss_mlp": 0.01045135, + "balance_loss_clip": 1.02674544, + "balance_loss_mlp": 1.02568007, + "epoch": 0.18106900354013117, + "flos": 45982253416320.0, + "grad_norm": 2.228659951784526, + "language_loss": 0.94731468, + "learning_rate": 3.7653575966510942e-06, + "loss": 0.96861982, + "num_input_tokens_seen": 178501045, + "router_z_loss_clip": 0.58642578, + "router_z_loss_mlp": 0.19445801, + "step": 6240, + "time_per_iteration": 2.6398935317993164 + }, + { + "auxiliary_loss_clip": 0.01009597, + "auxiliary_loss_mlp": 0.01000832, + "balance_loss_clip": 1.00253832, + "balance_loss_mlp": 1.00001538, + "epoch": 0.1810980210086472, + "flos": 56047606986240.0, + "grad_norm": 0.7420618266158774, + "language_loss": 0.52238625, + "learning_rate": 3.765269251204979e-06, + "loss": 0.54249054, + "num_input_tokens_seen": 178551160, + "router_z_loss_clip": 0.0703125, + "router_z_loss_mlp": 0.00817871, + "step": 6241, + "time_per_iteration": 2.7618720531463623 + }, + { + "auxiliary_loss_clip": 0.01010347, + "auxiliary_loss_mlp": 0.0100166, + "balance_loss_clip": 1.00322175, + "balance_loss_mlp": 1.00081372, + "epoch": 0.18112703847716324, + "flos": 67796474048640.0, + "grad_norm": 0.6677441364009484, + "language_loss": 0.53868967, + "learning_rate": 3.765180890167292e-06, + "loss": 0.55880976, + "num_input_tokens_seen": 178612890, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00848389, + "step": 6242, + "time_per_iteration": 3.015991449356079 + }, + { + "auxiliary_loss_clip": 0.01083269, + "auxiliary_loss_mlp": 0.01035599, + "balance_loss_clip": 1.02689016, + "balance_loss_mlp": 1.01862383, + "epoch": 0.1811560559456793, + "flos": 32230907746560.0, + "grad_norm": 2.07164122504035, + "language_loss": 0.93214655, + "learning_rate": 3.7650925135388125e-06, + "loss": 0.95333523, + "num_input_tokens_seen": 178629255, + "router_z_loss_clip": 0.56396484, + "router_z_loss_mlp": 0.16967773, + "step": 6243, + "time_per_iteration": 2.4754281044006348 + }, + { + "auxiliary_loss_clip": 0.01009995, + "auxiliary_loss_mlp": 0.01003294, + "balance_loss_clip": 1.00299549, + "balance_loss_mlp": 1.00242984, + "epoch": 0.18118507341419535, + "flos": 58193369280000.0, + "grad_norm": 0.6712869803467587, + "language_loss": 0.4875952, + "learning_rate": 3.7650041213203216e-06, + "loss": 0.5077281, + "num_input_tokens_seen": 178683205, + "router_z_loss_clip": 0.0703125, + "router_z_loss_mlp": 0.00866699, + "step": 6244, + "time_per_iteration": 2.939112424850464 + }, + { + "auxiliary_loss_clip": 0.01078144, + "auxiliary_loss_mlp": 0.01033066, + "balance_loss_clip": 1.02524734, + "balance_loss_mlp": 1.01554251, + "epoch": 0.1812140908827114, + "flos": 37450038913920.0, + "grad_norm": 1.7924161034049573, + "language_loss": 0.94967842, + "learning_rate": 3.7649157135126e-06, + "loss": 0.97079051, + "num_input_tokens_seen": 178708350, + "router_z_loss_clip": 0.52880859, + "router_z_loss_mlp": 0.17541504, + "step": 6245, + "time_per_iteration": 2.660475254058838 + }, + { + "auxiliary_loss_clip": 0.01010361, + "auxiliary_loss_mlp": 0.01002499, + "balance_loss_clip": 1.00347352, + "balance_loss_mlp": 1.00172377, + "epoch": 0.18124310835122745, + "flos": 55908058815360.0, + "grad_norm": 0.6617701310708153, + "language_loss": 0.43592501, + "learning_rate": 3.764827290116429e-06, + "loss": 0.45605358, + "num_input_tokens_seen": 178765950, + "router_z_loss_clip": 0.06884766, + "router_z_loss_mlp": 0.00775146, + "step": 6246, + "time_per_iteration": 2.9787862300872803 + }, + { + "auxiliary_loss_clip": 0.01009678, + "auxiliary_loss_mlp": 0.01001248, + "balance_loss_clip": 1.00292277, + "balance_loss_mlp": 1.00037742, + "epoch": 0.18127212581974347, + "flos": 61766343976320.0, + "grad_norm": 0.7053894112398273, + "language_loss": 0.50168467, + "learning_rate": 3.7647388511325888e-06, + "loss": 0.52179396, + "num_input_tokens_seen": 178829750, + "router_z_loss_clip": 0.06738281, + "router_z_loss_mlp": 0.00872803, + "step": 6247, + "time_per_iteration": 3.113276481628418 + }, + { + "auxiliary_loss_clip": 0.01009282, + "auxiliary_loss_mlp": 0.01001625, + "balance_loss_clip": 1.00251436, + "balance_loss_mlp": 1.00077868, + "epoch": 0.18130114328825953, + "flos": 57845284358400.0, + "grad_norm": 0.6777934495510506, + "language_loss": 0.499744, + "learning_rate": 3.764650396561861e-06, + "loss": 0.51985306, + "num_input_tokens_seen": 178890870, + "router_z_loss_clip": 0.06738281, + "router_z_loss_mlp": 0.00848389, + "step": 6248, + "time_per_iteration": 3.049962282180786 + }, + { + "auxiliary_loss_clip": 0.01008968, + "auxiliary_loss_mlp": 0.01001599, + "balance_loss_clip": 1.00212276, + "balance_loss_mlp": 1.00071692, + "epoch": 0.18133016075677558, + "flos": 73640969222400.0, + "grad_norm": 0.6564015423884246, + "language_loss": 0.47528517, + "learning_rate": 3.7645619264050267e-06, + "loss": 0.49539086, + "num_input_tokens_seen": 178952635, + "router_z_loss_clip": 0.06835938, + "router_z_loss_mlp": 0.0088501, + "step": 6249, + "time_per_iteration": 5.520312070846558 + }, + { + "auxiliary_loss_clip": 0.01009565, + "auxiliary_loss_mlp": 0.01001585, + "balance_loss_clip": 1.00294399, + "balance_loss_mlp": 1.00082457, + "epoch": 0.18135917822529163, + "flos": 62549586483840.0, + "grad_norm": 0.66305179108406, + "language_loss": 0.46487141, + "learning_rate": 3.764473440662868e-06, + "loss": 0.48498291, + "num_input_tokens_seen": 179014065, + "router_z_loss_clip": 0.06640625, + "router_z_loss_mlp": 0.00759888, + "step": 6250, + "time_per_iteration": 2.9825730323791504 + }, + { + "auxiliary_loss_clip": 0.010827, + "auxiliary_loss_mlp": 0.01035548, + "balance_loss_clip": 1.02548599, + "balance_loss_mlp": 1.0166893, + "epoch": 0.18138819569380768, + "flos": 54409658417280.0, + "grad_norm": 1.8804070971122058, + "language_loss": 0.69863212, + "learning_rate": 3.7643849393361654e-06, + "loss": 0.7198146, + "num_input_tokens_seen": 179034755, + "router_z_loss_clip": 0.57202148, + "router_z_loss_mlp": 0.18859863, + "step": 6251, + "time_per_iteration": 2.6161015033721924 + }, + { + "auxiliary_loss_clip": 0.01076766, + "auxiliary_loss_mlp": 0.01034976, + "balance_loss_clip": 1.02265644, + "balance_loss_mlp": 1.01518679, + "epoch": 0.18141721316232373, + "flos": 18836654129280.0, + "grad_norm": 2.0704292820291332, + "language_loss": 0.67209226, + "learning_rate": 3.764296422425701e-06, + "loss": 0.69320971, + "num_input_tokens_seen": 179053040, + "router_z_loss_clip": 0.54174805, + "router_z_loss_mlp": 0.19787598, + "step": 6252, + "time_per_iteration": 2.429532766342163 + }, + { + "auxiliary_loss_clip": 0.01084418, + "auxiliary_loss_mlp": 0.01035417, + "balance_loss_clip": 1.02824974, + "balance_loss_mlp": 1.01789331, + "epoch": 0.18144623063083976, + "flos": 12451458729600.0, + "grad_norm": 2.3341360553341133, + "language_loss": 0.70788586, + "learning_rate": 3.7642078899322568e-06, + "loss": 0.72908425, + "num_input_tokens_seen": 179067685, + "router_z_loss_clip": 0.56176758, + "router_z_loss_mlp": 0.17541504, + "step": 6253, + "time_per_iteration": 2.351201057434082 + }, + { + "auxiliary_loss_clip": 0.01009369, + "auxiliary_loss_mlp": 0.01004671, + "balance_loss_clip": 1.00278616, + "balance_loss_mlp": 1.00393188, + "epoch": 0.1814752480993558, + "flos": 74779695993600.0, + "grad_norm": 0.644189336398744, + "language_loss": 0.47205347, + "learning_rate": 3.764119341856615e-06, + "loss": 0.49219388, + "num_input_tokens_seen": 179133970, + "router_z_loss_clip": 0.06591797, + "router_z_loss_mlp": 0.00738525, + "step": 6254, + "time_per_iteration": 3.1243529319763184 + }, + { + "auxiliary_loss_clip": 0.01084644, + "auxiliary_loss_mlp": 0.01038872, + "balance_loss_clip": 1.02514255, + "balance_loss_mlp": 1.01972699, + "epoch": 0.18150426556787186, + "flos": 31349770053120.0, + "grad_norm": 3.8277236174785276, + "language_loss": 0.85689622, + "learning_rate": 3.764030778199557e-06, + "loss": 0.87813139, + "num_input_tokens_seen": 179151270, + "router_z_loss_clip": 0.59521484, + "router_z_loss_mlp": 0.19140625, + "step": 6255, + "time_per_iteration": 2.4685628414154053 + }, + { + "auxiliary_loss_clip": 0.01076712, + "auxiliary_loss_mlp": 0.01029254, + "balance_loss_clip": 1.02698588, + "balance_loss_mlp": 1.01372719, + "epoch": 0.1815332830363879, + "flos": 12814555536000.0, + "grad_norm": 2.890269039079833, + "language_loss": 0.82648271, + "learning_rate": 3.7639421989618653e-06, + "loss": 0.84754241, + "num_input_tokens_seen": 179161915, + "router_z_loss_clip": 0.49731445, + "router_z_loss_mlp": 0.15545654, + "step": 6256, + "time_per_iteration": 2.368527412414551 + }, + { + "auxiliary_loss_clip": 0.01009363, + "auxiliary_loss_mlp": 0.01003204, + "balance_loss_clip": 1.00275743, + "balance_loss_mlp": 1.0023632, + "epoch": 0.18156230050490396, + "flos": 74773272303360.0, + "grad_norm": 0.6238199172228692, + "language_loss": 0.43740976, + "learning_rate": 3.763853604144322e-06, + "loss": 0.45753542, + "num_input_tokens_seen": 179226700, + "router_z_loss_clip": 0.06640625, + "router_z_loss_mlp": 0.00842285, + "step": 6257, + "time_per_iteration": 3.0903890132904053 + }, + { + "auxiliary_loss_clip": 0.01081669, + "auxiliary_loss_mlp": 0.01042957, + "balance_loss_clip": 1.02687263, + "balance_loss_mlp": 1.02413392, + "epoch": 0.18159131797341999, + "flos": 34452187044480.0, + "grad_norm": 2.697039989369662, + "language_loss": 1.00941324, + "learning_rate": 3.76376499374771e-06, + "loss": 1.03065944, + "num_input_tokens_seen": 179245835, + "router_z_loss_clip": 0.54833984, + "router_z_loss_mlp": 0.18823242, + "step": 6258, + "time_per_iteration": 2.494382381439209 + }, + { + "auxiliary_loss_clip": 0.01081556, + "auxiliary_loss_mlp": 0.01031054, + "balance_loss_clip": 1.02565336, + "balance_loss_mlp": 1.01277888, + "epoch": 0.18162033544193604, + "flos": 32882596784640.0, + "grad_norm": 3.5231764770044456, + "language_loss": 0.95896882, + "learning_rate": 3.763676367772812e-06, + "loss": 0.98009491, + "num_input_tokens_seen": 179264610, + "router_z_loss_clip": 0.55859375, + "router_z_loss_mlp": 0.18273926, + "step": 6259, + "time_per_iteration": 2.5182135105133057 + }, + { + "auxiliary_loss_clip": 0.01078146, + "auxiliary_loss_mlp": 0.01034541, + "balance_loss_clip": 1.02516007, + "balance_loss_mlp": 1.01631355, + "epoch": 0.1816493529104521, + "flos": 33725050824960.0, + "grad_norm": 2.221614748398068, + "language_loss": 0.70503759, + "learning_rate": 3.763587726220411e-06, + "loss": 0.72616446, + "num_input_tokens_seen": 179281400, + "router_z_loss_clip": 0.53027344, + "router_z_loss_mlp": 0.18225098, + "step": 6260, + "time_per_iteration": 2.46738338470459 + }, + { + "auxiliary_loss_clip": 0.01009057, + "auxiliary_loss_mlp": 0.01001585, + "balance_loss_clip": 1.00217009, + "balance_loss_mlp": 1.00064874, + "epoch": 0.18167837037896814, + "flos": 71082010454400.0, + "grad_norm": 0.738588793975232, + "language_loss": 0.45410049, + "learning_rate": 3.7634990690912894e-06, + "loss": 0.47420692, + "num_input_tokens_seen": 179336110, + "router_z_loss_clip": 0.06933594, + "router_z_loss_mlp": 0.00933838, + "step": 6261, + "time_per_iteration": 2.9587478637695312 + }, + { + "auxiliary_loss_clip": 0.01086243, + "auxiliary_loss_mlp": 0.01038517, + "balance_loss_clip": 1.02792287, + "balance_loss_mlp": 1.01951516, + "epoch": 0.1817073878474842, + "flos": 33757520256000.0, + "grad_norm": 2.0020566917246883, + "language_loss": 1.07994306, + "learning_rate": 3.7634103963862304e-06, + "loss": 1.10119069, + "num_input_tokens_seen": 179356520, + "router_z_loss_clip": 0.58300781, + "router_z_loss_mlp": 0.19006348, + "step": 6262, + "time_per_iteration": 2.5226399898529053 + }, + { + "auxiliary_loss_clip": 0.01079159, + "auxiliary_loss_mlp": 0.01036462, + "balance_loss_clip": 1.02630305, + "balance_loss_mlp": 1.01936758, + "epoch": 0.18173640531600024, + "flos": 13071690766080.0, + "grad_norm": 2.8340221138752737, + "language_loss": 0.81825918, + "learning_rate": 3.7633217081060168e-06, + "loss": 0.83941531, + "num_input_tokens_seen": 179369545, + "router_z_loss_clip": 0.52783203, + "router_z_loss_mlp": 0.1708374, + "step": 6263, + "time_per_iteration": 2.4739267826080322 + }, + { + "auxiliary_loss_clip": 0.01008767, + "auxiliary_loss_mlp": 0.01000801, + "balance_loss_clip": 1.00215173, + "balance_loss_mlp": 0.9999665, + "epoch": 0.18176542278451627, + "flos": 72730469208960.0, + "grad_norm": 0.6910703671540956, + "language_loss": 0.49836206, + "learning_rate": 3.7632330042514325e-06, + "loss": 0.51845777, + "num_input_tokens_seen": 179430290, + "router_z_loss_clip": 0.06640625, + "router_z_loss_mlp": 0.00836182, + "step": 6264, + "time_per_iteration": 3.0432088375091553 + }, + { + "auxiliary_loss_clip": 0.01082172, + "auxiliary_loss_mlp": 0.01034773, + "balance_loss_clip": 1.02707529, + "balance_loss_mlp": 1.01813722, + "epoch": 0.18179444025303232, + "flos": 39885754982400.0, + "grad_norm": 3.017328275056029, + "language_loss": 1.02285814, + "learning_rate": 3.763144284823261e-06, + "loss": 1.04402745, + "num_input_tokens_seen": 179447695, + "router_z_loss_clip": 0.55126953, + "router_z_loss_mlp": 0.16638184, + "step": 6265, + "time_per_iteration": 2.5778117179870605 + }, + { + "auxiliary_loss_clip": 0.0100824, + "auxiliary_loss_mlp": 0.0100145, + "balance_loss_clip": 1.00158513, + "balance_loss_mlp": 1.00056827, + "epoch": 0.18182345772154837, + "flos": 66145082739840.0, + "grad_norm": 0.7203099741365571, + "language_loss": 0.47060892, + "learning_rate": 3.7630555498222856e-06, + "loss": 0.49070582, + "num_input_tokens_seen": 179497500, + "router_z_loss_clip": 0.06640625, + "router_z_loss_mlp": 0.0088501, + "step": 6266, + "time_per_iteration": 2.813424587249756 + }, + { + "auxiliary_loss_clip": 0.0107477, + "auxiliary_loss_mlp": 0.0103481, + "balance_loss_clip": 1.02448249, + "balance_loss_mlp": 1.01992059, + "epoch": 0.18185247519006442, + "flos": 23980338115200.0, + "grad_norm": 2.9195352633540863, + "language_loss": 0.8894124, + "learning_rate": 3.76296679924929e-06, + "loss": 0.91050816, + "num_input_tokens_seen": 179511440, + "router_z_loss_clip": 0.50317383, + "router_z_loss_mlp": 0.14880371, + "step": 6267, + "time_per_iteration": 2.4190516471862793 + }, + { + "auxiliary_loss_clip": 0.01075303, + "auxiliary_loss_mlp": 0.01035186, + "balance_loss_clip": 1.02398372, + "balance_loss_mlp": 1.01886678, + "epoch": 0.18188149265858047, + "flos": 22958081239680.0, + "grad_norm": 2.660982331974571, + "language_loss": 0.79298651, + "learning_rate": 3.7628780331050586e-06, + "loss": 0.81409144, + "num_input_tokens_seen": 179524475, + "router_z_loss_clip": 0.51293945, + "router_z_loss_mlp": 0.16320801, + "step": 6268, + "time_per_iteration": 2.3772008419036865 + }, + { + "auxiliary_loss_clip": 0.01083107, + "auxiliary_loss_mlp": 0.01043732, + "balance_loss_clip": 1.02700543, + "balance_loss_mlp": 1.02539158, + "epoch": 0.18191051012709653, + "flos": 35880274097280.0, + "grad_norm": 2.9909658103030474, + "language_loss": 0.89240748, + "learning_rate": 3.762789251390375e-06, + "loss": 0.9136759, + "num_input_tokens_seen": 179548470, + "router_z_loss_clip": 0.56225586, + "router_z_loss_mlp": 0.18322754, + "step": 6269, + "time_per_iteration": 2.5799813270568848 + }, + { + "auxiliary_loss_clip": 0.0107864, + "auxiliary_loss_mlp": 0.01034391, + "balance_loss_clip": 1.02482629, + "balance_loss_mlp": 1.01751113, + "epoch": 0.18193952759561255, + "flos": 34122676832640.0, + "grad_norm": 1.8763178578856126, + "language_loss": 0.87838775, + "learning_rate": 3.7627004541060233e-06, + "loss": 0.89951801, + "num_input_tokens_seen": 179567995, + "router_z_loss_clip": 0.53710938, + "router_z_loss_mlp": 0.16894531, + "step": 6270, + "time_per_iteration": 2.491736888885498 + }, + { + "auxiliary_loss_clip": 0.0107711, + "auxiliary_loss_mlp": 0.01034927, + "balance_loss_clip": 1.02343225, + "balance_loss_mlp": 1.01893461, + "epoch": 0.1819685450641286, + "flos": 33794982011520.0, + "grad_norm": 2.1963479911307062, + "language_loss": 0.80556059, + "learning_rate": 3.7626116412527876e-06, + "loss": 0.82668102, + "num_input_tokens_seen": 179583595, + "router_z_loss_clip": 0.53686523, + "router_z_loss_mlp": 0.15997314, + "step": 6271, + "time_per_iteration": 2.44101881980896 + }, + { + "auxiliary_loss_clip": 0.01080499, + "auxiliary_loss_mlp": 0.01038349, + "balance_loss_clip": 1.02493942, + "balance_loss_mlp": 1.02188623, + "epoch": 0.18199756253264465, + "flos": 43462711440000.0, + "grad_norm": 2.2041538355606503, + "language_loss": 0.74354959, + "learning_rate": 3.762522812831453e-06, + "loss": 0.76473802, + "num_input_tokens_seen": 179600390, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.16467285, + "step": 6272, + "time_per_iteration": 2.558619737625122 + }, + { + "auxiliary_loss_clip": 0.01074308, + "auxiliary_loss_mlp": 0.01029986, + "balance_loss_clip": 1.02532423, + "balance_loss_mlp": 1.01616371, + "epoch": 0.1820265800011607, + "flos": 30110283498240.0, + "grad_norm": 2.0549690665429, + "language_loss": 0.79683352, + "learning_rate": 3.762433968842804e-06, + "loss": 0.81787646, + "num_input_tokens_seen": 179614200, + "router_z_loss_clip": 0.49047852, + "router_z_loss_mlp": 0.13818359, + "step": 6273, + "time_per_iteration": 2.488596200942993 + }, + { + "auxiliary_loss_clip": 0.01077354, + "auxiliary_loss_mlp": 0.01033725, + "balance_loss_clip": 1.02394938, + "balance_loss_mlp": 1.01637387, + "epoch": 0.18205559746967676, + "flos": 19965570808320.0, + "grad_norm": 2.959545761381764, + "language_loss": 0.72510624, + "learning_rate": 3.762345109287624e-06, + "loss": 0.74621701, + "num_input_tokens_seen": 179627600, + "router_z_loss_clip": 0.53369141, + "router_z_loss_mlp": 0.17364502, + "step": 6274, + "time_per_iteration": 2.3787755966186523 + }, + { + "auxiliary_loss_clip": 0.01075188, + "auxiliary_loss_mlp": 0.01033703, + "balance_loss_clip": 1.02403522, + "balance_loss_mlp": 1.01833725, + "epoch": 0.18208461493819278, + "flos": 34740918921600.0, + "grad_norm": 2.2219881036551574, + "language_loss": 0.88095671, + "learning_rate": 3.7622562341666997e-06, + "loss": 0.90204561, + "num_input_tokens_seen": 179645390, + "router_z_loss_clip": 0.51098633, + "router_z_loss_mlp": 0.15362549, + "step": 6275, + "time_per_iteration": 2.502143144607544 + }, + { + "auxiliary_loss_clip": 0.01074345, + "auxiliary_loss_mlp": 0.01031372, + "balance_loss_clip": 1.02427256, + "balance_loss_mlp": 1.01523709, + "epoch": 0.18211363240670883, + "flos": 16684084120320.0, + "grad_norm": 2.745052319759808, + "language_loss": 0.64000666, + "learning_rate": 3.762167343480815e-06, + "loss": 0.66106379, + "num_input_tokens_seen": 179658815, + "router_z_loss_clip": 0.50146484, + "router_z_loss_mlp": 0.16125488, + "step": 6276, + "time_per_iteration": 2.3530807495117188 + }, + { + "auxiliary_loss_clip": 0.01008922, + "auxiliary_loss_mlp": 0.01000435, + "balance_loss_clip": 1.00209153, + "balance_loss_mlp": 0.99980313, + "epoch": 0.18214264987522488, + "flos": 74770409571840.0, + "grad_norm": 0.667041215686708, + "language_loss": 0.47193176, + "learning_rate": 3.762078437230755e-06, + "loss": 0.49202535, + "num_input_tokens_seen": 179721375, + "router_z_loss_clip": 0.06835938, + "router_z_loss_mlp": 0.00631714, + "step": 6277, + "time_per_iteration": 3.0486927032470703 + }, + { + "auxiliary_loss_clip": 0.01078473, + "auxiliary_loss_mlp": 0.0102954, + "balance_loss_clip": 1.02435279, + "balance_loss_mlp": 1.01325941, + "epoch": 0.18217166734374093, + "flos": 47888371071360.0, + "grad_norm": 2.5617125803860272, + "language_loss": 1.08907318, + "learning_rate": 3.761989515417306e-06, + "loss": 1.11015344, + "num_input_tokens_seen": 179744325, + "router_z_loss_clip": 0.54150391, + "router_z_loss_mlp": 0.1628418, + "step": 6278, + "time_per_iteration": 2.641336441040039 + }, + { + "auxiliary_loss_clip": 0.01075211, + "auxiliary_loss_mlp": 0.01037554, + "balance_loss_clip": 1.02556372, + "balance_loss_mlp": 1.02054906, + "epoch": 0.18220068481225699, + "flos": 12347452189440.0, + "grad_norm": 2.462002753023793, + "language_loss": 0.75441241, + "learning_rate": 3.761900578041252e-06, + "loss": 0.77554011, + "num_input_tokens_seen": 179756965, + "router_z_loss_clip": 0.49682617, + "router_z_loss_mlp": 0.17010498, + "step": 6279, + "time_per_iteration": 2.3662288188934326 + }, + { + "auxiliary_loss_clip": 0.01077266, + "auxiliary_loss_mlp": 0.01030515, + "balance_loss_clip": 1.02477026, + "balance_loss_mlp": 1.0135932, + "epoch": 0.18222970228077304, + "flos": 33466065292800.0, + "grad_norm": 2.4990426248994138, + "language_loss": 0.83291823, + "learning_rate": 3.7618116251033785e-06, + "loss": 0.85399604, + "num_input_tokens_seen": 179772335, + "router_z_loss_clip": 0.52514648, + "router_z_loss_mlp": 0.16918945, + "step": 6280, + "time_per_iteration": 2.4734408855438232 + }, + { + "auxiliary_loss_clip": 0.01008878, + "auxiliary_loss_mlp": 0.01000663, + "balance_loss_clip": 1.00211465, + "balance_loss_mlp": 0.99989438, + "epoch": 0.18225871974928906, + "flos": 62692660702080.0, + "grad_norm": 0.6104469583632961, + "language_loss": 0.49175748, + "learning_rate": 3.7617226566044727e-06, + "loss": 0.51185286, + "num_input_tokens_seen": 179833975, + "router_z_loss_clip": 0.06738281, + "router_z_loss_mlp": 0.00769043, + "step": 6281, + "time_per_iteration": 2.969233989715576 + }, + { + "auxiliary_loss_clip": 0.01071694, + "auxiliary_loss_mlp": 0.01028042, + "balance_loss_clip": 1.02237225, + "balance_loss_mlp": 1.01205051, + "epoch": 0.1822877372178051, + "flos": 35838588067200.0, + "grad_norm": 2.2188776166584363, + "language_loss": 0.71756721, + "learning_rate": 3.7616336725453197e-06, + "loss": 0.73856461, + "num_input_tokens_seen": 179860155, + "router_z_loss_clip": 0.49316406, + "router_z_loss_mlp": 0.15991211, + "step": 6282, + "time_per_iteration": 2.7836105823516846 + }, + { + "auxiliary_loss_clip": 0.0108107, + "auxiliary_loss_mlp": 0.01030745, + "balance_loss_clip": 1.02542913, + "balance_loss_mlp": 1.01311433, + "epoch": 0.18231675468632116, + "flos": 28546802726400.0, + "grad_norm": 1.6257004037852345, + "language_loss": 0.70440358, + "learning_rate": 3.761544672926704e-06, + "loss": 0.72552174, + "num_input_tokens_seen": 179884795, + "router_z_loss_clip": 0.5559082, + "router_z_loss_mlp": 0.17633057, + "step": 6283, + "time_per_iteration": 2.7359120845794678 + }, + { + "auxiliary_loss_clip": 0.01076382, + "auxiliary_loss_mlp": 0.0103699, + "balance_loss_clip": 1.02415323, + "balance_loss_mlp": 1.02018118, + "epoch": 0.18234577215483722, + "flos": 24783968856960.0, + "grad_norm": 2.35032290603186, + "language_loss": 0.82659012, + "learning_rate": 3.761455657749414e-06, + "loss": 0.84772384, + "num_input_tokens_seen": 179901810, + "router_z_loss_clip": 0.52197266, + "router_z_loss_mlp": 0.16821289, + "step": 6284, + "time_per_iteration": 2.4542441368103027 + }, + { + "auxiliary_loss_clip": 0.01082489, + "auxiliary_loss_mlp": 0.01043936, + "balance_loss_clip": 1.02646327, + "balance_loss_mlp": 1.02426624, + "epoch": 0.18237478962335327, + "flos": 23030351487360.0, + "grad_norm": 3.0260199094720086, + "language_loss": 0.97015882, + "learning_rate": 3.7613666270142347e-06, + "loss": 0.99142301, + "num_input_tokens_seen": 179915765, + "router_z_loss_clip": 0.56005859, + "router_z_loss_mlp": 0.19677734, + "step": 6285, + "time_per_iteration": 2.3866474628448486 + }, + { + "auxiliary_loss_clip": 0.0107991, + "auxiliary_loss_mlp": 0.0103853, + "balance_loss_clip": 1.02460623, + "balance_loss_mlp": 1.02144146, + "epoch": 0.18240380709186932, + "flos": 43983684881280.0, + "grad_norm": 1.9262487847226817, + "language_loss": 0.87530243, + "learning_rate": 3.7612775807219523e-06, + "loss": 0.89648688, + "num_input_tokens_seen": 179934435, + "router_z_loss_clip": 0.55273438, + "router_z_loss_mlp": 0.1708374, + "step": 6286, + "time_per_iteration": 2.508448362350464 + }, + { + "auxiliary_loss_clip": 0.01074441, + "auxiliary_loss_mlp": 0.01038336, + "balance_loss_clip": 1.02334011, + "balance_loss_mlp": 1.02255869, + "epoch": 0.18243282456038534, + "flos": 17377389365760.0, + "grad_norm": 2.4598910836512036, + "language_loss": 0.70473206, + "learning_rate": 3.761188518873354e-06, + "loss": 0.72585988, + "num_input_tokens_seen": 179947990, + "router_z_loss_clip": 0.51098633, + "router_z_loss_mlp": 0.15771484, + "step": 6287, + "time_per_iteration": 2.365744113922119 + }, + { + "auxiliary_loss_clip": 0.01074988, + "auxiliary_loss_mlp": 0.0102749, + "balance_loss_clip": 1.02441955, + "balance_loss_mlp": 1.01304197, + "epoch": 0.1824618420289014, + "flos": 9721878814080.0, + "grad_norm": 2.2285407409465305, + "language_loss": 0.73794085, + "learning_rate": 3.761099441469225e-06, + "loss": 0.75896561, + "num_input_tokens_seen": 179959645, + "router_z_loss_clip": 0.50561523, + "router_z_loss_mlp": 0.14447021, + "step": 6288, + "time_per_iteration": 2.4132630825042725 + }, + { + "auxiliary_loss_clip": 0.01079039, + "auxiliary_loss_mlp": 0.01032237, + "balance_loss_clip": 1.02571988, + "balance_loss_mlp": 1.01539314, + "epoch": 0.18249085949741745, + "flos": 11282217563520.0, + "grad_norm": 2.5767387685149217, + "language_loss": 0.91639304, + "learning_rate": 3.761010348510354e-06, + "loss": 0.93750584, + "num_input_tokens_seen": 179969945, + "router_z_loss_clip": 0.53393555, + "router_z_loss_mlp": 0.16845703, + "step": 6289, + "time_per_iteration": 2.3340694904327393 + }, + { + "auxiliary_loss_clip": 0.01081156, + "auxiliary_loss_mlp": 0.01031232, + "balance_loss_clip": 1.02552927, + "balance_loss_mlp": 1.01324892, + "epoch": 0.1825198769659335, + "flos": 12852296582400.0, + "grad_norm": 3.119949409815996, + "language_loss": 0.93447268, + "learning_rate": 3.7609212399975273e-06, + "loss": 0.95559657, + "num_input_tokens_seen": 179982260, + "router_z_loss_clip": 0.55566406, + "router_z_loss_mlp": 0.17980957, + "step": 6290, + "time_per_iteration": 2.3313708305358887 + }, + { + "auxiliary_loss_clip": 0.01073464, + "auxiliary_loss_mlp": 0.01030525, + "balance_loss_clip": 1.02520728, + "balance_loss_mlp": 1.01538539, + "epoch": 0.18254889443444955, + "flos": 12379781975040.0, + "grad_norm": 3.2419345027847406, + "language_loss": 0.87046719, + "learning_rate": 3.7608321159315315e-06, + "loss": 0.89150715, + "num_input_tokens_seen": 179993775, + "router_z_loss_clip": 0.48193359, + "router_z_loss_mlp": 0.15142822, + "step": 6291, + "time_per_iteration": 2.376725435256958 + }, + { + "auxiliary_loss_clip": 0.01068816, + "auxiliary_loss_mlp": 0.01030568, + "balance_loss_clip": 1.0214088, + "balance_loss_mlp": 1.01578057, + "epoch": 0.18257791190296557, + "flos": 13398303335040.0, + "grad_norm": 2.410717275515105, + "language_loss": 0.92076325, + "learning_rate": 3.7607429763131535e-06, + "loss": 0.94175708, + "num_input_tokens_seen": 180006425, + "router_z_loss_clip": 0.47387695, + "router_z_loss_mlp": 0.14782715, + "step": 6292, + "time_per_iteration": 2.3379907608032227 + }, + { + "auxiliary_loss_clip": 0.01076452, + "auxiliary_loss_mlp": 0.01030996, + "balance_loss_clip": 1.02361989, + "balance_loss_mlp": 1.01472974, + "epoch": 0.18260692937148162, + "flos": 23689302088320.0, + "grad_norm": 8.10324500218618, + "language_loss": 0.72722101, + "learning_rate": 3.760653821143181e-06, + "loss": 0.74829555, + "num_input_tokens_seen": 180022700, + "router_z_loss_clip": 0.52880859, + "router_z_loss_mlp": 0.16253662, + "step": 6293, + "time_per_iteration": 2.422792673110962 + }, + { + "auxiliary_loss_clip": 0.01080761, + "auxiliary_loss_mlp": 0.01040794, + "balance_loss_clip": 1.02549481, + "balance_loss_mlp": 1.02269161, + "epoch": 0.18263594683999768, + "flos": 16099498448640.0, + "grad_norm": 3.0717433805544214, + "language_loss": 0.78470296, + "learning_rate": 3.7605646504224017e-06, + "loss": 0.80591846, + "num_input_tokens_seen": 180034550, + "router_z_loss_clip": 0.55273438, + "router_z_loss_mlp": 0.18103027, + "step": 6294, + "time_per_iteration": 2.33123779296875 + }, + { + "auxiliary_loss_clip": 0.0100881, + "auxiliary_loss_mlp": 0.01001188, + "balance_loss_clip": 1.00207722, + "balance_loss_mlp": 1.00035906, + "epoch": 0.18266496430851373, + "flos": 56387066935680.0, + "grad_norm": 0.6952549723870333, + "language_loss": 0.46172154, + "learning_rate": 3.7604754641516026e-06, + "loss": 0.48182154, + "num_input_tokens_seen": 180089620, + "router_z_loss_clip": 0.06738281, + "router_z_loss_mlp": 0.00830078, + "step": 6295, + "time_per_iteration": 2.907711982727051 + }, + { + "auxiliary_loss_clip": 0.01008585, + "auxiliary_loss_mlp": 0.01002112, + "balance_loss_clip": 1.00192356, + "balance_loss_mlp": 1.00132513, + "epoch": 0.18269398177702978, + "flos": 69354438825600.0, + "grad_norm": 0.6231632962320552, + "language_loss": 0.42365465, + "learning_rate": 3.7603862623315723e-06, + "loss": 0.44376165, + "num_input_tokens_seen": 180146355, + "router_z_loss_clip": 0.06640625, + "router_z_loss_mlp": 0.00787354, + "step": 6296, + "time_per_iteration": 2.9753353595733643 + }, + { + "auxiliary_loss_clip": 0.01081062, + "auxiliary_loss_mlp": 0.01037576, + "balance_loss_clip": 1.02505815, + "balance_loss_mlp": 1.01995683, + "epoch": 0.18272299924554583, + "flos": 20443287208320.0, + "grad_norm": 2.127758262771822, + "language_loss": 0.84764647, + "learning_rate": 3.760297044963098e-06, + "loss": 0.86883283, + "num_input_tokens_seen": 180165855, + "router_z_loss_clip": 0.56054688, + "router_z_loss_mlp": 0.17614746, + "step": 6297, + "time_per_iteration": 2.3961894512176514 + }, + { + "auxiliary_loss_clip": 0.01076237, + "auxiliary_loss_mlp": 0.01032164, + "balance_loss_clip": 1.02481484, + "balance_loss_mlp": 1.01647627, + "epoch": 0.18275201671406185, + "flos": 34669451635200.0, + "grad_norm": 2.6245751610102483, + "language_loss": 0.94838172, + "learning_rate": 3.760207812046968e-06, + "loss": 0.96946573, + "num_input_tokens_seen": 180181675, + "router_z_loss_clip": 0.51416016, + "router_z_loss_mlp": 0.15692139, + "step": 6298, + "time_per_iteration": 2.527824640274048 + }, + { + "auxiliary_loss_clip": 0.01089942, + "auxiliary_loss_mlp": 0.01033541, + "balance_loss_clip": 1.02654839, + "balance_loss_mlp": 1.01328766, + "epoch": 0.1827810341825779, + "flos": 28030437584640.0, + "grad_norm": 2.4137873320955103, + "language_loss": 0.96840501, + "learning_rate": 3.7601185635839702e-06, + "loss": 0.98963976, + "num_input_tokens_seen": 180196365, + "router_z_loss_clip": 0.63427734, + "router_z_loss_mlp": 0.20239258, + "step": 6299, + "time_per_iteration": 2.444584608078003 + }, + { + "auxiliary_loss_clip": 0.01008605, + "auxiliary_loss_mlp": 0.01002766, + "balance_loss_clip": 1.00197196, + "balance_loss_mlp": 1.00194371, + "epoch": 0.18281005165109396, + "flos": 51095560786560.0, + "grad_norm": 0.6593795537032449, + "language_loss": 0.45363635, + "learning_rate": 3.760029299574893e-06, + "loss": 0.47375005, + "num_input_tokens_seen": 180253645, + "router_z_loss_clip": 0.06640625, + "router_z_loss_mlp": 0.00823975, + "step": 6300, + "time_per_iteration": 3.001793146133423 + }, + { + "auxiliary_loss_clip": 0.01008504, + "auxiliary_loss_mlp": 0.01002401, + "balance_loss_clip": 1.00164723, + "balance_loss_mlp": 1.00153649, + "epoch": 0.18283906911961, + "flos": 68674050783360.0, + "grad_norm": 0.6579973857216996, + "language_loss": 0.44428653, + "learning_rate": 3.759940020020525e-06, + "loss": 0.46439558, + "num_input_tokens_seen": 180312265, + "router_z_loss_clip": 0.06835938, + "router_z_loss_mlp": 0.00866699, + "step": 6301, + "time_per_iteration": 2.966952323913574 + }, + { + "auxiliary_loss_clip": 0.01085473, + "auxiliary_loss_mlp": 0.01035108, + "balance_loss_clip": 1.02819467, + "balance_loss_mlp": 1.01811504, + "epoch": 0.18286808658812606, + "flos": 34798452842880.0, + "grad_norm": 1.6155734798429973, + "language_loss": 0.85434932, + "learning_rate": 3.759850724921654e-06, + "loss": 0.87555516, + "num_input_tokens_seen": 180346215, + "router_z_loss_clip": 0.57324219, + "router_z_loss_mlp": 0.16992188, + "step": 6302, + "time_per_iteration": 2.963834762573242 + }, + { + "auxiliary_loss_clip": 0.01074393, + "auxiliary_loss_mlp": 0.01031183, + "balance_loss_clip": 1.02285337, + "balance_loss_mlp": 1.01640081, + "epoch": 0.18289710405664208, + "flos": 17050497505920.0, + "grad_norm": 3.1156052070646365, + "language_loss": 0.86236298, + "learning_rate": 3.75976141427907e-06, + "loss": 0.8834188, + "num_input_tokens_seen": 180359105, + "router_z_loss_clip": 0.51513672, + "router_z_loss_mlp": 0.14788818, + "step": 6303, + "time_per_iteration": 2.341581344604492 + }, + { + "auxiliary_loss_clip": 0.01008009, + "auxiliary_loss_mlp": 0.01001936, + "balance_loss_clip": 1.00126946, + "balance_loss_mlp": 1.00100577, + "epoch": 0.18292612152515814, + "flos": 69621558704640.0, + "grad_norm": 0.672018757379822, + "language_loss": 0.4592464, + "learning_rate": 3.759672088093561e-06, + "loss": 0.47934586, + "num_input_tokens_seen": 180420235, + "router_z_loss_clip": 0.06738281, + "router_z_loss_mlp": 0.00927734, + "step": 6304, + "time_per_iteration": 3.0776448249816895 + }, + { + "auxiliary_loss_clip": 0.01008797, + "auxiliary_loss_mlp": 0.0100422, + "balance_loss_clip": 1.00184798, + "balance_loss_mlp": 1.0034039, + "epoch": 0.1829551389936742, + "flos": 74762903629440.0, + "grad_norm": 0.6467787013732477, + "language_loss": 0.48216268, + "learning_rate": 3.7595827463659155e-06, + "loss": 0.50229287, + "num_input_tokens_seen": 180484395, + "router_z_loss_clip": 0.06933594, + "router_z_loss_mlp": 0.00817871, + "step": 6305, + "time_per_iteration": 3.0892698764801025 + }, + { + "auxiliary_loss_clip": 0.01072975, + "auxiliary_loss_mlp": 0.01026465, + "balance_loss_clip": 1.02275729, + "balance_loss_mlp": 1.01062834, + "epoch": 0.18298415646219024, + "flos": 28759843042560.0, + "grad_norm": 2.2018408430509635, + "language_loss": 0.65108907, + "learning_rate": 3.7594933890969232e-06, + "loss": 0.6720835, + "num_input_tokens_seen": 180502065, + "router_z_loss_clip": 0.50244141, + "router_z_loss_mlp": 0.1585083, + "step": 6306, + "time_per_iteration": 2.451166868209839 + }, + { + "auxiliary_loss_clip": 0.01074021, + "auxiliary_loss_mlp": 0.01024048, + "balance_loss_clip": 1.02343094, + "balance_loss_mlp": 1.00856829, + "epoch": 0.1830131739307063, + "flos": 11465546446080.0, + "grad_norm": 4.317133038988108, + "language_loss": 0.78893661, + "learning_rate": 3.759404016287374e-06, + "loss": 0.80991733, + "num_input_tokens_seen": 180512525, + "router_z_loss_clip": 0.50585938, + "router_z_loss_mlp": 0.15478516, + "step": 6307, + "time_per_iteration": 2.332641839981079 + }, + { + "auxiliary_loss_clip": 0.01075526, + "auxiliary_loss_mlp": 0.01032024, + "balance_loss_clip": 1.02313137, + "balance_loss_mlp": 1.01652634, + "epoch": 0.18304219139922234, + "flos": 74729947305600.0, + "grad_norm": 2.1033589323059116, + "language_loss": 0.92210448, + "learning_rate": 3.759314627938056e-06, + "loss": 0.94317997, + "num_input_tokens_seen": 180533805, + "router_z_loss_clip": 0.52392578, + "router_z_loss_mlp": 0.15509033, + "step": 6308, + "time_per_iteration": 2.7778255939483643 + }, + { + "auxiliary_loss_clip": 0.01074059, + "auxiliary_loss_mlp": 0.01033142, + "balance_loss_clip": 1.02323771, + "balance_loss_mlp": 1.01771069, + "epoch": 0.18307120886773837, + "flos": 34889858248320.0, + "grad_norm": 2.041859577410119, + "language_loss": 0.71019423, + "learning_rate": 3.7592252240497598e-06, + "loss": 0.73126626, + "num_input_tokens_seen": 180552535, + "router_z_loss_clip": 0.50878906, + "router_z_loss_mlp": 0.15429688, + "step": 6309, + "time_per_iteration": 2.5283799171447754 + }, + { + "auxiliary_loss_clip": 0.01079717, + "auxiliary_loss_mlp": 0.01028655, + "balance_loss_clip": 1.02570415, + "balance_loss_mlp": 1.01176858, + "epoch": 0.18310022633625442, + "flos": 15988998395520.0, + "grad_norm": 3.2119542729486446, + "language_loss": 0.95068622, + "learning_rate": 3.7591358046232744e-06, + "loss": 0.97176993, + "num_input_tokens_seen": 180564270, + "router_z_loss_clip": 0.54101562, + "router_z_loss_mlp": 0.16876221, + "step": 6310, + "time_per_iteration": 2.461205244064331 + }, + { + "auxiliary_loss_clip": 0.0107655, + "auxiliary_loss_mlp": 0.01030734, + "balance_loss_clip": 1.02462506, + "balance_loss_mlp": 1.01450324, + "epoch": 0.18312924380477047, + "flos": 15552130152960.0, + "grad_norm": 2.470130465172289, + "language_loss": 0.76191658, + "learning_rate": 3.7590463696593888e-06, + "loss": 0.78298944, + "num_input_tokens_seen": 180576640, + "router_z_loss_clip": 0.51953125, + "router_z_loss_mlp": 0.16235352, + "step": 6311, + "time_per_iteration": 2.417849540710449 + }, + { + "auxiliary_loss_clip": 0.01010429, + "auxiliary_loss_mlp": 0.01003434, + "balance_loss_clip": 1.00368512, + "balance_loss_mlp": 1.00270116, + "epoch": 0.18315826127328652, + "flos": 65468430213120.0, + "grad_norm": 0.655092677208183, + "language_loss": 0.46882361, + "learning_rate": 3.7589569191588945e-06, + "loss": 0.48896223, + "num_input_tokens_seen": 180638815, + "router_z_loss_clip": 0.06738281, + "router_z_loss_mlp": 0.00732422, + "step": 6312, + "time_per_iteration": 3.0305001735687256 + }, + { + "auxiliary_loss_clip": 0.01010635, + "auxiliary_loss_mlp": 0.01000665, + "balance_loss_clip": 1.00387192, + "balance_loss_mlp": 0.99994677, + "epoch": 0.18318727874180257, + "flos": 61709855529600.0, + "grad_norm": 0.7330195253598435, + "language_loss": 0.49248767, + "learning_rate": 3.7588674531225815e-06, + "loss": 0.51260066, + "num_input_tokens_seen": 180693490, + "router_z_loss_clip": 0.06738281, + "router_z_loss_mlp": 0.00717163, + "step": 6313, + "time_per_iteration": 2.9614415168762207 + }, + { + "auxiliary_loss_clip": 0.01077513, + "auxiliary_loss_mlp": 0.0103371, + "balance_loss_clip": 1.02589321, + "balance_loss_mlp": 1.01820636, + "epoch": 0.18321629621031862, + "flos": 45581729765760.0, + "grad_norm": 2.4675252347802292, + "language_loss": 0.82873809, + "learning_rate": 3.7587779715512386e-06, + "loss": 0.84985042, + "num_input_tokens_seen": 180720790, + "router_z_loss_clip": 0.51635742, + "router_z_loss_mlp": 0.15509033, + "step": 6314, + "time_per_iteration": 2.762634038925171 + }, + { + "auxiliary_loss_clip": 0.01071876, + "auxiliary_loss_mlp": 0.01027292, + "balance_loss_clip": 1.02424026, + "balance_loss_mlp": 1.01299894, + "epoch": 0.18324531367883465, + "flos": 13801166046720.0, + "grad_norm": 2.9332023049137144, + "language_loss": 0.94378793, + "learning_rate": 3.758688474445657e-06, + "loss": 0.96477968, + "num_input_tokens_seen": 180732655, + "router_z_loss_clip": 0.47680664, + "router_z_loss_mlp": 0.14300537, + "step": 6315, + "time_per_iteration": 4.488238334655762 + }, + { + "auxiliary_loss_clip": 0.01074602, + "auxiliary_loss_mlp": 0.01042144, + "balance_loss_clip": 1.02410388, + "balance_loss_mlp": 1.02537715, + "epoch": 0.1832743311473507, + "flos": 23980093735680.0, + "grad_norm": 2.4021646239719345, + "language_loss": 0.70711732, + "learning_rate": 3.7585989618066276e-06, + "loss": 0.72828478, + "num_input_tokens_seen": 180746185, + "router_z_loss_clip": 0.50488281, + "router_z_loss_mlp": 0.16772461, + "step": 6316, + "time_per_iteration": 4.744372606277466 + }, + { + "auxiliary_loss_clip": 0.01009222, + "auxiliary_loss_mlp": 0.01002008, + "balance_loss_clip": 1.0024972, + "balance_loss_mlp": 1.00119734, + "epoch": 0.18330334861586675, + "flos": 60461850602880.0, + "grad_norm": 2.308292397525182, + "language_loss": 0.47214365, + "learning_rate": 3.7585094336349405e-06, + "loss": 0.49225596, + "num_input_tokens_seen": 180807185, + "router_z_loss_clip": 0.06738281, + "router_z_loss_mlp": 0.00811768, + "step": 6317, + "time_per_iteration": 3.2442996501922607 + }, + { + "auxiliary_loss_clip": 0.01009788, + "auxiliary_loss_mlp": 0.01003265, + "balance_loss_clip": 1.0028019, + "balance_loss_mlp": 1.00247836, + "epoch": 0.1833323660843828, + "flos": 74762624338560.0, + "grad_norm": 0.6884063858382156, + "language_loss": 0.47075433, + "learning_rate": 3.7584198899313863e-06, + "loss": 0.49088487, + "num_input_tokens_seen": 180865900, + "router_z_loss_clip": 0.0703125, + "router_z_loss_mlp": 0.00787354, + "step": 6318, + "time_per_iteration": 3.023573160171509 + }, + { + "auxiliary_loss_clip": 0.01008875, + "auxiliary_loss_mlp": 0.01004311, + "balance_loss_clip": 1.00178933, + "balance_loss_mlp": 1.00339913, + "epoch": 0.18336138355289885, + "flos": 74777252198400.0, + "grad_norm": 0.7292595507666614, + "language_loss": 0.50350875, + "learning_rate": 3.758330330696756e-06, + "loss": 0.52364063, + "num_input_tokens_seen": 180927820, + "router_z_loss_clip": 0.07080078, + "router_z_loss_mlp": 0.00909424, + "step": 6319, + "time_per_iteration": 3.059572696685791 + }, + { + "auxiliary_loss_clip": 0.01008511, + "auxiliary_loss_mlp": 0.01001165, + "balance_loss_clip": 1.00142515, + "balance_loss_mlp": 1.00039005, + "epoch": 0.18339040102141488, + "flos": 67401466392960.0, + "grad_norm": 0.6485388591829511, + "language_loss": 0.4506157, + "learning_rate": 3.7582407559318404e-06, + "loss": 0.47071248, + "num_input_tokens_seen": 180990645, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00772095, + "step": 6320, + "time_per_iteration": 3.148622751235962 + }, + { + "auxiliary_loss_clip": 0.01008122, + "auxiliary_loss_mlp": 0.01009537, + "balance_loss_clip": 1.00105405, + "balance_loss_mlp": 1.00874472, + "epoch": 0.18341941848993093, + "flos": 59154876656640.0, + "grad_norm": 0.6640094084259638, + "language_loss": 0.44582957, + "learning_rate": 3.7581511656374313e-06, + "loss": 0.46600616, + "num_input_tokens_seen": 181052840, + "router_z_loss_clip": 0.07080078, + "router_z_loss_mlp": 0.00793457, + "step": 6321, + "time_per_iteration": 2.972442865371704 + }, + { + "auxiliary_loss_clip": 0.01008456, + "auxiliary_loss_mlp": 0.01006023, + "balance_loss_clip": 1.00146353, + "balance_loss_mlp": 1.00512898, + "epoch": 0.18344843595844698, + "flos": 74756898875520.0, + "grad_norm": 0.7391502483915472, + "language_loss": 0.52351695, + "learning_rate": 3.7580615598143198e-06, + "loss": 0.54366177, + "num_input_tokens_seen": 181107145, + "router_z_loss_clip": 0.0703125, + "router_z_loss_mlp": 0.00891113, + "step": 6322, + "time_per_iteration": 3.0397021770477295 + }, + { + "auxiliary_loss_clip": 0.01078297, + "auxiliary_loss_mlp": 0.01035626, + "balance_loss_clip": 1.02435887, + "balance_loss_mlp": 1.01812005, + "epoch": 0.18347745342696303, + "flos": 23871129782400.0, + "grad_norm": 2.381043867070682, + "language_loss": 0.99196541, + "learning_rate": 3.7579719384632973e-06, + "loss": 1.01310468, + "num_input_tokens_seen": 181120990, + "router_z_loss_clip": 0.5390625, + "router_z_loss_mlp": 0.1751709, + "step": 6323, + "time_per_iteration": 2.3385534286499023 + }, + { + "auxiliary_loss_clip": 0.01009541, + "auxiliary_loss_mlp": 0.01003065, + "balance_loss_clip": 1.00241935, + "balance_loss_mlp": 1.00230169, + "epoch": 0.18350647089547908, + "flos": 54844255555200.0, + "grad_norm": 0.6619127291163146, + "language_loss": 0.47683927, + "learning_rate": 3.757882301585155e-06, + "loss": 0.49696535, + "num_input_tokens_seen": 181177925, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00762939, + "step": 6324, + "time_per_iteration": 7.806495666503906 + }, + { + "auxiliary_loss_clip": 0.01010268, + "auxiliary_loss_mlp": 0.01000416, + "balance_loss_clip": 1.00312304, + "balance_loss_mlp": 0.99962884, + "epoch": 0.18353548836399514, + "flos": 62176714496640.0, + "grad_norm": 0.7073764760063478, + "language_loss": 0.49732035, + "learning_rate": 3.7577926491806846e-06, + "loss": 0.51742721, + "num_input_tokens_seen": 181232850, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00787354, + "step": 6325, + "time_per_iteration": 2.8556275367736816 + }, + { + "auxiliary_loss_clip": 0.01072982, + "auxiliary_loss_mlp": 0.01028221, + "balance_loss_clip": 1.02591491, + "balance_loss_mlp": 1.01350498, + "epoch": 0.18356450583251116, + "flos": 19056641806080.0, + "grad_norm": 2.4133458304022963, + "language_loss": 0.86852741, + "learning_rate": 3.7577029812506787e-06, + "loss": 0.88953948, + "num_input_tokens_seen": 181247770, + "router_z_loss_clip": 0.47045898, + "router_z_loss_mlp": 0.1472168, + "step": 6326, + "time_per_iteration": 2.380756378173828 + }, + { + "auxiliary_loss_clip": 0.01081867, + "auxiliary_loss_mlp": 0.01030208, + "balance_loss_clip": 1.0262537, + "balance_loss_mlp": 1.01318491, + "epoch": 0.1835935233010272, + "flos": 30512203603200.0, + "grad_norm": 1.7767663337328947, + "language_loss": 0.82653606, + "learning_rate": 3.757613297795928e-06, + "loss": 0.84765685, + "num_input_tokens_seen": 181264700, + "router_z_loss_clip": 0.55615234, + "router_z_loss_mlp": 0.17004395, + "step": 6327, + "time_per_iteration": 2.446949005126953 + }, + { + "auxiliary_loss_clip": 0.01014769, + "auxiliary_loss_mlp": 0.01000733, + "balance_loss_clip": 1.00729477, + "balance_loss_mlp": 1.00004411, + "epoch": 0.18362254076954326, + "flos": 59631894829440.0, + "grad_norm": 0.6409643723111883, + "language_loss": 0.44229859, + "learning_rate": 3.7575235988172266e-06, + "loss": 0.4624536, + "num_input_tokens_seen": 181325930, + "router_z_loss_clip": 0.07470703, + "router_z_loss_mlp": 0.00689697, + "step": 6328, + "time_per_iteration": 2.9568417072296143 + }, + { + "auxiliary_loss_clip": 0.01087107, + "auxiliary_loss_mlp": 0.01033089, + "balance_loss_clip": 1.03193367, + "balance_loss_mlp": 1.01570821, + "epoch": 0.18365155823805931, + "flos": 12706324721280.0, + "grad_norm": 4.594129373597458, + "language_loss": 0.75825721, + "learning_rate": 3.757433884315365e-06, + "loss": 0.77945918, + "num_input_tokens_seen": 181337180, + "router_z_loss_clip": 0.55126953, + "router_z_loss_mlp": 0.1739502, + "step": 6329, + "time_per_iteration": 2.333131790161133 + }, + { + "auxiliary_loss_clip": 0.01081543, + "auxiliary_loss_mlp": 0.01036147, + "balance_loss_clip": 1.02698839, + "balance_loss_mlp": 1.01884937, + "epoch": 0.18368057570657537, + "flos": 29677325328000.0, + "grad_norm": 3.533642210908693, + "language_loss": 0.97645175, + "learning_rate": 3.757344154291136e-06, + "loss": 0.99762863, + "num_input_tokens_seen": 181357175, + "router_z_loss_clip": 0.54614258, + "router_z_loss_mlp": 0.1730957, + "step": 6330, + "time_per_iteration": 2.4666974544525146 + }, + { + "auxiliary_loss_clip": 0.01014824, + "auxiliary_loss_mlp": 0.01008547, + "balance_loss_clip": 1.00737572, + "balance_loss_mlp": 1.00772452, + "epoch": 0.18370959317509142, + "flos": 74775506630400.0, + "grad_norm": 0.6490694817385962, + "language_loss": 0.50191307, + "learning_rate": 3.7572544087453325e-06, + "loss": 0.52214676, + "num_input_tokens_seen": 181424960, + "router_z_loss_clip": 0.07470703, + "router_z_loss_mlp": 0.00823975, + "step": 6331, + "time_per_iteration": 3.12043833732605 + }, + { + "auxiliary_loss_clip": 0.01015108, + "auxiliary_loss_mlp": 0.01012083, + "balance_loss_clip": 1.00767279, + "balance_loss_mlp": 1.01137662, + "epoch": 0.18373861064360744, + "flos": 70767616861440.0, + "grad_norm": 0.7378837690246486, + "language_loss": 0.49531302, + "learning_rate": 3.7571646476787474e-06, + "loss": 0.51558495, + "num_input_tokens_seen": 181487860, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00704956, + "step": 6332, + "time_per_iteration": 3.064058542251587 + }, + { + "auxiliary_loss_clip": 0.01080148, + "auxiliary_loss_mlp": 0.01041962, + "balance_loss_clip": 1.02693033, + "balance_loss_mlp": 1.02226853, + "epoch": 0.1837676281121235, + "flos": 11686127616000.0, + "grad_norm": 2.8122530454607206, + "language_loss": 0.73598868, + "learning_rate": 3.757074871092173e-06, + "loss": 0.75720978, + "num_input_tokens_seen": 181500145, + "router_z_loss_clip": 0.53198242, + "router_z_loss_mlp": 0.19689941, + "step": 6333, + "time_per_iteration": 2.459233283996582 + }, + { + "auxiliary_loss_clip": 0.01015174, + "auxiliary_loss_mlp": 0.01011633, + "balance_loss_clip": 1.00786507, + "balance_loss_mlp": 1.01084638, + "epoch": 0.18379664558063954, + "flos": 62943232596480.0, + "grad_norm": 0.689734600386481, + "language_loss": 0.46174225, + "learning_rate": 3.7569850789864017e-06, + "loss": 0.48201033, + "num_input_tokens_seen": 181551440, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00787354, + "step": 6334, + "time_per_iteration": 2.850586175918579 + }, + { + "auxiliary_loss_clip": 0.01015146, + "auxiliary_loss_mlp": 0.01002562, + "balance_loss_clip": 1.00798178, + "balance_loss_mlp": 1.00170982, + "epoch": 0.1838256630491556, + "flos": 67033761287040.0, + "grad_norm": 0.7116748580909297, + "language_loss": 0.47373194, + "learning_rate": 3.756895271362227e-06, + "loss": 0.493909, + "num_input_tokens_seen": 181610960, + "router_z_loss_clip": 0.07177734, + "router_z_loss_mlp": 0.00854492, + "step": 6335, + "time_per_iteration": 3.060059070587158 + }, + { + "auxiliary_loss_clip": 0.01072971, + "auxiliary_loss_mlp": 0.01030344, + "balance_loss_clip": 1.02525353, + "balance_loss_mlp": 1.01518679, + "epoch": 0.18385468051767165, + "flos": 24527322385920.0, + "grad_norm": 2.8242227984897172, + "language_loss": 0.91956842, + "learning_rate": 3.7568054482204433e-06, + "loss": 0.94060159, + "num_input_tokens_seen": 181624170, + "router_z_loss_clip": 0.47705078, + "router_z_loss_mlp": 0.15161133, + "step": 6336, + "time_per_iteration": 2.430103063583374 + }, + { + "auxiliary_loss_clip": 0.01080255, + "auxiliary_loss_mlp": 0.01034482, + "balance_loss_clip": 1.02605581, + "balance_loss_mlp": 1.01760197, + "epoch": 0.18388369798618767, + "flos": 33325783983360.0, + "grad_norm": 2.3729024447502396, + "language_loss": 1.00391507, + "learning_rate": 3.7567156095618426e-06, + "loss": 1.02506244, + "num_input_tokens_seen": 181644370, + "router_z_loss_clip": 0.54199219, + "router_z_loss_mlp": 0.16882324, + "step": 6337, + "time_per_iteration": 2.560471534729004 + }, + { + "auxiliary_loss_clip": 0.01081355, + "auxiliary_loss_mlp": 0.01031008, + "balance_loss_clip": 1.0279808, + "balance_loss_mlp": 1.01488531, + "epoch": 0.18391271545470372, + "flos": 31239828581760.0, + "grad_norm": 2.3099124507442133, + "language_loss": 0.79845023, + "learning_rate": 3.7566257553872182e-06, + "loss": 0.81957388, + "num_input_tokens_seen": 181659240, + "router_z_loss_clip": 0.53344727, + "router_z_loss_mlp": 0.16137695, + "step": 6338, + "time_per_iteration": 2.454197406768799 + }, + { + "auxiliary_loss_clip": 0.0107771, + "auxiliary_loss_mlp": 0.01040588, + "balance_loss_clip": 1.02794635, + "balance_loss_mlp": 1.02470374, + "epoch": 0.18394173292321978, + "flos": 28359354303360.0, + "grad_norm": 2.0523003283892227, + "language_loss": 1.02993381, + "learning_rate": 3.7565358856973648e-06, + "loss": 1.05111682, + "num_input_tokens_seen": 181677935, + "router_z_loss_clip": 0.49731445, + "router_z_loss_mlp": 0.15899658, + "step": 6339, + "time_per_iteration": 2.4548428058624268 + }, + { + "auxiliary_loss_clip": 0.0108711, + "auxiliary_loss_mlp": 0.01031384, + "balance_loss_clip": 1.02973318, + "balance_loss_mlp": 1.01395607, + "epoch": 0.18397075039173583, + "flos": 17157960270720.0, + "grad_norm": 2.688315805074016, + "language_loss": 0.94501865, + "learning_rate": 3.7564460004930754e-06, + "loss": 0.96620363, + "num_input_tokens_seen": 181691315, + "router_z_loss_clip": 0.57373047, + "router_z_loss_mlp": 0.17425537, + "step": 6340, + "time_per_iteration": 2.349017858505249 + }, + { + "auxiliary_loss_clip": 0.01087703, + "auxiliary_loss_mlp": 0.01034284, + "balance_loss_clip": 1.02967143, + "balance_loss_mlp": 1.01610422, + "epoch": 0.18399976786025188, + "flos": 28176933116160.0, + "grad_norm": 2.4072974530673683, + "language_loss": 0.8251369, + "learning_rate": 3.7563560997751447e-06, + "loss": 0.84635681, + "num_input_tokens_seen": 181708105, + "router_z_loss_clip": 0.58056641, + "router_z_loss_mlp": 0.18164062, + "step": 6341, + "time_per_iteration": 2.4843719005584717 + }, + { + "auxiliary_loss_clip": 0.0108215, + "auxiliary_loss_mlp": 0.01034386, + "balance_loss_clip": 1.02880561, + "balance_loss_mlp": 1.01723123, + "epoch": 0.18402878532876793, + "flos": 14276613208320.0, + "grad_norm": 2.9558490854819417, + "language_loss": 0.77375174, + "learning_rate": 3.756266183544366e-06, + "loss": 0.79491711, + "num_input_tokens_seen": 181720545, + "router_z_loss_clip": 0.53344727, + "router_z_loss_mlp": 0.17150879, + "step": 6342, + "time_per_iteration": 2.376757860183716 + }, + { + "auxiliary_loss_clip": 0.01083909, + "auxiliary_loss_mlp": 0.0103775, + "balance_loss_clip": 1.03077793, + "balance_loss_mlp": 1.02134669, + "epoch": 0.18405780279728395, + "flos": 20845277136000.0, + "grad_norm": 3.4355421648760722, + "language_loss": 0.90147567, + "learning_rate": 3.7561762518015334e-06, + "loss": 0.9226923, + "num_input_tokens_seen": 181735740, + "router_z_loss_clip": 0.53076172, + "router_z_loss_mlp": 0.1640625, + "step": 6343, + "time_per_iteration": 2.4273526668548584 + }, + { + "auxiliary_loss_clip": 0.01089135, + "auxiliary_loss_mlp": 0.01033538, + "balance_loss_clip": 1.03236127, + "balance_loss_mlp": 1.0152756, + "epoch": 0.1840868202658, + "flos": 13762063457280.0, + "grad_norm": 2.8141490433711343, + "language_loss": 0.95907557, + "learning_rate": 3.7560863045474414e-06, + "loss": 0.98030221, + "num_input_tokens_seen": 181745725, + "router_z_loss_clip": 0.56787109, + "router_z_loss_mlp": 0.18273926, + "step": 6344, + "time_per_iteration": 2.3446929454803467 + }, + { + "auxiliary_loss_clip": 0.01087811, + "auxiliary_loss_mlp": 0.01032522, + "balance_loss_clip": 1.03328347, + "balance_loss_mlp": 1.01565957, + "epoch": 0.18411583773431606, + "flos": 23215565583360.0, + "grad_norm": 2.511346741081558, + "language_loss": 0.71397716, + "learning_rate": 3.755996341782885e-06, + "loss": 0.7351805, + "num_input_tokens_seen": 181764640, + "router_z_loss_clip": 0.54516602, + "router_z_loss_mlp": 0.1685791, + "step": 6345, + "time_per_iteration": 2.5461559295654297 + }, + { + "auxiliary_loss_clip": 0.01079459, + "auxiliary_loss_mlp": 0.01037523, + "balance_loss_clip": 1.02986145, + "balance_loss_mlp": 1.02257991, + "epoch": 0.1841448552028321, + "flos": 27817117977600.0, + "grad_norm": 1.76075988009452, + "language_loss": 0.7593956, + "learning_rate": 3.755906363508658e-06, + "loss": 0.78056538, + "num_input_tokens_seen": 181784705, + "router_z_loss_clip": 0.49560547, + "router_z_loss_mlp": 0.1494751, + "step": 6346, + "time_per_iteration": 2.4459967613220215 + }, + { + "auxiliary_loss_clip": 0.0108568, + "auxiliary_loss_mlp": 0.01027719, + "balance_loss_clip": 1.03346658, + "balance_loss_mlp": 1.01299632, + "epoch": 0.18417387267134816, + "flos": 16246203448320.0, + "grad_norm": 2.5512256476569974, + "language_loss": 0.78405488, + "learning_rate": 3.7558163697255562e-06, + "loss": 0.80518878, + "num_input_tokens_seen": 181800060, + "router_z_loss_clip": 0.5222168, + "router_z_loss_mlp": 0.1472168, + "step": 6347, + "time_per_iteration": 2.3776206970214844 + }, + { + "auxiliary_loss_clip": 0.01086104, + "auxiliary_loss_mlp": 0.01037506, + "balance_loss_clip": 1.03386092, + "balance_loss_mlp": 1.02168095, + "epoch": 0.1842028901398642, + "flos": 33028743202560.0, + "grad_norm": 1.8967835650247793, + "language_loss": 0.67945969, + "learning_rate": 3.755726360434373e-06, + "loss": 0.70069575, + "num_input_tokens_seen": 181816035, + "router_z_loss_clip": 0.52148438, + "router_z_loss_mlp": 0.15826416, + "step": 6348, + "time_per_iteration": 2.4737415313720703 + }, + { + "auxiliary_loss_clip": 0.01088832, + "auxiliary_loss_mlp": 0.01037814, + "balance_loss_clip": 1.03281558, + "balance_loss_mlp": 1.01927066, + "epoch": 0.18423190760838024, + "flos": 27377736117120.0, + "grad_norm": 2.0042297611875854, + "language_loss": 0.78807801, + "learning_rate": 3.755636335635904e-06, + "loss": 0.80934441, + "num_input_tokens_seen": 181836565, + "router_z_loss_clip": 0.56005859, + "router_z_loss_mlp": 0.18560791, + "step": 6349, + "time_per_iteration": 2.52461838722229 + }, + { + "auxiliary_loss_clip": 0.01084469, + "auxiliary_loss_mlp": 0.01030239, + "balance_loss_clip": 1.03229547, + "balance_loss_mlp": 1.01422942, + "epoch": 0.1842609250768963, + "flos": 27882929623680.0, + "grad_norm": 2.2598887485544226, + "language_loss": 0.74608207, + "learning_rate": 3.755546295330945e-06, + "loss": 0.7672292, + "num_input_tokens_seen": 181852210, + "router_z_loss_clip": 0.52172852, + "router_z_loss_mlp": 0.16015625, + "step": 6350, + "time_per_iteration": 2.444620370864868 + }, + { + "auxiliary_loss_clip": 0.01080947, + "auxiliary_loss_mlp": 0.0103355, + "balance_loss_clip": 1.03086674, + "balance_loss_mlp": 1.01715863, + "epoch": 0.18428994254541234, + "flos": 27300264076800.0, + "grad_norm": 2.25672367049348, + "language_loss": 0.79055285, + "learning_rate": 3.75545623952029e-06, + "loss": 0.81169784, + "num_input_tokens_seen": 181865785, + "router_z_loss_clip": 0.50048828, + "router_z_loss_mlp": 0.16394043, + "step": 6351, + "time_per_iteration": 2.483778715133667 + }, + { + "auxiliary_loss_clip": 0.01080614, + "auxiliary_loss_mlp": 0.01033474, + "balance_loss_clip": 1.02998042, + "balance_loss_mlp": 1.01756573, + "epoch": 0.1843189600139284, + "flos": 29562182064000.0, + "grad_norm": 2.321478745486413, + "language_loss": 0.84625405, + "learning_rate": 3.7553661682047357e-06, + "loss": 0.86739492, + "num_input_tokens_seen": 181881200, + "router_z_loss_clip": 0.5065918, + "router_z_loss_mlp": 0.15905762, + "step": 6352, + "time_per_iteration": 2.4755918979644775 + }, + { + "auxiliary_loss_clip": 0.01024082, + "auxiliary_loss_mlp": 0.01007505, + "balance_loss_clip": 1.01594782, + "balance_loss_mlp": 1.00651526, + "epoch": 0.18434797748244444, + "flos": 74780743334400.0, + "grad_norm": 0.6300000122316044, + "language_loss": 0.46730566, + "learning_rate": 3.755276081385077e-06, + "loss": 0.48762149, + "num_input_tokens_seen": 181952110, + "router_z_loss_clip": 0.08154297, + "router_z_loss_mlp": 0.0098877, + "step": 6353, + "time_per_iteration": 3.2147130966186523 + }, + { + "auxiliary_loss_clip": 0.01023373, + "auxiliary_loss_mlp": 0.01002059, + "balance_loss_clip": 1.01533747, + "balance_loss_mlp": 1.00110567, + "epoch": 0.18437699495096047, + "flos": 64925006901120.0, + "grad_norm": 0.6919773567704907, + "language_loss": 0.46441039, + "learning_rate": 3.7551859790621092e-06, + "loss": 0.48466468, + "num_input_tokens_seen": 182017670, + "router_z_loss_clip": 0.08007812, + "router_z_loss_mlp": 0.00952148, + "step": 6354, + "time_per_iteration": 3.018270492553711 + }, + { + "auxiliary_loss_clip": 0.01019549, + "auxiliary_loss_mlp": 0.0100178, + "balance_loss_clip": 1.01166701, + "balance_loss_mlp": 1.00078416, + "epoch": 0.18440601241947652, + "flos": 74444809432320.0, + "grad_norm": 0.668478306423288, + "language_loss": 0.49094099, + "learning_rate": 3.755095861236629e-06, + "loss": 0.51115429, + "num_input_tokens_seen": 182085255, + "router_z_loss_clip": 0.07910156, + "router_z_loss_mlp": 0.00994873, + "step": 6355, + "time_per_iteration": 3.094874143600464 + }, + { + "auxiliary_loss_clip": 0.01075709, + "auxiliary_loss_mlp": 0.01030495, + "balance_loss_clip": 1.02663016, + "balance_loss_mlp": 1.01687574, + "epoch": 0.18443502988799257, + "flos": 29347291445760.0, + "grad_norm": 1.6917825841530767, + "language_loss": 0.71942991, + "learning_rate": 3.755005727909432e-06, + "loss": 0.74049187, + "num_input_tokens_seen": 182104955, + "router_z_loss_clip": 0.4909668, + "router_z_loss_mlp": 0.13604736, + "step": 6356, + "time_per_iteration": 2.5142552852630615 + }, + { + "auxiliary_loss_clip": 0.01077052, + "auxiliary_loss_mlp": 0.01035672, + "balance_loss_clip": 1.02829003, + "balance_loss_mlp": 1.02140307, + "epoch": 0.18446404735650862, + "flos": 26685513123840.0, + "grad_norm": 2.0674269303473567, + "language_loss": 0.7196641, + "learning_rate": 3.7549155790813144e-06, + "loss": 0.74079132, + "num_input_tokens_seen": 182119450, + "router_z_loss_clip": 0.48803711, + "router_z_loss_mlp": 0.14282227, + "step": 6357, + "time_per_iteration": 2.4479830265045166 + }, + { + "auxiliary_loss_clip": 0.01081469, + "auxiliary_loss_mlp": 0.01034022, + "balance_loss_clip": 1.02871072, + "balance_loss_mlp": 1.01897144, + "epoch": 0.18449306482502467, + "flos": 12851109596160.0, + "grad_norm": 2.791946022863689, + "language_loss": 0.76826251, + "learning_rate": 3.754825414753072e-06, + "loss": 0.78941739, + "num_input_tokens_seen": 182131300, + "router_z_loss_clip": 0.52783203, + "router_z_loss_mlp": 0.15063477, + "step": 6358, + "time_per_iteration": 2.354879856109619 + }, + { + "auxiliary_loss_clip": 0.01074907, + "auxiliary_loss_mlp": 0.01032712, + "balance_loss_clip": 1.02726626, + "balance_loss_mlp": 1.01896715, + "epoch": 0.18452208229354072, + "flos": 19237945829760.0, + "grad_norm": 2.3228414085204467, + "language_loss": 0.78884768, + "learning_rate": 3.754735234925501e-06, + "loss": 0.80992389, + "num_input_tokens_seen": 182147885, + "router_z_loss_clip": 0.47705078, + "router_z_loss_mlp": 0.13726807, + "step": 6359, + "time_per_iteration": 2.434389114379883 + }, + { + "auxiliary_loss_clip": 0.01017643, + "auxiliary_loss_mlp": 0.01046658, + "balance_loss_clip": 1.0097512, + "balance_loss_mlp": 1.04571009, + "epoch": 0.18455109976205675, + "flos": 59773572593280.0, + "grad_norm": 0.6731052315373084, + "language_loss": 0.46786219, + "learning_rate": 3.754645039599399e-06, + "loss": 0.48850521, + "num_input_tokens_seen": 182205440, + "router_z_loss_clip": 0.07910156, + "router_z_loss_mlp": 0.00946045, + "step": 6360, + "time_per_iteration": 2.8861916065216064 + }, + { + "auxiliary_loss_clip": 0.01082272, + "auxiliary_loss_mlp": 0.01031237, + "balance_loss_clip": 1.0290699, + "balance_loss_mlp": 1.01444042, + "epoch": 0.1845801172305728, + "flos": 55392742880640.0, + "grad_norm": 2.1222358896254305, + "language_loss": 0.77194953, + "learning_rate": 3.754554828775562e-06, + "loss": 0.79308462, + "num_input_tokens_seen": 182224455, + "router_z_loss_clip": 0.53125, + "router_z_loss_mlp": 0.16802979, + "step": 6361, + "time_per_iteration": 2.6964499950408936 + }, + { + "auxiliary_loss_clip": 0.01017061, + "auxiliary_loss_mlp": 0.01012055, + "balance_loss_clip": 1.00938594, + "balance_loss_mlp": 1.01125681, + "epoch": 0.18460913469908885, + "flos": 74781022625280.0, + "grad_norm": 0.6554885458170534, + "language_loss": 0.4708603, + "learning_rate": 3.7544646024547863e-06, + "loss": 0.49115148, + "num_input_tokens_seen": 182291470, + "router_z_loss_clip": 0.07714844, + "router_z_loss_mlp": 0.00799561, + "step": 6362, + "time_per_iteration": 3.184943437576294 + }, + { + "auxiliary_loss_clip": 0.01074849, + "auxiliary_loss_mlp": 0.01031549, + "balance_loss_clip": 1.0264492, + "balance_loss_mlp": 1.01690388, + "epoch": 0.1846381521676049, + "flos": 20807536089600.0, + "grad_norm": 1.8680321465096945, + "language_loss": 0.79497313, + "learning_rate": 3.7543743606378698e-06, + "loss": 0.81603718, + "num_input_tokens_seen": 182306055, + "router_z_loss_clip": 0.48388672, + "router_z_loss_mlp": 0.14642334, + "step": 6363, + "time_per_iteration": 2.403458833694458 + }, + { + "auxiliary_loss_clip": 0.01016409, + "auxiliary_loss_mlp": 0.01002982, + "balance_loss_clip": 1.00879753, + "balance_loss_mlp": 1.00213575, + "epoch": 0.18466716963612095, + "flos": 64732670887680.0, + "grad_norm": 0.6156021994082598, + "language_loss": 0.46118182, + "learning_rate": 3.7542841033256086e-06, + "loss": 0.48137575, + "num_input_tokens_seen": 182365425, + "router_z_loss_clip": 0.07617188, + "router_z_loss_mlp": 0.00848389, + "step": 6364, + "time_per_iteration": 2.987417697906494 + }, + { + "auxiliary_loss_clip": 0.01078412, + "auxiliary_loss_mlp": 0.01046151, + "balance_loss_clip": 1.02693665, + "balance_loss_mlp": 1.03032649, + "epoch": 0.184696187104637, + "flos": 13186799118720.0, + "grad_norm": 2.543233491872337, + "language_loss": 0.81718934, + "learning_rate": 3.7541938305188007e-06, + "loss": 0.83843505, + "num_input_tokens_seen": 182379450, + "router_z_loss_clip": 0.51489258, + "router_z_loss_mlp": 0.15826416, + "step": 6365, + "time_per_iteration": 2.518892765045166 + }, + { + "auxiliary_loss_clip": 0.01080948, + "auxiliary_loss_mlp": 0.01052448, + "balance_loss_clip": 1.02862501, + "balance_loss_mlp": 1.03549647, + "epoch": 0.18472520457315303, + "flos": 24351883470720.0, + "grad_norm": 2.296374846440954, + "language_loss": 0.71047795, + "learning_rate": 3.7541035422182424e-06, + "loss": 0.73181188, + "num_input_tokens_seen": 182396590, + "router_z_loss_clip": 0.52319336, + "router_z_loss_mlp": 0.16955566, + "step": 6366, + "time_per_iteration": 2.467895269393921 + }, + { + "auxiliary_loss_clip": 0.01079832, + "auxiliary_loss_mlp": 0.01050339, + "balance_loss_clip": 1.02910089, + "balance_loss_mlp": 1.0348295, + "epoch": 0.18475422204166908, + "flos": 29344289068800.0, + "grad_norm": 2.2438787081419806, + "language_loss": 0.97709405, + "learning_rate": 3.7540132384247323e-06, + "loss": 0.9983958, + "num_input_tokens_seen": 182413650, + "router_z_loss_clip": 0.5078125, + "router_z_loss_mlp": 0.15484619, + "step": 6367, + "time_per_iteration": 2.455747604370117 + }, + { + "auxiliary_loss_clip": 0.01016923, + "auxiliary_loss_mlp": 0.01031918, + "balance_loss_clip": 1.00931001, + "balance_loss_mlp": 1.03107119, + "epoch": 0.18478323951018513, + "flos": 67885990508160.0, + "grad_norm": 0.614522872090012, + "language_loss": 0.44821489, + "learning_rate": 3.753922919139067e-06, + "loss": 0.4687033, + "num_input_tokens_seen": 182481655, + "router_z_loss_clip": 0.07617188, + "router_z_loss_mlp": 0.00848389, + "step": 6368, + "time_per_iteration": 3.0842466354370117 + }, + { + "auxiliary_loss_clip": 0.01086595, + "auxiliary_loss_mlp": 0.01049864, + "balance_loss_clip": 1.03116047, + "balance_loss_mlp": 1.03210831, + "epoch": 0.18481225697870118, + "flos": 74729039610240.0, + "grad_norm": 2.6731070573316957, + "language_loss": 0.70576936, + "learning_rate": 3.7538325843620456e-06, + "loss": 0.72713399, + "num_input_tokens_seen": 182502055, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.17749023, + "step": 6369, + "time_per_iteration": 2.79563570022583 + }, + { + "auxiliary_loss_clip": 0.01019014, + "auxiliary_loss_mlp": 0.01047324, + "balance_loss_clip": 1.01111448, + "balance_loss_mlp": 1.04633427, + "epoch": 0.18484127444721724, + "flos": 70704355610880.0, + "grad_norm": 0.7554471765210825, + "language_loss": 0.46446764, + "learning_rate": 3.7537422340944643e-06, + "loss": 0.48513103, + "num_input_tokens_seen": 182554665, + "router_z_loss_clip": 0.07910156, + "router_z_loss_mlp": 0.0098877, + "step": 6370, + "time_per_iteration": 2.926539421081543 + }, + { + "auxiliary_loss_clip": 0.01091861, + "auxiliary_loss_mlp": 0.01048148, + "balance_loss_clip": 1.03436446, + "balance_loss_mlp": 1.03146458, + "epoch": 0.18487029191573326, + "flos": 16545967315200.0, + "grad_norm": 2.0052458438018275, + "language_loss": 0.6870296, + "learning_rate": 3.7536518683371226e-06, + "loss": 0.70842969, + "num_input_tokens_seen": 182572315, + "router_z_loss_clip": 0.57568359, + "router_z_loss_mlp": 0.16674805, + "step": 6371, + "time_per_iteration": 2.5218725204467773 + }, + { + "auxiliary_loss_clip": 0.01086002, + "auxiliary_loss_mlp": 0.01050717, + "balance_loss_clip": 1.03289461, + "balance_loss_mlp": 1.03440928, + "epoch": 0.1848993093842493, + "flos": 23799313382400.0, + "grad_norm": 2.011666353696283, + "language_loss": 0.70256865, + "learning_rate": 3.7535614870908177e-06, + "loss": 0.72393584, + "num_input_tokens_seen": 182593320, + "router_z_loss_clip": 0.53100586, + "router_z_loss_mlp": 0.16308594, + "step": 6372, + "time_per_iteration": 2.5359320640563965 + }, + { + "auxiliary_loss_clip": 0.0109061, + "auxiliary_loss_mlp": 0.01042597, + "balance_loss_clip": 1.03747857, + "balance_loss_mlp": 1.02645063, + "epoch": 0.18492832685276536, + "flos": 41128383559680.0, + "grad_norm": 2.0332046966221142, + "language_loss": 0.80804908, + "learning_rate": 3.753471090356348e-06, + "loss": 0.82938111, + "num_input_tokens_seen": 182613910, + "router_z_loss_clip": 0.53125, + "router_z_loss_mlp": 0.16143799, + "step": 6373, + "time_per_iteration": 2.574956178665161 + }, + { + "auxiliary_loss_clip": 0.01095535, + "auxiliary_loss_mlp": 0.01044813, + "balance_loss_clip": 1.03919625, + "balance_loss_mlp": 1.02558446, + "epoch": 0.18495734432128141, + "flos": 34525085696640.0, + "grad_norm": 2.038056758775125, + "language_loss": 0.77989304, + "learning_rate": 3.753380678134512e-06, + "loss": 0.80129647, + "num_input_tokens_seen": 182631700, + "router_z_loss_clip": 0.5637207, + "router_z_loss_mlp": 0.19226074, + "step": 6374, + "time_per_iteration": 2.5065295696258545 + }, + { + "auxiliary_loss_clip": 0.01089702, + "auxiliary_loss_mlp": 0.0103156, + "balance_loss_clip": 1.03833413, + "balance_loss_mlp": 1.01708186, + "epoch": 0.18498636178979747, + "flos": 26351220055680.0, + "grad_norm": 2.9657943177149853, + "language_loss": 0.78490728, + "learning_rate": 3.753290250426109e-06, + "loss": 0.80611992, + "num_input_tokens_seen": 182647175, + "router_z_loss_clip": 0.51416016, + "router_z_loss_mlp": 0.14483643, + "step": 6375, + "time_per_iteration": 2.4975576400756836 + }, + { + "auxiliary_loss_clip": 0.01103237, + "auxiliary_loss_mlp": 0.01026282, + "balance_loss_clip": 1.04437733, + "balance_loss_mlp": 1.00884175, + "epoch": 0.18501537925831352, + "flos": 21789468478080.0, + "grad_norm": 1.9817625187253805, + "language_loss": 0.72276652, + "learning_rate": 3.753199807231936e-06, + "loss": 0.74406171, + "num_input_tokens_seen": 182661845, + "router_z_loss_clip": 0.58813477, + "router_z_loss_mlp": 0.17437744, + "step": 6376, + "time_per_iteration": 2.421448230743408 + }, + { + "auxiliary_loss_clip": 0.0109754, + "auxiliary_loss_mlp": 0.01029411, + "balance_loss_clip": 1.04378915, + "balance_loss_mlp": 1.01539755, + "epoch": 0.18504439672682954, + "flos": 45615944764800.0, + "grad_norm": 2.4847223533636713, + "language_loss": 0.762541, + "learning_rate": 3.7531093485527938e-06, + "loss": 0.7838105, + "num_input_tokens_seen": 182679175, + "router_z_loss_clip": 0.53759766, + "router_z_loss_mlp": 0.1401062, + "step": 6377, + "time_per_iteration": 2.728379249572754 + }, + { + "auxiliary_loss_clip": 0.01032163, + "auxiliary_loss_mlp": 0.0100526, + "balance_loss_clip": 1.02180338, + "balance_loss_mlp": 1.00424123, + "epoch": 0.1850734141953456, + "flos": 63491089651200.0, + "grad_norm": 0.7117107790775742, + "language_loss": 0.48969412, + "learning_rate": 3.7530188743894797e-06, + "loss": 0.5100683, + "num_input_tokens_seen": 182736030, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01019287, + "step": 6378, + "time_per_iteration": 2.8730413913726807 + }, + { + "auxiliary_loss_clip": 0.0109356, + "auxiliary_loss_mlp": 0.01032669, + "balance_loss_clip": 1.04134631, + "balance_loss_mlp": 1.01801276, + "epoch": 0.18510243166386164, + "flos": 44811685618560.0, + "grad_norm": 1.9457873050712318, + "language_loss": 0.79196739, + "learning_rate": 3.752928384742794e-06, + "loss": 0.81322968, + "num_input_tokens_seen": 182754860, + "router_z_loss_clip": 0.52197266, + "router_z_loss_mlp": 0.14666748, + "step": 6379, + "time_per_iteration": 2.5459020137786865 + }, + { + "auxiliary_loss_clip": 0.01096248, + "auxiliary_loss_mlp": 0.01033654, + "balance_loss_clip": 1.04325068, + "balance_loss_mlp": 1.01815724, + "epoch": 0.1851314491323777, + "flos": 74731273937280.0, + "grad_norm": 2.414286328252297, + "language_loss": 0.66565549, + "learning_rate": 3.7528378796135354e-06, + "loss": 0.6869545, + "num_input_tokens_seen": 182779185, + "router_z_loss_clip": 0.53027344, + "router_z_loss_mlp": 0.1550293, + "step": 6380, + "time_per_iteration": 2.7955589294433594 + }, + { + "auxiliary_loss_clip": 0.0109662, + "auxiliary_loss_mlp": 0.01046224, + "balance_loss_clip": 1.04251277, + "balance_loss_mlp": 1.03035116, + "epoch": 0.18516046660089375, + "flos": 10955989019520.0, + "grad_norm": 3.0290405645729033, + "language_loss": 0.76080668, + "learning_rate": 3.7527473590025034e-06, + "loss": 0.78223515, + "num_input_tokens_seen": 182791460, + "router_z_loss_clip": 0.54174805, + "router_z_loss_mlp": 0.15863037, + "step": 6381, + "time_per_iteration": 2.4116458892822266 + }, + { + "auxiliary_loss_clip": 0.01096454, + "auxiliary_loss_mlp": 0.01049591, + "balance_loss_clip": 1.03942394, + "balance_loss_mlp": 1.03104758, + "epoch": 0.18518948406940977, + "flos": 12012425982720.0, + "grad_norm": 2.5034294638865453, + "language_loss": 0.82255495, + "learning_rate": 3.752656822910497e-06, + "loss": 0.84401536, + "num_input_tokens_seen": 182802510, + "router_z_loss_clip": 0.56982422, + "router_z_loss_mlp": 0.18554688, + "step": 6382, + "time_per_iteration": 2.3614494800567627 + }, + { + "auxiliary_loss_clip": 0.01091368, + "auxiliary_loss_mlp": 0.01050616, + "balance_loss_clip": 1.03883326, + "balance_loss_mlp": 1.03381371, + "epoch": 0.18521850153792582, + "flos": 50396811235200.0, + "grad_norm": 15.576265910794506, + "language_loss": 0.94155335, + "learning_rate": 3.752566271338317e-06, + "loss": 0.96297312, + "num_input_tokens_seen": 182825130, + "router_z_loss_clip": 0.52490234, + "router_z_loss_mlp": 0.16815186, + "step": 6383, + "time_per_iteration": 2.7034759521484375 + }, + { + "auxiliary_loss_clip": 0.01092955, + "auxiliary_loss_mlp": 0.01063275, + "balance_loss_clip": 1.03957808, + "balance_loss_mlp": 1.04702687, + "epoch": 0.18524751900644187, + "flos": 13837301170560.0, + "grad_norm": 2.4060844835476276, + "language_loss": 0.75727379, + "learning_rate": 3.7524757042867618e-06, + "loss": 0.77883613, + "num_input_tokens_seen": 182839605, + "router_z_loss_clip": 0.53417969, + "router_z_loss_mlp": 0.16259766, + "step": 6384, + "time_per_iteration": 2.358922004699707 + }, + { + "auxiliary_loss_clip": 0.01088485, + "auxiliary_loss_mlp": 0.01046986, + "balance_loss_clip": 1.03449082, + "balance_loss_mlp": 1.03002834, + "epoch": 0.18527653647495793, + "flos": 12485324615040.0, + "grad_norm": 2.8509624890263936, + "language_loss": 0.77029181, + "learning_rate": 3.7523851217566325e-06, + "loss": 0.79164654, + "num_input_tokens_seen": 182850825, + "router_z_loss_clip": 0.53930664, + "router_z_loss_mlp": 0.16955566, + "step": 6385, + "time_per_iteration": 2.4224908351898193 + }, + { + "auxiliary_loss_clip": 0.01082883, + "auxiliary_loss_mlp": 0.01053862, + "balance_loss_clip": 1.03548479, + "balance_loss_mlp": 1.03961658, + "epoch": 0.18530555394347398, + "flos": 10298783986560.0, + "grad_norm": 2.070187233444222, + "language_loss": 0.66700822, + "learning_rate": 3.7522945237487286e-06, + "loss": 0.68837565, + "num_input_tokens_seen": 182861690, + "router_z_loss_clip": 0.47460938, + "router_z_loss_mlp": 0.14245605, + "step": 6386, + "time_per_iteration": 2.3335695266723633 + }, + { + "auxiliary_loss_clip": 0.01086167, + "auxiliary_loss_mlp": 0.0106308, + "balance_loss_clip": 1.03335583, + "balance_loss_mlp": 1.04624796, + "epoch": 0.18533457141199003, + "flos": 16718159473920.0, + "grad_norm": 2.888205055248862, + "language_loss": 0.830953, + "learning_rate": 3.7522039102638506e-06, + "loss": 0.85244548, + "num_input_tokens_seen": 182873830, + "router_z_loss_clip": 0.52783203, + "router_z_loss_mlp": 0.16827393, + "step": 6387, + "time_per_iteration": 2.369011640548706 + }, + { + "auxiliary_loss_clip": 0.01095591, + "auxiliary_loss_mlp": 0.01068193, + "balance_loss_clip": 1.03627968, + "balance_loss_mlp": 1.04979932, + "epoch": 0.18536358888050605, + "flos": 20040564142080.0, + "grad_norm": 2.3348853797594415, + "language_loss": 1.02180123, + "learning_rate": 3.7521132813027984e-06, + "loss": 1.04343915, + "num_input_tokens_seen": 182890515, + "router_z_loss_clip": 0.59326172, + "router_z_loss_mlp": 0.18383789, + "step": 6388, + "time_per_iteration": 2.3899331092834473 + }, + { + "auxiliary_loss_clip": 0.01081222, + "auxiliary_loss_mlp": 0.01054802, + "balance_loss_clip": 1.03025651, + "balance_loss_mlp": 1.03913796, + "epoch": 0.1853926063490221, + "flos": 74734276314240.0, + "grad_norm": 1.8545012676526023, + "language_loss": 0.83169663, + "learning_rate": 3.752022636866372e-06, + "loss": 0.85305679, + "num_input_tokens_seen": 182914930, + "router_z_loss_clip": 0.50976562, + "router_z_loss_mlp": 0.15673828, + "step": 6389, + "time_per_iteration": 2.885655403137207 + }, + { + "auxiliary_loss_clip": 0.0108108, + "auxiliary_loss_mlp": 0.01059348, + "balance_loss_clip": 1.02882838, + "balance_loss_mlp": 1.04355252, + "epoch": 0.18542162381753816, + "flos": 41383738310400.0, + "grad_norm": 1.8796583319496247, + "language_loss": 0.73998618, + "learning_rate": 3.7519319769553735e-06, + "loss": 0.76139045, + "num_input_tokens_seen": 182935525, + "router_z_loss_clip": 0.5222168, + "router_z_loss_mlp": 0.15795898, + "step": 6390, + "time_per_iteration": 2.56709885597229 + }, + { + "auxiliary_loss_clip": 0.01016898, + "auxiliary_loss_mlp": 0.01044957, + "balance_loss_clip": 1.00843167, + "balance_loss_mlp": 1.04393756, + "epoch": 0.1854506412860542, + "flos": 64012447117440.0, + "grad_norm": 0.7042448235705114, + "language_loss": 0.47587729, + "learning_rate": 3.751841301570603e-06, + "loss": 0.49649581, + "num_input_tokens_seen": 182991195, + "router_z_loss_clip": 0.08496094, + "router_z_loss_mlp": 0.01019287, + "step": 6391, + "time_per_iteration": 7.528404951095581 + }, + { + "auxiliary_loss_clip": 0.01078926, + "auxiliary_loss_mlp": 0.01047897, + "balance_loss_clip": 1.02744269, + "balance_loss_mlp": 1.03136301, + "epoch": 0.18547965875457026, + "flos": 29271320593920.0, + "grad_norm": 2.111296329243705, + "language_loss": 0.76306498, + "learning_rate": 3.751750610712861e-06, + "loss": 0.78433323, + "num_input_tokens_seen": 183007285, + "router_z_loss_clip": 0.51489258, + "router_z_loss_mlp": 0.16540527, + "step": 6392, + "time_per_iteration": 2.4456570148468018 + }, + { + "auxiliary_loss_clip": 0.01082205, + "auxiliary_loss_mlp": 0.01040182, + "balance_loss_clip": 1.02877259, + "balance_loss_mlp": 1.02339101, + "epoch": 0.1855086762230863, + "flos": 16099777739520.0, + "grad_norm": 2.2594945156397257, + "language_loss": 0.78935027, + "learning_rate": 3.7516599043829485e-06, + "loss": 0.81057411, + "num_input_tokens_seen": 183023425, + "router_z_loss_clip": 0.53417969, + "router_z_loss_mlp": 0.16796875, + "step": 6393, + "time_per_iteration": 2.3720028400421143 + }, + { + "auxiliary_loss_clip": 0.01085434, + "auxiliary_loss_mlp": 0.01034762, + "balance_loss_clip": 1.03166842, + "balance_loss_mlp": 1.01964045, + "epoch": 0.18553769369160233, + "flos": 13434927217920.0, + "grad_norm": 18.448447555579634, + "language_loss": 0.70855367, + "learning_rate": 3.751569182581667e-06, + "loss": 0.72975564, + "num_input_tokens_seen": 183033950, + "router_z_loss_clip": 0.53662109, + "router_z_loss_mlp": 0.15118408, + "step": 6394, + "time_per_iteration": 2.3214659690856934 + }, + { + "auxiliary_loss_clip": 0.01020336, + "auxiliary_loss_mlp": 0.01002503, + "balance_loss_clip": 1.01109803, + "balance_loss_mlp": 1.00140071, + "epoch": 0.18556671116011839, + "flos": 74763741502080.0, + "grad_norm": 0.6510030636758103, + "language_loss": 0.47078201, + "learning_rate": 3.751478445309818e-06, + "loss": 0.4910104, + "num_input_tokens_seen": 183094520, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01104736, + "step": 6395, + "time_per_iteration": 3.1890552043914795 + }, + { + "auxiliary_loss_clip": 0.0107981, + "auxiliary_loss_mlp": 0.01037102, + "balance_loss_clip": 1.02980638, + "balance_loss_mlp": 1.02184296, + "epoch": 0.18559572862863444, + "flos": 25876261653120.0, + "grad_norm": 3.8591406910902917, + "language_loss": 0.76357234, + "learning_rate": 3.751387692568202e-06, + "loss": 0.78474134, + "num_input_tokens_seen": 183108615, + "router_z_loss_clip": 0.50073242, + "router_z_loss_mlp": 0.15258789, + "step": 6396, + "time_per_iteration": 2.4265687465667725 + }, + { + "auxiliary_loss_clip": 0.01023554, + "auxiliary_loss_mlp": 0.01017658, + "balance_loss_clip": 1.01413548, + "balance_loss_mlp": 1.01642454, + "epoch": 0.1856247460971505, + "flos": 66163341381120.0, + "grad_norm": 0.6509584990552675, + "language_loss": 0.45549434, + "learning_rate": 3.7512969243576222e-06, + "loss": 0.47590649, + "num_input_tokens_seen": 183171230, + "router_z_loss_clip": 0.09423828, + "router_z_loss_mlp": 0.0123291, + "step": 6397, + "time_per_iteration": 3.0234711170196533 + }, + { + "auxiliary_loss_clip": 0.01022996, + "auxiliary_loss_mlp": 0.01008943, + "balance_loss_clip": 1.01335728, + "balance_loss_mlp": 1.00779235, + "epoch": 0.18565376356566654, + "flos": 63568596602880.0, + "grad_norm": 0.6507826391427596, + "language_loss": 0.48685122, + "learning_rate": 3.7512061406788783e-06, + "loss": 0.50717056, + "num_input_tokens_seen": 183228935, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.01147461, + "step": 6398, + "time_per_iteration": 2.9134600162506104 + }, + { + "auxiliary_loss_clip": 0.01021432, + "auxiliary_loss_mlp": 0.01012658, + "balance_loss_clip": 1.01156425, + "balance_loss_mlp": 1.01148367, + "epoch": 0.18568278103418256, + "flos": 57433728718080.0, + "grad_norm": 0.6729172952356026, + "language_loss": 0.48388332, + "learning_rate": 3.751115341532774e-06, + "loss": 0.50422424, + "num_input_tokens_seen": 183283170, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01171875, + "step": 6399, + "time_per_iteration": 2.8159985542297363 + }, + { + "auxiliary_loss_clip": 0.01086419, + "auxiliary_loss_mlp": 0.01036057, + "balance_loss_clip": 1.03215528, + "balance_loss_mlp": 1.01955831, + "epoch": 0.18571179850269862, + "flos": 16062071604480.0, + "grad_norm": 3.974424753325305, + "language_loss": 0.7181735, + "learning_rate": 3.7510245269201094e-06, + "loss": 0.73939824, + "num_input_tokens_seen": 183296055, + "router_z_loss_clip": 0.54296875, + "router_z_loss_mlp": 0.1651001, + "step": 6400, + "time_per_iteration": 4.860055446624756 + }, + { + "auxiliary_loss_clip": 0.01081998, + "auxiliary_loss_mlp": 0.01037124, + "balance_loss_clip": 1.03092098, + "balance_loss_mlp": 1.02218127, + "epoch": 0.18574081597121467, + "flos": 20110809530880.0, + "grad_norm": 2.6406567507193426, + "language_loss": 0.74323452, + "learning_rate": 3.750933696841688e-06, + "loss": 0.76442575, + "num_input_tokens_seen": 183308225, + "router_z_loss_clip": 0.51049805, + "router_z_loss_mlp": 0.14935303, + "step": 6401, + "time_per_iteration": 4.8768110275268555 + }, + { + "auxiliary_loss_clip": 0.01085076, + "auxiliary_loss_mlp": 0.01032821, + "balance_loss_clip": 1.03197253, + "balance_loss_mlp": 1.01748502, + "epoch": 0.18576983343973072, + "flos": 30477569667840.0, + "grad_norm": 2.600792694199729, + "language_loss": 0.90022457, + "learning_rate": 3.750842851298312e-06, + "loss": 0.92140347, + "num_input_tokens_seen": 183323355, + "router_z_loss_clip": 0.5300293, + "router_z_loss_mlp": 0.15338135, + "step": 6402, + "time_per_iteration": 2.4865500926971436 + }, + { + "auxiliary_loss_clip": 0.01083371, + "auxiliary_loss_mlp": 0.0103328, + "balance_loss_clip": 1.02988219, + "balance_loss_mlp": 1.01780641, + "epoch": 0.18579885090824677, + "flos": 25148147915520.0, + "grad_norm": 2.1703458505264686, + "language_loss": 0.86591911, + "learning_rate": 3.7507519902907833e-06, + "loss": 0.88708556, + "num_input_tokens_seen": 183338615, + "router_z_loss_clip": 0.53466797, + "router_z_loss_mlp": 0.15484619, + "step": 6403, + "time_per_iteration": 2.429664373397827 + }, + { + "auxiliary_loss_clip": 0.01083134, + "auxiliary_loss_mlp": 0.01037071, + "balance_loss_clip": 1.02956653, + "balance_loss_mlp": 1.01985109, + "epoch": 0.18582786837676282, + "flos": 11538061073280.0, + "grad_norm": 3.7205632284884165, + "language_loss": 0.92659181, + "learning_rate": 3.7506611138199044e-06, + "loss": 0.94779384, + "num_input_tokens_seen": 183350115, + "router_z_loss_clip": 0.53613281, + "router_z_loss_mlp": 0.17236328, + "step": 6404, + "time_per_iteration": 2.3452236652374268 + }, + { + "auxiliary_loss_clip": 0.01080952, + "auxiliary_loss_mlp": 0.01036209, + "balance_loss_clip": 1.02806926, + "balance_loss_mlp": 1.01940095, + "epoch": 0.18585688584527885, + "flos": 12415358517120.0, + "grad_norm": 2.859802107533058, + "language_loss": 1.07764089, + "learning_rate": 3.7505702218864784e-06, + "loss": 1.09881258, + "num_input_tokens_seen": 183361080, + "router_z_loss_clip": 0.52880859, + "router_z_loss_mlp": 0.16809082, + "step": 6405, + "time_per_iteration": 2.4198365211486816 + }, + { + "auxiliary_loss_clip": 0.01082879, + "auxiliary_loss_mlp": 0.01036804, + "balance_loss_clip": 1.0300808, + "balance_loss_mlp": 1.01939964, + "epoch": 0.1858859033137949, + "flos": 35036772716160.0, + "grad_norm": 2.9362576549057486, + "language_loss": 0.96344519, + "learning_rate": 3.750479314491308e-06, + "loss": 0.98464203, + "num_input_tokens_seen": 183384035, + "router_z_loss_clip": 0.52783203, + "router_z_loss_mlp": 0.17407227, + "step": 6406, + "time_per_iteration": 2.5237574577331543 + }, + { + "auxiliary_loss_clip": 0.01021387, + "auxiliary_loss_mlp": 0.01022408, + "balance_loss_clip": 1.01137686, + "balance_loss_mlp": 1.0212338, + "epoch": 0.18591492078231095, + "flos": 66517710347520.0, + "grad_norm": 0.621102856440102, + "language_loss": 0.43884805, + "learning_rate": 3.750388391635195e-06, + "loss": 0.459286, + "num_input_tokens_seen": 183441240, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.01171875, + "step": 6407, + "time_per_iteration": 3.0037147998809814 + }, + { + "auxiliary_loss_clip": 0.01079631, + "auxiliary_loss_mlp": 0.01030567, + "balance_loss_clip": 1.02686405, + "balance_loss_mlp": 1.01388395, + "epoch": 0.185943938250827, + "flos": 11788912258560.0, + "grad_norm": 3.7572373401237864, + "language_loss": 0.94721907, + "learning_rate": 3.750297453318944e-06, + "loss": 0.96832108, + "num_input_tokens_seen": 183449125, + "router_z_loss_clip": 0.52832031, + "router_z_loss_mlp": 0.16668701, + "step": 6408, + "time_per_iteration": 2.3694846630096436 + }, + { + "auxiliary_loss_clip": 0.01079607, + "auxiliary_loss_mlp": 0.0103259, + "balance_loss_clip": 1.02690709, + "balance_loss_mlp": 1.01691365, + "epoch": 0.18597295571934305, + "flos": 16355202312960.0, + "grad_norm": 2.655751926822385, + "language_loss": 0.84034479, + "learning_rate": 3.750206499543358e-06, + "loss": 0.86146671, + "num_input_tokens_seen": 183461685, + "router_z_loss_clip": 0.52709961, + "router_z_loss_mlp": 0.15686035, + "step": 6409, + "time_per_iteration": 2.362541913986206 + }, + { + "auxiliary_loss_clip": 0.01017201, + "auxiliary_loss_mlp": 0.01014538, + "balance_loss_clip": 1.00652695, + "balance_loss_mlp": 1.01316679, + "epoch": 0.1860019731878591, + "flos": 62438667494400.0, + "grad_norm": 0.6456848260402497, + "language_loss": 0.5025028, + "learning_rate": 3.750115530309239e-06, + "loss": 0.52282023, + "num_input_tokens_seen": 183522720, + "router_z_loss_clip": 0.10693359, + "router_z_loss_mlp": 0.01373291, + "step": 6410, + "time_per_iteration": 2.9680335521698 + }, + { + "auxiliary_loss_clip": 0.01079748, + "auxiliary_loss_mlp": 0.01033697, + "balance_loss_clip": 1.02741241, + "balance_loss_mlp": 1.01811671, + "epoch": 0.18603099065637513, + "flos": 18216561738240.0, + "grad_norm": 2.744050647226742, + "language_loss": 0.92570364, + "learning_rate": 3.7500245456173927e-06, + "loss": 0.94683802, + "num_input_tokens_seen": 183534625, + "router_z_loss_clip": 0.52441406, + "router_z_loss_mlp": 0.15588379, + "step": 6411, + "time_per_iteration": 2.3852810859680176 + }, + { + "auxiliary_loss_clip": 0.0108244, + "auxiliary_loss_mlp": 0.01036205, + "balance_loss_clip": 1.02925587, + "balance_loss_mlp": 1.020082, + "epoch": 0.18606000812489118, + "flos": 14713795653120.0, + "grad_norm": 2.102611124910669, + "language_loss": 0.72935444, + "learning_rate": 3.749933545468621e-06, + "loss": 0.75054091, + "num_input_tokens_seen": 183547345, + "router_z_loss_clip": 0.53222656, + "router_z_loss_mlp": 0.16137695, + "step": 6412, + "time_per_iteration": 2.4820592403411865 + }, + { + "auxiliary_loss_clip": 0.01083779, + "auxiliary_loss_mlp": 0.01033906, + "balance_loss_clip": 1.02942204, + "balance_loss_mlp": 1.01715708, + "epoch": 0.18608902559340723, + "flos": 26861824823040.0, + "grad_norm": 2.9340827287378968, + "language_loss": 0.93056679, + "learning_rate": 3.7498425298637276e-06, + "loss": 0.95174366, + "num_input_tokens_seen": 183561495, + "router_z_loss_clip": 0.54370117, + "router_z_loss_mlp": 0.1673584, + "step": 6413, + "time_per_iteration": 2.433588743209839 + }, + { + "auxiliary_loss_clip": 0.01075605, + "auxiliary_loss_mlp": 0.01045451, + "balance_loss_clip": 1.02798855, + "balance_loss_mlp": 1.03075266, + "epoch": 0.18611804306192328, + "flos": 22631433759360.0, + "grad_norm": 2.011353012599798, + "language_loss": 0.69945711, + "learning_rate": 3.749751498803517e-06, + "loss": 0.72066772, + "num_input_tokens_seen": 183577220, + "router_z_loss_clip": 0.4765625, + "router_z_loss_mlp": 0.14697266, + "step": 6414, + "time_per_iteration": 2.3881235122680664 + }, + { + "auxiliary_loss_clip": 0.01084039, + "auxiliary_loss_mlp": 0.01042645, + "balance_loss_clip": 1.03085923, + "balance_loss_mlp": 1.02605128, + "epoch": 0.18614706053043933, + "flos": 28542857742720.0, + "grad_norm": 2.264952663516825, + "language_loss": 0.84077847, + "learning_rate": 3.7496604522887933e-06, + "loss": 0.86204529, + "num_input_tokens_seen": 183593415, + "router_z_loss_clip": 0.53222656, + "router_z_loss_mlp": 0.16595459, + "step": 6415, + "time_per_iteration": 2.4307851791381836 + }, + { + "auxiliary_loss_clip": 0.010856, + "auxiliary_loss_mlp": 0.0105121, + "balance_loss_clip": 1.02895331, + "balance_loss_mlp": 1.03391266, + "epoch": 0.18617607799895536, + "flos": 56926372573440.0, + "grad_norm": 1.9136097762202737, + "language_loss": 1.04700077, + "learning_rate": 3.7495693903203603e-06, + "loss": 1.06836891, + "num_input_tokens_seen": 183618400, + "router_z_loss_clip": 0.56640625, + "router_z_loss_mlp": 0.17297363, + "step": 6416, + "time_per_iteration": 2.750803232192993 + }, + { + "auxiliary_loss_clip": 0.0108344, + "auxiliary_loss_mlp": 0.01042198, + "balance_loss_clip": 1.02931488, + "balance_loss_mlp": 1.02552056, + "epoch": 0.1862050954674714, + "flos": 36204058846080.0, + "grad_norm": 1.8125954463806213, + "language_loss": 0.79746222, + "learning_rate": 3.749478312899023e-06, + "loss": 0.81871867, + "num_input_tokens_seen": 183638995, + "router_z_loss_clip": 0.54125977, + "router_z_loss_mlp": 0.16687012, + "step": 6417, + "time_per_iteration": 2.5127367973327637 + }, + { + "auxiliary_loss_clip": 0.01077709, + "auxiliary_loss_mlp": 0.01045038, + "balance_loss_clip": 1.02750647, + "balance_loss_mlp": 1.02963018, + "epoch": 0.18623411293598746, + "flos": 56889015552000.0, + "grad_norm": 1.8216000121517324, + "language_loss": 0.66114748, + "learning_rate": 3.749387220025585e-06, + "loss": 0.68237495, + "num_input_tokens_seen": 183658755, + "router_z_loss_clip": 0.50195312, + "router_z_loss_mlp": 0.15411377, + "step": 6418, + "time_per_iteration": 2.774153232574463 + }, + { + "auxiliary_loss_clip": 0.01018544, + "auxiliary_loss_mlp": 0.01016343, + "balance_loss_clip": 1.00792265, + "balance_loss_mlp": 1.01488853, + "epoch": 0.1862631304045035, + "flos": 64887405500160.0, + "grad_norm": 0.663735312465332, + "language_loss": 0.45559004, + "learning_rate": 3.749296111700851e-06, + "loss": 0.47593889, + "num_input_tokens_seen": 183721035, + "router_z_loss_clip": 0.10644531, + "router_z_loss_mlp": 0.01452637, + "step": 6419, + "time_per_iteration": 2.960115432739258 + }, + { + "auxiliary_loss_clip": 0.01073468, + "auxiliary_loss_mlp": 0.01037563, + "balance_loss_clip": 1.02482843, + "balance_loss_mlp": 1.02157712, + "epoch": 0.18629214787301956, + "flos": 23433807692160.0, + "grad_norm": 2.210937834524224, + "language_loss": 0.68974507, + "learning_rate": 3.7492049879256258e-06, + "loss": 0.71085542, + "num_input_tokens_seen": 183738425, + "router_z_loss_clip": 0.48730469, + "router_z_loss_mlp": 0.15985107, + "step": 6420, + "time_per_iteration": 2.408524990081787 + }, + { + "auxiliary_loss_clip": 0.010835, + "auxiliary_loss_mlp": 0.01039897, + "balance_loss_clip": 1.02859521, + "balance_loss_mlp": 1.02354765, + "epoch": 0.18632116534153562, + "flos": 19710355703040.0, + "grad_norm": 2.4384837262374175, + "language_loss": 1.1134814, + "learning_rate": 3.7491138487007144e-06, + "loss": 1.13471544, + "num_input_tokens_seen": 183752490, + "router_z_loss_clip": 0.55029297, + "router_z_loss_mlp": 0.16357422, + "step": 6421, + "time_per_iteration": 2.4037082195281982 + }, + { + "auxiliary_loss_clip": 0.01078087, + "auxiliary_loss_mlp": 0.01032596, + "balance_loss_clip": 1.027704, + "balance_loss_mlp": 1.01652098, + "epoch": 0.18635018281005164, + "flos": 15699673025280.0, + "grad_norm": 1.9027310436707006, + "language_loss": 0.64591485, + "learning_rate": 3.749022694026922e-06, + "loss": 0.66702163, + "num_input_tokens_seen": 183767930, + "router_z_loss_clip": 0.50366211, + "router_z_loss_mlp": 0.16082764, + "step": 6422, + "time_per_iteration": 2.391514539718628 + }, + { + "auxiliary_loss_clip": 0.01076112, + "auxiliary_loss_mlp": 0.01033008, + "balance_loss_clip": 1.02694082, + "balance_loss_mlp": 1.01877999, + "epoch": 0.1863792002785677, + "flos": 17267028958080.0, + "grad_norm": 2.6890358430376726, + "language_loss": 0.7627393, + "learning_rate": 3.7489315239050533e-06, + "loss": 0.78383052, + "num_input_tokens_seen": 183780460, + "router_z_loss_clip": 0.49169922, + "router_z_loss_mlp": 0.14239502, + "step": 6423, + "time_per_iteration": 2.3447799682617188 + }, + { + "auxiliary_loss_clip": 0.01081393, + "auxiliary_loss_mlp": 0.01033491, + "balance_loss_clip": 1.02720118, + "balance_loss_mlp": 1.01529956, + "epoch": 0.18640821774708374, + "flos": 36647490424320.0, + "grad_norm": 2.1337322294564927, + "language_loss": 0.83571541, + "learning_rate": 3.7488403383359134e-06, + "loss": 0.85686421, + "num_input_tokens_seen": 183800805, + "router_z_loss_clip": 0.54174805, + "router_z_loss_mlp": 0.18200684, + "step": 6424, + "time_per_iteration": 2.5650908946990967 + }, + { + "auxiliary_loss_clip": 0.01083983, + "auxiliary_loss_mlp": 0.01032973, + "balance_loss_clip": 1.03017271, + "balance_loss_mlp": 1.0166235, + "epoch": 0.1864372352155998, + "flos": 10954906767360.0, + "grad_norm": 3.0972496678217305, + "language_loss": 0.72069293, + "learning_rate": 3.748749137320308e-06, + "loss": 0.74186248, + "num_input_tokens_seen": 183811185, + "router_z_loss_clip": 0.53759766, + "router_z_loss_mlp": 0.16363525, + "step": 6425, + "time_per_iteration": 2.3316569328308105 + }, + { + "auxiliary_loss_clip": 0.01086388, + "auxiliary_loss_mlp": 0.0103649, + "balance_loss_clip": 1.03203678, + "balance_loss_mlp": 1.01838255, + "epoch": 0.18646625268411585, + "flos": 18362882712960.0, + "grad_norm": 5.269095583161515, + "language_loss": 0.74888134, + "learning_rate": 3.7486579208590426e-06, + "loss": 0.77011013, + "num_input_tokens_seen": 183825655, + "router_z_loss_clip": 0.54418945, + "router_z_loss_mlp": 0.18096924, + "step": 6426, + "time_per_iteration": 2.4117679595947266 + }, + { + "auxiliary_loss_clip": 0.01080658, + "auxiliary_loss_mlp": 0.010346, + "balance_loss_clip": 1.03161192, + "balance_loss_mlp": 1.0181973, + "epoch": 0.1864952701526319, + "flos": 33611513483520.0, + "grad_norm": 5.023036177984115, + "language_loss": 0.82031792, + "learning_rate": 3.7485666889529234e-06, + "loss": 0.84147054, + "num_input_tokens_seen": 183841125, + "router_z_loss_clip": 0.49121094, + "router_z_loss_mlp": 0.1640625, + "step": 6427, + "time_per_iteration": 2.4998157024383545 + }, + { + "auxiliary_loss_clip": 0.01088436, + "auxiliary_loss_mlp": 0.0103149, + "balance_loss_clip": 1.0326494, + "balance_loss_mlp": 1.01511073, + "epoch": 0.18652428762114792, + "flos": 15700126872960.0, + "grad_norm": 2.3802833775017502, + "language_loss": 0.75657028, + "learning_rate": 3.748475441602755e-06, + "loss": 0.77776957, + "num_input_tokens_seen": 183854640, + "router_z_loss_clip": 0.55786133, + "router_z_loss_mlp": 0.16369629, + "step": 6428, + "time_per_iteration": 2.372896909713745 + }, + { + "auxiliary_loss_clip": 0.01030461, + "auxiliary_loss_mlp": 0.01003273, + "balance_loss_clip": 1.0185076, + "balance_loss_mlp": 1.00202131, + "epoch": 0.18655330508966397, + "flos": 64266230856960.0, + "grad_norm": 1.2674216740593112, + "language_loss": 0.53032339, + "learning_rate": 3.7483841788093438e-06, + "loss": 0.55066073, + "num_input_tokens_seen": 183916125, + "router_z_loss_clip": 0.11962891, + "router_z_loss_mlp": 0.01251221, + "step": 6429, + "time_per_iteration": 2.967665433883667 + }, + { + "auxiliary_loss_clip": 0.01029633, + "auxiliary_loss_mlp": 0.01002092, + "balance_loss_clip": 1.01797438, + "balance_loss_mlp": 1.00081038, + "epoch": 0.18658232255818002, + "flos": 58938938697600.0, + "grad_norm": 0.6664578786236816, + "language_loss": 0.45412433, + "learning_rate": 3.7482929005734966e-06, + "loss": 0.47444159, + "num_input_tokens_seen": 183973985, + "router_z_loss_clip": 0.11669922, + "router_z_loss_mlp": 0.01281738, + "step": 6430, + "time_per_iteration": 2.934443712234497 + }, + { + "auxiliary_loss_clip": 0.01079181, + "auxiliary_loss_mlp": 0.01030341, + "balance_loss_clip": 1.02901077, + "balance_loss_mlp": 1.0152247, + "epoch": 0.18661134002669608, + "flos": 16901802558720.0, + "grad_norm": 4.744178598149714, + "language_loss": 0.78560907, + "learning_rate": 3.748201606896019e-06, + "loss": 0.80670422, + "num_input_tokens_seen": 183985465, + "router_z_loss_clip": 0.50268555, + "router_z_loss_mlp": 0.15124512, + "step": 6431, + "time_per_iteration": 2.3131072521209717 + }, + { + "auxiliary_loss_clip": 0.01075802, + "auxiliary_loss_mlp": 0.01036703, + "balance_loss_clip": 1.0271492, + "balance_loss_mlp": 1.02238631, + "epoch": 0.18664035749521213, + "flos": 16355062667520.0, + "grad_norm": 2.742400851108282, + "language_loss": 0.78015101, + "learning_rate": 3.748110297777717e-06, + "loss": 0.80127609, + "num_input_tokens_seen": 183999345, + "router_z_loss_clip": 0.48730469, + "router_z_loss_mlp": 0.14312744, + "step": 6432, + "time_per_iteration": 2.361572027206421 + }, + { + "auxiliary_loss_clip": 0.01020868, + "auxiliary_loss_mlp": 0.01001675, + "balance_loss_clip": 1.0103085, + "balance_loss_mlp": 1.00048852, + "epoch": 0.18666937496372815, + "flos": 69985982142720.0, + "grad_norm": 0.6775924364947435, + "language_loss": 0.49845135, + "learning_rate": 3.7480189732193973e-06, + "loss": 0.51867676, + "num_input_tokens_seen": 184060220, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01184082, + "step": 6433, + "time_per_iteration": 3.00406813621521 + }, + { + "auxiliary_loss_clip": 0.01078251, + "auxiliary_loss_mlp": 0.01038999, + "balance_loss_clip": 1.027632, + "balance_loss_mlp": 1.02400255, + "epoch": 0.1866983924322442, + "flos": 13910583847680.0, + "grad_norm": 3.1265720885308816, + "language_loss": 0.60946786, + "learning_rate": 3.7479276332218675e-06, + "loss": 0.63064039, + "num_input_tokens_seen": 184073510, + "router_z_loss_clip": 0.50585938, + "router_z_loss_mlp": 0.15008545, + "step": 6434, + "time_per_iteration": 2.423872470855713 + }, + { + "auxiliary_loss_clip": 0.01076584, + "auxiliary_loss_mlp": 0.01036044, + "balance_loss_clip": 1.02748251, + "balance_loss_mlp": 1.02046967, + "epoch": 0.18672740990076026, + "flos": 11867291994240.0, + "grad_norm": 2.290165599424304, + "language_loss": 0.86241806, + "learning_rate": 3.747836277785933e-06, + "loss": 0.88354433, + "num_input_tokens_seen": 184086205, + "router_z_loss_clip": 0.49047852, + "router_z_loss_mlp": 0.15570068, + "step": 6435, + "time_per_iteration": 2.352351427078247 + }, + { + "auxiliary_loss_clip": 0.01015474, + "auxiliary_loss_mlp": 0.01004765, + "balance_loss_clip": 1.00584936, + "balance_loss_mlp": 1.00341213, + "epoch": 0.1867564273692763, + "flos": 57436766006400.0, + "grad_norm": 0.6394823280058292, + "language_loss": 0.47362578, + "learning_rate": 3.747744906912401e-06, + "loss": 0.49382818, + "num_input_tokens_seen": 184153225, + "router_z_loss_clip": 0.09619141, + "router_z_loss_mlp": 0.0135498, + "step": 6436, + "time_per_iteration": 3.0978662967681885 + }, + { + "auxiliary_loss_clip": 0.01082807, + "auxiliary_loss_mlp": 0.010364, + "balance_loss_clip": 1.02830362, + "balance_loss_mlp": 1.01832139, + "epoch": 0.18678544483779236, + "flos": 34890102627840.0, + "grad_norm": 2.2969531044349383, + "language_loss": 0.9624244, + "learning_rate": 3.747653520602079e-06, + "loss": 0.98361647, + "num_input_tokens_seen": 184170150, + "router_z_loss_clip": 0.54541016, + "router_z_loss_mlp": 0.18066406, + "step": 6437, + "time_per_iteration": 2.529200553894043 + }, + { + "auxiliary_loss_clip": 0.01076778, + "auxiliary_loss_mlp": 0.01035395, + "balance_loss_clip": 1.02492499, + "balance_loss_mlp": 1.01929617, + "epoch": 0.1868144623063084, + "flos": 24491187262080.0, + "grad_norm": 2.3971496098750746, + "language_loss": 0.92940527, + "learning_rate": 3.7475621188557743e-06, + "loss": 0.95052707, + "num_input_tokens_seen": 184185535, + "router_z_loss_clip": 0.51928711, + "router_z_loss_mlp": 0.16088867, + "step": 6438, + "time_per_iteration": 2.4740941524505615 + }, + { + "auxiliary_loss_clip": 0.01077527, + "auxiliary_loss_mlp": 0.01034717, + "balance_loss_clip": 1.0273006, + "balance_loss_mlp": 1.01902938, + "epoch": 0.18684347977482443, + "flos": 31714228402560.0, + "grad_norm": 2.146256793091762, + "language_loss": 0.85106707, + "learning_rate": 3.7474707016742933e-06, + "loss": 0.87218946, + "num_input_tokens_seen": 184201550, + "router_z_loss_clip": 0.50268555, + "router_z_loss_mlp": 0.15698242, + "step": 6439, + "time_per_iteration": 2.5048093795776367 + }, + { + "auxiliary_loss_clip": 0.01080183, + "auxiliary_loss_mlp": 0.01029959, + "balance_loss_clip": 1.02779102, + "balance_loss_mlp": 1.01424682, + "epoch": 0.18687249724334049, + "flos": 30491185098240.0, + "grad_norm": 2.300863411519839, + "language_loss": 0.92592126, + "learning_rate": 3.7473792690584444e-06, + "loss": 0.94702262, + "num_input_tokens_seen": 184219125, + "router_z_loss_clip": 0.52392578, + "router_z_loss_mlp": 0.15710449, + "step": 6440, + "time_per_iteration": 2.583167552947998 + }, + { + "auxiliary_loss_clip": 0.01077485, + "auxiliary_loss_mlp": 0.01027786, + "balance_loss_clip": 1.02762794, + "balance_loss_mlp": 1.0137794, + "epoch": 0.18690151471185654, + "flos": 23580268312320.0, + "grad_norm": 1.9893715079887244, + "language_loss": 0.67040479, + "learning_rate": 3.747287821009034e-06, + "loss": 0.69145751, + "num_input_tokens_seen": 184237010, + "router_z_loss_clip": 0.49926758, + "router_z_loss_mlp": 0.14007568, + "step": 6441, + "time_per_iteration": 2.4522910118103027 + }, + { + "auxiliary_loss_clip": 0.01083553, + "auxiliary_loss_mlp": 0.01034595, + "balance_loss_clip": 1.0299046, + "balance_loss_mlp": 1.01730943, + "epoch": 0.1869305321803726, + "flos": 32407987495680.0, + "grad_norm": 3.6515808880990273, + "language_loss": 0.93080485, + "learning_rate": 3.7471963575268707e-06, + "loss": 0.95198631, + "num_input_tokens_seen": 184254870, + "router_z_loss_clip": 0.53686523, + "router_z_loss_mlp": 0.17272949, + "step": 6442, + "time_per_iteration": 2.46673583984375 + }, + { + "auxiliary_loss_clip": 0.01082842, + "auxiliary_loss_mlp": 0.01028191, + "balance_loss_clip": 1.03091741, + "balance_loss_mlp": 1.01320696, + "epoch": 0.18695954964888864, + "flos": 12343472294400.0, + "grad_norm": 2.962352973776175, + "language_loss": 0.70648932, + "learning_rate": 3.747104878612763e-06, + "loss": 0.72759968, + "num_input_tokens_seen": 184268155, + "router_z_loss_clip": 0.51928711, + "router_z_loss_mlp": 0.14990234, + "step": 6443, + "time_per_iteration": 2.3515214920043945 + }, + { + "auxiliary_loss_clip": 0.01082937, + "auxiliary_loss_mlp": 0.01033029, + "balance_loss_clip": 1.03116691, + "balance_loss_mlp": 1.01722813, + "epoch": 0.18698856711740466, + "flos": 13478952309120.0, + "grad_norm": 2.7918454643572965, + "language_loss": 0.73601693, + "learning_rate": 3.7470133842675173e-06, + "loss": 0.75717664, + "num_input_tokens_seen": 184281635, + "router_z_loss_clip": 0.51782227, + "router_z_loss_mlp": 0.15814209, + "step": 6444, + "time_per_iteration": 2.436497926712036 + }, + { + "auxiliary_loss_clip": 0.01090059, + "auxiliary_loss_mlp": 0.01039981, + "balance_loss_clip": 1.03550863, + "balance_loss_mlp": 1.02375126, + "epoch": 0.18701758458592072, + "flos": 47264019494400.0, + "grad_norm": 2.0294444556758253, + "language_loss": 0.95478201, + "learning_rate": 3.7469218744919423e-06, + "loss": 0.97608238, + "num_input_tokens_seen": 184302495, + "router_z_loss_clip": 0.54516602, + "router_z_loss_mlp": 0.16223145, + "step": 6445, + "time_per_iteration": 2.658668041229248 + }, + { + "auxiliary_loss_clip": 0.01091407, + "auxiliary_loss_mlp": 0.01033789, + "balance_loss_clip": 1.03346539, + "balance_loss_mlp": 1.01528811, + "epoch": 0.18704660205443677, + "flos": 74734695250560.0, + "grad_norm": 1.798783402304065, + "language_loss": 0.76024425, + "learning_rate": 3.7468303492868466e-06, + "loss": 0.78149623, + "num_input_tokens_seen": 184338970, + "router_z_loss_clip": 0.57983398, + "router_z_loss_mlp": 0.18505859, + "step": 6446, + "time_per_iteration": 2.830397844314575 + }, + { + "auxiliary_loss_clip": 0.01090271, + "auxiliary_loss_mlp": 0.01033321, + "balance_loss_clip": 1.03602099, + "balance_loss_mlp": 1.01709616, + "epoch": 0.18707561952295282, + "flos": 32480921059200.0, + "grad_norm": 2.695397432512475, + "language_loss": 0.89529711, + "learning_rate": 3.746738808653038e-06, + "loss": 0.91653311, + "num_input_tokens_seen": 184356650, + "router_z_loss_clip": 0.54248047, + "router_z_loss_mlp": 0.16217041, + "step": 6447, + "time_per_iteration": 2.480006217956543 + }, + { + "auxiliary_loss_clip": 0.01085771, + "auxiliary_loss_mlp": 0.01038331, + "balance_loss_clip": 1.03126597, + "balance_loss_mlp": 1.02219021, + "epoch": 0.18710463699146887, + "flos": 16283071710720.0, + "grad_norm": 2.835917378173042, + "language_loss": 0.91355336, + "learning_rate": 3.7466472525913266e-06, + "loss": 0.93479443, + "num_input_tokens_seen": 184367955, + "router_z_loss_clip": 0.54492188, + "router_z_loss_mlp": 0.16137695, + "step": 6448, + "time_per_iteration": 2.409749746322632 + }, + { + "auxiliary_loss_clip": 0.01078907, + "auxiliary_loss_mlp": 0.01021933, + "balance_loss_clip": 1.02988791, + "balance_loss_mlp": 1.00663841, + "epoch": 0.18713365445998492, + "flos": 22447930320000.0, + "grad_norm": 3.5264898574330554, + "language_loss": 0.98619628, + "learning_rate": 3.746555681102519e-06, + "loss": 1.00720477, + "num_input_tokens_seen": 184383145, + "router_z_loss_clip": 0.48999023, + "router_z_loss_mlp": 0.1529541, + "step": 6449, + "time_per_iteration": 2.346132516860962 + }, + { + "auxiliary_loss_clip": 0.0108206, + "auxiliary_loss_mlp": 0.01036787, + "balance_loss_clip": 1.02961946, + "balance_loss_mlp": 1.01927495, + "epoch": 0.18716267192850095, + "flos": 21251596072320.0, + "grad_norm": 1.664288384977464, + "language_loss": 0.62361825, + "learning_rate": 3.7464640941874247e-06, + "loss": 0.64480674, + "num_input_tokens_seen": 184401385, + "router_z_loss_clip": 0.52368164, + "router_z_loss_mlp": 0.17529297, + "step": 6450, + "time_per_iteration": 2.4501869678497314 + }, + { + "auxiliary_loss_clip": 0.01086961, + "auxiliary_loss_mlp": 0.0103552, + "balance_loss_clip": 1.03081048, + "balance_loss_mlp": 1.01813936, + "epoch": 0.187191689397017, + "flos": 12413124190080.0, + "grad_norm": 2.2132905373312073, + "language_loss": 0.88750196, + "learning_rate": 3.746372491846853e-06, + "loss": 0.90872669, + "num_input_tokens_seen": 184413160, + "router_z_loss_clip": 0.56176758, + "router_z_loss_mlp": 0.17382812, + "step": 6451, + "time_per_iteration": 2.3382489681243896 + }, + { + "auxiliary_loss_clip": 0.0108281, + "auxiliary_loss_mlp": 0.01026865, + "balance_loss_clip": 1.03111601, + "balance_loss_mlp": 1.01182652, + "epoch": 0.18722070686553305, + "flos": 30585311723520.0, + "grad_norm": 2.0975694382805234, + "language_loss": 0.69531828, + "learning_rate": 3.746280874081613e-06, + "loss": 0.71641505, + "num_input_tokens_seen": 184431410, + "router_z_loss_clip": 0.51733398, + "router_z_loss_mlp": 0.1505127, + "step": 6452, + "time_per_iteration": 2.5015039443969727 + }, + { + "auxiliary_loss_clip": 0.01088669, + "auxiliary_loss_mlp": 0.01031774, + "balance_loss_clip": 1.03234518, + "balance_loss_mlp": 1.01482892, + "epoch": 0.1872497243340491, + "flos": 11830528465920.0, + "grad_norm": 2.075868545125124, + "language_loss": 0.6371066, + "learning_rate": 3.7461892408925137e-06, + "loss": 0.65831101, + "num_input_tokens_seen": 184445055, + "router_z_loss_clip": 0.56347656, + "router_z_loss_mlp": 0.1696167, + "step": 6453, + "time_per_iteration": 2.34557843208313 + }, + { + "auxiliary_loss_clip": 0.01021433, + "auxiliary_loss_mlp": 0.01005421, + "balance_loss_clip": 1.01142085, + "balance_loss_mlp": 1.0042944, + "epoch": 0.18727874180256515, + "flos": 74777601312000.0, + "grad_norm": 0.6110308436194228, + "language_loss": 0.45591226, + "learning_rate": 3.746097592280364e-06, + "loss": 0.47618082, + "num_input_tokens_seen": 184514825, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.0112915, + "step": 6454, + "time_per_iteration": 3.135251522064209 + }, + { + "auxiliary_loss_clip": 0.01087483, + "auxiliary_loss_mlp": 0.01035735, + "balance_loss_clip": 1.03167081, + "balance_loss_mlp": 1.01757944, + "epoch": 0.1873077592710812, + "flos": 27227190867840.0, + "grad_norm": 1.9779200624303124, + "language_loss": 0.85217643, + "learning_rate": 3.7460059282459743e-06, + "loss": 0.87340862, + "num_input_tokens_seen": 184533310, + "router_z_loss_clip": 0.55761719, + "router_z_loss_mlp": 0.18145752, + "step": 6455, + "time_per_iteration": 2.4562385082244873 + }, + { + "auxiliary_loss_clip": 0.01019839, + "auxiliary_loss_mlp": 0.01001841, + "balance_loss_clip": 1.01016104, + "balance_loss_mlp": 1.00057733, + "epoch": 0.18733677673959723, + "flos": 74773795973760.0, + "grad_norm": 1.1134457449699968, + "language_loss": 0.4892292, + "learning_rate": 3.745914248790153e-06, + "loss": 0.50944602, + "num_input_tokens_seen": 184593675, + "router_z_loss_clip": 0.09667969, + "router_z_loss_mlp": 0.01263428, + "step": 6456, + "time_per_iteration": 3.0459351539611816 + }, + { + "auxiliary_loss_clip": 0.01079825, + "auxiliary_loss_mlp": 0.01037752, + "balance_loss_clip": 1.02716541, + "balance_loss_mlp": 1.02192664, + "epoch": 0.18736579420811328, + "flos": 15993083024640.0, + "grad_norm": 2.059196947823712, + "language_loss": 0.63975227, + "learning_rate": 3.745822553913711e-06, + "loss": 0.66092801, + "num_input_tokens_seen": 184613025, + "router_z_loss_clip": 0.52709961, + "router_z_loss_mlp": 0.15838623, + "step": 6457, + "time_per_iteration": 2.539135694503784 + }, + { + "auxiliary_loss_clip": 0.01072488, + "auxiliary_loss_mlp": 0.01028973, + "balance_loss_clip": 1.02713633, + "balance_loss_mlp": 1.01624727, + "epoch": 0.18739481167662933, + "flos": 37333289727360.0, + "grad_norm": 1.8383779306675, + "language_loss": 0.69278449, + "learning_rate": 3.745730843617458e-06, + "loss": 0.71379912, + "num_input_tokens_seen": 184632220, + "router_z_loss_clip": 0.45336914, + "router_z_loss_mlp": 0.12731934, + "step": 6458, + "time_per_iteration": 2.5152769088745117 + }, + { + "auxiliary_loss_clip": 0.01014996, + "auxiliary_loss_mlp": 0.01000963, + "balance_loss_clip": 1.00617409, + "balance_loss_mlp": 0.99974716, + "epoch": 0.18742382914514538, + "flos": 74770200103680.0, + "grad_norm": 0.7426370920901082, + "language_loss": 0.53945434, + "learning_rate": 3.7456391179022033e-06, + "loss": 0.55961394, + "num_input_tokens_seen": 184689715, + "router_z_loss_clip": 0.08837891, + "router_z_loss_mlp": 0.012146, + "step": 6459, + "time_per_iteration": 3.0099267959594727 + }, + { + "auxiliary_loss_clip": 0.01086352, + "auxiliary_loss_mlp": 0.01035567, + "balance_loss_clip": 1.02624822, + "balance_loss_mlp": 1.01620758, + "epoch": 0.18745284661366143, + "flos": 12266419190400.0, + "grad_norm": 2.551649196883425, + "language_loss": 0.95344824, + "learning_rate": 3.745547376768758e-06, + "loss": 0.97466743, + "num_input_tokens_seen": 184701450, + "router_z_loss_clip": 0.60058594, + "router_z_loss_mlp": 0.19372559, + "step": 6460, + "time_per_iteration": 2.3308894634246826 + }, + { + "auxiliary_loss_clip": 0.0101332, + "auxiliary_loss_mlp": 0.01001634, + "balance_loss_clip": 1.00447845, + "balance_loss_mlp": 1.00050128, + "epoch": 0.18748186408217746, + "flos": 64958523672960.0, + "grad_norm": 0.6881863931352472, + "language_loss": 0.51393753, + "learning_rate": 3.7454556202179318e-06, + "loss": 0.53408706, + "num_input_tokens_seen": 184759690, + "router_z_loss_clip": 0.08837891, + "router_z_loss_mlp": 0.01135254, + "step": 6461, + "time_per_iteration": 2.9817750453948975 + }, + { + "auxiliary_loss_clip": 0.01012275, + "auxiliary_loss_mlp": 0.01001456, + "balance_loss_clip": 1.00348973, + "balance_loss_mlp": 1.00037694, + "epoch": 0.1875108815506935, + "flos": 61339148046720.0, + "grad_norm": 0.6184368815279524, + "language_loss": 0.46721965, + "learning_rate": 3.745363848250535e-06, + "loss": 0.48735696, + "num_input_tokens_seen": 184822995, + "router_z_loss_clip": 0.08789062, + "router_z_loss_mlp": 0.01080322, + "step": 6462, + "time_per_iteration": 3.170581340789795 + }, + { + "auxiliary_loss_clip": 0.01071027, + "auxiliary_loss_mlp": 0.01027985, + "balance_loss_clip": 1.02275109, + "balance_loss_mlp": 1.01450288, + "epoch": 0.18753989901920956, + "flos": 32449429146240.0, + "grad_norm": 1.9455167437425918, + "language_loss": 0.69104636, + "learning_rate": 3.7452720608673785e-06, + "loss": 0.71203649, + "num_input_tokens_seen": 184843275, + "router_z_loss_clip": 0.48193359, + "router_z_loss_mlp": 0.13476562, + "step": 6463, + "time_per_iteration": 2.5302770137786865 + }, + { + "auxiliary_loss_clip": 0.01073354, + "auxiliary_loss_mlp": 0.01028039, + "balance_loss_clip": 1.02356589, + "balance_loss_mlp": 1.01261926, + "epoch": 0.1875689164877256, + "flos": 19017678862080.0, + "grad_norm": 2.2119903243995296, + "language_loss": 0.80759656, + "learning_rate": 3.745180258069273e-06, + "loss": 0.82861042, + "num_input_tokens_seen": 184855355, + "router_z_loss_clip": 0.49755859, + "router_z_loss_mlp": 0.15405273, + "step": 6464, + "time_per_iteration": 2.4199256896972656 + }, + { + "auxiliary_loss_clip": 0.01072519, + "auxiliary_loss_mlp": 0.0103263, + "balance_loss_clip": 1.02124333, + "balance_loss_mlp": 1.01629841, + "epoch": 0.18759793395624166, + "flos": 19201042656000.0, + "grad_norm": 2.4565445537803705, + "language_loss": 0.72043812, + "learning_rate": 3.745088439857029e-06, + "loss": 0.74148965, + "num_input_tokens_seen": 184868350, + "router_z_loss_clip": 0.51245117, + "router_z_loss_mlp": 0.16320801, + "step": 6465, + "time_per_iteration": 2.41282057762146 + }, + { + "auxiliary_loss_clip": 0.01073657, + "auxiliary_loss_mlp": 0.01026881, + "balance_loss_clip": 1.02369177, + "balance_loss_mlp": 1.01215875, + "epoch": 0.18762695142475772, + "flos": 11907197544960.0, + "grad_norm": 2.4157534853034592, + "language_loss": 0.79304188, + "learning_rate": 3.744996606231458e-06, + "loss": 0.81404728, + "num_input_tokens_seen": 184879375, + "router_z_loss_clip": 0.49951172, + "router_z_loss_mlp": 0.1472168, + "step": 6466, + "time_per_iteration": 2.5175631046295166 + }, + { + "auxiliary_loss_clip": 0.01078849, + "auxiliary_loss_mlp": 0.01027921, + "balance_loss_clip": 1.02542174, + "balance_loss_mlp": 1.0127275, + "epoch": 0.18765596889327374, + "flos": 74732670391680.0, + "grad_norm": 2.949152403875339, + "language_loss": 0.83901334, + "learning_rate": 3.744904757193371e-06, + "loss": 0.86008108, + "num_input_tokens_seen": 184903265, + "router_z_loss_clip": 0.53466797, + "router_z_loss_mlp": 0.15179443, + "step": 6467, + "time_per_iteration": 5.038075923919678 + }, + { + "auxiliary_loss_clip": 0.01012348, + "auxiliary_loss_mlp": 0.01004068, + "balance_loss_clip": 1.00347066, + "balance_loss_mlp": 1.00285244, + "epoch": 0.1876849863617898, + "flos": 58642177207680.0, + "grad_norm": 0.6971789158601263, + "language_loss": 0.47919819, + "learning_rate": 3.7448128927435788e-06, + "loss": 0.49936232, + "num_input_tokens_seen": 184961320, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.012146, + "step": 6468, + "time_per_iteration": 5.4188456535339355 + }, + { + "auxiliary_loss_clip": 0.01071911, + "auxiliary_loss_mlp": 0.01027931, + "balance_loss_clip": 1.02429175, + "balance_loss_mlp": 1.01410842, + "epoch": 0.18771400383030584, + "flos": 11542145702400.0, + "grad_norm": 2.620289575210716, + "language_loss": 0.81576365, + "learning_rate": 3.744721012882893e-06, + "loss": 0.83676207, + "num_input_tokens_seen": 184974175, + "router_z_loss_clip": 0.47680664, + "router_z_loss_mlp": 0.1383667, + "step": 6469, + "time_per_iteration": 2.41300106048584 + }, + { + "auxiliary_loss_clip": 0.01078939, + "auxiliary_loss_mlp": 0.01036126, + "balance_loss_clip": 1.0259645, + "balance_loss_mlp": 1.02018762, + "epoch": 0.1877430212988219, + "flos": 14019024130560.0, + "grad_norm": 2.774104276008207, + "language_loss": 0.77563608, + "learning_rate": 3.7446291176121255e-06, + "loss": 0.79678679, + "num_input_tokens_seen": 184987370, + "router_z_loss_clip": 0.52929688, + "router_z_loss_mlp": 0.15942383, + "step": 6470, + "time_per_iteration": 2.416393518447876 + }, + { + "auxiliary_loss_clip": 0.01074916, + "auxiliary_loss_mlp": 0.01023589, + "balance_loss_clip": 1.02569652, + "balance_loss_mlp": 1.01000524, + "epoch": 0.18777203876733795, + "flos": 27684063319680.0, + "grad_norm": 3.2186250054751784, + "language_loss": 0.7988047, + "learning_rate": 3.7445372069320877e-06, + "loss": 0.81978977, + "num_input_tokens_seen": 185008320, + "router_z_loss_clip": 0.49291992, + "router_z_loss_mlp": 0.13592529, + "step": 6471, + "time_per_iteration": 2.5607216358184814 + }, + { + "auxiliary_loss_clip": 0.01084604, + "auxiliary_loss_mlp": 0.01037974, + "balance_loss_clip": 1.02881408, + "balance_loss_mlp": 1.02131474, + "epoch": 0.187801056235854, + "flos": 16537030007040.0, + "grad_norm": 2.4508672454919593, + "language_loss": 0.76128268, + "learning_rate": 3.7444452808435906e-06, + "loss": 0.78250849, + "num_input_tokens_seen": 185020805, + "router_z_loss_clip": 0.55810547, + "router_z_loss_mlp": 0.16662598, + "step": 6472, + "time_per_iteration": 2.355705976486206 + }, + { + "auxiliary_loss_clip": 0.01014227, + "auxiliary_loss_mlp": 0.01001057, + "balance_loss_clip": 1.00526965, + "balance_loss_mlp": 0.9999488, + "epoch": 0.18783007370437002, + "flos": 66821489020800.0, + "grad_norm": 0.669236127359059, + "language_loss": 0.47616336, + "learning_rate": 3.7443533393474478e-06, + "loss": 0.49631619, + "num_input_tokens_seen": 185081065, + "router_z_loss_clip": 0.08984375, + "router_z_loss_mlp": 0.0111084, + "step": 6473, + "time_per_iteration": 3.0109939575195312 + }, + { + "auxiliary_loss_clip": 0.01077938, + "auxiliary_loss_mlp": 0.01033661, + "balance_loss_clip": 1.02608824, + "balance_loss_mlp": 1.01713252, + "epoch": 0.18785909117288607, + "flos": 41017918417920.0, + "grad_norm": 2.606792663090248, + "language_loss": 0.93012428, + "learning_rate": 3.74426138244447e-06, + "loss": 0.95124024, + "num_input_tokens_seen": 185096625, + "router_z_loss_clip": 0.51855469, + "router_z_loss_mlp": 0.16540527, + "step": 6474, + "time_per_iteration": 2.5579912662506104 + }, + { + "auxiliary_loss_clip": 0.0108279, + "auxiliary_loss_mlp": 0.01036364, + "balance_loss_clip": 1.0277015, + "balance_loss_mlp": 1.01922226, + "epoch": 0.18788810864140212, + "flos": 18002578815360.0, + "grad_norm": 2.364184488202524, + "language_loss": 0.8850705, + "learning_rate": 3.7441694101354697e-06, + "loss": 0.90626204, + "num_input_tokens_seen": 185114335, + "router_z_loss_clip": 0.55078125, + "router_z_loss_mlp": 0.17150879, + "step": 6475, + "time_per_iteration": 2.405642509460449 + }, + { + "auxiliary_loss_clip": 0.01082277, + "auxiliary_loss_mlp": 0.01032333, + "balance_loss_clip": 1.02762008, + "balance_loss_mlp": 1.0154891, + "epoch": 0.18791712610991818, + "flos": 25512501530880.0, + "grad_norm": 1.90134879333978, + "language_loss": 0.62966549, + "learning_rate": 3.7440774224212595e-06, + "loss": 0.65081167, + "num_input_tokens_seen": 185129305, + "router_z_loss_clip": 0.54614258, + "router_z_loss_mlp": 0.1685791, + "step": 6476, + "time_per_iteration": 4.774057865142822 + }, + { + "auxiliary_loss_clip": 0.01080986, + "auxiliary_loss_mlp": 0.01028962, + "balance_loss_clip": 1.02563393, + "balance_loss_mlp": 1.0112952, + "epoch": 0.18794614357843423, + "flos": 22047441580800.0, + "grad_norm": 2.609089791127758, + "language_loss": 0.82155049, + "learning_rate": 3.7439854193026523e-06, + "loss": 0.84264994, + "num_input_tokens_seen": 185144715, + "router_z_loss_clip": 0.55444336, + "router_z_loss_mlp": 0.17663574, + "step": 6477, + "time_per_iteration": 4.818073272705078 + }, + { + "auxiliary_loss_clip": 0.01091727, + "auxiliary_loss_mlp": 0.01039993, + "balance_loss_clip": 1.03104711, + "balance_loss_mlp": 1.02198017, + "epoch": 0.18797516104695025, + "flos": 34637191672320.0, + "grad_norm": 2.15616579461835, + "language_loss": 0.96199882, + "learning_rate": 3.743893400780459e-06, + "loss": 0.98331606, + "num_input_tokens_seen": 185163750, + "router_z_loss_clip": 0.60693359, + "router_z_loss_mlp": 0.18017578, + "step": 6478, + "time_per_iteration": 2.5029051303863525 + }, + { + "auxiliary_loss_clip": 0.01078109, + "auxiliary_loss_mlp": 0.01036348, + "balance_loss_clip": 1.02618623, + "balance_loss_mlp": 1.01930153, + "epoch": 0.1880041785154663, + "flos": 24017485668480.0, + "grad_norm": 2.4226384534930885, + "language_loss": 0.79314172, + "learning_rate": 3.7438013668554945e-06, + "loss": 0.81428623, + "num_input_tokens_seen": 185179065, + "router_z_loss_clip": 0.51928711, + "router_z_loss_mlp": 0.17028809, + "step": 6479, + "time_per_iteration": 2.4050111770629883 + }, + { + "auxiliary_loss_clip": 0.01085411, + "auxiliary_loss_mlp": 0.01034564, + "balance_loss_clip": 1.02908695, + "balance_loss_mlp": 1.01766062, + "epoch": 0.18803319598398235, + "flos": 74730401153280.0, + "grad_norm": 1.8150933236488112, + "language_loss": 0.92970896, + "learning_rate": 3.7437093175285702e-06, + "loss": 0.95090866, + "num_input_tokens_seen": 185204255, + "router_z_loss_clip": 0.56396484, + "router_z_loss_mlp": 0.16894531, + "step": 6480, + "time_per_iteration": 2.7832939624786377 + }, + { + "auxiliary_loss_clip": 0.01081151, + "auxiliary_loss_mlp": 0.01028849, + "balance_loss_clip": 1.02685666, + "balance_loss_mlp": 1.01245809, + "epoch": 0.1880622134524984, + "flos": 22229932590720.0, + "grad_norm": 3.066414643370651, + "language_loss": 0.85770541, + "learning_rate": 3.7436172528005e-06, + "loss": 0.8788054, + "num_input_tokens_seen": 185221540, + "router_z_loss_clip": 0.54345703, + "router_z_loss_mlp": 0.16387939, + "step": 6481, + "time_per_iteration": 2.406251907348633 + }, + { + "auxiliary_loss_clip": 0.01075673, + "auxiliary_loss_mlp": 0.01038363, + "balance_loss_clip": 1.02535605, + "balance_loss_mlp": 1.02368844, + "epoch": 0.18809123092101446, + "flos": 10662753576960.0, + "grad_norm": 3.1406750576567437, + "language_loss": 0.83438849, + "learning_rate": 3.743525172672097e-06, + "loss": 0.85552889, + "num_input_tokens_seen": 185232130, + "router_z_loss_clip": 0.50341797, + "router_z_loss_mlp": 0.14672852, + "step": 6482, + "time_per_iteration": 2.6090211868286133 + }, + { + "auxiliary_loss_clip": 0.01076357, + "auxiliary_loss_mlp": 0.01029143, + "balance_loss_clip": 1.02531421, + "balance_loss_mlp": 1.01355064, + "epoch": 0.1881202483895305, + "flos": 26936154840960.0, + "grad_norm": 2.146885583161588, + "language_loss": 1.09731209, + "learning_rate": 3.743433077144173e-06, + "loss": 1.11836708, + "num_input_tokens_seen": 185250475, + "router_z_loss_clip": 0.51000977, + "router_z_loss_mlp": 0.15594482, + "step": 6483, + "time_per_iteration": 2.447950839996338 + }, + { + "auxiliary_loss_clip": 0.01077885, + "auxiliary_loss_mlp": 0.0103492, + "balance_loss_clip": 1.02523184, + "balance_loss_mlp": 1.01941645, + "epoch": 0.18814926585804653, + "flos": 31209663300480.0, + "grad_norm": 2.081080934597082, + "language_loss": 0.77955461, + "learning_rate": 3.7433409662175434e-06, + "loss": 0.80068272, + "num_input_tokens_seen": 185271605, + "router_z_loss_clip": 0.52587891, + "router_z_loss_mlp": 0.1550293, + "step": 6484, + "time_per_iteration": 2.439586877822876 + }, + { + "auxiliary_loss_clip": 0.01073281, + "auxiliary_loss_mlp": 0.01026906, + "balance_loss_clip": 1.02489972, + "balance_loss_mlp": 1.01164103, + "epoch": 0.18817828332656258, + "flos": 43134876973440.0, + "grad_norm": 2.2765727724196188, + "language_loss": 0.72694778, + "learning_rate": 3.743248839893021e-06, + "loss": 0.74794966, + "num_input_tokens_seen": 185292130, + "router_z_loss_clip": 0.48339844, + "router_z_loss_mlp": 0.15258789, + "step": 6485, + "time_per_iteration": 2.588641405105591 + }, + { + "auxiliary_loss_clip": 0.01075028, + "auxiliary_loss_mlp": 0.01028866, + "balance_loss_clip": 1.02429879, + "balance_loss_mlp": 1.01386309, + "epoch": 0.18820730079507864, + "flos": 25294119776640.0, + "grad_norm": 1.9322655482256466, + "language_loss": 0.84722757, + "learning_rate": 3.743156698171419e-06, + "loss": 0.86826652, + "num_input_tokens_seen": 185308730, + "router_z_loss_clip": 0.50708008, + "router_z_loss_mlp": 0.14990234, + "step": 6486, + "time_per_iteration": 2.3949692249298096 + }, + { + "auxiliary_loss_clip": 0.01079639, + "auxiliary_loss_mlp": 0.01033093, + "balance_loss_clip": 1.02447677, + "balance_loss_mlp": 1.01700008, + "epoch": 0.1882363182635947, + "flos": 22560769434240.0, + "grad_norm": 1.9959039769294693, + "language_loss": 0.72792023, + "learning_rate": 3.7430645410535513e-06, + "loss": 0.74904758, + "num_input_tokens_seen": 185327600, + "router_z_loss_clip": 0.55126953, + "router_z_loss_mlp": 0.16088867, + "step": 6487, + "time_per_iteration": 2.4933154582977295 + }, + { + "auxiliary_loss_clip": 0.01076168, + "auxiliary_loss_mlp": 0.01028342, + "balance_loss_clip": 1.02605641, + "balance_loss_mlp": 1.01283908, + "epoch": 0.18826533573211074, + "flos": 17340695660160.0, + "grad_norm": 2.1442325899529395, + "language_loss": 0.79440296, + "learning_rate": 3.742972368540233e-06, + "loss": 0.81544805, + "num_input_tokens_seen": 185341170, + "router_z_loss_clip": 0.50097656, + "router_z_loss_mlp": 0.15509033, + "step": 6488, + "time_per_iteration": 2.346987009048462 + }, + { + "auxiliary_loss_clip": 0.0107427, + "auxiliary_loss_mlp": 0.01028705, + "balance_loss_clip": 1.02211118, + "balance_loss_mlp": 1.01319623, + "epoch": 0.1882943532006268, + "flos": 33757450433280.0, + "grad_norm": 3.2503021176302185, + "language_loss": 0.86712515, + "learning_rate": 3.7428801806322774e-06, + "loss": 0.88815492, + "num_input_tokens_seen": 185357070, + "router_z_loss_clip": 0.52148438, + "router_z_loss_mlp": 0.15509033, + "step": 6489, + "time_per_iteration": 2.503910541534424 + }, + { + "auxiliary_loss_clip": 0.01009704, + "auxiliary_loss_mlp": 0.01009212, + "balance_loss_clip": 1.00164855, + "balance_loss_mlp": 1.0081811, + "epoch": 0.18832337066914281, + "flos": 58294825424640.0, + "grad_norm": 0.7032372706700065, + "language_loss": 0.47512403, + "learning_rate": 3.7427879773304986e-06, + "loss": 0.4953132, + "num_input_tokens_seen": 185425485, + "router_z_loss_clip": 0.08056641, + "router_z_loss_mlp": 0.01031494, + "step": 6490, + "time_per_iteration": 3.220057249069214 + }, + { + "auxiliary_loss_clip": 0.01079439, + "auxiliary_loss_mlp": 0.0102498, + "balance_loss_clip": 1.02572811, + "balance_loss_mlp": 1.01039469, + "epoch": 0.18835238813765887, + "flos": 29233195522560.0, + "grad_norm": 4.96298053843981, + "language_loss": 1.01550436, + "learning_rate": 3.7426957586357108e-06, + "loss": 1.03654861, + "num_input_tokens_seen": 185438845, + "router_z_loss_clip": 0.53710938, + "router_z_loss_mlp": 0.14575195, + "step": 6491, + "time_per_iteration": 2.4791364669799805 + }, + { + "auxiliary_loss_clip": 0.01077632, + "auxiliary_loss_mlp": 0.01027809, + "balance_loss_clip": 1.02481055, + "balance_loss_mlp": 1.01174545, + "epoch": 0.18838140560617492, + "flos": 27335247125760.0, + "grad_norm": 2.4320636051640285, + "language_loss": 1.02362478, + "learning_rate": 3.7426035245487296e-06, + "loss": 1.04467916, + "num_input_tokens_seen": 185452280, + "router_z_loss_clip": 0.52807617, + "router_z_loss_mlp": 0.16046143, + "step": 6492, + "time_per_iteration": 2.4238078594207764 + }, + { + "auxiliary_loss_clip": 0.01080826, + "auxiliary_loss_mlp": 0.01035014, + "balance_loss_clip": 1.02499986, + "balance_loss_mlp": 1.01725793, + "epoch": 0.18841042307469097, + "flos": 26718576048000.0, + "grad_norm": 2.314364763505479, + "language_loss": 0.95341635, + "learning_rate": 3.742511275070368e-06, + "loss": 0.9745748, + "num_input_tokens_seen": 185468625, + "router_z_loss_clip": 0.55761719, + "router_z_loss_mlp": 0.17755127, + "step": 6493, + "time_per_iteration": 2.4224965572357178 + }, + { + "auxiliary_loss_clip": 0.01078529, + "auxiliary_loss_mlp": 0.01034657, + "balance_loss_clip": 1.02461147, + "balance_loss_mlp": 1.01740098, + "epoch": 0.18843944054320702, + "flos": 15698625684480.0, + "grad_norm": 2.156378829876167, + "language_loss": 0.75776863, + "learning_rate": 3.7424190102014423e-06, + "loss": 0.77890044, + "num_input_tokens_seen": 185483505, + "router_z_loss_clip": 0.53881836, + "router_z_loss_mlp": 0.17242432, + "step": 6494, + "time_per_iteration": 2.352879047393799 + }, + { + "auxiliary_loss_clip": 0.01072143, + "auxiliary_loss_mlp": 0.01026856, + "balance_loss_clip": 1.02414179, + "balance_loss_mlp": 1.01373124, + "epoch": 0.18846845801172304, + "flos": 24824712280320.0, + "grad_norm": 2.2244396339532932, + "language_loss": 0.66655624, + "learning_rate": 3.7423267299427667e-06, + "loss": 0.68754619, + "num_input_tokens_seen": 185499470, + "router_z_loss_clip": 0.47973633, + "router_z_loss_mlp": 0.13134766, + "step": 6495, + "time_per_iteration": 2.4320473670959473 + }, + { + "auxiliary_loss_clip": 0.01010367, + "auxiliary_loss_mlp": 0.0100235, + "balance_loss_clip": 1.00270247, + "balance_loss_mlp": 1.00123572, + "epoch": 0.1884974754802391, + "flos": 59904846771840.0, + "grad_norm": 0.7453277951370876, + "language_loss": 0.48647559, + "learning_rate": 3.7422344342951564e-06, + "loss": 0.50660276, + "num_input_tokens_seen": 185545560, + "router_z_loss_clip": 0.07666016, + "router_z_loss_mlp": 0.01116943, + "step": 6496, + "time_per_iteration": 2.9854629039764404 + }, + { + "auxiliary_loss_clip": 0.01072016, + "auxiliary_loss_mlp": 0.01026399, + "balance_loss_clip": 1.02465999, + "balance_loss_mlp": 1.01426959, + "epoch": 0.18852649294875515, + "flos": 16356389299200.0, + "grad_norm": 2.273773170432847, + "language_loss": 0.65703046, + "learning_rate": 3.742142123259427e-06, + "loss": 0.67801464, + "num_input_tokens_seen": 185558690, + "router_z_loss_clip": 0.47363281, + "router_z_loss_mlp": 0.12139893, + "step": 6497, + "time_per_iteration": 2.3195953369140625 + }, + { + "auxiliary_loss_clip": 0.01010521, + "auxiliary_loss_mlp": 0.01001106, + "balance_loss_clip": 1.00274444, + "balance_loss_mlp": 1.00005651, + "epoch": 0.1885555104172712, + "flos": 57218803188480.0, + "grad_norm": 0.7236464737323984, + "language_loss": 0.46072885, + "learning_rate": 3.7420497968363922e-06, + "loss": 0.48084509, + "num_input_tokens_seen": 185620775, + "router_z_loss_clip": 0.07763672, + "router_z_loss_mlp": 0.01049805, + "step": 6498, + "time_per_iteration": 2.9568183422088623 + }, + { + "auxiliary_loss_clip": 0.01080406, + "auxiliary_loss_mlp": 0.01029141, + "balance_loss_clip": 1.02563238, + "balance_loss_mlp": 1.01232648, + "epoch": 0.18858452788578725, + "flos": 27634347676800.0, + "grad_norm": 2.7341367628576165, + "language_loss": 0.87353343, + "learning_rate": 3.7419574550268694e-06, + "loss": 0.89462894, + "num_input_tokens_seen": 185640720, + "router_z_loss_clip": 0.54785156, + "router_z_loss_mlp": 0.16821289, + "step": 6499, + "time_per_iteration": 2.4707319736480713 + }, + { + "auxiliary_loss_clip": 0.01077597, + "auxiliary_loss_mlp": 0.01029746, + "balance_loss_clip": 1.02490008, + "balance_loss_mlp": 1.01469588, + "epoch": 0.1886135453543033, + "flos": 16465842011520.0, + "grad_norm": 2.3361625867711284, + "language_loss": 0.69859099, + "learning_rate": 3.7418650978316737e-06, + "loss": 0.71966439, + "num_input_tokens_seen": 185655870, + "router_z_loss_clip": 0.52734375, + "router_z_loss_mlp": 0.15063477, + "step": 6500, + "time_per_iteration": 2.408698320388794 + }, + { + "auxiliary_loss_clip": 0.01010947, + "auxiliary_loss_mlp": 0.01001323, + "balance_loss_clip": 1.00303686, + "balance_loss_mlp": 1.00038111, + "epoch": 0.18864256282281933, + "flos": 67786243153920.0, + "grad_norm": 0.6762383541676281, + "language_loss": 0.48013225, + "learning_rate": 3.7417727252516204e-06, + "loss": 0.50025499, + "num_input_tokens_seen": 185723505, + "router_z_loss_clip": 0.07910156, + "router_z_loss_mlp": 0.00939941, + "step": 6501, + "time_per_iteration": 3.1744096279144287 + }, + { + "auxiliary_loss_clip": 0.01076384, + "auxiliary_loss_mlp": 0.01033014, + "balance_loss_clip": 1.0235064, + "balance_loss_mlp": 1.01484656, + "epoch": 0.18867158029133538, + "flos": 24965447437440.0, + "grad_norm": 2.048660133009335, + "language_loss": 0.81202424, + "learning_rate": 3.7416803372875256e-06, + "loss": 0.83311826, + "num_input_tokens_seen": 185742225, + "router_z_loss_clip": 0.52905273, + "router_z_loss_mlp": 0.1817627, + "step": 6502, + "time_per_iteration": 2.500762701034546 + }, + { + "auxiliary_loss_clip": 0.0107797, + "auxiliary_loss_mlp": 0.01035935, + "balance_loss_clip": 1.02439594, + "balance_loss_mlp": 1.02083087, + "epoch": 0.18870059775985143, + "flos": 15880627935360.0, + "grad_norm": 2.399936220736505, + "language_loss": 0.75396049, + "learning_rate": 3.741587933940205e-06, + "loss": 0.77509958, + "num_input_tokens_seen": 185755155, + "router_z_loss_clip": 0.53588867, + "router_z_loss_mlp": 0.15081787, + "step": 6503, + "time_per_iteration": 2.318650245666504 + }, + { + "auxiliary_loss_clip": 0.01074695, + "auxiliary_loss_mlp": 0.01030688, + "balance_loss_clip": 1.02319384, + "balance_loss_mlp": 1.01595354, + "epoch": 0.18872961522836748, + "flos": 35877411365760.0, + "grad_norm": 1.7539632315602902, + "language_loss": 0.57060981, + "learning_rate": 3.7414955152104754e-06, + "loss": 0.5916636, + "num_input_tokens_seen": 185772685, + "router_z_loss_clip": 0.51489258, + "router_z_loss_mlp": 0.1473999, + "step": 6504, + "time_per_iteration": 2.5885097980499268 + }, + { + "auxiliary_loss_clip": 0.01080622, + "auxiliary_loss_mlp": 0.01028933, + "balance_loss_clip": 1.02384949, + "balance_loss_mlp": 1.01338184, + "epoch": 0.18875863269688353, + "flos": 18301225518720.0, + "grad_norm": 3.256538437152628, + "language_loss": 0.84875607, + "learning_rate": 3.7414030810991523e-06, + "loss": 0.86985159, + "num_input_tokens_seen": 185789565, + "router_z_loss_clip": 0.56835938, + "router_z_loss_mlp": 0.15539551, + "step": 6505, + "time_per_iteration": 2.352839469909668 + }, + { + "auxiliary_loss_clip": 0.01078739, + "auxiliary_loss_mlp": 0.01039444, + "balance_loss_clip": 1.02469826, + "balance_loss_mlp": 1.02152729, + "epoch": 0.18878765016539958, + "flos": 11648246924160.0, + "grad_norm": 2.8166857300764074, + "language_loss": 0.84333187, + "learning_rate": 3.741310631607053e-06, + "loss": 0.86451375, + "num_input_tokens_seen": 185800930, + "router_z_loss_clip": 0.54052734, + "router_z_loss_mlp": 0.17919922, + "step": 6506, + "time_per_iteration": 2.3829805850982666 + }, + { + "auxiliary_loss_clip": 0.01008173, + "auxiliary_loss_mlp": 0.0100543, + "balance_loss_clip": 1.00047708, + "balance_loss_mlp": 1.0042913, + "epoch": 0.1888166676339156, + "flos": 68248249441920.0, + "grad_norm": 0.6497646305545989, + "language_loss": 0.47581029, + "learning_rate": 3.7412181667349933e-06, + "loss": 0.49594629, + "num_input_tokens_seen": 185865620, + "router_z_loss_clip": 0.07714844, + "router_z_loss_mlp": 0.01141357, + "step": 6507, + "time_per_iteration": 3.0418262481689453 + }, + { + "auxiliary_loss_clip": 0.01077698, + "auxiliary_loss_mlp": 0.01034074, + "balance_loss_clip": 1.0251379, + "balance_loss_mlp": 1.01854718, + "epoch": 0.18884568510243166, + "flos": 15881395985280.0, + "grad_norm": 2.803724266417916, + "language_loss": 0.77695107, + "learning_rate": 3.74112568648379e-06, + "loss": 0.79806882, + "num_input_tokens_seen": 185882120, + "router_z_loss_clip": 0.52514648, + "router_z_loss_mlp": 0.15539551, + "step": 6508, + "time_per_iteration": 2.4990170001983643 + }, + { + "auxiliary_loss_clip": 0.01071964, + "auxiliary_loss_mlp": 0.01026767, + "balance_loss_clip": 1.02316308, + "balance_loss_mlp": 1.01227701, + "epoch": 0.1888747025709477, + "flos": 11756861763840.0, + "grad_norm": 3.544379944565129, + "language_loss": 0.79550344, + "learning_rate": 3.74103319085426e-06, + "loss": 0.81649077, + "num_input_tokens_seen": 185893295, + "router_z_loss_clip": 0.48779297, + "router_z_loss_mlp": 0.1449585, + "step": 6509, + "time_per_iteration": 2.4876840114593506 + }, + { + "auxiliary_loss_clip": 0.0107625, + "auxiliary_loss_mlp": 0.01027207, + "balance_loss_clip": 1.02450824, + "balance_loss_mlp": 1.01206148, + "epoch": 0.18890372003946376, + "flos": 12158432755200.0, + "grad_norm": 2.3001357358970163, + "language_loss": 0.5720197, + "learning_rate": 3.740940679847221e-06, + "loss": 0.59305429, + "num_input_tokens_seen": 185904960, + "router_z_loss_clip": 0.51733398, + "router_z_loss_mlp": 0.15133667, + "step": 6510, + "time_per_iteration": 2.4875917434692383 + }, + { + "auxiliary_loss_clip": 0.01074596, + "auxiliary_loss_mlp": 0.01028193, + "balance_loss_clip": 1.02401626, + "balance_loss_mlp": 1.01306581, + "epoch": 0.18893273750797981, + "flos": 17193746280960.0, + "grad_norm": 2.174726051379983, + "language_loss": 0.75273621, + "learning_rate": 3.7408481534634887e-06, + "loss": 0.77376407, + "num_input_tokens_seen": 185917515, + "router_z_loss_clip": 0.50585938, + "router_z_loss_mlp": 0.15118408, + "step": 6511, + "time_per_iteration": 2.3548810482025146 + }, + { + "auxiliary_loss_clip": 0.01071917, + "auxiliary_loss_mlp": 0.0103199, + "balance_loss_clip": 1.02265525, + "balance_loss_mlp": 1.01708889, + "epoch": 0.18896175497649584, + "flos": 29416140380160.0, + "grad_norm": 4.972430280702915, + "language_loss": 0.70492512, + "learning_rate": 3.7407556117038813e-06, + "loss": 0.72596425, + "num_input_tokens_seen": 185931520, + "router_z_loss_clip": 0.49291992, + "router_z_loss_mlp": 0.14892578, + "step": 6512, + "time_per_iteration": 2.453726291656494 + }, + { + "auxiliary_loss_clip": 0.01074454, + "auxiliary_loss_mlp": 0.01026414, + "balance_loss_clip": 1.02302623, + "balance_loss_mlp": 1.01243711, + "epoch": 0.1889907724450119, + "flos": 16685166372480.0, + "grad_norm": 2.154174286692505, + "language_loss": 0.74749064, + "learning_rate": 3.7406630545692164e-06, + "loss": 0.76849931, + "num_input_tokens_seen": 185946565, + "router_z_loss_clip": 0.5144043, + "router_z_loss_mlp": 0.13983154, + "step": 6513, + "time_per_iteration": 2.3600285053253174 + }, + { + "auxiliary_loss_clip": 0.01066291, + "auxiliary_loss_mlp": 0.01023221, + "balance_loss_clip": 1.02114308, + "balance_loss_mlp": 1.01084685, + "epoch": 0.18901978991352794, + "flos": 13071167095680.0, + "grad_norm": 4.389348069097552, + "language_loss": 0.8565644, + "learning_rate": 3.7405704820603105e-06, + "loss": 0.87745947, + "num_input_tokens_seen": 185958560, + "router_z_loss_clip": 0.45117188, + "router_z_loss_mlp": 0.12365723, + "step": 6514, + "time_per_iteration": 2.377488374710083 + }, + { + "auxiliary_loss_clip": 0.01078842, + "auxiliary_loss_mlp": 0.01033152, + "balance_loss_clip": 1.02489901, + "balance_loss_mlp": 1.01618862, + "epoch": 0.189048807382044, + "flos": 12342110751360.0, + "grad_norm": 2.6443863504401492, + "language_loss": 0.81531674, + "learning_rate": 3.7404778941779816e-06, + "loss": 0.83643663, + "num_input_tokens_seen": 185969935, + "router_z_loss_clip": 0.53881836, + "router_z_loss_mlp": 0.16967773, + "step": 6515, + "time_per_iteration": 2.3101775646209717 + }, + { + "auxiliary_loss_clip": 0.01008853, + "auxiliary_loss_mlp": 0.01004924, + "balance_loss_clip": 1.00069118, + "balance_loss_mlp": 1.00379753, + "epoch": 0.18907782485056004, + "flos": 69342148160640.0, + "grad_norm": 0.6478695724365393, + "language_loss": 0.51240528, + "learning_rate": 3.740385290923048e-06, + "loss": 0.53254306, + "num_input_tokens_seen": 186033810, + "router_z_loss_clip": 0.08154297, + "router_z_loss_mlp": 0.0112915, + "step": 6516, + "time_per_iteration": 3.08471417427063 + }, + { + "auxiliary_loss_clip": 0.01080593, + "auxiliary_loss_mlp": 0.01035554, + "balance_loss_clip": 1.02632034, + "balance_loss_mlp": 1.01840568, + "epoch": 0.1891068423190761, + "flos": 25111454209920.0, + "grad_norm": 4.042508508763273, + "language_loss": 0.85511458, + "learning_rate": 3.740292672296327e-06, + "loss": 0.87627602, + "num_input_tokens_seen": 186056415, + "router_z_loss_clip": 0.54321289, + "router_z_loss_mlp": 0.17150879, + "step": 6517, + "time_per_iteration": 2.626913070678711 + }, + { + "auxiliary_loss_clip": 0.01077825, + "auxiliary_loss_mlp": 0.01038625, + "balance_loss_clip": 1.0244503, + "balance_loss_mlp": 1.02132809, + "epoch": 0.18913585978759212, + "flos": 33066449337600.0, + "grad_norm": 2.46236489370507, + "language_loss": 0.71764541, + "learning_rate": 3.7402000382986373e-06, + "loss": 0.73880994, + "num_input_tokens_seen": 186074315, + "router_z_loss_clip": 0.53369141, + "router_z_loss_mlp": 0.17297363, + "step": 6518, + "time_per_iteration": 2.4971961975097656 + }, + { + "auxiliary_loss_clip": 0.01008765, + "auxiliary_loss_mlp": 0.01002111, + "balance_loss_clip": 1.00052714, + "balance_loss_mlp": 1.00102067, + "epoch": 0.18916487725610817, + "flos": 74780219664000.0, + "grad_norm": 0.6128965822045399, + "language_loss": 0.44046789, + "learning_rate": 3.7401073889307966e-06, + "loss": 0.46057665, + "num_input_tokens_seen": 186144145, + "router_z_loss_clip": 0.08251953, + "router_z_loss_mlp": 0.01092529, + "step": 6519, + "time_per_iteration": 3.1718032360076904 + }, + { + "auxiliary_loss_clip": 0.01074882, + "auxiliary_loss_mlp": 0.01028056, + "balance_loss_clip": 1.02344668, + "balance_loss_mlp": 1.01147377, + "epoch": 0.18919389472462422, + "flos": 33429336675840.0, + "grad_norm": 1.8777111088152094, + "language_loss": 0.96662217, + "learning_rate": 3.7400147241936227e-06, + "loss": 0.98765147, + "num_input_tokens_seen": 186163425, + "router_z_loss_clip": 0.51416016, + "router_z_loss_mlp": 0.16583252, + "step": 6520, + "time_per_iteration": 2.4843358993530273 + }, + { + "auxiliary_loss_clip": 0.01071952, + "auxiliary_loss_mlp": 0.01032786, + "balance_loss_clip": 1.02410352, + "balance_loss_mlp": 1.01732492, + "epoch": 0.18922291219314027, + "flos": 16245051373440.0, + "grad_norm": 2.573727368956591, + "language_loss": 0.82538557, + "learning_rate": 3.7399220440879355e-06, + "loss": 0.84643292, + "num_input_tokens_seen": 186176670, + "router_z_loss_clip": 0.47851562, + "router_z_loss_mlp": 0.15460205, + "step": 6521, + "time_per_iteration": 2.341745138168335 + }, + { + "auxiliary_loss_clip": 0.0107485, + "auxiliary_loss_mlp": 0.01038425, + "balance_loss_clip": 1.02326584, + "balance_loss_mlp": 1.02235591, + "epoch": 0.18925192966165633, + "flos": 74729493457920.0, + "grad_norm": 2.2751123472191868, + "language_loss": 0.63162601, + "learning_rate": 3.7398293486145524e-06, + "loss": 0.65275878, + "num_input_tokens_seen": 186199935, + "router_z_loss_clip": 0.51586914, + "router_z_loss_mlp": 0.16088867, + "step": 6522, + "time_per_iteration": 2.8254497051239014 + }, + { + "auxiliary_loss_clip": 0.01077883, + "auxiliary_loss_mlp": 0.01037425, + "balance_loss_clip": 1.024652, + "balance_loss_mlp": 1.02282214, + "epoch": 0.18928094713017235, + "flos": 28139890296960.0, + "grad_norm": 2.5468814802026407, + "language_loss": 0.78658724, + "learning_rate": 3.739736637774292e-06, + "loss": 0.80774033, + "num_input_tokens_seen": 186215060, + "router_z_loss_clip": 0.53271484, + "router_z_loss_mlp": 0.14593506, + "step": 6523, + "time_per_iteration": 2.7126567363739014 + }, + { + "auxiliary_loss_clip": 0.01080522, + "auxiliary_loss_mlp": 0.0103059, + "balance_loss_clip": 1.02453589, + "balance_loss_mlp": 1.01382923, + "epoch": 0.1893099645986884, + "flos": 47042879742720.0, + "grad_norm": 2.8348658590123534, + "language_loss": 0.8405143, + "learning_rate": 3.7396439115679744e-06, + "loss": 0.86162549, + "num_input_tokens_seen": 186232635, + "router_z_loss_clip": 0.56005859, + "router_z_loss_mlp": 0.16760254, + "step": 6524, + "time_per_iteration": 2.527564764022827 + }, + { + "auxiliary_loss_clip": 0.01080379, + "auxiliary_loss_mlp": 0.01037375, + "balance_loss_clip": 1.0251615, + "balance_loss_mlp": 1.02088785, + "epoch": 0.18933898206720445, + "flos": 16538601018240.0, + "grad_norm": 2.412413478988218, + "language_loss": 0.67029572, + "learning_rate": 3.7395511699964173e-06, + "loss": 0.69147325, + "num_input_tokens_seen": 186246215, + "router_z_loss_clip": 0.55249023, + "router_z_loss_mlp": 0.16503906, + "step": 6525, + "time_per_iteration": 2.3361432552337646 + }, + { + "auxiliary_loss_clip": 0.01076761, + "auxiliary_loss_mlp": 0.0103924, + "balance_loss_clip": 1.02506971, + "balance_loss_mlp": 1.02334929, + "epoch": 0.1893679995357205, + "flos": 30654928707840.0, + "grad_norm": 2.509699897924918, + "language_loss": 0.75190341, + "learning_rate": 3.73945841306044e-06, + "loss": 0.77306348, + "num_input_tokens_seen": 186261635, + "router_z_loss_clip": 0.51660156, + "router_z_loss_mlp": 0.15893555, + "step": 6526, + "time_per_iteration": 2.4457552433013916 + }, + { + "auxiliary_loss_clip": 0.01083234, + "auxiliary_loss_mlp": 0.01042314, + "balance_loss_clip": 1.02622962, + "balance_loss_mlp": 1.02413404, + "epoch": 0.18939701700423656, + "flos": 33576565345920.0, + "grad_norm": 2.1860082186016214, + "language_loss": 0.99357456, + "learning_rate": 3.7393656407608626e-06, + "loss": 1.01482999, + "num_input_tokens_seen": 186281960, + "router_z_loss_clip": 0.56958008, + "router_z_loss_mlp": 0.18188477, + "step": 6527, + "time_per_iteration": 2.492088556289673 + }, + { + "auxiliary_loss_clip": 0.01009456, + "auxiliary_loss_mlp": 0.01005909, + "balance_loss_clip": 1.00120759, + "balance_loss_mlp": 1.00494993, + "epoch": 0.1894260344727526, + "flos": 64710011548800.0, + "grad_norm": 0.6385811781436964, + "language_loss": 0.46301231, + "learning_rate": 3.7392728530985045e-06, + "loss": 0.48316598, + "num_input_tokens_seen": 186345930, + "router_z_loss_clip": 0.08251953, + "router_z_loss_mlp": 0.00958252, + "step": 6528, + "time_per_iteration": 3.0358803272247314 + }, + { + "auxiliary_loss_clip": 0.01076589, + "auxiliary_loss_mlp": 0.01032094, + "balance_loss_clip": 1.02404571, + "balance_loss_mlp": 1.01618564, + "epoch": 0.18945505194126863, + "flos": 22375206224640.0, + "grad_norm": 2.244184993642294, + "language_loss": 0.86969155, + "learning_rate": 3.739180050074184e-06, + "loss": 0.89077842, + "num_input_tokens_seen": 186360795, + "router_z_loss_clip": 0.52587891, + "router_z_loss_mlp": 0.15905762, + "step": 6529, + "time_per_iteration": 2.404268741607666 + }, + { + "auxiliary_loss_clip": 0.01009883, + "auxiliary_loss_mlp": 0.01001896, + "balance_loss_clip": 1.00186694, + "balance_loss_mlp": 1.00091815, + "epoch": 0.18948406940978468, + "flos": 68531919171840.0, + "grad_norm": 0.6406398602505705, + "language_loss": 0.47954535, + "learning_rate": 3.739087231688722e-06, + "loss": 0.49966317, + "num_input_tokens_seen": 186418360, + "router_z_loss_clip": 0.08007812, + "router_z_loss_mlp": 0.00976562, + "step": 6530, + "time_per_iteration": 2.983119010925293 + }, + { + "auxiliary_loss_clip": 0.01076651, + "auxiliary_loss_mlp": 0.01030673, + "balance_loss_clip": 1.02362299, + "balance_loss_mlp": 1.01446092, + "epoch": 0.18951308687830073, + "flos": 25659450910080.0, + "grad_norm": 1.8331995895168143, + "language_loss": 0.81987405, + "learning_rate": 3.7389943979429374e-06, + "loss": 0.84094727, + "num_input_tokens_seen": 186433070, + "router_z_loss_clip": 0.53076172, + "router_z_loss_mlp": 0.16204834, + "step": 6531, + "time_per_iteration": 2.4363760948181152 + }, + { + "auxiliary_loss_clip": 0.01073239, + "auxiliary_loss_mlp": 0.01026866, + "balance_loss_clip": 1.02335835, + "balance_loss_mlp": 1.01245904, + "epoch": 0.1895421043468168, + "flos": 19530133931520.0, + "grad_norm": 2.105041441707559, + "language_loss": 0.69215655, + "learning_rate": 3.738901548837651e-06, + "loss": 0.71315765, + "num_input_tokens_seen": 186446755, + "router_z_loss_clip": 0.4987793, + "router_z_loss_mlp": 0.14416504, + "step": 6532, + "time_per_iteration": 2.367215871810913 + }, + { + "auxiliary_loss_clip": 0.0100917, + "auxiliary_loss_mlp": 0.01002099, + "balance_loss_clip": 1.00150967, + "balance_loss_mlp": 1.00106823, + "epoch": 0.18957112181533284, + "flos": 60354806774400.0, + "grad_norm": 0.7062013162928775, + "language_loss": 0.53210664, + "learning_rate": 3.738808684373682e-06, + "loss": 0.55221933, + "num_input_tokens_seen": 186508145, + "router_z_loss_clip": 0.07666016, + "router_z_loss_mlp": 0.01031494, + "step": 6533, + "time_per_iteration": 3.0214390754699707 + }, + { + "auxiliary_loss_clip": 0.0107243, + "auxiliary_loss_mlp": 0.0102961, + "balance_loss_clip": 1.02262652, + "balance_loss_mlp": 1.01457167, + "epoch": 0.1896001392838489, + "flos": 12231645609600.0, + "grad_norm": 3.0125946676466637, + "language_loss": 1.09018326, + "learning_rate": 3.7387158045518517e-06, + "loss": 1.11120367, + "num_input_tokens_seen": 186519375, + "router_z_loss_clip": 0.49829102, + "router_z_loss_mlp": 0.15045166, + "step": 6534, + "time_per_iteration": 2.3228931427001953 + }, + { + "auxiliary_loss_clip": 0.01069754, + "auxiliary_loss_mlp": 0.01031705, + "balance_loss_clip": 1.02129161, + "balance_loss_mlp": 1.01566553, + "epoch": 0.1896291567523649, + "flos": 43245586494720.0, + "grad_norm": 2.6733479293950237, + "language_loss": 0.87808096, + "learning_rate": 3.7386229093729787e-06, + "loss": 0.89909548, + "num_input_tokens_seen": 186537580, + "router_z_loss_clip": 0.48510742, + "router_z_loss_mlp": 0.16027832, + "step": 6535, + "time_per_iteration": 2.6283655166625977 + }, + { + "auxiliary_loss_clip": 0.01072648, + "auxiliary_loss_mlp": 0.01040844, + "balance_loss_clip": 1.02467799, + "balance_loss_mlp": 1.02572203, + "epoch": 0.18965817422088097, + "flos": 18507597765120.0, + "grad_norm": 2.183714408515999, + "language_loss": 0.80011082, + "learning_rate": 3.738529998837886e-06, + "loss": 0.82124567, + "num_input_tokens_seen": 186551035, + "router_z_loss_clip": 0.47998047, + "router_z_loss_mlp": 0.15112305, + "step": 6536, + "time_per_iteration": 2.6101553440093994 + }, + { + "auxiliary_loss_clip": 0.01073587, + "auxiliary_loss_mlp": 0.01029332, + "balance_loss_clip": 1.0226959, + "balance_loss_mlp": 1.01497912, + "epoch": 0.18968719168939702, + "flos": 11356198467840.0, + "grad_norm": 2.9773944866869515, + "language_loss": 0.97715497, + "learning_rate": 3.7384370729473922e-06, + "loss": 0.99818426, + "num_input_tokens_seen": 186561495, + "router_z_loss_clip": 0.50878906, + "router_z_loss_mlp": 0.14367676, + "step": 6537, + "time_per_iteration": 2.4561972618103027 + }, + { + "auxiliary_loss_clip": 0.01075529, + "auxiliary_loss_mlp": 0.01034684, + "balance_loss_clip": 1.0245651, + "balance_loss_mlp": 1.01853108, + "epoch": 0.18971620915791307, + "flos": 16645959048960.0, + "grad_norm": 3.0156355760825675, + "language_loss": 0.66588604, + "learning_rate": 3.738344131702318e-06, + "loss": 0.68698812, + "num_input_tokens_seen": 186574660, + "router_z_loss_clip": 0.51074219, + "router_z_loss_mlp": 0.16149902, + "step": 6538, + "time_per_iteration": 2.3483803272247314 + }, + { + "auxiliary_loss_clip": 0.01073084, + "auxiliary_loss_mlp": 0.01033085, + "balance_loss_clip": 1.02363276, + "balance_loss_mlp": 1.01728392, + "epoch": 0.18974522662642912, + "flos": 42665609122560.0, + "grad_norm": 2.79394276205582, + "language_loss": 0.9427402, + "learning_rate": 3.738251175103486e-06, + "loss": 0.96380186, + "num_input_tokens_seen": 186596180, + "router_z_loss_clip": 0.49462891, + "router_z_loss_mlp": 0.15795898, + "step": 6539, + "time_per_iteration": 2.516695022583008 + }, + { + "auxiliary_loss_clip": 0.01075615, + "auxiliary_loss_mlp": 0.01032844, + "balance_loss_clip": 1.02381492, + "balance_loss_mlp": 1.01813996, + "epoch": 0.18977424409494514, + "flos": 20478270257280.0, + "grad_norm": 2.8422060812381904, + "language_loss": 0.89221215, + "learning_rate": 3.738158203151716e-06, + "loss": 0.9132967, + "num_input_tokens_seen": 186608745, + "router_z_loss_clip": 0.51855469, + "router_z_loss_mlp": 0.14709473, + "step": 6540, + "time_per_iteration": 2.34554123878479 + }, + { + "auxiliary_loss_clip": 0.01078005, + "auxiliary_loss_mlp": 0.01032599, + "balance_loss_clip": 1.02376616, + "balance_loss_mlp": 1.01450253, + "epoch": 0.1898032615634612, + "flos": 13582609735680.0, + "grad_norm": 3.5461584948858498, + "language_loss": 0.76369941, + "learning_rate": 3.7380652158478295e-06, + "loss": 0.78480542, + "num_input_tokens_seen": 186622140, + "router_z_loss_clip": 0.54174805, + "router_z_loss_mlp": 0.18103027, + "step": 6541, + "time_per_iteration": 2.3452131748199463 + }, + { + "auxiliary_loss_clip": 0.01076998, + "auxiliary_loss_mlp": 0.01035903, + "balance_loss_clip": 1.0231787, + "balance_loss_mlp": 1.01940477, + "epoch": 0.18983227903197725, + "flos": 24308556606720.0, + "grad_norm": 2.1600042803482933, + "language_loss": 0.86499578, + "learning_rate": 3.7379722131926474e-06, + "loss": 0.88612473, + "num_input_tokens_seen": 186638800, + "router_z_loss_clip": 0.53857422, + "router_z_loss_mlp": 0.16497803, + "step": 6542, + "time_per_iteration": 2.3785414695739746 + }, + { + "auxiliary_loss_clip": 0.01072706, + "auxiliary_loss_mlp": 0.01031874, + "balance_loss_clip": 1.02236247, + "balance_loss_mlp": 1.01657939, + "epoch": 0.1898612965004933, + "flos": 40288512960000.0, + "grad_norm": 2.3354633378563268, + "language_loss": 0.7524851, + "learning_rate": 3.7378791951869913e-06, + "loss": 0.7735309, + "num_input_tokens_seen": 186656005, + "router_z_loss_clip": 0.50341797, + "router_z_loss_mlp": 0.15283203, + "step": 6543, + "time_per_iteration": 4.663161516189575 + }, + { + "auxiliary_loss_clip": 0.01072272, + "auxiliary_loss_mlp": 0.01026923, + "balance_loss_clip": 1.02252328, + "balance_loss_mlp": 1.01304126, + "epoch": 0.18989031396900935, + "flos": 25987948692480.0, + "grad_norm": 2.6786881220806897, + "language_loss": 0.75129271, + "learning_rate": 3.737786161831683e-06, + "loss": 0.77228463, + "num_input_tokens_seen": 186678790, + "router_z_loss_clip": 0.49731445, + "router_z_loss_mlp": 0.13873291, + "step": 6544, + "time_per_iteration": 4.88755989074707 + }, + { + "auxiliary_loss_clip": 0.01008458, + "auxiliary_loss_mlp": 0.01012839, + "balance_loss_clip": 1.00090575, + "balance_loss_mlp": 1.01192713, + "epoch": 0.1899193314375254, + "flos": 60213198833280.0, + "grad_norm": 0.6217619584770381, + "language_loss": 0.44751859, + "learning_rate": 3.7376931131275447e-06, + "loss": 0.46773154, + "num_input_tokens_seen": 186741460, + "router_z_loss_clip": 0.07568359, + "router_z_loss_mlp": 0.00909424, + "step": 6545, + "time_per_iteration": 3.066804885864258 + }, + { + "auxiliary_loss_clip": 0.01008317, + "auxiliary_loss_mlp": 0.01007495, + "balance_loss_clip": 1.00089586, + "balance_loss_mlp": 1.00659478, + "epoch": 0.18994834890604143, + "flos": 61930890547200.0, + "grad_norm": 0.6894783528366126, + "language_loss": 0.49624676, + "learning_rate": 3.7376000490753974e-06, + "loss": 0.51640487, + "num_input_tokens_seen": 186801435, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00897217, + "step": 6546, + "time_per_iteration": 2.9321720600128174 + }, + { + "auxiliary_loss_clip": 0.01070729, + "auxiliary_loss_mlp": 0.01029671, + "balance_loss_clip": 1.02212667, + "balance_loss_mlp": 1.01377487, + "epoch": 0.18997736637455748, + "flos": 11683718732160.0, + "grad_norm": 2.960042704342877, + "language_loss": 0.89691585, + "learning_rate": 3.7375069696760627e-06, + "loss": 0.91791975, + "num_input_tokens_seen": 186812960, + "router_z_loss_clip": 0.48632812, + "router_z_loss_mlp": 0.15893555, + "step": 6547, + "time_per_iteration": 2.424307346343994 + }, + { + "auxiliary_loss_clip": 0.01079138, + "auxiliary_loss_mlp": 0.01033842, + "balance_loss_clip": 1.02605653, + "balance_loss_mlp": 1.01816022, + "epoch": 0.19000638384307353, + "flos": 17011360005120.0, + "grad_norm": 2.730952100695959, + "language_loss": 0.74852276, + "learning_rate": 3.737413874930364e-06, + "loss": 0.76965255, + "num_input_tokens_seen": 186826330, + "router_z_loss_clip": 0.53100586, + "router_z_loss_mlp": 0.15673828, + "step": 6548, + "time_per_iteration": 2.6766586303710938 + }, + { + "auxiliary_loss_clip": 0.01078787, + "auxiliary_loss_mlp": 0.01033922, + "balance_loss_clip": 1.02453494, + "balance_loss_mlp": 1.0157423, + "epoch": 0.19003540131158958, + "flos": 36166108331520.0, + "grad_norm": 3.262207782086176, + "language_loss": 0.93097675, + "learning_rate": 3.7373207648391226e-06, + "loss": 0.95210385, + "num_input_tokens_seen": 186842965, + "router_z_loss_clip": 0.54199219, + "router_z_loss_mlp": 0.18170166, + "step": 6549, + "time_per_iteration": 2.56880521774292 + }, + { + "auxiliary_loss_clip": 0.01085506, + "auxiliary_loss_mlp": 0.01050354, + "balance_loss_clip": 1.02533817, + "balance_loss_mlp": 1.0304637, + "epoch": 0.19006441878010563, + "flos": 44264980638720.0, + "grad_norm": 2.8622468427109475, + "language_loss": 0.84045208, + "learning_rate": 3.7372276394031614e-06, + "loss": 0.86181068, + "num_input_tokens_seen": 186860500, + "router_z_loss_clip": 0.6015625, + "router_z_loss_mlp": 0.19891357, + "step": 6550, + "time_per_iteration": 2.5552594661712646 + }, + { + "auxiliary_loss_clip": 0.01083001, + "auxiliary_loss_mlp": 0.0104676, + "balance_loss_clip": 1.02688086, + "balance_loss_mlp": 1.02892661, + "epoch": 0.19009343624862168, + "flos": 19274569712640.0, + "grad_norm": 2.0654150087804517, + "language_loss": 0.83521354, + "learning_rate": 3.7371344986233025e-06, + "loss": 0.85651124, + "num_input_tokens_seen": 186874780, + "router_z_loss_clip": 0.56176758, + "router_z_loss_mlp": 0.1784668, + "step": 6551, + "time_per_iteration": 2.5176777839660645 + }, + { + "auxiliary_loss_clip": 0.01080168, + "auxiliary_loss_mlp": 0.01039306, + "balance_loss_clip": 1.02652538, + "balance_loss_mlp": 1.02167535, + "epoch": 0.1901224537171377, + "flos": 35801021577600.0, + "grad_norm": 2.10157451580086, + "language_loss": 0.77059597, + "learning_rate": 3.737041342500369e-06, + "loss": 0.79179072, + "num_input_tokens_seen": 186895035, + "router_z_loss_clip": 0.53759766, + "router_z_loss_mlp": 0.17633057, + "step": 6552, + "time_per_iteration": 4.959331274032593 + }, + { + "auxiliary_loss_clip": 0.01008949, + "auxiliary_loss_mlp": 0.01028151, + "balance_loss_clip": 1.00141573, + "balance_loss_mlp": 1.02737021, + "epoch": 0.19015147118565376, + "flos": 73935356739840.0, + "grad_norm": 0.6779326648835863, + "language_loss": 0.47370559, + "learning_rate": 3.7369481710351833e-06, + "loss": 0.49407655, + "num_input_tokens_seen": 186961900, + "router_z_loss_clip": 0.07519531, + "router_z_loss_mlp": 0.0078125, + "step": 6553, + "time_per_iteration": 5.694151163101196 + }, + { + "auxiliary_loss_clip": 0.01074386, + "auxiliary_loss_mlp": 0.01041631, + "balance_loss_clip": 1.02381825, + "balance_loss_mlp": 1.02558506, + "epoch": 0.1901804886541698, + "flos": 28028761839360.0, + "grad_norm": 2.1729509703737926, + "language_loss": 0.93848604, + "learning_rate": 3.7368549842285684e-06, + "loss": 0.95964617, + "num_input_tokens_seen": 186979585, + "router_z_loss_clip": 0.50512695, + "router_z_loss_mlp": 0.16064453, + "step": 6554, + "time_per_iteration": 2.4290332794189453 + }, + { + "auxiliary_loss_clip": 0.01075222, + "auxiliary_loss_mlp": 0.01036493, + "balance_loss_clip": 1.02530885, + "balance_loss_mlp": 1.02252769, + "epoch": 0.19020950612268586, + "flos": 34050998211840.0, + "grad_norm": 1.720409056290353, + "language_loss": 0.88395512, + "learning_rate": 3.7367617820813474e-06, + "loss": 0.90507227, + "num_input_tokens_seen": 187008615, + "router_z_loss_clip": 0.49951172, + "router_z_loss_mlp": 0.13964844, + "step": 6555, + "time_per_iteration": 2.7062671184539795 + }, + { + "auxiliary_loss_clip": 0.01008497, + "auxiliary_loss_mlp": 0.01009103, + "balance_loss_clip": 1.00117469, + "balance_loss_mlp": 1.00828087, + "epoch": 0.1902385235912019, + "flos": 62408188010880.0, + "grad_norm": 0.6597798990976714, + "language_loss": 0.50473714, + "learning_rate": 3.736668564594344e-06, + "loss": 0.52491319, + "num_input_tokens_seen": 187067830, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00823975, + "step": 6556, + "time_per_iteration": 3.022799253463745 + }, + { + "auxiliary_loss_clip": 0.01086622, + "auxiliary_loss_mlp": 0.01038246, + "balance_loss_clip": 1.02765727, + "balance_loss_mlp": 1.01973271, + "epoch": 0.19026754105971794, + "flos": 20843112631680.0, + "grad_norm": 2.552012909255624, + "language_loss": 0.85653532, + "learning_rate": 3.7365753317683808e-06, + "loss": 0.87778401, + "num_input_tokens_seen": 187083890, + "router_z_loss_clip": 0.59033203, + "router_z_loss_mlp": 0.18505859, + "step": 6557, + "time_per_iteration": 2.5473506450653076 + }, + { + "auxiliary_loss_clip": 0.01075716, + "auxiliary_loss_mlp": 0.01033241, + "balance_loss_clip": 1.02422976, + "balance_loss_mlp": 1.01766598, + "epoch": 0.190296558528234, + "flos": 74738186386560.0, + "grad_norm": 1.4943035635789939, + "language_loss": 0.80614591, + "learning_rate": 3.736482083604281e-06, + "loss": 0.82723552, + "num_input_tokens_seen": 187116285, + "router_z_loss_clip": 0.51513672, + "router_z_loss_mlp": 0.15563965, + "step": 6558, + "time_per_iteration": 2.823439598083496 + }, + { + "auxiliary_loss_clip": 0.01082073, + "auxiliary_loss_mlp": 0.01035033, + "balance_loss_clip": 1.02659869, + "balance_loss_mlp": 1.01854062, + "epoch": 0.19032557599675004, + "flos": 28686176340480.0, + "grad_norm": 3.076317870137925, + "language_loss": 0.83699751, + "learning_rate": 3.7363888201028696e-06, + "loss": 0.85816848, + "num_input_tokens_seen": 187132870, + "router_z_loss_clip": 0.55541992, + "router_z_loss_mlp": 0.16503906, + "step": 6559, + "time_per_iteration": 2.64517879486084 + }, + { + "auxiliary_loss_clip": 0.01008592, + "auxiliary_loss_mlp": 0.01005041, + "balance_loss_clip": 1.0013566, + "balance_loss_mlp": 1.004058, + "epoch": 0.1903545934652661, + "flos": 74774354555520.0, + "grad_norm": 0.6419941290386458, + "language_loss": 0.51049185, + "learning_rate": 3.7362955412649688e-06, + "loss": 0.5306282, + "num_input_tokens_seen": 187199455, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.00982666, + "step": 6560, + "time_per_iteration": 3.1846978664398193 + }, + { + "auxiliary_loss_clip": 0.01080167, + "auxiliary_loss_mlp": 0.0103404, + "balance_loss_clip": 1.02560949, + "balance_loss_mlp": 1.01797938, + "epoch": 0.19038361093378214, + "flos": 37790790583680.0, + "grad_norm": 2.3235068041753553, + "language_loss": 0.65512669, + "learning_rate": 3.7362022470914034e-06, + "loss": 0.6762687, + "num_input_tokens_seen": 187221780, + "router_z_loss_clip": 0.54589844, + "router_z_loss_mlp": 0.16061401, + "step": 6561, + "time_per_iteration": 2.5593388080596924 + }, + { + "auxiliary_loss_clip": 0.01009302, + "auxiliary_loss_mlp": 0.01011621, + "balance_loss_clip": 1.00192285, + "balance_loss_mlp": 1.01066184, + "epoch": 0.1904126284022982, + "flos": 53320750156800.0, + "grad_norm": 1.8639758081404674, + "language_loss": 0.48562771, + "learning_rate": 3.7361089375829973e-06, + "loss": 0.50583696, + "num_input_tokens_seen": 187283200, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00958252, + "step": 6562, + "time_per_iteration": 2.9875497817993164 + }, + { + "auxiliary_loss_clip": 0.0100927, + "auxiliary_loss_mlp": 0.01013741, + "balance_loss_clip": 1.00188553, + "balance_loss_mlp": 1.01282954, + "epoch": 0.19044164587081422, + "flos": 62439714835200.0, + "grad_norm": 0.6157056772507377, + "language_loss": 0.48876077, + "learning_rate": 3.736015612740575e-06, + "loss": 0.50899088, + "num_input_tokens_seen": 187345580, + "router_z_loss_clip": 0.07373047, + "router_z_loss_mlp": 0.00909424, + "step": 6563, + "time_per_iteration": 2.978604555130005 + }, + { + "auxiliary_loss_clip": 0.01078822, + "auxiliary_loss_mlp": 0.01026529, + "balance_loss_clip": 1.02649617, + "balance_loss_mlp": 1.01082909, + "epoch": 0.19047066333933027, + "flos": 26898867642240.0, + "grad_norm": 2.0740150460613287, + "language_loss": 0.91631204, + "learning_rate": 3.7359222725649604e-06, + "loss": 0.93736559, + "num_input_tokens_seen": 187360060, + "router_z_loss_clip": 0.52392578, + "router_z_loss_mlp": 0.15698242, + "step": 6564, + "time_per_iteration": 2.4256653785705566 + }, + { + "auxiliary_loss_clip": 0.01008563, + "auxiliary_loss_mlp": 0.01000354, + "balance_loss_clip": 1.00123858, + "balance_loss_mlp": 0.99948937, + "epoch": 0.19049968080784632, + "flos": 63278503182720.0, + "grad_norm": 0.6553494771327243, + "language_loss": 0.45669511, + "learning_rate": 3.735828917056977e-06, + "loss": 0.47678426, + "num_input_tokens_seen": 187424760, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00866699, + "step": 6565, + "time_per_iteration": 3.1638405323028564 + }, + { + "auxiliary_loss_clip": 0.01075632, + "auxiliary_loss_mlp": 0.01028447, + "balance_loss_clip": 1.02475929, + "balance_loss_mlp": 1.0144695, + "epoch": 0.19052869827636237, + "flos": 24198859514880.0, + "grad_norm": 2.9440518572013876, + "language_loss": 0.71779507, + "learning_rate": 3.7357355462174504e-06, + "loss": 0.73883587, + "num_input_tokens_seen": 187439065, + "router_z_loss_clip": 0.50830078, + "router_z_loss_mlp": 0.13989258, + "step": 6566, + "time_per_iteration": 2.4225921630859375 + }, + { + "auxiliary_loss_clip": 0.01074726, + "auxiliary_loss_mlp": 0.0103064, + "balance_loss_clip": 1.02405643, + "balance_loss_mlp": 1.01584601, + "epoch": 0.19055771574487843, + "flos": 37481914851840.0, + "grad_norm": 3.3600177197457177, + "language_loss": 0.97461271, + "learning_rate": 3.735642160047205e-06, + "loss": 0.99566638, + "num_input_tokens_seen": 187458220, + "router_z_loss_clip": 0.5065918, + "router_z_loss_mlp": 0.14807129, + "step": 6567, + "time_per_iteration": 2.5432488918304443 + }, + { + "auxiliary_loss_clip": 0.01008065, + "auxiliary_loss_mlp": 0.01009354, + "balance_loss_clip": 1.00094557, + "balance_loss_mlp": 1.00848329, + "epoch": 0.19058673321339448, + "flos": 74765207779200.0, + "grad_norm": 0.6624447610694111, + "language_loss": 0.44307312, + "learning_rate": 3.735548758547066e-06, + "loss": 0.4632473, + "num_input_tokens_seen": 187524120, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00872803, + "step": 6568, + "time_per_iteration": 3.213107109069824 + }, + { + "auxiliary_loss_clip": 0.01070794, + "auxiliary_loss_mlp": 0.0102592, + "balance_loss_clip": 1.02231622, + "balance_loss_mlp": 1.01213944, + "epoch": 0.1906157506819105, + "flos": 22776044077440.0, + "grad_norm": 3.031160696220221, + "language_loss": 0.77470577, + "learning_rate": 3.735455341717858e-06, + "loss": 0.79567289, + "num_input_tokens_seen": 187538340, + "router_z_loss_clip": 0.48461914, + "router_z_loss_mlp": 0.13775635, + "step": 6569, + "time_per_iteration": 2.4138076305389404 + }, + { + "auxiliary_loss_clip": 0.01079326, + "auxiliary_loss_mlp": 0.01037255, + "balance_loss_clip": 1.02394998, + "balance_loss_mlp": 1.01972485, + "epoch": 0.19064476815042655, + "flos": 16390569386880.0, + "grad_norm": 3.794379696330834, + "language_loss": 0.84144026, + "learning_rate": 3.735361909560406e-06, + "loss": 0.86260605, + "num_input_tokens_seen": 187551825, + "router_z_loss_clip": 0.5534668, + "router_z_loss_mlp": 0.17529297, + "step": 6570, + "time_per_iteration": 2.3578808307647705 + }, + { + "auxiliary_loss_clip": 0.01075145, + "auxiliary_loss_mlp": 0.0102947, + "balance_loss_clip": 1.02352941, + "balance_loss_mlp": 1.01400256, + "epoch": 0.1906737856189426, + "flos": 38214008484480.0, + "grad_norm": 1.9491815205957947, + "language_loss": 0.71225035, + "learning_rate": 3.7352684620755356e-06, + "loss": 0.73329651, + "num_input_tokens_seen": 187571125, + "router_z_loss_clip": 0.5168457, + "router_z_loss_mlp": 0.15447998, + "step": 6571, + "time_per_iteration": 2.5521578788757324 + }, + { + "auxiliary_loss_clip": 0.01009482, + "auxiliary_loss_mlp": 0.01002622, + "balance_loss_clip": 1.00222695, + "balance_loss_mlp": 1.00180542, + "epoch": 0.19070280308745866, + "flos": 62335847940480.0, + "grad_norm": 0.6706745622989576, + "language_loss": 0.45093274, + "learning_rate": 3.735174999264072e-06, + "loss": 0.47105378, + "num_input_tokens_seen": 187630460, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.00817871, + "step": 6572, + "time_per_iteration": 3.1071460247039795 + }, + { + "auxiliary_loss_clip": 0.01009383, + "auxiliary_loss_mlp": 0.01002843, + "balance_loss_clip": 1.00180936, + "balance_loss_mlp": 1.00188363, + "epoch": 0.1907318205559747, + "flos": 74770130280960.0, + "grad_norm": 0.637764710226497, + "language_loss": 0.48332256, + "learning_rate": 3.7350815211268405e-06, + "loss": 0.50344479, + "num_input_tokens_seen": 187691940, + "router_z_loss_clip": 0.07617188, + "router_z_loss_mlp": 0.00958252, + "step": 6573, + "time_per_iteration": 3.032285213470459 + }, + { + "auxiliary_loss_clip": 0.01076355, + "auxiliary_loss_mlp": 0.01031465, + "balance_loss_clip": 1.02418327, + "balance_loss_mlp": 1.01503825, + "epoch": 0.19076083802449073, + "flos": 23323307639040.0, + "grad_norm": 1.7954095387861675, + "language_loss": 0.67720807, + "learning_rate": 3.734988027664667e-06, + "loss": 0.69828618, + "num_input_tokens_seen": 187707385, + "router_z_loss_clip": 0.52099609, + "router_z_loss_mlp": 0.16436768, + "step": 6574, + "time_per_iteration": 2.4081196784973145 + }, + { + "auxiliary_loss_clip": 0.01072017, + "auxiliary_loss_mlp": 0.01026852, + "balance_loss_clip": 1.02339768, + "balance_loss_mlp": 1.01237965, + "epoch": 0.19078985549300678, + "flos": 16248821800320.0, + "grad_norm": 5.976017139834481, + "language_loss": 0.78782642, + "learning_rate": 3.7348945188783772e-06, + "loss": 0.80881512, + "num_input_tokens_seen": 187723315, + "router_z_loss_clip": 0.48681641, + "router_z_loss_mlp": 0.14471436, + "step": 6575, + "time_per_iteration": 2.47041916847229 + }, + { + "auxiliary_loss_clip": 0.01073921, + "auxiliary_loss_mlp": 0.01030444, + "balance_loss_clip": 1.02378964, + "balance_loss_mlp": 1.0154413, + "epoch": 0.19081887296152283, + "flos": 12741796529280.0, + "grad_norm": 3.253653947202121, + "language_loss": 0.98182404, + "learning_rate": 3.7348009947687966e-06, + "loss": 1.0028677, + "num_input_tokens_seen": 187734475, + "router_z_loss_clip": 0.50146484, + "router_z_loss_mlp": 0.15002441, + "step": 6576, + "time_per_iteration": 2.3184244632720947 + }, + { + "auxiliary_loss_clip": 0.01010323, + "auxiliary_loss_mlp": 0.01001306, + "balance_loss_clip": 1.00294447, + "balance_loss_mlp": 1.00035191, + "epoch": 0.19084789043003889, + "flos": 67830584313600.0, + "grad_norm": 0.6802937413020712, + "language_loss": 0.4748092, + "learning_rate": 3.7347074553367515e-06, + "loss": 0.49492547, + "num_input_tokens_seen": 187793645, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00952148, + "step": 6577, + "time_per_iteration": 2.9963576793670654 + }, + { + "auxiliary_loss_clip": 0.01077583, + "auxiliary_loss_mlp": 0.0103172, + "balance_loss_clip": 1.02513909, + "balance_loss_mlp": 1.01523316, + "epoch": 0.19087690789855494, + "flos": 22592750106240.0, + "grad_norm": 2.483711757080598, + "language_loss": 0.71093291, + "learning_rate": 3.734613900583069e-06, + "loss": 0.73202598, + "num_input_tokens_seen": 187805175, + "router_z_loss_clip": 0.52392578, + "router_z_loss_mlp": 0.16485596, + "step": 6578, + "time_per_iteration": 2.371609687805176 + }, + { + "auxiliary_loss_clip": 0.01069413, + "auxiliary_loss_mlp": 0.01031299, + "balance_loss_clip": 1.02204955, + "balance_loss_mlp": 1.01754236, + "epoch": 0.190905925367071, + "flos": 42556924460160.0, + "grad_norm": 2.2677930483422553, + "language_loss": 1.02629483, + "learning_rate": 3.734520330508574e-06, + "loss": 1.04730201, + "num_input_tokens_seen": 187826980, + "router_z_loss_clip": 0.47436523, + "router_z_loss_mlp": 0.13757324, + "step": 6579, + "time_per_iteration": 2.57047963142395 + }, + { + "auxiliary_loss_clip": 0.01076206, + "auxiliary_loss_mlp": 0.01035416, + "balance_loss_clip": 1.02334523, + "balance_loss_mlp": 1.01882195, + "epoch": 0.190934942835587, + "flos": 11611657952640.0, + "grad_norm": 10.66168052261152, + "language_loss": 0.88119912, + "learning_rate": 3.7344267451140938e-06, + "loss": 0.90231526, + "num_input_tokens_seen": 187838195, + "router_z_loss_clip": 0.52905273, + "router_z_loss_mlp": 0.16589355, + "step": 6580, + "time_per_iteration": 2.3684170246124268 + }, + { + "auxiliary_loss_clip": 0.01010909, + "auxiliary_loss_mlp": 0.01006148, + "balance_loss_clip": 1.00361192, + "balance_loss_mlp": 1.00516486, + "epoch": 0.19096396030410306, + "flos": 53859774637440.0, + "grad_norm": 0.7355260468559067, + "language_loss": 0.52026129, + "learning_rate": 3.7343331444004542e-06, + "loss": 0.54043186, + "num_input_tokens_seen": 187898375, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00982666, + "step": 6581, + "time_per_iteration": 3.0191516876220703 + }, + { + "auxiliary_loss_clip": 0.010098, + "auxiliary_loss_mlp": 0.01006579, + "balance_loss_clip": 1.00252807, + "balance_loss_mlp": 1.00571477, + "epoch": 0.19099297777261912, + "flos": 74774982960000.0, + "grad_norm": 0.6465965786098483, + "language_loss": 0.4601582, + "learning_rate": 3.734239528368483e-06, + "loss": 0.48032197, + "num_input_tokens_seen": 187964160, + "router_z_loss_clip": 0.07275391, + "router_z_loss_mlp": 0.00866699, + "step": 6582, + "time_per_iteration": 3.053096294403076 + }, + { + "auxiliary_loss_clip": 0.01011014, + "auxiliary_loss_mlp": 0.01001262, + "balance_loss_clip": 1.00354528, + "balance_loss_mlp": 1.00027227, + "epoch": 0.19102199524113517, + "flos": 56203423850880.0, + "grad_norm": 2.2470524588344136, + "language_loss": 0.4383778, + "learning_rate": 3.7341458970190065e-06, + "loss": 0.45850056, + "num_input_tokens_seen": 188026835, + "router_z_loss_clip": 0.07519531, + "router_z_loss_mlp": 0.0098877, + "step": 6583, + "time_per_iteration": 2.969944715499878 + }, + { + "auxiliary_loss_clip": 0.01077422, + "auxiliary_loss_mlp": 0.01039329, + "balance_loss_clip": 1.02556443, + "balance_loss_mlp": 1.02345085, + "epoch": 0.19105101270965122, + "flos": 32346853971840.0, + "grad_norm": 1.4668462100117687, + "language_loss": 0.88582385, + "learning_rate": 3.7340522503528512e-06, + "loss": 0.90699142, + "num_input_tokens_seen": 188055020, + "router_z_loss_clip": 0.51904297, + "router_z_loss_mlp": 0.15881348, + "step": 6584, + "time_per_iteration": 2.562804698944092 + }, + { + "auxiliary_loss_clip": 0.01072042, + "auxiliary_loss_mlp": 0.01028058, + "balance_loss_clip": 1.02256858, + "balance_loss_mlp": 1.01349664, + "epoch": 0.19108003017816727, + "flos": 21351762362880.0, + "grad_norm": 2.440981145561138, + "language_loss": 0.84710252, + "learning_rate": 3.7339585883708457e-06, + "loss": 0.8681035, + "num_input_tokens_seen": 188069215, + "router_z_loss_clip": 0.49438477, + "router_z_loss_mlp": 0.14550781, + "step": 6585, + "time_per_iteration": 2.5922539234161377 + }, + { + "auxiliary_loss_clip": 0.01010016, + "auxiliary_loss_mlp": 0.01001277, + "balance_loss_clip": 1.00234199, + "balance_loss_mlp": 1.0001682, + "epoch": 0.1911090476466833, + "flos": 72621679812480.0, + "grad_norm": 0.7052672824815657, + "language_loss": 0.52798283, + "learning_rate": 3.7338649110738158e-06, + "loss": 0.5480957, + "num_input_tokens_seen": 188129955, + "router_z_loss_clip": 0.07714844, + "router_z_loss_mlp": 0.0111084, + "step": 6586, + "time_per_iteration": 3.042853832244873 + }, + { + "auxiliary_loss_clip": 0.01009442, + "auxiliary_loss_mlp": 0.01001333, + "balance_loss_clip": 1.00216508, + "balance_loss_mlp": 1.00031972, + "epoch": 0.19113806511519935, + "flos": 65668586371200.0, + "grad_norm": 0.6293735646394717, + "language_loss": 0.45193467, + "learning_rate": 3.733771218462589e-06, + "loss": 0.47204244, + "num_input_tokens_seen": 188200245, + "router_z_loss_clip": 0.07275391, + "router_z_loss_mlp": 0.01013184, + "step": 6587, + "time_per_iteration": 3.179457187652588 + }, + { + "auxiliary_loss_clip": 0.01010089, + "auxiliary_loss_mlp": 0.01006314, + "balance_loss_clip": 1.00250673, + "balance_loss_mlp": 1.00518787, + "epoch": 0.1911670825837154, + "flos": 60285294524160.0, + "grad_norm": 0.681563404313213, + "language_loss": 0.45591632, + "learning_rate": 3.7336775105379937e-06, + "loss": 0.47608036, + "num_input_tokens_seen": 188263695, + "router_z_loss_clip": 0.07568359, + "router_z_loss_mlp": 0.0112915, + "step": 6588, + "time_per_iteration": 3.1242191791534424 + }, + { + "auxiliary_loss_clip": 0.01009957, + "auxiliary_loss_mlp": 0.0100352, + "balance_loss_clip": 1.00234771, + "balance_loss_mlp": 1.00247145, + "epoch": 0.19119610005223145, + "flos": 60503327164800.0, + "grad_norm": 0.603986271971725, + "language_loss": 0.47290322, + "learning_rate": 3.7335837873008567e-06, + "loss": 0.49303803, + "num_input_tokens_seen": 188322555, + "router_z_loss_clip": 0.07617188, + "router_z_loss_mlp": 0.01049805, + "step": 6589, + "time_per_iteration": 2.8978798389434814 + }, + { + "auxiliary_loss_clip": 0.01077125, + "auxiliary_loss_mlp": 0.01039302, + "balance_loss_clip": 1.02460861, + "balance_loss_mlp": 1.02232111, + "epoch": 0.1912251175207475, + "flos": 37263917122560.0, + "grad_norm": 1.8604081727081323, + "language_loss": 0.79663211, + "learning_rate": 3.7334900487520063e-06, + "loss": 0.81779635, + "num_input_tokens_seen": 188341815, + "router_z_loss_clip": 0.52441406, + "router_z_loss_mlp": 0.16992188, + "step": 6590, + "time_per_iteration": 2.559145927429199 + }, + { + "auxiliary_loss_clip": 0.01074022, + "auxiliary_loss_mlp": 0.01027854, + "balance_loss_clip": 1.02215004, + "balance_loss_mlp": 1.01257777, + "epoch": 0.19125413498926352, + "flos": 12276473662080.0, + "grad_norm": 3.350507901310129, + "language_loss": 0.752599, + "learning_rate": 3.7333962948922705e-06, + "loss": 0.77361774, + "num_input_tokens_seen": 188356515, + "router_z_loss_clip": 0.51855469, + "router_z_loss_mlp": 0.152771, + "step": 6591, + "time_per_iteration": 2.4282939434051514 + }, + { + "auxiliary_loss_clip": 0.01077477, + "auxiliary_loss_mlp": 0.01031429, + "balance_loss_clip": 1.02560711, + "balance_loss_mlp": 1.01541901, + "epoch": 0.19128315245777958, + "flos": 70755329928960.0, + "grad_norm": 2.3214834286321993, + "language_loss": 0.91608799, + "learning_rate": 3.7333025257224772e-06, + "loss": 0.937177, + "num_input_tokens_seen": 188380815, + "router_z_loss_clip": 0.51904297, + "router_z_loss_mlp": 0.16015625, + "step": 6592, + "time_per_iteration": 2.7399728298187256 + }, + { + "auxiliary_loss_clip": 0.01082181, + "auxiliary_loss_mlp": 0.01033212, + "balance_loss_clip": 1.02748203, + "balance_loss_mlp": 1.01591432, + "epoch": 0.19131216992629563, + "flos": 15264096503040.0, + "grad_norm": 2.409773171230619, + "language_loss": 0.97305918, + "learning_rate": 3.733208741243454e-06, + "loss": 0.9942131, + "num_input_tokens_seen": 188395565, + "router_z_loss_clip": 0.54663086, + "router_z_loss_mlp": 0.17285156, + "step": 6593, + "time_per_iteration": 2.404744863510132 + }, + { + "auxiliary_loss_clip": 0.01071564, + "auxiliary_loss_mlp": 0.01031798, + "balance_loss_clip": 1.02254391, + "balance_loss_mlp": 1.01638675, + "epoch": 0.19134118739481168, + "flos": 36094606133760.0, + "grad_norm": 2.13012001584776, + "language_loss": 0.81303465, + "learning_rate": 3.733114941456031e-06, + "loss": 0.83406818, + "num_input_tokens_seen": 188411615, + "router_z_loss_clip": 0.4909668, + "router_z_loss_mlp": 0.1539917, + "step": 6594, + "time_per_iteration": 2.5140457153320312 + }, + { + "auxiliary_loss_clip": 0.0107472, + "auxiliary_loss_mlp": 0.01031377, + "balance_loss_clip": 1.02454412, + "balance_loss_mlp": 1.01651812, + "epoch": 0.19137020486332773, + "flos": 23870082441600.0, + "grad_norm": 1.898175386199621, + "language_loss": 0.71533918, + "learning_rate": 3.7330211263610354e-06, + "loss": 0.73640013, + "num_input_tokens_seen": 188425620, + "router_z_loss_clip": 0.50170898, + "router_z_loss_mlp": 0.14868164, + "step": 6595, + "time_per_iteration": 2.4435482025146484 + }, + { + "auxiliary_loss_clip": 0.01079503, + "auxiliary_loss_mlp": 0.01030667, + "balance_loss_clip": 1.0241791, + "balance_loss_mlp": 1.01398325, + "epoch": 0.19139922233184378, + "flos": 20952530432640.0, + "grad_norm": 3.0145196694575644, + "language_loss": 0.96792901, + "learning_rate": 3.7329272959592948e-06, + "loss": 0.98903072, + "num_input_tokens_seen": 188437815, + "router_z_loss_clip": 0.5534668, + "router_z_loss_mlp": 0.16668701, + "step": 6596, + "time_per_iteration": 2.3664939403533936 + }, + { + "auxiliary_loss_clip": 0.01009438, + "auxiliary_loss_mlp": 0.01004975, + "balance_loss_clip": 1.0022366, + "balance_loss_mlp": 1.00380695, + "epoch": 0.1914282398003598, + "flos": 74773342126080.0, + "grad_norm": 0.6296365614583747, + "language_loss": 0.47174501, + "learning_rate": 3.7328334502516396e-06, + "loss": 0.49188918, + "num_input_tokens_seen": 188504755, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.01165771, + "step": 6597, + "time_per_iteration": 3.1649439334869385 + }, + { + "auxiliary_loss_clip": 0.01080637, + "auxiliary_loss_mlp": 0.01041477, + "balance_loss_clip": 1.02494824, + "balance_loss_mlp": 1.02214098, + "epoch": 0.19145725726887586, + "flos": 20002020134400.0, + "grad_norm": 2.373010223446835, + "language_loss": 0.89089084, + "learning_rate": 3.732739589238898e-06, + "loss": 0.912112, + "num_input_tokens_seen": 188517895, + "router_z_loss_clip": 0.55712891, + "router_z_loss_mlp": 0.1932373, + "step": 6598, + "time_per_iteration": 2.6166906356811523 + }, + { + "auxiliary_loss_clip": 0.01010179, + "auxiliary_loss_mlp": 0.01008378, + "balance_loss_clip": 1.00301504, + "balance_loss_mlp": 1.0070312, + "epoch": 0.1914862747373919, + "flos": 74767127904000.0, + "grad_norm": 0.6488535988320514, + "language_loss": 0.50471503, + "learning_rate": 3.7326457129219e-06, + "loss": 0.52490062, + "num_input_tokens_seen": 188581265, + "router_z_loss_clip": 0.07177734, + "router_z_loss_mlp": 0.01348877, + "step": 6599, + "time_per_iteration": 3.1416351795196533 + }, + { + "auxiliary_loss_clip": 0.01079179, + "auxiliary_loss_mlp": 0.01032053, + "balance_loss_clip": 1.02526903, + "balance_loss_mlp": 1.01551831, + "epoch": 0.19151529220590796, + "flos": 16574945610240.0, + "grad_norm": 1.630087986935107, + "language_loss": 0.53708911, + "learning_rate": 3.7325518213014727e-06, + "loss": 0.55820149, + "num_input_tokens_seen": 188595440, + "router_z_loss_clip": 0.53930664, + "router_z_loss_mlp": 0.16540527, + "step": 6600, + "time_per_iteration": 2.37186598777771 + }, + { + "auxiliary_loss_clip": 0.0107101, + "auxiliary_loss_mlp": 0.01024978, + "balance_loss_clip": 1.02259898, + "balance_loss_mlp": 1.01128054, + "epoch": 0.191544309674424, + "flos": 24943940173440.0, + "grad_norm": 2.0239710556295782, + "language_loss": 0.93094432, + "learning_rate": 3.7324579143784474e-06, + "loss": 0.95190418, + "num_input_tokens_seen": 188613935, + "router_z_loss_clip": 0.48413086, + "router_z_loss_mlp": 0.13696289, + "step": 6601, + "time_per_iteration": 2.5197486877441406 + }, + { + "auxiliary_loss_clip": 0.0108145, + "auxiliary_loss_mlp": 0.01035517, + "balance_loss_clip": 1.02730775, + "balance_loss_mlp": 1.01889908, + "epoch": 0.19157332714294004, + "flos": 14967719038080.0, + "grad_norm": 3.505713853197459, + "language_loss": 1.12418699, + "learning_rate": 3.732363992153653e-06, + "loss": 1.14535666, + "num_input_tokens_seen": 188624715, + "router_z_loss_clip": 0.54125977, + "router_z_loss_mlp": 0.1661377, + "step": 6602, + "time_per_iteration": 2.3869400024414062 + }, + { + "auxiliary_loss_clip": 0.01077278, + "auxiliary_loss_mlp": 0.01036527, + "balance_loss_clip": 1.02563667, + "balance_loss_mlp": 1.02020693, + "epoch": 0.1916023446114561, + "flos": 28143346521600.0, + "grad_norm": 2.467335789496584, + "language_loss": 0.74322045, + "learning_rate": 3.732270054627918e-06, + "loss": 0.76435852, + "num_input_tokens_seen": 188643260, + "router_z_loss_clip": 0.51611328, + "router_z_loss_mlp": 0.16326904, + "step": 6603, + "time_per_iteration": 2.4745123386383057 + }, + { + "auxiliary_loss_clip": 0.01010066, + "auxiliary_loss_mlp": 0.01002181, + "balance_loss_clip": 1.0026474, + "balance_loss_mlp": 1.00104296, + "epoch": 0.19163136207997214, + "flos": 61716733067520.0, + "grad_norm": 0.6905805228939678, + "language_loss": 0.51730025, + "learning_rate": 3.7321761018020738e-06, + "loss": 0.53742266, + "num_input_tokens_seen": 188703380, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.01141357, + "step": 6604, + "time_per_iteration": 2.998775005340576 + }, + { + "auxiliary_loss_clip": 0.01010555, + "auxiliary_loss_mlp": 0.0100268, + "balance_loss_clip": 1.00326669, + "balance_loss_mlp": 1.00155354, + "epoch": 0.1916603795484882, + "flos": 55697392471680.0, + "grad_norm": 0.637414128833729, + "language_loss": 0.46934927, + "learning_rate": 3.7320821336769484e-06, + "loss": 0.4894816, + "num_input_tokens_seen": 188766235, + "router_z_loss_clip": 0.07275391, + "router_z_loss_mlp": 0.0112915, + "step": 6605, + "time_per_iteration": 3.0271730422973633 + }, + { + "auxiliary_loss_clip": 0.01079125, + "auxiliary_loss_mlp": 0.01034265, + "balance_loss_clip": 1.025033, + "balance_loss_mlp": 1.01697326, + "epoch": 0.19168939701700424, + "flos": 41863025721600.0, + "grad_norm": 3.5275706112560608, + "language_loss": 0.82531965, + "learning_rate": 3.7319881502533734e-06, + "loss": 0.84645355, + "num_input_tokens_seen": 188788130, + "router_z_loss_clip": 0.54101562, + "router_z_loss_mlp": 0.17279053, + "step": 6606, + "time_per_iteration": 2.6060609817504883 + }, + { + "auxiliary_loss_clip": 0.01076452, + "auxiliary_loss_mlp": 0.0103532, + "balance_loss_clip": 1.02560318, + "balance_loss_mlp": 1.02020478, + "epoch": 0.1917184144855203, + "flos": 39888687536640.0, + "grad_norm": 2.425478910737823, + "language_loss": 0.87736648, + "learning_rate": 3.7318941515321784e-06, + "loss": 0.89848423, + "num_input_tokens_seen": 188803490, + "router_z_loss_clip": 0.50927734, + "router_z_loss_mlp": 0.15112305, + "step": 6607, + "time_per_iteration": 2.4673242568969727 + }, + { + "auxiliary_loss_clip": 0.01009566, + "auxiliary_loss_mlp": 0.01001877, + "balance_loss_clip": 1.00241709, + "balance_loss_mlp": 1.00088787, + "epoch": 0.19174743195403632, + "flos": 61704060243840.0, + "grad_norm": 0.7736373516939467, + "language_loss": 0.43096668, + "learning_rate": 3.7318001375141926e-06, + "loss": 0.4510811, + "num_input_tokens_seen": 188860050, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.0098877, + "step": 6608, + "time_per_iteration": 2.8893778324127197 + }, + { + "auxiliary_loss_clip": 0.0108241, + "auxiliary_loss_mlp": 0.01035379, + "balance_loss_clip": 1.02662134, + "balance_loss_mlp": 1.0195179, + "epoch": 0.19177644942255237, + "flos": 31861876008960.0, + "grad_norm": 2.248795332824669, + "language_loss": 0.92554033, + "learning_rate": 3.731706108200248e-06, + "loss": 0.94671822, + "num_input_tokens_seen": 188877325, + "router_z_loss_clip": 0.55810547, + "router_z_loss_mlp": 0.15856934, + "step": 6609, + "time_per_iteration": 2.459843635559082 + }, + { + "auxiliary_loss_clip": 0.01009789, + "auxiliary_loss_mlp": 0.01001993, + "balance_loss_clip": 1.00254571, + "balance_loss_mlp": 1.00090194, + "epoch": 0.19180546689106842, + "flos": 66159570954240.0, + "grad_norm": 0.6363149320596486, + "language_loss": 0.4594039, + "learning_rate": 3.7316120635911733e-06, + "loss": 0.47952169, + "num_input_tokens_seen": 188938340, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.01092529, + "step": 6610, + "time_per_iteration": 2.9641268253326416 + }, + { + "auxiliary_loss_clip": 0.01073732, + "auxiliary_loss_mlp": 0.01027097, + "balance_loss_clip": 1.02391779, + "balance_loss_mlp": 1.01283967, + "epoch": 0.19183448435958447, + "flos": 31572026968320.0, + "grad_norm": 2.0093336045501595, + "language_loss": 0.89000118, + "learning_rate": 3.731518003687801e-06, + "loss": 0.91100955, + "num_input_tokens_seen": 188954410, + "router_z_loss_clip": 0.4987793, + "router_z_loss_mlp": 0.14251709, + "step": 6611, + "time_per_iteration": 2.592384099960327 + }, + { + "auxiliary_loss_clip": 0.01078844, + "auxiliary_loss_mlp": 0.01040306, + "balance_loss_clip": 1.02609801, + "balance_loss_mlp": 1.02450478, + "epoch": 0.19186350182810052, + "flos": 22630840266240.0, + "grad_norm": 3.113371592691988, + "language_loss": 0.81982696, + "learning_rate": 3.7314239284909606e-06, + "loss": 0.84101844, + "num_input_tokens_seen": 188969430, + "router_z_loss_clip": 0.52783203, + "router_z_loss_mlp": 0.15814209, + "step": 6612, + "time_per_iteration": 2.4131758213043213 + }, + { + "auxiliary_loss_clip": 0.01076477, + "auxiliary_loss_mlp": 0.01027223, + "balance_loss_clip": 1.02463627, + "balance_loss_mlp": 1.01114178, + "epoch": 0.19189251929661658, + "flos": 26098309100160.0, + "grad_norm": 1.9881350824959263, + "language_loss": 0.64478505, + "learning_rate": 3.7313298380014838e-06, + "loss": 0.66582203, + "num_input_tokens_seen": 188987035, + "router_z_loss_clip": 0.51831055, + "router_z_loss_mlp": 0.16094971, + "step": 6613, + "time_per_iteration": 2.41204571723938 + }, + { + "auxiliary_loss_clip": 0.0107392, + "auxiliary_loss_mlp": 0.0103268, + "balance_loss_clip": 1.02426744, + "balance_loss_mlp": 1.01839912, + "epoch": 0.1919215367651326, + "flos": 27920356467840.0, + "grad_norm": 2.064014905297189, + "language_loss": 0.63856494, + "learning_rate": 3.731235732220201e-06, + "loss": 0.65963101, + "num_input_tokens_seen": 189000300, + "router_z_loss_clip": 0.49609375, + "router_z_loss_mlp": 0.14294434, + "step": 6614, + "time_per_iteration": 2.5199570655822754 + }, + { + "auxiliary_loss_clip": 0.01072133, + "auxiliary_loss_mlp": 0.01029828, + "balance_loss_clip": 1.02475393, + "balance_loss_mlp": 1.01492667, + "epoch": 0.19195055423364865, + "flos": 12789592047360.0, + "grad_norm": 2.744610771304911, + "language_loss": 0.87426937, + "learning_rate": 3.7311416111479436e-06, + "loss": 0.895289, + "num_input_tokens_seen": 189014850, + "router_z_loss_clip": 0.47412109, + "router_z_loss_mlp": 0.14916992, + "step": 6615, + "time_per_iteration": 2.351038694381714 + }, + { + "auxiliary_loss_clip": 0.01085229, + "auxiliary_loss_mlp": 0.01040954, + "balance_loss_clip": 1.02964115, + "balance_loss_mlp": 1.02123666, + "epoch": 0.1919795717021647, + "flos": 30731248673280.0, + "grad_norm": 2.3311156432277667, + "language_loss": 0.91890562, + "learning_rate": 3.7310474747855434e-06, + "loss": 0.94016743, + "num_input_tokens_seen": 189032640, + "router_z_loss_clip": 0.55566406, + "router_z_loss_mlp": 0.19726562, + "step": 6616, + "time_per_iteration": 2.5262017250061035 + }, + { + "auxiliary_loss_clip": 0.01080961, + "auxiliary_loss_mlp": 0.0103726, + "balance_loss_clip": 1.02600026, + "balance_loss_mlp": 1.02004647, + "epoch": 0.19200858917068075, + "flos": 23724948453120.0, + "grad_norm": 2.105924001357739, + "language_loss": 0.7507295, + "learning_rate": 3.730953323133831e-06, + "loss": 0.77191174, + "num_input_tokens_seen": 189048555, + "router_z_loss_clip": 0.54931641, + "router_z_loss_mlp": 0.17211914, + "step": 6617, + "time_per_iteration": 2.3921072483062744 + }, + { + "auxiliary_loss_clip": 0.01082585, + "auxiliary_loss_mlp": 0.0103347, + "balance_loss_clip": 1.02708077, + "balance_loss_mlp": 1.01566005, + "epoch": 0.1920376066391968, + "flos": 26165028441600.0, + "grad_norm": 2.344879657330046, + "language_loss": 1.11432707, + "learning_rate": 3.7308591561936383e-06, + "loss": 1.13548756, + "num_input_tokens_seen": 189063310, + "router_z_loss_clip": 0.55517578, + "router_z_loss_mlp": 0.17816162, + "step": 6618, + "time_per_iteration": 2.465348243713379 + }, + { + "auxiliary_loss_clip": 0.01077042, + "auxiliary_loss_mlp": 0.01037634, + "balance_loss_clip": 1.02555633, + "balance_loss_mlp": 1.02185035, + "epoch": 0.19206662410771283, + "flos": 74728550851200.0, + "grad_norm": 2.8407144855706106, + "language_loss": 0.61167121, + "learning_rate": 3.7307649739657974e-06, + "loss": 0.63281792, + "num_input_tokens_seen": 189083930, + "router_z_loss_clip": 0.51489258, + "router_z_loss_mlp": 0.15765381, + "step": 6619, + "time_per_iteration": 5.067770957946777 + }, + { + "auxiliary_loss_clip": 0.01011666, + "auxiliary_loss_mlp": 0.01009973, + "balance_loss_clip": 1.00421929, + "balance_loss_mlp": 1.00906706, + "epoch": 0.19209564157622888, + "flos": 74482271187840.0, + "grad_norm": 0.8148283775156301, + "language_loss": 0.53115726, + "learning_rate": 3.7306707764511395e-06, + "loss": 0.55137366, + "num_input_tokens_seen": 189149295, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.0090332, + "step": 6620, + "time_per_iteration": 5.498613595962524 + }, + { + "auxiliary_loss_clip": 0.01083148, + "auxiliary_loss_mlp": 0.01035297, + "balance_loss_clip": 1.02621937, + "balance_loss_mlp": 1.01745093, + "epoch": 0.19212465904474493, + "flos": 13473436314240.0, + "grad_norm": 2.5507761880101185, + "language_loss": 0.78357112, + "learning_rate": 3.7305765636504977e-06, + "loss": 0.80475557, + "num_input_tokens_seen": 189162340, + "router_z_loss_clip": 0.56933594, + "router_z_loss_mlp": 0.17834473, + "step": 6621, + "time_per_iteration": 2.3436355590820312 + }, + { + "auxiliary_loss_clip": 0.01010346, + "auxiliary_loss_mlp": 0.01006264, + "balance_loss_clip": 1.0028007, + "balance_loss_mlp": 1.00542963, + "epoch": 0.19215367651326098, + "flos": 62111286875520.0, + "grad_norm": 0.6577668891295868, + "language_loss": 0.50127602, + "learning_rate": 3.7304823355647034e-06, + "loss": 0.52144206, + "num_input_tokens_seen": 189226640, + "router_z_loss_clip": 0.07519531, + "router_z_loss_mlp": 0.00836182, + "step": 6622, + "time_per_iteration": 3.281522750854492 + }, + { + "auxiliary_loss_clip": 0.01072847, + "auxiliary_loss_mlp": 0.01028198, + "balance_loss_clip": 1.02389526, + "balance_loss_mlp": 1.01415539, + "epoch": 0.19218269398177704, + "flos": 26826911596800.0, + "grad_norm": 2.1535021865862416, + "language_loss": 0.92118257, + "learning_rate": 3.7303880921945884e-06, + "loss": 0.94219309, + "num_input_tokens_seen": 189242570, + "router_z_loss_clip": 0.48950195, + "router_z_loss_mlp": 0.14031982, + "step": 6623, + "time_per_iteration": 2.4597082138061523 + }, + { + "auxiliary_loss_clip": 0.01080874, + "auxiliary_loss_mlp": 0.01034396, + "balance_loss_clip": 1.02590823, + "balance_loss_mlp": 1.01829672, + "epoch": 0.1922117114502931, + "flos": 32043459323520.0, + "grad_norm": 2.2508727958564823, + "language_loss": 0.79063165, + "learning_rate": 3.730293833540985e-06, + "loss": 0.81178439, + "num_input_tokens_seen": 189258585, + "router_z_loss_clip": 0.54980469, + "router_z_loss_mlp": 0.16094971, + "step": 6624, + "time_per_iteration": 2.504260301589966 + }, + { + "auxiliary_loss_clip": 0.01009695, + "auxiliary_loss_mlp": 0.01004397, + "balance_loss_clip": 1.00251222, + "balance_loss_mlp": 1.0036844, + "epoch": 0.1922407289188091, + "flos": 56819047587840.0, + "grad_norm": 0.736089103898097, + "language_loss": 0.52225477, + "learning_rate": 3.7301995596047274e-06, + "loss": 0.54239577, + "num_input_tokens_seen": 189314485, + "router_z_loss_clip": 0.07177734, + "router_z_loss_mlp": 0.0071106, + "step": 6625, + "time_per_iteration": 2.848489284515381 + }, + { + "auxiliary_loss_clip": 0.0107439, + "auxiliary_loss_mlp": 0.01030101, + "balance_loss_clip": 1.02541208, + "balance_loss_mlp": 1.01602197, + "epoch": 0.19226974638732516, + "flos": 24964993589760.0, + "grad_norm": 2.135047596563241, + "language_loss": 0.85875678, + "learning_rate": 3.7301052703866463e-06, + "loss": 0.87980163, + "num_input_tokens_seen": 189330830, + "router_z_loss_clip": 0.49047852, + "router_z_loss_mlp": 0.14080811, + "step": 6626, + "time_per_iteration": 2.4237468242645264 + }, + { + "auxiliary_loss_clip": 0.01072466, + "auxiliary_loss_mlp": 0.01038538, + "balance_loss_clip": 1.02426517, + "balance_loss_mlp": 1.02373242, + "epoch": 0.19229876385584121, + "flos": 74729702926080.0, + "grad_norm": 1.6377182906981291, + "language_loss": 0.57574081, + "learning_rate": 3.730010965887576e-06, + "loss": 0.59685087, + "num_input_tokens_seen": 189353705, + "router_z_loss_clip": 0.48242188, + "router_z_loss_mlp": 0.14807129, + "step": 6627, + "time_per_iteration": 2.8176498413085938 + }, + { + "auxiliary_loss_clip": 0.01009477, + "auxiliary_loss_mlp": 0.01002415, + "balance_loss_clip": 1.00229359, + "balance_loss_mlp": 1.00144911, + "epoch": 0.19232778132435727, + "flos": 62194379644800.0, + "grad_norm": 0.6460907523108498, + "language_loss": 0.53152776, + "learning_rate": 3.7299166461083483e-06, + "loss": 0.55164671, + "num_input_tokens_seen": 189420585, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.00964355, + "step": 6628, + "time_per_iteration": 3.1178579330444336 + }, + { + "auxiliary_loss_clip": 0.01009766, + "auxiliary_loss_mlp": 0.01002389, + "balance_loss_clip": 1.00292397, + "balance_loss_mlp": 1.00157845, + "epoch": 0.19235679879287332, + "flos": 74766220208640.0, + "grad_norm": 0.6599346471464902, + "language_loss": 0.48450413, + "learning_rate": 3.7298223110497966e-06, + "loss": 0.50462568, + "num_input_tokens_seen": 189482005, + "router_z_loss_clip": 0.06835938, + "router_z_loss_mlp": 0.00811768, + "step": 6629, + "time_per_iteration": 7.741138696670532 + }, + { + "auxiliary_loss_clip": 0.01010381, + "auxiliary_loss_mlp": 0.01002996, + "balance_loss_clip": 1.00345969, + "balance_loss_mlp": 1.00217307, + "epoch": 0.19238581626138937, + "flos": 59737228001280.0, + "grad_norm": 0.6914443051248417, + "language_loss": 0.51404911, + "learning_rate": 3.7297279607127548e-06, + "loss": 0.53418291, + "num_input_tokens_seen": 189543585, + "router_z_loss_clip": 0.06933594, + "router_z_loss_mlp": 0.00823975, + "step": 6630, + "time_per_iteration": 2.9406206607818604 + }, + { + "auxiliary_loss_clip": 0.0100956, + "auxiliary_loss_mlp": 0.01001027, + "balance_loss_clip": 1.00257707, + "balance_loss_mlp": 1.00009704, + "epoch": 0.1924148337299054, + "flos": 67736981358720.0, + "grad_norm": 0.672186788240634, + "language_loss": 0.53159302, + "learning_rate": 3.7296335950980558e-06, + "loss": 0.55169886, + "num_input_tokens_seen": 189613050, + "router_z_loss_clip": 0.06982422, + "router_z_loss_mlp": 0.00927734, + "step": 6631, + "time_per_iteration": 3.209536075592041 + }, + { + "auxiliary_loss_clip": 0.01077062, + "auxiliary_loss_mlp": 0.01033433, + "balance_loss_clip": 1.02349448, + "balance_loss_mlp": 1.0171783, + "epoch": 0.19244385119842145, + "flos": 31991507487360.0, + "grad_norm": 2.345625903656691, + "language_loss": 0.87455654, + "learning_rate": 3.7295392142065327e-06, + "loss": 0.89566147, + "num_input_tokens_seen": 189633925, + "router_z_loss_clip": 0.53588867, + "router_z_loss_mlp": 0.16247559, + "step": 6632, + "time_per_iteration": 2.665009021759033 + }, + { + "auxiliary_loss_clip": 0.01008685, + "auxiliary_loss_mlp": 0.0100653, + "balance_loss_clip": 1.00174129, + "balance_loss_mlp": 1.00568998, + "epoch": 0.1924728686669375, + "flos": 62146444481280.0, + "grad_norm": 0.7586333538321814, + "language_loss": 0.48699808, + "learning_rate": 3.7294448180390194e-06, + "loss": 0.50715023, + "num_input_tokens_seen": 189685955, + "router_z_loss_clip": 0.06933594, + "router_z_loss_mlp": 0.00842285, + "step": 6633, + "time_per_iteration": 3.1294915676116943 + }, + { + "auxiliary_loss_clip": 0.01072126, + "auxiliary_loss_mlp": 0.01026566, + "balance_loss_clip": 1.02437139, + "balance_loss_mlp": 1.01169491, + "epoch": 0.19250188613545355, + "flos": 27444525281280.0, + "grad_norm": 1.8556031240331312, + "language_loss": 0.56565922, + "learning_rate": 3.7293504065963494e-06, + "loss": 0.58664614, + "num_input_tokens_seen": 189701090, + "router_z_loss_clip": 0.47753906, + "router_z_loss_mlp": 0.14874268, + "step": 6634, + "time_per_iteration": 2.4590065479278564 + }, + { + "auxiliary_loss_clip": 0.01073619, + "auxiliary_loss_mlp": 0.01031578, + "balance_loss_clip": 1.02665174, + "balance_loss_mlp": 1.0170759, + "epoch": 0.1925309036039696, + "flos": 31094589859200.0, + "grad_norm": 2.416500503756808, + "language_loss": 0.72049415, + "learning_rate": 3.729255979879357e-06, + "loss": 0.74154615, + "num_input_tokens_seen": 189721075, + "router_z_loss_clip": 0.46948242, + "router_z_loss_mlp": 0.14501953, + "step": 6635, + "time_per_iteration": 2.4577932357788086 + }, + { + "auxiliary_loss_clip": 0.01084518, + "auxiliary_loss_mlp": 0.01034857, + "balance_loss_clip": 1.02804971, + "balance_loss_mlp": 1.0159142, + "epoch": 0.19255992107248562, + "flos": 40471597463040.0, + "grad_norm": 2.6375791554173618, + "language_loss": 0.82078165, + "learning_rate": 3.7291615378888763e-06, + "loss": 0.84197545, + "num_input_tokens_seen": 189740445, + "router_z_loss_clip": 0.56396484, + "router_z_loss_mlp": 0.18945312, + "step": 6636, + "time_per_iteration": 2.5138511657714844 + }, + { + "auxiliary_loss_clip": 0.01078636, + "auxiliary_loss_mlp": 0.0103542, + "balance_loss_clip": 1.02527714, + "balance_loss_mlp": 1.01846254, + "epoch": 0.19258893854100168, + "flos": 28903371108480.0, + "grad_norm": 1.7300177312643201, + "language_loss": 0.7737143, + "learning_rate": 3.729067080625741e-06, + "loss": 0.79485476, + "num_input_tokens_seen": 189755785, + "router_z_loss_clip": 0.53369141, + "router_z_loss_mlp": 0.16949463, + "step": 6637, + "time_per_iteration": 2.440951108932495 + }, + { + "auxiliary_loss_clip": 0.01079848, + "auxiliary_loss_mlp": 0.0102911, + "balance_loss_clip": 1.02696252, + "balance_loss_mlp": 1.0135529, + "epoch": 0.19261795600951773, + "flos": 74737383425280.0, + "grad_norm": 1.8322388650194261, + "language_loss": 0.83717752, + "learning_rate": 3.7289726080907854e-06, + "loss": 0.85826719, + "num_input_tokens_seen": 189780460, + "router_z_loss_clip": 0.52880859, + "router_z_loss_mlp": 0.15551758, + "step": 6638, + "time_per_iteration": 2.823500156402588 + }, + { + "auxiliary_loss_clip": 0.01084115, + "auxiliary_loss_mlp": 0.01035765, + "balance_loss_clip": 1.02616382, + "balance_loss_mlp": 1.0174067, + "epoch": 0.19264697347803378, + "flos": 29892355591680.0, + "grad_norm": 2.5203251318481725, + "language_loss": 0.78941405, + "learning_rate": 3.728878120284844e-06, + "loss": 0.81061286, + "num_input_tokens_seen": 189798110, + "router_z_loss_clip": 0.5793457, + "router_z_loss_mlp": 0.18365479, + "step": 6639, + "time_per_iteration": 2.432063341140747 + }, + { + "auxiliary_loss_clip": 0.01010086, + "auxiliary_loss_mlp": 0.01011758, + "balance_loss_clip": 1.00262117, + "balance_loss_mlp": 1.01083994, + "epoch": 0.19267599094654983, + "flos": 71082743592960.0, + "grad_norm": 0.690226085123578, + "language_loss": 0.48397174, + "learning_rate": 3.728783617208752e-06, + "loss": 0.50419021, + "num_input_tokens_seen": 189860725, + "router_z_loss_clip": 0.07470703, + "router_z_loss_mlp": 0.00915527, + "step": 6640, + "time_per_iteration": 3.0686731338500977 + }, + { + "auxiliary_loss_clip": 0.01081556, + "auxiliary_loss_mlp": 0.01034756, + "balance_loss_clip": 1.02716911, + "balance_loss_mlp": 1.0182637, + "epoch": 0.19270500841506588, + "flos": 26425619896320.0, + "grad_norm": 2.0369638810627597, + "language_loss": 0.72717392, + "learning_rate": 3.7286890988633434e-06, + "loss": 0.74833709, + "num_input_tokens_seen": 189877890, + "router_z_loss_clip": 0.54370117, + "router_z_loss_mlp": 0.16491699, + "step": 6641, + "time_per_iteration": 2.522932529449463 + }, + { + "auxiliary_loss_clip": 0.01009636, + "auxiliary_loss_mlp": 0.01011309, + "balance_loss_clip": 1.0023098, + "balance_loss_mlp": 1.01036739, + "epoch": 0.1927340258835819, + "flos": 65508300852480.0, + "grad_norm": 0.6423588110978261, + "language_loss": 0.5554716, + "learning_rate": 3.7285945652494527e-06, + "loss": 0.57568115, + "num_input_tokens_seen": 189941385, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00939941, + "step": 6642, + "time_per_iteration": 3.0299315452575684 + }, + { + "auxiliary_loss_clip": 0.01079801, + "auxiliary_loss_mlp": 0.01038663, + "balance_loss_clip": 1.02416742, + "balance_loss_mlp": 1.0203526, + "epoch": 0.19276304335209796, + "flos": 29928979474560.0, + "grad_norm": 3.182948464337824, + "language_loss": 0.86092615, + "learning_rate": 3.728500016367915e-06, + "loss": 0.88211077, + "num_input_tokens_seen": 189956405, + "router_z_loss_clip": 0.55712891, + "router_z_loss_mlp": 0.18304443, + "step": 6643, + "time_per_iteration": 2.4664502143859863 + }, + { + "auxiliary_loss_clip": 0.01008668, + "auxiliary_loss_mlp": 0.01007067, + "balance_loss_clip": 1.0016439, + "balance_loss_mlp": 1.0062561, + "epoch": 0.192792060820614, + "flos": 69116155729920.0, + "grad_norm": 0.6850601679741933, + "language_loss": 0.48776889, + "learning_rate": 3.728405452219567e-06, + "loss": 0.50792623, + "num_input_tokens_seen": 190015095, + "router_z_loss_clip": 0.0703125, + "router_z_loss_mlp": 0.00811768, + "step": 6644, + "time_per_iteration": 2.9750683307647705 + }, + { + "auxiliary_loss_clip": 0.01083329, + "auxiliary_loss_mlp": 0.01031585, + "balance_loss_clip": 1.02654612, + "balance_loss_mlp": 1.01505709, + "epoch": 0.19282107828913006, + "flos": 16362079850880.0, + "grad_norm": 2.039585318348212, + "language_loss": 0.74379432, + "learning_rate": 3.7283108728052416e-06, + "loss": 0.76494342, + "num_input_tokens_seen": 190028075, + "router_z_loss_clip": 0.56787109, + "router_z_loss_mlp": 0.1652832, + "step": 6645, + "time_per_iteration": 2.34171199798584 + }, + { + "auxiliary_loss_clip": 0.01073455, + "auxiliary_loss_mlp": 0.0103199, + "balance_loss_clip": 1.02426004, + "balance_loss_mlp": 1.01838791, + "epoch": 0.1928500957576461, + "flos": 74729982216960.0, + "grad_norm": 2.292665154157986, + "language_loss": 0.80073357, + "learning_rate": 3.728216278125775e-06, + "loss": 0.82178801, + "num_input_tokens_seen": 190050185, + "router_z_loss_clip": 0.49169922, + "router_z_loss_mlp": 0.1361084, + "step": 6646, + "time_per_iteration": 3.045830249786377 + }, + { + "auxiliary_loss_clip": 0.01076133, + "auxiliary_loss_mlp": 0.0103331, + "balance_loss_clip": 1.02560067, + "balance_loss_mlp": 1.01767623, + "epoch": 0.19287911322616216, + "flos": 26825445319680.0, + "grad_norm": 2.4852265027241467, + "language_loss": 0.86729968, + "learning_rate": 3.7281216681820034e-06, + "loss": 0.88839412, + "num_input_tokens_seen": 190066495, + "router_z_loss_clip": 0.50585938, + "router_z_loss_mlp": 0.15637207, + "step": 6647, + "time_per_iteration": 2.447636842727661 + }, + { + "auxiliary_loss_clip": 0.01076788, + "auxiliary_loss_mlp": 0.01030568, + "balance_loss_clip": 1.02531576, + "balance_loss_mlp": 1.01517808, + "epoch": 0.1929081306946782, + "flos": 12852715518720.0, + "grad_norm": 2.5828913684913304, + "language_loss": 0.94942665, + "learning_rate": 3.7280270429747623e-06, + "loss": 0.97050017, + "num_input_tokens_seen": 190081075, + "router_z_loss_clip": 0.51342773, + "router_z_loss_mlp": 0.15393066, + "step": 6648, + "time_per_iteration": 2.4251954555511475 + }, + { + "auxiliary_loss_clip": 0.01009926, + "auxiliary_loss_mlp": 0.01004335, + "balance_loss_clip": 1.00317883, + "balance_loss_mlp": 1.00337505, + "epoch": 0.19293714816319424, + "flos": 74772923189760.0, + "grad_norm": 0.6423209012269279, + "language_loss": 0.4419176, + "learning_rate": 3.7279324025048866e-06, + "loss": 0.46206021, + "num_input_tokens_seen": 190145165, + "router_z_loss_clip": 0.06738281, + "router_z_loss_mlp": 0.00958252, + "step": 6649, + "time_per_iteration": 3.0475642681121826 + }, + { + "auxiliary_loss_clip": 0.01069644, + "auxiliary_loss_mlp": 0.01026439, + "balance_loss_clip": 1.02163363, + "balance_loss_mlp": 1.01289642, + "epoch": 0.1929661656317103, + "flos": 29672088624000.0, + "grad_norm": 3.463505732753489, + "language_loss": 0.67913157, + "learning_rate": 3.727837746773213e-06, + "loss": 0.70009243, + "num_input_tokens_seen": 190160895, + "router_z_loss_clip": 0.48095703, + "router_z_loss_mlp": 0.13537598, + "step": 6650, + "time_per_iteration": 2.379192352294922 + }, + { + "auxiliary_loss_clip": 0.01010169, + "auxiliary_loss_mlp": 0.01002178, + "balance_loss_clip": 1.00329137, + "balance_loss_mlp": 1.00133193, + "epoch": 0.19299518310022634, + "flos": 70461778417920.0, + "grad_norm": 0.6947689150108088, + "language_loss": 0.47521949, + "learning_rate": 3.727743075780577e-06, + "loss": 0.49534297, + "num_input_tokens_seen": 190225085, + "router_z_loss_clip": 0.06884766, + "router_z_loss_mlp": 0.00848389, + "step": 6651, + "time_per_iteration": 3.108147144317627 + }, + { + "auxiliary_loss_clip": 0.01080013, + "auxiliary_loss_mlp": 0.01043631, + "balance_loss_clip": 1.0264008, + "balance_loss_mlp": 1.02547526, + "epoch": 0.1930242005687424, + "flos": 22927915958400.0, + "grad_norm": 2.6083127404669724, + "language_loss": 0.85305166, + "learning_rate": 3.7276483895278144e-06, + "loss": 0.87428808, + "num_input_tokens_seen": 190239105, + "router_z_loss_clip": 0.53637695, + "router_z_loss_mlp": 0.18151855, + "step": 6652, + "time_per_iteration": 2.3798575401306152 + }, + { + "auxiliary_loss_clip": 0.01074742, + "auxiliary_loss_mlp": 0.01032241, + "balance_loss_clip": 1.02234995, + "balance_loss_mlp": 1.01677942, + "epoch": 0.19305321803725842, + "flos": 27262802321280.0, + "grad_norm": 4.395912750061354, + "language_loss": 0.78739882, + "learning_rate": 3.7275536880157635e-06, + "loss": 0.80846858, + "num_input_tokens_seen": 190254435, + "router_z_loss_clip": 0.52368164, + "router_z_loss_mlp": 0.15460205, + "step": 6653, + "time_per_iteration": 2.460991144180298 + }, + { + "auxiliary_loss_clip": 0.01083156, + "auxiliary_loss_mlp": 0.01047906, + "balance_loss_clip": 1.02769101, + "balance_loss_mlp": 1.02949381, + "epoch": 0.19308223550577447, + "flos": 21175764865920.0, + "grad_norm": 2.2562507404209966, + "language_loss": 0.79008043, + "learning_rate": 3.7274589712452586e-06, + "loss": 0.81139106, + "num_input_tokens_seen": 190269070, + "router_z_loss_clip": 0.55493164, + "router_z_loss_mlp": 0.184021, + "step": 6654, + "time_per_iteration": 2.361466646194458 + }, + { + "auxiliary_loss_clip": 0.01076774, + "auxiliary_loss_mlp": 0.01037781, + "balance_loss_clip": 1.0258472, + "balance_loss_mlp": 1.02227783, + "epoch": 0.19311125297429052, + "flos": 26059520712960.0, + "grad_norm": 2.5817922158656357, + "language_loss": 0.91638935, + "learning_rate": 3.727364239217137e-06, + "loss": 0.93753487, + "num_input_tokens_seen": 190283555, + "router_z_loss_clip": 0.50952148, + "router_z_loss_mlp": 0.15490723, + "step": 6655, + "time_per_iteration": 2.458658218383789 + }, + { + "auxiliary_loss_clip": 0.01010516, + "auxiliary_loss_mlp": 0.01001819, + "balance_loss_clip": 1.00350118, + "balance_loss_mlp": 1.00110054, + "epoch": 0.19314027044280657, + "flos": 66747089180160.0, + "grad_norm": 0.657467943229982, + "language_loss": 0.48041484, + "learning_rate": 3.7272694919322354e-06, + "loss": 0.50053823, + "num_input_tokens_seen": 190346815, + "router_z_loss_clip": 0.0703125, + "router_z_loss_mlp": 0.00717163, + "step": 6656, + "time_per_iteration": 3.0011837482452393 + }, + { + "auxiliary_loss_clip": 0.01077748, + "auxiliary_loss_mlp": 0.01042797, + "balance_loss_clip": 1.02389419, + "balance_loss_mlp": 1.02818179, + "epoch": 0.19316928791132262, + "flos": 31788663154560.0, + "grad_norm": 2.388212664121327, + "language_loss": 0.85151094, + "learning_rate": 3.7271747293913904e-06, + "loss": 0.87271643, + "num_input_tokens_seen": 190361695, + "router_z_loss_clip": 0.5390625, + "router_z_loss_mlp": 0.14611816, + "step": 6657, + "time_per_iteration": 2.489976644515991 + }, + { + "auxiliary_loss_clip": 0.01079762, + "auxiliary_loss_mlp": 0.01042554, + "balance_loss_clip": 1.02576232, + "balance_loss_mlp": 1.02660406, + "epoch": 0.19319830537983868, + "flos": 25659276353280.0, + "grad_norm": 3.141988069165482, + "language_loss": 0.76106858, + "learning_rate": 3.72707995159544e-06, + "loss": 0.78229177, + "num_input_tokens_seen": 190380310, + "router_z_loss_clip": 0.53930664, + "router_z_loss_mlp": 0.1595459, + "step": 6658, + "time_per_iteration": 2.477921724319458 + }, + { + "auxiliary_loss_clip": 0.01077127, + "auxiliary_loss_mlp": 0.01034854, + "balance_loss_clip": 1.02618575, + "balance_loss_mlp": 1.01962495, + "epoch": 0.1932273228483547, + "flos": 33319430115840.0, + "grad_norm": 3.2785360110660173, + "language_loss": 0.96555066, + "learning_rate": 3.7269851585452205e-06, + "loss": 0.98667037, + "num_input_tokens_seen": 190396625, + "router_z_loss_clip": 0.51025391, + "router_z_loss_mlp": 0.15234375, + "step": 6659, + "time_per_iteration": 2.735100030899048 + }, + { + "auxiliary_loss_clip": 0.0107703, + "auxiliary_loss_mlp": 0.01043142, + "balance_loss_clip": 1.02553678, + "balance_loss_mlp": 1.02684665, + "epoch": 0.19325634031687075, + "flos": 15844178609280.0, + "grad_norm": 3.3444415063605217, + "language_loss": 0.86291391, + "learning_rate": 3.726890350241569e-06, + "loss": 0.88411564, + "num_input_tokens_seen": 190410485, + "router_z_loss_clip": 0.51489258, + "router_z_loss_mlp": 0.1630249, + "step": 6660, + "time_per_iteration": 2.3470945358276367 + }, + { + "auxiliary_loss_clip": 0.01072595, + "auxiliary_loss_mlp": 0.01033446, + "balance_loss_clip": 1.02224052, + "balance_loss_mlp": 1.01726329, + "epoch": 0.1932853577853868, + "flos": 28175117725440.0, + "grad_norm": 2.1429890157786993, + "language_loss": 0.76177323, + "learning_rate": 3.7267955266853226e-06, + "loss": 0.78283364, + "num_input_tokens_seen": 190425185, + "router_z_loss_clip": 0.50317383, + "router_z_loss_mlp": 0.1618042, + "step": 6661, + "time_per_iteration": 2.453306198120117 + }, + { + "auxiliary_loss_clip": 0.01074988, + "auxiliary_loss_mlp": 0.01029769, + "balance_loss_clip": 1.02405262, + "balance_loss_mlp": 1.01570201, + "epoch": 0.19331437525390285, + "flos": 16647390414720.0, + "grad_norm": 2.22337874967842, + "language_loss": 0.78556633, + "learning_rate": 3.72670068787732e-06, + "loss": 0.80661392, + "num_input_tokens_seen": 190438430, + "router_z_loss_clip": 0.50854492, + "router_z_loss_mlp": 0.140625, + "step": 6662, + "time_per_iteration": 2.3203773498535156 + }, + { + "auxiliary_loss_clip": 0.01010438, + "auxiliary_loss_mlp": 0.01003826, + "balance_loss_clip": 1.00344777, + "balance_loss_mlp": 1.00299776, + "epoch": 0.1933433927224189, + "flos": 59196841977600.0, + "grad_norm": 0.7371898051537468, + "language_loss": 0.47951812, + "learning_rate": 3.7266058338183985e-06, + "loss": 0.49966076, + "num_input_tokens_seen": 190501905, + "router_z_loss_clip": 0.0703125, + "router_z_loss_mlp": 0.00830078, + "step": 6663, + "time_per_iteration": 2.9736833572387695 + }, + { + "auxiliary_loss_clip": 0.0108048, + "auxiliary_loss_mlp": 0.01037701, + "balance_loss_clip": 1.02626562, + "balance_loss_mlp": 1.01952136, + "epoch": 0.19337241019093493, + "flos": 33393061906560.0, + "grad_norm": 1.9136803130992175, + "language_loss": 0.85782552, + "learning_rate": 3.7265109645093952e-06, + "loss": 0.87900734, + "num_input_tokens_seen": 190517540, + "router_z_loss_clip": 0.54174805, + "router_z_loss_mlp": 0.1817627, + "step": 6664, + "time_per_iteration": 2.522038221359253 + }, + { + "auxiliary_loss_clip": 0.01080647, + "auxiliary_loss_mlp": 0.01030696, + "balance_loss_clip": 1.0244416, + "balance_loss_mlp": 1.01370835, + "epoch": 0.19340142765945098, + "flos": 20369829974400.0, + "grad_norm": 2.412406132536206, + "language_loss": 0.87554085, + "learning_rate": 3.726416079951148e-06, + "loss": 0.89665431, + "num_input_tokens_seen": 190532560, + "router_z_loss_clip": 0.56152344, + "router_z_loss_mlp": 0.1697998, + "step": 6665, + "time_per_iteration": 2.429569721221924 + }, + { + "auxiliary_loss_clip": 0.01086185, + "auxiliary_loss_mlp": 0.01041089, + "balance_loss_clip": 1.02711058, + "balance_loss_mlp": 1.02261162, + "epoch": 0.19343044512796703, + "flos": 14312573775360.0, + "grad_norm": 3.0720032217218747, + "language_loss": 0.9291563, + "learning_rate": 3.726321180144496e-06, + "loss": 0.95042896, + "num_input_tokens_seen": 190543930, + "router_z_loss_clip": 0.59155273, + "router_z_loss_mlp": 0.18493652, + "step": 6666, + "time_per_iteration": 2.3043882846832275 + }, + { + "auxiliary_loss_clip": 0.0108591, + "auxiliary_loss_mlp": 0.01037393, + "balance_loss_clip": 1.02927637, + "balance_loss_mlp": 1.01884413, + "epoch": 0.19345946259648308, + "flos": 10115105990400.0, + "grad_norm": 3.12782434435448, + "language_loss": 0.79945403, + "learning_rate": 3.7262262650902762e-06, + "loss": 0.82068706, + "num_input_tokens_seen": 190556125, + "router_z_loss_clip": 0.56591797, + "router_z_loss_mlp": 0.18554688, + "step": 6667, + "time_per_iteration": 2.388632297515869 + }, + { + "auxiliary_loss_clip": 0.0107388, + "auxiliary_loss_mlp": 0.0103369, + "balance_loss_clip": 1.02627552, + "balance_loss_mlp": 1.02002871, + "epoch": 0.19348848006499914, + "flos": 16537448943360.0, + "grad_norm": 3.8724533381339765, + "language_loss": 0.78631246, + "learning_rate": 3.726131334789328e-06, + "loss": 0.80738819, + "num_input_tokens_seen": 190568295, + "router_z_loss_clip": 0.47583008, + "router_z_loss_mlp": 0.13665771, + "step": 6668, + "time_per_iteration": 2.3117916584014893 + }, + { + "auxiliary_loss_clip": 0.01013009, + "auxiliary_loss_mlp": 0.01004709, + "balance_loss_clip": 1.0057528, + "balance_loss_mlp": 1.00384498, + "epoch": 0.1935174975335152, + "flos": 65870804165760.0, + "grad_norm": 0.650739999331659, + "language_loss": 0.49268267, + "learning_rate": 3.726036389242489e-06, + "loss": 0.51285982, + "num_input_tokens_seen": 190632740, + "router_z_loss_clip": 0.07226562, + "router_z_loss_mlp": 0.00866699, + "step": 6669, + "time_per_iteration": 2.99702787399292 + }, + { + "auxiliary_loss_clip": 0.0107475, + "auxiliary_loss_mlp": 0.01032668, + "balance_loss_clip": 1.02548981, + "balance_loss_mlp": 1.01779091, + "epoch": 0.1935465150020312, + "flos": 31459606790400.0, + "grad_norm": 2.240803916750948, + "language_loss": 0.84121138, + "learning_rate": 3.725941428450599e-06, + "loss": 0.86228561, + "num_input_tokens_seen": 190647705, + "router_z_loss_clip": 0.49316406, + "router_z_loss_mlp": 0.14874268, + "step": 6670, + "time_per_iteration": 2.4375269412994385 + }, + { + "auxiliary_loss_clip": 0.0101301, + "auxiliary_loss_mlp": 0.01000798, + "balance_loss_clip": 1.00588918, + "balance_loss_mlp": 1.00003517, + "epoch": 0.19357553247054726, + "flos": 63500550629760.0, + "grad_norm": 0.7349141504113466, + "language_loss": 0.50283796, + "learning_rate": 3.7258464524144946e-06, + "loss": 0.52297598, + "num_input_tokens_seen": 190706095, + "router_z_loss_clip": 0.07128906, + "router_z_loss_mlp": 0.00762939, + "step": 6671, + "time_per_iteration": 2.945091485977173 + }, + { + "auxiliary_loss_clip": 0.01082777, + "auxiliary_loss_mlp": 0.01036405, + "balance_loss_clip": 1.02772284, + "balance_loss_mlp": 1.01852942, + "epoch": 0.19360454993906331, + "flos": 21460481936640.0, + "grad_norm": 2.0768740038298765, + "language_loss": 0.81671584, + "learning_rate": 3.725751461135017e-06, + "loss": 0.83790767, + "num_input_tokens_seen": 190720950, + "router_z_loss_clip": 0.55078125, + "router_z_loss_mlp": 0.17883301, + "step": 6672, + "time_per_iteration": 2.38278865814209 + }, + { + "auxiliary_loss_clip": 0.01011901, + "auxiliary_loss_mlp": 0.01000915, + "balance_loss_clip": 1.00453579, + "balance_loss_mlp": 1.0001049, + "epoch": 0.19363356740757937, + "flos": 74765242690560.0, + "grad_norm": 0.664965961744219, + "language_loss": 0.45607913, + "learning_rate": 3.7256564546130036e-06, + "loss": 0.47620732, + "num_input_tokens_seen": 190783085, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00811768, + "step": 6673, + "time_per_iteration": 3.2538084983825684 + }, + { + "auxiliary_loss_clip": 0.01075444, + "auxiliary_loss_mlp": 0.01035739, + "balance_loss_clip": 1.02511084, + "balance_loss_mlp": 1.02077222, + "epoch": 0.19366258487609542, + "flos": 10261252408320.0, + "grad_norm": 2.270566107572736, + "language_loss": 0.8134923, + "learning_rate": 3.7255614328492943e-06, + "loss": 0.83460414, + "num_input_tokens_seen": 190792860, + "router_z_loss_clip": 0.50366211, + "router_z_loss_mlp": 0.14971924, + "step": 6674, + "time_per_iteration": 2.373708724975586 + }, + { + "auxiliary_loss_clip": 0.01080222, + "auxiliary_loss_mlp": 0.01038402, + "balance_loss_clip": 1.02649152, + "balance_loss_mlp": 1.0206455, + "epoch": 0.19369160234461147, + "flos": 42121941431040.0, + "grad_norm": 4.9096425841669875, + "language_loss": 0.66233826, + "learning_rate": 3.7254663958447285e-06, + "loss": 0.68352449, + "num_input_tokens_seen": 190814865, + "router_z_loss_clip": 0.53808594, + "router_z_loss_mlp": 0.17755127, + "step": 6675, + "time_per_iteration": 2.5491604804992676 + }, + { + "auxiliary_loss_clip": 0.01071055, + "auxiliary_loss_mlp": 0.01036503, + "balance_loss_clip": 1.023754, + "balance_loss_mlp": 1.02275229, + "epoch": 0.1937206198131275, + "flos": 16172047987200.0, + "grad_norm": 3.9164779289422627, + "language_loss": 0.54598045, + "learning_rate": 3.7253713436001447e-06, + "loss": 0.567056, + "num_input_tokens_seen": 190826695, + "router_z_loss_clip": 0.47314453, + "router_z_loss_mlp": 0.13757324, + "step": 6676, + "time_per_iteration": 2.314528465270996 + }, + { + "auxiliary_loss_clip": 0.01077628, + "auxiliary_loss_mlp": 0.01047383, + "balance_loss_clip": 1.02562892, + "balance_loss_mlp": 1.03258276, + "epoch": 0.19374963728164354, + "flos": 30003868074240.0, + "grad_norm": 1.9359554298509307, + "language_loss": 0.84758461, + "learning_rate": 3.725276276116383e-06, + "loss": 0.86883473, + "num_input_tokens_seen": 190844010, + "router_z_loss_clip": 0.52001953, + "router_z_loss_mlp": 0.14788818, + "step": 6677, + "time_per_iteration": 2.466557741165161 + }, + { + "auxiliary_loss_clip": 0.01074539, + "auxiliary_loss_mlp": 0.01031735, + "balance_loss_clip": 1.02553868, + "balance_loss_mlp": 1.01697707, + "epoch": 0.1937786547501596, + "flos": 21244439243520.0, + "grad_norm": 2.4584670723789466, + "language_loss": 0.85105574, + "learning_rate": 3.7251811933942835e-06, + "loss": 0.87211847, + "num_input_tokens_seen": 190858365, + "router_z_loss_clip": 0.48999023, + "router_z_loss_mlp": 0.14764404, + "step": 6678, + "time_per_iteration": 2.3898391723632812 + }, + { + "auxiliary_loss_clip": 0.01082117, + "auxiliary_loss_mlp": 0.01042273, + "balance_loss_clip": 1.02702737, + "balance_loss_mlp": 1.02313447, + "epoch": 0.19380767221867565, + "flos": 14749337283840.0, + "grad_norm": 2.398977863161447, + "language_loss": 0.91468704, + "learning_rate": 3.725086095434685e-06, + "loss": 0.93593091, + "num_input_tokens_seen": 190871005, + "router_z_loss_clip": 0.55102539, + "router_z_loss_mlp": 0.19152832, + "step": 6679, + "time_per_iteration": 2.3827695846557617 + }, + { + "auxiliary_loss_clip": 0.01080764, + "auxiliary_loss_mlp": 0.01039525, + "balance_loss_clip": 1.02778935, + "balance_loss_mlp": 1.0245167, + "epoch": 0.1938366896871917, + "flos": 13727080408320.0, + "grad_norm": 2.2355442258359135, + "language_loss": 0.79032624, + "learning_rate": 3.7249909822384284e-06, + "loss": 0.8115291, + "num_input_tokens_seen": 190882780, + "router_z_loss_clip": 0.52954102, + "router_z_loss_mlp": 0.15014648, + "step": 6680, + "time_per_iteration": 2.4271328449249268 + }, + { + "auxiliary_loss_clip": 0.01012605, + "auxiliary_loss_mlp": 0.01007767, + "balance_loss_clip": 1.00519812, + "balance_loss_mlp": 1.00683117, + "epoch": 0.19386570715570772, + "flos": 64005080820480.0, + "grad_norm": 0.6994629007471143, + "language_loss": 0.49498922, + "learning_rate": 3.7248958538063536e-06, + "loss": 0.51519293, + "num_input_tokens_seen": 190942190, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00933838, + "step": 6681, + "time_per_iteration": 3.001615047454834 + }, + { + "auxiliary_loss_clip": 0.01013031, + "auxiliary_loss_mlp": 0.01009817, + "balance_loss_clip": 1.00562692, + "balance_loss_mlp": 1.00890529, + "epoch": 0.19389472462422377, + "flos": 46706175924480.0, + "grad_norm": 0.6893204619409676, + "language_loss": 0.44861352, + "learning_rate": 3.7248007101393002e-06, + "loss": 0.46884203, + "num_input_tokens_seen": 190993195, + "router_z_loss_clip": 0.07421875, + "router_z_loss_mlp": 0.00909424, + "step": 6682, + "time_per_iteration": 2.7881531715393066 + }, + { + "auxiliary_loss_clip": 0.01072132, + "auxiliary_loss_mlp": 0.01028222, + "balance_loss_clip": 1.02343667, + "balance_loss_mlp": 1.01300454, + "epoch": 0.19392374209273983, + "flos": 28323079534080.0, + "grad_norm": 1.9305547366990075, + "language_loss": 0.94349062, + "learning_rate": 3.7247055512381094e-06, + "loss": 0.96449411, + "num_input_tokens_seen": 191011830, + "router_z_loss_clip": 0.48706055, + "router_z_loss_mlp": 0.15209961, + "step": 6683, + "time_per_iteration": 2.420581340789795 + }, + { + "auxiliary_loss_clip": 0.01080978, + "auxiliary_loss_mlp": 0.010401, + "balance_loss_clip": 1.02575874, + "balance_loss_mlp": 1.02150297, + "epoch": 0.19395275956125588, + "flos": 28177212407040.0, + "grad_norm": 2.680467229909243, + "language_loss": 0.76509595, + "learning_rate": 3.724610377103621e-06, + "loss": 0.78630674, + "num_input_tokens_seen": 191028315, + "router_z_loss_clip": 0.55200195, + "router_z_loss_mlp": 0.18609619, + "step": 6684, + "time_per_iteration": 2.417604446411133 + }, + { + "auxiliary_loss_clip": 0.01081467, + "auxiliary_loss_mlp": 0.01034693, + "balance_loss_clip": 1.02626669, + "balance_loss_mlp": 1.01756263, + "epoch": 0.19398177702977193, + "flos": 19675023540480.0, + "grad_norm": 5.126615543344651, + "language_loss": 0.76873815, + "learning_rate": 3.7245151877366762e-06, + "loss": 0.78989971, + "num_input_tokens_seen": 191040630, + "router_z_loss_clip": 0.55249023, + "router_z_loss_mlp": 0.17126465, + "step": 6685, + "time_per_iteration": 2.371812582015991 + }, + { + "auxiliary_loss_clip": 0.01077313, + "auxiliary_loss_mlp": 0.01032245, + "balance_loss_clip": 1.02623248, + "balance_loss_mlp": 1.01592493, + "epoch": 0.19401079449828798, + "flos": 14240966843520.0, + "grad_norm": 2.8497858949130994, + "language_loss": 0.86950278, + "learning_rate": 3.7244199831381147e-06, + "loss": 0.8905983, + "num_input_tokens_seen": 191052515, + "router_z_loss_clip": 0.51025391, + "router_z_loss_mlp": 0.16308594, + "step": 6686, + "time_per_iteration": 2.386746406555176 + }, + { + "auxiliary_loss_clip": 0.01085328, + "auxiliary_loss_mlp": 0.01031443, + "balance_loss_clip": 1.02980506, + "balance_loss_mlp": 1.01486158, + "epoch": 0.194039811966804, + "flos": 27157818263040.0, + "grad_norm": 2.810167408087049, + "language_loss": 1.09106982, + "learning_rate": 3.724324763308779e-06, + "loss": 1.11223745, + "num_input_tokens_seen": 191066655, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.16564941, + "step": 6687, + "time_per_iteration": 2.6262989044189453 + }, + { + "auxiliary_loss_clip": 0.01075773, + "auxiliary_loss_mlp": 0.01029851, + "balance_loss_clip": 1.02796006, + "balance_loss_mlp": 1.01477671, + "epoch": 0.19406882943532006, + "flos": 12303636566400.0, + "grad_norm": 2.911627333447874, + "language_loss": 0.78651571, + "learning_rate": 3.7242295282495086e-06, + "loss": 0.80757195, + "num_input_tokens_seen": 191078655, + "router_z_loss_clip": 0.4777832, + "router_z_loss_mlp": 0.1506958, + "step": 6688, + "time_per_iteration": 2.3946166038513184 + }, + { + "auxiliary_loss_clip": 0.01086204, + "auxiliary_loss_mlp": 0.01042648, + "balance_loss_clip": 1.02920544, + "balance_loss_mlp": 1.02457643, + "epoch": 0.1940978469038361, + "flos": 11577896801280.0, + "grad_norm": 2.9270519199385197, + "language_loss": 0.76498425, + "learning_rate": 3.724134277961146e-06, + "loss": 0.78627276, + "num_input_tokens_seen": 191091670, + "router_z_loss_clip": 0.56982422, + "router_z_loss_mlp": 0.18054199, + "step": 6689, + "time_per_iteration": 2.3638789653778076 + }, + { + "auxiliary_loss_clip": 0.01082646, + "auxiliary_loss_mlp": 0.01035404, + "balance_loss_clip": 1.0277667, + "balance_loss_mlp": 1.01729631, + "epoch": 0.19412686437235216, + "flos": 25621395661440.0, + "grad_norm": 3.6202694334826204, + "language_loss": 1.01477134, + "learning_rate": 3.724039012444531e-06, + "loss": 1.03595185, + "num_input_tokens_seen": 191106000, + "router_z_loss_clip": 0.54882812, + "router_z_loss_mlp": 0.18103027, + "step": 6690, + "time_per_iteration": 2.449711322784424 + }, + { + "auxiliary_loss_clip": 0.0101979, + "auxiliary_loss_mlp": 0.01003589, + "balance_loss_clip": 1.01198125, + "balance_loss_mlp": 1.00263536, + "epoch": 0.1941558818408682, + "flos": 62044113686400.0, + "grad_norm": 0.7029772992055947, + "language_loss": 0.48186544, + "learning_rate": 3.7239437317005055e-06, + "loss": 0.50209922, + "num_input_tokens_seen": 191161260, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.00952148, + "step": 6691, + "time_per_iteration": 2.9690611362457275 + }, + { + "auxiliary_loss_clip": 0.01071973, + "auxiliary_loss_mlp": 0.01035694, + "balance_loss_clip": 1.02575743, + "balance_loss_mlp": 1.02184784, + "epoch": 0.19418489930938426, + "flos": 16245121196160.0, + "grad_norm": 2.7276233877263327, + "language_loss": 0.78293514, + "learning_rate": 3.7238484357299127e-06, + "loss": 0.80401182, + "num_input_tokens_seen": 191173775, + "router_z_loss_clip": 0.4621582, + "router_z_loss_mlp": 0.13848877, + "step": 6692, + "time_per_iteration": 2.4117677211761475 + }, + { + "auxiliary_loss_clip": 0.01075569, + "auxiliary_loss_mlp": 0.01039685, + "balance_loss_clip": 1.0256716, + "balance_loss_mlp": 1.0245986, + "epoch": 0.1942139167779003, + "flos": 24200500348800.0, + "grad_norm": 2.140759511608035, + "language_loss": 0.74487585, + "learning_rate": 3.7237531245335914e-06, + "loss": 0.7660284, + "num_input_tokens_seen": 191191875, + "router_z_loss_clip": 0.4987793, + "router_z_loss_mlp": 0.15075684, + "step": 6693, + "time_per_iteration": 2.4228615760803223 + }, + { + "auxiliary_loss_clip": 0.01082048, + "auxiliary_loss_mlp": 0.01032661, + "balance_loss_clip": 1.02767062, + "balance_loss_mlp": 1.01653218, + "epoch": 0.19424293424641634, + "flos": 30000900608640.0, + "grad_norm": 3.392556489593279, + "language_loss": 0.81632435, + "learning_rate": 3.723657798112386e-06, + "loss": 0.83747149, + "num_input_tokens_seen": 191206055, + "router_z_loss_clip": 0.54370117, + "router_z_loss_mlp": 0.16113281, + "step": 6694, + "time_per_iteration": 2.4872798919677734 + }, + { + "auxiliary_loss_clip": 0.01076981, + "auxiliary_loss_mlp": 0.01037121, + "balance_loss_clip": 1.02605808, + "balance_loss_mlp": 1.02047896, + "epoch": 0.1942719517149324, + "flos": 34123654350720.0, + "grad_norm": 1.876124925725152, + "language_loss": 0.87413818, + "learning_rate": 3.723562456467137e-06, + "loss": 0.89527917, + "num_input_tokens_seen": 191223865, + "router_z_loss_clip": 0.50927734, + "router_z_loss_mlp": 0.16650391, + "step": 6695, + "time_per_iteration": 2.498610019683838 + }, + { + "auxiliary_loss_clip": 0.01015243, + "auxiliary_loss_mlp": 0.01005136, + "balance_loss_clip": 1.00775385, + "balance_loss_mlp": 1.00417674, + "epoch": 0.19430096918344844, + "flos": 74773726151040.0, + "grad_norm": 1.0496463956502888, + "language_loss": 0.47448742, + "learning_rate": 3.7234670995986877e-06, + "loss": 0.49469119, + "num_input_tokens_seen": 191289200, + "router_z_loss_clip": 0.07470703, + "router_z_loss_mlp": 0.00958252, + "step": 6696, + "time_per_iteration": 5.158189058303833 + }, + { + "auxiliary_loss_clip": 0.01014427, + "auxiliary_loss_mlp": 0.01001686, + "balance_loss_clip": 1.00675941, + "balance_loss_mlp": 1.00066054, + "epoch": 0.1943299866519645, + "flos": 64731030053760.0, + "grad_norm": 0.703330948740711, + "language_loss": 0.47776687, + "learning_rate": 3.7233717275078787e-06, + "loss": 0.49792802, + "num_input_tokens_seen": 191351365, + "router_z_loss_clip": 0.07666016, + "router_z_loss_mlp": 0.01025391, + "step": 6697, + "time_per_iteration": 5.307248830795288 + }, + { + "auxiliary_loss_clip": 0.01078661, + "auxiliary_loss_mlp": 0.01031587, + "balance_loss_clip": 1.02785909, + "balance_loss_mlp": 1.01523805, + "epoch": 0.19435900412048052, + "flos": 27884221344000.0, + "grad_norm": 1.946101695190119, + "language_loss": 0.81526721, + "learning_rate": 3.723276340195554e-06, + "loss": 0.83636963, + "num_input_tokens_seen": 191369160, + "router_z_loss_clip": 0.5078125, + "router_z_loss_mlp": 0.16351318, + "step": 6698, + "time_per_iteration": 2.684835433959961 + }, + { + "auxiliary_loss_clip": 0.01012117, + "auxiliary_loss_mlp": 0.01000843, + "balance_loss_clip": 1.00433636, + "balance_loss_mlp": 0.99984771, + "epoch": 0.19438802158899657, + "flos": 59698579259520.0, + "grad_norm": 0.811852301444408, + "language_loss": 0.51630527, + "learning_rate": 3.7231809376625542e-06, + "loss": 0.53643489, + "num_input_tokens_seen": 191424155, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.00994873, + "step": 6699, + "time_per_iteration": 2.9262313842773438 + }, + { + "auxiliary_loss_clip": 0.01076889, + "auxiliary_loss_mlp": 0.01032194, + "balance_loss_clip": 1.02494514, + "balance_loss_mlp": 1.01652968, + "epoch": 0.19441703905751262, + "flos": 12086197418880.0, + "grad_norm": 2.357451792576738, + "language_loss": 0.70810485, + "learning_rate": 3.723085519909724e-06, + "loss": 0.72919571, + "num_input_tokens_seen": 191436615, + "router_z_loss_clip": 0.51953125, + "router_z_loss_mlp": 0.15667725, + "step": 6700, + "time_per_iteration": 2.4587149620056152 + }, + { + "auxiliary_loss_clip": 0.01010471, + "auxiliary_loss_mlp": 0.01001452, + "balance_loss_clip": 1.00265872, + "balance_loss_mlp": 1.00028992, + "epoch": 0.19444605652602867, + "flos": 74772190051200.0, + "grad_norm": 0.6165602794464917, + "language_loss": 0.46132177, + "learning_rate": 3.7229900869379048e-06, + "loss": 0.48144096, + "num_input_tokens_seen": 191498940, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.01159668, + "step": 6701, + "time_per_iteration": 3.030275344848633 + }, + { + "auxiliary_loss_clip": 0.01077912, + "auxiliary_loss_mlp": 0.01035443, + "balance_loss_clip": 1.02617335, + "balance_loss_mlp": 1.02008283, + "epoch": 0.19447507399454472, + "flos": 23469628613760.0, + "grad_norm": 4.250087730374693, + "language_loss": 0.99373341, + "learning_rate": 3.72289463874794e-06, + "loss": 1.01486707, + "num_input_tokens_seen": 191512015, + "router_z_loss_clip": 0.51757812, + "router_z_loss_mlp": 0.15362549, + "step": 6702, + "time_per_iteration": 2.4596478939056396 + }, + { + "auxiliary_loss_clip": 0.01010771, + "auxiliary_loss_mlp": 0.01002444, + "balance_loss_clip": 1.00315714, + "balance_loss_mlp": 1.00140727, + "epoch": 0.19450409146306077, + "flos": 71307095189760.0, + "grad_norm": 0.6367746505211989, + "language_loss": 0.43966964, + "learning_rate": 3.7227991753406727e-06, + "loss": 0.45980179, + "num_input_tokens_seen": 191580040, + "router_z_loss_clip": 0.07617188, + "router_z_loss_mlp": 0.01037598, + "step": 6703, + "time_per_iteration": 3.0657103061676025 + }, + { + "auxiliary_loss_clip": 0.01075692, + "auxiliary_loss_mlp": 0.01033186, + "balance_loss_clip": 1.02631569, + "balance_loss_mlp": 1.01801705, + "epoch": 0.1945331089315768, + "flos": 13624330677120.0, + "grad_norm": 4.30324029670929, + "language_loss": 0.72718918, + "learning_rate": 3.722703696716946e-06, + "loss": 0.74827802, + "num_input_tokens_seen": 191594375, + "router_z_loss_clip": 0.49316406, + "router_z_loss_mlp": 0.15155029, + "step": 6704, + "time_per_iteration": 2.359842538833618 + }, + { + "auxiliary_loss_clip": 0.01072881, + "auxiliary_loss_mlp": 0.01032676, + "balance_loss_clip": 1.02412927, + "balance_loss_mlp": 1.01838887, + "epoch": 0.19456212640009285, + "flos": 16206786656640.0, + "grad_norm": 2.6453789250812654, + "language_loss": 0.71370173, + "learning_rate": 3.722608202877603e-06, + "loss": 0.73475724, + "num_input_tokens_seen": 191605220, + "router_z_loss_clip": 0.48779297, + "router_z_loss_mlp": 0.14294434, + "step": 6705, + "time_per_iteration": 4.7709715366363525 + }, + { + "auxiliary_loss_clip": 0.01077566, + "auxiliary_loss_mlp": 0.01038644, + "balance_loss_clip": 1.02622926, + "balance_loss_mlp": 1.02255678, + "epoch": 0.1945911438686089, + "flos": 25476575875200.0, + "grad_norm": 2.834590273420485, + "language_loss": 0.74052542, + "learning_rate": 3.722512693823487e-06, + "loss": 0.76168752, + "num_input_tokens_seen": 191620190, + "router_z_loss_clip": 0.51318359, + "router_z_loss_mlp": 0.16088867, + "step": 6706, + "time_per_iteration": 4.916105508804321 + }, + { + "auxiliary_loss_clip": 0.01079088, + "auxiliary_loss_mlp": 0.01038934, + "balance_loss_clip": 1.02697515, + "balance_loss_mlp": 1.02283525, + "epoch": 0.19462016133712495, + "flos": 28175885775360.0, + "grad_norm": 2.6100054138948874, + "language_loss": 0.82065558, + "learning_rate": 3.7224171695554423e-06, + "loss": 0.8418358, + "num_input_tokens_seen": 191635515, + "router_z_loss_clip": 0.52172852, + "router_z_loss_mlp": 0.16094971, + "step": 6707, + "time_per_iteration": 2.4338696002960205 + }, + { + "auxiliary_loss_clip": 0.01013155, + "auxiliary_loss_mlp": 0.01001068, + "balance_loss_clip": 1.00552678, + "balance_loss_mlp": 1.0001086, + "epoch": 0.194649178805641, + "flos": 58793385772800.0, + "grad_norm": 0.6676206396763836, + "language_loss": 0.48119968, + "learning_rate": 3.7223216300743117e-06, + "loss": 0.50134194, + "num_input_tokens_seen": 191697065, + "router_z_loss_clip": 0.07617188, + "router_z_loss_mlp": 0.00958252, + "step": 6708, + "time_per_iteration": 3.0091159343719482 + }, + { + "auxiliary_loss_clip": 0.01012601, + "auxiliary_loss_mlp": 0.0100365, + "balance_loss_clip": 1.00489819, + "balance_loss_mlp": 1.00270879, + "epoch": 0.19467819627415706, + "flos": 63453278782080.0, + "grad_norm": 0.6651908842828362, + "language_loss": 0.43475461, + "learning_rate": 3.7222260753809403e-06, + "loss": 0.45491707, + "num_input_tokens_seen": 191754840, + "router_z_loss_clip": 0.07714844, + "router_z_loss_mlp": 0.00939941, + "step": 6709, + "time_per_iteration": 3.168332815170288 + }, + { + "auxiliary_loss_clip": 0.01078947, + "auxiliary_loss_mlp": 0.01035424, + "balance_loss_clip": 1.02666557, + "balance_loss_mlp": 1.01907444, + "epoch": 0.19470721374267308, + "flos": 38282752684800.0, + "grad_norm": 2.3384198998146593, + "language_loss": 0.73616159, + "learning_rate": 3.7221305054761705e-06, + "loss": 0.75730526, + "num_input_tokens_seen": 191775525, + "router_z_loss_clip": 0.52246094, + "router_z_loss_mlp": 0.16351318, + "step": 6710, + "time_per_iteration": 2.586630344390869 + }, + { + "auxiliary_loss_clip": 0.01084913, + "auxiliary_loss_mlp": 0.01049636, + "balance_loss_clip": 1.03079808, + "balance_loss_mlp": 1.03196883, + "epoch": 0.19473623121118913, + "flos": 14166008421120.0, + "grad_norm": 4.212657688004074, + "language_loss": 0.65910226, + "learning_rate": 3.7220349203608476e-06, + "loss": 0.68044776, + "num_input_tokens_seen": 191786490, + "router_z_loss_clip": 0.54052734, + "router_z_loss_mlp": 0.17675781, + "step": 6711, + "time_per_iteration": 2.4249002933502197 + }, + { + "auxiliary_loss_clip": 0.01013083, + "auxiliary_loss_mlp": 0.0100593, + "balance_loss_clip": 1.00536251, + "balance_loss_mlp": 1.00494671, + "epoch": 0.19476524867970518, + "flos": 74776623793920.0, + "grad_norm": 0.6104674676867055, + "language_loss": 0.44478604, + "learning_rate": 3.7219393200358153e-06, + "loss": 0.46497619, + "num_input_tokens_seen": 191855245, + "router_z_loss_clip": 0.07714844, + "router_z_loss_mlp": 0.00982666, + "step": 6712, + "time_per_iteration": 3.2234792709350586 + }, + { + "auxiliary_loss_clip": 0.01082481, + "auxiliary_loss_mlp": 0.01034364, + "balance_loss_clip": 1.0277065, + "balance_loss_mlp": 1.01643538, + "epoch": 0.19479426614822123, + "flos": 22485811011840.0, + "grad_norm": 2.333496811044981, + "language_loss": 0.74108934, + "learning_rate": 3.7218437045019185e-06, + "loss": 0.76225787, + "num_input_tokens_seen": 191870145, + "router_z_loss_clip": 0.5480957, + "router_z_loss_mlp": 0.17944336, + "step": 6713, + "time_per_iteration": 2.406668186187744 + }, + { + "auxiliary_loss_clip": 0.01077536, + "auxiliary_loss_mlp": 0.01031553, + "balance_loss_clip": 1.02640319, + "balance_loss_mlp": 1.01574588, + "epoch": 0.19482328361673729, + "flos": 31393131828480.0, + "grad_norm": 1.7746156914353177, + "language_loss": 0.74437869, + "learning_rate": 3.721748073760001e-06, + "loss": 0.76546955, + "num_input_tokens_seen": 191889295, + "router_z_loss_clip": 0.51171875, + "router_z_loss_mlp": 0.15820312, + "step": 6714, + "time_per_iteration": 2.45185923576355 + }, + { + "auxiliary_loss_clip": 0.01077794, + "auxiliary_loss_mlp": 0.01029157, + "balance_loss_clip": 1.02741611, + "balance_loss_mlp": 1.01453567, + "epoch": 0.1948523010852533, + "flos": 32735193557760.0, + "grad_norm": 1.603688464420547, + "language_loss": 0.64658058, + "learning_rate": 3.7216524278109076e-06, + "loss": 0.66764998, + "num_input_tokens_seen": 191907490, + "router_z_loss_clip": 0.50415039, + "router_z_loss_mlp": 0.1461792, + "step": 6715, + "time_per_iteration": 2.55126953125 + }, + { + "auxiliary_loss_clip": 0.01084911, + "auxiliary_loss_mlp": 0.01038379, + "balance_loss_clip": 1.02969527, + "balance_loss_mlp": 1.02170134, + "epoch": 0.19488131855376936, + "flos": 15952234867200.0, + "grad_norm": 2.55859907274347, + "language_loss": 0.69372916, + "learning_rate": 3.7215567666554834e-06, + "loss": 0.71496201, + "num_input_tokens_seen": 191923760, + "router_z_loss_clip": 0.55224609, + "router_z_loss_mlp": 0.16687012, + "step": 6716, + "time_per_iteration": 2.407365322113037 + }, + { + "auxiliary_loss_clip": 0.01017356, + "auxiliary_loss_mlp": 0.01015882, + "balance_loss_clip": 1.00954688, + "balance_loss_mlp": 1.01479733, + "epoch": 0.1949103360222854, + "flos": 59261292080640.0, + "grad_norm": 0.6774659187233945, + "language_loss": 0.48151881, + "learning_rate": 3.7214610902945735e-06, + "loss": 0.50185114, + "num_input_tokens_seen": 191982295, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.01086426, + "step": 6717, + "time_per_iteration": 2.9707019329071045 + }, + { + "auxiliary_loss_clip": 0.0107899, + "auxiliary_loss_mlp": 0.01031538, + "balance_loss_clip": 1.02749789, + "balance_loss_mlp": 1.01568294, + "epoch": 0.19493935349080146, + "flos": 74735428389120.0, + "grad_norm": 3.2665355249011263, + "language_loss": 0.71750784, + "learning_rate": 3.7213653987290227e-06, + "loss": 0.73861307, + "num_input_tokens_seen": 192009545, + "router_z_loss_clip": 0.51464844, + "router_z_loss_mlp": 0.15875244, + "step": 6718, + "time_per_iteration": 2.7783281803131104 + }, + { + "auxiliary_loss_clip": 0.01077124, + "auxiliary_loss_mlp": 0.01031802, + "balance_loss_clip": 1.02535605, + "balance_loss_mlp": 1.01575005, + "epoch": 0.19496837095931752, + "flos": 22745215480320.0, + "grad_norm": 1.940665442661792, + "language_loss": 0.73756289, + "learning_rate": 3.7212696919596757e-06, + "loss": 0.75865215, + "num_input_tokens_seen": 192028310, + "router_z_loss_clip": 0.51757812, + "router_z_loss_mlp": 0.16070557, + "step": 6719, + "time_per_iteration": 2.4107067584991455 + }, + { + "auxiliary_loss_clip": 0.01078472, + "auxiliary_loss_mlp": 0.01033786, + "balance_loss_clip": 1.02718663, + "balance_loss_mlp": 1.01821733, + "epoch": 0.19499738842783357, + "flos": 21316988782080.0, + "grad_norm": 2.2791268661462016, + "language_loss": 0.87578297, + "learning_rate": 3.7211739699873786e-06, + "loss": 0.89690554, + "num_input_tokens_seen": 192044180, + "router_z_loss_clip": 0.51220703, + "router_z_loss_mlp": 0.15576172, + "step": 6720, + "time_per_iteration": 2.4042742252349854 + }, + { + "auxiliary_loss_clip": 0.01084609, + "auxiliary_loss_mlp": 0.01032807, + "balance_loss_clip": 1.03042722, + "balance_loss_mlp": 1.01667845, + "epoch": 0.1950264058963496, + "flos": 14931199889280.0, + "grad_norm": 2.402770168403097, + "language_loss": 0.91219026, + "learning_rate": 3.7210782328129764e-06, + "loss": 0.93336439, + "num_input_tokens_seen": 192056835, + "router_z_loss_clip": 0.54150391, + "router_z_loss_mlp": 0.16149902, + "step": 6721, + "time_per_iteration": 2.427978754043579 + }, + { + "auxiliary_loss_clip": 0.01079375, + "auxiliary_loss_mlp": 0.01036264, + "balance_loss_clip": 1.03025711, + "balance_loss_mlp": 1.02211952, + "epoch": 0.19505542336486564, + "flos": 27118157091840.0, + "grad_norm": 2.084117723752052, + "language_loss": 0.8292858, + "learning_rate": 3.720982480437315e-06, + "loss": 0.85044217, + "num_input_tokens_seen": 192072945, + "router_z_loss_clip": 0.49169922, + "router_z_loss_mlp": 0.14129639, + "step": 6722, + "time_per_iteration": 2.460129737854004 + }, + { + "auxiliary_loss_clip": 0.01018327, + "auxiliary_loss_mlp": 0.0100191, + "balance_loss_clip": 1.0098927, + "balance_loss_mlp": 1.00081944, + "epoch": 0.1950844408333817, + "flos": 62690356552320.0, + "grad_norm": 0.7584665897893402, + "language_loss": 0.52305925, + "learning_rate": 3.7208867128612393e-06, + "loss": 0.54326165, + "num_input_tokens_seen": 192125760, + "router_z_loss_clip": 0.08447266, + "router_z_loss_mlp": 0.01092529, + "step": 6723, + "time_per_iteration": 3.0894548892974854 + }, + { + "auxiliary_loss_clip": 0.01080985, + "auxiliary_loss_mlp": 0.01043547, + "balance_loss_clip": 1.02725732, + "balance_loss_mlp": 1.02667308, + "epoch": 0.19511345830189775, + "flos": 28942962456960.0, + "grad_norm": 4.322933274310607, + "language_loss": 0.76597285, + "learning_rate": 3.7207909300855964e-06, + "loss": 0.78721821, + "num_input_tokens_seen": 192145865, + "router_z_loss_clip": 0.53808594, + "router_z_loss_mlp": 0.16864014, + "step": 6724, + "time_per_iteration": 2.564425230026245 + }, + { + "auxiliary_loss_clip": 0.0108314, + "auxiliary_loss_mlp": 0.01048335, + "balance_loss_clip": 1.02734184, + "balance_loss_mlp": 1.02978015, + "epoch": 0.1951424757704138, + "flos": 74726456169600.0, + "grad_norm": 3.1001712505582586, + "language_loss": 0.93124086, + "learning_rate": 3.720695132111231e-06, + "loss": 0.95255566, + "num_input_tokens_seen": 192167900, + "router_z_loss_clip": 0.55786133, + "router_z_loss_mlp": 0.1854248, + "step": 6725, + "time_per_iteration": 2.801874876022339 + }, + { + "auxiliary_loss_clip": 0.01076064, + "auxiliary_loss_mlp": 0.01038758, + "balance_loss_clip": 1.02695286, + "balance_loss_mlp": 1.02316546, + "epoch": 0.19517149323892985, + "flos": 28068143719680.0, + "grad_norm": 1.6418234862217478, + "language_loss": 0.69258785, + "learning_rate": 3.7205993189389905e-06, + "loss": 0.71373606, + "num_input_tokens_seen": 192183325, + "router_z_loss_clip": 0.4909668, + "router_z_loss_mlp": 0.15594482, + "step": 6726, + "time_per_iteration": 2.4847750663757324 + }, + { + "auxiliary_loss_clip": 0.01080309, + "auxiliary_loss_mlp": 0.0103799, + "balance_loss_clip": 1.02710736, + "balance_loss_mlp": 1.02107477, + "epoch": 0.19520051070744587, + "flos": 21901364985600.0, + "grad_norm": 1.8974598735852382, + "language_loss": 0.90707076, + "learning_rate": 3.7205034905697207e-06, + "loss": 0.92825377, + "num_input_tokens_seen": 192202335, + "router_z_loss_clip": 0.53198242, + "router_z_loss_mlp": 0.16918945, + "step": 6727, + "time_per_iteration": 2.579423427581787 + }, + { + "auxiliary_loss_clip": 0.01015174, + "auxiliary_loss_mlp": 0.01005985, + "balance_loss_clip": 1.00750363, + "balance_loss_mlp": 1.00507915, + "epoch": 0.19522952817596192, + "flos": 67445177281920.0, + "grad_norm": 0.6179281491104532, + "language_loss": 0.48886251, + "learning_rate": 3.7204076470042677e-06, + "loss": 0.50907415, + "num_input_tokens_seen": 192267885, + "router_z_loss_clip": 0.07666016, + "router_z_loss_mlp": 0.0090332, + "step": 6728, + "time_per_iteration": 3.1073546409606934 + }, + { + "auxiliary_loss_clip": 0.01075381, + "auxiliary_loss_mlp": 0.01036786, + "balance_loss_clip": 1.02585721, + "balance_loss_mlp": 1.0233748, + "epoch": 0.19525854564447798, + "flos": 30110981725440.0, + "grad_norm": 2.6891946869189645, + "language_loss": 0.88228846, + "learning_rate": 3.720311788243478e-06, + "loss": 0.90341008, + "num_input_tokens_seen": 192290955, + "router_z_loss_clip": 0.49560547, + "router_z_loss_mlp": 0.13397217, + "step": 6729, + "time_per_iteration": 2.6661875247955322 + }, + { + "auxiliary_loss_clip": 0.01073551, + "auxiliary_loss_mlp": 0.01029928, + "balance_loss_clip": 1.0239265, + "balance_loss_mlp": 1.01580822, + "epoch": 0.19528756311299403, + "flos": 46125013432320.0, + "grad_norm": 2.2783358065572474, + "language_loss": 1.02485299, + "learning_rate": 3.720215914288198e-06, + "loss": 1.04588783, + "num_input_tokens_seen": 192308635, + "router_z_loss_clip": 0.49658203, + "router_z_loss_mlp": 0.14123535, + "step": 6730, + "time_per_iteration": 2.6181604862213135 + }, + { + "auxiliary_loss_clip": 0.01081689, + "auxiliary_loss_mlp": 0.01044351, + "balance_loss_clip": 1.02709448, + "balance_loss_mlp": 1.02760804, + "epoch": 0.19531658058151008, + "flos": 24237124231680.0, + "grad_norm": 2.0476754832531077, + "language_loss": 0.85663283, + "learning_rate": 3.720120025139276e-06, + "loss": 0.87789327, + "num_input_tokens_seen": 192323045, + "router_z_loss_clip": 0.54589844, + "router_z_loss_mlp": 0.16729736, + "step": 6731, + "time_per_iteration": 2.3976778984069824 + }, + { + "auxiliary_loss_clip": 0.01081604, + "auxiliary_loss_mlp": 0.01039124, + "balance_loss_clip": 1.02603924, + "balance_loss_mlp": 1.02140927, + "epoch": 0.1953455980500261, + "flos": 21425359242240.0, + "grad_norm": 2.277476827346477, + "language_loss": 0.78531218, + "learning_rate": 3.720024120797557e-06, + "loss": 0.80651945, + "num_input_tokens_seen": 192337565, + "router_z_loss_clip": 0.5559082, + "router_z_loss_mlp": 0.17712402, + "step": 6732, + "time_per_iteration": 2.43581485748291 + }, + { + "auxiliary_loss_clip": 0.0108254, + "auxiliary_loss_mlp": 0.01039739, + "balance_loss_clip": 1.02676368, + "balance_loss_mlp": 1.02000952, + "epoch": 0.19537461551854216, + "flos": 40403795869440.0, + "grad_norm": 2.018128032521189, + "language_loss": 0.85898006, + "learning_rate": 3.719928201263889e-06, + "loss": 0.88020289, + "num_input_tokens_seen": 192359205, + "router_z_loss_clip": 0.55859375, + "router_z_loss_mlp": 0.1973877, + "step": 6733, + "time_per_iteration": 2.5434327125549316 + }, + { + "auxiliary_loss_clip": 0.01013235, + "auxiliary_loss_mlp": 0.01022214, + "balance_loss_clip": 1.00588632, + "balance_loss_mlp": 1.02125406, + "epoch": 0.1954036329870582, + "flos": 74701560637440.0, + "grad_norm": 0.6722211841060385, + "language_loss": 0.4821341, + "learning_rate": 3.71983226653912e-06, + "loss": 0.50248861, + "num_input_tokens_seen": 192416725, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00958252, + "step": 6734, + "time_per_iteration": 3.101858377456665 + }, + { + "auxiliary_loss_clip": 0.01015056, + "auxiliary_loss_mlp": 0.01011822, + "balance_loss_clip": 1.00772023, + "balance_loss_mlp": 1.01083899, + "epoch": 0.19543265045557426, + "flos": 74778090071040.0, + "grad_norm": 0.6219652604668703, + "language_loss": 0.47758681, + "learning_rate": 3.7197363166240957e-06, + "loss": 0.49785557, + "num_input_tokens_seen": 192485435, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00982666, + "step": 6735, + "time_per_iteration": 3.11666202545166 + }, + { + "auxiliary_loss_clip": 0.01016215, + "auxiliary_loss_mlp": 0.01020262, + "balance_loss_clip": 1.00885797, + "balance_loss_mlp": 1.01933181, + "epoch": 0.1954616679240903, + "flos": 64138973351040.0, + "grad_norm": 0.6821252316417135, + "language_loss": 0.47068927, + "learning_rate": 3.7196403515196647e-06, + "loss": 0.49105406, + "num_input_tokens_seen": 192535675, + "router_z_loss_clip": 0.07324219, + "router_z_loss_mlp": 0.00927734, + "step": 6736, + "time_per_iteration": 2.9484715461730957 + }, + { + "auxiliary_loss_clip": 0.01084893, + "auxiliary_loss_mlp": 0.01033638, + "balance_loss_clip": 1.03116679, + "balance_loss_mlp": 1.01713943, + "epoch": 0.19549068539260636, + "flos": 17192594206080.0, + "grad_norm": 2.975023545370008, + "language_loss": 0.83759707, + "learning_rate": 3.719544371226674e-06, + "loss": 0.85878241, + "num_input_tokens_seen": 192548055, + "router_z_loss_clip": 0.53759766, + "router_z_loss_mlp": 0.16503906, + "step": 6737, + "time_per_iteration": 2.3675124645233154 + }, + { + "auxiliary_loss_clip": 0.01019892, + "auxiliary_loss_mlp": 0.01004863, + "balance_loss_clip": 1.01199651, + "balance_loss_mlp": 1.00391507, + "epoch": 0.19551970286112239, + "flos": 70422885296640.0, + "grad_norm": 0.7214996629275459, + "language_loss": 0.51416701, + "learning_rate": 3.719448375745972e-06, + "loss": 0.53441459, + "num_input_tokens_seen": 192605645, + "router_z_loss_clip": 0.07910156, + "router_z_loss_mlp": 0.00946045, + "step": 6738, + "time_per_iteration": 3.021958827972412 + }, + { + "auxiliary_loss_clip": 0.01020259, + "auxiliary_loss_mlp": 0.01001148, + "balance_loss_clip": 1.01221681, + "balance_loss_mlp": 1.00022459, + "epoch": 0.19554872032963844, + "flos": 59514796529280.0, + "grad_norm": 0.643431656729413, + "language_loss": 0.46425247, + "learning_rate": 3.7193523650784054e-06, + "loss": 0.48446655, + "num_input_tokens_seen": 192665315, + "router_z_loss_clip": 0.08007812, + "router_z_loss_mlp": 0.00921631, + "step": 6739, + "time_per_iteration": 2.929166793823242 + }, + { + "auxiliary_loss_clip": 0.01089764, + "auxiliary_loss_mlp": 0.01047534, + "balance_loss_clip": 1.03188133, + "balance_loss_mlp": 1.02816272, + "epoch": 0.1955777377981545, + "flos": 74732844948480.0, + "grad_norm": 3.8424398571116245, + "language_loss": 0.72207904, + "learning_rate": 3.7192563392248235e-06, + "loss": 0.74345207, + "num_input_tokens_seen": 192691100, + "router_z_loss_clip": 0.57861328, + "router_z_loss_mlp": 0.19366455, + "step": 6740, + "time_per_iteration": 2.831786870956421 + }, + { + "auxiliary_loss_clip": 0.01024554, + "auxiliary_loss_mlp": 0.01000648, + "balance_loss_clip": 1.01589942, + "balance_loss_mlp": 0.99956346, + "epoch": 0.19560675526667054, + "flos": 54747894602880.0, + "grad_norm": 0.6766526068242428, + "language_loss": 0.4772684, + "learning_rate": 3.7191602981860737e-06, + "loss": 0.49752039, + "num_input_tokens_seen": 192750150, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.01086426, + "step": 6741, + "time_per_iteration": 3.006518840789795 + }, + { + "auxiliary_loss_clip": 0.01081613, + "auxiliary_loss_mlp": 0.01048923, + "balance_loss_clip": 1.03067124, + "balance_loss_mlp": 1.03375959, + "epoch": 0.1956357727351866, + "flos": 41821653893760.0, + "grad_norm": 2.733607058033153, + "language_loss": 0.95255649, + "learning_rate": 3.7190642419630043e-06, + "loss": 0.97386181, + "num_input_tokens_seen": 192767790, + "router_z_loss_clip": 0.50976562, + "router_z_loss_mlp": 0.1517334, + "step": 6742, + "time_per_iteration": 2.5552425384521484 + }, + { + "auxiliary_loss_clip": 0.01089447, + "auxiliary_loss_mlp": 0.01061165, + "balance_loss_clip": 1.03773522, + "balance_loss_mlp": 1.04474986, + "epoch": 0.19566479020370262, + "flos": 25401198516480.0, + "grad_norm": 2.2065527547810544, + "language_loss": 0.62631041, + "learning_rate": 3.7189681705564645e-06, + "loss": 0.64781654, + "num_input_tokens_seen": 192783605, + "router_z_loss_clip": 0.5168457, + "router_z_loss_mlp": 0.1640625, + "step": 6743, + "time_per_iteration": 2.441770076751709 + }, + { + "auxiliary_loss_clip": 0.0108982, + "auxiliary_loss_mlp": 0.01076113, + "balance_loss_clip": 1.03285539, + "balance_loss_mlp": 1.05690873, + "epoch": 0.19569380767221867, + "flos": 28759668485760.0, + "grad_norm": 2.0380680964058357, + "language_loss": 0.95450604, + "learning_rate": 3.718872083967302e-06, + "loss": 0.97616541, + "num_input_tokens_seen": 192802860, + "router_z_loss_clip": 0.56933594, + "router_z_loss_mlp": 0.19219971, + "step": 6744, + "time_per_iteration": 2.5437662601470947 + }, + { + "auxiliary_loss_clip": 0.01084465, + "auxiliary_loss_mlp": 0.0108131, + "balance_loss_clip": 1.03270602, + "balance_loss_mlp": 1.06444192, + "epoch": 0.19572282514073472, + "flos": 11720028412800.0, + "grad_norm": 2.6441692458164803, + "language_loss": 0.77441657, + "learning_rate": 3.7187759821963657e-06, + "loss": 0.79607433, + "num_input_tokens_seen": 192813230, + "router_z_loss_clip": 0.51806641, + "router_z_loss_mlp": 0.1685791, + "step": 6745, + "time_per_iteration": 2.3804173469543457 + }, + { + "auxiliary_loss_clip": 0.01082958, + "auxiliary_loss_mlp": 0.01067135, + "balance_loss_clip": 1.03123975, + "balance_loss_mlp": 1.051525, + "epoch": 0.19575184260925077, + "flos": 33327075703680.0, + "grad_norm": 1.7884495577660815, + "language_loss": 0.73128867, + "learning_rate": 3.7186798652445043e-06, + "loss": 0.75278962, + "num_input_tokens_seen": 192834315, + "router_z_loss_clip": 0.51611328, + "router_z_loss_mlp": 0.15600586, + "step": 6746, + "time_per_iteration": 2.5119118690490723 + }, + { + "auxiliary_loss_clip": 0.01022643, + "auxiliary_loss_mlp": 0.01026264, + "balance_loss_clip": 1.01335168, + "balance_loss_mlp": 1.02491677, + "epoch": 0.19578086007776682, + "flos": 73313867894400.0, + "grad_norm": 0.6997861231884336, + "language_loss": 0.58243084, + "learning_rate": 3.7185837331125665e-06, + "loss": 0.60291994, + "num_input_tokens_seen": 192900960, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.01348877, + "step": 6747, + "time_per_iteration": 3.134704828262329 + }, + { + "auxiliary_loss_clip": 0.01076789, + "auxiliary_loss_mlp": 0.01060128, + "balance_loss_clip": 1.02824819, + "balance_loss_mlp": 1.04541159, + "epoch": 0.19580987754628287, + "flos": 19529296058880.0, + "grad_norm": 2.872789910996726, + "language_loss": 0.91483545, + "learning_rate": 3.7184875858014022e-06, + "loss": 0.93620455, + "num_input_tokens_seen": 192912735, + "router_z_loss_clip": 0.48535156, + "router_z_loss_mlp": 0.14715576, + "step": 6748, + "time_per_iteration": 2.492729663848877 + }, + { + "auxiliary_loss_clip": 0.01082722, + "auxiliary_loss_mlp": 0.01064761, + "balance_loss_clip": 1.03166676, + "balance_loss_mlp": 1.0492698, + "epoch": 0.1958388950147989, + "flos": 37663079230080.0, + "grad_norm": 1.980286682276728, + "language_loss": 0.81951648, + "learning_rate": 3.7183914233118603e-06, + "loss": 0.84099132, + "num_input_tokens_seen": 192929295, + "router_z_loss_clip": 0.51000977, + "router_z_loss_mlp": 0.15490723, + "step": 6749, + "time_per_iteration": 2.656712293624878 + }, + { + "auxiliary_loss_clip": 0.01080973, + "auxiliary_loss_mlp": 0.0106465, + "balance_loss_clip": 1.02907777, + "balance_loss_mlp": 1.0479964, + "epoch": 0.19586791248331495, + "flos": 60064785043200.0, + "grad_norm": 2.225330674251466, + "language_loss": 0.86555535, + "learning_rate": 3.71829524564479e-06, + "loss": 0.88701153, + "num_input_tokens_seen": 192952070, + "router_z_loss_clip": 0.51953125, + "router_z_loss_mlp": 0.16650391, + "step": 6750, + "time_per_iteration": 2.6712276935577393 + }, + { + "auxiliary_loss_clip": 0.01017457, + "auxiliary_loss_mlp": 0.01037876, + "balance_loss_clip": 1.00919724, + "balance_loss_mlp": 1.03682697, + "epoch": 0.195896929951831, + "flos": 66089814324480.0, + "grad_norm": 0.6420950829933821, + "language_loss": 0.52692056, + "learning_rate": 3.71819905280104e-06, + "loss": 0.54747391, + "num_input_tokens_seen": 193015025, + "router_z_loss_clip": 0.08251953, + "router_z_loss_mlp": 0.01049805, + "step": 6751, + "time_per_iteration": 3.02704119682312 + }, + { + "auxiliary_loss_clip": 0.01015089, + "auxiliary_loss_mlp": 0.01022209, + "balance_loss_clip": 1.00703597, + "balance_loss_mlp": 1.02119601, + "epoch": 0.19592594742034705, + "flos": 67879217704320.0, + "grad_norm": 0.6751483743730132, + "language_loss": 0.49020708, + "learning_rate": 3.7181028447814613e-06, + "loss": 0.51058006, + "num_input_tokens_seen": 193081865, + "router_z_loss_clip": 0.08056641, + "router_z_loss_mlp": 0.01013184, + "step": 6752, + "time_per_iteration": 3.1890416145324707 + }, + { + "auxiliary_loss_clip": 0.01090208, + "auxiliary_loss_mlp": 0.01045711, + "balance_loss_clip": 1.03308713, + "balance_loss_mlp": 1.02778244, + "epoch": 0.1959549648888631, + "flos": 30146732824320.0, + "grad_norm": 2.5665781487286004, + "language_loss": 0.94244862, + "learning_rate": 3.718006621586903e-06, + "loss": 0.96380782, + "num_input_tokens_seen": 193097280, + "router_z_loss_clip": 0.57202148, + "router_z_loss_mlp": 0.17926025, + "step": 6753, + "time_per_iteration": 2.4789552688598633 + }, + { + "auxiliary_loss_clip": 0.0102021, + "auxiliary_loss_mlp": 0.01020778, + "balance_loss_clip": 1.01208031, + "balance_loss_mlp": 1.01990199, + "epoch": 0.19598398235737916, + "flos": 74761786465920.0, + "grad_norm": 0.6895709957924783, + "language_loss": 0.45549744, + "learning_rate": 3.717910383218215e-06, + "loss": 0.4759073, + "num_input_tokens_seen": 193152220, + "router_z_loss_clip": 0.08105469, + "router_z_loss_mlp": 0.00878906, + "step": 6754, + "time_per_iteration": 2.97517466545105 + }, + { + "auxiliary_loss_clip": 0.01023281, + "auxiliary_loss_mlp": 0.01015743, + "balance_loss_clip": 1.01503348, + "balance_loss_mlp": 1.01483727, + "epoch": 0.19601299982589518, + "flos": 74765696538240.0, + "grad_norm": 0.6631063879801591, + "language_loss": 0.47268164, + "learning_rate": 3.717814129676247e-06, + "loss": 0.49307188, + "num_input_tokens_seen": 193210835, + "router_z_loss_clip": 0.08251953, + "router_z_loss_mlp": 0.0090332, + "step": 6755, + "time_per_iteration": 3.071932554244995 + }, + { + "auxiliary_loss_clip": 0.01093243, + "auxiliary_loss_mlp": 0.01035036, + "balance_loss_clip": 1.04132926, + "balance_loss_mlp": 1.0187099, + "epoch": 0.19604201729441123, + "flos": 27920356467840.0, + "grad_norm": 3.505777795985707, + "language_loss": 0.71377194, + "learning_rate": 3.71771786096185e-06, + "loss": 0.73505473, + "num_input_tokens_seen": 193225640, + "router_z_loss_clip": 0.51879883, + "router_z_loss_mlp": 0.16308594, + "step": 6756, + "time_per_iteration": 2.474424362182617 + }, + { + "auxiliary_loss_clip": 0.0108999, + "auxiliary_loss_mlp": 0.01033248, + "balance_loss_clip": 1.03929567, + "balance_loss_mlp": 1.01815581, + "epoch": 0.19607103476292728, + "flos": 31936764608640.0, + "grad_norm": 1.992145730611596, + "language_loss": 0.6623823, + "learning_rate": 3.7176215770758734e-06, + "loss": 0.68361473, + "num_input_tokens_seen": 193241785, + "router_z_loss_clip": 0.50683594, + "router_z_loss_mlp": 0.15112305, + "step": 6757, + "time_per_iteration": 2.5200252532958984 + }, + { + "auxiliary_loss_clip": 0.01088808, + "auxiliary_loss_mlp": 0.01031359, + "balance_loss_clip": 1.04543495, + "balance_loss_mlp": 1.01944399, + "epoch": 0.19610005223144333, + "flos": 11539143325440.0, + "grad_norm": 2.428756169289822, + "language_loss": 0.78631067, + "learning_rate": 3.7175252780191683e-06, + "loss": 0.80751228, + "num_input_tokens_seen": 193254425, + "router_z_loss_clip": 0.43457031, + "router_z_loss_mlp": 0.11907959, + "step": 6758, + "time_per_iteration": 2.399430274963379 + }, + { + "auxiliary_loss_clip": 0.01095694, + "auxiliary_loss_mlp": 0.01044381, + "balance_loss_clip": 1.04514837, + "balance_loss_mlp": 1.02911043, + "epoch": 0.19612906969995939, + "flos": 28939890257280.0, + "grad_norm": 2.0441405198378217, + "language_loss": 0.72194099, + "learning_rate": 3.7174289637925843e-06, + "loss": 0.74334174, + "num_input_tokens_seen": 193268325, + "router_z_loss_clip": 0.50561523, + "router_z_loss_mlp": 0.15270996, + "step": 6759, + "time_per_iteration": 2.4571216106414795 + }, + { + "auxiliary_loss_clip": 0.01046022, + "auxiliary_loss_mlp": 0.01008017, + "balance_loss_clip": 1.03643227, + "balance_loss_mlp": 1.00714111, + "epoch": 0.1961580871684754, + "flos": 74773865796480.0, + "grad_norm": 0.6633298214626389, + "language_loss": 0.49496156, + "learning_rate": 3.7173326343969734e-06, + "loss": 0.51550192, + "num_input_tokens_seen": 193333915, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.00878906, + "step": 6760, + "time_per_iteration": 3.101440191268921 + }, + { + "auxiliary_loss_clip": 0.01101925, + "auxiliary_loss_mlp": 0.01033947, + "balance_loss_clip": 1.0487324, + "balance_loss_mlp": 1.01813364, + "epoch": 0.19618710463699146, + "flos": 35400881952000.0, + "grad_norm": 2.0178720446972194, + "language_loss": 0.89034164, + "learning_rate": 3.7172362898331856e-06, + "loss": 0.91170031, + "num_input_tokens_seen": 193351530, + "router_z_loss_clip": 0.53222656, + "router_z_loss_mlp": 0.15826416, + "step": 6761, + "time_per_iteration": 2.847574234008789 + }, + { + "auxiliary_loss_clip": 0.01100504, + "auxiliary_loss_mlp": 0.01044855, + "balance_loss_clip": 1.04542589, + "balance_loss_mlp": 1.02874374, + "epoch": 0.1962161221055075, + "flos": 24488603821440.0, + "grad_norm": 2.3117849250102, + "language_loss": 1.06055689, + "learning_rate": 3.7171399301020714e-06, + "loss": 1.08201051, + "num_input_tokens_seen": 193367590, + "router_z_loss_clip": 0.55053711, + "router_z_loss_mlp": 0.16107178, + "step": 6762, + "time_per_iteration": 2.4014909267425537 + }, + { + "auxiliary_loss_clip": 0.01102506, + "auxiliary_loss_mlp": 0.01037297, + "balance_loss_clip": 1.04699719, + "balance_loss_mlp": 1.02123392, + "epoch": 0.19624513957402356, + "flos": 27194686525440.0, + "grad_norm": 1.8110994310152146, + "language_loss": 0.99795943, + "learning_rate": 3.7170435552044834e-06, + "loss": 1.01935744, + "num_input_tokens_seen": 193389145, + "router_z_loss_clip": 0.5546875, + "router_z_loss_mlp": 0.1605835, + "step": 6763, + "time_per_iteration": 2.5953452587127686 + }, + { + "auxiliary_loss_clip": 0.01098241, + "auxiliary_loss_mlp": 0.01050133, + "balance_loss_clip": 1.04573226, + "balance_loss_mlp": 1.03463006, + "epoch": 0.19627415704253962, + "flos": 42295669689600.0, + "grad_norm": 2.1606287499408214, + "language_loss": 0.85351908, + "learning_rate": 3.7169471651412714e-06, + "loss": 0.87500286, + "num_input_tokens_seen": 193413790, + "router_z_loss_clip": 0.52490234, + "router_z_loss_mlp": 0.15515137, + "step": 6764, + "time_per_iteration": 2.5678744316101074 + }, + { + "auxiliary_loss_clip": 0.01048714, + "auxiliary_loss_mlp": 0.01005712, + "balance_loss_clip": 1.03853142, + "balance_loss_mlp": 1.00476432, + "epoch": 0.19630317451105567, + "flos": 68216617883520.0, + "grad_norm": 0.676592196299911, + "language_loss": 0.47269717, + "learning_rate": 3.716850759913287e-06, + "loss": 0.49324143, + "num_input_tokens_seen": 193481410, + "router_z_loss_clip": 0.10205078, + "router_z_loss_mlp": 0.00946045, + "step": 6765, + "time_per_iteration": 3.2624943256378174 + }, + { + "auxiliary_loss_clip": 0.01043202, + "auxiliary_loss_mlp": 0.01004557, + "balance_loss_clip": 1.03312922, + "balance_loss_mlp": 1.00354981, + "epoch": 0.1963321919795717, + "flos": 65472689399040.0, + "grad_norm": 0.6208662343203053, + "language_loss": 0.45453781, + "learning_rate": 3.7167543395213824e-06, + "loss": 0.4750154, + "num_input_tokens_seen": 193545285, + "router_z_loss_clip": 0.10058594, + "router_z_loss_mlp": 0.0100708, + "step": 6766, + "time_per_iteration": 3.031984329223633 + }, + { + "auxiliary_loss_clip": 0.01085028, + "auxiliary_loss_mlp": 0.01050365, + "balance_loss_clip": 1.03363776, + "balance_loss_mlp": 1.03527284, + "epoch": 0.19636120944808774, + "flos": 16355027756160.0, + "grad_norm": 2.4195495954493302, + "language_loss": 0.81799769, + "learning_rate": 3.716657903966409e-06, + "loss": 0.83935165, + "num_input_tokens_seen": 193559045, + "router_z_loss_clip": 0.51367188, + "router_z_loss_mlp": 0.15081787, + "step": 6767, + "time_per_iteration": 2.389909505844116 + }, + { + "auxiliary_loss_clip": 0.01090002, + "auxiliary_loss_mlp": 0.0106051, + "balance_loss_clip": 1.03556454, + "balance_loss_mlp": 1.0441128, + "epoch": 0.1963902269166038, + "flos": 13289932874880.0, + "grad_norm": 2.557881351235596, + "language_loss": 0.81399858, + "learning_rate": 3.716561453249218e-06, + "loss": 0.8355037, + "num_input_tokens_seen": 193570200, + "router_z_loss_clip": 0.54296875, + "router_z_loss_mlp": 0.16400146, + "step": 6768, + "time_per_iteration": 2.3680272102355957 + }, + { + "auxiliary_loss_clip": 0.01087401, + "auxiliary_loss_mlp": 0.01054185, + "balance_loss_clip": 1.03385437, + "balance_loss_mlp": 1.03741813, + "epoch": 0.19641924438511985, + "flos": 68161492846080.0, + "grad_norm": 2.6967933539816347, + "language_loss": 0.829588, + "learning_rate": 3.7164649873706617e-06, + "loss": 0.85100377, + "num_input_tokens_seen": 193590130, + "router_z_loss_clip": 0.53540039, + "router_z_loss_mlp": 0.16760254, + "step": 6769, + "time_per_iteration": 2.7800183296203613 + }, + { + "auxiliary_loss_clip": 0.01085546, + "auxiliary_loss_mlp": 0.01069512, + "balance_loss_clip": 1.03402865, + "balance_loss_mlp": 1.05408049, + "epoch": 0.1964482618536359, + "flos": 13034543212800.0, + "grad_norm": 4.980681865766263, + "language_loss": 0.67431831, + "learning_rate": 3.716368506331592e-06, + "loss": 0.69586891, + "num_input_tokens_seen": 193604825, + "router_z_loss_clip": 0.515625, + "router_z_loss_mlp": 0.15441895, + "step": 6770, + "time_per_iteration": 2.3684940338134766 + }, + { + "auxiliary_loss_clip": 0.01091883, + "auxiliary_loss_mlp": 0.01063962, + "balance_loss_clip": 1.03792691, + "balance_loss_mlp": 1.04760647, + "epoch": 0.19647727932215195, + "flos": 31824693544320.0, + "grad_norm": 1.962177999244711, + "language_loss": 0.84945679, + "learning_rate": 3.7162720101328607e-06, + "loss": 0.87101531, + "num_input_tokens_seen": 193624230, + "router_z_loss_clip": 0.54052734, + "router_z_loss_mlp": 0.16357422, + "step": 6771, + "time_per_iteration": 2.5155117511749268 + }, + { + "auxiliary_loss_clip": 0.01089831, + "auxiliary_loss_mlp": 0.01059671, + "balance_loss_clip": 1.03590608, + "balance_loss_mlp": 1.0447166, + "epoch": 0.19650629679066797, + "flos": 17012965927680.0, + "grad_norm": 2.1311826525987265, + "language_loss": 0.73565596, + "learning_rate": 3.716175498775321e-06, + "loss": 0.75715095, + "num_input_tokens_seen": 193641105, + "router_z_loss_clip": 0.5390625, + "router_z_loss_mlp": 0.1494751, + "step": 6772, + "time_per_iteration": 4.7238404750823975 + }, + { + "auxiliary_loss_clip": 0.01084938, + "auxiliary_loss_mlp": 0.01057451, + "balance_loss_clip": 1.0366559, + "balance_loss_mlp": 1.04279459, + "epoch": 0.19653531425918402, + "flos": 23361362887680.0, + "grad_norm": 1.8923266656977251, + "language_loss": 0.82364738, + "learning_rate": 3.716078972259825e-06, + "loss": 0.84507126, + "num_input_tokens_seen": 193655920, + "router_z_loss_clip": 0.48217773, + "router_z_loss_mlp": 0.14654541, + "step": 6773, + "time_per_iteration": 4.86262845993042 + }, + { + "auxiliary_loss_clip": 0.01032945, + "auxiliary_loss_mlp": 0.01046894, + "balance_loss_clip": 1.02358401, + "balance_loss_mlp": 1.0458982, + "epoch": 0.19656433172770008, + "flos": 74761367529600.0, + "grad_norm": 0.6853681257415173, + "language_loss": 0.47926468, + "learning_rate": 3.7159824305872247e-06, + "loss": 0.50006306, + "num_input_tokens_seen": 193711615, + "router_z_loss_clip": 0.09375, + "router_z_loss_mlp": 0.00994873, + "step": 6774, + "time_per_iteration": 2.957732677459717 + }, + { + "auxiliary_loss_clip": 0.01099224, + "auxiliary_loss_mlp": 0.01048044, + "balance_loss_clip": 1.0442512, + "balance_loss_mlp": 1.02889311, + "epoch": 0.19659334919621613, + "flos": 19456537052160.0, + "grad_norm": 2.3078725142027836, + "language_loss": 0.74032575, + "learning_rate": 3.7158858737583733e-06, + "loss": 0.76179838, + "num_input_tokens_seen": 193726740, + "router_z_loss_clip": 0.54980469, + "router_z_loss_mlp": 0.19165039, + "step": 6775, + "time_per_iteration": 2.3984341621398926 + }, + { + "auxiliary_loss_clip": 0.01102043, + "auxiliary_loss_mlp": 0.01045942, + "balance_loss_clip": 1.04829073, + "balance_loss_mlp": 1.02876365, + "epoch": 0.19662236666473218, + "flos": 36313371912960.0, + "grad_norm": 1.8503002099964, + "language_loss": 0.85226429, + "learning_rate": 3.7157893017741234e-06, + "loss": 0.87374407, + "num_input_tokens_seen": 193744105, + "router_z_loss_clip": 0.53808594, + "router_z_loss_mlp": 0.17175293, + "step": 6776, + "time_per_iteration": 2.525002956390381 + }, + { + "auxiliary_loss_clip": 0.01096945, + "auxiliary_loss_mlp": 0.0104254, + "balance_loss_clip": 1.04674053, + "balance_loss_mlp": 1.02794242, + "epoch": 0.1966513841332482, + "flos": 26171068106880.0, + "grad_norm": 3.201816354008393, + "language_loss": 0.82122594, + "learning_rate": 3.7156927146353284e-06, + "loss": 0.84262079, + "num_input_tokens_seen": 193759185, + "router_z_loss_clip": 0.50219727, + "router_z_loss_mlp": 0.14599609, + "step": 6777, + "time_per_iteration": 2.431908369064331 + }, + { + "auxiliary_loss_clip": 0.01101156, + "auxiliary_loss_mlp": 0.01043633, + "balance_loss_clip": 1.04598665, + "balance_loss_mlp": 1.02707469, + "epoch": 0.19668040160176425, + "flos": 31094973884160.0, + "grad_norm": 2.0443990038791044, + "language_loss": 0.90777647, + "learning_rate": 3.7155961123428407e-06, + "loss": 0.92922437, + "num_input_tokens_seen": 193783395, + "router_z_loss_clip": 0.55200195, + "router_z_loss_mlp": 0.16540527, + "step": 6778, + "time_per_iteration": 2.698418140411377 + }, + { + "auxiliary_loss_clip": 0.01103047, + "auxiliary_loss_mlp": 0.01034, + "balance_loss_clip": 1.05120063, + "balance_loss_mlp": 1.01922417, + "epoch": 0.1967094190702803, + "flos": 25513828162560.0, + "grad_norm": 1.8533351944890755, + "language_loss": 0.98321569, + "learning_rate": 3.7154994948975143e-06, + "loss": 1.00458622, + "num_input_tokens_seen": 193800720, + "router_z_loss_clip": 0.51806641, + "router_z_loss_mlp": 0.14764404, + "step": 6779, + "time_per_iteration": 2.4912056922912598 + }, + { + "auxiliary_loss_clip": 0.01052905, + "auxiliary_loss_mlp": 0.01006015, + "balance_loss_clip": 1.04304397, + "balance_loss_mlp": 1.00497818, + "epoch": 0.19673843653879636, + "flos": 64667557468800.0, + "grad_norm": 0.6710067230688077, + "language_loss": 0.49844778, + "learning_rate": 3.7154028623002016e-06, + "loss": 0.51903695, + "num_input_tokens_seen": 193860920, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.01037598, + "step": 6780, + "time_per_iteration": 3.0587708950042725 + }, + { + "auxiliary_loss_clip": 0.01054041, + "auxiliary_loss_mlp": 0.01005547, + "balance_loss_clip": 1.04417515, + "balance_loss_mlp": 1.00442052, + "epoch": 0.1967674540073124, + "flos": 73788547006080.0, + "grad_norm": 0.6857094429227036, + "language_loss": 0.46537709, + "learning_rate": 3.7153062145517565e-06, + "loss": 0.48597297, + "num_input_tokens_seen": 193924980, + "router_z_loss_clip": 0.09863281, + "router_z_loss_mlp": 0.0112915, + "step": 6781, + "time_per_iteration": 5.50628399848938 + }, + { + "auxiliary_loss_clip": 0.01048905, + "auxiliary_loss_mlp": 0.01004648, + "balance_loss_clip": 1.03854537, + "balance_loss_mlp": 1.00361097, + "epoch": 0.19679647147582846, + "flos": 74767861042560.0, + "grad_norm": 0.6673054614141039, + "language_loss": 0.49902669, + "learning_rate": 3.715209551653034e-06, + "loss": 0.51956224, + "num_input_tokens_seen": 193988520, + "router_z_loss_clip": 0.10351562, + "router_z_loss_mlp": 0.01037598, + "step": 6782, + "time_per_iteration": 5.309893369674683 + }, + { + "auxiliary_loss_clip": 0.01102777, + "auxiliary_loss_mlp": 0.01038208, + "balance_loss_clip": 1.04752672, + "balance_loss_mlp": 1.02097034, + "epoch": 0.19682548894434448, + "flos": 24125751394560.0, + "grad_norm": 2.241755817353307, + "language_loss": 0.73810434, + "learning_rate": 3.7151128736048855e-06, + "loss": 0.75951421, + "num_input_tokens_seen": 194004690, + "router_z_loss_clip": 0.55249023, + "router_z_loss_mlp": 0.17242432, + "step": 6783, + "time_per_iteration": 2.4169070720672607 + }, + { + "auxiliary_loss_clip": 0.01108389, + "auxiliary_loss_mlp": 0.01047509, + "balance_loss_clip": 1.05367851, + "balance_loss_mlp": 1.03195262, + "epoch": 0.19685450641286054, + "flos": 39523286580480.0, + "grad_norm": 1.7744992477700974, + "language_loss": 0.77958, + "learning_rate": 3.715016180408167e-06, + "loss": 0.801139, + "num_input_tokens_seen": 194022895, + "router_z_loss_clip": 0.54736328, + "router_z_loss_mlp": 0.15551758, + "step": 6784, + "time_per_iteration": 2.8442912101745605 + }, + { + "auxiliary_loss_clip": 0.01105885, + "auxiliary_loss_mlp": 0.01039383, + "balance_loss_clip": 1.04941154, + "balance_loss_mlp": 1.02205038, + "epoch": 0.1968835238813766, + "flos": 28795035559680.0, + "grad_norm": 2.186360321981528, + "language_loss": 0.84224063, + "learning_rate": 3.7149194720637313e-06, + "loss": 0.8636933, + "num_input_tokens_seen": 194040075, + "router_z_loss_clip": 0.56542969, + "router_z_loss_mlp": 0.17327881, + "step": 6785, + "time_per_iteration": 2.4775924682617188 + }, + { + "auxiliary_loss_clip": 0.01103299, + "auxiliary_loss_mlp": 0.0103995, + "balance_loss_clip": 1.0491128, + "balance_loss_mlp": 1.02452421, + "epoch": 0.19691254134989264, + "flos": 54478891376640.0, + "grad_norm": 3.1669901857622875, + "language_loss": 0.81607449, + "learning_rate": 3.714822748572432e-06, + "loss": 0.83750695, + "num_input_tokens_seen": 194058855, + "router_z_loss_clip": 0.54150391, + "router_z_loss_mlp": 0.15435791, + "step": 6786, + "time_per_iteration": 2.7512094974517822 + }, + { + "auxiliary_loss_clip": 0.01043651, + "auxiliary_loss_mlp": 0.01002123, + "balance_loss_clip": 1.03313112, + "balance_loss_mlp": 1.0011102, + "epoch": 0.1969415588184087, + "flos": 66385458650880.0, + "grad_norm": 0.6452100370275289, + "language_loss": 0.49518603, + "learning_rate": 3.7147260099351252e-06, + "loss": 0.51564372, + "num_input_tokens_seen": 194122215, + "router_z_loss_clip": 0.10546875, + "router_z_loss_mlp": 0.01013184, + "step": 6787, + "time_per_iteration": 3.0165679454803467 + }, + { + "auxiliary_loss_clip": 0.01103911, + "auxiliary_loss_mlp": 0.01048202, + "balance_loss_clip": 1.04757094, + "balance_loss_mlp": 1.03078568, + "epoch": 0.19697057628692474, + "flos": 31793131808640.0, + "grad_norm": 2.3542455599772194, + "language_loss": 0.86138588, + "learning_rate": 3.7146292561526648e-06, + "loss": 0.88290691, + "num_input_tokens_seen": 194141705, + "router_z_loss_clip": 0.5625, + "router_z_loss_mlp": 0.17419434, + "step": 6788, + "time_per_iteration": 2.520106315612793 + }, + { + "auxiliary_loss_clip": 0.01039157, + "auxiliary_loss_mlp": 0.01001549, + "balance_loss_clip": 1.02857256, + "balance_loss_mlp": 1.00069702, + "epoch": 0.19699959375544077, + "flos": 67320782507520.0, + "grad_norm": 0.7110931034649715, + "language_loss": 0.50031179, + "learning_rate": 3.714532487225904e-06, + "loss": 0.52071887, + "num_input_tokens_seen": 194202905, + "router_z_loss_clip": 0.10595703, + "router_z_loss_mlp": 0.00854492, + "step": 6789, + "time_per_iteration": 3.0678601264953613 + }, + { + "auxiliary_loss_clip": 0.01093698, + "auxiliary_loss_mlp": 0.01044461, + "balance_loss_clip": 1.04148769, + "balance_loss_mlp": 1.02879131, + "epoch": 0.19702861122395682, + "flos": 28249552477440.0, + "grad_norm": 2.004987784622358, + "language_loss": 0.77100736, + "learning_rate": 3.7144357031556986e-06, + "loss": 0.79238892, + "num_input_tokens_seen": 194222510, + "router_z_loss_clip": 0.52197266, + "router_z_loss_mlp": 0.15679932, + "step": 6790, + "time_per_iteration": 2.469311237335205 + }, + { + "auxiliary_loss_clip": 0.01094129, + "auxiliary_loss_mlp": 0.01049227, + "balance_loss_clip": 1.04064679, + "balance_loss_mlp": 1.03325868, + "epoch": 0.19705762869247287, + "flos": 32553610243200.0, + "grad_norm": 2.1718309975607775, + "language_loss": 0.89229298, + "learning_rate": 3.714338903942904e-06, + "loss": 0.91372657, + "num_input_tokens_seen": 194237620, + "router_z_loss_clip": 0.53466797, + "router_z_loss_mlp": 0.15960693, + "step": 6791, + "time_per_iteration": 2.5833213329315186 + }, + { + "auxiliary_loss_clip": 0.01099248, + "auxiliary_loss_mlp": 0.01050158, + "balance_loss_clip": 1.0421083, + "balance_loss_mlp": 1.03249156, + "epoch": 0.19708664616098892, + "flos": 27009751720320.0, + "grad_norm": 2.1247526497073714, + "language_loss": 0.90555823, + "learning_rate": 3.714242089588374e-06, + "loss": 0.92705238, + "num_input_tokens_seen": 194255135, + "router_z_loss_clip": 0.57177734, + "router_z_loss_mlp": 0.17675781, + "step": 6792, + "time_per_iteration": 2.5379388332366943 + }, + { + "auxiliary_loss_clip": 0.01029094, + "auxiliary_loss_mlp": 0.01002482, + "balance_loss_clip": 1.02020407, + "balance_loss_mlp": 1.00146878, + "epoch": 0.19711566362950497, + "flos": 61377238206720.0, + "grad_norm": 0.6616219496945035, + "language_loss": 0.46480322, + "learning_rate": 3.714145260092964e-06, + "loss": 0.48511899, + "num_input_tokens_seen": 194315000, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.01013184, + "step": 6793, + "time_per_iteration": 3.042858123779297 + }, + { + "auxiliary_loss_clip": 0.01093783, + "auxiliary_loss_mlp": 0.01050004, + "balance_loss_clip": 1.03879666, + "balance_loss_mlp": 1.03274822, + "epoch": 0.197144681098021, + "flos": 39235602044160.0, + "grad_norm": 2.4531005569563495, + "language_loss": 0.61764085, + "learning_rate": 3.7140484154575294e-06, + "loss": 0.63907868, + "num_input_tokens_seen": 194333560, + "router_z_loss_clip": 0.55078125, + "router_z_loss_mlp": 0.17248535, + "step": 6794, + "time_per_iteration": 2.57737135887146 + }, + { + "auxiliary_loss_clip": 0.01025272, + "auxiliary_loss_mlp": 0.01004008, + "balance_loss_clip": 1.0165571, + "balance_loss_mlp": 1.0029943, + "epoch": 0.19717369856653705, + "flos": 56792827290240.0, + "grad_norm": 0.6852760576581253, + "language_loss": 0.51701349, + "learning_rate": 3.7139515556829263e-06, + "loss": 0.53730631, + "num_input_tokens_seen": 194393075, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.01013184, + "step": 6795, + "time_per_iteration": 3.062408447265625 + }, + { + "auxiliary_loss_clip": 0.01084183, + "auxiliary_loss_mlp": 0.01045541, + "balance_loss_clip": 1.0348177, + "balance_loss_mlp": 1.03055012, + "epoch": 0.1972027160350531, + "flos": 26205702042240.0, + "grad_norm": 2.9609925259870744, + "language_loss": 0.58258837, + "learning_rate": 3.7138546807700085e-06, + "loss": 0.60388565, + "num_input_tokens_seen": 194410270, + "router_z_loss_clip": 0.49462891, + "router_z_loss_mlp": 0.14990234, + "step": 6796, + "time_per_iteration": 2.511719226837158 + }, + { + "auxiliary_loss_clip": 0.0108468, + "auxiliary_loss_mlp": 0.01051605, + "balance_loss_clip": 1.03707922, + "balance_loss_mlp": 1.03692997, + "epoch": 0.19723173350356915, + "flos": 26391544542720.0, + "grad_norm": 2.301516978710131, + "language_loss": 0.86662608, + "learning_rate": 3.7137577907196336e-06, + "loss": 0.88798898, + "num_input_tokens_seen": 194427010, + "router_z_loss_clip": 0.47607422, + "router_z_loss_mlp": 0.14672852, + "step": 6797, + "time_per_iteration": 2.5909900665283203 + }, + { + "auxiliary_loss_clip": 0.01087366, + "auxiliary_loss_mlp": 0.01045942, + "balance_loss_clip": 1.0387969, + "balance_loss_mlp": 1.0321672, + "epoch": 0.1972607509720852, + "flos": 20330413182720.0, + "grad_norm": 3.5452994386531658, + "language_loss": 0.7272836, + "learning_rate": 3.713660885532656e-06, + "loss": 0.74861664, + "num_input_tokens_seen": 194440035, + "router_z_loss_clip": 0.48535156, + "router_z_loss_mlp": 0.13775635, + "step": 6798, + "time_per_iteration": 2.3685340881347656 + }, + { + "auxiliary_loss_clip": 0.01084289, + "auxiliary_loss_mlp": 0.01045724, + "balance_loss_clip": 1.03838611, + "balance_loss_mlp": 1.03215253, + "epoch": 0.19728976844060125, + "flos": 15661582865280.0, + "grad_norm": 2.9932066403865303, + "language_loss": 0.77295792, + "learning_rate": 3.713563965209932e-06, + "loss": 0.79425812, + "num_input_tokens_seen": 194451795, + "router_z_loss_clip": 0.45922852, + "router_z_loss_mlp": 0.13562012, + "step": 6799, + "time_per_iteration": 2.4229304790496826 + }, + { + "auxiliary_loss_clip": 0.01085582, + "auxiliary_loss_mlp": 0.01038003, + "balance_loss_clip": 1.03782117, + "balance_loss_mlp": 1.02359009, + "epoch": 0.19731878590911728, + "flos": 12088571391360.0, + "grad_norm": 2.747671837512515, + "language_loss": 0.69348532, + "learning_rate": 3.7134670297523176e-06, + "loss": 0.7147212, + "num_input_tokens_seen": 194464480, + "router_z_loss_clip": 0.4765625, + "router_z_loss_mlp": 0.14422607, + "step": 6800, + "time_per_iteration": 2.39243483543396 + }, + { + "auxiliary_loss_clip": 0.01095224, + "auxiliary_loss_mlp": 0.01047626, + "balance_loss_clip": 1.0383954, + "balance_loss_mlp": 1.02907658, + "epoch": 0.19734780337763333, + "flos": 32994039444480.0, + "grad_norm": 2.473527838137862, + "language_loss": 1.19205403, + "learning_rate": 3.7133700791606693e-06, + "loss": 1.2134825, + "num_input_tokens_seen": 194485490, + "router_z_loss_clip": 0.56884766, + "router_z_loss_mlp": 0.18554688, + "step": 6801, + "time_per_iteration": 2.5374770164489746 + }, + { + "auxiliary_loss_clip": 0.01094028, + "auxiliary_loss_mlp": 0.01051943, + "balance_loss_clip": 1.03828394, + "balance_loss_mlp": 1.03337002, + "epoch": 0.19737682084614938, + "flos": 26823525194880.0, + "grad_norm": 2.6421064246103665, + "language_loss": 0.82377279, + "learning_rate": 3.7132731134358428e-06, + "loss": 0.84523249, + "num_input_tokens_seen": 194502270, + "router_z_loss_clip": 0.55810547, + "router_z_loss_mlp": 0.18566895, + "step": 6802, + "time_per_iteration": 2.4431145191192627 + }, + { + "auxiliary_loss_clip": 0.01097063, + "auxiliary_loss_mlp": 0.01052744, + "balance_loss_clip": 1.04320669, + "balance_loss_mlp": 1.03587556, + "epoch": 0.19740583831466543, + "flos": 28830891392640.0, + "grad_norm": 2.3828811385392985, + "language_loss": 0.87186766, + "learning_rate": 3.7131761325786947e-06, + "loss": 0.89336574, + "num_input_tokens_seen": 194517170, + "router_z_loss_clip": 0.53857422, + "router_z_loss_mlp": 0.16876221, + "step": 6803, + "time_per_iteration": 2.497260093688965 + }, + { + "auxiliary_loss_clip": 0.01088467, + "auxiliary_loss_mlp": 0.0103616, + "balance_loss_clip": 1.04047179, + "balance_loss_mlp": 1.02102613, + "epoch": 0.19743485578318148, + "flos": 41316006539520.0, + "grad_norm": 1.8521169971183669, + "language_loss": 0.81140113, + "learning_rate": 3.7130791365900823e-06, + "loss": 0.83264738, + "num_input_tokens_seen": 194535835, + "router_z_loss_clip": 0.47998047, + "router_z_loss_mlp": 0.15124512, + "step": 6804, + "time_per_iteration": 2.5880415439605713 + }, + { + "auxiliary_loss_clip": 0.01028733, + "auxiliary_loss_mlp": 0.01004217, + "balance_loss_clip": 1.01948118, + "balance_loss_mlp": 1.00326943, + "epoch": 0.1974638732516975, + "flos": 74759796518400.0, + "grad_norm": 0.8297608114068029, + "language_loss": 0.52961206, + "learning_rate": 3.7129821254708615e-06, + "loss": 0.54994154, + "num_input_tokens_seen": 194587335, + "router_z_loss_clip": 0.09277344, + "router_z_loss_mlp": 0.00946045, + "step": 6805, + "time_per_iteration": 2.9093613624572754 + }, + { + "auxiliary_loss_clip": 0.01093347, + "auxiliary_loss_mlp": 0.01042812, + "balance_loss_clip": 1.04041338, + "balance_loss_mlp": 1.02538371, + "epoch": 0.19749289072021356, + "flos": 26899391312640.0, + "grad_norm": 3.066188070011824, + "language_loss": 0.90506929, + "learning_rate": 3.712885099221889e-06, + "loss": 0.92643088, + "num_input_tokens_seen": 194602280, + "router_z_loss_clip": 0.52929688, + "router_z_loss_mlp": 0.17431641, + "step": 6806, + "time_per_iteration": 2.446120262145996 + }, + { + "auxiliary_loss_clip": 0.01087059, + "auxiliary_loss_mlp": 0.01035519, + "balance_loss_clip": 1.03882861, + "balance_loss_mlp": 1.02178025, + "epoch": 0.1975219081887296, + "flos": 14750349713280.0, + "grad_norm": 2.7243129899227996, + "language_loss": 0.81603873, + "learning_rate": 3.7127880578440226e-06, + "loss": 0.83726454, + "num_input_tokens_seen": 194616150, + "router_z_loss_clip": 0.48193359, + "router_z_loss_mlp": 0.13745117, + "step": 6807, + "time_per_iteration": 2.4403207302093506 + }, + { + "auxiliary_loss_clip": 0.0108948, + "auxiliary_loss_mlp": 0.01035961, + "balance_loss_clip": 1.03779876, + "balance_loss_mlp": 1.02017212, + "epoch": 0.19755092565724566, + "flos": 27858350160000.0, + "grad_norm": 2.69880347731124, + "language_loss": 0.85990655, + "learning_rate": 3.7126910013381194e-06, + "loss": 0.88116103, + "num_input_tokens_seen": 194631985, + "router_z_loss_clip": 0.51660156, + "router_z_loss_mlp": 0.15802002, + "step": 6808, + "time_per_iteration": 2.507070779800415 + }, + { + "auxiliary_loss_clip": 0.01093343, + "auxiliary_loss_mlp": 0.01030092, + "balance_loss_clip": 1.04279327, + "balance_loss_mlp": 1.01467252, + "epoch": 0.19757994312576171, + "flos": 12159340450560.0, + "grad_norm": 2.5024838512273035, + "language_loss": 0.82191539, + "learning_rate": 3.7125939297050356e-06, + "loss": 0.84314978, + "num_input_tokens_seen": 194647370, + "router_z_loss_clip": 0.50634766, + "router_z_loss_mlp": 0.15423584, + "step": 6809, + "time_per_iteration": 2.4892215728759766 + }, + { + "auxiliary_loss_clip": 0.0109078, + "auxiliary_loss_mlp": 0.01038324, + "balance_loss_clip": 1.04090953, + "balance_loss_mlp": 1.02212405, + "epoch": 0.19760896059427777, + "flos": 10189331274240.0, + "grad_norm": 2.771674259437634, + "language_loss": 0.69269967, + "learning_rate": 3.7124968429456295e-06, + "loss": 0.71399075, + "num_input_tokens_seen": 194659285, + "router_z_loss_clip": 0.49902344, + "router_z_loss_mlp": 0.16210938, + "step": 6810, + "time_per_iteration": 2.6115150451660156 + }, + { + "auxiliary_loss_clip": 0.01026037, + "auxiliary_loss_mlp": 0.01009126, + "balance_loss_clip": 1.01601732, + "balance_loss_mlp": 1.00815487, + "epoch": 0.1976379780627938, + "flos": 73058722611840.0, + "grad_norm": 0.6859037341590621, + "language_loss": 0.50786221, + "learning_rate": 3.712399741060758e-06, + "loss": 0.52821386, + "num_input_tokens_seen": 194720360, + "router_z_loss_clip": 0.10009766, + "router_z_loss_mlp": 0.00970459, + "step": 6811, + "time_per_iteration": 3.1134603023529053 + }, + { + "auxiliary_loss_clip": 0.01091826, + "auxiliary_loss_mlp": 0.01036695, + "balance_loss_clip": 1.0395602, + "balance_loss_mlp": 1.01936769, + "epoch": 0.19766699553130984, + "flos": 42587159564160.0, + "grad_norm": 2.287848961636158, + "language_loss": 0.874259, + "learning_rate": 3.712302624051279e-06, + "loss": 0.89554417, + "num_input_tokens_seen": 194737575, + "router_z_loss_clip": 0.5222168, + "router_z_loss_mlp": 0.17321777, + "step": 6812, + "time_per_iteration": 2.6264045238494873 + }, + { + "auxiliary_loss_clip": 0.01023715, + "auxiliary_loss_mlp": 0.01006579, + "balance_loss_clip": 1.01410985, + "balance_loss_mlp": 1.0055064, + "epoch": 0.1976960129998259, + "flos": 61931553863040.0, + "grad_norm": 0.6628356185890729, + "language_loss": 0.496777, + "learning_rate": 3.7122054919180506e-06, + "loss": 0.51707995, + "num_input_tokens_seen": 194800150, + "router_z_loss_clip": 0.09570312, + "router_z_loss_mlp": 0.01074219, + "step": 6813, + "time_per_iteration": 3.098623514175415 + }, + { + "auxiliary_loss_clip": 0.0109385, + "auxiliary_loss_mlp": 0.01034616, + "balance_loss_clip": 1.03928626, + "balance_loss_mlp": 1.01783752, + "epoch": 0.19772503046834194, + "flos": 41129151609600.0, + "grad_norm": 2.3296092784288094, + "language_loss": 0.92869532, + "learning_rate": 3.7121083446619306e-06, + "loss": 0.94998002, + "num_input_tokens_seen": 194817850, + "router_z_loss_clip": 0.54614258, + "router_z_loss_mlp": 0.16760254, + "step": 6814, + "time_per_iteration": 2.6499428749084473 + }, + { + "auxiliary_loss_clip": 0.0108987, + "auxiliary_loss_mlp": 0.01044726, + "balance_loss_clip": 1.03473163, + "balance_loss_mlp": 1.02815592, + "epoch": 0.197754047936858, + "flos": 33684865983360.0, + "grad_norm": 4.522736149635593, + "language_loss": 0.79138553, + "learning_rate": 3.7120111822837767e-06, + "loss": 0.8127315, + "num_input_tokens_seen": 194832875, + "router_z_loss_clip": 0.55175781, + "router_z_loss_mlp": 0.16577148, + "step": 6815, + "time_per_iteration": 2.5018701553344727 + }, + { + "auxiliary_loss_clip": 0.01020929, + "auxiliary_loss_mlp": 0.01001717, + "balance_loss_clip": 1.01142061, + "balance_loss_mlp": 1.00072205, + "epoch": 0.19778306540537405, + "flos": 51927750887040.0, + "grad_norm": 0.5972524153167859, + "language_loss": 0.42852068, + "learning_rate": 3.711914004784447e-06, + "loss": 0.44874716, + "num_input_tokens_seen": 194894840, + "router_z_loss_clip": 0.09472656, + "router_z_loss_mlp": 0.00994873, + "step": 6816, + "time_per_iteration": 3.040454626083374 + }, + { + "auxiliary_loss_clip": 0.01019428, + "auxiliary_loss_mlp": 0.01001287, + "balance_loss_clip": 1.01051402, + "balance_loss_mlp": 1.00025547, + "epoch": 0.19781208287389007, + "flos": 67397416675200.0, + "grad_norm": 0.6690589829304191, + "language_loss": 0.49104178, + "learning_rate": 3.7118168121648e-06, + "loss": 0.51124889, + "num_input_tokens_seen": 194956915, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.01031494, + "step": 6817, + "time_per_iteration": 3.042365789413452 + }, + { + "auxiliary_loss_clip": 0.01084014, + "auxiliary_loss_mlp": 0.01031455, + "balance_loss_clip": 1.03434443, + "balance_loss_mlp": 1.01661944, + "epoch": 0.19784110034240612, + "flos": 41056252957440.0, + "grad_norm": 1.9900414987274608, + "language_loss": 0.79040873, + "learning_rate": 3.7117196044256946e-06, + "loss": 0.81156343, + "num_input_tokens_seen": 194976380, + "router_z_loss_clip": 0.49682617, + "router_z_loss_mlp": 0.14837646, + "step": 6818, + "time_per_iteration": 2.6238455772399902 + }, + { + "auxiliary_loss_clip": 0.01087113, + "auxiliary_loss_mlp": 0.01040079, + "balance_loss_clip": 1.03459656, + "balance_loss_mlp": 1.02368748, + "epoch": 0.19787011781092217, + "flos": 22594565496960.0, + "grad_norm": 2.2710543917348764, + "language_loss": 0.828982, + "learning_rate": 3.7116223815679893e-06, + "loss": 0.85025394, + "num_input_tokens_seen": 194989220, + "router_z_loss_clip": 0.52514648, + "router_z_loss_mlp": 0.16394043, + "step": 6819, + "time_per_iteration": 2.438627243041992 + }, + { + "auxiliary_loss_clip": 0.01086224, + "auxiliary_loss_mlp": 0.01036581, + "balance_loss_clip": 1.03380263, + "balance_loss_mlp": 1.02049983, + "epoch": 0.19789913527943823, + "flos": 34560871706880.0, + "grad_norm": 2.052077300744173, + "language_loss": 0.95800638, + "learning_rate": 3.711525143592542e-06, + "loss": 0.9792344, + "num_input_tokens_seen": 195003950, + "router_z_loss_clip": 0.52539062, + "router_z_loss_mlp": 0.1607666, + "step": 6820, + "time_per_iteration": 2.5352396965026855 + }, + { + "auxiliary_loss_clip": 0.01079933, + "auxiliary_loss_mlp": 0.01029167, + "balance_loss_clip": 1.03087091, + "balance_loss_mlp": 1.01456356, + "epoch": 0.19792815274795428, + "flos": 28575046016640.0, + "grad_norm": 1.6566662175855558, + "language_loss": 0.82775426, + "learning_rate": 3.7114278905002122e-06, + "loss": 0.84884524, + "num_input_tokens_seen": 195030060, + "router_z_loss_clip": 0.4909668, + "router_z_loss_mlp": 0.14611816, + "step": 6821, + "time_per_iteration": 2.6804447174072266 + }, + { + "auxiliary_loss_clip": 0.01076723, + "auxiliary_loss_mlp": 0.01038631, + "balance_loss_clip": 1.03037834, + "balance_loss_mlp": 1.02386093, + "epoch": 0.1979571702164703, + "flos": 16719555928320.0, + "grad_norm": 4.761144797364591, + "language_loss": 0.55843937, + "learning_rate": 3.7113306222918586e-06, + "loss": 0.57959288, + "num_input_tokens_seen": 195042560, + "router_z_loss_clip": 0.46289062, + "router_z_loss_mlp": 0.14758301, + "step": 6822, + "time_per_iteration": 2.411241292953491 + }, + { + "auxiliary_loss_clip": 0.01087438, + "auxiliary_loss_mlp": 0.01037566, + "balance_loss_clip": 1.03328121, + "balance_loss_mlp": 1.0201261, + "epoch": 0.19798618768498635, + "flos": 16537658411520.0, + "grad_norm": 2.7570643242336894, + "language_loss": 0.69043201, + "learning_rate": 3.7112333389683405e-06, + "loss": 0.71168208, + "num_input_tokens_seen": 195055205, + "router_z_loss_clip": 0.54150391, + "router_z_loss_mlp": 0.17443848, + "step": 6823, + "time_per_iteration": 2.665344476699829 + }, + { + "auxiliary_loss_clip": 0.01083696, + "auxiliary_loss_mlp": 0.01033593, + "balance_loss_clip": 1.0324322, + "balance_loss_mlp": 1.01750624, + "epoch": 0.1980152051535024, + "flos": 17675163285120.0, + "grad_norm": 1.9127452123604511, + "language_loss": 0.83744502, + "learning_rate": 3.7111360405305173e-06, + "loss": 0.8586179, + "num_input_tokens_seen": 195071145, + "router_z_loss_clip": 0.51342773, + "router_z_loss_mlp": 0.16088867, + "step": 6824, + "time_per_iteration": 2.4300005435943604 + }, + { + "auxiliary_loss_clip": 0.01081684, + "auxiliary_loss_mlp": 0.01042866, + "balance_loss_clip": 1.03195846, + "balance_loss_mlp": 1.02688038, + "epoch": 0.19804422262201846, + "flos": 19893195826560.0, + "grad_norm": 2.395487577738934, + "language_loss": 0.72130692, + "learning_rate": 3.7110387269792474e-06, + "loss": 0.7425524, + "num_input_tokens_seen": 195085480, + "router_z_loss_clip": 0.49707031, + "router_z_loss_mlp": 0.15985107, + "step": 6825, + "time_per_iteration": 2.382371425628662 + }, + { + "auxiliary_loss_clip": 0.01021544, + "auxiliary_loss_mlp": 0.01012219, + "balance_loss_clip": 1.01369035, + "balance_loss_mlp": 1.01146543, + "epoch": 0.1980732400905345, + "flos": 69875098064640.0, + "grad_norm": 0.7270109262889322, + "language_loss": 0.46607447, + "learning_rate": 3.7109413983153922e-06, + "loss": 0.48641211, + "num_input_tokens_seen": 195137330, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.00753784, + "step": 6826, + "time_per_iteration": 2.921786308288574 + }, + { + "auxiliary_loss_clip": 0.01083556, + "auxiliary_loss_mlp": 0.01035656, + "balance_loss_clip": 1.03258944, + "balance_loss_mlp": 1.02015293, + "epoch": 0.19810225755905056, + "flos": 17997202465920.0, + "grad_norm": 3.0369172048027075, + "language_loss": 1.03027797, + "learning_rate": 3.7108440545398095e-06, + "loss": 1.05147004, + "num_input_tokens_seen": 195153415, + "router_z_loss_clip": 0.50976562, + "router_z_loss_mlp": 0.15509033, + "step": 6827, + "time_per_iteration": 2.4653451442718506 + }, + { + "auxiliary_loss_clip": 0.010217, + "auxiliary_loss_mlp": 0.01011127, + "balance_loss_clip": 1.01376212, + "balance_loss_mlp": 1.01035833, + "epoch": 0.19813127502756658, + "flos": 59742429793920.0, + "grad_norm": 0.7584622578423573, + "language_loss": 0.54143405, + "learning_rate": 3.71074669565336e-06, + "loss": 0.56176233, + "num_input_tokens_seen": 195211290, + "router_z_loss_clip": 0.07910156, + "router_z_loss_mlp": 0.00765991, + "step": 6828, + "time_per_iteration": 2.9216156005859375 + }, + { + "auxiliary_loss_clip": 0.01078914, + "auxiliary_loss_mlp": 0.01031733, + "balance_loss_clip": 1.03211486, + "balance_loss_mlp": 1.01679039, + "epoch": 0.19816029249608264, + "flos": 12085987950720.0, + "grad_norm": 5.324541619973608, + "language_loss": 0.68273509, + "learning_rate": 3.710649321656904e-06, + "loss": 0.70384157, + "num_input_tokens_seen": 195222885, + "router_z_loss_clip": 0.46801758, + "router_z_loss_mlp": 0.1494751, + "step": 6829, + "time_per_iteration": 2.4045681953430176 + }, + { + "auxiliary_loss_clip": 0.01087419, + "auxiliary_loss_mlp": 0.01043954, + "balance_loss_clip": 1.0330621, + "balance_loss_mlp": 1.02653146, + "epoch": 0.1981893099645987, + "flos": 28622284819200.0, + "grad_norm": 1.8820992533434275, + "language_loss": 0.64429688, + "learning_rate": 3.7105519325512997e-06, + "loss": 0.66561061, + "num_input_tokens_seen": 195242480, + "router_z_loss_clip": 0.54370117, + "router_z_loss_mlp": 0.17425537, + "step": 6830, + "time_per_iteration": 2.5273540019989014 + }, + { + "auxiliary_loss_clip": 0.01088244, + "auxiliary_loss_mlp": 0.0104603, + "balance_loss_clip": 1.03113902, + "balance_loss_mlp": 1.02683711, + "epoch": 0.19821832743311474, + "flos": 14785158205440.0, + "grad_norm": 2.20887446851494, + "language_loss": 0.85832787, + "learning_rate": 3.7104545283374097e-06, + "loss": 0.87967056, + "num_input_tokens_seen": 195255320, + "router_z_loss_clip": 0.57104492, + "router_z_loss_mlp": 0.19171143, + "step": 6831, + "time_per_iteration": 2.4673688411712646 + }, + { + "auxiliary_loss_clip": 0.01088333, + "auxiliary_loss_mlp": 0.0104655, + "balance_loss_clip": 1.0325315, + "balance_loss_mlp": 1.02744126, + "epoch": 0.1982473449016308, + "flos": 16207519795200.0, + "grad_norm": 2.63705418444838, + "language_loss": 0.69814622, + "learning_rate": 3.7103571090160926e-06, + "loss": 0.71949506, + "num_input_tokens_seen": 195269020, + "router_z_loss_clip": 0.55834961, + "router_z_loss_mlp": 0.19104004, + "step": 6832, + "time_per_iteration": 2.351926803588867 + }, + { + "auxiliary_loss_clip": 0.01019403, + "auxiliary_loss_mlp": 0.01000532, + "balance_loss_clip": 1.01136279, + "balance_loss_mlp": 0.99959654, + "epoch": 0.19827636237014684, + "flos": 59116367560320.0, + "grad_norm": 0.6975887559706118, + "language_loss": 0.4771753, + "learning_rate": 3.710259674588209e-06, + "loss": 0.49737465, + "num_input_tokens_seen": 195326225, + "router_z_loss_clip": 0.08007812, + "router_z_loss_mlp": 0.00933838, + "step": 6833, + "time_per_iteration": 2.980874538421631 + }, + { + "auxiliary_loss_clip": 0.01018673, + "auxiliary_loss_mlp": 0.01003651, + "balance_loss_clip": 1.0104022, + "balance_loss_mlp": 1.00287628, + "epoch": 0.19830537983866287, + "flos": 74763322565760.0, + "grad_norm": 0.6820415599504576, + "language_loss": 0.43220156, + "learning_rate": 3.7101622250546207e-06, + "loss": 0.45242479, + "num_input_tokens_seen": 195384550, + "router_z_loss_clip": 0.08300781, + "router_z_loss_mlp": 0.00772095, + "step": 6834, + "time_per_iteration": 3.0321109294891357 + }, + { + "auxiliary_loss_clip": 0.01087412, + "auxiliary_loss_mlp": 0.01048796, + "balance_loss_clip": 1.03581738, + "balance_loss_mlp": 1.0344727, + "epoch": 0.19833439730717892, + "flos": 27406923880320.0, + "grad_norm": 3.8191467650476794, + "language_loss": 0.79703641, + "learning_rate": 3.710064760416187e-06, + "loss": 0.81839848, + "num_input_tokens_seen": 195397325, + "router_z_loss_clip": 0.5168457, + "router_z_loss_mlp": 0.14318848, + "step": 6835, + "time_per_iteration": 2.5143892765045166 + }, + { + "auxiliary_loss_clip": 0.01088864, + "auxiliary_loss_mlp": 0.01045049, + "balance_loss_clip": 1.03455234, + "balance_loss_mlp": 1.0276444, + "epoch": 0.19836341477569497, + "flos": 48021286083840.0, + "grad_norm": 2.526025085141751, + "language_loss": 1.04709554, + "learning_rate": 3.70996728067377e-06, + "loss": 1.06843472, + "num_input_tokens_seen": 195415185, + "router_z_loss_clip": 0.54394531, + "router_z_loss_mlp": 0.17419434, + "step": 6836, + "time_per_iteration": 2.8511178493499756 + }, + { + "auxiliary_loss_clip": 0.01016022, + "auxiliary_loss_mlp": 0.01016369, + "balance_loss_clip": 1.00798821, + "balance_loss_mlp": 1.01542115, + "epoch": 0.19839243224421102, + "flos": 74772783544320.0, + "grad_norm": 0.6660393193409218, + "language_loss": 0.50502002, + "learning_rate": 3.7098697858282295e-06, + "loss": 0.52534389, + "num_input_tokens_seen": 195480245, + "router_z_loss_clip": 0.08007812, + "router_z_loss_mlp": 0.00946045, + "step": 6837, + "time_per_iteration": 3.087090492248535 + }, + { + "auxiliary_loss_clip": 0.01015815, + "auxiliary_loss_mlp": 0.01009563, + "balance_loss_clip": 1.00789118, + "balance_loss_mlp": 1.00866926, + "epoch": 0.19842144971272707, + "flos": 60655408513920.0, + "grad_norm": 0.6303434142177282, + "language_loss": 0.46595106, + "learning_rate": 3.709772275880427e-06, + "loss": 0.48620483, + "num_input_tokens_seen": 195546840, + "router_z_loss_clip": 0.07910156, + "router_z_loss_mlp": 0.00891113, + "step": 6838, + "time_per_iteration": 3.1276838779449463 + }, + { + "auxiliary_loss_clip": 0.01082003, + "auxiliary_loss_mlp": 0.01045097, + "balance_loss_clip": 1.02796531, + "balance_loss_mlp": 1.02803791, + "epoch": 0.1984504671812431, + "flos": 24470866984320.0, + "grad_norm": 2.2074707475716755, + "language_loss": 0.92836666, + "learning_rate": 3.7096747508312243e-06, + "loss": 0.94963771, + "num_input_tokens_seen": 195566325, + "router_z_loss_clip": 0.54150391, + "router_z_loss_mlp": 0.1706543, + "step": 6839, + "time_per_iteration": 2.4844131469726562 + }, + { + "auxiliary_loss_clip": 0.01082319, + "auxiliary_loss_mlp": 0.0104322, + "balance_loss_clip": 1.0313518, + "balance_loss_mlp": 1.02790725, + "epoch": 0.19847948464975915, + "flos": 16501558199040.0, + "grad_norm": 2.0209100394731245, + "language_loss": 0.60938078, + "learning_rate": 3.709577210681482e-06, + "loss": 0.63063616, + "num_input_tokens_seen": 195580195, + "router_z_loss_clip": 0.51025391, + "router_z_loss_mlp": 0.15307617, + "step": 6840, + "time_per_iteration": 2.4678361415863037 + }, + { + "auxiliary_loss_clip": 0.01015151, + "auxiliary_loss_mlp": 0.01012525, + "balance_loss_clip": 1.00761342, + "balance_loss_mlp": 1.01146364, + "epoch": 0.1985085021182752, + "flos": 74774494200960.0, + "grad_norm": 0.6784218483739743, + "language_loss": 0.48226738, + "learning_rate": 3.7094796554320624e-06, + "loss": 0.50254411, + "num_input_tokens_seen": 195642055, + "router_z_loss_clip": 0.07519531, + "router_z_loss_mlp": 0.01062012, + "step": 6841, + "time_per_iteration": 3.0608625411987305 + }, + { + "auxiliary_loss_clip": 0.01015231, + "auxiliary_loss_mlp": 0.01014527, + "balance_loss_clip": 1.0077064, + "balance_loss_mlp": 1.01342988, + "epoch": 0.19853751958679125, + "flos": 67875831302400.0, + "grad_norm": 0.6912515329413257, + "language_loss": 0.47031158, + "learning_rate": 3.7093820850838268e-06, + "loss": 0.4906092, + "num_input_tokens_seen": 195700545, + "router_z_loss_clip": 0.07519531, + "router_z_loss_mlp": 0.01098633, + "step": 6842, + "time_per_iteration": 3.0243303775787354 + }, + { + "auxiliary_loss_clip": 0.01088274, + "auxiliary_loss_mlp": 0.01034994, + "balance_loss_clip": 1.03129661, + "balance_loss_mlp": 1.01615894, + "epoch": 0.1985665370553073, + "flos": 26350731296640.0, + "grad_norm": 2.059204336183962, + "language_loss": 0.84927905, + "learning_rate": 3.7092844996376362e-06, + "loss": 0.87051177, + "num_input_tokens_seen": 195715330, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.18835449, + "step": 6843, + "time_per_iteration": 2.427828311920166 + }, + { + "auxiliary_loss_clip": 0.01079823, + "auxiliary_loss_mlp": 0.01029984, + "balance_loss_clip": 1.02944779, + "balance_loss_mlp": 1.01340842, + "epoch": 0.19859555452382335, + "flos": 46418528165760.0, + "grad_norm": 1.6758653100136205, + "language_loss": 0.75518119, + "learning_rate": 3.7091868990943544e-06, + "loss": 0.77627933, + "num_input_tokens_seen": 195741565, + "router_z_loss_clip": 0.50415039, + "router_z_loss_mlp": 0.16564941, + "step": 6844, + "time_per_iteration": 2.6516566276550293 + }, + { + "auxiliary_loss_clip": 0.01019383, + "auxiliary_loss_mlp": 0.01009788, + "balance_loss_clip": 1.0110544, + "balance_loss_mlp": 1.0088042, + "epoch": 0.19862457199233938, + "flos": 74778369361920.0, + "grad_norm": 0.5933192003309035, + "language_loss": 0.40977526, + "learning_rate": 3.7090892834548414e-06, + "loss": 0.43006697, + "num_input_tokens_seen": 195806420, + "router_z_loss_clip": 0.08300781, + "router_z_loss_mlp": 0.00982666, + "step": 6845, + "time_per_iteration": 3.0949082374572754 + }, + { + "auxiliary_loss_clip": 0.01084982, + "auxiliary_loss_mlp": 0.01038157, + "balance_loss_clip": 1.03204274, + "balance_loss_mlp": 1.02248645, + "epoch": 0.19865358946085543, + "flos": 24199313362560.0, + "grad_norm": 1.7542905555769044, + "language_loss": 0.69091386, + "learning_rate": 3.70899165271996e-06, + "loss": 0.71214527, + "num_input_tokens_seen": 195820200, + "router_z_loss_clip": 0.53051758, + "router_z_loss_mlp": 0.15686035, + "step": 6846, + "time_per_iteration": 2.461686134338379 + }, + { + "auxiliary_loss_clip": 0.01084718, + "auxiliary_loss_mlp": 0.01046331, + "balance_loss_clip": 1.0323875, + "balance_loss_mlp": 1.02991557, + "epoch": 0.19868260692937148, + "flos": 21644404312320.0, + "grad_norm": 2.0964980504088064, + "language_loss": 0.69473153, + "learning_rate": 3.7088940068905732e-06, + "loss": 0.71604204, + "num_input_tokens_seen": 195834175, + "router_z_loss_clip": 0.5234375, + "router_z_loss_mlp": 0.16418457, + "step": 6847, + "time_per_iteration": 2.382467031478882 + }, + { + "auxiliary_loss_clip": 0.01084817, + "auxiliary_loss_mlp": 0.01042438, + "balance_loss_clip": 1.03153062, + "balance_loss_mlp": 1.02547479, + "epoch": 0.19871162439788753, + "flos": 31534739769600.0, + "grad_norm": 2.093540971008143, + "language_loss": 0.83925867, + "learning_rate": 3.708796345967543e-06, + "loss": 0.86053121, + "num_input_tokens_seen": 195850265, + "router_z_loss_clip": 0.53295898, + "router_z_loss_mlp": 0.1696167, + "step": 6848, + "time_per_iteration": 4.94605016708374 + }, + { + "auxiliary_loss_clip": 0.01089589, + "auxiliary_loss_mlp": 0.01041212, + "balance_loss_clip": 1.03617072, + "balance_loss_mlp": 1.02530384, + "epoch": 0.19874064186640358, + "flos": 38974521830400.0, + "grad_norm": 2.561334847824093, + "language_loss": 0.79859024, + "learning_rate": 3.708698669951732e-06, + "loss": 0.81989825, + "num_input_tokens_seen": 195872100, + "router_z_loss_clip": 0.53393555, + "router_z_loss_mlp": 0.15893555, + "step": 6849, + "time_per_iteration": 2.5435707569122314 + }, + { + "auxiliary_loss_clip": 0.01083993, + "auxiliary_loss_mlp": 0.01039648, + "balance_loss_clip": 1.03371406, + "balance_loss_mlp": 1.02476478, + "epoch": 0.19876965933491964, + "flos": 15078009623040.0, + "grad_norm": 2.3633475299276383, + "language_loss": 0.88298708, + "learning_rate": 3.7086009788440026e-06, + "loss": 0.90422344, + "num_input_tokens_seen": 195886955, + "router_z_loss_clip": 0.50317383, + "router_z_loss_mlp": 0.14880371, + "step": 6850, + "time_per_iteration": 4.694230556488037 + }, + { + "auxiliary_loss_clip": 0.01080775, + "auxiliary_loss_mlp": 0.01031547, + "balance_loss_clip": 1.03222132, + "balance_loss_mlp": 1.01815343, + "epoch": 0.19879867680343566, + "flos": 14640582798720.0, + "grad_norm": 2.707714976776042, + "language_loss": 0.69203818, + "learning_rate": 3.7085032726452186e-06, + "loss": 0.71316141, + "num_input_tokens_seen": 195900725, + "router_z_loss_clip": 0.48535156, + "router_z_loss_mlp": 0.13391113, + "step": 6851, + "time_per_iteration": 2.357346296310425 + }, + { + "auxiliary_loss_clip": 0.01025936, + "auxiliary_loss_mlp": 0.0100113, + "balance_loss_clip": 1.01640344, + "balance_loss_mlp": 1.00033772, + "epoch": 0.1988276942719517, + "flos": 66570742569600.0, + "grad_norm": 0.6236728033495602, + "language_loss": 0.49329209, + "learning_rate": 3.7084055513562424e-06, + "loss": 0.51356274, + "num_input_tokens_seen": 195967160, + "router_z_loss_clip": 0.09521484, + "router_z_loss_mlp": 0.00793457, + "step": 6852, + "time_per_iteration": 3.0928962230682373 + }, + { + "auxiliary_loss_clip": 0.01081886, + "auxiliary_loss_mlp": 0.01044364, + "balance_loss_clip": 1.03087974, + "balance_loss_mlp": 1.02841365, + "epoch": 0.19885671174046776, + "flos": 19747363610880.0, + "grad_norm": 2.7485960915528027, + "language_loss": 0.92834079, + "learning_rate": 3.7083078149779363e-06, + "loss": 0.94960332, + "num_input_tokens_seen": 195979925, + "router_z_loss_clip": 0.51000977, + "router_z_loss_mlp": 0.1595459, + "step": 6853, + "time_per_iteration": 2.3512415885925293 + }, + { + "auxiliary_loss_clip": 0.0108241, + "auxiliary_loss_mlp": 0.01039856, + "balance_loss_clip": 1.03076887, + "balance_loss_mlp": 1.02376318, + "epoch": 0.1988857292089838, + "flos": 35145806492160.0, + "grad_norm": 2.2607633101333313, + "language_loss": 0.78785622, + "learning_rate": 3.7082100635111646e-06, + "loss": 0.80907887, + "num_input_tokens_seen": 195999745, + "router_z_loss_clip": 0.51635742, + "router_z_loss_mlp": 0.16094971, + "step": 6854, + "time_per_iteration": 2.525857925415039 + }, + { + "auxiliary_loss_clip": 0.01091202, + "auxiliary_loss_mlp": 0.01047929, + "balance_loss_clip": 1.03410935, + "balance_loss_mlp": 1.02926052, + "epoch": 0.19891474667749987, + "flos": 16792629137280.0, + "grad_norm": 2.429958814770607, + "language_loss": 0.80395281, + "learning_rate": 3.7081122969567905e-06, + "loss": 0.82534409, + "num_input_tokens_seen": 196012065, + "router_z_loss_clip": 0.5703125, + "router_z_loss_mlp": 0.18670654, + "step": 6855, + "time_per_iteration": 2.328615665435791 + }, + { + "auxiliary_loss_clip": 0.01090844, + "auxiliary_loss_mlp": 0.01044578, + "balance_loss_clip": 1.03133118, + "balance_loss_mlp": 1.02499747, + "epoch": 0.1989437641460159, + "flos": 30801563884800.0, + "grad_norm": 3.240270564276166, + "language_loss": 0.83457625, + "learning_rate": 3.7080145153156775e-06, + "loss": 0.85593051, + "num_input_tokens_seen": 196027310, + "router_z_loss_clip": 0.59521484, + "router_z_loss_mlp": 0.19580078, + "step": 6856, + "time_per_iteration": 2.4380836486816406 + }, + { + "auxiliary_loss_clip": 0.01078343, + "auxiliary_loss_mlp": 0.01034115, + "balance_loss_clip": 1.03122211, + "balance_loss_mlp": 1.01896966, + "epoch": 0.19897278161453194, + "flos": 12524811229440.0, + "grad_norm": 2.2441624151085304, + "language_loss": 0.73783642, + "learning_rate": 3.7079167185886887e-06, + "loss": 0.75896102, + "num_input_tokens_seen": 196040600, + "router_z_loss_clip": 0.47119141, + "router_z_loss_mlp": 0.15148926, + "step": 6857, + "time_per_iteration": 2.3590052127838135 + }, + { + "auxiliary_loss_clip": 0.01022216, + "auxiliary_loss_mlp": 0.0100276, + "balance_loss_clip": 1.01353765, + "balance_loss_mlp": 1.00196743, + "epoch": 0.199001799083048, + "flos": 62078084305920.0, + "grad_norm": 0.6665336777704342, + "language_loss": 0.47903645, + "learning_rate": 3.7078189067766886e-06, + "loss": 0.49928617, + "num_input_tokens_seen": 196102675, + "router_z_loss_clip": 0.08691406, + "router_z_loss_mlp": 0.00793457, + "step": 6858, + "time_per_iteration": 5.418502330780029 + }, + { + "auxiliary_loss_clip": 0.01078157, + "auxiliary_loss_mlp": 0.0103027, + "balance_loss_clip": 1.0287236, + "balance_loss_mlp": 1.01555943, + "epoch": 0.19903081655156404, + "flos": 22526903548800.0, + "grad_norm": 1.801315275648845, + "language_loss": 0.71572208, + "learning_rate": 3.7077210798805403e-06, + "loss": 0.73680633, + "num_input_tokens_seen": 196119545, + "router_z_loss_clip": 0.49462891, + "router_z_loss_mlp": 0.14709473, + "step": 6859, + "time_per_iteration": 5.067582607269287 + }, + { + "auxiliary_loss_clip": 0.01017747, + "auxiliary_loss_mlp": 0.01003961, + "balance_loss_clip": 1.00956428, + "balance_loss_mlp": 1.00317979, + "epoch": 0.1990598340200801, + "flos": 66374671040640.0, + "grad_norm": 0.6425863171889628, + "language_loss": 0.46608198, + "learning_rate": 3.707623237901109e-06, + "loss": 0.4862991, + "num_input_tokens_seen": 196180960, + "router_z_loss_clip": 0.08203125, + "router_z_loss_mlp": 0.0078125, + "step": 6860, + "time_per_iteration": 3.027850866317749 + }, + { + "auxiliary_loss_clip": 0.01085287, + "auxiliary_loss_mlp": 0.01031949, + "balance_loss_clip": 1.03088641, + "balance_loss_mlp": 1.01525331, + "epoch": 0.19908885148859615, + "flos": 16210068324480.0, + "grad_norm": 1.739446960781352, + "language_loss": 0.68789291, + "learning_rate": 3.7075253808392583e-06, + "loss": 0.70906532, + "num_input_tokens_seen": 196196625, + "router_z_loss_clip": 0.54492188, + "router_z_loss_mlp": 0.16687012, + "step": 6861, + "time_per_iteration": 2.3769690990448 + }, + { + "auxiliary_loss_clip": 0.01016753, + "auxiliary_loss_mlp": 0.01001303, + "balance_loss_clip": 1.00877118, + "balance_loss_mlp": 1.00046301, + "epoch": 0.19911786895711217, + "flos": 74780708423040.0, + "grad_norm": 0.6233339621008402, + "language_loss": 0.44541705, + "learning_rate": 3.7074275086958525e-06, + "loss": 0.46559763, + "num_input_tokens_seen": 196268375, + "router_z_loss_clip": 0.07958984, + "router_z_loss_mlp": 0.00842285, + "step": 6862, + "time_per_iteration": 3.24371337890625 + }, + { + "auxiliary_loss_clip": 0.01089238, + "auxiliary_loss_mlp": 0.01042955, + "balance_loss_clip": 1.03318882, + "balance_loss_mlp": 1.02361298, + "epoch": 0.19914688642562822, + "flos": 25112292082560.0, + "grad_norm": 2.4570429730223915, + "language_loss": 0.79270649, + "learning_rate": 3.7073296214717557e-06, + "loss": 0.81402838, + "num_input_tokens_seen": 196282655, + "router_z_loss_clip": 0.56005859, + "router_z_loss_mlp": 0.1932373, + "step": 6863, + "time_per_iteration": 2.4880635738372803 + }, + { + "auxiliary_loss_clip": 0.01082028, + "auxiliary_loss_mlp": 0.01030949, + "balance_loss_clip": 1.03074312, + "balance_loss_mlp": 1.01472449, + "epoch": 0.19917590389414427, + "flos": 33211967351040.0, + "grad_norm": 4.9260055716486235, + "language_loss": 0.79553223, + "learning_rate": 3.707231719167833e-06, + "loss": 0.81666195, + "num_input_tokens_seen": 196302055, + "router_z_loss_clip": 0.51269531, + "router_z_loss_mlp": 0.16217041, + "step": 6864, + "time_per_iteration": 2.4848527908325195 + }, + { + "auxiliary_loss_clip": 0.01091141, + "auxiliary_loss_mlp": 0.01035142, + "balance_loss_clip": 1.03489041, + "balance_loss_mlp": 1.01619911, + "epoch": 0.19920492136266033, + "flos": 21536243320320.0, + "grad_norm": 2.3155251618831754, + "language_loss": 0.84374404, + "learning_rate": 3.707133801784949e-06, + "loss": 0.86500692, + "num_input_tokens_seen": 196317145, + "router_z_loss_clip": 0.56152344, + "router_z_loss_mlp": 0.18951416, + "step": 6865, + "time_per_iteration": 2.4666292667388916 + }, + { + "auxiliary_loss_clip": 0.01017975, + "auxiliary_loss_mlp": 0.01004657, + "balance_loss_clip": 1.00970399, + "balance_loss_mlp": 1.00379837, + "epoch": 0.19923393883117638, + "flos": 74768559269760.0, + "grad_norm": 0.6231375127531572, + "language_loss": 0.49513143, + "learning_rate": 3.7070358693239683e-06, + "loss": 0.51535773, + "num_input_tokens_seen": 196381050, + "router_z_loss_clip": 0.08300781, + "router_z_loss_mlp": 0.00860596, + "step": 6866, + "time_per_iteration": 3.046351432800293 + }, + { + "auxiliary_loss_clip": 0.0108337, + "auxiliary_loss_mlp": 0.01037259, + "balance_loss_clip": 1.03084588, + "balance_loss_mlp": 1.02059317, + "epoch": 0.19926295629969243, + "flos": 11868723360000.0, + "grad_norm": 2.5472654234586467, + "language_loss": 0.88881773, + "learning_rate": 3.706937921785756e-06, + "loss": 0.91002405, + "num_input_tokens_seen": 196393040, + "router_z_loss_clip": 0.52514648, + "router_z_loss_mlp": 0.16674805, + "step": 6867, + "time_per_iteration": 2.4173007011413574 + }, + { + "auxiliary_loss_clip": 0.01090795, + "auxiliary_loss_mlp": 0.01034304, + "balance_loss_clip": 1.03730893, + "balance_loss_mlp": 1.01774645, + "epoch": 0.19929197376820845, + "flos": 11687035311360.0, + "grad_norm": 2.7489620891885935, + "language_loss": 0.69197989, + "learning_rate": 3.7068399591711773e-06, + "loss": 0.71323085, + "num_input_tokens_seen": 196407755, + "router_z_loss_clip": 0.53466797, + "router_z_loss_mlp": 0.16552734, + "step": 6868, + "time_per_iteration": 2.362987756729126 + }, + { + "auxiliary_loss_clip": 0.01021, + "auxiliary_loss_mlp": 0.01001597, + "balance_loss_clip": 1.01210999, + "balance_loss_mlp": 1.00078082, + "epoch": 0.1993209912367245, + "flos": 61603160814720.0, + "grad_norm": 0.6755665835816727, + "language_loss": 0.4596073, + "learning_rate": 3.706741981481097e-06, + "loss": 0.47983325, + "num_input_tokens_seen": 196468430, + "router_z_loss_clip": 0.08886719, + "router_z_loss_mlp": 0.00817871, + "step": 6869, + "time_per_iteration": 3.003126859664917 + }, + { + "auxiliary_loss_clip": 0.01084358, + "auxiliary_loss_mlp": 0.0102776, + "balance_loss_clip": 1.03363526, + "balance_loss_mlp": 1.01244199, + "epoch": 0.19935000870524056, + "flos": 12850935039360.0, + "grad_norm": 2.9814326583079196, + "language_loss": 0.77544051, + "learning_rate": 3.7066439887163816e-06, + "loss": 0.79656166, + "num_input_tokens_seen": 196480225, + "router_z_loss_clip": 0.50756836, + "router_z_loss_mlp": 0.15319824, + "step": 6870, + "time_per_iteration": 2.441291332244873 + }, + { + "auxiliary_loss_clip": 0.0102114, + "auxiliary_loss_mlp": 0.01003335, + "balance_loss_clip": 1.01231408, + "balance_loss_mlp": 1.00252426, + "epoch": 0.1993790261737566, + "flos": 74766743879040.0, + "grad_norm": 0.6200451246229215, + "language_loss": 0.45121494, + "learning_rate": 3.7065459808778954e-06, + "loss": 0.47145969, + "num_input_tokens_seen": 196542390, + "router_z_loss_clip": 0.08837891, + "router_z_loss_mlp": 0.00811768, + "step": 6871, + "time_per_iteration": 3.08479642868042 + }, + { + "auxiliary_loss_clip": 0.01086411, + "auxiliary_loss_mlp": 0.01031866, + "balance_loss_clip": 1.03144073, + "balance_loss_mlp": 1.01258397, + "epoch": 0.19940804364227266, + "flos": 16392384777600.0, + "grad_norm": 3.170306903341714, + "language_loss": 0.9292804, + "learning_rate": 3.706447957966505e-06, + "loss": 0.95046318, + "num_input_tokens_seen": 196555315, + "router_z_loss_clip": 0.55004883, + "router_z_loss_mlp": 0.19256592, + "step": 6872, + "time_per_iteration": 2.6394970417022705 + }, + { + "auxiliary_loss_clip": 0.01021144, + "auxiliary_loss_mlp": 0.01000629, + "balance_loss_clip": 1.01278257, + "balance_loss_mlp": 0.99978852, + "epoch": 0.19943706111078868, + "flos": 74775227339520.0, + "grad_norm": 0.6766451817915828, + "language_loss": 0.49341264, + "learning_rate": 3.7063499199830752e-06, + "loss": 0.51363039, + "num_input_tokens_seen": 196620950, + "router_z_loss_clip": 0.08349609, + "router_z_loss_mlp": 0.00842285, + "step": 6873, + "time_per_iteration": 3.1447558403015137 + }, + { + "auxiliary_loss_clip": 0.01081406, + "auxiliary_loss_mlp": 0.01034828, + "balance_loss_clip": 1.03109086, + "balance_loss_mlp": 1.01895559, + "epoch": 0.19946607857930473, + "flos": 16177179957120.0, + "grad_norm": 2.523324249721301, + "language_loss": 0.87882173, + "learning_rate": 3.7062518669284727e-06, + "loss": 0.89998412, + "num_input_tokens_seen": 196633935, + "router_z_loss_clip": 0.50366211, + "router_z_loss_mlp": 0.15881348, + "step": 6874, + "time_per_iteration": 2.4410452842712402 + }, + { + "auxiliary_loss_clip": 0.01086413, + "auxiliary_loss_mlp": 0.01041649, + "balance_loss_clip": 1.03066719, + "balance_loss_mlp": 1.0220871, + "epoch": 0.19949509604782079, + "flos": 19272754321920.0, + "grad_norm": 1.9035684656675111, + "language_loss": 0.69227517, + "learning_rate": 3.7061537988035633e-06, + "loss": 0.71355581, + "num_input_tokens_seen": 196647365, + "router_z_loss_clip": 0.55810547, + "router_z_loss_mlp": 0.19580078, + "step": 6875, + "time_per_iteration": 2.3792483806610107 + }, + { + "auxiliary_loss_clip": 0.01082773, + "auxiliary_loss_mlp": 0.01036245, + "balance_loss_clip": 1.03312707, + "balance_loss_mlp": 1.01966918, + "epoch": 0.19952411351633684, + "flos": 20950854687360.0, + "grad_norm": 2.385028241905197, + "language_loss": 0.68145931, + "learning_rate": 3.7060557156092127e-06, + "loss": 0.70264953, + "num_input_tokens_seen": 196661675, + "router_z_loss_clip": 0.49658203, + "router_z_loss_mlp": 0.16571045, + "step": 6876, + "time_per_iteration": 2.370370626449585 + }, + { + "auxiliary_loss_clip": 0.01086257, + "auxiliary_loss_mlp": 0.0104142, + "balance_loss_clip": 1.03216934, + "balance_loss_mlp": 1.02423608, + "epoch": 0.1995531309848529, + "flos": 32772794958720.0, + "grad_norm": 1.931220514408486, + "language_loss": 0.82610321, + "learning_rate": 3.7059576173462883e-06, + "loss": 0.84738004, + "num_input_tokens_seen": 196680805, + "router_z_loss_clip": 0.54077148, + "router_z_loss_mlp": 0.17175293, + "step": 6877, + "time_per_iteration": 2.5231921672821045 + }, + { + "auxiliary_loss_clip": 0.01077178, + "auxiliary_loss_mlp": 0.01032173, + "balance_loss_clip": 1.02726221, + "balance_loss_mlp": 1.01597261, + "epoch": 0.19958214845336894, + "flos": 27081672854400.0, + "grad_norm": 2.37305984627694, + "language_loss": 0.81977332, + "learning_rate": 3.705859504015655e-06, + "loss": 0.84086692, + "num_input_tokens_seen": 196694730, + "router_z_loss_clip": 0.49926758, + "router_z_loss_mlp": 0.16210938, + "step": 6878, + "time_per_iteration": 2.468736171722412 + }, + { + "auxiliary_loss_clip": 0.01081906, + "auxiliary_loss_mlp": 0.01037636, + "balance_loss_clip": 1.02963221, + "balance_loss_mlp": 1.02250814, + "epoch": 0.19961116592188496, + "flos": 24783165895680.0, + "grad_norm": 2.3523033540565645, + "language_loss": 0.81943071, + "learning_rate": 3.705761375618181e-06, + "loss": 0.84062612, + "num_input_tokens_seen": 196709120, + "router_z_loss_clip": 0.52319336, + "router_z_loss_mlp": 0.15136719, + "step": 6879, + "time_per_iteration": 2.379453659057617 + }, + { + "auxiliary_loss_clip": 0.01017976, + "auxiliary_loss_mlp": 0.01000901, + "balance_loss_clip": 1.01013255, + "balance_loss_mlp": 1.00012028, + "epoch": 0.19964018339040102, + "flos": 60532829130240.0, + "grad_norm": 0.7414184923015282, + "language_loss": 0.51309097, + "learning_rate": 3.705663232154732e-06, + "loss": 0.53327966, + "num_input_tokens_seen": 196762055, + "router_z_loss_clip": 0.07861328, + "router_z_loss_mlp": 0.0078125, + "step": 6880, + "time_per_iteration": 2.808915376663208 + }, + { + "auxiliary_loss_clip": 0.01017479, + "auxiliary_loss_mlp": 0.01000815, + "balance_loss_clip": 1.00966477, + "balance_loss_mlp": 1.00002182, + "epoch": 0.19966920085891707, + "flos": 74782104877440.0, + "grad_norm": 0.7016509077868096, + "language_loss": 0.50749707, + "learning_rate": 3.705565073626176e-06, + "loss": 0.52768004, + "num_input_tokens_seen": 196831425, + "router_z_loss_clip": 0.078125, + "router_z_loss_mlp": 0.00793457, + "step": 6881, + "time_per_iteration": 3.1574175357818604 + }, + { + "auxiliary_loss_clip": 0.01089017, + "auxiliary_loss_mlp": 0.01047235, + "balance_loss_clip": 1.03019762, + "balance_loss_mlp": 1.02768421, + "epoch": 0.19969821832743312, + "flos": 25402350591360.0, + "grad_norm": 2.2554752073745985, + "language_loss": 0.90079778, + "learning_rate": 3.705466900033378e-06, + "loss": 0.92216027, + "num_input_tokens_seen": 196848300, + "router_z_loss_clip": 0.58837891, + "router_z_loss_mlp": 0.19555664, + "step": 6882, + "time_per_iteration": 2.5575015544891357 + }, + { + "auxiliary_loss_clip": 0.01079813, + "auxiliary_loss_mlp": 0.01033825, + "balance_loss_clip": 1.02939868, + "balance_loss_mlp": 1.01729703, + "epoch": 0.19972723579594917, + "flos": 49993634321280.0, + "grad_norm": 1.8232687138368848, + "language_loss": 0.59416294, + "learning_rate": 3.705368711377207e-06, + "loss": 0.61529934, + "num_input_tokens_seen": 196868800, + "router_z_loss_clip": 0.50463867, + "router_z_loss_mlp": 0.16522217, + "step": 6883, + "time_per_iteration": 2.547452211380005 + }, + { + "auxiliary_loss_clip": 0.01018308, + "auxiliary_loss_mlp": 0.00998521, + "balance_loss_clip": 1.01068902, + "balance_loss_mlp": 0.99771631, + "epoch": 0.1997562532644652, + "flos": 74779102500480.0, + "grad_norm": 0.6646924775476888, + "language_loss": 0.47532678, + "learning_rate": 3.7052705076585285e-06, + "loss": 0.49549508, + "num_input_tokens_seen": 196938530, + "router_z_loss_clip": 0.07617188, + "router_z_loss_mlp": 0.00805664, + "step": 6884, + "time_per_iteration": 3.139465808868408 + }, + { + "auxiliary_loss_clip": 0.01082121, + "auxiliary_loss_mlp": 0.01039654, + "balance_loss_clip": 1.03077626, + "balance_loss_mlp": 1.0227505, + "epoch": 0.19978527073298125, + "flos": 23652887673600.0, + "grad_norm": 2.1355344031764467, + "language_loss": 0.80280769, + "learning_rate": 3.705172288878211e-06, + "loss": 0.82402551, + "num_input_tokens_seen": 196951335, + "router_z_loss_clip": 0.51367188, + "router_z_loss_mlp": 0.16906738, + "step": 6885, + "time_per_iteration": 2.656419515609741 + }, + { + "auxiliary_loss_clip": 0.01079331, + "auxiliary_loss_mlp": 0.01035977, + "balance_loss_clip": 1.02929688, + "balance_loss_mlp": 1.01969862, + "epoch": 0.1998142882014973, + "flos": 16027332935040.0, + "grad_norm": 18.191922920761733, + "language_loss": 0.71165884, + "learning_rate": 3.705074055037122e-06, + "loss": 0.73281193, + "num_input_tokens_seen": 196965700, + "router_z_loss_clip": 0.49975586, + "router_z_loss_mlp": 0.1628418, + "step": 6886, + "time_per_iteration": 2.4056224822998047 + }, + { + "auxiliary_loss_clip": 0.01080955, + "auxiliary_loss_mlp": 0.01040882, + "balance_loss_clip": 1.02999306, + "balance_loss_mlp": 1.02399623, + "epoch": 0.19984330567001335, + "flos": 16103513255040.0, + "grad_norm": 2.073123662207353, + "language_loss": 0.77481127, + "learning_rate": 3.7049758061361294e-06, + "loss": 0.79602963, + "num_input_tokens_seen": 196979550, + "router_z_loss_clip": 0.51049805, + "router_z_loss_mlp": 0.16870117, + "step": 6887, + "time_per_iteration": 2.3611371517181396 + }, + { + "auxiliary_loss_clip": 0.01089495, + "auxiliary_loss_mlp": 0.01043576, + "balance_loss_clip": 1.0358547, + "balance_loss_mlp": 1.02564669, + "epoch": 0.1998723231385294, + "flos": 37297049869440.0, + "grad_norm": 1.8890672618624689, + "language_loss": 1.0050838, + "learning_rate": 3.7048775421761006e-06, + "loss": 1.02641451, + "num_input_tokens_seen": 196999370, + "router_z_loss_clip": 0.53588867, + "router_z_loss_mlp": 0.17932129, + "step": 6888, + "time_per_iteration": 2.6247003078460693 + }, + { + "auxiliary_loss_clip": 0.01084131, + "auxiliary_loss_mlp": 0.0103828, + "balance_loss_clip": 1.03172469, + "balance_loss_mlp": 1.02203739, + "epoch": 0.19990134060704545, + "flos": 11613508254720.0, + "grad_norm": 2.3410950188812345, + "language_loss": 0.62157285, + "learning_rate": 3.7047792631579025e-06, + "loss": 0.64279699, + "num_input_tokens_seen": 197011485, + "router_z_loss_clip": 0.52319336, + "router_z_loss_mlp": 0.16247559, + "step": 6889, + "time_per_iteration": 2.3522119522094727 + }, + { + "auxiliary_loss_clip": 0.01083434, + "auxiliary_loss_mlp": 0.01040012, + "balance_loss_clip": 1.0308615, + "balance_loss_mlp": 1.02278638, + "epoch": 0.19993035807556148, + "flos": 23656797745920.0, + "grad_norm": 3.8591275722924974, + "language_loss": 0.93839097, + "learning_rate": 3.704680969082405e-06, + "loss": 0.95962554, + "num_input_tokens_seen": 197031645, + "router_z_loss_clip": 0.52612305, + "router_z_loss_mlp": 0.17224121, + "step": 6890, + "time_per_iteration": 2.4679245948791504 + }, + { + "auxiliary_loss_clip": 0.01086863, + "auxiliary_loss_mlp": 0.01035246, + "balance_loss_clip": 1.03428698, + "balance_loss_mlp": 1.0191412, + "epoch": 0.19995937554407753, + "flos": 20920410115200.0, + "grad_norm": 2.1068728803687673, + "language_loss": 0.72518122, + "learning_rate": 3.704582659950475e-06, + "loss": 0.74640226, + "num_input_tokens_seen": 197045930, + "router_z_loss_clip": 0.52587891, + "router_z_loss_mlp": 0.16101074, + "step": 6891, + "time_per_iteration": 2.3381032943725586 + }, + { + "auxiliary_loss_clip": 0.01080712, + "auxiliary_loss_mlp": 0.01039554, + "balance_loss_clip": 1.03137875, + "balance_loss_mlp": 1.02261996, + "epoch": 0.19998839301259358, + "flos": 20952041673600.0, + "grad_norm": 2.642796170589252, + "language_loss": 0.97655928, + "learning_rate": 3.7044843357629818e-06, + "loss": 0.99776196, + "num_input_tokens_seen": 197060565, + "router_z_loss_clip": 0.4934082, + "router_z_loss_mlp": 0.16931152, + "step": 6892, + "time_per_iteration": 2.4174280166625977 + }, + { + "auxiliary_loss_clip": 0.01087594, + "auxiliary_loss_mlp": 0.01039146, + "balance_loss_clip": 1.03402412, + "balance_loss_mlp": 1.02217615, + "epoch": 0.20001741048110963, + "flos": 74729493457920.0, + "grad_norm": 2.1788911314345647, + "language_loss": 0.87116241, + "learning_rate": 3.7043859965207924e-06, + "loss": 0.89242983, + "num_input_tokens_seen": 197082895, + "router_z_loss_clip": 0.53515625, + "router_z_loss_mlp": 0.16949463, + "step": 6893, + "time_per_iteration": 2.796370267868042 + } + ], + "logging_steps": 1.0, + "max_steps": 34462, + "num_input_tokens_seen": 197082895, + "num_train_epochs": 1, + "save_steps": 6893, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.289943254581903e+17, + "train_batch_size": 5, + "trial_name": null, + "trial_params": null +}